keycloak-aplcache

Merge pull request #2469 from mposolda/1.9.x KEYCLOAK-2613

4/1/2016 11:46:13 AM

Details

diff --git a/core/src/main/java/org/keycloak/representations/idm/RealmRepresentation.java b/core/src/main/java/org/keycloak/representations/idm/RealmRepresentation.java
index 54fa9be..de4bf38 100755
--- a/core/src/main/java/org/keycloak/representations/idm/RealmRepresentation.java
+++ b/core/src/main/java/org/keycloak/representations/idm/RealmRepresentation.java
@@ -121,6 +121,8 @@ public class RealmRepresentation {
     protected String resetCredentialsFlow;
     protected String clientAuthenticationFlow;
 
+    protected String keycloakVersion;
+
     @Deprecated
     protected Boolean social;
     @Deprecated
@@ -814,6 +816,14 @@ public class RealmRepresentation {
         this.clientAuthenticationFlow = clientAuthenticationFlow;
     }
 
+    public String getKeycloakVersion() {
+        return keycloakVersion;
+    }
+
+    public void setKeycloakVersion(String keycloakVersion) {
+        this.keycloakVersion = keycloakVersion;
+    }
+
     public List<GroupRepresentation> getGroups() {
         return groups;
     }
diff --git a/docbook/auth-server-docs/reference/en/en-US/modules/export-import.xml b/docbook/auth-server-docs/reference/en/en-US/modules/export-import.xml
index 3241d44..9a8a4bc 100755
--- a/docbook/auth-server-docs/reference/en/en-US/modules/export-import.xml
+++ b/docbook/auth-server-docs/reference/en/en-US/modules/export-import.xml
@@ -39,12 +39,21 @@
             </itemizedlist>
         </para>
         <para>
-            If you import to Directory, you can specify also the number of users to be stored in each JSON file. So if you have
-            very large amount of users in your database, you likely don't want to import them into single file as the file might be very big.
-            Processing of each file is done in separate transaction as exporting/importing all users at once could also lead to memory issues.
+            If you import to Directory, you can specify also the number of users to be stored in each JSON file.
         </para>
+        <warning>
+            <para>
+                If you have bigger amount of users in your database (500 or more), it's higly recommended to export into directory rather than to single file.
+                Exporting into single file may lead to the very big file. Also the directory provider is using separate transaction for each "page" (file with users),
+                which leads to much better performance. Default count of users per file (and transaction) is 50, which showed us best performance, but you have possibility to override (See below).
+            </para>
+            <para>
+                Exporting to single file is using one transaction per whole export and one per whole import, which leads to
+                bad performance with large amount of users - time increases exponentially with number of users.
+            </para>
+        </warning>
         <para>
-            To export into unencrypted directory you can use:
+            To export into the directory you can use:
             <programlisting><![CDATA[
 bin/standalone.sh -Dkeycloak.migration.action=export
 -Dkeycloak.migration.provider=dir -Dkeycloak.migration.dir=<DIR TO EXPORT TO>
@@ -98,7 +107,7 @@ bin/standalone.sh -Dkeycloak.migration.action=import
                     <listitem>
                         <para>
                             can be used to specify number of users per file (and also per DB transaction).
-                            It's 5000 by default. It's used only if usersExportStrategy is DIFFERENT_FILES
+                            It's 50 by default. It's used only if usersExportStrategy is DIFFERENT_FILES
                         </para>
                     </listitem>
                 </varlistentry>
diff --git a/services/src/main/java/org/keycloak/exportimport/dir/DirExportProvider.java b/services/src/main/java/org/keycloak/exportimport/dir/DirExportProvider.java
index dd1972d..906d246 100755
--- a/services/src/main/java/org/keycloak/exportimport/dir/DirExportProvider.java
+++ b/services/src/main/java/org/keycloak/exportimport/dir/DirExportProvider.java
@@ -17,7 +17,6 @@
 
 package org.keycloak.exportimport.dir;
 
-import org.keycloak.representations.VersionRepresentation;
 import org.keycloak.exportimport.util.ExportUtils;
 import org.keycloak.exportimport.util.MultipleStepsExportProvider;
 import org.keycloak.models.KeycloakSession;
@@ -87,13 +86,6 @@ public class DirExportProvider extends MultipleStepsExportProvider {
     }
 
     @Override
-    protected void writeVersion(String fileName, VersionRepresentation version) throws IOException {
-        File file = new File(this.rootDirectory, fileName);
-        FileOutputStream stream = new FileOutputStream(file);
-        JsonSerialization.prettyMapper.writeValue(stream, version);
-    }
-
-    @Override
     public void close() {
     }
 }
diff --git a/services/src/main/java/org/keycloak/exportimport/ExportImportConfig.java b/services/src/main/java/org/keycloak/exportimport/ExportImportConfig.java
index 7de7aac..2555a19 100644
--- a/services/src/main/java/org/keycloak/exportimport/ExportImportConfig.java
+++ b/services/src/main/java/org/keycloak/exportimport/ExportImportConfig.java
@@ -28,26 +28,24 @@ public class ExportImportConfig {
     public static final String ACTION_IMPORT = "import";
 
     public static final String PROVIDER = PREFIX + "provider";
-    public static final String PROVIDER_DEFAULT = "zip";
+    public static final String PROVIDER_DEFAULT = "dir";
 
     // Name of the realm to export. If null, then full export will be triggered
     public static final String REALM_NAME = PREFIX + "realmName";
 
     // used for "dir" provider
     public static final String DIR = PREFIX + "dir";
-    // used for "zip" provider
-    public static final String ZIP_FILE = PREFIX + "zipFile";
-    public static final String ZIP_PASSWORD = PREFIX + "zipPassword";
+
     // used for "singleFile" provider
     public static final String FILE = PREFIX + "file";
 
-    // How to export users when realm export is requested for "dir" and "zip" provider
+    // How to export users when realm export is requested for "dir" provider
     public static final String USERS_EXPORT_STRATEGY = PREFIX + "usersExportStrategy";
     public static final UsersExportStrategy DEFAULT_USERS_EXPORT_STRATEGY = UsersExportStrategy.DIFFERENT_FILES;
 
-    // Number of users per file used in "dir" and "zip" providers. Used if usersExportStrategy is DIFFERENT_FILES
+    // Number of users per file used in "dir" provider. Used if usersExportStrategy is DIFFERENT_FILES
     public static final String USERS_PER_FILE = PREFIX + "usersPerFile";
-    public static final Integer DEFAULT_USERS_PER_FILE = 5000;
+    public static final Integer DEFAULT_USERS_PER_FILE = 50;
 
     // Strategy used during import data
     public static final String STRATEGY = PREFIX + "strategy";
@@ -89,22 +87,6 @@ public class ExportImportConfig {
         return System.setProperty(DIR, dir);
     }
 
-    public static String getZipFile() {
-        return System.getProperty(ZIP_FILE);
-    }
-
-    public static void setZipFile(String exportImportZipFile) {
-        System.setProperty(ZIP_FILE, exportImportZipFile);
-    }
-
-    public static String getZipPassword() {
-        return System.getProperty(ZIP_PASSWORD);
-    }
-
-    public static void setZipPassword(String exportImportZipPassword) {
-        System.setProperty(ZIP_PASSWORD, exportImportZipPassword);
-    }
-
     public static String getFile() {
         return System.getProperty(FILE);
     }
diff --git a/services/src/main/java/org/keycloak/exportimport/ExportImportManager.java b/services/src/main/java/org/keycloak/exportimport/ExportImportManager.java
index b06e8a2..f8fb7ea 100644
--- a/services/src/main/java/org/keycloak/exportimport/ExportImportManager.java
+++ b/services/src/main/java/org/keycloak/exportimport/ExportImportManager.java
@@ -49,12 +49,12 @@ public class ExportImportManager {
         if (ExportImportConfig.ACTION_EXPORT.equals(exportImportAction)) {
             exportProvider = session.getProvider(ExportProvider.class, providerId);
             if (exportProvider == null) {
-                throw new RuntimeException("Export provider not found");
+                throw new RuntimeException("Export provider '" + providerId + "' not found");
             }
         } else if (ExportImportConfig.ACTION_IMPORT.equals(exportImportAction)) {
             importProvider = session.getProvider(ImportProvider.class, providerId);
             if (importProvider == null) {
-                throw new RuntimeException("Import provider not found");
+                throw new RuntimeException("Import provider '" + providerId + "' not found");
             }
         }
     }
diff --git a/services/src/main/java/org/keycloak/exportimport/util/ExportUtils.java b/services/src/main/java/org/keycloak/exportimport/util/ExportUtils.java
index bdea6f6..72cabb5 100755
--- a/services/src/main/java/org/keycloak/exportimport/util/ExportUtils.java
+++ b/services/src/main/java/org/keycloak/exportimport/util/ExportUtils.java
@@ -22,6 +22,7 @@ import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
+import org.keycloak.common.Version;
 import org.keycloak.common.util.Base64;
 import org.keycloak.models.*;
 import org.keycloak.models.utils.ModelToRepresentation;
@@ -39,6 +40,9 @@ public class ExportUtils {
     public static RealmRepresentation exportRealm(KeycloakSession session, RealmModel realm, boolean includeUsers) {
         RealmRepresentation rep = ModelToRepresentation.toRepresentation(realm, true);
 
+        // Project/product version
+        rep.setKeycloakVersion(Version.VERSION);
+
         // Client Templates
         List<ClientTemplateModel> templates = realm.getClientTemplates();
         List<ClientTemplateRepresentation> templateReps = new ArrayList<>();
diff --git a/services/src/main/java/org/keycloak/exportimport/util/MultipleStepsExportProvider.java b/services/src/main/java/org/keycloak/exportimport/util/MultipleStepsExportProvider.java
index 735659f..4537169 100755
--- a/services/src/main/java/org/keycloak/exportimport/util/MultipleStepsExportProvider.java
+++ b/services/src/main/java/org/keycloak/exportimport/util/MultipleStepsExportProvider.java
@@ -18,7 +18,6 @@
 package org.keycloak.exportimport.util;
 
 import org.jboss.logging.Logger;
-import org.keycloak.representations.VersionRepresentation;
 import org.keycloak.exportimport.ExportImportConfig;
 import org.keycloak.exportimport.ExportProvider;
 import org.keycloak.exportimport.UsersExportStrategy;
@@ -57,14 +56,11 @@ public abstract class MultipleStepsExportProvider implements ExportProvider {
         for (RealmModel realm : holder.realms) {
             exportRealmImpl(factory, realm.getName());
         }
-
-        writeVersion("version.json", VersionRepresentation.SINGLETON);
     }
 
     @Override
     public void exportRealm(KeycloakSessionFactory factory, String realmName) throws IOException {
         exportRealmImpl(factory, realmName);
-        writeVersion("version.json", VersionRepresentation.SINGLETON);
     }
 
     protected void exportRealmImpl(KeycloakSessionFactory factory, final String realmName) throws IOException {
@@ -127,8 +123,6 @@ public abstract class MultipleStepsExportProvider implements ExportProvider {
 
     protected abstract void writeUsers(String fileName, KeycloakSession session, RealmModel realm, List<UserModel> users) throws IOException;
 
-    protected abstract void writeVersion(String fileName, VersionRepresentation version) throws IOException;
-
     public static class RealmsHolder {
         List<RealmModel> realms;
 
diff --git a/testsuite/integration/src/test/java/org/keycloak/testsuite/exportimport/ExportImportTest.java b/testsuite/integration/src/test/java/org/keycloak/testsuite/exportimport/ExportImportTest.java
index f8f8e14..d3ada6e 100755
--- a/testsuite/integration/src/test/java/org/keycloak/testsuite/exportimport/ExportImportTest.java
+++ b/testsuite/integration/src/test/java/org/keycloak/testsuite/exportimport/ExportImportTest.java
@@ -166,8 +166,8 @@ public class ExportImportTest {
 
         testFullExportImport();
 
-        // There should be 6 files in target directory (3 realm, 3 user, 1 version)
-        Assert.assertEquals(7, new File(targetDirPath).listFiles().length);
+        // There should be 6 files in target directory (3 realm, 3 user)
+        Assert.assertEquals(6, new File(targetDirPath).listFiles().length);
     }
 
     @Test
@@ -180,9 +180,9 @@ public class ExportImportTest {
 
         testRealmExportImport();
 
-        // There should be 3 files in target directory (1 realm, 3 user, 1 version)
+        // There should be 3 files in target directory (1 realm, 3 user)
         File[] files = new File(targetDirPath).listFiles();
-        Assert.assertEquals(5, files.length);
+        Assert.assertEquals(4, files.length);
     }
 
     @Test