keycloak-aplcache

diff --git a/misc/CrossDataCenter.md b/misc/CrossDataCenter.md
index a57a7b9..5d9812d 100644
--- a/misc/CrossDataCenter.md
+++ b/misc/CrossDataCenter.md
@@ -98,7 +98,9 @@ Infinispan Server setup
             <transaction mode="NON_DURABLE_XA" locking="PESSIMISTIC"/>
             <locking acquire-timeout="0" />
             <backups>
-                <backup site="site2" failure-policy="FAIL" strategy="SYNC" enabled="true"/>
+                <backup site="site2" failure-policy="FAIL" strategy="SYNC" enabled="true">
+                    <take-offline min-wait="60000" after-failures="3" />
+                </backup>
             </backups>
         </replicated-cache-configuration>
 

diff --git a/misc/Testsuite.md b/misc/Testsuite.md
index f7b4e8e..142326a 100644
--- a/misc/Testsuite.md
+++ b/misc/Testsuite.md
@@ -117,7 +117,7 @@ But additionally you can enable Kerberos authentication in LDAP provider with th
 * KeyTab: $KEYCLOAK_SOURCES/testsuite/integration-arquillian/tests/base/src/test/resources/kerberos/http.keytab (Replace $KEYCLOAK_SOURCES with correct absolute path of your sources)
 
 Once you do this, you should also ensure that your Kerberos client configuration file is properly configured with KEYCLOAK.ORG domain. 
-See [../testsuite/integration-arquillian/src/test/resources/kerberos/test-krb5.conf](../testsuite/integration-arquillian/src/test/resources/kerberos/test-krb5.conf) for inspiration. The location of Kerberos configuration file 
+See [../testsuite/integration-arquillian/tests/base/src/test/resources/kerberos/test-krb5.conf](../testsuite/integration-arquillian/tests/base/src/test/resources/kerberos/test-krb5.conf) for inspiration. The location of Kerberos configuration file 
 is platform dependent (In linux it's file `/etc/krb5.conf` )
 
 Then you need to configure your browser to allow SPNEGO/Kerberos login from `localhost` .

diff --git a/model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProvider.java b/model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProvider.java
index 89140e3..c9022fc 100644
--- a/model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProvider.java
+++ b/model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProvider.java
@@ -22,6 +22,7 @@ import org.keycloak.cluster.ClusterEvent;
 import org.keycloak.cluster.ClusterListener;
 import org.keycloak.cluster.ClusterProvider;
 import org.keycloak.cluster.ExecutionResult;
+import org.keycloak.common.util.Retry;
 import org.keycloak.common.util.Time;
 
 import java.util.concurrent.Callable;
@@ -140,7 +141,7 @@ public class InfinispanClusterProvider implements ClusterProvider {
     private boolean tryLock(String cacheKey, int taskTimeoutInSeconds) {
         LockEntry myLock = createLockEntry();
 
-        LockEntry existingLock = (LockEntry) crossDCAwareCacheFactory.getCache().putIfAbsent(cacheKey, myLock, taskTimeoutInSeconds, TimeUnit.SECONDS);
+        LockEntry existingLock = InfinispanClusterProviderFactory.putIfAbsentWithRetries(crossDCAwareCacheFactory, cacheKey, myLock, taskTimeoutInSeconds);
         if (existingLock != null) {
             if (logger.isTraceEnabled()) {
                 logger.tracef("Task %s in progress already by node %s. Ignoring task.", cacheKey, existingLock.getNode());
@@ -156,22 +157,15 @@ public class InfinispanClusterProvider implements ClusterProvider {
 
 
     private void removeFromCache(String cacheKey) {
-        // 3 attempts to send the message (it may fail if some node fails in the meantime)
-        int retry = 3;
-        while (true) {
-            try {
-                crossDCAwareCacheFactory.getCache().remove(cacheKey);
-                if (logger.isTraceEnabled()) {
-                    logger.tracef("Task %s removed from the cache", cacheKey);
-                }
-                return;
-            } catch (RuntimeException e) {
-                retry--;
-                if (retry == 0) {
-                    throw e;
-                }
+        // More attempts to send the message (it may fail if some node fails in the meantime)
+        Retry.executeWithBackoff((int iteration) -> {
+
+            crossDCAwareCacheFactory.getCache().remove(cacheKey);
+            if (logger.isTraceEnabled()) {
+                logger.tracef("Task %s removed from the cache", cacheKey);
             }
-        }
+
+        }, 10, 10);
     }
 
 }

diff --git a/model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProviderFactory.java b/model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProviderFactory.java
index 54cfd04..4334c29 100644
--- a/model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProviderFactory.java
+++ b/model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProviderFactory.java
@@ -18,6 +18,7 @@
 package org.keycloak.cluster.infinispan;
 
 import org.infinispan.Cache;
+import org.infinispan.client.hotrod.exceptions.HotRodClientException;
 import org.infinispan.manager.EmbeddedCacheManager;
 import org.infinispan.notifications.Listener;
 import org.infinispan.notifications.cachemanagerlistener.annotation.ViewChanged;
@@ -29,6 +30,7 @@ import org.jboss.logging.Logger;
 import org.keycloak.Config;
 import org.keycloak.cluster.ClusterProvider;
 import org.keycloak.cluster.ClusterProviderFactory;
+import org.keycloak.common.util.Retry;
 import org.keycloak.common.util.Time;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.models.KeycloakSession;
@@ -42,6 +44,8 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
@@ -111,7 +115,7 @@ public class InfinispanClusterProviderFactory implements ClusterProviderFactory 
             // clusterStartTime not yet initialized. Let's try to put our startupTime
             int serverStartTime = (int) (session.getKeycloakSessionFactory().getServerStartupTimestamp() / 1000);
 
-            existingClusterStartTime = (Integer) crossDCAwareCacheFactory.getCache().putIfAbsent(InfinispanClusterProvider.CLUSTER_STARTUP_TIME_KEY, serverStartTime);
+            existingClusterStartTime = putIfAbsentWithRetries(crossDCAwareCacheFactory, InfinispanClusterProvider.CLUSTER_STARTUP_TIME_KEY, serverStartTime, -1);
             if (existingClusterStartTime == null) {
                 logger.debugf("Initialized cluster startup time to %s", Time.toDate(serverStartTime).toString());
                 return serverStartTime;
@@ -123,6 +127,35 @@ public class InfinispanClusterProviderFactory implements ClusterProviderFactory 
     }
 
 
+    // Will retry few times for the case when backup site not available in cross-dc environment.
+    // The site might be taken offline automatically if "take-offline" properly configured
+    static <V extends Serializable> V putIfAbsentWithRetries(CrossDCAwareCacheFactory crossDCAwareCacheFactory, String key, V value, int taskTimeoutInSeconds) {
+        AtomicReference<V> resultRef = new AtomicReference<>();
+
+        Retry.executeWithBackoff((int iteration) -> {
+
+            try {
+                V result;
+                if (taskTimeoutInSeconds > 0) {
+                    result = (V) crossDCAwareCacheFactory.getCache().putIfAbsent(key, value);
+                } else {
+                    result = (V) crossDCAwareCacheFactory.getCache().putIfAbsent(key, value, taskTimeoutInSeconds, TimeUnit.SECONDS);
+                }
+                resultRef.set(result);
+
+            } catch (HotRodClientException re) {
+                logger.warnf(re, "Failed to write key '%s' and value '%s' in iteration '%d' . Retrying", key, value, iteration);
+
+                // Rethrow the exception. Retry will take care of handle the exception and eventually retry the operation.
+                throw re;
+            }
+
+        }, 10, 10);
+
+        return resultRef.get();
+    }
+
+
 
     @Override
     public void init(Config.Scope config) {

diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java
index 76a010e..22b7382 100755
--- a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java
+++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java
@@ -276,7 +276,7 @@ public class InfinispanUserSessionProviderFactory implements UserSessionProvider
 
 
     private void loadSessionsFromRemoteCache(final KeycloakSessionFactory sessionFactory, String cacheName, final int sessionsPerSegment, final int maxErrors) {
-        log.debugf("Check pre-loading userSessions from remote cache '%s'", cacheName);
+        log.debugf("Check pre-loading sessions from remote cache '%s'", cacheName);
 
         KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
 
@@ -293,7 +293,7 @@ public class InfinispanUserSessionProviderFactory implements UserSessionProvider
 
         });
 
-        log.debugf("Pre-loading userSessions from remote cache '%s' finished", cacheName);
+        log.debugf("Pre-loading sessions from remote cache '%s' finished", cacheName);
     }
 
 

diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/OfflinePersistentUserSessionLoader.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/OfflinePersistentUserSessionLoader.java
index 5379e40..7b60dcb 100644
--- a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/OfflinePersistentUserSessionLoader.java
+++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/OfflinePersistentUserSessionLoader.java
@@ -18,15 +18,18 @@
 package org.keycloak.models.sessions.infinispan.initializer;
 
 import org.infinispan.Cache;
+import org.infinispan.client.hotrod.exceptions.HotRodClientException;
 import org.infinispan.context.Flag;
 import org.jboss.logging.Logger;
 import org.keycloak.cluster.ClusterProvider;
+import org.keycloak.common.util.Retry;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.UserSessionModel;
 import org.keycloak.models.session.UserSessionPersisterProvider;
 
 import java.io.Serializable;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
@@ -101,10 +104,24 @@ public class OfflinePersistentUserSessionLoader implements SessionLoader, Serial
     public void afterAllSessionsLoaded(BaseCacheInitializer initializer) {
         Cache<String, Serializable> workCache = initializer.getWorkCache();
 
-        // Cross-DC aware flag
-        workCache
-                .getAdvancedCache().withFlags(Flag.SKIP_REMOTE_LOOKUP)
-                .put(PERSISTENT_SESSIONS_LOADED, true);
+        // Will retry few times for the case when backup site not available in cross-dc environment.
+        // The site might be taken offline automatically if "take-offline" properly configured
+        Retry.executeWithBackoff((int iteration) -> {
+
+            try {
+                // Cross-DC aware flag
+                workCache
+                        .getAdvancedCache().withFlags(Flag.SKIP_REMOTE_LOOKUP)
+                        .put(PERSISTENT_SESSIONS_LOADED, true);
+
+            } catch (HotRodClientException re) {
+                log.warnf(re, "Failed to write flag PERSISTENT_SESSIONS_LOADED in iteration '%d' . Retrying", iteration);
+
+                // Rethrow the exception. Retry will take care of handle the exception and eventually retry the operation.
+                throw re;
+            }
+
+        }, 10, 10);
 
         // Just local-DC aware flag
         workCache

diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/RemoteCacheSessionsLoader.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/RemoteCacheSessionsLoader.java
index 128a7e9..1201563 100644
--- a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/RemoteCacheSessionsLoader.java
+++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/RemoteCacheSessionsLoader.java
@@ -142,7 +142,8 @@ public class RemoteCacheSessionsLoader implements SessionLoader {
                 .getAdvancedCache().withFlags(Flag.SKIP_CACHE_LOAD, Flag.SKIP_CACHE_STORE)
                 .get(OfflinePersistentUserSessionLoader.PERSISTENT_SESSIONS_LOADED_IN_CURRENT_DC);
 
-        if (cacheName.equals(InfinispanConnectionProvider.OFFLINE_USER_SESSION_CACHE_NAME) && sessionsLoaded != null && sessionsLoaded) {
+        if ((cacheName.equals(InfinispanConnectionProvider.OFFLINE_USER_SESSION_CACHE_NAME) || (cacheName.equals(InfinispanConnectionProvider.OFFLINE_CLIENT_SESSION_CACHE_NAME)))
+                && sessionsLoaded != null && sessionsLoaded) {
             log.debugf("Sessions already loaded in current DC. Skip sessions loading from remote cache '%s'", cacheName);
             return true;
         } else {

diff --git a/model/infinispan/src/test/java/org/keycloak/cluster/infinispan/ConcurrencyJDGOfflineBackupsTest.java b/model/infinispan/src/test/java/org/keycloak/cluster/infinispan/ConcurrencyJDGOfflineBackupsTest.java
new file mode 100644
index 0000000..19fe4f7
--- /dev/null
+++ b/model/infinispan/src/test/java/org/keycloak/cluster/infinispan/ConcurrencyJDGOfflineBackupsTest.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.cluster.infinispan;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.infinispan.Cache;
+import org.infinispan.client.hotrod.exceptions.HotRodClientException;
+import org.infinispan.context.Flag;
+import org.infinispan.manager.EmbeddedCacheManager;
+import org.infinispan.persistence.remote.configuration.RemoteStoreConfigurationBuilder;
+import org.jboss.logging.Logger;
+import org.keycloak.common.util.Time;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import org.keycloak.models.sessions.infinispan.changes.SessionEntityWrapper;
+import org.keycloak.models.sessions.infinispan.entities.UserSessionEntity;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class ConcurrencyJDGOfflineBackupsTest {
+
+    protected static final Logger logger = Logger.getLogger(ConcurrencyJDGOfflineBackupsTest.class);
+
+    public static void main(String[] args) throws Exception {
+
+        Cache<String, SessionEntityWrapper<UserSessionEntity>> cache1 = createManager(1).getCache(InfinispanConnectionProvider.USER_SESSION_CACHE_NAME);
+
+        try {
+            // Create initial item
+            UserSessionEntity session = new UserSessionEntity();
+            session.setId("123");
+            session.setRealmId("foo");
+            session.setBrokerSessionId("!23123123");
+            session.setBrokerUserId(null);
+            session.setUser("foo");
+            session.setLoginUsername("foo");
+            session.setIpAddress("123.44.143.178");
+            session.setStarted(Time.currentTime());
+            session.setLastSessionRefresh(Time.currentTime());
+
+//        AuthenticatedClientSessionEntity clientSession = new AuthenticatedClientSessionEntity();
+//        clientSession.setAuthMethod("saml");
+//        clientSession.setAction("something");
+//        clientSession.setTimestamp(1234);
+//        clientSession.setProtocolMappers(new HashSet<>(Arrays.asList("mapper1", "mapper2")));
+//        clientSession.setRoles(new HashSet<>(Arrays.asList("role1", "role2")));
+//        session.getAuthenticatedClientSessions().put(CLIENT_1_UUID.toString(), clientSession.getId());
+
+            SessionEntityWrapper<UserSessionEntity> wrappedSession = new SessionEntityWrapper<>(session);
+
+            // Some dummy testing of remoteStore behaviour
+            logger.info("Before put");
+
+
+            AtomicInteger successCount = new AtomicInteger(0);
+            AtomicInteger errorsCount = new AtomicInteger(0);
+            for (int i=0 ; i<100 ; i++) {
+                try {
+                    cache1
+                            .getAdvancedCache().withFlags(Flag.CACHE_MODE_LOCAL) // will still invoke remoteStore . Just doesn't propagate to cluster
+                            .put("123", wrappedSession);
+                    successCount.incrementAndGet();
+                    Thread.sleep(1000);
+                    logger.infof("Success in the iteration: %d", i);
+                } catch (HotRodClientException hrce) {
+                    logger.errorf("Failed to put the item in the iteration: %d ", i);
+                    errorsCount.incrementAndGet();
+                }
+            }
+
+            logger.infof("SuccessCount: %d, ErrorsCount: %d", successCount.get(), errorsCount.get());
+
+//            logger.info("After put");
+//
+//            cache1.replace("123", wrappedSession);
+//
+//            logger.info("After replace");
+//
+//            cache1.get("123");
+//
+//            logger.info("After cache1.get");
+
+//        cache2.get("123");
+//
+//        logger.info("After cache2.get");
+
+        } finally {
+            // Finish JVM
+            cache1.getCacheManager().stop();
+        }
+
+    }
+
+    private static EmbeddedCacheManager createManager(int threadId) {
+        return new TestCacheManagerFactory().createManager(threadId, InfinispanConnectionProvider.USER_SESSION_CACHE_NAME, RemoteStoreConfigurationBuilder.class);
+    }
+
+}

diff --git a/testsuite/integration-arquillian/HOW-TO-RUN.md b/testsuite/integration-arquillian/HOW-TO-RUN.md
index 8181561..23191b2 100644
--- a/testsuite/integration-arquillian/HOW-TO-RUN.md
+++ b/testsuite/integration-arquillian/HOW-TO-RUN.md
@@ -458,114 +458,115 @@ The cross DC requires setting a profile specifying used cache server by specifyi
 
 #### Run Cross-DC Tests from Maven
 
-a) First compile the Infinispan/JDG test server via the following command:
+a) Prepare the environment. Compile the infinispan server and eventually Keycloak on JBoss server.
+
+a1) If you want to use **Undertow** based Keycloak container, you just need to download and prepare the 
+Infinispan/JDG test server via the following command:
 
   `mvn -Pcache-server-infinispan,auth-servers-crossdc-undertow -f testsuite/integration-arquillian -DskipTests clean install`
 
-or
+*note: 'cache-server-infinispan' can be replaced by 'cache-server-jdg'*
 
-  `mvn -Pcache-server-jdg,auth-servers-crossdc-undertow -f testsuite/integration-arquillian -DskipTests clean install`
+a2) If you want to use **JBoss-based** Keycloak backend containers instead of containers on Embedded Undertow,
+ you need to prepare both the Infinispan/JDG test server and the Keycloak server on Wildfly/EAP. Run following command:
 
-b) Then in case you want to use **JBoss-based** Keycloak backend containers instead of containers on Embedded Undertow run following command:
+  `mvn -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly -f testsuite/integration-arquillian -DskipTests clean install`
 
-    `mvn -Pauth-servers-crossdc-jboss,auth-server-wildfly -f testsuite/integration-arquillian -DskipTests clean install`
+*note: 'cache-server-infinispan' can be replaced by 'cache-server-jdg'*
 
 *note: 'auth-server-wildfly' can be replaced by 'auth-server-eap'*
 
-By default JBoss-based containers use in-memory h2 database. It can be configured to use real DB, e.g. with following command:
+By default JBoss-based containers use TCP-based h2 database. It can be configured to use real DB, e.g. with following command:
 
-  `mvn -Pauth-servers-crossdc-jboss,auth-server-wildfly,jpa -f testsuite/integration-arquillian -DskipTests clean install -Djdbc.mvn.groupId=org.mariadb.jdbc -Djdbc.mvn.artifactId=mariadb-java-client -Djdbc.mvn.version=2.0.3 -Dkeycloak.connectionsJpa.url=jdbc:mariadb://localhost:3306/keycloak -Dkeycloak.connectionsJpa.password=keycloak -Dkeycloak.connectionsJpa.user=keycloak`
+  `mvn -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly,jpa -f testsuite/integration-arquillian -DskipTests clean install -Djdbc.mvn.groupId=org.mariadb.jdbc -Djdbc.mvn.artifactId=mariadb-java-client -Djdbc.mvn.version=2.0.3 -Dkeycloak.connectionsJpa.url=jdbc:mariadb://localhost:3306/keycloak -Dkeycloak.connectionsJpa.password=keycloak -Dkeycloak.connectionsJpa.user=keycloak`
 
-c1) Then you can run the tests using the following command (adjust the test specification according to your needs) for Keycloak backend containers on **Undertow**:
+b1) For **Undertow** Keycloak backend containers, you can run the tests using the following command (adjust the test specification according to your needs):
 
   `mvn -Pcache-server-infinispan,auth-servers-crossdc-undertow -Dtest=*.crossdc.* -pl testsuite/integration-arquillian/tests/base clean install`
 
-or
+*note: 'cache-server-infinispan' can be replaced by 'cache-server-jdg'*
 
-  `mvn -Pcache-server-jdg,auth-servers-crossdc-undertow -Dtest=*.crossdc.* -pl testsuite/integration-arquillian/tests/base clean install`
+*note: It can be useful to add additional system property to enable logging:*
+  
+  `-Dkeycloak.infinispan.logging.level=debug`
 
-c2) For **JBoss-based** Keycloak backend containers:
+b2) For **JBoss-based** Keycloak backend containers, you can run the tests like this:
 
   `mvn -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly -Dtest=*.crossdc.* -pl testsuite/integration-arquillian/tests/base clean install`
 
-or
-
-  `mvn -Pcache-server-jdg,auth-servers-crossdc-jboss,auth-server-wildfly -Dtest=*.crossdc.* -pl testsuite/integration-arquillian/tests/base clean install`
+*note: 'cache-server-infinispan' can be replaced by 'cache-server-jdg'*
 
 *note: 'auth-server-wildfly can be replaced by auth-server-eap'*
 
-**note**
-Previous commands can be "squashed" into one. E.g.:
+**note**:
+For **JBoss-based** Keycloak backend containers on real DB, the previous commands from (a2) and (b2) can be "squashed" into one. E.g.:
 
   `mvn -f testsuite/integration-arquillian clean install -Dtest=*.crossdc.* -Djdbc.mvn.groupId=org.mariadb.jdbc -Djdbc.mvn.artifactId=mariadb-java-client -Djdbc.mvn.version=2.0.3 -Dkeycloak.connectionsJpa.url=jdbc:mariadb://localhost:3306/keycloak -Dkeycloak.connectionsJpa.password=keycloak -Dkeycloak.connectionsJpa.user=keycloak -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly,jpa clean install`
 
-It can be useful to add additional system property to enable logging:
-  
-    -Dkeycloak.infinispan.logging.level=debug
     
-**Tests from package "manual"** uses manual lifecycle for all servers, so needs to be executed manually. Also needs to be executed with real DB like MySQL. You can run them with:
+#### Run "Manual" Cross-DC Tests from Maven
+    
+Tests from package "manual" uses manual lifecycle for all servers, so needs to be executed manually. 
+
+First prepare the environment and do the step (a) from previous paragraph. 
+
+c1) For **Undertow** Keycloak backend containers, you can run the test using following command:
+
+  `mvn -Pcache-server-infinispan,auth-servers-crossdc-undertow -Dtest=*.crossdc.manual.* -Dmanual.mode=true -Drun.h2=true -Dkeycloak.connectionsJpa.url.crossdc="jdbc:h2:tcp://localhost:9092/mem:keycloak-dc-shared;DB_CLOSE_DELAY=-1" -pl testsuite/integration-arquillian/tests/base clean install` 
+
+*note: As you can see, there is a need to run TCP-Based H2 for this test. In-memory H2 won't work due the data need 
+to persist the stop of all the Keycloak servers.*
+
+If you want to test with real DB like MySQL, you can run them with:
 
-    mvn -Pcache-server-infinispan -Dtest=*.crossdc.manual.* -Dmanual.mode=true \
-    -Dkeycloak.connectionsJpa.url.crossdc=jdbc:mysql://localhost/keycloak -Dkeycloak.connectionsJpa.driver.crossdc=com.mysql.jdbc.Driver \
-    -Dkeycloak.connectionsJpa.user=keycloak -Dkeycloak.connectionsJpa.password=keycloak \
-    -pl testsuite/integration-arquillian/tests/base test
+  `mvn -Pcache-server-infinispan,auth-servers-crossdc-undertow -Dtest=*.crossdc.manual.* -Dmanual.mode=true -Dkeycloak.connectionsJpa.url.crossdc=jdbc:mysql://localhost/keycloak -Dkeycloak.connectionsJpa.driver.crossdc=com.mysql.jdbc.Driver -Dkeycloak.connectionsJpa.user=keycloak -Dkeycloak.connectionsJpa.password=keycloak -pl testsuite/integration-arquillian/tests/base clean install`
 
+c2) For **JBoss-based** Keycloak backend containers, you can run the tests like this:
     
+  `mvn -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly -Dtest=*.crossdc.manual.* -Dmanual.mode=true -pl testsuite/integration-arquillian/tests/base clean install`
     
+*note: TCP-based H2 is used by default when running cross-dc tests on JBoss-based Keycloak container. 
+So no need to explicitly specify it like in (c1) for undertow.*       
      
   
 
 #### Run Cross-DC Tests from Intellij IDEA
 
-First we will manually download, configure and run infinispan server. Then we can run the tests from IDE against 1 server. It's more effective during
-development as there is no need to restart infinispan server(s) among test runs.
+First we will manually download, configure and run infinispan servers. Then we can run the tests from IDE against the servers. 
+It's more effective during development as there is no need to restart infinispan server(s) among test runs.
 
-1) Download infinispan server 8.2.X from http://infinispan.org/download/
+1) Download infinispan server 8.2.X from http://infinispan.org/download/ and go through the steps 
+from the [../../misc/CrossDataCenter.md](../../misc/CrossDataCenter.md) and the `Infinispan Server Setup` part.
 
-2) Edit `ISPN_SERVER_HOME/standalone/configuration/standalone.xml` and add these local-caches to the section under cache-container `local` :
+Assume you have both Infinispan/JDG servers up and running.
 
-    <cache-container name="local" ...
-    
-        ...
-        
-        <local-cache-configuration name="sessions-cfg" start="EAGER" batching="false">
-		    <transaction mode="NON_XA" locking="PESSIMISTIC"/>
-		</local-cache-configuration>
-
-        <local-cache name="sessions" configuration="sessions-cfg" />	
-		<local-cache name="offlineSessions" configuration="sessions-cfg" />	
-		<local-cache name="loginFailures" configuration="sessions-cfg" />	
-		<local-cache name="actionTokens" configuration="sessions-cfg" />	
-		<local-cache name="work" configuration="sessions-cfg" />
-    
-    </cache>
-    
-3) Run the server through `./standalone.sh` 
-       
-4) Setup MySQL database or some other shared database.  
+**TODO:** Change this once CrossDataCenter.md is removed and converted to the proper docs.
+
+2) Setup MySQL database or some other shared database.  
 
-5) Ensure that org.wildfly.arquillian:wildfly-arquillian-container-managed is on the classpath when running test. On Intellij, it can be 
-done by going to: View -> Tool Windows -> Maven projects. Then check profile "cache-server-infinispan". The tests will use this profile when executed.
+3) Ensure that `org.wildfly.arquillian:wildfly-arquillian-container-managed` is on the classpath when running test. On Intellij, it can be 
+done by going to: `View` -> `Tool Windows` -> `Maven projects`. Then check profile `cache-server-infinispan` and `auth-servers-crossdc-undertow`. 
+The tests will use this profile when executed.
 
-6) Run the LoginCrossDCTest (or any other test) with those properties. In shortcut, it's using MySQL database, disabled L1 lifespan and 
+4) Run the LoginCrossDCTest (or any other test) with those properties. In shortcut, it's using MySQL database and 
 connects to the remoteStore provided by infinispan server configured in previous steps:
 
-    -Dauth.server.crossdc=true -Dauth.server.undertow.crossdc=true -Dcache.server.lifecycle.skip=true -Dkeycloak.connectionsJpa.url.crossdc=jdbc:mysql://localhost/keycloak 
-    -Dkeycloak.connectionsJpa.driver.crossdc=com.mysql.jdbc.Driver -Dkeycloak.connectionsJpa.user=keycloak 
-    -Dkeycloak.connectionsJpa.password=keycloak -Dkeycloak.connectionsInfinispan.clustered=true -Dkeycloak.connectionsInfinispan.l1Lifespan=0 
-    -Dkeycloak.connectionsInfinispan.remoteStorePort=11222 -Dkeycloak.connectionsInfinispan.remoteStorePort.2=11222 -Dkeycloak.connectionsInfinispan.sessionsOwners=1 
-    -Dsession.cache.owners=1 -Dkeycloak.infinispan.logging.level=debug -Dresources
+  `-Dauth.server.crossdc=true -Dauth.server.undertow.crossdc=true -Dcache.server.lifecycle.skip=true -Dkeycloak.connectionsInfinispan.clustered=true -Dkeycloak.connectionsJpa.url.crossdc=jdbc:mysql://localhost/keycloak -Dkeycloak.connectionsJpa.driver.crossdc=com.mysql.jdbc.Driver -Dkeycloak.connectionsJpa.user=keycloak -Dkeycloak.connectionsJpa.password=keycloak -Dkeycloak.connectionsInfinispan.clustered=true -Dkeycloak.connectionsInfinispan.remoteStorePort=12232 -Dkeycloak.connectionsInfinispan.remoteStorePort.2=13232 -Dkeycloak.connectionsInfinispan.sessionsOwners=1 -Dsession.cache.owners=1 -Dkeycloak.infinispan.logging.level=debug -Dresources`    
     
-NOTE: Tests from package "manual" (eg. SessionsPreloadCrossDCTest) needs to be executed with managed containers. 
+**NOTE**: Tests from package `manual` (eg. SessionsPreloadCrossDCTest) needs to be executed with managed containers. 
 So skip steps 1,2 and add property `-Dmanual.mode=true` and change "cache.server.lifecycle.skip" to false `-Dcache.server.lifecycle.skip=false` or remove it.    
     
-7) If you want to debug and test manually, the servers are running on these ports (Note that not all backend servers are running by default and some might be also unused by loadbalancer):
+5) If you want to debug or test manually, the servers are running on these ports (Note that not all backend servers are running by default and some might be also unused by loadbalancer):
+
+* *Loadbalancer* -> "http://localhost:8180/auth"
+
+* *auth-server-undertow-cross-dc-0_1* -> "http://localhost:8101/auth"
+
+* *auth-server-undertow-cross-dc-0_2-manual* -> "http://localhost:8102/auth"
+
+* *auth-server-undertow-cross-dc-1_1* -> "http://localhost:8111/auth"
 
-    Loadbalancer -> "http://localhost:8180/auth"
-    auth-server-undertow-cross-dc-0_1 -> "http://localhost:8101/auth"
-    auth-server-undertow-cross-dc-0_2-manual -> "http://localhost:8102/auth"
-    auth-server-undertow-cross-dc-1_1 -> "http://localhost:8111/auth"
-    auth-server-undertow-cross-dc-1_2-manual -> "http://localhost:8112/auth"
+* *auth-server-undertow-cross-dc-1_2-manual* -> "http://localhost:8112/auth"
 
 
 ## Run Docker Authentication test

diff --git a/testsuite/integration-arquillian/servers/cache-server/jboss/common/add-keycloak-caches.xsl b/testsuite/integration-arquillian/servers/cache-server/jboss/common/add-keycloak-caches.xsl
index e6c16da..d3881a5 100644
--- a/testsuite/integration-arquillian/servers/cache-server/jboss/common/add-keycloak-caches.xsl
+++ b/testsuite/integration-arquillian/servers/cache-server/jboss/common/add-keycloak-caches.xsl
@@ -39,7 +39,9 @@
                 <transaction mode="NON_DURABLE_XA" locking="PESSIMISTIC"/>
                 <locking acquire-timeout="0" />
                 <backups>
-                    <backup site="{$remote.site}" failure-policy="FAIL" strategy="SYNC" enabled="true"/>
+                    <backup site="{$remote.site}" failure-policy="FAIL" strategy="SYNC" enabled="true">
+                        <take-offline min-wait="60000" after-failures="3" />
+                    </backup>
                 </backups>
             </replicated-cache-configuration>
 

diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/manual/SessionsPreloadCrossDCTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/manual/SessionsPreloadCrossDCTest.java
index bc1243b..14a508d 100644
--- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/manual/SessionsPreloadCrossDCTest.java
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/manual/SessionsPreloadCrossDCTest.java
@@ -52,8 +52,9 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
 
         stopAllCacheServersAndAuthServers();
 
-        // Start DC1 only
+        // Start DC1 and only the cache container from DC2. All Keycloak nodes on DC2 are stopped
         containerController.start(getCacheServer(DC.FIRST).getQualifier());
+        containerController.start(getCacheServer(DC.SECOND).getQualifier());
         startBackendNode(DC.FIRST, 0);
         enableLoadBalancerNode(DC.FIRST, 0);
 
@@ -119,7 +120,6 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
         List<OAuthClient.AccessTokenResponse> tokenResponses = createInitialSessions(false);
 
         // Start 2nd DC.
-        containerController.start(getCacheServer(DC.SECOND).getQualifier());
         startBackendNode(DC.SECOND, 0);
         enableLoadBalancerNode(DC.SECOND, 0);
 
@@ -130,7 +130,7 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
         Assert.assertEquals(sessions01, sessionsBefore + SESSIONS_COUNT);
         Assert.assertEquals(sessions02, sessionsBefore + SESSIONS_COUNT);
 
-        // On DC2 sessions were preloaded from from remoteCache
+        // On DC2 sessions were preloaded from remoteCache
         Assert.assertTrue(getTestingClientForStartedNodeInDc(1).testing().cache(InfinispanConnectionProvider.WORK_CACHE_NAME).contains("distributed::remoteCacheLoad::sessions"));
 
         // Assert refreshing works
@@ -157,13 +157,15 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
         // Stop Everything
         stopAllCacheServersAndAuthServers();
 
-        // Start DC1. Sessions should be preloaded from DB
+        // Start cache containers on both DC1 and DC2
         containerController.start(getCacheServer(DC.FIRST).getQualifier());
+        containerController.start(getCacheServer(DC.SECOND).getQualifier());
+
+        // Start Keycloak on DC1. Sessions should be preloaded from DB
         startBackendNode(DC.FIRST, 0);
         enableLoadBalancerNode(DC.FIRST, 0);
 
-        // Start DC2. Sessions should be preloaded from remoteCache
-        containerController.start(getCacheServer(DC.SECOND).getQualifier());
+        // Start Keycloak on DC2. Sessions should be preloaded from remoteCache
         startBackendNode(DC.SECOND, 0);
         enableLoadBalancerNode(DC.SECOND, 0);
 
@@ -210,7 +212,6 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
         }
 
         // Start 2nd DC.
-        containerController.start(getCacheServer(DC.SECOND).getQualifier());
         startBackendNode(DC.SECOND, 0);
         enableLoadBalancerNode(DC.SECOND, 0);
 

diff --git a/travis-run-tests.sh b/travis-run-tests.sh
index bee06b3..532eaf9 100755
--- a/travis-run-tests.sh
+++ b/travis-run-tests.sh
@@ -82,5 +82,17 @@ if [ $1 == "crossdc" ]; then
     cd tests/base
     mvn clean test -B -nsu -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly -Dtest=*.crossdc.**.* 2>&1 |
         java -cp ../../../utils/target/classes org.keycloak.testsuite.LogTrimmer
-    exit ${PIPESTATUS[0]}
+    BASE_TESTS_STATUS=${PIPESTATUS[0]}
+
+    mvn clean test -B -nsu -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly -Dtest=*.crossdc.manual.* -Dmanual.mode=true 2>&1 |
+        java -cp ../../../utils/target/classes org.keycloak.testsuite.LogTrimmer
+    MANUAL_TESTS_STATUS=${PIPESTATUS[0]}
+
+    echo "BASE_TESTS_STATUS=$BASE_TESTS_STATUS, MANUAL_TESTS_STATUS=$MANUAL_TESTS_STATUS";
+    if [ $BASE_TESTS_STATUS -eq 0 -a $MANUAL_TESTS_STATUS -eq 0 ]; then
+        exit 0;
+    else
+        exit 1;
+    fi;
+
 fi