keycloak-aplcache

diff --git a/testsuite/integration-arquillian/README.md b/testsuite/integration-arquillian/README.md
index 5f7f831..39168de 100644
--- a/testsuite/integration-arquillian/README.md
+++ b/testsuite/integration-arquillian/README.md
@@ -13,17 +13,28 @@ other options are: `auth-server-wildfly` and `auth-server-eap7`. The values corr
 **Note 1:** For the non-default options it's necessary to build a corresponding server module prior to running any of the test modules.
 This can be done by building the server module directly (from `servers/wildfly`/`servers/eap7`), 
 or by activating `auth-server-wildfly`/`auth-server-eap7` profile when building from the top level module.
-(The profiles will also set the proper value of the `auth.server.container` property.)
 
 **Note 2:** Most server-side configurations are done during the build of the server module
 and included in the output artifact - which is then consumed by the test modules( if a corresponding profile is activated).
 To reflect a change in server config in the test (e.g. a datasource) it's necessary to rebuild the server module after each change.
 
-### Migration
+#### Migration
 
-Migration tests can be enabled by setting `-Dmigrated.auth.server.container` property or activating a corresponding profile.
-When enabled, the `AuthServerTestEnricher` class will start/stop the selected *migrated* instance 
-even **before** the *current* auth server instance is started.
+Migration tests can be enabled by setting `-Dmigrated.auth.server.version` property. Supported versions can be found at the bottom of `tests/pom.xml`.
+When enabled, the `AuthServerTestEnricher` class will start and stop the selected migrated instance 
+*before* the current auth server instance is started.
+
+#### Cluster Setup
+
+Cluster setup can be enabled with profile `auth-server-wildfly-cluster`.
+(It is also necessary to build the server modules with this profile before running the test. See *Notes 1 and 2* above.)
+
+Clustering tests require MULTICAST to be enabled on machine's `loopback` network interface.
+This can be done by running the following commands under root privileges:
+```
+route add -net 224.0.0.0 netmask 240.0.0.0 dev lo
+ifconfig lo multicast
+```
 
 ### App Servers
 

diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractClusterTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractClusterTest.java
index 9dd6089..fc523b7 100644
--- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractClusterTest.java
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractClusterTest.java
@@ -3,12 +3,15 @@ package org.keycloak.testsuite.cluster;
 import java.util.ArrayList;
 import java.util.List;
 import org.jboss.arquillian.container.test.api.ContainerController;
+import org.jboss.arquillian.graphene.page.Page;
 import org.jboss.arquillian.test.api.ArquillianResource;
 import static org.junit.Assert.assertTrue;
 import org.keycloak.admin.client.Keycloak;
 import org.keycloak.models.Constants;
+import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.testsuite.AbstractKeycloakTest;
 import org.keycloak.testsuite.arquillian.ContainerInfo;
+import org.keycloak.testsuite.auth.page.AuthRealm;
 import static org.keycloak.testsuite.auth.page.AuthRealm.ADMIN;
 import static org.keycloak.testsuite.auth.page.AuthRealm.MASTER;
 
@@ -35,25 +38,38 @@ public abstract class AbstractClusterTest extends AbstractKeycloakTest {
             controller.start(backendNode.getQualifier());
             assertTrue(controller.isStarted(backendNode.getQualifier()));
 
-            log.info("Initializing admin client for: '" + backendNode.getContextRoot() + "/auth'");
-            backendAdminClients.add(Keycloak.getInstance(backendNode.getContextRoot() + "/auth",
-                    MASTER, ADMIN, ADMIN, Constants.ADMIN_CLI_CLIENT_ID));
+            backendAdminClients.add(createAdminClientFor(backendNode));
         }
     }
 
-    protected ContainerInfo backendInfo(int i) {
+    protected Keycloak createAdminClientFor(ContainerInfo backendNode) {
+        log.info("Initializing admin client for " + backendNode.getContextRoot() + "/auth");
+        return Keycloak.getInstance(backendNode.getContextRoot() + "/auth",
+                MASTER, ADMIN, ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
+    }
+
+    protected ContainerInfo backendNode(int i) {
         return suiteContext.getAuthServerBackendsInfo().get(i);
     }
 
     protected void startBackendNode(int i) {
-        String container = backendInfo(i).getQualifier();
+        String container = backendNode(i).getQualifier();
         if (!controller.isStarted(container)) {
             controller.start(container);
+            backendAdminClients.set(i, createAdminClientFor(backendNode(i)));
         }
     }
 
-    protected void stopBackendNode(int i) {
-        controller.kill(backendInfo(i).getQualifier());
+    protected void killBackendNode(int i) {
+        backendAdminClients.get(i).close();
+        controller.kill(backendNode(i).getQualifier());
     }
 
+    protected void listRealms(int i) {
+        log.info(String.format("Node %s: AccessTokenString: %s", i + 1, backendAdminClients.get(i).tokenManager().getAccessTokenString()));
+        for (RealmRepresentation r : backendAdminClients.get(i).realms().findAll()) {
+            log.info(String.format("Node %s: Realm: %s, Id: %s", i + 1, r.getRealm(), r.getId()));
+        }
+    }
+    
 }

diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractTwoNodeClusterTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractTwoNodeClusterTest.java
new file mode 100644
index 0000000..d88e616
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractTwoNodeClusterTest.java
@@ -0,0 +1,57 @@
+package org.keycloak.testsuite.cluster;
+
+import org.junit.Before;
+import org.keycloak.admin.client.Keycloak;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+import static org.keycloak.testsuite.util.WaitUtils.pause;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public abstract class AbstractTwoNodeClusterTest extends AbstractClusterTest {
+
+    @Before
+    public void beforeTwoNodeClusterTest() {
+        startBackendNodes(2);
+        pause(3000);
+    }
+
+    protected ContainerInfo backend1Info() {
+        return backendNode(0);
+    }
+
+    protected ContainerInfo backend2Info() {
+        return backendNode(1);
+    }
+
+    protected Keycloak backend1AdminClient() {
+        return backendAdminClients.get(0);
+    }
+
+    protected Keycloak backend2AdminClient() {
+        return backendAdminClients.get(1);
+    }
+
+    protected void startBackend1() {
+        startBackendNode(0);
+    }
+
+    protected void startBackend2() {
+        startBackendNode(1);
+    }
+
+    protected void failback() {
+        startBackend1();
+        startBackend2();
+    }
+
+    protected void killBackend1() {
+        killBackendNode(0);
+    }
+
+    protected void killBackend2() {
+        killBackendNode(1);
+    }
+
+}

diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/SessionFailoverClusterTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/SessionFailoverClusterTest.java
new file mode 100644
index 0000000..591e2e8
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/SessionFailoverClusterTest.java
@@ -0,0 +1,95 @@
+package org.keycloak.testsuite.cluster;
+
+import java.util.List;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.keycloak.representations.idm.RealmRepresentation;
+import static org.keycloak.testsuite.auth.page.AuthRealm.ADMIN;
+import static org.keycloak.testsuite.util.URLAssert.assertCurrentUrlDoesntStartWith;
+import static org.keycloak.testsuite.util.URLAssert.assertCurrentUrlStartsWith;
+import static org.keycloak.testsuite.util.WaitUtils.pause;
+import org.openqa.selenium.Cookie;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
+
+    public static final String KEYCLOAK_SESSION_COOKIE = "KEYCLOAK_SESSION";
+    public static final String KEYCLOAK_IDENTITY_COOKIE = "KEYCLOAK_IDENTITY";
+
+    @Override
+    public void addTestRealms(List<RealmRepresentation> testRealms) {
+    }
+
+    @Test
+    @Ignore("work in progress") // only works with owners="2" at the moment
+    public void sessionFailover() {
+        
+        // LOGOUT
+        accountPage.navigateTo();
+        driver.navigate().refresh();
+        pause(3000);
+        loginPage.form().login(ADMIN, ADMIN);
+        assertCurrentUrlStartsWith(accountPage);
+        
+        Cookie sessionCookie = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
+        assertNotNull(sessionCookie);
+
+        killBackend1();
+
+        // check if session survived backend failure
+        
+        driver.navigate().refresh();
+        pause(3000);
+        
+        assertCurrentUrlStartsWith(accountPage);
+        Cookie sessionCookieAfterFailover = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
+        assertNotNull(sessionCookieAfterFailover);
+        assertEquals(sessionCookieAfterFailover.getValue(), sessionCookie.getValue());
+
+        failback();
+
+        // check if session survived backend failback
+        driver.navigate().refresh();
+        pause(3000);
+        assertCurrentUrlStartsWith(accountPage);
+        Cookie sessionCookieAfterFailback = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
+        assertNotNull(sessionCookieAfterFailback);
+        assertEquals(sessionCookieAfterFailover.getValue(), sessionCookie.getValue());
+
+        // LOGOUT
+        accountPage.navigateTo();
+        accountPage.signOut();
+
+        assertCurrentUrlDoesntStartWith(accountPage);
+        masterRealmPage.navigateTo();
+        sessionCookie = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
+        assertNull(sessionCookie);
+
+        killBackend1();
+        
+        // check if session survived backend failure
+        driver.navigate().refresh();
+        pause(3000);
+        assertCurrentUrlDoesntStartWith(accountPage);
+        masterRealmPage.navigateTo();
+        sessionCookieAfterFailover = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
+        assertNull(sessionCookieAfterFailover);
+        
+        failback();
+    
+        // check if session survived backend failback
+        driver.navigate().refresh();
+        pause(3000);
+        assertCurrentUrlDoesntStartWith(accountPage);
+        masterRealmPage.navigateTo();
+        sessionCookieAfterFailback = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
+        assertNull(sessionCookieAfterFailback);
+    }
+
+}

diff --git a/testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml b/testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml
index 404f5b5..fc91116 100644
--- a/testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml
+++ b/testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml
@@ -72,10 +72,12 @@
                 <property name="enabled">${auth.server.wildfly.cluster}</property>
                 <property name="adapterImplClass">org.jboss.as.arquillian.container.managed.ManagedDeployableContainer</property>
                 <property name="jbossHome">${keycloak.balancer.home}</property>
-                <property name="javaVmArguments">
+                <property name="jbossArguments">
                     -Djboss.socket.binding.port-offset=${auth.server.port.offset} 
-                    -Xms64m -Xmx512m -XX:MaxPermSize=256m 
-                    ${adapter.test.props}
+                </property>
+                <property name="javaVmArguments">
+                    -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m
+                    -Djava.net.preferIPv4Stack=true
                 </property>
                 <property name="managementPort">${auth.server.management.port}</property>
                 <property name="startupTimeoutInSeconds">${startup.timeout.sec}</property>
@@ -87,11 +89,14 @@
                 <property name="adapterImplClass">org.jboss.as.arquillian.container.managed.ManagedDeployableContainer</property>
                 <property name="jbossHome">${keycloak.backend1.home}</property>
                 <property name="serverConfig">standalone-ha.xml</property>
-                <property name="javaVmArguments">
+                <property name="jbossArguments">
                     -Djboss.socket.binding.port-offset=${auth.server.backend1.port.offset} 
-                    -Xms64m -Xmx512m -XX:MaxPermSize=256m 
-                    ${adapter.test.props}
                     -Djboss.node.name=node1
+                    ${adapter.test.props}
+                </property>
+                <property name="javaVmArguments">
+                    -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m
+                    -Djava.net.preferIPv4Stack=true
                 </property>
                 <!--<property name="outputToConsole">false</property>-->
                 <property name="managementPort">${auth.server.backend1.management.port}</property>
@@ -104,11 +109,14 @@
                 <property name="adapterImplClass">org.jboss.as.arquillian.container.managed.ManagedDeployableContainer</property>
                 <property name="jbossHome">${keycloak.backend2.home}</property>
                 <property name="serverConfig">standalone-ha.xml</property>
-                <property name="javaVmArguments">
+                <property name="jbossArguments">
                     -Djboss.socket.binding.port-offset=${auth.server.backend2.port.offset} 
-                    -Xms64m -Xmx512m -XX:MaxPermSize=256m 
-                    ${adapter.test.props}
                     -Djboss.node.name=node2
+                    ${adapter.test.props}
+                </property>
+                <property name="javaVmArguments">
+                    -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m
+                    -Djava.net.preferIPv4Stack=true
                 </property>
                 <!--<property name="outputToConsole">false</property>-->
                 <property name="managementPort">${auth.server.backend2.management.port}</property>