keycloak-aplcache

diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractClusterTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractClusterTest.java
index fc523b7..80ed20f 100644
--- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractClusterTest.java
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractClusterTest.java
@@ -1,19 +1,20 @@
 package org.keycloak.testsuite.cluster;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 import org.jboss.arquillian.container.test.api.ContainerController;
-import org.jboss.arquillian.graphene.page.Page;
 import org.jboss.arquillian.test.api.ArquillianResource;
 import static org.junit.Assert.assertTrue;
+import org.junit.Before;
 import org.keycloak.admin.client.Keycloak;
 import org.keycloak.models.Constants;
-import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.testsuite.AbstractKeycloakTest;
 import org.keycloak.testsuite.arquillian.ContainerInfo;
-import org.keycloak.testsuite.auth.page.AuthRealm;
 import static org.keycloak.testsuite.auth.page.AuthRealm.ADMIN;
 import static org.keycloak.testsuite.auth.page.AuthRealm.MASTER;
+import static org.keycloak.testsuite.util.WaitUtils.pause;
 
 /**
  *
@@ -24,52 +25,91 @@ public abstract class AbstractClusterTest extends AbstractKeycloakTest {
     @ArquillianResource
     protected ContainerController controller;
 
-    protected List<Keycloak> backendAdminClients = new ArrayList<>();
+    protected Map<ContainerInfo, Keycloak> backendAdminClients = new HashMap<>();
 
-    public void startBackendNodes(int count) {
-        if (count < 0 || count > 10) {
-            throw new IllegalArgumentException();
+    private int currentFailNodeIndex = 0;
+
+    public int getClusterSize() {
+        return suiteContext.getAuthServerBackendsInfo().size();
+    }
+
+    protected void iterateCurrentFailNode() {
+        currentFailNodeIndex++;
+        if (currentFailNodeIndex >= getClusterSize()) {
+            currentFailNodeIndex = 0;
         }
-        assertTrue(suiteContext.getAuthServerBackendsInfo().size() >= count);
-        for (int i = 0; i < count; i++) {
+        logCurrentState();
+    }
 
-            ContainerInfo backendNode = suiteContext.getAuthServerBackendsInfo().get(i);
+    protected ContainerInfo getCurrentFailNode() {
+        return backendNode(currentFailNodeIndex);
+    }
 
-            controller.start(backendNode.getQualifier());
-            assertTrue(controller.isStarted(backendNode.getQualifier()));
+    protected Set<ContainerInfo> getCurrentSurvivorNodes() {
+        Set<ContainerInfo> survivors = new HashSet<>(suiteContext.getAuthServerBackendsInfo());
+        survivors.remove(getCurrentFailNode());
+        return survivors;
+    }
 
-            backendAdminClients.add(createAdminClientFor(backendNode));
+    protected void logCurrentState() {
+        boolean started = controller.isStarted(getCurrentFailNode().getQualifier());
+        log.info("Fail node: " + getCurrentFailNode() + (started ? "" : " (stopped)"));
+        for (ContainerInfo survivor : getCurrentSurvivorNodes()) {
+            started = controller.isStarted(survivor.getQualifier());
+            log.info("Survivor:  " + survivor + (started ? "" : " (stopped)"));
         }
     }
 
-    protected Keycloak createAdminClientFor(ContainerInfo backendNode) {
-        log.info("Initializing admin client for " + backendNode.getContextRoot() + "/auth");
-        return Keycloak.getInstance(backendNode.getContextRoot() + "/auth",
-                MASTER, ADMIN, ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
+    public void failure() {
+        log.info("Simulating failure");
+        killBackendNode(getCurrentFailNode());
+    }
+
+    public void failback() {
+        log.info("Bringing all backend nodes online");
+        for (ContainerInfo node : suiteContext.getAuthServerBackendsInfo()) {
+            startBackendNode(node);
+        }
     }
 
     protected ContainerInfo backendNode(int i) {
         return suiteContext.getAuthServerBackendsInfo().get(i);
     }
 
-    protected void startBackendNode(int i) {
-        String container = backendNode(i).getQualifier();
-        if (!controller.isStarted(container)) {
-            controller.start(container);
-            backendAdminClients.set(i, createAdminClientFor(backendNode(i)));
+    protected void startBackendNode(ContainerInfo node) {
+        if (!controller.isStarted(node.getQualifier())) {
+            log.info("Starting backend node: " + node);
+            controller.start(node.getQualifier());
+            assertTrue(controller.isStarted(node.getQualifier()));
+        }
+        log.info("Backend node " + node + " is started");
+        if (!backendAdminClients.containsKey(node)) {
+            backendAdminClients.put(node, createAdminClientFor(node));
         }
     }
 
-    protected void killBackendNode(int i) {
-        backendAdminClients.get(i).close();
-        controller.kill(backendNode(i).getQualifier());
+    protected Keycloak createAdminClientFor(ContainerInfo node) {
+        log.info("Initializing admin client for " + node.getContextRoot() + "/auth");
+        return Keycloak.getInstance(node.getContextRoot() + "/auth",
+                MASTER, ADMIN, ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
     }
 
-    protected void listRealms(int i) {
-        log.info(String.format("Node %s: AccessTokenString: %s", i + 1, backendAdminClients.get(i).tokenManager().getAccessTokenString()));
-        for (RealmRepresentation r : backendAdminClients.get(i).realms().findAll()) {
-            log.info(String.format("Node %s: Realm: %s, Id: %s", i + 1, r.getRealm(), r.getId()));
-        }
+    protected void killBackendNode(ContainerInfo node) {
+        backendAdminClients.get(node).close();
+        backendAdminClients.remove(node);
+        log.info("Killing backend node: " + node);
+        controller.kill(node.getQualifier());
     }
-    
+
+    protected Keycloak getAdminClientFor(ContainerInfo backendNode) {
+        return backendAdminClients.get(backendNode);
+    }
+
+    @Before
+    public void beforeClusterTest() {
+        failback();
+        logCurrentState();
+        pause(3000);
+    }
+
 }

diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/EntityInvalidationClusterTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/EntityInvalidationClusterTest.java
index 2175ed3..74e6d1f 100644
--- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/EntityInvalidationClusterTest.java
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/EntityInvalidationClusterTest.java
@@ -5,19 +5,43 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import org.junit.Test;
 import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
 import static org.keycloak.testsuite.auth.page.AuthRealm.TEST;
 
 /**
  *
  * @author tkyjovsk
  */
-public class EntityInvalidationClusterTest extends AbstractTwoNodeClusterTest {
+public class EntityInvalidationClusterTest extends AbstractClusterTest {
 
     @Override
     public void addTestRealms(List<RealmRepresentation> testRealms) {
     }
 
     @Test
+    public void createRealmViaFrontend() {
+        String realm = TEST + "_fe";
+
+        RealmRepresentation testRealm = new RealmRepresentation();
+        testRealm.setRealm(realm);
+        testRealm.setEnabled(true);
+
+        // CREATE on frontend
+        adminClient.realms().create(testRealm);
+
+        // check if created on frontend
+        RealmRepresentation testRealmOnFrontend = adminClient.realms().realm(realm).toRepresentation();
+        assertEquals(testRealmOnFrontend.getRealm(), testRealm.getRealm());
+
+        // check if created on backend nodes
+        for (ContainerInfo backend : suiteContext.getAuthServerBackendsInfo()) {
+            RealmRepresentation testRealmOnBackend = getAdminClientFor(backend).realms().realm(realm).toRepresentation();
+            assertEquals(testRealmOnBackend.getId(), testRealmOnFrontend.getId());
+            assertEquals(testRealmOnBackend.getRealm(), testRealmOnFrontend.getRealm());
+        }
+    }
+
+    @Test
     public void realmCRUDWithoutFailover() {
         realmCRUD(TEST + "_wofo", false);
     }
@@ -27,83 +51,84 @@ public class EntityInvalidationClusterTest extends AbstractTwoNodeClusterTest {
         realmCRUD(TEST + "_wfo", true);
     }
 
-    public void realmCRUD(String realm, boolean containerFailover) {
+    public void realmCRUD(String realmName, boolean backendFailover) {
+
+        // CREATE on current fail node
+        log.info("Creating realm on : " + getCurrentFailNode());
         RealmRepresentation testRealm = new RealmRepresentation();
-        testRealm.setRealm(realm);
+        testRealm.setRealm(realmName);
         testRealm.setEnabled(true);
+        getAdminClientFor(getCurrentFailNode()).realms().create(testRealm);
 
-        // CREATE on node1
-        backend1AdminClient().realms().create(testRealm);
+        // check if created on fail node
+        RealmRepresentation testRealmOnFailNode = getAdminClientFor(getCurrentFailNode()).realms().realm(realmName).toRepresentation();
+        assertEquals(testRealmOnFailNode.getRealm(), testRealm.getRealm());
 
-        // check if created on node1
-        RealmRepresentation testRealmOnBackend1 = backend1AdminClient().realms().realm(realm).toRepresentation();
-        assertEquals(testRealmOnBackend1.getRealm(), testRealm.getRealm());
-        if (containerFailover) {
-            killBackend1();
+        if (backendFailover) {
+            failure();
         }
 
-        // check if created on node2
-        RealmRepresentation testRealmOnBackend2 = backend2AdminClient().realms().realm(realm).toRepresentation();
-        assertEquals(testRealmOnBackend1.getId(), testRealmOnBackend2.getId());
-        assertEquals(testRealmOnBackend1.getRealm(), testRealmOnBackend2.getRealm());
+        // check if created on survivor nodes
+        for (ContainerInfo survivorNode : getCurrentSurvivorNodes()) {
+            RealmRepresentation testRealmOnSurvivorNode = getAdminClientFor(survivorNode).realms().realm(realmName).toRepresentation();
+            assertEquals(testRealmOnFailNode.getId(), testRealmOnSurvivorNode.getId());
+            assertEquals(testRealmOnFailNode.getRealm(), testRealmOnSurvivorNode.getRealm());
+            log.info("Creation on survivor: " + survivorNode + " verified");
+        }
 
         failback();
+        iterateCurrentFailNode();
+
+        // UPDATE on current fail node
+        log.info("Updating realm on: " + getCurrentFailNode());
+        String realmBeforeUpdate = realmName;
+        realmName += "_updated";
+        testRealm = testRealmOnFailNode;
+        testRealm.setRealm(realmName);
+        getAdminClientFor(getCurrentFailNode()).realms().realm(realmBeforeUpdate).update(testRealm);
+
+        // check if updated on fail node
+        testRealmOnFailNode = getAdminClientFor(getCurrentFailNode()).realms().realm(realmName).toRepresentation();
+        assertEquals(testRealmOnFailNode.getRealm(), testRealm.getRealm());
+
+        if (backendFailover) {
+            failure();
+        }
 
-        // UPDATE on node2
-        String realmUpdated = realm + "_updated";
-        testRealmOnBackend2.setRealm(realmUpdated);
-        backend2AdminClient().realms().realm(realm).update(testRealmOnBackend2);
-        if (containerFailover) {
-            killBackend2();
+        // check if updated on survivor nodes
+        for (ContainerInfo survivorNode : getCurrentSurvivorNodes()) {
+            RealmRepresentation testRealmOnSurvivorNode = getAdminClientFor(survivorNode).realms().realm(realmName).toRepresentation();
+            assertEquals(testRealmOnFailNode.getId(), testRealmOnSurvivorNode.getId());
+            assertEquals(testRealmOnFailNode.getRealm(), testRealmOnSurvivorNode.getRealm());
+            log.info("Update on survivor: " + survivorNode + " verified");
         }
-        // check if updated on node1
-        testRealmOnBackend1 = backend1AdminClient().realms().realm(realmUpdated).toRepresentation();
-        assertEquals(testRealmOnBackend1.getId(), testRealmOnBackend2.getId());
-        assertEquals(testRealmOnBackend1.getRealm(), testRealmOnBackend2.getRealm());
 
         failback();
+        iterateCurrentFailNode();
+
+        // DELETE on current fail node
+        log.info("Deleting realm on: " + getCurrentFailNode());
+        getAdminClientFor(getCurrentFailNode()).realms().realm(realmName).remove();
 
-        // DELETE on node1
-        backend1AdminClient().realms().realm(realmUpdated).remove();
-        if (containerFailover) {
-            killBackend1();
+        if (backendFailover) {
+            failure();
         }
-        // check if deleted on node2
-        boolean testRealmOnBackend2Exists = false;
-        for (RealmRepresentation realmOnBackend2 : backend2AdminClient().realms().findAll()) {
-            if (realmUpdated.equals(realmOnBackend2.getRealm())
-                    || testRealmOnBackend1.getId().equals(realmOnBackend2.getId())) {
-                testRealmOnBackend2Exists = true;
-                break;
+
+        // check if deleted from all survivor nodes
+        boolean realmStillExists = false;
+        for (ContainerInfo survivorNode : getCurrentSurvivorNodes()) {
+            boolean realmStillExistsOnSurvivor = false;
+            for (RealmRepresentation realmOnSurvivorNode : getAdminClientFor(survivorNode).realms().findAll()) {
+                if (realmName.equals(realmOnSurvivorNode.getRealm())
+                        || testRealmOnFailNode.getId().equals(realmOnSurvivorNode.getId())) {
+                    realmStillExistsOnSurvivor = true;
+                    realmStillExists = true;
+                    break;
+                }
             }
+            log.error("Deletion on survivor: " + survivorNode + (realmStillExistsOnSurvivor ? " FAILED" : " verified"));
         }
-        assertFalse(testRealmOnBackend2Exists);
+        assertFalse(realmStillExists);
     }
 
-    @Test
-    public void createRealmViaFrontend() {
-        String realm = TEST + "_fe";
-
-        RealmRepresentation testRealm = new RealmRepresentation();
-        testRealm.setRealm(realm);
-        testRealm.setEnabled(true);
-
-        // CREATE on frontend
-        adminClient.realms().create(testRealm);
-
-        // check if created on frontend
-        RealmRepresentation testRealmOnFrontend = adminClient.realms().realm(realm).toRepresentation();
-        assertEquals(testRealmOnFrontend.getRealm(), testRealm.getRealm());
-
-        // check if created on node1
-        RealmRepresentation testRealmOnBackend1 = backend1AdminClient().realms().realm(realm).toRepresentation();
-        assertEquals(testRealmOnBackend1.getId(), testRealmOnFrontend.getId());
-        assertEquals(testRealmOnBackend1.getRealm(), testRealmOnFrontend.getRealm());
-
-        // check if created on node2
-        RealmRepresentation testRealmOnBackend2 = backend2AdminClient().realms().realm(realm).toRepresentation();
-        assertEquals(testRealmOnBackend2.getId(), testRealmOnFrontend.getId());
-        assertEquals(testRealmOnBackend2.getRealm(), testRealmOnFrontend.getRealm());
-    }
-    
 }

diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/SessionFailoverClusterTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/SessionFailoverClusterTest.java
index 591e2e8..3597b75 100644
--- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/SessionFailoverClusterTest.java
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/SessionFailoverClusterTest.java
@@ -17,7 +17,7 @@ import org.openqa.selenium.Cookie;
  *
  * @author tkyjovsk
  */
-public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
+public class SessionFailoverClusterTest extends AbstractClusterTest {
 
     public static final String KEYCLOAK_SESSION_COOKIE = "KEYCLOAK_SESSION";
     public static final String KEYCLOAK_IDENTITY_COOKIE = "KEYCLOAK_IDENTITY";
@@ -30,7 +30,7 @@ public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
     @Ignore("work in progress") // only works with owners="2" at the moment
     public void sessionFailover() {
         
-        // LOGOUT
+        // LOGIN
         accountPage.navigateTo();
         driver.navigate().refresh();
         pause(3000);
@@ -40,7 +40,7 @@ public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
         Cookie sessionCookie = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
         assertNotNull(sessionCookie);
 
-        killBackend1();
+        failure();
 
         // check if session survived backend failure
         
@@ -53,6 +53,7 @@ public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
         assertEquals(sessionCookieAfterFailover.getValue(), sessionCookie.getValue());
 
         failback();
+        iterateCurrentFailNode();
 
         // check if session survived backend failback
         driver.navigate().refresh();
@@ -71,7 +72,7 @@ public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
         sessionCookie = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
         assertNull(sessionCookie);
 
-        killBackend1();
+        failure();
         
         // check if session survived backend failure
         driver.navigate().refresh();