keycloak-aplcache

diff --git a/testsuite/integration-arquillian/servers/wildfly/pom.xml b/testsuite/integration-arquillian/servers/wildfly/pom.xml
index 87f8674..21af0f2 100644
--- a/testsuite/integration-arquillian/servers/wildfly/pom.xml
+++ b/testsuite/integration-arquillian/servers/wildfly/pom.xml
@@ -415,6 +415,11 @@
         
         <profile>
             <id>auth-server-wildfly-cluster</id>
+            <properties>
+                <session.cache.owners>1</session.cache.owners>
+                <offline.session.cache.owners>1</offline.session.cache.owners>
+                <login.failure.cache.owners>1</login.failure.cache.owners>
+            </properties>
             <build>
                 <plugins>
                     <plugin>
@@ -448,6 +453,28 @@
                                                 </parameter>
                                             </parameters>
                                         </transformationSet>
+                                        <transformationSet>
+                                            <dir>${keycloak.server.home}/standalone/configuration</dir>
+                                            <includes>
+                                                <include>standalone-ha.xml</include>
+                                            </includes>
+                                            <stylesheet>src/main/xslt/ispn-cache-owners.xsl</stylesheet>
+                                            <outputDir>${keycloak.server.home}/standalone/configuration</outputDir>
+                                            <parameters>
+                                                <parameter>
+                                                    <name>sessionCacheOwners</name>
+                                                    <value>${session.cache.owners}</value>
+                                                </parameter>
+                                                <parameter>
+                                                    <name>offlineSessionCacheOwners</name>
+                                                    <value>${offline.session.cache.owners}</value>
+                                                </parameter>
+                                                <parameter>
+                                                    <name>loginFailureCacheOwners</name>
+                                                    <value>${login.failure.cache.owners}</value>
+                                                </parameter>
+                                            </parameters>
+                                        </transformationSet>
                                     </transformationSets>
                                 </configuration>
                             </execution>

diff --git a/testsuite/integration-arquillian/servers/wildfly/src/main/xslt/ispn-cache-owners.xsl b/testsuite/integration-arquillian/servers/wildfly/src/main/xslt/ispn-cache-owners.xsl
new file mode 100644
index 0000000..7237d89
--- /dev/null
+++ b/testsuite/integration-arquillian/servers/wildfly/src/main/xslt/ispn-cache-owners.xsl
@@ -0,0 +1,40 @@
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                xmlns:xalan="http://xml.apache.org/xalan"
+                xmlns:j="urn:jboss:domain:4.0"
+                xmlns:i="urn:jboss:domain:infinispan:4.0"
+                version="2.0"
+                exclude-result-prefixes="xalan i">
+
+    <xsl:output method="xml" version="1.0" encoding="UTF-8" indent="yes" xalan:indent-amount="4" standalone="no"/>
+    <xsl:strip-space elements="*"/>
+
+    <xsl:variable name="nsDS" select="'urn:jboss:domain:datasources:'"/>
+    
+    <xsl:param name="sessionCacheOwners" select="'1'"/>
+    <xsl:param name="offlineSessionCacheOwners" select="'1'"/>
+    <xsl:param name="loginFailureCacheOwners" select="'1'"/>
+
+    <xsl:template match="//i:cache-container/i:distributed-cache[@name='sessions']/@owners">
+        <xsl:attribute name="owners">
+            <xsl:value-of select="$sessionCacheOwners"/>
+        </xsl:attribute>
+    </xsl:template>
+    <xsl:template match="//i:cache-container/i:distributed-cache[@name='offlineSessions']/@owners">
+        <xsl:attribute name="owners">
+            <xsl:value-of select="$offlineSessionCacheOwners"/>
+        </xsl:attribute>
+    </xsl:template>
+    <xsl:template match="//i:cache-container/i:distributed-cache[@name='loginFailures']/@owners">
+        <xsl:attribute name="owners">
+            <xsl:value-of select="$loginFailureCacheOwners"/>
+        </xsl:attribute>
+    </xsl:template>
+
+    <!-- Copy everything else. -->
+    <xsl:template match="@*|node()">
+        <xsl:copy>
+            <xsl:apply-templates select="@*|node()" />
+        </xsl:copy>
+    </xsl:template>
+
+</xsl:stylesheet>
\ No newline at end of file

diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/admin/ApiUtil.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/admin/ApiUtil.java
index b7af504..275fd9c 100644
--- a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/admin/ApiUtil.java
+++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/admin/ApiUtil.java
@@ -30,8 +30,10 @@ import java.net.URI;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import org.apache.commons.lang.builder.EqualsBuilder;
 
 import static org.keycloak.representations.idm.CredentialRepresentation.PASSWORD;
+import org.keycloak.representations.idm.GroupRepresentation;
 
 /**
  * Created by st on 28.05.15.
@@ -39,7 +41,7 @@ import static org.keycloak.representations.idm.CredentialRepresentation.PASSWORD
 public class ApiUtil {
     
     private static final Logger log = Logger.getLogger(ApiUtil.class);
-
+    
     public static String getCreatedId(Response response) {
         URI location = response.getLocation();
         if (location == null) {
@@ -48,7 +50,7 @@ public class ApiUtil {
         String path = location.getPath();
         return path.substring(path.lastIndexOf('/') + 1);
     }
-
+    
     public static ClientResource findClientResourceByClientId(RealmResource realm, String clientId) {
         for (ClientRepresentation c : realm.clients().findAll()) {
             if (c.getClientId().equals(clientId)) {
@@ -57,7 +59,7 @@ public class ApiUtil {
         }
         return null;
     }
-
+    
     public static ClientResource findClientResourceByName(RealmResource realm, String name) {
         for (ClientRepresentation c : realm.clients().findAll()) {
             if (c.getName().equals(name)) {
@@ -66,7 +68,7 @@ public class ApiUtil {
         }
         return null;
     }
-
+    
     public static ClientRepresentation findClientByClientId(RealmResource realm, String clientId) {
         ClientRepresentation client = null;
         for (ClientRepresentation c : realm.clients().findAll()) {
@@ -76,7 +78,7 @@ public class ApiUtil {
         }
         return client;
     }
-
+    
     public static UserRepresentation findUserByUsername(RealmResource realm, String username) {
         UserRepresentation user = null;
         List<UserRepresentation> ur = realm.users().search(username, null, null);
@@ -85,7 +87,7 @@ public class ApiUtil {
         }
         return user;
     }
-
+    
     public static String createUserWithAdminClient(RealmResource realm, UserRepresentation user) {
         Response response = realm.users().create(user);
         String createdId = getCreatedId(response);
@@ -98,7 +100,7 @@ public class ApiUtil {
         resetUserPassword(realm.users().get(id), password, false);
         return id;
     }
-
+    
     public static void resetUserPassword(UserResource userResource, String newPassword, boolean temporary) {
         CredentialRepresentation newCredential = new CredentialRepresentation();
         newCredential.setType(PASSWORD);
@@ -106,7 +108,7 @@ public class ApiUtil {
         newCredential.setTemporary(temporary);
         userResource.resetPassword(newCredential);
     }
-
+    
     public static void assignClientRoles(RealmResource realm, String userId, String clientName, String... roles) {
         String realmName = realm.toRepresentation().getRealm();
         String clientId = "";
@@ -118,21 +120,32 @@ public class ApiUtil {
         
         if (!clientId.isEmpty()) {
             ClientResource clientResource = realm.clients().get(clientId);
-
+            
             List<RoleRepresentation> roleRepresentations = new ArrayList<>();
             for (String roleName : roles) {
                 RoleRepresentation role = clientResource.roles().get(roleName).toRepresentation();
                 roleRepresentations.add(role);
             }
-
+            
             UserResource userResource = realm.users().get(userId);
-            log.debug("assigning roles: " + Arrays.toString(roles) + " to user: \"" + 
-                    userResource.toRepresentation().getUsername() + "\" of client: \"" + 
-                    clientName + "\" in realm: \"" + realmName + "\"");
+            log.debug("assigning roles: " + Arrays.toString(roles) + " to user: \""
+                    + userResource.toRepresentation().getUsername() + "\" of client: \""
+                    + clientName + "\" in realm: \"" + realmName + "\"");
             userResource.roles().clientLevel(clientId).add(roleRepresentations);
         } else {
             log.warn("client with name " + clientName + "doesn't exist in realm " + realmName);
         }
     }
-
+    
+    public static boolean groupContainsSubgroup(GroupRepresentation group, GroupRepresentation subgroup) {
+        boolean contains = false;
+        for (GroupRepresentation sg : group.getSubGroups()) {
+            if (subgroup.getId().equals(sg.getId())) {
+                contains = true;
+                break;
+            }
+        }
+        return contains;
+    }
+    
 }

diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractClusterTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractClusterTest.java
index fc523b7..24b505a 100644
--- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractClusterTest.java
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractClusterTest.java
@@ -1,19 +1,22 @@
 package org.keycloak.testsuite.cluster;
 
-import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import org.jboss.arquillian.container.test.api.ContainerController;
-import org.jboss.arquillian.graphene.page.Page;
 import org.jboss.arquillian.test.api.ArquillianResource;
 import static org.junit.Assert.assertTrue;
+import org.junit.Before;
 import org.keycloak.admin.client.Keycloak;
 import org.keycloak.models.Constants;
 import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.testsuite.AbstractKeycloakTest;
 import org.keycloak.testsuite.arquillian.ContainerInfo;
-import org.keycloak.testsuite.auth.page.AuthRealm;
 import static org.keycloak.testsuite.auth.page.AuthRealm.ADMIN;
 import static org.keycloak.testsuite.auth.page.AuthRealm.MASTER;
+import static org.keycloak.testsuite.util.WaitUtils.pause;
 
 /**
  *
@@ -24,52 +27,103 @@ public abstract class AbstractClusterTest extends AbstractKeycloakTest {
     @ArquillianResource
     protected ContainerController controller;
 
-    protected List<Keycloak> backendAdminClients = new ArrayList<>();
+    protected Map<ContainerInfo, Keycloak> backendAdminClients = new HashMap<>();
 
-    public void startBackendNodes(int count) {
-        if (count < 0 || count > 10) {
-            throw new IllegalArgumentException();
+    private int currentFailNodeIndex = 0;
+
+    public int getClusterSize() {
+        return suiteContext.getAuthServerBackendsInfo().size();
+    }
+
+    protected void iterateCurrentFailNode() {
+        currentFailNodeIndex++;
+        if (currentFailNodeIndex >= getClusterSize()) {
+            currentFailNodeIndex = 0;
         }
-        assertTrue(suiteContext.getAuthServerBackendsInfo().size() >= count);
-        for (int i = 0; i < count; i++) {
+        logFailoverSetup();
+    }
 
-            ContainerInfo backendNode = suiteContext.getAuthServerBackendsInfo().get(i);
+    protected ContainerInfo getCurrentFailNode() {
+        return backendNode(currentFailNodeIndex);
+    }
 
-            controller.start(backendNode.getQualifier());
-            assertTrue(controller.isStarted(backendNode.getQualifier()));
+    protected Set<ContainerInfo> getCurrentSurvivorNodes() {
+        Set<ContainerInfo> survivors = new HashSet<>(suiteContext.getAuthServerBackendsInfo());
+        survivors.remove(getCurrentFailNode());
+        return survivors;
+    }
 
-            backendAdminClients.add(createAdminClientFor(backendNode));
+    protected void logFailoverSetup() {
+        log.info("Current failover setup");
+        boolean started = controller.isStarted(getCurrentFailNode().getQualifier());
+        log.info("Fail node: " + getCurrentFailNode() + (started ? "" : " (stopped)"));
+        for (ContainerInfo survivor : getCurrentSurvivorNodes()) {
+            started = controller.isStarted(survivor.getQualifier());
+            log.info("Survivor:  " + survivor + (started ? "" : " (stopped)"));
         }
     }
 
-    protected Keycloak createAdminClientFor(ContainerInfo backendNode) {
-        log.info("Initializing admin client for " + backendNode.getContextRoot() + "/auth");
-        return Keycloak.getInstance(backendNode.getContextRoot() + "/auth",
-                MASTER, ADMIN, ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
+    public void failure() {
+        log.info("Simulating failure");
+        killBackendNode(getCurrentFailNode());
+    }
+
+    public void failback() {
+        log.info("Bringing all backend nodes online");
+        for (ContainerInfo node : suiteContext.getAuthServerBackendsInfo()) {
+            startBackendNode(node);
+        }
+    }
+
+    protected ContainerInfo frontendNode() {
+        return suiteContext.getAuthServerInfo();
     }
 
     protected ContainerInfo backendNode(int i) {
         return suiteContext.getAuthServerBackendsInfo().get(i);
     }
 
-    protected void startBackendNode(int i) {
-        String container = backendNode(i).getQualifier();
-        if (!controller.isStarted(container)) {
-            controller.start(container);
-            backendAdminClients.set(i, createAdminClientFor(backendNode(i)));
+    protected void startBackendNode(ContainerInfo node) {
+        if (!controller.isStarted(node.getQualifier())) {
+            log.info("Starting backend node: " + node);
+            controller.start(node.getQualifier());
+            assertTrue(controller.isStarted(node.getQualifier()));
+        }
+        log.info("Backend node " + node + " is started");
+        if (!backendAdminClients.containsKey(node)) {
+            backendAdminClients.put(node, createAdminClientFor(node));
         }
     }
 
-    protected void killBackendNode(int i) {
-        backendAdminClients.get(i).close();
-        controller.kill(backendNode(i).getQualifier());
+    protected Keycloak createAdminClientFor(ContainerInfo node) {
+        log.info("Initializing admin client for " + node.getContextRoot() + "/auth");
+        return Keycloak.getInstance(node.getContextRoot() + "/auth",
+                MASTER, ADMIN, ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
+    }
+
+    protected void killBackendNode(ContainerInfo node) {
+        backendAdminClients.get(node).close();
+        backendAdminClients.remove(node);
+        log.info("Killing backend node: " + node);
+        controller.kill(node.getQualifier());
     }
 
-    protected void listRealms(int i) {
-        log.info(String.format("Node %s: AccessTokenString: %s", i + 1, backendAdminClients.get(i).tokenManager().getAccessTokenString()));
-        for (RealmRepresentation r : backendAdminClients.get(i).realms().findAll()) {
-            log.info(String.format("Node %s: Realm: %s, Id: %s", i + 1, r.getRealm(), r.getId()));
-        }
+    protected Keycloak getAdminClientFor(ContainerInfo node) {
+        return node.equals(suiteContext.getAuthServerInfo())
+                ? adminClient // frontend client
+                : backendAdminClients.get(node);
     }
-    
+
+    @Before
+    public void beforeClusterTest() {
+        failback();
+        logFailoverSetup();
+        pause(3000);
+    }
+
+    @Override
+    public void addTestRealms(List<RealmRepresentation> testRealms) {
+        // no test realms will be created by the default 
+    }
+
 }

diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractInvalidationClusterTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractInvalidationClusterTest.java
new file mode 100644
index 0000000..89ac450
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractInvalidationClusterTest.java
@@ -0,0 +1,165 @@
+package org.keycloak.testsuite.cluster;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.commons.lang.builder.ReflectionToStringBuilder;
+import org.apache.commons.lang.builder.ToStringStyle;
+import static org.junit.Assert.assertFalse;
+import org.junit.Test;
+import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+
+/**
+ *
+ * @author tkyjovsk
+ * @param <T> entity representation
+ * @param <TR> entity resource
+ */
+public abstract class AbstractInvalidationClusterTest<T, TR> extends AbstractClusterTest {
+
+    protected RealmRepresentation createTestRealmRepresentation() {
+        RealmRepresentation testRealm = new RealmRepresentation();
+        testRealm.setRealm("test_" + RandomStringUtils.randomAlphabetic(5));
+        testRealm.setEnabled(true);
+        return testRealm;
+    }
+
+    protected abstract T createTestEntityRepresentation();
+
+    @Test
+    public void crudWithoutFailover() {
+        crud(false);
+    }
+
+    @Test
+    public void crudWithFailover() {
+        crud(true);
+    }
+
+    public void crud(boolean backendFailover) {
+        T testEntity = createTestEntityRepresentation();
+
+        // CREATE 
+        testEntity = createEntityOnCurrentFailNode(testEntity);
+
+        if (backendFailover) {
+            failure();
+        }
+
+        assertEntityOnSurvivorNodesEqualsTo(testEntity);
+
+        failback();
+        iterateCurrentFailNode();
+
+        // UPDATE(s)
+        testEntity = testEntityUpdates(testEntity, backendFailover);
+
+        // DELETE 
+        deleteEntityOnCurrentFailNode(testEntity);
+
+        if (backendFailover) {
+            failure();
+        }
+
+        assertEntityOnSurvivorNodesIsDeleted(testEntity);
+    }
+
+    protected abstract TR entityResource(T testEntity, ContainerInfo node);
+
+    protected abstract TR entityResource(String idOrName, ContainerInfo node);
+    
+    protected abstract T createEntity(T testEntity, ContainerInfo node);
+
+    protected abstract T readEntity(T entity, ContainerInfo node);
+
+    protected abstract T updateEntity(T entity, ContainerInfo node);
+
+    protected abstract void deleteEntity(T testEntity, ContainerInfo node);
+
+    protected TR entityResourceOnCurrentFailNode(T testEntity) {
+        return entityResource(testEntity, getCurrentFailNode());
+    }
+
+    protected String getEntityType(T entity) {
+        return entity.getClass().getSimpleName().replace("Representation", "");
+    }
+
+    protected T createEntityOnCurrentFailNode(T entity) {
+        log.info("Creating " + getEntityType(entity) + " on " + getCurrentFailNode());
+        return createEntity(entity, getCurrentFailNode());
+    }
+
+    protected T readEntityOnCurrentFailNode(T entity) {
+        log.debug("Reading " + getEntityType(entity) + " on " + getCurrentFailNode());
+        return readEntity(entity, getCurrentFailNode());
+    }
+
+    protected T updateEntityOnCurrentFailNode(T entity) {
+        return updateEntityOnCurrentFailNode(entity, "");
+    }
+
+    protected T updateEntityOnCurrentFailNode(T entity, String updateType) {
+        log.info("Updating " + getEntityType(entity) + " " + updateType + " on " + getCurrentFailNode());
+        return updateEntity(entity, getCurrentFailNode());
+    }
+
+    protected void deleteEntityOnCurrentFailNode(T entity) {
+        log.info("Creating " + getEntityType(entity) + " on " + getCurrentFailNode());
+        deleteEntity(entity, getCurrentFailNode());
+    }
+
+    protected abstract T testEntityUpdates(T testEntity, boolean backendFailover);
+
+    protected void verifyEntityUpdateDuringFailover(T testEntity, boolean backendFailover) {
+        if (backendFailover) {
+            failure();
+        }
+
+        assertEntityOnSurvivorNodesEqualsTo(testEntity);
+
+        failback();
+        iterateCurrentFailNode();
+    }
+
+    protected List<String> excludedComparisonFields = new ArrayList<>();
+
+    protected void assertEntityOnSurvivorNodesEqualsTo(T testEntityOnFailNode) {
+        boolean entityDiffers = false;
+        for (ContainerInfo survivorNode : getCurrentSurvivorNodes()) {
+            T testEntityOnSurvivorNode = readEntity(testEntityOnFailNode, survivorNode);
+            if (EqualsBuilder.reflectionEquals(testEntityOnSurvivorNode, testEntityOnFailNode, excludedComparisonFields)) {
+                log.info(String.format("Verification of %s on survivor %s PASSED", getEntityType(testEntityOnFailNode), survivorNode));
+            } else {
+                entityDiffers = true;
+                log.error(String.format("Verification of %s on survivor %s FAILED", getEntityType(testEntityOnFailNode), survivorNode));
+                String tf = ReflectionToStringBuilder.reflectionToString(testEntityOnFailNode, ToStringStyle.SHORT_PREFIX_STYLE);
+                String ts = ReflectionToStringBuilder.reflectionToString(testEntityOnSurvivorNode, ToStringStyle.SHORT_PREFIX_STYLE);
+                log.error(String.format(
+                        "\nEntity on fail node: \n%s\n"
+                        + "\nEntity on survivor node: \n%s\n"
+                        + "\nDifference: \n%s\n",
+                        tf, ts, StringUtils.difference(tf, ts)));
+            }
+        }
+        assertFalse(entityDiffers);
+    }
+
+    private void assertEntityOnSurvivorNodesIsDeleted(T testEntityOnFailNode) {
+        // check if deleted from all survivor nodes
+        boolean entityExists = false;
+        for (ContainerInfo survivorNode : getCurrentSurvivorNodes()) {
+            T testEntityOnSurvivorNode = readEntity(testEntityOnFailNode, survivorNode);
+            if (testEntityOnSurvivorNode == null) {
+                log.info(String.format("Verification of %s deletion on survivor %s PASSED", getEntityType(testEntityOnFailNode), survivorNode));
+            } else {
+                entityExists = true;
+                log.error(String.format("Verification of %s deletion on survivor %s FAILED", getEntityType(testEntityOnFailNode), survivorNode));
+            }
+        }
+        assertFalse(entityExists);
+    }
+
+}

diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractInvalidationClusterTestWithTestRealm.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractInvalidationClusterTestWithTestRealm.java
new file mode 100644
index 0000000..0fe6c0d
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractInvalidationClusterTestWithTestRealm.java
@@ -0,0 +1,26 @@
+package org.keycloak.testsuite.cluster;
+
+import org.junit.Before;
+import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public abstract class AbstractInvalidationClusterTestWithTestRealm<T, TR> extends AbstractInvalidationClusterTest<T, TR> {
+
+    protected String testRealmName = null;
+    
+    @Before
+    public void createTestRealm() {
+        createTestRealm(frontendNode());
+    }
+    
+    protected void createTestRealm(ContainerInfo node) {
+        RealmRepresentation r = createTestRealmRepresentation();
+        getAdminClientFor(node).realms().create(r);
+        testRealmName = r.getRealm();
+    }
+    
+}

diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/ClientInvalidationClusterTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/ClientInvalidationClusterTest.java
new file mode 100644
index 0000000..506b626
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/ClientInvalidationClusterTest.java
@@ -0,0 +1,96 @@
+package org.keycloak.testsuite.cluster;
+
+import javax.ws.rs.NotFoundException;
+import javax.ws.rs.core.Response;
+import org.apache.commons.lang.RandomStringUtils;
+import static org.junit.Assert.assertNull;
+import org.junit.Before;
+import org.keycloak.admin.client.resource.ClientResource;
+import org.keycloak.admin.client.resource.ClientsResource;
+import org.keycloak.representations.idm.ClientRepresentation;
+import org.keycloak.testsuite.admin.ApiUtil;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public class ClientInvalidationClusterTest extends AbstractInvalidationClusterTestWithTestRealm<ClientRepresentation, ClientResource> {
+
+    @Before
+    public void setExcludedComparisonFields() {
+        excludedComparisonFields.add("protocolMappers");
+    }
+
+    @Override
+    protected ClientRepresentation createTestEntityRepresentation() {
+        ClientRepresentation client = new ClientRepresentation();
+        String s = RandomStringUtils.randomAlphabetic(5);
+        client.setClientId("client_" + s);
+        client.setName("name_" + s);
+        return client;
+    }
+
+    protected ClientsResource clients(ContainerInfo node) {
+        return getAdminClientFor(node).realm(testRealmName).clients();
+    }
+
+    @Override
+    protected ClientResource entityResource(ClientRepresentation client, ContainerInfo node) {
+        return entityResource(client.getId(), node);
+    }
+
+    @Override
+    protected ClientResource entityResource(String id, ContainerInfo node) {
+        return clients(node).get(id);
+    }
+    
+    @Override
+    protected ClientRepresentation createEntity(ClientRepresentation client, ContainerInfo node) {
+        Response response = clients(node).create(client);
+        String id = ApiUtil.getCreatedId(response);
+        response.close();
+        client.setId(id);
+        return readEntity(client, node);
+    }
+
+    @Override
+    protected ClientRepresentation readEntity(ClientRepresentation client, ContainerInfo node) {
+        ClientRepresentation u = null;
+        try {
+            u = entityResource(client, node).toRepresentation();
+        } catch (NotFoundException nfe) {
+            // expected when client doesn't exist
+        }
+        return u;
+    }
+
+    @Override
+    protected ClientRepresentation updateEntity(ClientRepresentation client, ContainerInfo node) {
+        entityResource(client, node).update(client);
+        return readEntity(client, node);
+    }
+
+    @Override
+    protected void deleteEntity(ClientRepresentation client, ContainerInfo node) {
+        entityResource(client, node).remove();
+        assertNull(readEntity(client, node));
+    }
+
+    @Override
+    protected ClientRepresentation testEntityUpdates(ClientRepresentation client, boolean backendFailover) {
+
+        // clientId
+        client.setClientId(client.getClientId() + "_updated");
+        client = updateEntityOnCurrentFailNode(client, "clientId");
+        verifyEntityUpdateDuringFailover(client, backendFailover);
+
+        // name
+        client.setName(client.getName() + "_updated");
+        client = updateEntityOnCurrentFailNode(client, "name");
+        verifyEntityUpdateDuringFailover(client, backendFailover);
+
+        return client;
+    }
+
+}

diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/GroupInvalidationClusterTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/GroupInvalidationClusterTest.java
new file mode 100644
index 0000000..4b9495a
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/GroupInvalidationClusterTest.java
@@ -0,0 +1,136 @@
+package org.keycloak.testsuite.cluster;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import javax.ws.rs.NotFoundException;
+import javax.ws.rs.core.Response;
+import org.apache.commons.lang.RandomStringUtils;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+import org.keycloak.admin.client.resource.GroupResource;
+import org.keycloak.admin.client.resource.GroupsResource;
+import org.keycloak.representations.idm.GroupRepresentation;
+import org.keycloak.testsuite.admin.ApiUtil;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public class GroupInvalidationClusterTest extends AbstractInvalidationClusterTestWithTestRealm<GroupRepresentation, GroupResource> {
+
+    @Before
+    public void setExcludedComparisonFields() {
+        excludedComparisonFields.add("subGroups");
+    }
+
+    @Override
+    protected GroupRepresentation createTestEntityRepresentation() {
+        GroupRepresentation group = new GroupRepresentation();
+        group.setName("group_" + RandomStringUtils.randomAlphabetic(5));
+        group.setAttributes(new HashMap<String, List<String>>());
+        group.getAttributes().put("attr1", Arrays.asList(new String[]{"attr1 value"}));
+        group.getAttributes().put("attr2", Arrays.asList(new String[]{"attr2 value", "attr2 value2"}));
+        return group;
+    }
+
+    protected GroupsResource groups(ContainerInfo node) {
+        return getAdminClientFor(node).realm(testRealmName).groups();
+    }
+
+    @Override
+    protected GroupResource entityResource(GroupRepresentation group, ContainerInfo node) {
+        return entityResource(group.getId(), node);
+    }
+
+    @Override
+    protected GroupResource entityResource(String id, ContainerInfo node) {
+        return groups(node).group(id);
+    }
+
+    @Override
+    protected GroupRepresentation createEntity(GroupRepresentation group, ContainerInfo node) {
+        Response response = groups(node).add(group);
+        String id = ApiUtil.getCreatedId(response);
+        response.close();
+        group.setId(id);
+        return readEntity(group, node);
+    }
+
+    @Override
+    protected GroupRepresentation readEntity(GroupRepresentation group, ContainerInfo node) {
+        GroupRepresentation u = null;
+        try {
+            u = entityResource(group, node).toRepresentation();
+        } catch (NotFoundException nfe) {
+            // expected when group doesn't exist
+        }
+        return u;
+    }
+
+    @Override
+    protected GroupRepresentation updateEntity(GroupRepresentation group, ContainerInfo node) {
+        entityResource(group, node).update(group);
+        return readEntity(group, node);
+    }
+
+    @Override
+    protected void deleteEntity(GroupRepresentation group, ContainerInfo node) {
+        entityResource(group, node).remove();
+        assertNull(readEntity(group, node));
+    }
+
+    @Override
+    protected GroupRepresentation testEntityUpdates(GroupRepresentation group, boolean backendFailover) {
+
+        // groupname
+        group.setName(group.getName() + "_updated");
+        group = updateEntityOnCurrentFailNode(group, "name");
+        verifyEntityUpdateDuringFailover(group, backendFailover);
+
+        // attributes - add new
+        group.getAttributes().put("attr3", Arrays.asList(new String[]{"attr3 value"}));
+        group = updateEntityOnCurrentFailNode(group, "attributes - adding");
+        verifyEntityUpdateDuringFailover(group, backendFailover);
+
+        // attributes - remove
+        group.getAttributes().remove("attr3");
+        group = updateEntityOnCurrentFailNode(group, "attributes - removing");
+        verifyEntityUpdateDuringFailover(group, backendFailover);
+
+        // attributes - update 1
+        group.getAttributes().get("attr1").set(0,
+                group.getAttributes().get("attr1").get(0) + " - updated");
+        group = updateEntityOnCurrentFailNode(group, "attributes");
+        verifyEntityUpdateDuringFailover(group, backendFailover);
+
+        // attributes - update 2
+        group.getAttributes().get("attr2").set(1,
+                group.getAttributes().get("attr2").get(1) + " - updated");
+        group = updateEntityOnCurrentFailNode(group, "attributes");
+        verifyEntityUpdateDuringFailover(group, backendFailover);
+
+        // move 
+        log.info("Updating Group parent on " + getCurrentFailNode());
+        GroupRepresentation parentGroup = new GroupRepresentation();
+        parentGroup.setName("parent");
+        parentGroup = createEntityOnCurrentFailNode(parentGroup);
+        assertEquals("/" + parentGroup.getName(), parentGroup.getPath());
+
+        Response r = entityResourceOnCurrentFailNode(parentGroup).subGroup(group);
+        r.close();
+        parentGroup = readEntityOnCurrentFailNode(parentGroup);
+        group = readEntityOnCurrentFailNode(group);
+
+        assertTrue(ApiUtil.groupContainsSubgroup(parentGroup, group));
+        assertEquals(parentGroup.getPath() + "/" + group.getName(), group.getPath());
+
+        verifyEntityUpdateDuringFailover(group, backendFailover);
+
+        return group;
+    }
+
+}

diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/RealmInvalidationClusterTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/RealmInvalidationClusterTest.java
new file mode 100644
index 0000000..667dde9
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/RealmInvalidationClusterTest.java
@@ -0,0 +1,109 @@
+package org.keycloak.testsuite.cluster;
+
+import javax.ws.rs.NotFoundException;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
+import org.keycloak.admin.client.resource.RealmResource;
+import org.keycloak.admin.client.resource.RealmsResource;
+import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public class RealmInvalidationClusterTest extends AbstractInvalidationClusterTest<RealmRepresentation, RealmResource> {
+
+    @Override
+    protected RealmRepresentation createTestEntityRepresentation() {
+        return createTestRealmRepresentation();
+    }
+
+    protected RealmsResource realms(ContainerInfo node) {
+        return getAdminClientFor(node).realms();
+    }
+
+    @Override
+    protected RealmResource entityResource(RealmRepresentation realm, ContainerInfo node) {
+        return entityResource(realm.getRealm(), node);
+    }
+
+    @Override
+    protected RealmResource entityResource(String name, ContainerInfo node) {
+        return getAdminClientFor(node).realm(name);
+    }
+
+    @Override
+    protected RealmRepresentation createEntity(RealmRepresentation realm, ContainerInfo node) {
+        realms(node).create(realm);
+        return readEntity(realm, node);
+    }
+
+    @Override
+    protected RealmRepresentation readEntity(RealmRepresentation realm, ContainerInfo node) {
+        RealmRepresentation realmOnNode = null;
+        try {
+            realmOnNode = entityResource(realm, node).toRepresentation();
+        } catch (NotFoundException nfe) {
+            // expected if realm not found
+        }
+        return realmOnNode;
+    }
+
+    @Override
+    protected RealmRepresentation updateEntity(RealmRepresentation realm, ContainerInfo node) {
+        return updateEntity(realm.getRealm(), realm, node);
+    }
+
+    private RealmRepresentation updateEntity(String realmName, RealmRepresentation realm, ContainerInfo node) {
+        entityResource(realmName, node).update(realm);
+        return readEntity(realm, node);
+    }
+
+    @Override
+    protected void deleteEntity(RealmRepresentation realm, ContainerInfo node) {
+        entityResource(realm, node).remove();
+        // check if deleted
+        assertNull(readEntity(realm, node));
+    }
+
+    @Override
+    protected RealmRepresentation testEntityUpdates(RealmRepresentation realm, boolean backendFailover) {
+
+        // realm name
+        String originalName = realm.getRealm();
+        realm.setRealm(realm.getRealm() + "_updated");
+        realm = updateEntity(originalName, realm, getCurrentFailNode());
+        verifyEntityUpdateDuringFailover(realm, backendFailover);
+
+        // enabled
+        realm.setEnabled(!realm.isEnabled());
+        realm = updateEntityOnCurrentFailNode(realm, "enabled");
+        verifyEntityUpdateDuringFailover(realm, backendFailover);
+
+        // public key
+        realm.setPublicKey("GENERATE");
+        realm = updateEntityOnCurrentFailNode(realm, "public key");
+        assertNotEquals("GENERATE", realm.getPublicKey());
+        verifyEntityUpdateDuringFailover(realm, backendFailover);
+
+        // require ssl
+        realm.setSslRequired("all");
+        realm = updateEntityOnCurrentFailNode(realm, "require ssl");
+        verifyEntityUpdateDuringFailover(realm, backendFailover);
+
+        // brute force detection
+        realm.setBruteForceProtected(!realm.isBruteForceProtected());
+        realm = updateEntityOnCurrentFailNode(realm, "brute force");
+        verifyEntityUpdateDuringFailover(realm, backendFailover);
+
+        // brute force detection - failure factor
+        realm.setBruteForceProtected(true);
+        realm.setFailureFactor(realm.getFailureFactor() + 1);
+        realm = updateEntityOnCurrentFailNode(realm, "brute force failure factor");
+        verifyEntityUpdateDuringFailover(realm, backendFailover);
+
+        return realm;
+    }
+
+}

diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/RoleInvalidationClusterTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/RoleInvalidationClusterTest.java
new file mode 100644
index 0000000..6ae2695
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/RoleInvalidationClusterTest.java
@@ -0,0 +1,86 @@
+package org.keycloak.testsuite.cluster;
+
+import javax.ws.rs.NotFoundException;
+import org.apache.commons.lang.RandomStringUtils;
+import static org.junit.Assert.assertNull;
+import org.keycloak.admin.client.resource.RoleResource;
+import org.keycloak.admin.client.resource.RolesResource;
+import org.keycloak.representations.idm.RoleRepresentation;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public class RoleInvalidationClusterTest extends AbstractInvalidationClusterTestWithTestRealm<RoleRepresentation, RoleResource> {
+
+    @Override
+    protected RoleRepresentation createTestEntityRepresentation() {
+        RoleRepresentation role = new RoleRepresentation();
+        role.setName("role_" + RandomStringUtils.randomAlphabetic(5));
+        role.setComposite(false);
+        role.setDescription("description of "+role.getName());
+        return role;
+    }
+
+    protected RolesResource roles(ContainerInfo node) {
+        return getAdminClientFor(node).realm(testRealmName).roles();
+    }
+
+    @Override
+    protected RoleResource entityResource(RoleRepresentation role, ContainerInfo node) {
+        return entityResource(role.getName(), node);
+    }
+
+    @Override
+    protected RoleResource entityResource(String name, ContainerInfo node) {
+        return roles(node).get(name);
+    }
+
+    @Override
+    protected RoleRepresentation createEntity(RoleRepresentation role, ContainerInfo node) {
+        roles(node).create(role);
+        return readEntity(role, node);
+    }
+
+    @Override
+    protected RoleRepresentation readEntity(RoleRepresentation role, ContainerInfo node) {
+        RoleRepresentation u = null;
+        try {
+            u = entityResource(role, node).toRepresentation();
+        } catch (NotFoundException nfe) {
+            // expected when role doesn't exist
+        }
+        return u;
+    }
+
+    @Override
+    protected RoleRepresentation updateEntity(RoleRepresentation role, ContainerInfo node) {
+        return updateEntity(role.getName(), role, node);
+    }
+
+    private RoleRepresentation updateEntity(String roleName, RoleRepresentation role, ContainerInfo node) {
+        entityResource(roleName, node).update(role);
+        return readEntity(role, node);
+    }
+
+    @Override
+    protected void deleteEntity(RoleRepresentation role, ContainerInfo node) {
+        entityResource(role, node).remove();
+        assertNull(readEntity(role, node));
+    }
+
+    @Override
+    protected RoleRepresentation testEntityUpdates(RoleRepresentation role, boolean backendFailover) {
+
+        // description
+        role.setDescription(role.getDescription()+"_- updated");
+        role = updateEntityOnCurrentFailNode(role, "description");
+        verifyEntityUpdateDuringFailover(role, backendFailover);
+        
+        // TODO composites
+
+        return role;
+    }
+
+}

diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/SessionFailoverClusterTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/SessionFailoverClusterTest.java
index 591e2e8..3597b75 100644
--- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/SessionFailoverClusterTest.java
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/SessionFailoverClusterTest.java
@@ -17,7 +17,7 @@ import org.openqa.selenium.Cookie;
  *
  * @author tkyjovsk
  */
-public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
+public class SessionFailoverClusterTest extends AbstractClusterTest {
 
     public static final String KEYCLOAK_SESSION_COOKIE = "KEYCLOAK_SESSION";
     public static final String KEYCLOAK_IDENTITY_COOKIE = "KEYCLOAK_IDENTITY";
@@ -30,7 +30,7 @@ public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
     @Ignore("work in progress") // only works with owners="2" at the moment
     public void sessionFailover() {
         
-        // LOGOUT
+        // LOGIN
         accountPage.navigateTo();
         driver.navigate().refresh();
         pause(3000);
@@ -40,7 +40,7 @@ public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
         Cookie sessionCookie = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
         assertNotNull(sessionCookie);
 
-        killBackend1();
+        failure();
 
         // check if session survived backend failure
         
@@ -53,6 +53,7 @@ public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
         assertEquals(sessionCookieAfterFailover.getValue(), sessionCookie.getValue());
 
         failback();
+        iterateCurrentFailNode();
 
         // check if session survived backend failback
         driver.navigate().refresh();
@@ -71,7 +72,7 @@ public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
         sessionCookie = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
         assertNull(sessionCookie);
 
-        killBackend1();
+        failure();
         
         // check if session survived backend failure
         driver.navigate().refresh();

diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/UserInvalidationClusterTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/UserInvalidationClusterTest.java
new file mode 100644
index 0000000..440848e
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/UserInvalidationClusterTest.java
@@ -0,0 +1,94 @@
+package org.keycloak.testsuite.cluster;
+
+import javax.ws.rs.NotFoundException;
+import javax.ws.rs.core.Response;
+import org.apache.commons.lang.RandomStringUtils;
+import static org.junit.Assert.assertNull;
+import org.keycloak.admin.client.resource.UserResource;
+import org.keycloak.admin.client.resource.UsersResource;
+import org.keycloak.representations.idm.UserRepresentation;
+import org.keycloak.testsuite.admin.ApiUtil;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public class UserInvalidationClusterTest extends AbstractInvalidationClusterTestWithTestRealm<UserRepresentation, UserResource> {
+
+    @Override
+    protected UserRepresentation createTestEntityRepresentation() {
+        String firstName = "user";
+        String lastName = RandomStringUtils.randomAlphabetic(5);
+        UserRepresentation user = new UserRepresentation();
+        user.setUsername(firstName + "_" + lastName);
+        user.setEmail(user.getUsername() + "@email.test");
+        user.setFirstName(firstName);
+        user.setLastName(lastName);
+        return user;
+    }
+
+    protected UsersResource users(ContainerInfo node) {
+        return getAdminClientFor(node).realm(testRealmName).users();
+    }
+
+    @Override
+    protected UserResource entityResource(UserRepresentation user, ContainerInfo node) {
+        return entityResource(user.getId(), node);
+    }
+
+    @Override
+    protected UserResource entityResource(String id, ContainerInfo node) {
+        return users(node).get(id);
+    }
+
+    @Override
+    protected UserRepresentation createEntity(UserRepresentation user, ContainerInfo node) {
+        Response response = users(node).create(user);
+        String id = ApiUtil.getCreatedId(response);
+        response.close();
+        user.setId(id);
+        return readEntity(user, node);
+    }
+
+    @Override
+    protected UserRepresentation readEntity(UserRepresentation user, ContainerInfo node) {
+        UserRepresentation u = null;
+        try {
+            u = entityResource(user, node).toRepresentation();
+        } catch (NotFoundException nfe) {
+            // expected when user doesn't exist
+        }
+        return u;
+    }
+
+    @Override
+    protected UserRepresentation updateEntity(UserRepresentation user, ContainerInfo node) {
+        entityResource(user, node).update(user);
+        return readEntity(user, node);
+    }
+
+    @Override
+    protected void deleteEntity(UserRepresentation user, ContainerInfo node) {
+        entityResource(user, node).remove();
+        assertNull(readEntity(user, node));
+    }
+
+    @Override
+    protected UserRepresentation testEntityUpdates(UserRepresentation user, boolean backendFailover) {
+
+        // username
+        user.setUsername(user.getUsername() + "_updated");
+        user = updateEntityOnCurrentFailNode(user, "username");
+        verifyEntityUpdateDuringFailover(user, backendFailover);
+
+        // first+lastName
+        user.setFirstName(user.getFirstName() + "_updated");
+        user.setLastName(user.getLastName() + "_updated");
+        user = updateEntityOnCurrentFailNode(user, "firstName/lastName");
+        verifyEntityUpdateDuringFailover(user, backendFailover);
+
+        return user;
+    }
+
+}

diff --git a/testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml b/testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml
index fc91116..d267f06 100644
--- a/testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml
+++ b/testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml
@@ -79,6 +79,7 @@
                     -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m
                     -Djava.net.preferIPv4Stack=true
                 </property>
+                <property name="outputToConsole">${frontend.console.output}</property>
                 <property name="managementPort">${auth.server.management.port}</property>
                 <property name="startupTimeoutInSeconds">${startup.timeout.sec}</property>
             </configuration>
@@ -98,7 +99,7 @@
                     -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m
                     -Djava.net.preferIPv4Stack=true
                 </property>
-                <!--<property name="outputToConsole">false</property>-->
+                <property name="outputToConsole">${backends.console.output}</property>
                 <property name="managementPort">${auth.server.backend1.management.port}</property>
                 <property name="startupTimeoutInSeconds">${startup.timeout.sec}</property>
             </configuration>
@@ -118,7 +119,7 @@
                     -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m
                     -Djava.net.preferIPv4Stack=true
                 </property>
-                <!--<property name="outputToConsole">false</property>-->
+                <property name="outputToConsole">${backends.console.output}</property>
                 <property name="managementPort">${auth.server.backend2.management.port}</property>
                 <property name="startupTimeoutInSeconds">${startup.timeout.sec}</property>
             </configuration>

diff --git a/testsuite/integration-arquillian/tests/pom.xml b/testsuite/integration-arquillian/tests/pom.xml
index 533745d..e569881 100644
--- a/testsuite/integration-arquillian/tests/pom.xml
+++ b/testsuite/integration-arquillian/tests/pom.xml
@@ -65,6 +65,9 @@
         <arquillian-wildfly-container.version>8.2.0.Final</arquillian-wildfly-container.version>
         <version.shrinkwrap.resolvers>2.1.1</version.shrinkwrap.resolvers>
         
+        <frontend.console.output>true</frontend.console.output>
+        <backends.console.output>true</backends.console.output>
+        
     </properties>
 
     <dependencyManagement>
@@ -119,6 +122,8 @@
                             <auth.server.ssl.required>${auth.server.ssl.required}</auth.server.ssl.required>
                             <startup.timeout.sec>${startup.timeout.sec}</startup.timeout.sec>
                             <jboss.server.config.dir>${jboss.server.config.dir}</jboss.server.config.dir>
+                            <frontend.console.output>${frontend.console.output}</frontend.console.output>
+                            <backends.console.output>${backend.console.output}</backends.console.output>
                         </systemPropertyVariables>
                         <properties>
                             <property>