keycloak-aplcache
Changes
model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java 15(+12 -3)
model/infinispan/src/main/java/org/keycloak/connections/infinispan/InfinispanConnectionProvider.java 1(+1 -0)
testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/JmxInfinispanCacheStatistics.java 66(+66 -0)
testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/JmxInfinispanChannelStatistics.java 55(+55 -0)
testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AuthServerTestEnricher.java 1(+1 -0)
testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CacheStatisticsControllerEnricher.java 356(+356 -0)
testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/containers/RegistryCreator.java 11(+8 -3)
testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/InfinispanStatistics.java 88(+88 -0)
testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/jmx/JmxConnectorRegistry.java 30(+30 -0)
testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/jmx/JmxConnectorRegistryCreator.java 73(+73 -0)
testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/KeycloakArquillianExtension.java 4(+4 -0)
testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/provider/LoadBalancerControllerProvider.java 6(+0 -6)
testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/Retry.java 0(+0 -0)
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractAdminCrossDCTest.java 34(+34 -0)
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractCrossDCTest.java 72(+70 -2)
Details
diff --git a/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java b/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java
index 76b0779..53e496f 100755
--- a/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java
+++ b/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java
@@ -156,8 +156,12 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
String nodeName = config.get("nodeName", System.getProperty(InfinispanConnectionProvider.JBOSS_NODE_NAME));
String jgroupsUdpMcastAddr = config.get("jgroupsUdpMcastAddr", System.getProperty(InfinispanConnectionProvider.JGROUPS_UDP_MCAST_ADDR));
configureTransport(gcb, nodeName, jgroupsUdpMcastAddr);
+ gcb.globalJmxStatistics()
+ .jmxDomain(InfinispanConnectionProvider.JMX_DOMAIN + "-" + nodeName);
}
- gcb.globalJmxStatistics().allowDuplicateDomains(allowDuplicateJMXDomains);
+ gcb.globalJmxStatistics()
+ .allowDuplicateDomains(allowDuplicateJMXDomains)
+ .enable();
cacheManager = new DefaultCacheManager(gcb.build());
containerManaged = false;
@@ -339,8 +343,13 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
channel.setName(nodeName);
JGroupsTransport transport = new JGroupsTransport(channel);
- gcb.transport().nodeName(nodeName);
- gcb.transport().transport(transport);
+ gcb.transport()
+ .nodeName(nodeName)
+ .transport(transport)
+ .globalJmxStatistics()
+ .jmxDomain(InfinispanConnectionProvider.JMX_DOMAIN + "-" + nodeName)
+ .enable()
+ ;
logger.infof("Configured jgroups transport with the channel name: %s", nodeName);
} catch (Exception e) {
diff --git a/model/infinispan/src/main/java/org/keycloak/connections/infinispan/InfinispanConnectionProvider.java b/model/infinispan/src/main/java/org/keycloak/connections/infinispan/InfinispanConnectionProvider.java
index 7fd2652..e8cdbf6 100755
--- a/model/infinispan/src/main/java/org/keycloak/connections/infinispan/InfinispanConnectionProvider.java
+++ b/model/infinispan/src/main/java/org/keycloak/connections/infinispan/InfinispanConnectionProvider.java
@@ -55,6 +55,7 @@ public interface InfinispanConnectionProvider extends Provider {
String JBOSS_NODE_NAME = "jboss.node.name";
String JGROUPS_UDP_MCAST_ADDR = "jgroups.udp.mcast_addr";
+ String JMX_DOMAIN = "jboss.datagrid-infinispan";
<K, V> Cache<K, V> getCache(String name);
diff --git a/testsuite/integration-arquillian/servers/cache-server/jboss/pom.xml b/testsuite/integration-arquillian/servers/cache-server/jboss/pom.xml
index 8c7f830..9c2d1f9 100644
--- a/testsuite/integration-arquillian/servers/cache-server/jboss/pom.xml
+++ b/testsuite/integration-arquillian/servers/cache-server/jboss/pom.xml
@@ -100,7 +100,7 @@
<artifactId>xml-maven-plugin</artifactId>
<executions>
<execution>
- <id>configure-adapter-debug-log</id>
+ <id>configure-keycloak-caches</id>
<phase>process-test-resources</phase>
<goals>
<goal>transform</goal>
@@ -111,8 +111,9 @@
<dir>${cache.server.jboss.home}/standalone/configuration</dir>
<includes>
<include>standalone.xml</include>
+ <include>clustered.xml</include>
</includes>
- <stylesheet>${common.resources}/add-keycloak-remote-store.xsl</stylesheet>
+ <stylesheet>${common.resources}/add-keycloak-caches.xsl</stylesheet>
<outputDir>${cache.server.jboss.home}/standalone/configuration</outputDir>
</transformationSet>
</transformationSets>
@@ -173,6 +174,23 @@
<overwrite>true</overwrite>
</configuration>
</execution>
+ <execution>
+ <id>copy-cache-server-configuration-for-dc-2</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${cache.server.jboss.home}/standalone-dc-2/deployments</outputDirectory>
+ <includeEmptyDirs>true</includeEmptyDirs>
+ <resources>
+ <resource>
+ <directory>${cache.server.jboss.home}/standalone/deployments</directory>
+ </resource>
+ </resources>
+ <overwrite>true</overwrite>
+ </configuration>
+ </execution>
</executions>
</plugin>
diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/JmxInfinispanCacheStatistics.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/JmxInfinispanCacheStatistics.java
new file mode 100644
index 0000000..2dd7bbc
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/JmxInfinispanCacheStatistics.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian.annotation;
+
+import org.keycloak.testsuite.arquillian.AuthServerTestEnricher;
+import org.keycloak.testsuite.arquillian.InfinispanStatistics;
+import org.keycloak.testsuite.arquillian.InfinispanStatistics.Constants;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation for a field / method parameter annotating {@link InfinispanStatistics} object that would be used
+ * to access statistics via JMX. By default, the access to "work" cache at remote infinispan / JDG server is requested
+ * yet the same annotation is used for other caches as well.
+ *
+ * @author hmlnarik
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.FIELD, ElementType.PARAMETER})
+public @interface JmxInfinispanCacheStatistics {
+
+ /** JMX domain. Should be set to default (@{code ""}) if the node to get the statistics from should be obtained from {@link #dcIndex()} and {@link #dcNodeIndex()}. */
+ String domain() default "";
+
+ // JMX address properties
+ String type() default Constants.TYPE_CACHE;
+ String cacheName() default "work";
+ String cacheMode() default "*";
+ String cacheManagerName() default "*";
+ String component() default Constants.COMPONENT_STATISTICS;
+
+ // Host address - either given by arrangement of DC ...
+
+ /** Index of the data center, starting from 0 */
+ int dcIndex() default -1;
+ /** Index of the node within data center, starting from 0. Nodes are ordered by arquillian qualifier as per {@link AuthServerTestEnricher} */
+ int dcNodeIndex() default -1;
+
+ // ... or by specific host/port
+
+ /** Port for management */
+ int managementPort() default -1;
+ /** Name of system property to obtain management port from */
+ String managementPortProperty() default "";
+ /** Host name to connect to */
+ String host() default "";
+ /** Name of system property to obtain host name from */
+ String hostProperty() default "";
+
+}
diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/JmxInfinispanChannelStatistics.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/JmxInfinispanChannelStatistics.java
new file mode 100644
index 0000000..41e9f20
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/JmxInfinispanChannelStatistics.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian.annotation;
+
+import org.keycloak.testsuite.arquillian.InfinispanStatistics.Constants;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ *
+ * @author hmlnarik
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.FIELD, ElementType.PARAMETER})
+public @interface JmxInfinispanChannelStatistics {
+
+ /** JMX domain. Should be set to default (@{code ""}) if the node to get the statistics from should be obtained from {@link #dcIndex()} and {@link #dcNodeIndex()}. */
+ String domain() default "";
+
+ // JMX address properties
+ String type() default Constants.TYPE_CHANNEL;
+ String cluster() default "*";
+
+ // Host address - either given by arrangement of DC ...
+
+ /** Index of the data center, starting from 0 */
+ int dcIndex() default -1;
+ /** Index of the node within data center, starting from 0. Nodes are ordered by arquillian qualifier as per {@link AuthServerTestEnricher} */
+ int dcNodeIndex() default -1;
+
+ /** Port for management */
+ int managementPort() default -1;
+ /** Name of system property to obtain management port from */
+ String managementPortProperty() default "";
+ /** Host name to connect to */
+ String host() default "";
+ /** Name of system property to obtain host name from */
+ String hostProperty() default "";
+}
diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AuthServerTestEnricher.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AuthServerTestEnricher.java
index 94293dd..97347d9 100644
--- a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AuthServerTestEnricher.java
+++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AuthServerTestEnricher.java
@@ -142,6 +142,7 @@ public class AuthServerTestEnricher {
containers.stream()
.filter(c -> c.getQualifier().startsWith(AUTH_SERVER_CONTAINER + "-cross-dc-"))
+ .sorted((a, b) -> a.getQualifier().compareTo(b.getQualifier()))
.forEach(c -> {
String portOffsetString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("bindHttpPortOffset", "0");
String dcString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("dataCenter", "0");
diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CacheStatisticsControllerEnricher.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CacheStatisticsControllerEnricher.java
new file mode 100644
index 0000000..4091ca4
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CacheStatisticsControllerEnricher.java
@@ -0,0 +1,356 @@
+package org.keycloak.testsuite.arquillian;
+
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import org.keycloak.testsuite.Retry;
+import java.util.Map;
+import org.jboss.arquillian.core.api.Instance;
+import org.jboss.arquillian.core.api.annotation.Inject;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.net.MalformedURLException;
+import javax.management.MBeanServerConnection;
+import javax.management.ObjectName;
+import javax.management.remote.JMXConnector;
+import javax.management.remote.JMXServiceURL;
+import org.jboss.arquillian.container.spi.Container;
+import org.jboss.arquillian.container.spi.ContainerRegistry;
+import org.jboss.arquillian.test.spi.TestEnricher;
+import java.io.IOException;
+import java.lang.reflect.Parameter;
+import java.util.Arrays;
+import java.util.stream.Collectors;
+import javax.management.Attribute;
+import javax.management.AttributeNotFoundException;
+import javax.management.InstanceNotFoundException;
+import javax.management.IntrospectionException;
+import javax.management.MBeanAttributeInfo;
+import javax.management.MBeanException;
+import javax.management.MBeanInfo;
+import javax.management.MalformedObjectNameException;
+import javax.management.ReflectionException;
+import org.keycloak.testsuite.arquillian.annotation.JmxInfinispanCacheStatistics;
+import java.util.Set;
+import org.keycloak.testsuite.arquillian.annotation.JmxInfinispanChannelStatistics;
+import org.keycloak.testsuite.arquillian.jmx.JmxConnectorRegistry;
+import org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow;
+import java.io.NotSerializableException;
+import java.lang.management.ManagementFactory;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.jboss.arquillian.core.spi.Validate;
+import org.jboss.logging.Logger;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public class CacheStatisticsControllerEnricher implements TestEnricher {
+
+ private static final Logger LOG = Logger.getLogger(CacheStatisticsControllerEnricher.class);
+
+ @Inject
+ private Instance<ContainerRegistry> registry;
+
+ @Inject
+ private Instance<JmxConnectorRegistry> jmxConnectorRegistry;
+
+ @Inject
+ private Instance<SuiteContext> suiteContext;
+
+ @Override
+ public void enrich(Object testCase) {
+ Validate.notNull(registry.get(), "registry should not be null");
+ Validate.notNull(jmxConnectorRegistry.get(), "jmxConnectorRegistry should not be null");
+ Validate.notNull(suiteContext.get(), "suiteContext should not be null");
+
+ for (Field field : FieldUtils.getAllFields(testCase.getClass())) {
+ JmxInfinispanCacheStatistics annotation = field.getAnnotation(JmxInfinispanCacheStatistics.class);
+
+ if (annotation == null) {
+ continue;
+ }
+
+ try {
+ FieldUtils.writeField(field, testCase, getInfinispanCacheStatistics(annotation), true);
+ } catch (IOException | IllegalAccessException | MalformedObjectNameException e) {
+ throw new RuntimeException("Could not set value on field " + field);
+ }
+ }
+ }
+
+ private InfinispanStatistics getInfinispanCacheStatistics(JmxInfinispanCacheStatistics annotation) throws MalformedObjectNameException, IOException, MalformedURLException {
+ MBeanServerConnection mbsc = getJmxServerConnection(annotation);
+
+ ObjectName mbeanName = new ObjectName(String.format(
+ "%s:type=%s,name=\"%s(%s)\",manager=\"%s\",component=%s",
+ annotation.domain().isEmpty() ? getDefaultDomain(annotation.dcIndex(), annotation.dcNodeIndex()) : InfinispanConnectionProvider.JMX_DOMAIN,
+ annotation.type(),
+ annotation.cacheName(),
+ annotation.cacheMode(),
+ annotation.cacheManagerName(),
+ annotation.component()
+ ));
+
+ InfinispanStatistics value = new InfinispanCacheStatisticsImpl(mbsc, mbeanName);
+
+ if (annotation.domain().isEmpty()) {
+ try {
+ Retry.execute(() -> value.reset(), 2, 150);
+ } catch (RuntimeException ex) {
+ if (annotation.dcIndex() != -1 && annotation.dcNodeIndex() != -1
+ && suiteContext.get().getAuthServerBackendsInfo(annotation.dcIndex()).get(annotation.dcNodeIndex()).isStarted()) {
+ LOG.warn("Could not reset statistics for " + mbeanName);
+ }
+ }
+ }
+
+ return value;
+ }
+
+ private InfinispanStatistics getJGroupsChannelStatistics(JmxInfinispanChannelStatistics annotation) throws MalformedObjectNameException, IOException, MalformedURLException {
+ MBeanServerConnection mbsc = getJmxServerConnection(annotation);
+
+ ObjectName mbeanName = new ObjectName(String.format(
+ "%s:type=%s,cluster=\"%s\"",
+ annotation.domain().isEmpty() ? getDefaultDomain(annotation.dcIndex(), annotation.dcNodeIndex()) : InfinispanConnectionProvider.JMX_DOMAIN,
+ annotation.type(),
+ annotation.cluster()
+ ));
+
+ InfinispanStatistics value = new InfinispanChannelStatisticsImpl(mbsc, mbeanName);
+
+ if (annotation.domain().isEmpty()) {
+ try {
+ Retry.execute(() -> value.reset(), 2, 150);
+ } catch (RuntimeException ex) {
+ if (annotation.dcIndex() != -1 && annotation.dcNodeIndex() != -1
+ && suiteContext.get().getAuthServerBackendsInfo(annotation.dcIndex()).get(annotation.dcNodeIndex()).isStarted()) {
+ LOG.warn("Could not reset statistics for " + mbeanName);
+ }
+ }
+ }
+
+ return value;
+ }
+
+ @Override
+ public Object[] resolve(Method method) {
+ Object[] values = new Object[method.getParameterCount()];
+
+ for (int i = 0; i < method.getParameterCount(); i ++) {
+ Parameter param = method.getParameters()[i];
+
+ JmxInfinispanCacheStatistics annotation = param.getAnnotation(JmxInfinispanCacheStatistics.class);
+ if (annotation != null) try {
+ values[i] = getInfinispanCacheStatistics(annotation);
+ } catch (IOException | MalformedObjectNameException e) {
+ throw new RuntimeException("Could not set value on field " + param);
+ }
+
+ JmxInfinispanChannelStatistics channelAnnotation = param.getAnnotation(JmxInfinispanChannelStatistics.class);
+ if (channelAnnotation != null) try {
+ values[i] = getJGroupsChannelStatistics(channelAnnotation);
+ } catch (IOException | MalformedObjectNameException e) {
+ throw new RuntimeException("Could not set value on field " + param);
+ }
+ }
+
+ return values;
+ }
+
+ private String getDefaultDomain(int dcIndex, int dcNodeIndex) {
+ if (dcIndex != -1 && dcNodeIndex != -1) {
+ return InfinispanConnectionProvider.JMX_DOMAIN + "-" + suiteContext.get().getAuthServerBackendsInfo(dcIndex).get(dcNodeIndex).getQualifier();
+ }
+ return InfinispanConnectionProvider.JMX_DOMAIN;
+ }
+
+ private MBeanServerConnection getJmxServerConnection(JmxInfinispanCacheStatistics annotation) throws MalformedURLException, IOException {
+ final String host;
+ final int port;
+
+ if (annotation.dcIndex() != -1 && annotation.dcNodeIndex() != -1) {
+ ContainerInfo node = suiteContext.get().getAuthServerBackendsInfo(annotation.dcIndex()).get(annotation.dcNodeIndex());
+ Container container = node.getArquillianContainer();
+ if (container.getDeployableContainer() instanceof KeycloakOnUndertow) {
+ return ManagementFactory.getPlatformMBeanServer();
+ }
+ host = "localhost";
+ port = container.getContainerConfiguration().getContainerProperties().containsKey("managementPort")
+ ? Integer.valueOf(container.getContainerConfiguration().getContainerProperties().get("managementPort"))
+ : 9990;
+ } else {
+ host = annotation.host().isEmpty()
+ ? System.getProperty((annotation.hostProperty().isEmpty()
+ ? "keycloak.connectionsInfinispan.remoteStoreServer"
+ : annotation.hostProperty()))
+ : annotation.host();
+
+ port = annotation.managementPort() == -1
+ ? Integer.valueOf(System.getProperty((annotation.managementPortProperty().isEmpty()
+ ? "cache.server.management.port"
+ : annotation.managementPortProperty())))
+ : annotation.managementPort();
+ }
+
+ JMXServiceURL url = new JMXServiceURL("service:jmx:remote+http://" + host + ":" + port);
+ JMXConnector jmxc = jmxConnectorRegistry.get().getConnection(url);
+
+ return jmxc.getMBeanServerConnection();
+ }
+
+ private MBeanServerConnection getJmxServerConnection(JmxInfinispanChannelStatistics annotation) throws MalformedURLException, IOException {
+ final String host;
+ final int port;
+
+ if (annotation.dcIndex() != -1 && annotation.dcNodeIndex() != -1) {
+ ContainerInfo node = suiteContext.get().getAuthServerBackendsInfo(annotation.dcIndex()).get(annotation.dcNodeIndex());
+ Container container = node.getArquillianContainer();
+ if (container.getDeployableContainer() instanceof KeycloakOnUndertow) {
+ return ManagementFactory.getPlatformMBeanServer();
+ }
+ host = "localhost";
+ port = container.getContainerConfiguration().getContainerProperties().containsKey("managementPort")
+ ? Integer.valueOf(container.getContainerConfiguration().getContainerProperties().get("managementPort"))
+ : 9990;
+ } else {
+ host = annotation.host().isEmpty()
+ ? System.getProperty((annotation.hostProperty().isEmpty()
+ ? "keycloak.connectionsInfinispan.remoteStoreServer"
+ : annotation.hostProperty()))
+ : annotation.host();
+
+ port = annotation.managementPort() == -1
+ ? Integer.valueOf(System.getProperty((annotation.managementPortProperty().isEmpty()
+ ? "cache.server.management.port"
+ : annotation.managementPortProperty())))
+ : annotation.managementPort();
+ }
+
+ JMXServiceURL url = new JMXServiceURL("service:jmx:remote+http://" + host + ":" + port);
+ JMXConnector jmxc = jmxConnectorRegistry.get().getConnection(url);
+
+ return jmxc.getMBeanServerConnection();
+ }
+
+ private static abstract class CacheStatisticsImpl implements InfinispanStatistics {
+
+ protected final MBeanServerConnection mbsc;
+ private final ObjectName mbeanNameTemplate;
+ private ObjectName mbeanName;
+
+ public CacheStatisticsImpl(MBeanServerConnection mbsc, ObjectName mbeanNameTemplate) {
+ this.mbsc = mbsc;
+ this.mbeanNameTemplate = mbeanNameTemplate;
+ }
+
+ @Override
+ public boolean exists() {
+ try {
+ getMbeanName();
+ return true;
+ } catch (Exception ex) {
+ return false;
+ }
+ }
+
+ @Override
+ public Map<String, Object> getStatistics() {
+ try {
+ MBeanInfo mBeanInfo = mbsc.getMBeanInfo(getMbeanName());
+ String[] statAttrs = Arrays.asList(mBeanInfo.getAttributes()).stream()
+ .filter(MBeanAttributeInfo::isReadable)
+ .map(MBeanAttributeInfo::getName)
+ .collect(Collectors.toList())
+ .toArray(new String[] {});
+ return mbsc.getAttributes(getMbeanName(), statAttrs)
+ .asList()
+ .stream()
+ .collect(Collectors.toMap(Attribute::getName, Attribute::getValue));
+ } catch (IOException | InstanceNotFoundException | ReflectionException | IntrospectionException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ protected ObjectName getMbeanName() throws IOException, RuntimeException {
+ if (this.mbeanName == null) {
+ Set<ObjectName> queryNames = mbsc.queryNames(mbeanNameTemplate, null);
+ if (queryNames.isEmpty()) {
+ throw new RuntimeException("No MBean of template " + mbeanNameTemplate + " found at JMX server");
+ }
+ this.mbeanName = queryNames.iterator().next();
+ }
+
+ return this.mbeanName;
+ }
+
+ @Override
+ public Comparable getSingleStatistics(String statisticsName) {
+ try {
+ return (Comparable) mbsc.getAttribute(getMbeanName(), statisticsName);
+ } catch (IOException | InstanceNotFoundException | MBeanException | ReflectionException | AttributeNotFoundException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @Override
+ public void waitToBecomeAvailable(int time, TimeUnit unit) {
+ long timeInMillis = TimeUnit.MILLISECONDS.convert(time, unit);
+ Retry.execute(() -> {
+ try {
+ getMbeanName();
+ if (! isAvailable()) throw new RuntimeException("Not available");
+ } catch (Exception ex) {
+ throw new RuntimeException("Timed out while waiting for " + mbeanNameTemplate + " to become available", ex);
+ }
+ }, 1 + (int) timeInMillis / 100, 100);
+ }
+
+ protected abstract boolean isAvailable();
+ }
+
+ private static class InfinispanCacheStatisticsImpl extends CacheStatisticsImpl {
+
+ public InfinispanCacheStatisticsImpl(MBeanServerConnection mbsc, ObjectName mbeanName) {
+ super(mbsc, mbeanName);
+ }
+
+ @Override
+ public void reset() {
+ try {
+ mbsc.invoke(getMbeanName(), "resetStatistics", new Object[] {}, new String[] {});
+ } catch (IOException | InstanceNotFoundException | MBeanException | ReflectionException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @Override
+ protected boolean isAvailable() {
+ return getSingleStatistics(Constants.STAT_CACHE_ELAPSED_TIME) != null;
+ }
+ }
+
+ private static class InfinispanChannelStatisticsImpl extends CacheStatisticsImpl {
+
+ public InfinispanChannelStatisticsImpl(MBeanServerConnection mbsc, ObjectName mbeanName) {
+ super(mbsc, mbeanName);
+ }
+
+ @Override
+ public void reset() {
+ try {
+ mbsc.invoke(getMbeanName(), "resetStats", new Object[] {}, new String[] {});
+ } catch (NotSerializableException ex) {
+ // Ignore return value not serializable, the invocation has already done its job
+ } catch (IOException | InstanceNotFoundException | MBeanException | ReflectionException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @Override
+ protected boolean isAvailable() {
+ return Objects.equals(getSingleStatistics(Constants.STAT_CHANNEL_CONNECTED), Boolean.TRUE);
+ }
+ }
+}
diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/containers/RegistryCreator.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/containers/RegistryCreator.java
index a2b6ea7..41278fc 100644
--- a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/containers/RegistryCreator.java
+++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/containers/RegistryCreator.java
@@ -35,6 +35,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
+import org.mvel2.MVEL;
import static org.keycloak.testsuite.arquillian.containers.SecurityActions.isClassPresent;
import static org.keycloak.testsuite.arquillian.containers.SecurityActions.loadClass;
@@ -97,10 +98,14 @@ public class RegistryCreator {
private static final String ENABLED = "enabled";
- private boolean isEnabled(ContainerDef containerDef) {
+ private static boolean isEnabled(ContainerDef containerDef) {
Map<String, String> props = containerDef.getContainerProperties();
- return !props.containsKey(ENABLED)
- || (props.containsKey(ENABLED) && props.get(ENABLED).equals("true"));
+ try {
+ return !props.containsKey(ENABLED)
+ || (props.containsKey(ENABLED) && ! props.get(ENABLED).isEmpty() && MVEL.evalToBoolean(props.get(ENABLED), (Object) null));
+ } catch (Exception ex) {
+ return false;
+ }
}
@SuppressWarnings("rawtypes")
diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/InfinispanStatistics.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/InfinispanStatistics.java
new file mode 100644
index 0000000..b315937
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/InfinispanStatistics.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian;
+
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public interface InfinispanStatistics {
+
+ public static class Constants {
+ public static final String DOMAIN_INFINISPAN_DATAGRID = InfinispanConnectionProvider.JMX_DOMAIN;
+
+ public static final String TYPE_CHANNEL = "channel";
+ public static final String TYPE_CACHE = "Cache";
+ public static final String TYPE_CACHE_MANAGER = "CacheManager";
+
+ public static final String COMPONENT_STATISTICS = "Statistics";
+
+ /** Cache statistics */
+ public static final String STAT_CACHE_AVERAGE_READ_TIME = "averageReadTime";
+ public static final String STAT_CACHE_AVERAGE_WRITE_TIME = "averageWriteTime";
+ public static final String STAT_CACHE_ELAPSED_TIME = "elapsedTime";
+ public static final String STAT_CACHE_EVICTIONS = "evictions";
+ public static final String STAT_CACHE_HITS = "hits";
+ public static final String STAT_CACHE_HIT_RATIO = "hitRatio";
+ public static final String STAT_CACHE_MISSES = "misses";
+ public static final String STAT_CACHE_NUMBER_OF_ENTRIES = "numberOfEntries";
+ public static final String STAT_CACHE_NUMBER_OF_ENTRIES_IN_MEMORY = "numberOfEntriesInMemory";
+ public static final String STAT_CACHE_READ_WRITE_RATIO = "readWriteRatio";
+ public static final String STAT_CACHE_REMOVE_HITS = "removeHits";
+ public static final String STAT_CACHE_REMOVE_MISSES = "removeMisses";
+ public static final String STAT_CACHE_STORES = "stores";
+ public static final String STAT_CACHE_TIME_SINCE_RESET = "timeSinceReset";
+
+ /** JGroups channel statistics */
+ public static final String STAT_CHANNEL_ADDRESS = "address";
+ public static final String STAT_CHANNEL_ADDRESS_UUID = "address_uuid";
+ public static final String STAT_CHANNEL_CLOSED = "closed";
+ public static final String STAT_CHANNEL_CLUSTER_NAME = "cluster_name";
+ public static final String STAT_CHANNEL_CONNECTED = "connected";
+ public static final String STAT_CHANNEL_CONNECTING = "connecting";
+ public static final String STAT_CHANNEL_DISCARD_OWN_MESSAGES = "discard_own_messages";
+ public static final String STAT_CHANNEL_OPEN = "open";
+ public static final String STAT_CHANNEL_RECEIVED_BYTES = "received_bytes";
+ public static final String STAT_CHANNEL_RECEIVED_MESSAGES = "received_messages";
+ public static final String STAT_CHANNEL_SENT_BYTES = "sent_bytes";
+ public static final String STAT_CHANNEL_SENT_MESSAGES = "sent_messages";
+ public static final String STAT_CHANNEL_STATE = "state";
+ public static final String STAT_CHANNEL_STATS = "stats";
+ public static final String STAT_CHANNEL_VIEW = "view";
+
+ }
+
+ Map<String, Object> getStatistics();
+
+ Comparable getSingleStatistics(String statisticsName);
+
+ void waitToBecomeAvailable(int time, TimeUnit unit);
+
+ /**
+ * Resets the statistics counters.
+ */
+ void reset();
+
+ /**
+ * Returns {@code true} iff the statistics represented by this object can be retrieved from the server.
+ */
+ boolean exists();
+}
diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/jmx/JmxConnectorRegistry.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/jmx/JmxConnectorRegistry.java
new file mode 100644
index 0000000..3a87c5b
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/jmx/JmxConnectorRegistry.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian.jmx;
+
+import javax.management.remote.JMXConnector;
+import javax.management.remote.JMXServiceURL;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public interface JmxConnectorRegistry {
+ JMXConnector getConnection(JMXServiceURL url);
+
+ void closeAll();
+}
diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/jmx/JmxConnectorRegistryCreator.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/jmx/JmxConnectorRegistryCreator.java
new file mode 100644
index 0000000..50c9b96
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/jmx/JmxConnectorRegistryCreator.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian.jmx;
+
+import java.io.IOException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import javax.management.remote.JMXConnector;
+import javax.management.remote.JMXConnectorFactory;
+import javax.management.remote.JMXServiceURL;
+import org.jboss.arquillian.core.api.InstanceProducer;
+import org.jboss.arquillian.core.api.annotation.ApplicationScoped;
+import org.jboss.arquillian.core.api.annotation.Inject;
+import org.jboss.arquillian.core.api.annotation.Observes;
+import org.jboss.arquillian.test.spi.event.suite.BeforeSuite;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public class JmxConnectorRegistryCreator {
+
+ @Inject
+ @ApplicationScoped
+ private InstanceProducer<JmxConnectorRegistry> connectorRegistry;
+
+ public void configureJmxConnectorRegistry(@Observes BeforeSuite event) {
+ if (connectorRegistry.get() == null) {
+ connectorRegistry.set(new JmxConnectorRegistry() {
+
+ private volatile ConcurrentMap<JMXServiceURL, JMXConnector> connectors = new ConcurrentHashMap<>();
+
+ @Override
+ public JMXConnector getConnection(JMXServiceURL url) {
+ JMXConnector res = connectors.get(url);
+ if (res == null) {
+ try {
+ final JMXConnector conn = JMXConnectorFactory.newJMXConnector(url, null);
+ res = connectors.putIfAbsent(url, conn);
+ if (res == null) {
+ res = conn;
+ }
+ res.connect();
+ } catch (IOException ex) {
+ throw new RuntimeException("Could not instantiate JMX connector for " + url, ex);
+ }
+ }
+ return res;
+ }
+
+ @Override
+ public void closeAll() {
+ connectors.values().forEach(c -> { try { c.close(); } catch (IOException e) {} });
+ connectors.clear();
+ }
+ });
+ }
+ }
+}
diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/KeycloakArquillianExtension.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/KeycloakArquillianExtension.java
index 7757b07..33dc8c2 100644
--- a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/KeycloakArquillianExtension.java
+++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/KeycloakArquillianExtension.java
@@ -32,6 +32,7 @@ import org.jboss.arquillian.graphene.location.CustomizableURLResourceProvider;
import org.jboss.arquillian.test.spi.enricher.resource.ResourceProvider;
import org.jboss.arquillian.test.spi.execution.TestExecutionDecider;
import org.keycloak.testsuite.arquillian.h2.H2TestEnricher;
+import org.keycloak.testsuite.arquillian.jmx.JmxConnectorRegistryCreator;
import org.keycloak.testsuite.arquillian.karaf.CustomKarafContainer;
import org.keycloak.testsuite.arquillian.migration.MigrationTestExecutionDecider;
import org.keycloak.testsuite.arquillian.provider.AdminClientProvider;
@@ -44,6 +45,7 @@ import org.keycloak.testsuite.drone.HtmlUnitScreenshots;
import org.keycloak.testsuite.drone.KeycloakDronePostSetup;
import org.keycloak.testsuite.drone.KeycloakHtmlUnitInstantiator;
import org.keycloak.testsuite.drone.KeycloakWebDriverConfigurator;
+import org.jboss.arquillian.test.spi.TestEnricher;
/**
*
@@ -65,6 +67,8 @@ public class KeycloakArquillianExtension implements LoadableExtension {
.service(DeploymentScenarioGenerator.class, DeploymentTargetModifier.class)
.service(ApplicationArchiveProcessor.class, DeploymentArchiveProcessor.class)
.service(DeployableContainer.class, CustomKarafContainer.class)
+ .service(TestEnricher.class, CacheStatisticsControllerEnricher.class)
+ .observer(JmxConnectorRegistryCreator.class)
.observer(AuthServerTestEnricher.class)
.observer(AppServerTestEnricher.class)
.observer(H2TestEnricher.class);
diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/provider/LoadBalancerControllerProvider.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/provider/LoadBalancerControllerProvider.java
index 4f99feb..af1703d 100644
--- a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/provider/LoadBalancerControllerProvider.java
+++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/provider/LoadBalancerControllerProvider.java
@@ -2,14 +2,8 @@ package org.keycloak.testsuite.arquillian.provider;
import org.keycloak.testsuite.arquillian.annotation.LoadBalancer;
import java.lang.annotation.Annotation;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import org.jboss.arquillian.container.spi.event.KillContainer;
-import org.jboss.arquillian.container.spi.event.StartContainer;
-import org.jboss.arquillian.container.spi.event.StopContainer;
import org.jboss.arquillian.core.api.Instance;
import org.jboss.arquillian.core.api.annotation.Inject;
-import org.jboss.arquillian.core.api.annotation.Observes;
import org.jboss.arquillian.test.api.ArquillianResource;
import org.jboss.arquillian.test.spi.enricher.resource.ResourceProvider;
import org.keycloak.testsuite.arquillian.LoadBalancerController;
diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractAdminCrossDCTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractAdminCrossDCTest.java
index 84527f7..2baa336 100644
--- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractAdminCrossDCTest.java
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractAdminCrossDCTest.java
@@ -19,13 +19,20 @@ package org.keycloak.testsuite.crossdc;
import org.keycloak.admin.client.resource.RealmResource;
import org.keycloak.events.log.JBossLoggingEventListenerProviderFactory;
import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.testsuite.Retry;
+import org.keycloak.testsuite.arquillian.InfinispanStatistics;
import org.keycloak.testsuite.events.EventsListenerProviderFactory;
import org.keycloak.testsuite.util.TestCleanup;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import org.hamcrest.Matcher;
import org.junit.Before;
+import static org.junit.Assert.assertThat;
/**
*
@@ -78,4 +85,31 @@ public abstract class AbstractAdminCrossDCTest extends AbstractCrossDCTest {
protected TestCleanup getCleanup() {
return getCleanup(REALM_NAME);
}
+
+ protected <T extends Comparable> void assertSingleStatistics(InfinispanStatistics stats, String key, Runnable testedCode, Function<T, Matcher<? super T>> matcherOnOldStat) {
+ stats.reset();
+
+ T oldStat = (T) stats.getSingleStatistics(key);
+ testedCode.run();
+
+ Retry.execute(() -> {
+ T newStat = (T) stats.getSingleStatistics(key);
+
+ Matcher<? super T> matcherInstance = matcherOnOldStat.apply(oldStat);
+ assertThat(newStat, matcherInstance);
+ }, 5, 200);
+ }
+
+ protected void assertStatistics(InfinispanStatistics stats, Runnable testedCode, BiConsumer<Map<String, Object>, Map<String, Object>> assertionOnStats) {
+ stats.reset();
+
+ Map<String, Object> oldStat = stats.getStatistics();
+ testedCode.run();
+
+ Retry.execute(() -> {
+ Map<String, Object> newStat = stats.getStatistics();
+ assertionOnStats.accept(oldStat, newStat);
+ }, 5, 200);
+ }
+
}
diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractCrossDCTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractCrossDCTest.java
index aa674ca..c88c0c1 100644
--- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractCrossDCTest.java
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractCrossDCTest.java
@@ -32,14 +32,21 @@ import org.jboss.arquillian.test.api.ArquillianResource;
import org.junit.After;
import org.junit.Before;
+import static org.hamcrest.Matchers.lessThan;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
/**
*
* @author hmlnarik
*/
public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest {
+ // Keep the following constants in sync with arquillian
+ public static final String QUALIFIER_NODE_BALANCER = "auth-server-balancer-cross-dc";
+
@ArquillianResource
- @LoadBalancer(value = "auth-server-balancer-cross-dc")
+ @LoadBalancer(value = QUALIFIER_NODE_BALANCER)
protected LoadBalancerController loadBalancerCtrl;
@ArquillianResource
@@ -103,6 +110,11 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
return Keycloak.getInstance(node.getContextRoot() + "/auth", AuthRealm.MASTER, AuthRealm.ADMIN, AuthRealm.ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
}
+ /**
+ * Creates admin client directed to the given node.
+ * @param node
+ * @return
+ */
protected Keycloak getAdminClientFor(ContainerInfo node) {
Keycloak adminClient = backendAdminClients.get(node);
if (adminClient == null && node.equals(suiteContext.getAuthServerInfo())) {
@@ -111,13 +123,17 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
return adminClient;
}
+ /**
+ * Disables routing requests to the given data center in the load balancer.
+ * @param dcIndex
+ */
public void disableDcOnLoadBalancer(int dcIndex) {
log.infof("Disabling load balancer for dc=%d", dcIndex);
this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).forEach(c -> loadBalancerCtrl.disableBackendNodeByName(c.getQualifier()));
}
/**
- * Enables all started nodes in the given data center
+ * Enables routing requests to all started nodes to the given data center in the load balancer.
* @param dcIndex
*/
public void enableDcOnLoadBalancer(int dcIndex) {
@@ -132,11 +148,21 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
}
}
+ /**
+ * Disables routing requests to the given node within the given data center in the load balancer.
+ * @param dcIndex
+ * @param nodeIndex
+ */
public void disableLoadBalancerNode(int dcIndex, int nodeIndex) {
log.infof("Disabling load balancer for dc=%d, node=%d", dcIndex, nodeIndex);
loadBalancerCtrl.disableBackendNodeByName(this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).get(nodeIndex).getQualifier());
}
+ /**
+ * Enables routing requests to the given node within the given data center in the load balancer.
+ * @param dcIndex
+ * @param nodeIndex
+ */
public void enableLoadBalancerNode(int dcIndex, int nodeIndex) {
log.infof("Enabling load balancer for dc=%d, node=%d", dcIndex, nodeIndex);
final ContainerInfo backendNode = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).get(nodeIndex);
@@ -149,11 +175,53 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
loadBalancerCtrl.enableBackendNodeByName(backendNode.getQualifier());
}
+ /**
+ * Starts a manually-controlled backend auth-server node in cross-DC scenario.
+ * @param dcIndex
+ * @param nodeIndex
+ * @return Started instance descriptor.
+ */
+ public ContainerInfo startBackendNode(int dcIndex, int nodeIndex) {
+ assertThat((Integer) dcIndex, lessThan(this.suiteContext.getDcAuthServerBackendsInfo().size()));
+ final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
+ assertThat((Integer) nodeIndex, lessThan(dcNodes.size()));
+ ContainerInfo dcNode = dcNodes.get(nodeIndex);
+ assertTrue("Node " + dcNode.getQualifier() + " has to be controlled manually", dcNode.isManual());
+ containerController.start(dcNode.getQualifier());
+ return dcNode;
+ }
+
+ /**
+ * Stops a manually-controlled backend auth-server node in cross-DC scenario.
+ * @param dcIndex
+ * @param nodeIndex
+ * @return Stopped instance descriptor.
+ */
+ public ContainerInfo stopBackendNode(int dcIndex, int nodeIndex) {
+ assertThat((Integer) dcIndex, lessThan(this.suiteContext.getDcAuthServerBackendsInfo().size()));
+ final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
+ assertThat((Integer) nodeIndex, lessThan(dcNodes.size()));
+ ContainerInfo dcNode = dcNodes.get(nodeIndex);
+ assertTrue("Node " + dcNode.getQualifier() + " has to be controlled manually", dcNode.isManual());
+ containerController.stop(dcNode.getQualifier());
+ return dcNode;
+ }
+
+ /**
+ * Returns stream of all nodes in the given dc that are started manually.
+ * @param dcIndex
+ * @return
+ */
public Stream<ContainerInfo> getManuallyStartedBackendNodes(int dcIndex) {
final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
return dcNodes.stream().filter(ContainerInfo::isManual);
}
+ /**
+ * Returns stream of all nodes in the given dc that are started automatically.
+ * @param dcIndex
+ * @return
+ */
public Stream<ContainerInfo> getAutomaticallyStartedBackendNodes(int dcIndex) {
final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
return dcNodes.stream().filter(c -> ! c.isManual());
diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java
index 45e7571..dbef2fc 100644
--- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java
@@ -17,16 +17,13 @@
package org.keycloak.testsuite.crossdc;
import org.keycloak.admin.client.resource.UserResource;
-import org.keycloak.events.admin.OperationType;
-import org.keycloak.events.admin.ResourceType;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
import org.keycloak.models.UserModel;
import org.keycloak.representations.idm.UserRepresentation;
import org.keycloak.testsuite.Retry;
import org.keycloak.testsuite.admin.ApiUtil;
-import org.keycloak.testsuite.arquillian.ContainerInfo;
import org.keycloak.testsuite.page.LoginPasswordUpdatePage;
import org.keycloak.testsuite.pages.ErrorPage;
-import org.keycloak.testsuite.util.AdminEventPaths;
import org.keycloak.testsuite.util.GreenMailRule;
import org.keycloak.testsuite.util.MailUtils;
import java.io.IOException;
@@ -36,12 +33,20 @@ import javax.mail.MessagingException;
import javax.mail.internet.MimeMessage;
import javax.ws.rs.core.Response;
import org.jboss.arquillian.graphene.page.Page;
-import org.jboss.arquillian.test.api.ArquillianResource;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
+import org.keycloak.testsuite.arquillian.annotation.JmxInfinispanCacheStatistics;
+import org.keycloak.testsuite.arquillian.annotation.JmxInfinispanChannelStatistics;
+import org.keycloak.testsuite.arquillian.InfinispanStatistics;
+import org.keycloak.testsuite.arquillian.InfinispanStatistics.Constants;
+import java.util.concurrent.TimeUnit;
+import org.hamcrest.Matchers;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
/**
*
@@ -69,7 +74,16 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
}
@Test
- public void sendResetPasswordEmailSuccessWorksInCrossDc() throws IOException, MessagingException {
+ public void sendResetPasswordEmailSuccessWorksInCrossDc(
+ @JmxInfinispanCacheStatistics(dcIndex=0, dcNodeIndex=0, cacheName=InfinispanConnectionProvider.ACTION_TOKEN_CACHE) InfinispanStatistics cacheDc0Node0Statistics,
+ @JmxInfinispanCacheStatistics(dcIndex=0, dcNodeIndex=1, cacheName=InfinispanConnectionProvider.ACTION_TOKEN_CACHE) InfinispanStatistics cacheDc0Node1Statistics,
+ @JmxInfinispanCacheStatistics(dcIndex=1, dcNodeIndex=0, cacheName=InfinispanConnectionProvider.ACTION_TOKEN_CACHE) InfinispanStatistics cacheDc1Node0Statistics,
+ @JmxInfinispanChannelStatistics() InfinispanStatistics channelStatisticsCrossDc) throws Exception {
+ startBackendNode(0, 1);
+ cacheDc0Node1Statistics.waitToBecomeAvailable(10, TimeUnit.SECONDS);
+
+ Comparable originalNumberOfEntries = cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES);
+
UserRepresentation userRep = new UserRepresentation();
userRep.setEnabled(true);
userRep.setUsername("user1");
@@ -88,21 +102,33 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
String link = MailUtils.getPasswordResetEmailLink(message);
- driver.navigate().to(link);
+ Retry.execute(() -> channelStatisticsCrossDc.reset(), 3, 100);
+
+ assertSingleStatistics(cacheDc0Node0Statistics, Constants.STAT_CACHE_NUMBER_OF_ENTRIES,
+ () -> driver.navigate().to(link),
+ Matchers::is
+ );
passwordUpdatePage.assertCurrent();
- passwordUpdatePage.changePassword("new-pass", "new-pass");
+ // Verify that there was at least one message sent via the channel
+ assertSingleStatistics(channelStatisticsCrossDc, Constants.STAT_CHANNEL_SENT_MESSAGES,
+ () -> passwordUpdatePage.changePassword("new-pass", "new-pass"),
+ old -> greaterThan((Comparable) 0l)
+ );
+
+ // Verify that the caches are synchronized
+ assertThat(cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES), greaterThan(originalNumberOfEntries));
+ assertThat(cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES),
+ is(cacheDc1Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES)));
assertEquals("Your account has been updated.", driver.getTitle());
disableDcOnLoadBalancer(0);
enableDcOnLoadBalancer(1);
- Retry.execute(() -> {
- driver.navigate().to(link);
- errorPage.assertCurrent();
- }, 3, 400);
+ driver.navigate().to(link);
+ errorPage.assertCurrent();
}
@Ignore("KEYCLOAK-5030")
@@ -144,9 +170,10 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
loadBalancerCtrl.enableBackendNodeByName(c.getQualifier());
});
- driver.navigate().to(link);
-
- errorPage.assertCurrent();
+ Retry.execute(() -> {
+ driver.navigate().to(link);
+ errorPage.assertCurrent();
+ }, 3, 400);
}
}
diff --git a/testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml b/testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml
index ecf986f..aa8649e 100644
--- a/testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml
+++ b/testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml
@@ -49,7 +49,7 @@
<container qualifier="auth-server-undertow" mode="suite" >
<configuration>
- <property name="enabled">${auth.server.undertow}</property>
+ <property name="enabled">${auth.server.undertow} && ! ${auth.server.undertow.crossdc}</property>
<property name="bindAddress">localhost</property>
<property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
<property name="bindHttpPort">${auth.server.http.port}</property>
@@ -169,12 +169,12 @@
<!-- Cross DC with embedded undertow. Node numbering is [centre #].[node #] -->
<group qualifier="auth-server-undertow-cross-dc">
- <container qualifier="cache-server-cross-dc" mode="suite" >
+ <container qualifier="cache-server-cross-dc-1" mode="suite" >
<configuration>
<property name="enabled">${auth.server.undertow.crossdc}</property>
<property name="adapterImplClass">org.jboss.as.arquillian.container.managed.ManagedDeployableContainer</property>
<property name="jbossHome">${cache.server.home}</property>
- <property name="serverConfig">standalone.xml</property>
+ <property name="serverConfig">clustered.xml</property>
<property name="jbossArguments">
-Djboss.socket.binding.port-offset=${cache.server.port.offset}
-Djboss.default.multicast.address=234.56.78.99
@@ -192,30 +192,54 @@
</configuration>
</container>
+ <container qualifier="cache-server-cross-dc-2" mode="suite" >
+ <configuration>
+ <property name="enabled">${auth.server.undertow.crossdc}</property>
+ <property name="adapterImplClass">org.jboss.as.arquillian.container.managed.ManagedDeployableContainer</property>
+ <property name="jbossHome">${cache.server.home}</property>
+ <property name="setupCleanServerBaseDir">true</property>
+ <property name="cleanServerBaseDir">${cache.server.home}/standalone-dc-2</property>
+ <property name="serverConfig">clustered.xml</property>
+ <property name="jbossArguments">
+ -Djboss.socket.binding.port-offset=${cache.server.2.port.offset}
+ -Djboss.default.multicast.address=234.56.78.99
+ -Djboss.node.name=cache-server-dc-2
+ ${adapter.test.props}
+ ${auth.server.profile}
+ </property>
+ <property name="javaVmArguments">
+ ${auth.server.memory.settings}
+ -Djava.net.preferIPv4Stack=true
+ </property>
+ <property name="outputToConsole">${cache.server.console.output}</property>
+ <property name="managementPort">${cache.server.2.management.port}</property>
+ <property name="startupTimeoutInSeconds">${auth.server.jboss.startup.timeout}</property>
+ </configuration>
+ </container>
+
<container qualifier="auth-server-balancer-cross-dc" mode="suite" >
<configuration>
<property name="enabled">${auth.server.undertow.crossdc}</property>
<property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.lb.SimpleUndertowLoadBalancerContainer</property>
<property name="bindAddress">localhost</property>
<property name="bindHttpPort">${auth.server.http.port}</property>
- <property name="bindHttpPortOffset">5</property>
<property name="nodes">auth-server-undertow-cross-dc-0.1=http://localhost:8101,auth-server-undertow-cross-dc-0.2-manual=http://localhost:8102,auth-server-undertow-cross-dc-1.1=http://localhost:8111,auth-server-undertow-cross-dc-1.2-manual=http://localhost:8112</property>
</configuration>
</container>
- <container qualifier="auth-server-undertow-cross-dc-0.1" mode="suite" >
+ <container qualifier="auth-server-undertow-cross-dc-0_1" mode="suite" >
<configuration>
<property name="enabled">${auth.server.undertow.crossdc}</property>
<property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
<property name="bindAddress">localhost</property>
<property name="bindHttpPort">${auth.server.http.port}</property>
<property name="bindHttpPortOffset">-79</property>
- <property name="route">auth-server-undertow-cross-dc-0.1</property>
+ <property name="route">auth-server-undertow-cross-dc-0_1</property>
<property name="remoteMode">${undertow.remote}</property>
<property name="dataCenter">0</property>
<property name="keycloakConfigPropertyOverrides">{
"keycloak.connectionsInfinispan.jgroupsUdpMcastAddr": "234.56.78.1",
- "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-0.1",
+ "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-0_1",
"keycloak.connectionsInfinispan.clustered": "${keycloak.connectionsInfinispan.clustered:true}",
"keycloak.connectionsInfinispan.remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
"keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}",
@@ -226,19 +250,19 @@
}</property>
</configuration>
</container>
- <container qualifier="auth-server-undertow-cross-dc-0.2-manual" mode="manual" >
+ <container qualifier="auth-server-undertow-cross-dc-0_2-manual" mode="manual" >
<configuration>
<property name="enabled">${auth.server.undertow.crossdc}</property>
<property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
<property name="bindAddress">localhost</property>
<property name="bindHttpPort">${auth.server.http.port}</property>
<property name="bindHttpPortOffset">-78</property>
- <property name="route">auth-server-undertow-cross-dc-0.2</property>
+ <property name="route">auth-server-undertow-cross-dc-0_2-manual</property>
<property name="remoteMode">${undertow.remote}</property>
<property name="dataCenter">0</property>
<property name="keycloakConfigPropertyOverrides">{
"keycloak.connectionsInfinispan.jgroupsUdpMcastAddr": "234.56.78.1",
- "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-0.2",
+ "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-0_2-manual",
"keycloak.connectionsInfinispan.clustered": "${keycloak.connectionsInfinispan.clustered:true}",
"keycloak.connectionsInfinispan.remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
"keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}",
@@ -250,22 +274,22 @@
</configuration>
</container>
- <container qualifier="auth-server-undertow-cross-dc-1.1" mode="suite" >
+ <container qualifier="auth-server-undertow-cross-dc-1_1" mode="suite" >
<configuration>
<property name="enabled">${auth.server.undertow.crossdc}</property>
<property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
<property name="bindAddress">localhost</property>
<property name="bindHttpPort">${auth.server.http.port}</property>
<property name="bindHttpPortOffset">-69</property>
- <property name="route">auth-server-undertow-cross-dc-1.1</property>
+ <property name="route">auth-server-undertow-cross-dc-1_1</property>
<property name="remoteMode">${undertow.remote}</property>
<property name="dataCenter">1</property>
<property name="keycloakConfigPropertyOverrides">{
"keycloak.connectionsInfinispan.jgroupsUdpMcastAddr": "234.56.78.2",
- "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-1.1",
+ "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-1_1",
"keycloak.connectionsInfinispan.clustered": "${keycloak.connectionsInfinispan.clustered:true}",
"keycloak.connectionsInfinispan.remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
- "keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}",
+ "keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort.2:11222}",
"keycloak.connectionsInfinispan.remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:true}",
"keycloak.connectionsJpa.url": "${keycloak.connectionsJpa.url.crossdc:jdbc:h2:mem:test-dc-shared}",
"keycloak.connectionsJpa.driver": "${keycloak.connectionsJpa.driver.crossdc:org.h2.Driver}",
@@ -273,22 +297,22 @@
}</property>
</configuration>
</container>
- <container qualifier="auth-server-undertow-cross-dc-1.2-manual" mode="manual" >
+ <container qualifier="auth-server-undertow-cross-dc-1_2-manual" mode="manual" >
<configuration>
<property name="enabled">${auth.server.undertow.crossdc}</property>
<property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
<property name="bindAddress">localhost</property>
<property name="bindHttpPort">${auth.server.http.port}</property>
<property name="bindHttpPortOffset">-68</property>
- <property name="route">auth-server-undertow-cross-dc-1.2</property>
+ <property name="route">auth-server-undertow-cross-dc-1_2-manual</property>
<property name="remoteMode">${undertow.remote}</property>
<property name="dataCenter">1</property>
<property name="keycloakConfigPropertyOverrides">{
"keycloak.connectionsInfinispan.jgroupsUdpMcastAddr": "234.56.78.2",
- "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-1.2",
+ "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-1_2-manual",
"keycloak.connectionsInfinispan.clustered": "${keycloak.connectionsInfinispan.clustered:true}",
"keycloak.connectionsInfinispan.remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
- "keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}",
+ "keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort.2:11222}",
"keycloak.connectionsInfinispan.remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:true}",
"keycloak.connectionsJpa.url": "${keycloak.connectionsJpa.url.crossdc:jdbc:h2:mem:test-dc-shared}",
"keycloak.connectionsJpa.driver": "${keycloak.connectionsJpa.driver.crossdc:org.h2.Driver}",
diff --git a/testsuite/integration-arquillian/tests/pom.xml b/testsuite/integration-arquillian/tests/pom.xml
index 5ff3e02..e5bba34 100755
--- a/testsuite/integration-arquillian/tests/pom.xml
+++ b/testsuite/integration-arquillian/tests/pom.xml
@@ -73,9 +73,12 @@
<cache.server.home>${containers.home}/${cache.server.container}</cache.server.home>
<cache.server.port.offset>1010</cache.server.port.offset>
<cache.server.management.port>11000</cache.server.management.port>
+ <cache.server.2.port.offset>2010</cache.server.2.port.offset>
+ <cache.server.2.management.port>12000</cache.server.2.management.port>
<cache.server.console.output>true</cache.server.console.output>
<keycloak.connectionsInfinispan.remoteStoreServer>localhost</keycloak.connectionsInfinispan.remoteStoreServer>
<keycloak.connectionsInfinispan.remoteStorePort>12232</keycloak.connectionsInfinispan.remoteStorePort>
+ <keycloak.connectionsInfinispan.remoteStorePort.2>13232</keycloak.connectionsInfinispan.remoteStorePort.2>
<keycloak.connectionsJpa.url.crossdc>jdbc:h2:mem:test-dc-shared</keycloak.connectionsJpa.url.crossdc>
<adapter.test.props/>
@@ -177,6 +180,23 @@
</executions>
</plugin>
<plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>clean-second-cache-server-arquillian-bug-workaround</id>
+ <phase>process-test-resources</phase>
+ <goals><goal>run</goal></goals>
+ <configuration>
+ <target>
+ <echo>${cache.server.home}/standalone-dc-2</echo>
+ <delete failonerror="false" dir="${cache.server.home}/standalone-dc-2" />
+ <mkdir dir="${cache.server.home}/standalone-dc-2/deployments" />
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables>
@@ -252,8 +272,11 @@
<cache.server.home>${cache.server.home}</cache.server.home>
<cache.server.console.output>${cache.server.console.output}</cache.server.console.output>
<cache.server.management.port>${cache.server.management.port}</cache.server.management.port>
+ <cache.server.2.port.offset>${cache.server.2.port.offset}</cache.server.2.port.offset>
+ <cache.server.2.management.port>${cache.server.2.management.port}</cache.server.2.management.port>
<keycloak.connectionsInfinispan.remoteStorePort>${keycloak.connectionsInfinispan.remoteStorePort}</keycloak.connectionsInfinispan.remoteStorePort>
+ <keycloak.connectionsInfinispan.remoteStorePort.2>${keycloak.connectionsInfinispan.remoteStorePort.2}</keycloak.connectionsInfinispan.remoteStorePort.2>
<keycloak.connectionsInfinispan.remoteStoreServer>${keycloak.connectionsInfinispan.remoteStoreServer}</keycloak.connectionsInfinispan.remoteStoreServer>
<keycloak.connectionsJpa.url.crossdc>${keycloak.connectionsJpa.url.crossdc}</keycloak.connectionsJpa.url.crossdc>