keycloak-uncached
Changes
testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/WaitUtils.java 1(+1 -0)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/htmlunit/HtmlUnitLoginLogoutPerfTest.java 40(+21 -19)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientLoginLogoutPerfTest.java 63(+32 -31)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientPerformanceTest.java 2(+2 -0)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/LoginLogoutParameters.java 18(+0 -18)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/LoginLogoutTestParameters.java 38(+38 -0)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/LoopingRunnable.java 32(+16 -16)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/Results.java 23(+0 -23)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/ResultsWithThroughput.java 48(+0 -48)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/OperationTimeoutException.java 14(+7 -7)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceMeasurement.java 145(+145 -0)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceStatistics.java 56(+56 -0)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceTest.java 190(+99 -91)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceTestMetrics.java 51(+0 -51)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/DataHoldingUpdatableStatistic.java 44(+24 -20)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/MovingUpdatableStatistic.java 34(+18 -16)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/SimpleStatistic.java 20(+10 -10)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/SimpleStatistics.java 17(+17 -0)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/Statistic.java 8(+5 -3)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/Statistics.java 5(+3 -2)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/UpdatableStatistic.java 11(+7 -4)
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/UpdatableStatistics.java 10(+5 -5)
Details
diff --git a/testsuite/integration-arquillian/servers/auth-server/jboss/common/keycloak-add-user.json b/testsuite/integration-arquillian/servers/auth-server/jboss/common/keycloak-add-user.json
new file mode 100644
index 0000000..60c0f09
--- /dev/null
+++ b/testsuite/integration-arquillian/servers/auth-server/jboss/common/keycloak-add-user.json
@@ -0,0 +1,15 @@
+[ {
+ "realm" : "master",
+ "users" : [ {
+ "username" : "admin",
+ "enabled" : true,
+ "credentials" : [ {
+ "type" : "password",
+ "hashedSaltedValue" : "dqalJHLkWhUJZO/q6+z1fvXOohTcGCXcvoU8xCEyvTxGN4wmLx7DtyhKuefggh6Bkx1I2eBTEX4tiWggwyXMDw==",
+ "salt" : "3fBAt5GAGGxFrV9fznpZHQ==",
+ "hashIterations" : 100000,
+ "algorithm" : "pbkdf2"
+ } ],
+ "realmRoles" : [ "admin" ]
+ } ]
+} ]
\ No newline at end of file
diff --git a/testsuite/integration-arquillian/servers/auth-server/jboss/common/mod_cluster.xsl b/testsuite/integration-arquillian/servers/auth-server/jboss/common/mod_cluster.xsl
index c5d983d..685211b 100644
--- a/testsuite/integration-arquillian/servers/auth-server/jboss/common/mod_cluster.xsl
+++ b/testsuite/integration-arquillian/servers/auth-server/jboss/common/mod_cluster.xsl
@@ -22,8 +22,26 @@
<xsl:output method="xml" version="1.0" encoding="UTF-8" indent="yes" xalan:indent-amount="4" standalone="no"/>
<xsl:strip-space elements="*"/>
-
- <!--add socket binding-->
+
+ <xsl:param name="load.metric" select="'simple'" />\
+
+ <!-- mod-cluster-config -->
+ <xsl:template match="//*[local-name()='mod-cluster-config']">
+ <mod-cluster-config advertise-socket="modcluster" connector="ajp">
+ <xsl:choose>
+ <xsl:when test="$load.metric='simple'">
+ <simple-load-provider factor="1"/>
+ </xsl:when>
+ <xsl:otherwise>
+ <dynamic-load-provider>
+ <load-metric type="{$load.metric}"/>
+ </dynamic-load-provider>
+ </xsl:otherwise>
+ </xsl:choose>
+ </mod-cluster-config>
+ </xsl:template>
+
+ <!--add socket-binding-->
<xsl:template match="//*[local-name()='socket-binding-group' and @name='standard-sockets']/*[local-name()='socket-binding' and @name='modcluster']">
<socket-binding name="modcluster" interface="private" port="0" multicast-address="${{jboss.default.multicast.address:230.0.0.4}}" multicast-port="23364"/>
</xsl:template>
diff --git a/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml b/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml
index 1958b8d..39062fb 100644
--- a/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml
+++ b/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml
@@ -406,60 +406,60 @@
</execution>
</executions>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.8</version>
- <executions>
- <execution>
- <id>inject-truststore-into-keycloak-server-json</id>
- <phase>process-resources</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <ant antfile="../build-truststore.xml" inheritRefs="true">
- <target name="inject-truststore"/>
- </ant>
- </target>
- </configuration>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>ant-contrib</groupId>
- <artifactId>ant-contrib</artifactId>
- <version>1.0b3</version>
- <exclusions>
- <exclusion>
- <groupId>ant</groupId>
- <artifactId>ant</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.ant</groupId>
- <artifactId>ant-apache-bsf</artifactId>
- <version>1.9.3</version>
- </dependency>
- <dependency>
- <groupId>org.apache.bsf</groupId>
- <artifactId>bsf-api</artifactId>
- <version>3.1</version>
- </dependency>
- <dependency>
- <groupId>rhino</groupId>
- <artifactId>js</artifactId>
- <version>1.7R2</version>
- </dependency>
- <dependency>
- <groupId>org.keycloak</groupId>
- <artifactId>keycloak-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>inject-truststore-into-keycloak-server-json</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <ant antfile="../build-truststore.xml" inheritRefs="true">
+ <target name="inject-truststore"/>
+ </ant>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>ant-contrib</groupId>
+ <artifactId>ant-contrib</artifactId>
+ <version>1.0b3</version>
+ <exclusions>
+ <exclusion>
+ <groupId>ant</groupId>
+ <artifactId>ant</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ant</groupId>
+ <artifactId>ant-apache-bsf</artifactId>
+ <version>1.9.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.bsf</groupId>
+ <artifactId>bsf-api</artifactId>
+ <version>3.1</version>
+ </dependency>
+ <dependency>
+ <groupId>rhino</groupId>
+ <artifactId>js</artifactId>
+ <version>1.7R2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.keycloak</groupId>
+ <artifactId>keycloak-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
</plugins>
</pluginManagement>
</build>
@@ -615,6 +615,12 @@
<session.cache.owners>1</session.cache.owners>
<offline.session.cache.owners>1</offline.session.cache.owners>
<login.failure.cache.owners>1</login.failure.cache.owners>
+
+ <load.metric>simple</load.metric>
+ <!-- The default value 'simple' configures mod-cluster with simple-load-provider.
+ Any other value configures it with dynamic-load-provider using the particular `load.metric`.
+ Supported metrics: https://docs.jboss.org/mod_cluster/1.2.0/html/java.AS7config.html#LoadMetric -->
+
</properties>
<build>
<pluginManagement>
@@ -701,6 +707,12 @@
</includes>
<stylesheet>${common.resources}/mod_cluster.xsl</stylesheet>
<outputDir>${auth.server.home}/standalone/configuration</outputDir>
+ <parameters>
+ <parameter>
+ <name>load.metric</name>
+ <value>${load.metric}</value>
+ </parameter>
+ </parameters>
</transformationSet>
</transformationSets>
</configuration>
@@ -711,6 +723,40 @@
</pluginManagement>
</build>
</profile>
+
+ <profile>
+ <id>admin</id>
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-keycloak-add-user-json</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${auth.server.home}/standalone/configuration</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${common.resources}</directory>
+ <includes>
+ <include>keycloak-add-user.json</include>
+ </includes>
+ </resource>
+ </resources>
+ <overwrite>true</overwrite>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+ </profile>
<profile>
<id>auth-server-wildfly</id>
diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/WaitUtils.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/WaitUtils.java
index c81a22c..5124377 100644
--- a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/WaitUtils.java
+++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/WaitUtils.java
@@ -51,6 +51,7 @@ public final class WaitUtils {
Thread.sleep(millis);
} catch (InterruptedException ex) {
Logger.getLogger(WaitUtils.class.getName()).log(Level.SEVERE, null, ex);
+ Thread.currentThread().interrupt();
}
}
diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/pom.xml b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/pom.xml
index 4264a04..4c7147c 100644
--- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/pom.xml
+++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.keycloak.testsuite</groupId>
<artifactId>integration-arquillian-tests-adapters-jboss</artifactId>
- <version>1.9.3.Final-SNAPSHOT</version>
+ <version>1.9.6.Final-SNAPSHOT</version>
</parent>
<artifactId>integration-arquillian-tests-adapters-remote</artifactId>
@@ -49,6 +49,11 @@
<artifactId>commons-io</artifactId>
<version>2.4</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-csv</artifactId>
+ <version>1.2</version>
+ </dependency>
</dependencies>
<build>
@@ -157,7 +162,7 @@
<warmup.load>1</warmup.load>
<warmup.duration>10</warmup.duration>
<max.iterations>0</max.iterations>
- <sleep.between.repeats>30</sleep.between.repeats>
+ <sleep.between.loops>30</sleep.between.loops>
</systemPropertyVariables>
</configuration>
</plugin>
@@ -166,4 +171,4 @@
</profile>
</profiles>
-</project>
\ No newline at end of file
+</project>
diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/README.md b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/README.md
index a9bd7fd..40926cb 100644
--- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/README.md
+++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/README.md
@@ -1,2 +1,65 @@
-#
+# Keycloak Adapter Tests - JBoss Remote
+
+## Performance Tests
+
+### Parameters
+
+* Warmup phase
+ - `warmup.load` Load during warmup phase (# of clients).
+ - `warmup.duration` Duration of warmup phase in seconds.
+* Measuremet iterations
+ - `initial.load` Load for the initial measurement iteration (# of clients).
+ - `load.increase` How many clients to add after each iteration.
+ - `load.increase.rate` How many clients to add per second.
+ - `measurement.duration` Duration of measurement iteration (in seconds).
+* Limits
+ - `max.iterations`
+ - `max.threads`
+* Other
+ - `sleep.between.loops` Sleep period between scenario loops.
+
+### Generated Load
+
+Warmup phase and measurement iterations with load-increase phases in between.
+
+ load
+
+ ^
+ │
+ │ /
+ │ _________/
+ │ /| |
+ │ / | |
+ │ _________/ | |
+ │ /| | | |
+ │ / | | | |
+ │ _________/ | | | |
+ │ /│ | | | | |
+ │ / | | | | | |
+ │ _________/ | | | | | |
+ │ /| | | | | | | |
+ │ ____________/ | | | | | | | |
+ │ /| | | | | | | | | |
+ │/ | | | | | | | | | |
+ └──|────────────|─|─────────|──|─────────|──|─────────|──|─────────|───────> time
+
+ <--warmup--> <--it.1-> <--it.2-> <--it.3-> <--it.4->
+
+
+### Login-Logout Test Scenario
+
+#### Collected Statistics
+
+ - ACCESS_REQUEST_TIME
+ - LOGIN_REQUEST_TIME
+ - LOGIN_VERIFY_REQUEST_TIME
+ - LOGOUT_REQUEST_TIME
+ - LOGOUT_VERIFY_REQUEST_TIME
+
+#### Parameters
+
+* Limits
+ - `max.login.time.average` Maximum accepted average value of LOGIN_REQUEST_TIME.
+ - `max.logout.time.average` Maximum accepted average value of LOGOUT_REQUEST_TIME.
+ - `max.timeout.percentage` Maximum accepted timeout percentage for all statistics.
diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/htmlunit/HtmlUnitLoginLogoutPerfTest.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/htmlunit/HtmlUnitLoginLogoutPerfTest.java
index b279098..de8e29f 100644
--- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/htmlunit/HtmlUnitLoginLogoutPerfTest.java
+++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/htmlunit/HtmlUnitLoginLogoutPerfTest.java
@@ -13,18 +13,19 @@ import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.testsuite.arquillian.annotation.AppServerContainer;
import org.keycloak.testsuite.performance.page.AppProfileJEE;
import org.openqa.selenium.By;
-import static org.keycloak.testsuite.performance.LoginLogoutParameters.AVERAGE_LOGIN_TIME_LIMIT;
-import static org.keycloak.testsuite.performance.LoginLogoutParameters.AVERAGE_LOGOUT_TIME_LIMIT;
-import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGIN_REQUEST_TIME;
-import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGOUT_REQUEST_TIME;
-import static org.keycloak.testsuite.performance.LoginLogoutParameters.ACCESS_REQUEST_TIME;
-import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGIN_VERIFY_REQUEST_TIME;
-import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGOUT_VERIFY_REQUEST_TIME;
+import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGIN_REQUEST_TIME;
+import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGOUT_REQUEST_TIME;
+import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.ACCESS_REQUEST_TIME;
+import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGIN_VERIFY_REQUEST_TIME;
+import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGOUT_VERIFY_REQUEST_TIME;
import org.keycloak.testsuite.performance.PerformanceTest;
-import static org.junit.Assert.assertTrue;
import org.keycloak.testsuite.performance.OperationTimeoutException;
-import static org.keycloak.testsuite.util.IOUtil.loadRealm;
import org.openqa.selenium.TimeoutException;
+import org.keycloak.testsuite.performance.PerformanceMeasurement;
+import org.keycloak.testsuite.performance.LoginLogoutTestParameters;
+import static org.junit.Assert.assertTrue;
+import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.PASSWORD_HASH_ITERATIONS;
+import static org.keycloak.testsuite.util.IOUtil.loadRealm;
/**
*
@@ -65,7 +66,9 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest {
@Override
public void addAdapterTestRealms(List<RealmRepresentation> testRealms) {
- testRealms.add(loadRealm("/examples-realm.json"));
+ RealmRepresentation examplesRealm = loadRealm("/examples-realm.json");
+ examplesRealm.setPasswordPolicy("hashIterations(" + PASSWORD_HASH_ITERATIONS + ")");
+ testRealms.add(examplesRealm);
}
@Before
@@ -83,16 +86,15 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest {
}
@Override
- protected boolean isLatestResultsWithinLimits() {
- return getLatestResults().get(LOGIN_REQUEST_TIME).getAverage() < AVERAGE_LOGIN_TIME_LIMIT
- && getLatestResults().get(LOGOUT_REQUEST_TIME).getAverage() < AVERAGE_LOGOUT_TIME_LIMIT;
+ protected boolean isMeasurementWithinLimits(PerformanceMeasurement measurement) {
+ return LoginLogoutTestParameters.isMeasurementWithinLimits(measurement);
}
public class Runnable extends HtmlUnitPerformanceTest.Runnable {
@Override
public void performanceScenario() throws Exception {
- LOG.debug(String.format("Starting login-logout scenario #%s", getRepeatCounter()));
+ LOG.trace(String.format("Starting login-logout scenario #%s", getLoopCounter()));
driver.manage().deleteAllCookies();
// ACCESS
@@ -104,7 +106,7 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest {
} catch (TimeoutException ex) {
throw new OperationTimeoutException(ACCESS_REQUEST_TIME, ex);
}
- metrics.addValue(ACCESS_REQUEST_TIME, timer.getElapsedTime());
+ statistics.addValue(ACCESS_REQUEST_TIME, timer.getElapsedTime());
// LOGIN
LOG.trace("Logging in");
@@ -119,7 +121,7 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest {
} catch (TimeoutException ex) {
throw new OperationTimeoutException(LOGIN_REQUEST_TIME, ex);
}
- metrics.addValue(LOGIN_REQUEST_TIME, timer.getElapsedTime());
+ statistics.addValue(LOGIN_REQUEST_TIME, timer.getElapsedTime());
// VERIFY LOGIN
LOG.trace("Verifying login");
@@ -130,7 +132,7 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest {
} catch (TimeoutException ex) {
throw new OperationTimeoutException(LOGIN_VERIFY_REQUEST_TIME, ex);
}
- metrics.addValue(LOGIN_VERIFY_REQUEST_TIME, timer.getElapsedTime());
+ statistics.addValue(LOGIN_VERIFY_REQUEST_TIME, timer.getElapsedTime());
// LOGOUT
LOG.trace("Logging out");
@@ -141,7 +143,7 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest {
} catch (TimeoutException ex) {
throw new OperationTimeoutException(LOGOUT_REQUEST_TIME, ex);
}
- metrics.addValue(LOGOUT_REQUEST_TIME, timer.getElapsedTime());
+ statistics.addValue(LOGOUT_REQUEST_TIME, timer.getElapsedTime());
// VERIFY LOGOUT
LOG.trace("Verifying logout");
@@ -152,7 +154,7 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest {
} catch (TimeoutException ex) {
throw new OperationTimeoutException(LOGOUT_VERIFY_REQUEST_TIME, ex);
}
- metrics.addValue(LOGOUT_VERIFY_REQUEST_TIME, timer.getElapsedTime());
+ statistics.addValue(LOGOUT_VERIFY_REQUEST_TIME, timer.getElapsedTime());
LOG.trace("Logged out");
}
diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientLoginLogoutPerfTest.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientLoginLogoutPerfTest.java
index 2a6e301..9590958 100644
--- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientLoginLogoutPerfTest.java
+++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientLoginLogoutPerfTest.java
@@ -27,18 +27,19 @@ import org.junit.Before;
import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.testsuite.arquillian.annotation.AppServerContainer;
import org.keycloak.testsuite.performance.page.AppProfileJEE;
-import static org.keycloak.testsuite.performance.LoginLogoutParameters.AVERAGE_LOGIN_TIME_LIMIT;
-import static org.keycloak.testsuite.performance.LoginLogoutParameters.AVERAGE_LOGOUT_TIME_LIMIT;
-import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGIN_REQUEST_TIME;
-import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGOUT_REQUEST_TIME;
-import static org.keycloak.testsuite.performance.LoginLogoutParameters.ACCESS_REQUEST_TIME;
-import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGIN_VERIFY_REQUEST_TIME;
-import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGOUT_VERIFY_REQUEST_TIME;
+import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.ACCESS_REQUEST_TIME;
+import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGIN_VERIFY_REQUEST_TIME;
+import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGOUT_VERIFY_REQUEST_TIME;
import org.keycloak.testsuite.performance.PerformanceTest;
import org.keycloak.testsuite.performance.OperationTimeoutException;
+import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGIN_REQUEST_TIME;
+import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGOUT_REQUEST_TIME;
+import org.keycloak.testsuite.performance.PerformanceMeasurement;
+import org.keycloak.testsuite.performance.LoginLogoutTestParameters;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.PASSWORD_HASH_ITERATIONS;
import static org.keycloak.testsuite.util.IOUtil.loadRealm;
/**
@@ -80,7 +81,9 @@ public class HttpClientLoginLogoutPerfTest extends HttpClientPerformanceTest {
@Override
public void addAdapterTestRealms(List<RealmRepresentation> testRealms) {
- testRealms.add(loadRealm("/examples-realm.json"));
+ RealmRepresentation examplesRealm = loadRealm("/examples-realm.json");
+ examplesRealm.setPasswordPolicy("hashIterations(" + PASSWORD_HASH_ITERATIONS + ")");
+ testRealms.add(examplesRealm);
}
@Before
@@ -98,17 +101,15 @@ public class HttpClientLoginLogoutPerfTest extends HttpClientPerformanceTest {
}
@Override
- protected boolean isLatestResultsWithinLimits() {
- return isLatestTimeoutsWithinLimits()
- && getLatestResults().get(LOGIN_REQUEST_TIME).getAverage() < AVERAGE_LOGIN_TIME_LIMIT
- && getLatestResults().get(LOGOUT_REQUEST_TIME).getAverage() < AVERAGE_LOGOUT_TIME_LIMIT;
+ protected boolean isMeasurementWithinLimits(PerformanceMeasurement measurement) {
+ return LoginLogoutTestParameters.isMeasurementWithinLimits(measurement);
}
public class Runnable extends HttpClientPerformanceTest.Runnable {
@Override
public void performanceScenario() throws IOException, OperationTimeoutException {
- LOG.debug(String.format("Starting login-logout scenario #%s", getRepeatCounter()));
+ LOG.trace(String.format("Starting login-logout scenario #%s", getLoopCounter()));
context.getCookieStore().clear();
// ACCESS
@@ -118,17 +119,17 @@ public class HttpClientLoginLogoutPerfTest extends HttpClientPerformanceTest {
LOG.trace(getSecuredPageRequest);
timer.reset();
try (CloseableHttpResponse r = client.execute(getSecuredPageRequest, context)) {
- assertEquals(HTTP_OK, r.getStatusLine().getStatusCode());
+ assertEquals("ACCESS_REQUEST OK", HTTP_OK, r.getStatusLine().getStatusCode());
logRedirects();
- assertEquals(1, context.getRedirectLocations().size());
- assertTrue(getLastRedirect().toASCIIString().startsWith(loginPageUrl));
+ assertEquals("ACCESS_REQUEST has 1 redirect", 1, context.getRedirectLocations().size());
+ assertTrue("ACCESS_REQUEST redirects to login page", getLastRedirect().toASCIIString().startsWith(loginPageUrl));
pageContent = EntityUtils.toString(r.getEntity());
} catch (SocketException ex) {
throw new OperationTimeoutException(ACCESS_REQUEST_TIME, ex);
} catch (SocketTimeoutException ex) {
throw new OperationTimeoutException(ACCESS_REQUEST_TIME, ex.bytesTransferred, ex);
}
- metrics.addValue(ACCESS_REQUEST_TIME, timer.getElapsedTime());
+ statistics.addValue(ACCESS_REQUEST_TIME, timer.getElapsedTime());
// LOGIN
final HttpPost loginRequest = new HttpPost(getLoginUrlFromPage(pageContent));
@@ -141,31 +142,31 @@ public class HttpClientLoginLogoutPerfTest extends HttpClientPerformanceTest {
LOG.trace(loginRequest);
timer.reset();
try (CloseableHttpResponse r = client.execute(loginRequest, context)) {
- assertEquals(HTTP_OK, r.getStatusLine().getStatusCode());
+ assertEquals("LOGIN_REQUEST OK", HTTP_OK, r.getStatusLine().getStatusCode());
logRedirects();
- assertEquals(2, context.getRedirectLocations().size());
- assertTrue(getLastRedirect().toASCIIString().equals(securedUrl));
+ assertEquals("LOGIN_REQUEST has 2 redirects", 2, context.getRedirectLocations().size());
+ assertTrue("LOGIN_REQUEST redirects to secured page", getLastRedirect().toASCIIString().equals(securedUrl));
} catch (SocketException ex) {
throw new OperationTimeoutException(LOGIN_REQUEST_TIME, ex);
} catch (SocketTimeoutException ex) {
throw new OperationTimeoutException(LOGIN_REQUEST_TIME, ex.bytesTransferred, ex);
}
- metrics.addValue(LOGIN_REQUEST_TIME, timer.getElapsedTime());
+ statistics.addValue(LOGIN_REQUEST_TIME, timer.getElapsedTime());
// VERIFY LOGIN
LOG.trace("Verifying login");
LOG.trace(getSecuredPageRequest);
timer.reset();
try (CloseableHttpResponse r = client.execute(getSecuredPageRequest, context)) {
- assertEquals(HTTP_OK, r.getStatusLine().getStatusCode());
+ assertEquals("LOGIN_VERIFY_REQUEST OK", HTTP_OK, r.getStatusLine().getStatusCode());
logRedirects();
- assertEquals(0, context.getRedirectLocations().size());
+ assertEquals("LOGIN_VERIFY_REQUEST has 0 redirects", 0, context.getRedirectLocations().size());
} catch (SocketException ex) {
throw new OperationTimeoutException(LOGIN_VERIFY_REQUEST_TIME, ex);
} catch (SocketTimeoutException ex) {
throw new OperationTimeoutException(LOGIN_VERIFY_REQUEST_TIME, ex.bytesTransferred, ex);
}
- metrics.addValue(LOGIN_VERIFY_REQUEST_TIME, timer.getElapsedTime());
+ statistics.addValue(LOGIN_VERIFY_REQUEST_TIME, timer.getElapsedTime());
// LOGOUT
final HttpGet logoutRequest = new HttpGet(logoutUrl);
@@ -173,31 +174,31 @@ public class HttpClientLoginLogoutPerfTest extends HttpClientPerformanceTest {
LOG.trace(logoutRequest);
timer.reset();
try (CloseableHttpResponse r = client.execute(logoutRequest, context)) {
- assertEquals(HTTP_OK, r.getStatusLine().getStatusCode());
+ assertEquals("LOGOUT_REQUEST OK", HTTP_OK, r.getStatusLine().getStatusCode());
logRedirects();
- assertEquals(0, context.getRedirectLocations().size());
+ assertEquals("LOGOUT_REQUEST has 0 redirects", 0, context.getRedirectLocations().size());
} catch (SocketException ex) {
throw new OperationTimeoutException(LOGOUT_REQUEST_TIME, ex);
} catch (SocketTimeoutException ex) {
throw new OperationTimeoutException(LOGOUT_REQUEST_TIME, ex.bytesTransferred, ex);
}
- metrics.addValue(LOGOUT_REQUEST_TIME, timer.getElapsedTime());
+ statistics.addValue(LOGOUT_REQUEST_TIME, timer.getElapsedTime());
// VERIFY LOGOUT
LOG.trace("Verifying logout");
LOG.trace(getSecuredPageRequest);
timer.reset();
try (CloseableHttpResponse r = client.execute(getSecuredPageRequest, context)) {
- assertEquals(HTTP_OK, r.getStatusLine().getStatusCode());
+ assertEquals("LOGOUT_VERIFY_REQUEST OK", HTTP_OK, r.getStatusLine().getStatusCode());
logRedirects();
- assertEquals(1, context.getRedirectLocations().size());
- assertTrue(getLastRedirect().toASCIIString().startsWith(loginPageUrl));
+ assertEquals("LOGOUT_VERIFY_REQUEST has 1 redirect", 1, context.getRedirectLocations().size());
+ assertTrue("LOGOUT_VERIFY_REQUEST redirects to login page", getLastRedirect().toASCIIString().startsWith(loginPageUrl));
} catch (SocketException ex) {
throw new OperationTimeoutException(LOGOUT_VERIFY_REQUEST_TIME, ex);
} catch (SocketTimeoutException ex) {
throw new OperationTimeoutException(LOGOUT_VERIFY_REQUEST_TIME, ex.bytesTransferred, ex);
}
- metrics.addValue(LOGOUT_VERIFY_REQUEST_TIME, timer.getElapsedTime());
+ statistics.addValue(LOGOUT_VERIFY_REQUEST_TIME, timer.getElapsedTime());
LOG.trace("Logged out");
diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientPerformanceTest.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientPerformanceTest.java
index d9aafaf..e6504e5 100644
--- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientPerformanceTest.java
+++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientPerformanceTest.java
@@ -12,6 +12,7 @@ import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.SocketConfig;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.client.DefaultRedirectStrategy;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
@@ -45,6 +46,7 @@ public abstract class HttpClientPerformanceTest extends PerformanceTest {
.setDefaultCookieStore(new BasicCookieStore())
.setDefaultRequestConfig(getDefaultRequestConfig())
.setRedirectStrategy(new CustomRedirectStrategy())
+ .setRetryHandler(new DefaultHttpRequestRetryHandler(0, false))
.build();
}
diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/LoginLogoutTestParameters.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/LoginLogoutTestParameters.java
new file mode 100644
index 0000000..70dbaec
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/LoginLogoutTestParameters.java
@@ -0,0 +1,38 @@
+package org.keycloak.testsuite.performance;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public class LoginLogoutTestParameters {
+
+ // Statistics
+ public static final String ACCESS_REQUEST_TIME = "ACCESS_REQUEST";
+ public static final String LOGIN_REQUEST_TIME = "LOGIN_REQUEST";
+ public static final String LOGIN_VERIFY_REQUEST_TIME = "LOGIN_VERIFY_REQUEST";
+ public static final String LOGOUT_REQUEST_TIME = "LOGOUT_REQUEST";
+ public static final String LOGOUT_VERIFY_REQUEST_TIME = "LOGOUT_VERIFY_REQUEST";
+
+ // Limits
+ public static final Integer MAX_LOGIN_TIME_AVERAGE = Integer.parseInt(System.getProperty("max.login.time.average", "500"));
+ public static final Integer MAX_LOGOUT_TIME_AVERAGE = Integer.parseInt(System.getProperty("max.logout.time.average", "500"));
+ public static final double MAX_TIMEOUT_PERCENTAGE = Double.parseDouble(System.getProperty("max.timeout.percentage", "0"));
+
+ // Other
+ public static final Integer PASSWORD_HASH_ITERATIONS = Integer.parseInt(System.getProperty("password.hash.iterations", "1"));
+
+ public static boolean isMeasurementWithinLimits(PerformanceMeasurement measurement) {
+ return isTimeoutPercentageWithinLimits(measurement)
+ && measurement.getStatistics().get(LOGIN_REQUEST_TIME).getAverage() < MAX_LOGIN_TIME_AVERAGE
+ && measurement.getStatistics().get(LOGOUT_REQUEST_TIME).getAverage() < MAX_LOGOUT_TIME_AVERAGE;
+ }
+
+ public static boolean isTimeoutPercentageWithinLimits(PerformanceMeasurement measurement) {
+ boolean withinLimits = true;
+ for (String statistic : measurement.getStatistics().keySet()) {
+ withinLimits = withinLimits && measurement.getTimeoutPercentage(statistic) <= MAX_TIMEOUT_PERCENTAGE;
+ }
+ return withinLimits;
+ }
+
+}
diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/OperationTimeoutException.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/OperationTimeoutException.java
index deb5026..c364766 100644
--- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/OperationTimeoutException.java
+++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/OperationTimeoutException.java
@@ -6,21 +6,21 @@ package org.keycloak.testsuite.performance;
*/
public class OperationTimeoutException extends Exception {
- private final String metric;
+ private final String statistic;
private final long value;
- public OperationTimeoutException(String metric, Throwable cause) {
- this(metric, 0, cause);
+ public OperationTimeoutException(String statistic, Throwable cause) {
+ this(statistic, 0, cause);
}
- public OperationTimeoutException(String metric, long value, Throwable cause) {
+ public OperationTimeoutException(String statistic, long value, Throwable cause) {
super(cause);
- this.metric = metric;
+ this.statistic = statistic;
this.value = value;
}
- public String getMetric() {
- return metric;
+ public String getStatistic() {
+ return statistic;
}
public long getValue() {
diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceMeasurement.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceMeasurement.java
new file mode 100644
index 0000000..32ec4a2
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceMeasurement.java
@@ -0,0 +1,145 @@
+package org.keycloak.testsuite.performance;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
+import static org.keycloak.testsuite.performance.PerformanceTest.LOG;
+import org.keycloak.testsuite.performance.statistics.SimpleStatistics;
+import static org.keycloak.testsuite.util.IOUtil.PROJECT_BUILD_DIRECTORY;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public class PerformanceMeasurement {
+
+ private final Date started;
+ private final int load;
+ private long durationMillis;
+ private SimpleStatistics statistics;
+ private SimpleStatistics timeoutStatistics;
+
+ public static final DateFormat ISO8601_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXX"); // should be compatible with `date --iso-8601=seconds`
+ public static final DateFormat RFC3339_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ssZ"); // should be compatible with `date --rfc-3339=seconds`
+
+ public PerformanceMeasurement(int load) {
+ this.started = new Date();
+ this.load = load;
+ }
+
+ public SimpleStatistics getStatistics() {
+ return this.statistics;
+ }
+
+ public SimpleStatistics getTimeoutStatistics() {
+ return this.timeoutStatistics;
+ }
+
+ public void setStatistics(SimpleStatistics statistics, SimpleStatistics timeoutStatistics) {
+ this.durationMillis = new Date().getTime() - started.getTime();
+ if (durationMillis < 0) {
+ throw new IllegalStateException("Cannot set a negative duration.");
+ }
+ this.statistics = statistics;
+ this.timeoutStatistics = timeoutStatistics;
+ }
+
+ private void checkStatisticsNotNull() {
+ if (statistics == null || timeoutStatistics == null) {
+ throw new IllegalStateException("Iteration doesn't have any statistics set.");
+ }
+ }
+
+ public double getThroughput(String statistic) {
+ checkStatisticsNotNull();
+ return (double) statistics.get(statistic).getCount() / durationMillis * 1000;
+ }
+
+ public double getTimeoutPercentage(String statistic) {
+ checkStatisticsNotNull();
+ long timeouts = timeoutStatistics.containsKey(statistic) ? timeoutStatistics.get(statistic).getCount() : 0;
+ return (double) timeouts / statistics.get(statistic).getCount();
+ }
+
+ public static final Object[] HEADER = new String[]{
+ "Timestamp",
+ "Load",
+ "Duration",
+ "Count",
+ "Min",
+ "Max",
+ "Average",
+ "Standard Deviation",
+ "Timeout Percentage",
+ "Throughput",};
+
+ public List toRecord(String statistic) {
+ checkStatisticsNotNull();
+ List record = new ArrayList();
+ record.add(ISO8601_DATE_FORMAT.format(started));
+ record.add(load);
+ record.add(durationMillis);
+ record.add(statistics.get(statistic).getCount());
+ record.add(statistics.get(statistic).getMin());
+ record.add(statistics.get(statistic).getMax());
+ record.add(statistics.get(statistic).getAverage());
+ record.add(statistics.get(statistic).getStandardDeviation());
+ record.add(getTimeoutPercentage(statistic));
+ record.add(getThroughput(statistic));
+ return record;
+ }
+
+ public void printToCSV() {
+ printToCSV(null);
+ }
+
+ public void printToCSV(String testName) {
+ checkStatisticsNotNull();
+ for (String statistic : statistics.keySet()) {
+
+ File csvFile = new File(PROJECT_BUILD_DIRECTORY + "/measurements" + (testName == null ? "" : "/" + testName),
+ statistic + ".csv");
+ boolean csvFileCreated = false;
+ if (!csvFile.exists()) {
+ try {
+ csvFile.getParentFile().mkdirs();
+ csvFileCreated = csvFile.createNewFile();
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(csvFile, true))) {
+
+ CSVPrinter printer = new CSVPrinter(writer, CSVFormat.RFC4180);
+
+ if (csvFileCreated) {
+ printer.printRecord(HEADER);
+ }
+ printer.printRecord(toRecord(statistic));
+
+ printer.flush();
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ }
+
+ public void printToLog() {
+ LOG.info("Measurement results:");
+ LOG.info("Operation " + Arrays.toString(HEADER));
+ for (String statistic : statistics.keySet()) {
+ LOG.info(statistic + " " + toRecord(statistic));
+ }
+ }
+
+}
diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceStatistics.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceStatistics.java
new file mode 100644
index 0000000..5209bbb
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceStatistics.java
@@ -0,0 +1,56 @@
+package org.keycloak.testsuite.performance;
+
+import java.util.concurrent.ConcurrentHashMap;
+import org.keycloak.testsuite.performance.statistics.DataHoldingUpdatableStatistic;
+import org.keycloak.testsuite.performance.statistics.MovingUpdatableStatistic;
+import org.keycloak.testsuite.performance.statistics.SimpleStatistics;
+import org.keycloak.testsuite.performance.statistics.UpdatableStatistics;
+import org.keycloak.testsuite.performance.statistics.UpdatableStatistic;
+
+/**
+ * PerformanceStatistics. Concurrent hash map of UpdatableStatistic objects,
+ * type of which can be selected by the "statistic.type" property.
+ *
+ * @author tkyjovsk
+ */
+public class PerformanceStatistics extends ConcurrentHashMap<String, UpdatableStatistic> implements UpdatableStatistics {
+
+ public static final String STATISTIC_TYPE = System.getProperty("statistic.type", MovingUpdatableStatistic.STATISTIC_TYPE_PROPERTY_VALUE);
+
+ @Override
+ public void reset() {
+ clear();
+ }
+
+ private UpdatableStatistic createIfNullAndGet(String statistic) {
+ UpdatableStatistic updatableStatistic = get(statistic);
+ if (updatableStatistic == null) {
+ switch (STATISTIC_TYPE) {
+ case DataHoldingUpdatableStatistic.STATISTIC_TYPE_PROPERTY_VALUE:
+ updatableStatistic = new DataHoldingUpdatableStatistic();
+ break;
+ case MovingUpdatableStatistic.STATISTIC_TYPE_PROPERTY_VALUE:
+ updatableStatistic = new DataHoldingUpdatableStatistic();
+ break;
+ default:
+ throw new IllegalStateException(String.format(
+ "Unknown statistic type: '%s'. Supported values: %s | %s",
+ STATISTIC_TYPE,
+ DataHoldingUpdatableStatistic.STATISTIC_TYPE_PROPERTY_VALUE,
+ MovingUpdatableStatistic.STATISTIC_TYPE_PROPERTY_VALUE));
+ }
+ put(statistic, updatableStatistic);
+ }
+ return updatableStatistic;
+ }
+
+ @Override
+ public void addValue(String statistic, long value) {
+ createIfNullAndGet(statistic).addValue(value);
+ }
+
+ public SimpleStatistics snapshot() {
+ return new SimpleStatistics(this);
+ }
+
+}
diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceTest.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceTest.java
index a4278e1..24c2d90 100644
--- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceTest.java
+++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceTest.java
@@ -11,8 +11,6 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.keycloak.testsuite.adapter.AbstractExampleAdapterTest;
-import org.keycloak.testsuite.performance.metrics.impl.Results;
-import org.keycloak.testsuite.performance.metrics.impl.ResultsWithThroughput;
import static org.keycloak.testsuite.util.WaitUtils.pause;
/**
@@ -21,12 +19,12 @@ import static org.keycloak.testsuite.util.WaitUtils.pause;
* @author tkyjovsk
*/
public abstract class PerformanceTest extends AbstractExampleAdapterTest {
-
- private final Logger LOG = Logger.getLogger(PerformanceTest.class);
-
+
+ public static final Logger LOG = Logger.getLogger(PerformanceTest.class);
+
public static final Integer WARMUP_LOAD = Integer.parseInt(System.getProperty("warmup.load", "5"));
public static final Integer WARMUP_DURATION = Integer.parseInt(System.getProperty("warmup.duration", "30"));
-
+
public static final Integer INITIAL_LOAD = Integer.parseInt(System.getProperty("initial.load", "10")); // load for the first iteration
public static final Integer LOAD_INCREASE = Integer.parseInt(System.getProperty("load.increase", "10")); // how many threads to add before each iteration
public static final Integer LOAD_INCREASE_RATE = Integer.parseInt(System.getProperty("load.increase.rate", "2")); // how fast to add the new threads per second
@@ -35,76 +33,83 @@ public abstract class PerformanceTest extends AbstractExampleAdapterTest {
public static final Integer MAX_ITERATIONS = Integer.parseInt(System.getProperty("max.iterations", "10"));
public static final Integer MAX_THREADS = Integer.parseInt(System.getProperty("max.threads", "1000"));
-
- public static final Integer SLEEP_BETWEEN_REPEATS = Integer.parseInt(System.getProperty("sleep.between.repeats", "0"));
-
- private final double AVERAGE_TIMEOUT_PERCENTAGE_LIMIT = Double.parseDouble(System.getProperty("average.timeout.percentage.limit", "0.01"));
-
+
+ public static final Integer SLEEP_BETWEEN_LOOPS = Integer.parseInt(System.getProperty("sleep.between.loops", "0"));
+ public static final Integer THREADPOOL_TERMINATION_TIMEOUT = Integer.parseInt(System.getProperty("threadpool.termination.timeout", "10"));
+ public static final Integer ADDITIONAL_SLEEP_AFTER = Integer.parseInt(System.getProperty("additional.sleep.after", "0"));
+
+ public static final String SCENARIO_TIME = "SCENARIO";
+
private int currentLoad;
-
+
private ExecutorService executorService;
+
+ protected PerformanceStatistics statistics = new PerformanceStatistics();
+ protected PerformanceStatistics timeoutStatistics = new PerformanceStatistics(); // for keeping track of # of conn. timeout exceptions
- protected PerformanceTestMetrics metrics = new PerformanceTestMetrics();
- protected PerformanceTestMetrics timeouts = new PerformanceTestMetrics();
-
- protected List<ResultsWithThroughput> resultsList = new ArrayList<>();
- protected List<ResultsWithThroughput> timeoutResultsList = new ArrayList<>();
-
+ protected List<PerformanceMeasurement> measurements = new ArrayList<>();
+
@Before
public void before() {
if (WARMUP_LOAD > INITIAL_LOAD) {
throw new IllegalArgumentException("'warmup.load' cannot be larger than 'initial.load'");
}
-
+
executorService = Executors.newFixedThreadPool(MAX_THREADS);
currentLoad = 0;
-
- metrics.clear();
+
+ statistics.clear();
+ timeoutStatistics.clear();
}
-
+
@After
public void after() throws IOException, InterruptedException {
executorService.shutdown();
+
LOG.info("Waiting for threadpool termination.");
- executorService.awaitTermination(10, TimeUnit.SECONDS);
+ executorService.awaitTermination(THREADPOOL_TERMINATION_TIMEOUT, TimeUnit.SECONDS);
+ pause(ADDITIONAL_SLEEP_AFTER * 1000);
+
+ LOG.info("Logging out all sessions.");
+ testRealmResource().logoutAll();
}
-
+
@Test
public void test() {
-
+
increaseLoadBy(WARMUP_LOAD); // increase to warmup load
warmup();
-
+
for (int i = 0; i < MAX_ITERATIONS; i++) {
-
+
int loadIncrease = (i == 0)
? INITIAL_LOAD - WARMUP_LOAD // increase from warmup to initial load
- : LOAD_INCREASE; // increase load between iterations
+ : LOAD_INCREASE; // increase load between measurements
increaseLoadBy(loadIncrease);
measurePerformance();
-
+
if (!isThereEnoughThreadsForNextIteration(LOAD_INCREASE)) {
LOG.warn("Threadpool capacity reached. Stopping the test.");
break;
}
- if (!isLatestResultsWithinLimits()) {
- LOG.warn("The latest measurement surpassed expected limit. Stopping the test.");
+ if (!isLatestMeasurementWithinLimits()) {
+ LOG.warn("The latest measurement exceeded expected limit. Stopping the test.");
break;
}
}
-
+
}
-
+
private void warmup() {
LOG.info("Warming up for " + WARMUP_DURATION + " s");
pauseWithErrorChecking(WARMUP_DURATION * 1000);
}
-
+
private boolean isThereEnoughThreadsForNextIteration(int loadIncrease) {
return currentLoad + loadIncrease <= MAX_THREADS;
}
-
+
private void increaseLoadBy(int loadIncrease) {
if (loadIncrease < 0) {
throw new IllegalArgumentException("Cannot increase load by a negative number (" + loadIncrease + ").");
@@ -113,7 +118,7 @@ public abstract class PerformanceTest extends AbstractExampleAdapterTest {
throw new IllegalArgumentException("Cannot increase load beyond threadpool capacity (" + MAX_THREADS + ").");
}
if (loadIncrease > 0) {
- LOG.info(String.format("Increasing load from %s to %s.", currentLoad, currentLoad + loadIncrease));
+ LOG.info(String.format("Increasing load from %s to %s at +%s clients/s.", currentLoad, currentLoad + loadIncrease, LOAD_INCREASE_RATE));
for (int t = 0; t < loadIncrease; t++) {
executorService.submit(newRunnable());
currentLoad++;
@@ -121,95 +126,98 @@ public abstract class PerformanceTest extends AbstractExampleAdapterTest {
}
}
}
-
+
private void measurePerformance() {
- LOG.info("Measuring performance");
- LOG.info("Iteration: " + (resultsList.size() + 1));
- LOG.info("Duration: " + MEASUREMENT_DURATION + " s");
- LOG.info("Load: " + currentLoad);
-
- metrics.reset();
+ PerformanceMeasurement measurement = new PerformanceMeasurement(currentLoad);
+ statistics.reset();
+ timeoutStatistics.reset();
+
+ LOG.info(String.format("Measuring performance. Iteration: %s, Load: %s, Duration: %s s", measurements.size() + 1, currentLoad, MEASUREMENT_DURATION));
+
pauseWithErrorChecking(MEASUREMENT_DURATION * 1000);
- resultsList.add(metrics.computeMetrics());
- timeoutResultsList.add(timeouts.computeMetrics());
-
- getLatestResults().logResults(); // to file
- LOG.info("Timeouts: " + getLatestTimeoutResults());
- }
-
- protected ResultsWithThroughput getLatestResults() {
- return resultsList.isEmpty() ? null : resultsList.get(resultsList.size() - 1);
+
+ measurement.setStatistics(
+ statistics.snapshot(),
+ timeoutStatistics.snapshot());
+ measurements.add(measurement);
+
+ measurement.printToCSV(getTestName());
+ measurement.printToLog();
}
-
- protected Results getLatestTimeoutResults() {
- return timeoutResultsList.isEmpty() ? null : timeoutResultsList.get(timeoutResultsList.size() - 1);
- }
-
+
private Throwable error = null;
-
+
public synchronized Throwable getError() {
return error;
}
-
+
public synchronized void setError(Throwable error) {
this.error = error;
}
-
+
protected void pauseWithErrorChecking(long millis) {
pauseWithErrorChecking(millis, 1000);
}
-
- protected void pauseWithErrorChecking(long millis, long checkIntervals) {
- long count = millis / checkIntervals;
- long remainder = millis % checkIntervals;
- for (int i = 0; i < count + 1; i++) { // repeat 'count' times + once for remainder
- if (i < count || remainder > 0) { // on last iteration check if any remainder
- pause(checkIntervals);
+
+ protected void pauseWithErrorChecking(long millis, long checkDurationMillis) {
+ long checkDurationMillisMin = Math.min(millis, checkDurationMillis);
+ long checkCount = millis / checkDurationMillis;
+ long remainder = millis % checkDurationMillis;
+ LOG.debug(String.format("Pause %s ms, checking errors once per %s ms", millis, checkDurationMillisMin));
+ for (int i = 0; i < checkCount + 1; i++) { // loop 'count' times + once for remainder
+ if (i < checkCount || remainder > 0) { // on last iteration check if any remainder
+ pause(checkDurationMillisMin);
if (getError() != null) {
throw new RuntimeException("PerformanceTestRunnable threw an exception. Stopping the test.", getError());
}
}
}
}
-
- protected abstract boolean isLatestResultsWithinLimits();
-
- protected boolean isLatestTimeoutsWithinLimits() {
- boolean socketTimeoutsWithinLimits = true;
- for (String metric : getLatestResults().keySet()) {
- long timemoutCount = getLatestTimeoutResults().containsKey(metric) ? getLatestTimeoutResults().get(metric).getCount() : 0;
- double timeoutPercentage = (double) timemoutCount / getLatestResults().get(metric).getCount();
- socketTimeoutsWithinLimits = socketTimeoutsWithinLimits && timeoutPercentage < AVERAGE_TIMEOUT_PERCENTAGE_LIMIT;
- }
- return socketTimeoutsWithinLimits;
+
+ protected PerformanceMeasurement getLatestMeasurement() {
+ return measurements.isEmpty() ? null : measurements.get(measurements.size() - 1);
}
-
+
+ protected boolean isLatestMeasurementWithinLimits() {
+ return isMeasurementWithinLimits(getLatestMeasurement());
+ }
+
+ protected abstract boolean isMeasurementWithinLimits(PerformanceMeasurement measurement);
+
protected abstract Runnable newRunnable();
-
- public abstract class Runnable extends RepeatRunnable {
-
- protected final Timer timer;
+
+ public abstract class Runnable extends LoopingRunnable {
+
+ protected final Timer timer; // for timing individual operations/requests
+ private final Timer scenarioTimer; // for timing the whole scenario
public Runnable() {
- super(SLEEP_BETWEEN_REPEATS * 1000);
+ super(SLEEP_BETWEEN_LOOPS * 1000);
this.timer = new Timer();
+ this.scenarioTimer = new Timer();
}
-
+
@Override
- public void repeat() {
+ public void loop() {
try {
+ scenarioTimer.reset();
performanceScenario();
+ statistics.addValue(SCENARIO_TIME, scenarioTimer.getElapsedTime());
} catch (OperationTimeoutException ex) {
- timeouts.addValue(ex.getMetric(), ex.getValue());
- LOG.debug(String.format("Operatin %s timed out. Cause: %s.", ex.getMetric(), ex.getCause()));
+ timeoutStatistics.addValue(ex.getStatistic(), ex.getValue());
+ LOG.debug(String.format("Operation %s timed out. Cause: %s.", ex.getStatistic(), ex.getCause()));
} catch (AssertionError | Exception ex) {
setError(ex);
throw new RuntimeException(ex);
}
}
-
+
public abstract void performanceScenario() throws Exception;
-
+
}
-
+
+ public String getTestName() {
+ return this.getClass().getSimpleName();
+ }
+
}
diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/SimpleStatistics.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/SimpleStatistics.java
new file mode 100644
index 0000000..185e0c6
--- /dev/null
+++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/SimpleStatistics.java
@@ -0,0 +1,17 @@
+package org.keycloak.testsuite.performance.statistics;
+
+import java.util.TreeMap;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public class SimpleStatistics extends TreeMap<String, SimpleStatistic> implements Statistics<SimpleStatistic> {
+
+ public SimpleStatistics(Statistics statistics) {
+ for (Object statistic : statistics.keySet()) {
+ put(statistic.toString(), new SimpleStatistic((Statistic) statistics.get(statistic)));
+ }
+ }
+
+}
diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/resources/log4j.properties b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/resources/log4j.properties
index 7c833ee..b931f73 100644
--- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/resources/log4j.properties
+++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/resources/log4j.properties
@@ -8,6 +8,7 @@ log4j.appender.DEFAULT.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p [%c] [%t]
log4j.logger.org.keycloak=OFF
log4j.logger.org.keycloak.testsuite=INFO
+log4j.logger.org.keycloak.testsuite.performance.PerformanceTest=INFO
# HtmlUnit
log4j.logger.org.keycloak.testsuite.performance.htmlunit.HtmlUnitLoginLogoutPerfTest=${logging.loginlogout}
@@ -17,14 +18,3 @@ log4j.logger.com.gargoylesoftware.htmlunit=OFF
log4j.logger.org.keycloak.testsuite.performance.httpclient.HttpClientLoginLogoutPerfTest=${logging.loginlogout}
log4j.logger.org.keycloak.testsuite.performance.httpclient.HttpClientPerformanceTest$CustomRedirectStrategy=OFF
-
-# RESULTS
-
-log4j.appender.RESULTS=org.apache.log4j.FileAppender
-log4j.appender.RESULTS.file=target/results.log
-log4j.appender.RESULTS.immediateFlush=true
-log4j.appender.RESULTS.append=true
-log4j.appender.RESULTS.layout=org.apache.log4j.PatternLayout
-log4j.appender.RESULTS.layout.ConversionPattern=%d{HHmmss} %m%n
-
-log4j.logger.org.keycloak.testsuite.performance.metrics.impl.ResultsWithThroughput=INFO, RESULTS
diff --git a/testsuite/integration-arquillian/tests/other/clean-start/pom.xml b/testsuite/integration-arquillian/tests/other/clean-start/pom.xml
index 16268ad..939ea39 100644
--- a/testsuite/integration-arquillian/tests/other/clean-start/pom.xml
+++ b/testsuite/integration-arquillian/tests/other/clean-start/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.keycloak.testsuite</groupId>
<artifactId>integration-arquillian-tests-other</artifactId>
- <version>1.9.5.Final-SNAPSHOT</version>
+ <version>1.9.6.Final-SNAPSHOT</version>
</parent>
<artifactId>integration-arquillian-tests-smoke-clean-start</artifactId>