keycloak-aplcache
Changes
testsuite/performance/db-failover/common.sh 18(+18 -0)
testsuite/performance/db-failover/loop.sh 35(+35 -0)
testsuite/performance/db-failover/README.md 67(+67 -0)
testsuite/performance/db-failover/setup.sh 38(+38 -0)
testsuite/performance/docker-compose-db-failover.yml 101(+101 -0)
testsuite/performance/keycloak/pom.xml 24(+21 -3)
Details
diff --git a/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml b/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml
index 26a3323..a0d2bf0 100644
--- a/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml
+++ b/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml
@@ -855,5 +855,23 @@
</modules>
</profile>
+
+ <profile>
+ <id>db-failover-mariadb</id>
+ <properties>
+ <jdbc.mvn.groupId>org.mariadb.jdbc</jdbc.mvn.groupId>
+ <jdbc.mvn.artifactId>mariadb-java-client</jdbc.mvn.artifactId>
+ <jdbc.mvn.version>2.0.3</jdbc.mvn.version>
+ <keycloak.connectionsJpa.user>keycloak</keycloak.connectionsJpa.user>
+ <keycloak.connectionsJpa.password>keycloak</keycloak.connectionsJpa.password>
+
+ <mariadb.ha.mode>replication</mariadb.ha.mode>
+ <mariadb.hosts>localhost:3316,localhost:3326</mariadb.hosts>
+ <mariadb.database>keycloak</mariadb.database>
+ <mariadb.options></mariadb.options>
+ <keycloak.connectionsJpa.url>jdbc:mariadb:${mariadb.ha.mode}://${mariadb.hosts}/${mariadb.database}${mariadb.options}</keycloak.connectionsJpa.url>
+ </properties>
+ </profile>
+
</profiles>
</project>
diff --git a/testsuite/performance/db/mariadb/wsrep.cnf b/testsuite/performance/db/mariadb/wsrep.cnf
index 6a461c8..52f0979 100644
--- a/testsuite/performance/db/mariadb/wsrep.cnf
+++ b/testsuite/performance/db/mariadb/wsrep.cnf
@@ -3,7 +3,7 @@ general_log=OFF
bind-address=0.0.0.0
innodb_flush_log_at_trx_commit=0
-query_cache_size=0
+query_cache_size=0
query_cache_type=0
binlog_format=ROW
@@ -15,7 +15,12 @@ innodb_buffer_pool_size=122M
wsrep_on=ON
wsrep_provider=/usr/lib/galera/libgalera_smm.so
+
wsrep_provider_options="gcache.size=300M; gcache.page_size=300M"
+#wsrep_provider_options="gcache.size=300M; gcache.page_size=300M; pc.bootstrap=YES"
+#wsrep_provider_options="gcache.size=300M; gcache.page_size=300M; pc.bootstrap=YES; pc.ignore_sb=TRUE"
+# See: http://galeracluster.com/documentation-webpages/twonode.html
+
wsrep_cluster_address="gcomm://"
wsrep_cluster_name="galera_cluster_keycloak"
wsrep_sst_method=rsync
diff --git a/testsuite/performance/db-failover/check-rows.sh b/testsuite/performance/db-failover/check-rows.sh
new file mode 100755
index 0000000..215d387
--- /dev/null
+++ b/testsuite/performance/db-failover/check-rows.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+. ./common.sh
+
+CONCAT_SQL=$(< db-failover/concat.sql)
+CONCAT_SQL_COMMAND='mysql -N -B -u keycloak --password=keycloak -e "$CONCAT_SQL" keycloak'
+ROWS_SQL=$(eval docker-compose -f docker-compose-db-failover.yml exec mariadb_1 $CONCAT_SQL_COMMAND | tr -dc '[:print:]')
+ROWS_SQL=${ROWS_SQL%UNION }
+ROWS_SQL_COMMAND='mysql -u keycloak --password=keycloak -e "$ROWS_SQL" keycloak'
+
+for (( i=1; i <= $NODES; i++)); do
+ ROWS[i]=$(eval docker-compose -f docker-compose-db-failover.yml exec mariadb_$i $ROWS_SQL_COMMAND)
+done
+
+DIFF=0
+for (( i=2; i <= $NODES; i++)); do
+ echo Node 1 vs Node $(( i )):
+ diff -y --suppress-common-lines <(echo "${ROWS[1]}") <(echo "${ROWS[i]}")
+ if [ $? -eq 0 ]; then echo No difference.; else DIFF=1; fi
+done
+
+exit $DIFF
testsuite/performance/db-failover/common.sh 18(+18 -0)
diff --git a/testsuite/performance/db-failover/common.sh b/testsuite/performance/db-failover/common.sh
new file mode 100755
index 0000000..abdb8f9
--- /dev/null
+++ b/testsuite/performance/db-failover/common.sh
@@ -0,0 +1,18 @@
+function killNode {
+ echo Killing mariadb_${1}
+ docker-compose -f docker-compose-db-failover.yml kill mariadb_${1}
+}
+
+function reconnectNode {
+ N=$1
+ NR=$(( N + 1 )); if [ "$NR" -gt "$NODES" ]; then NR=1; fi
+ export MARIADB_RUNNING_HOST=mariadb_${NR}
+ echo Attempting failback of mariadb_${N}, connecting to running cluster member mariadb_${NR}
+ docker-compose -f docker-compose-db-failover.yml up -d mariadb_${N}
+}
+
+if [ -z $NODES ]; then export NODES=2; fi
+if [ -z $MARIADB_OPTIONS ]; then export MARIADB_OPTIONS=""; fi
+if [ -z $START_KEYCLOAK ]; then export START_KEYCLOAK=false; fi
+
+cd ..
diff --git a/testsuite/performance/db-failover/concat.sql b/testsuite/performance/db-failover/concat.sql
new file mode 100644
index 0000000..f329ae8
--- /dev/null
+++ b/testsuite/performance/db-failover/concat.sql
@@ -0,0 +1,3 @@
+SELECT CONCAT(
+ 'SELECT "', table_name, '" AS table_name, COUNT(*) AS exact_row_count FROM `', table_schema, '`.`', table_name, '` UNION '
+) FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = 'keycloak';
diff --git a/testsuite/performance/db-failover/kill-node.sh b/testsuite/performance/db-failover/kill-node.sh
new file mode 100755
index 0000000..b47e39f
--- /dev/null
+++ b/testsuite/performance/db-failover/kill-node.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. ./common.sh
+
+if [ -z $1 ]; then echo "Specify DB node to kill."; exit 1; fi
+
+killNode $1
\ No newline at end of file
testsuite/performance/db-failover/loop.sh 35(+35 -0)
diff --git a/testsuite/performance/db-failover/loop.sh b/testsuite/performance/db-failover/loop.sh
new file mode 100755
index 0000000..79813b2
--- /dev/null
+++ b/testsuite/performance/db-failover/loop.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+. ./common.sh
+
+if [ -z "$TIME_BETWEEN_FAILURES" ]; then export TIME_BETWEEN_FAILURES=60; fi
+if [ -z "$FAILURE_DURATION" ]; then export FAILURE_DURATION=60; fi
+
+echo Running DB failover loop with the following parameters:
+echo NODES=$NODES
+echo TIME_BETWEEN_FAILURES=$TIME_BETWEEN_FAILURES
+echo FAILURE_DURATION=$FAILURE_DURATION
+echo
+echo Press Ctrl+C to interrupt.
+echo
+
+N=1
+
+while :
+do
+
+ killNode $N
+
+ echo Waiting $FAILURE_DURATION s before attempting to reconnect mariadb_${N}
+ sleep $FAILURE_DURATION
+
+ reconnectNode $N
+
+ echo Waiting $TIME_BETWEEN_FAILURES s before inducing another failure.
+ echo
+ sleep $TIME_BETWEEN_FAILURES
+
+ N=$((N+1))
+ if [ "$N" -gt "$NODES" ]; then N=1; fi
+
+done
\ No newline at end of file
testsuite/performance/db-failover/README.md 67(+67 -0)
diff --git a/testsuite/performance/db-failover/README.md b/testsuite/performance/db-failover/README.md
new file mode 100644
index 0000000..c378484
--- /dev/null
+++ b/testsuite/performance/db-failover/README.md
@@ -0,0 +1,67 @@
+# DB Failover Testing Utilities
+
+A set of scripts for testing DB failover scenarios.
+
+The scripts expect to be run with relative execution prefix `./` from within the `db-failover` directory.
+
+Provisioned services are defined in `../docker-compose-db-failover.yml` template.
+
+
+## Set the size of DB cluster
+
+Default size is 2 nodes. For a 3-node cluster run: `export NODES=3` before executing any of the scripts.
+
+For more than 3 nodes more service definitions need to be added to the docker-compose template.
+
+## Set up the environment
+
+Run `./setup.sh`
+
+This script will:
+1. Start a bootstrap DB instance of MariaDB cluster
+2. Start additional DB instances connected to the bootstrapped cluster
+3. Stop the bootstrap DB instance
+4. Optionally start Keycloak server
+
+Parameterized by environment variables:
+- `MARIADB_HA_MODE` See: [MariaDB HA parameters](https://mariadb.com/kb/en/library/failover-and-high-availability-with-mariadb-connector-j/#failover-high-availability-parameters)
+ Defaults to `replication`.
+- `MARIADB_OPTIONS` See: [MariaDB HA options](https://mariadb.com/kb/en/library/failover-and-high-availability-with-mariadb-connector-j/#failover-high-availability-options).
+ Use format: `?option1=value1[&option2=value2]...`. Default is an empty string.
+- `START_KEYCLOAK` Default is `false`. Use `export START_KEYCLOAK=true` to enable.
+
+More options relevant to MariaDB clustering can be found in `../db/mariadb/wsrep.cnf`.
+
+
+## Test the failover
+
+### Manual failover
+
+To induce a failure of specific DB node run: `./kill-node.sh X` where `X ∈ {1..3}`
+
+To reconnect the node back run: `./reconnect-node.sh X`
+
+
+### Automated failover loop
+
+Run `./loop.sh`
+
+This script will run an infinite loop of failover/failback of DB nodes, switching to the next node in each loop.
+
+Parameterized by environment variables:
+- `TIME_BETWEEN_FAILURES` Default is `60` (seconds).
+- `FAILURE_DURATION` Default is `60` (seconds).
+
+To exit the script press `Ctrl+C`.
+
+
+### Check number of table rows across the cluster
+
+Run: `./check-rows.sh`
+
+
+## Tear down the environment
+
+Run `./teardown.sh`
+
+This will stop all services and delete the database.
diff --git a/testsuite/performance/db-failover/reconnect-node.sh b/testsuite/performance/db-failover/reconnect-node.sh
new file mode 100755
index 0000000..6788a79
--- /dev/null
+++ b/testsuite/performance/db-failover/reconnect-node.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. ./common.sh
+
+if [ -z $1 ]; then echo "Specify DB node to reconnect to cluster."; exit 1; fi
+
+reconnectNode $1
\ No newline at end of file
testsuite/performance/db-failover/setup.sh 38(+38 -0)
diff --git a/testsuite/performance/db-failover/setup.sh b/testsuite/performance/db-failover/setup.sh
new file mode 100755
index 0000000..6f407ae
--- /dev/null
+++ b/testsuite/performance/db-failover/setup.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+. ./common.sh
+
+if [ -z "$DB_BOOTSTRAP_TIMEOUT" ]; then DB_BOOTSTRAP_TIMEOUT=10; fi
+if [ -z "$DB_JOIN_TIMEOUT" ]; then DB_JOIN_TIMEOUT=5; fi
+
+echo Setting up Keycloak DB failover environment:
+
+echo Starting DB bootstrap instance.
+docker-compose -f docker-compose-db-failover.yml up -d --build mariadb_bootstrap
+echo Waiting $DB_BOOTSTRAP_TIMEOUT s for the DB to initialize.
+sleep $DB_BOOTSTRAP_TIMEOUT
+
+MARIADB_HOSTS=""
+for (( i=1; i<=$NODES; i++ )); do
+
+ MARIADB_HOSTS=$MARIADB_HOSTS,mariadb_$i:3306
+
+ echo Starting DB node $i.
+ docker-compose -f docker-compose-db-failover.yml up -d mariadb_$i
+ echo Waiting $DB_JOIN_TIMEOUT s for the DB node to join
+ echo
+ sleep $DB_JOIN_TIMEOUT
+
+done
+
+echo Turning off the DB bootstrap instance.
+docker-compose -f docker-compose-db-failover.yml stop mariadb_bootstrap
+
+export MARIADB_HOSTS=${MARIADB_HOSTS/,/}
+echo MARIADB_HOSTS=$MARIADB_HOSTS
+
+if $START_KEYCLOAK; then
+ echo Starting Keycloak server.
+ docker-compose -f docker-compose-db-failover.yml up -d --build keycloak
+ ./healthcheck.sh
+fi
diff --git a/testsuite/performance/db-failover/teardown.sh b/testsuite/performance/db-failover/teardown.sh
new file mode 100755
index 0000000..ac47298
--- /dev/null
+++ b/testsuite/performance/db-failover/teardown.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+. ./common.sh
+
+echo Stopping Keycloak DB failover environment.
+
+docker-compose -f docker-compose-db-failover.yml down -v
+
diff --git a/testsuite/performance/docker-compose.yml b/testsuite/performance/docker-compose.yml
index 503f0d4..93cc97f 100644
--- a/testsuite/performance/docker-compose.yml
+++ b/testsuite/performance/docker-compose.yml
@@ -35,7 +35,7 @@ services:
networks:
- keycloak
environment:
- MARIADB_HOST: mariadb
+ MARIADB_HOSTS: mariadb:3306
MARIADB_DATABASE: keycloak
MARIADB_USER: keycloak
MARIADB_PASSWORD: keycloak
diff --git a/testsuite/performance/docker-compose-cluster.yml b/testsuite/performance/docker-compose-cluster.yml
index 4ecd95a..4909878 100644
--- a/testsuite/performance/docker-compose-cluster.yml
+++ b/testsuite/performance/docker-compose-cluster.yml
@@ -41,7 +41,7 @@ services:
CONFIGURATION: standalone-ha.xml
PUBLIC_SUBNET: 10.0.1.0/24
PRIVATE_SUBNET: 10.0.1.0/24
- MARIADB_HOST: mariadb
+ MARIADB_HOSTS: mariadb:3306
MARIADB_DATABASE: keycloak
MARIADB_USER: keycloak
MARIADB_PASSWORD: keycloak
diff --git a/testsuite/performance/docker-compose-crossdc.yml b/testsuite/performance/docker-compose-crossdc.yml
index 5075ad9..4115efe 100644
--- a/testsuite/performance/docker-compose-crossdc.yml
+++ b/testsuite/performance/docker-compose-crossdc.yml
@@ -131,7 +131,7 @@ services:
CONFIGURATION: standalone-ha.xml
PUBLIC_SUBNET: 10.1.1.0/24
PRIVATE_SUBNET: 10.1.1.0/24
- MARIADB_HOST: mariadb_dc1
+ MARIADB_HOSTS: mariadb_dc1:3306
MARIADB_DATABASE: keycloak
MARIADB_USER: keycloak
MARIADB_PASSWORD: keycloak
@@ -172,7 +172,7 @@ services:
CONFIGURATION: standalone-ha.xml
PUBLIC_SUBNET: 10.2.1.0/24
PRIVATE_SUBNET: 10.2.1.0/24
- MARIADB_HOST: mariadb_dc2
+ MARIADB_HOSTS: mariadb_dc2:3306
MARIADB_DATABASE: keycloak
MARIADB_USER: keycloak
MARIADB_PASSWORD: keycloak
testsuite/performance/docker-compose-db-failover.yml 101(+101 -0)
diff --git a/testsuite/performance/docker-compose-db-failover.yml b/testsuite/performance/docker-compose-db-failover.yml
new file mode 100644
index 0000000..23b0b71
--- /dev/null
+++ b/testsuite/performance/docker-compose-db-failover.yml
@@ -0,0 +1,101 @@
+version: "2.2"
+
+networks:
+ keycloak:
+ ipam:
+ config:
+ - subnet: 10.0.1.0/24
+
+ db_replication:
+ ipam:
+ config:
+ - subnet: 10.0.3.0/24
+
+services:
+
+ mariadb_bootstrap:
+ build: db/mariadb
+ image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest}
+ networks:
+ - db_replication
+ environment:
+ MYSQL_ROOT_PASSWORD: root
+ MYSQL_INITDB_SKIP_TZINFO: foo
+ MYSQL_DATABASE: keycloak
+ MYSQL_USER: keycloak
+ MYSQL_PASSWORD: keycloak
+ entrypoint: docker-entrypoint-wsrep.sh
+ command: --wsrep-new-cluster
+
+ mariadb_1:
+ build: db/mariadb
+ image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest}
+ networks:
+ - db_replication
+ - keycloak
+ environment:
+ MYSQL_ROOT_PASSWORD: root
+ MYSQL_INITDB_SKIP_TZINFO: foo
+ entrypoint: docker-entrypoint-wsrep.sh
+ command: --wsrep_cluster_address=gcomm://${MARIADB_RUNNING_HOST:-mariadb_bootstrap}
+ ports:
+ - "3316:3306"
+
+ mariadb_2:
+ build: db/mariadb
+ image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest}
+ networks:
+ - db_replication
+ - keycloak
+ environment:
+ MYSQL_ROOT_PASSWORD: root
+ MYSQL_INITDB_SKIP_TZINFO: foo
+ entrypoint: docker-entrypoint-wsrep.sh
+ command: --wsrep_cluster_address=gcomm://${MARIADB_RUNNING_HOST:-mariadb_bootstrap}
+ ports:
+ - "3326:3306"
+
+ mariadb_3:
+ build: db/mariadb
+ image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest}
+ networks:
+ - db_replication
+ - keycloak
+ environment:
+ MYSQL_ROOT_PASSWORD: root
+ MYSQL_INITDB_SKIP_TZINFO: foo
+ entrypoint: docker-entrypoint-wsrep.sh
+ command: --wsrep_cluster_address=gcomm://${MARIADB_RUNNING_HOST:-mariadb_bootstrap}
+ ports:
+ - "3336:3306"
+
+
+ keycloak:
+ build: keycloak
+ image: keycloak_test_keycloak:${KEYCLOAK_VERSION:-latest}
+ networks:
+ - keycloak
+ environment:
+ MARIADB_HA_MODE: ${MARIADB_HA_MODE:-replication}
+ MARIADB_HOSTS: ${MARIADB_HOSTS:-mariadb_1:3306,mariadb_2:3306}
+ MARIADB_OPTIONS: ${MARIADB_OPTIONS}
+ MARIADB_DATABASE: keycloak
+ MARIADB_USER: keycloak
+ MARIADB_PASSWORD: keycloak
+ KEYCLOAK_USER: admin
+ KEYCLOAK_PASSWORD: admin
+ # docker-compose syntax note: ${ENV_VAR:-<DEFAULT_VALUE>}
+ JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true
+ HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-500}
+ WORKER_IO_THREADS: ${KEYCLOAK_WORKER_IO_THREADS:-2}
+ WORKER_TASK_MAX_THREADS: ${KEYCLOAK_WORKER_TASK_MAX_THREADS:-16}
+ DS_MIN_POOL_SIZE: ${KEYCLOAK_DS_MIN_POOL_SIZE:-10}
+ DS_MAX_POOL_SIZE: ${KEYCLOAK_DS_MAX_POOL_SIZE:-100}
+ DS_POOL_PREFILL: "${KEYCLOAK_DS_POOL_PREFILL:-true}"
+ DS_PS_CACHE_SIZE: ${KEYCLOAK_DS_PS_CACHE_SIZE:-100}
+ ports:
+ - "8080:8080"
+ - "8443:8443"
+ - "7989:7989"
+ - "9990:9990"
+ - "9999:9999"
diff --git a/testsuite/performance/keycloak/configs/set-keycloak-ds-batch.cli b/testsuite/performance/keycloak/configs/set-keycloak-ds-batch.cli
index cffb6d8..35df67e 100644
--- a/testsuite/performance/keycloak/configs/set-keycloak-ds-batch.cli
+++ b/testsuite/performance/keycloak/configs/set-keycloak-ds-batch.cli
@@ -2,7 +2,7 @@
cd /subsystem=datasources/data-source=KeycloakDS
-:write-attribute(name=connection-url, value=jdbc:mariadb://${env.MARIADB_HOST:mariadb}:${env.MARIADB_PORT:3306}/${env.MARIADB_DATABASE:keycloak})
+:write-attribute(name=connection-url, value=jdbc:mariadb:${env.MARIADB_HA_MODE:}://${env.MARIADB_HOSTS:mariadb:3306}/${env.MARIADB_DATABASE:keycloak}${env.MARIADB_OPTIONS:})
:write-attribute(name=driver-name, value=mariadb)
:write-attribute(name=user-name, value=${env.MARIADB_USER:keycloak})
:write-attribute(name=password, value=${env.MARIADB_PASSWORD:keycloak})
testsuite/performance/keycloak/pom.xml 24(+21 -3)
diff --git a/testsuite/performance/keycloak/pom.xml b/testsuite/performance/keycloak/pom.xml
index 9c46b17..fb84672 100644
--- a/testsuite/performance/keycloak/pom.xml
+++ b/testsuite/performance/keycloak/pom.xml
@@ -34,6 +34,12 @@
Uses maven-dependency-plugin to unpack keycloak-server-dist artifact into `target/keycloak` which is then added to Docker image.
</description>
+ <properties>
+ <server.groupId>org.keycloak</server.groupId>
+ <server.artifactId>keycloak-server-dist</server.artifactId>
+ <server.unpacked.folder>keycloak-${product.version}</server.unpacked.folder>
+ </properties>
+
<build>
<plugins>
@@ -51,8 +57,8 @@
<configuration>
<artifactItems>
<artifactItem>
- <groupId>org.keycloak</groupId>
- <artifactId>keycloak-server-dist</artifactId>
+ <groupId>${server.groupId}</groupId>
+ <artifactId>${server.artifactId}</artifactId>
<version>${project.version}</version>
<type>zip</type>
<outputDirectory>${project.build.directory}</outputDirectory>
@@ -77,7 +83,7 @@
<configuration>
<target>
<move todir="${project.build.directory}/keycloak" failonerror="false">
- <fileset dir="${project.build.directory}/keycloak-${project.version}"/>
+ <fileset dir="${project.build.directory}/${server.unpacked.folder}"/>
</move>
</target>
</configuration>
@@ -87,4 +93,16 @@
</plugins>
</build>
+ <profiles>
+
+ <profile>
+ <id>integration-testsuite-server</id>
+ <properties>
+ <server.groupId>org.keycloak.testsuite</server.groupId>
+ <server.artifactId>integration-arquillian-servers-auth-server-wildfly</server.artifactId>
+ <server.unpacked.folder>auth-server-wildfly</server.unpacked.folder>
+ </properties>
+ </profile>
+ </profiles>
+
</project>
\ No newline at end of file