keycloak-uncached

diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProvider.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProvider.java
index c41b3e2..eb49db4 100755
--- a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProvider.java
+++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProvider.java
@@ -47,6 +47,7 @@ import org.keycloak.models.sessions.infinispan.events.RealmRemovedSessionEvent;
 import org.keycloak.models.sessions.infinispan.events.RemoveAllUserLoginFailuresEvent;
 import org.keycloak.models.sessions.infinispan.events.RemoveUserSessionsEvent;
 import org.keycloak.models.sessions.infinispan.events.SessionEventsSenderTransaction;
+import org.keycloak.models.sessions.infinispan.stream.AuthenticatedClientSessionPredicate;
 import org.keycloak.models.sessions.infinispan.stream.Comparators;
 import org.keycloak.models.sessions.infinispan.stream.Mappers;
 import org.keycloak.models.sessions.infinispan.stream.SessionPredicate;
@@ -160,6 +161,7 @@ public class InfinispanUserSessionProvider implements UserSessionProvider {
     public AuthenticatedClientSessionModel createClientSession(RealmModel realm, ClientModel client, UserSessionModel userSession) {
         AuthenticatedClientSessionEntity entity = new AuthenticatedClientSessionEntity();
         entity.setRealmId(realm.getId());
+        entity.setTimestamp(Time.currentTime());
         final UUID clientSessionId = entity.getId();
 
         InfinispanChangelogBasedTransaction<String, UserSessionEntity> userSessionUpdateTx = getTransaction(false);
@@ -468,6 +470,26 @@ public class InfinispanUserSessionProvider implements UserSessionProvider {
 
                 });
 
+        // Removing detached clientSessions. Ignore remoteStore for stream iteration. But we will invoke remoteStore for clientSession removal propagate
+        Cache<UUID, SessionEntityWrapper<AuthenticatedClientSessionEntity>> localCacheStoreIgnoreClientSessionCache = CacheDecorators.localCache(localClientSessionCache);
+
+        localCacheStoreIgnoreClientSessionCache
+                .entrySet()
+                .stream()
+                .filter(AuthenticatedClientSessionPredicate.create(realm.getId()).expired(expired))
+                .map(Mappers.clientSessionEntity())
+                .forEach(new Consumer<AuthenticatedClientSessionEntity>() {
+
+                    @Override
+                    public void accept(AuthenticatedClientSessionEntity clientSessionEntity) {
+                        clientSessionsSize.incrementAndGet();
+
+                        Future future = localClientSessionCache.removeAsync(clientSessionEntity.getId());
+                        futures.addTask(future);
+                    }
+
+                });
+
         futures.waitForAllToFinish();
 
         log.debugf("Removed %d expired user sessions and %d expired client sessions for realm '%s'", userSessionsSize.get(),
@@ -513,12 +535,28 @@ public class InfinispanUserSessionProvider implements UserSessionProvider {
 
                         // TODO:mposolda can be likely optimized to delete all expired at one step
                         persister.removeUserSession( userSessionEntity.getId(), true);
+                    }
+                });
+
 
-                        // TODO can be likely optimized to delete all at one step
-                        for (String clientUUID : userSessionEntity.getAuthenticatedClientSessions().keySet()) {
-                            persister.removeClientSession(userSessionEntity.getId(), clientUUID, true);
-                        }
+        // Removing detached clientSessions. Ignore remoteStore for stream iteration. But we will invoke remoteStore for clientSession removal propagate
+        Cache<UUID, SessionEntityWrapper<AuthenticatedClientSessionEntity>> localCacheStoreIgnoreClientSessionCache = CacheDecorators.localCache(localClientSessionCache);
+
+        localCacheStoreIgnoreClientSessionCache
+                .entrySet()
+                .stream()
+                .filter(AuthenticatedClientSessionPredicate.create(realm.getId()).expired(expiredOffline))
+                .map(Mappers.clientSessionEntity())
+                .forEach(new Consumer<AuthenticatedClientSessionEntity>() {
+
+                    @Override
+                    public void accept(AuthenticatedClientSessionEntity clientSessionEntity) {
+                        clientSessionsSize.incrementAndGet();
+
+                        Future future = localClientSessionCache.removeAsync(clientSessionEntity.getId());
+                        futures.addTask(future);
                     }
+
                 });
 
         futures.waitForAllToFinish();

diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/stream/AuthenticatedClientSessionPredicate.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/stream/AuthenticatedClientSessionPredicate.java
new file mode 100644
index 0000000..f3f1801
--- /dev/null
+++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/stream/AuthenticatedClientSessionPredicate.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.sessions.infinispan.stream;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Predicate;
+
+import org.infinispan.commons.marshall.Externalizer;
+import org.infinispan.commons.marshall.MarshallUtil;
+import org.infinispan.commons.marshall.SerializeWith;
+import org.keycloak.models.sessions.infinispan.changes.SessionEntityWrapper;
+import org.keycloak.models.sessions.infinispan.entities.AuthenticatedClientSessionEntity;
+import org.keycloak.models.sessions.infinispan.util.KeycloakMarshallUtil;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+@SerializeWith(AuthenticatedClientSessionPredicate.ExternalizerImpl.class)
+public class AuthenticatedClientSessionPredicate implements Predicate<Map.Entry<UUID, SessionEntityWrapper<AuthenticatedClientSessionEntity>>> {
+
+    private final String realm;
+
+    private Integer expired;
+
+    private AuthenticatedClientSessionPredicate(String realm) {
+        this.realm = realm;
+    }
+
+    /**
+     * Creates a client session predicate.
+     * @param realm
+     * @return
+     */
+    public static AuthenticatedClientSessionPredicate create(String realm) {
+        return new AuthenticatedClientSessionPredicate(realm);
+    }
+
+
+    public AuthenticatedClientSessionPredicate expired(Integer expired) {
+        this.expired = expired;
+        return this;
+    }
+
+
+    @Override
+    public boolean test(Map.Entry<UUID, SessionEntityWrapper<AuthenticatedClientSessionEntity>> entry) {
+        AuthenticatedClientSessionEntity entity = entry.getValue().getEntity();
+
+        if (!realm.equals(entity.getRealmId())) {
+            return false;
+        }
+
+        if (expired != null && entity.getTimestamp() > expired) {
+            return false;
+        }
+
+        return true;
+    }
+
+
+    public static class ExternalizerImpl implements Externalizer<AuthenticatedClientSessionPredicate> {
+
+        private static final int VERSION_1 = 1;
+
+        @Override
+        public void writeObject(ObjectOutput output, AuthenticatedClientSessionPredicate obj) throws IOException {
+            output.writeByte(VERSION_1);
+
+            MarshallUtil.marshallString(obj.realm, output);
+            KeycloakMarshallUtil.marshall(obj.expired, output);
+        }
+
+        @Override
+        public AuthenticatedClientSessionPredicate readObject(ObjectInput input) throws IOException, ClassNotFoundException {
+            switch (input.readByte()) {
+                case VERSION_1:
+                    return readObjectVersion1(input);
+                default:
+                    throw new IOException("Unknown version");
+            }
+        }
+
+        public AuthenticatedClientSessionPredicate readObjectVersion1(ObjectInput input) throws IOException, ClassNotFoundException {
+            AuthenticatedClientSessionPredicate res = new AuthenticatedClientSessionPredicate(MarshallUtil.unmarshallString(input));
+            res.expired(KeycloakMarshallUtil.unmarshallInteger(input));
+            return res;
+        }
+    }
+}

diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/stream/Mappers.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/stream/Mappers.java
index 815a54d..177fd23 100644
--- a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/stream/Mappers.java
+++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/stream/Mappers.java
@@ -18,6 +18,7 @@
 package org.keycloak.models.sessions.infinispan.stream;
 
 import org.keycloak.models.sessions.infinispan.changes.SessionEntityWrapper;
+import org.keycloak.models.sessions.infinispan.entities.AuthenticatedClientSessionEntity;
 import org.keycloak.models.sessions.infinispan.entities.LoginFailureEntity;
 import org.keycloak.models.sessions.infinispan.entities.LoginFailureKey;
 import org.keycloak.models.sessions.infinispan.entities.SessionEntity;
@@ -25,6 +26,7 @@ import org.keycloak.models.sessions.infinispan.entities.UserSessionEntity;
 
 import java.io.Serializable;
 import java.util.Map;
+import java.util.UUID;
 import java.util.function.Function;
 
 /**
@@ -48,6 +50,10 @@ public class Mappers {
         return new UserSessionEntityMapper();
     }
 
+    public static Function<Map.Entry<UUID, SessionEntityWrapper<AuthenticatedClientSessionEntity>>, AuthenticatedClientSessionEntity> clientSessionEntity() {
+        return new AuthenticatedClientSessionEntityMapper();
+    }
+
     public static Function<Map.Entry<LoginFailureKey, SessionEntityWrapper<LoginFailureEntity>>, LoginFailureKey> loginFailureId() {
         return new LoginFailureIdMapper();
     }
@@ -103,6 +109,15 @@ public class Mappers {
 
     }
 
+    private static class AuthenticatedClientSessionEntityMapper implements Function<Map.Entry<UUID, SessionEntityWrapper<AuthenticatedClientSessionEntity>>, AuthenticatedClientSessionEntity>, Serializable {
+
+        @Override
+        public AuthenticatedClientSessionEntity apply(Map.Entry<UUID, SessionEntityWrapper<AuthenticatedClientSessionEntity>> entry) {
+            return entry.getValue().getEntity();
+        }
+
+    }
+
     private static class LoginFailureIdMapper implements Function<Map.Entry<LoginFailureKey, SessionEntityWrapper<LoginFailureEntity>>, LoginFailureKey>, Serializable {
         @Override
         public LoginFailureKey apply(Map.Entry<LoginFailureKey, SessionEntityWrapper<LoginFailureEntity>> entry) {

diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/stream/UserSessionPredicate.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/stream/UserSessionPredicate.java
index 0b9a764..850d19e 100644
--- a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/stream/UserSessionPredicate.java
+++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/stream/UserSessionPredicate.java
@@ -73,7 +73,6 @@ public class UserSessionPredicate implements Predicate<Map.Entry<String, Session
      * from user session, only client session is deleted and user session is not updated for performance reason.
      *
      * @see AuthenticatedClientSessionAdapter#detachFromUserSession()
-     * @param clientSessionCache
      * @param clientUUID
      * @return
      */

diff --git a/testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/rest/resource/TestCacheResource.java b/testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/rest/resource/TestCacheResource.java
index d1e7be1..6c37063 100644
--- a/testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/rest/resource/TestCacheResource.java
+++ b/testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/rest/resource/TestCacheResource.java
@@ -25,9 +25,11 @@ import java.util.stream.Collectors;
 
 import javax.ws.rs.Consumes;
 import javax.ws.rs.GET;
+import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
 
 import org.infinispan.Cache;
 import org.infinispan.client.hotrod.RemoteCache;
@@ -96,6 +98,13 @@ public class TestCacheResource {
         cache.clear();
     }
 
+    @POST
+    @Path("/remove-key/{id}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public void removeKey(@PathParam("id") String id) {
+        cache.remove(id);
+    }
+
     @GET
     @Path("/jgroups-stats")
     @Produces(MediaType.APPLICATION_JSON)

diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/client/resources/TestingCacheResource.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/client/resources/TestingCacheResource.java
index c23d241..07c3a23 100644
--- a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/client/resources/TestingCacheResource.java
+++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/client/resources/TestingCacheResource.java
@@ -23,6 +23,7 @@ import org.keycloak.utils.MediaType;
 
 import javax.ws.rs.Consumes;
 import javax.ws.rs.GET;
+import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
@@ -60,6 +61,11 @@ public interface TestingCacheResource {
     @Consumes(MediaType.TEXT_PLAIN_UTF_8)
     void clear();
 
+    @POST
+    @Path("/remove-key/{id}")
+    @Produces(MediaType.APPLICATION_JSON)
+    void removeKey(@PathParam("id") String id);
+
     @GET
     @Path("/jgroups-stats")
     @Produces(MediaType.APPLICATION_JSON)

diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/OAuthClient.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/OAuthClient.java
index 29fb765..29dd244 100644
--- a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/OAuthClient.java
+++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/OAuthClient.java
@@ -1082,6 +1082,10 @@ public class OAuthClient {
         return publicKeys.get(realm);
     }
 
+    public void removeCachedPublicKeys() {
+        publicKeys.clear();
+    }
+
 
     private interface StateParamProvider {
 

diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/SessionExpirationCrossDCTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/SessionExpirationCrossDCTest.java
index ac71037..c32ab7b 100644
--- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/SessionExpirationCrossDCTest.java
+++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/SessionExpirationCrossDCTest.java
@@ -73,6 +73,7 @@ public class SessionExpirationCrossDCTest extends AbstractAdminCrossDCTest {
     @Before
     public void beforeTest() {
         try {
+            oauth.removeCachedPublicKeys();
             adminClient.realm(REALM_NAME).remove();
         } catch (NotFoundException ignore) {
         }
@@ -528,6 +529,68 @@ public class SessionExpirationCrossDCTest extends AbstractAdminCrossDCTest {
     }
 
 
+    // CLIENT SESSIONS
+
+    @Test
+    public void testClearDetachedClientSessions(
+            @JmxInfinispanCacheStatistics(dc=DC.FIRST, managementPortProperty = "cache.server.management.port", cacheName=InfinispanConnectionProvider.USER_SESSION_CACHE_NAME) InfinispanStatistics cacheDc1Statistics,
+            @JmxInfinispanCacheStatistics(dc=DC.SECOND, managementPortProperty = "cache.server.2.management.port", cacheName=InfinispanConnectionProvider.USER_SESSION_CACHE_NAME) InfinispanStatistics cacheDc2Statistics,
+            @JmxInfinispanChannelStatistics() InfinispanStatistics channelStatisticsCrossDc) throws Exception {
+
+        // Don't include remote stats. Size is smaller because of distributed cache
+        List<OAuthClient.AccessTokenResponse> responses = createInitialSessions(InfinispanConnectionProvider.USER_SESSION_CACHE_NAME, InfinispanConnectionProvider.CLIENT_SESSION_CACHE_NAME,
+                false, cacheDc1Statistics, cacheDc2Statistics, true);
+
+        // Directly remove the userSession entity on DC0. Should be propagated to DC1 as well, but clientSessions are not yet cleared (they become detached)
+        for (OAuthClient.AccessTokenResponse response : responses) {
+            String userSessionId = oauth.verifyToken(response.getAccessToken()).getSessionState();
+            getTestingClientForStartedNodeInDc(0).testing().cache(InfinispanConnectionProvider.USER_SESSION_CACHE_NAME).removeKey(userSessionId);
+        }
+
+        // Increase offset to big value like 100 hours
+        setTimeOffset(360000);
+
+        // Trigger removeExpired
+        getTestingClientForStartedNodeInDc(0).testing().removeExpired(REALM_NAME);
+        getTestingClientForStartedNodeInDc(1).testing().removeExpired(REALM_NAME);
+
+        // Ensure clientSessions were removed
+        assertStatisticsExpected("After remove expired", InfinispanConnectionProvider.USER_SESSION_CACHE_NAME, InfinispanConnectionProvider.CLIENT_SESSION_CACHE_NAME,
+                cacheDc1Statistics, cacheDc2Statistics, channelStatisticsCrossDc,
+                sessions01, sessions02, clientSessions01, clientSessions02,
+                remoteSessions01, remoteSessions02, true);
+    }
+
+
+    @Test
+    public void testClearDetachedOfflineClientSessions(
+            @JmxInfinispanCacheStatistics(dc=DC.FIRST, managementPortProperty = "cache.server.management.port", cacheName=InfinispanConnectionProvider.OFFLINE_USER_SESSION_CACHE_NAME) InfinispanStatistics cacheDc1Statistics,
+            @JmxInfinispanCacheStatistics(dc=DC.SECOND, managementPortProperty = "cache.server.2.management.port", cacheName=InfinispanConnectionProvider.OFFLINE_USER_SESSION_CACHE_NAME) InfinispanStatistics cacheDc2Statistics,
+            @JmxInfinispanChannelStatistics() InfinispanStatistics channelStatisticsCrossDc) throws Exception {
+
+        // Don't include remote stats. Size is smaller because of distributed cache
+        List<OAuthClient.AccessTokenResponse> responses = createInitialSessions(InfinispanConnectionProvider.OFFLINE_USER_SESSION_CACHE_NAME, InfinispanConnectionProvider.OFFLINE_CLIENT_SESSION_CACHE_NAME,
+                true, cacheDc1Statistics, cacheDc2Statistics, true);
+
+        // Directly remove the userSession entity on DC0. Should be propagated to DC1 as well, but clientSessions are not yet cleared (they become detached)
+        for (OAuthClient.AccessTokenResponse response : responses) {
+            String userSessionId = oauth.verifyToken(response.getAccessToken()).getSessionState();
+            getTestingClientForStartedNodeInDc(0).testing().cache(InfinispanConnectionProvider.OFFLINE_USER_SESSION_CACHE_NAME).removeKey(userSessionId);
+        }
+
+        // Increase offset to big value like 10000 hours (400+ days)
+        setTimeOffset(36000000);
+
+        // Trigger removeExpired
+        getTestingClientForStartedNodeInDc(0).testing().removeExpired(REALM_NAME);
+        getTestingClientForStartedNodeInDc(1).testing().removeExpired(REALM_NAME);
+
+        // Ensure clientSessions were removed
+        assertStatisticsExpected("After remove expired", InfinispanConnectionProvider.OFFLINE_USER_SESSION_CACHE_NAME, InfinispanConnectionProvider.OFFLINE_CLIENT_SESSION_CACHE_NAME,
+                cacheDc1Statistics, cacheDc2Statistics, channelStatisticsCrossDc,
+                sessions01, sessions02, clientSessions01, clientSessions02,
+                remoteSessions01, remoteSessions02, true);
+    }
 
     // AUTH SESSIONS