killbill-aplcache

payment: introduce per-plugin per-tenant state machines The

6/28/2016 1:34:42 PM

Changes

Details

diff --git a/api/src/main/java/org/killbill/billing/tenant/api/TenantInternalApi.java b/api/src/main/java/org/killbill/billing/tenant/api/TenantInternalApi.java
index 3a39d4f..2c6e6ea 100644
--- a/api/src/main/java/org/killbill/billing/tenant/api/TenantInternalApi.java
+++ b/api/src/main/java/org/killbill/billing/tenant/api/TenantInternalApi.java
@@ -1,6 +1,6 @@
 /*
- * Copyright 2014 Groupon, Inc
- * Copyright 2014 The Billing Project, LLC
+ * Copyright 2014-2016 Groupon, Inc
+ * Copyright 2014-2016 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -51,8 +51,9 @@ public interface TenantInternalApi {
 
     public String getPluginConfig(String pluginName, InternalTenantContext tenantContext);
 
+    public String getPluginPaymentStateMachineConfig(String pluginName, InternalTenantContext tenantContext);
+
     public List<String> getTenantValuesForKey(final String key, final InternalTenantContext tenantContext);
 
     public Tenant getTenantByApiKey(final String key) throws TenantApiException;
-
 }
diff --git a/jaxrs/src/main/java/org/killbill/billing/jaxrs/resources/JaxrsResource.java b/jaxrs/src/main/java/org/killbill/billing/jaxrs/resources/JaxrsResource.java
index 2fdd31e..118c54a 100644
--- a/jaxrs/src/main/java/org/killbill/billing/jaxrs/resources/JaxrsResource.java
+++ b/jaxrs/src/main/java/org/killbill/billing/jaxrs/resources/JaxrsResource.java
@@ -29,6 +29,7 @@ public interface JaxrsResource {
     public static final String TIMELINE = "timeline";
     public static final String REGISTER_NOTIFICATION_CALLBACK = "registerNotificationCallback";
     public static final String UPLOAD_PLUGIN_CONFIG = "uploadPluginConfig";
+    public static final String UPLOAD_PLUGIN_PAYMENT_STATE_MACHINE_CONFIG = "uploadPluginPaymentStateMachineConfig";
     public static final String USER_KEY_VALUE = "userKeyValue";
     public static final String SEARCH = "search";
 
diff --git a/jaxrs/src/main/java/org/killbill/billing/jaxrs/resources/TenantResource.java b/jaxrs/src/main/java/org/killbill/billing/jaxrs/resources/TenantResource.java
index 8ff2b14..3854b14 100644
--- a/jaxrs/src/main/java/org/killbill/billing/jaxrs/resources/TenantResource.java
+++ b/jaxrs/src/main/java/org/killbill/billing/jaxrs/resources/TenantResource.java
@@ -207,6 +207,46 @@ public class TenantResource extends JaxRsResourceBase {
         return deleteTenantKey(TenantKey.PLUGIN_CONFIG_, pluginName, createdBy, reason, comment, request);
     }
 
+    @TimedResource
+    @POST
+    @Path("/" + UPLOAD_PLUGIN_PAYMENT_STATE_MACHINE_CONFIG + "/{pluginName:" + ANYTHING_PATTERN + "}")
+    @Consumes(TEXT_PLAIN)
+    @Produces(APPLICATION_JSON)
+    @ApiOperation(value = "Add a per tenant payment state machine for a plugin")
+    @ApiResponses(value = {@ApiResponse(code = 400, message = "Invalid tenantId supplied")})
+    public Response uploadPluginPaymentStateMachineConfig(final String paymentStateMachineConfig,
+                                                          @PathParam("pluginName") final String pluginName,
+                                                          @HeaderParam(HDR_CREATED_BY) final String createdBy,
+                                                          @HeaderParam(HDR_REASON) final String reason,
+                                                          @HeaderParam(HDR_COMMENT) final String comment,
+                                                          @javax.ws.rs.core.Context final HttpServletRequest request,
+                                                          @javax.ws.rs.core.Context final UriInfo uriInfo) throws TenantApiException {
+        return insertTenantKey(TenantKey.PLUGIN_PAYMENT_STATE_MACHINE_, pluginName, paymentStateMachineConfig, uriInfo, "getPluginPaymentStateMachineConfig", createdBy, reason, comment, request);
+    }
+
+    @TimedResource
+    @GET
+    @Path("/" + UPLOAD_PLUGIN_PAYMENT_STATE_MACHINE_CONFIG + "/{pluginName:" + ANYTHING_PATTERN + "}")
+    @Produces(APPLICATION_JSON)
+    @ApiOperation(value = "Retrieve a per tenant payment state machine for a plugin", response = TenantKeyJson.class)
+    @ApiResponses(value = {@ApiResponse(code = 400, message = "Invalid tenantId supplied")})
+    public Response getPluginPaymentStateMachineConfig(@PathParam("pluginName") final String pluginName,
+                                                       @javax.ws.rs.core.Context final HttpServletRequest request) throws TenantApiException {
+        return getTenantKey(TenantKey.PLUGIN_PAYMENT_STATE_MACHINE_, pluginName, request);
+    }
+
+    @TimedResource
+    @DELETE
+    @Path("/" + UPLOAD_PLUGIN_PAYMENT_STATE_MACHINE_CONFIG + "/{pluginName:" + ANYTHING_PATTERN + "}")
+    @ApiOperation(value = "Delete a per tenant payment state machine for a plugin")
+    @ApiResponses(value = {@ApiResponse(code = 400, message = "Invalid tenantId supplied")})
+    public Response deletePluginPaymentStateMachineConfig(@PathParam("pluginName") final String pluginName,
+                                                          @HeaderParam(HDR_CREATED_BY) final String createdBy,
+                                                          @HeaderParam(HDR_REASON) final String reason,
+                                                          @HeaderParam(HDR_COMMENT) final String comment,
+                                                          @javax.ws.rs.core.Context final HttpServletRequest request) throws TenantApiException {
+        return deleteTenantKey(TenantKey.PLUGIN_PAYMENT_STATE_MACHINE_, pluginName, createdBy, reason, comment, request);
+    }
 
     @TimedResource
     @POST
diff --git a/payment/src/main/java/org/killbill/billing/payment/caching/EhCacheStateMachineConfigCache.java b/payment/src/main/java/org/killbill/billing/payment/caching/EhCacheStateMachineConfigCache.java
new file mode 100644
index 0000000..90f0863
--- /dev/null
+++ b/payment/src/main/java/org/killbill/billing/payment/caching/EhCacheStateMachineConfigCache.java
@@ -0,0 +1,142 @@
+/*
+ * Copyright 2016 Groupon, Inc
+ * Copyright 2016 The Billing Project, LLC
+ *
+ * The Billing Project licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at:
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.killbill.billing.payment.caching;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.net.URI;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+
+import org.killbill.automaton.DefaultStateMachineConfig;
+import org.killbill.automaton.StateMachineConfig;
+import org.killbill.billing.ErrorCode;
+import org.killbill.billing.ObjectType;
+import org.killbill.billing.callcontext.InternalTenantContext;
+import org.killbill.billing.payment.api.PaymentApiException;
+import org.killbill.billing.payment.glue.PaymentModule;
+import org.killbill.billing.tenant.api.TenantInternalApi;
+import org.killbill.billing.tenant.api.TenantInternalApi.CacheInvalidationCallback;
+import org.killbill.billing.tenant.api.TenantKV.TenantKey;
+import org.killbill.billing.util.cache.Cachable.CacheType;
+import org.killbill.billing.util.cache.CacheController;
+import org.killbill.billing.util.cache.CacheControllerDispatcher;
+import org.killbill.billing.util.cache.CacheLoaderArgument;
+import org.killbill.billing.util.cache.TenantStateMachineConfigCacheLoader.LoaderCallback;
+import org.killbill.billing.util.callcontext.InternalCallContextFactory;
+import org.killbill.xmlloader.XMLLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.io.Resources;
+
+public class EhCacheStateMachineConfigCache implements StateMachineConfigCache {
+
+    private static final Logger logger = LoggerFactory.getLogger(EhCacheStateMachineConfigCache.class);
+
+    private final TenantInternalApi tenantInternalApi;
+    private final CacheController cacheController;
+    private final CacheInvalidationCallback cacheInvalidationCallback;
+
+    private final LoaderCallback loaderCallback = new LoaderCallback() {
+        public Object loadStateMachineConfig(final String stateMachineConfigXML) throws PaymentApiException {
+            tenantInternalApi.initializeCacheInvalidationCallback(TenantKey.PLUGIN_PAYMENT_STATE_MACHINE_, cacheInvalidationCallback);
+
+            try {
+                final InputStream stream = new ByteArrayInputStream(stateMachineConfigXML.getBytes());
+                return XMLLoader.getObjectFromStream(new URI("dummy"), stream, DefaultStateMachineConfig.class);
+            } catch (final Exception e) {
+                // TODO 0.17 proper error code
+                throw new PaymentApiException(e, ErrorCode.PAYMENT_INTERNAL_ERROR, "Invalid payment state machine config");
+            }
+        }
+    };
+
+    private StateMachineConfig defaultPaymentStateMachineConfig;
+
+    @Inject
+    public EhCacheStateMachineConfigCache(final TenantInternalApi tenantInternalApi,
+                                          final CacheControllerDispatcher cacheControllerDispatcher,
+                                          @Named(PaymentModule.STATE_MACHINE_CONFIG_INVALIDATION_CALLBACK) final CacheInvalidationCallback cacheInvalidationCallback) {
+        this.tenantInternalApi = tenantInternalApi;
+        // Can be null if mis-configured (e.g. missing in ehcache.xml)
+        this.cacheController = cacheControllerDispatcher.getCacheController(CacheType.TENANT_PAYMENT_STATE_MACHINE_CONFIG);
+        this.cacheInvalidationCallback = cacheInvalidationCallback;
+    }
+
+    @Override
+    public void loadDefaultPaymentStateMachineConfig(final String url) throws PaymentApiException {
+        if (url != null) {
+            try {
+                defaultPaymentStateMachineConfig = XMLLoader.getObjectFromString(Resources.getResource(url).toExternalForm(), DefaultStateMachineConfig.class);
+            } catch (final Exception e) {
+                // TODO 0.17 proper error code
+                throw new PaymentApiException(e, ErrorCode.PAYMENT_INTERNAL_ERROR, "Invalid default payment state machine config");
+            }
+        }
+    }
+
+    @Override
+    public StateMachineConfig getPaymentStateMachineConfig(final String pluginName, final InternalTenantContext tenantContext) throws PaymentApiException {
+        if (tenantContext.getTenantRecordId() == InternalCallContextFactory.INTERNAL_TENANT_RECORD_ID || cacheController == null) {
+            return defaultPaymentStateMachineConfig;
+        }
+
+        final String pluginConfigKey = getCacheKeyName(pluginName, tenantContext);
+        final CacheLoaderArgument cacheLoaderArgument = createCacheLoaderArgument(pluginName);
+        try {
+            StateMachineConfig pluginPaymentStateMachineConfig = (StateMachineConfig) cacheController.get(pluginConfigKey, cacheLoaderArgument);
+            // It means we are using the default state machine config in a multi-tenant deployment
+            if (pluginPaymentStateMachineConfig == null) {
+                pluginPaymentStateMachineConfig = defaultPaymentStateMachineConfig;
+                cacheController.add(pluginConfigKey, pluginPaymentStateMachineConfig);
+            }
+            return pluginPaymentStateMachineConfig;
+        } catch (final IllegalStateException e) {
+            // TODO 0.17 proper error code
+            throw new PaymentApiException(e, ErrorCode.PAYMENT_INTERNAL_ERROR, "Invalid payment state machine");
+        }
+    }
+
+    // See also DefaultTenantUserApi - we use the same conventions as the main XML cache (so we can re-use the invalidation code)
+    private String getCacheKeyName(final String pluginName, final InternalTenantContext internalContext) {
+        final StringBuilder tenantKey = new StringBuilder(TenantKey.PLUGIN_PAYMENT_STATE_MACHINE_.toString());
+        tenantKey.append(pluginName);
+        tenantKey.append(CacheControllerDispatcher.CACHE_KEY_SEPARATOR);
+        tenantKey.append(internalContext.getTenantRecordId());
+        return tenantKey.toString();
+    }
+
+    @Override
+    public void clearPaymentStateMachineConfig(final String pluginName, final InternalTenantContext tenantContext) {
+        if (tenantContext.getTenantRecordId() != InternalCallContextFactory.INTERNAL_TENANT_RECORD_ID && cacheController != null) {
+            final String key = getCacheKeyName(pluginName, tenantContext);
+            cacheController.remove(key);
+        }
+    }
+
+    private CacheLoaderArgument createCacheLoaderArgument(final String pluginName) {
+        final Object[] args = new Object[2];
+        args[0] = loaderCallback;
+        args[1] = pluginName;
+        final ObjectType irrelevant = null;
+        final InternalTenantContext notUsed = null;
+        return new CacheLoaderArgument(irrelevant, args, notUsed);
+    }
+}
diff --git a/payment/src/main/java/org/killbill/billing/payment/caching/StateMachineConfigCache.java b/payment/src/main/java/org/killbill/billing/payment/caching/StateMachineConfigCache.java
new file mode 100644
index 0000000..64c77ad
--- /dev/null
+++ b/payment/src/main/java/org/killbill/billing/payment/caching/StateMachineConfigCache.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2016 Groupon, Inc
+ * Copyright 2016 The Billing Project, LLC
+ *
+ * The Billing Project licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at:
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.killbill.billing.payment.caching;
+
+import org.killbill.automaton.StateMachineConfig;
+import org.killbill.billing.callcontext.InternalTenantContext;
+import org.killbill.billing.payment.api.PaymentApiException;
+
+public interface StateMachineConfigCache {
+
+    public void loadDefaultPaymentStateMachineConfig(String url) throws PaymentApiException;
+
+    public StateMachineConfig getPaymentStateMachineConfig(String pluginName, InternalTenantContext tenantContext) throws PaymentApiException;
+
+    public void clearPaymentStateMachineConfig(String pluginName, InternalTenantContext tenantContext);
+}
diff --git a/payment/src/main/java/org/killbill/billing/payment/caching/StateMachineConfigCacheInvalidationCallback.java b/payment/src/main/java/org/killbill/billing/payment/caching/StateMachineConfigCacheInvalidationCallback.java
new file mode 100644
index 0000000..4a47495
--- /dev/null
+++ b/payment/src/main/java/org/killbill/billing/payment/caching/StateMachineConfigCacheInvalidationCallback.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2014-2016 Groupon, Inc
+ * Copyright 2014-2016 The Billing Project, LLC
+ *
+ * The Billing Project licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at:
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.killbill.billing.payment.caching;
+
+import javax.inject.Inject;
+
+import org.killbill.billing.callcontext.InternalTenantContext;
+import org.killbill.billing.tenant.api.TenantInternalApi.CacheInvalidationCallback;
+import org.killbill.billing.tenant.api.TenantKV.TenantKey;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+// Similar to TenantCacheInvalidationCallback
+public class StateMachineConfigCacheInvalidationCallback implements CacheInvalidationCallback {
+
+    private final Logger log = LoggerFactory.getLogger(StateMachineConfigCacheInvalidationCallback.class);
+
+    private final StateMachineConfigCache stateMachineConfigCache;
+
+    @Inject
+    public StateMachineConfigCacheInvalidationCallback(final StateMachineConfigCache stateMachineConfigCache) {
+        this.stateMachineConfigCache = stateMachineConfigCache;
+    }
+
+    @Override
+    public void invalidateCache(final TenantKey tenantKey, final Object cookie, final InternalTenantContext tenantContext) {
+        if (cookie == null) {
+            return;
+        }
+
+        log.info("Invalidate payment state machine config cache for pluginName='{}', tenantRecordId='{}'", cookie, tenantContext.getTenantRecordId());
+        stateMachineConfigCache.clearPaymentStateMachineConfig(cookie.toString(), tenantContext);
+    }
+}
diff --git a/payment/src/main/java/org/killbill/billing/payment/core/sm/PaymentAutomatonDAOHelper.java b/payment/src/main/java/org/killbill/billing/payment/core/sm/PaymentAutomatonDAOHelper.java
index 6800bbe..fa55ff0 100644
--- a/payment/src/main/java/org/killbill/billing/payment/core/sm/PaymentAutomatonDAOHelper.java
+++ b/payment/src/main/java/org/killbill/billing/payment/core/sm/PaymentAutomatonDAOHelper.java
@@ -57,6 +57,10 @@ public class PaymentAutomatonDAOHelper {
     private final OSGIServiceRegistration<PaymentPluginApi> pluginRegistry;
     private final PersistentBus eventBus;
 
+    // Cached
+    private String pluginName = null;
+    private PaymentPluginApi paymentPluginApi = null;
+
     // Used to build new payments and transactions
     public PaymentAutomatonDAOHelper(final PaymentStateContext paymentStateContext,
                                      final DateTime utcNow, final PaymentDao paymentDao,
@@ -146,27 +150,24 @@ public class PaymentAutomatonDAOHelper {
     }
 
     public String getPaymentProviderPluginName() throws PaymentApiException {
+        if (pluginName != null) {
+            return pluginName;
+        }
+
         final UUID paymentMethodId = paymentStateContext.getPaymentMethodId();
         final PaymentMethodModelDao methodDao = paymentDao.getPaymentMethodIncludedDeleted(paymentMethodId, internalCallContext);
         if (methodDao == null) {
             throw new PaymentApiException(ErrorCode.PAYMENT_NO_SUCH_PAYMENT_METHOD, paymentMethodId);
         }
-        return methodDao.getPluginName();
+        pluginName = methodDao.getPluginName();
+        return pluginName;
     }
 
-    public PaymentPluginApi getPaymentProviderPlugin() throws PaymentApiException {
+    public PaymentPluginApi getPaymentPluginApi() throws PaymentApiException {
         final String pluginName = getPaymentProviderPluginName();
         return getPaymentPluginApi(pluginName);
     }
 
-    public PaymentPluginApi getPaymentPluginApi(final String pluginName) throws PaymentApiException {
-        final PaymentPluginApi pluginApi = pluginRegistry.getServiceForName(pluginName);
-        if (pluginApi == null) {
-            throw new PaymentApiException(ErrorCode.PAYMENT_NO_SUCH_PAYMENT_PLUGIN, pluginName);
-        }
-        return pluginApi;
-    }
-
     public PaymentModelDao getPayment() throws PaymentApiException {
         final PaymentModelDao paymentModelDao;
         paymentModelDao = paymentDao.getPayment(paymentStateContext.getPaymentId(), internalCallContext);
@@ -184,6 +185,18 @@ public class PaymentAutomatonDAOHelper {
         return paymentDao;
     }
 
+    private PaymentPluginApi getPaymentPluginApi(final String pluginName) throws PaymentApiException {
+        if (paymentPluginApi != null) {
+            return paymentPluginApi;
+        }
+
+        paymentPluginApi = pluginRegistry.getServiceForName(pluginName);
+        if (paymentPluginApi == null) {
+            throw new PaymentApiException(ErrorCode.PAYMENT_NO_SUCH_PAYMENT_PLUGIN, pluginName);
+        }
+        return paymentPluginApi;
+    }
+
     private PaymentModelDao buildNewPaymentModelDao() {
         final DateTime createdDate = utcNow;
         final DateTime updatedDate = utcNow;
diff --git a/payment/src/main/java/org/killbill/billing/payment/core/sm/PaymentAutomatonRunner.java b/payment/src/main/java/org/killbill/billing/payment/core/sm/PaymentAutomatonRunner.java
index 957ecd1..0610adf 100644
--- a/payment/src/main/java/org/killbill/billing/payment/core/sm/PaymentAutomatonRunner.java
+++ b/payment/src/main/java/org/killbill/billing/payment/core/sm/PaymentAutomatonRunner.java
@@ -34,6 +34,7 @@ import org.killbill.automaton.State;
 import org.killbill.automaton.State.EnteringStateCallback;
 import org.killbill.automaton.State.LeavingStateCallback;
 import org.killbill.automaton.StateMachine;
+import org.killbill.automaton.StateMachineConfig;
 import org.killbill.billing.ErrorCode;
 import org.killbill.billing.account.api.Account;
 import org.killbill.billing.callcontext.InternalCallContext;
@@ -201,7 +202,7 @@ public class PaymentAutomatonRunner {
                 throw new IllegalStateException("Unsupported transaction type " + transactionType);
         }
 
-        runStateMachineOperation(currentStateName, transactionType, leavingStateCallback, operationCallback, enteringStateCallback);
+        runStateMachineOperation(currentStateName, transactionType, leavingStateCallback, operationCallback, enteringStateCallback, paymentStateContext, daoHelper);
 
         return paymentStateContext.getPaymentId();
     }
@@ -221,11 +222,14 @@ public class PaymentAutomatonRunner {
                                           final TransactionType transactionType,
                                           final LeavingStateCallback leavingStateCallback,
                                           final OperationCallback operationCallback,
-                                          final EnteringStateCallback enteringStateCallback) throws PaymentApiException {
+                                          final EnteringStateCallback enteringStateCallback,
+                                          final PaymentStateContext paymentStateContext,
+                                          final PaymentAutomatonDAOHelper daoHelper) throws PaymentApiException {
         try {
-            final StateMachine initialStateMachine = paymentSMHelper.getStateMachineForStateName(initialStateName);
+            final StateMachineConfig stateMachineConfig = paymentSMHelper.getStateMachineConfig(daoHelper.getPaymentProviderPluginName(), paymentStateContext.getInternalCallContext());
+            final StateMachine initialStateMachine = stateMachineConfig.getStateMachineForState(initialStateName);
             final State initialState = initialStateMachine.getState(initialStateName);
-            final Operation operation = paymentSMHelper.getOperationForTransaction(transactionType);
+            final Operation operation = paymentSMHelper.getOperationForTransaction(stateMachineConfig, transactionType);
 
             initialState.runOperation(operation, operationCallback, enteringStateCallback, leavingStateCallback);
         } catch (final MissingEntryException e) {
diff --git a/payment/src/main/java/org/killbill/billing/payment/core/sm/payments/PaymentOperation.java b/payment/src/main/java/org/killbill/billing/payment/core/sm/payments/PaymentOperation.java
index bfb1c21..434fd89 100644
--- a/payment/src/main/java/org/killbill/billing/payment/core/sm/payments/PaymentOperation.java
+++ b/payment/src/main/java/org/killbill/billing/payment/core/sm/payments/PaymentOperation.java
@@ -71,7 +71,7 @@ public abstract class PaymentOperation extends OperationCallbackBase<PaymentTran
         final String pluginName;
         try {
             pluginName = daoHelper.getPaymentProviderPluginName();
-            this.plugin = daoHelper.getPaymentPluginApi(pluginName);
+            this.plugin = daoHelper.getPaymentPluginApi();
         } catch (final PaymentApiException e) {
             throw convertToUnknownTransactionStatusAndErroredPaymentState(e);
         }
diff --git a/payment/src/main/java/org/killbill/billing/payment/core/sm/PaymentStateMachineHelper.java b/payment/src/main/java/org/killbill/billing/payment/core/sm/PaymentStateMachineHelper.java
index 064ed10..5ed95a9 100644
--- a/payment/src/main/java/org/killbill/billing/payment/core/sm/PaymentStateMachineHelper.java
+++ b/payment/src/main/java/org/killbill/billing/payment/core/sm/PaymentStateMachineHelper.java
@@ -23,8 +23,10 @@ import org.killbill.automaton.MissingEntryException;
 import org.killbill.automaton.Operation;
 import org.killbill.automaton.StateMachine;
 import org.killbill.automaton.StateMachineConfig;
+import org.killbill.billing.callcontext.InternalCallContext;
+import org.killbill.billing.payment.api.PaymentApiException;
 import org.killbill.billing.payment.api.TransactionType;
-import org.killbill.billing.payment.glue.PaymentModule;
+import org.killbill.billing.payment.caching.StateMachineConfigCache;
 
 /**
  * This class needs to know about the payment state machine xml file. All the knowledge about the xml file is encapsulated here.
@@ -73,11 +75,12 @@ public class PaymentStateMachineHelper {
     private static final String CREDIT_ERRORED = "CREDIT_ERRORED";
     private static final String VOID_ERRORED = "VOID_ERRORED";
     private static final String CHARGEBACK_ERRORED = "CHARGEBACK_ERRORED";
-    private final StateMachineConfig stateMachineConfig;
+
+    private final StateMachineConfigCache stateMachineConfigCache;
 
     @Inject
-    public PaymentStateMachineHelper(@javax.inject.Named(PaymentModule.STATE_MACHINE_PAYMENT) final StateMachineConfig stateMachineConfig) {
-        this.stateMachineConfig = stateMachineConfig;
+    public PaymentStateMachineHelper(final StateMachineConfigCache stateMachineConfigCache) {
+        this.stateMachineConfigCache = stateMachineConfigCache;
     }
 
     public String getInitStateNameForTransaction() {
@@ -168,17 +171,17 @@ public class PaymentStateMachineHelper {
         }
     }
 
-    public StateMachine getStateMachineForStateName(final String stateName) throws MissingEntryException {
-        return stateMachineConfig.getStateMachineForState(stateName);
+    public StateMachineConfig getStateMachineConfig(final String pluginName, final InternalCallContext internalCallContext) throws PaymentApiException {
+        return stateMachineConfigCache.getPaymentStateMachineConfig(pluginName, internalCallContext);
     }
 
-    public Operation getOperationForTransaction(final TransactionType transactionType) throws MissingEntryException {
-        final StateMachine stateMachine = getStateMachineForTransaction(transactionType);
+    public Operation getOperationForTransaction(final StateMachineConfig stateMachineConfig, final TransactionType transactionType) throws MissingEntryException {
+        final StateMachine stateMachine = getStateMachineForTransaction(stateMachineConfig, transactionType);
         // Only one operation defined, this is the current PaymentStates.xml model
         return stateMachine.getOperations()[0];
     }
 
-    public StateMachine getStateMachineForTransaction(final TransactionType transactionType) throws MissingEntryException {
+    private StateMachine getStateMachineForTransaction(final StateMachineConfig stateMachineConfig, final TransactionType transactionType) throws MissingEntryException {
         switch (transactionType) {
             case AUTHORIZE:
                 return stateMachineConfig.getStateMachine(AUTHORIZE_STATE_MACHINE_NAME);
diff --git a/payment/src/main/java/org/killbill/billing/payment/glue/DefaultPaymentService.java b/payment/src/main/java/org/killbill/billing/payment/glue/DefaultPaymentService.java
index e27a8b2..c0180be 100644
--- a/payment/src/main/java/org/killbill/billing/payment/glue/DefaultPaymentService.java
+++ b/payment/src/main/java/org/killbill/billing/payment/glue/DefaultPaymentService.java
@@ -1,7 +1,7 @@
 /*
- * Copyright 2010-2013 Ning, Inc.
- * Copyright 2014 Groupon, Inc
- * Copyright 2014 The Billing Project, LLC
+ * Copyright 2010-2014 Ning, Inc.
+ * Copyright 2014-2016 Groupon, Inc
+ * Copyright 2014-2016 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -19,11 +19,13 @@
 package org.killbill.billing.payment.glue;
 
 import org.killbill.billing.payment.api.PaymentApi;
+import org.killbill.billing.payment.api.PaymentApiException;
 import org.killbill.billing.payment.api.PaymentService;
 import org.killbill.billing.payment.bus.PaymentBusEventHandler;
+import org.killbill.billing.payment.caching.StateMachineConfigCache;
 import org.killbill.billing.payment.core.PaymentExecutors;
-import org.killbill.billing.payment.invoice.PaymentTagHandler;
 import org.killbill.billing.payment.core.janitor.Janitor;
+import org.killbill.billing.payment.invoice.PaymentTagHandler;
 import org.killbill.billing.payment.retry.DefaultRetryService;
 import org.killbill.billing.platform.api.LifecycleHandlerType;
 import org.killbill.billing.platform.api.LifecycleHandlerType.LifecycleLevel;
@@ -48,6 +50,7 @@ public class DefaultPaymentService implements PaymentService {
     private final DefaultRetryService retryService;
     private final Janitor janitor;
     private final PaymentExecutors paymentExecutors;
+    private final StateMachineConfigCache stateMachineConfigCache;
 
     @Inject
     public DefaultPaymentService(final PaymentBusEventHandler paymentBusEventHandler,
@@ -56,7 +59,8 @@ public class DefaultPaymentService implements PaymentService {
                                  final DefaultRetryService retryService,
                                  final PersistentBus eventBus,
                                  final Janitor janitor,
-                                 final PaymentExecutors paymentExecutors) {
+                                 final PaymentExecutors paymentExecutors,
+                                 final StateMachineConfigCache stateMachineConfigCache) {
         this.paymentBusEventHandler = paymentBusEventHandler;
         this.tagHandler = tagHandler;
         this.eventBus = eventBus;
@@ -64,6 +68,7 @@ public class DefaultPaymentService implements PaymentService {
         this.retryService = retryService;
         this.janitor = janitor;
         this.paymentExecutors = paymentExecutors;
+        this.stateMachineConfigCache = stateMachineConfigCache;
     }
 
     @Override
@@ -74,6 +79,12 @@ public class DefaultPaymentService implements PaymentService {
     @LifecycleHandlerType(LifecycleLevel.INIT_SERVICE)
     public void initialize() throws NotificationQueueAlreadyExists {
         try {
+            stateMachineConfigCache.loadDefaultPaymentStateMachineConfig(PaymentModule.DEFAULT_STATE_MACHINE_PAYMENT_XML);
+        } catch (final PaymentApiException e) {
+            log.error("Unable to load default payment state machine");
+        }
+
+        try {
             eventBus.register(paymentBusEventHandler);
             eventBus.register(tagHandler);
         } catch (final PersistentBus.EventBusException e) {
diff --git a/payment/src/main/java/org/killbill/billing/payment/glue/PaymentModule.java b/payment/src/main/java/org/killbill/billing/payment/glue/PaymentModule.java
index c9a0c74..0730b62 100644
--- a/payment/src/main/java/org/killbill/billing/payment/glue/PaymentModule.java
+++ b/payment/src/main/java/org/killbill/billing/payment/glue/PaymentModule.java
@@ -1,7 +1,7 @@
 /*
- * Copyright 2010-2013 Ning, Inc.
- * Copyright 2014-2015 Groupon, Inc
- * Copyright 2014-2015 The Billing Project, LLC
+ * Copyright 2010-2014 Ning, Inc.
+ * Copyright 2014-2016 Groupon, Inc
+ * Copyright 2014-2016 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -32,13 +32,14 @@ import org.killbill.billing.payment.api.PaymentApi;
 import org.killbill.billing.payment.api.PaymentGatewayApi;
 import org.killbill.billing.payment.api.PaymentService;
 import org.killbill.billing.payment.bus.PaymentBusEventHandler;
+import org.killbill.billing.payment.caching.EhCacheStateMachineConfigCache;
+import org.killbill.billing.payment.caching.StateMachineConfigCache;
+import org.killbill.billing.payment.caching.StateMachineConfigCacheInvalidationCallback;
 import org.killbill.billing.payment.core.PaymentExecutors;
 import org.killbill.billing.payment.core.PaymentGatewayProcessor;
 import org.killbill.billing.payment.core.PaymentMethodProcessor;
 import org.killbill.billing.payment.core.PaymentProcessor;
 import org.killbill.billing.payment.core.PluginControlPaymentProcessor;
-import org.killbill.billing.payment.core.janitor.IncompletePaymentAttemptTask;
-import org.killbill.billing.payment.core.janitor.IncompletePaymentTransactionTask;
 import org.killbill.billing.payment.core.janitor.Janitor;
 import org.killbill.billing.payment.core.sm.PaymentControlStateMachineHelper;
 import org.killbill.billing.payment.core.sm.PaymentStateMachineHelper;
@@ -54,6 +55,7 @@ import org.killbill.billing.payment.retry.DefaultRetryService;
 import org.killbill.billing.payment.retry.DefaultRetryService.DefaultRetryServiceScheduler;
 import org.killbill.billing.payment.retry.RetryService;
 import org.killbill.billing.platform.api.KillbillConfigSource;
+import org.killbill.billing.tenant.api.TenantInternalApi.CacheInvalidationCallback;
 import org.killbill.billing.util.config.PaymentConfig;
 import org.killbill.billing.util.glue.KillBillModule;
 import org.killbill.xmlloader.XMLLoader;
@@ -67,16 +69,16 @@ import com.google.inject.name.Names;
 
 public class PaymentModule extends KillBillModule {
 
-
     public static final String RETRYABLE_NAMED = "Retryable";
 
     public static final String STATE_MACHINE_RETRY = "RetryStateMachine";
-    public static final String STATE_MACHINE_PAYMENT = "PaymentStateMachine";
 
     @VisibleForTesting
-    static final String DEFAULT_STATE_MACHINE_RETRY_XML = "org/killbill/billing/payment/retry/RetryStates.xml";
+    public static final String DEFAULT_STATE_MACHINE_RETRY_XML = "org/killbill/billing/payment/retry/RetryStates.xml";
     @VisibleForTesting
-    static final String DEFAULT_STATE_MACHINE_PAYMENT_XML = "org/killbill/billing/payment/PaymentStates.xml";
+    public static final String DEFAULT_STATE_MACHINE_PAYMENT_XML = "org/killbill/billing/payment/PaymentStates.xml";
+
+    public static final String STATE_MACHINE_CONFIG_INVALIDATION_CALLBACK = "StateMachineConfigInvalidationCallback";
 
     public PaymentModule(final KillbillConfigSource configSource) {
         super(configSource);
@@ -104,13 +106,14 @@ public class PaymentModule extends KillBillModule {
     }
 
     protected void installStateMachines() {
-
         bind(StateMachineProvider.class).annotatedWith(Names.named(STATE_MACHINE_RETRY)).toInstance(new StateMachineProvider(DEFAULT_STATE_MACHINE_RETRY_XML));
         bind(StateMachineConfig.class).annotatedWith(Names.named(STATE_MACHINE_RETRY)).toProvider(Key.get(StateMachineProvider.class, Names.named(STATE_MACHINE_RETRY)));
+
         bind(PaymentControlStateMachineHelper.class).asEagerSingleton();
 
-        bind(StateMachineProvider.class).annotatedWith(Names.named(STATE_MACHINE_PAYMENT)).toInstance(new StateMachineProvider(DEFAULT_STATE_MACHINE_PAYMENT_XML));
-        bind(StateMachineConfig.class).annotatedWith(Names.named(STATE_MACHINE_PAYMENT)).toProvider(Key.get(StateMachineProvider.class, Names.named(STATE_MACHINE_PAYMENT)));
+        bind(StateMachineConfigCache.class).to(EhCacheStateMachineConfigCache.class).asEagerSingleton();
+        bind(CacheInvalidationCallback.class).annotatedWith(Names.named(STATE_MACHINE_CONFIG_INVALIDATION_CALLBACK)).to(StateMachineConfigCacheInvalidationCallback.class).asEagerSingleton();
+
         bind(PaymentStateMachineHelper.class).asEagerSingleton();
 
         bind(ControlPluginRunner.class).asEagerSingleton();
diff --git a/payment/src/test/java/org/killbill/billing/payment/caching/TestStateMachineConfigCache.java b/payment/src/test/java/org/killbill/billing/payment/caching/TestStateMachineConfigCache.java
new file mode 100644
index 0000000..9960e7a
--- /dev/null
+++ b/payment/src/test/java/org/killbill/billing/payment/caching/TestStateMachineConfigCache.java
@@ -0,0 +1,148 @@
+/*
+ * Copyright 2016 Groupon, Inc
+ * Copyright 2016 The Billing Project, LLC
+ *
+ * The Billing Project licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at:
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.killbill.billing.payment.caching;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.killbill.automaton.StateMachineConfig;
+import org.killbill.billing.callcontext.InternalCallContext;
+import org.killbill.billing.callcontext.InternalTenantContext;
+import org.killbill.billing.payment.PaymentTestSuiteNoDB;
+import org.killbill.billing.payment.api.PaymentApiException;
+import org.killbill.billing.payment.glue.PaymentModule;
+import org.killbill.xmlloader.UriAccessor;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import com.google.common.io.ByteStreams;
+import com.google.common.io.Resources;
+import net.sf.ehcache.CacheException;
+
+public class TestStateMachineConfigCache extends PaymentTestSuiteNoDB {
+
+    private InternalTenantContext multiTenantContext;
+    private InternalTenantContext otherMultiTenantContext;
+
+    @BeforeMethod(groups = "fast")
+    public void beforeMethod() throws Exception {
+        super.beforeMethod();
+
+        cacheControllerDispatcher.clearAll();
+
+        multiTenantContext = Mockito.mock(InternalTenantContext.class);
+        Mockito.when(multiTenantContext.getAccountRecordId()).thenReturn(456L);
+        Mockito.when(multiTenantContext.getTenantRecordId()).thenReturn(99L);
+
+        otherMultiTenantContext = Mockito.mock(InternalCallContext.class);
+        Mockito.when(otherMultiTenantContext.getAccountRecordId()).thenReturn(123L);
+        Mockito.when(otherMultiTenantContext.getTenantRecordId()).thenReturn(112233L);
+    }
+
+    @Test(groups = "fast")
+    public void testMissingPluginStateMachineConfig() throws PaymentApiException {
+        Assert.assertNotNull(stateMachineConfigCache.getPaymentStateMachineConfig(UUID.randomUUID().toString(), internalCallContext));
+        Assert.assertNotNull(stateMachineConfigCache.getPaymentStateMachineConfig(UUID.randomUUID().toString(), multiTenantContext));
+        Assert.assertNotNull(stateMachineConfigCache.getPaymentStateMachineConfig(UUID.randomUUID().toString(), otherMultiTenantContext));
+    }
+
+    @Test(groups = "fast")
+    public void testExistingTenantStateMachineConfig() throws PaymentApiException, URISyntaxException, IOException {
+        final String pluginName = UUID.randomUUID().toString();
+
+        final InternalCallContext differentMultiTenantContext = Mockito.mock(InternalCallContext.class);
+        Mockito.when(differentMultiTenantContext.getTenantRecordId()).thenReturn(55667788L);
+
+        final AtomicBoolean shouldThrow = new AtomicBoolean(false);
+        final Long multiTenantRecordId = multiTenantContext.getTenantRecordId();
+        final Long otherMultiTenantRecordId = otherMultiTenantContext.getTenantRecordId();
+
+        Mockito.when(tenantInternalApi.getPluginPaymentStateMachineConfig(Mockito.eq(pluginName), Mockito.any(InternalTenantContext.class))).thenAnswer(new Answer<String>() {
+            @Override
+            public String answer(final InvocationOnMock invocation) throws Throwable {
+                if (shouldThrow.get()) {
+                    throw new RuntimeException();
+                }
+                final InternalTenantContext internalContext = (InternalTenantContext) invocation.getArguments()[1];
+                if (multiTenantRecordId.equals(internalContext.getTenantRecordId())) {
+                    return new String(ByteStreams.toByteArray(UriAccessor.accessUri(Resources.getResource(PaymentModule.DEFAULT_STATE_MACHINE_PAYMENT_XML).toExternalForm())));
+                } else if (otherMultiTenantRecordId.equals(internalContext.getTenantRecordId())) {
+                    return new String(ByteStreams.toByteArray(UriAccessor.accessUri(Resources.getResource(PaymentModule.DEFAULT_STATE_MACHINE_RETRY_XML).toExternalForm())));
+                } else {
+                    return null;
+                }
+            }
+        });
+
+        // Verify the lookup for a non-cached tenant. No system config is set yet but EhCacheStateMachineConfigCache returns a default empty one
+        final StateMachineConfig defaultStateMachineConfig = stateMachineConfigCache.getPaymentStateMachineConfig(pluginName, differentMultiTenantContext);
+        Assert.assertNotNull(defaultStateMachineConfig);
+
+        // Make sure the cache loader isn't invoked, see https://github.com/killbill/killbill/issues/300
+        shouldThrow.set(true);
+
+        final StateMachineConfig defaultStateMachineConfig2 = stateMachineConfigCache.getPaymentStateMachineConfig(pluginName, differentMultiTenantContext);
+        Assert.assertNotNull(defaultStateMachineConfig2);
+        Assert.assertEquals(defaultStateMachineConfig2, defaultStateMachineConfig);
+
+        shouldThrow.set(false);
+
+        // Verify the lookup for this tenant
+        Assert.assertEquals(stateMachineConfigCache.getPaymentStateMachineConfig(UUID.randomUUID().toString(), multiTenantContext), defaultStateMachineConfig);
+        final StateMachineConfig result = stateMachineConfigCache.getPaymentStateMachineConfig(pluginName, multiTenantContext);
+        Assert.assertNotNull(result);
+        Assert.assertNotEquals(result, defaultStateMachineConfig);
+        Assert.assertEquals(result.getStateMachines().length, 8);
+
+        // Verify the lookup for another tenant
+        Assert.assertEquals(stateMachineConfigCache.getPaymentStateMachineConfig(UUID.randomUUID().toString(), otherMultiTenantContext), defaultStateMachineConfig);
+        final StateMachineConfig otherResult = stateMachineConfigCache.getPaymentStateMachineConfig(pluginName, otherMultiTenantContext);
+        Assert.assertNotNull(otherResult);
+        Assert.assertEquals(otherResult.getStateMachines().length, 1);
+
+        shouldThrow.set(true);
+
+        // Verify the lookup for this tenant
+        Assert.assertEquals(stateMachineConfigCache.getPaymentStateMachineConfig(pluginName, multiTenantContext), result);
+
+        // Verify the lookup with another context for the same tenant
+        final InternalCallContext sameMultiTenantContext = Mockito.mock(InternalCallContext.class);
+        Mockito.when(sameMultiTenantContext.getAccountRecordId()).thenReturn(9102L);
+        Mockito.when(sameMultiTenantContext.getTenantRecordId()).thenReturn(multiTenantRecordId);
+        Assert.assertEquals(stateMachineConfigCache.getPaymentStateMachineConfig(pluginName, sameMultiTenantContext), result);
+
+        // Verify the lookup with the other tenant
+        Assert.assertEquals(stateMachineConfigCache.getPaymentStateMachineConfig(pluginName, otherMultiTenantContext), otherResult);
+
+        // Verify clearing the cache works
+        stateMachineConfigCache.clearPaymentStateMachineConfig(pluginName, multiTenantContext);
+        Assert.assertEquals(stateMachineConfigCache.getPaymentStateMachineConfig(pluginName, otherMultiTenantContext), otherResult);
+        try {
+            stateMachineConfigCache.getPaymentStateMachineConfig(pluginName, multiTenantContext);
+            Assert.fail();
+        } catch (final CacheException exception) {
+            Assert.assertTrue(exception.getCause() instanceof RuntimeException);
+        }
+    }
+}
diff --git a/payment/src/test/java/org/killbill/billing/payment/caching/TestStateMachineConfigCacheInvalidationCallback.java b/payment/src/test/java/org/killbill/billing/payment/caching/TestStateMachineConfigCacheInvalidationCallback.java
new file mode 100644
index 0000000..9ead7cd
--- /dev/null
+++ b/payment/src/test/java/org/killbill/billing/payment/caching/TestStateMachineConfigCacheInvalidationCallback.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2016 Groupon, Inc
+ * Copyright 2016 The Billing Project, LLC
+ *
+ * The Billing Project licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at:
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.killbill.billing.payment.caching;
+
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.killbill.automaton.StateMachineConfig;
+import org.killbill.billing.callcontext.InternalCallContext;
+import org.killbill.billing.callcontext.InternalTenantContext;
+import org.killbill.billing.payment.PaymentTestSuiteNoDB;
+import org.killbill.billing.tenant.api.TenantKV.TenantKey;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import net.sf.ehcache.CacheException;
+
+public class TestStateMachineConfigCacheInvalidationCallback extends PaymentTestSuiteNoDB {
+
+    private InternalTenantContext multiTenantContext;
+    private InternalTenantContext otherMultiTenantContext;
+
+    @BeforeMethod(groups = "fast")
+    public void beforeMethod() throws Exception {
+        super.beforeMethod();
+
+        cacheControllerDispatcher.clearAll();
+
+        multiTenantContext = Mockito.mock(InternalTenantContext.class);
+        Mockito.when(multiTenantContext.getAccountRecordId()).thenReturn(456L);
+        Mockito.when(multiTenantContext.getTenantRecordId()).thenReturn(99L);
+
+        otherMultiTenantContext = Mockito.mock(InternalCallContext.class);
+        Mockito.when(otherMultiTenantContext.getAccountRecordId()).thenReturn(123L);
+        Mockito.when(otherMultiTenantContext.getTenantRecordId()).thenReturn(112233L);
+    }
+
+    @Test(groups = "fast")
+    public void testInvalidation() throws Exception {
+        final String pluginName = UUID.randomUUID().toString();
+
+        final StateMachineConfig defaultPaymentStateMachineConfig = stateMachineConfigCache.getPaymentStateMachineConfig(UUID.randomUUID().toString(), internalCallContext);
+        Assert.assertNotNull(defaultPaymentStateMachineConfig);
+
+        final AtomicBoolean shouldThrow = new AtomicBoolean(false);
+
+        Mockito.when(tenantInternalApi.getPluginPaymentStateMachineConfig(Mockito.eq(pluginName), Mockito.any(InternalTenantContext.class))).thenAnswer(new Answer<String>() {
+            @Override
+            public String answer(final InvocationOnMock invocation) throws Throwable {
+                if (shouldThrow.get()) {
+                    throw new RuntimeException();
+                }
+                return null;
+            }
+        });
+
+        // Prime caches
+        Assert.assertEquals(stateMachineConfigCache.getPaymentStateMachineConfig(pluginName, internalCallContext), defaultPaymentStateMachineConfig);
+        Assert.assertEquals(stateMachineConfigCache.getPaymentStateMachineConfig(pluginName, otherMultiTenantContext), defaultPaymentStateMachineConfig);
+
+        shouldThrow.set(true);
+
+        // No exception (cached)
+        Assert.assertEquals(stateMachineConfigCache.getPaymentStateMachineConfig(pluginName, internalCallContext), defaultPaymentStateMachineConfig);
+
+        cacheInvalidationCallback.invalidateCache(TenantKey.PLUGIN_PAYMENT_STATE_MACHINE_, pluginName, multiTenantContext);
+
+        try {
+            stateMachineConfigCache.getPaymentStateMachineConfig(pluginName, multiTenantContext);
+            Assert.fail();
+        } catch (final CacheException exception) {
+            Assert.assertTrue(exception.getCause() instanceof RuntimeException);
+        }
+
+        // No exception (cached)
+        Assert.assertEquals(stateMachineConfigCache.getPaymentStateMachineConfig(pluginName, otherMultiTenantContext), defaultPaymentStateMachineConfig);
+    }
+}
diff --git a/payment/src/test/java/org/killbill/billing/payment/core/sm/TestPaymentAutomatonDAOHelper.java b/payment/src/test/java/org/killbill/billing/payment/core/sm/TestPaymentAutomatonDAOHelper.java
index f6dcd0a..65586e8 100644
--- a/payment/src/test/java/org/killbill/billing/payment/core/sm/TestPaymentAutomatonDAOHelper.java
+++ b/payment/src/test/java/org/killbill/billing/payment/core/sm/TestPaymentAutomatonDAOHelper.java
@@ -100,7 +100,7 @@ public class TestPaymentAutomatonDAOHelper extends PaymentTestSuiteWithEmbeddedD
     public void testNoPaymentMethod() throws Exception {
         final PaymentAutomatonDAOHelper daoHelper = createDAOHelper(UUID.randomUUID(), paymentExternalKey, paymentTransactionExternalKey, amount, currency);
         try {
-            daoHelper.getPaymentProviderPlugin();
+            daoHelper.getPaymentPluginApi();
             Assert.fail();
         } catch (final PaymentApiException e) {
             Assert.assertEquals(e.getCode(), ErrorCode.PAYMENT_NO_SUCH_PAYMENT_METHOD.getCode());
diff --git a/payment/src/test/java/org/killbill/billing/payment/core/sm/TestPluginOperation.java b/payment/src/test/java/org/killbill/billing/payment/core/sm/TestPluginOperation.java
index a1a514f..b24ed53 100644
--- a/payment/src/test/java/org/killbill/billing/payment/core/sm/TestPluginOperation.java
+++ b/payment/src/test/java/org/killbill/billing/payment/core/sm/TestPluginOperation.java
@@ -209,7 +209,7 @@ public class TestPluginOperation extends PaymentTestSuiteNoDB {
                                                                                 callContext);
 
         final PaymentAutomatonDAOHelper daoHelper = Mockito.mock(PaymentAutomatonDAOHelper.class);
-        Mockito.when(daoHelper.getPaymentProviderPlugin()).thenReturn(null);
+        Mockito.when(daoHelper.getPaymentPluginApi()).thenReturn(null);
         return new PluginOperationTest(daoHelper, locker, paymentPluginDispatcher, paymentConfig, paymentStateContext);
     }
 
diff --git a/payment/src/test/java/org/killbill/billing/payment/core/sm/TestRetryablePayment.java b/payment/src/test/java/org/killbill/billing/payment/core/sm/TestRetryablePayment.java
index 9f6f8d8..ff72d21 100644
--- a/payment/src/test/java/org/killbill/billing/payment/core/sm/TestRetryablePayment.java
+++ b/payment/src/test/java/org/killbill/billing/payment/core/sm/TestRetryablePayment.java
@@ -80,9 +80,6 @@ import static org.testng.Assert.fail;
 public class TestRetryablePayment extends PaymentTestSuiteNoDB {
 
     @Inject
-    @Named(PaymentModule.STATE_MACHINE_PAYMENT)
-    private StateMachineConfig stateMachineConfig;
-    @Inject
     @Named(PaymentModule.STATE_MACHINE_RETRY)
     private StateMachineConfig retryStateMachineConfig;
     @Inject
diff --git a/payment/src/test/java/org/killbill/billing/payment/PaymentTestSuiteNoDB.java b/payment/src/test/java/org/killbill/billing/payment/PaymentTestSuiteNoDB.java
index 16ef9fe..74f2237 100644
--- a/payment/src/test/java/org/killbill/billing/payment/PaymentTestSuiteNoDB.java
+++ b/payment/src/test/java/org/killbill/billing/payment/PaymentTestSuiteNoDB.java
@@ -18,12 +18,15 @@
 
 package org.killbill.billing.payment;
 
+import javax.inject.Named;
+
 import org.killbill.billing.GuicyKillbillTestSuiteNoDB;
 import org.killbill.billing.account.api.AccountInternalApi;
 import org.killbill.billing.invoice.api.InvoiceInternalApi;
 import org.killbill.billing.osgi.api.OSGIServiceRegistration;
 import org.killbill.billing.payment.api.PaymentApi;
 import org.killbill.billing.payment.api.PaymentGatewayApi;
+import org.killbill.billing.payment.caching.StateMachineConfigCache;
 import org.killbill.billing.payment.core.PaymentExecutors;
 import org.killbill.billing.payment.core.PaymentMethodProcessor;
 import org.killbill.billing.payment.core.PaymentProcessor;
@@ -32,11 +35,14 @@ import org.killbill.billing.payment.core.sm.PaymentStateMachineHelper;
 import org.killbill.billing.payment.core.sm.PluginControlPaymentAutomatonRunner;
 import org.killbill.billing.payment.dao.MockPaymentDao;
 import org.killbill.billing.payment.dao.PaymentDao;
+import org.killbill.billing.payment.glue.PaymentModule;
 import org.killbill.billing.payment.glue.TestPaymentModuleNoDB;
 import org.killbill.billing.payment.plugin.api.PaymentPluginApi;
 import org.killbill.billing.payment.provider.MockPaymentProviderPlugin;
 import org.killbill.billing.payment.retry.DefaultRetryService;
 import org.killbill.billing.platform.api.KillbillConfigSource;
+import org.killbill.billing.tenant.api.TenantInternalApi;
+import org.killbill.billing.tenant.api.TenantInternalApi.CacheInvalidationCallback;
 import org.killbill.billing.util.cache.CacheControllerDispatcher;
 import org.killbill.billing.util.config.PaymentConfig;
 import org.killbill.bus.api.PersistentBus;
@@ -86,6 +92,13 @@ public abstract class PaymentTestSuiteNoDB extends GuicyKillbillTestSuiteNoDB {
     protected CacheControllerDispatcher cacheControllerDispatcher;
     @Inject
     protected PaymentExecutors paymentExecutors;
+    @Inject
+    protected StateMachineConfigCache stateMachineConfigCache;
+    @Inject
+    @Named(PaymentModule.STATE_MACHINE_CONFIG_INVALIDATION_CALLBACK)
+    protected CacheInvalidationCallback cacheInvalidationCallback;
+    @Inject
+    protected TenantInternalApi tenantInternalApi;
 
     @Override
     protected KillbillConfigSource getConfigSource() {
@@ -99,6 +112,8 @@ public abstract class PaymentTestSuiteNoDB extends GuicyKillbillTestSuiteNoDB {
     protected void beforeClass() throws Exception {
         final Injector injector = Guice.createInjector(new TestPaymentModuleNoDB(configSource, getClock()));
         injector.injectMembers(this);
+
+        stateMachineConfigCache.loadDefaultPaymentStateMachineConfig(PaymentModule.DEFAULT_STATE_MACHINE_PAYMENT_XML);
     }
 
     @BeforeMethod(groups = "fast")
diff --git a/payment/src/test/java/org/killbill/billing/payment/PaymentTestSuiteWithEmbeddedDB.java b/payment/src/test/java/org/killbill/billing/payment/PaymentTestSuiteWithEmbeddedDB.java
index 0d21398..d32f6ce 100644
--- a/payment/src/test/java/org/killbill/billing/payment/PaymentTestSuiteWithEmbeddedDB.java
+++ b/payment/src/test/java/org/killbill/billing/payment/PaymentTestSuiteWithEmbeddedDB.java
@@ -26,11 +26,13 @@ import org.killbill.billing.osgi.api.OSGIServiceRegistration;
 import org.killbill.billing.payment.api.AdminPaymentApi;
 import org.killbill.billing.payment.api.PaymentApi;
 import org.killbill.billing.payment.api.PaymentGatewayApi;
+import org.killbill.billing.payment.caching.StateMachineConfigCache;
 import org.killbill.billing.payment.core.PaymentExecutors;
 import org.killbill.billing.payment.core.PaymentProcessor;
 import org.killbill.billing.payment.core.PaymentMethodProcessor;
 import org.killbill.billing.payment.core.sm.PaymentStateMachineHelper;
 import org.killbill.billing.payment.dao.PaymentDao;
+import org.killbill.billing.payment.glue.PaymentModule;
 import org.killbill.billing.payment.glue.TestPaymentModuleWithEmbeddedDB;
 import org.killbill.billing.payment.plugin.api.PaymentPluginApi;
 import org.killbill.billing.payment.provider.MockPaymentProviderPlugin;
@@ -82,6 +84,8 @@ public abstract class PaymentTestSuiteWithEmbeddedDB extends GuicyKillbillTestSu
     protected PaymentExecutors paymentExecutors;
     @Inject
     protected NonEntityDao nonEntityDao;
+    @Inject
+    protected StateMachineConfigCache stateMachineConfigCache;
 
     @Override
     protected KillbillConfigSource getConfigSource() {
@@ -94,6 +98,8 @@ public abstract class PaymentTestSuiteWithEmbeddedDB extends GuicyKillbillTestSu
     protected void beforeClass() throws Exception {
         final Injector injector = Guice.createInjector(new TestPaymentModuleWithEmbeddedDB(configSource, getClock()));
         injector.injectMembers(this);
+
+        stateMachineConfigCache.loadDefaultPaymentStateMachineConfig(PaymentModule.DEFAULT_STATE_MACHINE_PAYMENT_XML);
     }
 
     @BeforeMethod(groups = "slow")
diff --git a/profiles/killbill/pom.xml b/profiles/killbill/pom.xml
index bc2bef9..99ccd1e 100644
--- a/profiles/killbill/pom.xml
+++ b/profiles/killbill/pom.xml
@@ -60,6 +60,11 @@
             <scope>compile</scope>
         </dependency>
         <dependency>
+            <groupId>com.jayway.awaitility</groupId>
+            <artifactId>awaitility</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
             <groupId>com.palominolabs.metrics</groupId>
             <artifactId>metrics-guice</artifactId>
             <scope>runtime</scope>
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/KillbillClient.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/KillbillClient.java
index 7b1c634..31b1729 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/KillbillClient.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/KillbillClient.java
@@ -60,6 +60,12 @@ public abstract class KillbillClient extends GuicyKillbillTestSuiteWithEmbeddedD
     protected static final String reason = "i am god";
     protected static final String comment = "no comment";
 
+    protected static RequestOptions requestOptions = RequestOptions.builder()
+                                                                   .withCreatedBy(createdBy)
+                                                                   .withReason(reason)
+                                                                   .withComment(comment)
+                                                                   .build();
+
     protected KillBillClient killBillClient;
     protected KillBillHttpClient killBillHttpClient;
 
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestTenantKV.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestTenantKV.java
index 54a51cc..94262cc 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestTenantKV.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestTenantKV.java
@@ -1,6 +1,6 @@
 /*
- * Copyright 2014-2015 Groupon, Inc
- * Copyright 2014-2015 The Billing Project, LLC
+ * Copyright 2014-2016 Groupon, Inc
+ * Copyright 2014-2016 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -17,32 +17,151 @@
 
 package org.killbill.billing.jaxrs;
 
+import java.math.BigDecimal;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.killbill.billing.ErrorCode;
+import org.killbill.billing.client.KillBillClientException;
+import org.killbill.billing.client.RequestOptions;
+import org.killbill.billing.client.model.Account;
+import org.killbill.billing.client.model.ComboPaymentTransaction;
+import org.killbill.billing.client.model.Payment;
+import org.killbill.billing.client.model.PaymentMethod;
+import org.killbill.billing.client.model.PaymentMethodPluginDetail;
+import org.killbill.billing.client.model.PaymentTransaction;
+import org.killbill.billing.client.model.PluginProperty;
+import org.killbill.billing.client.model.Tenant;
 import org.killbill.billing.client.model.TenantKey;
+import org.killbill.billing.payment.api.TransactionStatus;
 import org.killbill.billing.tenant.api.TenantKV;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.io.Resources;
+import com.jayway.awaitility.Awaitility;
+import com.jayway.awaitility.Duration;
 
 public class TestTenantKV extends TestJaxrsBase {
 
     @Test(groups = "slow", description = "Upload and retrieve a per plugin config")
     public void testPerTenantPluginConfig() throws Exception {
-
         final String pluginName = "PLUGIN_FOO";
 
         final String pluginPath = Resources.getResource("plugin.yml").getPath();
         final TenantKey tenantKey0 = killBillClient.registerPluginConfigurationForTenant(pluginName, pluginPath, createdBy, reason, comment);
         Assert.assertEquals(tenantKey0.getKey(), TenantKV.TenantKey.PLUGIN_CONFIG_.toString() + pluginName);
 
-        final TenantKey tenantKey1  = killBillClient.getPluginConfigurationForTenant(pluginName);
+        final TenantKey tenantKey1 = killBillClient.getPluginConfigurationForTenant(pluginName);
         Assert.assertEquals(tenantKey1.getKey(), TenantKV.TenantKey.PLUGIN_CONFIG_.toString() + pluginName);
         Assert.assertEquals(tenantKey1.getValues().size(), 1);
 
         killBillClient.unregisterPluginConfigurationForTenant(pluginName, createdBy, reason, comment);
-        final TenantKey tenantKey2  = killBillClient.getPluginConfigurationForTenant(pluginName);
+        final TenantKey tenantKey2 = killBillClient.getPluginConfigurationForTenant(pluginName);
         Assert.assertEquals(tenantKey2.getKey(), TenantKV.TenantKey.PLUGIN_CONFIG_.toString() + pluginName);
         Assert.assertEquals(tenantKey2.getValues().size(), 0);
     }
 
+    @Test(groups = "slow", description = "Upload and retrieve a per plugin payment state machine config")
+    public void testPerTenantPluginPaymentStateMachineConfig() throws Exception {
+        // Create another tenant - it will have a different state machine
+        final Tenant otherTenantWithDifferentStateMachine = new Tenant();
+        otherTenantWithDifferentStateMachine.setApiKey(UUID.randomUUID().toString());
+        otherTenantWithDifferentStateMachine.setApiSecret(UUID.randomUUID().toString());
+        killBillClient.createTenant(otherTenantWithDifferentStateMachine, requestOptions);
+        final RequestOptions requestOptionsOtherTenant = requestOptions.extend()
+                                                                       .withTenantApiKey(otherTenantWithDifferentStateMachine.getApiKey())
+                                                                       .withTenantApiSecret(otherTenantWithDifferentStateMachine.getApiSecret())
+                                                                       .build();
+
+        // Verify initial state
+        final TenantKey emptyTenantKey = killBillClient.getPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, requestOptions);
+        Assert.assertEquals(emptyTenantKey.getValues().size(), 0);
+        final TenantKey emptyTenantKeyOtherTenant = killBillClient.getPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, requestOptionsOtherTenant);
+        Assert.assertEquals(emptyTenantKeyOtherTenant.getValues().size(), 0);
+
+        final String stateMachineConfigPath = Resources.getResource("SimplePaymentStates.xml").getPath();
+        final TenantKey tenantKey0 = killBillClient.registerPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, stateMachineConfigPath, requestOptionsOtherTenant);
+        Assert.assertEquals(tenantKey0.getKey(), TenantKV.TenantKey.PLUGIN_PAYMENT_STATE_MACHINE_.toString() + PLUGIN_NAME);
+
+        // Verify only the other tenant has the new state machine
+        final TenantKey emptyTenantKey1 = killBillClient.getPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, requestOptions);
+        Assert.assertEquals(emptyTenantKey1.getValues().size(), 0);
+        final TenantKey tenantKey1OtherTenant = killBillClient.getPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, requestOptionsOtherTenant);
+        Assert.assertEquals(tenantKey1OtherTenant.getKey(), TenantKV.TenantKey.PLUGIN_PAYMENT_STATE_MACHINE_.toString() + PLUGIN_NAME);
+        Assert.assertEquals(tenantKey1OtherTenant.getValues().size(), 1);
+
+        // Create an auth in both tenant
+        final Payment payment = createComboPaymentTransaction(requestOptions);
+        final Payment paymentOtherTenant = createComboPaymentTransaction(requestOptionsOtherTenant);
+
+        // Void in the first tenant (allowed by the default state machine)
+        final Payment voidPayment = killBillClient.voidPayment(payment.getPaymentId(), payment.getPaymentExternalKey(), UUID.randomUUID().toString(), ImmutableList.<String>of(), ImmutableMap.<String, String>of(), requestOptions);
+        Assert.assertEquals(voidPayment.getTransactions().get(0).getStatus(), TransactionStatus.SUCCESS.toString());
+        Assert.assertEquals(voidPayment.getTransactions().get(1).getStatus(), TransactionStatus.SUCCESS.toString());
+
+        // Void in the other tenant (disallowed)
+        try {
+            killBillClient.voidPayment(paymentOtherTenant.getPaymentId(), paymentOtherTenant.getPaymentExternalKey(), UUID.randomUUID().toString(), ImmutableList.<String>of(), ImmutableMap.<String, String>of(), requestOptionsOtherTenant);
+            Assert.fail();
+        } catch (final KillBillClientException e) {
+            Assert.assertEquals((int) e.getBillingException().getCode(), ErrorCode.PAYMENT_INVALID_OPERATION.getCode());
+        }
+
+        // Remove the custom state machine
+        killBillClient.unregisterPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, requestOptionsOtherTenant);
+        final TenantKey tenantKey2 = killBillClient.getPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, requestOptionsOtherTenant);
+        Assert.assertEquals(tenantKey2.getKey(), TenantKV.TenantKey.PLUGIN_PAYMENT_STATE_MACHINE_.toString() + PLUGIN_NAME);
+        Assert.assertEquals(tenantKey2.getValues().size(), 0);
+
+        final AtomicReference<Payment> voidPaymentOtherTenant2Ref = new AtomicReference<Payment>();
+        Awaitility.await()
+                  .atMost(6, TimeUnit.SECONDS)
+                  .pollInterval(Duration.TWO_SECONDS)
+                  .until(new Callable<Boolean>() {
+                      @Override
+                      public Boolean call() throws Exception {
+                          // The void should now go through
+                          try {
+                              final Payment voidPaymentOtherTenant2 = killBillClient.voidPayment(paymentOtherTenant.getPaymentId(), paymentOtherTenant.getPaymentExternalKey(), UUID.randomUUID().toString(), ImmutableList.<String>of(), ImmutableMap.<String, String>of(), requestOptionsOtherTenant);
+                              voidPaymentOtherTenant2Ref.set(voidPaymentOtherTenant2);
+                              return voidPaymentOtherTenant2 != null;
+                          } catch (final KillBillClientException e) {
+                              // Invalidation hasn't happened yet
+                              return false;
+                          }
+                      }
+                  });
+        Assert.assertEquals(voidPaymentOtherTenant2Ref.get().getTransactions().get(0).getStatus(), TransactionStatus.SUCCESS.toString());
+        Assert.assertEquals(voidPaymentOtherTenant2Ref.get().getTransactions().get(1).getStatus(), TransactionStatus.SUCCESS.toString());
+    }
+
+    private Payment createComboPaymentTransaction(final RequestOptions requestOptions) throws KillBillClientException {
+        final Account accountJson = getAccount();
+        accountJson.setAccountId(null);
+
+        final PaymentMethodPluginDetail info = new PaymentMethodPluginDetail();
+        info.setProperties(null);
+
+        final String paymentMethodExternalKey = UUID.randomUUID().toString();
+        final PaymentMethod paymentMethodJson = new PaymentMethod(null, paymentMethodExternalKey, null, true, PLUGIN_NAME, info);
+
+        final String authTransactionExternalKey = UUID.randomUUID().toString();
+        final PaymentTransaction authTransactionJson = new PaymentTransaction();
+        authTransactionJson.setAmount(BigDecimal.TEN);
+        authTransactionJson.setCurrency(accountJson.getCurrency());
+        authTransactionJson.setPaymentExternalKey(UUID.randomUUID().toString());
+        authTransactionJson.setTransactionExternalKey(authTransactionExternalKey);
+        authTransactionJson.setTransactionType("AUTHORIZE");
+
+        final ComboPaymentTransaction comboAuthorization = new ComboPaymentTransaction(accountJson, paymentMethodJson, authTransactionJson, ImmutableList.<PluginProperty>of(), ImmutableList.<PluginProperty>of());
+        final Payment payment = killBillClient.createPayment(comboAuthorization, ImmutableMap.<String, String>of(), requestOptions);
+        Assert.assertEquals(payment.getTransactions().get(0).getStatus(), TransactionStatus.SUCCESS.toString());
+
+        return payment;
+    }
 }
diff --git a/profiles/killbill/src/test/resources/SimplePaymentStates.xml b/profiles/killbill/src/test/resources/SimplePaymentStates.xml
new file mode 100644
index 0000000..abf8e46
--- /dev/null
+++ b/profiles/killbill/src/test/resources/SimplePaymentStates.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright 2016 Groupon, Inc
+  ~ Copyright 2016 The Billing Project, LLC
+  ~
+  ~ The Billing Project licenses this file to you under the Apache License, version 2.0
+  ~ (the "License"); you may not use this file except in compliance with the
+  ~ License.  You may obtain a copy of the License at:
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+  ~ License for the specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<stateMachineConfig xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                    xsi:noNamespaceSchemaLocation="StateMachineConfig.xsd">
+
+    <stateMachines>
+        <stateMachine name="BIG_BANG">
+            <states>
+                <state name="BIG_BANG_INIT"/>
+            </states>
+            <transitions>
+                <transition>
+                    <initialState>BIG_BANG_INIT</initialState>
+                    <operation>OP_DUMMY</operation>
+                    <operationResult>SUCCESS</operationResult>
+                    <finalState>BIG_BANG_INIT</finalState>
+                </transition>
+            </transitions>
+            <operations>
+                <operation name="OP_DUMMY"/>
+            </operations>
+        </stateMachine>
+        <stateMachine name="AUTHORIZE">
+            <states>
+                <state name="AUTH_INIT"/>
+                <state name="AUTH_SUCCESS"/>
+                <state name="AUTH_FAILED"/>
+                <state name="AUTH_ERRORED"/>
+            </states>
+            <transitions>
+                <transition>
+                    <initialState>AUTH_INIT</initialState>
+                    <operation>OP_AUTHORIZE</operation>
+                    <operationResult>SUCCESS</operationResult>
+                    <finalState>AUTH_SUCCESS</finalState>
+                </transition>
+                <transition>
+                    <initialState>AUTH_INIT</initialState>
+                    <operation>OP_AUTHORIZE</operation>
+                    <operationResult>FAILURE</operationResult>
+                    <finalState>AUTH_FAILED</finalState>
+                </transition>
+                <transition>
+                    <initialState>AUTH_INIT</initialState>
+                    <operation>OP_AUTHORIZE</operation>
+                    <operationResult>EXCEPTION</operationResult>
+                    <finalState>AUTH_ERRORED</finalState>
+                </transition>
+            </transitions>
+            <operations>
+                <operation name="OP_AUTHORIZE"/>
+            </operations>
+        </stateMachine>
+    </stateMachines>
+
+    <linkStateMachines>
+        <linkStateMachine>
+            <initialStateMachine>BIG_BANG</initialStateMachine>
+            <initialState>BIG_BANG_INIT</initialState>
+            <finalStateMachine>AUTHORIZE</finalStateMachine>
+            <finalState>AUTH_INIT</finalState>
+        </linkStateMachine>
+    </linkStateMachines>
+</stateMachineConfig>
diff --git a/tenant/src/main/java/org/killbill/billing/tenant/api/DefaultTenantInternalApi.java b/tenant/src/main/java/org/killbill/billing/tenant/api/DefaultTenantInternalApi.java
index be25ccd..95c793e 100644
--- a/tenant/src/main/java/org/killbill/billing/tenant/api/DefaultTenantInternalApi.java
+++ b/tenant/src/main/java/org/killbill/billing/tenant/api/DefaultTenantInternalApi.java
@@ -1,6 +1,6 @@
 /*
- * Copyright 2014 Groupon, Inc
- * Copyright 2014 The Billing Project, LLC
+ * Copyright 2014-2016 Groupon, Inc
+ * Copyright 2014-2016 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -103,11 +103,17 @@ public class DefaultTenantInternalApi implements TenantInternalApi {
     }
 
     @Override
+    public String getPluginPaymentStateMachineConfig(final String pluginName, final InternalTenantContext tenantContext) {
+        final String pluginConfigKey = TenantKey.PLUGIN_PAYMENT_STATE_MACHINE_ + pluginName;
+        final List<String> values = tenantDao.getTenantValueForKey(pluginConfigKey, tenantContext);
+        return getUniqueValue(values, "payment state machine for plugin " + pluginConfigKey, tenantContext);
+    }
+
+    @Override
     public List<String> getTenantValuesForKey(final String key, final InternalTenantContext tenantContext) {
         return tenantDao.getTenantValueForKey(key, tenantContext);
     }
 
-
     @Override
     public Tenant getTenantByApiKey(final String key) throws TenantApiException {
         final TenantModelDao tenant = tenantDao.getTenantByApiKey(key);
@@ -117,8 +123,6 @@ public class DefaultTenantInternalApi implements TenantInternalApi {
         return new DefaultTenant(tenant);
     }
 
-
-
     private String getUniqueValue(final List<String> values, final String msg, final InternalTenantContext tenantContext) {
         if (values.isEmpty()) {
             return null;
diff --git a/tenant/src/main/java/org/killbill/billing/tenant/api/TenantCacheInvalidation.java b/tenant/src/main/java/org/killbill/billing/tenant/api/TenantCacheInvalidation.java
index da95a45..4b0d8b5 100644
--- a/tenant/src/main/java/org/killbill/billing/tenant/api/TenantCacheInvalidation.java
+++ b/tenant/src/main/java/org/killbill/billing/tenant/api/TenantCacheInvalidation.java
@@ -17,9 +17,8 @@
 
 package org.killbill.billing.tenant.api;
 
-import java.util.HashMap;
+import java.util.Collection;
 import java.util.List;
-import java.util.Map;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
@@ -46,8 +45,10 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Predicate;
+import com.google.common.collect.HashMultimap;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
 
 /**
  * This class manages the callbacks that have been registered when per tenant objects have been inserted into the
@@ -65,7 +66,7 @@ public class TenantCacheInvalidation {
 
     private static final Logger logger = LoggerFactory.getLogger(TenantCacheInvalidation.class);
 
-    private final Map<TenantKey, CacheInvalidationCallback> cache;
+    private final Multimap<TenantKey, CacheInvalidationCallback> cache;
     private final TenantBroadcastDao broadcastDao;
     private final TenantConfig tenantConfig;
     private final PersistentBus eventBus;
@@ -80,7 +81,7 @@ public class TenantCacheInvalidation {
                                    @Named(DefaultTenantModule.NO_CACHING_TENANT) final TenantDao tenantDao,
                                    final PersistentBus eventBus,
                                    final TenantConfig tenantConfig) {
-        this.cache = new HashMap<TenantKey, CacheInvalidationCallback>();
+        this.cache = HashMultimap.<TenantKey, CacheInvalidationCallback>create();
         this.broadcastDao = broadcastDao;
         this.tenantConfig = tenantConfig;
         this.tenantDao = tenantDao;
@@ -122,12 +123,11 @@ public class TenantCacheInvalidation {
     }
 
     public void registerCallback(final TenantKey key, final CacheInvalidationCallback value) {
-        if (!cache.containsKey(key)) {
-            cache.put(key, value);
-        }
+        cache.put(key, value);
+
     }
 
-    public CacheInvalidationCallback getCacheInvalidation(final TenantKey key) {
+    public Collection<CacheInvalidationCallback> getCacheInvalidations(final TenantKey key) {
         return cache.get(key);
     }
 
@@ -176,10 +176,12 @@ public class TenantCacheInvalidation {
                 try {
                     final TenantKeyAndCookie tenantKeyAndCookie = extractTenantKeyAndCookie(cur.getType());
                     if (tenantKeyAndCookie != null) {
-                        final CacheInvalidationCallback callback = parent.getCacheInvalidation(tenantKeyAndCookie.getTenantKey());
-                        if (callback != null) {
+                        final Collection<CacheInvalidationCallback> callbacks = parent.getCacheInvalidations(tenantKeyAndCookie.getTenantKey());
+                        if (!callbacks.isEmpty()) {
                             final InternalTenantContext tenantContext = new InternalTenantContext(cur.getTenantRecordId());
-                            callback.invalidateCache(tenantKeyAndCookie.getTenantKey(), tenantKeyAndCookie.getCookie(), tenantContext);
+                            for (final CacheInvalidationCallback callback : callbacks) {
+                                callback.invalidateCache(tenantKeyAndCookie.getTenantKey(), tenantKeyAndCookie.getCookie(), tenantContext);
+                            }
 
                             final Long tenantKvsTargetRecordId = cur.getTargetRecordId();
                             final BusInternalEvent event;
diff --git a/tenant/src/main/java/org/killbill/billing/tenant/api/user/DefaultTenantUserApi.java b/tenant/src/main/java/org/killbill/billing/tenant/api/user/DefaultTenantUserApi.java
index f8fc8a0..81d3df5 100644
--- a/tenant/src/main/java/org/killbill/billing/tenant/api/user/DefaultTenantUserApi.java
+++ b/tenant/src/main/java/org/killbill/billing/tenant/api/user/DefaultTenantUserApi.java
@@ -62,6 +62,7 @@ public class DefaultTenantUserApi implements TenantUserApi {
                                                                              .add(TenantKey.INVOICE_TEMPLATE)
                                                                              .add(TenantKey.INVOICE_TRANSLATION_)
                                                                              .add(TenantKey.PLUGIN_CONFIG_)
+                                                                             .add(TenantKey.PLUGIN_PAYMENT_STATE_MACHINE_)
                                                                              .add(TenantKey.PUSH_NOTIFICATION_CB).build();
 
     private final TenantDao tenantDao;
diff --git a/util/src/main/java/org/killbill/billing/util/cache/Cachable.java b/util/src/main/java/org/killbill/billing/util/cache/Cachable.java
index 6035ff3..5c5a854 100644
--- a/util/src/main/java/org/killbill/billing/util/cache/Cachable.java
+++ b/util/src/main/java/org/killbill/billing/util/cache/Cachable.java
@@ -32,6 +32,7 @@ public @interface Cachable {
     String AUDIT_LOG_CACHE_NAME = "audit-log";
     String AUDIT_LOG_VIA_HISTORY_CACHE_NAME = "audit-log-via-history";
     String TENANT_CATALOG_CACHE_NAME = "tenant-catalog";
+    String TENANT_PAYMENT_STATE_MACHINE_CONFIG_CACHE_NAME = "tenant-payment-state-machine-config";
     String TENANT_OVERDUE_CONFIG_CACHE_NAME = "tenant-overdue-config";
     String TENANT_KV_CACHE_NAME = "tenant-kv";
     String TENANT_CACHE_NAME = "tenant";
@@ -64,6 +65,9 @@ public @interface Cachable {
         /* Tenant catalog cache */
         TENANT_CATALOG(TENANT_CATALOG_CACHE_NAME, false),
 
+        /* Tenant payment state machine config cache */
+        TENANT_PAYMENT_STATE_MACHINE_CONFIG(TENANT_PAYMENT_STATE_MACHINE_CONFIG_CACHE_NAME, false),
+
         /* Tenant overdue config cache */
         TENANT_OVERDUE_CONFIG(TENANT_OVERDUE_CONFIG_CACHE_NAME, false),
 
diff --git a/util/src/main/java/org/killbill/billing/util/cache/CacheControllerDispatcher.java b/util/src/main/java/org/killbill/billing/util/cache/CacheControllerDispatcher.java
index bf2f6b1..bdf1302 100644
--- a/util/src/main/java/org/killbill/billing/util/cache/CacheControllerDispatcher.java
+++ b/util/src/main/java/org/killbill/billing/util/cache/CacheControllerDispatcher.java
@@ -22,10 +22,14 @@ import java.util.Map;
 import javax.inject.Inject;
 
 import org.killbill.billing.util.cache.Cachable.CacheType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 // Kill Bill generic cache dispatcher
 public class CacheControllerDispatcher {
 
+    private static final Logger logger = LoggerFactory.getLogger(CacheControllerDispatcher.class);
+
     public static final String CACHE_KEY_SEPARATOR = "::";
 
     private final Map<CacheType, CacheController<Object, Object>> caches;
diff --git a/util/src/main/java/org/killbill/billing/util/cache/EhCacheCacheManagerProvider.java b/util/src/main/java/org/killbill/billing/util/cache/EhCacheCacheManagerProvider.java
index f90b1d1..99429cf 100644
--- a/util/src/main/java/org/killbill/billing/util/cache/EhCacheCacheManagerProvider.java
+++ b/util/src/main/java/org/killbill/billing/util/cache/EhCacheCacheManagerProvider.java
@@ -63,7 +63,8 @@ public class EhCacheCacheManagerProvider implements Provider<CacheManager> {
                                        final TenantOverdueConfigCacheLoader tenantOverdueConfigCacheLoader,
                                        final TenantKVCacheLoader tenantKVCacheLoader,
                                        final TenantCacheLoader tenantCacheLoader,
-                                       final OverriddenPlanCacheLoader overriddenPlanCacheLoader) {
+                                       final OverriddenPlanCacheLoader overriddenPlanCacheLoader,
+                                       final TenantStateMachineConfigCacheLoader tenantStateMachineConfigCacheLoader) {
         this.metricRegistry = metricRegistry;
         this.cacheConfig = cacheConfig;
         cacheLoaders.add(accountCacheLoader);
@@ -79,6 +80,7 @@ public class EhCacheCacheManagerProvider implements Provider<CacheManager> {
         cacheLoaders.add(tenantKVCacheLoader);
         cacheLoaders.add(tenantCacheLoader);
         cacheLoaders.add(overriddenPlanCacheLoader);
+        cacheLoaders.add(tenantStateMachineConfigCacheLoader);
     }
 
     @Override
@@ -98,6 +100,11 @@ public class EhCacheCacheManagerProvider implements Provider<CacheManager> {
 
             final Ehcache cache = cacheManager.getEhcache(cacheLoader.getCacheType().getCacheName());
 
+            if (cache == null) {
+                logger.warn("Cache for cacheName='{}' not configured - check your ehcache.xml", cacheLoader.getCacheType().getCacheName());
+                continue;
+            }
+
             // Make sure we start from a clean state - this is mainly useful for tests
             for (final CacheLoader existingCacheLoader : cache.getRegisteredCacheLoaders()) {
                 cache.unregisterCacheLoader(existingCacheLoader);
diff --git a/util/src/main/java/org/killbill/billing/util/cache/TenantStateMachineConfigCacheLoader.java b/util/src/main/java/org/killbill/billing/util/cache/TenantStateMachineConfigCacheLoader.java
new file mode 100644
index 0000000..c6d5489
--- /dev/null
+++ b/util/src/main/java/org/killbill/billing/util/cache/TenantStateMachineConfigCacheLoader.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2016 Groupon, Inc
+ * Copyright 2016 The Billing Project, LLC
+ *
+ * The Billing Project licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at:
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.killbill.billing.util.cache;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.killbill.billing.callcontext.InternalTenantContext;
+import org.killbill.billing.payment.api.PaymentApiException;
+import org.killbill.billing.tenant.api.TenantInternalApi;
+import org.killbill.billing.tenant.api.TenantKV.TenantKey;
+import org.killbill.billing.util.cache.Cachable.CacheType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Singleton
+public class TenantStateMachineConfigCacheLoader extends BaseCacheLoader {
+
+    private static final Pattern PATTERN = Pattern.compile(TenantKey.PLUGIN_PAYMENT_STATE_MACHINE_.toString() + "(.*)");
+    private static final Logger log = LoggerFactory.getLogger(TenantStateMachineConfigCacheLoader.class);
+
+    private final TenantInternalApi tenantApi;
+
+    @Inject
+    public TenantStateMachineConfigCacheLoader(final TenantInternalApi tenantApi) {
+        super();
+        this.tenantApi = tenantApi;
+    }
+
+    @Override
+    public CacheType getCacheType() {
+        return CacheType.TENANT_PAYMENT_STATE_MACHINE_CONFIG;
+    }
+
+    @Override
+    public Object load(final Object key, final Object argument) {
+        checkCacheLoaderStatus();
+
+        if (!(key instanceof String)) {
+            throw new IllegalArgumentException("Unexpected key type of " + key.getClass().getName());
+        }
+        if (!(argument instanceof CacheLoaderArgument)) {
+            throw new IllegalArgumentException("Unexpected key type of " + argument.getClass().getName());
+        }
+
+        final String[] parts = ((String) key).split(CacheControllerDispatcher.CACHE_KEY_SEPARATOR);
+        final String rawKey = parts[0];
+        final Matcher matcher = PATTERN.matcher(rawKey);
+        if (!matcher.matches()) {
+            throw new IllegalArgumentException("Unexpected key " + rawKey);
+        }
+        final String pluginName = matcher.group(1);
+        final String tenantRecordId = parts[1];
+
+        final CacheLoaderArgument cacheLoaderArgument = (CacheLoaderArgument) argument;
+        final LoaderCallback callback = (LoaderCallback) cacheLoaderArgument.getArgs()[0];
+
+        final InternalTenantContext internalTenantContext = new InternalTenantContext(Long.valueOf(tenantRecordId));
+        final String stateMachineConfigXML = tenantApi.getPluginPaymentStateMachineConfig(pluginName, internalTenantContext);
+        if (stateMachineConfigXML == null) {
+            return null;
+        }
+
+        try {
+            log.info("Loading config state machine cache for pluginName='{}', tenantRecordId='{}'", pluginName, internalTenantContext.getTenantRecordId());
+            return callback.loadStateMachineConfig(stateMachineConfigXML);
+        } catch (final PaymentApiException e) {
+            throw new IllegalStateException(String.format("Failed to de-serialize state machine config for tenantRecordId='%s'", internalTenantContext.getTenantRecordId()), e);
+        }
+    }
+
+    public interface LoaderCallback {
+
+        public Object loadStateMachineConfig(final String stateMachineConfigXML) throws PaymentApiException;
+    }
+}
diff --git a/util/src/main/resources/ehcache.xml b/util/src/main/resources/ehcache.xml
index ced3336..fbf46af 100644
--- a/util/src/main/resources/ehcache.xml
+++ b/util/src/main/resources/ehcache.xml
@@ -1,9 +1,11 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
 <!--
-  ~ Copyright 2010-2013 Ning, Inc.
+  ~ Copyright 2010-2014 Ning, Inc.
+  ~ Copyright 2014-2016 Groupon, Inc
+  ~ Copyright 2014-2016 The Billing Project, LLC
   ~
-  ~ Ning licenses this file to you under the Apache License, version 2.0
+  ~ The Billing Project licenses this file to you under the Apache License, version 2.0
   ~ (the "License"); you may not use this file except in compliance with the
   ~ License.  You may obtain a copy of the License at:
   ~
@@ -208,6 +210,18 @@
                 properties=""/>
     </cache>
 
+    <cache name="tenant-payment-state-machine-config"
+           maxElementsInMemory="100"
+           maxElementsOnDisk="0"
+           overflowToDisk="false"
+           diskPersistent="false"
+           memoryStoreEvictionPolicy="LFU"
+           statistics="true"
+            >
+        <cacheEventListenerFactory
+                class="org.killbill.billing.util.cache.ExpirationListenerFactory"
+                properties=""/>
+    </cache>
 
 </ehcache>