killbill-aplcache

Merge remote-tracking branch 'origin/test-fixes' into test-fixes-doc Signed-off-by:

5/1/2018 4:15:05 PM

Changes

account/pom.xml 2(+1 -1)

api/pom.xml 2(+1 -1)

beatrix/pom.xml 2(+1 -1)

catalog/pom.xml 2(+1 -1)

currency/pom.xml 2(+1 -1)

invoice/pom.xml 2(+1 -1)

jaxrs/pom.xml 2(+1 -1)

junction/pom.xml 2(+1 -1)

NEWS 4(+4 -0)

overdue/pom.xml 2(+1 -1)

payment/pom.xml 2(+1 -1)

pom.xml 2(+1 -1)

profiles/pom.xml 2(+1 -1)

tenant/pom.xml 2(+1 -1)

usage/pom.xml 2(+1 -1)

util/pom.xml 2(+1 -1)

Details

account/pom.xml 2(+1 -1)

diff --git a/account/pom.xml b/account/pom.xml
index 03f02dd..d5faf24 100644
--- a/account/pom.xml
+++ b/account/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <artifactId>killbill</artifactId>
         <groupId>org.kill-bill.billing</groupId>
-        <version>0.19.13-SNAPSHOT</version>
+        <version>0.19.14-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>killbill-account</artifactId>

api/pom.xml 2(+1 -1)

diff --git a/api/pom.xml b/api/pom.xml
index c25649f..bb1e42c 100644
--- a/api/pom.xml
+++ b/api/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <artifactId>killbill</artifactId>
         <groupId>org.kill-bill.billing</groupId>
-        <version>0.19.13-SNAPSHOT</version>
+        <version>0.19.14-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>killbill-internal-api</artifactId>

beatrix/pom.xml 2(+1 -1)

diff --git a/beatrix/pom.xml b/beatrix/pom.xml
index e6c9447..fa183a3 100644
--- a/beatrix/pom.xml
+++ b/beatrix/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <artifactId>killbill</artifactId>
         <groupId>org.kill-bill.billing</groupId>
-        <version>0.19.13-SNAPSHOT</version>
+        <version>0.19.14-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>killbill-beatrix</artifactId>

catalog/pom.xml 2(+1 -1)

diff --git a/catalog/pom.xml b/catalog/pom.xml
index 2a45bf3..91f99c7 100644
--- a/catalog/pom.xml
+++ b/catalog/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <artifactId>killbill</artifactId>
         <groupId>org.kill-bill.billing</groupId>
-        <version>0.19.13-SNAPSHOT</version>
+        <version>0.19.14-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>killbill-catalog</artifactId>

currency/pom.xml 2(+1 -1)

diff --git a/currency/pom.xml b/currency/pom.xml
index 29f302d..8a414d3 100644
--- a/currency/pom.xml
+++ b/currency/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <artifactId>killbill</artifactId>
         <groupId>org.kill-bill.billing</groupId>
-        <version>0.19.13-SNAPSHOT</version>
+        <version>0.19.14-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>killbill-currency</artifactId>
diff --git a/entitlement/pom.xml b/entitlement/pom.xml
index 82352f1..cc4d0ed 100644
--- a/entitlement/pom.xml
+++ b/entitlement/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <artifactId>killbill</artifactId>
         <groupId>org.kill-bill.billing</groupId>
-        <version>0.19.13-SNAPSHOT</version>
+        <version>0.19.14-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>killbill-entitlement</artifactId>
diff --git a/entitlement/src/main/resources/org/killbill/billing/entitlement/dao/BlockingStateSqlDao.sql.stg b/entitlement/src/main/resources/org/killbill/billing/entitlement/dao/BlockingStateSqlDao.sql.stg
index 1e6d9dd..f5227a1 100644
--- a/entitlement/src/main/resources/org/killbill/billing/entitlement/dao/BlockingStateSqlDao.sql.stg
+++ b/entitlement/src/main/resources/org/killbill/billing/entitlement/dao/BlockingStateSqlDao.sql.stg
@@ -78,6 +78,7 @@ join (
 on t.record_id = tmp.record_id
 where t.type = :type
 and <accountRecordIdField("t.")> = :accountRecordId
+<AND_CHECK_TENANT("t.")>
 <defaultOrderBy("t.")>
 ;
  >>
@@ -99,6 +100,7 @@ join (
 ) tmp
 on t.record_id = tmp.record_id
 where <accountRecordIdField("t.")> = :accountRecordId
+<AND_CHECK_TENANT("t.")>
 <defaultOrderBy("t.")>
 ;
  >>

invoice/pom.xml 2(+1 -1)

diff --git a/invoice/pom.xml b/invoice/pom.xml
index 111e16c..3160299 100644
--- a/invoice/pom.xml
+++ b/invoice/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <artifactId>killbill</artifactId>
         <groupId>org.kill-bill.billing</groupId>
-        <version>0.19.13-SNAPSHOT</version>
+        <version>0.19.14-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>killbill-invoice</artifactId>

jaxrs/pom.xml 2(+1 -1)

diff --git a/jaxrs/pom.xml b/jaxrs/pom.xml
index cbde768..712326e 100644
--- a/jaxrs/pom.xml
+++ b/jaxrs/pom.xml
@@ -21,7 +21,7 @@
     <parent>
         <artifactId>killbill</artifactId>
         <groupId>org.kill-bill.billing</groupId>
-        <version>0.19.13-SNAPSHOT</version>
+        <version>0.19.14-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>killbill-jaxrs</artifactId>
diff --git a/jaxrs/src/main/java/org/killbill/billing/jaxrs/json/NotificationJson.java b/jaxrs/src/main/java/org/killbill/billing/jaxrs/json/NotificationJson.java
index 40e7c5c..241692a 100644
--- a/jaxrs/src/main/java/org/killbill/billing/jaxrs/json/NotificationJson.java
+++ b/jaxrs/src/main/java/org/killbill/billing/jaxrs/json/NotificationJson.java
@@ -1,7 +1,9 @@
 /*
  * Copyright 2010-2013 Ning, Inc.
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
  *
- * Ning licenses this file to you under the Apache License, version 2.0
+ * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
  * License.  You may obtain a copy of the License at:
  *
@@ -78,4 +80,52 @@ public class NotificationJson {
     public String getMetaData() {
         return metaData;
     }
+
+    @Override
+    public String toString() {
+        final StringBuilder sb = new StringBuilder("NotificationJson{");
+        sb.append("eventType='").append(eventType).append('\'');
+        sb.append(", accountId=").append(accountId);
+        sb.append(", objectType='").append(objectType).append('\'');
+        sb.append(", objectId=").append(objectId);
+        sb.append(", metaData='").append(metaData).append('\'');
+        sb.append('}');
+        return sb.toString();
+    }
+
+    @Override
+    public boolean equals(final Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        final NotificationJson that = (NotificationJson) o;
+
+        if (eventType != null ? !eventType.equals(that.eventType) : that.eventType != null) {
+            return false;
+        }
+        if (accountId != null ? !accountId.equals(that.accountId) : that.accountId != null) {
+            return false;
+        }
+        if (objectType != null ? !objectType.equals(that.objectType) : that.objectType != null) {
+            return false;
+        }
+        if (objectId != null ? !objectId.equals(that.objectId) : that.objectId != null) {
+            return false;
+        }
+        return metaData != null ? metaData.equals(that.metaData) : that.metaData == null;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = eventType != null ? eventType.hashCode() : 0;
+        result = 31 * result + (accountId != null ? accountId.hashCode() : 0);
+        result = 31 * result + (objectType != null ? objectType.hashCode() : 0);
+        result = 31 * result + (objectId != null ? objectId.hashCode() : 0);
+        result = 31 * result + (metaData != null ? metaData.hashCode() : 0);
+        return result;
+    }
 }

junction/pom.xml 2(+1 -1)

diff --git a/junction/pom.xml b/junction/pom.xml
index 6c36aed..51c6488 100644
--- a/junction/pom.xml
+++ b/junction/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <artifactId>killbill</artifactId>
         <groupId>org.kill-bill.billing</groupId>
-        <version>0.19.13-SNAPSHOT</version>
+        <version>0.19.14-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>killbill-junction</artifactId>

NEWS 4(+4 -0)

diff --git a/NEWS b/NEWS
index 28f753d..abe6ebb 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+0.19.13
+    Entitlement/subscription refactoring and incremental perf improvements
+    New implementation pass to allow a mix of RO and RW calls
+
 0.19.12
     New API to retrieve history info from any objects
     Entitlement/subscription refactoring and incremental perf improvements

overdue/pom.xml 2(+1 -1)

diff --git a/overdue/pom.xml b/overdue/pom.xml
index 9659f11..a8c8fc0 100644
--- a/overdue/pom.xml
+++ b/overdue/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <artifactId>killbill</artifactId>
         <groupId>org.kill-bill.billing</groupId>
-        <version>0.19.13-SNAPSHOT</version>
+        <version>0.19.14-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>killbill-overdue</artifactId>

payment/pom.xml 2(+1 -1)

diff --git a/payment/pom.xml b/payment/pom.xml
index 27f27cf..aa485cc 100644
--- a/payment/pom.xml
+++ b/payment/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <artifactId>killbill</artifactId>
         <groupId>org.kill-bill.billing</groupId>
-        <version>0.19.13-SNAPSHOT</version>
+        <version>0.19.14-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>killbill-payment</artifactId>

pom.xml 2(+1 -1)

diff --git a/pom.xml b/pom.xml
index 26a2b46..8f17096 100644
--- a/pom.xml
+++ b/pom.xml
@@ -24,7 +24,7 @@
         <version>0.141.61</version>
     </parent>
     <artifactId>killbill</artifactId>
-    <version>0.19.13-SNAPSHOT</version>
+    <version>0.19.14-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>killbill</name>
     <description>Library for managing recurring subscriptions and the associated billing</description>
diff --git a/profiles/killbill/pom.xml b/profiles/killbill/pom.xml
index d0a9c8c..0640371 100644
--- a/profiles/killbill/pom.xml
+++ b/profiles/killbill/pom.xml
@@ -21,7 +21,7 @@
     <parent>
         <artifactId>killbill-profiles</artifactId>
         <groupId>org.kill-bill.billing</groupId>
-        <version>0.19.13-SNAPSHOT</version>
+        <version>0.19.14-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>killbill-profiles-killbill</artifactId>
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/CallbackServer.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/CallbackServer.java
new file mode 100644
index 0000000..15654a2
--- /dev/null
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/CallbackServer.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
+ *
+ * The Billing Project licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at:
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.killbill.billing.jaxrs;
+
+import javax.servlet.Servlet;
+
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+
+public class CallbackServer {
+
+    private static final int SERVER_PORT = 8087;
+    private static final String CALLBACK_ENDPOINT = "/callmeback";
+
+    private final Server server;
+    private final String callbackEndpoint;
+    private final Servlet servlet;
+
+    public CallbackServer(final Servlet servlet) {
+        this.callbackEndpoint = CALLBACK_ENDPOINT;
+        this.servlet = servlet;
+        this.server = new Server(SERVER_PORT);
+    }
+
+    public void startServer() throws Exception {
+        final ServletContextHandler context = new ServletContextHandler();
+        context.setContextPath("/");
+        server.setHandler(context);
+        context.addServlet(new ServletHolder(servlet), callbackEndpoint);
+        server.start();
+    }
+
+    public void stopServer() throws Exception {
+        server.stop();
+    }
+
+    public static String getServletEndpoint() {
+        return "http://127.0.0.1:" + SERVER_PORT + CALLBACK_ENDPOINT;
+    }
+}
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/CallbackServlet.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/CallbackServlet.java
new file mode 100644
index 0000000..9f47378
--- /dev/null
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/CallbackServlet.java
@@ -0,0 +1,159 @@
+/*
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
+ *
+ * The Billing Project licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at:
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.killbill.billing.jaxrs;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Stack;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.killbill.billing.jaxrs.json.NotificationJson;
+import org.killbill.billing.notification.plugin.api.ExtBusEventType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Joiner;
+import com.google.common.io.CharStreams;
+
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+public class CallbackServlet extends HttpServlet {
+
+    private static final Logger log = LoggerFactory.getLogger(CallbackServlet.class);
+
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+    private static final Joiner SPACE_JOINER = Joiner.on(" ");
+    private static final long DELAY = 60000;
+
+    // Cross tenants (for now)
+    private final Collection<ExtBusEventType> nextExpectedEvent = new Stack<ExtBusEventType>();
+
+    private boolean isListenerFailed = false;
+    private String listenerFailedMsg;
+    private boolean completed = true;
+
+    @Override
+    protected void doPost(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException {
+        final String body = CharStreams.toString(new InputStreamReader(request.getInputStream(), "UTF-8"));
+        response.setStatus(HttpServletResponse.SC_OK);
+
+        final NotificationJson notification = objectMapper.readValue(body, NotificationJson.class);
+        log.info("Got notification: {}", notification);
+        assertEqualsNicely(notification.getEventType() == null ? null : ExtBusEventType.valueOf(notification.getEventType()));
+        notifyIfStackEmpty();
+    }
+
+    public void assertListenerStatus() {
+        // Bail early
+        if (isListenerFailed) {
+            log.error(listenerFailedMsg);
+            Assert.fail(listenerFailedMsg);
+        }
+
+        try {
+            assertTrue(isCompleted(DELAY));
+        } catch (final Exception e) {
+            fail("assertListenerStatus didn't complete", e);
+        }
+
+        if (isListenerFailed) {
+            log.error(listenerFailedMsg);
+            Assert.fail(listenerFailedMsg);
+        }
+    }
+
+    public synchronized void reset() {
+        nextExpectedEvent.clear();
+        completed = true;
+
+        isListenerFailed = false;
+        listenerFailedMsg = null;
+    }
+
+    public void pushExpectedEvents(final ExtBusEventType... events) {
+        for (final ExtBusEventType event : events) {
+            pushExpectedEvent(event);
+        }
+    }
+
+    public synchronized void pushExpectedEvent(final ExtBusEventType next) {
+        nextExpectedEvent.add(next);
+        log.info("Stacking expected event {}, got [{}]", next, SPACE_JOINER.join(nextExpectedEvent));
+        completed = false;
+    }
+
+    private synchronized boolean isCompleted(final long timeout) {
+        long waitTimeMs = timeout;
+        do {
+            try {
+                final long before = System.currentTimeMillis();
+                wait(100);
+                final long after = System.currentTimeMillis();
+                waitTimeMs -= (after - before);
+            } catch (final Exception ignore) {
+                return false;
+            }
+        } while (waitTimeMs > 0 && !completed);
+
+        if (!completed) {
+            log.error("CallbackServlet did not complete in " + timeout + " ms, remaining events are " + SPACE_JOINER.join(nextExpectedEvent));
+        }
+        return completed;
+    }
+
+    private synchronized void notifyIfStackEmpty() {
+        if (nextExpectedEvent.isEmpty()) {
+            log.debug("CallbackServlet EMPTY");
+            completed = true;
+            notify();
+        }
+    }
+
+    private synchronized void assertEqualsNicely(final ExtBusEventType received) {
+        boolean foundIt = false;
+        final Iterator<ExtBusEventType> it = nextExpectedEvent.iterator();
+        while (it.hasNext()) {
+            final ExtBusEventType ev = it.next();
+            if (ev == received) {
+                it.remove();
+                foundIt = true;
+                log.info("Found expected event: {}", received);
+                break;
+            }
+        }
+        if (!foundIt) {
+            final String errorMsg = "CallbackServlet: received unexpected event " + received + "; remaining expected events [" + SPACE_JOINER.join(nextExpectedEvent) + "]";
+            log.error(errorMsg);
+            failed(errorMsg);
+        }
+    }
+
+    private void failed(final String msg) {
+        this.isListenerFailed = true;
+        this.listenerFailedMsg = msg;
+    }
+}
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/KillbillClient.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/KillbillClient.java
index f09353e..4c378e9 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/KillbillClient.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/KillbillClient.java
@@ -1,7 +1,7 @@
 /*
  * Copyright 2010-2013 Ning, Inc.
- * Copyright 2014 Groupon, Inc
- * Copyright 2014 The Billing Project, LLC
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -62,6 +62,7 @@ import org.killbill.billing.client.model.gen.PaymentMethod;
 import org.killbill.billing.client.model.gen.PaymentMethodPluginDetail;
 import org.killbill.billing.client.model.gen.PluginProperty;
 import org.killbill.billing.client.model.gen.Subscription;
+import org.killbill.billing.notification.plugin.api.ExtBusEventType;
 import org.killbill.billing.payment.provider.ExternalPaymentProviderPlugin;
 import org.killbill.billing.util.UUIDs;
 import org.killbill.billing.util.tag.ControlTagType;
@@ -86,6 +87,9 @@ public abstract class KillbillClient extends GuicyKillbillTestSuiteWithEmbeddedD
 
     protected static final Currency DEFAULT_CURRENCY = Currency.USD;
 
+    // static to be shared across test class instances (initialized once in @BeforeSuite)
+    protected static CallbackServlet callbackServlet;
+
     // Multi-Tenancy information, if enabled
     protected String DEFAULT_API_KEY = UUID.randomUUID().toString();
     protected String DEFAULT_API_SECRET = UUID.randomUUID().toString();
@@ -166,10 +170,12 @@ public abstract class KillbillClient extends GuicyKillbillTestSuiteWithEmbeddedD
     protected Account createAccountWithDefaultPaymentMethod(final String externalkey, @Nullable final List<PluginProperty> pmProperties) throws Exception {
         final Account input = createAccount();
 
+        callbackServlet.pushExpectedEvent(ExtBusEventType.ACCOUNT_CHANGE);
         final PaymentMethodPluginDetail info = new PaymentMethodPluginDetail();
         info.setProperties(pmProperties);
         final PaymentMethod paymentMethodJson = new PaymentMethod(null, externalkey, input.getAccountId(), true, PLUGIN_NAME, info, EMPTY_AUDIT_LOGS);
         accountApi.createPaymentMethod(input.getAccountId(), paymentMethodJson, true, false, NULL_PLUGIN_NAMES, NULL_PLUGIN_PROPERTIES, requestOptions);
+        callbackServlet.assertListenerStatus();
         return accountApi.getAccount(input.getAccountId(), requestOptions);
     }
 
@@ -180,9 +186,14 @@ public abstract class KillbillClient extends GuicyKillbillTestSuiteWithEmbeddedD
     }
 
     protected PaymentMethod createPaymentMethod(final Account input, final boolean isDefault) throws KillBillClientException {
+        if (isDefault) {
+            callbackServlet.pushExpectedEvent(ExtBusEventType.ACCOUNT_CHANGE);
+        }
         final PaymentMethodPluginDetail info = new PaymentMethodPluginDetail();
         final PaymentMethod paymentMethodJson = new PaymentMethod(null, UUIDs.randomUUID().toString(), input.getAccountId(),
                                                                   isDefault, ExternalPaymentProviderPlugin.PLUGIN_NAME, info, EMPTY_AUDIT_LOGS);
+        callbackServlet.assertListenerStatus();
+
         return accountApi.createPaymentMethod(input.getAccountId(), paymentMethodJson, NULL_PLUGIN_NAMES, NULL_PLUGIN_PROPERTIES, requestOptions);
     }
 
@@ -191,12 +202,17 @@ public abstract class KillbillClient extends GuicyKillbillTestSuiteWithEmbeddedD
     }
 
     protected Account createAccount(final UUID parentAccountId) throws Exception {
+        callbackServlet.pushExpectedEvent(ExtBusEventType.ACCOUNT_CREATION);
         final Account input = getAccount(parentAccountId);
-        return accountApi.createAccount(input, requestOptions);
+        final Account account = accountApi.createAccount(input, requestOptions);
+        callbackServlet.assertListenerStatus();
+        return account;
     }
 
     protected Subscription createSubscription(final UUID accountId, final String bundleExternalKey, final String productName,
                                              final ProductCategory productCategory, final BillingPeriod billingPeriod, final boolean waitCompletion) throws Exception {
+        callbackServlet.pushExpectedEvents(ExtBusEventType.ACCOUNT_CHANGE, ExtBusEventType.ENTITLEMENT_CREATION, ExtBusEventType.SUBSCRIPTION_CREATION, ExtBusEventType.SUBSCRIPTION_CREATION, ExtBusEventType.INVOICE_CREATION);
+
         final Subscription input = new Subscription();
         input.setAccountId(accountId);
         input.setExternalKey(bundleExternalKey);
@@ -204,19 +220,37 @@ public abstract class KillbillClient extends GuicyKillbillTestSuiteWithEmbeddedD
         input.setProductCategory(productCategory);
         input.setBillingPeriod(billingPeriod);
         input.setPriceList(PriceListSet.DEFAULT_PRICELIST_NAME);
-        return subscriptionApi.createSubscription(input, null, null, true, false, null, waitCompletion, waitCompletion ? DEFAULT_WAIT_COMPLETION_TIMEOUT_SEC : -1L, NULL_PLUGIN_PROPERTIES, requestOptions);
+        final Subscription subscription = subscriptionApi.createSubscription(input, null, null, true, false, null, waitCompletion, waitCompletion ? DEFAULT_WAIT_COMPLETION_TIMEOUT_SEC : -1L, NULL_PLUGIN_PROPERTIES, requestOptions);
+        callbackServlet.assertListenerStatus();
+
+        return subscription;
     }
 
     protected Account createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoice() throws Exception {
+        return createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoice(true);
+    }
+
+    protected Account createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoice(final boolean paymentSuccess) throws Exception {
+        return createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoice("Shotgun", paymentSuccess);
+    }
+
+    protected Account createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoice(final String productName, final boolean paymentSuccess) throws Exception {
         final Account accountJson = createAccountWithDefaultPaymentMethod();
         assertNotNull(accountJson);
 
         // Add a bundle, subscription and move the clock to get the first invoice
-        final Subscription subscriptionJson = createSubscription(accountJson.getAccountId(), UUID.randomUUID().toString(), "Shotgun",
-                                                                ProductCategory.BASE, BillingPeriod.MONTHLY, true);
+        final Subscription subscriptionJson = createSubscription(accountJson.getAccountId(), UUID.randomUUID().toString(), productName,
+                                                                 ProductCategory.BASE, BillingPeriod.MONTHLY, true);
         assertNotNull(subscriptionJson);
+
+        callbackServlet.pushExpectedEvents(ExtBusEventType.SUBSCRIPTION_PHASE, ExtBusEventType.INVOICE_CREATION);
+        if (paymentSuccess) {
+            callbackServlet.pushExpectedEvents(ExtBusEventType.INVOICE_PAYMENT_SUCCESS, ExtBusEventType.PAYMENT_SUCCESS);
+        } else {
+            callbackServlet.pushExpectedEvents(ExtBusEventType.INVOICE_PAYMENT_FAILED, ExtBusEventType.PAYMENT_FAILED);
+        }
         clock.addDays(32);
-        crappyWaitForLackOfProperSynchonization();
+        callbackServlet.assertListenerStatus();
 
         return accountJson;
     }
@@ -225,7 +259,9 @@ public abstract class KillbillClient extends GuicyKillbillTestSuiteWithEmbeddedD
         final Account accountJson = createAccountWithExternalPaymentMethod();
         assertNotNull(accountJson);
 
+        callbackServlet.pushExpectedEvent(ExtBusEventType.TAG_CREATION);
         final Tags accountTag = accountApi.createAccountTags(accountJson.getAccountId(), ImmutableList.<UUID>of(ControlTagType.MANUAL_PAY.getId()), requestOptions);
+        callbackServlet.assertListenerStatus();
         assertNotNull(accountTag);
         assertEquals(accountTag.get(0).getTagDefinitionId(), ControlTagType.MANUAL_PAY.getId());
 
@@ -233,8 +269,10 @@ public abstract class KillbillClient extends GuicyKillbillTestSuiteWithEmbeddedD
         final Subscription subscriptionJson = createSubscription(accountJson.getAccountId(), UUID.randomUUID().toString(), "Shotgun",
                                                                 ProductCategory.BASE, BillingPeriod.MONTHLY, true);
         assertNotNull(subscriptionJson);
+
+        callbackServlet.pushExpectedEvents(ExtBusEventType.SUBSCRIPTION_PHASE, ExtBusEventType.INVOICE_CREATION);
         clock.addDays(32);
-        crappyWaitForLackOfProperSynchonization();
+        callbackServlet.assertListenerStatus();
 
         return accountJson;
     }
@@ -261,10 +299,11 @@ public abstract class KillbillClient extends GuicyKillbillTestSuiteWithEmbeddedD
         final Subscription subscriptionJson = createSubscription(accountJson.getAccountId(), UUID.randomUUID().toString(), "Shotgun",
                                                                 ProductCategory.BASE, BillingPeriod.MONTHLY, true);
         assertNotNull(subscriptionJson);
-        clock.addMonths(1);
-        crappyWaitForLackOfProperSynchonization();
 
         // No payment will be triggered as the account doesn't have a payment method
+        callbackServlet.pushExpectedEvent(ExtBusEventType.INVOICE_CREATION);
+        clock.addMonths(1);
+        callbackServlet.assertListenerStatus();
 
         return accountJson;
     }
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestAccount.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestAccount.java
index 59ca4b6..602c34a 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestAccount.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestAccount.java
@@ -69,12 +69,20 @@ public class TestAccount extends TestJaxrsBase {
 
     @BeforeMethod(groups = "slow")
     public void beforeMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         super.beforeMethod();
         mockPaymentProviderPlugin = (MockPaymentProviderPlugin) registry.getServiceForName(PLUGIN_NAME);
     }
 
     @AfterMethod(groups = "slow")
     public void tearDown() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         mockPaymentProviderPlugin.clear();
     }
 
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCache.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCache.java
index 38c7ff7..4adf0a7 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCache.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCache.java
@@ -21,20 +21,12 @@ import java.io.File;
 import java.nio.charset.Charset;
 import java.util.UUID;
 
-import org.joda.time.LocalDate;
 import org.killbill.automaton.StateMachineConfig;
 import org.killbill.billing.account.api.ImmutableAccountData;
-import org.killbill.billing.api.FlakyRetryAnalyzer;
-import org.killbill.billing.catalog.api.BillingPeriod;
 import org.killbill.billing.catalog.api.Catalog;
-import org.killbill.billing.catalog.api.PriceListSet;
-import org.killbill.billing.catalog.api.ProductCategory;
-import org.killbill.billing.client.RequestOptions;
 import org.killbill.billing.client.model.gen.Account;
-import org.killbill.billing.client.model.gen.PaymentMethod;
-import org.killbill.billing.client.model.gen.PaymentMethodPluginDetail;
-import org.killbill.billing.client.model.gen.Subscription;
 import org.killbill.billing.client.model.gen.Tenant;
+import org.killbill.billing.notification.plugin.api.ExtBusEventType;
 import org.killbill.billing.overdue.api.OverdueConfig;
 import org.killbill.billing.util.cache.Cachable.CacheType;
 import org.killbill.billing.util.cache.CacheController;
@@ -105,35 +97,21 @@ public class TestCache extends TestJaxrsBase {
         Assert.assertFalse(accountBcdCache.isKeyInCache(input.getAccountId()));
     }
 
-    // Flaky, see https://github.com/killbill/killbill/issues/860
-    @Test(groups = "slow", description = "Can Invalidate (clear) all Tenant Caches for current Tenant", retryAnalyzer = FlakyRetryAnalyzer.class)
+    @Test(groups = "slow", description = "Can Invalidate (clear) all Tenant Caches for current Tenant")
     public void testInvalidateCacheByTenant() throws Exception {
         // creating a new Tenant for this test
-        final String testApiKey = "testApiKey";
-        final String testApiSecret = "testApiSecret";
-        final Tenant tenant = new Tenant();
-        tenant.setApiKey(testApiKey);
-        tenant.setApiSecret(testApiSecret);
-        loginTenant(testApiKey, testApiSecret);
-        Tenant currentTenant = tenantApi.createTenant(tenant, false, requestOptions);
-
-        // using custom RequestOptions with the new Tenant created before
-        RequestOptions inputOptions = RequestOptions.builder()
-                                                    .withCreatedBy(createdBy)
-                                                    .withReason(reason)
-                                                    .withComment(comment)
-                                                    .withTenantApiKey(currentTenant.getApiKey())
-                                                    .withTenantApiSecret(currentTenant.getApiSecret())
-                                                    .build();
+        final Tenant currentTenant = createTenant("testApiKey", "testApiSecret", false);
 
         // Uploading the test catalog using the new Tenant created before
+        callbackServlet.pushExpectedEvent(ExtBusEventType.TENANT_CONFIG_CHANGE);
         final String catalogPath = Resources.getResource("SpyCarAdvanced.xml").getPath();
         final File catalogFile = new File(catalogPath);
         final String body = Files.toString(catalogFile, Charset.forName("UTF-8"));
-        catalogApi.uploadCatalogXml(body, inputOptions);
+        catalogApi.uploadCatalogXml(body, requestOptions);
+        callbackServlet.assertListenerStatus();
 
         // creating an Account with PaymentMethod and a Subscription
-        createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoiceWithInputOptions(inputOptions);
+        createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoice("Sports", true);
 
         // get all caches per tenant level
         final CacheController<String, Long> tenantRecordIdCache = cacheControllerDispatcher.getCacheController(CacheType.TENANT_RECORD_ID);
@@ -148,20 +126,20 @@ public class TestCache extends TestJaxrsBase {
         assertTrue(tenantRecordIdCache.isKeyInCache(currentTenant.getTenantId().toString()));
         final Long tenantRecordId = tenantRecordIdCache.get(currentTenant.getTenantId().toString(), null);
 
-        //assertTrue(hasKeysByTenantRecordId(tenantPaymentStateMachineConfigCache, tenantRecordId.toString()));
-        assertTrue(tenantCache.isKeyInCache(testApiKey));
+        assertTrue(hasKeysByTenantRecordId(tenantPaymentStateMachineConfigCache, tenantRecordId.toString()));
+        assertTrue(tenantCache.isKeyInCache(currentTenant.getApiKey()));
         assertTrue(hasKeysByTenantRecordId(tenantKvCache, tenantRecordId.toString()));
         assertTrue(tenantConfigCache.isKeyInCache(tenantRecordId));
         assertTrue(tenantOverdueConfigCache.isKeyInCache(tenantRecordId));
         assertTrue(tenantCatalogCache.isKeyInCache(tenantRecordId));
 
         // invalidate caches per tenant level
-        adminApi.invalidatesCache(null, inputOptions);
+        adminApi.invalidatesCache(null, requestOptions);
 
         // verify that now the caches don't have the previous values
         assertFalse(tenantRecordIdCache.isKeyInCache(currentTenant.getTenantId().toString()));
-        //assertFalse(hasKeysByTenantRecordId(tenantPaymentStateMachineConfigCache, tenantRecordId.toString()));
-        assertFalse(tenantCache.isKeyInCache(testApiKey));
+        assertFalse(hasKeysByTenantRecordId(tenantPaymentStateMachineConfigCache, tenantRecordId.toString()));
+        assertFalse(tenantCache.isKeyInCache(currentTenant.getApiKey()));
         assertFalse(hasKeysByTenantRecordId(tenantKvCache, tenantRecordId.toString()));
         assertFalse(tenantConfigCache.isKeyInCache(tenantRecordId));
         assertFalse(tenantOverdueConfigCache.isKeyInCache(tenantRecordId));
@@ -176,29 +154,4 @@ public class TestCache extends TestJaxrsBase {
         }
         return false;
     }
-
-    private void createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoiceWithInputOptions(final RequestOptions inputOptions) throws Exception {
-        Account account = accountApi.createAccount(getAccount(), inputOptions);
-
-        final PaymentMethodPluginDetail info = new PaymentMethodPluginDetail();
-        info.setProperties(null);
-        final PaymentMethod paymentMethodJson = new PaymentMethod(null, UUID.randomUUID().toString(), account.getAccountId(), true, PLUGIN_NAME, info, null);
-        accountApi.createPaymentMethod(account.getAccountId(), paymentMethodJson, NULL_PLUGIN_NAMES, NULL_PLUGIN_PROPERTIES, inputOptions);
-
-        final Subscription subscription = new Subscription();
-        subscription.setAccountId(account.getAccountId());
-        subscription.setExternalKey(UUID.randomUUID().toString());
-        subscription.setProductName("Sports");
-        subscription.setProductCategory(ProductCategory.BASE);
-        subscription.setBillingPeriod(BillingPeriod.MONTHLY);
-        subscription.setPriceList(PriceListSet.DEFAULT_PRICELIST_NAME);
-
-        clock.resetDeltaFromReality();
-        clock.setDay(new LocalDate(2013, 3, 1));
-        final Subscription subscriptionJson = subscriptionApi.createSubscription(subscription, null, null, null, NULL_PLUGIN_PROPERTIES, inputOptions);
-
-        assertNotNull(subscriptionJson);
-        clock.addDays(32);
-        crappyWaitForLackOfProperSynchonization();
-    }
 }
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCatalog.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCatalog.java
index fe9f6e6..7c2e38f 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCatalog.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCatalog.java
@@ -1,7 +1,7 @@
 /*
  * Copyright 2010-2013 Ning, Inc.
- * Copyright 2014-2017 Groupon, Inc
- * Copyright 2014-2017 The Billing Project, LLC
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -18,38 +18,31 @@
 
 package org.killbill.billing.jaxrs;
 
-import java.io.File;
-import java.io.IOException;
 import java.math.BigDecimal;
-import java.nio.charset.Charset;
-import java.sql.Date;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.UUID;
 
 import org.joda.time.DateTime;
+import org.joda.time.DateTime;
 import org.joda.time.LocalDate;
 import org.killbill.billing.catalog.api.BillingPeriod;
 import org.killbill.billing.catalog.api.Currency;
 import org.killbill.billing.catalog.api.ProductCategory;
 import org.killbill.billing.catalog.api.TimeUnit;
 import org.killbill.billing.client.KillBillClientException;
-import org.killbill.billing.client.RequestOptions;
 import org.killbill.billing.client.model.Catalogs;
 import org.killbill.billing.client.model.gen.Catalog;
 import org.killbill.billing.client.model.gen.Plan;
 import org.killbill.billing.client.model.gen.PlanDetail;
 import org.killbill.billing.client.model.gen.Product;
 import org.killbill.billing.client.model.gen.SimplePlan;
-import org.killbill.billing.client.model.gen.Tenant;
 import org.killbill.billing.client.model.gen.Usage;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.io.Files;
-import com.google.common.io.Resources;
 
 public class TestCatalog extends TestJaxrsBase {
 
@@ -183,22 +176,14 @@ public class TestCatalog extends TestJaxrsBase {
     @Test(groups = "slow", description = "Upload and retrieve a per plugin payment state machine config")
     public void testAddSimplePlanWithoutKBDefault() throws Exception {
         // Create another tenant initialized with no default catalog,...
-        final Tenant otherTenantNoKBDefault = new Tenant();
-        otherTenantNoKBDefault.setApiKey(UUID.randomUUID().toString());
-        otherTenantNoKBDefault.setApiSecret(UUID.randomUUID().toString());
-
-        tenantApi.createTenant(otherTenantNoKBDefault, false, requestOptions);
+        createTenant(UUID.randomUUID().toString(), UUID.randomUUID().toString(), false);
 
-        final RequestOptions requestOptionsOtherTenant = requestOptions.extend()
-                                                                       .withTenantApiKey(otherTenantNoKBDefault.getApiKey())
-                                                                       .withTenantApiSecret(otherTenantNoKBDefault.getApiSecret())
-                                                                       .build();
         // Verify the template catalog is not returned
-        List<Catalog> catalogsJson = catalogApi.getCatalogJson(null, requestOptionsOtherTenant);
+        List<Catalog> catalogsJson = catalogApi.getCatalogJson(null, requestOptions);
         Assert.assertEquals(catalogsJson.size(), 0);
 
-        catalogApi.addSimplePlan(new SimplePlan("foo-monthly", "Foo", ProductCategory.BASE, Currency.USD, BigDecimal.TEN, BillingPeriod.MONTHLY, 0, TimeUnit.UNLIMITED, ImmutableList.<String>of()), requestOptionsOtherTenant);
-        catalogsJson = catalogApi.getCatalogJson(null, requestOptionsOtherTenant);
+        catalogApi.addSimplePlan(new SimplePlan("foo-monthly", "Foo", ProductCategory.BASE, Currency.USD, BigDecimal.TEN, BillingPeriod.MONTHLY, 0, TimeUnit.UNLIMITED, ImmutableList.<String>of()), requestOptions);
+        catalogsJson = catalogApi.getCatalogJson(null, requestOptions);
         Assert.assertEquals(catalogsJson.size(), 1);
         Assert.assertEquals(catalogsJson.get(0).getProducts().size(), 1);
         Assert.assertEquals(catalogsJson.get(0).getProducts().get(0).getName(), "Foo");
@@ -207,9 +192,9 @@ public class TestCatalog extends TestJaxrsBase {
         Assert.assertEquals(catalogsJson.get(0).getPriceLists().get(0).getPlans().size(), 1);
         Assert.assertEquals(catalogsJson.get(0).getPriceLists().get(0).getPlans().get(0), "foo-monthly");
 
-        catalogApi.addSimplePlan(new SimplePlan("foo-annual", "Foo", ProductCategory.BASE, Currency.USD, new BigDecimal("100.00"), BillingPeriod.ANNUAL, 0, TimeUnit.UNLIMITED, ImmutableList.<String>of()), requestOptionsOtherTenant);
+        catalogApi.addSimplePlan(new SimplePlan("foo-annual", "Foo", ProductCategory.BASE, Currency.USD, new BigDecimal("100.00"), BillingPeriod.ANNUAL, 0, TimeUnit.UNLIMITED, ImmutableList.<String>of()), requestOptions);
 
-        catalogsJson = catalogApi.getCatalogJson(null, requestOptionsOtherTenant);
+        catalogsJson = catalogApi.getCatalogJson(null, requestOptions);
         Assert.assertEquals(catalogsJson.size(), 1);
         Assert.assertEquals(catalogsJson.get(0).getProducts().size(), 1);
         Assert.assertEquals(catalogsJson.get(0).getProducts().get(0).getName(), "Foo");
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCredit.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCredit.java
index 75b7be5..e414cc2 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCredit.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCredit.java
@@ -40,6 +40,10 @@ public class TestCredit extends TestJaxrsBase {
 
     @BeforeMethod(groups = "slow")
     public void setUp() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         accountJson = createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoice();
     }
 
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestInvoicePayment.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestInvoicePayment.java
index 9c96136..507d1c5 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestInvoicePayment.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestInvoicePayment.java
@@ -67,6 +67,10 @@ public class TestInvoicePayment extends TestJaxrsBase {
 
     @BeforeMethod(groups = "slow")
     public void beforeMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         super.beforeMethod();
         mockPaymentProviderPlugin = (MockPaymentProviderPlugin) registry.getServiceForName(PLUGIN_NAME);
     }
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestJaxrsBase.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestJaxrsBase.java
index 0fc073a..db0541f 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestJaxrsBase.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestJaxrsBase.java
@@ -76,6 +76,7 @@ import org.killbill.billing.jaxrs.resources.TestDBRouterResource;
 import org.killbill.billing.jetty.HttpServer;
 import org.killbill.billing.jetty.HttpServerConfig;
 import org.killbill.billing.lifecycle.glue.BusModule;
+import org.killbill.billing.notification.plugin.api.ExtBusEventType;
 import org.killbill.billing.osgi.api.OSGIServiceRegistration;
 import org.killbill.billing.payment.api.TransactionType;
 import org.killbill.billing.payment.glue.PaymentModule;
@@ -158,6 +159,7 @@ public class TestJaxrsBase extends KillbillClient {
 
     protected HttpServerConfig config;
     private HttpServer server;
+    private CallbackServer callbackServer;
 
     @Override
     protected KillbillConfigSource getConfigSource() {
@@ -269,6 +271,10 @@ public class TestJaxrsBase extends KillbillClient {
 
     @BeforeMethod(groups = "slow")
     public void beforeMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         super.beforeMethod();
 
         // Because we truncate the tables, the database record_id auto_increment will be reset
@@ -278,23 +284,56 @@ public class TestJaxrsBase extends KillbillClient {
         internalBus.start();
         cacheControllerDispatcher.clearAll();
         busHandler.reset();
+        callbackServlet.reset();
+
         clock.resetDeltaFromReality();
         clock.setDay(new LocalDate(2012, 8, 25));
 
         // Make sure to re-generate the api key and secret (could be cached by Shiro)
         DEFAULT_API_KEY = UUID.randomUUID().toString();
         DEFAULT_API_SECRET = UUID.randomUUID().toString();
-        loginTenant(DEFAULT_API_KEY, DEFAULT_API_SECRET);
 
         // Recreate the tenant (tables have been cleaned-up)
+        createTenant(DEFAULT_API_KEY, DEFAULT_API_SECRET, true);
+    }
+
+    protected Tenant createTenant(final String apiKey, final String apiSecret, final boolean useGlobalDefault) throws KillBillClientException {
+        callbackServlet.assertListenerStatus();
+        callbackServlet.reset();
+
+        loginTenant(apiKey, apiSecret);
         final Tenant tenant = new Tenant();
-        tenant.setApiKey(DEFAULT_API_KEY);
-        tenant.setApiSecret(DEFAULT_API_SECRET);
-        tenantApi.createTenant(tenant, true, requestOptions);
+        tenant.setApiKey(apiKey);
+        tenant.setApiSecret(apiSecret);
+
+        requestOptions = requestOptions.extend()
+                                       .withTenantApiKey(apiKey)
+                                       .withTenantApiSecret(apiSecret)
+                                       .build();
+
+        callbackServlet.pushExpectedEvent(ExtBusEventType.TENANT_CONFIG_CHANGE);
+        if (!useGlobalDefault) {
+            // Catalog
+            callbackServlet.pushExpectedEvent(ExtBusEventType.TENANT_CONFIG_CHANGE);
+        }
+
+        final Tenant createdTenant = tenantApi.createTenant(tenant, true, requestOptions);
+
+        // Register tenant for callback
+        final String callback = callbackServer.getServletEndpoint();
+        tenantApi.registerPushNotificationCallback(callback, requestOptions);
+        callbackServlet.assertListenerStatus();
+
+        createdTenant.setApiSecret(apiSecret);
+
+        return createdTenant;
     }
 
     @AfterMethod(groups = "slow")
     public void afterMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
 
         killBillHttpClient.close();
         externalBus.stop();
@@ -338,6 +377,10 @@ public class TestJaxrsBase extends KillbillClient {
         server = new HttpServer();
         server.configure(config, getListeners(), getFilters());
         server.start();
+
+        callbackServlet = new CallbackServlet();
+        callbackServer = new CallbackServer(callbackServlet);
+        callbackServer.startServer();
     }
 
     protected Iterable<EventListener> getListeners() {
@@ -359,6 +402,7 @@ public class TestJaxrsBase extends KillbillClient {
     public void afterSuite() {
         try {
             server.stop();
+            callbackServer.stopServer();
         } catch (final Exception ignored) {
         }
     }
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPayment.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPayment.java
index efce197..9538a4e 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPayment.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPayment.java
@@ -77,6 +77,10 @@ public class TestPayment extends TestJaxrsBase {
 
     @BeforeMethod(groups = "slow")
     public void beforeMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         super.beforeMethod();
         mockPaymentProviderPlugin = (MockPaymentProviderPlugin) registry.getServiceForName(PLUGIN_NAME);
 
@@ -101,6 +105,10 @@ public class TestPayment extends TestJaxrsBase {
 
     @AfterMethod(groups = "slow")
     public void tearDown() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         mockPaymentProviderPlugin.clear();
     }
 
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPaymentPluginProperties.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPaymentPluginProperties.java
index 0709ab5..de0326c 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPaymentPluginProperties.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPaymentPluginProperties.java
@@ -1,6 +1,6 @@
 /*
- * Copyright 2014-2016 Groupon, Inc
- * Copyright 2014-2016 The Billing Project, LLC
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -126,6 +126,10 @@ public class TestPaymentPluginProperties extends TestJaxrsBase {
 
     @BeforeMethod(groups = "slow")
     public void beforeMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         super.beforeMethod();
 
         mockPaymentControlProviderPlugin = new PluginPropertiesVerificator();
@@ -149,6 +153,10 @@ public class TestPaymentPluginProperties extends TestJaxrsBase {
 
     @AfterMethod(groups = "slow")
     public void tearDown() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         mockPaymentControlProviderPlugin.clearExpectPluginProperties();
     }
 
@@ -274,4 +282,4 @@ public class TestPaymentPluginProperties extends TestJaxrsBase {
         bodyProperties.add(new PluginProperty(key, value, false));
         expectProperties.add(new org.killbill.billing.payment.api.PluginProperty(key, value, false));
     }
-}
\ No newline at end of file
+}
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPerTenantConfig.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPerTenantConfig.java
index 944814a..36e1233 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPerTenantConfig.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPerTenantConfig.java
@@ -1,6 +1,6 @@
 /*
- * Copyright 2014-2016 Groupon, Inc
- * Copyright 2014-2016 The Billing Project, LLC
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -18,15 +18,11 @@
 package org.killbill.billing.jaxrs;
 
 import java.util.HashMap;
-import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
 
-import org.awaitility.Awaitility;
-import org.awaitility.Duration;
 import org.killbill.billing.client.model.Payments;
 import org.killbill.billing.client.model.gen.Account;
-import org.killbill.billing.client.model.gen.Tenant;
 import org.killbill.billing.client.model.gen.TenantKeyValue;
+import org.killbill.billing.notification.plugin.api.ExtBusEventType;
 import org.killbill.billing.osgi.api.OSGIServiceRegistration;
 import org.killbill.billing.payment.api.TransactionStatus;
 import org.killbill.billing.payment.plugin.api.PaymentPluginApi;
@@ -48,25 +44,27 @@ public class TestPerTenantConfig extends TestJaxrsBase {
 
     @BeforeMethod(groups = "slow")
     public void beforeMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         super.beforeMethod();
         mockPaymentProviderPlugin = (MockPaymentProviderPlugin) registry.getServiceForName(PLUGIN_NAME);
     }
 
     @AfterMethod(groups = "slow")
     public void tearDown() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         mockPaymentProviderPlugin.clear();
     }
 
     @Test(groups = "slow")
     public void testFailedPaymentWithPerTenantRetryConfig() throws Exception {
         // Create the tenant
-        final String apiKeyTenant1 = "tenantSuperTuned";
-        final String apiSecretTenant1 = "2367$$ffr79";
-        loginTenant(apiKeyTenant1, apiSecretTenant1);
-        final Tenant tenant1 = new Tenant();
-        tenant1.setApiKey(apiKeyTenant1);
-        tenant1.setApiSecret(apiSecretTenant1);
-        tenantApi.createTenant(tenant1, true, requestOptions);
+        createTenant("tenantSuperTuned", "2367$$ffr79", true);
 
         // Configure our plugin to fail
         mockPaymentProviderPlugin.makeAllInvoicesFailWithError(true);
@@ -77,9 +75,11 @@ public class TestPerTenantConfig extends TestJaxrsBase {
         perTenantProperties.put("org.killbill.payment.retry.days", "1,1,1");
         final String perTenantConfig = mapper.writeValueAsString(perTenantProperties);
 
+        callbackServlet.pushExpectedEvent(ExtBusEventType.TENANT_CONFIG_CHANGE);
         final TenantKeyValue tenantKey = tenantApi.uploadPerTenantConfiguration(perTenantConfig, requestOptions);
+        callbackServlet.assertListenerStatus();
 
-        final Account accountJson = createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoice();
+        final Account accountJson = createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoice(false);
 
         final Payments payments = accountApi.getPaymentsForAccount(accountJson.getAccountId(), NULL_PLUGIN_PROPERTIES, requestOptions);
         Assert.assertEquals(payments.size(), 1);
@@ -92,23 +92,14 @@ public class TestPerTenantConfig extends TestJaxrsBase {
         //
         // Now unregister special per tenant config and we the first retry occurs one day after (and still fails), it now sets a retry date of 8 days
         //
-
+        callbackServlet.pushExpectedEvents(ExtBusEventType.TENANT_CONFIG_DELETION);
         tenantApi.deletePerTenantConfiguration(requestOptions);
-        // org.killbill.tenant.broadcast.rate has been set to 1s
-        crappyWaitForLackOfProperSynchonization(2000);
+        callbackServlet.assertListenerStatus();
 
+        callbackServlet.pushExpectedEvents(ExtBusEventType.INVOICE_PAYMENT_FAILED, ExtBusEventType.PAYMENT_FAILED);
         clock.addDays(1);
+        callbackServlet.assertListenerStatus();
 
-        Awaitility.await()
-                  .atMost(4, TimeUnit.SECONDS)
-                  .pollInterval(Duration.ONE_SECOND)
-                  .until(new Callable<Boolean>() {
-                      @Override
-                      public Boolean call() throws Exception {
-
-                          return accountApi.getPaymentsForAccount(accountJson.getAccountId(), NULL_PLUGIN_PROPERTIES, requestOptions).get(0).getTransactions().size() == 2;
-                      }
-                  });
         final Payments payments2 = accountApi.getPaymentsForAccount(accountJson.getAccountId(), NULL_PLUGIN_PROPERTIES, requestOptions);
         Assert.assertEquals(payments2.size(), 1);
         Assert.assertEquals(payments2.get(0).getTransactions().size(), 2);
@@ -116,7 +107,7 @@ public class TestPerTenantConfig extends TestJaxrsBase {
         Assert.assertEquals(payments2.get(0).getTransactions().get(1).getStatus(), TransactionStatus.PAYMENT_FAILURE);
 
         clock.addDays(1);
-        crappyWaitForLackOfProperSynchonization(3000);
+        callbackServlet.assertListenerStatus();
 
         // No retry with default config
         final Payments payments3 = accountApi.getPaymentsForAccount(accountJson.getAccountId(), NULL_PLUGIN_PROPERTIES, requestOptions);
@@ -124,17 +115,10 @@ public class TestPerTenantConfig extends TestJaxrsBase {
         Assert.assertEquals(payments3.get(0).getTransactions().size(), 2);
 
         mockPaymentProviderPlugin.makeAllInvoicesFailWithError(false);
+        callbackServlet.pushExpectedEvents(ExtBusEventType.INVOICE_PAYMENT_SUCCESS, ExtBusEventType.PAYMENT_SUCCESS);
         clock.addDays(7);
+        callbackServlet.assertListenerStatus();
 
-        Awaitility.await()
-                  .atMost(4, TimeUnit.SECONDS)
-                  .pollInterval(Duration.ONE_SECOND)
-                  .until(new Callable<Boolean>() {
-                      @Override
-                      public Boolean call() throws Exception {
-                          return accountApi.getPaymentsForAccount(accountJson.getAccountId(), NULL_PLUGIN_PROPERTIES, requestOptions).get(0).getTransactions().size() == 3;
-                      }
-                  });
         final Payments payments4 = accountApi.getPaymentsForAccount(accountJson.getAccountId(), NULL_PLUGIN_PROPERTIES, requestOptions);
         Assert.assertEquals(payments4.size(), 1);
         Assert.assertEquals(payments4.get(0).getTransactions().size(), 3);
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPlugin.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPlugin.java
index 1817efe..8fd9966 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPlugin.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPlugin.java
@@ -60,6 +60,10 @@ public class TestPlugin extends TestJaxrsBase {
     @Override
     @BeforeMethod(groups = "slow")
     public void beforeMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         super.beforeMethod();
         setupOSGIPlugin();
         resetAllMarkers();
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPushNotification.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPushNotification.java
index 683c3c6..9e752f6 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPushNotification.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPushNotification.java
@@ -32,9 +32,6 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import org.awaitility.Awaitility;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
 import org.joda.time.DateTime;
 import org.killbill.CreatorName;
 import org.killbill.billing.api.FlakyRetryAnalyzer;
@@ -68,9 +65,6 @@ public class TestPushNotification extends TestJaxrsBase {
 
     private CallbackServer callbackServer;
 
-    private static final int SERVER_PORT = 8087;
-    private static final String CALLBACK_ENDPOINT = "/callmeback";
-
     private volatile boolean callbackCompleted;
     private volatile boolean callbackCompletedWithError;
     private volatile int expectedNbCalls = 1;
@@ -80,8 +74,12 @@ public class TestPushNotification extends TestJaxrsBase {
     @Override
     @BeforeMethod(groups = "slow")
     public void beforeMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         super.beforeMethod();
-        callbackServer = new CallbackServer(this, SERVER_PORT, CALLBACK_ENDPOINT);
+        callbackServer = new CallbackServer(new CallmebackServlet(this));
         resetCallbackStatusProperties();
         callbackServer.startServer();
         this.expectedNbCalls = 1;
@@ -89,6 +87,10 @@ public class TestPushNotification extends TestJaxrsBase {
 
     @AfterMethod(groups = "slow")
     public void afterMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         callbackServer.stopServer();
     }
 
@@ -221,7 +223,7 @@ public class TestPushNotification extends TestJaxrsBase {
     }
 
     private String registerTenantForCallback() throws KillBillClientException, InterruptedException {// Register tenant for callback
-        final String callback = "http://127.0.0.1:" + SERVER_PORT + CALLBACK_ENDPOINT;
+        final String callback = callbackServer.getServletEndpoint();
         final TenantKeyValue result0 = tenantApi.registerPushNotificationCallback(callback, requestOptions);
 
         Assert.assertTrue(waitForCallbacksToComplete());
@@ -367,31 +369,6 @@ public class TestPushNotification extends TestJaxrsBase {
         callbackCompletedWithError = withError;
     }
 
-    public static class CallbackServer {
-
-        private final Server server;
-        private final String callbackEndpoint;
-        private final TestPushNotification test;
-
-        public CallbackServer(final TestPushNotification test, final int port, final String callbackEndpoint) {
-            this.callbackEndpoint = callbackEndpoint;
-            this.test = test;
-            this.server = new Server(port);
-        }
-
-        public void startServer() throws Exception {
-            final ServletContextHandler context = new ServletContextHandler();
-            context.setContextPath("/");
-            server.setHandler(context);
-            context.addServlet(new ServletHolder(new CallmebackServlet(test)), callbackEndpoint);
-            server.start();
-        }
-
-        public void stopServer() throws Exception {
-            server.stop();
-        }
-    }
-
     public static class CallmebackServlet extends HttpServlet {
 
         private static final long serialVersionUID = -5181211514918217301L;
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestTenantKV.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestTenantKV.java
index c48d3e2..1acaec4 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestTenantKV.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestTenantKV.java
@@ -1,6 +1,6 @@
 /*
- * Copyright 2014-2016 Groupon, Inc
- * Copyright 2014-2016 The Billing Project, LLC
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -37,6 +37,7 @@ import org.killbill.billing.client.model.gen.PaymentTransaction;
 import org.killbill.billing.client.model.gen.PluginProperty;
 import org.killbill.billing.client.model.gen.Tenant;
 import org.killbill.billing.client.model.gen.TenantKeyValue;
+import org.killbill.billing.notification.plugin.api.ExtBusEventType;
 import org.killbill.billing.payment.api.TransactionStatus;
 import org.killbill.billing.payment.api.TransactionType;
 import org.killbill.billing.tenant.api.TenantKV;
@@ -51,8 +52,10 @@ public class TestTenantKV extends TestJaxrsBase {
     public void testPerTenantPluginConfig() throws Exception {
         final String pluginName = "PLUGIN_FOO";
 
+        callbackServlet.pushExpectedEvent(ExtBusEventType.TENANT_CONFIG_CHANGE);
         final String pluginConfig = getResourceBodyString("plugin.yml");
         final TenantKeyValue tenantKey0 = tenantApi.uploadPluginConfiguration(pluginName, pluginConfig, requestOptions);
+        callbackServlet.assertListenerStatus();
         Assert.assertEquals(tenantKey0.getKey(), TenantKV.TenantKey.PLUGIN_CONFIG_.toString() + pluginName);
 
         final TenantKeyValue tenantKey1 = tenantApi.getPluginConfiguration(pluginName, requestOptions);
@@ -67,54 +70,56 @@ public class TestTenantKV extends TestJaxrsBase {
 
     @Test(groups = "slow", description = "Upload and retrieve a per plugin payment state machine config")
     public void testPerTenantPluginPaymentStateMachineConfig() throws Exception {
+        final RequestOptions requestOptionsForOriginalTenant = requestOptions;
+
         // Create another tenant - it will have a different state machine
-        final Tenant otherTenantWithDifferentStateMachine = new Tenant();
-        otherTenantWithDifferentStateMachine.setApiKey(UUID.randomUUID().toString());
-        otherTenantWithDifferentStateMachine.setApiSecret(UUID.randomUUID().toString());
-        tenantApi.createTenant(otherTenantWithDifferentStateMachine, true, requestOptions);
-        final RequestOptions requestOptionsOtherTenant = requestOptions.extend()
-                                                                       .withTenantApiKey(otherTenantWithDifferentStateMachine.getApiKey())
-                                                                       .withTenantApiSecret(otherTenantWithDifferentStateMachine.getApiSecret())
-                                                                       .build();
+        final Tenant otherTenantWithDifferentStateMachine = createTenant(UUID.randomUUID().toString(), UUID.randomUUID().toString(), true);
 
         // Verify initial state
-        final TenantKeyValue emptyTenantKey = tenantApi.getPluginPaymentStateMachineConfig(PLUGIN_NAME, requestOptions);
+        final TenantKeyValue emptyTenantKey = tenantApi.getPluginPaymentStateMachineConfig(PLUGIN_NAME, requestOptionsForOriginalTenant);
         Assert.assertEquals(emptyTenantKey.getValues().size(), 0);
-        final TenantKeyValue emptyTenantKeyOtherTenant = tenantApi.getPluginPaymentStateMachineConfig(PLUGIN_NAME, requestOptionsOtherTenant);
+        final TenantKeyValue emptyTenantKeyOtherTenant = tenantApi.getPluginPaymentStateMachineConfig(PLUGIN_NAME, requestOptions);
         Assert.assertEquals(emptyTenantKeyOtherTenant.getValues().size(), 0);
 
+        callbackServlet.pushExpectedEvent(ExtBusEventType.TENANT_CONFIG_CHANGE);
         final String stateMachineConfig = getResourceBodyString("SimplePaymentStates.xml");
-        final TenantKeyValue tenantKey0 = tenantApi.uploadPluginPaymentStateMachineConfig(PLUGIN_NAME, stateMachineConfig, requestOptionsOtherTenant);
+        final TenantKeyValue tenantKey0 = tenantApi.uploadPluginPaymentStateMachineConfig(PLUGIN_NAME, stateMachineConfig, requestOptions);
+        callbackServlet.assertListenerStatus();
         Assert.assertEquals(tenantKey0.getKey(), TenantKV.TenantKey.PLUGIN_PAYMENT_STATE_MACHINE_.toString() + PLUGIN_NAME);
 
         // Verify only the other tenant has the new state machine
-        final TenantKeyValue emptyTenantKey1 = tenantApi.getPluginPaymentStateMachineConfig(PLUGIN_NAME, requestOptions);
+        final TenantKeyValue emptyTenantKey1 = tenantApi.getPluginPaymentStateMachineConfig(PLUGIN_NAME, requestOptionsForOriginalTenant);
         Assert.assertEquals(emptyTenantKey1.getValues().size(), 0);
-        final TenantKeyValue tenantKey1OtherTenant = tenantApi.getPluginPaymentStateMachineConfig(PLUGIN_NAME, requestOptionsOtherTenant);
+        final TenantKeyValue tenantKey1OtherTenant = tenantApi.getPluginPaymentStateMachineConfig(PLUGIN_NAME, requestOptions);
         Assert.assertEquals(tenantKey1OtherTenant.getKey(), TenantKV.TenantKey.PLUGIN_PAYMENT_STATE_MACHINE_.toString() + PLUGIN_NAME);
         Assert.assertEquals(tenantKey1OtherTenant.getValues().size(), 1);
 
         // Create an auth in both tenant
-        final Payment payment = createComboPaymentTransaction(requestOptions);
-        final Payment paymentOtherTenant = createComboPaymentTransaction(requestOptionsOtherTenant);
+        final Payment payment = createComboPaymentTransaction(requestOptionsForOriginalTenant);
+        final Payment paymentOtherTenant = createComboPaymentTransaction(requestOptions);
 
         // Void in the first tenant (allowed by the default state machine)
-        paymentApi.voidPayment(payment.getPaymentId(), new PaymentTransaction(), NULL_PLUGIN_NAMES, NULL_PLUGIN_PROPERTIES, requestOptions);
-        final Payment voidPayment = paymentApi.getPayment(payment.getPaymentId(), NULL_PLUGIN_PROPERTIES, requestOptions);
+        callbackServlet.pushExpectedEvent(ExtBusEventType.PAYMENT_SUCCESS);
+        paymentApi.voidPayment(payment.getPaymentId(), new PaymentTransaction(), NULL_PLUGIN_NAMES, NULL_PLUGIN_PROPERTIES, requestOptionsForOriginalTenant);
+        callbackServlet.assertListenerStatus();
+        final Payment voidPayment = paymentApi.getPayment(payment.getPaymentId(), NULL_PLUGIN_PROPERTIES, requestOptionsForOriginalTenant);
         Assert.assertEquals(voidPayment.getTransactions().get(0).getStatus(), TransactionStatus.SUCCESS);
         Assert.assertEquals(voidPayment.getTransactions().get(1).getStatus(), TransactionStatus.SUCCESS);
 
         // Void in the other tenant (disallowed)
         try {
-            paymentApi.voidPayment(paymentOtherTenant.getPaymentId(), new PaymentTransaction(), NULL_PLUGIN_NAMES, NULL_PLUGIN_PROPERTIES, requestOptionsOtherTenant);
+            paymentApi.voidPayment(paymentOtherTenant.getPaymentId(), new PaymentTransaction(), NULL_PLUGIN_NAMES, NULL_PLUGIN_PROPERTIES, requestOptions);
             Assert.fail();
         } catch (final KillBillClientException e) {
             Assert.assertEquals((int) e.getBillingException().getCode(), ErrorCode.PAYMENT_INVALID_OPERATION.getCode());
         }
+        callbackServlet.assertListenerStatus();
 
         // Remove the custom state machine
-        tenantApi.deletePluginPaymentStateMachineConfig(PLUGIN_NAME, requestOptionsOtherTenant);
-        final TenantKeyValue tenantKey2 = tenantApi.getPluginPaymentStateMachineConfig(PLUGIN_NAME, requestOptionsOtherTenant);
+        callbackServlet.pushExpectedEvent(ExtBusEventType.TENANT_CONFIG_DELETION);
+        tenantApi.deletePluginPaymentStateMachineConfig(PLUGIN_NAME, requestOptions);
+        final TenantKeyValue tenantKey2 = tenantApi.getPluginPaymentStateMachineConfig(PLUGIN_NAME, requestOptions);
+        callbackServlet.assertListenerStatus();
         Assert.assertEquals(tenantKey2.getKey(), TenantKV.TenantKey.PLUGIN_PAYMENT_STATE_MACHINE_.toString() + PLUGIN_NAME);
         Assert.assertEquals(tenantKey2.getValues().size(), 0);
 
@@ -127,8 +132,10 @@ public class TestTenantKV extends TestJaxrsBase {
                       public Boolean call() throws Exception {
                           // The void should now go through
                           try {
-                              paymentApi.voidPayment(paymentOtherTenant.getPaymentId(), new PaymentTransaction(), NULL_PLUGIN_NAMES, NULL_PLUGIN_PROPERTIES, requestOptionsOtherTenant);
-                              final Payment voidPaymentOtherTenant2 = paymentApi.getPayment(paymentOtherTenant.getPaymentId(), NULL_PLUGIN_PROPERTIES, requestOptionsOtherTenant);
+                              callbackServlet.pushExpectedEvent(ExtBusEventType.PAYMENT_SUCCESS);
+                              paymentApi.voidPayment(paymentOtherTenant.getPaymentId(), new PaymentTransaction(), NULL_PLUGIN_NAMES, NULL_PLUGIN_PROPERTIES, requestOptions);
+                              final Payment voidPaymentOtherTenant2 = paymentApi.getPayment(paymentOtherTenant.getPaymentId(), NULL_PLUGIN_PROPERTIES, requestOptions);
+                              callbackServlet.assertListenerStatus();
                               voidPaymentOtherTenant2Ref.set(voidPaymentOtherTenant2);
                               return voidPaymentOtherTenant2 != null;
                           } catch (final KillBillClientException e) {
@@ -159,8 +166,10 @@ public class TestTenantKV extends TestJaxrsBase {
         authTransactionJson.setTransactionExternalKey(authTransactionExternalKey);
         authTransactionJson.setTransactionType(TransactionType.AUTHORIZE);
 
+        callbackServlet.pushExpectedEvents(ExtBusEventType.ACCOUNT_CREATION, ExtBusEventType.ACCOUNT_CHANGE, ExtBusEventType.PAYMENT_SUCCESS);
         final ComboPaymentTransaction comboAuthorization = new ComboPaymentTransaction(accountJson, paymentMethodJson, authTransactionJson, ImmutableList.<PluginProperty>of(), ImmutableList.<PluginProperty>of(), null);
         final Payment payment = paymentApi.createComboPayment(comboAuthorization, NULL_PLUGIN_NAMES, requestOptions);
+        callbackServlet.assertListenerStatus();
         Assert.assertEquals(payment.getTransactions().get(0).getStatus(), TransactionStatus.SUCCESS);
 
         return payment;
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/server/security/TestKillbillJdbcTenantRealm.java b/profiles/killbill/src/test/java/org/killbill/billing/server/security/TestKillbillJdbcTenantRealm.java
index 292cd25..937c529 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/server/security/TestKillbillJdbcTenantRealm.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/server/security/TestKillbillJdbcTenantRealm.java
@@ -48,6 +48,10 @@ public class TestKillbillJdbcTenantRealm extends TestJaxrsBase {
     @Override
     @BeforeMethod(groups = "slow")
     public void beforeMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         super.beforeMethod();
 
         // Create the tenant
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/server/security/TestTenantFilter.java b/profiles/killbill/src/test/java/org/killbill/billing/server/security/TestTenantFilter.java
index 900364f..88bb985 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/server/security/TestTenantFilter.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/server/security/TestTenantFilter.java
@@ -1,7 +1,7 @@
 /*
  * Copyright 2010-2013 Ning, Inc.
- * Copyright 2014 Groupon, Inc
- * Copyright 2014 The Billing Project, LLC
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -32,6 +32,10 @@ public class TestTenantFilter extends TestJaxrsBase {
 
     @AfterMethod(groups = "slow")
     public void tearDown() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         // Default credentials
         loginTenant(DEFAULT_API_KEY, DEFAULT_API_SECRET);
     }
@@ -46,15 +50,10 @@ public class TestTenantFilter extends TestJaxrsBase {
         } catch (final KillBillClientException e) {
             Assert.assertEquals(e.getResponse().getStatusCode(), Status.UNAUTHORIZED.getStatusCode());
         }
+        callbackServlet.assertListenerStatus();
 
         // Create the tenant
-        final String apiKeyTenant1 = "pierre";
-        final String apiSecretTenant1 = "pierreIsFr3nch";
-        loginTenant(apiKeyTenant1, apiSecretTenant1);
-        final Tenant tenant1 = new Tenant();
-        tenant1.setApiKey(apiKeyTenant1);
-        tenant1.setApiSecret(apiSecretTenant1);
-        tenantApi.createTenant(tenant1, requestOptions);
+        final Tenant tenant1 = createTenant("pierre", "pierreIsFr3nch", true);
 
         final Account account1 = createAccount();
         Assert.assertEquals(accountApi.getAccountByKey(account1.getExternalKey(), requestOptions), account1);
@@ -62,13 +61,7 @@ public class TestTenantFilter extends TestJaxrsBase {
         logoutTenant();
 
         // Create another tenant
-        final String apiKeyTenant2 = "stephane";
-        final String apiSecretTenant2 = "stephane1sAlsoFr3nch";
-        loginTenant(apiKeyTenant2, apiSecretTenant2);
-        final Tenant tenant2 = new Tenant();
-        tenant2.setApiKey(apiKeyTenant2);
-        tenant2.setApiSecret(apiSecretTenant2);
-        tenantApi.createTenant(tenant2, requestOptions);
+        createTenant("stephane", "stephane1sAlsoFr3nch", true);
 
         final Account account2 = createAccount();
         Assert.assertEquals(accountApi.getAccountByKey(account2.getExternalKey(), requestOptions), account2);
@@ -77,7 +70,7 @@ public class TestTenantFilter extends TestJaxrsBase {
         Assert.assertNull(accountApi.getAccountByKey(account1.getExternalKey(), requestOptions));
 
         // Same for tenant1 and account2
-        loginTenant(apiKeyTenant1, apiSecretTenant1);
+        loginTenant(tenant1.getApiKey(), tenant1.getApiSecret());
         Assert.assertNull(accountApi.getAccountByKey(account2.getExternalKey(), requestOptions));
     }
 }
diff --git a/profiles/killpay/pom.xml b/profiles/killpay/pom.xml
index 048b66c..b4ecd51 100644
--- a/profiles/killpay/pom.xml
+++ b/profiles/killpay/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <artifactId>killbill-profiles</artifactId>
         <groupId>org.kill-bill.billing</groupId>
-        <version>0.19.13-SNAPSHOT</version>
+        <version>0.19.14-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>killbill-profiles-killpay</artifactId>

profiles/pom.xml 2(+1 -1)

diff --git a/profiles/pom.xml b/profiles/pom.xml
index b8b4fb4..d4a6789 100644
--- a/profiles/pom.xml
+++ b/profiles/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <artifactId>killbill</artifactId>
         <groupId>org.kill-bill.billing</groupId>
-        <version>0.19.13-SNAPSHOT</version>
+        <version>0.19.14-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>killbill-profiles</artifactId>
diff --git a/subscription/pom.xml b/subscription/pom.xml
index 1e4001f..9260907 100644
--- a/subscription/pom.xml
+++ b/subscription/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <artifactId>killbill</artifactId>
         <groupId>org.kill-bill.billing</groupId>
-        <version>0.19.13-SNAPSHOT</version>
+        <version>0.19.14-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>killbill-subscription</artifactId>

tenant/pom.xml 2(+1 -1)

diff --git a/tenant/pom.xml b/tenant/pom.xml
index 710dd0e..173271b 100644
--- a/tenant/pom.xml
+++ b/tenant/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <artifactId>killbill</artifactId>
         <groupId>org.kill-bill.billing</groupId>
-        <version>0.19.13-SNAPSHOT</version>
+        <version>0.19.14-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>killbill-tenant</artifactId>

usage/pom.xml 2(+1 -1)

diff --git a/usage/pom.xml b/usage/pom.xml
index 056478a..8166a7e 100644
--- a/usage/pom.xml
+++ b/usage/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <artifactId>killbill</artifactId>
         <groupId>org.kill-bill.billing</groupId>
-        <version>0.19.13-SNAPSHOT</version>
+        <version>0.19.14-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>killbill-usage</artifactId>

util/pom.xml 2(+1 -1)

diff --git a/util/pom.xml b/util/pom.xml
index 09fe09a..24ba132 100644
--- a/util/pom.xml
+++ b/util/pom.xml
@@ -21,7 +21,7 @@
     <parent>
         <artifactId>killbill</artifactId>
         <groupId>org.kill-bill.billing</groupId>
-        <version>0.19.13-SNAPSHOT</version>
+        <version>0.19.14-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>killbill-util</artifactId>
diff --git a/util/src/test/java/org/killbill/billing/api/TestApiListener.java b/util/src/test/java/org/killbill/billing/api/TestApiListener.java
index d730885..83cb4c2 100644
--- a/util/src/test/java/org/killbill/billing/api/TestApiListener.java
+++ b/util/src/test/java/org/killbill/billing/api/TestApiListener.java
@@ -294,7 +294,7 @@ public class TestApiListener {
         }
     }
 
-    public boolean isCompleted(final long timeout) {
+    private boolean isCompleted(final long timeout) {
         synchronized (this) {
             long waitTimeMs = timeout;
             do {
diff --git a/util/src/test/java/org/killbill/billing/GuicyKillbillTestSuite.java b/util/src/test/java/org/killbill/billing/GuicyKillbillTestSuite.java
index 135f7aa..eb0c634 100644
--- a/util/src/test/java/org/killbill/billing/GuicyKillbillTestSuite.java
+++ b/util/src/test/java/org/killbill/billing/GuicyKillbillTestSuite.java
@@ -42,7 +42,6 @@ import org.mockito.stubbing.Answer;
 import org.skife.config.ConfigSource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.testng.Assert;
 import org.testng.IHookCallBack;
 import org.testng.IHookable;
 import org.testng.ITestResult;
@@ -149,7 +148,7 @@ public class GuicyKillbillTestSuite implements IHookable {
 
     @BeforeMethod(alwaysRun = true)
     public void beforeMethodAlwaysRun(final Method method) throws Exception {
-        if (AbortAfterFirstFailureListener.hasFailures()) {
+        if (hasFailed()) {
             return;
         }
 
@@ -198,7 +197,7 @@ public class GuicyKillbillTestSuite implements IHookable {
 
     @AfterMethod(alwaysRun = true)
     public void afterMethodAlwaysRun(final Method method, final ITestResult result) throws Exception {
-        if (AbortAfterFirstFailureListener.hasFailures()) {
+        if (hasFailed()) {
             return;
         }
 
diff --git a/util/src/test/java/org/killbill/billing/KillbillTestSuite.java b/util/src/test/java/org/killbill/billing/KillbillTestSuite.java
index 7b4bdf7..8591c1f 100644
--- a/util/src/test/java/org/killbill/billing/KillbillTestSuite.java
+++ b/util/src/test/java/org/killbill/billing/KillbillTestSuite.java
@@ -1,7 +1,7 @@
 /*
  * Copyright 2010-2013 Ning, Inc.
- * Copyright 2014-2016 Groupon, Inc
- * Copyright 2014-2016 The Billing Project, LLC
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -20,6 +20,7 @@ package org.killbill.billing;
 
 import java.lang.reflect.Method;
 
+import org.killbill.billing.api.AbortAfterFirstFailureListener;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testng.ITestResult;
@@ -35,6 +36,10 @@ public class KillbillTestSuite {
 
     @BeforeMethod(alwaysRun = true)
     public void startTestSuite(final Method method) throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         log.info("***************************************************************************************************");
         log.info("*** Starting test {}:{}", method.getDeclaringClass().getName(), method.getName());
         log.info("***************************************************************************************************");
@@ -42,6 +47,10 @@ public class KillbillTestSuite {
 
     @AfterMethod(alwaysRun = true)
     public void endTestSuite(final Method method, final ITestResult result) throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         log.info("***************************************************************************************************");
         log.info("***   Ending test {}:{} {} ({} s.)", new Object[]{method.getDeclaringClass().getName(), method.getName(),
                                                                     result.isSuccess() ? "SUCCESS" : "!!! FAILURE !!!",
@@ -53,6 +62,6 @@ public class KillbillTestSuite {
     }
 
     public boolean hasFailed() {
-        return hasFailed;
+        return hasFailed || AbortAfterFirstFailureListener.hasFailures();
     }
 }