killbill-aplcache

server: first pass at synchronizing tests with push notifications Signed-off-by:

5/1/2018 1:46:55 PM

Changes

Details

diff --git a/jaxrs/src/main/java/org/killbill/billing/jaxrs/json/NotificationJson.java b/jaxrs/src/main/java/org/killbill/billing/jaxrs/json/NotificationJson.java
index 40e7c5c..241692a 100644
--- a/jaxrs/src/main/java/org/killbill/billing/jaxrs/json/NotificationJson.java
+++ b/jaxrs/src/main/java/org/killbill/billing/jaxrs/json/NotificationJson.java
@@ -1,7 +1,9 @@
 /*
  * Copyright 2010-2013 Ning, Inc.
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
  *
- * Ning licenses this file to you under the Apache License, version 2.0
+ * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
  * License.  You may obtain a copy of the License at:
  *
@@ -78,4 +80,52 @@ public class NotificationJson {
     public String getMetaData() {
         return metaData;
     }
+
+    @Override
+    public String toString() {
+        final StringBuilder sb = new StringBuilder("NotificationJson{");
+        sb.append("eventType='").append(eventType).append('\'');
+        sb.append(", accountId=").append(accountId);
+        sb.append(", objectType='").append(objectType).append('\'');
+        sb.append(", objectId=").append(objectId);
+        sb.append(", metaData='").append(metaData).append('\'');
+        sb.append('}');
+        return sb.toString();
+    }
+
+    @Override
+    public boolean equals(final Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        final NotificationJson that = (NotificationJson) o;
+
+        if (eventType != null ? !eventType.equals(that.eventType) : that.eventType != null) {
+            return false;
+        }
+        if (accountId != null ? !accountId.equals(that.accountId) : that.accountId != null) {
+            return false;
+        }
+        if (objectType != null ? !objectType.equals(that.objectType) : that.objectType != null) {
+            return false;
+        }
+        if (objectId != null ? !objectId.equals(that.objectId) : that.objectId != null) {
+            return false;
+        }
+        return metaData != null ? metaData.equals(that.metaData) : that.metaData == null;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = eventType != null ? eventType.hashCode() : 0;
+        result = 31 * result + (accountId != null ? accountId.hashCode() : 0);
+        result = 31 * result + (objectType != null ? objectType.hashCode() : 0);
+        result = 31 * result + (objectId != null ? objectId.hashCode() : 0);
+        result = 31 * result + (metaData != null ? metaData.hashCode() : 0);
+        return result;
+    }
 }
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/CallbackServer.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/CallbackServer.java
new file mode 100644
index 0000000..15654a2
--- /dev/null
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/CallbackServer.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
+ *
+ * The Billing Project licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at:
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.killbill.billing.jaxrs;
+
+import javax.servlet.Servlet;
+
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+
+public class CallbackServer {
+
+    private static final int SERVER_PORT = 8087;
+    private static final String CALLBACK_ENDPOINT = "/callmeback";
+
+    private final Server server;
+    private final String callbackEndpoint;
+    private final Servlet servlet;
+
+    public CallbackServer(final Servlet servlet) {
+        this.callbackEndpoint = CALLBACK_ENDPOINT;
+        this.servlet = servlet;
+        this.server = new Server(SERVER_PORT);
+    }
+
+    public void startServer() throws Exception {
+        final ServletContextHandler context = new ServletContextHandler();
+        context.setContextPath("/");
+        server.setHandler(context);
+        context.addServlet(new ServletHolder(servlet), callbackEndpoint);
+        server.start();
+    }
+
+    public void stopServer() throws Exception {
+        server.stop();
+    }
+
+    public static String getServletEndpoint() {
+        return "http://127.0.0.1:" + SERVER_PORT + CALLBACK_ENDPOINT;
+    }
+}
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/CallbackServlet.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/CallbackServlet.java
new file mode 100644
index 0000000..9f47378
--- /dev/null
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/CallbackServlet.java
@@ -0,0 +1,159 @@
+/*
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
+ *
+ * The Billing Project licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at:
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.killbill.billing.jaxrs;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Stack;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.killbill.billing.jaxrs.json.NotificationJson;
+import org.killbill.billing.notification.plugin.api.ExtBusEventType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Joiner;
+import com.google.common.io.CharStreams;
+
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+public class CallbackServlet extends HttpServlet {
+
+    private static final Logger log = LoggerFactory.getLogger(CallbackServlet.class);
+
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+    private static final Joiner SPACE_JOINER = Joiner.on(" ");
+    private static final long DELAY = 60000;
+
+    // Cross tenants (for now)
+    private final Collection<ExtBusEventType> nextExpectedEvent = new Stack<ExtBusEventType>();
+
+    private boolean isListenerFailed = false;
+    private String listenerFailedMsg;
+    private boolean completed = true;
+
+    @Override
+    protected void doPost(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException {
+        final String body = CharStreams.toString(new InputStreamReader(request.getInputStream(), "UTF-8"));
+        response.setStatus(HttpServletResponse.SC_OK);
+
+        final NotificationJson notification = objectMapper.readValue(body, NotificationJson.class);
+        log.info("Got notification: {}", notification);
+        assertEqualsNicely(notification.getEventType() == null ? null : ExtBusEventType.valueOf(notification.getEventType()));
+        notifyIfStackEmpty();
+    }
+
+    public void assertListenerStatus() {
+        // Bail early
+        if (isListenerFailed) {
+            log.error(listenerFailedMsg);
+            Assert.fail(listenerFailedMsg);
+        }
+
+        try {
+            assertTrue(isCompleted(DELAY));
+        } catch (final Exception e) {
+            fail("assertListenerStatus didn't complete", e);
+        }
+
+        if (isListenerFailed) {
+            log.error(listenerFailedMsg);
+            Assert.fail(listenerFailedMsg);
+        }
+    }
+
+    public synchronized void reset() {
+        nextExpectedEvent.clear();
+        completed = true;
+
+        isListenerFailed = false;
+        listenerFailedMsg = null;
+    }
+
+    public void pushExpectedEvents(final ExtBusEventType... events) {
+        for (final ExtBusEventType event : events) {
+            pushExpectedEvent(event);
+        }
+    }
+
+    public synchronized void pushExpectedEvent(final ExtBusEventType next) {
+        nextExpectedEvent.add(next);
+        log.info("Stacking expected event {}, got [{}]", next, SPACE_JOINER.join(nextExpectedEvent));
+        completed = false;
+    }
+
+    private synchronized boolean isCompleted(final long timeout) {
+        long waitTimeMs = timeout;
+        do {
+            try {
+                final long before = System.currentTimeMillis();
+                wait(100);
+                final long after = System.currentTimeMillis();
+                waitTimeMs -= (after - before);
+            } catch (final Exception ignore) {
+                return false;
+            }
+        } while (waitTimeMs > 0 && !completed);
+
+        if (!completed) {
+            log.error("CallbackServlet did not complete in " + timeout + " ms, remaining events are " + SPACE_JOINER.join(nextExpectedEvent));
+        }
+        return completed;
+    }
+
+    private synchronized void notifyIfStackEmpty() {
+        if (nextExpectedEvent.isEmpty()) {
+            log.debug("CallbackServlet EMPTY");
+            completed = true;
+            notify();
+        }
+    }
+
+    private synchronized void assertEqualsNicely(final ExtBusEventType received) {
+        boolean foundIt = false;
+        final Iterator<ExtBusEventType> it = nextExpectedEvent.iterator();
+        while (it.hasNext()) {
+            final ExtBusEventType ev = it.next();
+            if (ev == received) {
+                it.remove();
+                foundIt = true;
+                log.info("Found expected event: {}", received);
+                break;
+            }
+        }
+        if (!foundIt) {
+            final String errorMsg = "CallbackServlet: received unexpected event " + received + "; remaining expected events [" + SPACE_JOINER.join(nextExpectedEvent) + "]";
+            log.error(errorMsg);
+            failed(errorMsg);
+        }
+    }
+
+    private void failed(final String msg) {
+        this.isListenerFailed = true;
+        this.listenerFailedMsg = msg;
+    }
+}
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/KillbillClient.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/KillbillClient.java
index ffdbd07..c539938 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/KillbillClient.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/KillbillClient.java
@@ -1,7 +1,7 @@
 /*
  * Copyright 2010-2013 Ning, Inc.
- * Copyright 2014 Groupon, Inc
- * Copyright 2014 The Billing Project, LLC
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -38,6 +38,7 @@ import org.killbill.billing.client.model.PaymentMethodPluginDetail;
 import org.killbill.billing.client.model.PluginProperty;
 import org.killbill.billing.client.model.Subscription;
 import org.killbill.billing.client.model.Tags;
+import org.killbill.billing.notification.plugin.api.ExtBusEventType;
 import org.killbill.billing.payment.provider.ExternalPaymentProviderPlugin;
 import org.killbill.billing.util.UUIDs;
 import org.killbill.billing.util.tag.ControlTagType;
@@ -53,6 +54,9 @@ public abstract class KillbillClient extends GuicyKillbillTestSuiteWithEmbeddedD
 
     protected static final String DEFAULT_CURRENCY = "USD";
 
+    // static to be shared across test class instances (initialized once in @BeforeSuite)
+    protected static CallbackServlet callbackServlet;
+
     // Multi-Tenancy information, if enabled
     protected String DEFAULT_API_KEY = UUID.randomUUID().toString();
     protected String DEFAULT_API_SECRET = UUID.randomUUID().toString();
@@ -110,10 +114,12 @@ public abstract class KillbillClient extends GuicyKillbillTestSuiteWithEmbeddedD
     protected Account createAccountWithDefaultPaymentMethod(final String externalkey, @Nullable final List<PluginProperty> pmProperties) throws Exception {
         final Account input = createAccount();
 
+        callbackServlet.pushExpectedEvent(ExtBusEventType.ACCOUNT_CHANGE);
         final PaymentMethodPluginDetail info = new PaymentMethodPluginDetail();
         info.setProperties(pmProperties);
         final PaymentMethod paymentMethodJson = new PaymentMethod(null, externalkey, input.getAccountId(), true, PLUGIN_NAME, info);
         killBillClient.createPaymentMethod(paymentMethodJson, createdBy, reason, comment);
+        callbackServlet.assertListenerStatus();
         return killBillClient.getAccount(input.getExternalKey());
     }
 
@@ -126,10 +132,15 @@ public abstract class KillbillClient extends GuicyKillbillTestSuiteWithEmbeddedD
     }
 
     protected PaymentMethod createPaymentMethod(final Account input, final boolean isDefault) throws KillBillClientException {
+        if (isDefault) {
+            callbackServlet.pushExpectedEvent(ExtBusEventType.ACCOUNT_CHANGE);
+        }
         final PaymentMethodPluginDetail info = new PaymentMethodPluginDetail();
         final PaymentMethod paymentMethodJson = new PaymentMethod(null, UUIDs.randomUUID().toString(), input.getAccountId(),
                                                                   isDefault, ExternalPaymentProviderPlugin.PLUGIN_NAME, info);
-        return killBillClient.createPaymentMethod(paymentMethodJson, requestOptions);
+        final PaymentMethod paymentMethod = killBillClient.createPaymentMethod(paymentMethodJson, requestOptions);
+        callbackServlet.assertListenerStatus();
+        return paymentMethod;
     }
 
     protected Account createAccount() throws Exception {
@@ -137,12 +148,17 @@ public abstract class KillbillClient extends GuicyKillbillTestSuiteWithEmbeddedD
     }
 
     protected Account createAccount(final UUID parentAccountId) throws Exception {
+        callbackServlet.pushExpectedEvent(ExtBusEventType.ACCOUNT_CREATION);
         final Account input = getAccount(parentAccountId);
-        return killBillClient.createAccount(input, createdBy, reason, comment);
+        final Account account = killBillClient.createAccount(input, createdBy, reason, comment);
+        callbackServlet.assertListenerStatus();
+        return account;
     }
 
     protected Subscription createEntitlement(final UUID accountId, final String bundleExternalKey, final String productName,
                                              final ProductCategory productCategory, final BillingPeriod billingPeriod, final boolean waitCompletion) throws Exception {
+        callbackServlet.pushExpectedEvents(ExtBusEventType.ACCOUNT_CHANGE, ExtBusEventType.ENTITLEMENT_CREATION, ExtBusEventType.SUBSCRIPTION_CREATION, ExtBusEventType.SUBSCRIPTION_CREATION, ExtBusEventType.INVOICE_CREATION);
+
         final Subscription input = new Subscription();
         input.setAccountId(accountId);
         input.setExternalKey(bundleExternalKey);
@@ -151,19 +167,37 @@ public abstract class KillbillClient extends GuicyKillbillTestSuiteWithEmbeddedD
         input.setBillingPeriod(billingPeriod);
         input.setPriceList(PriceListSet.DEFAULT_PRICELIST_NAME);
 
-        return killBillClient.createSubscription(input, null, null,  waitCompletion ? DEFAULT_WAIT_COMPLETION_TIMEOUT_SEC : -1, false, requestOptions);
+        final Subscription subscription = killBillClient.createSubscription(input, null, null, waitCompletion ? DEFAULT_WAIT_COMPLETION_TIMEOUT_SEC : -1, false, requestOptions);
+        callbackServlet.assertListenerStatus();
+
+        return subscription;
     }
 
     protected Account createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoice() throws Exception {
+        return createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoice(true);
+    }
+
+    protected Account createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoice(final boolean paymentSuccess) throws Exception {
+        return createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoice("Shotgun", paymentSuccess);
+    }
+
+    protected Account createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoice(final String productName, final boolean paymentSuccess) throws Exception {
         final Account accountJson = createAccountWithDefaultPaymentMethod();
         assertNotNull(accountJson);
 
         // Add a bundle, subscription and move the clock to get the first invoice
-        final Subscription subscriptionJson = createEntitlement(accountJson.getAccountId(), UUID.randomUUID().toString(), "Shotgun",
+        final Subscription subscriptionJson = createEntitlement(accountJson.getAccountId(), UUID.randomUUID().toString(), productName,
                                                                 ProductCategory.BASE, BillingPeriod.MONTHLY, true);
         assertNotNull(subscriptionJson);
+
+        callbackServlet.pushExpectedEvents(ExtBusEventType.SUBSCRIPTION_PHASE, ExtBusEventType.INVOICE_CREATION);
+        if (paymentSuccess) {
+            callbackServlet.pushExpectedEvents(ExtBusEventType.INVOICE_PAYMENT_SUCCESS, ExtBusEventType.PAYMENT_SUCCESS);
+        } else {
+            callbackServlet.pushExpectedEvents(ExtBusEventType.INVOICE_PAYMENT_FAILED, ExtBusEventType.PAYMENT_FAILED);
+        }
         clock.addDays(32);
-        crappyWaitForLackOfProperSynchonization();
+        callbackServlet.assertListenerStatus();
 
         return accountJson;
     }
@@ -172,7 +206,9 @@ public abstract class KillbillClient extends GuicyKillbillTestSuiteWithEmbeddedD
         final Account accountJson = createAccountWithExternalPaymentMethod();
         assertNotNull(accountJson);
 
+        callbackServlet.pushExpectedEvent(ExtBusEventType.TAG_CREATION);
         final Tags accountTag = killBillClient.createAccountTag(accountJson.getAccountId(), ControlTagType.MANUAL_PAY.getId(), requestOptions);
+        callbackServlet.assertListenerStatus();
         assertNotNull(accountTag);
         assertEquals(accountTag.get(0).getTagDefinitionId(), ControlTagType.MANUAL_PAY.getId());
 
@@ -180,8 +216,10 @@ public abstract class KillbillClient extends GuicyKillbillTestSuiteWithEmbeddedD
         final Subscription subscriptionJson = createEntitlement(accountJson.getAccountId(), UUID.randomUUID().toString(), "Shotgun",
                                                                 ProductCategory.BASE, BillingPeriod.MONTHLY, true);
         assertNotNull(subscriptionJson);
+
+        callbackServlet.pushExpectedEvents(ExtBusEventType.SUBSCRIPTION_PHASE, ExtBusEventType.INVOICE_CREATION);
         clock.addDays(32);
-        crappyWaitForLackOfProperSynchonization();
+        callbackServlet.assertListenerStatus();
 
         return accountJson;
     }
@@ -208,10 +246,11 @@ public abstract class KillbillClient extends GuicyKillbillTestSuiteWithEmbeddedD
         final Subscription subscriptionJson = createEntitlement(accountJson.getAccountId(), UUID.randomUUID().toString(), "Shotgun",
                                                                 ProductCategory.BASE, BillingPeriod.MONTHLY, true);
         assertNotNull(subscriptionJson);
-        clock.addMonths(1);
-        crappyWaitForLackOfProperSynchonization();
 
         // No payment will be triggered as the account doesn't have a payment method
+        callbackServlet.pushExpectedEvent(ExtBusEventType.INVOICE_CREATION);
+        clock.addMonths(1);
+        callbackServlet.assertListenerStatus();
 
         return accountJson;
     }
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestAccount.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestAccount.java
index 66ff2e5..d479317 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestAccount.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestAccount.java
@@ -66,12 +66,20 @@ public class TestAccount extends TestJaxrsBase {
 
     @BeforeMethod(groups = "slow")
     public void beforeMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         super.beforeMethod();
         mockPaymentProviderPlugin = (MockPaymentProviderPlugin) registry.getServiceForName(PLUGIN_NAME);
     }
 
     @AfterMethod(groups = "slow")
     public void tearDown() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         mockPaymentProviderPlugin.clear();
     }
 
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCache.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCache.java
index ee38da4..83b74fe 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCache.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCache.java
@@ -19,20 +19,12 @@ package org.killbill.billing.jaxrs;
 
 import java.util.UUID;
 
-import org.joda.time.LocalDate;
 import org.killbill.automaton.StateMachineConfig;
 import org.killbill.billing.account.api.ImmutableAccountData;
-import org.killbill.billing.api.FlakyRetryAnalyzer;
-import org.killbill.billing.catalog.api.BillingPeriod;
 import org.killbill.billing.catalog.api.Catalog;
-import org.killbill.billing.catalog.api.PriceListSet;
-import org.killbill.billing.catalog.api.ProductCategory;
-import org.killbill.billing.client.RequestOptions;
 import org.killbill.billing.client.model.Account;
-import org.killbill.billing.client.model.PaymentMethod;
-import org.killbill.billing.client.model.PaymentMethodPluginDetail;
-import org.killbill.billing.client.model.Subscription;
 import org.killbill.billing.client.model.Tenant;
+import org.killbill.billing.notification.plugin.api.ExtBusEventType;
 import org.killbill.billing.overdue.api.OverdueConfig;
 import org.killbill.billing.util.cache.Cachable.CacheType;
 import org.killbill.billing.util.cache.CacheController;
@@ -102,32 +94,18 @@ public class TestCache extends TestJaxrsBase {
         Assert.assertFalse(accountBcdCache.isKeyInCache(input.getAccountId()));
     }
 
-    // Flaky, see https://github.com/killbill/killbill/issues/860
-    @Test(groups = "slow", description = "Can Invalidate (clear) all Tenant Caches for current Tenant", retryAnalyzer = FlakyRetryAnalyzer.class)
+    @Test(groups = "slow", description = "Can Invalidate (clear) all Tenant Caches for current Tenant")
     public void testInvalidateCacheByTenant() throws Exception {
         // creating a new Tenant for this test
-        final String testApiKey = "testApiKey";
-        final String testApiSecret = "testApiSecret";
-        final Tenant tenant = new Tenant();
-        tenant.setApiKey(testApiKey);
-        tenant.setApiSecret(testApiSecret);
-        loginTenant(testApiKey, testApiSecret);
-        Tenant currentTenant = killBillClient.createTenant(tenant, false, requestOptions);
-
-        // using custom RequestOptions with the new Tenant created before
-        RequestOptions inputOptions = RequestOptions.builder()
-                                                    .withCreatedBy(createdBy)
-                                                    .withReason(reason)
-                                                    .withComment(comment)
-                                                    .withTenantApiKey(currentTenant.getApiKey())
-                                                    .withTenantApiSecret(currentTenant.getApiSecret())
-                                                    .build();
+        final Tenant currentTenant = createTenant("testApiKey", "testApiSecret", false);
 
         // Uploading the test catalog using the new Tenant created before
-        killBillClient.uploadXMLCatalog(Resources.getResource("SpyCarAdvanced.xml").getPath(), inputOptions);
+        callbackServlet.pushExpectedEvent(ExtBusEventType.TENANT_CONFIG_CHANGE);
+        killBillClient.uploadXMLCatalog(Resources.getResource("SpyCarAdvanced.xml").getPath(), requestOptions);
+        callbackServlet.assertListenerStatus();
 
         // creating an Account with PaymentMethod and a Subscription
-        createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoiceWithInputOptions(inputOptions);
+        createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoice("Sports", true);
 
         // get all caches per tenant level
         final CacheController<String, Long> tenantRecordIdCache = cacheControllerDispatcher.getCacheController(CacheType.TENANT_RECORD_ID);
@@ -143,19 +121,19 @@ public class TestCache extends TestJaxrsBase {
         final Long tenantRecordId = tenantRecordIdCache.get(currentTenant.getTenantId().toString(), null);
 
         assertTrue(hasKeysByTenantRecordId(tenantPaymentStateMachineConfigCache, tenantRecordId.toString()));
-        assertTrue(tenantCache.isKeyInCache(testApiKey));
+        assertTrue(tenantCache.isKeyInCache(currentTenant.getApiKey()));
         assertTrue(hasKeysByTenantRecordId(tenantKvCache, tenantRecordId.toString()));
         assertTrue(tenantConfigCache.isKeyInCache(tenantRecordId));
         assertTrue(tenantOverdueConfigCache.isKeyInCache(tenantRecordId));
         assertTrue(tenantCatalogCache.isKeyInCache(tenantRecordId));
 
         // invalidate caches per tenant level
-        killBillClient.invalidateCacheByTenant(inputOptions);
+        killBillClient.invalidateCacheByTenant(requestOptions);
 
         // verify that now the caches don't have the previous values
         assertFalse(tenantRecordIdCache.isKeyInCache(currentTenant.getTenantId().toString()));
         assertFalse(hasKeysByTenantRecordId(tenantPaymentStateMachineConfigCache, tenantRecordId.toString()));
-        assertFalse(tenantCache.isKeyInCache(testApiKey));
+        assertFalse(tenantCache.isKeyInCache(currentTenant.getApiKey()));
         assertFalse(hasKeysByTenantRecordId(tenantKvCache, tenantRecordId.toString()));
         assertFalse(tenantConfigCache.isKeyInCache(tenantRecordId));
         assertFalse(tenantOverdueConfigCache.isKeyInCache(tenantRecordId));
@@ -170,29 +148,4 @@ public class TestCache extends TestJaxrsBase {
         }
         return false;
     }
-
-    private void createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoiceWithInputOptions(final RequestOptions inputOptions) throws Exception {
-        Account account = killBillClient.createAccount(getAccount(), inputOptions);
-
-        final PaymentMethodPluginDetail info = new PaymentMethodPluginDetail();
-        info.setProperties(null);
-        final PaymentMethod paymentMethodJson = new PaymentMethod(null, UUID.randomUUID().toString(), account.getAccountId(), true, PLUGIN_NAME, info);
-        killBillClient.createPaymentMethod(paymentMethodJson, inputOptions);
-
-        final Subscription subscription = new Subscription();
-        subscription.setAccountId(account.getAccountId());
-        subscription.setExternalKey(UUID.randomUUID().toString());
-        subscription.setProductName("Sports");
-        subscription.setProductCategory(ProductCategory.BASE);
-        subscription.setBillingPeriod(BillingPeriod.MONTHLY);
-        subscription.setPriceList(PriceListSet.DEFAULT_PRICELIST_NAME);
-
-        clock.resetDeltaFromReality();
-        clock.setDay(new LocalDate(2013, 3, 1));
-        final Subscription subscriptionJson = killBillClient.createSubscription(subscription, clock.getUTCToday(), DEFAULT_WAIT_COMPLETION_TIMEOUT_SEC, inputOptions);
-
-        assertNotNull(subscriptionJson);
-        clock.addDays(32);
-        crappyWaitForLackOfProperSynchonization();
-    }
 }
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCatalog.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCatalog.java
index 14563ed..cbb6486 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCatalog.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCatalog.java
@@ -1,7 +1,7 @@
 /*
  * Copyright 2010-2013 Ning, Inc.
- * Copyright 2014-2017 Groupon, Inc
- * Copyright 2014-2017 The Billing Project, LLC
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -31,21 +31,16 @@ import org.killbill.billing.catalog.api.Currency;
 import org.killbill.billing.catalog.api.ProductCategory;
 import org.killbill.billing.catalog.api.TimeUnit;
 import org.killbill.billing.client.KillBillClientException;
-import org.killbill.billing.client.KillBillHttpClient;
-import org.killbill.billing.client.RequestOptions;
 import org.killbill.billing.client.model.Catalog;
 import org.killbill.billing.client.model.Plan;
 import org.killbill.billing.client.model.PlanDetail;
 import org.killbill.billing.client.model.Product;
 import org.killbill.billing.client.model.SimplePlan;
-import org.killbill.billing.client.model.Tenant;
 import org.killbill.billing.client.model.Usage;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
-import com.google.common.collect.HashMultimap;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Multimap;
 import com.google.common.io.Resources;
 
 public class TestCatalog extends TestJaxrsBase {
@@ -187,22 +182,14 @@ public class TestCatalog extends TestJaxrsBase {
     @Test(groups = "slow", description = "Upload and retrieve a per plugin payment state machine config")
     public void testAddSimplePlanWithoutKBDefault() throws Exception {
         // Create another tenant initialized with no default catalog,...
-        final Tenant otherTenantNoKBDefault = new Tenant();
-        otherTenantNoKBDefault.setApiKey(UUID.randomUUID().toString());
-        otherTenantNoKBDefault.setApiSecret(UUID.randomUUID().toString());
+        createTenant(UUID.randomUUID().toString(), UUID.randomUUID().toString(), false);
 
-        killBillClient.createTenant(otherTenantNoKBDefault, false, requestOptions);
-
-        final RequestOptions requestOptionsOtherTenant = requestOptions.extend()
-                                                                       .withTenantApiKey(otherTenantNoKBDefault.getApiKey())
-                                                                       .withTenantApiSecret(otherTenantNoKBDefault.getApiSecret())
-                                                                       .build();
         // Verify the template catalog is not returned
-        List<Catalog> catalogsJson = killBillClient.getJSONCatalog(requestOptionsOtherTenant);
+        List<Catalog> catalogsJson = killBillClient.getJSONCatalog(requestOptions);
         Assert.assertEquals(catalogsJson.size(), 0);
 
-        killBillClient.addSimplePan(new SimplePlan("foo-monthly", "Foo", ProductCategory.BASE, Currency.USD, BigDecimal.TEN, BillingPeriod.MONTHLY, 0, TimeUnit.UNLIMITED, ImmutableList.<String>of()), requestOptionsOtherTenant);
-        catalogsJson = killBillClient.getJSONCatalog(requestOptionsOtherTenant);
+        killBillClient.addSimplePan(new SimplePlan("foo-monthly", "Foo", ProductCategory.BASE, Currency.USD, BigDecimal.TEN, BillingPeriod.MONTHLY, 0, TimeUnit.UNLIMITED, ImmutableList.<String>of()), requestOptions);
+        catalogsJson = killBillClient.getJSONCatalog(requestOptions);
         Assert.assertEquals(catalogsJson.size(),1);
         Assert.assertEquals(catalogsJson.get(0).getProducts().size(),1);
         Assert.assertEquals(catalogsJson.get(0).getProducts().get(0).getName(),"Foo");
@@ -212,9 +199,9 @@ public class TestCatalog extends TestJaxrsBase {
         Assert.assertEquals(catalogsJson.get(0).getPriceLists().get(0).getPlans().get(0), "foo-monthly");
 
 
-        killBillClient.addSimplePan(new SimplePlan("foo-annual", "Foo", ProductCategory.BASE, Currency.USD, new BigDecimal("100.00"), BillingPeriod.ANNUAL, 0, TimeUnit.UNLIMITED, ImmutableList.<String>of()), requestOptionsOtherTenant);
+        killBillClient.addSimplePan(new SimplePlan("foo-annual", "Foo", ProductCategory.BASE, Currency.USD, new BigDecimal("100.00"), BillingPeriod.ANNUAL, 0, TimeUnit.UNLIMITED, ImmutableList.<String>of()), requestOptions);
 
-        catalogsJson = killBillClient.getJSONCatalog(requestOptionsOtherTenant);
+        catalogsJson = killBillClient.getJSONCatalog(requestOptions);
         Assert.assertEquals(catalogsJson.size(),1);
         Assert.assertEquals(catalogsJson.get(0).getProducts().size(),1);
         Assert.assertEquals(catalogsJson.get(0).getProducts().get(0).getName(),"Foo");
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCredit.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCredit.java
index c7562dd..dddb0dc 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCredit.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestCredit.java
@@ -39,6 +39,10 @@ public class TestCredit extends TestJaxrsBase {
 
     @BeforeMethod(groups = "slow")
     public void setUp() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         accountJson = createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoice();
     }
 
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestInvoicePayment.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestInvoicePayment.java
index 14b0023..787558f 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestInvoicePayment.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestInvoicePayment.java
@@ -66,6 +66,10 @@ public class TestInvoicePayment extends TestJaxrsBase {
 
     @BeforeMethod(groups = "slow")
     public void beforeMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         super.beforeMethod();
         mockPaymentProviderPlugin = (MockPaymentProviderPlugin) registry.getServiceForName(PLUGIN_NAME);
     }
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestJaxrsBase.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestJaxrsBase.java
index 91d4f61..a464a62 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestJaxrsBase.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestJaxrsBase.java
@@ -40,6 +40,7 @@ import org.killbill.billing.GuicyKillbillTestWithEmbeddedDBModule;
 import org.killbill.billing.api.TestApiListener;
 import org.killbill.billing.beatrix.integration.db.TestDBRouterAPI;
 import org.killbill.billing.client.KillBillClient;
+import org.killbill.billing.client.KillBillClientException;
 import org.killbill.billing.client.KillBillHttpClient;
 import org.killbill.billing.client.model.Payment;
 import org.killbill.billing.client.model.PaymentTransaction;
@@ -49,6 +50,7 @@ import org.killbill.billing.jaxrs.resources.TestDBRouterResource;
 import org.killbill.billing.jetty.HttpServer;
 import org.killbill.billing.jetty.HttpServerConfig;
 import org.killbill.billing.lifecycle.glue.BusModule;
+import org.killbill.billing.notification.plugin.api.ExtBusEventType;
 import org.killbill.billing.osgi.api.OSGIServiceRegistration;
 import org.killbill.billing.payment.glue.PaymentModule;
 import org.killbill.billing.payment.provider.MockPaymentProviderPluginModule;
@@ -128,6 +130,7 @@ public class TestJaxrsBase extends KillbillClient {
 
     protected HttpServerConfig config;
     private HttpServer server;
+    private CallbackServer callbackServer;
 
     @Override
     protected KillbillConfigSource getConfigSource() {
@@ -217,6 +220,10 @@ public class TestJaxrsBase extends KillbillClient {
 
     @BeforeMethod(groups = "slow")
     public void beforeMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         super.beforeMethod();
 
         // Because we truncate the tables, the database record_id auto_increment will be reset
@@ -226,23 +233,57 @@ public class TestJaxrsBase extends KillbillClient {
         internalBus.start();
         cacheControllerDispatcher.clearAll();
         busHandler.reset();
+        callbackServlet.reset();
+
         clock.resetDeltaFromReality();
         clock.setDay(new LocalDate(2012, 8, 25));
 
         // Make sure to re-generate the api key and secret (could be cached by Shiro)
         DEFAULT_API_KEY = UUID.randomUUID().toString();
         DEFAULT_API_SECRET = UUID.randomUUID().toString();
-        loginTenant(DEFAULT_API_KEY, DEFAULT_API_SECRET);
 
         // Recreate the tenant (tables have been cleaned-up)
+        createTenant(DEFAULT_API_KEY, DEFAULT_API_SECRET, true);
+    }
+
+    protected Tenant createTenant(final String apiKey, final String apiSecret, final boolean useGlobalDefault) throws KillBillClientException {
+        callbackServlet.assertListenerStatus();
+        callbackServlet.reset();
+
+        loginTenant(apiKey, apiSecret);
         final Tenant tenant = new Tenant();
-        tenant.setApiKey(DEFAULT_API_KEY);
-        tenant.setApiSecret(DEFAULT_API_SECRET);
-        killBillClient.createTenant(tenant, createdBy, reason, comment);
+        tenant.setApiKey(apiKey);
+        tenant.setApiSecret(apiSecret);
+
+        requestOptions = requestOptions.extend()
+                                       .withTenantApiKey(apiKey)
+                                       .withTenantApiSecret(apiSecret)
+                                       .build();
+
+        callbackServlet.pushExpectedEvent(ExtBusEventType.TENANT_CONFIG_CHANGE);
+        if (!useGlobalDefault) {
+            // Catalog
+            callbackServlet.pushExpectedEvent(ExtBusEventType.TENANT_CONFIG_CHANGE);
+        }
+
+        final Tenant createdTenant = killBillClient.createTenant(tenant, useGlobalDefault, requestOptions);
+
+        // Register tenant for callback
+        final String callback = callbackServer.getServletEndpoint();
+        killBillClient.registerCallbackNotificationForTenant(callback, requestOptions);
+        callbackServlet.assertListenerStatus();
+
+        createdTenant.setApiSecret(apiSecret);
+
+        return createdTenant;
     }
 
     @AfterMethod(groups = "slow")
     public void afterMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         killBillClient.close();
         externalBus.stop();
         internalBus.stop();
@@ -285,6 +326,10 @@ public class TestJaxrsBase extends KillbillClient {
         server = new HttpServer();
         server.configure(config, getListeners(), getFilters());
         server.start();
+
+        callbackServlet = new CallbackServlet();
+        callbackServer = new CallbackServer(callbackServlet);
+        callbackServer.startServer();
     }
 
 
@@ -307,6 +352,7 @@ public class TestJaxrsBase extends KillbillClient {
     public void afterSuite() {
         try {
             server.stop();
+            callbackServer.stopServer();
         } catch (final Exception ignored) {
         }
     }
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPayment.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPayment.java
index 2b3f8a7..a713922 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPayment.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPayment.java
@@ -79,6 +79,10 @@ public class TestPayment extends TestJaxrsBase {
 
     @BeforeMethod(groups = "slow")
     public void beforeMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         super.beforeMethod();
         mockPaymentProviderPlugin = (MockPaymentProviderPlugin) registry.getServiceForName(PLUGIN_NAME);
 
@@ -103,6 +107,10 @@ public class TestPayment extends TestJaxrsBase {
 
     @AfterMethod(groups = "slow")
     public void tearDown() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         mockPaymentProviderPlugin.clear();
     }
 
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPaymentPluginProperties.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPaymentPluginProperties.java
index 3d87984..9904585 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPaymentPluginProperties.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPaymentPluginProperties.java
@@ -1,6 +1,6 @@
 /*
- * Copyright 2014-2016 Groupon, Inc
- * Copyright 2014-2016 The Billing Project, LLC
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -126,6 +126,10 @@ public class TestPaymentPluginProperties extends TestJaxrsBase {
 
     @BeforeMethod(groups = "slow")
     public void beforeMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         super.beforeMethod();
 
         mockPaymentControlProviderPlugin = new PluginPropertiesVerificator();
@@ -149,6 +153,10 @@ public class TestPaymentPluginProperties extends TestJaxrsBase {
 
     @AfterMethod(groups = "slow")
     public void tearDown() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         mockPaymentControlProviderPlugin.clearExpectPluginProperties();
     }
 
@@ -274,4 +282,4 @@ public class TestPaymentPluginProperties extends TestJaxrsBase {
         bodyProperties.add(new PluginProperty(key, value, false));
         expectProperties.add(new org.killbill.billing.payment.api.PluginProperty(key, value, false));
     }
-}
\ No newline at end of file
+}
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPerTenantConfig.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPerTenantConfig.java
index a8d572d..cbe1e97 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPerTenantConfig.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPerTenantConfig.java
@@ -1,6 +1,6 @@
 /*
- * Copyright 2014-2016 Groupon, Inc
- * Copyright 2014-2016 The Billing Project, LLC
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -18,13 +18,11 @@
 package org.killbill.billing.jaxrs;
 
 import java.util.HashMap;
-import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
 
 import org.killbill.billing.client.model.Account;
 import org.killbill.billing.client.model.Payments;
-import org.killbill.billing.client.model.Tenant;
 import org.killbill.billing.client.model.TenantKey;
+import org.killbill.billing.notification.plugin.api.ExtBusEventType;
 import org.killbill.billing.osgi.api.OSGIServiceRegistration;
 import org.killbill.billing.payment.api.TransactionStatus;
 import org.killbill.billing.payment.plugin.api.PaymentPluginApi;
@@ -36,8 +34,6 @@ import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
 import com.google.inject.Inject;
-import org.awaitility.Awaitility;
-import org.awaitility.Duration;
 
 public class TestPerTenantConfig extends TestJaxrsBase {
 
@@ -48,25 +44,27 @@ public class TestPerTenantConfig extends TestJaxrsBase {
 
     @BeforeMethod(groups = "slow")
     public void beforeMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         super.beforeMethod();
         mockPaymentProviderPlugin = (MockPaymentProviderPlugin) registry.getServiceForName(PLUGIN_NAME);
     }
 
     @AfterMethod(groups = "slow")
     public void tearDown() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         mockPaymentProviderPlugin.clear();
     }
 
     @Test(groups = "slow")
     public void testFailedPaymentWithPerTenantRetryConfig() throws Exception {
         // Create the tenant
-        final String apiKeyTenant1 = "tenantSuperTuned";
-        final String apiSecretTenant1 = "2367$$ffr79";
-        loginTenant(apiKeyTenant1, apiSecretTenant1);
-        final Tenant tenant1 = new Tenant();
-        tenant1.setApiKey(apiKeyTenant1);
-        tenant1.setApiSecret(apiSecretTenant1);
-        killBillClient.createTenant(tenant1, createdBy, reason, comment);
+        createTenant("tenantSuperTuned", "2367$$ffr79", true);
 
         // Configure our plugin to fail
         mockPaymentProviderPlugin.makeAllInvoicesFailWithError(true);
@@ -77,9 +75,11 @@ public class TestPerTenantConfig extends TestJaxrsBase {
         perTenantProperties.put("org.killbill.payment.retry.days", "1,1,1");
         final String perTenantConfig = mapper.writeValueAsString(perTenantProperties);
 
+        callbackServlet.pushExpectedEvent(ExtBusEventType.TENANT_CONFIG_CHANGE);
         final TenantKey tenantKey = killBillClient.postConfigurationPropertiesForTenant(perTenantConfig, requestOptions);
+        callbackServlet.assertListenerStatus();
 
-        final Account accountJson = createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoice();
+        final Account accountJson = createAccountWithPMBundleAndSubscriptionAndWaitForFirstInvoice(false);
 
         final Payments payments = killBillClient.getPaymentsForAccount(accountJson.getAccountId());
         Assert.assertEquals(payments.size(), 1);
@@ -92,22 +92,14 @@ public class TestPerTenantConfig extends TestJaxrsBase {
         //
         // Now unregister special per tenant config and we the first retry occurs one day after (and still fails), it now sets a retry date of 8 days
         //
+        callbackServlet.pushExpectedEvents(ExtBusEventType.TENANT_CONFIG_DELETION);
         killBillClient.unregisterConfigurationForTenant(requestOptions);
-        // org.killbill.tenant.broadcast.rate has been set to 1s
-        crappyWaitForLackOfProperSynchonization(2000);
+        callbackServlet.assertListenerStatus();
 
+        callbackServlet.pushExpectedEvents(ExtBusEventType.INVOICE_PAYMENT_FAILED, ExtBusEventType.PAYMENT_FAILED);
         clock.addDays(1);
+        callbackServlet.assertListenerStatus();
 
-        Awaitility.await()
-                  .atMost(4, TimeUnit.SECONDS)
-                  .pollInterval(Duration.ONE_SECOND)
-                  .until(new Callable<Boolean>() {
-                      @Override
-                      public Boolean call() throws Exception {
-
-                          return killBillClient.getPaymentsForAccount(accountJson.getAccountId()).get(0).getTransactions().size() == 2;
-                      }
-                  });
         final Payments payments2 = killBillClient.getPaymentsForAccount(accountJson.getAccountId());
         Assert.assertEquals(payments2.size(), 1);
         Assert.assertEquals(payments2.get(0).getTransactions().size(), 2);
@@ -115,7 +107,7 @@ public class TestPerTenantConfig extends TestJaxrsBase {
         Assert.assertEquals(payments2.get(0).getTransactions().get(1).getStatus(), TransactionStatus.PAYMENT_FAILURE.name());
 
         clock.addDays(1);
-        crappyWaitForLackOfProperSynchonization(3000);
+        callbackServlet.assertListenerStatus();
 
         // No retry with default config
         final Payments payments3 = killBillClient.getPaymentsForAccount(accountJson.getAccountId());
@@ -123,17 +115,10 @@ public class TestPerTenantConfig extends TestJaxrsBase {
         Assert.assertEquals(payments3.get(0).getTransactions().size(), 2);
 
         mockPaymentProviderPlugin.makeAllInvoicesFailWithError(false);
+        callbackServlet.pushExpectedEvents(ExtBusEventType.INVOICE_PAYMENT_SUCCESS, ExtBusEventType.PAYMENT_SUCCESS);
         clock.addDays(7);
+        callbackServlet.assertListenerStatus();
 
-        Awaitility.await()
-                  .atMost(4, TimeUnit.SECONDS)
-                  .pollInterval(Duration.ONE_SECOND)
-                  .until(new Callable<Boolean>() {
-                      @Override
-                      public Boolean call() throws Exception {
-                          return killBillClient.getPaymentsForAccount(accountJson.getAccountId()).get(0).getTransactions().size() == 3;
-                      }
-                  });
         final Payments payments4 = killBillClient.getPaymentsForAccount(accountJson.getAccountId());
         Assert.assertEquals(payments4.size(), 1);
         Assert.assertEquals(payments4.get(0).getTransactions().size(), 3);
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPlugin.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPlugin.java
index 3b4ce0c..9a03015 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPlugin.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPlugin.java
@@ -57,6 +57,10 @@ public class TestPlugin extends TestJaxrsBase {
     @Override
     @BeforeMethod(groups = "slow")
     public void beforeMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         super.beforeMethod();
         setupOSGIPlugin();
         resetAllMarkers();
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPushNotification.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPushNotification.java
index 5d08db5..81e8b07 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPushNotification.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestPushNotification.java
@@ -32,9 +32,6 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import org.awaitility.Awaitility;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
 import org.joda.time.DateTime;
 import org.killbill.CreatorName;
 import org.killbill.billing.api.FlakyRetryAnalyzer;
@@ -68,9 +65,6 @@ public class TestPushNotification extends TestJaxrsBase {
 
     private CallbackServer callbackServer;
 
-    private static final int SERVER_PORT = 8087;
-    private static final String CALLBACK_ENDPOINT = "/callmeback";
-
     private volatile boolean callbackCompleted;
     private volatile boolean callbackCompletedWithError;
     private volatile int expectedNbCalls = 1;
@@ -80,8 +74,12 @@ public class TestPushNotification extends TestJaxrsBase {
     @Override
     @BeforeMethod(groups = "slow")
     public void beforeMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         super.beforeMethod();
-        callbackServer = new CallbackServer(this, SERVER_PORT, CALLBACK_ENDPOINT);
+        callbackServer = new CallbackServer(new CallmebackServlet(this));
         resetCallbackStatusProperties();
         callbackServer.startServer();
         this.expectedNbCalls = 1;
@@ -89,6 +87,10 @@ public class TestPushNotification extends TestJaxrsBase {
 
     @AfterMethod(groups = "slow")
     public void afterMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         callbackServer.stopServer();
     }
 
@@ -220,8 +222,9 @@ public class TestPushNotification extends TestJaxrsBase {
         Assert.assertEquals(result2.getValues().size(), 0);
     }
 
-    private String registerTenantForCallback() throws KillBillClientException, InterruptedException {// Register tenant for callback
-        final String callback = "http://127.0.0.1:" + SERVER_PORT + CALLBACK_ENDPOINT;
+    // Register tenant for callback
+    private String registerTenantForCallback() throws KillBillClientException, InterruptedException {
+        final String callback = callbackServer.getServletEndpoint();
         final TenantKey result0 = killBillClient.registerCallbackNotificationForTenant(callback, requestOptions);
 
         Assert.assertTrue(waitForCallbacksToComplete());
@@ -366,31 +369,6 @@ public class TestPushNotification extends TestJaxrsBase {
         callbackCompletedWithError = withError;
     }
 
-    public static class CallbackServer {
-
-        private final Server server;
-        private final String callbackEndpoint;
-        private final TestPushNotification test;
-
-        public CallbackServer(final TestPushNotification test, final int port, final String callbackEndpoint) {
-            this.callbackEndpoint = callbackEndpoint;
-            this.test = test;
-            this.server = new Server(port);
-        }
-
-        public void startServer() throws Exception {
-            final ServletContextHandler context = new ServletContextHandler();
-            context.setContextPath("/");
-            server.setHandler(context);
-            context.addServlet(new ServletHolder(new CallmebackServlet(test)), callbackEndpoint);
-            server.start();
-        }
-
-        public void stopServer() throws Exception {
-            server.stop();
-        }
-    }
-
     public static class CallmebackServlet extends HttpServlet {
 
         private static final long serialVersionUID = -5181211514918217301L;
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestTenantKV.java b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestTenantKV.java
index fbff4b5..80a3363 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestTenantKV.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/jaxrs/TestTenantKV.java
@@ -1,6 +1,6 @@
 /*
- * Copyright 2014-2016 Groupon, Inc
- * Copyright 2014-2016 The Billing Project, LLC
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -35,6 +35,7 @@ import org.killbill.billing.client.model.PaymentTransaction;
 import org.killbill.billing.client.model.PluginProperty;
 import org.killbill.billing.client.model.Tenant;
 import org.killbill.billing.client.model.TenantKey;
+import org.killbill.billing.notification.plugin.api.ExtBusEventType;
 import org.killbill.billing.payment.api.TransactionStatus;
 import org.killbill.billing.tenant.api.TenantKV;
 import org.testng.Assert;
@@ -53,7 +54,9 @@ public class TestTenantKV extends TestJaxrsBase {
         final String pluginName = "PLUGIN_FOO";
 
         final String pluginPath = Resources.getResource("plugin.yml").getPath();
+        callbackServlet.pushExpectedEvent(ExtBusEventType.TENANT_CONFIG_CHANGE);
         final TenantKey tenantKey0 = killBillClient.registerPluginConfigurationForTenant(pluginName, pluginPath, createdBy, reason, comment);
+        callbackServlet.assertListenerStatus();
         Assert.assertEquals(tenantKey0.getKey(), TenantKV.TenantKey.PLUGIN_CONFIG_.toString() + pluginName);
 
         final TenantKey tenantKey1 = killBillClient.getPluginConfigurationForTenant(pluginName);
@@ -68,53 +71,55 @@ public class TestTenantKV extends TestJaxrsBase {
 
     @Test(groups = "slow", description = "Upload and retrieve a per plugin payment state machine config")
     public void testPerTenantPluginPaymentStateMachineConfig() throws Exception {
+        final RequestOptions requestOptionsForOriginalTenant = requestOptions;
+
         // Create another tenant - it will have a different state machine
-        final Tenant otherTenantWithDifferentStateMachine = new Tenant();
-        otherTenantWithDifferentStateMachine.setApiKey(UUID.randomUUID().toString());
-        otherTenantWithDifferentStateMachine.setApiSecret(UUID.randomUUID().toString());
-        killBillClient.createTenant(otherTenantWithDifferentStateMachine, true, requestOptions);
-        final RequestOptions requestOptionsOtherTenant = requestOptions.extend()
-                                                                       .withTenantApiKey(otherTenantWithDifferentStateMachine.getApiKey())
-                                                                       .withTenantApiSecret(otherTenantWithDifferentStateMachine.getApiSecret())
-                                                                       .build();
+        final Tenant otherTenantWithDifferentStateMachine = createTenant(UUID.randomUUID().toString(), UUID.randomUUID().toString(), true);
 
         // Verify initial state
-        final TenantKey emptyTenantKey = killBillClient.getPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, requestOptions);
+        final TenantKey emptyTenantKey = killBillClient.getPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, requestOptionsForOriginalTenant);
         Assert.assertEquals(emptyTenantKey.getValues().size(), 0);
-        final TenantKey emptyTenantKeyOtherTenant = killBillClient.getPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, requestOptionsOtherTenant);
+        final TenantKey emptyTenantKeyOtherTenant = killBillClient.getPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, requestOptions);
         Assert.assertEquals(emptyTenantKeyOtherTenant.getValues().size(), 0);
 
+        callbackServlet.pushExpectedEvent(ExtBusEventType.TENANT_CONFIG_CHANGE);
         final String stateMachineConfigPath = Resources.getResource("SimplePaymentStates.xml").getPath();
-        final TenantKey tenantKey0 = killBillClient.registerPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, stateMachineConfigPath, requestOptionsOtherTenant);
+        final TenantKey tenantKey0 = killBillClient.registerPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, stateMachineConfigPath, requestOptions);
+        callbackServlet.assertListenerStatus();
         Assert.assertEquals(tenantKey0.getKey(), TenantKV.TenantKey.PLUGIN_PAYMENT_STATE_MACHINE_.toString() + PLUGIN_NAME);
 
         // Verify only the other tenant has the new state machine
-        final TenantKey emptyTenantKey1 = killBillClient.getPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, requestOptions);
+        final TenantKey emptyTenantKey1 = killBillClient.getPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, requestOptionsForOriginalTenant);
         Assert.assertEquals(emptyTenantKey1.getValues().size(), 0);
-        final TenantKey tenantKey1OtherTenant = killBillClient.getPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, requestOptionsOtherTenant);
+        final TenantKey tenantKey1OtherTenant = killBillClient.getPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, requestOptions);
         Assert.assertEquals(tenantKey1OtherTenant.getKey(), TenantKV.TenantKey.PLUGIN_PAYMENT_STATE_MACHINE_.toString() + PLUGIN_NAME);
         Assert.assertEquals(tenantKey1OtherTenant.getValues().size(), 1);
 
         // Create an auth in both tenant
-        final Payment payment = createComboPaymentTransaction(requestOptions);
-        final Payment paymentOtherTenant = createComboPaymentTransaction(requestOptionsOtherTenant);
+        final Payment payment = createComboPaymentTransaction(requestOptionsForOriginalTenant);
+        final Payment paymentOtherTenant = createComboPaymentTransaction(requestOptions);
 
         // Void in the first tenant (allowed by the default state machine)
-        final Payment voidPayment = killBillClient.voidPayment(payment.getPaymentId(), payment.getPaymentExternalKey(), UUID.randomUUID().toString(), ImmutableList.<String>of(), ImmutableMap.<String, String>of(), requestOptions);
+        callbackServlet.pushExpectedEvent(ExtBusEventType.PAYMENT_SUCCESS);
+        final Payment voidPayment = killBillClient.voidPayment(payment.getPaymentId(), payment.getPaymentExternalKey(), UUID.randomUUID().toString(), ImmutableList.<String>of(), ImmutableMap.<String, String>of(), requestOptionsForOriginalTenant);
+        callbackServlet.assertListenerStatus();
         Assert.assertEquals(voidPayment.getTransactions().get(0).getStatus(), TransactionStatus.SUCCESS.toString());
         Assert.assertEquals(voidPayment.getTransactions().get(1).getStatus(), TransactionStatus.SUCCESS.toString());
 
         // Void in the other tenant (disallowed)
         try {
-            killBillClient.voidPayment(paymentOtherTenant.getPaymentId(), paymentOtherTenant.getPaymentExternalKey(), UUID.randomUUID().toString(), ImmutableList.<String>of(), ImmutableMap.<String, String>of(), requestOptionsOtherTenant);
+            killBillClient.voidPayment(paymentOtherTenant.getPaymentId(), paymentOtherTenant.getPaymentExternalKey(), UUID.randomUUID().toString(), ImmutableList.<String>of(), ImmutableMap.<String, String>of(), requestOptions);
             Assert.fail();
         } catch (final KillBillClientException e) {
             Assert.assertEquals((int) e.getBillingException().getCode(), ErrorCode.PAYMENT_INVALID_OPERATION.getCode());
         }
+        callbackServlet.assertListenerStatus();
 
         // Remove the custom state machine
-        killBillClient.unregisterPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, requestOptionsOtherTenant);
-        final TenantKey tenantKey2 = killBillClient.getPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, requestOptionsOtherTenant);
+        callbackServlet.pushExpectedEvent(ExtBusEventType.TENANT_CONFIG_DELETION);
+        killBillClient.unregisterPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, requestOptions);
+        callbackServlet.assertListenerStatus();
+        final TenantKey tenantKey2 = killBillClient.getPluginPaymentStateMachineConfigurationForTenant(PLUGIN_NAME, requestOptions);
         Assert.assertEquals(tenantKey2.getKey(), TenantKV.TenantKey.PLUGIN_PAYMENT_STATE_MACHINE_.toString() + PLUGIN_NAME);
         Assert.assertEquals(tenantKey2.getValues().size(), 0);
 
@@ -127,7 +132,9 @@ public class TestTenantKV extends TestJaxrsBase {
                       public Boolean call() throws Exception {
                           // The void should now go through
                           try {
-                              final Payment voidPaymentOtherTenant2 = killBillClient.voidPayment(paymentOtherTenant.getPaymentId(), paymentOtherTenant.getPaymentExternalKey(), UUID.randomUUID().toString(), ImmutableList.<String>of(), ImmutableMap.<String, String>of(), requestOptionsOtherTenant);
+                              callbackServlet.pushExpectedEvent(ExtBusEventType.PAYMENT_SUCCESS);
+                              final Payment voidPaymentOtherTenant2 = killBillClient.voidPayment(paymentOtherTenant.getPaymentId(), paymentOtherTenant.getPaymentExternalKey(), UUID.randomUUID().toString(), ImmutableList.<String>of(), ImmutableMap.<String, String>of(), requestOptions);
+                              callbackServlet.assertListenerStatus();
                               voidPaymentOtherTenant2Ref.set(voidPaymentOtherTenant2);
                               return voidPaymentOtherTenant2 != null;
                           } catch (final KillBillClientException e) {
@@ -158,8 +165,10 @@ public class TestTenantKV extends TestJaxrsBase {
         authTransactionJson.setTransactionExternalKey(authTransactionExternalKey);
         authTransactionJson.setTransactionType("AUTHORIZE");
 
+        callbackServlet.pushExpectedEvents(ExtBusEventType.ACCOUNT_CREATION, ExtBusEventType.ACCOUNT_CHANGE, ExtBusEventType.PAYMENT_SUCCESS);
         final ComboPaymentTransaction comboAuthorization = new ComboPaymentTransaction(accountJson, paymentMethodJson, authTransactionJson, ImmutableList.<PluginProperty>of(), ImmutableList.<PluginProperty>of());
         final Payment payment = killBillClient.createPayment(comboAuthorization, ImmutableMap.<String, String>of(), requestOptions);
+        callbackServlet.assertListenerStatus();
         Assert.assertEquals(payment.getTransactions().get(0).getStatus(), TransactionStatus.SUCCESS.toString());
 
         return payment;
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/server/security/TestKillbillJdbcTenantRealm.java b/profiles/killbill/src/test/java/org/killbill/billing/server/security/TestKillbillJdbcTenantRealm.java
index 292cd25..937c529 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/server/security/TestKillbillJdbcTenantRealm.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/server/security/TestKillbillJdbcTenantRealm.java
@@ -48,6 +48,10 @@ public class TestKillbillJdbcTenantRealm extends TestJaxrsBase {
     @Override
     @BeforeMethod(groups = "slow")
     public void beforeMethod() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         super.beforeMethod();
 
         // Create the tenant
diff --git a/profiles/killbill/src/test/java/org/killbill/billing/server/security/TestTenantFilter.java b/profiles/killbill/src/test/java/org/killbill/billing/server/security/TestTenantFilter.java
index fdc95ce..4e64cf8 100644
--- a/profiles/killbill/src/test/java/org/killbill/billing/server/security/TestTenantFilter.java
+++ b/profiles/killbill/src/test/java/org/killbill/billing/server/security/TestTenantFilter.java
@@ -1,7 +1,7 @@
 /*
  * Copyright 2010-2013 Ning, Inc.
- * Copyright 2014 Groupon, Inc
- * Copyright 2014 The Billing Project, LLC
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -32,6 +32,10 @@ public class TestTenantFilter extends TestJaxrsBase {
 
     @AfterMethod(groups = "slow")
     public void tearDown() throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         // Default credentials
         loginTenant(DEFAULT_API_KEY, DEFAULT_API_SECRET);
     }
@@ -46,15 +50,10 @@ public class TestTenantFilter extends TestJaxrsBase {
         } catch (final KillBillClientException e) {
             Assert.assertEquals(e.getResponse().getStatusCode(), Status.UNAUTHORIZED.getStatusCode());
         }
+        callbackServlet.assertListenerStatus();
 
         // Create the tenant
-        final String apiKeyTenant1 = "pierre";
-        final String apiSecretTenant1 = "pierreIsFr3nch";
-        loginTenant(apiKeyTenant1, apiSecretTenant1);
-        final Tenant tenant1 = new Tenant();
-        tenant1.setApiKey(apiKeyTenant1);
-        tenant1.setApiSecret(apiSecretTenant1);
-        killBillClient.createTenant(tenant1, createdBy, reason, comment);
+        final Tenant tenant1 = createTenant("pierre", "pierreIsFr3nch", true);
 
         final Account account1 = createAccount();
         Assert.assertEquals(killBillClient.getAccount(account1.getExternalKey()), account1);
@@ -62,13 +61,7 @@ public class TestTenantFilter extends TestJaxrsBase {
         logoutTenant();
 
         // Create another tenant
-        final String apiKeyTenant2 = "stephane";
-        final String apiSecretTenant2 = "stephane1sAlsoFr3nch";
-        loginTenant(apiKeyTenant2, apiSecretTenant2);
-        final Tenant tenant2 = new Tenant();
-        tenant2.setApiKey(apiKeyTenant2);
-        tenant2.setApiSecret(apiSecretTenant2);
-        killBillClient.createTenant(tenant2, createdBy, reason, comment);
+        createTenant("stephane", "stephane1sAlsoFr3nch", true);
 
         final Account account2 = createAccount();
         Assert.assertEquals(killBillClient.getAccount(account2.getExternalKey()), account2);
@@ -77,7 +70,7 @@ public class TestTenantFilter extends TestJaxrsBase {
         Assert.assertNull(killBillClient.getAccount(account1.getExternalKey()));
 
         // Same for tenant1 and account2
-        loginTenant(apiKeyTenant1, apiSecretTenant1);
+        loginTenant(tenant1.getApiKey(), tenant1.getApiSecret());
         Assert.assertNull(killBillClient.getAccount(account2.getExternalKey()));
     }
 }
diff --git a/util/src/test/java/org/killbill/billing/api/TestApiListener.java b/util/src/test/java/org/killbill/billing/api/TestApiListener.java
index d730885..83cb4c2 100644
--- a/util/src/test/java/org/killbill/billing/api/TestApiListener.java
+++ b/util/src/test/java/org/killbill/billing/api/TestApiListener.java
@@ -294,7 +294,7 @@ public class TestApiListener {
         }
     }
 
-    public boolean isCompleted(final long timeout) {
+    private boolean isCompleted(final long timeout) {
         synchronized (this) {
             long waitTimeMs = timeout;
             do {
diff --git a/util/src/test/java/org/killbill/billing/GuicyKillbillTestSuite.java b/util/src/test/java/org/killbill/billing/GuicyKillbillTestSuite.java
index 135f7aa..eb0c634 100644
--- a/util/src/test/java/org/killbill/billing/GuicyKillbillTestSuite.java
+++ b/util/src/test/java/org/killbill/billing/GuicyKillbillTestSuite.java
@@ -42,7 +42,6 @@ import org.mockito.stubbing.Answer;
 import org.skife.config.ConfigSource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.testng.Assert;
 import org.testng.IHookCallBack;
 import org.testng.IHookable;
 import org.testng.ITestResult;
@@ -149,7 +148,7 @@ public class GuicyKillbillTestSuite implements IHookable {
 
     @BeforeMethod(alwaysRun = true)
     public void beforeMethodAlwaysRun(final Method method) throws Exception {
-        if (AbortAfterFirstFailureListener.hasFailures()) {
+        if (hasFailed()) {
             return;
         }
 
@@ -198,7 +197,7 @@ public class GuicyKillbillTestSuite implements IHookable {
 
     @AfterMethod(alwaysRun = true)
     public void afterMethodAlwaysRun(final Method method, final ITestResult result) throws Exception {
-        if (AbortAfterFirstFailureListener.hasFailures()) {
+        if (hasFailed()) {
             return;
         }
 
diff --git a/util/src/test/java/org/killbill/billing/KillbillTestSuite.java b/util/src/test/java/org/killbill/billing/KillbillTestSuite.java
index 7b4bdf7..8591c1f 100644
--- a/util/src/test/java/org/killbill/billing/KillbillTestSuite.java
+++ b/util/src/test/java/org/killbill/billing/KillbillTestSuite.java
@@ -1,7 +1,7 @@
 /*
  * Copyright 2010-2013 Ning, Inc.
- * Copyright 2014-2016 Groupon, Inc
- * Copyright 2014-2016 The Billing Project, LLC
+ * Copyright 2014-2018 Groupon, Inc
+ * Copyright 2014-2018 The Billing Project, LLC
  *
  * The Billing Project licenses this file to you under the Apache License, version 2.0
  * (the "License"); you may not use this file except in compliance with the
@@ -20,6 +20,7 @@ package org.killbill.billing;
 
 import java.lang.reflect.Method;
 
+import org.killbill.billing.api.AbortAfterFirstFailureListener;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testng.ITestResult;
@@ -35,6 +36,10 @@ public class KillbillTestSuite {
 
     @BeforeMethod(alwaysRun = true)
     public void startTestSuite(final Method method) throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         log.info("***************************************************************************************************");
         log.info("*** Starting test {}:{}", method.getDeclaringClass().getName(), method.getName());
         log.info("***************************************************************************************************");
@@ -42,6 +47,10 @@ public class KillbillTestSuite {
 
     @AfterMethod(alwaysRun = true)
     public void endTestSuite(final Method method, final ITestResult result) throws Exception {
+        if (hasFailed()) {
+            return;
+        }
+
         log.info("***************************************************************************************************");
         log.info("***   Ending test {}:{} {} ({} s.)", new Object[]{method.getDeclaringClass().getName(), method.getName(),
                                                                     result.isSuccess() ? "SUCCESS" : "!!! FAILURE !!!",
@@ -53,6 +62,6 @@ public class KillbillTestSuite {
     }
 
     public boolean hasFailed() {
-        return hasFailed;
+        return hasFailed || AbortAfterFirstFailureListener.hasFailures();
     }
 }