killbill-memoizeit

payment: optimize pagination for payment methods Signed-off-by:

10/24/2013 6:42:32 PM

Details

diff --git a/account/src/main/java/com/ning/billing/account/api/svcs/DefaultAccountInternalApi.java b/account/src/main/java/com/ning/billing/account/api/svcs/DefaultAccountInternalApi.java
index 486057d..27d1601 100644
--- a/account/src/main/java/com/ning/billing/account/api/svcs/DefaultAccountInternalApi.java
+++ b/account/src/main/java/com/ning/billing/account/api/svcs/DefaultAccountInternalApi.java
@@ -67,21 +67,6 @@ public class DefaultAccountInternalApi implements AccountInternalApi {
     }
 
     @Override
-    public Pagination<Account> getAccounts(final Long offset, final Long limit, final InternalTenantContext context) {
-        final Pagination<AccountModelDao> accountModelDaos = accountDao.get(offset, limit, context);
-        return new DefaultPagination<Account>(accountModelDaos,
-                                              limit,
-                                              Iterators.<AccountModelDao, Account>transform(accountModelDaos.iterator(),
-                                                                                            new Function<AccountModelDao, Account>() {
-                                                                                                @Override
-                                                                                                public Account apply(final AccountModelDao input) {
-                                                                                                    return new DefaultAccount(input);
-                                                                                                }
-                                                                                            }));
-
-    }
-
-    @Override
     public void updateAccount(final String externalKey, final AccountData accountData,
                               final InternalCallContext context) throws AccountApiException {
         final Account currentAccount = getAccountByKey(externalKey, context);
diff --git a/api/src/main/java/com/ning/billing/account/api/AccountInternalApi.java b/api/src/main/java/com/ning/billing/account/api/AccountInternalApi.java
index 684ca30..19f818f 100644
--- a/api/src/main/java/com/ning/billing/account/api/AccountInternalApi.java
+++ b/api/src/main/java/com/ning/billing/account/api/AccountInternalApi.java
@@ -31,8 +31,6 @@ public interface AccountInternalApi {
 
     public Account getAccountByRecordId(Long recordId, InternalTenantContext context) throws AccountApiException;
 
-    public Pagination<Account> getAccounts(Long offset, Long limit, InternalTenantContext context);
-
     public void updateAccount(String key, AccountData accountData, InternalCallContext context) throws AccountApiException;
 
     public List<AccountEmail> getEmails(UUID accountId, InternalTenantContext context);
diff --git a/jaxrs/src/main/java/com/ning/billing/jaxrs/resources/JaxRsResourceBase.java b/jaxrs/src/main/java/com/ning/billing/jaxrs/resources/JaxRsResourceBase.java
index c24c399..150b97e 100644
--- a/jaxrs/src/main/java/com/ning/billing/jaxrs/resources/JaxRsResourceBase.java
+++ b/jaxrs/src/main/java/com/ning/billing/jaxrs/resources/JaxRsResourceBase.java
@@ -63,7 +63,7 @@ import com.google.common.collect.ImmutableList;
 
 public abstract class JaxRsResourceBase implements JaxrsResource {
 
-    private static final Logger log = LoggerFactory.getLogger(JaxRsResourceBase.class);
+    static final Logger log = LoggerFactory.getLogger(JaxRsResourceBase.class);
 
     protected static final ObjectMapper mapper = new ObjectMapper();
 
diff --git a/jaxrs/src/main/java/com/ning/billing/jaxrs/resources/PaymentMethodResource.java b/jaxrs/src/main/java/com/ning/billing/jaxrs/resources/PaymentMethodResource.java
index 1cdc630..b84898c 100644
--- a/jaxrs/src/main/java/com/ning/billing/jaxrs/resources/PaymentMethodResource.java
+++ b/jaxrs/src/main/java/com/ning/billing/jaxrs/resources/PaymentMethodResource.java
@@ -158,7 +158,8 @@ public class PaymentMethodResource extends JaxRsResourceBase {
                             account = accountUserApi.getAccountById(paymentMethod.getAccountId(), tenantContext);
                             accounts.put(paymentMethod.getAccountId(), account);
                         } catch (AccountApiException e) {
-                            throw new RuntimeException(e);
+                            log.warn("Unable to retrieve account", e);
+                            continue;
                         }
                     }
 
diff --git a/payment/src/main/java/com/ning/billing/payment/api/DefaultPaymentApi.java b/payment/src/main/java/com/ning/billing/payment/api/DefaultPaymentApi.java
index 579ded0..8352981 100644
--- a/payment/src/main/java/com/ning/billing/payment/api/DefaultPaymentApi.java
+++ b/payment/src/main/java/com/ning/billing/payment/api/DefaultPaymentApi.java
@@ -26,17 +26,14 @@ import java.util.UUID;
 
 import com.ning.billing.ErrorCode;
 import com.ning.billing.account.api.Account;
-import com.ning.billing.callcontext.DefaultCallContext;
 import com.ning.billing.callcontext.InternalCallContext;
 import com.ning.billing.clock.Clock;
 import com.ning.billing.payment.core.PaymentMethodProcessor;
 import com.ning.billing.payment.core.PaymentProcessor;
 import com.ning.billing.payment.core.RefundProcessor;
 import com.ning.billing.util.callcontext.CallContext;
-import com.ning.billing.util.callcontext.CallOrigin;
 import com.ning.billing.util.callcontext.InternalCallContextFactory;
 import com.ning.billing.util.callcontext.TenantContext;
-import com.ning.billing.util.callcontext.UserType;
 import com.ning.billing.util.entity.Pagination;
 
 import com.google.common.collect.ImmutableMap;
@@ -182,16 +179,12 @@ public class DefaultPaymentApi implements PaymentApi {
 
     @Override
     public Pagination<PaymentMethod> getPaymentMethods(final Long offset, final Long limit, final TenantContext context) {
-        // Lame, but required
-        final CallContext callContext = new DefaultCallContext(context.getTenantId(), DefaultPaymentApi.class.toString(), CallOrigin.EXTERNAL, UserType.CUSTOMER, UUID.randomUUID(), clock);
-        return methodProcessor.getPaymentMethods(offset, limit, context, callContext, internalCallContextFactory.createInternalTenantContext(context));
+        return methodProcessor.getPaymentMethods(offset, limit, context, internalCallContextFactory.createInternalTenantContext(context));
     }
 
     @Override
     public Pagination<PaymentMethod> getPaymentMethods(final Long offset, final Long limit, final String pluginName, final TenantContext context) throws PaymentApiException {
-        // Lame, but required
-        final CallContext callContext = new DefaultCallContext(context.getTenantId(), DefaultPaymentApi.class.toString(), CallOrigin.EXTERNAL, UserType.CUSTOMER, UUID.randomUUID(), clock);
-        return methodProcessor.getPaymentMethods(offset, limit, pluginName, context, callContext, internalCallContextFactory.createInternalTenantContext(context));
+        return methodProcessor.getPaymentMethods(offset, limit, pluginName, context, internalCallContextFactory.createInternalTenantContext(context));
     }
 
     @Override
diff --git a/payment/src/main/java/com/ning/billing/payment/core/PaymentMethodProcessor.java b/payment/src/main/java/com/ning/billing/payment/core/PaymentMethodProcessor.java
index f720133..c8d8786 100644
--- a/payment/src/main/java/com/ning/billing/payment/core/PaymentMethodProcessor.java
+++ b/payment/src/main/java/com/ning/billing/payment/core/PaymentMethodProcessor.java
@@ -53,7 +53,6 @@ import com.ning.billing.payment.provider.DefaultNoOpPaymentMethodPlugin;
 import com.ning.billing.payment.provider.DefaultPaymentMethodInfoPlugin;
 import com.ning.billing.payment.provider.ExternalPaymentProviderPlugin;
 import com.ning.billing.tag.TagInternalApi;
-import com.ning.billing.util.callcontext.CallContext;
 import com.ning.billing.util.callcontext.TenantContext;
 import com.ning.billing.util.dao.NonEntityDao;
 import com.ning.billing.util.entity.DefaultPagination;
@@ -159,71 +158,69 @@ public class PaymentMethodProcessor extends ProcessorBase {
         return new DefaultPaymentMethod(paymentMethodModelDao, paymentMethodPlugin);
     }
 
-    public Pagination<PaymentMethod> getPaymentMethods(final Long offset, final Long limit, final TenantContext tenantContext, final CallContext callContext, final InternalTenantContext internalTenantContext) {
+    public Pagination<PaymentMethod> getPaymentMethods(final Long offset, final Long limit, final TenantContext tenantContext, final InternalTenantContext internalTenantContext) {
         // Note that we cannot easily do streaming here, since we would have to rely on the statistics
         // returned by the Pagination objects from the plugins and we probably don't want to do that (if
         // one plugin gets it wrong, it may starve the others).
         final List<PaymentMethod> allResults = new LinkedList<PaymentMethod>();
+        Long totalNbRecords = 0L;
+        Long maxNbRecords = 0L;
 
         // Search in all plugins (we treat the full set of results as a union with respect to offset/limit)
+        boolean firstSearch = true;
         for (final String pluginName : getAvailablePlugins()) {
             try {
-                final Pagination<PaymentMethod> paymentMethods = getPaymentMethods(0L, Long.MAX_VALUE, pluginName, tenantContext, callContext, internalTenantContext);
-                allResults.addAll(ImmutableList.<PaymentMethod>copyOf(paymentMethods));
-                if (allResults.size() > offset + limit) {
-                    break;
+                final Pagination<PaymentMethod> paymentMethods;
+                if (allResults.size() >= limit) {
+                    // We have enough results, we just keep going (limit 1) to get the stats
+                    paymentMethods = getPaymentMethods(firstSearch ? offset : 0L, 1L, pluginName, tenantContext, internalTenantContext);
+                    // Required to close database connections
+                    ImmutableList.<PaymentMethod>copyOf(paymentMethods);
+                } else {
+                    paymentMethods = getPaymentMethods(firstSearch ? offset : 0L, limit - allResults.size(), pluginName, tenantContext, internalTenantContext);
+                    allResults.addAll(ImmutableList.<PaymentMethod>copyOf(paymentMethods));
                 }
+                firstSearch = false;
+                totalNbRecords += paymentMethods.getTotalNbRecords();
+                maxNbRecords += paymentMethods.getMaxNbRecords();
             } catch (PaymentApiException e) {
                 log.warn("Error while searching plugin " + pluginName, e);
                 // Non-fatal, continue to search other plugins
             }
         }
 
-        return DefaultPagination.<PaymentMethod>build(offset, limit, allResults);
+        return new DefaultPagination<PaymentMethod>(offset, limit, totalNbRecords, maxNbRecords, allResults.iterator());
     }
 
-    public Pagination<PaymentMethod> getPaymentMethods(final Long offset, final Long limit, final String pluginName, final TenantContext tenantContext, final CallContext callContext, final InternalTenantContext internalTenantContext) throws PaymentApiException {
+    public Pagination<PaymentMethod> getPaymentMethods(final Long offset, final Long limit, final String pluginName, final TenantContext tenantContext, final InternalTenantContext internalTenantContext) throws PaymentApiException {
         final PaymentPluginApi pluginApi = getPaymentPluginApi(pluginName);
-        final List<PaymentMethodPlugin> paymentMethods = new LinkedList<PaymentMethodPlugin>();
 
         // Find all payment methods for all accounts
-        final Pagination<Account> accounts = accountInternalApi.getAccounts(0L, Long.MAX_VALUE, internalTenantContext);
-        for (final Account account : accounts) {
-            try {
-                final List<PaymentMethodInfoPlugin> paymentMethodInfos = pluginApi.getPaymentMethods(account.getId(), true, callContext);
-                for (final PaymentMethodInfoPlugin paymentMethodInfoPlugin : paymentMethodInfos) {
-                    final PaymentMethodPlugin paymentMethodDetail = pluginApi.getPaymentMethodDetail(account.getId(), paymentMethodInfoPlugin.getPaymentMethodId(), tenantContext);
-                    paymentMethods.add(paymentMethodDetail);
-                    if (paymentMethods.size() > offset + limit) {
-                        break;
-                    }
-                }
-            } catch (PaymentPluginApiException e) {
-                throw new PaymentApiException(e, ErrorCode.PAYMENT_PLUGIN_GET_PAYMENT_METHODS, pluginName);
-            }
-        }
+        final Pagination<PaymentMethodModelDao> paymentMethodModelDaos = paymentDao.getPaymentMethods(pluginName, offset, limit, internalTenantContext);
 
-        return new DefaultPagination<PaymentMethod>(DefaultPagination.<PaymentMethodPlugin>build(offset, limit, paymentMethods),
+        return new DefaultPagination<PaymentMethod>(paymentMethodModelDaos,
                                                     limit,
-                                                    Iterators.<PaymentMethod>filter(Iterators.<PaymentMethodPlugin, PaymentMethod>transform(paymentMethods.iterator(),
-                                                                                                                                            new Function<PaymentMethodPlugin, PaymentMethod>() {
-                                                                                                                                                @Override
-                                                                                                                                                public PaymentMethod apply(final PaymentMethodPlugin paymentMethodPlugin) {
-                                                                                                                                                    if (paymentMethodPlugin.getKbPaymentMethodId() == null) {
-                                                                                                                                                        // Garbage from the plugin?
-                                                                                                                                                        log.debug("Plugin {} returned a payment method without a kbPaymentMethodId", pluginName);
-                                                                                                                                                        return null;
-                                                                                                                                                    }
-
-                                                                                                                                                    final PaymentMethodModelDao paymentMethodModelDao = paymentDao.getPaymentMethodIncludedDeleted(paymentMethodPlugin.getKbPaymentMethodId(), internalTenantContext);
-                                                                                                                                                    if (paymentMethodModelDao == null) {
-                                                                                                                                                        log.warn("Unable to find payment method id " + paymentMethodPlugin.getKbPaymentMethodId() + " present in plugin " + pluginName);
-                                                                                                                                                        return null;
-                                                                                                                                                    }
-
-                                                                                                                                                    return new DefaultPaymentMethod(paymentMethodModelDao, paymentMethodPlugin);
-                                                                                                                                                }
-                                                                                                                                            }),
+                                                    Iterators.<PaymentMethod>filter(Iterators.<PaymentMethodModelDao, PaymentMethod>transform(paymentMethodModelDaos.iterator(),
+                                                                                                                                              new Function<PaymentMethodModelDao, PaymentMethod>() {
+                                                                                                                                                  @Override
+                                                                                                                                                  public PaymentMethod apply(final PaymentMethodModelDao paymentMethodModelDao) {
+                                                                                                                                                      final PaymentMethodPlugin paymentMethodPlugin;
+                                                                                                                                                      try {
+                                                                                                                                                          paymentMethodPlugin = pluginApi.getPaymentMethodDetail(paymentMethodModelDao.getAccountId(), paymentMethodModelDao.getId(), tenantContext);
+                                                                                                                                                      } catch (PaymentPluginApiException e) {
+                                                                                                                                                          log.warn("Unable to find payment method id " + paymentMethodModelDao.getId() + " in plugin " + pluginName);
+                                                                                                                                                          return null;
+                                                                                                                                                      }
+
+                                                                                                                                                      if (paymentMethodPlugin.getKbPaymentMethodId() == null) {
+                                                                                                                                                          // Garbage from the plugin?
+                                                                                                                                                          log.debug("Plugin {} returned a payment method without a kbPaymentMethodId", pluginName);
+                                                                                                                                                          return null;
+                                                                                                                                                      }
+
+                                                                                                                                                      return new DefaultPaymentMethod(paymentMethodModelDao, paymentMethodPlugin);
+                                                                                                                                                  }
+                                                                                                                                              }),
                                                                                     Predicates.<PaymentMethod>notNull()));
     }
 
@@ -232,22 +229,33 @@ public class PaymentMethodProcessor extends ProcessorBase {
         // returned by the Pagination objects from the plugins and we probably don't want to do that (if
         // one plugin gets it wrong, it may starve the others).
         final List<PaymentMethod> allResults = new LinkedList<PaymentMethod>();
+        Long totalNbRecords = 0L;
+        Long maxNbRecords = 0L;
 
         // Search in all plugins (we treat the full set of results as a union with respect to offset/limit)
+        boolean firstSearch = true;
         for (final String pluginName : getAvailablePlugins()) {
             try {
-                final Pagination<PaymentMethod> paymentMethods = searchPaymentMethods(searchKey, 0L, Long.MAX_VALUE, pluginName, internalTenantContext);
-                allResults.addAll(ImmutableList.<PaymentMethod>copyOf(paymentMethods));
-                if (allResults.size() > offset + limit) {
-                    break;
+                final Pagination<PaymentMethod> paymentMethods;
+                if (allResults.size() >= limit) {
+                    // We have enough results, we just keep going (limit 1) to get the stats
+                    paymentMethods = searchPaymentMethods(searchKey, firstSearch ? offset : 0L, 1L, pluginName, internalTenantContext);
+                    // Required to close database connections
+                    ImmutableList.<PaymentMethod>copyOf(paymentMethods);
+                } else {
+                    paymentMethods = searchPaymentMethods(searchKey, firstSearch ? offset : 0L, limit - allResults.size(), pluginName, internalTenantContext);
+                    allResults.addAll(ImmutableList.<PaymentMethod>copyOf(paymentMethods));
                 }
+                firstSearch = false;
+                totalNbRecords += paymentMethods.getTotalNbRecords();
+                maxNbRecords += paymentMethods.getMaxNbRecords();
             } catch (PaymentApiException e) {
                 log.warn("Error while searching plugin " + pluginName, e);
                 // Non-fatal, continue to search other plugins
             }
         }
 
-        return DefaultPagination.<PaymentMethod>build(offset, limit, allResults);
+        return new DefaultPagination<PaymentMethod>(offset, limit, totalNbRecords, maxNbRecords, allResults.iterator());
     }
 
     public Pagination<PaymentMethod> searchPaymentMethods(final String searchKey, final Long offset, final Long limit, final String pluginName, final InternalTenantContext internalTenantContext) throws PaymentApiException {
diff --git a/payment/src/main/java/com/ning/billing/payment/dao/DefaultPaymentDao.java b/payment/src/main/java/com/ning/billing/payment/dao/DefaultPaymentDao.java
index 0b50ee1..7c0b657 100644
--- a/payment/src/main/java/com/ning/billing/payment/dao/DefaultPaymentDao.java
+++ b/payment/src/main/java/com/ning/billing/payment/dao/DefaultPaymentDao.java
@@ -17,6 +17,7 @@
 package com.ning.billing.payment.dao;
 
 import java.util.Collection;
+import java.util.Iterator;
 import java.util.List;
 import java.util.UUID;
 
@@ -24,14 +25,16 @@ import javax.inject.Inject;
 
 import org.skife.jdbi.v2.IDBI;
 
-import com.ning.billing.payment.api.PaymentStatus;
-import com.ning.billing.payment.dao.RefundModelDao.RefundStatus;
-import com.ning.billing.util.cache.CacheControllerDispatcher;
 import com.ning.billing.callcontext.InternalCallContext;
 import com.ning.billing.callcontext.InternalTenantContext;
 import com.ning.billing.clock.Clock;
-import com.ning.billing.util.dao.NonEntityDao;
 import com.ning.billing.entity.EntityPersistenceException;
+import com.ning.billing.payment.api.PaymentStatus;
+import com.ning.billing.payment.dao.RefundModelDao.RefundStatus;
+import com.ning.billing.util.cache.CacheControllerDispatcher;
+import com.ning.billing.util.dao.NonEntityDao;
+import com.ning.billing.util.entity.DefaultPagination;
+import com.ning.billing.util.entity.Pagination;
 import com.ning.billing.util.entity.dao.EntitySqlDao;
 import com.ning.billing.util.entity.dao.EntitySqlDaoTransactionWrapper;
 import com.ning.billing.util.entity.dao.EntitySqlDaoTransactionalJdbiWrapper;
@@ -215,6 +218,34 @@ public class DefaultPaymentDao implements PaymentDao {
     }
 
     @Override
+    public Pagination<PaymentMethodModelDao> getPaymentMethods(final String pluginName, final Long offset, final Long limit, final InternalTenantContext context) {
+        // Note: the connection will be busy as we stream the results out: hence we cannot use
+        // SQL_CALC_FOUND_ROWS / FOUND_ROWS on the actual query.
+        // We still need to know the actual number of results, mainly for the UI so that it knows if it needs to fetch
+        // more pages. To do that, we perform a dummy search query with SQL_CALC_FOUND_ROWS (but limit 1).
+        final Long count = transactionalSqlDao.execute(new EntitySqlDaoTransactionWrapper<Long>() {
+            @Override
+            public Long inTransaction(final EntitySqlDaoWrapperFactory<EntitySqlDao> entitySqlDaoWrapperFactory) throws Exception {
+                final PaymentMethodSqlDao sqlDao = entitySqlDaoWrapperFactory.become(PaymentMethodSqlDao.class);
+                final Iterator<PaymentMethodModelDao> dumbIterator = sqlDao.getByPluginName(pluginName, offset, 1L, context);
+                // Make sure to go through the results to close the connection
+                while (dumbIterator.hasNext()) {
+                    dumbIterator.next();
+                }
+                return sqlDao.getFoundRows(context);
+            }
+        });
+
+        // We usually always want to wrap our queries in an EntitySqlDaoTransactionWrapper... except here.
+        // Since we want to stream the results out, we don't want to auto-commit when this method returns.
+        final PaymentMethodSqlDao paymentMethodSqlDao = transactionalSqlDao.onDemand(PaymentMethodSqlDao.class);
+        final Long totalCount = paymentMethodSqlDao.getCount(context);
+        final Iterator<PaymentMethodModelDao> results = paymentMethodSqlDao.getByPluginName(pluginName, offset, limit, context);
+
+        return new DefaultPagination<PaymentMethodModelDao>(offset, limit, count, totalCount, results);
+    }
+
+    @Override
     public void deletedPaymentMethod(final UUID paymentMethodId, final InternalCallContext context) {
         transactionalSqlDao.execute(new EntitySqlDaoTransactionWrapper<Void>() {
             @Override
diff --git a/payment/src/main/java/com/ning/billing/payment/dao/PaymentDao.java b/payment/src/main/java/com/ning/billing/payment/dao/PaymentDao.java
index 3ea92e4..505df70 100644
--- a/payment/src/main/java/com/ning/billing/payment/dao/PaymentDao.java
+++ b/payment/src/main/java/com/ning/billing/payment/dao/PaymentDao.java
@@ -25,6 +25,7 @@ import com.ning.billing.payment.api.PaymentStatus;
 import com.ning.billing.payment.dao.RefundModelDao.RefundStatus;
 import com.ning.billing.callcontext.InternalCallContext;
 import com.ning.billing.callcontext.InternalTenantContext;
+import com.ning.billing.util.entity.Pagination;
 
 public interface PaymentDao {
 
@@ -65,6 +66,8 @@ public interface PaymentDao {
 
     public List<PaymentMethodModelDao> getPaymentMethods(UUID accountId, InternalTenantContext context);
 
+    public Pagination<PaymentMethodModelDao> getPaymentMethods(String pluginName, Long offset, Long limit, InternalTenantContext context);
+
     public void deletedPaymentMethod(UUID paymentMethodId, InternalCallContext context);
 
     public List<PaymentMethodModelDao> refreshPaymentMethods(UUID accountId, String pluginName, List<PaymentMethodModelDao> paymentMethods, InternalCallContext context);
diff --git a/payment/src/main/java/com/ning/billing/payment/dao/PaymentMethodSqlDao.java b/payment/src/main/java/com/ning/billing/payment/dao/PaymentMethodSqlDao.java
index cd62daf..d5af8bf 100644
--- a/payment/src/main/java/com/ning/billing/payment/dao/PaymentMethodSqlDao.java
+++ b/payment/src/main/java/com/ning/billing/payment/dao/PaymentMethodSqlDao.java
@@ -16,17 +16,20 @@
 
 package com.ning.billing.payment.dao;
 
+import java.util.Iterator;
 import java.util.List;
 
 import org.skife.jdbi.v2.sqlobject.Bind;
 import org.skife.jdbi.v2.sqlobject.BindBean;
 import org.skife.jdbi.v2.sqlobject.SqlQuery;
 import org.skife.jdbi.v2.sqlobject.SqlUpdate;
+import org.skife.jdbi.v2.sqlobject.customizers.Define;
+import org.skife.jdbi.v2.sqlobject.customizers.FetchSize;
 
-import com.ning.billing.payment.api.PaymentMethod;
-import com.ning.billing.util.audit.ChangeType;
 import com.ning.billing.callcontext.InternalCallContext;
 import com.ning.billing.callcontext.InternalTenantContext;
+import com.ning.billing.payment.api.PaymentMethod;
+import com.ning.billing.util.audit.ChangeType;
 import com.ning.billing.util.entity.dao.Audited;
 import com.ning.billing.util.entity.dao.EntitySqlDao;
 import com.ning.billing.util.entity.dao.EntitySqlDaoStringTemplate;
@@ -53,4 +56,13 @@ public interface PaymentMethodSqlDao extends EntitySqlDao<PaymentMethodModelDao,
 
     @SqlQuery
     List<PaymentMethodModelDao> getByAccountIdIncludedDelete(@Bind("accountId") final String accountId, @BindBean final InternalTenantContext context);
+
+    @SqlQuery
+    // Magic value to force MySQL to stream from the database
+    // See http://dev.mysql.com/doc/refman/5.0/en/connector-j-reference-implementation-notes.html (ResultSet)
+    @FetchSize(Integer.MIN_VALUE)
+    public Iterator<PaymentMethodModelDao> getByPluginName(@Bind("pluginName") final String pluginName,
+                                                           @Bind("offset") final Long offset,
+                                                           @Bind("rowCount") final Long rowCount,
+                                                           @BindBean final InternalTenantContext context);
 }
diff --git a/payment/src/main/java/com/ning/billing/payment/provider/DefaultNoOpPaymentMethodPlugin.java b/payment/src/main/java/com/ning/billing/payment/provider/DefaultNoOpPaymentMethodPlugin.java
index d431522..1ab82f0 100644
--- a/payment/src/main/java/com/ning/billing/payment/provider/DefaultNoOpPaymentMethodPlugin.java
+++ b/payment/src/main/java/com/ning/billing/payment/provider/DefaultNoOpPaymentMethodPlugin.java
@@ -19,6 +19,8 @@ package com.ning.billing.payment.provider;
 import java.util.List;
 import java.util.UUID;
 
+import javax.annotation.Nullable;
+
 import com.ning.billing.payment.api.PaymentMethodKVInfo;
 import com.ning.billing.payment.api.PaymentMethodPlugin;
 
@@ -39,7 +41,14 @@ public class DefaultNoOpPaymentMethodPlugin implements PaymentMethodPlugin {
     public DefaultNoOpPaymentMethodPlugin(final String externalId,
                                           final boolean isDefault,
                                           final List<PaymentMethodKVInfo> props) {
-        this.kbPaymentMethodId = null;
+        this(null, externalId, isDefault, props);
+    }
+
+    public DefaultNoOpPaymentMethodPlugin(@Nullable final UUID kbPaymentMethodId,
+                                          final String externalId,
+                                          final boolean isDefault,
+                                          final List<PaymentMethodKVInfo> props) {
+        this.kbPaymentMethodId = kbPaymentMethodId;
         this.externalId = externalId;
         this.isDefault = isDefault;
         this.props = props;
diff --git a/payment/src/main/java/com/ning/billing/payment/provider/DefaultNoOpPaymentProviderPlugin.java b/payment/src/main/java/com/ning/billing/payment/provider/DefaultNoOpPaymentProviderPlugin.java
index f934d59..f33a754 100644
--- a/payment/src/main/java/com/ning/billing/payment/provider/DefaultNoOpPaymentProviderPlugin.java
+++ b/payment/src/main/java/com/ning/billing/payment/provider/DefaultNoOpPaymentProviderPlugin.java
@@ -160,7 +160,7 @@ public class DefaultNoOpPaymentProviderPlugin implements NoOpPaymentPluginApi {
 
     @Override
     public Pagination<PaymentMethodPlugin> searchPaymentMethods(final String searchKey, final Long offset, final Long limit, final TenantContext tenantContext) throws PaymentPluginApiException {
-        final ImmutableList<PaymentMethodPlugin> results = ImmutableList.<PaymentMethodPlugin>copyOf(Iterables.<PaymentMethodPlugin>filter(Iterables.<PaymentMethodPlugin>concat(paymentMethods.values()), new Predicate<PaymentMethodPlugin>() {
+        final ImmutableList<PaymentMethodPlugin> allResults = ImmutableList.<PaymentMethodPlugin>copyOf(Iterables.<PaymentMethodPlugin>filter(Iterables.<PaymentMethodPlugin>concat(paymentMethods.values()), new Predicate<PaymentMethodPlugin>() {
             @Override
             public boolean apply(final PaymentMethodPlugin input) {
                 return (input.getAddress1() != null && input.getAddress1().contains(searchKey)) ||
@@ -172,7 +172,17 @@ public class DefaultNoOpPaymentProviderPlugin implements NoOpPaymentPluginApi {
                        (input.getCountry() != null && input.getCountry().contains(searchKey));
             }
         }));
-        return DefaultPagination.<PaymentMethodPlugin>build(offset, limit, results);
+
+        final List<PaymentMethodPlugin> results;
+        if (offset >= allResults.size()) {
+            results = ImmutableList.<PaymentMethodPlugin>of();
+        } else if (offset + limit > allResults.size()) {
+            results = allResults.subList(offset.intValue(), allResults.size());
+        } else {
+            results = allResults.subList(offset.intValue(), offset.intValue() + limit.intValue());
+        }
+
+        return new DefaultPagination<PaymentMethodPlugin>(offset, limit, (long) results.size(), (long) paymentMethods.values().size(), results.iterator());
     }
 
     @Override
diff --git a/payment/src/main/resources/com/ning/billing/payment/dao/PaymentMethodSqlDao.sql.stg b/payment/src/main/resources/com/ning/billing/payment/dao/PaymentMethodSqlDao.sql.stg
index 755a1ee..8652152 100644
--- a/payment/src/main/resources/com/ning/billing/payment/dao/PaymentMethodSqlDao.sql.stg
+++ b/payment/src/main/resources/com/ning/billing/payment/dao/PaymentMethodSqlDao.sql.stg
@@ -68,3 +68,14 @@ from <tableName()>
 where account_id = :accountId
 ;
 >>
+
+getByPluginName(pluginName, offset, rowCount) ::= <<
+select SQL_CALC_FOUND_ROWS
+<allTableFields()>
+from <tableName()>
+where plugin_name = :pluginName
+and is_active = 1
+order by record_id
+limit :offset, :rowCount
+;
+>>
diff --git a/payment/src/test/java/com/ning/billing/payment/dao/MockPaymentDao.java b/payment/src/test/java/com/ning/billing/payment/dao/MockPaymentDao.java
index 2c5e51f..418301b 100644
--- a/payment/src/test/java/com/ning/billing/payment/dao/MockPaymentDao.java
+++ b/payment/src/test/java/com/ning/billing/payment/dao/MockPaymentDao.java
@@ -25,10 +25,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 
-import com.ning.billing.payment.api.PaymentStatus;
-import com.ning.billing.payment.dao.RefundModelDao.RefundStatus;
 import com.ning.billing.callcontext.InternalCallContext;
 import com.ning.billing.callcontext.InternalTenantContext;
+import com.ning.billing.payment.api.PaymentStatus;
+import com.ning.billing.payment.dao.RefundModelDao.RefundStatus;
+import com.ning.billing.util.entity.Pagination;
 
 import com.google.common.collect.ImmutableList;
 
@@ -150,6 +151,11 @@ public class MockPaymentDao implements PaymentDao {
     }
 
     @Override
+    public Pagination<PaymentMethodModelDao> getPaymentMethods(final String pluginName, final Long offset, final Long limit, final InternalTenantContext context) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
     public void deletedPaymentMethod(final UUID paymentMethodId, final InternalCallContext context) {
         final Iterator<PaymentMethodModelDao> it = paymentMethods.iterator();
         while (it.hasNext()) {
diff --git a/util/src/main/java/com/ning/billing/util/entity/DefaultPagination.java b/util/src/main/java/com/ning/billing/util/entity/DefaultPagination.java
index e766255..71389d1 100644
--- a/util/src/main/java/com/ning/billing/util/entity/DefaultPagination.java
+++ b/util/src/main/java/com/ning/billing/util/entity/DefaultPagination.java
@@ -33,7 +33,7 @@ public class DefaultPagination<T> implements Pagination<T> {
     private final Long maxNbRecords;
     private final Iterator<T> delegateIterator;
 
-    // Builder when the streaming API can't be used
+    // Builder when the streaming API can't be used (should only be used for tests)
     // Notes: elements should be the entire records set (regardless of filtering) otherwise maxNbRecords won't be accurate
     public static <T> Pagination<T> build(final Long offset, final Long limit, final Collection<T> elements) {
         final List<T> allResults = ImmutableList.<T>copyOf(elements);