killbill-memoizeit
Details
diff --git a/api/src/main/java/com/ning/billing/util/api/ColumnInfo.java b/api/src/main/java/com/ning/billing/util/api/ColumnInfo.java
new file mode 100644
index 0000000..6896a2a
--- /dev/null
+++ b/api/src/main/java/com/ning/billing/util/api/ColumnInfo.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2010-2012 Ning, Inc.
+ *
+ * Ning licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.ning.billing.util.api;
+
+public interface ColumnInfo {
+
+ public String getTableName();
+
+ public String getColumnName();
+
+ public String getDataType();
+}
diff --git a/api/src/main/java/com/ning/billing/util/api/DatabaseExportOutputStream.java b/api/src/main/java/com/ning/billing/util/api/DatabaseExportOutputStream.java
new file mode 100644
index 0000000..4133459
--- /dev/null
+++ b/api/src/main/java/com/ning/billing/util/api/DatabaseExportOutputStream.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2010-2012 Ning, Inc.
+ *
+ * Ning licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.ning.billing.util.api;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public interface DatabaseExportOutputStream {
+
+ /**
+ * Notify the stream that the following data is for a different table
+ *
+ * @param tableName the following table name
+ * @param columnsForTable database columns
+ */
+ public void newTable(String tableName, List<ColumnInfo> columnsForTable);
+
+ /**
+ * Write one row of data
+ *
+ * @param row row of data
+ * @throws IOException generic IOException
+ */
+ public void write(Map<String, Object> row) throws IOException;
+}
diff --git a/api/src/main/java/com/ning/billing/util/api/ExportUserApi.java b/api/src/main/java/com/ning/billing/util/api/ExportUserApi.java
new file mode 100644
index 0000000..34cbe42
--- /dev/null
+++ b/api/src/main/java/com/ning/billing/util/api/ExportUserApi.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2010-2012 Ning, Inc.
+ *
+ * Ning licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.ning.billing.util.api;
+
+import java.io.OutputStream;
+import java.util.UUID;
+
+import com.ning.billing.util.callcontext.CallContext;
+
+// Although it's a read-only call, we want to know who triggered the export - hence the call context here
+public interface ExportUserApi {
+
+ public void exportDataForAccount(UUID accountId, DatabaseExportOutputStream out, CallContext context);
+
+ public void exportDataAsCSVForAccount(UUID accountId, OutputStream out, CallContext context);
+}
bin/import-account 40(+40 -0)
diff --git a/bin/import-account b/bin/import-account
new file mode 100755
index 0000000..f4e50c9
--- /dev/null
+++ b/bin/import-account
@@ -0,0 +1,40 @@
+#! /usr/bin/env bash
+
+###################################################################################
+# #
+# Copyright 2010-2012 Ning, Inc. #
+# #
+# Ning licenses this file to you under the Apache License, version 2.0 #
+# (the "License"); you may not use this file except in compliance with the #
+# License. You may obtain a copy of the License at: #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+# Unless required by applicable law or agreed to in writing, software #
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT #
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the #
+# License for the specific language governing permissions and limitations #
+# under the License. #
+# #
+###################################################################################
+
+set -e
+
+# Killbill server
+KILLBILL_URL=http://127.0.0.1:8080
+# Destination database
+DATABASE=killbill
+USERNAME=root
+PASSWORD=root
+# Temporary directory
+TMP_DIR=/var/tmp
+
+WHO=`whoami`
+cd $TMP_DIR
+rm -f xa*
+curl $KILLBILL_URL/1.0/kb/export/$1 -H"X-Killbill-CreatedBy: $WHO" | split -p '--' --
+for i in `ls xa*`; do
+ table_name=$(cat $i | head -1 | awk '{print $2}')
+ mv $i $table_name
+ mysqlimport --ignore-lines=1 --fields-terminated-by=, --fields-enclosed-by=\" --verbose -u$USERNAME -p$PASSWORD $DATABASE $TMP_DIR/$table_name
+done
diff --git a/jaxrs/src/main/java/com/ning/billing/jaxrs/resources/ExportResource.java b/jaxrs/src/main/java/com/ning/billing/jaxrs/resources/ExportResource.java
new file mode 100644
index 0000000..bdabb43
--- /dev/null
+++ b/jaxrs/src/main/java/com/ning/billing/jaxrs/resources/ExportResource.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2010-2012 Ning, Inc.
+ *
+ * Ning licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.ning.billing.jaxrs.resources;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.UUID;
+
+import javax.inject.Inject;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.GET;
+import javax.ws.rs.HeaderParam;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.StreamingOutput;
+
+import com.ning.billing.jaxrs.util.Context;
+import com.ning.billing.jaxrs.util.JaxrsUriBuilder;
+import com.ning.billing.util.api.AuditUserApi;
+import com.ning.billing.util.api.CustomFieldUserApi;
+import com.ning.billing.util.api.ExportUserApi;
+import com.ning.billing.util.api.TagUserApi;
+import com.ning.billing.util.callcontext.CallContext;
+
+import com.google.inject.Singleton;
+
+import static javax.ws.rs.core.MediaType.TEXT_PLAIN;
+
+@Singleton
+@Path(JaxrsResource.EXPORT_PATH)
+public class ExportResource extends JaxRsResourceBase {
+
+ private final ExportUserApi exportUserApi;
+
+ @Inject
+ public ExportResource(final ExportUserApi exportUserApi,
+ final JaxrsUriBuilder uriBuilder,
+ final TagUserApi tagUserApi,
+ final CustomFieldUserApi customFieldUserApi,
+ final AuditUserApi auditUserApi,
+ final Context context) {
+ super(uriBuilder, tagUserApi, customFieldUserApi, auditUserApi, context);
+ this.exportUserApi = exportUserApi;
+ }
+
+ @GET
+ @Path("/{accountId:" + UUID_PATTERN + "}")
+ @Produces(TEXT_PLAIN)
+ public StreamingOutput exportDataForAccount(@PathParam("accountId") final String accountId,
+ @HeaderParam(HDR_CREATED_BY) final String createdBy,
+ @HeaderParam(HDR_REASON) final String reason,
+ @HeaderParam(HDR_COMMENT) final String comment,
+ @javax.ws.rs.core.Context final HttpServletRequest request) {
+ final CallContext callContext = context.createContext(createdBy, reason, comment, request);
+ return new StreamingOutput() {
+ @Override
+ public void write(final OutputStream output) throws IOException, WebApplicationException {
+ // CSV by default for now
+ exportUserApi.exportDataAsCSVForAccount(UUID.fromString(accountId), output, callContext);
+ }
+ };
+ }
+}
diff --git a/jaxrs/src/main/java/com/ning/billing/jaxrs/resources/JaxrsResource.java b/jaxrs/src/main/java/com/ning/billing/jaxrs/resources/JaxrsResource.java
index e89acbe..fb16a50 100644
--- a/jaxrs/src/main/java/com/ning/billing/jaxrs/resources/JaxrsResource.java
+++ b/jaxrs/src/main/java/com/ning/billing/jaxrs/resources/JaxrsResource.java
@@ -126,4 +126,7 @@ public interface JaxrsResource {
public static final String TENANTS = "tenants";
public static final String TENANTS_PATH = PREFIX + "/" + TENANTS;
+
+ public static final String EXPORT = "export";
+ public static final String EXPORT_PATH = PREFIX + "/" + EXPORT;
}
pom.xml 5(+5 -0)
diff --git a/pom.xml b/pom.xml
index 3bed690..ae2af3a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -248,6 +248,11 @@
<version>2.0.0</version>
</dependency>
<dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-csv</artifactId>
+ <version>2.1.0</version>
+ </dependency>
+ <dependency>
<groupId>com.fasterxml.util</groupId>
<artifactId>low-gc-membuffers</artifactId>
<version>0.9.0</version>
diff --git a/server/src/main/java/com/ning/billing/server/modules/KillbillServerModule.java b/server/src/main/java/com/ning/billing/server/modules/KillbillServerModule.java
index bc27be2..4676cb4 100644
--- a/server/src/main/java/com/ning/billing/server/modules/KillbillServerModule.java
+++ b/server/src/main/java/com/ning/billing/server/modules/KillbillServerModule.java
@@ -20,7 +20,6 @@ package com.ning.billing.server.modules;
import org.skife.jdbi.v2.DBI;
import org.skife.jdbi.v2.IDBI;
-import com.google.inject.AbstractModule;
import com.ning.billing.account.glue.DefaultAccountModule;
import com.ning.billing.analytics.setup.AnalyticsModule;
import com.ning.billing.beatrix.glue.BeatrixModule;
@@ -49,11 +48,14 @@ import com.ning.billing.util.glue.BusModule;
import com.ning.billing.util.glue.CallContextModule;
import com.ning.billing.util.glue.ClockModule;
import com.ning.billing.util.glue.CustomFieldModule;
+import com.ning.billing.util.glue.ExportModule;
import com.ning.billing.util.glue.GlobalLockerModule;
import com.ning.billing.util.glue.NotificationQueueModule;
import com.ning.billing.util.glue.TagStoreModule;
import com.ning.jetty.jdbi.guice.providers.DBIProvider;
+import com.google.inject.AbstractModule;
+
public class KillbillServerModule extends AbstractModule {
@Override
protected void configure() {
@@ -111,6 +113,7 @@ public class KillbillServerModule extends AbstractModule {
install(new DefaultJunctionModule());
install(new DefaultOverdueModule());
install(new TenantModule());
+ install(new ExportModule());
installClock();
}
}
diff --git a/server/src/test/java/com/ning/billing/jaxrs/TestJaxrsBase.java b/server/src/test/java/com/ning/billing/jaxrs/TestJaxrsBase.java
index bf23bc3..eb0b9c1 100644
--- a/server/src/test/java/com/ning/billing/jaxrs/TestJaxrsBase.java
+++ b/server/src/test/java/com/ning/billing/jaxrs/TestJaxrsBase.java
@@ -62,6 +62,7 @@ import com.ning.billing.util.glue.AuditModule;
import com.ning.billing.util.glue.BusModule;
import com.ning.billing.util.glue.CallContextModule;
import com.ning.billing.util.glue.CustomFieldModule;
+import com.ning.billing.util.glue.ExportModule;
import com.ning.billing.util.glue.GlobalLockerModule;
import com.ning.billing.util.glue.NotificationQueueModule;
import com.ning.billing.util.glue.TagStoreModule;
@@ -182,6 +183,7 @@ public class TestJaxrsBase extends KillbillClient {
install(new DefaultJunctionModule());
install(new DefaultOverdueModule());
install(new TenantModule());
+ install(new ExportModule());
installClock();
}
util/pom.xml 4(+4 -0)
diff --git a/util/pom.xml b/util/pom.xml
index 5ed0228..a5ad4e0 100644
--- a/util/pom.xml
+++ b/util/pom.xml
@@ -104,6 +104,10 @@
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-joda</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-csv</artifactId>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/util/src/main/java/com/ning/billing/util/export/api/DefaultExportUserApi.java b/util/src/main/java/com/ning/billing/util/export/api/DefaultExportUserApi.java
new file mode 100644
index 0000000..5954883
--- /dev/null
+++ b/util/src/main/java/com/ning/billing/util/export/api/DefaultExportUserApi.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2010-2012 Ning, Inc.
+ *
+ * Ning licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.ning.billing.util.export.api;
+
+import java.io.OutputStream;
+import java.util.UUID;
+
+import javax.inject.Inject;
+
+import com.ning.billing.util.api.DatabaseExportOutputStream;
+import com.ning.billing.util.api.ExportUserApi;
+import com.ning.billing.util.callcontext.CallContext;
+import com.ning.billing.util.callcontext.InternalCallContext;
+import com.ning.billing.util.callcontext.InternalCallContextFactory;
+import com.ning.billing.util.export.dao.CSVExportOutputStream;
+import com.ning.billing.util.export.dao.DatabaseExportDao;
+
+public class DefaultExportUserApi implements ExportUserApi {
+
+ private final DatabaseExportDao exportDao;
+ private final InternalCallContextFactory internalCallContextFactory;
+
+ @Inject
+ public DefaultExportUserApi(final DatabaseExportDao exportDao,
+ final InternalCallContextFactory internalCallContextFactory) {
+ this.exportDao = exportDao;
+ this.internalCallContextFactory = internalCallContextFactory;
+ }
+
+ @Override
+ public void exportDataForAccount(final UUID accountId, final DatabaseExportOutputStream out, final CallContext context) {
+ final InternalCallContext internalContext = internalCallContextFactory.createInternalCallContext(accountId, context);
+ exportDao.exportDataForAccount(out, internalContext);
+ }
+
+ @Override
+ public void exportDataAsCSVForAccount(final UUID accountId, final OutputStream out, final CallContext context) {
+ exportDataForAccount(accountId, new CSVExportOutputStream(out), context);
+ }
+}
diff --git a/util/src/main/java/com/ning/billing/util/export/dao/CSVExportOutputStream.java b/util/src/main/java/com/ning/billing/util/export/dao/CSVExportOutputStream.java
new file mode 100644
index 0000000..ec7fb54
--- /dev/null
+++ b/util/src/main/java/com/ning/billing/util/export/dao/CSVExportOutputStream.java
@@ -0,0 +1,134 @@
+/*
+ * Copyright 2010-2012 Ning, Inc.
+ *
+ * Ning licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.ning.billing.util.export.dao;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.List;
+import java.util.Map;
+
+import com.ning.billing.util.api.ColumnInfo;
+import com.ning.billing.util.api.DatabaseExportOutputStream;
+
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.dataformat.csv.CsvMapper;
+import com.fasterxml.jackson.dataformat.csv.CsvSchema;
+import com.fasterxml.jackson.dataformat.csv.CsvSchema.ColumnType;
+
+public class CSVExportOutputStream extends OutputStream implements DatabaseExportOutputStream {
+
+ private static final CsvMapper mapper = new CsvMapper();
+
+ private final OutputStream delegate;
+
+ private String currentTableName;
+ private CsvSchema currentCSVSchema;
+ private ObjectWriter writer;
+ private boolean shouldWriteHeader = false;
+
+ public CSVExportOutputStream(final OutputStream delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public void write(final int b) throws IOException {
+ delegate.write(b);
+ }
+
+ @Override
+ public String toString() {
+ return delegate.toString();
+ }
+
+ @Override
+ public void newTable(final String tableName, final List<ColumnInfo> columnsForTable) {
+ currentTableName = tableName;
+
+ final CsvSchema.Builder builder = CsvSchema.builder();
+ for (final ColumnInfo columnInfo : columnsForTable) {
+ builder.addColumn(columnInfo.getColumnName(), getColumnTypeFromSqlType(columnInfo.getDataType()));
+ }
+ currentCSVSchema = builder.build();
+
+ writer = mapper.writer(currentCSVSchema);
+ shouldWriteHeader = true;
+ }
+
+ @Override
+ public void write(final Map<String, Object> row) throws IOException {
+ final byte[] bytes;
+ if (shouldWriteHeader) {
+ // Write the header once (mapper.writer will clone the writer). Add a small marker in front of the header
+ // to easily split it
+ write(String.format("-- %s ", currentTableName).getBytes());
+ bytes = mapper.writer(currentCSVSchema.withHeader()).writeValueAsBytes(row);
+ shouldWriteHeader = false;
+ } else {
+ bytes = writer.writeValueAsBytes(row);
+ }
+
+ write(bytes);
+ }
+
+ private ColumnType getColumnTypeFromSqlType(final String dataType) {
+ if (dataType == null) {
+ return ColumnType.STRING;
+ } else if ("bigint".equals(dataType)) {
+ return ColumnType.NUMBER_OR_STRING;
+ } else if ("blob".equals(dataType)) {
+ return ColumnType.STRING;
+ } else if ("char".equals(dataType)) {
+ return ColumnType.STRING;
+ } else if ("date".equals(dataType)) {
+ return ColumnType.STRING;
+ } else if ("datetime".equals(dataType)) {
+ return ColumnType.STRING;
+ } else if ("decimal".equals(dataType)) {
+ return ColumnType.NUMBER_OR_STRING;
+ } else if ("enum".equals(dataType)) {
+ return ColumnType.STRING;
+ } else if ("int".equals(dataType)) {
+ return ColumnType.NUMBER_OR_STRING;
+ } else if ("longblob".equals(dataType)) {
+ return ColumnType.STRING;
+ } else if ("longtext".equals(dataType)) {
+ return ColumnType.STRING;
+ } else if ("mediumblob".equals(dataType)) {
+ return ColumnType.STRING;
+ } else if ("mediumtext".equals(dataType)) {
+ return ColumnType.STRING;
+ } else if ("set".equals(dataType)) {
+ return ColumnType.STRING;
+ } else if ("smallint".equals(dataType)) {
+ return ColumnType.NUMBER_OR_STRING;
+ } else if ("text".equals(dataType)) {
+ return ColumnType.STRING;
+ } else if ("time".equals(dataType)) {
+ return ColumnType.STRING;
+ } else if ("timestamp".equals(dataType)) {
+ return ColumnType.STRING;
+ } else if ("tinyint".equals(dataType)) {
+ return ColumnType.NUMBER_OR_STRING;
+ } else if ("varbinary".equals(dataType)) {
+ return ColumnType.STRING;
+ } else if ("varchar".equals(dataType)) {
+ return ColumnType.STRING;
+ } else {
+ return ColumnType.STRING;
+ }
+ }
+}
diff --git a/util/src/main/java/com/ning/billing/util/export/dao/DatabaseExportDao.java b/util/src/main/java/com/ning/billing/util/export/dao/DatabaseExportDao.java
new file mode 100644
index 0000000..434b013
--- /dev/null
+++ b/util/src/main/java/com/ning/billing/util/export/dao/DatabaseExportDao.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2010-2012 Ning, Inc.
+ *
+ * Ning licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.ning.billing.util.export.dao;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.skife.jdbi.v2.Handle;
+import org.skife.jdbi.v2.IDBI;
+import org.skife.jdbi.v2.ResultIterator;
+import org.skife.jdbi.v2.tweak.HandleCallback;
+
+import com.ning.billing.util.api.ColumnInfo;
+import com.ning.billing.util.api.DatabaseExportOutputStream;
+import com.ning.billing.util.callcontext.InternalTenantContext;
+import com.ning.billing.util.validation.DefaultColumnInfo;
+import com.ning.billing.util.validation.dao.DatabaseSchemaDao;
+
+@Singleton
+public class DatabaseExportDao {
+
+ private final DatabaseSchemaDao databaseSchemaDao;
+ private final IDBI dbi;
+
+ @Inject
+ public DatabaseExportDao(final DatabaseSchemaDao databaseSchemaDao,
+ final IDBI dbi) {
+ this.databaseSchemaDao = databaseSchemaDao;
+ this.dbi = dbi;
+ }
+
+ public void exportDataForAccount(final DatabaseExportOutputStream out, final InternalTenantContext context) {
+ if (context.getAccountRecordId() == null || context.getTenantRecordId() == null) {
+ return;
+ }
+
+ final List<DefaultColumnInfo> columns = databaseSchemaDao.getColumnInfoList();
+ if (columns.size() == 0) {
+ return;
+ }
+
+ final List<ColumnInfo> columnsForTable = new ArrayList<ColumnInfo>();
+ // The list of columns is ordered by table name first
+ String lastSeenTableName = columns.get(0).getTableName();
+ for (final ColumnInfo column : columns) {
+ if (!column.getTableName().equals(lastSeenTableName)) {
+ exportDataForAccountAndTable(out, columnsForTable, context);
+ lastSeenTableName = column.getTableName();
+ columnsForTable.clear();
+ }
+ columnsForTable.add(column);
+ }
+ exportDataForAccountAndTable(out, columnsForTable, context);
+ }
+
+ private void exportDataForAccountAndTable(final DatabaseExportOutputStream out, final List<ColumnInfo> columnsForTable, final InternalTenantContext context) {
+ boolean hasAccountRecordIdColumn = false;
+ boolean firstColumn = true;
+ final StringBuilder queryBuilder = new StringBuilder("select ");
+ for (final ColumnInfo column : columnsForTable) {
+ if (!firstColumn) {
+ queryBuilder.append(", ");
+ } else {
+ firstColumn = false;
+ }
+
+ queryBuilder.append(column.getColumnName());
+ if (column.getColumnName().equals("account_record_id")) {
+ hasAccountRecordIdColumn = true;
+ }
+ }
+
+ // Don't export non-account specific tables - TODO accounts
+ if (!hasAccountRecordIdColumn) {
+ return;
+ }
+
+ // Build the query - make sure to filter by account and tenant!
+ final String tableName = columnsForTable.get(0).getTableName();
+ queryBuilder.append(" from ")
+ .append(tableName)
+ .append(" where account_record_id = :accountRecordId and tenant_record_id = :tenantRecordId");
+
+ // Notify the stream that we're about to write data for a different table
+ out.newTable(tableName, columnsForTable);
+
+ dbi.withHandle(new HandleCallback<Void>() {
+ @Override
+ public Void withHandle(final Handle handle) throws Exception {
+ final ResultIterator<Map<String, Object>> iterator = handle.createQuery(queryBuilder.toString())
+ .bind("accountRecordId", context.getAccountRecordId())
+ .bind("tenantRecordId", context.getTenantRecordId())
+ .iterator();
+ try {
+ while (iterator.hasNext()) {
+ final Map<String, Object> row = iterator.next();
+ out.write(row);
+ }
+ } finally {
+ iterator.close();
+ }
+
+ return null;
+ }
+ });
+ }
+}
diff --git a/util/src/main/java/com/ning/billing/util/glue/ExportModule.java b/util/src/main/java/com/ning/billing/util/glue/ExportModule.java
new file mode 100644
index 0000000..fe94020
--- /dev/null
+++ b/util/src/main/java/com/ning/billing/util/glue/ExportModule.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2010-2012 Ning, Inc.
+ *
+ * Ning licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.ning.billing.util.glue;
+
+import com.ning.billing.util.api.ExportUserApi;
+import com.ning.billing.util.export.api.DefaultExportUserApi;
+
+import com.google.inject.AbstractModule;
+
+public class ExportModule extends AbstractModule {
+
+ protected void installUserApi() {
+ bind(ExportUserApi.class).to(DefaultExportUserApi.class).asEagerSingleton();
+ }
+
+ @Override
+ protected void configure() {
+ installUserApi();
+ }
+}
diff --git a/util/src/main/java/com/ning/billing/util/validation/dao/DatabaseSchemaDao.java b/util/src/main/java/com/ning/billing/util/validation/dao/DatabaseSchemaDao.java
index a300e72..93f13c7 100644
--- a/util/src/main/java/com/ning/billing/util/validation/dao/DatabaseSchemaDao.java
+++ b/util/src/main/java/com/ning/billing/util/validation/dao/DatabaseSchemaDao.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2010-2011 Ning, Inc.
+ * Copyright 2010-2012 Ning, Inc.
*
* Ning licenses this file to you under the Apache License, version 2.0
* (the "License"); you may not use this file except in compliance with the
@@ -18,13 +18,18 @@ package com.ning.billing.util.validation.dao;
import java.util.List;
+import javax.annotation.Nullable;
+import javax.inject.Singleton;
+
import org.skife.jdbi.v2.IDBI;
-import com.ning.billing.util.validation.ColumnInfo;
+import com.ning.billing.util.validation.DefaultColumnInfo;
import com.google.inject.Inject;
+@Singleton
public class DatabaseSchemaDao {
+
private final DatabaseSchemaSqlDao dao;
@Inject
@@ -32,7 +37,11 @@ public class DatabaseSchemaDao {
this.dao = dbi.onDemand(DatabaseSchemaSqlDao.class);
}
- public List<ColumnInfo> getColumnInfoList(final String schemaName) {
+ public List<DefaultColumnInfo> getColumnInfoList() {
+ return getColumnInfoList(null);
+ }
+
+ public List<DefaultColumnInfo> getColumnInfoList(@Nullable final String schemaName) {
return dao.getSchemaInfo(schemaName);
}
}
diff --git a/util/src/main/java/com/ning/billing/util/validation/dao/DatabaseSchemaSqlDao.java b/util/src/main/java/com/ning/billing/util/validation/dao/DatabaseSchemaSqlDao.java
index 4d069b4..fad2b77 100644
--- a/util/src/main/java/com/ning/billing/util/validation/dao/DatabaseSchemaSqlDao.java
+++ b/util/src/main/java/com/ning/billing/util/validation/dao/DatabaseSchemaSqlDao.java
@@ -20,6 +20,8 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
+import javax.annotation.Nullable;
+
import org.skife.jdbi.v2.StatementContext;
import org.skife.jdbi.v2.sqlobject.Bind;
import org.skife.jdbi.v2.sqlobject.SqlQuery;
@@ -27,19 +29,19 @@ import org.skife.jdbi.v2.sqlobject.customizers.RegisterMapper;
import org.skife.jdbi.v2.sqlobject.stringtemplate.ExternalizedSqlViaStringTemplate3;
import org.skife.jdbi.v2.tweak.ResultSetMapper;
-import com.ning.billing.util.validation.ColumnInfo;
+import com.ning.billing.util.validation.DefaultColumnInfo;
@ExternalizedSqlViaStringTemplate3
@RegisterMapper(DatabaseSchemaSqlDao.ColumnInfoMapper.class)
public interface DatabaseSchemaSqlDao {
@SqlQuery
- List<ColumnInfo> getSchemaInfo(@Bind("schemaName") final String schemaName);
+ List<DefaultColumnInfo> getSchemaInfo(@Nullable @Bind("schemaName") final String schemaName);
- class ColumnInfoMapper implements ResultSetMapper<ColumnInfo> {
+ class ColumnInfoMapper implements ResultSetMapper<DefaultColumnInfo> {
@Override
- public ColumnInfo map(final int index, final ResultSet r, final StatementContext ctx) throws SQLException {
+ public DefaultColumnInfo map(final int index, final ResultSet r, final StatementContext ctx) throws SQLException {
final String tableName = r.getString("table_name");
final String columnName = r.getString("column_name");
final Integer scale = r.getInt("numeric_scale");
@@ -48,7 +50,7 @@ public interface DatabaseSchemaSqlDao {
final Integer maximumLength = r.getInt("character_maximum_length");
final String dataType = r.getString("data_type");
- return new ColumnInfo(tableName, columnName, scale, precision, isNullable, maximumLength, dataType);
+ return new DefaultColumnInfo(tableName, columnName, scale, precision, isNullable, maximumLength, dataType);
}
}
}
diff --git a/util/src/main/java/com/ning/billing/util/validation/ValidationConfiguration.java b/util/src/main/java/com/ning/billing/util/validation/ValidationConfiguration.java
index 9c27dfd..d716c9d 100644
--- a/util/src/main/java/com/ning/billing/util/validation/ValidationConfiguration.java
+++ b/util/src/main/java/com/ning/billing/util/validation/ValidationConfiguration.java
@@ -18,8 +18,8 @@ package com.ning.billing.util.validation;
import java.util.HashMap;
-public class ValidationConfiguration extends HashMap<String, ColumnInfo> {
- public void addMapping(final String propertyName, final ColumnInfo columnInfo) {
+public class ValidationConfiguration extends HashMap<String, DefaultColumnInfo> {
+ public void addMapping(final String propertyName, final DefaultColumnInfo columnInfo) {
super.put(propertyName, columnInfo);
}
diff --git a/util/src/main/java/com/ning/billing/util/validation/ValidationManager.java b/util/src/main/java/com/ning/billing/util/validation/ValidationManager.java
index 54e4bdf..1392f82 100644
--- a/util/src/main/java/com/ning/billing/util/validation/ValidationManager.java
+++ b/util/src/main/java/com/ning/billing/util/validation/ValidationManager.java
@@ -31,7 +31,7 @@ public class ValidationManager {
private final DatabaseSchemaDao dao;
// table name, string name, column info
- private final Map<String, Map<String, ColumnInfo>> columnInfoMap = new HashMap<String, Map<String, ColumnInfo>>();
+ private final Map<String, Map<String, DefaultColumnInfo>> columnInfoMap = new HashMap<String, Map<String, DefaultColumnInfo>>();
private final Map<Class, ValidationConfiguration> configurations = new HashMap<Class, ValidationConfiguration>();
@Inject
@@ -44,23 +44,23 @@ public class ValidationManager {
columnInfoMap.clear();
// get schema information and map it to columnInfo
- final List<ColumnInfo> columnInfoList = dao.getColumnInfoList(schemaName);
- for (final ColumnInfo columnInfo : columnInfoList) {
+ final List<DefaultColumnInfo> columnInfoList = dao.getColumnInfoList(schemaName);
+ for (final DefaultColumnInfo columnInfo : columnInfoList) {
final String tableName = columnInfo.getTableName();
if (!columnInfoMap.containsKey(tableName)) {
- columnInfoMap.put(tableName, new HashMap<String, ColumnInfo>());
+ columnInfoMap.put(tableName, new HashMap<String, DefaultColumnInfo>());
}
columnInfoMap.get(tableName).put(columnInfo.getColumnName(), columnInfo);
}
}
- public Collection<ColumnInfo> getTableInfo(final String tableName) {
+ public Collection<DefaultColumnInfo> getTableInfo(final String tableName) {
return columnInfoMap.get(tableName).values();
}
- public ColumnInfo getColumnInfo(final String tableName, final String columnName) {
+ public DefaultColumnInfo getColumnInfo(final String tableName, final String columnName) {
return (columnInfoMap.get(tableName) == null) ? null : columnInfoMap.get(tableName).get(columnName);
}
@@ -82,7 +82,7 @@ public class ValidationManager {
final Object value = field.get(o);
- final ColumnInfo columnInfo = configuration.get(propertyName);
+ final DefaultColumnInfo columnInfo = configuration.get(propertyName);
if (columnInfo == null) {
// no column info means the property hasn't been properly mapped; suppress validation
return true;
@@ -114,7 +114,7 @@ public class ValidationManager {
return true;
}
- private boolean hasValidNullability(final ColumnInfo columnInfo, final Object value) {
+ private boolean hasValidNullability(final DefaultColumnInfo columnInfo, final Object value) {
if (!columnInfo.getIsNullable()) {
if (value == null) {
return false;
@@ -124,7 +124,7 @@ public class ValidationManager {
return true;
}
- private boolean isValidLengthString(final ColumnInfo columnInfo, final Object value) {
+ private boolean isValidLengthString(final DefaultColumnInfo columnInfo, final Object value) {
if (columnInfo.getMaximumLength() != 0) {
if (value != null) {
if (value.toString().length() > columnInfo.getMaximumLength()) {
@@ -136,7 +136,7 @@ public class ValidationManager {
return true;
}
- private boolean isValidLengthChar(final ColumnInfo columnInfo, final Object value) {
+ private boolean isValidLengthChar(final DefaultColumnInfo columnInfo, final Object value) {
if (columnInfo.getDataType().equals("char")) {
if (value == null) {
return false;
@@ -150,7 +150,7 @@ public class ValidationManager {
return true;
}
- private boolean hasValidPrecision(final ColumnInfo columnInfo, final Object value) {
+ private boolean hasValidPrecision(final DefaultColumnInfo columnInfo, final Object value) {
if (columnInfo.getPrecision() != 0) {
if (value != null) {
final BigDecimal bigDecimalValue = new BigDecimal(value.toString());
@@ -163,7 +163,7 @@ public class ValidationManager {
return true;
}
- private boolean hasValidScale(final ColumnInfo columnInfo, final Object value) {
+ private boolean hasValidScale(final DefaultColumnInfo columnInfo, final Object value) {
if (columnInfo.getScale() != 0) {
if (value != null) {
final BigDecimal bigDecimalValue = new BigDecimal(value.toString());
@@ -184,7 +184,7 @@ public class ValidationManager {
return configurations.get(clazz);
}
- public void setConfiguration(final Class clazz, final String propertyName, final ColumnInfo columnInfo) {
+ public void setConfiguration(final Class clazz, final String propertyName, final DefaultColumnInfo columnInfo) {
if (!configurations.containsKey(clazz)) {
configurations.put(clazz, new ValidationConfiguration());
}
diff --git a/util/src/main/resources/com/ning/billing/util/validation/dao/DatabaseSchemaSqlDao.sql.stg b/util/src/main/resources/com/ning/billing/util/validation/dao/DatabaseSchemaSqlDao.sql.stg
index 1008369..e14db44 100644
--- a/util/src/main/resources/com/ning/billing/util/validation/dao/DatabaseSchemaSqlDao.sql.stg
+++ b/util/src/main/resources/com/ning/billing/util/validation/dao/DatabaseSchemaSqlDao.sql.stg
@@ -1,10 +1,10 @@
group DatabaseSchemaSqlDao;
-getSchemaInfo() ::= <<
+getSchemaInfo(schemaName) ::= <<
SELECT TABLE_NAME, COLUMN_NAME, IS_NULLABLE, DATA_TYPE,
CHARACTER_MAXIMUM_LENGTH, NUMERIC_PRECISION, NUMERIC_SCALE
FROM information_schema.columns
- WHERE TABLE_SCHEMA = :schemaName
- ORDER BY TABLE_NAME, COLUMN_NAME;
+ WHERE TABLE_SCHEMA = <if(schemaName)>schemaName<else>schema()<endif>
+ ORDER BY TABLE_NAME, ORDINAL_POSITION;
>>
diff --git a/util/src/test/java/com/ning/billing/util/export/dao/TestCSVExportOutputStream.java b/util/src/test/java/com/ning/billing/util/export/dao/TestCSVExportOutputStream.java
new file mode 100644
index 0000000..b55b487
--- /dev/null
+++ b/util/src/test/java/com/ning/billing/util/export/dao/TestCSVExportOutputStream.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2010-2012 Ning, Inc.
+ *
+ * Ning licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.ning.billing.util.export.dao;
+
+import java.io.ByteArrayOutputStream;
+import java.util.UUID;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import com.ning.billing.util.UtilTestSuite;
+import com.ning.billing.util.api.ColumnInfo;
+import com.ning.billing.util.validation.DefaultColumnInfo;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+public class TestCSVExportOutputStream extends UtilTestSuite {
+
+ @Test(groups = "fast")
+ public void testSimpleGenerator() throws Exception {
+ final CSVExportOutputStream out = new CSVExportOutputStream(new ByteArrayOutputStream());
+
+ // Create the schema
+ final String tableName = UUID.randomUUID().toString();
+ out.newTable(tableName,
+ ImmutableList.<ColumnInfo>of(
+ new DefaultColumnInfo(tableName, "first_name", 0, 0, true, 0, "varchar"),
+ new DefaultColumnInfo(tableName, "last_name", 0, 0, true, 0, "varchar"),
+ new DefaultColumnInfo(tableName, "age", 0, 0, true, 0, "tinyint"))
+ );
+
+ // Write some data
+ out.write(ImmutableMap.<String, Object>of("first_name", "jean",
+ "last_name", "dupond",
+ "age", 35));
+ // Don't assume "ordering"
+ out.write(ImmutableMap.<String, Object>of("last_name", "dujardin",
+ "first_name", "jack",
+ "age", 40));
+ out.write(ImmutableMap.<String, Object>of("age", 12,
+ "first_name", "pierre",
+ "last_name", "schmitt"));
+ // Verify the numeric parsing
+ out.write(ImmutableMap.<String, Object>of("first_name", "stephane",
+ "last_name", "dupont",
+ "age", "30"));
+
+ Assert.assertEquals(out.toString(), "-- " + tableName + " first_name,last_name,age\n" +
+ "jean,dupond,35\n" +
+ "jack,dujardin,40\n" +
+ "pierre,schmitt,12\n" +
+ "stephane,dupont,30\n");
+ }
+}
diff --git a/util/src/test/java/com/ning/billing/util/export/dao/TestDatabaseExportDao.java b/util/src/test/java/com/ning/billing/util/export/dao/TestDatabaseExportDao.java
new file mode 100644
index 0000000..8dd0322
--- /dev/null
+++ b/util/src/test/java/com/ning/billing/util/export/dao/TestDatabaseExportDao.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2010-2012 Ning, Inc.
+ *
+ * Ning licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.ning.billing.util.export.dao;
+
+import java.io.ByteArrayOutputStream;
+
+import org.skife.jdbi.v2.Handle;
+import org.skife.jdbi.v2.tweak.HandleCallback;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import com.ning.billing.util.UtilTestSuiteWithEmbeddedDB;
+import com.ning.billing.util.api.DatabaseExportOutputStream;
+import com.ning.billing.util.validation.dao.DatabaseSchemaDao;
+
+public class TestDatabaseExportDao extends UtilTestSuiteWithEmbeddedDB {
+
+ private DatabaseExportDao dao;
+
+ @BeforeMethod(groups = "slow")
+ public void setUp() throws Exception {
+ final DatabaseSchemaDao databaseSchemaDao = new DatabaseSchemaDao(getMysqlTestingHelper().getDBI());
+ dao = new DatabaseExportDao(databaseSchemaDao, getMysqlTestingHelper().getDBI());
+ }
+
+ @Test(groups = "slow")
+ public void testExportSimpleData() throws Exception {
+ // Empty database
+ final String dump = getDump();
+ Assert.assertEquals(dump, "");
+
+ final String tableNameA = "test_database_export_dao_a";
+ final String tableNameB = "test_database_export_dao_b";
+ getMysqlTestingHelper().getDBI().withHandle(new HandleCallback<Void>() {
+ @Override
+ public Void withHandle(final Handle handle) throws Exception {
+ handle.execute("create table " + tableNameA + "(record_id int(11) unsigned not null auto_increment," +
+ "a_column char default 'a'," +
+ "account_record_id int(11) unsigned not null," +
+ "tenant_record_id int(11) unsigned default 0," +
+ "primary key(record_id)) engine=innodb;");
+ handle.execute("create table " + tableNameB + "(record_id int(11) unsigned not null auto_increment," +
+ "b_column char default 'b'," +
+ "account_record_id int(11) unsigned not null," +
+ "tenant_record_id int(11) unsigned default 0," +
+ "primary key(record_id)) engine=innodb;");
+ handle.execute("insert into " + tableNameA + " (account_record_id, tenant_record_id) values (?, ?)",
+ internalCallContext.getAccountRecordId(), internalCallContext.getTenantRecordId());
+ handle.execute("insert into " + tableNameB + " (account_record_id, tenant_record_id) values (?, ?)",
+ internalCallContext.getAccountRecordId(), internalCallContext.getTenantRecordId());
+ return null;
+ }
+ });
+
+ // Verify new dump
+ final String newDump = getDump();
+ Assert.assertEquals(newDump, "-- " + tableNameA + " record_id,a_column,account_record_id,tenant_record_id\n" +
+ "1,a," + internalCallContext.getAccountRecordId() + "," + internalCallContext.getTenantRecordId() + "\n" +
+ "-- " + tableNameB + " record_id,b_column,account_record_id,tenant_record_id\n" +
+ "1,b," + internalCallContext.getAccountRecordId() + "," + internalCallContext.getTenantRecordId() + "\n");
+ }
+
+ private String getDump() {
+ final DatabaseExportOutputStream out = new CSVExportOutputStream(new ByteArrayOutputStream());
+ dao.exportDataForAccount(out, internalCallContext);
+ return out.toString();
+ }
+}
diff --git a/util/src/test/java/com/ning/billing/util/validation/TestValidationManager.java b/util/src/test/java/com/ning/billing/util/validation/TestValidationManager.java
index eac5741..a7690cd 100644
--- a/util/src/test/java/com/ning/billing/util/validation/TestValidationManager.java
+++ b/util/src/test/java/com/ning/billing/util/validation/TestValidationManager.java
@@ -63,12 +63,12 @@ public class TestValidationManager extends UtilTestSuiteWithEmbeddedDB {
@Test(groups = "slow")
public void testRetrievingColumnInfo() {
- final Collection<ColumnInfo> columnInfoList = vm.getTableInfo(TABLE_NAME);
+ final Collection<DefaultColumnInfo> columnInfoList = vm.getTableInfo(TABLE_NAME);
assertEquals(columnInfoList.size(), 4);
assertNotNull(vm.getColumnInfo(TABLE_NAME, "column1"));
assertNull(vm.getColumnInfo(TABLE_NAME, "bogus"));
- final ColumnInfo numericColumnInfo = vm.getColumnInfo(TABLE_NAME, "column3");
+ final DefaultColumnInfo numericColumnInfo = vm.getColumnInfo(TABLE_NAME, "column3");
assertNotNull(numericColumnInfo);
assertEquals(numericColumnInfo.getScale(), 4);
assertEquals(numericColumnInfo.getPrecision(), 10);