diff --git a/azkaban-common/src/main/java/azkaban/project/ProjectManager.java b/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
index 17865bb..9bcbf05 100644
--- a/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
+++ b/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
@@ -33,14 +33,14 @@ import org.apache.log4j.Logger;
import azkaban.flow.Flow;
import azkaban.project.ProjectLogEvent.EventType;
-import azkaban.project.validator.ValidationStatus;
import azkaban.project.validator.ValidationReport;
+import azkaban.project.validator.ValidationStatus;
import azkaban.project.validator.ValidatorConfigs;
import azkaban.project.validator.ValidatorManager;
import azkaban.project.validator.XmlValidatorManager;
import azkaban.user.Permission;
-import azkaban.user.User;
import azkaban.user.Permission.Type;
+import azkaban.user.User;
import azkaban.utils.DirectoryFlowLoader;
import azkaban.utils.Props;
import azkaban.utils.Utils;
@@ -48,7 +48,6 @@ import azkaban.utils.Utils;
public class ProjectManager {
private static final Logger logger = Logger.getLogger(ProjectManager.class);
-
private ConcurrentHashMap<Integer, Project> projectsById =
new ConcurrentHashMap<Integer, Project>();
private ConcurrentHashMap<String, Project> projectsByName =
@@ -75,9 +74,10 @@ public class ProjectManager {
tempDir.mkdirs();
}
- // The prop passed to XmlValidatorManager is used to initialize all the validators
- // Each validator will take certain key/value pairs from the prop to initialize
- // itself.
+ // The prop passed to XmlValidatorManager is used to initialize all the
+ // validators
+ // Each validator will take certain key/value pairs from the prop to
+ // initialize itself.
Props prop = new Props(props);
prop.put(ValidatorConfigs.PROJECT_ARCHIVE_FILE_PATH, "initialize");
loadAllProjects();
@@ -353,8 +353,32 @@ public class ProjectManager {
}
}
- public Map<String, ValidationReport> uploadProject(Project project, File archive, String fileType,
- User uploader, Props additionalProps) throws ProjectManagerException {
+ /**
+ * This method retrieves the uploaded project zip file from DB. A temporary
+ * file is created to hold the content of the uploaded zip file. This
+ * temporary file is provided in the ProjectFileHandler instance and the
+ * caller of this method should call method
+ * {@ProjectFileHandler.deleteLocalFile}
+ * to delete the temporary file.
+ *
+ * @param project
+ * @param version - latest version is used if value is -1
+ * @return ProjectFileHandler - null if can't find project zip file based on
+ * project name and version
+ * @throws ProjectManagerException
+ */
+ public ProjectFileHandler getProjectFileHandler(Project project, int version)
+ throws ProjectManagerException {
+
+ if (version == -1) {
+ version = projectLoader.getLatestProjectVersion(project);
+ }
+ return projectLoader.getUploadedFile(project, version);
+ }
+
+ public Map<String, ValidationReport> uploadProject(Project project,
+ File archive, String fileType, User uploader, Props additionalProps)
+ throws ProjectManagerException {
logger.info("Uploading files to " + project.getName());
// Unzip.
@@ -373,25 +397,34 @@ public class ProjectManager {
throw new ProjectManagerException("Error unzipping file.", e);
}
- // Since props is an instance variable of ProjectManager, and each invocation to the
- // uploadProject manager needs to pass a different value for the PROJECT_ARCHIVE_FILE_PATH
- // key, it is necessary to create a new instance of Props to make sure these different
- // values are isolated from each other.
+ // Since props is an instance variable of ProjectManager, and each
+ // invocation to the uploadProject manager needs to pass a different
+ // value for the PROJECT_ARCHIVE_FILE_PATH key, it is necessary to
+ // create a new instance of Props to make sure these different values
+ // are isolated from each other.
Props prop = new Props(props);
prop.putAll(additionalProps);
- prop.put(ValidatorConfigs.PROJECT_ARCHIVE_FILE_PATH, archive.getAbsolutePath());
- // Basically, we want to make sure that for different invocations to the uploadProject method,
- // the validators are using different values for the PROJECT_ARCHIVE_FILE_PATH configuration key.
- // In addition, we want to reload the validator objects for each upload, so that
- // we can change the validator configuration files without having to restart Azkaban web server.
- // If the XmlValidatorManager is an instance variable, 2 consecutive invocations to the uploadProject
- // method might cause the second one to overwrite the PROJECT_ARCHIVE_FILE_PATH configuration parameter
- // of the first, thus causing a wrong archive file path to be passed to the validators. Creating a
- // separate XmlValidatorManager object for each upload will prevent this issue without having to add
- // synchronization between uploads. Since we're already reloading the XML config file and creating
- // validator objects for each upload, this does not add too much additional overhead.
+ prop.put(ValidatorConfigs.PROJECT_ARCHIVE_FILE_PATH,
+ archive.getAbsolutePath());
+ // Basically, we want to make sure that for different invocations to the
+ // uploadProject method,
+ // the validators are using different values for the
+ // PROJECT_ARCHIVE_FILE_PATH configuration key.
+ // In addition, we want to reload the validator objects for each upload, so
+ // that we can change the validator configuration files without having to
+ // restart Azkaban web server. If the XmlValidatorManager is an instance
+ // variable, 2 consecutive invocations to the uploadProject
+ // method might cause the second one to overwrite the
+ // PROJECT_ARCHIVE_FILE_PATH configuration parameter
+ // of the first, thus causing a wrong archive file path to be passed to the
+ // validators. Creating a separate XmlValidatorManager object for each
+ // upload will prevent this issue without having to add
+ // synchronization between uploads. Since we're already reloading the XML
+ // config file and creating validator objects for each upload, this does
+ // not add too much additional overhead.
ValidatorManager validatorManager = new XmlValidatorManager(prop);
- logger.info("Validating project " + archive.getName() + " using the registered validators "
+ logger.info("Validating project " + archive.getName()
+ + " using the registered validators "
+ validatorManager.getValidatorsInfo().toString());
Map<String, ValidationReport> reports = validatorManager.validate(file);
ValidationStatus status = ValidationStatus.PASS;
@@ -414,7 +447,8 @@ public class ProjectManager {
return reports;
}
- DirectoryFlowLoader loader = (DirectoryFlowLoader) validatorManager.getDefaultValidator();
+ DirectoryFlowLoader loader =
+ (DirectoryFlowLoader) validatorManager.getDefaultValidator();
Map<String, Props> jobProps = loader.getJobProps();
List<Props> propProps = loader.getProps();
diff --git a/azkaban-webserver/src/main/java/azkaban/webapp/servlet/ProjectManagerServlet.java b/azkaban-webserver/src/main/java/azkaban/webapp/servlet/ProjectManagerServlet.java
index 3f7de9c..99e9ed3 100644
--- a/azkaban-webserver/src/main/java/azkaban/webapp/servlet/ProjectManagerServlet.java
+++ b/azkaban-webserver/src/main/java/azkaban/webapp/servlet/ProjectManagerServlet.java
@@ -18,6 +18,7 @@ package azkaban.webapp.servlet;
import java.io.BufferedOutputStream;
import java.io.File;
+import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
@@ -54,6 +55,7 @@ import azkaban.flow.Flow;
import azkaban.flow.FlowProps;
import azkaban.flow.Node;
import azkaban.project.Project;
+import azkaban.project.ProjectFileHandler;
import azkaban.project.ProjectLogEvent;
import azkaban.project.ProjectManager;
import azkaban.project.ProjectManagerException;
@@ -76,6 +78,7 @@ import azkaban.utils.Utils;
import azkaban.webapp.AzkabanWebServer;
public class ProjectManagerServlet extends LoginAbstractAzkabanServlet {
+ private static final String APPLICATION_ZIP_MIME_TYPE = "application/zip";
private static final long serialVersionUID = 1;
private static final Logger logger = Logger
.getLogger(ProjectManagerServlet.class);
@@ -84,10 +87,14 @@ public class ProjectManagerServlet extends LoginAbstractAzkabanServlet {
private static final String LOCKDOWN_CREATE_PROJECTS_KEY =
"lockdown.create.projects";
+ private static final String PROJECT_DOWNLOAD_BUFFER_SIZE_IN_BYTES =
+ "project.download.buffer.size";
+
private ProjectManager projectManager;
private ExecutorManagerAdapter executorManager;
private ScheduleManager scheduleManager;
private UserManager userManager;
+ private int downloadBufferSize;
private boolean lockdownCreateProjects = false;
@@ -112,6 +119,12 @@ public class ProjectManagerServlet extends LoginAbstractAzkabanServlet {
if (lockdownCreateProjects) {
logger.info("Creation of projects is locked down");
}
+
+ downloadBufferSize =
+ server.getServerProps().getInt(PROJECT_DOWNLOAD_BUFFER_SIZE_IN_BYTES,
+ 8192);
+
+ logger.info("downloadBufferSize: " + downloadBufferSize);
}
@Override
@@ -134,6 +147,8 @@ public class ProjectManagerServlet extends LoginAbstractAzkabanServlet {
handleFlowPage(req, resp, session);
} else if (hasParam(req, "delete")) {
handleRemoveProject(req, resp, session);
+ } else if (hasParam(req, "download")) {
+ handleDownloadProject(req, resp, session);
} else {
handleProjectPage(req, resp, session);
}
@@ -409,6 +424,97 @@ public class ProjectManagerServlet extends LoginAbstractAzkabanServlet {
ret.put("executions", history);
}
+ /**
+ * Download project zip file from DB and send it back client.
+ *
+ * This method requires a project name and an optional project version.
+ *
+ * @param req
+ * @param resp
+ * @param session
+ * @throws ServletException
+ * @throws IOException
+ */
+ private void handleDownloadProject(HttpServletRequest req,
+ HttpServletResponse resp, Session session) throws ServletException,
+ IOException {
+
+ User user = session.getUser();
+ String projectName = getParam(req, "project");
+ logger.info(user.getUserId() + " is downloading project: " + projectName);
+
+ Project project = projectManager.getProject(projectName);
+ if (project == null) {
+ this.setErrorMessageInCookie(resp, "Project " + projectName
+ + " doesn't exist.");
+ resp.sendRedirect(req.getContextPath());
+ return;
+ }
+
+ int version = -1;
+ if (hasParam(req, "version")) {
+ version = getIntParam(req, "version");
+ }
+
+ ProjectFileHandler projectFileHandler = null;
+ FileInputStream inStream = null;
+ OutputStream outStream = null;
+ try {
+ projectFileHandler =
+ projectManager.getProjectFileHandler(project, version);
+ if (projectFileHandler == null) {
+ this.setErrorMessageInCookie(resp, "Project " + projectName
+ + " with version " + version + " doesn't exist");
+ resp.sendRedirect(req.getContextPath());
+ return;
+ }
+ File projectZipFile = projectFileHandler.getLocalFile();
+ String logStr =
+ String.format(
+ "downloading project zip file for project \"%s\" at \"%s\""
+ + " size: %d type: %s fileName: \"%s\"",
+ projectFileHandler.getFileName(),
+ projectZipFile.getAbsolutePath(), projectZipFile.length(),
+ projectFileHandler.getFileType(),
+ projectFileHandler.getFileName());
+ logger.info(logStr);
+
+ // now set up HTTP response for downloading file
+ inStream = new FileInputStream(projectZipFile);
+
+ resp.setContentType(APPLICATION_ZIP_MIME_TYPE);
+
+ String headerKey = "Content-Disposition";
+ String headerValue =
+ String.format("attachment; filename=\"%s\"",
+ projectFileHandler.getFileName());
+ resp.setHeader(headerKey, headerValue);
+
+ outStream = resp.getOutputStream();
+
+ byte[] buffer = new byte[downloadBufferSize];
+ int bytesRead = -1;
+
+ while ((bytesRead = inStream.read(buffer)) != -1) {
+ outStream.write(buffer, 0, bytesRead);
+ }
+
+ } catch (Throwable e) {
+ logger.error(
+ "Encountered error while downloading project zip file for project: "
+ + projectName + " by user: " + user.getUserId(), e);
+ throw new ServletException(e);
+ } finally {
+ IOUtils.closeQuietly(inStream);
+ IOUtils.closeQuietly(outStream);
+
+ if (projectFileHandler != null) {
+ projectFileHandler.deleteLocalFile();
+ }
+ }
+
+ }
+
private void handleRemoveProject(HttpServletRequest req,
HttpServletResponse resp, Session session) throws ServletException,
IOException {
@@ -1341,13 +1447,19 @@ public class ProjectManagerServlet extends LoginAbstractAzkabanServlet {
project.getUsersWithPermission(Type.ADMIN), ","));
Permission perm = this.getPermissionObject(project, user, Type.ADMIN);
page.add("userpermission", perm);
- page.add("validatorFixPrompt", projectManager.getProps()
- .getBoolean(ValidatorConfigs.VALIDATOR_AUTO_FIX_PROMPT_FLAG_PARAM,
+ page.add(
+ "validatorFixPrompt",
+ projectManager.getProps().getBoolean(
+ ValidatorConfigs.VALIDATOR_AUTO_FIX_PROMPT_FLAG_PARAM,
ValidatorConfigs.DEFAULT_VALIDATOR_AUTO_FIX_PROMPT_FLAG));
- page.add("validatorFixLabel", projectManager.getProps()
- .get(ValidatorConfigs.VALIDATOR_AUTO_FIX_PROMPT_LABEL_PARAM));
- page.add("validatorFixLink", projectManager.getProps()
- .get(ValidatorConfigs.VALIDATOR_AUTO_FIX_PROMPT_LINK_PARAM));
+ page.add(
+ "validatorFixLabel",
+ projectManager.getProps().get(
+ ValidatorConfigs.VALIDATOR_AUTO_FIX_PROMPT_LABEL_PARAM));
+ page.add(
+ "validatorFixLink",
+ projectManager.getProps().get(
+ ValidatorConfigs.VALIDATOR_AUTO_FIX_PROMPT_LINK_PARAM));
boolean adminPerm = perm.isPermissionSet(Type.ADMIN);
if (adminPerm) {
@@ -1447,7 +1559,7 @@ public class ProjectManagerServlet extends LoginAbstractAzkabanServlet {
final String contentType = item.getContentType();
if (contentType != null
- && (contentType.startsWith("application/zip")
+ && (contentType.startsWith(APPLICATION_ZIP_MIME_TYPE)
|| contentType.startsWith("application/x-zip-compressed") || contentType
.startsWith("application/octet-stream"))) {
type = "zip";
@@ -1467,8 +1579,9 @@ public class ProjectManagerServlet extends LoginAbstractAzkabanServlet {
IOUtils.copy(item.getInputStream(), out);
out.close();
- Map<String, ValidationReport> reports = projectManager.uploadProject(
- project, archiveFile, type, user, props);
+ Map<String, ValidationReport> reports =
+ projectManager.uploadProject(project, archiveFile, type, user,
+ props);
StringBuffer message = new StringBuffer();
for (Entry<String, ValidationReport> reportEntry : reports.entrySet()) {
ValidationReport report = reportEntry.getValue();
@@ -1479,14 +1592,16 @@ public class ProjectManagerServlet extends LoginAbstractAzkabanServlet {
message.append("<br/>");
}
if (!report.getErrorMsgs().isEmpty()) {
- message.append("Validator " + reportEntry.getKey() + " reports errors:<ul>");
+ message.append("Validator " + reportEntry.getKey()
+ + " reports errors:<ul>");
for (String msg : report.getErrorMsgs()) {
message.append("<li>" + msg + "</li>");
}
message.append("</ul>");
}
if (!report.getWarningMsgs().isEmpty()) {
- message.append("Validator " + reportEntry.getKey() + " reports warnings:<ul>");
+ message.append("Validator " + reportEntry.getKey()
+ + " reports warnings:<ul>");
for (String msg : report.getWarningMsgs()) {
message.append("<li>" + msg + "</li>");
}
@@ -1500,7 +1615,8 @@ public class ProjectManagerServlet extends LoginAbstractAzkabanServlet {
logger.info("Installation Failed.", e);
String error = e.getMessage();
if (error.length() > 512) {
- error = error.substring(0, 512) + "<br>Too many errors to display.<br>";
+ error =
+ error.substring(0, 512) + "<br>Too many errors to display.<br>";
}
ret.put("error", "Installation Failed.<br>" + error);
} finally {
