azkaban-developers

diff --git a/.gitignore b/.gitignore
index a55a10d..60d8513 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 build/
+bin/
 .gradle/
 .settings/
 .idea/

diff --git a/azkaban-common/src/main/java/azkaban/project/ProjectManager.java b/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
index 54ab4dd..9acb811 100644
--- a/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
+++ b/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
@@ -32,6 +33,10 @@ import org.apache.log4j.Logger;
 
 import azkaban.flow.Flow;
 import azkaban.project.ProjectLogEvent.EventType;
+import azkaban.project.validator.Status;
+import azkaban.project.validator.ValidationReport;
+import azkaban.project.validator.ValidatorManager;
+import azkaban.project.validator.XmlValidatorManager;
 import azkaban.user.Permission;
 import azkaban.user.User;
 import azkaban.user.Permission.Type;
@@ -47,6 +52,7 @@ public class ProjectManager {
   private ConcurrentHashMap<String, Project> projectsByName =
       new ConcurrentHashMap<String, Project>();
   private final ProjectLoader projectLoader;
+  private final ValidatorManager validatorManager;
   private final Props props;
   private final File tempDir;
   private final int projectVersionRetention;
@@ -68,6 +74,7 @@ public class ProjectManager {
       tempDir.mkdirs();
     }
 
+    validatorManager = new XmlValidatorManager(props);
     loadAllProjects();
   }
 
@@ -337,7 +344,7 @@ public class ProjectManager {
     }
   }
 
-  public void uploadProject(Project project, File archive, String fileType,
+  public Map<String, ValidationReport> uploadProject(Project project, File archive, String fileType,
       User uploader) throws ProjectManagerException {
     logger.info("Uploading files to " + project.getName());
 
@@ -357,6 +364,31 @@ public class ProjectManager {
       throw new ProjectManagerException("Error unzipping file.", e);
     }
 
+    logger.info("Validating project using the registered validators "
+        + validatorManager.getValidatorsInfo().toString());
+    Map<String, ValidationReport> reports = validatorManager.validate(file);
+    Status status = Status.PASS;
+    for (Entry<String, ValidationReport> report : reports.entrySet()) {
+      logger.info("Before: " + report.getValue().getStatus());
+      if (report.getValue().getStatus().compareTo(status) > 0) {
+        status = report.getValue().getStatus();
+      }
+      logger.info("After: " + status);
+    }
+    if (status == Status.ERROR) {
+      logger.error("Error found in upload to " + project.getName()
+          + ". Cleaning up.");
+
+      try {
+        FileUtils.deleteDirectory(file);
+      } catch (IOException e) {
+        file.deleteOnExit();
+        e.printStackTrace();
+      }
+
+      return reports;
+    }
+
     logger.info("Validating Flow for upload " + archive.getName());
     DirectoryFlowLoader loader = new DirectoryFlowLoader(logger);
     loader.loadProjectFlow(file);
@@ -422,6 +454,8 @@ public class ProjectManager {
         + (project.getVersion() - projectVersionRetention));
     projectLoader.cleanOlderProjectVersion(project.getId(),
         project.getVersion() - projectVersionRetention);
+
+    return reports;
   }
 
   public void updateFlow(Project project, Flow flow)

diff --git a/azkaban-common/src/main/java/azkaban/project/validator/ProjectValidator.java b/azkaban-common/src/main/java/azkaban/project/validator/ProjectValidator.java
new file mode 100644
index 0000000..860fa2d
--- /dev/null
+++ b/azkaban-common/src/main/java/azkaban/project/validator/ProjectValidator.java
@@ -0,0 +1,10 @@
+package azkaban.project.validator;
+
+import java.io.File;
+import java.util.Properties;
+
+public interface ProjectValidator {
+  boolean initialize(Properties configuration);
+  String getValidatorInfo();
+  ValidationReport validateProject(File projectDir);
+}

diff --git a/azkaban-common/src/main/java/azkaban/project/validator/Status.java b/azkaban-common/src/main/java/azkaban/project/validator/Status.java
new file mode 100644
index 0000000..a8589c1
--- /dev/null
+++ b/azkaban-common/src/main/java/azkaban/project/validator/Status.java
@@ -0,0 +1,21 @@
+package azkaban.project.validator;
+
+/**
+ * Status of the ValidationReport. It also represents the severity of each rule.
+ */
+public enum Status {
+  PASS("PASS"),
+  WARN("WARN"),
+  ERROR("ERROR");
+
+  private final String _status;
+
+  private Status(final String status) {
+    _status = status;
+  }
+
+  @Override
+  public String toString() {
+    return _status;
+  }
+}

diff --git a/azkaban-common/src/main/java/azkaban/project/validator/ValidationReport.java b/azkaban-common/src/main/java/azkaban/project/validator/ValidationReport.java
new file mode 100644
index 0000000..819bc9a
--- /dev/null
+++ b/azkaban-common/src/main/java/azkaban/project/validator/ValidationReport.java
@@ -0,0 +1,20 @@
+package azkaban.project.validator;
+
+import java.util.Set;
+
+public interface ValidationReport {
+  void addPassMsgs(Set<String> msgs);
+
+  void addWarningMsgs(Set<String> msgs);
+
+  void addErrorMsgs(Set<String> msgs);
+
+  Status getStatus();
+
+  Set<String> getPassMsgs();
+
+  Set<String> getWarningMsgs();
+
+  Set<String> getErrorMsgs();
+
+}

diff --git a/azkaban-common/src/main/java/azkaban/project/validator/ValidatorManager.java b/azkaban-common/src/main/java/azkaban/project/validator/ValidatorManager.java
new file mode 100644
index 0000000..2ddf671
--- /dev/null
+++ b/azkaban-common/src/main/java/azkaban/project/validator/ValidatorManager.java
@@ -0,0 +1,15 @@
+package azkaban.project.validator;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+
+import azkaban.utils.Props;
+
+public interface ValidatorManager {
+  void loadValidators(Props props);
+
+  Map<String, ValidationReport> validate(File projectDir);
+
+  List<String> getValidatorsInfo();
+}

diff --git a/azkaban-common/src/main/java/azkaban/project/validator/XmlValidatorManager.java b/azkaban-common/src/main/java/azkaban/project/validator/XmlValidatorManager.java
new file mode 100644
index 0000000..aa46e99
--- /dev/null
+++ b/azkaban-common/src/main/java/azkaban/project/validator/XmlValidatorManager.java
@@ -0,0 +1,127 @@
+package azkaban.project.validator;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+
+import org.apache.log4j.Logger;
+import org.w3c.dom.Document;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import azkaban.utils.Props;
+
+public class XmlValidatorManager implements ValidatorManager {
+  private static final Logger logger = Logger.getLogger(XmlValidatorManager.class
+      .getName());
+
+  public static final String XML_FILE_PARAM = "project.validators.xml.file";
+  public static final String AZKABAN_VALIDATOR_TAG = "azkaban-validators";
+  public static final String VALIDATOR_TAG = "validator";
+  public static final String CLASSNAME_ATTR = "classname";
+
+  private Map<String, ProjectValidator> validators;
+
+  public XmlValidatorManager(Props props) {
+    validators = new LinkedHashMap<String, ProjectValidator>();
+    loadValidators(props);
+  }
+
+  @Override
+  public void loadValidators(Props props) {
+    String xmlPath = props.get(XML_FILE_PARAM);
+    File file = new File(xmlPath);
+    if (!file.exists()) {
+      throw new IllegalArgumentException("Validator xml file " + xmlPath
+          + " doesn't exist.");
+    }
+
+    // Creating the document builder to parse xml.
+    DocumentBuilderFactory docBuilderFactory =
+        DocumentBuilderFactory.newInstance();
+    DocumentBuilder builder = null;
+    try {
+      builder = docBuilderFactory.newDocumentBuilder();
+    } catch (ParserConfigurationException e) {
+      throw new IllegalArgumentException(
+          "Exception while parsing user xml. Document builder not created.", e);
+    }
+
+    Document doc = null;
+    try {
+      doc = builder.parse(file);
+    } catch (SAXException e) {
+      throw new IllegalArgumentException("Exception while parsing " + xmlPath
+          + ". Invalid XML.", e);
+    } catch (IOException e) {
+      throw new IllegalArgumentException("Exception while parsing " + xmlPath
+          + ". Error reading file.", e);
+    }
+
+    NodeList tagList = doc.getChildNodes();
+    Node azkabanValidators = tagList.item(0);
+
+    NodeList azkabanValidatorsList = azkabanValidators.getChildNodes();
+    for (int i = 0; i < azkabanValidatorsList.getLength(); ++i) {
+      Node node = azkabanValidatorsList.item(i);
+      if (node.getNodeType() == Node.ELEMENT_NODE) {
+        if (node.getNodeName().equals(VALIDATOR_TAG)) {
+          parseValidatorTag(node, props);
+        }
+      }
+    }
+  }
+
+  private void parseValidatorTag(Node node, Props props) {
+    NamedNodeMap validatorAttrMap = node.getAttributes();
+    Node classNameAttr = validatorAttrMap.getNamedItem(CLASSNAME_ATTR);
+    if (classNameAttr == null) {
+      throw new RuntimeException(
+          "Error loading validator. The validator 'classname' attribute doesn't exist");
+    }
+
+    String className = classNameAttr.getNodeValue();
+    try {
+      Class<?> validatorClass = Class.forName(className);
+      Constructor<?> validatorConstructor =
+          validatorClass.getConstructor();
+      ProjectValidator validator = (ProjectValidator) validatorConstructor.newInstance();
+      validator.initialize(props.toProperties());
+      validators.put(validator.getValidatorInfo(), validator);
+      logger.info("Added validator " + validator.getClass().getCanonicalName() + " to list of validators.");
+    } catch (Exception e) {
+      logger.error("Could not instantiate ProjectValidator " + className);
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public Map<String, ValidationReport> validate(File projectDir) {
+    Map<String, ValidationReport> reports = new LinkedHashMap<String, ValidationReport>();
+    for (Entry<String, ProjectValidator> validator : validators.entrySet()) {
+      reports.put(validator.getKey(), validator.getValue().validateProject(projectDir));
+    }
+    return reports;
+  }
+
+  @Override
+  public List<String> getValidatorsInfo() {
+    List<String> info = new ArrayList<String>();
+    for (String key : validators.keySet()) {
+      info.add(key);
+    }
+    return info;
+  }
+
+}

diff --git a/azkaban-soloserver/.gitignore b/azkaban-soloserver/.gitignore
new file mode 100644
index 0000000..2b7a31c
--- /dev/null
+++ b/azkaban-soloserver/.gitignore
@@ -0,0 +1,5 @@
+data/
+conf/
+temp/
+plugins/
+*.log

diff --git a/azkaban-webserver/src/main/java/azkaban/webapp/servlet/ProjectManagerServlet.java b/azkaban-webserver/src/main/java/azkaban/webapp/servlet/ProjectManagerServlet.java
index b80b18c..ac172a9 100644
--- a/azkaban-webserver/src/main/java/azkaban/webapp/servlet/ProjectManagerServlet.java
+++ b/azkaban-webserver/src/main/java/azkaban/webapp/servlet/ProjectManagerServlet.java
@@ -31,6 +31,7 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
 
 import javax.servlet.ServletConfig;
@@ -56,6 +57,7 @@ import azkaban.project.Project;
 import azkaban.project.ProjectLogEvent;
 import azkaban.project.ProjectManager;
 import azkaban.project.ProjectManagerException;
+import azkaban.project.validator.ValidationReport;
 import azkaban.scheduler.Schedule;
 import azkaban.scheduler.ScheduleManager;
 import azkaban.scheduler.ScheduleManagerException;
@@ -1450,7 +1452,27 @@ public class ProjectManagerServlet extends LoginAbstractAzkabanServlet {
         IOUtils.copy(item.getInputStream(), out);
         out.close();
 
-        projectManager.uploadProject(project, archiveFile, type, user);
+        Map<String, ValidationReport> reports = projectManager.uploadProject(
+            project, archiveFile, type, user);
+        StringBuffer message = new StringBuffer();
+        for (Entry<String, ValidationReport> reportEntry : reports.entrySet()) {
+          ValidationReport report = reportEntry.getValue();
+          if (!report.getErrorMsgs().isEmpty()) {
+            message.append("Validator " + reportEntry.getKey() + " reports errors:\n");
+            for (String msg : report.getErrorMsgs()) {
+              message.append(msg + "\n");
+            }
+          }
+          if (!report.getWarningMsgs().isEmpty()) {
+            message.append("Validator " + reportEntry.getKey() + " reports warnings:\n");
+            for (String msg : report.getWarningMsgs()) {
+              message.append(msg + "\n");
+            }
+          }
+        }
+        if (message.toString().length() > 0) {
+          ret.put("error", message.toString());
+        }
       } catch (Exception e) {
         logger.info("Installation Failed.", e);
         String error = e.getMessage();