azkaban-aplcache

Use customized classloader to load validator's classpath.

10/21/2014 10:12:43 PM

Details

diff --git a/azkaban-common/src/main/java/azkaban/project/ProjectManager.java b/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
index 64c450d..aaaaca0 100644
--- a/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
+++ b/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
@@ -76,7 +76,9 @@ public class ProjectManager {
       tempDir.mkdirs();
     }
 
-    validatorManager = new XmlValidatorManager(props);
+    Props prop = new Props(props);
+    prop.put(PROJECT_ARCHIVE_FILE_PATH, "initialize");
+    validatorManager = new XmlValidatorManager(prop);
     loadAllProjects();
   }
 
@@ -366,11 +368,17 @@ public class ProjectManager {
       throw new ProjectManagerException("Error unzipping file.", e);
     }
 
-    props.put(PROJECT_ARCHIVE_FILE_PATH, archive.getAbsolutePath());
-    validatorManager.loadValidators(props, logger);
+    Props prop = new Props(props);
+    prop.put(PROJECT_ARCHIVE_FILE_PATH, archive.getAbsolutePath());
+    validatorManager.loadValidators(prop, logger);
     logger.info("Validating project " + archive.getName() + " using the registered validators "
         + validatorManager.getValidatorsInfo().toString());
+    ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader();
+    // Switching the classloader to ValidatorClassLoader which prefers the validator's classpath
+    // over the parent classloader's classpath.
+    Thread.currentThread().setContextClassLoader(validatorManager.getClassLoader());
     Map<String, ValidationReport> reports = validatorManager.validate(file);
+    Thread.currentThread().setContextClassLoader(currentClassLoader);
     ValidationStatus status = ValidationStatus.PASS;
     for (Entry<String, ValidationReport> report : reports.entrySet()) {
       if (report.getValue().getStatus().compareTo(status) > 0) {
diff --git a/azkaban-common/src/main/java/azkaban/project/validator/ValidatorClassLoader.java b/azkaban-common/src/main/java/azkaban/project/validator/ValidatorClassLoader.java
new file mode 100644
index 0000000..4b0cd5c
--- /dev/null
+++ b/azkaban-common/src/main/java/azkaban/project/validator/ValidatorClassLoader.java
@@ -0,0 +1,145 @@
+package azkaban.project.validator;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.log4j.Logger;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Splitter;
+
+/**
+ * A {@link URLClassLoader} for YARN application isolation. Classes from
+ * the application JARs are loaded in preference to the parent loader.
+ */
+public class ValidatorClassLoader extends URLClassLoader {
+
+  private static final Logger logger = Logger.getLogger(XmlValidatorManager.class);
+
+  private static final FilenameFilter JAR_FILENAME_FILTER =
+    new FilenameFilter() {
+      @Override
+      public boolean accept(File dir, String name) {
+        return name.endsWith(".jar") || name.endsWith(".JAR");
+      }
+  };
+
+  private ClassLoader parent;
+
+  public ValidatorClassLoader(URL[] urls, ClassLoader parent) {
+    super(urls, parent);
+    this.parent = parent;
+    if (parent == null) {
+      throw new IllegalArgumentException("No parent classloader!");
+    }
+  }
+
+  public ValidatorClassLoader(String classpath, ClassLoader parent)
+      throws MalformedURLException {
+    this(constructUrlsFromClasspath(classpath), parent);
+  }
+
+  @VisibleForTesting
+  static URL[] constructUrlsFromClasspath(String classpath)
+      throws MalformedURLException {
+    List<URL> urls = new ArrayList<URL>();
+    for (String element : Splitter.on(File.pathSeparator).split(classpath)) {
+      if (element.endsWith("/*")) {
+        String dir = element.substring(0, element.length() - 1);
+        File[] files = new File(dir).listFiles(JAR_FILENAME_FILTER);
+        if (files != null) {
+          for (File file : files) {
+            urls.add(file.toURI().toURL());
+          }
+        }
+      } else {
+        File file = new File(element);
+        if (file.exists()) {
+          urls.add(new File(element).toURI().toURL());
+        }
+      }
+    }
+    return urls.toArray(new URL[urls.size()]);
+  }
+
+  @Override
+  public URL getResource(String name) {
+    URL url = null;
+
+    url= findResource(name);
+    if (url == null && name.startsWith("/")) {
+      if (logger.isDebugEnabled()) {
+        logger.debug("Remove leading / off " + name);
+      }
+      url= findResource(name.substring(1));
+    }
+
+    if (url == null) {
+      url= parent.getResource(name);
+    }
+
+    if (url != null) {
+      if (logger.isDebugEnabled()) {
+        logger.debug("getResource("+name+")=" + url);
+      }
+    }
+
+    return url;
+  }
+
+  @Override
+  public Class<?> loadClass(String name) throws ClassNotFoundException {
+    return this.loadClass(name, false);
+  }
+
+  @Override
+  protected synchronized Class<?> loadClass(String name, boolean resolve)
+      throws ClassNotFoundException {
+
+    if (logger.isDebugEnabled()) {
+      logger.debug("Loading class: " + name);
+    }
+
+    Class<?> c = findLoadedClass(name);
+    ClassNotFoundException ex = null;
+
+    if (c == null) {
+      // Try to load class from this classloader's URLs. Note that this is like
+      // the servlet spec, not the usual Java 2 behaviour where we ask the
+      // parent to attempt to load first.
+      try {
+        c = findClass(name);
+        if (logger.isDebugEnabled() && c != null) {
+          logger.debug("Loaded class: " + name + " ");
+        }
+      } catch (ClassNotFoundException e) {
+        if (logger.isDebugEnabled()) {
+          logger.debug(e);
+        }
+        ex = e;
+      }
+    }
+
+    if (c == null) { // try parent
+      c = parent.loadClass(name);
+      if (logger.isDebugEnabled() && c != null) {
+        logger.debug("Loaded class from parent: " + name + " ");
+      }
+    }
+
+    if (c == null) {
+      throw ex != null ? ex : new ClassNotFoundException(name);
+    }
+
+    if (resolve) {
+      resolveClass(c);
+    }
+
+    return c;
+  }
+}
\ No newline at end of file
diff --git a/azkaban-common/src/main/java/azkaban/project/validator/ValidatorManager.java b/azkaban-common/src/main/java/azkaban/project/validator/ValidatorManager.java
index 1be62fb..5bd9a8c 100644
--- a/azkaban-common/src/main/java/azkaban/project/validator/ValidatorManager.java
+++ b/azkaban-common/src/main/java/azkaban/project/validator/ValidatorManager.java
@@ -48,4 +48,9 @@ public interface ValidatorManager {
    * @return
    */
   List<String> getValidatorsInfo();
+
+  /**
+   * Returns the classloader used to load the validator classes
+   */
+  ClassLoader getClassLoader();
 }
diff --git a/azkaban-common/src/main/java/azkaban/project/validator/XmlValidatorManager.java b/azkaban-common/src/main/java/azkaban/project/validator/XmlValidatorManager.java
index dc05c1a..6723b41 100644
--- a/azkaban-common/src/main/java/azkaban/project/validator/XmlValidatorManager.java
+++ b/azkaban-common/src/main/java/azkaban/project/validator/XmlValidatorManager.java
@@ -5,7 +5,6 @@ import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.net.URLClassLoader;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -81,7 +80,8 @@ public class XmlValidatorManager implements ValidatorManager {
     } catch (MalformedURLException e) {
       throw new ValidatorManagerException(e);
     }
-    validatorLoader = new URLClassLoader(resources.toArray(new URL[resources.size()]));
+    validatorLoader = new ValidatorClassLoader(resources.toArray(new URL[resources.size()]),
+        XmlValidatorManager.class.getClassLoader());
 
     // Test loading the validators specified in the xml file.
     try {
@@ -223,4 +223,9 @@ public class XmlValidatorManager implements ValidatorManager {
     return info;
   }
 
+  @Override
+  public ClassLoader getClassLoader() {
+    return validatorLoader;
+  }
+
 }