azkaban-aplcache

Creating a Project Spec File. Phase I (#939) Added project-spec

3/14/2017 7:19:23 PM

Details

diff --git a/azkaban-common/build.gradle b/azkaban-common/build.gradle
index 0783917..50452ce 100644
--- a/azkaban-common/build.gradle
+++ b/azkaban-common/build.gradle
@@ -37,6 +37,7 @@ dependencies {
   compile('org.mortbay.jetty:jetty:6.1.26')
   compile('org.mortbay.jetty:jetty-util:6.1.26')
   compile('org.quartz-scheduler:quartz:2.2.1')
+  compile('org.yaml:snakeyaml:1.18')
   compile('io.dropwizard.metrics:metrics-core:3.1.0')
   compile('io.dropwizard.metrics:metrics-jvm:3.1.0')
 
diff --git a/azkaban-common/src/main/java/azkaban/project/ProjectSpec.java b/azkaban-common/src/main/java/azkaban/project/ProjectSpec.java
new file mode 100644
index 0000000..96789a6
--- /dev/null
+++ b/azkaban-common/src/main/java/azkaban/project/ProjectSpec.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2017 LinkedIn Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ */
+
+package azkaban.project;
+
+import java.io.Serializable;
+import java.net.URI;
+import java.util.Map;
+
+
+public class ProjectSpec implements Serializable {
+  private String version;
+  private PreExecutionSpec preExec;
+
+  public String getVersion() {
+    return version;
+  }
+
+  public void setVersion(String version) {
+    this.version = version;
+  }
+
+  public PreExecutionSpec getPreExec() {
+    return preExec;
+  }
+
+  public void setPreExec(PreExecutionSpec preExec) {
+    this.preExec = preExec;
+  }
+
+  @Override
+  public String toString() {
+    return "ProjectSpec{" + "version='" + version + '\'' + ", preExec=" + preExec + '}';
+  }
+
+  public static class PreExecutionSpec implements Serializable {
+    private Map<String, URI> fetch;
+
+    public Map<String, URI> getFetch() {
+      return fetch;
+    }
+
+    public void setFetch(Map<String, URI> fetch) {
+      this.fetch = fetch;
+    }
+
+    @Override
+    public String toString() {
+      return "PreExecutionSpec{" + "fetch=" + fetch + '}';
+    }
+  }
+
+}
diff --git a/azkaban-common/src/test/java/azkaban/project/ProjectSpecLoader.java b/azkaban-common/src/test/java/azkaban/project/ProjectSpecLoader.java
new file mode 100644
index 0000000..6fb9d25
--- /dev/null
+++ b/azkaban-common/src/test/java/azkaban/project/ProjectSpecLoader.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2017 LinkedIn Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ */
+
+package azkaban.project;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import org.yaml.snakeyaml.Yaml;
+
+
+public class ProjectSpecLoader {
+
+  public ProjectSpec load(File projectSpecFile) throws FileNotFoundException {
+    return new Yaml().loadAs(new FileInputStream(projectSpecFile), ProjectSpec.class);
+  }
+}
diff --git a/azkaban-common/src/test/java/azkaban/project/ProjectSpecTest.java b/azkaban-common/src/test/java/azkaban/project/ProjectSpecTest.java
new file mode 100644
index 0000000..a6a2c58
--- /dev/null
+++ b/azkaban-common/src/test/java/azkaban/project/ProjectSpecTest.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2017 LinkedIn Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ */
+
+package azkaban.project;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.net.URI;
+import java.util.Map;
+import org.junit.Test;
+import org.yaml.snakeyaml.Yaml;
+
+import static org.junit.Assert.*;
+
+
+public class ProjectSpecTest {
+
+  /**
+   * Loads spec.yaml from test/resources and asserts properties
+   *
+   */
+  @Test
+  public void testSpecLoad() throws Exception {
+    ClassLoader classLoader = getClass().getClassLoader();
+    File file = new File(classLoader.getResource("spec.yml").getFile());
+    ProjectSpec spec = new ProjectSpecLoader().load(file);
+
+    assertEquals("1.0", spec.getVersion());
+
+    Map<String, URI> fetchMap = spec.getPreExec().getFetch();
+    URI sampleUri = new URI("http://central.maven.org/maven2/log4j/log4j/1.2.17/log4j-1.2.17.jar");
+    assertEquals(sampleUri, fetchMap.get("lib"));
+    assertEquals(sampleUri, fetchMap.get("path/to/foo"));
+  }
+}
diff --git a/azkaban-common/src/test/resources/spec.yml b/azkaban-common/src/test/resources/spec.yml
new file mode 100644
index 0000000..d70961f
--- /dev/null
+++ b/azkaban-common/src/test/resources/spec.yml
@@ -0,0 +1,21 @@
+---
+version: 1.0
+# The version of the spec file.
+# This is the version of the YAML structure and not the project version.
+# This key will allow us to update the spec structure itself.
+
+preExec:
+    # Stuff to do before the execution phase, like fetching dependencies, etc
+    # fetch section can be used to fetch dependencies externally
+    # into a destination directory.
+    # The format is
+    # destination_dir:
+    #  - url1 # This can be an http url that will be fetched at runtime
+    #  - group:artifact:version # You can also mention coordinates to resource
+    fetch:
+        #  name of directory. This is created if doesn’t exist!
+        lib:
+         - http://central.maven.org/maven2/log4j/log4j/1.2.17/log4j-1.2.17.jar
+        #  path of directory. Again, entire path is created if doesn’t exist!
+        path/to/foo:
+         - http://central.maven.org/maven2/log4j/log4j/1.2.17/log4j-1.2.17.jar