azkaban-aplcache

fix hadoop jobtype flaky test (#1738) The two below classes

4/20/2018 6:45:48 PM

Details

diff --git a/az-hadoop-jobtype-plugin/src/test/java/azkaban/jobtype/TestHadoopJobUtilsExecutionJar.java b/az-hadoop-jobtype-plugin/src/test/java/azkaban/jobtype/TestHadoopJobUtilsExecutionJar.java
index feb3c2e..3b8ca9b 100644
--- a/az-hadoop-jobtype-plugin/src/test/java/azkaban/jobtype/TestHadoopJobUtilsExecutionJar.java
+++ b/az-hadoop-jobtype-plugin/src/test/java/azkaban/jobtype/TestHadoopJobUtilsExecutionJar.java
@@ -10,28 +10,30 @@ import org.apache.commons.io.FileUtils;
 import org.apache.log4j.Logger;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
-import azkaban.utils.Props;
-
+// TODO kunkun-tang: This test class needs more refactors.
 public class TestHadoopJobUtilsExecutionJar {
-  Props jobProps = null;
-
-  Logger logger = Logger.getRootLogger();
-
-  String workingDirString = "/tmp/TestHadoopSpark";
-
-  File workingDirFile = new File(workingDirString);
-
-  File libFolderFile = new File(workingDirFile, "lib");
-
-  String executionJarName = "hadoop-spark-job-test-execution-x.y.z-a.b.c.jar";
-
-  File executionJarFile = new File(libFolderFile, "hadoop-spark-job-test-execution-x.y.z-a.b.c.jar");
-
-  File libraryJarFile = new File(libFolderFile, "library.jar");
-
-  String delim = SparkJobArg.delimiter;
+  private Logger logger = Logger.getRootLogger();
+
+  private static String currentDirString = System.getProperty("user.dir");
+  private static String workingDirString = null;
+  private static File workingDirFile = null;
+  private static File libFolderFile = null;
+  private static File executionJarFile = null;
+  private static File libraryJarFile = null;
+
+  @BeforeClass
+  public static void setupFolder() {
+    workingDirString = currentDirString + "/../temp/TestHadoopSpark";
+    workingDirFile = new File(workingDirString);
+    libFolderFile = new File(workingDirFile, "lib");
+    executionJarFile = new File(libFolderFile,
+        "hadoop-spark-job-test-execution-x.y.z-a.b.c"
+            + ".jar");
+    libraryJarFile = new File(libFolderFile, "library.jar");
+  }
 
   @Before
   public void beforeMethod() throws IOException {
@@ -41,15 +43,13 @@ public class TestHadoopJobUtilsExecutionJar {
     libFolderFile.mkdirs();
     executionJarFile.createNewFile();
     libraryJarFile.createNewFile();
-
   }
 
   // nothing should happen
   @Test
   public void testNoLibFolder() throws IOException {
     FileUtils.deleteDirectory(libFolderFile);
-    String retval = HadoopJobUtils.resolveWildCardForJarSpec(workingDirString, "./lib/*", logger);
-
+    String retval = HadoopJobUtils.resolveWildCardForJarSpec(workingDirString, "lib/*", logger);
     Assert.assertEquals(retval, "");
   }
 
@@ -58,7 +58,7 @@ public class TestHadoopJobUtilsExecutionJar {
   public void testLibFolderHasNothingInIt() throws IOException {
     FileUtils.deleteDirectory(libFolderFile);
     libFolderFile.mkdirs();
-    String retval = HadoopJobUtils.resolveWildCardForJarSpec(workingDirString, "./lib/*", logger);
+    String retval = HadoopJobUtils.resolveWildCardForJarSpec(workingDirString, "lib/*", logger);
 
     Assert.assertEquals(retval, "");
   }
@@ -66,12 +66,12 @@ public class TestHadoopJobUtilsExecutionJar {
 
   @Test
   public void testOneLibFolderExpansion() throws IOException {
-    String retval = HadoopJobUtils.resolveWildCardForJarSpec(workingDirString, "./lib/*", logger);
+    String retval = HadoopJobUtils.resolveWildCardForJarSpec(workingDirString, "lib/*", logger);
     Set<String> retvalSet = new HashSet<String>(Arrays.asList(retval.split(",")));
 
     Set<String> expected = new HashSet<String>();
-    expected.add("/tmp/TestHadoopSpark/./lib/library.jar");
-    expected.add("/tmp/TestHadoopSpark/./lib/hadoop-spark-job-test-execution-x.y.z-a.b.c.jar");
+    expected.add(workingDirFile + "/lib/library.jar");
+    expected.add(workingDirFile + "/lib/hadoop-spark-job-test-execution-x.y.z-a.b.c.jar");
 
     Assert.assertTrue("Expected size is different from retrieval size. Expected: " + expected + " , Actual: " + retvalSet,
                       expected.size() == retvalSet.size());
@@ -88,13 +88,14 @@ public class TestHadoopJobUtilsExecutionJar {
     lib2test1Jar.createNewFile();
     File lib2test2Jar = new File(lib2FolderFile, "test2.jar");
     lib2test2Jar.createNewFile();
-    String retval = HadoopJobUtils.resolveWildCardForJarSpec(workingDirString, "./lib/*,./lib2/*",
+    String retval = HadoopJobUtils.resolveWildCardForJarSpec(workingDirString, "lib/*,lib2/*",
             logger);
 
-    Assert.assertTrue(retval.contains("/tmp/TestHadoopSpark/./lib/library.jar"));
-    Assert.assertTrue(retval.contains("/tmp/TestHadoopSpark/./lib/hadoop-spark-job-test-execution-x.y.z-a.b.c.jar"));
-    Assert.assertTrue(retval.contains("/tmp/TestHadoopSpark/./lib2/test1.jar"));
-    Assert.assertTrue(retval.contains("/tmp/TestHadoopSpark/./lib2/test2.jar"));
+    Assert.assertTrue(retval.contains(workingDirFile + "/lib/library.jar"));
+    Assert.assertTrue(retval.contains(workingDirFile + "/lib/hadoop-spark-job-test-execution-x.y"
+        + ".z-a.b.c.jar"));
+    Assert.assertTrue(retval.contains(workingDirFile + "/lib2/test1.jar"));
+    Assert.assertTrue(retval.contains(workingDirFile + "/lib2/test2.jar"));
   }
 
     @Test
@@ -107,11 +108,11 @@ public class TestHadoopJobUtilsExecutionJar {
       File lib2test1Jar = new File(lib2FolderFile, "test1.jar");
       lib2test1Jar.createNewFile();
 
-      String retval = HadoopJobUtils.resolveWildCardForJarSpec(workingDirString, "./lib/*,./lib2/*",
+      String retval = HadoopJobUtils.resolveWildCardForJarSpec(workingDirString, "lib/*,lib2/*",
               logger);
 
       Assert.assertEquals(
               retval,
-              "/tmp/TestHadoopSpark/./lib/library.jar,/tmp/TestHadoopSpark/./lib2/test1.jar");
+          workingDirFile + "/lib/library.jar," + workingDirFile + "/lib2/test1.jar");
   }
 }
diff --git a/az-hadoop-jobtype-plugin/src/test/java/azkaban/jobtype/TestHadoopJobUtilsResolveJarSpec.java b/az-hadoop-jobtype-plugin/src/test/java/azkaban/jobtype/TestHadoopJobUtilsResolveJarSpec.java
index a87bcc5..2988df7 100644
--- a/az-hadoop-jobtype-plugin/src/test/java/azkaban/jobtype/TestHadoopJobUtilsResolveJarSpec.java
+++ b/az-hadoop-jobtype-plugin/src/test/java/azkaban/jobtype/TestHadoopJobUtilsResolveJarSpec.java
@@ -5,30 +5,33 @@ import java.io.IOException;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.log4j.Logger;
-import org.junit.Assert;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import azkaban.utils.Props;
 
+// TODO kunkun-tang: This test class needs more refactors.
 public class TestHadoopJobUtilsResolveJarSpec {
-  Props jobProps = null;
-
-  Logger logger = Logger.getRootLogger();
-
-  String workingDirString = "/tmp/TestHadoopSpark";
-
-  File workingDirFile = new File(workingDirString);
-
-  File libFolderFile = new File(workingDirFile, "lib");
-
-  String executionJarName = "hadoop-spark-job-test-execution-x.y.z-a.b.c.jar";
-
-  File executionJarFile = new File(libFolderFile, "hadoop-spark-job-test-execution-x.y.z-a.b.c.jar");
-
-  File libraryJarFile = new File(libFolderFile, "library.jar");
-
-  String delim = SparkJobArg.delimiter;
+  private Logger logger = Logger.getRootLogger();
+
+  private static String currentDirString = System.getProperty("user.dir");
+  private static String workingDirString = null;
+  private static File workingDirFile = null;
+  private static File libFolderFile = null;
+  private static File executionJarFile = null;
+  private static File libraryJarFile = null;
+
+  @BeforeClass
+  public static void setupFolder() {
+    workingDirString = currentDirString + "/../temp/TestHadoopSpark";
+    workingDirFile = new File(workingDirString);
+    libFolderFile = new File(workingDirFile, "lib");
+    executionJarFile = new File(libFolderFile,
+        "hadoop-spark-job-test-execution-x.y.z-a.b.c"
+            + ".jar");
+    libraryJarFile = new File(libFolderFile, "library.jar");
+  }
 
   @Before
   public void beforeMethod() throws IOException {
@@ -44,12 +47,12 @@ public class TestHadoopJobUtilsResolveJarSpec {
   // nothing should happen
   @Test(expected = IllegalStateException.class)
   public void testJarDoesNotExist() throws IOException {
-    HadoopJobUtils.resolveExecutionJarName(workingDirString, "./lib/abc.jar", logger);
+    HadoopJobUtils.resolveExecutionJarName(workingDirString, "/lib/abc.jar", logger);
   }
 
   @Test(expected = IllegalStateException.class)
   public void testNoLibFolder() throws IOException {
     FileUtils.deleteDirectory(libFolderFile);
-    HadoopJobUtils.resolveExecutionJarName(workingDirString, "./lib/abc.jar", logger);
+    HadoopJobUtils.resolveExecutionJarName(workingDirString, "/lib/abc.jar", logger);
   }
 }