azkaban-aplcache

when loading project dir metadata into memory, create the project

8/8/2018 7:12:40 PM
3.51.0

Details

diff --git a/azkaban-exec-server/src/main/java/azkaban/execapp/FlowPreparer.java b/azkaban-exec-server/src/main/java/azkaban/execapp/FlowPreparer.java
index 5d93d4d..9c6c1a3 100644
--- a/azkaban-exec-server/src/main/java/azkaban/execapp/FlowPreparer.java
+++ b/azkaban-exec-server/src/main/java/azkaban/execapp/FlowPreparer.java
@@ -68,6 +68,23 @@ public class FlowPreparer {
 
   }
 
+  /**
+   * Creates a file which keeps the size of {@param dir} in bytes inside the {@param dir} and sets
+   * the dirSize for {@param pv}.
+   *
+   * @param dir the directory whose size needs to be kept in the file to be created.
+   * @param pv the projectVersion whose size needs to updated.
+   */
+  static void updateDirSize(final File dir, final ProjectVersion pv) {
+    final long sizeInByte = FileUtils.sizeOfDirectory(dir);
+    pv.setDirSizeInBytes(sizeInByte);
+    try {
+      FileIOUtils.dumpNumberToFile(Paths.get(dir.getPath(), PROJECT_DIR_SIZE_FILE_NAME),
+          sizeInByte);
+    } catch (final IOException e) {
+      log.error("error when dumping dir size to file", e);
+    }
+  }
 
   /**
    * Prepare the flow directory for execution.
@@ -180,24 +197,6 @@ public class FlowPreparer {
     }
   }
 
-  /**
-   * Creates a file which keeps the size of {@param dir} in bytes inside the {@param dir} and sets
-   * the dirSize for {@param pv}.
-   *
-   * @param dir the directory whose size needs to be kept in the file to be created.
-   * @param pv the projectVersion whose size needs to updated.
-   */
-  private void updateDirSize(final File dir, final ProjectVersion pv) {
-    final long sizeInByte = FileUtils.sizeOfDirectory(dir);
-    pv.setDirSizeInBytes(sizeInByte);
-    try {
-      FileIOUtils.dumpNumberToFile(Paths.get(dir.getPath(), PROJECT_DIR_SIZE_FILE_NAME),
-          sizeInByte);
-    } catch (final IOException e) {
-      log.error(e);
-    }
-  }
-
   private void copyCreateHardlinkDirectory(final File projectDir, final File execDir)
       throws IOException {
     FileIOUtils.createDeepHardlink(projectDir, execDir);
diff --git a/azkaban-exec-server/src/main/java/azkaban/execapp/FlowRunnerManager.java b/azkaban-exec-server/src/main/java/azkaban/execapp/FlowRunnerManager.java
index 405766b..42091ae 100644
--- a/azkaban-exec-server/src/main/java/azkaban/execapp/FlowRunnerManager.java
+++ b/azkaban-exec-server/src/main/java/azkaban/execapp/FlowRunnerManager.java
@@ -292,24 +292,29 @@ public class FlowRunnerManager implements EventListener,
   private Map<Pair<Integer, Integer>, ProjectVersion> loadExistingProjectsAsCache() {
     final Map<Pair<Integer, Integer>, ProjectVersion> allProjects =
         new ConcurrentHashMap<>();
+    logger.info("loading project dir metadata into memory");
     for (final Path project : this.loadExistingProjects()) {
       if (Files.isDirectory(project)) {
         try {
           final String fileName = project.getFileName().toString();
           final int projectId = Integer.parseInt(fileName.split("\\.")[0]);
           final int versionNum = Integer.parseInt(fileName.split("\\.")[1]);
-          final ProjectVersion version =
+          final ProjectVersion projVersion =
               new ProjectVersion(projectId, versionNum, project.toFile());
+          final Path projectDirSizeFile = Paths
+              .get(projVersion.getInstalledDir().toString(),
+                  FlowPreparer.PROJECT_DIR_SIZE_FILE_NAME);
+          if (!Files.exists(projectDirSizeFile)) {
+            FlowPreparer.updateDirSize(projVersion.getInstalledDir(), projVersion);
+          }
 
-          version.setDirSizeInBytes(
-              FileIOUtils.readNumberFromFile(Paths.get(version.getInstalledDir().toString(),
-                  FlowPreparer.PROJECT_DIR_SIZE_FILE_NAME)));
-
-          allProjects.put(new Pair<>(projectId, versionNum), version);
+          projVersion.setDirSizeInBytes(FileIOUtils.readNumberFromFile(projectDirSizeFile));
+          allProjects.put(new Pair<>(projectId, versionNum), projVersion);
         } catch (final Exception e) {
-          e.printStackTrace();
+          logger.error("error while loading project dir metadata", e);
         }
       }
+      logger.info("finish loading project dir metadata into memory");
     }
 
     return allProjects;