azkaban-aplcache

address feedback from #1803 (#1812) see comments from #1803:

6/20/2018 5:33:56 PM
3.48.0

Details

diff --git a/az-core/src/main/java/azkaban/Constants.java b/az-core/src/main/java/azkaban/Constants.java
index 2fe093d..58f4d96 100644
--- a/az-core/src/main/java/azkaban/Constants.java
+++ b/az-core/src/main/java/azkaban/Constants.java
@@ -229,19 +229,15 @@ public class Constants {
     public static final String SESSION_TIME_TO_LIVE = "session.time.to.live";
 
     // allowed max size of project dir in mb
-    public static final String PROJECT_DIR_MAX_SIZE = "azkaban.project.dir_max_size";
+    public static final String PROJECT_DIR_MAX_SIZE = "azkaban.project_cache_max_size_in_mb";
 
     // the disk usage threshold to trigger project dir cleanup.
-    // E.g, if set to 0.8, cleanup will trigger when project dir size >= 0.8 of
-    // {@value#PROJECT_DIR_MAX_SIZE}.
-    public static final String PROJECT_DIR_CLEANUP_START_THRESHOLD = "azkaban.project"
-        + ".start_cleanup_threshold";
+    // E.g, if set to 80, cleanup will trigger when project dir size >= 80% of {@value#PROJECT_DIR_MAX_SIZE}.
+    public static final String PROJECT_DIR_CLEANUP_START_THRESHOLD = "azkaban.project_cache_start_cleanup_threshold";
 
     // the disk usage threshold to stop project dir cleanup.
-    // E.g, if set to 0.6, cleanup will stop when project dir size < 0.6 of
-    // {@value#PROJECT_DIR_MAX_SIZE}.
-    public static final String PROJECT_DIR_CLEANUP_STOP_THRESHOLD = "azkaban.project"
-        + ".stop_cleanup_threshold";
+    // E.g, if set to 60, cleanup will stop when project dir size < 60% of {@value#PROJECT_DIR_MAX_SIZE}.
+    public static final String PROJECT_DIR_CLEANUP_STOP_THRESHOLD = "azkaban.project_cache.stop_cleanup_threshold";
   }
 
   public static class FlowProperties {
diff --git a/azkaban-exec-server/src/main/java/azkaban/execapp/FlowRunnerManager.java b/azkaban-exec-server/src/main/java/azkaban/execapp/FlowRunnerManager.java
index 523f815..c33ef3f 100644
--- a/azkaban-exec-server/src/main/java/azkaban/execapp/FlowRunnerManager.java
+++ b/azkaban-exec-server/src/main/java/azkaban/execapp/FlowRunnerManager.java
@@ -129,9 +129,9 @@ public class FlowRunnerManager implements EventListener,
   private final File executionDirectory;
   private final File projectDirectory;
 
-  private final long projectDirMaxSizeInMB;
-  private final double projectDirStartDeletionThreshold;
-  private final double projectDirStopDeletionThreshold;
+  private final long projectDirMaxSizeInMb;
+  private final int projectDirStartDeletionThreshold;
+  private final int projectDirStopDeletionThreshold;
 
   private final Object executionDirDeletionSync = new Object();
 
@@ -181,15 +181,15 @@ public class FlowRunnerManager implements EventListener,
       this.projectDirectory.mkdirs();
     }
 
-    this.projectDirMaxSizeInMB = props.getLong(ConfigurationKeys.PROJECT_DIR_MAX_SIZE,
-        2000000); // default value as 2TB
+    this.projectDirMaxSizeInMb = props.getLong(ConfigurationKeys.PROJECT_DIR_MAX_SIZE,
+        128000); // default value as 128GB
     this.projectDirStartDeletionThreshold = props
-        .getDouble(ConfigurationKeys.PROJECT_DIR_CLEANUP_START_THRESHOLD, 0.9);
+        .getInt(ConfigurationKeys.PROJECT_DIR_CLEANUP_START_THRESHOLD, 90);
     this.projectDirStopDeletionThreshold = props
-        .getDouble(ConfigurationKeys.PROJECT_DIR_CLEANUP_STOP_THRESHOLD, 0.8);
+        .getInt(ConfigurationKeys.PROJECT_DIR_CLEANUP_STOP_THRESHOLD, 60);
     Preconditions.checkArgument(this.projectDirStartDeletionThreshold >= 0 && this
-        .projectDirStartDeletionThreshold <= 1 && this.projectDirStopDeletionThreshold >= 0 &&
-        this.projectDirStopDeletionThreshold <= 1);
+        .projectDirStartDeletionThreshold <= 100 && this.projectDirStopDeletionThreshold >= 0 &&
+        this.projectDirStopDeletionThreshold < this.projectDirStartDeletionThreshold);
 
     this.installedProjects = loadExistingProjects();
 
@@ -1017,8 +1017,8 @@ public class FlowRunnerManager implements EventListener,
       final long dirSize;
       try {
         dirSize = FileIOUtils.sizeInKB(projectDir);
-        final long upperLimitInKB = (long) (FlowRunnerManager.this.projectDirMaxSizeInMB *
-            FlowRunnerManager.this.projectDirStartDeletionThreshold * 1024);
+        final long upperLimitInKB = (long) (FlowRunnerManager.this.projectDirMaxSizeInMb *
+            FlowRunnerManager.this.projectDirStartDeletionThreshold * 0.01 * 1024);
         return dirSize >= upperLimitInKB;
       } catch (final IOException e) {
         logger.error(e);
@@ -1030,8 +1030,8 @@ public class FlowRunnerManager implements EventListener,
       final long dirSize;
       try {
         dirSize = FileIOUtils.sizeInKB(projectDir);
-        final long lowerLimitInKB = (long) (FlowRunnerManager.this.projectDirMaxSizeInMB *
-            FlowRunnerManager.this.projectDirStopDeletionThreshold * 1024);
+        final long lowerLimitInKB = (long) (FlowRunnerManager.this.projectDirMaxSizeInMb *
+            FlowRunnerManager.this.projectDirStopDeletionThreshold * 0.01 * 1024);
         return dirSize >= lowerLimitInKB;
       } catch (final IOException e) {
         logger.error(e);