azkaban-aplcache

diff --git a/azkaban-common/build.gradle b/azkaban-common/build.gradle
index 658344f..d747d52 100644
--- a/azkaban-common/build.gradle
+++ b/azkaban-common/build.gradle
@@ -14,7 +14,13 @@
  * the License.
  */
 
+plugins {
+    id "io.freefair.aspectj.post-compile-weaving" version "3.1.4"
+}
+
 dependencies {
+    aspect 'br.ufrgs.inf.prosoft.applicationtracer:ApplicationTracer:1.0'
+
     compile project(':az-core')
     compile project(':azkaban-spi')
     compile project(':azkaban-db')

diff --git a/azkaban-common/ignored b/azkaban-common/ignored
new file mode 100644
index 0000000..a792717
--- /dev/null
+++ b/azkaban-common/ignored
@@ -0,0 +1,51 @@
+br.ufrgs.inf.prosoft.jsonserialiser.
+br.ufrgs.inf.prosoft.trace.
+com.cinnober.gradle.
+com.fasterxml.jackson.core.
+com.github.kt3k.coveralls.
+com.google.code.findbugs.
+com.google.code.gson.
+com.google.errorprone.
+com.google.guava.
+com.google.inject.
+com.gradle.build-scan.
+com.h2database.
+com.linkedin.pegasus.
+com.twitter.
+commons-cli.
+commons-collections.
+commons-dbutils.
+commons-fileupload.
+commons-io.
+commons-lang.
+io.dropwizard.metrics.
+io.freefair.aspectj.post-compile-weaving.
+jacoco.
+javax.mail.
+joda-time.
+junit.
+log4j.
+mysql.
+net.ltgt.errorprone.
+net.ltgt.gradle.
+net.sf.jopt-simple.
+org.apache.commons.
+org.apache.hadoop.
+org.apache.hive.
+org.apache.httpcomponents.
+org.apache.kafka.
+org.apache.pig.
+org.apache.spark.
+org.apache.velocity.
+org.aspectj.
+org.assertj.
+org.awaitility.
+org.bouncycastle.
+org.codehaus.jackson.
+org.jasypt.
+org.mockito.
+org.mongodb.
+org.mortbay.jetty.
+org.quartz-scheduler.
+org.slf4j.
+org.yaml.
\ No newline at end of file

diff --git a/azkaban-common/src/main/java/azkaban/executor/ActiveExecutors.java b/azkaban-common/src/main/java/azkaban/executor/ActiveExecutors.java
index c0aadce..67235f8 100644
--- a/azkaban-common/src/main/java/azkaban/executor/ActiveExecutors.java
+++ b/azkaban-common/src/main/java/azkaban/executor/ActiveExecutors.java
@@ -45,14 +45,15 @@ public class ActiveExecutors {
    * fails.
    */
   public void setupExecutors() throws ExecutorManagerException {
-    final ImmutableSet<Executor> newExecutors = loadExecutors();
-    if (newExecutors.isEmpty()) {
-      final String error = "No active executors found";
-      logger.error(error);
-      throw new ExecutorManagerException(error);
-    } else {
-      this.activeExecutors = newExecutors;
-    }
+    // meloca cache
+    // final ImmutableSet<Executor> newExecutors = loadExecutors();
+    // if (newExecutors.isEmpty()) {
+    //   final String error = "No active executors found";
+    //   logger.error(error);
+    //   throw new ExecutorManagerException(error);
+    // } else {
+    //   this.activeExecutors = newExecutors;
+    // }
   }
 
   /**
@@ -61,6 +62,19 @@ public class ActiveExecutors {
    * @return all executors
    */
   public Collection<Executor> getAll() {
+    // meloca cache
+    try{
+      final ImmutableSet<Executor> newExecutors = loadExecutors();
+      if (newExecutors.isEmpty()) {
+        final String error = "No active executors found";
+        logger.error(error);
+      } else {
+        this.activeExecutors = newExecutors;
+      }
+    } catch (ExecutorManagerException ex) {
+        logger.error("ExecutorManagerException: " + ex.getMessage());
+        this.activeExecutors = ImmutableSet.of();
+    }
     return this.activeExecutors;
   }
 

diff --git a/azkaban-common/src/main/java/azkaban/project/ProjectManager.java b/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
index 0b059ae..b5a838a 100644
--- a/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
+++ b/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
@@ -58,10 +58,10 @@ public class ProjectManager {
   private final boolean creatorDefaultPermissions;
   // Both projectsById and projectsByName cache need to be thread safe since they are accessed
   // from multiple threads concurrently without external synchronization for performance.
-  private final ConcurrentHashMap<Integer, Project> projectsById =
-      new ConcurrentHashMap<>();
-  private final CaseInsensitiveConcurrentHashMap<Project> projectsByName =
-      new CaseInsensitiveConcurrentHashMap<>();
+  // private final ConcurrentHashMap<Integer, Project> projectsById =
+      // new ConcurrentHashMap<>();
+  // private final CaseInsensitiveConcurrentHashMap<Project> projectsByName =
+      // new CaseInsensitiveConcurrentHashMap<>();
 
 
   @Inject
@@ -122,10 +122,10 @@ public class ProjectManager {
     } catch (final ProjectManagerException e) {
       throw new RuntimeException("Could not load projects from store.", e);
     }
-    for (final Project proj : projects) {
-      this.projectsByName.put(proj.getName(), proj);
-      this.projectsById.put(proj.getId(), proj);
-    }
+    // for (final Project proj : projects) {
+    //   this.projectsByName.put(proj.getName(), proj);
+    //   this.projectsById.put(proj.getId(), proj);
+    // }
 
     for (final Project proj : projects) {
       loadAllProjectFlows(proj);
@@ -152,7 +152,8 @@ public class ProjectManager {
 
   public List<Project> getUserProjects(final User user) {
     final ArrayList<Project> array = new ArrayList<>();
-    for (final Project project : this.projectsById.values()) {
+    for (final Project project : getProjects()) {
+    // for (final Project project : this.projectsById.values()) {
       final Permission perm = project.getUserPermission(user);
 
       if (perm != null
@@ -166,7 +167,8 @@ public class ProjectManager {
 
   public List<Project> getGroupProjects(final User user) {
     final List<Project> array = new ArrayList<>();
-    for (final Project project : this.projectsById.values()) {
+    // for (final Project project : this.projectsById.values()) {
+    for (final Project project : getProjects()) {
       if (project.hasGroupPermission(user, Type.READ)) {
         array.add(project);
       }
@@ -184,7 +186,8 @@ public class ProjectManager {
       return array;
     }
 
-    for (final Project project : this.projectsById.values()) {
+    for (final Project project : getProjects()) {
+    // for (final Project project : this.projectsById.values()) {
       final Permission perm = project.getUserPermission(user);
 
       if (perm != null
@@ -199,7 +202,12 @@ public class ProjectManager {
   }
 
   public List<Project> getProjects() {
-    return new ArrayList<>(this.projectsById.values());
+    // return new ArrayList<>(this.projectsById.values());
+    try {
+      return this.projectLoader.fetchAllActiveProjects();
+    } catch (final ProjectManagerException e) {
+      return new ArrayList<>();
+    }
   }
 
   public List<Project> getProjectsByRegex(final String regexPattern) {
@@ -223,14 +231,21 @@ public class ProjectManager {
    * Checks if a project is active using project_id
    */
   public Boolean isActiveProject(final int id) {
-    return this.projectsById.containsKey(id);
+    // return this.projectsById.containsKey(id);
+    try {
+        this.projectLoader.fetchProjectById(id);
+        return true;
+      } catch (final ProjectManagerException e) {
+        return false;
+      }
   }
 
   /**
    * fetch active project from cache and inactive projects from db by project_name
    */
   public Project getProject(final String name) {
-    Project fetchedProject = this.projectsByName.get(name);
+    // Project fetchedProject = this.projectsByName.get(name);
+    Project fetchedProject = null;
     if (fetchedProject == null) {
       try {
         logger.info("Project " + name + " doesn't exist in cache, fetching from DB now.");
@@ -246,7 +261,8 @@ public class ProjectManager {
    * fetch active project from cache and inactive projects from db by project_id
    */
   public Project getProject(final int id) {
-    Project fetchedProject = this.projectsById.get(id);
+    // Project fetchedProject = this.projectsById.get(id);
+    Project fetchedProject = null;
     if (fetchedProject == null) {
       try {
         fetchedProject = this.projectLoader.fetchProjectById(id);
@@ -272,15 +288,15 @@ public class ProjectManager {
 
     final Project newProject;
     synchronized (this) {
-      if (this.projectsByName.containsKey(projectName)) {
-        throw new ProjectManagerException("Project already exists.");
-      }
+      // if (this.projectsByName.containsKey(projectName)) {
+      //   throw new ProjectManagerException("Project already exists.");
+      // }
 
       logger.info("Trying to create " + projectName + " by user "
           + creator.getUserId());
       newProject = this.projectLoader.createNewProject(projectName, description, creator);
-      this.projectsByName.put(newProject.getName(), newProject);
-      this.projectsById.put(newProject.getId(), newProject);
+      // this.projectsByName.put(newProject.getName(), newProject);
+      // this.projectsById.put(newProject.getId(), newProject);
     }
 
     if (this.creatorDefaultPermissions) {
@@ -324,8 +340,8 @@ public class ProjectManager {
     this.projectLoader.postEvent(project, EventType.DELETED, deleter.getUserId(),
         null);
 
-    this.projectsByName.remove(project.getName());
-    this.projectsById.remove(project.getId());
+    // this.projectsByName.remove(project.getName());
+    // this.projectsById.remove(project.getId());
 
     return project;
   }

diff --git a/azkaban-exec-server/build.gradle b/azkaban-exec-server/build.gradle
index 8c01c99..ac5d5e6 100644
--- a/azkaban-exec-server/build.gradle
+++ b/azkaban-exec-server/build.gradle
@@ -14,9 +14,15 @@
  * the License.
  */
 
+plugins {
+    id "io.freefair.aspectj.post-compile-weaving" version "3.1.4"
+}
+
 apply plugin: 'distribution'
 
 dependencies {
+    aspect 'br.ufrgs.inf.prosoft.applicationtracer:ApplicationTracer:1.0'
+
     compile(project(':az-core'))
     compile(project(':azkaban-common'))
 

diff --git a/azkaban-exec-server/ignored b/azkaban-exec-server/ignored
new file mode 100644
index 0000000..a792717
--- /dev/null
+++ b/azkaban-exec-server/ignored
@@ -0,0 +1,51 @@
+br.ufrgs.inf.prosoft.jsonserialiser.
+br.ufrgs.inf.prosoft.trace.
+com.cinnober.gradle.
+com.fasterxml.jackson.core.
+com.github.kt3k.coveralls.
+com.google.code.findbugs.
+com.google.code.gson.
+com.google.errorprone.
+com.google.guava.
+com.google.inject.
+com.gradle.build-scan.
+com.h2database.
+com.linkedin.pegasus.
+com.twitter.
+commons-cli.
+commons-collections.
+commons-dbutils.
+commons-fileupload.
+commons-io.
+commons-lang.
+io.dropwizard.metrics.
+io.freefair.aspectj.post-compile-weaving.
+jacoco.
+javax.mail.
+joda-time.
+junit.
+log4j.
+mysql.
+net.ltgt.errorprone.
+net.ltgt.gradle.
+net.sf.jopt-simple.
+org.apache.commons.
+org.apache.hadoop.
+org.apache.hive.
+org.apache.httpcomponents.
+org.apache.kafka.
+org.apache.pig.
+org.apache.spark.
+org.apache.velocity.
+org.aspectj.
+org.assertj.
+org.awaitility.
+org.bouncycastle.
+org.codehaus.jackson.
+org.jasypt.
+org.mockito.
+org.mongodb.
+org.mortbay.jetty.
+org.quartz-scheduler.
+org.slf4j.
+org.yaml.
\ No newline at end of file

diff --git a/azkaban-exec-server/src/main/java/azkaban/execapp/ServerStatisticsServlet.java b/azkaban-exec-server/src/main/java/azkaban/execapp/ServerStatisticsServlet.java
index 0354bf3..e43bbff 100644
--- a/azkaban-exec-server/src/main/java/azkaban/execapp/ServerStatisticsServlet.java
+++ b/azkaban-exec-server/src/main/java/azkaban/execapp/ServerStatisticsServlet.java
@@ -63,11 +63,13 @@ public class ServerStatisticsServlet extends HttpServlet {
   protected void doPost(final HttpServletRequest req, final HttpServletResponse resp)
       throws ServletException, IOException {
 
-    final boolean noCache = null != req && Boolean.valueOf(req.getParameter(noCacheParamName));
+    // final boolean noCache = null != req && Boolean.valueOf(req.getParameter(noCacheParamName));
 
-    if (noCache || System.currentTimeMillis() - lastRefreshedTime > cacheTimeInMilliseconds) {
-      this.populateStatistics(noCache);
-    }
+    // if (noCache || System.currentTimeMillis() - lastRefreshedTime > cacheTimeInMilliseconds) {
+    //   this.populateStatistics(noCache);
+    // }
+
+    this.populateStatistics(true);
 
     JSONUtils.toJSON(cachedstats, resp.getOutputStream(), true);
   }

diff --git a/build.gradle b/build.gradle
index e825fdc..b63e397 100644
--- a/build.gradle
+++ b/build.gradle
@@ -28,15 +28,15 @@ buildscript {
 }
 
 plugins {
-    id 'com.gradle.build-scan' version '1.9'
+    id 'com.gradle.build-scan' version '2.0.2'
     id 'com.github.kt3k.coveralls' version '2.6.3'
     id 'jacoco'
 }
 
-buildScan {
-    licenseAgreementUrl = 'https://gradle.com/terms-of-service'
-    licenseAgree = 'yes'
-}
+// buildScan {
+//     licenseAgreementUrl = 'https://gradle.com/terms-of-service'
+//     licenseAgree = 'yes'
+// }
 
 apply plugin: 'com.cinnober.gradle.semver-git'
 
@@ -194,9 +194,9 @@ subprojects {
     /**
      * Terminate compilation if warnings occur.
      */
-    tasks.withType(JavaCompile) {
-        options.compilerArgs += ["-Werror"]
-    }
+    // tasks.withType(JavaCompile) {
+    //     options.compilerArgs += ["-Werror"]
+    // }
 
     /**
      * Print test execution summary when informational logging is enabled.
@@ -223,9 +223,9 @@ subprojects {
 /**
  * Gradle wrapper task.
  */
-task wrapper(type: Wrapper) {
-    distributionType('ALL')
-}
+// task wrapper(type: Wrapper) {
+//     distributionType('ALL')
+// }
 
 if (System.env.TRAVIS == 'true') {
     allprojects {

diff --git a/docker-compose.yml b/docker-compose.yml
index 3cc1fd6..e015d5a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -3,10 +3,16 @@ services:
   application:
     build: .
     container_name: azkaban
+    environment:
+      - TRACER_SERIALISE_INTERNALS=false
+      - TRACER_VERBOSE=true
+      - TRACER_ENABLE=true
+      - TRACER_TRACES=/caching-approaches-comparison/applications/traces/azkaban
     volumes:
       - application:/application
       - /root/.gradle:/home/gradle/.gradle
       - /root/.m2:/root/.m2
+      - ../../../:/caching-approaches-comparison
     restart: unless-stopped
     ports:
       - 8080:8081

diff --git a/Dockerfile b/Dockerfile
index 5e4bef6..67f0261 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM gradle:4.9
+FROM gradle:5.1.1
 
 USER root