azkaban-memoizeit
Changes
src/tl/flowsummary.tl 104(+50 -54)
src/tl/flowsummary-last-run.tl 2(+0 -2)
src/web/js/azkaban.flow.view.js 43(+31 -12)
Details
diff --git a/src/java/azkaban/webapp/servlet/ExecutorServlet.java b/src/java/azkaban/webapp/servlet/ExecutorServlet.java
index f5e4b4f..51c0dab 100644
--- a/src/java/azkaban/webapp/servlet/ExecutorServlet.java
+++ b/src/java/azkaban/webapp/servlet/ExecutorServlet.java
@@ -17,6 +17,7 @@
package azkaban.webapp.servlet;
import java.io.IOException;
+import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -45,6 +46,7 @@ import azkaban.user.User;
import azkaban.user.Permission.Type;
import azkaban.utils.FileIOUtils.LogData;
import azkaban.utils.LogSummary;
+import azkaban.utils.JSONUtils;
import azkaban.webapp.AzkabanWebServer;
import azkaban.webapp.session.Session;
@@ -55,6 +57,8 @@ public class ExecutorServlet extends LoginAbstractAzkabanServlet {
private ScheduleManager scheduleManager;
private ExecutorVelocityHelper velocityHelper;
+ private String statsDir;
+
@Override
public void init(ServletConfig config) throws ServletException {
super.init(config);
@@ -63,6 +67,7 @@ public class ExecutorServlet extends LoginAbstractAzkabanServlet {
executorManager = server.getExecutorManager();
scheduleManager = server.getScheduleManager();
velocityHelper = new ExecutorVelocityHelper();
+ statsDir = server.getServerProps().getString("azkaban.stats.dir");
}
@Override
@@ -513,8 +518,9 @@ public class ExecutorServlet extends LoginAbstractAzkabanServlet {
return;
}
- String jobId = this.getParam(req, "jobId");
+ String jobId = this.getParam(req, "jobid");
resp.setCharacterEncoding("utf-8");
+ String statsFilePath = null;
try {
ExecutableNode node = exFlow.getExecutableNode(jobId);
if (node == null) {
@@ -523,13 +529,16 @@ public class ExecutorServlet extends LoginAbstractAzkabanServlet {
return;
}
- // XXX
- outputDir = props.getString("azkaban.stats.dir",
- System.getProperty("java.io.tmpdir"));
-
- }
- catch (ExecutorManagerException e) {
- throw new ServletException(e);
+ statsFilePath = statsDir + "/" + exFlow.getExecutionId() + "-" +
+ jobId + "-stats.json";
+ File statsFile = new File(statsFilePath);
+ List<Object> jsonObj =
+ (ArrayList<Object>) JSONUtils.parseJSONFromFile(statsFile);
+ ret.put("jobStats", jsonObj);
+ }
+ catch (IOException e) {
+ ret.put("error", "Cannot open stats file: " + statsFilePath);
+ return;
}
}
src/tl/flowsummary.tl 104(+50 -54)
diff --git a/src/tl/flowsummary.tl b/src/tl/flowsummary.tl
index 132e660..4c287c3 100644
--- a/src/tl/flowsummary.tl
+++ b/src/tl/flowsummary.tl
@@ -20,65 +20,61 @@
Scheduling
{?schedule}
<div class="pull-right">
- <button type="button" id="removeSchedBtn" class="btn btn-xs btn-danger" onclick="removeSched({schedule.scheduleId})" >Remove Schedule</button>
+ <button type="button" id="removeSchedBtn" class="btn btn-sm btn-danger" onclick="removeSched({schedule.scheduleId})" >Remove Schedule</button>
</div>
{/schedule}
</h3>
- <div class="panel panel-default">
- <div class="panel-heading">
- Scheduling
- </div>
- {?schedule}
- <table class="table table-condensed table-bordered">
- <tbody>
- <tr>
- <td class="property-key">Schedule ID</td>
- <td class="property-value-half">{schedule.scheduleId}</td>
- <td class="property-key">Submitted By</td>
- <td class="property-value-half">{schedule.submitUser}</td>
- </tr>
- <tr>
- <td class="property-key">First Scheduled to Run</td>
- <td class="property-value-half">{schedule.firstSchedTime}</td>
- <td class="property-key">Repeats Every</td>
- <td class="property-value-half">{schedule.period}</td>
- </tr>
- <tr>
- <td class="property-key">Next Execution Time</td>
- <td class="property-value-half">{schedule.nextExecTime}</td>
- <td class="property-key">SLA</td>
- <td class="property-value-half">
- {?schedule.slaOptions}
- true
- {:else}
- false
- {/schedule.slaOptions}
- <div class="pull-right">
- <button type="button" id="addSlaBtn" class="btn btn-xs btn-primary" onclick="slaView.initFromSched({schedule.scheduleId}, '{flowName}')" >Set SLA</button>
- </div>
- </td>
- </tr>
- </tbody>
- </table>
- {:else}
- <div class="panel-body">
- <div class="alert alert-info">
- <h4>None</h4>
- <p>This flow has not been scheduled.</p>
- </div>
- </div>
- {/schedule}
- </div>
+ {?schedule}
+ <table class="table table-condensed table-bordered">
+ <tbody>
+ <tr>
+ <td class="property-key">Schedule ID</td>
+ <td class="property-value-half">{schedule.scheduleId}</td>
+ <td class="property-key">Submitted By</td>
+ <td class="property-value-half">{schedule.submitUser}</td>
+ </tr>
+ <tr>
+ <td class="property-key">First Scheduled to Run</td>
+ <td class="property-value-half">{schedule.firstSchedTime}</td>
+ <td class="property-key">Repeats Every</td>
+ <td class="property-value-half">{schedule.period}</td>
+ </tr>
+ <tr>
+ <td class="property-key">Next Execution Time</td>
+ <td class="property-value-half">{schedule.nextExecTime}</td>
+ <td class="property-key">SLA</td>
+ <td class="property-value-half">
+ {?schedule.slaOptions}
+ true
+ {:else}
+ false
+ {/schedule.slaOptions}
+ <div class="pull-right">
+ <button type="button" id="addSlaBtn" class="btn btn-xs btn-primary" onclick="slaView.initFromSched({schedule.scheduleId}, '{flowName}')" >Set SLA</button>
+ </div>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ {:else}
+ <div class="alert alert-info">
+ <h4>None</h4>
+ <p>This flow has not been scheduled.</p>
+ </div>
+ {/schedule}
</div>
- <div class="col-xs-12" id="last-run-container">
- <div class="alert alert-info" id="analyze-last-run">
- <h4>Analyze last run</h4>
- <p>Analyze the last run for aggregate performance statistics.</p>
- <p><strong>Note:</strong> this may take a few minutes, especially if your flow is large.</p>
- <p>
- <button type="button" id="analyze-btn" class="btn btn-primary">Analyze</button>
- </p>
+ <div class="col-xs-12">
+ <h3>Last Run Stats</h3>
+ <div id="last-run-container">
+ <div class="alert alert-info" id="analyze-last-run">
+ <h4>Analyze last run</h4>
+ <p>Analyze the last run for aggregate performance statistics.</p>
+ <p><strong>Note:</strong> this may take a few minutes, especially if your flow is large.</p>
+ <p>
+ <button type="button" id="analyze-btn" class="btn btn-primary">Analyze</button>
+ </p>
+ </div>
</div>
</div><!-- /.col-lg-12 -->
src/tl/flowsummary-last-run.tl 2(+0 -2)
diff --git a/src/tl/flowsummary-last-run.tl b/src/tl/flowsummary-last-run.tl
index 5a38710..8cf6455 100644
--- a/src/tl/flowsummary-last-run.tl
+++ b/src/tl/flowsummary-last-run.tl
@@ -1,5 +1,3 @@
- <h4>Last Run Stats</h4>
- <hr>
<div class="panel panel-default">
<div class="panel-heading">Resources</div>
<table class="table table-striped table-bordered table-condensed">
src/web/js/azkaban.flow.view.js 43(+31 -12)
diff --git a/src/web/js/azkaban.flow.view.js b/src/web/js/azkaban.flow.view.js
index 41b677f..959dc4a 100644
--- a/src/web/js/azkaban.flow.view.js
+++ b/src/web/js/azkaban.flow.view.js
@@ -346,7 +346,7 @@ azkaban.SummaryView = Backbone.View.extend({
};
var view = this;
var successHandler = function(data) {
- if (data.success == false || data.execId == null) {
+ if (data.success == "false" || data.execId == null) {
view.renderLastRun(data);
return;
}
@@ -365,10 +365,17 @@ azkaban.SummaryView = Backbone.View.extend({
jobs.push(node.id);
}
};
+ $.ajax({
+ url: requestURL,
+ data: requestData,
+ success: successHandler,
+ dataType: "json",
+ async: false
+ });
return jobs;
},
- fetchJobStats: function(jobId) {
+ fetchJobStats: function(jobId, execId) {
var requestURL = contextURL + "/executor";
var requestData = {
"execid": execId,
@@ -380,6 +387,13 @@ azkaban.SummaryView = Backbone.View.extend({
var successHandler = function(data) {
stats = data;
};
+ $.ajax({
+ url: requestURL,
+ data: requestData,
+ success: successHandler,
+ dataType: "json",
+ async: false
+ });
return stats;
},
@@ -387,14 +401,16 @@ azkaban.SummaryView = Backbone.View.extend({
var aggregateStats = data.stats;
var state = jobStats.state;
var conf = jobStats.conf;
- if (state.numMaps > aggregateStats.maxMapSlots) {
- aggregateStats.maxMapSlots = state.numMaps;
+ var mappers = parseInt(state.totalMappers);
+ var reducers = parseInt(state.totalReducers);
+ if (mappers > aggregateStats.maxMapSlots) {
+ aggregateStats.maxMapSlots = mappers;
}
- if (state.numReduces > aggregateStats.maxReduceSlots) {
- aggregateStats.maxReduceSlots = state.numReduces;
+ if (reducers > aggregateStats.maxReduceSlots) {
+ aggregateStats.maxReduceSlots = reducers;
}
- aggregateStats.totalMapSlots += state.numMaps;
- aggregateStats.totalReduceSlots += state.numReduces;
+ aggregateStats.totalMapSlots += mappers;
+ aggregateStats.totalReduceSlots += reducers;
},
analyzeLastRun: function(execId) {
@@ -420,11 +436,14 @@ azkaban.SummaryView = Backbone.View.extend({
for (var i = 0; i < jobs.length; ++i) {
var job = jobs[i];
- var jobStats = this.fetchJobStats(job.id);
- if (jobStats == null) {
+ var jobStats = this.fetchJobStats(job, execId);
+ if (jobStats.jobStats == null) {
data.warnings.push("No job stats available for job " + job.id);
+ continue;
+ }
+ for (var j = 0; j < jobStats.jobStats.length; ++j) {
+ this.updateStats(jobStats.jobStats[j], data);
}
- this.updateStats(jobStats, data);
}
data.success = true;
this.renderLastRun(data);
@@ -438,7 +457,7 @@ azkaban.SummaryView = Backbone.View.extend({
view.displayLastRun(out);
});
}
- else if (data.success == false) {
+ else if (data.success == "false") {
dust.render("flowsummary-no-data", data, function(err, out) {
view.displayLastRun(out);
});