azkaban-memoizeit
Changes
src/tl/flowstats.tl 107(+64 -43)
src/tl/flowsummary.tl 24(+17 -7)
src/web/js/azkaban.flow.stats.view.js 53(+39 -14)
Details
diff --git a/src/java/azkaban/webapp/servlet/velocity/flowpage.vm b/src/java/azkaban/webapp/servlet/velocity/flowpage.vm
index aca5158..887a597 100644
--- a/src/java/azkaban/webapp/servlet/velocity/flowpage.vm
+++ b/src/java/azkaban/webapp/servlet/velocity/flowpage.vm
@@ -140,8 +140,8 @@
## Summary view.
<div class="container-full" id="summaryView">
- <div class="row" id="summary-view-content">
- </div><!-- /.row -->
+ <div id="summary-view-content">
+ </div>
<div id="flow-stats-container">
<div class="row">
<div class="col-xs-12">
src/tl/flowstats.tl 107(+64 -43)
diff --git a/src/tl/flowstats.tl b/src/tl/flowstats.tl
index c8260d7..1f012c1 100644
--- a/src/tl/flowstats.tl
+++ b/src/tl/flowstats.tl
@@ -1,19 +1,32 @@
<div class="row">
<div class="col-xs-12">
<h4>Resources</h4>
- <table class="table table-bordered table-condensed">
+ <table class="table table-bordered table-condensed table-striped">
+ <thead>
+ <tr>
+ <th class="property-key">Resource</th>
+ <th class="property-key">Value</th>
+ <th>Job Name</th>
+ </tr>
+ </thead>
<tbody>
<tr>
<td class="property-key">Max Map Slots</td>
- <td class="property-value-half">{stats.maxMapSlots}</td>
+ <td>{stats.mapSlots.max}</td>
+ <td>{stats.mapSlots.job}</td>
+ </tr>
+ <tr>
<td class="property-key">Total Map Slots</td>
- <td class="property-value-half">{stats.totalMapSlots}</td>
+ <td colspan="2">{stats.totalMapSlots}</td>
</tr>
<tr>
<td class="property-key">Max Reduce Slots</td>
- <td class="property-value-half">{stats.maxReduceSlots}</td>
+ <td>{stats.reduceSlots.max}</td>
+ <td>{stats.reduceSlots.job}</td>
+ </tr>
+ <tr>
<td class="property-key">Total Reduce Slots</td>
- <td class="property-value-half">{stats.totalReduceSlots}</td>
+ <td colspan="2">{stats.totalReduceSlots}</td>
</tr>
</tbody>
</table>
@@ -23,60 +36,68 @@
<div class="row">
<div class="col-xs-12">
<h4>Parameters</h4>
- <table class="table table-bordered table-condensed">
+ <table class="table table-bordered table-condensed table-striped">
+ <thead>
+ <tr>
+ <th class="property-key">Parameter</th>
+ <th class="property-key">Value</th>
+ <th>Job Name</th>
+ </tr>
+ </thead>
<tbody>
<tr>
<td class="property-key">Max <code>-Xmx</code></td>
- <td class="property-value-half">{stats.xmx.str}</td>
- <td class="property-key">-Xms</td>
- <td class="property-value-half">
- {?stats.xms.set}
- {stats.xms.str}
- {:else}
- Not set.
- {/stats.xms.set}
+ <td>{stats.xmx.str}</td>
+ <td>{stats.xmx.job}</td>
+ </tr>
+ <tr>
+ <td class="property-key">Max -Xms</td>
+ {?stats.xms.set}
+ <td>
+ {stats.xms.str}
</td>
+ <td>
+ {stats.xms.job}
+ </td>
+ {:else}
+ <td colspan="2">
+ Not set.
+ </td>
+ {/stats.xms.set}
</tr>
<tr>
<td class="property-key">Max <code>mapred.job.map.memory.mb</code></td>
- <td class="property-value-half">{stats.maxJobMapMemoryMb}</td>
+ <td>{stats.jobMapMemoryMb.max}</td>
+ <td>{stats.jobMapMemoryMb.job}</td>
+ </tr>
+ <tr>
<td class="property-key">Max <code>mapred.job.reduce.memory.mb</code></td>
- <td class="property-value-half">{stats.maxJobReduceMemoryMb}</td>
+ <td>{stats.jobReduceMemoryMb.max}</td>
+ <td>{stats.jobReduceMemoryMb.job}</td>
</tr>
</tbody>
</table>
</div>
</div>
- <div class="row row-offcanvas row-offcanvas-left">
- <div class="col-xs-6 col-sm-3 sidebar-offcanvas">
- <ul class="nav nav-pills nav-stacked">
- <li class="active"><a href="#parameters-xmx" data-toggle="tab">Xmx</a></li>
- <li><a href="#parameters-xmx" data-toggle="tab">Xms</a></li>
- <li><a href="#parameters-map-memory" data-toggle="tab">mapred.job.map.memory.mb</a></li>
- <li><a href="#parameters-reduce-memory" data-toggle="tab">mapred.job.reduce.memory.mb</a></li>
- </ul>
- </div>
- <div class="col-xs-12 col-sm-9">
- <div class="tab-content">
- <div class="tab-pane active" id="parameters-xmx">
- Xmx
- </div>
- <div class="tab-pane" id="parameters-xms">
- Xms
- </div>
- <div class="tab-pane" id="parameters-map-memory">
- mapred.job.map.memory.mb
- </div>
- <div class="tab-pane" id="parameters-reduce-memory">
- mapred.job.reduce.memory.mb
- </div>
- </div>
- </div>
- </div>
<div class="row">
<div class="col-xs-12">
<h4>Counters</h4>
+ <table class="table table-bordered table-condensed">
+ <tbody>
+ <tr>
+ <td class="property-key">Max <code>FILE_BYTES_READ</code></td>
+ <td class="property-value-half">{stats.maxFileBytesRead}</td>
+ <td class="property-key">Max <code>HDFS_BYTES_READ</code></td>
+ <td class="property-value-half">{stats.maxHdfsBytesRead}</td>
+ </tr>
+ <tr>
+ <td class="property-key">Max <code>FILE_BYTES_WRITTEN</code></td>
+ <td class="property-value-half">{stats.}</td>
+ <td class="property-key">Max <code>mapred.job.reduce.memory.mb</code></td>
+ <td class="property-value-half">{stats.maxJobReduceMemoryMb}</td>
+ </tr>
+ </tbody>
+ </table>
</div>
</div>
-
src/tl/flowsummary.tl 24(+17 -7)
diff --git a/src/tl/flowsummary.tl b/src/tl/flowsummary.tl
index 5f9dbdd..12def49 100644
--- a/src/tl/flowsummary.tl
+++ b/src/tl/flowsummary.tl
@@ -1,21 +1,30 @@
- <div class="col-xs-12">
+ <div class="row">
+ <div class="col-xs-4">
+ <h1>3 <small>Successful</small> 4 <small>Failed</small></h1>
+ </div>
+ <div class="col-xs-8">
<table class="table table-bordered table-condensed">
<tbody>
<tr>
- <td class="property-key">Flow name</td>
- <td class="property-value-half">{flowName}</td>
<td class="property-key">Project name</td>
- <td class="property-value-half">{projectName}</td>
+ <td>{projectName}</td>
</tr>
<tr>
<td class="property-key">Run As</td>
- <td class="property-value-half">{user}</td>
+ <td>{user}</td>
+ </tr>
+ <tr>
<td class="property-key">Job Types Used</td>
- <td class="property-value-half">{#jobTypes}{.} {/jobTypes}</td>
+ <td>{#jobTypes}{.} {/jobTypes}</td>
</tr>
</tbody>
</table>
-
+ </div>
+ </div>
+ <hr>
+
+ <div class="row">
+ <div class="col-xs-12">
<h3>
Scheduling
{?schedule}
@@ -65,3 +74,4 @@
<h3>Last Run Stats</h3>
</div>
+ </div>
src/web/js/azkaban.flow.stats.view.js 53(+39 -14)
diff --git a/src/web/js/azkaban.flow.stats.view.js b/src/web/js/azkaban.flow.stats.view.js
index 76da587..eb9e97a 100644
--- a/src/web/js/azkaban.flow.stats.view.js
+++ b/src/web/js/azkaban.flow.stats.view.js
@@ -75,28 +75,37 @@ azkaban.FlowStatsView = Backbone.View.extend({
return stats;
},
- updateStats: function(jobStats, data) {
+ updateStats: function(jobStats, data, job) {
var stats = data.stats;
var state = jobStats.state;
var conf = jobStats.conf;
var mappers = parseInt(state.totalMappers);
var reducers = parseInt(state.totalReducers);
- if (mappers > stats.maxMapSlots) {
- stats.maxMapSlots = mappers;
+ if (mappers > stats.mapSlots.max) {
+ stats.mapSlots.max = mappers;
+ stats.mapSlots.job = job;
}
- if (reducers > stats.maxReduceSlots) {
- stats.maxReduceSlots = reducers;
+ if (reducers > stats.reduceSlots.max) {
+ stats.reduceSlots.max = reducers;
+ stats.reduceSlots.job = job;
}
stats.totalMapSlots += mappers;
stats.totalReduceSlots += reducers;
+ if (conf == null) {
+ data.warnings.push("No job conf available for job " + job);
+ return;
+ }
+
var jobMapMemoryMb = parseInt(conf['mapred.job.map.memory.mb']);
if (jobMapMemoryMb > stats.maxJobMapMemoryMb) {
- stats.maxJobMapMemoryMb = jobMapMemoryMb;
+ stats.jobMapMemoryMb.max = jobMapMemoryMb;
+ stats.jobMapMemoryMb.job = job;
}
var jobReduceMemoryMb = parseInt(conf['mapred.job.reduce.memory.mb']);
if (jobReduceMemoryMb > stats.maxJobReduceMemoryMb) {
- stats.maxJobReduceMemoryMb = jobReduceMemoryMb;
+ stats.jobReduceMemoryMb.max = jobReduceMemoryMb;
+ stats.jobReduceMemoryMb.job = job;
}
var childJavaOpts = conf['mapred.child.java.opts'];
@@ -112,6 +121,7 @@ azkaban.FlowStatsView = Backbone.View.extend({
if (val > stats.xmx.max) {
stats.xmx.max = val;
stats.xmx.str = size;
+ stats.xmx.job = job;
}
}
if (str.indexOf('Xms') > -1) {
@@ -124,6 +134,7 @@ azkaban.FlowStatsView = Backbone.View.extend({
if (val > stats.xms.max) {
stats.xms.max = val;
stats.xms.str = size;
+ stats.xms.job = job;
}
}
}
@@ -146,22 +157,36 @@ azkaban.FlowStatsView = Backbone.View.extend({
message: null,
warnings: [],
stats: {
- maxMapSlots: 0,
- maxReduceSlots: 0,
+ mapSlots: {
+ max: 0,
+ job: null
+ },
+ reduceSlots: {
+ max: 0,
+ job: null
+ },
totalMapSlots: 0,
totalReduceSlots: 0,
numJobs: jobs.length,
longestTaskTime: 0,
- maxJobMapMemoryMb: 0,
- maxJobReduceMemoryMb: 0,
+ jobMapMemoryMb: {
+ max: 0,
+ job: null
+ },
+ jobReduceMemoryMb: {
+ max: 0,
+ job: null
+ },
xmx: {
max: 0,
- str: null
+ str: null,
+ job: null
},
xms: {
set: false,
max: 0,
- str: null
+ str: null,
+ job: null
}
}
};
@@ -174,7 +199,7 @@ azkaban.FlowStatsView = Backbone.View.extend({
continue;
}
for (var j = 0; j < jobStats.jobStats.length; ++j) {
- this.updateStats(jobStats.jobStats[j], data);
+ this.updateStats(jobStats.jobStats[j], data, job);
}
}
this.finalizeStats(data);