azkaban-memoizeit
Changes
src/tl/flowstats.tl 117(+80 -37)
src/tl/flowstats-no-data.tl 4(+4 -0)
src/tl/flowsummary.tl 2(+2 -0)
src/web/js/azkaban.common.utils.js 19(+19 -0)
src/web/js/azkaban.exflow.view.js 2(+1 -1)
src/web/js/azkaban.flow.stats.view.js 71(+60 -11)
src/web/js/azkaban.flow.view.js 2(+1 -1)
Details
diff --git a/src/java/azkaban/webapp/servlet/velocity/executingflowpage.vm b/src/java/azkaban/webapp/servlet/velocity/executingflowpage.vm
index 75bd7d0..e373603 100644
--- a/src/java/azkaban/webapp/servlet/velocity/executingflowpage.vm
+++ b/src/java/azkaban/webapp/servlet/velocity/executingflowpage.vm
@@ -172,11 +172,13 @@
## Stats view.
<div class="container-full" id="statsView">
- <div class="row" id="stats-view-content">
- <div class="col-lg-12" id="flow-stats-container">
- <div class="alert alert-default">
- <h4>No stats available</h4>
- <p>Stats for this flow execution are not available.</p>
+ <div id="flow-stats-container">
+ <div class="row">
+ <div class="col-lg-12">
+ <div class="alert alert-default">
+ <h4>No stats available</h4>
+ <p>Stats for this flow execution are not available.</p>
+ </div>
</div>
</div>
</div><!-- /.row -->
diff --git a/src/java/azkaban/webapp/servlet/velocity/flowpage.vm b/src/java/azkaban/webapp/servlet/velocity/flowpage.vm
index af64cd7..aca5158 100644
--- a/src/java/azkaban/webapp/servlet/velocity/flowpage.vm
+++ b/src/java/azkaban/webapp/servlet/velocity/flowpage.vm
@@ -142,10 +142,9 @@
<div class="container-full" id="summaryView">
<div class="row" id="summary-view-content">
</div><!-- /.row -->
- <div class="row" id="last-run-stats">
- <div class="col-xs-12">
- <h3>Last Run Stats</h3>
- <div id="flow-stats-container">
+ <div id="flow-stats-container">
+ <div class="row">
+ <div class="col-xs-12">
<div class="alert alert-info">
<h4>Analyze last run</h4>
<p>Analyze the last run for aggregate performance statistics. <strong>Note:</strong> this may take a few minutes, especially if your flow is large.</p>
src/tl/flowstats.tl 117(+80 -37)
diff --git a/src/tl/flowstats.tl b/src/tl/flowstats.tl
index cbd1ec4..c8260d7 100644
--- a/src/tl/flowstats.tl
+++ b/src/tl/flowstats.tl
@@ -1,39 +1,82 @@
- <div class="panel panel-default">
- <div class="panel-heading">Resources</div>
- <table class="table table-striped table-bordered table-condensed">
- <tbody>
- <tr>
- <td class="property-key">Max Map Slots</td>
- <td class="property-value-half">{stats.maxMapSlots}</td>
- <td class="property-key">Total Map Slots</td>
- <td class="property-value-half">{stats.totalMapSlots}</td>
- </tr>
- <tr>
- <td class="property-key">Max Reduce Slots</td>
- <td class="property-value-half">{stats.maxReduceSlots}</td>
- <td class="property-key">Total Reduce Slots</td>
- <td class="property-value-half">{stats.totalReduceSlots}</td>
- </tr>
- </tbody>
- </table>
- </div>
+ <div class="row">
+ <div class="col-xs-12">
+ <h4>Resources</h4>
+ <table class="table table-bordered table-condensed">
+ <tbody>
+ <tr>
+ <td class="property-key">Max Map Slots</td>
+ <td class="property-value-half">{stats.maxMapSlots}</td>
+ <td class="property-key">Total Map Slots</td>
+ <td class="property-value-half">{stats.totalMapSlots}</td>
+ </tr>
+ <tr>
+ <td class="property-key">Max Reduce Slots</td>
+ <td class="property-value-half">{stats.maxReduceSlots}</td>
+ <td class="property-key">Total Reduce Slots</td>
+ <td class="property-value-half">{stats.totalReduceSlots}</td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </div>
- <div class="panel panel-default">
- <div class="panel-heading">Parameters</div>
- <table class="table table-striped table-bordered table-condensed">
- <tbody>
- <tr>
- <td class="property-key">Max -Xmx</td>
- <td class="property-value-half">{stats.maxXmx}</td>
- <td class="property-key">-Xms</td>
- <td class="property-value-half">{stats.maxXms}</td>
- </tr>
- <tr>
- <td class="property-key">Max <code>mapred.job.map.memory.mb</code></td>
- <td class="property-value-half">{stats.maxJobMapMemoryMb}</td>
- <td class="property-key">Max <code>mapred.job.reduce.memory.mb</code></td>
- <td class="property-value-half">{stats.maxJobReduceMemoryMb}</td>
- </tr>
- </tbody>
- </table>
+ <div class="row">
+ <div class="col-xs-12">
+ <h4>Parameters</h4>
+ <table class="table table-bordered table-condensed">
+ <tbody>
+ <tr>
+ <td class="property-key">Max <code>-Xmx</code></td>
+ <td class="property-value-half">{stats.xmx.str}</td>
+ <td class="property-key">-Xms</td>
+ <td class="property-value-half">
+ {?stats.xms.set}
+ {stats.xms.str}
+ {:else}
+ Not set.
+ {/stats.xms.set}
+ </td>
+ </tr>
+ <tr>
+ <td class="property-key">Max <code>mapred.job.map.memory.mb</code></td>
+ <td class="property-value-half">{stats.maxJobMapMemoryMb}</td>
+ <td class="property-key">Max <code>mapred.job.reduce.memory.mb</code></td>
+ <td class="property-value-half">{stats.maxJobReduceMemoryMb}</td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </div>
+ <div class="row row-offcanvas row-offcanvas-left">
+ <div class="col-xs-6 col-sm-3 sidebar-offcanvas">
+ <ul class="nav nav-pills nav-stacked">
+ <li class="active"><a href="#parameters-xmx" data-toggle="tab">Xmx</a></li>
+ <li><a href="#parameters-xmx" data-toggle="tab">Xms</a></li>
+ <li><a href="#parameters-map-memory" data-toggle="tab">mapred.job.map.memory.mb</a></li>
+ <li><a href="#parameters-reduce-memory" data-toggle="tab">mapred.job.reduce.memory.mb</a></li>
+ </ul>
+ </div>
+ <div class="col-xs-12 col-sm-9">
+ <div class="tab-content">
+ <div class="tab-pane active" id="parameters-xmx">
+ Xmx
+ </div>
+ <div class="tab-pane" id="parameters-xms">
+ Xms
+ </div>
+ <div class="tab-pane" id="parameters-map-memory">
+ mapred.job.map.memory.mb
+ </div>
+ <div class="tab-pane" id="parameters-reduce-memory">
+ mapred.job.reduce.memory.mb
+ </div>
</div>
+ </div>
+ </div>
+
+ <div class="row">
+ <div class="col-xs-12">
+ <h4>Counters</h4>
+ </div>
+ </div>
+
src/tl/flowstats-no-data.tl 4(+4 -0)
diff --git a/src/tl/flowstats-no-data.tl b/src/tl/flowstats-no-data.tl
index ac5e9f2..2f3605a 100644
--- a/src/tl/flowstats-no-data.tl
+++ b/src/tl/flowstats-no-data.tl
@@ -1,4 +1,8 @@
+ <div class="row">
+ <div class="col-xs-12">
<div class="alert alert-default">
<h4>No Flow Stats Available</h4>
<p>{message}</p>
</div>
+ </div>
+ </div>
src/tl/flowsummary.tl 2(+2 -0)
diff --git a/src/tl/flowsummary.tl b/src/tl/flowsummary.tl
index f9e8b84..5f9dbdd 100644
--- a/src/tl/flowsummary.tl
+++ b/src/tl/flowsummary.tl
@@ -62,4 +62,6 @@
<p>This flow has not been scheduled.</p>
</div>
{/schedule}
+
+ <h3>Last Run Stats</h3>
</div>
src/web/js/azkaban.common.utils.js 19(+19 -0)
diff --git a/src/web/js/azkaban.common.utils.js b/src/web/js/azkaban.common.utils.js
index 147bb94..974000c 100644
--- a/src/web/js/azkaban.common.utils.js
+++ b/src/web/js/azkaban.common.utils.js
@@ -36,3 +36,22 @@ function hasClass(el, name) {
}
return new RegExp('(\\s|^)'+name+'(\\s|$)').test(classes);
}
+
+function sizeStrToBytes(str) {
+ if (str.length == 0) {
+ return 0;
+ }
+ var unit = str.charAt(str.length - 1)
+ if (!isNaN(unit)) {
+ return parseInt(str);
+ }
+ var val = parseInt(str.substring(0, str.length - 1));
+ unit = unit.toUpperCase();
+ if (unit == 'M') {
+ val *= 0x100000;
+ }
+ else if (unit == 'G') {
+ val *= 0x40000000;
+ }
+ return val;
+}
src/web/js/azkaban.exflow.view.js 2(+1 -1)
diff --git a/src/web/js/azkaban.exflow.view.js b/src/web/js/azkaban.exflow.view.js
index 6c1a76a..a35ab7f 100644
--- a/src/web/js/azkaban.exflow.view.js
+++ b/src/web/js/azkaban.exflow.view.js
@@ -817,7 +817,7 @@ $(function() {
flowStatsModel = new azkaban.FlowStatsModel();
flowStatsView = new azkaban.FlowStatsView({
- el: $('#stats-view-content'),
+ el: $('#flow-stats-container'),
model: flowStatsModel
});
src/web/js/azkaban.flow.stats.view.js 71(+60 -11)
diff --git a/src/web/js/azkaban.flow.stats.view.js b/src/web/js/azkaban.flow.stats.view.js
index 64d0455..76da587 100644
--- a/src/web/js/azkaban.flow.stats.view.js
+++ b/src/web/js/azkaban.flow.stats.view.js
@@ -76,23 +76,61 @@ azkaban.FlowStatsView = Backbone.View.extend({
},
updateStats: function(jobStats, data) {
- var aggregateStats = data.stats;
+ var stats = data.stats;
var state = jobStats.state;
var conf = jobStats.conf;
var mappers = parseInt(state.totalMappers);
var reducers = parseInt(state.totalReducers);
- if (mappers > aggregateStats.maxMapSlots) {
- aggregateStats.maxMapSlots = mappers;
+ if (mappers > stats.maxMapSlots) {
+ stats.maxMapSlots = mappers;
}
- if (reducers > aggregateStats.maxReduceSlots) {
- aggregateStats.maxReduceSlots = reducers;
+ if (reducers > stats.maxReduceSlots) {
+ stats.maxReduceSlots = reducers;
+ }
+ stats.totalMapSlots += mappers;
+ stats.totalReduceSlots += reducers;
+
+ var jobMapMemoryMb = parseInt(conf['mapred.job.map.memory.mb']);
+ if (jobMapMemoryMb > stats.maxJobMapMemoryMb) {
+ stats.maxJobMapMemoryMb = jobMapMemoryMb;
+ }
+ var jobReduceMemoryMb = parseInt(conf['mapred.job.reduce.memory.mb']);
+ if (jobReduceMemoryMb > stats.maxJobReduceMemoryMb) {
+ stats.maxJobReduceMemoryMb = jobReduceMemoryMb;
}
- aggregateStats.totalMapSlots += mappers;
- aggregateStats.totalReduceSlots += reducers;
- var jobMapMemoryMb = conf['mapred.job.map.memory.mb'];
- var jobReduceMemoryMb = conf['mapred.job.reduce.memory.mb'];
var childJavaOpts = conf['mapred.child.java.opts'];
+ var parts = childJavaOpts.split(" ");
+ for (var i = 0; i < parts.length; ++i) {
+ var str = parts[i];
+ if (str.indexOf('Xmx') > -1) {
+ if (str.length <= 4) {
+ continue;
+ }
+ var size = str.substring(4, str.length);
+ var val = sizeStrToBytes(size);
+ if (val > stats.xmx.max) {
+ stats.xmx.max = val;
+ stats.xmx.str = size;
+ }
+ }
+ if (str.indexOf('Xms') > -1) {
+ if (str.length <= 4) {
+ continue;
+ }
+ var size = str.substring(4, str.length);
+ var val = sizeStrToBytes(size);
+ stats.xms.set = true;
+ if (val > stats.xms.max) {
+ stats.xms.max = val;
+ stats.xms.str = size;
+ }
+ }
+ }
+ },
+
+ finalizeStats: function(data) {
+ data.success = true;
},
analyzeExecution: function(execId) {
@@ -113,7 +151,18 @@ azkaban.FlowStatsView = Backbone.View.extend({
totalMapSlots: 0,
totalReduceSlots: 0,
numJobs: jobs.length,
- longestTaskTime: 0
+ longestTaskTime: 0,
+ maxJobMapMemoryMb: 0,
+ maxJobReduceMemoryMb: 0,
+ xmx: {
+ max: 0,
+ str: null
+ },
+ xms: {
+ set: false,
+ max: 0,
+ str: null
+ }
}
};
@@ -128,7 +177,7 @@ azkaban.FlowStatsView = Backbone.View.extend({
this.updateStats(jobStats.jobStats[j], data);
}
}
- data.success = true;
+ this.finalizeStats(data);
this.model.set({'data': data});
this.model.trigger('render');
},
src/web/js/azkaban.flow.view.js 2(+1 -1)
diff --git a/src/web/js/azkaban.flow.view.js b/src/web/js/azkaban.flow.view.js
index 1e8c0ef..75303a9 100644
--- a/src/web/js/azkaban.flow.view.js
+++ b/src/web/js/azkaban.flow.view.js
@@ -450,7 +450,7 @@ $(function() {
flowStatsModel = new azkaban.FlowStatsModel();
flowStatsView = new azkaban.FlowStatsView({
- el: $('#last-run-stats'),
+ el: $('#flow-stats-container'),
model: flowStatsModel
});