azkaban-memoizeit
Changes
src/web/js/azkaban.jobdetails.view.js 359(+147 -212)
src/web/js/azkaban.logdata.model.js 302(+302 -0)
Details
diff --git a/src/java/azkaban/webapp/servlet/ExecutorServlet.java b/src/java/azkaban/webapp/servlet/ExecutorServlet.java
index 6cc091b..3b3b8f8 100644
--- a/src/java/azkaban/webapp/servlet/ExecutorServlet.java
+++ b/src/java/azkaban/webapp/servlet/ExecutorServlet.java
@@ -44,7 +44,6 @@ import azkaban.user.Permission;
import azkaban.user.User;
import azkaban.user.Permission.Type;
import azkaban.utils.FileIOUtils.LogData;
-import azkaban.utils.LogSummary;
import azkaban.webapp.AzkabanWebServer;
import azkaban.webapp.session.Session;
@@ -128,9 +127,6 @@ public class ExecutorServlet extends LoginAbstractAzkabanServlet {
else if (ajaxName.equals("fetchExecJobLogs")) {
ajaxFetchJobLogs(req, resp, ret, session.getUser(), exFlow);
}
- else if (ajaxName.equals("fetchExecJobSummary")) {
- ajaxFetchJobSummary(req, resp, ret, session.getUser(), exFlow);
- }
else if (ajaxName.equals("retryFailedJobs")) {
ajaxRestartFailed(req, resp, ret, session.getUser(), exFlow);
}
@@ -451,53 +447,6 @@ public class ExecutorServlet extends LoginAbstractAzkabanServlet {
}
}
- /**
- * Gets the job summary.
- *
- * @param req
- * @param resp
- * @param user
- * @param exFlow
- * @throws ServletException
- */
- private void ajaxFetchJobSummary(HttpServletRequest req, HttpServletResponse resp, HashMap<String, Object> ret, User user, ExecutableFlow exFlow) throws ServletException {
- Project project = getProjectAjaxByPermission(ret, exFlow.getProjectId(), user, Type.READ);
- if (project == null) {
- return;
- }
-
- String jobId = this.getParam(req, "jobId");
- resp.setCharacterEncoding("utf-8");
-
- try {
- ExecutableNode node = exFlow.getExecutableNode(jobId);
- if (node == null) {
- ret.put("error", "Job " + jobId + " doesn't exist in " + exFlow.getExecutionId());
- return;
- }
-
- int attempt = this.getIntParam(req, "attempt", node.getAttempt());
- LogData data = executorManager.getExecutionJobLog(exFlow, jobId, 0, Integer.MAX_VALUE, attempt);
-
- LogSummary summary = new LogSummary(data);
- ret.put("commandProperties", summary.getCommandProperties());
-
- String jobType = summary.getJobType();
-
- if (jobType.contains("pig")) {
- ret.put("summaryTableHeaders", summary.getPigSummaryTableHeaders());
- ret.put("summaryTableData", summary.getPigSummaryTableData());
- ret.put("statTableHeaders", summary.getPigStatTableHeaders());
- ret.put("statTableData", summary.getPigStatTableData());
- } else if (jobType.contains("hive")) {
- ret.put("hiveQueries", summary.getHiveQueries());
- ret.put("hiveQueryJobs", summary.getHiveQueryJobs());
- }
- } catch (ExecutorManagerException e) {
- throw new ServletException(e);
- }
- }
-
private void ajaxFetchFlowInfo(HttpServletRequest req, HttpServletResponse resp, HashMap<String, Object> ret, User user, String projectName, String flowId) throws ServletException {
Project project = getProjectAjaxByPermission(ret, projectName, user, Type.READ);
if (project == null) {
diff --git a/src/java/azkaban/webapp/servlet/velocity/jobdetailspage.vm b/src/java/azkaban/webapp/servlet/velocity/jobdetailspage.vm
index a8df9c2..ed54afa 100644
--- a/src/java/azkaban/webapp/servlet/velocity/jobdetailspage.vm
+++ b/src/java/azkaban/webapp/servlet/velocity/jobdetailspage.vm
@@ -22,6 +22,7 @@
#parse("azkaban/webapp/servlet/velocity/javascript.vm")
<script type="text/javascript" src="${context}/js/azkaban.ajax.utils.js"></script>
+ <script type="text/javascript" src="${context}/js/azkaban.logdata.model.js"></script>
<script type="text/javascript" src="${context}/js/azkaban.jobdetails.view.js"></script>
<script type="text/javascript">
var contextURL = "${context}";
@@ -88,35 +89,46 @@
## Log content.
<div class="container-full container-fill" id="jobLogView">
- <div class="row">
- <div class="col-lg-12 col-content">
- <div class="log-viewer">
- <div class="panel panel-default">
- <div class="panel-heading">
- <div class="pull-right">
- <button type="button" id="updateLogBtn" class="btn btn-xs btn-default">Refresh</button>
- </div>
- Job Logs
- </div>
- <div class="panel-body">
- <pre id="logSection"></pre>
- </div>
- </div>
- </div>
- </div>
- </div>
+ <div class="row">
+ <div class="col-lg-12 col-content">
+ <div class="log-viewer">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ <div class="pull-right">
+ <button type="button" id="updateLogBtn" class="btn btn-xs btn-default">Refresh</button>
+ </div>
+ Job Logs
+ </div>
+ <div class="panel-body">
+ <pre id="logSection"></pre>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
</div>
## Job Summary
<div class="container-full" id="jobSummaryView">
- <div class="row">
- <div class="col-lg-12">
- <table id="commandTable" class="table table-striped table-bordered table-hover">
- </table>
+ <div class="row">
+ <div class="col-lg-12">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ <div class="pull-left">
+ <button type="button" id="updateSummaryBtn" class="btn btn-xs btn-default">Refresh</button>
+ </div>
+
+ </div>
+
+ <div class="panel panel-default" id="commandSummary">
+ <div class="panel-heading">Command Summary</div>
+ <table id="commandTable" class="table table-striped table-bordered table-hover">
+ </table>
+ </div>
- <div class="panel panel-default" id="jobsummary">
- <div class="panel-heading">Job Summary</div>
+ <div class="panel panel-default" id="pigJobSummary">
+ <div class="panel-heading">Pig Job Summary</div>
<table class="table table-striped table-bordered table-hover">
<thead id="summaryHeader">
</thead>
@@ -125,21 +137,21 @@
</table>
</div>
- <div class="panel panel-default" id="jobstats">
- <div class="panel-heading">Job Stats</div>
- <div class="panel-body panel-body-stats">
- <table class="table table-striped table-bordered table-hover table-condensed">
- <thead id="statsHeader">
- </thead>
- <tbody id="statsBody">
- </tbody>
- </table>
- </div>
+ <div class="panel panel-default" id="pigJobStats">
+ <div class="panel-heading">Pig Job Stats</div>
+ <div class="panel-body panel-body-stats">
+ <table class="table table-striped table-bordered table-hover table-condensed">
+ <thead id="statsHeader">
+ </thead>
+ <tbody id="statsBody">
+ </tbody>
+ </table>
+ </div>
</div>
-
- <div class="panel panel-default" id="hiveTable">
- <div class="panel-heading">Job Summary</div>
- <table class="table table-striped table-bordered table-hover">
+
+ <div class="panel panel-default" id="hiveJobSummary">
+ <div class="panel-heading">Hive Job Summary</div>
+ <table class="table table-striped table-bordered table-hover" id="hiveTable">
<thead id="hiveTableHeader">
</thead>
<tbody id="hiveTableBody">
@@ -148,6 +160,7 @@
</div>
</div>
</div>
+ </div>
</div>
## Error message message dialog.
src/web/js/azkaban.jobdetails.view.js 359(+147 -212)
diff --git a/src/web/js/azkaban.jobdetails.view.js b/src/web/js/azkaban.jobdetails.view.js
index 3f2b7a2..f191f37 100644
--- a/src/web/js/azkaban.jobdetails.view.js
+++ b/src/web/js/azkaban.jobdetails.view.js
@@ -16,247 +16,184 @@
$.namespace('azkaban');
-var logModel;
-azkaban.LogModel = Backbone.Model.extend({});
-
var jobLogView;
azkaban.JobLogView = Backbone.View.extend({
events: {
- "click #updateLogBtn" : "handleUpdate"
- },
- initialize: function(settings) {
- this.model.set({"offset": 0});
- this.handleUpdate();
+ "click #updateLogBtn" : "refresh"
},
- handleUpdate: function(evt) {
- var requestURL = contextURL + "/executor";
- var model = this.model;
- var finished = false;
-
- var date = new Date();
- var startTime = date.getTime();
-
- while (!finished) {
- var offset = this.model.get("offset");
- var requestData = {
- "execid": execId,
- "jobId": jobId,
- "ajax":"fetchExecJobLogs",
- "offset": offset,
- "length": 50000,
- "attempt": attempt
- };
-
- var successHandler = function(data) {
- console.log("fetchLogs");
- if (data.error) {
- console.log(data.error);
- finished = true;
- }
- else if (data.length == 0) {
- finished = true;
- }
- else {
- var date = new Date();
- var endTime = date.getTime();
- if ((endTime - startTime) > 10000) {
- finished = true;
- showDialog("Alert","The log is taking a long time to finish loading. Azkaban has stopped loading them. Please click Refresh to restart the load.");
- }
- var re = /(https?:\/\/(([-\w\.]+)+(:\d+)?(\/([\w/_\.]*(\?\S+)?)?)?))/g;
- var log = $("#logSection").text();
- if (!log) {
- log = data.data;
- }
- else {
- log += data.data;
- }
-
- var newOffset = data.offset + data.length;
- $("#logSection").text(log);
- log = $("#logSection").html();
- log = log.replace(re, "<a href=\"$1\" title=\"\">$1</a>");
- $("#logSection").html(log);
+ initialize: function() {
+ this.listenTo(this.model, "change:logData", this.render);
+ },
- model.set({"offset": newOffset, "log": log});
- $(".logViewer").scrollTop(9999);
- }
- }
+ refresh: function() {
+ this.model.refresh();
+ },
- $.ajax({
- url: requestURL,
- type: "get",
- async: false,
- data: requestData,
- dataType: "json",
- error: function(data) {
- console.log(data);
- finished = true;
- },
- success: successHandler
- });
- }
+ render: function() {
+ var re = /(https?:\/\/(([-\w\.]+)+(:\d+)?(\/([\w/_\.]*(\?\S+)?)?)?))/g;
+ var log = this.model.get("logData");
+ log = log.replace(re, "<a href=\"$1\" title=\"\">$1</a>");
+ $("#logSection").html(log);
}
});
-var summaryModel;
-azkaban.SummaryModel = Backbone.Model.extend({});
-
var jobSummaryView;
azkaban.JobSummaryView = Backbone.View.extend({
events: {
- "click #updateSummaryBtn" : "handleUpdate"
+ "click #updateSummaryBtn" : "refresh"
},
+
initialize: function(settings) {
- this.handleUpdate();
+ $("#commandSummary").hide();
+ $("#pigJobSummary").hide();
+ $("#pigJobStats").hide();
+ $("#hiveJobSummary").hide();
+
+ this.listenTo(this.model, "change:commandProperties", this.renderCommandTable);
+ this.listenTo(this.model, "change:pigSummary", this.renderPigSummaryTable);
+ this.listenTo(this.model, "change:pigStats", this.renderPigStatsTable);
+ this.listenTo(this.model, "change:hiveSummary", this.renderHiveTable);
},
- handleUpdate: function(evt) {
- var requestURL = contextURL + "/executor";
- var model = this.model;
- var self = this;
- var requestData = {
- "execid": execId,
- "jobId": jobId,
- "ajax":"fetchExecJobSummary",
- "attempt": attempt
- };
+ refresh: function() {
+ this.model.refresh();
+ },
- $.ajax({
- url: requestURL,
- dataType: "json",
- data: requestData,
- error: function(data) {
- console.log(data);
- },
- success: function(data) {
- console.log("fetchSummary");
- if (data.error) {
- console.log(data.error);
- }
- else {
- self.renderCommandTable(data.commandProperties);
- self.renderJobTable(data.summaryTableHeaders, data.summaryTableData, "summary");
- self.renderJobTable(data.statTableHeaders, data.statTableData, "stats");
- self.renderHiveTable(data.hiveQueries, data.hiveQueryJobs);
- }
- }
- });
+ handleUpdate: function(evt) {
+ renderJobTable(jobSummary.summaryTableHeaders, jobSummary.summaryTableData, "summary");
+ renderJobTable(jobSummary.statTableHeaders, jobSummary.statTableData, "stats");
+ renderHiveTable(jobSummary.hiveQueries, jobSummary.hiveQueryJobs);
},
- renderCommandTable: function(commandProperties) {
- if (commandProperties) {
- var commandTable = $("#commandTable");
-
- for (var i = 0; i < commandProperties.length; i++) {
- var prop = commandProperties[i];
+ renderCommandTable: function() {
+ var commandTable = $("#commandTable");
+ var commandProperties = this.model.get("commandProperties");
+
+ for (var key in commandProperties) {
+ if (commandProperties.hasOwnProperty(key)) {
+ var value = commandProperties[key];
+ if (Array.isArray(value)) {
+ value = value.join("<br/>");
+ }
var tr = document.createElement("tr");
- var name = document.createElement("td");
- var value = document.createElement("td");
- $(name).html("<b>" + prop.first + "</b>");
- $(value).html(prop.second);
- $(tr).append(name);
- $(tr).append(value);
+ var keyTd = document.createElement("td");
+ var valueTd = document.createElement("td");
+ $(keyTd).html("<b>" + key + "</b>");
+ $(valueTd).html(value);
+ $(tr).append(keyTd);
+ $(tr).append(valueTd);
commandTable.append(tr);
}
}
+
+ $("#commandSummary").show();
},
- renderJobTable: function(headers, data, prefix) {
- if (headers) {
- // Add table headers
- var header = $("#" + prefix + "Header");
- var tr = document.createElement("tr");
- var i;
- for (i = 0; i < headers.length; i++) {
- var th = document.createElement("th");
- $(th).text(headers[i]);
- $(tr).append(th);
- }
- header.append(tr);
-
- // Add table body
- var body = $("#" + prefix + "Body");
- for (i = 0; i < data.length; i++) {
- tr = document.createElement("tr");
- var row = data[i];
- for (var j = 0; j < row.length; j++) {
- var td = document.createElement("td");
- if (j == 0) {
- // first column is a link to job details page
- $(td).html(row[j]);
- } else {
- $(td).text(row[j]);
- }
- $(tr).append(td);
+ renderPigTable: function(tableName, data) {
+ // Add table headers
+ var header = $("#" + tableName + "Header");
+ var tr = document.createElement("tr");
+ var i;
+ var headers = data[0];
+ var numColumns = headers.length;
+ for (i = 0; i < numColumns; i++) {
+ var th = document.createElement("th");
+ $(th).text(headers[i]);
+ $(tr).append(th);
+ }
+ header.append(tr);
+
+ // Add table body
+ var body = $("#" + tableName + "Body");
+ for (i = 1; i < data.length; i++) {
+ tr = document.createElement("tr");
+ var row = data[i];
+ for (var j = 0; j < numColumns; j++) {
+ var td = document.createElement("td");
+ if (j == 0) {
+ // first column is a link to job details page
+ $(td).html(row[j]);
+ } else {
+ $(td).text(row[j]);
}
- body.append(tr);
+ $(tr).append(td);
}
- } else {
- $("#job" + prefix).hide();
+ body.append(tr);
}
+
+ $("#pigJob" + tableName.charAt(0).toUpperCase() + tableName.substring(1)).show();
},
- renderHiveTable: function(queries, queryJobs) {
- if (queries) {
- // Set up table column headers
- var header = $("#hiveTableHeader");
- var tr = document.createElement("tr");
- var headers = ["Query","Job","Map","Reduce","HDFS Read","HDFS Write"];
- var i;
-
- for (i = 0; i < headers.length; i++) {
- var th = document.createElement("th");
- $(th).text(headers[i]);
- $(tr).append(th);
- }
- header.append(tr);
+ renderPigSummaryTable: function() {
+ this.renderPigTable("summary", this.model.get("pigSummary"));
+ },
+ renderPigStatsTable: function() {
+ this.renderPigTable("stats", this.model.get("pigStats"));
+ },
+ renderHiveTable: function() {
+ var hiveSummary = this.model.get("hiveSummary");
+ var queries = hiveSummary.hiveQueries;
+ var queryJobs = hiveSummary.hiveQueryJobs;
+
+ // Set up table column headers
+ var header = $("#hiveTableHeader");
+ var tr = document.createElement("tr");
+ var headers = ["Query","Job","Map","Reduce","HDFS Read","HDFS Write"];
+ var i;
+
+ for (i = 0; i < headers.length; i++) {
+ var th = document.createElement("th");
+ $(th).text(headers[i]);
+ $(tr).append(th);
+ }
+ header.html(tr);
+
+ // Construct table body
+ var oldBody = $("#hiveTableBody");
+ var newBody = $(document.createElement("tbody")).attr("id", "hiveTableBody");
+ for (i = 0; i < queries.length; i++) {
+ // new query
+ tr = document.createElement("tr");
+ var td = document.createElement("td");
+ $(td).html("<b>" + queries[i] + "</b>");
+ $(tr).append(td);
- // Construct table body
- var body = $("#hiveTableBody");
- for (i = 0; i < queries.length; i++) {
- // new query
- tr = document.createElement("tr");
- var td = document.createElement("td");
- $(td).html("<b>" + queries[i] + "</b>");
- $(tr).append(td);
+ var jobs = queryJobs[i];
+ if (jobs != null) {
+ // add first job for this query
+ var jobValues = jobs[0];
+ var j;
+ for (j = 0; j < jobValues.length; j++) {
+ td = document.createElement("td");
+ $(td).html(jobValues[j]);
+ $(tr).append(td);
+ }
+ newBody.append(tr);
- var jobs = queryJobs[i];
- if (jobs != null) {
- // add first job for this query
- var jobValues = jobs[0];
- var j;
- for (j = 0; j < jobValues.length; j++) {
- td = document.createElement("td");
- $(td).html(jobValues[j]);
- $(tr).append(td);
- }
- body.append(tr);
+ // add remaining jobs for this query
+ for (j = 1; j < jobs.length; j++) {
+ jobValues = jobs[j];
+ tr = document.createElement("tr");
+
+ // add empty cell for query column
+ td = document.createElement("td");
+ $(td).html(" ");
+ $(tr).append(td);
- // add remaining jobs for this query
- for (j = 1; j < jobs.length; j++) {
- jobValues = jobs[j];
- tr = document.createElement("tr");
-
- // add empty cell for query column
+ // add job values
+ for (var k = 0; k < jobValues.length; k++) {
td = document.createElement("td");
- $(td).html(" ");
+ $(td).html(jobValues[k]);
$(tr).append(td);
-
- // add job values
- for (var k = 0; k < jobValues.length; k++) {
- td = document.createElement("td");
- $(td).html(jobValues[k]);
- $(tr).append(td);
- }
- body.append(tr);
}
-
- } else {
- body.append(tr);
+ newBody.append(tr);
}
+
+ } else {
+ newBody.append(tr);
}
- } else {
- $("#hiveTable").hide();
}
+ oldBody.replaceWith(newBody);
+
+ $("#hiveJobSummary").show();
}
});
@@ -277,9 +214,6 @@ azkaban.JobTabView = Backbone.View.extend({
}
},
- render: function() {
- },
-
handleJobLogViewLinkClick: function() {
$('#jobSummaryViewLink').removeClass('active');
$('#jobSummaryView').hide();
@@ -312,23 +246,24 @@ var showDialog = function(title, message) {
}
$(function() {
- var selected;
- logModel = new azkaban.LogModel();
+ var logDataModel = new azkaban.LogDataModel();
+
jobLogView = new azkaban.JobLogView({
el: $('#jobLogView'),
- model: logModel
+ model: logDataModel
});
- summaryModel = new azkaban.SummaryModel();
jobSummaryView = new azkaban.JobSummaryView({
el: $('#jobSummaryView'),
- model: summaryModel
+ model: logDataModel
});
jobTabView = new azkaban.JobTabView({
el: $('#headertabs')
});
+ logDataModel.refresh();
+
if (window.location.hash) {
var hash = window.location.hash;
if (hash == '#joblog') {
src/web/js/azkaban.logdata.model.js 302(+302 -0)
diff --git a/src/web/js/azkaban.logdata.model.js b/src/web/js/azkaban.logdata.model.js
new file mode 100644
index 0000000..2e21386
--- /dev/null
+++ b/src/web/js/azkaban.logdata.model.js
@@ -0,0 +1,302 @@
+$.namespace('azkaban');
+
+azkaban.LogDataModel = Backbone.Model.extend({
+ TIMESTAMP_REGEX: /^.*? - /gm,
+
+ JOB_TRACKER_URL_REGEX: /https?:\/\/[-\w\.]+(?::\d+)?\/[\w\/\.]*\?\S+(job_\d{12}_\d{4,})\S*/,
+
+ // Command properties
+ COMMAND_START: "Command: ",
+ CLASSPATH_REGEX: /(?:-cp|-classpath)\s+(\S+)/g,
+ ENVIRONMENT_VARIABLES_REGEX: /-D(\S+)/g,
+ JVM_MEMORY_REGEX: /(-Xm\S+)/g,
+ PIG_PARAMS_REGEX: /-param\s+(\S+)/g,
+
+ JOB_TYPE_REGEX: /Building (\S+) job executor/,
+
+ PIG_JOB_SUMMARY_START: "HadoopVersion",
+ PIG_JOB_STATS_START: "Job Stats (time in seconds):",
+
+ HIVE_PARSING_START: "Parsing command: ",
+ HIVE_PARSING_END: "Parse Completed",
+ HIVE_NUM_MAP_REDUCE_JOBS_STRING: "Total MapReduce jobs = ",
+ HIVE_MAP_REDUCE_JOB_START: "Starting Job",
+ HIVE_MAP_REDUCE_JOBS_SUMMARY: "MapReduce Jobs Launched:",
+ HIVE_MAP_REDUCE_SUMMARY_REGEX: /Job (\d+): Map: (\d+) Reduce: (\d+) HDFS Read: (\d+) HDFS Write: (\d+)/,
+
+ initialize: function() {
+ this.set("offset", 0 );
+ this.set("logData", "");
+ this.on("change:logData", this.parseLogData);
+ },
+
+ refresh: function() {
+ var requestURL = contextURL + "/executor";
+ var finished = false;
+
+ var date = new Date();
+ var startTime = date.getTime();
+
+ while (!finished) {
+ var requestData = {
+ "execid": execId,
+ "jobId": jobId,
+ "ajax":"fetchExecJobLogs",
+ "offset": this.get("offset"),
+ "length": 50000,
+ "attempt": attempt
+ };
+
+ var self = this;
+
+ var successHandler = function(data) {
+ console.log("fetchLogs");
+ if (data.error) {
+ console.log(data.error);
+ finished = true;
+ }
+ else if (data.length == 0) {
+ finished = true;
+ }
+ else {
+ var date = new Date();
+ var endTime = date.getTime();
+ if ((endTime - startTime) > 10000) {
+ finished = true;
+ showDialog("Alert","The log is taking a long time to finish loading. Azkaban has stopped loading them. Please click Refresh to restart the load.");
+ }
+
+ self.set("offset", data.offset + data.length);
+ self.set("logData", self.get("logData") + data.data);
+ }
+ }
+
+ $.ajax({
+ url: requestURL,
+ type: "get",
+ async: false,
+ data: requestData,
+ dataType: "json",
+ error: function(data) {
+ console.log(data);
+ finished = true;
+ },
+ success: successHandler
+ });
+ }
+ },
+
+ parseLogData: function() {
+ var data = this.get("logData").replace(this.TIMESTAMP_REGEX, "");
+ var lines = data.split("\n");
+
+ if (this.parseCommand(lines)) {
+ this.parseJobTrackerUrls(lines);
+
+ var jobType = this.parseJobType(lines);
+ if (jobType.indexOf("pig") !== -1) {
+ this.parsePigTable(lines, "pigSummary", this.PIG_JOB_SUMMARY_START, "", 0);
+ this.parsePigTable(lines, "pigStats", this.PIG_JOB_STATS_START, "", 1);
+ } else if (jobType.indexOf("hive") !== -1) {
+ this.parseHiveQueries(lines);
+ }
+ }
+ },
+
+ parseCommand: function(lines) {
+ var commandStartIndex = -1;
+ var numLines = lines.length;
+ for (var i = 0; i < numLines; i++) {
+ if (lines[i].indexOf(this.COMMAND_START) === 0) {
+ commandStartIndex = i;
+ break;
+ }
+ }
+
+ if (commandStartIndex != -1) {
+ var commandProperties = {};
+
+ var command = lines[commandStartIndex].substring(this.COMMAND_START.length);
+ commandProperties.Command = command;
+
+ this.parseCommandProperty(command, commandProperties, "Classpath", this.CLASSPATH_REGEX, ':');
+ this.parseCommandProperty(command, commandProperties, "-D", this.ENVIRONMENT_VARIABLES_REGEX);
+ this.parseCommandProperty(command, commandProperties, "Memory Settings", this.JVM_MEMORY_REGEX);
+ this.parseCommandProperty(command, commandProperties, "Params", this.PIG_PARAMS_REGEX);
+
+ this.set("commandProperties", commandProperties);
+
+ return true;
+ }
+
+ return false;
+ },
+
+ parseCommandProperty: function(command, commandProperties, propertyName, regex, split) {
+ var results = [];
+ var match;
+ while (match = regex.exec(command)) {
+ if (split) {
+ results = results.concat(match[1].split(split));
+ } else {
+ results.push(match[1]);
+ }
+ }
+
+ if (results.length > 0) {
+ commandProperties[propertyName] = results;
+ }
+ },
+
+ parseJobTrackerUrls: function(lines) {
+ var jobTrackerUrls = {};
+ var jobTrackerUrlsOrdered = [];
+ var numLines = lines.length;
+ var match;
+ for (var i = 0; i < numLines; i++) {
+ if ((match = this.JOB_TRACKER_URL_REGEX.exec(lines[i])) && !jobTrackerUrls[match[1]]) {
+ jobTrackerUrls[match[1]] = match[0];
+ jobTrackerUrlsOrdered.push(match[0]);
+ }
+ }
+ this.set("jobTrackerUrls", jobTrackerUrls);
+ this.set("jobTrackerUrlsOrdered", jobTrackerUrlsOrdered);
+ },
+
+ parseJobType: function(lines) {
+ var numLines = lines.length;
+ var match;
+ for (var i = 0; numLines; i++) {
+ if (match = this.JOB_TYPE_REGEX.exec(lines[i])) {
+ return match[1];
+ }
+ }
+
+ return null;
+ },
+
+ parsePigTable: function(lines, tableName, startPattern, endPattern, linesToSkipAfterStart) {
+ var index = -1;
+ var numLines = lines.length;
+ for (var i = 0; i < numLines; i++) {
+ if (lines[i].indexOf(startPattern) === 0) {
+ index = i + linesToSkipAfterStart;
+ break;
+ }
+ }
+
+ if (index != -1) {
+ var table = [];
+ var line;
+ while ((line = lines[index]) !== endPattern) {
+ var columns = line.split("\t");
+ // If first column is a job id, make it a link to the job tracker.
+ if (this.get("jobTrackerUrls")[columns[0]]) {
+ columns[0] = "<a href='" + this.get("jobTrackerUrls")[columns[0]] + "'>" + columns[0] + "</a>";
+ }
+ table.push(columns);
+ index++;
+ }
+
+ this.set(tableName, table);
+ }
+ },
+
+ parseHiveQueries: function(lines) {
+ var hiveQueries = [];
+ var hiveQueryJobs = [];
+
+ var currMapReduceJob = 0;
+ var numLines = lines.length;
+ for (var i = 0; i < numLines;) {
+ var line = lines[i];
+ var parsingCommandIndex = line.indexOf(this.HIVE_PARSING_START);
+ if (parsingCommandIndex === -1) {
+ i++;
+ continue;
+ }
+
+ // parse query text, which could span multiple lines
+ var queryStartIndex = parsingCommandIndex + this.HIVE_PARSING_START.length;
+ var query = line.substring(queryStartIndex) + "<br/>";
+
+ i++;
+ while (i < numLines && (line = lines[i]).indexOf(this.HIVE_PARSING_END) === -1) {
+ query += line + "<br/>";
+ i++;
+ }
+ hiveQueries.push(query);
+ i++;
+
+ // parse the query's Map-Reduce jobs, if any.
+ var numMRJobs = 0;
+ while (i < numLines) {
+ line = lines[i];
+ if (line.contains(this.HIVE_NUM_MAP_REDUCE_JOBS_STRING)) {
+ // query involves map reduce jobs
+ var numMRJobs = parseInt(line.substring(this.HIVE_NUM_MAP_REDUCE_JOBS_STRING.length),10);
+ i++;
+
+ // get the map reduce jobs summary
+ while (i < numLines) {
+ line = lines[i];
+ if (line.contains(this.HIVE_MAP_REDUCE_JOBS_SUMMARY)) {
+ // job summary table found
+ i++;
+
+ var queryJobs = [];
+
+ var previousJob = -1;
+ var numJobsSeen = 0;
+ while (numJobsSeen < numMRJobs && i < numLines) {
+ line = lines[i];
+ var match;
+ if (match = this.HIVE_MAP_REDUCE_SUMMARY_REGEX.exec(line)) {
+ var currJob = parseInt(match[1], 10);
+ if (currJob === previousJob) {
+ i++;
+ continue;
+ }
+
+ var job = [];
+ job.push("<a href='" + this.get("jobTrackerUrlsOrdered")[currMapReduceJob++] + "'>" + currJob + "</a>");
+ job.push(match[2]);
+ job.push(match[3]);
+ job.push(match[4]);
+ job.push(match[5]);
+ queryJobs.push(job);
+ previousJob = currJob;
+ numJobsSeen++;
+ }
+ i++;
+ }
+
+ if (numJobsSeen === numMRJobs) {
+ hiveQueryJobs.push(queryJobs);
+ }
+
+ break;
+ }
+ i++;
+ }
+ break;
+ }
+ else if (line.contains(this.HIVE_PARSING_START)) {
+ if (numMRJobs === 0) {
+ hiveQueryJobs.push(null);
+ }
+ break;
+ }
+ i++;
+ }
+ continue;
+ }
+
+ if (hiveQueries.length > 0) {
+ this.set("hiveSummary", {
+ hiveQueries: hiveQueries,
+ hiveQueryJobs: hiveQueryJobs
+ });
+ }
+ }
+});