azkaban-uncached
Changes
src/java/azkaban/utils/LogSummary.java 136(+136 -0)
src/web/js/azkaban.exflow.view.js 11(+11 -0)
src/web/js/azkaban.jobsummary.view.js 153(+153 -0)
Details
src/java/azkaban/utils/LogSummary.java 136(+136 -0)
diff --git a/src/java/azkaban/utils/LogSummary.java b/src/java/azkaban/utils/LogSummary.java
new file mode 100644
index 0000000..9d67e53
--- /dev/null
+++ b/src/java/azkaban/utils/LogSummary.java
@@ -0,0 +1,136 @@
+package azkaban.utils;
+
+import azkaban.utils.FileIOUtils.LogData;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class LogSummary {
+ private String command = null;
+ private List<String> classpath = new ArrayList<String>();
+ private List<String> params = new ArrayList<String>();
+
+ private String[] statTableHeaders = null;
+ private List<String[]> statTableData = new ArrayList<String[]>();
+
+ private String[] summaryTableHeaders = null;
+ private List<String[]> summaryTableData = new ArrayList<String[]>();
+
+ public LogSummary(LogData log) {
+ if (log != null) {
+ parseLogData(log.getData());
+ }
+ }
+
+ private void parseLogData(String data) {
+ data = data.replaceAll(".*? - ", "");
+ String[] lines = data.split("\n");
+
+ parseCommand(lines);
+ parseJobSummary(lines);
+ parseJobStats(lines);
+ }
+
+ private void parseCommand(String[] lines) {
+ int commandStartIndex = -1;
+ for (int i = 0; i < lines.length; i++) {
+ if (lines[i].startsWith("Command: ")) {
+ commandStartIndex = i;
+ break;
+ }
+ }
+
+ if (commandStartIndex != -1) {
+ command = lines[commandStartIndex].substring(9);
+
+ // Parse classpath
+ Pattern p = Pattern.compile("(?:-cp|-classpath)\\s+(\\S+)");
+ Matcher m = p.matcher(command);
+ if (m.find()) {
+ classpath = Arrays.asList(m.group(1).split(":"));
+ }
+
+ // Parse Pig params
+ p = Pattern.compile("-param\\s+(\\S+)");
+ m = p.matcher(command);
+ while (m.find()) {
+ params.add(m.group(1));
+ }
+ }
+ }
+
+ private void parseJobSummary(String[] lines) {
+ int jobSummaryStartIndex = -1;
+ for (int i = 0; i < lines.length; i++) {
+ if (lines[i].startsWith("HadoopVersion")) {
+ jobSummaryStartIndex = i;
+ break;
+ }
+ }
+
+ if (jobSummaryStartIndex != -1) {
+ String headerLine = lines[jobSummaryStartIndex];
+ summaryTableHeaders = headerLine.split("\t");
+
+ int tableRowIndex = jobSummaryStartIndex + 1;
+ String line;
+ while (!(line = lines[tableRowIndex]).equals("")) {
+ summaryTableData.add(line.split("\t"));
+ tableRowIndex++;
+ }
+ }
+ }
+
+ private void parseJobStats(String[] lines) {
+ int jobStatsStartIndex = -1;
+ for (int i = 0; i < lines.length; i++) {
+ if (lines[i].startsWith("Job Stats (time in seconds):")) {
+ jobStatsStartIndex = i+1;
+ break;
+ }
+ }
+
+ if (jobStatsStartIndex != -1) {
+ String headerLine = lines[jobStatsStartIndex];
+ statTableHeaders = headerLine.split("\t");
+
+ int tableRowIndex = jobStatsStartIndex + 1;
+ String line;
+ while (!(line = lines[tableRowIndex]).equals("")) {
+ statTableData.add(line.split("\t"));
+ tableRowIndex++;
+ }
+ }
+ }
+
+ public String[] getStatTableHeaders() {
+ return statTableHeaders;
+ }
+
+ public List<String[]> getStatTableData() {
+ return statTableData;
+ }
+
+ public String[] getSummaryTableHeaders() {
+ return summaryTableHeaders;
+ }
+
+ public List<String[]> getSummaryTableData() {
+ return summaryTableData;
+ }
+
+ public String getCommand() {
+ return command;
+ }
+
+ public List<String> getClasspath() {
+ return classpath;
+ }
+
+ public List<String> getParams() {
+ return params;
+ }
+}
diff --git a/src/java/azkaban/webapp/servlet/ExecutorServlet.java b/src/java/azkaban/webapp/servlet/ExecutorServlet.java
index 95847a1..7898864 100644
--- a/src/java/azkaban/webapp/servlet/ExecutorServlet.java
+++ b/src/java/azkaban/webapp/servlet/ExecutorServlet.java
@@ -44,6 +44,7 @@ import azkaban.user.Permission;
import azkaban.user.User;
import azkaban.user.Permission.Type;
import azkaban.utils.FileIOUtils.LogData;
+import azkaban.utils.LogSummary;
import azkaban.webapp.AzkabanWebServer;
import azkaban.webapp.session.Session;
@@ -70,8 +71,11 @@ public class ExecutorServlet extends LoginAbstractAzkabanServlet {
handleAJAXAction(req, resp, session);
}
else if (hasParam(req, "execid")) {
- if (hasParam(req, "job")) {
- handleExecutionJobPage(req, resp, session);
+ if (hasParam(req, "summary")) {
+ handleExecutionJobSummaryPage(req, resp, session);
+ }
+ else if (hasParam(req, "job")) {
+ handleExecutionJobLogPage(req, resp, session);
}
else {
handleExecutionFlowPage(req, resp, session);
@@ -82,7 +86,44 @@ public class ExecutorServlet extends LoginAbstractAzkabanServlet {
}
}
- private void handleExecutionJobPage(HttpServletRequest req, HttpServletResponse resp, Session session) throws ServletException, IOException {
+ private void handleExecutionJobSummaryPage(HttpServletRequest req, HttpServletResponse resp, Session session) throws ServletException, IOException {
+ Page page = newPage(req, resp, session, "azkaban/webapp/servlet/velocity/jobsummarypage.vm");
+ User user = session.getUser();
+ int execId = getIntParam(req, "execid");
+ String jobId = getParam(req, "job");
+ int attempt = getIntParam(req, "attempt", 0);
+ page.add("execid", execId);
+ page.add("jobid", jobId);
+ page.add("attempt", attempt);
+
+ ExecutableFlow flow = null;
+ try {
+ flow = executorManager.getExecutableFlow(execId);
+ if (flow == null) {
+ page.add("errorMsg", "Error loading executing flow " + execId + ": not found.");
+ page.render();
+ return;
+ }
+ } catch (ExecutorManagerException e) {
+ page.add("errorMsg", "Error loading executing flow: " + e.getMessage());
+ page.render();
+ return;
+ }
+
+ int projectId = flow.getProjectId();
+ Project project = getProjectPageByPermission(page, projectId, user, Type.READ);
+ if (project == null) {
+ page.render();
+ return;
+ }
+
+ page.add("projectName", project.getName());
+ page.add("flowid", flow.getFlowId());
+
+ page.render();
+ }
+
+ private void handleExecutionJobLogPage(HttpServletRequest req, HttpServletResponse resp, Session session) throws ServletException, IOException {
Page page = newPage(req, resp, session, "azkaban/webapp/servlet/velocity/joblogpage.vm");
User user = session.getUser();
int execId = getIntParam(req, "execid");
@@ -96,7 +137,7 @@ public class ExecutorServlet extends LoginAbstractAzkabanServlet {
try {
flow = executorManager.getExecutableFlow(execId);
if (flow == null) {
- page.add("errorMsg", "Error loading executing flow " + execId + " not found.");
+ page.add("errorMsg", "Error loading executing flow " + execId + ": not found.");
page.render();
return;
}
@@ -262,6 +303,9 @@ public class ExecutorServlet extends LoginAbstractAzkabanServlet {
else if (ajaxName.equals("fetchExecJobLogs")) {
ajaxFetchJobLogs(req, resp, ret, session.getUser(), exFlow);
}
+ else if (ajaxName.equals("fetchExecJobSummary")) {
+ ajaxFetchJobSummary(req, resp, ret, session.getUser(), exFlow);
+ }
else if (ajaxName.equals("retryFailedJobs")) {
ajaxRestartFailed(req, resp, ret, session.getUser(), exFlow);
}
@@ -442,6 +486,47 @@ public class ExecutorServlet extends LoginAbstractAzkabanServlet {
throw new ServletException(e);
}
}
+
+ /**
+ * Gets the job summary.
+ *
+ * @param req
+ * @param resp
+ * @param user
+ * @param exFlow
+ * @throws ServletException
+ */
+ private void ajaxFetchJobSummary(HttpServletRequest req, HttpServletResponse resp, HashMap<String, Object> ret, User user, ExecutableFlow exFlow) throws ServletException {
+ Project project = getProjectAjaxByPermission(ret, exFlow.getProjectId(), user, Type.READ);
+ if (project == null) {
+ return;
+ }
+
+ String jobId = this.getParam(req, "jobId");
+ resp.setCharacterEncoding("utf-8");
+
+ try {
+ ExecutableNode node = exFlow.getExecutableNode(jobId);
+ if (node == null) {
+ ret.put("error", "Job " + jobId + " doesn't exist in " + exFlow.getExecutionId());
+ return;
+ }
+
+ int attempt = this.getIntParam(req, "attempt", node.getAttempt());
+ LogData data = executorManager.getExecutionJobLog(exFlow, jobId, 0, Integer.MAX_VALUE, attempt);
+
+ LogSummary summary = new LogSummary(data);
+ ret.put("command", summary.getCommand());
+ ret.put("classpath", summary.getClasspath());
+ ret.put("params", summary.getParams());
+ ret.put("summaryTableHeaders", summary.getSummaryTableHeaders());
+ ret.put("summaryTableData", summary.getSummaryTableData());
+ ret.put("statTableHeaders", summary.getStatTableHeaders());
+ ret.put("statTableData", summary.getStatTableData());
+ } catch (ExecutorManagerException e) {
+ throw new ServletException(e);
+ }
+ }
private void ajaxFetchFlowInfo(HttpServletRequest req, HttpServletResponse resp, HashMap<String, Object> ret, User user, String projectName, String flowId) throws ServletException {
Project project = getProjectAjaxByPermission(ret, projectName, user, Type.READ);
diff --git a/src/java/azkaban/webapp/servlet/velocity/executingflowpage.vm b/src/java/azkaban/webapp/servlet/velocity/executingflowpage.vm
index 5c4f1d4..33e4c90 100644
--- a/src/java/azkaban/webapp/servlet/velocity/executingflowpage.vm
+++ b/src/java/azkaban/webapp/servlet/velocity/executingflowpage.vm
@@ -120,6 +120,7 @@
<th class="elapse">Elapsed</th>
<th class="status">Status</th>
<th class="logs">Logs</th>
+ <th class="summary">Summary</th>
</tr>
</thead>
<tbody id="executableBody">
diff --git a/src/java/azkaban/webapp/servlet/velocity/jobsummarypage.vm b/src/java/azkaban/webapp/servlet/velocity/jobsummarypage.vm
new file mode 100644
index 0000000..833ad22
--- /dev/null
+++ b/src/java/azkaban/webapp/servlet/velocity/jobsummarypage.vm
@@ -0,0 +1,111 @@
+#*
+ * Copyright 2012 LinkedIn Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+*#
+
+<!DOCTYPE html>
+<html>
+ <head>
+#parse( "azkaban/webapp/servlet/velocity/style.vm" )
+ <script type="text/javascript" src="${context}/js/jquery/jquery-1.9.1.js"></script>
+ <script type="text/javascript" src="${context}/js/underscore-1.4.4-min.js"></script>
+ <script type="text/javascript" src="${context}/js/namespace.js"></script>
+ <script type="text/javascript" src="${context}/js/backbone-0.9.10-min.js"></script>
+ <script type="text/javascript" src="${context}/js/jquery.simplemodal-1.4.4.js"></script>
+ <script type="text/javascript" src="${context}/js/azkaban.ajax.utils.js"></script>
+ <script type="text/javascript" src="${context}/js/azkaban.nav.js"></script>
+ <script type="text/javascript" src="${context}/js/azkaban.jobsummary.view.js"></script>
+ <script type="text/javascript">
+ var contextURL = "${context}";
+ var currentTime = ${currentTime};
+ var timezone = "${timezone}";
+ var errorMessage = null;
+ var successMessage = null;
+
+ var projectName = "${projectName}";
+ var flowName = "${flowid}";
+ var execId = "${execid}";
+ var jobId = "${jobid}";
+ var attempt = ${attempt};
+ </script>
+ </head>
+ <body>
+ #set($current_page="executing")
+#parse( "azkaban/webapp/servlet/velocity/nav.vm" )
+ <div class="content">
+#if($errorMsg)
+ <div class="box-error-message">$errorMsg</div>
+#else
+#if($error_message != "null")
+ <div class="box-error-message">$error_message</div>
+#elseif($success_message != "null")
+ <div class="box-success-message">$success_message</div>
+#end
+
+ <div id="all-jobs-content">
+ <div class="section-hd flow-header">
+ <h2><a href="${context}/executor?execid=${execid}&job=${jobid}">Job Execution<span>$jobid</span></a></h2>
+ <div class="section-sub-hd">
+ <h4><a href="${context}/manager?project=${projectName}">Project <span>$projectName</span></a></h4>
+ <h4 class="separator">></h4>
+ <h4><a href="${context}/manager?project=${projectName}&flow=${flowid}">Flow <span>$flowid</span></a></h4>
+ <h4 class="separator">></h4>
+ <h4><a href="${context}/executor?execid=${execid}#jobslist">Execution <span>$execid</span></a></h4>
+ <h4 class="separator">></h4>
+ <h4><a href="${context}/manager?project=${projectName}&flow=${flowid}&job=$jobid">Job <span>$jobid</span></a></h4>
+ </div>
+ </div>
+ </div>
+
+ <div id="headertabs" class="headertabs">
+ <ul>
+ <li>Summary</li>
+ </ul>
+ </div>
+
+ <div id="jobSummaryView" class="summaryView">
+ <table id="commandTable">
+ </table>
+
+ <br/>
+ <p>Job Summary
+ <table>
+ <thead id="summaryHeader">
+ </thead>
+ <tbody id="summaryBody">
+ </tbody>
+ </table>
+ </p>
+
+ <br/>
+ <p>Job Stats
+ <table>
+ <thead id="statsHeader">
+ </thead>
+ <tbody id="statsBody">
+ </tbody>
+ </table>
+ </p>
+ </div>
+#end
+
+ <div id="messageDialog" class="modal">
+ <h3 id="messageTitle">Error</h3>
+ <div class="messageDiv">
+ <p id="messageBox"></p>
+ </div>
+ </div>
+ </div>
+ </body>
+</html>
src/web/js/azkaban.exflow.view.js 11(+11 -0)
diff --git a/src/web/js/azkaban.exflow.view.js b/src/web/js/azkaban.exflow.view.js
index 377703b..955b648 100644
--- a/src/web/js/azkaban.exflow.view.js
+++ b/src/web/js/azkaban.exflow.view.js
@@ -467,6 +467,7 @@ azkaban.ExecutionListView = Backbone.View.extend({
var tdElapse = document.createElement("td");
var tdStatus = document.createElement("td");
var tdLog = document.createElement("td");
+ var tdSummary = document.createElement("td");
$(tr).append(tdName);
$(tr).append(tdTimeline);
@@ -475,6 +476,7 @@ azkaban.ExecutionListView = Backbone.View.extend({
$(tr).append(tdElapse);
$(tr).append(tdStatus);
$(tr).append(tdLog);
+ $(tr).append(tdSummary);
$(tr).attr("id", node.id + "-row");
$(tdTimeline).attr("id", node.id + "-timeline");
$(tdStart).attr("id", node.id + "-start");
@@ -516,6 +518,15 @@ azkaban.ExecutionListView = Backbone.View.extend({
$(a).text("Log");
$(tdLog).addClass("logLink");
$(tdLog).append(a);
+
+ var summaryURL = contextURL + "/executor?execid=" + execId +
+ "&job=" + node.id + "&summary";
+ a = document.createElement("a");
+ $(a).attr("href", summaryURL);
+ $(a).attr("id", node.id + "-summary-link");
+ $(a).text("Summary");
+ $(tdSummary).addClass("logSummary");
+ $(tdSummary).append(a);
executingBody.append(tr);
}
src/web/js/azkaban.jobsummary.view.js 153(+153 -0)
diff --git a/src/web/js/azkaban.jobsummary.view.js b/src/web/js/azkaban.jobsummary.view.js
new file mode 100644
index 0000000..9959502
--- /dev/null
+++ b/src/web/js/azkaban.jobsummary.view.js
@@ -0,0 +1,153 @@
+/*
+ * Copyright 2012 LinkedIn Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+$.namespace('azkaban');
+
+var summaryModel;
+azkaban.SummaryModel = Backbone.Model.extend({});
+
+var jobSummaryView;
+azkaban.JobSummaryView = Backbone.View.extend({
+ events: {
+ "click #updateSummaryBtn" : "handleUpdate"
+ },
+ initialize: function(settings) {
+ this.handleUpdate();
+ },
+ handleUpdate: function(evt) {
+ var requestURL = contextURL + "/executor";
+ var model = this.model;
+ var self = this;
+
+ $.ajax({
+ url: requestURL,
+ dataType: "json",
+ data: {"execid": execId, "jobId": jobId, "ajax":"fetchExecJobSummary", "attempt": attempt},
+ error: function(data) {
+ console.log(data);
+ },
+ success: function(data) {
+ console.log("fetchSummary");
+ if (data.error) {
+ console.log(data.error);
+ }
+ else {
+ self.renderCommandTable(data.command, data.classpath, data.params);
+ self.renderJobTable(data.summaryTableHeaders, data.summaryTableData, "summary");
+ self.renderJobTable(data.statTableHeaders, data.statTableData, "stats");
+ }
+ }
+ });
+ },
+ renderCommandTable: function(command, classpath, params) {
+ if (command) {
+ var commandTable = $("#commandTable");
+ var i;
+
+ // Add row for command
+ var tr = document.createElement("tr");
+ var td = document.createElement("td");
+ $(td).append("<b>Command</b>");
+ $(tr).append(td);
+ td = document.createElement("td");
+ $(td).text(command);
+ $(tr).append(td);
+ commandTable.append(tr);
+
+ // Add row for classpath
+ if (classpath && classpath.length > 0) {
+ tr = document.createElement("tr");
+ td = document.createElement("td");
+ $(td).append("<b>Classpath</b>");
+ $(tr).append(td);
+ td = document.createElement("td");
+ $(td).append(classpath[0]);
+ for (i = 1; i < classpath.length; i++) {
+ $(td).append("<br/>" + classpath[i]);
+ }
+ $(tr).append(td);
+ commandTable.append(tr);
+ }
+
+ // Add row for params
+ if (params && params.length > 0) {
+ tr = document.createElement("tr");
+ td = document.createElement("td");
+ $(td).append("<b>Params</b>");
+ $(tr).append(td);
+ td = document.createElement("td");
+ $(td).append(params[0]);
+ for (i = 1; i < params.length; i++) {
+ $(td).append("<br/>" + params[i]);
+ }
+ $(tr).append(td);
+ commandTable.append(tr);
+ }
+ }
+ },
+ renderJobTable: function(headers, data, prefix) {
+ if (headers) {
+ // Add table headers
+ var header = $("#" + prefix + "Header");
+ var tr = document.createElement("tr");
+ var i;
+ for (i = 0; i < headers.length; i++) {
+ var th = document.createElement("th");
+ $(th).text(headers[i]);
+ $(tr).append(th);
+ }
+ header.append(tr);
+
+ // Add table body
+ var body = $("#" + prefix + "Body");
+ for (i = 0; i < data.length; i++) {
+ tr = document.createElement("tr");
+ var row = data[i];
+ for (var j = 0; j < headers.length; j++) {
+ var td = document.createElement("td");
+ $(td).text(row[j]);
+ $(tr).append(td);
+ }
+ body.append(tr);
+ }
+ }
+ }
+});
+
+var showDialog = function(title, message) {
+ $('#messageTitle').text(title);
+
+ $('#messageBox').text(message);
+
+ $('#messageDialog').modal({
+ closeHTML: "<a href='#' title='Close' class='modal-close'>x</a>",
+ position: ["20%",],
+ containerId: 'confirm-container',
+ containerCss: {
+ 'height': '220px',
+ 'width': '565px'
+ },
+ onShow: function (dialog) {
+ }
+ });
+}
+
+$(function() {
+ var selected;
+
+ summaryModel = new azkaban.SummaryModel();
+ jobSummaryView = new azkaban.JobSummaryView({el:$('#jobSummaryView'), model: summaryModel});
+});