azkaban-developers
Changes
src/java/azkaban/webapp/AzkabanWebServer.java 39(+13 -26)
src/web/js/azkaban/model/log-data.js 53(+37 -16)
src/web/js/azkaban/view/job-details.js 44(+44 -0)
Details
src/java/azkaban/webapp/AzkabanWebServer.java 39(+13 -26)
diff --git a/src/java/azkaban/webapp/AzkabanWebServer.java b/src/java/azkaban/webapp/AzkabanWebServer.java
index 25a81c7..21460b2 100644
--- a/src/java/azkaban/webapp/AzkabanWebServer.java
+++ b/src/java/azkaban/webapp/AzkabanWebServer.java
@@ -93,8 +93,9 @@ import azkaban.webapp.servlet.HistoryServlet;
import azkaban.webapp.servlet.ProjectServlet;
import azkaban.webapp.servlet.ProjectManagerServlet;
import azkaban.webapp.servlet.TriggerManagerServlet;
-import azkaban.webapp.servlet.TriggerPlugin;
-import azkaban.webapp.servlet.ViewerPlugin;
+import azkaban.webapp.plugin.TriggerPlugin;
+import azkaban.webapp.plugin.ViewerPlugin;
+import azkaban.webapp.plugin.PluginRegistry;
import azkaban.webapp.session.SessionCache;
/**
@@ -153,7 +154,6 @@ public class AzkabanWebServer extends AzkabanServer {
private Props props;
private SessionCache sessionCache;
private File tempDir;
- private List<ViewerPlugin> viewerPlugins;
private Map<String, TriggerPlugin> triggerPlugins;
private MBeanServer mbeanServer;
@@ -209,10 +209,6 @@ public class AzkabanWebServer extends AzkabanServer {
configureMBeanServer();
}
- private void setViewerPlugins(List<ViewerPlugin> viewerPlugins) {
- this.viewerPlugins = viewerPlugins;
- }
-
private void setTriggerPlugins(Map<String, TriggerPlugin> triggerPlugins) {
this.triggerPlugins = triggerPlugins;
}
@@ -769,7 +765,7 @@ public class AzkabanWebServer extends AzkabanServer {
root.addServlet(new ServletHolder(new TriggerManagerServlet()),"/triggers");
String viewerPluginDir = azkabanSettings.getString("viewer.plugin.dir", "plugins/viewer");
- app.setViewerPlugins(loadViewerPlugins(root, viewerPluginDir, app.getVelocityEngine()));
+ loadViewerPlugins(root, viewerPluginDir, app.getVelocityEngine());
// triggerplugin
String triggerPluginDir = azkabanSettings.getString("trigger.plugin.dir", "plugins/triggers");
@@ -957,13 +953,12 @@ public class AzkabanWebServer extends AzkabanServer {
return triggerPlugins;
}
- private static List<ViewerPlugin> loadViewerPlugins(Context root, String pluginPath, VelocityEngine ve) {
+ private static void loadViewerPlugins(Context root, String pluginPath, VelocityEngine ve) {
File viewerPluginPath = new File(pluginPath);
if (!viewerPluginPath.exists()) {
- return Collections.<ViewerPlugin>emptyList();
+ return;
}
- ArrayList<ViewerPlugin> installedViewerPlugins = new ArrayList<ViewerPlugin>();
ClassLoader parentLoader = AzkabanWebServer.class.getClassLoader();
File[] pluginDirs = viewerPluginPath.listFiles();
ArrayList<String> jarPaths = new ArrayList<String>();
@@ -1005,6 +1000,7 @@ public class AzkabanWebServer extends AzkabanServer {
String pluginName = pluginProps.getString("viewer.name");
String pluginWebPath = pluginProps.getString("viewer.path");
+ String pluginJobType = pluginProps.getString("viewer.jobtype", null);
int pluginOrder = pluginProps.getInt("viewer.order", 0);
boolean pluginHidden = pluginProps.getBoolean("viewer.hidden", false);
List<String> extLibClasspath = pluginProps.getStringList("viewer.external.classpaths", (List<String>)null);
@@ -1108,27 +1104,18 @@ public class AzkabanWebServer extends AzkabanServer {
AbstractAzkabanServlet avServlet = (AbstractAzkabanServlet)obj;
root.addServlet(new ServletHolder(avServlet), "/" + pluginWebPath + "/*");
- installedViewerPlugins.add(new ViewerPlugin(pluginName, pluginWebPath, pluginOrder, pluginHidden));
+ PluginRegistry.getRegistry().register(new ViewerPlugin(
+ pluginName,
+ pluginWebPath,
+ pluginOrder,
+ pluginHidden,
+ pluginJobType));
}
// Velocity needs the jar resource paths to be set.
String jarResourcePath = StringUtils.join(jarPaths, ", ");
logger.info("Setting jar resource path " + jarResourcePath);
ve.addProperty("jar.resource.loader.path", jarResourcePath);
-
- // Sort plugins based on order
- Collections.sort(installedViewerPlugins, new Comparator<ViewerPlugin>() {
- @Override
- public int compare(ViewerPlugin o1, ViewerPlugin o2) {
- return o1.getOrder() - o2.getOrder();
- }
- });
-
- return installedViewerPlugins;
- }
-
- public List<ViewerPlugin> getViewerPlugins() {
- return viewerPlugins;
}
/**
diff --git a/src/java/azkaban/webapp/plugin/PluginRegistry.java b/src/java/azkaban/webapp/plugin/PluginRegistry.java
new file mode 100644
index 0000000..d49db54
--- /dev/null
+++ b/src/java/azkaban/webapp/plugin/PluginRegistry.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2014 LinkedIn Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package azkaban.webapp.plugin;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeSet;
+import java.util.Set;
+
+public class PluginRegistry {
+
+ private static PluginRegistry registry;
+
+ public TreeSet<ViewerPlugin> viewerPlugins;
+
+ public Map<String, TreeSet<ViewerPlugin>> jobTypeViewerPlugins;
+
+ private PluginRegistry() {
+ viewerPlugins = new TreeSet<ViewerPlugin>(ViewerPlugin.COMPARATOR);
+ jobTypeViewerPlugins = new HashMap<String, TreeSet<ViewerPlugin>>();
+ }
+
+ public void register(ViewerPlugin plugin) {
+ viewerPlugins.add(plugin);
+ String jobType = plugin.getJobType();
+ if (jobType == null) {
+ return;
+ }
+ TreeSet<ViewerPlugin> plugins = null;
+ if (!jobTypeViewerPlugins.containsKey(jobType)) {
+ plugins = new TreeSet<ViewerPlugin>(ViewerPlugin.COMPARATOR);
+ plugins.add(plugin);
+ jobTypeViewerPlugins.put(jobType, plugins);
+ }
+ else {
+ plugins = jobTypeViewerPlugins.get(jobType);
+ plugins.add(plugin);
+ }
+ }
+
+ public List<ViewerPlugin> getViewerPlugins() {
+ return new ArrayList<ViewerPlugin>(viewerPlugins);
+ }
+
+ public List<ViewerPlugin> getViewerPluginsForJobType(String jobType) {
+ TreeSet<ViewerPlugin> plugins = jobTypeViewerPlugins.get(jobType);
+ if (plugins == null) {
+ return null;
+ }
+ return new ArrayList<ViewerPlugin>(plugins);
+ }
+
+ public static PluginRegistry getRegistry() {
+ if (registry == null) {
+ registry = new PluginRegistry();
+ }
+ return registry;
+ }
+}
diff --git a/src/java/azkaban/webapp/servlet/AbstractAzkabanServlet.java b/src/java/azkaban/webapp/servlet/AbstractAzkabanServlet.java
index 8ac470e..2fc6ecc 100644
--- a/src/java/azkaban/webapp/servlet/AbstractAzkabanServlet.java
+++ b/src/java/azkaban/webapp/servlet/AbstractAzkabanServlet.java
@@ -39,6 +39,9 @@ import azkaban.utils.Props;
import azkaban.webapp.AzkabanServer;
import azkaban.webapp.AzkabanWebServer;
import azkaban.webapp.session.Session;
+import azkaban.webapp.plugin.ViewerPlugin;
+import azkaban.webapp.plugin.TriggerPlugin;
+import azkaban.webapp.plugin.PluginRegistry;
/**
* Base Servlet for pages
@@ -91,7 +94,7 @@ public abstract class AbstractAzkabanServlet extends HttpServlet {
if (application instanceof AzkabanWebServer) {
AzkabanWebServer server = (AzkabanWebServer)application;
- viewerPlugins = server.getViewerPlugins();
+ viewerPlugins = PluginRegistry.getRegistry().getViewerPlugins();
triggerPlugins = new ArrayList<TriggerPlugin>(server.getTriggerPlugins().values());
}
}
diff --git a/src/java/azkaban/webapp/servlet/ExecutorServlet.java b/src/java/azkaban/webapp/servlet/ExecutorServlet.java
index dadf693..bcab22d 100644
--- a/src/java/azkaban/webapp/servlet/ExecutorServlet.java
+++ b/src/java/azkaban/webapp/servlet/ExecutorServlet.java
@@ -50,6 +50,8 @@ import azkaban.utils.FileIOUtils.LogData;
import azkaban.utils.JSONUtils;
import azkaban.webapp.AzkabanWebServer;
import azkaban.webapp.session.Session;
+import azkaban.webapp.plugin.PluginRegistry;
+import azkaban.webapp.plugin.ViewerPlugin;
public class ExecutorServlet extends LoginAbstractAzkabanServlet {
private static final long serialVersionUID = 1L;
@@ -194,7 +196,18 @@ public class ExecutorServlet extends LoginAbstractAzkabanServlet {
page.render();
return;
}
- } catch (ExecutorManagerException e) {
+
+ ExecutableNode node = flow.getExecutableNode(jobId);
+ if (node == null) {
+ page.add("errorMsg", "Job " + jobId + " doesn't exist in " + flow.getExecutionId());
+ return;
+ }
+
+ List<ViewerPlugin> jobViewerPlugins = PluginRegistry.getRegistry()
+ .getViewerPluginsForJobType(node.getType());
+ page.add("jobViewerPlugins", jobViewerPlugins);
+ }
+ catch (ExecutorManagerException e) {
page.add("errorMsg", "Error loading executing flow: " + e.getMessage());
page.render();
return;
@@ -206,7 +219,7 @@ public class ExecutorServlet extends LoginAbstractAzkabanServlet {
page.render();
return;
}
-
+
page.add("projectName", project.getName());
page.add("flowid", flow.getFlowId());
diff --git a/src/java/azkaban/webapp/servlet/velocity/jobdetailsheader.vm b/src/java/azkaban/webapp/servlet/velocity/jobdetailsheader.vm
new file mode 100644
index 0000000..4bf06c3
--- /dev/null
+++ b/src/java/azkaban/webapp/servlet/velocity/jobdetailsheader.vm
@@ -0,0 +1,61 @@
+#*
+ * Copyright 2014 LinkedIn Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+*#
+
+ ## Page header.
+
+ <div class="az-page-header">
+ <div class="container-full">
+ <div class="row">
+ <div class="col-xs-6">
+ <h1><a href="${context}/executor?execid=${execid}&job=${jobid}">Job Execution <small>$jobid</small></a></h1>
+ </div>
+ <div class="col-xs-6">
+ <div class="pull-right az-page-header-form">
+ <a href="${context}/manager?project=${projectName}&flow=${flowid}&job=$jobid" class="btn btn-info">Job Properties</a>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+
+ <div class="container-full">
+
+ #parse ("azkaban/webapp/servlet/velocity/alerts.vm")
+
+ ## Breadcrumb
+
+ <ol class="breadcrumb">
+ <li><a href="${context}/manager?project=${projectName}"><strong>Project</strong> $projectName</a></li>
+ <li><a href="${context}/manager?project=${projectName}&flow=${flowid}"><strong>Flow</strong> $flowid</a></li>
+ <li><a href="${context}/executor?execid=${execid}#jobslist"><strong>Execution</strong> $execid</a></li>
+ <li class="active"><strong>Job</strong> $jobid</li>
+ </ol>
+
+ ## Tabs
+
+ <ul class="nav nav-tabs" id="headertabs">
+ #if ($current_page == "executing")
+ <li id="jobLogViewLink"><a href="#logs">Job Logs</a></li>
+ <li id="jobSummaryViewLink"><a href="#summary">Summary</a></li>
+ #else
+ <li id="jobLogViewLink"><a href="${context}/executor?execid=${execid}&job=${jobid}#logs">Job Logs</a></li>
+ <li id="jobSummaryViewLink"><a href="${context}/executor?execid=${execid}&job=${jobid}#summary">Summary</a></li>
+ #end
+ #foreach ($jobViewerPlugin in $jobViewerPlugins)
+ <li#if($current_page == $jobViewerPlugin.pluginName) class="active"#end><a href="$!context/${jobViewerPlugin.pluginPath}?execid=${execid}&jobid=${jobid}">$jobViewerPlugin.pluginName</a></li>
+ #end
+ </ul>
+ </div>
diff --git a/src/java/azkaban/webapp/servlet/velocity/jobdetailspage.vm b/src/java/azkaban/webapp/servlet/velocity/jobdetailspage.vm
index 50858db..f551245 100644
--- a/src/java/azkaban/webapp/servlet/velocity/jobdetailspage.vm
+++ b/src/java/azkaban/webapp/servlet/velocity/jobdetailspage.vm
@@ -47,44 +47,8 @@
#parse ("azkaban/webapp/servlet/velocity/errormsg.vm")
#else
- ## Page header.
- <div class="az-page-header">
- <div class="container-full">
- <div class="row">
- <div class="col-xs-6">
- <h1><a href="${context}/executor?execid=${execid}&job=${jobid}">Job Execution <small>$jobid</small></a></h1>
- </div>
- <div class="col-xs-6">
- <div class="pull-right az-page-header-form">
- <a href="${context}/manager?project=${projectName}&flow=${flowid}&job=$jobid" class="btn btn-info">Job Properties</a>
- </div>
- </div>
- </div>
- </div>
- </div>
-
- <div class="container-full">
-
- #parse ("azkaban/webapp/servlet/velocity/alerts.vm")
-
- ## Breadcrumb
-
- <ol class="breadcrumb">
- <li><a href="${context}/manager?project=${projectName}"><strong>Project</strong> $projectName</a></li>
- <li><a href="${context}/manager?project=${projectName}&flow=${flowid}"><strong>Flow</strong> $flowid</a></li>
- <li><a href="${context}/executor?execid=${execid}#jobslist"><strong>Execution</strong> $execid</a></li>
- <li class="active"><strong>Job</strong> $jobid</li>
- </ol>
-
- ## Tabs
-
- <ul class="nav nav-tabs" id="headertabs">
- <li id="jobLogViewLink"><a href="#logs">Job Logs</a></li>
- <li id="jobSummaryViewLink"><a href="#summary">Summary</a></li>
- <li><a href="${context}/pigvisualizer?execid=${execid}&jobid=${jobid}">Visualization</a></li>
- </ul>
- </div>
+ #parse ("azkaban/webapp/servlet/velocity/jobdetailsheader.vm")
## Log content.
@@ -120,6 +84,11 @@
</div>
</h3>
+ <div id="jobType">
+ <table id="jobTypeTable" class="table table-striped table-bordered table-hover">
+ </table>
+ </div>
+
<div id="command-summary">
<h4>Command Summary</h4>
<table id="commandTable" class="table table-striped table-bordered table-hover">
@@ -157,6 +126,13 @@
</tbody>
</table>
</div>
+
+ <div id="jobIds">
+ <h4>Map Reduce Jobs</h4>
+ <table class="table table-striped table-bordered table-hover">
+ <tbody id="jobIdsTableBody"></tbody>
+ </table>
+ </div>
</div>
</div>
</div>
src/web/js/azkaban/model/log-data.js 53(+37 -16)
diff --git a/src/web/js/azkaban/model/log-data.js b/src/web/js/azkaban/model/log-data.js
index 2fe3ac5..90a958f 100644
--- a/src/web/js/azkaban/model/log-data.js
+++ b/src/web/js/azkaban/model/log-data.js
@@ -24,6 +24,8 @@ azkaban.LogDataModel = Backbone.Model.extend({
HIVE_MAP_REDUCE_JOBS_SUMMARY: "MapReduce Jobs Launched:",
HIVE_MAP_REDUCE_SUMMARY_REGEX: /Job (\d+):\s+Map: (\d+)\s+Reduce: (\d+)\s+(?:Cumulative CPU: (.+?))?\s+HDFS Read: (\d+)\s+HDFS Write: (\d+)/,
+ JOB_ID_REGEX: /job_\d{12}_\d{4,}/,
+
initialize: function() {
this.set("offset", 0 );
this.set("logData", "");
@@ -91,14 +93,19 @@ azkaban.LogDataModel = Backbone.Model.extend({
var lines = data.split("\n");
if (this.parseCommand(lines)) {
+ this.parseJobType(lines);
this.parseJobTrackerUrls(lines);
- var jobType = this.parseJobType(lines);
- if (jobType.indexOf("pig") !== -1) {
- this.parsePigTable(lines, "pigSummary", this.PIG_JOB_SUMMARY_START, "", 0);
- this.parsePigTable(lines, "pigStats", this.PIG_JOB_STATS_START, "", 1);
- } else if (jobType.indexOf("hive") !== -1) {
- this.parseHiveQueries(lines);
+ var jobType = this.get("jobType");
+ if (jobType) {
+ if (jobType.indexOf("pig") !== -1) {
+ this.parsePigTable(lines, "pigSummary", this.PIG_JOB_SUMMARY_START, "", 0);
+ this.parsePigTable(lines, "pigStats", this.PIG_JOB_STATS_START, "", 1);
+ } else if (jobType.indexOf("hive") !== -1) {
+ this.parseHiveQueries(lines);
+ } else {
+ this.parseJobIds(lines);
+ }
}
}
},
@@ -163,16 +170,32 @@ azkaban.LogDataModel = Backbone.Model.extend({
this.set("jobTrackerUrlsOrdered", jobTrackerUrlsOrdered);
},
+ parseJobIds: function(lines) {
+ var seenJobIds = {};
+ var jobIds = [];
+ var numLines = lines.length;
+ var match;
+ for (var i = 0; i < numLines; i++) {
+ if ((match = this.JOB_ID_REGEX.exec(lines[i])) && !seenJobIds[match[0]]) {
+ seenJobIds[match[0]] = true;
+ jobIds.push(match[0]);
+ }
+ }
+
+ if (jobIds.length > 0) {
+ this.set("jobIds", jobIds);
+ }
+ },
+
parseJobType: function(lines) {
var numLines = lines.length;
var match;
- for (var i = 0; numLines; i++) {
+ for (var i = 0; i < numLines; i++) {
if (match = this.JOB_TYPE_REGEX.exec(lines[i])) {
- return match[1];
+ this.set("jobType", match[1]);
+ break;
}
}
-
- return null;
},
parsePigTable: function(lines, tableName, startPattern, endPattern, linesToSkipAfterStart) {
@@ -262,14 +285,12 @@ azkaban.LogDataModel = Backbone.Model.extend({
job.push("<a href='" + this.get("jobTrackerUrlsOrdered")[currMapReduceJob++] + "'>" + currJob + "</a>");
job.push(match[2]);
job.push(match[3]);
- job.push(match[4]);
- job.push(match[5]);
- job.push(match[6]);
-
- if (match[7]) {
+ if (match[4]) {
this.set("hasCumulativeCPU", true);
- job.push(match[7]);
+ job.push(match[4]);
}
+ job.push(match[5]);
+ job.push(match[6]);
queryJobs.push(job);
previousJob = currJob;
src/web/js/azkaban/view/job-details.js 44(+44 -0)
diff --git a/src/web/js/azkaban/view/job-details.js b/src/web/js/azkaban/view/job-details.js
index ab77045..07b823e 100644
--- a/src/web/js/azkaban/view/job-details.js
+++ b/src/web/js/azkaban/view/job-details.js
@@ -45,15 +45,19 @@ azkaban.JobSummaryView = Backbone.View.extend({
},
initialize: function(settings) {
+ $("#jobType").hide();
$("#commandSummary").hide();
$("#pigJobSummary").hide();
$("#pigJobStats").hide();
$("#hiveJobSummary").hide();
+ $("#jobIds").hide();
+ this.listenTo(this.model, "change:jobType", this.renderJobTypeTable);
this.listenTo(this.model, "change:commandProperties", this.renderCommandTable);
this.listenTo(this.model, "change:pigSummary", this.renderPigSummaryTable);
this.listenTo(this.model, "change:pigStats", this.renderPigStatsTable);
this.listenTo(this.model, "change:hiveSummary", this.renderHiveTable);
+ this.listenTo(this.model, "change:jobIds", this.renderJobIdsTable);
},
refresh: function() {
@@ -65,6 +69,46 @@ azkaban.JobSummaryView = Backbone.View.extend({
renderJobTable(jobSummary.statTableHeaders, jobSummary.statTableData, "stats");
renderHiveTable(jobSummary.hiveQueries, jobSummary.hiveQueryJobs);
},
+
+ renderJobTypeTable: function() {
+ var jobTypeTable = $("#jobTypeTable");
+ var jobType = this.model.get("jobType");
+
+ var tr = document.createElement("tr");
+ var td = document.createElement("td");
+ $(td).html("<b>Job Type</b>");
+ $(tr).append(td);
+ td = document.createElement("td");
+ $(td).html(jobType);
+ $(tr).append(td);
+
+ jobTypeTable.append(tr);
+
+ $("#jobType").show();
+ },
+
+ renderJobIdsTable: function() {
+ var oldBody = $("#jobIdsTableBody");
+ var newBody = $(document.createElement("tbody")).attr("id", "jobIdsTableBody");
+
+ var jobIds = this.model.get("jobIds");
+ var jobUrls = this.model.get("jobTrackerUrls");
+ var numJobs = jobIds.length;
+ for (var i = 0; i < numJobs; i++) {
+ var job = jobIds[i];
+ var tr = document.createElement("tr");
+ var td = document.createElement("td");
+ var html = jobUrls[job] ? "<a href='" + jobUrls[job] + "'>" + job + "</a>" : job;
+ $(td).html(html);
+ $(tr).append(td);
+ newBody.append(tr);
+ }
+
+ oldBody.replaceWith(newBody);
+
+ $("#jobIds").show();
+ },
+
renderCommandTable: function() {
var commandTable = $("#commandTable");
var commandProperties = this.model.get("commandProperties");