Details
diff --git a/azkaban-common/src/main/java/azkaban/jobcallback/JobCallbackValidator.java b/azkaban-common/src/main/java/azkaban/jobcallback/JobCallbackValidator.java
index 7b1462f..a85d1a3 100644
--- a/azkaban-common/src/main/java/azkaban/jobcallback/JobCallbackValidator.java
+++ b/azkaban-common/src/main/java/azkaban/jobcallback/JobCallbackValidator.java
@@ -54,8 +54,10 @@ public class JobCallbackValidator {
maxPostBodyLength);
}
- logger.info("Found " + totalCallbackCount + " job callbacks for job "
- + jobName);
+ if (logger.isDebugEnabled()) {
+ logger.debug("Found " + totalCallbackCount + " job callbacks for job "
+ + jobName);
+ }
return totalCallbackCount;
}
diff --git a/azkaban-common/src/main/java/azkaban/project/JdbcProjectLoader.java b/azkaban-common/src/main/java/azkaban/project/JdbcProjectLoader.java
index bde18ec..7111831 100644
--- a/azkaban-common/src/main/java/azkaban/project/JdbcProjectLoader.java
+++ b/azkaban-common/src/main/java/azkaban/project/JdbcProjectLoader.java
@@ -784,7 +784,6 @@ public class JdbcProjectLoader extends AbstractJdbcLoader implements
byte[] stringData = json.getBytes("UTF-8");
byte[] data = stringData;
- logger.info("UTF-8 size:" + data.length);
if (defaultEncodingType == EncodingType.GZIP) {
data = GZIPUtils.gzipBytes(stringData);
}
@@ -827,7 +826,6 @@ public class JdbcProjectLoader extends AbstractJdbcLoader implements
byte[] stringData = json.getBytes("UTF-8");
byte[] data = stringData;
- logger.info("UTF-8 size:" + data.length);
if (encType == EncodingType.GZIP) {
data = GZIPUtils.gzipBytes(stringData);
}
@@ -948,7 +946,6 @@ public class JdbcProjectLoader extends AbstractJdbcLoader implements
String propertyJSON = PropsUtils.toJSONString(props, true);
byte[] data = propertyJSON.getBytes("UTF-8");
- logger.info("UTF-8 size:" + data.length);
if (defaultEncodingType == EncodingType.GZIP) {
data = GZIPUtils.gzipBytes(data);
}
@@ -971,7 +968,6 @@ public class JdbcProjectLoader extends AbstractJdbcLoader implements
String propertyJSON = PropsUtils.toJSONString(props, true);
byte[] data = propertyJSON.getBytes("UTF-8");
- logger.info("UTF-8 size:" + data.length);
if (defaultEncodingType == EncodingType.GZIP) {
data = GZIPUtils.gzipBytes(data);
}
diff --git a/azkaban-common/src/main/java/azkaban/trigger/builtin/SlaChecker.java b/azkaban-common/src/main/java/azkaban/trigger/builtin/SlaChecker.java
index 1141176..27b9ba4 100644
--- a/azkaban-common/src/main/java/azkaban/trigger/builtin/SlaChecker.java
+++ b/azkaban-common/src/main/java/azkaban/trigger/builtin/SlaChecker.java
@@ -20,7 +20,6 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.log4j.Logger;
-
import org.joda.time.DateTime;
import org.joda.time.ReadablePeriod;
@@ -57,7 +56,6 @@ public class SlaChecker implements ConditionChecker {
private Boolean isSlaMissed(ExecutableFlow flow) {
String type = slaOption.getType();
- logger.info("flow is " + flow.getStatus());
if (flow.getStartTime() < 0) {
return Boolean.FALSE;
}
@@ -136,7 +134,6 @@ public class SlaChecker implements ConditionChecker {
private Boolean isSlaGood(ExecutableFlow flow) {
String type = slaOption.getType();
- logger.info("flow is " + flow.getStatus());
if (flow.getStartTime() < 0) {
return Boolean.FALSE;
}
@@ -218,13 +215,11 @@ public class SlaChecker implements ConditionChecker {
}
public Object isSlaFailed() {
- logger.info("Testing if sla failed for execution " + execId);
ExecutableFlow flow;
try {
flow = executorManager.getExecutableFlow(execId);
} catch (ExecutorManagerException e) {
logger.error("Can't get executable flow.", e);
- e.printStackTrace();
// something wrong, send out alerts
return Boolean.TRUE;
}
@@ -232,13 +227,11 @@ public class SlaChecker implements ConditionChecker {
}
public Object isSlaPassed() {
- logger.info("Testing if sla is good for execution " + execId);
ExecutableFlow flow;
try {
flow = executorManager.getExecutableFlow(execId);
} catch (ExecutorManagerException e) {
logger.error("Can't get executable flow.", e);
- e.printStackTrace();
// something wrong, send out alerts
return Boolean.TRUE;
}
diff --git a/azkaban-common/src/main/java/azkaban/trigger/Condition.java b/azkaban-common/src/main/java/azkaban/trigger/Condition.java
index 7bb275b..5e8e7b6 100644
--- a/azkaban-common/src/main/java/azkaban/trigger/Condition.java
+++ b/azkaban-common/src/main/java/azkaban/trigger/Condition.java
@@ -25,7 +25,6 @@ import org.apache.commons.jexl2.Expression;
import org.apache.commons.jexl2.JexlEngine;
import org.apache.commons.jexl2.MapContext;
import org.apache.log4j.Logger;
-
import org.joda.time.DateTime;
public class Condition {
@@ -119,7 +118,9 @@ public class Condition {
}
public boolean isMet() {
- logger.info("Testing condition " + expression);
+ if (logger.isDebugEnabled()) {
+ logger.debug("Testing condition " + expression);
+ }
return expression.evaluate(context).equals(Boolean.TRUE);
}
diff --git a/azkaban-common/src/main/java/azkaban/trigger/JdbcTriggerLoader.java b/azkaban-common/src/main/java/azkaban/trigger/JdbcTriggerLoader.java
index 68c0508..99b32f9 100644
--- a/azkaban-common/src/main/java/azkaban/trigger/JdbcTriggerLoader.java
+++ b/azkaban-common/src/main/java/azkaban/trigger/JdbcTriggerLoader.java
@@ -194,7 +194,9 @@ public class JdbcTriggerLoader extends AbstractJdbcLoader implements
@Override
public void updateTrigger(Trigger t) throws TriggerLoaderException {
- logger.info("Updating trigger " + t.getTriggerId() + " into db.");
+ if (logger.isDebugEnabled()) {
+ logger.debug("Updating trigger " + t.getTriggerId() + " into db.");
+ }
t.setLastModifyTime(System.currentTimeMillis());
Connection connection = getConnection();
try {
@@ -238,7 +240,9 @@ public class JdbcTriggerLoader extends AbstractJdbcLoader implements
if (updates == 0) {
throw new TriggerLoaderException("No trigger has been updated.");
} else {
- logger.info("Updated " + updates + " records.");
+ if (logger.isDebugEnabled()) {
+ logger.debug("Updated " + updates + " records.");
+ }
}
} catch (SQLException e) {
logger.error(UPDATE_TRIGGER + " failed.");
diff --git a/azkaban-common/src/main/java/azkaban/trigger/TriggerManager.java b/azkaban-common/src/main/java/azkaban/trigger/TriggerManager.java
index fa8d130..97e858c 100644
--- a/azkaban-common/src/main/java/azkaban/trigger/TriggerManager.java
+++ b/azkaban-common/src/main/java/azkaban/trigger/TriggerManager.java
@@ -281,7 +281,9 @@ public class TriggerManager extends EventHandler implements
logger.info("Skipping trigger" + t.getTriggerId() + " until " + t.getNextCheckTime());
}
- logger.info("Checking trigger " + t.getTriggerId());
+ if (logger.isDebugEnabled()) {
+ logger.info("Checking trigger " + t.getTriggerId());
+ }
if (t.getStatus().equals(TriggerStatus.READY)) {
if (t.triggerConditionMet()) {
onTriggerTrigger(t);
diff --git a/azkaban-common/src/main/java/azkaban/utils/StringUtils.java b/azkaban-common/src/main/java/azkaban/utils/StringUtils.java
index 17792a0..6e1c5d8 100644
--- a/azkaban-common/src/main/java/azkaban/utils/StringUtils.java
+++ b/azkaban-common/src/main/java/azkaban/utils/StringUtils.java
@@ -18,6 +18,7 @@ package azkaban.utils;
import java.util.Collection;
import java.util.List;
+import java.util.regex.Pattern;
public class StringUtils {
public static final char SINGLE_QUOTE = '\'';
@@ -88,4 +89,19 @@ public class StringUtils {
return buffer.toString();
}
+
+ private static final Pattern BROWSWER_PATTERN = Pattern
+ .compile(".*Chrome.*|.*Firefox.*|.*Safari.*");
+
+ public static boolean isFromBrowser(String userAgent) {
+ if (userAgent == null) {
+ return false;
+ }
+
+ if (BROWSWER_PATTERN.matcher(userAgent).matches()) {
+ return true;
+ } else {
+ return false;
+ }
+ }
}
diff --git a/azkaban-webserver/src/main/java/azkaban/webapp/AzkabanWebServer.java b/azkaban-webserver/src/main/java/azkaban/webapp/AzkabanWebServer.java
index 3171e9a..b5b8d0e 100644
--- a/azkaban-webserver/src/main/java/azkaban/webapp/AzkabanWebServer.java
+++ b/azkaban-webserver/src/main/java/azkaban/webapp/AzkabanWebServer.java
@@ -39,6 +39,7 @@ import javax.management.ObjectName;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
+import org.apache.log4j.jmx.HierarchyDynamicMBean;
import org.apache.velocity.app.VelocityEngine;
import org.apache.velocity.runtime.log.Log4JLogChute;
import org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader;
@@ -53,8 +54,6 @@ import org.mortbay.jetty.servlet.DefaultServlet;
import org.mortbay.jetty.servlet.ServletHolder;
import org.mortbay.thread.QueuedThreadPool;
-import com.linkedin.restli.server.RestliServlet;
-
import azkaban.alert.Alerter;
import azkaban.database.AzkabanDatabaseSetup;
import azkaban.executor.ExecutorManager;
@@ -88,19 +87,21 @@ import azkaban.utils.FileIOUtils;
import azkaban.utils.Props;
import azkaban.utils.PropsUtils;
import azkaban.utils.Utils;
+import azkaban.webapp.plugin.PluginRegistry;
+import azkaban.webapp.plugin.TriggerPlugin;
+import azkaban.webapp.plugin.ViewerPlugin;
import azkaban.webapp.servlet.AbstractAzkabanServlet;
import azkaban.webapp.servlet.ExecutorServlet;
+import azkaban.webapp.servlet.HistoryServlet;
import azkaban.webapp.servlet.IndexRedirectServlet;
import azkaban.webapp.servlet.JMXHttpServlet;
-import azkaban.webapp.servlet.ScheduleServlet;
-import azkaban.webapp.servlet.HistoryServlet;
-import azkaban.webapp.servlet.ProjectServlet;
import azkaban.webapp.servlet.ProjectManagerServlet;
+import azkaban.webapp.servlet.ProjectServlet;
+import azkaban.webapp.servlet.ScheduleServlet;
import azkaban.webapp.servlet.StatsServlet;
import azkaban.webapp.servlet.TriggerManagerServlet;
-import azkaban.webapp.plugin.TriggerPlugin;
-import azkaban.webapp.plugin.ViewerPlugin;
-import azkaban.webapp.plugin.PluginRegistry;
+
+import com.linkedin.restli.server.RestliServlet;
/**
* The Azkaban Jetty server class
@@ -123,6 +124,9 @@ import azkaban.webapp.plugin.PluginRegistry;
* Jetty truststore password
*/
public class AzkabanWebServer extends AzkabanServer {
+ private static final String AZKABAN_ACCESS_LOGGER_NAME =
+ "azkaban.webapp.servlet.LoginAbstractAzkabanServlet";
+
private static final Logger logger = Logger.getLogger(AzkabanWebServer.class);
public static final String AZKABAN_HOME = "AZKABAN_HOME";
@@ -823,23 +827,25 @@ public class AzkabanWebServer extends AzkabanServer {
public void logTopMemoryConsumers() throws Exception, IOException {
if (new File("/bin/bash").exists() && new File("/bin/ps").exists()
- && new File("/usr/bin/head").exists()) {
+ && new File("/usr/bin/head").exists()) {
logger.info("logging top memeory consumer");
java.lang.ProcessBuilder processBuilder =
- new java.lang.ProcessBuilder("/bin/bash", "-c", "/bin/ps aux --sort -rss | /usr/bin/head");
+ new java.lang.ProcessBuilder("/bin/bash", "-c",
+ "/bin/ps aux --sort -rss | /usr/bin/head");
Process p = processBuilder.start();
p.waitFor();
-
+
InputStream is = p.getInputStream();
- java.io.BufferedReader reader = new java.io.BufferedReader(new InputStreamReader(is));
+ java.io.BufferedReader reader =
+ new java.io.BufferedReader(new InputStreamReader(is));
String line = null;
while ((line = reader.readLine()) != null) {
logger.info(line);
}
is.close();
}
- }
+ }
});
logger.info("Server running on " + (ssl ? "ssl" : "") + " port " + port
+ ".");
@@ -1233,6 +1239,22 @@ public class AzkabanWebServer extends AzkabanServer {
registerMbean("executorManager", new JmxExecutorManager(
(ExecutorManager) executorManager));
}
+
+ // Register Log4J loggers as JMX beans so the log level can be
+ // updated via JConsole or Java VisualVM
+ HierarchyDynamicMBean log4jMBean = new HierarchyDynamicMBean();
+ registerMbean("log4jmxbean", log4jMBean);
+ ObjectName accessLogLoggerObjName =
+ log4jMBean.addLoggerMBean(AZKABAN_ACCESS_LOGGER_NAME);
+
+ if (accessLogLoggerObjName == null) {
+ System.out
+ .println("************* loginLoggerObjName is null, make sure there is a logger with name "
+ + AZKABAN_ACCESS_LOGGER_NAME);
+ } else {
+ System.out.println("******** loginLoggerObjName: "
+ + accessLogLoggerObjName.getCanonicalName());
+ }
}
public void close() {
diff --git a/azkaban-webserver/src/main/java/azkaban/webapp/servlet/LoginAbstractAzkabanServlet.java b/azkaban-webserver/src/main/java/azkaban/webapp/servlet/LoginAbstractAzkabanServlet.java
index e0b9e38..5eee1d4 100644
--- a/azkaban-webserver/src/main/java/azkaban/webapp/servlet/LoginAbstractAzkabanServlet.java
+++ b/azkaban-webserver/src/main/java/azkaban/webapp/servlet/LoginAbstractAzkabanServlet.java
@@ -25,6 +25,7 @@ import java.io.Writer;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
+import java.util.regex.Pattern;
import javax.servlet.ServletConfig;
import javax.servlet.ServletException;
@@ -43,6 +44,7 @@ import azkaban.user.Role;
import azkaban.user.User;
import azkaban.user.UserManager;
import azkaban.user.UserManagerException;
+import azkaban.utils.StringUtils;
/**
* Abstract Servlet that handles auto login when the session hasn't been
@@ -58,6 +60,9 @@ public abstract class LoginAbstractAzkabanServlet extends
private static final String SESSION_ID_NAME = "azkaban.browser.session.id";
private static final int DEFAULT_UPLOAD_DISK_SPOOL_SIZE = 20 * 1024 * 1024;
+ private static final Pattern BROWSWER_PATTERN = Pattern
+ .compile("[*Chrome*|*Firefox*|*Safari*]");
+
private static HashMap<String, String> contextType =
new HashMap<String, String>();
static {
@@ -77,11 +82,17 @@ public abstract class LoginAbstractAzkabanServlet extends
private MultipartParser multipartParser;
+ private boolean shouldLogRawUserAgent = false;
+
@Override
public void init(ServletConfig config) throws ServletException {
super.init(config);
multipartParser = new MultipartParser(DEFAULT_UPLOAD_DISK_SPOOL_SIZE);
+
+ shouldLogRawUserAgent =
+ getApplication().getServerProps().getBoolean("accesslog.raw.useragent",
+ false);
}
public void setResourceDirectory(File file) {
@@ -93,6 +104,7 @@ public abstract class LoginAbstractAzkabanServlet extends
throws ServletException, IOException {
// Set session id
Session session = getSessionFromRequest(req);
+ logRequest(req, session);
if (hasParam(req, "logout")) {
resp.sendRedirect(req.getContextPath());
if (session != null) {
@@ -103,7 +115,9 @@ public abstract class LoginAbstractAzkabanServlet extends
}
if (session != null) {
- logger.info("Found session " + session.getUser());
+ if (logger.isDebugEnabled()) {
+ logger.debug("Found session " + session.getUser());
+ }
if (handleFileGet(req, resp)) {
return;
}
@@ -120,6 +134,46 @@ public abstract class LoginAbstractAzkabanServlet extends
}
}
+ /**
+ * Log out request - the format should be close to Apache access log format
+ *
+ * @param req
+ * @param session
+ */
+ private void logRequest(HttpServletRequest req, Session session) {
+ StringBuilder buf = new StringBuilder();
+ buf.append(req.getRemoteAddr()).append(" ");
+ if (session != null && session.getUser() != null) {
+ buf.append(session.getUser().getUserId()).append(" ");
+ } else {
+ buf.append(" - ").append(" ");
+ }
+
+ buf.append("\"");
+ buf.append(req.getMethod()).append(" ");
+ buf.append(req.getRequestURI()).append(" ");
+ if (req.getQueryString() != null) {
+ buf.append(req.getQueryString()).append(" ");
+ } else {
+ buf.append("-").append(" ");
+ }
+ buf.append(req.getProtocol()).append("\" ");
+
+ String userAgent = req.getHeader("User-Agent");
+ if (shouldLogRawUserAgent) {
+ buf.append(userAgent);
+ } else {
+ // simply log a short string to indicate browser or not
+ if (StringUtils.isFromBrowser(userAgent)) {
+ buf.append("browser");
+ } else {
+ buf.append("not-browser");
+ }
+ }
+
+ logger.info(buf.toString());
+ }
+
private boolean handleFileGet(HttpServletRequest req, HttpServletResponse resp)
throws IOException {
if (webResourceDirectory == null) {
@@ -168,7 +222,6 @@ public abstract class LoginAbstractAzkabanServlet extends
if (cookie != null) {
sessionId = cookie.getValue();
- logger.info("Session id " + sessionId);
}
if (sessionId == null && hasParam(req, "session.id")) {
@@ -211,6 +264,8 @@ public abstract class LoginAbstractAzkabanServlet extends
throws ServletException, IOException {
Session session = getSessionFromRequest(req);
+ logRequest(req, session);
+
// Handle Multipart differently from other post messages
if (ServletFileUpload.isMultipartContent(req)) {
Map<String, Object> params = multipartParser.parseMultipart(req);
diff --git a/azkaban-webserver/src/main/resources/log4j.properties b/azkaban-webserver/src/main/resources/log4j.properties
index 0480c8c..4001d2c 100644
--- a/azkaban-webserver/src/main/resources/log4j.properties
+++ b/azkaban-webserver/src/main/resources/log4j.properties
@@ -1,20 +1,21 @@
+log_dir=${log4j.log.dir}
+
log4j.rootLogger=INFO, WebServer
log4j.logger.azkaban.webapp=INFO, WebServer
-log4j.logger.azkaban.webapp.servlet.AbstractAzkabanServlet=INFO, R
-log4j.logger.azkaban.webapp.servlet.LoginAbstractAzkabanServlet=INFO, R
+log4j.logger.azkaban.webapp.servlet.AbstractAzkabanServlet=INFO, Access
+log4j.logger.azkaban.webapp.servlet.LoginAbstractAzkabanServlet=INFO, Access
+log4j.additivity.azkaban.webapp.servlet.LoginAbstractAzkabanServlet=false
-log4j.appender.R=org.apache.log4j.RollingFileAppender
-log4j.appender.R.layout=org.apache.log4j.PatternLayout
-log4j.appender.R.File=azkaban-access.log
-log4j.appender.R.layout.ConversionPattern=%d{yyyy/MM/dd HH:mm:ss.SSS Z} %p [%c{1}] [Azkaban] %m%n
-log4j.appender.R.MaxFileSize=102400MB
-log4j.appender.R.MaxBackupIndex=2
+log4j.appender.Access=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.Access.layout=org.apache.log4j.PatternLayout
+log4j.appender.Access.File=${log_dir}/azkaban-access.log
+log4j.appender.Access.layout.ConversionPattern=%d{yyyy/MM/dd HH:mm:ss.SSS Z} %p [%c{1}] [Azkaban] %m%n
+log4j.appender.Access.DatePattern='.'yyyy-MM-dd
log4j.appender.WebServer=org.apache.log4j.DailyRollingFileAppender
log4j.appender.WebServer.layout=org.apache.log4j.PatternLayout
-log4j.appender.WebServer.File=azkaban-webserver.log
+log4j.appender.WebServer.File=${log_dir}/azkaban-webserver.log
log4j.appender.WebServer.layout.ConversionPattern=%d{yyyy/MM/dd HH:mm:ss.SSS Z} %p [%c{1}] [Azkaban] %m%n
-log4j.appender.WebServer.MaxBackupIndex=2
log4j.appender.WebServer.DatePattern='.'yyyy-MM-dd
log4j.appender.Console=org.apache.log4j.ConsoleAppender
