Prosoft Git

azkaban-uncached

generic triggers and some other fixes for 2.2 release

10/14/2013 7:08:20 PM
cjyu

cjyu

Commit: 1f37db1

Tree: 93aba2b

Parents: b639a5e

Changes

build.xml 42(+1 -41)

src/java/azkaban/jmx/JmxExecutorManagerAdapter.java 1(+1 -0)

src/java/azkaban/jmx/JmxExecutorManagerAdapterMBean.java 6(+6 -0)

src/java/azkaban/jobExecutor/NoopJob.java 2(+1 -1)

src/java/azkaban/trigger/builtin/ExecuteFlowAction.java 3(+2 -1)

src/java/azkaban/webapp/AzkabanWebServer.java 10(+5 -5)

src/java/azkaban/webapp/servlet/velocity/flowexecutionpanel.vm 6(+4 -2)

src/package/execserver/bin/azkaban-executor-start.sh 10(+6 -4)

src/package/execserver/bin/start-exec.sh 4(+4 -0)

src/package/webserver/bin/azkaban-web-start.sh 23(+19 -4)

src/package/webserver/bin/start-web.sh 3(+3 -0)

Details

build.xml 42(+1 -41) 

diff --git a/build.xml b/build.xml
index 1031487..86db40b 100644
--- a/build.xml
+++ b/build.xml
@@ -8,14 +8,12 @@
 	<property name="dist.packages.dir" value="${basedir}/dist/packages" />
 	<property name="dist.web.package.dir" value="${dist.packages.dir}/azkaban-web-server" />
 	<property name="dist.exec.package.dir" value="${dist.packages.dir}/azkaban-exec-server" />
-	<property name="dist.trigger.package.dir" value="${dist.packages.dir}/azkaban-trigger-server" />
 	<property name="dist.solo.package.dir" value="${dist.packages.dir}/azkaban-solo-server" />
 	<property name="dist.sql.package.dir" value="${dist.packages.dir}/sql" />	
 
 	<property name="conf.dir" value="${basedir}/conf" />
 	<property name="web.package.dir" value="${basedir}/src/package/webserver" />
 	<property name="exec.package.dir" value="${basedir}/src/package/execserver" />
-	<property name="trigger.package.dir" value="${basedir}/src/package/triggerserver" />
 	<property name="solo.package.dir" value="${basedir}/src/package/soloserver" />
 	
 	<property name="lib.dir" value="${basedir}/lib" />
@@ -210,44 +208,6 @@
 		</tar>
 	</target>
 	
-	<target name="package-trigger-server" depends="jars" description="Creates a package for the trigger server">
-		<delete dir="${dist.trigger.package.dir}" />
-		<mkdir dir="${dist.trigger.package.dir}" />
-		<mkdir dir="${dist.trigger.package.dir}/conf" />
-		<mkdir dir="${dist.trigger.package.dir}/bin" />
-		<mkdir dir="${dist.trigger.package.dir}/lib" />
-		<mkdir dir="${dist.trigger.package.dir}/plugins" />
-		<mkdir dir="${dist.trigger.package.dir}/extlib" />
-
-		<!-- Copy Azkaban jars and libs-->
-		<copy file="${azkaban.jar}" todir="${dist.trigger.package.dir}/lib" />
-		<copy todir="${dist.trigger.package.dir}/lib" >
-			<fileset dir="${lib.dir}" >
-				<exclude name="hadoop-core*.jar"/>
-			</fileset>
-		</copy>
-
-		<!-- Copy bin files for trigger server only-->
-		<copy todir="${dist.trigger.package.dir}/bin" >
-			<fileset dir="${trigger.package.dir}/bin"/>
-		</copy>
-
-		<!-- Copy conf files -->
-		<copy todir="${dist.trigger.package.dir}/conf" >
-			<fileset dir="${trigger.package.dir}/conf" />
-		</copy>
-
-		<!-- Tarball it -->
-		<tar destfile="${dist.trigger.package.dir}/${name}-trigger-server-${version}.tar.gz" compression="gzip" longfile="gnu">
-			<tarfileset dir="${dist.trigger.package.dir}" prefix="azkaban-${version}" filemode="755" includes="bin/*" />
-
-			<tarfileset dir="${dist.trigger.package.dir}" prefix="azkaban-${version}" includes="**">
-				<exclude name="bin/*"/>
-			</tarfileset>
-		</tar>
-	</target>
-
-
 	<target name="package-solo-server" depends="jars" description="Creates a package for the solo server">
 		<delete dir="${dist.solo.package.dir}" />
 		<mkdir dir="${dist.solo.package.dir}" />
@@ -297,6 +257,6 @@
 		</tar>
 	</target>
 	
-	<target name="package-all" depends="package-trigger-server, package-exec-server, package-web-server, package-solo-server, package-sql-scripts" description="Create all packages">
+	<target name="package-all" depends="package-exec-server, package-web-server, package-solo-server, package-sql-scripts" description="Create all packages">
 	</target>
 </project>

src/java/azkaban/jmx/JmxExecutorManagerAdapter.java 1(+1 -0) 

diff --git a/src/java/azkaban/jmx/JmxExecutorManagerAdapter.java b/src/java/azkaban/jmx/JmxExecutorManagerAdapter.java
index 1b63f26..fc0f8cf 100644
--- a/src/java/azkaban/jmx/JmxExecutorManagerAdapter.java
+++ b/src/java/azkaban/jmx/JmxExecutorManagerAdapter.java
@@ -43,4 +43,5 @@ public class JmxExecutorManagerAdapter implements JmxExecutorManagerAdapterMBean
 	public List<String> getPrimaryExecutorHostPorts() {
 		return new ArrayList<String>(manager.getPrimaryServerHosts());
 	}
+
 }

src/java/azkaban/jmx/JmxExecutorManagerAdapterMBean.java 6(+6 -0) 

diff --git a/src/java/azkaban/jmx/JmxExecutorManagerAdapterMBean.java b/src/java/azkaban/jmx/JmxExecutorManagerAdapterMBean.java
index b9742cb..197e721 100644
--- a/src/java/azkaban/jmx/JmxExecutorManagerAdapterMBean.java
+++ b/src/java/azkaban/jmx/JmxExecutorManagerAdapterMBean.java
@@ -17,4 +17,10 @@ public interface JmxExecutorManagerAdapterMBean {
 
 	@DisplayName("OPERATION: getPrimaryExecutorHostPorts")
 	public List<String> getPrimaryExecutorHostPorts();
+	
+//	@DisplayName("OPERATION: getExecutorThreadStage")
+//	public String getExecutorThreadStage();
+//	
+//	@DisplayName("OPERATION: getRunningFlows")
+//	public String getRunningFlows();
 }

src/java/azkaban/jobExecutor/NoopJob.java 2(+1 -1) 

diff --git a/src/java/azkaban/jobExecutor/NoopJob.java b/src/java/azkaban/jobExecutor/NoopJob.java
index 510c71b..a791845 100644
--- a/src/java/azkaban/jobExecutor/NoopJob.java
+++ b/src/java/azkaban/jobExecutor/NoopJob.java
@@ -25,7 +25,7 @@ import azkaban.utils.Props;
 public class NoopJob implements Job {
 	private String jobId;
 
-	public NoopJob(String jobid, Props props, Logger log) {
+	public NoopJob(String jobid, Props props, Props jobProps, Logger log) {
 		this.jobId = jobid;
 	}

src/java/azkaban/trigger/builtin/ExecuteFlowAction.java 3(+2 -1) 

diff --git a/src/java/azkaban/trigger/builtin/ExecuteFlowAction.java b/src/java/azkaban/trigger/builtin/ExecuteFlowAction.java
index f19adfe..d17aa9d 100644
--- a/src/java/azkaban/trigger/builtin/ExecuteFlowAction.java
+++ b/src/java/azkaban/trigger/builtin/ExecuteFlowAction.java
@@ -227,7 +227,7 @@ public class ExecuteFlowAction implements TriggerAction {
 		if(slaOptions != null && slaOptions.size() > 0) {
 			int execId = exflow.getExecutionId();
 			for(SlaOption sla : slaOptions) {
-				logger.info("Adding sla trigger " + sla.toString());
+				logger.info("Adding sla trigger " + sla.toString() + " to execution " + execId);
 				SlaChecker slaChecker = new SlaChecker("slaChecker", sla, execId);
 				Map<String, ConditionChecker> slaCheckers = new HashMap<String, ConditionChecker>();
 				slaCheckers.put(slaChecker.getId(), slaChecker);
@@ -251,6 +251,7 @@ public class ExecuteFlowAction implements TriggerAction {
 				Trigger slaTrigger = new Trigger("azkaban_sla", "azkaban", triggerCond, expireCond, actions);
 				slaTrigger.setResetOnTrigger(false);
 				slaTrigger.setResetOnExpire(false);
+				logger.info("Ready to put in the sla trigger");
 				triggerManager.insertTrigger(slaTrigger);
 			}
 		}

src/java/azkaban/webapp/AzkabanWebServer.java 10(+5 -5) 

diff --git a/src/java/azkaban/webapp/AzkabanWebServer.java b/src/java/azkaban/webapp/AzkabanWebServer.java
index d9d8ed0..699936d 100644
--- a/src/java/azkaban/webapp/AzkabanWebServer.java
+++ b/src/java/azkaban/webapp/AzkabanWebServer.java
@@ -56,7 +56,7 @@ import azkaban.database.AzkabanDatabaseSetup;
 import azkaban.executor.ExecutorManager;
 import azkaban.executor.ExecutorManagerAdapter;
 import azkaban.executor.JdbcExecutorLoader;
-import azkaban.jmx.JmxExecutorManagerAdapter;
+import azkaban.jmx.JmxExecutorManager;
 import azkaban.jmx.JmxJettyServer;
 import azkaban.jmx.JmxTriggerManager;
 import azkaban.project.JdbcProjectLoader;
@@ -1174,11 +1174,11 @@ public class AzkabanWebServer extends AzkabanServer {
 
 		registerMbean("jetty", new JmxJettyServer(server));
 		registerMbean("triggerManager", new JmxTriggerManager(triggerManager));
-//		if(executorManager instanceof ExecutorManagerLocalAdapter) {
-//			registerMbean("executorManager", new JmxExecutorManager(((ExecutorManagerLocalAdapter)executorManager).getExecutorManager()));
-//		}
+		if(executorManager instanceof ExecutorManager) {
+			registerMbean("executorManager", new JmxExecutorManager((ExecutorManager) executorManager));
+		}
+//		registerMbean("executorManager", new JmxExecutorManager(executorManager));
 //		registerMbean("executorManager", new JmxExecutorManager(executorManager));
-		registerMbean("executorManager", new JmxExecutorManagerAdapter(executorManager));
 	}
 	
 	public void close() {

src/java/azkaban/webapp/servlet/velocity/flowexecutionpanel.vm 6(+4 -2) 

diff --git a/src/java/azkaban/webapp/servlet/velocity/flowexecutionpanel.vm b/src/java/azkaban/webapp/servlet/velocity/flowexecutionpanel.vm
index f0cf6ac..b7f45fd 100644
--- a/src/java/azkaban/webapp/servlet/velocity/flowexecutionpanel.vm
+++ b/src/java/azkaban/webapp/servlet/velocity/flowexecutionpanel.vm
@@ -136,12 +136,13 @@
 		<a class="btn2" id="schedule-btn">Schedule</a>
 		#end
 		
+#*
 		#if( $triggerPlugins.size() > 0 ) 
 			#foreach( $triggerPlugin in $triggerPlugins )
 			<a class="btn2" id=set-$triggerPlugin.pluginName>$triggerPlugin.pluginName</a>
 			#end
 		#end
-
+*#
 		<a class="yes btn1" id="execute-btn">Execute</a>
 		<a class="no simplemodal-close btn3 closeExecPanel">Cancel</a>
 	</div>
@@ -152,6 +153,7 @@
 #parse( "azkaban/webapp/servlet/velocity/schedulepanel.vm" )
 #end
 
+#*
 #if( $triggerPlugins.size() > 0 ) 
 	#foreach( $triggerPlugin in $triggerPlugins )
 		#set ($prefix = $triggerPlugin.pluginName )
@@ -159,7 +161,7 @@
 		#parse( $triggerPlugin.inputPanelVM )
 	#end
 #end
-
+*#
 
 <div id="contextMenu">

src/package/execserver/bin/azkaban-executor-start.sh 10(+6 -4) 

diff --git a/src/package/execserver/bin/azkaban-executor-start.sh b/src/package/execserver/bin/azkaban-executor-start.sh
index 80261d7..f6313f7 100755
--- a/src/package/execserver/bin/azkaban-executor-start.sh
+++ b/src/package/execserver/bin/azkaban-executor-start.sh
@@ -1,7 +1,7 @@
 azkaban_dir=$(dirname $0)/..
 
 if [[ -z "$tmpdir" ]]; then
-tmpdir=temp
+tmpdir=/tmp
 fi
 
 for file in $azkaban_dir/lib/*.jar;
@@ -20,10 +20,12 @@ do
 done
 
 if [ "HADOOP_HOME" != "" ]; then
-	for file in $HADOOP_HOME/hadoop-core*.jar	do
+	for file in $HADOOP_HOME/hadoop-core*.jar ;
+	do
 		CLASSPATH=$CLASSPATH:$file
 	done
 	CLASSPATH=$CLASSPATH:$HADOOP_HOME/conf
+    JAVA_LIB_PATH="-Djava.library.path=$HADOOP_HOME/lib/native/Linux-amd64-64"
 else
 	echo "Error: HADOOP_HOME is not set. Hadoop job types will not run properly."
 fi
@@ -40,11 +42,11 @@ echo "Starting AzkabanExecutorServer on port $executorport ..."
 serverpath=`pwd`
 
 if [ -z $AZKABAN_OPTS ]; then
-  AZKABAN_OPTS=-Xmx3G
+  AZKABAN_OPTS="-Xmx3G"
 fi
 AZKABAN_OPTS="$AZKABAN_OPTS -server -Dcom.sun.management.jmxremote -Djava.io.tmpdir=$tmpdir -Dexecutorport=$executorport -Dserverpath=$serverpath"
 
-java $AZKABAN_OPTS -cp $CLASSPATH azkaban.execapp.AzkabanExecutorServer -conf $azkaban_dir/conf $@ &
+java $AZKABAN_OPTS $JAVA_LIB_PATH -cp $CLASSPATH azkaban.execapp.AzkabanExecutorServer -conf $azkaban_dir/conf $@ &
 
 echo $! > currentpid

src/package/execserver/bin/start-exec.sh 4(+4 -0) 

diff --git a/src/package/execserver/bin/start-exec.sh b/src/package/execserver/bin/start-exec.sh
new file mode 100644
index 0000000..f44b2ec
--- /dev/null
+++ b/src/package/execserver/bin/start-exec.sh
@@ -0,0 +1,4 @@
+base_dir=$(dirname $0)/..
+
+bin/azkaban-executor-start.sh $base_dir 2>&1>logs/executorServerLog__`date +%F+%T`.out &
+

src/package/webserver/bin/azkaban-web-start.sh 23(+19 -4) 

diff --git a/src/package/webserver/bin/azkaban-web-start.sh b/src/package/webserver/bin/azkaban-web-start.sh
index bdd19e0..fd16e47 100755
--- a/src/package/webserver/bin/azkaban-web-start.sh
+++ b/src/package/webserver/bin/azkaban-web-start.sh
@@ -1,7 +1,7 @@
 azkaban_dir=$(dirname $0)/..
 
 if [[ -z "$tmpdir" ]]; then
-tmpdir=temp
+tmpdir=/tmp
 fi
 
 for file in $azkaban_dir/lib/*.jar;
@@ -19,6 +19,21 @@ do
   CLASSPATH=$CLASSPATH:$file
 done
 
+if [ "HADOOP_HOME" != "" ]; then
+    for file in $HADOOP_HOME/hadoop-core*.jar;
+    do
+        CLASSPATH=$CLASSPATH:$file
+    done
+    CLASSPATH=$CLASSPATH:$HADOOP_HOME/conf
+    JAVA_LIB_PATH="-Djava.library.path=$HADOOP_HOME/lib/native/Linux-amd64-64"
+else
+    echo "Error: HADOOP_HOME is not set. Hadoop job types will not run properly."
+fi
+
+if [ "HIVE_HOME" != "" ]; then
+    CLASSPATH=$CLASSPATH:$HIVE_HOME/conf
+fi
+
 echo $azkaban_dir;
 echo $CLASSPATH;
 
@@ -26,11 +41,11 @@ executorport=`cat $azkaban_dir/conf/azkaban.properties | grep executor.port | cu
 serverpath=`pwd`
 
 if [ -z $AZKABAN_OPTS ]; then
-  AZKABAN_OPTS=-Xmx3G
+  AZKABAN_OPTS="-Xmx4G"
 fi
-AZKABAN_OPTS=$AZKABAN_OPTS -server -Dcom.sun.management.jmxremote -Djava.io.tmpdir=$tmpdir -Dexecutorport=$executorport -Dserverpath=$serverpath
+AZKABAN_OPTS="$AZKABAN_OPTS -server -Dcom.sun.management.jmxremote -Djava.io.tmpdir=$tmpdir -Dexecutorport=$executorport -Dserverpath=$serverpath"
 
-java $AZKABAN_OPTS -cp $CLASSPATH azkaban.webapp.AzkabanWebServer -conf $azkaban_dir/conf $@ &
+java $AZKABAN_OPTS $JAVA_LIB_PATH -cp $CLASSPATH azkaban.webapp.AzkabanWebServer -conf $azkaban_dir/conf $@ &
 
 echo $! > currentpid

src/package/webserver/bin/start-web.sh 3(+3 -0) 

diff --git a/src/package/webserver/bin/start-web.sh b/src/package/webserver/bin/start-web.sh
new file mode 100644
index 0000000..30d43fa
--- /dev/null
+++ b/src/package/webserver/bin/start-web.sh
@@ -0,0 +1,3 @@
+base_dir=$(dirname $0)/..
+
+bin/azkaban-web-start.sh $base_dir 2>&1>logs/webServerLog_`date +%F+%T`.out &

Bonobo Git Server (6.3.0.632) © 2015 Jakub Chodounský · Official Page · Github