azkaban-developers

diff --git a/azkaban-common/src/main/bash/util.sh b/azkaban-common/src/main/bash/util.sh
new file mode 100755
index 0000000..aebfd3e
--- /dev/null
+++ b/azkaban-common/src/main/bash/util.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Common utils
+set -o nounset
+
+# kill the process with retry
+# return 0 if kill succeeds or no process to kill,
+#        1 if kill fails
+
+function kill_process_with_retry {
+   pid="$1"
+   pname="$2"
+   maxattempt="$3"
+
+   if [[ -z $pid ]]; then
+     echo "pid doesn't exist, shutdown completed"
+     return 0
+   fi
+
+   for try in $(seq 1 $maxattempt); do
+     if [[ ! -z $pid ]]; then
+      echo "Killing $pname. [pid: $pid], attempt: $try"
+      kill ${pid}
+      sleep 5
+      if [[ -n "$(ps -p $pid -o pid=)" ]]; then
+        echo "$pname is not dead [pid: $pid]"
+        if [[ $try -lt $maxattempt ]]; then
+          echo "sleeping for a few seconds before retry"
+          sleep 10
+        fi
+      else
+        echo "shutdown succeeded"
+        return 0
+      fi
+     fi
+    done
+
+   echo "Error: unable to kill process for $maxattempt attempt(s), shutdown failed"
+   return 1
+}
\ No newline at end of file

diff --git a/azkaban-exec-server/build.gradle b/azkaban-exec-server/build.gradle
index 68e39ec..50feba0 100644
--- a/azkaban-exec-server/build.gradle
+++ b/azkaban-exec-server/build.gradle
@@ -17,6 +17,10 @@ distributions {
         into 'bin'
         fileMode = 0755
       }
+      from('../azkaban-common/src/main/bash') {
+        into 'bin'
+        fileMode = 0755
+      }
       from(configurations.runtime) {
         into 'lib'
       }

diff --git a/azkaban-exec-server/src/main/bash/azkaban-executor-shutdown.sh b/azkaban-exec-server/src/main/bash/azkaban-executor-shutdown.sh
index bfbca0b..cf318ad 100755
--- a/azkaban-exec-server/src/main/bash/azkaban-executor-shutdown.sh
+++ b/azkaban-exec-server/src/main/bash/azkaban-executor-shutdown.sh
@@ -1,19 +1,19 @@
 #!/usr/bin/env bash
 # Shutdown script for azkaban executor server
-installdir="$(dirname $0)/.."
+set -o nounset
+source "$(dirname $0)/util.sh"
 
+installdir="$(dirname $0)/.."
+maxattempt=3
 pid=`cat ${installdir}/currentpid`
-port=`cat ${installdir}/executor.port`
-
-echo "Killing Executor. [pid: $pid, port: $port]"
-
-kill ${pid}
-if [ $? -ne 0 ]; then
-    echo "Error: Shutdown failed"
-    exit 1;
-fi
+pname="exec server"
 
-rm  ${installdir}/currentpid
-rm  ${installdir}/executor.port
+kill_process_with_retry "${pid}" "${pname}" "${maxattempt}"
 
-echo "done."
\ No newline at end of file
+if [[ $? == 0 ]]; then
+  rm -f ${installdir}/currentpid
+  rm -f ${installdir}/executor.port
+  exit 0
+else
+  exit 1
+fi
\ No newline at end of file

diff --git a/azkaban-web-server/build.gradle b/azkaban-web-server/build.gradle
index bce9503..113c35f 100644
--- a/azkaban-web-server/build.gradle
+++ b/azkaban-web-server/build.gradle
@@ -159,6 +159,10 @@ distributions {
         into 'bin'
         fileMode = 0755
       }
+      from('../azkaban-common/src/main/bash') {
+        into 'bin'
+        fileMode = 0755
+      }
       from(configurations.runtime) {
         into 'lib'
       }

diff --git a/azkaban-web-server/src/main/bash/azkaban-web-shutdown.sh b/azkaban-web-server/src/main/bash/azkaban-web-shutdown.sh
index cfd0d40..ba9c0e9 100755
--- a/azkaban-web-server/src/main/bash/azkaban-web-shutdown.sh
+++ b/azkaban-web-server/src/main/bash/azkaban-web-shutdown.sh
@@ -1,15 +1,18 @@
 #!/usr/bin/env bash
 # Shutdown script for azkaban web server
-installdir="$(dirname $0)/.."
+set -o nounset
+source "$(dirname $0)/util.sh"
 
+installdir="$(dirname $0)/.."
+maxattempt=3
 pid=`cat ${installdir}/currentpid`
-echo "Killing Web Server. [pid: $pid]"
+pname="web server"
 
-kill ${pid}
-if [ $? -ne 0 ]; then
-    echo "Error: Shutdown failed"
-    exit 1;
-fi
+kill_process_with_retry "${pid}" "${pname}" "${maxattempt}"
 
-rm  ${installdir}/currentpid
-echo "done."
+if [[ $? == 0 ]]; then
+  rm -f ${installdir}/currentpid
+  exit 0
+else
+  exit 1
+fi
\ No newline at end of file

diff --git a/azkaban-web-server/src/main/bash/start-web.sh b/azkaban-web-server/src/main/bash/start-web.sh
index 9812790..b02a5b5 100755
--- a/azkaban-web-server/src/main/bash/start-web.sh
+++ b/azkaban-web-server/src/main/bash/start-web.sh
@@ -2,4 +2,4 @@
 
 base_dir=$(dirname $0)/..
 
-bin/azkaban-web-start.sh $base_dir >logs/webServerLog_`date +%F+%T`.out 2>&1 &
+$base_dir/bin/azkaban-web-start.sh $base_dir >$base_dir/logs/webServerLog_`date +%F+%T`.out 2>&1 &