azkaban-uncached
Changes
src/java/azkaban/utils/PropsUtils.java 89(+56 -33)
unit/java/azkaban/test/utils/PropsUtilsTest.java 101(+101 -0)
Details
src/java/azkaban/utils/PropsUtils.java 89(+56 -33)
diff --git a/src/java/azkaban/utils/PropsUtils.java b/src/java/azkaban/utils/PropsUtils.java
index e3c9c2d..16afb52 100644
--- a/src/java/azkaban/utils/PropsUtils.java
+++ b/src/java/azkaban/utils/PropsUtils.java
@@ -20,9 +20,11 @@ import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.StringTokenizer;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -30,6 +32,7 @@ import java.util.regex.Pattern;
import azkaban.executor.ExecutableFlow;
import azkaban.flow.CommonJobProperties;
+import org.apache.commons.lang.StringUtils;
import org.joda.time.DateTime;
public class PropsUtils {
@@ -146,47 +149,67 @@ public class PropsUtils {
.compile("\\$\\{([a-zA-Z_.0-9]+)\\}");
public static Props resolveProps(Props props) {
- if(props == null) return null;
+ if (props == null) return null;
Props resolvedProps = new Props();
-
+
+ LinkedHashSet<String> visitedVariables = new LinkedHashSet<String>();
for (String key : props.getKeySet()) {
- StringBuffer replaced = new StringBuffer();
String value = props.get(key);
- Matcher matcher = VARIABLE_PATTERN.matcher(value);
- while (matcher.find()) {
- String variableName = matcher.group(1);
-
- if (variableName.equals(key)) {
- throw new IllegalArgumentException(
- String.format(
- "Circular property definition starting from property[%s]",
- key));
- }
-
- String replacement = props.get(variableName);
- if (replacement == null)
+
+ visitedVariables.add(key);
+ String replacedValue = resolveVariableReplacement(value, props, visitedVariables);
+ visitedVariables.clear();
+
+ resolvedProps.put(key, replacedValue);
+ }
+
+ return resolvedProps;
+ };
+
+ private static String resolveVariableReplacement(String value, Props props, LinkedHashSet<String> visitedVariables) {
+ StringBuffer buffer = new StringBuffer();
+ int startIndex = 0;
+
+ Matcher matcher = VARIABLE_PATTERN.matcher(value);
+ while (matcher.find(startIndex)) {
+ if (startIndex < matcher.start()) {
+ // Copy everything up front to the buffer
+ buffer.append(value.substring(startIndex, matcher.start()));
+ }
+
+ String subVariable = matcher.group(1);
+ // Detected a cycle
+ if (visitedVariables.contains(subVariable)) {
+ throw new IllegalArgumentException(
+ String.format("Circular variable substitution found: [%s] -> [%s]",
+ StringUtils.join(visitedVariables, "->"), subVariable));
+ }
+ else {
+ // Add substitute variable and recurse.
+ String replacement = props.get(subVariable);
+ visitedVariables.add(subVariable);
+
+ if (replacement == null) {
throw new UndefinedPropertyException(
- "Could not find variable substitution for variable '"
- + variableName + "' in key '" + key + "'.");
-
- replacement = replacement.replaceAll("\\\\", "\\\\\\\\");
- replacement = replacement.replaceAll("\\$", "\\\\\\$");
-
- matcher.appendReplacement(replaced, replacement);
- matcher.appendTail(replaced);
-
- value = replaced.toString();
- replaced = new StringBuffer();
- matcher = VARIABLE_PATTERN.matcher(value);
+ String.format("Could not find variable substitution for variable(s) [%s]",
+ StringUtils.join(visitedVariables, "->")));
+ }
+
+ buffer.append(resolveVariableReplacement(replacement, props, visitedVariables));
+ visitedVariables.remove(subVariable);
}
- matcher.appendTail(replaced);
- resolvedProps.put(key, replaced.toString());
+
+ startIndex = matcher.end();
}
-
- return resolvedProps;
+
+ if (startIndex < value.length()) {
+ buffer.append(value.substring(startIndex));
+ }
+
+ return buffer.toString();
}
-
+
public static Props addCommonFlowProperties(final ExecutableFlow flow) {
Props parentProps = new Props();
diff --git a/src/java/azkaban/utils/StringUtils.java b/src/java/azkaban/utils/StringUtils.java
index 90fd688..6684b00 100644
--- a/src/java/azkaban/utils/StringUtils.java
+++ b/src/java/azkaban/utils/StringUtils.java
@@ -15,6 +15,7 @@
*/
package azkaban.utils;
+import java.util.Collection;
import java.util.List;
public class StringUtils {
@@ -45,7 +46,7 @@ public class StringUtils {
* @param delimiter
* @return
*/
- public static String join(List<String> list, String delimiter) {
+ public static String join(Collection<String> list, String delimiter) {
StringBuffer buffer = new StringBuffer();
for (String str: list) {
buffer.append(str);
@@ -54,4 +55,5 @@ public class StringUtils {
return buffer.toString();
}
+
}
diff --git a/src/java/azkaban/webapp/servlet/ScheduleServlet.java b/src/java/azkaban/webapp/servlet/ScheduleServlet.java
index aab3bde..4109d63 100644
--- a/src/java/azkaban/webapp/servlet/ScheduleServlet.java
+++ b/src/java/azkaban/webapp/servlet/ScheduleServlet.java
@@ -644,7 +644,7 @@ public class ScheduleServlet extends LoginAbstractAzkabanServlet {
int minutes = Integer.parseInt(parts[1]);
boolean isPm = parts[2].equalsIgnoreCase("pm");
- DateTimeZone timezone = parts[3].equals("UTC") ? DateTimeZone.UTC : DateTimeZone.forID("America/Los_Angeles");
+ DateTimeZone timezone = parts[3].equals("UTC") ? DateTimeZone.UTC : DateTimeZone.getDefault();
// scheduleDate: 02/10/2013
DateTime day = null;
diff --git a/src/java/azkaban/webapp/session/SessionCache.java b/src/java/azkaban/webapp/session/SessionCache.java
index 0058bf1..514606a 100644
--- a/src/java/azkaban/webapp/session/SessionCache.java
+++ b/src/java/azkaban/webapp/session/SessionCache.java
@@ -82,4 +82,4 @@ public class SessionCache {
public boolean removeSession(String id) {
return cache.remove(id);
}
-}
\ No newline at end of file
+}
diff --git a/src/package/execserver/bin/azkaban-executor-start.sh b/src/package/execserver/bin/azkaban-executor-start.sh
index 29f4241..912eaa0 100755
--- a/src/package/execserver/bin/azkaban-executor-start.sh
+++ b/src/package/execserver/bin/azkaban-executor-start.sh
@@ -29,9 +29,9 @@ serverpath=`pwd`
if [ -z $AZKABAN_OPTS ]; then
AZKABAN_OPTS=-Xmx3G
fi
-AZKABAN_OPTS=$AZKABAN_OPTS -server -Dcom.sun.management.jmxremote -Djava.io.tmpdir=$tmpdir -Dexecutorport=$executorport -Dserverpath=$serverpath
+AZKABAN_OPTS="$AZKABAN_OPTS -server -Dcom.sun.management.jmxremote -Djava.io.tmpdir=$tmpdir -Dexecutorport=$executorport -Dserverpath=$serverpath"
-java $AZKABAN_OPTS -cp $CLASSPATH azkaban.webapp.AzkabanExecutorServer -conf $azkaban_dir/conf $@ &
+java $AZKABAN_OPTS -cp $CLASSPATH azkaban.execapp.AzkabanExecutorServer -conf $azkaban_dir/conf $@ &
echo $! > currentpid
unit/java/azkaban/test/utils/PropsUtilsTest.java 101(+101 -0)
diff --git a/unit/java/azkaban/test/utils/PropsUtilsTest.java b/unit/java/azkaban/test/utils/PropsUtilsTest.java
new file mode 100644
index 0000000..e2c914f
--- /dev/null
+++ b/unit/java/azkaban/test/utils/PropsUtilsTest.java
@@ -0,0 +1,101 @@
+package azkaban.test.utils;
+
+import java.io.IOException;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import azkaban.utils.Props;
+import azkaban.utils.PropsUtils;
+import azkaban.utils.UndefinedPropertyException;
+
+public class PropsUtilsTest {
+ @Test
+ public void testGoodResolveProps() throws IOException {
+ Props propsGrandParent = new Props();
+ Props propsParent = new Props(propsGrandParent);
+ Props props = new Props(propsParent);
+
+ // Testing props in general
+ props.put("letter", "a");
+ propsParent.put("letter", "b");
+ propsGrandParent.put("letter", "c");
+
+ Assert.assertEquals("a", props.get("letter"));
+ propsParent.put("my", "name");
+ propsParent.put("your", "eyes");
+ propsGrandParent.put("their", "ears");
+ propsGrandParent.put("your", "hair");
+
+ Assert.assertEquals("name", props.get("my"));
+ Assert.assertEquals("eyes", props.get("your"));
+ Assert.assertEquals("ears", props.get("their"));
+
+ props.put("res1", "${my}");
+ props.put("res2", "${their} ${letter}");
+ props.put("res7", "${my} ${res5}");
+
+ propsParent.put("res3", "${your} ${their} ${res4}");
+ propsGrandParent.put("res4", "${letter}");
+ propsGrandParent.put("res5", "${their}");
+ propsParent.put("res6", " t ${your} ${your} ${their} ${res5}");
+
+ Props resolved = PropsUtils.resolveProps(props);
+ Assert.assertEquals("name", resolved.get("res1"));
+ Assert.assertEquals("ears a", resolved.get("res2"));
+ Assert.assertEquals("eyes ears a", resolved.get("res3"));
+ Assert.assertEquals("a", resolved.get("res4"));
+ Assert.assertEquals("ears", resolved.get("res5"));
+ Assert.assertEquals(" t eyes eyes ears ears", resolved.get("res6"));
+ Assert.assertEquals("name ears", resolved.get("res7"));
+ }
+
+ @Test
+ public void testCyclesResolveProps() throws IOException {
+ Props propsGrandParent = new Props();
+ Props propsParent = new Props(propsGrandParent);
+ Props props = new Props(propsParent);
+
+ // Testing props in general
+ props.put("a", "${a}");
+ failIfNotException(props);
+
+ props.put("a", "${b}");
+ props.put("b", "${a}");
+ failIfNotException(props);
+
+ props.clearLocal();
+ props.put("a", "${b}");
+ props.put("b", "${c}");
+ propsParent.put("d", "${a}");
+ failIfNotException(props);
+
+ props.clearLocal();
+ props.put("a", "testing ${b}");
+ props.put("b", "${c}");
+ propsGrandParent.put("c", "${d}");
+ propsParent.put("d", "${a}");
+ failIfNotException(props);
+
+ props.clearLocal();
+ props.put("a", "testing ${c} ${b}");
+ props.put("b", "${c} test");
+ propsGrandParent.put("c", "${d}");
+ propsParent.put("d", "${a}");
+ failIfNotException(props);
+ }
+
+ private void failIfNotException(Props props) {
+ try {
+ Props resolved = PropsUtils.resolveProps(props);
+ Assert.fail();
+ }
+ catch (UndefinedPropertyException e) {
+ e.printStackTrace();
+ }
+ catch (IllegalArgumentException e) {
+ e.printStackTrace();
+ }
+ }
+}