azkaban-aplcache

Saving and loading of flow files.

6/27/2012 9:19:39 PM

Details

diff --git a/src/java/azkaban/flow/ErrorEdge.java b/src/java/azkaban/flow/ErrorEdge.java
index 136e970..4a27616 100644
--- a/src/java/azkaban/flow/ErrorEdge.java
+++ b/src/java/azkaban/flow/ErrorEdge.java
@@ -27,6 +27,13 @@ public class ErrorEdge extends Edge {
 		this.error = error;
 	}
 	
+	public ErrorEdge(String source, String target, String error) {
+		super(null, null);
+		this.targetId = target;
+		this.sourceId = source;
+		this.error = error;
+	}
+	
 	@Override
 	public String getSourceId() {
 		return sourceId;
diff --git a/src/java/azkaban/flow/Flow.java b/src/java/azkaban/flow/Flow.java
index ab104c0..b0ba280 100644
--- a/src/java/azkaban/flow/Flow.java
+++ b/src/java/azkaban/flow/Flow.java
@@ -8,6 +8,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import azkaban.project.ProjectManager;
+import azkaban.project.ResourceLoader;
 import azkaban.utils.Props;
 
 public class Flow {
@@ -20,7 +22,8 @@ public class Flow {
     private HashMap<String, Edge> edges = new HashMap<String, Edge>();
     private HashMap<String, Set<Edge>> sourceEdges = new HashMap<String, Set<Edge>>();
     private HashMap<String, Set<Edge>> targetEdges = new HashMap<String, Set<Edge>>();
-    private ArrayList<Object> errors;
+    private HashMap<String, Props> flowProps = new HashMap<String, Props>(); 
+    private ArrayList<String> errors;
 
     public Flow(String id) {
         this.id = id;
@@ -32,7 +35,7 @@ public class Flow {
     
     public void addAllNodes(Collection<Node> nodes) {
         for (Node node: nodes) {
-            this.nodes.put(node.getId(), node);
+        	addNode(node);
         }
     }
     
@@ -40,19 +43,29 @@ public class Flow {
         nodes.put(node.getId(), node);
     }
 
+    public void addProperties(Props props) {
+    	flowProps.put(props.getSource(), props);
+    }
+    
+    public void addAllProperties(Collection<Props> props) {
+    	for (Props prop: props) {
+    		flowProps.put(prop.getSource(), prop);
+    	}
+    }
+    
     public String getId() {
         return id;
     }
     
-    public void addError(Object error) {
+    public void addError(String error) {
         if (errors == null) {
-            errors = new ArrayList<Object>();
+            errors = new ArrayList<String>();
         }
   
         errors.add(error);
     }
     
-    public List<Object> getErrors() {
+    public List<String> getErrors() {
     	return errors;
     }
     
@@ -68,12 +81,18 @@ public class Flow {
     	return edges.values();
     }
     
+    public void addAllEdges(Collection<Edge> edges) {
+    	for (Edge edge: edges) {
+    		addEdge(edge);
+    	}
+    }
+    
     public void addEdge(Edge edge) {
     	String source = edge.getSourceId();
     	String target = edge.getTargetId();
 
     	if (edge instanceof ErrorEdge) {
-    		addError(edge);
+    		addError("Error on " + edge.getId() + ". " + ((ErrorEdge)edge).getError());
     	}
 
     	Set<Edge> sourceSet = getEdgeSet(sourceEdges, source);
@@ -99,13 +118,135 @@ public class Flow {
 		HashMap<String, Object> flowObj = new HashMap<String, Object>();
 		flowObj.put("type", "flow");
 		flowObj.put("id", getId());
-		flowObj.put("properties", objectizeProperties());
+		flowObj.put("props", objectizeProperties());
 		flowObj.put("nodes", objectizeNodes());
 		flowObj.put("edges", objectizeEdges());
+		if (errors != null) {
+			flowObj.put("errors", errors);
+		}
 		
 		return flowObj;
     }
     
+    @SuppressWarnings("unchecked")
+	public static Flow flowFromObject(Object object, ResourceLoader loader) {
+    	Map<String, Object> flowObject = (Map<String,Object>)object;
+    	
+    	String id = (String)flowObject.get("id");
+    	Flow flow = new Flow(id);
+    	
+    	// Loading projects
+    	List<Object> propertiesList = (List<Object>)flowObject.get("props");
+    	Map<String, Props> properties = loadPropertiesFromObject(propertiesList, loader);
+    	flow.addAllProperties(properties.values());
+    	
+    	// Loading nodes
+    	List<Object> nodeList = (List<Object>)flowObject.get("nodes");
+    	Map<String, Node> nodes = loadNodesFromObjects(nodeList, properties, loader);
+    	flow.addAllNodes(nodes.values());
+    	
+    	// Loading edges
+    	List<Object> edgeList = (List<Object>)flowObject.get("edges");
+    	List<Edge> edges = loadEdgeFromObjects(edgeList, nodes, loader);
+    	flow.addAllEdges(edges);
+    	
+    	return flow;
+    }
+    
+    private static Map<String, Node> loadNodesFromObjects(List<Object> nodeList, Map<String, Props> properties, ResourceLoader loader) {
+    	Map<String, Node> nodeMap = new HashMap<String, Node>();
+    	
+    	for (Object obj: nodeList) {
+    		@SuppressWarnings("unchecked")
+			Map<String,Object> nodeObj = (Map<String,Object>)obj;
+    		String id = (String)nodeObj.get("id");
+    		String propsSource = (String)nodeObj.get("props.source");
+    		String inheritedSource = (String)nodeObj.get("inherited.source");
+
+    		Props inheritedProps = properties.get(inheritedSource);
+    		Props props = loader.loadPropsFromSource(inheritedProps, propsSource);
+    		
+    		Node node = new Node(id, props);
+    		nodeMap.put(id, node);
+    	}
+    	
+    	return nodeMap;
+    }
+    
+    private static List<Edge> loadEdgeFromObjects(List<Object> edgeList, Map<String, Node> nodes, ResourceLoader loader) {
+    	List<Edge> edgeResult = new ArrayList<Edge>();
+    	
+    	for (Object obj: edgeList) {
+    		@SuppressWarnings("unchecked")
+			Map<String,Object> edgeObj = (Map<String,Object>)obj;
+    		String id = (String)edgeObj.get("id");
+    		String source = (String)edgeObj.get("source");
+    		String target = (String)edgeObj.get("target");
+    		
+    		Node sourceNode = nodes.get(source);
+    		Node targetNode = nodes.get(target);
+    		String error = (String)edgeObj.get("error");
+    		
+    		Edge edge = null;
+    		if (sourceNode == null && targetNode != null) {
+    			edge = new ErrorEdge(source, target, "Edge Error: Neither source " + source + " nor " + target + " could be found.");
+    		}
+    		else if (sourceNode == null && targetNode != null) {
+    			edge = new ErrorEdge(source, target, "Edge Error: Source " + source + " could not be found. Target: " + target);
+    		}
+    		else if (sourceNode != null && targetNode == null) {
+    			edge = new ErrorEdge(source, target, "Edge Error: Source found " + source + ", but " + target + " could be found.");
+    		}
+    		else if (error != null) {
+    			edge = new ErrorEdge(source, target, error);
+    		}
+    		else {
+    			edge = new Edge(sourceNode, targetNode);
+    		}
+    		
+    		edgeResult.add(edge);
+    	}
+    	
+    	return edgeResult;    
+    }
+    
+    @SuppressWarnings("unchecked")
+	private static Map<String, Props> loadPropertiesFromObject(List<Object> propertyObjectList, ResourceLoader loader) {
+    	Map<String, Props> properties = new HashMap<String, Props>();
+    	
+    	Map<String, String> sourceToInherit = new HashMap<String,String>();
+    	for (Object propObj: propertyObjectList) {
+    		Map<String, Object> mapObj = (Map<String,Object>)propObj;
+    		String source = (String)mapObj.get("source");
+    		String inherits = (String)mapObj.get("inherits");
+    		
+    		sourceToInherit.put(source, inherits);
+    	}
+    	
+    	for (String source: sourceToInherit.keySet()) {
+    		recursiveResolveProps(source, sourceToInherit, loader, properties);
+    	}
+    	
+    	return properties;
+    }
+    
+    private static void recursiveResolveProps(String source, Map<String, String> sourceToInherit, ResourceLoader loader, Map<String, Props> properties) {
+    	Props prop = properties.get(source);
+    	if (prop != null) {
+    		return;
+    	}
+    	
+    	String inherits = sourceToInherit.get(source);
+    	Props parent = null;
+    	if (inherits != null) {
+    		recursiveResolveProps(inherits, sourceToInherit, loader, properties);
+        	parent = properties.get(inherits);
+    	}
+
+    	prop = loader.loadPropsFromSource(parent, source);
+    	properties.put(source, prop);
+    }
+    
 	private List<Map<String,Object>> objectizeNodes() {
 		ArrayList<Map<String,Object>> result = new ArrayList<Map<String,Object>>();
 		for (Node node : getNodes()) {
@@ -140,39 +281,19 @@ public class Flow {
 		return result;
 	}
 	
-	@SuppressWarnings("unchecked")
 	private List<Map<String,Object>> objectizeProperties() {
-		ArrayList<Map<String,Object>> result = new ArrayList<Map<String,Object>>();
 		
-		HashMap<String, Object> properties = new HashMap<String, Object>();
-		for (Node node: getNodes()) {
-			Props props = node.getProps().getParent();
-			if (props != null) {
-				traverseAndObjectizeProperties(properties, props);
+		ArrayList<Map<String,Object>> result = new ArrayList<Map<String,Object>>();
+		for (Props props: flowProps.values()) {
+			HashMap<String, Object> propObj = new HashMap<String, Object>();
+			propObj.put("source", props.getSource());
+			Props parent = props.getParent();
+			if (parent != null) {
+				propObj.put("inherits", parent.getSource());
 			}
-		}
-		
-		for (Object propMap : properties.values()) {
-			result.add((Map<String,Object>)propMap);
+			result.add(propObj);
 		}
 		
 		return result;
 	}
-	
-	private void traverseAndObjectizeProperties(HashMap<String, Object> properties, Props props) {
-		if (props.getSource() == null || properties.containsKey(props.getSource())) {
-			return;
-		}
-		
-		HashMap<String, Object> propObj = new HashMap<String,Object>();
-		propObj.put("source", props.getSource());
-		properties.put(props.getSource(), propObj);
-		
-		Props parent = props.getParent();
-		if (parent != null) {
-			propObj.put("inherits", parent.getSource());
-			
-			traverseAndObjectizeProperties(properties, parent);
-		}
-	}
 }
\ No newline at end of file
diff --git a/src/java/azkaban/flow/Node.java b/src/java/azkaban/flow/Node.java
index 68012f6..ce2fae7 100644
--- a/src/java/azkaban/flow/Node.java
+++ b/src/java/azkaban/flow/Node.java
@@ -12,7 +12,7 @@ public class Node {
     private State state = State.WAITING;
 
     private Props props;
-    
+
     public Node(String id, Props props) {
         this.id = id;
         this.props = props;
@@ -39,7 +39,7 @@ public class Node {
     public void setState(State state) {
         this.state = state;
     }
-
+    
     public Props getProps() {
         return props;
     }
diff --git a/src/java/azkaban/project/FileProjectManager.java b/src/java/azkaban/project/FileProjectManager.java
index 0ad73bf..98c6ecb 100644
--- a/src/java/azkaban/project/FileProjectManager.java
+++ b/src/java/azkaban/project/FileProjectManager.java
@@ -1,11 +1,13 @@
 package azkaban.project;
 
 import java.io.File;
+import java.io.FileFilter;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.security.AccessControlException;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -14,7 +16,6 @@ import org.apache.log4j.Logger;
 import org.joda.time.format.DateTimeFormat;
 import org.joda.time.format.DateTimeFormatter;
 
-import azkaban.flow.ErrorEdge;
 import azkaban.flow.Flow;
 import azkaban.user.Permission;
 import azkaban.user.Permission.Type;
@@ -36,6 +37,10 @@ public class FileProjectManager implements ProjectManager {
 	private static final String FLOW_EXTENSION = ".flow";
     private static final Logger logger = Logger.getLogger(FileProjectManager.class);
     private ConcurrentHashMap<String, Project> projects = new ConcurrentHashMap<String, Project>();
+    
+    // We store the flows for projects in the ProjectManager instead of the Project so we can employ different 
+    // loading/caching techniques.
+    private HashMap<String, Map<String, Flow>> projectFlows = new HashMap<String, Map<String, Flow>>();
 
 	private File projectDirectory;
 	
@@ -86,6 +91,41 @@ public class FileProjectManager implements ProjectManager {
     				Project project = Project.projectFromObject(obj);
     				logger.info("Loading project " + project.getName());
     				projects.put(project.getName(), project);
+    			
+    				String source = project.getSource();
+    				if (source == null) {
+    					logger.info(project.getName() + ": No flows uploaded");
+    					return;
+    				}
+    				
+    				File projectDir = new File(dir, source);
+    				if (!projectDir.exists()) {
+    					logger.error("ERROR project source dir " + projectDir + " doesn't exist.");
+    				}
+    				else if (!projectDir.isDirectory()) {
+    					logger.error("ERROR project source dir " + projectDir + " is not a directory.");
+    				}
+    				else {
+    					File projectSourceDir = new File(projectDir, PROJECT_DIRECTORY);
+    					FileResourceLoader loader = new FileResourceLoader(projectSourceDir);
+    					File[] flowFiles = projectDir.listFiles(new SuffixFilter(FLOW_EXTENSION));
+    					Map<String, Flow> flowMap = new LinkedHashMap<String, Flow>();
+    					for (File flowFile: flowFiles) {
+							Object objectizedFlow = null;
+    						try {
+    							objectizedFlow = JSONUtils.parseJSONFromFile(flowFile);
+							} catch (IOException e) {
+								logger.error("Error parsing flow file " + flowFile.toString());
+							}
+    						
+    						//Recreate Flow
+    						Flow flow = Flow.flowFromObject(objectizedFlow, loader);
+    						logger.debug("Loaded flow " + project.getName() + ": " + flow.getId());
+    						flowMap.put(flow.getId(), flow);
+    					}
+    					
+    					projectFlows.put(project.getName(), flowMap);
+    				}
     			}
     		}
     	}
@@ -106,7 +146,7 @@ public class FileProjectManager implements ProjectManager {
     	return array;
     }
     
-    public Project getProject(String name, User user) throws AccessControlException {
+    public Project getProject(String name, User user) {
     	Project project = projects.get(name);
     	if (project != null) {
     		Permission perm = project.getUserPermission(user);
@@ -133,14 +173,13 @@ public class FileProjectManager implements ProjectManager {
     	
 		Map<String, Flow> flows = new HashMap<String,Flow>();
 		List<String> errors = new ArrayList<String>();
-		List<Props> propsList = new ArrayList<Props>();
-		FlowUtils.loadProject(dir, flows, propsList, errors);
+		FlowUtils.loadProjectFlows(dir, flows, errors);
 		
     	File projectPath = new File(projectDirectory, projectName);
 		File installDir = new File(projectPath, FILE_DATE_FORMAT.print(System.currentTimeMillis()));
 		if (!installDir.mkdir()) {
 			throw new ProjectManagerException("Cannot create directory in " + projectDirectory);
-		}
+		}	
 		
 		for (Flow flow: flows.values()) {
 	    	try {
@@ -160,10 +199,19 @@ public class FileProjectManager implements ProjectManager {
 	    	// We synchronize on project so that we don't collide when uploading.
 	    	synchronized (project) {
 	    		logger.info("Uploading files to " + projectName);
-	    		project.setSource(projectDirectory.getName());
+	    		project.setSource(installDir.getName());
 	    		project.setLastModifiedTimestamp(System.currentTimeMillis());
 	    		project.setLastModifiedUser(uploader.getUserId());
+	    		projectFlows.put(projectName, flows);
 	    	}
+	    	
+	    	try {
+				writeProjectFile(projectPath, project);
+			} catch (IOException e) {
+	    		throw new ProjectManagerException(
+	    				"Project directory " + projectName + 
+	    				" cannot be created in " + projectDirectory, e);
+			}
 		}
 		else {
 			logger.info("Errors found loading project " + projectName);
@@ -283,4 +331,26 @@ public class FileProjectManager implements ProjectManager {
 	public synchronized Project removeProject(String projectName, User user) {
 		return null;
 	}
+
+	@Override
+	public List<Flow> getProjectFlows(String projectName, User user) throws ProjectManagerException {
+		
+		
+		return null;
+	}
+
+	private static class SuffixFilter implements FileFilter {
+		private String suffix;
+		
+		public SuffixFilter(String suffix) {
+			this.suffix = suffix;
+		}
+
+		@Override
+		public boolean accept(File pathname) {
+			String name = pathname.getName();
+			
+			return pathname.isFile() && !pathname.isHidden() && name.length() > suffix.length() && name.endsWith(suffix);
+		}
+	}
 }
\ No newline at end of file
diff --git a/src/java/azkaban/project/FileResourceLoader.java b/src/java/azkaban/project/FileResourceLoader.java
new file mode 100644
index 0000000..40af740
--- /dev/null
+++ b/src/java/azkaban/project/FileResourceLoader.java
@@ -0,0 +1,57 @@
+package azkaban.project;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+
+import azkaban.utils.Pair;
+import azkaban.utils.Props;
+
+public class FileResourceLoader implements ResourceLoader {
+	private HashMap<Pair<String, String>, Props> propsCache = new HashMap<Pair<String,String>, Props>();
+	private File basePath;
+	
+	public FileResourceLoader(File basePath) {
+		this.basePath = basePath;
+	}
+	
+	@Override
+	public Props loadPropsFromSource(String source) {
+		return loadPropsFromSource(null, source);
+	}
+	
+	@Override
+	public Props loadPropsFromSource(Props parent, String source) {
+		String parentSource = parent == null ? "null" : parent.getSource();
+		Pair<String, String> pair = new Pair<String,String>(parentSource, source);
+		Props props = propsCache.get(pair);
+		if (props != null) {
+			return props;
+		}
+
+		File path = new File(basePath, source);
+
+		if (!path.exists()) {
+			props = createErrorProps("Source file " + source + " doesn't exist.");
+		}
+		else if (!path.isFile()) {
+			props = createErrorProps("Source file " + source + " isn't a file.");
+		}
+		else {
+			try {
+				props = new Props(parent, path);
+			} catch (IOException e) {
+				props = createErrorProps("Error loading resource: " + e.getMessage());
+			}
+		}
+		
+		propsCache.put(pair, props);
+		return props;
+	}
+
+	private Props createErrorProps(String message) {
+		Props props = new Props();
+		props.put("error", message);
+		return props;
+	}
+}
diff --git a/src/java/azkaban/project/Project.java b/src/java/azkaban/project/Project.java
index 15ea37f..cb7337a 100644
--- a/src/java/azkaban/project/Project.java
+++ b/src/java/azkaban/project/Project.java
@@ -2,6 +2,7 @@ package azkaban.project;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 
@@ -128,7 +129,6 @@ public class Project {
 		if (source != null) {
 			project.setSource(source);
 		}
-
 		
 		List<Map<String, Object>> users = (List<Map<String, Object>>) projectObject
 				.get("users");
diff --git a/src/java/azkaban/project/ProjectManager.java b/src/java/azkaban/project/ProjectManager.java
index fbc0d92..f0ddabe 100644
--- a/src/java/azkaban/project/ProjectManager.java
+++ b/src/java/azkaban/project/ProjectManager.java
@@ -1,10 +1,13 @@
 package azkaban.project;
 
 import java.io.File;
+import java.io.IOException;
 import java.security.AccessControlException;
 import java.util.List;
 
+import azkaban.flow.Flow;
 import azkaban.user.User;
+import azkaban.utils.Props;
 
 public interface ProjectManager {
     
@@ -12,7 +15,9 @@ public interface ProjectManager {
     
     public List<Project> getProjects(User user);
     
-    public Project getProject(String name, User user) throws AccessControlException;
+    public List<Flow> getProjectFlows(String projectName, User user) throws ProjectManagerException;
+    
+    public Project getProject(String name, User user);
     
     public void uploadProject(String projectName, File projectDir, User uploader, boolean force) throws ProjectManagerException;
     
diff --git a/src/java/azkaban/project/ResourceLoader.java b/src/java/azkaban/project/ResourceLoader.java
new file mode 100644
index 0000000..c09df05
--- /dev/null
+++ b/src/java/azkaban/project/ResourceLoader.java
@@ -0,0 +1,9 @@
+package azkaban.project;
+
+import azkaban.utils.Props;
+
+public interface ResourceLoader {
+	public Props loadPropsFromSource(String source);
+	
+	public Props loadPropsFromSource(Props parent, String source);
+}
diff --git a/src/java/azkaban/utils/FlowUtils.java b/src/java/azkaban/utils/FlowUtils.java
index 35a7bce..b5cd295 100644
--- a/src/java/azkaban/utils/FlowUtils.java
+++ b/src/java/azkaban/utils/FlowUtils.java
@@ -21,9 +21,10 @@ public class FlowUtils {
 	private static final String DEPENDENCIES = "dependencies";
 	private static final String JOB_SUFFIX = ".job";
 	
-	public static void loadProject(File dir, Map<String, Flow> output, List<Props> propsList, List<String> projectErrors) {
+	public static void loadProjectFlows(File dir, Map<String, Flow> output, List<String> projectErrors) {
 		// Load all the project and job files.
 		Map<String,Node> jobMap = new HashMap<String,Node>();
+		List<Props> propsList = new ArrayList<Props>();
 		Set<String> duplicateJobs = new HashSet<String>();
 		Set<String> errors = new HashSet<String>();
 		loadProjectFromDir(dir.getPath(), dir, jobMap, propsList, duplicateJobs, errors);
@@ -32,7 +33,12 @@ public class FlowUtils {
 		Map<String, Set<Edge>> dependencies = new HashMap<String, Set<Edge>>();
 		resolveDependencies(jobMap, duplicateJobs, dependencies, errors);
 
+		// We add all the props for the flow. Each flow will be able to keep an independent list of dependencies.
 		HashMap<String, Flow> flows = buildFlowsFromDependencies(jobMap, dependencies, errors);
+		for (Flow flow: flows.values()) {
+			flow.addAllProperties(propsList);
+		}
+		
 		output.putAll(flows);
 		projectErrors.addAll(errors);
 	}
@@ -46,23 +52,24 @@ public class FlowUtils {
 		File[] propertyFiles = dir.listFiles(new SuffixFilter(PROPERTY_SUFFIX));
 		Props parent = null;
 		for (File file: propertyFiles) {
+			String relative = getRelativeFilePath(base, file.getPath());
 			try {
 				parent = new Props(parent, file);
-				String relative = getRelativeFilePath(base, file.getPath());
 				parent.setSource(relative);
 				
-				System.out.println("Adding " + relative);
-				propsList.add(parent);
 			} catch (IOException e) {
 				errors.add("Error loading properties " + file.getName() + ":" + e.getMessage());
 			}
+			
+			System.out.println("Adding " + relative);
+			propsList.add(parent);
 		}
 		
 		// Load all Job files. If there's a duplicate name, then we don't load
 		File[] jobFiles = dir.listFiles(new SuffixFilter(JOB_SUFFIX));
 		for (File file: jobFiles) {
+			String jobName = getNameWithoutExtension(file);
 			try {
-				String jobName = getNameWithoutExtension(file);
 				if (!duplicateJobs.contains(jobName)) {
 					if (jobMap.containsKey(jobName)) {
 						errors.add("Duplicate job names found '" + jobName + "'.");
diff --git a/src/java/azkaban/utils/Pair.java b/src/java/azkaban/utils/Pair.java
new file mode 100644
index 0000000..867f845
--- /dev/null
+++ b/src/java/azkaban/utils/Pair.java
@@ -0,0 +1,51 @@
+package azkaban.utils;
+
+public class Pair<F, S> {
+	private final F first;
+	private final S second;
+	
+	public Pair(F first, S second) {
+		this.first = first;
+		this.second = second;
+	}
+	
+	public F getFirst() {
+		return first;
+	}
+	
+	public S getSecond() {
+		return second;
+	}
+
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = 1;
+		result = prime * result + ((first == null) ? 0 : first.hashCode());
+		result = prime * result + ((second == null) ? 0 : second.hashCode());
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		@SuppressWarnings("rawtypes")
+		Pair other = (Pair) obj;
+		if (first == null) {
+			if (other.first != null)
+				return false;
+		} else if (!first.equals(other.first))
+			return false;
+		if (second == null) {
+			if (other.second != null)
+				return false;
+		} else if (!second.equals(other.second))
+			return false;
+		return true;
+	}
+}
diff --git a/src/java/azkaban/utils/Props.java b/src/java/azkaban/utils/Props.java
index e1fae6f..fa462fa 100644
--- a/src/java/azkaban/utils/Props.java
+++ b/src/java/azkaban/utils/Props.java
@@ -43,7 +43,7 @@ import org.apache.log4j.Logger;
  */
 public class Props {
     private final Map<String, String> _current;
-    private final Props _parent;
+    private Props _parent;
     private String source = null;
 
     /**
@@ -965,4 +965,8 @@ public class Props {
     public void setSource(String source) {
         this.source = source;
     }
+
+    public void setParent(Props prop) {
+    	this._parent = prop;
+    }
 }