Details
diff --git a/azkaban-common/src/main/java/azkaban/executor/ExecutionOptions.java b/azkaban-common/src/main/java/azkaban/executor/ExecutionOptions.java
index 90991f6..366939b 100644
--- a/azkaban-common/src/main/java/azkaban/executor/ExecutionOptions.java
+++ b/azkaban-common/src/main/java/azkaban/executor/ExecutionOptions.java
@@ -48,6 +48,7 @@ public class ExecutionOptions {
private static final String FAILURE_EMAILS_OVERRIDE = "failureEmailsOverride";
private static final String SUCCESS_EMAILS_OVERRIDE = "successEmailsOverride";
private static final String MAIL_CREATOR = "mailCreator";
+ private static final String MEMORY_CHECK = "memoryCheck";
private boolean notifyOnFirstFailure = true;
private boolean notifyOnLastFailure = false;
@@ -61,6 +62,7 @@ public class ExecutionOptions {
private Integer queueLevel = 0;
private String concurrentOption = CONCURRENT_OPTION_IGNORE;
private String mailCreator = DefaultMailCreator.DEFAULT_MAIL_CREATOR;
+ private boolean memoryCheck = false;
private Map<String, String> flowParameters = new HashMap<String, String>();
public enum FailureAction {
@@ -179,6 +181,14 @@ public class ExecutionOptions {
initiallyDisabledJobs = disabledJobs;
}
+ public boolean getMemoryCheck() {
+ return memoryCheck;
+ }
+
+ public void setMemoryCheck(boolean memoryCheck) {
+ this.memoryCheck = memoryCheck;
+ }
+
public Map<String, Object> toObject() {
HashMap<String, Object> flowOptionObj = new HashMap<String, Object>();
@@ -196,6 +206,7 @@ public class ExecutionOptions {
flowOptionObj.put(FAILURE_EMAILS_OVERRIDE, failureEmailsOverride);
flowOptionObj.put(SUCCESS_EMAILS_OVERRIDE, successEmailsOverride);
flowOptionObj.put(MAIL_CREATOR, mailCreator);
+ flowOptionObj.put(MEMORY_CHECK, memoryCheck);
return flowOptionObj;
}
@@ -252,6 +263,8 @@ public class ExecutionOptions {
options.setFailureEmailsOverridden(wrapper.getBool(FAILURE_EMAILS_OVERRIDE,
false));
+ options.setMemoryCheck(wrapper.getBool(MEMORY_CHECK, false));
+
return options;
}
}
diff --git a/azkaban-common/src/main/java/azkaban/executor/ExecutorManager.java b/azkaban-common/src/main/java/azkaban/executor/ExecutorManager.java
index 5f7bcd5..8061353 100644
--- a/azkaban-common/src/main/java/azkaban/executor/ExecutorManager.java
+++ b/azkaban-common/src/main/java/azkaban/executor/ExecutorManager.java
@@ -45,6 +45,7 @@ import azkaban.event.Event;
import azkaban.event.Event.Type;
import azkaban.event.EventHandler;
import azkaban.project.Project;
+import azkaban.project.ProjectWhitelist;
import azkaban.scheduler.ScheduleStatisticManager;
import azkaban.utils.FileIOUtils.JobMetaData;
import azkaban.utils.FileIOUtils.LogData;
@@ -585,6 +586,10 @@ public class ExecutorManager extends EventHandler implements
}
}
+ boolean memoryCheck = !ProjectWhitelist.isProjectWhitelisted(exflow.getProjectName(),
+ ProjectWhitelist.WhitelistType.MemoryCheck);
+ options.setMemoryCheck(memoryCheck);
+
// The exflow id is set by the loader. So it's unavailable until after
// this call.
executorLoader.uploadExecutableFlow(exflow);
diff --git a/azkaban-common/src/main/java/azkaban/jobExecutor/ProcessJob.java b/azkaban-common/src/main/java/azkaban/jobExecutor/ProcessJob.java
index 29596d1..bb92e01 100644
--- a/azkaban-common/src/main/java/azkaban/jobExecutor/ProcessJob.java
+++ b/azkaban-common/src/main/java/azkaban/jobExecutor/ProcessJob.java
@@ -40,6 +40,7 @@ public class ProcessJob extends AbstractProcessJob {
private volatile AzkabanProcess process;
private static final String MEMCHECK_ENABLED = "memCheck.enabled";
private static final String MEMCHECK_FREEMEMDECRAMT = "memCheck.freeMemDecrAmt";
+ public static final String AZKABAN_MEMORY_CHECK = "azkaban.memory.check";
public ProcessJob(final String jobId, final Props sysProps,
final Props jobProps, final Logger log) {
@@ -54,7 +55,7 @@ public class ProcessJob extends AbstractProcessJob {
handleError("Bad property definition! " + e.getMessage(), e);
}
- if (sysProps.getBoolean(MEMCHECK_ENABLED, true)) {
+ if (sysProps.getBoolean(MEMCHECK_ENABLED, true) && jobProps.getBoolean(AZKABAN_MEMORY_CHECK, false)) {
long freeMemDecrAmt = sysProps.getLong(MEMCHECK_FREEMEMDECRAMT, 0);
Pair<Long, Long> memPair = getProcMemoryRequirement();
boolean isMemGranted = SystemMemoryInfo.canSystemGrantMemory(memPair.getFirst(), memPair.getSecond(), freeMemDecrAmt);
diff --git a/azkaban-common/src/main/java/azkaban/project/ProjectManager.java b/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
index 4af1c9e..03c5594 100644
--- a/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
+++ b/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
@@ -33,6 +33,7 @@ import org.apache.log4j.Logger;
import azkaban.flow.Flow;
import azkaban.project.ProjectLogEvent.EventType;
+import azkaban.project.ProjectWhitelist.WhitelistType;
import azkaban.project.validator.ValidationReport;
import azkaban.project.validator.ValidationStatus;
import azkaban.project.validator.ValidatorConfigs;
@@ -84,6 +85,7 @@ public class ProjectManager {
// config files for the validators.
new XmlValidatorManager(prop);
loadAllProjects();
+ loadProjectWhiteList();
}
private void loadAllProjects() {
@@ -429,7 +431,7 @@ public class ProjectManager {
logger.info("Validating project " + archive.getName()
+ " using the registered validators "
+ validatorManager.getValidatorsInfo().toString());
- Map<String, ValidationReport> reports = validatorManager.validate(file);
+ Map<String, ValidationReport> reports = validatorManager.validate(project, file);
ValidationStatus status = ValidationStatus.PASS;
for (Entry<String, ValidationReport> report : reports.entrySet()) {
if (report.getValue().getStatus().compareTo(status) > 0) {
@@ -515,4 +517,7 @@ public class ProjectManager {
projectLoader.postEvent(project, type, user, message);
}
+ public void loadProjectWhiteList() {
+ ProjectWhitelist.load(props);
+ }
}
diff --git a/azkaban-common/src/main/java/azkaban/project/ProjectWhitelist.java b/azkaban-common/src/main/java/azkaban/project/ProjectWhitelist.java
new file mode 100644
index 0000000..b0be780
--- /dev/null
+++ b/azkaban-common/src/main/java/azkaban/project/ProjectWhitelist.java
@@ -0,0 +1,143 @@
+package azkaban.project;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import azkaban.utils.Props;
+
+/**
+ * @author wkang
+ *
+ * This class manages project whitelist defined in xml config file.
+ * An single xml config file contains different types of whitelisted
+ * projects. For additional type of whitelist, modify WhitelistType enum.
+ *
+ * The xml config file should in the following format. Please note
+ * the tag <MemoryCheck> is same as the defined enum MemoryCheck
+ *
+ * <ProjectWhitelists>
+ * <MemoryCheck>
+ * <project projectname="project1" />
+ * <project projectname="project2" />
+ * </MemoryCheck>
+ * <ProjectWhitelists>
+ *
+ */
+public class ProjectWhitelist {
+ private static final String XML_FILE_PARAM = "project.whitelists.xml.file";
+ private static final String PROJECT_WHITELISTS_TAG = "ProjectWhitelists";
+ private static final String PROJECT_TAG = "project";
+ private static final String PROJECTNAME_ATTR = "projectname";
+
+ private static AtomicReference<Map<WhitelistType, Set<String>>> projectsWhitelisted =
+ new AtomicReference<Map<WhitelistType, Set<String>>>();
+
+ static void load(Props props) {
+ String xmlFile = props.getString(XML_FILE_PARAM);
+ parseXMLFile(xmlFile);
+ }
+
+ private static void parseXMLFile(String xmlFile) {
+ File file = new File(xmlFile);
+ if (!file.exists()) {
+ throw new IllegalArgumentException("Project whitelist xml file " + xmlFile
+ + " doesn't exist.");
+ }
+
+ // Creating the document builder to parse xml.
+ DocumentBuilderFactory docBuilderFactory =
+ DocumentBuilderFactory.newInstance();
+ DocumentBuilder builder = null;
+ try {
+ builder = docBuilderFactory.newDocumentBuilder();
+ } catch (ParserConfigurationException e) {
+ throw new IllegalArgumentException(
+ "Exception while parsing project whitelist xml. Document builder not created.", e);
+ }
+
+ Document doc = null;
+ try {
+ doc = builder.parse(file);
+ } catch (SAXException e) {
+ throw new IllegalArgumentException("Exception while parsing " + xmlFile
+ + ". Invalid XML.", e);
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Exception while parsing " + xmlFile
+ + ". Error reading file.", e);
+ }
+
+ Map<WhitelistType, Set<String>> projsWhitelisted = new HashMap<WhitelistType, Set<String>>();
+ NodeList tagList = doc.getChildNodes();
+ if (!tagList.item(0).getNodeName().equals(PROJECT_WHITELISTS_TAG)) {
+ throw new RuntimeException("Cannot find tag '" + PROJECT_WHITELISTS_TAG + "' in " + xmlFile);
+ }
+
+ NodeList whitelists = tagList.item(0).getChildNodes();
+ for (int n = 0; n < whitelists.getLength(); ++n) {
+ if (whitelists.item(n).getNodeType() != Node.ELEMENT_NODE) {
+ continue;
+ }
+
+ String whitelistType = whitelists.item(n).getNodeName();
+ Set<String> projs = new HashSet<String>();
+
+ NodeList projectsList = whitelists.item(n).getChildNodes();
+ for (int i = 0; i < projectsList.getLength(); ++i) {
+ Node node = projectsList.item(i);
+ if (node.getNodeType() == Node.ELEMENT_NODE) {
+ if (node.getNodeName().equals(PROJECT_TAG)) {
+ parseProjectTag(node, projs);
+ }
+ }
+ }
+ projsWhitelisted.put(WhitelistType.valueOf(whitelistType), projs);
+ }
+ projectsWhitelisted.set(projsWhitelisted);
+ }
+
+ private static void parseProjectTag(Node node, Set<String> projects) {
+ NamedNodeMap projectAttrMap = node.getAttributes();
+ Node projectNameAttr = projectAttrMap.getNamedItem(PROJECTNAME_ATTR);
+ if (projectNameAttr == null) {
+ throw new RuntimeException("Error loading project. The '" + PROJECTNAME_ATTR
+ + "' attribute doesn't exist");
+ }
+
+ String projectName = projectNameAttr.getNodeValue();
+ projects.add(projectName);
+ }
+
+ public static boolean isProjectWhitelisted(String project, WhitelistType whitelistType) {
+ Map<WhitelistType, Set<String>> projsWhitelisted = projectsWhitelisted.get();
+ if (projsWhitelisted != null) {
+ Set<String> projs = projsWhitelisted.get(whitelistType);
+ if (projs != null) {
+ return projs.contains(project);
+ }
+ }
+ return false;
+ }
+
+ /**
+ * The tag in the project whitelist xml config file should be same as
+ * the defined enums.
+ */
+ public static enum WhitelistType {
+ MemoryCheck
+ }
+}
\ No newline at end of file
diff --git a/azkaban-common/src/main/java/azkaban/project/validator/ProjectValidator.java b/azkaban-common/src/main/java/azkaban/project/validator/ProjectValidator.java
index b1b10b8..3c4b0e2 100644
--- a/azkaban-common/src/main/java/azkaban/project/validator/ProjectValidator.java
+++ b/azkaban-common/src/main/java/azkaban/project/validator/ProjectValidator.java
@@ -2,6 +2,7 @@ package azkaban.project.validator;
import java.io.File;
+import azkaban.project.Project;
import azkaban.utils.Props;
/**
@@ -33,5 +34,5 @@ public interface ProjectValidator {
* @param projectDir
* @return
*/
- ValidationReport validateProject(File projectDir);
+ ValidationReport validateProject(Project project, File projectDir);
}
diff --git a/azkaban-common/src/main/java/azkaban/project/validator/ValidatorManager.java b/azkaban-common/src/main/java/azkaban/project/validator/ValidatorManager.java
index 1be62fb..e759ad2 100644
--- a/azkaban-common/src/main/java/azkaban/project/validator/ValidatorManager.java
+++ b/azkaban-common/src/main/java/azkaban/project/validator/ValidatorManager.java
@@ -6,6 +6,7 @@ import java.util.Map;
import org.apache.log4j.Logger;
+import azkaban.project.Project;
import azkaban.utils.Props;
/**
@@ -31,7 +32,7 @@ public interface ValidatorManager {
* @param projectDir
* @return
*/
- Map<String, ValidationReport> validate(File projectDir);
+ Map<String, ValidationReport> validate(Project project, File projectDir);
/**
* The ValidatorManager should have a default validator which checks for the most essential
diff --git a/azkaban-common/src/main/java/azkaban/project/validator/XmlValidatorManager.java b/azkaban-common/src/main/java/azkaban/project/validator/XmlValidatorManager.java
index 52d376c..0f240da 100644
--- a/azkaban-common/src/main/java/azkaban/project/validator/XmlValidatorManager.java
+++ b/azkaban-common/src/main/java/azkaban/project/validator/XmlValidatorManager.java
@@ -23,6 +23,7 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
+import azkaban.project.Project;
import azkaban.utils.DirectoryFlowLoader;
import azkaban.utils.Props;
@@ -226,10 +227,10 @@ public class XmlValidatorManager implements ValidatorManager {
}
@Override
- public Map<String, ValidationReport> validate(File projectDir) {
+ public Map<String, ValidationReport> validate(Project project, File projectDir) {
Map<String, ValidationReport> reports = new LinkedHashMap<String, ValidationReport>();
for (Entry<String, ProjectValidator> validator : validators.entrySet()) {
- reports.put(validator.getKey(), validator.getValue().validateProject(projectDir));
+ reports.put(validator.getKey(), validator.getValue().validateProject(project, projectDir));
logger.info("Validation status of validator " + validator.getKey() + " is "
+ reports.get(validator.getKey()).getStatus());
}
diff --git a/azkaban-common/src/main/java/azkaban/utils/DirectoryFlowLoader.java b/azkaban-common/src/main/java/azkaban/utils/DirectoryFlowLoader.java
index ad7a48f..8e402c5 100644
--- a/azkaban-common/src/main/java/azkaban/utils/DirectoryFlowLoader.java
+++ b/azkaban-common/src/main/java/azkaban/utils/DirectoryFlowLoader.java
@@ -36,6 +36,8 @@ import azkaban.flow.Flow;
import azkaban.flow.FlowProps;
import azkaban.flow.Node;
import azkaban.flow.SpecialJobTypes;
+import azkaban.project.Project;
+import azkaban.project.ProjectWhitelist;
import azkaban.project.validator.ProjectValidator;
import azkaban.project.validator.ValidationReport;
import azkaban.project.validator.XmlValidatorManager;
@@ -88,7 +90,7 @@ public class DirectoryFlowLoader implements ProjectValidator {
return propsList;
}
- public void loadProjectFlow(File baseDirectory) {
+ public void loadProjectFlow(Project project, File baseDirectory) {
propsList = new ArrayList<Props>();
flowPropsList = new ArrayList<FlowProps>();
jobPropsMap = new HashMap<String, Props>();
@@ -103,7 +105,7 @@ public class DirectoryFlowLoader implements ProjectValidator {
// Load all the props files and create the Node objects
loadProjectFromDir(baseDirectory.getPath(), baseDirectory, null);
- jobPropertiesCheck();
+ jobPropertiesCheck(project);
// Create edges and find missing dependencies
resolveDependencies();
@@ -376,7 +378,13 @@ public class DirectoryFlowLoader implements ProjectValidator {
visited.remove(node.getId());
}
- private void jobPropertiesCheck() {
+ private void jobPropertiesCheck(Project project) {
+ //if project is in the memory check whitelist, then we don't need to check its memory settings
+ if (ProjectWhitelist.isProjectWhitelisted(project.getName(),
+ ProjectWhitelist.WhitelistType.MemoryCheck)) {
+ return;
+ }
+
String maxXms = props.getString(JOB_MAX_XMS, MAX_XMS_DEFAULT);
String maxXmx = props.getString(JOB_MAX_XMX, MAX_XMX_DEFAULT);
long sizeMaxXms = Utils.parseMemString(maxXms);
@@ -444,8 +452,8 @@ public class DirectoryFlowLoader implements ProjectValidator {
}
@Override
- public ValidationReport validateProject(File projectDir) {
- loadProjectFlow(projectDir);
+ public ValidationReport validateProject(Project project, File projectDir) {
+ loadProjectFlow(project, projectDir);
ValidationReport report = new ValidationReport();
report.addErrorMsgs(errors);
return report;
diff --git a/azkaban-common/src/test/java/azkaban/executor/ExecutableFlowTest.java b/azkaban-common/src/test/java/azkaban/executor/ExecutableFlowTest.java
index 5948c7b..91a4290 100644
--- a/azkaban-common/src/test/java/azkaban/executor/ExecutableFlowTest.java
+++ b/azkaban-common/src/test/java/azkaban/executor/ExecutableFlowTest.java
@@ -43,12 +43,13 @@ public class ExecutableFlowTest {
@Before
public void setUp() throws Exception {
+ project = new Project(11, "myTestProject");
+
Logger logger = Logger.getLogger(this.getClass());
DirectoryFlowLoader loader = new DirectoryFlowLoader(new Props(), logger);
- loader.loadProjectFlow(new File("unit/executions/embedded"));
+ loader.loadProjectFlow(project, new File("unit/executions/embedded"));
Assert.assertEquals(0, loader.getErrors().size());
- project = new Project(11, "myTestProject");
project.setFlows(loader.getFlowMap());
project.setVersion(123);
}
diff --git a/azkaban-common/src/test/java/azkaban/utils/DirectoryFlowLoaderTest.java b/azkaban-common/src/test/java/azkaban/utils/DirectoryFlowLoaderTest.java
index b45b853..814744c 100644
--- a/azkaban-common/src/test/java/azkaban/utils/DirectoryFlowLoaderTest.java
+++ b/azkaban-common/src/test/java/azkaban/utils/DirectoryFlowLoaderTest.java
@@ -19,19 +19,22 @@ package azkaban.utils;
import java.io.File;
import org.apache.log4j.Logger;
-
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
+import azkaban.project.Project;
+
public class DirectoryFlowLoaderTest {
+ Project project = new Project(11, "myTestProject");
+
@Ignore @Test
public void testDirectoryLoad() {
Logger logger = Logger.getLogger(this.getClass());
DirectoryFlowLoader loader = new DirectoryFlowLoader(new Props(), logger);
- loader.loadProjectFlow(new File("unit/executions/exectest1"));
+ loader.loadProjectFlow(project, new File("unit/executions/exectest1"));
logger.info(loader.getFlowMap().size());
}
@@ -40,7 +43,7 @@ public class DirectoryFlowLoaderTest {
Logger logger = Logger.getLogger(this.getClass());
DirectoryFlowLoader loader = new DirectoryFlowLoader(new Props(), logger);
- loader.loadProjectFlow(new File("unit/executions/embedded"));
+ loader.loadProjectFlow(project, new File("unit/executions/embedded"));
Assert.assertEquals(0, loader.getErrors().size());
}
@@ -49,7 +52,7 @@ public class DirectoryFlowLoaderTest {
Logger logger = Logger.getLogger(this.getClass());
DirectoryFlowLoader loader = new DirectoryFlowLoader(new Props(), logger);
- loader.loadProjectFlow(new File("unit/executions/embeddedBad"));
+ loader.loadProjectFlow(project, new File("unit/executions/embeddedBad"));
for (String error : loader.getErrors()) {
System.out.println(error);
}
diff --git a/azkaban-execserver/src/main/java/azkaban/execapp/FlowRunner.java b/azkaban-execserver/src/main/java/azkaban/execapp/FlowRunner.java
index e98f5f2..dd18b3f 100644
--- a/azkaban-execserver/src/main/java/azkaban/execapp/FlowRunner.java
+++ b/azkaban-execserver/src/main/java/azkaban/execapp/FlowRunner.java
@@ -53,6 +53,7 @@ import azkaban.executor.ExecutorLoader;
import azkaban.executor.ExecutorManagerException;
import azkaban.executor.Status;
import azkaban.flow.FlowProps;
+import azkaban.jobExecutor.ProcessJob;
import azkaban.jobtype.JobTypeManager;
import azkaban.metric.MetricReportManager;
import azkaban.project.ProjectLoader;
@@ -651,6 +652,11 @@ public class FlowRunner extends EventHandler implements Runnable {
node.setInputProps(props);
}
+ private void customizeJobProperties(Props props) {
+ boolean memoryCheck = flow.getExecutionOptions().getMemoryCheck();
+ props.put(ProcessJob.AZKABAN_MEMORY_CHECK, Boolean.toString(memoryCheck));
+ }
+
private Props loadJobProps(ExecutableNode node) throws IOException {
Props props = null;
String source = node.getJobSource();
@@ -681,6 +687,9 @@ public class FlowRunner extends EventHandler implements Runnable {
if (path.getPath() != null) {
props.setSource(path.getPath());
}
+
+ customizeJobProperties(props);
+
return props;
}
diff --git a/azkaban-execserver/src/test/java/azkaban/execapp/FlowRunnerPipelineTest.java b/azkaban-execserver/src/test/java/azkaban/execapp/FlowRunnerPipelineTest.java
index 15cb90a..360d4f4 100644
--- a/azkaban-execserver/src/test/java/azkaban/execapp/FlowRunnerPipelineTest.java
+++ b/azkaban-execserver/src/test/java/azkaban/execapp/FlowRunnerPipelineTest.java
@@ -102,7 +102,7 @@ public class FlowRunnerPipelineTest {
project = new Project(1, "testProject");
File dir = new File("unit/executions/embedded2");
- prepareProject(dir);
+ prepareProject(project, dir);
InteractiveTestJob.clearTestJobs();
}
@@ -647,10 +647,10 @@ public class FlowRunnerPipelineTest {
}
}
- private void prepareProject(File directory) throws ProjectManagerException,
+ private void prepareProject(Project project, File directory) throws ProjectManagerException,
IOException {
DirectoryFlowLoader loader = new DirectoryFlowLoader(new Props(), logger);
- loader.loadProjectFlow(directory);
+ loader.loadProjectFlow(project, directory);
if (!loader.getErrors().isEmpty()) {
for (String error : loader.getErrors()) {
System.out.println(error);
diff --git a/azkaban-execserver/src/test/java/azkaban/execapp/FlowRunnerPropertyResolutionTest.java b/azkaban-execserver/src/test/java/azkaban/execapp/FlowRunnerPropertyResolutionTest.java
index a586f64..d5988bb 100644
--- a/azkaban-execserver/src/test/java/azkaban/execapp/FlowRunnerPropertyResolutionTest.java
+++ b/azkaban-execserver/src/test/java/azkaban/execapp/FlowRunnerPropertyResolutionTest.java
@@ -95,7 +95,7 @@ public class FlowRunnerPropertyResolutionTest {
project = new Project(1, "testProject");
File dir = new File("unit/executions/execpropstest");
- prepareProject(dir);
+ prepareProject(project, dir);
InteractiveTestJob.clearTestJobs();
}
@@ -207,10 +207,10 @@ public class FlowRunnerPropertyResolutionTest {
Assert.assertEquals("moo4", job3Props.get("props4"));
}
- private void prepareProject(File directory) throws ProjectManagerException,
+ private void prepareProject(Project project, File directory) throws ProjectManagerException,
IOException {
DirectoryFlowLoader loader = new DirectoryFlowLoader(new Props(), logger);
- loader.loadProjectFlow(directory);
+ loader.loadProjectFlow(project, directory);
if (!loader.getErrors().isEmpty()) {
for (String error : loader.getErrors()) {
System.out.println(error);
diff --git a/azkaban-execserver/src/test/java/azkaban/execapp/FlowRunnerTest2.java b/azkaban-execserver/src/test/java/azkaban/execapp/FlowRunnerTest2.java
index 00f61b4..c78abaa 100644
--- a/azkaban-execserver/src/test/java/azkaban/execapp/FlowRunnerTest2.java
+++ b/azkaban-execserver/src/test/java/azkaban/execapp/FlowRunnerTest2.java
@@ -124,7 +124,7 @@ public class FlowRunnerTest2 {
project = new Project(1, "testProject");
File dir = new File("unit/executions/embedded2");
- prepareProject(dir);
+ prepareProject(project, dir);
InteractiveTestJob.clearTestJobs();
}
@@ -1378,10 +1378,10 @@ public class FlowRunnerTest2 {
}
}
- private void prepareProject(File directory)
+ private void prepareProject(Project project, File directory)
throws ProjectManagerException, IOException {
DirectoryFlowLoader loader = new DirectoryFlowLoader(new Props(), logger);
- loader.loadProjectFlow(directory);
+ loader.loadProjectFlow(project, directory);
if (!loader.getErrors().isEmpty()) {
for (String error: loader.getErrors()) {
System.out.println(error);
diff --git a/azkaban-webserver/src/main/java/azkaban/webapp/servlet/ProjectManagerServlet.java b/azkaban-webserver/src/main/java/azkaban/webapp/servlet/ProjectManagerServlet.java
index 34a8fd8..b59dc9d 100644
--- a/azkaban-webserver/src/main/java/azkaban/webapp/servlet/ProjectManagerServlet.java
+++ b/azkaban-webserver/src/main/java/azkaban/webapp/servlet/ProjectManagerServlet.java
@@ -153,6 +153,8 @@ public class ProjectManagerServlet extends LoginAbstractAzkabanServlet {
handleProjectPage(req, resp, session);
}
return;
+ } else if (hasParam(req, "reloadProjectWhitelist")) {
+ handleReloadProjectWhitelist(req, resp, session);
}
Page page =
@@ -1738,4 +1740,23 @@ public class ProjectManagerServlet extends LoginAbstractAzkabanServlet {
return false;
}
+
+ private void handleReloadProjectWhitelist(HttpServletRequest req,
+ HttpServletResponse resp, Session session) throws ServletException {
+ if (hasPermission(session.getUser(), Permission.Type.ADMIN)) {
+ projectManager.loadProjectWhiteList();
+ }
+ }
+
+ protected boolean hasPermission(User user, Permission.Type type) {
+ for (String roleName : user.getRoles()) {
+ Role role = userManager.getRole(roleName);
+ if (role.getPermission().isPermissionSet(type)
+ || role.getPermission().isPermissionSet(Permission.Type.ADMIN)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
}
