Details
diff --git a/azkaban-common/src/main/java/azkaban/project/JdbcProjectLoader.java b/azkaban-common/src/main/java/azkaban/project/JdbcProjectLoader.java
index ced5eb0..6f55a9c 100644
--- a/azkaban-common/src/main/java/azkaban/project/JdbcProjectLoader.java
+++ b/azkaban-common/src/main/java/azkaban/project/JdbcProjectLoader.java
@@ -171,67 +171,68 @@ public class JdbcProjectLoader extends AbstractJdbcLoader implements
return project;
}
- /**
- * Fetch first project with a given name {@inheritDoc}
- *
- * @see azkaban.project.ProjectLoader#fetchProjectByName(java.lang.String)
- */
- @Override
- public Project fetchProjectByName(String name)
- throws ProjectManagerException {
- Connection connection = getConnection();
-
- Project project = null;
- try {
- project = fetchProjectByName(connection, name);
- } finally {
- DbUtils.closeQuietly(connection);
- }
+ @Override
+ public Project fetchProjectByName(String name)
+ throws ProjectManagerException {
+ Connection connection = getConnection();
- return project;
+ Project project = null;
+ try {
+ project = fetchProjectByName(connection, name);
+ } finally {
+ DbUtils.closeQuietly(connection);
}
- private Project fetchProjectByName(Connection connection, String name)
- throws ProjectManagerException {
- QueryRunner runner = new QueryRunner();
- // Fetch the project
- Project project = null;
- ProjectResultHandler handler = new ProjectResultHandler();
- try {
- List<Project> projects =
+ return project;
+ }
+
+ private Project fetchProjectByName(Connection connection, String name)
+ throws ProjectManagerException {
+ QueryRunner runner = new QueryRunner();
+ // Fetch the project
+ Project project;
+ ProjectResultHandler handler = new ProjectResultHandler();
+ // select active project from db first, if not exist, select inactive one.
+ // At most one active project with the same name exists in db.
+ try {
+ List<Project> projects =
+ runner.query(connection,
+ ProjectResultHandler.SELECT_ACTIVE_PROJECT_BY_NAME, handler, name);
+ if (projects.isEmpty()) {
+ projects =
runner.query(connection,
ProjectResultHandler.SELECT_PROJECT_BY_NAME, handler, name);
if (projects.isEmpty()) {
throw new ProjectManagerException(
"No project with name " + name + " exists in db.");
}
-
- project = projects.get(0);
- } catch (SQLException e) {
- logger.error(ProjectResultHandler.SELECT_PROJECT_BY_NAME
- + " failed.");
- throw new ProjectManagerException(
- "Query for existing project failed. Project " + name, e);
}
+ project = projects.get(0);
+ } catch (SQLException e) {
+ logger.error(ProjectResultHandler.SELECT_PROJECT_BY_NAME
+ + " failed.");
+ throw new ProjectManagerException(
+ "Query for existing project failed. Project " + name, e);
+ }
- // Fetch the user permissions
- List<Triple<String, Boolean, Permission>> permissions =
- fetchPermissionsForProject(connection, project);
+ // Fetch the user permissions
+ List<Triple<String, Boolean, Permission>> permissions =
+ fetchPermissionsForProject(connection, project);
- for (Triple<String, Boolean, Permission> perm : permissions) {
- if (perm.getThird().toFlags() != 0) {
- if (perm.getSecond()) {
- project
- .setGroupPermission(perm.getFirst(), perm.getThird());
- } else {
- project.setUserPermission(perm.getFirst(), perm.getThird());
- }
+ for (Triple<String, Boolean, Permission> perm : permissions) {
+ if (perm.getThird().toFlags() != 0) {
+ if (perm.getSecond()) {
+ project
+ .setGroupPermission(perm.getFirst(), perm.getThird());
+ } else {
+ project.setUserPermission(perm.getFirst(), perm.getThird());
}
}
-
- return project;
}
+ return project;
+ }
+
private List<Triple<String, Boolean, Permission>> fetchPermissionsForProject(
Connection connection, Project project) throws ProjectManagerException {
ProjectPermissionsResultHandler permHander =
diff --git a/azkaban-common/src/main/java/azkaban/project/ProjectManager.java b/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
index b079bbd..6ea28c3 100644
--- a/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
+++ b/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
@@ -47,11 +47,6 @@ import azkaban.utils.Utils;
public class ProjectManager {
private static final Logger logger = Logger.getLogger(ProjectManager.class);
-
- private ConcurrentHashMap<Integer, Project> projectsById =
- new ConcurrentHashMap<Integer, Project>();
- private ConcurrentHashMap<String, Project> projectsByName =
- new ConcurrentHashMap<String, Project>();
private final ProjectLoader projectLoader;
private final Props props;
private final File tempDir;
@@ -83,28 +78,10 @@ public class ProjectManager {
// By instantiating an object of XmlValidatorManager, this will verify the
// config files for the validators.
new XmlValidatorManager(prop);
- loadAllProjects();
loadProjectWhiteList();
}
- private void loadAllProjects() {
- List<Project> projects;
- try {
- projects = projectLoader.fetchAllActiveProjects();
- } catch (ProjectManagerException e) {
- throw new RuntimeException("Could not load projects from store.", e);
- }
- for (Project proj : projects) {
- projectsByName.put(proj.getName(), proj);
- projectsById.put(proj.getId(), proj);
- }
-
- for (Project proj : projects) {
- loadAllProjectFlows(proj);
- }
- }
-
- private void loadAllProjectFlows(Project project) {
+ public void loadAllProjectFlows(Project project) {
try {
List<Flow> flows = projectLoader.fetchAllProjectFlows(project);
Map<String, Flow> flowMap = new HashMap<String, Flow>();
@@ -118,64 +95,65 @@ public class ProjectManager {
}
}
- public List<String> getProjectNames() {
- return new ArrayList<String>(projectsByName.keySet());
- }
-
public Props getProps() {
return props;
}
public List<Project> getUserProjects(User user) {
- ArrayList<Project> array = new ArrayList<Project>();
- for (Project project : projectsById.values()) {
+ ArrayList<Project> userProjects = new ArrayList<>();
+ for (Project project : getProjects()) {
Permission perm = project.getUserPermission(user);
if (perm != null
&& (perm.isPermissionSet(Type.ADMIN) || perm
.isPermissionSet(Type.READ))) {
- array.add(project);
+ userProjects.add(project);
}
}
- return array;
+ return userProjects;
}
public List<Project> getGroupProjects(User user) {
- List<Project> array = new ArrayList<Project>();
- for (Project project : projectsById.values()) {
+ List<Project> groupProjects = new ArrayList<>();
+ for (Project project : getProjects()) {
if (project.hasGroupPermission(user, Type.READ)) {
- array.add(project);
+ groupProjects.add(project);
}
}
- return array;
+ return groupProjects;
}
public List<Project> getUserProjectsByRegex(User user, String regexPattern) {
- List<Project> array = new ArrayList<Project>();
+ List<Project> userProjects = new ArrayList<>();
Pattern pattern;
try {
pattern = Pattern.compile(regexPattern, Pattern.CASE_INSENSITIVE);
} catch (PatternSyntaxException e) {
logger.error("Bad regex pattern " + regexPattern);
- return array;
+ return userProjects;
}
-
- for (Project project : projectsById.values()) {
+ for (Project project : getProjects()) {
Permission perm = project.getUserPermission(user);
if (perm != null
&& (perm.isPermissionSet(Type.ADMIN) || perm
.isPermissionSet(Type.READ))) {
if (pattern.matcher(project.getName()).find()) {
- array.add(project);
+ userProjects.add(project);
}
}
}
- return array;
+ return userProjects;
}
public List<Project> getProjects() {
- return new ArrayList<Project>(projectsById.values());
+ List<Project> projects;
+ try {
+ projects = projectLoader.fetchAllActiveProjects();
+ } catch (ProjectManagerException e) {
+ throw new RuntimeException("Could not load projects from store.", e);
+ }
+ return projects;
}
public List<Project> getProjectsByRegex(String regexPattern) {
@@ -196,61 +174,44 @@ public class ProjectManager {
}
/**
- * Checks if a project is active using project_name
- *
- * @param name
- */
- public Boolean isActiveProject(String name) {
- return projectsByName.containsKey(name);
- }
-
- /**
* Checks if a project is active using project_id
*
- * @param name
+ * @param id
*/
public Boolean isActiveProject(int id) {
- return projectsById.containsKey(id);
+ return getProject(id) != null;
}
/**
- * fetch active project from cache and inactive projects from db by
- * project_name
+ * fetch active project (boolean active = true) from DB by project_name
*
* @param name
* @return
*/
public Project getProject(String name) {
Project fetchedProject = null;
- if (isActiveProject(name)) {
- fetchedProject = projectsByName.get(name);
- } else {
- try {
- fetchedProject = projectLoader.fetchProjectByName(name);
- } catch (ProjectManagerException e) {
- logger.error("Could not load project from store.", e);
- }
+ try {
+ fetchedProject = projectLoader.fetchProjectByName(name);
+ loadAllProjectFlows(fetchedProject);
+ } catch (ProjectManagerException e) {
+ logger.error("Could not load project" + name + " from store.", e);
}
return fetchedProject;
}
/**
- * fetch active project from cache and inactive projects from db by
- * project_id
+ * fetch active project (boolean active = true) from DB by project_id
*
* @param id
* @return
*/
public Project getProject(int id) {
Project fetchedProject = null;
- if (isActiveProject(id)) {
- fetchedProject = projectsById.get(id);
- } else {
- try {
- fetchedProject = projectLoader.fetchProjectById(id);
- } catch (ProjectManagerException e) {
- logger.error("Could not load project from store.", e);
- }
+ try {
+ fetchedProject = projectLoader.fetchProjectById(id);
+ loadAllProjectFlows(fetchedProject);
+ } catch (ProjectManagerException e) {
+ logger.error("Could not load project" + id + " from store.", e);
}
return fetchedProject;
}
@@ -268,16 +229,10 @@ public class ProjectManager {
"Project names must start with a letter, followed by any number of letters, digits, '-' or '_'.");
}
- if (projectsByName.containsKey(projectName)) {
- throw new ProjectManagerException("Project already exists.");
- }
-
logger.info("Trying to create " + projectName + " by user "
+ creator.getUserId());
Project newProject =
projectLoader.createNewProject(projectName, description, creator);
- projectsByName.put(newProject.getName(), newProject);
- projectsById.put(newProject.getId(), newProject);
if (creatorDefaultPermissions) {
// Add permission to project
@@ -324,10 +279,6 @@ public class ProjectManager {
projectLoader.removeProject(project, deleter.getUserId());
projectLoader.postEvent(project, EventType.DELETED, deleter.getUserId(),
null);
-
- projectsByName.remove(project.getName());
- projectsById.remove(project.getId());
-
return project;
}
diff --git a/azkaban-common/src/test/java/azkaban/project/MockProjectLoader.java b/azkaban-common/src/test/java/azkaban/project/MockProjectLoader.java
index 42d57b7..6d13146 100644
--- a/azkaban-common/src/test/java/azkaban/project/MockProjectLoader.java
+++ b/azkaban-common/src/test/java/azkaban/project/MockProjectLoader.java
@@ -18,6 +18,7 @@ package azkaban.project;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
@@ -29,6 +30,7 @@ import azkaban.user.Permission;
import azkaban.user.User;
import azkaban.utils.Props;
import azkaban.utils.Triple;
+import java.util.concurrent.ConcurrentHashMap;
public class MockProjectLoader implements ProjectLoader {
public File dir;
@@ -37,30 +39,52 @@ public class MockProjectLoader implements ProjectLoader {
this.dir = dir;
}
+ private ConcurrentHashMap<Integer, Project> projectsById =
+ new ConcurrentHashMap<>();
+ private ConcurrentHashMap<String, Project> projectsByName =
+ new ConcurrentHashMap<>();
+
+ private static int projectId = 0;
+
@Override
public List<Project> fetchAllActiveProjects() throws ProjectManagerException {
- // TODO Auto-generated method stub
- return null;
+ ArrayList<Project> activeProjects = new ArrayList<>();
+ for(Project project : projectsById.values()){
+ if(project.isActive()){
+ activeProjects.add(project);
+ }
+ }
+ return activeProjects;
}
@Override
public Project fetchProjectById(int id) throws ProjectManagerException {
- // TODO Auto-generated method stub
- return null;
+ System.out.println("MockProjectLoader: fetch project by id " + id);
+ if(!projectsById.containsKey(id)){
+ throw new ProjectManagerException("Could not get project by id.");
+ }
+ return projectsById.get(id);
}
@Override
public Project createNewProject(String name, String description, User creator)
throws ProjectManagerException {
- // TODO Auto-generated method stub
- return null;
+ Project project = new Project(++projectId, name);
+ project.setDescription(description);
+ project.setActive(true);
+ projectsById.put(project.getId(), project);
+ projectsByName.put(project.getName(), project);
+ System.out.println("MockProjectLoader: Created project " + project.getName() +
+ ", id: " + project.getId() + ", description: " + description +
+ ", user: " + creator.getUserId());
+ return project;
}
@Override
public void removeProject(Project project, String user)
throws ProjectManagerException {
- // TODO Auto-generated method stub
-
+ project.setActive(false);
+ System.out.println("MockProjectLoader: removed project " + project.getName());
}
@Override
@@ -144,8 +168,7 @@ public class MockProjectLoader implements ProjectLoader {
@Override
public List<Flow> fetchAllProjectFlows(Project project)
throws ProjectManagerException {
- // TODO Auto-generated method stub
- return null;
+ return new ArrayList<>();
}
@Override
@@ -244,9 +267,12 @@ public class MockProjectLoader implements ProjectLoader {
}
-@Override
-public Project fetchProjectByName(String name) throws ProjectManagerException {
- // TODO Auto-generated method stub
- return null;
-}
+ @Override
+ public Project fetchProjectByName(String name) throws ProjectManagerException {
+ System.out.println("MockProjectLoader: fetch project by name " + name);
+ if(!projectsByName.containsKey(name)){
+ throw new ProjectManagerException("Could not get project by name.");
+ }
+ return projectsByName.get(name);
+ }
}
diff --git a/azkaban-common/src/test/java/azkaban/project/ProjectManagerTest.java b/azkaban-common/src/test/java/azkaban/project/ProjectManagerTest.java
new file mode 100644
index 0000000..ca731d3
--- /dev/null
+++ b/azkaban-common/src/test/java/azkaban/project/ProjectManagerTest.java
@@ -0,0 +1,182 @@
+package azkaban.project;
+
+import azkaban.user.User;
+import azkaban.utils.Props;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import static org.mockito.Mockito.*;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+public class ProjectManagerTest {
+ private ProjectManager manager;
+ private ProjectLoader loader;
+ private User user;
+ private static final String PROJECT_NAME = "myTest";
+ private static final String PROJECT_NAME_2 = "myTest_2";
+ private static final String PROJECT_DESCRIPTION = "This is to test project manager";
+ private static final String TEST_USER = "testUser";
+ private static final String FILE_TYPE = "zip";
+ private static final int PROJECT_ID = 1;
+ private static final int PROJECT_ID_2 = 2;
+ private static final int PROJECT_VERSION = 5;
+ private static final int PROJECT_VERSION_RETENTIION = 3;
+
+ @Before
+ public void setUp() throws Exception {
+ Props props = new Props();
+ loader = mock(ProjectLoader.class);
+ manager = new ProjectManager(loader, props);
+ user = new User(TEST_USER);
+ Project project1 = new Project(PROJECT_ID, PROJECT_NAME);
+ project1.setDescription(PROJECT_DESCRIPTION);
+ project1.setActive(true);
+ project1.setVersion(PROJECT_VERSION);
+
+ when(loader.createNewProject(PROJECT_NAME, PROJECT_DESCRIPTION, user)).thenReturn(project1);
+ when(loader.fetchProjectById(PROJECT_ID)).thenReturn(project1);
+ when(loader.fetchProjectByName(PROJECT_NAME)).thenReturn(project1);
+ when(loader.fetchAllProjectFlows(project1)).thenReturn(new ArrayList<>());
+ when(loader.getLatestProjectVersion(project1)).thenReturn(PROJECT_VERSION);
+
+ doAnswer(new Answer<Void>() {
+ public Void answer(InvocationOnMock invocation) {
+ project1.setActive(false);
+ return null;
+ }
+ }).when(loader).removeProject(project1, user.getUserId());
+
+ doAnswer(new Answer<Void>() {
+ public Void answer(InvocationOnMock invocation) {
+ project1.setVersion(PROJECT_VERSION + 1);
+ return null;
+ }
+ }).when(loader).changeProjectVersion(project1, PROJECT_VERSION + 1, user.getUserId());
+
+ doThrow(ProjectManagerException.class).when(loader).fetchAllProjectFlows(null);
+
+ }
+
+ @Test
+ public void testCreateProject() throws Exception {
+ System.out.println("TestCreateProject");
+ Project project = manager.createProject(PROJECT_NAME, PROJECT_DESCRIPTION, user);
+ verify(loader).postEvent(project, ProjectLogEvent.EventType.CREATED, user.getUserId(), null);
+ Assert.assertEquals("Project Id", PROJECT_ID, project.getId());
+ Assert.assertEquals("Project name", PROJECT_NAME, project.getName());
+ Assert.assertEquals("Project description", PROJECT_DESCRIPTION,
+ project.getDescription());
+ Assert.assertTrue("Project is active", project.isActive());
+ }
+
+ @Test(expected = ProjectManagerException.class)
+ public void testCreateProjectWithEmptyName() throws Exception {
+ System.out.println("TestCreateProjectWithEmptyName");
+ manager.createProject(null, PROJECT_DESCRIPTION, user);
+ }
+
+ @Test(expected = ProjectManagerException.class)
+ public void testCreateProjectWithInvalidName() throws Exception {
+ System.out.println("TestCreateProjectWithInvalidName");
+ //Project name must start with a letter, test invalid project name "123", should throw exception
+ manager.createProject("123", PROJECT_DESCRIPTION, user);
+ }
+
+ @Test(expected = ProjectManagerException.class)
+ public void testCreateProjectWithEmptyDescription() throws Exception {
+ System.out.println("testCreateProjectWithEmptyDescription");
+ manager.createProject(PROJECT_NAME, null, user);
+ }
+
+ @Test(expected = ProjectManagerException.class)
+ public void testCreateProjectWithEmptyUser() throws Exception {
+ System.out.println("testCreateProjectWithEmptyUser");
+ manager.createProject(PROJECT_NAME, PROJECT_DESCRIPTION, null);
+ }
+
+ @Test
+ public void testRemoveProject() throws Exception {
+ System.out.println("TestRemoveProject");
+ Project project = manager.createProject(PROJECT_NAME, PROJECT_DESCRIPTION, user);
+ manager.removeProject(project, user);
+ verify(loader).removeProject(project, user.getUserId());
+ verify(loader).postEvent(project, ProjectLogEvent.EventType.DELETED, user.getUserId(),
+ null);
+ Project fetchedProject = manager.getProject(project.getId());
+ verify(loader).fetchProjectById(project.getId());
+ verify(loader).fetchAllProjectFlows(project);
+ Assert.assertFalse(fetchedProject.isActive());
+ }
+
+ @Test
+ public void testUploadProject() throws Exception {
+ System.out.println("TestUploadProject");
+ Project project = manager.createProject(PROJECT_NAME, PROJECT_DESCRIPTION, user);
+ File testDir = new File(this.getClass().getClassLoader().getResource("project/testjob/testjob.zip").getFile());
+ System.out.println("Uploading zip file: " + testDir.getAbsolutePath());
+ Props props = new Props();
+ manager.uploadProject(project, testDir, FILE_TYPE, user, props);
+ verify(loader).uploadProjectFile(project, PROJECT_VERSION + 1, FILE_TYPE, testDir.getName(),
+ testDir, user.getUserId());
+ verify(loader).uploadFlows(eq(project), eq(PROJECT_VERSION + 1), anyCollection());
+ verify(loader).changeProjectVersion(project, PROJECT_VERSION + 1, user.getUserId());
+ //uploadProjectProperties should be called twice, one for jobProps, the other for propProps
+ verify(loader, times(2)).uploadProjectProperties(eq(project), anyList());
+ verify(loader).postEvent(project, ProjectLogEvent.EventType.UPLOADED, user.getUserId(),
+ "Uploaded project files zip " + testDir.getName());
+ verify(loader).cleanOlderProjectVersion(project.getId(), PROJECT_VERSION + 1 - PROJECT_VERSION_RETENTIION);
+ }
+
+ @Test
+ public void testFetchProjectByName() throws Exception {
+ System.out.println("TestFetchProjectByName");
+ Project project = manager.createProject(PROJECT_NAME, PROJECT_DESCRIPTION, user);
+ Project fetchedProject = manager.getProject(project.getName());
+ Assert.assertEquals("Fetched project by name", project, fetchedProject);
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testFetchInvalidProjectByName() throws Exception {
+ System.out.println("TestFetchInvalidProjectByName");
+ manager.createProject(PROJECT_NAME, PROJECT_DESCRIPTION, user);
+ manager.getProject("Invalid_Project");
+ }
+
+ @Test
+ public void testFetchProjectById() throws Exception {
+ System.out.println("TestFetchProjectById");
+ Project project = manager.createProject(PROJECT_NAME, PROJECT_DESCRIPTION, user);
+ Project fetchedProject = manager.getProject(project.getId());
+ Assert.assertEquals("Fetched project by id", project, fetchedProject);
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testFetchInvalidProjectById() throws Exception {
+ System.out.println("TestFetchInvalidProjectById");
+ manager.createProject(PROJECT_NAME, PROJECT_DESCRIPTION, user);
+ manager.getProject(100);
+ }
+
+ @Test
+ public void testFetchAllProjects() throws Exception {
+ System.out.println("TestFetchAllProjects");
+ List<Project> projects = new ArrayList<>();
+ Project new_project1 = manager.createProject(PROJECT_NAME, PROJECT_DESCRIPTION, user);
+ Project project2 = new Project(PROJECT_ID_2, PROJECT_NAME_2);
+ project2.setDescription(PROJECT_DESCRIPTION);
+ project2.setActive(true);
+ project2.setVersion(PROJECT_VERSION);
+ when(loader.createNewProject(PROJECT_NAME_2, PROJECT_DESCRIPTION, user)).thenReturn(project2);
+ Project new_project2 = manager.createProject(PROJECT_NAME_2, PROJECT_DESCRIPTION, user);
+ projects.add(new_project1);
+ projects.add(new_project2);
+
+ when(loader.fetchAllActiveProjects()).thenReturn(projects);
+ List<Project> fetchedProjects = manager.getProjects();
+ Assert.assertEquals("Fetched projects: ", projects, fetchedProjects);
+ }
+}
