Details
diff --git a/azkaban-common/src/main/java/azkaban/project/JdbcProjectLoader.java b/azkaban-common/src/main/java/azkaban/project/JdbcProjectLoader.java
index 7bbd63f..a707ac2 100644
--- a/azkaban-common/src/main/java/azkaban/project/JdbcProjectLoader.java
+++ b/azkaban-common/src/main/java/azkaban/project/JdbcProjectLoader.java
@@ -171,6 +171,11 @@ public class JdbcProjectLoader extends AbstractJdbcLoader implements
return project;
}
+ /**
+ * Fetch first project with a given name {@inheritDoc}
+ *
+ * @see azkaban.project.ProjectLoader#fetchProjectByName(java.lang.String)
+ */
@Override
public Project fetchProjectByName(final String name)
throws ProjectManagerException {
@@ -190,23 +195,17 @@ public class JdbcProjectLoader extends AbstractJdbcLoader implements
throws ProjectManagerException {
final QueryRunner runner = new QueryRunner();
// Fetch the project
- final Project project;
+ Project project = null;
final ProjectResultHandler handler = new ProjectResultHandler();
- // select active project from db first, if not exist, select inactive one.
- // At most one active project with the same name exists in db.
try {
- List<Project> projects =
+ final List<Project> projects =
runner.query(connection,
- ProjectResultHandler.SELECT_ACTIVE_PROJECT_BY_NAME, handler, name);
+ ProjectResultHandler.SELECT_PROJECT_BY_NAME, handler, name);
if (projects.isEmpty()) {
- projects =
- runner.query(connection,
- ProjectResultHandler.SELECT_PROJECT_BY_NAME, handler, name);
- if (projects.isEmpty()) {
- throw new ProjectManagerException(
- "No project with name " + name + " exists in db.");
- }
+ throw new ProjectManagerException(
+ "No project with name " + name + " exists in db.");
}
+
project = projects.get(0);
} catch (final SQLException e) {
logger.error(ProjectResultHandler.SELECT_PROJECT_BY_NAME
@@ -410,20 +409,21 @@ public class JdbcProjectLoader extends AbstractJdbcLoader implements
/**
* Insert a new version record to TABLE project_versions before uploading files.
*
- * The reason for this operation: When error chunking happens in remote mysql server, incomplete
- * file data remains in DB, and an SQL exception is thrown. If we don't have this operation before
- * uploading file, the SQL exception prevents AZ from creating the new version record in Table
- * project_versions. However, the Table project_files still reserve the incomplete files, which
- * causes troubles when uploading a new file: Since the version in TABLE project_versions is still
- * old, mysql will stop inserting new files to db.
+ * The reason for this operation:
+ * When error chunking happens in remote mysql server, incomplete file data remains
+ * in DB, and an SQL exception is thrown. If we don't have this operation before uploading file,
+ * the SQL exception prevents AZ from creating the new version record in Table project_versions.
+ * However, the Table project_files still reserve the incomplete files, which causes troubles
+ * when uploading a new file: Since the version in TABLE project_versions is still old, mysql will stop
+ * inserting new files to db.
*
- * Why this operation is safe: When AZ uploads a new zip file, it always fetches the latest
- * version proj_v from TABLE project_version, proj_v+1 will be used as the new version for the
- * uploading files.
+ * Why this operation is safe:
+ * When AZ uploads a new zip file, it always fetches the latest version proj_v from TABLE project_version,
+ * proj_v+1 will be used as the new version for the uploading files.
*
- * Assume error chunking happens on day 1. proj_v is created for this bad file (old file version +
- * 1). When we upload a new project zip in day2, new file in day 2 will use the new version
- * (proj_v + 1). When file uploading completes, AZ will clean all old chunks in DB afterward.
+ * Assume error chunking happens on day 1. proj_v is created for this bad file (old file version + 1).
+ * When we upload a new project zip in day2, new file in day 2 will use the new version (proj_v + 1).
+ * When file uploading completes, AZ will clean all old chunks in DB afterward.
*/
private void addProjectToProjectVersions(final Connection connection,
final int projectId,
@@ -840,6 +840,10 @@ public class JdbcProjectLoader extends AbstractJdbcLoader implements
/**
* Get all the logs for a given project
+ *
+ * @param project
+ * @return
+ * @throws ProjectManagerException
*/
@Override
public List<ProjectLogEvent> getProjectEvents(final Project project, final int num,
diff --git a/azkaban-common/src/main/java/azkaban/project/ProjectManager.java b/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
index 15c1ce0..f051e85 100644
--- a/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
+++ b/azkaban-common/src/main/java/azkaban/project/ProjectManager.java
@@ -40,6 +40,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import java.util.zip.ZipFile;
@@ -56,6 +57,10 @@ public class ProjectManager {
private final File tempDir;
private final int projectVersionRetention;
private final boolean creatorDefaultPermissions;
+ private final ConcurrentHashMap<Integer, Project> projectsById =
+ new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<String, Project> projectsByName =
+ new ConcurrentHashMap<>();
@Inject
public ProjectManager(final ProjectLoader loader, final StorageManager storageManager,
@@ -86,10 +91,28 @@ public class ProjectManager {
// By instantiating an object of XmlValidatorManager, this will verify the
// config files for the validators.
new XmlValidatorManager(prop);
+ loadAllProjects();
loadProjectWhiteList();
}
- public void loadAllProjectFlows(final Project project) {
+ private void loadAllProjects() {
+ final List<Project> projects;
+ try {
+ projects = this.projectLoader.fetchAllActiveProjects();
+ } catch (final ProjectManagerException e) {
+ throw new RuntimeException("Could not load projects from store.", e);
+ }
+ for (final Project proj : projects) {
+ this.projectsByName.put(proj.getName(), proj);
+ this.projectsById.put(proj.getId(), proj);
+ }
+
+ for (final Project proj : projects) {
+ loadAllProjectFlows(proj);
+ }
+ }
+
+ private void loadAllProjectFlows(final Project project) {
try {
final List<Flow> flows = this.projectLoader.fetchAllProjectFlows(project);
final Map<String, Flow> flowMap = new HashMap<>();
@@ -103,65 +126,64 @@ public class ProjectManager {
}
}
+ public List<String> getProjectNames() {
+ return new ArrayList<>(this.projectsByName.keySet());
+ }
+
public Props getProps() {
return this.props;
}
public List<Project> getUserProjects(final User user) {
- final ArrayList<Project> userProjects = new ArrayList<>();
- for (final Project project : getProjects()) {
+ final ArrayList<Project> array = new ArrayList<>();
+ for (final Project project : this.projectsById.values()) {
final Permission perm = project.getUserPermission(user);
if (perm != null
&& (perm.isPermissionSet(Type.ADMIN) || perm
.isPermissionSet(Type.READ))) {
- userProjects.add(project);
+ array.add(project);
}
}
- return userProjects;
+ return array;
}
public List<Project> getGroupProjects(final User user) {
- final List<Project> groupProjects = new ArrayList<>();
- for (final Project project : getProjects()) {
+ final List<Project> array = new ArrayList<>();
+ for (final Project project : this.projectsById.values()) {
if (project.hasGroupPermission(user, Type.READ)) {
- groupProjects.add(project);
+ array.add(project);
}
}
- return groupProjects;
+ return array;
}
public List<Project> getUserProjectsByRegex(final User user, final String regexPattern) {
- final List<Project> userProjects = new ArrayList<>();
+ final List<Project> array = new ArrayList<>();
final Pattern pattern;
try {
pattern = Pattern.compile(regexPattern, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) {
logger.error("Bad regex pattern " + regexPattern);
- return userProjects;
+ return array;
}
- for (final Project project : getProjects()) {
+
+ for (final Project project : this.projectsById.values()) {
final Permission perm = project.getUserPermission(user);
if (perm != null
&& (perm.isPermissionSet(Type.ADMIN) || perm
.isPermissionSet(Type.READ))) {
if (pattern.matcher(project.getName()).find()) {
- userProjects.add(project);
+ array.add(project);
}
}
}
- return userProjects;
+ return array;
}
public List<Project> getProjects() {
- final List<Project> projects;
- try {
- projects = this.projectLoader.fetchAllActiveProjects();
- } catch (final ProjectManagerException e) {
- throw new RuntimeException("Could not load projects from store.", e);
- }
- return projects;
+ return new ArrayList<>(this.projectsById.values());
}
public List<Project> getProjectsByRegex(final String regexPattern) {
@@ -182,36 +204,61 @@ public class ProjectManager {
}
/**
+ * Checks if a project is active using project_name
+ *
+ * @param name
+ */
+ public Boolean isActiveProject(final String name) {
+ return this.projectsByName.containsKey(name);
+ }
+
+ /**
* Checks if a project is active using project_id
+ *
+ * @param name
*/
public Boolean isActiveProject(final int id) {
- return getProject(id) != null;
+ return this.projectsById.containsKey(id);
}
/**
- * fetch active project (boolean active = true) from DB by project_name
+ * fetch active project from cache and inactive projects from db by
+ * project_name
+ *
+ * @param name
+ * @return
*/
public Project getProject(final String name) {
Project fetchedProject = null;
- try {
- fetchedProject = this.projectLoader.fetchProjectByName(name);
- loadAllProjectFlows(fetchedProject);
- } catch (final ProjectManagerException e) {
- logger.error("Could not load project" + name + " from store.", e);
+ if (isActiveProject(name)) {
+ fetchedProject = this.projectsByName.get(name);
+ } else {
+ try {
+ fetchedProject = this.projectLoader.fetchProjectByName(name);
+ } catch (final ProjectManagerException e) {
+ logger.error("Could not load project from store.", e);
+ }
}
return fetchedProject;
}
/**
- * fetch active project (boolean active = true) from DB by project_id
+ * fetch active project from cache and inactive projects from db by
+ * project_id
+ *
+ * @param id
+ * @return
*/
public Project getProject(final int id) {
Project fetchedProject = null;
- try {
- fetchedProject = this.projectLoader.fetchProjectById(id);
- loadAllProjectFlows(fetchedProject);
- } catch (final ProjectManagerException e) {
- logger.error("Could not load project" + id + " from store.", e);
+ if (isActiveProject(id)) {
+ fetchedProject = this.projectsById.get(id);
+ } else {
+ try {
+ fetchedProject = this.projectLoader.fetchProjectById(id);
+ } catch (final ProjectManagerException e) {
+ logger.error("Could not load project from store.", e);
+ }
}
return fetchedProject;
}
@@ -229,10 +276,16 @@ public class ProjectManager {
"Project names must start with a letter, followed by any number of letters, digits, '-' or '_'.");
}
+ if (this.projectsByName.containsKey(projectName)) {
+ throw new ProjectManagerException("Project already exists.");
+ }
+
logger.info("Trying to create " + projectName + " by user "
+ creator.getUserId());
final Project newProject =
this.projectLoader.createNewProject(projectName, description, creator);
+ this.projectsByName.put(newProject.getName(), newProject);
+ this.projectsById.put(newProject.getId(), newProject);
if (this.creatorDefaultPermissions) {
// Add permission to project
@@ -258,6 +311,11 @@ public class ProjectManager {
/**
* Permanently delete all project files and properties data for all versions
* of a project and log event in project_events table
+ *
+ * @param project
+ * @param deleter
+ * @return
+ * @throws ProjectManagerException
*/
public synchronized Project purgeProject(final Project project, final User deleter)
throws ProjectManagerException {
@@ -274,6 +332,10 @@ public class ProjectManager {
this.projectLoader.removeProject(project, deleter.getUserId());
this.projectLoader.postEvent(project, EventType.DELETED, deleter.getUserId(),
null);
+
+ this.projectsByName.remove(project.getName());
+ this.projectsById.remove(project.getId());
+
return project;
}
@@ -389,9 +451,11 @@ public class ProjectManager {
* {@ProjectFileHandler.deleteLocalFile}
* to delete the temporary file.
*
+ * @param project
* @param version - latest version is used if value is -1
- * @return ProjectFileHandler - null if can't find project zip file based on project name and
- * version
+ * @return ProjectFileHandler - null if can't find project zip file based on
+ * project name and version
+ * @throws ProjectManagerException
*/
public ProjectFileHandler getProjectFileHandler(final Project project, int version)
throws ProjectManagerException {
diff --git a/azkaban-common/src/test/java/azkaban/project/MockProjectLoader.java b/azkaban-common/src/test/java/azkaban/project/MockProjectLoader.java
index c16e77d..c05b596 100644
--- a/azkaban-common/src/test/java/azkaban/project/MockProjectLoader.java
+++ b/azkaban-common/src/test/java/azkaban/project/MockProjectLoader.java
@@ -24,20 +24,13 @@ import azkaban.utils.Props;
import azkaban.utils.Triple;
import java.io.File;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
public class MockProjectLoader implements ProjectLoader {
- private static int projectId = 0;
- private final ConcurrentHashMap<Integer, Project> projectsById =
- new ConcurrentHashMap<>();
- private final ConcurrentHashMap<String, Project> projectsByName =
- new ConcurrentHashMap<>();
public File dir;
public MockProjectLoader(final File dir) {
@@ -46,43 +39,28 @@ public class MockProjectLoader implements ProjectLoader {
@Override
public List<Project> fetchAllActiveProjects() throws ProjectManagerException {
- final ArrayList<Project> activeProjects = new ArrayList<>();
- for (final Project project : this.projectsById.values()) {
- if (project.isActive()) {
- activeProjects.add(project);
- }
- }
- return activeProjects;
+ // TODO Auto-generated method stub
+ return null;
}
@Override
public Project fetchProjectById(final int id) throws ProjectManagerException {
- System.out.println("MockProjectLoader: fetch project by id " + id);
- if (!this.projectsById.containsKey(id)) {
- throw new ProjectManagerException("Could not get project by id.");
- }
- return this.projectsById.get(id);
+ // TODO Auto-generated method stub
+ return null;
}
@Override
public Project createNewProject(final String name, final String description, final User creator)
throws ProjectManagerException {
- final Project project = new Project(++projectId, name);
- project.setDescription(description);
- project.setActive(true);
- this.projectsById.put(project.getId(), project);
- this.projectsByName.put(project.getName(), project);
- System.out.println("MockProjectLoader: Created project " + project.getName() +
- ", id: " + project.getId() + ", description: " + description +
- ", user: " + creator.getUserId());
- return project;
+ // TODO Auto-generated method stub
+ return null;
}
@Override
public void removeProject(final Project project, final String user)
throws ProjectManagerException {
- project.setActive(false);
- System.out.println("MockProjectLoader: removed project " + project.getName());
+ // TODO Auto-generated method stub
+
}
@Override
@@ -172,7 +150,8 @@ public class MockProjectLoader implements ProjectLoader {
@Override
public List<Flow> fetchAllProjectFlows(final Project project)
throws ProjectManagerException {
- return new ArrayList<>();
+ // TODO Auto-generated method stub
+ return null;
}
@Override
@@ -273,10 +252,7 @@ public class MockProjectLoader implements ProjectLoader {
@Override
public Project fetchProjectByName(final String name) throws ProjectManagerException {
- System.out.println("MockProjectLoader: fetch project by name " + name);
- if (!this.projectsByName.containsKey(name)) {
- throw new ProjectManagerException("Could not get project by name.");
- }
- return this.projectsByName.get(name);
+ // TODO Auto-generated method stub
+ return null;
}
}
