azkaban-developers

diff --git a/build.gradle b/build.gradle
index aaea92f..168c745 100644
--- a/build.gradle
+++ b/build.gradle
@@ -41,6 +41,9 @@ configurations {
     compile {
         description = 'compile classpath'
     }
+    generateRestli {
+        transitive = true
+    }
     test {
         extendsFrom compile
     }
@@ -48,11 +51,13 @@ configurations {
 configurations.compile {
     description = 'compile classpath'
 }
+pegasusVersion = '1.15.7'
 
 dependencies {
   compile (
     [group: 'commons-collections', name:'commons-collections', version: '3.2.1'],
     [group: 'commons-configuration', name:'commons-configuration', version: '1.8'],
+    [group: 'commons-codec', name:'commons-codec', version: '1.9'],
     [group: 'commons-dbcp', name:'commons-dbcp', version: '1.4'],
     [group: 'commons-dbutils', name:'commons-dbutils', version: '1.5'],
     [group: 'org.apache.commons', name:'commons-email', version: '1.2'],
@@ -68,7 +73,6 @@ dependencies {
     [group: 'org.apache.httpcomponents', name:'httpcore', version: '4.2.1'],
     [group: 'org.codehaus.jackson', name:'jackson-core-asl', version: '1.9.5'],
     [group: 'org.codehaus.jackson', name:'jackson-mapper-asl',version: '1.9.5'],
-    [group: 'org.codehaus.jackson', name:'jackson-core-asl', version: '1.9.5'],
     [group: 'org.mortbay.jetty', name:'jetty', version: '6.1.26'],
     [group: 'org.mortbay.jetty', name:'jetty-util', version: '6.1.26'],
     [group: 'joda-time', name:'joda-time', version: '2.0'],
@@ -78,7 +82,21 @@ dependencies {
     [group: 'mysql', name:'mysql-connector-java', version: '5.1.28'],
     [group: 'javax.servlet', name:'servlet-api', version: '2.5'],
     [group: 'org.slf4j', name:'slf4j-api', version: '1.6.1'],
-    [group: 'org.apache.velocity', name:'velocity', version: '1.7']
+    [group: 'org.apache.velocity', name:'velocity', version: '1.7'],
+    [group: 'com.linkedin.pegasus', name: 'gradle-plugins', version: pegasusVersion],
+    [group: 'com.linkedin.pegasus', name: 'pegasus-common', version: pegasusVersion],
+    [group: 'com.linkedin.pegasus', name: 'restli-common', version: pegasusVersion],
+    [group: 'com.linkedin.pegasus', name: 'restli-server', version: pegasusVersion],
+    [group: 'com.linkedin.pegasus', name: 'data', version: pegasusVersion],
+    [group: 'com.linkedin.pegasus', name: 'r2', version: pegasusVersion],
+    [group: 'com.linkedin.pegasus', name: 'li-jersey-uri', version: pegasusVersion],
+    [group: 'com.linkedin.parseq', name: 'parseq', version: '1.3.7'],
+    [group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.3.2']
+  )
+  
+  generateRestli (
+      [group: 'com.linkedin.pegasus', name:'generator', version: pegasusVersion],
+      [group: 'com.linkedin.pegasus', name:'restli-tools', version: pegasusVersion]
   )
 
   testCompile (
@@ -86,6 +104,14 @@ dependencies {
   )
 }
 
+sourceSets {
+    main {
+        java {
+            srcDirs 'src/main/java', 'src/restli/generatedJava', 'src/restli/java'
+        }
+    }
+}
+
 jar {
     baseName =  'azkaban'
     manifest {
@@ -96,6 +122,27 @@ jar {
     }
 }
 
+task restliTemplateGenerator(type: JavaExec) {
+    mkdir 'src/restli/generatedJava'
+
+    main = 'com.linkedin.pegasus.generator.PegasusDataTemplateGenerator'
+    args = ['src/restli/generatedJava','src/restli/schemas']
+    classpath = configurations.generateRestli
+}
+
+task restliRestSpecGenerator(dependsOn: [restliTemplateGenerator], type: JavaExec) << {
+    mkdir 'src/restli/generatedRestSpec'
+
+    main = 'com.linkedin.restli.tools.idlgen.RestLiResourceModelExporterCmdLineApp'
+    args = ['-outdir', 'src/restli/generatedRestSpec', '-sourcepath', 'src/restli/java']
+    classpath = configurations.generateRestli
+}
+
+task restli(dependsOn: restliTemplateGenerator) << {
+}
+
+compileJava.dependsOn.add('restli')
+
 eclipse.classpath.file {
     // Erase the whole classpath
     beforeMerged {
@@ -243,7 +290,7 @@ task packageSql(type: Tar) {
     String packageDir = 'build/package/sql'
 
     println 'Creating Azkaban SQL Scripts into ' + packageDir
-    mkdir packageDir
+	mkdir packageDir
 
     println 'Copying SQL files'
     copy {
@@ -280,6 +327,7 @@ task packageExec(type: Tar, dependsOn: [jar, 'createVersionFile']) {
     appendix = 'exec-server'
     String packageDir = 'build/package/' + baseName + '-' + appendix
 
+    delete packageDir
     println 'Creating Azkaban Executor Server Package into ' + packageDir
     mkdir packageDir
     mkdir packageDir + '/extlib'
@@ -290,10 +338,11 @@ task packageExec(type: Tar, dependsOn: [jar, 'createVersionFile']) {
         from('src/package/execserver')
         into(packageDir)
     }
-
-    println 'Copying Azkaban lib'
+    
+    String jarFile = 'build/libs/' + baseName + '-' + version + '.jar'
+    println 'Copying Azkaban lib ' + jarFile
     copy {
-        from('build/libs')
+        from(jarFile)
         into(packageDir + '/lib')
     }
 
@@ -335,6 +384,7 @@ task packageExec(type: Tar, dependsOn: [jar, 'createVersionFile']) {
 task packageWeb(type: Tar, dependsOn: [jar, 'web', 'createVersionFile']) {
     appendix = 'web-server'
     String packageDir = 'build/package/' + baseName + '-' + appendix
+    delete packageDir
 
     println 'Creating Azkaban Web Server Package into ' + packageDir
     mkdir packageDir
@@ -346,10 +396,12 @@ task packageWeb(type: Tar, dependsOn: [jar, 'web', 'createVersionFile']) {
         from('src/package/webserver')
         into(packageDir)
     }
+    
+    String jarFile = 'build/libs/' + baseName + '-' + version + '.jar'
+    println 'Copying Azkaban lib ' + jarFile
 
-    println 'Copying Azkaban lib'
     copy {
-        from('build/libs')
+        from(jarFile)
         into(packageDir + '/lib')
     }
 

diff --git a/src/main/java/azkaban/user/UserManager.java b/src/main/java/azkaban/user/UserManager.java
index 071b307..253e817 100644
--- a/src/main/java/azkaban/user/UserManager.java
+++ b/src/main/java/azkaban/user/UserManager.java
@@ -34,7 +34,7 @@ public interface UserManager {
 	 * @throws UserManagerException If the username/password combination doesn't exist.
 	 */
 	public User getUser(String username, String password) throws UserManagerException;
-	
+
 	/**
 	 * Returns true if the user is valid. This is used when adding permissions for users
 	 * 

diff --git a/src/main/java/azkaban/webapp/AzkabanWebServer.java b/src/main/java/azkaban/webapp/AzkabanWebServer.java
index ebb0aa0..65b656b 100644
--- a/src/main/java/azkaban/webapp/AzkabanWebServer.java
+++ b/src/main/java/azkaban/webapp/AzkabanWebServer.java
@@ -50,6 +50,8 @@ import org.mortbay.jetty.servlet.DefaultServlet;
 import org.mortbay.jetty.servlet.ServletHolder;
 import org.mortbay.thread.QueuedThreadPool;
 
+import com.linkedin.restli.server.RestliServlet;
+
 import azkaban.alert.Alerter;
 import azkaban.database.AzkabanDatabaseSetup;
 import azkaban.executor.ExecutorManager;
@@ -158,6 +160,10 @@ public class AzkabanWebServer extends AzkabanServer {
 	private MBeanServer mbeanServer;
 	private ArrayList<ObjectName> registeredMBeans = new ArrayList<ObjectName>();
 
+	public static AzkabanWebServer getInstance() {
+		return app;
+	}
+	
 	/**
 	 * Constructor usually called by tomcat AzkabanServletContext to create the
 	 * initial server
@@ -762,6 +768,10 @@ public class AzkabanWebServer extends AzkabanServer {
 		root.addServlet(new ServletHolder(new ScheduleServlet()),"/schedule");
 		root.addServlet(new ServletHolder(new JMXHttpServlet()),"/jmx");
 		root.addServlet(new ServletHolder(new TriggerManagerServlet()),"/triggers");
+
+		ServletHolder restliHolder = new ServletHolder(new RestliServlet());
+		restliHolder.setInitParameter("resourcePackages", "azkaban.restli");
+		root.addServlet(restliHolder, "/restli/*");
 		
 		String viewerPluginDir = azkabanSettings.getString("viewer.plugin.dir", "plugins/viewer");
 		loadViewerPlugins(root, viewerPluginDir, app.getVelocityEngine());

diff --git a/src/restli/.gitignore b/src/restli/.gitignore
new file mode 100644
index 0000000..a6ad54c
--- /dev/null
+++ b/src/restli/.gitignore
@@ -0,0 +1,2 @@
+generatedJava
+generatedRestSpec

diff --git a/src/restli/java/azkaban/restli/ProjectManagerResource.java b/src/restli/java/azkaban/restli/ProjectManagerResource.java
new file mode 100644
index 0000000..faebbc8
--- /dev/null
+++ b/src/restli/java/azkaban/restli/ProjectManagerResource.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright 2014 LinkedIn Corp.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package azkaban.restli;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+import javax.servlet.ServletException;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.log4j.Logger;
+
+import azkaban.project.Project;
+import azkaban.project.ProjectManager;
+import azkaban.project.ProjectManagerException;
+import azkaban.user.Permission;
+import azkaban.user.User;
+import azkaban.user.UserManagerException;
+import azkaban.utils.Utils;
+import azkaban.webapp.AzkabanWebServer;
+
+import com.linkedin.restli.server.annotations.Action;
+import com.linkedin.restli.server.annotations.ActionParam;
+import com.linkedin.restli.server.annotations.RestLiActions;
+import com.linkedin.restli.server.resources.ResourceContextHolder;
+
+@RestLiActions(name = "project", namespace = "azkaban.restli")
+public class ProjectManagerResource extends ResourceContextHolder {
+	private static final Logger logger = Logger.getLogger(ProjectManagerResource.class);
+	
+	public AzkabanWebServer getAzkaban() {
+		return AzkabanWebServer.getInstance();
+	}
+	
+	@Action(name = "deploy")
+	public String deploy(
+			@ActionParam("sessionId") String sessionId,
+			@ActionParam("projectName") String projectName,
+			@ActionParam("packageUrl") String packageUrl)
+			throws ProjectManagerException, UserManagerException, ServletException, IOException {
+		logger.info("Deploy called. {sessionId: " + sessionId +
+				", projectName: " + projectName + 
+				", packageUrl:" + packageUrl + "}");
+		
+		String ip = (String)this.getContext().getRawRequestContext().getLocalAttr("REMOTE_ADDR");
+		User user = ResourceUtils.getUserFromSessionId(sessionId, ip);
+		ProjectManager projectManager = getAzkaban().getProjectManager();
+		Project project = projectManager.getProject(projectName);
+		if (project == null) {
+			throw new ProjectManagerException("Project '" + projectName + "' not found.");
+		}
+		
+		if (!ResourceUtils.hasPermission(project, user, Permission.Type.WRITE)) {
+			String errorMsg = "User " + user.getUserId() + " has no permission to write to project " + project.getName();
+			logger.error(errorMsg);
+			throw new ProjectManagerException(errorMsg);
+		}
+
+		// Deploy stuff here. Move the code to a more formal area later.
+		logger.info("Downloading file from " + packageUrl);
+		URL url = null;
+		InputStream urlFileInputStream = null;
+		try {
+			url = new URL(packageUrl);
+			InputStream in = url.openStream();
+			urlFileInputStream = new BufferedInputStream(in);
+		} catch (MalformedURLException e) {
+			String errorMsg = "Url " + packageUrl + " is malformed.";
+			logger.error(errorMsg, e);
+			throw new ProjectManagerException(errorMsg, e);
+		} catch (IOException e) {
+			String errorMsg = "Error opening input stream to " + packageUrl;
+			logger.error(errorMsg, e);
+			throw new ProjectManagerException(errorMsg, e);
+		}
+		
+		String filename = getFileName(url.getFile());
+
+		File tempDir = Utils.createTempDir();
+		OutputStream fileOutputStream = null;
+		try {
+			logger.error("Downloading " + filename);
+			File archiveFile = new File(tempDir, filename);
+			fileOutputStream = new BufferedOutputStream(new FileOutputStream(archiveFile));
+			IOUtils.copy(urlFileInputStream, fileOutputStream);
+			
+			logger.error("Downloaded to " + archiveFile.toString() + " " + archiveFile.length() + " bytes.");
+			projectManager.uploadProject(project, archiveFile, "zip", user);
+		} catch (Exception e) {
+			logger.info("Installation Failed.", e);
+			String error = e.getMessage();
+			if (error.length() > 512) {
+				error = error.substring(0, 512) + "\nToo many errors to display.\n";
+			}
+			
+			throw new ProjectManagerException("Installation failed: " + error);
+		}
+		finally {
+			if (tempDir.exists()) {
+				FileUtils.deleteDirectory(tempDir);
+			}
+			if (urlFileInputStream != null) {
+				urlFileInputStream.close();
+			}
+			if (fileOutputStream != null) {
+				fileOutputStream.close();
+			}
+		}
+		
+		return Integer.toString(project.getVersion());
+	}
+
+	private String getFileName(String file) {
+		return file.substring(file.lastIndexOf("/") + 1);
+	}
+}

diff --git a/src/restli/java/azkaban/restli/ResourceUtils.java b/src/restli/java/azkaban/restli/ResourceUtils.java
new file mode 100644
index 0000000..18229cf
--- /dev/null
+++ b/src/restli/java/azkaban/restli/ResourceUtils.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2014 LinkedIn Corp.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package azkaban.restli;
+
+import azkaban.project.Project;
+import azkaban.user.Permission;
+import azkaban.user.Role;
+import azkaban.user.User;
+import azkaban.user.UserManager;
+import azkaban.user.UserManagerException;
+import azkaban.webapp.AzkabanWebServer;
+import azkaban.webapp.session.Session;
+
+public class ResourceUtils {
+	
+	public static boolean hasPermission(Project project, User user, Permission.Type type) {
+		UserManager userManager = AzkabanWebServer.getInstance().getUserManager();
+		if (project.hasPermission(user, type)) {
+			return true;
+		}
+		
+		for (String roleName: user.getRoles()) {
+			Role role = userManager.getRole(roleName);
+			if (role.getPermission().isPermissionSet(type) || 
+				role.getPermission().isPermissionSet(Permission.Type.ADMIN)) {
+				return true;
+			}
+		}
+		
+		return false;
+	}
+	
+	public static User getUserFromSessionId(String sessionId, String ip) throws UserManagerException {
+		Session session = AzkabanWebServer.getInstance().getSessionCache().getSession(sessionId);
+		if (session == null) {
+			throw new UserManagerException("Invalid session. Login required");
+		}
+		else if (!session.getIp().equals(ip)) {
+			throw new UserManagerException("Invalid session. Session expired.");
+		}
+		
+		return session.getUser();
+	}
+}

diff --git a/src/restli/java/azkaban/restli/UserManagerResource.java b/src/restli/java/azkaban/restli/UserManagerResource.java
new file mode 100644
index 0000000..1a22263
--- /dev/null
+++ b/src/restli/java/azkaban/restli/UserManagerResource.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2014 LinkedIn Corp.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package azkaban.restli;
+
+import java.util.UUID;
+import javax.servlet.ServletException;
+import org.apache.log4j.Logger;
+
+import azkaban.restli.user.User;
+import azkaban.user.UserManager;
+import azkaban.user.UserManagerException;
+import azkaban.webapp.AzkabanWebServer;
+import azkaban.webapp.session.Session;
+
+import com.linkedin.restli.server.annotations.Action;
+import com.linkedin.restli.server.annotations.ActionParam;
+import com.linkedin.restli.server.annotations.RestLiActions;
+import com.linkedin.restli.server.resources.ResourceContextHolder;
+
+
+@RestLiActions(name = "user", namespace = "azkaban.restli")
+public class UserManagerResource extends ResourceContextHolder {
+	private static final Logger logger = Logger.getLogger(UserManagerResource.class);
+	
+	public AzkabanWebServer getAzkaban() {
+		return AzkabanWebServer.getInstance();
+	}
+	
+	@Action(name = "login")
+	public String login(
+			@ActionParam("username") String username,
+			@ActionParam("password") String password)
+			throws UserManagerException, ServletException {
+		String ip = (String)this.getContext().getRawRequestContext().getLocalAttr("REMOTE_ADDR");
+		logger.info("Attempting to login for " + username + " from ip '" + ip + "'");
+		
+		Session session = createSession(username, password, ip);
+		
+		logger.info("Session id " + session.getSessionId() + " created for user '" + username + "' and ip " + ip);
+		return session.getSessionId();
+	}
+
+	@Action(name = "getUserFromSessionId")
+	public User getUserFromSessionId(@ActionParam("sessionId") String sessionId) {
+		String ip = (String)this.getContext().getRawRequestContext().getLocalAttr("REMOTE_ADDR");
+		Session session = getSessionFromSessionId(sessionId, ip);
+		azkaban.user.User azUser = session.getUser();
+
+		// Fill out the restli object with properties from the Azkaban user
+		User user = new User();
+		user.setUserId(azUser.getUserId());
+		user.setEmail(azUser.getEmail());
+		return user;
+	}
+
+	private Session createSession(String username, String password, String ip)
+			throws UserManagerException, ServletException {
+		UserManager manager = getAzkaban().getUserManager();
+		azkaban.user.User user = manager.getUser(username, password);
+
+		String randomUID = UUID.randomUUID().toString();
+		Session session = new Session(randomUID, user, ip);
+		getAzkaban().getSessionCache().addSession(session);
+		
+		return session;
+	}
+
+	private Session getSessionFromSessionId(String sessionId, String remoteIp) {
+		if (sessionId == null) {
+			return null;
+		}
+
+		Session session = getAzkaban().getSessionCache().getSession(sessionId);
+		// Check if the IP's are equal. If not, we invalidate the sesson.
+		if (session == null || !remoteIp.equals(session.getIp())) {
+			return null;
+		}
+
+		return session;
+	}
+}
\ No newline at end of file

diff --git a/src/restli/schemas/azkaban/restli/user/User.pdsc b/src/restli/schemas/azkaban/restli/user/User.pdsc
new file mode 100644
index 0000000..156b380
--- /dev/null
+++ b/src/restli/schemas/azkaban/restli/user/User.pdsc
@@ -0,0 +1,10 @@
+{
+  "type": "record",
+  "name": "User",
+  "namespace": "azkaban.restli.user",
+  "doc": "Azkaban User restli info",
+  "fields": [
+    {"name": "userId", "type": "string","doc": "The username this session"},
+    {"name": "email", "type": "string","doc": "User email"}
+  ]
+}
\ No newline at end of file