azkaban-aplcache
Changes
.gitignore 1(+1 -0)
docs/conf.py 149(+149 -0)
docs/configuration.rst 7(+7 -0)
docs/getStarted.rst 267(+267 -0)
docs/index.rst 55(+55 -0)
docs/Makefile 20(+20 -0)
docs/README.rst 13(+13 -0)
docs/requirements.txt 2(+2 -0)
docs/userGuide.rst 7(+7 -0)
Details
.gitignore 1(+1 -0)
diff --git a/.gitignore b/.gitignore
index 97422aa..79d9b86 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@ build/
Gemfile.lock
vendor/
**/velocity.log
+docs/_build/
# local & temp directories are ignored so that they can be used to store temporary files such as local testing configurations.
local/
docs/conf.py 149(+149 -0)
diff --git a/docs/conf.py b/docs/conf.py
new file mode 100644
index 0000000..9ccf20a
--- /dev/null
+++ b/docs/conf.py
@@ -0,0 +1,149 @@
+# -*- coding: utf-8 -*-
+#
+# Azkaban documentation build configuration file, created by
+# sphinx-quickstart on Mon Jul 16 15:29:27 2018.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = []
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'Azkaban'
+copyright = u'2018, Linkedin'
+author = u'Azkaban Dev'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = u''
+# The full version, including alpha/beta/rc tags.
+release = u''
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_rtd_theme"
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# This is required for the alabaster theme
+# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
+html_sidebars = {
+ '**': [
+ 'about.html',
+ 'navigation.html',
+ 'relations.html', # needs 'show_related': True theme option to display
+ 'searchbox.html',
+ 'donate.html',
+ ]
+}
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'AzkabanDoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+ # The paper size ('letterpaper' or 'a4paper').
+ #
+ # 'papersize': 'letterpaper',
+
+ # The font size ('10pt', '11pt' or '12pt').
+ #
+ # 'pointsize': '10pt',
+
+ # Additional stuff for the LaTeX preamble.
+ #
+ # 'preamble': '',
+
+ # Latex figure (float) alignment
+ #
+ # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+# author, documentclass [howto, manual, or own class]).
+latex_documents = [
+ (master_doc, 'AzkabanDoc.tex', u'Azkaban Documentation',
+ u'Liang', 'manual'),
+]
+
+
+
docs/configuration.rst 7(+7 -0)
diff --git a/docs/configuration.rst b/docs/configuration.rst
new file mode 100644
index 0000000..63531aa
--- /dev/null
+++ b/docs/configuration.rst
@@ -0,0 +1,7 @@
+.. _configs:
+
+
+Configurations
+==================================
+
+BlaBlaBla
\ No newline at end of file
docs/getStarted.rst 267(+267 -0)
diff --git a/docs/getStarted.rst b/docs/getStarted.rst
new file mode 100644
index 0000000..27d1627
--- /dev/null
+++ b/docs/getStarted.rst
@@ -0,0 +1,267 @@
+.. _GetStartedHead:
+
+
+Getting Started
+==================================
+
+After version 3.0, we provide two modes: the stand alone "solo-server" mode and distributed multiple-executor mode. The following describes the differences between the two modes.
+
+In solo server mode, the DB is embedded H2 and both web server and executor server run in the same process. This should be useful if one just wants to try things out. It can also be used on small scale use cases.
+
+The multiple executor mode is for most serious production environment. Its DB should be backed by MySQL instances with master-slave set up. The web server and executor servers should ideally run in different hosts so that upgrading and maintenance shouldn't affect users. This multiple host setup brings in robust and scalable aspect to Azkaban.
+
+- Set up the database
+- Configure database to use multiple executors
+- Download and install the Executor Server for each executor configured in database
+- Install Azkaban Plugins
+- Install the Web Server
+
+Below are instructions on how to set Azkaban up.
+
+*****
+Building from Source
+*****
+
+Azkaban builds use Gradle (downloads automatically when run using gradlew which is the Gradle wrapper) and requires Java 8 or higher.
+
+The following commands run on *nix platforms like Linux, OS X.
+::
+ # Build Azkaban
+ ./gradlew build
+
+ # Clean the build
+ ./gradlew clean
+
+ # Build and install distributions
+ ./gradlew installDist
+
+ # Run tests
+ ./gradlew test
+
+ # Build without running tests
+ ./gradlew build -x test
+
+These are all standard Gradle commands. Please look at Gradle documentation for more info.
+
+Gradle creates .tar.gz files inside project directories. eg. ./azkaban-solo-server/build/distributions/azkaban-solo-server-0.1.0-SNAPSHOT.tar.gz. Untar using tar -xvzf path/to/azkaban-*.tar.gz.
+
+
+*****
+Getting started with the Solo Server
+*****
+The solo server is a standalone instance of Azkaban and the simplest to get started with. The solo server has the following advantages.
+
+- **Easy to install** - No MySQL instance is needed. It packages H2 as its main persistence storage.
+- **Easy to start up** - Both web server and executor server run in the same process.
+- **Full featured** - It packages all Azkaban features. You can use it in normal ways and install plugins for it.
+
+
+Installing the Solo Server
+########
+
+Follow these steps to get started:
+
+1. Clone the repo:
+::
+ git clone https://github.com/azkaban/azkaban.git
+2. Build Azkaban and create an installation package:
+::
+ cd azkaban; ./gradlew build installDist
+3. Start the solo server:
+::
+ cd azkaban-solo-server/build/install/azkaban-solo-server; bin/azkaban-solo-start.sh
+Azkaban solo server should be all set, by listening to ``8081`` port at default to accept incoming network request. So, open a web browser and check out ``http://localhost:8081/``
+
+4. Stop server:
+::
+ bin/azkaban-solo-shutdown.sh
+
+
+The solo-server installation should contain the following directories.
+
++----------+---------------------------------------------------------+
+| Folder | Description |
++==========+=========================================================+
+| bin | The scripts to start/stop Azkaban solo server |
+| | |
++----------+---------------------------------------------------------+
+| conf | The configuration files for Azkaban solo server |
+| | |
++----------+---------------------------------------------------------+
+| lib | The jar dependencies for Azkaban |
+| | |
++----------+---------------------------------------------------------+
+| extlib | Additional jars that are added to extlib will be added |
+| | to Azkaban's classpath |
++----------+---------------------------------------------------------+
+| plugins | the directory where plugins can be installed |
+| | |
++----------+---------------------------------------------------------+
+| web | The web (css, javascript, image) files for Azkaban web |
+| | server |
++----------+---------------------------------------------------------+
+
+
+Inside the ``conf`` directory, there should be three files:
+
+- ``azkaban.private.properties`` - Used by Azkaban to store secrets like Mysql password
+- ``azkaban.properties`` - Used by Azkaban for runtime parameters
+- ``global.properties`` - Global static properties that are passed as shared properties to every workflow and job.
+- ``azkaban-users.xml`` - Used to add users and roles for authentication. This file is not used if the XmLUserManager is not set up to use it.
+
+The ``The azkaban.properties`` file is the main configuration file.
+
+
+Configuring HTTPS server (*Optional*)
+########
+
+Azkaban solo server by default doesn't use SSL. But you could set it up the same way in a stand alone web server. Here is how:
+
+Azkaban web server supports SSL socket connectors, which means a keystore will have to be available. You can follow the steps to generate a valid jetty keystore provided at `here <https://wiki.eclipse.org/Jetty/Howto/Configure_SSL>`_. Once a keystore file has been created, Azkaban must be given its location and password. Within ``azkaban.properties`` or ``azkaban.private.properties`` (recommended), the following properties should be overridden.
+::
+ jetty.keystore=keystore
+ jetty.password=password
+ jetty.keypassword=password
+ jetty.truststore=keystore
+ jetty.trustpassword=password
+
+And configure ssl port in `azkaban.properties`:
+::
+ jetty.ssl.port=8443
+
+
+*****
+Getting started with the Multi Executor Server
+*****
+
+Databasea setup
+########
+
+We suggest users to opt for **Mysql** as Azkaban database, because we build up a few Mysql connection enhancements to facilitate AZ set up, and strengthen service reliability:
+
+
+- Install Mysql
+
+ Installation of MySQL DB won't be covered by these instructions, but you can access the instructions on `MySQL Documentation Site <https://dev.mysql.com/doc/>`_.
+
+- Set up Mysql
+
+ a. create database for Azkaban.::
+
+ # Example database creation command, although the db name doesn't need to be 'azkaban'
+ mysql> CREATE DATABASE azkaban;
+
+ b. create a mysql user for Azkaban. For example,::
+
+ # Example database creation command. The user name doesn't need to be 'azkaban'
+ mysql> CREATE USER 'username'@'%' IDENTIFIED BY 'password';
+ # give the user INSERT, SELECT, UPDATE, DELETE permission on all tables in the Azkaban db.
+ mysql> GRANT SELECT,INSERT,UPDATE,DELETE ON azkaban.* to '<username>'@'%' WITH GRANT OPTION;
+
+ c. Mysql Packet Size may need to be re-configured. MySQL may have, by default, a ridiculously low allowable packet size. To increase it, you'll need to have the property max_allowed_packet set to a higher number, say 1024M.
+ To configure this in linux, open /etc/my.cnf. Somewhere after mysqld, add the following::
+
+ [mysqld]
+ ...
+ max_allowed_packet=1024M
+
+ To restart MySQL, you can run::
+
+ $ sudo /sbin/service mysqld restart
+
+
+- Create the Azkaban Tables
+
+ Run individual table creation scripts from `latest table statements <https://github.com/azkaban/azkaban/tree/master/azkaban-db/src/main/sql>`_ on the MySQL instance to create your tables.
+
+ Alternatively, run create-all-sql-<version>.sql generated by build process. The location is the file is at ``/Users/latang/LNKDRepos/azkaban/azkaban-db/build/distributions/azkaban-db-<version>``, after you build `azkaban-db` module by ::
+
+ cd azkaban-db; ../gradlew build installDist
+
+Installing Azkaban Executor Server
+########
+
+Azkaban Executor Server handles the actual execution of the workflow and jobs. You can build the latest version from the master branch. See here for instructions on `Building from Source`_.
+
+Extract the package (executor distribution tar.gz from build folder) into a directory after gradle build. There should be the following directories.
+
++----------+---------------------------------------------------------+
+| Folder | Description |
++==========+=========================================================+
+| bin | The scripts to start/stop Azkaban solo server |
+| | |
++----------+---------------------------------------------------------+
+| conf | The configuration files for Azkaban solo server |
+| | |
++----------+---------------------------------------------------------+
+| lib | The jar dependencies for Azkaban |
+| | |
++----------+---------------------------------------------------------+
+| extlib | Additional jars that are added to extlib will be added |
+| | to Azkaban's classpath |
++----------+---------------------------------------------------------+
+| plugins | the directory where plugins can be installed |
+| | |
++----------+---------------------------------------------------------+
+
+For quick start, we may directly use the Installation directory `azkaban/azkaban-exec-server/build/install/azkaban-exec-server` generated by gradle. we only need to change mysql username and password inside ``azkaban.properties``::
+
+ # Mysql Configs
+ mysql.user=<username>
+ mysql.password=<password>
+
+Then run::
+
+ cd azkaban-solo-server/build/install/azkaban-exec-server
+ ./bin/start-exec.sh
+
+After that, remember to activate the executor by calling::
+
+ cd azkaban-exec-server/build/install/azkaban-exec-server
+ curl -G "localhost:$(<./executor.port)/executor?action=activate" && echo
+
+Then, one executor is ready for use. Users can set up multiple executors by distributing and deploying multiple executor installation distributions.
+
+
+Installing Azkaban Web Server
+########
+
+Azkaban Web Server handles project management, authentication, scheduling and trigger of executions. You can build the latest version from the master branch. See here for instructions on `Building from Source`_.
+
+Extract the package (executor distribution tar.gz from build folder) into a directory after gradle build. There should be the following directories.
+
++----------+---------------------------------------------------------+
+| Folder | Description |
++==========+=========================================================+
+| bin | The scripts to start/stop Azkaban solo server |
+| | |
++----------+---------------------------------------------------------+
+| conf | The configuration files for Azkaban solo server |
+| | |
++----------+---------------------------------------------------------+
+| lib | The jar dependencies for Azkaban |
+| | |
++----------+---------------------------------------------------------+
+| web | The web (css, javascript, image) files for Azkaban web |
+| | server |
++----------+---------------------------------------------------------+
+
+
+For quick start, we may directly use the Installation directory `azkaban/azkaban-web-server/build/install/azkaban-web-server` generated by gradle. we only need to change mysql username and password inside ``azkaban.properties``::
+
+ # Mysql Configs
+ mysql.user=<username>
+ mysql.password=<password>
+
+Then run ::
+
+ cd azkaban-web-server/build/install/azkaban-web-server
+ ./bin/start-web.sh
+
+Then, a multi-executor Azkaban instance is ready for use. Open a web browser and check out ``http://localhost:8081/``
+You are all set to login to Azkaban UI.
+
+
+
+
+
docs/index.rst 55(+55 -0)
diff --git a/docs/index.rst b/docs/index.rst
new file mode 100644
index 0000000..68b3068
--- /dev/null
+++ b/docs/index.rst
@@ -0,0 +1,55 @@
+.. Second Sphinx Proj documentation master file, created by
+ sphinx-quickstart on Mon Jul 16 15:29:27 2018.
+ You can adapt this file completely to your liking, but it should at least
+ contain the root `toctree` directive.
+
+Azkaban documentation!
+==============================================
+
+*****
+What is Azkaban
+*****
+Azkaban is a distributed Workflow Manager, implemented at LinkedIn to solve the problem of Hadoop job dependencies. We had jobs that needed to run in order, from ETL jobs to data analytics products.
+
+See the :ref:`GetStartedHead` for more details.
+
+*****
+Features
+*****
+
+- Distributed Multiple Executor
+- MySQL Retry
+- Friendly UI
+- Conditional Workflow
+- Data Triggers
+- High Security
+- Support plug-in extensions, from Web UI to job Execution
+- Full Authorship management system
+
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+
+ getStarted
+ configuration
+ userGuide
+
+
+
+*****
+Community
+*****
+Users and development team are in the `Gitter <https://gitter.im/azkaban-workflow-engine/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge>`_.
+
+Developers interested in getting more involved with Azkaban may join the mailing lists `mailing lists <https://groups.google.com/forum/?fromgroups#!forum/azkaban-dev>`_, `report bugs <https://github.com/azkaban/azkaban/issues>`_, and make `contributions <https://github.com/azkaban/azkaban/pulls>`_.
+
+
+
+
+.. Indices and tables
+.. ==================
+
+.. * :ref:`genindex`
+.. * :ref:`modindex`
+.. * :ref:`search`
docs/Makefile 20(+20 -0)
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 0000000..3d3d1f2
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS =
+SPHINXBUILD = python3 -msphinx
+SPHINXPROJ = Azkaban
+SOURCEDIR = .
+BUILDDIR = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+ @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+ @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
\ No newline at end of file
docs/README.rst 13(+13 -0)
diff --git a/docs/README.rst b/docs/README.rst
new file mode 100644
index 0000000..fcae7cf
--- /dev/null
+++ b/docs/README.rst
@@ -0,0 +1,13 @@
+Make sure python3 is installed in your environment. python virtual environment is recommended to run these scripts.
+
+To download the dependencies, run
+
+```bash
+pip3 install -r requirements.txt
+```
+
+After sphinx is successfully installed, run::
+
+make html
+
+Check out the generated ``_build`` folder, where the static html documentation site is built.
\ No newline at end of file
docs/requirements.txt 2(+2 -0)
diff --git a/docs/requirements.txt b/docs/requirements.txt
new file mode 100644
index 0000000..ab3f3dd
--- /dev/null
+++ b/docs/requirements.txt
@@ -0,0 +1,2 @@
+Sphinx
+sphinx_rtd_theme
docs/userGuide.rst 7(+7 -0)
diff --git a/docs/userGuide.rst b/docs/userGuide.rst
new file mode 100644
index 0000000..a5dadfa
--- /dev/null
+++ b/docs/userGuide.rst
@@ -0,0 +1,7 @@
+.. _userGuide:
+
+
+User Guide
+==================================
+
+BlaBlaBla
\ No newline at end of file