roadan closed pull request #38: Integration with RTD URL: https://github.com/apache/incubator-amaterasu/pull/38
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..7098622b --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = ApacheAmaterasuincubating +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..3a1b6862 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +from recommonmark.parser import CommonMarkParser + + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'Apache Amaterasu (incubating)' +copyright = '2018, Apache Amaterasu (incubating)' +author = 'Apache Amaterasu (incubating)' + +# The short X.Y version +version = '' +# The full version, including alpha/beta/rc tags +release = '0.2.0' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ +] + + +source_parsers = { + '.md': CommonMarkParser, +} + + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +source_suffix = ['.rst', '.md'] +# source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'ApacheAmaterasuincubatingdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'ApacheAmaterasuincubating.tex', 'Apache Amaterasu (incubating) Documentation', + 'Apache Amaterasu (incubating)', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'apacheamaterasuincubating', 'Apache Amaterasu (incubating) Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'ApacheAmaterasuincubating', 'Apache Amaterasu (incubating) Documentation', + author, 'ApacheAmaterasuincubating', 'One line description of project.', + 'Miscellaneous'), +] \ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 100755 index 00000000..640a9cfe --- /dev/null +++ b/docs/index.md @@ -0,0 +1,124 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +# Apache Amaterasu [](https://travis-ci.org/apache/incubator-amaterasu) + + + /\ + / \ /\ + / /\ / \ + _ _ / / / /\ \ + /_\ _ __ __ _ | |_ ___ _ _ __(_( _(_(_ )_) + / _ \ | ' \ / _` || _|/ -_)| '_|/ _` |(_-<| || | + /_/ \_\|_|_|_|\__,_| \__|\___||_| \__,_|/__/ \_,_| + + +Apache Amaterasu is an open-source, deployment tool for data pipelines. Amaterasu allows developers to write and easily deploy data pipelines, and clusters manage their configuration and dependencies. + +## Download + +For this preview version, we have packaged amaterasu nicely for you to just [download](https://s3-ap-southeast-2.amazonaws.com/amaterasu/amaterasu.tgz) and extract. +Once you do that, you are just a couple of easy steps away from running your first job. + +## Creating a dev/test Mesos cluster + +We have also created a Mesos cluster you can use to test Amaterasu or use for development purposes. +For more details, visit the [amaterasu-vagrant](https://github.com/shintoio/amaterasu-vagrant) repo + +## Configuration + +Configuring amaterasu is very simple. Before running amaterasu, open the `amaterasu.properties` file in the top-level amaterasu directory, and verify the following properties: + +| property | Description | Default value | +| ---------- | -------------------------- | -------------- | +| zk | The ZooKeeper connection<br> string to be used by<br> amaterasu | 192.168.33.11 | +| master | The clusters' Mesos master | 192.168.33.11 | +| user | The user that will be used<br> to run amaterasu | root | + +## Running a Job + +To run an amaterasu job, run the following command in the top-level amaterasu directory: + +``` +ama-start.sh --repo="https://github.com/shintoio/amaterasu-job-sample.git"; --branch="master" --env="test" --report="code" +``` + +We recommend you either fork or clone the job sample repo and use that as a starting point for creating your first job. + +# Apache Amaterasu Developers Information + +## Building Apache Amaterasu + +to build the amaterasu home dir (for dev purposes) run: +``` +./gradlew buildHomeDir test +``` + +to create a distributable jar (clean creates the home dir first) run: +``` +./gradlew buildDistribution test +``` + +## Architecture + +Amaterasu is an Apache Mesos framework with two levels of schedulers: + +* The ClusterScheduler manages the execution of all the jobs +* The JobScheduler manages the flow of a job + +The main clases in Amateraso are listed bellow: + + +-------------------------+ +------------------------+ + | ClusterScheduler | | Kami | + | |-->| | + | Manage jobs: | | Manages the jobs queue | + | Queue new jobs | | and Amaterasu cluster | + | Reload interrupted jobs | +------------------------+ + | Monitor cluster state | + +-------------------------+ + | + | +------------------------+ + | | JobExecutor | + | | | + +---->| Runs the Job Scheduler | + | Communicates with the | + | ClusterScheduler | + +------------------------+ + | + | + +------------------------+ +---------------------------+ + | JobScheduler | | JobParser | + | | | | + | Manages the execution |----->| Parses the kami.yaml file | + | of the job, by getting | | and create a JobManager | + | the execution flow | +---------------------------+ + | fron the JobManager | | + | and comunicating with | +---------------------------+ + | Mesos | | JobManager | + +------------------------+ | | + | | Manages the jobs workflow | + | | independently of mesos | + +------------------------+ +---------------------------+ + | ActionExecutor | + | | + | Executes ActionRunners | + | and manages state for | + | the executor | + +------------------------+ + + + diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..5c4c38cc --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=ApacheAmaterasuincubating + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
