fgreg closed pull request #17: SDAP-103 Update developer documentation URL: https://github.com/apache/incubator-sdap-nexus/pull/17
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/docker/solr-single-node/Dockerfile b/docker/solr-single-node/Dockerfile index fe1301d..49ab50d 100644 --- a/docker/solr-single-node/Dockerfile +++ b/docker/solr-single-node/Dockerfile @@ -1,3 +1,4 @@ + # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..fd39b78 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,3 @@ +_build +_static +_templates diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..a244ae3 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = incubator-sdap-nexus +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..8cca529 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,167 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = u'incubator-sdap-nexus' +copyright = u'2018, Apache SDAP' +author = u'Apache SDAP' + +# The short X.Y version +version = u'' +# The full version, including alpha/beta/rc tags +release = u'' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'sphinx.ext.viewcode', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'incubator-sdap-nexusdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'incubator-sdap-nexus.tex', u'incubator-sdap-nexus Documentation', + u'Apache SDAP', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'incubator-sdap-nexus', u'incubator-sdap-nexus Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'incubator-sdap-nexus', u'incubator-sdap-nexus Documentation', + author, 'incubator-sdap-nexus', 'One line description of project.', + 'Miscellaneous'), +] + + +# -- Extension configuration ------------------------------------------------- + +# -- Options for intersphinx extension --------------------------------------- + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'https://docs.python.org/': None} diff --git a/docs/images/Jupyter_Home.png b/docs/images/Jupyter_Home.png new file mode 100644 index 0000000..a7b0b49 Binary files /dev/null and b/docs/images/Jupyter_Home.png differ diff --git a/docs/images/Jupyter_Quickstart.png b/docs/images/Jupyter_Quickstart.png new file mode 100644 index 0000000..0782fe4 Binary files /dev/null and b/docs/images/Jupyter_Quickstart.png differ diff --git a/docs/images/nexus-summary.png b/docs/images/nexus-summary.png new file mode 100644 index 0000000..f19a69b Binary files /dev/null and b/docs/images/nexus-summary.png differ diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..5666094 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,13 @@ +Welcome to incubator-sdap-nexus's documentation! +================================================ + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + intro + quickstart + ningester + + +Check out the :ref:`quickstart`. diff --git a/docs/intro.rst b/docs/intro.rst new file mode 100644 index 0000000..6914726 --- /dev/null +++ b/docs/intro.rst @@ -0,0 +1,24 @@ +.. _intro: + + +******************* +About NEXUS +******************* + +NEXUS is a data-intensive analysis solution using a new approach for handling science data to enable large-scale data analysis. + +It supports a number of algorithms out of the box: + +* Time Series +* Latitude/Time Hovmöller +* Longitude/Time Hovmöller +* Latitude/Longitude Time Average +* Area Averaged Time Series +* Time Averaged Map +* Climatological Map +* Correlation Map +* Daily Difference Average + +.. image:: images/nexus-summary.png + +Check out the :ref:`quickstart`. diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..a9e146c --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=incubator-sdap-nexus + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/ningester.rst b/docs/ningester.rst new file mode 100644 index 0000000..82634ef --- /dev/null +++ b/docs/ningester.rst @@ -0,0 +1,89 @@ +.. _ningester: + +***************** +Ningester +***************** + +How to run this image +====================== + +The basic command is: + +.. code-block:: bash + + docker run -it --rm -v <absolute path to config directory on host>:/config/ -v <absolute path to granule on host>:/data/<granule name> sdap/ningester <profiles to activate> + +Replacing the following: + + * ``<absolute path to config directory on host>`` should be the absolute path on the host to the configuration for the job + * ``<absolute path to granule on host>`` should be the absolute path on the host to the granule intended for ingestion + * ``<granule name>`` should be the filename of the granule + * ``<profiles to activate>`` is a comma-separated list of profiles to activate + +The `ExampleJob.yml <https://github.com/apache/incubator-sdap-ningester/blob/bc596c2749a7a2b44a01558b60428f6d008f4f45/docker/example_config/ExampleJob.yml>`_ file shows an example Job configuration that would ingest an AVHRR granule. + +Configuration +============== + +Upon running the image, the ningester job will scan the ``/config`` directory for any files that end with the ``.yml`` extension. Specifically it uses find: + +.. code-block:: bash + + find /config -name "*.yml" | awk -vORS=, '{ print $1 }' + +Therefore, to configure the job, mount your configuration files into ``/config`` using a Docker volume. Alternatively, configuration is loaded via Spring Boot's `relaxed binding rules <https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-external-config.html#boot-features-external-config-relaxed-binding>`_. +So, you can also configure the job through environment variables where each 'level' of the yaml file gets replaced by an '_'. + +For example, given a configuration option in yaml that looks like: + +.. code-block:: yaml + + ningester: + tile_slicer: sliceFileByTilesDesired + sliceFileByTilesDesired: + tilesDesired: 1296 + timeDimension: time + dimensions: + - lat + - lon + +These could be replaced with the following corresponding Environment variables: + +.. code-block:: bash + + NINGESTER_TILE_SLICER=sliceFileByTilesDesired + NINGESTER_SLICE_FILE_BY_TILES_DESIRED_TILES_DESIRED=1296 + NINGESTER_SLICE_FILE_BY_TILES_DESIRED_TIME_DIMENSION=time + NINGESTER_SLICE_FILE_BY_TILES_DESIRED_DIMENSIONS[0]=lat + NINGESTER_SLICE_FILE_BY_TILES_DESIRED_DIMENSIONS[1]=lon + +However, because ningester has a lot of configuration options, it is recommended to use the yaml option. + +Data +===== + +Ningester is designed to ingest 1 granule per run. It looks for the granule to ingest in the ``/data`` directory of the container image. +Use a Docker volume to mount your data into ``/data``. + +The image relies on this command to find the first file in ``/data`` and it will use that file for ingestion: + +.. code-block:: bash + + find /data -type f -print -quit + +Examples +========== + +A few example commands are shared here. + +Docker for Mac +==================== + +The `ConnectionSettings-DockerForMac.yml <https://github.com/apache/incubator-sdap-ningester/blob/bc596c2749a7a2b44a01558b60428f6d008f4f45/docker/example_config/ConnectionSettings-DockerForMac.yml>`_ file shows an example of how to configure the connection settings +when running this job under Docker for Mac with Solr and Cassandra running on your host Mac. + +Replace ``<path to ningester>`` with the path on your local workstation to the ningester github project. + +.. code-block:: bash + + docker run -it --rm -v <path to ningester>/docker/example_config/:/config/ -v <path to ningester>/src/test/resources/granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc:/data/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc.nc sdap/ningester dockermachost,solr,cassandra diff --git a/docs/quickstart.rst b/docs/quickstart.rst index e5fb84f..1e66939 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -4,6 +4,8 @@ Quickstart Guide ***************** +This quickstart will take approximately 45 minutes to complete. + Introduction ============= @@ -289,7 +291,7 @@ Click on the ``Quickstart`` directory to open it. You should see a notebook call Click on the ``Time Series Example`` notebook to start it. This will open the notebook and allow you to run the two cells and execute a Time Series command against your local instance of NEXUS. -.. _quickstart-step8: +.. _quickstart-step9: Finished! ================ ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
