This is an automated email from the ASF dual-hosted git repository.
aviemzur pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git
The following commit(s) were added to refs/heads/master by this push:
new 2e3c080 add docs and examples
2e3c080 is described below
commit 2e3c08097118d12bfda2ed495461521baf29dd61
Author: Amihay Zer-Kavod <[email protected]>
AuthorDate: Fri Nov 20 12:20:09 2020 +0200
add docs and examples
---
.gitignore | 6 +
README.md | 3 +
docs/Makefile | 19 +++
docs/make.bat | 35 ++++
docs/source/How_to_build_a_local_liminal.md | 51 ++++++
docs/source/conf.py | 176 ++++++++++++++++++++
docs/source/getting_started_with_liminal.md | 182 +++++++++++++++++++++
docs/source/howto_contrib_to_liminal_docs.md | 58 +++++++
docs/source/index.rst | 41 +++++
docs/source/liminal_architecture.md | 65 ++++++++
docs/source/liminal_yaml_guide.md | 26 +++
docs/source/nstatic/airflow_main.png | Bin 0 -> 113607 bytes
docs/source/nstatic/airflow_task_log.png | Bin 0 -> 293074 bytes
docs/source/nstatic/airflow_view_dag.png | Bin 0 -> 182980 bytes
docs/source/nstatic/airflow_view_log.png | Bin 0 -> 172808 bytes
docs/source/nstatic/liminal_arch_001.png | Bin 0 -> 379411 bytes
docs/source/nstatic/liminal_arch_002.png | Bin 0 -> 49192 bytes
docs/source/nstatic/liminal_logo.png | Bin 0 -> 76075 bytes
examples/liminal-getting-started/__init__.py | 17 ++
examples/liminal-getting-started/hello_world.json | 1 +
.../liminal-getting-started/helloworld/__init__.py | 17 ++
.../helloworld/hello_world.py | 29 ++++
examples/liminal-getting-started/liminal.yml | 80 +++++++++
.../liminal-getting-started/myserver/__init__.py | 17 ++
.../liminal-getting-started/myserver/my_server.py | 21 +++
examples/liminal-getting-started/pip.conf | 17 ++
examples/liminal-getting-started/requirements.txt | 0
install.sh | 45 +++++
liminal/core/environment.py | 19 ++-
liminal/kubernetes/volume_util.py | 19 ++-
liminal/runners/airflow/tasks/python.py | 43 +++--
requirements.txt | 6 +-
run_tests.sh | 17 ++
scripts/__init__.py | 14 ++
scripts/liminal | 78 ++++++---
scripts/requirements-airflow.txt | 2 +-
36 files changed, 1043 insertions(+), 61 deletions(-)
diff --git a/.gitignore b/.gitignore
index e14e323..cf5f296 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,9 @@ venv
.Python
*.pyc
pip-selfcheck.json
+.DS_Store
+build
+apache_liminal.egg-info
+scripts/*.tar.gz
+scripts/*.whl
+dist
diff --git a/README.md b/README.md
index d1357fa..2f14fe4 100644
--- a/README.md
+++ b/README.md
@@ -34,6 +34,9 @@ non-functional tasks, and allowing them to focus on machine
learning code and ar
Using simple YAML configuration, create your own schedule data pipelines (a
sequence of tasks to
perform), application servers, and more.
+## Getting Started Documentation
+A simple getting stated guide for Liminal can be found
[here](docs/source/getting_started_with_liminal.md)
+
## Example YAML config file
```yaml
---
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 0000000..69fe55e
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,19 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS =
+SPHINXBUILD = sphinx-build
+SOURCEDIR = source
+BUILDDIR = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+ @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+ @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
\ No newline at end of file
diff --git a/docs/make.bat b/docs/make.bat
new file mode 100644
index 0000000..543c6b1
--- /dev/null
+++ b/docs/make.bat
@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+ set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+ echo.
+ echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+ echo.installed, then set the SPHINXBUILD environment variable to point
+ echo.to the full path of the 'sphinx-build' executable. Alternatively
you
+ echo.may add the Sphinx directory to PATH.
+ echo.
+ echo.If you don't have Sphinx installed, grab it from
+ echo.http://sphinx-doc.org/
+ exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd
diff --git a/docs/source/How_to_build_a_local_liminal.md
b/docs/source/How_to_build_a_local_liminal.md
new file mode 100644
index 0000000..5ddd534
--- /dev/null
+++ b/docs/source/How_to_build_a_local_liminal.md
@@ -0,0 +1,51 @@
+# Building a local installation of Liminal
+
+Start a venv in the liminal example folder
+```
+python3 -m venv env
+```
+
+And activate your virtual environment:
+```
+source env/bin/activate
+```
+clean up old versions
+```
+pip uninstall apache-liminal
+```
+make sure you have wheel install (should be OK)
+```
+pip install wheel
+```
+Set a version for the build:
+```
+export LIMINAL_BUILD_VERSION=0.0.1.MYVER
+```
+Build the liminal version in the base of the cloned liminal folder, go to
where you cloned it:
+```
+cd <liminal_path>
+git clone https://github.com/apache/incubator-liminal
+python3 setup.py sdist bdist_wheel
+```
+
+make sure the distribution created is copied to the ./scripts folder
+```
+cp ./dist/apache_liminal-0.0.1.MYVER-py3-none-any.whl ./scripts
+```
+
+local install in target directory
+```
+cd example_folder
+pip install <liminal_path>/scripts/apache_liminal-0.0.1.MYVER-py3-none-any.whl
+liminal build
+liminal deploy --clean
+liminal start
+```
+
+How to remove a version and rebuild:
+```
+cd example_folder
+pip uninstall apache-liminal
+cd <liminal_path>
+rm ./dist
+```
diff --git a/docs/source/conf.py b/docs/source/conf.py
new file mode 100644
index 0000000..152b2e7
--- /dev/null
+++ b/docs/source/conf.py
@@ -0,0 +1,176 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+sys.path.insert(0, os.path.abspath('.'))
+sys.path.insert(0, os.path.abspath('../'))
+
+# -- Project information -----------------------------------------------------
+
+project = u'Apchae Liminal'
+copyright = u'Apache Liminal'
+author = u'Amihay Zer-Kavod'
+
+# The short X.Y version
+version = u''
+# The full version, including alpha/beta/rc tags
+release = u'0.0.1'
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+ 'recommonmark',
+ 'sphinx.ext.autodoc',
+]
+
+html_logo = 'nstatic/liminal_logo.png'
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['ntemplates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'classic'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['nstatic']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself. Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'ApchaeLiminaldoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+ # The paper size ('letterpaper' or 'a4paper').
+ #
+ # 'papersize': 'letterpaper',
+
+ # The font size ('10pt', '11pt' or '12pt').
+ #
+ # 'pointsize': '10pt',
+
+ # Additional stuff for the LaTeX preamble.
+ #
+ # 'preamble': '',
+
+ # Latex figure (float) alignment
+ #
+ # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+# author, documentclass [howto, manual, or own class]).
+latex_documents = [
+ (master_doc, 'ApchaeLiminal.tex', u'Apchae Liminal Documentation',
+ u'Amihay Zer-Kavod', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ (master_doc, 'apchaeliminal', u'Apchae Liminal Documentation',
+ [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+# dir menu entry, description, category)
+texinfo_documents = [
+ (master_doc, 'ApchaeLiminal', u'Apchae Liminal Documentation',
+ author, 'ApchaeLiminal', 'One line description of project.',
+ 'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
diff --git a/docs/source/getting_started_with_liminal.md
b/docs/source/getting_started_with_liminal.md
new file mode 100644
index 0000000..dc46eb7
--- /dev/null
+++ b/docs/source/getting_started_with_liminal.md
@@ -0,0 +1,182 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required bgit y applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Getting Started with Apache Liminal
+
+Liminal is a data systems orchestration platform. Liminal enables data
scientists and data engineers to define their data systems and flow using
configuration.
+From feature engineering to production monitoring - and the framework takes
care of the infra behind the scenes and seamlessly integrates with the
infrastructure behind the scenes.
+
+
+## Quick Start
+
+This guide will allow you to set up your first apache Liminal environment and
allow you to create some simple ML pipelines. These will be very similar to the
ones you are going to build for real production scenarios. This is actually the
magic behind Liminal.
+
+## Prerequisites
+
+Python 3 (3.6 and up)
+
+[Docker Desktop](https://www.docker.com/products/docker-desktop)
+
+*Note: Make sure kubernetes cluster is running in docker desktop (or custom
kubernetes installation on your machine).*
+
+### Apache Liminal Hello World
+
+In this tutorial, we will go through setting up Liminal for the first time on
your local dev machine.
+
+I’m running this on my macBook, so this will cover mac related installation
aspects and kinks (if there are any).
+
+First, let’s build our examples project:
+
+In the dev folder, just clone the example code from liminal:
+
+
+```
+git clone https://github.com/apache/incubator-liminal
+```
+***Note:*** *You just cloned the entire Liminal Project, you actually need
just the examples folder.*
+
+Create a python virtualenv to isolate your runs, like this:
+
+```
+cd incubator-liminal/examples/liminal-getting-started
+python3 -m venv env
+```
+
+And activate your virtual environment:
+
+```
+source env/bin/activate
+```
+
+Now we are ready to install liminal:
+
+```
+pip install apache-liminal
+rm -rf ~/liminal_home
+```
+Let's build the images you need for the example:
+```
+liminal build
+```
+The build will create docker images based on the Liminal.yml file, each task
and service will have its docker image created.
+
+```
+liminal deploy --clean
+```
+The deploy will create missing containers from the images created above as
well as basic Apache AirFlow containers needed to run the pipelines, and deploy
them to your local docker (docker-compose)
+
+*Note: Liminal creates a bunch of assets, some are going to be located under:
the directory in your LIMINAL_HOME env variable. But default LIMINAL_HOME is
set to ~/liminal_home directory.*
+
+*Note: the --clean flag is for removing old containers from the previous
installs, it will not remove old images.*
+
+Now lets runs it:
+```
+liminal start
+```
+Liminal now spins up the Apache AirFlow containers it will run the dag created
based on the Liminal.yml file.
+It includes these three containers:
+* liminal-postgress
+* liminal-webserver
+* liminal-scheduler
+
+Once it finished loading the,
+Go to the Apache AirFlow admin in the browser:
+
+
+```
+http://localhost:8080/admin
+```
+You can just click: [http://localhost:8080/admin](http://localhost:8080/admin)
+
+
+
+
+***Important:** Click on the “On” button to activate the dag, nothing will
happen otherwise!*
+
+You can go to tree view to see all the tasks configured in the liminal.yml
file: \
+[http://localhost:8080/admin/Apache
AirFlow/tree?dag_id=example_pipeline](http://localhost:8080/admin/Apache
AirFlow/tree?dag_id=example_pipeline)
+
+Now lets see what actually happened to our task:
+
+
+
+Click on “hello_world_example” and you will get this popup: \
+
+ \
+Click on “view log” button and you can see the log of the current task run: \
+
+
+
+
+### mounted volumes
+All Tasks use a mounted volume as defined in the pipeline YAML:
+```YAML
+name: GettingStartedPipeline
+volumes:
+ - volume: gettingstartedvol
+ local:
+ path: ./
+```
+In our case the mounted volume will point to the liminal hello world example.
+The hello world task will read the **hello_world.json** file from the mounted
volume and will write the **hello_world_output.json** to it.
+
+*Note:* Each task will internally mount the volume defined above to an
internal representation, described under the task section in the yml:
+
+```YAML
+ task:
+ ...
+ mounts:
+ - mount: taskmount
+ volume: gettingstartedvol
+ path: /mnt/vol1
+```
+
+### Here are the entire list of commands, if you want to start from scratch:
+
+```
+git clone https://github.com/apache/incubator-liminal
+cd examples
+python3 -m venv env
+source env/bin/activate
+pip uninstall apache-liminal
+pip install apache-liminal
+rm -rf ~/liminal_home
+Liminal build
+liminal deploy --clean
+liminal start
+```
+
+### Closing up
+
+Making sure the airflow containers are closed, just click:
+
+```
+liminal stop
+```
+
+Or you can do it manually, stop the running containers:
+```
+docker container stop liminal-postgress liminal-webserver liminal-scheduler```
+```
+
+And don't forget to deactivate the python virtualenv, just write:
+
+```
+deactivate
+```
diff --git a/docs/source/howto_contrib_to_liminal_docs.md
b/docs/source/howto_contrib_to_liminal_docs.md
new file mode 100644
index 0000000..1a72620
--- /dev/null
+++ b/docs/source/howto_contrib_to_liminal_docs.md
@@ -0,0 +1,58 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required bgit y applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Contributing to Liminal docs
+
+Apache Liminal use Sphinx and readthedocs.org to create and publish
documentation to readthedocs.org
+
+## Basic setup
+Here is what you need to do in order to work and create docs locally
+
+## Installing sphinx and documentation
+
+More details about sphinx and readthedocs can be found
[here:](https://docs.readthedocs.io/en/stable/intro/getting-started-with-sphinx.html)
+
+Here is what you need to do:
+```
+pip install sphinx
+```
+Inside your project go to docs lib, update the docs you like and build them:
+```
+cd docs
+make html
+```
+
+## Automatically rebuild sphinx docs
+```
+pip install sphinx-autobuild
+cd docs
+sphinx-autobuild source/ build/html/
+```
+Now open a browser on
+http://localhost:8000/
+Your docs will automatically update
+
+## Rebuilding the pythondoc liminal code documentation
+```
+cd docs
+sphinx-apidoc -o source/ ../liminal
+```
+
+
+
diff --git a/docs/source/index.rst b/docs/source/index.rst
new file mode 100644
index 0000000..9161f75
--- /dev/null
+++ b/docs/source/index.rst
@@ -0,0 +1,41 @@
+.. Apchae Liminal documentation master file, created by
+ sphinx-quickstart on Sun Nov 15 08:45:27 2020.
+ You can adapt this file completely to your liking, but it should at least
+ contain the root `toctree` directive.
+
+..
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+..
+ http://www.apache.org/licenses/LICENSE-2.0
+..
+ Unless required bgit y applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+Welcome to Apchae Liminal's documentation!
+==========================================
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+
+ getting_started_with_liminal
+ liminal_architecture
+ liminal_yaml_guide
+ howto_contrib_to_liminal_docs
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/docs/source/liminal_architecture.md
b/docs/source/liminal_architecture.md
new file mode 100644
index 0000000..62e5558
--- /dev/null
+++ b/docs/source/liminal_architecture.md
@@ -0,0 +1,65 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required bgit y applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache Liminal High Level Architecture
+Liminal is an end-to-end platform for data engineers & scientists, allowing
them to build, train and deploy machine learning models in a robust and agile
way. The platform provides the abstractions and declarative capabilities for
data extraction & feature engineering followed by model training and serving.
Apache Liminal's goal is to operationalise the machine learning process,
allowing data scientists to quickly transition from a successful experiment to
an automated pipeline of model [...]
+
+## Motivation
+The challenges involved in operationalizing machine learning models are one of
the main reasons why many machine learning projects never make it to production.
+The process involves automating and orchestrating multiple steps which run on
heterogeneous infrastructure - different compute environments, data processing
platforms, ML frameworks, notebooks, containers and monitoring tools.
+
+There are no mature standards for this workflow, and most organizations do not
have the experience to build it in-house. In the best case, dev-ds-devops teams
form in order to accomplish this task together; in many cases, it's the data
scientists who try to deal with this themselves without the knowledge or the
inclination to become infrastructure experts.
+As a result, many projects never make it through the cycle. Those who do
suffer from a very long lead time from a successful experiment to an
operational, refreshable, deployed and monitored model in production.
+
+The goal of Apache Liminal is to simplify the creation and management of
machine learning pipelines by data engineers & scientists. The platform
provides declarative building blocks which define the workflow, orchestrate the
underlying infrastructure, take care of non functional concerns, enabling
focus in business logic / algorithm code.
+Some Commercial E2E solutions have started to emerge in the last few years,
however, they are limited to specific parts of the workflow, such as Databricks
MLFlow. Other solutions are tied to specific environments (e.g. SageMaker on
AWS).
+
+## High Level Architecture
+The platform is aimed to provide data engineers & scientists with a solution
for end to end flows from model training to real time inference in production.
It’s architecture enables and promotes adoption of specific components in
existing (non-Liminal) frameworks, as well as seamless integration with other
open source projects. Liminal was created to enable scalability in ML efforts
and after a thorough review of available solutions and frameworks, which did
not meet our main KPIs:
+- Provide an opinionated but customizable end-to-end workflow
+- Abstract away the complexity of underlying infrastructure
+- Support major open source tools and cloud-native infrastructure to carry out
many of the steps
+- Allow teams to leverage their existing investments or bring in their tools
of choice into the workflow
+
+We have found that other tech companies in the Hi-Tech ecosystem also have an
interest in such a platform, hence decided to share our work with the community.
+The following diagram depicts these main components and where Apache Liminal
comes in:
+
+
+
+A classical data scientist workflow includes some base phases:
+_Train, Deploy and Consume._
+
+**The Train phase includes the following tasks:**
+
+1. Fetch - get the data needed to build a model - usually using SQL
+1. Clean - make sure the data is useful for building the model
+1. Prepare - split data and encode features from the data according to model
needs
+1. Train - Build the model and tune it
+1. Evaluate - make sure the model is correct - run it on a test set, etc…
+1. Validate - make sure the model is up to the standards you need
+
+**The Deploy phase includes these tasks:**
+1. Deploy - make it available for usage in production
+1. Inference - Batch or Real-time - use the model to evaluate data by your
offline or online by your applications
+1. Consume - The actual use of the models created by applications and ETLs,
usually through APIs to the batch or real-time inference that usually rely on
Model and Feature stores.
+
+Liminal provides its users a declarative composition capabilities to
materialize these steps in a robust way, while exploiting existing frameworks
and tools. e.g. Data science frameworks such as scikit-learn, Tensor flow,
Keras and such, for running core data science algorithms; as numerous core
mechanisms as data stores, processing engines, parallelism, schedulers, code
deployment as well as batch and real-time inference.
+Liminal allows the creation and wiring of these kinds of functional and non
functional tasks while making the underlying infrastructure used by these tasks
very easy to use and even abstracted away entirely. While handling the
non-functional aspects as monitoring (in a standard fashion) deployment,
scheduling, resource management and execution.
+
+
\ No newline at end of file
diff --git a/docs/source/liminal_yaml_guide.md
b/docs/source/liminal_yaml_guide.md
new file mode 100644
index 0000000..9f9b9c2
--- /dev/null
+++ b/docs/source/liminal_yaml_guide.md
@@ -0,0 +1,26 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required bgit y applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Liminal yaml file explained
+
+## Main Structure of liminal.yml
+TDB
+## main properties
+### task
+### service
\ No newline at end of file
diff --git a/docs/source/nstatic/airflow_main.png
b/docs/source/nstatic/airflow_main.png
new file mode 100644
index 0000000..8b3d759
Binary files /dev/null and b/docs/source/nstatic/airflow_main.png differ
diff --git a/docs/source/nstatic/airflow_task_log.png
b/docs/source/nstatic/airflow_task_log.png
new file mode 100644
index 0000000..4747cb6
Binary files /dev/null and b/docs/source/nstatic/airflow_task_log.png differ
diff --git a/docs/source/nstatic/airflow_view_dag.png
b/docs/source/nstatic/airflow_view_dag.png
new file mode 100644
index 0000000..3088d30
Binary files /dev/null and b/docs/source/nstatic/airflow_view_dag.png differ
diff --git a/docs/source/nstatic/airflow_view_log.png
b/docs/source/nstatic/airflow_view_log.png
new file mode 100644
index 0000000..684cfde
Binary files /dev/null and b/docs/source/nstatic/airflow_view_log.png differ
diff --git a/docs/source/nstatic/liminal_arch_001.png
b/docs/source/nstatic/liminal_arch_001.png
new file mode 100644
index 0000000..696e052
Binary files /dev/null and b/docs/source/nstatic/liminal_arch_001.png differ
diff --git a/docs/source/nstatic/liminal_arch_002.png
b/docs/source/nstatic/liminal_arch_002.png
new file mode 100644
index 0000000..ac04a6a
Binary files /dev/null and b/docs/source/nstatic/liminal_arch_002.png differ
diff --git a/docs/source/nstatic/liminal_logo.png
b/docs/source/nstatic/liminal_logo.png
new file mode 100644
index 0000000..b3cf211
Binary files /dev/null and b/docs/source/nstatic/liminal_logo.png differ
diff --git a/examples/liminal-getting-started/__init__.py
b/examples/liminal-getting-started/__init__.py
new file mode 100644
index 0000000..217e5db
--- /dev/null
+++ b/examples/liminal-getting-started/__init__.py
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/examples/liminal-getting-started/hello_world.json
b/examples/liminal-getting-started/hello_world.json
new file mode 100644
index 0000000..0630236
--- /dev/null
+++ b/examples/liminal-getting-started/hello_world.json
@@ -0,0 +1 @@
+{ "hello" : "world" }
diff --git a/examples/liminal-getting-started/helloworld/__init__.py
b/examples/liminal-getting-started/helloworld/__init__.py
new file mode 100644
index 0000000..217e5db
--- /dev/null
+++ b/examples/liminal-getting-started/helloworld/__init__.py
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/examples/liminal-getting-started/helloworld/hello_world.py
b/examples/liminal-getting-started/helloworld/hello_world.py
new file mode 100644
index 0000000..78fbd9c
--- /dev/null
+++ b/examples/liminal-getting-started/helloworld/hello_world.py
@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import json
+import os
+
+print('Hello world!\n')
+
+# with open('/mnt/gettingstartedvol/hello_world.json') as file:
+# print(f'hello_world.json contents = {json.loads(file.readline())}')
+#
+# os.makedirs('/mnt/vol1/', exist_ok=True)
+
+with open('/mnt/gettingstartedvol/hello_world_output.json', 'w') as file:
+ file.write(json.dumps({'hello': 1, 'world': 2}))
\ No newline at end of file
diff --git a/examples/liminal-getting-started/liminal.yml
b/examples/liminal-getting-started/liminal.yml
new file mode 100644
index 0000000..66d787a
--- /dev/null
+++ b/examples/liminal-getting-started/liminal.yml
@@ -0,0 +1,80 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+name: GettingStartedPipeline
+volumes:
+ - volume: gettingstartedvol
+ local:
+ path: .
+pipelines:
+ - pipeline: getting_started_pipeline
+ owner: Bosco Albert Baracus
+ start_date: 1970-01-01
+ timeout_minutes: 10
+ schedule: 0 * 1 * *
+ default_arg_loaded: check
+ default_array_loaded: [2, 3, 4]
+ default_object_loaded:
+ key1: val1
+ key2: val2
+ metrics:
+ namespace: TestNamespace
+ backends: [ ]
+ tasks:
+ - task: python_hello_world_example
+ type: python
+ description: static input task
+ image: python_hello_world_example_image
+ source: helloworld
+ env_vars:
+ env1: "a"
+ env2: "b"
+ mounts:
+ - mount: mymount
+ volume: gettingstartedvol
+ path: /mnt/gettingstartedvol
+ cmd: python -u hello_world.py
+ - task: parallelized_hello_world_example_task
+ type: python
+ description: parallelized parallelized hello world example
+ image: python_hello_world_example_image
+ env_vars:
+ env1: "a"
+ env2: "b"
+ split_input: True
+ executors: 2
+ cmd: python -u hello_world.py
+ - task: python_hello_world_output_task
+ type: python
+ description: task with input from other task's output
+ image: python_hello_world_example_image
+ env_vars:
+ env1: "a"
+ env2: "b"
+ cmd: python -u hello_world.py
+services:
+ - service:
+ name: liminal_getting_started_python_server
+ type: python_server
+ description: my python server
+ image: liminal_getting_started_server_image
+ source: myserver
+ endpoints:
+ - endpoint: /myendpoint1
+ module: my_server
+ function: myendpoint1func
diff --git a/examples/liminal-getting-started/myserver/__init__.py
b/examples/liminal-getting-started/myserver/__init__.py
new file mode 100644
index 0000000..217e5db
--- /dev/null
+++ b/examples/liminal-getting-started/myserver/__init__.py
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/examples/liminal-getting-started/myserver/my_server.py
b/examples/liminal-getting-started/myserver/my_server.py
new file mode 100644
index 0000000..a3f0f2c
--- /dev/null
+++ b/examples/liminal-getting-started/myserver/my_server.py
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+def myendpoint1func():
+ return '1'
diff --git a/examples/liminal-getting-started/pip.conf
b/examples/liminal-getting-started/pip.conf
new file mode 100644
index 0000000..217e5db
--- /dev/null
+++ b/examples/liminal-getting-started/pip.conf
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/examples/liminal-getting-started/requirements.txt
b/examples/liminal-getting-started/requirements.txt
new file mode 100644
index 0000000..e69de29
diff --git a/install.sh b/install.sh
new file mode 100755
index 0000000..0ef53a3
--- /dev/null
+++ b/install.sh
@@ -0,0 +1,45 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required bgit y applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+if ! command -v liminal &> /dev/null
+then
+ liminal stop
+fi
+
+yes | pip uninstall apache-liminal
+
+cd "$DIR" || exit
+
+rm -rf build
+rm -rf dist
+
+python setup.py sdist bdist_wheel
+
+rm scripts/*.whl
+
+cp dist/*.whl scripts
+
+ver=$(ls scripts/*.whl)
+export LIMINAL_VERSION="apache-liminal @ file://$ver"
+
+pip install scripts/*.whl
+
+cd - || exit
diff --git a/liminal/core/environment.py b/liminal/core/environment.py
index a3dc3c9..42e14a4 100644
--- a/liminal/core/environment.py
+++ b/liminal/core/environment.py
@@ -44,15 +44,20 @@ def get_dags_dir():
def get_liminal_version():
result = os.environ.get(LIMINAL_VERSION_PARAM_NAME, None)
if not result:
- output = subprocess.run(['pip freeze | grep \'apache-liminal\''],
capture_output=True, env=os.environ, shell=True)
+ output = subprocess.run(['pip freeze | grep \'apache-liminal\''],
capture_output=True,
+ env=os.environ, shell=True)
pip_res = output.stdout.decode('UTF-8').strip()
- value = None
- if not pip_res:
- value = 'apache-liminal'
- if ' @ ' in pip_res:
- value = pip_res[pip_res.index(' @ ')+3:]
+ scripts_dir = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
+ 'scripts'
+ )
+ whl_files = [file for file in os.listdir(scripts_dir) if
file.endswith(".whl")]
+ if whl_files:
+ value = 'file://' + os.path.join(scripts_dir, whl_files[0])
+ elif ' @ ' in pip_res:
+ value = pip_res[pip_res.index(' @ ') + 3:]
else:
- value= pip_res
+ value = pip_res
print(f'LIMINAL_VERSION not set. Setting it to currently installed
version: {value}')
os.environ[LIMINAL_VERSION_PARAM_NAME] = value
return os.environ.get(LIMINAL_VERSION_PARAM_NAME, 'apache-liminal')
diff --git a/liminal/kubernetes/volume_util.py
b/liminal/kubernetes/volume_util.py
index 376385b..259faa8 100644
--- a/liminal/kubernetes/volume_util.py
+++ b/liminal/kubernetes/volume_util.py
@@ -17,6 +17,7 @@
# under the License.
import logging
+import os
from kubernetes import client, config
from kubernetes.client import V1PersistentVolume, V1PersistentVolumeClaim
@@ -28,6 +29,21 @@ _LOCAL_VOLUMES = set([])
_kubernetes = client.CoreV1Api()
+def create_local_volumes(liminal_config, base_dir):
+ volumes_config = liminal_config.get('volumes', [])
+
+ for volume_config in volumes_config:
+ if 'local' in volume_config:
+ print(f'Creating local kubernetes volume if needed:
{volume_config}')
+ path = volume_config['local']['path']
+ if path.startswith(".."):
+ path = os.path.join(base_dir, path)
+ if path.startswith("."):
+ path = os.path.join(base_dir, path[1:])
+ volume_config['local']['path'] = path
+ create_local_volume(volume_config)
+
+
def create_local_volume(conf, namespace='default') -> None:
name = conf['volume']
@@ -72,7 +88,8 @@ def delete_local_volume(name, namespace='default'):
_LOG.info(f'Deleting persistent volume claim {pvc_name}')
_kubernetes.delete_namespaced_persistent_volume_claim(pvc_name,
namespace)
- _LOCAL_VOLUMES.remove(name)
+ if name in _LOCAL_VOLUMES:
+ _LOCAL_VOLUMES.remove(name)
def _create_persistent_volume_claim(pvc_name, volume_name, namespace):
diff --git a/liminal/runners/airflow/tasks/python.py
b/liminal/runners/airflow/tasks/python.py
index a854852..1b9fd68 100644
--- a/liminal/runners/airflow/tasks/python.py
+++ b/liminal/runners/airflow/tasks/python.py
@@ -15,17 +15,19 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+import logging
from typing import Optional
-from airflow.contrib.kubernetes.volume import Volume
-from airflow.contrib.kubernetes.volume_mount import VolumeMount
+from airflow.kubernetes.volume import Volume
+from airflow.kubernetes.volume_mount import VolumeMount
from airflow.contrib.operators.kubernetes_pod_operator import
KubernetesPodOperator
from airflow.operators.dummy_operator import DummyOperator
-from liminal.kubernetes import volume_util
from liminal.runners.airflow.config.standalone_variable_backend import
get_variable
from liminal.runners.airflow.model import task
+_LOG = logging.getLogger(__name__)
+
class PythonTask(task.Task):
"""
@@ -36,9 +38,8 @@ class PythonTask(task.Task):
super().__init__(dag, liminal_config, pipeline_config, task_config,
parent, trigger_rule)
self.task_name = self.task_config['task']
self.image = self.task_config['image']
- self.volumes_config = self._volumes_config()
+ self.volumes = self._volumes()
self.mounts = self.task_config.get('mounts', [])
- self._create_local_volumes()
self.resources = self.__kubernetes_resources()
self.env_vars = self.__env_vars()
self.kubernetes_kwargs = self.__kubernetes_kwargs()
@@ -51,16 +52,21 @@ class PythonTask(task.Task):
else:
return self.__apply_task_to_dag_multiple_executors()
- def _volumes_config(self):
+ def _volumes(self):
volumes_config = self.liminal_config.get('volumes', [])
-
- for volume in volumes_config:
- if 'local' in volume:
- volume['persistentVolumeClaim'] = {
- 'claimName': f"{volume['volume']}-pvc"
+ volumes = []
+ for volume_config in volumes_config:
+ name = volume_config['volume']
+ volume = Volume(
+ name=name,
+ configs={
+ 'persistentVolumeClaim': {
+ 'claimName': f"{name}-pvc"
+ }
}
-
- return volumes_config
+ )
+ volumes.append(volume)
+ return volumes
def __apply_task_to_dag_multiple_executors(self):
start_task = DummyOperator(
@@ -141,12 +147,7 @@ class PythonTask(task.Task):
'image_pull_secrets': 'regcred',
'resources': self.resources,
'dag': self.dag,
- 'volumes': [
- Volume(volume['volume'], volume)
- for volume
- in self.volumes_config
- ],
- # TODO: aviem - mount with run_id as subpath
+ 'volumes': self.volumes,
'volume_mounts': [
VolumeMount(mount['volume'],
mount['path'],
@@ -174,7 +175,3 @@ class PythonTask(task.Task):
resources['limit_memory'] = self.task_config['limit_memory']
return resources
-
- def _create_local_volumes(self):
- for volume in [volume for volume in self.volumes_config if 'local' in
volume]:
- volume_util.create_local_volume(volume)
diff --git a/requirements.txt b/requirements.txt
index 3fef3a5..062b3c1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,10 +1,10 @@
docker==4.2.0
-apache-airflow==1.10.9
+apache-airflow==1.10.12
docker-pycreds==0.4.0
click==7.1.1
Flask==1.1.1
pyyaml
boto3==1.12.10
botocore==1.15.21
-kubernetes
-
+kubernetes==12.0.1
+wheel
diff --git a/run_tests.sh b/run_tests.sh
index 1761c01..59d801b 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -1,4 +1,21 @@
#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required bgit y applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
diff --git a/scripts/__init__.py b/scripts/__init__.py
new file mode 100644
index 0000000..ec20143
--- /dev/null
+++ b/scripts/__init__.py
@@ -0,0 +1,14 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/scripts/liminal b/scripts/liminal
index 0a794b5..3c7b132 100755
--- a/scripts/liminal
+++ b/scripts/liminal
@@ -17,6 +17,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
import os
import pathlib
import shutil
@@ -24,12 +25,15 @@ import subprocess
import sys
import click
+import yaml
+import scripts as s
from liminal.build import liminal_apps_builder
from liminal.core import environment
from liminal.core.util import files_util
+from liminal.kubernetes import volume_util
from liminal.runners.airflow import dag
-import scripts as s
+
try:
import importlib.resources as pkg_resources
except ImportError:
@@ -45,7 +49,7 @@ def cli():
def docker_is_running():
try:
return not subprocess.check_output("docker info >/dev/null 2>&1",
shell=True)
- except subprocess.CalledProcessError as e:
+ except subprocess.CalledProcessError:
msg = "Docker is not running. Please start docker service on your
machine\n"
sys.stderr.write(f"ERROR: {msg}")
raise RuntimeError(msg)
@@ -61,39 +65,53 @@ def build(path):
def deploy_liminal_core_internal(clean):
with pkg_resources.path(dag, 'liminal_dags.py') as p:
- dags_path=p
+ dags_path = p
os.makedirs(environment.get_dags_dir(), exist_ok=True)
dag_target_file = os.path.join(environment.get_liminal_home(),
'liminal_dags.py')
shutil.copyfile(dags_path, dag_target_file)
- #initialize the env. variable which indicates to the docke compose which
liminal to install in airflow docker
+ # initialize the env. variable which indicates to the docke compose which
+ # liminal to install in airflow docker
liminal_version = environment.get_liminal_version()
- # if liminal is installed from local file - teh developer needs to put it
in the /scripts folder, in which case
- # it will end up inside the container during build
- if liminal_version.find("file://")>-1:
+ print(f'Deploying liminal version: {liminal_version}')
+ # if liminal is installed from local file - the developer needs to put it
in the /scripts folder
+ # in which case it will end up inside the container during build
+ if liminal_version.find("file://") > -1:
local_file_name = os.path.basename(liminal_version)
- full_path = os.path.join('/opt/airflow/dags',local_file_name)
- print(f'Liminal was installed locally, changing the LIMINAL_VERSION
parameter to {full_path}')
- os.environ[environment.LIMINAL_VERSION_PARAM_NAME]= full_path
+ full_path = os.path.join('/opt/airflow/dags', local_file_name)
+ print(
+ f'Liminal was installed locally, changing the LIMINAL_VERSION
parameter to {full_path}')
+ os.environ[environment.LIMINAL_VERSION_PARAM_NAME] = full_path
if clean:
- docker_compose_command('down', ['--remove-orphans','--rmi', 'local'])
+ docker_compose_command('down', ['--remove-orphans', '--rmi', 'local'])
docker_compose_command('build', ['--no-cache'])
docker_compose_command('run', ['webserver', 'resetdb', '-y'])
- docker_compose_command('run', ['webserver','initdb'])
+ docker_compose_command('run', ['webserver', 'initdb'])
docker_compose_command('down', ['--remove-orphans'])
+
def docker_compose_command(command_name, args):
docker_compose_path, project_dir = get_docker_compose_paths()
concated_args = ' '.join(args)
- run_command = [f'docker-compose -f "{docker_compose_path}" -p liminal
--project-directory {project_dir} {command_name} {concated_args}']
+ run_command = [
+ 'docker-compose '
+ f'-f "{docker_compose_path}" '
+ '-p liminal --project-directory '
+ f'{project_dir} {command_name} {concated_args}'
+ ]
+ print(run_command[0])
if 'follow' in str(args):
subprocess.call(run_command, env=os.environ, shell=True)
+ return '', ''
else:
- output = subprocess.run(run_command,env=os.environ, shell=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
- return output.stdout, output.stderr
+ output = subprocess.run(run_command, env=os.environ, shell=True,
stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
universal_newlines=True)
+ return output.stdout, output.stderr
+
@cli.command("deploy", short_help="deploys your liminal.yaml files to
$LIMINAL_HOME folder")
@click.option('--path', default=os.getcwd(), help="folder containing
liminal.yaml files")
[email protected]('--clean', is_flag=True, default=False, help="re-deploy liminal
airflow components from scratch")
[email protected]('--clean', is_flag=True, default=False,
+ help="re-deploy liminal airflow components from scratch")
def deploy_liminal_apps(path, clean):
click.echo("deploying liminal yaml files")
liminal_home = environment.get_liminal_home()
@@ -106,10 +124,15 @@ def deploy_liminal_apps(path, clean):
yml_name = os.path.basename(config_file)
target_yml_name = os.path.join(environment.get_dags_dir(), yml_name)
shutil.copyfile(config_file, target_yml_name)
+ with open(config_file) as stream:
+ config = yaml.safe_load(stream)
+ volume_util.create_local_volumes(config,
os.path.dirname(config_file))
+
def liminal_is_running():
- stdout, stderr = docker_compose_command('ps', [])
- return "liminal" in stdout
+ stdout, stderr = docker_compose_command('ps', [])
+ return "liminal" in stdout
+
@cli.command("stop", short_help="stops the docker compose")
def stop():
@@ -118,27 +141,31 @@ def stop():
environment.get_liminal_home()
docker_compose_command('down', [])
+
@cli.command("logs", short_help="display the docker compose logs")
@click.option('--follow', '-f', is_flag=True, default=False, help="Follow log
output.")
[email protected]('--tail', '-t', default=0, type=int, help="Number of lines to
show from the end of the log")
[email protected]('--tail', '-t', default=0, type=int,
+ help="Number of lines to show from the end of the log")
def logs(follow, tail):
if docker_is_running() and liminal_is_running():
# initialize liminal home by default
environment.get_liminal_home()
- if (follow):
- docker_compose_command('logs', ['--follow'])
- if (tail>0):
- stdout, stderr = docker_compose_command('logs',
[f'--tail={tail}'])
- print(stdout)
+ if follow:
+ docker_compose_command('logs', ['--follow'])
+ if tail > 0:
+ stdout, stderr = docker_compose_command('logs', [f'--tail={tail}'])
+ print(stdout)
+
@cli.command("start",
short_help="starts a local airflow in docker compose. should be
run after deploy. " +
"Make sure docker is running on your machine")
def start():
+ liminal_version = environment.get_liminal_version()
+ print(f'starting liminal version: {liminal_version}')
if docker_is_running():
# initialize liminal home by default
environment.get_liminal_home()
- docker_compose_path, project_dir = get_docker_compose_paths()
docker_compose_command('up', ['-d'])
@@ -146,7 +173,6 @@ def get_docker_compose_paths():
with pkg_resources.path(s, 'docker-compose.yml') as p:
docker_compose_path = p
project_dir = pathlib.Path(docker_compose_path).parent.parent.absolute()
- print(f"starting docker compose from path {project_dir}")
return docker_compose_path, project_dir
diff --git a/scripts/requirements-airflow.txt b/scripts/requirements-airflow.txt
index 5191d2a..5cf3142 100644
--- a/scripts/requirements-airflow.txt
+++ b/scripts/requirements-airflow.txt
@@ -2,4 +2,4 @@ click==7.1.1
pyyaml
boto3==1.12.10
botocore==1.15.21
-kubernetes
\ No newline at end of file
+kubernetes
