This is an automated email from the ASF dual-hosted git repository.
assafpinhasi pushed a commit to branch official_docker
in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git
The following commit(s) were added to refs/heads/official_docker by this push:
new e9f25bf fixing missing logs and making deploy --clean reset the DB
first
e9f25bf is described below
commit e9f25bf27e5c2fdf33b016bcc24f08e754afb0e3
Author: Assaf Pinhasi <[email protected]>
AuthorDate: Wed Nov 4 01:35:10 2020 +0200
fixing missing logs and making deploy --clean reset the DB first
---
README.md | 2 ++
scripts/Dockerfile-airflow | 6 ++----
scripts/docker-compose.yml | 8 +++++---
scripts/liminal | 7 ++++---
4 files changed, 13 insertions(+), 10 deletions(-)
diff --git a/README.md b/README.md
index bd16c96..d283f13 100644
--- a/README.md
+++ b/README.md
@@ -125,6 +125,8 @@ When your pipeline code is ready, you can test it by
running it locally on your
1. Ensure you have The Docker engine running locally, and enable a local
Kubernetes cluster:
+And allocate it at least 3 CPUs (under "Resources" in the Docker preference
UI).
+
If you want to execute your pipeline on a remote kubernetes cluster, make sure
the cluster is configured
using :
```bash
diff --git a/scripts/Dockerfile-airflow b/scripts/Dockerfile-airflow
index c2095d6..4abb2a4 100644
--- a/scripts/Dockerfile-airflow
+++ b/scripts/Dockerfile-airflow
@@ -3,11 +3,9 @@ FROM apache/airflow
ARG LIMINAL_VERSION='apache-liminal'
ARG DAG_FOLDER='/opt/airflow/dags/'
ADD scripts/* ${DAG_FOLDER}
-ADD liminal/runners/airflow/dag/liminal_dags.py ${DAG_FOLDER}
-
-USER root
-RUN apt-get update && apt-get install -y --no-install-recommends git
+RUN ls -ls ${DAG_FOLDER}
+ADD liminal/runners/airflow/dag/liminal_dags.py ${DAG_FOLDER}
USER airflow
diff --git a/scripts/docker-compose.yml b/scripts/docker-compose.yml
index df4ea90..17e7faa 100644
--- a/scripts/docker-compose.yml
+++ b/scripts/docker-compose.yml
@@ -6,9 +6,11 @@
- POSTGRES_USER=airflow
- POSTGRES_PASSWORD=airflow
- POSTGRES_DB=
-
+ restart: always
ports:
- "5432:5432"
+ volumes:
+ - ${LIMINAL_HOME}/db:/var/lib/postgresql/data
logging:
options:
max-size: 10m
@@ -21,7 +23,6 @@
args:
LIMINAL_VERSION: ${LIMINAL_VERSION}
image: liminal-airflow
- restart: always
depends_on:
- postgres
environment:
@@ -40,6 +41,7 @@
max-file: "3"
volumes:
- ${LIMINAL_HOME}:/opt/airflow/dags
+ - ${LIMINAL_HOME}/logs:/opt/airflow/logs
- ${HOME}/.kube:/home/airflow/kube
ports:
- "8080:8080"
@@ -58,7 +60,6 @@
args:
LIMINAL_VERSION: ${LIMINAL_VERSION}
image: liminal-airflow
- restart: always
depends_on:
- postgres
- webserver
@@ -79,6 +80,7 @@
max-file: "3"
volumes:
- ${LIMINAL_HOME}:/opt/airflow/dags
+ - ${LIMINAL_HOME}/logs:/opt/airflow/logs
- ${HOME}/.kube:/home/airflow/kube
ports:
- "8793:8793"
diff --git a/scripts/liminal b/scripts/liminal
index 76eb0e3..958a00e 100755
--- a/scripts/liminal
+++ b/scripts/liminal
@@ -67,7 +67,7 @@ def deploy_liminal_core_internal(clean):
shutil.copyfile(dags_path, dag_target_file)
#initialize the env. variable which indicates to the docke compose which
liminal to install in airflow docker
liminal_version = environment.get_liminal_version()
- # if liminal is installed from local file - teh developer needs to put it
in the /scipts folder, in which case
+ # if liminal is installed from local file - teh developer needs to put it
in the /scripts folder, in which case
# it will end up inside the container during build
if liminal_version.find("file://")>-1:
local_file_name = os.path.basename(liminal_version)
@@ -77,14 +77,15 @@ def deploy_liminal_core_internal(clean):
if clean:
docker_compose_command('down', ['--rmi', 'local'])
docker_compose_command('build', ['--no-cache'])
+ docker_compose_command('run', ['webserver', 'resetdb', '-y'])
docker_compose_command('run', ['webserver','initdb'])
-
+ docker_compose_command('down', [])
def docker_compose_command(command_name, args):
docker_compose_path, project_dir = get_docker_compose_paths()
concated_args = ' '.join(args)
subprocess.call(
- [f'docker-compose -f "{docker_compose_path}" --project-directory
{project_dir} {command_name} {concated_args}'],
+ [f'docker-compose -f "{docker_compose_path}" -p liminal
--project-directory {project_dir} {command_name} {concated_args}'],
env=os.environ, shell=True)
@cli.command("deploy", short_help="deploys your liminal.yaml files to
$LIMINAL_HOME folder")
