This is an automated email from the ASF dual-hosted git repository. assafpinhasi pushed a commit to branch official_docker in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git
commit e9f25bf27e5c2fdf33b016bcc24f08e754afb0e3 Author: Assaf Pinhasi <[email protected]> AuthorDate: Wed Nov 4 01:35:10 2020 +0200 fixing missing logs and making deploy --clean reset the DB first --- README.md | 2 ++ scripts/Dockerfile-airflow | 6 ++---- scripts/docker-compose.yml | 8 +++++--- scripts/liminal | 7 ++++--- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index bd16c96..d283f13 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,8 @@ When your pipeline code is ready, you can test it by running it locally on your 1. Ensure you have The Docker engine running locally, and enable a local Kubernetes cluster:  +And allocate it at least 3 CPUs (under "Resources" in the Docker preference UI). + If you want to execute your pipeline on a remote kubernetes cluster, make sure the cluster is configured using : ```bash diff --git a/scripts/Dockerfile-airflow b/scripts/Dockerfile-airflow index c2095d6..4abb2a4 100644 --- a/scripts/Dockerfile-airflow +++ b/scripts/Dockerfile-airflow @@ -3,11 +3,9 @@ FROM apache/airflow ARG LIMINAL_VERSION='apache-liminal' ARG DAG_FOLDER='/opt/airflow/dags/' ADD scripts/* ${DAG_FOLDER} -ADD liminal/runners/airflow/dag/liminal_dags.py ${DAG_FOLDER} - -USER root -RUN apt-get update && apt-get install -y --no-install-recommends git +RUN ls -ls ${DAG_FOLDER} +ADD liminal/runners/airflow/dag/liminal_dags.py ${DAG_FOLDER} USER airflow diff --git a/scripts/docker-compose.yml b/scripts/docker-compose.yml index df4ea90..17e7faa 100644 --- a/scripts/docker-compose.yml +++ b/scripts/docker-compose.yml @@ -6,9 +6,11 @@ - POSTGRES_USER=airflow - POSTGRES_PASSWORD=airflow - POSTGRES_DB= - + restart: always ports: - "5432:5432" + volumes: + - ${LIMINAL_HOME}/db:/var/lib/postgresql/data logging: options: max-size: 10m @@ -21,7 +23,6 @@ args: LIMINAL_VERSION: ${LIMINAL_VERSION} image: liminal-airflow - restart: always depends_on: - postgres environment: @@ -40,6 +41,7 @@ max-file: "3" volumes: - ${LIMINAL_HOME}:/opt/airflow/dags + - ${LIMINAL_HOME}/logs:/opt/airflow/logs - ${HOME}/.kube:/home/airflow/kube ports: - "8080:8080" @@ -58,7 +60,6 @@ args: LIMINAL_VERSION: ${LIMINAL_VERSION} image: liminal-airflow - restart: always depends_on: - postgres - webserver @@ -79,6 +80,7 @@ max-file: "3" volumes: - ${LIMINAL_HOME}:/opt/airflow/dags + - ${LIMINAL_HOME}/logs:/opt/airflow/logs - ${HOME}/.kube:/home/airflow/kube ports: - "8793:8793" diff --git a/scripts/liminal b/scripts/liminal index 76eb0e3..958a00e 100755 --- a/scripts/liminal +++ b/scripts/liminal @@ -67,7 +67,7 @@ def deploy_liminal_core_internal(clean): shutil.copyfile(dags_path, dag_target_file) #initialize the env. variable which indicates to the docke compose which liminal to install in airflow docker liminal_version = environment.get_liminal_version() - # if liminal is installed from local file - teh developer needs to put it in the /scipts folder, in which case + # if liminal is installed from local file - teh developer needs to put it in the /scripts folder, in which case # it will end up inside the container during build if liminal_version.find("file://")>-1: local_file_name = os.path.basename(liminal_version) @@ -77,14 +77,15 @@ def deploy_liminal_core_internal(clean): if clean: docker_compose_command('down', ['--rmi', 'local']) docker_compose_command('build', ['--no-cache']) + docker_compose_command('run', ['webserver', 'resetdb', '-y']) docker_compose_command('run', ['webserver','initdb']) - + docker_compose_command('down', []) def docker_compose_command(command_name, args): docker_compose_path, project_dir = get_docker_compose_paths() concated_args = ' '.join(args) subprocess.call( - [f'docker-compose -f "{docker_compose_path}" --project-directory {project_dir} {command_name} {concated_args}'], + [f'docker-compose -f "{docker_compose_path}" -p liminal --project-directory {project_dir} {command_name} {concated_args}'], env=os.environ, shell=True) @cli.command("deploy", short_help="deploys your liminal.yaml files to $LIMINAL_HOME folder")
