This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 02f637face85f99f693d2ad668c7f4eb170783b7 Author: mucio <[email protected]> AuthorDate: Sun Sep 27 18:31:50 2020 +0200 New Breeze command start-airflow, it replaces the previous flag (#11157) (cherry picked from commit 0db7a307820639627930a9c0df219bc489b58867) --- BREEZE.rst | 46 ++++++++++--- breeze | 78 ++++++++++++++++++---- breeze-complete | 4 +- scripts/ci/docker-compose/base.yml | 3 + scripts/ci/libraries/_docker.env | 2 + scripts/ci/libraries/_initialization.sh | 5 ++ scripts/in_container/check_environment.sh | 32 ++------- scripts/in_container/entrypoint_ci.sh | 3 + scripts/in_container/entrypoint_exec.sh | 3 + scripts/in_container/run_tmux.sh | 53 +++++++++++++++ .../run_tmux_welcome.sh} | 21 ++---- .../stop_tmux_airflow.sh} | 23 +++---- 12 files changed, 194 insertions(+), 79 deletions(-) diff --git a/BREEZE.rst b/BREEZE.rst index 11f7cdb..4401da2 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -308,7 +308,7 @@ In case the problems are not solved, you can set the VERBOSE_COMMANDS variable t Then run the failed command, copy-and-paste the output from your terminal to the -`Airflow Slack <https://apache-airflow-slack.herokuapp.com/>`_ #airflow-breeze channel and +`Airflow Slack <https://s.apache.org/airflow-slack>`_ #airflow-breeze channel and describe your problem. Other uses of the Airflow Breeze environment @@ -1054,6 +1054,7 @@ This is the current syntax for `./breeze <./breeze>`_: push-image Pushes images to registry initialize-local-virtualenv Initializes local virtualenv setup-autocomplete Sets up autocomplete for breeze + start-airflow Starts Scheduler and Webserver and enters the shell stop Stops the docker-compose environment restart Stops the docker-compose environment including DB cleanup toggle-suppress-cheatsheet Toggles on/off cheatsheet @@ -1485,6 +1486,32 @@ This is the current syntax for `./breeze <./breeze>`_: #################################################################################################### + Detailed usage for command: start-airflow + + + breeze start-airflow + + Like the Shell command this will enter the interactive shell, but it will also start + automatically the Scheduler and the Webserver. It will leave you in a tmux session where you + can also observe what is happening in your Airflow. + + This is a convenient way to setup a development environment. Your dags will be loaded from the + folder 'files/dags' on your host machine (it could take some times). + + If you want to load default connections and example dags you can use the dedicated flags. + + Flags: + + --load-example-dags + Include Airflow example dags. + + --load-default-connections + Include Airflow Default Connections. + + + #################################################################################################### + + Detailed usage for command: stop @@ -1827,18 +1854,21 @@ This is the current syntax for `./breeze <./breeze>`_: cassandra kerberos mongo openldap presto rabbitmq redis all - --start-airflow - Starts the Airflow Scheduler and Webserver in two tmux panes, a third one will be - available for the usual use cases. In case the database is not initialized it will - run also 'airflow db init' and create an Admin user with credentials admin/admin. - --rbac-ui Enable RBAC UI. --init-script <INIT_SCRIPT_FILE> Initialization script name - Sourced from files/airflow-breeze-config. Default value - init.sh. It will be executed after the environment is configured and started (in case - --start-airflow is used). + init.sh. It will be executed after the environment is configured and started. + + **************************************************************************************************** + Additional actions executed starting Airflow + + --load-example-dags + Include Airflow example dags. + + --load-default-connections + Include Airflow Default Connections. **************************************************************************************************** Kind kubernetes and Kubernetes tests configuration(optional) diff --git a/breeze b/breeze index 6c476a7..bd80d1f 100755 --- a/breeze +++ b/breeze @@ -118,6 +118,12 @@ function breeze::setup_default_breeze_constants() { # If set to true, the database will be initialized, a user created and webserver and scheduler started export START_AIRFLOW="false" + # If set to true, the test connections will be created + export LOAD_DEFAULT_CONNECTIONS="false" + + # If set to true, the sample dags will be created + export LOAD_EXAMPLES="false" + # If set to true, RBAC mode is enabled export RBAC_UI="false" @@ -1006,12 +1012,6 @@ function breeze::parse_arguments() { export SKIP_CHECK_REMOTE_IMAGE="true" shift 2 ;; - --start-airflow) - echo "Starting Airflow" - echo - export START_AIRFLOW="true" - shift - ;; --init-script) export INIT_SCRIPT_FILE="${2}" echo "The initialization file is in ${INIT_SCRIPT_FILE}" @@ -1024,6 +1024,18 @@ function breeze::parse_arguments() { export RBAC_UI="true" shift ;; + --load-example-dags) + export LOAD_EXAMPLES="true" + echo "Include Airflow sample dags" + echo + shift + ;; + --load-default-connections) + export LOAD_DEFAULT_CONNECTIONS="true" + echo "Include Airflow default connections" + echo + shift + ;; --) shift break @@ -1151,6 +1163,11 @@ function breeze::parse_arguments() { shift 2 fi ;; + start-airflow) + last_subcommand="${1}" + export START_AIRFLOW="true" + shift + ;; stop) last_subcommand="${1}" command_to_run="run_docker_compose" @@ -1359,6 +1376,8 @@ function breeze::prepare_usage() { readonly USAGE_KIND_CLUSTER export USAGE_SETUP_AUTOCOMPLETE="Sets up autocomplete for breeze" readonly USAGE_SETUP_AUTOCOMPLETE + export USAGE_START_AIRFLOW="Starts Scheduler and Webserver and enters the shell" + readonly USAGE_START_AIRFLOW export USAGE_STOP="Stops the docker-compose environment" readonly USAGE_STOP export USAGE_RESTART="Stops the docker-compose environment including DB cleanup" @@ -1582,6 +1601,22 @@ ${CMDNAME} setup-autocomplete parameters and values. " readonly DETAILED_USAGE_SETUP_AUTOCOMPLETE + export DETAILED_USAGE_START_AIRFLOW=" +${CMDNAME} start-airflow + + Like the Shell command this will enter the interactive shell, but it will also start + automatically the Scheduler and the Webserver. It will leave you in a tmux session where you + can also observe what is happening in your Airflow. + + This is a convenient way to setup a development environment. Your dags will be loaded from the + folder 'files/dags' on your host machine (it could take some times). + + If you want to load default connections and example dags you can use the dedicated flags. + +Flags: +$(breeze::flag_start_airflow) +" + readonly DETAILED_USAGE_START_AIRFLOW export DETAILED_USAGE_STOP=" ${CMDNAME} stop @@ -1883,18 +1918,12 @@ function breeze::flag_breeze_actions() { ${FORMATTED_INTEGRATIONS} ---start-airflow - Starts the Airflow Scheduler and Webserver in two tmux panes, a third one will be - available for the usual use cases. In case the database is not initialized it will - run also 'airflow db init' and create an Admin user with credentials admin/admin. - --rbac-ui Enable RBAC UI. --init-script <INIT_SCRIPT_FILE> Initialization script name - Sourced from files/airflow-breeze-config. Default value - init.sh. It will be executed after the environment is configured and started (in case - --start-airflow is used). + init.sh. It will be executed after the environment is configured and started. " } @@ -2190,6 +2219,23 @@ function breeze::flag_pull_push_docker_images() { " } +##################################################################################################### +# +# Prints flags that control how Airflow should be populated with the command start-airflow +# +# Outputs: +# Flag information. +####################################################################################################### +function breeze::flag_start_airflow() { + echo " +--load-example-dags + Include Airflow example dags. + +--load-default-connections + Include Airflow Default Connections. + +" +} ####################################################################################################### # @@ -2221,6 +2267,10 @@ $(breeze::print_star_line) $(breeze::flag_breeze_actions) $(breeze::print_star_line) + Additional actions executed starting Airflow +$(breeze::flag_start_airflow) + +$(breeze::print_star_line) Kind kubernetes and Kubernetes tests configuration(optional) $(breeze::flag_kubernetes_configuration) @@ -2408,7 +2458,7 @@ function breeze::check_and_save_all_params() { ####################################################################################################### function breeze::print_cheatsheet() { if [[ ! -f ${SUPPRESS_CHEATSHEET_FILE} && ${command_to_run} == "enter_breeze" ]]; then - echo + echo breeze::print_line echo diff --git a/breeze-complete b/breeze-complete index 9abb033..e281e43 100644 --- a/breeze-complete +++ b/breeze-complete @@ -129,12 +129,13 @@ _breeze_long_options=" help python: backend: integration: kubernetes-mode: kubernetes-version: helm-version: kind-version: skip-mounting-local-sources install-airflow-version: install-airflow-reference: db-reset -verbose assume-yes assume-no assume-quit forward-credentials start-airflow rbac-ui init-script: +verbose assume-yes assume-no assume-quit forward-credentials rbac-ui init-script: force-build-images force-pull-images production-image extras: force-clean-images skip-rebuild-check build-cache-local build-cache-pulled build-cache-disabled disable-pip-cache dockerhub-user: dockerhub-repo: github-registry github-repository: github-image-id: postgres-version: mysql-version: additional-extras: additional-python-deps: additional-dev-deps: additional-runtime-deps: +load-default-connections load-example-dags " _breeze_commands=" @@ -147,6 +148,7 @@ generate-constraints push-image initialize-local-virtualenv setup-autocomplete +start-airflow stop restart toggle-suppress-cheatsheet diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml index 3a25a86..706d90c 100644 --- a/scripts/ci/docker-compose/base.yml +++ b/scripts/ci/docker-compose/base.yml @@ -41,6 +41,7 @@ services: - RUN_INTEGRATION_TESTS - ONLY_RUN_LONG_RUNNING_TESTS - ONLY_RUN_QUARANTINED_TESTS + - ONLY_RUN_HEISEN_TESTS - GITHUB_TOKEN - GITHUB_REPOSITORY - ISSUE_ID @@ -49,6 +50,8 @@ services: - INSTALL_AIRFLOW_VERSION - DB_RESET - START_AIRFLOW + - LOAD_EXAMPLES + - LOAD_DEFAULT_CONNECTIONS - RBAC_UI - ENABLED_SYSTEMS - RUN_SYSTEM_TESTS diff --git a/scripts/ci/libraries/_docker.env b/scripts/ci/libraries/_docker.env index d5ae770..986b7b4 100644 --- a/scripts/ci/libraries/_docker.env +++ b/scripts/ci/libraries/_docker.env @@ -28,3 +28,5 @@ VERSION_SUFFIX_FOR_PYPI VERSION_SUFFIX_FOR_SVN PRINT_INFO_FROM_SCRIPTS CI +LOAD_DEFAULT_CONNECTIONS +LOAD_EXAMPLES diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index afc8bee..7d28a10 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -111,6 +111,9 @@ function initialization::initialize_base_variables() { # If set to true, the database will be initialized, a user created and webserver and scheduler started export START_AIRFLOW=${START_AIRFLOW:="false"} + export LOAD_EXAMPLES=${LOAD_EXAMPLES:="false"} + + export LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS:="false"} # If set the specified file will be used to initialized Airflow after the environment is created, # otherwise it will use files/airflow-breeze-config/init.sh export INIT_SCRIPT_FILE=${INIT_SCRIPT_FILE:=""} @@ -519,6 +522,8 @@ Detected CI build environment: Initialization variables: INIT_SCRIPT_FILE: ${INIT_SCRIPT_FILE} + LOAD_DEFAULT_CONNECTIONS: ${LOAD_DEFAULT_CONNECTIONS} + LOAD_EXAMPLES: ${LOAD_EXAMPLES} EOF diff --git a/scripts/in_container/check_environment.sh b/scripts/in_container/check_environment.sh index abc0b11..0a040a9 100755 --- a/scripts/in_container/check_environment.sh +++ b/scripts/in_container/check_environment.sh @@ -105,6 +105,10 @@ function enable_rbacui_if_requested() { function startairflow_if_requested() { if [[ ${START_AIRFLOW:="false"} == "true" ]]; then + + export AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS} + export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} + . "$( dirname "${BASH_SOURCE[0]}" )/configure_environment.sh" # initialize db @@ -116,34 +120,6 @@ function startairflow_if_requested() { . "$( dirname "${BASH_SOURCE[0]}" )/run_init_script.sh" - #this is because I run docker in WSL - Hi Bill! - export TMUX_TMPDIR=~/.tmux/tmp - mkdir -p ~/.tmux/tmp - chmod 777 -R ~/.tmux/tmp - - # Set Session Name - SESSION="Airflow" - - # Start New Session with our name - tmux new-session -d -s $SESSION - - # Name first Pane and start bash - tmux rename-window -t 0 'Main' - tmux send-keys -t 'Main' 'bash' C-m 'clear' C-m - - tmux split-window -v - tmux select-pane -t 1 - tmux send-keys 'airflow scheduler' C-m - - tmux split-window -h - tmux select-pane -t 2 - tmux send-keys 'airflow webserver' C-m - - # Attach Session, on the Main window - tmux select-pane -t 0 - tmux send-keys 'cd /opt/airflow/' C-m 'clear' C-m - - tmux attach-session -t $SESSION:0 fi return $? } diff --git a/scripts/in_container/entrypoint_ci.sh b/scripts/in_container/entrypoint_ci.sh index b7a5e3e..e74fe40 100755 --- a/scripts/in_container/entrypoint_ci.sh +++ b/scripts/in_container/entrypoint_ci.sh @@ -159,6 +159,9 @@ ssh-keyscan -H localhost >> ~/.ssh/known_hosts 2>/dev/null # shellcheck source=scripts/in_container/run_init_script.sh . "${IN_CONTAINER_DIR}/run_init_script.sh" +# shellcheck source=scripts/in_container/run_tmux.sh +. "${IN_CONTAINER_DIR}/run_tmux.sh" + cd "${AIRFLOW_SOURCES}" set +u diff --git a/scripts/in_container/entrypoint_exec.sh b/scripts/in_container/entrypoint_exec.sh index a9d729c..bce0bfc 100755 --- a/scripts/in_container/entrypoint_exec.sh +++ b/scripts/in_container/entrypoint_exec.sh @@ -21,4 +21,7 @@ # shellcheck source=scripts/in_container/run_init_script.sh . "$( dirname "${BASH_SOURCE[0]}" )/run_init_script.sh" +# shellcheck source=scripts/in_container/run_tmux.sh +. "$( dirname "${BASH_SOURCE[0]}" )/run_tmux.sh" + exec /bin/bash "${@}" diff --git a/scripts/in_container/run_tmux.sh b/scripts/in_container/run_tmux.sh new file mode 100755 index 0000000..2f67b21 --- /dev/null +++ b/scripts/in_container/run_tmux.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +if [[ ${START_AIRFLOW:="false"} == "true" ]]; then + + ln -s "${IN_CONTAINER_DIR}/stop_tmux_airflow.sh" stop_airflow.sh + + export AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS} + export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} + + #this is because I run docker in WSL - Hi Bill! + export TMUX_TMPDIR=~/.tmux/tmp + mkdir -p ~/.tmux/tmp + chmod 777 -R ~/.tmux/tmp + + # Set Session Name + export TMUX_SESSION="Airflow" + + # Start New Session with our name + tmux new-session -d -s "${TMUX_SESSION}" + + # Name first Pane and start bash + tmux rename-window -t 0 'Main' + tmux send-keys -t 'Main' 'bash' C-m 'clear' C-m + + tmux split-window -v + tmux select-pane -t 1 + tmux send-keys 'airflow scheduler' C-m + + tmux split-window -h + tmux select-pane -t 2 + tmux send-keys 'airflow webserver' C-m + + # Attach Session, on the Main window + tmux select-pane -t 0 + tmux send-keys "./scripts/in_container/run_tmux_welcome.sh" C-m + + tmux attach-session -t "${TMUX_SESSION}":0 +fi diff --git a/scripts/ci/libraries/_docker.env b/scripts/in_container/run_tmux_welcome.sh old mode 100644 new mode 100755 similarity index 76% copy from scripts/ci/libraries/_docker.env copy to scripts/in_container/run_tmux_welcome.sh index d5ae770..8c7015d --- a/scripts/ci/libraries/_docker.env +++ b/scripts/in_container/run_tmux_welcome.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -14,17 +15,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -INSTALL_AIRFLOW_VERSION -PYTHONDONTWRITEBYTECODE -VERBOSE -VERBOSE_COMMANDS -HOST_USER_ID -HOST_GROUP_ID -HOST_OS -HOST_HOME -HOST_AIRFLOW_SOURCES -PYTHON_MAJOR_MINOR_VERSION -VERSION_SUFFIX_FOR_PYPI -VERSION_SUFFIX_FOR_SVN -PRINT_INFO_FROM_SCRIPTS -CI +cd /opt/airflow/ || exit +clear +echo "Welcome to your tmux based running Airflow environment (courtesy of Breeze)." +echo +echo " To stop Airflow and exit tmux just './stop_airflow.sh'." +echo diff --git a/scripts/ci/libraries/_docker.env b/scripts/in_container/stop_tmux_airflow.sh old mode 100644 new mode 100755 similarity index 76% copy from scripts/ci/libraries/_docker.env copy to scripts/in_container/stop_tmux_airflow.sh index d5ae770..17884f2 --- a/scripts/ci/libraries/_docker.env +++ b/scripts/in_container/stop_tmux_airflow.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -14,17 +15,11 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -INSTALL_AIRFLOW_VERSION -PYTHONDONTWRITEBYTECODE -VERBOSE -VERBOSE_COMMANDS -HOST_USER_ID -HOST_GROUP_ID -HOST_OS -HOST_HOME -HOST_AIRFLOW_SOURCES -PYTHON_MAJOR_MINOR_VERSION -VERSION_SUFFIX_FOR_PYPI -VERSION_SUFFIX_FOR_SVN -PRINT_INFO_FROM_SCRIPTS -CI + +tmux select-pane -t 2 +tmux send-keys C-c C-m 'exit' C-m + +tmux select-pane -t 1 +tmux send-keys C-c C-m 'exit' C-m + +tmux kill-session -t "${TMUX_SESSION}"
