This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 132e899a1132988e99281215fd6d9e9c37bd0369 Author: Jarek Potiuk <[email protected]> AuthorDate: Mon Aug 24 22:45:28 2020 +0200 Updated documentation for the CI with mermaid sequence diagrams (#10380) (cherry picked from commit f2da6b419fde71d335d133031f2c0fac28b87289) --- .pre-commit-config.yaml | 11 ++- .rat-excludes | 1 + images/ci/CI.png | Bin 0 -> 243004 bytes images/ci/pull_request_ci_flow.md5 | 1 + images/ci/pull_request_ci_flow.mermaid | 108 +++++++++++++++++++++++++ images/ci/pull_request_ci_flow.png | Bin 0 -> 195487 bytes images/ci/push_ci_flow.md5 | 1 + images/ci/push_ci_flow.mermaid | 117 ++++++++++++++++++++++++++++ images/ci/push_ci_flow.png | Bin 0 -> 209013 bytes images/ci/scheduled_ci_flow.md5 | 1 + images/ci/scheduled_ci_flow.mermaid | 111 ++++++++++++++++++++++++++ images/ci/scheduled_ci_flow.png | Bin 0 -> 201540 bytes scripts/ci/pre_commit/pre_commit_mermaid.sh | 97 +++++++++++++++++++++++ 13 files changed, 447 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1b5c436..6763164 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -128,6 +128,15 @@ repos: - --license-filepath - license-templates/LICENSE.txt - --fuzzy-match-generates-todo + - id: insert-license + name: Add license for all mermaid files + args: + - --comment-style + - "|%%|" + - --license-filepath + - license-templates/LICENSE.txt + - --fuzzy-match-generates-todo + files: \.mermaid$ - repo: https://github.com/thlorenz/doctoc.git rev: v1.4.0 hooks: @@ -277,7 +286,7 @@ repos: entry: ./scripts/ci/pre_commit/pre_commit_check_integrations.sh language: system pass_filenames: false - files: ^airflow/scripts/ci/libraries/_initialization.sh$|^breeze-complete$ + files: ^common/_common_values.sh$|^breeze-complete$ - id: check-apache-license name: Check if licenses are OK for Apache entry: "./scripts/ci/pre_commit/pre_commit_check_license.sh" diff --git a/.rat-excludes b/.rat-excludes index 497d7ed..69c8ccc 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -28,6 +28,7 @@ metastore_db .*sql .*svg .*csv +.*md5 CHANGELOG.txt .*zip .*lock diff --git a/images/ci/CI.png b/images/ci/CI.png new file mode 100644 index 0000000..7123519 Binary files /dev/null and b/images/ci/CI.png differ diff --git a/images/ci/pull_request_ci_flow.md5 b/images/ci/pull_request_ci_flow.md5 new file mode 100644 index 0000000..3fe05dd --- /dev/null +++ b/images/ci/pull_request_ci_flow.md5 @@ -0,0 +1 @@ +e3aa93e8b46ca97b4aad06de23a10ad5 images/ci/pull_request_ci_flow.mermaid diff --git a/images/ci/pull_request_ci_flow.mermaid b/images/ci/pull_request_ci_flow.mermaid new file mode 100644 index 0000000..2e4bf9c --- /dev/null +++ b/images/ci/pull_request_ci_flow.mermaid @@ -0,0 +1,108 @@ +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. + +sequenceDiagram + Contributor -->> Fork: Submit PR + Fork -->> Airflow Repo: Start Pull Request + activate Airflow Repo + Note over Airflow Repo: Trigger PR build + Note over CI Build: pull_request<br>[Read Token] + Note over Build Images: workflow_run<br>[Write Token] + Airflow Repo -->> CI Build: Trigger 'pull_request' event + activate CI Build + CI Build -->> Build Images: Trigger 'workflow_run' event + activate Build Images + Note over Build Images: Find Duplicate<br>[CI Build] + opt Duplicated builds + Build Images -->> CI Build: Cancel + end + Note over Build Images: Find Duplicates<br>[Build Image] + opt Duplicated builds + Build Images -->> Build Images: Cancel + end + Note over Build Images: Find failed<br>[CI Build] + opt Failed builds + Build Images -->> CI Build: Cancel + end + Note over Build Images: Find failed<br>[Build Image] + opt Failed builds + Build Images -->> Build Images: Cancel + end + Note over Build Images: Build info + par 3.6, 3.7, 3.8 + Note over Build Images: Build PROD Images<br>[GITHUB_RUN_ID] + and + Note over Build Images: Build CI Images<br>[GITHUB_RUN_ID] + end + par No CI image + Note over CI Build: Build info + and + Note over CI Build: Should trigger tests? + and + Note over CI Build: Helm tests + and + Note over CI Build: OpenAPI client gen + end + par 3.6, 3.7, 3.8 + Build Images ->> GitHub Registry: Push CI Images + activate GitHub Registry + and + Build Images ->> GitHub Registry: Push PROD Images + end + opt If any step failed + Build Images -->> CI Build: Cancel triggering run + end + deactivate Build Images + Note over GitHub Registry: Tagged Images<br>[GITHUB_RUN_ID] + loop Wait for images + par 3.6, 3.7, 3.8 + CI Build ->> CI Build: Pull CI Images + Note over CI Build: Wait for<br>[GITHUB_RUN_ID] + and + CI Build ->> CI Build: Pull PROD Images + Note over CI Build: Wait for<br>[GITHUB_RUN_ID] + end + end + par 3.6, 3.7, 3.8 + GitHub Registry ->> CI Build: Pull CI Images + and + GitHub Registry ->> CI Build: Pull PROD Images + end + deactivate GitHub Registry + par 3.6, 3.7, 3.8 + Note over CI Build: Run static checks + and + Note over CI Build: Build docs + and + Note over CI Build: Spell check docs + and + Note over CI Build: Backport packages + and + opt Triggered? + Note over CI Build: Tests + end + and + opt Triggered? + Note over CI Build: Kubernetes Tests + end + end + Note over CI Build: Quarantined tests + Note over CI Build: Merge Coverage + CI Build -->> Coverage.io: Upload Coverage + CI Build -->> Airflow Repo: Status Check for CI Build + deactivate Airflow Repo + deactivate CI Build diff --git a/images/ci/pull_request_ci_flow.png b/images/ci/pull_request_ci_flow.png new file mode 100644 index 0000000..7fec85a Binary files /dev/null and b/images/ci/pull_request_ci_flow.png differ diff --git a/images/ci/push_ci_flow.md5 b/images/ci/push_ci_flow.md5 new file mode 100644 index 0000000..edaa0cc --- /dev/null +++ b/images/ci/push_ci_flow.md5 @@ -0,0 +1 @@ +95db6b17e315a47318eaf6260b3f0fd3 images/ci/push_ci_flow.mermaid diff --git a/images/ci/push_ci_flow.mermaid b/images/ci/push_ci_flow.mermaid new file mode 100644 index 0000000..790f800 --- /dev/null +++ b/images/ci/push_ci_flow.mermaid @@ -0,0 +1,117 @@ +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. + +sequenceDiagram + Committer -->> Airflow Repo: Merge PR + activate Airflow Repo + Note over Airflow Repo: Trigger PR build + Note over CI Build: push<br>[Write Token] + Note over Build Images: workflow_run<br>[Write Token] + Airflow Repo -->> CI Build: Trigger 'pull_request' event + activate CI Build + CI Build -->> Build Images: Trigger 'workflow_run' event + activate Build Images + Note over Build Images: Find Duplicate<br>[CI Build] + opt Duplicated builds + Build Images -->> CI Build: Cancel + end + Note over Build Images: Find Duplicates<br>[Build Image] + opt Duplicated builds + Build Images -->> Build Images: Cancel + end + Note over Build Images: Find failed<br>[CI Build] + opt Failed builds + Build Images -->> CI Build: Cancel + end + Note over Build Images: Find failed<br>[Build Image] + opt Failed builds + Build Images -->> Build Images: Cancel + end + Note over Build Images: Build info + par 3.6, 3.7, 3.8 + Note over Build Images: Update constraints + Note over Build Images: Build PROD Images<br>[GITHUB_RUN_ID] + and + Note over Build Images: Update constraints + Note over Build Images: Build CI Images<br>[GITHUB_RUN_ID] + end + par No CI image + Note over CI Build: Build info + and + Note over CI Build: Test always! + and + Note over CI Build: Helm tests + and + Note over CI Build: OpenAPI client gen + end + par 3.6, 3.7, 3.8 + Build Images ->> GitHub Registry: Push CI Images + activate GitHub Registry + and + Build Images ->> GitHub Registry: Push PROD Images + end + opt If any step failed + Build Images -->> CI Build: Cancel triggering run + end + deactivate Build Images + Note over GitHub Registry: Tagged Images<br>[GITHUB_RUN_ID] + loop Wait for images + par 3.6, 3.7, 3.8 + CI Build ->> CI Build: Pull CI Images + Note over CI Build: Wait for<br>[GITHUB_RUN_ID] + and + CI Build ->> CI Build: Pull PROD Images + Note over CI Build: Wait for<br>[GITHUB_RUN_ID] + end + end + par 3.6, 3.7, 3.8 + GitHub Registry ->> CI Build: Pull CI Images + and + GitHub Registry ->> CI Build: Pull PROD Images + end + deactivate GitHub Registry + par 3.6, 3.7, 3.8 + Note over CI Build: Run static checks + and + Note over CI Build: Build docs + and + Note over CI Build: Spell check docs + and + Note over CI Build: Backport packages + and + Note over CI Build: Tests + and + Note over CI Build: Kubernetes Tests + end + Note over CI Build: Quarantined tests + Note over CI Build: Merge Coverage + CI Build -->> Coverage.io: Upload Coverage + Note over CI Build: Tag image<br>[COMMIT_SHA] + par 3.6, 3.7, 3.8 + CI Build ->> GitHub Registry: Push CI Images + activate GitHub Registry + and + CI Build ->> GitHub Registry: Push PROD Images + end + Note over GitHub Registry: Tagged Images<br>[COMMIT_SHA] + par 3.6, 3.7, 3.8 + CI Build ->> Airflow Repo: Push Constraints + end + deactivate GitHub Registry + CI Build -->> Airflow Repo: Status Check for CI Build + deactivate Airflow Repo + deactivate CI Build diff --git a/images/ci/push_ci_flow.png b/images/ci/push_ci_flow.png new file mode 100644 index 0000000..81c19bb Binary files /dev/null and b/images/ci/push_ci_flow.png differ diff --git a/images/ci/scheduled_ci_flow.md5 b/images/ci/scheduled_ci_flow.md5 new file mode 100644 index 0000000..8ae6938 --- /dev/null +++ b/images/ci/scheduled_ci_flow.md5 @@ -0,0 +1 @@ +f6ea87f0b4d990fd48a9dbec2a2a4f2d images/ci/scheduled_ci_flow.mermaid diff --git a/images/ci/scheduled_ci_flow.mermaid b/images/ci/scheduled_ci_flow.mermaid new file mode 100644 index 0000000..b892a7a --- /dev/null +++ b/images/ci/scheduled_ci_flow.mermaid @@ -0,0 +1,111 @@ +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. + +sequenceDiagram + CRON job -->> Airflow Repo: Nightly run + activate Airflow Repo + Note over Airflow Repo: Trigger PR build + Note over CI Build: push<br>[Write Token] + Note over Build Images: workflow_run<br>[Write Token] + Airflow Repo -->> CI Build: Trigger 'pull_request' event + activate CI Build + CI Build -->> Build Images: Trigger 'workflow_run' event + activate Build Images + Note over Build Images: Find Duplicate<br>[CI Build] + opt Duplicated builds + Build Images -->> CI Build: Cancel + end + Note over Build Images: Find Duplicates<br>[Build Image] + opt Duplicated builds + Build Images -->> Build Images: Cancel + end + Note over Build Images: Find failed<br>[CI Build] + opt Failed builds + Build Images -->> CI Build: Cancel + end + Note over Build Images: Find failed<br>[Build Image] + opt Failed builds + Build Images -->> Build Images: Cancel + end + Note over Build Images: Build info + par 3.6, 3.7, 3.8 + Note over Build Images: Build from scratch + Note over Build Images: Update constraints + Note over Build Images: Build PROD Images<br>[GITHUB_RUN_ID] + and + Note over Build Images: Build from scratch + Note over Build Images: Update constraints + Note over Build Images: Build CI Images<br>[GITHUB_RUN_ID] + end + par No CI image + Note over CI Build: Build info + and + Note over CI Build: Test always! + and + Note over CI Build: Helm tests + and + Note over CI Build: OpenAPI client gen + end + par 3.6, 3.7, 3.8 + Build Images ->> GitHub Registry: Push CI Images + activate GitHub Registry + and + Build Images ->> GitHub Registry: Push PROD Images + end + opt If any step failed + Build Images -->> CI Build: Cancel triggering run + end + deactivate Build Images + Note over GitHub Registry: Tagged Images<br>[GITHUB_RUN_ID] + loop Wait for images + par 3.6, 3.7, 3.8 + CI Build ->> CI Build: Pull CI Images + Note over CI Build: Wait for<br>[GITHUB_RUN_ID] + and + CI Build ->> CI Build: Pull PROD Images + Note over CI Build: Wait for<br>[GITHUB_RUN_ID] + end + end + par 3.6, 3.7, 3.8 + GitHub Registry ->> CI Build: Pull CI Images + and + GitHub Registry ->> CI Build: Pull PROD Images + end + deactivate GitHub Registry + par 3.6, 3.7, 3.8 + Note over CI Build: Run static checks + and + Note over CI Build: Build docs + and + Note over CI Build: Spell check docs + and + Note over CI Build: Backport packages + and + Note over CI Build: Tests + and + Note over CI Build: Kubernetes Tests + end + Note over CI Build: Quarantined tests + Note over CI Build: Merge Coverage + CI Build -->> Coverage.io: Upload Coverage + par 3.6, 3.7, 3.8 + CI Build ->> Airflow Repo: Push Constraints + end + CI Build ->> Airflow Repo: Push Nightly Tag + CI Build -->> Airflow Repo: Status Check for CI Build + deactivate Airflow Repo + deactivate CI Build diff --git a/images/ci/scheduled_ci_flow.png b/images/ci/scheduled_ci_flow.png new file mode 100644 index 0000000..553ccfc Binary files /dev/null and b/images/ci/scheduled_ci_flow.png differ diff --git a/scripts/ci/pre_commit/pre_commit_mermaid.sh b/scripts/ci/pre_commit/pre_commit_mermaid.sh new file mode 100755 index 0000000..fc52c91 --- /dev/null +++ b/scripts/ci/pre_commit/pre_commit_mermaid.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +set -euo pipefail +AIRFLOW_SOURCES="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd ../../../ && pwd )" +export NO_TERMINAL_OUTPUT_FROM_SCRIPTS="true" + +if ! command -v npm; then + echo 'You need to have npm installed in order to generate .mermaid graphs automatically.' + echo + # Do not fail. This is no problem if those images are not regenerated. + exit 0 +fi + +tmp_file=$(mktemp) + +cd "${AIRFLOW_SOURCES}" + +MERMAID_INSTALLATION_DIR="${AIRFLOW_SOURCES}/.build/mermaid/" +MERMAID_CONFIG_FILE="${MERMAID_INSTALLATION_DIR}/mermaid-config.json" +MERMAID_CLI="${MERMAID_INSTALLATION_DIR}/node_modules/.bin/mmdc" +export NODE_VIRTUAL_ENV="${MERMAID_INSTALLATION_DIR}" + +if [[ -f "${MERMAID_CLI}" ]]; then + MERMAID_INSTALLED="true" +else + MERMAID_INSTALLED="false" +fi + +# shellcheck disable=SC2064 +trap "rm -rf ${tmp_file}" EXIT + +for file in "${@}" +do + basename_file=${AIRFLOW_SOURCES}/"$(dirname "${file}")/$(basename "${file}" .mermaid)" + md5sum_file="${basename_file}.md5" + if ! diff "${md5sum_file}" <(md5sum "${file}"); then + if [[ ${MERMAID_INSTALLED} != "true" ]]; then + echo "Installing mermaid" + mkdir -p "${MERMAID_INSTALLATION_DIR}/node_modules" + pushd "${MERMAID_INSTALLATION_DIR}" + npm install mermaid.cli + cat >"${MERMAID_CONFIG_FILE}" <<EOF +{ + "theme": "default", + "themeCSS": ".label foreignObject { overflow: visible; }" +} +EOF + MERMAID_INSTALLED="true" + popd + fi + echo "Running generation for ${file}" + rm -f "${basename_file}.png" + rm -f "${basename_file}.md5" + # unfortunately mermaid does not handle well multiline comments and we need licence comment + # Stripping them manually :(. Multiline comments are coming in the future + # https://github.com/mermaid-js/mermaid/issues/1249 + grep -v "^%%" <"${file}" > "${tmp_file}" + mkdir -p "${MERMAID_INSTALLATION_DIR}" + + "${MERMAID_CLI}" \ + -i "${tmp_file}" \ + -w 2048 \ + -o "${basename_file}.png" \ + -c "${MERMAID_CONFIG_FILE}" + if [ -f "${basename_file}.png" ]; then + md5sum "${file}" >"${md5sum_file}" + echo + echo "Successfully generated: ${basename_file}.png" + echo "Successfully updated: ${basename_file}.md5" + echo + echo "Please add both files and commit them to repository" + echo + else + 1>&2 echo + 1>&2 echo "ERROR: Could not generate ${basename_file}.png" + 1>&2 echo + exit 1 + fi + else + echo "Skip regenerating file ${file} -> it's hash did not change in ${md5sum_file}" + fi +done
