This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 882535a8a2 Replace generation of docker volumes to be done from python
(#23985)
882535a8a2 is described below
commit 882535a8a2699af7d1d079ecebd8c31aa7fbaba9
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat May 28 17:19:14 2022 +0200
Replace generation of docker volumes to be done from python (#23985)
The pre-commit to generate docker volumes in docker compose
file is now written in Python and it also uses the newer "volume:"
syntax to define the volumes mounted in the docker-compose.
---
.pre-commit-config.yaml | 7 +-
.../src/airflow_breeze/commands/main_command.py | 4 +-
dev/breeze/src/airflow_breeze/utils/path_utils.py | 13 ++-
scripts/ci/docker-compose/local.yml | 128 +++++++++++++++------
scripts/ci/pre_commit/pre_commit_flake8.py | 1 -
.../ci/pre_commit/pre_commit_local_yml_mounts.py | 56 +++++++++
.../ci/pre_commit/pre_commit_local_yml_mounts.sh | 43 -------
scripts/ci/pre_commit/pre_commit_mypy.py | 1 -
8 files changed, 167 insertions(+), 86 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 13a51816fd..287d7d1c09 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -350,10 +350,11 @@ repos:
additional_dependencies: ['rich>=12.4.1', 'rich-click']
- id: update-local-yml-file
name: Update mounts in the local yml file
- entry: ./scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
- language: system
- files:
^scripts/ci/libraries/_local_mounts\.sh$|^scripts/ci/docker_compose/local\.yml$
+ entry: ./scripts/ci/pre_commit/pre_commit_local_yml_mounts.py
+ language: python
+ files:
^dev/breeze/src/airflow_breeze/utils/docker_command_utils\.py$|^scripts/ci/docker_compose/local\.yml$
pass_filenames: false
+ additional_dependencies: ['rich>=12.4.1']
- id: update-setup-cfg-file
name: Update setup.cfg file with all licenses
entry: ./scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh
diff --git a/dev/breeze/src/airflow_breeze/commands/main_command.py
b/dev/breeze/src/airflow_breeze/commands/main_command.py
index 4d805e7b7a..8ed451b0e5 100644
--- a/dev/breeze/src/airflow_breeze/commands/main_command.py
+++ b/dev/breeze/src/airflow_breeze/commands/main_command.py
@@ -37,7 +37,7 @@ from airflow_breeze.utils.common_options import (
option_use_packages_from_dist,
option_verbose,
)
-from airflow_breeze.utils.path_utils import create_directories
+from airflow_breeze.utils.path_utils import create_directories_and_files
@click.group(invoke_without_command=True,
context_settings={'help_option_names': ['-h', '--help']})
@@ -64,6 +64,6 @@ from airflow_breeze.utils.path_utils import create_directories
def main(ctx: click.Context, **kwargs):
from airflow_breeze.commands.developer_commands import shell
- create_directories()
+ create_directories_and_files()
if not ctx.invoked_subcommand:
ctx.forward(shell, extra_args={})
diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py
b/dev/breeze/src/airflow_breeze/utils/path_utils.py
index de2065b678..49d9d3bf9c 100644
--- a/dev/breeze/src/airflow_breeze/utils/path_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py
@@ -142,7 +142,11 @@ def print_warning_if_setup_changed() -> bool:
Prints warning if detected airflow sources are not the ones that Breeze
was installed with.
:return: True if warning was printed.
"""
- package_hash = get_package_setup_metadata_hash()
+ try:
+ package_hash = get_package_setup_metadata_hash()
+ except ModuleNotFoundError as e:
+ if "importlib_metadata" in e.msg:
+ return False
sources_hash = get_installation_sources_config_metadata_hash()
if sources_hash != package_hash:
installation_sources = get_installation_airflow_sources()
@@ -251,9 +255,9 @@ OUTPUT_LOG = Path(CACHE_TMP_FILE_DIR.name, 'out.log')
BREEZE_SOURCES_ROOT = AIRFLOW_SOURCES_ROOT / "dev" / "breeze"
-def create_directories() -> None:
+def create_directories_and_files() -> None:
"""
- Creates all directories that are needed for Breeze to work.
+ Creates all directories and files that are needed for Breeze to work via
docker-compose.
Checks if setup has been updates since last time and proposes to upgrade
if so.
"""
BUILD_CACHE_DIR.mkdir(parents=True, exist_ok=True)
@@ -263,3 +267,6 @@ def create_directories() -> None:
LOGS_DIR.mkdir(parents=True, exist_ok=True)
DIST_DIR.mkdir(parents=True, exist_ok=True)
OUTPUT_LOG.mkdir(parents=True, exist_ok=True)
+ (AIRFLOW_SOURCES_ROOT / ".bash_aliases").touch()
+ (AIRFLOW_SOURCES_ROOT / ".bash_history").touch()
+ (AIRFLOW_SOURCES_ROOT / ".inputrc").touch()
diff --git a/scripts/ci/docker-compose/local.yml
b/scripts/ci/docker-compose/local.yml
index 0e37d0c34a..e356cb8736 100644
--- a/scripts/ci/docker-compose/local.yml
+++ b/scripts/ci/docker-compose/local.yml
@@ -26,38 +26,100 @@ services:
# or those that might be useful to see in the host as output of the
# tests (such as logs)
volumes:
- - mypy-cache-volume:/opt/airflow/.mypy_cache/
- # START automatically generated volumes from LOCAL_MOUNTS in
_local_mounts.sh
- - ../../../.bash_aliases:/root/.bash_aliases:cached
- - ../../../.bash_history:/root/.bash_history:cached
- - ../../../.coveragerc:/opt/airflow/.coveragerc:cached
- - ../../../.dockerignore:/opt/airflow/.dockerignore:cached
- - ../../../.flake8:/opt/airflow/.flake8:cached
- - ../../../.github:/opt/airflow/.github:cached
- - ../../../.inputrc:/root/.inputrc:cached
- - ../../../.rat-excludes:/opt/airflow/.rat-excludes:cached
- - ../../../RELEASE_NOTES.rst:/opt/airflow/RELEASE_NOTES.rst:cached
- - ../../../LICENSE:/opt/airflow/LICENSE:cached
- - ../../../MANIFEST.in:/opt/airflow/MANIFEST.in:cached
- - ../../../NOTICE:/opt/airflow/NOTICE:cached
- - ../../../airflow:/opt/airflow/airflow:cached
- - ../../../provider_packages:/opt/airflow/provider_packages:cached
- - ../../../dags:/opt/airflow/dags:cached
- - ../../../dev:/opt/airflow/dev:cached
- - ../../../docs:/opt/airflow/docs:cached
- - ../../../hooks:/opt/airflow/hooks:cached
- - ../../../logs:/root/airflow/logs:cached
- - ../../../pyproject.toml:/opt/airflow/pyproject.toml:cached
- - ../../../pytest.ini:/opt/airflow/pytest.ini:cached
- - ../../../scripts:/opt/airflow/scripts:cached
- - ../../../scripts/docker/entrypoint_ci.sh:/entrypoint:cached
- - ../../../setup.cfg:/opt/airflow/setup.cfg:cached
- - ../../../setup.py:/opt/airflow/setup.py:cached
- - ../../../tests:/opt/airflow/tests:cached
- - ../../../kubernetes_tests:/opt/airflow/kubernetes_tests:cached
- - ../../../docker_tests:/opt/airflow/docker_tests:cached
- - ../../../chart:/opt/airflow/chart:cached
- - ../../../metastore_browser:/opt/airflow/metastore_browser:cached
- # END automatically generated volumes from LOCAL_MOUNTS in
_local_mounts.sh
+ - type: volume
+ source: mypy-cache-volume
+ target: /opt/airflow/.mypy_cache/
+ # START automatically generated volumes from NECESSARY_HOST_VOLUMES in
docker_command_utils.py
+ - type: bind
+ source: ../../../.bash_aliases
+ target: /root/.bash_aliases
+ - type: bind
+ source: ../../../.bash_history
+ target: /root/.bash_history
+ - type: bind
+ source: ../../../.coveragerc
+ target: /opt/airflow/.coveragerc
+ - type: bind
+ source: ../../../.dockerignore
+ target: /opt/airflow/.dockerignore
+ - type: bind
+ source: ../../../.flake8
+ target: /opt/airflow/.flake8
+ - type: bind
+ source: ../../../.github
+ target: /opt/airflow/.github
+ - type: bind
+ source: ../../../.inputrc
+ target: /root/.inputrc
+ - type: bind
+ source: ../../../.rat-excludes
+ target: /opt/airflow/.rat-excludes
+ - type: bind
+ source: ../../../RELEASE_NOTES.rst
+ target: /opt/airflow/RELEASE_NOTES.rst
+ - type: bind
+ source: ../../../LICENSE
+ target: /opt/airflow/LICENSE
+ - type: bind
+ source: ../../../MANIFEST.in
+ target: /opt/airflow/MANIFEST.in
+ - type: bind
+ source: ../../../NOTICE
+ target: /opt/airflow/NOTICE
+ - type: bind
+ source: ../../../airflow
+ target: /opt/airflow/airflow
+ - type: bind
+ source: ../../../provider_packages
+ target: /opt/airflow/provider_packages
+ - type: bind
+ source: ../../../dags
+ target: /opt/airflow/dags
+ - type: bind
+ source: ../../../dev
+ target: /opt/airflow/dev
+ - type: bind
+ source: ../../../docs
+ target: /opt/airflow/docs
+ - type: bind
+ source: ../../../hooks
+ target: /opt/airflow/hooks
+ - type: bind
+ source: ../../../logs
+ target: /root/airflow/logs
+ - type: bind
+ source: ../../../pyproject.toml
+ target: /opt/airflow/pyproject.toml
+ - type: bind
+ source: ../../../pytest.ini
+ target: /opt/airflow/pytest.ini
+ - type: bind
+ source: ../../../scripts
+ target: /opt/airflow/scripts
+ - type: bind
+ source: ../../../scripts/docker/entrypoint_ci.sh
+ target: /entrypoint
+ - type: bind
+ source: ../../../setup.cfg
+ target: /opt/airflow/setup.cfg
+ - type: bind
+ source: ../../../setup.py
+ target: /opt/airflow/setup.py
+ - type: bind
+ source: ../../../tests
+ target: /opt/airflow/tests
+ - type: bind
+ source: ../../../kubernetes_tests
+ target: /opt/airflow/kubernetes_tests
+ - type: bind
+ source: ../../../docker_tests
+ target: /opt/airflow/docker_tests
+ - type: bind
+ source: ../../../chart
+ target: /opt/airflow/chart
+ - type: bind
+ source: ../../../metastore_browser
+ target: /opt/airflow/metastore_browser
+ # END automatically generated volumes from NECESSARY_HOST_VOLUMES in
docker_command_utils.py
volumes:
mypy-cache-volume:
diff --git a/scripts/ci/pre_commit/pre_commit_flake8.py
b/scripts/ci/pre_commit/pre_commit_flake8.py
index 3d4a56060f..7adba2ffc6 100755
--- a/scripts/ci/pre_commit/pre_commit_flake8.py
+++ b/scripts/ci/pre_commit/pre_commit_flake8.py
@@ -32,7 +32,6 @@ AIRFLOW_SOURCES = Path(__file__).parents[3].resolve()
GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow")
if __name__ == '__main__':
- os.environ['SKIP_BREEZE_UPGRADE_CHECK'] = "true"
sys.path.insert(0, str(Path(__file__).parents[3].resolve() / "dev" /
"breeze" / "src"))
from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
from airflow_breeze.global_constants import MOUNT_SELECTED
diff --git a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.py
b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.py
new file mode 100755
index 0000000000..391214fa29
--- /dev/null
+++ b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import sys
+from pathlib import Path
+
+AIRFLOW_SOURCES_DIR = Path(__file__).parents[3].resolve()
+
+sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure
common_precommit_utils is imported
+sys.path.insert(0, str(AIRFLOW_SOURCES_DIR)) # make sure setup is imported
from Airflow
+sys.path.insert(
+ 0, str(AIRFLOW_SOURCES_DIR / "dev" / "breeze" / "src")
+) # make sure setup is imported from Airflow
+# flake8: noqa: F401
+
+from common_precommit_utils import insert_documentation # isort: skip
+
+sys.path.append(str(AIRFLOW_SOURCES_DIR))
+
+MOUNTS_HEADER = (
+ ' # START automatically generated volumes from '
+ 'NECESSARY_HOST_VOLUMES in docker_command_utils.py'
+)
+MOUNTS_FOOTER = (
+ ' # END automatically generated volumes from '
'NECESSARY_HOST_VOLUMES in docker_command_utils.py'
+)
+
+if __name__ == '__main__':
+ from airflow_breeze.utils.docker_command_utils import
NECESSARY_HOST_VOLUMES
+
+ local_mount_file_path = AIRFLOW_SOURCES_DIR / 'scripts' / 'ci' /
'docker-compose' / 'local.yml'
+ PREFIX = ' '
+ volumes = []
+ for (src, dest) in NECESSARY_HOST_VOLUMES:
+ volumes.extend(
+ [
+ PREFIX + "- type: bind\n",
+ PREFIX + f" source: ../../../{src}\n",
+ PREFIX + f" target: {dest}\n",
+ ]
+ )
+ insert_documentation(local_mount_file_path, volumes, MOUNTS_HEADER,
MOUNTS_FOOTER)
diff --git a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
deleted file mode 100755
index dbf82b7241..0000000000
--- a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-export PRINT_INFO_FROM_SCRIPTS="false"
-export SKIP_CHECK_REMOTE_IMAGE="true"
-
-# shellcheck source=scripts/ci/libraries/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-
-TMP_OUTPUT=$(mktemp)
-readonly TMP_OUTPUT
-
-# Remove temp file if it's hanging around
-traps::add_trap "rm -rf -- '${TMP_OUTPUT}' 2>/dev/null" EXIT HUP INT TERM
-
-LOCAL_YML_FILE="${AIRFLOW_SOURCES}/scripts/ci/docker-compose/local.yml"
-readonly LOCAL_YML_FILE
-
-lead_marker=' # START automatically generated volumes from LOCAL_MOUNTS
in _local_mounts.sh'
-tail_marker=' # END automatically generated volumes from LOCAL_MOUNTS in
_local_mounts.sh'
-
-local_mounts::generate_local_mounts_list " - ../../../"
-
-sed "/$lead_marker/q" "${LOCAL_YML_FILE}" > "${TMP_OUTPUT}"
-
-printf '%s\n' "${LOCAL_MOUNTS[@]}" >> "${TMP_OUTPUT}"
-sed -ne "/$tail_marker/,\$ p" "${LOCAL_YML_FILE}" >> "${TMP_OUTPUT}"
-
-mv "${TMP_OUTPUT}" "${LOCAL_YML_FILE}"
diff --git a/scripts/ci/pre_commit/pre_commit_mypy.py
b/scripts/ci/pre_commit/pre_commit_mypy.py
index 0075856325..9d92b99e80 100755
--- a/scripts/ci/pre_commit/pre_commit_mypy.py
+++ b/scripts/ci/pre_commit/pre_commit_mypy.py
@@ -33,7 +33,6 @@ AIRFLOW_SOURCES = Path(__file__).parents[3].resolve()
GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow")
if __name__ == '__main__':
- os.environ['SKIP_BREEZE_UPGRADE_CHECK'] = "true"
sys.path.insert(0, str(Path(__file__).parents[3].resolve() / "dev" /
"breeze" / "src"))
from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
from airflow_breeze.global_constants import MOUNT_SELECTED
