This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-2-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e3209f34c1c4701ab76720a36dca7d8fdf3061c2 Author: Kamil BreguĊa <[email protected]> AuthorDate: Sun Jan 23 14:15:45 2022 +0100 Add image labels required by ArtifactHub (#21040) (cherry picked from commit 7f02b4718bc9e282c1d59b0aab5dd46972b6f79c) --- Dockerfile | 6 +- docs/docker-stack/README.md | 75 ++++++++++++++++++++++ docs/docker-stack/index.rst | 5 +- scripts/ci/libraries/_build_images.sh | 1 + .../ci/pre_commit/pre_commit_update_versions.py | 7 +- 5 files changed, 91 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0edb037..86e6a0d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,6 +48,7 @@ ARG PYTHON_BASE_IMAGE="python:3.7-slim-buster" ARG AIRFLOW_PIP_VERSION=21.3.1 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"; +ARG AIRFLOW_IMAGE_README_URL="https://raw.githubusercontent.com/apache/airflow/main/docs/docker-stack/README.md"; # By default latest released version of airflow is installed (when empty) but this value can be overridden # and we can install version according to specification (For example ==2.0.2 or <3.0.0). @@ -396,6 +397,7 @@ ARG AIRFLOW_HOME # production image is prepared from sources rather than from package ARG AIRFLOW_INSTALLATION_METHOD="apache-airflow" ARG AIRFLOW_IMAGE_REPOSITORY +ARG AIRFLOW_IMAGE_README_URL ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \ ADDITIONAL_RUNTIME_APT_DEPS=${ADDITIONAL_RUNTIME_APT_DEPS} \ @@ -524,7 +526,9 @@ LABEL org.apache.airflow.distro="debian" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.ref.name="airflow" \ org.opencontainers.image.title="Production Airflow Image" \ - org.opencontainers.image.description="Reference, production-ready Apache Airflow image" + org.opencontainers.image.description="Reference, production-ready Apache Airflow image" \ + io.artifacthub.package.license='Apache-2.0' \ + io.artifacthub.package.readme-url='${AIRFLOW_IMAGE_README_URL}' ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"] CMD [] diff --git a/docs/docker-stack/README.md b/docs/docker-stack/README.md new file mode 100644 index 0000000..96010fe --- /dev/null +++ b/docs/docker-stack/README.md @@ -0,0 +1,75 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> +<!-- +This file was created for the purpose of publishing an image to ArtifactHub. +Please try to keep it in sync with index.rst +--> + +# Docker Image for Apache Airflow + +For the ease of deployment in production, the community releases a production-ready reference container +image. + +The Apache Airflow community, releases Docker Images which are `reference images` for Apache Airflow. +Every time a new version of Airflow is released, the images are prepared in the +[apache/airflow DockerHub](https://hub.docker.com/r/apache/airflow) +for all the supported Python versions. + +You can find the following images there (Assuming Airflow version `2.3.0.dev0`): + +* `apache/airflow:latest` - the latest released Airflow image with default Python version (3.7 currently) +* `apache/airflow:latest-pythonX.Y` - the latest released Airflow image with specific Python version +* `apache/airflow:2.3.0.dev0` - the versioned Airflow image with default Python version (3.7 currently) +* `apache/airflow:2.3.0.dev0-pythonX.Y` - the versioned Airflow image with specific Python version + +Those are "reference" images. They contain the most common set of extras, dependencies and providers that are +often used by the users and they are good to "try-things-out" when you want to just take Airflow for a spin, + +The Apache Airflow image provided as convenience package is optimized for size, and +it provides just a bare minimal set of the extras and dependencies installed and in most cases +you want to either extend or customize the image. You can see all possible extras in [Reference for package extras](https://airflow.apache.org/docs/apache-airflow/stable/extra-packages-ref.html). +The set of extras used in Airflow Production image are available in the +[Dockerfile](https://github.com/apache/airflow/blob/2c6c7fdb2308de98e142618836bdf414df9768c8/Dockerfile#L37). + +However, Airflow has more than 60 community-managed providers (installable via extras) and some of the +default extras/providers installed are not used by everyone, sometimes others extras/providers +are needed, sometimes (very often actually) you need to add your own custom dependencies, +packages or even custom providers. You can learn how to do it in [Building the image](https://airflow.apache.org/docs/docker-stack/build.html#build-build-image). + +The production images are build in DockerHub from released version and release candidates. There +are also images published from branches but they are used mainly for development and testing purpose. +See [Airflow Git Branching](https://github.com/apache/airflow/blob/main/CONTRIBUTING.rst#airflow-git-branches) +for details. + +## Usage + +The [`AIRFLOW_HOME`](https://airflow.apache.org/docs/apache-airflow/stable/cli-and-env-variables-ref.html#envvar-AIRFLOW_HOME) is set by default to ``/opt/airflow/`` - this means that DAGs +are in default in the ``/opt/airflow/dags`` folder and logs are in the ``/opt/airflow/logs`` + +The working directory is ``/opt/airflow`` by default. + +If no `AIRFLOW__CORE__SQL_ALCHEMY_CONN` variable is set then SQLite database is created in +``${AIRFLOW_HOME}/airflow.db``. + +For example commands that start Airflow see: [Executing commands](https://airflow.apache.org/docs/docker-stack/entrypoint.html#entrypoint-commands). + +Airflow requires many components to function as it is a distributed application. You may therefore also be interested +in launching Airflow in the Docker Compose environment, see: [Quick Start](https://airflow.apache.org/docs/apache-airflow/stable/start/index.html). + +You can use this image in [Helm Chart](https://airflow.apache.org/docs/helm-chart/stable/index.html) as well. diff --git a/docs/docker-stack/index.rst b/docs/docker-stack/index.rst index 3701844..411af82 100644 --- a/docs/docker-stack/index.rst +++ b/docs/docker-stack/index.rst @@ -15,6 +15,9 @@ specific language governing permissions and limitations under the License. + .. WARNING: + IF YOU ARE UPDATING THIS FILE, CONSIDER UPDATING README.MD TOO. + .. image:: /img/docker-logo.png :width: 100 @@ -44,7 +47,7 @@ Every time a new version of Airflow is released, the images are prepared in the `apache/airflow DockerHub <https://hub.docker.com/r/apache/airflow>`_ for all the supported Python versions. -You can find the following images there (Assuming Airflow version |airflow-version|): +You can find the following images there (Assuming Airflow version :subst-code:`|airflow-version|`): * :subst-code:`apache/airflow:latest` - the latest released Airflow image with default Python version (3.7 currently) * :subst-code:`apache/airflow:latest-pythonX.Y` - the latest released Airflow image with specific Python version diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index 2d84a92..c052261 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -673,6 +673,7 @@ function build_images::build_prod_images() { --build-arg AIRFLOW_CONSTRAINTS="${AIRFLOW_CONSTRAINTS}" \ --build-arg AIRFLOW_IMAGE_REPOSITORY="https://github.com/${GITHUB_REPOSITORY}"; \ --build-arg AIRFLOW_IMAGE_DATE_CREATED="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ + --build-arg AIRFLOW_IMAGE_README_URL="https://raw.githubusercontent.com/apache/airflow/${COMMIT_SHA}/docs/docker-stack/README.md"; \ "${additional_dev_args[@]}" \ "${additional_runtime_args[@]}" \ "${docker_cache_prod_directive[@]}" \ diff --git a/scripts/ci/pre_commit/pre_commit_update_versions.py b/scripts/ci/pre_commit/pre_commit_update_versions.py index 2af0698..3898d64 100755 --- a/scripts/ci/pre_commit/pre_commit_update_versions.py +++ b/scripts/ci/pre_commit/pre_commit_update_versions.py @@ -30,7 +30,7 @@ from setup import version # isort:skip def update_version(pattern: re.Pattern, v: str, file_path: str): - print(f"Replacing {pattern} to {version} in {file_path}") + print(f"Checking {pattern} in {file_path}") with open(file_path, "r+") as f: file_content = f.read() if not pattern.search(file_content): @@ -38,6 +38,7 @@ def update_version(pattern: re.Pattern, v: str, file_path: str): new_content = pattern.sub(fr'\g<1>{v}\g<2>', file_content) if file_content == new_content: return + print(" Updated.") f.seek(0) f.truncate() f.write(new_content) @@ -46,8 +47,12 @@ def update_version(pattern: re.Pattern, v: str, file_path: str): REPLACEMENTS = { r'^(FROM apache\/airflow:).*($)': "docs/docker-stack/docker-examples/extending/*/Dockerfile", r'(apache\/airflow:)[^-]*(\-)': "docs/docker-stack/entrypoint.rst", + r'(`apache/airflow:)[0-9].*?((?:-pythonX.Y)?`)': "docs/docker-stack/README.md", + r'(\(Assuming Airflow version `).*(`\))': "docs/docker-stack/README.md", } +print(f"Current version: {version}") + if __name__ == '__main__': for regexp, p in REPLACEMENTS.items(): text_pattern = re.compile(regexp, flags=re.MULTILINE)
