potiuk commented on a change in pull request #21956: URL: https://github.com/apache/airflow/pull/21956#discussion_r826958095
########## File path: dev/breeze/src/airflow_breeze/prod/build_prod_image.py ########## @@ -0,0 +1,153 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import Dict, List + +from airflow_breeze.cache import check_cache_and_write_if_not_cached, write_to_cache_file +from airflow_breeze.console import console +from airflow_breeze.prod.prod_params import ProdParams +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCE +from airflow_breeze.utils.run_utils import filter_out_none, run_command + +PARAMS_PROD_IMAGE = [ + "python_base_image", + "install_mysql_client", + "install_mssql_client", + "install_postgres_client", + "airflow_version", + "airflow_branch", + "airflow_extras", + "airflow_pre_cached_pip_packages", + "additional_airflow_extras", + "additional_python_deps", + "additional_dev_apt_command", + "additional_dev_apt_deps", + "additional_dev_apt_env", + "additional_runtime_apt_command", + "additional_runtime_apt_deps", + "additional_runtime_apt_env", + "upgrade_to_newer_dependencies", + "constraints_github_repository", + "airflow_constraints", + "airflow_image_repository", + "airflow_image_date_created", + "build_id", + "commit_sha", + "airflow_image_readme_url", + "install_providers_from_sources", + "install_from_pypi", + "install_from_docker_context_files", +] + +PARAMS_TO_VERIFY_PROD_IMAGE = [ + "dev_apt_command", + "dev_apt_deps", + "runtime_apt_command", + "runtime_apt_deps", +] + + +def construct_arguments_docker_command(prod_image: ProdParams) -> List[str]: + args_command = [] + for param in PARAMS_PROD_IMAGE: + args_command.append("--build-arg") + args_command.append(param.upper() + "=" + str(getattr(prod_image, param))) + for verify_param in PARAMS_TO_VERIFY_PROD_IMAGE: + param_value = str(getattr(prod_image, verify_param)) + if len(param_value) > 0: + args_command.append("--build-arg") + args_command.append(verify_param.upper() + "=" + param_value) + docker_cache = prod_image.docker_cache_prod_directive + if len(docker_cache) > 0: + args_command.extend(prod_image.docker_cache_prod_directive) + return args_command + + +def construct_docker_command(prod_image: ProdParams) -> List[str]: + arguments = construct_arguments_docker_command(prod_image) + build_command = prod_image.check_buildx_plugin_build_command() + build_flags = prod_image.extra_docker_build_flags + final_command = [] + final_command.extend(["docker"]) + final_command.extend(build_command) + final_command.extend(build_flags) + final_command.extend(["--pull"]) + final_command.extend(arguments) + final_command.extend(["-t", prod_image.airflow_prod_image_name, "--target", "main", "."]) + final_command.extend(["-f", 'Dockerfile']) + final_command.extend(["--platform", prod_image.platform]) + return final_command + + +def login_to_docker_registry(build_params: ProdParams): + if build_params.ci == "true": + if len(build_params.github_token) == 0: + console.print("\n[blue]Skip logging in to GitHub Registry. No Token available!") + elif build_params.airflow_login_to_github_registry != "true": + console.print( + "\n[blue]Skip logging in to GitHub Registry.\ + AIRFLOW_LOGIN_TO_GITHUB_REGISTRY is set as false" + ) + elif len(build_params.github_token) > 0: + run_command(['docker', 'logout', 'ghcr.io'], verbose=True, text=True) + # echo "${token}" | docker_v login \ + # --username "${GITHUB_USERNAME:-apache}" \ + # --password-stdin \ + # "ghcr.io" || true + run_command( Review comment: We should pass the token here to login from `GITHUB_TOKEN` variable (which might be actually added as a new `--github-token` option with GITHUB_TOKEN envvar). This can be done by adding a new feature to run_command to pass. Apparently this can be easily done with the `input` parameter: https://stackoverflow.com/questions/163542/how-do-i-pass-a-string-into-subprocess-popen-using-the-stdin-argument ########## File path: dev/breeze/src/airflow_breeze/prod/build_prod_image.py ########## @@ -0,0 +1,154 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import Dict, List + +from airflow_breeze.cache import check_cache_and_write_if_not_cached, write_to_cache_file +from airflow_breeze.console import console +from airflow_breeze.prod.prod_params import ProdParams +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCE +from airflow_breeze.utils.run_utils import filter_out_none, run_command + +PARAMS_PROD_IMAGE = [ + "python_base_image", + "install_mysql_client", + "install_mssql_client", + "install_postgres_client", + "airflow_version", + "airflow_branch", + "airflow_extras", + "airflow_pre_cached_pip_packages", + "additional_airflow_extras", + "additional_python_deps", + "additional_dev_apt_command", + "additional_dev_apt_deps", + "additional_dev_apt_env", + "additional_runtime_apt_command", + "additional_runtime_apt_deps", + "additional_runtime_apt_env", + "upgrade_to_newer_dependencies", + "constraints_github_repository", + "airflow_constraints", + "airflow_image_repository", + "airflow_image_date_created", + "build_id", + "commit_sha", + "airflow_image_readme_url", + "install_providers_from_sources", + "install_from_pypi", + "install_from_docker_context_files", +] + +PARAMS_TO_VERIFY_PROD_IMAGE = [ + "dev_apt_command", + "dev_apt_deps", + "runtime_apt_command", + "runtime_apt_deps", +] + + +def construct_arguments_docker_command(prod_image: ProdParams) -> List[str]: + args_command = [] + for param in PARAMS_PROD_IMAGE: + args_command.append("--build-arg") + args_command.append(param.upper() + "=" + str(getattr(prod_image, param))) + for verify_param in PARAMS_TO_VERIFY_PROD_IMAGE: + param_value = str(getattr(prod_image, verify_param)) + if len(param_value) > 0: + args_command.append("--build-arg") + args_command.append(verify_param.upper() + "=" + param_value) + docker_cache = prod_image.docker_cache_prod_directive + if len(docker_cache) > 0: + args_command.extend(prod_image.docker_cache_prod_directive) + return args_command + + +def construct_docker_command(prod_image: ProdParams) -> List[str]: + arguments = construct_arguments_docker_command(prod_image) + build_command = prod_image.check_buildx_plugin_build_command() + build_flags = prod_image.extra_docker_build_flags + final_command = [] + final_command.extend(["docker"]) + final_command.extend(build_command) + final_command.extend(build_flags) + final_command.extend(["--pull"]) + final_command.extend(arguments) + final_command.extend(["-t", prod_image.airflow_prod_image_name, "--target", "main", "."]) + final_command.extend(["-f", 'Dockerfile']) + final_command.extend(["--platform", prod_image.platform]) + return final_command + + +def login_to_docker_registry(build_params: ProdParams): + if build_params.ci == "true": + if len(build_params.github_token) == 0: + console.print("\n[blue]Skip logging in to GitHub Registry. No Token available!") + elif build_params.airflow_login_to_github_registry != "true": + console.print( + "\n[blue]Skip logging in to GitHub Registry.\ + AIRFLOW_LOGIN_TO_GITHUB_REGISTRY is set as false" + ) + elif len(build_params.github_token) > 0: + run_command(['docker', 'logout', 'ghcr.io'], verbose=True, text=True) + # echo "${token}" | docker_v login \ + # --username "${GITHUB_USERNAME:-apache}" \ + # --password-stdin \ + # "ghcr.io" || true + run_command( + ['docker', 'login', '--username', '', '--password-stdin', 'ghcr.io'], verbose=True, text=True + ) + else: + console.print('\n[blue]Skip Login to GitHub Container Registry as token is missing') + + +def check_docker_context_files(install_from_docker_context_files: bool): + pass + + +def build_production_image(verbose, **kwargs): + parameters_passed = filter_out_none(**kwargs) + prod_params = get_image_build_params(parameters_passed) + prod_params.print_info() + if prod_params.skip_building_prod_image: + console.print('[bright_yellow]\nSkip building production image. Assume the one we have is good!') + console.print('bright_yellow]\nYou must run Breeze2 build-prod-image before for all python versions!') + if prod_params.prepare_buildx_cache: + login_to_docker_registry(prod_params) + + cmd = construct_docker_command(prod_params) + print(cmd) + output = run_command(cmd, verbose=verbose, cwd=AIRFLOW_SOURCE, text=True) + console.print(f"[blue]{output}") + # if prod_params.prepare_buildx_cache: Review comment: we can uncomment this one I think, once the login to registry is sorted out. ########## File path: dev/breeze/src/airflow_breeze/breeze.py ########## @@ -312,11 +314,156 @@ def build_ci_image( @option_verbose @main.command(name='build-prod-image') -def build_prod_image(verbose: bool): [email protected]( Review comment: I think it woudl be great to add ENVVARS for all the parameters here already similarly to what @edithturn added in CI build. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
