jedcunningham commented on code in PR #36930:
URL: https://github.com/apache/airflow/pull/36930#discussion_r1462352420
##########
dev/breeze/src/airflow_breeze/commands/release_management_commands.py:
##########
@@ -2394,3 +2395,270 @@ def prepare_python_client(
finally:
if version_suffix_for_pypi:
VERSION_FILE.write_text(original_version)
+
+
+def _get_latest_airflow_version_from_pypi():
+ import requests
+
+ response = requests.get("https://pypi.org/pypi/apache-airflow/json";)
+ response.raise_for_status()
+ return response.json()["info"]["version"]
+
+
+CHART_DIR = AIRFLOW_SOURCES_ROOT / "chart"
+CHART_YAML_FILE = CHART_DIR / "Chart.yaml"
+
+
+@release_management.command(name="prepare-helm-chart-tarball", help="Prepares
helm chart tarball.")
[email protected](
+ "--version",
+ help="Version used for helm chart. This version has to be set and has to
match the version in "
+ "Chart.yaml, unless the --ignore-version-check flag is used.",
+ envvar="VERSION",
+)
[email protected](
+ "--version-suffix",
+ help="Version suffix used to publish the package. Needs to be present as
we always build "
+ "archive using release candidate tag.",
+ required=True,
+ envvar="VERSION_SUFFIX",
+)
[email protected](
+ "--ignore-version-check",
+ is_flag=True,
+ help="Ignores result of version update check. Produce tarball regardless
of "
+ "whether version is correctly set in the Chart.yaml.",
+)
[email protected](
+ "--skip-tagging",
+ is_flag=True,
+ help="Skip tagging the chart. Useful if the tag is already created, or
when you verify the chart.",
+)
[email protected](
+ "--skip-tag-signing",
+ is_flag=True,
+ help="Skip signing the tag. Useful for CI where we just tag without
signing the tag.",
+)
[email protected](
+ "--override-tag",
+ is_flag=True,
+ help="Override tag if it already exists. Useful when you want to re-create
the tag, usually when you"
+ "test the breeze command locally.",
+)
+@option_dry_run
+@option_verbose
+def prepare_helm_chart_tarball(
+ version: str | None,
+ version_suffix: str,
+ ignore_version_check: bool,
+ override_tag: bool,
+ skip_tagging: bool,
+ skip_tag_signing: bool,
+) -> None:
+ import yaml
+
+ chart_yaml_file_content = CHART_YAML_FILE.read_text()
+ chart_yaml_dict = yaml.safe_load(chart_yaml_file_content)
+ version_in_chart = chart_yaml_dict["version"]
+ airflow_version_in_chart = chart_yaml_dict["appVersion"]
+ airflow_version_from_pypi = _get_latest_airflow_version_from_pypi()
+ if ignore_version_check:
+ if not version:
+ version = version_in_chart
+ else:
+ if not version or not version_suffix:
+ get_console().print(
+ "[error]You need to provide --version and --version-suffix
parameter unless you "
+ "use --ignore-version-check[/]"
+ )
+ sys.exit(1)
+ get_console().print(f"[info]Latest Airflow version from PyPI:
{airflow_version_from_pypi}[/]")
+ get_console().print(f"[info]Airflow version in chart:
{airflow_version_in_chart}[/]")
+ updating = False
+ if version_in_chart != version:
+ get_console().print(
+ f"[warning]Version in chart.yaml ({version_in_chart}) does not
match the version "
+ f"passed as parameter ({version}). Updating[/]"
+ )
+ updating = True
+ chart_yaml_file_content = chart_yaml_file_content.replace(
+ f"version: {version_in_chart}", f"version: {version}"
+ )
+ else:
+ get_console().print(f"[success]Version in chart.yaml is good:
{version}[/]")
+ if airflow_version_from_pypi != airflow_version_in_chart:
+ get_console().print(
+ f"[warning]Airflow version in chart.yaml
({airflow_version_in_chart}) does not match the "
Review Comment:
Might be a bit nit picky, but we really need to check that the appVersion
matches the default tag in values, not the latest in pypi (or check both, but
just updating it to the latest pypi version is wrong).
##########
dev/README_RELEASE_HELM_CHART.md:
##########
@@ -103,101 +161,55 @@ the same between voted release candidate and final
release.
Because of this the version in the built artifacts that will become the
official Apache releases must not include the rcN suffix.
-- Set environment variables
-
- ```shell
- # Set Version
- export VERSION=1.0.1rc1
- export VERSION_WITHOUT_RC=${VERSION%rc?}
-
- # Set AIRFLOW_REPO_ROOT to the path of your git repo
- export AIRFLOW_REPO_ROOT=$(pwd)
+Make sure you have `apache` remote set up pointing to the apache git repo.
+If needed, add it with:
- # Example after cloning
- git clone https://github.com/apache/airflow.git airflow
- cd airflow
- export AIRFLOW_REPO_ROOT=$(pwd)
- ```
+```shell
+git remote add apache [email protected]:apache/airflow.git
+git fetch apache
+```
- We currently release Helm Chart from `main` branch:
- ```shell
- git checkout main
- ```
-
-- Clean the checkout: the sdist step below will
-
- ```shell
- git clean -fxd
- ```
-
-- Update Helm Chart version in `Chart.yaml`, example: `version: 1.0.0` (without
- the RC tag). If the default version of Airflow is different from
`appVersion` change it.
-
-- Add and commit the version change.
-
- ```shell
- git add chart
- git commit -m "Chart: Bump version to $VERSION_WITHOUT_RC"
- ```
-
- Note: You will tag this commit, you do not need to open a PR for it.
-
-- Tag your release
-
- ```shell
- git tag -s helm-chart/${VERSION} -m "Apache Airflow Helm Chart $VERSION"
- ```
-
-- Tarball the repo
-
- NOTE: Make sure your checkout is clean at this stage - any untracked or
changed files will otherwise be included
- in the file produced.
-
- ```shell
- git archive --format=tar.gz helm-chart/${VERSION}
--prefix=airflow-chart-${VERSION_WITHOUT_RC}/ \
- -o airflow-chart-${VERSION_WITHOUT_RC}-source.tar.gz chart
.rat-excludes
- ```
+```shell
+git checkout apache/main
+```
-- Generate chart binary
+- Clean the checkout: (note that this step will also clean any IDE settings
you might have so better to
+ do it in a checked out version you only use for releasing)
+```shell
+git clean -fxd
+rm -rf dist/*
+```
- ```shell
- helm package chart --dependency-update
- ```
+- Generate the source tarball:
-- Sign the chart binary
+```shell
+breeze release-management prepare-helm-chart-tarball --version ${VERSION}
--version-suffix ${VERSION_SUFFIX}
+```
- In the following command, replace the email address with your email
address or your KEY ID
- so GPG uses the right key to sign the chart.
- (If you have not generated a key yet, generate it by following
instructions on
- http://www.apache.org/dev/openpgp.html#key-gen-generate-key)
+- Generate the binary Helm Chart release:
- ```shell
- helm gpg sign -u [email protected] airflow-${VERSION_WITHOUT_RC}.tgz
- ```
+```shell
+breeze release-management prepare-helm-chart-packagte --sig
[email protected]
Review Comment:
```suggestion
breeze release-management prepare-helm-chart-package --sig
[email protected]
```
##########
dev/README_RELEASE_HELM_CHART.md:
##########
@@ -95,6 +143,16 @@ annotations:
url: https://github.com/apache/airflow/pull/19263
```
+Make sure that all the release notes changes are submitted as PR and merged.
Changes in release notes should
+also automatically (via `pre-commit` trigger updating of the
[reproducible_build.yaml](../chart/reproducible_build.yaml))
+file which is uses to reproducibly build the chart package and source tarball.
Review Comment:
```suggestion
file which is used to reproducibly build the chart package and source
tarball.
```
##########
dev/breeze/src/airflow_breeze/commands/release_management_commands.py:
##########
@@ -2394,3 +2395,270 @@ def prepare_python_client(
finally:
if version_suffix_for_pypi:
VERSION_FILE.write_text(original_version)
+
+
+def _get_latest_airflow_version_from_pypi():
+ import requests
+
+ response = requests.get("https://pypi.org/pypi/apache-airflow/json";)
+ response.raise_for_status()
+ return response.json()["info"]["version"]
+
+
+CHART_DIR = AIRFLOW_SOURCES_ROOT / "chart"
+CHART_YAML_FILE = CHART_DIR / "Chart.yaml"
+
+
+@release_management.command(name="prepare-helm-chart-tarball", help="Prepares
helm chart tarball.")
[email protected](
+ "--version",
+ help="Version used for helm chart. This version has to be set and has to
match the version in "
+ "Chart.yaml, unless the --ignore-version-check flag is used.",
+ envvar="VERSION",
+)
[email protected](
+ "--version-suffix",
+ help="Version suffix used to publish the package. Needs to be present as
we always build "
+ "archive using release candidate tag.",
+ required=True,
+ envvar="VERSION_SUFFIX",
+)
[email protected](
+ "--ignore-version-check",
+ is_flag=True,
+ help="Ignores result of version update check. Produce tarball regardless
of "
+ "whether version is correctly set in the Chart.yaml.",
+)
[email protected](
+ "--skip-tagging",
+ is_flag=True,
+ help="Skip tagging the chart. Useful if the tag is already created, or
when you verify the chart.",
+)
[email protected](
+ "--skip-tag-signing",
+ is_flag=True,
+ help="Skip signing the tag. Useful for CI where we just tag without
signing the tag.",
+)
[email protected](
+ "--override-tag",
+ is_flag=True,
+ help="Override tag if it already exists. Useful when you want to re-create
the tag, usually when you"
+ "test the breeze command locally.",
+)
+@option_dry_run
+@option_verbose
+def prepare_helm_chart_tarball(
+ version: str | None,
+ version_suffix: str,
+ ignore_version_check: bool,
+ override_tag: bool,
+ skip_tagging: bool,
+ skip_tag_signing: bool,
+) -> None:
+ import yaml
+
+ chart_yaml_file_content = CHART_YAML_FILE.read_text()
+ chart_yaml_dict = yaml.safe_load(chart_yaml_file_content)
+ version_in_chart = chart_yaml_dict["version"]
+ airflow_version_in_chart = chart_yaml_dict["appVersion"]
+ airflow_version_from_pypi = _get_latest_airflow_version_from_pypi()
+ if ignore_version_check:
+ if not version:
+ version = version_in_chart
+ else:
+ if not version or not version_suffix:
+ get_console().print(
+ "[error]You need to provide --version and --version-suffix
parameter unless you "
+ "use --ignore-version-check[/]"
+ )
+ sys.exit(1)
+ get_console().print(f"[info]Latest Airflow version from PyPI:
{airflow_version_from_pypi}[/]")
+ get_console().print(f"[info]Airflow version in chart:
{airflow_version_in_chart}[/]")
+ updating = False
+ if version_in_chart != version:
+ get_console().print(
+ f"[warning]Version in chart.yaml ({version_in_chart}) does not
match the version "
+ f"passed as parameter ({version}). Updating[/]"
+ )
+ updating = True
+ chart_yaml_file_content = chart_yaml_file_content.replace(
+ f"version: {version_in_chart}", f"version: {version}"
+ )
+ else:
+ get_console().print(f"[success]Version in chart.yaml is good:
{version}[/]")
+ if airflow_version_from_pypi != airflow_version_in_chart:
+ get_console().print(
+ f"[warning]Airflow version in chart.yaml
({airflow_version_in_chart}) does not match the "
+ f"latest version from PyPI ({airflow_version_from_pypi}).
Updating[/]"
+ )
+ updating = True
+ chart_yaml_file_content = chart_yaml_file_content.replace(
+ f"appVersion: {airflow_version_in_chart}", f"appVersion:
{airflow_version_from_pypi}"
+ )
+ else:
+ get_console().print(
+ f"[success]Airflow version in chart.yaml matches the latest
version from PyPI "
+ f"({airflow_version_from_pypi})[/]"
+ )
+ if updating:
+ CHART_YAML_FILE.write_text(chart_yaml_file_content)
+ get_console().print("\n[warning]Versions of the chart has been
updated[/]\n")
+ if ignore_version_check:
+ get_console().print(
+ "[warning]Ignoring the version check. "
+ "The tarball will be created but it should not be published[/]"
+ )
+ else:
+ get_console().print(
+ "\n[info]Please create a PR with that change, get it merged,
and try again.[/]\n"
+ )
+ sys.exit(1)
+ tag_with_suffix = f"helm-chart/{version}{version_suffix}"
+ if not skip_tagging:
+ get_console().print(f"[info]Tagging the chart with
{tag_with_suffix}[/]")
+ tag_command = [
+ "git",
+ "tag",
+ tag_with_suffix,
+ "-m",
+ f"Apache Airflow Helm Chart {version}{version_suffix}",
+ ]
+ if override_tag:
+ tag_command.append("--force")
+ if not skip_tag_signing:
+ tag_command.append("--sign")
+ result = run_command(tag_command, check=False)
+ if result.returncode != 0:
+ get_console().print(f"[error]Error tagging the chart with
{tag_with_suffix}.\n")
+ get_console().print(
+ "[warning]If you are sure the tag is set correctly, you can
add --skip-tagging"
+ " flag to the command[/]"
+ )
+ sys.exit(result.returncode)
+ else:
+ get_console().print(f"[warning]Skipping tagging the chart with
{tag_with_suffix}[/]")
+ get_console().print(f"[info]Creating tarball for Helm Chart
{tag_with_suffix}[/]")
+ archive_name = f"airflow-chart-{version}-source.tar.gz"
+ OUT_DIR.mkdir(parents=True, exist_ok=True)
+ source_archive = OUT_DIR / archive_name
+ source_archive.unlink(missing_ok=True)
+ result = run_command(
+ [
+ "git",
+ "-c",
+ "tar.umask=0077",
+ "archive",
+ "--format=tar.gz",
+ tag_with_suffix,
+ f"--prefix=airflow-chart-{version}/",
+ "-o",
+ source_archive.as_posix(),
+ "chart",
+ ".rat-excludes",
+ ],
+ check=False,
+ )
+ if result.returncode != 0:
+ get_console().print(f"[error]Error running git archive for Helm Chart
{tag_with_suffix}[/]")
+ sys.exit(result.returncode)
+ DIST_DIR.mkdir(parents=True, exist_ok=True)
+ final_archive = DIST_DIR / archive_name
+ final_archive.unlink(missing_ok=True)
+ result = repack_deterministically(
+ source_archive=source_archive,
+ dest_archive=final_archive,
+ prepend_path=None,
+ timestamp=get_source_date_epoch(CHART_DIR),
+ )
+ if result.returncode != 0:
+ get_console().print(
+ f"[error]Error repackaging source tarball for Helm Chart from
{source_archive} tp "
+ f"{tag_with_suffix}[/]"
+ )
+ sys.exit(result.returncode)
+ get_console().print(f"[success]Tarball created in {final_archive}")
+
+
+@release_management.command(name="prepare-helm-chart-package", help="Prepares
helm chart package.")
[email protected](
+ "--sign-email",
+ help="Eemail associated with the key used to sign the package.",
Review Comment:
```suggestion
help="Email associated with the key used to sign the package.",
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
