This is an automated email from the ASF dual-hosted git repository. maximebeauchemin pushed a commit to branch docker_multi_platform_releases in repository https://gitbox.apache.org/repos/asf/superset.git
commit 774077bfc72939a4c302c2a1c0aabc44ba11dfc6 Author: Maxime Beauchemin <[email protected]> AuthorDate: Thu Feb 8 12:47:26 2024 -0800 feat(docker): allow for docker release builds to be multi-platform Allow for release builds to be multi-platform --- .github/workflows/docker-release.yml | 14 +++----- scripts/build_docker.py | 34 +++++++++++------- tests/unit_tests/scripts/docker_build.py | 60 +++++++++++++++++++------------- 3 files changed, 60 insertions(+), 48 deletions(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 3ba7b6bf42..ae572c910e 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -43,14 +43,6 @@ jobs: strategy: matrix: build_preset: ["dev", "lean", "py310", "websocket", "dockerize"] - platform: ["linux/amd64", "linux/arm64"] - exclude: - # disabling because slow! no python wheels for arm/py39 and - # QEMU is slow! - - build_preset: "dev" - platform: "linux/arm64" - - build_preset: "lean" - platform: "linux/arm64" fail-fast: false steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -88,8 +80,10 @@ jobs: EVENT="release" fi pip install click + # Make a multi-platform image ./scripts/build_docker.py \ ${{ matrix.build_preset }} \ "$EVENT" \ - --build_context_ref "$RELEASE" \ - --platform ${{ matrix.platform }} $FORCE_LATEST + --build_context_ref "$RELEASE" $FORCE_LATEST \ + --platform "linux/arm64" + --platform "linux/amd64" diff --git a/scripts/build_docker.py b/scripts/build_docker.py index de6b8444f8..095a0f188d 100755 --- a/scripts/build_docker.py +++ b/scripts/build_docker.py @@ -79,7 +79,7 @@ def make_docker_tag(l: list[str]) -> str: def get_docker_tags( build_preset: str, - build_platform: str, + build_platforms: list[str], sha: str, build_context: str, build_context_ref: str, @@ -91,17 +91,18 @@ def get_docker_tags( tags: set[str] = set() tag_chunks: list[str] = [] - short_build_platform = build_platform.replace("linux/", "").replace("64", "") - is_latest = is_latest_release(build_context_ref) if build_preset != "lean": # Always add the preset_build name if different from default (lean) tag_chunks += [build_preset] - if short_build_platform != "amd": - # Always a platform indicator if different from default (amd) - tag_chunks += [short_build_platform] + if len(build_platforms) == 1: + build_platform = build_platforms[0] + short_build_platform = build_platform.replace("linux/", "").replace("64", "") + if short_build_platform != "amd": + # Always a platform indicator if different from default (amd) + tag_chunks += [short_build_platform] # Always craft a tag for the SHA tags.add(make_docker_tag([sha] + tag_chunks)) @@ -123,7 +124,7 @@ def get_docker_tags( def get_docker_command( build_preset: str, - build_platform: str, + build_platforms: list[str], is_authenticated: bool, sha: str, build_context: str, @@ -160,7 +161,7 @@ def get_docker_command( tags = get_docker_tags( build_preset, - build_platform, + build_platforms, sha, build_context, build_context_ref, @@ -170,8 +171,14 @@ def get_docker_command( docker_args = "--load" if not is_authenticated else "--push" target_argument = f"--target {build_target}" if build_target else "" - short_build_platform = build_platform.replace("linux/", "").replace("64", "") - cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}" + + cache_ref = f"{CACHE_REPO}:{py_ver}" + if len(build_platforms) == 1: + build_platform = build_platforms[0] + short_build_platform = build_platform.replace("linux/", "").replace("64", "") + cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}" + platform_arg = "--platform " + ",".join(build_platforms) + cache_from_arg = f"--cache-from=type=registry,ref={cache_ref}" cache_to_arg = ( f"--cache-to=type=registry,mode=max,ref={cache_ref}" if is_authenticated else "" @@ -187,7 +194,7 @@ def get_docker_command( {cache_from_arg} \\ {cache_to_arg} \\ {build_arg} \\ - --platform {build_platform} \\ + {platform_arg} \\ --label sha={sha} \\ --label target={build_target} \\ --label build_trigger={build_context} \\ @@ -206,7 +213,8 @@ def get_docker_command( @click.option( "--platform", type=click.Choice(["linux/arm64", "linux/amd64"]), - default="linux/amd64", + default=["linux/amd64"], + multiple=True, ) @click.option("--build_context_ref", help="a reference to the pr, release or branch") @click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.") @@ -217,7 +225,7 @@ def main( build_preset: str, build_context: str, build_context_ref: str, - platform: str, + platform: list[str], dry_run: bool, force_latest: bool, ) -> None: diff --git a/tests/unit_tests/scripts/docker_build.py b/tests/unit_tests/scripts/docker_build.py index ee9ad66ead..1f3fd708f3 100644 --- a/tests/unit_tests/scripts/docker_build.py +++ b/tests/unit_tests/scripts/docker_build.py @@ -56,12 +56,12 @@ def test_is_latest_release(release, expected_bool): @pytest.mark.parametrize( - "build_preset, build_platform, sha, build_context, build_context_ref, expected_tags", + "build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags", [ # PRs ( "lean", - "linux/arm64", + ["linux/arm64"], SHA, "pull_request", PR_ID, @@ -69,7 +69,7 @@ def test_is_latest_release(release, expected_bool): ), ( "ci", - "linux/amd64", + ["linux/amd64"], SHA, "pull_request", PR_ID, @@ -77,7 +77,7 @@ def test_is_latest_release(release, expected_bool): ), ( "lean", - "linux/amd64", + ["linux/amd64"], SHA, "pull_request", PR_ID, @@ -85,7 +85,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/arm64", + ["linux/arm64"], SHA, "pull_request", PR_ID, @@ -97,7 +97,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/amd64", + ["linux/amd64"], SHA, "pull_request", PR_ID, @@ -106,7 +106,7 @@ def test_is_latest_release(release, expected_bool): # old releases ( "lean", - "linux/arm64", + ["linux/arm64"], SHA, "release", OLD_REL, @@ -114,7 +114,7 @@ def test_is_latest_release(release, expected_bool): ), ( "lean", - "linux/amd64", + ["linux/amd64"], SHA, "release", OLD_REL, @@ -122,7 +122,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/arm64", + ["linux/arm64"], SHA, "release", OLD_REL, @@ -134,7 +134,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/amd64", + ["linux/amd64"], SHA, "release", OLD_REL, @@ -143,7 +143,7 @@ def test_is_latest_release(release, expected_bool): # new releases ( "lean", - "linux/arm64", + ["linux/arm64"], SHA, "release", NEW_REL, @@ -156,7 +156,7 @@ def test_is_latest_release(release, expected_bool): ), ( "lean", - "linux/amd64", + ["linux/amd64"], SHA, "release", NEW_REL, @@ -164,7 +164,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/arm64", + ["linux/arm64"], SHA, "release", NEW_REL, @@ -177,7 +177,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/amd64", + ["linux/amd64"], SHA, "release", NEW_REL, @@ -191,7 +191,7 @@ def test_is_latest_release(release, expected_bool): # merge on master ( "lean", - "linux/arm64", + ["linux/arm64"], SHA, "push", "master", @@ -199,7 +199,7 @@ def test_is_latest_release(release, expected_bool): ), ( "lean", - "linux/amd64", + ["linux/amd64"], SHA, "push", "master", @@ -207,7 +207,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/arm64", + ["linux/arm64"], SHA, "push", "master", @@ -219,7 +219,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/amd64", + ["linux/amd64"], SHA, "push", "master", @@ -228,21 +228,21 @@ def test_is_latest_release(release, expected_bool): ], ) def test_get_docker_tags( - build_preset, build_platform, sha, build_context, build_context_ref, expected_tags + build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags ): tags = docker_utils.get_docker_tags( - build_preset, build_platform, sha, build_context, build_context_ref + build_preset, build_platforms, sha, build_context, build_context_ref ) for tag in expected_tags: assert tag in tags @pytest.mark.parametrize( - "build_preset, build_platform, is_authenticated, sha, build_context, build_context_ref, contains", + "build_preset, build_platforms, is_authenticated, sha, build_context, build_context_ref, contains", [ ( "lean", - "linux/amd64", + ["linux/amd64"], True, SHA, "push", @@ -251,18 +251,28 @@ def test_get_docker_tags( ), ( "dev", - "linux/amd64", + ["linux/amd64"], False, SHA, "push", "master", ["--load", f"-t {REPO}:master-dev "], ), + # multi-platform + ( + "lean", + ["linux/arm64", "linux/amd64"], + True, + SHA, + "push", + "master", + [f"--platform linux/arm64,linux/amd64"], + ), ], ) def test_get_docker_command( build_preset, - build_platform, + build_platforms, is_authenticated, sha, build_context, @@ -271,7 +281,7 @@ def test_get_docker_command( ): cmd = docker_utils.get_docker_command( build_preset, - build_platform, + build_platforms, is_authenticated, sha, build_context,
