This is an automated email from the ASF dual-hosted git repository. husseinawala pushed a commit to branch test_arc_3 in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 959f9dcebec7201319eb5f4232ccb5ea96145ac3 Author: hussein-awala <[email protected]> AuthorDate: Sun Aug 25 18:35:17 2024 +0200 Test GHA ARC --- .github/workflows/basic-tests.yml | 16 +++---- .github/workflows/build-images.yml | 2 - .github/workflows/ci.yml | 4 +- dev/breeze/doc/ci/04_selective_checks.md | 2 - dev/breeze/src/airflow_breeze/global_constants.py | 6 +-- .../src/airflow_breeze/utils/selective_checks.py | 18 +------- dev/breeze/tests/test_pr_info.py | 12 ++---- dev/breeze/tests/test_selective_checks.py | 49 +++------------------- 8 files changed, 20 insertions(+), 89 deletions(-) diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index f5b1d54084..f171d5cce8 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -20,8 +20,8 @@ name: Basic tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: - description: "The array of labels (in json form) determining public runners." + runs-on-as-json-default: + description: "The array of labels (in json form) determining default runners." required: true type: string run-www-tests: @@ -62,7 +62,7 @@ jobs: run-breeze-tests: timeout-minutes: 10 name: Breeze unit tests - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} steps: - name: "Cleanup repo" shell: bash @@ -86,7 +86,7 @@ jobs: tests-www: timeout-minutes: 10 name: React WWW tests - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} if: inputs.run-www-tests == 'true' steps: - name: "Cleanup repo" @@ -115,7 +115,7 @@ jobs: test-openapi-client: timeout-minutes: 10 name: "Test OpenAPI client" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} if: inputs.needs-api-codegen == 'true' steps: - name: "Cleanup repo" @@ -206,7 +206,7 @@ jobs: static-checks-basic-checks-only: timeout-minutes: 30 name: "Static checks: basic checks only" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} if: inputs.basic-checks-only == 'true' steps: - name: "Cleanup repo" @@ -273,7 +273,7 @@ jobs: upgrade-check: timeout-minutes: 45 name: "Upgrade checks" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" if: inputs.canary-run == 'true' && inputs.latest-versions-only != 'true' @@ -341,7 +341,7 @@ jobs: test-airflow-release-commands: timeout-minutes: 80 name: "Test Airflow release commands" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" IMAGE_TAG: ${{ inputs.image-tag }} diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 1256fd2f0d..9205b0568d 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -76,8 +76,6 @@ jobs: is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }} is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }} is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }} - is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }} - is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }} chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} target-commit-sha: "${{steps.discover-pr-merge-commit.outputs.target-commit-sha || github.event.pull_request.head.sha || diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11cb390d32..2e74efd842 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,8 +116,6 @@ jobs: is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }} is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }} is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }} - is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }} - is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }} latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }} chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} has-migrations: ${{ steps.selective-checks.outputs.has-migrations }} @@ -167,7 +165,7 @@ jobs: needs: [build-info] uses: ./.github/workflows/basic-tests.yml with: - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} + runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} run-www-tests: ${{needs.build-info.outputs.run-www-tests}} needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}} default-python-version: ${{needs.build-info.outputs.default-python-version}} diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 819633d4c5..86b8fff5f8 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -194,9 +194,7 @@ Github Actions to pass the list of parameters to a command to execute | is-amd-runner | Whether runner used is an AMD one | true | | | is-arm-runner | Whether runner used is an ARM one | false | | | is-committer-build | Whether the build is triggered by a committer | false | | -| is-k8s-runner | Whether the build runs on our k8s infrastructure | false | | | is-self-hosted-runner | Whether the runner is self-hosted | false | | -| is-vm-runner | Whether the runner uses VM to run | true | | | kind-version | Which Kind version to use for tests | v0.16.0 | | | kubernetes-combos-list-as-string | All combinations of Python version and Kubernetes version to use for tests as space-separated string | 3.8-v1.25.2 3.9-v1.26.4 | * | | kubernetes-versions | All Kubernetes versions to use for tests as JSON array | ['v1.25.2'] | | diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 46b0c3d521..47b465de8e 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -32,12 +32,8 @@ from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT RUNS_ON_PUBLIC_RUNNER = '["ubuntu-22.04"]' # we should get more sophisticated logic here in the future, but for now we just check if # we use self airflow, vm-based, amd hosted runner as a default -# TODO: temporarily we need to switch to public runners to avoid issues with self-hosted runners -RUNS_ON_SELF_HOSTED_RUNNER = '["ubuntu-22.04"]' +RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "airflow-runner", "eks-runner", "X64", "small"]' RUNS_ON_SELF_HOSTED_ASF_RUNNER = '["self-hosted", "asf-runner"]' -# TODO: when we have it properly set-up with labels we should change it to -# RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "airflow-runner", "vm-runner", "X64"]' -# RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "Linux", "X64"]' SELF_HOSTED_RUNNERS_CPU_COUNT = 8 ANSWER = "" diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 224e76c251..209155ee81 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -1154,7 +1154,7 @@ class SelectiveChecks: @cached_property def runs_on_as_json_docs_build(self) -> str: if self._is_canary_run(): - return RUNS_ON_SELF_HOSTED_ASF_RUNNER + return RUNS_ON_SELF_HOSTED_RUNNER else: return RUNS_ON_PUBLIC_RUNNER @@ -1181,7 +1181,7 @@ class SelectiveChecks: # TODO: when we have it properly set-up with labels we should just check for # "airflow-runner" presence in runs_on runs_on_array = json.loads(self.runs_on_as_json_default) - return "Linux" in runs_on_array and "X64" in runs_on_array and "self-hosted" in runs_on_array + return "self-hosted" in runs_on_array and "airflow-runner" in runs_on_array @cached_property def is_amd_runner(self) -> bool: @@ -1221,20 +1221,6 @@ class SelectiveChecks: ] ) - @cached_property - def is_vm_runner(self) -> bool: - """Whether the runner is VM runner (managed by airflow).""" - # TODO: when we have it properly set-up with labels we should just check for - # "airflow-runner" presence in runs_on - return self.is_airflow_runner - - @cached_property - def is_k8s_runner(self) -> bool: - """Whether the runner is K8s runner (managed by airflow).""" - # TODO: when we have it properly set-up with labels we should just check for - # "k8s-runner" presence in runs_on - return False - @cached_property def has_migrations(self) -> bool: return any([file.startswith("airflow/migrations/") for file in self._files]) diff --git a/dev/breeze/tests/test_pr_info.py b/dev/breeze/tests/test_pr_info.py index 1c874e5c4d..292cef262a 100644 --- a/dev/breeze/tests/test_pr_info.py +++ b/dev/breeze/tests/test_pr_info.py @@ -81,9 +81,7 @@ def test_runs_on_self_hosted(): assert wi.head_repo == "apache/airflow" assert wi.event_name == "pull_request" assert wi.pr_number == 1234 - # TODO: fix it when we fix self-hosted runners - assert wi.get_runs_on() == '["ubuntu-22.04"]' - # assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]' + assert wi.get_runs_on() == '["self-hosted", "airflow-runner", "eks-runner", "X64", "small"]' assert wi.is_canary_run() == "false" assert wi.run_coverage() == "false" @@ -125,9 +123,7 @@ def test_runs_on_push_other_branch(): assert wi.head_repo == "apache/airflow" assert wi.event_name == "push" assert wi.pr_number is None - # TODO: revert it when we fix self-hosted runners - assert wi.get_runs_on() == '["ubuntu-22.04"]' - # assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]' + assert wi.get_runs_on() == '["self-hosted", "airflow-runner", "eks-runner", "X64", "small"]' assert wi.is_canary_run() == "false" assert wi.run_coverage() == "false" @@ -141,8 +137,6 @@ def test_runs_on_push_v_test_branch(): assert wi.head_repo == "apache/airflow" assert wi.event_name == "push" assert wi.pr_number is None - # TODO: revert it when we fix self-hosted runners - assert wi.get_runs_on() == '["ubuntu-22.04"]' - # assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]' + assert wi.get_runs_on() == '["self-hosted", "airflow-runner", "eks-runner", "X64", "small"]' assert wi.is_canary_run() == "true" assert wi.run_coverage() == "false" diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 09327e18ef..6434ce7a7d 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1752,7 +1752,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d ( "github_event, github_actor, github_repository, pr_labels, " "github_context_dict, runs_on_as_json_default, runs_on_as_docs_build, is_self_hosted_runner, " - "is_airflow_runner, is_amd_runner, is_arm_runner, is_vm_runner, is_k8s_runner, exception" + "is_airflow_runner, is_amd_runner, is_arm_runner, exception" ), [ pytest.param( @@ -1761,20 +1761,11 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d "apache/airflow", (), dict(), - # TODO: revert it when we fix self-hosted runners - '["ubuntu-22.04"]', - '["self-hosted", "asf-runner"]', - # '["self-hosted", "Linux", "X64"]', - # TODO: revert it when we fix self-hosted runners - "false", - "false", - # "true", - # "true", + '["self-hosted", "airflow-runner", "eks-runner", "X64", "small"]', + '["self-hosted", "airflow-runner", "eks-runner", "X64", "small"]', + "true", + "true", "true", - "false", - # TODO: revert it when we fix self-hosted runners - "false", - # "true", "false", False, id="Push event", @@ -1791,8 +1782,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d "false", "true", "false", - "false", - "false", False, id="Push event for private repo", ), @@ -1808,8 +1797,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d "false", "true", "false", - "false", - "false", False, id="Pull request", ), @@ -1825,8 +1812,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d "false", "true", "false", - "false", - "false", False, id="Pull request private repo", ), @@ -1842,8 +1827,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d "false", "true", "false", - "false", - "false", False, id="Pull request committer", ), @@ -1859,8 +1842,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d "false", "true", "false", - "false", - "false", False, id="Pull request committer pr non-committer", ), @@ -1876,8 +1857,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d "false", "true", "false", - "false", - "false", False, id="Pull request private repo committer", ), @@ -1893,8 +1872,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d "false", "true", "false", - "false", - "false", False, id="Pull request target", ), @@ -1910,8 +1887,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d "false", "true", "false", - "false", - "false", False, id="Pull request target private repo", ), @@ -1927,8 +1902,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d "false", "true", "false", - "false", - "false", False, id="Pull request target committer", ), @@ -1944,8 +1917,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d "false", "true", "false", - "false", - "false", False, id="Pull request target committer pr non-committer", ), @@ -1961,8 +1932,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d "false", "true", "false", - "false", - "false", False, id="Pull request targe private repo committer", ), @@ -1978,8 +1947,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d "false", "true", "false", - "false", - "false", True, id="Pull request by non committer with 'use self-hosted runners' label.", ), @@ -1995,8 +1962,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d "false", "true", "false", - "false", - "false", False, id="Pull request by committer with 'use public runners' label.", ), @@ -2014,8 +1979,6 @@ def test_runs_on( is_airflow_runner: str, is_amd_runner: str, is_arm_runner: str, - is_vm_runner: str, - is_k8s_runner: str, exception: bool, ): def get_output() -> str: @@ -2043,8 +2006,6 @@ def test_runs_on( assert_outputs_are_printed({"is-airflow-runner": is_airflow_runner}, str(stderr)) assert_outputs_are_printed({"is-amd-runner": is_amd_runner}, str(stderr)) assert_outputs_are_printed({"is-arm-runner": is_arm_runner}, str(stderr)) - assert_outputs_are_printed({"is-vm-runner": is_vm_runner}, str(stderr)) - assert_outputs_are_printed({"is-k8s-runner": is_k8s_runner}, str(stderr)) @pytest.mark.parametrize(
