This is an automated email from the ASF dual-hosted git repository. maximebeauchemin pushed a commit to branch no-op in repository https://gitbox.apache.org/repos/asf/superset.git
commit bbc3de5867a3c82f30195c88660d7256c51bff2d Author: Maxime Beauchemin <[email protected]> AuthorDate: Thu Apr 4 09:53:17 2024 -0700 chore: [proposal] de-matrix python-version in GHAs In this PR: - simplifying the single-item matrix (python-version) to NOT using a matrix. I'm guessing the reason we currently have a single-item matrix is an artifact of supporting multiple version in the past, and/or making it easy to go multi-python-version checks in the future, but there's a burden associated, especially around how this relates to "required checks" specified in .asf.yml - leveraging the `setup-backend`'s default for python version, making the main python version we use much more DRY. - fixing/simplifying the related no-op. We'll need new ones, but will be able to deprecate a bunch and simplify things. For instance, when we migrate to 3.11 in the future, we won't have to manage a bunch of python-version-specific no-ops About supporting multiple/future version of python, I'd argue that we should focus on a single one for a given CI run, and that if/when we need to CI against multiple version, we run a FULL test suite punctually in a dedicate PR/branch/ref. Point being, it's expensive for every commit to validate multiple versions of python and in many ways its not necessary. Currently our multi-python-version support is dubious at best, with only few checks that run against multiple versions. I really think we should pick a single version and support it very well. If/when we want to upgrade python version, we'd cut a PR and run CI for that purpose. If we want to continuously, actively support multiple python versions (and I don't think we should!), I'd suggest either a release-specific procedure (release manager using release branch, running full CI for that version/release) and/or a nightly job that would keep an eye on that version of python. --- .asf.yaml | 17 ++-- .github/actions/setup-backend/action.yml | 18 ++-- .github/workflows/no-op.yml | 103 ++++++++++++--------- .github/workflows/pre-commit.yml | 3 - .github/workflows/superset-cli.yml | 3 - .github/workflows/superset-helm-lint.yml | 5 +- .../workflows/superset-python-integrationtest.yml | 15 --- .github/workflows/superset-python-misc.yml | 10 -- .github/workflows/superset-python-presto-hive.yml | 6 -- .github/workflows/superset-python-unittest.yml | 5 - .github/workflows/superset-translations.yml | 5 - 11 files changed, 81 insertions(+), 109 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 9b2bfae806..106a9c39b7 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -67,15 +67,14 @@ github: - cypress-matrix (2, chrome) - cypress-matrix (3, chrome) - frontend-build - - pre-commit (3.10) - - python-lint (3.10) - - test-mysql (3.10) - - test-postgres (3.10) - - test-postgres (3.11) - - test-postgres-hive (3.10) - - test-postgres-presto (3.10) - - test-sqlite (3.10) - - unit-tests (3.10) + - pre-commit + - python-lint + - test-mysql + - test-postgres + - test-postgres-hive + - test-postgres-presto + - test-sqlite + - unit-tests required_pull_request_reviews: dismiss_stale_reviews: false diff --git a/.github/actions/setup-backend/action.yml b/.github/actions/setup-backend/action.yml index 5f6ba8d1a0..ce7f0208b5 100644 --- a/.github/actions/setup-backend/action.yml +++ b/.github/actions/setup-backend/action.yml @@ -13,6 +13,10 @@ inputs: description: 'Type of requirements to install. Options: base, development, default' required: false default: 'dev' + install-superset: + description: 'Whether to install Superset itself. If false, only python is installed' + required: false + default: 'true' runs: using: 'composite' @@ -24,11 +28,13 @@ runs: cache: ${{ inputs.cache }} - name: Install dependencies run: | - sudo apt-get update && sudo apt-get -y install libldap2-dev libsasl2-dev - pip install --upgrade pip setuptools wheel - if [ "${{ inputs.requirements-type }}" = "dev" ]; then - pip install -r requirements/development.txt - elif [ "${{ inputs.requirements-type }}" = "base" ]; then - pip install -r requirements/base.txt + if [ "${{ inputs.install-superset }}" = "true" ]; then + sudo apt-get update && sudo apt-get -y install libldap2-dev libsasl2-dev + pip install --upgrade pip setuptools wheel + if [ "${{ inputs.requirements-type }}" = "dev" ]; then + pip install -r requirements/development.txt + elif [ "${{ inputs.requirements-type }}" = "base" ]; then + pip install -r requirements/base.txt + fi fi shell: bash diff --git a/.github/workflows/no-op.yml b/.github/workflows/no-op.yml index 9cc9e8bec3..8927474d17 100644 --- a/.github/workflows/no-op.yml +++ b/.github/workflows/no-op.yml @@ -26,89 +26,102 @@ jobs: frontend-build: runs-on: ubuntu-latest steps: - - name: No-op for frontend-build - run: | - echo "This is a no-op step for frontend-build to ensure a successful status." - exit 0 - + - run: exit 0 +# ------------------------------------ +# DE-COMMENT POST MERGING +# ------------------------------------ +# +# pre-commit: +# runs-on: ubuntu-latest +# steps: +# - run: exit 0 +# python-lint: +# runs-on: ubuntu-latest +# steps: +# - run: exit 0 +# test-postgres-hive: +# runs-on: ubuntu-latest +# steps: +# - run: exit 0 +# test-postgres-presto: +# runs-on: ubuntu-latest +# steps: +# - run: exit 0 +# unit-tests: +# runs-on: ubuntu-latest +# steps: +# - run: exit 0 +# test-mysql: +# runs-on: ubuntu-latest +# steps: +# - run: exit 0 +# test-postgres: +# runs-on: ubuntu-latest +# steps: +# - run: exit 0 +# test-sqlite: +# runs-on: ubuntu-latest +# steps: +# - run: exit 0 +## ------------------------------------------------------ +## TO BE DEPRECATED +## These no-ops were used to deprecate/renaming checks +## and can be safely delete if seen on `master` +## ------------------------------------------------------ pre-commit: strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.10", ""] runs-on: ubuntu-latest steps: - - name: No-op for pre-commit - run: | - echo "This is a no-op step for pre-commit to ensure a successful status." - exit 0 + - run: exit 0 python-lint: strategy: matrix: - python-version: ["3.9", "3.10"] + python-version: ["3.9", "3.10", ""] runs-on: ubuntu-latest steps: - - name: No-op for python-lint - run: | - echo "This is a no-op step for python-lint to ensure a successful status." - exit 0 + - run: exit 0 test-postgres-hive: strategy: matrix: - python-version: ["3.9", "3.10"] + python-version: ["3.9", "3.10", ""] runs-on: ubuntu-latest steps: - - name: No-op for frontend-build - run: | - echo "This is a no-op step for test-postgres-postgres to ensure a successful status when skipped." - exit 0 + - run: exit 0 test-postgres-presto: strategy: matrix: - python-version: ["3.9", "3.10"] + python-version: ["3.9", "3.10", ""] runs-on: ubuntu-latest steps: - - name: No-op for frontend-build - run: | - echo "This is a no-op step for test-postgres-postgres to ensure a successful status when skipped." - exit 0 + - run: exit 0 unit-tests: strategy: matrix: - python-version: ["3.9", "3.10"] + python-version: ["3.9", "3.10", ""] runs-on: ubuntu-latest steps: - - name: No-op for frontend-build - run: | - echo "This is a no-op step for unit-tests to ensure a successful status when skipped." - exit 0 + - run: exit 0 test-mysql: strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.10", ""] runs-on: ubuntu-latest steps: - - name: No-op for test-mysql - run: | - echo "This is a no-op step for test-mysql to ensure a successful status when skipped." - exit 0 + - run: exit 0 test-postgres: strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.10", ""] runs-on: ubuntu-latest steps: - - name: No-op for test-postgres - run: | - echo "This is a no-op step for test-postgres to ensure a successful status when skipped." - exit 0 + - run: exit 0 test-sqlite: strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.10", ""] runs-on: ubuntu-latest steps: - - name: No-op for test-sqlite - run: | - echo "This is a no-op step for test-sqlite to ensure a successful status when skipped." - exit 0 + - run: exit 0 diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 27af14d224..541b4a8bfc 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -16,9 +16,6 @@ concurrency: jobs: pre-commit: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10"] steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 diff --git a/.github/workflows/superset-cli.yml b/.github/workflows/superset-cli.yml index f0e037197e..ffe446b4e2 100644 --- a/.github/workflows/superset-cli.yml +++ b/.github/workflows/superset-cli.yml @@ -16,9 +16,6 @@ concurrency: jobs: test-load-examples: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10"] env: PYTHONPATH: ${{ github.workspace }} SUPERSET_CONFIG: tests.integration_tests.superset_test_config diff --git a/.github/workflows/superset-helm-lint.yml b/.github/workflows/superset-helm-lint.yml index f16dd61fdd..f9e068318d 100644 --- a/.github/workflows/superset-helm-lint.yml +++ b/.github/workflows/superset-helm-lint.yml @@ -27,9 +27,10 @@ jobs: with: version: v3.5.4 - - uses: actions/setup-python@v5 + - name: Setup Python + uses: ./.github/actions/setup-backend/ with: - python-version: "3.10" + install-superset: 'false' - name: Set up chart-testing uses: ./.github/actions/chart-testing-action diff --git a/.github/workflows/superset-python-integrationtest.yml b/.github/workflows/superset-python-integrationtest.yml index 75eea5c70a..6bd6465aa5 100644 --- a/.github/workflows/superset-python-integrationtest.yml +++ b/.github/workflows/superset-python-integrationtest.yml @@ -16,9 +16,6 @@ concurrency: jobs: test-mysql: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10"] env: PYTHONPATH: ${{ github.workspace }} SUPERSET_CONFIG: tests.integration_tests.superset_test_config @@ -53,8 +50,6 @@ jobs: - name: Setup Python uses: ./.github/actions/setup-backend/ if: steps.check.outcome == 'failure' - with: - python-version: ${{ matrix.python-version }} - name: Setup MySQL if: steps.check.outcome == 'failure' uses: ./.github/actions/cached-dependencies @@ -74,9 +69,6 @@ jobs: bash .github/workflows/codecov.sh -c -F python -F mysql test-postgres: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10", "3.11"] env: PYTHONPATH: ${{ github.workspace }} SUPERSET_CONFIG: tests.integration_tests.superset_test_config @@ -112,8 +104,6 @@ jobs: - name: Setup Python uses: ./.github/actions/setup-backend/ if: steps.check.outcome == 'failure' - with: - python-version: ${{ matrix.python-version }} - name: Setup Postgres if: steps.check.outcome == 'failure' uses: ./.github/actions/cached-dependencies @@ -134,9 +124,6 @@ jobs: test-sqlite: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10"] env: PYTHONPATH: ${{ github.workspace }} SUPERSET_CONFIG: tests.integration_tests.superset_test_config @@ -166,8 +153,6 @@ jobs: - name: Setup Python uses: ./.github/actions/setup-backend/ if: steps.check.outcome == 'failure' - with: - python-version: ${{ matrix.python-version }} - name: Install dependencies if: steps.check.outcome == 'failure' uses: ./.github/actions/cached-dependencies diff --git a/.github/workflows/superset-python-misc.yml b/.github/workflows/superset-python-misc.yml index 8eb34d939a..9ab6fc99e3 100644 --- a/.github/workflows/superset-python-misc.yml +++ b/.github/workflows/superset-python-misc.yml @@ -21,9 +21,6 @@ concurrency: jobs: python-lint: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10"] steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 @@ -40,8 +37,6 @@ jobs: - name: Setup Python uses: ./.github/actions/setup-backend/ if: steps.check.outcome == 'failure' - with: - python-version: ${{ matrix.python-version }} - name: pylint if: steps.check.outcome == 'failure' # `-j 0` run Pylint in parallel @@ -49,9 +44,6 @@ jobs: babel-extract: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10"] steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 @@ -60,7 +52,5 @@ jobs: submodules: recursive - name: Setup Python uses: ./.github/actions/setup-backend/ - with: - python-version: ${{ matrix.python-version }} - name: Test babel extraction run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct diff --git a/.github/workflows/superset-python-presto-hive.yml b/.github/workflows/superset-python-presto-hive.yml index dd8d6612aa..7bc6bba2cc 100644 --- a/.github/workflows/superset-python-presto-hive.yml +++ b/.github/workflows/superset-python-presto-hive.yml @@ -21,9 +21,6 @@ concurrency: jobs: test-postgres-presto: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10"] env: PYTHONPATH: ${{ github.workspace }} SUPERSET_CONFIG: tests.integration_tests.superset_test_config @@ -88,9 +85,6 @@ jobs: test-postgres-hive: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10"] env: PYTHONPATH: ${{ github.workspace }} SUPERSET_CONFIG: tests.integration_tests.superset_test_config diff --git a/.github/workflows/superset-python-unittest.yml b/.github/workflows/superset-python-unittest.yml index c5bf8bf46d..a6bdb171f3 100644 --- a/.github/workflows/superset-python-unittest.yml +++ b/.github/workflows/superset-python-unittest.yml @@ -27,9 +27,6 @@ concurrency: jobs: unit-tests: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10", "3.11"] env: PYTHONPATH: ${{ github.workspace }} steps: @@ -48,8 +45,6 @@ jobs: - name: Setup Python uses: ./.github/actions/setup-backend/ if: steps.check.outcome == 'failure' - with: - python-version: ${{ matrix.python-version }} - name: Python unit tests if: steps.check.outcome == 'failure' env: diff --git a/.github/workflows/superset-translations.yml b/.github/workflows/superset-translations.yml index 9a8b8df1e3..0cc217bbd4 100644 --- a/.github/workflows/superset-translations.yml +++ b/.github/workflows/superset-translations.yml @@ -37,9 +37,6 @@ jobs: babel-extract: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10"] steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 @@ -48,7 +45,5 @@ jobs: submodules: recursive - name: Setup Python uses: ./.github/actions/setup-backend/ - with: - python-version: ${{ matrix.python-version }} - name: Test babel extraction run: ./scripts/babel_update.sh
