This is an automated email from the ASF dual-hosted git repository. maximebeauchemin pushed a commit to branch kill-pip-compile-multi in repository https://gitbox.apache.org/repos/asf/superset.git
commit 012d2b5c512482a39baa9b5e4889108686616a5a Author: Maxime Beauchemin <[email protected]> AuthorDate: Tue Mar 12 18:00:28 2024 -0700 drying actions --- .../workflows/superset-python-integrationtest.yml | 28 +--- .github/workflows/superset-python-unittest.yml | 18 +-- pyproject.toml | 6 + requirements/base.txt | 1 - requirements/development.txt | 2 - requirements/pip-compile-custom.sh | 32 ----- requirements/pip-compile-superset.py | 148 +++++++++++++++++++++ requirements/pip-compile-superset.sh | 33 ----- requirements/requirements-custom-example.in | 2 +- 9 files changed, 160 insertions(+), 110 deletions(-) diff --git a/.github/workflows/superset-python-integrationtest.yml b/.github/workflows/superset-python-integrationtest.yml index 8f753e1668..da9136713d 100644 --- a/.github/workflows/superset-python-integrationtest.yml +++ b/.github/workflows/superset-python-integrationtest.yml @@ -51,22 +51,8 @@ jobs: continue-on-error: true run: ./scripts/ci_check_no_file_changes.sh python - name: Setup Python + uses: ./.github/actions/setup-python.yml if: steps.check.outcome == 'failure' - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - cache-dependency-path: "requirements/development.txt" - - name: Install dependencies - if: steps.check.outcome == 'failure' - uses: ./.github/actions/cached-dependencies - with: - run: | - apt-get-install - pip-upgrade - pip install wheel - pip install -r requirements/development.txt - setup-mysql - name: Run celery if: steps.check.outcome == 'failure' run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 & @@ -117,12 +103,8 @@ jobs: continue-on-error: true run: ./scripts/ci_check_no_file_changes.sh python - name: Setup Python + uses: ./.github/actions/setup-python.yml if: steps.check.outcome == 'failure' - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - cache-dependency-path: "requirements/development.txt" - name: Install dependencies if: steps.check.outcome == 'failure' uses: ./.github/actions/cached-dependencies @@ -177,12 +159,8 @@ jobs: continue-on-error: true run: ./scripts/ci_check_no_file_changes.sh python - name: Setup Python + uses: ./.github/actions/setup-python.yml if: steps.check.outcome == 'failure' - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - cache-dependency-path: "requirements/development.txt" - name: Install dependencies if: steps.check.outcome == 'failure' uses: ./.github/actions/cached-dependencies diff --git a/.github/workflows/superset-python-unittest.yml b/.github/workflows/superset-python-unittest.yml index 3730c59c03..309cade866 100644 --- a/.github/workflows/superset-python-unittest.yml +++ b/.github/workflows/superset-python-unittest.yml @@ -46,24 +46,10 @@ jobs: continue-on-error: true run: ./scripts/ci_check_no_file_changes.sh python - name: Setup Python + uses: ./.github/actions/setup-python.yml if: steps.check.outcome == 'failure' - uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} - cache: "pip" - cache-dependency-path: "requirements/development.txt" - # TODO: separated requirements.txt file just for unit tests - - name: Install dependencies - if: steps.check.outcome == 'failure' - uses: ./.github/actions/cached-dependencies - with: - run: | - apt-get-install - pip-upgrade - pip install wheel - pip install -r requirements/development.txt - pip install shillelagh - mkdir ${{ github.workspace }}/.temp + requirements-type: 'dev' - name: Python unit tests if: steps.check.outcome == 'failure' env: diff --git a/pyproject.toml b/pyproject.toml index 75e9ecaf7d..e6c3db4b8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -205,3 +205,9 @@ development = [ "statsd", "tox", ] + +[tool.mypy] +exclude = '^requirements/*.py' + +[mypy-click] +ignore_missing_imports = True diff --git a/requirements/base.txt b/requirements/base.txt index 6fb321fb24..a5ee1da927 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -405,4 +405,3 @@ zipp==3.15.0 # The following packages are considered to be unsafe in a requirements file: # setuptools --e . diff --git a/requirements/development.txt b/requirements/development.txt index 71c8464d83..ea19e6401e 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -809,5 +809,3 @@ zope-interface==5.4.0 # The following packages are considered to be unsafe in a requirements file: # setuptools --e . --e . diff --git a/requirements/pip-compile-custom.sh b/requirements/pip-compile-custom.sh deleted file mode 100644 index bc04d19b33..0000000000 --- a/requirements/pip-compile-custom.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A simple example of how to customize your python dependencies while re-using -# as much as the pinned ones from the true-and-tested ones - -# first, create your own `requirements.in` file, as this example file -# see `requirements/requirements-custom-example.in` as an example - -# copy the pinned dependency so that we can make it the target outpout for pip-compile -# here we're getting pip-compile into using its output as an input -cp requirements/development.txt requirements/requirements-custom-example.txt - -# pip-compile mixing your input and output (which also acts as the basis) -pip-compile -o requirements-custom-example.in requirements-custom-example.in - -# this ideally is done as part of the release process, whenever any -# of the files referenced here change, including requirements/development.txt diff --git a/requirements/pip-compile-superset.py b/requirements/pip-compile-superset.py new file mode 100644 index 0000000000..9065004a94 --- /dev/null +++ b/requirements/pip-compile-superset.py @@ -0,0 +1,148 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import subprocess +import sys +from shutil import copyfile + +import click + +BASE_REQS = "requirements/base.txt" +DEV_REQS = "requirements/development.txt" + +DEV_EXTRAS = [ + "bigquery", + "cors", + "development", + "druid", + "hive", + "gevent", + "mysql", + "postgres", + "presto", + "prophet", + "trino", + "gsheets", + "playwright", + "thumbnails", +] + + +def read_requirements(path): + """Read requirements from a file and return them as a dictionary.""" + requirements = {} + with open(path) as file: + for line in file: + line = line.strip() + if line and not line.startswith("#"): + parts = line.split("==") + lib_name = parts[0].strip() + version = parts[1].strip() if len(parts) == 2 else None + requirements[lib_name] = version + return requirements + + +def compare_requirements(reqs1, reqs2): + """Compare two sets of requirements and identify differences.""" + added = {lib: ver for lib, ver in reqs2.items() if lib not in reqs1} + removed = {lib: ver for lib, ver in reqs1.items() if lib not in reqs2} + version_changed = { + lib: (reqs1[lib], reqs2[lib]) + for lib in reqs1 + if lib in reqs2 and reqs1[lib] != reqs2[lib] + } + return added, removed, version_changed + + +def bash(cmd): + print(f"RUN: {cmd}") + result = subprocess.run(cmd, shell=True) + if result.returncode != 0: + print(f"Error: Command '{cmd}' exited with {result.returncode}") + sys.exit(result.returncode) + + [email protected]() +def cli(): + pass + + [email protected]() [email protected]("--pip-flags", default="", help="Flags to pass directly to pip-compile.") +def compile_deps(pip_flags): + """Compile dependencies using pip-compile with optional flags.""" + # pip-compile commands + bash(f"pip-compile -o {BASE_REQS} {pip_flags}") + bash(f'pip-compile -o {DEV_REQS} -v {pip_flags} --extra {",".join(DEV_EXTRAS)}') + + click.echo("Dependencies compiled.") + + [email protected]() [email protected]( + "--dev", is_flag=True, help="Install development dependencies instead of base." +) +def install_deps(dev): + """Install dependencies from the compiled requirements file.""" + file_path = "requirements/development.txt" if dev else "requirements/base.txt" + bash(f"pip install -r {file_path}") + # Installing the project itself in editable mode + bash("pip install -e .") + click.echo(f"Dependencies from {file_path} and project installed.") + + [email protected]() [email protected]("file1", type=click.Path(exists=True), default=BASE_REQS) [email protected]("file2", type=click.Path(exists=True), default=DEV_REQS) +def compare_versions(file1, file2): + """Load two requirements files, compare them, and print differences.""" + reqs1 = read_requirements(file1) + reqs2 = read_requirements(file2) + added, removed, version_changed = compare_requirements(reqs1, reqs2) + + if added: + click.echo("Added:") + for lib, ver in added.items(): + click.echo(f"{lib}=={ver}") + if removed: + click.echo("\nRemoved:") + for lib, ver in removed.items(): + click.echo(f"{lib}=={ver}") + if version_changed: + click.echo("\nVersion Changed:") + for lib, versions in version_changed.items(): + click.echo(f"{lib}: from {versions[0]} to {versions[1]}") + + [email protected]() [email protected]("input_file") [email protected]("output_file") +def merge_compile(input_file, output_file): + """Merge, compile and check versions.""" + # Step 1: Copy development.txt to the output file + copyfile(DEV_REQS, output_file) + + # Step 2 & 3: Compile the user-defined requirements file appending to the output + bash(f"pip-compile -v {input_file} --output-file={output_file}") + + +cli.add_command(merge_compile) +cli.add_command(compile_deps) +cli.add_command(install_deps) +cli.add_command(compare_versions) + +if __name__ == "__main__": + cli() diff --git a/requirements/pip-compile-superset.sh b/requirements/pip-compile-superset.sh deleted file mode 100755 index 34f950ca42..0000000000 --- a/requirements/pip-compile-superset.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A simple bash script to "compile"/pin our python dependencies using pip-compile - -# Let's forward all script arguments to pip-compile -# you can pass things like `--no-upgrade` or target a specific package with `-P "flask"` -pip_compile_flags="$@" - -# Compile the base requirements -pip-compile -o requirements/base.txt $pip_compile_flags - -# Compile the development requirements with extras -pip-compile -o requirements/development.txt -v $pip_compile_flags \ - --extra bigquery,cors,development,druid,hive,gevent,mysql,postgres,presto,prophet,trino,gsheets,playwright,shillelagh,thumbnails - -# Append '-e .' to both requirements files to include the project itself -echo "-e ." >> requirements/base.txt -echo "-e ." >> requirements/development.txt diff --git a/requirements/requirements-custom-example.in b/requirements/requirements-custom-example.in index 4d1b630dfe..215256ab9f 100644 --- a/requirements/requirements-custom-example.in +++ b/requirements/requirements-custom-example.in @@ -5,4 +5,4 @@ cherrytree # pin an existing Superset reference to a specific version -boto3==1.18.44 +boto3==1.29.7
