This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-8-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 2fba7bd0b38e6f5e2ec842e24878d217e195101c Author: Jarek Potiuk <[email protected]> AuthorDate: Tue Nov 28 21:37:58 2023 +0100 Remove workaround for pymssql failing compilation with new Cython (#35924) Recent Cython release caused pymssql package failures when they were installed on ARM platform. This had been workarounded in the #32748, but since pymssql as of 2.1.8 supports new Cython, we can remove the workaround and bump the minimum version of pymsssql. This also makes it possible to remove the whole MSSQL client section from the image if we decide to - because this section will only install the odbc client that has been pre-installed to support MSSQL as metadata DB for Airflow core. (cherry picked from commit 4f060a482c3233504e7905b3ab2d00fe56ea43cd) --- Dockerfile | 42 ------------------------- Dockerfile.ci | 36 --------------------- airflow/providers/microsoft/mssql/provider.yaml | 2 +- generated/provider_dependencies.json | 2 +- scripts/docker/install_mssql.sh | 39 ----------------------- 5 files changed, 2 insertions(+), 119 deletions(-) diff --git a/Dockerfile b/Dockerfile index b9b358d0c7..7a7cb89225 100644 --- a/Dockerfile +++ b/Dockerfile @@ -347,8 +347,6 @@ COPY <<"EOF" /install_mssql.sh #!/usr/bin/env bash set -euo pipefail -. "$( dirname "${BASH_SOURCE[0]}" )/common.sh" - : "${AIRFLOW_PIP_VERSION:?Should be set}" : "${INSTALL_MSSQL_CLIENT:?Should be true or false}" @@ -384,40 +382,6 @@ function install_mssql_client() { rm -rf /var/lib/apt/lists/* apt-get autoremove -yqq --purge apt-get clean && rm -rf /var/lib/apt/lists/* - - # Workaround an issue with installing pymssql on ARM architecture triggered by Cython 3.0.0 release as of - # 18 July 2023. The problem is that pip uses latest Cython to compile pymssql and since we are using - # setuptools, there is no easy way to fix version of Cython used to compile packages. - # - # This triggers a problem with newer `pip` versions that have build isolation enabled by default because - # There is no (easy) way to pin build dependencies for dependent packages. If a package does not have - # limit on build dependencies, it will use the latest version of them to build that particular package. - # - # The workaround to the problem suggest in the last thread by Pradyun Gedam - pip maintainer - is to - # use PIP_CONSTRAINT environment variable and constraint the version of Cython used while installing - # the package. Which is precisely what we are doing here. - # - # Note that it does not work if we pass ``--constraint`` option to pip because it will not be passed to - # the package being build in isolation. The fact that the PIP_CONSTRAINT env variable works in the isolation - # is a bit of side-effect on how env variables work and that they are passed to subprocesses as pip - # launches a subprocess `pip` to build the package. - # - # This is a temporary solution until the issue is resolved in pymssql or Cython - # Issues/discussions that track it: - # - # * https://github.com/cython/cython/issues/5541 - # * https://github.com/pymssql/pymssql/pull/827 - # * https://discuss.python.org/t/no-way-to-pin-build-dependencies/29833 - # - # TODO: Remove this workaround when the issue is resolved. - # ALSO REMOVE THE TOP LINES ABOVE WITH common.sh IMPORT AS WELL AS COPYING common.sh ib - # Dockerfile AND Dockerfile.ci (look for capital PYMSSQL - there are several places to remove) - if [[ "${1}" == "dev" ]]; then - common::install_pip_version - echo "Cython==0.29.36" >> /tmp/mssql-constraints.txt - PIP_CONSTRAINT=/tmp/mssql-constraints.txt pip install pymssql - rm /tmp/mssql-constraints.txt - fi } install_mssql_client "${@}" @@ -1272,12 +1236,6 @@ ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \ # scripts which are needed much later will not invalidate the docker layer here COPY --from=scripts install_mysql.sh install_mssql.sh install_postgres.sh /scripts/docker/ -# THE 3 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON -# AND SHOULD BE REMOVED WHEN WORKAROUND IN install_mssql.sh IS REMOVED -ARG AIRFLOW_PIP_VERSION=23.3.1 -ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} -COPY --from=scripts common.sh /scripts/docker/ - RUN bash /scripts/docker/install_mysql.sh dev && \ bash /scripts/docker/install_mssql.sh dev && \ bash /scripts/docker/install_postgres.sh dev diff --git a/Dockerfile.ci b/Dockerfile.ci index c3e5e2e3a0..a22c363fa5 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -307,8 +307,6 @@ COPY <<"EOF" /install_mssql.sh #!/usr/bin/env bash set -euo pipefail -. "$( dirname "${BASH_SOURCE[0]}" )/common.sh" - : "${AIRFLOW_PIP_VERSION:?Should be set}" : "${INSTALL_MSSQL_CLIENT:?Should be true or false}" @@ -344,40 +342,6 @@ function install_mssql_client() { rm -rf /var/lib/apt/lists/* apt-get autoremove -yqq --purge apt-get clean && rm -rf /var/lib/apt/lists/* - - # Workaround an issue with installing pymssql on ARM architecture triggered by Cython 3.0.0 release as of - # 18 July 2023. The problem is that pip uses latest Cython to compile pymssql and since we are using - # setuptools, there is no easy way to fix version of Cython used to compile packages. - # - # This triggers a problem with newer `pip` versions that have build isolation enabled by default because - # There is no (easy) way to pin build dependencies for dependent packages. If a package does not have - # limit on build dependencies, it will use the latest version of them to build that particular package. - # - # The workaround to the problem suggest in the last thread by Pradyun Gedam - pip maintainer - is to - # use PIP_CONSTRAINT environment variable and constraint the version of Cython used while installing - # the package. Which is precisely what we are doing here. - # - # Note that it does not work if we pass ``--constraint`` option to pip because it will not be passed to - # the package being build in isolation. The fact that the PIP_CONSTRAINT env variable works in the isolation - # is a bit of side-effect on how env variables work and that they are passed to subprocesses as pip - # launches a subprocess `pip` to build the package. - # - # This is a temporary solution until the issue is resolved in pymssql or Cython - # Issues/discussions that track it: - # - # * https://github.com/cython/cython/issues/5541 - # * https://github.com/pymssql/pymssql/pull/827 - # * https://discuss.python.org/t/no-way-to-pin-build-dependencies/29833 - # - # TODO: Remove this workaround when the issue is resolved. - # ALSO REMOVE THE TOP LINES ABOVE WITH common.sh IMPORT AS WELL AS COPYING common.sh ib - # Dockerfile AND Dockerfile.ci (look for capital PYMSSQL - there are several places to remove) - if [[ "${1}" == "dev" ]]; then - common::install_pip_version - echo "Cython==0.29.36" >> /tmp/mssql-constraints.txt - PIP_CONSTRAINT=/tmp/mssql-constraints.txt pip install pymssql - rm /tmp/mssql-constraints.txt - fi } install_mssql_client "${@}" diff --git a/airflow/providers/microsoft/mssql/provider.yaml b/airflow/providers/microsoft/mssql/provider.yaml index ce4bb0ceb3..934a809171 100644 --- a/airflow/providers/microsoft/mssql/provider.yaml +++ b/airflow/providers/microsoft/mssql/provider.yaml @@ -48,7 +48,7 @@ versions: dependencies: - apache-airflow>=2.5.0 - apache-airflow-providers-common-sql>=1.3.1 - - pymssql>=2.1.5 + - pymssql>=2.1.8 integrations: - integration-name: Microsoft SQL Server (MSSQL) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 89ae8735bc..0ff1e0b32e 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -600,7 +600,7 @@ "deps": [ "apache-airflow-providers-common-sql>=1.3.1", "apache-airflow>=2.5.0", - "pymssql>=2.1.5" + "pymssql>=2.1.8" ], "cross-providers-deps": [ "common.sql" diff --git a/scripts/docker/install_mssql.sh b/scripts/docker/install_mssql.sh index 248ca68307..b5c74e44bc 100644 --- a/scripts/docker/install_mssql.sh +++ b/scripts/docker/install_mssql.sh @@ -18,11 +18,6 @@ # shellcheck shell=bash set -euo pipefail -# REMOVE THOSE 3 LINES BELOW TOGETHER WITH PYMSSQL INSTALLATION BELOW AFTER PYMSSQL -# INSTALLATION WITH CYTHON 3.0.0 IS FIXED SEE BELOW FOR MORE DETAILS -# shellcheck source=scripts/docker/common.sh -. "$( dirname "${BASH_SOURCE[0]}" )/common.sh" - : "${AIRFLOW_PIP_VERSION:?Should be set}" : "${INSTALL_MSSQL_CLIENT:?Should be true or false}" @@ -58,40 +53,6 @@ function install_mssql_client() { rm -rf /var/lib/apt/lists/* apt-get autoremove -yqq --purge apt-get clean && rm -rf /var/lib/apt/lists/* - - # Workaround an issue with installing pymssql on ARM architecture triggered by Cython 3.0.0 release as of - # 18 July 2023. The problem is that pip uses latest Cython to compile pymssql and since we are using - # setuptools, there is no easy way to fix version of Cython used to compile packages. - # - # This triggers a problem with newer `pip` versions that have build isolation enabled by default because - # There is no (easy) way to pin build dependencies for dependent packages. If a package does not have - # limit on build dependencies, it will use the latest version of them to build that particular package. - # - # The workaround to the problem suggest in the last thread by Pradyun Gedam - pip maintainer - is to - # use PIP_CONSTRAINT environment variable and constraint the version of Cython used while installing - # the package. Which is precisely what we are doing here. - # - # Note that it does not work if we pass ``--constraint`` option to pip because it will not be passed to - # the package being build in isolation. The fact that the PIP_CONSTRAINT env variable works in the isolation - # is a bit of side-effect on how env variables work and that they are passed to subprocesses as pip - # launches a subprocess `pip` to build the package. - # - # This is a temporary solution until the issue is resolved in pymssql or Cython - # Issues/discussions that track it: - # - # * https://github.com/cython/cython/issues/5541 - # * https://github.com/pymssql/pymssql/pull/827 - # * https://discuss.python.org/t/no-way-to-pin-build-dependencies/29833 - # - # TODO: Remove this workaround when the issue is resolved. - # ALSO REMOVE THE TOP LINES ABOVE WITH common.sh IMPORT AS WELL AS COPYING common.sh ib - # Dockerfile AND Dockerfile.ci (look for capital PYMSSQL - there are several places to remove) - if [[ "${1}" == "dev" ]]; then - common::install_pip_version - echo "Cython==0.29.36" >> /tmp/mssql-constraints.txt - PIP_CONSTRAINT=/tmp/mssql-constraints.txt pip install pymssql - rm /tmp/mssql-constraints.txt - fi } install_mssql_client "${@}"
