This is an automated email from the ASF dual-hosted git repository. vterentev pushed a commit to branch release-2.69.0-postrelease-cp-36583 in repository https://gitbox.apache.org/repos/asf/beam.git
commit c7e9df89cbdf72734668308863110f6fa8c0f75d Author: Danny McCormick <[email protected]> AuthorDate: Thu Oct 23 22:52:06 2025 +0400 Cherry-pick #36583 --- sdks/python/container/Dockerfile | 2 +- sdks/python/container/common.gradle | 2 +- .../ml/{py313 => py310}/ml_image_requirements.txt | 135 +++++++++---------- .../ml/{py313 => py311}/ml_image_requirements.txt | 128 +++++++++--------- .../ml/{py313 => py312}/ml_image_requirements.txt | 127 +++++++++--------- .../container/ml/py313/ml_image_requirements.txt | 47 +++---- .../ml/{py313 => py39}/ml_image_requirements.txt | 143 ++++++++++----------- sdks/python/container/run_generate_requirements.sh | 5 +- sdks/python/setup.py | 6 + 9 files changed, 277 insertions(+), 318 deletions(-) diff --git a/sdks/python/container/Dockerfile b/sdks/python/container/Dockerfile index efd5a4a90d8..56df970f75c 100644 --- a/sdks/python/container/Dockerfile +++ b/sdks/python/container/Dockerfile @@ -50,7 +50,7 @@ RUN \ # Install required packages for Beam Python SDK and common dependencies used by users. # use --no-deps to ensure the list includes all transitive dependencies. - pip install --no-deps -r /tmp/base_image_requirements.txt && \ + pip install --no-deps -r /tmp/base_image_requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu && \ rm -rf /tmp/base_image_requirements.txt && \ python -c "import nltk; nltk.download('stopwords')" && \ rm /root/nltk_data/corpora/stopwords.zip && \ diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index 0648bf4fa2e..a67a9f0a7d8 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -53,7 +53,7 @@ def generatePythonRequirements = tasks.register("generatePythonRequirements") { "${files(configurations.sdkSourceTarball.files).singleFile} " + "ml_image_requirements.txt " + "container/ml " + - "[gcp,dataframe,test,tensorflow,torch,transformers] " + + "[gcp,dataframe,test,ml_cpu] " + "${pipExtraOptions}" } } diff --git a/sdks/python/container/ml/py313/ml_image_requirements.txt b/sdks/python/container/ml/py310/ml_image_requirements.txt similarity index 70% copy from sdks/python/container/ml/py313/ml_image_requirements.txt copy to sdks/python/container/ml/py310/ml_image_requirements.txt index ebc712ee6a7..59fe869e02d 100644 --- a/sdks/python/container/ml/py313/ml_image_requirements.txt +++ b/sdks/python/container/ml/py310/ml_image_requirements.txt @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Autogenerated requirements file for Apache Beam py313 container image. +# Autogenerated requirements file for Apache Beam py310 container image. # Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. # Do not edit manually, adjust ../base_image_requirements_manual.txt or # Apache Beam's setup.py instead, and regenerate the list. @@ -22,87 +22,92 @@ # Reach out to a committer if you need help. absl-py==2.3.1 -aiofiles==24.1.0 +aiofiles==25.1.0 aiohappyeyeballs==2.6.1 -aiohttp==3.12.15 +aiohttp==3.13.1 aiosignal==1.4.0 annotated-types==0.7.0 anyio==4.11.0 asn1crypto==1.5.1 astunparse==1.6.3 -attrs==25.3.0 +async-timeout==5.0.1 +attrs==25.4.0 +backports.tarfile==1.2.0 beartype==0.21.0 beautifulsoup4==4.14.2 bs4==0.0.2 build==1.3.0 -cachetools==6.2.0 -certifi==2025.8.3 +cachetools==6.2.1 +certifi==2025.10.5 cffi==2.0.0 -charset-normalizer==3.4.3 +charset-normalizer==3.4.4 click==8.3.0 -cloud-sql-python-connector==1.18.4 +cloud-sql-python-connector==1.18.5 crcmod==1.7 -cryptography==46.0.2 -Cython==3.1.4 +cryptography==46.0.3 +Cython==3.1.5 dill==0.3.1.1 dnspython==2.8.0 docker==7.1.0 docopt==0.6.2 docstring_parser==0.17.0 +exceptiongroup==1.3.0 execnet==2.1.1 -fastavro==1.12.0 +fastavro==1.12.1 fasteners==0.20 -filelock==3.19.1 +filelock==3.20.0 flatbuffers==25.9.23 freezegun==1.5.5 -frozenlist==1.7.0 +frozenlist==1.8.0 fsspec==2025.9.0 future==1.0.0 gast==0.6.0 -google-api-core==2.25.1 -google-apitools==0.5.35 +google-api-core==2.26.0 +google-api-python-client==2.185.0 +google-apitools==0.5.31 google-auth==2.41.1 google-auth-httplib2==0.2.0 -google-cloud-aiplatform==1.118.0 +google-cloud-aiplatform==1.121.0 google-cloud-bigquery==3.38.0 google-cloud-bigquery-storage==2.33.1 -google-cloud-bigtable==2.32.0 +google-cloud-bigtable==2.33.0 google-cloud-core==2.4.3 google-cloud-datastore==2.21.0 -google-cloud-dlp==3.32.0 -google-cloud-language==2.17.2 +google-cloud-dlp==3.33.0 +google-cloud-language==2.18.0 +google-cloud-profiler==4.1.0 google-cloud-pubsub==2.31.1 google-cloud-pubsublite==1.12.0 google-cloud-recommendations-ai==0.10.18 -google-cloud-resource-manager==1.14.2 -google-cloud-secret-manager==2.24.0 +google-cloud-resource-manager==1.15.0 +google-cloud-secret-manager==2.25.0 google-cloud-spanner==3.58.0 google-cloud-storage==2.19.0 -google-cloud-videointelligence==2.16.2 -google-cloud-vision==3.10.2 +google-cloud-videointelligence==2.17.0 +google-cloud-vision==3.11.0 google-crc32c==1.7.1 -google-genai==1.39.1 +google-genai==1.45.0 google-pasta==0.2.0 google-resumable-media==2.7.2 -googleapis-common-protos==1.70.0 +googleapis-common-protos==1.71.0 greenlet==3.2.4 -grpc-google-iam-v1==0.14.2 +grpc-google-iam-v1==0.14.3 grpc-interceptor==0.15.4 -grpcio==1.75.1 -grpcio-status==1.75.1 +grpcio==1.65.5 +grpcio-status==1.65.5 guppy3==3.1.5 h11==0.16.0 -h5py==3.14.0 +h5py==3.15.1 hdfs==2.7.3 hf-xet==1.1.10 httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 huggingface-hub==0.35.3 -hypothesis==6.140.2 -idna==3.10 +hypothesis==6.142.2 +idna==3.11 importlib_metadata==8.7.0 -iniconfig==2.1.0 +iniconfig==2.3.0 jaraco.classes==3.4.0 jaraco.context==6.0.1 jaraco.functools==4.3.0 @@ -120,64 +125,51 @@ Markdown==3.9 markdown-it-py==4.0.0 MarkupSafe==3.0.3 mdurl==0.1.2 +milvus-lite==2.5.1 ml_dtypes==0.5.3 mmh3==5.2.0 mock==5.2.0 more-itertools==10.8.0 mpmath==1.3.0 -multidict==6.6.4 +multidict==6.7.0 namex==0.1.0 -networkx==3.5 +networkx==3.4.2 nltk==3.9.2 numpy==2.2.6 -nvidia-cublas-cu12==12.6.4.1 -nvidia-cuda-cupti-cu12==12.6.80 -nvidia-cuda-nvrtc-cu12==12.6.77 -nvidia-cuda-runtime-cu12==12.6.77 -nvidia-cudnn-cu12==9.5.1.17 -nvidia-cufft-cu12==11.3.0.4 -nvidia-cufile-cu12==1.11.1.6 -nvidia-curand-cu12==10.3.7.77 -nvidia-cusolver-cu12==11.7.1.2 -nvidia-cusparse-cu12==12.5.4.2 -nvidia-cusparselt-cu12==0.6.3 -nvidia-nccl-cu12==2.26.2 -nvidia-nvjitlink-cu12==12.6.85 -nvidia-nvtx-cu12==12.6.77 oauth2client==4.1.3 objsize==0.7.1 -opentelemetry-api==1.37.0 -opentelemetry-sdk==1.37.0 -opentelemetry-semantic-conventions==0.58b0 +opentelemetry-api==1.38.0 +opentelemetry-sdk==1.38.0 +opentelemetry-semantic-conventions==0.59b0 opt_einsum==3.4.0 optree==0.17.0 -oracledb==3.3.0 +oracledb==3.4.0 orjson==3.11.3 overrides==7.7.0 packaging==25.0 -pandas==2.3.3 +pandas==2.2.3 parameterized==0.9.0 pg8000==1.31.5 -pillow==11.3.0 +pillow==12.0.0 pip==25.2 pluggy==1.6.0 -propcache==0.3.2 +propcache==0.4.1 proto-plus==1.26.1 -protobuf==6.33.0rc1 -psycopg2-binary==2.9.10 +protobuf==5.29.5 +psycopg2-binary==2.9.11 pyarrow==18.1.0 pyarrow-hotfix==0.7 pyasn1==0.6.1 pyasn1_modules==0.4.2 pycparser==2.23 -pydantic==2.11.9 -pydantic_core==2.33.2 +pydantic==2.12.3 +pydantic_core==2.41.4 pydot==1.4.2 Pygments==2.19.2 PyHamcrest==2.1.0 PyJWT==2.10.1 -pymilvus==2.6.2 -pymongo==4.15.1 +pymilvus==2.5.16 +pymongo==4.15.3 PyMySQL==1.1.2 pyparsing==3.2.5 pyproject_hooks==1.2.0 @@ -190,16 +182,16 @@ python-tds==1.17.1 pytz==2025.2 PyYAML==6.0.3 redis==5.3.1 -referencing==0.36.2 -regex==2025.9.18 +referencing==0.37.0 +regex==2025.10.23 requests==2.32.5 requests-mock==1.12.1 -rich==14.1.0 +rich==14.2.0 rpds-py==0.27.1 rsa==4.9.1 safetensors==0.6.2 scikit-learn==1.7.2 -scipy==1.16.2 +scipy==1.15.3 scramp==1.4.6 SecretStorage==3.4.0 setuptools==80.9.0 @@ -208,7 +200,7 @@ six==1.17.0 sniffio==1.3.1 sortedcontainers==2.4.0 soupsieve==2.8 -SQLAlchemy==2.0.43 +SQLAlchemy==2.0.44 sqlalchemy_pytds==1.0.2 sqlparse==0.5.3 sympy==1.14.0 @@ -218,23 +210,24 @@ tensorboard-data-server==0.7.2 tensorflow==2.20.0 tensorflow-cpu-aws==2.20.0;platform_machine=="aarch64" termcolor==3.1.0 -testcontainers==4.13.1 +testcontainers==4.13.2 threadpoolctl==3.6.0 tokenizers==0.21.4 -torch==2.7.1 +tomli==2.3.0 +torch==2.8.0+cpu tqdm==4.67.1 transformers==4.55.4 -triton==3.3.1 typing-inspection==0.4.2 typing_extensions==4.15.0 tzdata==2025.2 ujson==5.11.0 +uritemplate==4.2.0 urllib3==2.5.0 virtualenv-clone==0.5.7 websockets==15.0.1 Werkzeug==3.1.3 wheel==0.45.1 -wrapt==1.17.3 -yarl==1.20.1 +wrapt==2.0.0 +yarl==1.22.0 zipp==3.23.0 zstandard==0.25.0 diff --git a/sdks/python/container/ml/py313/ml_image_requirements.txt b/sdks/python/container/ml/py311/ml_image_requirements.txt similarity index 71% copy from sdks/python/container/ml/py313/ml_image_requirements.txt copy to sdks/python/container/ml/py311/ml_image_requirements.txt index ebc712ee6a7..351eac254c0 100644 --- a/sdks/python/container/ml/py313/ml_image_requirements.txt +++ b/sdks/python/container/ml/py311/ml_image_requirements.txt @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Autogenerated requirements file for Apache Beam py313 container image. +# Autogenerated requirements file for Apache Beam py311 container image. # Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. # Do not edit manually, adjust ../base_image_requirements_manual.txt or # Apache Beam's setup.py instead, and regenerate the list. @@ -22,87 +22,90 @@ # Reach out to a committer if you need help. absl-py==2.3.1 -aiofiles==24.1.0 +aiofiles==25.1.0 aiohappyeyeballs==2.6.1 -aiohttp==3.12.15 +aiohttp==3.13.1 aiosignal==1.4.0 annotated-types==0.7.0 anyio==4.11.0 asn1crypto==1.5.1 astunparse==1.6.3 -attrs==25.3.0 +attrs==25.4.0 +backports.tarfile==1.2.0 beartype==0.21.0 beautifulsoup4==4.14.2 bs4==0.0.2 build==1.3.0 -cachetools==6.2.0 -certifi==2025.8.3 +cachetools==6.2.1 +certifi==2025.10.5 cffi==2.0.0 -charset-normalizer==3.4.3 +charset-normalizer==3.4.4 click==8.3.0 -cloud-sql-python-connector==1.18.4 +cloud-sql-python-connector==1.18.5 crcmod==1.7 -cryptography==46.0.2 -Cython==3.1.4 +cryptography==46.0.3 +Cython==3.1.5 dill==0.3.1.1 dnspython==2.8.0 docker==7.1.0 docopt==0.6.2 docstring_parser==0.17.0 execnet==2.1.1 -fastavro==1.12.0 +fastavro==1.12.1 fasteners==0.20 -filelock==3.19.1 +filelock==3.20.0 flatbuffers==25.9.23 freezegun==1.5.5 -frozenlist==1.7.0 +frozenlist==1.8.0 fsspec==2025.9.0 future==1.0.0 gast==0.6.0 -google-api-core==2.25.1 -google-apitools==0.5.35 +google-api-core==2.26.0 +google-api-python-client==2.185.0 +google-apitools==0.5.31 google-auth==2.41.1 google-auth-httplib2==0.2.0 -google-cloud-aiplatform==1.118.0 +google-cloud-aiplatform==1.121.0 google-cloud-bigquery==3.38.0 google-cloud-bigquery-storage==2.33.1 -google-cloud-bigtable==2.32.0 +google-cloud-bigtable==2.33.0 google-cloud-core==2.4.3 google-cloud-datastore==2.21.0 -google-cloud-dlp==3.32.0 -google-cloud-language==2.17.2 +google-cloud-dlp==3.33.0 +google-cloud-language==2.18.0 +google-cloud-profiler==4.1.0 google-cloud-pubsub==2.31.1 google-cloud-pubsublite==1.12.0 google-cloud-recommendations-ai==0.10.18 -google-cloud-resource-manager==1.14.2 -google-cloud-secret-manager==2.24.0 +google-cloud-resource-manager==1.15.0 +google-cloud-secret-manager==2.25.0 google-cloud-spanner==3.58.0 google-cloud-storage==2.19.0 -google-cloud-videointelligence==2.16.2 -google-cloud-vision==3.10.2 +google-cloud-videointelligence==2.17.0 +google-cloud-vision==3.11.0 google-crc32c==1.7.1 -google-genai==1.39.1 +google-genai==1.45.0 google-pasta==0.2.0 google-resumable-media==2.7.2 -googleapis-common-protos==1.70.0 +googleapis-common-protos==1.71.0 greenlet==3.2.4 -grpc-google-iam-v1==0.14.2 +grpc-google-iam-v1==0.14.3 grpc-interceptor==0.15.4 -grpcio==1.75.1 -grpcio-status==1.75.1 +grpcio==1.65.5 +grpcio-status==1.65.5 guppy3==3.1.5 h11==0.16.0 -h5py==3.14.0 +h5py==3.15.1 hdfs==2.7.3 hf-xet==1.1.10 httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 huggingface-hub==0.35.3 -hypothesis==6.140.2 -idna==3.10 +hypothesis==6.142.2 +idna==3.11 importlib_metadata==8.7.0 -iniconfig==2.1.0 +iniconfig==2.3.0 jaraco.classes==3.4.0 jaraco.context==6.0.1 jaraco.functools==4.3.0 @@ -120,64 +123,51 @@ Markdown==3.9 markdown-it-py==4.0.0 MarkupSafe==3.0.3 mdurl==0.1.2 +milvus-lite==2.5.1 ml_dtypes==0.5.3 mmh3==5.2.0 mock==5.2.0 more-itertools==10.8.0 mpmath==1.3.0 -multidict==6.6.4 +multidict==6.7.0 namex==0.1.0 networkx==3.5 nltk==3.9.2 numpy==2.2.6 -nvidia-cublas-cu12==12.6.4.1 -nvidia-cuda-cupti-cu12==12.6.80 -nvidia-cuda-nvrtc-cu12==12.6.77 -nvidia-cuda-runtime-cu12==12.6.77 -nvidia-cudnn-cu12==9.5.1.17 -nvidia-cufft-cu12==11.3.0.4 -nvidia-cufile-cu12==1.11.1.6 -nvidia-curand-cu12==10.3.7.77 -nvidia-cusolver-cu12==11.7.1.2 -nvidia-cusparse-cu12==12.5.4.2 -nvidia-cusparselt-cu12==0.6.3 -nvidia-nccl-cu12==2.26.2 -nvidia-nvjitlink-cu12==12.6.85 -nvidia-nvtx-cu12==12.6.77 oauth2client==4.1.3 objsize==0.7.1 -opentelemetry-api==1.37.0 -opentelemetry-sdk==1.37.0 -opentelemetry-semantic-conventions==0.58b0 +opentelemetry-api==1.38.0 +opentelemetry-sdk==1.38.0 +opentelemetry-semantic-conventions==0.59b0 opt_einsum==3.4.0 optree==0.17.0 -oracledb==3.3.0 +oracledb==3.4.0 orjson==3.11.3 overrides==7.7.0 packaging==25.0 -pandas==2.3.3 +pandas==2.2.3 parameterized==0.9.0 pg8000==1.31.5 -pillow==11.3.0 +pillow==12.0.0 pip==25.2 pluggy==1.6.0 -propcache==0.3.2 +propcache==0.4.1 proto-plus==1.26.1 -protobuf==6.33.0rc1 -psycopg2-binary==2.9.10 +protobuf==5.29.5 +psycopg2-binary==2.9.11 pyarrow==18.1.0 pyarrow-hotfix==0.7 pyasn1==0.6.1 pyasn1_modules==0.4.2 pycparser==2.23 -pydantic==2.11.9 -pydantic_core==2.33.2 +pydantic==2.12.3 +pydantic_core==2.41.4 pydot==1.4.2 Pygments==2.19.2 PyHamcrest==2.1.0 PyJWT==2.10.1 -pymilvus==2.6.2 -pymongo==4.15.1 +pymilvus==2.5.16 +pymongo==4.15.3 PyMySQL==1.1.2 pyparsing==3.2.5 pyproject_hooks==1.2.0 @@ -190,11 +180,11 @@ python-tds==1.17.1 pytz==2025.2 PyYAML==6.0.3 redis==5.3.1 -referencing==0.36.2 -regex==2025.9.18 +referencing==0.37.0 +regex==2025.10.23 requests==2.32.5 requests-mock==1.12.1 -rich==14.1.0 +rich==14.2.0 rpds-py==0.27.1 rsa==4.9.1 safetensors==0.6.2 @@ -208,7 +198,7 @@ six==1.17.0 sniffio==1.3.1 sortedcontainers==2.4.0 soupsieve==2.8 -SQLAlchemy==2.0.43 +SQLAlchemy==2.0.44 sqlalchemy_pytds==1.0.2 sqlparse==0.5.3 sympy==1.14.0 @@ -218,23 +208,23 @@ tensorboard-data-server==0.7.2 tensorflow==2.20.0 tensorflow-cpu-aws==2.20.0;platform_machine=="aarch64" termcolor==3.1.0 -testcontainers==4.13.1 +testcontainers==4.13.2 threadpoolctl==3.6.0 tokenizers==0.21.4 -torch==2.7.1 +torch==2.8.0+cpu tqdm==4.67.1 transformers==4.55.4 -triton==3.3.1 typing-inspection==0.4.2 typing_extensions==4.15.0 tzdata==2025.2 ujson==5.11.0 +uritemplate==4.2.0 urllib3==2.5.0 virtualenv-clone==0.5.7 websockets==15.0.1 Werkzeug==3.1.3 wheel==0.45.1 -wrapt==1.17.3 -yarl==1.20.1 +wrapt==2.0.0 +yarl==1.22.0 zipp==3.23.0 zstandard==0.25.0 diff --git a/sdks/python/container/ml/py313/ml_image_requirements.txt b/sdks/python/container/ml/py312/ml_image_requirements.txt similarity index 71% copy from sdks/python/container/ml/py313/ml_image_requirements.txt copy to sdks/python/container/ml/py312/ml_image_requirements.txt index ebc712ee6a7..e4b64d509dd 100644 --- a/sdks/python/container/ml/py313/ml_image_requirements.txt +++ b/sdks/python/container/ml/py312/ml_image_requirements.txt @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Autogenerated requirements file for Apache Beam py313 container image. +# Autogenerated requirements file for Apache Beam py312 container image. # Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. # Do not edit manually, adjust ../base_image_requirements_manual.txt or # Apache Beam's setup.py instead, and regenerate the list. @@ -22,87 +22,89 @@ # Reach out to a committer if you need help. absl-py==2.3.1 -aiofiles==24.1.0 +aiofiles==25.1.0 aiohappyeyeballs==2.6.1 -aiohttp==3.12.15 +aiohttp==3.13.1 aiosignal==1.4.0 annotated-types==0.7.0 anyio==4.11.0 asn1crypto==1.5.1 astunparse==1.6.3 -attrs==25.3.0 +attrs==25.4.0 beartype==0.21.0 beautifulsoup4==4.14.2 bs4==0.0.2 build==1.3.0 -cachetools==6.2.0 -certifi==2025.8.3 +cachetools==6.2.1 +certifi==2025.10.5 cffi==2.0.0 -charset-normalizer==3.4.3 +charset-normalizer==3.4.4 click==8.3.0 -cloud-sql-python-connector==1.18.4 +cloud-sql-python-connector==1.18.5 crcmod==1.7 -cryptography==46.0.2 -Cython==3.1.4 +cryptography==46.0.3 +Cython==3.1.5 dill==0.3.1.1 dnspython==2.8.0 docker==7.1.0 docopt==0.6.2 docstring_parser==0.17.0 execnet==2.1.1 -fastavro==1.12.0 +fastavro==1.12.1 fasteners==0.20 -filelock==3.19.1 +filelock==3.20.0 flatbuffers==25.9.23 freezegun==1.5.5 -frozenlist==1.7.0 +frozenlist==1.8.0 fsspec==2025.9.0 future==1.0.0 gast==0.6.0 -google-api-core==2.25.1 -google-apitools==0.5.35 +google-api-core==2.26.0 +google-api-python-client==2.185.0 +google-apitools==0.5.31 google-auth==2.41.1 google-auth-httplib2==0.2.0 -google-cloud-aiplatform==1.118.0 +google-cloud-aiplatform==1.121.0 google-cloud-bigquery==3.38.0 google-cloud-bigquery-storage==2.33.1 -google-cloud-bigtable==2.32.0 +google-cloud-bigtable==2.33.0 google-cloud-core==2.4.3 google-cloud-datastore==2.21.0 -google-cloud-dlp==3.32.0 -google-cloud-language==2.17.2 +google-cloud-dlp==3.33.0 +google-cloud-language==2.18.0 +google-cloud-profiler==4.1.0 google-cloud-pubsub==2.31.1 google-cloud-pubsublite==1.12.0 google-cloud-recommendations-ai==0.10.18 -google-cloud-resource-manager==1.14.2 -google-cloud-secret-manager==2.24.0 +google-cloud-resource-manager==1.15.0 +google-cloud-secret-manager==2.25.0 google-cloud-spanner==3.58.0 google-cloud-storage==2.19.0 -google-cloud-videointelligence==2.16.2 -google-cloud-vision==3.10.2 +google-cloud-videointelligence==2.17.0 +google-cloud-vision==3.11.0 google-crc32c==1.7.1 -google-genai==1.39.1 +google-genai==1.45.0 google-pasta==0.2.0 google-resumable-media==2.7.2 -googleapis-common-protos==1.70.0 +googleapis-common-protos==1.71.0 greenlet==3.2.4 -grpc-google-iam-v1==0.14.2 +grpc-google-iam-v1==0.14.3 grpc-interceptor==0.15.4 -grpcio==1.75.1 -grpcio-status==1.75.1 +grpcio==1.65.5 +grpcio-status==1.65.5 guppy3==3.1.5 h11==0.16.0 -h5py==3.14.0 +h5py==3.15.1 hdfs==2.7.3 hf-xet==1.1.10 httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 huggingface-hub==0.35.3 -hypothesis==6.140.2 -idna==3.10 +hypothesis==6.142.2 +idna==3.11 importlib_metadata==8.7.0 -iniconfig==2.1.0 +iniconfig==2.3.0 jaraco.classes==3.4.0 jaraco.context==6.0.1 jaraco.functools==4.3.0 @@ -120,64 +122,51 @@ Markdown==3.9 markdown-it-py==4.0.0 MarkupSafe==3.0.3 mdurl==0.1.2 +milvus-lite==2.5.1 ml_dtypes==0.5.3 mmh3==5.2.0 mock==5.2.0 more-itertools==10.8.0 mpmath==1.3.0 -multidict==6.6.4 +multidict==6.7.0 namex==0.1.0 networkx==3.5 nltk==3.9.2 numpy==2.2.6 -nvidia-cublas-cu12==12.6.4.1 -nvidia-cuda-cupti-cu12==12.6.80 -nvidia-cuda-nvrtc-cu12==12.6.77 -nvidia-cuda-runtime-cu12==12.6.77 -nvidia-cudnn-cu12==9.5.1.17 -nvidia-cufft-cu12==11.3.0.4 -nvidia-cufile-cu12==1.11.1.6 -nvidia-curand-cu12==10.3.7.77 -nvidia-cusolver-cu12==11.7.1.2 -nvidia-cusparse-cu12==12.5.4.2 -nvidia-cusparselt-cu12==0.6.3 -nvidia-nccl-cu12==2.26.2 -nvidia-nvjitlink-cu12==12.6.85 -nvidia-nvtx-cu12==12.6.77 oauth2client==4.1.3 objsize==0.7.1 -opentelemetry-api==1.37.0 -opentelemetry-sdk==1.37.0 -opentelemetry-semantic-conventions==0.58b0 +opentelemetry-api==1.38.0 +opentelemetry-sdk==1.38.0 +opentelemetry-semantic-conventions==0.59b0 opt_einsum==3.4.0 optree==0.17.0 -oracledb==3.3.0 +oracledb==3.4.0 orjson==3.11.3 overrides==7.7.0 packaging==25.0 -pandas==2.3.3 +pandas==2.2.3 parameterized==0.9.0 pg8000==1.31.5 -pillow==11.3.0 +pillow==12.0.0 pip==25.2 pluggy==1.6.0 -propcache==0.3.2 +propcache==0.4.1 proto-plus==1.26.1 -protobuf==6.33.0rc1 -psycopg2-binary==2.9.10 +protobuf==5.29.5 +psycopg2-binary==2.9.11 pyarrow==18.1.0 pyarrow-hotfix==0.7 pyasn1==0.6.1 pyasn1_modules==0.4.2 pycparser==2.23 -pydantic==2.11.9 -pydantic_core==2.33.2 +pydantic==2.12.3 +pydantic_core==2.41.4 pydot==1.4.2 Pygments==2.19.2 PyHamcrest==2.1.0 PyJWT==2.10.1 -pymilvus==2.6.2 -pymongo==4.15.1 +pymilvus==2.5.16 +pymongo==4.15.3 PyMySQL==1.1.2 pyparsing==3.2.5 pyproject_hooks==1.2.0 @@ -190,11 +179,11 @@ python-tds==1.17.1 pytz==2025.2 PyYAML==6.0.3 redis==5.3.1 -referencing==0.36.2 -regex==2025.9.18 +referencing==0.37.0 +regex==2025.10.23 requests==2.32.5 requests-mock==1.12.1 -rich==14.1.0 +rich==14.2.0 rpds-py==0.27.1 rsa==4.9.1 safetensors==0.6.2 @@ -208,7 +197,7 @@ six==1.17.0 sniffio==1.3.1 sortedcontainers==2.4.0 soupsieve==2.8 -SQLAlchemy==2.0.43 +SQLAlchemy==2.0.44 sqlalchemy_pytds==1.0.2 sqlparse==0.5.3 sympy==1.14.0 @@ -218,23 +207,23 @@ tensorboard-data-server==0.7.2 tensorflow==2.20.0 tensorflow-cpu-aws==2.20.0;platform_machine=="aarch64" termcolor==3.1.0 -testcontainers==4.13.1 +testcontainers==4.13.2 threadpoolctl==3.6.0 tokenizers==0.21.4 -torch==2.7.1 +torch==2.8.0+cpu tqdm==4.67.1 transformers==4.55.4 -triton==3.3.1 typing-inspection==0.4.2 typing_extensions==4.15.0 tzdata==2025.2 ujson==5.11.0 +uritemplate==4.2.0 urllib3==2.5.0 virtualenv-clone==0.5.7 websockets==15.0.1 Werkzeug==3.1.3 wheel==0.45.1 -wrapt==1.17.3 -yarl==1.20.1 +wrapt==2.0.0 +yarl==1.22.0 zipp==3.23.0 zstandard==0.25.0 diff --git a/sdks/python/container/ml/py313/ml_image_requirements.txt b/sdks/python/container/ml/py313/ml_image_requirements.txt index ebc712ee6a7..a974d860698 100644 --- a/sdks/python/container/ml/py313/ml_image_requirements.txt +++ b/sdks/python/container/ml/py313/ml_image_requirements.txt @@ -42,8 +42,8 @@ charset-normalizer==3.4.3 click==8.3.0 cloud-sql-python-connector==1.18.4 crcmod==1.7 -cryptography==46.0.2 -Cython==3.1.4 +cryptography==46.0.3 +Cython==3.1.5 dill==0.3.1.1 dnspython==2.8.0 docker==7.1.0 @@ -74,22 +74,22 @@ google-cloud-language==2.17.2 google-cloud-pubsub==2.31.1 google-cloud-pubsublite==1.12.0 google-cloud-recommendations-ai==0.10.18 -google-cloud-resource-manager==1.14.2 -google-cloud-secret-manager==2.24.0 +google-cloud-resource-manager==1.15.0 +google-cloud-secret-manager==2.25.0 google-cloud-spanner==3.58.0 google-cloud-storage==2.19.0 -google-cloud-videointelligence==2.16.2 -google-cloud-vision==3.10.2 +google-cloud-videointelligence==2.17.0 +google-cloud-vision==3.11.0 google-crc32c==1.7.1 google-genai==1.39.1 google-pasta==0.2.0 google-resumable-media==2.7.2 -googleapis-common-protos==1.70.0 +googleapis-common-protos==1.71.0 greenlet==3.2.4 grpc-google-iam-v1==0.14.2 grpc-interceptor==0.15.4 -grpcio==1.75.1 -grpcio-status==1.75.1 +grpcio==1.76.0 +grpcio-status==1.76.0 guppy3==3.1.5 h11==0.16.0 h5py==3.14.0 @@ -99,8 +99,8 @@ httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 huggingface-hub==0.35.3 -hypothesis==6.140.2 -idna==3.10 +hypothesis==6.142.2 +idna==3.11 importlib_metadata==8.7.0 iniconfig==2.1.0 jaraco.classes==3.4.0 @@ -130,20 +130,6 @@ namex==0.1.0 networkx==3.5 nltk==3.9.2 numpy==2.2.6 -nvidia-cublas-cu12==12.6.4.1 -nvidia-cuda-cupti-cu12==12.6.80 -nvidia-cuda-nvrtc-cu12==12.6.77 -nvidia-cuda-runtime-cu12==12.6.77 -nvidia-cudnn-cu12==9.5.1.17 -nvidia-cufft-cu12==11.3.0.4 -nvidia-cufile-cu12==1.11.1.6 -nvidia-curand-cu12==10.3.7.77 -nvidia-cusolver-cu12==11.7.1.2 -nvidia-cusparse-cu12==12.5.4.2 -nvidia-cusparselt-cu12==0.6.3 -nvidia-nccl-cu12==2.26.2 -nvidia-nvjitlink-cu12==12.6.85 -nvidia-nvtx-cu12==12.6.77 oauth2client==4.1.3 objsize==0.7.1 opentelemetry-api==1.37.0 @@ -190,8 +176,8 @@ python-tds==1.17.1 pytz==2025.2 PyYAML==6.0.3 redis==5.3.1 -referencing==0.36.2 -regex==2025.9.18 +referencing==0.37.0 +regex==2025.10.23 requests==2.32.5 requests-mock==1.12.1 rich==14.1.0 @@ -221,10 +207,9 @@ termcolor==3.1.0 testcontainers==4.13.1 threadpoolctl==3.6.0 tokenizers==0.21.4 -torch==2.7.1 +torch==2.8.0+cpu tqdm==4.67.1 transformers==4.55.4 -triton==3.3.1 typing-inspection==0.4.2 typing_extensions==4.15.0 tzdata==2025.2 @@ -234,7 +219,7 @@ virtualenv-clone==0.5.7 websockets==15.0.1 Werkzeug==3.1.3 wheel==0.45.1 -wrapt==1.17.3 -yarl==1.20.1 +wrapt==2.0.0 +yarl==1.22.0 zipp==3.23.0 zstandard==0.25.0 diff --git a/sdks/python/container/ml/py313/ml_image_requirements.txt b/sdks/python/container/ml/py39/ml_image_requirements.txt similarity index 69% copy from sdks/python/container/ml/py313/ml_image_requirements.txt copy to sdks/python/container/ml/py39/ml_image_requirements.txt index ebc712ee6a7..8e92499a3b1 100644 --- a/sdks/python/container/ml/py313/ml_image_requirements.txt +++ b/sdks/python/container/ml/py39/ml_image_requirements.txt @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Autogenerated requirements file for Apache Beam py313 container image. +# Autogenerated requirements file for Apache Beam py39 container image. # Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. # Do not edit manually, adjust ../base_image_requirements_manual.txt or # Apache Beam's setup.py instead, and regenerate the list. @@ -22,74 +22,79 @@ # Reach out to a committer if you need help. absl-py==2.3.1 -aiofiles==24.1.0 +aiofiles==25.1.0 aiohappyeyeballs==2.6.1 -aiohttp==3.12.15 +aiohttp==3.13.1 aiosignal==1.4.0 annotated-types==0.7.0 anyio==4.11.0 asn1crypto==1.5.1 astunparse==1.6.3 -attrs==25.3.0 +async-timeout==5.0.1 +attrs==25.4.0 +backports.tarfile==1.2.0 beartype==0.21.0 beautifulsoup4==4.14.2 bs4==0.0.2 build==1.3.0 -cachetools==6.2.0 -certifi==2025.8.3 +cachetools==6.2.1 +certifi==2025.10.5 cffi==2.0.0 -charset-normalizer==3.4.3 -click==8.3.0 -cloud-sql-python-connector==1.18.4 +charset-normalizer==3.4.4 +click==8.1.8 +cloud-sql-python-connector==1.18.5 crcmod==1.7 -cryptography==46.0.2 -Cython==3.1.4 +cryptography==46.0.3 +Cython==3.1.5 dill==0.3.1.1 -dnspython==2.8.0 +dnspython==2.7.0 docker==7.1.0 docopt==0.6.2 docstring_parser==0.17.0 +exceptiongroup==1.3.0 execnet==2.1.1 -fastavro==1.12.0 +fastavro==1.12.1 fasteners==0.20 filelock==3.19.1 flatbuffers==25.9.23 freezegun==1.5.5 -frozenlist==1.7.0 +frozenlist==1.8.0 fsspec==2025.9.0 future==1.0.0 gast==0.6.0 -google-api-core==2.25.1 -google-apitools==0.5.35 +google-api-core==2.26.0 +google-api-python-client==2.185.0 +google-apitools==0.5.31 google-auth==2.41.1 google-auth-httplib2==0.2.0 -google-cloud-aiplatform==1.118.0 +google-cloud-aiplatform==1.121.0 google-cloud-bigquery==3.38.0 google-cloud-bigquery-storage==2.33.1 -google-cloud-bigtable==2.32.0 +google-cloud-bigtable==2.33.0 google-cloud-core==2.4.3 google-cloud-datastore==2.21.0 -google-cloud-dlp==3.32.0 -google-cloud-language==2.17.2 +google-cloud-dlp==3.33.0 +google-cloud-language==2.18.0 +google-cloud-profiler==4.1.0 google-cloud-pubsub==2.31.1 google-cloud-pubsublite==1.12.0 google-cloud-recommendations-ai==0.10.18 -google-cloud-resource-manager==1.14.2 -google-cloud-secret-manager==2.24.0 +google-cloud-resource-manager==1.15.0 +google-cloud-secret-manager==2.25.0 google-cloud-spanner==3.58.0 google-cloud-storage==2.19.0 -google-cloud-videointelligence==2.16.2 -google-cloud-vision==3.10.2 +google-cloud-videointelligence==2.17.0 +google-cloud-vision==3.11.0 google-crc32c==1.7.1 -google-genai==1.39.1 +google-genai==1.45.0 google-pasta==0.2.0 google-resumable-media==2.7.2 -googleapis-common-protos==1.70.0 +googleapis-common-protos==1.71.0 greenlet==3.2.4 -grpc-google-iam-v1==0.14.2 +grpc-google-iam-v1==0.14.3 grpc-interceptor==0.15.4 -grpcio==1.75.1 -grpcio-status==1.75.1 +grpcio==1.65.5 +grpcio-status==1.65.5 guppy3==3.1.5 h11==0.16.0 h5py==3.14.0 @@ -99,8 +104,8 @@ httpcore==1.0.9 httplib2==0.22.0 httpx==0.28.1 huggingface-hub==0.35.3 -hypothesis==6.140.2 -idna==3.10 +hypothesis==6.141.1 +idna==3.11 importlib_metadata==8.7.0 iniconfig==2.1.0 jaraco.classes==3.4.0 @@ -112,72 +117,59 @@ joblib==1.5.2 jsonpickle==3.4.2 jsonschema==4.25.1 jsonschema-specifications==2025.9.1 -keras==3.11.3 +keras==3.10.0 keyring==25.6.0 keyrings.google-artifactregistry-auth==1.1.2 libclang==18.1.1 Markdown==3.9 -markdown-it-py==4.0.0 +markdown-it-py==3.0.0 MarkupSafe==3.0.3 mdurl==0.1.2 +milvus-lite==2.5.1 ml_dtypes==0.5.3 mmh3==5.2.0 mock==5.2.0 more-itertools==10.8.0 mpmath==1.3.0 -multidict==6.6.4 +multidict==6.7.0 namex==0.1.0 -networkx==3.5 +networkx==3.2.1 nltk==3.9.2 -numpy==2.2.6 -nvidia-cublas-cu12==12.6.4.1 -nvidia-cuda-cupti-cu12==12.6.80 -nvidia-cuda-nvrtc-cu12==12.6.77 -nvidia-cuda-runtime-cu12==12.6.77 -nvidia-cudnn-cu12==9.5.1.17 -nvidia-cufft-cu12==11.3.0.4 -nvidia-cufile-cu12==1.11.1.6 -nvidia-curand-cu12==10.3.7.77 -nvidia-cusolver-cu12==11.7.1.2 -nvidia-cusparse-cu12==12.5.4.2 -nvidia-cusparselt-cu12==0.6.3 -nvidia-nccl-cu12==2.26.2 -nvidia-nvjitlink-cu12==12.6.85 -nvidia-nvtx-cu12==12.6.77 +numpy==2.0.2 oauth2client==4.1.3 objsize==0.7.1 -opentelemetry-api==1.37.0 -opentelemetry-sdk==1.37.0 -opentelemetry-semantic-conventions==0.58b0 +opentelemetry-api==1.38.0 +opentelemetry-sdk==1.38.0 +opentelemetry-semantic-conventions==0.59b0 opt_einsum==3.4.0 optree==0.17.0 -oracledb==3.3.0 +oracledb==3.4.0 orjson==3.11.3 overrides==7.7.0 packaging==25.0 -pandas==2.3.3 +pandas==2.2.3 parameterized==0.9.0 pg8000==1.31.5 pillow==11.3.0 pip==25.2 pluggy==1.6.0 -propcache==0.3.2 +propcache==0.4.1 proto-plus==1.26.1 -protobuf==6.33.0rc1 -psycopg2-binary==2.9.10 +protobuf==5.29.5 +psycopg2-binary==2.9.9 pyarrow==18.1.0 pyarrow-hotfix==0.7 pyasn1==0.6.1 pyasn1_modules==0.4.2 pycparser==2.23 -pydantic==2.11.9 -pydantic_core==2.33.2 +pydantic==2.12.3 +pydantic_core==2.41.4 pydot==1.4.2 Pygments==2.19.2 PyHamcrest==2.1.0 PyJWT==2.10.1 -pymilvus==2.6.2 -pymongo==4.15.1 +pymilvus==2.5.16 +pymongo==4.15.3 PyMySQL==1.1.2 pyparsing==3.2.5 pyproject_hooks==1.2.0 @@ -191,24 +183,24 @@ pytz==2025.2 PyYAML==6.0.3 redis==5.3.1 referencing==0.36.2 -regex==2025.9.18 +regex==2025.10.23 requests==2.32.5 requests-mock==1.12.1 -rich==14.1.0 +rich==14.2.0 rpds-py==0.27.1 rsa==4.9.1 safetensors==0.6.2 -scikit-learn==1.7.2 -scipy==1.16.2 +scikit-learn==1.6.1 +scipy==1.13.1 scramp==1.4.6 -SecretStorage==3.4.0 +SecretStorage==3.3.3 setuptools==80.9.0 -shapely==2.1.2 +shapely==2.0.7 six==1.17.0 sniffio==1.3.1 sortedcontainers==2.4.0 soupsieve==2.8 -SQLAlchemy==2.0.43 +SQLAlchemy==2.0.44 sqlalchemy_pytds==1.0.2 sqlparse==0.5.3 sympy==1.14.0 @@ -218,23 +210,24 @@ tensorboard-data-server==0.7.2 tensorflow==2.20.0 tensorflow-cpu-aws==2.20.0;platform_machine=="aarch64" termcolor==3.1.0 -testcontainers==4.13.1 +testcontainers==4.13.2 threadpoolctl==3.6.0 tokenizers==0.21.4 -torch==2.7.1 +tomli==2.3.0 +torch==2.8.0+cpu tqdm==4.67.1 -transformers==4.55.4 -triton==3.3.1 +transformers==4.54.1 typing-inspection==0.4.2 typing_extensions==4.15.0 tzdata==2025.2 ujson==5.11.0 +uritemplate==4.2.0 urllib3==2.5.0 virtualenv-clone==0.5.7 websockets==15.0.1 Werkzeug==3.1.3 wheel==0.45.1 -wrapt==1.17.3 -yarl==1.20.1 +wrapt==2.0.0 +yarl==1.22.0 zipp==3.23.0 zstandard==0.25.0 diff --git a/sdks/python/container/run_generate_requirements.sh b/sdks/python/container/run_generate_requirements.sh index de14cbff2d5..d05783ebbdd 100755 --- a/sdks/python/container/run_generate_requirements.sh +++ b/sdks/python/container/run_generate_requirements.sh @@ -80,7 +80,10 @@ pip install --upgrade pip setuptools wheel # Install dataframe deps to add have Dataframe support in released images. # Install test deps since some integration tests need dependencies, # such as pytest, installed in the runner environment. -pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"} --no-cache-dir "$SDK_TARBALL""$EXTRAS" +# Force torch dependencies to be pulled from the PyTorch CPU wheel +# repository so that they don't include GPU dependencies with +# non-compliant licenses +pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"} --no-cache-dir "$SDK_TARBALL""$EXTRAS" --extra-index-url https://download.pytorch.org/whl/cpu pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"} --no-cache-dir -r "$PWD"/sdks/python/container/base_image_requirements_manual.txt pip uninstall -y apache-beam diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 719d188ed26..8feb1fe1f01 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -593,6 +593,12 @@ if __name__ == '__main__': 'tensorflow>=2.12.0', 'torch>=1.9.0' ], + 'ml_cpu': [ + 'tensorflow>=2.12.0', + 'torch==2.8.0+cpu', + 'transformers>=4.28.0,<4.55.0; python_version < "3.10"', + 'transformers>=4.28.0,<4.56.0; python_version >= "3.10"' + ], 'tft': [ 'tensorflow_transform>=1.14.0,<1.15.0' # tensorflow-transform requires dill, but doesn't set dill as a
