This is an automated email from the ASF dual-hosted git repository.
anandinguva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new f7ae6b41c37 Bump Pyarrow version to include 14.0.0 (#29536)
f7ae6b41c37 is described below
commit f7ae6b41c3754eb6c50352b98fb8f296ec62e58e
Author: Anand Inguva <[email protected]>
AuthorDate: Sun Dec 3 04:19:11 2023 +0000
Bump Pyarrow version to include 14.0.0 (#29536)
* generate requirements file
* update setup.py pyarrow
* Update pyarrow except for windows
* Skip tests for version 13 or greater.
* Add reason for unittest skipIf
* Fix pyarrow lower bound
* Update sdks/python/setup.py
Co-authored-by: tvalentyn <[email protected]>
---------
Co-authored-by: tvalentyn <[email protected]>
---
sdks/python/apache_beam/io/parquetio_test.py | 3 +
.../container/py310/base_image_requirements.txt | 80 +++++++++++-----------
.../container/py311/base_image_requirements.txt | 76 ++++++++++----------
.../container/py38/base_image_requirements.txt | 78 ++++++++++-----------
.../container/py39/base_image_requirements.txt | 80 +++++++++++-----------
sdks/python/setup.py | 14 +++-
6 files changed, 173 insertions(+), 158 deletions(-)
diff --git a/sdks/python/apache_beam/io/parquetio_test.py
b/sdks/python/apache_beam/io/parquetio_test.py
index 1cd5f1208cc..06d9a527140 100644
--- a/sdks/python/apache_beam/io/parquetio_test.py
+++ b/sdks/python/apache_beam/io/parquetio_test.py
@@ -334,6 +334,9 @@ class TestParquet(unittest.TestCase):
]
hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
+ @unittest.skipIf(
+ ARROW_MAJOR_VERSION >= 13,
+ 'pyarrow 13.x and above does not throw ArrowInvalid error')
def test_sink_transform_int96(self):
with tempfile.NamedTemporaryFile() as dst:
path = dst.name
diff --git a/sdks/python/container/py310/base_image_requirements.txt
b/sdks/python/container/py310/base_image_requirements.txt
index a9f94104374..a80e6611d9c 100644
--- a/sdks/python/container/py310/base_image_requirements.txt
+++ b/sdks/python/container/py310/base_image_requirements.txt
@@ -25,38 +25,38 @@ attrs==23.1.0
beautifulsoup4==4.12.2
bs4==0.0.1
build==1.0.3
-cachetools==5.3.1
-certifi==2023.7.22
+cachetools==5.3.2
+certifi==2023.11.17
cffi==1.16.0
-charset-normalizer==3.3.0
+charset-normalizer==3.3.2
click==8.1.7
cloudpickle==2.2.1
crcmod==1.7
-cryptography==41.0.4
+cryptography==41.0.6
Cython==0.29.36
deprecation==2.1.0
dill==0.3.1.1
dnspython==2.4.2
docker==6.1.3
docopt==0.6.2
-exceptiongroup==1.1.3
+exceptiongroup==1.2.0
execnet==2.0.2
-fastavro==1.8.4
+fastavro==1.9.0
fasteners==0.19
freezegun==1.2.2
future==0.18.3
-google-api-core==2.12.0
-google-api-python-client==2.104.0
+google-api-core==2.14.0
+google-api-python-client==2.108.0
google-apitools==0.5.31
-google-auth==2.23.3
+google-auth==2.23.4
google-auth-httplib2==0.1.1
-google-cloud-aiplatform==1.35.0
-google-cloud-bigquery==3.12.0
+google-cloud-aiplatform==1.36.4
+google-cloud-bigquery==3.13.0
google-cloud-bigquery-storage==2.22.0
google-cloud-bigtable==2.21.0
google-cloud-core==2.3.3
google-cloud-datastore==2.18.0
-google-cloud-dlp==3.12.3
+google-cloud-dlp==3.13.0
google-cloud-language==2.11.1
google-cloud-profiler==4.1.0
google-cloud-pubsub==2.18.4
@@ -64,26 +64,26 @@ google-cloud-pubsublite==1.8.3
google-cloud-recommendations-ai==0.10.5
google-cloud-resource-manager==1.10.4
google-cloud-spanner==3.40.1
-google-cloud-storage==2.12.0
+google-cloud-storage==2.13.0
google-cloud-videointelligence==2.11.4
google-cloud-vision==3.4.5
google-crc32c==1.5.0
google-resumable-media==2.6.0
googleapis-common-protos==1.61.0
-greenlet==3.0.0
-grpc-google-iam-v1==0.12.6
-grpcio==1.59.0
-grpcio-status==1.59.0
-guppy3==3.1.4
+greenlet==3.0.1
+grpc-google-iam-v1==0.12.7
+grpcio==1.59.3
+grpcio-status==1.59.3
+guppy3==3.1.4.post1
hdfs==2.7.3
httplib2==0.22.0
-hypothesis==6.88.1
-idna==3.4
+hypothesis==6.91.0
+idna==3.6
iniconfig==2.0.0
joblib==1.3.2
Js2Py==0.74
-jsonschema==4.19.1
-jsonschema-specifications==2023.7.1
+jsonschema==4.20.0
+jsonschema-specifications==2023.11.1
mmh3==4.0.1
mock==5.1.0
nltk==3.8.1
@@ -91,47 +91,47 @@ nose==1.3.7
numpy==1.24.4
oauth2client==4.1.3
objsize==0.6.1
-orjson==3.9.9
+orjson==3.9.10
overrides==6.5.0
packaging==23.2
pandas==1.5.3
parameterized==0.9.0
pluggy==1.3.0
proto-plus==1.22.3
-protobuf==4.25.0
+protobuf==4.25.1
psycopg2-binary==2.9.9
-pyarrow==11.0.0
-pyarrow-hotfix==0.4
-pyasn1==0.5.0
+pyarrow==14.0.1
+pyarrow-hotfix==0.6
+pyasn1==0.5.1
pyasn1-modules==0.3.0
pycparser==2.21
pydot==1.4.2
-PyHamcrest==2.0.4
+PyHamcrest==2.1.0
pyjsparser==2.7.1
-pymongo==4.5.0
+pymongo==4.6.0
PyMySQL==1.1.0
pyparsing==3.1.1
pyproject_hooks==1.0.0
-pytest==7.4.2
+pytest==7.4.3
pytest-timeout==2.2.0
-pytest-xdist==3.3.1
+pytest-xdist==3.5.0
python-dateutil==2.8.2
python-snappy==0.6.1
pytz==2023.3.post1
PyYAML==6.0.1
-referencing==0.30.2
+referencing==0.31.0
regex==2023.10.3
requests==2.31.0
requests-mock==1.11.0
-rpds-py==0.10.6
+rpds-py==0.13.1
rsa==4.9
-scikit-learn==1.3.1
-scipy==1.11.3
+scikit-learn==1.3.2
+scipy==1.11.4
shapely==2.0.2
six==1.16.0
sortedcontainers==2.4.0
soupsieve==2.5
-SQLAlchemy==1.4.49
+SQLAlchemy==1.4.50
sqlparse==0.4.4
tenacity==8.2.3
testcontainers==3.7.1
@@ -139,9 +139,9 @@ threadpoolctl==3.2.0
tomli==2.0.1
tqdm==4.66.1
typing_extensions==4.8.0
-tzlocal==5.1
+tzlocal==5.2
uritemplate==4.1.1
-urllib3==2.0.7
+urllib3==2.1.0
websocket-client==1.6.4
-wrapt==1.15.0
-zstandard==0.21.0
+wrapt==1.16.0
+zstandard==0.22.0
diff --git a/sdks/python/container/py311/base_image_requirements.txt
b/sdks/python/container/py311/base_image_requirements.txt
index 865b856683a..04d9fc28c27 100644
--- a/sdks/python/container/py311/base_image_requirements.txt
+++ b/sdks/python/container/py311/base_image_requirements.txt
@@ -25,14 +25,14 @@ attrs==23.1.0
beautifulsoup4==4.12.2
bs4==0.0.1
build==1.0.3
-cachetools==5.3.1
-certifi==2023.7.22
+cachetools==5.3.2
+certifi==2023.11.17
cffi==1.16.0
-charset-normalizer==3.3.0
+charset-normalizer==3.3.2
click==8.1.7
cloudpickle==2.2.1
crcmod==1.7
-cryptography==41.0.4
+cryptography==41.0.6
Cython==0.29.36
deprecation==2.1.0
dill==0.3.1.1
@@ -40,47 +40,47 @@ dnspython==2.4.2
docker==6.1.3
docopt==0.6.2
execnet==2.0.2
-fastavro==1.8.4
+fastavro==1.9.0
fasteners==0.19
freezegun==1.2.2
future==0.18.3
-google-api-core==2.12.0
+google-api-core==2.14.0
google-apitools==0.5.31
-google-auth==2.23.3
+google-auth==2.23.4
google-auth-httplib2==0.1.1
-google-cloud-aiplatform==1.35.0
-google-cloud-bigquery==3.12.0
+google-cloud-aiplatform==1.36.4
+google-cloud-bigquery==3.13.0
google-cloud-bigquery-storage==2.22.0
google-cloud-bigtable==2.21.0
google-cloud-core==2.3.3
google-cloud-datastore==2.18.0
-google-cloud-dlp==3.12.3
+google-cloud-dlp==3.13.0
google-cloud-language==2.11.1
google-cloud-pubsub==2.18.4
google-cloud-pubsublite==1.8.3
google-cloud-recommendations-ai==0.10.5
google-cloud-resource-manager==1.10.4
google-cloud-spanner==3.40.1
-google-cloud-storage==2.12.0
+google-cloud-storage==2.13.0
google-cloud-videointelligence==2.11.4
google-cloud-vision==3.4.5
google-crc32c==1.5.0
google-resumable-media==2.6.0
googleapis-common-protos==1.61.0
-greenlet==3.0.0
-grpc-google-iam-v1==0.12.6
-grpcio==1.59.0
-grpcio-status==1.59.0
-guppy3==3.1.4
+greenlet==3.0.1
+grpc-google-iam-v1==0.12.7
+grpcio==1.59.3
+grpcio-status==1.59.3
+guppy3==3.1.4.post1
hdfs==2.7.3
httplib2==0.22.0
-hypothesis==6.88.1
-idna==3.4
+hypothesis==6.91.0
+idna==3.6
iniconfig==2.0.0
joblib==1.3.2
Js2Py==0.74
-jsonschema==4.19.1
-jsonschema-specifications==2023.7.1
+jsonschema==4.20.0
+jsonschema-specifications==2023.11.1
mmh3==4.0.1
mock==5.1.0
nltk==3.8.1
@@ -88,54 +88,54 @@ nose==1.3.7
numpy==1.24.4
oauth2client==4.1.3
objsize==0.6.1
-orjson==3.9.9
+orjson==3.9.10
overrides==6.5.0
packaging==23.2
pandas==1.5.3
parameterized==0.9.0
pluggy==1.3.0
proto-plus==1.22.3
-protobuf==4.25.0
+protobuf==4.25.1
psycopg2-binary==2.9.9
-pyarrow==11.0.0
-pyarrow-hotfix==0.4
-pyasn1==0.5.0
+pyarrow==14.0.1
+pyarrow-hotfix==0.6
+pyasn1==0.5.1
pyasn1-modules==0.3.0
pycparser==2.21
pydot==1.4.2
-PyHamcrest==2.0.4
+PyHamcrest==2.1.0
pyjsparser==2.7.1
-pymongo==4.5.0
+pymongo==4.6.0
PyMySQL==1.1.0
pyparsing==3.1.1
pyproject_hooks==1.0.0
-pytest==7.4.2
+pytest==7.4.3
pytest-timeout==2.2.0
-pytest-xdist==3.3.1
+pytest-xdist==3.5.0
python-dateutil==2.8.2
pytz==2023.3.post1
PyYAML==6.0.1
-referencing==0.30.2
+referencing==0.31.0
regex==2023.10.3
requests==2.31.0
requests-mock==1.11.0
-rpds-py==0.10.6
+rpds-py==0.13.1
rsa==4.9
-scikit-learn==1.3.1
-scipy==1.11.3
+scikit-learn==1.3.2
+scipy==1.11.4
shapely==2.0.2
six==1.16.0
sortedcontainers==2.4.0
soupsieve==2.5
-SQLAlchemy==1.4.49
+SQLAlchemy==1.4.50
sqlparse==0.4.4
tenacity==8.2.3
testcontainers==3.7.1
threadpoolctl==3.2.0
tqdm==4.66.1
typing_extensions==4.8.0
-tzlocal==5.1
-urllib3==2.0.7
+tzlocal==5.2
+urllib3==2.1.0
websocket-client==1.6.4
-wrapt==1.15.0
-zstandard==0.21.0
+wrapt==1.16.0
+zstandard==0.22.0
diff --git a/sdks/python/container/py38/base_image_requirements.txt
b/sdks/python/container/py38/base_image_requirements.txt
index 83580f65f2d..f22bbfe7d71 100644
--- a/sdks/python/container/py38/base_image_requirements.txt
+++ b/sdks/python/container/py38/base_image_requirements.txt
@@ -26,10 +26,10 @@ backports.zoneinfo==0.2.1
beautifulsoup4==4.12.2
bs4==0.0.1
build==1.0.3
-cachetools==5.3.1
-certifi==2023.7.22
+cachetools==5.3.2
+certifi==2023.11.17
cffi==1.16.0
-charset-normalizer==3.3.0
+charset-normalizer==3.3.2
click==8.1.7
cloudpickle==2.2.1
crcmod==1.7
@@ -40,24 +40,24 @@ dill==0.3.1.1
dnspython==2.4.2
docker==6.1.3
docopt==0.6.2
-exceptiongroup==1.1.3
+exceptiongroup==1.2.0
execnet==2.0.2
-fastavro==1.8.4
+fastavro==1.9.0
fasteners==0.19
freezegun==1.2.2
future==0.18.3
-google-api-core==2.12.0
-google-api-python-client==2.104.0
+google-api-core==2.14.0
+google-api-python-client==2.108.0
google-apitools==0.5.31
-google-auth==2.23.3
+google-auth==2.23.4
google-auth-httplib2==0.1.1
-google-cloud-aiplatform==1.35.0
-google-cloud-bigquery==3.12.0
+google-cloud-aiplatform==1.36.4
+google-cloud-bigquery==3.13.0
google-cloud-bigquery-storage==2.22.0
google-cloud-bigtable==2.21.0
google-cloud-core==2.3.3
google-cloud-datastore==2.18.0
-google-cloud-dlp==3.12.3
+google-cloud-dlp==3.13.0
google-cloud-language==2.11.1
google-cloud-profiler==4.1.0
google-cloud-pubsub==2.18.4
@@ -65,28 +65,28 @@ google-cloud-pubsublite==1.8.3
google-cloud-recommendations-ai==0.10.5
google-cloud-resource-manager==1.10.4
google-cloud-spanner==3.40.1
-google-cloud-storage==2.12.0
+google-cloud-storage==2.13.0
google-cloud-videointelligence==2.11.4
google-cloud-vision==3.4.5
google-crc32c==1.5.0
google-resumable-media==2.6.0
googleapis-common-protos==1.61.0
-greenlet==3.0.0
-grpc-google-iam-v1==0.12.6
-grpcio==1.59.0
-grpcio-status==1.59.0
-guppy3==3.1.4
+greenlet==3.0.1
+grpc-google-iam-v1==0.12.7
+grpcio==1.59.3
+grpcio-status==1.59.3
+guppy3==3.1.4.post1
hdfs==2.7.3
httplib2==0.22.0
-hypothesis==6.88.1
-idna==3.4
+hypothesis==6.91.0
+idna==3.6
importlib-metadata==6.8.0
-importlib-resources==6.1.0
+importlib-resources==6.1.1
iniconfig==2.0.0
joblib==1.3.2
Js2Py==0.74
-jsonschema==4.19.1
-jsonschema-specifications==2023.7.1
+jsonschema==4.20.0
+jsonschema-specifications==2023.11.1
mmh3==4.0.1
mock==5.1.0
nltk==3.8.1
@@ -94,7 +94,7 @@ nose==1.3.7
numpy==1.24.4
oauth2client==4.1.3
objsize==0.6.1
-orjson==3.9.9
+orjson==3.9.10
overrides==6.5.0
packaging==23.2
pandas==1.5.3
@@ -102,40 +102,40 @@ parameterized==0.9.0
pkgutil_resolve_name==1.3.10
pluggy==1.3.0
proto-plus==1.22.3
-protobuf==4.25.0
+protobuf==4.25.1
psycopg2-binary==2.9.9
-pyarrow==11.0.0
-pyarrow-hotfix==0.4
-pyasn1==0.5.0
+pyarrow==14.0.1
+pyarrow-hotfix==0.6
+pyasn1==0.5.1
pyasn1-modules==0.3.0
pycparser==2.21
pydot==1.4.2
-PyHamcrest==2.0.4
+PyHamcrest==2.1.0
pyjsparser==2.7.1
-pymongo==4.5.0
+pymongo==4.6.0
PyMySQL==1.1.0
pyparsing==3.1.1
pyproject_hooks==1.0.0
-pytest==7.4.2
+pytest==7.4.3
pytest-timeout==2.2.0
-pytest-xdist==3.3.1
+pytest-xdist==3.5.0
python-dateutil==2.8.2
python-snappy==0.6.1
pytz==2023.3.post1
PyYAML==6.0.1
-referencing==0.30.2
+referencing==0.31.0
regex==2023.10.3
requests==2.31.0
requests-mock==1.11.0
-rpds-py==0.10.6
+rpds-py==0.13.1
rsa==4.9
-scikit-learn==1.3.1
+scikit-learn==1.3.2
scipy==1.10.1
shapely==2.0.2
six==1.16.0
sortedcontainers==2.4.0
soupsieve==2.5
-SQLAlchemy==1.4.49
+SQLAlchemy==1.4.50
sqlparse==0.4.4
tenacity==8.2.3
testcontainers==3.7.1
@@ -143,10 +143,10 @@ threadpoolctl==3.2.0
tomli==2.0.1
tqdm==4.66.1
typing_extensions==4.8.0
-tzlocal==5.1
+tzlocal==5.2
uritemplate==4.1.1
-urllib3==2.0.7
+urllib3==2.1.0
websocket-client==1.6.4
-wrapt==1.15.0
+wrapt==1.16.0
zipp==3.17.0
-zstandard==0.21.0
+zstandard==0.22.0
diff --git a/sdks/python/container/py39/base_image_requirements.txt
b/sdks/python/container/py39/base_image_requirements.txt
index 1b8ad7a2e74..f42b6f8c2b4 100644
--- a/sdks/python/container/py39/base_image_requirements.txt
+++ b/sdks/python/container/py39/base_image_requirements.txt
@@ -25,38 +25,38 @@ attrs==23.1.0
beautifulsoup4==4.12.2
bs4==0.0.1
build==1.0.3
-cachetools==5.3.1
-certifi==2023.7.22
+cachetools==5.3.2
+certifi==2023.11.17
cffi==1.16.0
-charset-normalizer==3.3.0
+charset-normalizer==3.3.2
click==8.1.7
cloudpickle==2.2.1
crcmod==1.7
-cryptography==41.0.4
+cryptography==41.0.6
Cython==0.29.36
deprecation==2.1.0
dill==0.3.1.1
dnspython==2.4.2
docker==6.1.3
docopt==0.6.2
-exceptiongroup==1.1.3
+exceptiongroup==1.2.0
execnet==2.0.2
-fastavro==1.8.4
+fastavro==1.9.0
fasteners==0.19
freezegun==1.2.2
future==0.18.3
-google-api-core==2.12.0
-google-api-python-client==2.104.0
+google-api-core==2.14.0
+google-api-python-client==2.108.0
google-apitools==0.5.31
-google-auth==2.23.3
+google-auth==2.23.4
google-auth-httplib2==0.1.1
-google-cloud-aiplatform==1.35.0
-google-cloud-bigquery==3.12.0
+google-cloud-aiplatform==1.36.4
+google-cloud-bigquery==3.13.0
google-cloud-bigquery-storage==2.22.0
google-cloud-bigtable==2.21.0
google-cloud-core==2.3.3
google-cloud-datastore==2.18.0
-google-cloud-dlp==3.12.3
+google-cloud-dlp==3.13.0
google-cloud-language==2.11.1
google-cloud-profiler==4.1.0
google-cloud-pubsub==2.18.4
@@ -64,27 +64,27 @@ google-cloud-pubsublite==1.8.3
google-cloud-recommendations-ai==0.10.5
google-cloud-resource-manager==1.10.4
google-cloud-spanner==3.40.1
-google-cloud-storage==2.12.0
+google-cloud-storage==2.13.0
google-cloud-videointelligence==2.11.4
google-cloud-vision==3.4.5
google-crc32c==1.5.0
google-resumable-media==2.6.0
googleapis-common-protos==1.61.0
-greenlet==3.0.0
-grpc-google-iam-v1==0.12.6
-grpcio==1.59.0
-grpcio-status==1.59.0
-guppy3==3.1.4
+greenlet==3.0.1
+grpc-google-iam-v1==0.12.7
+grpcio==1.59.3
+grpcio-status==1.59.3
+guppy3==3.1.4.post1
hdfs==2.7.3
httplib2==0.22.0
-hypothesis==6.88.1
-idna==3.4
+hypothesis==6.91.0
+idna==3.6
importlib-metadata==6.8.0
iniconfig==2.0.0
joblib==1.3.2
Js2Py==0.74
-jsonschema==4.19.1
-jsonschema-specifications==2023.7.1
+jsonschema==4.20.0
+jsonschema-specifications==2023.11.1
mmh3==4.0.1
mock==5.1.0
nltk==3.8.1
@@ -92,47 +92,47 @@ nose==1.3.7
numpy==1.24.4
oauth2client==4.1.3
objsize==0.6.1
-orjson==3.9.9
+orjson==3.9.10
overrides==6.5.0
packaging==23.2
pandas==1.5.3
parameterized==0.9.0
pluggy==1.3.0
proto-plus==1.22.3
-protobuf==4.25.0
+protobuf==4.25.1
psycopg2-binary==2.9.9
-pyarrow==11.0.0
-pyarrow-hotfix==0.4
-pyasn1==0.5.0
+pyarrow==14.0.1
+pyarrow-hotfix==0.6
+pyasn1==0.5.1
pyasn1-modules==0.3.0
pycparser==2.21
pydot==1.4.2
-PyHamcrest==2.0.4
+PyHamcrest==2.1.0
pyjsparser==2.7.1
-pymongo==4.5.0
+pymongo==4.6.0
PyMySQL==1.1.0
pyparsing==3.1.1
pyproject_hooks==1.0.0
-pytest==7.4.2
+pytest==7.4.3
pytest-timeout==2.2.0
-pytest-xdist==3.3.1
+pytest-xdist==3.5.0
python-dateutil==2.8.2
python-snappy==0.6.1
pytz==2023.3.post1
PyYAML==6.0.1
-referencing==0.30.2
+referencing==0.31.0
regex==2023.10.3
requests==2.31.0
requests-mock==1.11.0
-rpds-py==0.10.6
+rpds-py==0.13.1
rsa==4.9
-scikit-learn==1.3.1
-scipy==1.11.3
+scikit-learn==1.3.2
+scipy==1.11.4
shapely==2.0.2
six==1.16.0
sortedcontainers==2.4.0
soupsieve==2.5
-SQLAlchemy==1.4.49
+SQLAlchemy==1.4.50
sqlparse==0.4.4
tenacity==8.2.3
testcontainers==3.7.1
@@ -140,10 +140,10 @@ threadpoolctl==3.2.0
tomli==2.0.1
tqdm==4.66.1
typing_extensions==4.8.0
-tzlocal==5.1
+tzlocal==5.2
uritemplate==4.1.1
-urllib3==2.0.7
+urllib3==2.1.0
websocket-client==1.6.4
-wrapt==1.15.0
+wrapt==1.16.0
zipp==3.17.0
-zstandard==0.21.0
+zstandard==0.22.0
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 1785cd75df8..cdd3ecb1994 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -141,9 +141,21 @@ except ImportError:
# [BEAM-8181] pyarrow cannot be installed on 32-bit Windows platforms.
if sys.platform == 'win32' and sys.maxsize <= 2**32:
pyarrow_dependency = ['']
+elif sys.platform == 'win32' or sys.platform == 'cygwin':
+ # https://github.com/apache/beam/issues/28410 - pyarrow>=13 seeing issues
+ # on windows with error
+ # C:\arrow\cpp\src\arrow\filesystem\s3fs.cc:2904: arrow::fs::FinalizeS3 was
+ # not called even though S3 was initialized. This could lead to a
+ # segmentation fault at exit. Keep pyarrow<13 until this is resolved.
+ pyarrow_dependency = [
+ 'pyarrow>=3.0.0,<12.0.0',
+ # NOTE: We can remove this once Beam increases the pyarrow lower bound
+ # to a version that fixes CVE.
+ 'pyarrow-hotfix<1'
+ ]
else:
pyarrow_dependency = [
- 'pyarrow>=3.0.0,<12.0.0',
+ 'pyarrow>=3.0.0,<15.0.0',
# NOTE(https://github.com/apache/beam/issues/29392): We can remove this
# once Beam increases the pyarrow lower bound to a version that fixes
CVE.
'pyarrow-hotfix<1'
