This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 9b3bf08f78 GH-36752: [Python] Remove AWS SDK bundling when building
wheels (#36925)
9b3bf08f78 is described below
commit 9b3bf08f78f690605394ad3815177ece931f06d3
Author: Raúl Cumplido <[email protected]>
AuthorDate: Thu Aug 3 23:04:15 2023 +0200
GH-36752: [Python] Remove AWS SDK bundling when building wheels (#36925)
### Rationale for this change
In https://github.com/apache/arrow/pull/12227 we decided to use a bundled
version of the AWS SDK when compiling Python wheels, in order to downgrade the
AWS SDK version.
Now that we have fixed S3 finalization issues
(https://github.com/apache/arrow/pull/36442), it should be ok to rely on the
vcpkg-installed version of the AWS SDK again.
### What changes are included in this PR?
Remove use of bundled AWS SDK and use S3 vcpkg feature for requirements.
### Are these changes tested?
On CI and via crossbow
### Are there any user-facing changes?
No
* Closes: #36752
Authored-by: Raúl Cumplido <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
.env | 2 +-
ci/docker/python-wheel-manylinux.dockerfile | 3 ++-
ci/docker/python-wheel-windows-vs2017.dockerfile | 3 ++-
ci/scripts/java_jni_macos_build.sh | 1 -
ci/scripts/python_wheel_manylinux_build.sh | 4 ----
ci/scripts/python_wheel_windows_build.bat | 4 ----
6 files changed, 5 insertions(+), 12 deletions(-)
diff --git a/.env b/.env
index 25b2743f65..c9cd6c8094 100644
--- a/.env
+++ b/.env
@@ -98,7 +98,7 @@ VCPKG="501db0f17ef6df184fcdbfbe0f87cde2313b6ab1" #
2023.04.15 Release
# ci/docker/python-wheel-windows-vs2017.dockerfile.
# This is a workaround for our CI problem that "archery docker build" doesn't
# use pulled built images in dev/tasks/python-wheels/github.windows.yml.
-PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2022-06-12
+PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2023-08-02
# Use conanio/${CONAN} for "docker-compose run --rm conan". See
# https://github.com/conan-io/conan-docker-tools#readme for available
diff --git a/ci/docker/python-wheel-manylinux.dockerfile
b/ci/docker/python-wheel-manylinux.dockerfile
index ed329ecdef..0f7779c878 100644
--- a/ci/docker/python-wheel-manylinux.dockerfile
+++ b/ci/docker/python-wheel-manylinux.dockerfile
@@ -84,7 +84,8 @@ RUN vcpkg install \
--x-feature=flight \
--x-feature=gcs \
--x-feature=json \
- --x-feature=parquet
+ --x-feature=parquet \
+ --x-feature=s3
# Configure Python for applications running in the bash shell of this
Dockerfile
ARG python=3.8
diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile
b/ci/docker/python-wheel-windows-vs2017.dockerfile
index 01152dae23..531c4e678f 100644
--- a/ci/docker/python-wheel-windows-vs2017.dockerfile
+++ b/ci/docker/python-wheel-windows-vs2017.dockerfile
@@ -66,7 +66,8 @@ RUN vcpkg install \
--x-feature=flight \
--x-feature=gcs \
--x-feature=json \
- --x-feature=parquet
+ --x-feature=parquet \
+ --x-feature=s3
# Remove previous installations of python from the base image
# NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7
diff --git a/ci/scripts/java_jni_macos_build.sh
b/ci/scripts/java_jni_macos_build.sh
index 4a6f9444ec..d66c39a37c 100755
--- a/ci/scripts/java_jni_macos_build.sh
+++ b/ci/scripts/java_jni_macos_build.sh
@@ -81,7 +81,6 @@ cmake \
-DARROW_PARQUET=${ARROW_PARQUET} \
-DARROW_S3=${ARROW_S3} \
-DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \
- -DAWSSDK_SOURCE=BUNDLED \
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
-DCMAKE_INSTALL_LIBDIR=lib \
-DCMAKE_INSTALL_PREFIX=${install_dir} \
diff --git a/ci/scripts/python_wheel_manylinux_build.sh
b/ci/scripts/python_wheel_manylinux_build.sh
index cb5c2fbb7c..58e42fea88 100755
--- a/ci/scripts/python_wheel_manylinux_build.sh
+++ b/ci/scripts/python_wheel_manylinux_build.sh
@@ -85,9 +85,6 @@ fi
mkdir /tmp/arrow-build
pushd /tmp/arrow-build
-# ARROW-17501: We can remove -DAWSSDK_SOURCE=BUNDLED once
-# https://github.com/aws/aws-sdk-cpp/issues/1809 is fixed and vcpkg
-# ships the fix.
cmake \
-DARROW_ACERO=${ARROW_ACERO} \
-DARROW_BUILD_SHARED=ON \
@@ -120,7 +117,6 @@ cmake \
-DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
-DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
-DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
- -DAWSSDK_SOURCE=BUNDLED \
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
-DCMAKE_INSTALL_LIBDIR=lib \
-DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \
diff --git a/ci/scripts/python_wheel_windows_build.bat
b/ci/scripts/python_wheel_windows_build.bat
index ee879c7050..ffb43b3481 100644
--- a/ci/scripts/python_wheel_windows_build.bat
+++ b/ci/scripts/python_wheel_windows_build.bat
@@ -49,9 +49,6 @@ set ARROW_WITH_LZ4=ON
set ARROW_WITH_SNAPPY=ON
set ARROW_WITH_ZLIB=ON
set ARROW_WITH_ZSTD=ON
-@rem Workaround for https://github.com/aws/aws-sdk-cpp/issues/1809 .
-@rem Use (old) bundled AWS SDK C++ instead of (newer) AWS SDK C++.
-set AWSSDK_SOURCE=BUNDLED
set CMAKE_UNITY_BUILD=ON
set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
set VCPKG_ROOT=C:\vcpkg
@@ -90,7 +87,6 @@ cmake ^
-DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^
-DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^
-DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^
- -DAWSSDK_SOURCE=%AWSSDK_SOURCE% ^
-DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
-DCMAKE_CXX_COMPILER=clcache ^
-DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^
