This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new a0dec7f393 GH-39352: [FS][Azure] Enable azure in builds (#39971)
a0dec7f393 is described below
commit a0dec7f39394e619c8bdfe0eacb6ecde73a9ec12
Author: Thomas Newton <[email protected]>
AuthorDate: Fri Feb 9 01:41:36 2024 +0000
GH-39352: [FS][Azure] Enable azure in builds (#39971)
### Rationale for this change
### What changes are included in this PR?
Enable Azure in linux and mac os wheel builds. Tried to copy GCS
Don't enable Azure for windows builds because windows builds where all
failing. Failures were a combination of cmake version or `Could not find a
package configuration file provided by "wil"`. I think it makes sense to come
back to windows builds in another PR.
### Are these changes tested?
There is no new functionality to test.
### Are there any user-facing changes?
No
* Closes: #39352
Authored-by: Thomas Newton <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
ci/docker/python-wheel-manylinux.dockerfile | 1 +
ci/scripts/cpp_build.sh | 1 +
ci/scripts/python_build.sh | 1 +
ci/scripts/python_test.sh | 1 +
ci/scripts/python_wheel_macos_build.sh | 3 +++
ci/scripts/python_wheel_manylinux_build.sh | 3 +++
ci/scripts/python_wheel_unix_test.sh | 6 ++++--
ci/vcpkg/vcpkg.json | 10 ++++++++++
dev/tasks/python-wheels/github.osx.amd64.yml | 1 +
dev/tasks/python-wheels/github.osx.arm64.yml | 1 +
10 files changed, 26 insertions(+), 2 deletions(-)
diff --git a/ci/docker/python-wheel-manylinux.dockerfile
b/ci/docker/python-wheel-manylinux.dockerfile
index 2831440d5a..b1d9ed5ab8 100644
--- a/ci/docker/python-wheel-manylinux.dockerfile
+++ b/ci/docker/python-wheel-manylinux.dockerfile
@@ -82,6 +82,7 @@ RUN vcpkg install \
--clean-after-build \
--x-install-root=${VCPKG_ROOT}/installed \
--x-manifest-root=/arrow/ci/vcpkg \
+ --x-feature=azure \
--x-feature=flight \
--x-feature=gcs \
--x-feature=json \
diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh
index 69d86e871a..60cab1a9fe 100755
--- a/ci/scripts/cpp_build.sh
+++ b/ci/scripts/cpp_build.sh
@@ -54,6 +54,7 @@ if [ "${GITHUB_ACTIONS:-false}" = "true" ]; then
fi
if [ "${ARROW_ENABLE_THREADING:-ON}" = "OFF" ]; then
+ ARROW_AZURE=OFF
ARROW_FLIGHT=OFF
ARROW_FLIGHT_SQL=OFF
ARROW_GCS=OFF
diff --git a/ci/scripts/python_build.sh b/ci/scripts/python_build.sh
index c0a27e6e70..9bdcc4d687 100755
--- a/ci/scripts/python_build.sh
+++ b/ci/scripts/python_build.sh
@@ -55,6 +55,7 @@ export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja}
export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug}
export PYARROW_WITH_ACERO=${ARROW_ACERO:-OFF}
+export PYARROW_WITH_AZURE=${ARROW_AZURE:-OFF}
export PYARROW_WITH_CUDA=${ARROW_CUDA:-OFF}
export PYARROW_WITH_DATASET=${ARROW_DATASET:-ON}
export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT:-OFF}
diff --git a/ci/scripts/python_test.sh b/ci/scripts/python_test.sh
index 8dfedb2880..20ca3300c0 100755
--- a/ci/scripts/python_test.sh
+++ b/ci/scripts/python_test.sh
@@ -39,6 +39,7 @@ fi
# By default, force-test all optional components
: ${PYARROW_TEST_ACERO:=${ARROW_ACERO:-ON}}
+: ${PYARROW_TEST_AZURE:=${ARROW_AZURE:-ON}}
: ${PYARROW_TEST_CUDA:=${ARROW_CUDA:-ON}}
: ${PYARROW_TEST_DATASET:=${ARROW_DATASET:-ON}}
: ${PYARROW_TEST_FLIGHT:=${ARROW_FLIGHT:-ON}}
diff --git a/ci/scripts/python_wheel_macos_build.sh
b/ci/scripts/python_wheel_macos_build.sh
index 8123a9fdf1..bea5409100 100755
--- a/ci/scripts/python_wheel_macos_build.sh
+++ b/ci/scripts/python_wheel_macos_build.sh
@@ -63,6 +63,7 @@ pip install "delocate>=0.10.3"
echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
: ${ARROW_ACERO:=ON}
+: ${ARROW_AZURE:=ON}
: ${ARROW_DATASET:=ON}
: ${ARROW_FLIGHT:=ON}
: ${ARROW_GANDIVA:=OFF}
@@ -95,6 +96,7 @@ pushd ${build_dir}/build
cmake \
-DARROW_ACERO=${ARROW_ACERO} \
+ -DARROW_AZURE=${ARROW_AZURE} \
-DARROW_BUILD_SHARED=ON \
-DARROW_BUILD_STATIC=OFF \
-DARROW_BUILD_TESTS=OFF \
@@ -148,6 +150,7 @@ export PYARROW_BUNDLE_ARROW_CPP=1
export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR}
export PYARROW_INSTALL_TESTS=1
export PYARROW_WITH_ACERO=${ARROW_ACERO}
+export PYARROW_WITH_AZURE=${ARROW_AZURE}
export PYARROW_WITH_DATASET=${ARROW_DATASET}
export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT}
export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA}
diff --git a/ci/scripts/python_wheel_manylinux_build.sh
b/ci/scripts/python_wheel_manylinux_build.sh
index 58e42fea88..4d4d4fb694 100755
--- a/ci/scripts/python_wheel_manylinux_build.sh
+++ b/ci/scripts/python_wheel_manylinux_build.sh
@@ -49,6 +49,7 @@ rm -rf /arrow/python/pyarrow/*.so.*
echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
: ${ARROW_ACERO:=ON}
+: ${ARROW_AZURE:=ON}
: ${ARROW_DATASET:=ON}
: ${ARROW_FLIGHT:=ON}
: ${ARROW_GANDIVA:=OFF}
@@ -87,6 +88,7 @@ pushd /tmp/arrow-build
cmake \
-DARROW_ACERO=${ARROW_ACERO} \
+ -DARROW_AZURE=${ARROW_AZURE} \
-DARROW_BUILD_SHARED=ON \
-DARROW_BUILD_STATIC=OFF \
-DARROW_BUILD_TESTS=OFF \
@@ -141,6 +143,7 @@ export PYARROW_BUNDLE_ARROW_CPP=1
export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR}
export PYARROW_INSTALL_TESTS=1
export PYARROW_WITH_ACERO=${ARROW_ACERO}
+export PYARROW_WITH_AZURE=${ARROW_AZURE}
export PYARROW_WITH_DATASET=${ARROW_DATASET}
export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT}
export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA}
diff --git a/ci/scripts/python_wheel_unix_test.sh
b/ci/scripts/python_wheel_unix_test.sh
index 01250ff7ef..a25e5c51bd 100755
--- a/ci/scripts/python_wheel_unix_test.sh
+++ b/ci/scripts/python_wheel_unix_test.sh
@@ -28,15 +28,17 @@ fi
source_dir=${1}
+: ${ARROW_AZURE:=ON}
: ${ARROW_FLIGHT:=ON}
-: ${ARROW_SUBSTRAIT:=ON}
-: ${ARROW_S3:=ON}
: ${ARROW_GCS:=ON}
+: ${ARROW_S3:=ON}
+: ${ARROW_SUBSTRAIT:=ON}
: ${CHECK_IMPORTS:=ON}
: ${CHECK_UNITTESTS:=ON}
: ${INSTALL_PYARROW:=ON}
export PYARROW_TEST_ACERO=ON
+export PYARROW_TEST_AZURE=${ARROW_AZURE}
export PYARROW_TEST_CYTHON=OFF
export PYARROW_TEST_DATASET=ON
export PYARROW_TEST_FLIGHT=${ARROW_FLIGHT}
diff --git a/ci/vcpkg/vcpkg.json b/ci/vcpkg/vcpkg.json
index 99771728ec..e86479a7c3 100644
--- a/ci/vcpkg/vcpkg.json
+++ b/ci/vcpkg/vcpkg.json
@@ -105,6 +105,16 @@
}
]
},
+ "azure": {
+ "description": "Azure blob storage support",
+ "dependencies": [
+ "azure-core-cpp",
+ "azure-identity-cpp",
+ "azure-storage-blobs-cpp",
+ "azure-storage-common-cpp",
+ "azure-storage-files-datalake-cpp"
+ ]
+ },
"orc": {
"description": "ORC support",
"dependencies": [
diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml
b/dev/tasks/python-wheels/github.osx.amd64.yml
index 526412f842..e31a681653 100644
--- a/dev/tasks/python-wheels/github.osx.amd64.yml
+++ b/dev/tasks/python-wheels/github.osx.amd64.yml
@@ -85,6 +85,7 @@ jobs:
--clean-after-build \
--x-install-root=${VCPKG_ROOT}/installed \
--x-manifest-root=arrow/ci/vcpkg \
+ --x-feature=azure \
--x-feature=flight \
--x-feature=gcs \
--x-feature=json \
diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml
b/dev/tasks/python-wheels/github.osx.arm64.yml
index 35d74f1462..380c2e42f1 100644
--- a/dev/tasks/python-wheels/github.osx.arm64.yml
+++ b/dev/tasks/python-wheels/github.osx.arm64.yml
@@ -71,6 +71,7 @@ jobs:
--clean-after-build \
--x-install-root=${VCPKG_ROOT}/installed \
--x-manifest-root=arrow/ci/vcpkg \
+ --x-feature=azure \
--x-feature=flight \
--x-feature=gcs \
--x-feature=json \
