This is an automated email from the ASF dual-hosted git repository.
jorisvandenbossche pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new c9674bcc16 GH-37621: [Packaging][Conda] Sync conda recipes with
feedstocks (#37624)
c9674bcc16 is described below
commit c9674bcc16411d7ecfd3b5587f544758b9fc7062
Author: h-vetinari <[email protected]>
AuthorDate: Thu Sep 28 22:18:07 2023 +1100
GH-37621: [Packaging][Conda] Sync conda recipes with feedstocks (#37624)
Syncing after the release of 13.0.0 + a couple of migrations (state as of
https://github.com/conda-forge/arrow-cpp-feedstock/pull/1168 &
https://github.com/conda-forge/r-arrow-feedstock/pull/68)
Relevant updates:
* we're not building twice for different protobuf versions anymore
* new abseil version (fixes #36908)
* we've finally upgraded the aws-sdk to 1.11
* the default R versions (on unix) are now 4.2 & 4.3.
Also some further hardening of the activation scripts & clean-ups for
dependencies & test skips.
* Closes: #37621
Lead-authored-by: H. Vetinari <[email protected]>
Co-authored-by: h-vetinari <[email protected]>
Signed-off-by: Joris Van den Bossche <[email protected]>
---
.../linux_64_cuda_compiler_version11.2.yaml | 20 +++----
.../linux_64_cuda_compiler_versionNone.yaml | 20 +++----
.../linux_aarch64_cuda_compiler_version11.2.yaml | 20 +++----
.../linux_aarch64_cuda_compiler_versionNone.yaml | 20 +++----
.../linux_ppc64le_cuda_compiler_version11.2.yaml | 20 +++----
.../linux_ppc64le_cuda_compiler_versionNone.yaml | 20 +++----
dev/tasks/conda-recipes/.ci_support/osx_64_.yaml | 22 ++++----
.../conda-recipes/.ci_support/osx_arm64_.yaml | 20 +++----
...x_64_r_base4.1.yaml => linux_64_r_base4.3.yaml} | 2 +-
...r_base4.1.yaml => linux_aarch64_r_base4.3.yaml} | 2 +-
...osx_64_r_base4.1.yaml => osx_64_r_base4.3.yaml} | 2 +-
...m64_r_base4.1.yaml => osx_arm64_r_base4.3.yaml} | 2 +-
.../win_64_cuda_compiler_version11.2.yaml | 22 +++-----
.../win_64_cuda_compiler_versionNone.yaml | 22 +++-----
dev/tasks/conda-recipes/arrow-cpp/activate.sh | 17 ++++--
dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh | 4 +-
dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh | 4 ++
dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 65 +++++++++-------------
dev/tasks/tasks.yml | 24 ++++----
19 files changed, 151 insertions(+), 177 deletions(-)
diff --git
a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml
b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml
index 1cdcec199e..042e2364d1 100644
---
a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml
+++
b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml
@@ -1,7 +1,7 @@
aws_crt_cpp:
-- 0.20.3
+- 0.23.1
aws_sdk_cpp:
-- 1.10.57
+- 1.11.156
bzip2:
- '1'
c_compiler:
@@ -33,20 +33,18 @@ glog:
google_cloud_cpp:
- '2.12'
libabseil:
-- '20230125'
+- '20230802'
libgrpc:
-- '1.54'
-- '1.56'
+- '1.57'
libprotobuf:
-- '3.21'
-- 4.23.3
+- 4.23.4
lz4_c:
- 1.9.3
numpy:
-- '1.21'
+- '1.22'
- '1.23'
-- '1.21'
-- '1.21'
+- '1.22'
+- '1.22'
openssl:
- '3'
orc:
@@ -67,7 +65,7 @@ snappy:
target_platform:
- linux-64
thrift_cpp:
-- 0.18.1
+- 0.19.0
ucx:
- 1.14.0
zip_keys:
diff --git
a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml
b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml
index 5be5b58a73..9885e6db38 100644
---
a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml
+++
b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml
@@ -1,7 +1,7 @@
aws_crt_cpp:
-- 0.20.3
+- 0.23.1
aws_sdk_cpp:
-- 1.10.57
+- 1.11.156
bzip2:
- '1'
c_compiler:
@@ -33,20 +33,18 @@ glog:
google_cloud_cpp:
- '2.12'
libabseil:
-- '20230125'
+- '20230802'
libgrpc:
-- '1.54'
-- '1.56'
+- '1.57'
libprotobuf:
-- '3.21'
-- 4.23.3
+- 4.23.4
lz4_c:
- 1.9.3
numpy:
-- '1.21'
+- '1.22'
- '1.23'
-- '1.21'
-- '1.21'
+- '1.22'
+- '1.22'
openssl:
- '3'
orc:
@@ -67,7 +65,7 @@ snappy:
target_platform:
- linux-64
thrift_cpp:
-- 0.18.1
+- 0.19.0
ucx:
- 1.14.0
zip_keys:
diff --git
a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml
b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml
index 1677b03564..788b584504 100644
---
a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml
+++
b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml
@@ -1,9 +1,9 @@
BUILD:
- aarch64-conda_cos7-linux-gnu
aws_crt_cpp:
-- 0.20.3
+- 0.23.1
aws_sdk_cpp:
-- 1.10.57
+- 1.11.156
bzip2:
- '1'
c_compiler:
@@ -37,20 +37,18 @@ glog:
google_cloud_cpp:
- '2.12'
libabseil:
-- '20230125'
+- '20230802'
libgrpc:
-- '1.54'
-- '1.56'
+- '1.57'
libprotobuf:
-- '3.21'
-- 4.23.3
+- 4.23.4
lz4_c:
- 1.9.3
numpy:
-- '1.21'
+- '1.22'
- '1.23'
-- '1.21'
-- '1.21'
+- '1.22'
+- '1.22'
openssl:
- '3'
orc:
@@ -71,7 +69,7 @@ snappy:
target_platform:
- linux-aarch64
thrift_cpp:
-- 0.18.1
+- 0.19.0
ucx:
- 1.14.0
zip_keys:
diff --git
a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml
b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml
index 88fdf1254e..a1e4b8571a 100644
---
a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml
+++
b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml
@@ -1,9 +1,9 @@
BUILD:
- aarch64-conda_cos7-linux-gnu
aws_crt_cpp:
-- 0.20.3
+- 0.23.1
aws_sdk_cpp:
-- 1.10.57
+- 1.11.156
bzip2:
- '1'
c_compiler:
@@ -37,20 +37,18 @@ glog:
google_cloud_cpp:
- '2.12'
libabseil:
-- '20230125'
+- '20230802'
libgrpc:
-- '1.54'
-- '1.56'
+- '1.57'
libprotobuf:
-- '3.21'
-- 4.23.3
+- 4.23.4
lz4_c:
- 1.9.3
numpy:
-- '1.21'
+- '1.22'
- '1.23'
-- '1.21'
-- '1.21'
+- '1.22'
+- '1.22'
openssl:
- '3'
orc:
@@ -71,7 +69,7 @@ snappy:
target_platform:
- linux-aarch64
thrift_cpp:
-- 0.18.1
+- 0.19.0
ucx:
- 1.14.0
zip_keys:
diff --git
a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml
b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml
index 3585db7b99..e21c4cbe85 100644
---
a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml
+++
b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml
@@ -1,7 +1,7 @@
aws_crt_cpp:
-- 0.20.3
+- 0.23.1
aws_sdk_cpp:
-- 1.10.57
+- 1.11.156
bzip2:
- '1'
c_compiler:
@@ -33,20 +33,18 @@ glog:
google_cloud_cpp:
- '2.12'
libabseil:
-- '20230125'
+- '20230802'
libgrpc:
-- '1.54'
-- '1.56'
+- '1.57'
libprotobuf:
-- '3.21'
-- 4.23.3
+- 4.23.4
lz4_c:
- 1.9.3
numpy:
-- '1.21'
+- '1.22'
- '1.23'
-- '1.21'
-- '1.21'
+- '1.22'
+- '1.22'
openssl:
- '3'
orc:
@@ -67,7 +65,7 @@ snappy:
target_platform:
- linux-ppc64le
thrift_cpp:
-- 0.18.1
+- 0.19.0
ucx:
- 1.14.0
zip_keys:
diff --git
a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml
b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml
index c13a522254..89f1049ebd 100644
---
a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml
+++
b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml
@@ -1,7 +1,7 @@
aws_crt_cpp:
-- 0.20.3
+- 0.23.1
aws_sdk_cpp:
-- 1.10.57
+- 1.11.156
bzip2:
- '1'
c_compiler:
@@ -33,20 +33,18 @@ glog:
google_cloud_cpp:
- '2.12'
libabseil:
-- '20230125'
+- '20230802'
libgrpc:
-- '1.54'
-- '1.56'
+- '1.57'
libprotobuf:
-- '3.21'
-- 4.23.3
+- 4.23.4
lz4_c:
- 1.9.3
numpy:
-- '1.21'
+- '1.22'
- '1.23'
-- '1.21'
-- '1.21'
+- '1.22'
+- '1.22'
openssl:
- '3'
orc:
@@ -67,7 +65,7 @@ snappy:
target_platform:
- linux-ppc64le
thrift_cpp:
-- 0.18.1
+- 0.19.0
ucx:
- 1.14.0
zip_keys:
diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml
b/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml
index dd4a230760..2a5f8c5b36 100644
--- a/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml
@@ -1,9 +1,9 @@
MACOSX_DEPLOYMENT_TARGET:
-- '10.9'
+- '10.13'
aws_crt_cpp:
-- 0.20.3
+- 0.23.1
aws_sdk_cpp:
-- 1.10.57
+- 1.11.156
bzip2:
- '1'
c_compiler:
@@ -27,22 +27,20 @@ glog:
google_cloud_cpp:
- '2.12'
libabseil:
-- '20230125'
+- '20230802'
libgrpc:
-- '1.54'
-- '1.56'
+- '1.57'
libprotobuf:
-- '3.21'
-- 4.23.3
+- 4.23.4
lz4_c:
- 1.9.3
macos_machine:
- x86_64-apple-darwin13.4.0
numpy:
-- '1.21'
+- '1.22'
- '1.23'
-- '1.21'
-- '1.21'
+- '1.22'
+- '1.22'
openssl:
- '3'
orc:
@@ -63,7 +61,7 @@ snappy:
target_platform:
- osx-64
thrift_cpp:
-- 0.18.1
+- 0.19.0
zip_keys:
- - c_compiler_version
- cxx_compiler_version
diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml
b/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml
index 6a6713a54f..211b71226c 100644
--- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml
@@ -1,9 +1,9 @@
MACOSX_DEPLOYMENT_TARGET:
- '11.0'
aws_crt_cpp:
-- 0.20.3
+- 0.23.1
aws_sdk_cpp:
-- 1.10.57
+- 1.11.156
bzip2:
- '1'
c_compiler:
@@ -27,22 +27,20 @@ glog:
google_cloud_cpp:
- '2.12'
libabseil:
-- '20230125'
+- '20230802'
libgrpc:
-- '1.54'
-- '1.56'
+- '1.57'
libprotobuf:
-- '3.21'
-- 4.23.3
+- 4.23.4
lz4_c:
- 1.9.3
macos_machine:
- arm64-apple-darwin20.0.0
numpy:
-- '1.21'
+- '1.22'
- '1.23'
-- '1.21'
-- '1.21'
+- '1.22'
+- '1.22'
openssl:
- '3'
orc:
@@ -63,7 +61,7 @@ snappy:
target_platform:
- osx-arm64
thrift_cpp:
-- 0.18.1
+- 0.19.0
zip_keys:
- - c_compiler_version
- cxx_compiler_version
diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml
b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.3.yaml
similarity index 98%
rename from dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml
rename to dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.3.yaml
index e63767cbe9..a4d06c9f20 100644
--- a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.3.yaml
@@ -19,7 +19,7 @@ pin_run_as_build:
min_pin: x.x
max_pin: x.x
r_base:
-- '4.1'
+- '4.3'
target_platform:
- linux-64
zip_keys:
diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml
b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.3.yaml
similarity index 98%
rename from dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml
rename to dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.3.yaml
index 2b80b020fd..028b190bb1 100644
--- a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.3.yaml
@@ -23,7 +23,7 @@ pin_run_as_build:
min_pin: x.x
max_pin: x.x
r_base:
-- '4.1'
+- '4.3'
target_platform:
- linux-aarch64
zip_keys:
diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml
b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.3.yaml
similarity index 98%
rename from dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml
rename to dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.3.yaml
index 6be6c2f546..7b8b62d8e0 100644
--- a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.3.yaml
@@ -19,7 +19,7 @@ pin_run_as_build:
min_pin: x.x
max_pin: x.x
r_base:
-- '4.1'
+- '4.3'
target_platform:
- osx-64
zip_keys:
diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml
b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.3.yaml
similarity index 98%
rename from dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml
rename to dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.3.yaml
index 0ce856fccc..a8e8aab83d 100644
--- a/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.3.yaml
@@ -19,7 +19,7 @@ pin_run_as_build:
min_pin: x.x
max_pin: x.x
r_base:
-- '4.1'
+- '4.3'
target_platform:
- osx-arm64
zip_keys:
diff --git
a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml
b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml
index f75d92e276..32da33c072 100644
--- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml
@@ -1,11 +1,9 @@
aws_crt_cpp:
-- 0.20.3
+- 0.23.1
aws_sdk_cpp:
-- 1.10.57
+- 1.11.156
bzip2:
- '1'
-c_ares:
-- '1'
c_compiler:
- vs2019
channel_sources:
@@ -27,24 +25,22 @@ glog:
google_cloud_cpp:
- '2.12'
libabseil:
-- '20230125'
+- '20230802'
libcrc32c:
- '1.1'
libcurl:
- '8'
libgrpc:
-- '1.54'
-- '1.56'
+- '1.57'
libprotobuf:
-- '3.21'
-- 4.23.3
+- 4.23.4
lz4_c:
- 1.9.3
numpy:
-- '1.21'
+- '1.22'
- '1.23'
-- '1.21'
-- '1.21'
+- '1.22'
+- '1.22'
openssl:
- '3'
orc:
@@ -65,7 +61,7 @@ snappy:
target_platform:
- win-64
thrift_cpp:
-- 0.18.1
+- 0.19.0
zip_keys:
- - cuda_compiler
- cuda_compiler_version
diff --git
a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml
b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml
index 6d8fb15b15..6a33b86b9d 100644
--- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml
+++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml
@@ -1,11 +1,9 @@
aws_crt_cpp:
-- 0.20.3
+- 0.23.1
aws_sdk_cpp:
-- 1.10.57
+- 1.11.156
bzip2:
- '1'
-c_ares:
-- '1'
c_compiler:
- vs2019
channel_sources:
@@ -27,24 +25,22 @@ glog:
google_cloud_cpp:
- '2.12'
libabseil:
-- '20230125'
+- '20230802'
libcrc32c:
- '1.1'
libcurl:
- '8'
libgrpc:
-- '1.54'
-- '1.56'
+- '1.57'
libprotobuf:
-- '3.21'
-- 4.23.3
+- 4.23.4
lz4_c:
- 1.9.3
numpy:
-- '1.21'
+- '1.22'
- '1.23'
-- '1.21'
-- '1.21'
+- '1.22'
+- '1.22'
openssl:
- '3'
orc:
@@ -65,7 +61,7 @@ snappy:
target_platform:
- win-64
thrift_cpp:
-- 0.18.1
+- 0.19.0
zip_keys:
- - cuda_compiler
- cuda_compiler_version
diff --git a/dev/tasks/conda-recipes/arrow-cpp/activate.sh
b/dev/tasks/conda-recipes/arrow-cpp/activate.sh
index 8757612781..19d037ff41 100644
--- a/dev/tasks/conda-recipes/arrow-cpp/activate.sh
+++ b/dev/tasks/conda-recipes/arrow-cpp/activate.sh
@@ -23,6 +23,13 @@ _la_log "Beginning libarrow activation."
# where the GDB wrappers get installed
_la_gdb_prefix="$CONDA_PREFIX/share/gdb/auto-load"
+# If the directory is not writable, nothing can be done
+if [ ! -w "$_la_gdb_prefix" ]; then
+ _la_log 'No rights to modify $_la_gdb_prefix, cannot create symlink!'
+ _la_log 'Unless you plan to use the GDB debugger with libarrow, this
warning can be safely ignored.'
+ return
+fi
+
# this needs to be in sync with ARROW_GDB_INSTALL_DIR in build.sh
_la_placeholder="replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX"
# the paths here are intentionally stacked, see #935, resp.
@@ -44,7 +51,7 @@ for _la_target in "$_la_orig_install_dir/"*.py; do
# If the file doesn't exist, skip this iteration of the loop.
# (This happens when no files are found, in which case the
# loop runs with target equal to the pattern itself.)
- _la_log 'Folder $_la_orig_install_dir seems to not contain .py files,
skipping'
+ _la_log 'Folder $_la_orig_install_dir seems to not contain .py files,
skipping.'
continue
fi
_la_symlink="$_la_symlink_dir/$(basename "$_la_target")"
@@ -54,13 +61,13 @@ for _la_target in "$_la_orig_install_dir/"*.py; do
_la_log 'symlink $_la_symlink already exists and points to
$_la_target, skipping.'
continue
fi
- _la_log 'Creating symlink $_la_symlink pointing to $_la_target'
+ _la_log 'Creating symlink $_la_symlink pointing to $_la_target.'
mkdir -p "$_la_symlink_dir" || true
# this check also creates the symlink; if it fails, we enter the if-branch.
if ! ln -sf "$_la_target" "$_la_symlink"; then
- echo -n "${BASH_SOURCE[0]} ERROR: Failed to create symlink from "
- echo -n "'$_la_target' to '$_la_symlink'"
- echo
+ echo -n "${BASH_SOURCE[0]} WARNING: Failed to create symlink from "
+ echo "'$_la_target' to '$_la_symlink'!"
+ echo "Unless you plan to use the GDB debugger with libarrow, this
warning can be safely ignored."
continue
fi
done
diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh
b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh
index dc588f9473..ef0b038812 100755
--- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh
+++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh
@@ -30,7 +30,7 @@ fi
# Enable CUDA support
if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" !=
"None" ]]
then
- EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=ON
-DCUDA_TOOLKIT_ROOT_DIR=${CUDA_HOME}
-DCMAKE_LIBRARY_PATH=${CONDA_BUILD_SYSROOT}/lib"
+ EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=ON
-DCUDAToolkit_ROOT=${CUDA_HOME} -DCMAKE_LIBRARY_PATH=${CONDA_BUILD_SYSROOT}/lib"
else
EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=OFF"
fi
@@ -43,8 +43,8 @@ if [[ "${build_platform}" != "${target_platform}" ]]; then
fi
EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}
-DCLANG_EXECUTABLE=${BUILD_PREFIX}/bin/${CONDA_TOOLCHAIN_HOST}-clang"
EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}
-DLLVM_LINK_EXECUTABLE=${BUILD_PREFIX}/bin/llvm-link"
+ EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DARROW_JEMALLOC_LG_PAGE=16"
sed -ie
"s;protoc-gen-grpc.*$;protoc-gen-grpc=${BUILD_PREFIX}/bin/grpc_cpp_plugin\";g"
../src/arrow/flight/CMakeLists.txt
- sed -ie
's;"--with-jemalloc-prefix\=je_arrow_";"--with-jemalloc-prefix\=je_arrow_"
"--with-lg-page\=16";g' ../cmake_modules/ThirdpartyToolchain.cmake
fi
# disable -fno-plt, which causes problems with GCC on PPC
diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh
b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh
index 9c12321a1c..f39e06874c 100755
--- a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh
+++ b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh
@@ -24,6 +24,10 @@ BUILD_EXT_FLAGS=""
# Enable CUDA support
if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" !=
"None" ]]; then
export PYARROW_WITH_CUDA=1
+ if [[ "${build_platform}" != "${target_platform}" ]]; then
+ export CUDAToolkit_ROOT=${CUDA_HOME}
+ export CMAKE_LIBRARY_PATH=${CONDA_BUILD_SYSROOT}/lib
+ fi
else
export PYARROW_WITH_CUDA=0
fi
diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
index fbe40af3da..371b62245b 100644
--- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
+++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
@@ -61,7 +61,7 @@ outputs:
build:
string: h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }}
run_exports:
- - {{ pin_subpackage("libarrow", max_pin="x.x.x") }}
+ - {{ pin_subpackage("libarrow", max_pin="x") }}
ignore_run_exports_from:
- {{ compiler("cuda") }} # [cuda_compiler_version !=
"None"]
# arrow only uses headers, apparently
@@ -114,6 +114,8 @@ outputs:
- libgrpc
- libprotobuf
- libutf8proc
+ # gandiva requires shared libllvm
+ - llvm # [unix]
- lz4-c
- nlohmann_json
# gandiva depends on openssl
@@ -133,8 +135,6 @@ outputs:
# its host deps (which aren't yet covered above) leak into the build
here
- libcrc32c # [win]
- libcurl # [win]
- # same for libgrpc (before 1.55.0, which is coupled with libprotobuf
4.23.x)
- - c-ares # [win and libprotobuf == "3.21"]
run_constrained:
- apache-arrow-proc =*={{ build_ext }}
# make sure we don't co-install with old version of old package name
@@ -198,8 +198,6 @@ outputs:
requirements:
host:
- {{ pin_subpackage('libarrow', exact=True) }}
- # avoid wrappers for different builds colliding due to identical hashes
- - libprotobuf
run:
- {{ pin_subpackage('libarrow', exact=True) }}
test:
@@ -235,9 +233,7 @@ outputs:
- cmake
- ninja
host:
- # we're building for two protobuf versions, cannot pin exactly
- # - {{ pin_subpackage('libarrow', exact=True) }}
- - libarrow ={{ version }}=*_{{ PKG_BUILDNUM }}_{{ build_ext }}
+ - {{ pin_subpackage('libarrow', exact=True) }}
- clangdev {{ llvm_version }}
- llvmdev {{ llvm_version }}
- cython <3
@@ -246,8 +242,7 @@ outputs:
- setuptools
- setuptools_scm <8.0.0
run:
- # - {{ pin_subpackage('libarrow', exact=True) }}
- - libarrow ={{ version }}=*_{{ PKG_BUILDNUM }}_{{ build_ext }}
+ - {{ pin_subpackage('libarrow', exact=True) }}
- {{ pin_compatible('numpy') }}
- python
run_constrained:
@@ -336,28 +331,28 @@ outputs:
# crossbow CI: reduce to one python version, except on (unemulated) linux,
where it's fast enough
{% if linux64 or py == 311 %}
- # {% if not (aarch64 or ppc64le) or py in (310, 311) %}
- # only run the full test suite for one python version when in emulation
(each run takes ~45min);
- # there's essentially zero divergence in behaviour across python versions
anyway, and otherwise
- # CUDA builds for aarch/ppc consistently run out of disk space on azure
for some reason
+ # {% if not (aarch64 or ppc64le) or py == 311 %}
+ # only run the full test suite for one python version when in emulation
+ # (each run can take up to ~45min); there's essentially zero divergence
+ # in behaviour across python versions anyway
test:
requires:
- # vary protobuf version in test suite (historically, test failures
only have a very
- # weak dependency on python version, so we don't lose coverage by
doing half & half)
- - libprotobuf <4 # [py % 2 == 0]
# test_cpp_extension_in_python requires a compiler
- {{ compiler("cxx") }} # [linux]
- # temporary pin due to missing fixture
- - pytest <7.4.0
+ - pytest
- pytest-lazy-fixture
- backports.zoneinfo # [py<39]
+ - boto3
- cffi
- cloudpickle
- cython <3
- fastparquet
- fsspec
- hypothesis
+ # currently disabled due to GH-37692
+ # - minio-server
- pandas
+ - s3fs >=2023
- scipy
# these are generally (far) behind on migrating abseil/grpc/protobuf,
# and using them as test dependencies blocks the migrator unnecessarily
@@ -372,8 +367,8 @@ outputs:
source_files:
- testing/data
commands:
- - cd ${SP_DIR}/pyarrow/tests # [unix]
- - cd %SP_DIR%\pyarrow\tests # [win]
+ - cd ${SP_DIR} # [unix]
+ - cd %SP_DIR% # [win]
- export ARROW_TEST_DATA="${SRC_DIR}/testing/data" # [unix]
- set "ARROW_TEST_DATA=%SRC_DIR%\testing\data" # [win]
@@ -382,34 +377,26 @@ outputs:
{% set tests_to_skip = tests_to_skip + " or test_cuda" %}
# skip tests that raise SIGINT and crash the test suite
{% set tests_to_skip = tests_to_skip + " or (test_csv and
test_cancellation)" %} # [linux]
- {% set tests_to_skip = tests_to_skip + " or (test_flight and
test_interrupt)" %} # [linux]
- # tests that may crash the agent due to out-of-bound memory writes or
other risky stuff
- {% set tests_to_skip = tests_to_skip + " or test_debug_memory_pool" %}
# [aarch64 or ppc64le]
- # cannot pass -D_LIBCPP_DISABLE_AVAILABILITY to test suite for our
older macos sdk
- {% set tests_to_skip = tests_to_skip + " or
test_cpp_extension_in_python" %} # [osx]
+ # skip test that intentionally writes out of bounds and then expects
no error message
+ {% set tests_to_skip = tests_to_skip + " or
test_debug_memory_pool_disabled[system_memory_pool]" %} # [osx]
# skip tests that make invalid(-for-conda) assumptions about the
compilers setup
{% set tests_to_skip = tests_to_skip + " or test_cython_api" %}
# [unix]
{% set tests_to_skip = tests_to_skip + " or test_visit_strings" %}
# [unix]
# skip tests that cannot succeed in emulation
{% set tests_to_skip = tests_to_skip + " or
test_debug_memory_pool_disabled" %} # [aarch64 or ppc64le]
{% set tests_to_skip = tests_to_skip + " or
test_env_var_io_thread_count" %} # [aarch64 or ppc64le]
+ # XMinioInvalidObjectName on osx/win: "Object name contains
unsupported characters"
+ {% set tests_to_skip = tests_to_skip + " or
test_write_to_dataset_with_partitions_s3fs" %} # [osx or win]
# vvvvvvv TESTS THAT SHOULDN'T HAVE TO BE SKIPPED vvvvvvv
- {% set tests_to_skip = tests_to_skip + " or
test_extension_to_pandas_storage_type" %}
- # segfaults on OSX: to investigate ASAP
- {% set tests_to_skip = tests_to_skip + " or test_flight" %}
# [osx]
+ # currently broken
+ {% set tests_to_skip = tests_to_skip + " or
test_fastparquet_cross_compatibility" %}
# gandiva tests are segfaulting on ppc
- {% set tests_to_skip = tests_to_skip + " or test_gandiva" %}
# [ppc64le]
- # test failures on ppc
+ {% set tests_to_skip = tests_to_skip + " or test_gandiva" %}
# [ppc64le]
+ # test failures on ppc (both failing with: Float value was truncated
converting to int32)
{% set tests_to_skip = tests_to_skip + " or
test_safe_cast_from_float_with_nans_to_int" %} # [ppc64le]
- # gandiva tests are segfaulting on ppc
- {% set tests_to_skip = tests_to_skip + " or
test_float_with_null_as_integer" %} # [ppc64le]
- # test is broken; header is in $PREFIX, not $SP_DIR
- {% set tests_to_skip = tests_to_skip + " or (test_misc and
test_get_include)" %} # [unix]
- # flaky tests that fail occasionally
- {% set tests_to_skip = tests_to_skip + " or test_total_bytes_allocated
" %} # [linux]
- {% set tests_to_skip = tests_to_skip + " or test_feather_format " %}
# [linux]
+ {% set tests_to_skip = tests_to_skip + " or
test_float_with_null_as_integer" %} # [ppc64le]
# ^^^^^^^ TESTS THAT SHOULDN'T HAVE TO BE SKIPPED ^^^^^^^
- - pytest -rfEs -k "not ({{ tests_to_skip }})"
+ - pytest pyarrow/ -rfEs -k "not ({{ tests_to_skip }})"
{% endif %}
about:
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 29e038a922..859ff8ddb5 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -246,15 +246,15 @@ tasks:
# generated and to be synced regularly from the feedstock. We have no way
# yet to generate them inside the arrow repository automatically.
- conda-linux-x64-cpu-r41:
+ conda-linux-x64-cpu-r43:
ci: azure
template: conda-recipes/azure.linux.yml
params:
config: linux_64_cuda_compiler_versionNone
- r_config: linux_64_r_base4.1
+ r_config: linux_64_r_base4.3
artifacts:
- libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda
- - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda
+ - r-arrow-{no_rc_version}-r43(h[a-z0-9]+)_0.conda
conda-linux-x64-cpu-r42:
ci: azure
@@ -292,15 +292,15 @@ tasks:
########################### Conda Linux (aarch64) ###########################
- conda-linux-aarch64-cpu-r41:
+ conda-linux-aarch64-cpu-r43:
ci: azure
template: conda-recipes/azure.linux.yml
params:
config: linux_aarch64_cuda_compiler_versionNone
- r_config: linux_aarch64_r_base4.1
+ r_config: linux_aarch64_r_base4.3
artifacts:
- libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda
- - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda
+ - r-arrow-{no_rc_version}-r43(h[a-z0-9]+)_0.conda
conda-linux-aarch64-cpu-r42:
ci: azure
@@ -364,15 +364,15 @@ tasks:
############################## Conda OSX (x64) ##############################
- conda-osx-x64-cpu-r41:
+ conda-osx-x64-cpu-r43:
ci: azure
template: conda-recipes/azure.osx.yml
params:
config: osx_64_
- r_config: osx_64_r_base4.1
+ r_config: osx_64_r_base4.3
artifacts:
- libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda
- - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda
+ - r-arrow-{no_rc_version}-r43(h[a-z0-9]+)_0.conda
conda-osx-x64-cpu-r42:
ci: azure
@@ -398,15 +398,15 @@ tasks:
############################# Conda OSX (arm64) #############################
- conda-osx-arm64-cpu-r41:
+ conda-osx-arm64-cpu-r43:
ci: azure
template: conda-recipes/azure.osx.yml
params:
config: osx_arm64_
- r_config: osx_arm64_r_base4.1
+ r_config: osx_arm64_r_base4.3
artifacts:
- libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda
- - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda
+ - r-arrow-{no_rc_version}-r43(h[a-z0-9]+)_0.conda
conda-osx-arm64-cpu-r42:
ci: azure
