This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 8836535785 GH-43702: [C++][FS][Azure] Use the latest Azurite and
update the bundled Azure SDK for C++ to azure-identity_1.9.0 (#43723)
8836535785 is described below
commit 8836535785ba3dd4ba335818a34e0479929b70e6
Author: Sutou Kouhei <[email protected]>
AuthorDate: Sat Aug 17 11:20:16 2024 +0900
GH-43702: [C++][FS][Azure] Use the latest Azurite and update the bundled
Azure SDK for C++ to azure-identity_1.9.0 (#43723)
### Rationale for this change
Some our CI jobs (such as conda based jobs) use recent Azure SDK for C++
and they require latest Azurite. We need to update Azurite for these jobs.
I wanted to use the latest Azurite on all environments but I didn't.
Because I want to keep using `apt install nodejs` on old Ubuntu for easy to
maintain.
### What changes are included in this PR?
* Use the latest Azurite if possible
* Use `--skipApiVersionCheck` for old Azurite
* Update the bundled Azure SDK for C++
* This is not required. It's for detecting this problem in many CI jobs.
### Are these changes tested?
Yes.
### Are there any user-facing changes?
No.
* GitHub Issue: fix #41505
* GitHub Issue: #43702
Authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
ci/scripts/install_azurite.sh | 24 ++++++++++++++++++------
cpp/src/arrow/filesystem/azurefs_test.cc | 5 ++++-
cpp/thirdparty/versions.txt | 4 ++--
python/pyarrow/tests/conftest.py | 3 +++
4 files changed, 27 insertions(+), 9 deletions(-)
diff --git a/ci/scripts/install_azurite.sh b/ci/scripts/install_azurite.sh
index dda5e99405..b8b1618bed 100755
--- a/ci/scripts/install_azurite.sh
+++ b/ci/scripts/install_azurite.sh
@@ -19,20 +19,32 @@
set -e
-# Pin azurite to 3.29.0 due to https://github.com/apache/arrow/issues/41505
+node_version="$(node --version)"
+echo "node version = ${node_version}"
+
+case "${node_version}" in
+ v12*)
+ # Pin azurite to 3.29.0 due to https://github.com/apache/arrow/issues/41505
+ azurite_version=v3.29.0
+ ;;
+ *)
+ azurite_version=latest
+ ;;
+esac
+
case "$(uname)" in
Darwin)
- npm install -g [email protected]
+ npm install -g azurite@${azurite_version}
which azurite
;;
MINGW*)
choco install nodejs.install
- npm install -g [email protected]
+ npm install -g azurite@${azurite_version}
;;
Linux)
- npm install -g [email protected]
+ npm install -g azurite@${azurite_version}
which azurite
;;
esac
-echo "node version = $(node --version)"
-echo "azurite version = $(azurite --version)"
\ No newline at end of file
+
+echo "azurite version = $(azurite --version)"
diff --git a/cpp/src/arrow/filesystem/azurefs_test.cc
b/cpp/src/arrow/filesystem/azurefs_test.cc
index 36646f417c..5ff241b17f 100644
--- a/cpp/src/arrow/filesystem/azurefs_test.cc
+++ b/cpp/src/arrow/filesystem/azurefs_test.cc
@@ -198,7 +198,10 @@ class AzuriteEnv : public AzureEnvImpl<AzuriteEnv> {
self->temp_dir_->path().Join("debug.log"));
auto server_process = bp::child(
boost::this_process::environment(), exe_path, "--silent", "--location",
- self->temp_dir_->path().ToString(), "--debug",
self->debug_log_path_.ToString());
+ self->temp_dir_->path().ToString(), "--debug",
self->debug_log_path_.ToString(),
+ // For old Azurite. We can't install the latest Azurite with
+ // old Node.js on old Ubuntu.
+ "--skipApiVersionCheck");
if (!server_process.valid() || !server_process.running()) {
server_process.terminate();
server_process.wait();
diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt
index 16689c17fb..30fa24a209 100644
--- a/cpp/thirdparty/versions.txt
+++ b/cpp/thirdparty/versions.txt
@@ -54,8 +54,8 @@
ARROW_AWS_LC_BUILD_SHA256_CHECKSUM=ae96a3567161552744fc0cae8b4d68ed88b1ec0f3d3c9
ARROW_AWSSDK_BUILD_VERSION=1.10.55
ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=2d552fb1a84bef4a9b65e34aa7031851ed2aef5319e02cc6e4cb735c48aa30de
# Despite the confusing version name this is still the whole Azure SDK for C++
including core, keyvault, storage-common, etc.
-ARROW_AZURE_SDK_BUILD_VERSION=azure-core_1.10.3
-ARROW_AZURE_SDK_BUILD_SHA256_CHECKSUM=dd624c2f86adf474d2d0a23066be6e27af9cbd7e3f8d9d8fd7bf981e884b7b48
+ARROW_AZURE_SDK_BUILD_VERSION=azure-identity_1.9.0
+ARROW_AZURE_SDK_BUILD_SHA256_CHECKSUM=97065bfc971ac8df450853ce805f820f52b59457bd7556510186a1569502e4a1
ARROW_BOOST_BUILD_VERSION=1.81.0
ARROW_BOOST_BUILD_SHA256_CHECKSUM=9e0ffae35528c35f90468997bc8d99500bf179cbae355415a89a600c38e13574
ARROW_BROTLI_BUILD_VERSION=v1.0.9
diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py
index 343b602995..e1919497b5 100644
--- a/python/pyarrow/tests/conftest.py
+++ b/python/pyarrow/tests/conftest.py
@@ -263,6 +263,9 @@ def azure_server(tmpdir_factory):
tmpdir = tmpdir_factory.getbasetemp()
# We only need blob service emulator, not queue or table.
args = ['azurite-blob', "--location", tmpdir, "--blobPort", str(port)]
+ # For old Azurite. We can't install the latest Azurite with old
+ # Node.js on old Ubuntu.
+ args += ["--skipApiVersionCheck"]
proc = None
try:
proc = subprocess.Popen(args, env=env)
