This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 7cf158a332 GH-45195: [C++] Update bundled AWS SDK for C++ to 1.11.587
(#45306)
7cf158a332 is described below
commit 7cf158a33248c99d67d4d09be79d539c8b3837bd
Author: Sutou Kouhei <[email protected]>
AuthorDate: Tue Jun 17 11:04:48 2025 +0900
GH-45195: [C++] Update bundled AWS SDK for C++ to 1.11.587 (#45306)
### Rationale for this change
The current bundled version 1.10.55 was released on 2023-01-20:
https://github.com/aws/aws-sdk-cpp/releases/tag/1.10.55
### What changes are included in this PR?
* Use the latest AWS SDK for C++
* Migrate to FetchContent from ExternalProject
### Are these changes tested?
Yes.
### Are there any user-facing changes?
Yes.
* GitHub Issue: #45195
Authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
ci/rtools/README.md | 35 --
ci/rtools/aws_c_common_ep.patch | 39 --
ci/rtools/aws_c_io_ep.patch | 56 --
ci/rtools/awssdk_ep.patch | 181 ------
cpp/CMakeLists.txt | 11 +-
cpp/cmake_modules/AWSSDKVariables.cmake | 388 -------------
cpp/cmake_modules/BuildUtils.cmake | 12 +-
cpp/cmake_modules/SetupCxxFlags.cmake | 8 +-
cpp/cmake_modules/ThirdpartyToolchain.cmake | 645 +++++++--------------
cpp/cmake_modules/aws-c-common-1208.patch | 83 +++
.../aws_sdk_cpp_generate_variables.sh | 72 ---
cpp/thirdparty/README.md | 16 +
cpp/thirdparty/update.rb | 189 ++++++
cpp/thirdparty/versions.txt | 60 +-
python/pyarrow/tests/test_fs.py | 2 +
15 files changed, 547 insertions(+), 1250 deletions(-)
diff --git a/ci/rtools/README.md b/ci/rtools/README.md
deleted file mode 100644
index 08b5ea7f51..0000000000
--- a/ci/rtools/README.md
+++ /dev/null
@@ -1,35 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# rtools40 patches for AWS SDK and related libs
-
-The patches in this directory are solely for the purpose of building Arrow C++
-under [Rtools40](https://cran.r-project.org/bin/windows/Rtools/rtools40.html)
-and not used elsewhere. Once we've dropped support for Rtools40, we can
consider
-removing these patches.
-
-The larger reason these patches are needed is that Rtools provides their own
-packages and their versions of the AWS libraries weren't compatible with CMake
-3.25. Our solution was to bundle the AWS libs instead and these patches were
-required to get them building under the Rtools40 environment.
-
-The patches were added while upgrading the minimum required CMake version to
-3.25 in [GH-44950](https://github.com/apache/arrow/issues/44950). Please see
the
-associated PR, [GH-44989](https://github.com/apache/arrow/pull/44989), for more
-context.
diff --git a/ci/rtools/aws_c_common_ep.patch b/ci/rtools/aws_c_common_ep.patch
deleted file mode 100644
index 94c84d0fe1..0000000000
--- a/ci/rtools/aws_c_common_ep.patch
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-diff --git a/include/aws/common/byte_order.inl
b/include/aws/common/byte_order.inl
-index 1204be0..0abd9cb 100644
---- a/include/aws/common/byte_order.inl
-+++ b/include/aws/common/byte_order.inl
-@@ -13,7 +13,7 @@
- # include <stdlib.h>
- #else
- # include <netinet/in.h>
--#endif /* _MSC_VER */
-+#endif /* _WIN32 */
-
- AWS_EXTERN_C_BEGIN
-
-@@ -39,7 +39,7 @@ AWS_STATIC_IMPL uint64_t aws_hton64(uint64_t x) {
- uint64_t v;
- __asm__("bswap %q0" : "=r"(v) : "0"(x));
- return v;
--#elif defined(_MSC_VER)
-+#elif defined(_WIN32)
- return _byteswap_uint64(x);
- #else
- uint32_t low = x & UINT32_MAX;
diff --git a/ci/rtools/aws_c_io_ep.patch b/ci/rtools/aws_c_io_ep.patch
deleted file mode 100644
index a15d706ba1..0000000000
--- a/ci/rtools/aws_c_io_ep.patch
+++ /dev/null
@@ -1,56 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-diff --git a/source/windows/secure_channel_tls_handler.c
b/source/windows/secure_channel_tls_handler.c
-index 50caf02..29fe850 100644
---- a/source/windows/secure_channel_tls_handler.c
-+++ b/source/windows/secure_channel_tls_handler.c
-@@ -35,6 +36,25 @@
- # pragma warning(disable : 4306) /* Identifier is type cast to a larger
pointer. */
- #endif
-
-+#ifndef SP_PROT_TLS1_0_SERVER
-+#define SP_PROT_TLS1_0_SERVER SP_PROT_TLS1_SERVER
-+#endif
-+#ifndef SP_PROT_TLS1_0_CLIENT
-+#define SP_PROT_TLS1_0_CLIENT SP_PROT_TLS1_CLIENT
-+#endif
-+#ifndef SP_PROT_TLS1_1_SERVER
-+#define SP_PROT_TLS1_1_SERVER 0x00000100
-+#endif
-+#ifndef SP_PROT_TLS1_1_CLIENT
-+#define SP_PROT_TLS1_1_CLIENT 0x00000200
-+#endif
-+#ifndef SCH_USE_STRONG_CRYPTO
-+#define SCH_USE_STRONG_CRYPTO 0x00400000
-+#endif
-+#ifndef SECBUFFER_ALERT
-+#define SECBUFFER_ALERT 0x11
-+#endif
-+
- #define KB_1 1024
- #define READ_OUT_SIZE (16 * KB_1)
- #define READ_IN_SIZE READ_OUT_SIZE
-@@ -456,7 +476,7 @@ static int s_fillin_alpn_data(
-
- *extension_length += sizeof(uint32_t) + sizeof(uint16_t);
-
-- *extension_name = SecApplicationProtocolNegotiationExt_ALPN;
-+ *extension_name = 2;
- /*now add the protocols*/
- for (size_t i = 0; i < protocols_count; ++i) {
- struct aws_byte_cursor *protocol_ptr = NULL;
diff --git a/ci/rtools/awssdk_ep.patch b/ci/rtools/awssdk_ep.patch
deleted file mode 100644
index bd26f85329..0000000000
--- a/ci/rtools/awssdk_ep.patch
+++ /dev/null
@@ -1,181 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-diff --git a/aws-cpp-sdk-core/include/aws/core/utils/Array.h
b/aws-cpp-sdk-core/include/aws/core/utils/Array.h
-index 2b5bbc566..7cb93bdf0 100644
---- a/aws-cpp-sdk-core/include/aws/core/utils/Array.h
-+++ b/aws-cpp-sdk-core/include/aws/core/utils/Array.h
-@@ -54,7 +54,7 @@ namespace Aws
- {
- m_data.reset(Aws::NewArray<T>(m_size,
ARRAY_ALLOCATION_TAG));
-
--#ifdef _WIN32
-+#ifdef _MSC_VER
- std::copy(arrayToCopy, arrayToCopy + arraySize,
stdext::checked_array_iterator< T * >(m_data.get(), m_size));
- #else
- std::copy(arrayToCopy, arrayToCopy + arraySize,
m_data.get());
-@@ -82,7 +82,7 @@ namespace Aws
- if(arr->m_size > 0 && arr->m_data)
- {
- size_t arraySize = arr->m_size;
--#ifdef _WIN32
-+#ifdef _MSC_VER
- std::copy(arr->m_data.get(), arr->m_data.get() +
arraySize, stdext::checked_array_iterator< T * >(m_data.get() + location,
m_size));
- #else
- std::copy(arr->m_data.get(), arr->m_data.get() +
arraySize, m_data.get() + location);
-@@ -101,7 +101,7 @@ namespace Aws
- {
- m_data.reset(Aws::NewArray<T>(m_size,
ARRAY_ALLOCATION_TAG));
-
--#ifdef _WIN32
-+#ifdef _MSC_VER
- std::copy(other.m_data.get(), other.m_data.get() +
other.m_size, stdext::checked_array_iterator< T * >(m_data.get(), m_size));
- #else
- std::copy(other.m_data.get(), other.m_data.get() +
other.m_size, m_data.get());
-@@ -134,7 +134,7 @@ namespace Aws
- {
- m_data.reset(Aws::NewArray<T>(m_size,
ARRAY_ALLOCATION_TAG));
-
--#ifdef _WIN32
-+#ifdef _MSC_VER
- std::copy(other.m_data.get(), other.m_data.get() +
other.m_size, stdext::checked_array_iterator< T * >(m_data.get(), m_size));
- #else
- std::copy(other.m_data.get(), other.m_data.get() +
other.m_size, m_data.get());
-diff --git a/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp
b/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp
-index 4dade6489..a0456cf8e 100644
---- a/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp
-+++ b/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp
-@@ -22,6 +22,16 @@
- #include <sstream>
- #include <iostream>
-
-+#ifndef WINHTTP_OPTION_WEB_SOCKET_KEEPALIVE_INTERVAL
-+#define WINHTTP_OPTION_WEB_SOCKET_KEEPALIVE_INTERVAL 116
-+#endif
-+#ifndef WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1
-+#define WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 0x00000200
-+#endif
-+#ifndef WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2
-+#define WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2 0x00000800
-+#endif
-+
- using namespace Aws::Client;
- using namespace Aws::Http;
- using namespace Aws::Http::Standard;
-@@ -272,7 +282,7 @@ bool WinHttpSyncHttpClient::DoQueryHeaders(void*
hHttpRequest, std::shared_ptr<H
- wmemset(contentTypeStr, 0, static_cast<size_t>(dwSize / sizeof(wchar_t)));
-
- WinHttpQueryHeaders(hHttpRequest, WINHTTP_QUERY_CONTENT_TYPE, nullptr,
&contentTypeStr, &dwSize, 0);
-- if (contentTypeStr[0] != NULL)
-+ if (contentTypeStr[0])
- {
- Aws::String contentStr = StringUtils::FromWString(contentTypeStr);
- response->SetContentType(contentStr);
-@@ -303,7 +313,7 @@ bool WinHttpSyncHttpClient::DoQueryHeaders(void*
hHttpRequest, std::shared_ptr<H
-
- bool WinHttpSyncHttpClient::DoSendRequest(void* hHttpRequest) const
- {
-- return (WinHttpSendRequest(hHttpRequest, NULL, NULL, 0, 0, 0, NULL) != 0);
-+ return (WinHttpSendRequest(hHttpRequest, NULL, 0, NULL, 0, 0, 0) != 0);
- }
-
- bool WinHttpSyncHttpClient::DoReadData(void* hHttpRequest, char* body,
uint64_t size, uint64_t& read) const
-diff --git a/aws-cpp-sdk-core/source/http/windows/WinINetSyncHttpClient.cpp
b/aws-cpp-sdk-core/source/http/windows/WinINetSyncHttpClient.cpp
-index 5854cc334..70184a358 100644
---- a/aws-cpp-sdk-core/source/http/windows/WinINetSyncHttpClient.cpp
-+++ b/aws-cpp-sdk-core/source/http/windows/WinINetSyncHttpClient.cpp
-@@ -225,7 +225,7 @@ bool WinINetSyncHttpClient::DoQueryHeaders(void*
hHttpRequest, std::shared_ptr<H
- char contentTypeStr[1024];
- dwSize = sizeof(contentTypeStr);
- HttpQueryInfoA(hHttpRequest, HTTP_QUERY_CONTENT_TYPE, &contentTypeStr,
&dwSize, 0);
-- if (contentTypeStr[0] != NULL)
-+ if (contentTypeStr[0])
- {
- response->SetContentType(contentTypeStr);
- AWS_LOGSTREAM_DEBUG(GetLogTag(), "Received content type " <<
contentTypeStr);
-diff --git a/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp
b/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp
-index d7513cc3c..e390a8d4e 100644
---- a/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp
-+++ b/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp
-@@ -349,7 +349,7 @@ std::shared_ptr<HttpResponse>
WinSyncHttpClient::MakeRequest(const std::shared_p
- }
- }
-
-- if (!success && !IsRequestProcessingEnabled() ||
!ContinueRequest(*request))
-+ if ((!success && !IsRequestProcessingEnabled()) ||
!ContinueRequest(*request))
- {
- response->SetClientErrorType(CoreErrors::USER_CANCELLED);
- response->SetClientErrorMessage("Request processing disabled or
continuation cancelled by user's continuation handler.");
-diff --git a/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp
b/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp
-index 2ea82de6f..bc423441e 100644
---- a/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp
-+++ b/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp
-@@ -11,7 +11,9 @@
- #include <iostream>
- #include <Userenv.h>
-
-+#ifdef _MSC_VER
- #pragma warning( disable : 4996)
-+#endif
-
- using namespace Aws::Utils;
- namespace Aws
-@@ -304,6 +306,9 @@ Aws::String CreateTempFilePath()
- {
- #ifdef _MSC_VER
- #pragma warning(disable: 4996) // _CRT_SECURE_NO_WARNINGS
-+#elif !defined(L_tmpnam_s)
-+ // Definition from the MSVC stdio.h
-+ #define L_tmpnam_s (sizeof("\\") + 16)
- #endif
- char s_tempName[L_tmpnam_s+1];
-
-diff --git a/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp
b/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp
-index 0180f7fbf..3adbab313 100644
---- a/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp
-+++ b/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp
-@@ -9,7 +9,9 @@
-
- #include <iomanip>
-
-+#ifdef _MSC_VER
- #pragma warning(disable: 4996)
-+#endif
- #include <windows.h>
- #include <stdio.h>
- namespace Aws
-diff --git a/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp
b/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp
-index 2ee517b48..3b0dce665 100644
---- a/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp
-+++ b/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp
-@@ -939,7 +939,7 @@ std::shared_ptr<Aws::Utils::Crypto::HMAC>
Aws::Utils::Crypto::CreateSha256HMACIm
- return GetSha256HMACFactory()->CreateImplementation();
- }
-
--#ifdef _WIN32
-+#ifdef _MSC_VER
- #pragma warning( push )
- #pragma warning( disable : 4702 )
- #endif
-@@ -1032,7 +1032,7 @@ std::shared_ptr<SymmetricCipher>
Aws::Utils::Crypto::CreateAES_KeyWrapImplementa
- return GetAES_KeyWrapFactory()->CreateImplementation(key);
- }
-
--#ifdef _WIN32
-+#ifdef _MSC_VER
- #pragma warning(pop)
- #endif
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 60a77c815e..18ff93e965 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -56,6 +56,11 @@ cmake_policy(SET CMP0068 NEW)
# find_package() uses <PackageName>_ROOT variables.
cmake_policy(SET CMP0074 NEW)
+# https://cmake.org/cmake/help/latest/policy/CMP0077.html
+#
+# option() honors normal variables.
+cmake_policy(SET CMP0077 NEW)
+
# https://cmake.org/cmake/help/latest/policy/CMP0091.html
#
# MSVC runtime library flags are selected by an abstraction.
@@ -387,14 +392,10 @@ endif()
# where to put generated archives (.a files)
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-set(ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-
# where to put generated libraries (.so files)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-set(LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-
# where to put generated binaries
-set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
if(CMAKE_GENERATOR STREQUAL Xcode)
# Xcode projects support multi-configuration builds. This forces a single
output directory
diff --git a/cpp/cmake_modules/AWSSDKVariables.cmake
b/cpp/cmake_modules/AWSSDKVariables.cmake
deleted file mode 100644
index 729790dd0f..0000000000
--- a/cpp/cmake_modules/AWSSDKVariables.cmake
+++ /dev/null
@@ -1,388 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Generated by:
-# $ cpp/cmake_modules/aws_sdk_cpp_generate_variables.sh 1.10.55
-
-set(AWSSDK_UNUSED_DIRECTORIES
- .github
- AndroidSDKTesting
- CI
- Docs
- android-build
- android-unified-tests
- aws-cpp-sdk-AWSMigrationHub
- aws-cpp-sdk-access-management
- aws-cpp-sdk-accessanalyzer
- aws-cpp-sdk-account
- aws-cpp-sdk-acm
- aws-cpp-sdk-acm-pca
- aws-cpp-sdk-alexaforbusiness
- aws-cpp-sdk-amp
- aws-cpp-sdk-amplify
- aws-cpp-sdk-amplifybackend
- aws-cpp-sdk-amplifyuibuilder
- aws-cpp-sdk-apigateway
- aws-cpp-sdk-apigatewaymanagementapi
- aws-cpp-sdk-apigatewayv2
- aws-cpp-sdk-appconfig
- aws-cpp-sdk-appconfigdata
- aws-cpp-sdk-appflow
- aws-cpp-sdk-appintegrations
- aws-cpp-sdk-application-autoscaling
- aws-cpp-sdk-application-insights
- aws-cpp-sdk-applicationcostprofiler
- aws-cpp-sdk-appmesh
- aws-cpp-sdk-apprunner
- aws-cpp-sdk-appstream
- aws-cpp-sdk-appsync
- aws-cpp-sdk-arc-zonal-shift
- aws-cpp-sdk-athena
- aws-cpp-sdk-auditmanager
- aws-cpp-sdk-autoscaling
- aws-cpp-sdk-autoscaling-plans
- aws-cpp-sdk-awstransfer
- aws-cpp-sdk-backup
- aws-cpp-sdk-backup-gateway
- aws-cpp-sdk-backupstorage
- aws-cpp-sdk-batch
- aws-cpp-sdk-billingconductor
- aws-cpp-sdk-braket
- aws-cpp-sdk-budgets
- aws-cpp-sdk-ce
- aws-cpp-sdk-chime
- aws-cpp-sdk-chime-sdk-identity
- aws-cpp-sdk-chime-sdk-media-pipelines
- aws-cpp-sdk-chime-sdk-meetings
- aws-cpp-sdk-chime-sdk-messaging
- aws-cpp-sdk-chime-sdk-voice
- aws-cpp-sdk-cleanrooms
- aws-cpp-sdk-cloud9
- aws-cpp-sdk-cloudcontrol
- aws-cpp-sdk-clouddirectory
- aws-cpp-sdk-cloudformation
- aws-cpp-sdk-cloudfront
- aws-cpp-sdk-cloudfront-integration-tests
- aws-cpp-sdk-cloudhsm
- aws-cpp-sdk-cloudhsmv2
- aws-cpp-sdk-cloudsearch
- aws-cpp-sdk-cloudsearchdomain
- aws-cpp-sdk-cloudtrail
- aws-cpp-sdk-codeartifact
- aws-cpp-sdk-codebuild
- aws-cpp-sdk-codecatalyst
- aws-cpp-sdk-codecommit
- aws-cpp-sdk-codedeploy
- aws-cpp-sdk-codeguru-reviewer
- aws-cpp-sdk-codeguruprofiler
- aws-cpp-sdk-codepipeline
- aws-cpp-sdk-codestar
- aws-cpp-sdk-codestar-connections
- aws-cpp-sdk-codestar-notifications
- aws-cpp-sdk-cognito-idp
- aws-cpp-sdk-cognito-sync
- aws-cpp-sdk-cognitoidentity-integration-tests
- aws-cpp-sdk-comprehend
- aws-cpp-sdk-comprehendmedical
- aws-cpp-sdk-compute-optimizer
- aws-cpp-sdk-connect
- aws-cpp-sdk-connect-contact-lens
- aws-cpp-sdk-connectcampaigns
- aws-cpp-sdk-connectcases
- aws-cpp-sdk-connectparticipant
- aws-cpp-sdk-controltower
- aws-cpp-sdk-cur
- aws-cpp-sdk-custom-service-integration-tests
- aws-cpp-sdk-customer-profiles
- aws-cpp-sdk-databrew
- aws-cpp-sdk-dataexchange
- aws-cpp-sdk-datapipeline
- aws-cpp-sdk-datasync
- aws-cpp-sdk-dax
- aws-cpp-sdk-detective
- aws-cpp-sdk-devicefarm
- aws-cpp-sdk-devops-guru
- aws-cpp-sdk-directconnect
- aws-cpp-sdk-discovery
- aws-cpp-sdk-dlm
- aws-cpp-sdk-dms
- aws-cpp-sdk-docdb
- aws-cpp-sdk-docdb-elastic
- aws-cpp-sdk-drs
- aws-cpp-sdk-ds
- aws-cpp-sdk-dynamodb
- aws-cpp-sdk-dynamodb-integration-tests
- aws-cpp-sdk-dynamodbstreams
- aws-cpp-sdk-ebs
- aws-cpp-sdk-ec2
- aws-cpp-sdk-ec2-instance-connect
- aws-cpp-sdk-ec2-integration-tests
- aws-cpp-sdk-ecr
- aws-cpp-sdk-ecr-public
- aws-cpp-sdk-ecs
- aws-cpp-sdk-eks
- aws-cpp-sdk-elastic-inference
- aws-cpp-sdk-elasticache
- aws-cpp-sdk-elasticbeanstalk
- aws-cpp-sdk-elasticfilesystem
- aws-cpp-sdk-elasticfilesystem-integration-tests
- aws-cpp-sdk-elasticloadbalancing
- aws-cpp-sdk-elasticloadbalancingv2
- aws-cpp-sdk-elasticmapreduce
- aws-cpp-sdk-elastictranscoder
- aws-cpp-sdk-email
- aws-cpp-sdk-emr-containers
- aws-cpp-sdk-emr-serverless
- aws-cpp-sdk-es
- aws-cpp-sdk-eventbridge
- aws-cpp-sdk-eventbridge-tests
- aws-cpp-sdk-events
- aws-cpp-sdk-evidently
- aws-cpp-sdk-finspace
- aws-cpp-sdk-finspace-data
- aws-cpp-sdk-firehose
- aws-cpp-sdk-fis
- aws-cpp-sdk-fms
- aws-cpp-sdk-forecast
- aws-cpp-sdk-forecastquery
- aws-cpp-sdk-frauddetector
- aws-cpp-sdk-fsx
- aws-cpp-sdk-gamelift
- aws-cpp-sdk-gamesparks
- aws-cpp-sdk-glacier
- aws-cpp-sdk-globalaccelerator
- aws-cpp-sdk-glue
- aws-cpp-sdk-grafana
- aws-cpp-sdk-greengrass
- aws-cpp-sdk-greengrassv2
- aws-cpp-sdk-groundstation
- aws-cpp-sdk-guardduty
- aws-cpp-sdk-health
- aws-cpp-sdk-healthlake
- aws-cpp-sdk-honeycode
- aws-cpp-sdk-iam
- aws-cpp-sdk-identitystore
- aws-cpp-sdk-imagebuilder
- aws-cpp-sdk-importexport
- aws-cpp-sdk-inspector
- aws-cpp-sdk-inspector2
- aws-cpp-sdk-iot
- aws-cpp-sdk-iot-data
- aws-cpp-sdk-iot-jobs-data
- aws-cpp-sdk-iot-roborunner
- aws-cpp-sdk-iot1click-devices
- aws-cpp-sdk-iot1click-projects
- aws-cpp-sdk-iotanalytics
- aws-cpp-sdk-iotdeviceadvisor
- aws-cpp-sdk-iotevents
- aws-cpp-sdk-iotevents-data
- aws-cpp-sdk-iotfleethub
- aws-cpp-sdk-iotfleetwise
- aws-cpp-sdk-iotsecuretunneling
- aws-cpp-sdk-iotsitewise
- aws-cpp-sdk-iotthingsgraph
- aws-cpp-sdk-iottwinmaker
- aws-cpp-sdk-iotwireless
- aws-cpp-sdk-ivs
- aws-cpp-sdk-ivschat
- aws-cpp-sdk-kafka
- aws-cpp-sdk-kafkaconnect
- aws-cpp-sdk-kendra
- aws-cpp-sdk-kendra-ranking
- aws-cpp-sdk-keyspaces
- aws-cpp-sdk-kinesis
- aws-cpp-sdk-kinesis-integration-tests
- aws-cpp-sdk-kinesis-video-archived-media
- aws-cpp-sdk-kinesis-video-media
- aws-cpp-sdk-kinesis-video-signaling
- aws-cpp-sdk-kinesis-video-webrtc-storage
- aws-cpp-sdk-kinesisanalytics
- aws-cpp-sdk-kinesisanalyticsv2
- aws-cpp-sdk-kinesisvideo
- aws-cpp-sdk-kms
- aws-cpp-sdk-lakeformation
- aws-cpp-sdk-lambda
- aws-cpp-sdk-lambda-integration-tests
- aws-cpp-sdk-lex
- aws-cpp-sdk-lex-models
- aws-cpp-sdk-lexv2-models
- aws-cpp-sdk-lexv2-runtime
- aws-cpp-sdk-license-manager
- aws-cpp-sdk-license-manager-linux-subscriptions
- aws-cpp-sdk-license-manager-user-subscriptions
- aws-cpp-sdk-lightsail
- aws-cpp-sdk-location
- aws-cpp-sdk-logs
- aws-cpp-sdk-logs-integration-tests
- aws-cpp-sdk-lookoutequipment
- aws-cpp-sdk-lookoutmetrics
- aws-cpp-sdk-lookoutvision
- aws-cpp-sdk-m2
- aws-cpp-sdk-machinelearning
- aws-cpp-sdk-macie
- aws-cpp-sdk-macie2
- aws-cpp-sdk-managedblockchain
- aws-cpp-sdk-marketplace-catalog
- aws-cpp-sdk-marketplace-entitlement
- aws-cpp-sdk-marketplacecommerceanalytics
- aws-cpp-sdk-mediaconnect
- aws-cpp-sdk-mediaconvert
- aws-cpp-sdk-medialive
- aws-cpp-sdk-mediapackage
- aws-cpp-sdk-mediapackage-vod
- aws-cpp-sdk-mediastore
- aws-cpp-sdk-mediastore-data
- aws-cpp-sdk-mediastore-data-integration-tests
- aws-cpp-sdk-mediatailor
- aws-cpp-sdk-memorydb
- aws-cpp-sdk-meteringmarketplace
- aws-cpp-sdk-mgn
- aws-cpp-sdk-migration-hub-refactor-spaces
- aws-cpp-sdk-migrationhub-config
- aws-cpp-sdk-migrationhuborchestrator
- aws-cpp-sdk-migrationhubstrategy
- aws-cpp-sdk-mobile
- aws-cpp-sdk-monitoring
- aws-cpp-sdk-mq
- aws-cpp-sdk-mturk-requester
- aws-cpp-sdk-mwaa
- aws-cpp-sdk-neptune
- aws-cpp-sdk-network-firewall
- aws-cpp-sdk-networkmanager
- aws-cpp-sdk-nimble
- aws-cpp-sdk-oam
- aws-cpp-sdk-omics
- aws-cpp-sdk-opensearch
- aws-cpp-sdk-opensearchserverless
- aws-cpp-sdk-opsworks
- aws-cpp-sdk-opsworkscm
- aws-cpp-sdk-organizations
- aws-cpp-sdk-outposts
- aws-cpp-sdk-panorama
- aws-cpp-sdk-personalize
- aws-cpp-sdk-personalize-events
- aws-cpp-sdk-personalize-runtime
- aws-cpp-sdk-pi
- aws-cpp-sdk-pinpoint
- aws-cpp-sdk-pinpoint-email
- aws-cpp-sdk-pinpoint-sms-voice-v2
- aws-cpp-sdk-pipes
- aws-cpp-sdk-polly
- aws-cpp-sdk-polly-sample
- aws-cpp-sdk-pricing
- aws-cpp-sdk-privatenetworks
- aws-cpp-sdk-proton
- aws-cpp-sdk-qldb
- aws-cpp-sdk-qldb-session
- aws-cpp-sdk-queues
- aws-cpp-sdk-quicksight
- aws-cpp-sdk-ram
- aws-cpp-sdk-rbin
- aws-cpp-sdk-rds
- aws-cpp-sdk-rds-data
- aws-cpp-sdk-rds-integration-tests
- aws-cpp-sdk-redshift
- aws-cpp-sdk-redshift-data
- aws-cpp-sdk-redshift-integration-tests
- aws-cpp-sdk-redshift-serverless
- aws-cpp-sdk-rekognition
- aws-cpp-sdk-resiliencehub
- aws-cpp-sdk-resource-explorer-2
- aws-cpp-sdk-resource-groups
- aws-cpp-sdk-resourcegroupstaggingapi
- aws-cpp-sdk-robomaker
- aws-cpp-sdk-rolesanywhere
- aws-cpp-sdk-route53
- aws-cpp-sdk-route53-recovery-cluster
- aws-cpp-sdk-route53-recovery-control-config
- aws-cpp-sdk-route53-recovery-readiness
- aws-cpp-sdk-route53domains
- aws-cpp-sdk-route53resolver
- aws-cpp-sdk-rum
- aws-cpp-sdk-sagemaker
- aws-cpp-sdk-sagemaker-a2i-runtime
- aws-cpp-sdk-sagemaker-edge
- aws-cpp-sdk-sagemaker-featurestore-runtime
- aws-cpp-sdk-sagemaker-geospatial
- aws-cpp-sdk-sagemaker-metrics
- aws-cpp-sdk-sagemaker-runtime
- aws-cpp-sdk-savingsplans
- aws-cpp-sdk-scheduler
- aws-cpp-sdk-schemas
- aws-cpp-sdk-sdb
- aws-cpp-sdk-secretsmanager
- aws-cpp-sdk-securityhub
- aws-cpp-sdk-securitylake
- aws-cpp-sdk-serverlessrepo
- aws-cpp-sdk-service-quotas
- aws-cpp-sdk-servicecatalog
- aws-cpp-sdk-servicecatalog-appregistry
- aws-cpp-sdk-servicediscovery
- aws-cpp-sdk-sesv2
- aws-cpp-sdk-shield
- aws-cpp-sdk-signer
- aws-cpp-sdk-simspaceweaver
- aws-cpp-sdk-sms
- aws-cpp-sdk-sms-voice
- aws-cpp-sdk-snow-device-management
- aws-cpp-sdk-snowball
- aws-cpp-sdk-sns
- aws-cpp-sdk-sqs
- aws-cpp-sdk-sqs-integration-tests
- aws-cpp-sdk-ssm
- aws-cpp-sdk-ssm-contacts
- aws-cpp-sdk-ssm-incidents
- aws-cpp-sdk-ssm-sap
- aws-cpp-sdk-sso
- aws-cpp-sdk-sso-admin
- aws-cpp-sdk-sso-oidc
- aws-cpp-sdk-states
- aws-cpp-sdk-storagegateway
- aws-cpp-sdk-support
- aws-cpp-sdk-support-app
- aws-cpp-sdk-swf
- aws-cpp-sdk-synthetics
- aws-cpp-sdk-text-to-speech
- aws-cpp-sdk-text-to-speech-tests
- aws-cpp-sdk-textract
- aws-cpp-sdk-timestream-query
- aws-cpp-sdk-timestream-write
- aws-cpp-sdk-transcribe
- aws-cpp-sdk-transcribestreaming
- aws-cpp-sdk-transcribestreaming-integration-tests
- aws-cpp-sdk-translate
- aws-cpp-sdk-voice-id
- aws-cpp-sdk-waf
- aws-cpp-sdk-waf-regional
- aws-cpp-sdk-wafv2
- aws-cpp-sdk-wellarchitected
- aws-cpp-sdk-wisdom
- aws-cpp-sdk-workdocs
- aws-cpp-sdk-worklink
- aws-cpp-sdk-workmail
- aws-cpp-sdk-workmailmessageflow
- aws-cpp-sdk-workspaces
- aws-cpp-sdk-workspaces-web
- aws-cpp-sdk-xray
- code-generation
- crt
- doc_crosslinks
- doc_crosslinks_new
- doxygen
- generated
- scripts
- testing-resources)
diff --git a/cpp/cmake_modules/BuildUtils.cmake
b/cpp/cmake_modules/BuildUtils.cmake
index 32d962b0cf..134f47b12e 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -552,7 +552,7 @@ function(ADD_BENCHMARK REL_BENCHMARK_NAME)
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_BENCHMARK_NAME}.cc)
# This benchmark has a corresponding .cc file, set it up as an executable.
- set(BENCHMARK_PATH "${EXECUTABLE_OUTPUT_PATH}/${BENCHMARK_NAME}")
+ set(BENCHMARK_PATH "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${BENCHMARK_NAME}")
add_executable(${BENCHMARK_NAME} ${SOURCES})
if(ARG_STATIC_LINK_LIBS)
@@ -581,7 +581,8 @@ function(ADD_BENCHMARK REL_BENCHMARK_NAME)
PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE
INSTALL_RPATH_USE_LINK_PATH TRUE
INSTALL_RPATH
-
"$ENV{CONDA_PREFIX}/lib;${EXECUTABLE_OUTPUT_PATH}")
+
"$ENV{CONDA_PREFIX}/lib;${CMAKE_RUNTIME_OUTPUT_DIRECTORY}"
+ )
endif()
# Add test as dependency of relevant label targets
@@ -682,7 +683,7 @@ function(ADD_TEST_CASE REL_TEST_NAME)
# Make sure the executable name contains only hyphens, not underscores
string(REPLACE "_" "-" TEST_NAME ${TEST_NAME})
- set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
+ set(TEST_PATH "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TEST_NAME}")
add_executable(${TEST_NAME} ${SOURCES})
# With OSX and conda, we need to set the correct RPATH so that dependencies
@@ -695,7 +696,8 @@ function(ADD_TEST_CASE REL_TEST_NAME)
PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE
INSTALL_RPATH_USE_LINK_PATH TRUE
INSTALL_RPATH
-
"${EXECUTABLE_OUTPUT_PATH};$ENV{CONDA_PREFIX}/lib")
+
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY};$ENV{CONDA_PREFIX}/lib"
+ )
endif()
# Ensure using bundled GoogleTest when we use bundled GoogleTest.
@@ -826,7 +828,7 @@ function(ADD_ARROW_EXAMPLE REL_EXAMPLE_NAME)
if(EXISTS ${CMAKE_SOURCE_DIR}/examples/arrow/${REL_EXAMPLE_NAME}.cc)
# This example has a corresponding .cc file, set it up as an executable.
- set(EXAMPLE_PATH "${EXECUTABLE_OUTPUT_PATH}/${EXAMPLE_NAME}")
+ set(EXAMPLE_PATH "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${EXAMPLE_NAME}")
add_executable(${EXAMPLE_NAME} "${REL_EXAMPLE_NAME}.cc"
${ARG_EXTRA_SOURCES})
target_link_libraries(${EXAMPLE_NAME} ${ARROW_EXAMPLE_LINK_LIBS})
add_dependencies(runexample ${EXAMPLE_NAME})
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake
b/cpp/cmake_modules/SetupCxxFlags.cmake
index fdb28b540e..9ecca91ecd 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -152,8 +152,6 @@ set(CMAKE_CXX_EXTENSIONS OFF)
# shared libraries
set(CMAKE_POSITION_INDEPENDENT_CODE ${ARROW_POSITION_INDEPENDENT_CODE})
-string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
-
set(UNKNOWN_COMPILER_MESSAGE
"Unknown compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}")
@@ -265,7 +263,7 @@ endif()
# `RELEASE`, then it will default to `PRODUCTION`. The goal of defaulting to
# `CHECKIN` is to avoid friction with long response time from CI.
if(NOT BUILD_WARNING_LEVEL)
- if("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
+ if("${UPPERCASE_BUILD_TYPE}" STREQUAL "RELEASE")
set(BUILD_WARNING_LEVEL PRODUCTION)
else()
set(BUILD_WARNING_LEVEL CHECKIN)
@@ -602,7 +600,7 @@ if(NOT WIN32 AND NOT APPLE)
if(MUST_USE_GOLD)
message(STATUS "Using hard-wired gold linker (version ${GOLD_VERSION})")
if(ARROW_BUGGY_GOLD)
- if("${ARROW_LINK}" STREQUAL "d" AND "${CMAKE_BUILD_TYPE}" STREQUAL
"RELEASE")
+ if("${ARROW_LINK}" STREQUAL "d" AND "${UPPERCASE_BUILD_TYPE}" STREQUAL
"RELEASE")
message(SEND_ERROR "Configured to use buggy gold with dynamic
linking "
"in a RELEASE build")
endif()
@@ -808,7 +806,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
set(CMAKE_SHARED_LINKER_FLAGS "-sSIDE_MODULE=1
${ARROW_EMSCRIPTEN_LINKER_FLAGS}")
if(ARROW_TESTING)
# flags for building test executables for use in node
- if("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
+ if("${UPPERCASE_BUILD_TYPE}" STREQUAL "RELEASE")
set(CMAKE_EXE_LINKER_FLAGS
"${ARROW_EMSCRIPTEN_LINKER_FLAGS} -sALLOW_MEMORY_GROWTH -lnodefs.js
-lnoderawfs.js --pre-js ${BUILD_SUPPORT_DIR}/emscripten-test-init.js"
)
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake
b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 193711a6fb..5264beed78 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -1012,7 +1012,7 @@ endif()
set(MAKE_BUILD_ARGS "-j${NPROC}")
include(FetchContent)
-set(FC_DECLARE_COMMON_OPTIONS)
+set(FC_DECLARE_COMMON_OPTIONS SYSTEM)
if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.28)
list(APPEND FC_DECLARE_COMMON_OPTIONS EXCLUDE_FROM_ALL TRUE)
endif()
@@ -1020,13 +1020,17 @@ endif()
macro(prepare_fetchcontent)
set(BUILD_SHARED_LIBS OFF)
set(BUILD_STATIC_LIBS ON)
- set(CMAKE_COMPILE_WARNING_AS_ERROR FALSE)
- set(CMAKE_EXPORT_NO_PACKAGE_REGISTRY TRUE)
+ set(BUILD_TESTING OFF)
+ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "")
+ set(CMAKE_COMPILE_WARNING_AS_ERROR OFF)
+ set(CMAKE_EXPORT_NO_PACKAGE_REGISTRY OFF)
+ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "")
set(CMAKE_MACOSX_RPATH ${ARROW_INSTALL_NAME_RPATH})
# We set CMAKE_POLICY_VERSION_MINIMUM temporarily due to failures with CMake
4
# We should remove it once we have updated the dependencies:
# https://github.com/apache/arrow/issues/45985
set(CMAKE_POLICY_VERSION_MINIMUM 3.5)
+ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "")
if(MSVC)
string(REPLACE "/WX" "" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}")
@@ -1058,7 +1062,7 @@ macro(build_boost)
if(ARROW_BOOST_REQUIRE_LIBRARY)
set(BOOST_LIB_DIR "${BOOST_PREFIX}/stage/lib")
set(BOOST_BUILD_LINK "static")
- if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+ if("${UPPERCASE_BUILD_TYPE}" STREQUAL "DEBUG")
set(BOOST_BUILD_VARIANT "debug")
else()
set(BOOST_BUILD_VARIANT "release")
@@ -2657,11 +2661,13 @@ if(ARROW_WITH_ZLIB)
resolve_dependency(ZLIB PC_PACKAGE_NAMES zlib)
endif()
-macro(build_lz4)
+function(build_lz4)
message(STATUS "Building LZ4 from source using FetchContent")
# Set LZ4 as vendored
- set(LZ4_VENDORED TRUE)
+ set(LZ4_VENDORED
+ TRUE
+ PARENT_SCOPE)
# Declare the content
fetchcontent_declare(lz4
@@ -2692,7 +2698,10 @@ macro(build_lz4)
# Add to bundled static libs.
# We must use lz4_static (not imported target) not LZ4::lz4 (imported
target).
list(APPEND ARROW_BUNDLED_STATIC_LIBS lz4_static)
-endmacro()
+ set(ARROW_BUNDLED_STATIC_LIBS
+ ${ARROW_BUNDLED_STATIC_LIBS}
+ PARENT_SCOPE)
+endfunction()
if(ARROW_WITH_LZ4)
resolve_dependency(lz4
@@ -4778,7 +4787,7 @@ function(build_orc)
${ARROW_PROTOBUF_PROTOC}
${ARROW_ZSTD_LIBZSTD}
${Snappy_TARGET}
- LZ4::lz4
+ ${ORC_LZ4_TARGET}
ZLIB::ZLIB)
add_library(orc::orc STATIC IMPORTED)
set_target_properties(orc::orc PROPERTIES IMPORTED_LOCATION
"${ORC_STATIC_LIB}")
@@ -5055,439 +5064,217 @@ endif()
# ----------------------------------------------------------------------
# AWS SDK for C++
-include(AWSSDKVariables)
+function(build_awssdk)
+ message(STATUS "Building AWS SDK for C++ from source")
-macro(build_awssdk)
- message(STATUS "Building AWS C++ SDK from source")
- set(AWSSDK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/awssdk_ep-install")
- set(AWSSDK_INCLUDE_DIR "${AWSSDK_PREFIX}/include")
-
- # The AWS SDK has a few warnings around shortening lengths
- set(AWS_C_FLAGS "${EP_C_FLAGS}")
- set(AWS_CXX_FLAGS "${EP_CXX_FLAGS}")
- if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID
STREQUAL
- "Clang")
- # Negate warnings that AWS SDK cannot build under
- string(APPEND AWS_C_FLAGS " -Wno-error=shorten-64-to-32")
- string(APPEND AWS_CXX_FLAGS " -Wno-error=shorten-64-to-32")
- endif()
- if(NOT MSVC)
- string(APPEND AWS_C_FLAGS " -Wno-deprecated")
- string(APPEND AWS_CXX_FLAGS " -Wno-deprecated")
+ # aws-c-common must be the first product because others depend on
+ # this.
+ set(AWSSDK_PRODUCTS aws-c-common)
+ if(LINUX)
+ list(APPEND AWSSDK_PRODUCTS aws-lc s2n-tls)
endif()
- # GH-44950: This is required to build under Rtools40 and we may be able to
- # remove it if/when we no longer need to build under Rtools40
- if(WIN32 AND NOT MSVC)
- string(APPEND
- AWS_C_FLAGS
- " -D_WIN32_WINNT=0x0601 -D__USE_MINGW_ANSI_STDIO=1 -Wno-error
-Wno-error=format= -Wno-error=format-extra-args -Wno-unused-local-typedefs
-Wno-unused-variable"
- )
- string(APPEND
- AWS_CXX_FLAGS
- " -D_WIN32_WINNT=0x0601 -D__USE_MINGW_ANSI_STDIO=1 -Wno-error
-Wno-error=format= -Wno-error=format-extra-args -Wno-unused-local-typedefs
-Wno-unused-variable"
+ list(APPEND
+ AWSSDK_PRODUCTS
+ # We can't sort this in alphabetical order because some
+ # products depend on other products.
+ aws-checksums
+ aws-c-cal
+ aws-c-io
+ aws-c-event-stream
+ aws-c-sdkutils
+ aws-c-compression
+ aws-c-http
+ aws-c-mqtt
+ aws-c-auth
+ aws-c-s3
+ aws-crt-cpp
+ aws-sdk-cpp)
+ set(AWS_SDK_CPP_SOURCE_URL "${AWSSDK_SOURCE_URL}")
+ set(ARROW_AWS_SDK_CPP_BUILD_SHA256_CHECKSUM
"${ARROW_AWSSDK_BUILD_SHA256_CHECKSUM}")
+ foreach(AWSSDK_PRODUCT ${AWSSDK_PRODUCTS})
+ # aws-c-cal ->
+ # AWS-C-CAL
+ string(TOUPPER "${AWSSDK_PRODUCT}" BASE_VARIABLE_NAME)
+ # AWS-C-CAL ->
+ # AWS_C_CAL
+ string(REGEX REPLACE "-" "_" BASE_VARIABLE_NAME "${BASE_VARIABLE_NAME}")
+ if(MINGW AND AWSSDK_PRODUCT STREQUAL "aws-c-common")
+ find_program(PATCH patch REQUIRED)
+ set(${BASE_VARIABLE_NAME}_PATCH_COMMAND
+ ${PATCH} -p1 -i ${CMAKE_CURRENT_LIST_DIR}/aws-c-common-1208.patch)
+ endif()
+ fetchcontent_declare(${AWSSDK_PRODUCT}
+ ${FC_DECLARE_COMMON_OPTIONS} OVERRIDE_FIND_PACKAGE
+ PATCH_COMMAND ${${BASE_VARIABLE_NAME}_PATCH_COMMAND}
+ URL ${${BASE_VARIABLE_NAME}_SOURCE_URL}
+ URL_HASH
"SHA256=${ARROW_${BASE_VARIABLE_NAME}_BUILD_SHA256_CHECKSUM}"
)
- endif()
+ endforeach()
- set(AWSSDK_COMMON_CMAKE_ARGS
- ${EP_COMMON_CMAKE_ARGS}
- -DCMAKE_C_FLAGS=${AWS_C_FLAGS}
- -DCMAKE_CXX_FLAGS=${AWS_CXX_FLAGS}
- -DCPP_STANDARD=${CMAKE_CXX_STANDARD}
- -DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}
- -DCMAKE_PREFIX_PATH=${AWSSDK_PREFIX}
- -DENABLE_TESTING=OFF
- -DENABLE_UNITY_BUILD=ON
- -DOPENSSL_CRYPTO_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}
- -DOPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
- -DOPENSSL_SSL_LIBRARY=${OPENSSL_SSL_LIBRARY}
- -Dcrypto_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
- -Dcrypto_LIBRARY=${OPENSSL_CRYPTO_LIBRARY})
- if(ARROW_OPENSSL_USE_SHARED)
- list(APPEND AWSSDK_COMMON_CMAKE_ARGS
- -Dcrypto_SHARED_LIBRARY=${OPENSSL_CRYPTO_LIBRARY})
- else()
- list(APPEND AWSSDK_COMMON_CMAKE_ARGS
- -Dcrypto_STATIC_LIBRARY=${OPENSSL_CRYPTO_LIBRARY})
- endif()
- set(AWSSDK_CMAKE_ARGS
- ${AWSSDK_COMMON_CMAKE_ARGS}
- -DBUILD_DEPS=OFF
-
-DBUILD_ONLY=config\\$<SEMICOLON>s3\\$<SEMICOLON>transfer\\$<SEMICOLON>identity-management\\$<SEMICOLON>sts
- -DMINIMIZE_SIZE=ON)
- # Remove unused directories to save build directory storage.
- # 807MB -> 31MB
- set(AWSSDK_PATCH_COMMAND ${CMAKE_COMMAND} -E)
- if(CMAKE_VERSION VERSION_LESS 3.17)
- list(APPEND AWSSDK_PATCH_COMMAND remove_directory)
- else()
- list(APPEND AWSSDK_PATCH_COMMAND rm -rf)
+ prepare_fetchcontent()
+ set(BUILD_DEPS OFF)
+ set(BUILD_TOOL OFF)
+ set(ENABLE_TESTING OFF)
+ set(IN_SOURCE_BUILD ON)
+ set(MINIMIZE_SIZE ON)
+ set(USE_OPENSSL ON)
+
+ # For aws-c-common
+ if(MINGW)
+ # PPROCESSOR_NUMBER requires Windows 7 or later.
+ #
+ # 0x0601 == _WIN32_WINNT_WIN7
+ string(APPEND CMAKE_C_FLAGS " -D_WIN32_WINNT=0x0601")
+ string(APPEND CMAKE_CXX_FLAGS " -D_WIN32_WINNT=0x0601")
endif()
- list(APPEND AWSSDK_PATCH_COMMAND ${AWSSDK_UNUSED_DIRECTORIES})
- # Patch parts of the AWSSDK EP so it builds cleanly under Rtools40
- if(WIN32 AND NOT MSVC)
- find_program(PATCH patch REQUIRED)
- # Patch aws_c_common to build under Rtools40
- set(AWS_C_COMMON_PATCH_COMMAND ${PATCH} -p1 -i
-
${CMAKE_SOURCE_DIR}/../ci/rtools/aws_c_common_ep.patch)
- message(STATUS "Hello ${AWS_C_COMMON_PATCH_COMMAND}")
- # aws_c_io_ep to build under Rtools40
- set(AWS_C_IO_PATCH_COMMAND ${PATCH} -p1 -i
-
${CMAKE_SOURCE_DIR}/../ci/rtools/aws_c_io_ep.patch)
- message(STATUS "Hello ${AWS_C_IO_PATCH_COMMAND}")
- # awssdk_ep to build under Rtools40
- list(APPEND
- AWSSDK_PATCH_COMMAND
- &&
- ${PATCH}
- -p1
- -i
- ${CMAKE_SOURCE_DIR}/../ci/rtools/awssdk_ep.patch)
- message(STATUS "Hello ${AWSSDK_PATCH_COMMAND}")
- endif()
+ # For aws-lc
+ set(DISABLE_GO ON)
+ set(DISABLE_PERL ON)
- if(UNIX)
- # on Linux and macOS curl seems to be required
- find_curl()
- get_filename_component(CURL_ROOT_HINT "${CURL_INCLUDE_DIRS}" DIRECTORY)
- get_filename_component(ZLIB_ROOT_HINT "${ZLIB_INCLUDE_DIRS}" DIRECTORY)
+ # For s2n-tls
+ set(crypto_INCLUDE_DIR
"$<TARGET_PROPERTY:crypto,INTERFACE_INCLUDE_DIRECTORIES>")
+ set(crypto_STATIC_LIBRARY "$<TARGET_FILE:crypto>")
+ set(S2N_INTERN_LIBCRYPTO ON)
- # provide hint for AWS SDK to link with the already located libcurl and
zlib
- list(APPEND
- AWSSDK_CMAKE_ARGS
- -DCURL_INCLUDE_DIR=${CURL_ROOT_HINT}/include
- -DCURL_LIBRARY=${CURL_ROOT_HINT}/lib
- -DZLIB_INCLUDE_DIR=${ZLIB_ROOT_HINT}/include
- -DZLIB_LIBRARY=${ZLIB_ROOT_HINT}/lib)
- endif()
-
- file(MAKE_DIRECTORY ${AWSSDK_INCLUDE_DIR})
-
- # AWS C++ SDK related libraries to link statically
- set(_AWSSDK_LIBS
- aws-cpp-sdk-identity-management
- aws-cpp-sdk-sts
- aws-cpp-sdk-cognito-identity
- aws-cpp-sdk-s3
- aws-cpp-sdk-core
- aws-crt-cpp
- aws-c-s3
- aws-c-auth
- aws-c-mqtt
- aws-c-http
- aws-c-compression
- aws-c-sdkutils
- aws-c-event-stream
- aws-c-io
- aws-c-cal
- aws-checksums
- aws-c-common)
-
- # aws-lc needs to be installed on a separate folder to hide from unintended
use
- set(AWS_LC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/aws_lc_ep-install")
- set(AWS_LC_INCLUDE_DIR "${AWS_LC_PREFIX}/include")
-
- if(UNIX AND NOT APPLE) # aws-lc and s2n-tls only needed on linux
- file(MAKE_DIRECTORY ${AWS_LC_INCLUDE_DIR})
- list(APPEND _AWSSDK_LIBS s2n-tls aws-lc)
- endif()
-
- set(AWSSDK_LIBRARIES)
- foreach(_AWSSDK_LIB ${_AWSSDK_LIBS})
- # aws-c-common -> AWS-C-COMMON
- string(TOUPPER ${_AWSSDK_LIB} _AWSSDK_LIB_UPPER)
- # AWS-C-COMMON -> AWS_C_COMMON
- string(REPLACE "-" "_" _AWSSDK_LIB_NAME_PREFIX ${_AWSSDK_LIB_UPPER})
- set(_AWSSDK_STATIC_LIBRARY
-
"${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${_AWSSDK_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}"
- )
- if(${_AWSSDK_LIB} STREQUAL "s2n-tls") # Build output of s2n-tls is libs2n.a
- set(_AWSSDK_STATIC_LIBRARY
-
"${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}s2n${CMAKE_STATIC_LIBRARY_SUFFIX}"
- )
- elseif(${_AWSSDK_LIB} STREQUAL "aws-lc") # We only need libcrypto from
aws-lc
- set(_AWSSDK_STATIC_LIBRARY
-
"${AWS_LC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}crypto${CMAKE_STATIC_LIBRARY_SUFFIX}"
- )
- endif()
- if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-")
- set(_AWSSDK_TARGET_NAME ${_AWSSDK_LIB})
- elseif(${_AWSSDK_LIB} STREQUAL "aws-lc")
- set(_AWSSDK_TARGET_NAME AWS::crypto)
- else()
- set(_AWSSDK_TARGET_NAME AWS::${_AWSSDK_LIB})
- endif()
- add_library(${_AWSSDK_TARGET_NAME} STATIC IMPORTED)
- set_target_properties(${_AWSSDK_TARGET_NAME} PROPERTIES IMPORTED_LOCATION
-
${_AWSSDK_STATIC_LIBRARY})
- target_include_directories(${_AWSSDK_TARGET_NAME} BEFORE
- INTERFACE "${AWSSDK_INCLUDE_DIR}")
- if(${_AWSSDK_LIB} STREQUAL "aws-lc")
- set_target_properties(${_AWSSDK_TARGET_NAME} PROPERTIES IMPORTED_LOCATION
-
${_AWSSDK_STATIC_LIBRARY})
- target_include_directories(${_AWSSDK_TARGET_NAME} BEFORE
- INTERFACE "${AWS_LC_INCLUDE_DIR}")
- endif()
- set("${_AWSSDK_LIB_NAME_PREFIX}_STATIC_LIBRARY" ${_AWSSDK_STATIC_LIBRARY})
+ # For aws-lc and s2n-tls
+ #
+ # Link time optimization is causing trouble like GH-34349
+ string(REPLACE "-flto=auto" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
+ string(REPLACE "-ffat-lto-objects" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
- if(NOT ${_AWSSDK_LIB} STREQUAL "aws-lc")
- # aws-lc only linked against s2n but not arrow
- list(APPEND AWSSDK_LIBRARIES ${_AWSSDK_TARGET_NAME})
- endif()
- endforeach()
+ # For aws-c-io
+ if(MINGW AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9")
+ # This is for RTools 40. We can remove this after we dropped
+ # support for R < 4.2. schannel.h in RTools 40 is old.
- externalproject_add(aws_c_common_ep
- ${EP_COMMON_OPTIONS}
- URL ${AWS_C_COMMON_SOURCE_URL}
- URL_HASH
"SHA256=${ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM}"
- PATCH_COMMAND ${AWS_C_COMMON_PATCH_COMMAND}
- CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
- BUILD_BYPRODUCTS ${AWS_C_COMMON_STATIC_LIBRARY})
- add_dependencies(AWS::aws-c-common aws_c_common_ep)
-
- set(AWS_CHECKSUMS_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS})
- if(NOT WIN32)
- # On non-Windows, always build in release mode.
- # Especially with gcc, debug builds can fail with "asm constraint" errors:
- # https://github.com/TileDB-Inc/TileDB/issues/1351
- list(APPEND AWS_CHECKSUMS_CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release)
+ # For schannel.h
+ #
+ # See also:
+ #
https://learn.microsoft.com/en-us/windows/win32/api/schannel/ns-schannel-schannel_cred
+ string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_0_SERVER=0x00000040")
+ string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_0_CLIENT=0x00000080")
+ string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_1_SERVER=0x00000100")
+ string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_1_CLIENT=0x00000200")
+ string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_2_SERVER=0x00000400")
+ string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_2_CLIENT=0x00000800")
+ string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_3_SERVER=0x00001000")
+ string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_3_CLIENT=0x00002000")
+ string(APPEND CMAKE_C_FLAGS " -DSCH_USE_STRONG_CRYPTO=0x00400000")
+
+ # For sspi.h
+ #
+ # See also:
+ #
https://learn.microsoft.com/en-us/windows/win32/api/sspi/ne-sspi-sec_application_protocol_negotiation_ext
+ string(APPEND CMAKE_C_FLAGS "
-DSecApplicationProtocolNegotiationExt_ALPN=2")
+ # See also:
+ #
https://learn.microsoft.com/en-us/windows/win32/api/sspi/ns-sspi-secbuffer
+ string(APPEND CMAKE_C_FLAGS " -DSECBUFFER_ALERT=17")
endif()
- externalproject_add(aws_checksums_ep
- ${EP_COMMON_OPTIONS}
- URL ${AWS_CHECKSUMS_SOURCE_URL}
- URL_HASH
"SHA256=${ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM}"
- CMAKE_ARGS ${AWS_CHECKSUMS_CMAKE_ARGS}
- BUILD_BYPRODUCTS ${AWS_CHECKSUMS_STATIC_LIBRARY}
- DEPENDS aws_c_common_ep)
- add_dependencies(AWS::aws-checksums aws_checksums_ep)
-
- if("s2n-tls" IN_LIST _AWSSDK_LIBS)
- # Remove unused directories to save build directory storage.
- # 169MB -> 105MB
- set(AWS_LC_PATCH_COMMAND ${CMAKE_COMMAND} -E)
- if(CMAKE_VERSION VERSION_LESS 3.17)
- list(APPEND AWS_LC_PATCH_COMMAND remove_directory)
- else()
- list(APPEND AWS_LC_PATCH_COMMAND rm -rf)
- endif()
- list(APPEND AWS_LC_PATCH_COMMAND fuzz)
- set(AWS_LC_C_FLAGS ${EP_C_FLAGS})
- string(APPEND AWS_LC_C_FLAGS " -Wno-error=overlength-strings
-Wno-error=pedantic")
- # Link time optimization is causing trouble like #34349
- string(REPLACE "-flto=auto" "" AWS_LC_C_FLAGS "${AWS_LC_C_FLAGS}")
- string(REPLACE "-ffat-lto-objects" "" AWS_LC_C_FLAGS "${AWS_LC_C_FLAGS}")
-
- set(AWS_LC_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS})
- list(APPEND AWS_LC_CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${AWS_LC_PREFIX}
- -DCMAKE_C_FLAGS=${AWS_LC_C_FLAGS})
-
- externalproject_add(aws_lc_ep
- ${EP_COMMON_OPTIONS}
- URL ${AWS_LC_SOURCE_URL}
- URL_HASH "SHA256=${ARROW_AWS_LC_BUILD_SHA256_CHECKSUM}"
- PATCH_COMMAND ${AWS_LC_PATCH_COMMAND}
- CMAKE_ARGS ${AWS_LC_CMAKE_ARGS}
- BUILD_BYPRODUCTS ${AWS_LC_STATIC_LIBRARY})
- add_dependencies(AWS::crypto aws_lc_ep)
-
- set(S2N_TLS_C_FLAGS ${EP_C_FLAGS})
- # Link time optimization is causing trouble like #34349
- string(REPLACE "-flto=auto" "" S2N_TLS_C_FLAGS "${S2N_TLS_C_FLAGS}")
- string(REPLACE "-ffat-lto-objects" "" S2N_TLS_C_FLAGS "${S2N_TLS_C_FLAGS}")
-
- set(S2N_TLS_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS})
- list(APPEND
- S2N_TLS_CMAKE_ARGS
- # internalize libcrypto to avoid name conflict with OpenSSL
- -DS2N_INTERN_LIBCRYPTO=ON
- # path to find crypto provided by aws-lc
- -DCMAKE_PREFIX_PATH=${AWS_LC_PREFIX}
- -DCMAKE_C_FLAGS=${S2N_TLS_C_FLAGS}
- # paths to find crypto provided by aws-lc
- -Dcrypto_INCLUDE_DIR=${AWS_LC_PREFIX}/include
- -Dcrypto_LIBRARY=${AWS_LC_STATIC_LIBRARY}
- -Dcrypto_STATIC_LIBRARY=${AWS_LC_STATIC_LIBRARY})
-
- externalproject_add(s2n_tls_ep
- ${EP_COMMON_OPTIONS}
- URL ${S2N_TLS_SOURCE_URL}
- URL_HASH
"SHA256=${ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM}"
- CMAKE_ARGS ${S2N_TLS_CMAKE_ARGS}
- BUILD_BYPRODUCTS ${S2N_TLS_STATIC_LIBRARY}
- DEPENDS aws_lc_ep)
- add_dependencies(AWS::s2n-tls s2n_tls_ep)
+ # For aws-sdk-cpp
+ #
+ # We need to use CACHE variables because aws-sdk-cpp < 12.0.0 uses
+ # CMP0077 OLD policy. We can use normal variables when we use
+ # aws-sdk-cpp >= 12.0.0.
+ set(AWS_SDK_WARNINGS_ARE_ERRORS
+ OFF
+ CACHE BOOL "" FORCE)
+ set(BUILD_DEPS
+ OFF
+ CACHE BOOL "" FORCE)
+ set(BUILD_ONLY
+ ""
+ CACHE STRING "" FORCE)
+ list(APPEND
+ BUILD_ONLY
+ config
+ core
+ identity-management
+ s3
+ sts
+ transfer)
+ set(BUILD_SHARED_LIBS
+ OFF
+ CACHE BOOL "" FORCE)
+ set(ENABLE_TESTING
+ OFF
+ CACHE BOOL "" FORCE)
+ if(NOT WIN32)
+ set(ZLIB_INCLUDE_DIR
+ "$<TARGET_PROPERTY:ZLIB::ZLIB,INTERFACE_INCLUDE_DIRECTORIES>"
+ CACHE STRING "" FORCE)
+ set(ZLIB_LIBRARY
+ "$<TARGET_FILE:ZLIB::ZLIB>"
+ CACHE STRING "" FORCE)
endif()
+ if(MINGW AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9")
+ # This is for RTools 40. We can remove this after we dropped
+ # support for R < 4.2. schannel.h in RTools 40 is old.
- externalproject_add(aws_c_cal_ep
- ${EP_COMMON_OPTIONS}
- URL ${AWS_C_CAL_SOURCE_URL}
- URL_HASH
"SHA256=${ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM}"
- CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
- BUILD_BYPRODUCTS ${AWS_C_CAL_STATIC_LIBRARY}
- DEPENDS aws_c_common_ep)
- add_dependencies(AWS::aws-c-cal aws_c_cal_ep)
-
- set(AWS_C_IO_DEPENDS aws_c_common_ep aws_c_cal_ep)
- if(TARGET s2n_tls_ep)
- list(APPEND AWS_C_IO_DEPENDS s2n_tls_ep)
- endif()
- externalproject_add(aws_c_io_ep
- ${EP_COMMON_OPTIONS}
- URL ${AWS_C_IO_SOURCE_URL}
- URL_HASH "SHA256=${ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM}"
- PATCH_COMMAND ${AWS_C_IO_PATCH_COMMAND}
- CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
- BUILD_BYPRODUCTS ${AWS_C_IO_STATIC_LIBRARY}
- DEPENDS ${AWS_C_IO_DEPENDS})
- add_dependencies(AWS::aws-c-io aws_c_io_ep)
-
- externalproject_add(aws_c_event_stream_ep
- ${EP_COMMON_OPTIONS}
- URL ${AWS_C_EVENT_STREAM_SOURCE_URL}
- URL_HASH
"SHA256=${ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM}"
- CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
- BUILD_BYPRODUCTS ${AWS_C_EVENT_STREAM_STATIC_LIBRARY}
- DEPENDS aws_checksums_ep aws_c_io_ep)
- add_dependencies(AWS::aws-c-event-stream aws_c_event_stream_ep)
-
- externalproject_add(aws_c_sdkutils_ep
- ${EP_COMMON_OPTIONS}
- URL ${AWS_C_SDKUTILS_SOURCE_URL}
- URL_HASH
"SHA256=${ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM}"
- CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
- BUILD_BYPRODUCTS ${AWS_C_SDKUTILS_STATIC_LIBRARY}
- DEPENDS aws_c_common_ep)
- add_dependencies(AWS::aws-c-sdkutils aws_c_sdkutils_ep)
-
- externalproject_add(aws_c_compression_ep
- ${EP_COMMON_OPTIONS}
- URL ${AWS_C_COMPRESSION_SOURCE_URL}
- URL_HASH
"SHA256=${ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM}"
- CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
- BUILD_BYPRODUCTS ${AWS_C_COMPRESSION_STATIC_LIBRARY}
- DEPENDS aws_c_common_ep)
- add_dependencies(AWS::aws-c-compression aws_c_compression_ep)
-
- externalproject_add(aws_c_http_ep
- ${EP_COMMON_OPTIONS}
- URL ${AWS_C_HTTP_SOURCE_URL}
- URL_HASH
"SHA256=${ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM}"
- CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
- BUILD_BYPRODUCTS ${AWS_C_HTTP_STATIC_LIBRARY}
- DEPENDS aws_c_io_ep aws_c_compression_ep)
- add_dependencies(AWS::aws-c-http aws_c_http_ep)
-
- externalproject_add(aws_c_mqtt_ep
- ${EP_COMMON_OPTIONS}
- URL ${AWS_C_MQTT_SOURCE_URL}
- URL_HASH
"SHA256=${ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM}"
- CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
- BUILD_BYPRODUCTS ${AWS_C_MQTT_STATIC_LIBRARY}
- DEPENDS aws_c_http_ep)
- add_dependencies(AWS::aws-c-mqtt aws_c_mqtt_ep)
-
- externalproject_add(aws_c_auth_ep
- ${EP_COMMON_OPTIONS}
- URL ${AWS_C_AUTH_SOURCE_URL}
- URL_HASH
"SHA256=${ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM}"
- CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
- BUILD_BYPRODUCTS ${AWS_C_AUTH_STATIC_LIBRARY}
- DEPENDS aws_c_sdkutils_ep aws_c_cal_ep aws_c_http_ep)
- add_dependencies(AWS::aws-c-auth aws_c_auth_ep)
-
- externalproject_add(aws_c_s3_ep
- ${EP_COMMON_OPTIONS}
- URL ${AWS_C_S3_SOURCE_URL}
- URL_HASH "SHA256=${ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM}"
- CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
- BUILD_BYPRODUCTS ${AWS_C_S3_STATIC_LIBRARY}
- DEPENDS aws_checksums_ep aws_c_auth_ep)
- add_dependencies(AWS::aws-c-s3 aws_c_s3_ep)
-
- externalproject_add(aws_crt_cpp_ep
- ${EP_COMMON_OPTIONS}
- URL ${AWS_CRT_CPP_SOURCE_URL}
- URL_HASH
"SHA256=${ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM}"
- CMAKE_ARGS ${AWSSDK_CMAKE_ARGS}
- BUILD_BYPRODUCTS ${AWS_CRT_CPP_STATIC_LIBRARY}
- DEPENDS aws_c_auth_ep
- aws_c_cal_ep
- aws_c_common_ep
- aws_c_event_stream_ep
- aws_c_http_ep
- aws_c_io_ep
- aws_c_mqtt_ep
- aws_c_s3_ep
- aws_checksums_ep)
- add_dependencies(AWS::aws-crt-cpp aws_crt_cpp_ep)
-
- externalproject_add(awssdk_ep
- ${EP_COMMON_OPTIONS}
- URL ${AWSSDK_SOURCE_URL}
- URL_HASH "SHA256=${ARROW_AWSSDK_BUILD_SHA256_CHECKSUM}"
- PATCH_COMMAND ${AWSSDK_PATCH_COMMAND}
- CMAKE_ARGS ${AWSSDK_CMAKE_ARGS}
- BUILD_BYPRODUCTS
${AWS_CPP_SDK_COGNITO_IDENTITY_STATIC_LIBRARY}
- ${AWS_CPP_SDK_CORE_STATIC_LIBRARY}
-
${AWS_CPP_SDK_IDENTITY_MANAGEMENT_STATIC_LIBRARY}
- ${AWS_CPP_SDK_S3_STATIC_LIBRARY}
- ${AWS_CPP_SDK_STS_STATIC_LIBRARY}
- DEPENDS aws_crt_cpp_ep)
- foreach(_AWSSDK_LIB ${_AWSSDK_LIBS})
- if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-")
- add_dependencies(${_AWSSDK_LIB} awssdk_ep)
+ # For winhttp.h
+ #
+ # See also:
+ # https://learn.microsoft.com/en-us/windows/win32/winhttp/error-messages
+ string(APPEND CMAKE_CXX_FLAGS "
-DERROR_WINHTTP_UNHANDLED_SCRIPT_TYPE=12176")
+ string(APPEND CMAKE_CXX_FLAGS "
-DERROR_WINHTTP_SCRIPT_EXECUTION_ERROR=12177")
+ # See also:
+ #
https://learn.microsoft.com/en-us/windows/win32/api/winhttp/ns-winhttp-winhttp_async_result
+ string(APPEND CMAKE_CXX_FLAGS " -DAPI_GET_PROXY_FOR_URL=6")
+ # See also:
+ #
https://learn.microsoft.com/en-us/windows/win32/api/winhttp/nc-winhttp-winhttp_status_callback
+ string(APPEND CMAKE_CXX_FLAGS "
-DWINHTTP_CALLBACK_STATUS_CLOSE_COMPLETE=0x02000000")
+ string(APPEND CMAKE_CXX_FLAGS
+ " -DWINHTTP_CALLBACK_STATUS_SHUTDOWN_COMPLETE=0x04000000")
+ # See also:
+ # https://learn.microsoft.com/en-us/windows/win32/winhttp/option-flags
+ string(APPEND CMAKE_CXX_FLAGS "
-DWINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2=0x00000800")
+ string(APPEND CMAKE_CXX_FLAGS " -DWINHTTP_NO_CLIENT_CERT_CONTEXT=0")
+ endif()
+
+ set(AWSSDK_LINK_LIBRARIES)
+ foreach(AWSSDK_PRODUCT ${AWSSDK_PRODUCTS})
+ if("${AWSSDK_PRODUCT}" STREQUAL "s2n-tls")
+ # Use aws-lc's openssl/*.h not openssl/*.h in system.
+ set(ADDITIONAL_FLAGS
"-DCOMPILE_DEFINITIONS=-I${aws-lc_SOURCE_DIR}/include")
endif()
- endforeach()
-
- set(AWSSDK_VENDORED TRUE)
- list(APPEND ARROW_BUNDLED_STATIC_LIBS ${AWSSDK_LIBRARIES})
- set(AWSSDK_LINK_LIBRARIES ${AWSSDK_LIBRARIES})
- if(UNIX)
- # on Linux and macOS curl seems to be required
- set_property(TARGET aws-cpp-sdk-core
- APPEND
- PROPERTY INTERFACE_LINK_LIBRARIES CURL::libcurl)
- set_property(TARGET AWS::aws-c-cal
- APPEND
- PROPERTY INTERFACE_LINK_LIBRARIES OpenSSL::Crypto
OpenSSL::SSL)
- if(APPLE)
- set_property(TARGET AWS::aws-c-cal
- APPEND
- PROPERTY INTERFACE_LINK_LIBRARIES "-framework Security")
+ fetchcontent_makeavailable(${AWSSDK_PRODUCT})
+ if(CMAKE_VERSION VERSION_LESS 3.28)
+ set_property(DIRECTORY ${${AWSSDK_PRODUCT}_SOURCE_DIR} PROPERTY
EXCLUDE_FROM_ALL
+ TRUE)
endif()
- if(ZLIB_VENDORED)
- set_property(TARGET aws-cpp-sdk-core
- APPEND
- PROPERTY INTERFACE_LINK_LIBRARIES ZLIB::ZLIB)
- add_dependencies(awssdk_ep zlib_ep)
+ list(PREPEND CMAKE_MODULE_PATH "${${AWSSDK_PRODUCT}_SOURCE_DIR}/cmake")
+ if(NOT "${AWSSDK_PRODUCT}" STREQUAL "aws-sdk-cpp")
+ if("${AWSSDK_PRODUCT}" STREQUAL "aws-lc")
+ # We don't need to link aws-lc. It's used only by s2n-tls.
+ elseif("${AWSSDK_PRODUCT}" STREQUAL "s2n-tls")
+ list(PREPEND AWSSDK_LINK_LIBRARIES s2n)
+ else()
+ list(PREPEND AWSSDK_LINK_LIBRARIES ${AWSSDK_PRODUCT})
+ # This is for find_package(aws-*) in aws-crt-cpp and aws-sdk-cpp.
+ add_library(AWS::${AWSSDK_PRODUCT} ALIAS ${AWSSDK_PRODUCT})
+ endif()
endif()
- set_property(TARGET AWS::aws-c-io
- APPEND
- PROPERTY INTERFACE_LINK_LIBRARIES ${CMAKE_DL_LIBS})
- elseif(WIN32)
- set_property(TARGET aws-cpp-sdk-core
- APPEND
- PROPERTY INTERFACE_LINK_LIBRARIES
- "winhttp.lib"
- "bcrypt.lib"
- "wininet.lib"
- "userenv.lib"
- "version.lib")
- set_property(TARGET AWS::aws-c-cal
- APPEND
- PROPERTY INTERFACE_LINK_LIBRARIES
- "bcrypt.lib"
- "ncrypt.lib"
- "Secur32.lib"
- "Shlwapi.lib")
- set_property(TARGET AWS::aws-c-io
- APPEND
- PROPERTY INTERFACE_LINK_LIBRARIES "crypt32.lib")
- endif()
-
- # AWSSDK is static-only build
-endmacro()
+ endforeach()
+ list(PREPEND
+ AWSSDK_LINK_LIBRARIES
+ aws-cpp-sdk-identity-management
+ aws-cpp-sdk-sts
+ aws-cpp-sdk-cognito-identity
+ aws-cpp-sdk-s3
+ aws-cpp-sdk-core)
+
+ set(AWSSDK_VENDORED
+ TRUE
+ PARENT_SCOPE)
+ list(APPEND ARROW_BUNDLED_STATIC_LIBS ${AWSSDK_LINK_LIBRARIES})
+ set(ARROW_BUNDLED_STATIC_LIBS
+ ${ARROW_BUNDLED_STATIC_LIBS}
+ PARENT_SCOPE)
+ set(AWSSDK_LINK_LIBRARIES
+ ${AWSSDK_LINK_LIBRARIES}
+ PARENT_SCOPE)
+endfunction()
if(ARROW_S3)
resolve_dependency(AWSSDK HAVE_ALT TRUE)
@@ -5510,16 +5297,6 @@ if(ARROW_S3)
endif()
endif()
endif()
-
- if(APPLE)
- # CoreFoundation's path is hardcoded in the CMake files provided by
- # aws-sdk-cpp to use the macOS SDK provided by XCode which makes
- # XCode a hard dependency. Command Line Tools is often used instead
- # of the full XCode suite, so let the linker to find it.
- set_target_properties(AWS::aws-c-common
- PROPERTIES INTERFACE_LINK_LIBRARIES
- "-pthread;pthread;-framework
CoreFoundation")
- endif()
endif()
# ----------------------------------------------------------------------
diff --git a/cpp/cmake_modules/aws-c-common-1208.patch
b/cpp/cmake_modules/aws-c-common-1208.patch
new file mode 100644
index 0000000000..72a5f70776
--- /dev/null
+++ b/cpp/cmake_modules/aws-c-common-1208.patch
@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+From 06c435c6765833734c62ac45909b5ab1fc5276a7 Mon Sep 17 00:00:00 2001
+From: Sutou Kouhei <[email protected]>
+Date: Sat, 7 Jun 2025 20:40:36 +0900
+Subject: [PATCH] Use _WIN32 not _MSC_VER for MinGW
+
+This was missed in https://github.com/awslabs/aws-c-common/pull/801
+and https://github.com/awslabs/aws-c-common/pull/822 .
+
+MSYS2 also includes this change:
+https://github.com/msys2/MINGW-packages/blob/master/mingw-w64-aws-c-common/001-fix-build-on-mingw-aarch64.patch
+
+Error message without this:
+
+https://github.com/ursacomputing/crossbow/actions/runs/15502494926/job/43652596580#step:7:5884
+
+ cd /D/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-build &&
/D/a/crossbow/crossbow/sccache/sccache.exe /C/rtools40/mingw32/bin/cc.exe
-DARROW_HAVE_RUNTIME_SSE4_2 -DARROW_HAVE_SSE4_2 -DARROW_WITH_TIMING_TESTS
-DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_NONE -DCJSON_HIDE_SYMBOLS
-DINTEL_NO_ITTNOTIFY_API -DPSAPI_VERSION=1 -DWINDOWS_KERNEL_LIB=kernel32
-D_CRT_SECURE_NO_WARNINGS @CMakeFiles/aws-c-common.dir/includes_C.rsp -O3
-DNDEBUG -O2 -ftree-vectorize -std=gnu99 -fvisibil [...]
+ In file included from
D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_order.h:71,
+ from
D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_buf.h:9,
+ from
D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/thread.h:8,
+ from
D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/logging.h:11,
+ from
D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/source/allocator.c:8:
+
D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_order.inl:
In function 'aws_hton64':
+
D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_order.inl:47:23:
warning: implicit declaration of function 'htonl'; did you mean 'wtoll'?
[-Wimplicit-function-declaration]
+ return ((uint64_t)htonl(low)) << 32 | htonl(high);
+ ^~~~~
+ wtoll
+ In file included from
C:/rtools40/mingw32/i686-w64-mingw32/include/windows.h:92,
+ from
D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/source/allocator.c:15:
+ C:/rtools40/mingw32/i686-w64-mingw32/include/winsock.h: At top level:
+ C:/rtools40/mingw32/i686-w64-mingw32/include/winsock.h:286:37: error:
conflicting types for 'htonl'
+ WINSOCK_API_LINKAGE u_long WSAAPI htonl(u_long hostlong);
+ ^~~~~
+ In file included from
D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_order.h:71,
+ from
D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_buf.h:9,
+ from
D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/thread.h:8,
+ from
D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/logging.h:11,
+ from
D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/source/allocator.c:8:
+
D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_order.inl:47:23:
note: previous implicit declaration of 'htonl' was here
+ return ((uint64_t)htonl(low)) << 32 | htonl(high);
+ ^~~~~
+---
+ include/aws/common/byte_order.inl | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/include/aws/common/byte_order.inl
b/include/aws/common/byte_order.inl
+index 1204be06a1..0abd9cb8cd 100644
+--- a/include/aws/common/byte_order.inl
++++ b/include/aws/common/byte_order.inl
+@@ -13,7 +13,7 @@
+ # include <stdlib.h>
+ #else
+ # include <netinet/in.h>
+-#endif /* _MSC_VER */
++#endif /* _WIN32 */
+
+ AWS_EXTERN_C_BEGIN
+
+@@ -39,7 +39,7 @@ AWS_STATIC_IMPL uint64_t aws_hton64(uint64_t x) {
+ uint64_t v;
+ __asm__("bswap %q0" : "=r"(v) : "0"(x));
+ return v;
+-#elif defined(_MSC_VER)
++#elif defined(_WIN32)
+ return _byteswap_uint64(x);
+ #else
+ uint32_t low = x & UINT32_MAX;
diff --git a/cpp/cmake_modules/aws_sdk_cpp_generate_variables.sh
b/cpp/cmake_modules/aws_sdk_cpp_generate_variables.sh
deleted file mode 100755
index 79b560a4a1..0000000000
--- a/cpp/cmake_modules/aws_sdk_cpp_generate_variables.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -eu
-
-version=$1
-
-base_dir="$(dirname "$0")"
-output="${base_dir}/AWSSDKVariables.cmake"
-
-cat <<HEADER > ${output}
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Generated by:
-# $ cpp/cmake_modules/aws_sdk_cpp_generate_variables.sh ${version}
-
-HEADER
-
-rm -f ${version}.tar.gz
-wget https://github.com/aws/aws-sdk-cpp/archive/${version}.tar.gz
-base_name=aws-sdk-cpp-${version}
-rm -rf ${base_name}
-tar xf ${version}.tar.gz
-
-echo "set(AWSSDK_UNUSED_DIRECTORIES" >> ${output}
-find ${base_name} -mindepth 1 -maxdepth 1 -type d | \
- sort | \
- grep -v cmake | \
- grep -v toolchains | \
- grep -v aws-cpp-sdk-cognito-identity | \
- grep -v aws-cpp-sdk-core | \
- grep -v aws-cpp-sdk-config | \
- grep -v aws-cpp-sdk-s3 | \
- grep -v aws-cpp-sdk-transfer | \
- grep -v aws-cpp-sdk-identity-management | \
- grep -v aws-cpp-sdk-sts | \
- sed -E -e "s,^${base_name}/, ,g" >> ${output}
-echo ")" >> ${output}
-
-rm -rf ${base_name}
-rm -f ${version}.tar.gz
diff --git a/cpp/thirdparty/README.md b/cpp/thirdparty/README.md
index 8180681053..32b216861f 100644
--- a/cpp/thirdparty/README.md
+++ b/cpp/thirdparty/README.md
@@ -23,3 +23,19 @@ See the "Build Dependency Management" section in the [C++
Developer
Documentation][1].
[1]:
https://github.com/apache/arrow/blob/main/docs/source/developers/cpp/building.rst
+
+## Update versions automatically
+
+There is a convenient script that update versions in `versions.txt` to
+the latest version automatically. You can use it like the following:
+
+```console
+cpp/thirdparty/update.rb PRODUCT_PATTERN1 PRODUCT_PATTERN2 ...
+```
+
+For example, you can update AWS SDK for C++ related products' versions
+by the following command line:
+
+```console
+cpp/thirdparty/update.rb "AWS*" "S2N*"
+```
diff --git a/cpp/thirdparty/update.rb b/cpp/thirdparty/update.rb
new file mode 100755
index 0000000000..77f22de04f
--- /dev/null
+++ b/cpp/thirdparty/update.rb
@@ -0,0 +1,189 @@
+#!/usr/bin/env ruby
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require "digest/sha2"
+require "json"
+require "open-uri"
+require "optparse"
+
+option_parser = OptionParser.new
+option_parser.banner =
+ "Usage: #{$0} [options] PRODUCT_PATTERN1 PRODUCT_PATTERN2 ..."
+patterns = option_parser.parse!(ARGV)
+if patterns.empty?
+ puts(option_parser)
+ exit(false)
+end
+
+# Extract product information from the `cpp/thirdparty/versions.txt`
+# content.
+#
+# Output:
+#
+# {
+# "ABSL" => {
+# version: "20211102.0",
+# checksum:
"dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4",
+# url_template:
"https://github.com/abseil/abseil-cpp/archive/%{version}.tar.gz";
+# },
+# "AWS_C_AUTH" => {
+# version: "v0.9.0",
+# checksum:
"aa6e98864fefb95c249c100da4ae7aed36ba13a8a91415791ec6fad20bec0427",
+# url_template:
"https://github.com/awslabs/aws-c-auth/archive/%{version}.tar.gz";,
+# },
+# ...
+# }
+def parse_versions_txt_content(content)
+ products = {}
+ content.each_line(chomp: true) do |line|
+ case line
+ when /\AARROW_([A-Za-z0-9_-]+)_BUILD_VERSION=(.+?)\z/
+ product = Regexp.last_match[1]
+ version = Regexp.last_match[2]
+ products[product] = {version: version}
+ when /\AARROW_([A-Za-z0-9_-]+)_BUILD_SHA256_CHECKSUM=(.+?)\z/
+ product = Regexp.last_match[1]
+ checksum = Regexp.last_match[2]
+ products[product][:checksum] = checksum
+ when /\A "ARROW_([A-Za-z0-9_-]+)_URL (?:\S+) (\S+)"\z/
+ product = Regexp.last_match[1]
+ url_template = Regexp.last_match[2]
+ url_template.gsub!(/\${.+?}/) do |matched|
+ if matched.end_with?("//./_}")
+ "%{version_underscore}"
+ else
+ "%{version}"
+ end
+ end
+ products[product][:url_template] = url_template
+ end
+ end
+ products
+end
+
+# Update `metadata[:version]` and `metadata[:checksum]` for
+# `latest_version`.
+#
+# This is used by product specific `#update_product_*` such as
+# `#update_product_github` and `#update_product_apache`.
+def update_product_generic(product, metadata, latest_version)
+ version = metadata[:version]
+ url_template = metadata[:url_template]
+ url = url_template % {
+ version: latest_version,
+ version_underscore: latest_version.gsub(".", "_"),
+ }
+ $stderr.puts("Updating #{product}: #{version} -> #{latest_version}")
+ metadata[:version] = latest_version
+ URI.open(url, "rb") do |response|
+ metadata[:checksum] = Digest::SHA256.hexdigest(response.read)
+ end
+ $stderr.puts(" Checksum: #{metadata[:checksum]}")
+end
+
+# Update metadata to the latest version. This is for products hosted
+# on GitHub.
+def update_product_github(product, metadata, repository)
+ version = metadata[:version]
+ tags_url = "https://api.github.com/repos/#{repository}/tags";
+ tags = URI.open(tags_url) do |response|
+ JSON.parse(response.read)
+ end
+ latest_tag_name = tags[0]["name"]
+ if latest_tag_name.start_with?("v")
+ if metadata[:version].start_with?("v")
+ latest_version = latest_tag_name
+ else
+ latest_version = latest_tag_name[1..-1]
+ end
+ else
+ latest_version = latest_tag_name
+ end
+ return if version == latest_version
+
+ update_product_generic(product, metadata, latest_version)
+end
+
+# Update metadata to the latest version. This is for products
+# developed by Apache Software Foundation.
+def update_product_apache(product, metadata, project)
+ version = metadata[:version]
+ version_directory_pattern = metadata[:version_directory_template] % {
+ version: "(\\d+(?:\\.\\d+)+)",
+ }
+ versions = URI.open("https://downloads.apache.org/#{project}/";) do |response|
+ response.read.scan(/<a href="#{version_directory_pattern}\/">/).flatten
+ end
+ latest_version = versions.last
+ return if version == latest_version
+
+ update_product_generic(product, metadata, latest_version)
+end
+
+# Update one product to the latest version.
+def update_product(product, metadata)
+ url_template = metadata[:url_template]
+ if url_template.nil?
+ $stderr.puts("#{product} isn't supported " +
+ "because there is no associated URL")
+ return
+ end
+
+ case url_template
+ when /\Ahttps:\/\/github\.com\/((?:[^\/]+)\/(?:[^\/]+))\//
+ github_repository = Regexp.last_match[1]
+ update_product_github(product, metadata, github_repository)
+ when /\Ahttps:\/\/www\.apache\.org\/dyn\/closer\.lua\/
+ ((?:[^\/]+))\/((?:[^\/]+))\//x
+ apache_project = Regexp.last_match[1]
+ metadata[:version_directory_template] = Regexp.last_match[2]
+ update_product_apache(product, metadata, apache_project)
+ else
+ $stderr.puts("TODO: #{product} isn't supported yet: #{url_template}")
+ end
+end
+
+# Update `versions.txt` content with `products`. `products` must be
+# the same structure as `Hash` returned by
+# `#parse_versions_txt_content`.
+def update_versions_txt_content!(content, products)
+ products.each do |product, metadata|
+ prefix = "ARROW_#{Regexp.escape(product)}"
+ content.gsub!(/^#{prefix}_BUILD_VERSION=.*$/) do
+ "ARROW_#{product}_BUILD_VERSION=#{metadata[:version]}"
+ end
+ content.gsub!(/^#{prefix}_BUILD_SHA256_CHECKSUM=.*?$/) do
+ "ARROW_#{product}_BUILD_SHA256_CHECKSUM=#{metadata[:checksum]}"
+ end
+ end
+end
+
+versions_txt = File.join(__dir__, "versions.txt")
+versions_txt_content = File.read(versions_txt)
+products = parse_versions_txt_content(versions_txt_content)
+patterns.each do |pattern|
+ target_products = products.filter do |product, _|
+ File.fnmatch?(pattern, product)
+ end
+ target_products.each do |product, metadata|
+ update_product(product, metadata)
+ end
+end
+update_versions_txt_content!(versions_txt_content, products)
+File.write(versions_txt, versions_txt_content)
diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt
index 1fc53c4d5e..96ae7c6f03 100644
--- a/cpp/thirdparty/versions.txt
+++ b/cpp/thirdparty/versions.txt
@@ -25,34 +25,34 @@
ARROW_ABSL_BUILD_VERSION=20211102.0
ARROW_ABSL_BUILD_SHA256_CHECKSUM=dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4
-ARROW_AWS_C_AUTH_BUILD_VERSION=v0.6.22
-ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=691a6b4418afcd3dc141351b6ad33fccd8e3ff84df0e9e045b42295d284ee14c
-ARROW_AWS_C_CAL_BUILD_VERSION=v0.5.20
-ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM=acc352359bd06f8597415c366cf4ec4f00d0b0da92d637039a73323dd55b6cd0
-ARROW_AWS_C_COMMON_BUILD_VERSION=v0.8.9
-ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=2f3fbaf7c38eae5a00e2a816d09b81177f93529ae8ba1b82dc8f31407565327a
-ARROW_AWS_C_COMPRESSION_BUILD_VERSION=v0.2.16
-ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM=044b1dbbca431a07bde8255ef9ec443c300fc60d4c9408d4b862f65e496687f4
-ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.2.18
-ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=310ca617f713bf664e4c7485a3d42c1fb57813abd0107e49790d107def7cde4f
-ARROW_AWS_C_HTTP_BUILD_VERSION=v0.7.3
-ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=07e16c6bf5eba6f0dea96b6f55eae312a7c95b736f4d2e4a210000f45d8265ae
-ARROW_AWS_C_IO_BUILD_VERSION=v0.13.14
-ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM=12b66510c3d9a4f7e9b714e9cfab2a5bf835f8b9ce2f909d20ae2a2128608c71
-ARROW_AWS_C_MQTT_BUILD_VERSION=v0.8.4
-ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM=232eeac63e72883d460c686a09b98cdd811d24579affac47c5c3f696f956773f
-ARROW_AWS_C_S3_BUILD_VERSION=v0.2.3
-ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=a00b3c9f319cd1c9aa2c3fa15098864df94b066dcba0deaccbb3caa952d902fe
-ARROW_AWS_C_SDKUTILS_BUILD_VERSION=v0.1.6
-ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=8a2951344b2fb541eab1e9ca17c18a7fcbfd2aaff4cdd31d362d1fad96111b91
-ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.1.13
-ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=0f897686f1963253c5069a0e495b85c31635ba146cd3ac38cc2ea31eaf54694d
-ARROW_AWS_CRT_CPP_BUILD_VERSION=v0.18.16
-ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM=9e69bc1dc4b50871d1038aa9ff6ddeb4c9b28f7d6b5e5b1b69041ccf50a13483
-ARROW_AWS_LC_BUILD_VERSION=v1.3.0
-ARROW_AWS_LC_BUILD_SHA256_CHECKSUM=ae96a3567161552744fc0cae8b4d68ed88b1ec0f3d3c98700070115356da5a37
-ARROW_AWSSDK_BUILD_VERSION=1.10.55
-ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=2d552fb1a84bef4a9b65e34aa7031851ed2aef5319e02cc6e4cb735c48aa30de
+ARROW_AWS_C_AUTH_BUILD_VERSION=v0.9.0
+ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=aa6e98864fefb95c249c100da4ae7aed36ba13a8a91415791ec6fad20bec0427
+ARROW_AWS_C_CAL_BUILD_VERSION=v0.9.2
+ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM=f9f3bc6a069e2efe25fcdf73e4d2b16b5608c327d2eb57c8f7a8524e9e1fcad0
+ARROW_AWS_C_COMMON_BUILD_VERSION=v0.12.3
+ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=a4e7ac6c6f840cb6ab56b8ee0bcd94a61c59d68ca42570bca518432da4c94273
+ARROW_AWS_C_COMPRESSION_BUILD_VERSION=v0.3.1
+ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM=d89fca17a37de762dc34f332d2da402343078da8dbd2224c46a11a88adddf754
+ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.5.4
+ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=cef8b78e362836d15514110fb43a0a0c7a86b0a210d5fe25fd248a82027a7272
+ARROW_AWS_C_HTTP_BUILD_VERSION=v0.10.2
+ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=048d9d683459ade363fd7cc448c2b6329c78f67a2a0c0cb61c16de4634a2fc6b
+ARROW_AWS_C_IO_BUILD_VERSION=v0.19.1
+ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM=f2fea0c066924f7fe3c2b1c7b2fa9be640f5b16a6514854226330e63a1faacd0
+ARROW_AWS_C_MQTT_BUILD_VERSION=v0.13.1
+ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM=c54d02c1e46f55bae8d5e6f9c4b0d78d84c1c9d9ac16ba8d78c3361edcd8b5bb
+ARROW_AWS_C_S3_BUILD_VERSION=v0.8.1
+ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=c8b09780691d2b94e50d101c68f01fa2d1c3debb0ff3aed313d93f0d3c9af663
+ARROW_AWS_C_SDKUTILS_BUILD_VERSION=v0.2.4
+ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=493cbed4fa57e0d4622fcff044e11305eb4fc12445f32c8861025597939175fc
+ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.2.7
+ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=178e8398d98111f29150f7813a70c20ad97ab30be0de02525440355fe84ccb1d
+ARROW_AWS_CRT_CPP_BUILD_VERSION=v0.32.8
+ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM=db44260452a0296341fb8e7b987e4c328f08f7829b9f1c740fed9c963e081e93
+ARROW_AWS_LC_BUILD_VERSION=v1.52.1
+ARROW_AWS_LC_BUILD_SHA256_CHECKSUM=fe552e3c3522f73afc3c30011745c431c633f7b4e25dcd7b38325f194a7b3b75
+ARROW_AWSSDK_BUILD_VERSION=1.11.587
+ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=b9944ba9905a68d6e53abb4f36ab2b3bd18ac88d8571647bb9f2b8026b76f8cd
# Despite the confusing version name this is still the whole Azure SDK for C++
including core, keyvault, storage-common, etc.
ARROW_AZURE_SDK_BUILD_VERSION=azure-identity_1.9.0
ARROW_AZURE_SDK_BUILD_SHA256_CHECKSUM=97065bfc971ac8df450853ce805f820f52b59457bd7556510186a1569502e4a1
@@ -105,8 +105,8 @@ ARROW_SNAPPY_BUILD_VERSION=1.2.2
ARROW_SNAPPY_BUILD_SHA256_CHECKSUM=90f74bc1fbf78a6c56b3c4a082a05103b3a56bb17bca1a27e052ea11723292dc
ARROW_SUBSTRAIT_BUILD_VERSION=v0.44.0
ARROW_SUBSTRAIT_BUILD_SHA256_CHECKSUM=f989a862f694e7dbb695925ddb7c4ce06aa6c51aca945105c075139aed7e55a2
-ARROW_S2N_TLS_BUILD_VERSION=v1.3.35
-ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM=9d32b26e6bfcc058d98248bf8fc231537e347395dd89cf62bb432b55c5da990d
+ARROW_S2N_TLS_BUILD_VERSION=v1.5.21
+ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM=203d69d6f557f6ab303438ad186fca13fd2c60581b2cca6348a9fbee10d79995
ARROW_THRIFT_BUILD_VERSION=0.20.0
ARROW_THRIFT_BUILD_SHA256_CHECKSUM=b5d8311a779470e1502c027f428a1db542f5c051c8e1280ccd2163fa935ff2d6
ARROW_UTF8PROC_BUILD_VERSION=v2.10.0
diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py
index 2838372bd1..e90613a662 100644
--- a/python/pyarrow/tests/test_fs.py
+++ b/python/pyarrow/tests/test_fs.py
@@ -2043,6 +2043,8 @@ def test_concurrent_s3fs_init():
@pytest.mark.s3
[email protected](reason="atexit(ensure_s3_finalized) will be called too late "
+ "with bundled aws-sdk-cpp 1.11.587")
@pytest.mark.skipif(running_on_musllinux(), reason="Leaking S3ClientFinalizer
causes "
"segfault on musl based
systems")
def test_uwsgi_integration():
