This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 1cefde21c91 [bugfix](thirdparty) Upgrade aws s3 sdk to prevent mem
leak (#25106) (#25418)
1cefde21c91 is described below
commit 1cefde21c915ca4b2811c435aa9b9714aedebcd8
Author: AlexYue <[email protected]>
AuthorDate: Fri Jan 5 21:40:48 2024 +0800
[bugfix](thirdparty) Upgrade aws s3 sdk to prevent mem leak (#25106)
(#25418)
---
be/CMakeLists.txt | 1 +
be/cmake/thirdparty.cmake | 3 ++
thirdparty/CHANGELOG.md | 4 +++
thirdparty/download-thirdparty.sh | 7 ++--
thirdparty/patches/aws-sdk-cpp-1.11.119.patch | 52 +++++++++++++++++++++++++++
thirdparty/vars.sh | 8 ++---
6 files changed, 68 insertions(+), 7 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index d59ee18f8ed..b3d885faf19 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -464,6 +464,7 @@ set(AWS_LIBS
aws-c-compression
aws-c-http
aws-c-mqtt
+ aws-c-sdkutils
aws-sdk-s3-crt)
if (OS_MACOSX)
diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake
index 493cbb87555..4928b64b16b 100644
--- a/be/cmake/thirdparty.cmake
+++ b/be/cmake/thirdparty.cmake
@@ -212,6 +212,9 @@ set_target_properties(aws-checksums PROPERTIES
IMPORTED_LOCATION ${THIRDPARTY_DI
add_library(aws-c-s3 STATIC IMPORTED)
set_target_properties(aws-c-s3 PROPERTIES IMPORTED_LOCATION
${THIRDPARTY_DIR}/lib64/libaws-c-s3.a)
+add_library(aws-c-sdkutils STATIC IMPORTED)
+set_target_properties(aws-c-sdkutils PROPERTIES IMPORTED_LOCATION
${THIRDPARTY_DIR}/lib64/libaws-c-sdkutils.a)
+
if (NOT OS_MACOSX)
add_library(aws-s2n STATIC IMPORTED)
set_target_properties(aws-s2n PROPERTIES IMPORTED_LOCATION
${THIRDPARTY_DIR}/lib64/libs2n.a)
diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md
index c90363fcb75..065fd187da5 100644
--- a/thirdparty/CHANGELOG.md
+++ b/thirdparty/CHANGELOG.md
@@ -2,6 +2,10 @@
This file contains version of the third-party dependency libraries in the
build-env image. The docker build-env image is apache/doris, and the tag is
`build-env-${version}`
+## v20231008
+
+- Modified: aws sdk 1.9.211 -> 1.11.119
+
## v20230721
- Modified hadoop libhdfs 3.3.4.4 -> 3.3.4.5
diff --git a/thirdparty/download-thirdparty.sh
b/thirdparty/download-thirdparty.sh
index b63d3cbccb3..783c6fab007 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -376,12 +376,13 @@ echo "Finished patching ${HYPERSCAN_SOURCE}"
cd "${TP_SOURCE_DIR}/${AWS_SDK_SOURCE}"
if [[ ! -f "${PATCHED_MARK}" ]]; then
- if [[ "${AWS_SDK_SOURCE}" == "aws-sdk-cpp-1.9.272" ]]; then
- if wget --no-check-certificate -q
https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/aws-crt-cpp-1.9.272.tar.gz
-O aws-crt-cpp-1.9.272.tar.gz; then
- tar xzf aws-crt-cpp-1.9.272.tar.gz
+ if [[ "${AWS_SDK_SOURCE}" == "aws-sdk-cpp-1.11.119" ]]; then
+ if wget --no-check-certificate -q
https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/aws-crt-cpp-1.11.119.tar.gz
-O aws-crt-cpp-1.11.119.tar.gz; then
+ tar xzf aws-crt-cpp-1.11.119.tar.gz
else
bash ./prefetch_crt_dependency.sh
fi
+ patch -p1 <"${TP_PATCH_DIR}/aws-sdk-cpp-1.11.119.patch"
else
bash ./prefetch_crt_dependency.sh
fi
diff --git a/thirdparty/patches/aws-sdk-cpp-1.11.119.patch
b/thirdparty/patches/aws-sdk-cpp-1.11.119.patch
new file mode 100644
index 00000000000..b045721ecda
--- /dev/null
+++ b/thirdparty/patches/aws-sdk-cpp-1.11.119.patch
@@ -0,0 +1,52 @@
+diff --git
a/generated/src/aws-cpp-sdk-s3-crt/source/model/CopyObjectRequest.cpp
b/generated/src/aws-cpp-sdk-s3-crt/source/model/CopyObjectRequest.cpp
+index 2aa7869b51..9ba8877ed8 100644
+--- a/generated/src/aws-cpp-sdk-s3-crt/source/model/CopyObjectRequest.cpp
++++ b/generated/src/aws-cpp-sdk-s3-crt/source/model/CopyObjectRequest.cpp
+@@ -173,7 +173,7 @@ Aws::Http::HeaderValueCollection
CopyObjectRequest::GetRequestSpecificHeaders()
+ if(m_copySourceHasBeenSet)
+ {
+ ss << m_copySource;
+- headers.emplace("x-amz-copy-source", URI::URLEncodePath(ss.str()));
++ headers.emplace("x-amz-copy-source", ss.str());
+ ss.str("");
+ }
+
+diff --git
a/generated/src/aws-cpp-sdk-s3-crt/source/model/UploadPartCopyRequest.cpp
b/generated/src/aws-cpp-sdk-s3-crt/source/model/UploadPartCopyRequest.cpp
+index f301c47606..7f16e6afcd 100644
+--- a/generated/src/aws-cpp-sdk-s3-crt/source/model/UploadPartCopyRequest.cpp
++++ b/generated/src/aws-cpp-sdk-s3-crt/source/model/UploadPartCopyRequest.cpp
+@@ -112,7 +112,7 @@ Aws::Http::HeaderValueCollection
UploadPartCopyRequest::GetRequestSpecificHeader
+ if(m_copySourceHasBeenSet)
+ {
+ ss << m_copySource;
+- headers.emplace("x-amz-copy-source", URI::URLEncodePath(ss.str()));
++ headers.emplace("x-amz-copy-source", ss.str());
+ ss.str("");
+ }
+
+diff --git a/generated/src/aws-cpp-sdk-s3/source/model/CopyObjectRequest.cpp
b/generated/src/aws-cpp-sdk-s3/source/model/CopyObjectRequest.cpp
+index e1862b22ff..6bd6d0aea9 100644
+--- a/generated/src/aws-cpp-sdk-s3/source/model/CopyObjectRequest.cpp
++++ b/generated/src/aws-cpp-sdk-s3/source/model/CopyObjectRequest.cpp
+@@ -173,7 +173,7 @@ Aws::Http::HeaderValueCollection
CopyObjectRequest::GetRequestSpecificHeaders()
+ if(m_copySourceHasBeenSet)
+ {
+ ss << m_copySource;
+- headers.emplace("x-amz-copy-source", URI::URLEncodePath(ss.str()));
++ headers.emplace("x-amz-copy-source", ss.str());
+ ss.str("");
+ }
+
+diff --git
a/generated/src/aws-cpp-sdk-s3/source/model/UploadPartCopyRequest.cpp
b/generated/src/aws-cpp-sdk-s3/source/model/UploadPartCopyRequest.cpp
+index f918909a86..16af4d2797 100644
+--- a/generated/src/aws-cpp-sdk-s3/source/model/UploadPartCopyRequest.cpp
++++ b/generated/src/aws-cpp-sdk-s3/source/model/UploadPartCopyRequest.cpp
+@@ -112,7 +112,7 @@ Aws::Http::HeaderValueCollection
UploadPartCopyRequest::GetRequestSpecificHeader
+ if(m_copySourceHasBeenSet)
+ {
+ ss << m_copySource;
+- headers.emplace("x-amz-copy-source", URI::URLEncodePath(ss.str()));
++ headers.emplace("x-amz-copy-source", ss.str());
+ ss.str("");
+ }
+
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 87a4d2abd2a..a075309e099 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -332,10 +332,10 @@ BOOTSTRAP_TABLE_CSS_FILE="bootstrap-table.min.css"
BOOTSTRAP_TABLE_CSS_MD5SUM="23389d4456da412e36bae30c469a766a"
# aws sdk
-AWS_SDK_DOWNLOAD="https://github.com/aws/aws-sdk-cpp/archive/refs/tags/1.9.272.tar.gz";
-AWS_SDK_NAME="aws-sdk-cpp-1.9.272.tar.gz"
-AWS_SDK_SOURCE="aws-sdk-cpp-1.9.272"
-AWS_SDK_MD5SUM="3a9190e3d14b81bfdd9c41a89a419202"
+AWS_SDK_DOWNLOAD="https://github.com/aws/aws-sdk-cpp/archive/refs/tags/1.11.119.tar.gz";
+AWS_SDK_NAME="aws-sdk-cpp-1.11.119.tar.gz"
+AWS_SDK_SOURCE="aws-sdk-cpp-1.11.119"
+AWS_SDK_MD5SUM="3cd8bd51d39dc207a243a2074d11f439"
# tsan_header
TSAN_HEADER_DOWNLOAD="https://gcc.gnu.org/git/?p=gcc.git;a=blob_plain;f=libsanitizer/include/sanitizer/tsan_interface_atomic.h;hb=refs/heads/releases/gcc-7";
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
