This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e1b9854f90d [bugfix](thirdparty) Upgrade aws s3 sdk to prevent mem
leak (#25106)
e1b9854f90d is described below
commit e1b9854f90d62a95d442f7f775cfaa5232a26775
Author: AlexYue <[email protected]>
AuthorDate: Mon Oct 9 16:08:50 2023 +0800
[bugfix](thirdparty) Upgrade aws s3 sdk to prevent mem leak (#25106)
During the use of the AWS S3 SDK, we found that there is a memory leak.
According to the official issue, upgrading the SDK should resolve the issue.
---
be/cmake/thirdparty.cmake | 1 +
thirdparty/CHANGELOG.md | 3 ++
thirdparty/download-thirdparty.sh | 7 ++--
thirdparty/patches/aws-sdk-cpp-1.11.119.patch | 52 +++++++++++++++++++++++++++
thirdparty/vars.sh | 8 ++---
5 files changed, 64 insertions(+), 7 deletions(-)
diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake
index e55b413c242..60d9c65055b 100644
--- a/be/cmake/thirdparty.cmake
+++ b/be/cmake/thirdparty.cmake
@@ -137,6 +137,7 @@ add_thirdparty(aws-c-http LIB64)
add_thirdparty(aws-c-mqtt LIB64)
add_thirdparty(aws-checksums LIB64)
add_thirdparty(aws-c-s3 LIB64)
+add_thirdparty(aws-c-sdkutils LIB64)
if (NOT OS_MACOSX)
add_thirdparty(aws-s2n LIBNAME "lib/libs2n.a")
endif()
diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md
index 8e0b5416875..b0cb1f87704 100644
--- a/thirdparty/CHANGELOG.md
+++ b/thirdparty/CHANGELOG.md
@@ -2,6 +2,9 @@
This file contains version of the third-party dependency libraries in the
build-env image. The docker build-env image is apache/doris, and the tag is
`build-env-${version}`
+## v20231008
+- Modified: aws sdk 1.9.211 -> 1.11.119
+
## v20230824
- Modified: protobuf 3.15.0 -> 21.11
- Modified: glog 0.4.0 -> 0.6.0
diff --git a/thirdparty/download-thirdparty.sh
b/thirdparty/download-thirdparty.sh
index e31871f25dd..e1d7c328656 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -386,12 +386,13 @@ echo "Finished patching ${HYPERSCAN_SOURCE}"
cd "${TP_SOURCE_DIR}/${AWS_SDK_SOURCE}"
if [[ ! -f "${PATCHED_MARK}" ]]; then
- if [[ "${AWS_SDK_SOURCE}" == "aws-sdk-cpp-1.9.272" ]]; then
- if wget --no-check-certificate -q
https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/aws-crt-cpp-1.9.272.tar.gz
-O aws-crt-cpp-1.9.272.tar.gz; then
- tar xzf aws-crt-cpp-1.9.272.tar.gz
+ if [[ "${AWS_SDK_SOURCE}" == "aws-sdk-cpp-1.11.119" ]]; then
+ if wget --no-check-certificate -q
https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/aws-crt-cpp-1.11.119.tar.gz
-O aws-crt-cpp-1.11.119.tar.gz; then
+ tar xzf aws-crt-cpp-1.11.119.tar.gz
else
bash ./prefetch_crt_dependency.sh
fi
+ patch -p1 <"${TP_PATCH_DIR}/aws-sdk-cpp-1.11.119.patch"
else
bash ./prefetch_crt_dependency.sh
fi
diff --git a/thirdparty/patches/aws-sdk-cpp-1.11.119.patch
b/thirdparty/patches/aws-sdk-cpp-1.11.119.patch
new file mode 100644
index 00000000000..b045721ecda
--- /dev/null
+++ b/thirdparty/patches/aws-sdk-cpp-1.11.119.patch
@@ -0,0 +1,52 @@
+diff --git
a/generated/src/aws-cpp-sdk-s3-crt/source/model/CopyObjectRequest.cpp
b/generated/src/aws-cpp-sdk-s3-crt/source/model/CopyObjectRequest.cpp
+index 2aa7869b51..9ba8877ed8 100644
+--- a/generated/src/aws-cpp-sdk-s3-crt/source/model/CopyObjectRequest.cpp
++++ b/generated/src/aws-cpp-sdk-s3-crt/source/model/CopyObjectRequest.cpp
+@@ -173,7 +173,7 @@ Aws::Http::HeaderValueCollection
CopyObjectRequest::GetRequestSpecificHeaders()
+ if(m_copySourceHasBeenSet)
+ {
+ ss << m_copySource;
+- headers.emplace("x-amz-copy-source", URI::URLEncodePath(ss.str()));
++ headers.emplace("x-amz-copy-source", ss.str());
+ ss.str("");
+ }
+
+diff --git
a/generated/src/aws-cpp-sdk-s3-crt/source/model/UploadPartCopyRequest.cpp
b/generated/src/aws-cpp-sdk-s3-crt/source/model/UploadPartCopyRequest.cpp
+index f301c47606..7f16e6afcd 100644
+--- a/generated/src/aws-cpp-sdk-s3-crt/source/model/UploadPartCopyRequest.cpp
++++ b/generated/src/aws-cpp-sdk-s3-crt/source/model/UploadPartCopyRequest.cpp
+@@ -112,7 +112,7 @@ Aws::Http::HeaderValueCollection
UploadPartCopyRequest::GetRequestSpecificHeader
+ if(m_copySourceHasBeenSet)
+ {
+ ss << m_copySource;
+- headers.emplace("x-amz-copy-source", URI::URLEncodePath(ss.str()));
++ headers.emplace("x-amz-copy-source", ss.str());
+ ss.str("");
+ }
+
+diff --git a/generated/src/aws-cpp-sdk-s3/source/model/CopyObjectRequest.cpp
b/generated/src/aws-cpp-sdk-s3/source/model/CopyObjectRequest.cpp
+index e1862b22ff..6bd6d0aea9 100644
+--- a/generated/src/aws-cpp-sdk-s3/source/model/CopyObjectRequest.cpp
++++ b/generated/src/aws-cpp-sdk-s3/source/model/CopyObjectRequest.cpp
+@@ -173,7 +173,7 @@ Aws::Http::HeaderValueCollection
CopyObjectRequest::GetRequestSpecificHeaders()
+ if(m_copySourceHasBeenSet)
+ {
+ ss << m_copySource;
+- headers.emplace("x-amz-copy-source", URI::URLEncodePath(ss.str()));
++ headers.emplace("x-amz-copy-source", ss.str());
+ ss.str("");
+ }
+
+diff --git
a/generated/src/aws-cpp-sdk-s3/source/model/UploadPartCopyRequest.cpp
b/generated/src/aws-cpp-sdk-s3/source/model/UploadPartCopyRequest.cpp
+index f918909a86..16af4d2797 100644
+--- a/generated/src/aws-cpp-sdk-s3/source/model/UploadPartCopyRequest.cpp
++++ b/generated/src/aws-cpp-sdk-s3/source/model/UploadPartCopyRequest.cpp
+@@ -112,7 +112,7 @@ Aws::Http::HeaderValueCollection
UploadPartCopyRequest::GetRequestSpecificHeader
+ if(m_copySourceHasBeenSet)
+ {
+ ss << m_copySource;
+- headers.emplace("x-amz-copy-source", URI::URLEncodePath(ss.str()));
++ headers.emplace("x-amz-copy-source", ss.str());
+ ss.str("");
+ }
+
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 4e17117b6f8..e154a1b3420 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -346,10 +346,10 @@ BOOTSTRAP_TABLE_CSS_FILE="bootstrap-table.min.css"
BOOTSTRAP_TABLE_CSS_MD5SUM="23389d4456da412e36bae30c469a766a"
# aws sdk
-AWS_SDK_DOWNLOAD="https://github.com/aws/aws-sdk-cpp/archive/refs/tags/1.9.272.tar.gz";
-AWS_SDK_NAME="aws-sdk-cpp-1.9.272.tar.gz"
-AWS_SDK_SOURCE="aws-sdk-cpp-1.9.272"
-AWS_SDK_MD5SUM="3a9190e3d14b81bfdd9c41a89a419202"
+AWS_SDK_DOWNLOAD="https://github.com/aws/aws-sdk-cpp/archive/refs/tags/1.11.119.tar.gz";
+AWS_SDK_NAME="aws-sdk-cpp-1.11.119.tar.gz"
+AWS_SDK_SOURCE="aws-sdk-cpp-1.11.119"
+AWS_SDK_MD5SUM="3cd8bd51d39dc207a243a2074d11f439"
# tsan_header
TSAN_HEADER_DOWNLOAD="https://gcc.gnu.org/git/?p=gcc.git;a=blob_plain;f=libsanitizer/include/sanitizer/tsan_interface_atomic.h;hb=refs/heads/releases/gcc-7";
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
