This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 8080085d0aa branch-4.1: [chore](thirdparty) upgrade thrift from 0.16.0
to 0.24.0 (#67356)
8080085d0aa is described below
commit 8080085d0aa1b6efc0e0fe0b1dee53802da61b4d
Author: Calvin Kirs <[email protected]>
AuthorDate: Sat Sep 5 21:42:36 2026 +0800
branch-4.1: [chore](thirdparty) upgrade thrift from 0.16.0 to 0.24.0
(#67356)
picked from #65990
---
be/src/service/doris_main.cpp | 2 +-
be/src/util/thrift_util.cpp | 2 +-
.../java/org/apache/doris/qe/FEOpExecutor.java | 1 +
.../org/apache/doris/qe/PointQueryExecutor.java | 2 +-
.../java/org/apache/doris/qe/ResultReceiver.java | 13 +++++++++-
fe/pom.xml | 2 +-
fs_brokers/apache_hdfs_broker/pom.xml | 5 ++--
gensrc/thrift/Makefile | 2 +-
regression-test/framework/pom.xml | 4 ++-
thirdparty/CHANGELOG.md | 4 +++
thirdparty/build-thirdparty.sh | 14 +++++++----
thirdparty/download-thirdparty.sh | 16 ------------
.../patches/thrift-0.16-fix_mutex_include.patch | 9 -------
.../thrift-0.16-reset-consumed-message-size.patch | 29 ----------------------
thirdparty/vars.sh | 8 +++---
15 files changed, 41 insertions(+), 72 deletions(-)
diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp
index 12f5af0e2a4..346ab386b15 100644
--- a/be/src/service/doris_main.cpp
+++ b/be/src/service/doris_main.cpp
@@ -508,7 +508,7 @@ int main(int argc, char** argv) {
exit(-1);
}
// add logger for thrift internal
- apache::thrift::GlobalOutput.setOutputFunction(doris::thrift_output);
+
apache::thrift::TOutput::instance().setOutputFunction(doris::thrift_output);
Status status = Status::OK();
if (doris::config::enable_java_support) {
diff --git a/be/src/util/thrift_util.cpp b/be/src/util/thrift_util.cpp
index 015b5382cfc..6716f6dd7b2 100644
--- a/be/src/util/thrift_util.cpp
+++ b/be/src/util/thrift_util.cpp
@@ -100,7 +100,7 @@ static void thrift_output_function(const char* output) {
}
void init_thrift_logging() {
- apache::thrift::GlobalOutput.setOutputFunction(thrift_output_function);
+
apache::thrift::TOutput::instance().setOutputFunction(thrift_output_function);
}
Status wait_for_local_server(const ThriftServer& server, int num_retries, int
retry_interval_ms) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/FEOpExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/FEOpExecutor.java
index 7566d7c4a9a..b124ccb99e2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/FEOpExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/FEOpExecutor.java
@@ -302,6 +302,7 @@ public class FEOpExecutor {
+ "`query_timeout`/`insert_timeout`")
.put(TTransportException.END_OF_FILE, "EOF")
.put(TTransportException.CORRUPTED_DATA, "Corrupted
data")
+ .put(TTransportException.MESSAGE_SIZE_LIMIT, "Message
size exceeds limit")
.build();
private final String msg;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java
index 745e10af822..55e15295e15 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java
@@ -375,7 +375,7 @@ public class PointQueryExecutor implements CoordInterface {
try {
deserializer.deserialize(resultBatch, serialResult);
} catch (TException e) {
- if (e.getMessage().contains("MaxMessageSize reached")) {
+ if (ResultReceiver.isMessageSizeExceeded(e)) {
throw new TException("MaxMessageSize reached, try increase
max_msg_size_of_result_receiver");
} else {
throw e;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ResultReceiver.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ResultReceiver.java
index c5d1002e4cd..a67a5398f29 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ResultReceiver.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ResultReceiver.java
@@ -35,6 +35,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.thrift.TDeserializer;
import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
@@ -172,7 +173,7 @@ public class ResultReceiver {
try {
deserializer.deserialize(resultBatch, serialResult);
} catch (TException e) {
- if (e.getMessage().contains("MaxMessageSize reached"))
{
+ if (isMessageSizeExceeded(e)) {
throw new TException(
"MaxMessageSize reached, try increase
max_msg_size_of_result_receiver");
} else {
@@ -209,6 +210,16 @@ public class ResultReceiver {
return rowBatch;
}
+ // Thrift 0.24 reports an exceeded max message size as MESSAGE_SIZE_LIMIT;
older
+ // versions only carried the "MaxMessageSize reached" text on END_OF_FILE.
+ static boolean isMessageSizeExceeded(TException e) {
+ if (e instanceof TTransportException
+ && ((TTransportException) e).getType() ==
TTransportException.MESSAGE_SIZE_LIMIT) {
+ return true;
+ }
+ return e.getMessage() != null &&
e.getMessage().contains("MaxMessageSize reached");
+ }
+
public synchronized void cancel(Status reason) {
if (reason.isFinished()) {
return;
diff --git a/fe/pom.xml b/fe/pom.xml
index 0ed9e5bdba0..9e63c98bdac 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -277,7 +277,7 @@ under the License.
<json-simple.version>1.1.1</json-simple.version>
<junit.version>5.14.1</junit.version>
<hikaricp.version>6.0.0</hikaricp.version>
- <thrift.version>0.16.0</thrift.version>
+ <thrift.version>0.24.0</thrift.version>
<tomcat-embed.version>9.0.104</tomcat-embed.version>
<log4j2.version>2.25.4</log4j2.version>
<log4j-1.2.version>2.25.4</log4j-1.2.version>
diff --git a/fs_brokers/apache_hdfs_broker/pom.xml
b/fs_brokers/apache_hdfs_broker/pom.xml
index 2cb8d892dee..096f7af25d9 100644
--- a/fs_brokers/apache_hdfs_broker/pom.xml
+++ b/fs_brokers/apache_hdfs_broker/pom.xml
@@ -274,10 +274,11 @@ under the License.
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.thrift/libthrift -->
+ <!-- Must match the thrift compiler installed in thirdparty. -->
<dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
- <version>0.16.0</version>
+ <version>0.24.0</version>
</dependency>
<!--
https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-api -->
<dependency>
@@ -412,7 +413,7 @@ under the License.
<thriftExecutable>${env.DORIS_THIRDPARTY}/installed/bin/thrift</thriftExecutable>
<thriftSourceRoot>${basedir}/src/main/resources/thrift/</thriftSourceRoot>
<outputDirectory>${basedir}/src/main/thrift/</outputDirectory>
- <generator>java:fullcamel</generator>
+
<generator>java:fullcamel,generated_annotations=suppress</generator>
<skip>${skip.plugin}</skip>
</configuration>
<executions>
diff --git a/gensrc/thrift/Makefile b/gensrc/thrift/Makefile
index 689f4acb6d3..241ed3a4d55 100644
--- a/gensrc/thrift/Makefile
+++ b/gensrc/thrift/Makefile
@@ -32,7 +32,7 @@ all: ${GEN_OBJECTS} ${OBJECTS}
$(shell mkdir -p ${BUILD_DIR}/gen_java)
THRIFT_CPP_ARGS = -I ${CURDIR} -I ${BUILD_DIR}/thrift/ --gen
cpp:moveable_types,no_skeleton -out ${BUILD_DIR}/gen_cpp --allow-64bit-consts
-strict
-THRIFT_JAVA_ARGS = -I ${CURDIR} -I ${BUILD_DIR}/thrift/ --gen java:fullcamel
-out ${BUILD_DIR}/gen_java --allow-64bit-consts -strict
+THRIFT_JAVA_ARGS = -I ${CURDIR} -I ${BUILD_DIR}/thrift/ --gen
java:fullcamel,generated_annotations=suppress -out ${BUILD_DIR}/gen_java
--allow-64bit-consts -strict
${BUILD_DIR}/gen_cpp:
mkdir -p $@
diff --git a/regression-test/framework/pom.xml
b/regression-test/framework/pom.xml
index c5b81b39ce3..51547d11faa 100644
--- a/regression-test/framework/pom.xml
+++ b/regression-test/framework/pom.xml
@@ -66,6 +66,8 @@ under the License.
</mailingLists>
<properties>
<doris.home>${basedir}/../../</doris.home>
+ <!-- Must match the thrift compiler installed in thirdparty and
fe/pom.xml. -->
+ <thrift.version>0.24.0</thrift.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
@@ -184,7 +186,7 @@ under the License.
<dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
- <version>0.16.0</version>
+ <version>${thrift.version}</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md
index 7c7235b7e1e..0783980741b 100644
--- a/thirdparty/CHANGELOG.md
+++ b/thirdparty/CHANGELOG.md
@@ -2,6 +2,10 @@
This file contains version of the third-party dependency libraries in the
build-env image. The docker build-env image is apache/doris, and the tag is
`build-env-${version}`
+## 20260901
+
+- Modified: thrift 0.16.0 -> 0.24.0
+
## 20260429
- Added: timsort (cpp-TimSort 3.x.y)
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index 081985c03c1..38eeb7b2ac3 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -410,9 +410,13 @@ build_thrift() {
check_if_source_exist "${THRIFT_SOURCE}"
cd "${TP_SOURCE_DIR}/${THRIFT_SOURCE}"
+ # Headers of a previously installed thrift would shadow the in-tree ones
+ # via -I${TP_INCLUDE_DIR} and break an in-place version upgrade.
+ rm -rf "${TP_INSTALL_DIR}/include/thrift"
+
# FE UT can rebuild the release-branch Thrift in a build image that already
- # contains a newer Thrift. Prefer this source tree's headers so an in-place
- # downgrade does not compile old sources against the installed new headers.
+ # contains another Thrift version. Prefer this source tree's headers so an
+ # in-place rebuild does not compile against the installed headers.
local thrift_source_include="${TP_SOURCE_DIR}/${THRIFT_SOURCE}/lib/cpp/src"
if [[ "${KERNEL}" != 'Darwin' ]]; then
@@ -428,9 +432,9 @@ build_thrift() {
# NOTE(amos): libtool discard -static. --static works.
./configure CFLAGS="${cflags}" CXXFLAGS="${cxxflags}" LDFLAGS="${ldflags}"
LIBS="-lcrypto -ldl -lssl" \
--prefix="${TP_INSTALL_DIR}" --docdir="${TP_INSTALL_DIR}/doc"
--enable-static --disable-shared --disable-tests \
- --disable-tutorial --without-qt4 --without-qt5 --without-csharp
--without-erlang --without-nodejs --without-nodets --without-swift \
- --without-lua --without-perl --without-php --without-php_extension
--without-dart --without-ruby --without-cl \
- --without-haskell --without-go --without-haxe --without-d
--without-python -without-java --without-dotnetcore -without-rs --with-cpp \
+ --disable-tutorial --without-qt5 --without-c_glib --without-java
--without-kotlin --without-erlang --without-nodejs --without-nodets \
+ --without-lua --without-python --without-py3 --without-perl
--without-php --without-php_extension \
+ --without-dart --without-ruby --without-go --without-rs --without-cl
--without-netstd --without-d --with-cpp \
--with-libevent="${TP_INSTALL_DIR}" --with-boost="${TP_INSTALL_DIR}"
--with-openssl="${TP_INSTALL_DIR}"
# Thrift's generated Makefiles put dependency include paths before
CXXFLAGS.
diff --git a/thirdparty/download-thirdparty.sh
b/thirdparty/download-thirdparty.sh
index 6b2f6d1970d..c2c627e47a0 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -691,22 +691,6 @@ else
fi
fi
-# patch thrift
-if [[ " ${TP_ARCHIVES[*]} " =~ " THRIFT " ]]; then
- if [[ "${THRIFT_SOURCE}" == 'thrift-0.16.0' ]]; then
- cd "${TP_SOURCE_DIR}/${THRIFT_SOURCE}"
- if [[ ! -f "${PATCHED_MARK}" ]]; then
- for patch_file in "${TP_PATCH_DIR}"/thrift-0.16*; do
- echo "patch ${patch_file}"
- patch -p1 --ignore-whitespace <"${patch_file}"
- done
- touch "${PATCHED_MARK}"
- fi
- cd -
- fi
- echo "Finished patching ${THRIFT_SOURCE}"
-fi
-
# patch re2
if [[ " ${TP_ARCHIVES[*]} " =~ " RE2 " ]]; then
if [[ "${RE2_SOURCE}" == 're2-2021-02-02' ]]; then
diff --git a/thirdparty/patches/thrift-0.16-fix_mutex_include.patch
b/thirdparty/patches/thrift-0.16-fix_mutex_include.patch
deleted file mode 100644
index ca4315fde3a..00000000000
--- a/thirdparty/patches/thrift-0.16-fix_mutex_include.patch
+++ /dev/null
@@ -1,9 +0,0 @@
---- a/lib/cpp/src/thrift/concurrency/Mutex.h
-+++ b/lib/cpp/src/thrift/concurrency/Mutex.h
-@@ -19,4 +19,5 @@
- #ifndef _THRIFT_CONCURRENCY_MUTEX_H_
- #define _THRIFT_CONCURRENCY_MUTEX_H_ 1
-
-+#include <cstdint>
- #include <memory>
- #include <thrift/TNonCopyable.h>
diff --git a/thirdparty/patches/thrift-0.16-reset-consumed-message-size.patch
b/thirdparty/patches/thrift-0.16-reset-consumed-message-size.patch
deleted file mode 100644
index a760e3742db..00000000000
--- a/thirdparty/patches/thrift-0.16-reset-consumed-message-size.patch
+++ /dev/null
@@ -1,29 +0,0 @@
-From 89e0bc5fa4949b68503f7b6892128cc8fc5bc1d4 Mon Sep 17 00:00:00 2001
-From: Steve Licking <[email protected]>
-Date: Fri, 31 Dec 2021 10:54:05 -0800
-Subject: [PATCH] THRIFT-5492: Add readEnd to TBufferedTransport client: cpp
- Patch: Steve Licking
-
----
- lib/cpp/src/thrift/transport/TBufferTransports.h | 5 +++++
- 1 file changed, 5 insertions(+)
-
-diff --git a/lib/cpp/src/thrift/transport/TBufferTransports.h
b/lib/cpp/src/thrift/transport/TBufferTransports.h
-index 179934ba0..6feb540af 100644
---- a/lib/cpp/src/thrift/transport/TBufferTransports.h
-+++ b/lib/cpp/src/thrift/transport/TBufferTransports.h
-@@ -270,6 +270,11 @@ public:
- */
- uint32_t readAll(uint8_t* buf, uint32_t len) { return
TBufferBase::readAll(buf, len); }
-
-+ uint32_t readEnd() override {
-+ resetConsumedMessageSize();
-+ return 0;
-+ }
-+
- protected:
- void initPointers() {
- setReadBuffer(rBuf_.get(), 0);
---
-2.39.3
-
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 3bd4cd9e2fa..600181d5b20 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -68,10 +68,10 @@ OPENSSL_SOURCE=openssl-OpenSSL_1_1_1s
OPENSSL_MD5SUM="7e79a7560dee77c0758baa33c61af4b4"
# thrift
-THRIFT_DOWNLOAD="http://archive.apache.org/dist/thrift/0.16.0/thrift-0.16.0.tar.gz";
-THRIFT_NAME=thrift-0.16.0.tar.gz
-THRIFT_SOURCE=thrift-0.16.0
-THRIFT_MD5SUM="44cf1b54b4ec1890576c85804acfa637"
+THRIFT_DOWNLOAD="http://archive.apache.org/dist/thrift/0.24.0/thrift-0.24.0.tar.gz";
+THRIFT_NAME=thrift-0.24.0.tar.gz
+THRIFT_SOURCE=thrift-0.24.0
+THRIFT_MD5SUM="232e035ff80c5fb4b7243f0be3a76b02"
# protobuf
# brpc is not yet compatible with protobuf >= 22
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
