This is an automated email from the ASF dual-hosted git repository.
CalvinKirs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2cf32a3bbdc [chore](thirdparty) upgrade thrift from 0.16.0 to 0.24.0
(#65990)
2cf32a3bbdc is described below
commit 2cf32a3bbdc73d73e7912d44f90817eb81209ae7
Author: Calvin Kirs <[email protected]>
AuthorDate: Tue Sep 1 10:04:44 2026 +0800
[chore](thirdparty) upgrade thrift from 0.16.0 to 0.24.0 (#65990)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
Upgrade Apache Thrift from 0.16.0 (released 2022) to the latest 0.24.0,
for both the thirdparty toolchain and the FE Java runtime.
Changes:
- `thirdparty/vars.sh`: bump thrift 0.16.0 -> 0.24.0.
- Remove both `thrift-0.16-*` patches, they are already included
upstream:
- `fix_mutex_include`: `Mutex.h` includes `<cstdint>` in 0.24.
- `reset-consumed-message-size`: this was a cherry-pick of upstream
THRIFT-5492, included since 0.17.
- Also remove the now-dead 0.16-gated patch block in
`download-thirdparty.sh`.
- `fe/pom.xml`: bump libthrift 0.16.0 -> 0.24.0.
- `fe/fe-thrift/pom.xml`: add `generated_annotations=suppress` to the
Java generator options. Newer thrift compilers emit
`@javax.annotation.Generated` by default, which no longer exists on JDK
17; suppressing the annotation avoids introducing a new dependency. The
option is also supported by thrift 0.16, so the change is compatible
with both compilers.
Notes:
- FE <-> BE RPC stays wire-compatible across thrift versions
(TBinaryProtocol is stable), so mixed-version deployments during rollout
are safe.
- The BE C++ adaptation to the 0.24 headers/runtime is handled
separately in a follow-up PR.
- For developers upgrading an existing local thirdparty in place: remove
the stale `installed/include/thrift` headers before running
`build-thirdparty.sh thrift`, otherwise the 0.16 headers in the install
prefix shadow the 0.24 in-tree headers via `-I installed/include` and
the lib/cpp build fails. Clean CI builds are not affected.
---
be/src/service/doris_main.cpp | 2 +-
be/src/util/thrift_util.cpp | 2 +-
.../java/org/apache/doris/qe/FEOpExecutor.java | 1 +
.../org/apache/doris/qe/PointQueryExecutor.java | 2 +-
.../java/org/apache/doris/qe/ResultReceiver.java | 13 +++++++-
fe/fe-thrift/pom.xml | 38 ++++++++++++++++++++--
fe/pom.xml | 2 +-
regression-test/framework/pom.xml | 36 ++++++++++++++++++--
thirdparty/CHANGELOG.md | 8 ++---
thirdparty/build-thirdparty.sh | 10 ++++--
thirdparty/download-thirdparty.sh | 16 ---------
.../patches/thrift-0.16-fix_mutex_include.patch | 9 -----
.../thrift-0.16-reset-consumed-message-size.patch | 29 -----------------
thirdparty/vars.sh | 8 ++---
14 files changed, 101 insertions(+), 75 deletions(-)
diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp
index dc55d6ce142..f46914f9101 100644
--- a/be/src/service/doris_main.cpp
+++ b/be/src/service/doris_main.cpp
@@ -509,7 +509,7 @@ int main(int argc, char** argv) {
exit(-1);
}
// add logger for thrift internal
- apache::thrift::GlobalOutput.setOutputFunction(doris::thrift_output);
+
apache::thrift::TOutput::instance().setOutputFunction(doris::thrift_output);
Status status = Status::OK();
if (doris::config::enable_java_support) {
diff --git a/be/src/util/thrift_util.cpp b/be/src/util/thrift_util.cpp
index b0d6d3b3093..6177d81770a 100644
--- a/be/src/util/thrift_util.cpp
+++ b/be/src/util/thrift_util.cpp
@@ -100,7 +100,7 @@ static void thrift_output_function(const char* output) {
}
void init_thrift_logging() {
- apache::thrift::GlobalOutput.setOutputFunction(thrift_output_function);
+
apache::thrift::TOutput::instance().setOutputFunction(thrift_output_function);
}
Status wait_for_local_server(const ThriftServer& server, int num_retries, int
retry_interval_ms) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/FEOpExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/FEOpExecutor.java
index f0e31c4aa5f..cb1f6d5da9e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/FEOpExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/FEOpExecutor.java
@@ -321,6 +321,7 @@ public class FEOpExecutor {
+ "`query_timeout`/`insert_timeout`")
.put(TTransportException.END_OF_FILE, "EOF")
.put(TTransportException.CORRUPTED_DATA, "Corrupted
data")
+ .put(TTransportException.MESSAGE_SIZE_LIMIT, "Message
size exceeds limit")
.build();
private final String msg;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java
index ad78d2d075b..b52774d4d03 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java
@@ -384,7 +384,7 @@ public class PointQueryExecutor implements CoordInterface {
try {
deserializer.deserialize(resultBatch, serialResult);
} catch (TException e) {
- if (e.getMessage().contains("MaxMessageSize reached")) {
+ if (ResultReceiver.isMessageSizeExceeded(e)) {
throw new TException("MaxMessageSize reached, try increase
max_msg_size_of_result_receiver");
} else {
throw e;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ResultReceiver.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ResultReceiver.java
index c5d1002e4cd..a67a5398f29 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ResultReceiver.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ResultReceiver.java
@@ -35,6 +35,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.thrift.TDeserializer;
import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
@@ -172,7 +173,7 @@ public class ResultReceiver {
try {
deserializer.deserialize(resultBatch, serialResult);
} catch (TException e) {
- if (e.getMessage().contains("MaxMessageSize reached"))
{
+ if (isMessageSizeExceeded(e)) {
throw new TException(
"MaxMessageSize reached, try increase
max_msg_size_of_result_receiver");
} else {
@@ -209,6 +210,16 @@ public class ResultReceiver {
return rowBatch;
}
+ // Thrift 0.24 reports an exceeded max message size as MESSAGE_SIZE_LIMIT;
older
+ // versions only carried the "MaxMessageSize reached" text on END_OF_FILE.
+ static boolean isMessageSizeExceeded(TException e) {
+ if (e instanceof TTransportException
+ && ((TTransportException) e).getType() ==
TTransportException.MESSAGE_SIZE_LIMIT) {
+ return true;
+ }
+ return e.getMessage() != null &&
e.getMessage().contains("MaxMessageSize reached");
+ }
+
public synchronized void cancel(Status reason) {
if (reason.isFinished()) {
return;
diff --git a/fe/fe-thrift/pom.xml b/fe/fe-thrift/pom.xml
index 19bdde55edb..7eba4ca2576 100644
--- a/fe/fe-thrift/pom.xml
+++ b/fe/fe-thrift/pom.xml
@@ -27,8 +27,10 @@ under the License.
<relativePath>../pom.xml</relativePath>
</parent>
<properties>
-
<doris.thrift.executable>${project.parent.basedir}/../thirdparty/installed/bin/thrift</doris.thrift.executable>
-
<doris.thrift.source>${project.parent.basedir}/../gensrc/thrift</doris.thrift.source>
+ <!-- project.basedir (not project.parent.basedir): the latter is not
interpolated
+ at model-build time, so its raw value would leak into the antrun
check below. -->
+
<doris.thrift.executable>${project.basedir}/../../thirdparty/installed/bin/thrift</doris.thrift.executable>
+
<doris.thrift.source>${project.basedir}/../../gensrc/thrift</doris.thrift.source>
</properties>
<profiles>
<profile>
@@ -82,12 +84,42 @@ under the License.
<finalName>doris-fe-thrift</finalName>
<directory>${project.basedir}/target/</directory>
<plugins>
+ <plugin>
+ <!-- Generated code only compiles against the matching
libthrift; fail fast
+ with a clear message instead of hundreds of generic-arity
errors. -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>3.1.0</version>
+ <executions>
+ <execution>
+ <id>check-thrift-compiler-version</id>
+ <phase>initialize</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <exec executable="${doris.thrift.executable}"
outputproperty="thrift.compiler.version" failonerror="true">
+ <arg value="--version"/>
+ </exec>
+ <fail message="thrift compiler
'${doris.thrift.executable}' reports '${thrift.compiler.version}' but libthrift
is ${thrift.version}. Rebuild thirdparty, or point doris.thrift.executable /
DORIS_THIRDPARTY at a thrift ${thrift.version} binary.">
+ <condition>
+ <not>
+ <contains
string="${thrift.compiler.version}" substring="${thrift.version}"/>
+ </not>
+ </condition>
+ </fail>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.apache.thrift</groupId>
<artifactId>thrift-maven-plugin</artifactId>
<version>0.10.0</version>
<configuration>
- <generator>java:fullcamel</generator>
+
<generator>java:fullcamel,generated_annotations=suppress</generator>
<thriftExecutable>${doris.thrift.executable}</thriftExecutable>
<thriftSourceRoot>${doris.thrift.source}</thriftSourceRoot>
</configuration>
diff --git a/fe/pom.xml b/fe/pom.xml
index 73f70562657..e0aee6b61a7 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -304,7 +304,7 @@ under the License.
<open-json.version>1.8</open-json.version>
<junit.version>5.14.1</junit.version>
<hikaricp.version>6.0.0</hikaricp.version>
- <thrift.version>0.16.0</thrift.version>
+ <thrift.version>0.24.0</thrift.version>
<tomcat-embed.version>9.0.104</tomcat-embed.version>
<log4j2.version>2.25.4</log4j2.version>
<log4j-1.2.version>2.25.4</log4j-1.2.version>
diff --git a/regression-test/framework/pom.xml
b/regression-test/framework/pom.xml
index 52660d5bcff..08dc4758a34 100644
--- a/regression-test/framework/pom.xml
+++ b/regression-test/framework/pom.xml
@@ -68,6 +68,8 @@ under the License.
<doris.home>${basedir}/../../</doris.home>
<doris.thrift.executable>${doris.home}/thirdparty/installed/bin/thrift</doris.thrift.executable>
<doris.thrift.source>${doris.home}/gensrc/thrift</doris.thrift.source>
+ <!-- Must match the thrift compiler installed in thirdparty and
fe/pom.xml. -->
+ <thrift.version>0.24.0</thrift.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
@@ -174,12 +176,42 @@ under the License.
<visitor>true</visitor>
</configuration>
</plugin>
+ <plugin>
+ <!-- Generated code only compiles against the matching
libthrift; fail fast
+ with a clear message instead of hundreds of generic-arity
errors. -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>3.1.0</version>
+ <executions>
+ <execution>
+ <id>check-thrift-compiler-version</id>
+ <phase>initialize</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <exec executable="${doris.thrift.executable}"
outputproperty="thrift.compiler.version" failonerror="true">
+ <arg value="--version"/>
+ </exec>
+ <fail message="thrift compiler
'${doris.thrift.executable}' reports '${thrift.compiler.version}' but libthrift
is ${thrift.version}. Rebuild thirdparty, or point doris.thrift.executable /
DORIS_THIRDPARTY at a thrift ${thrift.version} binary.">
+ <condition>
+ <not>
+ <contains
string="${thrift.compiler.version}" substring="${thrift.version}"/>
+ </not>
+ </condition>
+ </fail>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.apache.thrift</groupId>
<artifactId>thrift-maven-plugin</artifactId>
<version>0.10.0</version>
<configuration>
- <generator>java:fullcamel</generator>
+
<generator>java:fullcamel,generated_annotations=suppress</generator>
<thriftExecutable>${doris.thrift.executable}</thriftExecutable>
<thriftSourceRoot>${doris.thrift.source}</thriftSourceRoot>
</configuration>
@@ -219,7 +251,7 @@ under the License.
<dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
- <version>0.16.0</version>
+ <version>${thrift.version}</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md
index b483a7ee34e..9d9f55f27bb 100644
--- a/thirdparty/CHANGELOG.md
+++ b/thirdparty/CHANGELOG.md
@@ -2,6 +2,10 @@
This file contains version of the third-party dependency libraries in the
build-env image. The docker build-env image is apache/doris, and the tag is
`build-env-${version}`
+## 20260824
+
+- Modified: thrift 0.16.0 -> 0.24.0
+
## 20260819
- Added: `installed/include/hadoop_hdfs` and `installed/lib/hadoop_hdfs` are
symlinks to
@@ -28,10 +32,6 @@ This file contains version of the third-party dependency
libraries in the build-
- Modified: hadoop-libs 3.4.2.3 -> 3.4.2.4
-## 20260814
-
-- Modified: hadoop-libs 3.4.2.2 -> 3.4.2.3
-
## 20260206
- Modified: jindofs 6.8.2 -> 6.10.4
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index 6597f91ed1e..c85f42a498f 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -447,6 +447,10 @@ build_thrift() {
check_if_source_exist "${THRIFT_SOURCE}"
cd "${TP_SOURCE_DIR}/${THRIFT_SOURCE}"
+ # Headers of a previously installed thrift would shadow the in-tree ones
+ # via -I${TP_INCLUDE_DIR} and break an in-place version upgrade.
+ rm -rf "${TP_INSTALL_DIR}/include/thrift"
+
if [[ "${KERNEL}" != 'Darwin' ]]; then
cflags="-I${TP_INCLUDE_DIR}"
cxxflags="-I${TP_INCLUDE_DIR} ${warning_unused_but_set_variable}
-Wno-inconsistent-missing-override"
@@ -460,9 +464,9 @@ build_thrift() {
# NOTE(amos): libtool discard -static. --static works.
./configure CFLAGS="${cflags}" CXXFLAGS="${cxxflags}" LDFLAGS="${ldflags}"
LIBS="-lcrypto -ldl -lssl" \
--prefix="${TP_INSTALL_DIR}" --docdir="${TP_INSTALL_DIR}/doc"
--enable-static --disable-shared --disable-tests \
- --disable-tutorial --without-qt4 --without-qt5 --without-csharp
--without-erlang --without-nodejs --without-nodets --without-swift \
- --without-lua --without-perl --without-php --without-php_extension
--without-dart --without-ruby --without-cl \
- --without-haskell --without-go --without-haxe --without-d
--without-python -without-java --without-dotnetcore -without-rs --with-cpp \
+ --disable-tutorial --without-qt5 --without-c_glib --without-java
--without-kotlin --without-erlang --without-nodejs --without-nodets \
+ --without-lua --without-python --without-py3 --without-perl
--without-php --without-php_extension \
+ --without-dart --without-ruby --without-go --without-rs --without-cl
--without-netstd --without-d --with-cpp \
--with-libevent="${TP_INSTALL_DIR}" --with-boost="${TP_INSTALL_DIR}"
--with-openssl="${TP_INSTALL_DIR}"
if [[ -f compiler/cpp/thrifty.hh ]]; then
diff --git a/thirdparty/download-thirdparty.sh
b/thirdparty/download-thirdparty.sh
index 5af7fdcc3e5..5ac55f7985f 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -796,22 +796,6 @@ else
fi
fi
-# patch thrift
-if [[ " ${TP_ARCHIVES[*]} " =~ " THRIFT " ]]; then
- if [[ "${THRIFT_SOURCE}" == 'thrift-0.16.0' ]]; then
- cd "${TP_SOURCE_DIR}/${THRIFT_SOURCE}"
- if [[ ! -f "${PATCHED_MARK}" ]]; then
- for patch_file in "${TP_PATCH_DIR}"/thrift-*; do
- echo "patch ${patch_file}"
- patch -p1 --ignore-whitespace <"${patch_file}"
- done
- touch "${PATCHED_MARK}"
- fi
- cd -
- fi
- echo "Finished patching ${THRIFT_SOURCE}"
-fi
-
# patch re2
if [[ " ${TP_ARCHIVES[*]} " =~ " RE2 " ]]; then
if [[ "${RE2_SOURCE}" == 're2-2021-02-02' ]]; then
diff --git a/thirdparty/patches/thrift-0.16-fix_mutex_include.patch
b/thirdparty/patches/thrift-0.16-fix_mutex_include.patch
deleted file mode 100644
index ca4315fde3a..00000000000
--- a/thirdparty/patches/thrift-0.16-fix_mutex_include.patch
+++ /dev/null
@@ -1,9 +0,0 @@
---- a/lib/cpp/src/thrift/concurrency/Mutex.h
-+++ b/lib/cpp/src/thrift/concurrency/Mutex.h
-@@ -19,4 +19,5 @@
- #ifndef _THRIFT_CONCURRENCY_MUTEX_H_
- #define _THRIFT_CONCURRENCY_MUTEX_H_ 1
-
-+#include <cstdint>
- #include <memory>
- #include <thrift/TNonCopyable.h>
diff --git a/thirdparty/patches/thrift-0.16-reset-consumed-message-size.patch
b/thirdparty/patches/thrift-0.16-reset-consumed-message-size.patch
deleted file mode 100644
index a760e3742db..00000000000
--- a/thirdparty/patches/thrift-0.16-reset-consumed-message-size.patch
+++ /dev/null
@@ -1,29 +0,0 @@
-From 89e0bc5fa4949b68503f7b6892128cc8fc5bc1d4 Mon Sep 17 00:00:00 2001
-From: Steve Licking <[email protected]>
-Date: Fri, 31 Dec 2021 10:54:05 -0800
-Subject: [PATCH] THRIFT-5492: Add readEnd to TBufferedTransport client: cpp
- Patch: Steve Licking
-
----
- lib/cpp/src/thrift/transport/TBufferTransports.h | 5 +++++
- 1 file changed, 5 insertions(+)
-
-diff --git a/lib/cpp/src/thrift/transport/TBufferTransports.h
b/lib/cpp/src/thrift/transport/TBufferTransports.h
-index 179934ba0..6feb540af 100644
---- a/lib/cpp/src/thrift/transport/TBufferTransports.h
-+++ b/lib/cpp/src/thrift/transport/TBufferTransports.h
-@@ -270,6 +270,11 @@ public:
- */
- uint32_t readAll(uint8_t* buf, uint32_t len) { return
TBufferBase::readAll(buf, len); }
-
-+ uint32_t readEnd() override {
-+ resetConsumedMessageSize();
-+ return 0;
-+ }
-+
- protected:
- void initPointers() {
- setReadBuffer(rBuf_.get(), 0);
---
-2.39.3
-
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 13a3a04e6a6..b23d737a299 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -72,10 +72,10 @@ OPENSSL_SOURCE=openssl-OpenSSL_1_1_1s
OPENSSL_MD5SUM="7e79a7560dee77c0758baa33c61af4b4"
# thrift
-THRIFT_DOWNLOAD="http://archive.apache.org/dist/thrift/0.16.0/thrift-0.16.0.tar.gz";
-THRIFT_NAME=thrift-0.16.0.tar.gz
-THRIFT_SOURCE=thrift-0.16.0
-THRIFT_MD5SUM="44cf1b54b4ec1890576c85804acfa637"
+THRIFT_DOWNLOAD="http://archive.apache.org/dist/thrift/0.24.0/thrift-0.24.0.tar.gz";
+THRIFT_NAME=thrift-0.24.0.tar.gz
+THRIFT_SOURCE=thrift-0.24.0
+THRIFT_MD5SUM="232e035ff80c5fb4b7243f0be3a76b02"
# protobuf
# brpc is not yet compatible with protobuf >= 22
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
