This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 8bbe4c5fad0 branch-4.1: [fix](fe) Upgrade gRPC and remove unused LZ4
dependencies (#68060)
8bbe4c5fad0 is described below
commit 8bbe4c5fad0f0af0c4e121b5d483e22bc215eb4c
Author: Calvin Kirs <[email protected]>
AuthorDate: Wed Sep 16 21:56:32 2026 +0800
branch-4.1: [fix](fe) Upgrade gRPC and remove unused LZ4 dependencies
(#68060)
https://github.com/apache/doris/pull/67585
---
.../max-compute-connector/pom.xml | 3 ++
.../doris/maxcompute/MaxComputeJniWriterTest.java | 52 ++++++++++++++++++++++
fe/fe-core/pom.xml | 1 +
fe/pom.xml | 17 ++++++-
4 files changed, 72 insertions(+), 1 deletion(-)
diff --git a/fe/be-java-extensions/max-compute-connector/pom.xml
b/fe/be-java-extensions/max-compute-connector/pom.xml
index 4662ab83a53..0fdd097e7de 100644
--- a/fe/be-java-extensions/max-compute-connector/pom.xml
+++ b/fe/be-java-extensions/max-compute-connector/pom.xml
@@ -98,6 +98,9 @@ under the License.
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
+ <argLine>
+ --add-opens=java.base/java.nio=ALL-UNNAMED
-Dio.netty.tryReflectionSetAccessible=true @{argLine}
+ </argLine>
<forkCount>${fe_ut_parallel}</forkCount>
<reuseForks>false</reuseForks>
<useFile>false</useFile>
diff --git
a/fe/be-java-extensions/max-compute-connector/src/test/java/org/apache/doris/maxcompute/MaxComputeJniWriterTest.java
b/fe/be-java-extensions/max-compute-connector/src/test/java/org/apache/doris/maxcompute/MaxComputeJniWriterTest.java
index 84cc4a79d6e..a795637240a 100644
---
a/fe/be-java-extensions/max-compute-connector/src/test/java/org/apache/doris/maxcompute/MaxComputeJniWriterTest.java
+++
b/fe/be-java-extensions/max-compute-connector/src/test/java/org/apache/doris/maxcompute/MaxComputeJniWriterTest.java
@@ -17,6 +17,16 @@
package org.apache.doris.maxcompute;
+import com.aliyun.odps.account.AliyunAccount;
+import com.aliyun.odps.table.arrow.ArrowReader;
+import com.aliyun.odps.table.arrow.ArrowReaderFactory;
+import com.aliyun.odps.table.arrow.ArrowWriter;
+import com.aliyun.odps.table.arrow.ArrowWriterFactory;
+import com.aliyun.odps.table.configuration.CompressionCodec;
+import com.aliyun.odps.table.configuration.ReaderOptions;
+import com.aliyun.odps.table.configuration.WriterOptions;
+import com.aliyun.odps.table.enviroment.Credentials;
+import com.aliyun.odps.table.enviroment.EnvironmentSettings;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.IntVector;
@@ -24,9 +34,51 @@ import org.apache.arrow.vector.VectorSchemaRoot;
import org.junit.Assert;
import org.junit.Test;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.util.Collections;
public class MaxComputeJniWriterTest {
+ @Test
+ public void testZstdArrowRoundTrip() throws Exception {
+ EnvironmentSettings settings = EnvironmentSettings.newBuilder()
+ .withCredentials(Credentials.newBuilder()
+ .withAccount(new AliyunAccount("test-access-key",
"test-secret-key")).build())
+ .build();
+ WriterOptions writerOptions =
WriterOptions.newBuilder().withSettings(settings)
+ .withCompressionCodec(CompressionCodec.ZSTD).build();
+ try (BufferAllocator allocator = new RootAllocator();
+ VectorSchemaRoot root = VectorSchemaRoot.of(new IntVector("c",
allocator))) {
+ IntVector vector = (IntVector) root.getVector(0);
+ vector.allocateNew(1024);
+ for (int i = 0; i < 1024; i++) {
+ vector.set(i, i % 7);
+ }
+ vector.setNull(5);
+ root.setRowCount(1024);
+
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ try (ArrowWriter writer =
ArrowWriterFactory.getRecordBatchWriter(bytes, writerOptions)) {
+ writer.writeBatch(root);
+ }
+
+ ReaderOptions readerOptions =
ReaderOptions.newBuilder().withSettings(settings)
+
.withBufferAllocator(allocator).withCompressionCodec(CompressionCodec.ZSTD)
+ .withReuseBatch(true).build();
+ try (ArrowReader reader = ArrowReaderFactory.getRecordBatchReader(
+ new ByteArrayInputStream(bytes.toByteArray()),
readerOptions)) {
+ Assert.assertTrue(reader.nextBatch());
+ VectorSchemaRoot decoded = reader.getCurrentValue();
+ Assert.assertEquals(root.getRowCount(), decoded.getRowCount());
+ IntVector decodedVector = (IntVector) decoded.getVector(0);
+ for (int i = 0; i < root.getRowCount(); i++) {
+ Assert.assertEquals(vector.getObject(i),
decodedVector.getObject(i));
+ }
+ Assert.assertFalse(reader.nextBatch());
+ }
+ }
+ }
+
@Test
public void testPrefixBufferBytesMeasuresLeadingRowsWithoutRebuild() {
try (BufferAllocator allocator = new RootAllocator();
diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml
index ef7b93414ef..848a731d4ba 100644
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@@ -753,6 +753,7 @@ under the License.
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>flight-sql-jdbc-driver</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
diff --git a/fe/pom.xml b/fe/pom.xml
index 4906cd98208..41c040cc869 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -311,7 +311,7 @@ under the License.
<!-- NOTE: Using grpc-java whose version is newer than 1.34.0 will
break the build on CentOS 6 due to the obsolete GLIBC -->
<grpc-java.version>1.34.0</grpc-java.version>
<!--Need to ensure that the version is the same as in
arrow/java/pom.xml or compatible with it.-->
- <grpc.version>1.65.1</grpc.version>
+ <grpc.version>1.75.0</grpc.version>
<check.freamework.version>3.53.0</check.freamework.version>
<!-- FE-only Arrow 19 startup POC: align protobuf runtime with Arrow
Java 19's protobuf BOM. -->
<protobuf.version>4.33.4</protobuf.version>
@@ -1474,6 +1474,10 @@ under the License.
<artifactId>odps-sdk-core</artifactId>
<version>${maxcompute.version}</version>
<exclusions>
+ <exclusion>
+ <groupId>org.lz4</groupId>
+ <artifactId>lz4-pure-java</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
@@ -1492,6 +1496,17 @@ under the License.
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>com.aliyun.odps</groupId>
+ <artifactId>odps-sdk-table-api</artifactId>
+ <version>${maxcompute.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.lz4</groupId>
+ <artifactId>lz4-pure-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<!-- Keep tea pinned for Alibaba SDK dependency chains that
allow com.aliyun:tea:[1.1.14,2.0.0). -->
<groupId>com.aliyun</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
