This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-4.2 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 608f2f533cad51d5d2d694eb04e9710481d0c20e Author: Calvin Kirs <[email protected]> AuthorDate: Wed Sep 16 21:56:32 2026 +0800 branch-4.1: [fix](fe) Upgrade gRPC and remove unused LZ4 dependencies (#68060) https://github.com/apache/doris/pull/67585 --- .../max-compute-connector/pom.xml | 3 ++ .../doris/maxcompute/MaxComputeJniWriterTest.java | 52 ++++++++++++++++++++++ fe/fe-core/pom.xml | 1 + fe/pom.xml | 17 ++++++- 4 files changed, 72 insertions(+), 1 deletion(-) diff --git a/fe/be-java-extensions/max-compute-connector/pom.xml b/fe/be-java-extensions/max-compute-connector/pom.xml index 4662ab83a53..0fdd097e7de 100644 --- a/fe/be-java-extensions/max-compute-connector/pom.xml +++ b/fe/be-java-extensions/max-compute-connector/pom.xml @@ -98,6 +98,9 @@ under the License. <plugin> <artifactId>maven-surefire-plugin</artifactId> <configuration> + <argLine> + --add-opens=java.base/java.nio=ALL-UNNAMED -Dio.netty.tryReflectionSetAccessible=true @{argLine} + </argLine> <forkCount>${fe_ut_parallel}</forkCount> <reuseForks>false</reuseForks> <useFile>false</useFile> diff --git a/fe/be-java-extensions/max-compute-connector/src/test/java/org/apache/doris/maxcompute/MaxComputeJniWriterTest.java b/fe/be-java-extensions/max-compute-connector/src/test/java/org/apache/doris/maxcompute/MaxComputeJniWriterTest.java index 84cc4a79d6e..a795637240a 100644 --- a/fe/be-java-extensions/max-compute-connector/src/test/java/org/apache/doris/maxcompute/MaxComputeJniWriterTest.java +++ b/fe/be-java-extensions/max-compute-connector/src/test/java/org/apache/doris/maxcompute/MaxComputeJniWriterTest.java @@ -17,6 +17,16 @@ package org.apache.doris.maxcompute; +import com.aliyun.odps.account.AliyunAccount; +import com.aliyun.odps.table.arrow.ArrowReader; +import com.aliyun.odps.table.arrow.ArrowReaderFactory; +import com.aliyun.odps.table.arrow.ArrowWriter; +import com.aliyun.odps.table.arrow.ArrowWriterFactory; +import com.aliyun.odps.table.configuration.CompressionCodec; +import com.aliyun.odps.table.configuration.ReaderOptions; +import com.aliyun.odps.table.configuration.WriterOptions; +import com.aliyun.odps.table.enviroment.Credentials; +import com.aliyun.odps.table.enviroment.EnvironmentSettings; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.IntVector; @@ -24,9 +34,51 @@ import org.apache.arrow.vector.VectorSchemaRoot; import org.junit.Assert; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.util.Collections; public class MaxComputeJniWriterTest { + @Test + public void testZstdArrowRoundTrip() throws Exception { + EnvironmentSettings settings = EnvironmentSettings.newBuilder() + .withCredentials(Credentials.newBuilder() + .withAccount(new AliyunAccount("test-access-key", "test-secret-key")).build()) + .build(); + WriterOptions writerOptions = WriterOptions.newBuilder().withSettings(settings) + .withCompressionCodec(CompressionCodec.ZSTD).build(); + try (BufferAllocator allocator = new RootAllocator(); + VectorSchemaRoot root = VectorSchemaRoot.of(new IntVector("c", allocator))) { + IntVector vector = (IntVector) root.getVector(0); + vector.allocateNew(1024); + for (int i = 0; i < 1024; i++) { + vector.set(i, i % 7); + } + vector.setNull(5); + root.setRowCount(1024); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ArrowWriter writer = ArrowWriterFactory.getRecordBatchWriter(bytes, writerOptions)) { + writer.writeBatch(root); + } + + ReaderOptions readerOptions = ReaderOptions.newBuilder().withSettings(settings) + .withBufferAllocator(allocator).withCompressionCodec(CompressionCodec.ZSTD) + .withReuseBatch(true).build(); + try (ArrowReader reader = ArrowReaderFactory.getRecordBatchReader( + new ByteArrayInputStream(bytes.toByteArray()), readerOptions)) { + Assert.assertTrue(reader.nextBatch()); + VectorSchemaRoot decoded = reader.getCurrentValue(); + Assert.assertEquals(root.getRowCount(), decoded.getRowCount()); + IntVector decodedVector = (IntVector) decoded.getVector(0); + for (int i = 0; i < root.getRowCount(); i++) { + Assert.assertEquals(vector.getObject(i), decodedVector.getObject(i)); + } + Assert.assertFalse(reader.nextBatch()); + } + } + } + @Test public void testPrefixBufferBytesMeasuresLeadingRowsWithoutRebuild() { try (BufferAllocator allocator = new RootAllocator(); diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index ef7b93414ef..848a731d4ba 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -753,6 +753,7 @@ under the License. <dependency> <groupId>org.apache.arrow</groupId> <artifactId>flight-sql-jdbc-driver</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>io.grpc</groupId> diff --git a/fe/pom.xml b/fe/pom.xml index 4906cd98208..41c040cc869 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -311,7 +311,7 @@ under the License. <!-- NOTE: Using grpc-java whose version is newer than 1.34.0 will break the build on CentOS 6 due to the obsolete GLIBC --> <grpc-java.version>1.34.0</grpc-java.version> <!--Need to ensure that the version is the same as in arrow/java/pom.xml or compatible with it.--> - <grpc.version>1.65.1</grpc.version> + <grpc.version>1.75.0</grpc.version> <check.freamework.version>3.53.0</check.freamework.version> <!-- FE-only Arrow 19 startup POC: align protobuf runtime with Arrow Java 19's protobuf BOM. --> <protobuf.version>4.33.4</protobuf.version> @@ -1474,6 +1474,10 @@ under the License. <artifactId>odps-sdk-core</artifactId> <version>${maxcompute.version}</version> <exclusions> + <exclusion> + <groupId>org.lz4</groupId> + <artifactId>lz4-pure-java</artifactId> + </exclusion> <exclusion> <groupId>org.apache.arrow</groupId> <artifactId>arrow-vector</artifactId> @@ -1492,6 +1496,17 @@ under the License. </exclusion> </exclusions> </dependency> + <dependency> + <groupId>com.aliyun.odps</groupId> + <artifactId>odps-sdk-table-api</artifactId> + <version>${maxcompute.version}</version> + <exclusions> + <exclusion> + <groupId>org.lz4</groupId> + <artifactId>lz4-pure-java</artifactId> + </exclusion> + </exclusions> + </dependency> <dependency> <!-- Keep tea pinned for Alibaba SDK dependency chains that allow com.aliyun:tea:[1.1.14,2.0.0). --> <groupId>com.aliyun</groupId> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
