This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new f3e26e571b0 branch-3.1: [fix](jni)Adjust the statistical time of JNI
appenddata. #58224 (#58385)
f3e26e571b0 is described below
commit f3e26e571b005cc6a77db5e4578d7b096c7652ab
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Nov 27 10:25:03 2025 +0800
branch-3.1: [fix](jni)Adjust the statistical time of JNI appenddata. #58224
(#58385)
Cherry-picked from #58224
Co-authored-by: daidai <[email protected]>
---
.../src/main/java/org/apache/doris/avro/AvroJNIScanner.java | 2 ++
.../main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java | 2 ++
.../java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java | 5 +++++
.../main/java/org/apache/doris/common/jni/MockJniScanner.java | 2 ++
.../java/org/apache/doris/maxcompute/MaxComputeJniScanner.java | 3 +++
.../src/main/java/org/apache/doris/paimon/PaimonJniScanner.java | 9 ++++++---
.../java/org/apache/doris/paimon/PaimonSysTableJniScanner.java | 9 ++++++---
.../apache/doris/trinoconnector/TrinoConnectorJniScanner.java | 3 +++
8 files changed, 29 insertions(+), 6 deletions(-)
diff --git
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroJNIScanner.java
b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroJNIScanner.java
index dc845f43cb8..2ef0bf1d45a 100644
---
a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroJNIScanner.java
+++
b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroJNIScanner.java
@@ -196,6 +196,7 @@ public class AvroJNIScanner extends JniScanner {
@Override
protected int getNext() throws IOException {
int numRows = 0;
+ long startTime = System.nanoTime();
for (; numRows < getBatchSize(); numRows++) {
if (!avroReader.hasNext(inputPair, ignore)) {
break;
@@ -211,6 +212,7 @@ public class AvroJNIScanner extends JniScanner {
}
}
}
+ appendDataTime += System.nanoTime() - startTime;
return numRows;
}
diff --git
a/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java
b/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java
index 5fa4c72951b..e4627c56f57 100644
---
a/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java
+++
b/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java
@@ -166,6 +166,7 @@ public class HadoopHudiJniScanner extends JniScanner {
return preExecutionAuthenticator.execute(() -> {
NullWritable key = reader.createKey();
ArrayWritable value = reader.createValue();
+ long startTime = System.nanoTime();
int numRows = 0;
for (; numRows < fetchSize; numRows++) {
if (!reader.next(key, value)) {
@@ -185,6 +186,7 @@ public class HadoopHudiJniScanner extends JniScanner {
if (fields.length == 0) {
vectorTable.appendVirtualData(numRows);
}
+ appendDataTime += System.nanoTime() - startTime;
return numRows;
});
} catch (Exception e) {
diff --git
a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java
b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java
index 977dfcd898b..d1b648a9801 100644
---
a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java
+++
b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java
@@ -106,9 +106,13 @@ public class IcebergSysTableJniScanner extends JniScanner {
protected int getNext() throws IOException {
try (ThreadClassLoaderContext ignored = new
ThreadClassLoaderContext(classLoader)) {
int rows = 0;
+ long startAppendDataTime = System.nanoTime();
+ long scanTime = 0;
while (rows < getBatchSize()) {
while (!reader.hasNext() && scanTasks.hasNext()) {
+ long startScanTaskTime = System.nanoTime();
nextScanTask();
+ scanTime = System.nanoTime() - startScanTaskTime;
}
if (!reader.hasNext()) {
break;
@@ -122,6 +126,7 @@ public class IcebergSysTableJniScanner extends JniScanner {
}
rows++;
}
+ appendDataTime += System.nanoTime() - startAppendDataTime -
scanTime;
return rows;
}
}
diff --git
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java
index 000f536e915..ecf3dd670cf 100644
---
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java
+++
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java
@@ -205,6 +205,7 @@ public class MockJniScanner extends JniScanner {
return 0;
}
int rows = Math.min(batchSize, mockRows - readRows);
+ long startTime = System.nanoTime();
for (int i = 0; i < rows; ++i) {
for (int j = 0; j < types.length; ++j) {
if ((i + j) % 16 == 0) {
@@ -215,6 +216,7 @@ public class MockJniScanner extends JniScanner {
}
}
}
+ appendDataTime += System.nanoTime() - startTime;
readRows += rows;
return rows;
}
diff --git
a/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java
b/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java
index 30376792546..4128d66f09d 100644
---
a/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java
+++
b/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java
@@ -263,6 +263,7 @@ public class MaxComputeJniScanner extends JniScanner {
List<FieldVector> fieldVectors = data.getFieldVectors();
int batchRows = 0;
+ long startTime = System.nanoTime();
for (FieldVector column : fieldVectors) {
Integer readColumnId =
readColumnsToId.get(column.getName());
batchRows = column.getValueCount();
@@ -275,6 +276,8 @@ public class MaxComputeJniScanner extends JniScanner {
appendData(readColumnId, columnValue);
}
}
+ appendDataTime += System.nanoTime() - startTime;
+
curReadRows += batchRows;
} catch (Exception e) {
String errorMsg = String.format("MaxComputeJniScanner Fail to
read arrow data. "
diff --git
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
index 4dc1fe90a33..5690b7f6505 100644
---
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
+++
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
@@ -172,20 +172,23 @@ public class PaimonJniScanner extends JniScanner {
}
while (recordIterator != null) {
+ long startTime = System.nanoTime();
+
InternalRow record;
while ((record = recordIterator.next()) != null) {
+ rows++;
columnValue.setOffsetRow(record);
for (int i = 0; i < fields.length; i++) {
columnValue.setIdx(i, types[i],
paimonDataTypeList.get(i));
- long l = System.nanoTime();
appendData(i, columnValue);
- appendDataTime += System.nanoTime() - l;
}
- rows++;
if (rows >= batchSize) {
+ appendDataTime += System.nanoTime() - startTime;
return rows;
}
}
+ appendDataTime += System.nanoTime() - startTime;
+
recordIterator.releaseBatch();
recordIterator = reader.readBatch();
}
diff --git
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonSysTableJniScanner.java
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonSysTableJniScanner.java
index 79289bb8256..6c2eab84129 100644
---
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonSysTableJniScanner.java
+++
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonSysTableJniScanner.java
@@ -163,19 +163,22 @@ public class PaimonSysTableJniScanner extends JniScanner {
int rows = 0;
while (recordIterator != null) {
InternalRow record;
+ long startTime = System.nanoTime();
+
while ((record = recordIterator.next()) != null) {
+ rows += 1;
columnValue.setOffsetRow(record);
for (int i = 0; i < fields.length; i++) {
columnValue.setIdx(i, types[i], paimonDataTypeList.get(i));
- long l = System.nanoTime();
appendData(i, columnValue);
- appendDataTime += System.nanoTime() - l;
}
- rows++;
+
if (rows >= batchSize) {
+ appendDataTime += System.nanoTime() - startTime;
return rows;
}
}
+
recordIterator.releaseBatch();
recordIterator = reader.readBatch();
if (recordIterator == null && paimonSplits.hasNext()) {
diff --git
a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java
b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java
index 2138a432037..4fb39c90b7e 100644
---
a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java
+++
b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java
@@ -179,6 +179,9 @@ public class TrinoConnectorJniScanner extends JniScanner {
@Override
public void close() throws IOException {
+ for (long appendDataTimeN : appendDataTimeNs) {
+ appendDataTime += appendDataTimeN;
+ }
if (source != null) {
source.close();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
