This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 02925398300 [fix](paimon)support readbatch #29039 (#29055)
02925398300 is described below
commit 029253983002301c315c394eb3127799f9a3946d
Author: wuwenchi <[email protected]>
AuthorDate: Wed Dec 27 13:00:08 2023 +0800
[fix](paimon)support readbatch #29039 (#29055)
---
.../java/org/apache/doris/paimon/PaimonJniScanner.java | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
index 237e06ad9c5..849f6fb67bb 100644
---
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
+++
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
@@ -57,6 +57,7 @@ public class PaimonJniScanner extends JniScanner {
private long dbId;
private long tblId;
private long lastUpdateTime;
+ private RecordReader.RecordIterator<InternalRow> recordIterator = null;
public PaimonJniScanner(int batchSize, Map<String, String> params) {
LOG.debug("params:{}", params);
@@ -133,18 +134,25 @@ public class PaimonJniScanner extends JniScanner {
protected int getNext() throws IOException {
int rows = 0;
try {
- RecordReader.RecordIterator<InternalRow> batch;
- while ((batch = reader.readBatch()) != null) {
+ if (recordIterator == null) {
+ recordIterator = reader.readBatch();
+ }
+
+ while (recordIterator != null) {
InternalRow record;
- while ((record = batch.next()) != null) {
+ while ((record = recordIterator.next()) != null) {
columnValue.setOffsetRow(record);
for (int i = 0; i < fields.length; i++) {
columnValue.setIdx(i, types[i]);
appendData(i, columnValue);
}
rows++;
+ if (rows >= batchSize) {
+ return rows;
+ }
}
- batch.releaseBatch();
+ recordIterator.releaseBatch();
+ recordIterator = reader.readBatch();
}
} catch (IOException e) {
LOG.warn("failed to getNext columnValue ", e);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
