(hudi) branch master updated: [HUDI-7062] Implement Caching Iterator for new filegroup reader (#10036)

[yihua]

This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 96cb780ec31 [HUDI-7062] Implement Caching Iterator for new filegroup 
reader (#10036)
96cb780ec31 is described below

commit 96cb780ec312e2f6d37e8b6e8468cfdd65a46058
Author: Jon Vexler <jbvex...@gmail.com>
AuthorDate: Thu Nov 9 13:53:27 2023 -0500

    [HUDI-7062] Implement Caching Iterator for new filegroup reader (#10036)
    
    Co-authored-by: Jonathan Vexler <=>
---
 .../table/read/HoodieBaseFileGroupRecordBuffer.java    | 18 ++++++++++++++++--
 .../read/HoodieKeyBasedFileGroupRecordBuffer.java      |  2 +-
 .../read/HoodiePositionBasedFileGroupRecordBuffer.java |  2 +-
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git 
a/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieBaseFileGroupRecordBuffer.java
 
b/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieBaseFileGroupRecordBuffer.java
index 06053293adf..f22e8f221f3 100644
--- 
a/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieBaseFileGroupRecordBuffer.java
+++ 
b/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieBaseFileGroupRecordBuffer.java
@@ -84,9 +84,23 @@ public abstract class HoodieBaseFileGroupRecordBuffer<T> 
implements HoodieFileGr
     this.baseFileIterator = baseFileIterator;
   }
 
+  /**
+   * This allows hasNext() to be called multiple times without incrementing 
the iterator by more than 1
+   * record. It does come with the caveat that hasNext() must be called every 
time before next(). But
+   * that is pretty expected behavior and every user should be doing that 
anyway.
+   */
+  protected abstract boolean doHasNext() throws IOException;
+
+  @Override
+  public final boolean hasNext() throws IOException {
+    return nextRecord != null || doHasNext();
+  }
+
   @Override
-  public T next() {
-    return nextRecord;
+  public final T next() {
+    T record = nextRecord;
+    nextRecord = null;
+    return record;
   }
 
   @Override
diff --git 
a/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieKeyBasedFileGroupRecordBuffer.java
 
b/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieKeyBasedFileGroupRecordBuffer.java
index 620c3d84940..2ab6048031f 100644
--- 
a/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieKeyBasedFileGroupRecordBuffer.java
+++ 
b/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieKeyBasedFileGroupRecordBuffer.java
@@ -121,7 +121,7 @@ public class HoodieKeyBasedFileGroupRecordBuffer<T> extends 
HoodieBaseFileGroupR
   }
 
   @Override
-  public boolean hasNext() throws IOException {
+  protected boolean doHasNext() throws IOException {
     ValidationUtils.checkState(baseFileIterator != null, "Base file iterator 
has not been set yet");
 
     // Handle merging.
diff --git 
a/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodiePositionBasedFileGroupRecordBuffer.java
 
b/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodiePositionBasedFileGroupRecordBuffer.java
index 4380567eacd..4bb70c1aa3d 100644
--- 
a/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodiePositionBasedFileGroupRecordBuffer.java
+++ 
b/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodiePositionBasedFileGroupRecordBuffer.java
@@ -163,7 +163,7 @@ public class HoodiePositionBasedFileGroupRecordBuffer<T> 
extends HoodieBaseFileG
   }
 
   @Override
-  public boolean hasNext() throws IOException {
+  protected boolean doHasNext() throws IOException {
     ValidationUtils.checkState(baseFileIterator != null, "Base file iterator 
has not been set yet");
 
     // Handle merging.