danny0405 commented on code in PR #12608:
URL: https://github.com/apache/hudi/pull/12608#discussion_r1920930956
##########
hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java:
##########
@@ -121,27 +121,43 @@ public interface RecordDeletionCallback {
/**
* Returns an iterator over the log records.
*/
- public Iterator<HoodieRecord<?>> iterator() {
- List<HoodieRecord<?>> records = new ArrayList<>();
- try {
- scan();
-
- while (!getCurrentInstantLogBlocks().isEmpty()) {
- HoodieLogBlock lastBlock = getCurrentInstantLogBlocks().pollLast();
- if (lastBlock instanceof HoodieDataBlock) {
- HoodieDataBlock dataBlock = (HoodieDataBlock) lastBlock;
- Pair<ClosableIterator<HoodieRecord>, Schema>
recordsIteratorSchemaPair = getRecordsIterator(dataBlock, Option.empty());
- try (ClosableIterator<HoodieRecord> recordIterator =
recordsIteratorSchemaPair.getLeft()) {
- while (recordIterator.hasNext()) {
- records.add(recordIterator.next());
+ public ClosableIterator<HoodieRecord<?>> iterator() {
+ return new ClosableIterator<HoodieRecord<?>>() {
+ private final Iterator<HoodieLogBlock> logBlockIterator =
getCurrentInstantLogBlocks().iterator();
+ private ClosableIterator<HoodieRecord> recordIterator = null;
+
+ @Override
+ public boolean hasNext() {
+ try {
+ while ((recordIterator == null || !recordIterator.hasNext()) &&
logBlockIterator.hasNext()) {
+ HoodieLogBlock logBlock = logBlockIterator.next();
+ if (logBlock instanceof HoodieDataBlock) {
+ HoodieDataBlock dataBlock = (HoodieDataBlock) logBlock;
+ Pair<ClosableIterator<HoodieRecord>, Schema>
recordsIteratorSchemaPair = getRecordsIterator(dataBlock, Option.empty());
+ recordIterator = recordsIteratorSchemaPair.getLeft();
}
}
+ return recordIterator != null && recordIterator.hasNext();
+ } catch (Exception e) {
+ throw new HoodieException("Error while iterating over log records",
e);
}
}
- } catch (Exception e) {
- throw new HoodieException("Error while iterating over log records", e);
- }
- return records.iterator();
+
+ @Override
+ public HoodieRecord<?> next() {
+ if (!hasNext()) {
Review Comment:
Maybe we remove this check because `hasNext` should always be invoked before
`next`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
