jonvex commented on code in PR #11415:
URL: https://github.com/apache/hudi/pull/11415#discussion_r1633546119
##########
hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodiePositionBasedFileGroupRecordBuffer.java:
##########
@@ -174,20 +189,97 @@ public boolean containsLogRecord(String recordKey) {
}
@Override
- protected boolean doHasNext() throws IOException {
- ValidationUtils.checkState(baseFileIterator != null, "Base file iterator
has not been set yet");
-
- // Handle merging.
- while (baseFileIterator.hasNext()) {
- T baseRecord = baseFileIterator.next();
- nextRecordPosition = readerContext.extractRecordPosition(baseRecord,
readerSchema, ROW_INDEX_TEMPORARY_COLUMN_NAME, nextRecordPosition);
- Pair<Option<T>, Map<String, Object>> logRecordInfo =
records.remove(nextRecordPosition++);
- if (hasNextBaseRecord(baseRecord, logRecordInfo)) {
- return true;
+ protected boolean hasNextBaseRecord(T baseRecord) throws IOException {
+ if (!readerContext.getShouldMergeUseRecordPosition()) {
+ return doHasNextFallbackBaseRecord(baseRecord);
+ }
+
+ nextRecordPosition = readerContext.extractRecordPosition(baseRecord,
readerSchema,
+ ROW_INDEX_COLUMN_NAME, nextRecordPosition);
+ Pair<Option<T>, Map<String, Object>> logRecordInfo =
records.remove(nextRecordPosition++);
+
+ Map<String, Object> metadata = readerContext.generateMetadataForRecord(
+ baseRecord, readerSchema);
+
+ Option<T> resultRecord = logRecordInfo != null
+ ? merge(Option.of(baseRecord), metadata, logRecordInfo.getLeft(),
logRecordInfo.getRight())
+ : merge(Option.empty(), Collections.emptyMap(), Option.of(baseRecord),
metadata);
+ if (resultRecord.isPresent()) {
+ nextRecord = readerContext.seal(resultRecord.get());
+ return true;
+ }
+ return false;
+ }
+
+ private boolean doHasNextFallbackBaseRecord(T baseRecord) throws IOException
{
+ if (needToDoHybridStrategy) {
+ //see if there is a delete block with record positions
+ nextRecordPosition = readerContext.extractRecordPosition(baseRecord,
readerSchema,
+ ROW_INDEX_TEMPORARY_COLUMN_NAME, nextRecordPosition);
+ Pair<Option<T>, Map<String, Object>> logRecordInfo =
records.remove(nextRecordPosition++);
Review Comment:
My understanding is that FG reader is single threaded
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
