cshuo commented on code in PR #19307:
URL: https://github.com/apache/hudi/pull/19307#discussion_r3612654147
##########
hudi-common/src/main/java/org/apache/hudi/common/table/read/lsm/LsmFileGroupRecordIterator.java:
##########
@@ -314,32 +315,85 @@ private ClosableIterator<BufferedRecord<T>>
createFileIterator(StoragePathInfo p
if (FSUtils.isNativeDeleteLogFile(storagePath.getName())) {
return createNativeDeleteLogIterator(pathInfo, storagePath, fileSize);
}
- Pair<HoodieSchema, Map<String, String>> requiredSchemaAndRenamedColumns =
-
readerContext.getSchemaHandler().getRequiredSchemaForFileAndRenamedColumns(storagePath);
- HoodieSchema fileRequiredSchema =
requiredSchemaAndRenamedColumns.getLeft();
- ClosableIterator<T> recordIterator;
+ return FSUtils.isLogFile(storagePath)
+ ? createLogFileIterator(pathInfo, storagePath, fileSize)
+ : createBaseFileIterator(pathInfo, storagePath, fileSize);
+ }
+
+ /**
+ * Reads a base file using the engine's schema-evolution support, matching
+ * {@code HoodieFileGroupReader#makeBaseFileIterator}.
+ */
+ private ClosableIterator<BufferedRecord<T>>
createBaseFileIterator(StoragePathInfo pathInfo,
+
StoragePath storagePath,
+ long
fileSize) throws IOException {
+ ClosableIterator<T> recordIterator = getFileRecordIterator(
+ pathInfo,
+ storagePath,
+ fileSize,
+ readerContext.getSchemaHandler().getTableSchema(),
+ readerSchema);
+ return toBufferedRecordIterator(recordIterator, readerSchema);
+ }
+
+ /**
+ * Reads a native data log with the same schema flow as an inline log block.
+ *
+ * <p>With schema evolution enabled, records are decoded with the writer
schema stored in the
+ * native log footer and then transformed once to the evolved schema for
that instant. Without
+ * schema evolution, the file reader projects directly to the required
reader schema.
+ */
+ private ClosableIterator<BufferedRecord<T>>
createLogFileIterator(StoragePathInfo pathInfo,
Review Comment:
https://github.com/apache/hudi/issues/19326
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
