alexeykudinkin commented on code in PR #7003:
URL: https://github.com/apache/hudi/pull/7003#discussion_r1035441012
##########
hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieParquetDataBlock.java:
##########
@@ -150,8 +151,17 @@ protected <T> ClosableIterator<HoodieRecord<T>>
readRecordsFromBlockPayload(Hood
blockContentLoc.getContentPositionInLogFile(),
blockContentLoc.getBlockSize());
+ Schema writerSchema = new
Schema.Parser().parse(this.getLogBlockHeader().get(HeaderMetadataType.SCHEMA));
+ if (!internalSchema.isEmptySchema()) {
Review Comment:
Same as https://github.com/apache/hudi/pull/7003/files#r1024817901
##########
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/ParquetBootstrapMetadataHandler.java:
##########
@@ -71,9 +71,10 @@ void executeBootstrap(HoodieBootstrapHandle<?, ?, ?, ?>
bootstrapHandle,
wrapper = new BoundedInMemoryExecutor<HoodieRecord, HoodieRecord,
Void>(config.getWriteBufferLimitBytes(),
reader.getRecordIterator(), new
BootstrapRecordConsumer(bootstrapHandle), record -> {
try {
- HoodieRecord recordCopy = record.copy();
- String recKey = recordCopy.getRecordKey(reader.getSchema(),
Option.of(keyGenerator));
- HoodieRecord hoodieRecord =
recordCopy.rewriteRecord(reader.getSchema(), config.getProps(),
HoodieAvroUtils.RECORD_KEY_SCHEMA);
+ String recKey = record.getRecordKey(reader.getSchema(),
Option.of(keyGenerator));
+ HoodieRecord hoodieRecord = record
+ .rewriteRecord(reader.getSchema(), config.getProps(),
HoodieAvroUtils.RECORD_KEY_SCHEMA)
+ .copy();
Review Comment:
Same comment as above
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java:
##########
@@ -132,20 +132,19 @@ public void runMerge(HoodieTable<T,
HoodieData<HoodieRecord<T>>, HoodieData<Hood
readerIterator = getMergingIterator(table, mergeHandle, baseFile,
reader, readSchema, externalSchemaTransformation);
} else {
if (needToReWriteRecord) {
- readerIterator = new RewriteIterator(reader.getRecordIterator(),
readSchema, readSchema, table.getConfig().getProps(), renameCols);
+ readerIterator = new RewriteIterator(reader.getRecordIterator(),
reader.getSchema(), readSchema, table.getConfig().getProps(), renameCols);
} else {
readerIterator = reader.getRecordIterator(readSchema);
}
}
wrapper = new
BoundedInMemoryExecutor<>(table.getConfig().getWriteBufferLimitBytes(),
readerIterator,
new UpdateHandler(mergeHandle), record -> {
- HoodieRecord recordCopy = record.copy();
if (!externalSchemaTransformation) {
- return recordCopy;
+ return record.copy();
Review Comment:
Let's add a comment elaborating why we're making a copy here
##########
hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java:
##########
@@ -394,7 +394,11 @@ private void processDataBlock(HoodieDataBlock dataBlock,
Option<KeySpec> keySpec
while (recordIterator.hasNext()) {
HoodieRecord currentRecord = recordIterator.next();
Schema schema = schemaOption.isPresent() ? schemaOption.get() :
dataBlock.getSchema();
- HoodieRecord record = schemaOption.isPresent() ?
currentRecord.rewriteRecordWithNewSchema(dataBlock.getSchema(), new
Properties(), schemaOption.get()) : currentRecord;
+ HoodieRecord record = schemaOption.isPresent()
+ ? currentRecord
+ .rewriteRecordWithNewSchema(dataBlock.getSchema(), new
Properties(), schemaOption.get())
Review Comment:
Please add a comment elaborating why copy is necessary (please ditto
everywhere, where we make a copy)
##########
hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java:
##########
@@ -61,7 +61,7 @@ public abstract class HoodieDataBlock extends HoodieLogBlock {
private final boolean enablePointLookups;
- protected final Schema readerSchema;
+ protected Schema readerSchema;
Review Comment:
@wzx140 we'd need to revert these changes after we rebase on the latest
master (these issues should have already been addressed on master)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
