This is an automated email from the ASF dual-hosted git repository.
mengtao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 21bcbfd792 [HUDI-5237] Support for HoodieUnMergedLogRecordScanner with
InternalSchema (#7237)
21bcbfd792 is described below
commit 21bcbfd7923c65acd7e458d6ca2a05f1ff9df109
Author: Alexander Trushev <[email protected]>
AuthorDate: Mon Nov 21 10:26:16 2022 +0700
[HUDI-5237] Support for HoodieUnMergedLogRecordScanner with InternalSchema
(#7237)
* [HUDI-5237] Support for HoodieUnMergedLogRecordScanner with InternalSchema
---
.../common/table/log/AbstractHoodieLogRecordReader.java | 2 ++
.../common/table/log/HoodieMergedLogRecordScanner.java | 3 ++-
.../common/table/log/HoodieUnMergedLogRecordScanner.java | 15 ++++++++++++---
.../metadata/HoodieMetadataMergedLogRecordReader.java | 5 +++++
4 files changed, 21 insertions(+), 4 deletions(-)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java
index 88da6aa1f0..0c8d8b3f6c 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java
@@ -817,6 +817,8 @@ public abstract class AbstractHoodieLogRecordReader {
public abstract Builder withReaderSchema(Schema schema);
+ public abstract Builder withInternalSchema(InternalSchema internalSchema);
+
public abstract Builder withLatestInstantTime(String latestInstantTime);
public abstract Builder withReadBlocksLazily(boolean readBlocksLazily);
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java
index e846637493..708015b1c1 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java
@@ -308,8 +308,9 @@ public class HoodieMergedLogRecordScanner extends
AbstractHoodieLogRecordReader
return this;
}
+ @Override
public Builder withInternalSchema(InternalSchema internalSchema) {
- this.internalSchema = internalSchema == null ?
InternalSchema.getEmptyInternalSchema() : internalSchema;
+ this.internalSchema = internalSchema;
return this;
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java
index 7ddb9f1236..b0d127c562 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java
@@ -40,9 +40,10 @@ public class HoodieUnMergedLogRecordScanner extends
AbstractHoodieLogRecordReade
private HoodieUnMergedLogRecordScanner(FileSystem fs, String basePath,
List<String> logFilePaths, Schema readerSchema,
String latestInstantTime, boolean
readBlocksLazily, boolean reverseReader, int bufferSize,
- LogRecordScannerCallback callback,
Option<InstantRange> instantRange, boolean useScanV2) {
+ LogRecordScannerCallback callback,
Option<InstantRange> instantRange, InternalSchema internalSchema,
+ boolean useScanV2) {
super(fs, basePath, logFilePaths, readerSchema, latestInstantTime,
readBlocksLazily, reverseReader, bufferSize, instantRange,
- false, true, Option.empty(), InternalSchema.getEmptyInternalSchema(),
useScanV2);
+ false, true, Option.empty(), internalSchema, useScanV2);
this.callback = callback;
}
@@ -81,6 +82,7 @@ public class HoodieUnMergedLogRecordScanner extends
AbstractHoodieLogRecordReade
private String basePath;
private List<String> logFilePaths;
private Schema readerSchema;
+ private InternalSchema internalSchema;
private String latestInstantTime;
private boolean readBlocksLazily;
private boolean reverseReader;
@@ -112,6 +114,12 @@ public class HoodieUnMergedLogRecordScanner extends
AbstractHoodieLogRecordReade
return this;
}
+ @Override
+ public Builder withInternalSchema(InternalSchema internalSchema) {
+ this.internalSchema = internalSchema;
+ return this;
+ }
+
public Builder withLatestInstantTime(String latestInstantTime) {
this.latestInstantTime = latestInstantTime;
return this;
@@ -151,7 +159,8 @@ public class HoodieUnMergedLogRecordScanner extends
AbstractHoodieLogRecordReade
@Override
public HoodieUnMergedLogRecordScanner build() {
return new HoodieUnMergedLogRecordScanner(fs, basePath, logFilePaths,
readerSchema,
- latestInstantTime, readBlocksLazily, reverseReader, bufferSize,
callback, instantRange, useScanV2);
+ latestInstantTime, readBlocksLazily, reverseReader, bufferSize,
callback, instantRange,
+ internalSchema, useScanV2);
}
}
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMergedLogRecordReader.java
b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMergedLogRecordReader.java
index b844f219cd..afcd726a0b 100644
---
a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMergedLogRecordReader.java
+++
b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMergedLogRecordReader.java
@@ -168,6 +168,11 @@ public class HoodieMetadataMergedLogRecordReader extends
HoodieMergedLogRecordSc
return this;
}
+ @Override
+ public Builder withInternalSchema(InternalSchema internalSchema) {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public Builder withLatestInstantTime(String latestInstantTime) {
this.latestInstantTime = latestInstantTime;
