nsivabalan commented on code in PR #12948:
URL: https://github.com/apache/hudi/pull/12948#discussion_r1997511013
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java:
##########
@@ -1734,4 +1654,387 @@ public boolean isInitialized() {
}
protected abstract BaseHoodieWriteClient<?, I, ?, ?> initializeWriteClient();
+
+ protected class MetadataWriteHandler implements Serializable {
Review Comment:
java docs.
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java:
##########
@@ -1734,4 +1654,387 @@ public boolean isInitialized() {
}
protected abstract BaseHoodieWriteClient<?, I, ?, ?> initializeWriteClient();
+
+ protected class MetadataWriteHandler implements Serializable {
+ private static final int PARTITION_INITIALIZATION_TIME_SUFFIX = 10;
+ HoodieTableVersion tableVersion;
+
+ public MetadataWriteHandler(HoodieTableVersion tableVersion) {
+ this.tableVersion = tableVersion;
+ }
+
+ public boolean shouldInitializeFromFilesystem(Set<String>
pendingDataInstants, Option<String> inflightInstantTimestamp) {
+ if (tableVersion.lesserThan(HoodieTableVersion.EIGHT) &&
pendingDataInstants.stream()
+ .anyMatch(i -> !inflightInstantTimestamp.isPresent() ||
!i.equals(inflightInstantTimestamp.get()))) {
+ metrics.ifPresent(m ->
m.updateMetrics(HoodieMetadataMetrics.BOOTSTRAP_ERR_STR, 1));
+ LOG.warn("Cannot initialize metadata table as operation(s) are in
progress on the dataset: {}",
+ Arrays.toString(pendingDataInstants.toArray()));
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ public String getTimelineHistoryPath() {
Review Comment:
can you revisit all access specifiers
##########
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java:
##########
@@ -101,10 +102,15 @@ public class TestHoodieBackedTableMetadata extends
TestHoodieMetadataBase {
private static final Logger LOG =
LoggerFactory.getLogger(TestHoodieBackedTableMetadata.class);
@ParameterizedTest
- @ValueSource(booleans = {true, false})
- public void testTableOperations(boolean reuseReaders) throws Exception {
+ @CsvSource({"true,6", "true,8", "false,6", "false,8"})
+ public void testTableOperations(boolean reuseReaders, int tableVersion)
throws Exception {
HoodieTableType tableType = HoodieTableType.COPY_ON_WRITE;
- init(tableType);
+ initPath();
+ HoodieWriteConfig config =
getWriteConfigBuilder(HoodieFailedWritesCleaningPolicy.EAGER, true, true,
false, true, false)
+ .build();
+ config.setValue(HoodieTableConfig.VERSION, String.valueOf(tableVersion));
+ config.setValue(HoodieWriteConfig.WRITE_TABLE_VERSION,
String.valueOf(tableVersion));
Review Comment:
we only need to set the `write table version`. can you avoid setting the
`table version` directly.
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala:
##########
@@ -418,48 +419,6 @@ class TestRecordLevelIndex extends
RecordLevelIndexTestBase {
validateDataAndRecordIndices(hudiOpts)
}
- @ParameterizedTest
- @CsvSource(value = Array(
- "COPY_ON_WRITE,6", "COPY_ON_WRITE,8", "MERGE_ON_READ,6", "MERGE_ON_READ,8"
- ))
- def testRLIWithDTClustering(tableType: String, tableVersion: Int): Unit = {
Review Comment:
why deleted this test?
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java:
##########
@@ -1734,4 +1654,387 @@ public boolean isInitialized() {
}
protected abstract BaseHoodieWriteClient<?, I, ?, ?> initializeWriteClient();
+
+ protected class MetadataWriteHandler implements Serializable {
+ private static final int PARTITION_INITIALIZATION_TIME_SUFFIX = 10;
+ HoodieTableVersion tableVersion;
Review Comment:
private final
##########
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java:
##########
@@ -101,10 +102,15 @@ public class TestHoodieBackedTableMetadata extends
TestHoodieMetadataBase {
private static final Logger LOG =
LoggerFactory.getLogger(TestHoodieBackedTableMetadata.class);
@ParameterizedTest
- @ValueSource(booleans = {true, false})
- public void testTableOperations(boolean reuseReaders) throws Exception {
+ @CsvSource({"true,6", "true,8", "false,6", "false,8"})
+ public void testTableOperations(boolean reuseReaders, int tableVersion)
throws Exception {
HoodieTableType tableType = HoodieTableType.COPY_ON_WRITE;
- init(tableType);
+ initPath();
+ HoodieWriteConfig config =
getWriteConfigBuilder(HoodieFailedWritesCleaningPolicy.EAGER, true, true,
false, true, false)
+ .build();
+ config.setValue(HoodieTableConfig.VERSION, String.valueOf(tableVersion));
+ config.setValue(HoodieWriteConfig.WRITE_TABLE_VERSION,
String.valueOf(tableVersion));
Review Comment:
at the end of every test, can we also add validation of table version in
table config in meta client.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
