Davis-Zhang-Onehouse commented on code in PR #12718:
URL: https://github.com/apache/hudi/pull/12718#discussion_r1932744911
##########
hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieIncrSourceE2E.java:
##########
@@ -253,11 +254,44 @@ public void
testSyncE2ENoPrevCkpThenSyncMultipleTimes(String tableVersion, Strin
ds = new HoodieDeltaStreamer(createConfig(basePath(), null, sourceClass),
jsc, Option.of(props));
ds.sync();
- // After upgrading, we still use checkpoint V1 since this is s3/Gcs
incremental source.
+ // Assert no upgrade happens.
+ if (tableVersion.equals("6")) {
+ metaClient = getHoodieMetaClientWithTableVersion(storageConf(),
basePath(), "6");
+ assertEquals(HoodieTableVersion.SIX,
metaClient.getTableConfig().getTableVersion());
+ assertEquals(TimelineLayoutVersion.LAYOUT_VERSION_1,
metaClient.getTableConfig().getTimelineLayoutVersion().get());
+ }
+
+ // After that, we still use checkpoint V1 since this is s3/Gcs incremental
source.
expectedMetadata = new HashMap<>();
expectedMetadata.put("schema", "");
expectedMetadata.put(STREAMER_CHECKPOINT_KEY_V1, "30");
verifyLastInstantCommitMetadata(expectedMetadata);
+
+ // In the forth round, enable auto upgrade, use table write version 8, the
upgrade is successful.
+ props = setupBaseProperties("8");
+ props.put("hoodie.metadata.enable", "false");
+ // Dummy behavior injection to return ckp 1.
+ props.put(OP_FETCH_NEXT_BATCH, OP_EMPTY_ROW_SET_NONE_NULL_CKP_V1_KEY);
+ props.put(RETURN_CHECKPOINT_KEY, "40");
+ props.put("hoodie.write.auto.upgrade", "true");
+ // Validate the given checkpoint is ckp 2 when doing the sync.
+ props.put(VAL_INPUT_CKP, VAL_NON_EMPTY_CKP_ALL_MEMBERS);
+ props.put(VAL_CKP_KEY_EQ_VAL, "30");
+ props.put(VAL_CKP_RESET_KEY_IS_NULL, "IGNORED");
+ props.put(VAL_CKP_IGNORE_KEY_IS_NULL, "IGNORED");
+
+ ds = new HoodieDeltaStreamer(createConfig(basePath(), null, sourceClass),
jsc, Option.of(props));
+ ds.sync();
+
+ // After upgrading, we still use checkpoint V1 since this is s3/Gcs
incremental source.
+ // Assert the table is upgraded.
+ metaClient = getHoodieMetaClientWithTableVersion(storageConf(),
basePath(), "8");
+ assertEquals(HoodieTableVersion.EIGHT,
metaClient.getTableConfig().getTableVersion());
+ assertEquals(TimelineLayoutVersion.LAYOUT_VERSION_2,
metaClient.getTableConfig().getTimelineLayoutVersion().get());
Review Comment:
will have a follow up PR for code maintainability items. Will create jira.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
