xiaoxuandev commented on code in PR #12143:
URL: https://github.com/apache/hudi/pull/12143#discussion_r1813796441
##########
hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java:
##########
@@ -333,56 +334,56 @@ public StreamSync(HoodieStreamer.Config cfg, SparkSession
sparkSession,
*
* @throws IOException in case of any IOException
*/
- public void refreshTimeline() throws IOException {
- if (storage.exists(new StoragePath(cfg.targetBasePath))) {
- try {
- HoodieTableMetaClient metaClient = getMetaClient();
- switch (metaClient.getTableType()) {
- case COPY_ON_WRITE:
- case MERGE_ON_READ:
- // we can use getCommitsTimeline for both COW and MOR here,
because for COW there is no deltacommit
- this.commitsTimelineOpt =
Option.of(metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants());
- this.allCommitsTimelineOpt =
Option.of(metaClient.getActiveTimeline().getAllCommitsTimeline());
- break;
- default:
- throw new HoodieException("Unsupported table type :" +
metaClient.getTableType());
- }
- } catch (HoodieIOException e) {
- LOG.warn("Full exception msg " + e.getMessage());
- if (e.getMessage().contains("Could not load Hoodie properties") &&
e.getMessage().contains(HoodieTableConfig.HOODIE_PROPERTIES_FILE)) {
- String basePathWithForwardSlash = cfg.targetBasePath.endsWith("/") ?
cfg.targetBasePath :
- String.format("%s/", cfg.targetBasePath);
- String pathToHoodieProps = String.format("%s%s/%s",
basePathWithForwardSlash,
- HoodieTableMetaClient.METAFOLDER_NAME,
HoodieTableConfig.HOODIE_PROPERTIES_FILE);
- String pathToHoodiePropsBackup = String.format("%s%s/%s",
basePathWithForwardSlash,
- HoodieTableMetaClient.METAFOLDER_NAME,
- HoodieTableConfig.HOODIE_PROPERTIES_FILE_BACKUP);
- boolean hoodiePropertiesExists =
- storage.exists(new StoragePath(basePathWithForwardSlash))
- && storage.exists(new StoragePath(pathToHoodieProps))
- && storage.exists(new StoragePath(pathToHoodiePropsBackup));
- if (!hoodiePropertiesExists) {
- LOG.warn("Base path exists, but table is not fully initialized.
Re-initializing again");
- initializeEmptyTable();
- // reload the timeline from metaClient and validate that its empty
table. If there are any instants found, then we should fail the pipeline, bcoz
hoodie.properties got deleted by mistake.
- HoodieTableMetaClient metaClientToValidate = getMetaClient();
- if (metaClientToValidate.reloadActiveTimeline().countInstants() >
0) {
- // Deleting the recreated hoodie.properties and throwing
exception.
- storage.deleteDirectory(new StoragePath(String.format("%s%s/%s",
basePathWithForwardSlash,
- HoodieTableMetaClient.METAFOLDER_NAME,
- HoodieTableConfig.HOODIE_PROPERTIES_FILE)));
- throw new HoodieIOException(
- "hoodie.properties is missing. Likely due to some external
entity. Please populate the hoodie.properties and restart the pipeline. ",
- e.getIOException());
- }
+ public HoodieTableMetaClient refreshTimeline() throws IOException {
+ HoodieTableMetaClient metaClient = null;
Review Comment:
Fixed, thanks for the review!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
