rmahindra123 commented on code in PR #11018:
URL: https://github.com/apache/hudi/pull/11018#discussion_r1570247366
##########
hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java:
##########
@@ -150,21 +159,27 @@ public HoodieStreamer(Config cfg, JavaSparkContext jssc,
FileSystem fs, Configur
public HoodieStreamer(Config cfg, JavaSparkContext jssc, FileSystem fs,
Configuration conf,
Option<TypedProperties> propsOverride,
Option<SourceProfileSupplier> sourceProfileSupplier) throws IOException {
this.properties = combineProperties(cfg, propsOverride,
jssc.hadoopConfiguration());
- if (cfg.initialCheckpointProvider != null && cfg.checkpoint == null) {
- InitialCheckPointProvider checkPointProvider =
-
UtilHelpers.createInitialCheckpointProvider(cfg.initialCheckpointProvider,
this.properties);
- checkPointProvider.init(conf);
- cfg.checkpoint = checkPointProvider.getCheckpoint();
- }
-
this.cfg = cfg;
+ initializeCheckpoint(cfg, conf, properties);
this.bootstrapExecutor = Option.ofNullable(
cfg.runBootstrap ? new BootstrapExecutor(cfg, jssc, fs, conf,
this.properties) : null);
HoodieSparkEngineContext sparkEngineContext = new
HoodieSparkEngineContext(jssc);
this.ingestionService = Option.ofNullable(
cfg.runBootstrap ? null : new StreamSyncService(cfg,
sparkEngineContext, fs, conf, Option.ofNullable(this.properties),
sourceProfileSupplier));
}
+ private void initializeCheckpoint(Config cfg, Configuration conf,
TypedProperties properties) throws IOException {
+ boolean ignoreCheckpoint = StringUtils.nonEmpty(cfg.ignoreCheckpoint);
+ if (ignoreCheckpoint) {
+ cfg.checkpoint = null;
Review Comment:
we should not do this right? Otherwise, we cannot reset the checkpoint if
ignore checkpoint is applied sometime before
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
