sampan-s-nayak commented on code in PR #11018:
URL: https://github.com/apache/hudi/pull/11018#discussion_r1570282467
##########
hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java:
##########
@@ -150,21 +159,27 @@ public HoodieStreamer(Config cfg, JavaSparkContext jssc,
FileSystem fs, Configur
public HoodieStreamer(Config cfg, JavaSparkContext jssc, FileSystem fs,
Configuration conf,
Option<TypedProperties> propsOverride,
Option<SourceProfileSupplier> sourceProfileSupplier) throws IOException {
this.properties = combineProperties(cfg, propsOverride,
jssc.hadoopConfiguration());
- if (cfg.initialCheckpointProvider != null && cfg.checkpoint == null) {
- InitialCheckPointProvider checkPointProvider =
-
UtilHelpers.createInitialCheckpointProvider(cfg.initialCheckpointProvider,
this.properties);
- checkPointProvider.init(conf);
- cfg.checkpoint = checkPointProvider.getCheckpoint();
- }
-
this.cfg = cfg;
+ initializeCheckpoint(cfg, conf, properties);
this.bootstrapExecutor = Option.ofNullable(
cfg.runBootstrap ? new BootstrapExecutor(cfg, jssc, fs, conf,
this.properties) : null);
HoodieSparkEngineContext sparkEngineContext = new
HoodieSparkEngineContext(jssc);
this.ingestionService = Option.ofNullable(
cfg.runBootstrap ? null : new StreamSyncService(cfg,
sparkEngineContext, fs, conf, Option.ofNullable(this.properties),
sourceProfileSupplier));
}
+ private void initializeCheckpoint(Config cfg, Configuration conf,
TypedProperties properties) throws IOException {
+ boolean ignoreCheckpoint = StringUtils.nonEmpty(cfg.ignoreCheckpoint);
+ if (ignoreCheckpoint) {
+ cfg.checkpoint = null;
Review Comment:
yes you are right, removing this code block
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
