nsivabalan commented on code in PR #11018:
URL: https://github.com/apache/hudi/pull/11018#discussion_r1567935589
##########
hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java:
##########
@@ -150,21 +159,27 @@ public HoodieStreamer(Config cfg, JavaSparkContext jssc,
FileSystem fs, Configur
public HoodieStreamer(Config cfg, JavaSparkContext jssc, FileSystem fs,
Configuration conf,
Option<TypedProperties> propsOverride,
Option<SourceProfileSupplier> sourceProfileSupplier) throws IOException {
this.properties = combineProperties(cfg, propsOverride,
jssc.hadoopConfiguration());
- if (cfg.initialCheckpointProvider != null && cfg.checkpoint == null) {
- InitialCheckPointProvider checkPointProvider =
-
UtilHelpers.createInitialCheckpointProvider(cfg.initialCheckpointProvider,
this.properties);
- checkPointProvider.init(conf);
- cfg.checkpoint = checkPointProvider.getCheckpoint();
- }
-
this.cfg = cfg;
+ initialiseCheckpoint(cfg, conf, properties);
Review Comment:
```suggestion
initializeCheckpoint(cfg, conf, properties);
```
##########
hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java:
##########
@@ -150,21 +159,27 @@ public HoodieStreamer(Config cfg, JavaSparkContext jssc,
FileSystem fs, Configur
public HoodieStreamer(Config cfg, JavaSparkContext jssc, FileSystem fs,
Configuration conf,
Option<TypedProperties> propsOverride,
Option<SourceProfileSupplier> sourceProfileSupplier) throws IOException {
this.properties = combineProperties(cfg, propsOverride,
jssc.hadoopConfiguration());
- if (cfg.initialCheckpointProvider != null && cfg.checkpoint == null) {
- InitialCheckPointProvider checkPointProvider =
-
UtilHelpers.createInitialCheckpointProvider(cfg.initialCheckpointProvider,
this.properties);
- checkPointProvider.init(conf);
- cfg.checkpoint = checkPointProvider.getCheckpoint();
- }
-
this.cfg = cfg;
+ initialiseCheckpoint(cfg, conf, properties);
this.bootstrapExecutor = Option.ofNullable(
cfg.runBootstrap ? new BootstrapExecutor(cfg, jssc, fs, conf,
this.properties) : null);
HoodieSparkEngineContext sparkEngineContext = new
HoodieSparkEngineContext(jssc);
this.ingestionService = Option.ofNullable(
cfg.runBootstrap ? null : new StreamSyncService(cfg,
sparkEngineContext, fs, conf, Option.ofNullable(this.properties),
sourceProfileSupplier));
}
+ private void initialiseCheckpoint(Config cfg, Configuration conf,
TypedProperties properties) throws IOException {
Review Comment:
```suggestion
private void initializeCheckpoint(Config cfg, Configuration conf,
TypedProperties properties) throws IOException {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
