This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new b349f999e730 refactor: Add Lombok Builder annotation to
TimelineService.Config (#17807)
b349f999e730 is described below
commit b349f999e730ff7d7903f769c033cae8db7e60a8
Author: voonhous <[email protected]>
AuthorDate: Fri Jan 9 13:34:50 2026 +0800
refactor: Add Lombok Builder annotation to TimelineService.Config (#17807)
---
.../client/embedded/EmbeddedTimelineService.java | 4 +-
.../hudi/timeline/service/TimelineService.java | 184 +++------------------
.../timeline/service/handlers/MarkerHandler.java | 3 +-
.../hudi/timeline/service/TestRequestHandler.java | 5 +-
.../hudi/timeline/service/TestTimelineService.java | 10 +-
.../TestRemoteHoodieTableFileSystemView.java | 4 +-
6 files changed, 46 insertions(+), 164 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
index ac78704cf2d5..2df9d0940d5b 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
@@ -144,7 +144,7 @@ public class EmbeddedTimelineService {
}
private void startServer(TimelineServiceCreator timelineServiceCreator)
throws IOException {
- TimelineService.Config.Builder timelineServiceConfBuilder =
TimelineService.Config.builder()
+ TimelineService.Config.ConfigBuilder timelineServiceConfBuilder =
TimelineService.Config.builder()
.serverPort(writeConfig.getEmbeddedTimelineServerPort())
.numThreads(writeConfig.getEmbeddedTimelineServerThreads())
.compress(writeConfig.getEmbeddedTimelineServerCompressOutput())
@@ -165,7 +165,7 @@ public class EmbeddedTimelineService {
.earlyConflictDetectionCheckCommitConflict(writeConfig.earlyConflictDetectionCheckCommitConflict())
.asyncConflictDetectorInitialDelayMs(writeConfig.getAsyncConflictDetectorInitialDelayMs())
.asyncConflictDetectorPeriodMs(writeConfig.getAsyncConflictDetectorPeriodMs())
- .earlyConflictDetectionMaxAllowableHeartbeatIntervalInMs(
+ .maxAllowableHeartbeatIntervalInMs(
writeConfig.getHoodieClientHeartbeatIntervalInMs()
* writeConfig.getHoodieClientHeartbeatTolerableMisses());
}
diff --git
a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java
b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java
index 968ceeb28cec..5df392fb946e 100644
---
a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java
+++
b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java
@@ -31,6 +31,8 @@ import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import io.javalin.Javalin;
import io.javalin.core.util.JavalinBindException;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -70,248 +72,116 @@ public class TimelineService {
/**
* Config for {@code TimelineService} class.
*/
+ @NoArgsConstructor
+ @AllArgsConstructor
+ @Builder
public static class Config implements Serializable {
+ @Builder.Default
@Parameter(names = {"--server-port", "-p"}, description = " Server Port")
public Integer serverPort = 26754;
+ @Builder.Default
@Parameter(names = {"--view-storage", "-st"}, description = "View Storage
Type. Default - SPILLABLE_DISK")
public FileSystemViewStorageType viewStorageType =
FileSystemViewStorageType.SPILLABLE_DISK;
+ @Builder.Default
@Parameter(names = {"--max-view-mem-per-table", "-mv"},
description = "Maximum view memory per table in MB to be used for
storing file-groups."
+ " Overflow file-groups will be spilled to disk. Used for
SPILLABLE_DISK storage type")
public Integer maxViewMemPerTableInMB = 2048;
+ @Builder.Default
@Parameter(names = {"--mem-overhead-fraction-pending-compaction", "-cf"},
description = "Memory Fraction of --max-view-mem-per-table to be
allocated for managing pending compaction"
+ " storage. Overflow entries will be spilled to disk. Used for
SPILLABLE_DISK storage type")
public Double memFractionForCompactionPerTable = 0.001;
+ @Builder.Default
@Parameter(names = {"--base-store-path", "-sp"},
description = "Directory where spilled view entries will be stored.
Used for SPILLABLE_DISK storage type")
public String baseStorePathForFileGroups =
FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue();
+ @Builder.Default
@Parameter(names = {"--rocksdb-path", "-rp"}, description = "Root
directory for RocksDB")
public String rocksDBPath =
FileSystemViewStorageConfig.ROCKSDB_BASE_PATH.defaultValue();
+ @Builder.Default
@Parameter(names = {"--threads", "-t"}, description = "Number of threads
to use for serving requests. The default number is 250")
public int numThreads = DEFAULT_NUM_THREADS;
+ @Builder.Default
@Parameter(names = {"--async"}, description = "Use asynchronous request
processing")
public boolean async = false;
+ @Builder.Default
@Parameter(names = {"--compress"}, description = "Compress output using
gzip")
public boolean compress = true;
+ @Builder.Default
@Parameter(names = {"--enable-marker-requests", "-em"}, description =
"Enable handling of marker-related requests")
public boolean enableMarkerRequests = false;
+ @Builder.Default
@Parameter(names = {"--enable-remote-partitioner"}, description = "Enable
remote partitioner")
public boolean enableRemotePartitioner = false;
+ @Builder.Default
@Parameter(names = {"--marker-batch-threads", "-mbt"}, description =
"Number of threads to use for batch processing marker creation requests")
public int markerBatchNumThreads = 20;
+ @Builder.Default
@Parameter(names = {"--marker-batch-interval-ms", "-mbi"}, description =
"The interval in milliseconds between two batch processing of marker creation
requests")
public long markerBatchIntervalMs = 50;
+ @Builder.Default
@Parameter(names = {"--marker-parallelism", "-mdp"}, description =
"Parallelism to use for reading and deleting marker files")
public int markerParallelism = 100;
+ @Builder.Default
@Parameter(names = {"--early-conflict-detection-strategy"}, description =
"The class name of the early conflict detection strategy to use. "
+ "This should be subclass of "
+
"`org.apache.hudi.common.conflict.detection.EarlyConflictDetectionStrategy`")
public String earlyConflictDetectionStrategy =
"org.apache.hudi.timeline.service.handlers.marker.AsyncTimelineServerBasedDetectionStrategy";
+ @Builder.Default
@Parameter(names = {"--early-conflict-detection-check-commit-conflict"},
description =
"Whether to enable commit conflict checking or not during early "
+ "conflict detection.")
- public boolean checkCommitConflict = false;
+ public boolean earlyConflictDetectionCheckCommitConflict = false;
+ @Builder.Default
@Parameter(names = {"--early-conflict-detection-enable"}, description =
"Whether to enable early conflict detection based on markers. "
+ "It eagerly detects writing conflict before create markers and
fails fast if a "
+ "conflict is detected, to release cluster compute resources as
soon as possible.")
public boolean earlyConflictDetectionEnable = false;
+ @Builder.Default
@Parameter(names = {"--async-conflict-detector-initial-delay-ms"},
description =
"Used for timeline-server-based markers with "
+ "`AsyncTimelineServerBasedDetectionStrategy`. "
+ "The time in milliseconds to delay the first execution of async
marker-based conflict detection.")
public Long asyncConflictDetectorInitialDelayMs = 0L;
+ @Builder.Default
@Parameter(names = {"--async-conflict-detector-period-ms"}, description =
"Used for timeline-server-based markers with "
+ "`AsyncTimelineServerBasedDetectionStrategy`. "
+ "The period in milliseconds between successive executions of
async marker-based conflict detection.")
public Long asyncConflictDetectorPeriodMs = 30000L;
+ @Builder.Default
@Parameter(names =
{"--early-conflict-detection-max-heartbeat-interval-ms"}, description =
"Used for timeline-server-based markers with "
+ "`AsyncTimelineServerBasedDetectionStrategy`. "
+ "Instants whose heartbeat is greater than the current value will
not be used in early conflict detection.")
public Long maxAllowableHeartbeatIntervalInMs = 120000L;
+ @Builder.Default
@Parameter(names = {"--help", "-h"})
public Boolean help = false;
-
- public static Builder builder() {
- return new Builder();
- }
-
- /**
- * Builder of Config class.
- */
- @NoArgsConstructor
- public static class Builder {
- private Integer serverPort = 26754;
- private FileSystemViewStorageType viewStorageType =
FileSystemViewStorageType.SPILLABLE_DISK;
- private Integer maxViewMemPerTableInMB = 2048;
- private Double memFractionForCompactionPerTable = 0.001;
- private String baseStorePathForFileGroups =
FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue();
- private String rocksDBPath =
FileSystemViewStorageConfig.ROCKSDB_BASE_PATH.defaultValue();
- private int numThreads = DEFAULT_NUM_THREADS;
- private boolean async = false;
- private boolean compress = true;
- private boolean enableMarkerRequests = false;
- private int markerBatchNumThreads = 20;
- private long markerBatchIntervalMs = 50L;
- private int markerParallelism = 100;
- private String earlyConflictDetectionStrategy =
"org.apache.hudi.timeline.service.handlers.marker.AsyncTimelineServerBasedDetectionStrategy";
- private Boolean checkCommitConflict = false;
- private Boolean earlyConflictDetectionEnable = false;
- private Long asyncConflictDetectorInitialDelayMs = 0L;
- private Long asyncConflictDetectorPeriodMs = 30000L;
- private Long maxAllowableHeartbeatIntervalInMs = 120000L;
- private boolean enableRemotePartitioner = false;
-
- public Builder serverPort(int serverPort) {
- this.serverPort = serverPort;
- return this;
- }
-
- public Builder viewStorageType(FileSystemViewStorageType
viewStorageType) {
- this.viewStorageType = viewStorageType;
- return this;
- }
-
- public Builder maxViewMemPerTableInMB(int maxViewMemPerTableInMB) {
- this.maxViewMemPerTableInMB = maxViewMemPerTableInMB;
- return this;
- }
-
- public Builder memFractionForCompactionPerTable(double
memFractionForCompactionPerTable) {
- this.memFractionForCompactionPerTable =
memFractionForCompactionPerTable;
- return this;
- }
-
- public Builder baseStorePathForFileGroups(String
baseStorePathForFileGroups) {
- this.baseStorePathForFileGroups = baseStorePathForFileGroups;
- return this;
- }
-
- public Builder rocksDBPath(String rocksDBPath) {
- this.rocksDBPath = rocksDBPath;
- return this;
- }
-
- public Builder numThreads(int numThreads) {
- this.numThreads = numThreads;
- return this;
- }
-
- public Builder async(boolean async) {
- this.async = async;
- return this;
- }
-
- public Builder compress(boolean compress) {
- this.compress = compress;
- return this;
- }
-
- public Builder enableMarkerRequests(boolean enableMarkerRequests) {
- this.enableMarkerRequests = enableMarkerRequests;
- return this;
- }
-
- public Builder enableRemotePartitioner(boolean enableRemotePartitioner) {
- this.enableRemotePartitioner = enableRemotePartitioner;
- return this;
- }
-
- public Builder markerBatchNumThreads(int markerBatchNumThreads) {
- this.markerBatchNumThreads = markerBatchNumThreads;
- return this;
- }
-
- public Builder markerBatchIntervalMs(long markerBatchIntervalMs) {
- this.markerBatchIntervalMs = markerBatchIntervalMs;
- return this;
- }
-
- public Builder markerParallelism(int markerParallelism) {
- this.markerParallelism = markerParallelism;
- return this;
- }
-
- public Builder earlyConflictDetectionStrategy(String
earlyConflictDetectionStrategy) {
- this.earlyConflictDetectionStrategy = earlyConflictDetectionStrategy;
- return this;
- }
-
- public Builder earlyConflictDetectionCheckCommitConflict(Boolean
checkCommitConflict) {
- this.checkCommitConflict = checkCommitConflict;
- return this;
- }
-
- public Builder earlyConflictDetectionEnable(Boolean
earlyConflictDetectionEnable) {
- this.earlyConflictDetectionEnable = earlyConflictDetectionEnable;
- return this;
- }
-
- public Builder asyncConflictDetectorInitialDelayMs(Long
asyncConflictDetectorInitialDelayMs) {
- this.asyncConflictDetectorInitialDelayMs =
asyncConflictDetectorInitialDelayMs;
- return this;
- }
-
- public Builder asyncConflictDetectorPeriodMs(Long
asyncConflictDetectorPeriodMs) {
- this.asyncConflictDetectorPeriodMs = asyncConflictDetectorPeriodMs;
- return this;
- }
-
- public Builder
earlyConflictDetectionMaxAllowableHeartbeatIntervalInMs(Long
maxAllowableHeartbeatIntervalInMs) {
- this.maxAllowableHeartbeatIntervalInMs =
maxAllowableHeartbeatIntervalInMs;
- return this;
- }
-
- public Config build() {
- Config config = new Config();
- config.serverPort = this.serverPort;
- config.viewStorageType = this.viewStorageType;
- config.maxViewMemPerTableInMB = this.maxViewMemPerTableInMB;
- config.memFractionForCompactionPerTable =
this.memFractionForCompactionPerTable;
- config.baseStorePathForFileGroups = this.baseStorePathForFileGroups;
- config.rocksDBPath = this.rocksDBPath;
- config.numThreads = this.numThreads;
- config.async = this.async;
- config.compress = this.compress;
- config.enableMarkerRequests = this.enableMarkerRequests;
- config.enableRemotePartitioner = this.enableRemotePartitioner;
- config.markerBatchNumThreads = this.markerBatchNumThreads;
- config.markerBatchIntervalMs = this.markerBatchIntervalMs;
- config.markerParallelism = this.markerParallelism;
- config.earlyConflictDetectionStrategy =
this.earlyConflictDetectionStrategy;
- config.checkCommitConflict = this.checkCommitConflict;
- config.earlyConflictDetectionEnable =
this.earlyConflictDetectionEnable;
- config.asyncConflictDetectorInitialDelayMs =
this.asyncConflictDetectorInitialDelayMs;
- config.asyncConflictDetectorPeriodMs =
this.asyncConflictDetectorPeriodMs;
- config.maxAllowableHeartbeatIntervalInMs =
this.maxAllowableHeartbeatIntervalInMs;
- return config;
- }
- }
}
private int startServiceOnPort(int port) throws IOException {
diff --git
a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/MarkerHandler.java
b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/MarkerHandler.java
index 2bcd4767de4f..7a35266720a6 100644
---
a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/MarkerHandler.java
+++
b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/MarkerHandler.java
@@ -201,7 +201,8 @@ public class MarkerHandler extends Handler {
earlyConflictDetectionStrategy =
(TimelineServerBasedDetectionStrategy)
ReflectionUtils.loadClass(
- strategyClassName, basePath, markerDir, markerName,
timelineServiceConfig.checkCommitConflict);
+ strategyClassName, basePath, markerDir, markerName,
+
timelineServiceConfig.earlyConflictDetectionCheckCommitConflict);
}
// markerDir => $base_path/.hoodie/.temp/$instant_time
diff --git
a/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/TestRequestHandler.java
b/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/TestRequestHandler.java
index f40b2212a2fe..acb8e4bd0d76 100644
---
a/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/TestRequestHandler.java
+++
b/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/TestRequestHandler.java
@@ -77,7 +77,10 @@ class TestRequestHandler extends HoodieCommonTestHarness {
}
TimelineServiceTestHarness.Builder builder =
TimelineServiceTestHarness.newBuilder();
server = builder.build(configuration,
-
TimelineService.Config.builder().serverPort(0).enableMarkerRequests(true).build(),
+ TimelineService.Config.builder()
+ .serverPort(0)
+ .enableMarkerRequests(true)
+ .build(),
FileSystemViewManager.createViewManager(localEngineContext,
metadataConfig, sConf, commonConfig));
server.startService();
} catch (Exception ex) {
diff --git
a/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/TestTimelineService.java
b/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/TestTimelineService.java
index c4ada81bdb9d..eb1184cf6bcd 100644
---
a/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/TestTimelineService.java
+++
b/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/TestTimelineService.java
@@ -45,7 +45,10 @@ class TestTimelineService {
try {
StorageConfiguration<Configuration> conf =
HoodieTestUtils.getDefaultStorageConf();
int originalServerPort = 8888;
- TimelineService.Config config =
TimelineService.Config.builder().enableMarkerRequests(true).serverPort(originalServerPort).build();
+ TimelineService.Config config = TimelineService.Config.builder()
+ .enableMarkerRequests(true)
+ .serverPort(originalServerPort)
+ .build();
FileSystemViewManager viewManager = mock(FileSystemViewManager.class);
timelineService = new TimelineService(conf, config, viewManager);
assertEquals(originalServerPort, timelineService.startService());
@@ -73,7 +76,10 @@ class TestTimelineService {
server.setExecutor(null);
server.start();
- TimelineService.Config config =
TimelineService.Config.builder().enableMarkerRequests(true).serverPort(originalServerPort).build();
+ TimelineService.Config config = TimelineService.Config.builder()
+ .enableMarkerRequests(true)
+ .serverPort(originalServerPort)
+ .build();
FileSystemViewManager viewManager = mock(FileSystemViewManager.class);
StorageConfiguration<Configuration> conf =
HoodieTestUtils.getDefaultStorageConf();
timelineService = new TimelineService(conf, config, viewManager);
diff --git
a/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/functional/TestRemoteHoodieTableFileSystemView.java
b/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/functional/TestRemoteHoodieTableFileSystemView.java
index c21e52f839a5..e2e8c5939cb3 100644
---
a/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/functional/TestRemoteHoodieTableFileSystemView.java
+++
b/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/functional/TestRemoteHoodieTableFileSystemView.java
@@ -84,7 +84,9 @@ public class TestRemoteHoodieTableFileSystemView extends
TestHoodieTableFileSyst
builder.withNumberOfSimulatedConnectionFailures(numberOfSimulatedConnectionFailures);
server = builder.build(
HadoopFSUtils.getStorageConf().unwrap(),
- TimelineService.Config.builder().serverPort(0).build(),
+ TimelineService.Config.builder()
+ .serverPort(0)
+ .build(),
FileSystemViewManager.createViewManager(localEngineContext,
metadataConfig, sConf, commonConfig));
server.startService();
} catch (Exception ex) {
