hudi-agent commented on code in PR #19641:
URL: https://github.com/apache/hudi/pull/19641#discussion_r3790006598
##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java:
##########
@@ -349,6 +349,61 @@ public class FlinkOptions extends HoodieConfig {
.noDefaultValue()
.withDescription("Parallelism of tasks that do the index writing,
default is the parallelism of the execution environment");
+ // ------------------------------------------------------------------------
+ // RocksDB Partitioned RLI Cache Options
+ // ------------------------------------------------------------------------
+
+ @AdvancedConfig
+ public static final ConfigOption<Boolean> INDEX_RLI_ROCKSDB_CACHE_ENABLED =
ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.enabled")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription("Enable RocksDB-based partitioned cache for record
level index lookups, default false.");
+
+ @AdvancedConfig
+ public static final ConfigOption<String> INDEX_RLI_ROCKSDB_CACHE_BASE_PATH =
ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.base.path")
+ .stringType()
+ .defaultValue("/tmp/hudi-index-cache")
+ .withDescription("Local directory for RocksDB partitioned record level
index cache data.");
+
+ @AdvancedConfig
+ public static final ConfigOption<Integer>
INDEX_RLI_ROCKSDB_CACHE_BOOTSTRAP_DAYS = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.bootstrap.days")
+ .intType()
+ .defaultValue(7)
+ .withDescription("Number of days of Partitioned Record Index to load
during bootstrap. Only partitions "
+ + "within this window are pre-loaded; older partitions are loaded on
demand when updates are observed.");
+
+ @AdvancedConfig
+ public static final ConfigOption<Long>
INDEX_RLI_ROCKSDB_CACHE_PARTITION_TTL_HOURS = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.partition.ttl.hours")
+ .longType()
+ .defaultValue(168L) // default 7 days
+ .withDescription("TTL for partition column families in the RocksDB
partitioned record level index cache, default 168 hours (7 days).");
+
+ @AdvancedConfig
+ public static final ConfigOption<Long>
INDEX_RLI_ROCKSDB_CACHE_BLOCK_CACHE_MB = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.block.cache.mb")
+ .longType()
+ .defaultValue(256L)
+ .withDescription("RocksDB block cache size (off-heap) in MB for the
partitioned record level index cache, default 256.");
+
+ @AdvancedConfig
+ public static final ConfigOption<String>
INDEX_RLI_ROCKSDB_CACHE_COMPACTION_STYLE = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.compaction.style")
+ .stringType()
+ .defaultValue("LEVEL")
+ .withDescription("RocksDB compaction style for the partitioned record
level index cache, default LEVEL.");
+
+ @AdvancedConfig
+ public static final ConfigOption<Boolean>
INDEX_RLI_ROCKSDB_CACHE_INVALIDATE_ON_REPLACE_COMMIT = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.invalidate.on.replacecommit")
Review Comment:
🤖 Before renaming to `...invalidate.on.replace.commit`, note that this token
mirrors the timeline action name, which is a single word:
`HoodieTimeline.REPLACE_COMMIT_ACTION = "replacecommit"`
(hudi-common/.../HoodieTimeline.java:64). Dotting it would diverge from the
action spelling used across the timeline, so `replacecommit` is arguably the
consistent form here.
##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java:
##########
@@ -349,6 +349,61 @@ public class FlinkOptions extends HoodieConfig {
.noDefaultValue()
.withDescription("Parallelism of tasks that do the index writing,
default is the parallelism of the execution environment");
+ // ------------------------------------------------------------------------
+ // RocksDB Partitioned RLI Cache Options
+ // ------------------------------------------------------------------------
+
+ @AdvancedConfig
+ public static final ConfigOption<Boolean> INDEX_RLI_ROCKSDB_CACHE_ENABLED =
ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.enabled")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription("Enable RocksDB-based partitioned cache for record
level index lookups, default false.");
+
+ @AdvancedConfig
+ public static final ConfigOption<String> INDEX_RLI_ROCKSDB_CACHE_BASE_PATH =
ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.base.path")
+ .stringType()
+ .defaultValue("/tmp/hudi-index-cache")
+ .withDescription("Local directory for RocksDB partitioned record level
index cache data.");
+
+ @AdvancedConfig
+ public static final ConfigOption<Integer>
INDEX_RLI_ROCKSDB_CACHE_BOOTSTRAP_DAYS = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.bootstrap.days")
+ .intType()
+ .defaultValue(7)
+ .withDescription("Number of days of Partitioned Record Index to load
during bootstrap. Only partitions "
+ + "within this window are pre-loaded; older partitions are loaded on
demand when updates are observed.");
+
+ @AdvancedConfig
+ public static final ConfigOption<Long>
INDEX_RLI_ROCKSDB_CACHE_PARTITION_TTL_HOURS = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.partition.ttl.hours")
+ .longType()
+ .defaultValue(168L) // default 7 days
+ .withDescription("TTL for partition column families in the RocksDB
partitioned record level index cache, default 168 hours (7 days).");
+
+ @AdvancedConfig
+ public static final ConfigOption<Long>
INDEX_RLI_ROCKSDB_CACHE_BLOCK_CACHE_MB = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.block.cache.mb")
+ .longType()
+ .defaultValue(256L)
+ .withDescription("RocksDB block cache size (off-heap) in MB for the
partitioned record level index cache, default 256.");
+
+ @AdvancedConfig
+ public static final ConfigOption<String>
INDEX_RLI_ROCKSDB_CACHE_COMPACTION_STYLE = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.compaction.style")
+ .stringType()
+ .defaultValue("LEVEL")
+ .withDescription("RocksDB compaction style for the partitioned record
level index cache, default LEVEL.");
+
+ @AdvancedConfig
+ public static final ConfigOption<Boolean>
INDEX_RLI_ROCKSDB_CACHE_INVALIDATE_ON_REPLACE_COMMIT = ConfigOptions
+ .key("hoodie.record.index.cache.rocksdb.invalidate.on.replacecommit")
Review Comment:
🤖 One thing worth factoring in here: this key derives from the timeline
action name, which is spelled as a single token —
`HoodieTimeline.REPLACE_COMMIT_ACTION = "replacecommit"`
(hudi-common/.../HoodieTimeline.java:64). Splitting it into `replace.commit`
would actually diverge from how the action is written everywhere in the
timeline, so `replacecommit` may be the more consistent choice despite looking
run-together.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
