zhedoubushishi commented on a change in pull request #2833:
URL: https://github.com/apache/hudi/pull/2833#discussion_r626258569
##########
File path:
hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
##########
@@ -33,53 +33,82 @@
public static final String METADATA_PREFIX = "hoodie.metadata";
// Enable the internal Metadata Table which saves file listings
- public static final String METADATA_ENABLE_PROP = METADATA_PREFIX +
".enable";
- public static final boolean DEFAULT_METADATA_ENABLE = false;
+ public static final ConfigOption<Boolean> METADATA_ENABLE_PROP = ConfigOption
+ .key(METADATA_PREFIX + ".enable")
+ .defaultValue(false)
+ .withDescription("Enable the internal Metadata Table which stores table
level metadata such as file listings");
// Validate contents of Metadata Table on each access against the actual
filesystem
- public static final String METADATA_VALIDATE_PROP = METADATA_PREFIX +
".validate";
- public static final boolean DEFAULT_METADATA_VALIDATE = false;
+ public static final ConfigOption<Boolean> METADATA_VALIDATE_PROP =
ConfigOption
+ .key(METADATA_PREFIX + ".validate")
+ .defaultValue(false)
+ .withDescription("Validate contents of Metadata Table on each access
against the actual listings from DFS");
+
public static final boolean DEFAULT_METADATA_ENABLE_FOR_READERS = false;
// Enable metrics for internal Metadata Table
- public static final String METADATA_METRICS_ENABLE_PROP = METADATA_PREFIX +
".metrics.enable";
- public static final boolean DEFAULT_METADATA_METRICS_ENABLE = false;
+ public static final ConfigOption<Boolean> METADATA_METRICS_ENABLE_PROP =
ConfigOption
+ .key(METADATA_PREFIX + ".metrics.enable")
+ .defaultValue(false)
+ .withDescription("");
// Parallelism for inserts
- public static final String METADATA_INSERT_PARALLELISM_PROP =
METADATA_PREFIX + ".insert.parallelism";
- public static final int DEFAULT_METADATA_INSERT_PARALLELISM = 1;
+ public static final ConfigOption<Integer> METADATA_INSERT_PARALLELISM_PROP =
ConfigOption
+ .key(METADATA_PREFIX + ".insert.parallelism")
+ .defaultValue(1)
+ .withDescription("Parallelism to use when writing to the metadata
table");
// Async clean
- public static final String METADATA_ASYNC_CLEAN_PROP = METADATA_PREFIX +
".clean.async";
- public static final boolean DEFAULT_METADATA_ASYNC_CLEAN = false;
+ public static final ConfigOption<Boolean> METADATA_ASYNC_CLEAN_PROP =
ConfigOption
+ .key(METADATA_PREFIX + ".clean.async")
+ .defaultValue(false)
+ .withDescription("");
// Maximum delta commits before compaction occurs
- public static final String METADATA_COMPACT_NUM_DELTA_COMMITS_PROP =
METADATA_PREFIX + ".compact.max.delta.commits";
- public static final int DEFAULT_METADATA_COMPACT_NUM_DELTA_COMMITS = 24;
+ public static final ConfigOption<Integer>
METADATA_COMPACT_NUM_DELTA_COMMITS_PROP = ConfigOption
+ .key(METADATA_PREFIX + ".compact.max.delta.commits")
+ .defaultValue(24)
+ .withDescription("Controls how often the metadata table is compacted.");
// Archival settings
- public static final String MIN_COMMITS_TO_KEEP_PROP = METADATA_PREFIX +
".keep.min.commits";
- public static final int DEFAULT_MIN_COMMITS_TO_KEEP = 20;
- public static final String MAX_COMMITS_TO_KEEP_PROP = METADATA_PREFIX +
".keep.max.commits";
- public static final int DEFAULT_MAX_COMMITS_TO_KEEP = 30;
+ public static final ConfigOption<Integer> MIN_COMMITS_TO_KEEP_PROP =
ConfigOption
+ .key(METADATA_PREFIX + ".keep.min.commits")
+ .defaultValue(20)
+ .withDescription("Controls the archival of the metadata table’s
timeline");
+
+ public static final ConfigOption<Integer> MAX_COMMITS_TO_KEEP_PROP =
ConfigOption
+ .key(METADATA_PREFIX + ".keep.max.commits")
+ .defaultValue(30)
+ .withDescription("Controls the archival of the metadata table’s
timeline");
// Cleaner commits retained
- public static final String CLEANER_COMMITS_RETAINED_PROP = METADATA_PREFIX +
".cleaner.commits.retained";
- public static final int DEFAULT_CLEANER_COMMITS_RETAINED = 3;
+ public static final ConfigOption<Integer> CLEANER_COMMITS_RETAINED_PROP =
ConfigOption
+ .key(METADATA_PREFIX + ".cleaner.commits.retained")
+ .defaultValue(3)
+ .withDescription("");
// Controls whether or not, upon failure to fetch from metadata table,
should fallback to listing.
- public static final String ENABLE_FALLBACK_PROP = METADATA_PREFIX +
".fallback.enable";
- public static final String DEFAULT_ENABLE_FALLBACK = "true";
+ public static final ConfigOption<String> ENABLE_FALLBACK_PROP = ConfigOption
+ .key(METADATA_PREFIX + ".fallback.enable")
+ .defaultValue("true")
+ .withDescription("Fallback to listing from DFS, if there are any errors
in fetching from metadata table");
// Regex to filter out matching directories during bootstrap
- public static final String DIRECTORY_FILTER_REGEX = METADATA_PREFIX +
".dir.filter.regex";
- public static final String DEFAULT_DIRECTORY_FILTER_REGEX = "";
-
- public static final String HOODIE_ASSUME_DATE_PARTITIONING_PROP =
"hoodie.assume.date.partitioning";
- public static final String DEFAULT_ASSUME_DATE_PARTITIONING = "false";
-
- public static final String FILE_LISTING_PARALLELISM_PROP =
"hoodie.file.listing.parallelism";
- public static final int DEFAULT_FILE_LISTING_PARALLELISM = 1500;
+ public static final ConfigOption<String> DIRECTORY_FILTER_REGEX =
ConfigOption
+ .key(METADATA_PREFIX + ".dir.filter.regex")
+ .defaultValue("")
+ .withDescription("");
+
+ public static final ConfigOption<String>
HOODIE_ASSUME_DATE_PARTITIONING_PROP = ConfigOption
+ .key("hoodie.assume.date.partitioning")
+ .defaultValue("false")
+ .withDescription("Should HoodieWriteClient assume the data is
partitioned by dates, i.e three levels from base path. "
+ + "This is a stop-gap to support tables created by versions < 0.3.1.
Will be removed eventually");
Review comment:
I copied it from
https://hudi.apache.org/docs/configurations.html#withAssumeDatePartitioning. I
took a quick look at the code, when it enabled, it just uses
```getAllPartitionFoldersThreeLevelsDown``` to list partitions. I don't know
what does the last sentense mean here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
