codope commented on code in PR #8758:
URL: https://github.com/apache/hudi/pull/8758#discussion_r1232142078
##########
hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java:
##########
@@ -694,17 +695,80 @@ private Long getTableChecksum() {
return getLong(TABLE_CHECKSUM);
}
- public List<String> getMetadataPartitionsInflight() {
- return StringUtils.split(
- getStringOrDefault(TABLE_METADATA_PARTITIONS_INFLIGHT,
StringUtils.EMPTY_STRING),
- CONFIG_VALUES_DELIMITER
- );
+ public Set<String> getMetadataPartitionsInflight() {
+ return new HashSet<>(StringUtils.split(
+ getStringOrDefault(TABLE_METADATA_PARTITIONS_INFLIGHT,
StringUtils.EMPTY_STRING),
+ CONFIG_VALUES_DELIMITER));
}
public Set<String> getMetadataPartitions() {
return new HashSet<>(
- StringUtils.split(getStringOrDefault(TABLE_METADATA_PARTITIONS,
StringUtils.EMPTY_STRING),
- CONFIG_VALUES_DELIMITER));
+ StringUtils.split(getStringOrDefault(TABLE_METADATA_PARTITIONS,
StringUtils.EMPTY_STRING),
+ CONFIG_VALUES_DELIMITER));
+ }
+
+ /**
+ * @returns true if metadata table has been created and is being used for
this dataset, else returns false.
+ */
+ public boolean isMetadataTableEnabled() {
+ return isMetadataPartitionEnabled(MetadataPartitionType.FILES);
+ }
+
+ /**
+ * Checks if metadata table is enabled and the specified partition has been
initialized.
+ *
+ * @param partition The partition to check
+ * @returns true if the specific partition has been initialized, else
returns false.
+ */
+ public boolean isMetadataPartitionEnabled(MetadataPartitionType partition) {
+ return getMetadataPartitions().contains(partition.getPartitionPath());
+ }
+
+ /**
+ * Enables or disables the specified metadata table partition.
+ *
+ * @param partitionType The partition
+ * @param enabled If true, the partition is enabled, else disabled
+ */
+ public void setMetadataPartitionState(HoodieTableMetaClient metaClient,
MetadataPartitionType partitionType, boolean enabled) {
+
ValidationUtils.checkArgument(!partitionType.getPartitionPath().contains(CONFIG_VALUES_DELIMITER),
+ "Metadata Table partition path cannot contain a comma: " +
partitionType.getPartitionPath());
+ Set<String> partitions = getMetadataPartitions();
+ Set<String> partitionsInflight = getMetadataPartitionsInflight();
+ if (enabled) {
+ partitions.add(partitionType.getPartitionPath());
+ partitionsInflight.remove(partitionType.getPartitionPath());
+ } else if (partitionType.equals(MetadataPartitionType.FILES)) {
+ // file listing partition is required for all other partitions to work
+ // Disabling file partition will also disable all partitions
+ partitions.clear();
+ partitionsInflight.clear();
+ } else {
+ partitions.remove(partitionType.getPartitionPath());
+ partitionsInflight.remove(partitionType.getPartitionPath());
+ }
+ setValue(TABLE_METADATA_PARTITIONS,
partitions.stream().sorted().collect(Collectors.joining(CONFIG_VALUES_DELIMITER)));
+ setValue(TABLE_METADATA_PARTITIONS_INFLIGHT,
partitionsInflight.stream().sorted().collect(Collectors.joining(CONFIG_VALUES_DELIMITER)));
+ update(metaClient.getFs(), new Path(metaClient.getMetaPath()), getProps());
+ LOG.info(String.format("MDT %s partition %s has been %s",
metaClient.getBasePathV2(), partitionType, enabled ? "enabled" : "disabled"));
+ }
+
+ /**
+ * Enables the specified metadata table partition as inflight.
+ *
+ * @param partitionTypes The list of partitions to enable as inflight.
+ */
+ public void setMetadataPartitionsInflight(HoodieTableMetaClient metaClient,
List<MetadataPartitionType> partitionTypes) {
Review Comment:
https://issues.apache.org/jira/browse/HUDI-6332
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
