FANNG1 commented on code in PR #10189:
URL: https://github.com/apache/gravitino/pull/10189#discussion_r2888239041
##########
api/src/main/java/org/apache/gravitino/policy/PolicyContents.java:
##########
@@ -128,4 +176,232 @@ public String toString() {
+ '}';
}
}
+
+ /** Built-in policy content for Iceberg compaction strategy. */
+ public static class IcebergCompactionContent implements PolicyContent {
+ /** Property key for strategy type. */
+ public static final String STRATEGY_TYPE_KEY = "strategy.type";
+ /** Strategy type value for compaction. */
+ public static final String STRATEGY_TYPE_VALUE = "compaction";
+ /** Property key for job template name. */
+ public static final String JOB_TEMPLATE_NAME_KEY = "job.template-name";
+ /** Built-in job template name for Iceberg rewrite data files. */
+ public static final String JOB_TEMPLATE_NAME_VALUE =
"builtin-iceberg-rewrite-data-files";
+ /** Prefix for rewrite options propagated to job options. */
+ public static final String JOB_OPTIONS_PREFIX = "job.options.";
+ /** Rule key for trigger expression. */
+ public static final String TRIGGER_EXPR_KEY = "trigger-expr";
+ /** Rule key for score expression. */
+ public static final String SCORE_EXPR_KEY = "score-expr";
+ /** Rule key for minimum data file MSE threshold. */
+ public static final String MIN_DATAFILE_MSE_KEY = "minDatafileMse";
+ /** Rule key for minimum delete file count threshold. */
+ public static final String MIN_DELETE_FILE_NUMBER_KEY =
"minDeleteFileNumber";
+ /** Rule key for data file MSE score weight. */
+ public static final String DATAFILE_MSE_WEIGHT_KEY = "datafileMseWeight";
+ /** Rule key for delete file number score weight. */
+ public static final String DELETE_FILE_NUMBER_WEIGHT_KEY =
"deleteFileNumberWeight";
+ /** Metric name for data file MSE. */
+ public static final String DATAFILE_MSE_METRIC = "custom-datafile_mse";
+ /** Metric name for delete file number. */
+ public static final String DELETE_FILE_NUMBER_METRIC =
"custom-delete_file_number";
+ /** Default score weight for data file MSE. */
+ public static final long DEFAULT_DATAFILE_MSE_WEIGHT = 1L;
+ /** Default score weight for delete file number. */
+ public static final long DEFAULT_DELETE_FILE_NUMBER_WEIGHT = 100L;
+
+ private static final Pattern OPTION_KEY_PATTERN =
Pattern.compile("[A-Za-z0-9._-]+");
+ private static final Set<MetadataObject.Type> SUPPORTED_OBJECT_TYPES =
+ ImmutableSet.of(MetadataObject.Type.TABLE);
+ private static final String TRIGGER_EXPR =
+ DATAFILE_MSE_METRIC
+ + " > "
+ + MIN_DATAFILE_MSE_KEY
+ + " || "
+ + DELETE_FILE_NUMBER_METRIC
+ + " > "
+ + MIN_DELETE_FILE_NUMBER_KEY;
+ private static final String SCORE_EXPR =
+ DATAFILE_MSE_METRIC
+ + " * "
+ + DATAFILE_MSE_WEIGHT_KEY
+ + " / 100 + "
+ + DELETE_FILE_NUMBER_METRIC
+ + " * "
+ + DELETE_FILE_NUMBER_WEIGHT_KEY;
+
+ private final Long minDatafileMse;
+ private final Long minDeleteFileNumber;
+ private final Long datafileMseWeight;
+ private final Long deleteFileNumberWeight;
+ private final Map<String, String> rewriteOptions;
+
+ /** Default constructor for Jackson deserialization only. */
+ private IcebergCompactionContent() {
+ this(null, null, null, null, null);
+ }
+
+ private IcebergCompactionContent(
+ Long minDatafileMse,
+ Long minDeleteFileNumber,
+ Long datafileMseWeight,
+ Long deleteFileNumberWeight,
+ Map<String, String> rewriteOptions) {
+ this.minDatafileMse = minDatafileMse;
+ this.minDeleteFileNumber = minDeleteFileNumber;
+ this.datafileMseWeight =
+ datafileMseWeight == null ? DEFAULT_DATAFILE_MSE_WEIGHT :
datafileMseWeight;
+ this.deleteFileNumberWeight =
+ deleteFileNumberWeight == null
+ ? DEFAULT_DELETE_FILE_NUMBER_WEIGHT
+ : deleteFileNumberWeight;
+ this.rewriteOptions =
+ rewriteOptions == null
+ ? Map.of()
+ : Collections.unmodifiableMap(new
LinkedHashMap<>(rewriteOptions));
+ }
Review Comment:
updated
##########
common/src/main/java/org/apache/gravitino/dto/policy/PolicyContentDTO.java:
##########
@@ -69,4 +76,148 @@ public Map<String, String> properties() {
return properties;
}
}
+
+ /** Represents a typed iceberg compaction policy content DTO. */
+ @EqualsAndHashCode
+ @ToString
+ @Builder(setterPrefix = "with")
+ @AllArgsConstructor(access = lombok.AccessLevel.PRIVATE)
+ class IcebergCompactionContentDTO implements PolicyContentDTO {
+
+ @JsonProperty("minDatafileMse")
+ private Long minDatafileMse;
+
+ @JsonProperty("minDeleteFileNumber")
+ private Long minDeleteFileNumber;
+
+ @JsonProperty("datafileMseWeight")
+ private Long datafileMseWeight;
+
+ @JsonProperty("deleteFileNumberWeight")
+ private Long deleteFileNumberWeight;
+
+ @JsonProperty("rewriteOptions")
+ private Map<String, String> rewriteOptions;
+
+ private static final Pattern OPTION_KEY_PATTERN =
Pattern.compile("[A-Za-z0-9._-]+");
+
+ // Default constructor for Jackson deserialization only.
+ private IcebergCompactionContentDTO() {}
+
+ /**
+ * Returns the minimum threshold for custom-datafile_mse metric.
+ *
+ * @return minimum data file MSE threshold
+ */
+ public Long minDatafileMse() {
+ return minDatafileMse;
+ }
+
+ /**
+ * Returns the minimum threshold for custom-delete_file_number metric.
+ *
+ * @return minimum delete file number threshold
+ */
+ public Long minDeleteFileNumber() {
+ return minDeleteFileNumber;
+ }
+
+ /**
+ * Returns the weight for custom-datafile_mse metric in score expression.
+ *
+ * @return data file MSE score weight
+ */
+ public Long datafileMseWeight() {
+ return datafileMseWeight == null
+ ? PolicyContents.IcebergCompactionContent.DEFAULT_DATAFILE_MSE_WEIGHT
+ : datafileMseWeight;
+ }
+
+ /**
+ * Returns the weight for custom-delete_file_number metric in score
expression.
+ *
+ * @return delete file number score weight
+ */
+ public Long deleteFileNumberWeight() {
+ return deleteFileNumberWeight == null
+ ?
PolicyContents.IcebergCompactionContent.DEFAULT_DELETE_FILE_NUMBER_WEIGHT
+ : deleteFileNumberWeight;
+ }
+
+ /**
+ * Returns rewrite options expanded to job.options.* during rule
generation.
+ *
+ * @return rewrite options map
+ */
+ public Map<String, String> rewriteOptions() {
+ return rewriteOptions == null
+ ? Map.of()
+ : Collections.unmodifiableMap(new LinkedHashMap<>(rewriteOptions));
+ }
Review Comment:
updated
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
