This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new ba424a793f [#14422] fix(policy): Reject invalid orphan locations and
coerced maintenance types (#14483)
ba424a793f is described below
commit ba424a793f1fb59cf5540fdc072b5a338b6cc9d5
Author: MaSai <[email protected]>
AuthorDate: Tue Nov 3 16:16:17 2026 +0800
[#14422] fix(policy): Reject invalid orphan locations and coerced
maintenance types (#14483)
### What changes were proposed in this pull request?
Tighten Iceberg maintenance policy write-time validation:
1. Share orphan `location` syntax checks via
`IcebergOrphanFileRemovalContent.parseAndValidateLocation` (includes
`URI.create` and absolute-path checks).
`OrphanFileLocationUtils.normalizeLocation` reuses the same helper so
create/update and job submission reject the same illegal values.
2. Introduce shared strict deserializers on `PolicyContentDTO`
(`StrictLong` / `StrictBoolean` / `StrictInteger` / `StrictString` /
`StrictStringMap`) and apply them across all four Iceberg maintenance
DTOs: compaction numerics and string/map fields, orphan `olderThanDays`
/ `dryRun` / `location`, rewrite-manifest thresholds / `spec_id` /
`use_caching`, snapshot fields, and peer `minIntervalMs` /
`commitTrigger` / `timeTrigger` / `jobOptions` / `updaterOptions`.
### Why are the changes needed?
Invalid orphan locations were stored and only failed when a cleanup job
ran. Wrong JSON types were silently truncated or coerced for most
maintenance fields (fractions truncated, numeric/boolean tokens coerced
into strings and map values), while snapshot `olderThanDays` was already
strict.
Fixes #14422
Fixes #14418
### Does this PR introduce _any_ user-facing change?
Yes. Policy create/update now returns 400 for:
1. Orphan `location` values that are not valid absolute URIs (for
example embedded spaces).
2. Fractional numbers, numeric strings, non-boolean tokens, non-string
tokens, and non-string map entry values on the maintenance fields listed
above.
### How was this patch tested?
```
./gradlew :common:test -PskipITs \
--tests org.apache.gravitino.dto.policy.TestCompactionPolicyStrictTypes \
--tests org.apache.gravitino.dto.policy.TestOrphanFileRemovalPolicyDTO \
--tests org.apache.gravitino.dto.policy.TestSnapshotExpirationPolicyDTO \
--tests org.apache.gravitino.dto.policy.TestManifestRewritePolicyDTO \
--tests org.apache.gravitino.dto.policy.TestPolicyDTO \
--tests
org.apache.gravitino.dto.policy.TestRewriteManifestsAndSnapshotExpirationPolicyDTO
./gradlew :api:test -PskipITs --tests
org.apache.gravitino.policy.TestIcebergOrphanFileRemovalContent
./gradlew :maintenance:optimizer-api:test -PskipITs
```
---------
Co-authored-by: Cursor <[email protected]>
---
.../policy/IcebergOrphanFileRemovalContent.java | 56 ++++--
.../TestIcebergOrphanFileRemovalContent.java | 4 +-
.../gravitino/dto/policy/PolicyContentDTO.java | 206 ++++++++++++++-------
.../policy/TestCompactionPolicyStrictTypes.java | 90 +++++++++
.../dto/policy/TestManifestRewritePolicyDTO.java | 10 +-
.../dto/policy/TestOrphanFileRemovalPolicyDTO.java | 83 +++++++++
.../policy/TestSnapshotExpirationPolicyDTO.java | 10 +-
.../common/util/OrphanFileLocationUtils.java | 18 +-
8 files changed, 374 insertions(+), 103 deletions(-)
diff --git
a/api/src/main/java/org/apache/gravitino/policy/IcebergOrphanFileRemovalContent.java
b/api/src/main/java/org/apache/gravitino/policy/IcebergOrphanFileRemovalContent.java
index ba184fd115..6cb1b87b7b 100644
---
a/api/src/main/java/org/apache/gravitino/policy/IcebergOrphanFileRemovalContent.java
+++
b/api/src/main/java/org/apache/gravitino/policy/IcebergOrphanFileRemovalContent.java
@@ -21,6 +21,7 @@ package org.apache.gravitino.policy;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import java.net.URI;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -143,6 +144,44 @@ public class IcebergOrphanFileRemovalContent extends
AbstractExecutablePolicyCon
MAX_OLDER_THAN_DAYS);
}
+ /**
+ * Validates orphan-scan location syntax shared by policy write-time checks
and job submission.
+ *
+ * <p>Mirrors {@code OrphanFileLocationUtils.normalizeLocation} lexical
rules so values that would
+ * fail {@link URI#create(String)} or absolute-path checks are rejected when
the policy is stored,
+ * not only when a cleanup job later runs.
+ *
+ * @param location non-null scan location
+ * @return parsed URI after syntax checks
+ * @throws IllegalArgumentException if the location is blank, ambiguous, or
not a usable URI
+ */
+ public static URI parseAndValidateLocation(String location) {
+ Preconditions.checkArgument(StringUtils.isNotBlank(location), "location
must not be blank");
+ Preconditions.checkArgument(
+ location.equals(StringUtils.strip(location.trim())),
+ "location must not have leading or trailing whitespace");
+ Preconditions.checkArgument(
+ !location.contains(".."), "location must not contain '..' path
segments: %s", location);
+ Preconditions.checkArgument(
+ !location.contains("\\"), "location must not contain '\\': %s",
location);
+ Preconditions.checkArgument(
+ !location.contains("%"), "location must not contain percent-encoding:
%s", location);
+ final URI uri;
+ try {
+ uri = URI.create(location);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("location must be a valid URI: " +
location, e);
+ }
+ Preconditions.checkArgument(
+ uri.getQuery() == null
+ && uri.getFragment() == null
+ && uri.getPath() != null
+ && uri.getPath().startsWith("/"),
+ "location must be an absolute path without query or fragment: %s",
+ location);
+ return uri;
+ }
+
@Override
public void validate() {
super.validate();
@@ -150,23 +189,8 @@ public class IcebergOrphanFileRemovalContent extends
AbstractExecutablePolicyCon
!Boolean.TRUE.equals(executionSpec().commitTrigger()),
"commitTrigger is not allowed for orphan file removal");
validateOlderThanDays(olderThanDays);
- Preconditions.checkArgument(
- location == null || StringUtils.isNotBlank(location), "location must
not be blank");
- Preconditions.checkArgument(
- location == null ||
location.equals(StringUtils.strip(location.trim())),
- "location must not have leading or trailing whitespace");
if (location != null) {
- // Mirror OrphanFileLocationUtils.normalizeLocation safety without
needing a table root.
- Preconditions.checkArgument(
- !location.contains(".."), "location must not contain '..' path
segments: %s", location);
- Preconditions.checkArgument(
- !location.contains("\\"), "location must not contain '\\': %s",
location);
- Preconditions.checkArgument(
- !location.contains("%"), "location must not contain
percent-encoding: %s", location);
- Preconditions.checkArgument(
- !location.contains("?") && !location.contains("#"),
- "location must not contain query or fragment: %s",
- location);
+ parseAndValidateLocation(location);
}
}
diff --git
a/api/src/test/java/org/apache/gravitino/policy/TestIcebergOrphanFileRemovalContent.java
b/api/src/test/java/org/apache/gravitino/policy/TestIcebergOrphanFileRemovalContent.java
index 87cea9eeda..b2318c16ed 100644
---
a/api/src/test/java/org/apache/gravitino/policy/TestIcebergOrphanFileRemovalContent.java
+++
b/api/src/test/java/org/apache/gravitino/policy/TestIcebergOrphanFileRemovalContent.java
@@ -103,7 +103,9 @@ class TestIcebergOrphanFileRemovalContent {
"s3://bucket/table/%2e%2e/other",
"s3://bucket/table\\data",
"s3://bucket/table/data?query",
- "s3://bucket/table/data#frag"
+ "s3://bucket/table/data#frag",
+ "s3://bucket/tbl/da ta",
+ "s3://bucket/tbl/da\tta"
}) {
IllegalArgumentException error =
Assertions.assertThrows(
diff --git
a/common/src/main/java/org/apache/gravitino/dto/policy/PolicyContentDTO.java
b/common/src/main/java/org/apache/gravitino/dto/policy/PolicyContentDTO.java
index 24562a9cb0..fc365b8807 100644
--- a/common/src/main/java/org/apache/gravitino/dto/policy/PolicyContentDTO.java
+++ b/common/src/main/java/org/apache/gravitino/dto/policy/PolicyContentDTO.java
@@ -118,6 +118,103 @@ public interface PolicyContentDTO extends PolicyContent {
}
}
+ /**
+ * Reads a long without silently coercing fractional values or strings.
+ *
+ * <p>Shared by Iceberg maintenance policy numeric fields so compaction,
orphan removal, snapshot
+ * expiration, rewrite-manifests thresholds, and peer {@code minIntervalMs}
reject the same wrong
+ * JSON tokens.
+ */
+ class StrictLongDeserializer extends JsonDeserializer<Long> {
+ @Override
+ public Long deserialize(JsonParser parser, DeserializationContext context)
throws IOException {
+ if (!parser.hasToken(JsonToken.VALUE_NUMBER_INT)) {
+ return context.reportInputMismatch(Long.class, "Value must be a 64-bit
integer");
+ }
+ return parser.getLongValue();
+ }
+ }
+
+ /**
+ * Reads a boolean without silently coercing numbers or strings.
+ *
+ * <p>Shared by Iceberg maintenance policy boolean fields ({@code dryRun},
{@code streamResults},
+ * {@code use_caching}, {@code commitTrigger}).
+ */
+ class StrictBooleanDeserializer extends JsonDeserializer<Boolean> {
+ @Override
+ public Boolean deserialize(JsonParser parser, DeserializationContext
context)
+ throws IOException {
+ if (!parser.hasToken(JsonToken.VALUE_TRUE) &&
!parser.hasToken(JsonToken.VALUE_FALSE)) {
+ return context.reportInputMismatch(Boolean.class, "Value must be a
boolean");
+ }
+ return parser.getBooleanValue();
+ }
+ }
+
+ /**
+ * Reads an int without silently coercing fractional values or strings.
+ *
+ * <p>Shared by Iceberg maintenance policy 32-bit integer fields (for
example rewrite-manifests
+ * {@code spec_id}).
+ */
+ class StrictIntegerDeserializer extends JsonDeserializer<Integer> {
+ @Override
+ public Integer deserialize(JsonParser parser, DeserializationContext
context)
+ throws IOException {
+ if (!parser.hasToken(JsonToken.VALUE_NUMBER_INT)) {
+ return context.reportInputMismatch(Integer.class, "Value must be a
32-bit integer");
+ }
+ return parser.getIntValue();
+ }
+ }
+
+ /**
+ * Reads a string without silently coercing numbers or booleans.
+ *
+ * <p>Shared by Iceberg maintenance policy string fields ({@code
rewriteStrategy}, {@code
+ * sortOrder}, orphan {@code location}, peer {@code timeTrigger}).
+ */
+ class StrictStringDeserializer extends JsonDeserializer<String> {
+ @Override
+ public String deserialize(JsonParser parser, DeserializationContext
context)
+ throws IOException {
+ if (!parser.hasToken(JsonToken.VALUE_STRING)) {
+ return context.reportInputMismatch(String.class, "Value must be a
string");
+ }
+ return parser.getText();
+ }
+ }
+
+ /**
+ * Reads a string-to-string map without silently coercing numeric or boolean
entry values.
+ *
+ * <p>Shared by Iceberg maintenance {@code rewriteOptions}, {@code
jobOptions}, and {@code
+ * updaterOptions}.
+ */
+ class StrictStringMapDeserializer extends JsonDeserializer<Map<String,
String>> {
+ @Override
+ public Map<String, String> deserialize(JsonParser parser,
DeserializationContext context)
+ throws IOException {
+ if (!parser.hasToken(JsonToken.START_OBJECT)) {
+ return context.reportInputMismatch(Map.class, "Value must be a JSON
object");
+ }
+ Map<String, String> result = new LinkedHashMap<>();
+ while (parser.nextToken() != JsonToken.END_OBJECT) {
+ String key = parser.currentName();
+ JsonToken valueToken = parser.nextToken();
+ if (valueToken == JsonToken.VALUE_NULL) {
+ result.put(key, null);
+ } else if (valueToken != JsonToken.VALUE_STRING) {
+ return context.reportInputMismatch(Map.class, "Map entry '%s' must
be a string", key);
+ } else {
+ result.put(key, parser.getText());
+ }
+ }
+ return result;
+ }
+ }
+
/** Represents a custom policy content DTO. */
@EqualsAndHashCode
@ToString
@@ -170,54 +267,67 @@ public interface PolicyContentDTO extends PolicyContent {
class IcebergCompactionContentDTO implements PolicyContentDTO,
ExecutionPeersDTO {
@JsonProperty("minDataFileMse")
+ @JsonDeserialize(using = StrictLongDeserializer.class)
private Long minDataFileMse;
@JsonProperty("minDeleteFileNumber")
+ @JsonDeserialize(using = StrictLongDeserializer.class)
private Long minDeleteFileNumber;
@JsonProperty("dataFileMseWeight")
+ @JsonDeserialize(using = StrictLongDeserializer.class)
private Long dataFileMseWeight;
@JsonProperty("deleteFileNumberWeight")
+ @JsonDeserialize(using = StrictLongDeserializer.class)
private Long deleteFileNumberWeight;
@JsonProperty("maxPartitionNum")
+ @JsonDeserialize(using = StrictLongDeserializer.class)
private Long maxPartitionNum;
@JsonProperty("rewriteStrategy")
+ @JsonDeserialize(using = StrictStringDeserializer.class)
private String rewriteStrategy;
@JsonProperty("sortOrder")
+ @JsonDeserialize(using = StrictStringDeserializer.class)
private String sortOrder;
@JsonProperty("rewriteOptions")
+ @JsonDeserialize(using = StrictStringMapDeserializer.class)
private Map<String, String> rewriteOptions;
@EqualsAndHashCode.Exclude
@Nullable
@JsonProperty("timeTrigger")
+ @JsonDeserialize(using = StrictStringDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_NULL)
private String timeTrigger;
@EqualsAndHashCode.Exclude
@Nullable
@JsonProperty("commitTrigger")
+ @JsonDeserialize(using = StrictBooleanDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_NULL)
private Boolean commitTrigger;
@EqualsAndHashCode.Exclude
@Nullable
@JsonProperty("minIntervalMs")
+ @JsonDeserialize(using = StrictLongDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_NULL)
private Long minIntervalMs;
@EqualsAndHashCode.Exclude
@JsonProperty("jobOptions")
+ @JsonDeserialize(using = StrictStringMapDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_EMPTY)
private Map<String, String> jobOptions;
@EqualsAndHashCode.Exclude
@JsonProperty("updaterOptions")
+ @JsonDeserialize(using = StrictStringMapDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_EMPTY)
private Map<String, String> updaterOptions;
@@ -420,39 +530,47 @@ public interface PolicyContentDTO extends PolicyContent {
@AllArgsConstructor(access = lombok.AccessLevel.PRIVATE)
class IcebergOrphanFileRemovalContentDTO implements PolicyContentDTO,
ExecutionPeersDTO {
@JsonProperty("olderThanDays")
+ @JsonDeserialize(using = StrictLongDeserializer.class)
private Long olderThanDays;
@JsonProperty("location")
+ @JsonDeserialize(using = StrictStringDeserializer.class)
private String location;
@JsonProperty("dryRun")
+ @JsonDeserialize(using = StrictBooleanDeserializer.class)
private Boolean dryRun;
@EqualsAndHashCode.Exclude
@Nullable
@JsonProperty("timeTrigger")
+ @JsonDeserialize(using = StrictStringDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_NULL)
private String timeTrigger;
@EqualsAndHashCode.Exclude
@Nullable
@JsonProperty("commitTrigger")
+ @JsonDeserialize(using = StrictBooleanDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_NULL)
private Boolean commitTrigger;
@EqualsAndHashCode.Exclude
@Nullable
@JsonProperty("minIntervalMs")
+ @JsonDeserialize(using = StrictLongDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_NULL)
private Long minIntervalMs;
@EqualsAndHashCode.Exclude
@JsonProperty("jobOptions")
+ @JsonDeserialize(using = StrictStringMapDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_EMPTY)
private Map<String, String> jobOptions;
@EqualsAndHashCode.Exclude
@JsonProperty("updaterOptions")
+ @JsonDeserialize(using = StrictStringMapDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_EMPTY)
private Map<String, String> updaterOptions;
@@ -590,50 +708,55 @@ public interface PolicyContentDTO extends PolicyContent {
@AllArgsConstructor(access = AccessLevel.PRIVATE)
class IcebergRewriteManifestsContentDTO implements PolicyContentDTO,
ExecutionPeersDTO {
@JsonProperty("manifest_count_critical")
- @JsonDeserialize(using = ManifestThresholdDeserializer.class)
+ @JsonDeserialize(using = StrictLongDeserializer.class)
private Long manifestCountCritical;
@JsonProperty("manifest_count_warning")
- @JsonDeserialize(using = ManifestThresholdDeserializer.class)
+ @JsonDeserialize(using = StrictLongDeserializer.class)
private Long manifestCountWarning;
@JsonProperty("avg_manifest_size_threshold_bytes")
- @JsonDeserialize(using = ManifestThresholdDeserializer.class)
+ @JsonDeserialize(using = StrictLongDeserializer.class)
private Long avgManifestSizeThresholdBytes;
@JsonProperty("spec_id")
- @JsonDeserialize(using = ManifestSpecIdDeserializer.class)
+ @JsonDeserialize(using = StrictIntegerDeserializer.class)
private Integer specId;
@JsonProperty("use_caching")
- @JsonDeserialize(using = ManifestUseCachingDeserializer.class)
+ @JsonDeserialize(using = StrictBooleanDeserializer.class)
private Boolean useCaching;
@EqualsAndHashCode.Exclude
@Nullable
@JsonProperty("timeTrigger")
+ @JsonDeserialize(using = StrictStringDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_NULL)
private String timeTrigger;
@EqualsAndHashCode.Exclude
@Nullable
@JsonProperty("commitTrigger")
+ @JsonDeserialize(using = StrictBooleanDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_NULL)
private Boolean commitTrigger;
@EqualsAndHashCode.Exclude
@Nullable
@JsonProperty("minIntervalMs")
+ @JsonDeserialize(using = StrictLongDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_NULL)
private Long minIntervalMs;
@EqualsAndHashCode.Exclude
@JsonProperty("jobOptions")
+ @JsonDeserialize(using = StrictStringMapDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_EMPTY)
private Map<String, String> jobOptions;
@EqualsAndHashCode.Exclude
@JsonProperty("updaterOptions")
+ @JsonDeserialize(using = StrictStringMapDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_EMPTY)
private Map<String, String> updaterOptions;
@@ -756,43 +879,6 @@ public interface PolicyContentDTO extends PolicyContent {
toDomainContent().validate();
}
- /** Reads a manifest threshold without silently coercing fractional values
or strings. */
- public static class ManifestThresholdDeserializer extends
JsonDeserializer<Long> {
- @Override
- public Long deserialize(JsonParser parser, DeserializationContext
context)
- throws IOException {
- if (!parser.hasToken(JsonToken.VALUE_NUMBER_INT)) {
- return context.reportInputMismatch(
- Long.class, "Manifest thresholds must be 64-bit integers");
- }
- return parser.getLongValue();
- }
- }
-
- /** Reads a partition spec ID without silently coercing fractional values
or strings. */
- public static class ManifestSpecIdDeserializer extends
JsonDeserializer<Integer> {
- @Override
- public Integer deserialize(JsonParser parser, DeserializationContext
context)
- throws IOException {
- if (!parser.hasToken(JsonToken.VALUE_NUMBER_INT)) {
- return context.reportInputMismatch(Integer.class, "spec_id must be a
32-bit integer");
- }
- return parser.getIntValue();
- }
- }
-
- /** Reads the caching option without silently coercing numbers or strings
to booleans. */
- public static class ManifestUseCachingDeserializer extends
JsonDeserializer<Boolean> {
- @Override
- public Boolean deserialize(JsonParser parser, DeserializationContext
context)
- throws IOException {
- if (!parser.hasToken(JsonToken.VALUE_TRUE) &&
!parser.hasToken(JsonToken.VALUE_FALSE)) {
- return context.reportInputMismatch(Boolean.class, "use_caching must
be a boolean");
- }
- return parser.getBooleanValue();
- }
- }
-
/**
* @return domain policy content including thresholds and execution peers
*/
@@ -850,42 +936,47 @@ public interface PolicyContentDTO extends PolicyContent {
@AllArgsConstructor(access = AccessLevel.PRIVATE)
class IcebergSnapshotExpirationContentDTO implements PolicyContentDTO,
ExecutionPeersDTO {
@JsonProperty("olderThanDays")
- @JsonDeserialize(using = SnapshotLongDeserializer.class)
+ @JsonDeserialize(using = StrictLongDeserializer.class)
private Long olderThanDays;
@JsonProperty("retainLast")
- @JsonDeserialize(using = SnapshotLongDeserializer.class)
+ @JsonDeserialize(using = StrictLongDeserializer.class)
private Long retainLast;
@JsonProperty("streamResults")
- @JsonDeserialize(using = SnapshotBooleanDeserializer.class)
+ @JsonDeserialize(using = StrictBooleanDeserializer.class)
private Boolean streamResults;
@EqualsAndHashCode.Exclude
@Nullable
@JsonProperty("timeTrigger")
+ @JsonDeserialize(using = StrictStringDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_NULL)
private String timeTrigger;
@EqualsAndHashCode.Exclude
@Nullable
@JsonProperty("commitTrigger")
+ @JsonDeserialize(using = StrictBooleanDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_NULL)
private Boolean commitTrigger;
@EqualsAndHashCode.Exclude
@Nullable
@JsonProperty("minIntervalMs")
+ @JsonDeserialize(using = StrictLongDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_NULL)
private Long minIntervalMs;
@EqualsAndHashCode.Exclude
@JsonProperty("jobOptions")
+ @JsonDeserialize(using = StrictStringMapDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_EMPTY)
private Map<String, String> jobOptions;
@EqualsAndHashCode.Exclude
@JsonProperty("updaterOptions")
+ @JsonDeserialize(using = StrictStringMapDeserializer.class)
@JsonInclude(JsonInclude.Include.NON_EMPTY)
private Map<String, String> updaterOptions;
@@ -1012,31 +1103,6 @@ public interface PolicyContentDTO extends PolicyContent {
return this;
}
}
-
- /** Reads a long option without silently coercing fractional values or
strings. */
- public static class SnapshotLongDeserializer extends
JsonDeserializer<Long> {
- @Override
- public Long deserialize(JsonParser parser, DeserializationContext
context)
- throws IOException {
- if (!parser.hasToken(JsonToken.VALUE_NUMBER_INT)) {
- return context.reportInputMismatch(
- Long.class, "olderThanDays and retainLast must be 64-bit
integers");
- }
- return parser.getLongValue();
- }
- }
-
- /** Reads streamResults without silently coercing numbers or strings to
booleans. */
- public static class SnapshotBooleanDeserializer extends
JsonDeserializer<Boolean> {
- @Override
- public Boolean deserialize(JsonParser parser, DeserializationContext
context)
- throws IOException {
- if (!parser.hasToken(JsonToken.VALUE_TRUE) &&
!parser.hasToken(JsonToken.VALUE_FALSE)) {
- return context.reportInputMismatch(Boolean.class, "streamResults
must be a boolean");
- }
- return parser.getBooleanValue();
- }
- }
}
/** Represents typed row-filter policy content. */
diff --git
a/common/src/test/java/org/apache/gravitino/dto/policy/TestCompactionPolicyStrictTypes.java
b/common/src/test/java/org/apache/gravitino/dto/policy/TestCompactionPolicyStrictTypes.java
new file mode 100644
index 0000000000..3fcececfe1
--- /dev/null
+++
b/common/src/test/java/org/apache/gravitino/dto/policy/TestCompactionPolicyStrictTypes.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.dto.policy;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.gravitino.dto.requests.PolicyCreateRequest;
+import org.apache.gravitino.dto.requests.PolicyUpdateRequest;
+import org.apache.gravitino.json.JsonUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+class TestCompactionPolicyStrictTypes {
+ @ParameterizedTest
+ @CsvSource({
+ "maxPartitionNum,7.9",
+ "maxPartitionNum,\"50\"",
+ "maxPartitionNum,true",
+ "minDataFileMse,1.5",
+ "minDeleteFileNumber,\"2\"",
+ "dataFileMseWeight,0.5",
+ "deleteFileNumberWeight,3.0",
+ "minIntervalMs,3600.5",
+ "minIntervalMs,\"100\"",
+ "commitTrigger,\"true\"",
+ "commitTrigger,1",
+ "rewriteStrategy,1",
+ "rewriteStrategy,true",
+ "sortOrder,2",
+ "sortOrder,true",
+ "timeTrigger,1",
+ "timeTrigger,true",
+ "rewriteOptions,1",
+ "jobOptions,1",
+ "updaterOptions,true"
+ })
+ void rejectInvalidValuesInCreateAndUpdate(String field, String value) {
+ assertRejected("{\"" + field + "\":" + value + "}");
+ }
+
+ @ParameterizedTest
+ @ValueSource(
+ strings = {
+ "{\"rewriteOptions\":{\"target-file-size-bytes\":134217728}}",
+ "{\"rewriteOptions\":{\"partial-progress-enabled\":true}}",
+ "{\"jobOptions\":{\"k\":1}}",
+ "{\"updaterOptions\":{\"metalake\":true}}"
+ })
+ void rejectNonStringMapEntryValues(String content) {
+ assertRejected(content);
+ }
+
+ private static void assertRejected(String content) {
+ String create =
+ "{\"name\":\"compact\",\"policyType\":\"system_iceberg_compaction\","
+ + "\"enabled\":true,\"content\":"
+ + content
+ + "}";
+ String update =
+
"{\"@type\":\"updateContent\",\"policyType\":\"system_iceberg_compaction\","
+ + "\"newContent\":"
+ + content
+ + "}";
+ Assertions.assertThrows(
+ JsonProcessingException.class,
+ () -> JsonUtils.objectMapper().readValue(create,
PolicyCreateRequest.class));
+ Assertions.assertThrows(
+ JsonProcessingException.class,
+ () ->
+ JsonUtils.objectMapper()
+ .readValue(update,
PolicyUpdateRequest.UpdatePolicyContentRequest.class));
+ }
+}
diff --git
a/common/src/test/java/org/apache/gravitino/dto/policy/TestManifestRewritePolicyDTO.java
b/common/src/test/java/org/apache/gravitino/dto/policy/TestManifestRewritePolicyDTO.java
index 9518c392ad..ee8d8359c9 100644
---
a/common/src/test/java/org/apache/gravitino/dto/policy/TestManifestRewritePolicyDTO.java
+++
b/common/src/test/java/org/apache/gravitino/dto/policy/TestManifestRewritePolicyDTO.java
@@ -73,7 +73,15 @@ class TestManifestRewritePolicyDTO {
"manifest_count_warning,100.5",
"manifest_count_warning,false",
"avg_manifest_size_threshold_bytes,4096.5",
- "avg_manifest_size_threshold_bytes,\"4096\""
+ "avg_manifest_size_threshold_bytes,\"4096\"",
+ "minIntervalMs,1.5",
+ "minIntervalMs,\"100\"",
+ "commitTrigger,1",
+ "commitTrigger,\"true\"",
+ "timeTrigger,1",
+ "timeTrigger,true",
+ "jobOptions,1",
+ "updaterOptions,true"
})
void testRejectInvalidValuesInCreateAndUpdate(String field, String value) {
String content = "{\"" + field + "\":" + value + "}";
diff --git
a/common/src/test/java/org/apache/gravitino/dto/policy/TestOrphanFileRemovalPolicyDTO.java
b/common/src/test/java/org/apache/gravitino/dto/policy/TestOrphanFileRemovalPolicyDTO.java
index 4286f7f31c..c032da6772 100644
---
a/common/src/test/java/org/apache/gravitino/dto/policy/TestOrphanFileRemovalPolicyDTO.java
+++
b/common/src/test/java/org/apache/gravitino/dto/policy/TestOrphanFileRemovalPolicyDTO.java
@@ -18,6 +18,7 @@
*/
package org.apache.gravitino.dto.policy;
+import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.collect.ImmutableMap;
import java.time.Instant;
import org.apache.gravitino.dto.AuditDTO;
@@ -30,6 +31,9 @@ import org.apache.gravitino.policy.PolicyContent;
import org.apache.gravitino.policy.PolicyContents;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
class TestOrphanFileRemovalPolicyDTO {
@Test
@@ -114,6 +118,85 @@ class TestOrphanFileRemovalPolicyDTO {
Assertions.assertEquals(domain, DTOConverters.fromDTO(restored.content()));
}
+ @ParameterizedTest
+ @CsvSource({
+ "olderThanDays,3.7",
+ "olderThanDays,\"3\"",
+ "olderThanDays,true",
+ "dryRun,1",
+ "dryRun,0",
+ "dryRun,\"true\"",
+ "minIntervalMs,1.9",
+ "minIntervalMs,\"100\"",
+ "commitTrigger,1",
+ "commitTrigger,\"false\"",
+ "location,123",
+ "location,true",
+ "timeTrigger,1",
+ "timeTrigger,true",
+ "jobOptions,1",
+ "updaterOptions,true"
+ })
+ void rejectInvalidValuesInCreateAndUpdate(String field, String value) {
+ assertRejected("{\"" + field + "\":" + value + "}");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"{\"jobOptions\":{\"k\":1}}",
"{\"updaterOptions\":{\"metalake\":true}}"})
+ void rejectNonStringMapEntryValues(String content) {
+ assertRejected(content);
+ }
+
+ private static void assertRejected(String content) {
+ String create =
+
"{\"name\":\"cleanup\",\"policyType\":\"system_iceberg_orphan_file_removal\","
+ + "\"enabled\":true,\"content\":"
+ + content
+ + "}";
+ String update =
+
"{\"@type\":\"updateContent\",\"policyType\":\"system_iceberg_orphan_file_removal\","
+ + "\"newContent\":"
+ + content
+ + "}";
+ Assertions.assertThrows(
+ JsonProcessingException.class,
+ () -> JsonUtils.objectMapper().readValue(create,
PolicyCreateRequest.class));
+ Assertions.assertThrows(
+ JsonProcessingException.class,
+ () ->
+ JsonUtils.objectMapper()
+ .readValue(update,
PolicyUpdateRequest.UpdatePolicyContentRequest.class));
+ }
+
+ @Test
+ void rejectNonUriLocationInCreateAndUpdate() throws Exception {
+ String content = "{\"olderThanDays\":3,\"location\":\"s3://bucket/tbl/da
ta\"}";
+ PolicyCreateRequest create =
+ JsonUtils.objectMapper()
+ .readValue(
+
"{\"name\":\"cleanup\",\"policyType\":\"system_iceberg_orphan_file_removal\","
+ + "\"enabled\":true,\"content\":"
+ + content
+ + "}",
+ PolicyCreateRequest.class);
+ PolicyUpdateRequest update =
+ JsonUtils.objectMapper()
+ .readValue(
+
"{\"@type\":\"updateContent\",\"policyType\":\"system_iceberg_orphan_file_removal\","
+ + "\"newContent\":"
+ + content
+ + "}",
+ PolicyUpdateRequest.class);
+ Assertions.assertTrue(
+ Assertions.assertThrows(IllegalArgumentException.class,
create::validate)
+ .getMessage()
+ .contains("location"));
+ Assertions.assertTrue(
+ Assertions.assertThrows(IllegalArgumentException.class,
update::validate)
+ .getMessage()
+ .contains("location"));
+ }
+
@Test
void createAndUpdateValidateRetentionUpperBoundary() throws Exception {
long maximum = IcebergOrphanFileRemovalContent.MAX_OLDER_THAN_DAYS;
diff --git
a/common/src/test/java/org/apache/gravitino/dto/policy/TestSnapshotExpirationPolicyDTO.java
b/common/src/test/java/org/apache/gravitino/dto/policy/TestSnapshotExpirationPolicyDTO.java
index 7046c74c23..8b33647883 100644
---
a/common/src/test/java/org/apache/gravitino/dto/policy/TestSnapshotExpirationPolicyDTO.java
+++
b/common/src/test/java/org/apache/gravitino/dto/policy/TestSnapshotExpirationPolicyDTO.java
@@ -121,7 +121,15 @@ class TestSnapshotExpirationPolicyDTO {
"streamResults,\"true\"",
"streamResults,\"false\"",
"streamResults,[]",
- "streamResults,{}"
+ "streamResults,{}",
+ "minIntervalMs,1.9",
+ "minIntervalMs,\"100\"",
+ "commitTrigger,1",
+ "commitTrigger,\"false\"",
+ "timeTrigger,1",
+ "timeTrigger,true",
+ "jobOptions,1",
+ "updaterOptions,true"
})
void rejectInvalidValuesInCreateAndUpdate(String field, String value) {
String content = "{\"" + field + "\":" + value + "}";
diff --git
a/maintenance/optimizer-api/src/main/java/org/apache/gravitino/maintenance/optimizer/common/util/OrphanFileLocationUtils.java
b/maintenance/optimizer-api/src/main/java/org/apache/gravitino/maintenance/optimizer/common/util/OrphanFileLocationUtils.java
index c09ea3bc7c..5149325c97 100644
---
a/maintenance/optimizer-api/src/main/java/org/apache/gravitino/maintenance/optimizer/common/util/OrphanFileLocationUtils.java
+++
b/maintenance/optimizer-api/src/main/java/org/apache/gravitino/maintenance/optimizer/common/util/OrphanFileLocationUtils.java
@@ -22,6 +22,7 @@ import com.google.common.base.Preconditions;
import java.net.URI;
import java.nio.file.Paths;
import java.util.Objects;
+import org.apache.gravitino.policy.IcebergOrphanFileRemovalContent;
/** Lexical containment checks shared by orphan cleanup submission and
execution. */
public final class OrphanFileLocationUtils {
@@ -57,20 +58,9 @@ public final class OrphanFileLocationUtils {
* @return normalized URI
*/
public static URI normalizeLocation(String value) {
- // Reject ambiguous encoded paths rather than allowing different
filesystem decoders to
- // interpret the containment check and the subsequent listing differently.
- Preconditions.checkArgument(
- !value.isEmpty() && !value.contains("%") && !value.contains("\\"),
- "Invalid scan location: %s",
- value);
- URI uri = URI.create(value);
- Preconditions.checkArgument(
- uri.getQuery() == null
- && uri.getFragment() == null
- && uri.getPath() != null
- && uri.getPath().startsWith("/"),
- "Scan location must be an absolute path without query or fragment: %s",
- value);
+ // Reuse policy write-time syntax checks so create/update and job
submission reject the same
+ // illegal characters (spaces, control chars, query/fragment, encodings).
+ URI uri = IcebergOrphanFileRemovalContent.parseAndValidateLocation(value);
if (uri.getScheme() == null || "file".equals(uri.getScheme())) {
return (uri.getScheme() == null ? Paths.get(value) :
Paths.get(uri)).normalize().toUri();
}