This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch branch-1.3
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/branch-1.3 by this push:
new 2ff005d1ff [Cherry-pick to branch-1.3] [#14461] fix(iceberg-rest):
follow the REST spec in scan planning requests and refuse plans it cannot
represent (#14463)
2ff005d1ff is described below
commit 2ff005d1ffda53bd5b8586807ec58c0243680ea3
Author: Mark Hoerth <[email protected]>
AuthorDate: Sun Nov 1 19:40:42 2026 -0800
[Cherry-pick to branch-1.3] [#14461] fix(iceberg-rest): follow the REST
spec in scan planning requests and refuse plans it cannot represent (#14463)
Backport to branch-1.3 of the fix for #14461 (same change as the PR to
main).
### What changes were proposed in this pull request?
- Incremental scans use `fromSnapshotExclusive`, and a range from a
snapshot to itself is empty.
- A `ValidationException` raised while files are planned (unknown
column, wrong case, wrong value type) is reported as 400 without the
table's schema struct. Unknown `stats-fields` names are refused up front
with a plain message.
- `IcebergObjectMapper` accepts the spec's constant filters `{"type":
"true"}` / `{"type": "false"}` anywhere in the filter, as well as
Iceberg's boolean form.
- `checkDeleteFilesCanBeReferenced`: when a plan contains different
delete files with the same location (deletion vectors sharing a Puffin
file), the plan is refused with an error, because Iceberg's response
serializer would reference the wrong deletion vector for every task but
one. Wrong results are worse than an error; the client can plan the scan
itself.
If #14458 is merged first, `IcebergObjectMapper` needs a trivial merge
(both register a module after `RESTSerializers.registerAll`).
### Why are the changes needed?
Spec-conformant clients received the start snapshot's files twice, could
not send the spec's constant filters, got 500 for invalid filters, and
could read wrong data from v3 tables with deletion vectors.
Fixes #14461
### Does this PR introduce _any_ user-facing change?
Yes. Incremental plans exclude the start snapshot. Invalid filters
return 400. Spec constant filters are accepted. Plans whose deletion
vectors cannot be referenced correctly return an error instead of wrong
references.
### How was this patch tested?
`TestIcebergScanPlanning`: exclusive start and the empty same-snapshot
range; unknown column, wrong case, and unknown stats field (400 without
the schema struct or NPE text); constant filters in the spec form and
nested; deletion vectors sharing a file (refused) and a shared delete
file (accepted).
`./gradlew :iceberg:iceberg-rest-server:test -PskipITs`
🤖 Generated with [Claude Code](https://claude.com/claude-code)
https://claude.ai/code/session_01Eu6fb9pLfmrSJzGQDVLi1g
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
.../iceberg/service/CatalogWrapperForREST.java | 61 ++++++-
.../iceberg/service/IcebergObjectMapper.java | 53 ++++++
.../iceberg/service/TestIcebergScanPlanning.java | 183 +++++++++++++++++++++
3 files changed, 293 insertions(+), 4 deletions(-)
diff --git
a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/CatalogWrapperForREST.java
b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/CatalogWrapperForREST.java
index e02c6d4f55..a8aff3076c 100644
---
a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/CatalogWrapperForREST.java
+++
b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/CatalogWrapperForREST.java
@@ -30,6 +30,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Stream;
@@ -48,6 +49,7 @@ import org.apache.gravitino.storage.GCSProperties;
import org.apache.gravitino.utils.ClassUtils;
import org.apache.gravitino.utils.MapUtils;
import org.apache.gravitino.utils.PrincipalUtils;
+import org.apache.iceberg.DeleteFile;
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.IncrementalAppendScan;
import org.apache.iceberg.Scan;
@@ -57,7 +59,9 @@ import org.apache.iceberg.TableProperties;
import org.apache.iceberg.TableScan;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.ServiceFailureException;
import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.rest.CatalogHandlers;
import org.apache.iceberg.rest.PlanStatus;
@@ -433,6 +437,7 @@ public class CatalogWrapperForREST extends
IcebergCatalogWrapper {
tableIdentifier);
}
+ checkDeleteFilesCanBeReferenced(tableIdentifier, fileScanTasks);
PlanTableScanResponse response;
try {
response = buildCompletedPlanTableScanResponse(table, fileScanTasks);
@@ -449,9 +454,12 @@ public class CatalogWrapperForREST extends
IcebergCatalogWrapper {
scanPlanCache.put(ScanPlanCacheKey.create(tableIdentifier, table,
scanRequest), response);
return response;
- } catch (IllegalArgumentException e) {
- LOG.error("Invalid scan request for table {}: {}", tableIdentifier,
e.getMessage());
- throw new IllegalArgumentException("Invalid scan parameters: " +
e.getMessage(), e);
+ } catch (IllegalArgumentException | ValidationException e) {
+ // A filter is bound to the schema only while files are planned, so an
unknown column or a
+ // value of the wrong type surfaces here as a ValidationException.
+ LOG.info("Invalid scan request for table {}: {}", tableIdentifier,
e.getMessage());
+ throw new IllegalArgumentException(
+ "Invalid scan parameters: " + withoutSchemaStruct(e.getMessage()),
e);
} catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
LOG.error("Table not found during scan planning: {}", tableIdentifier);
throw e;
@@ -500,6 +508,10 @@ public class CatalogWrapperForREST extends
IcebergCatalogWrapper {
Long startSnapshotId = scanRequest.startSnapshotId();
Long endSnapshotId = scanRequest.endSnapshotId();
// Use IncrementalAppendScan if both start and end snapshot IDs are
provided
+ if (startSnapshotId != null && startSnapshotId.equals(endSnapshotId)) {
+ // The start is exclusive, so the range from a snapshot to itself is
empty.
+ return CloseableIterable.empty();
+ }
if (startSnapshotId != null && endSnapshotId != null) {
LOG.debug(
"Using IncrementalAppendScan for table: {}, from snapshot: {} to
snapshot: {}",
@@ -509,7 +521,9 @@ public class CatalogWrapperForREST extends
IcebergCatalogWrapper {
IncrementalAppendScan incrementalScan =
table
.newIncrementalAppendScan()
- .fromSnapshotInclusive(startSnapshotId)
+ // The REST spec defines start-snapshot-id as exclusive, and
Iceberg's REST client
+ // sends it from appendsBetween/appendsAfter, which are
exclusive as well.
+ .fromSnapshotExclusive(startSnapshotId)
.toSnapshot(endSnapshotId);
incrementalScan = applyScanRequest(incrementalScan, scanRequest);
return incrementalScan.planFiles();
@@ -567,6 +581,15 @@ public class CatalogWrapperForREST extends
IcebergCatalogWrapper {
@SuppressWarnings("unchecked")
private <T extends Scan> T applyScanStatsFields(T scan, PlanTableScanRequest
scanRequest) {
if (scanRequest.statsFields() != null &&
!scanRequest.statsFields().isEmpty()) {
+ for (String field : scanRequest.statsFields()) {
+ boolean known =
+ scanRequest.caseSensitive()
+ ? scan.schema().findField(field) != null
+ : scan.schema().caseInsensitiveFindField(field) != null;
+ if (!known) {
+ throw new IllegalArgumentException("Invalid statistics fields:
unknown field " + field);
+ }
+ }
try {
scan = (T) scan.includeColumnStats(scanRequest.statsFields());
LOG.debug("Applied statistics fields: {}", scanRequest.statsFields());
@@ -624,4 +647,34 @@ public class CatalogWrapperForREST extends
IcebergCatalogWrapper {
properties.put(newProperty, deprecatedValue);
}
}
+ /**
+ * Iceberg's plan response serializer refers to delete files by their
location. Deletion vectors
+ * that share one Puffin file have the same location, so every task would
refer to whichever of
+ * them was indexed last, and a client would apply the wrong deletion
vector. Such a plan is
+ * refused rather than answered wrongly.
+ */
+ @VisibleForTesting
+ static void checkDeleteFilesCanBeReferenced(
+ TableIdentifier tableIdentifier, List<FileScanTask> fileScanTasks) {
+ Map<String, DeleteFile> byLocation = new HashMap<>();
+ for (FileScanTask task : fileScanTasks) {
+ for (DeleteFile deleteFile : task.deletes()) {
+ DeleteFile other = byLocation.putIfAbsent(deleteFile.location(),
deleteFile);
+ if (other != null
+ && !(Objects.equals(other.contentOffset(),
deleteFile.contentOffset())
+ && Objects.equals(other.referencedDataFile(),
deleteFile.referencedDataFile()))) {
+ throw new ServiceFailureException(
+ "Server-side scan planning cannot return table %s: several
deletion vectors are"
+ + " stored in %s, which the plan response format cannot
reference separately;"
+ + " plan the scan on the client",
+ tableIdentifier, deleteFile.location());
+ }
+ }
+ }
+ }
+
+ private static String withoutSchemaStruct(String message) {
+ // Iceberg's binding errors append the whole table struct; it is not
useful to a client.
+ return message == null ? null : message.replaceAll(" in struct: .*$", "");
+ }
}
diff --git
a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergObjectMapper.java
b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergObjectMapper.java
index 76fb9a6610..ed530ecef3 100644
---
a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergObjectMapper.java
+++
b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergObjectMapper.java
@@ -20,10 +20,20 @@ package org.apache.gravitino.iceberg.service;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.PropertyNamingStrategies;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.node.BooleanNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.io.IOException;
import org.apache.iceberg.rest.RESTSerializers;
+import org.apache.iceberg.rest.requests.PlanTableScanRequest;
+import org.apache.iceberg.rest.requests.PlanTableScanRequestParser;
public class IcebergObjectMapper {
private static final ObjectMapper objectMapper = createIcebergObjectMapper();
@@ -36,9 +46,52 @@ public class IcebergObjectMapper {
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
mapper.setPropertyNamingStrategy(new
PropertyNamingStrategies.KebabCaseStrategy());
RESTSerializers.registerAll(mapper);
+ // Registered after Iceberg's serializers, so it takes precedence for the
same type.
+ mapper.registerModule(
+ new SimpleModule("gravitino-iceberg-scan-request")
+ .addDeserializer(PlanTableScanRequest.class, new
PlanTableScanRequestDeserializer()));
return mapper;
}
+ /**
+ * The REST spec writes the constant expressions as {@code {"type": "true"}}
and {@code {"type":
+ * "false"}}, while Iceberg's parser accepts them only as JSON booleans.
Both forms are accepted.
+ */
+ private static class PlanTableScanRequestDeserializer
+ extends JsonDeserializer<PlanTableScanRequest> {
+ @Override
+ public PlanTableScanRequest deserialize(JsonParser parser,
DeserializationContext context)
+ throws IOException {
+ JsonNode node = parser.getCodec().readTree(parser);
+ if (node instanceof ObjectNode && node.has("filter")) {
+ ((ObjectNode) node).set("filter",
normalizeConstants(node.get("filter")));
+ }
+ return PlanTableScanRequestParser.fromJson(node);
+ }
+
+ private static JsonNode normalizeConstants(JsonNode expression) {
+ if (!(expression instanceof ObjectNode)) {
+ return expression;
+ }
+ ObjectNode object = (ObjectNode) expression;
+ JsonNode type = object.get("type");
+ if (type != null && type.isTextual() && object.size() == 1) {
+ if ("true".equals(type.asText())) {
+ return BooleanNode.TRUE;
+ }
+ if ("false".equals(type.asText())) {
+ return BooleanNode.FALSE;
+ }
+ }
+ for (String child : new String[] {"left", "right", "child"}) {
+ if (object.has(child)) {
+ object.set(child, normalizeConstants(object.get(child)));
+ }
+ }
+ return object;
+ }
+ }
+
public static ObjectMapper getInstance() {
return objectMapper;
}
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestIcebergScanPlanning.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestIcebergScanPlanning.java
new file mode 100644
index 0000000000..84b11e5ec1
--- /dev/null
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestIcebergScanPlanning.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.iceberg.service;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
+import org.apache.gravitino.iceberg.common.IcebergConfig;
+import org.apache.iceberg.DataFiles;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.rest.requests.PlanTableScanRequest;
+import org.apache.iceberg.rest.responses.PlanTableScanResponse;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+/** Tests server-side scan planning against the REST spec's request semantics.
*/
+public class TestIcebergScanPlanning {
+
+ private static final TableIdentifier IDENT =
TableIdentifier.of(Namespace.of("db"), "t");
+
+ private CatalogWrapperForREST wrapper;
+ private Table table;
+
+ @BeforeEach
+ public void setUp() {
+ wrapper =
+ new CatalogWrapperForREST(
+ "scan-test",
+ new IcebergConfig(
+ ImmutableMap.of(
+ IcebergConstants.CATALOG_BACKEND, "memory",
+ IcebergConstants.WAREHOUSE, "/tmp/warehouse")));
+ ((SupportsNamespaces)
wrapper.getCatalog()).createNamespace(Namespace.of("db"));
+ table =
+ wrapper
+ .getCatalog()
+ .createTable(
+ IDENT,
+ new Schema(
+ Types.NestedField.required(1, "id", Types.LongType.get()),
+ Types.NestedField.optional(2, "a",
Types.StringType.get())),
+ PartitionSpec.unpartitioned());
+ }
+
+ @Test
+ public void testIncrementalScanStartIsExclusive() {
+ long s1 = append("/data/f1.parquet");
+ append("/data/f2.parquet");
+ long s3 = append("/data/f3.parquet");
+
+ Assertions.assertEquals(
+ ImmutableList.of("/data/f2.parquet", "/data/f3.parquet"),
+
planFiles(PlanTableScanRequest.builder().withStartSnapshotId(s1).withEndSnapshotId(s3)));
+ Assertions.assertEquals(
+ ImmutableList.of(),
+
planFiles(PlanTableScanRequest.builder().withStartSnapshotId(s3).withEndSnapshotId(s3)));
+ }
+
+ @Test
+ public void testInvalidFilterAndStatsFieldsAreClientErrors() {
+ append("/data/f1.parquet");
+ IllegalArgumentException unknownColumn =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+
planFiles(PlanTableScanRequest.builder().withFilter(Expressions.equal("nope",
1))));
+ Assertions.assertFalse(
+ unknownColumn.getMessage().contains("struct<"),
unknownColumn.getMessage());
+
+ // Case-sensitive by default.
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
planFiles(PlanTableScanRequest.builder().withFilter(Expressions.equal("ID",
1L))));
+
+ IllegalArgumentException unknownStats =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ planFiles(
+
PlanTableScanRequest.builder().withStatsFields(ImmutableList.of("nope"))));
+ Assertions.assertTrue(unknownStats.getMessage().contains("nope"),
unknownStats.getMessage());
+ Assertions.assertFalse(
+ unknownStats.getMessage().contains("NullPointerException"),
unknownStats.getMessage());
+ }
+
+ @Test
+ public void testConstantFiltersInTheSpecFormAreAccepted() throws Exception {
+ PlanTableScanRequest request =
+ IcebergObjectMapper.getInstance()
+ .readValue(
+ "{\"filter\":{\"type\":\"not\",\"child\":{\"type\":\"or\","
+ +
"\"left\":{\"type\":\"false\"},\"right\":{\"type\":\"false\"}}}}",
+ PlanTableScanRequest.class);
+ Assertions.assertEquals(Expression.Operation.TRUE, request.filter().op());
+
+ PlanTableScanRequest alsoAccepted =
+ IcebergObjectMapper.getInstance()
+ .readValue("{\"filter\":true}", PlanTableScanRequest.class);
+ Assertions.assertEquals(Expression.Operation.TRUE,
alsoAccepted.filter().op());
+ }
+
+ @Test
+ public void testDeletionVectorsSharingAFileAreRefused() {
+ DeleteFile dv1 = deletionVector("/data/dv.puffin", 4L, "/data/f1.parquet");
+ DeleteFile dv2 = deletionVector("/data/dv.puffin", 120L,
"/data/f2.parquet");
+ FileScanTask task1 = Mockito.mock(FileScanTask.class);
+ Mockito.when(task1.deletes()).thenReturn(ImmutableList.of(dv1));
+ FileScanTask task2 = Mockito.mock(FileScanTask.class);
+ Mockito.when(task2.deletes()).thenReturn(ImmutableList.of(dv2));
+
+ Assertions.assertThrows(
+ ServiceFailureException.class,
+ () ->
+ CatalogWrapperForREST.checkDeleteFilesCanBeReferenced(
+ IDENT, ImmutableList.of(task1, task2)));
+ // The same delete file shared by tasks is fine.
+ Assertions.assertDoesNotThrow(
+ () ->
+ CatalogWrapperForREST.checkDeleteFilesCanBeReferenced(
+ IDENT, ImmutableList.of(task1, task1)));
+ }
+
+ private static DeleteFile deletionVector(String location, long offset,
String dataFile) {
+ DeleteFile file = Mockito.mock(DeleteFile.class);
+ Mockito.when(file.location()).thenReturn(location);
+ Mockito.when(file.contentOffset()).thenReturn(offset);
+ Mockito.when(file.referencedDataFile()).thenReturn(dataFile);
+ return file;
+ }
+
+ private long append(String path) {
+ table
+ .newFastAppend()
+ .appendFile(
+ DataFiles.builder(PartitionSpec.unpartitioned())
+ .withPath(path)
+ .withFileSizeInBytes(10)
+ .withRecordCount(1)
+ .build())
+ .commit();
+ table.refresh();
+ return table.currentSnapshot().snapshotId();
+ }
+
+ private List<String> planFiles(PlanTableScanRequest.Builder request) {
+ PlanTableScanResponse response = wrapper.planTableScan(IDENT,
request.build());
+ return response.fileScanTasks().stream()
+ .map(task -> task.file().location())
+ .sorted()
+ .collect(Collectors.toList());
+ }
+}