This is an automated email from the ASF dual-hosted git repository.
huaxingao pushed a commit to branch 1.10.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/1.10.x by this push:
new b0ee3de24c Core: Align ContentFile partition JSON with REST spec
(#14738)
b0ee3de24c is described below
commit b0ee3de24ced105e5e8f932fe44e60618aed6f3a
Author: Drew Gallardo <[email protected]>
AuthorDate: Tue Dec 2 14:16:08 2025 -0800
Core: Align ContentFile partition JSON with REST spec (#14738)
(cherry picked from commit 9896e8cccac2b245ed20bfae602b29fc640dcd36)
---
.../java/org/apache/iceberg/ContentFileParser.java | 70 ++++++++---
.../org/apache/iceberg/TestContentFileParser.java | 129 +++++++++++++++++++--
.../org/apache/iceberg/TestDataTaskParser.java | 21 +++-
.../org/apache/iceberg/TestFileScanTaskParser.java | 38 +++++-
.../TestFetchPlanningResultResponseParser.java | 6 +-
.../TestFetchScanTasksResponseParser.java | 6 +-
.../responses/TestPlanTableScanResponseParser.java | 32 ++---
7 files changed, 244 insertions(+), 58 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/ContentFileParser.java
b/core/src/main/java/org/apache/iceberg/ContentFileParser.java
index b48334d822..1f23d5227e 100644
--- a/core/src/main/java/org/apache/iceberg/ContentFileParser.java
+++ b/core/src/main/java/org/apache/iceberg/ContentFileParser.java
@@ -25,6 +25,7 @@ import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.JsonUtil;
public class ContentFileParser {
@@ -89,7 +90,7 @@ public class ContentFileParser {
if (contentFile.partition() != null) {
generator.writeFieldName(PARTITION);
- SingleValueParser.toJson(spec.partitionType(), contentFile.partition(),
generator);
+ partitionToJson(spec.partitionType(), contentFile.partition(),
generator);
}
generator.writeNumberField(FILE_SIZE, contentFile.fileSizeInBytes());
@@ -152,18 +153,7 @@ public class ContentFileParser {
PartitionData partitionData = null;
if (jsonNode.has(PARTITION)) {
- partitionData = new PartitionData(spec.partitionType());
- StructLike structLike =
- (StructLike) SingleValueParser.fromJson(spec.partitionType(),
jsonNode.get(PARTITION));
- Preconditions.checkState(
- partitionData.size() == structLike.size(),
- "Invalid partition data size: expected = %s, actual = %s",
- partitionData.size(),
- structLike.size());
- for (int pos = 0; pos < partitionData.size(); ++pos) {
- Class<?> javaClass =
spec.partitionType().fields().get(pos).type().typeId().javaClass();
- partitionData.set(pos, structLike.get(pos, javaClass));
- }
+ partitionData = partitionFromJson(spec.partitionType(),
jsonNode.get(PARTITION));
}
long fileSizeInBytes = JsonUtil.getLong(FILE_SIZE, jsonNode);
@@ -301,4 +291,58 @@ public class ContentFileParser {
lowerBounds,
upperBounds);
}
+
+ private static void partitionToJson(
+ Types.StructType partitionType, StructLike partitionData, JsonGenerator
generator)
+ throws IOException {
+ generator.writeStartArray();
+ List<Types.NestedField> fields = partitionType.fields();
+ for (int pos = 0; pos < fields.size(); ++pos) {
+ Types.NestedField field = fields.get(pos);
+ Object partitionValue = partitionData.get(pos, Object.class);
+ SingleValueParser.toJson(field.type(), partitionValue, generator);
+ }
+ generator.writeEndArray();
+ }
+
+ private static PartitionData partitionFromJson(
+ Types.StructType partitionType, JsonNode partitionNode) {
+ List<Types.NestedField> fields = partitionType.fields();
+ PartitionData partitionData = new PartitionData(partitionType);
+
+ if (partitionNode.isArray()) {
+ Preconditions.checkArgument(
+ partitionNode.size() == fields.size(),
+ "Invalid partition data size: expected = %s, actual = %s",
+ fields.size(),
+ partitionNode.size());
+
+ for (int pos = 0; pos < fields.size(); ++pos) {
+ Types.NestedField field = fields.get(pos);
+ Object partitionValue = SingleValueParser.fromJson(field.type(),
partitionNode.get(pos));
+ partitionData.set(pos, partitionValue);
+ }
+ } else if (partitionNode.isObject()) {
+ // Handle partition struct object format, which serializes by field ID
and skips
+ // null partition values
+ Preconditions.checkState(
+ partitionNode.size() <= fields.size(),
+ "Invalid partition data size: expected <= %s, actual = %s",
+ fields.size(),
+ partitionNode.size());
+
+ StructLike structLike = (StructLike)
SingleValueParser.fromJson(partitionType, partitionNode);
+ for (int pos = 0; pos < partitionData.size(); ++pos) {
+ Class<?> javaClass = fields.get(pos).type().typeId().javaClass();
+ partitionData.set(pos, structLike.get(pos, javaClass));
+ }
+ } else {
+ throw new IllegalArgumentException(
+ String.format(
+ "Invalid partition data for content file: expected array or
object (%s)",
+ partitionNode));
+ }
+
+ return partitionData;
+ }
}
diff --git a/core/src/test/java/org/apache/iceberg/TestContentFileParser.java
b/core/src/test/java/org/apache/iceberg/TestContentFileParser.java
index 3f463f722e..60805c70c8 100644
--- a/core/src/test/java/org/apache/iceberg/TestContentFileParser.java
+++ b/core/src/test/java/org/apache/iceberg/TestContentFileParser.java
@@ -72,7 +72,7 @@ public class TestContentFileParser {
assertThat(jsonStr).isEqualTo(expectedJson);
JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr);
ContentFile<?> deserializedContentFile =
- ContentFileParser.fromJson(jsonNode, Map.of(TestBase.SPEC.specId(),
spec));
+ ContentFileParser.fromJson(jsonNode, Map.of(spec.specId(), spec));
assertThat(deserializedContentFile).isInstanceOf(DataFile.class);
assertContentFileEquals(dataFile, deserializedContentFile, spec);
}
@@ -85,11 +85,116 @@ public class TestContentFileParser {
assertThat(jsonStr).isEqualTo(expectedJson);
JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr);
ContentFile<?> deserializedContentFile =
- ContentFileParser.fromJson(jsonNode, Map.of(spec.specId(),
TestBase.SPEC));
+ ContentFileParser.fromJson(jsonNode, Map.of(spec.specId(), spec));
assertThat(deserializedContentFile).isInstanceOf(DeleteFile.class);
assertContentFileEquals(deleteFile, deserializedContentFile, spec);
}
+ @Test
+ public void testPartitionJsonArrayWrongSize() throws Exception {
+ PartitionSpec spec =
PartitionSpec.builderFor(TestBase.SCHEMA).identity("data").build();
+ String jsonStr =
+
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data.parquet\","
+ +
"\"file-format\":\"PARQUET\",\"partition\":[],\"file-size-in-bytes\":10,"
+ + "\"record-count\":1}";
+
+ JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr);
+
+ assertThatThrownBy(() -> ContentFileParser.fromJson(jsonNode,
Map.of(spec.specId(), spec)))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Invalid partition data size");
+ }
+
+ @Test
+ public void testPartitionJsonInvalidType() throws Exception {
+ PartitionSpec spec =
PartitionSpec.builderFor(TestBase.SCHEMA).identity("data").build();
+ String jsonStr =
+
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data.parquet\","
+ +
"\"file-format\":\"PARQUET\",\"partition\":\"invalid\",\"file-size-in-bytes\":10,"
+ + "\"record-count\":1}";
+
+ JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr);
+
+ assertThatThrownBy(() -> ContentFileParser.fromJson(jsonNode,
Map.of(spec.specId(), spec)))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("expected array or object");
+ }
+
+ @Test
+ public void testParsesFieldIdPartitionMap() throws Exception {
+ PartitionSpec spec =
PartitionSpec.builderFor(TestBase.SCHEMA).identity("data").build();
+ String legacyJson =
+
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data.parquet\","
+ +
"\"file-format\":\"PARQUET\",\"partition\":{\"1000\":\"foo\"},\"file-size-in-bytes\":10,"
+ + "\"record-count\":1}";
+
+ JsonNode jsonNode = JsonUtil.mapper().readTree(legacyJson);
+ ContentFile<?> deserializedContentFile =
+ ContentFileParser.fromJson(jsonNode, Map.of(spec.specId(), spec));
+
+ assertThat(deserializedContentFile).isInstanceOf(DataFile.class);
+ assertThat(deserializedContentFile.partition().get(0,
String.class)).isEqualTo("foo");
+ }
+
+ @Test
+ public void testPartitionStructObjectContainsExtraField() throws Exception {
+ PartitionSpec spec =
PartitionSpec.builderFor(TestBase.SCHEMA).identity("data").build();
+ String jsonStr =
+
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data.parquet\","
+ +
"\"file-format\":\"PARQUET\",\"partition\":{\"1000\":\"foo\",\"9999\":\"bar\"},"
+ + "\"file-size-in-bytes\":10,\"record-count\":1}";
+
+ JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr);
+ assertThatThrownBy(() -> ContentFileParser.fromJson(jsonNode,
Map.of(spec.specId(), spec)))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("Invalid partition data size");
+ }
+
+ @Test
+ public void testPartitionStructObjectEmptyIsNull() throws Exception {
+ PartitionSpec spec =
PartitionSpec.builderFor(TestBase.SCHEMA).identity("data").build();
+ String jsonStr =
+
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data.parquet\","
+ +
"\"file-format\":\"PARQUET\",\"partition\":{},\"file-size-in-bytes\":10,"
+ + "\"record-count\":1}";
+
+ JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr);
+ ContentFile<?> contentFile = ContentFileParser.fromJson(jsonNode,
Map.of(spec.specId(), spec));
+ assertThat(contentFile).isInstanceOf(DataFile.class);
+ assertThat(contentFile.partition().get(0, String.class)).isNull();
+ }
+
+ @Test
+ public void testPartitionArrayRespectsSpecOrder() throws Exception {
+ PartitionSpec spec =
+
PartitionSpec.builderFor(TestBase.SCHEMA).identity("id").identity("data").build();
+
+ PartitionData partitionData = new PartitionData(spec.partitionType());
+ partitionData.set(0, 4);
+ partitionData.set(1, "foo");
+
+ DataFile dataFile =
+ DataFiles.builder(spec)
+ .withPath("/path/to/data.parquet")
+ .withFileSizeInBytes(10)
+ .withRecordCount(1)
+ .withPartition(partitionData)
+ .build();
+
+ String jsonStr = ContentFileParser.toJson(dataFile, spec);
+
+ // Verify partition values are serialized as array in correct order
+ assertThat(jsonStr).contains("\"partition\":[4,\"foo\"]");
+
+ JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr);
+ ContentFile<?> deserializedContentFile =
+ ContentFileParser.fromJson(jsonNode, Map.of(spec.specId(), spec));
+
+ assertThat(deserializedContentFile).isInstanceOf(DataFile.class);
+ assertThat(deserializedContentFile.partition().get(0,
Integer.class)).isEqualTo(4);
+ assertThat(deserializedContentFile.partition().get(1,
String.class)).isEqualTo("foo");
+ }
+
private static Stream<Arguments> provideSpecAndDataFile() {
return Stream.of(
Arguments.of(
@@ -128,17 +233,17 @@ public class TestContentFileParser {
private static String dataFileJsonWithRequiredOnly(PartitionSpec spec) {
if (spec.isUnpartitioned()) {
return
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\",\"file-format\":\"PARQUET\","
- +
"\"partition\":{},\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}";
+ +
"\"partition\":[],\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}";
} else {
return
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\",\"file-format\":\"PARQUET\","
- +
"\"partition\":{\"1000\":1},\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}";
+ +
"\"partition\":[1],\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}";
}
}
private static String dataFileJsonWithAllOptional(PartitionSpec spec) {
if (spec.isUnpartitioned()) {
return
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-with-stats.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":{},\"file-size-in-bytes\":350,\"record-count\":10,"
+ +
"\"file-format\":\"PARQUET\",\"partition\":[],\"file-size-in-bytes\":350,\"record-count\":10,"
+ "\"column-sizes\":{\"keys\":[3,4],\"values\":[100,200]},"
+ "\"value-counts\":{\"keys\":[3,4],\"values\":[90,180]},"
+ "\"null-value-counts\":{\"keys\":[3,4],\"values\":[10,20]},"
@@ -149,7 +254,7 @@ public class TestContentFileParser {
+ "\"split-offsets\":[128,256],\"sort-order-id\":1}";
} else {
return
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-with-stats.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":{\"1000\":1},\"file-size-in-bytes\":350,\"record-count\":10,"
+ +
"\"file-format\":\"PARQUET\",\"partition\":[1],\"file-size-in-bytes\":350,\"record-count\":10,"
+ "\"column-sizes\":{\"keys\":[3,4],\"values\":[100,200]},"
+ "\"value-counts\":{\"keys\":[3,4],\"values\":[90,180]},"
+ "\"null-value-counts\":{\"keys\":[3,4],\"values\":[10,20]},"
@@ -245,7 +350,7 @@ public class TestContentFileParser {
private static String deleteFileWithDataRefJson() {
return
"{\"spec-id\":0,\"content\":\"POSITION_DELETES\",\"file-path\":\"/path/to/delete.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":{\"1000\":4},\"file-size-in-bytes\":1234,"
+ +
"\"file-format\":\"PARQUET\",\"partition\":[4],\"file-size-in-bytes\":1234,"
+
"\"record-count\":10,\"referenced-data-file\":\"/path/to/data/file.parquet\"}";
}
@@ -271,7 +376,7 @@ public class TestContentFileParser {
private static String dvJson() {
return
"{\"spec-id\":0,\"content\":\"POSITION_DELETES\",\"file-path\":\"/path/to/delete.puffin\","
- +
"\"file-format\":\"PUFFIN\",\"partition\":{\"1000\":4},\"file-size-in-bytes\":1234,\"record-count\":10,"
+ +
"\"file-format\":\"PUFFIN\",\"partition\":[4],\"file-size-in-bytes\":1234,\"record-count\":10,"
+
"\"referenced-data-file\":\"/path/to/data/file.parquet\",\"content-offset\":4,\"content-size-in-bytes\":40}";
}
@@ -344,17 +449,17 @@ public class TestContentFileParser {
private static String deleteFileJsonWithRequiredOnly(PartitionSpec spec) {
if (spec.isUnpartitioned()) {
return
"{\"spec-id\":0,\"content\":\"POSITION_DELETES\",\"file-path\":\"/path/to/delete-a.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":{},\"file-size-in-bytes\":1234,\"record-count\":9}";
+ +
"\"file-format\":\"PARQUET\",\"partition\":[],\"file-size-in-bytes\":1234,\"record-count\":9}";
} else {
return
"{\"spec-id\":0,\"content\":\"POSITION_DELETES\",\"file-path\":\"/path/to/delete-a.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":{\"1000\":9},\"file-size-in-bytes\":1234,\"record-count\":9}";
+ +
"\"file-format\":\"PARQUET\",\"partition\":[9],\"file-size-in-bytes\":1234,\"record-count\":9}";
}
}
private static String deleteFileJsonWithAllOptional(PartitionSpec spec) {
if (spec.isUnpartitioned()) {
return
"{\"spec-id\":0,\"content\":\"EQUALITY_DELETES\",\"file-path\":\"/path/to/delete-with-stats.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":{},\"file-size-in-bytes\":1234,\"record-count\":10,"
+ +
"\"file-format\":\"PARQUET\",\"partition\":[],\"file-size-in-bytes\":1234,\"record-count\":10,"
+ "\"column-sizes\":{\"keys\":[3,4],\"values\":[100,200]},"
+ "\"value-counts\":{\"keys\":[3,4],\"values\":[90,180]},"
+ "\"null-value-counts\":{\"keys\":[3,4],\"values\":[10,20]},"
@@ -365,7 +470,7 @@ public class TestContentFileParser {
+
"\"split-offsets\":[128],\"equality-ids\":[3],\"sort-order-id\":1}";
} else {
return
"{\"spec-id\":0,\"content\":\"EQUALITY_DELETES\",\"file-path\":\"/path/to/delete-with-stats.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":{\"1000\":9},\"file-size-in-bytes\":1234,\"record-count\":10,"
+ +
"\"file-format\":\"PARQUET\",\"partition\":[9],\"file-size-in-bytes\":1234,\"record-count\":10,"
+ "\"column-sizes\":{\"keys\":[3,4],\"values\":[100,200]},"
+ "\"value-counts\":{\"keys\":[3,4],\"values\":[90,180]},"
+ "\"null-value-counts\":{\"keys\":[3,4],\"values\":[10,20]},"
diff --git a/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java
b/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java
index 8048b6d24e..758c11db78 100644
--- a/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java
+++ b/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java
@@ -146,7 +146,7 @@ public class TestDataTaskParser {
+ "\"value\":\"string\",\"value-required\":true}}]},"
+ "\"metadata-file\":{\"spec-id\":0,\"content\":\"DATA\","
+ "\"file-path\":\"/tmp/metadata2.json\","
- + "\"file-format\":\"METADATA\",\"partition\":{},"
+ + "\"file-format\":\"METADATA\",\"partition\":[],"
+
"\"file-size-in-bytes\":0,\"record-count\":2,\"sort-order-id\":0}"
+ "}";
JsonNode missingTableRowsNode =
mapper.reader().readTree(missingTableRowsStr);
@@ -164,6 +164,23 @@ public class TestDataTaskParser {
assertDataTaskEquals(dataTask, deserializedTask);
}
+ @Test
+ public void testDataTaskParsesFieldIdPartitionMap() {
+ String jsonStr =
+ "{\"task-type\":\"data-task\","
+ + "\"schema\":{\"type\":\"struct\",\"schema-id\":0,"
+ +
"\"fields\":[{\"id\":1,\"name\":\"committed_at\",\"required\":true,\"type\":\"timestamptz\"}]},"
+ + "\"projection\":{\"type\":\"struct\",\"schema-id\":0,"
+ +
"\"fields\":[{\"id\":1,\"name\":\"committed_at\",\"required\":true,\"type\":\"timestamptz\"}]},"
+ +
"\"metadata-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/tmp/metadata.json\","
+ +
"\"file-format\":\"METADATA\",\"partition\":{},\"file-size-in-bytes\":0,\"record-count\":1,\"sort-order-id\":0},"
+ + "\"rows\":[{\"1\":\"2009-02-13T23:31:30+00:00\"}]}";
+
+ StaticDataTask deserializedTask = (StaticDataTask)
ScanTaskParser.fromJson(jsonStr, true);
+
+
assertThat(deserializedTask.metadataFile().partition().size()).isEqualTo(0);
+ }
+
private DataTask createDataTask() {
Map<String, String> summary1 =
ImmutableMap.of(
@@ -248,7 +265,7 @@ public class TestDataTaskParser {
+ "\"value\":\"string\",\"value-required\":true}}]},"
+ "\"metadata-file\":{\"spec-id\":0,\"content\":\"DATA\","
+ "\"file-path\":\"/tmp/metadata2.json\","
- + "\"file-format\":\"METADATA\",\"partition\":{},"
+ + "\"file-format\":\"METADATA\",\"partition\":[],"
+ "\"file-size-in-bytes\":0,\"record-count\":2,\"sort-order-id\":0},"
+
"\"rows\":[{\"1\":\"2009-02-13T23:31:30+00:00\",\"2\":1,\"4\":\"append\","
+ "\"5\":\"file:/tmp/manifest1.avro\","
diff --git a/core/src/test/java/org/apache/iceberg/TestFileScanTaskParser.java
b/core/src/test/java/org/apache/iceberg/TestFileScanTaskParser.java
index c4a9fdf234..882c2b3349 100644
--- a/core/src/test/java/org/apache/iceberg/TestFileScanTaskParser.java
+++ b/core/src/test/java/org/apache/iceberg/TestFileScanTaskParser.java
@@ -62,6 +62,18 @@ public class TestFileScanTaskParser {
assertFileScanTaskEquals(fileScanTask, deserializedTask, spec,
caseSensitive);
}
+ @Test
+ public void testFileScanTaskParsesFieldIdPartitionMap() {
+ boolean caseSensitive = true;
+ PartitionSpec spec = TestBase.SPEC;
+ FileScanTask expected = createFileScanTask(spec, caseSensitive);
+
+ FileScanTask deserializedTask =
+ ScanTaskParser.fromJson(fileScanTaskFieldIdPartitionMapJson(),
caseSensitive);
+
+ assertFileScanTaskEquals(expected, deserializedTask, spec, caseSensitive);
+ }
+
private FileScanTask createFileScanTask(PartitionSpec spec, boolean
caseSensitive) {
ResidualEvaluator residualEvaluator;
if (spec.isUnpartitioned()) {
@@ -85,19 +97,39 @@ public class TestFileScanTaskParser {
+ "\"spec\":{\"spec-id\":0,\"fields\":[{\"name\":\"data_bucket\","
+ "\"transform\":\"bucket[16]\",\"source-id\":4,\"field-id\":1000}]},"
+
"\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\","
- + "\"file-format\":\"PARQUET\",\"partition\":{\"1000\":0},"
+ + "\"file-format\":\"PARQUET\",\"partition\":[0],"
+ "\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0},"
+ "\"start\":0,\"length\":10,"
+ "\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\","
+
"\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\","
- +
"\"partition\":{\"1000\":0},\"file-size-in-bytes\":10,\"record-count\":1},"
+ + "\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1},"
+
"{\"spec-id\":0,\"content\":\"EQUALITY_DELETES\",\"file-path\":\"/path/to/data-a2-deletes.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":{\"1000\":0},\"file-size-in-bytes\":10,"
+ +
"\"file-format\":\"PARQUET\",\"partition\":[0],\"file-size-in-bytes\":10,"
+ "\"record-count\":1,\"equality-ids\":[1],\"sort-order-id\":0}],"
+ "\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}";
}
private String fileScanTaskJson() {
+ return "{\"task-type\":\"file-scan-task\","
+ + "\"schema\":{\"type\":\"struct\",\"schema-id\":0,\"fields\":["
+ + "{\"id\":3,\"name\":\"id\",\"required\":true,\"type\":\"int\"},"
+ +
"{\"id\":4,\"name\":\"data\",\"required\":true,\"type\":\"string\"}]},"
+ + "\"spec\":{\"spec-id\":0,\"fields\":[{\"name\":\"data_bucket\","
+ + "\"transform\":\"bucket[16]\",\"source-id\":4,\"field-id\":1000}]},"
+ +
"\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\","
+ + "\"file-format\":\"PARQUET\",\"partition\":[0],"
+ + "\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0},"
+ + "\"start\":0,\"length\":10,"
+ + "\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\","
+ +
"\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\","
+ + "\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1},"
+ +
"{\"spec-id\":0,\"content\":\"EQUALITY_DELETES\",\"file-path\":\"/path/to/data-a2-deletes.parquet\","
+ +
"\"file-format\":\"PARQUET\",\"partition\":[0],\"file-size-in-bytes\":10,"
+ + "\"record-count\":1,\"equality-ids\":[1],\"sort-order-id\":0}],"
+ + "\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}";
+ }
+
+ private String fileScanTaskFieldIdPartitionMapJson() {
return "{\"task-type\":\"file-scan-task\","
+ "\"schema\":{\"type\":\"struct\",\"schema-id\":0,\"fields\":["
+ "{\"id\":3,\"name\":\"id\",\"required\":true,\"type\":\"int\"},"
diff --git
a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java
b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java
index 9db9e29530..8b82e9794d 100644
---
a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java
+++
b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java
@@ -157,7 +157,7 @@ public class TestFetchPlanningResultResponseParser {
"{\"status\":\"submitted\","
+
"\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\","
+
"\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\","
- +
"\"partition\":{\"1000\":0},\"file-size-in-bytes\":10,\"record-count\":1}]"
+ +
"\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}]"
+ "}";
assertThatThrownBy(
@@ -195,10 +195,10 @@ public class TestFetchPlanningResultResponseParser {
"{\"status\":\"completed\","
+
"\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\","
+
"\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\","
- +
"\"partition\":{\"1000\":0},\"file-size-in-bytes\":10,\"record-count\":1}],"
+ +
"\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}],"
+ "\"file-scan-tasks\":["
+
"{\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\","
- + "\"file-format\":\"PARQUET\",\"partition\":{\"1000\":0},"
+ + "\"file-format\":\"PARQUET\",\"partition\":[0],"
+
"\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0},"
+ "\"delete-file-references\":[0],"
+
"\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}]"
diff --git
a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java
b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java
index 01ca8288fb..62de75e381 100644
---
a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java
+++
b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java
@@ -97,7 +97,7 @@ public class TestFetchScanTasksResponseParser {
"{\"plan-tasks\":[\"task1\",\"task2\"],"
+
"\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\","
+
"\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\","
- +
"\"partition\":{\"1000\":0},\"file-size-in-bytes\":10,\"record-count\":1}]"
+ +
"\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}]"
+ "}";
assertThatThrownBy(
@@ -131,10 +131,10 @@ public class TestFetchScanTasksResponseParser {
"{"
+
"\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\","
+
"\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\","
- +
"\"partition\":{\"1000\":0},\"file-size-in-bytes\":10,\"record-count\":1}],"
+ +
"\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}],"
+ "\"file-scan-tasks\":["
+
"{\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\","
- + "\"file-format\":\"PARQUET\",\"partition\":{\"1000\":0},"
+ + "\"file-format\":\"PARQUET\",\"partition\":[0],"
+
"\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0},"
+ "\"delete-file-references\":[0],"
+
"\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}]"
diff --git
a/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java
b/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java
index 48f8a5dcb6..1bb23dd623 100644
---
a/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java
+++
b/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java
@@ -208,7 +208,7 @@ public class TestPlanTableScanResponseParser {
+ "\"plan-id\":\"somePlanId\","
+
"\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\","
+
"\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\","
- +
"\"partition\":{\"1000\":0},\"file-size-in-bytes\":10,\"record-count\":1}]"
+ +
"\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}]"
+ "}";
assertThatThrownBy(
@@ -242,10 +242,10 @@ public class TestPlanTableScanResponseParser {
"{\"status\":\"completed\","
+
"\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\","
+
"\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\","
- +
"\"partition\":{\"1000\":0},\"file-size-in-bytes\":10,\"record-count\":1}],"
+ +
"\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}],"
+ "\"file-scan-tasks\":["
+
"{\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\","
- + "\"file-format\":\"PARQUET\",\"partition\":{\"1000\":0},"
+ + "\"file-format\":\"PARQUET\",\"partition\":[0],"
+
"\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0},"
+ "\"delete-file-references\":[0],"
+
"\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}]"
@@ -315,9 +315,7 @@ public class TestPlanTableScanResponseParser {
+ " \"content\" : \"POSITION_DELETES\",\n"
+ " \"file-path\" : \"/path/to/data-a-deletes.parquet\",\n"
+ " \"file-format\" : \"PARQUET\",\n"
- + " \"partition\" : {\n"
- + " \"1000\" : 0\n"
- + " },\n"
+ + " \"partition\" : [ 0 ],\n"
+ " \"file-size-in-bytes\" : 10,\n"
+ " \"record-count\" : 1\n"
+ " }, {\n"
@@ -325,9 +323,7 @@ public class TestPlanTableScanResponseParser {
+ " \"content\" : \"POSITION_DELETES\",\n"
+ " \"file-path\" : \"/path/to/data-b-deletes.parquet\",\n"
+ " \"file-format\" : \"PARQUET\",\n"
- + " \"partition\" : {\n"
- + " \"1000\" : 1\n"
- + " },\n"
+ + " \"partition\" : [ 1 ],\n"
+ " \"file-size-in-bytes\" : 10,\n"
+ " \"record-count\" : 1\n"
+ " }, {\n"
@@ -335,9 +331,7 @@ public class TestPlanTableScanResponseParser {
+ " \"content\" : \"EQUALITY_DELETES\",\n"
+ " \"file-path\" : \"/path/to/data-c-deletes.parquet\",\n"
+ " \"file-format\" : \"PARQUET\",\n"
- + " \"partition\" : {\n"
- + " \"1000\" : 2\n"
- + " },\n"
+ + " \"partition\" : [ 2 ],\n"
+ " \"file-size-in-bytes\" : 10,\n"
+ " \"record-count\" : 1,\n"
+ " \"equality-ids\" : [ 1 ],\n"
@@ -349,9 +343,7 @@ public class TestPlanTableScanResponseParser {
+ " \"content\" : \"DATA\",\n"
+ " \"file-path\" : \"/path/to/data-a.parquet\",\n"
+ " \"file-format\" : \"PARQUET\",\n"
- + " \"partition\" : {\n"
- + " \"1000\" : 0\n"
- + " },\n"
+ + " \"partition\" : [ 0 ],\n"
+ " \"file-size-in-bytes\" : 10,\n"
+ " \"record-count\" : 1,\n"
+ " \"sort-order-id\" : 0\n"
@@ -364,9 +356,7 @@ public class TestPlanTableScanResponseParser {
+ " \"content\" : \"DATA\",\n"
+ " \"file-path\" : \"/path/to/data-b.parquet\",\n"
+ " \"file-format\" : \"PARQUET\",\n"
- + " \"partition\" : {\n"
- + " \"1000\" : 1\n"
- + " },\n"
+ + " \"partition\" : [ 1 ],\n"
+ " \"file-size-in-bytes\" : 10,\n"
+ " \"record-count\" : 1,\n"
+ " \"split-offsets\" : [ 1 ],\n"
@@ -380,9 +370,7 @@ public class TestPlanTableScanResponseParser {
+ " \"content\" : \"DATA\",\n"
+ " \"file-path\" : \"/path/to/data-c.parquet\",\n"
+ " \"file-format\" : \"PARQUET\",\n"
- + " \"partition\" : {\n"
- + " \"1000\" : 2\n"
- + " },\n"
+ + " \"partition\" : [ 2 ],\n"
+ " \"file-size-in-bytes\" : 10,\n"
+ " \"record-count\" : 1,\n"
+ " \"split-offsets\" : [ 2, 8 ],\n"
@@ -418,7 +406,7 @@ public class TestPlanTableScanResponseParser {
"{\"status\":\"completed\","
+ "\"file-scan-tasks\":["
+
"{\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\","
- + "\"file-format\":\"PARQUET\",\"partition\":{\"1000\":0},"
+ + "\"file-format\":\"PARQUET\",\"partition\":[0],"
+
"\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0},"
+
"\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}]"
+ "}";
