This is an automated email from the ASF dual-hosted git repository.
huaxingao pushed a commit to branch 1.10.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/1.10.x by this push:
new b14249e1c8 Revert "Core: Align ContentFile partition JSON with REST
spec (#14738)" (#14742)
b14249e1c8 is described below
commit b14249e1c8f8bc20a305ce922e542f88a3fb7767
Author: Drew Gallardo <[email protected]>
AuthorDate: Wed Dec 3 09:01:44 2025 -0800
Revert "Core: Align ContentFile partition JSON with REST spec (#14738)"
(#14742)
This reverts commit b0ee3de24ced105e5e8f932fe44e60618aed6f3a.
---
.../java/org/apache/iceberg/ContentFileParser.java | 70 +++--------
.../org/apache/iceberg/TestContentFileParser.java | 129 ++-------------------
.../org/apache/iceberg/TestDataTaskParser.java | 21 +---
.../org/apache/iceberg/TestFileScanTaskParser.java | 38 +-----
.../TestFetchPlanningResultResponseParser.java | 6 +-
.../TestFetchScanTasksResponseParser.java | 6 +-
.../responses/TestPlanTableScanResponseParser.java | 32 +++--
7 files changed, 58 insertions(+), 244 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/ContentFileParser.java
b/core/src/main/java/org/apache/iceberg/ContentFileParser.java
index 1f23d5227e..b48334d822 100644
--- a/core/src/main/java/org/apache/iceberg/ContentFileParser.java
+++ b/core/src/main/java/org/apache/iceberg/ContentFileParser.java
@@ -25,7 +25,6 @@ import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.JsonUtil;
public class ContentFileParser {
@@ -90,7 +89,7 @@ public class ContentFileParser {
if (contentFile.partition() != null) {
generator.writeFieldName(PARTITION);
- partitionToJson(spec.partitionType(), contentFile.partition(),
generator);
+ SingleValueParser.toJson(spec.partitionType(), contentFile.partition(),
generator);
}
generator.writeNumberField(FILE_SIZE, contentFile.fileSizeInBytes());
@@ -153,7 +152,18 @@ public class ContentFileParser {
PartitionData partitionData = null;
if (jsonNode.has(PARTITION)) {
- partitionData = partitionFromJson(spec.partitionType(),
jsonNode.get(PARTITION));
+ partitionData = new PartitionData(spec.partitionType());
+ StructLike structLike =
+ (StructLike) SingleValueParser.fromJson(spec.partitionType(),
jsonNode.get(PARTITION));
+ Preconditions.checkState(
+ partitionData.size() == structLike.size(),
+ "Invalid partition data size: expected = %s, actual = %s",
+ partitionData.size(),
+ structLike.size());
+ for (int pos = 0; pos < partitionData.size(); ++pos) {
+ Class<?> javaClass =
spec.partitionType().fields().get(pos).type().typeId().javaClass();
+ partitionData.set(pos, structLike.get(pos, javaClass));
+ }
}
long fileSizeInBytes = JsonUtil.getLong(FILE_SIZE, jsonNode);
@@ -291,58 +301,4 @@ public class ContentFileParser {
lowerBounds,
upperBounds);
}
-
- private static void partitionToJson(
- Types.StructType partitionType, StructLike partitionData, JsonGenerator
generator)
- throws IOException {
- generator.writeStartArray();
- List<Types.NestedField> fields = partitionType.fields();
- for (int pos = 0; pos < fields.size(); ++pos) {
- Types.NestedField field = fields.get(pos);
- Object partitionValue = partitionData.get(pos, Object.class);
- SingleValueParser.toJson(field.type(), partitionValue, generator);
- }
- generator.writeEndArray();
- }
-
- private static PartitionData partitionFromJson(
- Types.StructType partitionType, JsonNode partitionNode) {
- List<Types.NestedField> fields = partitionType.fields();
- PartitionData partitionData = new PartitionData(partitionType);
-
- if (partitionNode.isArray()) {
- Preconditions.checkArgument(
- partitionNode.size() == fields.size(),
- "Invalid partition data size: expected = %s, actual = %s",
- fields.size(),
- partitionNode.size());
-
- for (int pos = 0; pos < fields.size(); ++pos) {
- Types.NestedField field = fields.get(pos);
- Object partitionValue = SingleValueParser.fromJson(field.type(),
partitionNode.get(pos));
- partitionData.set(pos, partitionValue);
- }
- } else if (partitionNode.isObject()) {
- // Handle partition struct object format, which serializes by field ID
and skips
- // null partition values
- Preconditions.checkState(
- partitionNode.size() <= fields.size(),
- "Invalid partition data size: expected <= %s, actual = %s",
- fields.size(),
- partitionNode.size());
-
- StructLike structLike = (StructLike)
SingleValueParser.fromJson(partitionType, partitionNode);
- for (int pos = 0; pos < partitionData.size(); ++pos) {
- Class<?> javaClass = fields.get(pos).type().typeId().javaClass();
- partitionData.set(pos, structLike.get(pos, javaClass));
- }
- } else {
- throw new IllegalArgumentException(
- String.format(
- "Invalid partition data for content file: expected array or
object (%s)",
- partitionNode));
- }
-
- return partitionData;
- }
}
diff --git a/core/src/test/java/org/apache/iceberg/TestContentFileParser.java
b/core/src/test/java/org/apache/iceberg/TestContentFileParser.java
index 60805c70c8..3f463f722e 100644
--- a/core/src/test/java/org/apache/iceberg/TestContentFileParser.java
+++ b/core/src/test/java/org/apache/iceberg/TestContentFileParser.java
@@ -72,7 +72,7 @@ public class TestContentFileParser {
assertThat(jsonStr).isEqualTo(expectedJson);
JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr);
ContentFile<?> deserializedContentFile =
- ContentFileParser.fromJson(jsonNode, Map.of(spec.specId(), spec));
+ ContentFileParser.fromJson(jsonNode, Map.of(TestBase.SPEC.specId(),
spec));
assertThat(deserializedContentFile).isInstanceOf(DataFile.class);
assertContentFileEquals(dataFile, deserializedContentFile, spec);
}
@@ -85,116 +85,11 @@ public class TestContentFileParser {
assertThat(jsonStr).isEqualTo(expectedJson);
JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr);
ContentFile<?> deserializedContentFile =
- ContentFileParser.fromJson(jsonNode, Map.of(spec.specId(), spec));
+ ContentFileParser.fromJson(jsonNode, Map.of(spec.specId(),
TestBase.SPEC));
assertThat(deserializedContentFile).isInstanceOf(DeleteFile.class);
assertContentFileEquals(deleteFile, deserializedContentFile, spec);
}
- @Test
- public void testPartitionJsonArrayWrongSize() throws Exception {
- PartitionSpec spec =
PartitionSpec.builderFor(TestBase.SCHEMA).identity("data").build();
- String jsonStr =
-
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":[],\"file-size-in-bytes\":10,"
- + "\"record-count\":1}";
-
- JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr);
-
- assertThatThrownBy(() -> ContentFileParser.fromJson(jsonNode,
Map.of(spec.specId(), spec)))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("Invalid partition data size");
- }
-
- @Test
- public void testPartitionJsonInvalidType() throws Exception {
- PartitionSpec spec =
PartitionSpec.builderFor(TestBase.SCHEMA).identity("data").build();
- String jsonStr =
-
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":\"invalid\",\"file-size-in-bytes\":10,"
- + "\"record-count\":1}";
-
- JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr);
-
- assertThatThrownBy(() -> ContentFileParser.fromJson(jsonNode,
Map.of(spec.specId(), spec)))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("expected array or object");
- }
-
- @Test
- public void testParsesFieldIdPartitionMap() throws Exception {
- PartitionSpec spec =
PartitionSpec.builderFor(TestBase.SCHEMA).identity("data").build();
- String legacyJson =
-
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":{\"1000\":\"foo\"},\"file-size-in-bytes\":10,"
- + "\"record-count\":1}";
-
- JsonNode jsonNode = JsonUtil.mapper().readTree(legacyJson);
- ContentFile<?> deserializedContentFile =
- ContentFileParser.fromJson(jsonNode, Map.of(spec.specId(), spec));
-
- assertThat(deserializedContentFile).isInstanceOf(DataFile.class);
- assertThat(deserializedContentFile.partition().get(0,
String.class)).isEqualTo("foo");
- }
-
- @Test
- public void testPartitionStructObjectContainsExtraField() throws Exception {
- PartitionSpec spec =
PartitionSpec.builderFor(TestBase.SCHEMA).identity("data").build();
- String jsonStr =
-
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":{\"1000\":\"foo\",\"9999\":\"bar\"},"
- + "\"file-size-in-bytes\":10,\"record-count\":1}";
-
- JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr);
- assertThatThrownBy(() -> ContentFileParser.fromJson(jsonNode,
Map.of(spec.specId(), spec)))
- .isInstanceOf(IllegalStateException.class)
- .hasMessageContaining("Invalid partition data size");
- }
-
- @Test
- public void testPartitionStructObjectEmptyIsNull() throws Exception {
- PartitionSpec spec =
PartitionSpec.builderFor(TestBase.SCHEMA).identity("data").build();
- String jsonStr =
-
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":{},\"file-size-in-bytes\":10,"
- + "\"record-count\":1}";
-
- JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr);
- ContentFile<?> contentFile = ContentFileParser.fromJson(jsonNode,
Map.of(spec.specId(), spec));
- assertThat(contentFile).isInstanceOf(DataFile.class);
- assertThat(contentFile.partition().get(0, String.class)).isNull();
- }
-
- @Test
- public void testPartitionArrayRespectsSpecOrder() throws Exception {
- PartitionSpec spec =
-
PartitionSpec.builderFor(TestBase.SCHEMA).identity("id").identity("data").build();
-
- PartitionData partitionData = new PartitionData(spec.partitionType());
- partitionData.set(0, 4);
- partitionData.set(1, "foo");
-
- DataFile dataFile =
- DataFiles.builder(spec)
- .withPath("/path/to/data.parquet")
- .withFileSizeInBytes(10)
- .withRecordCount(1)
- .withPartition(partitionData)
- .build();
-
- String jsonStr = ContentFileParser.toJson(dataFile, spec);
-
- // Verify partition values are serialized as array in correct order
- assertThat(jsonStr).contains("\"partition\":[4,\"foo\"]");
-
- JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr);
- ContentFile<?> deserializedContentFile =
- ContentFileParser.fromJson(jsonNode, Map.of(spec.specId(), spec));
-
- assertThat(deserializedContentFile).isInstanceOf(DataFile.class);
- assertThat(deserializedContentFile.partition().get(0,
Integer.class)).isEqualTo(4);
- assertThat(deserializedContentFile.partition().get(1,
String.class)).isEqualTo("foo");
- }
-
private static Stream<Arguments> provideSpecAndDataFile() {
return Stream.of(
Arguments.of(
@@ -233,17 +128,17 @@ public class TestContentFileParser {
private static String dataFileJsonWithRequiredOnly(PartitionSpec spec) {
if (spec.isUnpartitioned()) {
return
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\",\"file-format\":\"PARQUET\","
- +
"\"partition\":[],\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}";
+ +
"\"partition\":{},\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}";
} else {
return
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\",\"file-format\":\"PARQUET\","
- +
"\"partition\":[1],\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}";
+ +
"\"partition\":{\"1000\":1},\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}";
}
}
private static String dataFileJsonWithAllOptional(PartitionSpec spec) {
if (spec.isUnpartitioned()) {
return
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-with-stats.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":[],\"file-size-in-bytes\":350,\"record-count\":10,"
+ +
"\"file-format\":\"PARQUET\",\"partition\":{},\"file-size-in-bytes\":350,\"record-count\":10,"
+ "\"column-sizes\":{\"keys\":[3,4],\"values\":[100,200]},"
+ "\"value-counts\":{\"keys\":[3,4],\"values\":[90,180]},"
+ "\"null-value-counts\":{\"keys\":[3,4],\"values\":[10,20]},"
@@ -254,7 +149,7 @@ public class TestContentFileParser {
+ "\"split-offsets\":[128,256],\"sort-order-id\":1}";
} else {
return
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-with-stats.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":[1],\"file-size-in-bytes\":350,\"record-count\":10,"
+ +
"\"file-format\":\"PARQUET\",\"partition\":{\"1000\":1},\"file-size-in-bytes\":350,\"record-count\":10,"
+ "\"column-sizes\":{\"keys\":[3,4],\"values\":[100,200]},"
+ "\"value-counts\":{\"keys\":[3,4],\"values\":[90,180]},"
+ "\"null-value-counts\":{\"keys\":[3,4],\"values\":[10,20]},"
@@ -350,7 +245,7 @@ public class TestContentFileParser {
private static String deleteFileWithDataRefJson() {
return
"{\"spec-id\":0,\"content\":\"POSITION_DELETES\",\"file-path\":\"/path/to/delete.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":[4],\"file-size-in-bytes\":1234,"
+ +
"\"file-format\":\"PARQUET\",\"partition\":{\"1000\":4},\"file-size-in-bytes\":1234,"
+
"\"record-count\":10,\"referenced-data-file\":\"/path/to/data/file.parquet\"}";
}
@@ -376,7 +271,7 @@ public class TestContentFileParser {
private static String dvJson() {
return
"{\"spec-id\":0,\"content\":\"POSITION_DELETES\",\"file-path\":\"/path/to/delete.puffin\","
- +
"\"file-format\":\"PUFFIN\",\"partition\":[4],\"file-size-in-bytes\":1234,\"record-count\":10,"
+ +
"\"file-format\":\"PUFFIN\",\"partition\":{\"1000\":4},\"file-size-in-bytes\":1234,\"record-count\":10,"
+
"\"referenced-data-file\":\"/path/to/data/file.parquet\",\"content-offset\":4,\"content-size-in-bytes\":40}";
}
@@ -449,17 +344,17 @@ public class TestContentFileParser {
private static String deleteFileJsonWithRequiredOnly(PartitionSpec spec) {
if (spec.isUnpartitioned()) {
return
"{\"spec-id\":0,\"content\":\"POSITION_DELETES\",\"file-path\":\"/path/to/delete-a.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":[],\"file-size-in-bytes\":1234,\"record-count\":9}";
+ +
"\"file-format\":\"PARQUET\",\"partition\":{},\"file-size-in-bytes\":1234,\"record-count\":9}";
} else {
return
"{\"spec-id\":0,\"content\":\"POSITION_DELETES\",\"file-path\":\"/path/to/delete-a.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":[9],\"file-size-in-bytes\":1234,\"record-count\":9}";
+ +
"\"file-format\":\"PARQUET\",\"partition\":{\"1000\":9},\"file-size-in-bytes\":1234,\"record-count\":9}";
}
}
private static String deleteFileJsonWithAllOptional(PartitionSpec spec) {
if (spec.isUnpartitioned()) {
return
"{\"spec-id\":0,\"content\":\"EQUALITY_DELETES\",\"file-path\":\"/path/to/delete-with-stats.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":[],\"file-size-in-bytes\":1234,\"record-count\":10,"
+ +
"\"file-format\":\"PARQUET\",\"partition\":{},\"file-size-in-bytes\":1234,\"record-count\":10,"
+ "\"column-sizes\":{\"keys\":[3,4],\"values\":[100,200]},"
+ "\"value-counts\":{\"keys\":[3,4],\"values\":[90,180]},"
+ "\"null-value-counts\":{\"keys\":[3,4],\"values\":[10,20]},"
@@ -470,7 +365,7 @@ public class TestContentFileParser {
+
"\"split-offsets\":[128],\"equality-ids\":[3],\"sort-order-id\":1}";
} else {
return
"{\"spec-id\":0,\"content\":\"EQUALITY_DELETES\",\"file-path\":\"/path/to/delete-with-stats.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":[9],\"file-size-in-bytes\":1234,\"record-count\":10,"
+ +
"\"file-format\":\"PARQUET\",\"partition\":{\"1000\":9},\"file-size-in-bytes\":1234,\"record-count\":10,"
+ "\"column-sizes\":{\"keys\":[3,4],\"values\":[100,200]},"
+ "\"value-counts\":{\"keys\":[3,4],\"values\":[90,180]},"
+ "\"null-value-counts\":{\"keys\":[3,4],\"values\":[10,20]},"
diff --git a/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java
b/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java
index 758c11db78..8048b6d24e 100644
--- a/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java
+++ b/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java
@@ -146,7 +146,7 @@ public class TestDataTaskParser {
+ "\"value\":\"string\",\"value-required\":true}}]},"
+ "\"metadata-file\":{\"spec-id\":0,\"content\":\"DATA\","
+ "\"file-path\":\"/tmp/metadata2.json\","
- + "\"file-format\":\"METADATA\",\"partition\":[],"
+ + "\"file-format\":\"METADATA\",\"partition\":{},"
+
"\"file-size-in-bytes\":0,\"record-count\":2,\"sort-order-id\":0}"
+ "}";
JsonNode missingTableRowsNode =
mapper.reader().readTree(missingTableRowsStr);
@@ -164,23 +164,6 @@ public class TestDataTaskParser {
assertDataTaskEquals(dataTask, deserializedTask);
}
- @Test
- public void testDataTaskParsesFieldIdPartitionMap() {
- String jsonStr =
- "{\"task-type\":\"data-task\","
- + "\"schema\":{\"type\":\"struct\",\"schema-id\":0,"
- +
"\"fields\":[{\"id\":1,\"name\":\"committed_at\",\"required\":true,\"type\":\"timestamptz\"}]},"
- + "\"projection\":{\"type\":\"struct\",\"schema-id\":0,"
- +
"\"fields\":[{\"id\":1,\"name\":\"committed_at\",\"required\":true,\"type\":\"timestamptz\"}]},"
- +
"\"metadata-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/tmp/metadata.json\","
- +
"\"file-format\":\"METADATA\",\"partition\":{},\"file-size-in-bytes\":0,\"record-count\":1,\"sort-order-id\":0},"
- + "\"rows\":[{\"1\":\"2009-02-13T23:31:30+00:00\"}]}";
-
- StaticDataTask deserializedTask = (StaticDataTask)
ScanTaskParser.fromJson(jsonStr, true);
-
-
assertThat(deserializedTask.metadataFile().partition().size()).isEqualTo(0);
- }
-
private DataTask createDataTask() {
Map<String, String> summary1 =
ImmutableMap.of(
@@ -265,7 +248,7 @@ public class TestDataTaskParser {
+ "\"value\":\"string\",\"value-required\":true}}]},"
+ "\"metadata-file\":{\"spec-id\":0,\"content\":\"DATA\","
+ "\"file-path\":\"/tmp/metadata2.json\","
- + "\"file-format\":\"METADATA\",\"partition\":[],"
+ + "\"file-format\":\"METADATA\",\"partition\":{},"
+ "\"file-size-in-bytes\":0,\"record-count\":2,\"sort-order-id\":0},"
+
"\"rows\":[{\"1\":\"2009-02-13T23:31:30+00:00\",\"2\":1,\"4\":\"append\","
+ "\"5\":\"file:/tmp/manifest1.avro\","
diff --git a/core/src/test/java/org/apache/iceberg/TestFileScanTaskParser.java
b/core/src/test/java/org/apache/iceberg/TestFileScanTaskParser.java
index 882c2b3349..c4a9fdf234 100644
--- a/core/src/test/java/org/apache/iceberg/TestFileScanTaskParser.java
+++ b/core/src/test/java/org/apache/iceberg/TestFileScanTaskParser.java
@@ -62,18 +62,6 @@ public class TestFileScanTaskParser {
assertFileScanTaskEquals(fileScanTask, deserializedTask, spec,
caseSensitive);
}
- @Test
- public void testFileScanTaskParsesFieldIdPartitionMap() {
- boolean caseSensitive = true;
- PartitionSpec spec = TestBase.SPEC;
- FileScanTask expected = createFileScanTask(spec, caseSensitive);
-
- FileScanTask deserializedTask =
- ScanTaskParser.fromJson(fileScanTaskFieldIdPartitionMapJson(),
caseSensitive);
-
- assertFileScanTaskEquals(expected, deserializedTask, spec, caseSensitive);
- }
-
private FileScanTask createFileScanTask(PartitionSpec spec, boolean
caseSensitive) {
ResidualEvaluator residualEvaluator;
if (spec.isUnpartitioned()) {
@@ -97,39 +85,19 @@ public class TestFileScanTaskParser {
+ "\"spec\":{\"spec-id\":0,\"fields\":[{\"name\":\"data_bucket\","
+ "\"transform\":\"bucket[16]\",\"source-id\":4,\"field-id\":1000}]},"
+
"\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\","
- + "\"file-format\":\"PARQUET\",\"partition\":[0],"
+ + "\"file-format\":\"PARQUET\",\"partition\":{\"1000\":0},"
+ "\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0},"
+ "\"start\":0,\"length\":10,"
+ "\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\","
+
"\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\","
- + "\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1},"
+ +
"\"partition\":{\"1000\":0},\"file-size-in-bytes\":10,\"record-count\":1},"
+
"{\"spec-id\":0,\"content\":\"EQUALITY_DELETES\",\"file-path\":\"/path/to/data-a2-deletes.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":[0],\"file-size-in-bytes\":10,"
+ +
"\"file-format\":\"PARQUET\",\"partition\":{\"1000\":0},\"file-size-in-bytes\":10,"
+ "\"record-count\":1,\"equality-ids\":[1],\"sort-order-id\":0}],"
+ "\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}";
}
private String fileScanTaskJson() {
- return "{\"task-type\":\"file-scan-task\","
- + "\"schema\":{\"type\":\"struct\",\"schema-id\":0,\"fields\":["
- + "{\"id\":3,\"name\":\"id\",\"required\":true,\"type\":\"int\"},"
- +
"{\"id\":4,\"name\":\"data\",\"required\":true,\"type\":\"string\"}]},"
- + "\"spec\":{\"spec-id\":0,\"fields\":[{\"name\":\"data_bucket\","
- + "\"transform\":\"bucket[16]\",\"source-id\":4,\"field-id\":1000}]},"
- +
"\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\","
- + "\"file-format\":\"PARQUET\",\"partition\":[0],"
- + "\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0},"
- + "\"start\":0,\"length\":10,"
- + "\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\","
- +
"\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\","
- + "\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1},"
- +
"{\"spec-id\":0,\"content\":\"EQUALITY_DELETES\",\"file-path\":\"/path/to/data-a2-deletes.parquet\","
- +
"\"file-format\":\"PARQUET\",\"partition\":[0],\"file-size-in-bytes\":10,"
- + "\"record-count\":1,\"equality-ids\":[1],\"sort-order-id\":0}],"
- + "\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}";
- }
-
- private String fileScanTaskFieldIdPartitionMapJson() {
return "{\"task-type\":\"file-scan-task\","
+ "\"schema\":{\"type\":\"struct\",\"schema-id\":0,\"fields\":["
+ "{\"id\":3,\"name\":\"id\",\"required\":true,\"type\":\"int\"},"
diff --git
a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java
b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java
index 8b82e9794d..9db9e29530 100644
---
a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java
+++
b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java
@@ -157,7 +157,7 @@ public class TestFetchPlanningResultResponseParser {
"{\"status\":\"submitted\","
+
"\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\","
+
"\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\","
- +
"\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}]"
+ +
"\"partition\":{\"1000\":0},\"file-size-in-bytes\":10,\"record-count\":1}]"
+ "}";
assertThatThrownBy(
@@ -195,10 +195,10 @@ public class TestFetchPlanningResultResponseParser {
"{\"status\":\"completed\","
+
"\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\","
+
"\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\","
- +
"\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}],"
+ +
"\"partition\":{\"1000\":0},\"file-size-in-bytes\":10,\"record-count\":1}],"
+ "\"file-scan-tasks\":["
+
"{\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\","
- + "\"file-format\":\"PARQUET\",\"partition\":[0],"
+ + "\"file-format\":\"PARQUET\",\"partition\":{\"1000\":0},"
+
"\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0},"
+ "\"delete-file-references\":[0],"
+
"\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}]"
diff --git
a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java
b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java
index 62de75e381..01ca8288fb 100644
---
a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java
+++
b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java
@@ -97,7 +97,7 @@ public class TestFetchScanTasksResponseParser {
"{\"plan-tasks\":[\"task1\",\"task2\"],"
+
"\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\","
+
"\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\","
- +
"\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}]"
+ +
"\"partition\":{\"1000\":0},\"file-size-in-bytes\":10,\"record-count\":1}]"
+ "}";
assertThatThrownBy(
@@ -131,10 +131,10 @@ public class TestFetchScanTasksResponseParser {
"{"
+
"\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\","
+
"\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\","
- +
"\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}],"
+ +
"\"partition\":{\"1000\":0},\"file-size-in-bytes\":10,\"record-count\":1}],"
+ "\"file-scan-tasks\":["
+
"{\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\","
- + "\"file-format\":\"PARQUET\",\"partition\":[0],"
+ + "\"file-format\":\"PARQUET\",\"partition\":{\"1000\":0},"
+
"\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0},"
+ "\"delete-file-references\":[0],"
+
"\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}]"
diff --git
a/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java
b/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java
index 1bb23dd623..48f8a5dcb6 100644
---
a/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java
+++
b/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java
@@ -208,7 +208,7 @@ public class TestPlanTableScanResponseParser {
+ "\"plan-id\":\"somePlanId\","
+
"\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\","
+
"\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\","
- +
"\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}]"
+ +
"\"partition\":{\"1000\":0},\"file-size-in-bytes\":10,\"record-count\":1}]"
+ "}";
assertThatThrownBy(
@@ -242,10 +242,10 @@ public class TestPlanTableScanResponseParser {
"{\"status\":\"completed\","
+
"\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\","
+
"\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\","
- +
"\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}],"
+ +
"\"partition\":{\"1000\":0},\"file-size-in-bytes\":10,\"record-count\":1}],"
+ "\"file-scan-tasks\":["
+
"{\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\","
- + "\"file-format\":\"PARQUET\",\"partition\":[0],"
+ + "\"file-format\":\"PARQUET\",\"partition\":{\"1000\":0},"
+
"\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0},"
+ "\"delete-file-references\":[0],"
+
"\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}]"
@@ -315,7 +315,9 @@ public class TestPlanTableScanResponseParser {
+ " \"content\" : \"POSITION_DELETES\",\n"
+ " \"file-path\" : \"/path/to/data-a-deletes.parquet\",\n"
+ " \"file-format\" : \"PARQUET\",\n"
- + " \"partition\" : [ 0 ],\n"
+ + " \"partition\" : {\n"
+ + " \"1000\" : 0\n"
+ + " },\n"
+ " \"file-size-in-bytes\" : 10,\n"
+ " \"record-count\" : 1\n"
+ " }, {\n"
@@ -323,7 +325,9 @@ public class TestPlanTableScanResponseParser {
+ " \"content\" : \"POSITION_DELETES\",\n"
+ " \"file-path\" : \"/path/to/data-b-deletes.parquet\",\n"
+ " \"file-format\" : \"PARQUET\",\n"
- + " \"partition\" : [ 1 ],\n"
+ + " \"partition\" : {\n"
+ + " \"1000\" : 1\n"
+ + " },\n"
+ " \"file-size-in-bytes\" : 10,\n"
+ " \"record-count\" : 1\n"
+ " }, {\n"
@@ -331,7 +335,9 @@ public class TestPlanTableScanResponseParser {
+ " \"content\" : \"EQUALITY_DELETES\",\n"
+ " \"file-path\" : \"/path/to/data-c-deletes.parquet\",\n"
+ " \"file-format\" : \"PARQUET\",\n"
- + " \"partition\" : [ 2 ],\n"
+ + " \"partition\" : {\n"
+ + " \"1000\" : 2\n"
+ + " },\n"
+ " \"file-size-in-bytes\" : 10,\n"
+ " \"record-count\" : 1,\n"
+ " \"equality-ids\" : [ 1 ],\n"
@@ -343,7 +349,9 @@ public class TestPlanTableScanResponseParser {
+ " \"content\" : \"DATA\",\n"
+ " \"file-path\" : \"/path/to/data-a.parquet\",\n"
+ " \"file-format\" : \"PARQUET\",\n"
- + " \"partition\" : [ 0 ],\n"
+ + " \"partition\" : {\n"
+ + " \"1000\" : 0\n"
+ + " },\n"
+ " \"file-size-in-bytes\" : 10,\n"
+ " \"record-count\" : 1,\n"
+ " \"sort-order-id\" : 0\n"
@@ -356,7 +364,9 @@ public class TestPlanTableScanResponseParser {
+ " \"content\" : \"DATA\",\n"
+ " \"file-path\" : \"/path/to/data-b.parquet\",\n"
+ " \"file-format\" : \"PARQUET\",\n"
- + " \"partition\" : [ 1 ],\n"
+ + " \"partition\" : {\n"
+ + " \"1000\" : 1\n"
+ + " },\n"
+ " \"file-size-in-bytes\" : 10,\n"
+ " \"record-count\" : 1,\n"
+ " \"split-offsets\" : [ 1 ],\n"
@@ -370,7 +380,9 @@ public class TestPlanTableScanResponseParser {
+ " \"content\" : \"DATA\",\n"
+ " \"file-path\" : \"/path/to/data-c.parquet\",\n"
+ " \"file-format\" : \"PARQUET\",\n"
- + " \"partition\" : [ 2 ],\n"
+ + " \"partition\" : {\n"
+ + " \"1000\" : 2\n"
+ + " },\n"
+ " \"file-size-in-bytes\" : 10,\n"
+ " \"record-count\" : 1,\n"
+ " \"split-offsets\" : [ 2, 8 ],\n"
@@ -406,7 +418,7 @@ public class TestPlanTableScanResponseParser {
"{\"status\":\"completed\","
+ "\"file-scan-tasks\":["
+
"{\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\","
- + "\"file-format\":\"PARQUET\",\"partition\":[0],"
+ + "\"file-format\":\"PARQUET\",\"partition\":{\"1000\":0},"
+
"\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0},"
+
"\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}]"
+ "}";
