This is an automated email from the ASF dual-hosted git repository.
aokolnychyi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new ccaeb2f4d5 Core: Disallow setting equality field IDs for data (#8970)
ccaeb2f4d5 is described below
commit ccaeb2f4d5be44b4ab177b8fb790194eff8070a5
Author: Anton Okolnychyi <[email protected]>
AuthorDate: Thu Nov 16 08:33:02 2023 -0800
Core: Disallow setting equality field IDs for data (#8970)
---
core/src/main/java/org/apache/iceberg/ContentFileParser.java | 1 -
core/src/main/java/org/apache/iceberg/DataFiles.java | 11 +++--------
core/src/main/java/org/apache/iceberg/GenericDataFile.java | 3 +--
.../test/java/org/apache/iceberg/TestContentFileParser.java | 6 ++----
.../java/org/apache/iceberg/TestManifestWriterVersions.java | 2 +-
5 files changed, 7 insertions(+), 16 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/ContentFileParser.java
b/core/src/main/java/org/apache/iceberg/ContentFileParser.java
index b3edf2927f..dd08c5c69e 100644
--- a/core/src/main/java/org/apache/iceberg/ContentFileParser.java
+++ b/core/src/main/java/org/apache/iceberg/ContentFileParser.java
@@ -156,7 +156,6 @@ class ContentFileParser {
metrics,
keyMetadata,
splitOffsets,
- equalityFieldIds,
sortOrderId);
} else {
return new GenericDeleteFile(
diff --git a/core/src/main/java/org/apache/iceberg/DataFiles.java
b/core/src/main/java/org/apache/iceberg/DataFiles.java
index a6ea80d366..34f830be4d 100644
--- a/core/src/main/java/org/apache/iceberg/DataFiles.java
+++ b/core/src/main/java/org/apache/iceberg/DataFiles.java
@@ -29,7 +29,6 @@ import org.apache.iceberg.io.InputFile;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.types.Conversions;
-import org.apache.iceberg.util.ArrayUtil;
import org.apache.iceberg.util.ByteBuffers;
public class DataFiles {
@@ -154,7 +153,6 @@ public class DataFiles {
private Map<Integer, ByteBuffer> upperBounds = null;
private ByteBuffer keyMetadata = null;
private List<Long> splitOffsets = null;
- private List<Integer> equalityFieldIds = null;
private Integer sortOrderId = SortOrder.unsorted().orderId();
public Builder(PartitionSpec spec) {
@@ -301,12 +299,10 @@ public class DataFiles {
return this;
}
+ /** @deprecated since 1.5.0, will be removed in 1.6.0; must not be set for
data files. */
+ @Deprecated
public Builder withEqualityFieldIds(List<Integer> equalityIds) {
- if (equalityIds != null) {
- this.equalityFieldIds = ImmutableList.copyOf(equalityIds);
- }
-
- return this;
+ throw new UnsupportedOperationException("Equality field IDs must not be
set for data files");
}
public Builder withEncryptionKeyMetadata(ByteBuffer newKeyMetadata) {
@@ -350,7 +346,6 @@ public class DataFiles {
upperBounds),
keyMetadata,
splitOffsets,
- ArrayUtil.toIntArray(equalityFieldIds),
sortOrderId);
}
}
diff --git a/core/src/main/java/org/apache/iceberg/GenericDataFile.java
b/core/src/main/java/org/apache/iceberg/GenericDataFile.java
index f78bb2ce9f..8fe7ec756a 100644
--- a/core/src/main/java/org/apache/iceberg/GenericDataFile.java
+++ b/core/src/main/java/org/apache/iceberg/GenericDataFile.java
@@ -41,7 +41,6 @@ class GenericDataFile extends BaseFile<DataFile> implements
DataFile {
Metrics metrics,
ByteBuffer keyMetadata,
List<Long> splitOffsets,
- int[] equalityFieldIds,
Integer sortOrderId) {
super(
specId,
@@ -58,7 +57,7 @@ class GenericDataFile extends BaseFile<DataFile> implements
DataFile {
metrics.lowerBounds(),
metrics.upperBounds(),
splitOffsets,
- equalityFieldIds,
+ null /* no equality field IDs */,
sortOrderId,
keyMetadata);
}
diff --git a/core/src/test/java/org/apache/iceberg/TestContentFileParser.java
b/core/src/test/java/org/apache/iceberg/TestContentFileParser.java
index 9360f571c5..f3b5e7b60c 100644
--- a/core/src/test/java/org/apache/iceberg/TestContentFileParser.java
+++ b/core/src/test/java/org/apache/iceberg/TestContentFileParser.java
@@ -21,7 +21,6 @@ package org.apache.iceberg;
import com.fasterxml.jackson.databind.JsonNode;
import java.nio.ByteBuffer;
import java.util.Arrays;
-import java.util.Collections;
import java.util.stream.Stream;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.types.Comparators;
@@ -141,7 +140,7 @@ public class TestContentFileParser {
+
"\"lower-bounds\":{\"keys\":[3,4],\"values\":[\"01000000\",\"02000000\"]},"
+
"\"upper-bounds\":{\"keys\":[3,4],\"values\":[\"05000000\",\"0A000000\"]},"
+ "\"key-metadata\":\"00000000000000000000000000000000\","
- +
"\"split-offsets\":[128,256],\"equality-ids\":[1],\"sort-order-id\":1}";
+ + "\"split-offsets\":[128,256],\"sort-order-id\":1}";
} else {
return
"{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-with-stats.parquet\","
+
"\"file-format\":\"PARQUET\",\"partition\":{\"1000\":1},\"file-size-in-bytes\":350,\"record-count\":10,"
@@ -152,7 +151,7 @@ public class TestContentFileParser {
+
"\"lower-bounds\":{\"keys\":[3,4],\"values\":[\"01000000\",\"02000000\"]},"
+
"\"upper-bounds\":{\"keys\":[3,4],\"values\":[\"05000000\",\"0A000000\"]},"
+ "\"key-metadata\":\"00000000000000000000000000000000\","
- +
"\"split-offsets\":[128,256],\"equality-ids\":[1],\"sort-order-id\":1}";
+ + "\"split-offsets\":[128,256],\"sort-order-id\":1}";
}
}
@@ -180,7 +179,6 @@ public class TestContentFileParser {
))
.withFileSizeInBytes(350)
.withSplitOffsets(Arrays.asList(128L, 256L))
- .withEqualityFieldIds(Collections.singletonList(1))
.withEncryptionKeyMetadata(ByteBuffer.wrap(new byte[16]))
.withSortOrder(
SortOrder.builderFor(TableTestBase.SCHEMA)
diff --git
a/core/src/test/java/org/apache/iceberg/TestManifestWriterVersions.java
b/core/src/test/java/org/apache/iceberg/TestManifestWriterVersions.java
index 08b27d7460..740791b255 100644
--- a/core/src/test/java/org/apache/iceberg/TestManifestWriterVersions.java
+++ b/core/src/test/java/org/apache/iceberg/TestManifestWriterVersions.java
@@ -78,7 +78,7 @@ public class TestManifestWriterVersions {
private static final DataFile DATA_FILE =
new GenericDataFile(
- 0, PATH, FORMAT, PARTITION, 150972L, METRICS, null, OFFSETS, null,
SORT_ORDER_ID);
+ 0, PATH, FORMAT, PARTITION, 150972L, METRICS, null, OFFSETS,
SORT_ORDER_ID);
private static final List<Integer> EQUALITY_IDS = ImmutableList.of(1);
private static final int[] EQUALITY_ID_ARR = new int[] {1};
