This is an automated email from the ASF dual-hosted git repository.
aokolnychyi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 62a23a3775 Core: Fix null partitions in PartitionSet (#9248)
62a23a3775 is described below
commit 62a23a37758792b31e2590bd7d1aa46da1074512
Author: Anton Okolnychyi <[email protected]>
AuthorDate: Fri Dec 8 12:52:29 2023 -0800
Core: Fix null partitions in PartitionSet (#9248)
---
.../java/org/apache/iceberg/util/PartitionSet.java | 4 +-
.../org/apache/iceberg/util/TestPartitionMap.java | 2 +
.../org/apache/iceberg/util/TestPartitionSet.java | 83 ++++++++++++++++++++++
3 files changed, 87 insertions(+), 2 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/util/PartitionSet.java
b/core/src/main/java/org/apache/iceberg/util/PartitionSet.java
index 893c320707..eff37fa5a9 100644
--- a/core/src/main/java/org/apache/iceberg/util/PartitionSet.java
+++ b/core/src/main/java/org/apache/iceberg/util/PartitionSet.java
@@ -63,7 +63,7 @@ public class PartitionSet extends AbstractSet<Pair<Integer,
StructLike>> {
if (o instanceof Pair) {
Object first = ((Pair<?, ?>) o).first();
Object second = ((Pair<?, ?>) o).second();
- if (first instanceof Integer && second instanceof StructLike) {
+ if (first instanceof Integer && (second == null || second instanceof
StructLike)) {
return contains((Integer) first, (StructLike) second);
}
}
@@ -98,7 +98,7 @@ public class PartitionSet extends AbstractSet<Pair<Integer,
StructLike>> {
if (o instanceof Pair) {
Object first = ((Pair<?, ?>) o).first();
Object second = ((Pair<?, ?>) o).second();
- if (first instanceof Integer && second instanceof StructLike) {
+ if (first instanceof Integer && (second == null || second instanceof
StructLike)) {
return remove((Integer) first, (StructLike) second);
}
}
diff --git a/core/src/test/java/org/apache/iceberg/util/TestPartitionMap.java
b/core/src/test/java/org/apache/iceberg/util/TestPartitionMap.java
index e528a1e70e..268f7eada8 100644
--- a/core/src/test/java/org/apache/iceberg/util/TestPartitionMap.java
+++ b/core/src/test/java/org/apache/iceberg/util/TestPartitionMap.java
@@ -274,9 +274,11 @@ public class TestPartitionMap {
map1.put(BY_DATA_SPEC.specId(), Row.of("aaa"), "v1");
map1.put(BY_DATA_SPEC.specId(), Row.of("bbb"), "v2");
+ map1.put(UNPARTITIONED_SPEC.specId(), null, "v3");
map2.put(BY_DATA_SPEC.specId(), CustomRow.of("aaa"), "v1");
map2.put(BY_DATA_SPEC.specId(), CustomRow.of("bbb"), "v2");
+ map2.put(UNPARTITIONED_SPEC.specId(), null, "v3");
assertThat(map1.keySet()).isEqualTo(map2.keySet());
assertThat(map1.entrySet()).isEqualTo(map2.entrySet());
diff --git a/core/src/test/java/org/apache/iceberg/util/TestPartitionSet.java
b/core/src/test/java/org/apache/iceberg/util/TestPartitionSet.java
new file mode 100644
index 0000000000..533e590423
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/util/TestPartitionSet.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.util;
+
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.Map;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.TestHelpers.CustomRow;
+import org.apache.iceberg.TestHelpers.Row;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.Test;
+
+public class TestPartitionSet {
+ private static final Schema SCHEMA =
+ new Schema(
+ required(1, "id", Types.IntegerType.get()),
+ required(2, "data", Types.StringType.get()),
+ required(3, "category", Types.StringType.get()));
+ private static final PartitionSpec UNPARTITIONED_SPEC =
PartitionSpec.unpartitioned();
+ private static final PartitionSpec BY_DATA_SPEC =
+ PartitionSpec.builderFor(SCHEMA).identity("data").withSpecId(1).build();
+ private static final PartitionSpec BY_DATA_CATEGORY_BUCKET_SPEC =
+ PartitionSpec.builderFor(SCHEMA).identity("data").bucket("category",
8).withSpecId(3).build();
+ private static final Map<Integer, PartitionSpec> SPECS =
+ ImmutableMap.of(
+ UNPARTITIONED_SPEC.specId(),
+ UNPARTITIONED_SPEC,
+ BY_DATA_SPEC.specId(),
+ BY_DATA_SPEC,
+ BY_DATA_CATEGORY_BUCKET_SPEC.specId(),
+ BY_DATA_CATEGORY_BUCKET_SPEC);
+
+ @Test
+ public void testGet() {
+ PartitionSet set = PartitionSet.create(SPECS);
+ set.add(BY_DATA_SPEC.specId(), Row.of("a"));
+ set.add(UNPARTITIONED_SPEC.specId(), null);
+ set.add(UNPARTITIONED_SPEC.specId(), Row.of());
+ set.add(BY_DATA_CATEGORY_BUCKET_SPEC.specId(), CustomRow.of("a", 1));
+
+ assertThat(set).hasSize(4);
+ assertThat(set.contains(BY_DATA_SPEC.specId(),
CustomRow.of("a"))).isTrue();
+ assertThat(set.contains(UNPARTITIONED_SPEC.specId(), null)).isTrue();
+ assertThat(set.contains(UNPARTITIONED_SPEC.specId(),
CustomRow.of())).isTrue();
+ assertThat(set.contains(BY_DATA_CATEGORY_BUCKET_SPEC.specId(), Row.of("a",
1))).isTrue();
+ }
+
+ @Test
+ public void testRemove() {
+ PartitionSet set = PartitionSet.create(SPECS);
+ set.add(BY_DATA_SPEC.specId(), Row.of("a"));
+ set.add(UNPARTITIONED_SPEC.specId(), null);
+ set.add(UNPARTITIONED_SPEC.specId(), Row.of());
+ set.add(BY_DATA_CATEGORY_BUCKET_SPEC.specId(), CustomRow.of("a", 1));
+
+ assertThat(set).hasSize(4);
+ assertThat(set.remove(BY_DATA_SPEC.specId(), CustomRow.of("a"))).isTrue();
+ assertThat(set.remove(UNPARTITIONED_SPEC.specId(), null)).isTrue();
+ assertThat(set.remove(UNPARTITIONED_SPEC.specId(),
CustomRow.of())).isTrue();
+ assertThat(set.remove(BY_DATA_CATEGORY_BUCKET_SPEC.specId(), Row.of("a",
1))).isTrue();
+ assertThat(set).isEmpty();
+ }
+}
