This is an automated email from the ASF dual-hosted git repository.
amogh-jahagirdar pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 574e44670b Core: Return all metrics field IDs, in schema order (#18331)
574e44670b is described below
commit 574e44670b8502479b5c74d0edf0b9190d235af4
Author: Ryan Blue <[email protected]>
AuthorDate: Thu Oct 1 10:01:01 2026 -0700
Core: Return all metrics field IDs, in schema order (#18331)
---
.../java/org/apache/iceberg/MetricsConfig.java | 31 +--
.../main/java/org/apache/iceberg/SchemaOrder.java | 123 +++++++++++
.../org/apache/iceberg/util/SerializableMap.java | 8 +-
.../java/org/apache/iceberg/TestMetricsConfig.java | 238 +++++++++++++++++++++
.../java/org/apache/iceberg/TestMetricsModes.java | 233 +-------------------
.../java/org/apache/iceberg/TestSchemaOrder.java | 121 +++++++++++
.../java/org/apache/iceberg/TestStatsUtil.java | 16 +-
7 files changed, 512 insertions(+), 258 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/MetricsConfig.java
b/core/src/main/java/org/apache/iceberg/MetricsConfig.java
index eba0eb7cee..5bb5d9bcc7 100644
--- a/core/src/main/java/org/apache/iceberg/MetricsConfig.java
+++ b/core/src/main/java/org/apache/iceberg/MetricsConfig.java
@@ -77,9 +77,9 @@ public final class MetricsConfig implements Serializable {
Map<String, MetricsMode> columnModes,
MetricsMode defaultMode,
Map<Integer, String> idToName) {
- this.columnModes = SerializableMap.copyOf(columnModes).immutableMap();
+ this.columnModes = SerializableMap.copyOf(columnModes);
this.defaultMode = defaultMode;
- this.idToName = idToName != null ?
SerializableMap.copyOf(idToName).immutableMap() : null;
+ this.idToName = idToName != null ? SerializableMap.copyOf(idToName) : null;
}
public static MetricsConfig getDefault() {
@@ -266,7 +266,11 @@ public final class MetricsConfig implements Serializable {
// Handle configured default mode
MetricsMode configuredDefault = configuredDefault(props);
- Map<String, MetricsMode> defaultColumnConf = defaultColumnModes(schema,
maxDefaultColumns);
+ Map<String, MetricsMode> defaultColumnConf =
+ defaultColumnModes(
+ schema,
+ configuredDefault != null ? configuredDefault : DEFAULT_MODE,
+ maxDefaultColumns);
MetricsMode defaultMode;
if (configuredDefault != null) {
@@ -281,9 +285,7 @@ public final class MetricsConfig implements Serializable {
Map<String, MetricsMode> columnModes = Maps.newHashMap();
- if (configuredDefault == null) {
- columnModes.putAll(defaultColumnConf);
- }
+ columnModes.putAll(defaultColumnConf);
// Default sort columns to at least truncate (overridden by config)
columnModes.putAll(sortColumnModes(order, configuredDefault));
@@ -309,11 +311,12 @@ public final class MetricsConfig implements Serializable {
return null;
}
- private static Map<String, MetricsMode> defaultColumnModes(Schema schema,
int maxColumns) {
+ private static Map<String, MetricsMode> defaultColumnModes(
+ Schema schema, MetricsMode defaultMode, int maxColumns) {
ImmutableMap.Builder<String, MetricsMode> builder = ImmutableMap.builder();
if (schema != null) {
for (int id : limitFieldIds(schema, maxColumns)) {
- builder.put(schema.findColumnName(id), DEFAULT_MODE);
+ builder.put(schema.findColumnName(id), defaultMode);
}
}
@@ -367,15 +370,15 @@ public final class MetricsConfig implements Serializable {
private static Map<Integer, String> idToName(
Schema schema, Map<String, MetricsMode> columnModes) {
if (schema != null) {
- ImmutableMap.Builder<Integer, String> builder = ImmutableMap.builder();
- for (String name : columnModes.keySet()) {
- Types.NestedField field = schema.findField(name);
- if (field != null) {
- builder.put(field.fieldId(), name);
+ Map<Integer, String> idToName = Maps.newLinkedHashMap();
+ for (int fieldId : SchemaOrder.allFieldIds(schema)) {
+ String name = schema.findColumnName(fieldId);
+ if (columnModes.containsKey(name)) {
+ idToName.put(fieldId, name);
}
}
- return builder.buildKeepingLast();
+ return idToName;
}
return null;
diff --git a/core/src/main/java/org/apache/iceberg/SchemaOrder.java
b/core/src/main/java/org/apache/iceberg/SchemaOrder.java
new file mode 100644
index 0000000000..4f5559fbae
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/SchemaOrder.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.util.List;
+import java.util.function.Supplier;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.TypeUtil;
+import org.apache.iceberg.types.Types;
+
+/**
+ * Utility methods for working with fields in schema definition order.
+ *
+ * <p>Schema order is a pre-order traversal of fields in the schema. This
produces fields in the
+ * same left-to-right order as a type definition. For example, the schema
<code>
+ * id bigint, data string, point struct<x float, y float></code>
produces fields in schema
+ * order: [ id, data, point, point.x, point.y ]
+ */
+class SchemaOrder {
+ private SchemaOrder() {}
+
+ /**
+ * Produces a list of all field IDs in schema order.
+ *
+ * @param schema a schema
+ * @return all field IDs in schema order
+ */
+ public static List<Integer> allFieldIds(Schema schema) {
+ return TypeUtil.visit(schema, new IdVisitor(true));
+ }
+
+ /**
+ * Produces a list of leaf field IDs in schema order.
+ *
+ * @param schema a schema
+ * @return leaf field IDs in schema order
+ */
+ public static List<Integer> leafFieldIds(Schema schema) {
+ return TypeUtil.visit(schema, new IdVisitor(false));
+ }
+
+ private static class IdVisitor extends
TypeUtil.CustomOrderSchemaVisitor<List<Integer>> {
+ private final List<Integer> orderedIds = Lists.newArrayList();
+ private final boolean includeNonLeafIds;
+
+ private IdVisitor(boolean includeNonLeafIds) {
+ this.includeNonLeafIds = includeNonLeafIds;
+ }
+
+ @Override
+ @SuppressWarnings("ReturnValueIgnored")
+ public List<Integer> schema(Schema schema, Supplier<List<Integer>>
structResult) {
+ structResult.get(); // visit the top-level struct
+ return orderedIds;
+ }
+
+ @Override
+ public List<Integer> struct(Types.StructType struct,
Iterable<List<Integer>> fieldResults) {
+ for (List<Integer> ignored : fieldResults) {
+ // no handling is required; iterating visits the struct's fields
+ }
+
+ return null;
+ }
+
+ @Override
+ @SuppressWarnings("ReturnValueIgnored")
+ public List<Integer> field(Types.NestedField field,
Supplier<List<Integer>> fieldResult) {
+ if (field.type().isNestedType()) {
+ if (includeNonLeafIds) {
+ orderedIds.add(field.fieldId());
+ }
+
+ fieldResult.get(); // visit the field
+ } else {
+ orderedIds.add(field.fieldId());
+ }
+
+ return null;
+ }
+
+ @Override
+ public List<Integer> list(Types.ListType list, Supplier<List<Integer>>
elementResult) {
+ field(list.fields().get(0), elementResult);
+ return null;
+ }
+
+ @Override
+ public List<Integer> map(
+ Types.MapType map, Supplier<List<Integer>> keyResult,
Supplier<List<Integer>> valueResult) {
+ field(map.fields().get(0), keyResult);
+ field(map.fields().get(1), valueResult);
+ return null;
+ }
+
+ @Override
+ public List<Integer> variant(Types.VariantType variant) {
+ return null;
+ }
+
+ @Override
+ public List<Integer> primitive(Type.PrimitiveType primitive) {
+ return null;
+ }
+ }
+}
diff --git a/core/src/main/java/org/apache/iceberg/util/SerializableMap.java
b/core/src/main/java/org/apache/iceberg/util/SerializableMap.java
index 908829daea..a5a7ac6344 100644
--- a/core/src/main/java/org/apache/iceberg/util/SerializableMap.java
+++ b/core/src/main/java/org/apache/iceberg/util/SerializableMap.java
@@ -21,6 +21,7 @@ package org.apache.iceberg.util;
import java.io.Serializable;
import java.util.Collection;
import java.util.Collections;
+import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
@@ -36,12 +37,15 @@ public class SerializableMap<K, V> implements Map<K, V>,
Serializable {
}
private SerializableMap(Map<K, V> map) {
- this.copiedMap = Maps.newHashMap();
+ this.copiedMap = map instanceof LinkedHashMap ? Maps.newLinkedHashMap() :
Maps.newHashMap();
this.copiedMap.putAll(map);
}
private SerializableMap(Map<K, V> map, Set<K> keys) {
- Map<K, V> filteredMap = Maps.newHashMapWithExpectedSize(keys.size());
+ Map<K, V> filteredMap =
+ map instanceof LinkedHashMap
+ ? Maps.newLinkedHashMapWithExpectedSize(keys.size())
+ : Maps.newHashMapWithExpectedSize(keys.size());
for (K key : keys) {
if (map.containsKey(key)) {
diff --git a/core/src/test/java/org/apache/iceberg/TestMetricsConfig.java
b/core/src/test/java/org/apache/iceberg/TestMetricsConfig.java
index 7f85326f0c..f046f08ad9 100644
--- a/core/src/test/java/org/apache/iceberg/TestMetricsConfig.java
+++ b/core/src/test/java/org/apache/iceberg/TestMetricsConfig.java
@@ -24,6 +24,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.types.Types;
import org.junit.jupiter.api.Test;
@@ -42,6 +43,181 @@ public class TestMetricsConfig {
optional(CATEGORY, "category", Types.StringType.get()),
optional(DATA, "data", Types.StringType.get()));
+ @Test
+ public void testInvalidColumnModeValue() {
+ Map<String, String> properties =
+ ImmutableMap.of(
+ TableProperties.DEFAULT_WRITE_METRICS_MODE,
+ "full",
+ TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col",
+ "troncate(5)");
+
+ Schema schema = new Schema(required(1, "col", Types.StringType.get()));
+
+ MetricsConfig config = MetricsTestUtil.from(properties, schema);
+ assertThat(config.columnMode(1))
+ .as("Invalid mode should be defaulted to table default (full)")
+ .isEqualTo(MetricsModes.Full.get());
+
+ assertThat(config.metricsFieldIds()).containsExactly(1);
+ }
+
+ @Test
+ public void testInvalidDefaultColumnModeValue() {
+ Map<String, String> properties =
+ ImmutableMap.of(
+ TableProperties.DEFAULT_WRITE_METRICS_MODE,
+ "fuull",
+ TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col",
+ "troncate(5)");
+
+ Schema schema = new Schema(required(1, "col", Types.StringType.get()));
+
+ MetricsConfig config = MetricsTestUtil.from(properties, schema);
+ assertThat(config.columnMode(1))
+ .as("Invalid mode should be defaulted to library default
(truncate(16))")
+ .isEqualTo(MetricsModes.Truncate.withLength(16));
+
+ assertThat(config.metricsFieldIds()).containsExactly(1);
+ }
+
+ @Test
+ public void testMetricsConfigSortedColsDefault() {
+ Schema schema =
+ new Schema(
+ required(1, "col1", Types.IntegerType.get()),
+ required(2, "col2", Types.IntegerType.get()),
+ required(3, "col3", Types.IntegerType.get()),
+ required(4, "col4", Types.IntegerType.get()));
+ SortOrder sortOrder =
SortOrder.builderFor(schema).asc("col2").asc("col3").build();
+ Map<String, String> properties =
+ Map.of(
+ TableProperties.DEFAULT_WRITE_METRICS_MODE,
+ "counts",
+ TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col1",
+ "counts",
+ TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col2",
+ "none");
+
+ MetricsConfig config = MetricsTestUtil.from(properties, schema, sortOrder);
+ assertThat(config.columnMode(1))
+ .as("Non-sorted existing column should not be overridden")
+ .isEqualTo(MetricsModes.Counts.get());
+ assertThat(config.columnMode(2))
+ .as("Sorted column defaults should not override user specified config")
+ .isEqualTo(MetricsModes.None.get());
+ assertThat(config.columnMode(3))
+ .as("Unspecified sorted column should use default")
+ .isEqualTo(MetricsModes.Truncate.withLength(16));
+ assertThat(config.columnMode(4))
+ .as("Unspecified normal column should use default")
+ .isEqualTo(MetricsModes.Counts.get());
+
+ assertThat(config.metricsFieldIds()).containsExactly(1, 2, 3, 4);
+ }
+
+ @Test
+ public void testMetricsConfigSortedColsDefaultByInvalid() {
+ Schema schema =
+ new Schema(
+ required(1, "col1", Types.IntegerType.get()),
+ required(2, "col2", Types.IntegerType.get()),
+ required(3, "col3", Types.IntegerType.get()));
+ SortOrder sortOrder =
SortOrder.builderFor(schema).asc("col2").asc("col3").build();
+ Map<String, String> properties =
+ Map.of(
+ TableProperties.DEFAULT_WRITE_METRICS_MODE,
+ "counts",
+ TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col1",
+ "full",
+ TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col2",
+ "invalid");
+
+ MetricsConfig config = MetricsTestUtil.from(properties, schema, sortOrder);
+ assertThat(config.columnMode(1))
+ .as("Non-sorted existing column should not be overridden by sorted
column")
+ .isEqualTo(MetricsModes.Full.get());
+ assertThat(config.columnMode(2))
+ .as("Original default applies as user entered invalid mode for sorted
column")
+ .isEqualTo(MetricsModes.Truncate.withLength(16));
+
+ assertThat(config.metricsFieldIds()).containsExactly(1, 2, 3);
+ }
+
+ @Test
+ public void testMetricsConfigInferredDefaultModeLimit() {
+ Schema schema =
+ new Schema(
+ required(1, "col1", Types.IntegerType.get()),
+ required(2, "col2", Types.IntegerType.get()),
+ required(3, "col3", Types.IntegerType.get()));
+
+ // only infer a default for the first two columns
+ Map<String, String> properties =
+ Map.of(TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, "2");
+
+ MetricsConfig config = MetricsTestUtil.from(properties, schema);
+
+
assertThat(config.columnMode(1)).isEqualTo(MetricsModes.Truncate.withLength(16));
+
assertThat(config.columnMode(2)).isEqualTo(MetricsModes.Truncate.withLength(16));
+ assertThat(config.columnMode(3)).isEqualTo(MetricsModes.None.get());
+
+ assertThat(config.metricsFieldIds()).containsExactly(1, 2);
+ }
+
+ @Test
+ public void testMetricsVariantSupported() {
+ Schema schema =
+ new Schema(
+ required(1, "variant", Types.VariantType.get()),
+ required(2, "int", Types.IntegerType.get()));
+
+ // only infer a default for the first column
+ Map<String, String> properties =
+ Map.of(TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, "1");
+
+ MetricsConfig config = MetricsTestUtil.from(properties, schema);
+
+ Map<Integer, MetricsModes.MetricsMode> metricModes =
+ schema.idToName().keySet().stream().collect(Collectors.toMap(id -> id,
config::columnMode));
+
+ assertThat(metricModes)
+ .containsOnly(
+ Map.entry(1, MetricsModes.Truncate.withLength(16)),
+ Map.entry(2, MetricsModes.None.get()));
+
+ assertThat(config.metricsFieldIds()).containsExactly(1);
+ }
+
+ @Test
+ public void testMetricsConfigNestedTypesStructs() {
+ Schema schema =
+ new Schema(
+ required(
+ 5,
+ "col_struct",
+ Types.StructType.of(
+ required(33, "a", Types.IntegerType.get()),
+ required(1, "b", Types.IntegerType.get()))),
+ required(4, "top", Types.IntegerType.get()));
+
+ Map<String, String> properties =
+ Map.of(TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, "2");
+
+ MetricsConfig config = MetricsTestUtil.from(properties, schema);
+
+ Map<Integer, MetricsModes.MetricsMode> metricModes =
+ schema.idToName().keySet().stream().collect(Collectors.toMap(id -> id,
config::columnMode));
+
+ assertThat(metricModes).containsOnlyKeys(33, 5, 1, 4);
+
+ assertThat(metricModes).containsEntry(33,
MetricsModes.Truncate.withLength(16));
+ assertThat(metricModes).containsEntry(1, MetricsModes.None.get());
+ assertThat(metricModes).containsEntry(4,
MetricsModes.Truncate.withLength(16));
+
+ assertThat(config.metricsFieldIds()).containsExactly(33, 4);
+ }
+
@Test
void configCannotDisablePartitionSourceMetrics() {
PartitionSpec spec =
PartitionSpec.builderFor(SCHEMA).identity("category").build();
@@ -288,4 +464,66 @@ public class TestMetricsConfig {
.isEqualTo(MetricsModes.Truncate.withLength(16));
assertThat(wider.columnMode(4)).isEqualTo(MetricsModes.None.get());
}
+
+ @Test
+ public void testMetricsConfigKryoSerialization() throws Exception {
+ Map<String, String> metricsConfig =
+ ImmutableMap.of(
+ TableProperties.DEFAULT_WRITE_METRICS_MODE,
+ "counts",
+ TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col1",
+ "full",
+ TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col2",
+ "truncate(16)");
+
+ Schema schema =
+ new Schema(
+ Types.NestedField.required(1, "col1", Types.IntegerType.get()),
+ Types.NestedField.optional(2, "col2", Types.StringType.get()),
+ Types.NestedField.optional(3, "col3", Types.StringType.get()));
+
+ MetricsConfig config = MetricsTestUtil.from(metricsConfig, schema);
+ MetricsConfig deserialized =
TestHelpers.KryoHelpers.roundTripSerialize(config);
+
+
assertThat(deserialized.columnMode(1)).asString().isEqualTo(MetricsModes.Full.get().toString());
+ assertThat(deserialized.columnMode(2))
+ .asString()
+ .isEqualTo(MetricsModes.Truncate.withLength(16).toString());
+ assertThat(deserialized.columnMode(3))
+ .asString()
+ .isEqualTo(MetricsModes.Counts.get().toString());
+
+
assertThat(deserialized.metricsFieldIds()).containsExactlyElementsOf(config.metricsFieldIds());
+ }
+
+ @Test
+ public void testMetricsConfigJavaSerialization() throws Exception {
+ Map<String, String> metricsConfig =
+ ImmutableMap.of(
+ TableProperties.DEFAULT_WRITE_METRICS_MODE,
+ "counts",
+ TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col1",
+ "full",
+ TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col2",
+ "truncate(16)");
+
+ Schema schema =
+ new Schema(
+ Types.NestedField.required(1, "col1", Types.IntegerType.get()),
+ Types.NestedField.optional(2, "col2", Types.StringType.get()),
+ Types.NestedField.optional(3, "col3", Types.StringType.get()));
+
+ MetricsConfig config = MetricsTestUtil.from(metricsConfig, schema);
+ MetricsConfig deserialized = TestHelpers.roundTripSerialize(config);
+
+
assertThat(deserialized.columnMode(1)).asString().isEqualTo(MetricsModes.Full.get().toString());
+ assertThat(deserialized.columnMode(2))
+ .asString()
+ .isEqualTo(MetricsModes.Truncate.withLength(16).toString());
+ assertThat(deserialized.columnMode(3))
+ .asString()
+ .isEqualTo(MetricsModes.Counts.get().toString());
+
+
assertThat(deserialized.metricsFieldIds()).containsExactlyElementsOf(config.metricsFieldIds());
+ }
}
diff --git a/core/src/test/java/org/apache/iceberg/TestMetricsModes.java
b/core/src/test/java/org/apache/iceberg/TestMetricsModes.java
index 6c873ec21b..20125f0e19 100644
--- a/core/src/test/java/org/apache/iceberg/TestMetricsModes.java
+++ b/core/src/test/java/org/apache/iceberg/TestMetricsModes.java
@@ -18,48 +18,17 @@
*/
package org.apache.iceberg;
-import static org.apache.iceberg.TestHelpers.ALL_VERSIONS;
-import static org.apache.iceberg.types.Types.NestedField.required;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.assertj.core.api.Assumptions.assumeThat;
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
import org.apache.iceberg.MetricsModes.Counts;
import org.apache.iceberg.MetricsModes.Full;
import org.apache.iceberg.MetricsModes.None;
import org.apache.iceberg.MetricsModes.Truncate;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
-import org.apache.iceberg.types.Types;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.TestTemplate;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.api.Test;
-@ExtendWith(ParameterizedTestExtension.class)
public class TestMetricsModes {
- @TempDir private File tableDir;
-
- @Parameter private int formatVersion;
-
- @Parameters(name = "formatVersion = {0}")
- protected static List<Integer> formatVersions() {
- return ALL_VERSIONS;
- }
-
- @TempDir private Path temp;
-
- @AfterEach
- public void after() {
- TestTables.clearTables();
- }
-
- @TestTemplate
+ @Test
public void testMetricsModeParsing() {
assertThat(MetricsModes.fromString("none")).isEqualTo(None.get());
assertThat(MetricsModes.fromString("nOnE")).isEqualTo(None.get());
@@ -71,206 +40,10 @@ public class TestMetricsModes {
assertThat(MetricsModes.fromString("FULL")).isEqualTo(Full.get());
}
- @TestTemplate
+ @Test
public void testInvalidTruncationLength() {
assertThatThrownBy(() -> MetricsModes.fromString("truncate(0)"))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Truncate length should be positive");
}
-
- @TestTemplate
- public void testInvalidColumnModeValue() {
- Map<String, String> properties =
- ImmutableMap.of(
- TableProperties.DEFAULT_WRITE_METRICS_MODE,
- "full",
- TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col",
- "troncate(5)");
-
- Schema schema = new Schema(required(1, "col", Types.StringType.get()));
-
- MetricsConfig config = MetricsTestUtil.from(properties, schema);
- assertThat(config.columnMode("col"))
- .as("Invalid mode should be defaulted to table default (full)")
- .isEqualTo(MetricsModes.Full.get());
- }
-
- @TestTemplate
- public void testInvalidDefaultColumnModeValue() {
- Map<String, String> properties =
- ImmutableMap.of(
- TableProperties.DEFAULT_WRITE_METRICS_MODE,
- "fuull",
- TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col",
- "troncate(5)");
-
- Schema schema = new Schema(required(1, "col", Types.StringType.get()));
-
- MetricsConfig config = MetricsTestUtil.from(properties, schema);
- assertThat(config.columnMode("col"))
- .as("Invalid mode should be defaulted to library default
(truncate(16))")
- .isEqualTo(MetricsModes.Truncate.withLength(16));
- }
-
- @TestTemplate
- public void testMetricsConfigSortedColsDefault() {
- Schema schema =
- new Schema(
- required(1, "col1", Types.IntegerType.get()),
- required(2, "col2", Types.IntegerType.get()),
- required(3, "col3", Types.IntegerType.get()),
- required(4, "col4", Types.IntegerType.get()));
- SortOrder sortOrder =
SortOrder.builderFor(schema).asc("col2").asc("col3").build();
- Table testTable =
- TestTables.create(
- tableDir, "test", schema, PartitionSpec.unpartitioned(),
sortOrder, formatVersion);
- testTable
- .updateProperties()
- .set(TableProperties.DEFAULT_WRITE_METRICS_MODE, "counts")
- .set(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col1",
"counts")
- .set(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col2", "none")
- .commit();
-
- MetricsConfig config = MetricsConfig.forTable(testTable);
- assertThat(config.columnMode("col1"))
- .as("Non-sorted existing column should not be overridden")
- .isEqualTo(Counts.get());
- assertThat(config.columnMode("col2"))
- .as("Sorted column defaults should not override user specified config")
- .isEqualTo(None.get());
- assertThat(config.columnMode("col3"))
- .as("Unspecified sorted column should use default")
- .isEqualTo(Truncate.withLength(16));
- assertThat(config.columnMode("col4"))
- .as("Unspecified normal column should use default")
- .isEqualTo(Counts.get());
- }
-
- @TestTemplate
- public void testMetricsConfigSortedColsDefaultByInvalid() throws Exception {
- Schema schema =
- new Schema(
- required(1, "col1", Types.IntegerType.get()),
- required(2, "col2", Types.IntegerType.get()),
- required(3, "col3", Types.IntegerType.get()));
- SortOrder sortOrder =
SortOrder.builderFor(schema).asc("col2").asc("col3").build();
- Table testTable =
- TestTables.create(
- tableDir, "test", schema, PartitionSpec.unpartitioned(),
sortOrder, formatVersion);
- testTable
- .updateProperties()
- .set(TableProperties.DEFAULT_WRITE_METRICS_MODE, "counts")
- .set(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col1", "full")
- .set(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col2",
"invalid")
- .commit();
-
- MetricsConfig config = MetricsConfig.forTable(testTable);
- assertThat(config.columnMode("col1"))
- .as("Non-sorted existing column should not be overridden by sorted
column")
- .isEqualTo(Full.get());
- assertThat(config.columnMode("col2"))
- .as("Original default applies as user entered invalid mode for sorted
column")
- .isEqualTo(Truncate.withLength(16));
- }
-
- @TestTemplate
- public void testMetricsConfigInferredDefaultModeLimit() throws IOException {
- Schema schema =
- new Schema(
- required(1, "col1", Types.IntegerType.get()),
- required(2, "col2", Types.IntegerType.get()),
- required(3, "col3", Types.IntegerType.get()));
-
- Table table =
- TestTables.create(
- tableDir,
- "test",
- schema,
- PartitionSpec.unpartitioned(),
- SortOrder.unsorted(),
- formatVersion);
-
- // only infer a default for the first two columns
- table
- .updateProperties()
- .set(TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, "2")
- .commit();
-
- MetricsConfig config = MetricsConfig.forTable(table);
-
- assertThat(config.columnMode("col1")).isEqualTo(Truncate.withLength(16));
- assertThat(config.columnMode("col2")).isEqualTo(Truncate.withLength(16));
- assertThat(config.columnMode("col3")).isEqualTo(None.get());
- }
-
- @TestTemplate
- public void testMetricsVariantSupported() {
- assumeThat(formatVersion).isGreaterThanOrEqualTo(3);
- Schema schema =
- new Schema(
- required(1, "variant", Types.VariantType.get()),
- required(2, "int", Types.IntegerType.get()));
-
- Table table =
- TestTables.create(
- tableDir,
- "test",
- schema,
- PartitionSpec.unpartitioned(),
- SortOrder.unsorted(),
- formatVersion);
-
- // only infer a default for the first column
- table
- .updateProperties()
- .set(TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, "1")
- .commit();
-
- MetricsConfig config = MetricsConfig.forTable(table);
-
- Map<String, MetricsModes.MetricsMode> metricModes =
- schema.idToName().values().stream().collect(Collectors.toMap(k -> k,
config::columnMode));
-
- assertThat(metricModes)
- .containsOnly(Map.entry("variant", Truncate.withLength(16)),
Map.entry("int", None.get()));
- }
-
- @TestTemplate
- public void testMetricsConfigNestedTypesStructs() {
- Schema schema =
- new Schema(
- required(
- 5,
- "col_struct",
- Types.StructType.of(
- required(33, "a", Types.IntegerType.get()),
- required(1, "b", Types.IntegerType.get()))),
- required(4, "top", Types.IntegerType.get()));
-
- Table table =
- TestTables.create(
- tableDir,
- "test",
- schema,
- PartitionSpec.unpartitioned(),
- SortOrder.unsorted(),
- formatVersion);
-
- // only infer a default for the first two columns
- table
- .updateProperties()
- .set(TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, "2")
- .commit();
-
- MetricsConfig config = MetricsConfig.forTable(table);
-
- Map<String, MetricsModes.MetricsMode> metricModes =
- schema.idToName().values().stream().collect(Collectors.toMap(k -> k,
config::columnMode));
-
- assertThat(metricModes).containsOnlyKeys("col_struct.a", "col_struct",
"col_struct.b", "top");
-
- assertThat(metricModes).containsEntry("col_struct.a",
Truncate.withLength(16));
- assertThat(metricModes).containsEntry("col_struct.b", None.get());
- assertThat(metricModes).containsEntry("top", Truncate.withLength(16));
- }
}
diff --git a/core/src/test/java/org/apache/iceberg/TestSchemaOrder.java
b/core/src/test/java/org/apache/iceberg/TestSchemaOrder.java
new file mode 100644
index 0000000000..2f319dd678
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/TestSchemaOrder.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.Test;
+
+class TestSchemaOrder {
+ @Test
+ public void exampleStructOrder() {
+ Schema schema =
+ new Schema(
+ Types.NestedField.required(1, "id", Types.LongType.get()),
+ Types.NestedField.optional(2, "data", Types.StringType.get()),
+ Types.NestedField.optional(
+ 3,
+ "point",
+ Types.StructType.of(
+ Types.NestedField.required(4, "x", Types.FloatType.get()),
+ Types.NestedField.required(5, "y",
Types.FloatType.get()))));
+
+ assertThat(SchemaOrder.allFieldIds(schema)).containsExactly(1, 2, 3, 4, 5);
+ assertThat(SchemaOrder.leafFieldIds(schema)).containsExactly(1, 2, 4, 5);
+ }
+
+ @Test
+ public void listOrder() {
+ Schema schema =
+ new Schema(
+ Types.NestedField.required(1, "id", Types.LongType.get()),
+ Types.NestedField.optional(
+ 2, "numbers", Types.ListType.ofRequired(3,
Types.DoubleType.get())),
+ Types.NestedField.optional(4, "data", Types.StringType.get()));
+
+ assertThat(SchemaOrder.allFieldIds(schema)).containsExactly(1, 2, 3, 4);
+ assertThat(SchemaOrder.leafFieldIds(schema)).containsExactly(1, 3, 4);
+ }
+
+ @Test
+ public void mapOrder() {
+ Schema schema =
+ new Schema(
+ Types.NestedField.required(1, "id", Types.LongType.get()),
+ Types.NestedField.optional(
+ 2,
+ "properties",
+ Types.MapType.ofRequired(3, 4, Types.StringType.get(),
Types.StringType.get())),
+ Types.NestedField.optional(5, "data", Types.StringType.get()));
+
+ assertThat(SchemaOrder.allFieldIds(schema)).containsExactly(1, 2, 3, 4, 5);
+ assertThat(SchemaOrder.leafFieldIds(schema)).containsExactly(1, 3, 4, 5);
+ }
+
+ @Test
+ public void mixedTypeOrder() {
+ Schema schema =
+ new Schema(
+ required(1, "id", Types.LongType.get()),
+ optional(2, "data", Types.StringType.get()),
+ optional(
+ 3,
+ "preferences",
+ Types.StructType.of(
+ required(4, "feature1", Types.BooleanType.get()),
+ optional(5, "feature2", Types.BooleanType.get()))),
+ required(
+ 6,
+ "locations",
+ Types.MapType.ofRequired(
+ 7,
+ 8, // Map value ID comes after key fields (between 12 and
13)
+ Types.StructType.of(
+ required(9, "address", Types.StringType.get()),
+ required(10, "city", Types.StringType.get()),
+ required(11, "state", Types.StringType.get()),
+ required(12, "zip", Types.IntegerType.get())),
+ Types.StructType.of(
+ required(13, "lat", Types.FloatType.get()),
+ required(14, "long", Types.FloatType.get())))),
+ optional(
+ 15,
+ "points",
+ Types.ListType.ofOptional(
+ 16,
+ Types.StructType.of(
+ required(17, "x", Types.LongType.get()),
+ required(18, "y", Types.LongType.get())))),
+ required(19, "numbers", Types.ListType.ofRequired(20,
Types.DoubleType.get())),
+ optional(
+ 21,
+ "properties",
+ Types.MapType.ofOptional(22, 23, Types.StringType.get(),
Types.StringType.get())),
+ optional(24, "variant", Types.VariantType.get()));
+
+ assertThat(SchemaOrder.allFieldIds(schema))
+ .containsExactly(
+ 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 8, 13, 14, 15, 16, 17, 18, 19,
20, 21, 22, 23, 24);
+ assertThat(SchemaOrder.leafFieldIds(schema))
+ .containsExactly(1, 2, 4, 5, 9, 10, 11, 12, 13, 14, 17, 18, 20, 22,
23, 24);
+ }
+}
diff --git a/core/src/test/java/org/apache/iceberg/TestStatsUtil.java
b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java
index a0db67642f..183c4336f7 100644
--- a/core/src/test/java/org/apache/iceberg/TestStatsUtil.java
+++ b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java
@@ -21,11 +21,9 @@ package org.apache.iceberg;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import java.util.Comparator;
import java.util.List;
import java.util.Map;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.junit.jupiter.api.Test;
@@ -567,20 +565,14 @@ public class TestStatsUtil {
}
/**
- * Assert that two struct types match in field IDs, names, optionality, and
types, ignoring field
- * order and docs.
+ * Assert that two struct types match in field IDs, names, optionality, and
types, ignoring docs.
*/
private static void assertSameStructure(Types.StructType expected,
Types.StructType actual) {
assertThat(actual.fields()).as("Number of
fields").hasSameSizeAs(expected.fields());
- List<Types.NestedField> expectedFields =
Lists.newArrayList(expected.fields());
- List<Types.NestedField> actualFields = Lists.newArrayList(actual.fields());
- expectedFields.sort(Comparator.comparingInt(Types.NestedField::fieldId));
- actualFields.sort(Comparator.comparingInt(Types.NestedField::fieldId));
-
- for (int i = 0; i < expectedFields.size(); i += 1) {
- Types.NestedField expectedField = expectedFields.get(i);
- Types.NestedField actualField = actualFields.get(i);
+ for (int i = 0; i < expected.fields().size(); i += 1) {
+ Types.NestedField expectedField = expected.fields().get(i);
+ Types.NestedField actualField = actual.fields().get(i);
assertThat(actualField.fieldId()).as("Field
ID").isEqualTo(expectedField.fieldId());
assertThat(actualField.name()).as("Field
name").isEqualTo(expectedField.name());
