This is an automated email from the ASF dual-hosted git repository.
russellspitzer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 49cf9d9881 Core: V3 Metadata Upgrade Validation and Testing (#10861)
49cf9d9881 is described below
commit 49cf9d98818b7d40c9fb7602274b4ba0fd83d82a
Author: Jonathan Leang <[email protected]>
AuthorDate: Fri Aug 16 16:08:25 2024 -0700
Core: V3 Metadata Upgrade Validation and Testing (#10861)
Co-authored-by: Jonathan Leang <[email protected]>
---
.../org/apache/iceberg/TestFormatVersions.java | 66 ++++++++++++++++------
.../java/org/apache/iceberg/TestTableMetadata.java | 43 ++++++++++----
.../org/apache/iceberg/TestUpdateRequirements.java | 9 ++-
3 files changed, 88 insertions(+), 30 deletions(-)
diff --git a/core/src/test/java/org/apache/iceberg/TestFormatVersions.java
b/core/src/test/java/org/apache/iceberg/TestFormatVersions.java
index 4a9f18581d..3414f1858e 100644
--- a/core/src/test/java/org/apache/iceberg/TestFormatVersions.java
+++ b/core/src/test/java/org/apache/iceberg/TestFormatVersions.java
@@ -28,51 +28,83 @@ import org.junit.jupiter.api.TestTemplate;
public class TestFormatVersions extends TestBase {
@Parameters(name = "formatVersion = {0}")
protected static List<Object> parameters() {
- return Arrays.asList(1);
+ return Arrays.asList(1, 2);
}
@TestTemplate
public void testDefaultFormatVersion() {
- assertThat(table.ops().current().formatVersion()).isEqualTo(1);
+ assertThat(table.ops().current().formatVersion()).isEqualTo(formatVersion);
}
@TestTemplate
public void testFormatVersionUpgrade() {
TableOperations ops = table.ops();
- TableMetadata base = ops.current();
- ops.commit(base, base.upgradeToFormatVersion(2));
+ int newFormatVersion = formatVersion + 1;
+
+ TableMetadata newTableMetadata =
ops.current().upgradeToFormatVersion(newFormatVersion);
+
+ assertThat(
+ newTableMetadata.changes().stream()
+ .filter(MetadataUpdate.UpgradeFormatVersion.class::isInstance)
+ .map(MetadataUpdate.UpgradeFormatVersion.class::cast)
+ .map(MetadataUpdate.UpgradeFormatVersion::formatVersion))
+ .containsExactly(newFormatVersion);
+
+ ops.commit(ops.current(), newTableMetadata);
- assertThat(ops.current().formatVersion()).isEqualTo(2);
+ assertThat(ops.current().formatVersion()).isEqualTo(newFormatVersion);
+ }
+
+ @TestTemplate
+ public void testFormatVersionUpgradeToLatest() {
+ TableOperations ops = table.ops();
+
+ TableMetadata newTableMetadata =
+
ops.current().upgradeToFormatVersion(TableMetadata.SUPPORTED_TABLE_FORMAT_VERSION);
+
+ assertThat(
+ newTableMetadata.changes().stream()
+ .filter(MetadataUpdate.UpgradeFormatVersion.class::isInstance)
+ .map(MetadataUpdate.UpgradeFormatVersion.class::cast)
+ .map(MetadataUpdate.UpgradeFormatVersion::formatVersion))
+ .isEqualTo(List.of(TableMetadata.SUPPORTED_TABLE_FORMAT_VERSION));
+
+ ops.commit(ops.current(), newTableMetadata);
+
+ assertThat(ops.current().formatVersion())
+ .isEqualTo(TableMetadata.SUPPORTED_TABLE_FORMAT_VERSION);
}
@TestTemplate
public void testFormatVersionDowngrade() {
TableOperations ops = table.ops();
- TableMetadata base = ops.current();
- ops.commit(base, base.upgradeToFormatVersion(2));
+ int newFormatVersion = formatVersion + 1;
+ ops.commit(ops.current(),
ops.current().upgradeToFormatVersion(newFormatVersion));
- assertThat(ops.current().formatVersion()).isEqualTo(2);
+ assertThat(ops.current().formatVersion()).isEqualTo(newFormatVersion);
- assertThatThrownBy(() -> ops.current().upgradeToFormatVersion(1))
+ assertThatThrownBy(() ->
ops.current().upgradeToFormatVersion(formatVersion))
.isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Cannot downgrade v2 table to v1");
+ .hasMessage(
+ String.format("Cannot downgrade v%d table to v%d",
newFormatVersion, formatVersion));
- assertThat(ops.current().formatVersion()).isEqualTo(2);
+ assertThat(ops.current().formatVersion()).isEqualTo(newFormatVersion);
}
@TestTemplate
public void testFormatVersionUpgradeNotSupported() {
TableOperations ops = table.ops();
TableMetadata base = ops.current();
+ int unsupportedFormatVersion =
TableMetadata.SUPPORTED_TABLE_FORMAT_VERSION + 1;
assertThatThrownBy(
- () ->
- ops.commit(
- base,
-
base.upgradeToFormatVersion(TableMetadata.SUPPORTED_TABLE_FORMAT_VERSION + 1)))
+ () -> ops.commit(base,
base.upgradeToFormatVersion(unsupportedFormatVersion)))
.isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Cannot upgrade table to unsupported format version: v4
(supported: v3)");
+ .hasMessage(
+ String.format(
+ "Cannot upgrade table to unsupported format version: v%d
(supported: v%d)",
+ unsupportedFormatVersion,
TableMetadata.SUPPORTED_TABLE_FORMAT_VERSION));
- assertThat(ops.current().formatVersion()).isEqualTo(1);
+ assertThat(ops.current().formatVersion()).isEqualTo(formatVersion);
}
}
diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java
b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java
index e11cc500df..809b1ba963 100644
--- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java
+++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java
@@ -32,6 +32,7 @@ import static
org.apache.iceberg.TestHelpers.assertSameSchemaList;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.api.Assertions.entry;
+import static org.junit.jupiter.params.provider.Arguments.arguments;
import com.fasterxml.jackson.core.JsonGenerator;
import java.io.File;
@@ -49,6 +50,8 @@ import java.util.Random;
import java.util.Set;
import java.util.SortedSet;
import java.util.UUID;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
import org.apache.iceberg.TableMetadata.MetadataLogEntry;
import org.apache.iceberg.TableMetadata.SnapshotLogEntry;
import org.apache.iceberg.exceptions.ValidationException;
@@ -63,6 +66,9 @@ import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.JsonUtil;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
public class TestTableMetadata {
private static final String TEST_LOCATION = "s3://bucket/test/location";
@@ -1451,8 +1457,20 @@ public class TestTableMetadata {
.doesNotContainKey(TableProperties.FORMAT_VERSION);
}
- @Test
- public void testReplaceV1MetadataToV2ThroughTableProperty() {
+ private static Stream<Arguments> upgradeFormatVersionProvider() {
+ // return a stream of all valid upgrade paths
+ return IntStream.range(1, TableMetadata.SUPPORTED_TABLE_FORMAT_VERSION)
+ .boxed()
+ .flatMap(
+ baseFormatVersion ->
+ IntStream.rangeClosed(
+ baseFormatVersion + 1,
TableMetadata.SUPPORTED_TABLE_FORMAT_VERSION)
+ .mapToObj(newFormatVersion -> arguments(baseFormatVersion,
newFormatVersion)));
+ }
+
+ @ParameterizedTest
+ @MethodSource("upgradeFormatVersionProvider")
+ public void testReplaceMetadataThroughTableProperty(int baseFormatVersion,
int newFormatVersion) {
Schema schema = new Schema(Types.NestedField.required(10, "x",
Types.StringType.get()));
TableMetadata meta =
@@ -1460,7 +1478,8 @@ public class TestTableMetadata {
schema,
PartitionSpec.unpartitioned(),
null,
- ImmutableMap.of(TableProperties.FORMAT_VERSION, "1", "key",
"val"));
+ ImmutableMap.of(
+ TableProperties.FORMAT_VERSION,
String.valueOf(baseFormatVersion), "key", "val"));
meta =
meta.buildReplacement(
@@ -1468,17 +1487,19 @@ public class TestTableMetadata {
meta.spec(),
meta.sortOrder(),
meta.location(),
- ImmutableMap.of(TableProperties.FORMAT_VERSION, "2", "key2",
"val2"));
+ ImmutableMap.of(
+ TableProperties.FORMAT_VERSION,
String.valueOf(newFormatVersion), "key2", "val2"));
- assertThat(meta.formatVersion()).isEqualTo(2);
+ assertThat(meta.formatVersion()).isEqualTo(newFormatVersion);
assertThat(meta.properties())
.containsEntry("key", "val")
.containsEntry("key2", "val2")
.doesNotContainKey(TableProperties.FORMAT_VERSION);
}
- @Test
- public void testUpgradeV1MetadataToV2ThroughTableProperty() {
+ @ParameterizedTest
+ @MethodSource("upgradeFormatVersionProvider")
+ public void testUpgradeMetadataThroughTableProperty(int baseFormatVersion,
int newFormatVersion) {
Schema schema = new Schema(Types.NestedField.required(10, "x",
Types.StringType.get()));
TableMetadata meta =
@@ -1486,15 +1507,17 @@ public class TestTableMetadata {
schema,
PartitionSpec.unpartitioned(),
null,
- ImmutableMap.of(TableProperties.FORMAT_VERSION, "1", "key",
"val"));
+ ImmutableMap.of(
+ TableProperties.FORMAT_VERSION,
String.valueOf(baseFormatVersion), "key", "val"));
meta =
meta.replaceProperties(
- ImmutableMap.of(TableProperties.FORMAT_VERSION, "2", "key2",
"val2"));
+ ImmutableMap.of(
+ TableProperties.FORMAT_VERSION,
String.valueOf(newFormatVersion), "key2", "val2"));
assertThat(meta.formatVersion())
.as("format version should be configured based on the format-version
key")
- .isEqualTo(2);
+ .isEqualTo(newFormatVersion);
assertThat(meta.properties())
.as("should not contain format-version but should contain new
properties")
.containsExactly(entry("key2", "val2"));
diff --git a/core/src/test/java/org/apache/iceberg/TestUpdateRequirements.java
b/core/src/test/java/org/apache/iceberg/TestUpdateRequirements.java
index ed11424417..1a6c289ea2 100644
--- a/core/src/test/java/org/apache/iceberg/TestUpdateRequirements.java
+++ b/core/src/test/java/org/apache/iceberg/TestUpdateRequirements.java
@@ -37,6 +37,8 @@ import org.apache.iceberg.view.ViewMetadata;
import org.assertj.core.api.InstanceOfAssertFactories;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
public class TestUpdateRequirements {
private final TableMetadata metadata = mock(TableMetadata.class);
@@ -184,11 +186,12 @@ public class TestUpdateRequirements {
updatedViewMetadata.uuid(), viewMetadata.uuid()));
}
- @Test
- public void upgradeFormatVersion() {
+ @ParameterizedTest
+ @ValueSource(ints = {2, 3})
+ public void upgradeFormatVersion(int formatVersion) {
List<UpdateRequirement> requirements =
UpdateRequirements.forUpdateTable(
- metadata, ImmutableList.of(new
MetadataUpdate.UpgradeFormatVersion(2)));
+ metadata, ImmutableList.of(new
MetadataUpdate.UpgradeFormatVersion(formatVersion)));
requirements.forEach(req -> req.validate(metadata));
assertThat(requirements)
