This is an automated email from the ASF dual-hosted git repository. adutra pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push: new f0428804f Upgrade Polaris to Iceberg 1.9.0 (#1309) f0428804f is described below commit f0428804f91dd2216c8dafc934976be45569962f Author: Alexandre Dutra <adu...@users.noreply.github.com> AuthorDate: Mon Apr 28 16:55:52 2025 +0200 Upgrade Polaris to Iceberg 1.9.0 (#1309) --- gradle/libs.versions.toml | 2 +- .../polaris/service/it/env/IcebergHelper.java | 11 +------- .../it/test/PolarisApplicationIntegrationTest.java | 18 ++++++------ .../it/test/PolarisRestCatalogIntegrationTest.java | 33 +++++++++++++++++----- .../PolarisRestCatalogViewIntegrationBase.java | 20 +++++++------ .../spark/quarkus/it/SparkCatalogBaseIT.java | 11 ++++++-- .../quarkus/catalog/IcebergCatalogTest.java | 14 +++++++++ .../quarkus/catalog/IcebergCatalogViewTest.java | 23 +++++++++------ .../service/quarkus/task/TaskTestUtils.java | 2 +- regtests/setup.sh | 2 +- regtests/t_pyspark/src/iceberg_spark.py | 2 +- 11 files changed, 88 insertions(+), 50 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index bc7eede8b..9b8db8dd3 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -19,7 +19,7 @@ [versions] hadoop = "3.4.1" -iceberg = "1.8.1" # Ensure to update the iceberg version in regtests to keep regtests up-to-date +iceberg = "1.9.0" # Ensure to update the iceberg version in regtests to keep regtests up-to-date quarkus = "3.21.2" immutables = "2.10.1" picocli = "4.7.7" diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java index 4d0c987b2..a119bf615 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java @@ -20,8 +20,6 @@ package org.apache.polaris.service.it.env; import com.google.common.collect.ImmutableMap; import java.util.Map; -import org.apache.iceberg.catalog.SessionCatalog; -import org.apache.iceberg.rest.HTTPClient; import org.apache.iceberg.rest.RESTCatalog; import org.apache.iceberg.rest.auth.OAuth2Properties; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; @@ -36,14 +34,7 @@ public final class IcebergHelper { String catalog, Map<String, String> extraProperties) { String authToken = client.obtainToken(credentials); - SessionCatalog.SessionContext context = SessionCatalog.SessionContext.createEmpty(); - RESTCatalog restCatalog = - new RESTCatalog( - context, - (config) -> - HTTPClient.builder(config) - .uri(config.get(org.apache.iceberg.CatalogProperties.URI)) - .build()); + RESTCatalog restCatalog = new RESTCatalog(); ImmutableMap.Builder<String, String> propertiesBuilder = ImmutableMap.<String, String>builder() diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java index 2788567da..ac1584254 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java @@ -371,8 +371,8 @@ public class PolarisApplicationIntegrationTest { TableIdentifier.of(ns, "the_table"), new Schema( List.of( - Types.NestedField.of( - 1, false, "theField", Types.StringType.get())))) + Types.NestedField.required( + 1, "theField", Types.StringType.get())))) .withLocation("file:///tmp/tables") .withSortOrder(SortOrder.unsorted()) .withPartitionSpec(PartitionSpec.unpartitioned()) @@ -399,8 +399,8 @@ public class PolarisApplicationIntegrationTest { TableIdentifier.of(ns, "the_table"), new Schema( List.of( - Types.NestedField.of( - 1, false, "theField", Types.StringType.get())))) + Types.NestedField.required( + 1, "theField", Types.StringType.get())))) .withSortOrder(SortOrder.unsorted()) .withPartitionSpec(PartitionSpec.unpartitioned()) .withProperties(Map.of("write.data.path", "s3://my-bucket/path/to/data")) @@ -416,8 +416,8 @@ public class PolarisApplicationIntegrationTest { TableIdentifier.of(ns, "the_table"), new Schema( List.of( - Types.NestedField.of( - 1, false, "theField", Types.StringType.get())))) + Types.NestedField.required( + 1, "theField", Types.StringType.get())))) .withSortOrder(SortOrder.unsorted()) .withPartitionSpec(PartitionSpec.unpartitioned()) .withProperties(Map.of("write.metadata.path", "s3://my-bucket/path/to/data")) @@ -454,7 +454,7 @@ public class PolarisApplicationIntegrationTest { .assignUUID() .addPartitionSpec(PartitionSpec.unpartitioned()) .addSortOrder(SortOrder.unsorted()) - .addSchema(new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get()))) + .addSchema(new Schema(Types.NestedField.required(1, "col1", Types.StringType.get()))) .build(); TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation)); @@ -490,7 +490,7 @@ public class PolarisApplicationIntegrationTest { String location = baseLocation.resolve("testIcebergUpdateTableInExternalCatalog").toString(); String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json"; - Types.NestedField col1 = Types.NestedField.of(1, false, "col1", Types.StringType.get()); + Types.NestedField col1 = Types.NestedField.required(1, "col1", Types.StringType.get()); TableMetadata tableMetadata = TableMetadata.buildFromEmpty() .setLocation(location) @@ -545,7 +545,7 @@ public class PolarisApplicationIntegrationTest { .assignUUID() .addPartitionSpec(PartitionSpec.unpartitioned()) .addSortOrder(SortOrder.unsorted()) - .addSchema(new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get()))) + .addSchema(new Schema(Types.NestedField.required(1, "col1", Types.StringType.get()))) .build(); TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation)); diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java index e5c8619e1..e55ddf1c5 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java @@ -148,6 +148,19 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog> private final String catalogBaseLocation = s3BucketBase + "/" + System.getenv("USER") + "/path/to/data"; + private static final Map<String, String> DEFAULT_REST_CATALOG_CONFIG = + Map.of( + org.apache.iceberg.CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1", + "catalog-default-key1", + org.apache.iceberg.CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2", + "catalog-default-key2", + org.apache.iceberg.CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3", + "catalog-default-key3", + org.apache.iceberg.CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3", + "catalog-override-key3", + org.apache.iceberg.CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4", + "catalog-override-key4"); + private static final String[] DEFAULT_CATALOG_PROPERTIES = { "allow.unstructured.table.location", "true", "allow.external.table.location", "true" @@ -235,7 +248,7 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog> managementApi.createCatalog(principalRoleName, catalog); - restCatalogConfig = + Map<String, String> dynamicConfig = testInfo .getTestMethod() .map(m -> m.getAnnotation(RestCatalogConfig.class)) @@ -254,6 +267,12 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog> }) .orElse(ImmutableMap.of()); + restCatalogConfig = + ImmutableMap.<String, String>builder() + .putAll(DEFAULT_REST_CATALOG_CONFIG) + .putAll(dynamicConfig) + .build(); + restCatalog = initCatalog(currentCatalogName, ImmutableMap.of()); } @@ -581,7 +600,7 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog> restCatalog.createNamespace(ns1); TableMetadata tableMetadata = TableMetadata.newTableMetadata( - new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))), + new Schema(List.of(Types.NestedField.required(1, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), externalCatalogBase + "/ns1/my_table", Map.of()); @@ -616,7 +635,7 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog> restCatalog.createNamespace(ns1); TableMetadata tableMetadata = TableMetadata.newTableMetadata( - new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))), + new Schema(List.of(Types.NestedField.required(1, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), externalCatalogBase + "/ns1/my_table", Map.of()); @@ -650,7 +669,7 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog> restCatalog.createNamespace(ns1); TableMetadata tableMetadata = TableMetadata.newTableMetadata( - new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))), + new Schema(List.of(Types.NestedField.required(1, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), externalCatalogBase + "/ns1/my_table", Map.of()); @@ -681,7 +700,7 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog> restCatalog.createNamespace(ns1); TableMetadata tableMetadata = TableMetadata.newTableMetadata( - new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))), + new Schema(List.of(Types.NestedField.required(1, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), "file:///tmp/ns1/my_table", Map.of()); @@ -728,7 +747,7 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog> restCatalog.createNamespace(ns1); TableMetadata tableMetadata = TableMetadata.newTableMetadata( - new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))), + new Schema(List.of(Types.NestedField.required(1, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), "file:///tmp/ns1/my_table", Map.of()); @@ -774,7 +793,7 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog> restCatalog.createNamespace(ns1); TableMetadata tableMetadata = TableMetadata.newTableMetadata( - new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))), + new Schema(List.of(Types.NestedField.required(1, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), "file:///tmp/ns1/my_table", Map.of()); diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java index c5df1c5d8..bbe21bda6 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java @@ -62,6 +62,16 @@ public abstract class PolarisRestCatalogViewIntegrationBase extends ViewCatalogT Assumptions.setPreferredAssumptionException(PreferredAssumptionException.JUNIT5); } + public static Map<String, String> DEFAULT_REST_CATALOG_CONFIG = + Map.of( + org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1", + org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", "catalog-default-key2", + org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key3", "catalog-default-key3", + org.apache.iceberg.CatalogProperties.VIEW_OVERRIDE_PREFIX + "key3", + "catalog-override-key3", + org.apache.iceberg.CatalogProperties.VIEW_OVERRIDE_PREFIX + "key4", + "catalog-override-key4"); + private static ClientCredentials adminCredentials; private static PolarisApiEndpoints endpoints; private static PolarisClient client; @@ -120,15 +130,7 @@ public abstract class PolarisRestCatalogViewIntegrationBase extends ViewCatalogT restCatalog = IcebergHelper.restCatalog( - client, - endpoints, - principalCredentials, - catalogName, - Map.of( - org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", - "catalog-default-key1", - org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", - "catalog-default-key2")); + client, endpoints, principalCredentials, catalogName, DEFAULT_REST_CATALOG_CONFIG); } @AfterEach diff --git a/plugins/spark/v3.5/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java b/plugins/spark/v3.5/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java index 575436e33..2e8d0e4a9 100644 --- a/plugins/spark/v3.5/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java +++ b/plugins/spark/v3.5/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java @@ -27,6 +27,7 @@ import io.quarkus.test.junit.QuarkusIntegrationTest; import java.util.Arrays; import java.util.Map; import org.apache.iceberg.exceptions.BadRequestException; +import org.apache.iceberg.exceptions.NamespaceNotEmptyException; import org.apache.iceberg.spark.SupportsReplaceView; import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException; import org.apache.spark.sql.catalyst.analysis.NoSuchViewException; @@ -107,7 +108,10 @@ public abstract class SparkCatalogBaseIT extends SparkIntegrationBase { // directly drop lv1ns[0] should fail assertThatThrownBy(() -> namespaceCatalog.dropNamespace(lv1ns[0], true)) - .isInstanceOf(BadRequestException.class); + .isInstanceOfAny( + BadRequestException.class, // Iceberg < 1.9.0 + NamespaceNotEmptyException.class // Iceberg >= 1.9.0 + ); for (String[] namespace : lv2ns1) { namespaceCatalog.dropNamespace(namespace, true); } @@ -249,7 +253,10 @@ public abstract class SparkCatalogBaseIT extends SparkIntegrationBase { // drop namespace fails since there are views under it assertThatThrownBy(() -> namespaceCatalog.dropNamespace(l2ns, true)) - .isInstanceOf(BadRequestException.class); + .isInstanceOfAny( + BadRequestException.class, // Iceberg < 1.9.0 + NamespaceNotEmptyException.class // Iceberg >= 1.9.0 + ); // drop the views for (String name : nsl2ViewNames) { viewCatalog.dropView(Identifier.of(l2ns, name)); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index ea37631d7..39c8950fe 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -180,6 +180,19 @@ public abstract class IcebergCatalogTest extends CatalogTests<IcebergCatalog> { public static final String SECRET_ACCESS_KEY = "secret_access_key"; public static final String SESSION_TOKEN = "session_token"; + public static Map<String, String> TABLE_PREFIXES = + Map.of( + CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1", + "catalog-default-key1", + CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2", + "catalog-default-key2", + CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3", + "catalog-default-key3", + CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3", + "catalog-override-key3", + CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4", + "catalog-override-key4"); + @Inject MetaStoreManagerFactory managerFactory; @Inject PolarisConfigurationStore configurationStore; @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; @@ -344,6 +357,7 @@ public abstract class IcebergCatalogTest extends CatalogTests<IcebergCatalog> { ImmutableMap.Builder<String, String> propertiesBuilder = ImmutableMap.<String, String>builder() .put(CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO") + .putAll(TABLE_PREFIXES) .putAll(additionalProperties); icebergCatalog.initialize(CATALOG_NAME, propertiesBuilder.buildKeepingLast()); return icebergCatalog; diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java index 5123ea0e1..127dd7f23 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java @@ -104,6 +104,14 @@ public class IcebergCatalogViewTest extends ViewCatalogTests<IcebergCatalog> { public static final String CATALOG_NAME = "polaris-catalog"; + public static Map<String, String> VIEW_PREFIXES = + Map.of( + CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1", + CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", "catalog-default-key2", + CatalogProperties.VIEW_DEFAULT_PREFIX + "key3", "catalog-default-key3", + CatalogProperties.VIEW_OVERRIDE_PREFIX + "key3", "catalog-override-key3", + CatalogProperties.VIEW_OVERRIDE_PREFIX + "key4", "catalog-override-key4"); + @Inject MetaStoreManagerFactory managerFactory; @Inject UserSecretsManagerFactory userSecretsManagerFactory; @Inject PolarisConfigurationStore configurationStore; @@ -211,15 +219,12 @@ public class IcebergCatalogViewTest extends ViewCatalogTests<IcebergCatalog> { securityContext, Mockito.mock(), fileIOFactory); - this.catalog.initialize( - CATALOG_NAME, - ImmutableMap.of( - CatalogProperties.FILE_IO_IMPL, - "org.apache.iceberg.inmemory.InMemoryFileIO", - CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", - "catalog-default-key1", - CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", - "catalog-default-key2")); + Map<String, String> properties = + ImmutableMap.<String, String>builder() + .put(CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO") + .putAll(VIEW_PREFIXES) + .build(); + this.catalog.initialize(CATALOG_NAME, properties); } @AfterEach diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java index d7538b923..83f32f9b3 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java @@ -100,7 +100,7 @@ public class TaskTestUtils { tmBuilder .setLocation("path/to/table") .addSchema( - new Schema(List.of(Types.NestedField.of(1, false, "field1", Types.StringType.get())))) + new Schema(List.of(Types.NestedField.required(1, "field1", Types.StringType.get())))) .addSortOrder(SortOrder.unsorted()) .assignUUID(UUID.randomUUID().toString()) .addPartitionSpec(PartitionSpec.unpartitioned()); diff --git a/regtests/setup.sh b/regtests/setup.sh index e65e5a337..b2d792fa1 100755 --- a/regtests/setup.sh +++ b/regtests/setup.sh @@ -31,7 +31,7 @@ if [ -z "${SPARK_HOME}" ]; then fi SPARK_CONF="${SPARK_HOME}/conf/spark-defaults.conf" DERBY_HOME="/tmp/derby" -ICEBERG_VERSION="1.8.1" +ICEBERG_VERSION="1.9.0" export PYTHONPATH="${SPARK_HOME}/python/:${SPARK_HOME}/python/lib/py4j-0.10.9.7-src.zip:$PYTHONPATH" # Ensure binaries are downloaded locally diff --git a/regtests/t_pyspark/src/iceberg_spark.py b/regtests/t_pyspark/src/iceberg_spark.py index 339888ef8..897180d5f 100644 --- a/regtests/t_pyspark/src/iceberg_spark.py +++ b/regtests/t_pyspark/src/iceberg_spark.py @@ -74,7 +74,7 @@ class IcebergSparkSession: """Initial method for Iceberg Spark session. Creates a Spark session with specified configs. """ packages = [ - "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1", + "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.9.0", "org.apache.hadoop:hadoop-aws:3.4.0", "software.amazon.awssdk:bundle:2.23.19", "software.amazon.awssdk:url-connection-client:2.23.19",