nastra commented on code in PR #7412: URL: https://github.com/apache/iceberg/pull/7412#discussion_r1261357299
########## gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java: ########## @@ -0,0 +1,385 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.grpc.testing.LocalChannelProvider; +import com.google.api.gax.grpc.testing.MockGrpcService; +import com.google.api.gax.grpc.testing.MockServiceHelper; +import com.google.cloud.bigquery.biglake.v1.Catalog; +import com.google.cloud.bigquery.biglake.v1.CatalogName; +import com.google.cloud.bigquery.biglake.v1.Database; +import com.google.cloud.bigquery.biglake.v1.DatabaseName; +import com.google.cloud.bigquery.biglake.v1.HiveDatabaseOptions; +import com.google.cloud.bigquery.biglake.v1.MetastoreServiceSettings; +import java.io.File; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.catalog.CatalogTests; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.gcp.GCPProperties; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.junit.Rule; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +public class BigLakeCatalogTest extends CatalogTests<BigLakeCatalog> { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); Review Comment: I'm not sure this will work because `@Rule` is a JUnit4 thing but these tests are running with JUnit5. Also could you please elaborate why this rule is needed? ########## gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java: ########## @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import com.google.cloud.bigquery.biglake.v1.Catalog; +import com.google.cloud.bigquery.biglake.v1.CatalogName; +import com.google.cloud.bigquery.biglake.v1.Database; +import com.google.cloud.bigquery.biglake.v1.DatabaseName; +import com.google.cloud.bigquery.biglake.v1.HiveDatabaseOptions; +import com.google.cloud.bigquery.biglake.v1.Table; +import com.google.cloud.bigquery.biglake.v1.TableName; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.iceberg.BaseMetastoreCatalog; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.MetadataTableType; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.ServiceFailureException; +import org.apache.iceberg.gcp.GCPProperties; +import org.apache.iceberg.hadoop.Configurable; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.ResolvingFileIO; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.base.Strings; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.relocated.com.google.common.collect.Streams; +import org.apache.iceberg.util.LocationUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Iceberg BigLake Metastore (BLMS) Catalog implementation. */ +public final class BigLakeCatalog extends BaseMetastoreCatalog + implements SupportsNamespaces, Configurable<Object> { + + public static final String DEFAULT_BIGLAKE_SERVICE_ENDPOINT = "biglake.googleapis.com:443"; + + private static final Logger LOG = LoggerFactory.getLogger(BigLakeCatalog.class); + + // The name of this Iceberg catalog plugin: spark.sql.catalog.<catalog_plugin>. + private String catalogPulginName; + private Map<String, String> properties; + private FileIO io; + private Object conf; + private String projectId; + private String region; + // BLMS catalog ID and fully qualified name. + private String catalogId; + private CatalogName catalogName; + private BigLakeClient client; + + // Must have a no-arg constructor to be dynamically loaded + // initialize(String name, Map<String, String> properties) will be called to complete + // initialization + public BigLakeCatalog() {} + + @Override + public void initialize(String inputName, Map<String, String> properties) { + Preconditions.checkArgument( + properties.containsKey(GCPProperties.BIGLAKE_PROJECT_ID), + "GCP project ID must be specified"); + String projectId = properties.get(GCPProperties.BIGLAKE_PROJECT_ID); + + Preconditions.checkArgument( + properties.containsKey(GCPProperties.BIGLAKE_PROJECT_ID), "GCP region must be specified"); Review Comment: ```suggestion properties.containsKey(GCPProperties.BIGLAKE_GCP_REGION), "GCP region must be specified"); ``` ########## gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java: ########## @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import com.google.cloud.bigquery.biglake.v1.Catalog; +import com.google.cloud.bigquery.biglake.v1.CatalogName; +import com.google.cloud.bigquery.biglake.v1.Database; +import com.google.cloud.bigquery.biglake.v1.DatabaseName; +import com.google.cloud.bigquery.biglake.v1.HiveDatabaseOptions; +import com.google.cloud.bigquery.biglake.v1.Table; +import com.google.cloud.bigquery.biglake.v1.TableName; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.iceberg.BaseMetastoreCatalog; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.MetadataTableType; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.ServiceFailureException; +import org.apache.iceberg.gcp.GCPProperties; +import org.apache.iceberg.hadoop.Configurable; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.ResolvingFileIO; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.base.Strings; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.relocated.com.google.common.collect.Streams; +import org.apache.iceberg.util.LocationUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Iceberg BigLake Metastore (BLMS) Catalog implementation. */ +public final class BigLakeCatalog extends BaseMetastoreCatalog + implements SupportsNamespaces, Configurable<Object> { + + public static final String DEFAULT_BIGLAKE_SERVICE_ENDPOINT = "biglake.googleapis.com:443"; + + private static final Logger LOG = LoggerFactory.getLogger(BigLakeCatalog.class); + + // The name of this Iceberg catalog plugin: spark.sql.catalog.<catalog_plugin>. + private String catalogPulginName; + private Map<String, String> properties; + private FileIO io; + private Object conf; + private String projectId; + private String region; + // BLMS catalog ID and fully qualified name. + private String catalogId; + private CatalogName catalogName; + private BigLakeClient client; + + // Must have a no-arg constructor to be dynamically loaded + // initialize(String name, Map<String, String> properties) will be called to complete + // initialization + public BigLakeCatalog() {} + + @Override + public void initialize(String inputName, Map<String, String> properties) { Review Comment: ```suggestion public void initialize(String name, Map<String, String> properties) { ``` ########## gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java: ########## @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import com.google.cloud.bigquery.biglake.v1.Catalog; +import com.google.cloud.bigquery.biglake.v1.CatalogName; +import com.google.cloud.bigquery.biglake.v1.Database; +import com.google.cloud.bigquery.biglake.v1.DatabaseName; +import com.google.cloud.bigquery.biglake.v1.HiveDatabaseOptions; +import com.google.cloud.bigquery.biglake.v1.Table; +import com.google.cloud.bigquery.biglake.v1.TableName; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.iceberg.BaseMetastoreCatalog; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.MetadataTableType; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.ServiceFailureException; +import org.apache.iceberg.gcp.GCPProperties; +import org.apache.iceberg.hadoop.Configurable; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.ResolvingFileIO; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.base.Strings; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.relocated.com.google.common.collect.Streams; +import org.apache.iceberg.util.LocationUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Iceberg BigLake Metastore (BLMS) Catalog implementation. */ +public final class BigLakeCatalog extends BaseMetastoreCatalog + implements SupportsNamespaces, Configurable<Object> { + + public static final String DEFAULT_BIGLAKE_SERVICE_ENDPOINT = "biglake.googleapis.com:443"; + + private static final Logger LOG = LoggerFactory.getLogger(BigLakeCatalog.class); + + // The name of this Iceberg catalog plugin: spark.sql.catalog.<catalog_plugin>. + private String catalogPulginName; + private Map<String, String> properties; + private FileIO io; + private Object conf; + private String projectId; + private String region; + // BLMS catalog ID and fully qualified name. + private String catalogId; + private CatalogName catalogName; + private BigLakeClient client; + + // Must have a no-arg constructor to be dynamically loaded + // initialize(String name, Map<String, String> properties) will be called to complete + // initialization + public BigLakeCatalog() {} + + @Override + public void initialize(String inputName, Map<String, String> properties) { + Preconditions.checkArgument( + properties.containsKey(GCPProperties.BIGLAKE_PROJECT_ID), + "GCP project ID must be specified"); + String projectId = properties.get(GCPProperties.BIGLAKE_PROJECT_ID); + + Preconditions.checkArgument( + properties.containsKey(GCPProperties.BIGLAKE_PROJECT_ID), "GCP region must be specified"); + String region = properties.get(GCPProperties.BIGLAKE_GCP_REGION); + + BigLakeClient client; + try { + // TODO: to add more auth options of the client. Currently it uses default auth + // (https://github.com/googleapis/google-cloud-java#application-default-credentials) + // that works on GCP services (e.g., GCE, GKE, Dataproc). + client = + new BigLakeClient( + properties.getOrDefault( + GCPProperties.BIGLAKE_ENDPOINT, DEFAULT_BIGLAKE_SERVICE_ENDPOINT), + projectId, + region); + } catch (IOException e) { + throw new ServiceFailureException(e, "Creating BigLake client failed"); + } + + initialize(inputName, properties, projectId, region, client); + } + + @VisibleForTesting + void initialize( + String inputName, + Map<String, String> properties, + String projectId, + String region, + BigLakeClient client) { + this.catalogPulginName = inputName; + this.properties = ImmutableMap.copyOf(properties); + this.projectId = projectId; + this.region = region; + Preconditions.checkNotNull(client, "BigLake client must not be null"); + this.client = client; + + // Users can specify the BigLake catalog ID, otherwise catalog plugin name will be used. + // For example, "spark.sql.catalog.<plugin-name>=org.apache.iceberg.spark.SparkCatalog" + // specifies the plugin name "<plugin-name>". + this.catalogId = this.properties.getOrDefault(GCPProperties.BIGLAKE_CATALOG_ID, inputName); + this.catalogName = CatalogName.of(projectId, region, catalogId); + + String ioImpl = + this.properties.getOrDefault( + CatalogProperties.FILE_IO_IMPL, ResolvingFileIO.class.getName()); + this.io = CatalogUtil.loadFileIO(ioImpl, this.properties, conf); + } + + @Override + protected TableOperations newTableOps(TableIdentifier identifier) { + // The identifier of metadata tables is like "ns.table.files". + // Return a non-existing table in this case (empty table ID is disallowed in BigLake + // Metastore), loadTable will try loadMetadataTable. + if (identifier.namespace().levels().length > 1 + && MetadataTableType.from(identifier.name()) != null) { + return new BigLakeTableOperations( + client, io, tableName(identifier.namespace().level(0), /* tableId= */ "")); + } + + return new BigLakeTableOperations( + client, io, tableName(databaseId(identifier.namespace()), identifier.name())); + } + + @Override + protected String defaultWarehouseLocation(TableIdentifier identifier) { + String locationUri = loadDatabase(identifier.namespace()).getHiveOptions().getLocationUri(); + return String.format( + "%s/%s", + Strings.isNullOrEmpty(locationUri) + ? databaseLocation(databaseId(identifier.namespace())) + : locationUri, + identifier.name()); + } + + @Override + public List<TableIdentifier> listTables(Namespace namespace) { + // When deleting a BLMS catalog via `DROP NAMESPACE <catalog>`, this method is called for + // verifying catalog emptiness. `namespace` is empty in this case, we list databases in + // this catalog instead. + // TODO: to return all tables in all databases in a BLMS catalog instead of a "placeholder". + if (namespace.levels().length == 0) { + return Iterables.isEmpty(client.listDatabases(catalogName)) + ? ImmutableList.of() + : ImmutableList.of(TableIdentifier.of("placeholder")); + } + + return Streams.stream(client.listTables(databaseName(namespace))) + .map(BigLakeCatalog::tableIdentifier) + .collect(ImmutableList.toImmutableList()); + } + + @Override + public boolean dropTable(TableIdentifier identifier, boolean purge) { + TableOperations ops = newTableOps(identifier); + // TODO: to catch NotFoundException as in https://github.com/apache/iceberg/pull/5510. + TableMetadata lastMetadata = ops.current(); + try { + client.deleteTable(tableName(databaseId(identifier.namespace()), identifier.name())); + } catch (NoSuchTableException e) { + return false; + } + + if (purge && lastMetadata != null) { + CatalogUtil.dropTableData(ops.io(), lastMetadata); + } + + return true; + } + + @Override + public void renameTable(TableIdentifier from, TableIdentifier to) { + String fromDbId = databaseId(from.namespace()); + String toDbId = databaseId(to.namespace()); + + Preconditions.checkArgument( + fromDbId.equals(toDbId), + "Cannot rename table %s to %s: database must match", + from.toString(), + to.toString()); + client.renameTable(tableName(fromDbId, from.name()), tableName(toDbId, to.name())); + } + + @Override + public void createNamespace(Namespace namespace, Map<String, String> metadata) { + if (namespace.levels().length == 0) { + // Used by `CREATE NAMESPACE <catalog>`. Create a BLMS catalog linked with Iceberg catalog. + client.createCatalog(catalogName, Catalog.getDefaultInstance()); + LOG.info("Created BigLake catalog: {}", catalogName.toString()); + } else if (namespace.levels().length == 1) { + // Create a database. + String dbId = databaseId(namespace); + Database.Builder builder = Database.newBuilder().setType(Database.Type.HIVE); + builder + .getHiveOptionsBuilder() + .putAllParameters(metadata) + .setLocationUri(databaseLocation(dbId)); + + client.createDatabase(DatabaseName.of(projectId, region, catalogId, dbId), builder.build()); + } else { + throw new IllegalArgumentException( + String.format("Invalid namespace (too long): %s", namespace)); + } + } + + @Override + public List<Namespace> listNamespaces(Namespace namespace) { + if (namespace.levels().length != 0) { + // BLMS does not support namespaces under database or tables, returns empty. + // It is called when dropping a namespace to make sure it's empty (listTables is called as + // well), returns empty to unblock deletion. + return ImmutableList.of(); + } + + return Streams.stream(client.listDatabases(catalogName)) + .map(BigLakeCatalog::namespace) + .collect(ImmutableList.toImmutableList()); + } + + @Override + public boolean dropNamespace(Namespace namespace) { + try { + if (namespace.levels().length == 0) { + // Used by `DROP NAMESPACE <catalog>`. Deletes the BLMS catalog linked by Iceberg catalog. + client.deleteCatalog(catalogName); + LOG.info("Deleted BigLake catalog: {}", catalogName.toString()); + } else if (namespace.levels().length == 1) { + client.deleteDatabase(databaseName(namespace)); + // Don't delete the data file folder for safety. It aligns with HMS's default behavior. + // To support database or catalog level config controlling file deletion in future. + } else { + LOG.warn("Invalid namespace (too long): %s", namespace); + return false; + } + } catch (NoSuchNamespaceException e) { + return false; + } + + return true; + } + + @Override + public boolean setProperties(Namespace namespace, Map<String, String> properties) { + HiveDatabaseOptions.Builder optionsBuilder = + loadDatabase(namespace).toBuilder().getHiveOptionsBuilder(); + properties.forEach(optionsBuilder::putParameters); + client.updateDatabaseParameters(databaseName(namespace), optionsBuilder.getParametersMap()); + return true; + } + + @Override + public boolean removeProperties(Namespace namespace, Set<String> properties) { + HiveDatabaseOptions.Builder optionsBuilder = + loadDatabase(namespace).toBuilder().getHiveOptionsBuilder(); + properties.forEach(optionsBuilder::removeParameters); + client.updateDatabaseParameters(databaseName(namespace), optionsBuilder.getParametersMap()); + return true; + } + + @Override + public Map<String, String> loadNamespaceMetadata(Namespace namespace) { + if (namespace.levels().length == 0) { Review Comment: `namespace.isEmpty()` ########## gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java: ########## @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import com.google.cloud.bigquery.biglake.v1.Catalog; +import com.google.cloud.bigquery.biglake.v1.CatalogName; +import com.google.cloud.bigquery.biglake.v1.Database; +import com.google.cloud.bigquery.biglake.v1.DatabaseName; +import com.google.cloud.bigquery.biglake.v1.HiveDatabaseOptions; +import com.google.cloud.bigquery.biglake.v1.Table; +import com.google.cloud.bigquery.biglake.v1.TableName; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.iceberg.BaseMetastoreCatalog; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.MetadataTableType; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.ServiceFailureException; +import org.apache.iceberg.gcp.GCPProperties; +import org.apache.iceberg.hadoop.Configurable; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.ResolvingFileIO; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.base.Strings; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.relocated.com.google.common.collect.Streams; +import org.apache.iceberg.util.LocationUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Iceberg BigLake Metastore (BLMS) Catalog implementation. */ +public final class BigLakeCatalog extends BaseMetastoreCatalog + implements SupportsNamespaces, Configurable<Object> { + + public static final String DEFAULT_BIGLAKE_SERVICE_ENDPOINT = "biglake.googleapis.com:443"; + + private static final Logger LOG = LoggerFactory.getLogger(BigLakeCatalog.class); + + // The name of this Iceberg catalog plugin: spark.sql.catalog.<catalog_plugin>. + private String catalogPulginName; + private Map<String, String> properties; + private FileIO io; + private Object conf; + private String projectId; + private String region; + // BLMS catalog ID and fully qualified name. + private String catalogId; + private CatalogName catalogName; + private BigLakeClient client; + + // Must have a no-arg constructor to be dynamically loaded + // initialize(String name, Map<String, String> properties) will be called to complete + // initialization + public BigLakeCatalog() {} + + @Override + public void initialize(String inputName, Map<String, String> properties) { + Preconditions.checkArgument( + properties.containsKey(GCPProperties.BIGLAKE_PROJECT_ID), + "GCP project ID must be specified"); + String projectId = properties.get(GCPProperties.BIGLAKE_PROJECT_ID); + + Preconditions.checkArgument( + properties.containsKey(GCPProperties.BIGLAKE_PROJECT_ID), "GCP region must be specified"); + String region = properties.get(GCPProperties.BIGLAKE_GCP_REGION); + + BigLakeClient client; + try { + // TODO: to add more auth options of the client. Currently it uses default auth + // (https://github.com/googleapis/google-cloud-java#application-default-credentials) + // that works on GCP services (e.g., GCE, GKE, Dataproc). + client = + new BigLakeClient( + properties.getOrDefault( + GCPProperties.BIGLAKE_ENDPOINT, DEFAULT_BIGLAKE_SERVICE_ENDPOINT), + projectId, + region); + } catch (IOException e) { + throw new ServiceFailureException(e, "Creating BigLake client failed"); + } + + initialize(inputName, properties, projectId, region, client); + } + + @VisibleForTesting + void initialize( + String inputName, + Map<String, String> properties, + String projectId, + String region, + BigLakeClient client) { + this.catalogPulginName = inputName; + this.properties = ImmutableMap.copyOf(properties); + this.projectId = projectId; + this.region = region; + Preconditions.checkNotNull(client, "BigLake client must not be null"); + this.client = client; + + // Users can specify the BigLake catalog ID, otherwise catalog plugin name will be used. + // For example, "spark.sql.catalog.<plugin-name>=org.apache.iceberg.spark.SparkCatalog" + // specifies the plugin name "<plugin-name>". + this.catalogId = this.properties.getOrDefault(GCPProperties.BIGLAKE_CATALOG_ID, inputName); + this.catalogName = CatalogName.of(projectId, region, catalogId); + + String ioImpl = + this.properties.getOrDefault( + CatalogProperties.FILE_IO_IMPL, ResolvingFileIO.class.getName()); + this.io = CatalogUtil.loadFileIO(ioImpl, this.properties, conf); + } + + @Override + protected TableOperations newTableOps(TableIdentifier identifier) { + // The identifier of metadata tables is like "ns.table.files". + // Return a non-existing table in this case (empty table ID is disallowed in BigLake + // Metastore), loadTable will try loadMetadataTable. + if (identifier.namespace().levels().length > 1 + && MetadataTableType.from(identifier.name()) != null) { + return new BigLakeTableOperations( + client, io, tableName(identifier.namespace().level(0), /* tableId= */ "")); + } + + return new BigLakeTableOperations( + client, io, tableName(databaseId(identifier.namespace()), identifier.name())); + } + + @Override + protected String defaultWarehouseLocation(TableIdentifier identifier) { + String locationUri = loadDatabase(identifier.namespace()).getHiveOptions().getLocationUri(); + return String.format( + "%s/%s", + Strings.isNullOrEmpty(locationUri) + ? databaseLocation(databaseId(identifier.namespace())) + : locationUri, + identifier.name()); + } + + @Override + public List<TableIdentifier> listTables(Namespace namespace) { + // When deleting a BLMS catalog via `DROP NAMESPACE <catalog>`, this method is called for + // verifying catalog emptiness. `namespace` is empty in this case, we list databases in + // this catalog instead. + // TODO: to return all tables in all databases in a BLMS catalog instead of a "placeholder". + if (namespace.levels().length == 0) { + return Iterables.isEmpty(client.listDatabases(catalogName)) + ? ImmutableList.of() + : ImmutableList.of(TableIdentifier.of("placeholder")); + } + + return Streams.stream(client.listTables(databaseName(namespace))) + .map(BigLakeCatalog::tableIdentifier) + .collect(ImmutableList.toImmutableList()); + } + + @Override + public boolean dropTable(TableIdentifier identifier, boolean purge) { + TableOperations ops = newTableOps(identifier); + // TODO: to catch NotFoundException as in https://github.com/apache/iceberg/pull/5510. + TableMetadata lastMetadata = ops.current(); + try { + client.deleteTable(tableName(databaseId(identifier.namespace()), identifier.name())); + } catch (NoSuchTableException e) { + return false; + } + + if (purge && lastMetadata != null) { + CatalogUtil.dropTableData(ops.io(), lastMetadata); + } + + return true; + } + + @Override + public void renameTable(TableIdentifier from, TableIdentifier to) { + String fromDbId = databaseId(from.namespace()); + String toDbId = databaseId(to.namespace()); + + Preconditions.checkArgument( + fromDbId.equals(toDbId), + "Cannot rename table %s to %s: database must match", + from.toString(), + to.toString()); + client.renameTable(tableName(fromDbId, from.name()), tableName(toDbId, to.name())); + } + + @Override + public void createNamespace(Namespace namespace, Map<String, String> metadata) { + if (namespace.levels().length == 0) { + // Used by `CREATE NAMESPACE <catalog>`. Create a BLMS catalog linked with Iceberg catalog. + client.createCatalog(catalogName, Catalog.getDefaultInstance()); + LOG.info("Created BigLake catalog: {}", catalogName.toString()); + } else if (namespace.levels().length == 1) { + // Create a database. + String dbId = databaseId(namespace); + Database.Builder builder = Database.newBuilder().setType(Database.Type.HIVE); + builder + .getHiveOptionsBuilder() + .putAllParameters(metadata) + .setLocationUri(databaseLocation(dbId)); + + client.createDatabase(DatabaseName.of(projectId, region, catalogId, dbId), builder.build()); + } else { + throw new IllegalArgumentException( + String.format("Invalid namespace (too long): %s", namespace)); + } + } + + @Override + public List<Namespace> listNamespaces(Namespace namespace) { + if (namespace.levels().length != 0) { Review Comment: `!namespace.isEmpty()` ########## gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java: ########## @@ -0,0 +1,385 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; Review Comment: Iceberg is currently migrating from JUnit4 to JUnit5, so it would be great to not use any JUnit4 style assertions and rather switch directly to AssertJ: https://iceberg.apache.org/contribute/#assertj ########## gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java: ########## @@ -0,0 +1,385 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.grpc.testing.LocalChannelProvider; +import com.google.api.gax.grpc.testing.MockGrpcService; +import com.google.api.gax.grpc.testing.MockServiceHelper; +import com.google.cloud.bigquery.biglake.v1.Catalog; +import com.google.cloud.bigquery.biglake.v1.CatalogName; +import com.google.cloud.bigquery.biglake.v1.Database; +import com.google.cloud.bigquery.biglake.v1.DatabaseName; +import com.google.cloud.bigquery.biglake.v1.HiveDatabaseOptions; +import com.google.cloud.bigquery.biglake.v1.MetastoreServiceSettings; +import java.io.File; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.catalog.CatalogTests; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.gcp.GCPProperties; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.junit.Rule; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +public class BigLakeCatalogTest extends CatalogTests<BigLakeCatalog> { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + @TempDir public Path temp; + + private static final String GCP_PROJECT = "my-project"; + private static final String GCP_REGION = "us"; + private static final String CATALOG_ID = "biglake"; + + private String warehouseLocation; + + // For tests using a BigLake catalog connecting to a mocked service. + private MockMetastoreService mockMetastoreService; + private MockServiceHelper mockServiceHelper; + private LocalChannelProvider channelProvider; + private BigLakeCatalog bigLakeCatalogUsingMockService; + + // For tests using a BigLake catalog with a mocked client. + private BigLakeClient mockBigLakeClient; + private BigLakeCatalog bigLakeCatalogUsingMockClient; + + @BeforeEach + public void before() throws Exception { + mockMetastoreService = new MockMetastoreService(); + mockServiceHelper = + new MockServiceHelper( + UUID.randomUUID().toString(), Arrays.<MockGrpcService>asList(mockMetastoreService)); + mockServiceHelper.start(); + + File warehouse = temp.toFile(); + warehouseLocation = warehouse.getAbsolutePath(); + + ImmutableMap<String, String> properties = + ImmutableMap.of( + GCPProperties.BIGLAKE_PROJECT_ID, + GCP_PROJECT, + CatalogProperties.WAREHOUSE_LOCATION, + warehouseLocation); + + channelProvider = mockServiceHelper.createChannelProvider(); + MetastoreServiceSettings settings = + MetastoreServiceSettings.newBuilder() + .setTransportChannelProvider(channelProvider) + .setCredentialsProvider(NoCredentialsProvider.create()) + .build(); + + bigLakeCatalogUsingMockService = new BigLakeCatalog(); + bigLakeCatalogUsingMockService.setConf(new Configuration()); + bigLakeCatalogUsingMockService.initialize( + CATALOG_ID, + properties, + GCP_PROJECT, + GCP_REGION, + new BigLakeClient(settings, GCP_PROJECT, GCP_REGION)); + + mockBigLakeClient = mock(BigLakeClient.class); + bigLakeCatalogUsingMockClient = new BigLakeCatalog(); + bigLakeCatalogUsingMockClient.setConf(new Configuration()); + bigLakeCatalogUsingMockClient.initialize( + CATALOG_ID, properties, GCP_PROJECT, GCP_REGION, mockBigLakeClient); + } + + @AfterEach + public void after() { + mockServiceHelper.stop(); + } + + @Override + protected boolean requiresNamespaceCreate() { + return true; + } + + @Override + protected BigLakeCatalog catalog() { + return bigLakeCatalogUsingMockService; + } + + @Override + protected boolean supportsNamesWithSlashes() { + return false; + } + + @Test + public void testDefaultWarehouseWithDatabaseLocation_asExpected() { + when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) + .thenReturn( + Database.newBuilder() + .setHiveOptions(HiveDatabaseOptions.newBuilder().setLocationUri("db_folder")) + .build()); + + assertEquals( + "db_folder/table", + bigLakeCatalogUsingMockClient.defaultWarehouseLocation(TableIdentifier.of("db", "table"))); + } + + @Test + public void testDefaultWarehouseWithoutDatabaseLocation_asExpected() { + when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, "us", CATALOG_ID, "db"))) + .thenReturn( + Database.newBuilder().setHiveOptions(HiveDatabaseOptions.getDefaultInstance()).build()); + + assertEquals( + warehouseLocation + "/db.db/table", + bigLakeCatalogUsingMockClient.defaultWarehouseLocation(TableIdentifier.of("db", "table"))); + } + + @Test + public void testRenameTable_differentDatabase_fail() { + Exception exception = Review Comment: please use AssertJ-style assertions for this. This would be something like: `assertThatThrownBy(() -> client.renameTable(...)).isInstanceOf(IllegalArgumentException.class).hasMessage(...)`. ########## gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java: ########## @@ -0,0 +1,385 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.grpc.testing.LocalChannelProvider; +import com.google.api.gax.grpc.testing.MockGrpcService; +import com.google.api.gax.grpc.testing.MockServiceHelper; +import com.google.cloud.bigquery.biglake.v1.Catalog; +import com.google.cloud.bigquery.biglake.v1.CatalogName; +import com.google.cloud.bigquery.biglake.v1.Database; +import com.google.cloud.bigquery.biglake.v1.DatabaseName; +import com.google.cloud.bigquery.biglake.v1.HiveDatabaseOptions; +import com.google.cloud.bigquery.biglake.v1.MetastoreServiceSettings; +import java.io.File; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.catalog.CatalogTests; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.gcp.GCPProperties; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.junit.Rule; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +public class BigLakeCatalogTest extends CatalogTests<BigLakeCatalog> { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + @TempDir public Path temp; Review Comment: I believe this can be made `private` ########## gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java: ########## @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; Review Comment: same as above regarding JUnit4-style assertions ########## gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java: ########## @@ -0,0 +1,385 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.grpc.testing.LocalChannelProvider; +import com.google.api.gax.grpc.testing.MockGrpcService; +import com.google.api.gax.grpc.testing.MockServiceHelper; +import com.google.cloud.bigquery.biglake.v1.Catalog; +import com.google.cloud.bigquery.biglake.v1.CatalogName; +import com.google.cloud.bigquery.biglake.v1.Database; +import com.google.cloud.bigquery.biglake.v1.DatabaseName; +import com.google.cloud.bigquery.biglake.v1.HiveDatabaseOptions; +import com.google.cloud.bigquery.biglake.v1.MetastoreServiceSettings; +import java.io.File; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.catalog.CatalogTests; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.gcp.GCPProperties; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.junit.Rule; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +public class BigLakeCatalogTest extends CatalogTests<BigLakeCatalog> { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + @TempDir public Path temp; + + private static final String GCP_PROJECT = "my-project"; + private static final String GCP_REGION = "us"; + private static final String CATALOG_ID = "biglake"; + + private String warehouseLocation; + + // For tests using a BigLake catalog connecting to a mocked service. + private MockMetastoreService mockMetastoreService; + private MockServiceHelper mockServiceHelper; + private LocalChannelProvider channelProvider; + private BigLakeCatalog bigLakeCatalogUsingMockService; + + // For tests using a BigLake catalog with a mocked client. + private BigLakeClient mockBigLakeClient; + private BigLakeCatalog bigLakeCatalogUsingMockClient; + + @BeforeEach + public void before() throws Exception { + mockMetastoreService = new MockMetastoreService(); + mockServiceHelper = + new MockServiceHelper( + UUID.randomUUID().toString(), Arrays.<MockGrpcService>asList(mockMetastoreService)); + mockServiceHelper.start(); + + File warehouse = temp.toFile(); + warehouseLocation = warehouse.getAbsolutePath(); + + ImmutableMap<String, String> properties = + ImmutableMap.of( + GCPProperties.BIGLAKE_PROJECT_ID, + GCP_PROJECT, + CatalogProperties.WAREHOUSE_LOCATION, + warehouseLocation); + + channelProvider = mockServiceHelper.createChannelProvider(); + MetastoreServiceSettings settings = + MetastoreServiceSettings.newBuilder() + .setTransportChannelProvider(channelProvider) + .setCredentialsProvider(NoCredentialsProvider.create()) + .build(); + + bigLakeCatalogUsingMockService = new BigLakeCatalog(); + bigLakeCatalogUsingMockService.setConf(new Configuration()); + bigLakeCatalogUsingMockService.initialize( + CATALOG_ID, + properties, + GCP_PROJECT, + GCP_REGION, + new BigLakeClient(settings, GCP_PROJECT, GCP_REGION)); + + mockBigLakeClient = mock(BigLakeClient.class); + bigLakeCatalogUsingMockClient = new BigLakeCatalog(); + bigLakeCatalogUsingMockClient.setConf(new Configuration()); + bigLakeCatalogUsingMockClient.initialize( + CATALOG_ID, properties, GCP_PROJECT, GCP_REGION, mockBigLakeClient); + } + + @AfterEach + public void after() { + mockServiceHelper.stop(); + } + + @Override + protected boolean requiresNamespaceCreate() { + return true; + } + + @Override + protected BigLakeCatalog catalog() { + return bigLakeCatalogUsingMockService; + } + + @Override + protected boolean supportsNamesWithSlashes() { + return false; + } + + @Test + public void testDefaultWarehouseWithDatabaseLocation_asExpected() { + when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) + .thenReturn( + Database.newBuilder() + .setHiveOptions(HiveDatabaseOptions.newBuilder().setLocationUri("db_folder")) + .build()); + + assertEquals( + "db_folder/table", + bigLakeCatalogUsingMockClient.defaultWarehouseLocation(TableIdentifier.of("db", "table"))); + } + + @Test + public void testDefaultWarehouseWithoutDatabaseLocation_asExpected() { + when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, "us", CATALOG_ID, "db"))) + .thenReturn( + Database.newBuilder().setHiveOptions(HiveDatabaseOptions.getDefaultInstance()).build()); + + assertEquals( + warehouseLocation + "/db.db/table", + bigLakeCatalogUsingMockClient.defaultWarehouseLocation(TableIdentifier.of("db", "table"))); + } + + @Test + public void testRenameTable_differentDatabase_fail() { + Exception exception = + assertThrows( + IllegalArgumentException.class, + () -> + bigLakeCatalogUsingMockClient.renameTable( + TableIdentifier.of("db0", "t1"), TableIdentifier.of("db1", "t2"))); + assertEquals( + "Cannot rename table db0.t1 to db1.t2: database must match", exception.getMessage()); + } + + @Test + public void testCreateNamespace_createCatalogWhenEmptyNamespace() throws Exception { Review Comment: I believe test methods shouldn't contain `_` and rather use camelCase ########## gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java: ########## @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.gax.grpc.GrpcStatusCode; +import com.google.api.gax.rpc.AbortedException; +import com.google.cloud.bigquery.biglake.v1.HiveTableOptions; +import com.google.cloud.bigquery.biglake.v1.HiveTableOptions.StorageDescriptor; +import com.google.cloud.bigquery.biglake.v1.Table; +import com.google.cloud.bigquery.biglake.v1.TableName; +import io.grpc.Status.Code; +import java.io.File; +import java.io.IOException; +import java.util.Optional; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.TrueFileFilter; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.Schema; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.CommitFailedException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.gcp.GCPProperties; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.types.Types; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; Review Comment: same as above, new tests being added should be JUnit5: https://iceberg.apache.org/contribute/#junit4--junit5 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
