Samrat002 commented on code in PR #191: URL: https://github.com/apache/flink-connector-aws/pull/191#discussion_r2045157187
########## flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/GlueCatalogFactory.java: ########## @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue; + +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; +import org.apache.flink.table.catalog.Catalog; +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.factories.CatalogFactory; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class GlueCatalogFactory implements CatalogFactory { + + // Define configuration options that users must provide + public static final ConfigOption<String> REGION = + ConfigOptions.key("region") + .stringType() + .noDefaultValue() + .withDescription("AWS region for the Glue catalog"); + + public static final ConfigOption<String> DEFAULT_DATABASE = + ConfigOptions.key("default-database") + .stringType() + .defaultValue("default") + .withDescription("Default database to use in Glue catalog"); + + @Override + public String factoryIdentifier() { + return "glue"; // This allows users to use `type = 'glue'` in SQL Review Comment: nit: redundant comment . ########## flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/operations/GlueDatabaseOperations.java: ########## @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.operations; + +import org.apache.flink.table.catalog.CatalogDatabase; +import org.apache.flink.table.catalog.CatalogDatabaseImpl; +import org.apache.flink.table.catalog.exceptions.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.model.*; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Handles all database-related operations for the Glue catalog. + * Provides functionality for listing, retrieving, creating, and deleting databases in AWS Glue. + */ +public class GlueDatabaseOperations extends AbstractGlueOperations { + + /** Logger for logging database operations. */ + private static final Logger LOG = LoggerFactory.getLogger(GlueDatabaseOperations.class); + + /** + * Constructor for GlueDatabaseOperations. + * Initializes the Glue client and catalog name. + * + * @param glueClient The Glue client to interact with AWS Glue. + * @param catalogName The name of the catalog. + */ + public GlueDatabaseOperations(GlueClient glueClient, String catalogName) { + super(glueClient, catalogName); + } + + /** + * Lists all the databases in the Glue catalog. + * + * @return A list of database names. + * @throws CatalogException if there is an error fetching the list of databases. + */ + public List<String> listDatabases() throws CatalogException { + try { + LOG.debug("Listing databases from Glue catalog"); + return glueClient.getDatabases(GetDatabasesRequest.builder().build()) + .databaseList() + .stream() + .map(Database::name) + .collect(Collectors.toList()); + } catch (OperationTimeoutException e) { + LOG.error("Timeout while listing databases in Glue", e); + throw new CatalogException("Timeout while listing databases: " + e.getMessage(), e); + } catch (ResourceNumberLimitExceededException e) { + LOG.error("Resource limit exceeded while listing databases in Glue", e); + throw new CatalogException("Resource limit exceeded while listing databases: " + e.getMessage(), e); + } catch (GlueException e) { + LOG.error("Failed to list databases in Glue", e); + throw new CatalogException("Failed to list databases: " + e.getMessage(), e); + } + } + + /** + * Retrieves the specified database from the Glue catalog. + * + * @param databaseName The name of the database to fetch. + * @return The CatalogDatabase object representing the Glue database. + * @throws DatabaseNotExistException If the database does not exist in the Glue catalog. + * @throws CatalogException If there is any error retrieving the database. + */ + public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException { + try { + GetDatabaseResponse response = glueClient.getDatabase( + GetDatabaseRequest.builder() + .name(databaseName) + .build() + ); + + Database glueDatabase = response.database(); + return convertGlueDatabase(glueDatabase); + } catch (EntityNotFoundException e) { + throw new DatabaseNotExistException(catalogName, databaseName); + } catch (InvalidInputException e) { + LOG.error("Invalid input while getting database: {}", databaseName, e); + throw new CatalogException("Invalid database name: " + databaseName, e); + } catch (OperationTimeoutException e) { + LOG.error("Timeout while getting database: {}", databaseName, e); + throw new CatalogException("Timeout while getting database: " + databaseName, e); + } catch (GlueException e) { + LOG.error("Error getting database: {}", databaseName, e); + throw new CatalogException("Error getting database: " + databaseName, e); + } + } + + /** + * Converts the Glue database model to a Flink CatalogDatabase. + * + * @param glueDatabase The Glue database model. + * @return A CatalogDatabase representing the Glue database. + */ + private CatalogDatabase convertGlueDatabase(Database glueDatabase) { + Map<String, String> properties = new HashMap<>(glueDatabase.parameters()); + return new CatalogDatabaseImpl( + properties, + glueDatabase.description() + ); + } + + /** + * Checks whether a database exists in Glue. + * + * @param databaseName The name of the database to check. + * @return true if the database exists, false otherwise. + */ + public boolean glueDatabaseExists(String databaseName) { Review Comment: should be a private method ########## flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/operations/GlueTableOperations.java: ########## @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.operations; + +import org.apache.flink.table.catalog.CatalogTable; +import org.apache.flink.table.catalog.ObjectPath; +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.catalog.exceptions.TableNotExistException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.model.*; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * Handles all table-related operations for the Glue catalog. + * Provides functionality for checking existence, listing, creating, getting, and dropping tables in AWS Glue. + */ +public class GlueTableOperations extends AbstractGlueOperations { + + /** + * Logger for logging table operations. + */ + private static final Logger LOG = LoggerFactory.getLogger(GlueTableOperations.class); + + /** + * Constructor for GlueTableOperations. + * Initializes the Glue client and catalog name. + * + * @param glueClient The Glue client to interact with AWS Glue. + * @param catalogName The name of the catalog. + */ + public GlueTableOperations(GlueClient glueClient, String catalogName) { + super(glueClient, catalogName); + } + + /** + * Checks whether a table exists in the Glue catalog. + * + * @param databaseName The name of the database where the table should exist. + * @param tableName The name of the table to check. + * @return true if the table exists, false otherwise. + */ + public boolean glueTableExists(String databaseName, String tableName) { + try { + glueClient.getTable(builder -> builder.databaseName(databaseName).name(tableName)); + return true; + } catch (EntityNotFoundException e) { + return false; + } catch (InvalidInputException | OperationTimeoutException | ResourceNumberLimitExceededException e) { + throw new CatalogException("Error checking table existence: " + databaseName + "." + tableName, e); + } catch (GlueException e) { + throw new CatalogException("Error checking table existence: " + databaseName + "." + tableName, e); + } + } + + /** + * Lists all tables in a given database. + * + * @param databaseName The name of the database from which to list tables. + * @return A list of table names. + * @throws CatalogException if there is an error fetching the table list. + */ + public List<String> listTables(String databaseName) { + try { + GetTablesRequest request = GetTablesRequest.builder() + .databaseName(databaseName) + .build(); + GetTablesResponse response = glueClient.getTables(request); + return response.tableList().stream() + .map(Table::name) + .collect(Collectors.toList()); + } catch (InvalidInputException e) { Review Comment: Use paginated token based request to fetch tables from glue . see the code here -> https://github.com/apache/flink-connector-aws/pull/47/files#diff-71f77e6798d0a666c50a5fce09787991e9a299dfc82f5b8f20ccf9f9e34cf66fR204 ########## flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/GlueCatalogFactory.java: ########## @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue; + +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; +import org.apache.flink.table.catalog.Catalog; +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.factories.CatalogFactory; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class GlueCatalogFactory implements CatalogFactory { + + // Define configuration options that users must provide + public static final ConfigOption<String> REGION = + ConfigOptions.key("region") + .stringType() + .noDefaultValue() + .withDescription("AWS region for the Glue catalog"); + + public static final ConfigOption<String> DEFAULT_DATABASE = + ConfigOptions.key("default-database") + .stringType() + .defaultValue("default") + .withDescription("Default database to use in Glue catalog"); + + @Override + public String factoryIdentifier() { + return "glue"; // This allows users to use `type = 'glue'` in SQL + } + + @Override + public Set<ConfigOption<?>> requiredOptions() { + Set<ConfigOption<?>> options = new HashSet<>(); + options.add(REGION); + return options; + } + + @Override + public Set<ConfigOption<?>> optionalOptions() { + Set<ConfigOption<?>> options = new HashSet<>(); + options.add(DEFAULT_DATABASE); + return options; + } + + @Override + public Catalog createCatalog(Context context) { + // Read the configuration options + Map<String,String> config = context.getOptions(); + String name = context.getName(); + String region = config.get("region"); Review Comment: `"region"` && `default-database` can be moved to final constants in the class ########## flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/GlueCatalogFactory.java: ########## @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue; + +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; +import org.apache.flink.table.catalog.Catalog; +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.factories.CatalogFactory; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class GlueCatalogFactory implements CatalogFactory { + + // Define configuration options that users must provide + public static final ConfigOption<String> REGION = + ConfigOptions.key("region") + .stringType() + .noDefaultValue() + .withDescription("AWS region for the Glue catalog"); + + public static final ConfigOption<String> DEFAULT_DATABASE = + ConfigOptions.key("default-database") + .stringType() + .defaultValue("default") + .withDescription("Default database to use in Glue catalog"); + + @Override + public String factoryIdentifier() { + return "glue"; // This allows users to use `type = 'glue'` in SQL + } + + @Override + public Set<ConfigOption<?>> requiredOptions() { + Set<ConfigOption<?>> options = new HashSet<>(); + options.add(REGION); + return options; + } + + @Override + public Set<ConfigOption<?>> optionalOptions() { + Set<ConfigOption<?>> options = new HashSet<>(); + options.add(DEFAULT_DATABASE); + return options; + } + + @Override + public Catalog createCatalog(Context context) { + // Read the configuration options Review Comment: nit: redundant comment ########## flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/operations/GlueTableOperations.java: ########## @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.operations; + +import org.apache.flink.table.catalog.CatalogTable; +import org.apache.flink.table.catalog.ObjectPath; +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.catalog.exceptions.TableNotExistException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.model.*; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * Handles all table-related operations for the Glue catalog. + * Provides functionality for checking existence, listing, creating, getting, and dropping tables in AWS Glue. + */ +public class GlueTableOperations extends AbstractGlueOperations { + + /** + * Logger for logging table operations. + */ + private static final Logger LOG = LoggerFactory.getLogger(GlueTableOperations.class); + + /** + * Constructor for GlueTableOperations. + * Initializes the Glue client and catalog name. + * + * @param glueClient The Glue client to interact with AWS Glue. + * @param catalogName The name of the catalog. + */ + public GlueTableOperations(GlueClient glueClient, String catalogName) { + super(glueClient, catalogName); + } + + /** + * Checks whether a table exists in the Glue catalog. + * + * @param databaseName The name of the database where the table should exist. + * @param tableName The name of the table to check. + * @return true if the table exists, false otherwise. + */ + public boolean glueTableExists(String databaseName, String tableName) { + try { + glueClient.getTable(builder -> builder.databaseName(databaseName).name(tableName)); + return true; Review Comment: does it make more sense to have an explicit check if table object is not null ? I am not sure if entity returned is null or matches with different tableName in same database ? glue supports only lowercase , how 2 different tablename `TestABC` is distinguised from `testAbC` ? ########## flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/operations/GlueDatabaseOperations.java: ########## @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.operations; + +import org.apache.flink.table.catalog.CatalogDatabase; +import org.apache.flink.table.catalog.CatalogDatabaseImpl; +import org.apache.flink.table.catalog.exceptions.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.model.*; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Handles all database-related operations for the Glue catalog. + * Provides functionality for listing, retrieving, creating, and deleting databases in AWS Glue. + */ +public class GlueDatabaseOperations extends AbstractGlueOperations { + + /** Logger for logging database operations. */ + private static final Logger LOG = LoggerFactory.getLogger(GlueDatabaseOperations.class); + + /** + * Constructor for GlueDatabaseOperations. + * Initializes the Glue client and catalog name. + * + * @param glueClient The Glue client to interact with AWS Glue. + * @param catalogName The name of the catalog. + */ + public GlueDatabaseOperations(GlueClient glueClient, String catalogName) { + super(glueClient, catalogName); + } + + /** + * Lists all the databases in the Glue catalog. + * + * @return A list of database names. + * @throws CatalogException if there is an error fetching the list of databases. + */ + public List<String> listDatabases() throws CatalogException { + try { + LOG.debug("Listing databases from Glue catalog"); + return glueClient.getDatabases(GetDatabasesRequest.builder().build()) Review Comment: `GetDatabasesRequest` supports paginated request . `listDatabases` operation will be slow in case of large no. of registered database. convert the listing in a loop with batch size and passing nextTokenId ########## flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/StreamingJob.java: ########## @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.EnvironmentSettings; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StreamingJob { Review Comment: Delete test job class ? this can be added to e2etest for catalog. ########## flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/operations/GlueFunctionsOperations.java: ########## @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.operations; + +import org.apache.flink.table.catalog.glue.util.GlueCatalogConstants; +import org.apache.flink.table.catalog.glue.util.GlueFunctionsUtil; +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.catalog.CatalogFunction; +import org.apache.flink.table.catalog.CatalogFunctionImpl; +import org.apache.flink.table.catalog.ObjectPath; +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.catalog.exceptions.FunctionAlreadyExistException; +import org.apache.flink.table.catalog.exceptions.FunctionNotExistException; +import org.apache.flink.table.resource.ResourceUri; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.model.AlreadyExistsException; +import software.amazon.awssdk.services.glue.model.CreateUserDefinedFunctionRequest; +import software.amazon.awssdk.services.glue.model.CreateUserDefinedFunctionResponse; +import software.amazon.awssdk.services.glue.model.DeleteUserDefinedFunctionRequest; +import software.amazon.awssdk.services.glue.model.DeleteUserDefinedFunctionResponse; +import software.amazon.awssdk.services.glue.model.EntityNotFoundException; +import software.amazon.awssdk.services.glue.model.GetUserDefinedFunctionRequest; +import software.amazon.awssdk.services.glue.model.GetUserDefinedFunctionResponse; +import software.amazon.awssdk.services.glue.model.GetUserDefinedFunctionsRequest; +import software.amazon.awssdk.services.glue.model.GetUserDefinedFunctionsResponse; +import software.amazon.awssdk.services.glue.model.GlueException; +import software.amazon.awssdk.services.glue.model.PrincipalType; +import software.amazon.awssdk.services.glue.model.UpdateUserDefinedFunctionRequest; +import software.amazon.awssdk.services.glue.model.UpdateUserDefinedFunctionResponse; +import software.amazon.awssdk.services.glue.model.UserDefinedFunction; +import software.amazon.awssdk.services.glue.model.UserDefinedFunctionInput; + +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** Utilities for Glue catalog Function related operations. */ +@Internal +public class GlueFunctionsOperations extends AbstractGlueOperations { Review Comment: IMO, function support can be part of followup pr. WDUT ? ########## flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/operations/GlueDatabaseOperations.java: ########## @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.operations; + +import org.apache.flink.table.catalog.CatalogDatabase; +import org.apache.flink.table.catalog.CatalogDatabaseImpl; +import org.apache.flink.table.catalog.exceptions.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.model.*; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Handles all database-related operations for the Glue catalog. + * Provides functionality for listing, retrieving, creating, and deleting databases in AWS Glue. + */ +public class GlueDatabaseOperations extends AbstractGlueOperations { + + /** Logger for logging database operations. */ + private static final Logger LOG = LoggerFactory.getLogger(GlueDatabaseOperations.class); + + /** + * Constructor for GlueDatabaseOperations. + * Initializes the Glue client and catalog name. + * + * @param glueClient The Glue client to interact with AWS Glue. + * @param catalogName The name of the catalog. + */ + public GlueDatabaseOperations(GlueClient glueClient, String catalogName) { + super(glueClient, catalogName); + } + + /** + * Lists all the databases in the Glue catalog. + * + * @return A list of database names. + * @throws CatalogException if there is an error fetching the list of databases. + */ + public List<String> listDatabases() throws CatalogException { + try { + LOG.debug("Listing databases from Glue catalog"); + return glueClient.getDatabases(GetDatabasesRequest.builder().build()) + .databaseList() + .stream() + .map(Database::name) + .collect(Collectors.toList()); + } catch (OperationTimeoutException e) { + LOG.error("Timeout while listing databases in Glue", e); + throw new CatalogException("Timeout while listing databases: " + e.getMessage(), e); + } catch (ResourceNumberLimitExceededException e) { + LOG.error("Resource limit exceeded while listing databases in Glue", e); + throw new CatalogException("Resource limit exceeded while listing databases: " + e.getMessage(), e); + } catch (GlueException e) { + LOG.error("Failed to list databases in Glue", e); + throw new CatalogException("Failed to list databases: " + e.getMessage(), e); + } + } + + /** + * Retrieves the specified database from the Glue catalog. + * + * @param databaseName The name of the database to fetch. + * @return The CatalogDatabase object representing the Glue database. + * @throws DatabaseNotExistException If the database does not exist in the Glue catalog. + * @throws CatalogException If there is any error retrieving the database. + */ + public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException { + try { + GetDatabaseResponse response = glueClient.getDatabase( + GetDatabaseRequest.builder() + .name(databaseName) + .build() + ); + + Database glueDatabase = response.database(); + return convertGlueDatabase(glueDatabase); + } catch (EntityNotFoundException e) { + throw new DatabaseNotExistException(catalogName, databaseName); + } catch (InvalidInputException e) { + LOG.error("Invalid input while getting database: {}", databaseName, e); + throw new CatalogException("Invalid database name: " + databaseName, e); + } catch (OperationTimeoutException e) { + LOG.error("Timeout while getting database: {}", databaseName, e); + throw new CatalogException("Timeout while getting database: " + databaseName, e); + } catch (GlueException e) { + LOG.error("Error getting database: {}", databaseName, e); + throw new CatalogException("Error getting database: " + databaseName, e); + } + } + + /** + * Converts the Glue database model to a Flink CatalogDatabase. + * + * @param glueDatabase The Glue database model. + * @return A CatalogDatabase representing the Glue database. + */ + private CatalogDatabase convertGlueDatabase(Database glueDatabase) { + Map<String, String> properties = new HashMap<>(glueDatabase.parameters()); + return new CatalogDatabaseImpl( + properties, + glueDatabase.description() + ); + } + + /** + * Checks whether a database exists in Glue. + * + * @param databaseName The name of the database to check. + * @return true if the database exists, false otherwise. + */ + public boolean glueDatabaseExists(String databaseName) { + try { + glueClient.getDatabase(builder -> builder.name(databaseName)); + return true; + } catch (EntityNotFoundException e) { + return false; + } catch (InvalidInputException | OperationTimeoutException | ResourceNumberLimitExceededException e) { + throw new CatalogException("Error checking database existence: " + databaseName, e); + } catch (GlueException e) { + throw new CatalogException("Error checking database existence: " + databaseName, e); + } + } + + /** + * Creates a new database in Glue. + * + * @param databaseName The name of the database to create. + * @param catalogDatabase The CatalogDatabase containing properties and description. + * @throws DatabaseAlreadyExistException If the database already exists. + * @throws CatalogException If there is any error creating the database. + */ + public void createDatabase(String databaseName, CatalogDatabase catalogDatabase) + throws DatabaseAlreadyExistException, CatalogException { + try { + glueClient.createDatabase(builder -> builder.databaseInput(db -> + db.name(databaseName) + .description(String.valueOf(catalogDatabase.getDescription())) + .parameters(catalogDatabase.getProperties()))); + } catch (AlreadyExistsException e) { + throw new DatabaseAlreadyExistException(catalogName, databaseName); + } catch (InvalidInputException | ResourceNumberLimitExceededException | OperationTimeoutException e) { + throw new CatalogException("Error creating database: " + databaseName, e); + } catch (GlueException e) { + throw new CatalogException("Error creating database: " + databaseName, e); + } Review Comment: Merge into one catch block ########## flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/operations/GlueDatabaseOperations.java: ########## @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.operations; + +import org.apache.flink.table.catalog.CatalogDatabase; +import org.apache.flink.table.catalog.CatalogDatabaseImpl; +import org.apache.flink.table.catalog.exceptions.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.model.*; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Handles all database-related operations for the Glue catalog. + * Provides functionality for listing, retrieving, creating, and deleting databases in AWS Glue. + */ +public class GlueDatabaseOperations extends AbstractGlueOperations { + + /** Logger for logging database operations. */ + private static final Logger LOG = LoggerFactory.getLogger(GlueDatabaseOperations.class); + + /** + * Constructor for GlueDatabaseOperations. + * Initializes the Glue client and catalog name. + * + * @param glueClient The Glue client to interact with AWS Glue. + * @param catalogName The name of the catalog. + */ + public GlueDatabaseOperations(GlueClient glueClient, String catalogName) { + super(glueClient, catalogName); + } + + /** + * Lists all the databases in the Glue catalog. + * + * @return A list of database names. + * @throws CatalogException if there is an error fetching the list of databases. + */ + public List<String> listDatabases() throws CatalogException { + try { + LOG.debug("Listing databases from Glue catalog"); + return glueClient.getDatabases(GetDatabasesRequest.builder().build()) + .databaseList() + .stream() + .map(Database::name) + .collect(Collectors.toList()); + } catch (OperationTimeoutException e) { + LOG.error("Timeout while listing databases in Glue", e); + throw new CatalogException("Timeout while listing databases: " + e.getMessage(), e); + } catch (ResourceNumberLimitExceededException e) { + LOG.error("Resource limit exceeded while listing databases in Glue", e); + throw new CatalogException("Resource limit exceeded while listing databases: " + e.getMessage(), e); + } catch (GlueException e) { + LOG.error("Failed to list databases in Glue", e); + throw new CatalogException("Failed to list databases: " + e.getMessage(), e); + } + } + + /** + * Retrieves the specified database from the Glue catalog. + * + * @param databaseName The name of the database to fetch. + * @return The CatalogDatabase object representing the Glue database. + * @throws DatabaseNotExistException If the database does not exist in the Glue catalog. + * @throws CatalogException If there is any error retrieving the database. + */ + public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException { + try { + GetDatabaseResponse response = glueClient.getDatabase( + GetDatabaseRequest.builder() + .name(databaseName) + .build() + ); + + Database glueDatabase = response.database(); + return convertGlueDatabase(glueDatabase); + } catch (EntityNotFoundException e) { + throw new DatabaseNotExistException(catalogName, databaseName); + } catch (InvalidInputException e) { + LOG.error("Invalid input while getting database: {}", databaseName, e); + throw new CatalogException("Invalid database name: " + databaseName, e); + } catch (OperationTimeoutException e) { + LOG.error("Timeout while getting database: {}", databaseName, e); + throw new CatalogException("Timeout while getting database: " + databaseName, e); + } catch (GlueException e) { + LOG.error("Error getting database: {}", databaseName, e); + throw new CatalogException("Error getting database: " + databaseName, e); + } + } + + /** + * Converts the Glue database model to a Flink CatalogDatabase. + * + * @param glueDatabase The Glue database model. + * @return A CatalogDatabase representing the Glue database. + */ + private CatalogDatabase convertGlueDatabase(Database glueDatabase) { + Map<String, String> properties = new HashMap<>(glueDatabase.parameters()); + return new CatalogDatabaseImpl( + properties, + glueDatabase.description() + ); + } + + /** + * Checks whether a database exists in Glue. + * + * @param databaseName The name of the database to check. + * @return true if the database exists, false otherwise. + */ + public boolean glueDatabaseExists(String databaseName) { + try { + glueClient.getDatabase(builder -> builder.name(databaseName)); + return true; + } catch (EntityNotFoundException e) { + return false; + } catch (InvalidInputException | OperationTimeoutException | ResourceNumberLimitExceededException e) { + throw new CatalogException("Error checking database existence: " + databaseName, e); + } catch (GlueException e) { + throw new CatalogException("Error checking database existence: " + databaseName, e); + } + } + + /** + * Creates a new database in Glue. + * + * @param databaseName The name of the database to create. + * @param catalogDatabase The CatalogDatabase containing properties and description. + * @throws DatabaseAlreadyExistException If the database already exists. + * @throws CatalogException If there is any error creating the database. + */ + public void createDatabase(String databaseName, CatalogDatabase catalogDatabase) + throws DatabaseAlreadyExistException, CatalogException { + try { + glueClient.createDatabase(builder -> builder.databaseInput(db -> + db.name(databaseName) + .description(String.valueOf(catalogDatabase.getDescription())) + .parameters(catalogDatabase.getProperties()))); + } catch (AlreadyExistsException e) { + throw new DatabaseAlreadyExistException(catalogName, databaseName); + } catch (InvalidInputException | ResourceNumberLimitExceededException | OperationTimeoutException e) { + throw new CatalogException("Error creating database: " + databaseName, e); + } catch (GlueException e) { + throw new CatalogException("Error creating database: " + databaseName, e); + } + } + + /** + * Deletes the specified database from Glue. + * + * @param databaseName The name of the database to delete. + * @throws DatabaseNotExistException If the database does not exist in the Glue catalog. + * @throws CatalogException If there is any error deleting the database. + */ + public void dropGlueDatabase(String databaseName) throws DatabaseNotExistException, CatalogException { + try { + DeleteDatabaseRequest deleteDatabaseRequest = DeleteDatabaseRequest.builder() + .name(databaseName) + .build(); + + glueClient.deleteDatabase(deleteDatabaseRequest); Review Comment: ```suggestion DeleteDatabaseResult result = glueClient.deleteDatabase(deleteDatabaseRequest); if(result.getSdkResponseMetadata().getHttpStatusCode() != 200) throw new CatalogException("Error dropping database"); ``` ########## flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/operations/GlueDatabaseOperations.java: ########## @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.operations; + +import org.apache.flink.table.catalog.CatalogDatabase; +import org.apache.flink.table.catalog.CatalogDatabaseImpl; +import org.apache.flink.table.catalog.exceptions.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.model.*; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Handles all database-related operations for the Glue catalog. + * Provides functionality for listing, retrieving, creating, and deleting databases in AWS Glue. + */ +public class GlueDatabaseOperations extends AbstractGlueOperations { + + /** Logger for logging database operations. */ + private static final Logger LOG = LoggerFactory.getLogger(GlueDatabaseOperations.class); + + /** + * Constructor for GlueDatabaseOperations. + * Initializes the Glue client and catalog name. + * + * @param glueClient The Glue client to interact with AWS Glue. + * @param catalogName The name of the catalog. + */ + public GlueDatabaseOperations(GlueClient glueClient, String catalogName) { + super(glueClient, catalogName); + } + + /** + * Lists all the databases in the Glue catalog. + * + * @return A list of database names. + * @throws CatalogException if there is an error fetching the list of databases. + */ + public List<String> listDatabases() throws CatalogException { + try { + LOG.debug("Listing databases from Glue catalog"); + return glueClient.getDatabases(GetDatabasesRequest.builder().build()) + .databaseList() + .stream() + .map(Database::name) + .collect(Collectors.toList()); + } catch (OperationTimeoutException e) { + LOG.error("Timeout while listing databases in Glue", e); + throw new CatalogException("Timeout while listing databases: " + e.getMessage(), e); + } catch (ResourceNumberLimitExceededException e) { + LOG.error("Resource limit exceeded while listing databases in Glue", e); + throw new CatalogException("Resource limit exceeded while listing databases: " + e.getMessage(), e); Review Comment: `OperationTimeoutException` `ResourceNumberLimitExceededException` can be eleminated with the help of paginated request calls ########## flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/operations/GlueTableOperations.java: ########## @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.operations; + +import org.apache.flink.table.catalog.CatalogTable; +import org.apache.flink.table.catalog.ObjectPath; +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.catalog.exceptions.TableNotExistException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.model.*; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * Handles all table-related operations for the Glue catalog. + * Provides functionality for checking existence, listing, creating, getting, and dropping tables in AWS Glue. + */ +public class GlueTableOperations extends AbstractGlueOperations { + + /** + * Logger for logging table operations. + */ + private static final Logger LOG = LoggerFactory.getLogger(GlueTableOperations.class); + + /** + * Constructor for GlueTableOperations. + * Initializes the Glue client and catalog name. + * + * @param glueClient The Glue client to interact with AWS Glue. + * @param catalogName The name of the catalog. + */ + public GlueTableOperations(GlueClient glueClient, String catalogName) { + super(glueClient, catalogName); + } + + /** + * Checks whether a table exists in the Glue catalog. + * + * @param databaseName The name of the database where the table should exist. + * @param tableName The name of the table to check. + * @return true if the table exists, false otherwise. + */ + public boolean glueTableExists(String databaseName, String tableName) { + try { + glueClient.getTable(builder -> builder.databaseName(databaseName).name(tableName)); + return true; + } catch (EntityNotFoundException e) { + return false; + } catch (InvalidInputException | OperationTimeoutException | ResourceNumberLimitExceededException e) { + throw new CatalogException("Error checking table existence: " + databaseName + "." + tableName, e); + } catch (GlueException e) { + throw new CatalogException("Error checking table existence: " + databaseName + "." + tableName, e); + } + } + + /** + * Lists all tables in a given database. + * + * @param databaseName The name of the database from which to list tables. + * @return A list of table names. + * @throws CatalogException if there is an error fetching the table list. + */ + public List<String> listTables(String databaseName) { + try { + GetTablesRequest request = GetTablesRequest.builder() + .databaseName(databaseName) + .build(); + GetTablesResponse response = glueClient.getTables(request); + return response.tableList().stream() + .map(Table::name) + .collect(Collectors.toList()); + } catch (InvalidInputException e) { + throw new CatalogException("Invalid input while listing tables: " + e.getMessage(), e); + } catch (OperationTimeoutException e) { + throw new CatalogException("Operation timed out while listing tables: " + e.getMessage(), e); + } catch (ResourceNumberLimitExceededException e) { + throw new CatalogException("Resource limit exceeded while listing tables: " + e.getMessage(), e); + } catch (GlueException e) { + throw new CatalogException("AWS Glue error while listing tables: " + e.getMessage(), e); + } + } + + /** + * Creates a new table in Glue. + * + * @param databaseName The name of the database where the table should be created. + * @param tableInput The input data for creating the table. + * @throws CatalogException if there is an error creating the table. + */ + public void createTable(String databaseName, TableInput tableInput) { + try { + CreateTableRequest request = CreateTableRequest.builder() + .databaseName(databaseName) + .tableInput(tableInput) + .build(); + glueClient.createTable(request); Review Comment: use ```suggestion var response = glueClient.createTable(request); if(response == null || !response.sdkHttpResponse().isSuccessful()) { throw new CatalogException("Error"); } ``` ########## flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/util/GlueTypeConverter.java: ########## @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.util; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Utility class for converting Flink types to Glue types and vice versa. + * Supports the conversion of common primitive, array, map, and struct types. + */ +public class GlueTypeConverter { Review Comment: UnitTest for Type Converter would be helpful for taking care of errors and bug. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
