dimas-b commented on code in PR #1287: URL: https://github.com/apache/polaris/pull/1287#discussion_r2054856001
########## extension/persistence/relational-jdbc/build.gradle.kts: ########## @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { id("polaris-server") } + +dependencies { + implementation(project(":polaris-core")) + implementation(libs.slf4j.api) + implementation("com.google.guava:guava:33.0.0-jre") // Use the latest version Review Comment: please move version to the toml file ########## extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/DatasourceOperations.java: ########## @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.extension.persistence.relational.jdbc; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import jakarta.annotation.Nonnull; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; +import java.util.function.Predicate; +import javax.sql.DataSource; +import javax.swing.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DatasourceOperations { + private static final Logger LOGGER = LoggerFactory.getLogger(DatasourceOperations.class); + + private static final String ALREADY_EXISTS_STATE_POSTGRES = "42P07"; + private static final String CONSTRAINT_VIOLATION_SQL_CODE = "23505"; + + private final DataSource datasource; + + public DatasourceOperations(DataSource datasource) { + this.datasource = datasource; + } + + public void executeScript(String scriptFilePath) throws SQLException { + ClassLoader classLoader = DatasourceOperations.class.getClassLoader(); + try (Connection connection = borrowConnection(); + Statement statement = connection.createStatement()) { + boolean autoCommit = connection.getAutoCommit(); + connection.setAutoCommit(true); + BufferedReader reader = + new BufferedReader( + new InputStreamReader( + Objects.requireNonNull(classLoader.getResourceAsStream(scriptFilePath)), UTF_8)); + StringBuilder sqlBuffer = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + line = line.trim(); + if (!line.isEmpty() && !line.startsWith("--")) { // Ignore empty lines and comments + sqlBuffer.append(line).append("\n"); + if (line.endsWith(";")) { // Execute statement when semicolon is found + String sql = sqlBuffer.toString().trim(); + try { + int rowsUpdated = statement.executeUpdate(sql); + LOGGER.debug("Query {} executed {} rows affected", sql, rowsUpdated); + } catch (SQLException e) { + LOGGER.error("Error executing query {}", sql, e); + // re:throw this as unhandled exception + throw new RuntimeException(e); + } + sqlBuffer.setLength(0); // Clear the buffer for the next statement + } + } + } + connection.setAutoCommit(autoCommit); + } catch (IOException e) { + LOGGER.error("Error reading the script file", e); + throw new RuntimeException(e); + } catch (SQLException e) { + LOGGER.error("Error executing the script file", e); + throw e; + } + } + + public <T, R> List<R> executeSelect( + @Nonnull String query, + @Nonnull Class<T> targetClass, + @Nonnull Function<T, R> transformer, + Predicate<R> entityFilter, + int limit) + throws SQLException { + try (Connection connection = borrowConnection(); + Statement statement = connection.createStatement(); + ResultSet s = statement.executeQuery(query)) { + return ResultSetToObjectConverter.collect(s, targetClass, transformer, entityFilter, limit); + } catch (SQLException e) { + LOGGER.error("Error executing query {}", query, e); + throw e; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public int executeUpdate(String query) throws SQLException { + try (Connection connection = borrowConnection(); + Statement statement = connection.createStatement()) { + boolean autoCommit = connection.getAutoCommit(); + connection.setAutoCommit(true); + int result = statement.executeUpdate(query); + connection.setAutoCommit(autoCommit); + return result; + } catch (SQLException e) { + LOGGER.error("Error executing query {}", query, e); + throw e; + } + } + + public int executeUpdate(String query, Statement statement) throws SQLException { + LOGGER.debug("Executing query {} within transaction", query); + try { + return statement.executeUpdate(query); + } catch (SQLException e) { + LOGGER.error("Error executing query {}", query, e); + throw e; + } + } + + public void runWithinTransaction(TransactionCallback callback) throws SQLException { + try (Connection connection = borrowConnection()) { + boolean autoCommit = connection.getAutoCommit(); + connection.setAutoCommit(false); + boolean success = false; + try { + try (Statement statement = connection.createStatement()) { + success = callback.execute(statement); + } + } finally { + if (success) { + connection.commit(); + } else { + connection.rollback(); + } + connection.setAutoCommit(autoCommit); + } + } catch (SQLException e) { + LOGGER.error("Caught Error while executing transaction", e); Review Comment: I see a "thumbs up" reaction on my comment, but the code is unchanged :sweat_smile: Do you prefer to keep this as an ERROR log? Same concern about lines 160 an d138. ########## extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java: ########## @@ -0,0 +1,303 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.extension.persistence.relational.jdbc; + +import io.smallrye.common.annotation.Identifier; +import jakarta.annotation.Nullable; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; +import javax.sql.DataSource; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntityConstants; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.persistence.BasePersistence; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.PrincipalSecretsGenerator; +import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.core.persistence.cache.EntityCache; +import org.apache.polaris.core.persistence.dao.entity.BaseResult; +import org.apache.polaris.core.persistence.dao.entity.EntityResult; +import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult; +import org.apache.polaris.core.persistence.transactional.TransactionalMetaStoreManagerImpl; +import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; +import org.apache.polaris.core.storage.cache.StorageCredentialCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The implementation of Configuration interface for configuring the {@link PolarisMetaStoreManager} + * using a JDBC backed by SQL metastore. TODO: refactor - <a + * href="https://github.com/apache/polaris/pull/1287/files#r2047487588";>...</a> + */ +@ApplicationScoped +@Identifier("relational-jdbc") +public class JdbcMetaStoreManagerFactory implements MetaStoreManagerFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(JdbcMetaStoreManagerFactory.class); + + final Map<String, PolarisMetaStoreManager> metaStoreManagerMap = new HashMap<>(); + final Map<String, StorageCredentialCache> storageCredentialCacheMap = new HashMap<>(); + final Map<String, EntityCache> entityCacheMap = new HashMap<>(); + final Map<String, Supplier<BasePersistence>> sessionSupplierMap = new HashMap<>(); + protected final PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); + // TODO: Pending discussion of if we should have one Database per realm or 1 schema per realm + // or realm should be a primary key on all the tables. + @Inject DataSource dataSource; + @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; + + protected JdbcMetaStoreManagerFactory() {} + + protected PrincipalSecretsGenerator secretsGenerator( + RealmContext realmContext, @Nullable RootCredentialsSet rootCredentialsSet) { + if (rootCredentialsSet != null) { + return PrincipalSecretsGenerator.bootstrap( + realmContext.getRealmIdentifier(), rootCredentialsSet); + } else { + return PrincipalSecretsGenerator.RANDOM_SECRETS; + } + } + + protected PolarisMetaStoreManager createNewMetaStoreManager() { + return new TransactionalMetaStoreManagerImpl(); + } + + private void initializeForRealm( + RealmContext realmContext, RootCredentialsSet rootCredentialsSet, boolean isBootstrap) { + DatasourceOperations databaseOperations = getDatasourceOperations(isBootstrap); + sessionSupplierMap.put( + realmContext.getRealmIdentifier(), + () -> + new JdbcBasePersistenceImpl( + databaseOperations, + secretsGenerator(realmContext, rootCredentialsSet), + storageIntegrationProvider, + realmContext.getRealmIdentifier())); + + PolarisMetaStoreManager metaStoreManager = createNewMetaStoreManager(); + metaStoreManagerMap.put(realmContext.getRealmIdentifier(), metaStoreManager); + } + + private DatasourceOperations getDatasourceOperations(boolean isBootstrap) { + DatasourceOperations databaseOperations = new DatasourceOperations(dataSource); + if (isBootstrap) { + // TODO: see if we need to take script from Quarkus or can we just + // use the script committed in the repo. + try { + databaseOperations.executeScript("scripts/postgres/schema-v1-postgres.sql"); Review Comment: nit: the class name suggests that all JDBC backends are supported, but this script is apparently just for PG... It would be preferable to make it pluggable/configurable. ########## extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java: ########## @@ -0,0 +1,303 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.extension.persistence.relational.jdbc; + +import io.smallrye.common.annotation.Identifier; +import jakarta.annotation.Nullable; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; +import javax.sql.DataSource; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntityConstants; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.persistence.BasePersistence; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.PrincipalSecretsGenerator; +import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.core.persistence.cache.EntityCache; +import org.apache.polaris.core.persistence.dao.entity.BaseResult; +import org.apache.polaris.core.persistence.dao.entity.EntityResult; +import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult; +import org.apache.polaris.core.persistence.transactional.TransactionalMetaStoreManagerImpl; +import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; +import org.apache.polaris.core.storage.cache.StorageCredentialCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The implementation of Configuration interface for configuring the {@link PolarisMetaStoreManager} + * using a JDBC backed by SQL metastore. TODO: refactor - <a + * href="https://github.com/apache/polaris/pull/1287/files#r2047487588";>...</a> + */ +@ApplicationScoped +@Identifier("relational-jdbc") +public class JdbcMetaStoreManagerFactory implements MetaStoreManagerFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(JdbcMetaStoreManagerFactory.class); + + final Map<String, PolarisMetaStoreManager> metaStoreManagerMap = new HashMap<>(); + final Map<String, StorageCredentialCache> storageCredentialCacheMap = new HashMap<>(); + final Map<String, EntityCache> entityCacheMap = new HashMap<>(); + final Map<String, Supplier<BasePersistence>> sessionSupplierMap = new HashMap<>(); + protected final PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); + // TODO: Pending discussion of if we should have one Database per realm or 1 schema per realm + // or realm should be a primary key on all the tables. + @Inject DataSource dataSource; + @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; + + protected JdbcMetaStoreManagerFactory() {} + + protected PrincipalSecretsGenerator secretsGenerator( + RealmContext realmContext, @Nullable RootCredentialsSet rootCredentialsSet) { + if (rootCredentialsSet != null) { + return PrincipalSecretsGenerator.bootstrap( + realmContext.getRealmIdentifier(), rootCredentialsSet); + } else { + return PrincipalSecretsGenerator.RANDOM_SECRETS; + } + } + + protected PolarisMetaStoreManager createNewMetaStoreManager() { + return new TransactionalMetaStoreManagerImpl(); + } + + private void initializeForRealm( + RealmContext realmContext, RootCredentialsSet rootCredentialsSet, boolean isBootstrap) { + DatasourceOperations databaseOperations = getDatasourceOperations(isBootstrap); + sessionSupplierMap.put( + realmContext.getRealmIdentifier(), + () -> + new JdbcBasePersistenceImpl( + databaseOperations, + secretsGenerator(realmContext, rootCredentialsSet), + storageIntegrationProvider, + realmContext.getRealmIdentifier())); + + PolarisMetaStoreManager metaStoreManager = createNewMetaStoreManager(); + metaStoreManagerMap.put(realmContext.getRealmIdentifier(), metaStoreManager); + } + + private DatasourceOperations getDatasourceOperations(boolean isBootstrap) { + DatasourceOperations databaseOperations = new DatasourceOperations(dataSource); + if (isBootstrap) { + // TODO: see if we need to take script from Quarkus or can we just + // use the script committed in the repo. + try { + databaseOperations.executeScript("scripts/postgres/schema-v1-postgres.sql"); Review Comment: nit: I believe it would be nicer to do script execution outside the hot call path (and remove the `if` above), but it's ok to refactor that later. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@polaris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
