lasdf1234 commented on code in PR #11066: URL: https://github.com/apache/gravitino/pull/11066#discussion_r3239451969
########## plugins/idp-basic/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/IdpUserMetaBaseSQLProvider.java: ########## @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper.provider.base; + +import java.util.List; +import org.apache.gravitino.storage.relational.mapper.IdpUserMetaMapper; +import org.apache.gravitino.storage.relational.po.IdpUserPO; +import org.apache.ibatis.annotations.Param; + +public class IdpUserMetaBaseSQLProvider { + + public String selectIdpUser(@Param("userName") String userName) { + return "SELECT user_id as userId, user_name as userName, password_hash as passwordHash," + + " current_version as currentVersion," + + " last_version as lastVersion, deleted_at as deletedAt" + + " FROM " + + IdpUserMetaMapper.IDP_USER_TABLE_NAME + + " WHERE user_name = #{userName} AND deleted_at = 0"; + } + + public String selectIdpUsers(@Param("userNames") List<String> userNames) { + return "<script>" + + "SELECT user_id as userId, user_name as userName, password_hash as passwordHash," + + " current_version as currentVersion," + + " last_version as lastVersion, deleted_at as deletedAt" + + " FROM " + + IdpUserMetaMapper.IDP_USER_TABLE_NAME + + " WHERE deleted_at = 0 " + + "<choose>" + + "<when test='userNames != null and userNames.size() > 0'>" + + "AND user_name IN (" + + "<foreach item='item' collection='userNames' separator=','>" + + "#{item}" + + "</foreach>" + + ") " + + "</when>" + + "<otherwise>" + + "AND 1 = 0 " + + "</otherwise>" + + "</choose>" + + "</script>"; + } + + public String insertIdpUser(@Param("userMeta") IdpUserPO userPO) { + return "INSERT INTO " + + IdpUserMetaMapper.IDP_USER_TABLE_NAME + + " (user_id, user_name, password_hash, current_version, last_version, deleted_at)" + + " VALUES (" + + " #{userMeta.userId}," + + " #{userMeta.userName}," + + " #{userMeta.passwordHash}," + + " #{userMeta.currentVersion}," + + " #{userMeta.lastVersion}," + + " #{userMeta.deletedAt}" + + " )"; + } + + public String updateIdpUserPassword( + @Param("userId") Long userId, + @Param("passwordHash") String passwordHash, + @Param("currentVersion") Long currentVersion, + @Param("newCurrentVersion") Long newCurrentVersion, + @Param("newLastVersion") Long newLastVersion) { + return "UPDATE " + + IdpUserMetaMapper.IDP_USER_TABLE_NAME + + " SET password_hash = #{passwordHash}," + + " current_version = #{newCurrentVersion}," + + " last_version = #{newLastVersion}" + + " WHERE user_id = #{userId}" + + " AND current_version = #{currentVersion}" + + " AND deleted_at = 0"; + } + + public String softDeleteIdpUser( + @Param("userId") Long userId, @Param("deletedAt") Long deletedAt) { + return "UPDATE " + + IdpUserMetaMapper.IDP_USER_TABLE_NAME + + " SET deleted_at = #{deletedAt}," Review Comment: Resolved ########## plugins/idp-basic/src/test/java/org/apache/gravitino/storage/relational/mapper/IdpMapperTestBase.java: ########## @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Comparator; +import java.util.UUID; +import java.util.stream.Stream; +import org.apache.gravitino.Config; +import org.apache.gravitino.Configs; +import org.apache.gravitino.storage.relational.JDBCBackend; +import org.apache.gravitino.storage.relational.po.IdpUserPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.session.SqlSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; + +abstract class IdpMapperTestBase { + private static final String JDBC_STORE_PATH = + "/tmp/gravitino_jdbc_idpMappers_" + UUID.randomUUID().toString().replace("-", ""); + private static final String DB_DIR = JDBC_STORE_PATH + "/testdb"; + + protected static JDBCBackend backend; + protected static SqlSession sharedSession; + protected static IdpUserMetaMapper idpUserMetaMapper; + + @BeforeAll + static void setup() throws Exception { + File dir = new File(DB_DIR); + if (dir.exists()) { + deleteDirectory(dir.toPath()); + } + dir.mkdirs(); Review Comment: Resolved ########## plugins/idp-basic/src/main/java/org/apache/gravitino/storage/relational/mapper/IdpUserMetaSQLProviderFactory.java: ########## @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.provider.base.IdpUserMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.mapper.provider.h2.IdpUserMetaH2Provider; +import org.apache.gravitino.storage.relational.mapper.provider.postgresql.IdpUserMetaPostgreSQLProvider; +import org.apache.gravitino.storage.relational.po.IdpUserPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class IdpUserMetaSQLProviderFactory { + private static final Map<JDBCBackendType, IdpUserMetaBaseSQLProvider> + IDP_USER_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new IdpUserMetaMySQLProvider(), + JDBCBackendType.H2, new IdpUserMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new IdpUserMetaPostgreSQLProvider()); + + public static IdpUserMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return IDP_USER_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } Review Comment: The current implementation matches many existing *SQLProviderFactory classes in the repo, such as TableColumnSQLProviderFactory.In normal execution, databaseId is controlled and should not be missing ########## plugins/idp-basic/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/IdpUserMetaPostgreSQLProvider.java: ########## @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper.provider.postgresql; + +import org.apache.gravitino.storage.relational.mapper.IdpUserMetaMapper; +import org.apache.gravitino.storage.relational.mapper.provider.base.IdpUserMetaBaseSQLProvider; +import org.apache.ibatis.annotations.Param; + +public class IdpUserMetaPostgreSQLProvider extends IdpUserMetaBaseSQLProvider { + + @Override + public String softDeleteIdpUser(Long userId) { + return "UPDATE " + + IdpUserMetaMapper.IDP_USER_TABLE_NAME + + " SET deleted_at = CAST(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) * 1000 AS BIGINT)," + + " current_version = current_version + 1," + + " last_version = last_version + 1" + + " WHERE user_id = #{userId} AND deleted_at = 0"; + } + + @Override + public String deleteIdpUserMetasByLegacyTimeline(Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + IdpUserMetaMapper.IDP_USER_TABLE_NAME + + " WHERE user_id IN (SELECT user_id FROM " + + IdpUserMetaMapper.IDP_USER_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit})"; Review Comment: The same pattern already exists broadly in core and in sibling providers.The intent of these cleanup methods is to delete any batch of eligible soft-deleted rows, not a specifically ordered batch. ########## plugins/idp-basic/src/main/java/org/apache/gravitino/storage/relational/mapper/IdpUserMetaMapper.java: ########## @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import java.util.List; +import org.apache.gravitino.storage.relational.po.IdpUserPO; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; + +/** + * A MyBatis Mapper for table meta operation SQLs. + * + * <p>This interface class is a specification defined by MyBatis. It requires this interface class + * to identify the corresponding SQLs for execution. We can write SQLs in an additional XML file, or + * write SQLs with annotations in this interface Mapper. See: <a + * href="https://mybatis.org/mybatis-3/getting-started.html";></a> Review Comment: Resolved, I have modified doc. ########## plugins/idp-basic/src/test/java/org/apache/gravitino/storage/relational/mapper/IdpMapperTestBase.java: ########## @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Comparator; +import java.util.UUID; +import java.util.stream.Stream; +import org.apache.gravitino.Config; +import org.apache.gravitino.Configs; +import org.apache.gravitino.integration.test.util.BaseIT; +import org.apache.gravitino.integration.test.util.CloseContainerExtension; +import org.apache.gravitino.integration.test.util.PrintFuncNameExtension; +import org.apache.gravitino.storage.relational.JDBCBackend; +import org.apache.gravitino.storage.relational.po.IdpUserPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.session.SqlSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.extension.ExtendWith; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@ExtendWith({PrintFuncNameExtension.class, CloseContainerExtension.class}) +abstract class IdpMapperTestBase { + private final BaseIT baseIT = new BaseIT(); + private Path h2Path; + + protected String backendType; + protected JDBCBackend backend; + protected SqlSession sharedSession; + protected IdpUserMetaMapper idpUserMetaMapper; + + @BeforeAll + void startBackend() throws SQLException { + backendType = backendType(); + backend = createBackend(backendType); + } + + @BeforeEach + void openSession() { + sharedSession = SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true); + idpUserMetaMapper = sharedSession.getMapper(IdpUserMetaMapper.class); + truncateTables(); + } + + @AfterEach + void closeSession() { + if (sharedSession != null) { + sharedSession.close(); + sharedSession = null; + } + } + + @AfterAll + void stopBackend() throws IOException { + SqlSessionFactoryHelper.getInstance().close(); + if (backend != null) { + backend.close(); + backend = null; + } + + if (h2Path != null && Files.exists(h2Path)) { + deleteDirectory(h2Path); + h2Path = null; + } + } + + void truncateTables() { + try (SqlSession sqlSession = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true)) { + try (Connection connection = sqlSession.getConnection(); + Statement statement = connection.createStatement()) { + if ("postgresql".equalsIgnoreCase(backendType)) { + statement.execute("TRUNCATE TABLE idp_user_meta RESTART IDENTITY CASCADE"); + } else { + statement.execute("TRUNCATE TABLE idp_user_meta"); + } + } + } catch (SQLException e) { + throw new RuntimeException("Truncate table failed", e); + } + } + + protected IdpUserPO insertUser( + long userId, + String userName, + String passwordHash, + long currentVersion, + long lastVersion, + long deletedAt) { + IdpUserPO userPO = + IdpUserPO.builder() + .withUserId(userId) + .withUserName(userName) + .withPasswordHash(passwordHash) + .withCurrentVersion(currentVersion) + .withLastVersion(lastVersion) + .withDeletedAt(deletedAt) + .build(); + idpUserMetaMapper.insertIdpUser(userPO); + return userPO; + } + + protected long queryLongValue(String table, String column, String idColumn, long idValue) { + try (SqlSession sqlSession = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true)) { + try (Connection connection = sqlSession.getConnection()) { + String query = "SELECT " + column + " FROM " + table + " WHERE " + idColumn + " = ?"; + try (PreparedStatement statement = connection.prepareStatement(query)) { + statement.setLong(1, idValue); + try (ResultSet resultSet = statement.executeQuery()) { + assertTrue(resultSet.next()); + return resultSet.getLong(1); + } + } + } + } catch (SQLException e) { + throw new RuntimeException("Query " + column + " from " + table + " failed", e); + } + } + + protected int countRows(String table, String idColumn, long idValue) { + try (SqlSession sqlSession = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true)) { + try (Connection connection = sqlSession.getConnection()) { + String query = "SELECT COUNT(1) FROM " + table + " WHERE " + idColumn + " = ?"; + try (PreparedStatement statement = connection.prepareStatement(query)) { + statement.setLong(1, idValue); + try (ResultSet resultSet = statement.executeQuery()) { + assertTrue(resultSet.next()); + return resultSet.getInt(1); + } + } + } + } catch (SQLException e) { + throw new RuntimeException("Count rows from " + table + " failed", e); + } + } + + protected abstract String backendType(); + + private JDBCBackend createBackend(String backendType) throws SQLException { + Config config = new Config(false) {}; + config.set(Configs.ENTITY_STORE, Configs.RELATIONAL_ENTITY_STORE); + config.set(Configs.ENTITY_RELATIONAL_STORE, backendType); + config.set(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_MAX_CONNECTIONS, 20); + config.set(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_WAIT_MILLISECONDS, 1000L); + + if ("mysql".equals(backendType)) { + config.set(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL, baseIT.startAndInitMySQLBackend()); + config.set(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER, "root"); + config.set(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD, "root"); + config.set(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER, "com.mysql.cj.jdbc.Driver"); + } else if ("postgresql".equals(backendType)) { + config.set(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL, baseIT.startAndInitPGBackend()); + config.set(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER, "root"); + config.set(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD, "root"); + config.set(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER, "org.postgresql.Driver"); + } else { + String jdbcStorePath = + "/tmp/gravitino_jdbc_idpMappers_" + UUID.randomUUID().toString().replace("-", ""); + h2Path = Path.of(jdbcStorePath); + try { + Files.createDirectories(h2Path); + } catch (IOException e) { + throw new RuntimeException("Create H2 test directory failed: " + h2Path, e); + } + + config.set( + Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL, + String.format("jdbc:h2:file:%s/testdb;DB_CLOSE_DELAY=-1;MODE=MYSQL", jdbcStorePath)); Review Comment: It has been changed to the default temporary directory creation method provided by JDK. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
