This is an automated email from the ASF dual-hosted git repository. yuqi1129 pushed a commit to branch review-11012 in repository https://gitbox.apache.org/repos/asf/gravitino.git
commit 0d181d7196500fe9bd73f99d113941b4ef815726 Merge: 859634b682 d52363c264 Author: yuqi <[email protected]> AuthorDate: Fri May 29 20:11:05 2026 +0800 fix(core): handle concurrent import race at the dispatcher layer Move EntityAlreadyExistsException recovery from importTable/importSchema (entity-store interaction level) up to loadTable/loadSchema (dispatcher level). On an HA race the losing node now retries internalLoadTable / internalLoadSchema; if the retry confirms the entity is properly imported (same stringId) it succeeds, otherwise it still raises UnsupportedOperationException for genuine multi-catalog conflicts. Also resolve all merge conflicts in GlueCatalogOperations.java (take upstream Iceberg SDK refactor) and in the four dispatcher/test files. Co-Authored-By: Claude Sonnet 4.6 <[email protected]> .asf.yaml | 1 + .../workflows/backend-integration-test-action.yml | 8 +- .../workflows/flink-integration-test-action.yml | 22 +- .github/workflows/idp-basic-test.yml | 76 + .github/workflows/trino-integration-test.yml | 8 +- .github/workflows/trino-multi-version-test.yml | 8 +- AGENTS.md | 3 +- README.md | 2 +- .../java/org/apache/gravitino/SupportsSchemas.java | 22 + .../apache/gravitino/authorization/Privileges.java | 2 +- .../gravitino/credential/JdbcCredential.java | 116 ++ .../org.apache.gravitino.credential.Credential | 1 + .../authorization/TestSecurableObjects.java | 2 +- .../test/RangerAuthorizationPluginIT.java | 15 +- build.gradle.kts | 4 +- .../oss/credential/OSSSecretKeyProvider.java | 5 + .../gravitino/oss/credential/OSSTokenProvider.java | 5 + .../s3/credential/AwsIrsaCredentialGenerator.java | 20 +- .../s3/credential/AwsIrsaCredentialProvider.java | 7 + .../s3/credential/S3SecretKeyProvider.java | 7 + .../gravitino/s3/credential/S3TokenGenerator.java | 20 +- .../gravitino/s3/credential/S3TokenProvider.java | 7 + .../s3/credential/TestS3PolicyPrefix.java | 65 + .../abs/credential/ADLSTokenProvider.java | 8 + .../abs/credential/AzureAccountKeyProvider.java | 8 + .../gravitino/gcs/credential/GCSTokenProvider.java | 5 + .../operations/ClickHouseTableOperations.java | 16 - .../operations/ClickHouseTableSqlUtils.java | 34 +- .../integration/test/CatalogClickHouseIT.java | 46 +- .../operations/TestClickHouseTableOperations.java | 64 + .../catalog/hive/HiveStorageConstants.java | 22 +- .../fileset/TestFilesetCatalogOperations.java | 2 +- catalogs/catalog-glue/build.gradle.kts | 5 + .../catalog/glue/GlueCatalogOperations.java | 344 +++-- .../glue/GlueCatalogPropertiesMetadata.java | 14 +- .../gravitino/catalog/glue/GlueConstants.java | 22 +- .../catalog/glue/GlueIcebergTableHelper.java | 818 +++++++++++ .../apache/gravitino/catalog/glue/GlueTable.java | 49 +- .../catalog/glue/GlueTablePropertiesMetadata.java | 78 +- .../gravitino/catalog/glue/GlueTypeConverter.java | 4 +- .../glue/GravitinoGlueCredentialsProvider.java | 68 + .../catalog-glue/src/main/resources/glue.conf | 6 +- .../gravitino/catalog/glue/GlueTableTestBase.java | 3 +- .../catalog/glue/TestAwsGlueCatalogOperations.java | 234 ++++ .../catalog/glue/TestAwsGlueTableConversion.java | 14 +- .../glue/TestGlueCatalogOperationsForIceberg.java | 281 ++++ .../glue/TestGlueCatalogPropertiesMetadata.java | 12 +- .../glue/TestGlueCatalogTableOperations.java | 29 +- .../catalog/glue/TestGlueIcebergTableHelper.java | 177 +++ .../catalog/glue/TestGlueTableConversion.java | 14 +- .../integration/test/AbstractGlueCatalogIT.java | 44 +- .../glue/integration/test/AwsGlueCatalogIT.java | 74 +- .../glue/integration/test/MotoGlueCatalogIT.java | 4 +- .../catalog/hive/HiveCatalogOperations.java | 218 ++- .../hive/HiveCatalogPropertiesMetadata.java | 5 +- .../apache/gravitino/catalog/hive/HiveView.java | 159 +++ .../catalog/hive/HiveViewCatalogOperations.java | 460 ++++++ .../catalog/hive/TestHiveCatalogOperations.java | 796 +++++++++++ .../gravitino/catalog/hive/TestHiveView.java | 80 ++ .../hive/integration/test/CatalogHiveViewIT.java | 584 ++++++++ .../apache/gravitino/catalog/jdbc/JdbcCatalog.java | 27 + .../catalog/jdbc/TestJdbcCatalogCredential.java | 195 +++ .../integration/test/CatalogMysqlCredentialIT.java | 163 +++ .../mysql/integration/test/CatalogMysqlIT.java | 12 + .../converter/PostgreSqlTypeConverter.java | 12 +- .../converter/TestPostgreSqlTypeConverter.java | 15 + .../kafka/integration/test/CatalogKafkaIT.java | 74 +- .../generic/GenericCatalogOperations.java | 16 +- .../generic/GenericCatalogPropertiesMetadata.java | 11 +- .../lakehouse/lance/LanceTableDelegator.java | 6 +- .../lakehouse/lance/LanceTableOperations.java | 52 +- .../lakehouse/generic/TestPropertiesMetadata.java | 9 +- .../lakehouse/lance/TestLanceTableOperations.java | 44 +- .../test/CatalogGenericCatalogLanceIT.java | 19 +- .../catalog/lakehouse/iceberg/IcebergCatalog.java | 46 +- .../iceberg/IcebergCatalogCapability.java | 55 + .../iceberg/IcebergCatalogOperations.java | 129 +- .../iceberg/IcebergCatalogPropertiesMetadata.java | 9 +- .../catalog/lakehouse/iceberg/IcebergView.java | 133 +- .../iceberg/IcebergViewCatalogOperations.java | 446 ++++++ .../lakehouse/iceberg/converter/ConvertUtil.java | 23 + .../iceberg/ops/IcebergCatalogWrapperHelper.java | 33 +- .../lakehouse/iceberg/TestIcebergCatalog.java | 215 +++ .../iceberg/TestIcebergCatalogCapability.java | 106 ++ .../iceberg/TestIcebergCatalogOperations.java | 55 +- .../TestIcebergCatalogPropertiesMetadata.java | 75 + .../catalog/lakehouse/iceberg/TestIcebergView.java | 150 ++ .../iceberg/TestIcebergViewCatalogOperations.java | 377 +++++ .../iceberg/converter/TestConvertUtil.java | 8 + .../integration/test/CatalogIcebergBaseIT.java | 543 +++++++ .../integration/test/TestMultipleJDBCLoad.java | 12 + catalogs/catalog-lakehouse-paimon/build.gradle.kts | 34 +- .../catalog/lakehouse/paimon/PaimonCatalog.java | 49 + .../lakehouse/paimon/PaimonCatalogOperations.java | 91 +- .../paimon/PaimonTablePropertiesMetadata.java | 6 +- .../catalog/lakehouse/paimon/PaimonView.java | 221 +++ .../lakehouse/paimon/PaimonViewCatalogOps.java | 310 ++++ .../lakehouse/paimon/ops/PaimonCatalogOps.java | 41 + .../catalog/lakehouse/paimon/utils/TypeUtils.java | 25 +- .../lakehouse/paimon/TestPaimonCatalog.java | 249 ++++ .../lakehouse/paimon/TestPaimonViewCatalogOps.java | 526 +++++++ .../integration/test/CatalogPaimonBaseIT.java | 75 + .../integration/test/CatalogPaimonHiveIT.java | 91 ++ .../test/CatalogPaimonJdbcCredentialIT.java | 155 ++ .../lakehouse/paimon/ops/TestPaimonCatalogOps.java | 103 +- .../paimon/utils/PaimonViewTestCatalogHelper.java | 178 +++ .../java/org/apache/gravitino/hive/HiveTable.java | 24 + .../apache/gravitino/hive/client/HiveClient.java | 3 + .../gravitino/hive/client/HiveClientImpl.java | 6 + .../org/apache/gravitino/hive/client/HiveShim.java | 3 + .../apache/gravitino/hive/client/HiveShimV2.java | 11 + .../apache/gravitino/hive/client/HiveShimV3.java | 20 + .../hive/converter/HiveTableConverter.java | 60 +- .../apache/gravitino/hive/client/TestHive2HMS.java | 6 + .../hive/converter/TestHiveTableConverter.java | 76 + catalogs/hive-metastore2-libs/build.gradle.kts | 8 + catalogs/hive-metastore3-libs/build.gradle.kts | 8 + clients/client-java/build.gradle.kts | 5 + .../apache/gravitino/client/BaseSchemaCatalog.java | 42 +- .../apache/gravitino/client/RelationalCatalog.java | 15 +- .../gravitino/client/TestRelationalCatalog.java | 37 + .../HierarchicalSchemaAuthorizationIT.java | 330 +++++ clients/client-python/MANIFEST.in | 1 + .../gravitino/api/authorization/__init__.py | 4 + .../api/authorization/{__init__.py => group.py} | 37 +- .../api/authorization/{__init__.py => owner.py} | 35 +- .../gravitino/api/authorization/privileges.py | 1479 +++++++++++++++++++- .../api/authorization/{__init__.py => user.py} | 37 +- .../client-python/gravitino/api/metadata_object.py | 6 + .../gravitino/api/metadata_objects.py | 6 +- clients/client-python/gravitino/api/rel/table.py | 6 + .../gravitino/api/supports_schemas.py | 18 +- .../gravitino/client/base_schema_catalog.py | 21 +- .../gravitino/client/gravitino_client.py | 175 +++ .../gravitino/client/gravitino_metalake.py | 298 +++- .../metadata_object_statistics_operations.py | 102 ++ .../gravitino/client/relational_table.py | 24 +- .../gravitino/dto/authorization/__init__.py | 8 +- .../gravitino/dto/authorization/group_dto.py | 91 ++ .../__init__.py => dto/authorization/owner_dto.py} | 32 +- .../gravitino/dto/authorization/user_dto.py | 93 ++ .../requests/group_add_request.py} | 28 +- .../gravitino/dto/requests/owner_set_request.py | 43 + .../requests/statistics_drop_request.py} | 36 +- .../dto/requests/statistics_update_request.py | 64 + .../requests/user_add_request.py} | 28 +- .../gravitino/dto/responses/group_response.py | 79 ++ .../gravitino/dto/responses/owner_response.py | 47 + .../responses/remove_response.py} | 37 +- .../__init__.py => dto/responses/set_response.py} | 22 +- .../dto/responses/statistic_list_response.py | 45 + .../gravitino/dto/responses/user_response.py | 77 + .../client-python/gravitino/dto/stats/__init__.py | 6 +- .../gravitino/dto/stats/statistic_dto.py | 99 ++ clients/client-python/gravitino/exceptions/base.py | 16 + .../exceptions/handlers/group_error_handler.py | 67 + .../exceptions/handlers/owner_error_handler.py | 58 + .../handlers/statistics_error_handler.py | 76 + .../exceptions/handlers/user_error_handler.py | 68 + .../{MANIFEST.in => requirements-lance.txt} | 15 +- clients/client-python/setup.py | 1 + .../tests/integration/test_group_management.py | 124 ++ .../tests/integration/test_lance_ray.py | 332 +++++ .../client-python/tests/integration/test_owner.py | 182 +++ .../tests/integration/test_relational_table.py | 38 + .../client-python/tests/integration/test_user.py | 156 +++ .../unittests/authorization/test_privileges.py | 407 ++++++ .../test_metadata_object_statistics_operations.py | 113 ++ .../client/test_metalake_group_operations.py | 268 ++++ .../client/test_metalake_user_operations.py | 268 ++++ .../dto/requests/test_group_add_request.py} | 28 +- .../dto/requests/test_statistics_drop_request.py | 60 + .../dto/requests/test_statistics_update_request.py | 98 ++ .../unittests/dto/responses/test_group_response.py | 76 + .../dto/responses/test_statistic_list_response.py | 76 + .../unittests/dto/responses/test_user_response.py | 92 ++ .../tests/unittests/dto/stats/__init__.py | 6 +- .../unittests/dto/stats/test_statistic_dto.py | 70 + .../tests/unittests/dto/test_group_dto.py | 126 ++ .../tests/unittests/dto/test_user_dto.py | 141 ++ .../tests/unittests/test_base_schema_catalog.py | 100 ++ .../tests/unittests/test_error_handler.py | 117 ++ .../tests/unittests/test_metadata_objects.py | 59 +- .../client-python/tests/unittests/test_owner.py | 361 +++++ .../tests/unittests/test_relational_table.py | 22 + .../gravitino/credential/CredentialProvider.java | 30 +- .../credential/CredentialProviderDelegator.java | 4 +- .../credential/TestCredentialFactory.java | 22 + conf/log4j2.properties.template | 29 + core/build.gradle.kts | 7 + .../main/java/org/apache/gravitino/Configs.java | 45 +- .../java/org/apache/gravitino/GravitinoEnv.java | 11 +- .../gravitino/SupportsRelationOperations.java | 28 + .../java/org/apache/gravitino/audit/AuditLog.java | 20 + .../apache/gravitino/audit/FileAuditWriter.java | 92 +- .../gravitino/audit/v2/SimpleAuditLogV2.java | 16 +- .../authorization/AuthorizationRequestContext.java | 160 ++- .../authorization/AuthorizationUtils.java | 23 +- .../authorization/GravitinoAuthorizer.java | 27 +- .../gravitino/authorization/OwnerDispatcher.java | 32 + .../gravitino/authorization/OwnerEventManager.java | 40 + .../gravitino/authorization/OwnerManager.java | 188 ++- .../gravitino/cache/CaffeineGravitinoCache.java | 191 +++ .../org/apache/gravitino/cache/GravitinoCache.java | 112 ++ .../gravitino/cache/NoOpsGravitinoCache.java | 73 + .../catalog/SchemaOperationDispatcher.java | 92 +- .../catalog/TableOperationDispatcher.java | 36 +- .../apache/gravitino/connector/BaseCatalog.java | 54 +- .../credential/CatalogCredentialManager.java | 68 +- .../gravitino/credential/CredentialCache.java | 25 +- .../credential/CredentialOperationDispatcher.java | 78 +- .../credential/JdbcCredentialProvider.java | 38 +- .../gravitino/hook/SchemaHookDispatcher.java | 108 +- .../gravitino/listener/ViewEventDispatcher.java | 174 +++ .../apache/gravitino/listener/api/event/Event.java | 25 +- .../event/{Event.java => view/AlterViewEvent.java} | 30 +- .../AlterViewFailureEvent.java} | 25 +- .../listener/api/event/view/AlterViewPreEvent.java | 34 +- .../{Event.java => view/CreateViewEvent.java} | 22 +- .../CreateViewFailureEvent.java} | 25 +- .../api/event/view/CreateViewPreEvent.java | 34 +- .../event/{Event.java => view/DropViewEvent.java} | 21 +- .../{Event.java => view/DropViewFailureEvent.java} | 16 +- .../listener/api/event/view/DropViewPreEvent.java | 25 +- .../event/{Event.java => view/ListViewEvent.java} | 27 +- .../{Event.java => view/ListViewFailureEvent.java} | 24 +- .../listener/api/event/view/ListViewPreEvent.java | 34 +- .../event/{Event.java => view/LoadViewEvent.java} | 27 +- .../{Event.java => view/LoadViewFailureEvent.java} | 16 +- .../listener/api/event/view/LoadViewPreEvent.java | 25 +- .../api/event/{Event.java => view/ViewEvent.java} | 16 +- .../{Event.java => view/ViewFailureEvent.java} | 12 +- .../listener/api/event/view/ViewPreEvent.java | 26 +- .../gravitino/listener/api/info/ViewInfo.java | 134 ++ .../storage/LancePartitionStatisticStorage.java | 75 +- .../gravitino/storage/relational/JDBCBackend.java | 35 +- .../storage/relational/RelationalEntityStore.java | 22 +- .../relational/mapper/EntityChangeLogMapper.java | 5 +- .../mapper/EntityChangeLogSQLProviderFactory.java | 8 +- .../relational/mapper/FilesetMetaMapper.java | 4 +- .../mapper/FilesetMetaSQLProviderFactory.java | 4 +- .../relational/mapper/FilesetVersionMapper.java | 4 +- .../mapper/FilesetVersionSQLProviderFactory.java | 5 +- .../relational/mapper/FunctionMetaMapper.java | 4 +- .../mapper/FunctionMetaSQLProviderFactory.java | 5 +- .../mapper/FunctionVersionMetaMapper.java | 4 +- .../FunctionVersionMetaSQLProviderFactory.java | 6 +- .../storage/relational/mapper/GroupMetaMapper.java | 14 + .../mapper/GroupMetaSQLProviderFactory.java | 15 + .../storage/relational/mapper/ModelMetaMapper.java | 4 +- .../mapper/ModelMetaSQLProviderFactory.java | 4 +- .../mapper/ModelVersionAliasRelMapper.java | 4 +- .../ModelVersionAliasSQLProviderFactory.java | 5 +- .../relational/mapper/ModelVersionMetaMapper.java | 4 +- .../mapper/ModelVersionMetaSQLProviderFactory.java | 5 +- .../storage/relational/mapper/OwnerMetaMapper.java | 21 +- .../mapper/OwnerMetaSQLProviderFactory.java | 22 +- .../mapper/PolicyMetadataObjectRelMapper.java | 4 +- .../PolicyMetadataObjectRelSQLProviderFactory.java | 6 +- .../relational/mapper/SchemaMetaMapper.java | 20 +- .../mapper/SchemaMetaSQLProviderFactory.java | 21 +- .../relational/mapper/SecurableObjectMapper.java | 4 +- .../mapper/SecurableObjectSQLProviderFactory.java | 4 +- .../relational/mapper/StatisticMetaMapper.java | 4 +- .../mapper/StatisticSQLProviderFactory.java | 4 +- .../relational/mapper/TableColumnMapper.java | 4 +- .../mapper/TableColumnSQLProviderFactory.java | 4 +- .../storage/relational/mapper/TableMetaMapper.java | 4 +- .../mapper/TableMetaSQLProviderFactory.java | 4 +- .../mapper/TagMetadataObjectRelMapper.java | 4 +- .../TagMetadataObjectRelSQLProviderFactory.java | 5 +- .../storage/relational/mapper/TopicMetaMapper.java | 4 +- .../mapper/TopicMetaSQLProviderFactory.java | 4 +- .../storage/relational/mapper/ViewMetaMapper.java | 6 +- .../mapper/ViewMetaSQLProviderFactory.java | 4 +- .../base/EntityChangeLogBaseSQLProvider.java | 27 +- .../provider/base/FilesetMetaBaseSQLProvider.java | 12 +- .../base/FilesetVersionBaseSQLProvider.java | 12 +- .../provider/base/FunctionMetaBaseSQLProvider.java | 12 +- .../base/FunctionVersionMetaBaseSQLProvider.java | 14 +- .../provider/base/GroupMetaBaseSQLProvider.java | 57 + .../provider/base/ModelMetaBaseSQLProvider.java | 12 +- .../base/ModelVersionAliasRelBaseSQLProvider.java | 13 +- .../base/ModelVersionMetaBaseSQLProvider.java | 12 +- .../provider/base/OwnerMetaBaseSQLProvider.java | 108 +- .../PolicyMetadataObjectRelBaseSQLProvider.java | 39 +- .../provider/base/SchemaMetaBaseSQLProvider.java | 71 +- .../base/SecurableObjectBaseSQLProvider.java | 55 +- .../provider/base/StatisticBaseSQLProvider.java | 41 +- .../provider/base/TableColumnBaseSQLProvider.java | 12 +- .../provider/base/TableMetaBaseSQLProvider.java | 12 +- .../base/TagMetadataObjectRelBaseSQLProvider.java | 51 +- .../provider/base/TopicMetaBaseSQLProvider.java | 12 +- .../provider/base/ViewMetaBaseSQLProvider.java | 12 +- .../mapper/provider/h2/GroupMetaH2Provider.java | 48 + .../postgresql/FilesetMetaPostgreSQLProvider.java | 13 +- .../FilesetVersionPostgreSQLProvider.java | 12 +- .../postgresql/FunctionMetaPostgreSQLProvider.java | 13 +- .../FunctionVersionMetaPostgreSQLProvider.java | 14 +- .../postgresql/GroupMetaPostgreSQLProvider.java | 44 + .../postgresql/ModelMetaPostgreSQLProvider.java | 13 +- .../ModelVersionAliasRelPostgreSQLProvider.java | 14 +- .../ModelVersionMetaPostgreSQLProvider.java | 13 +- .../postgresql/OwnerMetaPostgreSQLProvider.java | 72 +- .../PolicyMetadataObjectRelPostgreSQLProvider.java | 31 +- .../postgresql/SchemaMetaPostgreSQLProvider.java | 39 +- .../SecurableObjectPostgreSQLProvider.java | 55 +- .../postgresql/TableColumnPostgreSQLProvider.java | 13 +- .../postgresql/TableMetaPostgreSQLProvider.java | 13 +- .../TagMetadataObjectRelPostgreSQLProvider.java | 51 +- .../postgresql/TopicMetaPostgreSQLProvider.java | 13 +- .../postgresql/ViewMetaPostgreSQLProvider.java | 13 +- ...ngedOwnerInfo.java => OwnerRelForDeletion.java} | 15 +- .../storage/relational/po/StatisticPO.java | 15 +- .../relational/po/auth/ChangedOwnerInfo.java | 1 + .../{ChangedOwnerInfo.java => GroupUpdatedAt.java} | 7 +- .../relational/service/BasePOStorageOps.java | 88 ++ .../relational/service/FunctionMetaService.java | 127 +- .../relational/service/FunctionPOStorageOps.java | 98 ++ .../relational/service/GroupMetaService.java | 43 +- .../HierarchicalConversionPOStorageOps.java | 199 +++ .../relational/service/MetadataObjectService.java | 33 +- .../relational/service/OwnerMetaService.java | 46 + .../relational/service/POStorageReadRouting.java | 118 ++ .../relational/service/SchemaMetaService.java | 321 +++-- .../relational/service/SchemaPOStorageOps.java | 148 ++ .../relational/service/StatisticMetaService.java | 4 +- .../relational/service/TableMetaService.java | 139 +- .../relational/service/TablePOStorageOps.java | 128 ++ .../relational/service/ViewMetaService.java | 122 +- .../relational/service/ViewPOStorageOps.java | 120 ++ .../gravitino/utils/HierarchicalSchemaUtil.java | 181 +++ .../org/apache/gravitino/utils/RequestContext.java | 62 + ....apache.gravitino.credential.CredentialProvider | 3 +- .../apache/gravitino/audit/TestAuditManager.java | 132 +- .../gravitino/audit/TestFileAuditWriter.java | 187 +++ .../org/apache/gravitino/audit/TestOperation.java | 77 + .../gravitino/audit/v2/TestSimpleAuditLogV2.java | 103 ++ .../TestAuthorizationRequestContext.java | 226 +++ .../authorization/TestAuthorizationUtils.java | 28 + .../gravitino/authorization/TestOwnerManager.java | 41 + .../apache/gravitino/cache/TestGravitinoCache.java | 484 +++++++ .../catalog/TestSchemaOperationDispatcher.java | 99 +- .../catalog/TestTableOperationDispatcher.java | 70 +- .../credential/Dummy2CredentialProvider.java | 7 + .../credential/DummyCredentialProvider.java | 7 + .../gravitino/credential/TestCredentialCache.java | 90 ++ .../TestCredentialOperationDispatcher.java | 103 ++ .../credential/TestCredentialProvider.java | 3 +- .../credential/TestJdbcCredentialProvider.java | 194 +++ .../gravitino/hook/TestSchemaHookDispatcher.java | 133 +- .../listener/api/event/TestEventRemoteAddress.java | 89 ++ .../listener/api/event/TestViewEvent.java | 326 +++++ .../TestLancePartitionStatisticStorage.java | 45 +- .../storage/memory/TestMemoryEntityStore.java | 14 + .../storage/relational/BackendTestExtension.java | 3 + .../relational/TestJDBCBackendBatchGet.java | 98 ++ .../relational/TestJDBCBackendBatchInsert.java | 170 +++ .../mapper/provider/base/TestAuthMappers.java | 55 +- .../provider/base/TestEntityChangeLogMapper.java | 3 +- .../service/TestEntityChangeLogService.java | 81 +- .../relational/service/TestGroupMetaService.java | 21 + .../TestHierarchicalConversionPOStorageOps.java | 327 +++++ .../relational/service/TestOwnerMetaService.java | 149 ++ .../service/TestPOStorageReadRouting.java | 152 ++ .../relational/service/TestSchemaMetaService.java | 218 +++ .../relational/service/TestTableMetaService.java | 21 +- .../storage/relational/utils/TestPOConverters.java | 5 +- .../utils/TestHierarchicalSchemaUtil.java | 144 ++ .../apache/gravitino/utils/TestRequestContext.java | 70 + ....apache.gravitino.credential.CredentialProvider | 1 + design-docs/gravitino-local-authentication.md | 42 +- design-docs/gravitino-logical-view-management.md | 317 +++-- docs/apache-hive-catalog.md | 46 +- docs/gravitino-server-config.md | 52 +- docs/iceberg-rest-engine/trino.md | 32 + docs/iceberg-rest-service.md | 51 +- docs/index.md | 2 + docs/lakehouse-generic-lance-table.md | 26 +- docs/lakehouse-iceberg-catalog.md | 111 +- docs/lakehouse-paimon-catalog.md | 14 + docs/lance-rest-integration.md | 55 +- docs/lance-rest-service.md | 31 +- docs/manage-relational-metadata-using-gravitino.md | 9 +- docs/manage-view-metadata-using-gravitino.md | 288 ++++ docs/open-api/openapi.yaml | 14 + docs/open-api/schemas.yaml | 24 +- docs/open-api/views.yaml | 544 +++++++ docs/security/access-control.md | 8 + docs/trino-connector/catalog-hive.md | 6 +- docs/trino-connector/catalog-iceberg.md | 9 +- docs/trino-connector/catalog-mysql.md | 5 +- docs/trino-connector/catalog-postgresql.md | 5 +- docs/trino-connector/sql-support.md | 1 + .../{flink => flink-common}/build.gradle.kts | 40 +- .../connector/CatalogPropertiesConverter.java | 0 .../flink/connector/DefaultPartitionConverter.java | 0 .../flink/connector/PartitionConverter.java | 0 .../SchemaAndTablePropertiesConverter.java | 0 .../connector/UnsupportPartitionConverter.java | 0 .../flink/connector/catalog/BaseCatalog.java | 18 +- .../connector/catalog/BaseCatalogFactory.java | 0 .../connector/catalog/GravitinoCatalogManager.java | 0 .../connector/hive/FlinkGenericTableUtil.java | 15 +- .../flink/connector/hive/GravitinoHiveCatalog.java | 18 +- .../hive/GravitinoHiveCatalogFactory.java | 22 +- .../hive/GravitinoHiveCatalogFactoryOptions.java | 0 .../hive/HiveCatalogPropertiesConverter.java | 0 .../HiveSchemaAndTablePropertiesConverter.java | 0 .../connector/iceberg/GravitinoIcebergCatalog.java | 38 +- .../iceberg/GravitinoIcebergCatalogFactory.java | 38 +- .../GravitinoIcebergCatalogFactoryOptions.java | 0 .../iceberg/IcebergPropertiesConstants.java | 0 .../iceberg/IcebergPropertiesConverter.java | 0 .../flink/connector/jdbc/GravitinoJdbcCatalog.java | 20 +- .../jdbc/GravitinoJdbcCatalogFactory.java | 19 +- .../jdbc/GravitinoJdbcCatalogFactoryOptions.java | 4 +- .../connector/jdbc/JdbcPropertiesConstants.java | 0 .../connector/jdbc/JdbcPropertiesConverter.java | 0 .../mysql/GravitinoMysqlJdbcCatalogFactory.java | 0 .../jdbc/mysql/MysqlPropertiesConverter.java | 0 .../GravitinoPostgresJdbcCatalogFactory.java | 0 .../postgresql/PostgresqlPropertiesConverter.java | 0 .../connector/paimon/GravitinoPaimonCatalog.java | 0 .../paimon/GravitinoPaimonCatalogFactory.java | 11 +- .../GravitinoPaimonCatalogFactoryOptions.java | 0 .../paimon/PaimonPropertiesConverter.java | 0 .../connector/store/GravitinoCatalogStore.java | 38 +- .../store/GravitinoCatalogStoreFactory.java | 0 .../store/GravitinoCatalogStoreFactoryOptions.java | 0 .../store/GravitinoSessionCatalogStore.java | 0 .../flink/connector/utils/CatalogCompat.java | 50 + .../connector/utils/DefaultCatalogCompat.java} | 21 +- .../flink/connector/utils/FactoryUtils.java | 0 .../flink/connector/utils/PropertyUtils.java | 0 .../flink/connector/utils/TableUtils.java | 0 .../gravitino/flink/connector/utils/TypeUtils.java | 15 + .../org.apache.flink.table.factories.Factory | 4 +- .../flink/connector/catalog/TestBaseCatalog.java | 6 +- .../connector/hive/TestFlinkGenericTableUtil.java | 17 +- .../hive/TestHivePropertiesConverter.java | 0 .../TestHiveSchemaAndTablePropertiesConverter.java | 0 .../iceberg/TestGravitinoIcebergCatalog.java} | 31 +- .../iceberg/TestIcebergPropertiesConverter.java | 0 .../connector/integration/test/FlinkCommonIT.java | 48 +- .../connector/integration/test/FlinkEnvIT.java | 0 .../test/catalog/GravitinoCatalogManagerIT.java | 2 +- .../integration/test/hive/FlinkHiveCatalogIT.java | 36 +- .../test/hive/FlinkHiveKerberosClientIT.java | 24 +- .../test/iceberg/FlinkIcebergCatalogIT.java | 0 .../test/iceberg/FlinkIcebergHiveCatalogIT.java | 4 +- .../test/iceberg/FlinkIcebergRestCatalogIT.java | 68 +- .../test/jdbc/FlinkJdbcMysqlCatalogIT.java | 2 +- .../test/paimon/FlinkPaimonCatalogIT.java | 65 + .../test/paimon/FlinkPaimonHiveBackendIT.java | 2 +- .../test/paimon/FlinkPaimonJdbcBackendIT.java | 2 +- .../FlinkPaimonLocalFileSystemBackendIT.java | 2 +- .../integration/test/utils/TestUtils.java | 0 .../AbstractJdbcPropertiesConverterTestSuite.java | 0 .../jdbc/TestMysqlPropertiesConverter.java | 0 .../jdbc/TestPostgresqlPropertiesConverter.java | 0 .../paimon/TestPaimonPropertiesConverter.java | 0 .../connector/store/TestGravitinoCatalogStore.java | 37 + .../store/TestGravitinoCatalogStoreFactory.java | 0 .../connector/store/TestGravitinoFlinkConfig.java | 0 .../store/TestGravitinoSessionCatalogStore.java | 0 .../connector/utils/TestDefaultCatalogCompat.java | 72 + .../flink/connector/utils/TestTypeUtils.java | 32 + .../src/test/resources/flink-tests/hive-site.xml | 0 .../src/test/resources/log4j2.properties | 0 .../{ => v1.18}/flink-runtime/build.gradle.kts | 17 +- .../flink/runtime/TestRuntimeJarDependencies.java | 26 + flink-connector/{ => v1.18}/flink/build.gradle.kts | 79 +- .../hive/GravitinoHiveCatalogFactoryFlink118.java} | 45 +- .../hive/GravitinoHiveCatalogFlink118.java} | 42 +- .../GravitinoIcebergCatalogFactoryFlink118.java} | 39 +- .../iceberg/GravitinoIcebergCatalogFlink118.java} | 36 +- .../jdbc/GravitinoJdbcCatalogFlink118.java} | 33 +- .../GravitinoMysqlJdbcCatalogFactoryFlink118.java} | 32 +- ...avitinoPostgresJdbcCatalogFactoryFlink118.java} | 33 +- .../GravitinoPaimonCatalogFactoryFlink118.java} | 24 +- .../paimon/GravitinoPaimonCatalogFlink118.java} | 33 +- .../connector/utils/CatalogCompatFlink118.java} | 21 +- .../org.apache.flink.table.factories.Factory | 11 +- .../test/catalog/GravitinoCatalogManagerIT118.java | 9 +- .../test/hive/FlinkHiveCatalogIT118.java | 9 +- .../test/hive/FlinkHiveKerberosClientIT118.java | 9 +- .../test/iceberg/FlinkIcebergHiveCatalogIT118.java | 9 +- .../test/iceberg/FlinkIcebergRestCatalogIT118.java | 9 +- .../test/jdbc/FlinkJdbcMysqlCatalogIT118.java | 9 +- .../test/paimon/FlinkPaimonHiveBackendIT118.java | 9 +- .../test/paimon/FlinkPaimonJdbcBackendIT118.java | 9 +- .../FlinkPaimonLocalFileSystemBackendIT118.java | 9 +- .../{ => v1.19}/flink-runtime/build.gradle.kts | 17 +- .../flink/runtime/TestRuntimeJarDependencies.java | 26 + flink-connector/{ => v1.19}/flink/build.gradle.kts | 85 +- .../hive/GravitinoHiveCatalogFactoryFlink119.java} | 46 +- .../hive/GravitinoHiveCatalogFlink119.java} | 43 +- .../GravitinoIcebergCatalogFactoryFlink119.java} | 40 +- .../iceberg/GravitinoIcebergCatalogFlink119.java} | 37 +- .../jdbc/GravitinoJdbcCatalogFlink119.java} | 35 +- .../GravitinoMysqlJdbcCatalogFactoryFlink119.java} | 33 +- ...avitinoPostgresJdbcCatalogFactoryFlink119.java} | 34 +- .../GravitinoPaimonCatalogFactoryFlink119.java} | 25 +- .../paimon/GravitinoPaimonCatalogFlink119.java} | 34 +- .../org.apache.flink.table.factories.Factory | 11 +- .../test/catalog/GravitinoCatalogManagerIT119.java | 9 +- .../test/hive/FlinkHiveCatalogIT119.java | 9 +- .../test/hive/FlinkHiveKerberosClientIT119.java | 9 +- .../test/iceberg/FlinkIcebergHiveCatalogIT119.java | 9 +- .../test/iceberg/FlinkIcebergRestCatalogIT119.java | 9 +- .../test/jdbc/FlinkJdbcMysqlCatalogIT119.java | 9 +- .../test/paimon/FlinkPaimonHiveBackendIT119.java | 9 +- .../test/paimon/FlinkPaimonJdbcBackendIT119.java | 9 +- .../FlinkPaimonLocalFileSystemBackendIT119.java | 9 +- .../{ => v1.20}/flink-runtime/build.gradle.kts | 17 +- .../flink/runtime/TestRuntimeJarDependencies.java | 26 + flink-connector/{ => v1.20}/flink/build.gradle.kts | 85 +- .../hive/GravitinoHiveCatalogFactoryFlink120.java} | 46 +- .../hive/GravitinoHiveCatalogFlink120.java} | 43 +- .../GravitinoIcebergCatalogFactoryFlink120.java} | 40 +- .../iceberg/GravitinoIcebergCatalogFlink120.java} | 37 +- .../jdbc/GravitinoJdbcCatalogFlink120.java} | 35 +- .../GravitinoMysqlJdbcCatalogFactoryFlink120.java} | 33 +- ...avitinoPostgresJdbcCatalogFactoryFlink120.java} | 34 +- .../GravitinoPaimonCatalogFactoryFlink120.java} | 25 +- .../paimon/GravitinoPaimonCatalogFlink120.java} | 34 +- .../connector/utils/CatalogCompatFlink120.java | 49 + .../org.apache.flink.table.factories.Factory | 11 +- .../test/catalog/GravitinoCatalogManagerIT120.java | 9 +- .../test/hive/FlinkHiveCatalogIT120.java | 78 ++ .../test/hive/FlinkHiveKerberosClientIT120.java | 9 +- .../test/iceberg/FlinkIcebergHiveCatalogIT120.java | 9 +- .../test/iceberg/FlinkIcebergRestCatalogIT120.java | 9 +- .../test/jdbc/FlinkJdbcMysqlCatalogIT120.java | 51 + .../test/paimon/FlinkPaimonHiveBackendIT120.java | 9 +- .../test/paimon/FlinkPaimonJdbcBackendIT120.java | 9 +- .../FlinkPaimonLocalFileSystemBackendIT120.java | 9 +- gradle/libs.versions.toml | 31 +- .../gravitino/iceberg/common/IcebergConfig.java | 9 +- .../iceberg/common/ops/IcebergCatalogWrapper.java | 14 +- .../iceberg/common/utils/IcebergCatalogUtil.java | 17 +- .../common/utils/IcebergIdentifierUtils.java | 56 +- .../iceberg/common/TestIcebergConfig.java | 16 + .../common/utils/TestIcebergCatalogUtil.java | 24 + .../common/utils/TestIcebergIdentifierUtils.java | 27 +- .../org/apache/gravitino/iceberg/RESTService.java | 6 + .../iceberg/service/CatalogWrapperForREST.java | 169 ++- .../service/IcebergAuthenticationFilter.java | 12 + .../iceberg/service/IcebergExceptionMapper.java | 5 +- .../dispatcher/IcebergNamespaceHookDispatcher.java | 152 +- .../service/dispatcher/IcebergOwnershipUtils.java | 50 +- .../dispatcher/IcebergTableHookDispatcher.java | 102 +- .../dispatcher/IcebergTableOperationExecutor.java | 4 +- .../dispatcher/IcebergViewHookDispatcher.java | 121 +- .../service/rest/IcebergHealthOperations.java | 138 ++ .../service/rest/IcebergNamespaceOperations.java | 25 +- .../service/rest/IcebergTableOperations.java | 13 +- .../service/rest/IcebergViewOperations.java | 13 +- .../listener/api/event/IcebergRequestContext.java | 14 +- .../annotations/IcebergAuthorizationMetadata.java | 1 + .../web/filter/CreateNamespaceAuthzHandler.java | 95 ++ ...bergMetadataAuthorizationMethodInterceptor.java | 86 +- .../server/web/filter/RenameTableAuthzHandler.java | 7 +- .../server/web/filter/RenameViewAuthzHandler.java | 7 +- .../integration/test/IcebergAuthorizationIT.java | 56 + .../iceberg/integration/test/IcebergCacheIT.java | 230 +++ .../test/IcebergNamespaceAuthorizationIT.java | 150 +- .../test/IcebergTableAuthorizationIT.java | 143 +- .../test/IcebergViewAuthorizationIT.java | 185 ++- .../iceberg/service/TestCatalogWrapperForREST.java | 272 +++- .../service/TestIcebergAuthenticationFilter.java | 47 + .../service/TestIcebergExceptionMapper.java | 2 +- .../TestIcebergNamespaceHookDispatcher.java | 123 +- .../TestIcebergNamespaceOperationExecutor.java | 110 ++ .../dispatcher/TestIcebergOwnershipUtils.java | 75 +- .../dispatcher/TestIcebergTableHookDispatcher.java | 134 +- .../dispatcher/TestIcebergViewHookDispatcher.java | 112 +- .../service/extension/DummyCredentialProvider.java | 7 +- .../service/rest/TestIcebergHealthOperations.java | 94 ++ .../rest/TestIcebergNamespaceOperations.java | 33 + .../service/rest/TestIcebergTableOperations.java | 2 +- ...bergMetadataAuthorizationMethodInterceptor.java | 84 ++ .../gravitino/integration/test/util/BaseIT.java | 29 +- .../integration/test/util/TestDatabaseName.java | 3 + .../lance/common/ops/LanceTableOperations.java | 23 +- .../gravitino/GravitinoLanceTableOperations.java | 45 +- .../lance/common/utils/LanceConstants.java | 4 +- .../lance/common/utils/LancePropertiesUtils.java | 56 +- .../common/utils/TestLancePropertiesUtils.java | 63 + lance/lance-rest-server/build.gradle.kts | 2 +- .../service/rest/JsonNullableMapperProvider.java | 44 + .../lance/service/rest/LanceTableOperations.java | 78 +- .../gravitino/TestGravitinoLanceModeParsing.java | 61 + .../lance/integration/test/LanceRESTServiceIT.java | 190 +-- .../service/rest/TestLanceNamespaceOperations.java | 162 +-- lineage/build.gradle.kts | 4 + plugins/idp-basic/build.gradle.kts | 45 + .../apache/gravitino/idp/IdpUserGroupManager.java | 192 +++ .../org/apache/gravitino/idp/dto/IdpGroupDTO.java | 82 ++ .../org/apache/gravitino/idp/dto/IdpUserDTO.java | 83 ++ .../idp/dto/requests/AddGroupRequest.java | 68 + .../gravitino/idp/dto/requests/AddUserRequest.java | 76 + .../dto/requests/GroupMembershipChangeRequest.java | 84 ++ .../idp/dto/requests/ResetPasswordRequest.java | 69 + .../idp/dto/responses/IdpGroupResponse.java | 79 ++ .../idp/dto/responses/IdpUserResponse.java | 79 ++ .../idp/exception/AlreadyExistsException.java | 49 + .../gravitino/idp/exception/NotFoundException.java | 49 + .../org/apache/gravitino/idp/model/IdpGroup.java | 72 + .../org/apache/gravitino/idp/model/IdpUser.java | 72 + .../idp/storage/mapper/IdpGroupMetaMapper.java | 54 + .../mapper/IdpGroupMetaSQLProviderFactory.java | 74 + .../idp/storage/mapper/IdpUserGroupRelMapper.java | 77 + .../mapper/IdpUserGroupRelSQLProviderFactory.java | 88 ++ .../idp/storage/mapper/IdpUserMetaMapper.java | 62 + .../mapper/IdpUserMetaSQLProviderFactory.java | 83 ++ .../storage/mapper/SQLProviderFactoryHelper.java | 68 + .../provider/IdpBasicMapperPackageProvider.java | 28 +- .../provider/base/IdpGroupMetaBaseSQLProvider.java | 68 + .../base/IdpUserGroupRelBaseSQLProvider.java | 130 ++ .../provider/base/IdpUserMetaBaseSQLProvider.java | 94 ++ .../mapper/provider/h2/IdpGroupMetaH2Provider.java | 12 +- .../provider/h2/IdpUserGroupRelH2Provider.java | 87 ++ .../mapper/provider/h2/IdpUserMetaH2Provider.java | 12 +- .../postgresql/IdpGroupMetaPostgreSQLProvider.java | 42 + .../IdpUserGroupRelPostgreSQLProvider.java | 98 ++ .../postgresql/IdpUserMetaPostgreSQLProvider.java | 42 + .../gravitino/idp/storage/po/IdpGroupPO.java | 26 +- .../idp/storage/po/IdpUserGroupRelPO.java | 27 +- .../apache/gravitino/idp/storage/po/IdpUserPO.java | 27 +- .../storage/relational/IdpGarbageCollector.java | 121 ++ .../storage/relational/IdpRelationalStorage.java | 81 ++ .../converters/IdpSQLExceptionConverter.java | 94 ++ .../IdpSQLExceptionConverterFactory.java | 84 ++ .../relational/utils/IdpExceptionUtils.java | 44 + .../idp/storage/service/IdpGroupMetaService.java | 215 +++ .../idp/storage/service/IdpUserMetaService.java | 167 +++ ...elational.mapper.provider.MapperPackageProvider | 6 +- .../gravitino/idp/TestIdpUserGroupManager.java | 206 +++ .../basic/password/TestArgon2idPasswordHasher.java | 14 + .../apache/gravitino/idp/dto/TestIdpGroupDTO.java | 77 + .../apache/gravitino/idp/dto/TestIdpUserDTO.java | 77 + .../idp/dto/requests/TestAddGroupRequest.java | 58 + .../idp/dto/requests/TestAddUserRequest.java | 71 + .../requests/TestGroupMembershipChangeRequest.java | 95 ++ .../idp/dto/requests/TestResetPasswordRequest.java | 68 + .../idp/dto/responses/TestIdpGroupResponse.java | 94 ++ .../idp/dto/responses/TestIdpUserResponse.java | 94 ++ .../storage/mapper/AbstractIdpMetaStorageTest.java | 259 ++++ .../mapper/TestIdpBasicMapperPackageProvider.java | 56 + .../storage/mapper/TestIdpGroupMetaStorage.java | 140 ++ .../storage/mapper/TestIdpUserGroupRelStorage.java | 401 ++++++ .../idp/storage/mapper/TestIdpUserMetaStorage.java | 196 +++ .../mapper/TestSQLProviderFactoryHelper.java | 68 + .../gravitino/idp/storage/po/TestIdpGroupPO.java | 84 ++ .../idp/storage/po/TestIdpUserGroupRelPO.java | 89 ++ .../gravitino/idp/storage/po/TestIdpUserPO.java | 105 ++ .../relational/TestIdpGarbageCollector.java | 180 +++ .../service/AbstractIdpMetaServiceTest.java | 203 +++ .../storage/service/TestIdpGroupMetaService.java | 198 +++ .../storage/service/TestIdpUserMetaService.java | 158 +++ scripts/h2/schema-1.3.0-h2.sql | 14 +- scripts/h2/upgrade-1.2.0-to-1.3.0-h2.sql | 12 +- scripts/mysql/schema-1.3.0-mysql.sql | 16 +- scripts/mysql/upgrade-1.2.0-to-1.3.0-mysql.sql | 18 +- scripts/postgresql/schema-1.3.0-postgresql.sql | 28 +- .../upgrade-1.2.0-to-1.3.0-postgresql.sql | 27 +- .../authentication/AuthenticationFilter.java | 9 +- .../server/authorization/MetadataIdConverter.java | 10 +- .../authorization/PassThroughAuthorizer.java | 2 +- .../annotations/AuthorizationRequest.java | 3 +- .../AuthorizationExpressionConstants.java | 3 +- .../AuthorizationExpressionConverter.java | 8 +- .../authorization/jcasbin/CachedGroupRoleRels.java | 37 +- .../authorization/jcasbin/CachedUserRoleRels.java | 37 +- .../jcasbin/JcasbinAuthorizationCacheKeys.java | 125 ++ .../jcasbin/JcasbinAuthorizationLookups.java | 117 ++ .../authorization/jcasbin/JcasbinAuthorizer.java | 871 ++++++++---- .../authorization/jcasbin/JcasbinChangePoller.java | 320 +++++ .../jcasbin/JcasbinLoadedRolesCache.java | 94 ++ .../gravitino/server/web/HealthAliasServlet.java | 31 +- .../gravitino/server/web/RequestContextFilter.java | 75 + .../org/apache/gravitino/server/web/Utils.java | 7 + .../authorization/MockGravitinoAuthorizer.java | 2 +- .../authorization/TestMetadataIdConverter.java | 29 +- .../authorization/TestPassThroughAuthorizer.java | 3 +- .../TestAuthorizationExpressionConverter.java | 9 + .../jcasbin/TestJcasbinAuthorizationCacheKeys.java | 117 ++ .../jcasbin/TestJcasbinAuthorizationLookups.java | 117 ++ .../jcasbin/TestJcasbinAuthorizer.java | 757 ++++++---- .../jcasbin/TestJcasbinAuthorizerCacheHelpers.java | 46 + .../jcasbin/TestJcasbinChangePoller.java | 188 +++ .../server/web/TestHealthAliasServlet.java | 20 + .../server/web/TestRequestContextFilter.java | 120 ++ .../apache/gravitino/server/GravitinoServer.java | 2 + .../web/filter/GravitinoInterceptionService.java | 8 +- .../AssociatePolicyAuthorizationExecutor.java | 3 +- .../AssociateTagAuthorizationExecutor.java | 3 +- .../authorization/AuthorizationExecutor.java | 4 +- .../authorization/AuthorizeExecutorFactory.java | 2 + .../authorization/CommonAuthorizerExecutor.java | 5 +- .../CreateSchemaAuthorizationExecutor.java | 105 ++ .../authorization/RunJobAuthorizationExecutor.java | 6 +- .../server/web/rest/HealthOperations.java | 24 +- .../server/web/rest/SchemaOperations.java | 55 +- .../server/web/rest/VersionOperations.java | 3 +- .../gravitino/server/web/ui/WebUIFilter.java | 5 + .../filter/TestGravitinoInterceptionService.java | 4 +- .../server/web/rest/TestSchemaOperations.java | 40 + .../gravitino/server/web/ui/WebUIFilterTest.java | 15 + settings.gradle.kts | 20 +- .../iceberg/SparkIcebergCatalogHiveBackendIT.java | 1 + .../connector/integration/test/TrinoQueryIT.java | 37 +- .../integration/test/TrinoQueryTestTool.java | 17 +- .../trino-ci-testset/testsets/hive/00018_ctas.sql | 35 + .../trino-ci-testset/testsets/hive/00018_ctas.txt | 30 + .../testsets/jdbc-mysql/00014_ctas.sql | 35 + .../testsets/jdbc-mysql/00014_ctas.txt | 30 + .../testsets/jdbc-postgresql/00010_ctas.sql | 35 + .../testsets/jdbc-postgresql/00010_ctas.txt | 30 + .../testsets/lakehouse-iceberg/00013_ctas.sql | 57 + .../testsets/lakehouse-iceberg/00013_ctas.txt | 71 + .../trino/connector/GravitinoMetadata435.java | 12 + .../trino/connector/GravitinoMetadata440.java | 13 + .../trino/connector/GravitinoMetadata446.java | 13 + .../trino/connector/GravitinoMetadata452.java | 13 + .../trino/connector/GravitinoMetadata469.java | 13 + .../trino/connector/GravitinoMetadata478.java | 13 + .../trino/connector/GravitinoMetadata.java | 92 ++ .../connector/GravitinoOutputTableHandle.java | 102 ++ .../trino/connector/GravitinoPageSinkProvider.java | 8 +- .../catalog/CatalogConnectorMetadataAdapter.java | 2 +- .../catalog/DefaultCatalogConnectorFactory.java | 5 + .../catalog/glue/GlueConnectorAdapter.java | 127 ++ .../catalog/glue/GlueDataTypeTransformer.java | 66 + .../catalog/glue/GlueMetadataAdapter.java | 181 +++ .../catalog/glue/GlueTablePropertyConverter.java | 44 + .../catalog/hive/HiveMetadataAdapter.java | 42 +- .../connector/catalog/hive/HivePropertyMeta.java | 11 +- .../trino/connector/util/json/JsonCodec.java | 5 + .../trino/connector/TestGravitinoConnector.java | 53 + .../TestGravitinoMetadataGetNewTableLayout.java | 118 ++ .../catalog/glue/TestGlueConnectorAdapter.java | 76 + .../catalog/glue/TestGlueDataTypeTransformer.java | 85 ++ .../catalog/glue/TestGlueMetadataAdapter.java | 272 ++++ .../catalog/hive/TestHivePropertyMeta.java | 30 +- web-v2/web/package.json | 17 +- web-v2/web/pnpm-lock.yaml | 694 ++++----- web-v2/web/src/app/catalogs/TreeComponent.js | 9 +- web-v2/web/src/app/catalogs/page.js | 24 +- .../app/catalogs/rightContent/CreateTableDialog.js | 99 +- .../src/app/catalogs/rightContent/RightContent.js | 10 + .../entitiesContent/FunctionDetailsPage.js | 39 +- .../entitiesContent/SchemaDetailsPage.js | 100 +- .../entitiesContent/ViewDetailsPage.js | 353 +++++ .../src/components/SecurableObjectFormFields.js | 27 +- web-v2/web/src/config/index.js | 16 +- web-v2/web/src/config/security.js | 12 + web-v2/web/src/lib/api/views/index.js | 53 + web-v2/web/src/lib/store/metalakes/index.js | 142 +- web-v2/web/src/lib/styles/antdStyles/globals.css | 31 +- 762 files changed, 43807 insertions(+), 5108 deletions(-) diff --cc core/src/main/java/org/apache/gravitino/catalog/SchemaOperationDispatcher.java index f034c40c3d,ed91328d45..9ab9df5588 --- a/core/src/main/java/org/apache/gravitino/catalog/SchemaOperationDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/catalog/SchemaOperationDispatcher.java @@@ -189,13 -192,13 +192,27 @@@ public class SchemaOperationDispatcher TreeLockUtils.doWithTreeLock(ident, LockType.READ, () -> internalLoadSchema(ident)); if (!schema.imported()) { -- TreeLockUtils.doWithTreeLock( -- NameIdentifier.of(ident.namespace().levels()), -- LockType.WRITE, -- () -> { -- importSchema(ident); -- return null; -- }); ++ try { ++ TreeLockUtils.doWithTreeLock( ++ NameIdentifier.of(ident.namespace().levels()), ++ LockType.WRITE, ++ () -> { ++ importSchema(ident); ++ return null; ++ }); ++ } catch (EntityAlreadyExistsException e) { ++ // HA race: another Gravitino node concurrently imported this schema. Reload from the ++ // entity store to verify the entity stored by the winning node is consistent. ++ LOG.info( ++ "Schema {} was concurrently imported by another node; reloading from store.", ident); ++ EntityCombinedSchema reloaded = ++ TreeLockUtils.doWithTreeLock(ident, LockType.READ, () -> internalLoadSchema(ident)); ++ if (!reloaded.imported()) { ++ throw new UnsupportedOperationException( ++ "Schema managed by multiple catalogs. This may cause unexpected issues such as privilege conflicts. " ++ + "To resolve: Remove all catalogs managing this schema, then recreate one catalog to ensure single-catalog management."); ++ } ++ } } return schema; @@@ -387,18 -438,22 +452,7 @@@ try { store.put(schemaEntity, true); } catch (EntityAlreadyExistsException e) { - // HA race: another Gravitino node finished importing the same schema between - // our internalLoadSchema check above and this put. The entity is already in - // the store, so the import is effectively a no-op — let the caller's load - // request succeed instead of failing with "managed by multiple catalogs". -- SchemaEntity concurrentSchemaEntity = getEntity(identifier, SCHEMA, SchemaEntity.class); - if (concurrentSchemaEntity != null) { - if (isSameImportedSchema(concurrentSchemaEntity, stringId)) { -- LOG.info( -- "Schema {} was imported concurrently, reusing the existing entity in Gravitino.", -- identifier); -- return; -- } -- -- LOG.error("Failed to import schema {} with id {} to the store.", identifier, uid, e); -- throw new UnsupportedOperationException( -- "Schema managed by multiple catalogs. This may cause unexpected issues such as privilege conflicts. " -- + "To resolve: Remove all catalogs managing this schema, then recreate one catalog to ensure single-catalog management."); ++ throw e; } catch (Exception e) { LOG.error(FormattedErrorMessages.STORE_OP_FAILURE, "put", identifier, e); throw new RuntimeException("Fail to import schema entity to the store.", e); diff --cc core/src/main/java/org/apache/gravitino/catalog/TableOperationDispatcher.java index cb4aef4792,0233f610f5..244edec88d --- a/core/src/main/java/org/apache/gravitino/catalog/TableOperationDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/catalog/TableOperationDispatcher.java @@@ -123,8 -123,8 +123,22 @@@ public class TableOperationDispatcher e schemaDispatcher.loadSchema(schemaIdent); // Import the table. -- entityCombinedTable = -- TreeLockUtils.doWithTreeLock(schemaIdent, LockType.WRITE, () -> importTable(ident)); ++ try { ++ entityCombinedTable = ++ TreeLockUtils.doWithTreeLock(schemaIdent, LockType.WRITE, () -> importTable(ident)); ++ } catch (EntityAlreadyExistsException e) { ++ // HA race: another Gravitino node concurrently imported this table. Reload from the ++ // entity store to pick up the entity stored by the winning node. ++ LOG.info( ++ "Table {} was concurrently imported by another node; reloading from store.", ident); ++ entityCombinedTable = ++ TreeLockUtils.doWithTreeLock(ident, LockType.READ, () -> internalLoadTable(ident)); ++ if (!entityCombinedTable.imported()) { ++ throw new UnsupportedOperationException( ++ "Table managed by multiple catalogs. This may cause unexpected issues such as privilege conflicts. " ++ + "To resolve: Remove all catalogs managing this table, then recreate one catalog to ensure single-catalog management."); ++ } ++ } } // Update the column entities in Gravitino store if the columns are different from the ones @@@ -477,23 -477,26 +491,7 @@@ try { store.put(tableEntity, true); } catch (EntityAlreadyExistsException e) { - // HA race: another Gravitino node finished importing the same table between - // our internalLoadTable check above and this put. Reuse the existing entity - // instead of failing the caller's load with "managed by multiple catalogs". -- TableEntity concurrentTableEntity = getEntity(identifier, TABLE, TableEntity.class); - if (concurrentTableEntity != null) { - if (isSameImportedTable(concurrentTableEntity, stringId)) { -- LOG.info( -- "Table {} was imported concurrently, reusing the existing entity in Gravitino.", -- identifier); -- return EntityCombinedTable.of(table.tableFromCatalog(), concurrentTableEntity) -- .withHiddenProperties( -- getHiddenPropertyNames( -- getCatalogIdentifier(identifier), -- HasPropertyMetadata::tablePropertiesMetadata, -- table.tableFromCatalog().properties())); -- } -- -- LOG.error("Failed to import table {} with id {} to the store.", identifier, uid, e); -- throw new UnsupportedOperationException( -- "Table managed by multiple catalogs. This may cause unexpected issues such as privilege conflicts. " -- + "To resolve: Remove all catalogs managing this table, then recreate one catalog to ensure single-catalog management."); ++ throw e; } catch (Exception e) { LOG.error(FormattedErrorMessages.STORE_OP_FAILURE, "put", identifier, e); throw new RuntimeException("Fail to import the table entity to the store.", e); diff --cc core/src/test/java/org/apache/gravitino/catalog/TestSchemaOperationDispatcher.java index 880450569c,fb10bcba9e..455e428c34 --- a/core/src/test/java/org/apache/gravitino/catalog/TestSchemaOperationDispatcher.java +++ b/core/src/test/java/org/apache/gravitino/catalog/TestSchemaOperationDispatcher.java @@@ -222,8 -223,8 +223,13 @@@ public class TestSchemaOperationDispatc .withAuditInfo(concurrentAudit) .build(); ++ // Simulate HA race: first two gets return not-found (so both the pre-import check and the ++ // internalLoadSchema inside importSchema proceed to store.put), then put throws ++ // EntityAlreadyExistsException, and the dispatcher-level retry sees the entity on the third ++ // get. reset(entityStore); doThrow(new NoSuchEntityException("mock error")) ++ .doThrow(new NoSuchEntityException("mock error")) .doReturn(concurrentSchemaEntity) .when(entityStore) .get(any(), eq(Entity.EntityType.SCHEMA), any()); @@@ -236,6 -237,38 +242,42 @@@ Assertions.assertEquals("comment", loadedSchema.comment()); } + @Test + public void testConcurrentImportSchemaFailsOnMismatchedIdentifier() throws IOException { + NameIdentifier schemaIdent = NameIdentifier.of(metalake, catalog, "schemaConcurrentMismatch"); + Map<String, String> props = ImmutableMap.of("k1", "v1", "k2", "v2"); + dispatcher.createSchema(schemaIdent, "comment", props); + SchemaEntity importedSchemaEntity = entityStore.get(schemaIdent, SCHEMA, SchemaEntity.class); + + AuditInfo concurrentAudit = + AuditInfo.builder().withCreator("concurrent").withCreateTime(Instant.now()).build(); + SchemaEntity mismatchedSchemaEntity = + SchemaEntity.builder() + .withId(importedSchemaEntity.id() + 1) + .withName(schemaIdent.name()) + .withNamespace(schemaIdent.namespace()) + .withAuditInfo(concurrentAudit) + .build(); + ++ // Simulate genuine multi-catalog conflict: put fails, and the dispatcher-level retry finds ++ // an entity with a mismatched ID (operateOnEntity returns null → imported=false → error ++ // thrown). + reset(entityStore); + doThrow(new NoSuchEntityException("mock error")) ++ .doThrow(new NoSuchEntityException("mock error")) + .doReturn(mismatchedSchemaEntity) + .when(entityStore) + .get(any(), eq(Entity.EntityType.SCHEMA), any()); + doThrow(new EntityAlreadyExistsException("mock conflict")) + .when(entityStore) + .put(any(), anyBoolean()); + + UnsupportedOperationException exception = + Assertions.assertThrows( + UnsupportedOperationException.class, () -> dispatcher.loadSchema(schemaIdent)); + Assertions.assertTrue(exception.getMessage().contains("Schema managed by multiple catalogs")); + } + @Test public void testCreateAndAlterSchema() throws IOException { NameIdentifier schemaIdent = NameIdentifier.of(metalake, catalog, "schema21"); diff --cc core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java index 7e89a94747,2bcad2e22f..9c44e46c7e --- a/core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java +++ b/core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java @@@ -299,8 -300,8 +300,13 @@@ public class TestTableOperationDispatch .withAuditInfo(concurrentAudit) .build(); ++ // Simulate HA race: first two gets return not-found (so both the pre-import check and the ++ // internalLoadTable inside importTable proceed to store.put), then put throws ++ // EntityAlreadyExistsException, and the dispatcher-level retry sees the entity on the third ++ // get. reset(entityStore); doThrow(new NoSuchEntityException("mock error")) ++ .doThrow(new NoSuchEntityException("mock error")) .doReturn(concurrentTableEntity) .when(entityStore) .get(any(), eq(Entity.EntityType.TABLE), any()); @@@ -314,6 -315,64 +320,68 @@@ Assertions.assertEquals("comment", loadedTable.comment()); } + @Test + public void testConcurrentImportTableFailsOnMismatchedIdentifier() throws IOException { + Namespace tableNs = Namespace.of(metalake, catalog, "schemaConcurrentMismatch"); + Map<String, String> props = ImmutableMap.of("k1", "v1", "k2", "v2"); + schemaOperationDispatcher.createSchema(NameIdentifier.of(tableNs.levels()), "comment", props); + + NameIdentifier tableIdent = NameIdentifier.of(tableNs, "tableConcurrentMismatch"); + Column[] columns = + new Column[] { + TestColumn.builder() + .withName("col1") + .withPosition(0) + .withType(Types.StringType.get()) + .build(), + TestColumn.builder() + .withName("col2") + .withPosition(1) + .withType(Types.StringType.get()) + .build() + }; + + Table table = + tableOperationDispatcher.createTable( + tableIdent, columns, "comment", props, new Transform[0]); + TableEntity importedTableEntity = entityStore.get(tableIdent, TABLE, TableEntity.class); + + AuditInfo concurrentAudit = + AuditInfo.builder().withCreator("concurrent").withCreateTime(Instant.now()).build(); + TableEntity mismatchedTableEntity = + TableEntity.builder() + .withId(importedTableEntity.id() + 1) + .withName(tableIdent.name()) + .withNamespace(tableIdent.namespace()) + .withColumns( + IntStream.range(0, table.columns().length) + .mapToObj( + i -> + ColumnEntity.toColumnEntity(table.columns()[i], i, 0L, concurrentAudit)) + .collect(Collectors.toList())) + .withAuditInfo(concurrentAudit) + .build(); + ++ // Simulate genuine multi-catalog conflict: put fails, and the dispatcher-level retry finds ++ // an entity with a mismatched ID (operateOnEntity returns null → imported=false → error ++ // thrown). + reset(entityStore); + doThrow(new NoSuchEntityException("mock error")) ++ .doThrow(new NoSuchEntityException("mock error")) + .doReturn(mismatchedTableEntity) + .when(entityStore) + .get(any(), eq(Entity.EntityType.TABLE), any()); + doThrow(new EntityAlreadyExistsException("mock conflict")) + .when(entityStore) + .put(any(), anyBoolean()); + + UnsupportedOperationException exception = + Assertions.assertThrows( + UnsupportedOperationException.class, + () -> tableOperationDispatcher.loadTable(tableIdent)); + Assertions.assertTrue(exception.getMessage().contains("Table managed by multiple catalogs")); + } + @Test public void testCreateAndAlterTable() throws IOException { Namespace tableNs = Namespace.of(metalake, catalog, "schema61");
