This is an automated email from the ASF dual-hosted git repository.
yuqi1129 pushed a change to branch issue-10464-iceberg-listener-decouple
in repository https://gitbox.apache.org/repos/asf/gravitino.git
from a28a57194f improvement(iceberg): decouple listener events from Iceberg
APIs
add efe98bb652 [#11232][#11233] improvement(core): Global
EntityChangeLogPoller with CatalogManager cache invalidation and automatic
retention cleanup (#11254)
add ae1a05046c [#10983] test(lance): add Lance compatibility matrix
automation (#11119)
add df532dbf03 [#11298] feat(iceberg-rest): add async cleanup manager
(#11299)
add b5ea86aa57 [#11337] deps(paimon): Exclude transitive tomcat-embed-core
from hive-exec (#11338)
add 9170a770b9 [#11368] fix(docker): Build bundle jars for Docker image
(#11388)
add 2311c4be8f [11391] fix(bundles): run FilesetS3CatalogIT hangs on macOS
(#11393)
add aef7acaf84 [#11212] test(authz): add CI workflow for multi-instance
consistency (#11213)
add 62e135a78f [#11141] improvement(ui):Change the OIDC callback address
based on NEXT_PUBLIC_BASE and catch error (#11142)
add 2adfe2321b [#11363] feat(iceberg-rest): wire async cleanup into the
REST drop flow (#11364)
add dff496720b Merge origin/main: forward-port asyncPurge to
iceberg-listener IcebergRequestContext
No new revisions were added by this update.
Summary of changes:
.../workflows/lance-compatibility-matrix-test.yml | 96 ++
.../workflows/multi-instance-consistency-test.yml | 117 +++
.../gravitino/s3/fs/S3FileSystemProvider.java | 3 +-
.../fileset/TestFilesetCatalogOperations.java | 6 +
.../integration/test/FilesetS3CatalogIT.java | 5 +-
.../catalog/kafka/TestKafkaCatalogOperations.java | 6 +
.../generic/TestGenericCatalogOperations.java | 6 +
catalogs/catalog-lakehouse-paimon/build.gradle.kts | 1 +
.../catalog/model/TestModelCatalogOperations.java | 6 +
clients/client-python/build.gradle.kts | 73 ++
clients/client-python/requirements-dev.txt | 8 +
clients/client-python/requirements-lance.txt | 8 +-
.../client-python/scripts/run_lance_ray_matrix.py | 355 +++++++
.../tests/integration/test_lance_ray.py | 41 +-
.../cache/it/AbstractEntityStorageBenchmark.java | 6 +
.../main/java/org/apache/gravitino/Configs.java | 28 +
.../java/org/apache/gravitino/GravitinoEnv.java | 2 +
.../catalog/CatalogChangeLogListener.java | 103 ++
.../apache/gravitino/catalog/CatalogManager.java | 82 ++
...CDatabase.java => EntityChangeLogListener.java} | 15 +-
.../storage/relational/EntityChangeLogPoller.java | 248 +++++
.../storage/relational/RelationalEntityStore.java | 27 +-
.../relational/SupportsEntityChangeLog.java | 50 +
.../relational/service/CatalogMetaService.java | 4 +-
.../authorization/TestAccessControlManager.java | 6 +
.../gravitino/authorization/TestOwnerManager.java | 6 +
.../gravitino/catalog/TestCatalogManager.java | 183 ++++
.../gravitino/hook/TestFilesetHookDispatcher.java | 6 +
.../gravitino/hook/TestTableHookDispatcher.java | 6 +
.../apache/gravitino/policy/TestPolicyManager.java | 6 +
.../gravitino/stats/TestStatisticManager.java | 6 +
.../storage/AbstractEntityStorageTest.java | 6 +
.../relational/TestEntityChangeLogPoller.java | 242 +++++
.../service/TestEntityChangeLogService.java | 31 +-
.../org/apache/gravitino/tag/TestTagManager.java | 6 +
design-docs/async-iceberg-rest-hard-deletion.md | 253 +++--
dev/ci/setup_multi_instance.sh | 157 +++
dev/ci/test_multi_instance_consistency.sh | 1051 ++++++++++++++++++++
dev/docker/gravitino/gravitino-dependency.sh | 14 +-
docs/gravitino-server-config.md | 3 +
docs/iceberg-rest-service.md | 20 +
docs/lance-rest-integration.md | 57 +-
.../iceberg/common/ops/IcebergCatalogWrapper.java | 32 +
.../listener/api/event/IcebergRequestContext.java | 21 +
.../api/event/TestIcebergRequestContext.java | 68 ++
.../org/apache/gravitino/iceberg/RESTService.java | 31 +-
.../service/cleanup/IcebergCleanupJobStore.java | 69 +-
.../service/cleanup/IcebergCleanupManager.java | 389 ++++++++
.../cleanup/mapper/IcebergCleanupJobMapper.java | 6 +-
.../IcebergCleanupJobSQLProviderFactory.java | 10 +-
.../IcebergCleanupMapperPackageProvider.java | 12 +-
.../base/IcebergCleanupJobBaseSQLProvider.java | 20 +-
.../service/dispatcher/IcebergCleanupHelper.java | 79 ++
.../IcebergNamespaceOperationExecutor.java | 12 +-
.../dispatcher/IcebergTableOperationExecutor.java | 57 +-
...elational.mapper.provider.MapperPackageProvider | 19 -
.../integration/test/IcebergRESTAsyncPurgeIT.java | 263 +++++
.../AbstractIcebergCleanupJobStoreBackendTest.java | 46 +-
.../service/cleanup/TestIcebergCleanupManager.java | 406 ++++++++
.../service/dispatcher/TestIcebergAsyncPurge.java | 260 +++++
.../TestIcebergNamespaceOperationExecutor.java | 3 +-
.../TestIcebergTableOperationExecutor.java | 3 +-
.../iceberg/service/rest/IcebergRestTestUtil.java | 8 +-
lance/lance-rest-server/build.gradle.kts | 131 ++-
.../jcasbin/JcasbinAuthorizationLookups.java | 33 +-
.../authorization/jcasbin/JcasbinAuthorizer.java | 24 +-
...hangePoller.java => JcasbinChangeListener.java} | 53 +-
.../jcasbin/TestJcasbinAuthorizationLookups.java | 40 +-
.../jcasbin/TestJcasbinChangePoller.java | 72 +-
.../TestIcebergCatalogPropertyConverter.java | 9 +-
web-v2/web/src/app/login/components/OidcLogin.js | 19 +-
web-v2/web/src/lib/auth/providers/oidc.js | 77 +-
web-v2/web/src/lib/auth/providers/oidc.test.js | 262 ++++-
73 files changed, 5489 insertions(+), 400 deletions(-)
create mode 100644 .github/workflows/lance-compatibility-matrix-test.yml
create mode 100644 .github/workflows/multi-instance-consistency-test.yml
create mode 100644 clients/client-python/scripts/run_lance_ray_matrix.py
create mode 100644
core/src/main/java/org/apache/gravitino/catalog/CatalogChangeLogListener.java
copy
core/src/main/java/org/apache/gravitino/storage/relational/{JDBCDatabase.java
=> EntityChangeLogListener.java} (68%)
create mode 100644
core/src/main/java/org/apache/gravitino/storage/relational/EntityChangeLogPoller.java
create mode 100644
core/src/main/java/org/apache/gravitino/storage/relational/SupportsEntityChangeLog.java
create mode 100644
core/src/test/java/org/apache/gravitino/storage/relational/TestEntityChangeLogPoller.java
create mode 100755 dev/ci/setup_multi_instance.sh
create mode 100755 dev/ci/test_multi_instance_consistency.sh
create mode 100644
iceberg/iceberg-listener/src/test/java/org/apache/gravitino/listener/api/event/TestIcebergRequestContext.java
create mode 100644
iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/cleanup/IcebergCleanupManager.java
create mode 100644
iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/dispatcher/IcebergCleanupHelper.java
delete mode 100644
iceberg/iceberg-rest-server/src/main/resources/META-INF/services/org.apache.gravitino.storage.relational.mapper.provider.MapperPackageProvider
create mode 100644
iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAsyncPurgeIT.java
create mode 100644
iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/cleanup/TestIcebergCleanupManager.java
create mode 100644
iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/dispatcher/TestIcebergAsyncPurge.java
rename
server-common/src/main/java/org/apache/gravitino/server/authorization/jcasbin/{JcasbinChangePoller.java
=> JcasbinChangeListener.java} (86%)
