This is an automated email from the ASF dual-hosted git repository.
mchades pushed a change to branch codex/issue-13540-ossie-import-export
in repository https://gitbox.apache.org/repos/asf/gravitino.git
discard 3bc95093d9 [#13540] feat(semantic-model): Expose typed Ossie import
and export APIs
discard 03e8826eec [#13540] fix(semantic-model): Address Ossie review comments
omit da4c2503cd [#13540] feat(semantic-model): Add Ossie import and export
add 8e9ca0009d [#13565] fix: Validate table and column field lengths
(#13551)
add 014fa11d58 [#13385] feat(core): Support default values for job
template placeholders (#13468)
add 97bb4a28c8 [#13523] fix(core): replace the Lance dataset cache with a
shared Session (#13528)
add 2e915108f5 [#13583] fix(spark-connector): Skip backend JDBC driver
preload when routing through Iceberg REST server (#13584)
add 502bd5f372 [#13595] improvement(mcp): move pylint/pytest/parameterized
to a dev group (#13596)
add 4aa6f2170c [#13502] fix(core): Replace a stale table registration
instead of inheriting it (#13503)
add b9f4786c46 [#13346] fix(doris): Preserve table comments on Doris 2.1.0
(#13588)
add 9f7be9d25e [#12206] improvement(core): decouple the fileset and policy
OCC version from their history version (#13484)
add 7a04352992 [#12740] fix(doris): Preserve default value on column type
change (#12747)
add c4453641e8 [#12176] feat(mcp-server): support policy-on-tag tools
(#13477)
add 0b091e12db [#11284] docs: Add design doc for Iceberg REST server-side
scan planning results (#12241)
add 2f95e46741 [#13538] fix(authz): Hide access key IDs and split
USE_SECRET from USE_CREDENTIAL (#13539)
add 279ed981b0 [#13004] fix(core): Fence live endpoints for relation
writes (#13486)
add 393924be59 [MINOR] fix(license): ship the binary LICENSE and NOTICE in
connector images (#13578)
add 2b514d5cd5 [#12608] feat(server): Add Semantic Model list, alter, and
drop REST APIs (#12626)
add e792572587 [#13540] feat(semantic-model): Add Ossie import and export
add 95e4b437f3 [#13540] fix(semantic-model): Address Ossie review comments
add 95b532832d [#13540] feat(semantic-model): Expose typed Ossie import
and export APIs
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (3bc95093d9)
\
N -- N -- N refs/heads/codex/issue-13540-ossie-import-export
(95b532832d)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.../apache/gravitino/authorization/Privilege.java | 13 +-
.../apache/gravitino/authorization/Privileges.java | 76 +-
.../gravitino/credential/CredentialInfos.java | 59 ++
.../apache/gravitino/secret/SupportsSecrets.java | 24 +-
.../authorization/TestSecurableObjects.java | 40 +-
.../gravitino/credential/TestCredentialInfos.java | 61 ++
.../gravitino/catalog/glue/GlueConstants.java | 4 +-
.../apache/gravitino/storage/AWSProperties.java | 2 +-
.../CloudStorageCredentialPropertyKeys.java | 88 +-
.../TestCloudStorageCredentialPropertyKeys.java | 71 +-
.../fileset/TestFilesetCatalogCredential.java | 11 +-
.../TestFilesetCloudPropertiesMetadata.java | 10 +-
.../glue/TestGlueCatalogPropertiesMetadata.java | 6 +-
.../catalog/hive/TestHiveCatalogOperations.java | 2 +-
.../apache/gravitino/catalog/jdbc/JdbcCatalog.java | 19 +
.../catalog/jdbc/TestJdbcCatalogCredential.java | 7 +-
.../DorisColumnDefaultValueConverter.java | 49 ++
.../doris/operation/DorisTableOperations.java | 175 +++-
.../TestDorisColumnDefaultValueConverter.java | 40 +
.../doris/integration/test/CatalogDoris2xIT.java | 171 ++++
.../doris/integration/test/CatalogDoris3xIT.java | 234 ++++++
.../doris/integration/test/CatalogDoris4xIT.java | 213 +++++
.../operation/TestDorisTableCommentOperations.java | 72 ++
.../doris/operation/TestDorisTableCreation.java | 234 ++++++
.../TestDorisTableOperationsSqlGeneration.java | 209 +++++
.../catalog/lakehouse/iceberg/IcebergCatalog.java | 12 +
.../lakehouse/iceberg/TestIcebergCatalog.java | 7 +-
.../TestIcebergCatalogPropertiesMetadata.java | 6 +-
.../TestPaimonCatalogPropertiesMetadata.java | 9 +-
.../test/authorization/FilesetAuthorizationIT.java | 13 +-
.../gravitino/api/authorization/privileges.py | 60 +-
.../client-python/gravitino/api/catalog_change.py | 3 +-
.../gravitino/filesystem/gvfs_base_operations.py | 84 +-
.../tests/unittests/test_filesystem_cache.py | 13 +-
.../tests/unittests/test_gvfs_merge_secrets.py | 164 ++++
.../tests/unittests/test_gvfs_with_hook.py | 4 +
.../tests/unittests/test_gvfs_with_local.py | 4 +
.../filesystem/hadoop/BaseGVFSOperations.java | 63 ++
.../filesystem/hadoop/GravitinoMockServerBase.java | 21 +
.../hadoop/TestBaseGVFSOperationsSecrets.java | 113 ++-
.../dto/requests/SemanticModelCreateRequest.java | 1 +
.../dto/requests/SemanticModelUpdateRequest.java | 245 ++++++
...quest.java => SemanticModelUpdatesRequest.java} | 32 +-
.../dto/semantic/SemanticModelDefinitionDTO.java | 9 +
.../semantic/SemanticModelDefinitionValidator.java | 217 +++++
.../requests/TestSemanticModelCreateRequest.java | 3 +-
.../requests/TestSemanticModelUpdateRequest.java | 168 ++++
.../semantic/TestSemanticModelDefinitionDTO.java | 63 ++
.../org/apache/gravitino/EntityFieldLimits.java | 3 +
.../catalog/TableOperationDispatcher.java | 39 +-
.../cloud/storage/AWSPropertiesMetadata.java | 4 +-
.../cloud/storage/AzurePropertiesMetadata.java | 1 +
.../cloud/storage/COSPropertiesMetadata.java | 2 +-
.../cloud/storage/OSSPropertiesMetadata.java | 2 +-
.../cloud/storage/S3PropertiesMetadata.java | 2 +-
.../apache/gravitino/connector/BaseCatalog.java | 62 ++
.../connector/HiddenPropertyMaskUtils.java | 8 +-
.../java/org/apache/gravitino/job/JobManager.java | 203 +----
.../gravitino/job/JobTemplatePlaceholderUtils.java | 366 +++++++++
.../apache/gravitino/job/JobTemplateResolver.java | 247 ++++++
.../org/apache/gravitino/meta/ColumnEntity.java | 7 +-
.../org/apache/gravitino/meta/TableEntity.java | 4 +-
.../secret/FallbackPropertiesMetadata.java | 14 +-
.../gravitino/secret/SecretPropertyUtils.java | 54 +-
.../storage/LancePartitionStatisticStorage.java | 182 +----
.../relational/mapper/FilesetMetaMapper.java | 18 +-
.../mapper/FilesetMetaSQLProviderFactory.java | 11 +-
.../relational/mapper/LiveEndpointMapper.java | 21 +-
.../relational/mapper/LiveEndpointSQLProvider.java | 136 ++++
.../relational/mapper/PolicyMetaMapper.java | 6 +-
.../mapper/PolicyMetaSQLProviderFactory.java | 4 +-
.../provider/DefaultMapperPackageProvider.java | 2 +
.../provider/base/FilesetMetaBaseSQLProvider.java | 134 ++-
.../provider/base/PolicyMetaBaseSQLProvider.java | 25 +-
.../postgresql/FilesetMetaPostgreSQLProvider.java | 44 +-
.../postgresql/PolicyMetaPostgreSQLProvider.java | 4 +-
.../gravitino/storage/relational/po/FilesetPO.java | 33 +-
.../gravitino/storage/relational/po/PolicyPO.java | 17 +
.../relational/service/FilesetMetaService.java | 16 +-
.../relational/service/LiveEndpointService.java | 131 +++
.../relational/service/PolicyMetaService.java | 26 +-
.../relational/service/PolicyTagRelService.java | 5 +
.../relational/service/RoleMetaService.java | 87 +-
.../relational/service/TableMetaService.java | 41 +-
.../storage/relational/service/TagMetaService.java | 18 +-
.../storage/relational/utils/POConverters.java | 164 +++-
.../catalog/TestTableOperationDispatcher.java | 223 +++++
.../cloud/storage/TestCloudPropertiesMetadata.java | 8 +-
.../TestBaseCatalogPropertiesMetadata.java | 17 +-
.../TestBaseCatalogCredentialSecrets.java | 68 ++
.../org/apache/gravitino/job/TestJobManager.java | 232 +++---
.../job/TestJobTemplatePlaceholderUtils.java | 433 ++++++++++
...bTemplate.java => TestJobTemplateResolver.java} | 357 +++++---
.../gravitino/job/local/TestLocalJobExecutor.java | 59 +-
.../gravitino/meta/TestEntityFieldLimits.java | 32 +
.../TestSecretPropertyOperationDispatcher.java | 20 +-
.../gravitino/secret/TestSecretPropertyUtils.java | 51 +-
.../TestLancePartitionStatisticStorage.java | 265 ------
.../apache/gravitino/storage/TestSQLScripts.java | 54 ++
.../storage/relational/TestJDBCBackend.java | 26 +
.../base/TestFilesetMetaBaseSQLProvider.java | 48 +-
.../TestFilesetMetaPostgreSQLProvider.java | 20 +-
.../relational/service/TestFilesetMetaService.java | 278 ++++++-
.../service/TestLiveEndpointService.java | 171 ++++
.../relational/service/TestPolicyMetaService.java | 119 ++-
.../relational/service/TestRoleMetaService.java | 398 +++++++++
.../relational/service/TestTableMetaService.java | 70 +-
.../relational/service/TestTagMetaService.java | 71 ++
.../storage/relational/utils/TestPOConverters.java | 103 ++-
design-docs/gravitino-glue-catalog.md | 4 +-
.../iceberg-expire-snapshots-maintenance-job.md | 2 +-
.../iceberg-rest-server-side-scan-planning.md | 894 +++++++++++++++++++++
dev/docker/build-docker.sh | 4 +-
dev/docker/flink-connectors/Dockerfile | 5 +-
dev/docker/flink-connectors/README.md | 12 +-
.../flink-connectors-dependency.sh | 23 +-
dev/docker/flink-connectors/licenses/.gitignore | 5 +-
.../licenses/THIRD_PARTY_LICENSES.txt | 20 -
dev/docker/mcp-server/Dockerfile | 7 +-
dev/docker/mcp-server/start-mcp-server.sh | 5 +-
dev/docker/spark-connectors/Dockerfile | 6 +-
dev/docker/spark-connectors/README.md | 12 +-
dev/docker/spark-connectors/licenses/.gitignore | 5 +-
.../licenses/THIRD_PARTY_LICENSES.txt | 21 -
.../spark-connectors-dependency.sh | 23 +-
dev/docker/trino-connectors/Dockerfile | 6 +-
dev/docker/trino-connectors/README.md | 8 +-
dev/docker/trino-connectors/licenses/.gitignore | 5 +-
.../licenses/THIRD_PARTY_LICENSES.txt | 24 -
.../trino-connectors-dependency.sh | 25 +-
docs/apache-hive-catalog.md | 2 +-
docs/fileset-catalog-with-cos.md | 10 +-
docs/fileset-catalog-with-oss.md | 7 +
docs/fileset-catalog-with-s3.md | 4 +
docs/fileset-catalog.md | 8 +-
docs/gravitino-mcp-server.md | 135 ++--
docs/jdbc-doris-catalog.md | 19 +-
docs/jdbc-hologres-catalog.md | 2 +-
docs/jdbc-mysql-catalog.md | 2 +-
docs/jdbc-oceanbase-catalog.md | 2 +-
docs/jdbc-postgresql-catalog.md | 2 +-
docs/jdbc-starrocks-catalog.md | 2 +-
docs/lakehouse-iceberg-catalog.md | 2 +-
docs/manage-jobs-in-gravitino.md | 51 +-
docs/manage-statistics-in-gravitino.md | 1 -
docs/open-api/roles.yaml | 3 +-
docs/security/access-control.md | 11 +-
docs/security/credential-vending.md | 27 +-
.../optimizer-cli-reference.md | 21 +-
.../optimizer-configuration.md | 15 +-
.../connector/store/GravitinoCatalogStore.java | 3 +-
.../flink/connector/utils/PropertyUtils.java | 76 +-
.../connector/store/TestGravitinoCatalogStore.java | 32 +-
.../flink/connector/utils/TestPropertyUtils.java | 29 +-
.../iceberg/common/ops/IcebergCatalogWrapper.java | 15 +
.../common/ops/TestIcebergCatalogWrapper.java | 26 +
.../dispatcher/IcebergColumnFieldValidator.java | 49 ++
.../IcebergNamespaceOperationExecutor.java | 15 +-
.../dispatcher/IcebergTableOperationExecutor.java | 10 +
.../provider/DynamicIcebergConfigProvider.java | 49 +-
.../service/dispatcher/TestIcebergAsyncPurge.java | 4 +
.../TestIcebergNamespaceOperationExecutor.java | 141 ++++
.../TestIcebergTableOperationExecutor.java | 139 ++++
.../provider/TestDynamicIcebergConfigProvider.java | 51 +-
.../service/rest/CatalogWrapperForTest.java | 25 +-
.../integration/test/container/DorisImageName.java | 2 +
maintenance/jobs/build.gradle.kts | 2 +
.../jobs/iceberg/IcebergExpireSnapshotsJob.java | 45 +-
.../jobs/iceberg/IcebergRemoveOrphanFilesJob.java | 10 +-
.../jobs/iceberg/IcebergRewriteDataFilesJob.java | 12 +-
.../jobs/iceberg/IcebergRewriteManifestsJob.java | 8 +-
.../iceberg/IcebergUpdateStatsAndMetricsJob.java | 8 +-
.../maintenance/jobs/spark/SparkPiJob.java | 14 +-
.../maintenance/jobs/TemplateArguments.java | 70 ++
.../jobs/TestBuiltInJobTemplateProvider.java | 64 ++
.../iceberg/TestIcebergExpireSnapshotsJob.java | 60 +-
.../iceberg/TestIcebergRemoveOrphanFilesJob.java | 3 +-
.../TestIcebergRemoveOrphanFilesJobMain.java | 7 +-
.../iceberg/TestIcebergRewriteDataFilesJob.java | 40 +-
.../iceberg/TestIcebergRewriteManifestsJob.java | 32 +-
.../TestIcebergRewriteManifestsJobWithSpark.java | 14 +-
.../jobs/iceberg/TestIcebergUpdateStatsJob.java | 28 +-
.../maintenance/jobs/spark/TestSparkPiJob.java | 14 +-
.../common/util/IcebergSparkConfigUtils.java | 11 +-
mcp-server/build.gradle.kts | 39 +-
.../plain/plain_rest_client_policy_operation.py | 42 +-
mcp-server/mcp_server/client/policy_operation.py | 54 +-
mcp-server/mcp_server/tools/policy.py | 157 +++-
mcp-server/pyproject.toml | 13 +
mcp-server/tests/unit/client/test_url_encoding.py | 58 ++
mcp-server/tests/unit/tools/mock_operation.py | 17 +-
mcp-server/tests/unit/tools/test_policy.py | 83 +-
mcp-server/tests/unit/tools/test_statistic.py | 1 -
mcp-server/uv.lock | 209 +++--
scripts/h2/schema-2.0.0-h2.sql | 2 +
scripts/h2/upgrade-1.3.0-to-2.0.0-h2.sql | 12 +
scripts/mysql/schema-2.0.0-mysql.sql | 2 +
scripts/mysql/upgrade-1.3.0-to-2.0.0-mysql.sql | 12 +
scripts/postgresql/schema-2.0.0-postgresql.sql | 4 +
.../upgrade-1.3.0-to-2.0.0-postgresql.sql | 12 +
.../AuthorizationExpressionConstants.java | 21 +-
.../AuthorizationExpressionConverter.java | 13 +-
.../TestAuthorizationExpressionConverter.java | 20 +-
.../rest/MetadataObjectCredentialOperations.java | 6 -
.../web/rest/MetadataObjectSecretOperations.java | 20 +-
.../server/web/rest/SemanticModelOperations.java | 146 +++-
.../TestMetadataObjectCredentialOperations.java | 46 --
.../rest/TestMetadataObjectSecretOperations.java | 123 ++-
.../web/rest/TestSemanticModelOperations.java | 168 +++-
.../spark/connector/catalog/BaseCatalog.java | 31 +-
.../connector/iceberg/GravitinoIcebergCatalog.java | 53 +-
.../connector/catalog/TestBaseCatalogSecrets.java | 104 ++-
.../TestGravitinoIcebergCatalogRestRouting.java | 68 ++
.../test/authorization/SparkAuthorizationIT.java | 8 +-
.../authorization/SparkJwksAuthorizationIT.java | 6 +-
.../connector/catalog/CatalogConnectorManager.java | 62 +-
.../trino/connector/GravitinoMockServer.java | 5 +
.../catalog/TestCatalogConnectorManager.java | 68 +-
218 files changed, 10599 insertions(+), 2106 deletions(-)
create mode 100644
api/src/main/java/org/apache/gravitino/credential/CredentialInfos.java
create mode 100644
api/src/test/java/org/apache/gravitino/credential/TestCredentialInfos.java
create mode 100644
catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDoris2xIT.java
create mode 100644
catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableCreation.java
create mode 100644
common/src/main/java/org/apache/gravitino/dto/requests/SemanticModelUpdateRequest.java
copy
common/src/main/java/org/apache/gravitino/dto/requests/{TableUpdatesRequest.java
=> SemanticModelUpdatesRequest.java} (69%)
create mode 100644
common/src/main/java/org/apache/gravitino/dto/semantic/SemanticModelDefinitionValidator.java
create mode 100644
common/src/test/java/org/apache/gravitino/dto/requests/TestSemanticModelUpdateRequest.java
create mode 100644
core/src/main/java/org/apache/gravitino/job/JobTemplatePlaceholderUtils.java
create mode 100644
core/src/main/java/org/apache/gravitino/job/JobTemplateResolver.java
copy api/src/main/java/org/apache/gravitino/Auditable.java =>
core/src/main/java/org/apache/gravitino/storage/relational/mapper/LiveEndpointMapper.java
(59%)
create mode 100644
core/src/main/java/org/apache/gravitino/storage/relational/mapper/LiveEndpointSQLProvider.java
create mode 100644
core/src/main/java/org/apache/gravitino/storage/relational/service/LiveEndpointService.java
create mode 100644
core/src/test/java/org/apache/gravitino/job/TestJobTemplatePlaceholderUtils.java
rename core/src/test/java/org/apache/gravitino/job/{TestJobTemplate.java =>
TestJobTemplateResolver.java} (65%)
create mode 100644
core/src/test/java/org/apache/gravitino/storage/relational/service/TestLiveEndpointService.java
create mode 100644 design-docs/iceberg-rest-server-side-scan-planning.md
delete mode 100644
dev/docker/flink-connectors/licenses/THIRD_PARTY_LICENSES.txt
delete mode 100644
dev/docker/spark-connectors/licenses/THIRD_PARTY_LICENSES.txt
delete mode 100644
dev/docker/trino-connectors/licenses/THIRD_PARTY_LICENSES.txt
create mode 100644
iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/dispatcher/IcebergColumnFieldValidator.java
create mode 100644
maintenance/jobs/src/test/java/org/apache/gravitino/maintenance/jobs/TemplateArguments.java
