This is an automated email from the ASF dual-hosted git repository.
czy006 pushed a change to branch 0.9.x
in repository https://gitbox.apache.org/repos/asf/amoro.git
from 09d8364ca [Hotfix][AMS] Unify authentication handling for dashboard
requests (#4312)
add cbd7d6565 [hotfix] Fix S3 credentials not vended when loading Lance
table (#4279)
add d29cbf4bc [AMORO-4271] AIP-5 Phase 2: demand-driven scale-up for
dynamic allocation groups (#4272)
add 0cb625072 [AMORO-4274] Remove Roaring64Bitmap.isEmpty() check in
CombinedDelete… (#4275)
add 0704a1ee9 [AMORO-4281]Fix cleanning process trigger issue for mixed
format tables (#4282)
add da51a2d58 [AMORO-4240][AMS] Support namespace properties in REST
catalog (#4283)
add eee2e3573 [AMORO-4126][ams] Default waitAppCompletion=false for Spark
optimizer on Kubernetes (#4289)
add 7be0710b3 [AMORO-4277] imporove expire_snapshots performance (#4278)
add 0dfaa4b15 [AMORO-4290][AMS] Propagate internal Iceberg table commit
failures (#4291)
add 436a3173b [AMORO-4286]Change process status to failed when recover
from AMS restarting (#4288)
add ea4ea4cd3 [Hotfix] Pin all external actions to the specific git hash
(#4294)
add ec28554a2 [Hotfix] Change the docker action version to match ASF
policy (#4299)
add b7bf0d39d [AMORO-4297][AMS] Persist failure reasons in task quota
history (#4298)
add 38b2b49e7 [AMORO-4240][AMS] Support staged table creation in REST
catalog (#4284)
add acb9a49a4 [AMORO-4300] Recover active processes after table ownership
transfer (#4304)
add efbedc396 [AMORO-4301][ams] Fix Kubernetes executor label prefix
(#4302)
add a0640e0c9 [AMORO-2099][amoro-web] Remove default Glue lock
configuration (#4305)
add 7e9f9a769 [Hotfix][tools] Include Helm chart in change-version script
(#4306)
add 4953d9c90 [AMORO-4303][AMS] Stabilize optimizer heartbeat tests (#4307)
add 7c791a5f5 [AMORO-4308][lance] Remove redundant getTables call (#4309)
add c26fd6e78 [AMORO-4266] Introduce Amoro MCP server (#4285)
add 287f4586b [AMORO-4295] AIP-5 Phase 3+4: idle-driven scale-down with
graceful drain for dynamic allocation groups (#4296)
add 9f2a45e88 [AMORO-4139] Delete resource and optimizer atomically when
releasing optimizer (#4310)
add 1b094b410 [Hotfix][AMS] Unify authentication handling for dashboard
requests (#4312)
add 219550114 [Docs] Fix the broken Star History chart in the README
(#4315)
add 56c34f61e [AMORO-4313] AIP-5 Phase 5: observability metrics for
dynamic allocation (#4314)
add 072560927 [Hotfix][AMS] Stabilize optimizer scale-down floor test
(#4332)
add f5098f8d0 [hotfix][Process] Preserve recovery failure context (#4321)
add 4c141b393 [AMORO-4318] Add mixed Flink 1.20 support (#4319)
add 6ab54c515 [AMORO-4316][AMS] Support monthly period for auto-creating
Iceberg tags (#4317)
add de6205c0e [AMORO-4264][AMS] Use a shared pool for Iceberg maintenance
planning (#4311)
add 96a4d781a [hotfix][Optimizer] Recognize CLOSED optimizing processes
(#4331)
add 7333642cf [hotfix][AMS] Clean up orphan optimizer records at startup
(#4320)
add 72a772986 [hotfix][Optimizer] Remove zero-valued in-flight task
counters (#4328)
add 1c894691b [hotfix][Optimizer] Guard invalid fragment ratio and target
quota (#4324)
add 4f26c4574 [hotfix][Optimizer] Preserve keeper visibility on lookup
failure (#4325)
add 505a66e1e [AMORO-4280] Add support for Lance format on REST metastore
(#4334)
add f4e73310b [Hotfix] Change log4j impl dependency to `log4j-slf4j2-impl`
(#4336)
add 18859a662 [AMORO-4337] Upgrade Iceberg dependency to 1.8.1 (#4338)
add 52ef6b190 [Hotfix] Remove slf4j classes from kyuubi-hive-jdbc-shaded
(#4342)
add 5d976fc71 [Refactor] Unify the main process of amoro service container
(#4339)
add fc0c2e8e3 [AMORO-4343][ams] Fix runtime creation for unsupported table
formats (#4344)
new 6239b3917 Merge remote-tracking branch 'origin/master' into 0.9.x
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/mcp-server-ci.yml | 77 ++
.gitignore | 3 +
README.md | 4 +-
amoro-ams/pom.xml | 46 +-
.../apache/amoro/server/AmoroManagementConf.java | 7 +
.../apache/amoro/server/AmoroServiceContainer.java | 155 +--
.../amoro/server/DefaultOptimizingService.java | 647 ++++++++++-
.../apache/amoro/server/RestCatalogService.java | 175 ++-
.../amoro/server/catalog/CatalogBuilder.java | 9 +-
.../DatabaseMetadata.java} | 33 +-
.../amoro/server/catalog/InternalCatalog.java | 58 +-
.../amoro/server/catalog/InternalCatalogImpl.java | 7 +-
.../dashboard/controller/CatalogController.java | 16 +-
.../server/optimizing/OptimizerGroupMetrics.java | 42 +
.../amoro/server/optimizing/OptimizingQueue.java | 69 +-
.../amoro/server/optimizing/SchedulingPolicy.java | 16 +
.../optimizing/dra/DynamicAllocationConfig.java | 90 ++
.../optimizing/dra/DynamicAllocationMetrics.java | 142 +++
.../optimizing/dra/DynamicAllocationState.java | 332 ++++++
.../optimizing/dra/PendingRegistrations.java | 71 ++
.../server/persistence/mapper/OptimizerMapper.java | 7 +
.../server/persistence/mapper/ResourceMapper.java | 7 +
.../server/persistence/mapper/TableMetaMapper.java | 48 +-
.../amoro/server/process/ProcessService.java | 12 +-
.../server/resource/DefaultOptimizerManager.java | 23 +-
.../amoro/server/resource/OptimizerInstance.java | 4 +-
.../amoro/server/table/DefaultTableRuntime.java | 6 +-
.../amoro/server/table/DefaultTableService.java | 7 +
.../amoro/server/table/TableConfigurations.java | 3 +
.../internal/IcebergInternalTableOperations.java | 12 +-
.../table/internal/InternalIcebergCreator.java | 47 +-
.../internal/InternalMixedIcebergCreator.java | 77 +-
.../table/internal/InternalTableCreator.java | 15 +-
.../src/main/resources/derby/ams-derby-init.sql | 3 +-
.../upgrade.sql} | 3 +-
.../src/main/resources/mysql/ams-mysql-init.sql | 1 +
amoro-ams/src/main/resources/mysql/upgrade.sql | 3 +
.../main/resources/postgres/ams-postgres-init.sql | 4 +-
amoro-ams/src/main/resources/postgres/upgrade.sql | 6 +-
.../org/apache/amoro/server/AmsEnvironment.java | 4 +-
.../amoro/server/TestAmoroManagementConf.java | 8 +
.../amoro/server/TestDefaultOptimizingService.java | 224 ++++
.../server/TestInternalIcebergCatalogService.java | 215 +++-
.../amoro/server/TestOptimizerGroupKeeper.java | 50 +-
.../TestOptimizerResourceGroupConcurrency.java | 203 ++++
.../amoro/server/TestOptimizerScaleKeeper.java | 829 ++++++++++++++
.../amoro/server/TestRestCatalogService.java | 43 +
.../server/optimizing/TestOptimizingQueue.java | 178 +++
.../optimizing/dra/TestComputeScaleDown.java | 208 ++++
.../server/optimizing/dra/TestComputeScaleUp.java | 282 +++++
.../dra/TestDynamicAllocationConfig.java | 130 +++
.../dra/TestDynamicAllocationMetrics.java | 134 +++
.../optimizing/dra/TestDynamicAllocationState.java | 175 +++
.../optimizing/dra/TestPendingRegistrations.java | 87 ++
.../amoro/server/persistence/TestDerbyUpgrade.java | 80 ++
.../persistence/mapper/TestTableMetaMapper.java | 166 +++
.../table/TestSyncUnsupportedTableFormat.java | 59 +
.../TestIcebergInternalTableOperations.java | 96 ++
.../java/org/apache/amoro/OptimizerProperties.java | 6 +
.../org/apache/amoro/config/OptimizingConfig.java | 2 +-
.../org/apache/amoro/config/TagConfiguration.java | 18 +-
.../java/org/apache/amoro/utils/CatalogUtil.java | 12 +
.../apache/amoro/config/TestOptimizingConfig.java | 47 +
amoro-format-hudi/pom.xml | 4 +
.../iceberg/maintainer/IcebergTableMaintainer.java | 2 +
.../optimizing/plan/CommonPartitionEvaluator.java | 2 +-
.../org/apache/amoro/table/TableProperties.java | 1 +
.../org/apache/amoro/utils/IcebergThreadPools.java | 30 +
.../maintainer/TestAutoCreateIcebergTagAction.java | 88 ++
.../maintainer/TestIcebergTableMaintainer.java | 62 ++
.../optimizing/IcebergRewriteExecutorTest.java | 2 +-
.../plan/TestCommonPartitionEvaluator.java | 82 ++
.../apache/amoro/utils/TestIcebergThreadPools.java | 14 +
.../amoro/formats/lance/AbstractLanceCatalog.java | 111 ++
.../amoro/formats/lance/LanceCatalogFactory.java | 4 +
.../formats/lance/LanceDirectoryV1Catalog.java | 121 +-
.../amoro/formats/lance/LanceRestCatalog.java | 134 +++
.../flink/source/RowDataFileScanTaskReader.java | 6 +-
.../amoro-mixed-flink-common/pom.xml | 8 +-
.../lookup/filter/TestRowDataPredicateBase.java | 5 +-
.../flink/util/MixedFormatMockEnvironment.java | 3 +
amoro-format-mixed/amoro-mixed-flink/pom.xml | 2 +
.../amoro-mixed-flink-1.20}/pom.xml | 12 +-
.../amoro-mixed-flink-runtime-1.20}/pom.xml | 12 +-
.../amoro-mixed-spark-3-common/pom.xml | 12 +
.../v3.3/amoro-mixed-spark-3.3/pom.xml | 17 +
.../v3.4/amoro-mixed-spark-3.4/pom.xml | 8 +
.../v3.5/amoro-mixed-spark-3.5/pom.xml | 8 +
amoro-format-mixed/amoro-mixed-trino/pom.xml | 14 +
amoro-format-paimon/pom.xml | 10 +
amoro-mcp-server/README.md | 319 ++++++
amoro-mcp-server/pyproject.toml | 68 ++
.../src/amoro_mcp_server/__init__.py | 6 +-
amoro-mcp-server/src/amoro_mcp_server/actions.py | 235 ++++
amoro-mcp-server/src/amoro_mcp_server/auth.py | 388 +++++++
amoro-mcp-server/src/amoro_mcp_server/client.py | 439 ++++++++
.../src/amoro_mcp_server/compatibility.py | 55 +
amoro-mcp-server/src/amoro_mcp_server/config.py | 121 ++
.../src/amoro_mcp_server/logging_config.py | 74 ++
amoro-mcp-server/src/amoro_mcp_server/models.py | 83 ++
.../src/amoro_mcp_server/request_context.py | 62 ++
amoro-mcp-server/src/amoro_mcp_server/sanitize.py | 198 ++++
amoro-mcp-server/src/amoro_mcp_server/server.py | 140 +++
.../src/amoro_mcp_server/tools/__init__.py | 57 +
.../tools/check_ams_connectivity.py | 40 +
.../src/amoro_mcp_server/tools/common.py | 19 +-
.../amoro_mcp_server/tools/get_table_details.py | 38 +
.../src/amoro_mcp_server/tools/list_catalogs.py | 19 +-
.../src/amoro_mcp_server/tools/list_databases.py | 42 +-
.../src/amoro_mcp_server/tools/list_optimizers.py | 45 +
.../tools/list_optimizing_processes.py | 58 +
.../tools/list_optimizing_tables.py | 48 +
.../tools/list_optimizing_tasks.py | 58 +
.../amoro_mcp_server/tools/list_resource_groups.py | 19 +-
.../amoro_mcp_server/tools/list_table_snapshots.py | 64 ++
.../src/amoro_mcp_server/tools/list_tables.py | 46 +
.../tests/conftest.py | 17 +-
amoro-mcp-server/tests/test_actions.py | 295 +++++
amoro-mcp-server/tests/test_auth.py | 172 +++
amoro-mcp-server/tests/test_client.py | 279 +++++
amoro-mcp-server/tests/test_compatibility.py | 156 +++
.../tests/test_logging.py | 16 +-
amoro-mcp-server/tests/test_sanitize.py | 132 +++
amoro-mcp-server/tests/test_server.py | 340 ++++++
amoro-mcp-server/uv.lock | 1159 ++++++++++++++++++++
amoro-optimizer/amoro-optimizer-spark/pom.xml | 8 +
amoro-optimizer/amoro-optimizer-standalone/pom.xml | 2 +-
amoro-web/src/views/catalogs/Detail.vue | 280 +++--
charts/amoro/templates/amoro-configmap.yaml | 1 +
dev/deps/dependencies-hadoop-2-spark-3.3 | 34 +-
dev/deps/dependencies-hadoop-3-spark-3.5 | 34 +-
dist/src/main/amoro-bin/conf/config.yaml | 1 +
docs/_index.md | 2 +-
docs/admin-guides/managing-optimizers.md | 11 +-
docs/configuration/ams-config.md | 1 +
docs/engines/flink/flink-cdc-ingestion.md | 4 +-
docs/engines/flink/flink-get-started.md | 2 +-
docs/engines/flink/using-logstore.md | 2 +
docs/user-guides/configurations.md | 16 +-
docs/user-guides/metrics.md | 17 +
pom.xml | 45 +-
141 files changed, 11568 insertions(+), 516 deletions(-)
create mode 100644 .github/workflows/mcp-server-ci.yml
copy
amoro-ams/src/main/java/org/apache/amoro/server/{resource/ContainerMetadata.java
=> catalog/DatabaseMetadata.java} (60%)
create mode 100644
amoro-ams/src/main/java/org/apache/amoro/server/optimizing/dra/DynamicAllocationMetrics.java
create mode 100644
amoro-ams/src/main/java/org/apache/amoro/server/optimizing/dra/DynamicAllocationState.java
create mode 100644
amoro-ams/src/main/java/org/apache/amoro/server/optimizing/dra/PendingRegistrations.java
copy amoro-ams/src/main/resources/{mysql/upgrade-0.5.0-to-0.5.1.sql =>
derby/upgrade.sql} (84%)
create mode 100644
amoro-ams/src/test/java/org/apache/amoro/server/TestOptimizerResourceGroupConcurrency.java
create mode 100644
amoro-ams/src/test/java/org/apache/amoro/server/TestOptimizerScaleKeeper.java
create mode 100644
amoro-ams/src/test/java/org/apache/amoro/server/TestRestCatalogService.java
create mode 100644
amoro-ams/src/test/java/org/apache/amoro/server/optimizing/dra/TestComputeScaleDown.java
create mode 100644
amoro-ams/src/test/java/org/apache/amoro/server/optimizing/dra/TestComputeScaleUp.java
create mode 100644
amoro-ams/src/test/java/org/apache/amoro/server/optimizing/dra/TestDynamicAllocationMetrics.java
create mode 100644
amoro-ams/src/test/java/org/apache/amoro/server/optimizing/dra/TestDynamicAllocationState.java
create mode 100644
amoro-ams/src/test/java/org/apache/amoro/server/optimizing/dra/TestPendingRegistrations.java
create mode 100644
amoro-ams/src/test/java/org/apache/amoro/server/persistence/TestDerbyUpgrade.java
create mode 100644
amoro-ams/src/test/java/org/apache/amoro/server/persistence/mapper/TestTableMetaMapper.java
create mode 100644
amoro-ams/src/test/java/org/apache/amoro/server/table/TestSyncUnsupportedTableFormat.java
create mode 100644
amoro-ams/src/test/java/org/apache/amoro/server/table/internal/TestIcebergInternalTableOperations.java
create mode 100644
amoro-common/src/test/java/org/apache/amoro/config/TestOptimizingConfig.java
create mode 100644
amoro-format-iceberg/src/test/java/org/apache/amoro/formats/iceberg/maintainer/TestIcebergTableMaintainer.java
create mode 100644
amoro-format-iceberg/src/test/java/org/apache/amoro/optimizing/plan/TestCommonPartitionEvaluator.java
create mode 100644
amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/AbstractLanceCatalog.java
create mode 100644
amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceRestCatalog.java
copy amoro-format-mixed/amoro-mixed-flink/{v1.19/amoro-mixed-flink-1.19 =>
v1.20/amoro-mixed-flink-1.20}/pom.xml (91%)
copy
amoro-format-mixed/amoro-mixed-flink/{v1.19/amoro-mixed-flink-runtime-1.19 =>
v1.20/amoro-mixed-flink-runtime-1.20}/pom.xml (97%)
create mode 100644 amoro-mcp-server/README.md
create mode 100644 amoro-mcp-server/pyproject.toml
copy amoro-ams/src/test/resources/docker-java.properties =>
amoro-mcp-server/src/amoro_mcp_server/__init__.py (85%)
create mode 100644 amoro-mcp-server/src/amoro_mcp_server/actions.py
create mode 100644 amoro-mcp-server/src/amoro_mcp_server/auth.py
create mode 100644 amoro-mcp-server/src/amoro_mcp_server/client.py
create mode 100644 amoro-mcp-server/src/amoro_mcp_server/compatibility.py
create mode 100644 amoro-mcp-server/src/amoro_mcp_server/config.py
create mode 100644 amoro-mcp-server/src/amoro_mcp_server/logging_config.py
create mode 100644 amoro-mcp-server/src/amoro_mcp_server/models.py
create mode 100644 amoro-mcp-server/src/amoro_mcp_server/request_context.py
create mode 100644 amoro-mcp-server/src/amoro_mcp_server/sanitize.py
create mode 100644 amoro-mcp-server/src/amoro_mcp_server/server.py
create mode 100644 amoro-mcp-server/src/amoro_mcp_server/tools/__init__.py
create mode 100644
amoro-mcp-server/src/amoro_mcp_server/tools/check_ams_connectivity.py
copy amoro-ams/src/test/resources/docker-java.properties =>
amoro-mcp-server/src/amoro_mcp_server/tools/common.py (58%)
create mode 100644
amoro-mcp-server/src/amoro_mcp_server/tools/get_table_details.py
copy
amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/resources/docker-java.properties
=> amoro-mcp-server/src/amoro_mcp_server/tools/list_catalogs.py (56%)
copy .github/workflows/take.yml =>
amoro-mcp-server/src/amoro_mcp_server/tools/list_databases.py (50%)
create mode 100644
amoro-mcp-server/src/amoro_mcp_server/tools/list_optimizers.py
create mode 100644
amoro-mcp-server/src/amoro_mcp_server/tools/list_optimizing_processes.py
create mode 100644
amoro-mcp-server/src/amoro_mcp_server/tools/list_optimizing_tables.py
create mode 100644
amoro-mcp-server/src/amoro_mcp_server/tools/list_optimizing_tasks.py
copy
amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/resources/docker-java.properties
=> amoro-mcp-server/src/amoro_mcp_server/tools/list_resource_groups.py (54%)
create mode 100644
amoro-mcp-server/src/amoro_mcp_server/tools/list_table_snapshots.py
create mode 100644 amoro-mcp-server/src/amoro_mcp_server/tools/list_tables.py
copy amoro-ams/src/test/resources/docker-java.properties =>
amoro-mcp-server/tests/conftest.py (63%)
create mode 100644 amoro-mcp-server/tests/test_actions.py
create mode 100644 amoro-mcp-server/tests/test_auth.py
create mode 100644 amoro-mcp-server/tests/test_client.py
create mode 100644 amoro-mcp-server/tests/test_compatibility.py
copy amoro-ams/src/test/resources/docker-java.properties =>
amoro-mcp-server/tests/test_logging.py (58%)
create mode 100644 amoro-mcp-server/tests/test_sanitize.py
create mode 100644 amoro-mcp-server/tests/test_server.py
create mode 100644 amoro-mcp-server/uv.lock
