This is an automated email from the ASF dual-hosted git repository. xxubai pushed a commit to branch 0.9.x in repository https://gitbox.apache.org/repos/asf/amoro.git
commit 30580a3aee87265ec5d97192e7c60f91ef0b58ef Merge: 7a65790f4 a93a32b4d Author: xuba <[email protected]> AuthorDate: Fri Jul 31 14:48:36 2026 +0800 Merge remote-tracking branch 'origin/0.9.0-rc4' into 0.9.x # Conflicts: # amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java # amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConfValidator.java # amoro-ams/src/main/java/org/apache/amoro/server/process/iceberg/DanglingDeleteFilesCleaningProcess.java # amoro-ams/src/main/java/org/apache/amoro/server/process/iceberg/DataExpiringProcess.java # amoro-ams/src/main/java/org/apache/amoro/server/process/iceberg/IcebergProcessFactory.java # amoro-ams/src/main/java/org/apache/amoro/server/process/iceberg/OrphanFilesCleaningProcess.java # amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/InlineTableExecutors.java # amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultTableRuntime.java # amoro-ams/src/test/java/org/apache/amoro/server/AmsEnvironment.java # amoro-ams/src/test/java/org/apache/amoro/server/TestAmoroManagementConfValidator.java # amoro-ams/src/test/java/org/apache/amoro/server/process/iceberg/TestIcebergProcessFactory.java # amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/PeriodicTableSchedulerTestBase.java # amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/TestConfigurableIntervalExecutors.java # amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/TestPeriodicTableSchedulerCleanup.java # amoro-common/src/main/java/org/apache/amoro/IcebergActions.java # amoro-common/src/test/java/org/apache/amoro/process/TestLocalExecutionEngine.java # charts/amoro/templates/amoro-configmap.yaml # dist/src/main/amoro-bin/conf/config.yaml # dist/src/main/amoro-bin/conf/plugins/execute-engines.yaml # dist/src/main/amoro-bin/conf/plugins/process-factories.yaml # docs/admin-guides/deployment.md .github/workflows/core-hadoop3-ci.yml | 7 +- README.md | 2 +- amoro-ams/pom.xml | 36 ++ .../apache/amoro/server/AmoroManagementConf.java | 48 --- .../amoro/server/AmoroManagementConfValidator.java | 8 - .../apache/amoro/server/AmoroServiceContainer.java | 15 +- .../amoro/server/DefaultOptimizingService.java | 28 +- .../amoro/server/catalog/CatalogBuilder.java | 12 +- .../amoro/server/dashboard/DashboardServer.java | 4 +- .../dashboard/MixedAndIcebergTableDescriptor.java | 88 ++++- .../server/dashboard/ServerTableDescriptor.java | 14 +- .../dashboard/controller/CatalogController.java | 20 +- .../controller/OptimizerGroupController.java | 18 +- .../dashboard/controller/TableController.java | 25 +- .../server/manager/AbstractOptimizerContainer.java | 5 + .../server/manager/FlinkOptimizerContainer.java | 6 +- .../manager/KubernetesOptimizerContainer.java | 46 ++- .../amoro/server/optimizing/TaskRuntime.java | 98 +++++- .../optimizing/dra/DynamicAllocationConfig.java | 362 ++++++++++++++++++++ .../persistence/mapper/TableProcessMapper.java | 2 + .../server/process/DefaultTableProcessStore.java | 50 +-- .../amoro/server/process/TableProcessMeta.java | 4 +- .../DanglingDeleteFilesCleaningProcess.java | 6 +- .../process/iceberg/DataExpiringProcess.java | 6 +- .../iceberg/HiveCommitSyncProcess.java} | 66 ++-- .../process/iceberg/IcebergProcessFactory.java | 57 +++- .../iceberg/OrphanFilesCleaningProcess.java | 6 +- .../process/iceberg/SnapshotsExpiringProcess.java | 10 +- ...ngProcess.java => TagsAutoCreatingProcess.java} | 18 +- .../server/scheduler/PeriodicTableScheduler.java | 121 +------ .../scheduler/inline/InlineTableExecutors.java | 22 -- .../inline/SnapshotsExpiringExecutor.java | 86 ----- .../scheduler/inline/TagsAutoCreatingExecutor.java | 76 ----- .../amoro/server/table/DefaultTableRuntime.java | 27 -- .../amoro/server/utils/IcebergTableUtil.java | 3 +- .../amoro/server/utils/SnowflakeIdGenerator.java | 3 + .../apache/amoro/server/AMSServiceTestBase.java | 3 + .../org/apache/amoro/server/AmsEnvironment.java | 1 - .../amoro/server/TestAmoroManagementConf.java | 5 - .../server/TestAmoroManagementConfValidator.java | 16 - .../amoro/server/TestDefaultOptimizingService.java | 47 ++- .../amoro/server/TestOptimizerGroupController.java | 44 +++ .../amoro/server/TestOptimizerGroupKeeper.java | 61 ++++ .../TestIcebergServerTableDescriptor.java | 117 ++++++- .../manager/TestFlinkOptimizerContainer.java | 23 ++ .../manager/TestKubernetesOptimizerContainer.java | 277 +++++++++++++++- .../server/optimizing/BaseOptimizingChecker.java | 6 +- .../server/optimizing/TestOptimizingQueue.java | 115 +++++++ .../dra/TestDynamicAllocationConfig.java | 245 ++++++++++++++ .../process/iceberg/TestIcebergProcessFactory.java | 29 ++ .../inline/PeriodicTableSchedulerTestBase.java | 82 ----- .../inline/TestConfigurableIntervalExecutors.java | 34 +- .../inline/TestPeriodicTableSchedulerCleanup.java | 250 -------------- .../inline/TestProcessDataExpiringExecutor.java | 3 +- .../src/test/resources/config-with-units.yaml | 22 -- .../src/test/resources/config-without-units.yaml | 3 - .../org/apache/amoro/CommonUnifiedCatalog.java | 6 +- .../main/java/org/apache/amoro/IcebergActions.java | 3 +- .../java/org/apache/amoro/OptimizerProperties.java | 60 +++- .../main/java/org/apache/amoro/TableFormat.java | 1 + .../apache/amoro/maintainer/TableMaintainer.java | 14 +- .../apache/amoro/process/StagedTaskDescriptor.java | 4 + .../table/descriptor/FormatTableDescriptor.java | 21 +- .../amoro/table/descriptor/ProcessCategory.java | 55 +++ .../apache/amoro/MockAmoroManagementServer.java | 15 + .../amoro/process/TestLocalExecutionEngine.java | 4 + .../org/apache/amoro/utils/JacksonUtilTest.java | 6 +- amoro-format-hudi/pom.xml | 2 - .../amoro/formats/hudi/HudiTableDescriptor.java | 25 +- .../iceberg/maintainer/IcebergTableMaintainer.java | 101 ++++-- .../iceberg/maintainer/MixedTableMaintainer.java | 74 ++++- .../formats/iceberg/utils/RollingFileCleaner.java | 55 +-- .../optimizing/AbstractRewriteFilesExecutor.java | 2 +- .../amoro/optimizing/IcebergRewriteExecutor.java | 358 ++++++++++++++++++++ .../amoro/optimizing/ParquetFileMergeRunner.java | 198 +++++++++++ .../apache/amoro/optimizing/TaskProperties.java | 7 + .../org/apache/amoro/table/TableProperties.java | 12 + .../org/apache/amoro/utils/IcebergThreadPools.java | 153 ++++++--- .../java/org/apache/amoro/utils/TableFileUtil.java | 6 + .../apache/iceberg/parquet/ParquetIOBridge.java | 27 +- .../optimizing/IcebergRewriteExecutorTest.java | 369 +++++++++++++++++++++ .../apache/amoro/utils/TestIcebergThreadPools.java | 65 ++++ .../pom.xml | 63 ++-- .../amoro/formats/lance/LanceCatalogFactory.java | 58 ++++ .../formats/lance/LanceDirectoryV1Catalog.java | 215 ++++++++++++ .../org/apache/amoro/formats/lance/LanceTable.java | 71 ++++ .../amoro/formats/lance/LanceTableDescriptor.java | 329 ++++++++++++++++++ .../services/org.apache.amoro.FormatCatalogFactory | 19 ++ ...he.amoro.table.descriptor.FormatTableDescriptor | 18 + .../amoro-mixed-flink-common/pom.xml | 10 +- .../interceptor/KerberosInvocationHandler.java | 1 - .../hybrid/reader/MixedFormatSourceReader.java | 38 ++- .../org/apache/amoro/flink/table/FlinkSource.java | 35 +- .../flink/table/KerberosAwareInputFormat.java | 165 +++++++++ .../amoro/flink/table/MixedFormatTableLoader.java | 247 +++++++++++++- .../table/UnkeyedInputFormatOperatorFactory.java | 8 +- .../table/UnkeyedStreamingReaderOperator.java | 175 ++++++++++ .../amoro/flink/util/FlinkClassReflectionUtil.java | 65 ---- .../apache/amoro/flink/util/IcebergClassUtil.java | 119 +++---- .../org/apache/amoro/flink/write/FlinkSink.java | 8 +- .../apache/amoro/flink/table/TestWatermark.java | 66 +--- .../amoro/flink/util/TestIcebergClassUtil.java | 75 +++++ .../amoro/flink/write/TestAutomaticLogWriter.java | 14 +- .../flink/write/TestMixedFormatFileWriter.java | 11 +- amoro-format-mixed/amoro-mixed-trino/pom.xml | 1 - amoro-format-paimon/pom.xml | 9 + .../formats/paimon/PaimonTableDescriptor.java | 34 +- .../common/AbstractOptimizerOperator.java | 44 ++- .../apache/amoro/optimizer/common/Optimizer.java | 88 ++++- .../amoro/optimizer/common/OptimizerConfig.java | 18 + .../amoro/optimizer/common/OptimizerExecutor.java | 39 ++- .../amoro/optimizer/common/OptimizerToucher.java | 55 ++- .../amoro/optimizer/common/OptimizerTestBase.java | 1 + .../common/TestAbstractOptimizerOperator.java | 57 ++++ .../amoro/optimizer/common/TestOptimizer.java | 139 ++++++++ .../optimizer/common/TestOptimizerExecutor.java | 135 ++++++++ .../optimizer/common/TestOptimizerToucher.java | 32 ++ .../amoro/optimizer/flink/FlinkExecutor.java | 94 +++++- .../amoro/optimizer/flink/FlinkOptimizer.java | 3 +- .../optimizer/flink/FlinkOptimizerExecutor.java | 21 ++ .../amoro/optimizer/flink/TestFlinkExecutor.java | 178 ++++++++++ .../flink/TestFlinkOptimizerExecutor.java | 66 ++++ .../amoro/optimizer/spark/SparkOptimizer.java | 26 ++ .../optimizer/standalone/StandaloneOptimizer.java | 24 ++ amoro-web/mock/modules/table.js | 33 +- amoro-web/src/assets/icons/svg/lance.svg | 25 ++ amoro-web/src/language/en.ts | 5 + amoro-web/src/language/zh.ts | 5 + amoro-web/src/services/table.service.ts | 14 +- amoro-web/src/types/common.type.ts | 1 + amoro-web/src/views/tables/components/Cleanup.vue | 26 ++ .../src/views/tables/components/Optimizing.vue | 51 ++- .../src/views/tables/components/Profiling.vue | 26 ++ amoro-web/src/views/tables/index.vue | 6 + charts/amoro/templates/amoro-configmap.yaml | 13 - charts/amoro/templates/amoro-httproute.yaml | 54 +++ charts/amoro/tests/amoro-httproute_test.yaml | 192 +++++++++++ charts/amoro/values.yaml | 48 +++ dev/deps/dependencies-hadoop-2-spark-3.3 | 13 +- dev/deps/dependencies-hadoop-3-spark-3.5 | 13 +- dist/src/main/amoro-bin/bin/optimizer.sh | 2 +- dist/src/main/amoro-bin/conf/config.yaml | 13 - .../amoro-bin/conf/plugins/execute-engines.yaml | 4 +- .../amoro-bin/conf/plugins/process-factories.yaml | 4 + docs/admin-guides/deployment.md | 10 +- docs/admin-guides/managing-optimizers.md | 4 + docs/configuration/ams-config.md | 8 - docs/user-guides/configurations.md | 51 +-- pom.xml | 81 ++++- 149 files changed, 6515 insertions(+), 1492 deletions(-)
