This is an automated email from the ASF dual-hosted git repository.
xxubai pushed a change to branch 0.9.x
in repository https://gitbox.apache.org/repos/asf/amoro.git
from 7a65790f4 [0.9.x][Hotfix] Force ReachableFileCleanup when snapshots
exist outside main ancestry (backport #4231) (#4241)
add 75dd6570c [AMORO-4208] Refactor orphan-files-cleaning via
ProcessFactory plugin (#4209)
add 0c5d3d84c [AMORO-4216] Refactor dangling-delete-files-cleaning via
ProcessFactory plugin (#4214)
add 22092632f [Hotfix] Update release validation instructions to use
./mvnw for building (#4222)
add cba754f59 [AMORO-4220] [Improvement] Support BETWEEN and reversed
comparison SQL filters in ExpressionUtil. (#4221)
add 1d4bc38be [AMORO-4223][ams] Fix AMS startup crash when recovering
Iceberg maintenance processes (#4224)
add 38ef38134 [AMORO-4215] Refactor data expiring via ProcessFactory
plugin (#4218)
add 467930937 [hotfix] Unify action names, pool names, and constants for
SnapshotsExpiringProcess and OrphanFilesCleaningProcess (#4225)
add e75b4a0b3 [AMORO-4186] Add support for Parquet row-group merging in
IcebergRewriteExecutor (#4188)
add 23b0ab98e [AMORO-4217] Clean up invalid code and parameters left over
from the snapshot-expiring refactoring (#4226)
add 99fcc0889 [AMORO-4227] Refactor auto-creating iceberg tags via
ProcessFactory plugin (#4228)
add be6663bcf [AMORO-4229] Refactor synchronizing Hive tables process via
ProcessFactory plugin (#4230)
add d8f379346 [Hotfix] Force ReachableFileCleanup when snapshots exist
outside main ancestry (#4231)
add 4ed2e91e3 [AMORO-4236] Splitting the “Optimizing” tab into the
“Optimizing” tab,“Cleanup” tab and “Profiling” tab (#4238)
add c812511fb [AMORO-4242][Helm] Add Gateway API HTTPRoute support to the
Amoro chart (#4243)
add acf8ef47a [AMORO-4246] Refactor process data expiring via
ProcessFactory plugin (#4247)
add 374ed06bd Revert "[AMORO-4246] Refactor process data expiring via
ProcessFactory plugin" (#4252)
add b3bf85ab5 [hotfix] Cleanup empty columns from Cleanup and Profiling
pages (#4260)
add 675c222c3 [AMORO-4257][AMS] Fix duplicate process ID caused by
multiple SnowflakeIdGenerator instances (#4258)
add bc83f0834 Fix expireSnapshots fails with IllegalArgumentException on
object-store-backed tables (#4259)
add 9126f7318 [AMORO-3853] Support Java 17 build for Amoro (#4244)
add 2b3027d9c [AMORO-4211] Fix legacy Flink config loading (#4249)
add d1dca2749 [AMORO-3837] Integrate lance format (#4051)
add 89d27bcd6 [AMORO-4198][optimizer] Support graceful shutdown for
in-progress tasks (#4197)
add 498ea2716 [AMORO-4263] Persist cleanup operation metrics for Dashboard
display (#4265)
add 8b03fec11 [AMORO-4253] AIP-5 Phase 1: dynamic allocation configuration
and validation (#4254)
add 27ae8d55f [AMORO-4235] Recognize stale optimizer responses on the AMS
side (#4261)
add 6318c4084 [AMORO-4269] Refactor Iceberg thread pool initialization and
scope (#4270)
add 92187c525 Change project version to 0.9.0-incubating
add 27f19d58d [hotfix] Change project version to 0.9.0-incubating
add 7aac7fa42 [hotfix] fix helm chart to 0.9.0-incubating and release guide
add a93a32b4d [hotfix] fix lance maven pom to 0.9.0-incubating
new 30580a3ae Merge remote-tracking branch 'origin/0.9.0-rc4' into 0.9.x
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/core-hadoop3-ci.yml | 7 +-
README.md | 2 +-
amoro-ams/pom.xml | 36 ++
.../apache/amoro/server/AmoroManagementConf.java | 48 ---
.../amoro/server/AmoroManagementConfValidator.java | 8 -
.../apache/amoro/server/AmoroServiceContainer.java | 15 +-
.../amoro/server/DefaultOptimizingService.java | 28 +-
.../amoro/server/catalog/CatalogBuilder.java | 12 +-
.../amoro/server/dashboard/DashboardServer.java | 4 +-
.../dashboard/MixedAndIcebergTableDescriptor.java | 88 ++++-
.../server/dashboard/ServerTableDescriptor.java | 14 +-
.../dashboard/controller/CatalogController.java | 20 +-
.../controller/OptimizerGroupController.java | 18 +-
.../dashboard/controller/TableController.java | 25 +-
.../server/manager/AbstractOptimizerContainer.java | 5 +
.../server/manager/FlinkOptimizerContainer.java | 6 +-
.../manager/KubernetesOptimizerContainer.java | 46 ++-
.../amoro/server/optimizing/TaskRuntime.java | 98 +++++-
.../optimizing/dra/DynamicAllocationConfig.java | 362 ++++++++++++++++++++
.../persistence/mapper/TableProcessMapper.java | 2 +
.../server/process/DefaultTableProcessStore.java | 50 +--
.../amoro/server/process/TableProcessMeta.java | 4 +-
.../DanglingDeleteFilesCleaningProcess.java | 6 +-
.../process/iceberg/DataExpiringProcess.java | 6 +-
.../iceberg/HiveCommitSyncProcess.java} | 66 ++--
.../process/iceberg/IcebergProcessFactory.java | 57 +++-
.../iceberg/OrphanFilesCleaningProcess.java | 6 +-
.../process/iceberg/SnapshotsExpiringProcess.java | 10 +-
...ngProcess.java => TagsAutoCreatingProcess.java} | 18 +-
.../server/scheduler/PeriodicTableScheduler.java | 121 +------
.../scheduler/inline/InlineTableExecutors.java | 22 --
.../inline/SnapshotsExpiringExecutor.java | 86 -----
.../scheduler/inline/TagsAutoCreatingExecutor.java | 76 -----
.../amoro/server/table/DefaultTableRuntime.java | 27 --
.../server/table/cleanup/CleanupOperation.java | 27 --
.../amoro/server/utils/IcebergTableUtil.java | 3 +-
.../amoro/server/utils/SnowflakeIdGenerator.java | 3 +
.../apache/amoro/server/AMSServiceTestBase.java | 3 +
.../org/apache/amoro/server/AmsEnvironment.java | 1 -
.../amoro/server/TestAmoroManagementConf.java | 5 -
.../server/TestAmoroManagementConfValidator.java | 16 -
.../amoro/server/TestDefaultOptimizingService.java | 47 ++-
.../amoro/server/TestOptimizerGroupController.java | 44 +++
.../amoro/server/TestOptimizerGroupKeeper.java | 61 ++++
.../TestIcebergServerTableDescriptor.java | 117 ++++++-
.../manager/TestFlinkOptimizerContainer.java | 23 ++
.../manager/TestKubernetesOptimizerContainer.java | 277 +++++++++++++++-
.../server/optimizing/BaseOptimizingChecker.java | 6 +-
.../server/optimizing/TestOptimizingQueue.java | 115 +++++++
.../dra/TestDynamicAllocationConfig.java | 245 ++++++++++++++
.../process/iceberg/TestIcebergProcessFactory.java | 29 ++
.../inline/PeriodicTableSchedulerTestBase.java | 82 -----
.../inline/TestConfigurableIntervalExecutors.java | 34 +-
.../inline/TestPeriodicTableSchedulerCleanup.java | 250 --------------
.../inline/TestProcessDataExpiringExecutor.java | 3 +-
.../src/test/resources/config-with-units.yaml | 22 --
.../src/test/resources/config-without-units.yaml | 3 -
.../org/apache/amoro/CommonUnifiedCatalog.java | 6 +-
.../main/java/org/apache/amoro/IcebergActions.java | 3 +-
.../java/org/apache/amoro/OptimizerProperties.java | 60 +++-
.../main/java/org/apache/amoro/TableFormat.java | 1 +
.../apache/amoro/maintainer/TableMaintainer.java | 14 +-
.../apache/amoro/process/StagedTaskDescriptor.java | 4 +
.../table/descriptor/FormatTableDescriptor.java | 21 +-
.../descriptor/ProcessCategory.java} | 49 ++-
.../apache/amoro/MockAmoroManagementServer.java | 15 +
.../amoro/process/TestLocalExecutionEngine.java | 4 +
.../org/apache/amoro/utils/JacksonUtilTest.java | 6 +-
amoro-format-hudi/pom.xml | 2 -
.../amoro/formats/hudi/HudiTableDescriptor.java | 25 +-
.../iceberg/maintainer/IcebergTableMaintainer.java | 101 ++++--
.../iceberg/maintainer/MixedTableMaintainer.java | 74 ++++-
.../formats/iceberg/utils/RollingFileCleaner.java | 55 +--
.../optimizing/AbstractRewriteFilesExecutor.java | 2 +-
.../amoro/optimizing/IcebergRewriteExecutor.java | 358 ++++++++++++++++++++
.../amoro/optimizing/ParquetFileMergeRunner.java | 198 +++++++++++
.../apache/amoro/optimizing/TaskProperties.java | 7 +
.../org/apache/amoro/table/TableProperties.java | 12 +
.../org/apache/amoro/utils/IcebergThreadPools.java | 153 ++++++---
.../java/org/apache/amoro/utils/TableFileUtil.java | 6 +
.../apache/iceberg/parquet/ParquetIOBridge.java | 22 +-
.../optimizing/IcebergRewriteExecutorTest.java | 369 +++++++++++++++++++++
.../apache/amoro/utils/TestIcebergThreadPools.java | 65 ++++
.../pom.xml | 63 ++--
.../amoro/formats/lance/LanceCatalogFactory.java | 29 +-
.../formats/lance/LanceDirectoryV1Catalog.java | 215 ++++++++++++
.../org/apache/amoro/formats/lance/LanceTable.java | 44 +--
.../amoro/formats/lance/LanceTableDescriptor.java | 329 ++++++++++++++++++
.../services/org.apache.amoro.FormatCatalogFactory | 2 +-
...he.amoro.table.descriptor.FormatTableDescriptor | 2 +-
.../amoro-mixed-flink-common/pom.xml | 10 +-
.../interceptor/KerberosInvocationHandler.java | 1 -
.../hybrid/reader/MixedFormatSourceReader.java | 38 ++-
.../org/apache/amoro/flink/table/FlinkSource.java | 35 +-
.../flink/table/KerberosAwareInputFormat.java | 165 +++++++++
.../amoro/flink/table/MixedFormatTableLoader.java | 247 +++++++++++++-
.../table/UnkeyedInputFormatOperatorFactory.java | 8 +-
.../table/UnkeyedStreamingReaderOperator.java | 175 ++++++++++
.../amoro/flink/util/FlinkClassReflectionUtil.java | 65 ----
.../apache/amoro/flink/util/IcebergClassUtil.java | 119 +++----
.../org/apache/amoro/flink/write/FlinkSink.java | 8 +-
.../apache/amoro/flink/table/TestWatermark.java | 66 +---
.../amoro/flink/util/TestIcebergClassUtil.java | 75 +++++
.../amoro/flink/write/TestAutomaticLogWriter.java | 14 +-
.../flink/write/TestMixedFormatFileWriter.java | 11 +-
amoro-format-mixed/amoro-mixed-trino/pom.xml | 1 -
amoro-format-paimon/pom.xml | 9 +
.../formats/paimon/PaimonTableDescriptor.java | 34 +-
.../common/AbstractOptimizerOperator.java | 44 ++-
.../apache/amoro/optimizer/common/Optimizer.java | 88 ++++-
.../amoro/optimizer/common/OptimizerConfig.java | 18 +
.../amoro/optimizer/common/OptimizerExecutor.java | 39 ++-
.../amoro/optimizer/common/OptimizerToucher.java | 55 ++-
.../amoro/optimizer/common/OptimizerTestBase.java | 1 +
.../common/TestAbstractOptimizerOperator.java | 57 ++++
.../amoro/optimizer/common/TestOptimizer.java | 139 ++++++++
.../optimizer/common/TestOptimizerExecutor.java | 135 ++++++++
.../optimizer/common/TestOptimizerToucher.java | 32 ++
.../amoro/optimizer/flink/FlinkExecutor.java | 94 +++++-
.../amoro/optimizer/flink/FlinkOptimizer.java | 3 +-
.../optimizer/flink/FlinkOptimizerExecutor.java | 21 ++
.../amoro/optimizer/flink/TestFlinkExecutor.java | 178 ++++++++++
.../flink/TestFlinkOptimizerExecutor.java | 66 ++++
.../amoro/optimizer/spark/SparkOptimizer.java | 26 ++
.../optimizer/standalone/StandaloneOptimizer.java | 24 ++
amoro-web/mock/modules/table.js | 33 +-
.../{views/Home.vue => assets/icons/svg/lance.svg} | 16 +-
amoro-web/src/language/en.ts | 5 +
amoro-web/src/language/zh.ts | 5 +
amoro-web/src/services/table.service.ts | 14 +-
amoro-web/src/types/common.type.ts | 1 +
.../{Home.vue => tables/components/Cleanup.vue} | 15 +-
.../src/views/tables/components/Optimizing.vue | 51 ++-
.../{Home.vue => tables/components/Profiling.vue} | 15 +-
amoro-web/src/views/tables/index.vue | 6 +
charts/amoro/templates/amoro-configmap.yaml | 13 -
charts/amoro/templates/amoro-httproute.yaml | 54 +++
charts/amoro/tests/amoro-httproute_test.yaml | 192 +++++++++++
charts/amoro/values.yaml | 48 +++
dev/deps/dependencies-hadoop-2-spark-3.3 | 13 +-
dev/deps/dependencies-hadoop-3-spark-3.5 | 13 +-
dist/src/main/amoro-bin/bin/optimizer.sh | 2 +-
dist/src/main/amoro-bin/conf/config.yaml | 13 -
.../amoro-bin/conf/plugins/execute-engines.yaml | 4 +-
.../amoro-bin/conf/plugins/process-factories.yaml | 4 +
docs/admin-guides/deployment.md | 10 +-
docs/admin-guides/managing-optimizers.md | 4 +
docs/configuration/ams-config.md | 8 -
docs/user-guides/configurations.md | 51 +--
pom.xml | 81 ++++-
150 files changed, 6297 insertions(+), 1606 deletions(-)
create mode 100644
amoro-ams/src/main/java/org/apache/amoro/server/optimizing/dra/DynamicAllocationConfig.java
rename
amoro-ams/src/main/java/org/apache/amoro/server/{scheduler/inline/HiveCommitSyncExecutor.java
=> process/iceberg/HiveCommitSyncProcess.java} (60%)
copy
amoro-ams/src/main/java/org/apache/amoro/server/process/iceberg/{DataExpiringProcess.java
=> TagsAutoCreatingProcess.java} (74%)
delete mode 100644
amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/SnapshotsExpiringExecutor.java
delete mode 100644
amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/TagsAutoCreatingExecutor.java
delete mode 100644
amoro-ams/src/main/java/org/apache/amoro/server/table/cleanup/CleanupOperation.java
create mode 100644
amoro-ams/src/test/java/org/apache/amoro/server/optimizing/dra/TestDynamicAllocationConfig.java
delete mode 100644
amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/PeriodicTableSchedulerTestBase.java
delete mode 100644
amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/TestPeriodicTableSchedulerCleanup.java
copy
amoro-common/src/main/java/org/apache/amoro/{authentication/BasicPrincipal.java
=> table/descriptor/ProcessCategory.java} (52%)
create mode 100644
amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/ParquetFileMergeRunner.java
copy
amoro-format-mixed/amoro-mixed-hive/src/main/java/org/apache/iceberg/parquet/AdaptHiveApplyNameMapping.java
=>
amoro-format-iceberg/src/main/java/org/apache/iceberg/parquet/ParquetIOBridge.java
(53%)
create mode 100644
amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestIcebergThreadPools.java
copy {amoro-format-paimon => amoro-format-lance}/pom.xml (58%)
mode change 100644 => 100755
copy
amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiCatalogFactory.java
=>
amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceCatalogFactory.java
(64%)
mode change 100644 => 100755
create mode 100755
amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceDirectoryV1Catalog.java
copy
amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTable.java
=>
amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceTable.java
(58%)
mode change 100644 => 100755
create mode 100755
amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceTableDescriptor.java
copy {amoro-format-hudi =>
amoro-format-lance}/src/main/resources/META-INF/services/org.apache.amoro.FormatCatalogFactory
(94%)
copy {amoro-format-hudi =>
amoro-format-lance}/src/main/resources/META-INF/services/org.apache.amoro.table.descriptor.FormatTableDescriptor
(93%)
create mode 100644
amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/main/java/org/apache/amoro/flink/table/KerberosAwareInputFormat.java
create mode 100644
amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/main/java/org/apache/amoro/flink/table/UnkeyedStreamingReaderOperator.java
delete mode 100644
amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/main/java/org/apache/amoro/flink/util/FlinkClassReflectionUtil.java
create mode 100644
amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/util/TestIcebergClassUtil.java
create mode 100644
amoro-optimizer/amoro-optimizer-common/src/test/java/org/apache/amoro/optimizer/common/TestAbstractOptimizerOperator.java
create mode 100644
amoro-optimizer/amoro-optimizer-flink/src/test/java/org/apache/amoro/optimizer/flink/TestFlinkExecutor.java
create mode 100644
amoro-optimizer/amoro-optimizer-flink/src/test/java/org/apache/amoro/optimizer/flink/TestFlinkOptimizerExecutor.java
copy amoro-web/src/{views/Home.vue => assets/icons/svg/lance.svg} (73%)
mode change 100644 => 100755
copy amoro-web/src/views/{Home.vue => tables/components/Cleanup.vue} (53%)
copy amoro-web/src/views/{Home.vue => tables/components/Profiling.vue} (53%)
create mode 100644 charts/amoro/templates/amoro-httproute.yaml
create mode 100644 charts/amoro/tests/amoro-httproute_test.yaml
