This is an automated email from the ASF dual-hosted git repository.
jinsongzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new d4ea2fa18 [AMORO-3192] Add the
"clean-orphan-file.interval-time-minutes" parameter (#3193)
d4ea2fa18 is described below
commit d4ea2fa18bf3472d1b279c95fad2ac1dfe5329b3
Author: Darcy <[email protected]>
AuthorDate: Wed Sep 11 19:53:05 2024 +0800
[AMORO-3192] Add the "clean-orphan-file.interval-time-minutes" parameter
(#3193)
* Add the "clean-orphan-file.interval-time-minutes" parameter
* Rename the parameter to "clean-orphan-files.interval" and use the new
parameter type.
* rollback configurations doc and add example to config.yaml
---
.../main/java/org/apache/amoro/server/AmoroManagementConf.java | 6 ++++++
.../amoro/server/table/executor/AsyncTableExecutors.java | 4 +++-
.../server/table/executor/OrphanFilesCleaningExecutor.java | 10 ++++++----
.../src/main/java/org/apache/amoro/table/TableProperties.java | 1 +
dist/src/main/amoro-bin/conf/config.yaml | 1 +
5 files changed, 17 insertions(+), 5 deletions(-)
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java
b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java
index 871ab5bc9..415995d81 100644
--- a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java
+++ b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java
@@ -102,6 +102,12 @@ public class AmoroManagementConf {
.defaultValue(10)
.withDescription("The number of threads used for orphan files
cleaning.");
+ public static final ConfigOption<Duration> CLEAN_ORPHAN_FILES_INTERVAL =
+ ConfigOptions.key("clean-orphan-files.interval")
+ .durationType()
+ .defaultValue(Duration.ofDays(1))
+ .withDescription("Interval for cleaning orphan files.");
+
public static final ConfigOption<Boolean>
CLEAN_DANGLING_DELETE_FILES_ENABLED =
ConfigOptions.key("clean-dangling-delete-files.enabled")
.booleanType()
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/AsyncTableExecutors.java
b/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/AsyncTableExecutors.java
index 09f7a4d3c..a45385026 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/AsyncTableExecutors.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/AsyncTableExecutors.java
@@ -49,7 +49,9 @@ public class AsyncTableExecutors {
if (conf.getBoolean(AmoroManagementConf.CLEAN_ORPHAN_FILES_ENABLED)) {
this.orphanFilesCleaningExecutor =
new OrphanFilesCleaningExecutor(
- tableManager,
conf.getInteger(AmoroManagementConf.CLEAN_ORPHAN_FILES_THREAD_COUNT));
+ tableManager,
+
conf.getInteger(AmoroManagementConf.CLEAN_ORPHAN_FILES_THREAD_COUNT),
+ conf.get(AmoroManagementConf.CLEAN_ORPHAN_FILES_INTERVAL));
}
if
(conf.getBoolean(AmoroManagementConf.CLEAN_DANGLING_DELETE_FILES_ENABLED)) {
this.danglingDeleteFilesCleaningExecutor =
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/OrphanFilesCleaningExecutor.java
b/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/OrphanFilesCleaningExecutor.java
index 9d40090ad..6d6cf01dc 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/OrphanFilesCleaningExecutor.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/OrphanFilesCleaningExecutor.java
@@ -26,18 +26,20 @@ import org.apache.amoro.server.table.TableRuntime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.time.Duration;
+
public class OrphanFilesCleaningExecutor extends BaseTableExecutor {
private static final Logger LOG =
LoggerFactory.getLogger(OrphanFilesCleaningExecutor.class);
+ private final Duration interval;
- private static final long INTERVAL = 24 * 60 * 60 * 1000L;
-
- public OrphanFilesCleaningExecutor(TableManager tableRuntimes, int poolSize)
{
+ public OrphanFilesCleaningExecutor(TableManager tableRuntimes, int poolSize,
Duration interval) {
super(tableRuntimes, poolSize);
+ this.interval = interval;
}
@Override
protected long getNextExecutingTime(TableRuntime tableRuntime) {
- return INTERVAL;
+ return interval.toMillis();
}
@Override
diff --git
a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java
b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java
index debe3f0ca..9f2aeca5d 100644
---
a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java
+++
b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java
@@ -173,6 +173,7 @@ public class TableProperties {
public static final String MIN_ORPHAN_FILE_EXISTING_TIME =
"clean-orphan-file.min-existing-time-minutes";
public static final long MIN_ORPHAN_FILE_EXISTING_TIME_DEFAULT = 2880; // 2
Days
+
public static final String ENABLE_DANGLING_DELETE_FILES_CLEAN =
"clean-dangling-delete-files.enabled";
public static final boolean ENABLE_DANGLING_DELETE_FILES_CLEAN_DEFAULT =
true;
diff --git a/dist/src/main/amoro-bin/conf/config.yaml
b/dist/src/main/amoro-bin/conf/config.yaml
index d16027c34..ae1de24e4 100644
--- a/dist/src/main/amoro-bin/conf/config.yaml
+++ b/dist/src/main/amoro-bin/conf/config.yaml
@@ -66,6 +66,7 @@ ams:
clean-orphan-files:
enabled: true
thread-count: 10
+ interval: 1d
clean-dangling-delete-files:
enabled: true
