This is an automated email from the ASF dual-hosted git repository.
xuba pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new e3671b528 [AMORO-4080] Make periodic executor intervals configurable
(#4079)
e3671b528 is described below
commit e3671b52890eed298901eef44a606e6a93e96e7b
Author: Jiwon Park <[email protected]>
AuthorDate: Wed Feb 11 14:43:59 2026 +0900
[AMORO-4080] Make periodic executor intervals configurable (#4079)
Make hardcoded intervals configurable for
DanglingDeleteFilesCleaningExecutor and SnapshotsExpiringExecutor
Add ConfigOption<Duration> for expire-snapshots.interval (default 1h) and
clean-dangling-delete-files.interval (default 1d) to allow runtime
configuration,
consistent with other periodic executors like OrphanFilesCleaningExecutor.
Signed-off-by: Jiwon Park <[email protected]>
---
.../apache/amoro/server/AmoroManagementConf.java | 12 +++
.../DanglingDeleteFilesCleaningExecutor.java | 13 +--
.../scheduler/inline/InlineTableExecutors.java | 7 +-
.../inline/SnapshotsExpiringExecutor.java | 12 +--
.../inline/TestConfigurableIntervalExecutors.java | 99 ++++++++++++++++++++++
docs/configuration/ams-config.md | 2 +
6 files changed, 133 insertions(+), 12 deletions(-)
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java
b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java
index 7877ee76a..96e54b7df 100644
--- a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java
+++ b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java
@@ -123,6 +123,12 @@ public class AmoroManagementConf {
.defaultValue(10)
.withDescription("The number of threads used for snapshots
expiring.");
+ public static final ConfigOption<Duration> EXPIRE_SNAPSHOTS_INTERVAL =
+ ConfigOptions.key("expire-snapshots.interval")
+ .durationType()
+ .defaultValue(Duration.ofHours(1))
+ .withDescription("Interval for expiring snapshots.");
+
public static final ConfigOption<Boolean> CLEAN_ORPHAN_FILES_ENABLED =
ConfigOptions.key("clean-orphan-files.enabled")
.booleanType()
@@ -153,6 +159,12 @@ public class AmoroManagementConf {
.defaultValue(10)
.withDescription("The number of threads used for dangling delete
files cleaning.");
+ public static final ConfigOption<Duration>
CLEAN_DANGLING_DELETE_FILES_INTERVAL =
+ ConfigOptions.key("clean-dangling-delete-files.interval")
+ .durationType()
+ .defaultValue(Duration.ofDays(1))
+ .withDescription("Interval for cleaning dangling delete files.");
+
public static final ConfigOption<Boolean> SYNC_HIVE_TABLES_ENABLED =
ConfigOptions.key("sync-hive-tables.enabled")
.booleanType()
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/DanglingDeleteFilesCleaningExecutor.java
b/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/DanglingDeleteFilesCleaningExecutor.java
index 337481439..3ad669092 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/DanglingDeleteFilesCleaningExecutor.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/DanglingDeleteFilesCleaningExecutor.java
@@ -30,6 +30,7 @@ import org.apache.amoro.server.table.cleanup.CleanupOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.time.Duration;
import java.util.concurrent.ThreadLocalRandom;
/** Clean table dangling delete files */
@@ -38,20 +39,22 @@ public class DanglingDeleteFilesCleaningExecutor extends
PeriodicTableScheduler
private static final Logger LOG =
LoggerFactory.getLogger(DanglingDeleteFilesCleaningExecutor.class);
- private static final long INTERVAL = 24 * 60 * 60 * 1000L;
+ private final Duration interval;
- protected DanglingDeleteFilesCleaningExecutor(TableService tableService, int
poolSize) {
+ protected DanglingDeleteFilesCleaningExecutor(
+ TableService tableService, int poolSize, Duration interval) {
super(tableService, poolSize);
+ this.interval = interval;
}
@Override
protected long getNextExecutingTime(TableRuntime tableRuntime) {
- return INTERVAL;
+ return interval.toMillis();
}
@Override
protected boolean shouldExecute(Long lastCleanupEndTime) {
- return System.currentTimeMillis() - lastCleanupEndTime >= INTERVAL;
+ return System.currentTimeMillis() - lastCleanupEndTime >=
interval.toMillis();
}
@Override
@@ -72,7 +75,7 @@ public class DanglingDeleteFilesCleaningExecutor extends
PeriodicTableScheduler
@Override
protected long getExecutorDelay() {
- return ThreadLocalRandom.current().nextLong(INTERVAL);
+ return ThreadLocalRandom.current().nextLong(interval.toMillis());
}
@Override
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/InlineTableExecutors.java
b/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/InlineTableExecutors.java
index d51b02713..17205bfbf 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/InlineTableExecutors.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/InlineTableExecutors.java
@@ -44,7 +44,9 @@ public class InlineTableExecutors {
if (conf.getBoolean(AmoroManagementConf.EXPIRE_SNAPSHOTS_ENABLED)) {
this.snapshotsExpiringExecutor =
new SnapshotsExpiringExecutor(
- tableService,
conf.getInteger(AmoroManagementConf.EXPIRE_SNAPSHOTS_THREAD_COUNT));
+ tableService,
+
conf.getInteger(AmoroManagementConf.EXPIRE_SNAPSHOTS_THREAD_COUNT),
+ conf.get(AmoroManagementConf.EXPIRE_SNAPSHOTS_INTERVAL));
}
if (conf.getBoolean(AmoroManagementConf.CLEAN_ORPHAN_FILES_ENABLED)) {
this.orphanFilesCleaningExecutor =
@@ -57,7 +59,8 @@ public class InlineTableExecutors {
this.danglingDeleteFilesCleaningExecutor =
new DanglingDeleteFilesCleaningExecutor(
tableService,
-
conf.getInteger(AmoroManagementConf.CLEAN_DANGLING_DELETE_FILES_THREAD_COUNT));
+
conf.getInteger(AmoroManagementConf.CLEAN_DANGLING_DELETE_FILES_THREAD_COUNT),
+
conf.get(AmoroManagementConf.CLEAN_DANGLING_DELETE_FILES_INTERVAL));
}
this.optimizingCommitExecutor =
new OptimizingCommitExecutor(
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/SnapshotsExpiringExecutor.java
b/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/SnapshotsExpiringExecutor.java
index d137b98a7..c1788436d 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/SnapshotsExpiringExecutor.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/SnapshotsExpiringExecutor.java
@@ -29,21 +29,23 @@ import
org.apache.amoro.server.table.cleanup.CleanupOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.time.Duration;
import java.util.concurrent.ThreadLocalRandom;
/** Service for expiring tables periodically. */
public class SnapshotsExpiringExecutor extends PeriodicTableScheduler {
private static final Logger LOG =
LoggerFactory.getLogger(SnapshotsExpiringExecutor.class);
- private static final long INTERVAL = 60 * 60 * 1000L; // 1 hour
+ private final Duration interval;
- public SnapshotsExpiringExecutor(TableService tableService, int poolSize) {
+ public SnapshotsExpiringExecutor(TableService tableService, int poolSize,
Duration interval) {
super(tableService, poolSize);
+ this.interval = interval;
}
@Override
protected long getNextExecutingTime(TableRuntime tableRuntime) {
- return INTERVAL;
+ return interval.toMillis();
}
@Override
@@ -58,7 +60,7 @@ public class SnapshotsExpiringExecutor extends
PeriodicTableScheduler {
@Override
protected boolean shouldExecute(Long lastCleanupEndTime) {
- return System.currentTimeMillis() - lastCleanupEndTime >= INTERVAL;
+ return System.currentTimeMillis() - lastCleanupEndTime >=
interval.toMillis();
}
@Override
@@ -68,7 +70,7 @@ public class SnapshotsExpiringExecutor extends
PeriodicTableScheduler {
@Override
protected long getExecutorDelay() {
- return ThreadLocalRandom.current().nextLong(INTERVAL);
+ return ThreadLocalRandom.current().nextLong(interval.toMillis());
}
@Override
diff --git
a/amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/TestConfigurableIntervalExecutors.java
b/amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/TestConfigurableIntervalExecutors.java
new file mode 100644
index 000000000..3a20c6101
--- /dev/null
+++
b/amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/TestConfigurableIntervalExecutors.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.scheduler.inline;
+
+import org.apache.amoro.TableRuntime;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.time.Duration;
+
+/**
+ * Tests for configurable interval in DanglingDeleteFilesCleaningExecutor and
+ * SnapshotsExpiringExecutor.
+ */
+public class TestConfigurableIntervalExecutors {
+
+ @Test
+ public void testDanglingDeleteFilesDefaultInterval() {
+ Duration interval = Duration.ofDays(1);
+ DanglingDeleteFilesCleaningExecutor executor =
+ new DanglingDeleteFilesCleaningExecutor(null, 1, interval);
+
+ TableRuntime tableRuntime = Mockito.mock(TableRuntime.class);
+ Assert.assertEquals(Duration.ofDays(1).toMillis(),
executor.getNextExecutingTime(tableRuntime));
+ }
+
+ @Test
+ public void testDanglingDeleteFilesCustomInterval() {
+ Duration interval = Duration.ofHours(12);
+ DanglingDeleteFilesCleaningExecutor executor =
+ new DanglingDeleteFilesCleaningExecutor(null, 1, interval);
+
+ TableRuntime tableRuntime = Mockito.mock(TableRuntime.class);
+ Assert.assertEquals(
+ Duration.ofHours(12).toMillis(),
executor.getNextExecutingTime(tableRuntime));
+ }
+
+ @Test
+ public void testDanglingDeleteFilesShouldExecuteAfterInterval() {
+ Duration interval = Duration.ofHours(6);
+ DanglingDeleteFilesCleaningExecutor executor =
+ new DanglingDeleteFilesCleaningExecutor(null, 1, interval);
+
+ long now = System.currentTimeMillis();
+ // 7 hours ago - should execute
+ Assert.assertTrue(executor.shouldExecute(now -
Duration.ofHours(7).toMillis()));
+ // 5 hours ago - should not execute
+ Assert.assertFalse(executor.shouldExecute(now -
Duration.ofHours(5).toMillis()));
+ }
+
+ @Test
+ public void testSnapshotsExpiringDefaultInterval() {
+ Duration interval = Duration.ofHours(1);
+ SnapshotsExpiringExecutor executor = new SnapshotsExpiringExecutor(null,
1, interval);
+
+ TableRuntime tableRuntime = Mockito.mock(TableRuntime.class);
+ Assert.assertEquals(
+ Duration.ofHours(1).toMillis(),
executor.getNextExecutingTime(tableRuntime));
+ }
+
+ @Test
+ public void testSnapshotsExpiringCustomInterval() {
+ Duration interval = Duration.ofMinutes(30);
+ SnapshotsExpiringExecutor executor = new SnapshotsExpiringExecutor(null,
1, interval);
+
+ TableRuntime tableRuntime = Mockito.mock(TableRuntime.class);
+ Assert.assertEquals(
+ Duration.ofMinutes(30).toMillis(),
executor.getNextExecutingTime(tableRuntime));
+ }
+
+ @Test
+ public void testSnapshotsExpiringShouldExecuteAfterInterval() {
+ Duration interval = Duration.ofHours(2);
+ SnapshotsExpiringExecutor executor = new SnapshotsExpiringExecutor(null,
1, interval);
+
+ long now = System.currentTimeMillis();
+ // 3 hours ago - should execute
+ Assert.assertTrue(executor.shouldExecute(now -
Duration.ofHours(3).toMillis()));
+ // 1 hour ago - should not execute
+ Assert.assertFalse(executor.shouldExecute(now -
Duration.ofHours(1).toMillis()));
+ }
+}
diff --git a/docs/configuration/ams-config.md b/docs/configuration/ams-config.md
index d976978c2..24b518d0c 100644
--- a/docs/configuration/ams-config.md
+++ b/docs/configuration/ams-config.md
@@ -50,6 +50,7 @@ table td:last-child, table th:last-child { width: 40%;
word-break: break-all; }
| blocker.timeout | 1 min | Session timeout. Default unit is milliseconds if
not specified. |
| catalog-meta-cache.expiration-interval | 1 min | TTL for catalog metadata. |
| clean-dangling-delete-files.enabled | true | Enable dangling delete files
cleaning. |
+| clean-dangling-delete-files.interval | 1 d | Interval for cleaning dangling
delete files. |
| clean-dangling-delete-files.thread-count | 10 | The number of threads used
for dangling delete files cleaning. |
| clean-orphan-files.enabled | true | Enable orphan files cleaning. |
| clean-orphan-files.interval | 1 d | Interval for cleaning orphan files. |
@@ -67,6 +68,7 @@ table td:last-child, table th:last-child { width: 40%;
word-break: break-all; }
| database.url | jdbc:derby:/tmp/amoro/derby;create=true | Database connection
address |
| database.username | root | The username for connecting to the database. |
| expire-snapshots.enabled | true | Enable snapshots expiring. |
+| expire-snapshots.interval | 1 h | Interval for expiring snapshots. |
| expire-snapshots.thread-count | 10 | The number of threads used for
snapshots expiring. |
| ha.cluster-name | default | Amoro management service cluster name. |
| ha.connection-timeout | 5 min | The Zookeeper connection timeout in
milliseconds. |
