This is an automated email from the ASF dual-hosted git repository.
dspavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new ce6159bd IGNITE-28729 Enable cleaner as maintenance job and add
progress reporting (#234)
ce6159bd is described below
commit ce6159bd00bae9c3871ecee1a32c9fd4fa806b56
Author: ignitetcbot <[email protected]>
AuthorDate: Fri May 29 00:01:55 2026 +0300
IGNITE-28729 Enable cleaner as maintenance job and add progress reporting
(#234)
Enabling the cleaner background job, adding progress reporting and
improving cleanup observability.
Codex co-authored-by: Dmitriy Pavlov <[email protected]>
---
.gitignore | 3 +
conf/branches.json | 2 +-
.../ci/web/rest/monitoring/MonitoringService.java | 10 +-
.../tcbot/engine/board/TeamcityIgnitedModule.java | 6 +
.../engine/cleaner/TeamcityIgnitedModule.java | 6 +
.../app/guice/GuiceTcBotApplicationContext.java | 1 +
.../interceptor/MonitoredTaskInterceptor.java | 22 +-
.../ignite/tcbot/engine/TcBotEngineModule.java | 2 +
.../tcbot/common/monitoring/MonitoredTasks.java | 13 ++
.../ignite/tcbot/engine/cleaner/Cleaner.java | 257 +++++++++++++++++++--
.../tcbot/engine/process/ProgressReporter.java | 83 +++++++
.../integrationTest/python/teamcity_emulator.py | 29 ++-
.../apache/ignite/tcignited/build/FatBuildDao.java | 99 ++++++--
13 files changed, 488 insertions(+), 45 deletions(-)
diff --git a/.gitignore b/.gitignore
index 09bf248f..fb124ed9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -34,6 +34,9 @@ __pycache__/
# virtual machine crash logs, see
http://www.java.com/en/download/help/error_hotspot.xml
hs_err_pid*
/conf/branches-prom.json
+/conf/diagnostic/
+/conf/tcbot_logs/
+/conf/work/
/ignite-tc-helper-web/ignite/**
/ignite-tc-helper-web/src/test/tmp/**
/migrator/src/test/work/**
diff --git a/conf/branches.json b/conf/branches.json
index f9f8d1cf..ab5c6553 100644
--- a/conf/branches.json
+++ b/conf/branches.json
@@ -9,7 +9,7 @@
"confidence": 0.995,
"cleanerConfig": {
"numOfItemsToDel": 100000,
- "safeDaysForCaches": 180,
+ "safeDaysForCaches": 150,
"safeDaysForLogs": 90,
"period": 1440
},
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java
index 72e7b504..d9484728 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java
@@ -77,6 +77,7 @@ import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.conf.NotificationsConfig;
import org.apache.ignite.tcbot.engine.process.BotProcessMonitor;
import org.apache.ignite.tcbot.engine.process.BotProcessStatus;
+import org.apache.ignite.tcbot.engine.process.ProgressReporter;
import org.apache.ignite.tcbot.notify.IEmailSender;
import org.apache.ignite.tcbot.notify.ISendEmailConfig;
import org.apache.ignite.tcbot.notify.ISlackSender;
@@ -232,6 +233,7 @@ public class MonitoringService {
MaintenanceActionRegistry actions =
instance(MaintenanceActionRegistry.class);
IScheduler scheduler = instance(IScheduler.class);
BotProcessMonitor process = instance(BotProcessMonitor.class);
+ ProgressReporter progress = instance(ProgressReporter.class);
if (!actions.hasAction(name)) {
process.fail(processId, "Maintenance action is not registered: " +
name);
@@ -243,14 +245,10 @@ public class MonitoringService {
boolean accepted = scheduler.runNamedNow(name, () -> {
try {
- process.status(processId, "Running maintenance action: " +
name);
-
- String result = actions.run(name);
-
- process.finish(processId, result);
+ progress.run(processId, "maintenanceAction", "Maintenance
action request accepted: " + name,
+ () -> actions.run(name));
}
catch (Exception e) {
- process.fail(processId, e);
throw new RuntimeException(e);
}
}, processId);
diff --git
a/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcbot/engine/board/TeamcityIgnitedModule.java
b/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcbot/engine/board/TeamcityIgnitedModule.java
index 108f9d03..dfc9324d 100644
---
a/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcbot/engine/board/TeamcityIgnitedModule.java
+++
b/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcbot/engine/board/TeamcityIgnitedModule.java
@@ -29,7 +29,10 @@ import org.apache.ignite.tcbot.engine.cleaner.Cleaner;
import org.apache.ignite.tcbot.engine.defect.DefectsStorage;
import org.apache.ignite.tcbot.engine.issue.IIssuesStorage;
import org.apache.ignite.tcbot.engine.issue.IssuesStorage;
+import org.apache.ignite.tcbot.engine.process.BotProcessMonitor;
+import org.apache.ignite.tcbot.engine.process.ProgressReporter;
import org.apache.ignite.tcbot.engine.user.IUserStorage;
+import org.apache.ignite.tcbot.persistence.scheduler.MaintenanceActionRegistry;
import org.apache.ignite.tcignited.ITeamcityIgnitedProvider;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.build.ProactiveFatBuildSync;
@@ -76,6 +79,9 @@ public class TeamcityIgnitedModule extends AbstractModule {
bind(BoardService.class).in(Scopes.SINGLETON);
bind(ITeamcityIgnitedProvider.class).toInstance(mock(ITeamcityIgnitedProvider.class));
bind(IUserStorage.class).toInstance(mock(IUserStorage.class));
+ bind(BotProcessMonitor.class).in(Scopes.SINGLETON);
+ bind(ProgressReporter.class).in(Scopes.SINGLETON);
+ bind(MaintenanceActionRegistry.class).in(Scopes.SINGLETON);
TcRealConnectionModule module = new TcRealConnectionModule();
diff --git
a/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcbot/engine/cleaner/TeamcityIgnitedModule.java
b/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcbot/engine/cleaner/TeamcityIgnitedModule.java
index 5df8cc68..d44906a8 100644
---
a/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcbot/engine/cleaner/TeamcityIgnitedModule.java
+++
b/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcbot/engine/cleaner/TeamcityIgnitedModule.java
@@ -28,6 +28,9 @@ import
org.apache.ignite.ci.teamcity.ignited.change.ChangeSync;
import org.apache.ignite.tcbot.engine.defect.DefectsStorage;
import org.apache.ignite.tcbot.engine.issue.IIssuesStorage;
import org.apache.ignite.tcbot.engine.issue.IssuesStorage;
+import org.apache.ignite.tcbot.engine.process.BotProcessMonitor;
+import org.apache.ignite.tcbot.engine.process.ProgressReporter;
+import org.apache.ignite.tcbot.persistence.scheduler.MaintenanceActionRegistry;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.build.ProactiveFatBuildSync;
import org.apache.ignite.tcignited.build.UpdateCountersStorage;
@@ -68,6 +71,9 @@ public class TeamcityIgnitedModule extends AbstractModule {
bind(Cleaner.class).in(Scopes.SINGLETON);
bind(DefectsStorage.class).in(Scopes.SINGLETON);
bind(IIssuesStorage.class).to(IssuesStorage.class).in(Scopes.SINGLETON);
+ bind(BotProcessMonitor.class).in(Scopes.SINGLETON);
+ bind(ProgressReporter.class).in(Scopes.SINGLETON);
+ bind(MaintenanceActionRegistry.class).in(Scopes.SINGLETON);
TcRealConnectionModule module = new TcRealConnectionModule();
diff --git
a/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/app/guice/GuiceTcBotApplicationContext.java
b/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/app/guice/GuiceTcBotApplicationContext.java
index 49167a01..494610b5 100644
---
a/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/app/guice/GuiceTcBotApplicationContext.java
+++
b/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/app/guice/GuiceTcBotApplicationContext.java
@@ -90,6 +90,7 @@ class GuiceTcBotApplicationContext implements
TcBotApplicationContext {
return;
getInstance(BuildObserver.class);
+ getInstance(Cleaner.class).startBackgroundClean();
getInstance(UserAdminRefreshService.class).start();
ready.set(true);
}
diff --git
a/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/common/interceptor/MonitoredTaskInterceptor.java
b/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/common/interceptor/MonitoredTaskInterceptor.java
index cb43706e..80da268b 100644
---
a/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/common/interceptor/MonitoredTaskInterceptor.java
+++
b/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/common/interceptor/MonitoredTaskInterceptor.java
@@ -77,6 +77,7 @@ public class MonitoredTaskInterceptor implements
MethodInterceptor, MonitoredTas
private final AtomicLong lastStartTs = new AtomicLong();
private final AtomicLong lastEndTs = new AtomicLong();
private final AtomicReference<Object> lastResult = new
AtomicReference<>();
+ private final AtomicReference<String> currentStatus = new
AtomicReference<>();
private final AtomicInteger callsCnt = new AtomicInteger();
/** Name and full key for monitored task. */
@@ -92,11 +93,18 @@ public class MonitoredTaskInterceptor implements
MethodInterceptor, MonitoredTas
lastStartTs.set(startTs);
lastEndTs.set(0);
+ currentStatus.set(null);
}
void saveEnd(long ts, Object res) {
- lastEndTs.set(ts);
lastResult.set(res);
+ currentStatus.set(null);
+ lastEndTs.set(ts);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void reportCurrentStatus(String status) {
+ currentStatus.set(status);
}
public String name() {
@@ -142,8 +150,10 @@ public class MonitoredTaskInterceptor implements
MethodInterceptor, MonitoredTas
public String result() {
if (lastEndTs.get() == 0) {
long time = System.currentTimeMillis() - lastStartTs.get();
+ String duration = "(running for " +
TimeUtil.millisToDurationPrintable(time) + ")";
+ String status = currentStatus.get();
- return "(running for " +
TimeUtil.millisToDurationPrintable(time) + ")";
+ return status == null ? duration : status + " " + duration;
}
return Objects.toString(lastResult.get());
@@ -177,7 +187,10 @@ public class MonitoredTaskInterceptor implements
MethodInterceptor, MonitoredTas
log(monitoredInvoke.toString(), -1);
Object res = null;
+ MonitoredTasks.Invocation prevInvocation =
MonitoredTasks.CURRENT_INVOCATION.get();
try {
+ MonitoredTasks.CURRENT_INVOCATION.set(monitoredInvoke);
+
res = invocation.proceed();
return res;
@@ -188,6 +201,11 @@ public class MonitoredTaskInterceptor implements
MethodInterceptor, MonitoredTas
throw t;
}
finally {
+ if (prevInvocation == null)
+ MonitoredTasks.CURRENT_INVOCATION.remove();
+ else
+ MonitoredTasks.CURRENT_INVOCATION.set(prevInvocation);
+
long end = System.currentTimeMillis();
monitoredInvoke.saveEnd(end, res);
diff --git
a/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/engine/TcBotEngineModule.java
b/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/engine/TcBotEngineModule.java
index b3d1abee..c88f5f40 100644
---
a/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/engine/TcBotEngineModule.java
+++
b/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/engine/TcBotEngineModule.java
@@ -29,6 +29,7 @@ import org.apache.ignite.tcbot.engine.issue.IIssuesStorage;
import org.apache.ignite.tcbot.engine.issue.IssuesStorage;
import org.apache.ignite.tcbot.engine.newtests.NewTestsStorage;
import org.apache.ignite.tcbot.engine.process.BotProcessMonitor;
+import org.apache.ignite.tcbot.engine.process.ProgressReporter;
import org.apache.ignite.tcbot.engine.tracked.IDetailedStatusForTrackedBranch;
import org.apache.ignite.tcbot.engine.tracked.TrackedBranchChainsProcessor;
import org.apache.ignite.tcbot.engine.user.IUserStorage;
@@ -55,6 +56,7 @@ public class TcBotEngineModule extends AbstractModule {
bind(MutedIssuesDao.class).in(Scopes.SINGLETON);
bind(NewTestsStorage.class).in(Scopes.SINGLETON);
bind(BotProcessMonitor.class).in(Scopes.SINGLETON);
+ bind(ProgressReporter.class).in(Scopes.SINGLETON);
install(new TcBotCommonModule());
}
diff --git
a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/monitoring/MonitoredTasks.java
b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/monitoring/MonitoredTasks.java
index 9ac8df5a..3c83aa90 100644
---
a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/monitoring/MonitoredTasks.java
+++
b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/monitoring/MonitoredTasks.java
@@ -20,10 +20,19 @@ package org.apache.ignite.tcbot.common.monitoring;
import java.util.Collection;
public interface MonitoredTasks extends AutoCloseable {
+ ThreadLocal<Invocation> CURRENT_INVOCATION = new ThreadLocal<>();
+
Collection<? extends Invocation> getList();
long startedTs();
+ static void reportCurrentTaskStatus(String status) {
+ Invocation invocation = CURRENT_INVOCATION.get();
+
+ if (invocation != null)
+ invocation.reportCurrentStatus(status);
+ }
+
@Override void close() throws Exception;
interface Invocation {
@@ -40,5 +49,9 @@ public interface MonitoredTasks extends AutoCloseable {
String end();
String result();
+
+ default void reportCurrentStatus(String status) {
+ // No-op for implementations that do not expose in-flight progress.
+ }
}
}
diff --git
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/cleaner/Cleaner.java
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/cleaner/Cleaner.java
index 387293c9..85260758 100644
---
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/cleaner/Cleaner.java
+++
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/cleaner/Cleaner.java
@@ -19,6 +19,7 @@ package org.apache.ignite.tcbot.engine.cleaner;
import java.io.File;
import java.time.ZonedDateTime;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -27,6 +28,7 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.inject.Inject;
+import javax.inject.Provider;
import org.apache.ignite.ci.teamcity.ignited.buildcondition.BuildConditionDao;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.tcbot.common.conf.TcBotWorkDir;
@@ -36,6 +38,8 @@ import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.defect.DefectsStorage;
import org.apache.ignite.tcbot.engine.issue.IIssuesStorage;
import org.apache.ignite.tcbot.engine.newtests.NewTestsStorage;
+import org.apache.ignite.tcbot.engine.process.ProgressReporter;
+import org.apache.ignite.tcbot.persistence.scheduler.MaintenanceActionRegistry;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.buildlog.BuildLogCheckResultDao;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
@@ -50,7 +54,11 @@ import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
public class Cleaner {
+ /** Progress reporting step for scanning old fat builds. */
+ private static final int OLD_BUILD_SCAN_PROGRESS_STEP = 1_000;
+
private final AtomicBoolean init = new AtomicBoolean();
+ private final AtomicBoolean maintenanceActionRegistered = new
AtomicBoolean();
@Inject private IIssuesStorage issuesStorage;
@Inject private FatBuildDao fatBuildDao;
@@ -62,15 +70,21 @@ public class Cleaner {
@Inject private DefectsStorage defectsStorage;
@Inject private NewTestsStorage newTestsStorage;
@Inject private ITcBotConfig cfg;
+ @Inject private ProgressReporter progress;
+ @Inject private MaintenanceActionRegistry maintenanceActions;
+ @Inject private Provider<Cleaner> self;
/** Logger. */
private static final Logger logger =
LoggerFactory.getLogger(Cleaner.class);
private ScheduledExecutorService executorService;
+ /** Maintenance action name. */
+ public static final String MAINTENANCE_ACTION_NAME = "Cleaner.clean";
+
@AutoProfiling
@MonitoredTask(name = "Clean old cache data and log files")
- public void clean() {
+ public String clean() {
try {
if (cfg.getCleanerConfig().enabled()) {
int numOfItemsToDel = cfg.getCleanerConfig().numOfItemsToDel();
@@ -83,33 +97,116 @@ public class Cleaner {
ZonedDateTime thresholdDateForLogs =
ZonedDateTime.now().minusDays(safeDaysForLogs);
- logger.info("Some data from caches (numOfItemsToDel=" +
numOfItemsToDel + ") older than " + thresholdDateForCaches + " will be
removed.");
+ logger.info("Some data from caches (numOfItemsToDel=" +
numOfItemsToDel + ") older than "
+ + thresholdDateForCaches + " will be removed.");
logger.info("Some log files (numOfItemsToDel=" +
numOfItemsToDel + ") older than " + thresholdDateForLogs + " will be removed.");
- removeCacheEntries(thresholdDateForCaches, numOfItemsToDel);
+ CacheCleanResult cacheRes =
removeCacheEntries(thresholdDateForCaches, numOfItemsToDel);
+
+ LogCleanResult logRes = removeLogFiles(thresholdDateForLogs,
numOfItemsToDel);
- removeLogFiles(thresholdDateForLogs, numOfItemsToDel);
+ String res = cacheRes.summary() + "; " + logRes.summary();
+
+ report(res);
+
+ return res;
}
- else
+ else {
logger.info("Periodic cache clean disabled.");
+
+ return "Periodic cache clean disabled.";
+ }
}
catch (Throwable e) {
logger.error("Periodic cache and log clean failed: " +
e.getMessage(), e);
e.printStackTrace();
+
+ return "Periodic cache and log clean failed: " + e.getMessage();
}
}
- private int removeCacheEntries(ZonedDateTime thresholdDate, int
numOfItemsToDel) {
+ private CacheCleanResult removeCacheEntries(ZonedDateTime thresholdDate,
int numOfItemsToDel) {
long thresholdEpochMilli = thresholdDate.toInstant().toEpochMilli();
- Set<Long> oldBuildsKeys =
fatBuildDao.getOldBuilds(thresholdEpochMilli, numOfItemsToDel);
+ report("Checking " + FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME + " for
builds older than " + thresholdDate);
+
+ int totalPartitions = fatBuildDao.affinity().partitions();
+ int selected = 0;
+ int removed = 0;
+ int scannedEntries = 0;
+ int scannedPartitions = 0;
+ boolean limitReached = false;
+
+ for (int part = 0; part < totalPartitions && removed <
numOfItemsToDel; part++) {
+ scannedPartitions++;
+ Set<Long> checkedInPartition = new HashSet<>();
+
+ while (removed < numOfItemsToDel) {
+ int remainingLimit = numOfItemsToDel - removed;
+
+ report("Checking " + FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME
+ " partition " + (part + 1) + "/"
+ + totalPartitions + ", remaining delete limit " +
remainingLimit + ", removed so far " + removed);
+
+ FatBuildDao.OldBuildsSearchResult oldBuilds =
fatBuildDao.getOldBuildsFromPartition(
+ thresholdEpochMilli,
+ part,
+ remainingLimit,
+ OLD_BUILD_SCAN_PROGRESS_STEP,
+ checkedInPartition,
+ this::report);
+
+ selected += oldBuilds.selected();
+ scannedEntries += oldBuilds.scanned();
+ checkedInPartition.addAll(oldBuilds.keys());
+
+ if (oldBuilds.keys().isEmpty()) {
+ report("Checked " +
FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME + " partition " + (part + 1) + "/"
+ + totalPartitions + ": scanned " + oldBuilds.scanned()
+ + " entries, no more old build candidates");
+
+ break;
+ }
+
+ report("Checked " + FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME
+ " partition " + (part + 1) + "/"
+ + totalPartitions + ": cursor closed, scanned " +
oldBuilds.scanned()
+ + " entries, selected " + oldBuilds.selected() + ",
deleting "
+ + oldBuilds.keys().size() + " candidate records");
+
+ int partitionRemoved =
removeCacheEntriesForPartition(oldBuilds.keys(), part, totalPartitions);
+
+ removed += partitionRemoved;
+
+ report("Finished " + FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME
+ " partition " + (part + 1) + "/"
+ + totalPartitions + ": selected " + oldBuilds.selected() +
", removed " + partitionRemoved
+ + ", total selected " + selected + ", total removed " +
removed);
+
+ if (!oldBuilds.deleteLimitReached())
+ break;
+ }
+ }
+
+ if (removed >= numOfItemsToDel && scannedPartitions < totalPartitions)
+ limitReached = true;
+
+ removeInconsistentRecords(thresholdDate, numOfItemsToDel);
+
+ return new CacheCleanResult(selected, removed, scannedEntries,
scannedPartitions, totalPartitions, limitReached);
+ }
+
+ private int removeCacheEntriesForPartition(List<Long> oldBuildsKeysList,
int part, int totalPartitions) {
+ Set<Long> oldBuildsKeys = new HashSet<>(oldBuildsKeysList);
Map<Integer, List<Integer>> oldBuildsTeamCityAndBuildIds =
oldBuildsKeys.stream()
.map(FatBuildDao::cacheKeyToSrvIdAndBuildId)
.collect(groupingBy(IgniteBiTuple::get1,
mapping(IgniteBiTuple::get2, toList())));
+ String partition = "partition " + (part + 1) + "/" + totalPartitions;
+
+ report("Checking defects before cache cleanup " + partition + ": " +
oldBuildsKeys.size()
+ + " candidate builds");
+
defectsStorage.checkIfPossibleToRemove(oldBuildsTeamCityAndBuildIds);
oldBuildsKeys = oldBuildsTeamCityAndBuildIds.entrySet().stream()
@@ -119,60 +216,172 @@ public class Cleaner {
logger.info("Builds will be removed (" + oldBuildsKeys.size() + ")");
+ report("Removing " + partition + " from teamcitySuiteHistory: " +
oldBuildsKeys.size()
+ + " build records");
suiteInvocationHistoryDao.removeAll(oldBuildsKeys);
+
+ report("Removing " + partition + " from buildLogCheckResult: " +
oldBuildsKeys.size()
+ + " build records");
buildLogCheckResultDao.removeAll(oldBuildsKeys);
+
+ report("Removing " + partition + " from teamcityBuildRef: " +
oldBuildsKeys.size()
+ + " build records");
buildRefDao.removeAll(oldBuildsKeys);
+
+ report("Removing " + partition + " from teamcityBuildStartTime: " +
oldBuildsKeys.size()
+ + " build records");
buildStartTimeStorage.removeAll(oldBuildsKeys);
+
+ report("Removing " + partition + " from buildsConditions: " +
oldBuildsKeys.size()
+ + " build records");
buildConditionDao.removeAll(oldBuildsKeys);
+
+ report("Removing old defects " + partition + ": " +
oldBuildsKeys.size() + " build records");
defectsStorage.removeOldDefects(oldBuildsTeamCityAndBuildIds);
+
+ report("Removing old issues " + partition + ": " +
oldBuildsKeys.size() + " build records");
issuesStorage.removeOldIssues(oldBuildsTeamCityAndBuildIds);
+
+ report("Removing " + partition + " from " +
FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME + ": "
+ + oldBuildsKeys.size() + " build records");
fatBuildDao.removeAll(oldBuildsKeys);
+ return oldBuildsKeys.size();
+ }
+
+ private void removeInconsistentRecords(ZonedDateTime thresholdDate, int
numOfItemsToDel) {
+ int deleteLimit = Math.max(1, numOfItemsToDel);
+
//Need to eventually delete data with broken consistency
-
defectsStorage.removeOldDefects(thresholdDate.minusDays(60).toInstant().toEpochMilli(),
numOfItemsToDel);
-
issuesStorage.removeOldIssues(thresholdDate.minusDays(60).toInstant().toEpochMilli(),
numOfItemsToDel);
+ report("Removing inconsistent old defects older than " +
thresholdDate.minusDays(60));
+
defectsStorage.removeOldDefects(thresholdDate.minusDays(60).toInstant().toEpochMilli(),
deleteLimit);
-
newTestsStorage.removeOldTests(ZonedDateTime.now().minusDays(5).toInstant().toEpochMilli());
+ report("Removing inconsistent old issues older than " +
thresholdDate.minusDays(60));
+
issuesStorage.removeOldIssues(thresholdDate.minusDays(60).toInstant().toEpochMilli(),
deleteLimit);
- return oldBuildsKeys.size();
+ report("Removing old new-tests records");
+
newTestsStorage.removeOldTests(ZonedDateTime.now().minusDays(5).toInstant().toEpochMilli());
}
- private void removeLogFiles(ZonedDateTime thresholdDate, int
numOfItemsToDel) {
+ private LogCleanResult removeLogFiles(ZonedDateTime thresholdDate, int
numOfItemsToDel) {
long thresholdEpochMilli = thresholdDate.toInstant().toEpochMilli();
final File workDir = TcBotWorkDir.resolveWorkDir();
+ LogCleanResult res = new LogCleanResult();
+
for (String srvId : cfg.getServerIds()) {
File srvIdLogDir = new File(workDir,
cfg.getTeamcityConfig(srvId).logsDirectory());
- removeFiles(srvIdLogDir, thresholdEpochMilli, numOfItemsToDel);
+ res.add(removeFiles(srvIdLogDir, thresholdEpochMilli,
numOfItemsToDel));
}
File tcBotLogDir = new File(workDir, "tcbot_logs");
- removeFiles(tcBotLogDir, thresholdEpochMilli, numOfItemsToDel);
+ res.add(removeFiles(tcBotLogDir, thresholdEpochMilli,
numOfItemsToDel));
+
+ return res;
}
- private void removeFiles(File dir, long thresholdDate, int
numOfItemsToDel) {
+ private LogCleanResult removeFiles(File dir, long thresholdDate, int
numOfItemsToDel) {
+ report("Checking log directory " + dir);
+
File[] logFiles = dir.listFiles();
List<File> filesToRmv = new ArrayList<>(numOfItemsToDel);
+ int checked = 0;
+
+ if (logFiles != null) {
+ for (File file : logFiles) {
+ checked++;
- if (logFiles != null)
- for (File file : logFiles)
if (file.lastModified() < thresholdDate && numOfItemsToDel-- >
0)
filesToRmv.add(file);
+ }
+ }
logger.info("In the directory " + dir + " files will be removed (" +
filesToRmv.size() + ")");
+ int removed = 0;
+
for (File file : filesToRmv) {
- file.delete();
+ if (file.delete())
+ removed++;
+ }
+
+ report("Checked log directory " + dir + ": removed " + removed + "/" +
filesToRmv.size()
+ + " old files, checked " + checked + " files");
+
+ return new LogCleanResult(checked, filesToRmv.size(), removed);
+ }
+
+ private void report(String status) {
+ progress.report(status);
+ }
+
+ private static class CacheCleanResult {
+ private final int selected;
+
+ private final int removed;
+
+ private final int scannedEntries;
+
+ private final int scannedPartitions;
+
+ private final int totalPartitions;
+
+ private final boolean limitReached;
+
+ CacheCleanResult(int selected, int removed, int scannedEntries, int
scannedPartitions, int totalPartitions,
+ boolean limitReached) {
+ this.selected = selected;
+ this.removed = removed;
+ this.scannedEntries = scannedEntries;
+ this.scannedPartitions = scannedPartitions;
+ this.totalPartitions = totalPartitions;
+ this.limitReached = limitReached;
+ }
+
+ String summary() {
+ return "Caches: removed " + removed + "/" + selected + " selected
old build records"
+ + ", scanned entries " + scannedEntries
+ + ", scanned partitions " + scannedPartitions + "/" +
totalPartitions
+ + (limitReached ? ", delete limit reached, more old builds may
remain" : "");
+ }
+ }
+
+ private static class LogCleanResult {
+ private int checked;
+
+ private int oldFiles;
+
+ private int removed;
+
+ LogCleanResult() {
+ // No-op.
+ }
+
+ LogCleanResult(int checked, int oldFiles, int removed) {
+ this.checked = checked;
+ this.oldFiles = oldFiles;
+ this.removed = removed;
+ }
+
+ void add(LogCleanResult res) {
+ checked += res.checked;
+ oldFiles += res.oldFiles;
+ removed += res.removed;
}
+ String summary() {
+ return "Logs: removed " + removed + "/" + oldFiles + " old files,
checked " + checked + " files";
+ }
}
public void startBackgroundClean() {
if (init.compareAndSet(false, true)) {
+ registerMaintenanceAction();
+
suiteInvocationHistoryDao.init();
buildLogCheckResultDao.init();
buildRefDao.init();
@@ -182,7 +391,17 @@ public class Cleaner {
executorService = Executors.newSingleThreadScheduledExecutor();
- executorService.scheduleAtFixedRate(this::clean, 5,
cfg.getCleanerConfig().period(), TimeUnit.MINUTES);
+ executorService.scheduleAtFixedRate(() -> self.get().clean(), 5,
cfg.getCleanerConfig().period(),
+ TimeUnit.MINUTES);
+ }
+ }
+
+ /** Registers manual cleaner action for monitoring management UI. */
+ private void registerMaintenanceAction() {
+ if (maintenanceActionRegistered.compareAndSet(false, true)) {
+ maintenanceActions.register(MAINTENANCE_ACTION_NAME,
+ "Run cleaner for old cache data and log files",
+ () -> self.get().clean());
}
}
diff --git
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/process/ProgressReporter.java
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/process/ProgressReporter.java
new file mode 100644
index 00000000..29e79e35
--- /dev/null
+++
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/process/ProgressReporter.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.tcbot.engine.process;
+
+import java.util.concurrent.Callable;
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import org.apache.ignite.tcbot.common.monitoring.MonitoredTasks;
+
+/**
+ * Shared progress reporter for monitored tasks and user-visible UI processes.
+ */
+@Singleton
+public class ProgressReporter {
+ /** Current user-visible process id, when a UI-triggered operation is
running. */
+ private final ThreadLocal<Long> currentProcessId = new ThreadLocal<>();
+
+ /** User-visible process monitor. */
+ @Inject private BotProcessMonitor processMonitor;
+
+ /**
+ * Reports progress to every status surface available in the current
thread.
+ *
+ * @param status Status text.
+ */
+ public void report(String status) {
+ MonitoredTasks.reportCurrentTaskStatus(status);
+ processMonitor.status(currentProcessId.get(), status);
+ }
+
+ /**
+ * Runs an action with a current user-visible process id.
+ *
+ * @param processId Process id supplied by UI.
+ * @param kind Process kind.
+ * @param acceptedStatus Initial status.
+ * @param action Action.
+ * @return Action result.
+ */
+ public String run(@Nullable Long processId, String kind, String
acceptedStatus, Callable<String> action)
+ throws Exception {
+ processMonitor.start(processId, kind, acceptedStatus);
+
+ Long prevProcessId = currentProcessId.get();
+
+ try {
+ currentProcessId.set(processId);
+ report(acceptedStatus);
+
+ String result = action.call();
+
+ processMonitor.finish(processId, result);
+
+ return result;
+ }
+ catch (Exception e) {
+ processMonitor.fail(processId, e);
+
+ throw e;
+ }
+ finally {
+ if (prevProcessId == null)
+ currentProcessId.remove();
+ else
+ currentProcessId.set(prevProcessId);
+ }
+ }
+}
diff --git
a/tcbot-integration-tests/src/integrationTest/python/teamcity_emulator.py
b/tcbot-integration-tests/src/integrationTest/python/teamcity_emulator.py
index d052221f..95aaad19 100644
--- a/tcbot-integration-tests/src/integrationTest/python/teamcity_emulator.py
+++ b/tcbot-integration-tests/src/integrationTest/python/teamcity_emulator.py
@@ -52,6 +52,8 @@ RUN_ALL = "IgniteTests24Java17_RunAll"
RUN_ALL_NIGHTLY = "IgniteTests24Java17_RunAllNightly"
PROJECT_ID = "ApacheIgnite"
PROJECT_NAME = "Apache Ignite"
+CLEANER_OLD_MASTER_CHAINS = 8
+CLEANER_OLD_MASTER_BASE_ID = 700000
SUITES = [
"IgniteTests24Java17_Cache1",
"IgniteTests24Java17_ComputeGrid",
@@ -850,6 +852,7 @@ def initial_builds():
builds = {}
create_master_history(builds)
+ create_old_master_cleaner_history(builds)
create_run_all_chain(builds, "800101", "pull/12005/head", "SUCCESS",
"finished",
queued="20260510T090000+0000",
started="20260510T090010+0000",
finished="20260510T090130+0000")
@@ -868,6 +871,26 @@ def initial_builds():
return builds
+def create_old_master_cleaner_history(builds):
+ base_date = datetime.now(timezone.utc) - timedelta(days=365 +
CLEANER_OLD_MASTER_CHAINS)
+
+ for idx in range(CLEANER_OLD_MASTER_CHAINS):
+ build_id = str(CLEANER_OLD_MASTER_BASE_ID + idx * 10)
+ build_date = base_date + timedelta(days=idx)
+ started_date = build_date + timedelta(seconds=10)
+ finished_date = started_date + timedelta(minutes=2)
+ model = SUITE_MODELS[idx % len(SUITE_MODELS)]
+ failed = idx % 5 == 0
+ failed_tests = {first_randomized_test(model)} if failed else set()
+
+ create_run_all_chain(builds, build_id, "<default>", "FAILURE" if
failed else "SUCCESS", "finished",
+ queued=tc_date_from(build_date),
+ started=tc_date_from(started_date),
+ finished=tc_date_from(finished_date),
+ suite_statuses={model["buildTypeId"]: "FAILURE"
if failed else "SUCCESS"},
+ failed_tests=failed_tests)
+
+
def create_master_history(builds):
base_id = 810000
@@ -1379,7 +1402,11 @@ def build_statistics(build):
def tc_date():
- return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S+0000")
+ return tc_date_from(datetime.now(timezone.utc))
+
+
+def tc_date_from(dt):
+ return dt.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%S+0000")
def main():
diff --git
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index e70b32ff..9aade6e7 100644
---
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -29,6 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
@@ -45,6 +46,7 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.binary.BinaryObject;
import org.apache.ignite.cache.CacheEntryProcessor;
import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.lang.IgniteBiPredicate;
@@ -419,31 +421,96 @@ public class FatBuildDao {
}
}
- public Set<Long> getOldBuilds(long thresholdDate, int numOfItemsToDel) {
+ public OldBuildsSearchResult getOldBuildsFromPartition(long thresholdDate,
int part, int numOfItemsToDel,
+ int progressStep, @Nullable Consumer<String> progressReporter) {
+ return getOldBuildsFromPartition(thresholdDate, part, numOfItemsToDel,
progressStep, null, progressReporter);
+ }
+
+ public OldBuildsSearchResult getOldBuildsFromPartition(long thresholdDate,
int part, int numOfItemsToDel,
+ int progressStep, @Nullable Set<Long> ignoredKeys, @Nullable
Consumer<String> progressReporter) {
IgniteCache<Long, BinaryObject> cacheWithBinary =
buildsCache.withKeepBinary();
- ScanQuery<Long, BinaryObject> scan = new ScanQuery<>((key, fatBuild)
-> {
- Long startDate = 0L;
+ ScanQuery<Long, BinaryObject> scan = new ScanQuery<>();
+ scan.setPartition(part);
- if (fatBuild.hasField("startDate"))
- startDate = fatBuild.<Long>field("startDate");
+ List<Long> oldBuildsKeys = new ArrayList<>();
+ int scanned = 0;
+ int selected = 0;
+ int progressStep0 = Math.max(1, progressStep);
+ boolean deleteLimitReached = false;
- return (startDate > 0 && startDate < thresholdDate) ||
- !fatBuild.hasField("startDate");
- }
- );
+ try (QueryCursor<Cache.Entry<Long, BinaryObject>> cursor =
cacheWithBinary.query(scan)) {
+ for (Cache.Entry<Long, BinaryObject> entry : cursor) {
+ scanned++;
- Set<Long> oldBuildsKeys = new HashSet<>(numOfItemsToDel);
+ if (ignoredKeys != null &&
ignoredKeys.contains(entry.getKey()))
+ continue;
- for (Cache.Entry<Long, BinaryObject> entry :
cacheWithBinary.query(scan)) {
- if (numOfItemsToDel > 0) {
- numOfItemsToDel--;
+ if (!isOldBuild(entry.getValue(), thresholdDate))
+ continue;
+
+ if (oldBuildsKeys.size() >= numOfItemsToDel) {
+ deleteLimitReached = true;
+ break;
+ }
+
+ selected++;
oldBuildsKeys.add(entry.getKey());
+
+ if (progressReporter != null && (selected == 1 || selected %
progressStep0 == 0))
+ progressReporter.accept("Checking " +
TEAMCITY_FAT_BUILD_CACHE_NAME + " partition " + part
+ + ": scanned " + scanned + " entries, selected " +
selected + " old build candidates");
}
- else
- break;
}
- return oldBuildsKeys;
+
+ if (progressReporter != null)
+ progressReporter.accept("Checked " + TEAMCITY_FAT_BUILD_CACHE_NAME
+ " partition " + part
+ + ": scanned " + scanned + " entries, selected " + selected +
" old build candidates"
+ + (deleteLimitReached ? ", delete limit reached, more old
builds may remain" : ""));
+
+ return new OldBuildsSearchResult(oldBuildsKeys, selected, scanned,
deleteLimitReached);
+ }
+
+ private boolean isOldBuild(BinaryObject fatBuild, long thresholdDate) {
+ if (!fatBuild.hasField("startDate"))
+ return true;
+
+ Long startDate = fatBuild.<Long>field("startDate");
+
+ return startDate != null && startDate > 0 && startDate < thresholdDate;
+ }
+
+ public static class OldBuildsSearchResult {
+ private final List<Long> keys;
+
+ private final int selected;
+
+ private final int scanned;
+
+ private final boolean deleteLimitReached;
+
+ public OldBuildsSearchResult(List<Long> keys, int selected, int
scanned, boolean deleteLimitReached) {
+ this.keys = keys;
+ this.selected = selected;
+ this.scanned = scanned;
+ this.deleteLimitReached = deleteLimitReached;
+ }
+
+ public List<Long> keys() {
+ return keys;
+ }
+
+ public int selected() {
+ return selected;
+ }
+
+ public int scanned() {
+ return scanned;
+ }
+
+ public boolean deleteLimitReached() {
+ return deleteLimitReached;
+ }
}
public void remove(long key) {
