This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch time-consumer-detection
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/time-consumer-detection by
this push:
new e0b4e78 Time consuming builds detection: moved to async computation
in scheduler
e0b4e78 is described below
commit e0b4e780ee5bc976b583abf44189562457e1d910
Author: Dmitriy Pavlov <[email protected]>
AuthorDate: Sat Jul 6 14:09:16 2019 +0300
Time consuming builds detection: moved to async computation in scheduler
---
.../ci/tcbot/TcBotBusinessServicesModule.java | 4 +
ignite-tc-helper-web/src/main/webapp/index.html | 3 -
.../tcbot/engine/buildtime/BuildTimeService.java | 122 +++++----------------
.../ignite/tcignited/buildref/BuildRefDao.java | 47 --------
.../tcignited/buildtime/BuildTimeResult.java | 23 ++--
.../ignite/tcignited/history/HistoryCollector.java | 106 +++++++++++++++---
6 files changed, 130 insertions(+), 175 deletions(-)
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotBusinessServicesModule.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotBusinessServicesModule.java
index 2134a3f..f0d636f 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotBusinessServicesModule.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotBusinessServicesModule.java
@@ -19,6 +19,7 @@ package org.apache.ignite.ci.tcbot;
import com.google.inject.AbstractModule;
import com.google.inject.internal.SingletonScope;
import org.apache.ignite.ci.issue.IssuesStorage;
+import org.apache.ignite.tcbot.engine.buildtime.BuildTimeService;
import org.apache.ignite.tcbot.engine.chain.BuildChainProcessor;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.ci.tcbot.conf.LocalFilesBasedConfig;
@@ -42,5 +43,8 @@ public class TcBotBusinessServicesModule extends
AbstractModule {
bind(MasterTrendsService.class).in(new SingletonScope());
bind(ITcBotBgAuth.class).to(TcBotBgAuthImpl.class).in(new
SingletonScope());
bind(BuildChainProcessor.class).in(new SingletonScope());
+
+ //todo move to bot engine module
+ bind(BuildTimeService.class).in(new SingletonScope());
}
}
diff --git a/ignite-tc-helper-web/src/main/webapp/index.html
b/ignite-tc-helper-web/src/main/webapp/index.html
index 6d0f5b8..86f12f3 100644
--- a/ignite-tc-helper-web/src/main/webapp/index.html
+++ b/ignite-tc-helper-web/src/main/webapp/index.html
@@ -40,7 +40,6 @@ function loadData() {
$.ajax({ url: "/rest/buildtime/analytics",
success: function (data) {
- $("#buildTime").html("Test");
$("#loadStatus").html("");
}, error: showErrInLoadStatus });
}
@@ -57,8 +56,6 @@ function loadData() {
<a href="index0.html"><button class="idxpgbutton"><font
size="30px">λ</font><br>I like old home page</button></a>
</div>
-<div id="buildTime"></div>
-
<div id="loadStatus"></div>
<div id="version"></div>
diff --git
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java
index 6da5eda..18d03f1 100644
---
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java
+++
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java
@@ -17,15 +17,13 @@
package org.apache.ignite.tcbot.engine.buildtime;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.binary.BinaryObject;
-import org.apache.ignite.cache.query.QueryCursor;
-import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.tcbot.common.interceptor.MonitoredTask;
import org.apache.ignite.tcbot.common.util.TimeUtil;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.ui.BuildTimeRecordUi;
import org.apache.ignite.tcbot.engine.ui.BuildTimeResultUi;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcbot.persistence.scheduler.IScheduler;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.ITeamcityIgnitedProvider;
import org.apache.ignite.tcignited.build.FatBuildDao;
@@ -34,18 +32,17 @@ import
org.apache.ignite.tcignited.buildtime.BuildTimeRecord;
import org.apache.ignite.tcignited.buildtime.BuildTimeResult;
import org.apache.ignite.tcignited.creds.ICredentialsProv;
import org.apache.ignite.tcignited.history.HistoryCollector;
-import org.apache.ignite.tcignited.history.RunHistCompactedDao;
-import org.apache.ignite.tcservice.model.hist.BuildRef;
-import org.apache.ignite.tcservice.model.result.stat.Statistics;
-import javax.cache.Cache;
import javax.inject.Inject;
import java.time.Duration;
-import java.util.*;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
public class BuildTimeService {
-
@Inject ITeamcityIgnitedProvider tcProv;
/** Config. */
@@ -53,27 +50,24 @@ public class BuildTimeService {
@Inject FatBuildDao fatBuildDao;
- @Inject BuildRefDao buildRefDao;
-
- @Inject RunHistCompactedDao runHistCompactedDao;
-
@Inject IStringCompactor compactor;
@Inject HistoryCollector historyCollector;
- public BuildTimeResultUi analytics(ICredentialsProv prov) {
- String serverCode = cfg.primaryServerCode();
+ @Inject IScheduler scheduler;
- ITeamcityIgnited server = tcProv.server(serverCode, prov);
+ private volatile BuildTimeResult lastRes1d = new BuildTimeResult();
- // fatBuildDao.loadBuildTimeResult();
+ @Inject private BuildRefDao buildRefDao;
- Collection<String> allServers = cfg.getServerIds();
+ public BuildTimeResultUi analytics(ICredentialsProv prov) {
+ if (buildRefDao.buildRefsCache() == null)
+ return new BuildTimeResultUi();
- int days = 1;
- List<Long> idsToCheck = forEachBuildRef(days, allServers);
+ Collection<String> allServers = cfg.getServerIds();
- BuildTimeResult res = fatBuildDao.loadBuildTimeResult(days,
idsToCheck);
+ scheduler.sheduleNamed("BuildTimeService.loadAnalytics",
+ this::loadAnalytics, 15, TimeUnit.MINUTES);
Set<Integer> availableServers = allServers.stream()
.filter(prov::hasAccess)
@@ -83,8 +77,10 @@ public class BuildTimeService {
BuildTimeResultUi resultUi = new BuildTimeResultUi();
long minDuration = Duration.ofHours(1).toMillis();
- List<Map.Entry<Long, BuildTimeRecord>> entries =
res.topBuildTypes(availableServers, minDuration);
- entries.forEach(e->{
+ BuildTimeResult res = lastRes1d;
+ List<Map.Entry<Long, BuildTimeRecord>> entries =
res.topByBuildTypes(availableServers, minDuration, 5);
+
+ entries.forEach(e -> {
BuildTimeRecordUi buildTimeRecordUi = new BuildTimeRecordUi();
Long key = e.getKey();
int btId = BuildTimeResult.cacheKeyToBuildType(key);
@@ -94,82 +90,18 @@ public class BuildTimeService {
resultUi.byBuildType.add(buildTimeRecordUi);
});
-
-
return resultUi;
}
- public List<Long> forEachBuildRef(int days, Collection<String> allServers)
{
- IgniteCache<Long, BinaryObject> cacheBin =
buildRefDao.buildRefsCache().withKeepBinary();
-
- Set<Integer> availableServers = allServers.stream()
- .map(ITeamcityIgnited::serverIdToInt)
- .collect(Collectors.toSet());
-
- Map<Integer, Integer> preBorder = new HashMap<>();
-
- availableServers.forEach(srvId -> {
- Integer borderForAgeForBuildId =
runHistCompactedDao.getBorderForAgeForBuildId(srvId, days);
- if (borderForAgeForBuildId != null)
- preBorder.put(srvId, borderForAgeForBuildId);
- });
-
- int stateRunning = compactor.getStringId(BuildRef.STATE_RUNNING);
- final int stateQueued = compactor.getStringId(BuildRef.STATE_QUEUED);
- Integer buildDurationId =
compactor.getStringIdIfPresent(Statistics.BUILD_DURATION);
-
- long minTs = System.currentTimeMillis() -
Duration.ofDays(days).toMillis();
- QueryCursor<Cache.Entry<Long, BinaryObject>> query = cacheBin.query(
- new ScanQuery<Long, BinaryObject>()
- .setFilter((key, v) -> {
- int srvId = BuildRefDao.cacheKeyToSrvId(key);
- Integer buildIdBorder = preBorder.get(srvId);
- if (buildIdBorder != null) {
- int id = v.field("id");
- if (id < buildIdBorder)
- return false;// pre-filtered build out of scope
- }
- int state = v.field("state");
-
- return stateQueued != state;
- }));
-
- int cnt = 0;
- List<Long> idsToCheck = new ArrayList<>();
-
- try (QueryCursor<Cache.Entry<Long, BinaryObject>> cursor = query) {
- for (Cache.Entry<Long, BinaryObject> next : cursor) {
- Long key = next.getKey();
- int srvId = BuildRefDao.cacheKeyToSrvId(key);
-
- int buildId = BuildRefDao.cacheKeyToBuildId(key);
-
- Integer borderBuildId =
runHistCompactedDao.getBorderForAgeForBuildId(srvId, days);
-
- boolean passesDate = borderBuildId == null || buildId >=
borderBuildId;
-
- if (!passesDate)
- continue;
-
- Long startTs = historyCollector.getBuildStartTime(srvId,
buildId);
- if (startTs == null || startTs < minTs)
- continue; //time not saved in the DB, skip
-
- BinaryObject buildBinary = next.getValue();
- long runningTime = 0l;// getBuildRunningTime(stateRunning,
buildDurationId, buildBinary);
-
- System.err.println("Found build at srv [" + srvId + "]: [" +
buildId + "] to analyze, ts="+ startTs);
-
- cnt++;
-
- idsToCheck.add(key);
- }
- }
+ @SuppressWarnings("WeakerAccess")
+ @MonitoredTask(name = "Load Build Time Analytics")
+ protected void loadAnalytics() {
+ int days = 1;
- System.err.println("Total builds to load " + cnt);
+ List<Long> idsToCheck = historyCollector.findAllRecentBuilds(days,
cfg.getServerIds());
- // serversCompute.call(new BuildTimeIgniteCallable(cacheBin,
stateRunning, buildDurationId));
+ BuildTimeResult res = fatBuildDao.loadBuildTimeResult(days,
idsToCheck);
- return idsToCheck;
+ lastRes1d = res;
}
}
diff --git
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
index 5cca271..ffad600 100644
---
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
@@ -322,51 +322,4 @@ public class BuildRefDao {
public IgniteCache<Long, BuildRefCompacted> buildRefsCache() {
return buildRefsCache;
}
-
- public static class BuildTimeIgniteCallable implements
IgniteCallable<Long> {
- private final int stateRunning;
- private final Integer buildDurationId;
- @IgniteInstanceResource
- Ignite ignite;
-
- public BuildTimeIgniteCallable(IgniteCache<Long, BinaryObject>
cacheBin, int stateRunning,
- Integer buildDurationId) {
- this.stateRunning = stateRunning;
- this.buildDurationId = buildDurationId;
- }
-
- @Override public Long call() throws Exception {
- IgniteCache<Long, BuildRefCompacted> cache =
ignite.cache(TEAMCITY_BUILD_CACHE_NAME);
-
- IgniteCache<Long, BuildRefCompacted>cacheBin =
cache.withKeepBinary();
- QueryCursor<Cache.Entry<Long, BinaryObject>> query =
cacheBin.query(
- new ScanQuery<Long, BinaryObject>()
- .setFilter((k,v)->{
- int srvId = cacheKeyToSrvId(k);
-
-return false;
- })
- .setLocal(true));
-
- /*.query(new SqlQuery<Long, BinaryObject>(
- FatBuildCompacted.class,
- " _KEY > ?")
- .setLocal(true)
- .setArgs(0L));*/
-
-// Iterate over the result set.
- try (QueryCursor<Cache.Entry<Long, BinaryObject>> cursor = query) {
-
- for (Cache.Entry<Long, BinaryObject> next : cursor) {
-
- BinaryObject buildBinary = next.getValue();
- long runningTime = 0l;// getBuildRunningTime(stateRunning,
buildDurationId, buildBinary);
-
- System.err.println("Running " + runningTime);
- }
- }
-
- return null;
- }
- }
}
diff --git
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeResult.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeResult.java
index 4472866..0a05e74 100644
---
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeResult.java
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeResult.java
@@ -26,25 +26,20 @@ public class BuildTimeResult {
}
- public List<Map.Entry<Long, BuildTimeRecord>> topBuildTypes(Set<Integer>
availableServers, long minAvgDurationMs) {
+ public List<Map.Entry<Long, BuildTimeRecord>> topByBuildTypes(Set<Integer>
availableServers,
+ long
minAvgDurationMs,
+ int maxCnt) {
return btByBuildType.entrySet().stream()
- .filter(e->{
+ .filter(e -> {
Long key = e.getKey();
int srvId = cacheKeyToSrvId(key);
return availableServers.contains(srvId);
})
- .filter(e->{
- return e.getValue().avgDuration() > minAvgDurationMs;
- })
- .sorted(Comparator.comparing(new Function<Map.Entry<Long,
BuildTimeRecord>,
- Long>() {
- @Override
- public Long apply( Map.Entry<Long, BuildTimeRecord>
entry) {
- return entry.getValue().avgDuration();
- }
- })
- .reversed())
- .limit(5)
+ .filter(e -> e.getValue().avgDuration() > minAvgDurationMs)
+ .sorted(Comparator.comparing(
+ (Function<Map.Entry<Long, BuildTimeRecord>, Long>)
entry -> entry.getValue().avgDuration())
+ .reversed())
+ .limit(maxCnt)
.collect(Collectors.toList());
}
}
diff --git
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
index de43b6a..1246a3e 100644
---
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
@@ -18,39 +18,44 @@ package org.apache.ignite.tcignited.history;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Iterables;
-import java.time.Duration;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.BiPredicate;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import javax.inject.Inject;
-import javax.inject.Provider;
import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
import org.apache.ignite.tcbot.common.TcBotConst;
import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
+import org.apache.ignite.tcbot.common.exeption.ServicesStartingException;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.build.SuiteHistory;
import org.apache.ignite.tcignited.buildref.BranchEquivalence;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
+import org.apache.ignite.tcservice.model.hist.BuildRef;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.cache.Cache;
+import javax.inject.Inject;
+import javax.inject.Provider;
+import java.time.Duration;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiPredicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
/**
*
*/
@@ -351,4 +356,73 @@ public class HistoryCollector {
return time;
}
+
+ public List<Long> findAllRecentBuilds(int days, Collection<String>
allServers) {
+ IgniteCache<Long, BuildRefCompacted> cache =
buildRefDao.buildRefsCache();
+ if (cache == null)
+ throw new ServicesStartingException(new RuntimeException("Ignite
is not yet available"));
+
+ IgniteCache<Long, BinaryObject> cacheBin = cache.withKeepBinary();
+
+ final Map<Integer, Integer> preBorder = new HashMap<>();
+
+ allServers.stream()
+ .map(ITeamcityIgnited::serverIdToInt)
+ .forEach(srvId -> {
+ Integer borderForAgeForBuildId =
runHistCompactedDao.getBorderForAgeForBuildId(srvId, days);
+ if (borderForAgeForBuildId != null)
+ preBorder.put(srvId, borderForAgeForBuildId);
+ });
+
+ final int stateQueued = compactor.getStringId(BuildRef.STATE_QUEUED);
+
+ long minTs = System.currentTimeMillis() -
Duration.ofDays(days).toMillis();
+ QueryCursor<Cache.Entry<Long, BinaryObject>> query = cacheBin.query(
+ new ScanQuery<Long, BinaryObject>()
+ .setFilter((key, v) -> {
+ int srvId = BuildRefDao.cacheKeyToSrvId(key);
+ Integer buildIdBorder = preBorder.get(srvId);
+ if (buildIdBorder != null) {
+ int id = v.field("id");
+ if (id < buildIdBorder)
+ return false;// pre-filtered build out of
scope
+ }
+ int state = v.field("state");
+
+ return stateQueued != state;
+ }));
+
+ int cnt = 0;
+ List<Long> idsToCheck = new ArrayList<>();
+
+ try (QueryCursor<Cache.Entry<Long, BinaryObject>> cursor = query) {
+ for (Cache.Entry<Long, BinaryObject> next : cursor) {
+ Long key = next.getKey();
+ int srvId = BuildRefDao.cacheKeyToSrvId(key);
+
+ int buildId = BuildRefDao.cacheKeyToBuildId(key);
+
+ Integer borderBuildId =
runHistCompactedDao.getBorderForAgeForBuildId(srvId, days);
+
+ boolean passesDate = borderBuildId == null || buildId >=
borderBuildId;
+
+ if (!passesDate)
+ continue;
+
+ Long startTs = getBuildStartTime(srvId, buildId);
+ if (startTs == null || startTs < minTs)
+ continue; //time not saved in the DB, skip
+
+ System.err.println("Found build at srv [" + srvId + "]: [" +
buildId + "] to analyze, ts=" + startTs);
+
+ cnt++;
+
+ idsToCheck.add(key);
+ }
+ }
+
+ System.err.println("Total builds to load " + cnt);
+
+ return idsToCheck;
+ }
}
