This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch time-consumer-detection in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 72fb44196367f0e850d5d45dd67b50ad6aa39382 Author: Dmitriy Pavlov <[email protected]> AuthorDate: Fri Jul 5 21:47:52 2019 +0300 Build time collection callable: change scan flow to build ref instead of fat build --- .../org/apache/ignite/tcbot/common/TcBotConst.java | 3 + .../tcbot/engine/buildtime/BuildTimeService.java | 84 +++++++++++++++++++++- .../ignite/tcignited/TeamcityIgnitedImpl.java | 11 ++- .../apache/ignite/tcignited/build/FatBuildDao.java | 27 ++++--- .../ignite/tcignited/buildref/BuildRefDao.java | 64 ++++++++++++++++- .../ignite/tcignited/history/HistoryCollector.java | 32 ++++++++- .../tcignited/history/RunHistCompactedDao.java | 78 ++++++++++++++++++-- 7 files changed, 275 insertions(+), 24 deletions(-) diff --git a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/TcBotConst.java b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/TcBotConst.java index 3dd56ee..44433db 100644 --- a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/TcBotConst.java +++ b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/TcBotConst.java @@ -26,6 +26,9 @@ public class TcBotConst { /** Max days to keep test invocatoin data in run statistics: affects Bot Visa. */ public static final int HISTORY_MAX_DAYS = 21; + /** Absulte Max days to keep track about build existence, 42 */ + public static final int BUILD_MAX_DAYS = HISTORY_MAX_DAYS * 2; + /** History collection process: build id per server ID border days. */ public static final int HISTORY_BUILD_ID_BORDER_DAYS = HISTORY_MAX_DAYS + 2; diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java index 6b33379..95dca93 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java @@ -17,15 +17,26 @@ package org.apache.ignite.tcbot.engine.buildtime; -import java.util.List; +import java.time.Duration; +import java.util.Collection; +import javax.cache.Cache; import javax.inject.Inject; -import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.cache.query.QueryCursor; +import org.apache.ignite.cache.query.ScanQuery; import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; import org.apache.ignite.tcbot.engine.ui.BuildTimeSummaryUi; +import org.apache.ignite.tcbot.persistence.IStringCompactor; import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcignited.ITeamcityIgnitedProvider; import org.apache.ignite.tcignited.build.FatBuildDao; +import org.apache.ignite.tcignited.buildref.BuildRefDao; import org.apache.ignite.tcignited.creds.ICredentialsProv; +import org.apache.ignite.tcignited.history.HistoryCollector; +import org.apache.ignite.tcignited.history.RunHistCompactedDao; +import org.apache.ignite.tcservice.model.hist.BuildRef; +import org.apache.ignite.tcservice.model.result.stat.Statistics; public class BuildTimeService { @@ -36,13 +47,82 @@ public class BuildTimeService { @Inject FatBuildDao fatBuildDao; + @Inject BuildRefDao buildRefDao; + + @Inject RunHistCompactedDao runHistCompactedDao; + + @Inject IStringCompactor compactor; + + @Inject HistoryCollector historyCollector; + public BuildTimeSummaryUi analytics(ICredentialsProv prov) { String serverCode = cfg.primaryServerCode(); ITeamcityIgnited server = tcProv.server(serverCode, prov); + // fatBuildDao.forEachFatBuild(); + + Collection<String> allServers = cfg.getServerIds(); + + forEachBuildRef(1, allServers); + fatBuildDao.forEachFatBuild(); return null; } + + public void forEachBuildRef(int days, Collection<String> allServers) { + IgniteCache<Long, BinaryObject> cacheBin = buildRefDao.buildRefsCache().withKeepBinary(); + + // Ignite ignite = igniteProvider.get(); + + //IgniteCompute serversCompute = ignite.compute(ignite.cluster().forServers()); + + int stateRunning = compactor.getStringId(BuildRef.STATE_RUNNING); + final int stateQueued = compactor.getStringId(BuildRef.STATE_QUEUED); + Integer buildDurationId = compactor.getStringIdIfPresent(Statistics.BUILD_DURATION); + + long minTs = System.currentTimeMillis() - Duration.ofDays(days).toMillis(); + QueryCursor<Cache.Entry<Long, BinaryObject>> query = cacheBin.query( + new ScanQuery<Long, BinaryObject>() + .setFilter((k, v) -> { + + int state = v.field("state"); + + return stateQueued != state; + })); + + int cnt = 0; + + try (QueryCursor<Cache.Entry<Long, BinaryObject>> cursor = query) { + for (Cache.Entry<Long, BinaryObject> next : cursor) { + Long key = next.getKey(); + int srvId = BuildRefDao.cacheKeyToSrvId(key); + + int buildId = BuildRefDao.cacheKeyToBuildId(key); + + Integer borderBuildId = runHistCompactedDao.getBorderForAgeForBuildId(srvId, days); + + boolean passesDate = borderBuildId == null || buildId >= borderBuildId; + + if (!passesDate) + continue; + + Long startTs = historyCollector.getBuildStartTime(srvId, buildId); + if (startTs != null && startTs < minTs) + continue; //time not saved in the DB, skip + + BinaryObject buildBinary = next.getValue(); + long runningTime = 0l;// getBuildRunningTime(stateRunning, buildDurationId, buildBinary); + + System.err.println("Found build at srv [" + srvId + "]: [" + buildId + "] to analyze, ts="+ startTs); + + cnt++; + } + } + + System.err.println("Total builds to load " + cnt); + + // serversCompute.call(new BuildTimeIgniteCallable(cacheBin, stateRunning, buildDurationId)); + } } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java index 892b2e2..ce10502 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java @@ -569,12 +569,17 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { long ts = highBuild.getStartDateTs(); - return ts > 0 ? ts : null; + if (ts > 0) { + runHistCompactedDao.setBuildStartTime(srvIdMaskHigh, buildId, ts); + + return ts; + } else + return null; } - @GuavaCached(maximumSize = 100000, expireAfterAccessSecs = 90, softValues = true) + //@GuavaCached(maximumSize = 100000, expireAfterAccessSecs = 90, softValues = true) public Long getBuildStartTime(int buildId) { - return runHistCompactedDao.getBuildStartTime(srvIdMaskHigh, buildId); + return histCollector.getBuildStartTime(srvIdMaskHigh, buildId); } /** {@inheritDoc} */ diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java index 4bc074d..c8b46cf 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java @@ -44,7 +44,6 @@ import org.apache.ignite.cache.CacheEntryProcessor; import org.apache.ignite.cache.query.QueryCursor; import org.apache.ignite.cache.query.ScanQuery; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; -import org.apache.ignite.ci.teamcity.ignited.fatbuild.StatisticsCompacted; import org.apache.ignite.lang.IgniteCallable; import org.apache.ignite.resources.IgniteInstanceResource; import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; @@ -213,6 +212,17 @@ public class FatBuildDao { /** * @param srvId Server id. + * @param buildId Build Id. + */ + @Nullable public Long getBuildStartTime(int srvId, Integer buildId) { + IgniteCache<Long, BinaryObject> cacheBin = buildsCache.withKeepBinary(); + long key = buildIdToCacheKey(srvId, buildId); + + return cacheBin.invoke(key, new GetStartTimeProc()); + } + + /** + * @param srvId Server id. * @param ids Ids. */ public Map<Integer, Long> getBuildStartTime(int srvId, Set<Integer> ids) { @@ -244,10 +254,7 @@ public class FatBuildDao { int stateRunning = compactor.getStringId(BuildRef.STATE_RUNNING); Integer buildDurationId = compactor.getStringIdIfPresent(Statistics.BUILD_DURATION); - serversCompute.call(new BuiltTimeIgniteCallable(cacheBin, stateRunning, buildDurationId)); - - - + serversCompute.call(new BuiltTimeIgniteCallable(stateRunning, buildDurationId)); } public static long getBuildRunningTime(int stateRunning, Integer buildDurationId, @@ -312,24 +319,22 @@ public class FatBuildDao { } public static class BuiltTimeIgniteCallable implements IgniteCallable<Long> { - private final IgniteCache<Long, BinaryObject> cacheBin; private final int stateRunning; private final Integer buildDurationId; @IgniteInstanceResource Ignite ignite; - public BuiltTimeIgniteCallable(IgniteCache<Long, BinaryObject> cacheBin, int stateRunning, + public BuiltTimeIgniteCallable( int stateRunning, Integer buildDurationId) { - this.cacheBin = cacheBin; this.stateRunning = stateRunning; this.buildDurationId = buildDurationId; } @Override public Long call() throws Exception { - IgniteCache<Object, Object> cache = ignite.cache(TEAMCITY_FAT_BUILD_CACHE_NAME); + IgniteCache<Long, FatBuildCompacted> cache = ignite.cache(TEAMCITY_FAT_BUILD_CACHE_NAME); - IgniteCache<Object, Object> cacheBin = cache.withKeepBinary(); + IgniteCache<Long, BinaryObject> cacheBin = cache.withKeepBinary(); QueryCursor<Cache.Entry<Long, BinaryObject>> query = cacheBin.query( new ScanQuery<Long, BinaryObject>() .setLocal(true)); @@ -370,7 +375,7 @@ public class FatBuildDao { if (1 != 2) return null; - Iterable<Cache.Entry<Long, BinaryObject>> entries = this.cacheBin.localEntries(); + Iterable<Cache.Entry<Long, BinaryObject>> entries = cacheBin.localEntries(); for (Cache.Entry<Long, BinaryObject> next : entries) { Long srvAndBuild = next.getKey(); diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java index e8c8ae0..5cca271 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java @@ -36,13 +36,17 @@ import javax.inject.Inject; import javax.inject.Provider; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.cache.QueryEntity; import org.apache.ignite.cache.query.QueryCursor; +import org.apache.ignite.cache.query.ScanQuery; import org.apache.ignite.cache.query.SqlQuery; import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted; import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.util.GridIntList; +import org.apache.ignite.lang.IgniteCallable; +import org.apache.ignite.resources.IgniteInstanceResource; import org.apache.ignite.tcbot.common.exeption.ExceptionUtil; import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.common.interceptor.GuavaCached; @@ -104,7 +108,7 @@ public class BuildRefDao { * @param srvId Server id. */ public static boolean isKeyForServer(Long key, int srvId) { - return key!=null && key >> 32 == srvId; + return key!=null && cacheKeyToSrvId(key) == srvId; } /** @@ -169,6 +173,13 @@ public class BuildRefDao { } /** + * @param cacheKey Cache key. + */ + public static int cacheKeyToSrvId(long cacheKey) { + return (int)(cacheKey >> 32); + } + + /** * @param srvId Server id mask high. * @param buildTypeId Build type (suite) id. * @param bracnhNameQry Bracnh name query. @@ -307,4 +318,55 @@ public class BuildRefDao { return StreamSupport.stream(buildRefsCache.spliterator(), false) .filter(entry -> isKeyForServer(entry.getKey(), srvId)); } + + public IgniteCache<Long, BuildRefCompacted> buildRefsCache() { + return buildRefsCache; + } + + public static class BuildTimeIgniteCallable implements IgniteCallable<Long> { + private final int stateRunning; + private final Integer buildDurationId; + @IgniteInstanceResource + Ignite ignite; + + public BuildTimeIgniteCallable(IgniteCache<Long, BinaryObject> cacheBin, int stateRunning, + Integer buildDurationId) { + this.stateRunning = stateRunning; + this.buildDurationId = buildDurationId; + } + + @Override public Long call() throws Exception { + IgniteCache<Long, BuildRefCompacted> cache = ignite.cache(TEAMCITY_BUILD_CACHE_NAME); + + IgniteCache<Long, BuildRefCompacted>cacheBin = cache.withKeepBinary(); + QueryCursor<Cache.Entry<Long, BinaryObject>> query = cacheBin.query( + new ScanQuery<Long, BinaryObject>() + .setFilter((k,v)->{ + int srvId = cacheKeyToSrvId(k); + +return false; + }) + .setLocal(true)); + + /*.query(new SqlQuery<Long, BinaryObject>( + FatBuildCompacted.class, + " _KEY > ?") + .setLocal(true) + .setArgs(0L));*/ + +// Iterate over the result set. + try (QueryCursor<Cache.Entry<Long, BinaryObject>> cursor = query) { + + for (Cache.Entry<Long, BinaryObject> next : cursor) { + + BinaryObject buildBinary = next.getValue(); + long runningTime = 0l;// getBuildRunningTime(stateRunning, buildDurationId, buildBinary); + + System.err.println("Running " + runningTime); + } + } + + return null; + } + } } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java index e9ca205..de43b6a 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java @@ -16,7 +16,6 @@ */ package org.apache.ignite.tcignited.history; -import com.google.common.base.Preconditions; import com.google.common.cache.CacheBuilder; import com.google.common.collect.Iterables; import java.time.Duration; @@ -143,9 +142,18 @@ public class HistoryCollector { long curTs = System.currentTimeMillis(); Set<Integer> buildIds = bRefsList.stream() .filter(b -> { - AtomicInteger biggestIdOutOfScope = biggestBuildIdOutOfHistoryScope.get(srvId); + /* AtomicInteger biggestIdOutOfScope = biggestBuildIdOutOfHistoryScope.get(srvId); int outOfScopeBuildId = biggestIdOutOfScope == null ? -1 : biggestIdOutOfScope.get(); return b.id() > outOfScopeBuildId; + */ + + Integer maxBuildIdForDay = runHistCompactedDao.getBorderForAgeForBuildId(srvId, TcBotConst.HISTORY_BUILD_ID_BORDER_DAYS); + + if (maxBuildIdForDay == null) + return true; + + return b.id()>maxBuildIdForDay; + }) .filter(this::applicableForHistory) .map(BuildRefCompacted::id) @@ -225,7 +233,7 @@ public class HistoryCollector { int normalizedBaseBranch) { Map<Integer, SuiteInvocation> suiteRunHist = histDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch); - System.out.println("***** Found history for suite " + logger.info("***** Found history for suite " + compactor.getStringFromId(buildTypeId) + " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": " + suiteRunHist.size() ); @@ -325,4 +333,22 @@ public class HistoryCollector { return getSuiteHist(srvId, buildTypeId, normalizedBaseBranch); } + + /** + * @param srvId Server id. + * @param buildId Build id. + */ + public Long getBuildStartTime(int srvId, int buildId) { + Long time = runHistCompactedDao.getBuildStartTime(srvId, buildId); + + if (time != null) + return time; + + time = fatBuildDao.getBuildStartTime(srvId, buildId); + + if (time != null) + runHistCompactedDao.setBuildStartTime(srvId, buildId, time); + + return time; + } } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java index 5a18465..aa49966 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java @@ -17,6 +17,7 @@ package org.apache.ignite.tcignited.history; +import java.time.Duration; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -24,7 +25,10 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerArray; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -42,6 +46,7 @@ import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation; import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted; import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey; import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.tcbot.common.TcBotConst; import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.common.interceptor.GuavaCached; import org.apache.ignite.tcbot.persistence.CacheConfigs; @@ -51,7 +56,7 @@ import org.apache.ignite.tcignited.buildref.BuildRefDao; import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch; /** - * + * TODO: rename to build start time storage */ public class RunHistCompactedDao { /** Cache name. */ @@ -78,6 +83,17 @@ public class RunHistCompactedDao { /** Build start time. */ private IgniteCache<Long, Long> buildStartTime; + /** + * Biggest build ID, which is older than particular days count. + * Map: server ID-> Array of build Ids + * Array[0] = max build ID + * Array[1] = max build ID older than 1 day + */ + private final ConcurrentMap<Integer, AtomicIntegerArray> maxBuildIdOlderThanDays = new ConcurrentHashMap<>(); + + /** Millis in day. */ + private static final long MILLIS_IN_DAY = Duration.ofDays(1).toMillis(); + /** Compactor. */ @Inject private IStringCompactor compactor; @@ -146,14 +162,27 @@ public class RunHistCompactedDao { if (ts == null || ts <= 0) return null; + processBuildForBorder(srvId, buildId, ts); + return ts; } + public boolean setBuildStartTime(int srvId, int buildId, long ts) { + if (ts <= 0) + return false; + + processBuildForBorder(srvId, buildId, ts); + + return buildStartTime.putIfAbsent(buildIdToCacheKey(srvId, buildId), ts); + } + @AutoProfiling public boolean setBuildProcessed(int srvId, int buildId, long ts) { if (ts <= 0) return false; + processBuildForBorder(srvId, buildId, ts); + return buildStartTime.putIfAbsent(buildIdToCacheKey(srvId, buildId), ts); } @@ -263,8 +292,13 @@ public class RunHistCompactedDao { Map<Integer, Long> res = new HashMap<>(); buildStartTime.getAll(cacheKeys).forEach((k, ts) -> { - if (ts != null && ts > 0) - res.put(BuildRefDao.cacheKeyToBuildId(k), ts); + if (ts != null && ts > 0) { + int buildId = BuildRefDao.cacheKeyToBuildId(k); + + res.put(buildId, ts); + + processBuildForBorder(srvId, buildId, ts); + } }); return res; @@ -274,10 +308,46 @@ public class RunHistCompactedDao { Map<Long, Long> res = new HashMap<>(); builds.forEach((buildId, ts) -> { - if (ts != null && ts > 0) + if (ts != null && ts > 0) { res.put(buildIdToCacheKey(srvId, buildId), ts); + + processBuildForBorder(srvId, buildId, ts); + } }); buildStartTime.putAll(res); } + + private void processBuildForBorder(int srvId, Integer buildId, Long ts) { + if (ts == null || ts <= 0) + return; + + AtomicIntegerArray arr = maxBuildIdOlderThanDays.computeIfAbsent(srvId, + k -> new AtomicIntegerArray(TcBotConst.BUILD_MAX_DAYS + 1)); + + long ageMs = System.currentTimeMillis() - ts; + if (ageMs < 0) + return; + + long days = ageMs / MILLIS_IN_DAY; + if (days > TcBotConst.BUILD_MAX_DAYS) + days = TcBotConst.BUILD_MAX_DAYS; + + arr.accumulateAndGet((int)days, buildId, Math::max); + } + + @Nullable public Integer getBorderForAgeForBuildId(int srvId, int ageDays) { + AtomicIntegerArray arr = maxBuildIdOlderThanDays.get(srvId); + if (arr == null) + return null; + + for (int i = ageDays; i < TcBotConst.BUILD_MAX_DAYS; i++) { + int buildId = arr.get(i); + if (buildId != 0) + return buildId; + } + + return null; + } + }
