This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push: new 47f23e7 Trusted tests & suite history performance fixes: Storage into persisted cache + TTL (#129) 47f23e7 is described below commit 47f23e711abf43de86afcf8a9e4a7150988cb94a Author: Dmitriy Pavlov <dpav...@apache.org> AuthorDate: Tue Jun 25 17:49:07 2019 +0300 Trusted tests & suite history performance fixes: Storage into persisted cache + TTL (#129) --- build.gradle | 5 +- ignite-tc-helper-web/build.gradle | 4 - .../ignite/ci/tcbot/issue/IssueDetector.java | 17 +- .../org/apache/ignite/ci/web/model/Version.java | 2 +- .../ci/teamcity/ignited/TeamcityIgnitedMock.java | 46 ++- jetty-launcher/build.gradle | 4 - .../tcbot/engine/chain/BuildChainProcessor.java | 50 ++-- .../ignite/tcbot/engine/chain/MultBuildRunCtx.java | 83 ++++-- .../tcbot/engine/chain/SingleBuildRunCtx.java | 4 + .../tcbot/engine/chain/TestCompactedMult.java | 25 +- .../ignite/tcbot/engine/pr/PrChainsProcessor.java | 8 +- .../tracked/TrackedBranchChainsProcessor.java | 7 +- .../apache/ignite/tcbot/engine/ui/DsChainUi.java | 15 +- .../apache/ignite/tcbot/engine/ui/DsSuiteUi.java | 62 ++-- .../ignite/tcbot/engine/ui/DsTestFailureUi.java | 39 ++- .../ignite/tcbot/engine/ui/DsTestHistoryUi.java | 2 +- tcbot-server-node/build.gradle | 3 - .../apache/ignite/tcignited/ITeamcityIgnited.java | 10 +- .../ignite/tcignited/TeamcityIgnitedImpl.java | 83 +++--- .../ignite/tcignited/TeamcityIgnitedModule.java | 4 + .../apache/ignite/tcignited/build/FatBuildDao.java | 155 +++------- .../tcignited/build/ProactiveFatBuildSync.java | 3 +- .../ignite/tcignited/build/SuiteHistory.java | 39 ++- .../BranchEquivalence.java} | 30 +- .../ignite/tcignited/buildref/BuildRefDao.java | 2 +- .../ignite/tcignited/history/HistoryCollector.java | 326 +++++++++++++++++++++ .../ISuiteRunHistory.java} | 21 +- .../tcignited/history/RunHistCompactedDao.java | 36 +++ .../ignite/tcignited/history/RunHistSync.java | 2 + .../ignite/tcignited/history/SuiteInvocation.java | 77 +++++ .../history/SuiteInvocationHistoryDao.java | 87 ++++++ 31 files changed, 919 insertions(+), 332 deletions(-) diff --git a/build.gradle b/build.gradle index edf7b84..43f7f5a 100644 --- a/build.gradle +++ b/build.gradle @@ -38,10 +38,11 @@ subprojects { } allprojects { + apply plugin: 'java' + sourceCompatibility = '1.8' + targetCompatibility = '1.8' ext { - sourceCompatibility = '1.8' - targetCompatibility = '1.8' jettyVer = '9.4.12.v20180830' diff --git a/ignite-tc-helper-web/build.gradle b/ignite-tc-helper-web/build.gradle index a9cb41f..e4fa011 100644 --- a/ignite-tc-helper-web/build.gradle +++ b/ignite-tc-helper-web/build.gradle @@ -18,10 +18,6 @@ apply plugin: 'java' apply plugin: 'war' -sourceCompatibility = '1.8' -targetCompatibility = '1.8' - - // https://www.apache.org/legal/resolved.html#category-a dependencies { compile (project(":tcbot-common")); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java index e3800a7..62842f0 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java @@ -261,13 +261,14 @@ public class IssueDetector { final String trackedBranch = res.getTrackedBranch(); + String suiteId = suiteCurrentStatus.suiteId; for (DsTestFailureUi testFailure : suiteCurrentStatus.testFailures) { - if (registerTestFailIssues(tcIgnited, srvCode, normalizeBranch, testFailure, trackedBranch, + if (registerTestFailIssues(tcIgnited, srvCode, suiteId, normalizeBranch, testFailure, trackedBranch, suiteCurrentStatus.tags)) newIssues++; } - if (registerSuiteFailIssues(tcIgnited, srvCode, normalizeBranch, suiteCurrentStatus, trackedBranch)) + if (registerSuiteFailIssues(tcIgnited, srvCode, suiteId, normalizeBranch, suiteCurrentStatus, trackedBranch)) newIssues++; } } @@ -286,13 +287,15 @@ public class IssueDetector { */ private boolean registerSuiteFailIssues(ITeamcityIgnited tcIgnited, String srvCode, + String suiteId, String normalizeBranch, DsSuiteUi suiteFailure, String trackedBranch) { - String suiteId = suiteFailure.suiteId; + Integer btId = compactor.getStringIdIfPresent(suiteId); + Integer brNormId = compactor.getStringIdIfPresent(normalizeBranch); - IRunHistory runStat = tcIgnited.getSuiteRunHist(suiteId, normalizeBranch); + IRunHistory runStat = tcIgnited.getSuiteRunHist(btId, brNormId).self(); if (runStat == null) return false; @@ -361,13 +364,17 @@ public class IssueDetector { private boolean registerTestFailIssues(ITeamcityIgnited tcIgnited, String srvCode, + String suiteId, String normalizeBranch, DsTestFailureUi testFailure, String trackedBranch, @Nonnull Set<String> suiteTags) { String name = testFailure.name; + int tname = compactor.getStringId(name); + Integer btId = compactor.getStringIdIfPresent(suiteId); + Integer brNormId = compactor.getStringIdIfPresent(normalizeBranch); - IRunHistory runStat = tcIgnited.getTestRunHist(name, normalizeBranch); + IRunHistory runStat = tcIgnited.getTestRunHist(tname, btId, brNormId); if (runStat == null) return false; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java index dcc646e..c1bd22d 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java @@ -28,7 +28,7 @@ package org.apache.ignite.ci.web.model; public static final String GITHUB_REF = "https://github.com/apache/ignite-teamcity-bot"; /** TC Bot Version. */ - public static final String VERSION = "20190613"; + public static final String VERSION = "20190621"; /** Java version, where Web App is running. */ public String javaVer; diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java index 4f9eacd..9a0d174 100644 --- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java +++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java @@ -31,6 +31,7 @@ import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey; import org.apache.ignite.tcbot.persistence.IStringCompactor; import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcignited.SyncMode; +import org.apache.ignite.tcignited.history.ISuiteRunHistory; import org.apache.ignite.tcservice.model.result.tests.TestOccurrence; import org.jetbrains.annotations.NotNull; import org.mockito.Mockito; @@ -90,10 +91,10 @@ public class TeamcityIgnitedMock { .collect(Collectors.toList()); }); - when(tcIgnited.getTestRunHist(anyString(), anyString())) + when(tcIgnited.getTestRunHist(anyInt(), anyInt(), anyInt())) .thenAnswer((inv) -> { - final String name = inv.getArgument(0); - final String branch = inv.getArgument(1); + final Integer tstName = inv.getArgument(0); + final Integer branchId = inv.getArgument(2); // System.out.println("Search history " + name + " in " + branch + ": " ); if (histCache.isEmpty()) { @@ -103,11 +104,9 @@ public class TeamcityIgnitedMock { } } - final Integer tstName = c.getStringIdIfPresent(name); if (tstName == null) return null; - final Integer branchId = c.getStringIdIfPresent(branch); if (branchId == null) return null; @@ -115,11 +114,46 @@ public class TeamcityIgnitedMock { final RunHistCompacted runHistCompacted = histCache.get(key); - System.out.println("Test history " + name + " in " + branch + " => " + runHistCompacted); + System.out.println("Test history " + c.getStringFromId(tstName) + " in " + c.getStringFromId(branchId) + " => " + runHistCompacted); return runHistCompacted; }); + when(tcIgnited.getSuiteRunHist(anyInt(), anyInt())) + .thenAnswer((inv) -> { + final Integer suiteName = inv.getArgument(0); + final Integer branchId = inv.getArgument(1); + // System.out.println("Search history " + name + " in " + branch + ": " ); + if (histCache.isEmpty()) { + synchronized (histCache) { + if (histCache.isEmpty()) + initHistory(c, histCache, builds, srvId); + } + } + + ISuiteRunHistory mock = Mockito.mock(ISuiteRunHistory.class); + + when(mock.getTestRunHist(anyInt())).thenAnswer((inv2)-> { + final Integer tstName = inv2.getArgument(0); + + if (tstName == null) + return null; + + if (branchId == null) + return null; + + final RunHistKey key = new RunHistKey(srvId, tstName, branchId); + + final RunHistCompacted runHistCompacted = histCache.get(key); + + System.out.println("Test history " + c.getStringFromId(tstName) + " in " + c.getStringFromId(branchId) + " => " + runHistCompacted); + + return runHistCompacted; + }); + + return mock; + }); + // when(tcIgnited.gitBranchPrefix()).thenReturn("ignite-"); ITcServerConfig mock = mock(ITcServerConfig.class); diff --git a/jetty-launcher/build.gradle b/jetty-launcher/build.gradle index edfb84d..5df3d7b 100644 --- a/jetty-launcher/build.gradle +++ b/jetty-launcher/build.gradle @@ -18,10 +18,6 @@ apply plugin: 'java' apply plugin: 'application' -sourceCompatibility = '1.8' -targetCompatibility = '1.8' - - mainClassName = 'org.apache.ignite.ci.TcHelperJettyLauncher' applicationDefaultJvmArgs = ["-Dteamcity.helper.home=../work", "-Dteamcity.bot.regionsize=16", // 16g Durable Memory region diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java index 62bb209..579dbb0 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java @@ -19,12 +19,32 @@ package org.apache.ignite.tcbot.engine.chain; import com.google.common.base.Preconditions; import com.google.common.util.concurrent.Futures; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.inject.Inject; import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted; import org.apache.ignite.ci.teamcity.ignited.buildtype.ParametersCompacted; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; +import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.common.util.FutureUtil; import org.apache.ignite.tcbot.engine.pool.TcUpdatePool; -import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.engine.ui.LrTestUi; import org.apache.ignite.tcbot.engine.ui.LrTestsSuiteSummaryUi; import org.apache.ignite.tcbot.persistence.IStringCompactor; @@ -38,18 +58,6 @@ import org.apache.ignite.tcservice.model.result.Build; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import javax.inject.Inject; -import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Future; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; - /** * Process whole Build Chain, E.g. runAll at particular server, including all builds involved */ @@ -144,6 +152,8 @@ public class BuildChainProcessor { if (entryPoints.isEmpty()) return new FullChainRunCtx(Build.createFakeStub()); + Integer failRateBranchId = compactor.getStringIdIfPresent(RunHistSync.normalizeBranch(failRateBranch)); + Map<Integer, Future<FatBuildCompacted>> builds = loadAllBuildsInChains(entryPoints, mode, tcIgn); Map<String, List<Future<FatBuildCompacted>>> freshRebuilds = new ConcurrentHashMap<>(); @@ -177,6 +187,11 @@ public class BuildChainProcessor { buildsForSuite.forEach(buildCompacted -> ctx.addBuild(loadChanges(buildCompacted, tcIgn))); + //ask for history for the suite in parallel + tcUpdatePool.getService().submit(() -> { + ctx.history(tcIgn, failRateBranchId); + }); + analyzeTests(ctx, tcIgn, procLog); fillBuildCounts(ctx, tcIgn, includeScheduledInfo); @@ -185,9 +200,7 @@ public class BuildChainProcessor { }); Function<MultBuildRunCtx, Float> function = ctx -> { - - //todo cache RunStat instance into suite context to compare - IRunHistory runStat = tcIgn.getSuiteRunHist(ctx.suiteId(), RunHistSync.normalizeBranch(failRateBranch)); + IRunHistory runStat = ctx.history(tcIgn, failRateBranchId); if (runStat == null) return 0f; @@ -230,7 +243,8 @@ public class BuildChainProcessor { .peek(val -> Preconditions.checkNotNull(val, "Build future should be in context")) .flatMap(ref -> dependencies(ref, mode, builds, tcIgn).stream()).collect(Collectors.toSet()); - logger.info("Level [" + level + "] dependencies:" + depsNextLevel); + if(logger.isDebugEnabled()) + logger.debug("Level [" + level + "] dependencies:" + depsNextLevel); remainedUnloadedDeps = depsNextLevel; } @@ -330,7 +344,7 @@ public class BuildChainProcessor { protected void fillBuildCounts(MultBuildRunCtx outCtx, ITeamcityIgnited teamcityIgnited, boolean includeScheduledInfo) { if (includeScheduledInfo && !outCtx.hasScheduledBuildsInfo()) { - final List<BuildRefCompacted> runAllBuilds = teamcityIgnited.getAllBuildsCompacted(outCtx.buildTypeId(), outCtx.branchName()); + List<BuildRefCompacted> runAllBuilds = teamcityIgnited.getAllBuildsCompacted(outCtx.suiteId(), outCtx.branchName()); long cntRunning = runAllBuilds .stream() diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java index a794110..f965db3 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java @@ -17,7 +17,9 @@ package org.apache.ignite.tcbot.engine.chain; +import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.common.cache.CacheBuilder; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; @@ -28,23 +30,25 @@ import java.util.Optional; import java.util.OptionalDouble; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; - import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted; import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted; import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted; import org.apache.ignite.tcbot.common.conf.ITcServerConfig; +import org.apache.ignite.tcbot.common.exeption.ExceptionUtil; import org.apache.ignite.tcbot.common.util.CollectionUtil; import org.apache.ignite.tcbot.persistence.IStringCompactor; import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcignited.buildlog.ILogCheckResult; import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult; import org.apache.ignite.tcignited.history.IRunHistory; +import org.apache.ignite.tcignited.history.ISuiteRunHistory; import org.apache.ignite.tcservice.model.hist.BuildRef; import org.apache.ignite.tcservice.model.result.problems.ProblemOccurrence; import org.apache.ignite.tcservice.model.result.stat.Statistics; @@ -67,6 +71,9 @@ public class MultBuildRunCtx implements ISuiteResults { /** Builds: Single execution. */ private List<SingleBuildRunCtx> builds = new CopyOnWriteArrayList<>(); + private final com.google.common.cache.Cache<Integer, Optional<ISuiteRunHistory>> historyCacheMap + = CacheBuilder.newBuilder().build(); + public void addBuild(SingleBuildRunCtx ctx) { builds.add(ctx); } @@ -98,8 +105,9 @@ public class MultBuildRunCtx implements ISuiteResults { return buildsStream().map(SingleBuildRunCtx::getTestLogCheckResult).filter(Objects::nonNull); } - public String suiteId() { - return firstBuildInfo.suiteId(); + /** {@inheritDoc} */ + @Override public String suiteId() { + return firstBuild().map(SingleBuildRunCtx::suiteId).orElse(null); } /** {@inheritDoc} */ @@ -107,9 +115,6 @@ public class MultBuildRunCtx implements ISuiteResults { return getBuildMessageProblemCount() > 0; } - public String buildTypeId() { - return firstBuildInfo.buildTypeId; - } public boolean hasAnyBuildProblemExceptTestOrSnapshot() { return allProblemsInAllBuilds() @@ -170,6 +175,7 @@ public class MultBuildRunCtx implements ISuiteResults { return buildsStream().filter(ISuiteResults::hasCompilationProblem).count(); } + /** {@inheritDoc} */ public boolean hasTimeoutProblem() { return getExecutionTimeoutCount() > 0; } @@ -178,6 +184,7 @@ public class MultBuildRunCtx implements ISuiteResults { return buildsStream().filter(SingleBuildRunCtx::hasTimeoutProblem).count(); } + /** {@inheritDoc} */ public boolean hasJvmCrashProblem() { return getJvmCrashProblemCount() > 0; } @@ -186,10 +193,12 @@ public class MultBuildRunCtx implements ISuiteResults { return buildsCntHavingBuildProblem(ProblemOccurrence.TC_JVM_CRASH); } + /** {@inheritDoc} */ public boolean hasOomeProblem() { return getOomeProblemCount() > 0; } + /** {@inheritDoc} */ @Override public boolean hasExitCodeProblem() { return getExitCodeProblemsCount() > 0; } @@ -303,8 +312,8 @@ public class MultBuildRunCtx implements ISuiteResults { return CollectionUtil.top(logSizeBytes.entrySet().stream(), 3, comparing).stream(); } - public Stream<? extends IMultTestOccurrence> getTopLongRunning() { - Comparator<IMultTestOccurrence> comparing = Comparator.comparing(IMultTestOccurrence::getAvgDurationMs); + public Stream<TestCompactedMult> getTopLongRunning() { + Comparator<TestCompactedMult> comparing = Comparator.comparing(TestCompactedMult::getAvgDurationMs); Map<Integer, TestCompactedMult> res = new HashMap<>(); @@ -315,7 +324,7 @@ public class MultBuildRunCtx implements ISuiteResults { return CollectionUtil.top(res.values().stream(), 3, comparing).stream(); } - public List<IMultTestOccurrence> getFailedTests() { + public List<TestCompactedMult> getFailedTests() { Map<Integer, TestCompactedMult> res = new HashMap<>(); builds.forEach(singleBuildRunCtx -> { @@ -327,7 +336,7 @@ public class MultBuildRunCtx implements ISuiteResults { public void saveToMap(Map<Integer, TestCompactedMult> res, Stream<TestCompacted> tests) { tests.forEach(testCompacted -> { - res.computeIfAbsent(testCompacted.testName(), k -> new TestCompactedMult(compactor)) + res.computeIfAbsent(testCompacted.testName(), k -> new TestCompactedMult(compactor, this)) .add(testCompacted); }); } @@ -607,17 +616,13 @@ public class MultBuildRunCtx implements ISuiteResults { //todo can cache mult occurrences in ctx builds.forEach(singleBuildRunCtx -> { - saveToMap(res, singleBuildRunCtx.getAllTests() - .filter(t -> !t.isIgnoredTest() && !t.isMutedTest())); + saveToMap(res, + singleBuildRunCtx.getAllTests().filter(t -> !t.isIgnoredTest() && !t.isMutedTest())); }); Integer branchName = compactor.getStringIdIfPresent(normalizedBaseBranch); - Integer suiteName = compactor.getStringIdIfPresent( buildTypeId()); - - // res.clear(); //todo enable feature back - //todo can cache fail rate in mult occur - res.keySet().forEach((testNameId) -> { - IRunHistory stat = tcIgnited.getTestRunHist(testNameId, suiteName, branchName); + res.forEach((testNameId, compactedMult) -> { + IRunHistory stat = compactedMult.history(tcIgnited, branchName); String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat); boolean b = testBlockerComment != null; if (b) // this test will be considered as blocker if will fail @@ -627,4 +632,46 @@ public class MultBuildRunCtx implements ISuiteResults { return trustedCnt.get(); } + /** + * Returns suite name non compacted. + */ + public Integer buildTypeIdId() { + return firstBuild().map(SingleBuildRunCtx::buildTypeIdId).orElse(null); + } + + public Optional<SingleBuildRunCtx> firstBuild() { + return builds.stream().findFirst(); + } + + /** + * @param tcIgn Tc ign. + * @param baseBranchId Base branch id. + */ + public IRunHistory history(ITeamcityIgnited tcIgn, Integer baseBranchId) { + if (baseBranchId == null) + return null; + + ISuiteRunHistory suiteHist = suiteHist(tcIgn, baseBranchId); + if (suiteHist == null) + return null; + + return suiteHist.self(); + } + + @Nullable + ISuiteRunHistory suiteHist(ITeamcityIgnited tcIgn, Integer baseBranchId) { + Integer buildTypeIdId = buildTypeIdId(); + Preconditions.checkNotNull(buildTypeIdId, "Build type ID should be filled"); + + try { + return historyCacheMap.get(baseBranchId, + () -> { + return Optional.ofNullable(tcIgn.getSuiteRunHist(buildTypeIdId, baseBranchId)); + }) + .orElse(null); + } + catch (ExecutionException e) { + throw ExceptionUtil.propagateException(e); + } + } } diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java index dab646e..e41d65f 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java @@ -346,4 +346,8 @@ public class SingleBuildRunCtx implements ISuiteResults { public int totalNotMutedTests() { return buildCompacted.totalNotMutedTests(); } + + public int buildTypeIdId() { + return buildCompacted.buildTypeId(); + } } diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java index 908bc5f..7d5d3c3 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java @@ -21,10 +21,14 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import javax.annotation.Nullable; import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted; import org.apache.ignite.tcbot.persistence.IStringCompactor; +import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcignited.history.IRunHistSummary; +import org.apache.ignite.tcignited.history.IRunHistory; import org.apache.ignite.tcignited.history.IRunStat; +import org.apache.ignite.tcignited.history.ISuiteRunHistory; import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull; /** @@ -33,12 +37,17 @@ import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull; public class TestCompactedMult implements IMultTestOccurrence { private final List<TestCompacted> occurrences = new ArrayList<>(); private IStringCompactor compactor; + private MultBuildRunCtx ctx; private long avgDuration = -1; - public TestCompactedMult(IStringCompactor compactor) { + public TestCompactedMult(IStringCompactor compactor, MultBuildRunCtx ctx) { this.compactor = compactor; + this.ctx = ctx; } + @Nullable public Integer testName() { + return occurrences.isEmpty() ? null : occurrences.iterator().next().testName(); + } /** {@inheritDoc} */ @Override public String getName() { return occurrences.isEmpty() ? "" : occurrences.iterator().next().testName(compactor); @@ -109,4 +118,18 @@ public class TestCompactedMult implements IMultTestOccurrence { public void add(TestCompacted next) { occurrences.add(next); } + + + public IRunHistory history(ITeamcityIgnited ignited, Integer baseBranchId) { + Integer name = testName(); + if (name == null) + return null; + + ISuiteRunHistory suiteRunHist = ctx.suiteHist(ignited, baseBranchId); + + if (suiteRunHist == null) + return null; + + return suiteRunHist.getTestRunHist(name); + } } diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java index 7d4b4d3..5a9246c 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java @@ -258,19 +258,19 @@ public class PrChainsProcessor { .failedChildSuites() .map((ctx) -> { String normalizedBaseBranch = RunHistSync.normalizeBranch(baseBranch); - IRunHistory statInBaseBranch = tcIgnited.getSuiteRunHist(ctx.suiteId(), normalizedBaseBranch); + Integer baseBranchId = compactor.getStringIdIfPresent(normalizedBaseBranch); + IRunHistory statInBaseBranch = ctx.history(tcIgnited, baseBranchId); String suiteComment = ctx.getPossibleBlockerComment(compactor, statInBaseBranch, tcIgnited.config()); List<DsTestFailureUi> failures = ctx.getFailedTests().stream().map(occurrence -> { - IRunHistory stat = tcIgnited.getTestRunHist(occurrence.getName(), normalizedBaseBranch); - + IRunHistory stat = occurrence.history(tcIgnited, baseBranchId); String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat); if (!Strings.isNullOrEmpty(testBlockerComment)) { final DsTestFailureUi failure = new DsTestFailureUi(); - failure.initFromOccurrence(occurrence, tcIgnited, ctx.projectId(), ctx.branchName(), baseBranch); + failure.initFromOccurrence(occurrence, tcIgnited, ctx.projectId(), ctx.branchName(), baseBranch, baseBranchId); return failure; } diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java index bda0b02..a0b40af 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java @@ -22,18 +22,17 @@ import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.inject.Inject; - import org.apache.ignite.tcbot.common.conf.ITcServerConfig; +import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.engine.chain.BuildChainProcessor; import org.apache.ignite.tcbot.engine.chain.FullChainRunCtx; import org.apache.ignite.tcbot.engine.chain.LatestRebuildMode; import org.apache.ignite.tcbot.engine.chain.ProcessLogsMode; -import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; import org.apache.ignite.tcbot.engine.conf.ITrackedBranch; import org.apache.ignite.tcbot.engine.ui.DsChainUi; -import org.apache.ignite.tcbot.engine.ui.LrTestsFullSummaryUi; import org.apache.ignite.tcbot.engine.ui.DsSummaryUi; +import org.apache.ignite.tcbot.engine.ui.LrTestsFullSummaryUi; import org.apache.ignite.tcbot.persistence.IStringCompactor; import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcignited.ITeamcityIgnitedProvider; @@ -107,7 +106,7 @@ public class TrackedBranchChainsProcessor { boolean includeScheduled = buildResMergeCnt == 1; final FullChainRunCtx ctx = chainProc.loadFullChainContext( - tcIgnited, + tcIgnited, chains, rebuild, logs, diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java index 913c08a..d8c8746 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java @@ -24,13 +24,12 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Stream; import javax.annotation.Nullable; - +import org.apache.ignite.internal.util.typedef.T2; +import org.apache.ignite.tcbot.common.util.CollectionUtil; import org.apache.ignite.tcbot.common.util.UrlUtil; import org.apache.ignite.tcbot.engine.chain.FullChainRunCtx; -import org.apache.ignite.tcbot.engine.chain.IMultTestOccurrence; import org.apache.ignite.tcbot.engine.chain.MultBuildRunCtx; -import org.apache.ignite.tcbot.common.util.CollectionUtil; -import org.apache.ignite.internal.util.typedef.T2; +import org.apache.ignite.tcbot.engine.chain.TestCompactedMult; import org.apache.ignite.tcbot.persistence.IStringCompactor; import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcservice.model.conf.BuildType; @@ -208,18 +207,18 @@ public class DsChainUi { webToHist = buildWebLink(tcIgnited, ctx); webToBuild = buildWebLinkToBuild(tcIgnited, ctx); - Stream<T2<MultBuildRunCtx, IMultTestOccurrence>> allLongRunning = ctx.suites().flatMap( + Stream<T2<MultBuildRunCtx, TestCompactedMult>> allLongRunning = ctx.suites().flatMap( suite -> suite.getTopLongRunning().map(t -> new T2<>(suite, t)) ); - Comparator<T2<MultBuildRunCtx, IMultTestOccurrence>> durationComp + Comparator<T2<MultBuildRunCtx, TestCompactedMult>> durationComp = Comparator.comparing((pair) -> pair.get2().getAvgDurationMs()); CollectionUtil.top(allLongRunning, 3, durationComp).forEach( pairCtxAndOccur -> { MultBuildRunCtx suite = pairCtxAndOccur.get1(); - IMultTestOccurrence longRunningOccur = pairCtxAndOccur.get2(); + TestCompactedMult longRunningOccur = pairCtxAndOccur.get2(); - DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, suite, longRunningOccur, baseBranchTc); + DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, compactor, suite, longRunningOccur, baseBranchTc); failure.testName = "[" + suite.suiteName() + "] " + failure.testName; //may be separate field diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java index 8d25cf2..eea9212 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java @@ -29,19 +29,18 @@ import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; - import org.apache.ignite.tcbot.common.util.UrlUtil; -import org.apache.ignite.tcbot.engine.chain.IMultTestOccurrence; import org.apache.ignite.tcbot.engine.chain.MultBuildRunCtx; +import org.apache.ignite.tcbot.engine.chain.TestCompactedMult; import org.apache.ignite.tcbot.engine.issue.EventTemplates; import org.apache.ignite.tcbot.engine.ui.BotUrls.GetBuildLog; -import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult; -import org.apache.ignite.tcignited.history.IRunHistory; import org.apache.ignite.tcbot.persistence.IStringCompactor; import org.apache.ignite.tcignited.ITeamcityIgnited; +import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult; +import org.apache.ignite.tcignited.history.IRunHistory; -import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch; import static org.apache.ignite.tcbot.common.util.TimeUtil.millisToDurationPrintable; +import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch; /** @@ -164,9 +163,12 @@ public class DsSuiteUi extends DsHistoryStatUi { name = suite.suiteName(); String failRateNormalizedBranch = normalizeBranch(baseBranch); + Integer baseBranchId = compactor.getStringIdIfPresent(failRateNormalizedBranch); + String curBranchNormalized = normalizeBranch(suite.branchName()); + Integer curBranchId = compactor.getStringIdIfPresent(curBranchNormalized); - IRunHistory baseBranchHist = initSuiteStat(tcIgnited, failRateNormalizedBranch, curBranchNormalized, suite.suiteId()); + IRunHistory baseBranchHist = initSuiteStat(tcIgnited, baseBranchId, curBranchId, suite); Set<String> collect = suite.lastChangeUsers().collect(Collectors.toSet()); @@ -186,26 +188,28 @@ public class DsSuiteUi extends DsHistoryStatUi { webToHistBaseBranch = buildWebLink(tcIgnited, suite, baseBranch); webToBuild = buildWebLinkToBuild(tcIgnited, suite); + Integer buildTypeIdId = suite.buildTypeIdId(); if (includeTests) { - List<IMultTestOccurrence> tests = suite.getFailedTests(); - Function<IMultTestOccurrence, Float> function = foccur -> { - IRunHistory apply = tcIgnited.getTestRunHist(foccur.getName(), failRateNormalizedBranch); + List<TestCompactedMult> tests = suite.getFailedTests(); + Function<TestCompactedMult, Float> function = testCompactedMult -> { + IRunHistory res = testCompactedMult.history(tcIgnited, baseBranchId); - return apply == null ? 0f : apply.getFailRate(); + return res == null ? 0f : res.getFailRate(); }; tests.sort(Comparator.comparing(function).reversed()); tests.forEach(occurrence -> { final DsTestFailureUi failure = new DsTestFailureUi(); - failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(), suite.branchName(), baseBranch); - failure.initStat(tcIgnited, failRateNormalizedBranch, curBranchNormalized); + failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(), + suite.branchName(), baseBranch, baseBranchId); + failure.initStat(occurrence, buildTypeIdId, tcIgnited, baseBranchId, curBranchId); testFailures.add(failure); }); suite.getTopLongRunning().forEach(occurrence -> { - final DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, suite, occurrence, baseBranch); + final DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, compactor, suite, occurrence, baseBranch); topLongRunning.add(failure); }); @@ -260,13 +264,10 @@ public class DsSuiteUi extends DsHistoryStatUi { } private IRunHistory initSuiteStat(ITeamcityIgnited tcIgnited, - String failRateNormalizedBranch, - String curBranchNormalized, - String suiteId) { - if (Strings.isNullOrEmpty(suiteId)) - return null; - - final IRunHistory statInBaseBranch = tcIgnited.getSuiteRunHist(suiteId, failRateNormalizedBranch); + Integer failRateNormalizedBranch, + Integer curBranchNormalized, + MultBuildRunCtx suite) { + IRunHistory statInBaseBranch = suite.history(tcIgnited, failRateNormalizedBranch); if (statInBaseBranch != null) { failures = statInBaseBranch.getFailuresCount(); @@ -285,9 +286,8 @@ public class DsSuiteUi extends DsHistoryStatUi { } IRunHistory latestRunsSrc = null; - if (!failRateNormalizedBranch.equals(curBranchNormalized)) { - - final IRunHistory statForStripe = tcIgnited.getSuiteRunHist(suiteId, curBranchNormalized); + if (!Objects.equals(failRateNormalizedBranch, curBranchNormalized)) { + IRunHistory statForStripe = suite.history(tcIgnited, curBranchNormalized); latestRunsSrc = statForStripe; latestRuns = statForStripe != null ? statForStripe.getLatestRunResults() : null; @@ -315,16 +315,18 @@ public class DsSuiteUi extends DsHistoryStatUi { } @Nonnull public static DsTestFailureUi createOrrucForLongRun(ITeamcityIgnited tcIgnited, - @Nonnull MultBuildRunCtx suite, - final IMultTestOccurrence occurrence, - @Nullable final String failRateBranch) { + IStringCompactor compactor, @Nonnull MultBuildRunCtx suite, + final TestCompactedMult occurrence, + @Nullable final String failRateBranch) { final DsTestFailureUi failure = new DsTestFailureUi(); - failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(), suite.branchName(), failRateBranch); + Integer baseBranchId = compactor.getStringIdIfPresent(normalizeBranch(failRateBranch)); + Integer buildTypeIdId = suite.buildTypeIdId(); + failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(), suite.branchName(), + failRateBranch, baseBranchId); - failure.initStat(tcIgnited, - normalizeBranch(failRateBranch), - normalizeBranch(suite.branchName())); + failure.initStat(occurrence, buildTypeIdId, tcIgnited, baseBranchId, + compactor.getStringIdIfPresent(normalizeBranch(suite.branchName()))); return failure; } diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java index b0cb5d9..6f9f127 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java @@ -25,17 +25,15 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.Nonnull; import javax.annotation.Nullable; - import org.apache.ignite.tcbot.common.util.UrlUtil; -import org.apache.ignite.tcbot.engine.chain.IMultTestOccurrence; import org.apache.ignite.tcbot.engine.chain.TestCompactedMult; import org.apache.ignite.tcbot.engine.issue.EventTemplates; +import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcignited.buildlog.LogMsgToWarn; import org.apache.ignite.tcignited.history.IRunHistory; -import org.apache.ignite.tcignited.ITeamcityIgnited; -import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch; import static org.apache.ignite.tcbot.common.util.TimeUtil.millisToDurationPrintable; +import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch; /** @@ -94,14 +92,16 @@ public class DsTestFailureUi { * @param failure test ocurrence (probably multiple) * @param tcIgn Teamcity. * @param projectId project ID. - * @param branchName - * @param baseBranchName base branch name (e.g. master). + * @param branchName current branch name. + * @param baseBranchName base branch name (e.g. master), without normalization. + * @param baseBranchId Normalized base branch ID (from compactor). */ - public void initFromOccurrence(@Nonnull final IMultTestOccurrence failure, + public void initFromOccurrence(@Nonnull final TestCompactedMult failure, @Nonnull final ITeamcityIgnited tcIgn, @Nullable final String projectId, @Nullable final String branchName, - @Nullable final String baseBranchName) { + @Nullable final String baseBranchName, + Integer baseBranchId) { name = failure.getName(); investigated = failure.isInvestigated(); curFailures = failure.failuresCount(); @@ -144,8 +144,7 @@ public class DsTestFailureUi { webUrlBaseBranch = buildWebLink(tcIgn, full.test.id, projectId, baseBranchName); }); - final IRunHistory stat = tcIgn.getTestRunHist(name, normalizeBranch(baseBranchName)); - + final IRunHistory stat = failure.history(tcIgn, baseBranchId); blockerComment = TestCompactedMult.getPossibleBlockerComment(stat); } @@ -191,22 +190,22 @@ public class DsTestFailureUi { } /** + * @param occurrence + * @param buildTypeIdId * @param tcIgnited TC service as Run stat supplier. - * @param failRateNormalizedBranch Base branch: Fail rate and flakyness detection normalized branch. + * @param baseBranchId Base branch: Fail rate and flakyness detection normalized branch. * @param curBranchNormalized Cur branch normalized. */ - public void initStat(ITeamcityIgnited tcIgnited, - String failRateNormalizedBranch, - String curBranchNormalized) { - - final IRunHistory stat = tcIgnited.getTestRunHist(name, failRateNormalizedBranch); - + public void initStat(TestCompactedMult occurrence, Integer buildTypeIdId, ITeamcityIgnited tcIgnited, + @Nullable Integer baseBranchId, + @Nullable Integer curBranchNormalized) { + final IRunHistory stat = occurrence.history(tcIgnited, baseBranchId); histBaseBranch.init(stat); - IRunHistory statForProblemsDetection = null; + IRunHistory statForProblemsDetection; - if (!curBranchNormalized.equals(failRateNormalizedBranch)) { - statForProblemsDetection = tcIgnited.getTestRunHist(name, curBranchNormalized); + if (!Objects.equals(curBranchNormalized, baseBranchId)) { + statForProblemsDetection = occurrence.history(tcIgnited, curBranchNormalized); if (statForProblemsDetection != null) { histCurBranch = new DsTestHistoryUi(); diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java index 8b8597f..e0fe975 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java @@ -27,7 +27,7 @@ import org.apache.ignite.tcignited.history.IRunHistory; */ public class DsTestHistoryUi { /** 'All the time' runs history statistic. */ - public DsHistoryStatUi allTime = new DsHistoryStatUi(); + @Deprecated public DsHistoryStatUi allTime = new DsHistoryStatUi(); /** Latest runs history statistic. */ public DsHistoryStatUi recent = new DsHistoryStatUi(); diff --git a/tcbot-server-node/build.gradle b/tcbot-server-node/build.gradle index 3ffcf74..0bc43f8 100644 --- a/tcbot-server-node/build.gradle +++ b/tcbot-server-node/build.gradle @@ -18,9 +18,6 @@ apply plugin: 'java' apply plugin: 'application' -sourceCompatibility = '1.8' -targetCompatibility = '1.8' - repositories { mavenCentral() mavenLocal() diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java index 91c4850..5ffd315 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java @@ -33,6 +33,7 @@ import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted; import org.apache.ignite.tcbot.common.conf.ITcServerConfig; import org.apache.ignite.tcignited.history.IRunHistory; import org.apache.ignite.tcignited.history.IRunStat; +import org.apache.ignite.tcignited.history.ISuiteRunHistory; import org.apache.ignite.tcservice.model.agent.Agent; import org.apache.ignite.tcservice.model.mute.MuteInfo; import org.apache.ignite.tcservice.model.result.Build; @@ -200,19 +201,22 @@ public interface ITeamcityIgnited { */ public BuildTypeCompacted getBuildType(String buildTypeId); + @Deprecated @Nullable public IRunHistory getTestRunHist(String testName, @Nullable String branch); + @Deprecated @Nullable public IRunHistory getSuiteRunHist(String suiteId, @Nullable String branch); + @Nullable public ISuiteRunHistory getSuiteRunHist(@Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch); /** * V.3.0 run history implementation based on scan of fat builds. * * @param testName Test name. - * @param suiteName Suite name. - * @param branchName Branch name. + * @param buildTypeId Suite (Build Type) ID, ID for compactor. Null suite name means suite not found. + * @param normalizedBaseBranch Branch name. This branch name does not support branches equivalence, only exact query will work. */ - @Nullable public IRunHistory getTestRunHist(int testName, @Nullable Integer suiteName, @Nullable Integer branchName); + @Nullable public IRunHistory getTestRunHist(int testName, @Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch); /** * @param suiteBuildTypeId Suite id. diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java index a6e37e5..ea68f59 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java @@ -16,10 +16,8 @@ */ package org.apache.ignite.tcignited; -import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.io.File; -import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -32,10 +30,8 @@ import java.util.Map; import java.util.Objects; import java.util.OptionalInt; import java.util.Set; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Supplier; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -52,9 +48,6 @@ import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted; import org.apache.ignite.ci.teamcity.ignited.change.ChangeDao; import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; -import org.apache.ignite.tcignited.mute.MuteDao; -import org.apache.ignite.tcignited.mute.MuteSync; -import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData; import org.apache.ignite.tcbot.common.conf.ITcServerConfig; import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.common.interceptor.GuavaCached; @@ -64,13 +57,18 @@ import org.apache.ignite.tcbot.persistence.scheduler.IScheduler; import org.apache.ignite.tcignited.build.FatBuildDao; import org.apache.ignite.tcignited.build.ProactiveFatBuildSync; import org.apache.ignite.tcignited.buildlog.BuildLogCheckResultDao; +import org.apache.ignite.tcignited.buildref.BranchEquivalence; import org.apache.ignite.tcignited.buildref.BuildRefDao; import org.apache.ignite.tcignited.buildref.BuildRefSync; +import org.apache.ignite.tcignited.history.HistoryCollector; import org.apache.ignite.tcignited.history.IRunHistory; import org.apache.ignite.tcignited.history.IRunStat; +import org.apache.ignite.tcignited.history.ISuiteRunHistory; import org.apache.ignite.tcignited.history.RunHistCompactedDao; import org.apache.ignite.tcignited.history.RunHistSync; -import org.apache.ignite.tcservice.ITeamcity; +import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao; +import org.apache.ignite.tcignited.mute.MuteDao; +import org.apache.ignite.tcignited.mute.MuteSync; import org.apache.ignite.tcservice.ITeamcityConn; import org.apache.ignite.tcservice.model.agent.Agent; import org.apache.ignite.tcservice.model.conf.Project; @@ -95,11 +93,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { /** Max build id diff to enforce reload during incremental refresh. */ public static final int MAX_ID_DIFF_TO_ENFORCE_CONTINUE_SCAN = 3000; - /** Default synonyms. */ - private static final List<String> DEFAULT_SYNONYMS - = Collections.unmodifiableList( - Lists.newArrayList(ITeamcity.DEFAULT, ITeamcity.REFS_HEADS_MASTER, ITeamcity.MASTER)); - /** Server (service) code. */ private String srvCode; @@ -151,11 +144,20 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { /** Run history sync. */ @Inject private RunHistSync runHistSync; - @Inject private BuildLogCheckResultDao logCheckResultDao; + /** Logger check result DAO. */ + @Inject private BuildLogCheckResultDao logCheckResDao; + + /** History DAO. */ + @Inject private SuiteInvocationHistoryDao histDao; + + /** History collector. */ + @Inject private HistoryCollector histCollector; /** Strings compactor. */ @Inject private IStringCompactor compactor; + @Inject private BranchEquivalence branchEquivalence; + /** Server ID mask for cache Entries. */ private int srvIdMaskHigh; @@ -172,7 +174,8 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { changesDao.init(); runHistCompactedDao.init(); muteDao.init(); - logCheckResultDao.init(); + logCheckResDao.init(); + histDao.init(); } /** @@ -352,7 +355,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { @Nullable String branchName) { ensureActualizeRequested(); - return buildRefDao.getAllBuildsCompacted(srvIdMaskHigh, buildTypeId, branchForQuery(branchName)); + return buildRefDao.getAllBuildsCompacted(srvIdMaskHigh, buildTypeId, branchEquivalence.branchForQuery(branchName)); } /** {@inheritDoc} */ @@ -365,7 +368,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { if (stateQueuedId == null) return Collections.emptyList(); - Set<Integer> branchNameIds = branchForQuery(branchName).stream().map(str -> compactor.getStringIdIfPresent(str)) + Set<Integer> branchNameIds = branchEquivalence.branchForQuery(branchName).stream().map(str -> compactor.getStringIdIfPresent(str)) .filter(Objects::nonNull).collect(Collectors.toSet()); List<BuildRefCompacted> res = new ArrayList<>(); @@ -422,7 +425,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { /** {@inheritDoc} */ @Nullable - @AutoProfiling @Override public IRunHistory getTestRunHist(String testName, @Nullable String branch) { return runHistCompactedDao.getTestRunHist(srvIdMaskHigh, testName, branch); } @@ -434,32 +436,27 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { return runHistCompactedDao.getSuiteRunHist(srvIdMaskHigh, suiteId, branch); } - @Nullable @Override - public IRunHistory getTestRunHist(int testName, @Nullable Integer suiteName, @Nullable Integer branchName) { - if (suiteName == null || branchName == null) + /** {@inheritDoc} */ + @Nullable @Override public ISuiteRunHistory getSuiteRunHist(@Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch) { + if (buildTypeId == null || normalizedBaseBranch == null) return null; - Supplier<Set<Integer>> supplier = () -> { - String btId = compactor.getStringFromId(suiteName); - String branchId = compactor.getStringFromId(branchName); - List<BuildRefCompacted> compacted = getAllBuildsCompacted(btId, branchId); - long curTs = System.currentTimeMillis(); - Set<Integer> buildIds = compacted.stream().filter( - bRef -> { - Long startTime = getBuildStartTime(bRef.id()); - if (startTime == null) - return false; + if (buildTypeId < 0 || normalizedBaseBranch < 0) + return null; - return Duration.ofMillis(curTs - startTime).toDays() < InvocationData.MAX_DAYS; - } - ).map(BuildRefCompacted::id).collect(Collectors.toSet()); + return histCollector.getSuiteRunHist(srvIdMaskHigh, buildTypeId, normalizedBaseBranch); + } - System.err.println("Build " + btId + " branch " + branchId + " builds in scope " + buildIds.size()); + /** {@inheritDoc} */ + @Nullable @Override public IRunHistory getTestRunHist(int testName, @Nullable Integer buildTypeId, + @Nullable Integer normalizedBaseBranch) { + if (buildTypeId == null || normalizedBaseBranch == null) + return null; - return buildIds; - }; + if (testName < 0 || buildTypeId < 0 || normalizedBaseBranch < 0) + return null; - return fatBuildDao.getTestRunHist(srvIdMaskHigh, supplier, testName, suiteName, branchName); + return histCollector.getTestRunHist(srvIdMaskHigh, testName, buildTypeId, normalizedBaseBranch); } /** {@inheritDoc} */ @@ -504,13 +501,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { return buildTypeDao.getFatBuildType(srvIdMaskHigh, buildTypeId); } - public List<String> branchForQuery(@Nullable String branchName) { - if (ITeamcity.DEFAULT.equals(branchName)) - return DEFAULT_SYNONYMS; - else - return Collections.singletonList(branchName); - } - /** * Enables scheduling for build refs/builds/history sync */ @@ -522,7 +512,8 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { // schedule find missing later fatBuildSync.ensureActualizationRequested(srvCode, conn); - runHistSync.invokeLaterFindMissingHistory(srvCode); + //todo remove unused code + // runHistSync.invokeLaterFindMissingHistory(srvCode); } /** {@inheritDoc} */ diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java index 707cab8..bc0b203 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java @@ -29,6 +29,8 @@ import org.apache.ignite.ci.teamcity.ignited.change.ChangeDao; import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync; import org.apache.ignite.tcignited.build.FatBuildDao; import org.apache.ignite.tcignited.build.ProactiveFatBuildSync; +import org.apache.ignite.tcignited.history.HistoryCollector; +import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao; import org.apache.ignite.tcignited.mute.MuteDao; import org.apache.ignite.tcignited.mute.MuteSync; import org.apache.ignite.tcignited.buildlog.BuildLogProcessorModule; @@ -65,6 +67,8 @@ public class TeamcityIgnitedModule extends AbstractModule { bind(MuteDao.class).in(new SingletonScope()); bind(MuteSync.class).in(new SingletonScope()); bind(BuildLogCheckResultDao.class).in(new SingletonScope()); + bind(SuiteInvocationHistoryDao.class).in(new SingletonScope()); + bind(HistoryCollector.class).in(new SingletonScope()); TcRealConnectionModule module = new TcRealConnectionModule(); if (conn != null) diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java index ad08fe3..b40e204 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java @@ -18,16 +18,13 @@ package org.apache.ignite.tcignited.build; import com.google.common.base.Preconditions; -import com.google.common.cache.CacheBuilder; +import com.google.common.collect.Iterables; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -41,23 +38,18 @@ import javax.inject.Inject; import javax.inject.Provider; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.cache.CacheEntryProcessor; -import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; -import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted; -import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation; -import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted; -import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey; -import org.apache.ignite.tcbot.common.exeption.ExceptionUtil; import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.persistence.CacheConfigs; import org.apache.ignite.tcbot.persistence.IStringCompactor; -import org.apache.ignite.tcignited.history.IRunHistory; +import org.apache.ignite.tcignited.buildref.BuildRefDao; +import org.apache.ignite.tcignited.history.HistoryCollector; import org.apache.ignite.tcservice.model.changes.ChangesList; import org.apache.ignite.tcservice.model.result.Build; import org.apache.ignite.tcservice.model.result.problems.ProblemOccurrence; import org.apache.ignite.tcservice.model.result.stat.Statistics; -import org.apache.ignite.tcservice.model.result.tests.TestOccurrence; import org.apache.ignite.tcservice.model.result.tests.TestOccurrencesFull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,24 +70,11 @@ public class FatBuildDao { /** Builds cache. */ private IgniteCache<Long, FatBuildCompacted> buildsCache; - - /** Suite history cache. */ - private IgniteCache<RunHistKey, SuiteHistory> suiteHistory; - /** Compactor. */ @Inject private IStringCompactor compactor; - /** - * Non persistence cache for all suite RunHistory for particular branch. - * RunHistKey(ServerId||BranchId||suiteId)-> Build reference - */ - private final com.google.common.cache.Cache<RunHistKey, SuiteHistory> runHistInMemCache - = CacheBuilder.newBuilder() - .maximumSize(8000) - .expireAfterAccess(16, TimeUnit.MINUTES) - .softValues() - .build(); - + /** History collector. */ + @Inject private HistoryCollector histCollector; /** * @@ -155,7 +134,7 @@ public class FatBuildDao { public void putFatBuild(int srvIdMaskHigh, int buildId, FatBuildCompacted newBuild) { buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId), newBuild); - invalidateHistoryInMem(srvIdMaskHigh, Stream.of(newBuild)); + histCollector.invalidateHistoryInMem(srvIdMaskHigh, newBuild); } public static int[] extractChangeIds(@Nonnull ChangesList changesList) { @@ -172,7 +151,7 @@ public class FatBuildDao { } /** - * @param srvIdMaskHigh Server id mask high. + * @param srvIdMaskHigh Server id mask to be placed at high bits of the key. * @param buildId Build id. */ public static long buildIdToCacheKey(int srvIdMaskHigh, int buildId) { @@ -197,10 +176,7 @@ public class FatBuildDao { public Map<Long, FatBuildCompacted> getAllFatBuilds(int srvIdMaskHigh, Collection<Integer> buildsIds) { Preconditions.checkNotNull(buildsCache, "init() was not called"); - Set<Long> ids = buildsIds.stream() - .filter(Objects::nonNull) - .map(buildId -> buildIdToCacheKey(srvIdMaskHigh, buildId)) - .collect(Collectors.toSet()); + Set<Long> ids = buildsIdsToCacheKeys(srvIdMaskHigh, buildsIds); return buildsCache.getAll(ids); } @@ -223,108 +199,47 @@ public class FatBuildDao { .filter(entry -> isKeyForServer(entry.getKey(), srvId)); } - public IRunHistory getTestRunHist(int srvIdMaskHigh, - Supplier<Set<Integer>> buildIdsSupplier, int testName, int suiteName, int branchName) { - - - RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, suiteName, branchName); - - SuiteHistory history; - - try { - history = runHistInMemCache.get(runHistKey, - () -> { - Set<Integer> buildIds = determineLatestBuilds(buildIdsSupplier); - - return calcSuiteHistory(srvIdMaskHigh, buildIds); - }); - } - catch (ExecutionException e) { - throw ExceptionUtil.propagateException(e); - } - - return history.testsHistory.get(testName); + private static Set<Long> buildsIdsToCacheKeys(int srvId, Collection<Integer> stream) { + return stream.stream() + .filter(Objects::nonNull).map(id -> buildIdToCacheKey(srvId, id)).collect(Collectors.toSet()); } - @AutoProfiling - protected SuiteHistory calcSuiteHistory(int srvIdMaskHigh, Set<Integer> buildIds) { - Set<Long> cacheKeys = buildIds.stream().map(id -> buildIdToCacheKey(srvIdMaskHigh, id)).collect(Collectors.toSet()); - - int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS); - - CacheEntryProcessor<Long, FatBuildCompacted, Map<Integer, Invocation>> processor = new HistoryCollectProcessor(successStatusStrId); - - Map<Long, EntryProcessorResult<Map<Integer, Invocation>>> map = buildsCache.invokeAll(cacheKeys, processor); - - SuiteHistory hist = new SuiteHistory(); - - map.values().forEach( - res-> { - if(res==null) - return; - - Map<Integer, Invocation> invocationMap = res.get(); - - if(invocationMap == null) - return; - - invocationMap.forEach((k, v) -> { - RunHistCompacted compacted = hist.testsHistory.computeIfAbsent(k, - k_ -> new RunHistCompacted()); - - compacted.innerAddInvocation(v); + /** + * @param srvId Server id. + * @param ids Ids. + */ + public Map<Integer, Long> getBuildStartTime(int srvId, Set<Integer> ids) { + IgniteCache<Long, BinaryObject> cacheBin = buildsCache.withKeepBinary(); + Set<Long> keys = buildsIdsToCacheKeys(srvId, ids); + HashMap<Integer, Long> res = new HashMap<>(); + + Iterables.partition(keys, 32 * 10).forEach( + chunk -> { + Map<Long, EntryProcessorResult<Long>> map = cacheBin.invokeAll(keys, new GetStartTimeProc()); + map.forEach((k, r) -> { + Long ts = r.get(); + if (ts != null) + res.put(BuildRefDao.cacheKeyToBuildId(k), ts); }); - } ); - System.err.println("Suite history: tests in scope " - + hist.testsHistory.size() - + " for " +buildIds.size() + " builds checked" - + " size " + hist.size(igniteProvider.get())); - - return hist; + return res; } - @AutoProfiling - protected Set<Integer> determineLatestBuilds(Supplier<Set<Integer>> buildIdsSupplier) { - return buildIdsSupplier.get(); - } - - public void invalidateHistoryInMem(int srvId, Stream<BuildRefCompacted> stream) { - Iterable<RunHistKey> objects = - stream - .map(b -> new RunHistKey(srvId, b.buildTypeId(), b.branchName())) - .collect(Collectors.toSet()); - - runHistInMemCache.invalidateAll(objects); - } - - - private static class HistoryCollectProcessor implements CacheEntryProcessor<Long, FatBuildCompacted, Map<Integer, Invocation>> { - private final int successStatusStrId; - - public HistoryCollectProcessor(int successStatusStrId) { - this.successStatusStrId = successStatusStrId; + private static class GetStartTimeProc implements CacheEntryProcessor<Long, BinaryObject, Long> { + public GetStartTimeProc() { } - @Override public Map<Integer, Invocation> process(MutableEntry<Long, FatBuildCompacted> entry, + /** {@inheritDoc} */ + @Override public Long process(MutableEntry<Long, BinaryObject> entry, Object... arguments) throws EntryProcessorException { if (entry.getValue() == null) return null; - Map<Integer, Invocation> hist = new HashMap<>(); - FatBuildCompacted fatBuildCompacted = entry.getValue(); - Stream<TestCompacted> tests = fatBuildCompacted.getAllTests(); - tests.forEach( - testCompacted -> { - Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, (k, v) -> false, successStatusStrId); - - hist.put(testCompacted.testName(), invocation); - } - ); + BinaryObject buildBinary = entry.getValue(); - return hist; + return buildBinary.field("startDate"); } } } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java index 3e01c05..8ad06a7 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java @@ -315,7 +315,8 @@ public class ProactiveFatBuildSync { buildRefDao.save(srvIdMask, refCompacted); - runHistSync.saveToHistoryLater(srvCode, savedVer); + //todo remove unused code + // runHistSync.saveToHistoryLater(srvCode, savedVer); return savedVer; } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java index 8969736..0e2d010 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java @@ -21,15 +21,50 @@ import java.util.HashMap; import java.util.Map; import org.apache.ignite.Ignite; +import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation; import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted; import org.apache.ignite.internal.binary.BinaryObjectExImpl; +import org.apache.ignite.tcignited.history.IRunHistory; +import org.apache.ignite.tcignited.history.ISuiteRunHistory; +import org.apache.ignite.tcignited.history.SuiteInvocation; -public class SuiteHistory { +/** + * Suite run history summary. + */ +public class SuiteHistory implements ISuiteRunHistory { /** Tests history: Test name ID->RunHistory */ - Map<Integer, RunHistCompacted> testsHistory = new HashMap<>(); + private Map<Integer, RunHistCompacted> testsHistory = new HashMap<>(); + + private RunHistCompacted suiteHist = new RunHistCompacted(); public int size(Ignite ignite) { BinaryObjectExImpl binary = ignite.binary().toBinary(this); return binary.length(); } + + public IRunHistory getTestRunHist(int testName) { + return testsHistory.get(testName); + } + + public RunHistCompacted getOrAddTestsHistory(Integer tName) { + return testsHistory.computeIfAbsent(tName, k_ -> new RunHistCompacted()); + } + + public void addTestInvocation(Integer tName, Invocation invocation) { + getOrAddTestsHistory(tName).innerAddInvocation(invocation); + } + + public void addSuiteInvocation(SuiteInvocation suiteInv) { + suiteInv.tests().forEach(this::addTestInvocation); + + suiteHist.innerAddInvocation(suiteInv.suiteInvocation()); + } + + public RunHistCompacted getSuiteHist() { + return suiteHist; + } + + @Override public IRunHistory self() { + return suiteHist; + } } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java similarity index 53% copy from tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java copy to tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java index 8969736..ab5f0b9 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java @@ -14,22 +14,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.ignite.tcignited.buildref; -package org.apache.ignite.tcignited.build; +import com.google.common.collect.Lists; +import java.util.Collections; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.ignite.tcservice.ITeamcity; -import java.util.HashMap; -import java.util.Map; +public class BranchEquivalence { + /** Default synonyms. */ + private static final List<String> DEFAULT_SYNONYMS + = Collections.unmodifiableList( + Lists.newArrayList(ITeamcity.DEFAULT, ITeamcity.REFS_HEADS_MASTER, ITeamcity.MASTER)); -import org.apache.ignite.Ignite; -import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted; -import org.apache.ignite.internal.binary.BinaryObjectExImpl; -public class SuiteHistory { - /** Tests history: Test name ID->RunHistory */ - Map<Integer, RunHistCompacted> testsHistory = new HashMap<>(); - - public int size(Ignite ignite) { - BinaryObjectExImpl binary = ignite.binary().toBinary(this); - return binary.length(); + public List<String> branchForQuery(@Nullable String branchName) { + if (ITeamcity.DEFAULT.equals(branchName)) + return DEFAULT_SYNONYMS; + else + return Collections.singletonList(branchName); } + } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java index f3a3e89..e8c8ae0 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java @@ -175,7 +175,7 @@ public class BuildRefDao { */ @AutoProfiling @Nonnull public List<BuildRefCompacted> getAllBuildsCompacted(int srvId, - String buildTypeId, + String buildTypeId, List<String> bracnhNameQry) { Integer buildTypeIdId = compactor.getStringIdIfPresent(buildTypeId); if (buildTypeIdId == null) diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java new file mode 100644 index 0000000..7254774 --- /dev/null +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java @@ -0,0 +1,326 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.tcignited.history; + +import com.google.common.base.Preconditions; +import com.google.common.cache.CacheBuilder; +import com.google.common.collect.Iterables; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiPredicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.inject.Inject; +import javax.inject.Provider; +import org.apache.ignite.Ignite; +import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted; +import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted; +import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation; +import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData; +import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey; +import org.apache.ignite.tcbot.common.exeption.ExceptionUtil; +import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; +import org.apache.ignite.tcbot.persistence.IStringCompactor; +import org.apache.ignite.tcignited.build.FatBuildDao; +import org.apache.ignite.tcignited.build.SuiteHistory; +import org.apache.ignite.tcignited.buildref.BranchEquivalence; +import org.apache.ignite.tcignited.buildref.BuildRefDao; +import org.apache.ignite.tcservice.model.result.tests.TestOccurrence; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + */ +public class HistoryCollector { + /** Logger. */ + private static final Logger logger = LoggerFactory.getLogger(HistoryCollector.class); + + /** History DAO. */ + @Inject private SuiteInvocationHistoryDao histDao; + + /** Fat build DAO. */ + @Inject private FatBuildDao fatBuildDao; + + /** Build reference DAO. */ + @Inject private BuildRefDao buildRefDao; + + /** Compactor. */ + @Inject private IStringCompactor compactor; + + /** Ignite provider. */ + @Inject private Provider<Ignite> igniteProvider; + + /** Branch equivalence. */ + @Inject private BranchEquivalence branchEquivalence; + + /** Run history DAO. */ + @Inject private RunHistCompactedDao runHistCompactedDao; + + /** + * Non persistence cache for all suite RunHistory for particular branch. RunHistKey(ServerId||BranchId||suiteId)-> + * Build reference + */ + private final com.google.common.cache.Cache<RunHistKey, SuiteHistory> runHistInMemCache + = CacheBuilder.newBuilder() + .maximumSize(8000) + .expireAfterAccess(16, TimeUnit.MINUTES) + .softValues() + .build(); + + /** Biggest build ID, which out of history scope (MAX days + 2). */ + private final ConcurrentMap<Integer, AtomicInteger> biggestBuildIdOutOfHistoryScope = new ConcurrentHashMap<>(); + + /** + * @param srvIdMaskHigh Server id mask to be placed at high bits in the key. + * @param testName Test name. + * @param buildTypeId Suite (Build type) id. + * @param normalizedBaseBranch Branch name. + */ + public IRunHistory getTestRunHist(int srvIdMaskHigh, int testName, int buildTypeId, + int normalizedBaseBranch) { + + SuiteHistory hist = getSuiteHist(srvIdMaskHigh, buildTypeId, normalizedBaseBranch); + + return hist.getTestRunHist(testName); + } + + @AutoProfiling + protected SuiteHistory getSuiteHist(int srvIdMaskHigh, int buildTypeId, int normalizedBaseBranch) { + RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, buildTypeId, normalizedBaseBranch); + + SuiteHistory hist; + try { + hist = runHistInMemCache.get(runHistKey, + () -> loadSuiteHistory(srvIdMaskHigh, buildTypeId, normalizedBaseBranch)); + } + catch (ExecutionException e) { + throw ExceptionUtil.propagateException(e); + } + + return hist; + } + + /** + * Latest actual Build ids supplier. This supplier should handle all equivalent branches in + * it. + * @param srvId + * @param buildTypeId + * @param normalizedBaseBranch + * @param knownBuilds Known builds, which already present in run history. + */ + @AutoProfiling + protected Set<Integer> determineLatestBuilds( + int srvId, int buildTypeId, int normalizedBaseBranch, Set<Integer> knownBuilds) { + String btId = compactor.getStringFromId(buildTypeId); + String branchId = compactor.getStringFromId(normalizedBaseBranch); + List<BuildRefCompacted> bRefsList = buildRefDao.getAllBuildsCompacted(srvId, btId, + branchEquivalence.branchForQuery(branchId)); + + long curTs = System.currentTimeMillis(); + Set<Integer> buildIds = bRefsList.stream() + .filter(b -> { + AtomicInteger biggestIdOutOfScope = biggestBuildIdOutOfHistoryScope.get(srvId); + int outOfScopeBuildId = biggestIdOutOfScope == null ? -1 : biggestIdOutOfScope.get(); + return b.id() > outOfScopeBuildId; + }) + .filter(this::applicableForHistory) + .map(BuildRefCompacted::id) + .filter(bId -> !knownBuilds.contains(bId)).collect(Collectors.toSet()); + + System.out.println("***** Loading build start time history for suite " + + compactor.getStringFromId(buildTypeId) + + " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": " + buildIds.size() + " builds" ); + + Map<Integer, Long> buildStartTimes = getStartTimeFromSpecialCache(srvId, buildIds); + + Set<Integer> notFoundKeys = new HashSet<>(buildIds); + notFoundKeys.removeAll(buildStartTimes.keySet()); + + if (!notFoundKeys.isEmpty()) { + Map<Integer, Long> buildStartTimeFromFatBuild = getStartTimeFromFatBuild(srvId, notFoundKeys); + + buildStartTimes.putAll(buildStartTimeFromFatBuild); + + runHistCompactedDao.setBuildsStartTime(srvId, buildStartTimeFromFatBuild); + } + + Set<Integer> buildInScope = buildIds.stream().filter( + bId -> { + Long startTime = buildStartTimes.get(bId); + if (startTime == null) + return false; + + long ageInDays = Duration.ofMillis(curTs - startTime).toDays(); + + if (ageInDays > InvocationData.MAX_DAYS + 2) { + AtomicInteger integer = biggestBuildIdOutOfHistoryScope.computeIfAbsent(srvId, + s -> { + AtomicInteger atomicInteger = new AtomicInteger(); + atomicInteger.set(-1); + return atomicInteger; + }); + + int newBorder = integer.accumulateAndGet(bId, Math::max); + + if (newBorder == bId) + logger.info("History Collector: New border for server was set " + bId); + } + + return ageInDays < InvocationData.MAX_DAYS; + } + ).collect(Collectors.toSet()); + + System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " + + buildInScope.size() + " from " + bRefsList.size()); + + return buildInScope; + } + + @AutoProfiling + protected Map<Integer, Long> getStartTimeFromSpecialCache(int srvId, Set<Integer> buildIds) { + return runHistCompactedDao.getBuildsStartTime(srvId, buildIds); + } + + @AutoProfiling + protected Map<Integer, Long> getStartTimeFromFatBuild(int srvId, Set<Integer> buildIds) { + return fatBuildDao.getBuildStartTime(srvId, buildIds); + } + + /** + * @param ref Build Reference or fat build. + */ + private boolean applicableForHistory(BuildRefCompacted ref) { + return !ref.isFakeStub() && !ref.isCancelled(compactor) && ref.isFinished(compactor); + } + + @AutoProfiling + protected SuiteHistory loadSuiteHistory(int srvId, + int buildTypeId, + int normalizedBaseBranch) { + Map<Integer, SuiteInvocation> suiteRunHist = histDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch); + + System.out.println("***** Found history for suite " + + compactor.getStringFromId(buildTypeId) + + " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": " + suiteRunHist.size() ); + + Set<Integer> buildIds = determineLatestBuilds(srvId, buildTypeId, normalizedBaseBranch, suiteRunHist.keySet()); + + HashSet<Integer> missedBuildsIds = new HashSet<>(buildIds); + + missedBuildsIds.removeAll(suiteRunHist.keySet()); + + if (!missedBuildsIds.isEmpty()) { + Map<Integer, SuiteInvocation> addl = addSuiteInvocationsToHistory(srvId, missedBuildsIds, normalizedBaseBranch); + + suiteRunHist.putAll(addl); + + /* + Map<Integer, SuiteInvocation> reloaded = histDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch); + + addl.keySet().forEach((k) -> { + Preconditions.checkState( reloaded.containsKey(k)); + }); + */ + } + + SuiteHistory sumary = new SuiteHistory(); + + suiteRunHist.forEach((buildId, suiteInv) -> sumary.addSuiteInvocation(suiteInv)); + + if (logger.isDebugEnabled()) { + logger.debug("***** History for suite " + + compactor.getStringFromId(buildTypeId) + + " branch" + compactor.getStringFromId(normalizedBaseBranch) + " requires " + + sumary.size(igniteProvider.get()) + " bytes"); + } + + return sumary; + } + + /** + * @param srvId Server id. + * @param b Build ref to invalidate. + */ + public void invalidateHistoryInMem(int srvId, BuildRefCompacted b) { + RunHistKey inv = new RunHistKey(srvId, b.buildTypeId(), b.branchName()); + + runHistInMemCache.invalidate(inv); + } + + + @AutoProfiling + protected Map<Integer, SuiteInvocation> addSuiteInvocationsToHistory(int srvId, + HashSet<Integer> missedBuildsIds, int normalizedBaseBranch) { + Map<Integer, SuiteInvocation> suiteRunHist = new HashMap<>(); + int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS); + + System.err.println(Thread.currentThread().getName() + ": GET ALL: " + missedBuildsIds.size()); + + Iterables.partition(missedBuildsIds, 32 * 10).forEach( + chunk -> { + fatBuildDao.getAllFatBuilds(srvId, chunk).forEach((buildCacheKey, fatBuildCompacted) -> { + if (!applicableForHistory(fatBuildCompacted)) + return; + + BiPredicate<Integer, Integer> paramsFilter = (k, v) -> false; + + SuiteInvocation sinv = new SuiteInvocation(srvId, normalizedBaseBranch, fatBuildCompacted, compactor, paramsFilter); + + Stream<TestCompacted> tests = fatBuildCompacted.getAllTests(); + tests.forEach( + testCompacted -> { + Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, paramsFilter, successStatusStrId); + + sinv.addTest(testCompacted.testName(), invocation); + } + ); + + suiteRunHist.put(fatBuildCompacted.id(), sinv); + }); + } + ); + + + System.err.println("***** + Adding to persisted history " + + " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": added " + + suiteRunHist.size() + " invocations from " + missedBuildsIds.size() + " builds checked"); + + histDao.putAll(srvId, suiteRunHist); + + return suiteRunHist; + } + + /** + * @param srvId Server id. + * @param buildTypeId Build type id. + * @param normalizedBaseBranch Normalized base branch. + */ + public ISuiteRunHistory getSuiteRunHist(int srvId, int buildTypeId, int normalizedBaseBranch) { + return getSuiteHist(srvId, buildTypeId, normalizedBaseBranch); + } + +} diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java similarity index 59% copy from tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java copy to tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java index 8969736..38dff1c 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java @@ -14,22 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.ignite.tcignited.history; -package org.apache.ignite.tcignited.build; - -import java.util.HashMap; -import java.util.Map; - -import org.apache.ignite.Ignite; -import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted; -import org.apache.ignite.internal.binary.BinaryObjectExImpl; - -public class SuiteHistory { - /** Tests history: Test name ID->RunHistory */ - Map<Integer, RunHistCompacted> testsHistory = new HashMap<>(); - - public int size(Ignite ignite) { - BinaryObjectExImpl binary = ignite.binary().toBinary(this); - return binary.length(); - } +public interface ISuiteRunHistory { + IRunHistory self(); + IRunHistory getTestRunHist(int testName); } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java index 8fed725..ffc0c47 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java @@ -17,9 +17,15 @@ package org.apache.ignite.tcignited.history; +import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.cache.Cache; @@ -40,6 +46,7 @@ import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.common.interceptor.GuavaCached; import org.apache.ignite.tcbot.persistence.CacheConfigs; import org.apache.ignite.tcbot.persistence.IStringCompactor; +import org.apache.ignite.tcignited.buildref.BuildRefDao; import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch; @@ -61,6 +68,7 @@ public class RunHistCompactedDao { private Provider<Ignite> igniteProvider; /** Test history cache. */ + @Deprecated private IgniteCache<RunHistKey, RunHistCompacted> testHistCache; /** Suite history cache. */ @@ -234,4 +242,32 @@ public class RunHistCompactedDao { cluster.disableWal(testHistCache.getName()); cluster.disableWal(suiteHistCache.getName()); } + + private static Set<Long> buildsIdsToCacheKeys(int srvId, Collection<Integer> ids) { + return ids.stream() + .filter(Objects::nonNull).map(id -> buildIdToCacheKey(srvId, id)).collect(Collectors.toSet()); + } + + public Map<Integer, Long> getBuildsStartTime(int srvId, Set<Integer> ids) { + Set<Long> cacheKeys = buildsIdsToCacheKeys(srvId, ids); + + Map<Integer, Long> res = new HashMap<>(); + + buildStartTime.getAll(cacheKeys).forEach((k, r) -> { + res.put(BuildRefDao.cacheKeyToBuildId(k), r); + }); + + return res; + } + + public void setBuildsStartTime(int srvId, Map<Integer, Long> builds) { + Map<Long, Long> res = new HashMap<>(); + + builds.forEach((buildId, ts) -> { + if (ts != null) + res.put(buildIdToCacheKey(srvId, buildId), ts); + }); + + buildStartTime.putAll(res); + } } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java index a0a90f4..9a621ee 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java @@ -205,6 +205,8 @@ public class RunHistSync { @Nonnull protected String saveInvocationsMap( Map<RunHistKey, List<Invocation>> buildsSaveThisRun, Map<RunHistKey, List<Invocation>> testsSaveThisRun) { + if (Boolean.valueOf(System.getProperty(TcBotSystemProperties.DEV_MODE))) + return "Skipped"; if (Boolean.valueOf(System.getProperty(TcBotSystemProperties.DEV_MODE))) if (testsSaveThisRun.size() > 100) diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java new file mode 100644 index 0000000..7342af7 --- /dev/null +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.tcignited.history; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.function.BiPredicate; +import org.apache.ignite.cache.affinity.AffinityKeyMapped; +import org.apache.ignite.cache.query.annotations.QuerySqlField; +import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; +import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation; +import org.apache.ignite.tcbot.persistence.IStringCompactor; +import org.apache.ignite.tcbot.persistence.Persisted; + +/** + * Shorter verison of FatBuild with less data: created only if run history was required, + * has time limitation of MAX_DAYS, may have TTL. + */ +@Persisted +public class SuiteInvocation { + /** Server ID for queries */ + @QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 0)}) + private int srvId; + + /** Suite name for queries */ + @AffinityKeyMapped + @QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 1)}) + private int buildTypeId; + + /** Teamcity branch name for queries */ + @QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 2)}) + private int normalizedBranchName; + + private Invocation suite; + + private Map<Integer, Invocation> tests = new HashMap<>(); + + Long buildStartTime; + + public SuiteInvocation() {} + + public SuiteInvocation(int srvId, int normalizedBaseBranch, FatBuildCompacted buildCompacted, IStringCompactor comp, + BiPredicate<Integer, Integer> filter) { + this.srvId = srvId; + this.normalizedBranchName = normalizedBaseBranch; + this.buildStartTime = buildCompacted.getStartDateTs(); + this.suite = buildCompacted.toInvocation(comp, filter); + this.buildTypeId = buildCompacted.buildTypeId(); + } + + public void addTest(int testName, Invocation invocation) { + tests.put(testName, invocation); + } + + public Map<Integer, Invocation> tests() { + return Collections.unmodifiableMap(tests); + } + + public Invocation suiteInvocation() { + return suite; + } +} diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java new file mode 100644 index 0000000..dd93c5e --- /dev/null +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.tcignited.history; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import javax.cache.Cache; +import javax.cache.expiry.AccessedExpiryPolicy; +import javax.cache.expiry.Duration; +import javax.inject.Inject; +import javax.inject.Provider; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.query.QueryCursor; +import org.apache.ignite.cache.query.SqlQuery; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; +import org.apache.ignite.tcbot.persistence.CacheConfigs; +import org.apache.ignite.tcignited.buildref.BuildRefDao; + +import static java.util.concurrent.TimeUnit.HOURS; + +/** + * Suite invocation history access object. + */ +public class SuiteInvocationHistoryDao { + /** Ignite provider. */ + @Inject + private Provider<Ignite> igniteProvider; + + /** Suite history cache. */ + private IgniteCache<Long, SuiteInvocation> suiteHistory; + + public void init() { + CacheConfiguration<Long , SuiteInvocation> ccfg = CacheConfigs.getCacheV2Config("teamcitySuiteHistory"); + ccfg.setExpiryPolicyFactory(AccessedExpiryPolicy.factoryOf(new Duration(HOURS, 12))); + ccfg.setEagerTtl(true); + + ccfg.setQueryEntities(Collections.singletonList(new QueryEntity(Long.class, SuiteInvocation.class))); + + Ignite ignite = igniteProvider.get(); + + suiteHistory = ignite.getOrCreateCache(ccfg); + } + + @AutoProfiling + public Map<Integer, SuiteInvocation> getSuiteRunHist(int srvId, int buildTypeId, int normalizedBranchName) { + java.util.Map<Integer, SuiteInvocation> map = new HashMap<>(); + try (QueryCursor<Cache.Entry<Long, SuiteInvocation>> qryCursor = suiteHistory.query( + new SqlQuery<Long, SuiteInvocation>(SuiteInvocation.class, "srvId = ? and buildTypeId = ? and normalizedBranchName = ?") + .setArgs(srvId, buildTypeId, normalizedBranchName))) { + + for (Cache.Entry<Long, SuiteInvocation> next : qryCursor) { + Long key = next.getKey(); + int buildId = BuildRefDao.cacheKeyToBuildId(key); + map.put(buildId, next.getValue()); + } + } + + return map; + } + + @AutoProfiling + public void putAll(int srvId, Map<Integer, SuiteInvocation> addl) { + Map<Long, SuiteInvocation> data = new HashMap<>(); + + addl.forEach((k, v) -> data.put(BuildRefDao.buildIdToCacheKey(srvId, k), v)); + + suiteHistory.putAll(data); + } +}