This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch ignite-10372 in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit eb825aadcb66831d153408154ec27cd69f76a862 Author: Dmitriy Pavlov <[email protected]> AuthorDate: Thu Nov 22 19:25:55 2018 +0300 IGNITE-10372 Optimize master trends page step 3: failed tests move to be sourced from chain content --- .../main/java/org/apache/ignite/ci/ITeamcity.java | 2 - .../apache/ignite/ci/IgnitePersistentTeamcity.java | 17 ---- .../apache/ignite/ci/IgniteTeamcityConnection.java | 12 --- .../java/org/apache/ignite/ci/db/DbMigrations.java | 3 + .../ci/tcbot/trends/MasterTrendsService.java | 33 ++++---- .../teamcity/ignited/fatbuild/TestCompacted.java | 4 +- .../web/model/current/BuildStatisticsSummary.java | 39 ++++++++- .../ignite/ci/web/model/hist/BuildsHistory.java | 98 +++++----------------- 8 files changed, 75 insertions(+), 133 deletions(-) diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java index 1820d14..cedb8d4 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java @@ -86,8 +86,6 @@ public interface ITeamcity extends ITeamcityConn { @Deprecated CompletableFuture<TestRef> getTestRef(FullQueryParams key); - Configurations getConfigurations(FullQueryParams key); - /** * List of build's related issues. * diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java index 102a9c5..1a3e1d2 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java @@ -102,7 +102,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea public static final String BOT_DETECTED_ISSUES = "botDetectedIssues"; public static final String TEST_REFS = "testRefs"; - public static final String CONFIGURATIONS = "configurations"; //todo need separate cache or separate key for 'execution time' because it is placed in statistics private static final String BUILDS_FAILURE_RUN_STAT = "buildsFailureRunStat"; @@ -198,14 +197,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea return getOrCreateCacheV2(ignCacheNme(TESTS_OCCURRENCES)); } - - /** - * @return {@link Configurations} instances cache, 32 parts. - */ - private IgniteCache<String, Configurations> configurationsCache() { - return getOrCreateCacheV2(ignCacheNme(CONFIGURATIONS)); - } - /** * @return {@link TestRef} instances cache, 32 parts. */ @@ -460,14 +451,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea return getTests(href + ",muted:false,status:FAILURE,count:" + cnt + "&fields=testOccurrence(id,name)"); } - /** {@inheritDoc} */ - @AutoProfiling - @Override public Configurations getConfigurations(FullQueryParams key) { - return loadIfAbsent(configurationsCache(), - key.toString(), - k -> teamcity.getConfigurations(key)); - } - private void addTestOccurrencesToStat(TestOccurrences val) { for (TestOccurrence next : val.getTests()) addTestOccurrenceToStat(next, ITeamcity.DEFAULT, null); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java index 7a80d96..914905e 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java @@ -126,8 +126,6 @@ public class IgniteTeamcityConnection implements ITeamcity { /** Build logger processing running. */ private ConcurrentHashMap<Integer, CompletableFuture<LogCheckTask>> buildLogProcessingRunning = new ConcurrentHashMap<>(); - private static int MAX_CFG_CNT = 500; - public Executor getExecutor() { return executor; } @@ -485,16 +483,6 @@ public class IgniteTeamcityConnection implements ITeamcity { }, executor); } - - /** {@inheritDoc} */ - @AutoProfiling - @Override public Configurations getConfigurations(FullQueryParams key) { - Configurations configurations = getJaxbUsingHref("app/rest/latest/builds?locator=snapshotDependency:(to:(id:" + key.getBuildId() - + "),includeInitial:true),defaultFilter:false,count:" + MAX_CFG_CNT, Configurations.class); - - return configurations.setBuild(key.getBuildId()); - } - /** {@inheritDoc} */ @AutoProfiling @Override public IssuesUsagesList getIssuesUsagesList(String href) { return getJaxbUsingHref(href, IssuesUsagesList.class); } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java index 0486f2b..9b695b6 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java @@ -115,6 +115,7 @@ public class DbMigrations { String CHANGES_LIST = "changesList"; String CHANGE_INFO_FULL = "changeInfoFull"; String CURRENT_PR_FAILURES = "currentPrFailures"; + String CONFIGURATIONS = "configurations"; } public static final int SUITES_CNT = 100; @@ -449,6 +450,8 @@ public class DbMigrations { applyDestroyIgnCacheMigration(Old.BUILD_PROBLEMS); applyDestroyIgnCacheMigration(Old.TEST_FULL); + + applyDestroyIgnCacheMigration(Old.CONFIGURATIONS); } private void applyDestroyIgnCacheMigration(String cacheName) { diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/trends/MasterTrendsService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/trends/MasterTrendsService.java index 2e1b8bc..0bad554 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/trends/MasterTrendsService.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/trends/MasterTrendsService.java @@ -21,6 +21,7 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -32,6 +33,8 @@ import javax.inject.Inject; import org.apache.ignite.ci.di.AutoProfiling; import org.apache.ignite.ci.di.cache.GuavaCached; import org.apache.ignite.ci.tcbot.chain.BuildChainProcessor; +import org.apache.ignite.ci.tcmodel.hist.BuildRef; +import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence; import org.apache.ignite.ci.teamcity.ignited.IStringCompactor; import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited; import org.apache.ignite.ci.teamcity.ignited.SyncMode; @@ -41,12 +44,6 @@ import org.apache.ignite.ci.util.FutureUtil; import org.apache.ignite.ci.web.model.current.BuildStatisticsSummary; import org.jetbrains.annotations.NotNull; -import static org.apache.ignite.ci.tcmodel.hist.BuildRef.STATUS_SUCCESS; -import static org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence.TC_EXECUTION_TIMEOUT; -import static org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence.TC_EXIT_CODE; -import static org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence.TC_JVM_CRASH; -import static org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence.TC_OOME; - /** * */ @@ -70,13 +67,7 @@ public class MasterTrendsService { /** Initialize build statistics. */ public void initialize(BuildStatisticsSummary s, @Nonnull final ITeamcityIgnited tcIgn) { - if (s.strIds.isEmpty()) { - s.strIds.put(STATUS_SUCCESS, compactor.getStringId(STATUS_SUCCESS)); - s.strIds.put(TC_EXIT_CODE, compactor.getStringId(TC_EXIT_CODE)); - s.strIds.put(TC_OOME, compactor.getStringId(TC_OOME)); - s.strIds.put(TC_JVM_CRASH, compactor.getStringId(TC_JVM_CRASH)); - s.strIds.put(TC_EXECUTION_TIMEOUT, compactor.getStringId(TC_EXECUTION_TIMEOUT)); - } + BuildStatisticsSummary.initStrings(compactor); FatBuildCompacted build = tcIgn.getFatBuild(s.buildId); @@ -106,14 +97,21 @@ public class MasterTrendsService { chainBuilds.stream().filter(b -> !b.isComposite()) .forEach(b -> { b.getAllTests().forEach(t -> { - if (t.getIgnoredFlag()) + if (t.isIgnoredTest()) arr[0]++; - else if (t.getMutedFlag()) + else if (t.isMutedTest()) arr[1]++; - else if (t.status() != s.strIds.get(STATUS_SUCCESS)) + else if (t.status() != BuildStatisticsSummary.getStringId(TestOccurrence.STATUS_SUCCESS)) arr[2]++; arr[3]++; + + if (t.status() == BuildStatisticsSummary.getStringId(TestOccurrence.STATUS_FAILURE) + && !t.isIgnoredTest() && !t.isMutedTest()) { + Map<Integer, Integer> btTests = s.failedTests.computeIfAbsent(b.buildTypeId(), k -> new HashMap<>()); + + btTests.merge(t.testName(), 1, (x, y) -> (x == null ? 0 : x) + (y == null ? 0 : y)); + } }); }); @@ -127,7 +125,7 @@ public class MasterTrendsService { Stream<FatBuildCompacted> snapshotDependenciesWithProblems = chainBuilds.stream() .filter(b -> !b.isComposite()) - .filter(b -> b.status() != BuildStatisticsSummary.strIds.get(STATUS_SUCCESS)); + .filter(b -> b.status() != BuildStatisticsSummary.getStringId(BuildRef.STATUS_SUCCESS)); s.duration = chainBuilds.stream() .filter(b -> !b.isComposite()) @@ -140,5 +138,4 @@ public class MasterTrendsService { s.totalProblems = s.getBuildTypeProblemsCount(problems); } - } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java index 902dbee..26b64a4 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java @@ -315,13 +315,13 @@ public class TestCompacted { return isFailedTest(compactor) && !(isMutedTest() || isIgnoredTest()); } - private boolean isIgnoredTest() { + public boolean isIgnoredTest() { Boolean flag = getIgnoredFlag(); return flag != null && flag; } - private boolean isMutedTest() { + public boolean isMutedTest() { Boolean flag = getMutedFlag(); return flag != null && flag; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/BuildStatisticsSummary.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/BuildStatisticsSummary.java index 80f1661..4ea34b4 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/BuildStatisticsSummary.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/BuildStatisticsSummary.java @@ -17,6 +17,7 @@ package org.apache.ignite.ci.web.model.current; +import com.google.common.base.Preconditions; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import java.util.ArrayList; @@ -24,15 +25,18 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; +import org.apache.ignite.ci.tcmodel.hist.BuildRef; import org.apache.ignite.ci.tcmodel.result.TestOccurrencesRef; +import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence; +import org.apache.ignite.ci.teamcity.ignited.IStringCompactor; import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted; -import static org.apache.ignite.ci.tcmodel.hist.BuildRef.STATUS_SUCCESS; import static org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence.*; /** @@ -40,7 +44,7 @@ import static org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence.*; */ public class BuildStatisticsSummary { /** String ids. */ - public static HashMap<String, Integer> strIds = new HashMap<>(); + private static final Map<String, Integer> strIds = new ConcurrentHashMap<>(); /** Short problem names. */ public static final String TOTAL = "TOTAL"; @@ -79,6 +83,9 @@ public class BuildStatisticsSummary { /** Is fake stub. */ public boolean isFakeStub; + /** Failed tests: Map from build type string ID -> Map of test name (full) string ID to count of failure. */ + public Map<Integer, Map<Integer, Integer>> failedTests = new HashMap<>(); + /** Is valid. */ public boolean isValid = true; @@ -89,6 +96,32 @@ public class BuildStatisticsSummary { this.buildId = buildId; } + /** + * @param compactor Compactor. + */ + public static void initStrings(IStringCompactor compactor) { + if (strIds.isEmpty()) { + synchronized (BuildStatisticsSummary.class) { + if (strIds.isEmpty()) { + strIds.put(TestOccurrence.STATUS_SUCCESS, compactor.getStringId(TestOccurrence.STATUS_SUCCESS)); + strIds.put(TestOccurrence.STATUS_FAILURE, compactor.getStringId(TestOccurrence.STATUS_FAILURE)); + strIds.put(TC_EXIT_CODE, compactor.getStringId(TC_EXIT_CODE)); + strIds.put(TC_OOME, compactor.getStringId(TC_OOME)); + strIds.put(TC_JVM_CRASH, compactor.getStringId(TC_JVM_CRASH)); + strIds.put(TC_EXECUTION_TIMEOUT, compactor.getStringId(TC_EXECUTION_TIMEOUT)); + //key is the same with tests. strIds.put(BuildRef.STATUS_SUCCESS, compactor.getStringId(BuildRef.STATUS_SUCCESS)); + } + } + } + } + + public static int getStringId(String failure) { + Preconditions.checkState(!strIds.isEmpty()); + + Integer integer = strIds.get(failure); + + return Preconditions.checkNotNull(integer, "No data for [" + failure + "]"); + } /** * @param problemName Problem name. @@ -146,7 +179,7 @@ public class BuildStatisticsSummary { */ private List<FatBuildCompacted> getBuildsWithProblems(List<FatBuildCompacted> builds) { return builds.stream() - .filter(b -> b.status() != strIds.get(STATUS_SUCCESS)) + .filter(b -> b.status() != strIds.get(BuildRef.STATUS_SUCCESS)) .collect(Collectors.toList()); } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/hist/BuildsHistory.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/hist/BuildsHistory.java index 59e6e6c..9ea5d10 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/hist/BuildsHistory.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/hist/BuildsHistory.java @@ -22,12 +22,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; import com.google.inject.Injector; import org.apache.ignite.ci.ITeamcity; -import org.apache.ignite.ci.tcbot.chain.BuildChainProcessor; import org.apache.ignite.ci.tcbot.trends.MasterTrendsService; -import org.apache.ignite.ci.tcmodel.result.Build; -import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence; -import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrences; import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted; +import org.apache.ignite.ci.teamcity.ignited.IStringCompactor; import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited; import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnitedProvider; import org.apache.ignite.ci.teamcity.restcached.ITcServerProvider; @@ -93,6 +90,8 @@ public class BuildsHistory { @Inject private MasterTrendsService masterTrendsService; + @Inject private IStringCompactor compactor; + /** */ public void initialize(ICredentialsProv prov) { ITeamcity teamcity = tcServerProvider.server(srvId, prov); @@ -115,7 +114,7 @@ public class BuildsHistory { .collect(Collectors.toList()); if (!skipTests) - initFailedTests(teamcity, validBuilds); + initFailedTests(validBuilds, buildIdsWithConditions); if (MasterTrendsService.DEBUG) System.out.println("Preparing response"); @@ -169,85 +168,26 @@ public class BuildsHistory { } /** */ - private Map<Integer, String> getConfigurations(ITeamcity teamcity, int buildId) { - Map<Integer, String> configurations = new HashMap<>(); - - FullQueryParams key = new FullQueryParams(); - - key.setServerId(teamcity.serverId()); - key.setBuildId(buildId); - - teamcity.getConfigurations(key).getBuilds().forEach(buildRef -> { - Integer id = buildRef.getId(); - - String configurationName = buildRef.buildTypeId; - - if (id != null && configurationName != null) - configurations.put(id, configurationName); - }); - - return configurations; - } - - /** */ - private void initFailedTests(ITeamcity teamcity, List<Integer> buildIds) { - List<Future<Void>> buildProcessorFutures = new ArrayList<>(); - - for (int buildId : buildIds) { - Future<Void> buildFut = CompletableFuture.supplyAsync(() -> { - Map<Integer, String> configurations = getConfigurations(teamcity, buildId); - - Build build = teamcity.getBuild(teamcity.getBuildHrefById(buildId)); - - TestOccurrences testOccurrences = teamcity.getFailedTests(build.testOccurrences.href, - build.testOccurrences.failed, BuildChainProcessor.normalizeBranch(build.branchName)); - - for (TestOccurrence testOccurrence : testOccurrences.getTests()) { - String configurationName = configurations.get(testOccurrence.getBuildId()); - - if(configurationName == null) - continue; - - Map<String, Float> tests = mergedTestsBySuites.computeIfAbsent(configurationName, - k -> new HashMap<>()); - - String testName = testOccurrence.getName(); - - if (!tests.containsKey(testName)) { - tests.put(testName, 0F); - - FullQueryParams key = new FullQueryParams(); - - key.setServerId(srvId); - key.setProjectId(projectId); - key.setTestName(testOccurrence.getName()); - key.setSuiteId(configurationName); + private void initFailedTests(List<Integer> buildIds, + Map<Integer, Boolean> buildIdsWithConditions) { - teamcity.getTestRef(key); - } + for (BuildStatisticsSummary buildStat : buildsStatistics) { + Boolean valid = buildIdsWithConditions.get(buildStat.buildId); + if(!Boolean.TRUE.equals(valid)) + continue; - tests.put(testName, tests.get(testName) + 1F / buildIds.size()); - } + buildStat.failedTests.forEach((btId,v)->{ + String configurationName = compactor.getStringFromId(btId); + Map<String, Float> tests = mergedTestsBySuites.computeIfAbsent(configurationName, + k -> new HashMap<>()); - return null; - }, teamcity.getExecutor()); + v.forEach((t, c) -> { + String testName = compactor.getStringFromId(t); - buildProcessorFutures.add(buildFut); + tests.merge(testName, 1F / buildIds.size(), (a, b) -> a + b); + }); + }); } - - buildProcessorFutures.forEach(v -> { - try { - v.get(); - } catch (ExecutionException e) { - if (e.getCause() instanceof UncheckedIOException) - logger.error(Arrays.toString(e.getStackTrace())); - - else - throw new RuntimeException(e); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - }); } private BuildsHistory withParameters(Builder builder) {
