This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch ignite-10030 in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit e24abae77954d2056389d03931dcbcbca869190f Author: Dmitriy Pavlov <[email protected]> AuthorDate: Sat Oct 27 18:35:54 2018 +0300 IGNITE-10030 problems done in prod: Background upload of changes, problems, and statistics --- .../main/java/org/apache/ignite/ci/ITeamcity.java | 5 +- .../apache/ignite/ci/IgnitePersistentTeamcity.java | 3 ++ .../apache/ignite/ci/IgniteTeamcityConnection.java | 1 + .../apache/ignite/ci/analysis/FullChainRunCtx.java | 12 ----- .../apache/ignite/ci/analysis/MultBuildRunCtx.java | 53 ++++++++-------------- .../ignite/ci/analysis/SingleBuildRunCtx.java | 33 ++++---------- .../ci/tcbot/builds/CompareBuildsService.java | 4 +- .../ignite/ci/tcbot/chain/BuildChainProcessor.java | 9 ++-- .../ignited/fatbuild/FatBuildCompacted.java | 13 +++++- .../ignited/fatbuild/ProactiveFatBuildSync.java | 3 +- .../ignited/fatbuild/ProblemCompacted.java | 29 ++++++++++-- 11 files changed, 78 insertions(+), 87 deletions(-) diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java index 79c1b2b..5cdfaea 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java @@ -133,10 +133,7 @@ public interface ITeamcity extends ITeamcityConn { return "app/rest/latest/builds/id:" + Integer.toString(id); } - /** - * @param build - * @return - */ + @Deprecated ProblemOccurrences getProblems(BuildRef build); @Deprecated diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java index 8cdf0cd..9c321ad 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java @@ -1122,6 +1122,9 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea /** {@inheritDoc} */ @AutoProfiling @Override public void calculateBuildStatistic(SingleBuildRunCtx ctx) { + if (ctx.buildId() == null) + return; + if (calculatedStatistic().containsKey(ctx.buildId())) return; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java index 4a801ff..09f0e54 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java @@ -318,6 +318,7 @@ public class IgniteTeamcityConnection implements ITeamcity { } @Override + @AutoProfiling public ProblemOccurrences getProblems(int buildId) { return getJaxbUsingHref("app/rest/latest/problemOccurrences" + "?locator=build:(id:" + buildId + ")" + diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/FullChainRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/FullChainRunCtx.java index 0e04ccb..0d01506 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/FullChainRunCtx.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/FullChainRunCtx.java @@ -40,18 +40,6 @@ public class FullChainRunCtx { fakeStub = chainResults.isFakeStub(); } - @Deprecated - public int buildProblems() { - return (int)buildCfgsResults.stream().filter(MultBuildRunCtx::hasNontestBuildProblem).count(); - } - - public int timeoutsOomeCrashBuildProblems() { - return (int)buildCfgsResults.stream().filter(context -> - context.hasJvmCrashProblem() - || context.hasTimeoutProblem() - || context.hasOomeProblem()).count(); - } - public List<MultBuildRunCtx> suites() { return buildCfgsResults; } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java index 0411fe2..9076484 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java @@ -24,29 +24,23 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Future; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; -import org.apache.ignite.ci.tcmodel.conf.BuildType; + import org.apache.ignite.ci.tcmodel.hist.BuildRef; -import org.apache.ignite.ci.tcmodel.result.Build; -import org.apache.ignite.ci.tcmodel.result.TestOccurrencesRef; import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence; import org.apache.ignite.ci.tcmodel.result.stat.Statistics; -import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence; import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull; +import org.apache.ignite.ci.teamcity.ignited.IStringCompactor; +import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted; import org.apache.ignite.ci.util.CollectionUtil; -import org.apache.ignite.ci.util.FutureUtil; -import org.apache.ignite.ci.util.TimeUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import static java.util.stream.Stream.concat; - /** * Run configuration execution results loaded from different API URLs. * Includes tests and problem occurrences; if logs processing is done also contains last started test @@ -55,6 +49,8 @@ public class MultBuildRunCtx implements ISuiteResults { /** First build info. */ @Nonnull private final BuildRef firstBuildInfo; + private IStringCompactor compactor; + /** Builds: Single execution. */ private List<SingleBuildRunCtx> builds = new CopyOnWriteArrayList<>(); @@ -77,8 +73,9 @@ public class MultBuildRunCtx implements ISuiteResults { /** Currently scheduled builds */ private Integer queuedBuildCount; - public MultBuildRunCtx(@Nonnull final BuildRef buildInfo) { + public MultBuildRunCtx(@Nonnull final BuildRef buildInfo, IStringCompactor compactor) { this.firstBuildInfo = buildInfo; + this.compactor = compactor; } public Stream<String> getCriticalFailLastStartedTest() { @@ -106,23 +103,13 @@ public class MultBuildRunCtx implements ISuiteResults { return firstBuildInfo.buildTypeId; } - - @Deprecated - //currently used only in old metrics - public boolean hasNontestBuildProblem() { - return allProblemsInAllBuilds().anyMatch(problem -> - !problem.isFailedTests() - && !problem.isSnapshotDepProblem() - && !ProblemOccurrence.BUILD_FAILURE_ON_MESSAGE.equals(problem.type)); - //todo what to do with BuildFailureOnMessage, now it is ignored - } - public boolean hasAnyBuildProblemExceptTestOrSnapshot() { return allProblemsInAllBuilds() - .anyMatch(p -> !p.isFailedTests() && !p.isSnapshotDepProblem()); + .anyMatch(p -> !p.isFailedTests(compactor) && !p.isSnapshotDepProblem(compactor)); } - @NotNull public Stream<ProblemOccurrence> allProblemsInAllBuilds() { + @NotNull + private Stream<ProblemCompacted> allProblemsInAllBuilds() { return buildsStream().flatMap(SingleBuildRunCtx::getProblemsStream); } @@ -184,21 +171,21 @@ public class MultBuildRunCtx implements ISuiteResults { addKnownProblemCnt(res, "Exit Code", getExitCodeProblemsCount()); { - Stream<ProblemOccurrence> stream = + Stream<ProblemCompacted> stream = allProblemsInAllBuilds().filter(p -> - !p.isFailedTests() - && !p.isSnapshotDepProblem() - && !p.isExecutionTimeout() - && !p.isJvmCrash() - && !p.isExitCode() - && !p.isJavaLevelDeadlock() - && !p.isOome()); - Optional<ProblemOccurrence> bpOpt = stream.findAny(); + !p.isFailedTests(compactor) + && !p.isSnapshotDepProblem(compactor) + && !p.isExecutionTimeout(compactor) + && !p.isJvmCrash(compactor) + && !p.isExitCode(compactor) + //&& !p.isJavaLevelDeadlock(compactor) + && !p.isOome(compactor)); + Optional<ProblemCompacted> bpOpt = stream.findAny(); if (bpOpt.isPresent()) { if (res.length() > 0) res.append(", "); - res.append(bpOpt.get().type).append(" "); + res.append(bpOpt.get().type(compactor)).append(" "); } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java index f34a53d..4bc48cd 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java @@ -26,12 +26,10 @@ import java.util.concurrent.Future; import java.util.stream.Stream; import javax.annotation.Nonnull; import org.apache.ignite.ci.tcmodel.changes.Change; -import org.apache.ignite.ci.tcmodel.result.Build; -import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence; -import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence; import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull; import org.apache.ignite.ci.teamcity.ignited.IStringCompactor; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; +import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted; import org.apache.ignite.ci.util.FutureUtil; import org.jetbrains.annotations.Nullable; @@ -41,26 +39,22 @@ import org.jetbrains.annotations.Nullable; public class SingleBuildRunCtx implements ISuiteResults { /** Build compacted. */ private FatBuildCompacted buildCompacted; + /** Compactor. */ private IStringCompactor compactor; /** Logger check result future. */ private CompletableFuture<LogCheckResult> logCheckResFut; - /** Build problems occurred during single build run. */ - @Nullable private List<ProblemOccurrence> problems; - /** Changes. */ private List<Change> changes = new ArrayList<>(); /** - * @param build Build. * @param buildCompacted Build compacted. * @param compactor Compactor. */ - public SingleBuildRunCtx(Build build, - FatBuildCompacted buildCompacted, - IStringCompactor compactor) { + public SingleBuildRunCtx(FatBuildCompacted buildCompacted, + IStringCompactor compactor) { this.buildCompacted = buildCompacted; this.compactor = compactor; } @@ -78,26 +72,23 @@ public class SingleBuildRunCtx implements ISuiteResults { } private long getExecutionTimeoutCount() { - return getProblemsStream().filter(ProblemOccurrence::isExecutionTimeout).count(); + return getProblemsStream().filter(p -> p.isExecutionTimeout(compactor)).count(); } - Stream<ProblemOccurrence> getProblemsStream() { - if (problems == null) - return Stream.empty(); - - return problems.stream().filter(Objects::nonNull); + Stream<ProblemCompacted> getProblemsStream() { + return buildCompacted.problems().stream(); } @Override public boolean hasJvmCrashProblem() { - return getProblemsStream().anyMatch(ProblemOccurrence::isJvmCrash); + return getProblemsStream().anyMatch(p -> p.isJvmCrash(compactor)); } @Override public boolean hasOomeProblem() { - return getProblemsStream().anyMatch(ProblemOccurrence::isOome); + return getProblemsStream().anyMatch(p -> p.isOome(compactor)); } @Override public boolean hasExitCodeProblem() { - return getProblemsStream().anyMatch(ProblemOccurrence::isExitCode); + return getProblemsStream().anyMatch(p -> p.isExitCode(compactor)); } @Override public String suiteId() { @@ -149,10 +140,6 @@ public class SingleBuildRunCtx implements ISuiteResults { return logCheckRes; } - public void setProblems(@Nullable List<ProblemOccurrence> problems) { - this.problems = problems; - } - public void addChange(Change change) { if (change.isFakeStub()) return; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java index b383dc5..da17398 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java @@ -26,6 +26,7 @@ import org.apache.ignite.ci.analysis.MultBuildRunCtx; import org.apache.ignite.ci.tcbot.chain.BuildChainProcessor; import org.apache.ignite.ci.tcmodel.hist.BuildRef; import org.apache.ignite.ci.tcmodel.result.Build; +import org.apache.ignite.ci.teamcity.ignited.IStringCompactor; import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited; import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnitedProvider; import org.apache.ignite.ci.teamcity.restcached.ITcServerProvider; @@ -41,6 +42,7 @@ public class CompareBuildsService { @Inject ICredentialsProv prov; @Inject BuildChainProcessor bcp; @Inject ITeamcityIgnitedProvider tcIgnitedProv; + @Inject IStringCompactor compactor; public List<String> tests0(String srvId, Integer buildId ) { IAnalyticsEnabledTeamcity teamcity = helper.server(srvId, prov); @@ -74,7 +76,7 @@ public class CompareBuildsService { else { logger.info("Loading tests for build {}.", build.getId()); - MultBuildRunCtx buildCtx = new MultBuildRunCtx(build); + MultBuildRunCtx buildCtx = new MultBuildRunCtx(build, compactor); buildCtx.addBuild(bcp.loadTestsAndProblems(tc, build, buildCtx, server)); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java index cb7a886..7bdd497 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java @@ -23,7 +23,6 @@ import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -158,7 +157,8 @@ public class BuildChainProcessor { ITeamcityIgnited teamcityIgnited, Map<String, MultBuildRunCtx> buildsCtxMap, Stream<? extends BuildRef> list) { list.forEach((BuildRef ref) -> { - MultBuildRunCtx ctx = buildsCtxMap.computeIfAbsent(ref.buildTypeId, k -> new MultBuildRunCtx(ref)); + MultBuildRunCtx ctx = buildsCtxMap.computeIfAbsent(ref.buildTypeId, + k -> new MultBuildRunCtx(ref, compactor)); ctx.addBuild(loadTestsAndProblems(teamcity, ref, ctx, teamcityIgnited)); }); @@ -177,9 +177,8 @@ public class BuildChainProcessor { @Deprecated MultBuildRunCtx mCtx, ITeamcityIgnited tcIgnited) { FatBuildCompacted buildCompacted = tcIgnited.getFatBuild(buildRef.getId()); - Build buildFromFat = buildCompacted.toBuild(compactor); - SingleBuildRunCtx ctx = new SingleBuildRunCtx(buildFromFat, buildCompacted, compactor); + SingleBuildRunCtx ctx = new SingleBuildRunCtx(buildCompacted, compactor); if (!buildCompacted.isComposite()) mCtx.addTests(buildCompacted.getTestOcurrences(compactor).getTests()); @@ -190,8 +189,6 @@ public class BuildChainProcessor { if (build == null || build.isFakeStub()) return ctx; - if (build.problemOccurrences != null) - ctx.setProblems(teamcity.getProblems(build).getProblemsNonNull()); if (build.lastChanges != null) { for (ChangeRef next : build.lastChanges.changes) { diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java index dbb23b3..3cc65db 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java @@ -262,12 +262,14 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn name == that.name && Objects.equal(tests, that.tests) && Objects.equal(snapshotDeps, that.snapshotDeps) && - Objects.equal(flags, that.flags); + Objects.equal(flags, that.flags) && + Objects.equal(problems, that.problems); } /** {@inheritDoc} */ @Override public int hashCode() { - return Objects.hashCode(super.hashCode(), _ver, startDate, finishDate, queuedDate, projectId, name, tests, snapshotDeps, flags); + return Objects.hashCode(super.hashCode(), _ver, startDate, finishDate, queuedDate, projectId, name, tests, + snapshotDeps, flags, problems); } /** @@ -312,6 +314,13 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn .collect(Collectors.toList()); } + public List<ProblemCompacted> problems() { + if (this.problems == null) + return Collections.emptyList(); + + return Collections.unmodifiableList(this.problems); + } + public void addProblems(IStringCompactor compactor, List<ProblemOccurrence> occurrences) { if (occurrences.isEmpty()) return; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java index e916891..3cd337a 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java @@ -232,8 +232,7 @@ public class ProactiveFatBuildSync { } if (build.problemOccurrences != null) { - ProblemOccurrences po = conn.getProblems(buildId); - problems = new ArrayList<>(po.getProblemsNonNull()); + problems = conn.getProblems(buildId).getProblemsNonNull(); } else problems = Collections.emptyList(); } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java index 85e6792..961dbe8 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java @@ -45,6 +45,9 @@ import static org.apache.ignite.ci.analysis.RunStat.extractIdPrefixed; * */ public class ProblemCompacted { + /** Logger. */ + private static final Logger logger = LoggerFactory.getLogger(ProblemCompacted.class); + /** Id */ private int id = -1; private int type = -1; @@ -55,9 +58,6 @@ public class ProblemCompacted { // later details may be needed @Nullable private byte[] details; - /** Logger. */ - private static final Logger logger = LoggerFactory.getLogger(ProblemCompacted.class); - /** * Default constructor. */ @@ -74,7 +74,7 @@ public class ProblemCompacted { try { final Integer problemId = extractIdPrefixed(problemIdStr, "problem:(id:", ")"); if (problemId != null) - id = problemId ; + id = problemId; } catch (Exception e) { logger.error("Failed to handle TC response: " + problemIdStr, e); } @@ -139,4 +139,25 @@ public class ProblemCompacted { public String type(IStringCompactor compactor) { return compactor.getStringFromId(type); } + + public boolean isJvmCrash(IStringCompactor compactor) { + return compactor.getStringId(ProblemOccurrence.TC_JVM_CRASH) == type; + } + + public boolean isOome(IStringCompactor compactor) { + return compactor.getStringId(ProblemOccurrence.TC_OOME) == type; + } + + public boolean isExitCode(IStringCompactor compactor) { + return compactor.getStringId(ProblemOccurrence.TC_EXIT_CODE) == type; + } + + public boolean isFailedTests(IStringCompactor compactor) { + return compactor.getStringId(ProblemOccurrence.TC_FAILED_TESTS) == type; + } + + public boolean isSnapshotDepProblem(IStringCompactor compactor) { + return compactor.getStringId(ProblemOccurrence.SNAPSHOT_DEPENDENCY_ERROR) == type + || compactor.getStringId(ProblemOccurrence.SNAPSHOT_DEPENDENCY_ERROR_BUILD_PROCEEDS_TYPE) == type; + } }
