This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 9e3cb0bb05d7205dad68a12f9edb0df78cca3749 Author: Nikolai Kulagin <[email protected]> AuthorDate: Tue Sep 11 14:17:32 2018 +0300 Refactor save of value in cache and fix NPE - Fixes #3. Signed-off-by: Dmitriy Pavlov <[email protected]> --- .../main/java/org/apache/ignite/ci/ITeamcity.java | 36 ++- .../apache/ignite/ci/IgnitePersistentTeamcity.java | 75 ++++-- .../org/apache/ignite/ci/IgniteTeamcityHelper.java | 61 ++++- .../org/apache/ignite/ci/analysis/Expirable.java | 14 +- .../apache/ignite/ci/analysis/MultBuildRunCtx.java | 6 +- .../ci/issue/{IssueRef.java => ProblemRef.java} | 8 +- .../apache/ignite/ci/tcmodel/hist/BuildRef.java | 2 + .../org/apache/ignite/ci/tcmodel/result/Build.java | 16 +- .../ci/tcmodel/result/ProblemOccurrencesRef.java | 4 + .../result/RelatedIssuesRef.java} | 14 +- .../ci/tcmodel/result/TestOccurrencesRef.java | 2 + .../IssueRef.java} | 23 +- .../IssueUsage.java} | 28 +- .../IssuesUsagesList.java} | 30 ++- .../tcmodel/result/problems/ProblemOccurrence.java | 30 ++- .../result/problems/ProblemOccurrences.java | 3 +- .../web/model/current/BuildStatisticsSummary.java | 286 +++++++++++++++++++++ .../ci/web/model/current/SuiteCurrentStatus.java | 8 +- .../ignite/ci/web/model/current/TestFailure.java | 8 +- .../ci/web/rest/build/GetBuildTestFailures.java | 79 +++++- ignite-tc-helper-web/src/main/webapp/index.html | 7 +- .../src/main/webapp/statistics.html | 203 +++++++++++++++ 22 files changed, 836 insertions(+), 107 deletions(-) diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java index 9782cbf..d9d8d1d 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java @@ -33,6 +33,7 @@ import org.apache.ignite.ci.tcmodel.changes.ChangesList; import org.apache.ignite.ci.tcmodel.conf.BuildType; import org.apache.ignite.ci.tcmodel.hist.BuildRef; import org.apache.ignite.ci.tcmodel.result.Build; +import org.apache.ignite.ci.tcmodel.result.issues.IssuesUsagesList; import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence; import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrences; import org.apache.ignite.ci.tcmodel.result.stat.Statistics; @@ -52,6 +53,7 @@ import static org.apache.ignite.ci.db.DbMigrations.TESTS_COUNT_7700; public interface ITeamcity extends AutoCloseable { String DEFAULT = "<default>"; + long DEFAULT_BUILDS_COUNT = 1000; CompletableFuture<List<BuildType>> getProjectSuites(String projectId); @@ -62,7 +64,17 @@ public interface ITeamcity extends AutoCloseable { * @param branch * @return list of builds in historical order, recent builds coming last */ - List<BuildRef> getFinishedBuilds(String projectId, String branch); + default List<BuildRef> getFinishedBuilds(String projectId, String branch) { + return getFinishedBuilds(projectId, branch, null); + }; + + /** + * @param projectId suite ID (string without spaces) + * @param branch + * @param cnt builds count + * @return list of builds in historical order, recent builds coming last + */ + List<BuildRef> getFinishedBuilds(String projectId, String branch, Long cnt); /** * Includes snapshot dependencies failed builds into list @@ -71,7 +83,19 @@ public interface ITeamcity extends AutoCloseable { * @param branch branch in TC identification * @return list of builds in historical order, recent builds coming last */ - List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String projectId, String branch); + default List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String projectId, String branch){ + return getFinishedBuilds(projectId, branch, null); + }; + + /** + * Includes snapshot dependencies failed builds into list + * + * @param projectId suite ID (string without spaces) + * @param branch branch in TC identification + * @param cnt builds count + * @return list of builds in historical order, recent builds coming last + */ + List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String projectId, String branch, Long cnt); /** */ CompletableFuture<List<BuildRef>> getRunningBuilds(@Nullable String branch); @@ -80,7 +104,11 @@ public interface ITeamcity extends AutoCloseable { CompletableFuture<List<BuildRef>> getQueuedBuilds(@Nullable String branch); default int[] getBuildNumbersFromHistory(String projectId, String branchNameForHist) { - return getFinishedBuilds(projectId, branchNameForHist).stream().mapToInt(BuildRef::getId).toArray(); + return getBuildNumbersFromHistory(projectId, branchNameForHist, null); + } + + default int[] getBuildNumbersFromHistory(String projectId, String branchNameForHist, Long cnt) { + return getFinishedBuilds(projectId, branchNameForHist, cnt).stream().mapToInt(BuildRef::getId).toArray(); } Build getBuild(String href); @@ -114,6 +142,8 @@ public interface ITeamcity extends AutoCloseable { ChangesList getChangesList(String href); + IssuesUsagesList getIssuesUsagesList(String href); + /** * Runs deep collection of all related statistics for particular build * diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java index 99f61bb..e89e2f2 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java @@ -36,6 +36,7 @@ import java.util.concurrent.locks.Lock; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Predicate; +import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; import javax.annotation.Nullable; @@ -57,6 +58,7 @@ import org.apache.ignite.ci.tcmodel.changes.ChangesList; import org.apache.ignite.ci.tcmodel.conf.BuildType; import org.apache.ignite.ci.tcmodel.hist.BuildRef; import org.apache.ignite.ci.tcmodel.result.Build; +import org.apache.ignite.ci.tcmodel.result.issues.IssuesUsagesList; import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrences; import org.apache.ignite.ci.tcmodel.result.stat.Statistics; import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence; @@ -84,6 +86,7 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea public static final String LOG_CHECK_RESULT = "logCheckResult"; public static final String CHANGE_INFO_FULL = "changeInfoFull"; public static final String CHANGES_LIST = "changesList"; + public static final String ISSUES_USAGES_LIST = "issuesUsagesList"; public static final String TEST_FULL = "testFull"; public static final String BUILD_PROBLEMS = "buildProblems"; public static final String BUILD_STATISTICS = "buildStatistics"; @@ -197,7 +200,7 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea } /** - * @return Build history: {@link BuildRef} lists cache, 32 parts, transaactional + * @return Build history: {@link BuildRef} lists cache, 32 parts, transactional */ public IgniteCache<SuiteInBranch, Expirable<List<BuildRef>>> buildHistIncFailedCache() { return getOrCreateCacheV2Tx(ignCacheNme(BUILD_HIST_FINISHED_OR_FAILED)); @@ -240,14 +243,15 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea return loaded; } - private <K, V> V timedLoadIfAbsentOrMerge(IgniteCache<K, Expirable<V>> cache, int seconds, K key, + private <K, V extends List> V timedLoadIfAbsentOrMerge(IgniteCache<K, Expirable<V>> cache, int seconds, Long cnt, K key, BiFunction<K, V, V> loadWithMerge) { @Nullable final Expirable<V> persistedBuilds = cache.get(key); int fields = ObjectInterner.internFields(persistedBuilds); if (persistedBuilds != null) { - if (persistedBuilds.isAgeLessThanSecs(seconds)) + if (persistedBuilds.isAgeLessThanSecs(seconds) && + (cnt == null || persistedBuilds.hasCounterGreaterThan(cnt))) return persistedBuilds.getData(); } @@ -258,7 +262,7 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea try { apply = loadWithMerge.apply(key, persistedBuilds != null ? persistedBuilds.getData() : null); - final Expirable<V> newVal = new Expirable<>(System.currentTimeMillis(), apply); + final Expirable<V> newVal = new Expirable<>(System.currentTimeMillis(), apply.size(), apply); cache.put(key, newVal); } @@ -270,14 +274,14 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea } /** {@inheritDoc} */ - @Override public List<BuildRef> getFinishedBuilds(String projectId, String branch) { + @Override public List<BuildRef> getFinishedBuilds(String projectId, String branch, Long cnt) { final SuiteInBranch suiteInBranch = new SuiteInBranch(projectId, branch); - return timedLoadIfAbsentOrMerge(buildHistCache(), 60, suiteInBranch, + List<BuildRef> buildRefs = timedLoadIfAbsentOrMerge(buildHistCache(), 60, cnt, suiteInBranch, (key, persistedValue) -> { List<BuildRef> builds; try { - builds = teamcity.getFinishedBuilds(projectId, branch); + builds = teamcity.getFinishedBuilds(projectId, branch, cnt); } catch (Exception e) { if (Throwables.getRootCause(e) instanceof FileNotFoundException) { @@ -290,6 +294,13 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea return mergeByIdToHistoricalOrder(persistedValue, builds); }); + + if (cnt == null) + return buildRefs; + + return buildRefs.stream() + .skip(cnt < buildRefs.size() ? buildRefs.size() - cnt : 0) + .collect(Collectors.toList()); } @NotNull @@ -307,12 +318,12 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea //loads build history with following parameter: defaultFilter:false,state:finished /** {@inheritDoc} */ - @Override public List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String projectId, String branch) { + @Override public List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String projectId, String branch, Long cnt) { final SuiteInBranch suiteInBranch = new SuiteInBranch(projectId, branch); - return timedLoadIfAbsentOrMerge(buildHistIncFailedCache(), 60, suiteInBranch, + return timedLoadIfAbsentOrMerge(buildHistIncFailedCache(), 60, cnt, suiteInBranch, (key, persistedValue) -> { - List<BuildRef> failed = teamcity.getFinishedBuildsIncludeSnDepFailed(projectId, branch); + List<BuildRef> failed = teamcity.getFinishedBuildsIncludeSnDepFailed(projectId, branch, cnt); return mergeByIdToHistoricalOrder(persistedValue, failed); }); @@ -335,8 +346,8 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea public static int getTriggerRelCacheValidSecs(int defaultSecs) { long msSinceTrigger = System.currentTimeMillis() - lastTriggerMs; - long secondsSinceTigger = TimeUnit.MILLISECONDS.toSeconds(msSinceTrigger); - return Math.min((int)secondsSinceTigger, defaultSecs); + long secondsSinceTrigger = TimeUnit.MILLISECONDS.toSeconds(msSinceTrigger); + return Math.min((int)secondsSinceTrigger, defaultSecs); } /** {@inheritDoc} */ @@ -446,18 +457,22 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea /** {@inheritDoc}*/ @Override public ProblemOccurrences getProblems(Build build) { - String href = build.problemOccurrences.href; + if (build.problemOccurrences != null) { + String href = build.problemOccurrences.href; - return loadIfAbsent( - buildProblemsCache(), - href, - k -> { - ProblemOccurrences problems = teamcity.getProblems(build); + return loadIfAbsent( + buildProblemsCache(), + href, + k -> { + ProblemOccurrences problems = teamcity.getProblems(build); - registerCriticalBuildProblemInStat(build, problems); + registerCriticalBuildProblemInStat(build, problems); - return problems; - }); + return problems; + }); + } + else + return new ProblemOccurrences(); } private void registerCriticalBuildProblemInStat(Build build, ProblemOccurrences problems) { @@ -591,6 +606,24 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea }); } + @Override public IssuesUsagesList getIssuesUsagesList(String href) { + IssuesUsagesList issuesUsages = loadIfAbsentV2(ISSUES_USAGES_LIST, href, href1 -> { + try { + return teamcity.getIssuesUsagesList(href1); + } + catch (Exception e) { + if (Throwables.getRootCause(e) instanceof FileNotFoundException) { + System.err.println("Issues Usage List not found for href : " + href); + + return new IssuesUsagesList(); + } + else + throw e; + } + }); + return issuesUsages; + } + /** {@inheritDoc} */ @Override public List<RunStat> topTestFailing(int cnt) { return CollectionUtil.top(allTestAnalysis(), cnt, Comparator.comparing(RunStat::getFailRate)); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java index 079c80b..d3421b7 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java @@ -62,6 +62,7 @@ import org.apache.ignite.ci.tcmodel.conf.bt.BuildTypeFull; import org.apache.ignite.ci.tcmodel.hist.BuildRef; import org.apache.ignite.ci.tcmodel.hist.Builds; import org.apache.ignite.ci.tcmodel.result.Build; +import org.apache.ignite.ci.tcmodel.result.issues.IssuesUsagesList; import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrences; import org.apache.ignite.ci.tcmodel.result.stat.Statistics; import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull; @@ -354,18 +355,28 @@ public class IgniteTeamcityHelper implements ITeamcity { private List<BuildRef> getBuildHistory(@Nullable String buildTypeId, @Nullable String branchName, boolean dfltFilter, - @Nullable String state) { + @Nullable String state){ + + return getBuildHistory(buildTypeId, branchName, dfltFilter, state, null); + } + + private List<BuildRef> getBuildHistory(@Nullable String buildTypeId, + @Nullable String branchName, + boolean dfltFilter, + @Nullable String state, + @Nullable Long cnt) { String btFilter = isNullOrEmpty(buildTypeId) ? "" : ",buildType:" + buildTypeId + ""; String stateFilter = isNullOrEmpty(state) ? "" : (",state:" + state); - String brachFilter = isNullOrEmpty(branchName) ? "" :",branch:" + branchName; + String branchFilter = isNullOrEmpty(branchName) ? "" :",branch:" + branchName; + long cntFilter = cnt == null ? DEFAULT_BUILDS_COUNT : cnt; return sendGetXmlParseJaxb(host + "app/rest/latest/builds" + "?locator=" + "defaultFilter:" + dfltFilter + btFilter + stateFilter - + brachFilter - + ",count:1000", Builds.class).getBuildsNonNull(); + + branchFilter + + ",count:" + cntFilter, Builds.class).getBuildsNonNull(); } public BuildTypeFull getBuildType(String buildTypeId) { @@ -378,7 +389,16 @@ public class IgniteTeamcityHelper implements ITeamcity { } public ProblemOccurrences getProblems(Build build) { - return getJaxbUsingHref(build.problemOccurrences.href, ProblemOccurrences.class); + if (build.problemOccurrences != null) { + ProblemOccurrences problemOccurrences = getJaxbUsingHref(build.problemOccurrences.href, ProblemOccurrences.class); + + problemOccurrences.problemOccurrences + .forEach(p -> p.buildRef = build); + + return problemOccurrences; + } + else + return new ProblemOccurrences(); } public TestOccurrences getTests(String href, String normalizedBranch) { @@ -401,6 +421,8 @@ public class IgniteTeamcityHelper implements ITeamcity { return getJaxbUsingHref(href, ChangesList.class); } + public IssuesUsagesList getIssuesUsagesList(String href) { return getJaxbUsingHref(href, IssuesUsagesList.class); } + private <T> T getJaxbUsingHref(String href, Class<T> elem) { return sendGetXmlParseJaxb(host + (href.startsWith("/") ? href.substring(1) : href), elem); } @@ -411,17 +433,29 @@ public class IgniteTeamcityHelper implements ITeamcity { /** {@inheritDoc} */ @Override public List<BuildRef> getFinishedBuilds(String projectId, String branch) { + + return getFinishedBuilds(projectId, branch, null); + } + + /** {@inheritDoc} */ + public List<BuildRef> getFinishedBuilds(String projectId, + String branch, Long cnt) { List<BuildRef> finished = getBuildHistory(projectId, UrlUtil.escape(branch), true, - null); + null, cnt); return finished.stream().filter(BuildRef::isNotCancelled).collect(Collectors.toList()); } /** {@inheritDoc} */ @Override public List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String projectId, String branch) { - return getBuildsInState(projectId, branch, BuildRef.STATE_FINISHED); + return getBuildsInState(projectId, branch, BuildRef.STATE_FINISHED, null); + } + + /** {@inheritDoc} */ + @Override public List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String projectId, String branch, Long cnt) { + return getBuildsInState(projectId, branch, BuildRef.STATE_FINISHED, cnt); } /** {@inheritDoc} */ @@ -437,14 +471,23 @@ public class IgniteTeamcityHelper implements ITeamcity { private List<BuildRef> getBuildsInState( @Nullable final String projectId, @Nullable final String branch, - @Nonnull final String state) { + @Nonnull final String state, + @Nullable final Long cnt) { List<BuildRef> finished = getBuildHistory(projectId, UrlUtil.escape(branch), false, - state); + state, cnt); return finished.stream().filter(BuildRef::isNotCancelled).collect(Collectors.toList()); } + private List<BuildRef> getBuildsInState( + @Nullable final String projectId, + @Nullable final String branch, + @Nonnull final String state) { + + return getBuildsInState(projectId, branch, state, null); + } + public String serverId() { return tcName; } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/Expirable.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/Expirable.java index 16a9f2b..32d176a 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/Expirable.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/Expirable.java @@ -27,14 +27,16 @@ import org.apache.ignite.ci.db.Persisted; public class Expirable<D> { private final long ts; private final D data; + private final long cnt; public Expirable(D data) { - this(System.currentTimeMillis(), data); + this(System.currentTimeMillis(), 1, data); } - public Expirable(long ts, D data) { + public Expirable(long ts, long cnt, D data) { this.ts = ts; this.data = data; + this.cnt = cnt; } public long getTs() { @@ -45,6 +47,10 @@ public class Expirable<D> { return data; } + public long getCnt(){ + return cnt; + } + public long getAgeMs() { return System.currentTimeMillis() - ts; } @@ -52,4 +58,8 @@ public class Expirable<D> { public boolean isAgeLessThanSecs(int seconds) { return getAgeMs() < TimeUnit.SECONDS.toMillis(seconds); } + + public boolean hasCounterGreaterThan(long cnt){ + return cnt < this.cnt; + } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java index 7b62e23..4a9d47c 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java @@ -125,7 +125,7 @@ public class MultBuildRunCtx implements ISuiteResults { public boolean hasNontestBuildProblem() { return problems != null && problems.stream().anyMatch(problem -> !problem.isFailedTests() - && !problem.isShaphotDepProblem() + && !problem.isSnapshotDepProblem() && !ProblemOccurrence.BUILD_FAILURE_ON_MESSAGE.equals(problem.type)); //todo what to do with BuildFailureOnMessage, now it is ignored } @@ -135,7 +135,7 @@ public class MultBuildRunCtx implements ISuiteResults { } private Optional<ProblemOccurrence> getBuildProblemExceptTestOrSnapshot() { - return problems.stream().filter(p -> !p.isFailedTests() && !p.isShaphotDepProblem()).findAny(); + return problems.stream().filter(p -> !p.isFailedTests() && !p.isSnapshotDepProblem()).findAny(); } public List<SingleBuildRunCtx> getBuilds() { @@ -244,7 +244,7 @@ public class MultBuildRunCtx implements ISuiteResults { Stream<ProblemOccurrence> stream = problems.stream().filter(p -> !p.isFailedTests() - && !p.isShaphotDepProblem() + && !p.isSnapshotDepProblem() && !p.isExecutionTimeout() && !p.isJvmCrash() && !p.isExitCode() diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/IssueRef.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/ProblemRef.java similarity index 83% copy from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/IssueRef.java copy to ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/ProblemRef.java index d64ddf6..7c04fbd 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/IssueRef.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/ProblemRef.java @@ -17,11 +17,15 @@ package org.apache.ignite.ci.issue; -public class IssueRef { +/** + * Reference to some Issue with current suite or test detected by the Bot. + * Currently contains only display name. + */ +public class ProblemRef { public String name; public String webUrl; - public IssueRef(String name) { + public ProblemRef(String name) { this.name = name; } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/hist/BuildRef.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/hist/BuildRef.java index 5022985..0e6202b 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/hist/BuildRef.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/hist/BuildRef.java @@ -52,6 +52,8 @@ public class BuildRef extends AbstractRef { @XmlAttribute public Boolean composite; + @XmlAttribute public String webUrl; + /** * @return Build ID */ diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/Build.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/Build.java index ece1c8f..9c6a9cd 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/Build.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/Build.java @@ -58,6 +58,8 @@ public class Build extends BuildRef implements IVersionedEntity { @XmlElement(name = "statistics") public StatisticsRef statisticsRef; + @XmlElement(name = "relatedIssues") public RelatedIssuesRef relatedIssuesRef; + /** Changes not included into build.*/ @XmlElement(name = "lastChanges") public ChangesList lastChanges; @@ -87,11 +89,19 @@ public class Build extends BuildRef implements IVersionedEntity { } public Date getFinishDate() { + return getDate(finishDate); + } + + public Date getStartDate() { + return getDate(startDate); + } + + private Date getDate(String date) { try { - if (finishDate == null) + if (date == null) return null; SimpleDateFormat f = new SimpleDateFormat("yyyyMMdd'T'HHmmssZ"); - return f.parse(finishDate); + return f.parse(date); } catch (ParseException e) { throw new IllegalStateException(e); @@ -127,4 +137,4 @@ public class Build extends BuildRef implements IVersionedEntity { public void setTriggered(Triggered triggered) { this.triggered = triggered; } -} +} \ No newline at end of file diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/ProblemOccurrencesRef.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/ProblemOccurrencesRef.java index 7bc3ff1..a382ddb 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/ProblemOccurrencesRef.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/ProblemOccurrencesRef.java @@ -25,4 +25,8 @@ import javax.xml.bind.annotation.XmlAttribute; public class ProblemOccurrencesRef { /** Href without host name to obtain full problems list. */ @XmlAttribute public String href; + + @XmlAttribute public long count; + + @XmlAttribute public long newFailed; } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/IssueRef.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/RelatedIssuesRef.java similarity index 82% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/IssueRef.java rename to ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/RelatedIssuesRef.java index d64ddf6..81e1157 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/IssueRef.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/RelatedIssuesRef.java @@ -15,13 +15,11 @@ * limitations under the License. */ -package org.apache.ignite.ci.issue; +package org.apache.ignite.ci.tcmodel.result; -public class IssueRef { - public String name; - public String webUrl; - - public IssueRef(String name) { - this.name = name; - } +/** + * Related issues reference. + */ +public class RelatedIssuesRef extends AbstractRef { } + diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/TestOccurrencesRef.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/TestOccurrencesRef.java index 6cbda4a..b250e98 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/TestOccurrencesRef.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/TestOccurrencesRef.java @@ -29,5 +29,7 @@ public class TestOccurrencesRef { @XmlAttribute public Integer count; @XmlAttribute public Integer passed; @XmlAttribute public Integer failed; + @XmlAttribute public Integer newFailed; + @XmlAttribute public Integer ignored; @XmlAttribute public Integer muted; } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/TestOccurrencesRef.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/issues/IssueRef.java similarity index 63% copy from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/TestOccurrencesRef.java copy to ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/issues/IssueRef.java index 6cbda4a..09ea91a 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/TestOccurrencesRef.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/issues/IssueRef.java @@ -15,19 +15,24 @@ * limitations under the License. */ -package org.apache.ignite.ci.tcmodel.result; +package org.apache.ignite.ci.tcmodel.result.issues; import javax.xml.bind.annotation.XmlAttribute; /** - * Test occurrences reference + * Issue short version from list of build's related issues. + * + * See example of XML, e.g. here + * https://ci.ignite.apache.org/app/rest/latest/builds/id:1694977/relatedIssues */ -public class TestOccurrencesRef { - /** Href without host name to obtain full tests list. */ - @XmlAttribute public String href; +public class IssueRef { + @XmlAttribute public String id; + @XmlAttribute public String url; - @XmlAttribute public Integer count; - @XmlAttribute public Integer passed; - @XmlAttribute public Integer failed; - @XmlAttribute public Integer muted; + @Override public String toString() { + return "IssueRef{" + + "id='" + id + '\'' + + ", url='" + url + '\'' + + '}'; + } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/problems/ProblemOccurrences.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/issues/IssueUsage.java similarity index 58% copy from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/problems/ProblemOccurrences.java copy to ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/issues/IssueUsage.java index 43166d4..ae57c4d 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/problems/ProblemOccurrences.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/issues/IssueUsage.java @@ -15,23 +15,29 @@ * limitations under the License. */ -package org.apache.ignite.ci.tcmodel.result.problems; +package org.apache.ignite.ci.tcmodel.result.issues; -import java.util.Collections; -import java.util.List; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; -import org.apache.ignite.ci.tcmodel.result.ProblemOccurrencesRef; +import org.apache.ignite.ci.tcmodel.changes.ChangesList; /** - * List of problems associated with build. + * Build's related issue from TC. + * + * See example of XML, e.g. here + * https://ci.ignite.apache.org/app/rest/latest/builds/id:1694977/relatedIssues */ -@XmlRootElement(name = "problemOccurrences") -public class ProblemOccurrences extends ProblemOccurrencesRef { - @XmlElement(name = "problemOccurrence") - private List<ProblemOccurrence> problemOccurrences; +@XmlRootElement(name = "IssueUsage") +@XmlAccessorType(XmlAccessType.FIELD) +public class IssueUsage { + @XmlElement(name = "issue") + private IssueRef issue; + @XmlElement(name = "changes") + private ChangesList changesList; - public List<ProblemOccurrence> getProblemsNonNull() { - return problemOccurrences == null ? Collections.emptyList() : problemOccurrences; + public IssueRef getIssue(){ + return issue; } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/problems/ProblemOccurrences.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/issues/IssuesUsagesList.java similarity index 57% copy from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/problems/ProblemOccurrences.java copy to ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/issues/IssuesUsagesList.java index 43166d4..1212ab6 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/problems/ProblemOccurrences.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/issues/IssuesUsagesList.java @@ -15,23 +15,35 @@ * limitations under the License. */ -package org.apache.ignite.ci.tcmodel.result.problems; +package org.apache.ignite.ci.tcmodel.result.issues; import java.util.Collections; import java.util.List; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; -import org.apache.ignite.ci.tcmodel.result.ProblemOccurrencesRef; /** - * List of problems associated with build. + * List of build's related issues from TC. + * + * See example of XML, e.g. here + * https://ci.ignite.apache.org/app/rest/latest/builds/id:1694977/relatedIssues */ -@XmlRootElement(name = "problemOccurrences") -public class ProblemOccurrences extends ProblemOccurrencesRef { - @XmlElement(name = "problemOccurrence") - private List<ProblemOccurrence> problemOccurrences; +@XmlRootElement(name = "issuesUsages") +public class IssuesUsagesList { + @XmlElement(name = "issueUsage") + private List<IssueUsage> issuesUsages; + + @XmlElement Integer count; + + @XmlElement String href; + + public List<IssueUsage> getIssuesUsagesNonNull() { + return issuesUsages == null ? Collections.emptyList() : issuesUsages; + } - public List<ProblemOccurrence> getProblemsNonNull() { - return problemOccurrences == null ? Collections.emptyList() : problemOccurrences; + @Override public String toString() { + return "IssuesUsagesList{" + + "issuesUsages=" + issuesUsages + + '}'; } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/problems/ProblemOccurrence.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/problems/ProblemOccurrence.java index 92868bc..b8ae4f0 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/problems/ProblemOccurrence.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/problems/ProblemOccurrence.java @@ -18,34 +18,41 @@ package org.apache.ignite.ci.tcmodel.result.problems; import javax.xml.bind.annotation.XmlAttribute; +import org.apache.ignite.ci.tcmodel.hist.BuildRef; /** * One build problem. Contains its type. */ public class ProblemOccurrence { public static final String BUILD_FAILURE_ON_MESSAGE = "BuildFailureOnMessage"; - private static final String TC_EXIT_CODE = "TC_EXIT_CODE"; - private static final String TC_OOME = "TC_OOME"; + public static final String TC_EXIT_CODE = "TC_EXIT_CODE"; + public static final String TC_OOME = "TC_OOME"; + public static final String TC_EXECUTION_TIMEOUT = "TC_EXECUTION_TIMEOUT"; + public static final String TC_FAILED_TESTS = "TC_FAILED_TESTS"; + public static final String TC_JVM_CRASH = "TC_JVM_CRASH"; + public static final String OTHER = "OTHER"; + public static final String SNAPSHOT_DEPENDENCY_ERROR = "SNAPSHOT_DEPENDENCY_ERROR_BUILD_PROCEEDS_TYPE"; @XmlAttribute public String id; @XmlAttribute public String identity; @XmlAttribute public String type; @XmlAttribute public String href; + public BuildRef buildRef; public boolean isExecutionTimeout() { - return "TC_EXECUTION_TIMEOUT".equals(type); + return TC_EXECUTION_TIMEOUT.equals(type); } public boolean isFailedTests() { - return "TC_FAILED_TESTS".equals(type); + return TC_FAILED_TESTS.equals(type); } - public boolean isShaphotDepProblem() { - return "SNAPSHOT_DEPENDENCY_ERROR_BUILD_PROCEEDS_TYPE".equals(type); + public boolean isSnapshotDepProblem() { + return SNAPSHOT_DEPENDENCY_ERROR.equals(type); } public boolean isJvmCrash() { - return "TC_JVM_CRASH".equals(type); + return TC_JVM_CRASH.equals(type); } public boolean isOome() { @@ -55,4 +62,13 @@ public class ProblemOccurrence { public boolean isExitCode() { return TC_EXIT_CODE.equals(type); } + + public boolean isOther(){ + return !isFailedTests() + && !isSnapshotDepProblem() + && !isExecutionTimeout() + && !isJvmCrash() + && !isExitCode() + && !isOome(); + } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/problems/ProblemOccurrences.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/problems/ProblemOccurrences.java index 43166d4..c95a7bc 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/problems/ProblemOccurrences.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/result/problems/ProblemOccurrences.java @@ -29,9 +29,10 @@ import org.apache.ignite.ci.tcmodel.result.ProblemOccurrencesRef; @XmlRootElement(name = "problemOccurrences") public class ProblemOccurrences extends ProblemOccurrencesRef { @XmlElement(name = "problemOccurrence") - private List<ProblemOccurrence> problemOccurrences; + public List<ProblemOccurrence> problemOccurrences; public List<ProblemOccurrence> getProblemsNonNull() { return problemOccurrences == null ? Collections.emptyList() : problemOccurrences; } } + diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/BuildStatisticsSummary.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/BuildStatisticsSummary.java new file mode 100644 index 0000000..2b73771 --- /dev/null +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/BuildStatisticsSummary.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ci.web.model.current; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import org.apache.ignite.ci.ITeamcity; +import org.apache.ignite.ci.tcmodel.hist.BuildRef; +import org.apache.ignite.ci.tcmodel.result.Build; +import org.apache.ignite.ci.tcmodel.result.issues.IssueRef; +import org.apache.ignite.ci.tcmodel.result.issues.IssueUsage; +import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence; +import org.apache.ignite.ci.util.TimeUtil; +import org.apache.ignite.ci.web.IBackgroundUpdatable; + +/** + * Summary of build statistics. + */ +public class BuildStatisticsSummary extends UpdateInfo implements IBackgroundUpdatable { + /** Build with test and problems references. */ + public Build build; + + /** List of problem occurrences. */ + private List<ProblemOccurrence> problems; + + /** List of related issues. */ + public List<IssueRef> relatedIssues; + + /** Duration printable. */ + public String durationPrintable; + + /** Short build run result (without snapshot-dependencies printable result). */ + public String shortRes; + + /** Snapshot-dependencies build run result. */ + public List<String> fullRes; + + /** Snapshot-dependency. */ + private List<BuildRef> snapshotDependencies; + + /** Build problems count. */ + public long problemsCount; + + public BuildStatisticsSummary(Build build){ + this.build = build; + } + + /** Initialize build statistics. */ + public void initialize(@Nonnull final ITeamcity teamcity) { + + if (build.isFakeStub()) + return; + + relatedIssues = teamcity.getIssuesUsagesList(build.relatedIssuesRef.href).getIssuesUsagesNonNull().stream() + .map(IssueUsage::getIssue).collect(Collectors.toList()); + + durationPrintable = TimeUtil + .getDurationPrintable(build.getFinishDate().getTime() - build.getStartDate().getTime()); + + snapshotDependencies = getSnapshotDependencies(teamcity, build); + + problems = getProblems(teamcity); + + shortRes = getShortRes(); + + fullRes = getFullRes(); + + problemsCount = getAllProblemsCount(null); + } + + private long getExecutionTimeoutCount(String buildTypeId) { + return getProblemsStream(buildTypeId).filter(ProblemOccurrence::isExecutionTimeout).count(); + } + + private long getJvmCrashProblemCount(String buildTypeId) { + return getProblemsStream(buildTypeId).filter(ProblemOccurrence::isJvmCrash).count(); + } + + private long getExitCodeProblemsCount(String buildTypeId) { + return getProblemsStream(buildTypeId).filter(ProblemOccurrence::isExitCode).count(); + } + + private long getOomeProblemCount(String buildTypeId) { + return getProblemsStream(buildTypeId).filter(ProblemOccurrence::isOome).count(); + } + + private long getFailedTestsProblemCount(String buildTypeId) { + return getProblemsStream(buildTypeId).filter(ProblemOccurrence::isFailedTests).count(); + } + + private long getSnapshotDepProblemCount(String buildTypeId) { + return getProblemsStream(buildTypeId).filter(ProblemOccurrence::isSnapshotDepProblem).count(); + } + + private long getOtherProblemCount(String buildTypeId) { + return getProblemsStream(buildTypeId).filter(ProblemOccurrence::isOther).count(); + } + + private long getAllProblemsCount(String buildTypeId) { + return getProblemsStream(buildTypeId).count(); + } + + /** + * Problems for all snapshot-dependencies. + * + * @param teamcity Teamcity. + */ + private List<ProblemOccurrence> getProblems(@Nonnull final ITeamcity teamcity){ + if (snapshotDependencies == null) + return Collections.emptyList(); + + List<ProblemOccurrence> problemOccurrences = new ArrayList<>(); + + List<BuildRef> snapshotDependencyWithProblems = getSnapshotDependenciesWithProblems(); + + for (BuildRef buildRef : snapshotDependencyWithProblems) + problemOccurrences.addAll(teamcity + .getProblems(teamcity.getBuild(buildRef.href)) + .getProblemsNonNull()); + + return problemOccurrences; + } + + /** + * Snapshot-dependencies for build. + * + * @param teamcity Teamcity. + * @param buildRef Build reference. + */ + private List<BuildRef> getSnapshotDependencies(@Nonnull final ITeamcity teamcity, BuildRef buildRef){ + List<BuildRef> snapshotDependencies = new ArrayList<>(); + + if (buildRef.isComposite()){ + Build build = teamcity.getBuild(buildRef.href); + + for (BuildRef snDep : build.getSnapshotDependenciesNonNull()) + snapshotDependencies.addAll(getSnapshotDependencies(teamcity, snDep)); + } else + snapshotDependencies.add(buildRef); + + return snapshotDependencies; + } + + /** + * Snapshot-dependencies without status "Success". + */ + private List<BuildRef> getSnapshotDependenciesWithProblems(){ + if (snapshotDependencies == null) + return Collections.emptyList(); + + return snapshotDependencies.stream() + .filter(b -> !b.isSuccess()) + .collect(Collectors.toList()); + } + + /** + * @param buildTypeId Build type id (if null - for all problems). + */ + private Stream<ProblemOccurrence> getProblemsStream(String buildTypeId) { + if (problems == null) + return Stream.empty(); + + return problems.stream() + .filter(Objects::nonNull) + .filter(p -> buildTypeId == null || buildTypeId.equals(p.buildRef.buildTypeId) + ); + } + + /** + * Full build run result (snapshot-dependencies printable result). + * + * @return printable result; + */ + private List<String> getFullRes(){ + List<String> fullRes = new ArrayList<>(); + + List<BuildRef> snapshotDependencyWithProblems = getSnapshotDependenciesWithProblems(); + + for (BuildRef build : snapshotDependencyWithProblems) + fullRes.add(getRes(build.buildTypeId)); + + return fullRes.stream() + .sorted() + .collect(Collectors.toList()); + } + + /** + * Short build run result (without snapshot-dependencies printable result). + * + * @return printable result; + */ + private String getShortRes(){ + return getRes(null); + } + + /** + * Build run result for buildTypeId. + * + * @param buildTypeId buildTypeId. + * + * @return printable result. + */ + private String getRes(String buildTypeId){ + StringBuilder res = new StringBuilder(); + + addKnownProblemCnt(res, ProblemOccurrence.TC_EXECUTION_TIMEOUT, getExecutionTimeoutCount(buildTypeId)); + addKnownProblemCnt(res, ProblemOccurrence.TC_JVM_CRASH, getJvmCrashProblemCount(buildTypeId)); + addKnownProblemCnt(res, ProblemOccurrence.TC_OOME, getOomeProblemCount(buildTypeId)); + addKnownProblemCnt(res, ProblemOccurrence.TC_EXIT_CODE, getExitCodeProblemsCount(buildTypeId)); + addKnownProblemCnt(res, ProblemOccurrence.TC_FAILED_TESTS, getFailedTestsProblemCount(buildTypeId)); + addKnownProblemCnt(res, ProblemOccurrence.SNAPSHOT_DEPENDENCY_ERROR, getSnapshotDepProblemCount(buildTypeId)); + addKnownProblemCnt(res, ProblemOccurrence.OTHER, getOtherProblemCount(buildTypeId)); + + res.insert(0, (buildTypeId != null ? buildTypeId : "TOTAL") + " [" + getAllProblemsCount(buildTypeId) + "]" + + (res.length() != 0 ? ": " : " ")); + + return res.toString(); + } + + /** + * @param res Response. + * @param nme Name of problem. + * @param execToCnt Execute to count. + */ + private void addKnownProblemCnt(StringBuilder res, String nme, long execToCnt) { + if (execToCnt > 0) { + if (res.length() > 0) + res.append(", "); + + res.append(nme) + .append(execToCnt > 1 ? " [" + execToCnt + "]" : ""); + } + } + + /** {@inheritDoc} */ + @Override public void setUpdateRequired(boolean update) { + updateRequired = update; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (!(o instanceof BuildStatisticsSummary)) + return false; + + BuildStatisticsSummary that = (BuildStatisticsSummary)o; + + return problemsCount == that.problemsCount && + Objects.equals(build, that.build) && + Objects.equals(problems, that.problems) && + Objects.equals(relatedIssues, that.relatedIssues) && + Objects.equals(durationPrintable, that.durationPrintable) && + Objects.equals(getShortRes(), that.getShortRes()) && + Objects.equals(getFullRes(), that.getFullRes()) && + Objects.equals(snapshotDependencies, that.snapshotDependencies); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + + return Objects.hash(build, problems, relatedIssues, durationPrintable, getShortRes(), getFullRes(), + snapshotDependencies, problemsCount); + } +} \ No newline at end of file diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java index 1a6b6c6..91945c1 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java @@ -38,7 +38,7 @@ import org.apache.ignite.ci.analysis.SuiteInBranch; import org.apache.ignite.ci.analysis.TestInBranch; import org.apache.ignite.ci.analysis.TestLogCheckResult; import org.apache.ignite.ci.issue.EventTemplates; -import org.apache.ignite.ci.issue.IssueRef; +import org.apache.ignite.ci.issue.ProblemRef; import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull; import org.apache.ignite.ci.web.rest.GetBuildLog; import org.jetbrains.annotations.NotNull; @@ -105,7 +105,7 @@ import static org.apache.ignite.ci.util.UrlUtil.escape; public String durationPrintable; - @Nullable public IssueRef problemRef; + @Nullable public ProblemRef problemRef; public void initFromContext(@Nonnull final ITeamcity teamcity, @@ -244,12 +244,12 @@ import static org.apache.ignite.ci.util.UrlUtil.escape; RunStat.TestId testId = latestRunsSrc.detectTemplate(EventTemplates.newFailure); if (testId != null) - problemRef = new IssueRef("New Failure"); + problemRef = new ProblemRef("New Failure"); RunStat.TestId buildIdCritical = latestRunsSrc.detectTemplate(EventTemplates.newCriticalFailure); if (buildIdCritical != null) - problemRef = new IssueRef("New Critical Failure"); + problemRef = new ProblemRef("New Critical Failure"); } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java index 38b9399..559fb3e 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java @@ -32,7 +32,7 @@ import org.apache.ignite.ci.analysis.ITestFailureOccurrences; import org.apache.ignite.ci.analysis.RunStat; import org.apache.ignite.ci.analysis.TestInBranch; import org.apache.ignite.ci.issue.EventTemplates; -import org.apache.ignite.ci.issue.IssueRef; +import org.apache.ignite.ci.issue.ProblemRef; import org.apache.ignite.ci.logs.LogMsgToWarn; import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull; @@ -78,7 +78,7 @@ import static org.apache.ignite.ci.web.model.current.SuiteCurrentStatus.branchFo public List<String> warnings = new ArrayList<>(); - @Nullable public IssueRef problemRef; + @Nullable public ProblemRef problemRef; @Nullable public String flakyComments; @@ -209,12 +209,12 @@ import static org.apache.ignite.ci.web.model.current.SuiteCurrentStatus.branchFo RunStat.TestId testId = latestRunsSrcInBranch.detectTemplate(EventTemplates.newFailure); if (testId != null) - problemRef = new IssueRef("New Failure"); + problemRef = new ProblemRef("New Failure"); RunStat.TestId recentContributedTestId = latestRunsSrcInBranch.detectTemplate(EventTemplates.newContributedTestFailure); if (recentContributedTestId != null) - problemRef = new IssueRef("Recently contributed test failure"); + problemRef = new ProblemRef("Recently contributed test failure"); flakyComments = latestRunsSrcInBranch.getFlakyComments(); } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/build/GetBuildTestFailures.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/build/GetBuildTestFailures.java index 7ba09a2..5c860c6 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/build/GetBuildTestFailures.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/build/GetBuildTestFailures.java @@ -17,13 +17,11 @@ package org.apache.ignite.ci.web.rest.build; +import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; -import javax.annotation.Nonnull; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.GET; @@ -36,27 +34,30 @@ import org.apache.ignite.ci.BuildChainProcessor; import org.apache.ignite.ci.IAnalyticsEnabledTeamcity; import org.apache.ignite.ci.ITcHelper; import org.apache.ignite.ci.ITeamcity; -import org.apache.ignite.ci.IgnitePersistentTeamcity; import org.apache.ignite.ci.analysis.FullChainRunCtx; import org.apache.ignite.ci.analysis.mode.LatestRebuildMode; import org.apache.ignite.ci.analysis.mode.ProcessLogsMode; import org.apache.ignite.ci.tcmodel.hist.BuildRef; import org.apache.ignite.ci.user.ICredentialsProv; -import org.apache.ignite.ci.web.rest.login.ServiceUnauthorizedException; import org.apache.ignite.ci.web.BackgroundUpdater; import org.apache.ignite.ci.web.CtxListener; +import org.apache.ignite.ci.web.model.current.BuildStatisticsSummary; import org.apache.ignite.ci.web.model.current.ChainAtServerCurrentStatus; import org.apache.ignite.ci.web.model.current.TestFailuresSummary; import org.apache.ignite.ci.web.model.current.UpdateInfo; +import org.apache.ignite.ci.web.rest.login.ServiceUnauthorizedException; import org.apache.ignite.ci.web.rest.parms.FullQueryParams; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import static com.google.common.base.Strings.isNullOrEmpty; + @Path(GetBuildTestFailures.BUILD) @Produces(MediaType.APPLICATION_JSON) public class GetBuildTestFailures { public static final String BUILD = "build"; public static final String TEST_FAILURES_SUMMARY_CACHE_NAME = BUILD + "TestFailuresSummary"; + public static final String BUILDS_STATISTICS_SUMMARY_CACHE_NAME = BUILD + "sStatisticsSummary"; @Context private ServletContext context; @@ -88,7 +89,7 @@ public class GetBuildTestFailures { @QueryParam("serverId") String serverId, @QueryParam("buildId") Integer buildId, @Nullable @QueryParam("checkAllLogs") Boolean checkAllLogs) - throws ServiceUnauthorizedException { + throws ServiceUnauthorizedException { final BackgroundUpdater updater = CtxListener.getBackgroundUpdater(context); @@ -131,7 +132,6 @@ public class GetBuildTestFailures { (checkAllLogs != null && checkAllLogs) ? ProcessLogsMode.ALL : ProcessLogsMode.SUITE_NOT_COMPLETE, false, false, teamcity, failRateBranch); - pubCtx.ifPresent(ctx -> { final ChainAtServerCurrentStatus chainStatus = new ChainAtServerCurrentStatus(serverId, ctx.branchName()); @@ -149,4 +149,67 @@ public class GetBuildTestFailures { return res; } -} + + @GET + @Path("history") + public List<BuildStatisticsSummary> getBuildsHistory( + @Nullable @QueryParam("server") String server, + @Nullable @QueryParam("buildType") String buildType, + @Nullable @QueryParam("branch") String branch, + @Nullable @QueryParam("count") Long count) + throws ServiceUnauthorizedException { + + String srvId = isNullOrEmpty(server) ? "apache" : server; + String buildTypeId = isNullOrEmpty(buildType) ? "IgniteTests24Java8_RunAll" : buildType; + String branchName = isNullOrEmpty(branch) ? "refs/heads/master" : branch; + long cnt = count == null ? 50 : count; + + final BackgroundUpdater updater = CtxListener.getBackgroundUpdater(context); + + final ITcHelper tcHelper = CtxListener.getTcHelper(context); + + final ICredentialsProv prov = ICredentialsProv.get(req); + + try (IAnalyticsEnabledTeamcity teamcity = tcHelper.server(srvId, prov)) { + + int[] finishedBuilds = teamcity.getBuildNumbersFromHistory(buildTypeId, branchName, cnt); + + List<BuildStatisticsSummary> buildsStatistics = new ArrayList<>(); + + for (int i = 0; i < finishedBuilds.length; i++) { + int buildId = finishedBuilds[i]; + + FullQueryParams param = new FullQueryParams(); + param.setBuildId(buildId); + param.setBranch(branchName); + param.setServerId(srvId); + + BuildStatisticsSummary buildsStatistic = updater.get( + BUILDS_STATISTICS_SUMMARY_CACHE_NAME, prov, param, + (k) -> getBuildStatisticsSummaryNoCache(srvId, buildId), false); + + if (!buildsStatistic.build.isFakeStub()) + buildsStatistics.add(buildsStatistic); + } + + return buildsStatistics; + } + } + + private BuildStatisticsSummary getBuildStatisticsSummaryNoCache(String server, int buildId) { + + String srvId = isNullOrEmpty(server) ? "apache" : server; + + final ITcHelper tcHelper = CtxListener.getTcHelper(context); + + final ICredentialsProv creds = ICredentialsProv.get(req); + + try (IAnalyticsEnabledTeamcity teamcity = tcHelper.server(srvId, creds)) { + + BuildStatisticsSummary stat = new BuildStatisticsSummary(teamcity.getBuild(buildId)); + stat.initialize(teamcity); + + return stat; + } + } +} \ No newline at end of file diff --git a/ignite-tc-helper-web/src/main/webapp/index.html b/ignite-tc-helper-web/src/main/webapp/index.html index 2466f17..c642863 100644 --- a/ignite-tc-helper-web/src/main/webapp/index.html +++ b/ignite-tc-helper-web/src/main/webapp/index.html @@ -6,7 +6,7 @@ <link rel="stylesheet" href="https://code.jquery.com/ui/1.12.1/themes/base/jquery-ui.css";> <link rel="stylesheet" href="css/style-1.5.css"> - + <script src="https://code.jquery.com/jquery-1.12.4.js";></script> <script src="https://code.jquery.com/ui/1.12.1/jquery-ui.js";></script> <script src="js/common-1.6.js"></script> @@ -45,7 +45,7 @@ function loadData() { error: showErrInLoadStatus }); - $.ajax({ + $.ajax({ url: "rest/branches/getServerIds", success: function(result) { $("#loadStatus").html(""); @@ -122,7 +122,8 @@ Statistics: <br> <a href="restpretty.html?url=top/failingSuite">Top failing suites</a> (JSON) <br> <!--<a href="./status">Current Build Status (obsolete)</a><br>--> <br> - +<a href="statistics.html?buildType=IgniteTests24Java8_RunAll&branch=refs/heads/master&count=50">Master's branch statistics</a><br> +<br> Check branch/PR: <br> <div id="suitesForPrCheck"></div> <br> diff --git a/ignite-tc-helper-web/src/main/webapp/statistics.html b/ignite-tc-helper-web/src/main/webapp/statistics.html new file mode 100644 index 0000000..ffdf9bb --- /dev/null +++ b/ignite-tc-helper-web/src/main/webapp/statistics.html @@ -0,0 +1,203 @@ +<html> +<head> + <title>Ignite Teamcity - statistics master's branch</title> + <link rel="icon" href="img/leaf-icon-png-7066.png"> + <link rel="stylesheet" href="https://code.jquery.com/ui/1.12.1/themes/base/jquery-ui.css";> + <link rel="stylesheet" href="css/style-1.5.css"> + + <script src="https://code.jquery.com/jquery-1.12.4.js";></script> + <script src="https://code.jquery.com/ui/1.12.1/jquery-ui.js";></script> + <script src="js/common-1.6.js"></script> + <script src="js/testfails-2.0.js"></script> + <style> + table { + width: 70%; + border-collapse: collapse; + } + td, th { + + padding: 10px 5px 10px 5px; + } + th { + text-align: left; + padding: 5px; + background-color: #f5f5ff; + color: #000000; + } + tr:nth-child(odd) { background-color: #fafaff; } + </style> +</head> +<body> +<script> + var g_shownDataHashCodeHex = "" + var g_updTimer = null; + + $(document).ready(function() { + $.getScript("js/testfails-2.0.js", function(data, textStatus, jqxhr){ }); + + $( document ).tooltip(); + loadData(); + //todo fix setInterval( function() { checkForUpdate(); }, 30000); + + $.ajax({ url: "rest/branches/version", success: showVersionInfo, error: showErrInLoadStatus }); + + if(g_updTimer==null) { + g_updTimer=setTimeout(tstTimeout, 3200); + } + setInterval(tstTimeout, 10000); + }); + + function tstTimeout() { + + if(g_updTimer!=null) { + clearTimeout(g_updTimer); + g_updTimer=null; + } + + + var d = new Date(); + var n = d.getTime(); + + // $(document.body).prepend("timeout at "+n+"<br>"); + + + if(g_updTimer==null) { + g_updTimer=setTimeout(tstTimeout, 3200); + } + } + + function parmsForRest() { + var curReqParms = ""; + var server = findGetParameter("server"); + if(server!=null) { + curReqParms += "&server=" + server; + } + + var buildType = findGetParameter("buildType"); + if(buildType!=null) { + curReqParms += "&buildType=" + buildType; + } + + var branch = findGetParameter("branch"); + if(branch!=null) { + curReqParms += "&branch=" + branch; + } + + var count = findGetParameter("count"); + if(count!=null) { + curReqParms += "&count=" + count; + } + + curReqParms = curReqParms.replace("&","?"); + + return curReqParms; + } + + function loadData() { + var curFailuresUrl = "rest/build/history" + parmsForRest(); + + $("#loadStatus").html("<img src='https://www.wallies.com/filebin/images/loading_apple.gif' width=20px height=20px> Please wait"); + $.ajax({ + url: curFailuresUrl, + success: function(result) { + if(result.updateRequired || (isDefinedAndFilled(result.runningUpdates) && result.runningUpdates>0)) { + setTimeout(checkForUpdate, 3000) + $("#loadStatus").html("<img src='https://www.wallies.com/filebin/images/loading_apple.gif' width=20px height=20px> Updating"); + } else { + $("#loadStatus").html(""); + } + showData(result); + g_shownDataHashCodeHex = isDefinedAndFilled(result.hashCodeHex) ? result.hashCodeHex : ""; + }, + error: showErrInLoadStatus + }); + } + function showData(result) { + $("#statistics").html(showBuildHistory(result)); + } + + function median(arr){ + arr = arr.sort(function(a, b){ return a - b; }); + var i = arr.length / 2; + return i % 1 == 0 ? (arr[i - 1] + arr[i]) / 2 : arr[Math.floor(i)]; + } + + function formatString(string) { + if (string.indexOf("_") < string.indexOf(":")) + string = string.substring(string.indexOf("_") + 1); + + return "<b>" + string.substring(0, string.indexOf(":") + 1) + "</b>" + string.substring(string.indexOf(":") + 1); + } + + function showBuildHistory(result) { + var res = "<table><tr><th>#</th><th>Id</th><th>Total tests</th><th>Failed tests</th><th>Ignored tests</th>" + + "<th>Muted tests</th><th>Total issues</th><th>Total run time</th></tr>"; + + var problemCountMin = result[0].problemsCount; + var problemCountMax = problemCountMin; + var problemCounts = []; + var average = 0; + + for (var i = 0; i < result.length; i++) { + var buildStatistics = result[i]; + + problemCounts[i] = buildStatistics.problemsCount; + + if (problemCounts[i] > problemCountMax) problemCountMax = problemCounts[i]; + + if (problemCounts[i] < problemCountMin) problemCountMin = problemCounts[i]; + + average += problemCounts[i] / buildStatistics.build.testOccurrences.failed; + + res += "<tr><td>" + (i + 1) + "</td>" + + "<td><a href='" + buildStatistics.build.webUrl + "'>" + buildStatistics.build.id + "</a></td>" + + "<td>" + buildStatistics.build.testOccurrences.count + "</td>" + + "<td>" + buildStatistics.build.testOccurrences.failed + "</td>" + + "<td>" + buildStatistics.build.testOccurrences.ignored + "</td>" + + "<td>" + buildStatistics.build.testOccurrences.muted + "</td><td>"; + + if (buildStatistics.fullRes.length > 0) { + res += "<details><summary>" + formatString(buildStatistics.shortRes) + "</summary><p>"; + + for (var k = 0; k < buildStatistics.fullRes.length; k++) { + res += (k + 1) + ". " + formatString(buildStatistics.fullRes[k]) + "<br>"; + } + + res += "</p></details><br>"; + } else { + res += "<p>" + buildStatistics.shortRes + "</p>"; + } + + var issuesArr = buildStatistics.relatedIssues; + + if (issuesArr.length > 0) { + res += "<details><summary>Related issues [" + issuesArr.length + "]</summary><p>"; + + for (var j = 0; j < issuesArr.length; j++) { + var issue = issuesArr[j]; + + if (j !== 0) res += ", "; + + res += "<a href='" + issue.url + "'>" + issue.id + "</a>"; + } + + res += "</p></details>"; + } + + res += "</td><td>" + buildStatistics.durationPrintable + "</td></tr>"; + } + + res += "</table><br>min: " + problemCountMin + "; max: " + problemCountMax + + "; median: " + (Math.round(median(problemCounts) * 1000) / 1000) + + "; average: " + (Math.round(average / result.length * 1000) / 1000) + "<br><br>"; + + return res; + + } +</script> +<br> +<div id="loadStatus"></div> +<div id="statistics"></div> +<div id="version"></div> +</body> +</html> \ No newline at end of file
