This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch ignite-10030 in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 2d6f65709213479fd2affb1a29df119d2bbff54b Author: Dmitriy Pavlov <[email protected]> AuthorDate: Sat Oct 27 17:34:39 2018 +0300 IGNITE-10030 Started Background upload of changes, problems, and statistics --- .../main/java/org/apache/ignite/ci/ITeamcity.java | 4 --- .../apache/ignite/ci/IgnitePersistentTeamcity.java | 27 ++++-------------- .../apache/ignite/ci/IgniteTeamcityConnection.java | 21 +++++++------- .../ignited/fatbuild/FatBuildCompacted.java | 10 +++++++ .../ci/teamcity/ignited/fatbuild/FatBuildDao.java | 18 ++++++------ .../ignited/fatbuild/ProactiveFatBuildSync.java | 33 ++++++++++++++++------ .../ignite/ci/teamcity/pure/ITeamcityConn.java | 3 ++ .../ignited/IgnitedTcInMemoryIntegrationTest.java | 9 +++++- .../src/test/resources/problemList.xml | 3 ++ 9 files changed, 74 insertions(+), 54 deletions(-) diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java index 12446b9..79c1b2b 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java @@ -105,10 +105,6 @@ public interface ITeamcity extends ITeamcityConn { /** */ @Deprecated - CompletableFuture<List<BuildRef>> getRunningBuilds(@Nullable String branch); - - /** */ - @Deprecated CompletableFuture<List<BuildRef>> getQueuedBuilds(@Nullable String branch); /** diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java index 3487566..8cdf0cd 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java @@ -142,12 +142,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea /** cached loads of queued builds for branch. */ private ConcurrentMap<String, CompletableFuture<List<BuildRef>>> queuedBuildsFuts = new ConcurrentHashMap<>(); - /** cached running builds for branch. */ - private ConcurrentMap<String, Expirable<List<BuildRef>>> runningBuilds = new ConcurrentHashMap<>(); - - /** cached loads of running builds for branch. */ - private ConcurrentMap<String, CompletableFuture<List<BuildRef>>> runningBuildsFuts = new ConcurrentHashMap<>(); - /** */ private ConcurrentMap<SuiteInBranch, Long> lastQueuedHistory = new ConcurrentHashMap<>(); @@ -667,20 +661,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea } - /** {@inheritDoc} */ - @Override public CompletableFuture<List<BuildRef>> getRunningBuilds(String branch) { - int defaultSecs = 60; - int secondsUseCached = getTriggerRelCacheValidSecs(defaultSecs); - - return loadAsyncIfAbsentOrExpired( - runningBuilds, - Strings.nullToEmpty(branch), - runningBuildsFuts, - teamcity::getRunningBuilds, - secondsUseCached, - secondsUseCached == defaultSecs); - } - public static int getTriggerRelCacheValidSecs(int defaultSecs) { long msSinceTrigger = System.currentTimeMillis() - lastTriggerMs; long secondsSinceTrigger = TimeUnit.MILLISECONDS.toSeconds(msSinceTrigger); @@ -1191,12 +1171,17 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea /** {@inheritDoc} */ @AutoProfiling - @Override public Build triggerBuild(String buildTypeId, String branchName, boolean cleanRebuild, boolean queueAtTop) { + @Override public Build triggerBuild(String buildTypeId, @NotNull String branchName, boolean cleanRebuild, boolean queueAtTop) { lastTriggerMs = System.currentTimeMillis(); return teamcity.triggerBuild(buildTypeId, branchName, cleanRebuild, queueAtTop); } + @Override + public ProblemOccurrences getProblems(int buildId) { + return teamcity.getProblems(buildId); + } + /** {@inheritDoc} */ @Override public void setAuthToken(String tok) { teamcity.setAuthToken(tok); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java index 12810df..4a801ff 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java @@ -272,10 +272,10 @@ public class IgniteTeamcityConnection implements ITeamcity { /** {@inheritDoc} */ @AutoProfiling @Override public Build triggerBuild( - String buildTypeId, - @Nonnull String branchName, - boolean cleanRebuild, - boolean queueAtTop + String buildTypeId, + @NotNull @Nonnull String branchName, + boolean cleanRebuild, + boolean queueAtTop ) { String triggeringOptions = " <triggeringOptions" + @@ -317,6 +317,13 @@ public class IgniteTeamcityConnection implements ITeamcity { } } + @Override + public ProblemOccurrences getProblems(int buildId) { + return getJaxbUsingHref("app/rest/latest/problemOccurrences" + + "?locator=build:(id:" + buildId + ")" + + "&fields=problemOccurrence(id,type,identity,href,details,build(id))", ProblemOccurrences.class); + } + private CompletableFuture<List<File>> unzip(CompletableFuture<File> zipFileFut) { return zipFileFut.thenApplyAsync(ZipUtil::unZipToSameFolder, executor); } @@ -549,12 +556,6 @@ public class IgniteTeamcityConnection implements ITeamcity { /** {@inheritDoc} */ @Override - @AutoProfiling public CompletableFuture<List<BuildRef>> getRunningBuilds(@Nullable String branch) { - return supplyAsync(() -> getBuildsInState(null, branch, BuildRef.STATE_RUNNING), executor); - } - - /** {@inheritDoc} */ - @Override @AutoProfiling public CompletableFuture<List<BuildRef>> getQueuedBuilds(@Nullable String branch) { return supplyAsync(() -> getBuildsInState(null, branch, BuildRef.STATE_QUEUED), executor); } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java index 231ed5d..bcad28e 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java @@ -19,6 +19,7 @@ package org.apache.ignite.ci.teamcity.ignited.fatbuild; import com.google.common.base.Objects; import java.util.ArrayList; import java.util.BitSet; +import java.util.Collections; import java.util.List; import java.util.stream.Stream; import org.apache.ignite.ci.analysis.IVersionedEntity; @@ -27,6 +28,7 @@ import org.apache.ignite.ci.tcmodel.conf.BuildType; import org.apache.ignite.ci.tcmodel.hist.BuildRef; import org.apache.ignite.ci.tcmodel.result.Build; import org.apache.ignite.ci.tcmodel.result.TestOccurrencesRef; +import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence; import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull; import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull; import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted; @@ -297,4 +299,12 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn public String projectId(IStringCompactor compactor) { return compactor.getStringFromId(projectId); } + + public List<ProblemOccurrence> problems(IStringCompactor compactor) { + return Collections.emptyList(); //todo + } + + public void addProblems(IStringCompactor compactor, List<ProblemOccurrence> problems) { + //todo + } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java index afe4812..6469311 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java @@ -24,20 +24,16 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.StreamSupport; -import javax.cache.Cache; import javax.inject.Inject; import javax.inject.Provider; import javax.validation.constraints.NotNull; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.ci.db.TcHelperDb; -import org.apache.ignite.ci.di.AutoProfiling; import org.apache.ignite.ci.tcmodel.result.Build; +import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence; import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull; -import org.apache.ignite.ci.teamcity.ignited.BuildRefDao; import org.apache.ignite.ci.teamcity.ignited.IStringCompactor; -import org.apache.ignite.internal.util.GridIntList; import org.jetbrains.annotations.Nullable; /** @@ -68,14 +64,16 @@ public class FatBuildDao { * @param buildId * @param build Build data. * @param tests TestOccurrences one or several pages. + * @param problems * @param existingBuild existing version of build in the DB. * @return Fat Build saved (if modifications detected), otherwise null. */ public FatBuildCompacted saveBuild(long srvIdMaskHigh, - int buildId, - @NotNull Build build, - List<TestOccurrencesFull> tests, - @Nullable FatBuildCompacted existingBuild) { + int buildId, + @NotNull Build build, + @NotNull List<TestOccurrencesFull> tests, + @NotNull List<ProblemOccurrence> problems, + @Nullable FatBuildCompacted existingBuild) { Preconditions.checkNotNull(buildsCache, "init() was not called"); Preconditions.checkNotNull(build, "build can't be null"); @@ -84,6 +82,8 @@ public class FatBuildDao { for (TestOccurrencesFull next : tests) newBuild.addTests(compactor, next.getTests()); + newBuild.addProblems(compactor, problems); + if (existingBuild == null || !existingBuild.equals(newBuild)) { buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId), newBuild); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java index 8eac870..e916891 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java @@ -22,6 +22,8 @@ import org.apache.ignite.ci.di.AutoProfiling; import org.apache.ignite.ci.di.MonitoredTask; import org.apache.ignite.ci.di.scheduler.IScheduler; import org.apache.ignite.ci.tcmodel.result.Build; +import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence; +import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrences; import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull; import org.apache.ignite.ci.teamcity.ignited.BuildRefDao; import org.apache.ignite.ci.teamcity.ignited.IStringCompactor; @@ -213,18 +215,27 @@ public class ProactiveFatBuildSync { Build build; List<TestOccurrencesFull> tests = new ArrayList<>(); + List<ProblemOccurrence> problems; try { build = conn.getBuild(buildId); - String nextHref = null; - do { - boolean testDtls = !build.isComposite(); // don't query test details for compoite - TestOccurrencesFull page = conn.getTestsPage(buildId, nextHref, testDtls); - nextHref = page.nextHref(); + if(build.testOccurrences!=null) { + String nextHref = null; + do { + boolean testDtls = !build.isComposite(); // don't query test details for compoite + TestOccurrencesFull page = conn.getTestsPage(buildId, nextHref, testDtls); + nextHref = page.nextHref(); - tests.add(page); + tests.add(page); + } + while (!Strings.isNullOrEmpty(nextHref)); } - while (!Strings.isNullOrEmpty(nextHref)); + + if (build.problemOccurrences != null) { + ProblemOccurrences po = conn.getProblems(buildId); + problems = new ArrayList<>(po.getProblemsNonNull()); + } else + problems = Collections.emptyList(); } catch (Exception e) { if (Throwables.getRootCause(e) instanceof FileNotFoundException) { @@ -237,9 +248,13 @@ public class ProactiveFatBuildSync { build.setCancelled(); tests = Collections.singletonList(existingBuild.getTestOcurrences(compactor)); + + problems = existingBuild.problems(compactor); } - else + else { build = Build.createFakeStub(); + problems = Collections.emptyList(); + } } else { logger.error("Loading build [" + buildId + "] for server [" + srvNme + "] failed:" + e.getMessage(), e); @@ -251,6 +266,6 @@ public class ProactiveFatBuildSync { //if we are here because of some sort of outdated version of build, // new save will be performed with new entity version for compacted build - return fatBuildDao.saveBuild(srvIdMask, buildId, build, tests, existingBuild); + return fatBuildDao.saveBuild(srvIdMask, buildId, build, tests, problems, existingBuild); } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java index f832088..9737011 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java @@ -23,6 +23,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.apache.ignite.ci.tcmodel.hist.BuildRef; import org.apache.ignite.ci.tcmodel.result.Build; +import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrences; import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull; /** @@ -59,4 +60,6 @@ public interface ITeamcityConn { * @param queueAtTop Put at the top of the build queue. */ public Build triggerBuild(String buildTypeId, @Nonnull String branchName, boolean cleanRebuild, boolean queueAtTop); + + ProblemOccurrences getProblems(int buildId); } diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java index 173a959..6427cb4 100644 --- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java +++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java @@ -40,6 +40,8 @@ import org.apache.ignite.ci.di.scheduler.NoOpSheduler; import org.apache.ignite.ci.tcmodel.conf.BuildType; import org.apache.ignite.ci.tcmodel.hist.BuildRef; import org.apache.ignite.ci.tcmodel.result.Build; +import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence; +import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrences; import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao; @@ -252,6 +254,7 @@ public class IgnitedTcInMemoryIntegrationTest { public void testFatBuild() throws JAXBException, IOException { Build refBuild = jaxbTestXml("/build.xml", Build.class); TestOccurrencesFull testsRef = jaxbTestXml("/testList.xml", TestOccurrencesFull.class); + ProblemOccurrences problemsList = jaxbTestXml("/problemList.xml", ProblemOccurrences.class); Injector injector = Guice.createInjector(new AbstractModule() { @Override protected void configure() { @@ -265,7 +268,8 @@ public class IgnitedTcInMemoryIntegrationTest { int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(APACHE); List<TestOccurrencesFull> occurrences = Collections.singletonList(testsRef); - FatBuildCompacted buildCompacted = stor.saveBuild(srvIdMaskHigh, refBuild.getId(), refBuild, occurrences, null); + FatBuildCompacted buildCompacted = stor.saveBuild(srvIdMaskHigh, refBuild.getId(), refBuild, occurrences, + problemsList.getProblemsNonNull(), null); assertNotNull(buildCompacted); FatBuildCompacted fatBuild = stor.getFatBuild(srvIdMaskHigh, 2153237); @@ -300,6 +304,9 @@ public class IgnitedTcInMemoryIntegrationTest { Set<String> testNamesRef = new TreeSet<>(); testsRef.getTests().forEach(testOccurrence -> testNamesRef.add(testOccurrence.name)); assertEquals(testNamesRef, testNamesAct); + + final List<ProblemOccurrence> problems = buildCompacted.problems(compactor); + assertEquals(1, problems.size()); } public void saveTmpFile(Object obj, String name) throws IOException, JAXBException { diff --git a/ignite-tc-helper-web/src/test/resources/problemList.xml b/ignite-tc-helper-web/src/test/resources/problemList.xml new file mode 100644 index 0000000..6bf4ca9 --- /dev/null +++ b/ignite-tc-helper-web/src/test/resources/problemList.xml @@ -0,0 +1,3 @@ +<problemOccurrences count="1" href="http://ci.ignite.apache.org:443/app/rest/problemOccurrences?locator=build:(id:2153237)"> + <problemOccurrence id="problem:(id:3733),build:(id:2153237)" type="TC_FAILED_TESTS" identity="TC_FAILED_TESTS_bt1053" href="/app/rest/latest/problemOccurrences/problem:(id:3733),build:(id:2153237)"/> +</problemOccurrences> \ No newline at end of file
