This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch ignite-9848-2 in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 9da98d150c463bffede8afbbcec5551a55f69d79 Author: Dmitriy Pavlov <[email protected]> AuthorDate: Sat Oct 27 13:14:50 2018 +0300 Standalone syncer class started; Find missing builds by refs started --- .../main/java/org/apache/ignite/ci/ITeamcity.java | 1 - .../ignite/ci/teamcity/ignited/BuildRefDao.java | 21 +- .../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 190 ++++-------------- .../ci/teamcity/ignited/TeamcityIgnitedModule.java | 2 + .../ci/teamcity/ignited/fatbuild/FatBuildDao.java | 26 +++ .../ignited/fatbuild/ProactiveFatBuildSync.java | 221 +++++++++++++++++++++ .../ignite/ci/teamcity/pure/ITeamcityConn.java | 5 + .../ignited/IgnitedTcInMemoryIntegrationTest.java | 2 +- 8 files changed, 306 insertions(+), 162 deletions(-) diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java index 92cdab8..12446b9 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java @@ -59,7 +59,6 @@ public interface ITeamcity extends ITeamcityConn { CompletableFuture<List<BuildType>> getProjectSuites(String projectId); - String serverId(); /** * @param projectId Suite ID (string without spaces). diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java index d11f148..92b1684 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java @@ -17,13 +17,7 @@ package org.apache.ignite.ci.teamcity.ignited; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.TreeMap; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -223,4 +217,17 @@ public class BuildRefDao { return false; } + + @AutoProfiling + public int[] getAllIds(int srvId) { + GridIntList res = new GridIntList(buildRefsCache.size()); + + StreamSupport.stream(buildRefsCache.spliterator(), false) + .map(Cache.Entry::getKey) + .filter(entry -> isKeyForServer(entry, srvId)) + .map(BuildRefDao::cacheKeyToBuildId) + .forEach(res::add); + + return res.array(); + } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java index f32d335..08e1b5f 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java @@ -20,23 +20,6 @@ package org.apache.ignite.ci.teamcity.ignited; import com.google.common.base.Strings; import com.google.common.base.Throwables; import com.google.common.collect.Sets; -import java.io.FileNotFoundException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.OptionalInt; -import java.util.Set; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; -import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; -import javax.inject.Inject; import org.apache.ignite.ci.ITeamcity; import org.apache.ignite.ci.di.AutoProfiling; import org.apache.ignite.ci.di.MonitoredTask; @@ -48,23 +31,30 @@ import org.apache.ignite.ci.tcmodel.result.Build; import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao; -import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted; +import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProactiveFatBuildSync; import org.apache.ignite.ci.teamcity.pure.ITeamcityConn; import org.apache.ignite.ci.util.ExceptionUtil; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; +import javax.inject.Inject; +import java.io.FileNotFoundException; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + public class TeamcityIgnitedImpl implements ITeamcityIgnited { /** Logger. */ - private static final Logger logger = LoggerFactory.getLogger(TestCompacted.class); + private static final Logger logger = LoggerFactory.getLogger(TeamcityIgnitedImpl.class); /** Max build id diff to enforce reload during incremental refresh. */ public static final int MAX_ID_DIFF_TO_ENFORCE_CONTINUE_SCAN = 3000; - public static final int FAT_BUILD_PROACTIVE_TASKS = 4; /** Server id. */ - private String srvId; + private String srvNme; /** Pure HTTP Connection API. */ private ITeamcityConn conn; @@ -81,45 +71,27 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { /** Build DAO. */ @Inject private FatBuildDao fatBuildDao; - @Inject private IStringCompactor compactor; + @Inject private ProactiveFatBuildSync buildSync; /** Server ID mask for cache Entries. */ private int srvIdMaskHigh; - @GuardedBy("this") - private Set<Integer> buildToLoad = new HashSet<>(); - public void init(String srvId, ITeamcityConn conn) { - this.srvId = srvId; + this.srvNme = srvId; this.conn = conn; srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId); buildRefDao.init(); //todo init somehow in auto buildConditionDao.init(); fatBuildDao.init(); - } - - /** - * Invoke load fat builds later, re-load provided builds. - * - * @param buildsToAskFromTc Builds to ask from tc. - */ - public void scheduleBuildsLoad(Collection<Integer> buildsToAskFromTc) { - if (buildsToAskFromTc.isEmpty()) - return; - - synchronized (this) { - buildToLoad.addAll(buildsToAskFromTc); - } - - int ldrToActivate = ThreadLocalRandom.current().nextInt(FAT_BUILD_PROACTIVE_TASKS); - - scheduler.sheduleNamed(taskName("loadFatBuilds" + ldrToActivate), () -> loadFatBuilds(ldrToActivate), 2, TimeUnit.MINUTES); + buildSync.invokeLaterFindMissingByBuildRef(srvNme); } - @NotNull public String taskName(String taskName) { - return ITeamcityIgnited.class.getSimpleName() +"." + taskName + "." + srvId; + + @NotNull + private String taskName(String taskName) { + return ITeamcityIgnited.class.getSimpleName() +"." + taskName + "." + srvNme; } /** {@inheritDoc} */ @@ -132,7 +104,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { @Override public List<BuildRef> getBuildHistory( @Nullable String buildTypeId, @Nullable String branchName) { - scheduler.sheduleNamed(taskName("actualizeRecentBuilds"), this::actualizeRecentBuilds, 2, TimeUnit.MINUTES); + scheduler.sheduleNamed(taskName("actualizeRecentBuildRefs"), this::actualizeRecentBuildRefs, 2, TimeUnit.MINUTES); String bracnhNameQry ; if (ITeamcity.DEFAULT.equals(branchName)) @@ -148,7 +120,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { Build build = conn.triggerBuild(buildTypeId, branchName, cleanRebuild, queueAtTop); //todo may add additional parameter: load builds into DB in sync/async fashion - runActualizeBuilds(srvId, false, Sets.newHashSet(build.getId())); + runActualizeBuildRefs(srvNme, false, Sets.newHashSet(build.getId())); return build; } @@ -172,7 +144,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { if (existingBuild != null && !existingBuild.isOutdatedEntityVersion()) return existingBuild; - FatBuildCompacted savedVer = reloadBuild(buildId, existingBuild); + FatBuildCompacted savedVer = buildSync.reloadBuild(conn, buildId, existingBuild); //build was modified, probably we need also to update reference accordindly if (savedVer != null) @@ -181,61 +153,12 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { return savedVer == null ? existingBuild : savedVer; } - /** - * @param buildId - * @param existingBuild - * @return new build if it was updated or null if no updates detected - */ - public FatBuildCompacted reloadBuild(int buildId, @Nullable FatBuildCompacted existingBuild) { - // System.err.println(Thread.currentThread().getName()+ ": Build " + buildId); - //todo some sort of locking to avoid double requests - Build build; - List<TestOccurrencesFull> tests = new ArrayList<>(); - try { - build = conn.getBuild(buildId); - - String nextHref = null; - do { - boolean testDtls = !build.isComposite(); // don't query test details for compoite - TestOccurrencesFull page = conn.getTestsPage(buildId, nextHref, testDtls); - nextHref = page.nextHref(); - - tests.add(page); - } - while (!Strings.isNullOrEmpty(nextHref)); - } - catch (Exception e) { - if (Throwables.getRootCause(e) instanceof FileNotFoundException) { - logger.info("Loading build [" + buildId + "] for server [" + srvId + "] failed:" + e.getMessage(), e); - - if (existingBuild != null) { - build = existingBuild.toBuild(compactor); - - if(build.isRunning() || build.isQueued()) - build.setCancelled(); - - tests = Collections.singletonList(existingBuild.getTestOcurrences(compactor)); - } - else - build = Build.createFakeStub(); - } else { - logger.error("Loading build [" + buildId + "] for server [" + srvId + "] failed:" + e.getMessage(), e); - - e.printStackTrace(); - - throw ExceptionUtil.propagateException(e); - } - } - //if we are here because of some sort of outdated version of build, - // new save will be performed with new entity version for compacted build - return fatBuildDao.saveBuild(srvIdMaskHigh, buildId, build, tests, existingBuild); - } /** * */ - void actualizeRecentBuilds() { + void actualizeRecentBuildRefs() { List<BuildRefCompacted> running = buildRefDao.getQueuedAndRunning(srvIdMaskHigh); Set<Integer> paginateUntil = new HashSet<>(); @@ -252,23 +175,23 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { }); } //schedule direct reload for Fat Builds for all queued too-old builds - scheduleBuildsLoad(directUpload); + buildSync.scheduleBuildsLoad(srvNme, directUpload); - runActualizeBuilds(srvId, false, paginateUntil); + runActualizeBuildRefs(srvNme, false, paginateUntil); if(!paginateUntil.isEmpty()) { //some builds may stuck in the queued or running, enforce loading as well - scheduleBuildsLoad(paginateUntil); + buildSync.scheduleBuildsLoad(srvNme, paginateUntil); } // schedule full resync later - scheduler.invokeLater(this::sheduleResync, 15, TimeUnit.MINUTES); + scheduler.invokeLater(this::sheduleResyncBuildRefs, 15, TimeUnit.MINUTES); } /** * */ - private void sheduleResync() { + private void sheduleResyncBuildRefs() { scheduler.sheduleNamed(taskName("fullReindex"), this::fullReindex, 120, TimeUnit.MINUTES); } @@ -276,25 +199,29 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { * */ void fullReindex() { - runActualizeBuilds(srvId, true, null); + runActualizeBuildRefs(srvNme, true, null); + + buildSync.invokeLaterFindMissingByBuildRef(srvNme); } + /** - * @param srvId Server id. todo to be added as composite name extend + * @param srvId Server id. * @param fullReindex Reindex all builds from TC history. * @param mandatoryToReload [in/out] Build ID should be found before end of sync. Ignored if fullReindex mode. * */ + @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"}) @MonitoredTask(name = "Actualize BuildRefs(srv, full resync)", nameExtArgsIndexes = {0, 1}) @AutoProfiling - protected String runActualizeBuilds(String srvId, boolean fullReindex, - @Nullable Set<Integer> mandatoryToReload) { + protected String runActualizeBuildRefs(String srvId, boolean fullReindex, + @Nullable Set<Integer> mandatoryToReload) { AtomicReference<String> outLinkNext = new AtomicReference<>(); List<BuildRef> tcDataFirstPage = conn.getBuildRefs(null, outLinkNext); Set<Long> buildsUpdated = buildRefDao.saveChunk(srvIdMaskHigh, tcDataFirstPage); int totalUpdated = buildsUpdated.size(); - scheduleBuildsLoad(cacheKeysToBuildIds(buildsUpdated)); + buildSync.scheduleBuildsLoad(srvNme, cacheKeysToBuildIds(buildsUpdated)); int totalChecked = tcDataFirstPage.size(); int neededToFind = 0; @@ -310,7 +237,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { List<BuildRef> tcDataNextPage = conn.getBuildRefs(nextPageUrl, outLinkNext); Set<Long> curChunkBuildsSaved = buildRefDao.saveChunk(srvIdMaskHigh, tcDataNextPage); totalUpdated += curChunkBuildsSaved.size(); - scheduleBuildsLoad(cacheKeysToBuildIds(curChunkBuildsSaved)); + buildSync.scheduleBuildsLoad(srvNme, cacheKeysToBuildIds(curChunkBuildsSaved)); int savedCurChunk = curChunkBuildsSaved.size(); @@ -332,47 +259,4 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { @NotNull private List<Integer> cacheKeysToBuildIds(Collection<Long> cacheKeysUpdated) { return cacheKeysUpdated.stream().map(BuildRefDao::cacheKeyToBuildId).collect(Collectors.toList()); } - - /** */ - private void loadFatBuilds(int ldrNo) { - Set<Integer> load; - - synchronized (this) { - load = buildToLoad; - buildToLoad = new HashSet<>(); - } - - doLoadBuilds(ldrNo, srvId, load); - } - - @MonitoredTask(name = "Proactive Builds Loading (agent,server)", nameExtArgsIndexes = {0, 1}) - @AutoProfiling - protected String doLoadBuilds(int ldrNo, String srvId, Set<Integer> load) { - if(load.isEmpty()) - return "Nothing to load"; - - AtomicInteger err = new AtomicInteger(); - AtomicInteger ld = new AtomicInteger(); - - Map<Long, FatBuildCompacted> builds = fatBuildDao.getAllFatBuilds(srvIdMaskHigh, load); - - load.forEach( - buildId -> { - try { - FatBuildCompacted existingBuild = builds.get(FatBuildDao.buildIdToCacheKey(srvIdMaskHigh, buildId)); - - FatBuildCompacted savedVer = reloadBuild(buildId, existingBuild); - - if (savedVer != null) - ld.incrementAndGet(); - } - catch (Exception e) { - logger.error("", e); - err.incrementAndGet(); - } - } - ); - - return "Builds updated " + ld.get() + " from " + load.size() + " requested, errors: " + err; - } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java index e397d8f..9a5d68c 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java @@ -20,6 +20,7 @@ import com.google.inject.AbstractModule; import com.google.inject.internal.SingletonScope; import org.apache.ignite.ci.tcbot.condition.BuildConditionDao; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao; +import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProactiveFatBuildSync; import org.apache.ignite.ci.teamcity.pure.ITeamcityHttpConnection; import org.apache.ignite.ci.teamcity.restcached.TcRestCachedModule; import org.jetbrains.annotations.Nullable; @@ -37,6 +38,7 @@ public class TeamcityIgnitedModule extends AbstractModule { bind(BuildRefDao.class).in(new SingletonScope()); bind(BuildConditionDao.class).in(new SingletonScope()); bind(FatBuildDao.class).in(new SingletonScope()); + bind(ProactiveFatBuildSync.class).in(new SingletonScope()); bind(IStringCompactor.class).to(IgniteStringCompactor.class).in(new SingletonScope()); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java index fd8b8df..725c074 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java @@ -24,15 +24,20 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import javax.cache.Cache; import javax.inject.Inject; import javax.inject.Provider; import javax.validation.constraints.NotNull; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.ci.db.TcHelperDb; +import org.apache.ignite.ci.di.AutoProfiling; import org.apache.ignite.ci.tcmodel.result.Build; import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull; +import org.apache.ignite.ci.teamcity.ignited.BuildRefDao; import org.apache.ignite.ci.teamcity.ignited.IStringCompactor; +import org.apache.ignite.internal.util.GridIntList; import org.jetbrains.annotations.Nullable; /** @@ -120,4 +125,25 @@ public class FatBuildDao { return buildsCache.getAll(ids); } + + /** + * @param key Key. + * @param srvId Server id. + */ + private boolean isKeyForServer(Long key, int srvId) { + return key!=null && key >> 32 == srvId; + } + + @AutoProfiling + public int[] getAllIds(int srvId) { + GridIntList res = new GridIntList(buildsCache.size()); + + StreamSupport.stream(buildsCache.spliterator(), false) + .map(Cache.Entry::getKey) + .filter(entry -> isKeyForServer(entry, srvId)) + .map(BuildRefDao::cacheKeyToBuildId) + .forEach(res::add); + + return res.array(); + } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java new file mode 100644 index 0000000..2dd1757 --- /dev/null +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.ci.teamcity.ignited.fatbuild; + +import com.google.common.base.Strings; +import com.google.common.base.Throwables; +import org.apache.ignite.ci.di.AutoProfiling; +import org.apache.ignite.ci.di.MonitoredTask; +import org.apache.ignite.ci.di.scheduler.IScheduler; +import org.apache.ignite.ci.tcmodel.result.Build; +import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull; +import org.apache.ignite.ci.teamcity.ignited.BuildRefDao; +import org.apache.ignite.ci.teamcity.ignited.IStringCompactor; +import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited; +import org.apache.ignite.ci.teamcity.pure.ITeamcityConn; +import org.apache.ignite.ci.util.ExceptionUtil; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.GuardedBy; +import javax.inject.Inject; +import java.io.FileNotFoundException; +import java.util.*; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class ProactiveFatBuildSync { + public static final int FAT_BUILD_PROACTIVE_TASKS = 4; + + /** Logger. */ + private static final Logger logger = LoggerFactory.getLogger(ProactiveFatBuildSync.class); + + /** Build reference DAO. */ + @Inject + private BuildRefDao buildRefDao; + + /** Build DAO. */ + @Inject private FatBuildDao fatBuildDao; + + /** Scheduler. */ + @Inject private IScheduler scheduler; + + @Inject private IStringCompactor compactor; + + + @GuardedBy("this") + private Set<Integer> buildToLoad = new HashSet<>(); + + + /** + * Invoke load fat builds later, re-load provided builds. + * @param srvNme + * @param buildsToAskFromTc Builds to ask from tc. + */ + public void scheduleBuildsLoad(String srvNme , Collection<Integer> buildsToAskFromTc) { + if (buildsToAskFromTc.isEmpty()) + return; + + synchronized (this) { + buildToLoad.addAll(buildsToAskFromTc); + } + + int ldrToActivate = ThreadLocalRandom.current().nextInt(FAT_BUILD_PROACTIVE_TASKS); + + scheduler.sheduleNamed(taskName("loadFatBuilds" + ldrToActivate, srvNme), + () -> loadFatBuilds(ldrToActivate, srvNme), 2, TimeUnit.MINUTES); + + } + + + @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"}) + @MonitoredTask(name = "Find missing builds", nameExtArgsIndexes = {0}) + @AutoProfiling + protected String findMissingBuildsFromBuildRef(String srvId) { + int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId); + + final int[] buildRefKeys = buildRefDao.getAllIds(srvIdMaskHigh); + final int[] fatBuildKeys = fatBuildDao.getAllIds(srvIdMaskHigh); + + Arrays.parallelSort(buildRefKeys); + Arrays.parallelSort(fatBuildKeys); + /* ; + */ + + return ""; + } + + + + /** */ + private void loadFatBuilds(int ldrNo, String srvNme) { + Set<Integer> load; + + synchronized (this) { + load = buildToLoad; + buildToLoad = new HashSet<>(); + } + + doLoadBuilds(ldrNo, srvNme, conn, load); + } + + @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"}) + @MonitoredTask(name = "Proactive Builds Loading (agent,server)", nameExtArgsIndexes = {0, 1}) + @AutoProfiling + protected String doLoadBuilds(int ldrNo, String srvId, ITeamcityConn conn, Set<Integer> load) { + if(load.isEmpty()) + return "Nothing to load"; + + final int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId); + + AtomicInteger err = new AtomicInteger(); + AtomicInteger ld = new AtomicInteger(); + + Map<Long, FatBuildCompacted> builds = fatBuildDao.getAllFatBuilds(srvIdMaskHigh, load); + + load.forEach( + buildId -> { + try { + FatBuildCompacted existingBuild = builds.get(FatBuildDao.buildIdToCacheKey(srvIdMaskHigh, buildId)); + + FatBuildCompacted savedVer = reloadBuild(conn, buildId, existingBuild); + + if (savedVer != null) + ld.incrementAndGet(); + } + catch (Exception e) { + logger.error("", e); + err.incrementAndGet(); + } + } + ); + + return "Builds updated " + ld.get() + " from " + load.size() + " requested, errors: " + err; + } + + @NotNull + private String taskName(String taskName, String srvName) { + return ProactiveFatBuildSync.class.getSimpleName() +"." + taskName + "." + srvName; + } + + public void invokeLaterFindMissingByBuildRef(String srvName) { + scheduler.sheduleNamed(taskName("findMissingBuildsFromBuildRef", srvName), + () -> findMissingBuildsFromBuildRef(srvName), 360, TimeUnit.MINUTES); + } + + /** + * + * @param conn + * @param buildId + * @param existingBuild + * @return new build if it was updated or null if no updates detected + */ + @SuppressWarnings({"WeakerAccess"}) + @AutoProfiling + public FatBuildCompacted reloadBuild(ITeamcityConn conn, int buildId, @Nullable FatBuildCompacted existingBuild) { + // System.err.println(Thread.currentThread().getName()+ ": Build " + buildId); + //todo some sort of locking to avoid double requests + + final String srvNme = conn.serverId(); + final int srvIdMask = ITeamcityIgnited.serverIdToInt(srvNme); + + Build build; + List<TestOccurrencesFull> tests = new ArrayList<>(); + try { + build = conn.getBuild(buildId); + + String nextHref = null; + do { + boolean testDtls = !build.isComposite(); // don't query test details for compoite + TestOccurrencesFull page = conn.getTestsPage(buildId, nextHref, testDtls); + nextHref = page.nextHref(); + + tests.add(page); + } + while (!Strings.isNullOrEmpty(nextHref)); + } + catch (Exception e) { + if (Throwables.getRootCause(e) instanceof FileNotFoundException) { + logger.info("Loading build [" + buildId + "] for server [" + srvNme + "] failed:" + e.getMessage(), e); + + if (existingBuild != null) { + build = existingBuild.toBuild(compactor); + + if(build.isRunning() || build.isQueued()) + build.setCancelled(); + + tests = Collections.singletonList(existingBuild.getTestOcurrences(compactor)); + } + else + build = Build.createFakeStub(); + } else { + logger.error("Loading build [" + buildId + "] for server [" + srvNme + "] failed:" + e.getMessage(), e); + + e.printStackTrace(); + + throw ExceptionUtil.propagateException(e); + } + } + + //if we are here because of some sort of outdated version of build, + // new save will be performed with new entity version for compacted build + return fatBuildDao.saveBuild(srvIdMask, buildId, build, tests, existingBuild); + } +} diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java index 78f3d34..f832088 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java @@ -30,6 +30,11 @@ import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull; */ public interface ITeamcityConn { /** + * @return Internal server ID as string + */ + String serverId(); + + /** * @return Normalized Host address, ends with '/'. */ public String host(); diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java index b07e102..173a959 100644 --- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java +++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java @@ -214,7 +214,7 @@ public class IgnitedTcInMemoryIntegrationTest { for (int i = queuedBuildIdx; i < tcBuilds.size(); i++) tcBuilds.get(i).state = BuildRef.STATE_FINISHED; - teamcityIgnited.actualizeRecentBuilds(); + teamcityIgnited.actualizeRecentBuildRefs(); List<BuildRef> hist = srv.getBuildHistory(buildTypeId, branchName);
