This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch ignite-9848-load-all-builds in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 2717617cafa3de0e9cb406474487ddd60d8f93d8 Author: Dmitriy Pavlov <[email protected]> AuthorDate: Fri Oct 26 20:37:48 2018 +0300 Several updator's threads implemented --- .../ignite/ci/di/MonitoredTaskInterceptor.java | 13 +++++----- .../ignite/ci/di/scheduler/TcBotScheduler.java | 17 ++++++++----- .../ignite/ci/teamcity/ignited/BuildRefDao.java | 11 +++++--- .../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 29 ++++++++++++++++------ .../ci/teamcity/ignited/TeamcityIgnitedModule.java | 1 + .../ignited/{ => fatbuild}/FatBuildDao.java | 3 ++- .../ignited/IgnitedTcInMemoryIntegrationTest.java | 1 + 7 files changed, 51 insertions(+), 24 deletions(-) diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/MonitoredTaskInterceptor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/MonitoredTaskInterceptor.java index a0840da..c77816a 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/MonitoredTaskInterceptor.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/MonitoredTaskInterceptor.java @@ -17,23 +17,22 @@ package org.apache.ignite.ci.di; import com.google.common.base.Strings; -import org.aopalliance.intercept.MethodInterceptor; -import org.aopalliance.intercept.MethodInvocation; -import org.apache.ignite.ci.util.TimeUtil; -import org.jetbrains.annotations.NotNull; - import java.lang.reflect.Method; import java.util.Collection; import java.util.Collections; import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import org.aopalliance.intercept.MethodInterceptor; +import org.aopalliance.intercept.MethodInvocation; +import org.apache.ignite.ci.util.TimeUtil; +import org.jetbrains.annotations.NotNull; public class MonitoredTaskInterceptor implements MethodInterceptor { - private final ConcurrentMap<String, Invocation> totalTime = new ConcurrentHashMap<>(); + private final ConcurrentMap<String, Invocation> totalTime = new ConcurrentSkipListMap<>(); public static class Invocation { private final AtomicLong lastStartTs = new AtomicLong(); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java index cedb00b..2193666 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java @@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; class TcBotScheduler implements IScheduler { - public static final int POOL_SIZE = 3; + public static final int POOL_SIZE = 8; /** Logger. */ private static final Logger logger = LoggerFactory.getLogger(TcBotScheduler.class); @@ -50,17 +50,21 @@ class TcBotScheduler implements IScheduler { task.sheduleWithQuitePeriod(cmd, queitPeriod, unit); if (tickGuard.compareAndSet(false, true)) { - for (int i = 0; i < POOL_SIZE; i++) - service().scheduleAtFixedRate(this::checkNamedTasks, 0, 5, TimeUnit.SECONDS); + for (int i = 0; i < POOL_SIZE; i++) { + int threadNo = i; + + service().scheduleAtFixedRate(() -> checkNamedTasks(threadNo), 0, 20, TimeUnit.SECONDS); + } } } /** * + * @param threadNo */ @SuppressWarnings({"UnusedReturnValue", "WeakerAccess"}) - @MonitoredTask(name = "Run Named Scheduled Tasks") - protected String checkNamedTasks() { + @MonitoredTask(name = "Scheduled, runner", nameExtArgIndex = 0) + protected String checkNamedTasks(int threadNo) { AtomicInteger run = new AtomicInteger(); List<Throwable> problems = new ArrayList<>(); namedTasks.forEach((s, task) -> { @@ -73,7 +77,8 @@ class TcBotScheduler implements IScheduler { problems.add(e); } }); - return "Finished " + run.get() + (problems.isEmpty() ? ", exceptions: " : problems.toString()); + + return "Finished " + run.get() + " task(s) " + (problems.isEmpty() ? "" : (", exceptions: " + problems.toString())); } /** {@inheritDoc} */ diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java index f6f7c44..d11f148 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java @@ -72,12 +72,14 @@ public class BuildRefDao { */ @NotNull protected Stream<BuildRefCompacted> compactedBuildsForServer(int srvId) { return StreamSupport.stream(buildRefsCache.spliterator(), false) - .filter(entry -> { - return isKeyForServer(entry.getKey(), srvId); - }) + .filter(entry -> isKeyForServer(entry.getKey(), srvId)) .map(javax.cache.Cache.Entry::getValue); } + /** + * @param key Key. + * @param srvId Server id. + */ private boolean isKeyForServer(Long key, int srvId) { return key!=null && key >> 32 == srvId; } @@ -86,6 +88,7 @@ public class BuildRefDao { * @param srvId Server id mask high. * @param ghData Gh data. */ + @AutoProfiling public Set<Long> saveChunk(long srvId, List<BuildRef> ghData) { Set<Long> ids = ghData.stream().map(BuildRef::getId) .filter(Objects::nonNull) @@ -132,6 +135,7 @@ public class BuildRefDao { * @param buildTypeId Build type id. * @param bracnhNameQry Bracnh name query. */ + @AutoProfiling @NotNull public List<BuildRef> findBuildsInHistory(int srvId, @Nullable String buildTypeId, String bracnhNameQry) { @@ -206,6 +210,7 @@ public class BuildRefDao { * @param srvId Server id. * @param refCompacted Reference compacted. */ + @AutoProfiling public boolean save(int srvId, BuildRefCompacted refCompacted) { long cacheKey = buildIdToCacheKey(srvId, refCompacted.id()); BuildRefCompacted buildPersisted = buildRefsCache.get(cacheKey); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java index 840178e..bcd1476 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.OptionalInt; import java.util.Set; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -46,6 +47,7 @@ import org.apache.ignite.ci.tcmodel.hist.BuildRef; import org.apache.ignite.ci.tcmodel.result.Build; import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; +import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao; import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted; import org.apache.ignite.ci.teamcity.pure.ITeamcityConn; import org.apache.ignite.ci.util.ExceptionUtil; @@ -97,11 +99,17 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { } public void scheduleBuildsLoad(List<Integer> buildsToAskFromTc) { + if (buildsToAskFromTc.isEmpty()) + return; + synchronized (this) { buildToLoad.addAll(buildsToAskFromTc); } - scheduler.sheduleNamed(taskName("loadFatBuilds"), this::loadFatBuilds, 2, TimeUnit.MINUTES); + int ldrToActivate = ThreadLocalRandom.current().nextInt(5); + + scheduler.sheduleNamed(taskName("loadFatBuilds" + ldrToActivate), () -> loadFatBuilds(ldrToActivate), 2, TimeUnit.MINUTES); + } @NotNull public String taskName(String taskName) { @@ -235,7 +243,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { runActualizeBuilds(srvId, false, collect); // schedule full resync later - scheduler.invokeLater(this::sheduleResync, 20, TimeUnit.MINUTES); + scheduler.invokeLater(this::sheduleResync, 15, TimeUnit.MINUTES); } /** @@ -302,18 +310,24 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { return cacheKeysUpdated.stream().map(BuildRefDao::cacheKeyToBuildId).collect(Collectors.toList()); } - private void loadFatBuilds() { + /** */ + private void loadFatBuilds(int ldrNo) { Set<Integer> load; + synchronized (this) { load = buildToLoad; buildToLoad = new HashSet<>(); } - doLoadBuilds(srvId, load); + + doLoadBuilds(ldrNo, srvId, load); } - @MonitoredTask(name = "Proactive Builds Loading", nameExtArgIndex = 0) + @MonitoredTask(name = "Proactive Builds Loading (agent,server)", nameExtArgsIndexes = {0, 1}) @AutoProfiling - protected String doLoadBuilds(String srvId, Set<Integer> load) { + protected String doLoadBuilds(int ldrNo, String srvId, Set<Integer> load) { + if(load.isEmpty()) + return "Nothing to load"; + AtomicInteger err = new AtomicInteger(); AtomicInteger ld = new AtomicInteger(); @@ -335,6 +349,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { } } ); - return "Builds Loaded " + ld.get() + " from " + load.size() + " requested, errors: " + err; + + return "Builds updated " + ld.get() + " from " + load.size() + " requested, errors: " + err; } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java index 6ef440b..e397d8f 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java @@ -19,6 +19,7 @@ package org.apache.ignite.ci.teamcity.ignited; import com.google.inject.AbstractModule; import com.google.inject.internal.SingletonScope; import org.apache.ignite.ci.tcbot.condition.BuildConditionDao; +import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao; import org.apache.ignite.ci.teamcity.pure.ITeamcityHttpConnection; import org.apache.ignite.ci.teamcity.restcached.TcRestCachedModule; import org.jetbrains.annotations.Nullable; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/FatBuildDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java similarity index 96% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/FatBuildDao.java rename to ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java index 43a1573..fc8b993 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/FatBuildDao.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.ci.teamcity.ignited; +package org.apache.ignite.ci.teamcity.ignited.fatbuild; import java.util.Collection; import java.util.List; @@ -30,6 +30,7 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.ci.db.TcHelperDb; import org.apache.ignite.ci.tcmodel.result.Build; import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull; +import org.apache.ignite.ci.teamcity.ignited.IStringCompactor; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; import org.jetbrains.annotations.Nullable; diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java index aa2e7eb..9672abf 100644 --- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java +++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java @@ -42,6 +42,7 @@ import org.apache.ignite.ci.tcmodel.hist.BuildRef; import org.apache.ignite.ci.tcmodel.result.Build; import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; +import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao; import org.apache.ignite.ci.teamcity.pure.BuildHistoryEmulator; import org.apache.ignite.ci.teamcity.pure.ITeamcityHttpConnection; import org.apache.ignite.ci.user.ICredentialsProv;
