This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9848-load-all-builds
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-9848-load-all-builds by
this push:
new bdfce20 IGNITE-9848: Background upload of all builds from TC to the
bot DB
bdfce20 is described below
commit bdfce2058a7c616ec3b7d1cfc5deb96e4a81d69d
Author: Dmitriy Pavlov <[email protected]>
AuthorDate: Fri Oct 26 21:27:41 2018 +0300
IGNITE-9848: Background upload of all builds from TC to the bot DB
---
.../ignite/ci/di/scheduler/TcBotScheduler.java | 3 +-
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 62 +++++++++++++++-------
.../ci/teamcity/ignited/fatbuild/FatBuildDao.java | 17 ++++--
.../ignited/IgnitedTcInMemoryIntegrationTest.java | 2 +-
4 files changed, 59 insertions(+), 25 deletions(-)
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
index 2193666..2e1ef46 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
@@ -53,7 +53,8 @@ class TcBotScheduler implements IScheduler {
for (int i = 0; i < POOL_SIZE; i++) {
int threadNo = i;
- service().scheduleAtFixedRate(() -> checkNamedTasks(threadNo),
0, 20, TimeUnit.SECONDS);
+ int period = 15000 +
ThreadLocalRandom.current().nextInt(10000);
+ service().scheduleAtFixedRate(() -> checkNamedTasks(threadNo),
0, period, TimeUnit.MILLISECONDS);
}
}
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index bcd1476..4abf7cd 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -60,7 +60,8 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
private static final Logger logger =
LoggerFactory.getLogger(TestCompacted.class);
/** Max build id diff to enforce reload during incremental refresh. */
- public static final int MAX_ID_DIFF_TO_ENFORCE = 5000;
+ public static final int MAX_ID_DIFF_TO_ENFORCE_CONTINUE_SCAN = 3000;
+ public static final int FAT_BUILD_PROACTIVE_TASKS = 4;
/** Server id. */
private String srvId;
@@ -98,7 +99,12 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited
{
fatBuildDao.init();
}
- public void scheduleBuildsLoad(List<Integer> buildsToAskFromTc) {
+ /**
+ * Invoke load fat builds later, re-load provided builds.
+ *
+ * @param buildsToAskFromTc Builds to ask from tc.
+ */
+ public void scheduleBuildsLoad(Collection<Integer> buildsToAskFromTc) {
if (buildsToAskFromTc.isEmpty())
return;
@@ -106,7 +112,7 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
buildToLoad.addAll(buildsToAskFromTc);
}
- int ldrToActivate = ThreadLocalRandom.current().nextInt(5);
+ int ldrToActivate =
ThreadLocalRandom.current().nextInt(FAT_BUILD_PROACTIVE_TASKS);
scheduler.sheduleNamed(taskName("loadFatBuilds" + ldrToActivate), ()
-> loadFatBuilds(ldrToActivate), 2, TimeUnit.MINUTES);
@@ -223,7 +229,7 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
//if we are here because of some sort of outdated version of build,
// new save will be performed with new entity version for compacted
build
- return fatBuildDao.saveBuild(srvIdMaskHigh, build, tests,
existingBuild);
+ return fatBuildDao.saveBuild(srvIdMaskHigh, buildId, build, tests,
existingBuild);
}
/**
@@ -232,15 +238,29 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
void actualizeRecentBuilds() {
List<BuildRefCompacted> running =
buildRefDao.getQueuedAndRunning(srvIdMaskHigh);
- //todo intersect with ever found during current launch
- Set<Integer> collect =
running.stream().map(BuildRefCompacted::id).collect(Collectors.toSet());
+ List<Integer> runningIds =
running.stream().map(BuildRefCompacted::id).collect(Collectors.toList());
+
+ Set<Integer> paginateUntil = new HashSet<>();
+ Set<Integer> directUpload = new HashSet<>();
+
+ OptionalInt max = runningIds.stream().mapToInt(i -> i).max();
+ if (max.isPresent()) {
+ runningIds.forEach(id->{
+ if(id > (max.getAsInt() -
MAX_ID_DIFF_TO_ENFORCE_CONTINUE_SCAN))
+ paginateUntil.add(id);
+ else
+ directUpload.add(id);
+ });
+ }
+ //schedule direct reload for Fat Builds for all queued too-old builds
+ scheduleBuildsLoad(directUpload);
- OptionalInt max = collect.stream().mapToInt(i -> i).max();
- if (max.isPresent())
- collect = collect.stream().filter(id -> id > (max.getAsInt() -
MAX_ID_DIFF_TO_ENFORCE)).collect(Collectors.toSet());
- //drop stucked builds.
+ runActualizeBuilds(srvId, false, paginateUntil);
- runActualizeBuilds(srvId, false, collect);
+ if(!paginateUntil.isEmpty()) {
+ //some builds may stuck in the queued or running, enforce loading
as well
+ directUpload.addAll(paginateUntil);
+ }
// schedule full resync later
scheduler.invokeLater(this::sheduleResync, 15, TimeUnit.MINUTES);
@@ -263,7 +283,8 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
/**
* @param srvId Server id. todo to be added as composite name extend
* @param fullReindex Reindex all builds from TC history.
- * @param mandatoryToReload Build ID can be used as end of syncing.
Ignored if fullReindex mode.
+ * @param mandatoryToReload [in/out] Build ID should be found before end
of sync. Ignored if fullReindex mode.
+ *
*/
@MonitoredTask(name = "Actualize BuildRefs(srv, full resync)",
nameExtArgsIndexes = {0, 1})
@AutoProfiling
@@ -277,10 +298,12 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
scheduleBuildsLoad(cacheKeysToBuildIds(buildsUpdated));
int totalChecked = tcDataFirstPage.size();
+ int neededToFind = 0;
+ if (mandatoryToReload != null) {
+ neededToFind = mandatoryToReload.size();
- final Set<Integer> stillNeedToFind =
- mandatoryToReload == null ? Collections.emptySet() :
Sets.newHashSet(mandatoryToReload);
-
tcDataFirstPage.stream().map(BuildRef::getId).forEach(stillNeedToFind::remove);
+
tcDataFirstPage.stream().map(BuildRef::getId).forEach(mandatoryToReload::remove);
+ }
while (outLinkNext.get() != null) {
String nextPageUrl = outLinkNext.get();
@@ -295,15 +318,16 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
totalChecked += tcDataNextPage.size();
if (!fullReindex) {
- if (!stillNeedToFind.isEmpty())
-
tcDataNextPage.stream().map(BuildRef::getId).forEach(stillNeedToFind::remove);
+ if (mandatoryToReload!=null && !mandatoryToReload.isEmpty())
+
tcDataNextPage.stream().map(BuildRef::getId).forEach(mandatoryToReload::remove);
- if (savedCurChunk == 0 && stillNeedToFind.isEmpty())
+ if (savedCurChunk == 0 && (mandatoryToReload==null ||
mandatoryToReload.isEmpty()))
break; // There are no modification at current page,
hopefully no modifications at all
}
}
- return "Entries saved " + totalUpdated + " Builds checked " +
totalChecked;
+ int leftToFind = mandatoryToReload == null ? 0 :
mandatoryToReload.size();
+ return "Entries saved " + totalUpdated + " Builds checked " +
totalChecked + " Needed to find " + neededToFind + " remained to find " +
leftToFind;
}
@NotNull private List<Integer> cacheKeysToBuildIds(Collection<Long>
cacheKeysUpdated) {
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
index fc8b993..fd8b8df 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
@@ -17,6 +17,7 @@
package org.apache.ignite.ci.teamcity.ignited.fatbuild;
+import com.google.common.base.Preconditions;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -25,13 +26,13 @@ import java.util.Set;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Provider;
+import javax.validation.constraints.NotNull;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.ci.db.TcHelperDb;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.jetbrains.annotations.Nullable;
/**
@@ -59,15 +60,19 @@ public class FatBuildDao {
/**
* @param srvIdMaskHigh Server id mask high.
+ * @param buildId
* @param build Build data.
* @param tests TestOccurrences one or several pages.
* @param existingBuild existing version of build in the DB.
* @return Fat Build saved (if modifications detected), otherwise null.
*/
public FatBuildCompacted saveBuild(long srvIdMaskHigh,
- Build build,
+ int buildId,
+ @NotNull Build build,
List<TestOccurrencesFull> tests,
@Nullable FatBuildCompacted existingBuild) {
+ Preconditions.checkNotNull(buildsCache, "init() was not called");
+ Preconditions.checkNotNull(build, "build can't be null");
FatBuildCompacted newBuild = new FatBuildCompacted(compactor, build);
@@ -75,7 +80,7 @@ public class FatBuildDao {
newBuild.addTests(compactor, next.getTests());
if (existingBuild == null || !existingBuild.equals(newBuild)) {
- buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, build.getId()),
newBuild);
+ buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId),
newBuild);
return newBuild;
}
@@ -96,6 +101,8 @@ public class FatBuildDao {
* @param buildId Build id.
*/
public FatBuildCompacted getFatBuild(int srvIdMaskHigh, int buildId) {
+ Preconditions.checkNotNull(buildsCache, "init() was not called");
+
return buildsCache.get(buildIdToCacheKey(srvIdMaskHigh, buildId));
}
@@ -103,7 +110,9 @@ public class FatBuildDao {
* @param srvIdMaskHigh Server id mask high.
* @param buildsIds Builds ids.
*/
- public Map<Long, FatBuildCompacted> getAllFatBuilds(int srvIdMaskHigh,
Collection<Integer> buildsIds ) {
+ public Map<Long, FatBuildCompacted> getAllFatBuilds(int srvIdMaskHigh,
Collection<Integer> buildsIds) {
+ Preconditions.checkNotNull(buildsCache, "init() was not called");
+
Set<Long> ids = buildsIds.stream()
.filter(Objects::nonNull)
.map(buildId -> buildIdToCacheKey(srvIdMaskHigh, buildId))
diff --git
a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
index 9672abf..b07e102 100644
---
a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
+++
b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
@@ -265,7 +265,7 @@ public class IgnitedTcInMemoryIntegrationTest {
int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(APACHE);
List<TestOccurrencesFull> occurrences =
Collections.singletonList(testsRef);
- FatBuildCompacted buildCompacted = stor.saveBuild(srvIdMaskHigh,
refBuild, occurrences, null);
+ FatBuildCompacted buildCompacted = stor.saveBuild(srvIdMaskHigh,
refBuild.getId(), refBuild, occurrences, null);
assertNotNull(buildCompacted);
FatBuildCompacted fatBuild = stor.getFatBuild(srvIdMaskHigh, 2153237);
