[ignite-teamcity-bot] branch ignite-9848-3 updated: IGNITE-9848: Background upload of all builds from TC to the bot DB
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9848-3
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-9848-3 by this push:
new d19262f IGNITE-9848: Background upload of all builds from TC to the
bot DB
d19262f is described below
commit d19262faf99f52645dbaf751f95f76c3e5c80fe5
Author: Dmitriy Pavlov
AuthorDate: Mon Oct 29 17:56:30 2018 +0300
IGNITE-9848: Background upload of all builds from TC to the bot DB
---
.../ignite/ci/tcbot/chain/BuildChainProcessor.java | 22 +++---
.../ignite/ci/teamcity/ignited/BuildRefDao.java| 10 ++
.../ci/teamcity/ignited/ITeamcityIgnited.java | 5 +
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 5 +
4 files changed, 31 insertions(+), 11 deletions(-)
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
index 07f5fe0..56a9bd1 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
@@ -44,13 +44,13 @@ import org.apache.ignite.ci.analysis.mode.ProcessLogsMode;
import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefDao;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.util.FutureUtil;
import org.apache.ignite.ci.web.TcUpdatePool;
-import org.apache.lucene.util.IntsRef;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
@@ -69,6 +69,8 @@ public class BuildChainProcessor {
/** Compactor. */
@Inject private IStringCompactor compactor;
+@Inject private BuildRefDao buildRefDao;
+
/**
* @param teamcity Teamcity.
* @param teamcityIgnited
@@ -97,8 +99,8 @@ public class BuildChainProcessor {
Map buildsCtxMap = new ConcurrentHashMap<>();
Stream uniqueBuldsInvolved = entryPoints.stream()
-.flatMap(ref -> dependencies(teamcityIgnited, builds,
teamcity, ref)).filter(Objects::nonNull)
-.flatMap(ref -> dependencies(teamcityIgnited, builds,
teamcity, ref)).filter(Objects::nonNull)
+.flatMap(ref -> dependencies(teamcityIgnited, builds,
ref)).filter(Objects::nonNull)
+.flatMap(ref -> dependencies(teamcityIgnited, builds,
ref)).filter(Objects::nonNull)
.filter(ref -> ensureUnique(unique, ref))
;
@@ -283,10 +285,10 @@ public class BuildChainProcessor {
return branch;
}
-@Nullable private static Stream dependencies(
+@Nullable private Stream dependencies(
ITeamcityIgnited teamcityIgnited,
Map builds,
-ITeamcity teamcity, BuildRef ref) {
+BuildRef ref) {
FatBuildCompacted buildCompacted = builds.get(ref.getId());
if (buildCompacted != null)
return Stream.of(ref); // already processed build, so just keep
current ID in stream
@@ -298,12 +300,10 @@ public class BuildChainProcessor {
int[] ints = build.snapshotDependencies();
-List deps = IntStream.of(ints).mapToObj(i-> {
-BuildRef ref1 = new BuildRef();
-ref1.setId(i);
-
-return ref1;
-}).collect(Collectors.toList());
+//todo
+List deps = IntStream.of(ints).mapToObj(i -> {
+return buildRefDao.getBuildRef(teamcityIgnited.serverId(), i);
+}).filter(Objects::nonNull).collect(Collectors.toList());
if (deps.isEmpty())
return Stream.of(ref);
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
index 92b1684..87439fa 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
@@ -230,4 +230,14 @@ public class BuildRefDao {
return res.array();
}
+
+public BuildRef getBuildRef(String serverId, int i) {
+BuildRefCompacted ref = buildRefsCache.get(buildIdToCacheKey(
+ITeamcityIgnited.serverIdToInt(serverId), i));
+
+if(ref==null)
+return null;
+
+return ref.toBuildRef(compactor);
+}
}
[ignite-teamcity-bot] branch ignite-9848-3 updated: IGNITE-9848: Background upload of all builds from TC to the bot DB
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9848-3
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-9848-3 by this push:
new 6326920 IGNITE-9848: Background upload of all builds from TC to the
bot DB
6326920 is described below
commit 6326920c45aa73cab55d4426dbdd81243cfd3287
Author: Dmitriy Pavlov
AuthorDate: Mon Oct 29 17:47:38 2018 +0300
IGNITE-9848: Background upload of all builds from TC to the bot DB
---
.../java/org/apache/ignite/ci/db/DbMigrations.java | 8 ++--
.../ignite/ci/tcbot/chain/BuildChainProcessor.java | 45 +++---
.../ci/teamcity/ignited/BuildRefCompacted.java | 5 +++
.../ignited/fatbuild/FatBuildCompacted.java| 12 +-
.../ignited/fatbuild/ProactiveFatBuildSync.java| 3 --
5 files changed, 51 insertions(+), 22 deletions(-)
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
index 6441ab1..824dc4a 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
@@ -404,10 +404,10 @@ public class DbMigrations {
applyDestroyIgnCacheMigration(FINISHED_BUILDS_INCLUDE_FAILED);
applyDestroyIgnCacheMigration(TEST_OCCURRENCE_FULL);
-applyDestroyCacheMigration(TEAMCITY_BUILD_CACHE_NAME_OLD,
TEAMCITY_BUILD_CACHE_NAME_OLD);
applyDestroyIgnCacheMigration(TESTS);
applyDestroyIgnCacheMigration(STAT);
applyDestroyIgnCacheMigration(BUILD_STATISTICS);
+applyDestroyCacheMigration(TEAMCITY_BUILD_CACHE_NAME_OLD,
TEAMCITY_BUILD_CACHE_NAME_OLD);
}
private void applyDestroyIgnCacheMigration(String cacheName) {
@@ -415,12 +415,12 @@ public class DbMigrations {
applyDestroyCacheMigration(cacheName, ignCacheNme);
}
-private void applyDestroyCacheMigration(String dispCacheName, String
ignCacheNme) {
+private void applyDestroyCacheMigration(String dispCacheName, String
cacheNme) {
applyMigration("destroy-" + dispCacheName, () -> {
-IgniteCache cache = ignite.cache(ignCacheNme);
+IgniteCache cache = ignite.cache(cacheNme);
if (cache == null) {
-System.err.println("cache not found");
+System.err.println("cache [" + cacheNme + "] not found");
return;
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
index fe26954..07f5fe0 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
@@ -28,6 +28,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.function.Function;
import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.inject.Inject;
@@ -49,6 +50,7 @@ import
org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.util.FutureUtil;
import org.apache.ignite.ci.web.TcUpdatePool;
+import org.apache.lucene.util.IntsRef;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
@@ -89,18 +91,14 @@ public class BuildChainProcessor {
if (entryPoints.isEmpty())
return new FullChainRunCtx(Build.createFakeStub());
-BuildRef next = entryPoints.iterator().next();
-Build results = teamcity.getBuild(next.href);
-FullChainRunCtx fullChainRunCtx = new FullChainRunCtx(results);
Map unique = new ConcurrentHashMap<>();
+Map builds = new ConcurrentHashMap<>();
Map buildsCtxMap = new ConcurrentHashMap<>();
Stream uniqueBuldsInvolved = entryPoints.stream()
-.parallel()
-.unordered()
-.flatMap(ref -> dependencies(teamcity,
ref)).filter(Objects::nonNull)
-.flatMap(ref -> dependencies(teamcity,
ref)).filter(Objects::nonNull)
+.flatMap(ref -> dependencies(teamcityIgnited, builds,
teamcity, ref)).filter(Objects::nonNull)
+.flatMap(ref -> dependencies(teamcityIgnited, builds,
teamcity, ref)).filter(Objects::nonNull)
.filter(ref -> ensureUnique(unique, ref))
;
@@ -140,6 +138,12 @@ public class BuildChainProcessor {
return runStat.getCriticalFailRate() * 3.14f +
