This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new 061bfe2 IGNITE-9848: Standalone syncer class; Find missing builds by
refs done; Different compression types for details added - Fixes #48.
061bfe2 is described below
commit 061bfe2a8a75dff489c404910c5b8455b729c98a
Author: Dmitriy Pavlov <[email protected]>
AuthorDate: Sat Oct 27 15:22:40 2018 +0300
IGNITE-9848: Standalone syncer class; Find missing builds by refs done;
Different compression types for details added - Fixes #48.
Signed-off-by: Dmitriy Pavlov <[email protected]>
---
.../main/java/org/apache/ignite/ci/ITeamcity.java | 1 -
.../java/org/apache/ignite/ci/db/DbMigrations.java | 15 +-
.../ignite/ci/di/scheduler/TcBotScheduler.java | 10 +-
.../ignite/ci/teamcity/ignited/BuildRefDao.java | 21 +-
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 192 +++-------------
.../ci/teamcity/ignited/TeamcityIgnitedModule.java | 2 +
.../ignited/fatbuild/FatBuildCompacted.java | 2 +-
.../ci/teamcity/ignited/fatbuild/FatBuildDao.java | 17 ++
.../ignited/fatbuild/ProactiveFatBuildSync.java | 256 +++++++++++++++++++++
.../teamcity/ignited/fatbuild/TestCompacted.java | 127 ++++++++--
.../ignite/ci/teamcity/pure/ITeamcityConn.java | 5 +
.../org/apache/ignite/ci/web/model/Version.java | 2 +-
.../ignited/IgnitedTcInMemoryIntegrationTest.java | 2 +-
13 files changed, 457 insertions(+), 195 deletions(-)
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
index 92cdab8..12446b9 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
@@ -59,7 +59,6 @@ public interface ITeamcity extends ITeamcityConn {
CompletableFuture<List<BuildType>> getProjectSuites(String projectId);
- String serverId();
/**
* @param projectId Suite ID (string without spaces).
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
index 4a16f79..8b06abb 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
@@ -150,7 +150,9 @@ public class DbMigrations {
});
applyMigration(TESTS + "-to-" + testOccurrencesCache.getName(), () -> {
- IgniteCache<String, TestOccurrences> tests =
getOrCreateIgnCacheV1(TESTS);
+ IgniteCache<String, TestOccurrences> tests =
ignite.cache(ignCacheNme(TESTS));
+ if(tests==null)
+ return;
int size = tests.size();
if (size > 0) {
@@ -402,17 +404,18 @@ public class DbMigrations {
applyDestroyIgnCacheMigration(FINISHED_BUILDS_INCLUDE_FAILED);
applyDestroyIgnCacheMigration(TEST_OCCURRENCE_FULL);
- //todo destroy
- // applyDestroyIgnCacheMigration(TEAMCITY_BUILD_CACHE_NAME_OLD);
+
+ applyDestroyCacheMigration(TEAMCITY_BUILD_CACHE_NAME_OLD,
TEAMCITY_BUILD_CACHE_NAME_OLD);
+ applyDestroyIgnCacheMigration(TESTS);
}
- public void applyDestroyIgnCacheMigration(String cacheName) {
+ private void applyDestroyIgnCacheMigration(String cacheName) {
String ignCacheNme = ignCacheNme(cacheName);
applyDestroyCacheMigration(cacheName, ignCacheNme);
}
- public void applyDestroyCacheMigration(String cacheName, String
ignCacheNme) {
- applyMigration("destroy-" + cacheName, () -> {
+ private void applyDestroyCacheMigration(String dispCacheName, String
ignCacheNme) {
+ applyMigration("destroy-" + dispCacheName, () -> {
IgniteCache<Object, Object> cache = ignite.cache(ignCacheNme);
if (cache == null) {
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
index 2e1ef46..045ea2a 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
@@ -84,8 +84,16 @@ class TcBotScheduler implements IScheduler {
/** {@inheritDoc} */
@Override public void stop() {
- if (executorSvc != null)
+ if (executorSvc != null) {
executorSvc.shutdown();
+ try {
+ if(!executorSvc.awaitTermination(10, TimeUnit.SECONDS))
+ executorSvc.shutdownNow();
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
}
/**
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
index d11f148..92b1684 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
@@ -17,13 +17,7 @@
package org.apache.ignite.ci.teamcity.ignited;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.TreeMap;
+import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
@@ -223,4 +217,17 @@ public class BuildRefDao {
return false;
}
+
+ @AutoProfiling
+ public int[] getAllIds(int srvId) {
+ GridIntList res = new GridIntList(buildRefsCache.size());
+
+ StreamSupport.stream(buildRefsCache.spliterator(), false)
+ .map(Cache.Entry::getKey)
+ .filter(entry -> isKeyForServer(entry, srvId))
+ .map(BuildRefDao::cacheKeyToBuildId)
+ .forEach(res::add);
+
+ return res.array();
+ }
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index f32d335..5603b44 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -17,26 +17,7 @@
package org.apache.ignite.ci.teamcity.ignited;
-import com.google.common.base.Strings;
-import com.google.common.base.Throwables;
import com.google.common.collect.Sets;
-import java.io.FileNotFoundException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.OptionalInt;
-import java.util.Set;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
-import javax.annotation.Nullable;
-import javax.annotation.concurrent.GuardedBy;
-import javax.inject.Inject;
import org.apache.ignite.ci.ITeamcity;
import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.di.MonitoredTask;
@@ -45,26 +26,30 @@ import org.apache.ignite.ci.tcbot.condition.BuildCondition;
import org.apache.ignite.ci.tcbot.condition.BuildConditionDao;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProactiveFatBuildSync;
import org.apache.ignite.ci.teamcity.pure.ITeamcityConn;
-import org.apache.ignite.ci.util.ExceptionUtil;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** Logger. */
- private static final Logger logger =
LoggerFactory.getLogger(TestCompacted.class);
+ private static final Logger logger =
LoggerFactory.getLogger(TeamcityIgnitedImpl.class);
/** Max build id diff to enforce reload during incremental refresh. */
public static final int MAX_ID_DIFF_TO_ENFORCE_CONTINUE_SCAN = 3000;
- public static final int FAT_BUILD_PROACTIVE_TASKS = 4;
/** Server id. */
- private String srvId;
+ private String srvNme;
/** Pure HTTP Connection API. */
private ITeamcityConn conn;
@@ -81,16 +66,13 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
/** Build DAO. */
@Inject private FatBuildDao fatBuildDao;
- @Inject private IStringCompactor compactor;
+ @Inject private ProactiveFatBuildSync buildSync;
/** Server ID mask for cache Entries. */
private int srvIdMaskHigh;
- @GuardedBy("this")
- private Set<Integer> buildToLoad = new HashSet<>();
-
public void init(String srvId, ITeamcityConn conn) {
- this.srvId = srvId;
+ this.srvNme = srvId;
this.conn = conn;
srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
@@ -99,27 +81,10 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
fatBuildDao.init();
}
- /**
- * Invoke load fat builds later, re-load provided builds.
- *
- * @param buildsToAskFromTc Builds to ask from tc.
- */
- public void scheduleBuildsLoad(Collection<Integer> buildsToAskFromTc) {
- if (buildsToAskFromTc.isEmpty())
- return;
-
- synchronized (this) {
- buildToLoad.addAll(buildsToAskFromTc);
- }
-
- int ldrToActivate =
ThreadLocalRandom.current().nextInt(FAT_BUILD_PROACTIVE_TASKS);
-
- scheduler.sheduleNamed(taskName("loadFatBuilds" + ldrToActivate), ()
-> loadFatBuilds(ldrToActivate), 2, TimeUnit.MINUTES);
- }
-
- @NotNull public String taskName(String taskName) {
- return ITeamcityIgnited.class.getSimpleName() +"." + taskName + "." +
srvId;
+ @NotNull
+ private String taskName(String taskName) {
+ return ITeamcityIgnited.class.getSimpleName() +"." + taskName + "." +
srvNme;
}
/** {@inheritDoc} */
@@ -132,7 +97,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited
{
@Override public List<BuildRef> getBuildHistory(
@Nullable String buildTypeId,
@Nullable String branchName) {
- scheduler.sheduleNamed(taskName("actualizeRecentBuilds"),
this::actualizeRecentBuilds, 2, TimeUnit.MINUTES);
+ scheduler.sheduleNamed(taskName("actualizeRecentBuildRefs"),
this::actualizeRecentBuildRefs, 2, TimeUnit.MINUTES);
String bracnhNameQry ;
if (ITeamcity.DEFAULT.equals(branchName))
@@ -148,7 +113,7 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
Build build = conn.triggerBuild(buildTypeId, branchName, cleanRebuild,
queueAtTop);
//todo may add additional parameter: load builds into DB in sync/async
fashion
- runActualizeBuilds(srvId, false, Sets.newHashSet(build.getId()));
+ runActualizeBuildRefs(srvNme, false, Sets.newHashSet(build.getId()));
return build;
}
@@ -172,7 +137,7 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
if (existingBuild != null && !existingBuild.isOutdatedEntityVersion())
return existingBuild;
- FatBuildCompacted savedVer = reloadBuild(buildId, existingBuild);
+ FatBuildCompacted savedVer = buildSync.reloadBuild(conn, buildId,
existingBuild);
//build was modified, probably we need also to update reference
accordindly
if (savedVer != null)
@@ -181,61 +146,15 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
return savedVer == null ? existingBuild : savedVer;
}
- /**
- * @param buildId
- * @param existingBuild
- * @return new build if it was updated or null if no updates detected
- */
- public FatBuildCompacted reloadBuild(int buildId, @Nullable
FatBuildCompacted existingBuild) {
- // System.err.println(Thread.currentThread().getName()+ ": Build " +
buildId);
- //todo some sort of locking to avoid double requests
- Build build;
- List<TestOccurrencesFull> tests = new ArrayList<>();
- try {
- build = conn.getBuild(buildId);
-
- String nextHref = null;
- do {
- boolean testDtls = !build.isComposite(); // don't query test
details for compoite
- TestOccurrencesFull page = conn.getTestsPage(buildId,
nextHref, testDtls);
- nextHref = page.nextHref();
-
- tests.add(page);
- }
- while (!Strings.isNullOrEmpty(nextHref));
- }
- catch (Exception e) {
- if (Throwables.getRootCause(e) instanceof FileNotFoundException) {
- logger.info("Loading build [" + buildId + "] for server [" +
srvId + "] failed:" + e.getMessage(), e);
-
- if (existingBuild != null) {
- build = existingBuild.toBuild(compactor);
- if(build.isRunning() || build.isQueued())
- build.setCancelled();
-
- tests =
Collections.singletonList(existingBuild.getTestOcurrences(compactor));
- }
- else
- build = Build.createFakeStub();
- } else {
- logger.error("Loading build [" + buildId + "] for server [" +
srvId + "] failed:" + e.getMessage(), e);
-
- e.printStackTrace();
-
- throw ExceptionUtil.propagateException(e);
- }
- }
-
- //if we are here because of some sort of outdated version of build,
- // new save will be performed with new entity version for compacted
build
- return fatBuildDao.saveBuild(srvIdMaskHigh, buildId, build, tests,
existingBuild);
- }
/**
*
*/
- void actualizeRecentBuilds() {
+ void actualizeRecentBuildRefs() {
+ // schedule find missing later
+ buildSync.invokeLaterFindMissingByBuildRef(srvNme, conn);
+
List<BuildRefCompacted> running =
buildRefDao.getQueuedAndRunning(srvIdMaskHigh);
Set<Integer> paginateUntil = new HashSet<>();
@@ -252,23 +171,23 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
});
}
//schedule direct reload for Fat Builds for all queued too-old builds
- scheduleBuildsLoad(directUpload);
+ buildSync.scheduleBuildsLoad(conn, directUpload);
- runActualizeBuilds(srvId, false, paginateUntil);
+ runActualizeBuildRefs(srvNme, false, paginateUntil);
if(!paginateUntil.isEmpty()) {
//some builds may stuck in the queued or running, enforce loading
as well
- scheduleBuildsLoad(paginateUntil);
+ buildSync.scheduleBuildsLoad(conn, paginateUntil);
}
// schedule full resync later
- scheduler.invokeLater(this::sheduleResync, 15, TimeUnit.MINUTES);
+ scheduler.invokeLater(this::sheduleResyncBuildRefs, 15,
TimeUnit.MINUTES);
}
/**
*
*/
- private void sheduleResync() {
+ private void sheduleResyncBuildRefs() {
scheduler.sheduleNamed(taskName("fullReindex"), this::fullReindex,
120, TimeUnit.MINUTES);
}
@@ -276,25 +195,27 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
*
*/
void fullReindex() {
- runActualizeBuilds(srvId, true, null);
+ runActualizeBuildRefs(srvNme, true, null);
}
+
/**
- * @param srvId Server id. todo to be added as composite name extend
+ * @param srvId Server id.
* @param fullReindex Reindex all builds from TC history.
* @param mandatoryToReload [in/out] Build ID should be found before end
of sync. Ignored if fullReindex mode.
*
*/
+ @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
@MonitoredTask(name = "Actualize BuildRefs(srv, full resync)",
nameExtArgsIndexes = {0, 1})
@AutoProfiling
- protected String runActualizeBuilds(String srvId, boolean fullReindex,
- @Nullable Set<Integer> mandatoryToReload) {
+ protected String runActualizeBuildRefs(String srvId, boolean fullReindex,
+ @Nullable Set<Integer>
mandatoryToReload) {
AtomicReference<String> outLinkNext = new AtomicReference<>();
List<BuildRef> tcDataFirstPage = conn.getBuildRefs(null, outLinkNext);
Set<Long> buildsUpdated = buildRefDao.saveChunk(srvIdMaskHigh,
tcDataFirstPage);
int totalUpdated = buildsUpdated.size();
- scheduleBuildsLoad(cacheKeysToBuildIds(buildsUpdated));
+ buildSync.scheduleBuildsLoad(conn, cacheKeysToBuildIds(buildsUpdated));
int totalChecked = tcDataFirstPage.size();
int neededToFind = 0;
@@ -310,7 +231,7 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
List<BuildRef> tcDataNextPage = conn.getBuildRefs(nextPageUrl,
outLinkNext);
Set<Long> curChunkBuildsSaved =
buildRefDao.saveChunk(srvIdMaskHigh, tcDataNextPage);
totalUpdated += curChunkBuildsSaved.size();
- scheduleBuildsLoad(cacheKeysToBuildIds(curChunkBuildsSaved));
+ buildSync.scheduleBuildsLoad(conn,
cacheKeysToBuildIds(curChunkBuildsSaved));
int savedCurChunk = curChunkBuildsSaved.size();
@@ -332,47 +253,4 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
@NotNull private List<Integer> cacheKeysToBuildIds(Collection<Long>
cacheKeysUpdated) {
return
cacheKeysUpdated.stream().map(BuildRefDao::cacheKeyToBuildId).collect(Collectors.toList());
}
-
- /** */
- private void loadFatBuilds(int ldrNo) {
- Set<Integer> load;
-
- synchronized (this) {
- load = buildToLoad;
- buildToLoad = new HashSet<>();
- }
-
- doLoadBuilds(ldrNo, srvId, load);
- }
-
- @MonitoredTask(name = "Proactive Builds Loading (agent,server)",
nameExtArgsIndexes = {0, 1})
- @AutoProfiling
- protected String doLoadBuilds(int ldrNo, String srvId, Set<Integer> load) {
- if(load.isEmpty())
- return "Nothing to load";
-
- AtomicInteger err = new AtomicInteger();
- AtomicInteger ld = new AtomicInteger();
-
- Map<Long, FatBuildCompacted> builds =
fatBuildDao.getAllFatBuilds(srvIdMaskHigh, load);
-
- load.forEach(
- buildId -> {
- try {
- FatBuildCompacted existingBuild =
builds.get(FatBuildDao.buildIdToCacheKey(srvIdMaskHigh, buildId));
-
- FatBuildCompacted savedVer = reloadBuild(buildId,
existingBuild);
-
- if (savedVer != null)
- ld.incrementAndGet();
- }
- catch (Exception e) {
- logger.error("", e);
- err.incrementAndGet();
- }
- }
- );
-
- return "Builds updated " + ld.get() + " from " + load.size() + "
requested, errors: " + err;
- }
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java
index e397d8f..9a5d68c 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java
@@ -20,6 +20,7 @@ import com.google.inject.AbstractModule;
import com.google.inject.internal.SingletonScope;
import org.apache.ignite.ci.tcbot.condition.BuildConditionDao;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProactiveFatBuildSync;
import org.apache.ignite.ci.teamcity.pure.ITeamcityHttpConnection;
import org.apache.ignite.ci.teamcity.restcached.TcRestCachedModule;
import org.jetbrains.annotations.Nullable;
@@ -37,6 +38,7 @@ public class TeamcityIgnitedModule extends AbstractModule {
bind(BuildRefDao.class).in(new SingletonScope());
bind(BuildConditionDao.class).in(new SingletonScope());
bind(FatBuildDao.class).in(new SingletonScope());
+ bind(ProactiveFatBuildSync.class).in(new SingletonScope());
bind(IStringCompactor.class).to(IgniteStringCompactor.class).in(new
SingletonScope());
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index 59701f2..231ed5d 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -39,7 +39,7 @@ import org.jetbrains.annotations.Nullable;
@Persisted
public class FatBuildCompacted extends BuildRefCompacted implements
IVersionedEntity {
/** Latest version. */
- private static final int LATEST_VERSION = 1;
+ private static final int LATEST_VERSION = 2;
/** Default branch flag offset. */
public static final int DEF_BR_F = 0;
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
index fd8b8df..afe4812 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
@@ -24,15 +24,20 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+import javax.cache.Cache;
import javax.inject.Inject;
import javax.inject.Provider;
import javax.validation.constraints.NotNull;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.ci.db.TcHelperDb;
+import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefDao;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.apache.ignite.internal.util.GridIntList;
import org.jetbrains.annotations.Nullable;
/**
@@ -120,4 +125,16 @@ public class FatBuildDao {
return buildsCache.getAll(ids);
}
+
+ /**
+ * @param key Key.
+ * @param srvId Server id.
+ */
+ private boolean isKeyForServer(Long key, int srvId) {
+ return key!=null && key >> 32 == srvId;
+ }
+
+ public boolean containsKey(int srvIdMaskHigh, int buildRefKey) {
+ return buildsCache.containsKey(buildIdToCacheKey(srvIdMaskHigh,
buildRefKey));
+ }
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
new file mode 100644
index 0000000..8eac870
--- /dev/null
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ci.teamcity.ignited.fatbuild;
+
+import com.google.common.base.Strings;
+import com.google.common.base.Throwables;
+import org.apache.ignite.ci.di.AutoProfiling;
+import org.apache.ignite.ci.di.MonitoredTask;
+import org.apache.ignite.ci.di.scheduler.IScheduler;
+import org.apache.ignite.ci.tcmodel.result.Build;
+import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefDao;
+import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
+import org.apache.ignite.ci.teamcity.pure.ITeamcityConn;
+import org.apache.ignite.ci.util.ExceptionUtil;
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.GuardedBy;
+import javax.inject.Inject;
+import java.io.FileNotFoundException;
+import java.util.*;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class ProactiveFatBuildSync {
+ public static final int FAT_BUILD_PROACTIVE_TASKS = 4;
+
+ /** Logger. */
+ private static final Logger logger =
LoggerFactory.getLogger(ProactiveFatBuildSync.class);
+
+ /** Build reference DAO. */
+ @Inject
+ private BuildRefDao buildRefDao;
+
+ /** Build DAO. */
+ @Inject private FatBuildDao fatBuildDao;
+
+ /** Scheduler. */
+ @Inject private IScheduler scheduler;
+
+ @Inject private IStringCompactor compactor;
+
+ @GuardedBy("this")
+ private Map<String, SyncTask> buildToLoad = new HashMap<>();
+
+ private static class SyncTask {
+ ITeamcityConn conn;
+ Set<Integer> ids = new HashSet<>();
+ }
+
+ /**
+ * Invoke load fat builds later, re-load provided builds.
+ * @param conn
+ * @param buildsToAskFromTc Builds to ask from tc.
+ */
+ public void scheduleBuildsLoad(ITeamcityConn conn, Collection<Integer>
buildsToAskFromTc) {
+ if (buildsToAskFromTc.isEmpty())
+ return;
+
+
+ synchronized (this) {
+ final String serverId = conn.serverId();
+ final SyncTask syncTask = buildToLoad.computeIfAbsent(serverId, s
-> new SyncTask());
+ syncTask.conn = conn;
+ syncTask.ids.addAll(buildsToAskFromTc);
+ }
+
+ int ldrToActivate =
ThreadLocalRandom.current().nextInt(FAT_BUILD_PROACTIVE_TASKS);
+
+ scheduler.sheduleNamed(taskName("loadFatBuilds" + ldrToActivate,
conn.serverId()),
+ () -> loadFatBuilds(ldrToActivate, conn.serverId()), 2,
TimeUnit.MINUTES);
+
+ }
+
+
+ @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
+ @MonitoredTask(name = "Find missing builds", nameExtArgsIndexes = {0})
+ @AutoProfiling
+ protected String findMissingBuildsFromBuildRef(String srvId, ITeamcityConn
conn) {
+ int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
+
+ final int[] buildRefKeys = buildRefDao.getAllIds(srvIdMaskHigh);
+
+ Arrays.parallelSort(buildRefKeys);
+
+ List<Integer> buildsIdsToLoad = new ArrayList<>();
+ int totalAskedToLoad = 0;
+
+ for (int buildRefKey : buildRefKeys) {
+ if (!fatBuildDao.containsKey(srvIdMaskHigh, buildRefKey))
+ buildsIdsToLoad.add(buildRefKey);
+
+ if (buildsIdsToLoad.size() >= 100) {
+ totalAskedToLoad += buildsIdsToLoad.size();
+ scheduleBuildsLoad(conn, buildsIdsToLoad);
+ buildsIdsToLoad.clear();
+ }
+ }
+
+ if (!buildsIdsToLoad.isEmpty()) {
+ totalAskedToLoad += buildsIdsToLoad.size();
+ scheduleBuildsLoad(conn, buildsIdsToLoad);
+ }
+
+ return "Invoked later load for " + totalAskedToLoad + " builds from "
+ srvId;
+ }
+
+ /** */
+ private void loadFatBuilds(int ldrNo, String serverId) {
+ Set<Integer> load;
+ ITeamcityConn conn;
+ synchronized (this) {
+ final SyncTask syncTask = buildToLoad.get(serverId);
+ if (syncTask == null)
+ return;
+
+ if (syncTask.ids.isEmpty()) {
+ syncTask.conn = null;
+ return;
+ }
+
+ if (syncTask.conn == null)
+ return;
+
+ load = syncTask.ids;
+ syncTask.ids = new HashSet<>();
+
+ conn = syncTask.conn;
+ syncTask.conn = null;
+ }
+
+ doLoadBuilds(ldrNo, serverId, conn, load);
+ }
+
+ @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
+ @MonitoredTask(name = "Proactive Builds Loading (srv,agent)",
nameExtArgsIndexes = {1, 0})
+ @AutoProfiling
+ protected String doLoadBuilds(int ldrNo, String srvId, ITeamcityConn conn,
Set<Integer> load) {
+ if(load.isEmpty())
+ return "Nothing to load";
+
+ final int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
+
+ AtomicInteger err = new AtomicInteger();
+ AtomicInteger ld = new AtomicInteger();
+
+ Map<Long, FatBuildCompacted> builds =
fatBuildDao.getAllFatBuilds(srvIdMaskHigh, load);
+
+ load.forEach(
+ buildId -> {
+ try {
+ FatBuildCompacted existingBuild =
builds.get(FatBuildDao.buildIdToCacheKey(srvIdMaskHigh, buildId));
+
+ FatBuildCompacted savedVer = reloadBuild(conn,
buildId, existingBuild);
+
+ if (savedVer != null)
+ ld.incrementAndGet();
+ }
+ catch (Exception e) {
+ logger.error("", e);
+ err.incrementAndGet();
+ }
+ }
+ );
+
+ return "Builds updated " + ld.get() + " from " + load.size() + "
requested, errors: " + err;
+ }
+
+ @NotNull
+ private String taskName(String taskName, String srvName) {
+ return ProactiveFatBuildSync.class.getSimpleName() +"." + taskName +
"." + srvName;
+ }
+
+ public void invokeLaterFindMissingByBuildRef(String srvName, ITeamcityConn
conn) {
+ scheduler.sheduleNamed(taskName("findMissingBuildsFromBuildRef",
srvName),
+ () -> findMissingBuildsFromBuildRef(srvName, conn), 360,
TimeUnit.MINUTES);
+ }
+
+ /**
+ *
+ * @param conn
+ * @param buildId
+ * @param existingBuild
+ * @return new build if it was updated or null if no updates detected
+ */
+ @SuppressWarnings({"WeakerAccess"})
+ @AutoProfiling
+ public FatBuildCompacted reloadBuild(ITeamcityConn conn, int buildId,
@Nullable FatBuildCompacted existingBuild) {
+ // System.err.println(Thread.currentThread().getName()+ ": Build " +
buildId);
+ //todo some sort of locking to avoid double requests
+
+ final String srvNme = conn.serverId();
+ final int srvIdMask = ITeamcityIgnited.serverIdToInt(srvNme);
+
+ Build build;
+ List<TestOccurrencesFull> tests = new ArrayList<>();
+ try {
+ build = conn.getBuild(buildId);
+
+ String nextHref = null;
+ do {
+ boolean testDtls = !build.isComposite(); // don't query test
details for compoite
+ TestOccurrencesFull page = conn.getTestsPage(buildId,
nextHref, testDtls);
+ nextHref = page.nextHref();
+
+ tests.add(page);
+ }
+ while (!Strings.isNullOrEmpty(nextHref));
+ }
+ catch (Exception e) {
+ if (Throwables.getRootCause(e) instanceof FileNotFoundException) {
+ logger.info("Loading build [" + buildId + "] for server [" +
srvNme + "] failed:" + e.getMessage(), e);
+
+ if (existingBuild != null) {
+ build = existingBuild.toBuild(compactor);
+
+ if(build.isRunning() || build.isQueued())
+ build.setCancelled();
+
+ tests =
Collections.singletonList(existingBuild.getTestOcurrences(compactor));
+ }
+ else
+ build = Build.createFakeStub();
+ } else {
+ logger.error("Loading build [" + buildId + "] for server [" +
srvNme + "] failed:" + e.getMessage(), e);
+
+ e.printStackTrace();
+
+ throw ExceptionUtil.propagateException(e);
+ }
+ }
+
+ //if we are here because of some sort of outdated version of build,
+ // new save will be performed with new entity version for compacted
build
+ return fatBuildDao.saveBuild(srvIdMask, buildId, build, tests,
existingBuild);
+ }
+}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
index 13e4c5b..37a32bc 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
@@ -19,9 +19,16 @@ package org.apache.ignite.ci.teamcity.ignited.fatbuild;
import com.google.common.base.Objects;
import com.google.common.base.Strings;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.BitSet;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
import org.apache.ignite.ci.analysis.RunStat;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
@@ -41,6 +48,13 @@ public class TestCompacted {
public static final int CUR_MUTED_F = 2;
public static final int CUR_INV_F = 4;
public static final int IGNORED_F = 6;
+ /** true when kept uncompressed */
+ public static final int COMPRESS_TYPE_FLAG1 = 8;
+ /** true when kept gzip */
+ public static final int COMPRESS_TYPE_FLAG2 = 9;
+ public static final int COMPRESS_TYPE_RFU3 = 10;
+ public static final int COMPRESS_TYPE_RFU4 = 11;
+
/** Id in this build only. Does not idenfity test for its history */
private int idInBuild = -1;
@@ -56,9 +70,8 @@ public class TestCompacted {
/** Actual build id. */
private int actualBuildId = -1;
- /** Snappy comressed test log Details. */
- private byte[] details;
-
+ /** Uncompressesd/ZIP/Snappy compressed test log Details. */
+ @Nullable private byte[] details;
/** Logger. */
private static final Logger logger =
LoggerFactory.getLogger(TestCompacted.class);
@@ -77,9 +90,10 @@ public class TestCompacted {
String testOccurrenceId = testOccurrence.getId();
if (!Strings.isNullOrEmpty(testOccurrenceId)) {
try {
- idInBuild =
RunStat.extractFullId(testOccurrenceId).getTestId();
- }
- catch (Exception e) {
+ final RunStat.TestId testId =
RunStat.extractFullId(testOccurrenceId);
+ if (testId != null)
+ idInBuild = testId.getTestId();
+ } catch (Exception e) {
logger.error("Failed to handle TC response: " +
testOccurrenceId, e);
}
}
@@ -99,15 +113,7 @@ public class TestCompacted {
if (testOccurrence.test != null && testOccurrence.test.id != null)
testId = testOccurrence.test.id;
- String details = testOccurrence.details;
- if (!Strings.isNullOrEmpty(details)) {
- try {
- this.details =
Snappy.compress(details.getBytes(StandardCharsets.UTF_8));
- }
- catch (Exception e) {
- logger.error("Snappy.compress failed: " + e.getMessage(), e);
- }
- }
+ setDetails(testOccurrence.details);
}
private void setFlag(int off, Boolean val) {
@@ -177,17 +183,98 @@ public class TestCompacted {
if (details == null)
return "";
+ final boolean flag1 = flags.get(COMPRESS_TYPE_FLAG1);
+ final boolean flag2 = flags.get(COMPRESS_TYPE_FLAG2);
+ if(!flag1 && !flag2) {
+ try {
+ byte[] uncompressed = Snappy.uncompress(details);
+ return new String(uncompressed, StandardCharsets.UTF_8);
+ } catch (IOException e) {
+ logger.error("Snappy.uncompress failed: " + e.getMessage(), e);
+ return null;
+ }
+ } else if(flag1 && !flag2) {
+ return new String(details, StandardCharsets.UTF_8);
+ } else if (!flag1 && flag2) {
+ try {
+ final ByteArrayInputStream in = new
ByteArrayInputStream(details);
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ try (final GZIPInputStream gzi = new GZIPInputStream(in)) {
+ byte[] outbuf = new byte[details.length];
+ int len;
+ while ((len = gzi.read(outbuf, 0, outbuf.length)) != -1) {
+ bos.write(outbuf, 0, len);
+ }
+ }
+
+ return new String(bos.toByteArray(), StandardCharsets.UTF_8);
+ } catch (Exception e) {
+ logger.error("GZip.uncompress failed: " + e.getMessage(), e);
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+
+ public void setDetails(String dtlsString) {
+ if (Strings.isNullOrEmpty(dtlsString)) {
+ this.details = null;
+ return;
+ }
+
+
+ byte[] uncompressed;
+ byte[] snappy = null;
+ byte[] gzip = null;
+ try {
+ uncompressed = dtlsString.getBytes(StandardCharsets.UTF_8);
+ } catch (Exception e) {
+ logger.error("Set details failed: " + e.getMessage(), e);
+ return;
+ }
try {
- byte[] uncompressed = Snappy.uncompress(details);
- return new String(uncompressed, StandardCharsets.UTF_8);
+ snappy = Snappy.compress(uncompressed);
+ }
+ catch (Exception e) {
+ logger.error("Snappy.compress failed: " + e.getMessage(), e);
}
- catch (IOException e) {
- logger.error("Snappy.uncompress failed: " + e.getMessage(), e);
- return null;
+ try {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try(final GZIPOutputStream gzipOutputStream = new
GZIPOutputStream(out)) {
+ gzipOutputStream.write(uncompressed);
+ }
+ gzip = out.toByteArray();
+ }
+ catch (Exception e) {
+ logger.error("Snappy.compress failed: " + e.getMessage(), e);
+ }
+
+ final int snappyLen = snappy!=null ? snappy.length : -1;
+ final int gzipLen = gzip!=null ? gzip.length : -1;
+ flags.set(COMPRESS_TYPE_FLAG1, true);
+ flags.set(COMPRESS_TYPE_FLAG2, false);
+ //uncompressed
+ details = uncompressed;
+
+ if (snappyLen > 0 && snappyLen < details.length) {
+ flags.set(COMPRESS_TYPE_FLAG1, false);
+ flags.set(COMPRESS_TYPE_FLAG2, false);
+ details = snappy;
+ }
+
+ if (gzipLen > 0 && gzipLen < details.length) {
+ flags.set(COMPRESS_TYPE_FLAG1, false);
+ flags.set(COMPRESS_TYPE_FLAG2, true);
+ details = gzip;
}
+
+ logger.info("U " + uncompressed.length + " S " + snappyLen + " Z " +
gzipLen + ": F (" +
+ flags.get(COMPRESS_TYPE_FLAG1) + ", " +
+ flags.get(COMPRESS_TYPE_FLAG2) +")");
}
public Boolean getIgnoredFlag() {
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
index 78f3d34..f832088 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
@@ -30,6 +30,11 @@ import
org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
*/
public interface ITeamcityConn {
/**
+ * @return Internal server ID as string
+ */
+ String serverId();
+
+ /**
* @return Normalized Host address, ends with '/'.
*/
public String host();
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
index b58b699..c082100 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
@@ -23,7 +23,7 @@ package org.apache.ignite.ci.web.model;
public static final String GITHUB_REF =
"https://github.com/apache/ignite-teamcity-bot";;
/** TC Bot Version. */
- public static final String VERSION = "20181026";
+ public static final String VERSION = "20181027";
/** TC Bot Version. */
public String version = VERSION;
diff --git
a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
index b07e102..173a959 100644
---
a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
+++
b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
@@ -214,7 +214,7 @@ public class IgnitedTcInMemoryIntegrationTest {
for (int i = queuedBuildIdx; i < tcBuilds.size(); i++)
tcBuilds.get(i).state = BuildRef.STATE_FINISHED;
- teamcityIgnited.actualizeRecentBuilds();
+ teamcityIgnited.actualizeRecentBuildRefs();
List<BuildRef> hist = srv.getBuildHistory(buildTypeId, branchName);
