This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-10562
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-10562 by this push:
new 68c0b05 IGNITE-10562 Change logic of build diverged handling: TC Bot
DB contains invalid values for build references ID maps to value with other ID.
68c0b05 is described below
commit 68c0b05bda5a13809b56d32aa180ac351d45f92d
Author: Dmitriy Pavlov <[email protected]>
AuthorDate: Thu Dec 6 20:04:49 2018 +0300
IGNITE-10562 Change logic of build diverged handling: TC Bot DB contains
invalid values for build references ID maps to value with other ID.
---
.../ci/teamcity/ignited/BuildRefCompacted.java | 12 +++-
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 21 +++++--
.../ignited/fatbuild/FatBuildCompacted.java | 12 +++-
.../ci/teamcity/ignited/fatbuild/FatBuildDao.java | 30 ++-------
.../ignited/fatbuild/ProactiveFatBuildSync.java | 73 ++++++++++++++--------
5 files changed, 91 insertions(+), 57 deletions(-)
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
index ada9317..599a4cb 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
@@ -66,7 +66,7 @@ public class BuildRefCompacted {
state = compactor.getStringId(ref.state());
}
- public void setId(Integer buildId) {
+ public void setId(@Nullable Integer buildId) {
this.id = buildId == null ? -1 : buildId;
}
@@ -158,11 +158,21 @@ public class BuildRefCompacted {
}
/** */
+ protected void status(int status) {
+ this.status = status;
+ }
+
+ /** */
public int state() {
return state;
}
/** */
+ public void state(int state) {
+ this.state = state;
+ }
+
+ /** */
public boolean isFakeStub() {
return id() < 0;
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 8112ff5..f3125cb 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -386,7 +386,7 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
* Enables scheduleing for build refs/builds/history sync
*/
public void ensureActualizeRequested() {
- scheduler.sheduleNamed(taskName("actualizeRecentBuildRefs"),
this::actualizeRecentBuildRefs, 2, TimeUnit.MINUTES);
+ scheduler.sheduleNamed(taskName("actualizeRecentBuildRefs"), () ->
actualizeRecentBuildRefs(srvNme), 2, TimeUnit.MINUTES);
// schedule find missing later
buildSync.ensureActualizationRequested(srvNme, conn);
@@ -488,11 +488,16 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
return changes.values();
}
+ String actualizeRecentBuildRefs() {
+ return actualizeRecentBuildRefs(srvNme);
+ }
/**
*
+ * @param srvNme TC service name
*/
- void actualizeRecentBuildRefs() {
+ @MonitoredTask(name = "Prepare Actualize BuildRefs(srv, full resync)",
nameExtArgsIndexes = {0})
+ String actualizeRecentBuildRefs(String srvNme) {
List<BuildRefCompacted> running =
buildRefDao.getQueuedAndRunning(srvIdMaskHigh);
Set<Integer> paginateUntil = new HashSet<>();
@@ -501,18 +506,22 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
List<Integer> runningIds =
running.stream().map(BuildRefCompacted::id).collect(Collectors.toList());
OptionalInt max = runningIds.stream().mapToInt(i -> i).max();
if (max.isPresent()) {
- runningIds.forEach(id->{
- if(id > (max.getAsInt() -
MAX_ID_DIFF_TO_ENFORCE_CONTINUE_SCAN))
+ runningIds.forEach(id -> {
+ if (id > (max.getAsInt() -
MAX_ID_DIFF_TO_ENFORCE_CONTINUE_SCAN))
paginateUntil.add(id);
else
directUpload.add(id);
});
}
+
+ int cntFreshBuilds = paginateUntil.size();
+
//schedule direct reload for Fat Builds for all queued too-old builds
buildSync.scheduleBuildsLoad(conn, directUpload);
runActualizeBuildRefs(srvNme, false, paginateUntil);
+ int freshButNotFoundByBuildsRefsScan = paginateUntil.size();
if (!paginateUntil.isEmpty()) {
//some builds may stuck in the queued or running, enforce loading
now
buildSync.doLoadBuilds(-1, srvNme, conn, paginateUntil);
@@ -520,6 +529,10 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
// schedule full resync later
scheduler.invokeLater(this::sheduleResyncBuildRefs, 15,
TimeUnit.MINUTES);
+
+ return "Build queue " + running.size() + ", relatively fresh: " +
cntFreshBuilds +
+ ", fresh but not found by scan: " +
freshButNotFoundByBuildsRefsScan +
+ ", old builds sheduled " + directUpload.size();
}
/**
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index 3f4ff2b..31cba27 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -506,7 +506,15 @@ public class FatBuildCompacted extends BuildRefCompacted
implements IVersionedEn
return invocation;
}
- public void setVersion(short version) {
- this._ver = version;
+ public void setVersion(short ver) {
+ this._ver = ver;
}
+
+ public FatBuildCompacted setCancelled(IStringCompactor compactor) {
+ status(compactor.getStringId(BuildRef.STATUS_UNKNOWN));
+ state(compactor.getStringId(BuildRef.STATE_FINISHED));
+
+ return this;
+ }
+
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
index b7257e1..6a3381d 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
@@ -23,8 +23,9 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
-import java.util.function.BiConsumer;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
import javax.annotation.Nullable;
import javax.cache.Cache;
import javax.inject.Inject;
@@ -32,8 +33,6 @@ import javax.inject.Provider;
import javax.validation.constraints.NotNull;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.query.QueryCursor;
-import org.apache.ignite.cache.query.SqlQuery;
import org.apache.ignite.ci.db.TcHelperDb;
import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.tcmodel.changes.ChangesList;
@@ -123,12 +122,6 @@ public class FatBuildDao {
buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId), newBuild);
}
-
- @AutoProfiling
- public boolean removeFatBuild(int srvIdMaskHigh, int buildId) {
- return buildsCache.remove(buildIdToCacheKey(srvIdMaskHigh, buildId));
- }
-
public static int[] extractChangeIds(@NotNull ChangesList changesList) {
return changesList.changes().stream().mapToInt(
ch -> {
@@ -188,20 +181,9 @@ public class FatBuildDao {
return buildsCache.containsKey(buildIdToCacheKey(srvIdMaskHigh,
buildId));
}
- public void forEachBuildsV5(int srvId, BiConsumer<Long, FatBuildCompacted>
processor) {
- try (QueryCursor<Cache.Entry<Long, FatBuildCompacted>> qryCursor =
buildsCache.query(
- new SqlQuery<Long, FatBuildCompacted>(FatBuildCompacted.class,
"_ver < ?")
- .setArgs(FatBuildCompacted.LATEST_VERSION))) {
-
- for (Cache.Entry<Long, FatBuildCompacted> next : qryCursor) {
- Long key = next.getKey();
-
- if (!isKeyForServer(key, srvId))
- continue;
-
- processor.accept(next.getKey(), next.getValue());
- }
- }
-
+ public Stream<Cache.Entry<Long, FatBuildCompacted>>
outdatedVersionEntries(int srvId) {
+ return StreamSupport.stream(buildsCache.spliterator(), false)
+ .filter(entry -> entry.getValue().isOutdatedEntityVersion())
+ .filter(entry -> isKeyForServer(entry.getKey(), srvId));
}
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
index a77b899..b41ed9b 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -16,11 +16,9 @@
*/
package org.apache.ignite.ci.teamcity.ignited.fatbuild;
-import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.base.Throwables;
import java.util.stream.Stream;
-import org.apache.ignite.ci.ITeamcity;
import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.di.MonitoredTask;
import org.apache.ignite.ci.di.scheduler.IScheduler;
@@ -245,22 +243,36 @@ public class ProactiveFatBuildSync {
*/
public void ensureActualizationRequested(String srvName, ITeamcityConn
conn) {
scheduler.sheduleNamed(taskName("findMissingBuildsFromBuildRef",
srvName),
- () -> findMissingBuildsFromBuildRef(srvName, conn), 360,
TimeUnit.MINUTES);
+ () -> findMissingBuildsFromBuildRef(srvName, conn), 360,
TimeUnit.MINUTES);
scheduler.sheduleNamed(taskName("migrateBuildsToV6", srvName),
() -> migrateBuildsToV6(srvName, conn), 8, TimeUnit.HOURS);
}
+ /**
+ * @param srvName Server name.
+ * @param conn Connection.
+ */
@MonitoredTask(name = "Migrate Builds to V6", nameExtArgsIndexes = {0})
public String migrateBuildsToV6(String srvName, ITeamcityConn conn) {
int srvId = ITeamcityIgnited.serverIdToInt(srvName);
AtomicInteger cnt = new AtomicInteger();
- fatBuildDao.forEachBuildsV5(srvId, (id, fatBuild) -> {
+ AtomicInteger divergedIds = new AtomicInteger();
+ fatBuildDao.outdatedVersionEntries(srvId).forEach(entry -> {
cnt.incrementAndGet();
+ int buildId = BuildRefDao.cacheKeyToBuildId(entry.getKey());
+ FatBuildCompacted transformed = transformV5Build(
+ srvId,
+ buildId,
+ entry.getValue());
+
+ if (transformed != null)
+ divergedIds.incrementAndGet();
});
- return cnt.get() + " builds found having version < 6";
+ return "Found: " + cnt.get() + " builds found having version < 6 and "
+ + divergedIds.get() + " with ID divergence.";
}
/**
@@ -320,23 +332,8 @@ public class ProactiveFatBuildSync {
final int srvIdMask = ITeamcityIgnited.serverIdToInt(srvName);
if (existingBuild != null && existingBuild.isOutdatedEntityVersion()) {
- int ver = existingBuild.version();
- if (ver ==
FatBuildCompacted.VER_FULL_DATA_BUT_ID_CONFLICTS_POSSIBLE) {
- if (Objects.equals(buildId, existingBuild.id())) {
- existingBuild.setVersion(FatBuildCompacted.LATEST_VERSION);
-
- fatBuildDao.putFatBuild(srvIdMask, buildId, existingBuild);
-
- return null;
- }
- else {
- logger.warn("Build inconsistency found in the DB, removing
build " + existingBuild.getId());
-
- existingBuild = null;
-
- fatBuildDao.removeFatBuild(srvIdMask, buildId);
- }
- }
+ if (existingBuild.version() ==
FatBuildCompacted.VER_FULL_DATA_BUT_ID_CONFLICTS_POSSIBLE)
+ return transformV5Build(srvIdMask, buildId, existingBuild);
}
Build build;
@@ -349,9 +346,11 @@ public class ProactiveFatBuildSync {
if (build.isFakeStub())
build.setCancelled(); // probably now it will not happen
because of direct connection to TC.
- else
- Preconditions.checkState(Objects.equals(build.getId(),
buildId),
- "Build IDs are not consistent: returned " + build.getId()
+ " queued is " + buildId);
+ else {
+ if(!Objects.equals(build.getId(), buildId))
+ throw new FileNotFoundException(
+ "Build IDs are not consistent: returned " +
build.getId() + " queued is " + buildId);
+ }
if (build.testOccurrences != null && !build.isComposite()) { //
don't query tests for compoite
String nextHref = null;
@@ -397,7 +396,7 @@ public class ProactiveFatBuildSync {
problems = existingBuild.problems(compactor);
- //todo extract new parameters or save fat build without XML
+ //todo extract new parameters or save fat build without
XML convertions
// - existingBuild.statistics();
// - int[] changes = existingBuild.changes();
}
@@ -419,4 +418,26 @@ public class ProactiveFatBuildSync {
// new save will be performed with new entity version for compacted
build
return fatBuildDao.saveBuild(srvIdMask, buildId, build, tests,
problems, statistics, changesList, existingBuild);
}
+
+ @Nullable
+ public FatBuildCompacted transformV5Build(int srvIdMask, int buildId,
@NotNull FatBuildCompacted existingBuild) {
+ if (Objects.equals(buildId, existingBuild.id())) {
+ existingBuild.setVersion(FatBuildCompacted.LATEST_VERSION);
+
+ fatBuildDao.putFatBuild(srvIdMask, buildId, existingBuild);
+
+ return null;
+ }
+ else {
+ logger.warn("Build inconsistency found in the DB, removing build "
+ existingBuild.getId());
+
+ FatBuildCompacted buildCompacted = new FatBuildCompacted()
+ .setFakeStub(true)
+ .setCancelled(compactor);
+
+ fatBuildDao.putFatBuild(srvIdMask, buildId, buildCompacted);
+
+ return buildCompacted;
+ }
+ }
}
