This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new c861b45 IGNITE-10562 TC Bot DB contains invalid values for build
references ID maps to value with other ID. - Fixes #92.
c861b45 is described below
commit c861b457c1dbd35a2a2d30998b827a98916b049b
Author: Dmitriy Pavlov <[email protected]>
AuthorDate: Thu Dec 6 14:57:28 2018 +0300
IGNITE-10562 TC Bot DB contains invalid values for build references ID maps
to value with other ID. - Fixes #92.
Signed-off-by: Dmitriy Pavlov <[email protected]>
---
.../main/java/org/apache/ignite/ci/ITeamcity.java | 10 +-------
.../apache/ignite/ci/IgnitePersistentTeamcity.java | 6 +++++
.../apache/ignite/ci/IgniteTeamcityConnection.java | 6 +++++
.../apache/ignite/ci/runners/ClientTmpHelper.java | 14 ++++++++--
.../ignite/ci/runners/RemoteClientTmpHelper.java | 30 +++++++++++++++++-----
.../ci/tcbot/builds/CompareBuildsService.java | 2 +-
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 6 ++---
.../ignited/fatbuild/FatBuildCompacted.java | 18 +++++++++++--
.../ci/teamcity/ignited/fatbuild/FatBuildDao.java | 6 +++++
.../ignited/fatbuild/ProactiveFatBuildSync.java | 24 ++++++++++++++---
.../org/apache/ignite/ci/web/model/Version.java | 2 +-
11 files changed, 95 insertions(+), 29 deletions(-)
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
index 62332e1..80ed20d 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
@@ -68,15 +68,7 @@ public interface ITeamcity extends ITeamcityConn {
@Deprecated
Build getBuild(String href);
- default Build getBuild(int buildId) {
- return getBuild(getBuildHrefById(buildId));
- }
-
- @NotNull default String getBuildHrefById(int id) {
- return buildHref(id);
- }
-
- @NotNull static String buildHref(int id) {
+ @NotNull public static String buildHref(int id) {
return "app/rest/latest/builds/id:" + id;
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
index 4b256ce..88f486c 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
@@ -325,6 +325,7 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
return new SuiteInBranch(loaded.suiteId(),
normalizeBranch(loaded.branchName));
}
+ @Deprecated
private Build realLoadBuild(String href1) {
try {
return teamcity.getBuild(href1);
@@ -352,6 +353,11 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
return teamcity.host();
}
+ /** {@inheritDoc} */
+ @Override public Build getBuild(int buildId) {
+ return teamcity.getBuild(buildId);
+ }
+
@Deprecated
private void registerCriticalBuildProblemInStat(BuildRef build,
ProblemOccurrences problems) {
boolean criticalFail =
problems.getProblemsNonNull().stream().anyMatch(occurrence ->
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
index ce5a319..40e5c3a 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
@@ -446,6 +446,12 @@ public class IgniteTeamcityConnection implements ITeamcity
{
return getJaxbUsingHref(href, Build.class);
}
+ /** {@inheritDoc} */
+ @AutoProfiling
+ @Override public Build getBuild(int buildId) {
+ return getJaxbUsingHref(ITeamcity.buildHref(buildId), Build.class);
+ }
+
/**
* @param href Href.
* @param elem Element class.
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/ClientTmpHelper.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/ClientTmpHelper.java
index 14207a0..f05804f 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/ClientTmpHelper.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/ClientTmpHelper.java
@@ -28,14 +28,24 @@ import org.apache.ignite.ci.user.UserAndSessionsStorage;
* Utility class for local connection to TC helper DB (server) and any
manipulations with data needed.
*/
public class ClientTmpHelper {
+ /**
+ * @param args Args.
+ */
public static void main(String[] args) {
- try (Ignite ignite = TcHelperDb.startClient()){
+ int inconsistent;
+ try (Ignite ignite = TcHelperDb.startClient()) {
RemoteClientTmpHelper.DUMPS = "dumpsLocal";
- RemoteClientTmpHelper.validateBuildIdConsistency(ignite);
+ inconsistent =
RemoteClientTmpHelper.validateBuildIdConsistency(ignite);
}
+
+ System.err.println("Inconsistent builds in queue found [" +
+ +inconsistent + "]");
}
+ /**
+ * @param args Args.
+ */
public static void main0(String[] args) {
Ignite ignite = TcHelperDb.startClient();
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java
index 07c7ef9..d987067 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java
@@ -23,6 +23,7 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.UncheckedIOException;
+import java.util.concurrent.atomic.AtomicInteger;
import javax.cache.Cache;
import javax.xml.bind.JAXBException;
import org.apache.ignite.Ignite;
@@ -49,13 +50,21 @@ public class RemoteClientTmpHelper {
public static String DUMPS = "dumps";
private static boolean dumpDict = false;
+ /**
+ * @param args Args.
+ */
public static void main(String[] args) {
+ int inconsistent;
+
try (Ignite ignite = tcbotServer()) {
- validateBuildIdConsistency(ignite);
+ inconsistent = validateBuildIdConsistency(ignite);
}
+
+ System.err.println("Inconsistent builds in queue found [" +
inconsistent + "]");
}
- public static void validateBuildIdConsistency(Ignite ignite) {
+ public static int validateBuildIdConsistency(Ignite ignite) {
+ AtomicInteger inconsistent = new AtomicInteger();
String apacheSrvName = "apache";
int apache = ITeamcityIgnited.serverIdToInt(apacheSrvName);
@@ -65,8 +74,8 @@ public class RemoteClientTmpHelper {
IgniteCache<Long, BuildRefCompacted> cacheRef =
ignite.cache(BuildRefDao.TEAMCITY_BUILD_CACHE_NAME);
IgniteCache<Long, FatBuildCompacted> cacheFat =
ignite.cache(FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME);
- String oldBuild =
IgnitePersistentTeamcity.ignCacheNme(IgnitePersistentTeamcity.BUILDS,
apacheSrvName);
- IgniteCache<String, Build> buildCache = ignite.cache(oldBuild);
+ String cacheOldBuild =
IgnitePersistentTeamcity.ignCacheNme(IgnitePersistentTeamcity.BUILDS,
apacheSrvName);
+ IgniteCache<String, Build> buildCache = ignite.cache(cacheOldBuild);
cacheRef.forEach(
entry -> {
BuildRefCompacted buildRef = entry.getValue();
@@ -80,13 +89,20 @@ public class RemoteClientTmpHelper {
dumpBuildRef(buildId, buildRef);
dumpFatBuild(cacheFat, apache, buildId);
String href = ITeamcity.buildHref(buildId);
- Build fatBuild = buildCache.get(href);
- if (fatBuild != null)
- dumpOldBuild(buildId, href, fatBuild);
+ Build oldBuild = buildCache.get(href);
+ if (oldBuild != null)
+ dumpOldBuild(buildId, href, oldBuild);
+
+ inconsistent.incrementAndGet();
+
+ if(!fat.isOutdatedEntityVersion())
+ Preconditions.checkState(false, oldBuild);
}
}
}
);
+
+ return inconsistent.get();
}
public static void dumpOldBuild(int buildId, String href, Build fatBuild) {
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
index 8ca0b24..f513809 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
@@ -53,7 +53,7 @@ public class CompareBuildsService {
public List<String> tests0(String srvId, Integer buildId, ICredentialsProv
prov ) {
IAnalyticsEnabledTeamcity teamcity = helper.server(srvId, prov);
- String hrefById = teamcity.getBuildHrefById(buildId);
+ String hrefById = ITeamcity.buildHref(buildId);
BuildRef buildRef = new BuildRef();
buildRef.setId(buildId);
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 1d0b4bc..0a2ed37 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -441,10 +441,8 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
FatBuildCompacted existingBuild = getFatBuildFromIgnite(buildId);
if (mode == SyncMode.NONE) {
- if (existingBuild != null)
- return existingBuild;
- else
- return new FatBuildCompacted().setFakeStub(true); // providing
fake builds
+ // providing fake builds
+ return existingBuild != null ? existingBuild : new
FatBuildCompacted().setFakeStub(true);
}
FatBuildCompacted savedVer = buildSync.loadBuild(conn, buildId,
existingBuild, mode);
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index 1a50a5d..3f4ff2b 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -50,7 +50,10 @@ import java.util.stream.Stream;
@Persisted
public class FatBuildCompacted extends BuildRefCompacted implements
IVersionedEntity {
/** Latest version. */
- private static final int LATEST_VERSION = 5;
+ public static final short LATEST_VERSION = 6;
+
+ /** Latest version. */
+ public static final short VER_FULL_DATA_BUT_ID_CONFLICTS_POSSIBLE = 5;
/** Default branch flag offset. */
public static final int DEF_BR_F = 0;
@@ -66,7 +69,14 @@ public class FatBuildCompacted extends BuildRefCompacted
implements IVersionedEn
public static final int[] EMPTY = new int[0];
- /** Entity fields version. */
+ /**
+ * Entity fields version.
+ * <ul>
+ * <li>{@link #VER_FULL_DATA_BUT_ID_CONFLICTS_POSSIBLE} - fully supported
field set, tests, problems. </li>
+ * <li>6 - done double check if build ID is consistent with a key. If this
check passes, version is set to 6, if
+ * not-build is deleted.</li>
+ * </ul>
+ */
private short _ver = LATEST_VERSION;
/** Start date. The number of milliseconds since January 1, 1970, 00:00:00
GMT */
@@ -495,4 +505,8 @@ public class FatBuildCompacted extends BuildRefCompacted
implements IVersionedEn
invocation.changesPresent(changes().length > 0 ? 1 : 0);
return invocation;
}
+
+ public void setVersion(short version) {
+ this._ver = version;
+ }
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
index 9979bed..ccafc68 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
@@ -119,6 +119,12 @@ public class FatBuildDao {
buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId), newBuild);
}
+
+ @AutoProfiling
+ public boolean removeFatBuild(int srvIdMaskHigh, int buildId) {
+ return buildsCache.remove(buildIdToCacheKey(srvIdMaskHigh, buildId));
+ }
+
public static int[] extractChangeIds(@NotNull ChangesList changesList) {
return changesList.changes().stream().mapToInt(
ch -> {
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
index 753a12f..3f68542 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -16,6 +16,7 @@
*/
package org.apache.ignite.ci.teamcity.ignited.fatbuild;
+import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.base.Throwables;
import java.util.stream.Stream;
@@ -294,12 +295,26 @@ public class ProactiveFatBuildSync {
@SuppressWarnings({"WeakerAccess"})
@AutoProfiling
public FatBuildCompacted reloadBuild(ITeamcityConn conn, int buildId,
@Nullable FatBuildCompacted existingBuild) {
- // System.err.println(Thread.currentThread().getName()+ ": Build " +
buildId);
//todo some sort of locking to avoid double requests
final String srvName = conn.serverId();
final int srvIdMask = ITeamcityIgnited.serverIdToInt(srvName);
+ if (existingBuild != null && existingBuild.isOutdatedEntityVersion()) {
+ int ver = existingBuild.version();
+ if (ver ==
FatBuildCompacted.VER_FULL_DATA_BUT_ID_CONFLICTS_POSSIBLE) {
+ if (Objects.equals(buildId, existingBuild.id()))
+ existingBuild.setVersion(FatBuildCompacted.LATEST_VERSION);
+ else {
+ logger.warn("Build inconsistency found in the DB, removing
build " + existingBuild.getId());
+
+ existingBuild = null;
+
+ fatBuildDao.removeFatBuild(srvIdMask, buildId);
+ }
+ }
+ }
+
Build build;
List<TestOccurrencesFull> tests = new ArrayList<>();
List<ProblemOccurrence> problems = null;
@@ -309,9 +324,12 @@ public class ProactiveFatBuildSync {
build = conn.getBuild(buildId);
if (build.isFakeStub())
- build.setCancelled();
+ build.setCancelled(); // probably now it will not happen
because of direct connection to TC.
+ else
+ Preconditions.checkState(Objects.equals(build.getId(),
buildId),
+ "Build IDs are not consistent: returned " + build.getId()
+ " queued is " + buildId);
- if(build.testOccurrences != null && !build.isComposite()) { //
don't query tests for compoite
+ if (build.testOccurrences != null && !build.isComposite()) { //
don't query tests for compoite
String nextHref = null;
do {
TestOccurrencesFull page = conn.getTestsPage(buildId,
nextHref, true);
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
index dd0b9ef..ea6df15 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
@@ -25,7 +25,7 @@ package org.apache.ignite.ci.web.model;
public static final String GITHUB_REF =
"https://github.com/apache/ignite-teamcity-bot";;
/** TC Bot Version. */
- public static final String VERSION = "20181205";
+ public static final String VERSION = "20181206";
/** TC Bot Version. */
public String version = VERSION;
