This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch ignite-10071 in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit ff7ff10930c6d1dc8592ba462d1c5651a7f33d9d Author: Dmitriy Pavlov <dpav...@apache.org> AuthorDate: Tue Oct 30 21:12:03 2018 +0300 IGNITE-10071 Background upload of a build and simultaneous bot restart may result in a queued build persisted but actually build is finished --- .../ignite/ci/runners/RemoteClientTmpHelper.java | 72 ++++++++++++++-------- .../ci/teamcity/ignited/BuildRefCompacted.java | 11 ++++ .../ignite/ci/teamcity/ignited/BuildRefDao.java | 2 +- .../ignited/fatbuild/FatBuildCompacted.java | 19 ++++++ .../ignited/fatbuild/ProblemCompacted.java | 24 +++----- .../teamcity/ignited/fatbuild/TestCompacted.java | 33 ++++++---- 6 files changed, 108 insertions(+), 53 deletions(-) diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java index 043883a..9c40870 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java @@ -17,20 +17,28 @@ package org.apache.ignite.ci.runners; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import java.io.FileWriter; +import java.io.IOException; import javax.cache.Cache; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.Ignition; -import org.apache.ignite.ci.IgnitePersistentTeamcity; -import org.apache.ignite.ci.user.TcHelperUser; -import org.apache.ignite.ci.user.UserAndSessionsStorage; +import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted; +import org.apache.ignite.ci.teamcity.ignited.BuildRefDao; +import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited; +import org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor; +import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; +import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.logger.slf4j.Slf4jLogger; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; public class RemoteClientTmpHelper { + private static boolean dumpDict = false; + private static void setupDisco(IgniteConfiguration cfg) { final TcpDiscoverySpi spi = new TcpDiscoverySpi(); final int locPort = 54433; @@ -44,7 +52,7 @@ public class RemoteClientTmpHelper { cfg.setDiscoverySpi(spi); } - public static void main(String[] args) { + public static void main(String[] args) throws IOException { final IgniteConfiguration cfg = new IgniteConfiguration(); setupDisco(cfg); @@ -54,36 +62,48 @@ public class RemoteClientTmpHelper { cfg.setClientMode(true); final Ignite ignite = Ignition.start(cfg); - ignite.cluster().active(true); - - IgniteCache<Object, Object> cache1 = ignite.cache(UserAndSessionsStorage.USERS); - for (Cache.Entry<Object, Object> next : cache1) { - System.out.println(next.getKey() + ": " + next.getValue()); - - if (next.getKey().equals("someusername")) { - TcHelperUser u = (TcHelperUser)next.getValue(); - u.resetCredentials(); - - cache1.put(next.getKey(), u); + if(dumpDict) { + IgniteCache<String, Object> strings = ignite.cache(IgniteStringCompactor.STRINGS_CACHE); + try (FileWriter writer = new FileWriter("Dictionary.txt")) { + for (Cache.Entry<String, Object> next1 : strings) { + writer.write(next1.getValue().toString() + + "\n"); + } } } + IgniteCache<Long, FatBuildCompacted> cache1 = ignite.cache(FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME); - IgniteCache<Object, Object> cache = ignite.cache(IgnitePersistentTeamcity.BOT_DETECTED_ISSUES); - for (Cache.Entry<Object, Object> next : cache) { - Object key = next.getKey(); - Object value = next.getValue(); + int apache = ITeamcityIgnited.serverIdToInt("apache"); - if (key.toString().contains("GridCacheLifecycleAwareSelfTest.testLifecycleAware")) { - /*boolean remove = cache.remove(key); + int id = 2200135; + int id1 = 2200209; + dumpFatBuild(cache1, apache, id); + dumpFatBuild(cache1, apache, id1); - if (remove) - System.err.println("Removed issue " + value);*/ - System.err.println("Issue: " + value); - } + IgniteCache<Long, BuildRefCompacted> cache2 = ignite.cache(BuildRefDao.TEAMCITY_BUILD_CACHE_NAME); + dumpBuildRef(cache2, apache, id); + dumpBuildRef(cache2, apache, id1); + ignite.close(); + + } + + public static void dumpBuildRef(IgniteCache<Long, BuildRefCompacted> cache, int apache, int id) throws IOException { + long l = BuildRefDao.buildIdToCacheKey(apache, id); + BuildRefCompacted compacted = cache.get(l); + Preconditions.checkNotNull(compacted, "Can't find build by ID " + id); + try (FileWriter writer = new FileWriter("BuildRef " + id + ".txt")) { + writer.write(compacted.toString()); } + } - ignite.close(); + public static void dumpFatBuild(IgniteCache<Long, FatBuildCompacted> cache, int apache, int id) throws IOException { + long l = FatBuildDao.buildIdToCacheKey(apache, id); + FatBuildCompacted compacted = cache.get(l); + Preconditions.checkNotNull(compacted, "Can't find build by ID " + id); + try (FileWriter writer = new FileWriter("Build " + id + ".txt")) { + writer.write(compacted.toString()); + } } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java index 6b84357..a3aff88 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java @@ -16,6 +16,7 @@ */ package org.apache.ignite.ci.teamcity.ignited; +import com.google.common.base.MoreObjects; import com.google.common.base.Objects; import org.apache.ignite.cache.query.annotations.QuerySqlField; import org.apache.ignite.ci.db.Persisted; @@ -172,4 +173,14 @@ public class BuildRefCompacted { public boolean isQueued(IStringCompactor compactor) { return compactor.getStringId(STATE_QUEUED) == state(); } + + @Override public String toString() { + return MoreObjects.toStringHelper(this) + .add("id", id) + .add("buildTypeId", buildTypeId) + .add("branchName", branchName) + .add("status", status) + .add("state", state) + .toString(); + } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java index d871f84..0d7467a 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java @@ -115,7 +115,7 @@ public class BuildRefDao { * @param srvId Server id mask high. * @param buildId Build id. */ - public long buildIdToCacheKey(long srvId, int buildId) { + public static long buildIdToCacheKey(long srvId, int buildId) { return (long)buildId | srvId << 32; } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java index aeeb644..04f2a63 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java @@ -16,6 +16,7 @@ */ package org.apache.ignite.ci.teamcity.ignited.fatbuild; +import com.google.common.base.MoreObjects; import com.google.common.base.Objects; import org.apache.ignite.ci.analysis.IVersionedEntity; import org.apache.ignite.ci.db.Persisted; @@ -374,4 +375,22 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn return snapshotDeps.clone(); } + + @Override public String toString() { + return MoreObjects.toStringHelper(this) + .add("_", super.toString()) + .add("_ver", _ver) + .add("startDate", startDate) + .add("finishDate", finishDate) + .add("queuedDate", queuedDate) + .add("projectId", projectId) + .add("name", name) + .add("tests", tests) + .add("snapshotDeps", snapshotDeps) + .add("flags", flags) + .add("problems", problems) + .add("statistics", statistics) + .add("changesIds", changesIds) + .toString(); + } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java index 961dbe8..f15c418 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java @@ -17,27 +17,14 @@ package org.apache.ignite.ci.teamcity.ignited.fatbuild; +import com.google.common.base.MoreObjects; import com.google.common.base.Objects; import com.google.common.base.Strings; -import org.apache.ignite.ci.analysis.RunStat; import org.apache.ignite.ci.tcmodel.hist.BuildRef; import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence; -import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence; -import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull; -import org.apache.ignite.ci.tcmodel.result.tests.TestRef; import org.apache.ignite.ci.teamcity.ignited.IStringCompactor; -import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.xerial.snappy.Snappy; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.BitSet; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; import static org.apache.ignite.ci.analysis.RunStat.extractIdPrefixed; @@ -160,4 +147,13 @@ public class ProblemCompacted { return compactor.getStringId(ProblemOccurrence.SNAPSHOT_DEPENDENCY_ERROR) == type || compactor.getStringId(ProblemOccurrence.SNAPSHOT_DEPENDENCY_ERROR_BUILD_PROCEEDS_TYPE) == type; } + + @Override public String toString() { + return MoreObjects.toStringHelper(this) + .add("id", id) + .add("type", type) + .add("identity", identity) + .add("actualBuildId", actualBuildId) + .toString() + "\n"; + } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java index dfd0c1e..753a0ba 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java @@ -17,18 +17,16 @@ package org.apache.ignite.ci.teamcity.ignited.fatbuild; +import com.google.common.base.MoreObjects; import com.google.common.base.Objects; import com.google.common.base.Strings; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.BitSet; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; - import org.apache.ignite.ci.analysis.RunStat; import org.apache.ignite.ci.tcmodel.hist.BuildRef; import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence; @@ -197,18 +195,17 @@ public class TestCompacted { logger.error("Snappy.uncompress failed: " + e.getMessage(), e); return null; } - } else if(flag1 && !flag2) { + } else if(flag1 && !flag2) return new String(details, StandardCharsets.UTF_8); - } else if (!flag1 && flag2) { + else if (!flag1 && flag2) { try { final ByteArrayInputStream in = new ByteArrayInputStream(details); ByteArrayOutputStream bos = new ByteArrayOutputStream(); try (final GZIPInputStream gzi = new GZIPInputStream(in)) { byte[] outbuf = new byte[details.length]; int len; - while ((len = gzi.read(outbuf, 0, outbuf.length)) != -1) { + while ((len = gzi.read(outbuf, 0, outbuf.length)) != -1) bos.write(outbuf, 0, len); - } } return new String(bos.toByteArray(), StandardCharsets.UTF_8); @@ -216,13 +213,12 @@ public class TestCompacted { logger.error("GZip.uncompress failed: " + e.getMessage(), e); return null; } - } else { + } else return null; - } } - public void setDetails(String dtlsString) { - if (Strings.isNullOrEmpty(dtlsString)) { + public void setDetails(String dtlsStr) { + if (Strings.isNullOrEmpty(dtlsStr)) { this.details = null; return; } @@ -232,7 +228,7 @@ public class TestCompacted { byte[] snappy = null; byte[] gzip = null; try { - uncompressed = dtlsString.getBytes(StandardCharsets.UTF_8); + uncompressed = dtlsStr.getBytes(StandardCharsets.UTF_8); } catch (Exception e) { logger.error("Set details failed: " + e.getMessage(), e); return; @@ -353,4 +349,17 @@ public class TestCompacted { public Integer getDuration() { return duration < 0 ? null : duration; } + + @Override public String toString() { + return MoreObjects.toStringHelper(this) + .add("idInBuild", idInBuild) + .add("name", name) + .add("status", status) + .add("duration", duration) + .add("flags", flags) + .add("testId", testId) + .add("actualBuildId", actualBuildId) + .add("details", details) + .toString() + "\n"; + } }