This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-10372
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-10372 by this push:
new 599e4f5 IGNITE-10372 Optimize master trends page step 3: failed tests
move to be sourced from chain content
599e4f5 is described below
commit 599e4f50cca466377e4b9e20199e186bc2047d45
Author: Dmitriy Pavlov <[email protected]>
AuthorDate: Thu Nov 22 19:40:57 2018 +0300
IGNITE-10372 Optimize master trends page step 3: failed tests move to be
sourced from chain content
---
.../main/java/org/apache/ignite/ci/ITeamcity.java | 4 --
.../apache/ignite/ci/IgnitePersistentTeamcity.java | 45 +------------
.../apache/ignite/ci/IgniteTeamcityConnection.java | 12 ----
.../java/org/apache/ignite/ci/db/DbMigrations.java | 77 +++-------------------
.../web/model/current/BuildStatisticsSummary.java | 2 +-
.../ignite/ci/web/model/hist/BuildsHistory.java | 9 +--
6 files changed, 17 insertions(+), 132 deletions(-)
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
index cedb8d4..8be2cf6 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
@@ -77,11 +77,7 @@ public interface ITeamcity extends ITeamcityConn {
@NotNull static String buildHref(int id) {
return "app/rest/latest/builds/id:" + id;
}
- @Deprecated
- public TestOccurrences getTests(String fullUrl);
- @Deprecated
- TestOccurrences getFailedTests(String href, int cnt, String
normalizedBranch);
@Deprecated
CompletableFuture<TestRef> getTestRef(FullQueryParams key);
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
index 1a3e1d2..02efe12 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
@@ -22,13 +22,8 @@ import com.google.common.base.Throwables;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
import java.util.Comparator;
-import java.util.Date;
import java.util.List;
-import java.util.SortedMap;
-import java.util.TreeMap;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -36,8 +31,6 @@ import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.locks.Lock;
-import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Stream;
@@ -45,7 +38,6 @@ import java.util.stream.StreamSupport;
import javax.annotation.Nullable;
import javax.cache.Cache;
import javax.inject.Inject;
-import javax.ws.rs.BadRequestException;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
@@ -66,12 +58,10 @@ import org.apache.ignite.ci.tcmodel.changes.ChangesList;
import org.apache.ignite.ci.tcmodel.conf.BuildType;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
-import org.apache.ignite.ci.tcmodel.result.Configurations;
import org.apache.ignite.ci.tcmodel.result.issues.IssuesUsagesList;
import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrences;
import org.apache.ignite.ci.tcmodel.result.stat.Statistics;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrences;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.tcmodel.result.tests.TestRef;
@@ -94,7 +84,6 @@ import static
org.apache.ignite.ci.tcbot.chain.BuildChainProcessor.normalizeBran
public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity,
ITeamcity, ITcAnalytics {
//V2 caches, 32 parts (V1 caches were 1024 parts)
@Deprecated
- private static final String TESTS_OCCURRENCES = "testOccurrences";
private static final String TESTS_RUN_STAT = "testsRunStat";
private static final String CALCULATED_STATISTIC = "calculatedStatistic";
private static final String LOG_CHECK_RESULT = "logCheckResult";
@@ -141,8 +130,7 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
DbMigrations migrations = new DbMigrations(ignite, conn.serverId());
migrations.dataMigration(
- testOccurrencesCache(), this::addTestOccurrencesToStat,
- this::migrateOccurrencesToLatest,
+ this::migrateOccurrencesToLatest,
buildsCache(), this::addBuildOccurrenceToFailuresStat,
buildsFailureRunStatCache(), testRunStatCache(),
visasHistStorage.visas());
@@ -190,14 +178,6 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
}
/**
- * @return {@link TestOccurrences} instances cache, 32 parts.
- */
- @Deprecated
- private IgniteCache<String, TestOccurrences> testOccurrencesCache() {
- return getOrCreateCacheV2(ignCacheNme(TESTS_OCCURRENCES));
- }
-
- /**
* @return {@link TestRef} instances cache, 32 parts.
*/
private IgniteCache<String, TestRef> testRefsCache() {
@@ -435,29 +415,6 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
/** {@inheritDoc} */
@AutoProfiling
- @Deprecated
- @Override public TestOccurrences getTests(String fullUrl) {
- String hrefForDb = DbMigrations.removeCountFromRef(fullUrl);
-
- return loadIfAbsent(testOccurrencesCache(),
- hrefForDb, //hack to avoid test reloading from store in case of
href filter replaced
- hrefIgnored -> teamcity.getTests(fullUrl));
- }
-
- /** {@inheritDoc} */
- @AutoProfiling
- @Deprecated
- @Override public TestOccurrences getFailedTests(String href, int cnt,
String normalizedBranch) {
- return getTests(href + ",muted:false,status:FAILURE,count:" + cnt +
"&fields=testOccurrence(id,name)");
- }
-
- private void addTestOccurrencesToStat(TestOccurrences val) {
- for (TestOccurrence next : val.getTests())
- addTestOccurrenceToStat(next, ITeamcity.DEFAULT, null);
- }
-
- /** {@inheritDoc} */
- @AutoProfiling
@Override public CompletableFuture<TestRef> getTestRef(FullQueryParams
key) {
return CacheUpdateUtil.loadAsyncIfAbsent(
testRefsCache(),
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
index 914905e..b1089ab 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
@@ -465,18 +465,6 @@ public class IgniteTeamcityConnection implements ITeamcity
{
/** {@inheritDoc} */
@AutoProfiling
- @Override public TestOccurrences getTests(String fullUrl) {
- return getJaxbUsingHref(fullUrl, TestOccurrences.class);
- }
-
- /** {@inheritDoc} */
- @AutoProfiling
- @Override public TestOccurrences getFailedTests(String href, int count,
String normalizedBranch) {
- return getTests(href + ",muted:false,status:FAILURE,count:" + count +
"&fields=testOccurrence(id,name)");
- }
-
- /** {@inheritDoc} */
- @AutoProfiling
@Override public CompletableFuture<TestRef> getTestRef(FullQueryParams
key) {
return supplyAsync(() -> {
return getJaxbUsingHref("app/rest/latest/tests/name:" +
key.getTestName(), TestRef.class);
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
index 9b695b6..48188e4 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
@@ -64,16 +64,12 @@ public class DbMigrations {
public static final String DONE_MIGRATIONS = "doneMigrations";
@Deprecated
- public static final String TESTS = "tests";
- @Deprecated
private static final String BUILD_RESULTS = "buildResults";
private static final String BUILD_STATISTICS = "buildStatistics";
private static final String BUILD_CONDITIONS_CACHE_NAME =
"buildConditions";
- public static final String TESTS_COUNT_7700 = ",count:7700";
-
//V1 caches, 1024 parts
@Deprecated
public static final String RUN_STAT_CACHE = "runStat";
@@ -116,6 +112,8 @@ public class DbMigrations {
String CHANGE_INFO_FULL = "changeInfoFull";
String CURRENT_PR_FAILURES = "currentPrFailures";
String CONFIGURATIONS = "configurations";
+ String TESTS_OCCURRENCES = "testOccurrences";
+ String TESTS = "tests";
}
public static final int SUITES_CNT = 100;
@@ -129,18 +127,12 @@ public class DbMigrations {
this.serverId = srvId;
}
- public static String removeCountFromRef(String href) {
- return href.replace(TESTS_COUNT_7700, "")
- .replace(",count:7500", "");
- }
-
public void dataMigration(
- IgniteCache<String, TestOccurrences> testOccurrencesCache,
Consumer<TestOccurrences> saveTestToStat,
- Consumer<TestOccurrences> saveTestToLatest,
- Cache<String, Build> buildCache, Consumer<Build> saveBuildToStat,
- IgniteCache<SuiteInBranch, RunStat> suiteHistCache,
- IgniteCache<TestInBranch, RunStat> testHistCache,
- Cache<CompactContributionKey, List<CompactVisaRequest>>
visasCache) {
+ Consumer<TestOccurrences> saveTestToLatest,
+ Cache<String, Build> buildCache, Consumer<Build> saveBuildToStat,
+ IgniteCache<SuiteInBranch, RunStat> suiteHistCache,
+ IgniteCache<TestInBranch, RunStat> testHistCache,
+ Cache<CompactContributionKey, List<CompactVisaRequest>> visasCache) {
doneMigrations = doneMigrationsCache();
@@ -195,58 +187,8 @@ public class DbMigrations {
}
});
- applyMigration("InitialFillLatestRunsV3", () -> {
- int size = testOccurrencesCache.size();
- if (size > 0) {
- int i = 0;
- int maxFoundBuildId = 0;
- for (Cache.Entry<String, TestOccurrences> entry :
testOccurrencesCache) {
- String key = entry.getKey();
-
- Integer buildId = RunStat.extractIdPrefixed(key,
"locator=build:(id:", ")");
- if (buildId != null) {
- if (buildId > maxFoundBuildId)
- maxFoundBuildId = buildId;
-
- if (buildId < maxFoundBuildId -
(RunStat.MAX_LATEST_RUNS * SUITES_CNT * 3))
- System.out.println(serverId + " - Skipping entry "
+ i + " from " + size + ": " + key);
- else {
- System.out.println(serverId + " - Migrating entry
" + i + " from " + size + ": " + key);
-
- saveTestToLatest.accept(entry.getValue());
- }
- }
-
- i++;
- }
- }
- });
-
- applyMigration(TESTS + "-to-" + testOccurrencesCache.getName(), () -> {
- IgniteCache<String, TestOccurrences> tests =
ignite.cache(ignCacheNme(TESTS));
- if(tests==null)
- return;
-
- int size = tests.size();
- if (size > 0) {
- int i = 0;
- for (Cache.Entry<String, TestOccurrences> entry : tests) {
- System.out.println("Migrating entry " + i + " from " +
size + ": " + entry.getKey());
+ applyMigration("InitialFillLatestRunsV3", () -> {});
- String transformedKey = removeCountFromRef(entry.getKey());
- TestOccurrences val = entry.getValue();
-
- if (testOccurrencesCache.putIfAbsent(transformedKey, val))
- saveTestToStat.accept(val);
-
- i++;
- }
-
- tests.clear();
-
- tests.destroy();
- }
- });
String newBuildsCache = BUILD_RESULTS + "-to-" +
IgnitePersistentTeamcity.BUILDS + "V2";
applyMigration("RemoveStatisticsFromBuildCache", ()->{
@@ -433,7 +375,7 @@ public class DbMigrations {
applyDestroyIgnCacheMigration(FINISHED_BUILDS_INCLUDE_FAILED);
applyDestroyIgnCacheMigration(TEST_OCCURRENCE_FULL);
- applyDestroyIgnCacheMigration(TESTS);
+ applyDestroyIgnCacheMigration(Old.TESTS);
applyDestroyIgnCacheMigration(STAT);
applyDestroyIgnCacheMigration(BUILD_STATISTICS);
applyDestroyCacheMigration(BUILD_CONDITIONS_CACHE_NAME,
BUILD_CONDITIONS_CACHE_NAME);
@@ -452,6 +394,7 @@ public class DbMigrations {
applyDestroyIgnCacheMigration(Old.TEST_FULL);
applyDestroyIgnCacheMigration(Old.CONFIGURATIONS);
+ applyDestroyIgnCacheMigration(Old.TESTS_OCCURRENCES);
}
private void applyDestroyIgnCacheMigration(String cacheName) {
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/BuildStatisticsSummary.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/BuildStatisticsSummary.java
index 4ea34b4..e11bccc 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/BuildStatisticsSummary.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/BuildStatisticsSummary.java
@@ -83,7 +83,7 @@ public class BuildStatisticsSummary {
/** Is fake stub. */
public boolean isFakeStub;
- /** Failed tests: Map from build type string ID -> Map of test name (full)
string ID to count of failure. */
+ /** Failed tests: Map from build type string ID -> Map of test name (full)
string ID -> to count of failure. */
public Map<Integer, Map<Integer, Integer>> failedTests = new HashMap<>();
/** Is valid. */
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/hist/BuildsHistory.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/hist/BuildsHistory.java
index 9ea5d10..0ea27cf 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/hist/BuildsHistory.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/hist/BuildsHistory.java
@@ -176,15 +176,16 @@ public class BuildsHistory {
if(!Boolean.TRUE.equals(valid))
continue;
- buildStat.failedTests.forEach((btId,v)->{
+ buildStat.failedTests.forEach((btId, map) -> {
String configurationName = compactor.getStringFromId(btId);
Map<String, Float> tests =
mergedTestsBySuites.computeIfAbsent(configurationName,
k -> new HashMap<>());
- v.forEach((t, c) -> {
- String testName = compactor.getStringFromId(t);
+ map.forEach((tn, cnt) -> {
+ String testName = compactor.getStringFromId(tn);
+ float i = cnt != null ? cnt : 1F;
- tests.merge(testName, 1F / buildIds.size(), (a, b) -> a +
b);
+ tests.merge(testName, i / buildIds.size(), (a, b) -> a +
b);
});
});
}
