This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new 18d2624 Don't save build queue data to persistence; removing old
caches created by mistake during migrations
18d2624 is described below
commit 18d262473a9ee0eed793c8c6dc5638d4de4ecdd2
Author: Dmitriy Pavlov <[email protected]>
AuthorDate: Fri Sep 21 21:21:58 2018 +0300
Don't save build queue data to persistence; removing old caches created by
mistake during migrations
---
.../apache/ignite/ci/IgnitePersistentTeamcity.java | 66 +++++++++++++++++++---
.../java/org/apache/ignite/ci/db/DbMigrations.java | 41 +++++++++++++-
.../org/apache/ignite/ci/util/CacheUpdateUtil.java | 46 ---------------
.../ci/web/rest/monitoring/MonitoringService.java | 6 +-
4 files changed, 102 insertions(+), 57 deletions(-)
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
index 3983e81..0d92e4f 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
@@ -32,6 +32,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.function.BiFunction;
import java.util.function.Function;
@@ -105,9 +106,6 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
private static final String BUILDS_FAILURE_RUN_STAT =
"buildsFailureRunStat";
public static final String BUILDS = "builds";
- private static final String BUILD_QUEUE = "buildQueue";
- private static final String RUNNING_BUILDS = "runningBuilds";
-
/** Number of builds to re-query from TC to be sure some builds in the
middle are not lost. */
private static final int MAX_BUILDS_IN_PAST_TO_RELOAD = 5;
@@ -124,9 +122,15 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
*/
private ConcurrentMap<String, CompletableFuture<TestOccurrenceFull>>
testOccFullFutures = new ConcurrentHashMap<>();
+ /** cached running builds for branch. */
+ private ConcurrentMap<String, Expirable<List<BuildRef>>> queuedBuilds =
new ConcurrentHashMap<>();
+
/** cached loads of queued builds for branch. */
private ConcurrentMap<String, CompletableFuture<List<BuildRef>>>
queuedBuildsFuts = new ConcurrentHashMap<>();
+ /** cached running builds for branch. */
+ private ConcurrentMap<String, Expirable<List<BuildRef>>> runningBuilds =
new ConcurrentHashMap<>();
+
/** cached loads of running builds for branch. */
private ConcurrentMap<String, CompletableFuture<List<BuildRef>>>
runningBuildsFuts = new ConcurrentHashMap<>();
@@ -441,13 +445,61 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
}
+ public <K, V> CompletableFuture<V>
loadAsyncIfAbsentOrExpired(ConcurrentMap<K, Expirable<V>> cache,
+ K key,
+
ConcurrentMap<K, CompletableFuture<V>> cachedComputations,
+ Function<K,
CompletableFuture<V>> realLoadFunction,
+ int
maxAgeSecs,
+ boolean
alwaysProvidePersisted) {
+ @Nullable final Expirable<V> persistedValue = cache.get(key);
+
+ int fields = ObjectInterner.internFields(persistedValue);
+
+ // if (fields > 0)
+ // System.out.println("Interned " + fields + " after get()");
+
+
+ if (persistedValue != null &&
persistedValue.isAgeLessThanSecs(maxAgeSecs))
+ return CompletableFuture.completedFuture(persistedValue.getData());
+
+ AtomicReference<CompletableFuture<V>> submitRef = new
AtomicReference<>();
+
+ CompletableFuture<V> loadFut = cachedComputations.computeIfAbsent(key,
+ k -> {
+ CompletableFuture<V> future = realLoadFunction.apply(k)
+ .thenApplyAsync(valueLoaded -> {
+ cache.put(k, new Expirable<V>(valueLoaded));
+
+ return valueLoaded;
+ });
+
+ submitRef.set(future);
+
+ return future;
+ }
+ ).thenApply(res -> {
+ CompletableFuture<V> f = submitRef.get();
+
+ if (f != null)
+ cachedComputations.remove(key, f);
+
+ return res;
+ });
+
+ if (alwaysProvidePersisted && persistedValue != null)
+ return CompletableFuture.completedFuture(persistedValue.getData());
+
+ return loadFut;
+ }
+
+
/** {@inheritDoc} */
@Override public CompletableFuture<List<BuildRef>> getRunningBuilds(String
branch) {
int defaultSecs = 60;
int secondsUseCached = getTriggerRelCacheValidSecs(defaultSecs);
- return CacheUpdateUtil.loadAsyncIfAbsentOrExpired(
- getOrCreateCacheV2(ignCacheNme(RUNNING_BUILDS)),
+ return loadAsyncIfAbsentOrExpired(
+ runningBuilds, //getOrCreateCacheV2(ignCacheNme(RUNNING_BUILDS)),
Strings.nullToEmpty(branch),
runningBuildsFuts,
teamcity::getRunningBuilds,
@@ -466,8 +518,8 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
int defaultSecs = 60;
int secondsUseCached = getTriggerRelCacheValidSecs(defaultSecs);
- return CacheUpdateUtil.loadAsyncIfAbsentOrExpired(
- getOrCreateCacheV2(ignCacheNme(BUILD_QUEUE)),
+ return loadAsyncIfAbsentOrExpired(
+ queuedBuilds, // getOrCreateCacheV2(ignCacheNme(BUILD_QUEUE)),
Strings.nullToEmpty(branch),
queuedBuildsFuts,
teamcity::getQueuedBuilds,
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
index 32a23b6..0364e71 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
@@ -54,6 +54,8 @@ import org.slf4j.LoggerFactory;
* Migrations to be applied to each TC related caches.
*/
public class DbMigrations {
+ public static final String BUILD_QUEUE = "buildQueue";
+ public static final String RUNNING_BUILDS = "runningBuilds";
/** Logger. */
private static final Logger logger =
LoggerFactory.getLogger(DbMigrations.class);
@@ -183,7 +185,10 @@ public class DbMigrations {
});
applyMigration(newBuildsCache, () -> {
- IgniteCache<String, Build> oldBuilds =
getOrCreateIgnCacheV1(BUILD_RESULTS);
+ IgniteCache<String, Build> oldBuilds =
ignite.cache(ignCacheNme(BUILD_RESULTS));
+
+ if (oldBuilds == null)
+ return;
int size = oldBuilds.size();
if (size > 0) {
@@ -278,7 +283,13 @@ public class DbMigrations {
applyMigration(TEST_OCCURRENCE_FULL + "-to-" + testFullCache.getName()
+ "V2", () -> {
String cacheNme = ignCacheNme(TEST_OCCURRENCE_FULL);
- IgniteCache<String, TestOccurrenceFull> oldTestCache =
ignite.getOrCreateCache(cacheNme);
+ IgniteCache<String, TestOccurrenceFull> oldTestCache =
ignite.cache(cacheNme);
+
+ if (oldTestCache == null) {
+ System.err.println("cache not found");
+
+ return;
+ }
int size = oldTestCache.size();
if (size > 0) {
@@ -380,6 +391,27 @@ public class DbMigrations {
i++;
}
});
+
+ applyDestroyCacheMigration(RUNNING_BUILDS);
+
+ applyDestroyCacheMigration(BUILD_QUEUE);
+
+ applyDestroyCacheMigration(FINISHED_BUILDS_INCLUDE_FAILED);
+ applyDestroyCacheMigration(TEST_OCCURRENCE_FULL);
+ }
+
+ public void applyDestroyCacheMigration(String cacheName) {
+ applyMigration("destroy-" + cacheName, () -> {
+ IgniteCache<Object, Object> cache =
ignite.cache(ignCacheNme(cacheName));
+
+ if (cache == null) {
+ System.err.println("cache not found");
+
+ return;
+ }
+
+ cache.destroy();
+ });
}
private <K, V> void applyV1toV2Migration(String full, Cache<K, V> cache) {
@@ -394,8 +426,11 @@ public class DbMigrations {
private <K,V> void v1tov2cacheMigrate(String deprecatedCache, Cache<K, V>
newCache) {
String cacheNme = ignCacheNme(deprecatedCache);
- IgniteCache<K, V> tests = ignite.getOrCreateCache(cacheNme);
+ IgniteCache<K, V> tests = ignite.cache(cacheNme);
+ if (tests == null) {
+ System.err.println("Cache not found");
+ }
int size = tests.size();
if (size > 0) {
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/CacheUpdateUtil.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/CacheUpdateUtil.java
index a94d47c..9c7f385 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/CacheUpdateUtil.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/CacheUpdateUtil.java
@@ -77,50 +77,4 @@ public class CacheUpdateUtil {
});
}
- public static <K, V> CompletableFuture<V>
loadAsyncIfAbsentOrExpired(IgniteCache<K, Expirable<V>> cache,
- K key,
- ConcurrentMap<K, CompletableFuture<V>> cachedComputations,
- Function<K, CompletableFuture<V>> realLoadFunction,
- int maxAgeSecs,
- boolean alwaysProvidePersisted) {
- @Nullable final Expirable<V> persistedValue = cache.get(key);
-
- int fields = ObjectInterner.internFields(persistedValue);
-
- // if (fields > 0)
- // System.out.println("Interned " + fields + " after get()");
-
-
- if (persistedValue != null &&
persistedValue.isAgeLessThanSecs(maxAgeSecs))
- return CompletableFuture.completedFuture(persistedValue.getData());
-
- AtomicReference<CompletableFuture<V>> submitRef = new
AtomicReference<>();
-
- CompletableFuture<V> loadFut = cachedComputations.computeIfAbsent(key,
- k -> {
- CompletableFuture<V> future = realLoadFunction.apply(k)
- .thenApplyAsync(valueLoaded -> {
- cache.put(k, new Expirable<V>(valueLoaded));
-
- return valueLoaded;
- });
-
- submitRef.set(future);
-
- return future;
- }
- ).thenApply(res -> {
- CompletableFuture<V> f = submitRef.get();
-
- if (f != null)
- cachedComputations.remove(key, f);
-
- return res;
- });
-
- if (alwaysProvidePersisted && persistedValue != null)
- return CompletableFuture.completedFuture(persistedValue.getData());
-
- return loadFut;
- }
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java
index 82b202d..62406e2 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java
@@ -19,6 +19,7 @@ package org.apache.ignite.ci.web.rest.monitoring;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.CacheMetrics;
+import org.apache.ignite.cache.affinity.Affinity;
import org.apache.ignite.ci.di.ProfilingInterceptor;
import org.apache.ignite.ci.web.CtxListener;
@@ -84,7 +85,10 @@ public class MonitoringService {
// res.add(next + ": " + size + " get " + averageGetTime + " put "
+ averagePutTime);
- res.add(next + ": " + size);
+
+ Affinity<Object> affinity = ignite.affinity(next);
+
+ res.add(next + ": " + size + " parts " + affinity.partitions());
}
return res;
}
