This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch time-consumer-detection
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/time-consumer-detection by
this push:
new f5c7802 Time consuming builds detection
f5c7802 is described below
commit f5c78023d3f95d791345d509c2e6232d6c28cea7
Author: Dmitriy Pavlov <[email protected]>
AuthorDate: Sat Jul 6 14:59:38 2019 +0300
Time consuming builds detection
---
.../src/main/webapp/buildtime.html | 123 ++++++++++++++
.../tcbot/engine/buildtime/BuildTimeService.java | 7 +-
.../ignited/fatbuild/FatBuildCompacted.java | 8 +
.../ignited/fatbuild/StatisticsCompacted.java | 38 ++---
.../apache/ignite/tcignited/build/FatBuildDao.java | 180 ++++++++-------------
.../tcignited/buildtime/BuildTimeRecord.java | 4 +
6 files changed, 215 insertions(+), 145 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/webapp/buildtime.html
b/ignite-tc-helper-web/src/main/webapp/buildtime.html
new file mode 100644
index 0000000..2e1dcc4
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/webapp/buildtime.html
@@ -0,0 +1,123 @@
+<html>
+<head>
+ <title>Apache Ignite Teamcity Bot - Build Time Analytics</title>
+ <link rel="icon" href="img/leaf-icon-png-7066.png">
+
+ <link rel="stylesheet"
href="https://code.jquery.com/ui/1.12.1/themes/base/jquery-ui.css";>
+
+
+ <script src="https://code.jquery.com/jquery-1.12.4.js";></script>
+ <script src="https://code.jquery.com/ui/1.12.1/jquery-ui.js";></script>
+
+
+ <script src="https://cdn.jsdelivr.net/npm/vue/dist/vue.js";></script>
+ <!-- production version, optimized for size and speed -->
+ <!--<script src="https://cdn.jsdelivr.net/npm/vue";></script>-->
+
+ <script
src="https://cdn.jsdelivr.net/npm/vuetify/dist/vuetify.js";></script>
+ <link
href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900|Material+Icons"
rel="stylesheet">
+ <link href="https://cdn.jsdelivr.net/npm/vuetify/dist/vuetify.min.css";
rel="stylesheet">
+
+ <link rel="stylesheet" href="css/style-1.5.css">
+
+ <script src="js/common-1.6.js"></script>
+ <script>
+$(document).ready(function() {
+ $.getScript("js/common-1.6.js", function(data, textStatus, jqxhr){ });
+
+ $( document ).tooltip();
+ loadData();
+});
+
+function loadAnalytics() {
+ let app = new Vue({
+ el: '#app',
+ data () {
+ return {
+ headers: [
+ { text: 'Build Type', value: 'buildType' },
+ { text: 'Average Duration', value: 'averageDuration' },
+ { text: 'Duration', value: 'totalDuration' }
+ ],
+ byBuildType: [
+ ]
+ }
+ },
+ created () {
+ this.initialize()
+ },
+
+ methods: {
+ setBuildTimeStat(data) {
+ this.byBuildType = data.byBuildType;
+
+ $("#loadStatus").html("");
+ },
+ initialize() {
+ $.ajax({ url: "/rest/buildtime/analytics",
+ success: this.setBuildTimeStat,
+ error: showErrInLoadStatus });
+ }
+ }
+ })
+}
+
+function loadData() {
+ $("#loadStatus").html("<img
src='https://www.wallies.com/filebin/images/loading_apple.gif' width=20px
height=20px> Please wait");
+
+ $("#version").html(" " + "<a href=\"monitoring.html\">TC Bot Moniroting
Page</a> <br>");
+ $.ajax({
+ url: "rest/branches/version",
+ success: showVersionInfo,
+ error: showErrInLoadStatus
+ });
+
+
+ loadAnalytics();
+}
+
+
+</script>
+</head>
+<body>
+
+<div id="loadStatus"></div>
+
+
+<div class="formgroup" id="app">
+ <v-app id="readyForReview">
+ <!--<v-expansion-panel>-->
+ <!--<v-expansion-panel-content-->
+ <!--v-for="(item,i) in 1"-->
+ <!--:key="i"-->
+ <!-->-->
+ <template v-slot:header>
+ <div>Build types longest avg.duration</div>
+ </template>
+ <v-card>
+ <v-data-table
+ :headers="headers"
+ :items="byBuildType"
+ class="elevation-1"
+ >
+ <template v-slot:items="props">
+ <!--<td>-->
+ <!--<a :href="props.item.prHtmlUrl">{{
props.item.prNumber }} </a> {{ props.item.prTitle }}</td>-->
+ <!--<td class="text-xs-right">{{ props.item.b
}}</td>-->
+ <!--<td class="text-xs-right">-->
+ <!--<img :src="props.item.prAuthorAvatarUrl"
width='20px' height='20px'> {{ props.item.prAuthor }}</td>-->
+ <td class="text-xs-right">{{ props.item.buildType
}}</td>
+ <td class="text-xs-right">{{
props.item.averageDuration }}</td>
+ <td class="text-xs-right">{{
props.item.totalDuration }}</td>
+ </template>
+ </v-data-table>
+ </v-card>
+ <!--</v-expansion-panel-content>-->
+ <!--</v-expansion-panel>-->
+
+ </v-app>
+</div>
+
+<div id="version"></div>
+</body>
+</html>
\ No newline at end of file
diff --git
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java
index 18d03f1..7001e35 100644
---
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java
+++
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java
@@ -76,9 +76,10 @@ public class BuildTimeService {
BuildTimeResultUi resultUi = new BuildTimeResultUi();
- long minDuration = Duration.ofHours(1).toMillis();
+ long minDuration = Duration.ofMinutes(60).toMillis();
+ int cntToInclude = 50;
BuildTimeResult res = lastRes1d;
- List<Map.Entry<Long, BuildTimeRecord>> entries =
res.topByBuildTypes(availableServers, minDuration, 5);
+ List<Map.Entry<Long, BuildTimeRecord>> entries =
res.topByBuildTypes(availableServers, minDuration, cntToInclude);
entries.forEach(e -> {
BuildTimeRecordUi buildTimeRecordUi = new BuildTimeRecordUi();
@@ -87,6 +88,8 @@ public class BuildTimeService {
buildTimeRecordUi.buildType = compactor.getStringFromId(btId);
buildTimeRecordUi.averageDuration =
TimeUtil.millisToDurationPrintable(e.getValue().avgDuration());
+ buildTimeRecordUi.totalDuration =
TimeUtil.millisToDurationPrintable(e.getValue().totalDuration());
+
resultUi.byBuildType.add(buildTimeRecordUi);
});
diff --git
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index eaeeb54..c47d8e9 100644
---
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -554,6 +554,10 @@ public class FatBuildCompacted extends BuildRefCompacted
implements IVersionedEn
.forEach(this.problems::add);
}
+ public Long statisticValue(Integer propCode) {
+ return statistics == null ? null : statistics.statisticValue(propCode);
+ }
+
public Long buildDuration(IStringCompactor compactor) {
return statistics == null ? null : statistics.buildDuration(compactor);
}
@@ -711,4 +715,8 @@ public class FatBuildCompacted extends BuildRefCompacted
implements IVersionedEn
return cnt;
}
+
+ public long getFinishDateTs() {
+ return finishDate;
+ }
}
diff --git
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java
index 15955bf..514542b 100644
---
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java
@@ -29,6 +29,8 @@ import org.apache.ignite.internal.util.GridLongList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
+
/**
* Statistics values to be saved in compacted form.
*/
@@ -85,10 +87,14 @@ public class StatisticsCompacted {
public Long buildDuration(IStringCompactor compactor) {
Integer buildDurationId =
compactor.getStringIdIfPresent(Statistics.BUILD_DURATION);
- if (buildDurationId == null)
+ return statisticValue(buildDurationId);
+ }
+
+ @Nullable public Long statisticValue(@Nullable Integer propCode) {
+ if (propCode == null)
return null;
- long val = findPropertyValue(buildDurationId);
+ long val = findPropertyValue(propCode);
return val >= 0 ? val : null;
}
@@ -100,12 +106,7 @@ public class StatisticsCompacted {
public Long buildDurationNetTime(IStringCompactor compactor) {
Integer buildDurationNetId =
compactor.getStringIdIfPresent(Statistics.BUILD_DURATION_NET_TIME);
- if (buildDurationNetId == null)
- return null;
-
- long val = findPropertyValue(buildDurationNetId);
-
- return val >= 0 ? val : null;
+ return statisticValue(buildDurationNetId);
}
/**
@@ -115,12 +116,7 @@ public class StatisticsCompacted {
public Long artifcactPublishingDuration(IStringCompactor compactor) {
Integer buildDurationNetId =
compactor.getStringIdIfPresent(Statistics.ARTIFACTS_PUBLISHING_DURATION);
- if (buildDurationNetId == null)
- return null;
-
- long val = findPropertyValue(buildDurationNetId);
-
- return val >= 0 ? val : null;
+ return statisticValue(buildDurationNetId);
}
/**
@@ -130,12 +126,7 @@ public class StatisticsCompacted {
public Long dependeciesResolvingDuration(IStringCompactor compactor) {
Integer buildDurationNetId =
compactor.getStringIdIfPresent(Statistics.DEPENDECIES_RESOLVING_DURATION);
- if (buildDurationNetId == null)
- return null;
-
- long val = findPropertyValue(buildDurationNetId);
-
- return val >= 0 ? val : null;
+ return statisticValue(buildDurationNetId);
}
/**
@@ -145,12 +136,7 @@ public class StatisticsCompacted {
public Long sourceUpdateDuration(IStringCompactor compactor) {
Integer buildDurationNetId =
compactor.getStringIdIfPresent(Statistics.SOURCES_UPDATE_DURATION);
- if (buildDurationNetId == null)
- return null;
-
- long val = findPropertyValue(buildDurationNetId);
-
- return val >= 0 ? val : null;
+ return statisticValue(buildDurationNetId);
}
public long findPropertyValue(int propCode) {
diff --git
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index e9f02c0..d3ef135 100644
---
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -19,34 +19,11 @@ package org.apache.ignite.tcignited.build;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterables;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import java.util.stream.StreamSupport;
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-import javax.cache.Cache;
-import javax.cache.processor.EntryProcessorException;
-import javax.cache.processor.EntryProcessorResult;
-import javax.cache.processor.MutableEntry;
-import javax.inject.Inject;
-import javax.inject.Provider;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCompute;
import org.apache.ignite.binary.BinaryObject;
import org.apache.ignite.cache.CacheEntryProcessor;
-import org.apache.ignite.cache.query.QueryCursor;
-import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
-import org.apache.ignite.lang.IgniteCallable;
-import org.apache.ignite.resources.IgniteInstanceResource;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.persistence.CacheConfigs;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
@@ -62,6 +39,19 @@ import
org.apache.ignite.tcservice.model.result.tests.TestOccurrencesFull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import javax.cache.Cache;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.EntryProcessorResult;
+import javax.cache.processor.MutableEntry;
+import javax.inject.Inject;
+import javax.inject.Provider;
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
/**
*
*/
@@ -248,39 +238,67 @@ public class FatBuildDao {
}
public BuildTimeResult loadBuildTimeResult(int ageDays, List<Long>
idsToCheck) {
- IgniteCache<Long, BinaryObject> cacheBin =
buildsCache.withKeepBinary();
-
int stateRunning = compactor.getStringId(BuildRef.STATE_RUNNING);
Integer buildDurationId =
compactor.getStringIdIfPresent(Statistics.BUILD_DURATION);
- BuildTimeResult res =new BuildTimeResult();
+ BuildTimeResult res = new BuildTimeResult();
+
+ // also may take affinity into account
Iterables.partition(idsToCheck, MAX_FAT_BUILD_CHUNK).forEach(
- chunk -> {
- HashSet<Long> keys = new HashSet<>(chunk);
- Map<Long, BinaryObject> all = cacheBin.getAll(keys);
- all.forEach((key, buildBinary) -> {
- long runningTime = getBuildRunningTime(stateRunning,
buildDurationId, buildBinary);
- if (runningTime > 0) {
- int buildTypeId = buildBinary.field("buildTypeId");
- System.err.println("Running " + runningTime + "BT: " +
buildTypeId);
-
- int srvId = BuildRefDao.cacheKeyToSrvId(key);
- res.add(srvId, buildTypeId, runningTime);
- }
- });
- }
+ chunk -> {
+ HashSet<Long> keys = new HashSet<>(chunk);
+ Map<Long, FatBuildCompacted> all =
buildsCache.getAll(keys);
+ all.forEach((key, build) -> {
+ if (build.isComposite())
+ return;
+
+ long runningTime = getBuildRunningTime(stateRunning,
buildDurationId, build);
+ if (runningTime > 0) {
+ int buildTypeId = build.buildTypeId();
+ System.err.println("Running " + runningTime + "
BT: " + buildTypeId);
+
+ int srvId = BuildRefDao.cacheKeyToSrvId(key);
+ res.add(srvId, buildTypeId, runningTime);
+ }
+ });
+ }
);
+ return res;
- if(0!=1) return res;
+ }
- Ignite ignite = igniteProvider.get();
+ public static long getBuildRunningTime(int stateRunning, Integer
buildDurationId,
+ FatBuildCompacted buildBinary) {
+ long startTs = buildBinary.getStartDateTs();
- IgniteCompute serversCompute =
ignite.compute(ignite.cluster().forServers());
+ if (startTs <= 0)
+ return -1;
- serversCompute.call(new BuiltTimeIgniteCallable(stateRunning,
buildDurationId));
- return res;
+ int state = buildBinary.state();
+
+ long runningTime = -1;
+ if (stateRunning == state)
+ runningTime = System.currentTimeMillis() - startTs;
+ if (runningTime < 0) {
+ if (buildDurationId != null) {
+ Long val = buildBinary.statisticValue(buildDurationId);
+
+ runningTime = (val != null && val >= 0) ? val : -1;
+ }
+
+
+ }
+
+ if (runningTime < 0) {
+ long finishTs = buildBinary.getFinishDateTs();
+
+ if (finishTs > 0)
+ runningTime = finishTs - startTs;
+ }
+
+ return runningTime;
}
public static long getBuildRunningTime(int stateRunning, Integer
buildDurationId,
@@ -343,76 +361,4 @@ public class FatBuildDao {
return startDate;
}
}
-
- public static class BuiltTimeIgniteCallable implements
IgniteCallable<Long> {
- private final int stateRunning;
- private final Integer buildDurationId;
- @IgniteInstanceResource
- Ignite ignite;
-
- public BuiltTimeIgniteCallable( int stateRunning,
- Integer buildDurationId) {
- this.stateRunning = stateRunning;
- this.buildDurationId = buildDurationId;
- }
-
- @Override public Long call() throws Exception {
-
- IgniteCache<Long, FatBuildCompacted> cache =
ignite.cache(TEAMCITY_FAT_BUILD_CACHE_NAME);
-
- IgniteCache<Long, BinaryObject> cacheBin = cache.withKeepBinary();
- QueryCursor<Cache.Entry<Long, BinaryObject>> query =
cacheBin.query(
- new ScanQuery<Long, BinaryObject>()
- .setLocal(true));
-
- /*.query(new SqlQuery<Long, BinaryObject>(
- FatBuildCompacted.class,
- " _KEY > ?")
- .setLocal(true)
- .setArgs(0L));*/
-
-// Iterate over the result set.
- try (QueryCursor<Cache.Entry<Long, BinaryObject>> cursor = query) {
-
- for (Cache.Entry<Long, BinaryObject> next : cursor) {
-
- BinaryObject buildBinary = next.getValue();
- long runningTime = getBuildRunningTime(stateRunning,
buildDurationId, buildBinary);
-
- System.err.println("Running " + runningTime);
- }
- }
-
- /*
- FieldsQueryCursor<List<?>> query = cacheBin.query(new
SqlFieldsQuery("" +
-
- "select startDate, buildTypeId from FatBuildCompacted where _KEY >
?")
- .setLocal(true)
- .setArgs(0L));
-
-// Iterate over the result set.
- try (QueryCursor<List<?>> cursor = query) {
- for (List<?> row : cursor)
- System.out.println("startDate=" + row.get(0));
-
- }
- */
-
- if (1 != 2)
- return null;
-
- Iterable<Cache.Entry<Long, BinaryObject>> entries =
cacheBin.localEntries();
- for (Cache.Entry<Long, BinaryObject> next : entries) {
- Long srvAndBuild = next.getKey();
-
- BinaryObject buildBinary = next.getValue();
-
- Long val = getBuildRunningTime(stateRunning, buildDurationId,
buildBinary);
- if (val != null)
- return val;
- }
-
- return null;
- }
- }
}
diff --git
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeRecord.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeRecord.java
index 285e04d..63e8026 100644
---
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeRecord.java
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeRecord.java
@@ -31,4 +31,8 @@ public class BuildTimeRecord {
return totaltime / cnt;
}
+
+ public long totalDuration() {
+ return totaltime;
+ }
}
