This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new 5c2d80c Time consuming builds detection, including timed out suites -
Fixes #132.
5c2d80c is described below
commit 5c2d80c3ae7eb0153b2131abc9729b8d89b0a1b6
Author: Dmitriy Pavlov <[email protected]>
AuthorDate: Sat Jul 6 15:30:32 2019 +0300
Time consuming builds detection, including timed out suites - Fixes #132.
Signed-off-by: Dmitriy Pavlov <[email protected]>
---
.../ci/tcbot/TcBotBusinessServicesModule.java | 4 +
.../org/apache/ignite/ci/web/model/Version.java | 2 +-
.../web/rest/buildtime/BuildTimeRestService.java | 57 ++++++++
.../src/main/webapp/buildtime.html | 139 +++++++++++++++++++
ignite-tc-helper-web/src/main/webapp/index.html | 5 +
.../org/apache/ignite/tcbot/common/TcBotConst.java | 3 +
.../tcbot/engine/buildtime/BuildTimeService.java | 115 ++++++++++++++++
.../{DsProblemRef.java => BuildTimeRecordUi.java} | 17 +--
.../{DsProblemRef.java => BuildTimeResultUi.java} | 16 +--
.../ignite/tcbot/engine/ui/DsHistoryStatUi.java | 2 +-
.../ignite/tcbot/engine/ui/DsProblemRef.java | 1 +
.../ignited/fatbuild/FatBuildCompacted.java | 8 ++
.../ignited/fatbuild/StatisticsCompacted.java | 40 ++----
.../ignite/tcignited/TeamcityIgnitedImpl.java | 11 +-
.../apache/ignite/tcignited/build/FatBuildDao.java | 153 ++++++++++++++++++---
.../ignite/tcignited/buildref/BuildRefDao.java | 13 +-
.../tcignited/buildtime/BuildTimeRecord.java | 27 ++--
.../tcignited/buildtime/BuildTimeResult.java | 81 +++++++++++
.../ignite/tcignited/history/HistoryCollector.java | 138 ++++++++++++++++---
.../tcignited/history/RunHistCompactedDao.java | 78 ++++++++++-
20 files changed, 804 insertions(+), 106 deletions(-)
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotBusinessServicesModule.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotBusinessServicesModule.java
index 2134a3f..f0d636f 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotBusinessServicesModule.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotBusinessServicesModule.java
@@ -19,6 +19,7 @@ package org.apache.ignite.ci.tcbot;
import com.google.inject.AbstractModule;
import com.google.inject.internal.SingletonScope;
import org.apache.ignite.ci.issue.IssuesStorage;
+import org.apache.ignite.tcbot.engine.buildtime.BuildTimeService;
import org.apache.ignite.tcbot.engine.chain.BuildChainProcessor;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.ci.tcbot.conf.LocalFilesBasedConfig;
@@ -42,5 +43,8 @@ public class TcBotBusinessServicesModule extends
AbstractModule {
bind(MasterTrendsService.class).in(new SingletonScope());
bind(ITcBotBgAuth.class).to(TcBotBgAuthImpl.class).in(new
SingletonScope());
bind(BuildChainProcessor.class).in(new SingletonScope());
+
+ //todo move to bot engine module
+ bind(BuildTimeService.class).in(new SingletonScope());
}
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
index d284a46..27b0990 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
@@ -28,7 +28,7 @@ package org.apache.ignite.ci.web.model;
public static final String GITHUB_REF =
"https://github.com/apache/ignite-teamcity-bot";;
/** TC Bot Version. */
- public static final String VERSION = "20190704";
+ public static final String VERSION = "20190706";
/** Java version, where Web App is running. */
public String javaVer;
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/buildtime/BuildTimeRestService.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/buildtime/BuildTimeRestService.java
new file mode 100644
index 0000000..43c39de
--- /dev/null
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/buildtime/BuildTimeRestService.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ci.web.rest.buildtime;
+
+import org.apache.ignite.ci.user.ITcBotUserCreds;
+import org.apache.ignite.ci.web.CtxListener;
+import org.apache.ignite.tcbot.engine.buildtime.BuildTimeService;
+import org.apache.ignite.tcbot.engine.ui.BuildTimeResultUi;
+import org.jetbrains.annotations.Nullable;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+
+@Path(BuildTimeRestService.LONG_RUNNING_SUMMARY)
+@Produces(MediaType.APPLICATION_JSON)
+public class BuildTimeRestService {
+ public static final String LONG_RUNNING_SUMMARY = "buildtime";
+
+ /** Servlet Context. */
+ @Context
+ private ServletContext ctx;
+
+ /** Current Request. */
+ @Context
+ private HttpServletRequest req;
+
+ @GET
+ @Path("analytics")
+ public BuildTimeResultUi loadAnalytics(@Nullable @QueryParam("branch")
String branchOrNull) {
+ final ITcBotUserCreds creds = ITcBotUserCreds.get(req);
+
+ final BuildTimeService tbProc =
CtxListener.getInjector(ctx).getInstance(BuildTimeService.class);
+
+ return tbProc.analytics(creds);
+ }
+}
diff --git a/ignite-tc-helper-web/src/main/webapp/buildtime.html
b/ignite-tc-helper-web/src/main/webapp/buildtime.html
new file mode 100644
index 0000000..7090beb
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/webapp/buildtime.html
@@ -0,0 +1,139 @@
+<html>
+<head>
+ <title>Apache Ignite Teamcity Bot - Build Time Analytics</title>
+ <link rel="icon" href="img/leaf-icon-png-7066.png">
+
+ <link rel="stylesheet"
href="https://code.jquery.com/ui/1.12.1/themes/base/jquery-ui.css";>
+
+
+ <script src="https://code.jquery.com/jquery-1.12.4.js";></script>
+ <script src="https://code.jquery.com/ui/1.12.1/jquery-ui.js";></script>
+
+
+ <script src="https://cdn.jsdelivr.net/npm/vue/dist/vue.js";></script>
+ <!-- production version, optimized for size and speed -->
+ <!--<script src="https://cdn.jsdelivr.net/npm/vue";></script>-->
+
+ <script
src="https://cdn.jsdelivr.net/npm/vuetify/dist/vuetify.js";></script>
+ <link
href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900|Material+Icons"
rel="stylesheet">
+ <link href="https://cdn.jsdelivr.net/npm/vuetify/dist/vuetify.min.css";
rel="stylesheet">
+
+ <link rel="stylesheet" href="css/style-1.5.css">
+
+ <script src="js/common-1.6.js"></script>
+ <script>
+$(document).ready(function() {
+ $.getScript("js/common-1.6.js", function(data, textStatus, jqxhr){ });
+
+ $( document ).tooltip();
+ loadData();
+});
+
+function loadAnalytics() {
+ let app = new Vue({
+ el: '#app',
+ data () {
+ return {
+ headers: [
+ { text: 'Build Type', value: 'buildType' },
+ { text: 'Average Duration', value: 'averageDuration' },
+ { text: 'Duration', value: 'totalDuration' }
+ ],
+ byBuildType: [
+ ]
+ }
+ },
+ created () {
+ this.initialize()
+ },
+
+ methods: {
+ setBuildTimeStat(data) {
+ this.byBuildType = data.byBuildType;
+ this.timedOutByBuildType = data.timedOutByBuildType;
+
+ $("#loadStatus").html("");
+ },
+ initialize() {
+ $.ajax({ url: "/rest/buildtime/analytics",
+ success: this.setBuildTimeStat,
+ error: showErrInLoadStatus });
+ }
+ }
+ })
+}
+
+function loadData() {
+ $("#loadStatus").html("<img
src='https://www.wallies.com/filebin/images/loading_apple.gif' width=20px
height=20px> Please wait");
+
+ $("#version").html(" " + "<a href=\"monitoring.html\">TC Bot Moniroting
Page</a> <br>");
+ $.ajax({
+ url: "rest/branches/version",
+ success: showVersionInfo,
+ error: showErrInLoadStatus
+ });
+
+
+ loadAnalytics();
+}
+
+
+</script>
+</head>
+<body>
+
+<div id="loadStatus"></div>
+
+
+<div class="formgroup" id="app">
+
+ <v-app id="readyForReview">
+ <!--<v-expansion-panel>-->
+ <!--<v-expansion-panel-content-->
+ <!--v-for="(item,i) in 1"-->
+ <!--:key="i"-->
+ <!-->-->
+ <template v-slot:header>
+ <div>Build types longest avg.duration</div>
+ </template>
+ <v-card>
+ <div>Longest average duration (more than 90 minutes)</div>
+ <v-data-table
+ :headers="headers"
+ :items="byBuildType"
+ class="elevation-1"
+ >
+ <template v-slot:items="props">
+ <!--<td>-->
+ <!--<a :href="props.item.prHtmlUrl">{{
props.item.prNumber }} </a> {{ props.item.prTitle }}</td>-->
+ <!--<td class="text-xs-right">{{ props.item.b
}}</td>-->
+ <!--<td class="text-xs-right">-->
+ <!--<img :src="props.item.prAuthorAvatarUrl"
width='20px' height='20px'> {{ props.item.prAuthor }}</td>-->
+ <td class="text-xs-right">{{ props.item.buildType
}}</td>
+ <td class="text-xs-right">{{
props.item.averageDuration }}</td>
+ <td class="text-xs-right">{{
props.item.totalDuration }}</td>
+ </template>
+ </v-data-table>
+
+ <div>Timed out suites average duration (more than 60
minutes)</div>
+ <v-data-table
+ :headers="headers"
+ :items="timedOutByBuildType"
+ class="elevation-1"
+ >
+ <template v-slot:items="props">
+ <td class="text-xs-right">{{ props.item.buildType
}}</td>
+ <td class="text-xs-right">{{
props.item.averageDuration }}</td>
+ <td class="text-xs-right">{{
props.item.totalDuration }}</td>
+ </template>
+ </v-data-table>
+ </v-card>
+ <!--</v-expansion-panel-content>-->
+ <!--</v-expansion-panel>-->
+
+ </v-app>
+</div>
+
+<div id="version"></div>
+</body>
+</html>
\ No newline at end of file
diff --git a/ignite-tc-helper-web/src/main/webapp/index.html
b/ignite-tc-helper-web/src/main/webapp/index.html
index e60ab52..86f12f3 100644
--- a/ignite-tc-helper-web/src/main/webapp/index.html
+++ b/ignite-tc-helper-web/src/main/webapp/index.html
@@ -37,6 +37,11 @@ function loadData() {
},
error: showErrInLoadStatus
});
+
+ $.ajax({ url: "/rest/buildtime/analytics",
+ success: function (data) {
+ $("#loadStatus").html("");
+ }, error: showErrInLoadStatus });
}
diff --git
a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/TcBotConst.java
b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/TcBotConst.java
index 3dd56ee..44433db 100644
--- a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/TcBotConst.java
+++ b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/TcBotConst.java
@@ -26,6 +26,9 @@ public class TcBotConst {
/** Max days to keep test invocatoin data in run statistics: affects Bot
Visa. */
public static final int HISTORY_MAX_DAYS = 21;
+ /** Absulte Max days to keep track about build existence, 42 */
+ public static final int BUILD_MAX_DAYS = HISTORY_MAX_DAYS * 2;
+
/** History collection process: build id per server ID border days. */
public static final int HISTORY_BUILD_ID_BORDER_DAYS = HISTORY_MAX_DAYS +
2;
diff --git
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java
new file mode 100644
index 0000000..09dfaa9
--- /dev/null
+++
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tcbot.engine.buildtime;
+
+import org.apache.ignite.tcbot.common.interceptor.MonitoredTask;
+import org.apache.ignite.tcbot.common.util.TimeUtil;
+import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
+import org.apache.ignite.tcbot.engine.ui.BuildTimeRecordUi;
+import org.apache.ignite.tcbot.engine.ui.BuildTimeResultUi;
+import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcbot.persistence.scheduler.IScheduler;
+import org.apache.ignite.tcignited.ITeamcityIgnited;
+import org.apache.ignite.tcignited.ITeamcityIgnitedProvider;
+import org.apache.ignite.tcignited.build.FatBuildDao;
+import org.apache.ignite.tcignited.buildref.BuildRefDao;
+import org.apache.ignite.tcignited.buildtime.BuildTimeRecord;
+import org.apache.ignite.tcignited.buildtime.BuildTimeResult;
+import org.apache.ignite.tcignited.creds.ICredentialsProv;
+import org.apache.ignite.tcignited.history.HistoryCollector;
+
+import javax.inject.Inject;
+import java.time.Duration;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+public class BuildTimeService {
+ @Inject private ITeamcityIgnitedProvider tcProv;
+
+ /** Config. */
+ @Inject private ITcBotConfig cfg;
+
+ @Inject private FatBuildDao fatBuildDao;
+
+ @Inject private IStringCompactor compactor;
+
+ @Inject private HistoryCollector historyCollector;
+
+ @Inject private IScheduler scheduler;
+
+ private volatile BuildTimeResult lastRes1d = new BuildTimeResult();
+
+ @Inject private BuildRefDao buildRefDao;
+
+ public BuildTimeResultUi analytics(ICredentialsProv prov) {
+ if (buildRefDao.buildRefsCache() == null)
+ return new BuildTimeResultUi();
+
+ Collection<String> allServers = cfg.getServerIds();
+
+ scheduler.sheduleNamed("BuildTimeService.loadAnalytics",
+ this::loadAnalytics, 15, TimeUnit.MINUTES);
+
+ Set<Integer> availableServers = allServers.stream()
+ .filter(prov::hasAccess)
+ .map(ITeamcityIgnited::serverIdToInt)
+ .collect(Collectors.toSet());
+
+ BuildTimeResultUi resultUi = new BuildTimeResultUi();
+
+ long minDuration = Duration.ofMinutes(90).toMillis();
+ long minDurationTimeout = Duration.ofMinutes(60).toMillis();
+ int cntToInclude = 50;
+ BuildTimeResult res = lastRes1d;
+
+ res.topByBuildTypes(availableServers, minDuration, cntToInclude)
+ .stream().map(this::convertToUi).forEach(e ->
resultUi.byBuildType.add(e));
+
+ res.topTimeoutsByBuildTypes(availableServers, minDurationTimeout,
cntToInclude)
+ .stream().map(this::convertToUi).forEach(e ->
resultUi.timedOutByBuildType.add(e));
+
+ return resultUi;
+ }
+
+ public BuildTimeRecordUi convertToUi(Map.Entry<Long, BuildTimeRecord> e) {
+ BuildTimeRecordUi buildTimeRecordUi = new BuildTimeRecordUi();
+ Long key = e.getKey();
+ int btId = BuildTimeResult.cacheKeyToBuildType(key);
+ buildTimeRecordUi.buildType = compactor.getStringFromId(btId);
+
+ buildTimeRecordUi.averageDuration =
TimeUtil.millisToDurationPrintable(e.getValue().avgDuration());
+ buildTimeRecordUi.totalDuration =
TimeUtil.millisToDurationPrintable(e.getValue().totalDuration());
+ return buildTimeRecordUi;
+ }
+
+ @SuppressWarnings("WeakerAccess")
+ @MonitoredTask(name = "Load Build Time Analytics")
+ protected void loadAnalytics() {
+ int days = 1;
+
+ List<Long> idsToCheck = historyCollector.findAllRecentBuilds(days,
cfg.getServerIds());
+
+ BuildTimeResult res = fatBuildDao.loadBuildTimeResult(days,
idsToCheck);
+
+ lastRes1d = res;
+ }
+}
diff --git
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeRecordUi.java
similarity index 73%
copy from
tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
copy to
tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeRecordUi.java
index 23fe3bd..f411d83 100644
---
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
+++
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeRecordUi.java
@@ -14,18 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.ignite.tcbot.engine.ui;
-/**
- * Detailed status of failures: Reference to some Issue with current suite or
test detected by the Bot.
- * Currently contains only display name.
- */
-public class DsProblemRef {
- public String name;
- public String webUrl;
-
- public DsProblemRef(String name) {
- this.name = name;
- }
+@SuppressWarnings({"WeakerAccess", "PublicField"})
+public class BuildTimeRecordUi {
+ public String buildType;
+ public String averageDuration;
+ public String totalDuration;
}
diff --git
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeResultUi.java
similarity index 73%
copy from
tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
copy to
tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeResultUi.java
index 23fe3bd..37e9a21 100644
---
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
+++
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeResultUi.java
@@ -17,15 +17,11 @@
package org.apache.ignite.tcbot.engine.ui;
-/**
- * Detailed status of failures: Reference to some Issue with current suite or
test detected by the Bot.
- * Currently contains only display name.
- */
-public class DsProblemRef {
- public String name;
- public String webUrl;
+import java.util.ArrayList;
+import java.util.List;
- public DsProblemRef(String name) {
- this.name = name;
- }
+@SuppressWarnings({"WeakerAccess", "PublicField"})
+public class BuildTimeResultUi {
+ public List<BuildTimeRecordUi> byBuildType = new ArrayList<>();
+ public List<BuildTimeRecordUi> timedOutByBuildType = new ArrayList<>();
}
diff --git
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsHistoryStatUi.java
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsHistoryStatUi.java
index a718e47..6107c70 100644
---
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsHistoryStatUi.java
+++
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsHistoryStatUi.java
@@ -23,7 +23,7 @@ import javax.annotation.Nullable;
/**
* Detailed status of failures: Test/suite failure summary: contains statistic
of failures and total runs for suite or for test.
*/
-public class DsHistoryStatUi {
+@SuppressWarnings("PublicField") public class DsHistoryStatUi {
/** Registered number of failures from TC helper DB */
@Nullable public Integer failures;
diff --git
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
index 23fe3bd..3c98b34 100644
---
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
+++
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
@@ -21,6 +21,7 @@ package org.apache.ignite.tcbot.engine.ui;
* Detailed status of failures: Reference to some Issue with current suite or
test detected by the Bot.
* Currently contains only display name.
*/
+@SuppressWarnings({"WeakerAccess", "PublicField"})
public class DsProblemRef {
public String name;
public String webUrl;
diff --git
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index eaeeb54..c47d8e9 100644
---
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -554,6 +554,10 @@ public class FatBuildCompacted extends BuildRefCompacted
implements IVersionedEn
.forEach(this.problems::add);
}
+ public Long statisticValue(Integer propCode) {
+ return statistics == null ? null : statistics.statisticValue(propCode);
+ }
+
public Long buildDuration(IStringCompactor compactor) {
return statistics == null ? null : statistics.buildDuration(compactor);
}
@@ -711,4 +715,8 @@ public class FatBuildCompacted extends BuildRefCompacted
implements IVersionedEn
return cnt;
}
+
+ public long getFinishDateTs() {
+ return finishDate;
+ }
}
diff --git
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java
index 12a0f67..514542b 100644
---
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java
@@ -29,6 +29,8 @@ import org.apache.ignite.internal.util.GridLongList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
+
/**
* Statistics values to be saved in compacted form.
*/
@@ -85,10 +87,14 @@ public class StatisticsCompacted {
public Long buildDuration(IStringCompactor compactor) {
Integer buildDurationId =
compactor.getStringIdIfPresent(Statistics.BUILD_DURATION);
- if (buildDurationId == null)
+ return statisticValue(buildDurationId);
+ }
+
+ @Nullable public Long statisticValue(@Nullable Integer propCode) {
+ if (propCode == null)
return null;
- long val = findPropertyValue(buildDurationId);
+ long val = findPropertyValue(propCode);
return val >= 0 ? val : null;
}
@@ -100,12 +106,7 @@ public class StatisticsCompacted {
public Long buildDurationNetTime(IStringCompactor compactor) {
Integer buildDurationNetId =
compactor.getStringIdIfPresent(Statistics.BUILD_DURATION_NET_TIME);
- if (buildDurationNetId == null)
- return null;
-
- long val = findPropertyValue(buildDurationNetId);
-
- return val >= 0 ? val : null;
+ return statisticValue(buildDurationNetId);
}
/**
@@ -115,12 +116,7 @@ public class StatisticsCompacted {
public Long artifcactPublishingDuration(IStringCompactor compactor) {
Integer buildDurationNetId =
compactor.getStringIdIfPresent(Statistics.ARTIFACTS_PUBLISHING_DURATION);
- if (buildDurationNetId == null)
- return null;
-
- long val = findPropertyValue(buildDurationNetId);
-
- return val >= 0 ? val : null;
+ return statisticValue(buildDurationNetId);
}
/**
@@ -130,12 +126,7 @@ public class StatisticsCompacted {
public Long dependeciesResolvingDuration(IStringCompactor compactor) {
Integer buildDurationNetId =
compactor.getStringIdIfPresent(Statistics.DEPENDECIES_RESOLVING_DURATION);
- if (buildDurationNetId == null)
- return null;
-
- long val = findPropertyValue(buildDurationNetId);
-
- return val >= 0 ? val : null;
+ return statisticValue(buildDurationNetId);
}
/**
@@ -145,15 +136,10 @@ public class StatisticsCompacted {
public Long sourceUpdateDuration(IStringCompactor compactor) {
Integer buildDurationNetId =
compactor.getStringIdIfPresent(Statistics.SOURCES_UPDATE_DURATION);
- if (buildDurationNetId == null)
- return null;
-
- long val = findPropertyValue(buildDurationNetId);
-
- return val >= 0 ? val : null;
+ return statisticValue(buildDurationNetId);
}
- private long findPropertyValue(int propCode) {
+ public long findPropertyValue(int propCode) {
if (keys == null)
return -1L;
diff --git
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
index 892b2e2..ce10502 100644
---
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
@@ -569,12 +569,17 @@ public class TeamcityIgnitedImpl implements
ITeamcityIgnited {
long ts = highBuild.getStartDateTs();
- return ts > 0 ? ts : null;
+ if (ts > 0) {
+ runHistCompactedDao.setBuildStartTime(srvIdMaskHigh, buildId, ts);
+
+ return ts;
+ } else
+ return null;
}
- @GuavaCached(maximumSize = 100000, expireAfterAccessSecs = 90, softValues
= true)
+ //@GuavaCached(maximumSize = 100000, expireAfterAccessSecs = 90,
softValues = true)
public Long getBuildStartTime(int buildId) {
- return runHistCompactedDao.getBuildStartTime(srvIdMaskHigh, buildId);
+ return histCollector.getBuildStartTime(srvIdMaskHigh, buildId);
}
/** {@inheritDoc} */
diff --git
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index 82da2c8..5b4740d 100644
---
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -19,23 +19,6 @@ package org.apache.ignite.tcignited.build;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterables;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import java.util.stream.StreamSupport;
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-import javax.cache.Cache;
-import javax.cache.processor.EntryProcessorException;
-import javax.cache.processor.EntryProcessorResult;
-import javax.cache.processor.MutableEntry;
-import javax.inject.Inject;
-import javax.inject.Provider;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.binary.BinaryObject;
@@ -45,8 +28,10 @@ import
org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.persistence.CacheConfigs;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
+import org.apache.ignite.tcignited.buildtime.BuildTimeResult;
import org.apache.ignite.tcignited.history.HistoryCollector;
import org.apache.ignite.tcservice.model.changes.ChangesList;
+import org.apache.ignite.tcservice.model.hist.BuildRef;
import org.apache.ignite.tcservice.model.result.Build;
import org.apache.ignite.tcservice.model.result.problems.ProblemOccurrence;
import org.apache.ignite.tcservice.model.result.stat.Statistics;
@@ -54,6 +39,19 @@ import
org.apache.ignite.tcservice.model.result.tests.TestOccurrencesFull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import javax.cache.Cache;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.EntryProcessorResult;
+import javax.cache.processor.MutableEntry;
+import javax.inject.Inject;
+import javax.inject.Provider;
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
/**
*
*/
@@ -63,6 +61,7 @@ public class FatBuildDao {
/** Cache name */
public static final String TEAMCITY_FAT_BUILD_CACHE_NAME =
"teamcityFatBuild";
+ public static final int MAX_FAT_BUILD_CHUNK = 32 * 10;
/** Ignite provider. */
@Inject private Provider<Ignite> igniteProvider;
@@ -206,6 +205,17 @@ public class FatBuildDao {
/**
* @param srvId Server id.
+ * @param buildId Build Id.
+ */
+ @Nullable public Long getBuildStartTime(int srvId, Integer buildId) {
+ IgniteCache<Long, BinaryObject> cacheBin =
buildsCache.withKeepBinary();
+ long key = buildIdToCacheKey(srvId, buildId);
+
+ return cacheBin.invoke(key, new GetStartTimeProc());
+ }
+
+ /**
+ * @param srvId Server id.
* @param ids Ids.
*/
public Map<Integer, Long> getBuildStartTime(int srvId, Set<Integer> ids) {
@@ -213,7 +223,7 @@ public class FatBuildDao {
Set<Long> keys = buildsIdsToCacheKeys(srvId, ids);
HashMap<Integer, Long> res = new HashMap<>();
- Iterables.partition(keys, 32 * 10).forEach(
+ Iterables.partition(keys, MAX_FAT_BUILD_CHUNK).forEach(
chunk -> {
Map<Long, EntryProcessorResult<Long>> map =
cacheBin.invokeAll(keys, new GetStartTimeProc());
map.forEach((k, r) -> {
@@ -227,6 +237,113 @@ public class FatBuildDao {
return res;
}
+ public BuildTimeResult loadBuildTimeResult(int ageDays, List<Long>
idsToCheck) {
+ int stateRunning = compactor.getStringId(BuildRef.STATE_RUNNING);
+ Integer buildDurationId =
compactor.getStringIdIfPresent(Statistics.BUILD_DURATION);
+ int timeoutProblemCode =
compactor.getStringId(ProblemOccurrence.TC_EXECUTION_TIMEOUT);
+
+ BuildTimeResult res = new BuildTimeResult();
+
+ // also may take affinity into account
+ Iterables.partition(idsToCheck, MAX_FAT_BUILD_CHUNK).forEach(
+ chunk -> {
+ HashSet<Long> keys = new HashSet<>(chunk);
+ Map<Long, FatBuildCompacted> all =
buildsCache.getAll(keys);
+ all.forEach((key, build) -> {
+ if (build.isComposite())
+ return;
+
+ long runningTime = getBuildRunningTime(stateRunning,
buildDurationId, build);
+ if (runningTime > 0) {
+ int buildTypeId = build.buildTypeId();
+ System.err.println("Running " + runningTime + "
BT: " + buildTypeId);
+
+ int srvId = BuildRefDao.cacheKeyToSrvId(key);
+ boolean hasTimeout =
build.hasBuildProblemType(timeoutProblemCode);
+
+ res.add(srvId, buildTypeId, runningTime,
hasTimeout);
+ }
+ });
+ }
+ );
+
+ return res;
+
+ }
+
+ public static long getBuildRunningTime(int stateRunning, Integer
buildDurationId,
+ FatBuildCompacted buildBinary) {
+ long startTs = buildBinary.getStartDateTs();
+
+ if (startTs <= 0)
+ return -1;
+
+ int state = buildBinary.state();
+
+ long runningTime = -1;
+ if (stateRunning == state)
+ runningTime = System.currentTimeMillis() - startTs;
+
+ if (runningTime < 0) {
+ if (buildDurationId != null) {
+ Long val = buildBinary.statisticValue(buildDurationId);
+
+ runningTime = (val != null && val >= 0) ? val : -1;
+ }
+
+
+ }
+
+ if (runningTime < 0) {
+ long finishTs = buildBinary.getFinishDateTs();
+
+ if (finishTs > 0)
+ runningTime = finishTs - startTs;
+ }
+
+ return runningTime;
+ }
+
+ public static long getBuildRunningTime(int stateRunning, Integer
buildDurationId,
+ BinaryObject buildBinary) {
+ Long startTs = buildBinary.field("startDate");
+
+ if (startTs == null || startTs <= 0)
+ return -1;
+
+
+ int status = buildBinary.field("status");
+ int state = buildBinary.field("state");
+
+ long runningTime = -1;
+ if(stateRunning == state)
+ runningTime = System.currentTimeMillis() - startTs;
+
+ if(runningTime<0){
+
+ if (buildDurationId != null) {
+ BinaryObject statistics = buildBinary.field("statistics");
+ if(statistics!=null) {
+ // statistics.field()
+ }
+ long val = -1; //statistics.findPropertyValue(buildDurationId);
+
+ runningTime = val >= 0 ? val : -1;
+ }
+
+
+ }
+
+ if(runningTime<0) {
+ Long finishTs= buildBinary.field("finishDate");
+
+ if(finishTs!=null)
+ runningTime = finishTs - startTs;
+ }
+
+ return runningTime;
+ }
+
private static class GetStartTimeProc implements CacheEntryProcessor<Long,
BinaryObject, Long> {
public GetStartTimeProc() {
}
diff --git
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
index e8c8ae0..72ad31a 100644
---
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
@@ -104,7 +104,7 @@ public class BuildRefDao {
* @param srvId Server id.
*/
public static boolean isKeyForServer(Long key, int srvId) {
- return key!=null && key >> 32 == srvId;
+ return key!=null && cacheKeyToSrvId(key) == srvId;
}
/**
@@ -169,6 +169,13 @@ public class BuildRefDao {
}
/**
+ * @param cacheKey Cache key.
+ */
+ public static int cacheKeyToSrvId(long cacheKey) {
+ return (int)(cacheKey >> 32);
+ }
+
+ /**
* @param srvId Server id mask high.
* @param buildTypeId Build type (suite) id.
* @param bracnhNameQry Bracnh name query.
@@ -307,4 +314,8 @@ public class BuildRefDao {
return StreamSupport.stream(buildRefsCache.spliterator(), false)
.filter(entry -> isKeyForServer(entry.getKey(), srvId));
}
+
+ public IgniteCache<Long, BuildRefCompacted> buildRefsCache() {
+ return buildRefsCache;
+ }
}
diff --git
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeRecord.java
similarity index 66%
copy from
tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
copy to
tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeRecord.java
index 23fe3bd..63e8026 100644
---
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeRecord.java
@@ -14,18 +14,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.ignite.tcignited.buildtime;
-package org.apache.ignite.tcbot.engine.ui;
+public class BuildTimeRecord {
+ private long totaltime;
+ private int cnt;
-/**
- * Detailed status of failures: Reference to some Issue with current suite or
test detected by the Bot.
- * Currently contains only display name.
- */
-public class DsProblemRef {
- public String name;
- public String webUrl;
+ public void addInvocation(long runningTimeMs) {
+ totaltime += runningTimeMs;
+ cnt++;
+ }
+
+ public long avgDuration() {
+ if (cnt == 0)
+ return 0;
+
+ return totaltime / cnt;
+ }
- public DsProblemRef(String name) {
- this.name = name;
+ public long totalDuration() {
+ return totaltime;
}
}
diff --git
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeResult.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeResult.java
new file mode 100644
index 0000000..0721fd9
--- /dev/null
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeResult.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.tcignited.buildtime;
+
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class BuildTimeResult {
+ private Map<Long, BuildTimeRecord> btByBuildType = new HashMap<>();
+ private Map<Long, BuildTimeRecord> timedOutByBuildType = new HashMap<>();
+
+ public void add(int srvId, int buildTypeId, long runningTime, boolean
hasTimeout) {
+ long cacheKey = buildTypeToCacheKey(srvId, buildTypeId);
+ btByBuildType.computeIfAbsent(cacheKey, k -> new
BuildTimeRecord()).addInvocation(runningTime);
+
+ if (hasTimeout)
+ timedOutByBuildType.computeIfAbsent(cacheKey, k -> new
BuildTimeRecord()).addInvocation(runningTime);
+ }
+
+ public static long buildTypeToCacheKey(long srvId, int btId) {
+ return (long)btId | srvId << 32;
+ }
+
+ public static int cacheKeyToSrvId(long cacheKey) {
+ return (int)(cacheKey >> 32);
+ }
+
+ public static int cacheKeyToBuildType(Long cacheKey) {
+ long l = cacheKey << 32;
+ return (int)(l >> 32);
+ }
+
+
+ public List<Map.Entry<Long, BuildTimeRecord>> topByBuildTypes(Set<Integer>
availableServers,
+ long
minAvgDurationMs,
+ int maxCnt) {
+ return filtered(btByBuildType, availableServers, minAvgDurationMs)
+ .sorted(Comparator.comparing(
+ (Function<Map.Entry<Long, BuildTimeRecord>, Long>)
entry -> entry.getValue().avgDuration())
+ .reversed())
+ .limit(maxCnt)
+ .collect(Collectors.toList());
+ }
+
+ public List<Map.Entry<Long, BuildTimeRecord>>
topTimeoutsByBuildTypes(Set<Integer> availableServers,
+ long
minAvgDurationMs,
+ int maxCnt) {
+ return filtered(timedOutByBuildType, availableServers,
minAvgDurationMs)
+ .sorted(Comparator.comparing(
+ (Function<Map.Entry<Long, BuildTimeRecord>, Long>)
entry -> entry.getValue().avgDuration())
+ .reversed())
+ .limit(maxCnt)
+ .collect(Collectors.toList());
+ }
+
+ private Stream<Map.Entry<Long, BuildTimeRecord>> filtered(Map<Long,
BuildTimeRecord> map, Set<Integer> availableServers, long minAvgDurationMs) {
+ return map.entrySet().stream()
+ .filter(e -> {
+ Long key = e.getKey();
+ int srvId = cacheKeyToSrvId(key);
+ return availableServers.contains(srvId);
+ })
+ .filter(e -> e.getValue().avgDuration() > minAvgDurationMs);
+ }
+}
diff --git
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
index e9ca205..1246a3e 100644
---
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
@@ -16,42 +16,46 @@
*/
package org.apache.ignite.tcignited.history;
-import com.google.common.base.Preconditions;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Iterables;
-import java.time.Duration;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.BiPredicate;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import javax.inject.Inject;
-import javax.inject.Provider;
import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
import org.apache.ignite.tcbot.common.TcBotConst;
import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
+import org.apache.ignite.tcbot.common.exeption.ServicesStartingException;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.build.SuiteHistory;
import org.apache.ignite.tcignited.buildref.BranchEquivalence;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
+import org.apache.ignite.tcservice.model.hist.BuildRef;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.cache.Cache;
+import javax.inject.Inject;
+import javax.inject.Provider;
+import java.time.Duration;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiPredicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
/**
*
*/
@@ -143,9 +147,18 @@ public class HistoryCollector {
long curTs = System.currentTimeMillis();
Set<Integer> buildIds = bRefsList.stream()
.filter(b -> {
- AtomicInteger biggestIdOutOfScope =
biggestBuildIdOutOfHistoryScope.get(srvId);
+ /* AtomicInteger biggestIdOutOfScope =
biggestBuildIdOutOfHistoryScope.get(srvId);
int outOfScopeBuildId = biggestIdOutOfScope == null ? -1 :
biggestIdOutOfScope.get();
return b.id() > outOfScopeBuildId;
+ */
+
+ Integer maxBuildIdForDay =
runHistCompactedDao.getBorderForAgeForBuildId(srvId,
TcBotConst.HISTORY_BUILD_ID_BORDER_DAYS);
+
+ if (maxBuildIdForDay == null)
+ return true;
+
+ return b.id()>maxBuildIdForDay;
+
})
.filter(this::applicableForHistory)
.map(BuildRefCompacted::id)
@@ -225,7 +238,7 @@ public class HistoryCollector {
int normalizedBaseBranch) {
Map<Integer, SuiteInvocation> suiteRunHist =
histDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch);
- System.out.println("***** Found history for suite "
+ logger.info("***** Found history for suite "
+ compactor.getStringFromId(buildTypeId)
+ " branch " + compactor.getStringFromId(normalizedBaseBranch) +
": " + suiteRunHist.size() );
@@ -325,4 +338,91 @@ public class HistoryCollector {
return getSuiteHist(srvId, buildTypeId, normalizedBaseBranch);
}
+
+ /**
+ * @param srvId Server id.
+ * @param buildId Build id.
+ */
+ public Long getBuildStartTime(int srvId, int buildId) {
+ Long time = runHistCompactedDao.getBuildStartTime(srvId, buildId);
+
+ if (time != null)
+ return time;
+
+ time = fatBuildDao.getBuildStartTime(srvId, buildId);
+
+ if (time != null)
+ runHistCompactedDao.setBuildStartTime(srvId, buildId, time);
+
+ return time;
+ }
+
+ public List<Long> findAllRecentBuilds(int days, Collection<String>
allServers) {
+ IgniteCache<Long, BuildRefCompacted> cache =
buildRefDao.buildRefsCache();
+ if (cache == null)
+ throw new ServicesStartingException(new RuntimeException("Ignite
is not yet available"));
+
+ IgniteCache<Long, BinaryObject> cacheBin = cache.withKeepBinary();
+
+ final Map<Integer, Integer> preBorder = new HashMap<>();
+
+ allServers.stream()
+ .map(ITeamcityIgnited::serverIdToInt)
+ .forEach(srvId -> {
+ Integer borderForAgeForBuildId =
runHistCompactedDao.getBorderForAgeForBuildId(srvId, days);
+ if (borderForAgeForBuildId != null)
+ preBorder.put(srvId, borderForAgeForBuildId);
+ });
+
+ final int stateQueued = compactor.getStringId(BuildRef.STATE_QUEUED);
+
+ long minTs = System.currentTimeMillis() -
Duration.ofDays(days).toMillis();
+ QueryCursor<Cache.Entry<Long, BinaryObject>> query = cacheBin.query(
+ new ScanQuery<Long, BinaryObject>()
+ .setFilter((key, v) -> {
+ int srvId = BuildRefDao.cacheKeyToSrvId(key);
+ Integer buildIdBorder = preBorder.get(srvId);
+ if (buildIdBorder != null) {
+ int id = v.field("id");
+ if (id < buildIdBorder)
+ return false;// pre-filtered build out of
scope
+ }
+ int state = v.field("state");
+
+ return stateQueued != state;
+ }));
+
+ int cnt = 0;
+ List<Long> idsToCheck = new ArrayList<>();
+
+ try (QueryCursor<Cache.Entry<Long, BinaryObject>> cursor = query) {
+ for (Cache.Entry<Long, BinaryObject> next : cursor) {
+ Long key = next.getKey();
+ int srvId = BuildRefDao.cacheKeyToSrvId(key);
+
+ int buildId = BuildRefDao.cacheKeyToBuildId(key);
+
+ Integer borderBuildId =
runHistCompactedDao.getBorderForAgeForBuildId(srvId, days);
+
+ boolean passesDate = borderBuildId == null || buildId >=
borderBuildId;
+
+ if (!passesDate)
+ continue;
+
+ Long startTs = getBuildStartTime(srvId, buildId);
+ if (startTs == null || startTs < minTs)
+ continue; //time not saved in the DB, skip
+
+ System.err.println("Found build at srv [" + srvId + "]: [" +
buildId + "] to analyze, ts=" + startTs);
+
+ cnt++;
+
+ idsToCheck.add(key);
+ }
+ }
+
+ System.err.println("Total builds to load " + cnt);
+
+ return idsToCheck;
+ }
}
diff --git
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
index 5a18465..aa49966 100644
---
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
@@ -17,6 +17,7 @@
package org.apache.ignite.tcignited.history;
+import java.time.Duration;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -24,7 +25,10 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicIntegerArray;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -42,6 +46,7 @@ import
org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.tcbot.common.TcBotConst;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.GuavaCached;
import org.apache.ignite.tcbot.persistence.CacheConfigs;
@@ -51,7 +56,7 @@ import org.apache.ignite.tcignited.buildref.BuildRefDao;
import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
/**
- *
+ * TODO: rename to build start time storage
*/
public class RunHistCompactedDao {
/** Cache name. */
@@ -78,6 +83,17 @@ public class RunHistCompactedDao {
/** Build start time. */
private IgniteCache<Long, Long> buildStartTime;
+ /**
+ * Biggest build ID, which is older than particular days count.
+ * Map: server ID-> Array of build Ids
+ * Array[0] = max build ID
+ * Array[1] = max build ID older than 1 day
+ */
+ private final ConcurrentMap<Integer, AtomicIntegerArray>
maxBuildIdOlderThanDays = new ConcurrentHashMap<>();
+
+ /** Millis in day. */
+ private static final long MILLIS_IN_DAY = Duration.ofDays(1).toMillis();
+
/** Compactor. */
@Inject private IStringCompactor compactor;
@@ -146,14 +162,27 @@ public class RunHistCompactedDao {
if (ts == null || ts <= 0)
return null;
+ processBuildForBorder(srvId, buildId, ts);
+
return ts;
}
+ public boolean setBuildStartTime(int srvId, int buildId, long ts) {
+ if (ts <= 0)
+ return false;
+
+ processBuildForBorder(srvId, buildId, ts);
+
+ return buildStartTime.putIfAbsent(buildIdToCacheKey(srvId, buildId),
ts);
+ }
+
@AutoProfiling
public boolean setBuildProcessed(int srvId, int buildId, long ts) {
if (ts <= 0)
return false;
+ processBuildForBorder(srvId, buildId, ts);
+
return buildStartTime.putIfAbsent(buildIdToCacheKey(srvId, buildId),
ts);
}
@@ -263,8 +292,13 @@ public class RunHistCompactedDao {
Map<Integer, Long> res = new HashMap<>();
buildStartTime.getAll(cacheKeys).forEach((k, ts) -> {
- if (ts != null && ts > 0)
- res.put(BuildRefDao.cacheKeyToBuildId(k), ts);
+ if (ts != null && ts > 0) {
+ int buildId = BuildRefDao.cacheKeyToBuildId(k);
+
+ res.put(buildId, ts);
+
+ processBuildForBorder(srvId, buildId, ts);
+ }
});
return res;
@@ -274,10 +308,46 @@ public class RunHistCompactedDao {
Map<Long, Long> res = new HashMap<>();
builds.forEach((buildId, ts) -> {
- if (ts != null && ts > 0)
+ if (ts != null && ts > 0) {
res.put(buildIdToCacheKey(srvId, buildId), ts);
+
+ processBuildForBorder(srvId, buildId, ts);
+ }
});
buildStartTime.putAll(res);
}
+
+ private void processBuildForBorder(int srvId, Integer buildId, Long ts) {
+ if (ts == null || ts <= 0)
+ return;
+
+ AtomicIntegerArray arr = maxBuildIdOlderThanDays.computeIfAbsent(srvId,
+ k -> new AtomicIntegerArray(TcBotConst.BUILD_MAX_DAYS + 1));
+
+ long ageMs = System.currentTimeMillis() - ts;
+ if (ageMs < 0)
+ return;
+
+ long days = ageMs / MILLIS_IN_DAY;
+ if (days > TcBotConst.BUILD_MAX_DAYS)
+ days = TcBotConst.BUILD_MAX_DAYS;
+
+ arr.accumulateAndGet((int)days, buildId, Math::max);
+ }
+
+ @Nullable public Integer getBorderForAgeForBuildId(int srvId, int ageDays)
{
+ AtomicIntegerArray arr = maxBuildIdOlderThanDays.get(srvId);
+ if (arr == null)
+ return null;
+
+ for (int i = ageDays; i < TcBotConst.BUILD_MAX_DAYS; i++) {
+ int buildId = arr.get(i);
+ if (buildId != 0)
+ return buildId;
+ }
+
+ return null;
+ }
+
}
