This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new 1604b22 Incremental build upload from TC implemented - Fixes #15.
1604b22 is described below
commit 1604b221bcb984add815421d88e930ee217d7d08
Author: Dmitriy Pavlov <dpav...@apache.org>
AuthorDate: Thu Sep 20 16:08:08 2018 +0300
Incremental build upload from TC implemented - Fixes #15.
Signed-off-by: Dmitriy Pavlov <dpav...@apache.org>
---
.../main/java/org/apache/ignite/ci/ITeamcity.java | 16 ++--
.../apache/ignite/ci/IgnitePersistentTeamcity.java | 94 ++++++++++++++--------
.../org/apache/ignite/ci/IgniteTeamcityHelper.java | 52 ++++++------
.../org/apache/ignite/ci/issue/IssueDetector.java | 2 +-
.../runners/IgniteTeamcityHelperRunnerExample.java | 4 +-
5 files changed, 97 insertions(+), 71 deletions(-)
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
index 4e9f9d6..ccc3fea 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
@@ -66,16 +66,17 @@ public interface ITeamcity extends AutoCloseable {
* @return list of builds in historical order, recent builds coming last
*/
default List<BuildRef> getFinishedBuilds(String projectId, String branch) {
- return getFinishedBuilds(projectId, branch, null);
+ return getFinishedBuilds(projectId, branch, null, null);
};
/**
* @param projectId suite ID (string without spaces)
* @param branch
* @param cnt builds count
+ * @param sinceBuildNumber
* @return list of builds in historical order, recent builds coming last
*/
- List<BuildRef> getFinishedBuilds(String projectId, String branch, Long
cnt);
+ List<BuildRef> getFinishedBuilds(String projectId, String branch, Long
cnt, Integer sinceBuildNumber);
/**
* Includes snapshot dependencies failed builds into list
@@ -85,7 +86,7 @@ public interface ITeamcity extends AutoCloseable {
* @return list of builds in historical order, recent builds coming last
*/
default List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String
projectId, String branch){
- return getFinishedBuildsIncludeSnDepFailed(projectId, branch, null);
+ return getFinishedBuildsIncludeSnDepFailed(projectId, branch, null,
null);
};
/**
@@ -94,9 +95,10 @@ public interface ITeamcity extends AutoCloseable {
* @param projectId suite ID (string without spaces)
* @param branch branch in TC identification
* @param cnt builds count
+ * @param sinceBuildNumber limit builds export with some build number, not
operational for Persistent connection
* @return list of builds in historical order, recent builds coming last
*/
- List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String projectId,
String branch, Long cnt);
+ List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String projectId,
String branch, Long cnt, Integer sinceBuildNumber);
/** */
CompletableFuture<List<BuildRef>> getRunningBuilds(@Nullable String
branch);
@@ -104,12 +106,8 @@ public interface ITeamcity extends AutoCloseable {
/** */
CompletableFuture<List<BuildRef>> getQueuedBuilds(@Nullable String branch);
- default int[] getBuildNumbersFromHistory(String projectId, String
branchNameForHist) {
- return getBuildNumbersFromHistory(projectId, branchNameForHist, null);
- }
-
default int[] getBuildNumbersFromHistory(String projectId, String
branchNameForHist, Long cnt) {
- return getFinishedBuilds(projectId, branchNameForHist,
cnt).stream().mapToInt(BuildRef::getId).toArray();
+ return getFinishedBuilds(projectId, branchNameForHist, cnt,
null).stream().mapToInt(BuildRef::getId).toArray();
}
Build getBuild(String href);
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
index d3b038d..4d49121 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
@@ -33,6 +33,7 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
+import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@@ -71,37 +72,43 @@ import org.apache.ignite.ci.util.CacheUpdateUtil;
import org.apache.ignite.ci.util.CollectionUtil;
import org.apache.ignite.ci.util.ObjectInterner;
import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.xml.sax.SAXParseException;
import static org.apache.ignite.ci.BuildChainProcessor.normalizeBranch;
/**
- *
+ * Apache Ignite based cache over teamcity responses
*/
public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity,
ITeamcity, ITcAnalytics {
- //V1 caches, 1024 parts
-
- //V2 caches, 32 parts
- public static final String TESTS_OCCURRENCES = "testOccurrences";
- public static final String TESTS_RUN_STAT = "testsRunStat";
- public static final String CALCULATED_STATISTIC = "calculatedStatistic";
- public static final String LOG_CHECK_RESULT = "logCheckResult";
- public static final String CHANGE_INFO_FULL = "changeInfoFull";
- public static final String CHANGES_LIST = "changesList";
- public static final String ISSUES_USAGES_LIST = "issuesUsagesList";
- public static final String TEST_FULL = "testFull";
- public static final String BUILD_PROBLEMS = "buildProblems";
- public static final String BUILD_STATISTICS = "buildStatistics";
- public static final String BUILD_HIST_FINISHED = "buildHistFinished";
- public static final String BUILD_HIST_FINISHED_OR_FAILED =
"buildHistFinishedOrFailed";
+ /** Logger. */
+ private static final Logger logger =
LoggerFactory.getLogger(IgnitePersistentTeamcity.class);
+
+ //V2 caches, 32 parts (V1 caches were 1024 parts)
+ private static final String TESTS_OCCURRENCES = "testOccurrences";
+ private static final String TESTS_RUN_STAT = "testsRunStat";
+ private static final String CALCULATED_STATISTIC = "calculatedStatistic";
+ private static final String LOG_CHECK_RESULT = "logCheckResult";
+ private static final String CHANGE_INFO_FULL = "changeInfoFull";
+ private static final String CHANGES_LIST = "changesList";
+ private static final String ISSUES_USAGES_LIST = "issuesUsagesList";
+ private static final String TEST_FULL = "testFull";
+ private static final String BUILD_PROBLEMS = "buildProblems";
+ private static final String BUILD_STATISTICS = "buildStatistics";
+ private static final String BUILD_HIST_FINISHED = "buildHistFinished";
+ private static final String BUILD_HIST_FINISHED_OR_FAILED =
"buildHistFinishedOrFailed";
public static final String BOT_DETECTED_ISSUES = "botDetectedIssues";
//todo need separate cache or separate key for 'execution time' because it
is placed in statistics
- public static final String BUILDS_FAILURE_RUN_STAT =
"buildsFailureRunStat";
+ private static final String BUILDS_FAILURE_RUN_STAT =
"buildsFailureRunStat";
public static final String BUILDS = "builds";
- public static final String BUILD_QUEUE = "buildQueue";
- public static final String RUNNING_BUILDS = "runningBuilds";
+ private static final String BUILD_QUEUE = "buildQueue";
+ private static final String RUNNING_BUILDS = "runningBuilds";
+
+ /** Number of builds to re-query from TC to be sure some builds in the
middle are not lost. */
+ private static final int MAX_BUILDS_IN_PAST_TO_RELOAD = 5;
@Inject
private Ignite ignite;
@@ -135,7 +142,6 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
//for DI
public IgnitePersistentTeamcity() {}
-
private IgnitePersistentTeamcity(Ignite ignite, IgniteTeamcityHelper
teamcity) {
init(teamcity);
this.ignite = ignite;
@@ -267,12 +273,12 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
return loaded;
}
- protected <K> List<BuildRef> loadBuildHistory(IgniteCache<K,
Expirable<List<BuildRef>>> cache,
- int seconds,
- Long cnt,
- K key,
- Function<K,
List<BuildRef>> realLoad) {
- @Nullable Expirable<List<BuildRef>> persistedBuilds =
readBuildHistEntry( cache, (K) key);
+ protected <K> List<BuildRef> loadBuildHistory(IgniteCache<K,
Expirable<List<BuildRef>>> cache,
+ int seconds,
+ Long cnt,
+ K key,
+ BiFunction<K, Integer,
List<BuildRef>> realLoad) {
+ @Nullable Expirable<List<BuildRef>> persistedBuilds =
readBuildHistEntry(cache, (K) key);
if (persistedBuilds != null
&& (persistedBuilds.isAgeLessThanSecs(seconds)
@@ -297,9 +303,24 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
//todo sinceBuild:(number:) // --todo -10 build numbers
+ Integer sinceBuildNum = null;
+ if (persistedBuilds != null) {
+ List<BuildRef> prevData = persistedBuilds.getData();
+ if (prevData.size() >= MAX_BUILDS_IN_PAST_TO_RELOAD) {
+ BuildRef buildRef = prevData.get(prevData.size() -
MAX_BUILDS_IN_PAST_TO_RELOAD);
+
+ if (!Strings.isNullOrEmpty(buildRef.buildNumber)) {
+ try {
+ sinceBuildNum =
Integer.valueOf(buildRef.buildNumber);
+ } catch (NumberFormatException e) {
+ logger.info("", e);
+ }
+ }
+ }
+ }
List<BuildRef> dataFromRest;
try {
- dataFromRest = realLoad.apply(key);
+ dataFromRest = realLoad.apply(key, sinceBuildNum);
}
catch (Exception e) {
if (Throwables.getRootCause(e) instanceof
FileNotFoundException) {
@@ -340,8 +361,8 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
@AutoProfiling
@SuppressWarnings("WeakerAccess")
- protected <K> Lock lockBuildHistEntry(IgniteCache<K,
Expirable<List<BuildRef>>> cache, K key) {
- if(noLocks)
+ protected <K> Lock lockBuildHistEntry(IgniteCache<K,
Expirable<List<BuildRef>>> cache, K key) {
+ if (noLocks)
return null;
Lock lock = cache.lock(key);
@@ -353,11 +374,15 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
/** {@inheritDoc} */
@AutoProfiling
- @Override public List<BuildRef> getFinishedBuilds(String projectId, String
branch, Long cnt) {
+ @Override public List<BuildRef> getFinishedBuilds(String projectId,
+ String branch,
+ Long cnt,
+ Integer ignored) {
+ //todo may be support sinceBuildNo
final SuiteInBranch suiteInBranch = new SuiteInBranch(projectId,
branch);
List<BuildRef> buildRefs = loadBuildHistory(buildHistCache(), 60, cnt,
suiteInBranch,
- (key) -> teamcity.getFinishedBuilds(projectId, branch, cnt));
+ (key, sinceBuildNumber) -> teamcity.getFinishedBuilds(projectId,
branch, cnt, sinceBuildNumber));
if (cnt == null)
return buildRefs;
@@ -385,11 +410,14 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
/** {@inheritDoc} */
@AutoProfiling
- @Override public List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String
projectId, String branch, Long cnt) {
+ @Override public List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String
projectId,
+ String
branch,
+ Long
cnt,
+
Integer ignored) {
final SuiteInBranch suiteInBranch = new SuiteInBranch(projectId,
branch);
return loadBuildHistory(buildHistIncFailedCache(), 60, cnt,
suiteInBranch,
- (key) -> teamcity.getFinishedBuildsIncludeSnDepFailed(projectId,
branch, cnt));
+ (key, sinceBuildNumber) ->
teamcity.getFinishedBuildsIncludeSnDepFailed(projectId, branch, cnt,
sinceBuildNumber));
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java
index ba8a7d6..41da839 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java
@@ -462,36 +462,32 @@ public class IgniteTeamcityHelper implements ITeamcity {
}
}
+ @SuppressWarnings("WeakerAccess")
@AutoProfiling
- protected <T> T loadXml(Class<T> rootElem, InputStreamReader reader)
throws JAXBException {
+ protected <T> T loadXml(Class<T> rootElem, InputStreamReader reader)
throws JAXBException {
return XmlUtil.load(rootElem, reader);
}
private List<BuildRef> getBuildHistory(@Nullable String buildTypeId,
- @Nullable String branchName,
- boolean dfltFilter,
- @Nullable String state){
-
- return getBuildHistory(buildTypeId, branchName, dfltFilter, state,
null);
- }
-
- private List<BuildRef> getBuildHistory(@Nullable String buildTypeId,
- @Nullable String branchName,
- boolean dfltFilter,
- @Nullable String state,
- @Nullable Long cnt) {
- String btFilter = isNullOrEmpty(buildTypeId) ? "" : ",buildType:" +
buildTypeId + "";
+ @Nullable String branchName,
+ boolean dfltFilter,
+ @Nullable String state,
+ @Nullable Long cnt,
+ @Nullable Integer sinceBuildNumber)
{
+ String btFilter = isNullOrEmpty(buildTypeId) ? "" : ",buildType:" +
buildTypeId;
String stateFilter = isNullOrEmpty(state) ? "" : (",state:" + state);
- String branchFilter = isNullOrEmpty(branchName) ? "" :",branch:" +
branchName;
+ String branchFilter = isNullOrEmpty(branchName) ? "" : ",branch:" +
branchName;
+ String buildNoFilter = sinceBuildNumber == null ? "" :
",sinceBuild:(number:" + sinceBuildNumber + ")";
long cntFilter = cnt == null ? DEFAULT_BUILDS_COUNT : cnt;
return sendGetXmlParseJaxb(host + "app/rest/latest/builds"
- + "?locator="
- + "defaultFilter:" + dfltFilter
- + btFilter
- + stateFilter
- + branchFilter
- + ",count:" + cntFilter, Builds.class).getBuildsNonNull();
+ + "?locator="
+ + "defaultFilter:" + dfltFilter
+ + btFilter
+ + stateFilter
+ + branchFilter
+ + buildNoFilter
+ + ",count:" + cntFilter, Builds.class).getBuildsNonNull();
}
@AutoProfiling
@@ -566,18 +562,22 @@ public class IgniteTeamcityHelper implements ITeamcity {
@Override public List<BuildRef> getFinishedBuilds(String projectId,
String branch) {
- return getFinishedBuilds(projectId, branch, null);
+ return getFinishedBuilds(projectId, branch, null, null);
}
/** {@inheritDoc} */
@Override
@AutoProfiling
public List<BuildRef> getFinishedBuilds(String projectId,
- String branch, Long cnt) {
+ String branch,
+ Long cnt,
+ @Nullable Integer
sinceBuildNumber) {
List<BuildRef> finished = getBuildHistory(projectId,
UrlUtil.escape(branch),
true,
- null, cnt);
+ null,
+ cnt,
+ sinceBuildNumber);
return
finished.stream().filter(BuildRef::isNotCancelled).collect(Collectors.toList());
}
@@ -590,7 +590,7 @@ public class IgniteTeamcityHelper implements ITeamcity {
/** {@inheritDoc} */
@Override
- @AutoProfiling public List<BuildRef>
getFinishedBuildsIncludeSnDepFailed(String projectId, String branch, Long cnt) {
+ @AutoProfiling public List<BuildRef>
getFinishedBuildsIncludeSnDepFailed(String projectId, String branch, Long cnt,
Integer sinceBuildNumber) {
return getBuildsInState(projectId, branch, BuildRef.STATE_FINISHED,
cnt);
}
@@ -614,7 +614,7 @@ public class IgniteTeamcityHelper implements ITeamcity {
List<BuildRef> finished = getBuildHistory(projectId,
UrlUtil.escape(branch),
false,
- state, cnt);
+ state, cnt, null);
return
finished.stream().filter(BuildRef::isNotCancelled).collect(Collectors.toList());
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/IssueDetector.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/IssueDetector.java
index e6b3eb6..eb12138 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/IssueDetector.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/IssueDetector.java
@@ -397,7 +397,7 @@ public class IssueDetector {
1,
backgroundOpsTcHelper,
backgroundOpsCreds,
- executor);
+ executor);
registerIssuesLater(failures, backgroundOpsTcHelper,
backgroundOpsCreds);
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/IgniteTeamcityHelperRunnerExample.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/IgniteTeamcityHelperRunnerExample.java
index 92ff0c8..bdfc34a 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/IgniteTeamcityHelperRunnerExample.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/IgniteTeamcityHelperRunnerExample.java
@@ -19,7 +19,6 @@ package org.apache.ignite.ci.runners;
import com.google.common.base.Throwables;
import java.io.File;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -30,6 +29,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
+
import org.apache.ignite.ci.IgniteTeamcityHelper;
import org.apache.ignite.ci.tcmodel.conf.BuildType;
import org.apache.ignite.ci.tcmodel.conf.bt.BuildTypeFull;
@@ -57,7 +57,7 @@ public class IgniteTeamcityHelperRunnerExample {
if (bt.getName().toLowerCase().contains("pds")
// || bt.getName().toLowerCase().contains("cache")
) {
- int[] ints =
helper.getBuildNumbersFromHistory(bt.getName(), branchNameForHist);
+ int[] ints =
helper.getBuildNumbersFromHistory(bt.getName(), branchNameForHist, null);
List<CompletableFuture<File>> fileFutList =
helper.standardProcessLogs(ints);
List<File> collect = getFuturesResults(fileFutList);
