This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch incremental-build-upload
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/incremental-build-upload by
this push:
new 4ac2525 Incremental builds implemented
4ac2525 is described below
commit 4ac252579d1be88a2f89cac3416071bf840f1976
Author: Dmitriy Pavlov <[email protected]>
AuthorDate: Wed Sep 19 20:08:42 2018 +0300
Incremental builds implemented
---
.../main/java/org/apache/ignite/ci/ITeamcity.java | 16 +++----
.../apache/ignite/ci/IgnitePersistentTeamcity.java | 51 +++++++++++++++++-----
.../org/apache/ignite/ci/IgniteTeamcityHelper.java | 48 ++++++++++----------
.../runners/IgniteTeamcityHelperRunnerExample.java | 4 +-
4 files changed, 72 insertions(+), 47 deletions(-)
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
index 4e9f9d6..ccc3fea 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
@@ -66,16 +66,17 @@ public interface ITeamcity extends AutoCloseable {
* @return list of builds in historical order, recent builds coming last
*/
default List<BuildRef> getFinishedBuilds(String projectId, String branch) {
- return getFinishedBuilds(projectId, branch, null);
+ return getFinishedBuilds(projectId, branch, null, null);
};
/**
* @param projectId suite ID (string without spaces)
* @param branch
* @param cnt builds count
+ * @param sinceBuildNumber
* @return list of builds in historical order, recent builds coming last
*/
- List<BuildRef> getFinishedBuilds(String projectId, String branch, Long
cnt);
+ List<BuildRef> getFinishedBuilds(String projectId, String branch, Long
cnt, Integer sinceBuildNumber);
/**
* Includes snapshot dependencies failed builds into list
@@ -85,7 +86,7 @@ public interface ITeamcity extends AutoCloseable {
* @return list of builds in historical order, recent builds coming last
*/
default List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String
projectId, String branch){
- return getFinishedBuildsIncludeSnDepFailed(projectId, branch, null);
+ return getFinishedBuildsIncludeSnDepFailed(projectId, branch, null,
null);
};
/**
@@ -94,9 +95,10 @@ public interface ITeamcity extends AutoCloseable {
* @param projectId suite ID (string without spaces)
* @param branch branch in TC identification
* @param cnt builds count
+ * @param sinceBuildNumber limit builds export with some build number, not
operational for Persistent connection
* @return list of builds in historical order, recent builds coming last
*/
- List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String projectId,
String branch, Long cnt);
+ List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String projectId,
String branch, Long cnt, Integer sinceBuildNumber);
/** */
CompletableFuture<List<BuildRef>> getRunningBuilds(@Nullable String
branch);
@@ -104,12 +106,8 @@ public interface ITeamcity extends AutoCloseable {
/** */
CompletableFuture<List<BuildRef>> getQueuedBuilds(@Nullable String branch);
- default int[] getBuildNumbersFromHistory(String projectId, String
branchNameForHist) {
- return getBuildNumbersFromHistory(projectId, branchNameForHist, null);
- }
-
default int[] getBuildNumbersFromHistory(String projectId, String
branchNameForHist, Long cnt) {
- return getFinishedBuilds(projectId, branchNameForHist,
cnt).stream().mapToInt(BuildRef::getId).toArray();
+ return getFinishedBuilds(projectId, branchNameForHist, cnt,
null).stream().mapToInt(BuildRef::getId).toArray();
}
Build getBuild(String href);
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
index d3b038d..e9d1150 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
@@ -33,6 +33,7 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
+import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@@ -71,6 +72,8 @@ import org.apache.ignite.ci.util.CacheUpdateUtil;
import org.apache.ignite.ci.util.CollectionUtil;
import org.apache.ignite.ci.util.ObjectInterner;
import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.xml.sax.SAXParseException;
import static org.apache.ignite.ci.BuildChainProcessor.normalizeBranch;
@@ -79,6 +82,9 @@ import static
org.apache.ignite.ci.BuildChainProcessor.normalizeBranch;
*
*/
public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity,
ITeamcity, ITcAnalytics {
+ /** Logger. */
+ private static final Logger logger =
LoggerFactory.getLogger(IgniteTeamcityHelper.class);
+
//V1 caches, 1024 parts
//V2 caches, 32 parts
@@ -267,12 +273,12 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
return loaded;
}
- protected <K> List<BuildRef> loadBuildHistory(IgniteCache<K,
Expirable<List<BuildRef>>> cache,
- int seconds,
- Long cnt,
- K key,
- Function<K,
List<BuildRef>> realLoad) {
- @Nullable Expirable<List<BuildRef>> persistedBuilds =
readBuildHistEntry( cache, (K) key);
+ protected <K> List<BuildRef> loadBuildHistory(IgniteCache<K,
Expirable<List<BuildRef>>> cache,
+ int seconds,
+ Long cnt,
+ K key,
+ BiFunction<K, Integer,
List<BuildRef>> realLoad) {
+ @Nullable Expirable<List<BuildRef>> persistedBuilds =
readBuildHistEntry(cache, (K) key);
if (persistedBuilds != null
&& (persistedBuilds.isAgeLessThanSecs(seconds)
@@ -297,9 +303,25 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
//todo sinceBuild:(number:) // --todo -10 build numbers
+ Integer sinceBuildNumber = null;
+ if(persistedBuilds!=null) {
+ List<BuildRef> prevData = persistedBuilds.getData();
+ if (prevData.size() > 10) {
+ BuildRef buildRef = prevData.get(prevData.size() - 10);
+
+ if (!Strings.isNullOrEmpty(buildRef.buildNumber)) {
+ try {
+ sinceBuildNumber =
Integer.valueOf(buildRef.buildNumber);
+ } catch (NumberFormatException e) {
+ logger.info("", e);
+ }
+ }
+ }
+ }
List<BuildRef> dataFromRest;
try {
- dataFromRest = realLoad.apply(key);
+ dataFromRest = realLoad.apply(key, sinceBuildNumber);
+
}
catch (Exception e) {
if (Throwables.getRootCause(e) instanceof
FileNotFoundException) {
@@ -353,11 +375,15 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
/** {@inheritDoc} */
@AutoProfiling
- @Override public List<BuildRef> getFinishedBuilds(String projectId, String
branch, Long cnt) {
+ @Override public List<BuildRef> getFinishedBuilds(String projectId,
+ String branch,
+ Long cnt,
+ Integer ignored) {
+ //todo may be support sinceBuildNo
final SuiteInBranch suiteInBranch = new SuiteInBranch(projectId,
branch);
List<BuildRef> buildRefs = loadBuildHistory(buildHistCache(), 60, cnt,
suiteInBranch,
- (key) -> teamcity.getFinishedBuilds(projectId, branch, cnt));
+ (key, sinceBuildNumber) -> teamcity.getFinishedBuilds(projectId,
branch, cnt, sinceBuildNumber));
if (cnt == null)
return buildRefs;
@@ -385,11 +411,14 @@ public class IgnitePersistentTeamcity implements
IAnalyticsEnabledTeamcity, ITea
/** {@inheritDoc} */
@AutoProfiling
- @Override public List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String
projectId, String branch, Long cnt) {
+ @Override public List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String
projectId,
+ String
branch,
+ Long
cnt,
+
Integer ignored) {
final SuiteInBranch suiteInBranch = new SuiteInBranch(projectId,
branch);
return loadBuildHistory(buildHistIncFailedCache(), 60, cnt,
suiteInBranch,
- (key) -> teamcity.getFinishedBuildsIncludeSnDepFailed(projectId,
branch, cnt));
+ (key, sinceBuildNumber) ->
teamcity.getFinishedBuildsIncludeSnDepFailed(projectId, branch, cnt,
sinceBuildNumber));
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java
index ba8a7d6..871d9e9 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java
@@ -468,30 +468,25 @@ public class IgniteTeamcityHelper implements ITeamcity {
}
private List<BuildRef> getBuildHistory(@Nullable String buildTypeId,
- @Nullable String branchName,
- boolean dfltFilter,
- @Nullable String state){
-
- return getBuildHistory(buildTypeId, branchName, dfltFilter, state,
null);
- }
-
- private List<BuildRef> getBuildHistory(@Nullable String buildTypeId,
- @Nullable String branchName,
- boolean dfltFilter,
- @Nullable String state,
- @Nullable Long cnt) {
- String btFilter = isNullOrEmpty(buildTypeId) ? "" : ",buildType:" +
buildTypeId + "";
+ @Nullable String branchName,
+ boolean dfltFilter,
+ @Nullable String state,
+ @Nullable Long cnt,
+ @Nullable Integer sinceBuildNumber)
{
+ String btFilter = isNullOrEmpty(buildTypeId) ? "" : ",buildType:" +
buildTypeId;
String stateFilter = isNullOrEmpty(state) ? "" : (",state:" + state);
- String branchFilter = isNullOrEmpty(branchName) ? "" :",branch:" +
branchName;
+ String branchFilter = isNullOrEmpty(branchName) ? "" : ",branch:" +
branchName;
+ String buildNoFilter = sinceBuildNumber == null ? "" :
",sinceBuild:(number:" + sinceBuildNumber + ")";
long cntFilter = cnt == null ? DEFAULT_BUILDS_COUNT : cnt;
return sendGetXmlParseJaxb(host + "app/rest/latest/builds"
- + "?locator="
- + "defaultFilter:" + dfltFilter
- + btFilter
- + stateFilter
- + branchFilter
- + ",count:" + cntFilter, Builds.class).getBuildsNonNull();
+ + "?locator="
+ + "defaultFilter:" + dfltFilter
+ + btFilter
+ + stateFilter
+ + branchFilter
+ + buildNoFilter
+ + ",count:" + cntFilter, Builds.class).getBuildsNonNull();
}
@AutoProfiling
@@ -566,18 +561,21 @@ public class IgniteTeamcityHelper implements ITeamcity {
@Override public List<BuildRef> getFinishedBuilds(String projectId,
String branch) {
- return getFinishedBuilds(projectId, branch, null);
+ return getFinishedBuilds(projectId, branch, null, null);
}
/** {@inheritDoc} */
@Override
@AutoProfiling
public List<BuildRef> getFinishedBuilds(String projectId,
- String branch, Long cnt) {
+ String branch, Long cnt,
+ @Nullable Integer
sinceBuildNumber) {
List<BuildRef> finished = getBuildHistory(projectId,
UrlUtil.escape(branch),
true,
- null, cnt);
+ null,
+ cnt,
+ sinceBuildNumber);
return
finished.stream().filter(BuildRef::isNotCancelled).collect(Collectors.toList());
}
@@ -590,7 +588,7 @@ public class IgniteTeamcityHelper implements ITeamcity {
/** {@inheritDoc} */
@Override
- @AutoProfiling public List<BuildRef>
getFinishedBuildsIncludeSnDepFailed(String projectId, String branch, Long cnt) {
+ @AutoProfiling public List<BuildRef>
getFinishedBuildsIncludeSnDepFailed(String projectId, String branch, Long cnt,
Integer sinceBuildNumber) {
return getBuildsInState(projectId, branch, BuildRef.STATE_FINISHED,
cnt);
}
@@ -614,7 +612,7 @@ public class IgniteTeamcityHelper implements ITeamcity {
List<BuildRef> finished = getBuildHistory(projectId,
UrlUtil.escape(branch),
false,
- state, cnt);
+ state, cnt, null);
return
finished.stream().filter(BuildRef::isNotCancelled).collect(Collectors.toList());
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/IgniteTeamcityHelperRunnerExample.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/IgniteTeamcityHelperRunnerExample.java
index 92ff0c8..bdfc34a 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/IgniteTeamcityHelperRunnerExample.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/IgniteTeamcityHelperRunnerExample.java
@@ -19,7 +19,6 @@ package org.apache.ignite.ci.runners;
import com.google.common.base.Throwables;
import java.io.File;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -30,6 +29,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
+
import org.apache.ignite.ci.IgniteTeamcityHelper;
import org.apache.ignite.ci.tcmodel.conf.BuildType;
import org.apache.ignite.ci.tcmodel.conf.bt.BuildTypeFull;
@@ -57,7 +57,7 @@ public class IgniteTeamcityHelperRunnerExample {
if (bt.getName().toLowerCase().contains("pds")
// || bt.getName().toLowerCase().contains("cache")
) {
- int[] ints =
helper.getBuildNumbersFromHistory(bt.getName(), branchNameForHist);
+ int[] ints =
helper.getBuildNumbersFromHistory(bt.getName(), branchNameForHist, null);
List<CompletableFuture<File>> fileFutList =
helper.standardProcessLogs(ints);
List<File> collect = getFuturesResults(fileFutList);
