This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-10095
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-10095 by this push:
new ae1d17c Saving reverse.dep.*.env.JAVA_HOME implemented as parameter
coming from suites config
ae1d17c is described below
commit ae1d17c6bd082208895787c86154a8511080bf08
Author: Dmitriy Pavlov <[email protected]>
AuthorDate: Wed May 8 19:23:05 2019 +0300
Saving reverse.dep.*.env.JAVA_HOME implemented as parameter coming from
suites config
---
.../ignite/ci/tcbot/conf/ChainAtServerTracked.java | 34 ++++++---
.../ignited/buildtype/ParametersCompacted.java | 30 ++++++++
.../ignited/fatbuild/FatBuildCompacted.java | 17 +++--
.../ignited/fatbuild/ProactiveFatBuildSync.java | 6 +-
.../ignited/fatbuild/PropertiesCompacted.java | 89 ----------------------
.../teamcity/ignited/fatbuild/TestCompacted.java | 22 +++++-
.../ci/teamcity/ignited/runhist/Invocation.java | 14 ++++
.../ci/teamcity/ignited/runhist/RunHistSync.java | 32 ++++++--
.../ci/teamcity/ignited/TeamcityIgnitedMock.java | 2 +-
9 files changed, 123 insertions(+), 123 deletions(-)
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/ChainAtServerTracked.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/ChainAtServerTracked.java
index 92088b3..82d3570 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/ChainAtServerTracked.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/ChainAtServerTracked.java
@@ -18,13 +18,16 @@
package org.apache.ignite.ci.tcbot.conf;
import com.google.common.base.Strings;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
+import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
+import org.jetbrains.annotations.NotNull;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.isNullOrEmpty;
@@ -113,25 +116,32 @@ public class ChainAtServerTracked extends ChainAtServer {
/**
* @return Map with parameter values for current run.
*/
- public Map<String, Object> buildParameters() {
- Map<String, Object> values = new HashMap<>();
+ @NotNull public Map<String, Object> buildParameters() {
+ if (triggerParameters == null || triggerParameters.isEmpty())
+ return Collections.emptyMap();
- if (triggerParameters != null) {
+ Map<String, Object> values = new HashMap<>();
- triggerParameters.forEach(
- p -> {
- String name = p.name();
- Object val = p.generateValue();
+ triggerParameters.forEach(
+ p -> {
+ String name = p.name();
+ Object val = p.generateValue();
- if (!Strings.isNullOrEmpty(name) && val != null)
- values.put(name, val);
- }
- );
- }
+ if (!Strings.isNullOrEmpty(name) && val != null)
+ values.put(name, val);
+ }
+ );
return values;
}
+ public Stream<String> buildParametersKeys() {
+ if (triggerParameters == null || triggerParameters.isEmpty())
+ return Stream.empty();
+
+ return triggerParameters.stream().map(BuildParameter::name);
+ }
+
/** */
public String branchForRest() {
return branchForRest;
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildtype/ParametersCompacted.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildtype/ParametersCompacted.java
index dd7180f..212bbb9 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildtype/ParametersCompacted.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildtype/ParametersCompacted.java
@@ -22,15 +22,25 @@ import com.google.common.base.Strings;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
+import java.util.function.BiConsumer;
+import org.apache.ignite.ci.db.Persisted;
import org.apache.ignite.ci.tcmodel.conf.bt.Parameters;
import org.apache.ignite.ci.tcmodel.conf.bt.Property;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.internal.util.GridIntList;
+/**
+ * Properties (Build/Build Type parameters) compacted value for storing in TC
Bot DB
+ */
+@Persisted
public class ParametersCompacted {
+ /** Property Keys (Names), int value is coming from the compatcor. */
private GridIntList keys;
+
+ /** Property Values, int value is coming from the compatcor. */
private GridIntList values;
+
/**
* Default constructor.
*/
@@ -124,4 +134,24 @@ public class ParametersCompacted {
.add("values", values)
.toString();
}
+
+
+ public void forEach(IStringCompactor compactor, BiConsumer<String, String>
consumer) {
+ forEach((k,v)-> consumer.accept(compactor.getStringFromId(k),
compactor.getStringFromId(v)));
+ }
+
+ public void forEach(BiConsumer<Integer, Integer> consumer) {
+ int size = keys.size();
+
+ for (int i = 0; i < size; i++) {
+ int nameid = keys.get(i);
+
+ if (i >= values.size())
+ break;
+
+ int valId = values.get(i);
+
+ consumer.accept(nameid, valId);
+ }
+ }
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index 4b44d92..7bfa146 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -48,6 +48,7 @@ import org.apache.ignite.ci.tcmodel.vcs.Revisions;
import org.apache.ignite.ci.tcmodel.vcs.VcsRootInstance;
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.apache.ignite.ci.teamcity.ignited.buildtype.ParametersCompacted;
import org.apache.ignite.ci.teamcity.ignited.change.RevisionCompacted;
import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
@@ -121,7 +122,7 @@ public class FatBuildCompacted extends BuildRefCompacted
implements IVersionedEn
@Nullable private RevisionCompacted revisions[];
/** Build parameters compacted, excluding dynamic parameters. */
- @Nullable private PropertiesCompacted parameters;
+ @Nullable private ParametersCompacted buildParameters;
/** {@inheritDoc} */
@Override public int version() {
@@ -223,7 +224,7 @@ public class FatBuildCompacted extends BuildRefCompacted
implements IVersionedEn
.filter(prop ->
!ITeamcity.AVOID_SAVE_PROPERTIES.contains(prop.name())).collect(Collectors.toList());
if (!propList.isEmpty())
- this.parameters = new PropertiesCompacted(compactor, propList);
+ this.buildParameters = new ParametersCompacted(compactor,
propList);
}
}
@@ -341,10 +342,10 @@ public class FatBuildCompacted extends BuildRefCompacted
implements IVersionedEn
}).collect(Collectors.toList()));
}
- if (parameters != null) {
+ if (buildParameters != null) {
List<Property> props = new ArrayList<>();
- parameters.forEach(compactor, (k, v) -> props.add(new Property(k,
v)));
+ buildParameters.forEach(compactor, (k, v) -> props.add(new
Property(k, v)));
res.parameters(new Parameters(props));
}
@@ -444,12 +445,12 @@ public class FatBuildCompacted extends BuildRefCompacted
implements IVersionedEn
Arrays.equals(changesIds, that.changesIds) &&
Objects.equals(triggered, that.triggered) &&
Arrays.equals(revisions, that.revisions) &&
- Objects.equals(parameters, that.parameters);
+ Objects.equals(buildParameters, that.buildParameters);
}
/** {@inheritDoc} */
@Override public int hashCode() {
- int res = Objects.hash(super.hashCode(), _ver, startDate, finishDate,
queuedDate, projectId, name, tests, flags, problems, statistics, triggered,
parameters);
+ int res = Objects.hash(super.hashCode(), _ver, startDate, finishDate,
queuedDate, projectId, name, tests, flags, problems, statistics, triggered,
buildParameters);
res = 31 * res + Arrays.hashCode(snapshotDeps);
res = 31 * res + Arrays.hashCode(changesIds);
res = 31 * res + Arrays.hashCode(revisions);
@@ -665,7 +666,7 @@ public class FatBuildCompacted extends BuildRefCompacted
implements IVersionedEn
return Collections.unmodifiableList(Arrays.asList(revisions));
}
- @Nullable public PropertiesCompacted parameters() {
- return parameters;
+ @Nullable public ParametersCompacted parameters() {
+ return buildParameters;
}
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
index 94a1970..073c9c3 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -308,12 +308,12 @@ public class ProactiveFatBuildSync {
if (savedVer.isFakeStub())
refCompacted.setId(buildId); //to provide possiblity to save the
build
- final String srvName = conn.serverId();
- final int srvIdMask = ITeamcityIgnited.serverIdToInt(srvName);
+ final String srvCode = conn.serverId();
+ final int srvIdMask = ITeamcityIgnited.serverIdToInt(srvCode);
buildRefDao.save(srvIdMask, refCompacted);
- runHistSync.saveToHistoryLater(srvName, savedVer);
+ runHistSync.saveToHistoryLater(srvCode, savedVer);
return savedVer;
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/PropertiesCompacted.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/PropertiesCompacted.java
deleted file mode 100644
index e81096c..0000000
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/PropertiesCompacted.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ci.teamcity.ignited.fatbuild;
-
-import com.google.common.base.Strings;
-import java.util.List;
-import java.util.function.BiConsumer;
-import org.apache.ignite.ci.db.Persisted;
-import org.apache.ignite.ci.tcmodel.conf.bt.Property;
-import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
-import org.apache.ignite.internal.util.GridIntList;
-
-/**
- * Properties (Build parameters) compacted value for storing in TC Bot DB
- */
-@Persisted
-public class PropertiesCompacted {
- /** Property Keys (Names), int value is coming from the compatcor. */
- private GridIntList keys;
-
- /** Property Values, int value is coming from the compatcor. */
- private GridIntList values;
-
- public PropertiesCompacted() {
- }
-
- public PropertiesCompacted(IStringCompactor compactor, List<Property>
props) {
- int size = props.size();
- keys = new GridIntList(size);
- values = new GridIntList(size);
-
- for (Property next : props) {
- String name = next.name();
- if (Strings.isNullOrEmpty(name))
- continue;
-
- String valStr = next.value();
-
- keys.add(compactor.getStringId(name));
- values.add(compactor.getStringId(valStr));
- }
- }
-
- private long findPropertyValue(int propCode) {
- if (keys == null)
- return -1L;
-
- int size = keys.size();
-
- for (int i = 0; i < size; i++) {
- int nameid = keys.get(i);
-
- if (nameid == propCode)
- return i < values.size() ? values.get(i) : -1;
- }
-
- return -1L;
- }
-
- public void forEach(IStringCompactor compactor, BiConsumer<String, String>
consumer) {
- int size = keys.size();
-
- for (int i = 0; i < size; i++) {
- int nameid = keys.get(i);
-
- if (i >= values.size())
- break;
-
- int valId = values.get(i);
-
- consumer.accept(compactor.getStringFromId(nameid),
compactor.getStringFromId(valId));
- }
- }
-}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
index 62415cb..11a2502 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
@@ -24,6 +24,8 @@ import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.BitSet;
import java.util.Objects;
+import java.util.TreeMap;
+import java.util.function.BiPredicate;
import org.apache.ignite.ci.analysis.RunStat;
import org.apache.ignite.ci.tcbot.common.StringFieldCompacted;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
@@ -364,7 +366,15 @@ public class TestCompacted {
return null;
}
- public Invocation toInvocation(IStringCompactor compactor,
FatBuildCompacted build) {
+ /**
+ * @param compactor
+ * @param build
+ * @param paramsFilter parameters filter to find out parameters to be
saved in RunHistory (for future filtering).
+ * @return
+ */
+ public Invocation toInvocation(IStringCompactor compactor,
+ FatBuildCompacted build,
+ BiPredicate<Integer, Integer> paramsFilter) {
final boolean failedTest = isFailedTest(compactor);
final int failCode = failedTest
@@ -378,7 +388,13 @@ public class TestCompacted {
.withStartDate(build.getStartDateTs())
.withChanges(build.changes());
- return invocation;
- // .withParameters(build.parameters().forEach(compactor,
(k,v)->{}););
+ java.util.Map<Integer, Integer> importantParms = new TreeMap<>();
+
+ build.parameters().forEach((k, v) -> {
+ if (paramsFilter.test(k, v))
+ importantParms.put(k, v);
+ });
+
+ return invocation.withParameters(importantParms);
}
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/Invocation.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/Invocation.java
index 05e858f..f3fbbe0 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/Invocation.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/Invocation.java
@@ -20,6 +20,8 @@ package org.apache.ignite.ci.teamcity.ignited.runhist;
import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
+import java.util.Map;
+import javax.annotation.Nullable;
import org.apache.ignite.ci.db.Persisted;
/**
@@ -48,6 +50,9 @@ public class Invocation {
/** Build Start date as timestamp. */
private long startDate;
+ /** Additional (important) build Pareters, which can be used for
filtering. */
+ @Nullable private Map<Integer, Integer> parms;
+
/**
* Creates invocation.
* @param buildId Build id.
@@ -133,4 +138,13 @@ public class Invocation {
@Override public int hashCode() {
return Objects.hashCode(buildId, status, changePresent, startDate);
}
+
+ public Invocation withParameters(Map<Integer, Integer> parms) {
+ if (parms == null || parms.isEmpty())
+ return this;
+
+ this.parms = parms;
+
+ return this;
+ }
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
index eb40e36..aecd618 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
@@ -35,6 +36,8 @@ import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.di.MonitoredTask;
import org.apache.ignite.ci.di.scheduler.IScheduler;
import org.apache.ignite.ci.tcbot.TcBotSystemProperties;
+import org.apache.ignite.ci.tcbot.conf.ChainAtServerTracked;
+import org.apache.ignite.ci.tcbot.conf.ITcBotConfig;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
import org.apache.ignite.ci.teamcity.ignited.buildref.BuildRefDao;
@@ -69,6 +72,8 @@ public class RunHistSync {
/** Build DAO. */
@Inject private FatBuildDao fatBuildDao;
+ @Inject private ITcBotConfig cfg;
+
/** Build to save to history. */
@GuardedBy("this")
private final Map<String, SyncTask> buildToSave = new HashMap<>();
@@ -89,17 +94,19 @@ public class RunHistSync {
}
/**
- * @param srvVame Server id.
+ * @param srvCode Server code (internal identification).
* @param build Build.
*/
- public void saveToHistoryLater(String srvVame, FatBuildCompacted build) {
+ public void saveToHistoryLater(String srvCode, FatBuildCompacted build) {
if (!validForStatistics(build))
return;
- int srvId = ITeamcityIgnited.serverIdToInt(srvVame);
+ int srvId = ITeamcityIgnited.serverIdToInt(srvCode);
if (histDao.buildWasProcessed(srvId, build.id()))
return;
+ Set<Integer> allTriggeringBuildParameters =
getSuitePossibleParameters(srvCode);
+
boolean saveNow = false;
int branchNameNormalized =
compactor.getStringId(normalizeBranch(build.branchName(compactor)));
@@ -109,7 +116,7 @@ public class RunHistSync {
build.getAllTests().forEach(t -> {
RunHistKey histKey = new RunHistKey(srvId, t.testName(),
branchNameNormalized);
List<Invocation> list = testInvMap.computeIfAbsent(histKey, k ->
new ArrayList<>());
- list.add(t.toInvocation(compactor, build));
+ list.add(t.toInvocation(compactor, build, (k, v) ->
allTriggeringBuildParameters.contains(k)));
cntTests.incrementAndGet();
});
@@ -120,7 +127,7 @@ public class RunHistSync {
int cnt = cntTests.get();
synchronized (this) {
- final SyncTask syncTask = buildToSave.computeIfAbsent(srvVame, s
-> new SyncTask());
+ final SyncTask syncTask = buildToSave.computeIfAbsent(srvCode, s
-> new SyncTask());
if (syncTask.sheduledTestsCnt() + cnt <= MAX_TESTS_QUEUE)
syncTask.addLater(testInvMap, cnt, buildInvKey, buildInv);
@@ -138,11 +145,22 @@ public class RunHistSync {
else {
int ldrToActivate =
ThreadLocalRandom.current().nextInt(HIST_LDR_TASKS) + 1;
- scheduler.sheduleNamed(taskName("saveBuildToHistory." +
ldrToActivate, srvVame),
- () -> saveBuildToHistory(srvVame, ldrToActivate), 1,
TimeUnit.MINUTES);
+ scheduler.sheduleNamed(taskName("saveBuildToHistory." +
ldrToActivate, srvCode),
+ () -> saveBuildToHistory(srvCode, ldrToActivate), 1,
TimeUnit.MINUTES);
}
}
+ @NotNull public Set<Integer> getSuitePossibleParameters(String srvCode) {
+ return cfg.getTrackedBranches().getBranches().stream().flatMap(
+ b -> b.getChainsStream()
+ .filter(ChainAtServerTracked::isTriggerBuild)
+ .filter(chain -> Objects.equals(chain.getServerId(),
srvCode))
+ .flatMap(ChainAtServerTracked::buildParametersKeys)
+ )
+ .map(k -> compactor.getStringId(k))
+ .collect(Collectors.toSet());
+ }
+
@MonitoredTask(name = "Save Builds To History(srv, runner)",
nameExtArgsIndexes = {0, 1})
@SuppressWarnings("WeakerAccess")
protected String saveBuildToHistory(String srvName, int ldrToActivate) {
diff --git
a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
index 5cf9604..2354722 100644
---
a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
+++
b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
@@ -134,7 +134,7 @@ public class TeamcityIgnitedMock {
final RunHistCompacted hist =
histCache.computeIfAbsent(histKey, RunHistCompacted::new);
- Invocation inv = testCompacted.toInvocation(c, build);
+ Invocation inv = testCompacted.toInvocation(c, build, (k, v)
-> true);
hist.addInvocation(inv);
});
