This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 3c3927e69e6c1f8a6dd5af539d1550617d1de535 Author: Dmitrii Ryabov <[email protected]> AuthorDate: Wed Sep 12 18:22:20 2018 +0300 IGNITE-9377 Handle print crit failures from RunAll to the JIRA ticket - Fixes #6. --- .../java/org/apache/ignite/ci/HelperConfig.java | 22 +++ .../main/java/org/apache/ignite/ci/ITcHelper.java | 6 + .../main/java/org/apache/ignite/ci/ITeamcity.java | 28 ++- .../apache/ignite/ci/IgnitePersistentTeamcity.java | 36 ++-- .../org/apache/ignite/ci/IgniteTeamcityHelper.java | 93 ++++++---- .../main/java/org/apache/ignite/ci/TcHelper.java | 13 ++ .../org/apache/ignite/ci/github/PullRequest.java | 2 +- .../org/apache/ignite/ci/observer/BuildInfo.java | 81 ++++++++ .../apache/ignite/ci/observer/BuildObserver.java | 63 +++++++ .../apache/ignite/ci/observer/ObserverTask.java | 204 +++++++++++++++++++++ .../apache/ignite/ci/tcmodel/hist/BuildRef.java | 4 +- .../java/org/apache/ignite/ci/util/HttpUtil.java | 42 ++++- .../java/org/apache/ignite/ci/util/XmlUtil.java | 26 +++ .../ignite/ci/web/model/current/TestFailure.java | 12 ++ .../ignite/ci/web/model/current/UpdateInfo.java | 6 + .../apache/ignite/ci/web/rest/TriggerBuild.java | 44 ++++- .../ignite/ci/web/rest/pr/GetPrTestFailures.java | 49 +++-- ignite-tc-helper-web/src/main/webapp/index.html | 55 ++++++ 18 files changed, 708 insertions(+), 78 deletions(-) diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/HelperConfig.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/HelperConfig.java index b3b51c7..37880c1 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/HelperConfig.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/HelperConfig.java @@ -48,7 +48,14 @@ public class HelperConfig { @Deprecated private static final String PASSWORD = "password"; public static final String ENCODED_PASSWORD = "encoded_password"; + + /** GitHub authorization token property name. */ public static final String GITHUB_AUTH_TOKEN = "github.auth_token"; + + /** JIRA authorization token property name. */ + public static final String JIRA_AUTH_TOKEN = "jira.auth_token"; + + /** Slack authorization token property name. */ public static final String SLACK_AUTH_TOKEN = "slack.auth_token"; public static final String SLACK_CHANNEL = "slack.channel"; public static final String LOGS = "logs"; @@ -136,6 +143,21 @@ public class HelperConfig { } /** + * Extract JIRA basic authorization token from properties. + * + * @param props Properties, where token is placed. + * @return Null or decoded auth token for Github. + */ + @Nullable static String prepareJiraHttpAuthToken(Properties props) { + String pwd = props.getProperty(JIRA_AUTH_TOKEN); + + if (isNullOrEmpty(pwd)) + return null; + + return pwd; + } + + /** * Extract TeamCity authorization token from properties. * * @param props Properties, where token is placed. diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITcHelper.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITcHelper.java index b2beb51..69df43d 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITcHelper.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITcHelper.java @@ -18,6 +18,7 @@ package org.apache.ignite.ci; import java.util.List; +import org.apache.ignite.ci.observer.BuildObserver; import org.apache.ignite.ci.issue.IssueDetector; import org.apache.ignite.ci.issue.IssuesStorage; import org.apache.ignite.ci.user.ICredentialsProv; @@ -42,6 +43,11 @@ public interface ITcHelper { IssueDetector issueDetector(); + /** + * @return Build observer. + */ + BuildObserver buildObserver(); + IAnalyticsEnabledTeamcity server(String srvId, @Nullable ICredentialsProv prov); ITcAnalytics tcAnalytics(String serverId); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java index 7e105c7..8201f5c 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java @@ -239,7 +239,12 @@ public interface ITeamcity extends AutoCloseable { * @param cleanRebuild Rebuild all dependencies. * @param queueAtTop Put at the top of the build queue. */ - void triggerBuild(String id, String name, boolean cleanRebuild, boolean queueAtTop); + Build triggerBuild(String id, String name, boolean cleanRebuild, boolean queueAtTop); + + /** + * @param tok TeamCity authorization token. + */ + void setAuthToken(String tok); /** * @return {@code True} if TeamCity authorization token is available. @@ -247,9 +252,9 @@ public interface ITeamcity extends AutoCloseable { boolean isTeamCityTokenAvailable(); /** - * @param token TeamCity authorization token. + * @param token GitHub authorization token. */ - void setAuthToken(String token); + void setGitToken(String token); /** * @return {@code True} if GitHub authorization token is available. @@ -257,9 +262,14 @@ public interface ITeamcity extends AutoCloseable { boolean isGitTokenAvailable(); /** - * @param token GitHub authorization token. + * @param tok Jira authorization token. */ - void setGitToken(String token); + void setJiraToken(String tok); + + /** + * @return {@code True} if JIRA authorization token is available. + */ + boolean isJiraTokenAvailable(); /** * Send POST request with given body. @@ -276,6 +286,14 @@ public interface ITeamcity extends AutoCloseable { */ PullRequest getPullRequest(String branch); + /** + * @param ticket JIRA ticket name. + * @param comment Comment to be placed in the ticket conversation. + * @return {@code True} if ticket was succesfully commented. Otherwise - {@code false}. + */ + boolean commentJiraTicket(String ticket, String comment); + + default void setAuthData(String user, String password) { setAuthToken( Base64Util.encodeUtf8String(user + ":" + password)); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java index 16e3a22..1e10d78 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java @@ -77,7 +77,6 @@ import static org.apache.ignite.ci.BuildChainProcessor.normalizeBranch; * */ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITeamcity, ITcAnalytics { - //V1 caches, 1024 parts //V2 caches, 32 parts @@ -817,10 +816,15 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea } /** {@inheritDoc} */ - @Override public void triggerBuild(String id, String name, boolean cleanRebuild, boolean queueAtTop) { + @Override public Build triggerBuild(String id, String name, boolean cleanRebuild, boolean queueAtTop) { lastTriggerMs = System.currentTimeMillis(); - teamcity.triggerBuild(id, name, cleanRebuild, queueAtTop); + return teamcity.triggerBuild(id, name, cleanRebuild, queueAtTop); + } + + /** {@inheritDoc} */ + @Override public void setAuthToken(String tok) { + teamcity.setAuthToken(tok); } /** {@inheritDoc} */ @@ -829,8 +833,8 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea } /** {@inheritDoc} */ - @Override public void setAuthToken(String token) { - teamcity.setAuthToken(token); + @Override public void setGitToken(String tok) { + teamcity.setGitToken(tok); } /** {@inheritDoc} */ @@ -839,18 +843,28 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea } /** {@inheritDoc} */ - @Override public void setGitToken(String token) { - teamcity.setGitToken(token); + @Override public void setJiraToken(String tok) { + teamcity.setJiraToken(tok); } /** {@inheritDoc} */ - @Override public boolean notifyGit(String url, String body) { - return teamcity.notifyGit(url, body); + @Override public boolean isJiraTokenAvailable() { + return teamcity.isJiraTokenAvailable(); + } + + /** {@inheritDoc} */ + @Override public boolean commentJiraTicket(String ticket, String comment) { + return teamcity.commentJiraTicket(ticket, comment); } /** {@inheritDoc} */ - @Override public PullRequest getPullRequest(String branch) { - return teamcity.getPullRequest(branch); + @Override public PullRequest getPullRequest(String branchForTc) { + return teamcity.getPullRequest(branchForTc); + } + + /** {@inheritDoc} */ + @Override public boolean notifyGit(String url, String body) { + return teamcity.notifyGit(url, body); } /** {@inheritDoc} */ diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java index 0813e2d..3bbc78e 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java @@ -27,6 +27,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.StringReader; import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.List; @@ -83,6 +84,7 @@ import org.slf4j.LoggerFactory; import static com.google.common.base.Strings.isNullOrEmpty; import static java.util.concurrent.CompletableFuture.supplyAsync; import static org.apache.ignite.ci.HelperConfig.ensureDirExist; +import static org.apache.ignite.ci.util.XmlUtil.xmlEscapeText; /** * Class for access to Teamcity REST API without any caching. @@ -106,6 +108,9 @@ public class IgniteTeamcityHelper implements ITeamcity { /** GitHub authorization token. */ private String gitAuthTok; + /** JIRA authorization token. */ + private String jiraBasicAuthTok; + private final String configName; //main properties file name private final String tcName; @@ -131,6 +136,8 @@ public class IgniteTeamcityHelper implements ITeamcity { setGitToken(HelperConfig.prepareGithubHttpAuthToken(props)); + setJiraToken(HelperConfig.prepareJiraHttpAuthToken(props)); + final File logsDirFile = HelperConfig.resolveLogs(workDir, props); logsDir = ensureDirExist(logsDirFile); @@ -139,13 +146,18 @@ public class IgniteTeamcityHelper implements ITeamcity { } /** {@inheritDoc} */ + @Override public void setAuthToken(String tok) { + basicAuthTok = tok; + } + + /** {@inheritDoc} */ @Override public boolean isTeamCityTokenAvailable() { return basicAuthTok != null; } /** {@inheritDoc} */ - @Override public void setAuthToken(String token) { - basicAuthTok = token; + @Override public void setGitToken(String tok) { + gitAuthTok = tok; } /** {@inheritDoc} */ @@ -154,33 +166,40 @@ public class IgniteTeamcityHelper implements ITeamcity { } /** {@inheritDoc} */ - @Override public void setGitToken(String token) { - gitAuthTok = token; + @Override public void setJiraToken(String tok) { + jiraBasicAuthTok = tok; } /** {@inheritDoc} */ - @Override public boolean notifyGit(String url, String body) { + @Override public boolean isJiraTokenAvailable() { + return jiraBasicAuthTok != null; + } + + /** {@inheritDoc} */ + @Override public boolean commentJiraTicket(String ticket, String comment) { try { - HttpUtil.sendPostAsStringToGit(gitAuthTok, url, body); + String url = "https://issues.apache.org/jira/rest/api/2/issue/"; + ticket + "/comment"; + + HttpUtil.sendPostAsStringToJira(jiraBasicAuthTok, url, "{\"body\": \"" + comment + "\"}"); return true; } catch (IOException e) { - logger.error("Failed to notify Git [errMsg="+e.getMessage()+']'); + logger.error("Failed to notify JIRA [errMsg="+e.getMessage()+']'); return false; } } /** {@inheritDoc} */ - @Override public PullRequest getPullRequest(String branch) { + @Override public PullRequest getPullRequest(String branchForTc) { String id = null; - for (int i = 5; i < branch.length(); i++) { - char c = branch.charAt(i); + for (int i = 5; i < branchForTc.length(); i++) { + char c = branchForTc.charAt(i); if (!Character.isDigit(c)) { - id = branch.substring(5, i); + id = branchForTc.substring(5, i); break; } @@ -199,6 +218,20 @@ public class IgniteTeamcityHelper implements ITeamcity { } /** {@inheritDoc} */ + @Override public boolean notifyGit(String url, String body) { + try { + HttpUtil.sendPostAsStringToGit(gitAuthTok, url, body); + + return true; + } + catch (IOException e) { + logger.error("Failed to notify Git [errMsg="+e.getMessage()+']'); + + return false; + } + } + + /** {@inheritDoc} */ @Override public List<Agent> agents(boolean connected, boolean authorized) { String url = "app/rest/agents?locator=connected:" + connected + ",authorized:" + authorized; @@ -259,31 +292,13 @@ public class IgniteTeamcityHelper implements ITeamcity { .thenApply(LogCheckTask::getResult); } - /** - * @param t Text to process. - */ - private String xmlEscapeText(CharSequence t) { - StringBuilder sb = new StringBuilder(); - for(int i = 0; i < t.length(); i++){ - char c = t.charAt(i); - switch(c){ - case '<': sb.append("<"); break; - case '>': sb.append(">"); break; - case '\"': sb.append("""); break; - case '&': sb.append("&"); break; - case '\'': sb.append("'"); break; - default: - if(c>0x7e) - sb.append("&#").append((int)c).append(";"); - else - sb.append(c); - } - } - return sb.toString(); - } - /** {@inheritDoc} */ - @Override public void triggerBuild(String buildTypeId, String branchName, boolean cleanRebuild, boolean queueAtTop) { + @Override public Build triggerBuild( + String buildTypeId, + String branchName, + boolean cleanRebuild, + boolean queueAtTop + ) { String triggeringOptions = " <triggeringOptions" + " cleanSources=\"" + cleanRebuild + "\"" + @@ -307,11 +322,17 @@ public class IgniteTeamcityHelper implements ITeamcity { "</build>"; String url = host + "app/rest/buildQueue"; + try { logger.info("Triggering build: buildTypeId={}, branchName={}, cleanRebuild={}, queueAtTop={}", buildTypeId, branchName, cleanRebuild, queueAtTop); - HttpUtil.sendPostAsString(basicAuthTok, url, param); + try (StringReader reader = new StringReader(HttpUtil.sendPostAsString(basicAuthTok, url, param))) { + return XmlUtil.load(Build.class, reader); + } + catch (JAXBException e) { + throw Throwables.propagate(e); + } } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/TcHelper.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/TcHelper.java index 1f6bc0d..a95483b 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/TcHelper.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/TcHelper.java @@ -23,6 +23,7 @@ import com.google.common.cache.CacheBuilder; import java.util.List; import org.apache.ignite.Ignite; import org.apache.ignite.ci.conf.BranchesTracked; +import org.apache.ignite.ci.observer.BuildObserver; import org.apache.ignite.ci.issue.IssueDetector; import org.apache.ignite.ci.issue.IssuesStorage; import org.apache.ignite.ci.user.ICredentialsProv; @@ -56,6 +57,10 @@ public class TcHelper implements ITcHelper { private TcUpdatePool tcUpdatePool = new TcUpdatePool(); private IssuesStorage issuesStorage; private IssueDetector detector; + + /** Build observer. */ + private BuildObserver buildObserver; + private UserAndSessionsStorage userAndSessionsStorage; public TcHelper(Ignite ignite) { @@ -65,6 +70,7 @@ public class TcHelper implements ITcHelper { userAndSessionsStorage = new UserAndSessionsStorage(ignite); detector = new IssueDetector(ignite, issuesStorage, userAndSessionsStorage); + buildObserver = new BuildObserver(this); } /** {@inheritDoc} */ @@ -78,6 +84,11 @@ public class TcHelper implements ITcHelper { } /** {@inheritDoc} */ + @Override public BuildObserver buildObserver() { + return buildObserver; + } + + /** {@inheritDoc} */ @Override public IAnalyticsEnabledTeamcity server(String srvId, @Nullable ICredentialsProv prov) { if (stop.get()) throw new IllegalStateException("Shutdown"); @@ -150,6 +161,8 @@ public class TcHelper implements ITcHelper { tcUpdatePool.stop(); detector.stop(); + + buildObserver.stop(); } public ExecutorService getService() { diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/PullRequest.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/PullRequest.java index d381d12..43a6b83 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/PullRequest.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/PullRequest.java @@ -32,7 +32,7 @@ public class PullRequest { /** Pull Request title. */ private String title; - /** Pull Request url to get statuses. */ + /** Pull Request statuses URL. */ @SerializedName("statuses_url") private String statusesUrl; /** diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/BuildInfo.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/BuildInfo.java new file mode 100644 index 0000000..445cd06 --- /dev/null +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/BuildInfo.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ci.observer; + +import org.apache.ignite.ci.tcmodel.result.Build; +import org.apache.ignite.ci.user.ICredentialsProv; + +/** + * + */ +class BuildInfo { + /** Build. */ + final Build build; + + /** Server id. */ + final String srvId; + + /** */ + final ICredentialsProv prov; + + /** JIRA ticket name. */ + final String ticket; + + /** + * @param build Build. + * @param srvId Server id. + * @param prov Credentials. + * @param ticket JIRA ticket name. + */ + BuildInfo(Build build, String srvId, ICredentialsProv prov, String ticket) { + this.build = build; + this.srvId = srvId; + this.prov = prov; + this.ticket = ticket; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + BuildInfo info = (BuildInfo)o; + + if (!build.equals(info.build)) + return false; + if (!srvId.equals(info.srvId)) + return false; + if (!prov.equals(info.prov)) + return false; + + return ticket.equals(info.ticket); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + int res = build.hashCode(); + + res = 31 * res + srvId.hashCode(); + res = 31 * res + prov.hashCode(); + res = 31 * res + ticket.hashCode(); + + return res; + } +} diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/BuildObserver.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/BuildObserver.java new file mode 100644 index 0000000..a7a7b1b --- /dev/null +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/BuildObserver.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ci.observer; + +import java.util.Timer; +import org.apache.ignite.ci.ITcHelper; +import org.apache.ignite.ci.tcmodel.result.Build; +import org.apache.ignite.ci.user.ICredentialsProv; + +/** + * + */ +public class BuildObserver { + /** Time between observing actions in milliseconds. */ + private final long period = 30 * 60 * 1_000; + + /** Timer. */ + private final Timer timer; + + /** Task, which should be done periodically. */ + private final ObserverTask task; + + /** + * @param helper Helper. + */ + public BuildObserver(ITcHelper helper) { + timer = new Timer(); + + timer.schedule(task = new ObserverTask(helper), period, period); + } + + /** + * Stop observer. + */ + public void stop() { + timer.cancel(); + } + + /** + * @param build Build id. + * @param srvId Server id. + * @param prov Credentials. + * @param ticket JIRA ticket name. + */ + public void observe(Build build, String srvId, ICredentialsProv prov, String ticket) { + task.builds.add(new BuildInfo(build, srvId, prov, ticket)); + } +} diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/ObserverTask.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/ObserverTask.java new file mode 100644 index 0000000..7466a30 --- /dev/null +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/ObserverTask.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ci.observer; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.TimerTask; +import java.util.concurrent.ConcurrentLinkedQueue; +import org.apache.ignite.ci.IAnalyticsEnabledTeamcity; +import org.apache.ignite.ci.ITcHelper; +import org.apache.ignite.ci.tcmodel.result.Build; +import org.apache.ignite.ci.web.model.current.ChainAtServerCurrentStatus; +import org.apache.ignite.ci.web.model.current.SuiteCurrentStatus; +import org.apache.ignite.ci.web.model.current.TestFailure; +import org.apache.ignite.ci.web.model.current.TestFailuresSummary; +import org.apache.ignite.ci.web.model.hist.FailureSummary; +import org.apache.ignite.ci.web.rest.pr.GetPrTestFailures; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.ignite.ci.analysis.RunStat.MAX_LATEST_RUNS; +import static org.apache.ignite.ci.util.XmlUtil.xmlEscapeText; + +/** + * Checks observed builds for finished status and comments JIRA ticket. + */ +public class ObserverTask extends TimerTask { + /** Logger. */ + private static final Logger logger = LoggerFactory.getLogger(ObserverTask.class); + + /** Helper. */ + private final ITcHelper helper; + + /** Builds. */ + final Queue<BuildInfo> builds; + + /** + * @param helper Helper. + */ + ObserverTask(ITcHelper helper) { + this.helper = helper; + builds = new ConcurrentLinkedQueue<>(); + } + + /** {@inheritDoc} */ + @Override public void run() { + for (BuildInfo info : builds) { + IAnalyticsEnabledTeamcity teamcity = helper.server(info.srvId, info.prov); + Build build = teamcity.getBuild(info.build.getId()); + String comment; + + try { + comment = generateComment(build, info); + } + catch (RuntimeException e) { + logger.error("Exception happened during generating comment for JIRA " + + "[build=" + build.getId() + ", errMsg=" + e.getMessage() + ']'); + + continue; + } + + if (build.state.equals("finished")) { + if (teamcity.commentJiraTicket(info.ticket, comment)) + builds.remove(info); + } + } + } + + /** + * @param build Build. + * @param info Info. + */ + private String generateComment(Build build, BuildInfo info) { + StringBuilder res = new StringBuilder(); + TestFailuresSummary summary = GetPrTestFailures.getTestFailuresSummary( + helper, info.prov, info.srvId, build.getBuildType().getId(), build.branchName, "Latest", null); + + if (summary != null) { + for (ChainAtServerCurrentStatus server : summary.servers) { + if (!server.serverName().equals("apache")) + continue; + + Map<String, List<SuiteCurrentStatus>> fails = findFailures(server); + + for (List<SuiteCurrentStatus> suites : fails.values()) { + for (SuiteCurrentStatus suite : suites) { + res.append("{color:#d04437}").append(suite.name).append("{color}"); + res.append(" [[tests ").append(suite.failedTests); + + if (suite.result != null && !suite.result.equals("")) + res.append(' ').append(suite.result); + + res.append('|').append(suite.webToBuild).append("]]\\n"); + + for (TestFailure failure : suite.testFailures) { + res.append("* "); + + if (failure.suiteName != null && failure.testName != null) + res.append(failure.suiteName).append(": ").append(failure.testName); + else + res.append(failure.name); + + FailureSummary recent = failure.histBaseBranch.recent; + + if (recent != null) { + if (recent.failureRate != null) { + res.append(" - ").append(recent.failureRate).append("% fails in last ") + .append(MAX_LATEST_RUNS).append(" master runs."); + } + else if (recent.failures != null && recent.runs != null) { + res.append(" - ").append(recent.failures).append(" fails / ") + .append(recent.runs).append(" runs."); + } + } + + res.append("\\n"); + } + + res.append("\\n"); + } + } + + if (res.length() > 0) { + res.insert(0, "{panel:title=Possible Blockers|" + + "borderStyle=dashed|borderColor=#ccc|titleBGColor=#F7D6C1}\\n") + .append("{panel}"); + } + else { + res.append("{panel:title=No blockers found!|" + + "borderStyle=dashed|borderColor=#ccc|titleBGColor=#D6F7C1}{panel}"); + } + } + } + + res.append("\\n").append("[TeamCity Run All|").append(build.webUrl).append(']'); + + return xmlEscapeText(res.toString()); + } + + /** + * @param srv Server. + * @return Failures for given server. + */ + private Map<String, List<SuiteCurrentStatus>> findFailures(ChainAtServerCurrentStatus srv) { + Map<String, List<SuiteCurrentStatus>> fails = new LinkedHashMap<>(); + + fails.put("compilation", new ArrayList<>()); + fails.put("timeout", new ArrayList<>()); + fails.put("exit code", new ArrayList<>()); + fails.put("failed tests", new ArrayList<>()); + + for (SuiteCurrentStatus suite : srv.suites) { + String suiteRes = suite.result.toLowerCase(); + String failType = null; + + if (suiteRes.contains("compilation")) + failType = "compilation"; + + if (suiteRes.contains("timeout")) + failType = "timeout"; + + if (suiteRes.contains("exit code")) + failType = "exit code"; + + if (failType == null) { + List<TestFailure> failures = new ArrayList<>(); + + for (TestFailure testFailure : suite.testFailures) { + if (testFailure.isNewFailedTest()) + failures.add(testFailure); + } + + if (!failures.isEmpty()) { + suite.testFailures = failures; + + failType = "failed tests"; + } + } + + if (failType != null) + fails.get(failType).add(suite); + } + + return fails; + } +} diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/hist/BuildRef.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/hist/BuildRef.java index 0e6202b..2305fc7 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/hist/BuildRef.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcmodel/hist/BuildRef.java @@ -44,7 +44,8 @@ public class BuildRef extends AbstractRef { public static final String STATE_QUEUED = "queued"; - @XmlAttribute private String state; + /** Current state of build. */ + @XmlAttribute public String state; @XmlAttribute(name = "number") public String buildNumber; @@ -52,6 +53,7 @@ public class BuildRef extends AbstractRef { @XmlAttribute public Boolean composite; + /** Build page URL. */ @XmlAttribute public String webUrl; /** diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/HttpUtil.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/HttpUtil.java index 8234f8a..45f249b 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/HttpUtil.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/HttpUtil.java @@ -20,6 +20,7 @@ package org.apache.ignite.ci.util; import com.google.common.base.Stopwatch; import java.io.BufferedReader; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -81,18 +82,18 @@ public class HttpUtil { /** * Send GET request to the GitHub url. * - * @param githubAuthToken Authorization token. + * @param githubAuthTok Authorization OAuth token. * @param url URL. * @return Input stream from connection. * @throws IOException If failed. */ - public static InputStream sendGetToGit(String githubAuthToken, String url) throws IOException { + public static InputStream sendGetToGit(String githubAuthTok, String url) throws IOException { Stopwatch started = Stopwatch.createStarted(); URL obj = new URL(url); HttpURLConnection con = (HttpURLConnection)obj.openConnection(); con.setRequestProperty("accept-charset", StandardCharsets.UTF_8.toString()); - con.setRequestProperty("Authorization", "token " + githubAuthToken); + con.setRequestProperty("Authorization", "token " + githubAuthTok); con.setRequestProperty("Connection", "Keep-Alive"); con.setRequestProperty("Keep-Alive", "header"); @@ -193,4 +194,39 @@ public class HttpUtil { return readIsToString(inputStream); } } + + /** + * Send POST request to the GitHub url. + * + * @param jiraAuthTok Authorization Base64 token. + * @param url URL. + * @param body Request POST params. + * @return Response body from given url. + * @throws IOException If failed. + */ + public static String sendPostAsStringToJira(String jiraAuthTok, String url, String body) throws IOException { + URL obj = new URL(url); + HttpURLConnection con = (HttpURLConnection)obj.openConnection(); + Charset charset = StandardCharsets.UTF_8; + + con.setRequestProperty("accept-charset", charset.toString()); + con.setRequestProperty("Authorization", "Basic " + jiraAuthTok); + con.setRequestProperty("content-type", "application/json"); + con.setRequestProperty("Connection", "Keep-Alive"); + con.setRequestProperty("Keep-Alive", "header"); + + con.setRequestMethod("POST"); + + con.setDoOutput(true); + + try (OutputStreamWriter writer = new OutputStreamWriter(con.getOutputStream(), charset)){ + writer.write(body); // Write POST query string (if any needed). + } + + logger.info("\nSending 'POST' request to URL : " + url + "\n" + body); + + try (InputStream inputStream = getInputStream(con)){ + return readIsToString(inputStream); + } + } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/XmlUtil.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/XmlUtil.java index c636e0b..5eb1510 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/XmlUtil.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/XmlUtil.java @@ -48,4 +48,30 @@ public class XmlUtil { return unmarshal; } + + /** + * @param t Text to process. + */ + public static String xmlEscapeText(CharSequence t) { + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < t.length(); i++) { + char c = t.charAt(i); + + switch(c){ + case '<': sb.append("<"); break; + case '>': sb.append(">"); break; + case '\"': sb.append("""); break; + case '&': sb.append("&"); break; + case '\'': sb.append("'"); break; + default: + if (c>0x7e) + sb.append("&#").append((int)c).append(";"); + else + sb.append(c); + } + } + + return sb.toString(); + } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java index 795cc63..9325f60 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java @@ -242,6 +242,18 @@ import static org.apache.ignite.ci.web.model.current.SuiteCurrentStatus.branchFo } } + /** + * @return {@code True} if this failure is appeared in the current branch. + */ + public boolean isNewFailedTest() { + FailureSummary recent = histBaseBranch.recent; + + boolean lowFailureRate = recent != null && recent.failureRate != null && + Float.valueOf(recent.failureRate.replace(',', '.')) < 4.; + + return lowFailureRate && flakyComments == null; + } + /** {@inheritDoc} */ @Override public boolean equals(Object o) { if (this == o) diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/UpdateInfo.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/UpdateInfo.java index 50b71f7..59e64ec 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/UpdateInfo.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/UpdateInfo.java @@ -29,6 +29,9 @@ import org.apache.ignite.ci.IAnalyticsEnabledTeamcity; /** GitHub auth token availability flag. */ public static int GITHUB_FLAG = 2; + /** JIRA auth token availability flag. */ + public static int JIRA_FLAG = 2; + /** Flags to use in javascript. */ public Integer javaFlags = 0; @@ -59,5 +62,8 @@ import org.apache.ignite.ci.IAnalyticsEnabledTeamcity; if (teamcity.isGitTokenAvailable()) javaFlags = javaFlags | GITHUB_FLAG; + + if (teamcity.isJiraTokenAvailable()) + javaFlags = javaFlags | JIRA_FLAG; } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/TriggerBuild.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/TriggerBuild.java index 4986f3d..2148a35 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/TriggerBuild.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/TriggerBuild.java @@ -27,7 +27,10 @@ import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import org.apache.ignite.ci.ITcHelper; import org.apache.ignite.ci.ITeamcity; +import org.apache.ignite.ci.github.PullRequest; +import org.apache.ignite.ci.tcmodel.result.Build; import org.apache.ignite.ci.user.ICredentialsProv; import org.apache.ignite.ci.web.CtxListener; import org.apache.ignite.ci.web.rest.login.ServiceUnauthorizedException; @@ -47,19 +50,44 @@ public class TriggerBuild { @GET @Path("trigger") public SimpleResult triggerBuild( - @Nullable @QueryParam("serverId") String serverId, + @Nullable @QueryParam("serverId") String srvId, @Nullable @QueryParam("branchName") String branchName, @Nullable @QueryParam("suiteId") String suiteId, - @Nullable @QueryParam("top") Boolean top) { - + @Nullable @QueryParam("top") Boolean top, + @Nullable @QueryParam("observe") Boolean observe + ) { final ICredentialsProv prov = ICredentialsProv.get(req); - if(!prov.hasAccess(serverId)) { - throw ServiceUnauthorizedException.noCreds(serverId); - } + if (!prov.hasAccess(srvId)) + throw ServiceUnauthorizedException.noCreds(srvId); - try (final ITeamcity helper = CtxListener.getTcHelper(context).server(serverId, prov)) { - helper.triggerBuild(suiteId, branchName, false, top != null && top); + ITcHelper helper = CtxListener.getTcHelper(context); + + try (final ITeamcity teamcity = helper.server(srvId, prov)) { + PullRequest pr = teamcity.getPullRequest(branchName); + + String ticketId = ""; + + if (pr.getTitle().startsWith("IGNITE-")) { + int beginIdx = 7; + int endIdx = 7; + + while (endIdx < pr.getTitle().length() && Character.isDigit(pr.getTitle().charAt(endIdx))) + endIdx++; + + ticketId = pr.getTitle().substring(beginIdx, endIdx); + } + + if (ticketId.equals("")) + return new SimpleResult("PR title \"" + pr.getTitle() + "\" should starts with \"IGNITE-XXXX\"." + + " Please, rename PR according to the" + + " <a href='https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute"; + + "#HowtoContribute-1.CreateGitHubpull-request'>contributing guide</a>."); + + Build build = teamcity.triggerBuild(suiteId, branchName, false, top != null && top); + + if (observe != null && observe) + helper.buildObserver().observe(build, srvId, prov, "ignite-" + ticketId); } return new SimpleResult("OK"); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/pr/GetPrTestFailures.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/pr/GetPrTestFailures.java index 12344b4..eba2b58 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/pr/GetPrTestFailures.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/pr/GetPrTestFailures.java @@ -103,22 +103,44 @@ public class GetPrTestFailures { @Nonnull @QueryParam("action") String action, @Nullable @QueryParam("count") Integer count) { - final TestFailuresSummary res = new TestFailuresSummary(); - final AtomicInteger runningUpdates = new AtomicInteger(); - final ITcHelper tcHelper = CtxListener.getTcHelper(context); final ICredentialsProv creds = ICredentialsProv.get(req); + return getTestFailuresSummary(tcHelper, creds, srvId, suiteId, branchForTc, action, count); + } + + /** + * @param helper Helper. + * @param creds Credentials. + * @param srvId Server id. + * @param suiteId Suite id. + * @param branchForTc Branch name. + * @param act Action. + * @param cnt Count. + * @return Test failures summary. + */ + public static TestFailuresSummary getTestFailuresSummary( + ITcHelper helper, + ICredentialsProv creds, + String srvId, + String suiteId, + String branchForTc, + String act, + Integer cnt + ) { + final TestFailuresSummary res = new TestFailuresSummary(); + final AtomicInteger runningUpdates = new AtomicInteger(); + //using here non persistent TC allows to skip update statistic - try (IAnalyticsEnabledTeamcity teamcity = tcHelper.server(srvId, creds)) { + try (IAnalyticsEnabledTeamcity teamcity = helper.server(srvId, creds)) { res.setJavaFlags(teamcity); LatestRebuildMode rebuild; - if (FullQueryParams.HISTORY.equals(action)) + if (FullQueryParams.HISTORY.equals(act)) rebuild = LatestRebuildMode.ALL; - else if (FullQueryParams.LATEST.equals(action)) + else if (FullQueryParams.LATEST.equals(act)) rebuild = LatestRebuildMode.LATEST; - else if (FullQueryParams.CHAIN.equals(action)) + else if (FullQueryParams.CHAIN.equals(act)) rebuild = LatestRebuildMode.NONE; else rebuild = LatestRebuildMode.LATEST; @@ -129,7 +151,7 @@ public class GetPrTestFailures { long limit; if (rebuild == LatestRebuildMode.ALL) - limit = count == null ? 10 : count; + limit = cnt == null ? 10 : cnt; else limit = 1; @@ -156,9 +178,10 @@ public class GetPrTestFailures { if (ctx.isFakeStub()) chainStatus.setBuildNotFound(true); else { - int cnt = (int)ctx.getRunningUpdates().count(); - if (cnt > 0) - runningUpdates.addAndGet(cnt); + int cnt0 = (int)ctx.getRunningUpdates().count(); + + if (cnt0 > 0) + runningUpdates.addAndGet(cnt0); //fail rate reference is always default (master) chainStatus.initFromContext(teamcity, ctx, teamcity, failRateBranch); @@ -179,8 +202,8 @@ public class GetPrTestFailures { @Nullable @QueryParam("serverId") String srvId, @Nonnull @QueryParam("suiteId") String suiteId, @Nonnull @QueryParam("branchForTc") String branchForTc, - @Nonnull @QueryParam("action") String action, - @Nullable @QueryParam("count") Integer count, + @Nonnull @QueryParam("action") String act, + @Nullable @QueryParam("count") Integer cnt, @Nonnull @FormParam("notifyMsg") String msg) { if (!branchForTc.startsWith("pull/")) return "Given branch is not a pull request. Notify works only for pull requests."; diff --git a/ignite-tc-helper-web/src/main/webapp/index.html b/ignite-tc-helper-web/src/main/webapp/index.html index c642863..481706e 100644 --- a/ignite-tc-helper-web/src/main/webapp/index.html +++ b/ignite-tc-helper-web/src/main/webapp/index.html @@ -9,6 +9,7 @@ <script src="https://code.jquery.com/jquery-1.12.4.js";></script> <script src="https://code.jquery.com/ui/1.12.1/jquery-ui.js";></script> + <script src="js/common-1.6.js"></script> <script> $(document).ready(function() { @@ -21,6 +22,7 @@ $(document).ready(function() { function loadData() { $("#loadStatus").html("<img src='https://www.wallies.com/filebin/images/loading_apple.gif' width=20px height=20px> Please wait"); + $.ajax({ url: "rest/branches/version", success: showVersionInfo, @@ -50,6 +52,7 @@ function loadData() { success: function(result) { $("#loadStatus").html(""); showBuildsOnServers(result); + showRunAllForm(result); }, error: showErrInLoadStatus }); @@ -108,6 +111,55 @@ function showBuildsOnServers(result) { } $("#buildsCheck").html(res); } + +/** + * This form allows user to start TeamCity Run All build. + */ +function showRunAllForm(result) { + var res = ""; + + for (var i = 0; i < result.length; i++) { + var serverId = result[i]; + + res+="Server: <input type='text' name='serverId' value=" + serverId +" readonly>" ; + res+="Pull Request #<input type='text' name='prId' onkeypress='return trigBuild(event)'> "; + res+="<button onclick='trigBuild()'>Run All</button>"; + } + + $("#runAll").html(res); +} + +/** + * Start Run All build on TeamCity and comment in JIRA ticket when build will be finished. + */ +function trigBuild(event) { + if (event != null && event.key != "Enter") + return; + + var fields = document.getElementById("runAll").children; + var url = "rest/build/trigger?suiteId=IgniteTests24Java8_RunAll"; + var prId = null; + + for (let field of fields) { + if (field.name == "serverId") + url += "&serverId=" + field.value; + + if (field.name == "prId") { + url += "&branchName=pull%2F" + field.value + "%2Fhead"; + prId = field.value; + } + } + + url += "&observe=true" + + $.ajax({ + url: url, + success: function(result) { + $("#runAll").html("Run All test build was started for PR #" + prId); + }, + error: showErrInLoadStatus + }); +} </script> </head> <body> @@ -132,6 +184,9 @@ Check build: <br> <div id="buildsCheck"></div> <br> +TeamCity Run All: <br> +<div id="runAll"></div> +<br> <a href="ignval.html">Ignite Log Values pretty-print</a><br>
