This is an automated email from the ASF dual-hosted git repository.
olamy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/maven-dist-tool.git
The following commit(s) were added to refs/heads/master by this push:
new cb20e2b Use Jenkins Json API rather than html scrapping (#161)
cb20e2b is described below
commit cb20e2b2f1415471e1d63299f161aee365d73685
Author: Olivier Lamy <[email protected]>
AuthorDate: Sun Mar 15 19:12:06 2026 +1000
Use Jenkins Json API rather than html scrapping (#161)
* Use Json data for ListMasterJobsReport
Signed-off-by: Olivier Lamy <[email protected]>
* real testing
Signed-off-by: Olivier Lamy <[email protected]>
* limit concurrency call
Signed-off-by: Olivier Lamy <[email protected]>
* fix this withMaven reporting
Signed-off-by: Olivier Lamy <[email protected]>
* Make ListBranchesReport faster
Signed-off-by: Olivier Lamy <[email protected]>
* spotless
Signed-off-by: Olivier Lamy <[email protected]>
* use virtual thread
Signed-off-by: Olivier Lamy <[email protected]>
* upgrade github api to fix incompatibility with jackson
Signed-off-by: Olivier Lamy <[email protected]>
* add needed jenkins repo
Signed-off-by: Olivier Lamy <[email protected]>
* spotless
Signed-off-by: Olivier Lamy <[email protected]>
* add timeout
Signed-off-by: Olivier Lamy <[email protected]>
* we will need some token
Signed-off-by: Olivier Lamy <[email protected]>
* spotless
Signed-off-by: Olivier Lamy <[email protected]>
* Makes MLStats way faster (reactive and parallel)
Signed-off-by: Olivier Lamy <[email protected]>
---------
Signed-off-by: Olivier Lamy <[email protected]>
---
pom.xml | 30 ++++-
.../org/apache/maven/dist/tools/JsonRetry.java | 84 ++++++++++++
.../tools/committers/CommittersStatsReport.java | 25 ++--
.../maven/dist/tools/committers/MLStats.java | 121 ++++++++---------
.../maven/dist/tools/jobs/AbstractJobsReport.java | 4 +
.../tools/jobs/branches/ListBranchesReport.java | 143 ++++++++++++---------
.../tools/jobs/master/ListMasterJobsReport.java | 85 ++++++------
.../dist/tools/memorycheck/MemoryCheckReport.java | 8 +-
.../committers/MavenCommittersRepositoryTest.java | 63 ++++++---
9 files changed, 364 insertions(+), 199 deletions(-)
diff --git a/pom.xml b/pom.xml
index cebeb27..4ba5f08 100644
--- a/pom.xml
+++ b/pom.xml
@@ -152,13 +152,28 @@
<dependency>
<groupId>org.kohsuke</groupId>
<artifactId>github-api</artifactId>
- <version>1.129</version>
+ <version>2.0-rc.5</version>
</dependency>
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
<version>7.5.0.202512021534-r</version>
</dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-client</artifactId>
+ <version>12.1.7</version>
+ </dependency>
+ <dependency>
+ <groupId>io.projectreactor</groupId>
+ <artifactId>reactor-core</artifactId>
+ <version>3.7.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-reactive-httpclient</artifactId>
+ <version>4.1.4</version>
+ </dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
@@ -187,9 +202,9 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.wiremock</groupId>
- <artifactId>wiremock</artifactId>
- <version>3.13.2</version>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>12.1.7</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -199,6 +214,13 @@
<scope>test</scope>
</dependency>
</dependencies>
+ <repositories>
+ <!-- https://github.com/hub4j/github-api/issues/2206 -->
+ <repository>
+ <id>repo.jenkins-ci.org</id>
+ <url>https://repo.jenkins-ci.org/public/</url>
+ </repository>
+ </repositories>
<build>
<pluginManagement>
diff --git a/src/main/java/org/apache/maven/dist/tools/JsonRetry.java
b/src/main/java/org/apache/maven/dist/tools/JsonRetry.java
new file mode 100644
index 0000000..7948a91
--- /dev/null
+++ b/src/main/java/org/apache/maven/dist/tools/JsonRetry.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.maven.dist.tools;
+
+import java.util.concurrent.TimeUnit;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.lang3.StringUtils;
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.Request;
+import org.eclipse.jetty.reactive.client.ReactiveRequest;
+import org.eclipse.jetty.reactive.client.ReactiveResponse;
+import reactor.core.publisher.Mono;
+
+public class JsonRetry {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private HttpClient httpClient = new HttpClient();
+
+ private JsonRetry() {
+ try {
+ this.httpClient.start();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static class JsonRetryHolder {
+ private static final JsonRetry INSTANCE = new JsonRetry();
+ }
+
+ public static JsonRetry getInstance() {
+ return JsonRetryHolder.INSTANCE;
+ }
+
+ public HttpClient getHttpClient() {
+ return httpClient;
+ }
+
+ public static JsonNode get(String url) throws Exception {
+ Request request = getInstance().httpClient.newRequest(url);
+ String apiToken = System.getenv("API_TOKEN");
+ if (StringUtils.isNotBlank(apiToken)) {
+ request.headers(httpFields -> httpFields.add("Authorization",
"Basic " + apiToken));
+ }
+ String json = request.send().getContentAsString();
+ return json != null ? OBJECT_MAPPER.readTree(json) : null;
+ }
+
+ public static Mono<JsonNode> getAsync(String url) {
+ Request request = getInstance().httpClient.newRequest(url).timeout(60,
TimeUnit.SECONDS);
+ String apiToken = System.getenv("API_TOKEN");
+ if (StringUtils.isNotBlank(apiToken)) {
+ request.headers(httpFields -> httpFields.add("Authorization",
"Basic " + apiToken));
+ }
+ ReactiveRequest reactiveRequest =
ReactiveRequest.newBuilder(request).build();
+ return
Mono.from(reactiveRequest.response(ReactiveResponse.Content.asString()))
+ .flatMap(json -> {
+ try {
+ return Mono.justOrEmpty(OBJECT_MAPPER.readTree(json));
+ } catch (Exception e) {
+ return Mono.error(e);
+ }
+ });
+ }
+}
diff --git
a/src/main/java/org/apache/maven/dist/tools/committers/CommittersStatsReport.java
b/src/main/java/org/apache/maven/dist/tools/committers/CommittersStatsReport.java
index 5794e63..1265826 100644
---
a/src/main/java/org/apache/maven/dist/tools/committers/CommittersStatsReport.java
+++
b/src/main/java/org/apache/maven/dist/tools/committers/CommittersStatsReport.java
@@ -38,9 +38,11 @@ import
org.apache.maven.dist.tools.committers.MavenCommittersRepository.Committe
import org.apache.maven.doxia.sink.Sink;
import org.apache.maven.doxia.sink.impl.SinkEventAttributeSet.Semantics;
import org.apache.maven.plugins.annotations.Mojo;
+import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.reporting.AbstractMavenReport;
import org.apache.maven.reporting.AbstractMavenReportRenderer;
import org.apache.maven.reporting.MavenReportException;
+import reactor.core.publisher.Flux;
/**
* Generate a Committers statistic
@@ -52,6 +54,9 @@ public class CommittersStatsReport extends
AbstractMavenReport {
public static final int LAST_ACTIVITY_MONTHS_WARNING = 2 * 12;
+ @Parameter(defaultValue = "4", property =
"dist-tool.committers.concurrency")
+ private int concurrency;
+
private final Map<String, MLStats> mlStats;
private final MavenCommittersRepository mavenCommitters;
@@ -110,14 +115,18 @@ public class CommittersStatsReport extends
AbstractMavenReport {
}
private Map<Committer, List<String>> retrieveCommitterStats() {
- Map<Committer, List<String>> result = new LinkedHashMap<>();
- for (Committer committer : mavenCommitters.getCommitters()) {
- List<String> lastDateList = mlStats.values().stream()
- .map(ml -> ml.getLast(committer))
- .toList();
- result.put(committer, lastDateList);
- }
- return result;
+ List<Committer> committers = new
ArrayList<>(mavenCommitters.getCommitters());
+ List<MLStats> statsList = new ArrayList<>(mlStats.values());
+
+ return Flux.fromIterable(committers)
+ .flatMapSequential(
+ committer -> Flux.fromIterable(statsList)
+ .flatMapSequential(ml ->
ml.getLastAsync(committer))
+ .collectList()
+ .map(dates -> Map.entry(committer, dates)),
+ concurrency)
+ .collectMap(Map.Entry::getKey, Map.Entry::getValue,
LinkedHashMap::new)
+ .block();
}
private void renderStatsTable(Map<Committer, List<String>>
committerStats) {
diff --git a/src/main/java/org/apache/maven/dist/tools/committers/MLStats.java
b/src/main/java/org/apache/maven/dist/tools/committers/MLStats.java
index ede54c7..b3d4ea9 100644
--- a/src/main/java/org/apache/maven/dist/tools/committers/MLStats.java
+++ b/src/main/java/org/apache/maven/dist/tools/committers/MLStats.java
@@ -18,29 +18,29 @@
*/
package org.apache.maven.dist.tools.committers;
-import java.io.IOException;
-import java.io.InputStream;
import java.net.URI;
import java.net.URLEncoder;
-import java.net.http.HttpClient;
-import java.net.http.HttpRequest;
-import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
-import java.time.Duration;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.maven.dist.tools.JsonRetry;
import
org.apache.maven.dist.tools.committers.MavenCommittersRepository.Committer;
import org.apache.maven.doxia.sink.Sink;
+import org.eclipse.jetty.client.Request;
+import org.eclipse.jetty.reactive.client.ReactiveRequest;
+import org.eclipse.jetty.reactive.client.ReactiveResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
import static java.util.Map.entry;
@@ -48,6 +48,8 @@ public abstract class MLStats {
private final Logger log = LoggerFactory.getLogger(MLStats.class);
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
private static final String ML_STATS_ADDRES =
"https://lists.apache.org/api/stats.lua";;
private static final Map<String, String> STANDARD_QUERY_PARAMS =
Map.ofEntries(
@@ -70,72 +72,55 @@ public abstract class MLStats {
sink.text(" and header_from " + (name ? "committer name" :
"<committerId>@apache.org"));
}
- public String getLast(Committer committer) {
-
- List<Map<String, String>> queryParamsList =
getQueryParamsList(committer);
- return queryParamsList.stream()
+ public Mono<String> getLastAsync(Committer committer) {
+ List<URI> uris = getQueryParamsList(committer).stream()
.map(this::prepareStatsURI)
- .map(this::getLastFromML)
- .filter(Optional::isPresent)
- .map(Optional::get)
- .max(Comparator.naturalOrder())
- .orElse("-");
+ .toList();
+
+ return Flux.fromIterable(uris)
+ .flatMapSequential(uri -> {
+ Request request = JsonRetry.getInstance()
+ .getHttpClient()
+ .newRequest(uri.toString())
+ .headers(f -> f.add("Accept", "application/json"))
+ .timeout(60, TimeUnit.SECONDS);
+ ReactiveRequest reactiveRequest =
+ ReactiveRequest.newBuilder(request).build();
+ return
Mono.from(reactiveRequest.response(ReactiveResponse.Content.asString()))
+ .flatMap(json -> {
+ try {
+ Optional<String> last =
parseLastFromNode(OBJECT_MAPPER.readTree(json));
+ log.info("Query: {}, returns: {}", uri,
last);
+ return Mono.justOrEmpty(last);
+ } catch (Exception e) {
+ return Mono.error(e);
+ }
+ })
+ .onErrorResume(e -> {
+ log.warn("Query: {}, error: {}", uri,
e.getMessage());
+ return Mono.empty();
+ });
+ })
+ .collect(Collectors.maxBy(Comparator.naturalOrder()))
+ .map(opt -> opt.orElse("-"));
}
- private Optional<String> getLastFromML(URI statsURI) {
- try (HttpClient client = HttpClient.newHttpClient()) {
- HttpRequest request = HttpRequest.newBuilder()
- .GET()
- .header("Accept", "application/json")
- .uri(statsURI)
- .timeout(Duration.ofSeconds(60))
- .build();
- HttpResponse<InputStream> response = client.send(request,
HttpResponse.BodyHandlers.ofInputStream());
- Optional<String> last = parseLast(response.body());
- log.info("Query: {}, returns: {}", statsURI, last);
- return last;
-
- } catch (IOException e) {
- log.warn("Query: {}, error: {}", statsURI, e.getMessage());
- // try next one ...
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new RuntimeException(e);
- }
- return Optional.empty();
+ public String getLast(Committer committer) {
+ return getLastAsync(committer).block();
}
- private Optional<String> parseLast(InputStream input) throws IOException {
- JsonFactory factory = new JsonFactory();
- Integer lastYear = null;
- Integer lastMonth = null;
-
- try (JsonParser parser = factory.createParser(input)) {
- while ((lastYear == null || lastMonth == null) &&
parser.nextToken() != JsonToken.END_OBJECT) {
- if (parser.currentToken() != JsonToken.VALUE_NUMBER_INT) {
- continue;
- }
- String name = parser.currentName();
- switch (name) {
- case "lastYear":
- lastYear = parser.getValueAsInt();
- break;
- case "lastMonth":
- lastMonth = parser.getValueAsInt();
- break;
- default:
- // ignore
- }
- }
+ private Optional<String> parseLastFromNode(JsonNode node) {
+ JsonNode lastYearNode = node.get("lastYear");
+ JsonNode lastMonthNode = node.get("lastMonth");
+ if (lastYearNode == null || lastMonthNode == null) {
+ return Optional.empty();
}
-
- if (lastYear != null && lastMonth != null) {
- if (lastYear == 1970 && lastMonth == 1) {
- return Optional.empty();
- }
- return Optional.of(String.format("%04d-%02d", lastYear,
lastMonth));
+ int lastYear = lastYearNode.asInt();
+ int lastMonth = lastMonthNode.asInt();
+ if (lastYear == 1970 && lastMonth == 1) {
+ return Optional.empty();
}
- return Optional.empty();
+ return Optional.of(String.format("%04d-%02d", lastYear, lastMonth));
}
private URI prepareStatsURI(Map<String, String> queryParams) {
diff --git
a/src/main/java/org/apache/maven/dist/tools/jobs/AbstractJobsReport.java
b/src/main/java/org/apache/maven/dist/tools/jobs/AbstractJobsReport.java
index 3b5bd9d..eb7e0c6 100644
--- a/src/main/java/org/apache/maven/dist/tools/jobs/AbstractJobsReport.java
+++ b/src/main/java/org/apache/maven/dist/tools/jobs/AbstractJobsReport.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.stream.Collectors;
import org.apache.maven.dist.tools.JsoupRetry;
+import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.reporting.AbstractMavenReport;
import org.apache.maven.reporting.MavenReportException;
import org.jsoup.nodes.Document;
@@ -35,6 +36,9 @@ import org.jsoup.select.Elements;
public abstract class AbstractJobsReport extends AbstractMavenReport {
protected static final String GITBOX_URL =
"https://gitbox.apache.org/repos/asf";;
+ @Parameter(defaultValue = "8", property = "dist-tool.jobs.concurrency")
+ protected int concurrency;
+
protected static final String MAVENBOX_JOBS_BASE_URL =
"https://ci-maven.apache.org/job/Maven/job/maven-box/job/";;
private static final Collection<String> EXCLUDED = Arrays.asList(
diff --git
a/src/main/java/org/apache/maven/dist/tools/jobs/branches/ListBranchesReport.java
b/src/main/java/org/apache/maven/dist/tools/jobs/branches/ListBranchesReport.java
index 631a158..c749f4b 100644
---
a/src/main/java/org/apache/maven/dist/tools/jobs/branches/ListBranchesReport.java
+++
b/src/main/java/org/apache/maven/dist/tools/jobs/branches/ListBranchesReport.java
@@ -21,7 +21,6 @@ package org.apache.maven.dist.tools.jobs.branches;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
-import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
@@ -29,10 +28,12 @@ import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
-import org.apache.maven.dist.tools.JsoupRetry;
+import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.maven.dist.tools.JsonRetry;
import org.apache.maven.dist.tools.jobs.AbstractJobsReport;
import org.apache.maven.doxia.sink.Sink;
import org.apache.maven.doxia.sink.SinkEventAttributes;
@@ -42,7 +43,10 @@ import org.apache.maven.reporting.MavenReportException;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.Ref;
-import org.jsoup.nodes.Document;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
+import reactor.core.scheduler.Scheduler;
+import reactor.core.scheduler.Schedulers;
/**
* Generate report with build status of the Jenkins job for the master branch
of every Git repository in
@@ -183,66 +187,87 @@ public class ListBranchesReport extends
AbstractJobsReport {
protected void executeReport(Locale locale) throws MavenReportException {
Collection<String> repositoryNames = repositoryNames();
- List<Result> repoStatus = new ArrayList<>(repositoryNames.size());
-
- for (String repository : repositoryNames) {
- getLog().info("processing " + repository);
- final String repositoryJobUrl = MAVENBOX_JOBS_BASE_URL +
repository;
-
- try {
- Document jenkinsBranchesDoc = JsoupRetry.get(repositoryJobUrl);
- Result result = new Result(repository, repositoryJobUrl);
- int masterBranchesGit = 0;
- int masterBranchesJenkins = 0;
- Collection<String> jiraBranchesGit = new ArrayList<>();
- Collection<String> jiraBranchesJenkins = new ArrayList<>();
- Collection<String> dependabotBranchesGit = new ArrayList<>();
- Collection<String> dependabotBranchesJenkins = new
ArrayList<>();
- Collection<String> restGit = new ArrayList<>();
- Collection<String> restJenkins = new ArrayList<>();
-
- for (String branch : getBranches(repository)) {
- if ("master".equals(branch)) {
- masterBranchesGit++;
-
- if (jenkinsBranchesDoc.getElementById("job_master") !=
null) {
- masterBranchesJenkins++;
- }
- } else if (JIRAPROJECTS.containsKey(repository)
- &&
branch.toUpperCase().startsWith(JIRAPROJECTS.get(repository) + '-')) {
- jiraBranchesGit.add(branch);
- if
(jenkinsBranchesDoc.getElementById(URLEncoder.encode("job_" + branch, "UTF-8"))
!= null) {
- jiraBranchesJenkins.add(branch);
- }
- } else if (branch.startsWith("dependabot/")) {
- dependabotBranchesGit.add(branch);
- if
(jenkinsBranchesDoc.getElementById(URLEncoder.encode("job_" + branch, "UTF-8"))
!= null) {
- dependabotBranchesJenkins.add(branch);
- }
- } else {
- restGit.add(branch);
- if
(jenkinsBranchesDoc.getElementById(URLEncoder.encode("job_" + branch, "UTF-8"))
!= null) {
- restJenkins.add(branch);
- }
+ List<Result> repoStatus = Flux.fromIterable(repositoryNames)
+ .flatMap(
+ repo -> fetchResult(repo).onErrorResume(e -> {
+ getLog().warn("Failed to read status for " + repo
+ " Jenkins job " + MAVENBOX_JOBS_BASE_URL
+ + repo);
+ return Mono.empty();
+ }),
+ concurrency)
+ .collectList()
+ .block();
+
+ generateReport(repoStatus);
+ }
+
+ private Mono<Result> fetchResult(String repository) {
+ String repositoryJobUrl = MAVENBOX_JOBS_BASE_URL + repository +
"/api/json?tree=jobs[name]";
+
+ Mono<JsonNode> jenkinsMono = JsonRetry.getAsync(repositoryJobUrl);
+ Scheduler scheduler =
Schedulers.fromExecutorService(Executors.newVirtualThreadPerTaskExecutor());
+ Mono<Collection<String>> branchesMono =
+ Mono.fromCallable(() ->
getBranches(repository)).subscribeOn(scheduler);
+
+ return Mono.zip(jenkinsMono, branchesMono).map(tuple -> {
+ JsonNode jenkinsBranchesDoc = tuple.getT1();
+ Collection<String> gitBranches = tuple.getT2();
+
+ Result result = new Result(repository, repositoryJobUrl);
+ int masterBranchesGit = 0;
+ int masterBranchesJenkins = 0;
+ Collection<String> jiraBranchesGit = new ArrayList<>();
+ Collection<String> jiraBranchesJenkins = new ArrayList<>();
+ Collection<String> dependabotBranchesGit = new ArrayList<>();
+ Collection<String> dependabotBranchesJenkins = new ArrayList<>();
+ Collection<String> restGit = new ArrayList<>();
+ Collection<String> restJenkins = new ArrayList<>();
+
+ for (String branch : gitBranches) {
+ if ("master".equals(branch)) {
+ masterBranchesGit++;
+
+ if
(jenkinsBranchesDoc.get("jobs").valueStream().anyMatch(n -> n.get("name")
+ .asText()
+ .equals("master"))) {
+ masterBranchesJenkins++;
+ }
+ } else if (JIRAPROJECTS.containsKey(repository)
+ &&
branch.toUpperCase().startsWith(JIRAPROJECTS.get(repository) + '-')) {
+ jiraBranchesGit.add(branch);
+ if
(jenkinsBranchesDoc.get("jobs").valueStream().anyMatch(n -> n.get("name")
+ .asText()
+ .equals(branch))) {
+ jiraBranchesJenkins.add(branch);
+ }
+ } else if (branch.startsWith("dependabot/")) {
+ dependabotBranchesGit.add(branch);
+ if
(jenkinsBranchesDoc.get("jobs").valueStream().anyMatch(n -> n.get("name")
+ .asText()
+ .equals(branch))) {
+ dependabotBranchesJenkins.add(branch);
+ }
+ } else {
+ restGit.add(branch);
+ if
(jenkinsBranchesDoc.get("jobs").valueStream().anyMatch(n -> n.get("name")
+ .asText()
+ .equals(branch))) {
+ restJenkins.add(branch);
}
}
-
- result.setMasterBranchesGit(masterBranchesGit);
- result.setMasterBranchesJenkins(masterBranchesJenkins);
- result.setJiraBranchesGit(jiraBranchesGit);
- result.setJiraBranchesJenkins(jiraBranchesJenkins);
- result.setDependabotBranchesGit(dependabotBranchesGit);
- result.setDependabotBranchesJenkins(dependabotBranchesJenkins);
- result.setRestGit(restGit);
- result.setRestJenkins(restJenkins);
-
- repoStatus.add(result);
- } catch (IOException | GitAPIException e) {
- getLog().warn("Failed to read status for " + repository + "
Jenkins job " + repositoryJobUrl);
}
- }
- generateReport(repoStatus);
+ result.setMasterBranchesGit(masterBranchesGit);
+ result.setMasterBranchesJenkins(masterBranchesJenkins);
+ result.setJiraBranchesGit(jiraBranchesGit);
+ result.setJiraBranchesJenkins(jiraBranchesJenkins);
+ result.setDependabotBranchesGit(dependabotBranchesGit);
+ result.setDependabotBranchesJenkins(dependabotBranchesJenkins);
+ result.setRestGit(restGit);
+ result.setRestJenkins(restJenkins);
+
+ return result;
+ });
}
private String getGitHubBranchesUrl(String repository) {
diff --git
a/src/main/java/org/apache/maven/dist/tools/jobs/master/ListMasterJobsReport.java
b/src/main/java/org/apache/maven/dist/tools/jobs/master/ListMasterJobsReport.java
index c53cc30..92b9b17 100644
---
a/src/main/java/org/apache/maven/dist/tools/jobs/master/ListMasterJobsReport.java
+++
b/src/main/java/org/apache/maven/dist/tools/jobs/master/ListMasterJobsReport.java
@@ -18,10 +18,8 @@
*/
package org.apache.maven.dist.tools.jobs.master;
-import java.io.IOException;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
@@ -30,13 +28,15 @@ import java.util.Locale;
import java.util.Map;
import java.util.stream.Collectors;
-import org.apache.maven.dist.tools.JsoupRetry;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.maven.dist.tools.JsonRetry;
import org.apache.maven.dist.tools.jobs.AbstractJobsReport;
import org.apache.maven.doxia.sink.Sink;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.reporting.MavenReportException;
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
/**
* Generate report with build status of the Jenkins job for the master branch
of every Git repository in
@@ -76,46 +76,49 @@ public class ListMasterJobsReport extends
AbstractJobsReport {
protected void executeReport(Locale locale) throws MavenReportException {
Collection<String> repositoryNames = repositoryNames();
- List<Result> repoStatus = new ArrayList<>(repositoryNames.size());
-
- for (String repository : repositoryNames) {
- final String repositoryJobUrl = MAVENBOX_JOBS_BASE_URL +
repository;
-
- try {
- Document doc = JsoupRetry.get(repositoryJobUrl);
-
- Result result = new Result(repository, repositoryJobUrl);
-
- Element masterRow = doc.getElementById("job_master");
- if (masterRow == null) {
- getLog().warn(MAVENBOX_JOBS_BASE_URL + repository + " is
missing id job_master");
- continue;
- } else if (masterRow.hasClass("job-status-red") ||
masterRow.hasClass("job-status-red-anime")) {
- result.setStatus("FAILURE");
- } else if (masterRow.hasClass("job-status-yellow") ||
masterRow.hasClass("job-status-yellow-anime")) {
- result.setStatus("UNSTABLE");
- } else if (masterRow.hasClass("job-status-blue") ||
masterRow.hasClass("job-status-blue-anime")) {
- result.setStatus("SUCCESS");
- } else {
- result.setStatus("UNKNOWN");
- }
- result.setIcon(masterRow
- .select("span.build-status-icon__wrapper")
- .first()
- .outerHtml());
-
- result.setLastBuild(getLastBuild(
- masterRow.child(3).attr("data"),
masterRow.child(4).attr("data")));
-
- repoStatus.add(result);
- } catch (IOException e) {
- getLog().warn("Failed to read status for " + repository + "
Jenkins job " + repositoryJobUrl);
- }
- }
+ List<Result> repoStatus = Flux.fromIterable(repositoryNames)
+ .flatMap(
+ repo -> JsonRetry.getAsync(MAVENBOX_JOBS_BASE_URL +
repo
+ +
"/api/json?tree=jobs[name,url,color,lastBuild[result,number]]")
+ .flatMap(jsonNode -> buildResult(repo,
jsonNode))
+ .onErrorResume(e -> {
+ getLog().warn("Failed to read status for "
+ repo + " Jenkins job "
+ + MAVENBOX_JOBS_BASE_URL + repo);
+ return Mono.empty();
+ }),
+ concurrency)
+ .collectList()
+ .block();
generateReport(repoStatus);
}
+ private Mono<Result> buildResult(String repository, JsonNode jsonNode) {
+ if (!(jsonNode instanceof ObjectNode objectNode)) {
+ getLog().warn("Failed to read JSON for " + repository + " Jenkins
job " + MAVENBOX_JOBS_BASE_URL
+ + repository);
+ return Mono.empty();
+ }
+ // find the master node
+ return Mono.justOrEmpty(objectNode
+ .get("jobs")
+ .valueStream()
+ .filter(n -> n.get("name").asText().equals("master"))
+ .findFirst()
+ .map(n -> {
+ JsonNode lastBuild = n.get("lastBuild");
+ String status = lastBuild != null ?
lastBuild.get("result").asText() : "UNKNOWN";
+ String buildUrl = n.get("url").asText()
+ + n.get("lastBuild").get("number").asText();
+ Result result = new Result(repository, buildUrl);
+ result.setStatus(status);
+ //
https://ci-maven.apache.org/static/67d6365a/images/24x24/blue.png
+ result.setIcon("https://ci-maven.apache.org/static/48x48/";
+ + n.get("color").asText().toLowerCase() + ".png");
+ return result;
+ }));
+ }
+
private void generateReport(List<Result> repoStatus) {
Sink sink = getSink();
diff --git
a/src/main/java/org/apache/maven/dist/tools/memorycheck/MemoryCheckReport.java
b/src/main/java/org/apache/maven/dist/tools/memorycheck/MemoryCheckReport.java
index 145fbb6..b01f518 100644
---
a/src/main/java/org/apache/maven/dist/tools/memorycheck/MemoryCheckReport.java
+++
b/src/main/java/org/apache/maven/dist/tools/memorycheck/MemoryCheckReport.java
@@ -118,9 +118,8 @@ public class MemoryCheckReport extends
AbstractDistCheckReport {
status.forEach(s -> {
sink.listItem();
sink.link(s.getHtmlUrl().toString());
- sink.text(DateTimeFormatter.ISO_LOCAL_DATE
- .withZone(ZoneId.of("UTC"))
- .format(s.getStartedAt().toInstant()));
+ sink.text(
+
DateTimeFormatter.ISO_LOCAL_DATE.withZone(ZoneId.of("UTC")).format(s.getStartedAt()));
if (s.getConclusion() == GHWorkflowRun.Conclusion.SUCCESS) {
iconSuccess(sink);
} else {
@@ -168,8 +167,9 @@ public class MemoryCheckReport extends
AbstractDistCheckReport {
private List<GHWorkflowJob> getLatestBuildStatus() {
try {
+ // FIXME we should use some GH token
return StreamSupport.stream(
- GitHub.connect()
+ GitHub.connectAnonymously()
.getRepository(GITHUB_REPOSITORY)
.queryWorkflowRuns()
.list()
diff --git
a/src/test/java/org/apache/maven/dist/tools/committers/MavenCommittersRepositoryTest.java
b/src/test/java/org/apache/maven/dist/tools/committers/MavenCommittersRepositoryTest.java
index 0e81d28..65afa5f 100644
---
a/src/test/java/org/apache/maven/dist/tools/committers/MavenCommittersRepositoryTest.java
+++
b/src/test/java/org/apache/maven/dist/tools/committers/MavenCommittersRepositoryTest.java
@@ -18,19 +18,24 @@
*/
package org.apache.maven.dist.tools.committers;
+import java.net.InetSocketAddress;
import java.util.List;
-import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
-import com.github.tomakehurst.wiremock.junit5.WireMockTest;
import
org.apache.maven.dist.tools.committers.MavenCommittersRepository.Committer;
+import org.eclipse.jetty.http.HttpHeader;
+import org.eclipse.jetty.io.Content;
+import org.eclipse.jetty.server.Handler;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.Response;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.util.Callback;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
-import static com.github.tomakehurst.wiremock.client.WireMock.get;
-import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
import static org.assertj.core.api.Assertions.assertThat;
-@WireMockTest
class MavenCommittersRepositoryTest {
private static final String GROUP = """
@@ -63,17 +68,45 @@ class MavenCommittersRepositoryTest {
}
""";
- @Test
- void dataLoad(WireMockRuntimeInfo wireMockRuntimeInfo) {
+ private Server server;
+ private String baseUrl;
+
+ @BeforeEach
+ void startServer() throws Exception {
+ server = new Server(new InetSocketAddress("localhost", 0));
+ server.setHandler(new Handler.Abstract() {
+ @Override
+ public boolean handle(Request request, Response response, Callback
callback) throws Exception {
+ String path = request.getHttpURI().getPath();
+ String body;
+ if ("/json/foundation/groups.json".equals(path)) {
+ body = GROUP;
+ } else if ("/json/foundation/people_name.json".equals(path)) {
+ body = NAMES;
+ } else {
+ Response.writeError(request, response, callback, 404);
+ return true;
+ }
+ response.setStatus(200);
+ response.getHeaders().put(HttpHeader.CONTENT_TYPE,
"application/json");
+ Content.Sink.write(response, true, body, callback);
+ return true;
+ }
+ });
+ server.start();
+ int port = ((ServerConnector)
server.getConnectors()[0]).getLocalPort();
+ baseUrl = "http://localhost:"; + port;
+ }
- stubFor(get("/json/foundation/groups.json")
- .willReturn(aResponse().withStatus(200).withBody(GROUP)));
- stubFor(get("/json/foundation/people_name.json")
- .willReturn(aResponse().withStatus(200).withBody(NAMES)));
+ @AfterEach
+ void stopServer() throws Exception {
+ server.stop();
+ }
- MavenCommittersRepository mavenCommittersRepository =
- new
MavenCommittersRepository(wireMockRuntimeInfo.getHttpBaseUrl());
- assertThat(mavenCommittersRepository.getCommitters())
+ @Test
+ void dataLoad() {
+ MavenCommittersRepository repo = new
MavenCommittersRepository(baseUrl);
+ assertThat(repo.getCommitters())
.containsExactly(
new Committer("cstamas", List.of("Tamas Cservenak",
"Tamás Cservenák"), true),
new Committer("m1", List.of("M1 name"), false),
