This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 0383d103ea00 CAMEL-27004: camel-jbang - an example from GitHub with a
file that cannot be downloaded fails, and a failed refresh of the Quarkus
platforms uses the cached copy (#30124)
0383d103ea00 is described below
commit 0383d103ea00b95fbbf884d24e5d1a069183f3e7
Author: Claus Ibsen <[email protected]>
AuthorDate: Mon Nov 16 08:56:02 2026 +0100
CAMEL-27004: camel-jbang - an example from GitHub with a file that cannot
be downloaded fails, and a failed refresh of the Quarkus platforms uses the
cached copy (#30124)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../camel/dsl/jbang/core/common/ExampleHelper.java | 60 ++++++--
.../core/common/ExampleHelperDownloadTest.java | 171 +++++++++++++++++++++
.../camel/dsl/jbang/run/common/QuarkusHelper.java | 72 ++++++++-
.../dsl/jbang/run/common/QuarkusHelperTest.java | 61 ++++++++
4 files changed, 349 insertions(+), 15 deletions(-)
diff --git
a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/ExampleHelper.java
b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/ExampleHelper.java
index a72292720ae8..39f3bba2f6fb 100644
---
a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/ExampleHelper.java
+++
b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/ExampleHelper.java
@@ -16,6 +16,7 @@
*/
package org.apache.camel.dsl.jbang.core.common;
+import java.io.IOException;
import java.io.InputStream;
import java.net.ProxySelector;
import java.net.URI;
@@ -34,6 +35,9 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import org.apache.camel.main.util.SuggestSimilarHelper;
import org.apache.camel.util.FileUtil;
@@ -543,21 +547,34 @@ public final class ExampleHelper {
* @param deleteOnExit whether the folder is deleted when this JVM exits;
false when the example may outlive it
*/
public static Path downloadGithubExample(JsonObject entry, boolean
deleteOnExit) throws Exception {
+ return downloadGithubExample(entry, deleteOnExit, GITHUB_RAW_URL,
Duration.ofSeconds(20));
+ }
+
+ /**
+ * Downloads each file of the example, and fails when one of them cannot
be downloaded, so the example does not run
+ * without it (CAMEL-27004: a file that GitHub did not answer with 200,
such as at a rate limit, was skipped).
+ *
+ * @param rawUrl the URL of a file, with the name of the example and the
name of the file
+ * @param timeout how long to wait for a file, its content included
+ */
+ static Path downloadGithubExample(JsonObject entry, boolean deleteOnExit,
String rawUrl, Duration timeout)
+ throws Exception {
String name = entry.getString("name");
List<String> fileNames = getFiles(entry);
Path tempDir = Files.createTempDirectory(EXAMPLE_DIR_PREFIX);
-
- HttpClient hc =
HttpClient.newBuilder().proxy(ProxySelector.getDefault()).build();
- for (String fileName : fileNames) {
- String rawUrl = String.format(GITHUB_RAW_URL, name, fileName);
- HttpResponse<String> res = hc.send(
- HttpRequest.newBuilder(new
URI(rawUrl)).timeout(Duration.ofSeconds(20)).build(),
- HttpResponse.BodyHandlers.ofString());
- if (res.statusCode() == 200) {
+ try {
+ HttpClient hc =
HttpClient.newBuilder().proxy(ProxySelector.getDefault()).build();
+ for (String fileName : fileNames) {
+ String url = String.format(rawUrl, name, fileName);
+ String content = downloadFile(hc, url, timeout);
Path targetFile = tempDir.resolve(fileName);
Files.createDirectories(targetFile.getParent());
- Files.writeString(targetFile, res.body());
+ Files.writeString(targetFile, content);
}
+ } catch (Exception e) {
+ // not half an example
+ FileUtil.removeDir(tempDir.toFile());
+ throw e;
}
if (deleteOnExit) {
@@ -566,6 +583,31 @@ public final class ExampleHelper {
return tempDir;
}
+ private static String downloadFile(HttpClient hc, String url, Duration
timeout)
+ throws IOException, InterruptedException {
+ // sendAsync + get(timeout) bounds the content too;
HttpRequest.timeout() only bounds the headers
+ var future =
hc.sendAsync(HttpRequest.newBuilder(URI.create(url)).timeout(timeout).build(),
+ HttpResponse.BodyHandlers.ofString());
+ HttpResponse<String> res;
+ try {
+ res = future.get(timeout.toMillis(), TimeUnit.MILLISECONDS);
+ } catch (TimeoutException e) {
+ future.cancel(true);
+ throw new IOException("Downloading " + url + " timed out after " +
timeout.toSeconds() + " seconds", e);
+ } catch (ExecutionException e) {
+ if (e.getCause() instanceof IOException ioe) {
+ throw ioe;
+ }
+ throw new IOException("Downloading " + url + " failed",
e.getCause());
+ }
+ int status = res.statusCode();
+ if (status != 200) {
+ String hint = status == 403 || status == 429 ? " (GitHub rate
limit, try again later)" : "";
+ throw new IOException("Downloading " + url + " failed with HTTP "
+ status + hint);
+ }
+ return res.body();
+ }
+
/**
* Deletes the folder of an example, with what the example wrote in it
(out/, a done folder), when this JVM exits.
* File.deleteOnExit leaves the folder behind once anything else is in it.
diff --git
a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/common/ExampleHelperDownloadTest.java
b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/common/ExampleHelperDownloadTest.java
new file mode 100644
index 000000000000..96e3f3a7446f
--- /dev/null
+++
b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/common/ExampleHelperDownloadTest.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.dsl.jbang.core.common;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.stream.Stream;
+
+import com.sun.net.httpserver.HttpServer;
+import org.apache.camel.util.FileUtil;
+import org.apache.camel.util.json.JsonObject;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * The download of an example from GitHub fails when a file of it cannot be
downloaded, instead of running the example
+ * without that file (CAMEL-27004), and a file whose content stalls does not
hang the download.
+ */
+class ExampleHelperDownloadTest {
+
+ private HttpServer server;
+ private ExecutorService executor;
+ private String rawUrl;
+ // the status per file name; 200 when not given
+ private final Map<String, Integer> status = new ConcurrentHashMap<>();
+ private final CountDownLatch release = new CountDownLatch(1);
+ private volatile String stalled;
+ // the content of the first file, to tell whether a folder of it was left
behind
+ private final String marker = UUID.randomUUID().toString();
+
+ @BeforeEach
+ void startServer() throws IOException {
+ server = HttpServer.create(new InetSocketAddress("localhost", 0), 0);
+ executor = Executors.newCachedThreadPool();
+ server.setExecutor(executor);
+ server.createContext("/", exchange -> {
+ String file =
exchange.getRequestURI().getPath().substring("/hello/".length());
+ int code = status.getOrDefault(file, 200);
+ if (code != 200) {
+ exchange.sendResponseHeaders(code, -1);
+ exchange.close();
+ return;
+ }
+ byte[] body = ("route.yaml".equals(file) ? marker : "content of "
+ file).getBytes(StandardCharsets.UTF_8);
+ if (file.equals(stalled)) {
+ // the headers, and then the content only when the test ends
+ exchange.sendResponseHeaders(200, body.length + 1);
+ try (OutputStream out = exchange.getResponseBody()) {
+ out.write(body);
+ out.flush();
+ release.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ return;
+ }
+ exchange.sendResponseHeaders(200, body.length);
+ try (OutputStream out = exchange.getResponseBody()) {
+ out.write(body);
+ }
+ });
+ server.start();
+ rawUrl = "http://localhost:" + server.getAddress().getPort() +
"/%s/%s";
+ }
+
+ @AfterEach
+ void stopServer() {
+ release.countDown();
+ server.stop(0);
+ executor.shutdownNow();
+ }
+
+ @Test
+ void downloadsEachFile() throws Exception {
+ Path dir = ExampleHelper.downloadGithubExample(example(), false,
rawUrl, Duration.ofSeconds(10));
+ try {
+ assertThat(dir.resolve("route.yaml")).hasContent(marker);
+ assertThat(dir.resolve("input/order.xml")).hasContent("content of
input/order.xml");
+ } finally {
+ FileUtil.removeDir(dir.toFile());
+ }
+ }
+
+ @Test
+ void aFileThatIsNotThereFailsTheDownload() throws Exception {
+ status.put("input/order.xml", 404);
+
+ assertThatThrownBy(() ->
ExampleHelper.downloadGithubExample(example(), false, rawUrl,
Duration.ofSeconds(10)))
+ .isInstanceOf(IOException.class)
+ .hasMessageContaining("/hello/input/order.xml")
+ .hasMessageContaining("HTTP 404");
+ // and not half of the example is left
+ assertThat(foldersWithTheFirstFile()).isEmpty();
+ }
+
+ @Test
+ void aRateLimitSaysSo() {
+ status.put("route.yaml", 429);
+
+ assertThatThrownBy(() ->
ExampleHelper.downloadGithubExample(example(), false, rawUrl,
Duration.ofSeconds(10)))
+ .isInstanceOf(IOException.class)
+ .hasMessageContaining("HTTP 429")
+ .hasMessageContaining("rate limit");
+ }
+
+ @Test
+ void aStalledFileTimesOut() throws Exception {
+ stalled = "input/order.xml";
+
+ long start = System.nanoTime();
+ assertThatThrownBy(() ->
ExampleHelper.downloadGithubExample(example(), false, rawUrl,
Duration.ofSeconds(1)))
+ .isInstanceOf(IOException.class)
+ .hasMessageContaining("timed out");
+ assertThat(Duration.ofNanos(System.nanoTime() -
start)).isLessThan(Duration.ofSeconds(10));
+ assertThat(foldersWithTheFirstFile()).isEmpty();
+ }
+
+ private static JsonObject example() {
+ JsonObject entry = new JsonObject();
+ entry.put("name", "hello");
+ entry.put("files", List.of("route.yaml", "input/order.xml"));
+ return entry;
+ }
+
+ // the example folders in the temporary directory that hold the first file
of this test
+ private List<Path> foldersWithTheFirstFile() throws IOException {
+ Path tmp = Path.of(System.getProperty("java.io.tmpdir"));
+ try (Stream<Path> dirs = Files.list(tmp)) {
+ return dirs.filter(d ->
d.getFileName().toString().startsWith(ExampleHelper.EXAMPLE_DIR_PREFIX))
+ .filter(d -> {
+ try {
+ Path f = d.resolve("route.yaml");
+ return Files.isRegularFile(f) &&
marker.equals(Files.readString(f));
+ } catch (IOException e) {
+ return false;
+ }
+ })
+ .toList();
+ }
+ }
+}
diff --git
a/dsl/camel-jbang/camel-jbang-run/src/main/java/org/apache/camel/dsl/jbang/run/common/QuarkusHelper.java
b/dsl/camel-jbang/camel-jbang-run/src/main/java/org/apache/camel/dsl/jbang/run/common/QuarkusHelper.java
index 7f076301e336..b6023932ad23 100644
---
a/dsl/camel-jbang/camel-jbang-run/src/main/java/org/apache/camel/dsl/jbang/run/common/QuarkusHelper.java
+++
b/dsl/camel-jbang/camel-jbang-run/src/main/java/org/apache/camel/dsl/jbang/run/common/QuarkusHelper.java
@@ -44,6 +44,9 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.stream.Stream;
@@ -68,6 +71,8 @@ import org.apache.camel.util.json.JsonArray;
import org.apache.camel.util.json.JsonObject;
import org.apache.camel.util.json.Jsoner;
import org.apache.maven.artifact.versioning.ComparableVersion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static
org.apache.camel.dsl.jbang.run.common.CamelJBangConstants.QUARKUS_EXTENSION_REGISTRY_BASE_URI;
import static
org.apache.camel.dsl.jbang.run.common.CamelJBangConstants.QUARKUS_GROUP_ID;
@@ -88,6 +93,11 @@ public final class QuarkusHelper {
*/
private static final Duration PLATFORM_MAPPING_TTL = Duration.ofDays(7);
+ private static final Logger LOG =
LoggerFactory.getLogger(QuarkusHelper.class);
+
+ // how long the registry may take to answer, its document included
+ private static final Duration REGISTRY_TIMEOUT = Duration.ofSeconds(20);
+
private QuarkusHelper() {
}
@@ -344,12 +354,17 @@ public final class QuarkusHelper {
}
}
- HttpClient hc =
HttpClient.newBuilder().proxy(ProxySelector.getDefault()).build();
- HttpResponse<String> res = hc.send(
- HttpRequest.newBuilder(uri)
- .timeout(Duration.ofSeconds(20))
- .build(),
- HttpResponse.BodyHandlers.ofString());
+ HttpResponse<String> res;
+ try {
+ res = get(uri);
+ } catch (IOException e) {
+ // offline, or the registry does not answer: the document of
an earlier day is better than none
+ JsonArray stale = staleCache(uri, registriesDir, fresh,
e.getMessage());
+ if (stale != null) {
+ return stale;
+ }
+ throw e;
+ }
if (res.statusCode() == 200) {
final String jsonBody = res.body();
@@ -361,6 +376,10 @@ public final class QuarkusHelper {
return deserialize(jsonBody);
} else {
+ JsonArray stale = staleCache(uri, registriesDir, fresh, "HTTP
" + res.statusCode());
+ if (stale != null) {
+ return stale;
+ }
throw new RuntimeException(
"Could not get " + uri + ": " + res.statusCode() + "
to determine the Quarkus Platform version."
+ " You may want to pass
--download=false to use the cached resource if available"
@@ -378,6 +397,47 @@ public final class QuarkusHelper {
}
}
+ /**
+ * Gets the document of the registry. HttpRequest.timeout() only bounds
the headers, so the document is bounded with
+ * sendAsync and get(timeout): a registry that sent the headers and then
stalled hung the command.
+ */
+ private static HttpResponse<String> get(URI uri) throws IOException,
InterruptedException {
+ HttpClient hc =
HttpClient.newBuilder().proxy(ProxySelector.getDefault()).build();
+ var future =
hc.sendAsync(HttpRequest.newBuilder(uri).timeout(REGISTRY_TIMEOUT).build(),
+ HttpResponse.BodyHandlers.ofString());
+ try {
+ return future.get(REGISTRY_TIMEOUT.toMillis(),
TimeUnit.MILLISECONDS);
+ } catch (TimeoutException e) {
+ future.cancel(true);
+ throw new IOException("timed out after " +
REGISTRY_TIMEOUT.toSeconds() + " seconds", e);
+ } catch (ExecutionException e) {
+ if (e.getCause() instanceof IOException ioe) {
+ throw ioe;
+ }
+ throw new IOException(e.getCause());
+ }
+ }
+
+ /**
+ * The document of the registry cached on an earlier day, when the daily
refresh failed (CAMEL-27004: the command
+ * failed although it had the document). Not with --fresh, which asks for
the registry and nothing else.
+ *
+ * @return the platform streams of the cached document, or null when there
is none
+ */
+ private static JsonArray staleCache(URI uri, Path registriesDir, boolean
fresh, String reason)
+ throws IOException, DeserializationException {
+ if (fresh) {
+ return null;
+ }
+ Path cacheFile = cacheFile(uri, registriesDir);
+ if (!Files.isRegularFile(cacheFile)) {
+ return null;
+ }
+ LOG.warn("Could not refresh the Quarkus platforms from {} ({}): using
the copy cached on {}", uri, reason,
+
LocalDate.ofInstant(Files.getLastModifiedTime(cacheFile).toInstant(),
ZoneId.systemDefault()));
+ return deserialize(Files.readString(cacheFile));
+ }
+
private static boolean updatedToday(Path cacheFile) {
try {
FileTime lastModified = Files.getLastModifiedTime(cacheFile);
diff --git
a/dsl/camel-jbang/camel-jbang-run/src/test/java/org/apache/camel/dsl/jbang/run/common/QuarkusHelperTest.java
b/dsl/camel-jbang/camel-jbang-run/src/test/java/org/apache/camel/dsl/jbang/run/common/QuarkusHelperTest.java
index 398960cba8c8..716b2d4e3878 100644
---
a/dsl/camel-jbang/camel-jbang-run/src/test/java/org/apache/camel/dsl/jbang/run/common/QuarkusHelperTest.java
+++
b/dsl/camel-jbang/camel-jbang-run/src/test/java/org/apache/camel/dsl/jbang/run/common/QuarkusHelperTest.java
@@ -33,6 +33,7 @@ import java.util.stream.Stream;
import com.github.tomakehurst.wiremock.client.WireMock;
import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
+import com.github.tomakehurst.wiremock.http.Fault;
import com.github.tomakehurst.wiremock.junit5.WireMockExtension;
import
org.apache.camel.dsl.jbang.run.common.QuarkusHelper.CamelVersionInPlatformRelease;
import org.apache.camel.dsl.jbang.run.common.QuarkusHelper.MajorMinor;
@@ -586,4 +587,64 @@ public class QuarkusHelperTest {
.containsExactlyInAnyOrder("all.json",
"platform-mapping.json");
}
}
+
+ // CAMEL-27004: the daily refresh of the registry document failed the
command, although it had the document
+
+ @Test
+ void staleCacheIsUsedWhenTheRegistryFails() throws Exception {
+ stubRegistry();
+ Path registriesDir = newRegistriesDir();
+ JsonArray cached =
QuarkusHelper.fetchPlatformStreams(wireMock.baseUrl(), true, false,
registriesDir);
+ backdate(registryCacheFile(registriesDir));
+
+
wireMock.stubFor(WireMock.get(WireMock.urlEqualTo("/client/platforms/all"))
+ .willReturn(WireMock.serverError()));
+ JsonArray streams =
QuarkusHelper.fetchPlatformStreams(wireMock.baseUrl(), true, false,
registriesDir);
+
+ Assertions.assertThat(streams).isNotNull().isEqualTo(cached);
+ }
+
+ @Test
+ void staleCacheIsUsedWhenTheRegistryCannotBeReached() throws Exception {
+ stubRegistry();
+ Path registriesDir = newRegistriesDir();
+ JsonArray cached =
QuarkusHelper.fetchPlatformStreams(wireMock.baseUrl(), true, false,
registriesDir);
+ backdate(registryCacheFile(registriesDir));
+
+ // the connection is reset, as when offline or behind a proxy that
refuses it
+
wireMock.stubFor(WireMock.get(WireMock.urlEqualTo("/client/platforms/all"))
+ .willReturn(WireMock.aResponse().withFault(
+ Fault.CONNECTION_RESET_BY_PEER)));
+ JsonArray streams =
QuarkusHelper.fetchPlatformStreams(wireMock.baseUrl(), true, false,
registriesDir);
+
+ Assertions.assertThat(streams).isNotNull().isEqualTo(cached);
+ }
+
+ @Test
+ void withoutACacheAFailingRegistryStillFails() throws Exception {
+ wireMock.resetAll();
+
wireMock.stubFor(WireMock.get(WireMock.urlEqualTo("/client/platforms/all"))
+ .willReturn(WireMock.serverError()));
+ Path registriesDir = newRegistriesDir();
+
+ Assertions.assertThatThrownBy(
+ () -> QuarkusHelper.fetchPlatformStreams(wireMock.baseUrl(),
true, false, registriesDir))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessageContaining("500");
+ }
+
+ @Test
+ void freshDoesNotUseTheStaleCache() throws Exception {
+ stubRegistry();
+ Path registriesDir = newRegistriesDir();
+ QuarkusHelper.fetchPlatformStreams(wireMock.baseUrl(), true, false,
registriesDir);
+
+
wireMock.stubFor(WireMock.get(WireMock.urlEqualTo("/client/platforms/all"))
+ .willReturn(WireMock.serverError()));
+
+ Assertions.assertThatThrownBy(
+ () -> QuarkusHelper.fetchPlatformStreams(wireMock.baseUrl(),
true, true, registriesDir))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessageContaining("500");
+ }
}