This is an automated email from the ASF dual-hosted git repository.

davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git


The following commit(s) were added to refs/heads/main by this push:
     new 0383d103ea00 CAMEL-27004: camel-jbang - an example from GitHub with a 
file that cannot be downloaded fails, and a failed refresh of the Quarkus 
platforms uses the cached copy (#30124)
0383d103ea00 is described below

commit 0383d103ea00b95fbbf884d24e5d1a069183f3e7
Author: Claus Ibsen <[email protected]>
AuthorDate: Mon Nov 16 08:56:02 2026 +0100

    CAMEL-27004: camel-jbang - an example from GitHub with a file that cannot 
be downloaded fails, and a failed refresh of the Quarkus platforms uses the 
cached copy (#30124)
    
    Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
 .../camel/dsl/jbang/core/common/ExampleHelper.java |  60 ++++++--
 .../core/common/ExampleHelperDownloadTest.java     | 171 +++++++++++++++++++++
 .../camel/dsl/jbang/run/common/QuarkusHelper.java  |  72 ++++++++-
 .../dsl/jbang/run/common/QuarkusHelperTest.java    |  61 ++++++++
 4 files changed, 349 insertions(+), 15 deletions(-)

diff --git 
a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/ExampleHelper.java
 
b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/ExampleHelper.java
index a72292720ae8..39f3bba2f6fb 100644
--- 
a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/ExampleHelper.java
+++ 
b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/ExampleHelper.java
@@ -16,6 +16,7 @@
  */
 package org.apache.camel.dsl.jbang.core.common;
 
+import java.io.IOException;
 import java.io.InputStream;
 import java.net.ProxySelector;
 import java.net.URI;
@@ -34,6 +35,9 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 import org.apache.camel.main.util.SuggestSimilarHelper;
 import org.apache.camel.util.FileUtil;
@@ -543,21 +547,34 @@ public final class ExampleHelper {
      * @param deleteOnExit whether the folder is deleted when this JVM exits; 
false when the example may outlive it
      */
     public static Path downloadGithubExample(JsonObject entry, boolean 
deleteOnExit) throws Exception {
+        return downloadGithubExample(entry, deleteOnExit, GITHUB_RAW_URL, 
Duration.ofSeconds(20));
+    }
+
+    /**
+     * Downloads each file of the example, and fails when one of them cannot 
be downloaded, so the example does not run
+     * without it (CAMEL-27004: a file that GitHub did not answer with 200, 
such as at a rate limit, was skipped).
+     *
+     * @param rawUrl  the URL of a file, with the name of the example and the 
name of the file
+     * @param timeout how long to wait for a file, its content included
+     */
+    static Path downloadGithubExample(JsonObject entry, boolean deleteOnExit, 
String rawUrl, Duration timeout)
+            throws Exception {
         String name = entry.getString("name");
         List<String> fileNames = getFiles(entry);
         Path tempDir = Files.createTempDirectory(EXAMPLE_DIR_PREFIX);
-
-        HttpClient hc = 
HttpClient.newBuilder().proxy(ProxySelector.getDefault()).build();
-        for (String fileName : fileNames) {
-            String rawUrl = String.format(GITHUB_RAW_URL, name, fileName);
-            HttpResponse<String> res = hc.send(
-                    HttpRequest.newBuilder(new 
URI(rawUrl)).timeout(Duration.ofSeconds(20)).build(),
-                    HttpResponse.BodyHandlers.ofString());
-            if (res.statusCode() == 200) {
+        try {
+            HttpClient hc = 
HttpClient.newBuilder().proxy(ProxySelector.getDefault()).build();
+            for (String fileName : fileNames) {
+                String url = String.format(rawUrl, name, fileName);
+                String content = downloadFile(hc, url, timeout);
                 Path targetFile = tempDir.resolve(fileName);
                 Files.createDirectories(targetFile.getParent());
-                Files.writeString(targetFile, res.body());
+                Files.writeString(targetFile, content);
             }
+        } catch (Exception e) {
+            // not half an example
+            FileUtil.removeDir(tempDir.toFile());
+            throw e;
         }
 
         if (deleteOnExit) {
@@ -566,6 +583,31 @@ public final class ExampleHelper {
         return tempDir;
     }
 
+    private static String downloadFile(HttpClient hc, String url, Duration 
timeout)
+            throws IOException, InterruptedException {
+        // sendAsync + get(timeout) bounds the content too; 
HttpRequest.timeout() only bounds the headers
+        var future = 
hc.sendAsync(HttpRequest.newBuilder(URI.create(url)).timeout(timeout).build(),
+                HttpResponse.BodyHandlers.ofString());
+        HttpResponse<String> res;
+        try {
+            res = future.get(timeout.toMillis(), TimeUnit.MILLISECONDS);
+        } catch (TimeoutException e) {
+            future.cancel(true);
+            throw new IOException("Downloading " + url + " timed out after " + 
timeout.toSeconds() + " seconds", e);
+        } catch (ExecutionException e) {
+            if (e.getCause() instanceof IOException ioe) {
+                throw ioe;
+            }
+            throw new IOException("Downloading " + url + " failed", 
e.getCause());
+        }
+        int status = res.statusCode();
+        if (status != 200) {
+            String hint = status == 403 || status == 429 ? " (GitHub rate 
limit, try again later)" : "";
+            throw new IOException("Downloading " + url + " failed with HTTP " 
+ status + hint);
+        }
+        return res.body();
+    }
+
     /**
      * Deletes the folder of an example, with what the example wrote in it 
(out/, a done folder), when this JVM exits.
      * File.deleteOnExit leaves the folder behind once anything else is in it.
diff --git 
a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/common/ExampleHelperDownloadTest.java
 
b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/common/ExampleHelperDownloadTest.java
new file mode 100644
index 000000000000..96e3f3a7446f
--- /dev/null
+++ 
b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/common/ExampleHelperDownloadTest.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.dsl.jbang.core.common;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.stream.Stream;
+
+import com.sun.net.httpserver.HttpServer;
+import org.apache.camel.util.FileUtil;
+import org.apache.camel.util.json.JsonObject;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * The download of an example from GitHub fails when a file of it cannot be 
downloaded, instead of running the example
+ * without that file (CAMEL-27004), and a file whose content stalls does not 
hang the download.
+ */
+class ExampleHelperDownloadTest {
+
+    private HttpServer server;
+    private ExecutorService executor;
+    private String rawUrl;
+    // the status per file name; 200 when not given
+    private final Map<String, Integer> status = new ConcurrentHashMap<>();
+    private final CountDownLatch release = new CountDownLatch(1);
+    private volatile String stalled;
+    // the content of the first file, to tell whether a folder of it was left 
behind
+    private final String marker = UUID.randomUUID().toString();
+
+    @BeforeEach
+    void startServer() throws IOException {
+        server = HttpServer.create(new InetSocketAddress("localhost", 0), 0);
+        executor = Executors.newCachedThreadPool();
+        server.setExecutor(executor);
+        server.createContext("/", exchange -> {
+            String file = 
exchange.getRequestURI().getPath().substring("/hello/".length());
+            int code = status.getOrDefault(file, 200);
+            if (code != 200) {
+                exchange.sendResponseHeaders(code, -1);
+                exchange.close();
+                return;
+            }
+            byte[] body = ("route.yaml".equals(file) ? marker : "content of " 
+ file).getBytes(StandardCharsets.UTF_8);
+            if (file.equals(stalled)) {
+                // the headers, and then the content only when the test ends
+                exchange.sendResponseHeaders(200, body.length + 1);
+                try (OutputStream out = exchange.getResponseBody()) {
+                    out.write(body);
+                    out.flush();
+                    release.await();
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                }
+                return;
+            }
+            exchange.sendResponseHeaders(200, body.length);
+            try (OutputStream out = exchange.getResponseBody()) {
+                out.write(body);
+            }
+        });
+        server.start();
+        rawUrl = "http://localhost:"; + server.getAddress().getPort() + 
"/%s/%s";
+    }
+
+    @AfterEach
+    void stopServer() {
+        release.countDown();
+        server.stop(0);
+        executor.shutdownNow();
+    }
+
+    @Test
+    void downloadsEachFile() throws Exception {
+        Path dir = ExampleHelper.downloadGithubExample(example(), false, 
rawUrl, Duration.ofSeconds(10));
+        try {
+            assertThat(dir.resolve("route.yaml")).hasContent(marker);
+            assertThat(dir.resolve("input/order.xml")).hasContent("content of 
input/order.xml");
+        } finally {
+            FileUtil.removeDir(dir.toFile());
+        }
+    }
+
+    @Test
+    void aFileThatIsNotThereFailsTheDownload() throws Exception {
+        status.put("input/order.xml", 404);
+
+        assertThatThrownBy(() -> 
ExampleHelper.downloadGithubExample(example(), false, rawUrl, 
Duration.ofSeconds(10)))
+                .isInstanceOf(IOException.class)
+                .hasMessageContaining("/hello/input/order.xml")
+                .hasMessageContaining("HTTP 404");
+        // and not half of the example is left
+        assertThat(foldersWithTheFirstFile()).isEmpty();
+    }
+
+    @Test
+    void aRateLimitSaysSo() {
+        status.put("route.yaml", 429);
+
+        assertThatThrownBy(() -> 
ExampleHelper.downloadGithubExample(example(), false, rawUrl, 
Duration.ofSeconds(10)))
+                .isInstanceOf(IOException.class)
+                .hasMessageContaining("HTTP 429")
+                .hasMessageContaining("rate limit");
+    }
+
+    @Test
+    void aStalledFileTimesOut() throws Exception {
+        stalled = "input/order.xml";
+
+        long start = System.nanoTime();
+        assertThatThrownBy(() -> 
ExampleHelper.downloadGithubExample(example(), false, rawUrl, 
Duration.ofSeconds(1)))
+                .isInstanceOf(IOException.class)
+                .hasMessageContaining("timed out");
+        assertThat(Duration.ofNanos(System.nanoTime() - 
start)).isLessThan(Duration.ofSeconds(10));
+        assertThat(foldersWithTheFirstFile()).isEmpty();
+    }
+
+    private static JsonObject example() {
+        JsonObject entry = new JsonObject();
+        entry.put("name", "hello");
+        entry.put("files", List.of("route.yaml", "input/order.xml"));
+        return entry;
+    }
+
+    // the example folders in the temporary directory that hold the first file 
of this test
+    private List<Path> foldersWithTheFirstFile() throws IOException {
+        Path tmp = Path.of(System.getProperty("java.io.tmpdir"));
+        try (Stream<Path> dirs = Files.list(tmp)) {
+            return dirs.filter(d -> 
d.getFileName().toString().startsWith(ExampleHelper.EXAMPLE_DIR_PREFIX))
+                    .filter(d -> {
+                        try {
+                            Path f = d.resolve("route.yaml");
+                            return Files.isRegularFile(f) && 
marker.equals(Files.readString(f));
+                        } catch (IOException e) {
+                            return false;
+                        }
+                    })
+                    .toList();
+        }
+    }
+}
diff --git 
a/dsl/camel-jbang/camel-jbang-run/src/main/java/org/apache/camel/dsl/jbang/run/common/QuarkusHelper.java
 
b/dsl/camel-jbang/camel-jbang-run/src/main/java/org/apache/camel/dsl/jbang/run/common/QuarkusHelper.java
index 7f076301e336..b6023932ad23 100644
--- 
a/dsl/camel-jbang/camel-jbang-run/src/main/java/org/apache/camel/dsl/jbang/run/common/QuarkusHelper.java
+++ 
b/dsl/camel-jbang/camel-jbang-run/src/main/java/org/apache/camel/dsl/jbang/run/common/QuarkusHelper.java
@@ -44,6 +44,9 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.Properties;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
 import java.util.stream.Stream;
@@ -68,6 +71,8 @@ import org.apache.camel.util.json.JsonArray;
 import org.apache.camel.util.json.JsonObject;
 import org.apache.camel.util.json.Jsoner;
 import org.apache.maven.artifact.versioning.ComparableVersion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import static 
org.apache.camel.dsl.jbang.run.common.CamelJBangConstants.QUARKUS_EXTENSION_REGISTRY_BASE_URI;
 import static 
org.apache.camel.dsl.jbang.run.common.CamelJBangConstants.QUARKUS_GROUP_ID;
@@ -88,6 +93,11 @@ public final class QuarkusHelper {
      */
     private static final Duration PLATFORM_MAPPING_TTL = Duration.ofDays(7);
 
+    private static final Logger LOG = 
LoggerFactory.getLogger(QuarkusHelper.class);
+
+    // how long the registry may take to answer, its document included
+    private static final Duration REGISTRY_TIMEOUT = Duration.ofSeconds(20);
+
     private QuarkusHelper() {
     }
 
@@ -344,12 +354,17 @@ public final class QuarkusHelper {
                 }
             }
 
-            HttpClient hc = 
HttpClient.newBuilder().proxy(ProxySelector.getDefault()).build();
-            HttpResponse<String> res = hc.send(
-                    HttpRequest.newBuilder(uri)
-                            .timeout(Duration.ofSeconds(20))
-                            .build(),
-                    HttpResponse.BodyHandlers.ofString());
+            HttpResponse<String> res;
+            try {
+                res = get(uri);
+            } catch (IOException e) {
+                // offline, or the registry does not answer: the document of 
an earlier day is better than none
+                JsonArray stale = staleCache(uri, registriesDir, fresh, 
e.getMessage());
+                if (stale != null) {
+                    return stale;
+                }
+                throw e;
+            }
 
             if (res.statusCode() == 200) {
                 final String jsonBody = res.body();
@@ -361,6 +376,10 @@ public final class QuarkusHelper {
 
                 return deserialize(jsonBody);
             } else {
+                JsonArray stale = staleCache(uri, registriesDir, fresh, "HTTP 
" + res.statusCode());
+                if (stale != null) {
+                    return stale;
+                }
                 throw new RuntimeException(
                         "Could not get " + uri + ": " + res.statusCode() + " 
to determine the Quarkus Platform version."
                                            + " You may want to pass 
--download=false to use the cached resource if available"
@@ -378,6 +397,47 @@ public final class QuarkusHelper {
         }
     }
 
+    /**
+     * Gets the document of the registry. HttpRequest.timeout() only bounds 
the headers, so the document is bounded with
+     * sendAsync and get(timeout): a registry that sent the headers and then 
stalled hung the command.
+     */
+    private static HttpResponse<String> get(URI uri) throws IOException, 
InterruptedException {
+        HttpClient hc = 
HttpClient.newBuilder().proxy(ProxySelector.getDefault()).build();
+        var future = 
hc.sendAsync(HttpRequest.newBuilder(uri).timeout(REGISTRY_TIMEOUT).build(),
+                HttpResponse.BodyHandlers.ofString());
+        try {
+            return future.get(REGISTRY_TIMEOUT.toMillis(), 
TimeUnit.MILLISECONDS);
+        } catch (TimeoutException e) {
+            future.cancel(true);
+            throw new IOException("timed out after " + 
REGISTRY_TIMEOUT.toSeconds() + " seconds", e);
+        } catch (ExecutionException e) {
+            if (e.getCause() instanceof IOException ioe) {
+                throw ioe;
+            }
+            throw new IOException(e.getCause());
+        }
+    }
+
+    /**
+     * The document of the registry cached on an earlier day, when the daily 
refresh failed (CAMEL-27004: the command
+     * failed although it had the document). Not with --fresh, which asks for 
the registry and nothing else.
+     *
+     * @return the platform streams of the cached document, or null when there 
is none
+     */
+    private static JsonArray staleCache(URI uri, Path registriesDir, boolean 
fresh, String reason)
+            throws IOException, DeserializationException {
+        if (fresh) {
+            return null;
+        }
+        Path cacheFile = cacheFile(uri, registriesDir);
+        if (!Files.isRegularFile(cacheFile)) {
+            return null;
+        }
+        LOG.warn("Could not refresh the Quarkus platforms from {} ({}): using 
the copy cached on {}", uri, reason,
+                
LocalDate.ofInstant(Files.getLastModifiedTime(cacheFile).toInstant(), 
ZoneId.systemDefault()));
+        return deserialize(Files.readString(cacheFile));
+    }
+
     private static boolean updatedToday(Path cacheFile) {
         try {
             FileTime lastModified = Files.getLastModifiedTime(cacheFile);
diff --git 
a/dsl/camel-jbang/camel-jbang-run/src/test/java/org/apache/camel/dsl/jbang/run/common/QuarkusHelperTest.java
 
b/dsl/camel-jbang/camel-jbang-run/src/test/java/org/apache/camel/dsl/jbang/run/common/QuarkusHelperTest.java
index 398960cba8c8..716b2d4e3878 100644
--- 
a/dsl/camel-jbang/camel-jbang-run/src/test/java/org/apache/camel/dsl/jbang/run/common/QuarkusHelperTest.java
+++ 
b/dsl/camel-jbang/camel-jbang-run/src/test/java/org/apache/camel/dsl/jbang/run/common/QuarkusHelperTest.java
@@ -33,6 +33,7 @@ import java.util.stream.Stream;
 
 import com.github.tomakehurst.wiremock.client.WireMock;
 import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
+import com.github.tomakehurst.wiremock.http.Fault;
 import com.github.tomakehurst.wiremock.junit5.WireMockExtension;
 import 
org.apache.camel.dsl.jbang.run.common.QuarkusHelper.CamelVersionInPlatformRelease;
 import org.apache.camel.dsl.jbang.run.common.QuarkusHelper.MajorMinor;
@@ -586,4 +587,64 @@ public class QuarkusHelperTest {
                     .containsExactlyInAnyOrder("all.json", 
"platform-mapping.json");
         }
     }
+
+    // CAMEL-27004: the daily refresh of the registry document failed the 
command, although it had the document
+
+    @Test
+    void staleCacheIsUsedWhenTheRegistryFails() throws Exception {
+        stubRegistry();
+        Path registriesDir = newRegistriesDir();
+        JsonArray cached = 
QuarkusHelper.fetchPlatformStreams(wireMock.baseUrl(), true, false, 
registriesDir);
+        backdate(registryCacheFile(registriesDir));
+
+        
wireMock.stubFor(WireMock.get(WireMock.urlEqualTo("/client/platforms/all"))
+                .willReturn(WireMock.serverError()));
+        JsonArray streams = 
QuarkusHelper.fetchPlatformStreams(wireMock.baseUrl(), true, false, 
registriesDir);
+
+        Assertions.assertThat(streams).isNotNull().isEqualTo(cached);
+    }
+
+    @Test
+    void staleCacheIsUsedWhenTheRegistryCannotBeReached() throws Exception {
+        stubRegistry();
+        Path registriesDir = newRegistriesDir();
+        JsonArray cached = 
QuarkusHelper.fetchPlatformStreams(wireMock.baseUrl(), true, false, 
registriesDir);
+        backdate(registryCacheFile(registriesDir));
+
+        // the connection is reset, as when offline or behind a proxy that 
refuses it
+        
wireMock.stubFor(WireMock.get(WireMock.urlEqualTo("/client/platforms/all"))
+                .willReturn(WireMock.aResponse().withFault(
+                        Fault.CONNECTION_RESET_BY_PEER)));
+        JsonArray streams = 
QuarkusHelper.fetchPlatformStreams(wireMock.baseUrl(), true, false, 
registriesDir);
+
+        Assertions.assertThat(streams).isNotNull().isEqualTo(cached);
+    }
+
+    @Test
+    void withoutACacheAFailingRegistryStillFails() throws Exception {
+        wireMock.resetAll();
+        
wireMock.stubFor(WireMock.get(WireMock.urlEqualTo("/client/platforms/all"))
+                .willReturn(WireMock.serverError()));
+        Path registriesDir = newRegistriesDir();
+
+        Assertions.assertThatThrownBy(
+                () -> QuarkusHelper.fetchPlatformStreams(wireMock.baseUrl(), 
true, false, registriesDir))
+                .isInstanceOf(RuntimeException.class)
+                .hasMessageContaining("500");
+    }
+
+    @Test
+    void freshDoesNotUseTheStaleCache() throws Exception {
+        stubRegistry();
+        Path registriesDir = newRegistriesDir();
+        QuarkusHelper.fetchPlatformStreams(wireMock.baseUrl(), true, false, 
registriesDir);
+
+        
wireMock.stubFor(WireMock.get(WireMock.urlEqualTo("/client/platforms/all"))
+                .willReturn(WireMock.serverError()));
+
+        Assertions.assertThatThrownBy(
+                () -> QuarkusHelper.fetchPlatformStreams(wireMock.baseUrl(), 
true, true, registriesDir))
+                .isInstanceOf(RuntimeException.class)
+                .hasMessageContaining("500");
+    }
 }

Reply via email to