This is an automated email from the ASF dual-hosted git repository. mraible pushed a commit to branch planet-fetch-status-and-interrupts-6.1.x in repository https://gitbox.apache.org/repos/asf/roller.git
commit ecdc5f2fd66502c7b1009feca1a58f6c8b000eca Author: Matt Raible <[email protected]> AuthorDate: Tue Nov 17 20:20:09 2026 -0700 Keep interrupts, check the HTTP status and stop abandoned feed downloads Review follow-ups for the planet feed fetcher: - An interrupted fetch restores the thread's interrupt flag before throwing FetcherException. - A malformed feed URL fails with a FetcherException instead of the IllegalArgumentException from URI.create. - A response other than 2xx fails with its status instead of being parsed as a feed. - On Java 11, cancelling the response future doesn't stop the exchange, so a stalled feed kept downloading after the fetcher gave up. The body subscriber now cancels its subscription when the fetcher stops waiting, which closes the connection. - The timing tests allow more time, so a slow build machine doesn't fail them. --- .../planet/business/fetcher/RomeFeedFetcher.java | 80 ++++++++++++++++++---- .../fetcher/RomeFeedFetcherLimitsTest.java | 71 +++++++++++++++++-- 2 files changed, 132 insertions(+), 19 deletions(-) diff --git a/app/src/main/java/org/apache/roller/planet/business/fetcher/RomeFeedFetcher.java b/app/src/main/java/org/apache/roller/planet/business/fetcher/RomeFeedFetcher.java index c47c27866..762d68ea2 100644 --- a/app/src/main/java/org/apache/roller/planet/business/fetcher/RomeFeedFetcher.java +++ b/app/src/main/java/org/apache/roller/planet/business/fetcher/RomeFeedFetcher.java @@ -47,6 +47,8 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Flow; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.text.StringEscapeUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; @@ -119,7 +121,11 @@ public class RomeFeedFetcher implements FeedFetcher { SyndFeed feed; try { feed = fetchFeed(feedURL); - } catch (FeedException | IOException | InterruptedException ex) { + } catch (InterruptedException ex) { + // let the caller see that it was asked to stop + Thread.currentThread().interrupt(); + throw new FetcherException("Interrupted fetching subscription - "+feedURL, ex); + } catch (FeedException | IOException ex) { throw new FetcherException("Error fetching subscription - "+feedURL, ex); } @@ -255,22 +261,34 @@ public class RomeFeedFetcher implements FeedFetcher { private SyndFeed fetchFeed(String url) throws IOException, InterruptedException, FeedException { - HttpRequest request = requestBuilder.copy().uri(URI.create(url)).build(); + HttpRequest request; + try { + request = requestBuilder.copy().uri(URI.create(url)).build(); + } catch (IllegalArgumentException e) { + throw new IOException("Feed URL is not valid - " + url, e); + } // the request timeout only covers the response headers, so bound the // whole download, including the body, here + AtomicBoolean abandoned = new AtomicBoolean(); + AtomicReference<LimitedBodySubscriber> subscriber = new AtomicReference<>(); CompletableFuture<HttpResponse<byte[]>> response = client.sendAsync(request, - info -> new LimitedBodySubscriber(maxBytes, - info.headers().firstValueAsLong("Content-Length").orElse(-1))); + info -> { + LimitedBodySubscriber s = new LimitedBodySubscriber(maxBytes, + info.headers().firstValueAsLong("Content-Length").orElse(-1), + info.statusCode(), abandoned); + subscriber.set(s); + return s; + }); byte[] body; try { body = response.get(maxTime.toMillis(), TimeUnit.MILLISECONDS).body(); } catch (TimeoutException e) { - response.cancel(true); + abandon(response, abandoned, subscriber); throw new IOException("Feed was not received within " + maxTime.toSeconds() + " seconds"); } catch (InterruptedException e) { - response.cancel(true); + abandon(response, abandoned, subscriber); throw e; } catch (ExecutionException e) { if (e.getCause() instanceof IOException) { @@ -286,6 +304,20 @@ public class RomeFeedFetcher implements FeedFetcher { } + // stops a download we no longer wait for. On Java 11, cancelling the + // future doesn't stop the exchange, so the body subscriber also cancels + // its subscription, which closes the connection. + private static void abandon(CompletableFuture<?> response, AtomicBoolean abandoned, + AtomicReference<LimitedBodySubscriber> subscriber) { + abandoned.set(true); + LimitedBodySubscriber s = subscriber.get(); + if (s != null) { + s.cancel(new IOException("Feed download abandoned")); + } + response.cancel(true); + } + + // a property from roller.properties, or the default if it isn't positive private static int positiveProperty(String name, int defaultValue) { try { @@ -298,7 +330,8 @@ public class RomeFeedFetcher implements FeedFetcher { } - // collects a response body, failing once it grows past maxBytes + // collects a successful response's body, failing once it grows past + // maxBytes or once the fetcher stops waiting for it private static final class LimitedBodySubscriber implements HttpResponse.BodySubscriber<byte[]> { @@ -306,17 +339,27 @@ public class RomeFeedFetcher implements FeedFetcher { private final ByteArrayOutputStream body = new ByteArrayOutputStream(); private final long maxBytes; private final long contentLength; + private final int statusCode; + private final AtomicBoolean abandoned; private Flow.Subscription subscription; + private boolean cancelled; - LimitedBodySubscriber(long maxBytes, long contentLength) { + LimitedBodySubscriber(long maxBytes, long contentLength, int statusCode, + AtomicBoolean abandoned) { this.maxBytes = maxBytes; this.contentLength = contentLength; + this.statusCode = statusCode; + this.abandoned = abandoned; } @Override - public void onSubscribe(Flow.Subscription subscription) { + public synchronized void onSubscribe(Flow.Subscription subscription) { this.subscription = subscription; - if (contentLength > maxBytes) { + if (abandoned.get()) { + cancel(new IOException("Feed download abandoned")); + } else if (statusCode < 200 || statusCode > 299) { + cancel(new IOException("Feed request returned HTTP status " + statusCode)); + } else if (contentLength > maxBytes) { tooLarge(); } else { subscription.request(Long.MAX_VALUE); @@ -324,8 +367,8 @@ public class RomeFeedFetcher implements FeedFetcher { } @Override - public void onNext(List<ByteBuffer> buffers) { - if (result.isDone()) { + public synchronized void onNext(List<ByteBuffer> buffers) { + if (cancelled || result.isDone()) { return; } for (ByteBuffer buffer : buffers) { @@ -355,9 +398,16 @@ public class RomeFeedFetcher implements FeedFetcher { } private void tooLarge() { - subscription.cancel(); - result.completeExceptionally(new IOException( - "Feed is larger than " + maxBytes + " bytes")); + cancel(new IOException("Feed is larger than " + maxBytes + " bytes")); + } + + // stops reading the body; safe to call from any thread + synchronized void cancel(IOException reason) { + if (!cancelled && subscription != null) { + cancelled = true; + subscription.cancel(); + } + result.completeExceptionally(reason); } } diff --git a/app/src/test/java/org/apache/roller/planet/business/fetcher/RomeFeedFetcherLimitsTest.java b/app/src/test/java/org/apache/roller/planet/business/fetcher/RomeFeedFetcherLimitsTest.java index 3684bc09d..205fa48e1 100644 --- a/app/src/test/java/org/apache/roller/planet/business/fetcher/RomeFeedFetcherLimitsTest.java +++ b/app/src/test/java/org/apache/roller/planet/business/fetcher/RomeFeedFetcherLimitsTest.java @@ -28,8 +28,10 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -61,6 +63,8 @@ class RomeFeedFetcherLimitsTest { private static HttpServer server; private static ExecutorService executor; private static volatile boolean stopping; + // counted down when the client closes the /trickle connection + private static final CountDownLatch trickleClosed = new CountDownLatch(1); private final RomeFeedFetcher fetcher = new RomeFeedFetcher(MAX_BYTES, MAX_TIME); @@ -83,6 +87,15 @@ class RomeFeedFetcherLimitsTest { server.createContext("/large-chunked", exchange -> send(exchange, feed(MAX_BYTES * 16), false)); server.createContext("/drip", RomeFeedFetcherLimitsTest::drip); server.createContext("/stall", RomeFeedFetcherLimitsTest::stall); + server.createContext("/trickle", RomeFeedFetcherLimitsTest::trickle); + server.createContext("/missing", exchange -> { + // a feed body, so only the status says the request failed + byte[] body = feed(1024); + exchange.sendResponseHeaders(404, body.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(body); + } + }); server.start(); } @@ -133,20 +146,38 @@ class RomeFeedFetcherLimitsTest { } } + // sends headers, then a byte every 100 ms until the client goes away + private static void trickle(HttpExchange exchange) throws IOException { + exchange.sendResponseHeaders(200, 0); + try (OutputStream out = exchange.getResponseBody()) { + out.write(HEAD.getBytes(StandardCharsets.UTF_8)); + while (!stopping) { + out.write('x'); + out.flush(); + Thread.sleep(100); + } + } catch (IOException e) { + trickleClosed.countDown(); + } catch (InterruptedException e) { + // server stopping + } + } + private static String url(String path) { return "http://" + server.getAddress().getHostString() + ":" + server.getAddress().getPort() + path; } - // fetching fails within a few seconds, with the cause's message - // containing the expected text + // fetching fails well before the server would finish, with the cause's + // message containing the expected text. The bound leaves room for a slow + // build machine; the slow feeds would take minutes. private void assertFailsFast(String path, String message) { long start = System.nanoTime(); - FetcherException ex = assertTimeoutPreemptively(Duration.ofSeconds(10), + FetcherException ex = assertTimeoutPreemptively(Duration.ofSeconds(30), () -> assertThrows(FetcherException.class, () -> fetcher.fetchSubscription(url(path)))); Duration elapsed = Duration.ofNanos(System.nanoTime() - start); - assertTrue(elapsed.compareTo(MAX_TIME.plusSeconds(2)) < 0, + assertTrue(elapsed.compareTo(MAX_TIME.plusSeconds(10)) < 0, "took " + elapsed); Throwable cause = ex.getRootCause() != null ? ex.getRootCause() : ex; assertTrue(cause.getMessage() != null @@ -178,6 +209,38 @@ class RomeFeedFetcherLimitsTest { assertFailsFast("/stall", "not received within 2 seconds"); } + @Test + void abandonedFeedIsNoLongerDownloaded() throws Exception { + assertFailsFast("/trickle", "not received within 2 seconds"); + assertTrue(trickleClosed.await(10, TimeUnit.SECONDS), + "the fetcher kept downloading the feed after giving up"); + } + + @Test + void unsuccessfulStatusFails() { + assertFailsFast("/missing", "HTTP status 404"); + } + + @Test + void malformedUrlFails() { + FetcherException ex = assertThrows(FetcherException.class, + () -> fetcher.fetchSubscription("http://exa mple.com/feed")); + assertTrue(ex.getMessage().contains("exa mple.com"), ex.getMessage()); + } + + @Test + void interruptIsKept() { + Thread.currentThread().interrupt(); + try { + assertThrows(FetcherException.class, + () -> fetcher.fetchSubscription(url("/stall"))); + assertTrue(Thread.currentThread().isInterrupted(), + "the interrupt was lost"); + } finally { + Thread.interrupted(); + } + } + @Test void onlyHttpAndHttpsUrlsAreFetched() { for (String url : new String[] {"file:///etc/hosts", "ftp://localhost/feed",
