This is an automated email from the ASF dual-hosted git repository.
markt-asf pushed a commit to branch 10.1.x
in repository https://gitbox.apache.org/repos/asf/tomcat.git
The following commit(s) were added to refs/heads/10.1.x by this push:
new 1d680ff24d HTTP/2 error handling improvements
1d680ff24d is described below
commit 1d680ff24d4710842eee6ea8dd45e1f74fd63e2f
Author: Mark Thomas <[email protected]>
AuthorDate: Fri Nov 20 15:50:30 2026 +0000
HTTP/2 error handling improvements
---
.../apache/coyote/http2/Http2UpgradeHandler.java | 2 +-
java/org/apache/coyote/http2/Stream.java | 74 ++++
java/org/apache/coyote/http2/StreamProcessor.java | 32 +-
test/org/apache/coyote/http2/TestStream.java | 306 +++++++++++++-
.../http2/TestStreamProcessorAsyncStreamError.java | 463 +++++++++++++++++++++
webapps/docs/changelog.xml | 10 +
6 files changed, 882 insertions(+), 5 deletions(-)
diff --git a/java/org/apache/coyote/http2/Http2UpgradeHandler.java
b/java/org/apache/coyote/http2/Http2UpgradeHandler.java
index 10e964a9c7..8aa99f1d08 100644
--- a/java/org/apache/coyote/http2/Http2UpgradeHandler.java
+++ b/java/org/apache/coyote/http2/Http2UpgradeHandler.java
@@ -480,7 +480,7 @@ class Http2UpgradeHandler extends AbstractStream implements
InternalHttpUpgradeH
if (stream == null) {
sendStreamReset(null, se);
} else {
- stream.close(se);
+ stream.closeOnStreamError(se);
}
} finally {
if (isOverheadLimitExceeded()) {
diff --git a/java/org/apache/coyote/http2/Stream.java
b/java/org/apache/coyote/http2/Stream.java
index 496f395ad5..8237997a38 100644
--- a/java/org/apache/coyote/http2/Stream.java
+++ b/java/org/apache/coyote/http2/Stream.java
@@ -55,6 +55,7 @@ import org.apache.tomcat.util.http.parser.Host;
import org.apache.tomcat.util.http.parser.HttpParser;
import org.apache.tomcat.util.http.parser.Priority;
import org.apache.tomcat.util.net.ApplicationBufferHandler;
+import org.apache.tomcat.util.net.SocketEvent;
import org.apache.tomcat.util.net.WriteBuffer;
import org.apache.tomcat.util.res.StringManager;
@@ -113,6 +114,11 @@ class Stream extends AbstractNonZeroStream implements
HeaderEmitter {
private final Object recycledLock = new Object();
private volatile boolean recycled = false;
+ // The processor handling the request for this stream. Set when the
processor is created.
+ private volatile StreamProcessor streamProcessor = null;
+ // A stream error raised by the connection thread while the processor was
still processing the request.
+ private volatile StreamException streamError = null;
+ private final AtomicBoolean streamErrorNotified = new AtomicBoolean(false);
Stream(Integer identifier, Http2UpgradeHandler handler) {
@@ -886,6 +892,74 @@ class Stream extends AbstractNonZeroStream implements
HeaderEmitter {
}
+ /**
+ * Closes the stream following a stream error detected by the connection
thread while processing incoming frames.
+ * <p>
+ * If a {@link StreamProcessor} is still processing the request, the
request is, or may become, asynchronous. A
+ * waiting processor can only complete its clean-up (remove itself from
the waiting processors and recycle itself)
+ * when it processes an event so it must be notified of the error. If the
request is already in asynchronous mode
+ * that is done here. Otherwise it is done by the processor once the
request becomes asynchronous.
+ *
+ * @param streamException The stream error
+ */
+ @SuppressWarnings("null") // processor is not null when it is referenced
+ final void closeOnStreamError(StreamException streamException) {
+ StreamProcessor processor = streamProcessor;
+ close(streamException);
+ if (processor != null && !recycled) {
+ /*
+ * Record the error before checking if the request is
asynchronous. The processor checks for a recorded
+ * error after it has started asynchronous processing (see
notifyProcessorOfStreamError()). With both
+ * fields volatile, at least one of the two threads is guaranteed
to see the other's update so the processor
+ * is always notified, whichever thread wins the race.
+ */
+ streamError = streamException;
+ if (processor.isAsync()) {
+ notifyProcessorOfStreamError(processor);
+ }
+ }
+ }
+
+
+ /**
+ * Called by the {@link StreamProcessor} once the request has started
asynchronous processing. If the connection
+ * thread raised a stream error before asynchronous processing started it
will not have notified the processor, so
+ * do so now. Without the notification, nothing would clean up the
processor until the asynchronous timeout fired
+ * (or never, if there is no timeout).
+ *
+ * @param processor The processor for this stream
+ */
+ final void notifyProcessorOfStreamError(StreamProcessor processor) {
+ // Notify once only, whichever thread gets here first
+ if (streamError != null && !recycled &&
streamErrorNotified.compareAndSet(false, true)) {
+ /*
+ * The request is deliberately not touched here. This may be
called from a thread that does not own the
+ * request and the attributes of the request are not thread-safe.
The processor sets the error attribute on
+ * the request when it processes the event (see
StreamProcessor.process()).
+ *
+ * If the processor has already cleaned up before the event is
processed, it ignores the event.
+ */
+ processor.processSocketEvent(SocketEvent.ERROR, true);
+ }
+ }
+
+
+ /**
+ * Obtain the stream error, if any, that was raised by the connection
thread while the request for this stream was
+ * still being processed.
+ *
+ * @return The stream error or {@code null}
+ */
+ final StreamException getStreamError() {
+ return streamError;
+ }
+
+
+ final void setStreamProcessor(StreamProcessor streamProcessor) {
+ this.streamProcessor = streamProcessor;
+ }
+
+
final void close(Http2Exception http2Exception) {
if (http2Exception instanceof ConnectionException) {
handler.closeConnection(http2Exception);
diff --git a/java/org/apache/coyote/http2/StreamProcessor.java
b/java/org/apache/coyote/http2/StreamProcessor.java
index 6152ec36f7..2c5733e3d3 100644
--- a/java/org/apache/coyote/http2/StreamProcessor.java
+++ b/java/org/apache/coyote/http2/StreamProcessor.java
@@ -63,6 +63,8 @@ class StreamProcessor extends AbstractProcessor implements
NonPipeliningProcesso
private final Lock processLock = new ReentrantLock();
private final Http2UpgradeHandler handler;
private final Stream stream;
+ // True while this processor is registered as a waiting processor. Only
modified while holding processLock.
+ private volatile boolean waiting = false;
private SendfileData sendfileData = null;
private SendfileState sendfileState = null;
@@ -77,6 +79,7 @@ class StreamProcessor extends AbstractProcessor implements
NonPipeliningProcesso
super(adapter, stream.getCoyoteRequest(), stream.getCoyoteResponse());
this.handler = handler;
this.stream = stream;
+ stream.setStreamProcessor(this);
setSocketWrapper(socketWrapper);
}
@@ -93,7 +96,13 @@ class StreamProcessor extends AbstractProcessor implements
NonPipeliningProcesso
* replaced it should not be used for processing any further
events. When it is known that processing is
* going to be a NO-OP, exit early.
*/
- if (!stream.equals(handler.getStream(stream.getIdAsInt()))) {
+ /*
+ * The exception is a processor that is registered as a
waiting processor (asynchronous processing).
+ * The connection thread replaces the stream when it raises a
stream error. A waiting processor can only
+ * complete its clean-up (remove itself from the waiting
processors and recycle itself) by processing
+ * an event so it must not ignore events just because the
stream has been replaced.
+ */
+ if (!waiting &&
!stream.equals(handler.getStream(stream.getIdAsInt()))) {
return;
}
// Avoid duplicate timeout events
@@ -103,6 +112,17 @@ class StreamProcessor extends AbstractProcessor implements
NonPipeliningProcesso
SocketState state = SocketState.CLOSED;
boolean reuseRequestAndResponse = false;
try {
+ if (event == SocketEvent.ERROR) {
+ /*
+ * If the connection thread raised a stream error
while the request was being processed, make
+ * the error visible to the application. It is set
here, on the thread that owns the request,
+ * and not by the connection thread, since the
attributes of the request are not thread-safe.
+ */
+ StreamException streamError = stream.getStreamError();
+ if (streamError != null &&
request.getAttribute(RequestDispatcher.ERROR_EXCEPTION) == null) {
+
request.setAttribute(RequestDispatcher.ERROR_EXCEPTION, streamError);
+ }
+ }
state = process(socketWrapper, event);
/*
* The request and response may only be returned to the
pool for re-use if the stream ended cleanly.
@@ -117,14 +137,20 @@ class StreamProcessor extends AbstractProcessor
implements NonPipeliningProcesso
}
if (state == SocketState.LONG) {
+ waiting = true;
handler.getProtocol().getHttp11Protocol().addWaitingProcessor(this);
+ // The connection thread may have raised a stream
error before the request became async
+ stream.notifyProcessorOfStreamError(this);
} else if (state == SocketState.CLOSED) {
+ waiting = false;
handler.getProtocol().getHttp11Protocol().removeWaitingProcessor(this);
- if (!stream.isInputFinished() &&
getErrorState().isIoAllowed()) {
+ if (!stream.isInputFinished() &&
getErrorState().isIoAllowed() && stream.canReset()) {
// The request has been processed but the request
body has not been
// fully read. This typically occurs when Tomcat
rejects an upload
// of some form (e.g. PUT or POST). Need to tell
the client not to
- // send any more data on this stream (reset).
+ // send any more data on this stream (reset). If
the stream can no
+ // longer be reset (e.g. a reset has already been
sent) there is
+ // nothing to send. The final else block performs
the clean-up.
StreamException se =
new
StreamException(sm.getString("streamProcessor.cancel", stream.getConnectionId(),
stream.getIdAsString()),
Http2Error.NO_ERROR, stream.getIdAsInt());
diff --git a/test/org/apache/coyote/http2/TestStream.java
b/test/org/apache/coyote/http2/TestStream.java
index 5503d3be05..2e238d6f22 100644
--- a/test/org/apache/coyote/http2/TestStream.java
+++ b/test/org/apache/coyote/http2/TestStream.java
@@ -18,7 +18,21 @@ package org.apache.coyote.http2;
import java.io.IOException;
import java.nio.ByteBuffer;
-
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import jakarta.servlet.RequestDispatcher;
+import jakarta.servlet.ServletConnection;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServlet;
import jakarta.servlet.http.HttpServletRequest;
@@ -29,6 +43,11 @@ import org.junit.Test;
import org.apache.catalina.Context;
import org.apache.catalina.startup.Tomcat;
+import org.apache.coyote.ActionCode;
+import org.apache.coyote.AsyncContextCallback;
+import org.apache.tomcat.util.net.SocketEvent;
+import org.apache.tomcat.util.net.SocketWrapperBase;
+import org.easymock.EasyMock;
import trailers.ResponseTrailers;
@@ -114,6 +133,291 @@ public class TestStream extends Http2TestBase {
}
+ @Test
+ public void testErrorDoesNotNotifyRecycledRequest() throws Exception {
+ SocketWrapperBase<?> socket =
EasyMock.createNiceMock(SocketWrapperBase.class);
+ ServletConnection connection =
EasyMock.createNiceMock(ServletConnection.class);
+ AsyncContextCallback callback =
EasyMock.createNiceMock(AsyncContextCallback.class);
+ EasyMock.expect(socket.getServletConnection("h2c",
"")).andStubReturn(connection);
+ EasyMock.expect(connection.getConnectionId()).andStubReturn("0");
+ EasyMock.replay(socket, connection, callback);
+
+ CountDownLatch resetStarted = new CountDownLatch(1);
+ CountDownLatch resumeReset = new CountDownLatch(1);
+ Http2Protocol protocol = new Http2Protocol();
+ Http2UpgradeHandler handler = new Http2UpgradeHandler(protocol, null,
null, socket) {
+ @Override
+ void sendStreamReset(StreamStateMachine state, StreamException
error) throws IOException {
+ // Pause after closeOnStreamError() has checked the
processor's async state.
+ resetStarted.countDown();
+ try {
+ Assert.assertTrue("Reset was not released",
resumeReset.await(15, TimeUnit.SECONDS));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException(e);
+ }
+ state.sendReset();
+ }
+ };
+ Stream original = new Stream(Integer.valueOf(3), handler);
+ original.state.receivedStartOfHeaders();
+ original.receivedEndOfStream();
+ AtomicBoolean errorNotified = new AtomicBoolean();
+ StreamProcessor processor = new StreamProcessor(handler, original,
null, socket) {
+ @Override
+ protected void processSocketEvent(SocketEvent event, boolean
dispatch) {
+ if (event == SocketEvent.ERROR) {
+ errorNotified.set(true);
+ }
+ }
+ };
+ processor.action(ActionCode.ASYNC_START, callback);
+ processor.asyncPostProcess();
+ Assert.assertTrue(processor.isAsync());
+
+ AtomicReference<Throwable> failure = new AtomicReference<>();
+ StreamException error = new StreamException("late DATA",
Http2Error.STREAM_CLOSED, 3);
+ Thread connectionThread = new Thread(() -> {
+ try {
+ original.closeOnStreamError(error);
+ } catch (Throwable t) {
+ failure.set(t);
+ }
+ });
+ Stream next;
+ connectionThread.start();
+ try {
+ Assert.assertTrue("Reset did not start", resetStarted.await(15,
TimeUnit.SECONDS));
+
+ // Complete and recycle the async request while the connection
thread is sending the reset.
+ original.state.sentEndOfStream();
+ processor.action(ActionCode.ASYNC_COMPLETE, null);
+ processor.asyncPostProcess();
+ Assert.assertFalse(processor.isAsync());
+ processor.recycle();
+ original.replace();
+ original.recycle(true);
+
+ // Another connection can re-use the request from the protocol's
shared pool.
+ Http2UpgradeHandler otherConnection = new
Http2UpgradeHandler(protocol, null, null, socket);
+ next = new Stream(Integer.valueOf(5), otherConnection);
+ Assert.assertSame(original.getCoyoteRequest(),
next.getCoyoteRequest());
+
Assert.assertNull(next.getCoyoteRequest().getAttribute(RequestDispatcher.ERROR_EXCEPTION));
+ } finally {
+ resumeReset.countDown();
+ connectionThread.join(15000);
+ }
+ Assert.assertFalse("Connection thread did not finish",
connectionThread.isAlive());
+ Assert.assertNull(failure.get());
+ Assert.assertNull("The old stream error must not be set on the re-used
request",
+
next.getCoyoteRequest().getAttribute(RequestDispatcher.ERROR_EXCEPTION));
+ Assert.assertFalse("The recycled processor must not receive an error
event", errorNotified.get());
+ }
+
+
+ private static final int STRESS_ITERATIONS = 20000;
+
+
+ /*
+ * Stress test for the race between the connection thread, which raises a
stream error, and the thread processing
+ * the request, which starts asynchronous processing. Whichever thread
wins, the processor must be notified of the
+ * error once, and only once.
+ */
+ @Test
+ public void testStreamErrorNotificationRaceStress() throws Exception {
+ SocketWrapperBase<?> socket = createStressSocket();
+ AsyncContextCallback callback =
EasyMock.createNiceMock(AsyncContextCallback.class);
+ EasyMock.replay(callback);
+ Http2UpgradeHandler handler = createStressHandler(socket);
+ StreamException error = new StreamException("late DATA",
Http2Error.STREAM_CLOSED, 3);
+
+ ExecutorService executor = Executors.newFixedThreadPool(2);
+ try {
+ for (int i = 0; i < STRESS_ITERATIONS; i++) {
+ Stream stream = new Stream(Integer.valueOf(3), handler);
+ Assert.assertNull("Request re-used with a stale error on
iteration " + i,
+
stream.getCoyoteRequest().getAttribute(RequestDispatcher.ERROR_EXCEPTION));
+ stream.state.receivedStartOfHeaders();
+ stream.receivedEndOfStream();
+ AtomicInteger errorEvents = new AtomicInteger();
+ StreamProcessor processor = createStressProcessor(handler,
stream, socket, errorEvents);
+ CyclicBarrier barrier = new CyclicBarrier(2);
+
+ // Connection thread
+ Future<?> connection = executor.submit(() -> {
+ awaitBarrier(barrier);
+ jitter();
+ stream.closeOnStreamError(error);
+ });
+ // Thread processing the request. Mirrors
StreamProcessor.process() when the request becomes async.
+ Future<?> processing = executor.submit(() -> {
+ awaitBarrier(barrier);
+ jitter();
+ startStressAsync(processor, callback);
+ stream.notifyProcessorOfStreamError(processor);
+ });
+ connection.get(15, TimeUnit.SECONDS);
+ processing.get(15, TimeUnit.SECONDS);
+
+ Assert.assertEquals("Iteration " + i, 1, errorEvents.get());
+ // The threads notifying the processor must not write to the
request. The processor does that when it
+ // processes the event on the thread that owns the request.
+ Assert.assertNull("Iteration " + i,
+
stream.getCoyoteRequest().getAttribute(RequestDispatcher.ERROR_EXCEPTION));
+
+ // Clean up so the request can be re-used by the next iteration
+ cleanUpStressProcessor(processor, stream);
+ }
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
+
+ /*
+ * Stress test for the race between the connection thread, which raises a
stream error for an asynchronous request,
+ * and the thread completing and recycling the request. The connection
thread must never write to a request that
+ * has been recycled (and may have been re-used by another stream).
+ */
+ @Test
+ public void testStreamErrorRecycleRaceStress() throws Exception {
+ SocketWrapperBase<?> socket = createStressSocket();
+ AsyncContextCallback callback =
EasyMock.createNiceMock(AsyncContextCallback.class);
+ EasyMock.replay(callback);
+ Http2UpgradeHandler handler = createStressHandler(socket);
+ StreamException error = new StreamException("late DATA",
Http2Error.STREAM_CLOSED, 3);
+
+ int notifiedBeforeRecycle = 0;
+ int skipped = 0;
+ ExecutorService executor = Executors.newFixedThreadPool(2);
+ try {
+ for (int i = 0; i < STRESS_ITERATIONS; i++) {
+ Stream stream = new Stream(Integer.valueOf(3), handler);
+ // Any error set on this request after it was recycled would
still be visible here
+ Assert.assertNull("Request re-used with a stale error on
iteration " + i,
+
stream.getCoyoteRequest().getAttribute(RequestDispatcher.ERROR_EXCEPTION));
+ stream.state.receivedStartOfHeaders();
+ stream.receivedEndOfStream();
+ AtomicInteger errorEvents = new AtomicInteger();
+ StreamProcessor processor = createStressProcessor(handler,
stream, socket, errorEvents);
+ startStressAsync(processor, callback);
+ CyclicBarrier barrier = new CyclicBarrier(2);
+
+ Future<?> connection = executor.submit(() -> {
+ awaitBarrier(barrier);
+ stream.closeOnStreamError(error);
+ });
+ Future<?> processing = executor.submit(() -> {
+ awaitBarrier(barrier);
+ // Closing the stream takes longer than the clean-up so
delay the clean-up to get a good mix of
+ // outcomes
+ jitter(60000);
+ cleanUpStressProcessor(processor, stream);
+ });
+ connection.get(15, TimeUnit.SECONDS);
+ processing.get(15, TimeUnit.SECONDS);
+
+ Assert.assertTrue("Iteration " + i, errorEvents.get() <= 1);
+ if (errorEvents.get() == 1) {
+ notifiedBeforeRecycle++;
+ } else {
+ skipped++;
+ }
+ }
+ } finally {
+ executor.shutdownNow();
+ }
+ // Not an assertion on the exact split (it depends on timing) but both
outcomes should have been exercised
+ // for the stress test to be meaningful.
+ System.out.println("Recycle race stress test: notified [" +
notifiedBeforeRecycle + "], skipped [" + skipped + "]");
+ }
+
+
+ private SocketWrapperBase<?> createStressSocket() {
+ SocketWrapperBase<?> socket =
EasyMock.createNiceMock(SocketWrapperBase.class);
+ ServletConnection connection =
EasyMock.createNiceMock(ServletConnection.class);
+ EasyMock.expect(socket.getServletConnection("h2c",
"")).andStubReturn(connection);
+ EasyMock.expect(connection.getConnectionId()).andStubReturn("0");
+ EasyMock.replay(socket, connection);
+ return socket;
+ }
+
+
+ private Http2UpgradeHandler createStressHandler(SocketWrapperBase<?>
socket) {
+ return new Http2UpgradeHandler(new Http2Protocol(), null, null,
socket) {
+ @Override
+ void sendStreamReset(StreamStateMachine state, StreamException
error) throws IOException {
+ state.sendReset();
+ }
+ };
+ }
+
+
+ private StreamProcessor createStressProcessor(Http2UpgradeHandler handler,
Stream stream,
+ SocketWrapperBase<?> socket, AtomicInteger errorEvents) {
+ return new StreamProcessor(handler, stream, null, socket) {
+ @Override
+ protected void processSocketEvent(SocketEvent event, boolean
dispatch) {
+ if (event == SocketEvent.ERROR) {
+ errorEvents.incrementAndGet();
+ }
+ }
+ };
+ }
+
+
+ /*
+ * Mirrors what StreamProcessor.process() does when asynchronous
processing completes.
+ */
+ private void cleanUpStressProcessor(StreamProcessor processor, Stream
stream) {
+ stream.state.sentEndOfStream();
+ processor.action(ActionCode.ASYNC_COMPLETE, null);
+ try {
+ processor.asyncPostProcess();
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ processor.recycle();
+ stream.replace();
+ stream.recycle(true);
+ }
+
+
+ private void startStressAsync(StreamProcessor processor,
AsyncContextCallback callback) {
+ processor.action(ActionCode.ASYNC_START, callback);
+ try {
+ processor.asyncPostProcess();
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+
+ /*
+ * Random delay so that, over many iterations, the two threads interleave
in many different ways.
+ */
+ private static void jitter() {
+ jitter(3000);
+ }
+
+
+ private static void jitter(int maxSpins) {
+ int spins = ThreadLocalRandom.current().nextInt(maxSpins);
+ for (int i = 0; i < spins; i++) {
+ Thread.onSpinWait();
+ }
+ }
+
+
+ private static void awaitBarrier(CyclicBarrier barrier) {
+ try {
+ barrier.await(15, TimeUnit.SECONDS);
+ } catch (InterruptedException | BrokenBarrierException |
TimeoutException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+
private static final class PathParam extends HttpServlet {
private static final long serialVersionUID = 1L;
diff --git
a/test/org/apache/coyote/http2/TestStreamProcessorAsyncStreamError.java
b/test/org/apache/coyote/http2/TestStreamProcessorAsyncStreamError.java
new file mode 100644
index 0000000000..2dbe1ab311
--- /dev/null
+++ b/test/org/apache/coyote/http2/TestStreamProcessorAsyncStreamError.java
@@ -0,0 +1,463 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.coyote.http2;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import jakarta.servlet.AsyncContext;
+import jakarta.servlet.AsyncEvent;
+import jakarta.servlet.AsyncListener;
+import jakarta.servlet.ServletException;
+import jakarta.servlet.http.HttpServlet;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletResponse;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.catalina.Context;
+import org.apache.catalina.Wrapper;
+import org.apache.catalina.startup.Tomcat;
+import org.apache.coyote.AbstractProtocol;
+
+/**
+ * Tests that the resources associated with a request that is in asynchronous
mode are released when the connection
+ * thread raises a stream level error for the stream.
+ */
+public class TestStreamProcessorAsyncStreamError extends Http2TestBase {
+
+ // Generous so slow CI systems do not cause false failures. The test
returns as soon as the expected state is seen.
+ private static final long WAIT_LIMIT = 15000;
+
+ private long asyncTimeout = 2000;
+ private long maxConcurrentStreams = 200;
+ private int maxConcurrentStreamExecution = -1;
+ private final CountDownLatch blockStarted = new CountDownLatch(1);
+ private final CountDownLatch blockLatch = new CountDownLatch(1);
+ private final CountDownLatch delayedInService = new CountDownLatch(1);
+ private final CountDownLatch delayedRelease = new CountDownLatch(1);
+ private final CountDownLatch asyncCompleted = new CountDownLatch(1);
+ private final CountDownLatch syncInService = new CountDownLatch(1);
+ private final CountDownLatch syncRelease = new CountDownLatch(1);
+
+ private final AtomicReference<AsyncContext> asyncContext = new
AtomicReference<>();
+ private final AtomicReference<Throwable> asyncError = new
AtomicReference<>();
+
+
+ /*
+ * Control. No error. The async timeout fires and the application
completes the request.
+ */
+ @Test
+ public void testTimeoutWithoutStreamError() throws Exception {
+ startAsyncRequest();
+ waitForWaitingProcessors(getBaseline());
+ }
+
+
+ /*
+ * An empty DATA frame on a half-closed (remote) stream is a stream error
that is detected by the connection thread.
+ * The async timeout must still be processed so the processor is removed
from the waiting processors and recycled.
+ */
+ @Test
+ public void testStreamErrorThenTimeout() throws Exception {
+ startAsyncRequest();
+ sendStreamError();
+ waitForWaitingProcessors(getBaseline());
+ assertApplicationSawStreamError();
+ }
+
+
+ /*
+ * As above but the application completes the request after the stream
error rather than waiting for the timeout.
+ *
+ * Only one stream may be processed at a time and that is occupied, so the
ERROR event for the stream error is
+ * queued. That makes the order deterministic: the application completes
the request before the container has
+ * processed the error, so complete() must succeed. Both events then have
to be handled correctly.
+ */
+ @Test
+ public void testStreamErrorThenComplete() throws Exception {
+ maxConcurrentStreamExecution = 1;
+ startAsyncRequest();
+
+ // Occupy the only execution slot
+ sendSimpleGetRequestTo(5, "/block");
+ Assert.assertTrue("Blocking request did not start",
+ blockStarted.await(WAIT_LIMIT, TimeUnit.MILLISECONDS));
+
+ // The ERROR event is queued because there is no free execution slot
+ sendStreamError();
+
+ AsyncContext ac = asyncContext.get();
+ Assert.assertNotNull(ac);
+ ac.complete();
+
+ // Free the execution slot. The queued events are now processed.
+ blockLatch.countDown();
+ waitForWaitingProcessors(getBaseline());
+ }
+
+
+ /*
+ * The ERROR event for a stream error is processed on a container thread.
If the number of streams being processed
+ * has reached the limit, the event is queued. While it is queued the
closed stream is replaced and may be pruned
+ * from the connection's stream map. The processor is a waiting processor
so it must still process the event when
+ * it is eventually run and clean up, even though its stream is no longer
in the map.
+ */
+ @Test
+ public void testStreamErrorThenStreamPrunedBeforeEventProcessed() throws
Exception {
+ // Retain at most 5 * 2 streams
+ maxConcurrentStreams = 2;
+ // Only one stream may be processed at a time
+ maxConcurrentStreamExecution = 1;
+ startAsyncRequest();
+
+ // Occupy the only execution slot
+ sendSimpleGetRequestTo(5, "/block");
+ Assert.assertTrue("Blocking request did not start",
+ blockStarted.await(WAIT_LIMIT, TimeUnit.MILLISECONDS));
+
+ // Stream error for the async stream. The ERROR event is queued
because there is no free execution slot.
+ sendStreamError();
+
+ // Open enough streams for the closed async stream to be a candidate
for pruning. Reset each one immediately so
+ // they don't count against the concurrent streams limit.
+ for (int streamId = 7; streamId < 7 + 2 * 30; streamId += 2) {
+ sendSimpleGetRequestTo(streamId, "/simple");
+ sendRst(streamId, Http2Error.CANCEL.getCode());
+ }
+ // The connection thread processes frames in order so, once it has
answered a ping, it has processed every
+ // frame sent before it (including the creation and pruning of the
streams above).
+ waitForPingAck();
+
+ // Free the execution slot. The queued events are now processed.
+ blockLatch.countDown();
+
+ // The blocking request ends and the stream 3 processor must clean up.
Expect only the connection's own
+ // processor to remain.
+ waitForWaitingProcessors(getBaseline());
+ }
+
+
+ /*
+ * The stream error is raised while the request is being processed but
before the application has started
+ * asynchronous processing. There is no async timeout so the processor
must be notified of the error, once the
+ * application has started asynchronous processing, else nothing would
ever clean it up.
+ */
+ @Test
+ public void testStreamErrorBeforeStartAsync() throws Exception {
+ asyncTimeout = 0;
+ startAsyncRequest("/delayed", false);
+ Assert.assertTrue("Request did not reach the servlet",
+ delayedInService.await(WAIT_LIMIT, TimeUnit.MILLISECONDS));
+
+ // The request is being processed but is not (yet) asynchronous
+ Assert.assertEquals(getBaseline(), waitingProcessors());
+ sendStreamError();
+
+ // Now allow the application to start asynchronous processing. There
is no timeout so the container can only
+ // complete the async processing by processing the error notification.
Waiting for that avoids checking the
+ // waiting processors before the request has entered async mode.
+ delayedRelease.countDown();
+ Assert.assertTrue("Asynchronous processing was never completed",
+ asyncCompleted.await(WAIT_LIMIT, TimeUnit.MILLISECONDS));
+
+ // The processor must have cleaned up
+ waitForWaitingProcessors(getBaseline());
+ assertApplicationSawStreamError();
+ }
+
+
+ /*
+ * The stream error is raised while the request is being processed and the
application never starts asynchronous
+ * processing. The processor must complete normally, replace the stream
and leave the connection usable without
+ * sending a second reset for the stream.
+ */
+ @Test
+ public void testStreamErrorWithoutAsync() throws Exception {
+ startAsyncRequest("/delayed-sync", false);
+ Assert.assertTrue("Request did not reach the servlet",
+ syncInService.await(WAIT_LIMIT, TimeUnit.MILLISECONDS));
+
+ sendStreamError();
+
+ // Let the application return without having used asynchronous
processing
+ syncRelease.countDown();
+
+ // The connection must still be usable
+ sendSimpleGetRequestTo(5, "/simple");
+ StringBuilder trace = new StringBuilder();
+ boolean ended = false;
+ int frames = 0;
+ while (!ended && frames++ < 100) {
+ output.clearTrace();
+ try {
+ parser.readFrame();
+ } catch (IOException ioe) {
+ Assert.fail("Connection closed before the response was
complete. Trace so far: " + trace + " " + ioe);
+ }
+ trace.append(output.getTrace());
+ ended = output.getTrace().contains("5-EndOfStream") ||
output.getTrace().contains("5-RST");
+ }
+ String result = trace.toString();
+ Assert.assertTrue(result, result.contains("5-EndOfStream"));
+ Assert.assertFalse(result, result.contains("5-RST"));
+ Assert.assertFalse(result, result.contains("Goaway"));
+ // sendStreamError() has already consumed the reset for stream 3.
There must not be another.
+ Assert.assertFalse(result, result.contains("3-RST"));
+
+ // No async processing so nothing should be left registered
+ Assert.assertEquals(getBaseline(), waitingProcessors());
+ }
+
+
+ /*
+ * The stream error is set as the error for the request by the thread that
processes the ERROR event so the
+ * application should have been told about it.
+ */
+ private void assertApplicationSawStreamError() {
+ Throwable t = asyncError.get();
+ Assert.assertTrue("The application should have been given the stream
error but got [" + t + "]",
+ t instanceof StreamException);
+ }
+
+
+ private void waitForPingAck() throws Exception {
+ byte[] payload = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 };
+ sendPing(0, false, payload);
+ String expected = "0-Ping-Ack-[1,2,3,4,5,6,7,8]";
+ StringBuilder trace = new StringBuilder();
+ int frames = 0;
+ while (frames++ < 1000) {
+ output.clearTrace();
+ try {
+ parser.readFrame();
+ } catch (IOException ioe) {
+ Assert.fail("Connection closed before the ping was
acknowledged. Trace so far: " + trace + " " + ioe);
+ }
+ trace.append(output.getTrace());
+ if (output.getTrace().contains(expected)) {
+ return;
+ }
+ }
+ Assert.fail("Ping was not acknowledged. Trace: " + trace);
+ }
+
+
+ private void sendSimpleGetRequestTo(int streamId, String path) throws
IOException {
+ byte[] frameHeader = new byte[9];
+ ByteBuffer headersPayload = ByteBuffer.allocate(128);
+ buildGetRequest(frameHeader, headersPayload, null, streamId, path);
+ writeFrame(frameHeader, headersPayload);
+ }
+
+
+ private int baseline = -1;
+
+ private int getBaseline() {
+ return baseline;
+ }
+
+
+ /*
+ * The number of stream processors that are registered as waiting
processors. The total number of waiting
+ * processors can't be used because it also includes the connection's own
processor which is added and removed as
+ * the connection is processed so it would make the checks in these tests
unreliable.
+ */
+ private int waitingProcessors() throws Exception {
+ Field field =
AbstractProtocol.class.getDeclaredField("waitingProcessors");
+ field.setAccessible(true);
+ Set<?> waiting = (Set<?>)
field.get(getTomcatInstance().getConnector().getProtocolHandler());
+ int count = 0;
+ for (Object processor : waiting) {
+ if (processor instanceof StreamProcessor) {
+ count++;
+ }
+ }
+ return count;
+ }
+
+
+ private void startAsyncRequest() throws Exception {
+ startAsyncRequest("/async", true);
+ }
+
+
+ private void startAsyncRequest(String path, boolean waitForAsync) throws
Exception {
+ enableHttp2(maxConcurrentStreams);
+ if (maxConcurrentStreamExecution > 0) {
+
http2Protocol.setMaxConcurrentStreamExecution(maxConcurrentStreamExecution);
+ // The test resets a lot of streams. Don't let the overhead
protection close the connection.
+ http2Protocol.setOverheadResetFactor(0);
+ }
+
+ Tomcat tomcat = getTomcatInstance();
+ Context ctxt = getProgrammaticRootContext();
+ // Target of the HTTP/2 upgrade request
+ Tomcat.addServlet(ctxt, "simple", new SimpleServlet());
+ ctxt.addServletMapping("/simple", "simple");
+ Wrapper w = Tomcat.addServlet(ctxt, "async", new
AsyncServlet(asyncContext, asyncTimeout, asyncCompleted, asyncError));
+ w.setAsyncSupported(true);
+ ctxt.addServletMapping("/async", "async");
+ Tomcat.addServlet(ctxt, "block", new HttpServlet() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected void doGet(HttpServletRequest req, HttpServletResponse
resp)
+ throws ServletException, IOException {
+ blockStarted.countDown();
+ try {
+ blockLatch.await(WAIT_LIMIT, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ // Ignore
+ }
+ }
+ });
+ ctxt.addServletMapping("/block", "block");
+ Wrapper delayed = Tomcat.addServlet(ctxt, "delayed", new HttpServlet()
{
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected void doGet(HttpServletRequest req, HttpServletResponse
resp)
+ throws ServletException, IOException {
+ // Not yet async. Wait until the test has triggered the stream
error.
+ delayedInService.countDown();
+ try {
+ delayedRelease.await(WAIT_LIMIT, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ // Ignore
+ }
+ new AsyncServlet(asyncContext, asyncTimeout, asyncCompleted,
asyncError).doGet(req, resp);
+ }
+ });
+ delayed.setAsyncSupported(true);
+ ctxt.addServletMapping("/delayed", "delayed");
+ Tomcat.addServlet(ctxt, "delayedSync", new HttpServlet() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected void doGet(HttpServletRequest req, HttpServletResponse
resp)
+ throws ServletException, IOException {
+ // Never goes async. Wait until the test has triggered the
stream error.
+ syncInService.countDown();
+ try {
+ syncRelease.await(WAIT_LIMIT, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ // Ignore
+ }
+ }
+ });
+ ctxt.addServletMapping("/delayed-sync", "delayedSync");
+ tomcat.start();
+
+ openClientConnection();
+ doHttpUpgrade();
+ sendClientPreface();
+ validateHttp2InitialResponse(maxConcurrentStreams);
+
+ // Only stream processors are counted
+ baseline = waitingProcessors();
+
+ byte[] frameHeader = new byte[9];
+ ByteBuffer headersPayload = ByteBuffer.allocate(128);
+ buildGetRequest(frameHeader, headersPayload, null, 3, path);
+ writeFrame(frameHeader, headersPayload);
+
+ if (!waitForAsync) {
+ return;
+ }
+
+ // Wait for the request to enter async mode
+ long end = System.currentTimeMillis() + WAIT_LIMIT;
+ while (waitingProcessors() <= baseline && System.currentTimeMillis() <
end) {
+ Thread.sleep(50);
+ }
+ Assert.assertTrue("Request did not enter async mode",
waitingProcessors() > baseline);
+ }
+
+
+ private void sendStreamError() throws Exception {
+ // Stream 3 is half-closed (remote). An empty DATA frame is a stream
error.
+ sendData(3, new byte[0]);
+
+ // Server should reset the stream
+ output.clearTrace();
+ parser.readFrame();
+ Assert.assertTrue(output.getTrace(),
output.getTrace().startsWith("3-RST-"));
+ }
+
+
+ private void waitForWaitingProcessors(int expected) throws Exception {
+ long end = System.currentTimeMillis() + WAIT_LIMIT;
+ while (waitingProcessors() != expected && System.currentTimeMillis() <
end) {
+ Thread.sleep(100);
+ }
+ Assert.assertEquals("Processor for the stream was not removed from the
waiting processors", expected,
+ waitingProcessors());
+ }
+
+
+ private static class AsyncServlet extends HttpServlet {
+
+ private static final long serialVersionUID = 1L;
+
+ private final transient AtomicReference<AsyncContext> asyncContext;
+ private final long asyncTimeout;
+ private final transient CountDownLatch asyncCompleted;
+ private final transient AtomicReference<Throwable> asyncError;
+
+ AsyncServlet(AtomicReference<AsyncContext> asyncContext, long
asyncTimeout, CountDownLatch asyncCompleted,
+ AtomicReference<Throwable> asyncError) {
+ this.asyncContext = asyncContext;
+ this.asyncTimeout = asyncTimeout;
+ this.asyncCompleted = asyncCompleted;
+ this.asyncError = asyncError;
+ }
+
+ @Override
+ protected void doGet(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
+ AsyncContext ac = req.startAsync();
+ ac.setTimeout(asyncTimeout);
+ ac.addListener(new AsyncListener() {
+ @Override
+ public void onComplete(AsyncEvent event) {
+ asyncCompleted.countDown();
+ }
+
+ @Override
+ public void onTimeout(AsyncEvent event) {
+ ac.complete();
+ }
+
+ @Override
+ public void onError(AsyncEvent event) {
+ asyncError.set(event.getThrowable());
+ }
+
+ @Override
+ public void onStartAsync(AsyncEvent event) {
+ }
+ });
+ asyncContext.set(ac);
+ }
+ }
+}
diff --git a/webapps/docs/changelog.xml b/webapps/docs/changelog.xml
index 9b91d133be..dbe18d2fb8 100644
--- a/webapps/docs/changelog.xml
+++ b/webapps/docs/changelog.xml
@@ -222,6 +222,16 @@
<fix>
Refactor the strict SNI check for consistency. (markt)
</fix>
+ <fix>
+ When using HTTP/2, do not send a second reset for a stream that has
+ already been reset when the request finishes processing with the
+ request body not fully read. (markt)
+ </fix>
+ <fix>
+ When using HTTP/2, ensure that the resources associated with a request
+ that is in asynchronous mode are released if the connection thread
+ raises a stream error for that stream. (markt)
+ </fix>
</changelog>
</subsection>
<subsection name="Jasper">
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]