This is an automated email from the ASF dual-hosted git repository.
olamy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/maven-surefire.git
The following commit(s) were added to refs/heads/master by this push:
new 1fdaa085c Dump and kill timed-out forks in parallel (#3551)
1fdaa085c is described below
commit 1fdaa085c1cac9a9c0452af0d73996ce785e41f1
Author: Olivier Lamy <[email protected]>
AuthorDate: Sun Nov 1 07:12:05 2026 +1000
Dump and kill timed-out forks in parallel (#3551)
* Dump and kill timed-out forks in parallel
When several forks time out together, each one now runs its timeout
extensions and gets killed on its own thread, so a slow thread dump
of one fork no longer delays the kill of the others.
Signed-off-by: Olivier Lamy <[email protected]>
* cleanup
Signed-off-by: Olivier Lamy <[email protected]>
* Fix missing jstack dump when forks time out together
Signed-off-by: Olivier Lamy <[email protected]>
---------
Signed-off-by: Olivier Lamy <[email protected]>
---
.../plugin/surefire/booterclient/ForkStarter.java | 15 +++-
.../surefire/booterclient/output/ForkClient.java | 41 ++++++----
.../extensions/timeout/JstackTimeoutExtension.java | 5 +-
.../booterclient/output/ForkClientTest.java | 87 +++++++++++++++++++++-
.../timeout/JstackTimeoutExtensionTest.java | 16 ++++
.../site/markdown/examples/timeout-extension.md | 2 +
6 files changed, 146 insertions(+), 20 deletions(-)
diff --git
a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/ForkStarter.java
b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/ForkStarter.java
index b3dac8d2a..76826121a 100644
---
a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/ForkStarter.java
+++
b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/ForkStarter.java
@@ -95,6 +95,7 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Collections.addAll;
import static java.util.UUID.randomUUID;
+import static java.util.concurrent.Executors.newCachedThreadPool;
import static java.util.concurrent.Executors.newScheduledThreadPool;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
@@ -155,6 +156,8 @@ public class ForkStarter {
private final ScheduledExecutorService timeoutCheckScheduler;
+ private final ExecutorService timeoutHandlerExecutor;
+
private final Queue<ForkClient> currentForkClients;
private final int forkedProcessTimeoutInSeconds;
@@ -277,6 +280,7 @@ public ForkStarter(
defaultReporterFactories = new ConcurrentLinkedQueue<>();
currentForkClients = new ConcurrentLinkedQueue<>();
timeoutCheckScheduler = createTimeoutCheckScheduler();
+ timeoutHandlerExecutor = createTimeoutHandlerExecutor();
timeoutExtensionDispatcher = new TimeoutExtensionDispatcher(log);
triggerTimeoutCheck();
}
@@ -292,6 +296,7 @@ public RunResult run(@Nonnull SurefireProperties
effectiveSystemProperties, @Non
reportMerger.close();
pingThreadScheduler.shutdownNow();
timeoutCheckScheduler.shutdownNow();
+ timeoutHandlerExecutor.shutdownNow();
timeoutExtensionDispatcher.close();
for (String line : logsAtEnd) {
log.warning(line);
@@ -930,6 +935,14 @@ private static ScheduledExecutorService
createTimeoutCheckScheduler() {
return newScheduledThreadPool(1, threadFactory);
}
+ /**
+ * One thread per timed-out fork, so a slow thread dump of one fork
doesn't delay killing the others.
+ */
+ private static ExecutorService createTimeoutHandlerExecutor() {
+ ThreadFactory threadFactory =
newDaemonThreadFactory("fork-timeout-handler");
+ return newCachedThreadPool(threadFactory);
+ }
+
private ScheduledFuture<?> triggerPingTimerForShutdown(final
TestLessInputStreamBuilder builder) {
return pingThreadScheduler.scheduleWithFixedDelay(
new Runnable() {
@@ -965,7 +978,7 @@ private ScheduledFuture<?> triggerTimeoutCheck() {
public void run() {
long systemTime = currentTimeMillis();
for (ForkClient forkClient : currentForkClients) {
- forkClient.tryToTimeout(systemTime,
forkedProcessTimeoutInSeconds);
+ forkClient.tryToTimeout(systemTime,
forkedProcessTimeoutInSeconds, timeoutHandlerExecutor);
}
}
},
diff --git
a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/output/ForkClient.java
b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/output/ForkClient.java
index 7f12f5a48..9ff370c7f 100644
---
a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/output/ForkClient.java
+++
b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/output/ForkClient.java
@@ -27,6 +27,8 @@
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.atomic.AtomicLong;
import
org.apache.maven.plugin.surefire.booterclient.lazytestprovider.NotifiableTestStream;
@@ -125,10 +127,10 @@ public void
setStopOnNextTestListener(ForkedProcessEventListener listener) {
}
/**
- * Registers a one-shot callback invoked synchronously when the forked
- * process timeout has been reached, immediately before the KILL command is
- * sent. The forked JVM is still alive when the callback runs, so it may
- * collect live diagnostics (for example a {@code jstack} dump).
+ * Sets a callback that runs once, just before a timed-out fork is killed.
+ * The fork is still alive then, so the callback can grab a {@code jstack}
dump.
+ * It runs on the executor given to {@link #tryToTimeout}, so callbacks of
+ * different forks can run at the same time.
*
* @param listener the callback to run, or {@code null} to clear; callback
* must not throw checked exceptions
@@ -301,30 +303,43 @@ public void kill() {
/**
* Called in concurrent Thread.
* Will shutdown if timeout was reached.
+ * <p>
+ * The listener and the kill run on {@code executor}, so a slow listener
+ * doesn't hold up other forks.
*
* @param currentTimeMillis current time in milliseconds
* @param forkedProcessTimeoutInSeconds timeout in seconds given by MOJO
+ * @param executor where the listener and the kill run
*/
- public void tryToTimeout(long currentTimeMillis, int
forkedProcessTimeoutInSeconds) {
+ public void tryToTimeout(long currentTimeMillis, int
forkedProcessTimeoutInSeconds, Executor executor) {
if (forkedProcessTimeoutInSeconds > 0) {
final long forkedProcessTimeoutInMillis = 1000L *
forkedProcessTimeoutInSeconds;
final long startedAt = testSetStartedAt.get();
if (startedAt > START_TIME_ZERO && currentTimeMillis - startedAt
>= forkedProcessTimeoutInMillis) {
if (testSetStartedAt.compareAndSet(startedAt,
START_TIME_NEGATIVE_TIMEOUT)) {
- Runnable listener = timeoutDetectedListener;
- if (listener != null) {
- try {
- listener.run();
- } catch (RuntimeException ignored) {
- // listener failures must never prevent the kill
- }
+ try {
+ executor.execute(this::timeout);
+ } catch (RejectedExecutionException e) {
+ // executor is shut down, so kill the fork right here
+ timeout();
}
- notifiableTestStream.shutdown(KILL);
}
}
}
}
+ private void timeout() {
+ Runnable listener = timeoutDetectedListener;
+ if (listener != null) {
+ try {
+ listener.run();
+ } catch (RuntimeException ignored) {
+ // listener failures must never prevent the kill
+ }
+ }
+ notifiableTestStream.shutdown(KILL);
+ }
+
public DefaultReporterFactory getDefaultReporterFactory() {
return defaultReporterFactory;
}
diff --git
a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/extensions/timeout/JstackTimeoutExtension.java
b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/extensions/timeout/JstackTimeoutExtension.java
index 7417b6668..1e18b6499 100644
---
a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/extensions/timeout/JstackTimeoutExtension.java
+++
b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/extensions/timeout/JstackTimeoutExtension.java
@@ -107,7 +107,7 @@ private static boolean
isEnabled(ForkedProcessTimeoutContext context) {
return fromCtx != null && Boolean.parseBoolean(fromCtx.trim());
}
- private static File resolveOutputDirectory(ForkedProcessTimeoutContext
context, ConsoleLogger logger) {
+ static File resolveOutputDirectory(ForkedProcessTimeoutContext context,
ConsoleLogger logger) {
String configured =
context.getExtensionContext().get(OUTPUT_LOCATION_KEY);
File target;
if (configured != null && !configured.trim().isEmpty()) {
@@ -115,7 +115,8 @@ private static File
resolveOutputDirectory(ForkedProcessTimeoutContext context,
} else {
target = context.getReportsDirectory();
}
- if (!target.isDirectory() && !target.mkdirs()) {
+ // Another fork may have just created it, so mkdirs() can return false
+ if (!target.mkdirs() && !target.isDirectory()) {
logger.warning("JstackTimeoutExtension: cannot create output
directory " + target.getAbsolutePath());
return null;
}
diff --git
a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/output/ForkClientTest.java
b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/output/ForkClientTest.java
index 1c36807cd..6b8d35f79 100644
---
a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/output/ForkClientTest.java
+++
b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/output/ForkClientTest.java
@@ -29,6 +29,9 @@
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -95,6 +98,7 @@
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
@@ -689,7 +693,7 @@ public void shouldSendTestsetStartingKilled() {
ForkClient client = new ForkClient(factory, notifiableTestStream, 0);
client.handleEvent(new TestsetStartingEvent(reportEntry));
- client.tryToTimeout(System.currentTimeMillis() + 1000L, 1);
+ client.tryToTimeout(System.currentTimeMillis() + 1000L, 1,
Runnable::run);
verify(notifiableTestStream).shutdown(Shutdown.KILL);
verifyNoMoreInteractions(notifiableTestStream);
@@ -772,7 +776,7 @@ public void shouldSendTestsetStarting() {
ForkClient client = new ForkClient(factory, notifiableTestStream, 0);
client.handleEvent(new TestsetStartingEvent(reportEntry));
- client.tryToTimeout(System.currentTimeMillis(), 1);
+ client.tryToTimeout(System.currentTimeMillis(), 1, Runnable::run);
verifyNoInteractions(notifiableTestStream);
verify(factory).createTestReportListener();
@@ -1629,8 +1633,8 @@ public void timeoutListenerInvokedBeforeKillAndOnlyOnce()
{
client.handleEvent(new TestsetStartingEvent(reportEntry));
- client.tryToTimeout(System.currentTimeMillis() + 1000L, 1);
- client.tryToTimeout(System.currentTimeMillis() + 2000L, 1);
+ client.tryToTimeout(System.currentTimeMillis() + 1000L, 1,
Runnable::run);
+ client.tryToTimeout(System.currentTimeMillis() + 2000L, 1,
Runnable::run);
assertThat(listenerCalls).hasValue(1);
verify(notifiableTestStream).shutdown(Shutdown.KILL);
@@ -1638,4 +1642,79 @@ public void
timeoutListenerInvokedBeforeKillAndOnlyOnce() {
assertThat(order).containsExactly("listener", "shutdown");
assertThat(client.hadTimeout()).isTrue();
}
+
+ @Test
+ public void slowTimeoutListenerDoesNotDelayOtherForks() throws Exception {
+ NotifiableTestStream slowStream = mock(NotifiableTestStream.class);
+ NotifiableTestStream otherStream = mock(NotifiableTestStream.class);
+ ForkClient slowFork = startedForkClient(slowStream, 1);
+ ForkClient otherFork = startedForkClient(otherStream, 2);
+ CountDownLatch dumping = new CountDownLatch(1);
+ CountDownLatch dumpDone = new CountDownLatch(1);
+ slowFork.setTimeoutDetectedListener(() -> {
+ dumping.countDown();
+ try {
+ dumpDone.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ });
+ ExecutorService timer = Executors.newSingleThreadExecutor();
+ ExecutorService executor = Executors.newCachedThreadPool();
+ try {
+ // one round of the timeout check, like ForkStarter's timer
+ long now = System.currentTimeMillis() + 1000L;
+ timer.execute(() -> {
+ slowFork.tryToTimeout(now, 1, executor);
+ otherFork.tryToTimeout(now, 1, executor);
+ });
+
+ assertThat(dumping.await(5L, TimeUnit.SECONDS)).isTrue();
+ // the other fork is killed while the slow one is still dumping
+ verify(otherStream, timeout(5000L)).shutdown(Shutdown.KILL);
+ verify(slowStream, never()).shutdown(Shutdown.KILL);
+
+ dumpDone.countDown();
+ verify(slowStream, timeout(5000L)).shutdown(Shutdown.KILL);
+ assertThat(slowFork.hadTimeout()).isTrue();
+ assertThat(otherFork.hadTimeout()).isTrue();
+ } finally {
+ dumpDone.countDown();
+ timer.shutdownNow();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void killIsSentEvenWhenExecutorRejects() {
+ NotifiableTestStream stream = mock(NotifiableTestStream.class);
+ ForkClient client = startedForkClient(stream, 1);
+ AtomicInteger listenerCalls = new AtomicInteger();
+ client.setTimeoutDetectedListener(listenerCalls::incrementAndGet);
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ executor.shutdown();
+
+ client.tryToTimeout(System.currentTimeMillis() + 1000L, 1, executor);
+
+ assertThat(listenerCalls).hasValue(1);
+ verify(stream).shutdown(Shutdown.KILL);
+ assertThat(client.hadTimeout()).isTrue();
+ }
+
+ private static ForkClient startedForkClient(NotifiableTestStream stream,
int forkNumber) {
+ DefaultReporterFactory factory = mock(DefaultReporterFactory.class);
+ when(factory.getReportsDirectory()).thenReturn(new File("."));
+ when(factory.createTestReportListener()).thenReturn(new
MockReporter());
+
+ TestSetReportEntry reportEntry = mock(TestSetReportEntry.class);
+ when(reportEntry.getRunMode()).thenReturn(NORMAL_RUN);
+ when(reportEntry.getTestRunId()).thenReturn(1L);
+ when(reportEntry.getElapsed()).thenReturn(ELAPSED_TIME);
+ when(reportEntry.getName()).thenReturn("my test");
+ when(reportEntry.getSourceName()).thenReturn("pkg.MyTest");
+
+ ForkClient client = new ForkClient(factory, stream, forkNumber);
+ client.handleEvent(new TestsetStartingEvent(reportEntry));
+ return client;
+ }
}
diff --git
a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/extensions/timeout/JstackTimeoutExtensionTest.java
b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/extensions/timeout/JstackTimeoutExtensionTest.java
index e1882499b..97b78add1 100644
---
a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/extensions/timeout/JstackTimeoutExtensionTest.java
+++
b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/extensions/timeout/JstackTimeoutExtensionTest.java
@@ -143,6 +143,22 @@ void
outputLocationFromExtensionContextIsHonoredWhenJstackAvailable() throws Exc
assertThat(customDir.toFile().listFiles()).isNotNull().isNotEmpty();
}
+ @Test
+ void outputDirectoryMadeByAnotherForkIsUsed() {
+ // Pretend another fork created the directory first
+ File dir = new File(tmp.toFile(), "jstacks") {
+ @Override
+ public boolean mkdirs() {
+ super.mkdirs();
+ return false;
+ }
+ };
+ DefaultForkedProcessTimeoutContext ctx =
+ new DefaultForkedProcessTimeoutContext(123L, 1, null, dir, 60,
logger, null);
+
+ assertThat(JstackTimeoutExtension.resolveOutputDirectory(ctx,
logger)).isSameAs(dir);
+ }
+
@Test
void blankOutputLocationFallsBackToReportsDirectory() {
Map<String, String> ctxMap =
Collections.singletonMap(JstackTimeoutExtension.OUTPUT_LOCATION_KEY, " ");
diff --git
a/maven-surefire-plugin/src/site/markdown/examples/timeout-extension.md
b/maven-surefire-plugin/src/site/markdown/examples/timeout-extension.md
index d6964e227..10aae063e 100644
--- a/maven-surefire-plugin/src/site/markdown/examples/timeout-extension.md
+++ b/maven-surefire-plugin/src/site/markdown/examples/timeout-extension.md
@@ -168,5 +168,7 @@ It works the same way with `maven-failsafe-plugin`.
- All registered extensions run, including the `jstack` one when it's enabled.
- Each callback runs once per timed-out fork.
+- If several forks time out together, their extensions run at the same time,
so make your extension safe to call
+ from several threads. For one fork, extensions still run one by one, before
it's killed.
- Each callback gets 30 seconds, then it's cancelled.
- If an extension throws, Surefire logs a warning. The test results don't
change.