This is an automated email from the ASF dual-hosted git repository.
sebastian-nagel pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new a0814a7b1 NUTCH-3177 Fetcher to report idle threads not as hung threads
a0814a7b1 is described below
commit a0814a7b116dbf5b0053f7b2fa2428f3d868c95f
Author: Sebastian Nagel <[email protected]>
AuthorDate: Wed May 13 22:56:43 2026 +0200
NUTCH-3177 Fetcher to report idle threads not as hung threads
- reset reprUrl in FetcherThread after fetch is finished
- report idle threads properly
---
src/java/org/apache/nutch/fetcher/Fetcher.java | 13 +++++++++----
src/java/org/apache/nutch/fetcher/FetcherThread.java | 3 +++
2 files changed, 12 insertions(+), 4 deletions(-)
diff --git a/src/java/org/apache/nutch/fetcher/Fetcher.java
b/src/java/org/apache/nutch/fetcher/Fetcher.java
index 029d95ff7..d8f83a96b 100644
--- a/src/java/org/apache/nutch/fetcher/Fetcher.java
+++ b/src/java/org/apache/nutch/fetcher/Fetcher.java
@@ -448,16 +448,21 @@ public class Fetcher extends NutchTool implements Tool {
* fetcher.threads.timeout.divisor.
*/
if ((System.currentTimeMillis() - lastRequestStart.get()) > timeout)
{
- LOG.warn("Timeout reached with no new requests since {} seconds.",
+ LOG.warn(
+ "Timeout reached with no new requests since {} milliseconds.",
timeout);
- LOG.warn("Aborting with {} hung threads{}.", activeThreads,
+ LOG.warn("Aborting with {} hung or idle threads{}.", activeThreads,
feeder.isAlive() ? " (queue feeder still alive)" : "");
hungThreadsCounter.increment(activeThreads.get());
for (int i = 0; i < fetcherThreads.size(); i++) {
FetcherThread thread = fetcherThreads.get(i);
if (thread.isAlive()) {
- LOG.warn("Thread #{} hung while processing {}", i,
- thread.getReprUrl());
+ if (thread.getReprUrl() != null) {
+ LOG.warn("Thread #{} hung while processing {}", i,
+ thread.getReprUrl());
+ } else {
+ LOG.warn("Thread #{} idle", i);
+ }
StackTraceElement[] stack = thread.getStackTrace();
StringBuilder sb = new StringBuilder();
sb.append("Stack of thread #").append(i).append(":\n");
diff --git a/src/java/org/apache/nutch/fetcher/FetcherThread.java
b/src/java/org/apache/nutch/fetcher/FetcherThread.java
index e3ee092b4..e894f01f1 100644
--- a/src/java/org/apache/nutch/fetcher/FetcherThread.java
+++ b/src/java/org/apache/nutch/fetcher/FetcherThread.java
@@ -537,6 +537,9 @@ public class FetcherThread extends Thread {
output(fit.url, fit.datum, null, ProtocolStatus.STATUS_FAILED,
CrawlDatum.STATUS_FETCH_RETRY);
}
+
+ // done: unset reprUrl for reporting
+ setReprUrl(null);
}
} catch (Throwable e) {
