This is an automated email from the ASF dual-hosted git repository.
sebastian-nagel pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new a6159ea8e NUTCH-3174 protocol-okhttp: request may hang despite
http.time.limit is set
a6159ea8e is described below
commit a6159ea8e071735a553f7b2758096ef7b071adb4
Author: Sebastian Nagel <[email protected]>
AuthorDate: Thu May 7 15:16:10 2026 +0200
NUTCH-3174 protocol-okhttp: request may hang despite http.time.limit is set
- set OkHttp's call timeout to the value of http.time.limit (if not -1)
- add check whether http.time.limit is longer than http.timeout
- more verbose logging
---
.../org/apache/nutch/protocol/http/api/HttpBase.java | 19 +++++++++++++------
.../java/org/apache/nutch/protocol/okhttp/OkHttp.java | 5 +++++
.../apache/nutch/protocol/okhttp/OkHttpResponse.java | 8 +++++++-
3 files changed, 25 insertions(+), 7 deletions(-)
diff --git
a/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
b/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
index caa3f861e..63b9224b0 100755
---
a/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
+++
b/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
@@ -223,6 +223,12 @@ public abstract class HttpBase implements Protocol {
this.timeout = conf.getInt("http.timeout", 10000);
this.maxContent = conf.getInt("http.content.limit", 1024 * 1024);
this.maxDuration = conf.getInt("http.time.limit", -1);
+ if (maxDuration >= 0 && (maxDuration * 1000) < timeout) {
+ LOG.warn(
+ "The configuration property http.time.limit ({} seconds) is less
than http.timeout ({} ms), "
+ + "the entire request will time out before individual reads are
timed out.",
+ maxDuration, timeout);
+ }
this.partialAsTruncated = conf.getBoolean("http.partial.truncated", false);
this.userAgent = getAgentString(conf.get("http.agent.name"),
conf.get("http.agent.version"), conf.get("http.agent.description"),
@@ -272,8 +278,8 @@ public abstract class HttpBase implements Protocol {
}
} catch (Exception e) {
- this.logger.warn("Failed to read http.agent.rotate.file {}: {}",
agentsFile,
- StringUtils.stringifyException(e));
+ this.logger.warn("Failed to read http.agent.rotate.file {}:",
+ agentsFile, e);
this.userAgentNames = null;
} finally {
if (br != null) {
@@ -314,8 +320,8 @@ public abstract class HttpBase implements Protocol {
}
}
} catch (Exception e) {
- this.logger.warn("Failed to read http.agent.host.cookie.file {}: {}",
- cookieFile, StringUtils.stringifyException(e));
+ this.logger.warn("Failed to read http.agent.host.cookie.file {}:",
+ cookieFile, e);
this.hostCookies = null;
} finally {
if (br != null) {
@@ -614,8 +620,9 @@ public abstract class HttpBase implements Protocol {
this.logger.info("http.proxy.host = {}", this.proxyHost);
this.logger.info("http.proxy.port = {}", this.proxyPort);
this.logger.info("http.proxy.exception.list = {}", this.useProxy);
- this.logger.info("http.timeout = {}", this.timeout);
- this.logger.info("http.content.limit = {}", this.maxContent);
+ this.logger.info("http.timeout = {} ms", this.timeout);
+ this.logger.info("http.time.limit = {} seconds", this.maxDuration);
+ this.logger.info("http.content.limit = {} bytes", this.maxContent);
this.logger.info("http.agent = {}", this.userAgent);
this.logger.info("http.accept.language = {}", this.acceptLanguage);
this.logger.info("http.accept = {}", this.accept);
diff --git
a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java
b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java
index 8c898a0e2..be26886fe 100644
---
a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java
+++
b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java
@@ -121,6 +121,11 @@ public class OkHttp extends HttpBase {
.writeTimeout(this.timeout, TimeUnit.MILLISECONDS)
.readTimeout(this.timeout, TimeUnit.MILLISECONDS);
+ if (this.maxDuration >= 0) {
+ // timeout for the entire request
+ builder.callTimeout(this.maxDuration, TimeUnit.SECONDS);
+ }
+
if (!this.tlsCheckCertificate) {
try {
SSLContext trustAllSslContext = SSLContext.getInstance("TLS");
diff --git
a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttpResponse.java
b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttpResponse.java
index 7616bc886..8d7966199 100644
---
a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttpResponse.java
+++
b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttpResponse.java
@@ -17,6 +17,7 @@
package org.apache.nutch.protocol.okhttp;
import java.io.IOException;
+import java.io.InterruptedIOException;
import java.lang.invoke.MethodHandles;
import java.net.URL;
import java.util.Base64;
@@ -186,7 +187,12 @@ public class OkHttpResponse implements Response {
} catch (IOException e) {
if (partialAsTruncated && source.getBuffer().size() > 0) {
// treat already fetched content as truncated
- truncated.setReason(TruncatedContentReason.DISCONNECT);
+ if (e instanceof InterruptedIOException) {
+ // thrown by OkHttp if the call timeout is hit
+ truncated.setReason(TruncatedContentReason.TIME);
+ } else {
+ truncated.setReason(TruncatedContentReason.DISCONNECT);
+ }
LOG.info("Truncated content for {}, partial fetch caused by:",
this.url,
e);
} else {
