This is an automated email from the ASF dual-hosted git repository.
sebastian-nagel pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new 9f78c72cb NUTCH-3173 protocol-okhttp: store OkHttp's internal URL in
response metadata (#919)
9f78c72cb is described below
commit 9f78c72cb1a5b69205d33e2b29039a45b680404b
Author: Luca <[email protected]>
AuthorDate: Sun Jun 7 16:57:46 2026 +0200
NUTCH-3173 protocol-okhttp: store OkHttp's internal URL in response
metadata (#919)
---
src/java/org/apache/nutch/net/protocols/Response.java | 2 +-
src/java/org/apache/nutch/protocol/Protocol.java | 13 +++++++++++++
.../nutch/protocol/http/api/HttpRobotRulesParser.java | 3 ++-
.../apache/nutch/protocol/httpclient/HttpResponse.java | 11 +++++++++++
.../java/org/apache/nutch/protocol/okhttp/OkHttp.java | 18 ++++++++++++++++++
.../apache/nutch/protocol/okhttp/OkHttpResponse.java | 11 +++++++++--
6 files changed, 54 insertions(+), 4 deletions(-)
diff --git a/src/java/org/apache/nutch/net/protocols/Response.java
b/src/java/org/apache/nutch/net/protocols/Response.java
index 3fbe93266..2f5278437 100644
--- a/src/java/org/apache/nutch/net/protocols/Response.java
+++ b/src/java/org/apache/nutch/net/protocols/Response.java
@@ -85,7 +85,7 @@ public interface Response extends HttpHeaders {
};
/**
- * Get the URL used to retrieve this response.
+ * Get the URL the protocol actually used when requesting the Response.
* @return {@link java.net.URL}
*/
public URL getUrl();
diff --git a/src/java/org/apache/nutch/protocol/Protocol.java
b/src/java/org/apache/nutch/protocol/Protocol.java
index 2514eae33..34805febb 100644
--- a/src/java/org/apache/nutch/protocol/Protocol.java
+++ b/src/java/org/apache/nutch/protocol/Protocol.java
@@ -16,6 +16,7 @@
*/
package org.apache.nutch.protocol;
+import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
@@ -40,6 +41,18 @@ public interface Protocol extends Pluggable, Configurable {
*/
ProtocolOutput getProtocolOutput(Text url, CrawlDatum datum);
+/**
+ * Resolve a relative URL against a base URL using the protocol's URL
+ * library.
+ *
+ * @param base the base URL the relative URL is resolved against
+ * @param relative the relative URL string (typically a Location: header
value)
+ * @return resolved absolute URL
+ * @throws MalformedURLException if the URL is malformed
+ */
+ default URL resolveUrl(URL base, String relative) throws
MalformedURLException {
+ return new URL(base, relative);
+ }
/**
* Retrieve robot rules applicable for this URL.
*
diff --git
a/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
b/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
index 9da92698f..7c5d8aa0a 100644
---
a/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
+++
b/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
@@ -170,7 +170,8 @@ public class HttpRobotRulesParser extends RobotRulesParser {
LOG.debug("Following robots.txt redirect: {} -> {}",
robotsUrlRedir,
redirectionLocation);
try {
- robotsUrlRedir = new URL(robotsUrlRedir, redirectionLocation);
+ robotsUrlRedir = ((HttpBase) http).resolveUrl(robotsUrlRedir,
+ redirectionLocation);
} catch (MalformedURLException e) {
LOG.info(
"Failed to resolve redirect location for robots.txt: {} ->
{} ({})",
diff --git
a/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java
b/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java
index 87ee0bb8a..a7109bce4 100644
---
a/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java
+++
b/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java
@@ -121,6 +121,17 @@ public class HttpResponse implements Response {
client.getParams().setParameter("http.useragent", http.getUserAgent());
// NUTCH-1941
code = client.executeMethod(get);
+ // When followRedirects=true HC3 walks the redirect chain internally;
+ // getURI() returns the final URI. Capture it so getUrl() honors the
+ // contract — without this, robots.txt redirects via this plugin
+ // report the original URL even though a different URL was fetched.
+ try {
+ this.url = new URL(get.getURI().toString());
+ } catch (org.apache.commons.httpclient.URIException
+ | java.net.MalformedURLException e) {
+ // Keep the input URL or try to normalize it?
+ }
+
Header[] heads = get.getResponseHeaders();
for (int i = 0; i < heads.length; i++) {
diff --git
a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java
b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java
index a9d2b14d4..8c898a0e2 100644
---
a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java
+++
b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java
@@ -20,6 +20,7 @@ import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.InetAddress;
import java.net.InetSocketAddress;
+import java.net.MalformedURLException;
import java.net.Proxy;
import java.net.ProxySelector;
import java.net.SocketAddress;
@@ -58,6 +59,7 @@ import okhttp3.ConnectionPool;
import okhttp3.Gzip;
import okhttp3.Handshake;
import okhttp3.Headers;
+import okhttp3.HttpUrl;
import okhttp3.Interceptor;
import okhttp3.OkHttpClient;
import okhttp3.Protocol;
@@ -436,6 +438,22 @@ public class OkHttp extends HttpBase {
return new OkHttpResponse(this, url, datum);
}
+ /**
+ * Resolve a relative URL using OkHttp's {@link HttpUrl} parser, which is
+ * more lenient than Java's {@link URL} (handles malformed protocol-relative
+ * slashes such as {@code https:////host/path}, IDN→punycode, host case
+ * normalization, etc.). Falls back to Java URL if HttpUrl cannot parse.
+ */
+ @Override
+ public URL resolveUrl(URL base, String relative) throws
MalformedURLException {
+ HttpUrl baseHttpUrl = HttpUrl.get(base);
+ HttpUrl resolved = baseHttpUrl.resolve(relative);
+ if (resolved != null) {
+ return resolved.url();
+ }
+ return super.resolveUrl(base, relative);
+ }
+
public static void main(String[] args) throws Exception {
OkHttp okhttp = new OkHttp();
okhttp.setConf(NutchConfiguration.create());
diff --git
a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttpResponse.java
b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttpResponse.java
index 605c03390..7616bc886 100644
---
a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttpResponse.java
+++
b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttpResponse.java
@@ -71,7 +71,7 @@ public class OkHttpResponse implements Response {
public OkHttpResponse(OkHttp okhttp, URL url, CrawlDatum datum)
throws ProtocolException, IOException {
- this.url = url;
+ this.url = url; // provisional; overwritten below with the normalized form
Request.Builder rb = new Request.Builder().url(url);
@@ -102,7 +102,14 @@ public class OkHttpResponse implements Response {
}
Request request = rb.build();
- okhttp3.Call call = okhttp.getClient(url).newCall(request);
+
+ // OkHttp parsed the URL via HttpUrl; that's the form actually going on
+ // the wire (IDN→punycode, repeated-slash repair, host lowercasing).
+ this.url = request.url().url();
+ if (LOG.isDebugEnabled() && !this.url.toString().equals(url.toString())) {
+ LOG.debug("The normalized URL different from the requested URL: {} ->
{}", url, this.url);
+ }
+ okhttp3.Call call = okhttp.getClient(this.url).newCall(request);
// ensure that Response and underlying ResponseBody are closed
try (okhttp3.Response response = call.execute()) {
