This is an automated email from the ASF dual-hosted git repository.
snagel pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new 00bf8c463 NUTCH-3139 protocol-okhttp: add support for zstd
content-encoding - upgrade to OkHttp 5.3.2 - enable support for zstd
content-encoding
00bf8c463 is described below
commit 00bf8c46319d079b086af09b1052f80ff7a7dbd5
Author: Sebastian Nagel <[email protected]>
AuthorDate: Mon Dec 15 23:30:29 2025 +0100
NUTCH-3139 protocol-okhttp: add support for zstd content-encoding
- upgrade to OkHttp 5.3.2
- enable support for zstd content-encoding
---
src/plugin/protocol-okhttp/ivy.xml | 7 ++++---
src/plugin/protocol-okhttp/plugin.xml | 16 +++++++++-------
.../java/org/apache/nutch/protocol/okhttp/OkHttp.java | 13 ++++++++-----
3 files changed, 21 insertions(+), 15 deletions(-)
diff --git a/src/plugin/protocol-okhttp/ivy.xml
b/src/plugin/protocol-okhttp/ivy.xml
index 0768def78..28f355d7b 100644
--- a/src/plugin/protocol-okhttp/ivy.xml
+++ b/src/plugin/protocol-okhttp/ivy.xml
@@ -37,8 +37,9 @@
</publications>
<dependencies>
- <dependency org="com.squareup.okhttp3" name="okhttp" rev="4.9.3"/>
- <dependency org="com.squareup.okhttp3" name="okhttp-brotli" rev="4.9.3"/>
+ <dependency org="com.squareup.okhttp3" name="okhttp" rev="5.3.2"/>
+ <dependency org="com.squareup.okhttp3" name="okhttp-brotli" rev="5.3.2"/>
+ <dependency org="com.squareup.okhttp3" name="okhttp-zstd" rev="5.3.2"/>
</dependencies>
-
+
</ivy-module>
diff --git a/src/plugin/protocol-okhttp/plugin.xml
b/src/plugin/protocol-okhttp/plugin.xml
index e2183d2b5..51f65f5d2 100755
--- a/src/plugin/protocol-okhttp/plugin.xml
+++ b/src/plugin/protocol-okhttp/plugin.xml
@@ -28,13 +28,15 @@
<!-- dependencies of OkHttp -->
<library name="annotations-13.0.jar"/>
<library name="dec-0.1.2.jar"/>
- <library name="kotlin-stdlib-1.4.10.jar"/>
- <library name="kotlin-stdlib-common-1.4.10.jar"/>
- <library name="kotlin-stdlib-jdk7-1.4.10.jar"/>
- <library name="kotlin-stdlib-jdk8-1.4.10.jar"/>
- <library name="okhttp-4.9.3.jar"/>
- <library name="okhttp-brotli-4.9.3.jar"/>
- <library name="okio-2.8.0.jar"/>
+ <library name="kotlin-stdlib-2.2.21.jar"/>
+ <library name="okhttp-5.3.2.jar"/>
+ <library name="okhttp-brotli-5.3.2.jar"/>
+ <library name="okhttp-jvm-5.3.2.jar"/>
+ <library name="okhttp-zstd-5.3.2.jar"/>
+ <library name="okio-3.16.4.jar"/>
+ <library name="okio-jvm-3.16.4.jar"/>
+ <library name="zstd-kmp-jvm-0.4.0.jar"/>
+ <library name="zstd-kmp-okio-jvm-0.4.0.jar"/>
<!-- end of dependencies of OkHttp -->
</runtime>
diff --git
a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java
b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java
index 954c3f6df..a9d2b14d4 100644
---
a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java
+++
b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java
@@ -52,15 +52,19 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import okhttp3.Authenticator;
+import okhttp3.CompressionInterceptor;
import okhttp3.Connection;
import okhttp3.ConnectionPool;
+import okhttp3.Gzip;
import okhttp3.Handshake;
import okhttp3.Headers;
import okhttp3.Interceptor;
import okhttp3.OkHttpClient;
import okhttp3.Protocol;
import okhttp3.Request;
-import okhttp3.brotli.BrotliInterceptor;
+import okhttp3.brotli.Brotli;
+import okhttp3.zstd.Zstd;
+
public class OkHttp extends HttpBase {
@@ -156,13 +160,11 @@ public class OkHttp extends HttpBase {
String proxyUsername = conf.get("http.proxy.username");
if (proxyUsername == null) {
ProxySelector selector = new ProxySelector() {
- @SuppressWarnings("serial")
private final List<Proxy> noProxyList = new ArrayList<Proxy>() {
{
add(Proxy.NO_PROXY);
}
};
- @SuppressWarnings("serial")
private final List<Proxy> proxyList = new ArrayList<Proxy>() {
{
add(proxy);
@@ -224,8 +226,9 @@ public class OkHttp extends HttpBase {
builder.addNetworkInterceptor(new HTTPHeadersInterceptor());
}
- // enable support for Brotli compression (Content-Encoding)
- builder.addInterceptor(BrotliInterceptor.INSTANCE);
+ // enable support for Zstd, Brotli, Gzip Content-Encoding
+ builder.addInterceptor(new CompressionInterceptor(Zstd.INSTANCE,
+ Brotli.INSTANCE, Gzip.INSTANCE));
// instantiate connection pool(s), cf.
// https://square.github.io/okhttp/3.x/okhttp/okhttp3/ConnectionPool.html
