This is an automated email from the ASF dual-hosted git repository.
jnioche pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
The following commit(s) were added to refs/heads/main by this push:
new 4d49466d Removed apache http protocol implementation. Closes #1995
(#2071)
4d49466d is described below
commit 4d49466df34dbd094d09021d1bc3bdc4293f01d0
Author: Julien Nioche <[email protected]>
AuthorDate: Tue Aug 25 13:08:23 2026 +0100
Removed apache http protocol implementation. Closes #1995 (#2071)
Signed-off-by: Julien Nioche <[email protected]>
---
.../protocol/httpclient/HttpProtocol.java | 421 ---------------------
core/src/main/resources/crawler-default.yaml | 3 -
.../protocol/DelegationProtocolTest.java | 4 -
.../HttpProtocolProxyManagerTest.java} | 5 +-
docs/src/main/asciidoc/configuration.adoc | 8 +-
docs/src/main/asciidoc/internals.adoc | 40 +-
6 files changed, 23 insertions(+), 458 deletions(-)
diff --git
a/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java
b/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java
deleted file mode 100644
index 6c4a137d..00000000
---
a/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java
+++ /dev/null
@@ -1,421 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.protocol.httpclient;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.MalformedURLException;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Base64;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Optional;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.mutable.MutableBoolean;
-import org.apache.http.Header;
-import org.apache.http.HeaderIterator;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpHeaders;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpResponse;
-import org.apache.http.StatusLine;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.ResponseHandler;
-import org.apache.http.client.config.AuthSchemes;
-import org.apache.http.client.config.CookieSpecs;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpHead;
-import org.apache.http.client.methods.HttpRequestBase;
-import org.apache.http.cookie.Cookie;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.impl.conn.DefaultProxyRoutePlanner;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
-import org.apache.http.message.BasicHeader;
-import org.apache.http.util.Args;
-import org.apache.http.util.ByteArrayBuffer;
-import org.apache.storm.Config;
-import org.apache.stormcrawler.Constants;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.persistence.Status;
-import org.apache.stormcrawler.protocol.AbstractHttpProtocol;
-import org.apache.stormcrawler.protocol.Protocol;
-import org.apache.stormcrawler.protocol.ProtocolResponse;
-import org.apache.stormcrawler.proxy.SCProxy;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.apache.stormcrawler.util.CookieConverter;
-import org.slf4j.LoggerFactory;
-
-/**
- * Uses Apache httpclient to handle http and https.
- *
- * @deprecated The Apache HttpClient protocol implementation is deprecated and
will be removed in
- * the next major release of StormCrawler. Use the OkHttp implementation
({@link
- * org.apache.stormcrawler.protocol.okhttp.HttpProtocol}) instead.
- */
-@Deprecated
-public class HttpProtocol extends AbstractHttpProtocol
- implements ResponseHandler<ProtocolResponse> {
-
- private static final org.slf4j.Logger LOG =
LoggerFactory.getLogger(HttpProtocol.class);
-
- private static final PoolingHttpClientConnectionManager CONNECTION_MANAGER
=
- new PoolingHttpClientConnectionManager();
-
- private int globalMaxContent;
-
- private HttpClientBuilder builder;
-
- private CloseableHttpClient client;
-
- private String userAgent;
- private Collection<BasicHeader> defaultHeaders;
-
- private RequestConfig requestConfig;
- private RequestConfig.Builder requestConfigBuilder;
-
- @Override
- public void configure(final Config conf) {
-
- super.configure(conf);
-
- // allow up to 200 connections or same as the number of threads used
for
- // fetching
- int maxFetchThreads = ConfUtils.getInt(conf, "fetcher.threads.number",
200);
- CONNECTION_MANAGER.setMaxTotal(maxFetchThreads);
- int maxPerRoute = ConfUtils.getInt(conf, "fetcher.threads.per.queue",
1);
- if (maxPerRoute < 20) {
- maxPerRoute = 20;
- }
- CONNECTION_MANAGER.setDefaultMaxPerRoute(maxPerRoute);
-
- globalMaxContent = ConfUtils.getInt(conf, "http.content.limit", -1);
-
- userAgent = getAgentString(conf);
-
- defaultHeaders = new LinkedList<>();
-
- String accept = ConfUtils.getString(conf, "http.accept");
- if (StringUtils.isNotBlank(accept)) {
- defaultHeaders.add(new BasicHeader(HttpHeaders.ACCEPT, accept));
- }
-
- customHeaders.forEach(
- h -> {
- defaultHeaders.add(new BasicHeader(h.getKey(),
h.getValue()));
- });
-
- String basicAuthUser = ConfUtils.getString(conf,
"http.basicauth.user", null);
-
- // use a basic auth?
- if (StringUtils.isNotBlank(basicAuthUser)) {
- String basicAuthPass = ConfUtils.getString(conf,
"http.basicauth.password", "");
- String encoding =
- Base64.getEncoder()
- .encodeToString(
- (basicAuthUser + ":" + basicAuthPass)
- .getBytes(StandardCharsets.UTF_8));
- defaultHeaders.add(new BasicHeader(HttpHeaders.AUTHORIZATION,
"Basic " + encoding));
- }
-
- String acceptLanguage = ConfUtils.getString(conf,
"http.accept.language");
- if (StringUtils.isNotBlank(acceptLanguage)) {
- defaultHeaders.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE,
acceptLanguage));
- }
-
- builder =
- HttpClients.custom()
- .setUserAgent(userAgent)
- .setDefaultHeaders(defaultHeaders)
- .setConnectionManager(CONNECTION_MANAGER)
- .setConnectionManagerShared(true)
- .disableRedirectHandling()
- .disableAutomaticRetries();
-
- int timeout = ConfUtils.getInt(conf, "http.timeout", 10000);
-
- requestConfigBuilder =
- RequestConfig.custom()
- .setSocketTimeout(timeout)
- .setConnectTimeout(timeout)
- .setConnectionRequestTimeout(timeout)
- .setCookieSpec(CookieSpecs.STANDARD);
-
- requestConfig = requestConfigBuilder.build();
-
- client = builder.build();
- }
-
- @Override
- public ProtocolResponse getProtocolOutput(String url, Metadata md) throws
Exception {
-
- LOG.debug("HTTP connection manager stats {}",
CONNECTION_MANAGER.getTotalStats());
-
- // set default request config to global config
- RequestConfig reqConfig = requestConfig;
-
- // default to the shared client for non-proxy requests
- CloseableHttpClient httpClient = client;
-
- // conditionally add a dynamic proxy
- if (proxyManager != null) {
- // retrieve proxy from proxy manager
- Optional<SCProxy> proxOptional = proxyManager.getProxy(md);
- if (proxOptional.isPresent()) {
- SCProxy prox = proxOptional.get();
-
- // create a new builder with the same defaults (user-agent,
- // headers) to avoid losing them in proxied requests
- HttpClientBuilder localBuilder =
- HttpClients.custom()
- .setUserAgent(userAgent)
- .setDefaultHeaders(defaultHeaders)
- .setConnectionManager(CONNECTION_MANAGER)
- .setConnectionManagerShared(true)
- .disableRedirectHandling()
- .disableAutomaticRetries();
- RequestConfig.Builder localRequestConfigBuilder =
RequestConfig.copy(requestConfig);
-
- // conditionally configure proxy authentication
- if (StringUtils.isNotBlank(prox.getUsername())) {
- List<String> authSchemes = new ArrayList<>();
-
- // Can make configurable and add more in future
- authSchemes.add(AuthSchemes.BASIC);
-
localRequestConfigBuilder.setProxyPreferredAuthSchemes(authSchemes);
-
- BasicCredentialsProvider basicAuthCreds = new
BasicCredentialsProvider();
- basicAuthCreds.setCredentials(
- new AuthScope(prox.getAddress(),
Integer.parseInt(prox.getPort())),
- new UsernamePasswordCredentials(
- prox.getUsername(), prox.getPassword()));
- localBuilder.setDefaultCredentialsProvider(basicAuthCreds);
- }
-
- HttpHost proxy = new HttpHost(prox.getAddress(),
Integer.parseInt(prox.getPort()));
- DefaultProxyRoutePlanner routePlanner = new
DefaultProxyRoutePlanner(proxy);
- localBuilder.setRoutePlanner(routePlanner);
-
- // save start time for debugging speed impact of request config
- // build
- long buildStart = System.currentTimeMillis();
-
- // set request config to new configuration with dynamic proxy
- reqConfig = localRequestConfigBuilder.build();
-
- LOG.debug(
- "time to build http request config with proxy: {}ms",
- System.currentTimeMillis() - buildStart);
-
- LOG.debug("fetching with " + prox.toString());
- httpClient = localBuilder.build();
- }
- }
-
- HttpRequestBase request = new HttpGet(url);
- ResponseHandler<ProtocolResponse> responseHandler = this;
-
- if (md != null) {
-
- addHeadersToRequest(request, md);
-
- String useHead = md.getFirstValue("http.method.head");
- if (Boolean.parseBoolean(useHead)) {
- request = new HttpHead(url);
- }
-
- String lastModified = md.getFirstValue(HttpHeaders.LAST_MODIFIED);
- if (StringUtils.isNotBlank(lastModified)) {
- request.addHeader(HttpHeaders.IF_MODIFIED_SINCE,
formatHttpDate(lastModified));
- }
-
- String ifNoneMatch = md.getFirstValue(HttpHeaders.ETAG,
protocolMetadataPrefix);
- if (StringUtils.isNotBlank(ifNoneMatch)) {
- request.addHeader(HttpHeaders.IF_NONE_MATCH, ifNoneMatch);
- }
-
- String accept = md.getFirstValue("http.accept");
- if (StringUtils.isNotBlank(accept)) {
- request.setHeader(new BasicHeader(HttpHeaders.ACCEPT, accept));
- }
-
- String acceptLanguage = md.getFirstValue("http.accept.language");
- if (StringUtils.isNotBlank(acceptLanguage)) {
- request.setHeader(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE,
acceptLanguage));
- }
-
- String pageMaxContentStr = md.getFirstValue("http.content.limit");
- if (StringUtils.isNotBlank(pageMaxContentStr)) {
- try {
- int pageMaxContent = Integer.parseInt(pageMaxContentStr);
- responseHandler =
getResponseHandlerWithContentLimit(pageMaxContent);
- } catch (NumberFormatException e) {
- LOG.warn("Invalid http.content.limit in metadata: {}",
pageMaxContentStr);
- }
- }
-
- if (useCookies) {
- addCookiesToRequest(request, md);
- }
- }
-
- request.setConfig(reqConfig);
-
- return httpClient.execute(request, responseHandler);
- }
-
- private void addCookiesToRequest(HttpRequestBase request, Metadata md) {
- String[] cookieStrings = md.getValues(RESPONSE_COOKIES_HEADER,
protocolMetadataPrefix);
- if (cookieStrings != null && cookieStrings.length > 0) {
- List<Cookie> cookies;
- try {
- cookies = CookieConverter.getCookies(cookieStrings,
request.getURI().toURL());
- for (Cookie c : cookies) {
- request.addHeader("Cookie", c.getName() + "=" +
c.getValue());
- }
- } catch (MalformedURLException e) { // Bad url , nothing to do
- }
- }
- }
-
- protected void addHeadersToRequest(HttpRequestBase request, Metadata md) {
- String[] headerStrings = md.getValues(SET_HEADER_BY_REQUEST,
protocolMetadataPrefix);
- if ((headerStrings != null) && (headerStrings.length > 0)) {
- for (String hs : headerStrings) {
- KeyValue h = KeyValue.build(hs);
- request.addHeader(h.getKey(), h.getValue());
- }
- }
- }
-
- @Override
- public ProtocolResponse handleResponse(HttpResponse response) throws
IOException {
- return handleResponseWithContentLimit(response, globalMaxContent);
- }
-
- public ProtocolResponse handleResponseWithContentLimit(HttpResponse
response, int maxContent)
- throws IOException {
- StatusLine statusLine = response.getStatusLine();
- int status = statusLine.getStatusCode();
-
- StringBuilder verbatim = new StringBuilder();
- if (storeHttpHeaders) {
- verbatim.append(statusLine).append("\r\n");
- }
-
- Metadata metadata = new Metadata();
- HeaderIterator iter = response.headerIterator();
- while (iter.hasNext()) {
- Header header = iter.nextHeader();
- if (storeHttpHeaders) {
- verbatim.append(header.toString()).append("\r\n");
- }
- metadata.addValue(header.getName().toLowerCase(Locale.ROOT),
header.getValue());
- }
-
- MutableBoolean trimmed = new MutableBoolean();
-
- byte[] bytes = new byte[] {};
-
- if (!Status.REDIRECTION.equals(Status.fromHTTPCode(status))) {
- bytes = HttpProtocol.toByteArray(response.getEntity(), maxContent,
trimmed);
- if (trimmed.booleanValue()) {
- metadata.setValue(ProtocolResponse.TRIMMED_RESPONSE_KEY,
"true");
- LOG.warn("HTTP content trimmed to {}", bytes.length);
- }
- }
-
- if (storeHttpHeaders) {
- verbatim.append("\r\n");
- metadata.setValue(ProtocolResponse.RESPONSE_HEADERS_KEY,
verbatim.toString());
- }
-
- return new ProtocolResponse(bytes, status, metadata);
- }
-
- private ResponseHandler<ProtocolResponse>
getResponseHandlerWithContentLimit(
- int pageMaxContent) {
- return new ResponseHandler<>() {
- public ProtocolResponse handleResponse(final HttpResponse
response) throws IOException {
- return handleResponseWithContentLimit(response,
pageMaxContent);
- }
- };
- }
-
- private static byte[] toByteArray(
- final HttpEntity entity, int maxContent, MutableBoolean trimmed)
throws IOException {
-
- if (entity == null) {
- return new byte[] {};
- }
-
- final InputStream instream = entity.getContent();
- if (instream == null) {
- return new byte[] {};
- }
- try (instream) {
- Args.check(
- (entity.getContentLength() <= Constants.MAX_ARRAY_SIZE)
- || (maxContent >= 0 && maxContent <=
Constants.MAX_ARRAY_SIZE),
- "HTTP entity too large to be buffered in memory");
- int reportedLength = (int) entity.getContentLength();
- // set default size for buffer: 100 KB
- int bufferInitSize = 102400;
- if (reportedLength != -1) {
- bufferInitSize = reportedLength;
- }
- // avoid init of too large a buffer when we will trim anyway
- if (maxContent != -1 && bufferInitSize > maxContent) {
- bufferInitSize = maxContent;
- }
- final ByteArrayBuffer buffer = new ByteArrayBuffer(bufferInitSize);
- final byte[] tmp = new byte[4096];
- int lengthRead;
- while ((lengthRead = instream.read(tmp)) != -1) {
- // check whether we need to trim
- if (maxContent != -1 && buffer.length() + lengthRead >
maxContent) {
- buffer.append(tmp, 0, maxContent - buffer.length());
- trimmed.setValue(true);
- break;
- }
- buffer.append(tmp, 0, lengthRead);
- }
- return buffer.toByteArray();
- }
- }
-
- @Override
- public void cleanup() {
- try {
- client.close();
- } catch (IOException e) {
- LOG.error("Error closing HTTP client", e);
- }
- }
-
- public static void main(String[] args) throws Exception {
- Protocol.main(new HttpProtocol(), args);
- }
-}
diff --git a/core/src/main/resources/crawler-default.yaml
b/core/src/main/resources/crawler-default.yaml
index 27092814..0343f644 100644
--- a/core/src/main/resources/crawler-default.yaml
+++ b/core/src/main/resources/crawler-default.yaml
@@ -197,9 +197,6 @@ config:
robots.error.cache.spec: "maximumSize=10000,expireAfterWrite=1h"
protocols: "http,https,file"
- # NOTE: the Apache HttpClient protocol implementation
- # (org.apache.stormcrawler.protocol.httpclient.HttpProtocol) is deprecated
and will be removed in
- # the next major release of StormCrawler. OkHttp is now the default.
http.protocol.implementation:
"org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
https.protocol.implementation:
"org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
file.protocol.implementation:
"org.apache.stormcrawler.protocol.file.FileProtocol"
diff --git
a/core/src/test/java/org/apache/stormcrawler/protocol/DelegationProtocolTest.java
b/core/src/test/java/org/apache/stormcrawler/protocol/DelegationProtocolTest.java
index 3333b378..9b6ba8bc 100644
---
a/core/src/test/java/org/apache/stormcrawler/protocol/DelegationProtocolTest.java
+++
b/core/src/test/java/org/apache/stormcrawler/protocol/DelegationProtocolTest.java
@@ -27,10 +27,6 @@ import org.junit.jupiter.api.Test;
class DelegationProtocolTest {
- private static final String OKHTTP =
"org.apache.stormcrawler.protocol.okhttp.HttpProtocol";
-
- private static final String APACHE =
"org.apache.stormcrawler.protocol.httpclient.HttpProtocol";
-
@Test
void getProtocolTest() throws FileNotFoundException {
Config conf = new Config();
diff --git
a/core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java
b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolProxyManagerTest.java
similarity index 95%
rename from
core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java
rename to
core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolProxyManagerTest.java
index d350e8f8..cb43d0e7 100644
---
a/core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java
+++
b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolProxyManagerTest.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.stormcrawler.protocol.httpclient;
+package org.apache.stormcrawler.protocol.okhttp;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -26,8 +26,7 @@ import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.proxy.SingleProxyManager;
import org.junit.jupiter.api.Test;
-@SuppressWarnings("deprecation")
-class HttpClientProtocolProxyManagerTest {
+class HttpProtocolProxyManagerTest {
@Test
void omittedProxyManagerWithoutProxyHostDoesNotCreateDefaultManager() {
diff --git a/docs/src/main/asciidoc/configuration.adoc
b/docs/src/main/asciidoc/configuration.adoc
index 40dec13b..7aaa0ccf 100644
--- a/docs/src/main/asciidoc/configuration.adoc
+++ b/docs/src/main/asciidoc/configuration.adoc
@@ -202,9 +202,7 @@ is defined.
| http.basicauth.user | - | Username for Basic Authentication.
| http.content.limit | -1 | Maximum HTTP response body size (bytes). Default:
no limit.
| http.protocol.implementation |
org.apache.stormcrawler.protocol.okhttp.HttpProtocol | HTTP Protocol
-implementation. Note: the Apache HttpClient implementation
-(`org.apache.stormcrawler.protocol.httpclient.HttpProtocol`) is *deprecated*
and will be removed in
-the next major release.
+implementation.
| http.proxy | - | Full **SCProxy** connection string read by
SingleProxyManager when that manager is selected, e.g.
`http://user:[email protected]:8080`.
| http.proxy.file | - | Proxy connection-string file read by MultiProxyManager
when that manager is selected.
| http.proxy.host | - | SingleProxyManager component proxy host.
@@ -228,9 +226,7 @@ the next major release.
| http.timeout | 10000 | Connection timeout (ms).
| http.use.cookies | false | Use cookies in subsequent requests.
| https.protocol.implementation |
org.apache.stormcrawler.protocol.okhttp.HttpProtocol | HTTPS Protocol
-implementation. Note: the Apache HttpClient implementation
-(`org.apache.stormcrawler.protocol.httpclient.HttpProtocol`) is *deprecated*
and will be removed in
-the next major release.
+implementation.
| partition.url.mode | byHost | Defines how URLs are partitioned: byHost,
byDomain, or byIP.
| protocols | http,https,file | Supported protocols.
| redirections.allowed | true | If true, emit redirect target URLs as
"outlinks" to the status stream. If false, do not follow redirects. See also
`http.allow.redirects`.
diff --git a/docs/src/main/asciidoc/internals.adoc
b/docs/src/main/asciidoc/internals.adoc
index 262e7c07..ff74e9a4 100644
--- a/docs/src/main/asciidoc/internals.adoc
+++ b/docs/src/main/asciidoc/internals.adoc
@@ -421,38 +421,36 @@ http.protocol.implementation:
"org.apache.stormcrawler.protocol.okhttp.HttpProto
https.protocol.implementation:
"org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
----
-*
link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java[HttpClient]
(*deprecated* — will be removed in the next major release; use OKHttp instead)
*
link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java[OKHttp]
*
link:https://github.com/apache/stormcrawler/blob/main/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java[Playwright]
(browser-based, in the `playwright` external module)
==== Feature grid
-[cols="2,1,1", options="header"]
+[cols="2,1", options="header"]
|===
-| Features | HTTPClient | OKHttp
-
-| Basic authentication |
link:https://github.com/apache/stormcrawler/pull/589[Y] |
link:https://github.com/apache/stormcrawler/issues/792[Y]
-| Proxy (w. credentials?) | Y / Y | Y /
link:https://github.com/apache/stormcrawler/issues/751[Y]
-| Interruptible / trimmable
link:https://github.com/apache/stormcrawler/issues/463[#463] | N / Y | Y / Y
-| Cookies | Y | link:https://github.com/apache/stormcrawler/issues/632[Y]
-| Response headers | Y | Y
-| Trust all certificates | N |
link:https://github.com/apache/stormcrawler/issues/615[Y]
-| HEAD method | link:https://github.com/apache/stormcrawler/issues/485[Y] |
link:https://github.com/apache/stormcrawler/pull/923[Y]
-| POST method | N | link:https://github.com/apache/stormcrawler/issues/641[Y]
-| Verbatim response header |
link:https://github.com/apache/stormcrawler/issues/317[Y] |
link:https://github.com/apache/stormcrawler/issues/506[Y]
-| Verbatim request header | N |
link:https://github.com/apache/stormcrawler/issues/506[Y]
-| IP address capture | N |
link:https://github.com/apache/stormcrawler/pull/691[Y]
-| Navigation and javascript | N | N
-| HTTP/2 | N | Y
-| Configurable connection pool | N |
link:https://github.com/apache/stormcrawler/issues/918[Y]
+| Features | OKHttp
+
+| Basic authentication |
link:https://github.com/apache/stormcrawler/issues/792[Y]
+| Proxy (w. credentials?) | Y /
link:https://github.com/apache/stormcrawler/issues/751[Y]
+| Interruptible / trimmable
link:https://github.com/apache/stormcrawler/issues/463[#463] | Y / Y
+| Cookies | link:https://github.com/apache/stormcrawler/issues/632[Y]
+| Response headers | Y
+| Trust all certificates |
link:https://github.com/apache/stormcrawler/issues/615[Y]
+| HEAD method | link:https://github.com/apache/stormcrawler/pull/923[Y]
+| POST method | link:https://github.com/apache/stormcrawler/issues/641[Y]
+| Verbatim response header |
link:https://github.com/apache/stormcrawler/issues/506[Y]
+| Verbatim request header |
link:https://github.com/apache/stormcrawler/issues/506[Y]
+| IP address capture | link:https://github.com/apache/stormcrawler/pull/691[Y]
+| Navigation and javascript | N
+| HTTP/2 | Y
+| Configurable connection pool |
link:https://github.com/apache/stormcrawler/issues/918[Y]
|===
-Neither of the HTTP clients executes JavaScript: use the browser-based
link:https://github.com/apache/stormcrawler/tree/main/external/playwright[Playwright]
protocol for pages that require rendering or navigation.
+The OKHttp client does not execute JavaScript: use the browser-based
link:https://github.com/apache/stormcrawler/tree/main/external/playwright[Playwright]
protocol for pages that require rendering or navigation.
==== HTTP/2
* The
link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java[OKHttp]
protocol supports link:https://en.wikipedia.org/wiki/HTTP/2[HTTP/2] if the JDK
includes
link:https://en.wikipedia.org/wiki/Application-Layer_Protocol_Negotiation[ALPN]
(Java 9 and upwards or Java 8 builds starting early/mid 2020).
-*
link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java[HttpClient]
does not yet support HTTP/2.
*
link:https://github.com/apache/stormcrawler/blob/main/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java[Playwright]:
whether HTTP/2 is used or not depends on the browser.
Since link:https://github.com/apache/stormcrawler/pull/829[#829], the HTTP
protocol version used is configurable via `http.protocol.versions` (see also
comments in
link:https://github.com/apache/stormcrawler/blob/main/core/src/main/resources/crawler-default.yaml[crawler-default.yaml]).
@@ -479,7 +477,7 @@ The `metadata` argument to
link:https://github.com/apache/stormcrawler/blob/main
* `protocol.set-cookie`: If this key is present in `metadata` and
`http.use.cookies` is true, the protocol will send cookies stored from the
response this page was linked to, given the cookie is applicable to the domain
of the link.
-* `http.method.head`: If this key is present in `metadata`, the protocol sends
a HEAD request. (Available in v1.12+ only for httpclient, see
link:https://github.com/apache/stormcrawler/issues/485[#485])
+* `http.method.head`: If this key is present in `metadata`, the protocol sends
a HEAD request. (see link:https://github.com/apache/stormcrawler/pull/923[#923])
* `http.post.json`: If this key is present in `metadata`, the protocol sends a
POST request. (Available in v1.12+ only for okhttp, see
link:https://github.com/apache/stormcrawler/issues/641[#641])
