abhinav-phi commented on code in PR #2126:
URL: https://github.com/apache/stormcrawler/pull/2126#discussion_r3951761737
##########
core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java:
##########
@@ -158,6 +175,37 @@ public void configure(Config conf) {
this.partialContentAsTrimmed =
ConfUtils.getBoolean(conf, "http.content.partial.as.trimmed",
false);
+ /*
+ * certificate trust and hostname verification are separate decisions:
+ * accepting any certificate does not imply accepting any name
+ */
+ final boolean trustEverything = ConfUtils.getBoolean(conf,
"http.trust.everything", false);
+ final boolean verifyHostnames = ConfUtils.getBoolean(conf,
"http.verify.hostnames", true);
+ // credentials are withheld over connections whose server certificate
+ // was not validated, unless explicitly opted in
+ final boolean insecureCredentialsAllowed =
+ ConfUtils.getBoolean(conf, "http.credentials.allow.insecure",
false);
+ this.sendCredentials = !trustEverything || insecureCredentialsAllowed;
Review Comment:
Addressed in a5d7ba15 — see the review summary comment.
##########
core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java:
##########
@@ -108,6 +107,23 @@ public class HttpProtocol extends AbstractHttpProtocol {
// makes sure that a missing cookie origin is reported once and not for
every url
private final AtomicBoolean missingCookieOriginLogged = new
AtomicBoolean();
+ // makes sure that withheld cookies are reported once and not for every url
+ private final AtomicBoolean withheldCookiesLogged = new AtomicBoolean();
+
+ // whether credentials (basic auth, credential headers, cookies) may be
sent:
+ // false when the servers are not authenticated and
+ // http.credentials.allow.insecure is not enabled
+ private boolean sendCredentials = true;
+
+ /** Header names carrying credentials, see {@link
#isCredentialHeader(String)}. */
+ private static final Set<String> CREDENTIAL_HEADERS =
Review Comment:
Addressed in a5d7ba15 — see the review summary comment.
##########
core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java:
##########
@@ -285,6 +347,15 @@ public EventListener create(Call call) {
}
private void addCookiesToRequest(Builder rb, String url, Metadata md) {
+ if (!sendCredentials) {
Review Comment:
Addressed in a5d7ba15 — see the review summary comment.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
