This is an automated email from the ASF dual-hosted git repository. dzamo pushed a commit to branch 1.20 in repository https://gitbox.apache.org/repos/asf/drill.git
commit fb1be8d2044c9981611e03a1835cb8cab4c664cb Author: Charles S. Givre <[email protected]> AuthorDate: Mon Sep 5 19:44:08 2022 +0800 DRILL-8291: Allow Case Sensitive Filters in HTTP Plugin (#2639) * DRILL-8291: Allow Case Sensitive Filters in HTTP Plugin --- contrib/storage-http/README.md | 7 +++++-- .../apache/drill/exec/store/http/HttpApiConfig.java | 19 ++++++++++++++++++- .../drill/exec/store/http/HttpPushDownListener.java | 9 ++++++++- .../apache/drill/exec/store/http/util/SimpleHttp.java | 4 +--- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/contrib/storage-http/README.md b/contrib/storage-http/README.md index 79c76f6491..35105554fd 100644 --- a/contrib/storage-http/README.md +++ b/contrib/storage-http/README.md @@ -305,6 +305,9 @@ on 400 series errors, `false` will return an empty result set (with implicit fie Default is `true`, but when set to false, Drill will trust all SSL certificates. Useful for debugging or on internal corporate networks using self-signed certificates or private certificate authorities. +#### caseSensitiveFilters +Some APIs are case sensitive with the fields which are pushed down. If the endpoint that you are working with is in fact case sensitive, simply set this to `true`. Defaults to `false`. + ## Usage This plugin is different from other plugins in that it the table component of the `FROM` clause @@ -576,9 +579,9 @@ ORDER BY issue_count DESC you may encounter errors or empty responses if you are counting on the endpoint for redirection. -2. At this time, the plugin does not support any authentication other than basic authentication. +~~2. At this time, the plugin does not support any authentication other than basic authentication.~~ -3. This plugin does not implement join filter pushdowns (only constant plushdowns are +3. This plugin does not implement join filter pushdowns (only constant pushdowns are supported). Join pushdown has the potential to improve performance if you use the HTTP service joined to another table. diff --git a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java index 09d6b2ff63..83dd631f09 100644 --- a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java +++ b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java @@ -94,6 +94,8 @@ public class HttpApiConfig { private final String limitQueryParam; @JsonProperty private final boolean errorOn400; + @JsonProperty + private final boolean caseSensitiveFilters; // Enables the user to configure JSON options at the connection level rather than globally. @JsonProperty @@ -186,6 +188,7 @@ public class HttpApiConfig { && errorOn400 == that.errorOn400 && verifySSLCert == that.verifySSLCert && directCredentials == that.directCredentials + && caseSensitiveFilters == that.caseSensitiveFilters && Objects.equals(url, that.url) && Objects.equals(method, that.method) && Objects.equals(postBody, that.postBody) @@ -204,7 +207,7 @@ public class HttpApiConfig { public int hashCode() { return Objects.hash(url, requireTail, method, postBody, headers, params, dataPath, authType, inputType, xmlDataLevel, limitQueryParam, errorOn400, jsonOptions, verifySSLCert, - credentialsProvider, paginator, directCredentials); + credentialsProvider, paginator, directCredentials, caseSensitiveFilters); } @Override @@ -217,6 +220,7 @@ public class HttpApiConfig { .field("headers", headers) .field("params", params) .field("dataPath", dataPath) + .field("caseSensitiveFilters", caseSensitiveFilters) .field("authType", authType) .field("inputType", inputType) .field("xmlDataLevel", xmlDataLevel) @@ -285,6 +289,7 @@ public class HttpApiConfig { this.xmlDataLevel = Math.max(1, builder.xmlDataLevel); this.errorOn400 = builder.errorOn400; + this.caseSensitiveFilters = builder.caseSensitiveFilters; this.credentialsProvider = CredentialProviderUtils.getCredentialsProvider(builder.userName, builder.password, builder.credentialsProvider); this.directCredentials = builder.credentialsProvider == null; @@ -331,6 +336,11 @@ public class HttpApiConfig { return credentialsProvider; } + @JsonProperty + public boolean caseSensitiveFilters() { + return this.caseSensitiveFilters; + } + @JsonPOJOBuilder(withPrefix = "") public static class HttpApiConfigBuilder { private String userName; @@ -359,6 +369,8 @@ public class HttpApiConfig { private int xmlDataLevel; + private boolean caseSensitiveFilters; + private String limitQueryParam; private boolean errorOn400; @@ -425,6 +437,11 @@ public class HttpApiConfig { return this; } + public HttpApiConfigBuilder caseSensitiveFilters(boolean caseSensitiveFilters) { + this.caseSensitiveFilters = caseSensitiveFilters; + return this; + } + public HttpApiConfigBuilder method(String method) { this.method = method; return this; diff --git a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java index c4f55967e6..30f4be57fa 100644 --- a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java +++ b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java @@ -35,6 +35,7 @@ import org.apache.drill.exec.store.http.HttpPaginatorConfig.PaginatorMethod; import org.apache.drill.exec.store.http.util.SimpleHttp; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -175,7 +176,13 @@ public class HttpPushDownListener implements FilterPushDownListener { */ @Override public Pair<GroupScan, List<RexNode>> transform(AndNode andNode) { - Map<String, String> filters = CaseInsensitiveMap.newHashMap(); + Map<String, String> filters; + if (groupScan.getHttpConfig().caseSensitiveFilters()) { + filters = new HashMap<>(); + } else { + filters = CaseInsensitiveMap.newHashMap(); + } + double selectivity = 1; for (ExprNode expr : andNode.children) { ColRelOpConstNode relOp = (ColRelOpConstNode) expr; diff --git a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java index 777f49d207..03e4d5e5cc 100644 --- a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java +++ b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java @@ -28,7 +28,6 @@ import okhttp3.Request; import okhttp3.Response; import org.apache.commons.lang3.StringUtils; -import org.apache.drill.common.map.CaseInsensitiveMap; import org.apache.drill.common.exceptions.CustomErrorContext; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.oauth.PersistentTokenTable; @@ -529,7 +528,6 @@ public class SimpleHttp { .message("API Query with URL Parameters must be populated.") .build(logger); } - CaseInsensitiveMap<String>caseInsensitiveFilterMap = (CaseInsensitiveMap<String>)filters; List<String> params = SimpleHttp.getURLParameters(url); String tempUrl = SimpleHttp.decodedURL(url); @@ -543,7 +541,7 @@ public class SimpleHttp { // of providing helpful errors in strange cases, it is there. - String value = caseInsensitiveFilterMap.get(param); + String value = filters.get(param); // Check and see if there is a default for this parameter. If not throw an error. if (StringUtils.isEmpty(value)) {
