This is an automated email from the ASF dual-hosted git repository.
cgivre pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new d9c4227e80 DRILL-8291: Allow Case Sensitive Filters in HTTP Plugin
(#2639)
d9c4227e80 is described below
commit d9c4227e80aae4259cf8eb293de1d51209aee5a3
Author: Charles S. Givre <[email protected]>
AuthorDate: Mon Sep 5 07:44:08 2022 -0400
DRILL-8291: Allow Case Sensitive Filters in HTTP Plugin (#2639)
* DRILL-8291: Allow Case Sensitive Filters in HTTP Plugin
---
contrib/storage-http/README.md | 7 +++++--
.../apache/drill/exec/store/http/HttpApiConfig.java | 19 +++++++++++++++++--
.../drill/exec/store/http/HttpPushDownListener.java | 9 ++++++++-
.../apache/drill/exec/store/http/util/SimpleHttp.java | 4 +---
4 files changed, 31 insertions(+), 8 deletions(-)
diff --git a/contrib/storage-http/README.md b/contrib/storage-http/README.md
index 0f188ba0cc..d384bedd11 100644
--- a/contrib/storage-http/README.md
+++ b/contrib/storage-http/README.md
@@ -306,6 +306,9 @@ on 400 series errors, `false` will return an empty result
set (with implicit fie
Default is `true`, but when set to false, Drill will trust all SSL
certificates. Useful for debugging or on internal corporate networks using
self-signed certificates or
private certificate authorities.
+#### caseSensitiveFilters
+Some APIs are case sensitive with the fields which are pushed down. If the
endpoint that you are working with is in fact case sensitive, simply set this
to `true`. Defaults to `false`.
+
## Usage
This plugin is different from other plugins in that it the table component of
the `FROM` clause
@@ -577,9 +580,9 @@ ORDER BY issue_count DESC
you may encounter errors or empty responses if you are counting on the
endpoint for
redirection.
-2. At this time, the plugin does not support any authentication other than
basic authentication.
+~~2. At this time, the plugin does not support any authentication other than
basic authentication.~~
-3. This plugin does not implement join filter pushdowns (only constant
plushdowns are
+3. This plugin does not implement join filter pushdowns (only constant
pushdowns are
supported). Join pushdown has the potential to improve performance if you
use the HTTP service
joined to another table.
diff --git
a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java
b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java
index 581b61cfcc..91af33e36f 100644
---
a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java
+++
b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java
@@ -104,6 +104,9 @@ public class HttpApiConfig {
@JsonProperty
private final boolean errorOn400;
+ @JsonProperty
+ private final boolean caseSensitiveFilters;
+
// Enables the user to configure JSON options at the connection level rather
than globally.
@JsonProperty
private final HttpJsonOptions jsonOptions;
@@ -157,6 +160,10 @@ public class HttpApiConfig {
return this.inputType;
}
+ public boolean caseSensitiveFilters() {
+ return this.caseSensitiveFilters;
+ }
+
public int xmlDataLevel() {
return this.xmlDataLevel;
}
@@ -199,6 +206,7 @@ public class HttpApiConfig {
&& errorOn400 == that.errorOn400
&& verifySSLCert == that.verifySSLCert
&& directCredentials == that.directCredentials
+ && caseSensitiveFilters == that.caseSensitiveFilters
&& Objects.equals(url, that.url)
&& Objects.equals(method, that.method)
&& Objects.equals(postBody, that.postBody)
@@ -218,7 +226,7 @@ public class HttpApiConfig {
public int hashCode() {
return Objects.hash(url, requireTail, method, postBody, headers, params,
dataPath,
authType, inputType, xmlDataLevel, limitQueryParam, errorOn400,
jsonOptions, verifySSLCert,
- credentialsProvider, paginator, directCredentials,
postParameterLocation);
+ credentialsProvider, paginator, directCredentials,
postParameterLocation, caseSensitiveFilters);
}
@Override
@@ -232,6 +240,7 @@ public class HttpApiConfig {
.field("headers", headers)
.field("params", params)
.field("dataPath", dataPath)
+ .field("caseSensitiveFilters", caseSensitiveFilters)
.field("authType", authType)
.field("inputType", inputType)
.field("xmlDataLevel", xmlDataLevel)
@@ -328,6 +337,7 @@ public class HttpApiConfig {
this.xmlDataLevel = Math.max(1, builder.xmlDataLevel);
this.errorOn400 = builder.errorOn400;
+ this.caseSensitiveFilters = builder.caseSensitiveFilters;
this.credentialsProvider =
CredentialProviderUtils.getCredentialsProvider(builder.userName,
builder.password, builder.credentialsProvider);
this.directCredentials = builder.credentialsProvider == null;
@@ -422,7 +432,7 @@ public class HttpApiConfig {
private String authType;
private int xmlDataLevel;
-
+ private boolean caseSensitiveFilters;
private String limitQueryParam;
private boolean errorOn400;
@@ -489,6 +499,11 @@ public class HttpApiConfig {
return this;
}
+ public HttpApiConfigBuilder caseSensitiveFilters(boolean
caseSensitiveFilters) {
+ this.caseSensitiveFilters = caseSensitiveFilters;
+ return this;
+ }
+
public HttpApiConfigBuilder method(String method) {
this.method = method;
return this;
diff --git
a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java
b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java
index c4f55967e6..30f4be57fa 100644
---
a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java
+++
b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java
@@ -35,6 +35,7 @@ import
org.apache.drill.exec.store.http.HttpPaginatorConfig.PaginatorMethod;
import org.apache.drill.exec.store.http.util.SimpleHttp;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -175,7 +176,13 @@ public class HttpPushDownListener implements
FilterPushDownListener {
*/
@Override
public Pair<GroupScan, List<RexNode>> transform(AndNode andNode) {
- Map<String, String> filters = CaseInsensitiveMap.newHashMap();
+ Map<String, String> filters;
+ if (groupScan.getHttpConfig().caseSensitiveFilters()) {
+ filters = new HashMap<>();
+ } else {
+ filters = CaseInsensitiveMap.newHashMap();
+ }
+
double selectivity = 1;
for (ExprNode expr : andNode.children) {
ColRelOpConstNode relOp = (ColRelOpConstNode) expr;
diff --git
a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java
b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java
index e5f0ce70d5..ee5b285f1e 100644
---
a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java
+++
b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java
@@ -35,7 +35,6 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.drill.common.exceptions.EmptyErrorContext;
import org.apache.drill.common.logical.OAuthConfig;
import org.apache.drill.common.logical.StoragePluginConfig.AuthMode;
-import org.apache.drill.common.map.CaseInsensitiveMap;
import org.apache.drill.common.exceptions.CustomErrorContext;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.ExecConstants;
@@ -712,7 +711,6 @@ public class SimpleHttp {
.message("API Query with URL Parameters must be populated.")
.build(logger);
}
- CaseInsensitiveMap<String>caseInsensitiveFilterMap =
(CaseInsensitiveMap<String>)filters;
List<String> params = SimpleHttp.getURLParameters(url);
String tempUrl = SimpleHttp.decodedURL(url);
@@ -726,7 +724,7 @@ public class SimpleHttp {
// of providing helpful errors in strange cases, it is there.
- String value = caseInsensitiveFilterMap.get(param);
+ String value = filters.get(param);
// Check and see if there is a default for this parameter. If not throw
an error.
if (StringUtils.isEmpty(value)) {
