This is an automated email from the ASF dual-hosted git repository.
cgivre pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 4e827e9 DRILL-8161: Add Global Credentials to HTTP Storage Plugin
(#2486)
4e827e9 is described below
commit 4e827e905b02bdbcacda2ff075610bf9449e8736
Author: Charles S. Givre <[email protected]>
AuthorDate: Wed Mar 9 17:43:08 2022 -0500
DRILL-8161: Add Global Credentials to HTTP Storage Plugin (#2486)
* DRILL-8161: Add Global Credentials to HTTP Storage Plugin
* Removed unused import
* Formatting from code review
* Updated README
---
contrib/storage-http/README.md | 16 +++++++
.../exec/store/http/HttpStoragePluginConfig.java | 35 ++++++++++++---
.../drill/exec/store/http/util/SimpleHttp.java | 33 +++++++++++++-
.../drill/exec/store/http/TestHttpPlugin.java | 17 +++++++-
.../drill/exec/store/http/TestOAuthProcess.java | 2 +-
.../exec/store/http/TestOAuthTokenUpdate.java | 2 +-
.../drill/exec/store/http/TestPagination.java | 10 ++++-
.../store/security/CredentialProviderUtils.java | 12 +++++-
.../UsernamePasswordWithProxyCredentials.java | 50 ++++++++++++++++++++++
.../security/oauth/OAuthTokenCredentials.java | 2 +
10 files changed, 164 insertions(+), 15 deletions(-)
diff --git a/contrib/storage-http/README.md b/contrib/storage-http/README.md
index 40a57bf..8154f0a 100644
--- a/contrib/storage-http/README.md
+++ b/contrib/storage-http/README.md
@@ -282,6 +282,22 @@ If the `authType` is set to `basic`, `username` and
`password` must be set in th
`password`: The password for basic authentication.
+##### Global Credentials
+If you have an HTTP plugin with multiple endpoints that all use the same
credentials, you can set the `authType` to `basic` and set global
+credentials in the storage plugin configuration.
+
+Simply add the following to the storage plugin configuration:
+```json
+ "credentialsProvider": {
+ "credentialsProviderType": "PlainCredentialsProvider",
+ "credentials": {
+ "username": "user1",
+ "password": "user1Pass"
+ }
+ }
+```
+Note that the `authType` still must be set to `basic` and that any endpoint
credentials will override the global credentials.
+
#### Limiting Results
Some APIs support a query parameter which is used to limit the number of
results returned by the API. In this case you can set the `limitQueryParam`
config variable to the query parameter name and Drill will automatically
include this in your query. For instance, if you have an API which supports a
limit query parameter called `maxRecords` and you set the abovementioned config
variable then execute the following query:
diff --git
a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpStoragePluginConfig.java
b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpStoragePluginConfig.java
index ba32c82..721db6a 100644
---
a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpStoragePluginConfig.java
+++
b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpStoragePluginConfig.java
@@ -28,8 +28,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import org.apache.drill.exec.store.security.CredentialProviderUtils;
import org.apache.drill.common.logical.security.CredentialsProvider;
+import
org.apache.drill.exec.store.security.UsernamePasswordWithProxyCredentials;
import org.apache.drill.exec.store.security.oauth.OAuthTokenCredentials;
-import org.apache.drill.exec.store.security.UsernamePasswordCredentials;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,6 +60,8 @@ public class HttpStoragePluginConfig extends
AbstractSecuredStoragePluginConfig
public HttpStoragePluginConfig(@JsonProperty("cacheResults") Boolean
cacheResults,
@JsonProperty("connections") Map<String,
HttpApiConfig> connections,
@JsonProperty("timeout") Integer timeout,
+ @JsonProperty("username") String username,
+ @JsonProperty("password") String password,
@JsonProperty("proxyHost") String proxyHost,
@JsonProperty("proxyPort") Integer proxyPort,
@JsonProperty("proxyType") String proxyType,
@@ -72,6 +74,8 @@ public class HttpStoragePluginConfig extends
AbstractSecuredStoragePluginConfig
getClientID(new OAuthTokenCredentials(credentialsProvider)),
getClientSecret(new OAuthTokenCredentials(credentialsProvider)),
getTokenURL(new OAuthTokenCredentials(credentialsProvider)),
+ normalize(username),
+ normalize(password),
normalize(proxyUsername),
normalize(proxyPassword),
credentialsProvider),
@@ -139,7 +143,10 @@ public class HttpStoragePluginConfig extends
AbstractSecuredStoragePluginConfig
public HttpStoragePluginConfig copyForPlan(String connectionName) {
return new HttpStoragePluginConfig(
cacheResults, configFor(connectionName), timeout,
- proxyHost, proxyPort, proxyType, null, null, oAuthConfig,
credentialsProvider);
+ getUsernamePasswordCredentials().getUsername(),
+ getUsernamePasswordCredentials().getPassword(),
+ proxyHost, proxyPort, proxyType,
getUsernamePasswordCredentials().getProxyUsername(),
+ getUsernamePasswordCredentials().getProxyPassword(), oAuthConfig,
credentialsProvider);
}
private Map<String, HttpApiConfig> configFor(String connectionName) {
@@ -205,10 +212,26 @@ public class HttpStoragePluginConfig extends
AbstractSecuredStoragePluginConfig
return oAuthConfig;
}
+ @JsonProperty("username")
+ public String username() {
+ if (directCredentials) {
+ return getUsernamePasswordCredentials().getUsername();
+ }
+ return null;
+ }
+
+ @JsonProperty("password")
+ public String password() {
+ if (directCredentials) {
+ return getUsernamePasswordCredentials().getPassword();
+ }
+ return null;
+ }
+
@JsonProperty("proxyUsername")
public String proxyUsername() {
if (directCredentials) {
- return getUsernamePasswordCredentials().getUsername();
+ return getUsernamePasswordCredentials().getProxyUsername();
}
return null;
}
@@ -216,7 +239,7 @@ public class HttpStoragePluginConfig extends
AbstractSecuredStoragePluginConfig
@JsonProperty("proxyPassword")
public String proxyPassword() {
if (directCredentials) {
- return getUsernamePasswordCredentials().getPassword();
+ return getUsernamePasswordCredentials().getProxyPassword();
}
return null;
}
@@ -245,8 +268,8 @@ public class HttpStoragePluginConfig extends
AbstractSecuredStoragePluginConfig
}
@JsonIgnore
- public UsernamePasswordCredentials getUsernamePasswordCredentials() {
- return new UsernamePasswordCredentials(credentialsProvider);
+ public UsernamePasswordWithProxyCredentials getUsernamePasswordCredentials()
{
+ return new UsernamePasswordWithProxyCredentials(credentialsProvider);
}
@JsonIgnore
diff --git
a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java
b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java
index 80dac2d..971e039 100644
---
a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java
+++
b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java
@@ -146,9 +146,10 @@ public class SimpleHttp {
builder.authenticator(new AccessTokenAuthenticator(repository));
builder.addInterceptor(new AccessTokenInterceptor(repository));
} else if (apiConfig.authType().equalsIgnoreCase("basic")) {
- // If the API uses basic authentication add the authentication code.
+ // If the API uses basic authentication add the authentication code.
Use the global credentials unless there are credentials
+ // for the specific endpoint.
logger.debug("Adding Interceptor");
- UsernamePasswordCredentials credentials =
apiConfig.getUsernamePasswordCredentials();
+ UsernamePasswordCredentials credentials = getCredentials();
builder.addInterceptor(new
BasicAuthInterceptor(credentials.getUsername(), credentials.getPassword()));
}
@@ -362,6 +363,34 @@ public class SimpleHttp {
}
/**
+ * Logic to determine whether the API connection has global credentials or
credentials specific for the
+ * API endpoint.
+ * @param endpointConfig The API endpoint configuration
+ * @return True if the endpoint has credentials, false if not.
+ */
+ private boolean hasEndpointCredentials(HttpApiConfig endpointConfig) {
+ UsernamePasswordCredentials credentials =
endpointConfig.getUsernamePasswordCredentials();
+ if (StringUtils.isNotEmpty(credentials.getUsername()) &&
+ StringUtils.isNotEmpty(credentials.getPassword())) {
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * If the user has defined username/password for the specific API endpoint,
pass the API endpoint credentials.
+ * Otherwise, use the global connection credentials.
+ * @return A UsernamePasswordCredentials collection with the correct
username/password
+ */
+ private UsernamePasswordCredentials getCredentials() {
+ if (hasEndpointCredentials(apiConfig)) {
+ return apiConfig.getUsernamePasswordCredentials();
+ } else {
+ return scanDefn.tableSpec().config().getUsernamePasswordCredentials();
+ }
+ }
+
+ /**
* Gets the HTTP response code from the HTTP call. Note that this value
* is only available after the getInputStream() method has been called.
*
diff --git
a/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestHttpPlugin.java
b/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestHttpPlugin.java
index 38e1f96..1ddbb05 100644
---
a/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestHttpPlugin.java
+++
b/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestHttpPlugin.java
@@ -52,6 +52,7 @@ import java.util.concurrent.TimeUnit;
import static org.apache.drill.test.rowSet.RowSetUtilities.mapArray;
import static org.apache.drill.test.rowSet.RowSetUtilities.mapValue;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -128,7 +129,7 @@ public class TestHttpPlugin extends ClusterTest {
configs.put("pokemon", pokemonConfig);
HttpStoragePluginConfig mockStorageConfigWithWorkspace =
- new HttpStoragePluginConfig(false, configs, 10, "", 80, "", "", "",
null, PlainCredentialsProvider.EMPTY_CREDENTIALS_PROVIDER);
+ new HttpStoragePluginConfig(false, configs, 10, null, null, "", 80,
"", "", "", null, PlainCredentialsProvider.EMPTY_CREDENTIALS_PROVIDER);
mockStorageConfigWithWorkspace.setEnabled(true);
cluster.defineStoragePlugin("live", mockStorageConfigWithWorkspace);
}
@@ -240,6 +241,7 @@ public class TestHttpPlugin extends ClusterTest {
HttpApiConfig mockPostConfigWithoutPostBody = HttpApiConfig.builder()
.url("http://localhost:8091/";)
.method("POST")
+ .authType("basic")
.headers(headers)
.build();
@@ -332,7 +334,10 @@ public class TestHttpPlugin extends ClusterTest {
configs.put("mockJsonAllText", mockTableWithJsonOptions);
HttpStoragePluginConfig mockStorageConfigWithWorkspace =
- new HttpStoragePluginConfig(false, configs, 2, "", 80, "", "", "",
null, PlainCredentialsProvider.EMPTY_CREDENTIALS_PROVIDER);
+ new HttpStoragePluginConfig(false, configs, 2, "globaluser",
"globalpass", "",
+ 80, "", "", "", null, new PlainCredentialsProvider(ImmutableMap.of(
+ UsernamePasswordCredentials.USERNAME, "globaluser",
+ UsernamePasswordCredentials.PASSWORD, "globalpass")));
mockStorageConfigWithWorkspace.setEnabled(true);
cluster.defineStoragePlugin("local", mockStorageConfigWithWorkspace);
}
@@ -785,6 +790,11 @@ public class TestHttpPlugin extends ClusterTest {
.build();
RowSetUtilities.verify(expected, results);
+
+ // Verify correct username/password from endpoint configuration
+ RecordedRequest recordedRequest = server.takeRequest();
+ assertNotNull(recordedRequest.getHeader("Authorization"));
+ assertEquals("Basic dXNlcjpwYXNz",
recordedRequest.getHeader("Authorization"));
}
}
@@ -961,6 +971,9 @@ public class TestHttpPlugin extends ClusterTest {
RecordedRequest recordedRequest = server.takeRequest();
assertEquals("POST", recordedRequest.getMethod());
+ // Verify correct username/password from global configuration
+ assertNotNull(recordedRequest.getHeader("Authorization"));
+ assertEquals("Basic Z2xvYmFsdXNlcjpnbG9iYWxwYXNz",
recordedRequest.getHeader("Authorization"));
}
}
diff --git
a/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestOAuthProcess.java
b/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestOAuthProcess.java
index 943a331..521dcb2 100644
---
a/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestOAuthProcess.java
+++
b/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestOAuthProcess.java
@@ -109,7 +109,7 @@ public class TestOAuthProcess extends ClusterTest {
// Add storage plugin for test OAuth
HttpStoragePluginConfig mockStorageConfigWithWorkspace =
- new HttpStoragePluginConfig(false, configs, TIMEOUT, "", 80, "", "", "",
+ new HttpStoragePluginConfig(false, configs, TIMEOUT, null, null, "", 80,
"", "", "",
oAuthConfig, credentialsProvider);
mockStorageConfigWithWorkspace.setEnabled(true);
cluster.defineStoragePlugin("localOauth", mockStorageConfigWithWorkspace);
diff --git
a/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestOAuthTokenUpdate.java
b/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestOAuthTokenUpdate.java
index 4171af9..dbadcd1 100644
---
a/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestOAuthTokenUpdate.java
+++
b/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestOAuthTokenUpdate.java
@@ -89,7 +89,7 @@ public class TestOAuthTokenUpdate extends ClusterTest {
// Add storage plugin for test OAuth
HttpStoragePluginConfig mockStorageConfigWithWorkspace =
- new HttpStoragePluginConfig(false, configs, TIMEOUT, "", 80, "", "", "",
+ new HttpStoragePluginConfig(false, configs, TIMEOUT,null, null, "", 80,
"", "", "",
oAuthConfig, credentialsProvider);
mockStorageConfigWithWorkspace.setEnabled(true);
cluster.defineStoragePlugin("localOauth", mockStorageConfigWithWorkspace);
diff --git
a/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestPagination.java
b/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestPagination.java
index d5ad618..45f6263 100644
---
a/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestPagination.java
+++
b/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestPagination.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.store.http;
import okhttp3.mockwebserver.MockResponse;
import okhttp3.mockwebserver.MockWebServer;
+import okhttp3.mockwebserver.RecordedRequest;
import org.apache.drill.common.logical.security.PlainCredentialsProvider;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.util.DrillFileUtils;
@@ -47,6 +48,7 @@ import java.util.List;
import java.util.Map;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
public class TestPagination extends ClusterTest {
private static final int MOCK_SERVER_PORT = 8092;
@@ -113,7 +115,7 @@ public class TestPagination extends ClusterTest {
configs.put("github", githubConfig);
HttpStoragePluginConfig mockStorageConfigWithWorkspace =
- new HttpStoragePluginConfig(false, configs, 10, "", 80, "", "", "",
null, PlainCredentialsProvider.EMPTY_CREDENTIALS_PROVIDER);
+ new HttpStoragePluginConfig(false, configs, 10, null, null, "", 80, "",
"", "", null, PlainCredentialsProvider.EMPTY_CREDENTIALS_PROVIDER);
mockStorageConfigWithWorkspace.setEnabled(true);
cluster.defineStoragePlugin("live", mockStorageConfigWithWorkspace);
}
@@ -193,7 +195,7 @@ public class TestPagination extends ClusterTest {
configs.put("xml_paginator_url_params",
mockXmlConfigWithPaginatorAndUrlParams);
HttpStoragePluginConfig mockStorageConfigWithWorkspace =
- new HttpStoragePluginConfig(false, configs, 2, "", 80, "", "", "", null,
PlainCredentialsProvider.EMPTY_CREDENTIALS_PROVIDER);
+ new HttpStoragePluginConfig(false, configs, 2, null, null, "", 80, "",
"", "", null, PlainCredentialsProvider.EMPTY_CREDENTIALS_PROVIDER);
mockStorageConfigWithWorkspace.setEnabled(true);
cluster.defineStoragePlugin("local", mockStorageConfigWithWorkspace);
}
@@ -349,6 +351,10 @@ public class TestPagination extends ClusterTest {
b.release();
}
assertEquals(6, count);
+
+ // Verify that there are no random headers being inserted if
authorization is not defined.
+ RecordedRequest recordedRequest = server.takeRequest();
+ assertNull(recordedRequest.getHeader("Authorization"));
}
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/security/CredentialProviderUtils.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/security/CredentialProviderUtils.java
index 4ad2463..297c193 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/security/CredentialProviderUtils.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/security/CredentialProviderUtils.java
@@ -54,8 +54,10 @@ public class CredentialProviderUtils {
* @param clientID The OAuth Client ID. This is provided by the application
during signup.
* @param clientSecret The OAUth Client Secret. This is provided by the
application during signup.
* @param tokenURI The URI from which you swap the auth code for access and
refresh tokens.
- * @param username Optional username for proxy or other services
+ * @param username Optional username for other services
* @param password Optional password for proxy or other services
+ * @param proxyUsername Optional username for a proxy server.
+ * @param proxyPassword Optional password for a proxy server.
* @param credentialsProvider The credential store which retains the
credentials.
* @return A credential provider with the access tokens
*/
@@ -65,6 +67,8 @@ public class CredentialProviderUtils {
String tokenURI,
String username,
String password,
+ String proxyUsername,
+ String proxyPassword,
CredentialsProvider credentialsProvider) {
if (credentialsProvider != null) {
@@ -86,6 +90,12 @@ public class CredentialProviderUtils {
if (tokenURI != null) {
mapBuilder.put(OAuthTokenCredentials.TOKEN_URI, tokenURI);
}
+ if (proxyUsername != null) {
+ mapBuilder.put(OAuthTokenCredentials.PROXY_USERNAME, proxyUsername);
+ }
+ if (proxyPassword != null) {
+ mapBuilder.put(OAuthTokenCredentials.PROXY_PASSWORD, proxyPassword);
+ }
return new PlainCredentialsProvider(mapBuilder.build());
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/security/UsernamePasswordWithProxyCredentials.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/security/UsernamePasswordWithProxyCredentials.java
new file mode 100644
index 0000000..7ae56cc
--- /dev/null
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/security/UsernamePasswordWithProxyCredentials.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.store.security;
+
+import org.apache.drill.common.logical.security.CredentialsProvider;
+import org.apache.drill.exec.store.security.oauth.OAuthTokenCredentials;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class UsernamePasswordWithProxyCredentials extends
UsernamePasswordCredentials {
+ private final String proxyUsername;
+ private final String proxyPassword;
+
+ public UsernamePasswordWithProxyCredentials(CredentialsProvider
credentialsProvider) {
+ super(credentialsProvider);
+ if (credentialsProvider == null || credentialsProvider.getCredentials() ==
null) {
+ this.proxyUsername = null;
+ this.proxyPassword = null;
+ } else {
+ Map<String, String> credentials = credentialsProvider.getCredentials()
== null ? new HashMap<>() : credentialsProvider.getCredentials();
+ this.proxyUsername =
credentials.get(OAuthTokenCredentials.PROXY_USERNAME);
+ this.proxyPassword =
credentials.get(OAuthTokenCredentials.PROXY_PASSWORD);
+ }
+ }
+
+ public String getProxyUsername() {
+ return proxyUsername;
+ }
+
+ public String getProxyPassword() {
+ return proxyPassword;
+ }
+}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/security/oauth/OAuthTokenCredentials.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/security/oauth/OAuthTokenCredentials.java
index 2e39ee3..516d872 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/security/oauth/OAuthTokenCredentials.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/security/oauth/OAuthTokenCredentials.java
@@ -32,6 +32,8 @@ public class OAuthTokenCredentials extends
UsernamePasswordCredentials {
public static final String ACCESS_TOKEN = "accessToken";
public static final String REFRESH_TOKEN = "refreshToken";
public static final String TOKEN_URI = "tokenURI";
+ public static final String PROXY_USERNAME = "proxyUsername";
+ public static final String PROXY_PASSWORD = "proxyPassword";
private final String clientID;
private final String clientSecret;
