This is an automated email from the ASF dual-hosted git repository.
cgivre pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new fe1d15f DRILL-8047: Add a custom authn provider for HashiCorp Vault
(#2394)
fe1d15f is described below
commit fe1d15f1941334792e3f1c1dd53d932dd1b66279
Author: James Turton <[email protected]>
AuthorDate: Fri Dec 10 00:59:32 2021 +0200
DRILL-8047: Add a custom authn provider for HashiCorp Vault (#2394)
* Add Vault user authenticator and unit tests.
* Fix test failures.
* Enable Vault token auth method.
* Fix bug affecting token owner check.
* Add Vault token auth method unit test.
* Get token auth unit test passing.
---
.../rpc/user/security/VaultUserAuthenticator.java | 228 +++++++++++++++++++++
.../server/rest/auth/DrillRestLoginService.java | 2 +-
.../security/vault/VaultCredentialsProvider.java | 1 +
.../TestHtpasswdFileUserAuthenticator.java | 3 +
.../user/security/TestVaultUserAuthenticator.java | 157 ++++++++++++++
5 files changed, 390 insertions(+), 1 deletion(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/security/VaultUserAuthenticator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/security/VaultUserAuthenticator.java
new file mode 100644
index 0000000..110d5f7
--- /dev/null
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/security/VaultUserAuthenticator.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc.user.security;
+
+import com.bettercloud.vault.response.LookupResponse;
+import com.bettercloud.vault.Vault;
+import com.bettercloud.vault.VaultConfig;
+import com.bettercloud.vault.VaultException;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.exception.DrillbitStartupException;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import lombok.EqualsAndHashCode;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Implement {@link org.apache.drill.exec.rpc.user.security.UserAuthenticator}
+ * based on HashiCorp Vault. Configure the Vault client using the Drill BOOT
+ * options that appear below.
+ */
+@Slf4j
+@EqualsAndHashCode
+@UserAuthenticatorTemplate(type = "vault")
+public class VaultUserAuthenticator implements UserAuthenticator {
+
+ // Drill boot options used to configure Vault auth.
+ public static final String VAULT_ADDRESS =
"drill.exec.security.user.auth.vault.address";
+ public static final String VAULT_TOKEN =
"drill.exec.security.user.auth.vault.token";
+ public static final String VAULT_AUTH_METHOD =
"drill.exec.security.user.auth.vault.method";
+
+ // The subset of Vault auth methods that are supported by this authenticator
+ public enum VaultAuthMethod {
+ APP_ROLE,
+ LDAP,
+ USER_PASS,
+ VAULT_TOKEN
+ }
+
+ private VaultConfig vaultConfig;
+
+ private Vault vault;
+
+ private VaultAuthMethod authMethod;
+
+ /**
+ * Reads Drill BOOT options and uses them to set up a Vault client.
+ * @param config object providing Drill config settings for Vault
+ * @throws DrillbitStartupException if the provided Vault configuration is
invalid
+ */
+ @Override
+ public void setup(DrillConfig config) throws DrillbitStartupException {
+ // Read config values
+ String vaultAddress = Objects.requireNonNull(
+ config.getString(VAULT_ADDRESS),
+ String.format(
+ "Vault address BOOT option is not specified. Please set [%s] config " +
+ "option.",
+ VAULT_ADDRESS
+ )
+ );
+
+ this.authMethod = VaultAuthMethod.valueOf(
+ Objects.requireNonNull(
+ config.getString(VAULT_AUTH_METHOD),
+ String.format(
+ "Vault auth method is not specified. Please set [%s] config option.",
+ VAULT_AUTH_METHOD
+ )
+ )
+ );
+
+ VaultConfig vaultConfBuilder = new VaultConfig().address(vaultAddress);
+ String vaultToken = config.hasPath(VAULT_TOKEN)
+ ? config.getString(VAULT_TOKEN)
+ : null;
+
+ if (this.authMethod == VaultAuthMethod.VAULT_TOKEN) {
+ // Drill will use end users' Vault tokens for Vault operations
+ if (vaultToken != null) {
+ logger.warn(
+ "When Drill is set to authenticate using end user Vault tokens the "
+
+ "[{}] BOOT option is ignored.",
+ VAULT_TOKEN
+ );
+ }
+ } else {
+ // Drill needs its own Vault token set in a BOOT option
+ if (vaultToken == null) {
+ throw new DrillbitStartupException(String.format(
+ "Only the %s method is supported if you do not set a token in the " +
+ "[%s] config option.",
+ VaultAuthMethod.VAULT_TOKEN,
+ VAULT_TOKEN
+ ));
+ }
+ vaultConfBuilder.token(vaultToken);
+ }
+
+ // Initialise Vault client
+ try {
+ logger.debug(
+ "Tries to init a Vault client with Vault addr = {}, auth method = {}",
+ vaultAddress,
+ authMethod
+ );
+
+ this.vaultConfig = vaultConfBuilder.build();
+ this.vault = new Vault(this.vaultConfig);
+ } catch (VaultException e) {
+ logger.error(String.join(System.lineSeparator(),
+ "Error initialising the Vault client library using configuration: ",
+ "\tvaultAddress: {}",
+ "\tvaultToken: {}",
+ "\tauthMethod: {}"
+ ),
+ vaultAddress,
+ vaultToken,
+ authMethod,
+ e
+ );
+ throw new DrillbitStartupException(
+ "Error initialising the Vault client library: " + e.getMessage(),
+ e
+ );
+ }
+ }
+
+ /**
+ * Attempts to authenticate a Drill user with the provided password and the
+ * configured Vault auth method. Only auth methods that have a natural
+ * mapping to PLAIN authentication's user and password strings are supported.
+ * @param user username
+ * @param password password
+ * @throws UserAuthenticationException if the authentication attempt fails
+ */
+ @Override
+ public void authenticate(
+ String user,
+ String password
+ ) throws UserAuthenticationException {
+
+ try {
+ logger.debug("Tries to authenticate user {} using {}", user, authMethod);
+
+ switch (authMethod) {
+ case APP_ROLE:
+ // user = role id, password = secret id
+ vault.auth().loginByAppRole(user, password);
+ break;
+ case LDAP:
+ // user = username, password = password
+ vault.auth().loginByLDAP(user, password);
+ break;
+ case USER_PASS:
+ // user = username, password = password
+ vault.auth().loginByUserPass(user, password);
+ break;
+ case VAULT_TOKEN:
+ // user = username, password = vault token
+
+ // Create a throwaway Vault client using the provided token and send
a
+ // token lookup request to Vault which will fail if the token is
+ // invalid.
+ VaultConfig lookupConfig = new VaultConfig()
+ .address(this.vaultConfig.getAddress())
+ .token(password)
+ .build();
+
+ LookupResponse lookupResp = new
Vault(lookupConfig).auth().lookupSelf();
+ // Check token owner using getPath() because getUsername() sometimes
+ // returns null.
+ if (!lookupResp.getPath().endsWith("/" + user)) {
+ throw new UserAuthenticationException(String.format(
+ "Attempted to authenticate user %s with a Vault token that is " +
+ " valid but has path %s!",
+ user,
+ lookupResp.getPath()
+ ));
+ }
+ break;
+
+ default:
+ throw new UserAuthenticationException(String.format(
+ "The Vault authentication method '%s' is not supported",
+ authMethod
+ ));
+ }
+ } catch (VaultException e) {
+ logger.warn("Failed to authenticate user {} using {}: {}.", user,
authMethod, e);
+ throw new UserAuthenticationException(
+ String.format(
+ "Failed to authenticate user %s using %s: %s",
+ user,
+ authMethod,
+ e.getMessage()
+ )
+ );
+ }
+
+ logger.info(
+ "User {} authenticated against Vault successfully.",
+ user
+ );
+ }
+
+ @Override
+ public void close() throws IOException {
+ this.vault = null;
+ logger.debug("Has been closed.");
+ }
+}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/auth/DrillRestLoginService.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/auth/DrillRestLoginService.java
index a21a0f1..6f3b969 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/auth/DrillRestLoginService.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/auth/DrillRestLoginService.java
@@ -104,7 +104,7 @@ public class DrillRestLoginService implements LoginService {
if (e instanceof UserAuthenticationException) {
logger.debug("Authentication failed for WebUser '{}'", username, e);
} else {
- logger.error("UnExpected failure occurred for WebUser {} during
login.", username, e);
+ logger.error("Unexpected failure occurred for WebUser {} during
login.", username, e);
}
return null;
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/security/vault/VaultCredentialsProvider.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/security/vault/VaultCredentialsProvider.java
index aff2acc..8de6282 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/security/vault/VaultCredentialsProvider.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/security/vault/VaultCredentialsProvider.java
@@ -37,6 +37,7 @@ import java.util.Objects;
*/
public class VaultCredentialsProvider implements CredentialsProvider {
+ // Drill boot options used to configure a Vault credentials provider
public static final String VAULT_ADDRESS =
"drill.exec.storage.vault.address";
public static final String VAULT_TOKEN = "drill.exec.storage.vault.token";
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestHtpasswdFileUserAuthenticator.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestHtpasswdFileUserAuthenticator.java
index 9c27408..6b6b3ca 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestHtpasswdFileUserAuthenticator.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestHtpasswdFileUserAuthenticator.java
@@ -17,12 +17,14 @@
*/
package org.apache.drill.exec.rpc.user.security;
+import org.apache.drill.categories.SecurityTest;
import org.apache.drill.common.config.DrillProperties;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.test.ClientFixture;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterTest;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
import java.io.File;
import java.io.IOException;
@@ -33,6 +35,7 @@ import java.util.List;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+@Category(SecurityTest.class)
public class TestHtpasswdFileUserAuthenticator extends ClusterTest {
private File tempPasswdFile;
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestVaultUserAuthenticator.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestVaultUserAuthenticator.java
new file mode 100644
index 0000000..d5e33c2
--- /dev/null
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestVaultUserAuthenticator.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc.user.security;
+
+import com.bettercloud.vault.Vault;
+import com.bettercloud.vault.VaultConfig;
+import com.bettercloud.vault.response.AuthResponse;
+import org.apache.drill.categories.SecurityTest;
+import org.apache.drill.common.config.DrillProperties;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.test.ClientFixture;
+import org.apache.drill.test.ClusterFixture;
+import org.apache.drill.test.ClusterTest;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.testcontainers.utility.DockerImageName;
+import org.testcontainers.vault.VaultContainer;
+import org.testcontainers.vault.VaultLogLevel;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.fail;
+
+@Category(SecurityTest.class)
+public class TestVaultUserAuthenticator extends ClusterTest {
+
+ private static final String ROOT_TOKEN_VALUE = "vault-token";
+
+ private static String vaultAddr;
+
+ @ClassRule
+ public static final VaultContainer<?> vaultContainer =
+ new VaultContainer<>(DockerImageName.parse("vault").withTag("1.1.3"))
+ .withLogLevel(VaultLogLevel.Debug)
+ .withVaultToken(ROOT_TOKEN_VALUE)
+ .withInitCommand(
+ "auth enable userpass",
+ "write auth/userpass/users/alice password=pass1 policies=admins",
+ "write auth/userpass/users/bob password=buzzkill policies=admins"
+ );
+
+ @BeforeClass
+ public static void init() throws Exception {
+ vaultAddr = String.format(
+ "http://%s:%d";,
+ vaultContainer.getHost(),
+ vaultContainer.getMappedPort(8200)
+ );
+ }
+
+ @Test
+ public void testUserPassAuth() throws Exception {
+ cluster = ClusterFixture.bareBuilder(dirTestWatcher)
+ .clusterSize(3)
+ .configProperty(ExecConstants.ALLOW_LOOPBACK_ADDRESS_BINDING, true)
+ .configProperty(ExecConstants.USER_AUTHENTICATION_ENABLED, true)
+ .configProperty(ExecConstants.USER_AUTHENTICATOR_IMPL, "vault")
+ .configProperty(VaultUserAuthenticator.VAULT_ADDRESS, vaultAddr)
+ .configProperty(VaultUserAuthenticator.VAULT_TOKEN, ROOT_TOKEN_VALUE)
+ .configProperty(
+ VaultUserAuthenticator.VAULT_AUTH_METHOD,
+ VaultUserAuthenticator.VaultAuthMethod.USER_PASS
+ )
+ .build();
+
+ tryCredentials("notalice", "pass1", cluster, false);
+ tryCredentials("notbob", "buzzkill", cluster, false);
+ tryCredentials("alice", "wrong", cluster, false);
+ tryCredentials("bob", "incorrect", cluster, false);
+ tryCredentials("alice", "pass1", cluster, true);
+ tryCredentials("bob", "buzzkill", cluster, true);
+ }
+
+ @Test
+ public void testVaultTokenAuth() throws Exception {
+ // Use the Vault client lib to obtain Vault tokens for our test users.
+ VaultConfig vaultConfig = new VaultConfig()
+ .address(vaultAddr)
+ .token(ROOT_TOKEN_VALUE)
+ .build();
+
+ Vault vault = new Vault(vaultConfig);
+ AuthResponse aliceResp = vault.auth().loginByUserPass("alice", "pass1");
+ String aliceAuthToken = aliceResp.getAuthClientToken();
+ AuthResponse bobResp = vault.auth().loginByUserPass("bob", "buzzkill");
+ String bobAuthToken = bobResp.getAuthClientToken();
+
+ // set up a new cluster with a config option selecting Vault token auth
+ cluster = ClusterFixture.bareBuilder(dirTestWatcher)
+ .clusterSize(3)
+ .configProperty(ExecConstants.ALLOW_LOOPBACK_ADDRESS_BINDING, true)
+ .configProperty(ExecConstants.USER_AUTHENTICATION_ENABLED, true)
+ .configProperty(ExecConstants.USER_AUTHENTICATOR_IMPL, "vault")
+ .configProperty(VaultUserAuthenticator.VAULT_ADDRESS, vaultAddr)
+ // test without any VAULT_TOKEN boot option for token auth method
+ .configProperty(
+ VaultUserAuthenticator.VAULT_AUTH_METHOD,
+ VaultUserAuthenticator.VaultAuthMethod.VAULT_TOKEN
+ )
+ .build();
+
+ tryCredentials("notalice", aliceAuthToken, cluster, false);
+ tryCredentials("notbob", bobAuthToken, cluster, false);
+ tryCredentials("alice", "wrong", cluster, false);
+ tryCredentials("bob", "incorrect", cluster, false);
+ tryCredentials("alice", aliceAuthToken, cluster, true);
+ tryCredentials("bob", bobAuthToken, cluster, true);
+ }
+
+ private static void tryCredentials(String user, String password,
ClusterFixture cluster, boolean shouldSucceed) throws Exception {
+ try {
+ ClientFixture client = cluster.clientBuilder()
+ .property(DrillProperties.USER, user)
+ .property(DrillProperties.PASSWORD, password)
+ .build();
+
+ // Run few queries using the new client
+ List<String> queries = Arrays.asList(
+ "SHOW SCHEMAS",
+ "USE INFORMATION_SCHEMA",
+ "SHOW TABLES",
+ "SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME LIKE
'COLUMNS'",
+ "SELECT * FROM cp.`region.json` LIMIT 5");
+
+ for (String query : queries) {
+ client.queryBuilder().sql(query).run();
+ }
+
+ if (!shouldSucceed) {
+ fail("Expected connect to fail because of incorrect username /
password combination, but it succeeded");
+ }
+ } catch (IllegalStateException e) {
+ if (shouldSucceed) {
+ throw e;
+ }
+ }
+ }
+
+}
