asfgit closed pull request #1419: DRILL-6662: Access AWS access key ID and
secret access key using Cred…
URL: https://github.com/apache/drill/pull/1419
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/distribution/src/resources/core-site-example.xml
b/distribution/src/resources/core-site-example.xml
index 854e54dbe67..c7225a14784 100644
--- a/distribution/src/resources/core-site-example.xml
+++ b/distribution/src/resources/core-site-example.xml
@@ -30,4 +30,18 @@
<value>ENTER_YOUR_SECRETKEY</value>
</property>
+ <!--Use this property to specify one or more credential provider URIs
instead of
+ configuring above credentials in plain text-->
+ <!--<property>
+ <name>hadoop.security.credential.provider.path</name>
+ <value>ENTER_YOUR_PROVIDER_PATH</value>
+ </property>-->
+
+ <!--Set this property to true to avoid caching of S3 file system
configuration properties,
+ so when you add/update a property (e.g. fs.s3a.secret.key) in S3
storage plugin its new value will be taken-->
+ <!--<property>
+ <name>fs.s3a.impl.disable.cache</name>
+ <value>true</value>
+ </property>-->
+
</configuration>
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
index b1f41a41451..cb66913ad13 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
@@ -20,6 +20,7 @@
import static
org.apache.drill.exec.store.dfs.FileSystemSchemaFactory.DEFAULT_WS_NAME;
import java.io.IOException;
+import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -77,6 +78,10 @@ public FileSystemPlugin(FileSystemConfig config,
DrillbitContext context, String
fsConf.set("fs.classpath.impl", ClassPathFileSystem.class.getName());
fsConf.set("fs.drill-local.impl",
LocalSyncableFileSystem.class.getName());
+ if (isS3Connection(fsConf)) {
+ handleS3Credentials(fsConf);
+ }
+
formatCreator = newFormatCreator(config, context, fsConf);
List<FormatMatcher> matchers = new ArrayList<>();
formatPluginsByConfig = new HashMap<>();
@@ -104,6 +109,33 @@ public FileSystemPlugin(FileSystemConfig config,
DrillbitContext context, String
}
}
+ private boolean isS3Connection(Configuration conf) {
+ URI uri = FileSystem.getDefaultUri(conf);
+ return uri.getScheme().equals("s3a");
+ }
+
+ /**
+ * Retrieve secret and access keys from configured (with
+ * {@link
org.apache.hadoop.security.alias.CredentialProviderFactory#CREDENTIAL_PROVIDER_PATH}
property)
+ * credential providers and set it into {@code conf}. If provider path is
not configured or credential
+ * is absent in providers, it will conditionally fallback to configuration
setting. The fallback will occur unless
+ * {@link
org.apache.hadoop.security.alias.CredentialProvider#CLEAR_TEXT_FALLBACK} is set
to {@code false}.
+ *
+ * @param conf {@code Configuration} which will be updated with credentials
from provider
+ * @throws IOException thrown if a credential cannot be retrieved from
provider
+ */
+ private void handleS3Credentials(Configuration conf) throws IOException {
+ String[] credentialKeys = {"fs.s3a.secret.key", "fs.s3a.access.key"};
+ for (String key : credentialKeys) {
+ char[] credentialChars = conf.getPassword(key);
+ if (credentialChars == null) {
+ logger.warn(String.format("Property '%s' is absent.", key));
+ } else {
+ conf.set(key, String.valueOf(credentialChars));
+ }
+ }
+ }
+
/**
* Creates a new FormatCreator instance.
*
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
