dongjoon-hyun commented on code in PR #38084:
URL: https://github.com/apache/spark/pull/38084#discussion_r989711342
##########
core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala:
##########
@@ -438,26 +440,48 @@ private[spark] object SparkHadoopUtil extends Logging {
// the behavior of the old implementation of this code, for backwards
compatibility.
if (conf != null) {
// Explicitly check for S3 environment variables
- val keyId = System.getenv("AWS_ACCESS_KEY_ID")
- val accessKey = System.getenv("AWS_SECRET_ACCESS_KEY")
- if (keyId != null && accessKey != null) {
- hadoopConf.set("fs.s3.awsAccessKeyId", keyId)
- hadoopConf.set("fs.s3n.awsAccessKeyId", keyId)
- hadoopConf.set("fs.s3a.access.key", keyId)
- hadoopConf.set("fs.s3.awsSecretAccessKey", accessKey)
- hadoopConf.set("fs.s3n.awsSecretAccessKey", accessKey)
- hadoopConf.set("fs.s3a.secret.key", accessKey)
-
- val sessionToken = System.getenv("AWS_SESSION_TOKEN")
- if (sessionToken != null) {
- hadoopConf.set("fs.s3a.session.token", sessionToken)
- }
- }
+ val env: util.Map[String, String] = System.getenv
+ appendS3CredentialsFromEnvironment(hadoopConf, env)
appendHiveConfigs(hadoopConf)
appendSparkHadoopConfigs(conf, hadoopConf)
appendSparkHiveConfigs(conf, hadoopConf)
val bufferSize = conf.get(BUFFER_SIZE).toString
- hadoopConf.set("io.file.buffer.size", bufferSize)
+ hadoopConf.set("io.file.buffer.size", bufferSize, BUFFER_SIZE.key)
+ }
+ }
+
+ /**
+ * Append any AWS secrets from the environment variables
+ * if both `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` are set.
+ * If these two are set and `AWS_SESSION_TOKEN` is also set,
+ * then `fs.s3a.session.token`.
+ * The option is set with a source string which includes the hostname
+ * on which it was set. This can help debug propagation issues.
+ * @param hadoopConf configuration to patch
+ * @param env environment.
+ */
+ // Exposed for testing
+ private[deploy] def appendS3CredentialsFromEnvironment(
+ hadoopConf: Configuration,
+ env: util.Map[String, String]): Unit = {
Review Comment:
```scala
- hadoopConf: Configuration,
- env: util.Map[String, String]): Unit = {
+ hadoopConf: Configuration,
+ env: JMap[String, String]): Unit = {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
