This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 8dea9b8 [SPARK-34811][CORE] Redact fs.s3a.access.key like secret and
token
8dea9b8 is described below
commit 8dea9b8071609c6690bd8f3e9d5621d7bed4786a
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sun Mar 21 14:08:34 2021 -0700
[SPARK-34811][CORE] Redact fs.s3a.access.key like secret and token
### What changes were proposed in this pull request?
Like we redact secrets and tokens, this PR aims to redact access key.
### Why are the changes needed?
Access key is also worth to hide.
### Does this PR introduce _any_ user-facing change?
This will hide this information from SparkUI (`Spark Properties` and
`Hadoop Properties` and logs).
### How was this patch tested?
Pass the newly updated UT.
Closes #31912 from dongjoon-hyun/SPARK-34811.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 3c32b54a0fbdc55c503bc72a3d39d58bf99e3bfa)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
core/src/main/scala/org/apache/spark/internal/config/package.scala | 2 +-
core/src/test/scala/org/apache/spark/util/UtilsSuite.scala | 5 ++++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala
b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index f6de5e4..3daa9f5 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -1015,7 +1015,7 @@ package object config {
"like YARN and event logs.")
.version("2.1.2")
.regexConf
- .createWithDefault("(?i)secret|password|token".r)
+ .createWithDefault("(?i)secret|password|token|access[.]key".r)
private[spark] val STRING_REDACTION_PATTERN =
ConfigBuilder("spark.redaction.string.regex")
diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
index 18ff960..208e729 100644
--- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
@@ -1024,11 +1024,13 @@ class UtilsSuite extends SparkFunSuite with
ResetSystemProperties with Logging {
// Set some secret keys
val secretKeys = Seq(
"spark.executorEnv.HADOOP_CREDSTORE_PASSWORD",
+ "spark.hadoop.fs.s3a.access.key",
"spark.my.password",
"spark.my.sECreT")
secretKeys.foreach { key => sparkConf.set(key, "sensitive_value") }
// Set a non-secret key
sparkConf.set("spark.regular.property", "regular_value")
+ sparkConf.set("spark.hadoop.fs.s3a.access_key", "regular_value")
// Set a property with a regular key but secret in the value
sparkConf.set("spark.sensitive.property", "has_secret_in_value")
@@ -1039,7 +1041,8 @@ class UtilsSuite extends SparkFunSuite with
ResetSystemProperties with Logging {
secretKeys.foreach { key => assert(redactedConf(key) ===
Utils.REDACTION_REPLACEMENT_TEXT) }
assert(redactedConf("spark.regular.property") === "regular_value")
assert(redactedConf("spark.sensitive.property") ===
Utils.REDACTION_REPLACEMENT_TEXT)
-
+ assert(redactedConf("spark.hadoop.fs.s3a.access.key") ===
Utils.REDACTION_REPLACEMENT_TEXT)
+ assert(redactedConf("spark.hadoop.fs.s3a.access_key") === "regular_value")
}
test("redact sensitive information in command line args") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
