This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 29b981b [SPARK-34811][CORE] Redact fs.s3a.access.key like secret and
token
29b981b is described below
commit 29b981b3feacadabf5cef4793506c20897ccd535
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sun Mar 21 14:08:34 2021 -0700
[SPARK-34811][CORE] Redact fs.s3a.access.key like secret and token
### What changes were proposed in this pull request?
Like we redact secrets and tokens, this PR aims to redact access key.
### Why are the changes needed?
Access key is also worth to hide.
### Does this PR introduce _any_ user-facing change?
This will hide this information from SparkUI (`Spark Properties` and
`Hadoop Properties` and logs).
### How was this patch tested?
Pass the newly updated UT.
Closes #31912 from dongjoon-hyun/SPARK-34811.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 3c32b54a0fbdc55c503bc72a3d39d58bf99e3bfa)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
core/src/main/scala/org/apache/spark/internal/config/package.scala | 2 +-
core/src/test/scala/org/apache/spark/util/UtilsSuite.scala | 5 ++++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala
b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 91689df..4e5b8f0 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -358,7 +358,7 @@ package object config {
"a property key or value, the value is redacted from the environment
UI and various logs " +
"like YARN and event logs.")
.regexConf
- .createWithDefault("(?i)secret|password|token".r)
+ .createWithDefault("(?i)secret|password|token|access[.]key".r)
private[spark] val STRING_REDACTION_PATTERN =
ConfigBuilder("spark.redaction.string.regex")
diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
index 00f96b9..a1a6721 100644
--- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
@@ -1013,11 +1013,13 @@ class UtilsSuite extends SparkFunSuite with
ResetSystemProperties with Logging {
// Set some secret keys
val secretKeys = Seq(
"spark.executorEnv.HADOOP_CREDSTORE_PASSWORD",
+ "spark.hadoop.fs.s3a.access.key",
"spark.my.password",
"spark.my.sECreT")
secretKeys.foreach { key => sparkConf.set(key, "sensitive_value") }
// Set a non-secret key
sparkConf.set("spark.regular.property", "regular_value")
+ sparkConf.set("spark.hadoop.fs.s3a.access_key", "regular_value")
// Set a property with a regular key but secret in the value
sparkConf.set("spark.sensitive.property", "has_secret_in_value")
@@ -1028,7 +1030,8 @@ class UtilsSuite extends SparkFunSuite with
ResetSystemProperties with Logging {
secretKeys.foreach { key => assert(redactedConf(key) ===
Utils.REDACTION_REPLACEMENT_TEXT) }
assert(redactedConf("spark.regular.property") === "regular_value")
assert(redactedConf("spark.sensitive.property") ===
Utils.REDACTION_REPLACEMENT_TEXT)
-
+ assert(redactedConf("spark.hadoop.fs.s3a.access.key") ===
Utils.REDACTION_REPLACEMENT_TEXT)
+ assert(redactedConf("spark.hadoop.fs.s3a.access_key") === "regular_value")
}
test("tryWithSafeFinally") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
