Github user gatorsmile commented on a diff in the pull request:
https://github.com/apache/spark/pull/19985#discussion_r157667492
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
---
@@ -52,4 +53,40 @@ class DataSourceScanExecRedactionSuite extends QueryTest
with SharedSQLContext {
assert(df.queryExecution.simpleString.contains(replacement))
}
}
+
+ private def isIncluded(queryExecution: QueryExecution, msg: String):
Boolean = {
+ queryExecution.toString.contains(msg) ||
+ queryExecution.simpleString.contains(msg) ||
+ queryExecution.stringWithStats.contains(msg)
+ }
+
+ test("explain is redacted using SQLConf") {
+ withTempDir { dir =>
+ val basePath = dir.getCanonicalPath
+ spark.range(0, 10).toDF("a").write.parquet(new Path(basePath,
"foo=1").toString)
+ val df = spark.read.parquet(basePath)
+ val replacement = "*********"
+
+ // Respect SparkConf and replace file:/
+ assert(isIncluded(df.queryExecution, replacement))
+
+ assert(isIncluded(df.queryExecution, "FileScan"))
+ assert(!isIncluded(df.queryExecution, "file:/"))
+
+ withSQLConf(SQLConf.SQL_STRING_REDACTION_PATTERN.key ->
"(?i)FileScan") {
+ // Respect SQLConf and replace FileScan
+ assert(isIncluded(df.queryExecution, replacement))
+
+ assert(!isIncluded(df.queryExecution, "FileScan"))
+ assert(isIncluded(df.queryExecution, "file:/"))
+ }
+
+ // Respect SparkConf and replace file:/
+ assert(isIncluded(df.queryExecution, replacement))
+
+ assert(isIncluded(df.queryExecution, "FileScan"))
+ assert(!isIncluded(df.queryExecution, "file:/"))
--- End diff --
Removed
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
