Github user zsxwing commented on a diff in the pull request:
https://github.com/apache/spark/pull/19985#discussion_r157272539
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala ---
@@ -418,6 +418,35 @@ class StreamSuite extends StreamTest {
assert(OutputMode.Update === InternalOutputModes.Update)
}
+ override protected def sparkConf: SparkConf = super.sparkConf
+ .set("spark.redaction.string.regex", "file:/[\\w_]+")
+
+ test("explain - redaction") {
+ val replacement = "*********"
+
+ val inputData = MemoryStream[String]
+ val df = inputData.toDS().map(_ +
"foo").groupBy("value").agg(count("*"))
+ // Test StreamingQuery.display
+ val q =
df.writeStream.queryName("memory_explain").outputMode("complete").format("memory")
+ .start()
+ .asInstanceOf[StreamingQueryWrapper]
+ .streamingQuery
+ try {
+ inputData.addData("abc")
+ q.processAllAvailable()
+
+ val explainWithoutExtended = q.explainInternal(false)
+ assert(explainWithoutExtended.contains(replacement))
+ assert(explainWithoutExtended.contains("StateStoreRestore"))
--- End diff --
nit: add `assert(!explainWithoutExtended.contains("file:/"))` to verify it
does replace the correct content.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
