Repository: spark Updated Branches: refs/heads/branch-2.2 50f86e1fe -> fb1b5f08a
[SPARK-21418][SQL] NoSuchElementException: None.get in DataSourceScanExec with sun.io.serialization.extendedDebugInfo=true ## What changes were proposed in this pull request? If no SparkConf is available to Utils.redact, simply don't redact. ## How was this patch tested? Existing tests Author: Sean Owen <[email protected]> Closes #19123 from srowen/SPARK-21418. (cherry picked from commit ca59445adb30ed796189532df2a2898ecd33db68) Signed-off-by: Herman van Hovell <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fb1b5f08 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fb1b5f08 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fb1b5f08 Branch: refs/heads/branch-2.2 Commit: fb1b5f08adaf4ec7c786b7a8b6283b62683f1324 Parents: 50f86e1 Author: Sean Owen <[email protected]> Authored: Mon Sep 4 23:02:59 2017 +0200 Committer: Herman van Hovell <[email protected]> Committed: Mon Sep 4 23:03:10 2017 +0200 ---------------------------------------------------------------------- core/src/main/scala/org/apache/spark/util/Utils.scala | 9 ++++++--- .../org/apache/spark/sql/execution/DataSourceScanExec.scala | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/fb1b5f08/core/src/main/scala/org/apache/spark/util/Utils.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 69c6c33..a5f44bd 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -2614,9 +2614,12 @@ private[spark] object Utils extends Logging { * Redact the sensitive information in the given string. */ def redact(conf: SparkConf, text: String): String = { - if (text == null || text.isEmpty || !conf.contains(STRING_REDACTION_PATTERN)) return text - val regex = conf.get(STRING_REDACTION_PATTERN).get - regex.replaceAllIn(text, REDACTION_REPLACEMENT_TEXT) + if (text == null || text.isEmpty || conf == null || !conf.contains(STRING_REDACTION_PATTERN)) { + text + } else { + val regex = conf.get(STRING_REDACTION_PATTERN).get + regex.replaceAllIn(text, REDACTION_REPLACEMENT_TEXT) + } } private def redact(redactionPattern: Regex, kvs: Seq[(String, String)]): Seq[(String, String)] = { http://git-wip-us.apache.org/repos/asf/spark/blob/fb1b5f08/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala index 74fc23a..6fb41b6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala @@ -67,7 +67,7 @@ trait DataSourceScanExec extends LeafExecNode with CodegenSupport { * Shorthand for calling redactString() without specifying redacting rules */ private def redact(text: String): String = { - Utils.redact(SparkSession.getActiveSession.get.sparkContext.conf, text) + Utils.redact(SparkSession.getActiveSession.map(_.sparkContext.conf).orNull, text) } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
