Repository: spark Updated Branches: refs/heads/branch-2.1 707630147 -> 42386e796
[SPARK-18260] Make from_json null safe ## What changes were proposed in this pull request? `from_json` is currently not safe against `null` rows. This PR adds a fix and a regression test for it. ## How was this patch tested? Regression test Author: Burak Yavuz <[email protected]> Closes #15771 from brkyvz/json_fix. (cherry picked from commit 6e2701815761d5870111cb56300e30d3059b39ed) Signed-off-by: Reynold Xin <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/42386e79 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/42386e79 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/42386e79 Branch: refs/heads/branch-2.1 Commit: 42386e796f6519d22092fba88a8c42cba6511d7c Parents: 7076301 Author: Burak Yavuz <[email protected]> Authored: Sat Nov 5 00:07:51 2016 -0700 Committer: Reynold Xin <[email protected]> Committed: Sat Nov 5 00:08:00 2016 -0700 ---------------------------------------------------------------------- .../spark/sql/catalyst/expressions/jsonExpressions.scala | 4 +++- .../sql/catalyst/expressions/JsonExpressionsSuite.scala | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/42386e79/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala index e034735..89fe7c4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala @@ -498,7 +498,9 @@ case class JsonToStruct(schema: StructType, options: Map[String, String], child: override def children: Seq[Expression] = child :: Nil override def eval(input: InternalRow): Any = { - try parser.parse(child.eval(input).toString).head catch { + val json = child.eval(input) + if (json == null) return null + try parser.parse(json.toString).head catch { case _: SparkSQLJsonProcessingException => null } } http://git-wip-us.apache.org/repos/asf/spark/blob/42386e79/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala index f9db649..3bfa0bf 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala @@ -344,6 +344,14 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { ) } + test("from_json null input column") { + val schema = StructType(StructField("a", IntegerType) :: Nil) + checkEvaluation( + JsonToStruct(schema, Map.empty, Literal(null)), + null + ) + } + test("to_json") { val schema = StructType(StructField("a", IntegerType) :: Nil) val struct = Literal.create(create_row(1), schema) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
