Repository: spark Updated Branches: refs/heads/branch-1.5 ea23e54ff -> 57596fb8c
[SPARK-9581][SQL] Add unit test for JSON UDT This brings #7416 up-to-date by drubbo. Author: Emiliano Leporati <[email protected]> Author: Reynold Xin <[email protected]> Closes #7917 from rxin/udt-json-test and squashes the following commits: 93e3954 [Reynold Xin] Fix test. 7035308 [Reynold Xin] Merge pull request #7416 from drubbo/master b5bcd94 [Emiliano Leporati] removed unneded case in MyDenseVector::equals 508a399 [Emiliano Leporati] Merge remote branch 'upstream/master' 7569e42 [Emiliano Leporati] using checkAnswer 62daccd [Emiliano Leporati] added coverage for UDTs in JSON RDDs (cherry picked from commit 1d1a76c8c56148beec70dd14e4fdeb86de65018a) Signed-off-by: Reynold Xin <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/57596fb8 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/57596fb8 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/57596fb8 Branch: refs/heads/branch-1.5 Commit: 57596fb8c13f2c9d7f326cad84744dad16e0fc95 Parents: ea23e54 Author: Emiliano Leporati <[email protected]> Authored: Wed Aug 5 00:42:08 2015 -0700 Committer: Reynold Xin <[email protected]> Committed: Wed Aug 5 00:42:16 2015 -0700 ---------------------------------------------------------------------- .../apache/spark/sql/json/JacksonParser.scala | 2 +- .../apache/spark/sql/UserDefinedTypeSuite.scala | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/57596fb8/sql/core/src/main/scala/org/apache/spark/sql/json/JacksonParser.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JacksonParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/json/JacksonParser.scala index f1a66c8..b8fd3b9 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/json/JacksonParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/json/JacksonParser.scala @@ -125,7 +125,7 @@ private[sql] object JacksonParser { convertMap(factory, parser, kt) case (_, udt: UserDefinedType[_]) => - udt.deserialize(convertField(factory, parser, udt.sqlType)) + convertField(factory, parser, udt.sqlType) } } http://git-wip-us.apache.org/repos/asf/spark/blob/57596fb8/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala index f299352..9181222 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala @@ -138,4 +138,24 @@ class UserDefinedTypeSuite extends QueryTest { val actual = openHashSetUDT.deserialize(openHashSetUDT.serialize(set)) assert(actual.iterator.toSet === set.iterator.toSet) } + + test("UDTs with JSON") { + val data = Seq( + "{\"id\":1,\"vec\":[1.1,2.2,3.3,4.4]}", + "{\"id\":2,\"vec\":[2.25,4.5,8.75]}" + ) + val schema = StructType(Seq( + StructField("id", IntegerType, false), + StructField("vec", new MyDenseVectorUDT, false) + )) + + val stringRDD = ctx.sparkContext.parallelize(data) + val jsonRDD = ctx.read.schema(schema).json(stringRDD) + checkAnswer( + jsonRDD, + Row(1, new MyDenseVector(Array(1.1, 2.2, 3.3, 4.4))) :: + Row(2, new MyDenseVector(Array(2.25, 4.5, 8.75))) :: + Nil + ) + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
