Repository: spark Updated Branches: refs/heads/master 2aa16d03d -> 46d30ac48
[SPARK-18717][SQL] Make code generation for Scala Map work with immutable.Map also ## What changes were proposed in this pull request? Fixes compile errors in generated code when user has case class with a `scala.collections.immutable.Map` instead of a `scala.collections.Map`. Since ArrayBasedMapData.toScalaMap returns the immutable version we can make it work with both. ## How was this patch tested? Additional unit tests. Author: Andrew Ray <[email protected]> Closes #16161 from aray/fix-map-codegen. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/46d30ac4 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/46d30ac4 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/46d30ac4 Branch: refs/heads/master Commit: 46d30ac4846b3ec94426cc482c42cff72ebd6d92 Parents: 2aa16d0 Author: Andrew Ray <[email protected]> Authored: Tue Dec 13 15:49:22 2016 +0800 Committer: Wenchen Fan <[email protected]> Committed: Tue Dec 13 15:49:22 2016 +0800 ---------------------------------------------------------------------- .../org/apache/spark/sql/catalyst/ScalaReflection.scala | 2 +- .../test/scala/org/apache/spark/sql/DatasetSuite.scala | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/46d30ac4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala index 6e20096..ad218cf 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala @@ -342,7 +342,7 @@ object ScalaReflection extends ScalaReflection { StaticInvoke( ArrayBasedMapData.getClass, - ObjectType(classOf[Map[_, _]]), + ObjectType(classOf[scala.collection.immutable.Map[_, _]]), "toScalaMap", keyData :: valueData :: Nil) http://git-wip-us.apache.org/repos/asf/spark/blob/46d30ac4/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala index 37d5667..3742115 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala @@ -1120,8 +1120,20 @@ class DatasetSuite extends QueryTest with SharedSQLContext { // sizeInBytes is 2404280404, before the fix, it overflows to a negative number assert(sizeInBytes > 0) } + + test("SPARK-18717: code generation works for both scala.collection.Map" + + " and scala.collection.imutable.Map") { + val ds = Seq(WithImmutableMap("hi", Map(42L -> "foo"))).toDS + checkDataset(ds.map(t => t), WithImmutableMap("hi", Map(42L -> "foo"))) + + val ds2 = Seq(WithMap("hi", Map(42L -> "foo"))).toDS + checkDataset(ds2.map(t => t), WithMap("hi", Map(42L -> "foo"))) + } } +case class WithImmutableMap(id: String, map_test: scala.collection.immutable.Map[Long, String]) +case class WithMap(id: String, map_test: scala.collection.Map[Long, String]) + case class Generic[T](id: T, value: Double) case class OtherTuple(_1: String, _2: Int) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
