Repository: spark Updated Branches: refs/heads/branch-1.0 7e3e9afdb -> 8285993a9
[Spark-2137][SQL] Timestamp UDFs broken https://issues.apache.org/jira/browse/SPARK-2137 Author: Yin Huai <[email protected]> Closes #1081 from yhuai/SPARK-2137 and squashes the following commits: c04f910 [Yin Huai] Merge remote-tracking branch 'upstream/master' into SPARK-2137 205f17b [Yin Huai] Make Hive UDF wrapper support Timestamp. (cherry picked from commit 891968509105d8d8cf5a608ad9473aeeed747089) Signed-off-by: Reynold Xin <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8285993a Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8285993a Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8285993a Branch: refs/heads/branch-1.0 Commit: 8285993a981f8a9e2e123834c4a869accb9f1a04 Parents: 7e3e9af Author: Yin Huai <[email protected]> Authored: Fri Jun 13 23:28:57 2014 -0700 Committer: Reynold Xin <[email protected]> Committed: Fri Jun 13 23:29:07 2014 -0700 ---------------------------------------------------------------------- .../src/main/scala/org/apache/spark/sql/hive/HiveContext.scala | 2 +- sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala | 3 ++- .../golden/timestamp_udf-10-dbc23736a61d9482d13cacada02a7a09 | 1 + .../golden/timestamp_udf-11-442cf850a0cc1f1dcfdeaeffbffb2c35 | 1 + .../golden/timestamp_udf-12-51959036fd4ac4f1e24f4e06eb9b0b6 | 1 + .../golden/timestamp_udf-13-6ab3f356deaf807e8accc37e1f4849a | 1 + .../golden/timestamp_udf-14-c745a1016461403526d44928a269c1de | 1 + .../golden/timestamp_udf-15-7ab76c4458c7f78038c8b1df0fdeafbe | 1 + .../golden/timestamp_udf-16-b36e87e17ca24d82072220bff559c718 | 1 + .../golden/timestamp_udf-17-dad44d2d4a421286e9da080271bd2639 | 1 + .../golden/timestamp_udf-18-cb033ecad964a2623bc633ac1d3f752a | 1 + .../golden/timestamp_udf-19-79914c5347620c6e62a8e0b9a95984af | 0 .../golden/timestamp_udf-20-59fc1842a23369235d42ed040d45fb3d | 0 .../golden/timestamp_udf-4-80ce02ec84ee8abcb046367ca37279cc | 0 .../golden/timestamp_udf-5-1124399033bcadf3874fb48f593392d | 1 + .../golden/timestamp_udf-6-5810193ce35d38c23f4fc4b4979d60a4 | 1 + .../golden/timestamp_udf-7-250e640a6a818f989f3f3280b00f64f9 | 1 + .../golden/timestamp_udf-8-975df43df015d86422965af456f87a94 | 1 + .../golden/timestamp_udf-9-287614364eaa3fb82aad08c6b62cc938 | 1 + 19 files changed, 17 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala index 96e0ec5..cc95b7a 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala @@ -250,7 +250,7 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) { protected val primitiveTypes = Seq(StringType, IntegerType, LongType, DoubleType, FloatType, BooleanType, ByteType, - ShortType, DecimalType) + ShortType, DecimalType, TimestampType) protected def toHiveString(a: (Any, DataType)): String = a match { case (struct: Row, StructType(fields)) => http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala index 5729020..771d2bc 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala @@ -187,7 +187,8 @@ private[hive] case class HiveSimpleUdf(name: String, children: Seq[Expression]) val primitiveClasses = Seq( Integer.TYPE, classOf[java.lang.Integer], classOf[java.lang.String], java.lang.Double.TYPE, classOf[java.lang.Double], java.lang.Long.TYPE, classOf[java.lang.Long], - classOf[HiveDecimal], java.lang.Byte.TYPE, classOf[java.lang.Byte] + classOf[HiveDecimal], java.lang.Byte.TYPE, classOf[java.lang.Byte], + classOf[java.sql.Timestamp] ) val matchingConstructor = argClass.getConstructors.find { c => c.getParameterTypes.size == 1 && primitiveClasses.contains(c.getParameterTypes.head) http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/test/resources/golden/timestamp_udf-10-dbc23736a61d9482d13cacada02a7a09 ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-10-dbc23736a61d9482d13cacada02a7a09 b/sql/hive/src/test/resources/golden/timestamp_udf-10-dbc23736a61d9482d13cacada02a7a09 new file mode 100644 index 0000000..b3c4eec --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-10-dbc23736a61d9482d13cacada02a7a09 @@ -0,0 +1 @@ +2011-05-06 07:08:09.1234567 2011-05-06 02:08:09.1234567 http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/test/resources/golden/timestamp_udf-11-442cf850a0cc1f1dcfdeaeffbffb2c35 ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-11-442cf850a0cc1f1dcfdeaeffbffb2c35 b/sql/hive/src/test/resources/golden/timestamp_udf-11-442cf850a0cc1f1dcfdeaeffbffb2c35 new file mode 100644 index 0000000..f69f13e --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-11-442cf850a0cc1f1dcfdeaeffbffb2c35 @@ -0,0 +1 @@ +2011-05-06 07:08:09.1234567 2011-05-06 02:08:09.1234567 2011-05-06 07:08:09.1234567 2011-05-06 02:08:09.1234567 http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/test/resources/golden/timestamp_udf-12-51959036fd4ac4f1e24f4e06eb9b0b6 ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-12-51959036fd4ac4f1e24f4e06eb9b0b6 b/sql/hive/src/test/resources/golden/timestamp_udf-12-51959036fd4ac4f1e24f4e06eb9b0b6 new file mode 100644 index 0000000..f14f17e --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-12-51959036fd4ac4f1e24f4e06eb9b0b6 @@ -0,0 +1 @@ +2011-05-06 07:08:09.1234567 2011-05-06 12:08:09.1234567 http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/test/resources/golden/timestamp_udf-13-6ab3f356deaf807e8accc37e1f4849a ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-13-6ab3f356deaf807e8accc37e1f4849a b/sql/hive/src/test/resources/golden/timestamp_udf-13-6ab3f356deaf807e8accc37e1f4849a new file mode 100644 index 0000000..7881bff --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-13-6ab3f356deaf807e8accc37e1f4849a @@ -0,0 +1 @@ +2011-05-06 07:08:09.1234567 2011-05-06 12:08:09.1234567 2011-05-06 07:08:09.1234567 2011-05-06 12:08:09.1234567 http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/test/resources/golden/timestamp_udf-14-c745a1016461403526d44928a269c1de ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-14-c745a1016461403526d44928a269c1de b/sql/hive/src/test/resources/golden/timestamp_udf-14-c745a1016461403526d44928a269c1de new file mode 100644 index 0000000..2c5e9e9 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-14-c745a1016461403526d44928a269c1de @@ -0,0 +1 @@ +1304690889 2011 5 6 6 18 7 8 9 2011-05-06 http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/test/resources/golden/timestamp_udf-15-7ab76c4458c7f78038c8b1df0fdeafbe ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-15-7ab76c4458c7f78038c8b1df0fdeafbe b/sql/hive/src/test/resources/golden/timestamp_udf-15-7ab76c4458c7f78038c8b1df0fdeafbe new file mode 100644 index 0000000..1949725 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-15-7ab76c4458c7f78038c8b1df0fdeafbe @@ -0,0 +1 @@ +2011-05-11 2011-04-26 http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/test/resources/golden/timestamp_udf-16-b36e87e17ca24d82072220bff559c718 ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-16-b36e87e17ca24d82072220bff559c718 b/sql/hive/src/test/resources/golden/timestamp_udf-16-b36e87e17ca24d82072220bff559c718 new file mode 100644 index 0000000..816f56e --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-16-b36e87e17ca24d82072220bff559c718 @@ -0,0 +1 @@ +0 3333 -3333 http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/test/resources/golden/timestamp_udf-17-dad44d2d4a421286e9da080271bd2639 ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-17-dad44d2d4a421286e9da080271bd2639 b/sql/hive/src/test/resources/golden/timestamp_udf-17-dad44d2d4a421286e9da080271bd2639 new file mode 100644 index 0000000..a4182d1 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-17-dad44d2d4a421286e9da080271bd2639 @@ -0,0 +1 @@ +2011-05-06 02:08:09.1234567 http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/test/resources/golden/timestamp_udf-18-cb033ecad964a2623bc633ac1d3f752a ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-18-cb033ecad964a2623bc633ac1d3f752a b/sql/hive/src/test/resources/golden/timestamp_udf-18-cb033ecad964a2623bc633ac1d3f752a new file mode 100644 index 0000000..02ccd3a --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-18-cb033ecad964a2623bc633ac1d3f752a @@ -0,0 +1 @@ +2011-05-06 12:08:09.1234567 http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/test/resources/golden/timestamp_udf-19-79914c5347620c6e62a8e0b9a95984af ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-19-79914c5347620c6e62a8e0b9a95984af b/sql/hive/src/test/resources/golden/timestamp_udf-19-79914c5347620c6e62a8e0b9a95984af new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/test/resources/golden/timestamp_udf-20-59fc1842a23369235d42ed040d45fb3d ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-20-59fc1842a23369235d42ed040d45fb3d b/sql/hive/src/test/resources/golden/timestamp_udf-20-59fc1842a23369235d42ed040d45fb3d new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/test/resources/golden/timestamp_udf-4-80ce02ec84ee8abcb046367ca37279cc ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-4-80ce02ec84ee8abcb046367ca37279cc b/sql/hive/src/test/resources/golden/timestamp_udf-4-80ce02ec84ee8abcb046367ca37279cc new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/test/resources/golden/timestamp_udf-5-1124399033bcadf3874fb48f593392d ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-5-1124399033bcadf3874fb48f593392d b/sql/hive/src/test/resources/golden/timestamp_udf-5-1124399033bcadf3874fb48f593392d new file mode 100644 index 0000000..2c5e9e9 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-5-1124399033bcadf3874fb48f593392d @@ -0,0 +1 @@ +1304690889 2011 5 6 6 18 7 8 9 2011-05-06 http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/test/resources/golden/timestamp_udf-6-5810193ce35d38c23f4fc4b4979d60a4 ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-6-5810193ce35d38c23f4fc4b4979d60a4 b/sql/hive/src/test/resources/golden/timestamp_udf-6-5810193ce35d38c23f4fc4b4979d60a4 new file mode 100644 index 0000000..1949725 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-6-5810193ce35d38c23f4fc4b4979d60a4 @@ -0,0 +1 @@ +2011-05-11 2011-04-26 http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/test/resources/golden/timestamp_udf-7-250e640a6a818f989f3f3280b00f64f9 ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-7-250e640a6a818f989f3f3280b00f64f9 b/sql/hive/src/test/resources/golden/timestamp_udf-7-250e640a6a818f989f3f3280b00f64f9 new file mode 100644 index 0000000..816f56e --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-7-250e640a6a818f989f3f3280b00f64f9 @@ -0,0 +1 @@ +0 3333 -3333 http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/test/resources/golden/timestamp_udf-8-975df43df015d86422965af456f87a94 ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-8-975df43df015d86422965af456f87a94 b/sql/hive/src/test/resources/golden/timestamp_udf-8-975df43df015d86422965af456f87a94 new file mode 100644 index 0000000..a4182d1 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-8-975df43df015d86422965af456f87a94 @@ -0,0 +1 @@ +2011-05-06 02:08:09.1234567 http://git-wip-us.apache.org/repos/asf/spark/blob/8285993a/sql/hive/src/test/resources/golden/timestamp_udf-9-287614364eaa3fb82aad08c6b62cc938 ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-9-287614364eaa3fb82aad08c6b62cc938 b/sql/hive/src/test/resources/golden/timestamp_udf-9-287614364eaa3fb82aad08c6b62cc938 new file mode 100644 index 0000000..02ccd3a --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-9-287614364eaa3fb82aad08c6b62cc938 @@ -0,0 +1 @@ +2011-05-06 12:08:09.1234567
