Github user adrian-wang commented on a diff in the pull request:
https://github.com/apache/spark/pull/7643#discussion_r36051537
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeFunctions.scala
---
@@ -258,3 +258,89 @@ case class DateFormatClass(left: Expression, right:
Expression) extends BinaryEx
})
}
}
+
+/**
+ * Assumes given timestamp is UTC and converts to given timezone.
+ */
+case class FromUTCTimestamp(left: Expression, right: Expression)
+ extends BinaryExpression with ImplicitCastInputTypes {
+
+ override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType,
StringType)
+
+ override def dataType: DataType = TimestampType
+
+ override def nullSafeEval(time: Any, timezone: Any): Any = {
+ DateTimeUtils.fromUTCTime(time.asInstanceOf[Long],
timezone.asInstanceOf[UTF8String])
+ }
+
+ override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
+ val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
+ defineCodeGen(ctx, ev, (timestamp, format) => {
+ s"""$dtu.fromUTCTime($timestamp, $format)"""
+ })
+ }
+}
+
+/**
+ * Assumes given timestamp is in given timezone and converts to UTC.
+ */
+case class ToUTCTimestamp(left: Expression, right: Expression)
+ extends BinaryExpression with ImplicitCastInputTypes {
+
+ override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType,
StringType)
+
+ override def dataType: DataType = TimestampType
+
+ override def nullSafeEval(time: Any, timezone: Any): Any = {
+ DateTimeUtils.toUTCTime(time.asInstanceOf[Long],
timezone.asInstanceOf[UTF8String])
+ }
+
+ override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode):
String = {
+ val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
+ defineCodeGen(ctx, ev, (timestamp, format) => {
+ s"""$dtu.toUTCTime($timestamp, $format)"""
+ })
+ }
+}
+
+/**
+ * Returns the number of days from startdate to enddate. If input type is
String, will be
+ * considered as UTC.
+ */
+case class DateDiff(left: Expression, right: Expression)
+ extends BinaryExpression with ExpectsInputTypes {
--- End diff --
This is because hive would pass the string into timestamp/date type by
supposing the string is in UTC timestamp. but our implicit casting would use
local time zone to cast. This is where this 2 implementations differs, and I'm
not quite sure whether we should follow this in order to be compatible with
hive, or just do what we think is correct.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
