yaooqinn commented on a change in pull request #28650:
URL: https://github.com/apache/spark/pull/28650#discussion_r431693463
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
##########
@@ -859,127 +865,80 @@ case class UnixTimestamp(timeExp: Expression, format:
Expression, timeZoneId: Op
}
abstract class ToTimestamp
- extends BinaryExpression with TimeZoneAwareExpression with ExpectsInputTypes
{
+ extends BinaryExpression with TimestampFormatterHelper with
ExpectsInputTypes {
// The result of the conversion to timestamp is microseconds divided by this
factor.
// For example if the factor is 1000000, the result of the expression is in
seconds.
protected def downScaleFactor: Long
+ override protected def formatString: Expression = right
+ override protected def isParsing = true
+
override def inputTypes: Seq[AbstractDataType] =
Seq(TypeCollection(StringType, DateType, TimestampType), StringType)
override def dataType: DataType = LongType
override def nullable: Boolean = true
- private lazy val constFormat: UTF8String =
right.eval().asInstanceOf[UTF8String]
- private lazy val formatter: TimestampFormatter =
- try {
- TimestampFormatter(
- constFormat.toString,
- zoneId,
- legacyFormat = SIMPLE_DATE_FORMAT,
- needVarLengthSecondFraction = true)
- } catch {
- case e: SparkUpgradeException => throw e
- case NonFatal(_) => null
- }
-
override def eval(input: InternalRow): Any = {
- val t = left.eval(input)
- if (t == null) {
- null
- } else {
+ Option(left.eval(input)).map { t =>
left.dataType match {
- case DateType =>
- epochDaysToMicros(t.asInstanceOf[Int], zoneId) / downScaleFactor
- case TimestampType =>
- t.asInstanceOf[Long] / downScaleFactor
- case StringType if right.foldable =>
- if (constFormat == null || formatter == null) {
- null
- } else {
- try {
- formatter.parse(
- t.asInstanceOf[UTF8String].toString) / downScaleFactor
- } catch {
- case e: SparkUpgradeException => throw e
- case NonFatal(_) => null
- }
- }
+ case DateType => epochDaysToMicros(t.asInstanceOf[Int], zoneId) /
downScaleFactor
+ case TimestampType => t.asInstanceOf[Long] / downScaleFactor
case StringType =>
- val f = right.eval(input)
- if (f == null) {
- null
- } else {
- val formatString = f.asInstanceOf[UTF8String].toString
+ Option(right.eval(input)).map { fmt =>
+ val formatter =
formatterOption.getOrElse(getFormatter(fmt.toString))
try {
- TimestampFormatter(
- formatString,
- zoneId,
- legacyFormat = SIMPLE_DATE_FORMAT,
- needVarLengthSecondFraction = true)
- .parse(t.asInstanceOf[UTF8String].toString) / downScaleFactor
+ formatter.parse(t.asInstanceOf[UTF8String].toString) /
downScaleFactor
} catch {
- case e: SparkUpgradeException => throw e
- case NonFatal(_) => null
+ case _: DateTimeParseException |
+ _: DateTimeException |
+ _: ParseException => null
}
- }
+ }.orNull
}
- }
+ }.orNull
}
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val javaType = CodeGenerator.javaType(dataType)
left.dataType match {
- case StringType if right.foldable =>
+ case StringType => formatterOption.map { fmt =>
val df = classOf[TimestampFormatter].getName
- if (formatter == null) {
- ExprCode.forNullValue(dataType)
- } else {
- val formatterName = ctx.addReferenceObj("formatter", formatter, df)
- val eval1 = left.genCode(ctx)
- ev.copy(code = code"""
- ${eval1.code}
- boolean ${ev.isNull} = ${eval1.isNull};
- $javaType ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
- if (!${ev.isNull}) {
- try {
- ${ev.value} = $formatterName.parse(${eval1.value}.toString())
/ $downScaleFactor;
- } catch (java.lang.IllegalArgumentException e) {
- ${ev.isNull} = true;
- } catch (java.text.ParseException e) {
- ${ev.isNull} = true;
- } catch (java.time.format.DateTimeParseException e) {
- ${ev.isNull} = true;
- } catch (java.time.DateTimeException e) {
- ${ev.isNull} = true;
- }
- }""")
- }
- case StringType =>
+ val formatterName = ctx.addReferenceObj("formatter", fmt, df)
+ nullSafeCodeGen(ctx, ev, (datetimeStr, _) =>
+ s"""
+ |try {
+ | ${ev.value} = $formatterName.parse($datetimeStr.toString()) /
$downScaleFactor;
+ |} catch (java.time.DateTimeException e) {
+ | ${ev.isNull} = true;
+ |} catch (java.time.format.DateTimeParseException e) {
+ | ${ev.isNull} = true;
+ |} catch (java.text.ParseException e) {
+ | ${ev.isNull} = true;
+ |}
+ |""".stripMargin)
+ }.getOrElse {
val zid = ctx.addReferenceObj("zoneId", zoneId,
classOf[ZoneId].getName)
val tf = TimestampFormatter.getClass.getName.stripSuffix("$")
val ldf = LegacyDateFormats.getClass.getName.stripSuffix("$")
- nullSafeCodeGen(ctx, ev, (string, format) => {
+ nullSafeCodeGen(ctx, ev, (string, format) =>
s"""
- try {
- ${ev.value} = $tf$$.MODULE$$.apply(
- $format.toString(),
- $zid,
- $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT(),
- true)
- .parse($string.toString()) / $downScaleFactor;
- } catch (java.lang.IllegalArgumentException e) {
- ${ev.isNull} = true;
- } catch (java.text.ParseException e) {
- ${ev.isNull} = true;
- } catch (java.time.format.DateTimeParseException e) {
- ${ev.isNull} = true;
- } catch (java.time.DateTimeException e) {
- ${ev.isNull} = true;
- }
- """
- })
+ |try {
+ | ${ev.value} = $tf$$.MODULE$$.apply(
Review comment:
SGTM
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
