AngersZhuuuu commented on a change in pull request #31979:
URL: https://github.com/apache/spark/pull/31979#discussion_r602980428
##########
File path:
sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
##########
@@ -528,4 +530,65 @@ class HiveScriptTransformationSuite extends
BaseScriptTransformationSuite with T
checkAnswer(query2, identity, Row("\\N,\\N,\\N") :: Nil)
}
+
+ test("SPARK-34879: HiveInspector supports DayTimeIntervalType and
YearMonthIntervalType") {
+ assume(TestUtils.testCommandAvailable("/bin/bash"))
+ withTempView("v") {
+ val df = Seq(
+ (Duration.ofDays(1),
+ Duration.ofSeconds(100).plusNanos(123456),
+ Duration.ofSeconds(Long.MaxValue /
DateTimeConstants.MICROS_PER_SECOND),
+ Period.ofMonths(10)),
+ (Duration.ofDays(1),
+ Duration.ofSeconds(100).plusNanos(1123456789),
+ Duration.ofSeconds(Long.MaxValue /
DateTimeConstants.MICROS_PER_SECOND),
+ Period.ofMonths(10))
+ ).toDF("a", "b", "c", "d")
+ .select('a, 'b, 'c.cast(DayTimeIntervalType).as("c_1"), 'd)
+ df.createTempView("v")
+
+ // Hive serde supports DayTimeIntervalType/YearMonthIntervalType as
input and output data type
+ checkAnswer(
+ df,
+ (child: SparkPlan) => createScriptTransformationExec(
+ input = Seq(
+ df.col("a").expr,
+ df.col("b").expr,
+ df.col("c_1").expr,
+ df.col("d").expr),
+ script = "cat",
+ output = Seq(
+ AttributeReference("a", DayTimeIntervalType)(),
+ AttributeReference("b", DayTimeIntervalType)(),
+ AttributeReference("c_1", DayTimeIntervalType)(),
+ AttributeReference("d", YearMonthIntervalType)()),
+ child = child,
+ ioschema = hiveIOSchema),
+ df.select('a, 'b, 'c_1, 'd).collect())
+ }
+ }
+
+ test("SPARK-34879: HiveInceptor throw overflow when" +
+ " HiveIntervalDayTime overflow then DayTimeIntervalType") {
+ withTempView("v") {
+ val df = Seq(
+ ("579025220 15:30:06.000001000")
+ ).toDF("a")
+ df.createTempView("v")
+
+ val e = intercept[Exception] {
+ checkAnswer(
+ df,
+ (child: SparkPlan) => createScriptTransformationExec(
+ input = Seq(
+ df.col("a").expr),
Review comment:
> nit:
Done
##########
File path:
sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
##########
@@ -528,4 +530,65 @@ class HiveScriptTransformationSuite extends
BaseScriptTransformationSuite with T
checkAnswer(query2, identity, Row("\\N,\\N,\\N") :: Nil)
}
+
+ test("SPARK-34879: HiveInspector supports DayTimeIntervalType and
YearMonthIntervalType") {
+ assume(TestUtils.testCommandAvailable("/bin/bash"))
+ withTempView("v") {
+ val df = Seq(
+ (Duration.ofDays(1),
+ Duration.ofSeconds(100).plusNanos(123456),
+ Duration.ofSeconds(Long.MaxValue /
DateTimeConstants.MICROS_PER_SECOND),
+ Period.ofMonths(10)),
+ (Duration.ofDays(1),
+ Duration.ofSeconds(100).plusNanos(1123456789),
+ Duration.ofSeconds(Long.MaxValue /
DateTimeConstants.MICROS_PER_SECOND),
+ Period.ofMonths(10))
+ ).toDF("a", "b", "c", "d")
+ .select('a, 'b, 'c.cast(DayTimeIntervalType).as("c_1"), 'd)
+ df.createTempView("v")
+
+ // Hive serde supports DayTimeIntervalType/YearMonthIntervalType as
input and output data type
+ checkAnswer(
+ df,
+ (child: SparkPlan) => createScriptTransformationExec(
+ input = Seq(
+ df.col("a").expr,
+ df.col("b").expr,
+ df.col("c_1").expr,
+ df.col("d").expr),
+ script = "cat",
+ output = Seq(
+ AttributeReference("a", DayTimeIntervalType)(),
+ AttributeReference("b", DayTimeIntervalType)(),
+ AttributeReference("c_1", DayTimeIntervalType)(),
+ AttributeReference("d", YearMonthIntervalType)()),
+ child = child,
+ ioschema = hiveIOSchema),
+ df.select('a, 'b, 'c_1, 'd).collect())
+ }
+ }
+
+ test("SPARK-34879: HiveInceptor throw overflow when" +
+ " HiveIntervalDayTime overflow then DayTimeIntervalType") {
+ withTempView("v") {
+ val df = Seq(
+ ("579025220 15:30:06.000001000")
+ ).toDF("a")
Review comment:
> nit:
Done
##########
File path:
sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
##########
@@ -528,4 +530,65 @@ class HiveScriptTransformationSuite extends
BaseScriptTransformationSuite with T
checkAnswer(query2, identity, Row("\\N,\\N,\\N") :: Nil)
}
+
+ test("SPARK-34879: HiveInspector supports DayTimeIntervalType and
YearMonthIntervalType") {
+ assume(TestUtils.testCommandAvailable("/bin/bash"))
+ withTempView("v") {
+ val df = Seq(
+ (Duration.ofDays(1),
+ Duration.ofSeconds(100).plusNanos(123456),
+ Duration.ofSeconds(Long.MaxValue /
DateTimeConstants.MICROS_PER_SECOND),
+ Period.ofMonths(10)),
+ (Duration.ofDays(1),
+ Duration.ofSeconds(100).plusNanos(1123456789),
+ Duration.ofSeconds(Long.MaxValue /
DateTimeConstants.MICROS_PER_SECOND),
+ Period.ofMonths(10))
+ ).toDF("a", "b", "c", "d")
+ .select('a, 'b, 'c.cast(DayTimeIntervalType).as("c_1"), 'd)
Review comment:
> `'a` is deprecated in the future scala versions. I wouldn't use it.
Let's replace it by `$"a"`
Done
##########
File path:
sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
##########
@@ -528,4 +530,65 @@ class HiveScriptTransformationSuite extends
BaseScriptTransformationSuite with T
checkAnswer(query2, identity, Row("\\N,\\N,\\N") :: Nil)
}
+
+ test("SPARK-34879: HiveInspector supports DayTimeIntervalType and
YearMonthIntervalType") {
+ assume(TestUtils.testCommandAvailable("/bin/bash"))
+ withTempView("v") {
+ val df = Seq(
+ (Duration.ofDays(1),
+ Duration.ofSeconds(100).plusNanos(123456),
+ Duration.ofSeconds(Long.MaxValue /
DateTimeConstants.MICROS_PER_SECOND),
Review comment:
> I would propose to test the corner case `Duration.of(Long.MaxValue,
ChronoUnit.MICROS)`
Done
##########
File path:
sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
##########
@@ -346,6 +347,20 @@ private[hive] trait HiveInspectors {
withNullSafe(o => getTimestampWritable(o))
case _: TimestampObjectInspector =>
withNullSafe(o => DateTimeUtils.toJavaTimestamp(o.asInstanceOf[Long]))
+ case _: HiveIntervalDayTimeObjectInspector if x.preferWritable() =>
+ withNullSafe(o => getDayTimeIntervalWritable(o))
+ case _: HiveIntervalDayTimeObjectInspector =>
+ withNullSafe(o => {
+ val duration = IntervalUtils.microsToDuration(o.asInstanceOf[Long])
+ new HiveIntervalDayTime(duration.getSeconds, duration.getNano)
+ })
+ case _: HiveIntervalYearMonthObjectInspector if x.preferWritable() =>
+ withNullSafe(o => getYearMonthIntervalWritable(o))
+ case _: HiveIntervalYearMonthObjectInspector =>
+ withNullSafe(o => {
+ val period = IntervalUtils.monthsToPeriod(o.asInstanceOf[Int])
+ new HiveIntervalYearMonth(period.getYears, period.getMonths)
+ })
Review comment:
>
Done
##########
File path:
sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
##########
@@ -512,6 +527,13 @@ private[hive] trait HiveInspectors {
_ => constant
case poi: VoidObjectInspector =>
_ => null // always be null for void object inspector
+ case ym: WritableConstantHiveIntervalDayTimeObjectInspector =>
Review comment:
> `ym` -> `dt`
Done
##########
File path:
sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
##########
@@ -512,6 +527,13 @@ private[hive] trait HiveInspectors {
_ => constant
case poi: VoidObjectInspector =>
_ => null // always be null for void object inspector
+ case ym: WritableConstantHiveIntervalDayTimeObjectInspector =>
+ val constant =
ym.getWritableConstantValue.asInstanceOf[HiveIntervalDayTime]
+ _ => IntervalUtils.durationToMicros(
+
Duration.ofSeconds(constant.getTotalSeconds).plusNanos(constant.getNanos.toLong))
+ case dt: WritableConstantHiveIntervalYearMonthObjectInspector =>
Review comment:
> `dt` -> `ym`
Done
##########
File path:
sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
##########
@@ -1024,6 +1102,24 @@ private[hive] trait HiveInspectors {
new
hiveIo.TimestampWritable(DateTimeUtils.toJavaTimestamp(value.asInstanceOf[Long]))
}
+ private def getDayTimeIntervalWritable(value: Any):
hiveIo.HiveIntervalDayTimeWritable =
+ if (value == null) {
+ null
+ } else {
+ val duration = IntervalUtils.microsToDuration(value.asInstanceOf[Long])
+ new hiveIo.HiveIntervalDayTimeWritable(
+ new HiveIntervalDayTime(duration.getSeconds, duration.getNano))
+ }
+
+ private def getYearMonthIntervalWritable(value: Any):
hiveIo.HiveIntervalYearMonthWritable =
+ if (value == null) {
+ null
+ } else {
+ val period = IntervalUtils.monthsToPeriod(value.asInstanceOf[Int])
+ new hiveIo.HiveIntervalYearMonthWritable(
+ new HiveIntervalYearMonth(period.getYears, period.getMonths))
Review comment:
>
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
