This is an automated email from the ASF dual-hosted git repository.
csy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/auron.git
The following commit(s) were added to refs/heads/master by this push:
new 4f015cfc [AURON #1686]Fix IllegalArgumentException in buildTimePartExt
leading to fallback to Spark (#1687)
4f015cfc is described below
commit 4f015cfcac9b0cc8727d33a53de00439a604e15e
Author: Thomas <[email protected]>
AuthorDate: Fri Dec 5 17:11:07 2025 +0800
[AURON #1686]Fix IllegalArgumentException in buildTimePartExt leading to
fallback to Spark (#1687)
---
.../scala/org.apache.auron/AuronQuerySuite.scala | 36 ++++++++--------------
.../apache/spark/sql/auron/NativeConverters.scala | 2 +-
2 files changed, 14 insertions(+), 24 deletions(-)
diff --git
a/spark-extension-shims-spark/src/test/scala/org.apache.auron/AuronQuerySuite.scala
b/spark-extension-shims-spark/src/test/scala/org.apache.auron/AuronQuerySuite.scala
index 9151eec8..ccd9f0e5 100644
---
a/spark-extension-shims-spark/src/test/scala/org.apache.auron/AuronQuerySuite.scala
+++
b/spark-extension-shims-spark/src/test/scala/org.apache.auron/AuronQuerySuite.scala
@@ -345,6 +345,7 @@ class AuronQuerySuite extends AuronQueryTest with
BaseAuronSQLSuite with AuronSQ
checkAnswer(sql(q), Seq(expected))
}
}
+
test("test filter with hour function") {
withEnvConf("spark.auron.datetime.extract.enabled" -> "true") {
withTable("t_hour") {
@@ -356,15 +357,13 @@ class AuronQuerySuite extends AuronQueryTest with
BaseAuronSQLSuite with AuronSQ
|""".stripMargin)
// Keep rows where HOUR >= 8, then group by hour
- checkAnswer(
- sql("""
+ checkSparkAnswerAndOperator("""
|select h, count(*)
|from (select hour(event_time) as h from t_hour) t
|where h >= 8
|group by h
|order by h
- |""".stripMargin),
- Seq(Row(8, 2)))
+ |""".stripMargin)
}
}
}
@@ -380,14 +379,12 @@ class AuronQuerySuite extends AuronQueryTest with
BaseAuronSQLSuite with AuronSQ
|""".stripMargin)
// Keep rows where MINUTE = 30, then group by minute
- checkAnswer(
- sql("""
+ checkSparkAnswerAndOperator("""
|select m, count(*)
|from (select minute(event_time) as m from t_minute) t
|where m = 30
|group by m
- |""".stripMargin),
- Seq(Row(30, 2)))
+ |""".stripMargin)
}
}
}
@@ -403,14 +400,12 @@ class AuronQuerySuite extends AuronQueryTest with
BaseAuronSQLSuite with AuronSQ
|""".stripMargin)
// Keep rows where SECOND = 0, then group by second
- checkAnswer(
- sql("""
+ checkSparkAnswerAndOperator("""
|select s, count(*)
|from (select second(event_time) as s from t_second) t
|where s = 0
|group by s
- |""".stripMargin),
- Seq(Row(0, 2)))
+ |""".stripMargin)
}
}
}
@@ -421,16 +416,14 @@ class AuronQuerySuite extends AuronQueryTest with
BaseAuronSQLSuite with AuronSQ
withTable("t_date_parts") {
sql(
"create table t_date_parts using parquet as select date'2024-12-18'
as d union all select date'2024-12-19'")
- checkAnswer(
- sql("""
+ checkSparkAnswerAndOperator("""
|select
| hour(d) as h,
| minute(d) as m,
| second(d) as s
|from t_date_parts
|order by d
- |""".stripMargin),
- Seq(Row(0, 0, 0), Row(0, 0, 0)))
+ |""".stripMargin)
}
}
}
@@ -444,12 +437,10 @@ class AuronQuerySuite extends AuronQueryTest with
BaseAuronSQLSuite with AuronSQ
|select from_utc_timestamp(to_timestamp('1970-01-01 00:00:00'),
'Asia/Shanghai') as ts
|""".stripMargin)
- checkAnswer(
- sql("""
+ checkSparkAnswerAndOperator("""
|select hour(ts), minute(ts), second(ts)
|from t_tz
- |""".stripMargin),
- Seq(Row(8, 0, 0)))
+ |""".stripMargin)
}
}
}
@@ -464,9 +455,8 @@ class AuronQuerySuite extends AuronQueryTest with
BaseAuronSQLSuite with AuronSQ
|""".stripMargin)
// Kolkata -> 05:30:00; Kathmandu -> 05:45:00
- checkAnswer(
- sql("select minute(ts1), second(ts1), minute(ts2), second(ts2) from
t_tz2"),
- Seq(Row(30, 0, 45, 0)))
+ checkSparkAnswerAndOperator(
+ "select minute(ts1), second(ts1), minute(ts2), second(ts2) from
t_tz2")
}
}
}
diff --git
a/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala
b/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala
index 65d6040d..677de319 100644
---
a/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala
+++
b/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala
@@ -1336,7 +1336,7 @@ object NativeConverters extends Logging {
fallback: Expression => pb.PhysicalExprNode): pb.PhysicalExprNode = {
val tzArg: Expression = child.dataType match {
case TimestampType =>
- Literal(SQLConf.get.sessionLocalTimeZone, StringType)
+ Literal.create(SQLConf.get.sessionLocalTimeZone, StringType)
case _ =>
Literal.create(null, StringType)
}
