This is an automated email from the ASF dual-hosted git repository.
andygrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 268849c0cb feat: add `make_interval` support (codegen dispatch +
native) (#5039)
268849c0cb is described below
commit 268849c0cb52c471b205e1dbbb81af98465fc024
Author: Peter Lee <[email protected]>
AuthorDate: Thu Aug 6 19:31:19 2026 +0800
feat: add `make_interval` support (codegen dispatch + native) (#5039)
---
docs/source/user-guide/latest/expressions.md | 4 +-
native/Cargo.lock | 1 +
native/spark-expr/Cargo.toml | 1 +
native/spark-expr/src/comet_scalar_funcs.rs | 7 +-
.../spark-expr/src/datetime_funcs/make_interval.rs | 87 ++++++++++++++++++++++
native/spark-expr/src/datetime_funcs/mod.rs | 2 +
native/spark-expr/src/lib.rs | 5 +-
.../org/apache/comet/serde/QueryPlanSerde.scala | 1 +
.../scala/org/apache/comet/serde/datetime.scala | 37 ++++++++-
.../expressions/datetime/make_interval.sql | 67 +++++++++++++++++
.../expressions/datetime/make_interval_ansi.sql | 41 ++++++++++
.../datetime/make_interval_dispatch.sql | 49 ++++++++++++
.../datetime/make_interval_dispatch_ansi.sql | 48 ++++++++++++
.../expressions/datetime/try_make_interval.sql | 39 ++++++++++
.../CometDatetimeExpressionBenchmark.scala | 55 ++++++++++++++
15 files changed, 436 insertions(+), 8 deletions(-)
diff --git a/docs/source/user-guide/latest/expressions.md
b/docs/source/user-guide/latest/expressions.md
index fd4324babc..192eae807e 100644
--- a/docs/source/user-guide/latest/expressions.md
+++ b/docs/source/user-guide/latest/expressions.md
@@ -277,7 +277,7 @@ The type-name conversion functions (`bigint`, `binary`,
`boolean`, `date`, `deci
| `localtimestamp` | ✅ | — | |
| `make_date` | ✅ | Native | |
| `make_dt_interval` | ✅ | Codegen dispatch | |
-| `make_interval` | 🔜 | — | Produces legacy CalendarInterval; tracked by
[#5061](https://github.com/apache/datafusion-comet/issues/5061) |
+| `make_interval` | ✅ | Hybrid | Routes through the JVM codegen dispatcher by
default; intervals outside Arrow's nanosecond range are tracked by
[#5279](https://github.com/apache/datafusion-comet/issues/5279); the native
path is opt-in via allowIncompatible
([details](compatibility/expressions/datetime.md)) |
| `make_time` | 🔜 | — | Spark 4.1 TIME type; tracked by
[#4288](https://github.com/apache/datafusion-comet/issues/4288) |
| `make_timestamp` | ✅ | Hybrid | |
| `make_timestamp_ltz` | ✅ | — | 2-arg TIME form falls back |
@@ -308,7 +308,7 @@ The type-name conversion functions (`bigint`, `binary`,
`boolean`, `date`, `deci
| `to_unix_timestamp` | ✅ | Hybrid | |
| `to_utc_timestamp` | ✅ | Hybrid | Routes through the JVM codegen dispatcher
by default (handles all timezone forms); the native path is opt-in via
allowIncompatible ([details](compatibility/expressions/datetime.md)) |
| `trunc` | ✅ | Hybrid | |
-| `try_make_interval` | 🔜 | — | Produces legacy CalendarInterval; tracked by
[#5061](https://github.com/apache/datafusion-comet/issues/5061) |
+| `try_make_interval` | ✅ | — | Rewrites to `MakeInterval`; same support as
`make_interval` (Spark 4.0+) |
| `try_make_timestamp` | ✅ | — | |
| `try_to_date` | ✅ | — | Rewrites to `Cast`/`GetTimestamp` before Comet sees
the plan; same support as `to_date` |
| `try_to_time` | 🔜 | — | Spark 4.1 TIME type; tracked by
[#4288](https://github.com/apache/datafusion-comet/issues/4288) |
diff --git a/native/Cargo.lock b/native/Cargo.lock
index eeaa9da5ce..ae8a2f7be7 100644
--- a/native/Cargo.lock
+++ b/native/Cargo.lock
@@ -2066,6 +2066,7 @@ dependencies = [
"datafusion",
"datafusion-comet-common",
"datafusion-comet-jni-bridge",
+ "datafusion-spark",
"futures",
"jni 0.22.4",
"num",
diff --git a/native/spark-expr/Cargo.toml b/native/spark-expr/Cargo.toml
index 2333d2f0e6..6faa9fec4e 100644
--- a/native/spark-expr/Cargo.toml
+++ b/native/spark-expr/Cargo.toml
@@ -30,6 +30,7 @@ edition = { workspace = true }
arrow = { workspace = true }
chrono = { workspace = true }
datafusion = { workspace = true }
+datafusion-spark = { workspace = true }
chrono-tz = { workspace = true }
num = { workspace = true }
regex = { workspace = true }
diff --git a/native/spark-expr/src/comet_scalar_funcs.rs
b/native/spark-expr/src/comet_scalar_funcs.rs
index 50732959c8..e90e01de95 100644
--- a/native/spark-expr/src/comet_scalar_funcs.rs
+++ b/native/spark-expr/src/comet_scalar_funcs.rs
@@ -28,8 +28,8 @@ use crate::{
spark_isnan, spark_lpad, spark_make_decimal, spark_month_name,
spark_read_side_padding,
spark_round, spark_rpad, spark_to_time, spark_unhex, spark_unscaled_value,
EvalMode,
SparkArrayPositionFunc, SparkArraySlice, SparkArraysOverlap,
SparkContains, SparkDateDiff,
- SparkDateFromUnixDate, SparkDateTrunc, SparkFlatten, SparkMakeDate,
SparkMakeTime,
- SparkNextDay, SparkSecondsToTimestamp, SparkSizeFunc,
+ SparkDateFromUnixDate, SparkDateTrunc, SparkFlatten, SparkMakeDate,
SparkMakeInterval,
+ SparkMakeTime, SparkNextDay, SparkSecondsToTimestamp, SparkSizeFunc,
};
use arrow::datatypes::DataType;
use datafusion::common::{DataFusionError, Result as DataFusionResult};
@@ -250,6 +250,9 @@ pub fn create_comet_physical_fun_with_eval_mode(
"make_date" => Ok(Arc::new(ScalarUDF::new_from_impl(SparkMakeDate::new(
fail_on_error,
)))),
+ "make_interval" =>
Ok(Arc::new(ScalarUDF::new_from_impl(SparkMakeInterval::new(
+ fail_on_error,
+ )))),
"next_day" => Ok(Arc::new(ScalarUDF::new_from_impl(SparkNextDay::new(
fail_on_error,
)))),
diff --git a/native/spark-expr/src/datetime_funcs/make_interval.rs
b/native/spark-expr/src/datetime_funcs/make_interval.rs
new file mode 100644
index 0000000000..9f7b21ca2a
--- /dev/null
+++ b/native/spark-expr/src/datetime_funcs/make_interval.rs
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::arithmetic_overflow_error;
+use arrow::array::Array;
+use arrow::datatypes::DataType;
+use datafusion::common::Result;
+use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs,
ScalarUDFImpl, Signature};
+use datafusion_spark::function::datetime::make_interval::SparkMakeInterval as
DataFusionMakeInterval;
+
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkMakeInterval {
+ inner: DataFusionMakeInterval,
+ fail_on_error: bool,
+}
+
+impl SparkMakeInterval {
+ pub fn new(fail_on_error: bool) -> Self {
+ Self {
+ inner: DataFusionMakeInterval::new(),
+ fail_on_error,
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkMakeInterval {
+ fn name(&self) -> &str {
+ self.inner.name()
+ }
+
+ fn signature(&self) -> &Signature {
+ self.inner.signature()
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ self.inner.return_type(arg_types)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ let inputs = if self.fail_on_error {
+ Some(args.args.clone())
+ } else {
+ None
+ };
+ let result = self.inner.invoke_with_args(args)?;
+
+ if let Some(inputs) = inputs {
+ let inputs_are_valid = |i| {
+ inputs.iter().all(|input| match input {
+ ColumnarValue::Array(values) => values.is_valid(i),
+ ColumnarValue::Scalar(value) => !value.is_null(),
+ })
+ };
+ let overflow = match &result {
+ ColumnarValue::Array(values) =>
values.nulls().is_some_and(|nulls| {
+ nulls.null_count() != 0
+ && nulls
+ .iter()
+ .enumerate()
+ .any(|(i, is_valid)| !is_valid &&
inputs_are_valid(i))
+ }),
+ ColumnarValue::Scalar(value) => value.is_null() &&
inputs_are_valid(0),
+ };
+ if overflow {
+ // Spark identifies the integer or long operation that
overflowed. The native
+ // wrapper only sees the result null mask, so it can only
report interval overflow.
+ return Err(arithmetic_overflow_error("interval").into());
+ }
+ }
+
+ Ok(result)
+ }
+}
diff --git a/native/spark-expr/src/datetime_funcs/mod.rs
b/native/spark-expr/src/datetime_funcs/mod.rs
index 37c5fa5dd2..05530f29c2 100644
--- a/native/spark-expr/src/datetime_funcs/mod.rs
+++ b/native/spark-expr/src/datetime_funcs/mod.rs
@@ -22,6 +22,7 @@ mod day_month_name;
mod extract_date_part;
mod hours;
mod make_date;
+mod make_interval;
mod make_time;
mod next_day;
mod seconds_to_timestamp;
@@ -38,6 +39,7 @@ pub use extract_date_part::SparkMinute;
pub use extract_date_part::SparkSecond;
pub use hours::SparkHoursTransform;
pub use make_date::SparkMakeDate;
+pub use make_interval::SparkMakeInterval;
pub use make_time::SparkMakeTime;
pub use next_day::SparkNextDay;
pub use seconds_to_timestamp::SparkSecondsToTimestamp;
diff --git a/native/spark-expr/src/lib.rs b/native/spark-expr/src/lib.rs
index 42ab43361a..2b5c29befc 100644
--- a/native/spark-expr/src/lib.rs
+++ b/native/spark-expr/src/lib.rs
@@ -77,8 +77,9 @@ pub use comet_scalar_funcs::{
pub use csv_funcs::*;
pub use datetime_funcs::{
spark_day_name, spark_month_name, spark_to_time, SparkDateDiff,
SparkDateFromUnixDate,
- SparkDateTrunc, SparkHour, SparkHoursTransform, SparkMakeDate,
SparkMakeTime, SparkMinute,
- SparkNextDay, SparkSecond, SparkSecondsToTimestamp, SparkUnixTimestamp,
TimestampTruncExpr,
+ SparkDateTrunc, SparkHour, SparkHoursTransform, SparkMakeDate,
SparkMakeInterval,
+ SparkMakeTime, SparkMinute, SparkNextDay, SparkSecond,
SparkSecondsToTimestamp,
+ SparkUnixTimestamp, TimestampTruncExpr,
};
pub use error::{decimal_overflow_error, SparkError, SparkErrorWithContext,
SparkResult};
pub use hash_funcs::*;
diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
index 4299eb14ba..85601a9e0c 100644
--- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
@@ -303,6 +303,7 @@ object QueryPlanSerde extends Logging with CometExprShim
with CometTypeShim {
classOf[MakeTimestamp] -> CometMakeTimestamp,
classOf[MakeYMInterval] -> CometMakeYMInterval,
classOf[MakeDTInterval] -> CometMakeDTInterval,
+ classOf[MakeInterval] -> CometMakeInterval,
classOf[MultiplyDTInterval] -> CometMultiplyDTInterval,
classOf[TimestampAdd] -> CometTimestampAdd,
classOf[TimestampDiff] -> CometTimestampDiff,
diff --git a/spark/src/main/scala/org/apache/comet/serde/datetime.scala
b/spark/src/main/scala/org/apache/comet/serde/datetime.scala
index 946c34a857..a74a158e97 100644
--- a/spark/src/main/scala/org/apache/comet/serde/datetime.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/datetime.scala
@@ -21,9 +21,9 @@ package org.apache.comet.serde
import java.util.Locale
-import org.apache.spark.sql.catalyst.expressions.{AddMonths, Attribute,
ConvertTimezone, DateAdd, DateDiff, DateFormatClass, DateFromUnixDate, DateSub,
DayOfMonth, DayOfWeek, DayOfYear, Days, Expression, FromUTCTimestamp,
GetDateField, GetTimestamp, Hour, Hours, LastDay, Literal, MakeDate,
MakeDTInterval, MakeTimestamp, MakeYMInterval, MicrosToTimestamp,
MillisToTimestamp, Minute, Month, MonthsBetween, MultiplyDTInterval, NextDay,
PreciseTimestampConversion, Quarter, Second, SecondsToTim [...]
+import org.apache.spark.sql.catalyst.expressions.{AddMonths, Attribute, Cast,
ConvertTimezone, DateAdd, DateDiff, DateFormatClass, DateFromUnixDate, DateSub,
DayOfMonth, DayOfWeek, DayOfYear, Days, Expression, FromUTCTimestamp,
GetDateField, GetTimestamp, Hour, Hours, LastDay, Literal, MakeDate,
MakeDTInterval, MakeInterval, MakeTimestamp, MakeYMInterval, MicrosToTimestamp,
MillisToTimestamp, Minute, Month, MonthsBetween, MultiplyDTInterval, NextDay,
PreciseTimestampConversion, Quarter, [...]
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{DataType, DateType, DoubleType, FloatType,
IntegerType, LongType, StringType, TimestampNTZType, TimestampType}
+import org.apache.spark.sql.types.{CalendarIntervalType, DataType, DateType,
DoubleType, FloatType, IntegerType, LongType, StringType, TimestampNTZType,
TimestampType}
import org.apache.spark.unsafe.types.UTF8String
import org.apache.comet.CometConf
@@ -963,6 +963,39 @@ object CometMakeYMInterval extends
CometCodegenDispatch[MakeYMInterval]
object CometMakeDTInterval extends CometCodegenDispatch[MakeDTInterval]
+object CometMakeInterval extends CometExpressionSerde[MakeInterval] with
CodegenDispatchFallback {
+ private val incompatReason =
+ "The native implementation converts seconds to `Float64`, which can lose
microsecond" +
+ " precision, and stores time in nanoseconds, which overflows for large
time components" +
+ " (hours, minutes, seconds) that Spark can represent."
+
+ override def getCompatibleNotes(): Seq[String] = Seq(
+ "Both the default JVM codegen-dispatch path and the native path currently
limit the" +
+ " elapsed-time component to about 292 years in either direction. This
only affects" +
+ " extreme intervals and is tracked in" +
+ " [#5279](https://github.com/apache/datafusion-comet/issues/5279).")
+
+ override def getIncompatibleReasons(): Seq[String] = Seq(incompatReason)
+
+ override def getSupportLevel(expr: MakeInterval): SupportLevel =
+ Incompatible(Some(incompatReason))
+
+ override def convert(
+ expr: MakeInterval,
+ inputs: Seq[Attribute],
+ binding: Boolean): Option[Expr] = {
+ // The explicit return type skips DataFusion's registry coercion, but its
kernel needs Float64.
+ val children = expr.children.updated(6, Cast(expr.secs, DoubleType))
+ val childExprs = children.map(exprToProtoInternal(_, inputs, binding))
+ val optExpr = scalarFunctionExprToProtoWithReturnType(
+ "make_interval",
+ CalendarIntervalType,
+ expr.failOnError,
+ childExprs: _*)
+ optExpr
+ }
+}
+
object CometMultiplyDTInterval extends CometCodegenDispatch[MultiplyDTInterval]
object CometTimestampAdd extends CometCodegenDispatch[TimestampAdd]
diff --git
a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql
b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql
new file mode 100644
index 0000000000..952a9085ee
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql
@@ -0,0 +1,67 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Config: spark.comet.expression.MakeInterval.allowIncompatible=true
+
+statement
+CREATE TABLE test_make_interval(
+ years int,
+ months int,
+ weeks int,
+ days int,
+ hours int,
+ mins int,
+ secs decimal(18, 6)) USING parquet
+
+statement
+INSERT INTO test_make_interval VALUES
+ (1, 2, 3, 4, 5, 6, 7.123456),
+ (0, 1, 0, 1, 0, 0, 100.000001),
+ (-1, -2, -1, -1, -1, -1, -1.500000),
+ (NULL, 1, 2, 3, 4, 5, 6.000000),
+ (2, NULL, 2, 3, 4, 5, 6.000000),
+ (3, 1, 2, 3, 4, 5, NULL),
+ (-2147483648, 0, 0, 0, 0, 0, 0.000000)
+
+query
+SELECT make_interval(years, months, weeks, days, hours, mins, secs)
+FROM test_make_interval
+ORDER BY years
+
+query
+SELECT make_interval(1, 2), make_interval(3), make_interval()
+
+query
+SELECT make_interval(0, 1, 0, 1, 0, 0, 100.000001)
+
+query
+SELECT make_interval(2147483647)
+
+query ignore(https://github.com/apache/datafusion-comet/issues/5131)
+SELECT make_interval(1, 2, 3, 4, 0, 0, 123456789012.123456)
+
+query
+SELECT make_interval(0, 0, 0, 0, 0, 0, 999999999.999999)
+
+query ignore(https://github.com/apache/datafusion-comet/issues/5131)
+SELECT make_interval(0, 0, 0, 0, 0, 0, 999999999.000001)
+
+query ignore(https://github.com/apache/datafusion-comet/issues/5131)
+SELECT make_interval(0, 0, 0, 0, 2562048)
+
+query
+SELECT make_interval(0, 0, 0, 0, 0, 0, 1234567890123456789)
diff --git
a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql
b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql
new file mode 100644
index 0000000000..175b9ecd08
--- /dev/null
+++
b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql
@@ -0,0 +1,41 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Native ANSI execution must preserve Spark's overflow exception.
+-- Config: spark.sql.ansi.enabled=true
+-- Config: spark.comet.expression.MakeInterval.allowIncompatible=true
+
+statement
+CREATE TABLE test_make_interval_ansi(years int) USING parquet
+
+statement
+INSERT INTO test_make_interval_ansi VALUES (NULL)
+
+query
+SELECT make_interval(1, 2, 3, 4, 5, 6, 7.123456)
+
+query
+SELECT make_interval(years) FROM test_make_interval_ansi
+
+query expect_error(overflow. If necessary set)
+SELECT make_interval(2147483647)
+
+query expect_error(overflow. If necessary set)
+SELECT make_interval(0, 0, 2147483647)
+
+query ignore(https://github.com/apache/datafusion-comet/issues/5131)
+SELECT make_interval(0, 0, 0, 0, 2562048)
diff --git
a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch.sql
b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch.sql
new file mode 100644
index 0000000000..ee2d5e8160
--- /dev/null
+++
b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch.sql
@@ -0,0 +1,49 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- With allowIncompatible unset, MakeInterval uses Spark's JVM codegen
dispatcher.
+
+statement
+CREATE TABLE test_make_interval_dispatch(
+ years int,
+ months int,
+ weeks int,
+ days int,
+ hours int,
+ mins int,
+ secs decimal(18, 6)) USING parquet
+
+statement
+INSERT INTO test_make_interval_dispatch VALUES
+ (1, 2, 3, 4, 5, 6, 7.123456),
+ (0, 1, 0, 1, 0, 0, 100.000001),
+ (-1, -2, -1, -1, -1, -1, -1.500000),
+ (NULL, 1, 2, 3, 4, 5, 6.000000),
+ (2, NULL, 2, 3, 4, 5, 6.000000),
+ (3, 1, 2, 3, 4, 5, NULL),
+ (0, 0, 0, 0, 2562048, 0, 0.000000)
+
+query
+SELECT make_interval(years, months, weeks, days, hours, mins, secs)
+FROM test_make_interval_dispatch
+WHERE hours != 2562048
+ORDER BY years
+
+query ignore(https://github.com/apache/datafusion-comet/issues/5279)
+SELECT make_interval(0, 0, 0, 0, hours)
+FROM test_make_interval_dispatch
+WHERE hours = 2562048
diff --git
a/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch_ansi.sql
b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch_ansi.sql
new file mode 100644
index 0000000000..f199a8fb66
--- /dev/null
+++
b/spark/src/test/resources/sql-tests/expressions/datetime/make_interval_dispatch_ansi.sql
@@ -0,0 +1,48 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Config: spark.sql.ansi.enabled=true
+
+statement
+CREATE TABLE test_make_interval_dispatch_ansi(years int, weeks int, hours int)
USING parquet
+
+statement
+INSERT INTO test_make_interval_dispatch_ansi VALUES
+ (1, 0, 5),
+ (2147483647, 0, 0),
+ (0, 2147483647, 0),
+ (0, 0, 2562048)
+
+query
+SELECT make_interval(years, 0, weeks, 0, hours)
+FROM test_make_interval_dispatch_ansi
+WHERE years = 1
+
+query expect_error(overflow. If necessary set)
+SELECT make_interval(years)
+FROM test_make_interval_dispatch_ansi
+WHERE years = 2147483647
+
+query expect_error(overflow. If necessary set)
+SELECT make_interval(0, 0, weeks)
+FROM test_make_interval_dispatch_ansi
+WHERE weeks = 2147483647
+
+query ignore(https://github.com/apache/datafusion-comet/issues/5279)
+SELECT make_interval(0, 0, 0, 0, hours)
+FROM test_make_interval_dispatch_ansi
+WHERE hours = 2562048
diff --git
a/spark/src/test/resources/sql-tests/expressions/datetime/try_make_interval.sql
b/spark/src/test/resources/sql-tests/expressions/datetime/try_make_interval.sql
new file mode 100644
index 0000000000..5e481cb9ff
--- /dev/null
+++
b/spark/src/test/resources/sql-tests/expressions/datetime/try_make_interval.sql
@@ -0,0 +1,39 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- MinSparkVersion: 4.0
+-- ConfigMatrix: spark.sql.ansi.enabled=true,false
+
+statement
+CREATE TABLE test_try_make_interval(
+ years int,
+ months int,
+ weeks int,
+ days int,
+ hours int,
+ mins int,
+ secs decimal(18, 6)) USING parquet
+
+statement
+INSERT INTO test_try_make_interval VALUES
+ (1, 2, 3, 4, 5, 6, 7.123456),
+ (2147483647, 0, 0, 0, 0, 0, 0.000000)
+
+query
+SELECT try_make_interval(years, months, weeks, days, hours, mins, secs)
+FROM test_try_make_interval
+ORDER BY years
diff --git
a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala
b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala
index 224f2ad344..a9d5b35e17 100644
---
a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala
+++
b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala
@@ -19,9 +19,12 @@
package org.apache.spark.sql.benchmark
+import org.apache.spark.benchmark.Benchmark
import
org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA}
import org.apache.spark.sql.internal.SQLConf
+import org.apache.comet.CometConf
+
// spotless:off
/**
* Benchmark to measure Comet execution performance. To run this benchmark:
@@ -131,6 +134,54 @@ object CometDatetimeExpressionBenchmark extends
CometBenchmarkBase {
}
}
+ def makeIntervalBenchmark(values: Int): Unit = {
+ withTempPath { dir =>
+ withTempTable("parquetV1Table") {
+ prepareTable(
+ dir,
+ spark.sql(s"""SELECT
+ | CAST(ABS(value) % 10 AS INT) AS y,
+ | CAST(ABS(value) % 12 AS INT) AS mo,
+ | CAST(ABS(value) % 4 AS INT) AS w,
+ | CAST(ABS(value) % 28 AS INT) AS d,
+ | CAST(ABS(value) % 24 AS INT) AS h,
+ | CAST(ABS(value) % 60 AS INT) AS mi,
+ | CAST(ABS(value) % 60 AS DECIMAL(18, 6)) AS s
+ |FROM $tbl""".stripMargin))
+
+ val query = "SELECT make_interval(y, mo, w, d, h, mi, s) FROM
parquetV1Table"
+ def consumeIntervals(): Unit = {
+
spark.sql(query).queryExecution.toRdd.foreachPartition(_.foreach(_.getInterval(0)))
+ }
+ val benchmark = new Benchmark("MakeInterval", values, output = output)
+ val cometConfigs = Map(
+ CometConf.COMET_ENABLED.key -> "true",
+ CometConf.COMET_EXEC_ENABLED.key -> "true",
+ "spark.sql.optimizer.excludedRules" ->
+ "org.apache.spark.sql.catalyst.optimizer.ConstantFolding")
+
+ benchmark.addCase("Spark") { _ =>
+ withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
+ consumeIntervals()
+ }
+ }
+ benchmark.addCase("Comet (codegen dispatch)") { _ =>
+ withSQLConf(cometConfigs.toSeq: _*) {
+ consumeIntervals()
+ }
+ }
+ benchmark.addCase("Comet (native)") { _ =>
+ val configs =
+ cometConfigs ++
Map(CometConf.getExprAllowIncompatConfigKey("MakeInterval") -> "true")
+ withSQLConf(configs.toSeq: _*) {
+ consumeIntervals()
+ }
+ }
+ benchmark.run()
+ }
+ }
+ }
+
override def runCometBenchmark(mainArgs: Array[String]): Unit = {
val values = 1024 * 1024;
@@ -167,6 +218,10 @@ object CometDatetimeExpressionBenchmark extends
CometBenchmarkBase {
makeTimeBenchmark(v)
}
}
+
+ runBenchmarkWithTable("MakeInterval", values) { v =>
+ makeIntervalBenchmark(v)
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
