comphead commented on code in PR #72: URL: https://github.com/apache/datafusion-spark/pull/72#discussion_r4566027875
########## crates/sqllogictest/test_files/datetime/make_time.slt: ########## @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +# Tests for Spark-compatible make_time (Spark 4.1+) +# Reference: https://spark.apache.org/docs/latest/api/sql/index.html#make_time +# Expected values come from Spark's sql-tests/results/time.sql.out. + +query D +SELECT make_time(1, 18, 19.87); +---- +01:18:19.870 + +query D +SELECT make_time(6, 30, 45.887); +---- +06:30:45.887 + +query D +SELECT make_time(0, 0, 0); +---- +00:00:00 + +query D +SELECT make_time(23, 59, 59.999999); +---- +23:59:59.999999 + +# NULL in any argument gives NULL +query D +SELECT make_time(NULL, 18, 19.87); +---- +NULL + +query D +SELECT make_time(1, NULL, 19.87); +---- +NULL + +query D +SELECT make_time(1, 18, NULL); +---- +NULL + +# Strings are cast to the argument types, like Spark's implicit casts +query D +SELECT make_time('6', '30', '45.5'); +---- +06:30:45.500 + +# Result type +query T +SELECT arrow_typeof(make_time(1, 18, 19.87)); +---- +Time64(ns) + +# Out-of-range fields raise DATETIME_FIELD_OUT_OF_BOUNDS +statement error Invalid value for HourOfDay \(valid values 0 - 23\): -1 +SELECT make_time(-1, 18, 19.87); + +statement error Invalid value for HourOfDay \(valid values 0 - 23\): 24 +SELECT make_time(24, 18, 19.87); + +statement error Invalid value for MinuteOfHour \(valid values 0 - 59\): 60 +SELECT make_time(1, 60, 19.87); + +statement error Invalid value for SecondOfMinute \(valid values 0 - 59\): 60 +SELECT make_time(1, 18, 60.0); + +statement error Invalid value for SecondOfMinute \(valid values 0 - 59\): 9999999999 +SELECT make_time(1, 18, 9999999999.999999); + +statement error Invalid value for SecondOfMinute \(valid values 0 - 59\): -999999999 +SELECT make_time(1, 18, -999999999.999999); + +statement error Invalid value for SecondOfMinute \(valid values 0 - 59\): 4294967297 +SELECT make_time(1, 18, 4294967297.999999); Review Comment: Cases worth adding here: negative fractional seconds (`make_time(1, 1, -0.5)`, where Spark floors for the check but prints `0`), a seconds decimal with scale above 6 next to 60 (`59.9999995::DECIMAL(10, 7)`, which Spark's cast to `DECIMAL(16, 6)` rounds half up to 60), and BIGINT or DOUBLE arguments to pin `coerce_types`. CI runs only `cargo test -p datafusion-spark-sqllogictest`, not `cargo test -p datafusion-spark`, so the Rust unit tests here do not run in CI. So `valid_times`, `out_of_range_fields_report_the_spark_message` and `coerce_types_rules` would be better as `.slt` cases too. ########## crates/spark/src/function/datetime/make_time.rs: ########## @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::sync::Arc; + +use arrow::array::{Array, ArrayRef, AsArray, Time64NanosecondArray}; +use arrow::datatypes::{DataType, Decimal128Type, Int32Type, TimeUnit}; +use datafusion_common::utils::take_function_args; +use datafusion_common::{DataFusionError, Result, exec_datafusion_err, exec_err}; +use datafusion_expr::{ + ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, +}; + +const MICROS_PER_SECOND: i128 = 1_000_000; +const NANOS_PER_MICRO: i64 = 1_000; + +/// Spark's seconds argument type, `DecimalType(16, 6)`. +const SECONDS_PRECISION: u8 = 16; +const SECONDS_SCALE: i8 = 6; + +/// Spark-compatible `make_time(hour, minute, second)` (Spark 4.1+). +/// <https://spark.apache.org/docs/latest/api/sql/index.html#make_time> +/// +/// Creates a `TIME(6)` from hour (0-23), minute (0-59) and second +/// (0-59.999999) fields, as nanoseconds since midnight (`Time64(Nanosecond)`). +/// The result is NULL if any argument is NULL; out-of-range fields raise +/// `DATETIME_FIELD_OUT_OF_BOUNDS`, regardless of ANSI mode. +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkMakeTime { + signature: Signature, +} + +impl Default for SparkMakeTime { + fn default() -> Self { + Self::new() + } +} + +impl SparkMakeTime { + pub fn new() -> Self { + Self { + signature: Signature::user_defined(Volatility::Immutable), + } + } +} + +fn out_of_bounds( + field: &str, + range: &str, + value: impl std::fmt::Display, +) -> DataFusionError { + exec_datafusion_err!( + "[DATETIME_FIELD_OUT_OF_BOUNDS.WITHOUT_SUGGESTION] Invalid value for {field} (valid values {range}): {value}" + ) +} + +/// `secs_micros` is the unscaled value of the `DECIMAL(16, 6)` seconds argument. +fn make_time(hours: i32, minutes: i32, secs_micros: i128) -> Result<i64> { + let full_secs = secs_micros.div_euclid(MICROS_PER_SECOND); + let frac_micros = secs_micros.rem_euclid(MICROS_PER_SECOND) as i64; + + // Checked first, like Spark, so the message reports the whole seconds + // (truncated toward zero) rather than a wrapped value. + if !(0..=59).contains(&full_secs) { Review Comment: Spark checks seconds first only when `fullSecs > Int.MaxValue || fullSecs < 0` (`DateTimeUtils.makeTime`). For 60 up to `Int.MaxValue` it calls `LocalTime.of(hours, minutes, secs, nanos)`, which checks hour, then minute, then second. So `make_time(24, 0, 60)` reports `HourOfDay` in Spark and `SecondOfMinute` here, and `make_time(1, 60, 60)` reports `MinuteOfHour` in Spark. The Comet source had Spark's order. Could we keep the `< 0 || > i32::MAX` check first, then hour and minute, then `> 59` on seconds, and add those two cases to the `.slt`? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
