Ritesh17rb commented on code in PR #381:
URL: https://github.com/apache/datafusion-spark/pull/381#discussion_r4643816265
##########
crates/spark/src/function/datetime/timestamp_micros.rs:
##########
@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::datatypes::{DataType, Field, FieldRef, TimeUnit};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, internal_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
ScalarUDF,
+ ScalarUDFImpl, Signature, Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Spark-compatible `timestamp_micros` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#timestamp_micros>
+#[user_doc(
+ doc_section(label = "Time and Date Functions"),
+ description = "Returns the timestamp that is `microseconds` microseconds
after \
+ 1970-01-01 00:00:00 UTC. Negative values precede the epoch. Unlike
DataFusion's \
+ `to_timestamp_micros`, only signed integer inputs are accepted;
floating-point, \
+ decimal and string inputs are rejected. The result has microsecond
precision \
+ and the session time zone (`datafusion.execution.time_zone`, default
UTC). \
+ The time zone affects display, not the stored microsecond count. Every
BIGINT \
+ value is valid, including its minimum and maximum. Returns NULL for
NULL input.",
+ syntax_example = "timestamp_micros(microseconds)",
+ sql_example = r#"```sql
+> SELECT timestamp_micros(0::BIGINT);
+1970-01-01T00:00:00Z
+> SELECT timestamp_micros(-1::BIGINT);
+1969-12-31T23:59:59.999999Z
+> SELECT timestamp_micros(NULL);
+NULL
+```"#,
+ argument(
+ name = "microseconds",
+ description = "TINYINT, SMALLINT, INT or BIGINT count of microseconds
since the UTC epoch. NULL returns NULL."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkTimestampMicros {
+ signature: Signature,
+ timezone: Arc<str>,
+}
+
+impl Default for SparkTimestampMicros {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkTimestampMicros {
+ pub fn new() -> Self {
+ Self::new_with_config(&ConfigOptions::default())
+ }
+
+ pub fn new_with_config(config: &ConfigOptions) -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Stable),
+ timezone: config
+ .execution
+ .time_zone
+ .as_ref()
+ .map_or("UTC", |tz| tz.as_str())
+ .into(),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkTimestampMicros {
+ fn name(&self) -> &str {
+ "timestamp_micros"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
Review Comment:
Implemented in 1526fc0. Added `SparkInputTypes::check_only` in
`implicit_cast.rs` and switched `timestamp_micros` to use it. It validates the
declared input types without implicit casts, handles NULLs and dictionary
layouts, and shares the Spark-style argument errors. The helper is available
for `timestamp_millis` and `timestamp_seconds` when those are implemented.
Added coverage for validation in both ANSI modes and the error message.
Workspace Clippy and all 516 SQL test runs pass.
##########
crates/spark/src/function/datetime/timestamp_micros.rs:
##########
@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::datatypes::{DataType, Field, FieldRef, TimeUnit};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, internal_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
ScalarUDF,
+ ScalarUDFImpl, Signature, Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Spark-compatible `timestamp_micros` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#timestamp_micros>
+#[user_doc(
+ doc_section(label = "Time and Date Functions"),
+ description = "Returns the timestamp that is `microseconds` microseconds
after \
+ 1970-01-01 00:00:00 UTC. Negative values precede the epoch. Unlike
DataFusion's \
+ `to_timestamp_micros`, only signed integer inputs are accepted;
floating-point, \
+ decimal and string inputs are rejected. The result has microsecond
precision \
+ and the session time zone (`datafusion.execution.time_zone`, default
UTC). \
+ The time zone affects display, not the stored microsecond count. Every
BIGINT \
+ value is valid, including its minimum and maximum. Returns NULL for
NULL input.",
+ syntax_example = "timestamp_micros(microseconds)",
+ sql_example = r#"```sql
+> SELECT timestamp_micros(0::BIGINT);
+1970-01-01T00:00:00Z
+> SELECT timestamp_micros(-1::BIGINT);
+1969-12-31T23:59:59.999999Z
+> SELECT timestamp_micros(NULL);
+NULL
+```"#,
+ argument(
+ name = "microseconds",
+ description = "TINYINT, SMALLINT, INT or BIGINT count of microseconds
since the UTC epoch. NULL returns NULL."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkTimestampMicros {
+ signature: Signature,
+ timezone: Arc<str>,
+}
+
+impl Default for SparkTimestampMicros {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkTimestampMicros {
+ pub fn new() -> Self {
+ Self::new_with_config(&ConfigOptions::default())
+ }
+
+ pub fn new_with_config(config: &ConfigOptions) -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Stable),
+ timezone: config
+ .execution
+ .time_zone
+ .as_ref()
+ .map_or("UTC", |tz| tz.as_str())
+ .into(),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkTimestampMicros {
+ fn name(&self) -> &str {
+ "timestamp_micros"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ let [data_type] = take_function_args(self.name(), arg_types)?;
+ let value_type = match data_type {
+ DataType::Dictionary(_, value_type) => value_type.as_ref(),
+ other => other,
+ };
+ // Spark's ExpectsInputTypes accepts integers and NULL, without the
+ // numeric/string casts allowed by ImplicitCastInputTypes.
+ if value_type.is_signed_integer() || matches!(value_type,
DataType::Null) {
Review Comment:
The signed-only check was intentional to follow Spark's native input types,
but I agree that accepting the safely widenable Arrow unsigned types is more
consistent here. Updated in 1526fc0 to accept `UInt8`, `UInt16`, and `UInt32`
and widen them losslessly to Int64. `UInt64` remains rejected because its full
range cannot fit in the signed timestamp count; this is documented. Added SQL
coverage for each accepted unsigned type's maximum value, dictionary input with
NULLs, and UInt64 rejection. All 516 SQL test runs pass, including both modes
for `timestamp_micros`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]