andygrove commented on code in PR #381:
URL: https://github.com/apache/datafusion-spark/pull/381#discussion_r4642834910


##########
crates/spark/src/function/datetime/timestamp_micros.rs:
##########
@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::datatypes::{DataType, Field, FieldRef, TimeUnit};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, internal_err, plan_err};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, 
ScalarUDF,
+    ScalarUDFImpl, Signature, Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Spark-compatible `timestamp_micros` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#timestamp_micros>
+#[user_doc(
+    doc_section(label = "Time and Date Functions"),
+    description = "Returns the timestamp that is `microseconds` microseconds 
after \
+        1970-01-01 00:00:00 UTC. Negative values precede the epoch. Unlike 
DataFusion's \
+        `to_timestamp_micros`, only signed integer inputs are accepted; 
floating-point, \
+        decimal and string inputs are rejected. The result has microsecond 
precision \
+        and the session time zone (`datafusion.execution.time_zone`, default 
UTC). \
+        The time zone affects display, not the stored microsecond count. Every 
BIGINT \
+        value is valid, including its minimum and maximum. Returns NULL for 
NULL input.",
+    syntax_example = "timestamp_micros(microseconds)",
+    sql_example = r#"```sql
+> SELECT timestamp_micros(0::BIGINT);
+1970-01-01T00:00:00Z
+> SELECT timestamp_micros(-1::BIGINT);
+1969-12-31T23:59:59.999999Z
+> SELECT timestamp_micros(NULL);
+NULL
+```"#,
+    argument(
+        name = "microseconds",
+        description = "TINYINT, SMALLINT, INT or BIGINT count of microseconds 
since the UTC epoch. NULL returns NULL."
+    )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkTimestampMicros {
+    signature: Signature,
+    timezone: Arc<str>,
+}
+
+impl Default for SparkTimestampMicros {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkTimestampMicros {
+    pub fn new() -> Self {
+        Self::new_with_config(&ConfigOptions::default())
+    }
+
+    pub fn new_with_config(config: &ConfigOptions) -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Stable),
+            timezone: config
+                .execution
+                .time_zone
+                .as_ref()
+                .map_or("UTC", |tz| tz.as_str())
+                .into(),
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkTimestampMicros {
+    fn name(&self) -> &str {
+        "timestamp_micros"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {

Review Comment:
   I agree `SparkInputTypes` isn't a fit as it stands, since it models 
`ImplicitCastInputTypes` and would cast `'1'`. `timestamp_millis` and 
`timestamp_seconds` will need the same `ExpectsInputTypes` treatment soon, 
though. What do you think about adding a check-only mode to `implicit_cast.rs` 
so all three share it? That would also give this error the same Spark-shaped 
message the other functions use ("The first parameter of `timestamp_micros` 
requires the "INTEGRAL" type..."). If you'd rather keep this PR focused, a 
tracking issue linked here would work too.



##########
crates/spark/src/function/datetime/timestamp_micros.rs:
##########
@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::datatypes::{DataType, Field, FieldRef, TimeUnit};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, internal_err, plan_err};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, 
ScalarUDF,
+    ScalarUDFImpl, Signature, Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Spark-compatible `timestamp_micros` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#timestamp_micros>
+#[user_doc(
+    doc_section(label = "Time and Date Functions"),
+    description = "Returns the timestamp that is `microseconds` microseconds 
after \
+        1970-01-01 00:00:00 UTC. Negative values precede the epoch. Unlike 
DataFusion's \
+        `to_timestamp_micros`, only signed integer inputs are accepted; 
floating-point, \
+        decimal and string inputs are rejected. The result has microsecond 
precision \
+        and the session time zone (`datafusion.execution.time_zone`, default 
UTC). \
+        The time zone affects display, not the stored microsecond count. Every 
BIGINT \
+        value is valid, including its minimum and maximum. Returns NULL for 
NULL input.",
+    syntax_example = "timestamp_micros(microseconds)",
+    sql_example = r#"```sql
+> SELECT timestamp_micros(0::BIGINT);
+1970-01-01T00:00:00Z
+> SELECT timestamp_micros(-1::BIGINT);
+1969-12-31T23:59:59.999999Z
+> SELECT timestamp_micros(NULL);
+NULL
+```"#,
+    argument(
+        name = "microseconds",
+        description = "TINYINT, SMALLINT, INT or BIGINT count of microseconds 
since the UTC epoch. NULL returns NULL."
+    )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkTimestampMicros {
+    signature: Signature,
+    timezone: Arc<str>,
+}
+
+impl Default for SparkTimestampMicros {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkTimestampMicros {
+    pub fn new() -> Self {
+        Self::new_with_config(&ConfigOptions::default())
+    }
+
+    pub fn new_with_config(config: &ConfigOptions) -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Stable),
+            timezone: config
+                .execution
+                .time_zone
+                .as_ref()
+                .map_or("UTC", |tz| tz.as_str())
+                .into(),
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkTimestampMicros {
+    fn name(&self) -> &str {
+        "timestamp_micros"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+        let [data_type] = take_function_args(self.name(), arg_types)?;
+        let value_type = match data_type {
+            DataType::Dictionary(_, value_type) => value_type.as_ref(),
+            other => other,
+        };
+        // Spark's ExpectsInputTypes accepts integers and NULL, without the
+        // numeric/string casts allowed by ImplicitCastInputTypes.
+        if value_type.is_signed_integer() || matches!(value_type, 
DataType::Null) {

Review Comment:
   The `Integral` rule in `implicit_cast.rs` uses `is_integer()`, so it also 
accepts `UInt8`/`UInt16`/`UInt32`. Was rejecting unsigned types here 
intentional? Spark never produces them, but a DataFusion caller with a `UInt32` 
column would see different behavior here than from the other integral-argument 
functions. Widening those three to Int64 would be lossless.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to