andygrove commented on code in PR #381:
URL: https://github.com/apache/datafusion-spark/pull/381#discussion_r4642834910
##########
crates/spark/src/function/datetime/timestamp_micros.rs:
##########
@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::datatypes::{DataType, Field, FieldRef, TimeUnit};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, internal_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
ScalarUDF,
+ ScalarUDFImpl, Signature, Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Spark-compatible `timestamp_micros` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#timestamp_micros>
+#[user_doc(
+ doc_section(label = "Time and Date Functions"),
+ description = "Returns the timestamp that is `microseconds` microseconds
after \
+ 1970-01-01 00:00:00 UTC. Negative values precede the epoch. Unlike
DataFusion's \
+ `to_timestamp_micros`, only signed integer inputs are accepted;
floating-point, \
+ decimal and string inputs are rejected. The result has microsecond
precision \
+ and the session time zone (`datafusion.execution.time_zone`, default
UTC). \
+ The time zone affects display, not the stored microsecond count. Every
BIGINT \
+ value is valid, including its minimum and maximum. Returns NULL for
NULL input.",
+ syntax_example = "timestamp_micros(microseconds)",
+ sql_example = r#"```sql
+> SELECT timestamp_micros(0::BIGINT);
+1970-01-01T00:00:00Z
+> SELECT timestamp_micros(-1::BIGINT);
+1969-12-31T23:59:59.999999Z
+> SELECT timestamp_micros(NULL);
+NULL
+```"#,
+ argument(
+ name = "microseconds",
+ description = "TINYINT, SMALLINT, INT or BIGINT count of microseconds
since the UTC epoch. NULL returns NULL."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkTimestampMicros {
+ signature: Signature,
+ timezone: Arc<str>,
+}
+
+impl Default for SparkTimestampMicros {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkTimestampMicros {
+ pub fn new() -> Self {
+ Self::new_with_config(&ConfigOptions::default())
+ }
+
+ pub fn new_with_config(config: &ConfigOptions) -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Stable),
+ timezone: config
+ .execution
+ .time_zone
+ .as_ref()
+ .map_or("UTC", |tz| tz.as_str())
+ .into(),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkTimestampMicros {
+ fn name(&self) -> &str {
+ "timestamp_micros"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
Review Comment:
I agree `SparkInputTypes` isn't a fit as it stands, since it models
`ImplicitCastInputTypes` and would cast `'1'`. `timestamp_millis` and
`timestamp_seconds` will need the same `ExpectsInputTypes` treatment soon,
though. What do you think about adding a check-only mode to `implicit_cast.rs`
so all three share it? That would also give this error the same Spark-shaped
message the other functions use ("The first parameter of `timestamp_micros`
requires the "INTEGRAL" type..."). If you'd rather keep this PR focused, a
tracking issue linked here would work too.
##########
crates/spark/src/function/datetime/timestamp_micros.rs:
##########
@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::datatypes::{DataType, Field, FieldRef, TimeUnit};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, internal_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
ScalarUDF,
+ ScalarUDFImpl, Signature, Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Spark-compatible `timestamp_micros` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#timestamp_micros>
+#[user_doc(
+ doc_section(label = "Time and Date Functions"),
+ description = "Returns the timestamp that is `microseconds` microseconds
after \
+ 1970-01-01 00:00:00 UTC. Negative values precede the epoch. Unlike
DataFusion's \
+ `to_timestamp_micros`, only signed integer inputs are accepted;
floating-point, \
+ decimal and string inputs are rejected. The result has microsecond
precision \
+ and the session time zone (`datafusion.execution.time_zone`, default
UTC). \
+ The time zone affects display, not the stored microsecond count. Every
BIGINT \
+ value is valid, including its minimum and maximum. Returns NULL for
NULL input.",
+ syntax_example = "timestamp_micros(microseconds)",
+ sql_example = r#"```sql
+> SELECT timestamp_micros(0::BIGINT);
+1970-01-01T00:00:00Z
+> SELECT timestamp_micros(-1::BIGINT);
+1969-12-31T23:59:59.999999Z
+> SELECT timestamp_micros(NULL);
+NULL
+```"#,
+ argument(
+ name = "microseconds",
+ description = "TINYINT, SMALLINT, INT or BIGINT count of microseconds
since the UTC epoch. NULL returns NULL."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkTimestampMicros {
+ signature: Signature,
+ timezone: Arc<str>,
+}
+
+impl Default for SparkTimestampMicros {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkTimestampMicros {
+ pub fn new() -> Self {
+ Self::new_with_config(&ConfigOptions::default())
+ }
+
+ pub fn new_with_config(config: &ConfigOptions) -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Stable),
+ timezone: config
+ .execution
+ .time_zone
+ .as_ref()
+ .map_or("UTC", |tz| tz.as_str())
+ .into(),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkTimestampMicros {
+ fn name(&self) -> &str {
+ "timestamp_micros"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ let [data_type] = take_function_args(self.name(), arg_types)?;
+ let value_type = match data_type {
+ DataType::Dictionary(_, value_type) => value_type.as_ref(),
+ other => other,
+ };
+ // Spark's ExpectsInputTypes accepts integers and NULL, without the
+ // numeric/string casts allowed by ImplicitCastInputTypes.
+ if value_type.is_signed_integer() || matches!(value_type,
DataType::Null) {
Review Comment:
The `Integral` rule in `implicit_cast.rs` uses `is_integer()`, so it also
accepts `UInt8`/`UInt16`/`UInt32`. Was rejecting unsigned types here
intentional? Spark never produces them, but a DataFusion caller with a `UInt32`
column would see different behavior here than from the other integral-argument
functions. Widening those three to Int64 would be lossless.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]