mbutrovich commented on code in PR #317:
URL: https://github.com/apache/datafusion-spark/pull/317#discussion_r4602472123


##########
crates/spark/src/function/conversion/cast/data_type.rs:
##########
@@ -0,0 +1,516 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+//! Parses Spark type names with Spark's SQL type grammar, the `dataType` rule
+//! of
+//! 
[`SqlBaseParser.g4`](https://github.com/apache/spark/blob/v4.2.0/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4#L1477-L1525).
+
+use std::sync::Arc;
+
+use arrow::datatypes::{DECIMAL128_MAX_PRECISION, DataType, Field, Fields, 
TimeUnit};
+use datafusion_common::{DataFusionError, Result, plan_datafusion_err};
+
+/// Parses a Spark type name, such as `int`, `decimal(10, 2)` or
+/// `struct<a: int, b: array<string>>`, into the Arrow type that represents it.
+///
+/// Keywords are case-insensitive, as in Spark. `TIMESTAMP` is
+/// `TIMESTAMP_LTZ`, Spark's default `spark.sql.timestampType`, and is tagged
+/// with `session_timezone`. `CHAR(n)` and `VARCHAR(n)` are `STRING`, as Spark
+/// casts them by default. Types Spark can name but this crate does not 
support,
+/// such as intervals, `TIME`, `VARIANT` and collated strings, are an error.
+pub(crate) fn parse_spark_type(
+    type_name: &str,
+    session_timezone: &str,
+) -> Result<DataType> {
+    let mut parser = Parser {
+        tokens: tokenize(type_name)?,
+        position: 0,
+        type_name,
+        session_timezone,
+    };
+    let data_type = parser.data_type()?;
+    match parser.peek() {
+        None => Ok(data_type),
+        Some(token) => Err(parser.error(&format!("unexpected {token}"))),
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+enum Token {
+    /// An unquoted identifier or keyword.
+    Word(String),
+    /// A backquoted identifier, with its quotes removed.
+    Quoted(String),
+    /// A single-quoted string, as in `COMMENT 'text'`.
+    Text,
+    Integer(u64),
+    Symbol(char),
+}
+
+impl std::fmt::Display for Token {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Token::Word(word) => write!(f, "'{word}'"),
+            Token::Quoted(name) => write!(f, "`{name}`"),
+            Token::Text => write!(f, "string literal"),
+            Token::Integer(value) => write!(f, "'{value}'"),
+            Token::Symbol(symbol) => write!(f, "'{symbol}'"),
+        }
+    }
+}
+
+fn invalid(type_name: &str, message: &str) -> DataFusionError {
+    plan_datafusion_err!("Invalid spark_cast target type '{type_name}': 
{message}")
+}
+
+fn tokenize(type_name: &str) -> Result<Vec<Token>> {
+    let error = |message: &str| invalid(type_name, message);
+    let mut tokens = vec![];
+    let mut chars = type_name.chars().peekable();
+    while let Some(&c) = chars.peek() {
+        if c.is_whitespace() {
+            chars.next();
+        } else if c.is_ascii_alphabetic() || c == '_' {
+            let mut word = String::new();
+            while let Some(&c) = chars.peek()
+                && (c.is_ascii_alphanumeric() || c == '_')
+            {
+                word.push(c);
+                chars.next();
+            }
+            tokens.push(Token::Word(word));
+        } else if c.is_ascii_digit() {
+            let mut digits = String::new();
+            while let Some(&c) = chars.peek()
+                && c.is_ascii_digit()
+            {
+                digits.push(c);
+                chars.next();
+            }
+            let value = digits
+                .parse()
+                .map_err(|_| error(&format!("'{digits}' is out of range")))?;
+            tokens.push(Token::Integer(value));
+        } else if c == '`' || c == '\'' {
+            // A doubled quote inside a quoted identifier or string is a 
literal
+            // quote.
+            chars.next();
+            let mut text = String::new();
+            loop {
+                match chars.next() {
+                    Some(q) if q == c => {
+                        if chars.peek() == Some(&c) {
+                            text.push(c);
+                            chars.next();
+                        } else {
+                            break;
+                        }
+                    }
+                    Some(other) => text.push(other),
+                    None => return Err(error("unterminated quote")),
+                }
+            }
+            tokens.push(if c == '`' {
+                Token::Quoted(text)
+            } else {
+                Token::Text
+            });
+        } else if "<>(),:".contains(c) {
+            tokens.push(Token::Symbol(c));
+            chars.next();
+        } else {
+            return Err(error(&format!("unexpected character '{c}'")));
+        }
+    }
+    Ok(tokens)
+}
+
+const TIMESTAMP_NTZ: DataType = DataType::Timestamp(TimeUnit::Microsecond, 
None);
+
+struct Parser<'a> {
+    tokens: Vec<Token>,
+    position: usize,
+    type_name: &'a str,
+    session_timezone: &'a str,
+}
+
+impl Parser<'_> {
+    fn error(&self, message: &str) -> DataFusionError {
+        invalid(self.type_name, message)
+    }
+
+    fn unsupported(&self, what: &str) -> DataFusionError {
+        plan_datafusion_err!(
+            "Unsupported spark_cast target type '{}': {what} is not supported",
+            self.type_name
+        )
+    }
+
+    fn peek(&self) -> Option<&Token> {
+        self.tokens.get(self.position)
+    }
+
+    /// Applies `f` to the next token, and consumes it if `f` returns `Some`.
+    fn eat_map<T>(&mut self, f: impl FnOnce(&Token) -> Option<T>) -> Option<T> 
{
+        let value = self.peek().and_then(f);
+        if value.is_some() {
+            self.position += 1;
+        }
+        value
+    }
+
+    fn peek_keyword(&self, keyword: &str) -> bool {
+        matches!(self.peek(), Some(Token::Word(word)) if 
word.eq_ignore_ascii_case(keyword))
+    }
+
+    fn eat_keyword(&mut self, keyword: &str) -> bool {
+        let found = self.peek_keyword(keyword);
+        if found {
+            self.position += 1;
+        }
+        found
+    }
+
+    fn expect_keyword(&mut self, keyword: &str) -> Result<()> {
+        if self.eat_keyword(keyword) {
+            Ok(())
+        } else {
+            Err(self.expected(&format!("'{keyword}'")))
+        }
+    }
+
+    fn eat_symbol(&mut self, symbol: char) -> bool {
+        let found = self.peek() == Some(&Token::Symbol(symbol));
+        if found {
+            self.position += 1;
+        }
+        found
+    }
+
+    fn expect_symbol(&mut self, symbol: char) -> Result<()> {
+        if self.eat_symbol(symbol) {
+            Ok(())
+        } else {
+            Err(self.expected(&format!("'{symbol}'")))
+        }
+    }
+
+    fn expected(&self, what: &str) -> DataFusionError {
+        match self.peek() {
+            Some(token) => self.error(&format!("expected {what}, found 
{token}")),
+            None => self.error(&format!("expected {what}")),
+        }
+    }
+
+    fn integer(&mut self) -> Result<u64> {
+        self.eat_map(|token| match token {
+            Token::Integer(value) => Some(*value),
+            _ => None,
+        })
+        .ok_or_else(|| self.expected("an integer"))
+    }
+
+    /// The parenthesized integers after a type name, if any.
+    fn parameters(&mut self) -> Result<Vec<u64>> {
+        let mut parameters = vec![];
+        if self.eat_symbol('(') {
+            loop {
+                parameters.push(self.integer()?);
+                if !self.eat_symbol(',') {
+                    break;
+                }
+            }
+            self.expect_symbol(')')?;
+        }
+        Ok(parameters)
+    }
+
+    fn data_type(&mut self) -> Result<DataType> {
+        let Some(word) = self.eat_map(|token| match token {
+            Token::Word(word) => Some(word.to_ascii_lowercase()),
+            _ => None,
+        }) else {
+            return Err(self.expected("a type"));
+        };
+        let data_type = match word.as_str() {
+            "array" => {
+                self.expect_symbol('<')?;
+                let element = self.data_type()?;
+                self.expect_symbol('>')?;
+                DataType::List(Arc::new(Field::new_list_field(element, true)))
+            }
+            "map" => {
+                self.expect_symbol('<')?;
+                let key = self.data_type()?;
+                self.expect_symbol(',')?;
+                let value = self.data_type()?;
+                self.expect_symbol('>')?;
+                // The field names Spark's `ArrowUtils` uses for a map.
+                let entries = Field::new_struct(
+                    "entries",
+                    vec![
+                        Field::new("key", key, false),
+                        Field::new("value", value, true),
+                    ],
+                    false,
+                );
+                DataType::Map(Arc::new(entries), false)
+            }
+            "struct" => DataType::Struct(self.struct_fields()?),
+            "boolean" => DataType::Boolean,
+            "tinyint" | "byte" => DataType::Int8,
+            "smallint" | "short" => DataType::Int16,
+            "int" | "integer" => DataType::Int32,
+            "bigint" | "long" => DataType::Int64,
+            "float" | "real" => DataType::Float32,
+            "double" => DataType::Float64,
+            "date" => DataType::Date32,
+            "binary" => DataType::Binary,
+            "timestamp_ltz" => self.timestamp_ltz(),
+            "timestamp_ntz" => TIMESTAMP_NTZ,
+            "timestamp" => {
+                if self.eat_keyword("with") {
+                    for keyword in ["local", "time", "zone"] {
+                        self.expect_keyword(keyword)?;
+                    }
+                    self.timestamp_ltz()
+                } else if self.eat_keyword("without") {
+                    for keyword in ["time", "zone"] {
+                        self.expect_keyword(keyword)?;
+                    }
+                    TIMESTAMP_NTZ
+                } else {
+                    self.timestamp_ltz()
+                }
+            }
+            "string" => DataType::Utf8,
+            "char" | "character" | "varchar" => {

Review Comment:
   Spark requires a length for `CHAR` and `VARCHAR` 
([`DataTypeAstBuilder.scala`](https://github.com/apache/spark/blob/32f7299601108917fb01920a54e084595b7b3bf8/sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/DataTypeAstBuilder.scala#L306-L311)).
 In Spark 4.2.0, `CAST(1 AS CHAR)` fails with `[DATATYPE_MISSING_SIZE] DataType 
"CHAR" requires a length parameter`, and `VARCHAR` fails the same way. Here 
`spark_cast(NULL, 'char')` returns `Utf8`, because `parameters()` returns an 
empty list and only more than one length is rejected. Could this require 
exactly one length?



##########
crates/sqllogictest/test_files/conversion/cast_float_to_timestamp.slt:
##########
@@ -189,15 +192,15 @@ statement ok
 SET datafusion.execution.enable_ansi_mode = true;
 
 # NaN should error in ANSI mode
-statement error
+statement error \[CAST_INVALID_INPUT\] The value 'NaN' of the type "DOUBLE" 
cannot be cast to "TIMESTAMP" because it is malformed
 SELECT spark_cast(arrow_cast('NaN', 'Float64'), 'timestamp');
 
 # Infinity should error in ANSI mode
-statement error
+statement error \[CAST_INVALID_INPUT\] The value 'Infinity' of the type 
"DOUBLE" cannot be cast to "TIMESTAMP"

Review Comment:
   Spark 4.2.0 doesn't quote a double in this message. With ANSI mode on, 
`CAST(CAST('NaN' AS DOUBLE) AS TIMESTAMP)` fails with:
   
   ```
   [CAST_INVALID_INPUT] The value NaN of the type "DOUBLE" cannot be cast to 
"TIMESTAMP" because it is malformed. Correct the value as per the syntax, or 
change its target type. Use `try_cast` to tolerate malformed input and return 
NULL instead.
   ```
   
   `Infinity` is unquoted the same way. The PySpark runner doesn't compare 
error text, so it passes on these lines. The quotes come from `SparkError`'s 
format string in #304 (see my comment on `crates/spark/src/error.rs` there). 
Once that changes, could these two expectations drop the quotes?



##########
crates/spark/src/function/conversion/cast/mod.rs:
##########
@@ -0,0 +1,664 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+//! Spark-compatible `CAST`.
+//!
+//! [`spark_cast`] is the kernel, for engines that build physical plans
+//! themselves, such as Comet. The `spark_cast(expr, '<type>')` SQL function is
+//! built on it.
+//!
+//! The kernel is being ported from Comet's
+//! `native/spark-expr/src/conversion_funcs`, one source type family at a time
+//! (<https://github.com/apache/datafusion-spark/issues/88>). A cast with no
+//! Spark-compatible implementation yet returns a "not implemented" error, 
rather
+//! than falling through to Arrow's cast and producing a result that differs
+//! from Spark's.
+
+mod data_type;
+mod numeric;
+mod udf;
+mod utils;
+
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, AsArray};
+use arrow::compute::{CastOptions, cast_with_options};
+use arrow::datatypes::{DataType, Int64Type, TimeUnit, 
TimestampMicrosecondType};
+use arrow::temporal_conversions::MICROSECONDS;
+use arrow::util::display::FormatOptions;
+use datafusion_common::{
+    Result, ScalarValue, exec_datafusion_err, internal_err, not_impl_err,
+};
+use datafusion_expr::ColumnarValue;
+
+use crate::function::timezone_utils::{local_to_timestamp, timestamp_to_local};
+use crate::{EvalMode, SparkTimeZone};
+use numeric::{cast_float_to_timestamp, cast_int_to_timestamp};
+use utils::remove_trailing_zeroes;
+
+pub(crate) use data_type::parse_spark_type;
+pub use udf::SparkCast;
+
+/// The format Spark uses for a timestamp cast to a string. `%.f` writes the
+/// fraction of a second only when it is not zero, padded to 3, 6 or 9 digits,
+/// and [`remove_trailing_zeroes`] then drops the padding.
+const TIMESTAMP_FORMAT: Option<&str> = Some("%Y-%m-%d %H:%M:%S%.f");
+
+/// Options for [`spark_cast`].
+///
+/// More options, such as the target Spark version, are added as the casts that
+/// depend on them are ported.
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+#[non_exhaustive]
+pub struct SparkCastOptions {
+    /// How a value that cannot be cast is handled.
+    pub eval_mode: EvalMode,
+    /// The session time zone (`spark.sql.session.timeZone`). A cast from
+    /// `TIMESTAMP` to `TIMESTAMP_NTZ`, `STRING` or `DATE`, or from
+    /// `TIMESTAMP_NTZ` to `TIMESTAMP`, returns an error without one.
+    pub timezone: Option<SparkTimeZone>,
+    /// Why `timezone` is missing, for the error of a cast that needs it.
+    missing_timezone_reason: Option<String>,
+}
+
+impl SparkCastOptions {
+    /// Options with no session time zone.
+    pub fn new(eval_mode: EvalMode) -> Self {
+        Self {
+            eval_mode,
+            timezone: None,
+            missing_timezone_reason: None,
+        }
+    }
+
+    pub fn with_timezone(self, timezone: SparkTimeZone) -> Self {
+        Self {
+            timezone: Some(timezone),
+            missing_timezone_reason: None,
+            ..self
+        }
+    }
+
+    /// Options with no session time zone because the configured one could not
+    /// be used, for `reason`.
+    pub(crate) fn without_timezone_because(self, reason: String) -> Self {
+        Self {
+            timezone: None,
+            missing_timezone_reason: Some(reason),
+            ..self
+        }
+    }
+
+    fn session_timezone(&self) -> Result<&SparkTimeZone> {
+        self.timezone
+            .as_ref()
+            .ok_or_else(|| match &self.missing_timezone_reason {
+                Some(reason) => exec_datafusion_err!(
+                    "Casting a timestamp requires a session time zone: 
{reason}"
+                ),
+                None => exec_datafusion_err!(
+                    "Casting a timestamp requires a session time zone"
+                ),
+            })
+    }
+}
+
+/// Casts `arg` to `data_type` the way Spark's `Cast` expression does.
+///
+/// An error is returned for a cast that Spark does not support, or that is not
+/// implemented yet. Spark can never name a dictionary type, so a dictionary
+/// `data_type` is an internal error.
+pub fn spark_cast(
+    arg: ColumnarValue,
+    data_type: &DataType,
+    cast_options: &SparkCastOptions,
+) -> Result<ColumnarValue> {
+    match arg {
+        ColumnarValue::Array(array) => Ok(ColumnarValue::Array(cast_array(
+            array,
+            data_type,
+            cast_options,
+        )?)),
+        ColumnarValue::Scalar(scalar) => {
+            let array = cast_array(scalar.to_array()?, data_type, 
cast_options)?;
+            Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+                &array, 0,
+            )?))
+        }
+    }
+}
+
+pub(crate) fn cast_array(
+    array: ArrayRef,
+    to_type: &DataType,
+    cast_options: &SparkCastOptions,
+) -> Result<ArrayRef> {
+    use DataType::*;
+    use EvalMode::*;
+    let from_type = array.data_type();
+
+    // Spark's SQL type grammar cannot express a dictionary as a cast target:
+    // 
https://github.com/apache/spark/blob/v4.2.0/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4#L1477-L1525
+    if matches!(to_type, Dictionary(_, _)) {
+        return internal_err!("Spark cannot specify dictionary types as cast 
targets");
+    }
+
+    if from_type == to_type {
+        return Ok(array);
+    }
+
+    // Spark's TIMESTAMP and TIMESTAMP_NTZ are microseconds since the epoch, 
and
+    // the timestamp casts below assume that unit.
+    let is_other_timestamp_unit =
+        |t: &DataType| matches!(t, Timestamp(unit, _) if *unit != 
TimeUnit::Microsecond);
+    if is_other_timestamp_unit(from_type) || is_other_timestamp_unit(to_type) {
+        return not_supported(from_type, to_type);
+    }
+
+    let eval_mode = cast_options.eval_mode;
+    let native_cast_options = CastOptions {
+        // Arrow returns NULL for a value it cannot cast when `safe` is set, 
and
+        // an error otherwise.
+        safe: eval_mode != Ansi,
+        format_options: 
FormatOptions::new().with_timestamp_format(TIMESTAMP_FORMAT),
+    };
+    let arrow_cast =
+        |array: &dyn Array| Ok(cast_with_options(array, to_type, 
&native_cast_options)?);
+
+    // Engines such as Comet can read a string or binary Parquet column as a
+    // dictionary, although Spark sees its value type. Cast the dictionary 
values
+    // and unpack the result.
+    //
+    // TODO: this casts values that no row refers to, so once a cast from 
string
+    // can fail, ANSI mode could raise an error that Spark would not.
+    if let Dictionary(_, value_type) = from_type
+        && matches!(value_type.as_ref(), Utf8 | LargeUtf8 | Binary | 
LargeBinary)
+    {
+        let dictionary = array.as_any_dictionary();
+        let values = cast_array(Arc::clone(dictionary.values()), to_type, 
cast_options)?;
+        return arrow_cast(&dictionary.with_values(values));
+    }
+
+    match (from_type, to_type) {
+        // A NULL array has no values, so Arrow's cast only changes the type.
+        (Null, _) => arrow_cast(&array),
+
+        // Numbers are seconds since the epoch. Spark has no cast from a number
+        // to TIMESTAMP_NTZ.
+        (Int8 | Int16 | Int32 | Int64, Timestamp(_, tz @ Some(_))) => {
+            cast_int_to_timestamp(&array, tz)
+        }
+        (Float32 | Float64, Timestamp(_, tz @ Some(_))) => {
+            cast_float_to_timestamp(&array, tz, eval_mode)
+        }
+
+        // Timestamps. Arrow's casts ignore the session time zone.
+        (Timestamp(_, Some(_)), Int64) => {
+            // Spark returns whole seconds, rounded down.
+            let micros = array.as_primitive::<TimestampMicrosecondType>();
+            Ok(Arc::new(
+                micros.unary::<_, Int64Type>(|v| v.div_euclid(MICROSECONDS)),
+            ))
+        }
+        (Timestamp(_, Some(_)), Utf8 | Date32 | Timestamp(_, None)) => {
+            let local = timestamp_to_local(
+                array.as_primitive(),
+                cast_options.session_timezone()?,
+            )?;
+            match to_type {
+                Utf8 => Ok(remove_trailing_zeroes(&arrow_cast(&local)?)),
+                Date32 => arrow_cast(&local),
+                _ => Ok(Arc::new(local)),
+            }
+        }
+        (Timestamp(_, None), Utf8) => 
Ok(remove_trailing_zeroes(&arrow_cast(&array)?)),
+        (Timestamp(_, None), Timestamp(_, Some(target_tz))) => {
+            let utc = local_to_timestamp(
+                array.as_primitive(),
+                cast_options.session_timezone()?,
+            )?;
+            Ok(Arc::new(utc.with_timezone(Arc::clone(target_tz))))
+        }
+
+        // Casts that Arrow supports, but where Spark's result differs, and 
that
+        // have not been ported from Comet yet. Each is replaced by its
+        // implementation as its family is ported.
+        //
+        // LEGACY wraps on overflow and ANSI raises CAST_OVERFLOW, while Arrow
+        // returns NULL. TRY matches Arrow.
+        (Int64, Int32 | Int16 | Int8) | (Int32, Int16 | Int8) | (Int16, Int8)
+            if eval_mode != Try =>
+        {
+            not_yet_ported(from_type, to_type, eval_mode)
+        }
+        (Float32 | Float64 | Decimal128(_, _), Int8 | Int16 | Int32 | Int64)
+            if eval_mode != Try =>
+        {
+            not_yet_ported(from_type, to_type, eval_mode)
+        }
+        // ANSI raises NUMERIC_VALUE_OUT_OF_RANGE, with Spark's message.
+        (Decimal128(_, _), Decimal128(_, _)) if eval_mode == Ansi => {
+            not_yet_ported(from_type, to_type, eval_mode)
+        }
+        // LEGACY uses Java's BigDecimal.toString, as in "0E-18".
+        (Decimal128(_, _), Utf8) if eval_mode == Legacy => {
+            not_yet_ported(from_type, to_type, eval_mode)
+        }
+        // Spark replaces invalid UTF-8, and Arrow raises an error.
+        (Binary, Utf8) => not_yet_ported(from_type, to_type, eval_mode),
+        // Casts Spark supports that Arrow does not do the same way.
+        (
+            Utf8 | LargeUtf8 | Utf8View,
+            Boolean
+            | Int8
+            | Int16
+            | Int32
+            | Int64
+            | Float32
+            | Float64
+            | Decimal128(_, _)
+            | Date32
+            | Timestamp(_, _),
+        )
+        | (Boolean | Decimal128(_, _), Timestamp(_, Some(_)))
+        | (Boolean | Int8 | Int16 | Int32 | Int64, Decimal128(_, _))
+        | (Int8 | Int16 | Int32 | Int64, Binary)
+        | (Float32 | Float64, Decimal128(_, _) | Utf8)
+        | (Decimal128(_, _), Boolean | Float32 | Float64)
+        | (Date32, Timestamp(_, _))
+        | (Struct(_) | List(_) | Map(_, _), Utf8)
+        | (Struct(_), Struct(_))
+        | (List(_), List(_))
+        | (Map(_, _), Map(_, _)) => not_yet_ported(from_type, to_type, 
eval_mode),

Review Comment:
   Several casts that Spark supports aren't in this arm, so they fall through 
to `not_supported`. The error then says Spark has no such cast. Spark 4.2.0 
evaluates `CAST(TIMESTAMP'2020-01-01 00:00:00' AS INT)`, `AS SMALLINT`, `AS 
DOUBLE`, and `AS DECIMAL(20,2)` in both modes, and `AS BOOLEAN` with ANSI mode 
off. With ANSI mode off, every cast from DATE to a numeric type or BOOLEAN 
returns NULL. Here `spark_cast(spark_cast(1, 'timestamp'), 'int')` fails with 
`Spark-compatible cast from Timestamp(µs, "UTC") to Int32 is not supported`. 
Could we add these pairs to this arm?
   
   ```rust
   | (Timestamp(_, Some(_)), Boolean | Int8 | Int16 | Int32 | Float32 | Float64 
| Decimal128(_, _))
   | (Date32, Boolean | Int8 | Int16 | Int64 | Float32 | Float64 | 
Decimal128(_, _))
   ```
   
   `(Date32, Int32)` belongs here as well. My #304 review raises that pair in 
the allowlist.



##########
crates/spark/src/function/conversion/cast/udf.rs:
##########
@@ -0,0 +1,299 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion_common::config::ConfigOptions;
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, internal_err, plan_err};
+use datafusion_expr::{
+    ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, 
ScalarUDFImpl,
+    Signature, Volatility,
+};
+
+use super::{SparkCastOptions, parse_spark_type, spark_cast};
+use crate::{EvalMode, SparkTimeZone};
+
+/// The time zone DataFusion uses when `datafusion.execution.time_zone` is not
+/// set.
+const DEFAULT_TIMEZONE: &str = "UTC";
+
+/// `spark_cast(expr, '<type>')`: Spark's `CAST(expr AS <type>)`.
+///
+/// The target type is a string literal in Spark's SQL type syntax, such as
+/// `'int'`, `'decimal(10, 2)'` or `'array<timestamp>'`. The cast follows
+/// `datafusion.execution.enable_ansi_mode`, as Spark's follows
+/// `spark.sql.ansi.enabled`, and uses `datafusion.execution.time_zone` as the
+/// session time zone.
+///
+/// # References
+/// - <https://spark.apache.org/docs/latest/api/sql/index.html#cast>
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkCast {

Review Comment:
   `spark_cast` has no `#[user_doc]`, and 
[`CONTRIBUTING.md`](https://github.com/apache/datafusion-spark/blob/125600164aa825e2fe805887dfdb384d4182f2a8/CONTRIBUTING.md?plain=1#L41-L44)
 asks for one on every function. Should we add it here, since this PR defines 
the target-type syntax and the casts that return "not implemented yet"? Without 
it, the function is missing from the reference that #115 generates.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to