andygrove commented on code in PR #304:
URL: https://github.com/apache/datafusion-spark/pull/304#discussion_r4603705404


##########
crates/spark/src/error.rs:
##########
@@ -0,0 +1,243 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+//! Typed Spark errors.
+//!
+//! A [`SparkError`] renders the message Spark raises for the same condition,
+//! including its error class (such as `[CAST_OVERFLOW]`). It reaches callers 
as
+//! [`DataFusionError::External`], so an engine that maps errors back to Spark
+//! exceptions, such as Comet, can recover the variant with
+//! [`SparkError::find`] instead of parsing the message.
+
+use std::error::Error;
+use std::fmt::{Display, Formatter};
+
+use datafusion_common::DataFusionError;
+
+/// An error that Spark raises with a specific error class.
+///
+/// The messages follow Spark 4.x's
+/// 
[`error-conditions.json`](https://github.com/apache/spark/blob/v4.2.0/common/utils/src/main/resources/error/error-conditions.json).
+/// Spark 3.x appended `If necessary set "spark.sql.ansi.enabled" to "false" to
+/// bypass this error.` to the `CAST_INVALID_INPUT` and `CAST_OVERFLOW`
+/// messages, and 4.0 dropped it.
+/// Spark quotes values and types in the forms its `toSQLValue` and `toSQLType`
+/// produce. The caller passes them already formatted, such as `2147483648L` 
and
+/// `"BIGINT"` without the quotes, since only the caller knows the Spark type.
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum SparkError {

Review Comment:
   Done in ba24438. Each variant is now `#[non_exhaustive]`.



##########
crates/spark/src/error.rs:
##########
@@ -0,0 +1,243 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+//! Typed Spark errors.
+//!
+//! A [`SparkError`] renders the message Spark raises for the same condition,
+//! including its error class (such as `[CAST_OVERFLOW]`). It reaches callers 
as
+//! [`DataFusionError::External`], so an engine that maps errors back to Spark
+//! exceptions, such as Comet, can recover the variant with
+//! [`SparkError::find`] instead of parsing the message.
+
+use std::error::Error;
+use std::fmt::{Display, Formatter};
+
+use datafusion_common::DataFusionError;
+
+/// An error that Spark raises with a specific error class.
+///
+/// The messages follow Spark 4.x's
+/// 
[`error-conditions.json`](https://github.com/apache/spark/blob/v4.2.0/common/utils/src/main/resources/error/error-conditions.json).
+/// Spark 3.x appended `If necessary set "spark.sql.ansi.enabled" to "false" to
+/// bypass this error.` to the `CAST_INVALID_INPUT` and `CAST_OVERFLOW`
+/// messages, and 4.0 dropped it.
+/// Spark quotes values and types in the forms its `toSQLValue` and `toSQLType`
+/// produce. The caller passes them already formatted, such as `2147483648L` 
and
+/// `"BIGINT"` without the quotes, since only the caller knows the Spark type.
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum SparkError {
+    /// `CAST_INVALID_INPUT`: a value that cannot be parsed as the target type,
+    /// in ANSI mode. Spark raises it as a `SparkNumberFormatException`.
+    CastInvalidValue {
+        value: String,
+        from_type: String,
+        to_type: String,
+    },
+    /// The same error class and message as [`SparkError::CastInvalidValue`],
+    /// for a cast to a date or timestamp. Spark raises it as a
+    /// `SparkDateTimeException` rather than a `SparkNumberFormatException`.
+    InvalidInputInCastToDatetime {
+        value: String,
+        from_type: String,
+        to_type: String,
+    },
+    /// `CAST_OVERFLOW`: a value outside the range of the target type, in ANSI
+    /// mode.
+    CastOverflow {
+        value: String,
+        from_type: String,
+        to_type: String,
+    },
+    /// `NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION`: a value that does not fit
+    /// the target decimal precision and scale, in ANSI mode.
+    NumericValueOutOfRange {
+        value: String,
+        precision: u8,
+        scale: i8,
+    },
+    /// `NUMERIC_OUT_OF_SUPPORTED_RANGE`: a string with more than 38 digits 
cast
+    /// to a decimal.
+    NumericOutOfSupportedRange { value: String },
+}
+
+impl SparkError {
+    /// The Spark error class, such as `CAST_OVERFLOW`.
+    pub fn error_class(&self) -> &'static str {
+        match self {
+            Self::CastInvalidValue { .. } | Self::InvalidInputInCastToDatetime 
{ .. } => {
+                "CAST_INVALID_INPUT"
+            }
+            Self::CastOverflow { .. } => "CAST_OVERFLOW",
+            Self::NumericValueOutOfRange { .. } => {
+                "NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION"
+            }
+            Self::NumericOutOfSupportedRange { .. } => 
"NUMERIC_OUT_OF_SUPPORTED_RANGE",
+        }
+    }
+
+    /// Finds a `SparkError` in `error`, looking through the
+    /// [`DataFusionError::Context`], [`DataFusionError::Shared`] and
+    /// [`DataFusionError::ArrowError`] wrappers that DataFusion and Arrow
+    /// kernels add around an external error.
+    pub fn find(error: &DataFusionError) -> Option<&SparkError> {
+        match error {
+            DataFusionError::External(e) => e.downcast_ref::<SparkError>(),
+            DataFusionError::ArrowError(e, _) => match e.as_ref() {
+                arrow::error::ArrowError::ExternalError(e) => e.downcast_ref(),
+                _ => None,
+            },
+            DataFusionError::Context(_, e) => Self::find(e),
+            DataFusionError::Shared(e) => Self::find(e),
+            _ => None,
+        }
+    }
+}
+
+impl Display for SparkError {

Review Comment:
   Done in ba24438. `sql_state()` returns 22018 for `CAST_INVALID_INPUT` and 
22003 for the other three, matching `error-conditions.json`, and every message 
ends with ` SQLSTATE: <code>`.



##########
crates/spark/src/error.rs:
##########
@@ -0,0 +1,243 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+//! Typed Spark errors.
+//!
+//! A [`SparkError`] renders the message Spark raises for the same condition,
+//! including its error class (such as `[CAST_OVERFLOW]`). It reaches callers 
as
+//! [`DataFusionError::External`], so an engine that maps errors back to Spark
+//! exceptions, such as Comet, can recover the variant with
+//! [`SparkError::find`] instead of parsing the message.
+
+use std::error::Error;
+use std::fmt::{Display, Formatter};
+
+use datafusion_common::DataFusionError;
+
+/// An error that Spark raises with a specific error class.
+///
+/// The messages follow Spark 4.x's
+/// 
[`error-conditions.json`](https://github.com/apache/spark/blob/v4.2.0/common/utils/src/main/resources/error/error-conditions.json).
+/// Spark 3.x appended `If necessary set "spark.sql.ansi.enabled" to "false" to
+/// bypass this error.` to the `CAST_INVALID_INPUT` and `CAST_OVERFLOW`
+/// messages, and 4.0 dropped it.
+/// Spark quotes values and types in the forms its `toSQLValue` and `toSQLType`
+/// produce. The caller passes them already formatted, such as `2147483648L` 
and
+/// `"BIGINT"` without the quotes, since only the caller knows the Spark type.
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum SparkError {
+    /// `CAST_INVALID_INPUT`: a value that cannot be parsed as the target type,
+    /// in ANSI mode. Spark raises it as a `SparkNumberFormatException`.
+    CastInvalidValue {
+        value: String,
+        from_type: String,
+        to_type: String,
+    },
+    /// The same error class and message as [`SparkError::CastInvalidValue`],
+    /// for a cast to a date or timestamp. Spark raises it as a
+    /// `SparkDateTimeException` rather than a `SparkNumberFormatException`.
+    InvalidInputInCastToDatetime {
+        value: String,
+        from_type: String,
+        to_type: String,
+    },
+    /// `CAST_OVERFLOW`: a value outside the range of the target type, in ANSI
+    /// mode.
+    CastOverflow {
+        value: String,
+        from_type: String,
+        to_type: String,
+    },
+    /// `NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION`: a value that does not fit
+    /// the target decimal precision and scale, in ANSI mode.
+    NumericValueOutOfRange {
+        value: String,
+        precision: u8,
+        scale: i8,
+    },
+    /// `NUMERIC_OUT_OF_SUPPORTED_RANGE`: a string with more than 38 digits 
cast
+    /// to a decimal.
+    NumericOutOfSupportedRange { value: String },
+}
+
+impl SparkError {
+    /// The Spark error class, such as `CAST_OVERFLOW`.
+    pub fn error_class(&self) -> &'static str {
+        match self {
+            Self::CastInvalidValue { .. } | Self::InvalidInputInCastToDatetime 
{ .. } => {
+                "CAST_INVALID_INPUT"
+            }
+            Self::CastOverflow { .. } => "CAST_OVERFLOW",
+            Self::NumericValueOutOfRange { .. } => {
+                "NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION"
+            }
+            Self::NumericOutOfSupportedRange { .. } => 
"NUMERIC_OUT_OF_SUPPORTED_RANGE",
+        }
+    }
+
+    /// Finds a `SparkError` in `error`, looking through the
+    /// [`DataFusionError::Context`], [`DataFusionError::Shared`] and
+    /// [`DataFusionError::ArrowError`] wrappers that DataFusion and Arrow
+    /// kernels add around an external error.
+    pub fn find(error: &DataFusionError) -> Option<&SparkError> {
+        match error {
+            DataFusionError::External(e) => e.downcast_ref::<SparkError>(),
+            DataFusionError::ArrowError(e, _) => match e.as_ref() {
+                arrow::error::ArrowError::ExternalError(e) => e.downcast_ref(),
+                _ => None,
+            },
+            DataFusionError::Context(_, e) => Self::find(e),
+            DataFusionError::Shared(e) => Self::find(e),
+            _ => None,
+        }
+    }
+}
+
+impl Display for SparkError {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let class = self.error_class();
+        match self {
+            Self::CastInvalidValue {
+                value,
+                from_type,
+                to_type,
+            }
+            | Self::InvalidInputInCastToDatetime {
+                value,
+                from_type,
+                to_type,
+            } => write!(
+                f,
+                "[{class}] The value '{value}' of the type \"{from_type}\" 
cannot be cast to \

Review Comment:
   Done in ba24438. The format string no longer quotes the value. 
`SparkError::sql_string` does what `toSQLValue` does for strings (so `a'b` 
renders as `'a\'b'`), and #317 passes doubles such as `NaN` unquoted. A test 
covers both.



##########
crates/spark/src/error.rs:
##########
@@ -0,0 +1,243 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+//! Typed Spark errors.
+//!
+//! A [`SparkError`] renders the message Spark raises for the same condition,
+//! including its error class (such as `[CAST_OVERFLOW]`). It reaches callers 
as
+//! [`DataFusionError::External`], so an engine that maps errors back to Spark
+//! exceptions, such as Comet, can recover the variant with
+//! [`SparkError::find`] instead of parsing the message.
+
+use std::error::Error;
+use std::fmt::{Display, Formatter};
+
+use datafusion_common::DataFusionError;
+
+/// An error that Spark raises with a specific error class.
+///
+/// The messages follow Spark 4.x's
+/// 
[`error-conditions.json`](https://github.com/apache/spark/blob/v4.2.0/common/utils/src/main/resources/error/error-conditions.json).
+/// Spark 3.x appended `If necessary set "spark.sql.ansi.enabled" to "false" to
+/// bypass this error.` to the `CAST_INVALID_INPUT` and `CAST_OVERFLOW`
+/// messages, and 4.0 dropped it.
+/// Spark quotes values and types in the forms its `toSQLValue` and `toSQLType`
+/// produce. The caller passes them already formatted, such as `2147483648L` 
and
+/// `"BIGINT"` without the quotes, since only the caller knows the Spark type.
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum SparkError {
+    /// `CAST_INVALID_INPUT`: a value that cannot be parsed as the target type,
+    /// in ANSI mode. Spark raises it as a `SparkNumberFormatException`.
+    CastInvalidValue {
+        value: String,
+        from_type: String,
+        to_type: String,
+    },
+    /// The same error class and message as [`SparkError::CastInvalidValue`],
+    /// for a cast to a date or timestamp. Spark raises it as a
+    /// `SparkDateTimeException` rather than a `SparkNumberFormatException`.
+    InvalidInputInCastToDatetime {
+        value: String,
+        from_type: String,
+        to_type: String,
+    },
+    /// `CAST_OVERFLOW`: a value outside the range of the target type, in ANSI
+    /// mode.
+    CastOverflow {
+        value: String,
+        from_type: String,
+        to_type: String,
+    },
+    /// `NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION`: a value that does not fit
+    /// the target decimal precision and scale, in ANSI mode.
+    NumericValueOutOfRange {
+        value: String,
+        precision: u8,
+        scale: i8,
+    },
+    /// `NUMERIC_OUT_OF_SUPPORTED_RANGE`: a string with more than 38 digits 
cast
+    /// to a decimal.
+    NumericOutOfSupportedRange { value: String },
+}
+
+impl SparkError {
+    /// The Spark error class, such as `CAST_OVERFLOW`.
+    pub fn error_class(&self) -> &'static str {
+        match self {
+            Self::CastInvalidValue { .. } | Self::InvalidInputInCastToDatetime 
{ .. } => {
+                "CAST_INVALID_INPUT"
+            }
+            Self::CastOverflow { .. } => "CAST_OVERFLOW",
+            Self::NumericValueOutOfRange { .. } => {
+                "NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION"
+            }
+            Self::NumericOutOfSupportedRange { .. } => 
"NUMERIC_OUT_OF_SUPPORTED_RANGE",
+        }
+    }
+
+    /// Finds a `SparkError` in `error`, looking through the
+    /// [`DataFusionError::Context`], [`DataFusionError::Shared`] and
+    /// [`DataFusionError::ArrowError`] wrappers that DataFusion and Arrow
+    /// kernels add around an external error.
+    pub fn find(error: &DataFusionError) -> Option<&SparkError> {
+        match error {
+            DataFusionError::External(e) => e.downcast_ref::<SparkError>(),
+            DataFusionError::ArrowError(e, _) => match e.as_ref() {
+                arrow::error::ArrowError::ExternalError(e) => e.downcast_ref(),
+                _ => None,
+            },
+            DataFusionError::Context(_, e) => Self::find(e),
+            DataFusionError::Shared(e) => Self::find(e),
+            _ => None,
+        }
+    }
+}
+
+impl Display for SparkError {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let class = self.error_class();
+        match self {
+            Self::CastInvalidValue {
+                value,
+                from_type,
+                to_type,
+            }
+            | Self::InvalidInputInCastToDatetime {
+                value,
+                from_type,
+                to_type,
+            } => write!(
+                f,
+                "[{class}] The value '{value}' of the type \"{from_type}\" 
cannot be cast to \
+                 \"{to_type}\" because it is malformed. Correct the value as 
per the syntax, or \
+                 change its target type. Use `try_cast` to tolerate malformed 
input and return \
+                 NULL instead."
+            ),
+            Self::CastOverflow {
+                value,
+                from_type,
+                to_type,
+            } => write!(
+                f,
+                "[{class}] The value {value} of the type \"{from_type}\" 
cannot be cast to \
+                 \"{to_type}\" due to an overflow. Use `try_cast` to tolerate 
overflow and return \
+                 NULL instead."
+            ),
+            Self::NumericValueOutOfRange {
+                value,
+                precision,
+                scale,
+            } => write!(
+                f,
+                "[{class}] {value} cannot be represented as 
Decimal({precision}, {scale}). If \

Review Comment:
   Applied in ba24438, with the test assertion updated.



##########
crates/spark/src/function/conversion/cast/mod.rs:
##########
@@ -0,0 +1,596 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+//! Spark-compatible `CAST`.
+//!
+//! [`spark_cast`] is the kernel, for engines that build physical plans
+//! themselves, such as Comet. The `spark_cast(expr, '<type>')` SQL function is
+//! built on it.
+//!
+//! The kernel is being ported from Comet's
+//! `native/spark-expr/src/conversion_funcs`, one source type family at a time
+//! (<https://github.com/apache/datafusion-spark/issues/88>). A cast with no
+//! Spark-compatible implementation yet returns a "not implemented" error, 
rather
+//! than falling through to Arrow's cast and producing a result that differs
+//! from Spark's.
+
+mod udf;
+mod utils;
+
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, AsArray};
+use arrow::compute::{CastOptions, cast_with_options};
+use arrow::datatypes::{DataType, Int64Type, TimeUnit, 
TimestampMicrosecondType};
+use arrow::temporal_conversions::MICROSECONDS;
+use arrow::util::display::FormatOptions;
+use datafusion_common::{
+    Result, ScalarValue, exec_datafusion_err, internal_err, not_impl_err,
+};
+use datafusion_expr::ColumnarValue;
+
+use crate::function::timezone_utils::{local_to_timestamp, timestamp_to_local};
+use crate::{EvalMode, SparkTimeZone};
+use utils::remove_trailing_zeroes;
+
+pub use udf::SparkCast;
+
+/// The format Spark uses for a timestamp cast to a string. `%.f` writes the
+/// fraction of a second only when it is not zero, padded to 3, 6 or 9 digits,
+/// and [`remove_trailing_zeroes`] then drops the padding.
+const TIMESTAMP_FORMAT: Option<&str> = Some("%Y-%m-%d %H:%M:%S%.f");
+
+/// Options for [`spark_cast`].
+///
+/// More options, such as the target Spark version, are added as the casts that
+/// depend on them are ported.
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+#[non_exhaustive]
+pub struct SparkCastOptions {
+    /// How a value that cannot be cast is handled.
+    pub eval_mode: EvalMode,
+    /// The session time zone (`spark.sql.session.timeZone`). A cast from
+    /// `TIMESTAMP` to `TIMESTAMP_NTZ`, `STRING` or `DATE`, or from
+    /// `TIMESTAMP_NTZ` to `TIMESTAMP`, returns an error without one.
+    pub timezone: Option<SparkTimeZone>,
+}
+
+impl SparkCastOptions {
+    /// Options with no session time zone.
+    pub fn new(eval_mode: EvalMode) -> Self {
+        Self {
+            eval_mode,
+            timezone: None,
+        }
+    }
+
+    pub fn with_timezone(self, timezone: SparkTimeZone) -> Self {
+        Self {
+            timezone: Some(timezone),
+            ..self
+        }
+    }
+
+    fn session_timezone(&self) -> Result<&SparkTimeZone> {
+        self.timezone.as_ref().ok_or_else(|| {
+            exec_datafusion_err!("Casting a timestamp requires a session time 
zone")
+        })
+    }
+}
+
+/// Casts `arg` to `data_type` the way Spark's `Cast` expression does.
+///
+/// An error is returned for a cast that Spark does not support, or that is not
+/// implemented yet. Spark can never name a dictionary type, so a dictionary
+/// `data_type` is an internal error.
+pub fn spark_cast(
+    arg: ColumnarValue,
+    data_type: &DataType,
+    cast_options: &SparkCastOptions,
+) -> Result<ColumnarValue> {
+    match arg {
+        ColumnarValue::Array(array) => Ok(ColumnarValue::Array(cast_array(
+            array,
+            data_type,
+            cast_options,
+        )?)),
+        ColumnarValue::Scalar(scalar) => {
+            let array = cast_array(scalar.to_array()?, data_type, 
cast_options)?;
+            Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+                &array, 0,
+            )?))
+        }
+    }
+}
+
+pub(crate) fn cast_array(
+    array: ArrayRef,
+    to_type: &DataType,
+    cast_options: &SparkCastOptions,
+) -> Result<ArrayRef> {
+    use DataType::*;
+    use EvalMode::*;
+    let from_type = array.data_type();
+
+    // Spark's SQL type grammar cannot express a dictionary as a cast target:
+    // 
https://github.com/apache/spark/blob/v4.2.0/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4#L1477-L1525
+    if matches!(to_type, Dictionary(_, _)) {
+        return internal_err!("Spark cannot specify dictionary types as cast 
targets");
+    }
+
+    if from_type == to_type {
+        return Ok(array);
+    }
+
+    // Spark's TIMESTAMP and TIMESTAMP_NTZ are microseconds since the epoch, 
and
+    // the timestamp casts below assume that unit.
+    let is_other_timestamp_unit =
+        |t: &DataType| matches!(t, Timestamp(unit, _) if *unit != 
TimeUnit::Microsecond);
+    if is_other_timestamp_unit(from_type) || is_other_timestamp_unit(to_type) {
+        return not_supported(from_type, to_type);
+    }
+
+    let eval_mode = cast_options.eval_mode;
+    let native_cast_options = CastOptions {
+        // Arrow returns NULL for a value it cannot cast when `safe` is set, 
and
+        // an error otherwise.
+        safe: eval_mode != Ansi,
+        format_options: 
FormatOptions::new().with_timestamp_format(TIMESTAMP_FORMAT),
+    };
+    let arrow_cast =
+        |array: &dyn Array| Ok(cast_with_options(array, to_type, 
&native_cast_options)?);
+
+    // Engines such as Comet can read a string or binary Parquet column as a
+    // dictionary, although Spark sees its value type. Cast the dictionary 
values
+    // and unpack the result.
+    //
+    // TODO: this casts values that no row refers to, so once a cast from 
string
+    // can fail, ANSI mode could raise an error that Spark would not.
+    if let Dictionary(_, value_type) = from_type
+        && matches!(value_type.as_ref(), Utf8 | LargeUtf8 | Binary | 
LargeBinary)
+    {
+        let dictionary = array.as_any_dictionary();
+        let values = cast_array(Arc::clone(dictionary.values()), to_type, 
cast_options)?;
+        return arrow_cast(&dictionary.with_values(values));
+    }
+
+    match (from_type, to_type) {
+        // A NULL array has no values, so Arrow's cast only changes the type.
+        (Null, _) => arrow_cast(&array),
+
+        // Timestamps. Arrow's casts ignore the session time zone.
+        (Timestamp(_, Some(_)), Int64) => {
+            // Spark returns whole seconds, rounded down.
+            let micros = array.as_primitive::<TimestampMicrosecondType>();
+            Ok(Arc::new(
+                micros.unary::<_, Int64Type>(|v| v.div_euclid(MICROSECONDS)),
+            ))
+        }
+        (Timestamp(_, Some(_)), Utf8 | Date32 | Timestamp(_, None)) => {
+            let local = timestamp_to_local(
+                array.as_primitive(),
+                cast_options.session_timezone()?,
+            )?;
+            match to_type {
+                Utf8 => Ok(remove_trailing_zeroes(&arrow_cast(&local)?)),
+                Date32 => arrow_cast(&local),
+                _ => Ok(Arc::new(local)),
+            }
+        }
+        (Timestamp(_, None), Utf8) => 
Ok(remove_trailing_zeroes(&arrow_cast(&array)?)),
+        (Timestamp(_, None), Timestamp(_, Some(target_tz))) => {
+            let utc = local_to_timestamp(
+                array.as_primitive(),
+                cast_options.session_timezone()?,
+            )?;
+            Ok(Arc::new(utc.with_timezone(Arc::clone(target_tz))))
+        }
+
+        // Casts that Arrow supports, but where Spark's result differs, and 
that
+        // have not been ported from Comet yet. Each is replaced by its
+        // implementation as its family is ported.
+        //
+        // LEGACY wraps on overflow and ANSI raises CAST_OVERFLOW, while Arrow
+        // returns NULL. TRY matches Arrow.
+        (Int64, Int32 | Int16 | Int8) | (Int32, Int16 | Int8) | (Int16, Int8)
+            if eval_mode != Try =>
+        {
+            not_yet_ported(from_type, to_type, eval_mode)
+        }
+        (Float32 | Float64 | Decimal128(_, _), Int8 | Int16 | Int32 | Int64)
+            if eval_mode != Try =>
+        {
+            not_yet_ported(from_type, to_type, eval_mode)
+        }
+        // ANSI raises NUMERIC_VALUE_OUT_OF_RANGE, with Spark's message.
+        (Decimal128(_, _), Decimal128(_, _)) if eval_mode == Ansi => {
+            not_yet_ported(from_type, to_type, eval_mode)
+        }
+        // LEGACY uses Java's BigDecimal.toString, as in "0E-18".
+        (Decimal128(_, _), Utf8) if eval_mode == Legacy => {
+            not_yet_ported(from_type, to_type, eval_mode)
+        }
+        // Spark replaces invalid UTF-8, and Arrow raises an error.
+        (Binary, Utf8) => not_yet_ported(from_type, to_type, eval_mode),
+
+        _ if is_datafusion_spark_compatible(from_type, to_type) => 
arrow_cast(&array),
+        _ => not_supported(from_type, to_type),
+    }
+}
+
+fn not_supported(from_type: &DataType, to_type: &DataType) -> Result<ArrayRef> 
{
+    not_impl_err!("Spark-compatible cast from {from_type} to {to_type} is not 
supported")
+}
+
+fn not_yet_ported(
+    from_type: &DataType,
+    to_type: &DataType,
+    eval_mode: EvalMode,
+) -> Result<ArrayRef> {
+    not_impl_err!(
+        "Spark-compatible cast from {from_type} to {to_type} in {eval_mode:?} 
mode is \
+         not implemented yet"
+    )
+}
+
+/// Whether Arrow's cast from `from_type` to `to_type` matches Spark's in every
+/// eval mode. [`cast_array`] handles the casts it needs to before consulting
+/// this.
+fn is_datafusion_spark_compatible(from_type: &DataType, to_type: &DataType) -> 
bool {
+    use DataType::*;
+    match from_type {
+        Boolean => matches!(
+            to_type,
+            Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Utf8
+        ),
+        Int8 | Int16 | Int32 | Int64 => matches!(
+            to_type,
+            Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Utf8
+        ),
+        Float32 | Float64 => matches!(
+            to_type,
+            Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64
+        ),
+        // Float32 and Float64 are absent: Arrow computes
+        // `(unscaled as f64) / 10^scale`, which rounds twice and can differ 
from
+        // Spark's correctly rounded BigDecimal.doubleValue by one ulp. 
Boolean is
+        // absent because Arrow does not implement that cast.
+        Decimal128(_, _) | Decimal256(_, _) => matches!(
+            to_type,
+            Int8 | Int16 | Int32 | Int64 | Decimal128(_, _) | Decimal256(_, _) 
| Utf8
+        ),
+        Utf8 => matches!(to_type, Binary),
+        // Arrow reinterprets the days since the epoch, as Spark does.
+        Date32 => matches!(to_type, Int32 | Utf8),

Review Comment:
   Done in ffc46a2. `(Date32, Int32)` is now a placeholder arm that returns 
"not implemented yet" in every mode, with a unit test. The "date to int" item 
in #88 can decide how Comet gets the day count.



##########
crates/spark/src/function/datetime/timezone.rs:
##########
@@ -105,6 +174,68 @@ impl SparkTimeZone {
             }
         }
     }
+
+    /// Converts `local`, a local date-time in this time zone encoded as if it
+    /// were UTC, to UTC, the way Java's `LocalDateTime.atZone` does. The
+    /// inverse of [`Self::adjust_to_local_time`].
+    ///
+    /// A local time that happens twice, when clocks go back, resolves to the
+    /// earlier offset. A local time in the gap when clocks go forward moves
+    /// forward by the length of the gap.
+    pub(crate) fn local_to_utc<T: ArrowTimestampType>(self, local: i64) -> 
Result<i64> {
+        let units = units_per_second::<T>();
+        let shift = |offset_seconds: i32| {
+            local
+                .checked_sub(i64::from(offset_seconds) * units)
+                .ok_or_else(|| exec_datafusion_err!("Timestamp {local} 
overflows"))
+        };
+        match self {
+            Self::FixedOffset(seconds) => shift(seconds),
+            Self::Region(tz) => {
+                let local_datetime =
+                    DateTime::<Utc>::from_timestamp(local.div_euclid(units), 0)
+                        .ok_or_else(|| {
+                            exec_datafusion_err!(
+                                "Timestamp is outside Chrono's calendar range"
+                            )
+                        })?
+                        .naive_utc();
+                let offset = region_offset_at_local(&tz, local_datetime);
+                shift(offset)
+            }
+            Self::SystemV(standard_offset) => {
+                // Use daylight time if the result is in daylight time. That 
picks
+                // the earlier offset in the overlap, and standard time, the
+                // offset before the transition, in the gap.
+                let daylight_offset = standard_offset + 3600;
+                let utc = shift(daylight_offset)?;
+                if system_v_offset::<T>(utc, standard_offset)? == 
daylight_offset {
+                    Ok(utc)
+                } else {
+                    shift(standard_offset)
+                }
+            }
+        }
+    }
+}
+
+/// The offset of `tz`, in seconds, at the local date-time `local`, resolved as
+/// Java's `LocalDateTime.atZone` does. The fraction of a second does not 
affect
+/// the offset, so `local` is in whole seconds.
+fn region_offset_at_local(tz: &Tz, local: NaiveDateTime) -> i32 {
+    let offset_of = |result: LocalResult<DateTime<Tz>>| match result {
+        LocalResult::Single(dt) | LocalResult::Ambiguous(dt, _) => {
+            Some(dt.offset().fix().local_minus_utc())
+        }
+        LocalResult::None => None,
+    };
+    offset_of(tz.from_local_datetime(&local))
+        // In a gap, apply the offset in effect before it, which moves the time
+        // forward by the length of the gap.
+        .or_else(|| {
+            offset_of(tz.from_local_datetime(&(local - 
chrono::Duration::hours(3))))
+        })
+        .unwrap_or(0)

Review Comment:
   Done in 21c490c, as suggested. Taking the offset from the UTC side 14 hours 
back can't fail, so there's no fallback to 0 any more. 
`local_times_in_a_gap_use_the_offset_before_it` covers your two cases (Apia and 
Kwajalein, with Spark's values) and an ordinary New York DST gap. It fails on 
the old code with 1325246400 for Apia.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to