andygrove commented on code in PR #304:
URL: https://github.com/apache/datafusion-spark/pull/304#discussion_r4607462675
##########
crates/spark/src/error.rs:
##########
@@ -0,0 +1,293 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! Typed Spark errors.
+//!
+//! A [`SparkError`] renders the message Spark raises for the same condition,
+//! including its error class (such as `[CAST_OVERFLOW]`). It reaches callers
as
+//! [`DataFusionError::External`], so an engine that maps errors back to Spark
+//! exceptions, such as Comet, can recover the variant with
+//! [`SparkError::find`] instead of parsing the message.
+
+use std::error::Error;
+use std::fmt::{Display, Formatter};
+
+use datafusion_common::DataFusionError;
+
+/// An error that Spark raises with a specific error class.
+///
+/// The messages follow Spark 4.x's
+///
[`error-conditions.json`](https://github.com/apache/spark/blob/v4.2.0/common/utils/src/main/resources/error/error-conditions.json).
+/// Spark 3.x appended `If necessary set "spark.sql.ansi.enabled" to "false" to
+/// bypass this error.` to the `CAST_INVALID_INPUT` and `CAST_OVERFLOW`
+/// messages, and 4.0 dropped it.
+/// Spark quotes values and types in the forms its `toSQLValue` and `toSQLType`
+/// produce. The caller passes them already formatted, such as `2147483648L`,
+/// `NaN`, or `'abc'` from [`SparkError::sql_string`], and `"BIGINT"` without
the
+/// quotes, since only the caller knows the Spark type. Every message ends with
+/// the error's SQLSTATE, as Spark's do.
+///
+/// The variants are `#[non_exhaustive]` so that fields can be added later.
+/// Code outside this crate matches them with `..`.
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum SparkError {
+ /// `CAST_INVALID_INPUT`: a value that cannot be parsed as the target type,
+ /// in ANSI mode. Spark raises it as a `SparkNumberFormatException`.
+ #[non_exhaustive]
+ CastInvalidValue {
+ value: String,
+ from_type: String,
+ to_type: String,
+ },
+ /// The same error class and message as [`SparkError::CastInvalidValue`],
+ /// for a cast to a date or timestamp. Spark raises it as a
+ /// `SparkDateTimeException` rather than a `SparkNumberFormatException`.
+ #[non_exhaustive]
+ InvalidInputInCastToDatetime {
+ value: String,
+ from_type: String,
+ to_type: String,
+ },
+ /// `CAST_OVERFLOW`: a value outside the range of the target type, in ANSI
+ /// mode.
+ #[non_exhaustive]
+ CastOverflow {
+ value: String,
+ from_type: String,
+ to_type: String,
+ },
+ /// `NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION`: a value that does not fit
+ /// the target decimal precision and scale, in ANSI mode.
+ #[non_exhaustive]
+ NumericValueOutOfRange {
+ value: String,
+ precision: u8,
+ scale: i8,
+ },
+ /// `NUMERIC_OUT_OF_SUPPORTED_RANGE`: a string with more than 38 digits
cast
+ /// to a decimal.
+ #[non_exhaustive]
+ NumericOutOfSupportedRange { value: String },
+}
+
+impl SparkError {
+ /// The Spark error class, such as `CAST_OVERFLOW`.
+ pub fn error_class(&self) -> &'static str {
+ match self {
+ Self::CastInvalidValue { .. } | Self::InvalidInputInCastToDatetime
{ .. } => {
+ "CAST_INVALID_INPUT"
+ }
+ Self::CastOverflow { .. } => "CAST_OVERFLOW",
+ Self::NumericValueOutOfRange { .. } => {
+ "NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION"
+ }
+ Self::NumericOutOfSupportedRange { .. } =>
"NUMERIC_OUT_OF_SUPPORTED_RANGE",
+ }
+ }
+
+ /// The SQLSTATE that Spark reports with the error class.
+ pub fn sql_state(&self) -> &'static str {
+ match self {
+ Self::CastInvalidValue { .. } | Self::InvalidInputInCastToDatetime
{ .. } => {
+ "22018"
+ }
+ Self::CastOverflow { .. }
+ | Self::NumericValueOutOfRange { .. }
+ | Self::NumericOutOfSupportedRange { .. } => "22003",
+ }
+ }
+
+ /// A string value as Spark's `toSQLValue` renders it in a message: quoted,
+ /// with backslashes and quotes escaped, so `a'b` becomes `'a\'b'`.
+ pub fn sql_string(value: &str) -> String {
+ format!("'{}'", value.replace('\\', "\\\\").replace('\'', "\\'"))
+ }
Review Comment:
Removed in 259b798. The tests pass the quoted literal directly now.
sql_string will come back as pub(crate) with the string cast port that calls it.
##########
crates/spark/src/function/datetime/timezone.rs:
##########
@@ -105,6 +174,68 @@ impl SparkTimeZone {
}
}
}
+
+ /// Converts `local`, a local date-time in this time zone encoded as if it
+ /// were UTC, to UTC, the way Java's `LocalDateTime.atZone` does. The
+ /// inverse of [`Self::adjust_to_local_time`].
+ ///
+ /// A local time that happens twice, when clocks go back, resolves to the
+ /// earlier offset. A local time in the gap when clocks go forward moves
+ /// forward by the length of the gap.
+ pub(crate) fn local_to_utc<T: ArrowTimestampType>(self, local: i64) ->
Result<i64> {
+ let units = units_per_second::<T>();
+ let shift = |offset_seconds: i32| {
+ local
+ .checked_sub(i64::from(offset_seconds) * units)
+ .ok_or_else(|| exec_datafusion_err!("Timestamp {local}
overflows"))
+ };
+ match self {
+ Self::FixedOffset(seconds) => shift(seconds),
+ Self::Region(tz) => {
+ let local_datetime =
+ DateTime::<Utc>::from_timestamp(local.div_euclid(units), 0)
+ .ok_or_else(|| {
+ exec_datafusion_err!(
+ "Timestamp is outside Chrono's calendar range"
+ )
+ })?
+ .naive_utc();
+ let offset = region_offset_at_local(&tz, local_datetime);
+ shift(offset)
+ }
+ Self::SystemV(standard_offset) => {
+ // Use daylight time if the result is in daylight time. That
picks
+ // the earlier offset in the overlap, and standard time, the
+ // offset before the transition, in the gap.
+ let daylight_offset = standard_offset + 3600;
+ let utc = shift(daylight_offset)?;
+ if system_v_offset::<T>(utc, standard_offset)? ==
daylight_offset {
+ Ok(utc)
+ } else {
+ shift(standard_offset)
+ }
+ }
+ }
+ }
+}
+
+/// The offset of `tz`, in seconds, at the local date-time `local`, resolved as
+/// Java's `LocalDateTime.atZone` does. The fraction of a second does not
affect
+/// the offset, so `local` is in whole seconds.
+fn region_offset_at_local(tz: &Tz, local: NaiveDateTime) -> i32 {
+ let offset_of = |result: LocalResult<DateTime<Tz>>| match result {
+ LocalResult::Single(dt) | LocalResult::Ambiguous(dt, _) => {
+ Some(dt.offset().fix().local_minus_utc())
+ }
+ LocalResult::None => None,
+ };
+ offset_of(tz.from_local_datetime(&local))
+ // In a gap, apply the offset in effect before it, which moves the time
+ // forward by the length of the gap.
+ .or_else(|| {
+ offset_of(tz.from_local_datetime(&(local -
chrono::Duration::hours(3))))
+ })
+ .unwrap_or(0)
Review Comment:
Thanks for checking this against Java across all the zones.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]