andygrove commented on code in PR #1191:
URL: https://github.com/apache/datafusion-comet/pull/1191#discussion_r1894276646
##########
native/spark-expr/src/schema_adapter.rs:
##########
@@ -284,6 +287,237 @@ impl SchemaMapper for SchemaMapping {
}
}
+/// Determine if Comet supports a cast, taking options such as EvalMode and
Timezone into account.
+fn cast_supported(from_type: &DataType, to_type: &DataType, options:
&SparkCastOptions) -> bool {
+ use DataType::*;
+
+ let from_type = if let Dictionary(_, dt) = from_type {
+ dt
+ } else {
+ from_type
+ };
+
+ let to_type = if let Dictionary(_, dt) = to_type {
+ dt
+ } else {
+ to_type
+ };
+
+ if from_type == to_type {
+ return true;
+ }
+
+ match (from_type, to_type) {
+ (Boolean, _) => can_cast_from_boolean(to_type, options),
+ (UInt8 | UInt16 | UInt32 | UInt64, Int8 | Int16 | Int32 | Int64)
+ if options.allow_cast_unsigned_ints =>
+ {
+ true
+ }
+ (Int8, _) => can_cast_from_byte(to_type, options),
+ (Int16, _) => can_cast_from_short(to_type, options),
+ (Int32, _) => can_cast_from_int(to_type, options),
+ (Int64, _) => can_cast_from_long(to_type, options),
+ (Float32, _) => can_cast_from_float(to_type, options),
+ (Float64, _) => can_cast_from_double(to_type, options),
+ (Decimal128(p, s), _) => can_cast_from_decimal(p, s, to_type, options),
+ (Timestamp(_, None), _) => can_cast_from_timestamp_ntz(to_type,
options),
+ (Timestamp(_, Some(_)), _) => can_cast_from_timestamp(to_type,
options),
+ (Utf8 | LargeUtf8, _) => can_cast_from_string(to_type, options),
+ (_, Utf8 | LargeUtf8) => can_cast_to_string(from_type, options),
+ (Struct(from_fields), Struct(to_fields)) => from_fields
+ .iter()
+ .zip(to_fields.iter())
+ .all(|(a, b)| cast_supported(a.data_type(), b.data_type(),
options)),
+ _ => false,
+ }
+}
+
+fn can_cast_from_string(to_type: &DataType, options: &SparkCastOptions) ->
bool {
+ use DataType::*;
+ match to_type {
+ Boolean | Int8 | Int16 | Int32 | Int64 | Binary => true,
+ Float32 | Float64 => {
+ // https://github.com/apache/datafusion-comet/issues/326
+ // Does not support inputs ending with 'd' or 'f'. Does not
support 'inf'.
+ // Does not support ANSI mode.
+ options.allow_incompat
+ }
+ Decimal128(_, _) => {
+ // https://github.com/apache/datafusion-comet/issues/325
+ // Does not support inputs ending with 'd' or 'f'. Does not
support 'inf'.
+ // Does not support ANSI mode. Returns 0.0 instead of null if
input contains no digits
+
+ options.allow_incompat
+ }
+ Date32 | Date64 => {
+ // https://github.com/apache/datafusion-comet/issues/327
+ // Only supports years between 262143 BC and 262142 AD
+ options.allow_incompat
+ }
+ Timestamp(_, _) if options.eval_mode == EvalMode::Ansi => {
+ // ANSI mode not supported
+ false
+ }
+ Timestamp(_, Some(tz)) if tz.as_ref() != "UTC" => {
+ // Cast will use UTC instead of $timeZoneId
+ options.allow_incompat
+ }
+ Timestamp(_, _) => {
+ // https://github.com/apache/datafusion-comet/issues/328
+ // Not all valid formats are supported
+ options.allow_incompat
+ }
+ _ => false,
+ }
+}
+
+fn can_cast_to_string(from_type: &DataType, options: &SparkCastOptions) ->
bool {
+ use DataType::*;
+ match from_type {
+ Boolean | Int8 | Int16 | Int32 | Int64 | Date32 | Date64 |
Timestamp(_, _) => true,
+ Float32 | Float64 => {
+ // There can be differences in precision.
+ // For example, the input \"1.4E-45\" will produce 1.0E-45 " +
+ // instead of 1.4E-45"))
+ true
+ }
+ Decimal128(_, _) => {
+ // https://github.com/apache/datafusion-comet/issues/1068
+ // There can be formatting differences in some case due to Spark
using
+ // scientific notation where Comet does not
+ true
+ }
+ Binary => {
+ // https://github.com/apache/datafusion-comet/issues/377
+ // Only works for binary data representing valid UTF-8 strings
+ options.allow_incompat
+ }
+ Struct(fields) => fields
+ .iter()
+ .all(|f| can_cast_to_string(f.data_type(), options)),
+ _ => false,
+ }
+}
+
+fn can_cast_from_timestamp_ntz(to_type: &DataType, options: &SparkCastOptions)
-> bool {
+ use DataType::*;
+ match to_type {
+ Timestamp(_, _) | Date32 | Date64 | Utf8 => {
+ // incompatible
+ options.allow_incompat
+ }
+ _ => {
+ // unsupported
+ false
+ }
+ }
+}
+
+fn can_cast_from_timestamp(to_type: &DataType, _options: &SparkCastOptions) ->
bool {
+ use DataType::*;
+ match to_type {
+ Timestamp(_, _) => true,
Review Comment:
Adding this line is the actual fix
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
