This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 89cb6a2810 feat: use Substrait's PrecisionTimestamp and
PrecisionTimestampTz instead of deprecated Timestamp (#11597)
89cb6a2810 is described below
commit 89cb6a28108f00d2b01fd978c85fb1cb5e639349
Author: Arttu <[email protected]>
AuthorDate: Thu Aug 22 12:13:51 2024 +0200
feat: use Substrait's PrecisionTimestamp and PrecisionTimestampTz instead
of deprecated Timestamp (#11597)
* bump substrait-rs
* consume and produce precisiontimestamps
* bump substrait to latest
* clippy
* deprecate in 42, since we're already on 41
---
datafusion/substrait/Cargo.toml | 6 +-
datafusion/substrait/src/logical_plan/consumer.rs | 244 ++++++++++++++++------
datafusion/substrait/src/logical_plan/producer.rs | 141 +++++++++----
datafusion/substrait/src/variation_const.rs | 6 +
4 files changed, 292 insertions(+), 105 deletions(-)
diff --git a/datafusion/substrait/Cargo.toml b/datafusion/substrait/Cargo.toml
index 9e7ef9632a..ff02ef8c7e 100644
--- a/datafusion/substrait/Cargo.toml
+++ b/datafusion/substrait/Cargo.toml
@@ -38,9 +38,9 @@ chrono = { workspace = true }
datafusion = { workspace = true, default-features = true }
itertools = { workspace = true }
object_store = { workspace = true }
-pbjson-types = "0.6"
-prost = "0.12"
-substrait = { version = "0.36.0", features = ["serde"] }
+pbjson-types = "0.7"
+prost = "0.13"
+substrait = { version = "0.41", features = ["serde"] }
url = { workspace = true }
[dev-dependencies]
diff --git a/datafusion/substrait/src/logical_plan/consumer.rs
b/datafusion/substrait/src/logical_plan/consumer.rs
index f2756bb06d..b1b510f179 100644
--- a/datafusion/substrait/src/logical_plan/consumer.rs
+++ b/datafusion/substrait/src/logical_plan/consumer.rs
@@ -42,14 +42,14 @@ use crate::variation_const::{
DECIMAL_128_TYPE_VARIATION_REF, DECIMAL_256_TYPE_VARIATION_REF,
DEFAULT_CONTAINER_TYPE_VARIATION_REF, DEFAULT_TYPE_VARIATION_REF,
INTERVAL_MONTH_DAY_NANO_TYPE_NAME, LARGE_CONTAINER_TYPE_VARIATION_REF,
- TIMESTAMP_MICRO_TYPE_VARIATION_REF, TIMESTAMP_MILLI_TYPE_VARIATION_REF,
- TIMESTAMP_NANO_TYPE_VARIATION_REF, TIMESTAMP_SECOND_TYPE_VARIATION_REF,
UNSIGNED_INTEGER_TYPE_VARIATION_REF,
};
#[allow(deprecated)]
use crate::variation_const::{
INTERVAL_DAY_TIME_TYPE_REF, INTERVAL_MONTH_DAY_NANO_TYPE_REF,
- INTERVAL_YEAR_MONTH_TYPE_REF,
+ INTERVAL_YEAR_MONTH_TYPE_REF, TIMESTAMP_MICRO_TYPE_VARIATION_REF,
+ TIMESTAMP_MILLI_TYPE_VARIATION_REF, TIMESTAMP_NANO_TYPE_VARIATION_REF,
+ TIMESTAMP_SECOND_TYPE_VARIATION_REF,
};
use datafusion::arrow::array::{new_empty_array, AsArray};
use datafusion::common::scalar::ScalarStructBuilder;
@@ -69,6 +69,7 @@ use datafusion::{
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use substrait::proto::exchange_rel::ExchangeKind;
+use
substrait::proto::expression::literal::interval_day_to_second::PrecisionMode;
use substrait::proto::expression::literal::user_defined::Val;
use substrait::proto::expression::literal::{
IntervalDayToSecond, IntervalYearToMonth, UserDefined,
@@ -95,6 +96,13 @@ use substrait::proto::{
};
use substrait::proto::{FunctionArgument, SortField};
+// Substrait PrecisionTimestampTz indicates that the timestamp is relative to
UTC, which
+// is the same as the expectation for any non-empty timezone in DF, so any
non-empty timezone
+// results in correct points on the timeline, and we pick UTC as a reasonable
default.
+// However, DF uses the timezone also for some arithmetic and display purposes
(see e.g.
+//
https://github.com/apache/arrow-rs/blob/ee5694078c86c8201549654246900a4232d531a9/arrow-cast/src/cast/mod.rs#L1749).
+const DEFAULT_TIMEZONE: &str = "UTC";
+
pub fn name_to_op(name: &str) -> Option<Operator> {
match name {
"equal" => Some(Operator::Eq),
@@ -877,8 +885,8 @@ fn from_substrait_jointype(join_type: i32) ->
Result<JoinType> {
join_rel::JoinType::Left => Ok(JoinType::Left),
join_rel::JoinType::Right => Ok(JoinType::Right),
join_rel::JoinType::Outer => Ok(JoinType::Full),
- join_rel::JoinType::Anti => Ok(JoinType::LeftAnti),
- join_rel::JoinType::Semi => Ok(JoinType::LeftSemi),
+ join_rel::JoinType::LeftAnti => Ok(JoinType::LeftAnti),
+ join_rel::JoinType::LeftSemi => Ok(JoinType::LeftSemi),
_ => plan_err!("unsupported join type {substrait_join_type:?}"),
}
} else {
@@ -1369,23 +1377,51 @@ fn from_substrait_type(
},
r#type::Kind::Fp32(_) => Ok(DataType::Float32),
r#type::Kind::Fp64(_) => Ok(DataType::Float64),
- r#type::Kind::Timestamp(ts) => match ts.type_variation_reference {
- TIMESTAMP_SECOND_TYPE_VARIATION_REF => {
- Ok(DataType::Timestamp(TimeUnit::Second, None))
- }
- TIMESTAMP_MILLI_TYPE_VARIATION_REF => {
- Ok(DataType::Timestamp(TimeUnit::Millisecond, None))
- }
- TIMESTAMP_MICRO_TYPE_VARIATION_REF => {
- Ok(DataType::Timestamp(TimeUnit::Microsecond, None))
- }
- TIMESTAMP_NANO_TYPE_VARIATION_REF => {
- Ok(DataType::Timestamp(TimeUnit::Nanosecond, None))
+ r#type::Kind::Timestamp(ts) => {
+ // Kept for backwards compatibility, new plans should use
PrecisionTimestamp(Tz) instead
+ #[allow(deprecated)]
+ match ts.type_variation_reference {
+ TIMESTAMP_SECOND_TYPE_VARIATION_REF => {
+ Ok(DataType::Timestamp(TimeUnit::Second, None))
+ }
+ TIMESTAMP_MILLI_TYPE_VARIATION_REF => {
+ Ok(DataType::Timestamp(TimeUnit::Millisecond, None))
+ }
+ TIMESTAMP_MICRO_TYPE_VARIATION_REF => {
+ Ok(DataType::Timestamp(TimeUnit::Microsecond, None))
+ }
+ TIMESTAMP_NANO_TYPE_VARIATION_REF => {
+ Ok(DataType::Timestamp(TimeUnit::Nanosecond, None))
+ }
+ v => not_impl_err!(
+ "Unsupported Substrait type variation {v} of type
{s_kind:?}"
+ ),
}
- v => not_impl_err!(
- "Unsupported Substrait type variation {v} of type
{s_kind:?}"
- ),
- },
+ }
+ r#type::Kind::PrecisionTimestamp(pts) => {
+ let unit = match pts.precision {
+ 0 => Ok(TimeUnit::Second),
+ 3 => Ok(TimeUnit::Millisecond),
+ 6 => Ok(TimeUnit::Microsecond),
+ 9 => Ok(TimeUnit::Nanosecond),
+ p => not_impl_err!(
+ "Unsupported Substrait precision {p} for
PrecisionTimestamp"
+ ),
+ }?;
+ Ok(DataType::Timestamp(unit, None))
+ }
+ r#type::Kind::PrecisionTimestampTz(pts) => {
+ let unit = match pts.precision {
+ 0 => Ok(TimeUnit::Second),
+ 3 => Ok(TimeUnit::Millisecond),
+ 6 => Ok(TimeUnit::Microsecond),
+ 9 => Ok(TimeUnit::Nanosecond),
+ p => not_impl_err!(
+ "Unsupported Substrait precision {p} for
PrecisionTimestampTz"
+ ),
+ }?;
+ Ok(DataType::Timestamp(unit, Some(DEFAULT_TIMEZONE.into())))
+ }
r#type::Kind::Date(date) => match date.type_variation_reference {
DATE_32_TYPE_VARIATION_REF => Ok(DataType::Date32),
DATE_64_TYPE_VARIATION_REF => Ok(DataType::Date64),
@@ -1465,22 +1501,10 @@ fn from_substrait_type(
"Unsupported Substrait type variation {v} of type
{s_kind:?}"
),
},
- r#type::Kind::IntervalYear(i) => match i.type_variation_reference {
- DEFAULT_TYPE_VARIATION_REF => {
- Ok(DataType::Interval(IntervalUnit::YearMonth))
- }
- v => not_impl_err!(
- "Unsupported Substrait type variation {v} of type
{s_kind:?}"
- ),
- },
- r#type::Kind::IntervalDay(i) => match i.type_variation_reference {
- DEFAULT_TYPE_VARIATION_REF => {
- Ok(DataType::Interval(IntervalUnit::DayTime))
- }
- v => not_impl_err!(
- "Unsupported Substrait type variation {v} of type
{s_kind:?}"
- ),
- },
+ r#type::Kind::IntervalYear(_) => {
+ Ok(DataType::Interval(IntervalUnit::YearMonth))
+ }
+ r#type::Kind::IntervalDay(_) =>
Ok(DataType::Interval(IntervalUnit::DayTime)),
r#type::Kind::UserDefined(u) => {
if let Some(name) = extensions.types.get(&u.type_reference) {
match name.as_ref() {
@@ -1676,21 +1700,59 @@ fn from_substrait_literal(
},
Some(LiteralType::Fp32(f)) => ScalarValue::Float32(Some(*f)),
Some(LiteralType::Fp64(f)) => ScalarValue::Float64(Some(*f)),
- Some(LiteralType::Timestamp(t)) => match lit.type_variation_reference {
- TIMESTAMP_SECOND_TYPE_VARIATION_REF => {
- ScalarValue::TimestampSecond(Some(*t), None)
- }
- TIMESTAMP_MILLI_TYPE_VARIATION_REF => {
- ScalarValue::TimestampMillisecond(Some(*t), None)
- }
- TIMESTAMP_MICRO_TYPE_VARIATION_REF => {
- ScalarValue::TimestampMicrosecond(Some(*t), None)
+ Some(LiteralType::Timestamp(t)) => {
+ // Kept for backwards compatibility, new plans should use
PrecisionTimestamp(Tz) instead
+ #[allow(deprecated)]
+ match lit.type_variation_reference {
+ TIMESTAMP_SECOND_TYPE_VARIATION_REF => {
+ ScalarValue::TimestampSecond(Some(*t), None)
+ }
+ TIMESTAMP_MILLI_TYPE_VARIATION_REF => {
+ ScalarValue::TimestampMillisecond(Some(*t), None)
+ }
+ TIMESTAMP_MICRO_TYPE_VARIATION_REF => {
+ ScalarValue::TimestampMicrosecond(Some(*t), None)
+ }
+ TIMESTAMP_NANO_TYPE_VARIATION_REF => {
+ ScalarValue::TimestampNanosecond(Some(*t), None)
+ }
+ others => {
+ return substrait_err!("Unknown type variation reference
{others}");
+ }
}
- TIMESTAMP_NANO_TYPE_VARIATION_REF => {
- ScalarValue::TimestampNanosecond(Some(*t), None)
+ }
+ Some(LiteralType::PrecisionTimestamp(pt)) => match pt.precision {
+ 0 => ScalarValue::TimestampSecond(Some(pt.value), None),
+ 3 => ScalarValue::TimestampMillisecond(Some(pt.value), None),
+ 6 => ScalarValue::TimestampMicrosecond(Some(pt.value), None),
+ 9 => ScalarValue::TimestampNanosecond(Some(pt.value), None),
+ p => {
+ return not_impl_err!(
+ "Unsupported Substrait precision {p} for
PrecisionTimestamp"
+ );
}
- others => {
- return substrait_err!("Unknown type variation reference
{others}");
+ },
+ Some(LiteralType::PrecisionTimestampTz(pt)) => match pt.precision {
+ 0 => ScalarValue::TimestampSecond(
+ Some(pt.value),
+ Some(DEFAULT_TIMEZONE.into()),
+ ),
+ 3 => ScalarValue::TimestampMillisecond(
+ Some(pt.value),
+ Some(DEFAULT_TIMEZONE.into()),
+ ),
+ 6 => ScalarValue::TimestampMicrosecond(
+ Some(pt.value),
+ Some(DEFAULT_TIMEZONE.into()),
+ ),
+ 9 => ScalarValue::TimestampNanosecond(
+ Some(pt.value),
+ Some(DEFAULT_TIMEZONE.into()),
+ ),
+ p => {
+ return not_impl_err!(
+ "Unsupported Substrait precision {p} for
PrecisionTimestamp"
+ );
}
},
Some(LiteralType::Date(d)) => ScalarValue::Date32(Some(*d)),
@@ -1881,10 +1943,24 @@ fn from_substrait_literal(
Some(LiteralType::IntervalDayToSecond(IntervalDayToSecond {
days,
seconds,
- microseconds,
+ subseconds,
+ precision_mode,
})) => {
- // DF only supports millisecond precision, so we lose the micros
here
- ScalarValue::new_interval_dt(*days, (seconds * 1000) +
(microseconds / 1000))
+ // DF only supports millisecond precision, so for any more
granular type we lose precision
+ let milliseconds = match precision_mode {
+ Some(PrecisionMode::Microseconds(ms)) => ms / 1000,
+ Some(PrecisionMode::Precision(0)) => *subseconds as i32 * 1000,
+ Some(PrecisionMode::Precision(3)) => *subseconds as i32,
+ Some(PrecisionMode::Precision(6)) => (subseconds / 1000) as
i32,
+ Some(PrecisionMode::Precision(9)) => (subseconds / 1000 /
1000) as i32,
+ _ => {
+ return not_impl_err!(
+ "Unsupported Substrait interval day to second
precision mode"
+ )
+ }
+ };
+
+ ScalarValue::new_interval_dt(*days, (seconds * 1000) +
milliseconds)
}
Some(LiteralType::IntervalYearToMonth(IntervalYearToMonth { years,
months })) => {
ScalarValue::new_interval_ym(*years, *months)
@@ -2026,21 +2102,55 @@ fn from_substrait_null(
},
r#type::Kind::Fp32(_) => Ok(ScalarValue::Float32(None)),
r#type::Kind::Fp64(_) => Ok(ScalarValue::Float64(None)),
- r#type::Kind::Timestamp(ts) => match ts.type_variation_reference {
- TIMESTAMP_SECOND_TYPE_VARIATION_REF => {
- Ok(ScalarValue::TimestampSecond(None, None))
- }
- TIMESTAMP_MILLI_TYPE_VARIATION_REF => {
- Ok(ScalarValue::TimestampMillisecond(None, None))
- }
- TIMESTAMP_MICRO_TYPE_VARIATION_REF => {
- Ok(ScalarValue::TimestampMicrosecond(None, None))
- }
- TIMESTAMP_NANO_TYPE_VARIATION_REF => {
- Ok(ScalarValue::TimestampNanosecond(None, None))
+ r#type::Kind::Timestamp(ts) => {
+ // Kept for backwards compatibility, new plans should use
PrecisionTimestamp(Tz) instead
+ #[allow(deprecated)]
+ match ts.type_variation_reference {
+ TIMESTAMP_SECOND_TYPE_VARIATION_REF => {
+ Ok(ScalarValue::TimestampSecond(None, None))
+ }
+ TIMESTAMP_MILLI_TYPE_VARIATION_REF => {
+ Ok(ScalarValue::TimestampMillisecond(None, None))
+ }
+ TIMESTAMP_MICRO_TYPE_VARIATION_REF => {
+ Ok(ScalarValue::TimestampMicrosecond(None, None))
+ }
+ TIMESTAMP_NANO_TYPE_VARIATION_REF => {
+ Ok(ScalarValue::TimestampNanosecond(None, None))
+ }
+ v => not_impl_err!(
+ "Unsupported Substrait type variation {v} of type
{kind:?}"
+ ),
}
- v => not_impl_err!(
- "Unsupported Substrait type variation {v} of type {kind:?}"
+ }
+ r#type::Kind::PrecisionTimestamp(pts) => match pts.precision {
+ 0 => Ok(ScalarValue::TimestampSecond(None, None)),
+ 3 => Ok(ScalarValue::TimestampMillisecond(None, None)),
+ 6 => Ok(ScalarValue::TimestampMicrosecond(None, None)),
+ 9 => Ok(ScalarValue::TimestampNanosecond(None, None)),
+ p => not_impl_err!(
+ "Unsupported Substrait precision {p} for
PrecisionTimestamp"
+ ),
+ },
+ r#type::Kind::PrecisionTimestampTz(pts) => match pts.precision {
+ 0 => Ok(ScalarValue::TimestampSecond(
+ None,
+ Some(DEFAULT_TIMEZONE.into()),
+ )),
+ 3 => Ok(ScalarValue::TimestampMillisecond(
+ None,
+ Some(DEFAULT_TIMEZONE.into()),
+ )),
+ 6 => Ok(ScalarValue::TimestampMicrosecond(
+ None,
+ Some(DEFAULT_TIMEZONE.into()),
+ )),
+ 9 => Ok(ScalarValue::TimestampNanosecond(
+ None,
+ Some(DEFAULT_TIMEZONE.into()),
+ )),
+ p => not_impl_err!(
+ "Unsupported Substrait precision {p} for
PrecisionTimestamp"
),
},
r#type::Kind::Date(date) => match date.type_variation_reference {
diff --git a/datafusion/substrait/src/logical_plan/producer.rs
b/datafusion/substrait/src/logical_plan/producer.rs
index ee04749f5e..72b6760be2 100644
--- a/datafusion/substrait/src/logical_plan/producer.rs
+++ b/datafusion/substrait/src/logical_plan/producer.rs
@@ -38,8 +38,6 @@ use crate::variation_const::{
DECIMAL_128_TYPE_VARIATION_REF, DECIMAL_256_TYPE_VARIATION_REF,
DEFAULT_CONTAINER_TYPE_VARIATION_REF, DEFAULT_TYPE_VARIATION_REF,
INTERVAL_MONTH_DAY_NANO_TYPE_NAME, LARGE_CONTAINER_TYPE_VARIATION_REF,
- TIMESTAMP_MICRO_TYPE_VARIATION_REF, TIMESTAMP_MILLI_TYPE_VARIATION_REF,
- TIMESTAMP_NANO_TYPE_VARIATION_REF, TIMESTAMP_SECOND_TYPE_VARIATION_REF,
UNSIGNED_INTEGER_TYPE_VARIATION_REF,
};
use datafusion::arrow::array::{Array, GenericListArray, OffsetSizeTrait};
@@ -55,10 +53,11 @@ use datafusion::logical_expr::{expr, Between,
JoinConstraint, LogicalPlan, Opera
use datafusion::prelude::Expr;
use pbjson_types::Any as ProtoAny;
use substrait::proto::exchange_rel::{ExchangeKind, RoundRobin, ScatterFields};
+use
substrait::proto::expression::literal::interval_day_to_second::PrecisionMode;
use substrait::proto::expression::literal::map::KeyValue;
use substrait::proto::expression::literal::{
- user_defined, IntervalDayToSecond, IntervalYearToMonth, List, Map, Struct,
- UserDefined,
+ user_defined, IntervalDayToSecond, IntervalYearToMonth, List, Map,
+ PrecisionTimestamp, Struct, UserDefined,
};
use substrait::proto::expression::subquery::InPredicate;
use substrait::proto::expression::window_function::BoundsType;
@@ -658,8 +657,8 @@ fn to_substrait_jointype(join_type: JoinType) ->
join_rel::JoinType {
JoinType::Left => join_rel::JoinType::Left,
JoinType::Right => join_rel::JoinType::Right,
JoinType::Full => join_rel::JoinType::Outer,
- JoinType::LeftAnti => join_rel::JoinType::Anti,
- JoinType::LeftSemi => join_rel::JoinType::Semi,
+ JoinType::LeftAnti => join_rel::JoinType::LeftAnti,
+ JoinType::LeftSemi => join_rel::JoinType::LeftSemi,
JoinType::RightAnti | JoinType::RightSemi => unimplemented!(),
}
}
@@ -1376,20 +1375,31 @@ fn to_substrait_type(
nullability,
})),
}),
- // Timezone is ignored.
- DataType::Timestamp(unit, _) => {
- let type_variation_reference = match unit {
- TimeUnit::Second => TIMESTAMP_SECOND_TYPE_VARIATION_REF,
- TimeUnit::Millisecond => TIMESTAMP_MILLI_TYPE_VARIATION_REF,
- TimeUnit::Microsecond => TIMESTAMP_MICRO_TYPE_VARIATION_REF,
- TimeUnit::Nanosecond => TIMESTAMP_NANO_TYPE_VARIATION_REF,
+ DataType::Timestamp(unit, tz) => {
+ let precision = match unit {
+ TimeUnit::Second => 0,
+ TimeUnit::Millisecond => 3,
+ TimeUnit::Microsecond => 6,
+ TimeUnit::Nanosecond => 9,
};
- Ok(substrait::proto::Type {
- kind: Some(r#type::Kind::Timestamp(r#type::Timestamp {
- type_variation_reference,
+ let kind = match tz {
+ None =>
r#type::Kind::PrecisionTimestamp(r#type::PrecisionTimestamp {
+ type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
nullability,
- })),
- })
+ precision,
+ }),
+ Some(_) => {
+ // If timezone is present, no matter what the actual tz
value is, it indicates the
+ // value of the timestamp is tied to UTC epoch. That's all
that Substrait cares about.
+ // As the timezone is lost, this conversion may be lossy
for downstream use of the value.
+
r#type::Kind::PrecisionTimestampTz(r#type::PrecisionTimestampTz {
+ type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
+ nullability,
+ precision,
+ })
+ }
+ };
+ Ok(substrait::proto::Type { kind: Some(kind) })
}
DataType::Date32 => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Date(r#type::Date {
@@ -1415,6 +1425,7 @@ fn to_substrait_type(
kind: Some(r#type::Kind::IntervalDay(r#type::IntervalDay {
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
nullability,
+ precision: Some(3), // DayTime precision is always
milliseconds
})),
}),
IntervalUnit::MonthDayNano => {
@@ -1798,21 +1809,64 @@ fn to_substrait_literal(
ScalarValue::Float64(Some(f)) => {
(LiteralType::Fp64(*f), DEFAULT_TYPE_VARIATION_REF)
}
- ScalarValue::TimestampSecond(Some(t), _) => (
- LiteralType::Timestamp(*t),
- TIMESTAMP_SECOND_TYPE_VARIATION_REF,
+ ScalarValue::TimestampSecond(Some(t), None) => (
+ LiteralType::PrecisionTimestamp(PrecisionTimestamp {
+ precision: 0,
+ value: *t,
+ }),
+ DEFAULT_TYPE_VARIATION_REF,
),
- ScalarValue::TimestampMillisecond(Some(t), _) => (
- LiteralType::Timestamp(*t),
- TIMESTAMP_MILLI_TYPE_VARIATION_REF,
+ ScalarValue::TimestampMillisecond(Some(t), None) => (
+ LiteralType::PrecisionTimestamp(PrecisionTimestamp {
+ precision: 3,
+ value: *t,
+ }),
+ DEFAULT_TYPE_VARIATION_REF,
),
- ScalarValue::TimestampMicrosecond(Some(t), _) => (
- LiteralType::Timestamp(*t),
- TIMESTAMP_MICRO_TYPE_VARIATION_REF,
+ ScalarValue::TimestampMicrosecond(Some(t), None) => (
+ LiteralType::PrecisionTimestamp(PrecisionTimestamp {
+ precision: 6,
+ value: *t,
+ }),
+ DEFAULT_TYPE_VARIATION_REF,
),
- ScalarValue::TimestampNanosecond(Some(t), _) => (
- LiteralType::Timestamp(*t),
- TIMESTAMP_NANO_TYPE_VARIATION_REF,
+ ScalarValue::TimestampNanosecond(Some(t), None) => (
+ LiteralType::PrecisionTimestamp(PrecisionTimestamp {
+ precision: 9,
+ value: *t,
+ }),
+ DEFAULT_TYPE_VARIATION_REF,
+ ),
+ // If timezone is present, no matter what the actual tz value is, it
indicates the
+ // value of the timestamp is tied to UTC epoch. That's all that
Substrait cares about.
+ // As the timezone is lost, this conversion may be lossy for
downstream use of the value.
+ ScalarValue::TimestampSecond(Some(t), Some(_)) => (
+ LiteralType::PrecisionTimestampTz(PrecisionTimestamp {
+ precision: 0,
+ value: *t,
+ }),
+ DEFAULT_TYPE_VARIATION_REF,
+ ),
+ ScalarValue::TimestampMillisecond(Some(t), Some(_)) => (
+ LiteralType::PrecisionTimestampTz(PrecisionTimestamp {
+ precision: 3,
+ value: *t,
+ }),
+ DEFAULT_TYPE_VARIATION_REF,
+ ),
+ ScalarValue::TimestampMicrosecond(Some(t), Some(_)) => (
+ LiteralType::PrecisionTimestampTz(PrecisionTimestamp {
+ precision: 6,
+ value: *t,
+ }),
+ DEFAULT_TYPE_VARIATION_REF,
+ ),
+ ScalarValue::TimestampNanosecond(Some(t), Some(_)) => (
+ LiteralType::PrecisionTimestampTz(PrecisionTimestamp {
+ precision: 9,
+ value: *t,
+ }),
+ DEFAULT_TYPE_VARIATION_REF,
),
ScalarValue::Date32(Some(d)) => {
(LiteralType::Date(*d), DATE_32_TYPE_VARIATION_REF)
@@ -1847,7 +1901,8 @@ fn to_substrait_literal(
LiteralType::IntervalDayToSecond(IntervalDayToSecond {
days: i.days,
seconds: i.milliseconds / 1000,
- microseconds: (i.milliseconds % 1000) * 1000,
+ subseconds: (i.milliseconds % 1000) as i64,
+ precision_mode: Some(PrecisionMode::Precision(3)), // 3 for
milliseconds
}),
DEFAULT_TYPE_VARIATION_REF,
),
@@ -2142,6 +2197,18 @@ mod test {
round_trip_literal(ScalarValue::UInt64(Some(u64::MIN)))?;
round_trip_literal(ScalarValue::UInt64(Some(u64::MAX)))?;
+ for (ts, tz) in [
+ (Some(12345), None),
+ (None, None),
+ (Some(12345), Some("UTC".into())),
+ (None, Some("UTC".into())),
+ ] {
+ round_trip_literal(ScalarValue::TimestampSecond(ts, tz.clone()))?;
+ round_trip_literal(ScalarValue::TimestampMillisecond(ts,
tz.clone()))?;
+ round_trip_literal(ScalarValue::TimestampMicrosecond(ts,
tz.clone()))?;
+ round_trip_literal(ScalarValue::TimestampNanosecond(ts, tz))?;
+ }
+
round_trip_literal(ScalarValue::List(ScalarValue::new_list_nullable(
&[ScalarValue::Float32(Some(1.0))],
&DataType::Float32,
@@ -2271,10 +2338,14 @@ mod test {
round_trip_type(DataType::UInt64)?;
round_trip_type(DataType::Float32)?;
round_trip_type(DataType::Float64)?;
- round_trip_type(DataType::Timestamp(TimeUnit::Second, None))?;
- round_trip_type(DataType::Timestamp(TimeUnit::Millisecond, None))?;
- round_trip_type(DataType::Timestamp(TimeUnit::Microsecond, None))?;
- round_trip_type(DataType::Timestamp(TimeUnit::Nanosecond, None))?;
+
+ for tz in [None, Some("UTC".into())] {
+ round_trip_type(DataType::Timestamp(TimeUnit::Second,
tz.clone()))?;
+ round_trip_type(DataType::Timestamp(TimeUnit::Millisecond,
tz.clone()))?;
+ round_trip_type(DataType::Timestamp(TimeUnit::Microsecond,
tz.clone()))?;
+ round_trip_type(DataType::Timestamp(TimeUnit::Nanosecond, tz))?;
+ }
+
round_trip_type(DataType::Date32)?;
round_trip_type(DataType::Date64)?;
round_trip_type(DataType::Binary)?;
diff --git a/datafusion/substrait/src/variation_const.rs
b/datafusion/substrait/src/variation_const.rs
index c94ad2d669..1525da7645 100644
--- a/datafusion/substrait/src/variation_const.rs
+++ b/datafusion/substrait/src/variation_const.rs
@@ -38,10 +38,16 @@
/// The "system-preferred" variation (i.e., no variation).
pub const DEFAULT_TYPE_VARIATION_REF: u32 = 0;
pub const UNSIGNED_INTEGER_TYPE_VARIATION_REF: u32 = 1;
+
+#[deprecated(since = "42.0.0", note = "Use `PrecisionTimestamp(Tz)` type
instead")]
pub const TIMESTAMP_SECOND_TYPE_VARIATION_REF: u32 = 0;
+#[deprecated(since = "42.0.0", note = "Use `PrecisionTimestamp(Tz)` type
instead")]
pub const TIMESTAMP_MILLI_TYPE_VARIATION_REF: u32 = 1;
+#[deprecated(since = "42.0.0", note = "Use `PrecisionTimestamp(Tz)` type
instead")]
pub const TIMESTAMP_MICRO_TYPE_VARIATION_REF: u32 = 2;
+#[deprecated(since = "42.0.0", note = "Use `PrecisionTimestamp(Tz)` type
instead")]
pub const TIMESTAMP_NANO_TYPE_VARIATION_REF: u32 = 3;
+
pub const DATE_32_TYPE_VARIATION_REF: u32 = 0;
pub const DATE_64_TYPE_VARIATION_REF: u32 = 1;
pub const DEFAULT_CONTAINER_TYPE_VARIATION_REF: u32 = 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
