sdf-jkl commented on code in PR #10114:
URL: https://github.com/apache/arrow-rs/pull/10114#discussion_r4648184595
##########
parquet-variant-compute/src/variant_to_arrow.rs:
##########
@@ -1276,12 +1342,79 @@ macro_rules! define_variant_to_primitive_builder {
}
}
-define_variant_to_primitive_builder!(
- struct VariantToStringArrowBuilder<'a, B: StringLikeArrayBuilder>
- |capacity| -> B { B::with_capacity(capacity) },
- |value| value.as_string(),
- type_name: B::type_name()
-);
+pub(crate) struct VariantToStringArrowBuilder<'a, B: StringLikeArrayBuilder> {
+ builder: B,
+ cast_options: &'a CastOptions<'a>,
+ formats: CompiledCastFormats<'a>,
+ cast_mode: VariantCastMode,
+ buffer: String,
+}
+
+impl<'a, B: StringLikeArrayBuilder> VariantToStringArrowBuilder<'a, B> {
+ fn new(cast_options: &'a CastOptions<'a>, capacity: usize, cast_mode:
VariantCastMode) -> Self {
+ Self {
+ builder: B::with_capacity(capacity),
+ cast_options,
+ formats: CompiledCastFormats::new(cast_options),
+ cast_mode,
+ buffer: String::new(),
+ }
+ }
+
+ fn append_null(&mut self) -> Result<()> {
+ self.builder.append_null();
+ Ok(())
+ }
+
+ fn append_value(&mut self, value: &Variant<'_, '_>) -> Result<bool> {
+ let value = match value {
+ // short-circuit the common case of string-like variants to avoid
the overhead of casting
+ Variant::String(value) => value,
+ Variant::ShortString(value) => value.as_str(),
+ // can't cast to string, so append null
+ Variant::Null | Variant::Object(_) => {
Review Comment:
With `safe: false`, a top-level object now becomes NULL instead of an error,
because this arm appends NULL without going through
`variant_cast_with_options`. On main the same input errors. Could
`Variant::Object(_)` go through the normal failure path so strict mode still
rejects it? `Variant::Null` can stay here.
##########
parquet-variant-compute/src/type_conversion.rs:
##########
@@ -720,6 +726,222 @@ pub(crate) fn variant_to_boolean(
}
}
+#[inline]
+fn write_float_to_string<F: Float>(f: F, out: &mut String) {
+ let mut buffer = ryu::Buffer::new();
+ out.push_str(buffer.format(f));
+}
+
+#[derive(Default)]
+pub(crate) struct CompiledCastFormats<'a> {
+ date: CompiledTimeFormat<'a>,
+ time: CompiledTimeFormat<'a>,
+ timestamp: CompiledTimeFormat<'a>,
+ timestamp_tz: CompiledTimeFormat<'a>,
+ null: &'a str,
+ quoted_strings: bool,
+}
+
+impl<'a> CompiledCastFormats<'a> {
+ pub(crate) fn new(options: &CastOptions<'a>) -> Self {
+ let options = &options.format_options;
+ Self {
+ date: CompiledTimeFormat::new(options.date_format()),
+ time: CompiledTimeFormat::new(options.time_format()),
+ timestamp: CompiledTimeFormat::new(options.timestamp_format()),
+ timestamp_tz:
CompiledTimeFormat::new(options.timestamp_tz_format()),
+ null: options.null(),
+ quoted_strings: options.quoted_strings(),
+ }
+ }
+}
+
+fn write_lexical_to_string<N: lexical_core::ToLexical>(out: &mut String, n: N)
{
+ // i64::FORMATTED_SIZE is the upper bound for all integer types we support
+ // (i8/i16/i32/i64). With power-of-two feature it's 128, otherwise 20.
+ // We can't use N::FORMATTED_SIZE in a generic function
(generic_const_exprs).
+ let mut buf = [0u8; i64::FORMATTED_SIZE];
+ let written = lexical_core::write(n, &mut buf);
+ // Lexical core produces valid UTF-8
+ out.push_str(unsafe { std::str::from_utf8_unchecked(written) });
+}
+
+fn format_to_arrow_err(e: FormatError) -> ArrowError {
+ match e {
+ FormatError::Arrow(e) => e,
+ FormatError::Format(e) => ArrowError::CastError(format!("Format error:
{e}")),
+ }
+}
+
+// Convert a variant into the output with the given format.
+pub(crate) fn write_variant_to_string(
+ variant: &Variant<'_, '_>,
+ formats: &CompiledCastFormats<'_>,
+ out: &mut String,
+) -> Result<bool> {
+ match variant {
+ Variant::Null => {
+ out.push_str(formats.null);
+ }
+ Variant::String(s) => {
+ if formats.quoted_strings {
+ write!(out, "{s:?}").expect("writing string to string should
never fail");
+ } else {
+ out.push_str(s);
+ }
+ }
+ Variant::ShortString(s) => {
+ if formats.quoted_strings {
+ write!(out, "{:?}", s.as_str())
+ .expect("writing string to string should never fail");
+ } else {
+ out.push_str(s);
+ }
+ }
+ Variant::BooleanTrue => {
+ out.push_str("true");
+ }
+ Variant::BooleanFalse => {
+ out.push_str("false");
+ }
+ Variant::Int8(i) => {
+ write_lexical_to_string(out, *i);
+ }
+ Variant::Int16(i) => {
+ write_lexical_to_string(out, *i);
+ }
+ Variant::Int32(i) => {
+ write_lexical_to_string(out, *i);
+ }
+ Variant::Int64(i) => {
+ write_lexical_to_string(out, *i);
+ }
+ Variant::Float(f) => {
+ write_float_to_string(*f, out);
+ }
+ Variant::Double(f) => {
+ write_float_to_string(*f, out);
+ }
+ Variant::Decimal4(d) => {
+ datatypes::write_decimal(out, d.integer(), d.scale() as _)
+ .expect("writing decimal to string should never fail");
+ }
+ Variant::Decimal8(d) => {
+ // would never fail because we write into a string here
+ datatypes::write_decimal(out, d.integer(), d.scale() as _)
+ .expect("writing decimal to string should never fail");
+ }
+ Variant::Decimal16(d) => {
+ // would never fail because we write into a string here
+ datatypes::write_decimal(out, d.integer(), d.scale() as _)
+ .expect("writing decimal to string should never fail");
+ }
+ Variant::Date(d) => {
+ write_temporal_display(out, d,
&formats.date).map_err(format_to_arrow_err)?;
+ }
+ Variant::Time(t) => {
+ write_temporal_display(out, t,
&formats.time).map_err(format_to_arrow_err)?;
+ }
+ Variant::TimestampMicros(t) => {
+ write_timestamp(
+ out,
+ t.naive_utc(),
+ "+00:00".parse().ok(),
+ &formats.timestamp_tz,
+ )
+ .map_err(format_to_arrow_err)?;
+ }
+ Variant::TimestampNtzMicros(t) => {
+ write_timestamp(out, *t, None,
&formats.timestamp).map_err(format_to_arrow_err)?;
+ }
+ Variant::TimestampNanos(t) => {
+ write_timestamp(
+ out,
+ t.naive_utc(),
+ "+00:00".parse().ok(),
+ &formats.timestamp_tz,
+ )
+ .map_err(format_to_arrow_err)?;
+ }
+ Variant::TimestampNtzNanos(t) => {
+ write_timestamp(out, *t, None,
&formats.timestamp).map_err(format_to_arrow_err)?;
+ }
+ Variant::Uuid(u) => {
+ write!(out, "{u}").expect("write to string should not fail");
+ }
+ Variant::Binary(v) => {
+ for byte in *v {
+ write!(out, "{byte:02x}").expect("write to string should never
fail");
+ }
+ }
+ Variant::List(l) => {
+ write_list_to_string(l.iter(), formats, out)?;
+ }
+ Variant::Object(o) => {
+ write_map_to_string(o.iter(), formats, out)?;
+ }
+ }
+ Ok(true)
+}
+
+fn write_list_to_string<'m, 'v>(
+ mut iter: impl Iterator<Item = Variant<'m, 'v>>,
+ formats: &CompiledCastFormats,
+ out: &mut String,
+) -> Result<()> {
+ out.push('[');
+ if let Some(item) = iter.next()
+ && !write_variant_to_string(&item, formats, out)?
+ {
+ out.push_str(formats.null);
+ }
+ for item in iter {
+ out.push_str(", ");
+ if !write_variant_to_string(&item, formats, out)? {
+ out.push_str(formats.null);
+ }
+ }
+ out.push(']');
+ Ok(())
+}
+
+fn write_map_to_string<'m, 'v>(
+ mut iter: impl Iterator<Item = (&'m str, Variant<'m, 'v>)>,
+ formats: &CompiledCastFormats,
+ out: &mut String,
+) -> Result<()> {
+ out.push('{');
+
+ if let Some((key, value)) = iter.next() {
+ out.push_str(key);
Review Comment:
With `with_quoted_strings(true)`, object keys are written without quotes:
`[{"k,1": "v,1"}]` becomes `[{k,1: "v,1"}]`. Casting the equivalent
`List<Map<Utf8, Utf8>>` gives `[{"k,1": "v,1"}]`. Could keys follow
`quoted_strings` too (here and at line 925)? The existing quoted-strings test
only covers a list of strings.
##########
parquet-variant-compute/src/variant_to_arrow.rs:
##########
@@ -207,6 +207,52 @@ pub(crate) fn make_variant_to_arrow_row_builder<'a>(
Ok(builder)
}
+pub(crate) struct VariantToBinaryArrowRowBuilder<'a, B:
BinaryLikeArrayBuilder> {
Review Comment:
`define_variant_to_primitive_builder!` already supports a `cast_mode`
closure (see `VariantToBooleanArrowRowBuilder`), so this hand-written builder
could stay a macro invocation:
```rust
define_variant_to_primitive_builder!(
struct VariantToBinaryArrowRowBuilder<'a, B: BinaryLikeArrayBuilder>
|capacity| -> B { B::with_capacity(capacity) },
|value, cast_mode| match cast_mode {
VariantCastMode::Get => variant_to_binary(value),
VariantCastMode::Shred => value.as_u8_slice(),
},
type_name: B::type_name()
);
```
That also keeps the strict-mode error message consistent with the other
builders.
##########
parquet-variant-compute/src/type_conversion.rs:
##########
@@ -720,6 +726,222 @@ pub(crate) fn variant_to_boolean(
}
}
+#[inline]
+fn write_float_to_string<F: Float>(f: F, out: &mut String) {
+ let mut buffer = ryu::Buffer::new();
+ out.push_str(buffer.format(f));
+}
+
+#[derive(Default)]
+pub(crate) struct CompiledCastFormats<'a> {
+ date: CompiledTimeFormat<'a>,
+ time: CompiledTimeFormat<'a>,
+ timestamp: CompiledTimeFormat<'a>,
+ timestamp_tz: CompiledTimeFormat<'a>,
+ null: &'a str,
+ quoted_strings: bool,
+}
+
+impl<'a> CompiledCastFormats<'a> {
+ pub(crate) fn new(options: &CastOptions<'a>) -> Self {
+ let options = &options.format_options;
+ Self {
+ date: CompiledTimeFormat::new(options.date_format()),
+ time: CompiledTimeFormat::new(options.time_format()),
+ timestamp: CompiledTimeFormat::new(options.timestamp_format()),
+ timestamp_tz:
CompiledTimeFormat::new(options.timestamp_tz_format()),
+ null: options.null(),
+ quoted_strings: options.quoted_strings(),
+ }
+ }
+}
+
+fn write_lexical_to_string<N: lexical_core::ToLexical>(out: &mut String, n: N)
{
+ // i64::FORMATTED_SIZE is the upper bound for all integer types we support
+ // (i8/i16/i32/i64). With power-of-two feature it's 128, otherwise 20.
+ // We can't use N::FORMATTED_SIZE in a generic function
(generic_const_exprs).
+ let mut buf = [0u8; i64::FORMATTED_SIZE];
+ let written = lexical_core::write(n, &mut buf);
+ // Lexical core produces valid UTF-8
+ out.push_str(unsafe { std::str::from_utf8_unchecked(written) });
+}
+
+fn format_to_arrow_err(e: FormatError) -> ArrowError {
+ match e {
+ FormatError::Arrow(e) => e,
+ FormatError::Format(e) => ArrowError::CastError(format!("Format error:
{e}")),
+ }
+}
+
+// Convert a variant into the output with the given format.
+pub(crate) fn write_variant_to_string(
+ variant: &Variant<'_, '_>,
+ formats: &CompiledCastFormats<'_>,
+ out: &mut String,
+) -> Result<bool> {
+ match variant {
+ Variant::Null => {
+ out.push_str(formats.null);
+ }
+ Variant::String(s) => {
+ if formats.quoted_strings {
+ write!(out, "{s:?}").expect("writing string to string should
never fail");
+ } else {
+ out.push_str(s);
+ }
+ }
+ Variant::ShortString(s) => {
+ if formats.quoted_strings {
+ write!(out, "{:?}", s.as_str())
+ .expect("writing string to string should never fail");
+ } else {
+ out.push_str(s);
+ }
+ }
+ Variant::BooleanTrue => {
+ out.push_str("true");
+ }
+ Variant::BooleanFalse => {
+ out.push_str("false");
+ }
+ Variant::Int8(i) => {
+ write_lexical_to_string(out, *i);
+ }
+ Variant::Int16(i) => {
+ write_lexical_to_string(out, *i);
+ }
+ Variant::Int32(i) => {
+ write_lexical_to_string(out, *i);
+ }
+ Variant::Int64(i) => {
+ write_lexical_to_string(out, *i);
+ }
+ Variant::Float(f) => {
+ write_float_to_string(*f, out);
+ }
+ Variant::Double(f) => {
+ write_float_to_string(*f, out);
+ }
+ Variant::Decimal4(d) => {
+ datatypes::write_decimal(out, d.integer(), d.scale() as _)
+ .expect("writing decimal to string should never fail");
+ }
+ Variant::Decimal8(d) => {
+ // would never fail because we write into a string here
+ datatypes::write_decimal(out, d.integer(), d.scale() as _)
+ .expect("writing decimal to string should never fail");
+ }
+ Variant::Decimal16(d) => {
+ // would never fail because we write into a string here
+ datatypes::write_decimal(out, d.integer(), d.scale() as _)
+ .expect("writing decimal to string should never fail");
+ }
+ Variant::Date(d) => {
+ write_temporal_display(out, d,
&formats.date).map_err(format_to_arrow_err)?;
+ }
+ Variant::Time(t) => {
+ write_temporal_display(out, t,
&formats.time).map_err(format_to_arrow_err)?;
+ }
+ Variant::TimestampMicros(t) => {
+ write_timestamp(
+ out,
+ t.naive_utc(),
+ "+00:00".parse().ok(),
+ &formats.timestamp_tz,
+ )
+ .map_err(format_to_arrow_err)?;
+ }
+ Variant::TimestampNtzMicros(t) => {
+ write_timestamp(out, *t, None,
&formats.timestamp).map_err(format_to_arrow_err)?;
+ }
+ Variant::TimestampNanos(t) => {
+ write_timestamp(
+ out,
+ t.naive_utc(),
+ "+00:00".parse().ok(),
+ &formats.timestamp_tz,
+ )
+ .map_err(format_to_arrow_err)?;
+ }
+ Variant::TimestampNtzNanos(t) => {
+ write_timestamp(out, *t, None,
&formats.timestamp).map_err(format_to_arrow_err)?;
+ }
+ Variant::Uuid(u) => {
+ write!(out, "{u}").expect("write to string should not fail");
+ }
+ Variant::Binary(v) => {
Review Comment:
This arm is also reached for a top-level `Binary`, so `variant_get(..,
Utf8)` returns hex (`"6162"`). `cast(Binary, Utf8)` decodes the bytes as UTF-8
instead (`"ab"`), with invalid UTF-8 as NULL in safe mode and an error in
strict mode. Hex is right for binary nested in a list (that matches
`ArrayFormatter`), but the top-level case should use `std::str::from_utf8` and
append the borrowed `&str`, with invalid bytes going through
`variant_cast_with_options`. Could you add a test with `b"ab"` and `[0xff,
0x00]` in both modes?
##########
parquet-variant-compute/src/type_conversion.rs:
##########
@@ -720,6 +726,222 @@ pub(crate) fn variant_to_boolean(
}
}
+#[inline]
+fn write_float_to_string<F: Float>(f: F, out: &mut String) {
+ let mut buffer = ryu::Buffer::new();
+ out.push_str(buffer.format(f));
+}
+
+#[derive(Default)]
+pub(crate) struct CompiledCastFormats<'a> {
+ date: CompiledTimeFormat<'a>,
+ time: CompiledTimeFormat<'a>,
+ timestamp: CompiledTimeFormat<'a>,
+ timestamp_tz: CompiledTimeFormat<'a>,
+ null: &'a str,
+ quoted_strings: bool,
+}
+
+impl<'a> CompiledCastFormats<'a> {
+ pub(crate) fn new(options: &CastOptions<'a>) -> Self {
+ let options = &options.format_options;
+ Self {
+ date: CompiledTimeFormat::new(options.date_format()),
+ time: CompiledTimeFormat::new(options.time_format()),
+ timestamp: CompiledTimeFormat::new(options.timestamp_format()),
+ timestamp_tz:
CompiledTimeFormat::new(options.timestamp_tz_format()),
+ null: options.null(),
+ quoted_strings: options.quoted_strings(),
+ }
+ }
+}
+
+fn write_lexical_to_string<N: lexical_core::ToLexical>(out: &mut String, n: N)
{
+ // i64::FORMATTED_SIZE is the upper bound for all integer types we support
+ // (i8/i16/i32/i64). With power-of-two feature it's 128, otherwise 20.
+ // We can't use N::FORMATTED_SIZE in a generic function
(generic_const_exprs).
+ let mut buf = [0u8; i64::FORMATTED_SIZE];
+ let written = lexical_core::write(n, &mut buf);
+ // Lexical core produces valid UTF-8
+ out.push_str(unsafe { std::str::from_utf8_unchecked(written) });
+}
+
+fn format_to_arrow_err(e: FormatError) -> ArrowError {
+ match e {
+ FormatError::Arrow(e) => e,
+ FormatError::Format(e) => ArrowError::CastError(format!("Format error:
{e}")),
+ }
+}
+
+// Convert a variant into the output with the given format.
+pub(crate) fn write_variant_to_string(
Review Comment:
`write_variant_to_string` always returns `Ok(true)`, so the `if
!write_variant_to_string(..)? { out.push_str(formats.null) }` branches in the
list and map writers never run. Could it return `Result<()>`?
That also fixes the remaining formatting error case: `append_cast_value`
turns a formatting `Err` into `None` with `.ok()`, so an invalid date format
gives NULL with `safe: true`, while arrow-cast returns `Cast error: Format
error` in both modes. If unsupported types return `None` and formatting errors
propagate as `Err`, both cases match the kernel. A test with
`with_date_format(Some("%Y-%m-%d %H"))` in both modes would cover it.
##########
parquet-variant-compute/src/shred_variant.rs:
##########
@@ -1178,7 +1178,7 @@ mod tests {
#[test]
fn test_already_shredded_input_error() {
- // Create a VariantArray that already has typed_value_field
+ // Create a VariantArray that already has typed_value_column
Review Comment:
nit: unrelated change. Same for the import reorder in
`parquet-variant/src/variant.rs` and the trailing comma in
`define_variant_to_primitive_builder!`.
##########
arrow-cast/src/display.rs:
##########
@@ -36,29 +36,42 @@ use arrow_buffer::ArrowNativeType;
use arrow_data::decimal::write_decimal;
use arrow_schema::*;
use chrono::format::{Item, StrftimeItems};
-use chrono::{NaiveDate, NaiveDateTime, SecondsFormat, TimeZone, Utc};
+use chrono::{NaiveDate, NaiveDateTime, NaiveTime, SecondsFormat, TimeZone,
Utc};
use lexical_core::FormattedSize;
type TimeFormat<'a> = Option<&'a str>;
struct CompiledItems<'a>(Vec<Item<'a>>);
-enum CompiledTimeFormat<'a> {
+enum CompiledTimeFormatInner<'a> {
Default,
Custom(Box<CompiledItems<'a>>),
}
+/// A compiled time format that can be used to format temporal values.
+///
+/// Use [`CompiledTimeFormat::new` to compile a custom strftime format,
Review Comment:
nit: the link is missing its closing bracket: [`CompiledTimeFormat::new`].
##########
parquet-variant-compute/src/type_conversion.rs:
##########
@@ -762,4 +988,580 @@ macro_rules! primitive_conversion_single_value {
)
}};
}
+use crate::variant_to_arrow::TemporalFormats;
pub(crate) use primitive_conversion_single_value;
+
+#[cfg(test)]
+mod tests {
+ use crate::type_conversion::variant_to_string;
+ use crate::variant_to_arrow::TemporalFormats;
+ use arrow::array::{
+ Array, AsArray, BooleanArray, Date32Array, Float32Array, Float64Array,
Int32Builder,
+ ListBuilder, MapBuilder, StringBuilder, Time64MicrosecondArray,
TimestampMicrosecondArray,
+ TimestampNanosecondArray,
+ };
+ use arrow::compute::{CastOptions, cast, cast_with_options};
+ use arrow::util::display::FormatOptions;
+ use arrow_schema::DataType;
+ use chrono::{DateTime, NaiveDate, NaiveTime};
+ use parquet_variant::{Variant, VariantBuilder, VariantBuilderExt};
+ use std::borrow::Cow;
+ use std::iter::zip;
+
+ #[test]
+ fn test_compatible_cast_logic_with_cast_kernel() {
Review Comment:
Following up on this one: I think it is worth doing now. The comparison
tests call `convert_variant_to_string` directly, so they never reach
`VariantToStringArrowBuilder::append_value`, which is where the top-level
object, top-level binary, and safe/strict cases above live. A helper that runs
`variant_get(.., Utf8)` with given `CastOptions` and compares against
`cast_with_options` on the equivalent Arrow array would cover the builder too,
and the list/map tests could shrink to their inputs.
`convert_variant_to_string` could then go.
##########
parquet-variant-compute/src/type_conversion.rs:
##########
@@ -720,6 +726,222 @@ pub(crate) fn variant_to_boolean(
}
}
+#[inline]
+fn write_float_to_string<F: Float>(f: F, out: &mut String) {
+ let mut buffer = ryu::Buffer::new();
+ out.push_str(buffer.format(f));
+}
+
+#[derive(Default)]
+pub(crate) struct CompiledCastFormats<'a> {
+ date: CompiledTimeFormat<'a>,
+ time: CompiledTimeFormat<'a>,
+ timestamp: CompiledTimeFormat<'a>,
+ timestamp_tz: CompiledTimeFormat<'a>,
+ null: &'a str,
+ quoted_strings: bool,
+}
+
+impl<'a> CompiledCastFormats<'a> {
+ pub(crate) fn new(options: &CastOptions<'a>) -> Self {
+ let options = &options.format_options;
+ Self {
+ date: CompiledTimeFormat::new(options.date_format()),
+ time: CompiledTimeFormat::new(options.time_format()),
+ timestamp: CompiledTimeFormat::new(options.timestamp_format()),
+ timestamp_tz:
CompiledTimeFormat::new(options.timestamp_tz_format()),
+ null: options.null(),
+ quoted_strings: options.quoted_strings(),
+ }
+ }
+}
+
+fn write_lexical_to_string<N: lexical_core::ToLexical>(out: &mut String, n: N)
{
+ // i64::FORMATTED_SIZE is the upper bound for all integer types we support
+ // (i8/i16/i32/i64). With power-of-two feature it's 128, otherwise 20.
+ // We can't use N::FORMATTED_SIZE in a generic function
(generic_const_exprs).
+ let mut buf = [0u8; i64::FORMATTED_SIZE];
+ let written = lexical_core::write(n, &mut buf);
+ // Lexical core produces valid UTF-8
+ out.push_str(unsafe { std::str::from_utf8_unchecked(written) });
+}
+
+fn format_to_arrow_err(e: FormatError) -> ArrowError {
+ match e {
+ FormatError::Arrow(e) => e,
+ FormatError::Format(e) => ArrowError::CastError(format!("Format error:
{e}")),
+ }
+}
+
+// Convert a variant into the output with the given format.
+pub(crate) fn write_variant_to_string(
+ variant: &Variant<'_, '_>,
+ formats: &CompiledCastFormats<'_>,
+ out: &mut String,
+) -> Result<bool> {
+ match variant {
+ Variant::Null => {
+ out.push_str(formats.null);
+ }
+ Variant::String(s) => {
+ if formats.quoted_strings {
+ write!(out, "{s:?}").expect("writing string to string should
never fail");
+ } else {
+ out.push_str(s);
+ }
+ }
+ Variant::ShortString(s) => {
+ if formats.quoted_strings {
+ write!(out, "{:?}", s.as_str())
+ .expect("writing string to string should never fail");
+ } else {
+ out.push_str(s);
+ }
+ }
+ Variant::BooleanTrue => {
+ out.push_str("true");
+ }
+ Variant::BooleanFalse => {
+ out.push_str("false");
+ }
+ Variant::Int8(i) => {
+ write_lexical_to_string(out, *i);
+ }
+ Variant::Int16(i) => {
+ write_lexical_to_string(out, *i);
+ }
+ Variant::Int32(i) => {
+ write_lexical_to_string(out, *i);
+ }
+ Variant::Int64(i) => {
+ write_lexical_to_string(out, *i);
+ }
+ Variant::Float(f) => {
+ write_float_to_string(*f, out);
+ }
+ Variant::Double(f) => {
+ write_float_to_string(*f, out);
+ }
+ Variant::Decimal4(d) => {
+ datatypes::write_decimal(out, d.integer(), d.scale() as _)
+ .expect("writing decimal to string should never fail");
+ }
+ Variant::Decimal8(d) => {
+ // would never fail because we write into a string here
+ datatypes::write_decimal(out, d.integer(), d.scale() as _)
+ .expect("writing decimal to string should never fail");
+ }
+ Variant::Decimal16(d) => {
+ // would never fail because we write into a string here
+ datatypes::write_decimal(out, d.integer(), d.scale() as _)
+ .expect("writing decimal to string should never fail");
+ }
+ Variant::Date(d) => {
+ write_temporal_display(out, d,
&formats.date).map_err(format_to_arrow_err)?;
+ }
+ Variant::Time(t) => {
+ write_temporal_display(out, t,
&formats.time).map_err(format_to_arrow_err)?;
+ }
+ Variant::TimestampMicros(t) => {
+ write_timestamp(
+ out,
+ t.naive_utc(),
+ "+00:00".parse().ok(),
+ &formats.timestamp_tz,
+ )
+ .map_err(format_to_arrow_err)?;
+ }
+ Variant::TimestampNtzMicros(t) => {
+ write_timestamp(out, *t, None,
&formats.timestamp).map_err(format_to_arrow_err)?;
+ }
+ Variant::TimestampNanos(t) => {
+ write_timestamp(
+ out,
+ t.naive_utc(),
+ "+00:00".parse().ok(),
+ &formats.timestamp_tz,
+ )
+ .map_err(format_to_arrow_err)?;
+ }
+ Variant::TimestampNtzNanos(t) => {
+ write_timestamp(out, *t, None,
&formats.timestamp).map_err(format_to_arrow_err)?;
+ }
+ Variant::Uuid(u) => {
+ write!(out, "{u}").expect("write to string should not fail");
+ }
+ Variant::Binary(v) => {
+ for byte in *v {
+ write!(out, "{byte:02x}").expect("write to string should never
fail");
+ }
+ }
+ Variant::List(l) => {
+ write_list_to_string(l.iter(), formats, out)?;
+ }
+ Variant::Object(o) => {
+ write_map_to_string(o.iter(), formats, out)?;
+ }
+ }
+ Ok(true)
+}
+
+fn write_list_to_string<'m, 'v>(
Review Comment:
std's `Formatter::debug_list` / `debug_map` produce exactly the `[a, b]` and
`{k: v, k2: v2}` layout that `ArrayFormatter` uses, and arrow's quoted strings
are `{value:?}`. If the writer takes `&mut dyn Write` and returns
`fmt::Result`, `write_list_to_string` and `write_map_to_string` can go:
```rust
fn write_str(out: &mut dyn Write, s: &str, quoted: bool) -> std::fmt::Result
{
if quoted { write!(out, "{s:?}") } else { out.write_str(s) }
}
// in write_variant (now `-> std::fmt::Result`)
Variant::String(s) => write_str(out, s, formats.quoted_strings),
Variant::ShortString(s) => write_str(out, s.as_str(),
formats.quoted_strings),
Variant::List(_) | Variant::Object(_) => write!(out, "{:?}", Nested(variant,
formats)),
// `debug_list` / `debug_map` produce the same `[a, b]` / `{k: v}` layout as
`ArrayFormatter`.
struct Nested<'a>(&'a Variant<'a, 'a>, &'a CompiledCastFormats<'a>);
impl std::fmt::Debug for Nested<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let formats = self.1;
match self.0 {
Variant::List(l) => f
.debug_list()
.entries(l.iter().map(|v| NestedValue(v, formats)))
.finish(),
Variant::Object(o) => f
.debug_map()
.entries(o.iter().map(|(k, v)| (Key(k,
formats.quoted_strings), NestedValue(v, formats))))
.finish(),
v => write_variant(v, formats, f),
}
}
}
struct NestedValue<'a, 'm, 'v>(Variant<'m, 'v>, &'a CompiledCastFormats<'a>);
impl std::fmt::Debug for NestedValue<'_, '_, '_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write_variant(&self.0, self.1, f)
}
}
struct Key<'a>(&'a str, bool);
impl std::fmt::Debug for Key<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write_str(f, self.0, self.1)
}
}
```
`write_variant_to_string` then wraps `write_variant` and maps `fmt::Error`
to `Cast error: Format error`, like the kernel. I tried this on top of this
head: all `parquet-variant-compute` tests pass, including the nested list/map,
custom null, quoted string, and nested binary comparisons, and it is about 70
lines shorter. Since every arm becomes a single `fmt::Result` expression, the
micros/nanos timestamp arms can also merge (`Variant::TimestampMicros(t) |
Variant::TimestampNanos(t)`), and the `.expect(..)` calls go away. It also
fixes the key quoting above.
nit: `"+00:00".parse()` runs for every timestamp value. Could the UTC `Tz`
be parsed once in `CompiledCastFormats::new`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]