sdf-jkl commented on code in PR #12303:
URL: https://github.com/apache/arrow-rs/pull/12303#discussion_r4647105400
##########
parquet-variant/src/canonicalize.rs:
##########
@@ -0,0 +1,552 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! Canonical encoding of [`Variant`] values.
+//!
+//! The Variant binary encoding allows one logical value to be written in many
ways: the metadata
+//! dictionary may hold unused or unsorted field names, an integer may use a
wider width than it
+//! needs, a decimal may carry trailing zeros, and so on. Byte-oriented
operations such as hashing
+//! for `GROUP BY` or a join therefore treat equal values as different.
+//!
+//! [`Variant::canonicalize`] rebuilds a value so that equal values share one
encoding.
+
+use std::collections::BTreeSet;
+
+use arrow_schema::ArrowError;
+
+use crate::{
+ ObjectFieldBuilder, Variant, VariantBuilder, VariantBuilderExt,
VariantDecimal4,
+ VariantDecimal8, VariantDecimal16, VariantObject,
+};
+
+impl Variant<'_, '_> {
+ /// Returns the canonical `(metadata, value)` encoding of this variant.
+ ///
+ /// Two variants that are logically equal canonicalize to identical bytes,
so the result can be
+ /// hashed or compared byte-wise. The rules follow the canonical form
proposed for Apache Spark
+ /// ([apache/spark#59669]):
+ ///
+ /// * The metadata dictionary holds only the field names the value
references, sorted by their
+ /// unsigned UTF-8 bytes, with the smallest offset width that fits.
+ /// * Object fields are written in dictionary (key) order; list elements
keep their order.
+ /// * Integers use the smallest of `INT8`, `INT16`, `INT32`, `INT64` that
holds the value.
+ /// * Decimals drop trailing zeros. An integral decimal that fits an `i64`
becomes an integer;
+ /// otherwise the smallest of `DECIMAL4`, `DECIMAL8`, `DECIMAL16` that
holds it is used.
+ /// * `FLOAT` and `DOUBLE` keep their type; `-0.0` becomes `0.0` and every
NaN becomes the
+ /// same bit pattern ([`f32::NAN`] / [`f64::NAN`]).
+ /// * Strings use the short-string encoding when they fit (at most 63
bytes).
+ /// * Booleans, `NULL`, dates, times, timestamps, binary and UUID have one
encoding already and
+ /// are copied as-is.
+ ///
+ /// Canonicalizing an already canonical value returns the same bytes.
+ ///
+ /// Note that the canonical form may change the Rust-level type of a
value: `Variant::Int64(1)`
+ /// canonicalizes to the bytes of `Variant::Int8(1)`, and
`Variant::Decimal4(1.0)` to those of
+ /// `Variant::Int8(1)`.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error if the variant is malformed or an object holds the
same field name twice.
+ ///
+ /// # Example
+ /// ```
+ /// # use parquet_variant::{Variant, VariantBuilder};
+ /// // Two builders write the same object with a different dictionary order
+ /// let mut a = VariantBuilder::new().with_field_names(["b", "a",
"unused"]);
+ /// a.new_object().with_field("a", 1i64).with_field("b", 2i8).finish();
+ /// let (am, av) = a.finish();
+ ///
+ /// let mut b = VariantBuilder::new();
+ /// b.new_object().with_field("a", 1i8).with_field("b", 2i32).finish();
+ /// let (bm, bv) = b.finish();
+ ///
+ /// assert_ne!((&am, &av), (&bm, &bv));
+ /// let a_canonical = Variant::try_new(&am,
&av).unwrap().canonicalize().unwrap();
+ /// let b_canonical = Variant::try_new(&bm,
&bv).unwrap().canonicalize().unwrap();
+ /// assert_eq!(a_canonical, b_canonical);
+ /// ```
+ ///
+ /// [apache/spark#59669]: https://github.com/apache/spark/pull/59669
+ pub fn canonicalize(&self) -> Result<(Vec<u8>, Vec<u8>), ArrowError> {
+ let mut field_names = BTreeSet::new();
+ collect_field_names(self, &mut field_names)?;
+ // `str` orders by bytes, so the set is already in unsigned UTF-8
order.
+ let mut builder =
VariantBuilder::new().with_field_names(field_names.iter().copied());
+ append_canonical(&mut builder, self)?;
+ builder.try_finish()
+ }
+}
+
+/// Collects every field name referenced anywhere inside `value`.
+fn collect_field_names<'m>(
+ value: &Variant<'m, '_>,
+ names: &mut BTreeSet<&'m str>,
+) -> Result<(), ArrowError> {
+ match value {
+ Variant::Object(object) => {
+ for field in object.iter_try() {
+ let (name, value) = field?;
+ names.insert(name);
+ collect_field_names(&value, names)?;
+ }
+ }
+ Variant::List(list) => {
+ for item in list.iter_try() {
+ collect_field_names(&item?, names)?;
+ }
+ }
+ _ => {}
+ }
+ Ok(())
+}
+
+/// Appends `value` to `builder` in canonical form. The builder's metadata
dictionary must already
+/// hold every field name `value` references, in sorted order.
+fn append_canonical<B: VariantBuilderExt>(
+ builder: &mut B,
+ value: &Variant<'_, '_>,
+) -> Result<(), ArrowError> {
+ match value {
+ // Narrowed to the smallest width that fits
+ Variant::Int8(v) => builder.append_value(canonical_int(*v as i64)),
+ Variant::Int16(v) => builder.append_value(canonical_int(*v as i64)),
+ Variant::Int32(v) => builder.append_value(canonical_int(*v as i64)),
+ Variant::Int64(v) => builder.append_value(canonical_int(*v)),
+ Variant::Decimal4(d) => {
+ builder.append_value(canonical_decimal(d.integer() as i128,
d.scale())?)
+ }
+ Variant::Decimal8(d) => {
+ builder.append_value(canonical_decimal(d.integer() as i128,
d.scale())?)
+ }
+ Variant::Decimal16(d) =>
builder.append_value(canonical_decimal(d.integer(), d.scale())?),
+ Variant::Float(v) => builder.append_value(canonical_f32(*v)),
+ Variant::Double(v) => builder.append_value(canonical_f64(*v)),
+ // `From<&str>` picks the short-string encoding whenever the string
fits
+ Variant::String(s) => builder.append_value(*s),
+ Variant::ShortString(s) => builder.append_value(s.as_str()),
+ Variant::Object(object) => append_canonical_object(builder, object)?,
+ Variant::List(list) => {
+ let mut list_builder = builder.try_new_list()?;
+ for item in list.iter_try() {
+ append_canonical(&mut list_builder, &item?)?;
+ }
+ list_builder.finish();
+ }
+ // One encoding in the spec: copy unchanged. Listed one by one so that
a type added to the
+ // spec has to be classified here.
+ Variant::Null => builder.append_value(Variant::Null),
+ Variant::BooleanTrue => builder.append_value(Variant::BooleanTrue),
+ Variant::BooleanFalse => builder.append_value(Variant::BooleanFalse),
+ Variant::Date(v) => builder.append_value(*v),
+ Variant::Time(v) => builder.append_value(*v),
+ Variant::TimestampMicros(v) =>
builder.append_value(Variant::TimestampMicros(*v)),
+ Variant::TimestampNtzMicros(v) =>
builder.append_value(Variant::TimestampNtzMicros(*v)),
+ Variant::TimestampNanos(v) =>
builder.append_value(Variant::TimestampNanos(*v)),
+ Variant::TimestampNtzNanos(v) =>
builder.append_value(Variant::TimestampNtzNanos(*v)),
Review Comment:
The spec puts micros and nanos timestamps in one equivalence class (types
12/18, and 13/19 for NTZ), and shredding converts between them, so the same
instant can arrive here in either form.
Could these arms use `builder.append_value(*v)`? The `From<DateTime<Utc>>` /
`From<NaiveDateTime>` impls already pick micros unless there are
sub-microsecond digits. The doc line listing timestamps as single-encoding
needs the same update.
##########
parquet-variant/src/canonicalize.rs:
##########
@@ -0,0 +1,552 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! Canonical encoding of [`Variant`] values.
+//!
+//! The Variant binary encoding allows one logical value to be written in many
ways: the metadata
+//! dictionary may hold unused or unsorted field names, an integer may use a
wider width than it
+//! needs, a decimal may carry trailing zeros, and so on. Byte-oriented
operations such as hashing
+//! for `GROUP BY` or a join therefore treat equal values as different.
+//!
+//! [`Variant::canonicalize`] rebuilds a value so that equal values share one
encoding.
+
+use std::collections::BTreeSet;
+
+use arrow_schema::ArrowError;
+
+use crate::{
+ ObjectFieldBuilder, Variant, VariantBuilder, VariantBuilderExt,
VariantDecimal4,
+ VariantDecimal8, VariantDecimal16, VariantObject,
+};
+
+impl Variant<'_, '_> {
+ /// Returns the canonical `(metadata, value)` encoding of this variant.
+ ///
+ /// Two variants that are logically equal canonicalize to identical bytes,
so the result can be
+ /// hashed or compared byte-wise. The rules follow the canonical form
proposed for Apache Spark
+ /// ([apache/spark#59669]):
+ ///
+ /// * The metadata dictionary holds only the field names the value
references, sorted by their
+ /// unsigned UTF-8 bytes, with the smallest offset width that fits.
+ /// * Object fields are written in dictionary (key) order; list elements
keep their order.
+ /// * Integers use the smallest of `INT8`, `INT16`, `INT32`, `INT64` that
holds the value.
+ /// * Decimals drop trailing zeros. An integral decimal that fits an `i64`
becomes an integer;
+ /// otherwise the smallest of `DECIMAL4`, `DECIMAL8`, `DECIMAL16` that
holds it is used.
+ /// * `FLOAT` and `DOUBLE` keep their type; `-0.0` becomes `0.0` and every
NaN becomes the
+ /// same bit pattern ([`f32::NAN`] / [`f64::NAN`]).
+ /// * Strings use the short-string encoding when they fit (at most 63
bytes).
+ /// * Booleans, `NULL`, dates, times, timestamps, binary and UUID have one
encoding already and
+ /// are copied as-is.
+ ///
+ /// Canonicalizing an already canonical value returns the same bytes.
+ ///
+ /// Note that the canonical form may change the Rust-level type of a
value: `Variant::Int64(1)`
+ /// canonicalizes to the bytes of `Variant::Int8(1)`, and
`Variant::Decimal4(1.0)` to those of
+ /// `Variant::Int8(1)`.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error if the variant is malformed or an object holds the
same field name twice.
+ ///
+ /// # Example
+ /// ```
+ /// # use parquet_variant::{Variant, VariantBuilder};
+ /// // Two builders write the same object with a different dictionary order
+ /// let mut a = VariantBuilder::new().with_field_names(["b", "a",
"unused"]);
+ /// a.new_object().with_field("a", 1i64).with_field("b", 2i8).finish();
+ /// let (am, av) = a.finish();
+ ///
+ /// let mut b = VariantBuilder::new();
+ /// b.new_object().with_field("a", 1i8).with_field("b", 2i32).finish();
+ /// let (bm, bv) = b.finish();
+ ///
+ /// assert_ne!((&am, &av), (&bm, &bv));
+ /// let a_canonical = Variant::try_new(&am,
&av).unwrap().canonicalize().unwrap();
+ /// let b_canonical = Variant::try_new(&bm,
&bv).unwrap().canonicalize().unwrap();
+ /// assert_eq!(a_canonical, b_canonical);
+ /// ```
+ ///
+ /// [apache/spark#59669]: https://github.com/apache/spark/pull/59669
+ pub fn canonicalize(&self) -> Result<(Vec<u8>, Vec<u8>), ArrowError> {
+ let mut field_names = BTreeSet::new();
+ collect_field_names(self, &mut field_names)?;
+ // `str` orders by bytes, so the set is already in unsigned UTF-8
order.
+ let mut builder =
VariantBuilder::new().with_field_names(field_names.iter().copied());
+ append_canonical(&mut builder, self)?;
+ builder.try_finish()
+ }
+}
+
+/// Collects every field name referenced anywhere inside `value`.
+fn collect_field_names<'m>(
+ value: &Variant<'m, '_>,
+ names: &mut BTreeSet<&'m str>,
+) -> Result<(), ArrowError> {
+ match value {
+ Variant::Object(object) => {
+ for field in object.iter_try() {
+ let (name, value) = field?;
+ names.insert(name);
+ collect_field_names(&value, names)?;
+ }
+ }
+ Variant::List(list) => {
+ for item in list.iter_try() {
+ collect_field_names(&item?, names)?;
+ }
+ }
+ _ => {}
+ }
+ Ok(())
+}
+
+/// Appends `value` to `builder` in canonical form. The builder's metadata
dictionary must already
+/// hold every field name `value` references, in sorted order.
+fn append_canonical<B: VariantBuilderExt>(
+ builder: &mut B,
+ value: &Variant<'_, '_>,
+) -> Result<(), ArrowError> {
+ match value {
+ // Narrowed to the smallest width that fits
+ Variant::Int8(v) => builder.append_value(canonical_int(*v as i64)),
+ Variant::Int16(v) => builder.append_value(canonical_int(*v as i64)),
+ Variant::Int32(v) => builder.append_value(canonical_int(*v as i64)),
+ Variant::Int64(v) => builder.append_value(canonical_int(*v)),
+ Variant::Decimal4(d) => {
+ builder.append_value(canonical_decimal(d.integer() as i128,
d.scale())?)
+ }
+ Variant::Decimal8(d) => {
+ builder.append_value(canonical_decimal(d.integer() as i128,
d.scale())?)
+ }
+ Variant::Decimal16(d) =>
builder.append_value(canonical_decimal(d.integer(), d.scale())?),
+ Variant::Float(v) => builder.append_value(canonical_f32(*v)),
+ Variant::Double(v) => builder.append_value(canonical_f64(*v)),
+ // `From<&str>` picks the short-string encoding whenever the string
fits
+ Variant::String(s) => builder.append_value(*s),
+ Variant::ShortString(s) => builder.append_value(s.as_str()),
+ Variant::Object(object) => append_canonical_object(builder, object)?,
+ Variant::List(list) => {
+ let mut list_builder = builder.try_new_list()?;
+ for item in list.iter_try() {
+ append_canonical(&mut list_builder, &item?)?;
+ }
+ list_builder.finish();
+ }
+ // One encoding in the spec: copy unchanged. Listed one by one so that
a type added to the
+ // spec has to be classified here.
+ Variant::Null => builder.append_value(Variant::Null),
+ Variant::BooleanTrue => builder.append_value(Variant::BooleanTrue),
+ Variant::BooleanFalse => builder.append_value(Variant::BooleanFalse),
+ Variant::Date(v) => builder.append_value(*v),
+ Variant::Time(v) => builder.append_value(*v),
+ Variant::TimestampMicros(v) =>
builder.append_value(Variant::TimestampMicros(*v)),
+ Variant::TimestampNtzMicros(v) =>
builder.append_value(Variant::TimestampNtzMicros(*v)),
+ Variant::TimestampNanos(v) =>
builder.append_value(Variant::TimestampNanos(*v)),
+ Variant::TimestampNtzNanos(v) =>
builder.append_value(Variant::TimestampNtzNanos(*v)),
+ Variant::Binary(v) => builder.append_value(*v),
+ Variant::Uuid(v) => builder.append_value(*v),
+ }
+ Ok(())
+}
+
+/// Appends an object with its fields sorted by key, each value in canonical
form.
+fn append_canonical_object<'m, 'v, B: VariantBuilderExt>(
+ builder: &mut B,
+ object: &VariantObject<'m, 'v>,
+) -> Result<(), ArrowError> {
+ let mut fields: Vec<(&'m str, Variant<'m, 'v>)> =
+ object.iter_try().collect::<Result<_, _>>()?;
+ fields.sort_by(|(a, _), (b, _)| a.as_bytes().cmp(b.as_bytes()));
+ if let Some(pair) = fields.windows(2).find(|pair| pair[0].0 == pair[1].0) {
+ return Err(ArrowError::InvalidArgumentError(format!(
+ "Cannot canonicalize object with duplicate field name '{}'",
+ pair[0].0
+ )));
+ }
Review Comment:
Full validation already rejects unsorted or repeated field names, so this
sort and duplicate check never fire on validated input
(`duplicate_field_names_are_rejected` needs `Variant::new` to reach them).
Could `canonicalize` start with `self.clone().with_full_validation()?`
instead? Fields then iterate in key order, `iter()` replaces `iter_try()`, and
this becomes a plain loop. It's free for values from `try_new`, since
validation is cached. I tried it locally: 17 lines added, 35 removed, tests and
clippy pass.
##########
parquet-variant/src/canonicalize.rs:
##########
@@ -0,0 +1,552 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! Canonical encoding of [`Variant`] values.
+//!
+//! The Variant binary encoding allows one logical value to be written in many
ways: the metadata
+//! dictionary may hold unused or unsorted field names, an integer may use a
wider width than it
+//! needs, a decimal may carry trailing zeros, and so on. Byte-oriented
operations such as hashing
+//! for `GROUP BY` or a join therefore treat equal values as different.
+//!
+//! [`Variant::canonicalize`] rebuilds a value so that equal values share one
encoding.
+
+use std::collections::BTreeSet;
+
+use arrow_schema::ArrowError;
+
+use crate::{
+ ObjectFieldBuilder, Variant, VariantBuilder, VariantBuilderExt,
VariantDecimal4,
+ VariantDecimal8, VariantDecimal16, VariantObject,
+};
+
+impl Variant<'_, '_> {
+ /// Returns the canonical `(metadata, value)` encoding of this variant.
+ ///
+ /// Two variants that are logically equal canonicalize to identical bytes,
so the result can be
+ /// hashed or compared byte-wise. The rules follow the canonical form
proposed for Apache Spark
Review Comment:
no longer proposed, but used in Spark now
##########
parquet-variant/src/canonicalize.rs:
##########
@@ -0,0 +1,552 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! Canonical encoding of [`Variant`] values.
+//!
+//! The Variant binary encoding allows one logical value to be written in many
ways: the metadata
+//! dictionary may hold unused or unsorted field names, an integer may use a
wider width than it
+//! needs, a decimal may carry trailing zeros, and so on. Byte-oriented
operations such as hashing
+//! for `GROUP BY` or a join therefore treat equal values as different.
+//!
+//! [`Variant::canonicalize`] rebuilds a value so that equal values share one
encoding.
+
+use std::collections::BTreeSet;
+
+use arrow_schema::ArrowError;
+
+use crate::{
+ ObjectFieldBuilder, Variant, VariantBuilder, VariantBuilderExt,
VariantDecimal4,
+ VariantDecimal8, VariantDecimal16, VariantObject,
+};
+
+impl Variant<'_, '_> {
+ /// Returns the canonical `(metadata, value)` encoding of this variant.
+ ///
+ /// Two variants that are logically equal canonicalize to identical bytes,
so the result can be
+ /// hashed or compared byte-wise. The rules follow the canonical form
proposed for Apache Spark
+ /// ([apache/spark#59669]):
+ ///
+ /// * The metadata dictionary holds only the field names the value
references, sorted by their
Review Comment:
This is also kinda an explanation of equivalence classes
##########
parquet-variant/src/canonicalize.rs:
##########
@@ -0,0 +1,552 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! Canonical encoding of [`Variant`] values.
+//!
+//! The Variant binary encoding allows one logical value to be written in many
ways: the metadata
+//! dictionary may hold unused or unsorted field names, an integer may use a
wider width than it
+//! needs, a decimal may carry trailing zeros, and so on. Byte-oriented
operations such as hashing
+//! for `GROUP BY` or a join therefore treat equal values as different.
+//!
+//! [`Variant::canonicalize`] rebuilds a value so that equal values share one
encoding.
+
+use std::collections::BTreeSet;
+
+use arrow_schema::ArrowError;
+
+use crate::{
+ ObjectFieldBuilder, Variant, VariantBuilder, VariantBuilderExt,
VariantDecimal4,
+ VariantDecimal8, VariantDecimal16, VariantObject,
+};
+
+impl Variant<'_, '_> {
+ /// Returns the canonical `(metadata, value)` encoding of this variant.
+ ///
+ /// Two variants that are logically equal canonicalize to identical bytes,
so the result can be
+ /// hashed or compared byte-wise. The rules follow the canonical form
proposed for Apache Spark
+ /// ([apache/spark#59669]):
+ ///
+ /// * The metadata dictionary holds only the field names the value
references, sorted by their
+ /// unsigned UTF-8 bytes, with the smallest offset width that fits.
+ /// * Object fields are written in dictionary (key) order; list elements
keep their order.
+ /// * Integers use the smallest of `INT8`, `INT16`, `INT32`, `INT64` that
holds the value.
+ /// * Decimals drop trailing zeros. An integral decimal that fits an `i64`
becomes an integer;
+ /// otherwise the smallest of `DECIMAL4`, `DECIMAL8`, `DECIMAL16` that
holds it is used.
+ /// * `FLOAT` and `DOUBLE` keep their type; `-0.0` becomes `0.0` and every
NaN becomes the
+ /// same bit pattern ([`f32::NAN`] / [`f64::NAN`]).
+ /// * Strings use the short-string encoding when they fit (at most 63
bytes).
+ /// * Booleans, `NULL`, dates, times, timestamps, binary and UUID have one
encoding already and
+ /// are copied as-is.
+ ///
+ /// Canonicalizing an already canonical value returns the same bytes.
+ ///
+ /// Note that the canonical form may change the Rust-level type of a
value: `Variant::Int64(1)`
+ /// canonicalizes to the bytes of `Variant::Int8(1)`, and
`Variant::Decimal4(1.0)` to those of
+ /// `Variant::Int8(1)`.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error if the variant is malformed or an object holds the
same field name twice.
+ ///
+ /// # Example
+ /// ```
+ /// # use parquet_variant::{Variant, VariantBuilder};
+ /// // Two builders write the same object with a different dictionary order
+ /// let mut a = VariantBuilder::new().with_field_names(["b", "a",
"unused"]);
+ /// a.new_object().with_field("a", 1i64).with_field("b", 2i8).finish();
+ /// let (am, av) = a.finish();
+ ///
+ /// let mut b = VariantBuilder::new();
+ /// b.new_object().with_field("a", 1i8).with_field("b", 2i32).finish();
+ /// let (bm, bv) = b.finish();
+ ///
+ /// assert_ne!((&am, &av), (&bm, &bv));
+ /// let a_canonical = Variant::try_new(&am,
&av).unwrap().canonicalize().unwrap();
+ /// let b_canonical = Variant::try_new(&bm,
&bv).unwrap().canonicalize().unwrap();
+ /// assert_eq!(a_canonical, b_canonical);
+ /// ```
+ ///
+ /// [apache/spark#59669]: https://github.com/apache/spark/pull/59669
+ pub fn canonicalize(&self) -> Result<(Vec<u8>, Vec<u8>), ArrowError> {
+ let mut field_names = BTreeSet::new();
+ collect_field_names(self, &mut field_names)?;
+ // `str` orders by bytes, so the set is already in unsigned UTF-8
order.
+ let mut builder =
VariantBuilder::new().with_field_names(field_names.iter().copied());
+ append_canonical(&mut builder, self)?;
+ builder.try_finish()
+ }
+}
+
+/// Collects every field name referenced anywhere inside `value`.
+fn collect_field_names<'m>(
+ value: &Variant<'m, '_>,
+ names: &mut BTreeSet<&'m str>,
+) -> Result<(), ArrowError> {
+ match value {
+ Variant::Object(object) => {
+ for field in object.iter_try() {
+ let (name, value) = field?;
+ names.insert(name);
+ collect_field_names(&value, names)?;
+ }
+ }
+ Variant::List(list) => {
+ for item in list.iter_try() {
+ collect_field_names(&item?, names)?;
+ }
+ }
+ _ => {}
+ }
+ Ok(())
+}
+
+/// Appends `value` to `builder` in canonical form. The builder's metadata
dictionary must already
+/// hold every field name `value` references, in sorted order.
+fn append_canonical<B: VariantBuilderExt>(
+ builder: &mut B,
+ value: &Variant<'_, '_>,
+) -> Result<(), ArrowError> {
Review Comment:
These primitive arms are effectively the spec's equivalence classes in code:
one representative encoding per value in each class.
Could they live on their own, e.g. a `pub(crate) fn
canonical_primitive(&self) -> Variant` in a new `variant/equivalence.rs`, with
`append_canonical` keeping only the object and list walk?
That gives the class rules one home to reuse later (the JSON crate's integer
narrowing in `variant_from_number` is the same rule), and the exhaustive match
still forces new spec types to be classified.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]