alamb commented on code in PR #8122:
URL: https://github.com/apache/arrow-rs/pull/8122#discussion_r2274138690
##########
parquet-variant-compute/src/variant_get/mod.rs:
##########
@@ -15,50 +15,208 @@
// specific language governing permissions and limitations
// under the License.
use arrow::{
- array::{Array, ArrayRef},
+ array::{self, Array, ArrayRef, BinaryViewArray, StructArray},
compute::CastOptions,
error::Result,
};
-use arrow_schema::{ArrowError, FieldRef};
-use parquet_variant::VariantPath;
+use arrow_schema::{ArrowError, DataType, FieldRef};
+use parquet_variant::{VariantPath, VariantPathElement};
use crate::variant_array::ShreddingState;
-use crate::variant_get::output::instantiate_output_builder;
-use crate::VariantArray;
+use crate::{variant_array::ShreddedVariantFieldArray, VariantArray};
+
+use std::sync::Arc;
mod output;
+pub(crate) enum ShreddedPathStep<'a> {
Review Comment:
yes, this makes lots of sense
##########
parquet-variant-compute/src/variant_array.rs:
##########
@@ -229,107 +328,138 @@ pub enum ShreddingState {
// TODO: add missing state where there is neither value nor typed_value
// Missing { metadata: BinaryViewArray },
/// This variant has no typed_value field
- Unshredded {
- metadata: BinaryViewArray,
- value: BinaryViewArray,
- },
+ Unshredded { value: BinaryViewArray },
/// This variant has a typed_value field and no value field
/// meaning it is the shredded type
- Typed {
- metadata: BinaryViewArray,
- typed_value: ArrayRef,
- },
- /// Partially shredded:
- /// * value is an object
- /// * typed_value is a shredded object.
+ PerfectlyShredded { typed_value: ArrayRef },
+ /// Imperfectly shredded: Shredded values reside in `typed_value` while
those that failed to
+ /// shred reside in `value`. Missing field values are NULL in both
columns, while NULL primitive
+ /// values have NULL `typed_value` and `Variant::Null` in `value`.
///
- /// Note the spec says "Writers must not produce data where both value and
- /// typed_value are non-null, unless the Variant value is an object."
- PartiallyShredded {
- metadata: BinaryViewArray,
+ /// NOTE: A partially shredded struct is a special kind of imperfect
shredding, where
+ /// `typed_value` and `value` are both non-NULL. The `typed_value` is a
struct containing the
+ /// subset of fields for which shredding was attempted (each field will
then have its own value
+ /// and/or typed_value sub-fields that indicate how shredding actually
turned out). Meanwhile,
+ /// the `value` is a variant object containing the subset of fields for
which shredding was
+ /// not even attempted.
+ ImperfectlyShredded {
value: BinaryViewArray,
typed_value: ArrayRef,
},
}
impl ShreddingState {
/// try to create a new `ShreddingState` from the given fields
- pub fn try_new(
- metadata: BinaryViewArray,
- value: Option<BinaryViewArray>,
- typed_value: Option<ArrayRef>,
- ) -> Result<Self, ArrowError> {
- match (metadata, value, typed_value) {
- (metadata, Some(value), Some(typed_value)) =>
Ok(Self::PartiallyShredded {
- metadata,
- value,
- typed_value,
- }),
- (metadata, Some(value), None) => Ok(Self::Unshredded { metadata,
value }),
- (metadata, None, Some(typed_value)) => Ok(Self::Typed {
- metadata,
- typed_value,
- }),
- (_metadata_field, None, None) =>
Err(ArrowError::InvalidArgumentError(String::from(
+ pub fn try_new(inner: &StructArray) -> Result<Self, ArrowError> {
+ // Note the specification allows for any order so we must search by
name
+
+ // Find the value field, if present
+ let value = inner
+ .column_by_name("value")
+ .map(|v| {
+ v.as_binary_view_opt().ok_or_else(|| {
+ ArrowError::NotYetImplemented(format!(
+ "VariantArray 'value' field must be BinaryView, got
{}",
+ v.data_type()
+ ))
+ })
+ })
+ .transpose()?
+ .cloned();
+
+ // Find the typed_value field, if present
+ let typed_value = inner.column_by_name("typed_value").cloned();
+
+ match (value, typed_value) {
+ (Some(value), Some(typed_value)) => {
+ Ok(Self::ImperfectlyShredded { value, typed_value })
+ }
+ (Some(value), None) => Ok(Self::Unshredded { value }),
+ (None, Some(typed_value)) => Ok(Self::PerfectlyShredded {
typed_value }),
+ (None, None) => Err(ArrowError::InvalidArgumentError(String::from(
"VariantArray has neither value nor typed_value field",
))),
}
}
- /// Return a reference to the metadata field
- pub fn metadata_field(&self) -> &BinaryViewArray {
- match self {
- ShreddingState::Unshredded { metadata, .. } => metadata,
- ShreddingState::Typed { metadata, .. } => metadata,
- ShreddingState::PartiallyShredded { metadata, .. } => metadata,
- }
- }
-
/// Return a reference to the value field, if present
pub fn value_field(&self) -> Option<&BinaryViewArray> {
match self {
ShreddingState::Unshredded { value, .. } => Some(value),
- ShreddingState::Typed { .. } => None,
- ShreddingState::PartiallyShredded { value, .. } => Some(value),
+ ShreddingState::PerfectlyShredded { .. } => None,
+ ShreddingState::ImperfectlyShredded { value, .. } => Some(value),
}
}
/// Return a reference to the typed_value field, if present
pub fn typed_value_field(&self) -> Option<&ArrayRef> {
match self {
ShreddingState::Unshredded { .. } => None,
- ShreddingState::Typed { typed_value, .. } => Some(typed_value),
- ShreddingState::PartiallyShredded { typed_value, .. } =>
Some(typed_value),
+ ShreddingState::PerfectlyShredded { typed_value, .. } =>
Some(typed_value),
+ ShreddingState::ImperfectlyShredded { typed_value, .. } =>
Some(typed_value),
}
}
/// Slice all the underlying arrays
pub fn slice(&self, offset: usize, length: usize) -> Self {
match self {
- ShreddingState::Unshredded { metadata, value } =>
ShreddingState::Unshredded {
- metadata: metadata.slice(offset, length),
- value: value.slice(offset, length),
- },
- ShreddingState::Typed {
- metadata,
- typed_value,
- } => ShreddingState::Typed {
- metadata: metadata.slice(offset, length),
- typed_value: typed_value.slice(offset, length),
- },
- ShreddingState::PartiallyShredded {
- metadata,
- value,
- typed_value,
- } => ShreddingState::PartiallyShredded {
- metadata: metadata.slice(offset, length),
+ ShreddingState::Unshredded { value } => ShreddingState::Unshredded
{
value: value.slice(offset, length),
- typed_value: typed_value.slice(offset, length),
},
+ ShreddingState::PerfectlyShredded { typed_value } => {
+ ShreddingState::PerfectlyShredded {
+ typed_value: typed_value.slice(offset, length),
+ }
+ }
+ ShreddingState::ImperfectlyShredded { value, typed_value } => {
+ ShreddingState::ImperfectlyShredded {
+ value: value.slice(offset, length),
+ typed_value: typed_value.slice(offset, length),
+ }
+ }
}
}
}
+/// Builds struct arrays from component fields
+///
+/// TODO: move to arrow crate
+#[derive(Debug, Default, Clone)]
+pub struct StructArrayBuilder {
Review Comment:
Yeah, there is already a class called `StructBuilder` which helps build
struct arrays row by row.
https://docs.rs/arrow/latest/arrow/array/struct.StructBuilder.html
Maybe we could add `StructArrayBuilder` to the same location. With
sufficient documentation examples, I think it would be straightforward to add
Is this something it would help if I looked into?
##########
parquet-variant-compute/src/variant_get/mod.rs:
##########
@@ -15,50 +15,208 @@
// specific language governing permissions and limitations
// under the License.
use arrow::{
- array::{Array, ArrayRef},
+ array::{self, Array, ArrayRef, BinaryViewArray, StructArray},
compute::CastOptions,
error::Result,
};
-use arrow_schema::{ArrowError, FieldRef};
-use parquet_variant::VariantPath;
+use arrow_schema::{ArrowError, DataType, FieldRef};
+use parquet_variant::{VariantPath, VariantPathElement};
use crate::variant_array::ShreddingState;
-use crate::variant_get::output::instantiate_output_builder;
-use crate::VariantArray;
+use crate::{variant_array::ShreddedVariantFieldArray, VariantArray};
+
+use std::sync::Arc;
mod output;
+pub(crate) enum ShreddedPathStep<'a> {
+ /// Path step succeeded, return the new shredding state
+ Success(&'a ShreddingState),
+ /// The path element is not present in the `typed_value` column and there
is no `value` column,
+ /// so we we know it does not exist. It, and all paths under it, are
all-NULL.
+ Missing,
+ /// The path element is not present in the `typed_value` and must be
retrieved from the `value`
+ /// column instead. The caller should be prepared to handle any value,
including the requested
+ /// type, an arbitrary "wrong" type, or `Variant::Null`.
+ NotShredded,
+}
+
+/// Given a shredded variant field -- a `(value?, typed_value?)` pair -- try
to take one path step
+/// deeper. For a `VariantPathElement::Field`, the step fails if there is no
`typed_value` at this
+/// level, or if `typed_value` is not a struct, or if the requested field name
does not exist.
+///
+/// TODO: Support `VariantPathElement::Index`? It wouldn't be easy, and maybe
not even possible.
+pub(crate) fn follow_shredded_path_element<'a>(
+ shredding_state: &'a ShreddingState,
+ path_element: &VariantPathElement<'_>,
+) -> Result<ShreddedPathStep<'a>> {
+ // If the requested path element 's not present in `typed_value`, and
`value` is missing, then
+ // we know it does not exist; it, and all paths under it, are all-NULL.
+ let missing_path_step = || {
+ if shredding_state.value_field().is_none() {
+ ShreddedPathStep::Missing
+ } else {
+ ShreddedPathStep::NotShredded
+ }
+ };
+
+ let Some(typed_value) = shredding_state.typed_value_field() else {
+ return Ok(missing_path_step());
+ };
+
+ match path_element {
+ VariantPathElement::Field { name } => {
+ // Try to step into the requested field name of a struct.
+ let Some(field) = typed_value
+ .as_any()
+ .downcast_ref::<StructArray>()
+ .and_then(|typed_value| typed_value.column_by_name(name))
+ else {
+ return Ok(missing_path_step());
+ };
+
+ let field = field
+ .as_any()
+ .downcast_ref::<ShreddedVariantFieldArray>()
Review Comment:
If we could somehow figure out to make this be `VariantArray` rather than
`ShreddedVariantFieldArray` I think that would be the most elegant /
understandable
##########
parquet-variant-compute/src/variant_array.rs:
##########
@@ -192,7 +202,96 @@ impl VariantArray {
/// Return a reference to the metadata field of the [`StructArray`]
pub fn metadata_field(&self) -> &BinaryViewArray {
- self.shredding_state.metadata_field()
+ &self.metadata
+ }
+
+ /// Return a reference to the value field of the `StructArray`
+ pub fn value_field(&self) -> Option<&BinaryViewArray> {
+ self.shredding_state.value_field()
+ }
+
+ /// Return a reference to the typed_value field of the `StructArray`, if
present
+ pub fn typed_value_field(&self) -> Option<&ArrayRef> {
+ self.shredding_state.typed_value_field()
+ }
+}
+
+/// One shredded field of a partially or prefectly shredded variant. For
example, suppose the
+/// shredding schema for variant `v` treats it as an object with a single
field `a`, where `a` is
+/// itself a struct with the single field `b` of type INT. Then the physical
layout of the column
+/// is:
+///
+/// ```text
+/// v: VARIANT {
+/// metadata: BINARY,
+/// value: BINARY,
+/// typed_value: STRUCT {
+/// a: SHREDDED_VARIANT_FIELD {
+/// value: BINARY,
+/// typed_value: STRUCT {
+/// a: SHREDDED_VARIANT_FIELD {
+/// value: BINARY,
+/// typed_value: INT,
+/// },
+/// },
+/// },
+/// },
+/// }
+/// ```
+///
+/// In the above, each row of `v.value` is either a variant value (shredding
failed, `v` was not an
Review Comment:
In going through this explanation, I think it might be helpful to make some
specific examples of valid values. Maybe we can add them to the docs too --
something like
## Fields could be shredded
```json
{
"a" : {
"b": 42 <-- stored as an integer in typed_value field
}
}
```
## Field could not be shredded
```json
{
"a" : {
"b": "foo" <-- stored as a variant
}
}
```
## Extra non shredded fields
```json
{
"a" : {
"b": 42, <-- stored as an integer in typed_value field
"z": "foo" <-- stored as a variant in sub field?
}
}
```
But I am not sure about the last example, to be honest
##########
parquet-variant-compute/src/variant_get/mod.rs:
##########
@@ -15,50 +15,208 @@
// specific language governing permissions and limitations
// under the License.
use arrow::{
- array::{Array, ArrayRef},
+ array::{self, Array, ArrayRef, BinaryViewArray, StructArray},
compute::CastOptions,
error::Result,
};
-use arrow_schema::{ArrowError, FieldRef};
-use parquet_variant::VariantPath;
+use arrow_schema::{ArrowError, DataType, FieldRef};
+use parquet_variant::{VariantPath, VariantPathElement};
use crate::variant_array::ShreddingState;
-use crate::variant_get::output::instantiate_output_builder;
-use crate::VariantArray;
+use crate::{variant_array::ShreddedVariantFieldArray, VariantArray};
+
+use std::sync::Arc;
mod output;
+pub(crate) enum ShreddedPathStep<'a> {
+ /// Path step succeeded, return the new shredding state
+ Success(&'a ShreddingState),
+ /// The path element is not present in the `typed_value` column and there
is no `value` column,
+ /// so we we know it does not exist. It, and all paths under it, are
all-NULL.
+ Missing,
+ /// The path element is not present in the `typed_value` and must be
retrieved from the `value`
+ /// column instead. The caller should be prepared to handle any value,
including the requested
+ /// type, an arbitrary "wrong" type, or `Variant::Null`.
+ NotShredded,
+}
+
+/// Given a shredded variant field -- a `(value?, typed_value?)` pair -- try
to take one path step
+/// deeper. For a `VariantPathElement::Field`, the step fails if there is no
`typed_value` at this
+/// level, or if `typed_value` is not a struct, or if the requested field name
does not exist.
+///
+/// TODO: Support `VariantPathElement::Index`? It wouldn't be easy, and maybe
not even possible.
Review Comment:
Yeah, I think we would likely have to copy the relevant bytes to a new arary
##########
parquet-variant-compute/src/variant_get/output/struct_output.rs:
##########
@@ -0,0 +1,371 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::{ArrayRef, AsArray as _, NullBufferBuilder};
+use arrow::datatypes;
+use arrow::datatypes::{ArrowPrimitiveType, FieldRef};
+use arrow::error::{ArrowError, Result};
+use parquet_variant::{Variant, VariantObject, VariantPath};
+
+use std::sync::Arc;
+
+#[allow(unused)]
+pub(crate) fn make_shredding_row_builder(
+ //metadata: &BinaryViewArray,
+ path: VariantPath<'_>,
+ data_type: Option<&datatypes::DataType>,
+) -> Result<Box<dyn VariantShreddingRowBuilder>> {
+ todo!() // wire it all up!
+}
+
+/// Builder for shredding variant values into strongly typed Arrow arrays.
+///
+/// Useful for variant_get kernels that need to extract specific paths from
variant values, possibly
+/// with casting of leaf values to specific types.
+#[allow(unused)]
+pub(crate) trait VariantShreddingRowBuilder {
+ fn append_null(&mut self) -> Result<()>;
+
+ fn append_value(&mut self, value: &Variant<'_, '_>) -> Result<bool>;
+
+ fn finish(&mut self) -> Result<ArrayRef>;
+}
+
+/// A thin wrapper whose only job is to extract a specific path from a variant
value and pass the
+/// result to a nested builder.
+#[allow(unused)]
+struct VariantPathRowBuilder<'a, T: VariantShreddingRowBuilder> {
+ builder: T,
+ path: VariantPath<'a>,
+}
+
+impl<T: VariantShreddingRowBuilder> VariantShreddingRowBuilder for
VariantPathRowBuilder<'_, T> {
+ fn append_null(&mut self) -> Result<()> {
+ self.builder.append_null()
+ }
+
+ fn append_value(&mut self, value: &Variant<'_, '_>) -> Result<bool> {
+ if let Some(v) = value.get_path(&self.path) {
+ self.builder.append_value(&v)
+ } else {
+ self.builder.append_null()?;
+ Ok(false)
+ }
+ }
+ fn finish(&mut self) -> Result<ArrayRef> {
+ self.builder.finish()
+ }
+}
+
+/// Helper trait for converting `Variant` values to arrow primitive values.
+#[allow(unused)]
+trait VariantAsPrimitive<T: ArrowPrimitiveType> {
+ fn as_primitive(&self) -> Option<T::Native>;
+}
+impl VariantAsPrimitive<datatypes::Int32Type> for Variant<'_, '_> {
+ fn as_primitive(&self) -> Option<i32> {
+ self.as_int32()
+ }
+}
+impl VariantAsPrimitive<datatypes::Float64Type> for Variant<'_, '_> {
+ fn as_primitive(&self) -> Option<f64> {
+ self.as_f64()
+ }
+}
+
+/// Builder for shredding variant values to primitive values
+#[allow(unused)]
+struct PrimitiveVariantShreddingRowBuilder<T: ArrowPrimitiveType> {
+ builder: arrow::array::PrimitiveBuilder<T>,
+}
+
+impl<T> VariantShreddingRowBuilder for PrimitiveVariantShreddingRowBuilder<T>
+where
+ T: ArrowPrimitiveType,
+ for<'m, 'v> Variant<'m, 'v>: VariantAsPrimitive<T>,
+{
+ fn append_null(&mut self) -> Result<()> {
+ self.builder.append_null();
+ Ok(())
+ }
+
+ fn append_value(&mut self, value: &Variant<'_, '_>) -> Result<bool> {
+ if let Some(v) = value.as_primitive() {
Review Comment:
this is very clever
##########
parquet-variant-compute/src/variant_array.rs:
##########
@@ -229,107 +328,138 @@ pub enum ShreddingState {
// TODO: add missing state where there is neither value nor typed_value
// Missing { metadata: BinaryViewArray },
/// This variant has no typed_value field
- Unshredded {
- metadata: BinaryViewArray,
- value: BinaryViewArray,
- },
+ Unshredded { value: BinaryViewArray },
/// This variant has a typed_value field and no value field
/// meaning it is the shredded type
- Typed {
- metadata: BinaryViewArray,
- typed_value: ArrayRef,
- },
- /// Partially shredded:
- /// * value is an object
- /// * typed_value is a shredded object.
+ PerfectlyShredded { typed_value: ArrayRef },
Review Comment:
My understanding was that if `typed_value` was non null, it contains the
value and an implementation doesn't even have to check the `value` field
If `typed_value` is null, then we have to check the `value` field and
produce a null in the output when needed
##########
parquet-variant-compute/src/variant_get/mod.rs:
##########
@@ -15,50 +15,208 @@
// specific language governing permissions and limitations
// under the License.
use arrow::{
- array::{Array, ArrayRef},
+ array::{self, Array, ArrayRef, BinaryViewArray, StructArray},
compute::CastOptions,
error::Result,
};
-use arrow_schema::{ArrowError, FieldRef};
-use parquet_variant::VariantPath;
+use arrow_schema::{ArrowError, DataType, FieldRef};
+use parquet_variant::{VariantPath, VariantPathElement};
use crate::variant_array::ShreddingState;
-use crate::variant_get::output::instantiate_output_builder;
-use crate::VariantArray;
+use crate::{variant_array::ShreddedVariantFieldArray, VariantArray};
+
+use std::sync::Arc;
mod output;
+pub(crate) enum ShreddedPathStep<'a> {
+ /// Path step succeeded, return the new shredding state
+ Success(&'a ShreddingState),
+ /// The path element is not present in the `typed_value` column and there
is no `value` column,
+ /// so we we know it does not exist. It, and all paths under it, are
all-NULL.
+ Missing,
+ /// The path element is not present in the `typed_value` and must be
retrieved from the `value`
+ /// column instead. The caller should be prepared to handle any value,
including the requested
+ /// type, an arbitrary "wrong" type, or `Variant::Null`.
+ NotShredded,
+}
+
+/// Given a shredded variant field -- a `(value?, typed_value?)` pair -- try
to take one path step
+/// deeper. For a `VariantPathElement::Field`, the step fails if there is no
`typed_value` at this
+/// level, or if `typed_value` is not a struct, or if the requested field name
does not exist.
+///
+/// TODO: Support `VariantPathElement::Index`? It wouldn't be easy, and maybe
not even possible.
+pub(crate) fn follow_shredded_path_element<'a>(
+ shredding_state: &'a ShreddingState,
+ path_element: &VariantPathElement<'_>,
+) -> Result<ShreddedPathStep<'a>> {
+ // If the requested path element 's not present in `typed_value`, and
`value` is missing, then
+ // we know it does not exist; it, and all paths under it, are all-NULL.
+ let missing_path_step = || {
+ if shredding_state.value_field().is_none() {
+ ShreddedPathStep::Missing
+ } else {
+ ShreddedPathStep::NotShredded
+ }
+ };
+
+ let Some(typed_value) = shredding_state.typed_value_field() else {
+ return Ok(missing_path_step());
+ };
+
+ match path_element {
+ VariantPathElement::Field { name } => {
+ // Try to step into the requested field name of a struct.
+ let Some(field) = typed_value
+ .as_any()
+ .downcast_ref::<StructArray>()
+ .and_then(|typed_value| typed_value.column_by_name(name))
+ else {
+ return Ok(missing_path_step());
+ };
+
+ let field = field
+ .as_any()
+ .downcast_ref::<ShreddedVariantFieldArray>()
+ .ok_or_else(|| {
+ // TODO: Should we blow up? Or just end the traversal and
let the normal
+ // variant pathing code sort out the mess that it must
anyway be
+ // prepared to handle?
+ ArrowError::InvalidArgumentError(format!(
+ "Expected a ShreddedVariantFieldArray, got {:?}
instead",
+ field.data_type(),
+ ))
+ })?;
+
+ Ok(ShreddedPathStep::Success(field.shredding_state()))
+ }
+ VariantPathElement::Index { .. } => {
+ // TODO: Support array indexing. Among other things, it will
require slicing not
+ // only the array we have here, but also the corresponding
metadata and null masks.
+ Err(ArrowError::NotYetImplemented(
+ "Pathing into shredded variant array index".into(),
+ ))
+ }
+ }
+}
+
+/// Follows the given path as far as possible through shredded variant fields.
If the path ends on a
+/// shredded field, return it directly. Otherwise, use a row shredder to
follow the rest of the path
+/// and extract the requested value on a per-row basis.
+fn shredded_get_path(
+ input: &VariantArray,
+ path: &[VariantPathElement<'_>],
+ as_type: Option<&DataType>,
+) -> Result<ArrayRef> {
+ // Helper that creates a new VariantArray from the given nested value and
typed_value columns,
+ let make_target_variant = |value: Option<BinaryViewArray>, typed_value:
Option<ArrayRef>| {
+ let metadata = input.metadata_field().clone();
+ let nulls = input.inner().nulls().cloned();
+ VariantArray::from_parts(
+ metadata,
+ value,
+ typed_value,
+ nulls,
+ )
+ };
+
+ // Helper that shreds a VariantArray to a specific type.
+ let shred_basic_variant = |target: VariantArray, path: VariantPath<'_>,
as_type: Option<&DataType>| {
+ let mut builder =
output::struct_output::make_shredding_row_builder(path, as_type)?;
+ for i in 0..target.len() {
+ if target.is_null(i) {
+ builder.append_null()?;
+ } else {
+ builder.append_value(&target.value(i))?;
+ }
+ }
+ builder.finish()
+ };
+
+ // Peel away the prefix of path elements that traverses the shredded parts
of this variant
+ // column. Shredding will traverse the rest of the path on a per-row basis.
+ let mut shredding_state = input.shredding_state();
+ let mut path_index = 0;
+ for path_element in path {
+ match follow_shredded_path_element(shredding_state, path_element)? {
+ ShreddedPathStep::Success(state) => {
+ shredding_state = state;
+ path_index += 1;
+ continue;
+ }
+ ShreddedPathStep::Missing => {
+ let num_rows = input.len();
+ let arr = match as_type {
+ Some(data_type) =>
Arc::new(array::new_null_array(data_type, num_rows)) as _,
+ None => Arc::new(array::NullArray::new(num_rows)) as _,
+ };
+ return Ok(arr);
+ }
+ ShreddedPathStep::NotShredded => {
+ let target =
make_target_variant(shredding_state.value_field().cloned(), None);
+ return shred_basic_variant(target, path[path_index..].into(),
as_type);
+ }
+ };
+ }
+
+ // Path exhausted! Create a new `VariantArray` for the location we landed
on.
+ let target = make_target_variant(
+ shredding_state.value_field().cloned(),
+ shredding_state.typed_value_field().cloned(),
+ );
+
+ // If our caller did not request any specific type, we can just return
whatever we landed on.
+ let Some(data_type) = as_type else {
+ return Ok(Arc::new(target));
+ };
+
+ // Structs are special. Recurse into each field separately, hoping to
follow the shredding even
+ // further, and build up the final struct from those individually shredded
results.
+ if let DataType::Struct(fields) = data_type {
+ let children = fields
+ .iter()
+ .map(|field| {
+ shredded_get_path(
+ &target,
+ &[VariantPathElement::from(field.name().as_str())],
+ Some(field.data_type()),
+ )
+ })
+ .collect::<Result<Vec<_>>>()?;
+
+ return Ok(Arc::new(StructArray::try_new(
+ fields.clone(),
+ children,
+ target.nulls().cloned(),
+ )?));
+ }
+
+ // Not a struct, so directly shred the variant as the requested type
+ shred_basic_variant(target, VariantPath::default(), as_type)
+}
+
/// Returns an array with the specified path extracted from the variant values.
///
/// The return array type depends on the `as_type` field of the options
parameter
/// 1. `as_type: None`: a VariantArray is returned. The values in this new
VariantArray will point
/// to the specified path.
/// 2. `as_type: Some(<specific field>)`: an array of the specified type is
returned.
+///
+/// TODO: How would a caller request a struct or list type where the
fields/elements can be any
+/// variant? Caller can pass None as the requested type to fetch a specific
path, but it would
+/// quickly become annoying (and inefficient) to call `variant_get` for each
leaf value in a struct or
+/// list and then try to assemble the results.
Review Comment:
Maybe one way to handle this case would be to extract the fields your code
wants to work with as individual arrays, rather than trying to get back a
single StructArray with the relevant fields
We don't really have a great story at the moment even for casting one struct
array to another when the fields don't match up. See, for example, the
discussion in
- https://github.com/apache/arrow-rs/issues/7176
Therefore I think maybe we can actually punt on casting the result to a
proper struct array recursively (we can certainly add a TODO item for a follow
on PR)
##########
parquet-variant-compute/src/variant_get/output/struct_output.rs:
##########
@@ -0,0 +1,371 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::{ArrayRef, AsArray as _, NullBufferBuilder};
+use arrow::datatypes;
+use arrow::datatypes::{ArrowPrimitiveType, FieldRef};
+use arrow::error::{ArrowError, Result};
+use parquet_variant::{Variant, VariantObject, VariantPath};
+
+use std::sync::Arc;
+
+#[allow(unused)]
+pub(crate) fn make_shredding_row_builder(
+ //metadata: &BinaryViewArray,
+ path: VariantPath<'_>,
+ data_type: Option<&datatypes::DataType>,
+) -> Result<Box<dyn VariantShreddingRowBuilder>> {
+ todo!() // wire it all up!
+}
+
+/// Builder for shredding variant values into strongly typed Arrow arrays.
+///
+/// Useful for variant_get kernels that need to extract specific paths from
variant values, possibly
+/// with casting of leaf values to specific types.
+#[allow(unused)]
+pub(crate) trait VariantShreddingRowBuilder {
+ fn append_null(&mut self) -> Result<()>;
+
+ fn append_value(&mut self, value: &Variant<'_, '_>) -> Result<bool>;
+
+ fn finish(&mut self) -> Result<ArrayRef>;
+}
+
+/// A thin wrapper whose only job is to extract a specific path from a variant
value and pass the
+/// result to a nested builder.
+#[allow(unused)]
+struct VariantPathRowBuilder<'a, T: VariantShreddingRowBuilder> {
+ builder: T,
+ path: VariantPath<'a>,
+}
+
+impl<T: VariantShreddingRowBuilder> VariantShreddingRowBuilder for
VariantPathRowBuilder<'_, T> {
+ fn append_null(&mut self) -> Result<()> {
+ self.builder.append_null()
+ }
+
+ fn append_value(&mut self, value: &Variant<'_, '_>) -> Result<bool> {
+ if let Some(v) = value.get_path(&self.path) {
+ self.builder.append_value(&v)
+ } else {
+ self.builder.append_null()?;
+ Ok(false)
+ }
+ }
+ fn finish(&mut self) -> Result<ArrayRef> {
+ self.builder.finish()
+ }
+}
+
+/// Helper trait for converting `Variant` values to arrow primitive values.
+#[allow(unused)]
+trait VariantAsPrimitive<T: ArrowPrimitiveType> {
+ fn as_primitive(&self) -> Option<T::Native>;
+}
+impl VariantAsPrimitive<datatypes::Int32Type> for Variant<'_, '_> {
+ fn as_primitive(&self) -> Option<i32> {
+ self.as_int32()
+ }
+}
+impl VariantAsPrimitive<datatypes::Float64Type> for Variant<'_, '_> {
+ fn as_primitive(&self) -> Option<f64> {
+ self.as_f64()
+ }
+}
+
+/// Builder for shredding variant values to primitive values
+#[allow(unused)]
+struct PrimitiveVariantShreddingRowBuilder<T: ArrowPrimitiveType> {
+ builder: arrow::array::PrimitiveBuilder<T>,
+}
+
+impl<T> VariantShreddingRowBuilder for PrimitiveVariantShreddingRowBuilder<T>
+where
+ T: ArrowPrimitiveType,
+ for<'m, 'v> Variant<'m, 'v>: VariantAsPrimitive<T>,
+{
+ fn append_null(&mut self) -> Result<()> {
+ self.builder.append_null();
+ Ok(())
+ }
+
+ fn append_value(&mut self, value: &Variant<'_, '_>) -> Result<bool> {
+ if let Some(v) = value.as_primitive() {
+ self.builder.append_value(v);
+ Ok(true)
+ } else {
+ self.builder.append_null(); // TODO: handle casting failure
+ Ok(false)
+ }
+ }
+
+ fn finish(&mut self) -> Result<ArrayRef> {
+ Ok(Arc::new(self.builder.finish()))
+ }
+}
+
+/// Builder for appending raw binary variant values to a BinaryViewArray. It
copies the bytes
+/// as-is, without any decoding.
+#[allow(unused)]
+struct BinaryVariantRowBuilder {
+ nulls: NullBufferBuilder,
+}
+
+impl VariantShreddingRowBuilder for BinaryVariantRowBuilder {
+ fn append_null(&mut self) -> Result<()> {
+ self.nulls.append_null();
+ Ok(())
+ }
+ fn append_value(&mut self, _value: &Variant<'_, '_>) -> Result<bool> {
+ // We need a way to convert a Variant directly to bytes. In
particular, we want to just copy
+ // across the underlying value byte slice of a `Variant::Object` or
`Variant::List`, without
+ // any interaction with a `VariantMetadata` (because we will just
reuse the existing one).
+ //
+ // One could _probably_ emulate this with
parquet_variant::VariantBuilder, but it would do a
+ // lot of unnecessary work and would also create a new metadata column
we don't need.
Review Comment:
Yes, I agree that we are missing some way to copy a variant or some subpart
of the variant to a new Variant.
I wonder if we could make something that implements `VariantBuilderExt` that
is designed for this usecase
https://github.com/apache/arrow-rs/blob/e845411dbf26a10da072af772b7cd98f9f05d0b5/parquet-variant/src/builder.rs#L1542-L1541
Something like
```rust
let (metadata, value) = get_variant(); // exitsting variant at metadata
let variant = Variant::new(metadata, value);
// Create a new variant builder for copying variants without recreating the
metadata
let mut builder = CopyingVariantBuilder::new(existing_metadata, capacity);
// copying this value just copies the bytes, does not re-create the metadata
// errors if the metadata pointer doens't match maybe?
builder.append_value(&variant)?;
```
🤔
##########
parquet-variant-compute/src/variant_get/output/struct_output.rs:
##########
@@ -0,0 +1,371 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::{ArrayRef, AsArray as _, NullBufferBuilder};
+use arrow::datatypes;
+use arrow::datatypes::{ArrowPrimitiveType, FieldRef};
+use arrow::error::{ArrowError, Result};
+use parquet_variant::{Variant, VariantObject, VariantPath};
+
+use std::sync::Arc;
+
+#[allow(unused)]
+pub(crate) fn make_shredding_row_builder(
+ //metadata: &BinaryViewArray,
+ path: VariantPath<'_>,
+ data_type: Option<&datatypes::DataType>,
+) -> Result<Box<dyn VariantShreddingRowBuilder>> {
+ todo!() // wire it all up!
+}
+
+/// Builder for shredding variant values into strongly typed Arrow arrays.
+///
+/// Useful for variant_get kernels that need to extract specific paths from
variant values, possibly
+/// with casting of leaf values to specific types.
+#[allow(unused)]
+pub(crate) trait VariantShreddingRowBuilder {
+ fn append_null(&mut self) -> Result<()>;
+
+ fn append_value(&mut self, value: &Variant<'_, '_>) -> Result<bool>;
+
+ fn finish(&mut self) -> Result<ArrayRef>;
+}
+
+/// A thin wrapper whose only job is to extract a specific path from a variant
value and pass the
+/// result to a nested builder.
+#[allow(unused)]
+struct VariantPathRowBuilder<'a, T: VariantShreddingRowBuilder> {
+ builder: T,
+ path: VariantPath<'a>,
+}
+
+impl<T: VariantShreddingRowBuilder> VariantShreddingRowBuilder for
VariantPathRowBuilder<'_, T> {
+ fn append_null(&mut self) -> Result<()> {
+ self.builder.append_null()
+ }
+
+ fn append_value(&mut self, value: &Variant<'_, '_>) -> Result<bool> {
+ if let Some(v) = value.get_path(&self.path) {
+ self.builder.append_value(&v)
+ } else {
+ self.builder.append_null()?;
+ Ok(false)
+ }
+ }
+ fn finish(&mut self) -> Result<ArrayRef> {
+ self.builder.finish()
+ }
+}
+
+/// Helper trait for converting `Variant` values to arrow primitive values.
+#[allow(unused)]
+trait VariantAsPrimitive<T: ArrowPrimitiveType> {
+ fn as_primitive(&self) -> Option<T::Native>;
+}
+impl VariantAsPrimitive<datatypes::Int32Type> for Variant<'_, '_> {
+ fn as_primitive(&self) -> Option<i32> {
+ self.as_int32()
+ }
+}
+impl VariantAsPrimitive<datatypes::Float64Type> for Variant<'_, '_> {
+ fn as_primitive(&self) -> Option<f64> {
+ self.as_f64()
+ }
+}
+
+/// Builder for shredding variant values to primitive values
+#[allow(unused)]
+struct PrimitiveVariantShreddingRowBuilder<T: ArrowPrimitiveType> {
+ builder: arrow::array::PrimitiveBuilder<T>,
+}
+
+impl<T> VariantShreddingRowBuilder for PrimitiveVariantShreddingRowBuilder<T>
+where
+ T: ArrowPrimitiveType,
+ for<'m, 'v> Variant<'m, 'v>: VariantAsPrimitive<T>,
+{
+ fn append_null(&mut self) -> Result<()> {
+ self.builder.append_null();
+ Ok(())
+ }
+
+ fn append_value(&mut self, value: &Variant<'_, '_>) -> Result<bool> {
+ if let Some(v) = value.as_primitive() {
+ self.builder.append_value(v);
+ Ok(true)
+ } else {
+ self.builder.append_null(); // TODO: handle casting failure
+ Ok(false)
+ }
+ }
+
+ fn finish(&mut self) -> Result<ArrayRef> {
+ Ok(Arc::new(self.builder.finish()))
+ }
+}
+
+/// Builder for appending raw binary variant values to a BinaryViewArray. It
copies the bytes
+/// as-is, without any decoding.
+#[allow(unused)]
+struct BinaryVariantRowBuilder {
+ nulls: NullBufferBuilder,
+}
+
+impl VariantShreddingRowBuilder for BinaryVariantRowBuilder {
+ fn append_null(&mut self) -> Result<()> {
+ self.nulls.append_null();
+ Ok(())
+ }
+ fn append_value(&mut self, _value: &Variant<'_, '_>) -> Result<bool> {
+ // We need a way to convert a Variant directly to bytes. In
particular, we want to just copy
+ // across the underlying value byte slice of a `Variant::Object` or
`Variant::List`, without
+ // any interaction with a `VariantMetadata` (because we will just
reuse the existing one).
+ //
+ // One could _probably_ emulate this with
parquet_variant::VariantBuilder, but it would do a
+ // lot of unnecessary work and would also create a new metadata column
we don't need.
+ todo!()
+ }
+
+ fn finish(&mut self) -> Result<ArrayRef> {
+ // What `finish` does will depend strongly on how `append_value` ends
up working. But
+ // ultimately we'll create and return a `VariantArray` instance.
+ todo!()
+ }
+}
+
+/// Builder that extracts a struct. Casting failures produce NULL or error
according to options.
+#[allow(unused)]
+struct StructVariantShreddingRowBuilder {
+ nulls: NullBufferBuilder,
+ field_builders: Vec<(FieldRef, Box<dyn VariantShreddingRowBuilder>)>,
+}
+
+impl VariantShreddingRowBuilder for StructVariantShreddingRowBuilder {
+ fn append_null(&mut self) -> Result<()> {
+ for (_, builder) in &mut self.field_builders {
+ builder.append_null()?;
+ }
+ self.nulls.append_null();
+ Ok(())
+ }
+
+ fn append_value(&mut self, value: &Variant<'_, '_>) -> Result<bool> {
+ // Casting failure if it's not even an object.
+ let Variant::Object(value) = value else {
+ // TODO: handle casting failure
+ self.append_null()?;
+ return Ok(false);
+ };
+
+ // Process each field. If the field is missing, it becomes NULL. If
the field is present,
+ // the child builder handles it from there, and a failed cast could
produce NULL or error.
+ //
+ // TODO: This loop costs `O(m lg n)` where `m` is the number of fields
in this builder and
+ // `n` is the number of fields in the variant object we're probing.
Given that `m` and `n`
+ // could both be large -- indepentently of each other -- we should
consider doing something
+ // more clever that bounds the cost to O(m + n).
+ for (field, builder) in &mut self.field_builders {
+ match value.get(field.name()) {
+ None => builder.append_null()?,
+ Some(v) => {
+ builder.append_value(&v)?;
+ }
+ }
+ }
+ self.nulls.append_non_null();
+ Ok(true)
+ }
+
+ fn finish(&mut self) -> Result<ArrayRef> {
+ let mut fields = Vec::with_capacity(self.field_builders.len());
+ let mut arrays = Vec::with_capacity(self.field_builders.len());
+ for (field, mut builder) in std::mem::take(&mut self.field_builders) {
+ fields.push(field);
+ arrays.push(builder.finish()?);
+ }
+ Ok(Arc::new(arrow::array::StructArray::try_new(
+ fields.into(),
+ arrays,
+ self.nulls.finish(),
+ )?))
+ }
+}
+
+/// Used for actual shredding of binary variant values into shredded variant
values
+#[allow(unused)]
+struct ShreddedVariantRowBuilder {
Review Comment:
would this also be used to shred unshredded variant values to shredded
arrays?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
