This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 32b385b946 [Variant] VariantArrayBuilder uses MetadataBuilder and
ValueBuilder (#8206)
32b385b946 is described below
commit 32b385b9465c6512c66f95f397acfa126368840c
Author: Ryan Johnson <scov...@users.noreply.github.com>
AuthorDate: Sat Aug 23 04:07:57 2025 -0700
[Variant] VariantArrayBuilder uses MetadataBuilder and ValueBuilder (#8206)
# Which issue does this PR close?
- Closes https://github.com/apache/arrow-rs/issues/8205
# Rationale for this change
`VariantArrayBuilder` had a very complex choreography with the
`VariantBuilder` API, that required lots of manual drop glue to deal
with ownership transfers between it and the `VariantArrayVariantBuilder`
it delegates the actual work to. Rework the whole thing to use a
(now-reusable) `MetadataBuilder` and `ValueBuilder`, with rollbacks
largely handled by `ParentState` -- just like the other builders in the
parquet-variant crate.
# What changes are included in this PR?
Five changes (curated as five commits that reviewers may want to examine
individually):
1. Make a bunch of parquet-variant builder infrastructure public, so
that `VariantArrayBuilder` can access it from the
parquet-variant-compute crate.
2. Make `MetadataBuilder` reusable. Its `finish` method appends the
bytes of a new serialized metadata dictionary to the underlying buffer
and resets the remaining builder state. The builder is thus ready to
create a brand new metadata dictionary whose serialized bytes will also
be appended to the underlying buffer once finished.
3. Rework `VariantArrayBuilder` to use `MetadataBuilder` and
`ValueBuilder`, coordinated via `ParentState`. This is the main feature
of the PR and also the most complicated/subtle.
4. Delete now-unused code that had been added previously in order to
support the old implementation of `VariantArrayBuilder`.
5. Add missing doc comments for now-public types and methods
# Are these changes tested?
Existing variant array builder tests cover the change.
# Are there any user-facing changes?
A lot of builder-related types and methods from the parquet-variant
crate are now public.
---
.../src/variant_array_builder.rs | 165 +++++---------
parquet-variant/src/builder.rs | 243 +++++----------------
2 files changed, 111 insertions(+), 297 deletions(-)
diff --git a/parquet-variant-compute/src/variant_array_builder.rs
b/parquet-variant-compute/src/variant_array_builder.rs
index 969dc3776a..69f631e34d 100644
--- a/parquet-variant-compute/src/variant_array_builder.rs
+++ b/parquet-variant-compute/src/variant_array_builder.rs
@@ -20,7 +20,8 @@
use crate::VariantArray;
use arrow::array::{ArrayRef, BinaryViewArray, BinaryViewBuilder,
NullBufferBuilder, StructArray};
use arrow_schema::{ArrowError, DataType, Field, Fields};
-use parquet_variant::{ListBuilder, ObjectBuilder, Variant, VariantBuilder,
VariantBuilderExt};
+use parquet_variant::{ListBuilder, ObjectBuilder, Variant, VariantBuilderExt};
+use parquet_variant::{MetadataBuilder, ParentState, ValueBuilder};
use std::sync::Arc;
/// A builder for [`VariantArray`]
@@ -72,12 +73,12 @@ use std::sync::Arc;
pub struct VariantArrayBuilder {
/// Nulls
nulls: NullBufferBuilder,
- /// buffer for all the metadata
- metadata_buffer: Vec<u8>,
+ /// builder for all the metadata
+ metadata_builder: MetadataBuilder,
/// ending offset for each serialized metadata dictionary in the buffer
metadata_offsets: Vec<usize>,
- /// buffer for values
- value_buffer: Vec<u8>,
+ /// builder for values
+ value_builder: ValueBuilder,
/// ending offset for each serialized variant value in the buffer
value_offsets: Vec<usize>,
/// The fields of the final `StructArray`
@@ -95,9 +96,9 @@ impl VariantArrayBuilder {
Self {
nulls: NullBufferBuilder::new(row_capacity),
- metadata_buffer: Vec::new(), // todo allocation capacity
+ metadata_builder: MetadataBuilder::default(),
metadata_offsets: Vec::with_capacity(row_capacity),
- value_buffer: Vec::new(),
+ value_builder: ValueBuilder::new(),
value_offsets: Vec::with_capacity(row_capacity),
fields: Fields::from(vec![metadata_field, value_field]),
}
@@ -107,15 +108,17 @@ impl VariantArrayBuilder {
pub fn build(self) -> VariantArray {
let Self {
mut nulls,
- metadata_buffer,
+ metadata_builder,
metadata_offsets,
- value_buffer,
+ value_builder,
value_offsets,
fields,
} = self;
+ let metadata_buffer = metadata_builder.into_inner();
let metadata_array = binary_view_array_from_buffers(metadata_buffer,
metadata_offsets);
+ let value_buffer = value_builder.into_inner();
let value_array = binary_view_array_from_buffers(value_buffer,
value_offsets);
// The build the final struct array
@@ -136,14 +139,14 @@ impl VariantArrayBuilder {
pub fn append_null(&mut self) {
self.nulls.append_null();
// The subfields are expected to be non-nullable according to the
parquet variant spec.
- self.metadata_offsets.push(self.metadata_buffer.len());
- self.value_offsets.push(self.value_buffer.len());
+ self.metadata_offsets.push(self.metadata_builder.offset());
+ self.value_offsets.push(self.value_builder.offset());
}
/// Append the [`Variant`] to the builder as the next row
pub fn append_variant(&mut self, variant: Variant) {
let mut direct_builder = self.variant_builder();
- direct_builder.variant_builder.append_value(variant);
+ direct_builder.append_value(variant);
direct_builder.finish()
}
@@ -194,32 +197,23 @@ impl VariantArrayBuilder {
///
/// See [`VariantArrayBuilder::variant_builder`] for an example
pub struct VariantArrayVariantBuilder<'a> {
- /// was finish called?
- finished: bool,
- /// starting offset in the variant_builder's `metadata` buffer
- metadata_offset: usize,
- /// starting offset in the variant_builder's `value` buffer
- value_offset: usize,
- /// Parent array builder that this variant builder writes to. Buffers
- /// have been moved into the variant builder, and must be returned on
- /// drop
- array_builder: &'a mut VariantArrayBuilder,
- /// Builder for the in progress variant value, temporarily owns the buffers
- /// from `array_builder`
- variant_builder: VariantBuilder,
+ parent_state: ParentState<'a>,
+ metadata_offsets: &'a mut Vec<usize>,
+ value_offsets: &'a mut Vec<usize>,
+ nulls: &'a mut NullBufferBuilder,
}
impl VariantBuilderExt for VariantArrayVariantBuilder<'_> {
fn append_value<'m, 'v>(&mut self, value: impl Into<Variant<'m, 'v>>) {
- self.variant_builder.append_value(value);
+ ValueBuilder::append_variant(self.parent_state(), value.into());
}
fn try_new_list(&mut self) -> Result<ListBuilder<'_>, ArrowError> {
- Ok(self.variant_builder.new_list())
+ Ok(ListBuilder::new(self.parent_state(), false))
}
fn try_new_object(&mut self) -> Result<ObjectBuilder<'_>, ArrowError> {
- Ok(self.variant_builder.new_object())
+ Ok(ObjectBuilder::new(self.parent_state(), false))
}
}
@@ -228,103 +222,40 @@ impl<'a> VariantArrayVariantBuilder<'a> {
///
/// Note this is not public as this is a structure that is logically
/// part of the [`VariantArrayBuilder`] and relies on its internal
structure
- fn new(array_builder: &'a mut VariantArrayBuilder) -> Self {
- // append directly into the metadata and value buffers
- let metadata_buffer = std::mem::take(&mut
array_builder.metadata_buffer);
- let value_buffer = std::mem::take(&mut array_builder.value_buffer);
- let metadata_offset = metadata_buffer.len();
- let value_offset = value_buffer.len();
+ fn new(builder: &'a mut VariantArrayBuilder) -> Self {
+ let parent_state =
+ ParentState::variant(&mut builder.value_builder, &mut
builder.metadata_builder);
VariantArrayVariantBuilder {
- finished: false,
- metadata_offset,
- value_offset,
- variant_builder: VariantBuilder::new_with_buffers(metadata_buffer,
value_buffer),
- array_builder,
+ parent_state,
+ metadata_offsets: &mut builder.metadata_offsets,
+ value_offsets: &mut builder.value_offsets,
+ nulls: &mut builder.nulls,
}
}
- /// Return a reference to the underlying `VariantBuilder`
- pub fn inner(&self) -> &VariantBuilder {
- &self.variant_builder
- }
-
- /// Return a mutable reference to the underlying `VariantBuilder`
- pub fn inner_mut(&mut self) -> &mut VariantBuilder {
- &mut self.variant_builder
- }
-
/// Called to finish the in progress variant and write it to the underlying
/// buffers
///
/// Note if you do not call finish, on drop any changes made to the
/// underlying buffers will be rolled back.
pub fn finish(mut self) {
- self.finished = true;
-
- let metadata_offset = self.metadata_offset;
- let value_offset = self.value_offset;
- // get the buffers back from the variant builder
- let (metadata_buffer, value_buffer) = std::mem::take(&mut
self.variant_builder).finish();
-
- // Sanity Check: if the buffers got smaller, something went wrong
(previous data was lost)
- assert!(
- metadata_offset <= metadata_buffer.len(),
- "metadata length decreased unexpectedly"
- );
- assert!(
- value_offset <= value_buffer.len(),
- "value length decreased unexpectedly"
- );
-
- // commit the changes by putting the
- // ending offsets into the parent array builder.
- let builder = &mut self.array_builder;
- builder.metadata_offsets.push(metadata_buffer.len());
- builder.value_offsets.push(value_buffer.len());
- builder.nulls.append_non_null();
+ // Record the ending offsets after finishing metadata and finish the
parent state.
+ let (value_builder, metadata_builder) =
self.parent_state.value_and_metadata_builders();
+ self.metadata_offsets.push(metadata_builder.finish());
+ self.value_offsets.push(value_builder.offset());
+ self.nulls.append_non_null();
+ self.parent_state.finish();
+ }
- // put the buffers back into the array builder
- builder.metadata_buffer = metadata_buffer;
- builder.value_buffer = value_buffer;
+ fn parent_state(&mut self) -> ParentState<'_> {
+ let (value_builder, metadata_builder) =
self.parent_state.value_and_metadata_builders();
+ ParentState::variant(value_builder, metadata_builder)
}
}
+// Empty Drop to help with borrow checking - warns users if they forget to
call finish()
impl Drop for VariantArrayVariantBuilder<'_> {
- /// If the builder was not finished, roll back any changes made to the
- /// underlying buffers (by truncating them)
- fn drop(&mut self) {
- if self.finished {
- return;
- }
-
- // if the object was not finished, need to rollback any changes by
- // truncating the buffers to the original offsets
- let metadata_offset = self.metadata_offset;
- let value_offset = self.value_offset;
-
- // get the buffers back from the variant builder
- let (mut metadata_buffer, mut value_buffer) =
- std::mem::take(&mut self.variant_builder).into_buffers();
-
- // Sanity Check: if the buffers got smaller, something went wrong
(previous data was lost) so panic immediately
- metadata_buffer
- .len()
- .checked_sub(metadata_offset)
- .expect("metadata length decreased unexpectedly");
- value_buffer
- .len()
- .checked_sub(value_offset)
- .expect("value length decreased unexpectedly");
-
- // Note this truncate is fast because truncate doesn't free any memory:
- // it just has to drop elements (and u8 doesn't have a destructor)
- metadata_buffer.truncate(metadata_offset);
- value_buffer.truncate(value_offset);
-
- // put the buffers back into the array builder
- self.array_builder.metadata_buffer = metadata_buffer;
- self.array_builder.value_buffer = value_buffer;
- }
+ fn drop(&mut self) {}
}
fn binary_view_array_from_buffers(buffer: Vec<u8>, offsets: Vec<usize>) ->
BinaryViewArray {
@@ -457,12 +388,18 @@ mod test {
assert_eq!(variant_array.len(), 2);
assert!(!variant_array.is_null(0));
let variant = variant_array.value(0);
- let variant = variant.as_object().expect("variant to be an object");
- assert_eq!(variant.get("foo").unwrap(), Variant::from(1i32));
+ assert_eq!(
+ variant.get_object_field("foo"),
+ Some(Variant::from(1i32)),
+ "Expected an object with field \"foo\", got: {variant:?}"
+ );
assert!(!variant_array.is_null(1));
let variant = variant_array.value(1);
- let variant = variant.as_object().expect("variant to be an object");
- assert_eq!(variant.get("baz").unwrap(), Variant::from(3i32));
+ assert_eq!(
+ variant.get_object_field("baz"),
+ Some(Variant::from(3i32)),
+ "Expected an object with field \"baz\", got: {variant:?}"
+ );
}
}
diff --git a/parquet-variant/src/builder.rs b/parquet-variant/src/builder.rs
index aa202460a4..2d51b6d2fd 100644
--- a/parquet-variant/src/builder.rs
+++ b/parquet-variant/src/builder.rs
@@ -86,27 +86,15 @@ fn append_packed_u32(dest: &mut Vec<u8>, value: u32,
value_size: usize) {
///
/// You can reuse an existing `Vec<u8>` by using the `from` impl
#[derive(Debug, Default)]
-struct ValueBuilder(Vec<u8>);
+pub struct ValueBuilder(Vec<u8>);
impl ValueBuilder {
/// Construct a ValueBuffer that will write to a new underlying `Vec`
- fn new() -> Self {
+ pub fn new() -> Self {
Default::default()
}
}
-impl From<Vec<u8>> for ValueBuilder {
- fn from(value: Vec<u8>) -> Self {
- Self(value)
- }
-}
-
-impl From<ValueBuilder> for Vec<u8> {
- fn from(value_buffer: ValueBuilder) -> Self {
- value_buffer.0
- }
-}
-
impl ValueBuilder {
fn append_u8(&mut self, term: u8) {
self.0.push(term);
@@ -120,8 +108,9 @@ impl ValueBuilder {
self.0.push(primitive_header(primitive_type));
}
- fn into_inner(self) -> Vec<u8> {
- self.into()
+ /// Returns the underlying buffer, consuming self
+ pub fn into_inner(self) -> Vec<u8> {
+ self.0
}
fn inner_mut(&mut self) -> &mut Vec<u8> {
@@ -292,7 +281,8 @@ impl ValueBuilder {
Ok(())
}
- fn offset(&self) -> usize {
+ /// Returns the current size of the underlying buffer
+ pub fn offset(&self) -> usize {
self.0.len()
}
@@ -302,7 +292,7 @@ impl ValueBuilder {
///
/// This method will panic if the variant contains duplicate field names
in objects
/// when validation is enabled. For a fallible version, use
[`ValueBuilder::try_append_variant`]
- fn append_variant(mut state: ParentState<'_>, variant: Variant<'_, '_>) {
+ pub fn append_variant(mut state: ParentState<'_>, variant: Variant<'_,
'_>) {
let builder = state.value_builder();
match variant {
Variant::Null => builder.append_null(),
@@ -437,7 +427,7 @@ impl ValueBuilder {
///
/// You can use an existing `Vec<u8>` as the metadata buffer by using the
`from` impl.
#[derive(Default, Debug)]
-struct MetadataBuilder {
+pub struct MetadataBuilder {
// Field names -- field_ids are assigned in insert order
field_names: IndexSet<String>,
@@ -448,16 +438,6 @@ struct MetadataBuilder {
metadata_buffer: Vec<u8>,
}
-/// Create a new MetadataBuilder that will write to the specified metadata
buffer
-impl From<Vec<u8>> for MetadataBuilder {
- fn from(metadata_buffer: Vec<u8>) -> Self {
- Self {
- metadata_buffer,
- ..Default::default()
- }
- }
-}
-
impl MetadataBuilder {
/// Upsert field name to dictionary, return its ID
fn upsert_field_name(&mut self, field_name: &str) -> u32 {
@@ -477,6 +457,11 @@ impl MetadataBuilder {
id as u32
}
+ /// The current length of the underlying metadata buffer
+ pub fn offset(&self) -> usize {
+ self.metadata_buffer.len()
+ }
+
/// Returns the number of field names stored in the metadata builder.
/// Note: this method should be the only place to call
`self.field_names.len()`
///
@@ -498,17 +483,18 @@ impl MetadataBuilder {
self.field_names.iter().map(|k| k.len()).sum()
}
- fn finish(self) -> Vec<u8> {
+ /// Finalizes the metadata dictionary and appends its serialized bytes to
the underlying buffer,
+ /// returning the resulting [`Self::offset`]. The builder state is reset
and ready to start
+ /// building a new metadata dictionary.
+ pub fn finish(&mut self) -> usize {
let nkeys = self.num_field_names();
// Calculate metadata size
let total_dict_size: usize = self.metadata_size();
- let Self {
- field_names,
- is_sorted,
- mut metadata_buffer,
- } = self;
+ let metadata_buffer = &mut self.metadata_buffer;
+ let is_sorted = std::mem::take(&mut self.is_sorted);
+ let field_names = std::mem::take(&mut self.field_names);
// Determine appropriate offset size based on the larger of dict size
or total string size
let max_offset = std::cmp::max(total_dict_size, nkeys);
@@ -524,27 +510,27 @@ impl MetadataBuilder {
metadata_buffer.push(0x01 | (is_sorted as u8) << 4 | ((offset_size -
1) << 6));
// Write dictionary size
- write_offset(&mut metadata_buffer, nkeys, offset_size);
+ write_offset(metadata_buffer, nkeys, offset_size);
// Write offsets
let mut cur_offset = 0;
for key in field_names.iter() {
- write_offset(&mut metadata_buffer, cur_offset, offset_size);
+ write_offset(metadata_buffer, cur_offset, offset_size);
cur_offset += key.len();
}
// Write final offset
- write_offset(&mut metadata_buffer, cur_offset, offset_size);
+ write_offset(metadata_buffer, cur_offset, offset_size);
// Write string data
for key in field_names {
metadata_buffer.extend_from_slice(key.as_bytes());
}
- metadata_buffer
+ metadata_buffer.len()
}
- /// Return the inner buffer, without finalizing any in progress metadata.
- pub(crate) fn into_inner(self) -> Vec<u8> {
+ /// Returns the inner buffer, consuming self without finalizing any in
progress metadata.
+ pub fn into_inner(self) -> Vec<u8> {
self.metadata_buffer
}
}
@@ -585,7 +571,7 @@ impl<S: AsRef<str>> Extend<S> for MetadataBuilder {
/// treat the variants as a union, so that accessing a `value_builder` or
`metadata_builder` is
/// branch-free.
#[derive(Debug)]
-enum ParentState<'a> {
+pub enum ParentState<'a> {
Variant {
value_builder: &'a mut ValueBuilder,
saved_value_builder_offset: usize,
@@ -614,7 +600,10 @@ enum ParentState<'a> {
}
impl<'a> ParentState<'a> {
- fn variant(
+ /// Creates a new instance suitable for a top-level variant builder
+ /// (e.g. [`VariantBuilder`]). The value and metadata builder state is
checkpointed and will
+ /// roll back on drop, unless [`Self::finish`] is called.
+ pub fn variant(
value_builder: &'a mut ValueBuilder,
metadata_builder: &'a mut MetadataBuilder,
) -> Self {
@@ -627,7 +616,10 @@ impl<'a> ParentState<'a> {
}
}
- fn list(
+ /// Creates a new instance suitable for a [`ListBuilder`]. The value and
metadata builder state
+ /// is checkpointed and will roll back on drop, unless [`Self::finish`] is
called. The new
+ /// element's offset is also captured eagerly and will also roll back if
not finished.
+ pub fn list(
value_builder: &'a mut ValueBuilder,
metadata_builder: &'a mut MetadataBuilder,
offsets: &'a mut Vec<usize>,
@@ -651,7 +643,12 @@ impl<'a> ParentState<'a> {
}
}
- fn try_object(
+ /// Creates a new instance suitable for an [`ObjectBuilder`]. The value
and metadata builder state
+ /// is checkpointed and will roll back on drop, unless [`Self::finish`] is
called. The new
+ /// field's name and offset are also captured eagerly and will also roll
back if not finished.
+ ///
+ /// The call fails if the field name is invalid (e.g. because it
duplicates an existing field).
+ pub fn try_object(
value_builder: &'a mut ValueBuilder,
metadata_builder: &'a mut MetadataBuilder,
fields: &'a mut IndexMap<u32, usize>,
@@ -717,8 +714,8 @@ impl<'a> ParentState<'a> {
}
}
- // Mark the insertion as having succeeded.
- fn finish(&mut self) {
+ /// Mark the insertion as having succeeded. Internal state will no longer
roll back on drop.
+ pub fn finish(&mut self) {
*self.is_finished() = true
}
@@ -778,7 +775,7 @@ impl<'a> ParentState<'a> {
/// Return mutable references to the value and metadata builders that this
/// parent state is using.
- fn value_and_metadata_builders(&mut self) -> (&mut ValueBuilder, &mut
MetadataBuilder) {
+ pub fn value_and_metadata_builders(&mut self) -> (&mut ValueBuilder, &mut
MetadataBuilder) {
match self {
ParentState::Variant {
value_builder,
@@ -986,41 +983,6 @@ impl Drop for ParentState<'_> {
/// );
///
/// ```
-/// # Example: Reusing Buffers
-///
-/// You can use the [`VariantBuilder`] to write into existing buffers (for
-/// example to write multiple variants back to back in the same buffer)
-///
-/// ```
-/// // we will write two variants back to back
-/// use parquet_variant::{Variant, VariantBuilder};
-/// // Append 12345
-/// let mut builder = VariantBuilder::new();
-/// builder.append_value(12345);
-/// let (metadata, value) = builder.finish();
-/// // remember where the first variant ends
-/// let (first_meta_offset, first_meta_len) = (0, metadata.len());
-/// let (first_value_offset, first_value_len) = (0, value.len());
-///
-/// // now, append a second variant to the same buffers
-/// let mut builder = VariantBuilder::new_with_buffers(metadata, value);
-/// builder.append_value("Foo");
-/// let (metadata, value) = builder.finish();
-///
-/// // The variants can be referenced in their appropriate location
-/// let variant1 = Variant::new(
-/// &metadata[first_meta_offset..first_meta_len],
-/// &value[first_value_offset..first_value_len]
-/// );
-/// assert_eq!(variant1, Variant::Int32(12345));
-///
-/// let variant2 = Variant::new(
-/// &metadata[first_meta_len..],
-/// &value[first_value_len..]
-/// );
-/// assert_eq!(variant2, Variant::from("Foo"));
-/// ```
-///
/// # Example: Unique Field Validation
///
/// This example shows how enabling unique field validation will cause an error
@@ -1100,16 +1062,6 @@ impl VariantBuilder {
self
}
- /// Create a new VariantBuilder that will write the metadata and values to
- /// the specified buffers.
- pub fn new_with_buffers(metadata_buffer: Vec<u8>, value_buffer: Vec<u8>)
-> Self {
- Self {
- value_builder: ValueBuilder::from(value_buffer),
- metadata_builder: MetadataBuilder::from(metadata_buffer),
- validate_unique_fields: false,
- }
- }
-
/// Enables validation of unique field keys in nested objects.
///
/// This setting is propagated to all [`ObjectBuilder`]s created through
this [`VariantBuilder`]
@@ -1215,19 +1167,8 @@ impl VariantBuilder {
}
/// Finish the builder and return the metadata and value buffers.
- pub fn finish(self) -> (Vec<u8>, Vec<u8>) {
- (
- self.metadata_builder.finish(),
- self.value_builder.into_inner(),
- )
- }
-
- /// Return the inner metadata buffers and value buffer.
- ///
- /// This can be used to get the underlying buffers provided via
- /// [`VariantBuilder::new_with_buffers`] without finalizing the metadata or
- /// values (for rolling back changes).
- pub fn into_buffers(self) -> (Vec<u8>, Vec<u8>) {
+ pub fn finish(mut self) -> (Vec<u8>, Vec<u8>) {
+ self.metadata_builder.finish();
(
self.metadata_builder.into_inner(),
self.value_builder.into_inner(),
@@ -1246,7 +1187,8 @@ pub struct ListBuilder<'a> {
}
impl<'a> ListBuilder<'a> {
- fn new(parent_state: ParentState<'a>, validate_unique_fields: bool) ->
Self {
+ /// Creates a new list builder, nested on top of the given parent state.
+ pub fn new(parent_state: ParentState<'a>, validate_unique_fields: bool) ->
Self {
Self {
parent_state,
offsets: vec![],
@@ -1388,7 +1330,8 @@ pub struct ObjectBuilder<'a> {
}
impl<'a> ObjectBuilder<'a> {
- fn new(parent_state: ParentState<'a>, validate_unique_fields: bool) ->
Self {
+ /// Creates a new object builder, nested on top of the given parent state.
+ pub fn new(parent_state: ParentState<'a>, validate_unique_fields: bool) ->
Self {
Self {
parent_state,
fields: IndexMap::new(),
@@ -1589,18 +1532,27 @@ impl<'a> ObjectBuilder<'a> {
/// Allows users to append values to a [`VariantBuilder`], [`ListBuilder`] or
/// [`ObjectBuilder`]. using the same interface.
pub trait VariantBuilderExt {
+ /// Appends a new variant value to this builder. See e.g.
[`VariantBuilder::append_value`].
fn append_value<'m, 'v>(&mut self, value: impl Into<Variant<'m, 'v>>);
+ /// Creates a nested list builder. See e.g. [`VariantBuilder::new_list`].
Panics if the nested
+ /// builder cannot be created, see e.g. [`ObjectBuilder::new_list`].
fn new_list(&mut self) -> ListBuilder<'_> {
self.try_new_list().unwrap()
}
+ /// Creates a nested object builder. See e.g.
[`VariantBuilder::new_object`]. Panics if the
+ /// nested builder cannot be created, see e.g.
[`ObjectBuilder::new_object`].
fn new_object(&mut self) -> ObjectBuilder<'_> {
self.try_new_object().unwrap()
}
+ /// Creates a nested list builder. See e.g. [`VariantBuilder::new_list`].
Returns an error if
+ /// the nested builder cannot be created, see e.g.
[`ObjectBuilder::try_new_list`].
fn try_new_list(&mut self) -> Result<ListBuilder<'_>, ArrowError>;
+ /// Creates a nested object builder. See e.g.
[`VariantBuilder::new_object`]. Returns an error
+ /// if the nested builder cannot be created, see e.g.
[`ObjectBuilder::try_new_object`].
fn try_new_object(&mut self) -> Result<ObjectBuilder<'_>, ArrowError>;
}
@@ -2779,81 +2731,6 @@ mod tests {
assert_eq!(metadata.num_field_names(), 3);
}
- /// Test reusing buffers with nested objects
- #[test]
- fn test_with_existing_buffers_nested() {
- let mut builder = VariantBuilder::new();
- append_test_list(&mut builder);
- let (m1, v1) = builder.finish();
- let variant1 = Variant::new(&m1, &v1);
-
- let mut builder = VariantBuilder::new();
- append_test_object(&mut builder);
- let (m2, v2) = builder.finish();
- let variant2 = Variant::new(&m2, &v2);
-
- let mut builder = VariantBuilder::new();
- builder.append_value("This is a string");
- let (m3, v3) = builder.finish();
- let variant3 = Variant::new(&m3, &v3);
-
- // Now, append those three variants to the a new buffer that is reused
- let mut builder = VariantBuilder::new();
- append_test_list(&mut builder);
- let (metadata, value) = builder.finish();
- let (meta1_offset, meta1_end) = (0, metadata.len());
- let (value1_offset, value1_end) = (0, value.len());
-
- // reuse same buffer
- let mut builder = VariantBuilder::new_with_buffers(metadata, value);
- append_test_object(&mut builder);
- let (metadata, value) = builder.finish();
- let (meta2_offset, meta2_end) = (meta1_end, metadata.len());
- let (value2_offset, value2_end) = (value1_end, value.len());
-
- // Append a string
- let mut builder = VariantBuilder::new_with_buffers(metadata, value);
- builder.append_value("This is a string");
- let (metadata, value) = builder.finish();
- let (meta3_offset, meta3_end) = (meta2_end, metadata.len());
- let (value3_offset, value3_end) = (value2_end, value.len());
-
- // verify we can read the variants back correctly
- let roundtrip1 = Variant::new(
- &metadata[meta1_offset..meta1_end],
- &value[value1_offset..value1_end],
- );
- assert_eq!(roundtrip1, variant1,);
-
- let roundtrip2 = Variant::new(
- &metadata[meta2_offset..meta2_end],
- &value[value2_offset..value2_end],
- );
- assert_eq!(roundtrip2, variant2,);
-
- let roundtrip3 = Variant::new(
- &metadata[meta3_offset..meta3_end],
- &value[value3_offset..value3_end],
- );
- assert_eq!(roundtrip3, variant3);
- }
-
- /// append a simple List variant
- fn append_test_list(builder: &mut VariantBuilder) {
- builder
- .new_list()
- .with_value(1234)
- .with_value("a string value")
- .finish();
- }
-
- /// append an object variant
- fn append_test_object(builder: &mut VariantBuilder) {
- let mut obj = builder.new_object();
- obj.insert("a", true);
- obj.finish().unwrap();
- }
-
#[test]
fn test_variant_builder_to_list_builder_no_finish() {
// Create a list builder but never finish it
