tustvold commented on code in PR #1588:
URL: https://github.com/apache/arrow-rs/pull/1588#discussion_r864990665
##########
parquet/src/arrow/array_reader/builder.rs:
##########
@@ -295,96 +297,80 @@ impl<'a> TypeVisitor<Option<Box<dyn ArrayReader>>, &'a
ArrayReaderBuilderContext
.first()
.ok_or_else(|| ArrowError("List field must have a
child.".to_string()))?
.clone();
- let mut new_context = context.clone();
+ // If the list can contain nulls
+ let nullable = match list_type.get_basic_info().repetition() {
+ Repetition::REQUIRED => false,
+ Repetition::OPTIONAL => true,
+ Repetition::REPEATED => {
+ return Err(general_err!("List type cannot be repeated"))
+ }
+ };
+
+ let mut new_context = context.clone();
new_context.path.append(vec![list_type.name().to_string()]);
- // We need to know at what definition a list or its child is null
- let list_null_def = new_context.def_level;
- let mut list_empty_def = new_context.def_level;
+
+ // The repeated field
+ new_context.rep_level += 1;
+ new_context.def_level += 1;
// If the list's root is nullable
- if let Repetition::OPTIONAL = list_type.get_basic_info().repetition() {
+ if nullable {
new_context.def_level += 1;
- // current level is nullable, increment to get level for empty
list slot
- list_empty_def += 1;
}
- match list_child.get_basic_info().repetition() {
- Repetition::REPEATED => {
- new_context.def_level += 1;
- new_context.rep_level += 1;
- }
- Repetition::OPTIONAL => {
- new_context.def_level += 1;
- }
- _ => (),
- }
+ match self.dispatch(item_type, &new_context) {
+ Ok(Some(item_reader)) => {
+ let item_type = item_reader.get_data_type().clone();
+
+ // a list is a group type with a single child. The list child's
+ // name comes from the child's field name.
+ // if the child's name is "list" and it has a child, then use
this child
+ if list_child.name() == "list" &&
!list_child.get_fields().is_empty() {
+ list_child = list_child.get_fields().first().unwrap();
+ }
- let reader = self.dispatch(item_type.clone(), &new_context);
- if let Ok(Some(item_reader)) = reader {
- let item_reader_type = item_reader.get_data_type().clone();
-
- match item_reader_type {
- ArrowType::List(_)
- | ArrowType::FixedSizeList(_, _)
- | ArrowType::Dictionary(_, _) => Err(ArrowError(format!(
- "reading List({:?}) into arrow not supported yet",
Review Comment:
This is the error that would previously be returned on a nested list
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
