novemberkilo commented on a change in pull request #1063:
URL: https://github.com/apache/arrow-rs/pull/1063#discussion_r774220465
##########
File path: parquet/src/arrow/arrow_writer.rs
##########
@@ -1574,6 +1570,9 @@ mod tests {
.unwrap();
let a = ListArray::from(a_list_data);
+ // let values = Arc::new(a);
+ // one_column_roundtrip("null_list_single_column", values, true,
Some(SMALL_SIZE / 2));
+
let batch = RecordBatch::try_new(Arc::new(schema),
vec![Arc::new(a)]).unwrap();
roundtrip("test_null_list_single_column.parquet", batch, None);
Review comment:
The error I am now seeing comes from the reader being unable to read the
parquet that is written out.
```
thread 'arrow::arrow_writer::tests::null_list_single_column' panicked at
'called `Result::unwrap()` on an `Err` value: ArrowError("Unable to convert
parquet INT32 logical type Some(UNKNOWN(NullType)) or converted type NONE")',
parquet/src/arrow/array_reader.rs:1502:14
```
I have peppered the code with `dbg!` to see if I can make sense of what's
going on but so far all I have is a hunch that we've also got a potential bug
in the reader. Detailed output from this test appears below:
```
---- arrow::arrow_writer::tests::null_list_single_column stdout ----
[parquet/src/arrow/levels.rs:754] &self = LevelInfo {
definition: [
1,
],
repetition: Some(
[
0,
],
),
array_offsets: [
0,
0,
],
array_mask: [
true,
],
max_definition: 2,
level_type: List(
true,
),
offset: 0,
length: 0,
}
[parquet/src/arrow/levels.rs:788] &filtered = []
[parquet/src/arrow/array_reader.rs:1844] &context =
ArrayReaderBuilderContext {
def_level: 0,
rep_level: 0,
path: ColumnPath {
parts: [
"arrow_schema",
],
},
}
[parquet/src/arrow/array_reader.rs:1845] &child = GroupType {
basic_info: BasicTypeInfo {
name: "emptylist",
repetition: Some(
OPTIONAL,
),
converted_type: LIST,
logical_type: Some(
LIST(
ListType,
),
),
id: None,
},
fields: [
GroupType {
basic_info: BasicTypeInfo {
name: "list",
repetition: Some(
REPEATED,
),
converted_type: NONE,
logical_type: None,
id: None,
},
fields: [
PrimitiveType {
basic_info: BasicTypeInfo {
name: "item",
repetition: Some(
OPTIONAL,
),
converted_type: NONE,
logical_type: Some(
UNKNOWN(
NullType,
),
),
id: None,
},
physical_type: INT32,
type_length: -1,
scale: -1,
precision: -1,
},
],
},
],
}
[parquet/src/arrow/array_reader.rs:1495] &list_type = GroupType {
basic_info: BasicTypeInfo {
name: "emptylist",
repetition: Some(
OPTIONAL,
),
converted_type: LIST,
logical_type: Some(
LIST(
ListType,
),
),
id: None,
},
fields: [
GroupType {
basic_info: BasicTypeInfo {
name: "list",
repetition: Some(
REPEATED,
),
converted_type: NONE,
logical_type: None,
id: None,
},
fields: [
PrimitiveType {
basic_info: BasicTypeInfo {
name: "item",
repetition: Some(
OPTIONAL,
),
converted_type: NONE,
logical_type: Some(
UNKNOWN(
NullType,
),
),
id: None,
},
physical_type: INT32,
type_length: -1,
scale: -1,
precision: -1,
},
],
},
],
}
[parquet/src/arrow/array_reader.rs:1496] &context =
ArrayReaderBuilderContext {
def_level: 0,
rep_level: 0,
path: ColumnPath {
parts: [
"arrow_schema",
],
},
}
[parquet/src/arrow/array_reader.rs:1497] &item_type = PrimitiveType {
basic_info: BasicTypeInfo {
name: "item",
repetition: Some(
OPTIONAL,
),
converted_type: NONE,
logical_type: Some(
UNKNOWN(
NullType,
),
),
id: None,
},
physical_type: INT32,
type_length: -1,
scale: -1,
precision: -1,
}
[parquet/src/arrow/array_reader.rs:1498] &new_context =
ArrayReaderBuilderContext {
def_level: 2,
rep_level: 1,
path: ColumnPath {
parts: [
"arrow_schema",
"emptylist",
],
},
}
thread 'arrow::arrow_writer::tests::null_list_single_column' panicked at
'called `Result::unwrap()` on an `Err` value: ArrowError("Unable to convert
parquet INT32 logical type Some(UNKNOWN(NullType)) or converted type NONE")',
parquet/src/arrow/array_reader.rs:1502:14
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
