tustvold commented on issue #2984:
URL: https://github.com/apache/arrow-rs/issues/2984#issuecomment-1302554218
> Could it be that the parquet crate is inferring the converted-type from
the logic type
This is definitely possible, looking at the code it will use the same type
builder that will set the type. However, the following test passes
```
#[test]
fn test_writes_converted_type() {
use crate::format::{MicroSeconds, FileMetaData, TimeUnit};
let mut data = Vec::with_capacity(1024);
let logical_type = LogicalType::Timestamp {
is_adjusted_to_u_t_c: false,
unit: TimeUnit::MICROS(MicroSeconds {}),
};
let field = Arc::new(
types::Type::primitive_type_builder("col1", Type::INT64)
.with_logical_type(Some(logical_type.clone()))
.with_repetition(Repetition::REQUIRED)
.build()
.unwrap(),
);
assert_eq!(
field.get_basic_info().converted_type(),
ConvertedType::TIMESTAMP_MICROS
);
let schema = Arc::new(
types::Type::group_type_builder("schema")
.with_fields(&mut vec![field.clone()])
.build()
.unwrap(),
);
// Write data
let props = Arc::new(WriterProperties::builder().build());
let mut writer =
SerializedFileWriter::new(&mut data, schema.clone(),
props).unwrap();
let mut row_group_writer = writer.next_row_group().unwrap();
let mut column_writer =
row_group_writer.next_column().unwrap().unwrap();
column_writer
.typed::<Int64Type>()
.write_batch(&[1, 2, 3, 4], None, None)
.unwrap();
column_writer.close().unwrap();
row_group_writer.close().unwrap();
writer.close().unwrap();
// Must read footer manually as default logic automatically infers
let footer = data[data.len() - 8..].try_into().unwrap();
let metadata_len =
crate::file::footer::decode_footer(footer).unwrap();
let metadata = &data[data.len() - 8 - metadata_len..data.len() - 8];
let mut prot = TCompactInputProtocol::new(metadata);
let t_file_metadata = FileMetaData::read_from_in_protocol(&mut
prot).unwrap();
assert_eq!(t_file_metadata.schema.len(), 2);
let element = &t_file_metadata.schema[1];
assert_eq!(&element.name, "col1");
let actual =
ConvertedType::try_from(element.converted_type).unwrap();
assert_eq!(actual, ConvertedType::TIMESTAMP_MICROS);
let actual =
LogicalType::from(element.logical_type.clone().unwrap());
assert_eq!(actual, logical_type);
}
```
> Would you prefer I would open a new bug ticket for this and leaving this
one closed
If you could that would be grand, I'm struggling to understand what is going
on
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
