liukun4515 commented on code in PR #2960:
URL: https://github.com/apache/arrow-datafusion/pull/2960#discussion_r928343239
##########
datafusion/core/src/datasource/file_format/parquet.rs:
##########
@@ -1023,6 +1023,49 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn read_decimal_parquet() -> Result<()> {
+ let session_ctx = SessionContext::new();
+ let task_ctx = session_ctx.task_ctx();
+
+ // parquet use the int32 as the physical type to store decimal
+ let exec = get_exec("int32_decimal.parquet", None, None).await?;
+ let batches = collect(exec, task_ctx.clone()).await?;
+ assert_eq!(1, batches.len());
+ assert_eq!(1, batches[0].num_columns());
+ let column = batches[0].column(0);
+ assert_eq!(&DataType::Decimal(4, 2), column.data_type());
+
+ // parquet use the int64 as the physical type to store decimal
+ let exec = get_exec("int64_decimal.parquet", None, None).await?;
+ let batches = collect(exec, task_ctx.clone()).await?;
+ assert_eq!(1, batches.len());
+ assert_eq!(1, batches[0].num_columns());
+ let column = batches[0].column(0);
+ assert_eq!(&DataType::Decimal(10, 2), column.data_type());
+
+ // parquet use the fixed length binary as the physical type to store
decimal
+ let exec = get_exec("fixed_length_decimal.parquet", None, None).await?;
+ let batches = collect(exec, task_ctx.clone()).await?;
+ assert_eq!(1, batches.len());
+ assert_eq!(1, batches[0].num_columns());
+ let column = batches[0].column(0);
+ assert_eq!(&DataType::Decimal(25, 2), column.data_type());
+
+ let exec = get_exec("fixed_length_decimal_legacy.parquet", None,
None).await?;
+ let batches = collect(exec, task_ctx.clone()).await?;
+ assert_eq!(1, batches.len());
+ assert_eq!(1, batches[0].num_columns());
+ let column = batches[0].column(0);
+ assert_eq!(&DataType::Decimal(25, 2), column.data_type());
+
+ // parquet use the fixed length binary as the physical type to store
decimal
+ // TODO: arrow-rs don't support convert the physical type of binary to
decimal
+ // let exec = get_exec("byte_array_decimal.parquet", None,
None).await?;
Review Comment:
@tustvold
In the parquet of Java version, we can write decimal with the binary or
fixed-length binary, so we can get the decimal using the physical type of
binary.
we can get the definition of decimal from the [format
spec](https://github.com/apache/parquet-format/blob/54e53e5d7794d383529dd30746378f19a12afd58/src/main/thrift/parquet.thrift#L67)
Decimal can be stored in the parquet as INT32, INT64, FIXED_LEN_BYTE_ARRAY
or BYTE_ARRAY.
I can help to implement the BYTE_ARRAY in the arrow-rs.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
