0lai0 commented on code in PR #1393:
URL: https://github.com/apache/mahout/pull/1393#discussion_r3389145939
##########
qdp/qdp-core/src/readers/parquet.rs:
##########
@@ -17,24 +17,180 @@
//! Parquet format reader implementation.
use std::fs::File;
+use std::marker::PhantomData;
use std::path::Path;
-use arrow::array::{Array, FixedSizeListArray, Float64Array, ListArray};
+use arrow::array::{Array, FixedSizeListArray, Float32Array, Float64Array,
ListArray};
+use arrow::compute;
use arrow::datatypes::DataType;
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use crate::error::{MahoutError, Result};
-use crate::reader::{DataReader, NullHandling, StreamingDataReader,
handle_float64_nulls};
+use crate::reader::{
+ DataReader, FloatElem, NullHandling, StreamingDataReader,
handle_float32_nulls,
+ handle_float64_nulls,
+};
-/// Reader for Parquet files containing List<Float64> or
FixedSizeList<Float64> columns.
-pub struct ParquetReader {
+// ---------------------------------------------------------------------------
+// Internal sealed helper trait
+// ---------------------------------------------------------------------------
+
+/// Zero-copy or Arrow-cast extraction from an Arrow array into an output
`Vec<Self>`.
+///
+/// Implemented for `f32` and `f64` only:
+/// - same dtype → `extend_from_slice` directly from the Arrow buffer (zero
alloc)
+/// - cross dtype → `arrow::compute::cast` first, then `extend_from_slice`
+/// - f32 → f64: exact, NaN preserved
+/// - f64 → f32: values outside f32 range become ±Inf, NaN preserved
+pub(crate) trait ArrowPrimitive: FloatElem {
+ fn extend_from_arrow_array(
+ output: &mut Vec<Self>,
+ array: &dyn Array,
+ null_handling: NullHandling,
+ ) -> Result<()>;
+
+ fn collect_from_arrow_array(
+ array: &dyn Array,
+ null_handling: NullHandling,
+ ) -> Result<Vec<Self>> {
+ let mut out = Vec::new();
+ Self::extend_from_arrow_array(&mut out, array, null_handling)?;
+ Ok(out)
+ }
+}
+
+impl ArrowPrimitive for f64 {
+ fn extend_from_arrow_array(
+ output: &mut Vec<f64>,
+ array: &dyn Array,
+ null_handling: NullHandling,
+ ) -> Result<()> {
+ match array.data_type() {
+ DataType::Float64 => {
+ let arr = array
+ .as_any()
+ .downcast_ref::<Float64Array>()
+ .ok_or_else(|| MahoutError::Io("Float64 downcast
failed".to_string()))?;
+ handle_float64_nulls(output, arr, null_handling)?;
+ }
+ DataType::Float32 => {
+ let casted = compute::cast(array, &DataType::Float64)
+ .map_err(|e| MahoutError::Io(format!("Arrow cast f32→f64:
{e}")))?;
+ let arr = casted
+ .as_any()
+ .downcast_ref::<Float64Array>()
+ .ok_or_else(|| MahoutError::Io("Cast to Float64
failed".to_string()))?;
+ handle_float64_nulls(output, arr, null_handling)?;
+ }
+ other => {
+ return Err(MahoutError::InvalidInput(format!(
+ "Expected Float32 or Float64 values, got {other:?}"
+ )));
+ }
+ }
+ Ok(())
+ }
+}
+
+impl ArrowPrimitive for f32 {
Review Comment:
Thank @rich7420 for review. I'll update in next commit.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
