0lai0 commented on code in PR #1407:
URL: https://github.com/apache/mahout/pull/1407#discussion_r3473396582
##########
qdp/qdp-core/src/pipeline_runner.rs:
##########
@@ -1365,4 +1474,190 @@ mod tests {
"normalize() must set prefetch_depth > 0"
);
}
+
+ //
-------------------------------------------------------------------------
+ // dtype file-load tests
+ //
+ // These tests verify that PipelineConfig.dtype is respected when loading
+ // from file sources. They stop at the BatchData variant boundary — the
+ // encode kernel (encode_batch_f32_for_pipeline) is CUDA-gated and cannot
+ // be exercised in CPU-only CI. BatchData::F32 is the observable proxy
that
+ // confirms the f32 kernel would be called on a GPU host; this mirrors the
+ // existing convention in test_synthetic_producer_f32_*.
+ //
-------------------------------------------------------------------------
+
+ mod dtype_file_tests {
+ use super::*;
+ use arrow::array::{ArrayRef, FixedSizeListBuilder, Float32Builder,
RecordBatch};
+ use arrow::datatypes::{DataType, Field, Schema};
+ use parquet::arrow::ArrowWriter;
+ use std::fs;
+ use std::sync::Arc;
+
+ fn write_f32_parquet(path: &std::path::Path) {
+ // 8 samples, each 4 features — matches amplitude encoding with 2
qubits (2^2=4)
+ let item_field = Arc::new(Field::new("item", DataType::Float32,
true));
+ let list_field = Field::new("data",
DataType::FixedSizeList(item_field, 4), true);
+ let schema = Arc::new(Schema::new(vec![list_field]));
+ let mut builder = FixedSizeListBuilder::new(Float32Builder::new(),
4);
+ for _ in 0..8 {
+ builder.values().append_slice(&[0.25_f32, 0.5, 0.75, 1.0]);
+ builder.append(true);
+ }
+ let array = Arc::new(builder.finish()) as ArrayRef;
+ let batch = RecordBatch::try_new(schema.clone(),
vec![array]).unwrap();
+ let file = fs::File::create(path).unwrap();
+ let mut writer = ArrowWriter::try_new(file, schema, None).unwrap();
+ writer.write(&batch).unwrap();
+ writer.close().unwrap();
+ }
+
+ static FILE_COUNTER: std::sync::atomic::AtomicUsize =
+ std::sync::atomic::AtomicUsize::new(0);
+
+ fn temp_parquet_path(tag: &str) -> std::path::PathBuf {
+ let n = FILE_COUNTER.fetch_add(1,
std::sync::atomic::Ordering::Relaxed);
+ std::env::temp_dir().join(format!(
+ "mahout_pipeline_dtype_{tag}_{pid}_{n}.parquet",
+ pid = std::process::id(),
+ ))
+ }
+
+ #[test]
+ fn test_read_file_by_extension_f32_parquet_returns_f32_batch_data() {
+ let path = temp_parquet_path("f32");
+ write_f32_parquet(&path);
+ let result = read_file_by_extension(&path, NullHandling::FillZero,
Precision::Float32);
+ let _ = fs::remove_file(&path);
+ let (batch_data, num_samples, sample_size) = result.unwrap();
+ assert!(
+ matches!(batch_data, BatchData::F32(_)),
Review Comment:
Tests now assert the actual values (first element ≈ 0.25), not just the
variant. Thanks for review.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
