alamb commented on code in PR #7522:
URL: https://github.com/apache/arrow-rs/pull/7522#discussion_r2093552973
##########
parquet/src/arrow/async_reader/mod.rs:
##########
@@ -1555,6 +1537,68 @@ mod tests {
#[tokio::test]
async fn test_row_filter() {
+ let a = StringArray::from_iter_values(["a", "b", "b", "b", "c", "c"]);
+ let b = StringArray::from_iter_values(["1", "2", "3", "4", "5", "6"]);
+ let data = RecordBatch::try_from_iter([
+ ("a", Arc::new(a) as ArrayRef),
+ ("b", Arc::new(b) as ArrayRef),
+ ])
+ .unwrap();
+
+ let mut buf = Vec::with_capacity(1024);
+ let mut writer = ArrowWriter::try_new(&mut buf, data.schema(),
None).unwrap();
+ writer.write(&data).unwrap();
+ writer.close().unwrap();
+
+ let data: Bytes = buf.into();
+ let metadata = ParquetMetaDataReader::new()
+ .parse_and_finish(&data)
+ .unwrap();
+ let parquet_schema = metadata.file_metadata().schema_descr_ptr();
+
+ let test = TestReader::new(data);
+ let requests = test.requests.clone();
+
+ let a_scalar = StringArray::from_iter_values(["b"]);
+ let a_filter = ArrowPredicateFn::new(
+ ProjectionMask::leaves(&parquet_schema, vec![0]),
+ move |batch| eq(batch.column(0), &Scalar::new(&a_scalar)),
+ );
+
+ let filter = RowFilter::new(vec![Box::new(a_filter)]);
+
+ let mask = ProjectionMask::leaves(&parquet_schema, vec![0, 1]);
+ let stream = ParquetRecordBatchStreamBuilder::new(test)
+ .await
+ .unwrap()
+ .with_projection(mask.clone())
+ .with_batch_size(1024)
+ .with_row_filter(filter)
+ .build()
+ .unwrap();
+
+ let batches: Vec<_> = stream.try_collect().await.unwrap();
+ assert_eq!(batches.len(), 1);
+
+ let batch = &batches[0];
+ assert_eq!(batch.num_columns(), 2);
+
+ // Filter should have kept only rows with "b" in column 0
+ assert_eq!(
+ batch.column(0).as_ref(),
+ &StringArray::from_iter_values(["b", "b", "b"])
+ );
+ assert_eq!(
+ batch.column(1).as_ref(),
+ &StringArray::from_iter_values(["2", "3", "4"])
+ );
+
+ // Should only have made 2 requests
Review Comment:
That is an excellent point. I believe the rationale is like this:
```suggestion
// Should only have made 2 requests. First request fetches the
// data for evaluating the predicate and second request fetches
// the data for evaluating the projection
```
This test is largely copied from `test_two_row_filters` so we should also
update that test's comments too (it has 2 predicates, so 3 requests (one each
for the predicates and then for the projection)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
