adriangb commented on code in PR #16086:
URL: https://github.com/apache/datafusion/pull/16086#discussion_r2094868000
##########
datafusion/core/src/datasource/physical_plan/parquet.rs:
##########
@@ -848,6 +848,68 @@ mod tests {
assert_eq!(read.len(), 0);
}
+ #[tokio::test]
+ async fn evolved_schema_column_type_filter_strings() {
+ // The table and filter have a common data type, but the file schema
differs
+ let c1: ArrayRef =
+ Arc::new(StringViewArray::from(vec![Some("foo"), Some("bar")]));
+ let batch = create_batch(vec![("c1", c1.clone())]);
+
+ let schema = Arc::new(Schema::new(vec![Field::new("c1",
DataType::Utf8, false)]));
+
+ // Predicate should prune all row groups
+ let filter =
col("c1").eq(lit(ScalarValue::Utf8(Some("aaa".to_string()))));
+ let read = RoundTrip::new()
+ .with_predicate(filter)
+ .with_schema(schema.clone())
+ .round_trip_to_batches(vec![batch.clone()])
+ .await
+ .unwrap();
+ assert_eq!(read.len(), 0);
+
+ // Predicate should prune no row groups
+ let filter =
col("c1").eq(lit(ScalarValue::Utf8(Some("foo".to_string()))));
+ let read = RoundTrip::new()
+ .with_predicate(filter)
+ .with_schema(schema)
+ .round_trip_to_batches(vec![batch])
+ .await
+ .unwrap();
+ assert_eq!(read.len(), 1);
+ }
+
+ #[tokio::test]
+ async fn evolved_schema_column_type_filter_ints() {
+ // The table and filter have a common data type, but the file schema
differs
+ let c1: ArrayRef = Arc::new(Int8Array::from(vec![Some(1), Some(2)]));
+ let batch = create_batch(vec![("c1", c1.clone())]);
+
+ let schema =
+ Arc::new(Schema::new(vec![Field::new("c1", DataType::UInt64,
false)]));
+
+ // Predicate should prune all row groups
+ let filter = col("c1").eq(lit(ScalarValue::UInt64(Some(5))));
+ let read = RoundTrip::new()
+ .with_predicate(filter)
+ .with_schema(schema.clone())
+ .round_trip_to_batches(vec![batch.clone()])
+ .await
+ .unwrap();
+ assert_eq!(read.len(), 0);
+
+ // TODO: this is failing on main, and has been for a long time!
+ // See <comment on PR>
Review Comment:
```suggestion
// See
https://github.com/apache/datafusion/pull/16086#discussion_r2094817127
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
