shahwazreza commented on code in PR #27118: URL: https://github.com/apache/datafusion/pull/27118#discussion_r4497768387
########## datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt: ########## @@ -1163,3 +1163,93 @@ SET datafusion.execution.target_partitions = 4; statement ok RESET datafusion.execution.parquet.pushdown_filters; + +# Regression for https://github.com/apache/datafusion/issues/26029. +# When the TopK filter is pushed into a scan, each sort key is rewritten in +# terms of the file's columns. One row per row group lets TopK tighten its +# filter before the answer rows, which are last in each file, are read. +statement ok +SET datafusion.execution.target_partitions = 1; + +statement ok +SET datafusion.execution.parquet.pushdown_filters = true; + +statement ok +SET datafusion.explain.physical_plan_only = true; + +query I +COPY (SELECT value AS b FROM generate_series(0, 19)) +TO 'test_files/scratch/dynamic_filter_pushdown_config/issue_26029.parquet' +STORED AS PARQUET OPTIONS ('format.max_row_group_size' '1'); +---- +20 + +statement ok +CREATE EXTERNAL TABLE issue_26029 STORED AS PARQUET +LOCATION 'test_files/scratch/dynamic_filter_pushdown_config/issue_26029.parquet'; + +# The first sort key contains the second one, `b`. The whole CASE must be +# rewritten, not only the `b` inside it. `p` and `q` are equal, so the +# expected rows are easy to work out. +# +# The middle projection's column order is not stable between runs, so those +# parts are ignored. The check is that TopK's filter reaches the scan. +query TT +EXPLAIN WITH s AS (SELECT b, 19 - b AS p, 19 - b AS q FROM issue_26029) +SELECT p FROM s ORDER BY CASE WHEN b % 2 = 0 THEN p ELSE q END, b LIMIT 3; +---- +physical_plan +01)ProjectionExec: expr=[p@0 as p] +02)--SortExec: TopK(fetch=3), expr=[<slt:ignore>], preserve_partitioning=[false] +03)----ProjectionExec: expr=[<slt:ignore>] Review Comment: Issue submitted #27177 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
