This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23406-5e80f6cda3a24ed827e51328b094060e3a0d02bd in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 761f7f691894ee093efc9e610a108782c3450af9 Author: Prateek Ganigi <[email protected]> AuthorDate: Sat Jul 11 19:17:15 2026 -0700 test: add sqllogictest coverage for DISTINCT / GROUP BY / aggregation on map columns (#23406) ## Which issue does this PR close? - Closes #15428. ## Rationale for this change `SELECT DISTINCT` on map columns used to fail with `ArrowError(NotYetImplemented)` because arrow-rs's `RowConverter` did not support Map types. That support has since landed upstream (apache/arrow-rs#7879) and is available on DataFusion main via arrow 59.1, so these queries now work. As suggested in https://github.com/apache/datafusion/issues/15428#issuecomment-4911477288 (maintainer comment), this PR adds sqllogictest coverage to lock in the behavior and close the issue. ## What changes are included in this PR? Adds a new test section to `datafusion/sqllogictest/test_files/map.slt` covering: - `SELECT DISTINCT` on a map column, including the exact reproducer from the issue (`DISTINCT ... LIMIT`), collapsing of duplicate maps and duplicate NULLs, and `DISTINCT` over map + scalar columns together - `GROUP BY` on a map column, `GROUP BY` map + scalar keys, and `HAVING` with a map grouping key - Aggregations with map inputs: `COUNT(map)`, `COUNT(DISTINCT map)`, `SUM` grouped by map, and `array_agg` of map values - Edge cases: empty maps under `DISTINCT` (equal to each other, distinct from `NULL`), NULL maps, and unsorted maps with the same entries in different order being treated as distinct values - `UNION` (distinct) on map columns - `DISTINCT` / `GROUP BY` over the existing 209-row `parquet_map.parquet` file for coverage on real Parquet-backed map data Note: a bare `MAP {'key': NULL}` literal infers a `Null` value type and fails type coercion across a VALUES list against `Map(Utf8, Int64)` rows, so the test uses `CAST(NULL AS BIGINT)` for the null map value. That coercion gap may be worth a separate follow-up issue. ## Are these changes tested? Yes, this PR is test-only. All tests pass locally via `cargo test -p datafusion-sqllogictest --test sqllogictests -- map.slt`. ## Are there any user-facing changes? No. Test-only change; no API or behavior changes. --- datafusion/sqllogictest/test_files/map.slt | 132 +++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/datafusion/sqllogictest/test_files/map.slt b/datafusion/sqllogictest/test_files/map.slt index 970ae2707d..f8cbb395cb 100644 --- a/datafusion/sqllogictest/test_files/map.slt +++ b/datafusion/sqllogictest/test_files/map.slt @@ -916,3 +916,135 @@ SELECT map([column1, column1 * 10], ['x','y']) FROM (VALUES (1), (2), (3)) t; {1: x, 10: y} {2: x, 20: y} {3: x, 30: y} + +# tests for DISTINCT / GROUP BY / aggregation on map columns +# https://github.com/apache/datafusion/issues/15428 + +statement ok +CREATE TABLE map_distinct_table AS VALUES + (MAP {'k1': 1, 'k2': 2}, 'a', 1), + (MAP {'k1': 1, 'k2': 2}, 'a', 2), + (MAP {'k1': 1, 'k2': 2}, 'b', 3), + (MAP {'k1': 3}, 'a', 4), + (MAP {'k1': CAST(NULL AS BIGINT)}, 'b', 5); + +statement ok +INSERT INTO map_distinct_table VALUES (NULL, 'a', 6), (NULL, 'a', 7), (NULL, 'b', 8); + +# distinct on a map column collapses duplicate maps and duplicate NULLs +query ? rowsort +SELECT DISTINCT column1 FROM map_distinct_table; +---- +NULL +{k1: 1, k2: 2} +{k1: 3} +{k1: NULL} + +# exact reproducer from #15428: DISTINCT on a map column with LIMIT +query ? rowsort +SELECT DISTINCT column1 FROM map_distinct_table LIMIT 10; +---- +NULL +{k1: 1, k2: 2} +{k1: 3} +{k1: NULL} + +# distinct over a map column together with a scalar column +query ?T rowsort +SELECT DISTINCT column1, column2 FROM map_distinct_table; +---- +NULL a +NULL b +{k1: 1, k2: 2} a +{k1: 1, k2: 2} b +{k1: 3} a +{k1: NULL} b + +# group by a map column +query ?II rowsort +SELECT column1, COUNT(*), SUM(column3) FROM map_distinct_table GROUP BY column1; +---- +NULL 3 21 +{k1: 1, k2: 2} 3 6 +{k1: 3} 1 4 +{k1: NULL} 1 5 + +# group by a map column and a scalar column +query ?TI rowsort +SELECT column1, column2, COUNT(*) FROM map_distinct_table GROUP BY column1, column2; +---- +NULL a 2 +NULL b 1 +{k1: 1, k2: 2} a 2 +{k1: 1, k2: 2} b 1 +{k1: 3} a 1 +{k1: NULL} b 1 + +# empty maps compare equal under DISTINCT and are distinct from NULL +query ? rowsort +SELECT DISTINCT column1 FROM (VALUES (MAP {}), (MAP {}), (NULL)) t(column1); +---- +NULL +{} + +# HAVING clause with a map grouping key +query ?I rowsort +SELECT column1, COUNT(*) FROM map_distinct_table GROUP BY column1 HAVING COUNT(*) > 1; +---- +NULL 3 +{k1: 1, k2: 2} 3 + +# count and count distinct on a map column +query II +SELECT COUNT(column1), COUNT(DISTINCT column1) FROM map_distinct_table; +---- +5 3 + +# map column as input to an aggregate function +query T? +SELECT column2, array_agg(column1 ORDER BY column3) FROM map_distinct_table GROUP BY column2 ORDER BY column2; +---- +a [{k1: 1, k2: 2}, {k1: 1, k2: 2}, {k1: 3}, NULL, NULL] +b [{k1: 1, k2: 2}, {k1: NULL}, NULL] + +# UNION (distinct) on map columns +query ? +SELECT MAP {'a': 1} UNION SELECT MAP {'a': 1}; +---- +{a: 1} + +# unsorted maps are compared by entry order: maps with the same entries in a +# different order are treated as distinct values +query ? rowsort +SELECT DISTINCT column1 FROM (SELECT MAP {'k1': 1, 'k2': 2} AS column1 UNION ALL SELECT MAP {'k2': 2, 'k1': 1}); +---- +{k1: 1, k2: 2} +{k2: 2, k1: 1} + +statement ok +DROP TABLE map_distinct_table; + +# distinct / group by on map columns read from parquet +statement ok +CREATE EXTERNAL TABLE map_data +STORED AS PARQUET +LOCATION '../core/tests/data/parquet_map.parquet'; + +query I +SELECT COUNT(*) FROM (SELECT DISTINCT ints, strings FROM map_data); +---- +209 + +query TI rowsort +SELECT strings['method'] AS method, COUNT(*) FROM (SELECT DISTINCT strings FROM map_data) GROUP BY method; +---- +DELETE 24 +GET 27 +HEAD 33 +OPTION 29 +PATCH 30 +POST 41 +PUT 25 + +statement ok +DROP TABLE map_data; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
