uros-b commented on code in PR #57629:
URL: https://github.com/apache/spark/pull/57629#discussion_r3700541803
##########
sql/core/src/test/resources/sql-tests/inputs/distinct-map-aggregates.sql:
##########
@@ -0,0 +1,85 @@
+-- Test DISTINCT aggregates with MapType arguments.
+
+CREATE OR REPLACE TEMPORARY VIEW distinct_map_data AS SELECT * FROM VALUES
+ (2, map('a', 1, 'b', 2), 1, true),
+ (2, map('b', 2, 'a', 1), 1, true),
+ (1, map('a', 1, 'b', 2), 1, true),
+ (1, map('a', 3), 2, false)
+AS distinct_map_data(g, m, id, should_keep);
+
+SELECT COUNT(DISTINCT m) FROM distinct_map_data;
+
+SELECT SIZE(COLLECT_LIST(DISTINCT m)) FROM distinct_map_data;
+
+SELECT COLLECT_LIST(DISTINCT m) FROM distinct_map_data;
+
+SELECT map_entries(FIRST(DISTINCT m)), map_entries(LAST(DISTINCT m)),
COUNT(DISTINCT m)
+FROM VALUES (map('b', 2, 'a', 1)) AS single_map_data(m);
+
+SELECT COUNT(DISTINCT m, id) FROM distinct_map_data;
+
+SELECT COUNT(DISTINCT m), COUNT(DISTINCT id) FROM distinct_map_data;
+
+SELECT g, COUNT(DISTINCT m)
+FROM distinct_map_data
+GROUP BY g
+ORDER BY g;
+
+SELECT m, COUNT(DISTINCT m), COLLECT_LIST(DISTINCT m)
+FROM distinct_map_data
+GROUP BY m
+ORDER BY element_at(m, 'a');
+
+SELECT COUNT(DISTINCT m) FILTER (WHERE should_keep) FROM distinct_map_data;
+
+SELECT MAX(map_values(m)[0])
+FROM distinct_map_data
+WHERE id = 1;
+
+SELECT MAX(map_values(m)[0]), COUNT(DISTINCT m)
+FROM distinct_map_data
+WHERE id = 1;
+
+SELECT g
+FROM distinct_map_data
+GROUP BY g
+ORDER BY COUNT(DISTINCT m), g;
+
+SELECT g
+FROM distinct_map_data
+GROUP BY g
+HAVING COUNT(DISTINCT m) = 1
+ORDER BY g;
+
+SELECT COUNT(DISTINCT named_struct('m', m)) FROM distinct_map_data;
+
+SELECT COUNT(DISTINCT array(m)) FROM distinct_map_data;
+
+SELECT COUNT(DISTINCT map('m', m)) FROM distinct_map_data;
+
+SELECT COUNT(DISTINCT m), COLLECT_LIST(DISTINCT m)
+FROM VALUES
+ (CAST(map() AS MAP<STRING, INT>)),
+ (CAST(map() AS MAP<STRING, INT>)),
+ (CAST(NULL AS MAP<STRING, INT>))
+AS null_and_empty_map_data(m);
+
+SELECT g, GROUPING(g), COUNT(DISTINCT m)
+FROM distinct_map_data
+GROUP BY GROUPING SETS ((g), ())
+ORDER BY GROUPING(g), g;
+
+SELECT COUNT(DISTINCT named_struct('m', m, 'n', n))
+FROM VALUES
+ (map('a', 1, 'b', 2), map('x', 1, 'y', 2)),
+ (map('b', 2, 'a', 1), map('y', 2, 'x', 1))
+AS grouped_distinct_map_data(m, n)
+GROUP BY m;
+
+SET spark.sql.optimizer.insertMapSortInDistinctAggregates.enabled=false;
+
+SELECT COUNT(DISTINCT m), COLLECT_LIST(DISTINCT m) FROM distinct_map_data;
+
+SELECT COUNT(DISTINCT named_struct('m', m)) FROM distinct_map_data;
+
Review Comment:
The killswitch isn't a complete revert, and that deserves an additional
test. When the map is both a grouping key and a distinct argument, turning the
config off still yields a normalized value: distinctExpressions is empty, but
the grouping-alias substitution in the transformDown catch-all rewrites m
inside COUNT(DISTINCT m) to _groupingmapsort anyway. That's exactly the pre-PR
behavior, so it's the correct thing for a killswitch to do, but the config doc
("MapSort is not added specifically for distinct aggregate arguments") is easy
to misread as a full revert. A SELECT m, COUNT(DISTINCT m) ... GROUP BY m case
in the enabled=false block would lock the intent down.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
