cloud-fan commented on code in PR #45782:
URL: https://github.com/apache/spark/pull/45782#discussion_r1551349926
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala:
##########
@@ -284,12 +284,15 @@ object DataSourceUtils extends PredicateHelper {
* Determines whether a filter should be pushed down to the data source or
not.
*
* @param expression The filter expression to be evaluated.
+ * @param isCollationPushDownSupported Whether the data source supports
collation push down.
* @return A boolean indicating whether the filter should be pushed down or
not.
*/
- def shouldPushFilter(expression: Expression): Boolean = {
- expression.deterministic && !expression.exists {
+ def shouldPushFilter(expression: Expression, isCollationPushDownSupported:
Boolean): Boolean = {
+ if (!expression.deterministic) return false
+
+ isCollationPushDownSupported || !expression.exists {
case childExpression @ (_: Attribute | _: GetStructField) =>
- // don't push down filters for types with non-default collation
+ // don't push down filters for types with utf8 binary collation
Review Comment:
non-binary-sortable?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
