beliefer commented on code in PR #36332:
URL: https://github.com/apache/spark/pull/36332#discussion_r857304020
##########
sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/filter/Predicate.java:
##########
@@ -146,4 +149,210 @@ public class Predicate extends GeneralScalarExpression {
public Predicate(String name, Expression[] children) {
super(name, children);
}
+
+ public org.apache.spark.sql.sources.Filter toV1() {
+ String expressionStr = "";
+ for (Expression e : children()) {
+ expressionStr += e.describe() + ", ";
+ }
+
+ if (name().equals("=")) {
+ if (children()[1] instanceof LiteralValue) {
+ // e.g. a = 1
+ return new EqualTo(children()[0].describe(),
+
CatalystTypeConverters.convertToScala(((LiteralValue)children()[1]).value(),
Review Comment:
Two indents
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVScan.scala:
##########
@@ -70,8 +70,8 @@ case class CSVScan(
ExprUtils.verifyColumnNameOfCorruptRecord(dataSchema,
parsedOptions.columnNameOfCorruptRecord)
// Don't push any filter which refers to the "virtual" column which cannot
present in the input.
// Such filters will be applied later on the upper layer.
- val actualFilters =
-
pushedFilters.filterNot(_.references.contains(parsedOptions.columnNameOfCorruptRecord))
+ val actualFilters = pushedFilters.map(_.toV1)
+
.filterNot(_.references.contains(parsedOptions.columnNameOfCorruptRecord))
Review Comment:
+1
##########
sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/filter/Predicate.java:
##########
@@ -146,4 +149,210 @@ public class Predicate extends GeneralScalarExpression {
public Predicate(String name, Expression[] children) {
super(name, children);
}
+
+ public org.apache.spark.sql.sources.Filter toV1() {
+ String expressionStr = "";
+ for (Expression e : children()) {
+ expressionStr += e.describe() + ", ";
+ }
+
+ if (name().equals("=")) {
+ if (children()[1] instanceof LiteralValue) {
+ // e.g. a = 1
+ return new EqualTo(children()[0].describe(),
+
CatalystTypeConverters.convertToScala(((LiteralValue)children()[1]).value(),
+ ((LiteralValue)children()[1]).dataType()));
+ } else if (children()[0] instanceof LiteralValue) {
+ // e.g. 1 = a
+ return new EqualTo(children()[1].describe(),
+
CatalystTypeConverters.convertToScala(((LiteralValue)children()[0]).value(),
Review Comment:
ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
