kazuyukitanimura commented on code in PR #585:
URL: https://github.com/apache/datafusion-comet/pull/585#discussion_r1693605259
##########
spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala:
##########
@@ -985,23 +984,82 @@ object QueryPlanSerde extends Logging with
ShimQueryPlanSerde with CometExprShim
None
case EqualTo(left, right) =>
- val leftExpr = exprToProtoInternal(left, inputs)
- val rightExpr = exprToProtoInternal(right, inputs)
+ // this is a workaround for handling -0.0 in double and float
+ // untill https://github.com/apache/datafusion/issues/11108 is fixed
+ val leftZero = Literal.default(left.dataType)
+ val rightZero = Literal.default(right.dataType)
+ val negZeroLeft = UnaryMinus(leftZero)
+ val negZeroRight = UnaryMinus(rightZero)
+
+ def buildEqualExpr(
+ leftExpr: Option[Expr],
+ rightExpr: Option[Expr]): Option[ExprOuterClass.Expr] = {
+ if (leftExpr.isDefined && rightExpr.isDefined) {
+ Some(
+ ExprOuterClass.Expr
+ .newBuilder()
+ .setEq(
+ ExprOuterClass.Equal
+ .newBuilder()
+ .setLeft(leftExpr.get)
+ .setRight(rightExpr.get))
+ .build())
+ } else {
+ withInfo(expr, left, right)
+ None
+ }
+ }
+ if ((left.dataType == DoubleType &&
+ right.dataType == DoubleType) ||
+ (left.dataType == FloatType &&
+ right.dataType == FloatType)) {
+ (left, right) match {
+ case (`negZeroLeft`, `negZeroRight`) =>
+ return buildEqualExpr(
+ exprToProtoInternal(Abs(left).child, inputs),
+ exprToProtoInternal(Abs(right).child, inputs))
+ case (`negZeroLeft`, _) =>
+ return buildEqualExpr(
+ exprToProtoInternal(Abs(left).child, inputs),
+ exprToProtoInternal(right, inputs))
+ case (_, `negZeroRight`) =>
+ return buildEqualExpr(
+ exprToProtoInternal(left, inputs),
+ exprToProtoInternal(Abs(right).child, inputs))
+ case _ =>
+ val doubleNan = Literal(Double.NaN, DoubleType)
+ val floatNan = Literal(Float.NaN, FloatType)
+
+ // Ensure neither left nor right is -0.0 or 0.0 or NaN
+ // also return none if one side is nullable and the other is
not
+ if ((left.nullable && !right.nullable) &&
Review Comment:
What is the reason of the `left.nullable && !right.nullable` requirement?
##########
spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala:
##########
@@ -985,23 +984,82 @@ object QueryPlanSerde extends Logging with
ShimQueryPlanSerde with CometExprShim
None
case EqualTo(left, right) =>
- val leftExpr = exprToProtoInternal(left, inputs)
- val rightExpr = exprToProtoInternal(right, inputs)
+ // this is a workaround for handling -0.0 in double and float
+ // untill https://github.com/apache/datafusion/issues/11108 is fixed
+ val leftZero = Literal.default(left.dataType)
+ val rightZero = Literal.default(right.dataType)
+ val negZeroLeft = UnaryMinus(leftZero)
+ val negZeroRight = UnaryMinus(rightZero)
+
+ def buildEqualExpr(
+ leftExpr: Option[Expr],
+ rightExpr: Option[Expr]): Option[ExprOuterClass.Expr] = {
+ if (leftExpr.isDefined && rightExpr.isDefined) {
+ Some(
+ ExprOuterClass.Expr
+ .newBuilder()
+ .setEq(
+ ExprOuterClass.Equal
+ .newBuilder()
+ .setLeft(leftExpr.get)
+ .setRight(rightExpr.get))
+ .build())
+ } else {
+ withInfo(expr, left, right)
+ None
+ }
+ }
+ if ((left.dataType == DoubleType &&
+ right.dataType == DoubleType) ||
+ (left.dataType == FloatType &&
+ right.dataType == FloatType)) {
+ (left, right) match {
+ case (`negZeroLeft`, `negZeroRight`) =>
+ return buildEqualExpr(
+ exprToProtoInternal(Abs(left).child, inputs),
+ exprToProtoInternal(Abs(right).child, inputs))
+ case (`negZeroLeft`, _) =>
+ return buildEqualExpr(
+ exprToProtoInternal(Abs(left).child, inputs),
+ exprToProtoInternal(right, inputs))
+ case (_, `negZeroRight`) =>
+ return buildEqualExpr(
+ exprToProtoInternal(left, inputs),
+ exprToProtoInternal(Abs(right).child, inputs))
Review Comment:
Do you mind explaining why we need `Abs(right).child`? I thought
`Abs(right).child == right`
##########
spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala:
##########
@@ -982,23 +981,80 @@ object QueryPlanSerde extends Logging with
ShimQueryPlanSerde with CometExprShim
None
case EqualTo(left, right) =>
- val leftExpr = exprToProtoInternal(left, inputs)
- val rightExpr = exprToProtoInternal(right, inputs)
+ // this is a workaround for handling -0.0 in double and float
+ // untill https://github.com/apache/datafusion/issues/11108 is fixed
+ val leftZero = Literal.default(left.dataType)
+ val rightZero = Literal.default(right.dataType)
+ val negZeroLeft = UnaryMinus(leftZero)
+ val negZeroRight = UnaryMinus(rightZero)
+
+ def buildEqualExpr(
+ leftExpr: Option[Expr],
+ rightExpr: Option[Expr]): Option[ExprOuterClass.Expr] = {
+ if (leftExpr.isDefined && rightExpr.isDefined) {
+ Some(
+ ExprOuterClass.Expr
+ .newBuilder()
+ .setEq(
+ ExprOuterClass.Equal
+ .newBuilder()
+ .setLeft(leftExpr.get)
+ .setRight(rightExpr.get))
+ .build())
+ } else {
+ withInfo(expr, left, right)
+ None
+ }
+ }
+ if ((left.dataType == DoubleType &&
+ right.dataType == DoubleType) ||
+ (left.dataType == FloatType &&
+ right.dataType == FloatType)) {
+ (left, right) match {
+ case (`negZeroLeft`, `negZeroRight`) =>
+ return buildEqualExpr(
+ exprToProtoInternal(Abs(left).child, inputs),
+ exprToProtoInternal(Abs(right).child, inputs))
+ case (`negZeroLeft`, _) =>
+ return buildEqualExpr(
+ exprToProtoInternal(Abs(left).child, inputs),
+ exprToProtoInternal(right, inputs))
+ case (_, `negZeroRight`) =>
+ return buildEqualExpr(
+ exprToProtoInternal(left, inputs),
+ exprToProtoInternal(Abs(right).child, inputs))
+ case _ =>
+ val doubleNan = Literal(Double.NaN, DoubleType)
+ val floatNan = Literal(Float.NaN, FloatType)
+
+ if ((left.nullable && !right.nullable) &&
+ (left != negZeroLeft && right != negZeroRight) &&
+ (left != leftZero && right != rightZero) &&
+ (left != doubleNan && right != doubleNan) &&
+ (left != floatNan && right != floatNan)) {
+ withInfo(expr, left, right)
+ return None
+ }
+ buildEqualExpr(
+ exprToProtoInternal(left, inputs),
+ exprToProtoInternal(right, inputs))
+ }
+ }
- if (leftExpr.isDefined && rightExpr.isDefined) {
- val builder = ExprOuterClass.Equal.newBuilder()
- builder.setLeft(leftExpr.get)
- builder.setRight(rightExpr.get)
+ val leftExpr =
+ if (left.dataType == DoubleType || left.dataType == FloatType) {
+ exprToProtoInternal(If(EqualTo(left, negZeroLeft), leftZero,
left), inputs)
Review Comment:
Hmmm still trying to understand how this recursion works...
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
