manuzhang commented on code in PR #4775:
URL: https://github.com/apache/datafusion-comet/pull/4775#discussion_r3900869807
##########
spark/src/main/scala/org/apache/comet/serde/aggregates.scala:
##########
@@ -763,6 +763,131 @@ object CometCorr extends
CometAggregateExpressionSerde[Corr] {
}
}
+/**
+ * Shared serialization for the simple linear regression aggregates. `child1`
is the dependent
+ * variable (y) and `child2` is the independent variable (x), matching the
native accumulator's
+ * `regr_*(y, x)` convention.
+ */
+trait CometRegrBase {
+ def convertRegr(
+ aggExpr: AggregateExpression,
+ regrType: ExprOuterClass.Regr.RegrType,
+ y: Expression,
+ x: Expression,
+ inputs: Seq[Attribute],
+ binding: Boolean): Option[ExprOuterClass.AggExpr] = {
+ val child1Expr = exprToProto(y, inputs, binding)
+ val child2Expr = exprToProto(x, inputs, binding)
+ val dataType = serializeDataType(DoubleType)
+
+ if (child1Expr.isDefined && child2Expr.isDefined && dataType.isDefined) {
+ val builder = ExprOuterClass.Regr.newBuilder()
+ builder.setChild1(child1Expr.get)
+ builder.setChild2(child2Expr.get)
+ builder.setRegrType(regrType)
+ builder.setDatatype(dataType.get)
+ // Spark 3.5 fixed regr_slope/regr_intercept so VariancePop(x) only
counts
+ // rows where both y and x are non-null. Spark 3.4 counts every row
where x
+ // is non-null. The native accumulator only consults this for
slope/intercept.
+ builder.setFilterVarByPairNulls(isSpark35Plus)
+ // Spark swapped regr_r2's degenerate-case handling: a constant dependent
+ // variable now yields 1.0 (was null) and a constant independent variable
+ // yields null (was 1.0). The swap is present in the Spark versions Comet
+ // builds against for 3.5 and later (3.5.9, 4.0.3+, 4.1, 4.2) but not in
3.4.
Review Comment:
same here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
