manuzhang commented on code in PR #4775:
URL: https://github.com/apache/datafusion-comet/pull/4775#discussion_r3900887644
##########
spark/src/main/scala/org/apache/comet/serde/aggregates.scala:
##########
@@ -763,6 +763,131 @@ object CometCorr extends
CometAggregateExpressionSerde[Corr] {
}
}
+/**
+ * Shared serialization for the simple linear regression aggregates. `child1`
is the dependent
+ * variable (y) and `child2` is the independent variable (x), matching the
native accumulator's
+ * `regr_*(y, x)` convention.
+ */
+trait CometRegrBase {
+ def convertRegr(
+ aggExpr: AggregateExpression,
+ regrType: ExprOuterClass.Regr.RegrType,
+ y: Expression,
+ x: Expression,
+ inputs: Seq[Attribute],
+ binding: Boolean): Option[ExprOuterClass.AggExpr] = {
+ val child1Expr = exprToProto(y, inputs, binding)
+ val child2Expr = exprToProto(x, inputs, binding)
+ val dataType = serializeDataType(DoubleType)
+
+ if (child1Expr.isDefined && child2Expr.isDefined && dataType.isDefined) {
+ val builder = ExprOuterClass.Regr.newBuilder()
+ builder.setChild1(child1Expr.get)
+ builder.setChild2(child2Expr.get)
+ builder.setRegrType(regrType)
+ builder.setDatatype(dataType.get)
+ // Spark 3.5 fixed regr_slope/regr_intercept so VariancePop(x) only
counts
Review Comment:
Maybe better (describing what not how)
```
VariancePop(x) only counts rows where both y and x are non-null in Spark
3.5+, but counts every row where x is non-null in 3.4
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
