Github user ueshin commented on a diff in the pull request:
https://github.com/apache/spark/pull/21054#discussion_r186020071
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/regression.scala
---
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions.aggregate
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.types.{AbstractDataType, DoubleType}
+
+/**
+ * Base trait for all regression functions.
+ */
+trait RegrLike extends AggregateFunction with ImplicitCastInputTypes {
+ def y: Expression
+ def x: Expression
+
+ override def children: Seq[Expression] = Seq(y, x)
+ override def inputTypes: Seq[AbstractDataType] = Seq(DoubleType,
DoubleType)
+
+ protected def updateIfNotNull(exprs: Seq[Expression]): Seq[Expression] =
{
+ assert(aggBufferAttributes.length == exprs.length)
+ val nullableChildren = children.filter(_.nullable)
+ if (nullableChildren.isEmpty) {
+ exprs
+ } else {
+ exprs.zip(aggBufferAttributes).map { case (e, a) =>
+ If(nullableChildren.map(IsNull).reduce(Or), a, e)
+ }
+ }
+ }
+}
+
+
+@ExpressionDescription(
+ usage = "_FUNC_(y, x) - Returns the number of non-null pairs.",
+ since = "2.4.0")
+case class RegrCount(y: Expression, x: Expression)
+ extends CountLike with RegrLike {
+
+ override lazy val updateExpressions: Seq[Expression] =
updateIfNotNull(Seq(count + 1L))
+
+ override def prettyName: String = "regr_count"
+}
+
+
+@ExpressionDescription(
+ usage = "_FUNC_(y, x) - Returns SUM(x*x)-SUM(x)*SUM(x)/N. Any pair with
a NULL is ignored.",
--- End diff --
@gatorsmile What do you think?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
