hvanhovell commented on a change in pull request #26127: [SPARK-29348][SQL] Add
observable Metrics for Streaming queries
URL: https://github.com/apache/spark/pull/26127#discussion_r351946481
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
##########
@@ -961,3 +961,51 @@ case class Deduplicate(
* This is used to whitelist such commands in the subquery-related checks.
*/
trait SupportsSubquery extends LogicalPlan
+
+/**
+ * Collect arbitrary (named) metrics from a dataset. As soon as the query
reaches a completion
+ * point (batch query completes or streaming query epoch completes) an event
is emitted on the
+ * driver which can be observed by attaching a listener to the spark session.
The metrics are named
+ * so we can collect metrics at multiple places in a single dataset.
+ *
+ * This node behaves like a global aggregate. All the metrics collected must
be aggregate functions
+ * or be literals.
+ */
+case class CollectMetrics(
+ name: String,
+ metrics: Seq[NamedExpression],
+ child: LogicalPlan)
+ extends UnaryNode {
+
+ /**
+ * Check if an expression is a valid metric. A metric must meet the
following criteria:
+ * - Is not a window function;
+ * - Is not nested aggregate function;
+ * - Is not a distinct aggregate function;
+ * - Has only non-deterministic functions that are nested inside an
aggregate function;
+ * - Has only attributes that are nested inside an aggregate function.
+ *
+ * @param e expression to check.
+ * @param seenAggregate `true` iff one of the parents on the expression is
an aggregate function.
Review comment:
Done.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
