HeartSaVioR commented on a change in pull request #28040: [SPARK-31278][SS] Fix
StreamingQuery output rows metric
URL: https://github.com/apache/spark/pull/28040#discussion_r399130697
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala
##########
@@ -203,46 +203,41 @@ class StreamingAggregationSuite extends
StateStoreMetricsTest with Assertions {
}
def stateOperatorProgresses: Seq[StateOperatorProgress] = {
- val operatorProgress = mutable.ArrayBuffer[StateOperatorProgress]()
- var progress = query.recentProgress.last
-
- operatorProgress ++= progress.stateOperators.map { op =>
op.copy(op.numRowsUpdated) }
- if (progress.numInputRows == 0) {
- // empty batch, merge metrics from previous batch as well
- progress = query.recentProgress.takeRight(2).head
- operatorProgress.zipWithIndex.foreach { case (sop, index) =>
- // "numRowsUpdated" should be merged, as it could be updated in
both batches.
- // (for now it is only updated from previous batch, but things can
be changed.)
- // other metrics represent current status of state so picking up
the latest values.
- val newOperatorProgress = sop.copy(
- sop.numRowsUpdated +
progress.stateOperators(index).numRowsUpdated)
- operatorProgress(index) = newOperatorProgress
- }
- }
-
- operatorProgress
+ query.recentProgress.last.stateOperators
}
}
+ val clock = new StreamManualClock()
+
testStream(aggWithWatermark)(
+ StartStream(Trigger.ProcessingTime("interval 1 second"), clock),
AddData(inputData, 15),
+ AdvanceManualClock(1000L),
CheckAnswer(), // watermark = 5
AssertOnQuery { _.stateNodes.size === 1 },
AssertOnQuery { _.stateNodes.head.metrics("numOutputRows").value === 0 },
AssertOnQuery { _.stateOperatorProgresses.head.numRowsUpdated === 1 },
AssertOnQuery { _.stateOperatorProgresses.head.numRowsTotal === 1 },
AddData(inputData, 10, 12, 14),
+ AdvanceManualClock(1000L),
CheckAnswer(), // watermark = 5
AssertOnQuery { _.stateNodes.size === 1 },
AssertOnQuery { _.stateNodes.head.metrics("numOutputRows").value === 0 },
AssertOnQuery { _.stateOperatorProgresses.head.numRowsUpdated === 1 },
AssertOnQuery { _.stateOperatorProgresses.head.numRowsTotal === 2 },
AddData(inputData, 25),
+ AdvanceManualClock(1000L),
+ CheckAnswer(), // watermark = 15, but nothing yet
+ AssertOnQuery { _.lastProgress.sink.numOutputRows == 0 },
+ AdvanceManualClock(1000L),
+ AssertOnQuery { _.lastProgress.sink.numOutputRows == 0 },
+ AssertOnQuery { _.stateOperatorProgresses.head.numRowsUpdated === 1 },
Review comment:
This confused me - this value is actually updated in previous time so the
better place of this line is after L231.
This makes me thinking the delay of output if the trigger interval is quite
huge. It should be uncommon, but it's not impossible with cron-ed continuous
execution of once trigger.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
