HeartSaVioR commented on code in PR #42822:
URL: https://github.com/apache/spark/pull/42822#discussion_r1325236875
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/statefulOperators.scala:
##########
@@ -183,7 +184,6 @@ trait StateStoreWriter extends StatefulOperator with
PythonSQLMetrics { self: Sp
assert(numStateStoreInstances >= 1, s"invalid number of stores:
$numStateStoreInstances")
// Shuffle partitions capture the number of tasks that have this stateful
operator instance.
// For each task instance this number is incremented by one.
- longMetric("numShufflePartitions") += 1
longMetric("numStateStoreInstances") += numStateStoreInstances
Review Comment:
This is also something which could be off, but let's deal with this
separately as for most operators it's just the same.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/statefulOperators.scala:
##########
@@ -183,7 +184,6 @@ trait StateStoreWriter extends StatefulOperator with
PythonSQLMetrics { self: Sp
assert(numStateStoreInstances >= 1, s"invalid number of stores:
$numStateStoreInstances")
// Shuffle partitions capture the number of tasks that have this stateful
operator instance.
// For each task instance this number is incremented by one.
- longMetric("numShufflePartitions") += 1
longMetric("numStateStoreInstances") += numStateStoreInstances
Review Comment:
This is also something which could be off, but let's deal with this
separately as for most operators it's just the same with numShufflePartitions.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
