zecookiez commented on code in PR #50123:
URL: https://github.com/apache/spark/pull/50123#discussion_r2005150616
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreProvider.scala:
##########
@@ -966,3 +969,40 @@ class RocksDBStateStoreChangeDataReader(
}
}
}
+
+/**
+ * Object used to relay events reported from a RocksDB instance to the state
store coordinator.
+ *
+ * We pass this into the RocksDB instance to report specific events like
snapshot uploads.
+ * This should only be used to report back to the coordinator for metrics and
monitoring purposes.
+ */
+private[state] case class RocksDBEventListener(
+ queryRunId: String,
+ stateStoreId: StateStoreId,
+ storeConf: StateStoreConf) {
+
+ /** ID of the state store provider managing the RocksDB instance */
+ private val stateStoreProviderId: StateStoreProviderId =
+ StateStoreProviderId(stateStoreId, UUID.fromString(queryRunId))
+
+ /** Whether the event listener should relay these messages to the state
store coordinator */
+ private val coordinatorReportUploadEnabled: Boolean =
+ storeConf.stateStoreCoordinatorReportUploadEnabled
+
+ /**
+ * Callback function from RocksDB to report events to the coordinator.
+ * Additional information such as the state store ID and the query run ID are
+ * attached here to report back to the coordinator.
+ *
+ * @param version The snapshot version that was just uploaded from RocksDB
+ */
+ def reportSnapshotUploaded(version: Long): Unit = {
+ // Only report to the coordinator if this is enabled, as sometimes we do
not need
+ // to track for lagging instances.
+ // Also ignore message if we are missing the provider ID from lack of
initialization.
Review Comment:
Good catch, this is an outdated comment that I forgot to remove, thanks
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
