viirya commented on a change in pull request #33455:
URL: https://github.com/apache/spark/pull/33455#discussion_r673773677
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreProvider.scala
##########
@@ -215,40 +239,79 @@ object RocksDBStateStoreProvider {
// Native operation latencies report as latency per 1000 calls
// as SQLMetrics support ms latency whereas RocksDB reports it in
microseconds.
- val CUSTOM_METRIC_GET_TIME = StateStoreCustomTimingMetric(
- "rocksdbGetLatency", "RocksDB: avg get latency (per 1000 calls)")
- val CUSTOM_METRIC_PUT_TIME = StateStoreCustomTimingMetric(
- "rocksdbPutLatency", "RocksDB: avg put latency (per 1000 calls)")
+ val CUSTOM_METRIC_GET_TIME = StateStoreCustomTimingNsMetric(
+ "rocksdbGetLatency", "RocksDB: avg get latency")
+ val CUSTOM_METRIC_PUT_TIME = StateStoreCustomTimingNsMetric(
+ "rocksdbPutLatency", "RocksDB: avg put latency")
+
+ val CUSTOM_METRIC_GET_COUNT = StateStoreCustomSumMetric(
+ "rocksdbGetCount", "RocksDB: number of get calls")
+ val CUSTOM_METRIC_PUT_COUNT = StateStoreCustomSumMetric(
+ "rocksdbPutCount", "RocksDB: number of put calls")
// Commit latency detailed breakdown
val CUSTOM_METRIC_WRITEBATCH_TIME = StateStoreCustomTimingMetric(
"rocksdbCommitWriteBatchLatency", "RocksDB: commit - write batch time")
val CUSTOM_METRIC_FLUSH_TIME = StateStoreCustomTimingMetric(
"rocksdbCommitFlushLatency", "RocksDB: commit - flush time")
+ val CUSTOM_METRIC_COMMIT_COMPACT_TIME = StateStoreCustomTimingMetric(
+ "rocksdbCommitCompactLatency", "RocksDB: commit - compact time")
val CUSTOM_METRIC_PAUSE_TIME = StateStoreCustomTimingMetric(
"rocksdbCommitPauseLatency", "RocksDB: commit - pause bg time")
val CUSTOM_METRIC_CHECKPOINT_TIME = StateStoreCustomTimingMetric(
"rocksdbCommitCheckpointLatency", "RocksDB: commit - checkpoint time")
val CUSTOM_METRIC_FILESYNC_TIME = StateStoreCustomTimingMetric(
- "rocksdbFileSyncTime", "RocksDB: commit - file sync time")
- val CUSTOM_METRIC_FILES_COPIED = StateStoreCustomSizeMetric(
+ "rocksdbCommitFileSyncLatencyMs", "RocksDB: commit - file sync to external
storage time")
+ val CUSTOM_METRIC_FILES_COPIED = StateStoreCustomSumMetric(
Review comment:
Good catch.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
