ericm-db commented on code in PR #54567:
URL: https://github.com/apache/spark/pull/54567#discussion_r2874831850
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -244,6 +244,9 @@ class RocksDB(
// Was snapshot auto repair performed when loading the current version
@volatile private var performedSnapshotAutoRepair = false
+ // Was a DFS (cloud) fetch performed when loading the current version
+ @volatile private var loadedFromCloud = 0
Review Comment:
Also, can we change the name to numCloudLoads or something?
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -244,6 +244,9 @@ class RocksDB(
// Was snapshot auto repair performed when loading the current version
@volatile private var performedSnapshotAutoRepair = false
+ // Was a DFS (cloud) fetch performed when loading the current version
+ @volatile private var loadedFromCloud = 0
Review Comment:
Let's make this a Long, 0L
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -680,17 +698,15 @@ class RocksDB(
override protected def beforeLoad(): Unit = closeDB(ignoreException =
false)
override protected def loadSnapshotFromCheckpoint(snapshotVersion:
Long): Unit = {
- val remoteMetaData = fileManager.loadCheckpointFromDfs(snapshotVersion,
- workingDir, rocksDBFileMapping)
+ val metadata = fetchCheckpointFromDfs(snapshotVersion)
- loadedVersion = snapshotVersion
// Initialize maxVersion upon successful load from DFS
fileManager.setMaxSeenVersion(snapshotVersion)
- openLocalRocksDB(remoteMetaData)
+ openLocalRocksDB(metadata)
// By setting this to the snapshot version we successfully loaded,
- // if auto snapshot repair is enabled, and we end up skipping the
latest snapshot
+ // if auto snapshot repair is enabled and we end up skipping the
latest snapshot
Review Comment:
undo this change to reduce diff
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
