chaoqin-li1123 commented on code in PR #46942:
URL: https://github.com/apache/spark/pull/46942#discussion_r1635337283
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala:
##########
@@ -874,6 +875,48 @@ class RocksDBSuite extends
AlsoTestWithChangelogCheckpointingEnabled with Shared
)
}
+ testWithChangelogCheckpointingEnabled("RocksDBFileManager: deepCopy") {
+ val dfsRootDir = new File(Utils.createTempDir().getAbsolutePath +
"/state/1/1")
+ val originalFileManager = new RocksDBFileManager(
+ dfsRootDir.getAbsolutePath, Utils.createTempDir(), new Configuration)
+
+ val file1 = RocksDBSstFile("001.sst", "dfs1", 100L)
+ val file2 = RocksDBSstFile("002.sst", "dfs2", 100L)
+
+ // Get access to the private ConcurrentHashMap
+ val origVersionToRocksDBFilesMap: ConcurrentHashMap[Long,
Seq[RocksDBImmutableFile]]
+ = getPrivateField(originalFileManager, "versionToRocksDBFiles")
Review Comment:
Try to verify the independence without accessing private field.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBFileManager.scala:
##########
@@ -135,7 +135,6 @@ class RocksDBFileManager(
private val versionToRocksDBFiles = new ConcurrentHashMap[Long,
Seq[RocksDBImmutableFile]]
-
Review Comment:
Unnecessary change.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -181,6 +190,7 @@ class RocksDB(
try {
if (loadedVersion != version) {
closeDB()
+ fileManager = fileManager.deepCopy()
Review Comment:
Can we comment here why we want need to make a deep copy here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
