anishshri-db commented on code in PR #52960:
URL: https://github.com/apache/spark/pull/52960#discussion_r2515225233
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala:
##########
@@ -1393,6 +1393,70 @@ class RocksDBSuite extends AlsoTestWithRocksDBFeatures
with SharedSparkSession
encodeMethod.invoke(db, key.getBytes, cfName).asInstanceOf[Array[Byte]]
}
+ testWithStateStoreCheckpointIdsAndColumnFamilies(
+ "RocksDB: keyExists over 1000 random keys across CFs",
+ TestWithBothChangelogCheckpointingEnabledAndDisabled) {
+ case (enableStateStoreCheckpointIds, colFamiliesEnabled) =>
+ val remoteDir = Utils.createTempDir().toString
+ new File(remoteDir).delete()
+
+ val conf = dbConf.copy(compactOnCommit = false)
+ withDB(
+ remoteDir,
+ conf = conf,
+ useColumnFamilies = colFamiliesEnabled,
+ enableStateStoreCheckpointIds = enableStateStoreCheckpointIds) { db =>
+ val totalPresent = 500
+ val totalAbsent = 500
+
+ // Generate present and absent keys using simple disjoint prefixes
+ val presentKeysAll = (0 until totalPresent).map(i => s"present_$i")
+
+ // Insert present keys
+ db.load(0)
+ // If column families are enabled, create a CF and use it uniformly
(after load)
+ val cfNameOpt =
+ if (colFamiliesEnabled) {
+ val cf = "test_cf_random"
+ db.createColFamilyIfAbsent(cf, isInternal = false)
+ Some(cf)
+ } else {
+ None
+ }
+ cfNameOpt match {
+ case Some(cf) =>
+ presentKeysAll.foreach { k => db.put(k, s"v_$k", cf) }
+ case None =>
+ presentKeysAll.foreach { k => db.put(k, s"v_$k") }
+ }
+ db.commit()
+
+ // Generate absent keys using a different prefix to avoid overlap
+ val absentKeysAll = (0 until totalAbsent).map(i => s"absent_$i")
+
+ // Reload and test
+ db.load(0)
+ db.load(1)
Review Comment:
Why are we repeating the `load` ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
