zifeif2 commented on code in PR #53386:
URL: https://github.com/apache/spark/pull/53386#discussion_r2621206131
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StatePartitionAllColumnFamiliesWriterSuite.scala:
##########
@@ -146,18 +148,47 @@ class StatePartitionAllColumnFamiliesWriterSuite extends
StateDataSourceTestBase
assert(!checkpointFileExists(new File(targetDir, storeNamePath),
versionToCheck, ".changelog"))
assert(checkpointFileExists(new File(targetDir, storeNamePath),
versionToCheck, ".zip"))
- // Step 4: Read from target using normal reader
- val targetReader = spark.read
- .format("statestore")
- .option(StateSourceOptions.PATH, targetDir)
- val targetNormalData = (storeName match {
- case Some(name) => targetReader.option(StateSourceOptions.STORE_NAME,
name)
- case None => targetReader
- }).load()
- .selectExpr("key", "value", "partition_id")
- .collect()
+ // Step 3: Validate by reading from both source and target using normal
reader"
+ // Default selectExprs for most column families
+ val defaultSelectExprs = Seq("key", "value", "partition_id")
+
+ def shouldCheckColumnFamilyName: String => Boolean = name => {
+ (!name.startsWith("$")
+ || (columnFamilyToStateSourceOptions.contains(name) &&
+
columnFamilyToStateSourceOptions(name).contains(StateSourceOptions.READ_REGISTERED_TIMERS)))
+ }
+ // Validate each column family separately (skip internal column families
starting with $)
+ columnFamiliesToValidate
+ // TODO: How to validate that internal columns are written correctly?
Review Comment:
TODO: will rebase this on latest multi-cf reader and directly query internal
column with spark.read...
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
