zifeif2 commented on code in PR #53104:
URL: https://github.com/apache/spark/pull/53104#discussion_r2557236283
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/state/StatePartitionReader.scala:
##########
@@ -50,7 +50,16 @@ class StatePartitionReaderFactory(
override def createReader(partition: InputPartition):
PartitionReader[InternalRow] = {
val stateStoreInputPartition =
partition.asInstanceOf[StateStoreInputPartition]
if
(stateStoreInputPartition.sourceOptions.internalOnlyReadAllColumnFamilies) {
- new StatePartitionReaderAllColumnFamilies(storeConf, hadoopConf,
+ // Disable format validation because the schema returned by
+ // StatePartitionAllColumnFamiliesReader does not contain the
corresponding
+ // keySchema or valueSchema.
+ // It's safe to do so we also don't expect the caller of
StatePartitionAllColumnFamiliesReader
+ // to extract specific fields out of the returning row.
+ val modifiedStoreConf = storeConf.withExtraOptions(Map(
+ StateStoreConf.FORMAT_VALIDATION_ENABLED_CONFIG -> "false",
Review Comment:
this version disable format validation so that we don't need the keySchema
and valueSchema
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
