HeartSaVioR commented on code in PR #47528:
URL: https://github.com/apache/spark/pull/47528#discussion_r1704813542
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/OperatorStateMetadata.scala:
##########
@@ -291,30 +312,54 @@ class OperatorStateMetadataV2Writer(
class OperatorStateMetadataV2Reader(
stateCheckpointPath: Path,
- hadoopConf: Configuration) extends OperatorStateMetadataReader {
+ hadoopConf: Configuration,
+ batchId: Long) extends OperatorStateMetadataReader {
+
+ // Check that the requested batchId is available in the checkpoint directory
+ val baseCheckpointDir = stateCheckpointPath.getParent.getParent
+ val lastAvailOffset =
listOffsets(baseCheckpointDir).lastOption.getOrElse(-1L)
+ if (batchId > lastAvailOffset) {
+ throw
StateDataSourceErrors.failedToReadOperatorMetadata(baseCheckpointDir.toString,
batchId)
+ }
private val metadataDirPath =
OperatorStateMetadataV2.metadataDirPath(stateCheckpointPath)
private lazy val fm = CheckpointFileManager.create(metadataDirPath,
hadoopConf)
fm.mkdirs(metadataDirPath.getParent)
+
override def version: Int = 2
- private def listBatches(): Array[Long] = {
+ // List the available offsets in the offset directory
+ private def listOffsets(baseCheckpointDir: Path): Array[Long] = {
+ val offsetLog = new Path(baseCheckpointDir, "offsets")
Review Comment:
Ah OK, fine for me. Just wanted to see whether we could avoid the code
duplication.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
