WweiL commented on code in PR #48355:
URL: https://github.com/apache/spark/pull/48355#discussion_r1870463098
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -431,16 +592,27 @@ class RocksDB(
/**
* Replay change log from the loaded version to the target version.
*/
- private def replayChangelog(endVersion: Long): Unit = {
+ private def replayChangelog(
+ endVersion: Long,
+ stateStoreCkptIdLineage: Option[Array[LineageItem]] = None): Unit = {
+
+ val versionsAndUniqueIds = stateStoreCkptIdLineage match {
+ // First entry of lineage corresponds to loadedVersion
+ case Some(lineage) => lineage.map(i => (i.version,
Some(i.checkpointUniqueId)))
+ case None => (loadedVersion + 1 to endVersion).map((_, None)).toArray
+ }
+
Review Comment:
sure change
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala:
##########
@@ -431,16 +592,27 @@ class RocksDB(
/**
* Replay change log from the loaded version to the target version.
*/
- private def replayChangelog(endVersion: Long): Unit = {
+ private def replayChangelog(
+ endVersion: Long,
+ stateStoreCkptIdLineage: Option[Array[LineageItem]] = None): Unit = {
+
+ val versionsAndUniqueIds = stateStoreCkptIdLineage match {
+ // First entry of lineage corresponds to loadedVersion
+ case Some(lineage) => lineage.map(i => (i.version,
Some(i.checkpointUniqueId)))
+ case None => (loadedVersion + 1 to endVersion).map((_, None)).toArray
+ }
+
Review Comment:
sure changed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
