hubcio commented on code in PR #4278:
URL: https://github.com/apache/iggy/pull/4278#discussion_r4255810929
##########
core/partitions/src/state_transfer.rs:
##########
@@ -3331,16 +3334,27 @@ where
}
InstallPhase::Drain => {
if let Some(persistence) = &self.persistence {
+ if install.drain.is_none() {
+ // Checkpoint syncs may be skipped when the frontier
is unchanged.
+ // The backup still needs proof from each original
writer.
+ install.synced_files =
self.barrier_install_files_locked();
+ install.disposition = InstallDisposition::Backup;
+ }
self.start_persistence();
let drain = install
.drain
.get_or_insert_with(|| persistence.begin_drain());
match persistence.observe_drain(drain) {
Ok(true) => {
- install.disposition = InstallDisposition::Backup;
install.phase = InstallPhase::Backup;
}
Ok(false) => step = crate::PartitionIoStep::Pending,
+ Err(source)
Review Comment:
warning: ignoring the drain deadline lets a stalled sync block partition
recovery and shutdown indefinitely. honor the deadline and fence the partition
while retaining the active writer.
##########
core/partitions/src/install_backup.rs:
##########
@@ -150,15 +163,23 @@ async fn link_tree<S: DurableStorage>(
} else {
// Transfer unlinks or atomically replaces these frozen files.
// Hard links retain the old bytes without copying segment
data.
- storage.hard_link(&source.join(&name), &destination).await?;
- storage
- .open(&destination, OpenMode::Read)
- .await?
- .sync()
- .await?;
+ let source_file = source.join(&name);
+ storage.hard_link(&source_file, &destination).await?;
+ if !synced_files.contains(&source_file) {
+ files_to_sync.push(destination);
Review Comment:
warning: keeping every unsynced path until the walk ends lets large
partitions exceed the job's memory reservation. sync paths during the walk in
batches bounded by `FILE_SYNC_CONCURRENCY`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
