Hi Ayush,
On Thu, Oct 22, 2026 at 11:29 PM Ayush Tiwari
<[email protected]> wrote:
> My worry was that a truncation in that summary could drop block
> references from the new timeline when we merge it. We'd then miss
> those blocks in the incremental backup and take stale ones from the
> earlier backup instead.
I could reproduce it. The setup is in the attached script: the standby
pauses at recovery_target_lsn = the redo LSN of a checkpoint, with
recovery_target_inclusive = false. Before that point the primary
updates rows 5..8 of an 8-block table, which the standby replays.
After it, the primary runs DELETE + VACUUM, truncating the table to 4
blocks, and checkpoints. The standby summarizes that WAL but never
replays it, and is then promoted.
The timeline 1 summary that starts at the switch point holds "limit 4"
for the table. It gets merged and drops blocks 4..11, the INCREMENTAL
file ends up with 0 blocks and truncation_block_length 12, and after
pg_combinebackup rows 5..8 come back with their pre-update contents.
pg_verifybackup reports both backups as valid.
On master with Fujii's v1-0001 from the #19774 thread that happened in
5 of 5 runs on tmpfs and 5 of 5 on btrfs. Without v1-0001 it was 5 of
5 on btrfs and 0 of 5 on tmpfs, where readdir happens to merge the
old-timeline summary first. With your v1: 0 of 10, with and without
v1-0001. Your patch applies cleanly either way, and make check,
pg_walsummary, pg_combinebackup and the recovery suite pass with it.
One note for anyone running the script: reading the table on the
promoted server before the backup can hide the loss, because with
data checksums the hint-bit FPIs mark those blocks again on the new
timeline. The script also keeps the target off page boundaries, which
hit a different error [1].
[1]
https://postgr.es/m/cak49s1a28ofeloo++vo2nr_jh0kqcv5gwus7dxnldw7qliz...@mail.gmail.com
Regards,
Manu
#!/bin/bash
# Incremental backup after promoting a standby at a recovery target that is
# the start of a WAL summary. The standby had already summarized WAL it
# received but never replayed, so it has an old-timeline summary that starts
# exactly at the switch point. FilterWalSummaries() keeps it, and a
# truncation in that never-replayed WAL drops blocks from the backup.
#
# usage: PGBIN=/path/to/bin bash repro-tie-A.sh WORKDIR [PORT]
# exit: 0 restored data matches the source, 1 it does not, 2 setup problem
set -u
: "${PGBIN:?set PGBIN to the bin directory of the build under test}"
W=${1:?usage: PGBIN=... bash repro-tie-A.sh WORKDIR [PORT]}
P=${2:-5440}; P1=$((P + 1)); P2=$((P + 2))
die() { echo "SETUP: $*"; exit 2; }
q() { "$PGBIN/psql" -X -h "$W" -p "$1" -d postgres -Atq -c "$2"; }
poll() { # poll PORT SQL: wait up to 60 s for SQL to return t
for _ in $(seq 600); do [ "$(q "$1" "$2" 2>/dev/null)" = t ] && return 0;
sleep 0.1; done
return 1
}
stopall() { for d in src standby restored; do
[ -f "$W/$d/postmaster.pid" ] && "$PGBIN/pg_ctl" -D "$W/$d" -m immediate -w
stop >/dev/null 2>&1
done; }
trap stopall EXIT
rm -rf "$W"; mkdir -p "$W" || die "mkdir $W"
"$PGBIN/initdb" -D "$W/src" -A trust --no-locale >/dev/null || die initdb
cat >> "$W/src/postgresql.conf" <<EOC
port = $P
unix_socket_directories = '$W'
listen_addresses = ''
summarize_wal = on
autovacuum = off
EOC
"$PGBIN/pg_ctl" -D "$W/src" -l "$W/src.log" -w start >/dev/null || die "start
primary"
# One row per page: 8 rows, 8 blocks.
q $P "CREATE TABLE t (id int, pad text STORAGE PLAIN)"
q $P "INSERT INTO t SELECT g, repeat('x', 4096) FROM generate_series(1, 8) g"
q $P "VACUUM t"
"$PGBIN/pg_basebackup" -h "$W" -p $P -D "$W/full" -c fast || die "full backup"
# The standby starts from a copy of the full backup, so the full backup's WAL
# range is in its history.
cp -a "$W/full" "$W/standby"
touch "$W/standby/standby.signal"
# Some WAL first, so that the redo LSN below is not right after a page header
# (a switch point on a page boundary is a different problem: the first
# summary on the new timeline then starts after the header).
q $P "CREATE TABLE marker0 ()"
# Replayed by the standby: rows 5..8 get new versions (blocks 4..11).
q $P "UPDATE t SET pad = pad || 'o' WHERE id > 4"
# The recovery target is the redo LSN of this checkpoint, which is also where
# a WAL summary starts.
q $P "CHECKPOINT"
T=$(q $P "SELECT redo_lsn FROM pg_control_checkpoint()")
# Received and summarized by the standby, never replayed: truncate to 4 blocks.
q $P "DELETE FROM t WHERE id > 4"
q $P "VACUUM t"
echo "primary after target: t has $(q $P "SELECT pg_relation_size('t') / 8192")
blocks"
q $P "CHECKPOINT"
X=$(q $P "SELECT redo_lsn FROM pg_control_checkpoint()")
echo "target $T, last primary redo $X"
cat >> "$W/standby/postgresql.conf" <<EOC
port = $P1
primary_conninfo = 'host=$W port=$P'
recovery_target_lsn = '$T'
recovery_target_inclusive = false
recovery_target_action = 'pause'
log_min_messages = debug1
EOC
"$PGBIN/pg_ctl" -D "$W/standby" -l "$W/standby.log" -w start >/dev/null || die
"start standby"
poll $P1 "SELECT pg_get_wal_replay_pause_state() = 'paused'" || die "standby
did not pause"
# The summarizer follows the walreceiver, not replay: wait until it has
# summarized the primary's WAL up to its last checkpoint, past the target.
poll $P1 "SELECT EXISTS (SELECT FROM pg_available_wal_summaries()
WHERE tli = 1 AND end_lsn >= '$X')" || die "no timeline 1 summary up
to $X"
echo "standby paused at $(q $P1 "SELECT pg_last_wal_replay_lsn()"), timeline 1
summaries:"
q $P1 "SELECT ' ' || start_lsn || ' .. ' || end_lsn FROM
pg_available_wal_summaries()
WHERE tli = 1 ORDER BY start_lsn"
"$PGBIN/pg_ctl" -D "$W/src" -w stop >/dev/null
"$PGBIN/pg_ctl" -D "$W/standby" -w promote >/dev/null || die promote
q $P1 "CHECKPOINT"
# Don't read t before the backup: with data checksums, setting hint bits
# WAL-logs those blocks again on the new timeline and hides the loss.
"$PGBIN/pg_basebackup" -h "$W" -p $P1 -D "$W/incr" -c fast \
--incremental "$W/full/backup_manifest" || die "incremental backup"
FP="SELECT count(*) || '|' || coalesce(max(id), 0) || '|' ||
pg_relation_size('t') / 8192
|| '|' || md5(coalesce(string_agg(id || ':' || md5(pad), ',' ORDER BY id),
'')) FROM t"
REL=$(q $P1 "SELECT pg_relation_filepath('t')")
SRC=$(q $P1 "$FP")
"$PGBIN/pg_ctl" -D "$W/standby" -w stop >/dev/null
echo "summaries with an entry for t, in readdir order (ls -U):"
for f in $(ls -U "$W/standby/pg_wal/summaries"); do
s=$("$PGBIN/pg_walsummary" "$W/standby/pg_wal/summaries/$f" \
| grep "REL ${REL##*/}, FORK main" | sed 's/.*FORK main: //' | tr '\n' '
')
echo " $f: ${s:--}"
done
echo "merge order (DEBUG1 of the incremental backup):"
grep -o 'reading WAL summary file "[^"]*"' "$W/standby.log" | sed
's/.*summaries\// /; s/"$//'
set -- $(od -A n -t u4 -N 12 "$W/incr/$(dirname "$REL")/INCREMENTAL.$(basename
"$REL")" 2>/dev/null)
if [ $# -eq 3 ]; then
echo "incr/$(dirname "$REL")/INCREMENTAL.$(basename "$REL"): $2 blocks," \
"truncation_block_length $3"
else
echo "incr: $REL sent in full"
fi
"$PGBIN/pg_verifybackup" -q "$W/full" && "$PGBIN/pg_verifybackup" -q "$W/incr" \
&& echo "pg_verifybackup: both backups OK"
"$PGBIN/pg_combinebackup" -o "$W/restored" "$W/full" "$W/incr" || die
pg_combinebackup
sed -i "s/^port = $P1\$/port = $P2/; /^recovery_target/d; /^primary_conninfo/d"
\
"$W/restored/postgresql.conf"
"$PGBIN/pg_ctl" -D "$W/restored" -l "$W/restored.log" -w start >/dev/null ||
die "start restored"
DST=$(q $P2 "$FP")
"$PGBIN/pg_ctl" -D "$W/restored" -w stop >/dev/null
echo "source: $SRC (count|max(id)|blocks|md5 of rows)"
echo "restored: $DST"
if [ "$SRC" = "$DST" ]; then echo "RESULT: OK"; exit 0; fi
echo "RESULT: DATA LOSS"; exit 1
