Monthly PITR restore drill for the messenger database
Restores the newest base backup plus every WAL segment since into a throwaway postgres container, compares table count, devices, users and key_log (md5 over seq and leaf_hash up to the restored seq) with live, and writes PASS or FAIL to LAST_PITR_DRILL. Installed on the box 2026-10-14: first run PASS, and pointing it at a missing archive gives FAIL with exit 1. Timer: the 2nd of each month, 06:30 UTC. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Paul Hitt
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
8485339b5c
commit
5150a1b45e
@@ -0,0 +1,8 @@
|
||||
[Unit]
|
||||
Description=Restore the messenger PITR archive into a throwaway container and compare it with live
|
||||
After=docker.service
|
||||
Requires=docker.service
|
||||
[Service]
|
||||
Type=oneshot
|
||||
TimeoutStartSec=3600
|
||||
ExecStart=/srv/prod/backups/messenger-pitr-drill.sh
|
||||
Executable
+125
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env bash
|
||||
# Prove, once a month, that the Messenger's point-in-time archive restores.
|
||||
#
|
||||
# Takes the newest base backup in b2enc:messenger-pitr/, replays every WAL
|
||||
# segment archived since, into a throwaway postgres:17-alpine container, then
|
||||
# compares it with the live database:
|
||||
#
|
||||
# - the same public tables exist;
|
||||
# - devices and users are not empty;
|
||||
# - key_log matches row for row up to the last seq the restore reached
|
||||
# (md5 over seq and leaf_hash). key_log only ever grows, so any difference
|
||||
# means the archive lost or changed history.
|
||||
#
|
||||
# b2enc:messenger-pitr/base/<day>.tar.gz --+
|
||||
# b2enc:messenger-pitr/wal/<segment> -----+--> scratch dir --> drill container
|
||||
# | replay to end
|
||||
# messenger-db-1 (live) <----- compare tables, counts, key_log ---+
|
||||
#
|
||||
# Writes one PASS or FAIL line to LAST_PITR_DRILL and the log, exits non-zero
|
||||
# on FAIL, and always removes the container with -v (postgres:17 declares a
|
||||
# VOLUME, so a plain rm leaves an anonymous volume behind) and the scratch dir.
|
||||
# Run as root (the systemd unit beside this file).
|
||||
set -euo pipefail
|
||||
|
||||
BACKUPS=/srv/prod/backups
|
||||
RCLONE_CONF="${RCLONE_CONF:-$BACKUPS/rclone.conf}"
|
||||
PITR="${PITR:-b2enc:messenger-pitr}"
|
||||
LOG="${LOG:-$BACKUPS/messenger-pitr-drill.log}"
|
||||
LAST="$BACKUPS/LAST_PITR_DRILL"
|
||||
IMAGE="${IMAGE:-$(docker inspect messenger-db-1 --format '{{.Config.Image}}')}"
|
||||
NAME="pitr-drill-$$"
|
||||
|
||||
# Scratch lives on the data disk: /tmp is a 12 GB tmpfs and a day of WAL can
|
||||
# be several GB.
|
||||
work=$(mktemp -d "$BACKUPS/pitr-drill.XXXXXX")
|
||||
cleanup() {
|
||||
docker rm -fv "$NAME" >/dev/null 2>&1 || true
|
||||
rm -rf "$work"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
log() { echo "$(date -Is) $*" | tee -a "$LOG"; }
|
||||
fail() {
|
||||
log "FAIL $*"
|
||||
printf 'FAIL %s :: %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >"$LAST"
|
||||
exit 1
|
||||
}
|
||||
rc() {
|
||||
docker run --rm -v "$RCLONE_CONF":/config/rclone/rclone.conf:ro -v "$work":/work \
|
||||
rclone/rclone "$@"
|
||||
}
|
||||
live() {
|
||||
docker exec messenger-db-1 sh -c \
|
||||
'psql -U "${POSTGRES_USER:-flamenet}" -tAc "$1" "${POSTGRES_DB:-flamenet}"' _ "$1"
|
||||
}
|
||||
drill() {
|
||||
docker exec "$NAME" psql -U "$DBUSER" -tAc "$1" "$DBNAME"
|
||||
}
|
||||
|
||||
DBUSER=$(docker exec messenger-db-1 sh -c 'echo "${POSTGRES_USER:-flamenet}"')
|
||||
DBNAME=$(docker exec messenger-db-1 sh -c 'echo "${POSTGRES_DB:-flamenet}"')
|
||||
|
||||
log "drill starting (image $IMAGE)"
|
||||
|
||||
# 1. Newest base backup.
|
||||
base=$(rc lsf "$PITR/base/" | grep -E '^[0-9]{8}\.tar\.gz$' | sort | tail -1) \
|
||||
|| fail "cannot list $PITR/base/"
|
||||
[ -n "$base" ] || fail "no base backup in $PITR/base/"
|
||||
rc copyto "$PITR/base/$base" "/work/$base" || fail "cannot download base $base"
|
||||
|
||||
mkdir -p "$work/data" "$work/wal" "$work/unpack"
|
||||
# Bases from before 2026-10-09 begin with a NOTICE line; strip it if present.
|
||||
gzip -dc "$work/$base" | sed '1{/^NOTICE:/d}' | tar -x -C "$work/unpack" \
|
||||
|| fail "base $base does not unpack"
|
||||
tar -xzf "$work/unpack/base/base.tar.gz" -C "$work/data" || fail "base.tar.gz in $base does not unpack"
|
||||
start_wal=$(sed -n 's/^START WAL LOCATION: .* (file \([0-9A-F]\{24\}\))$/\1/p' "$work/data/backup_label")
|
||||
[ -n "$start_wal" ] || fail "no START WAL file in backup_label of $base"
|
||||
log "base $base starts at WAL $start_wal"
|
||||
|
||||
# 2. Every WAL segment from the start file on (names sort in WAL order on one
|
||||
# timeline), plus .backup and .history files.
|
||||
rc lsf "$PITR/wal/" >"$work/wal.list" || fail "cannot list $PITR/wal/"
|
||||
awk -v s="$start_wal" '($0 >= s) || /\.history$/' "$work/wal.list" >"$work/wal.need"
|
||||
n=$(wc -l <"$work/wal.need")
|
||||
[ "$n" -gt 0 ] || fail "no WAL at or after $start_wal"
|
||||
rc copy "$PITR/wal/" /work/wal --files-from /work/wal.need >/dev/null 2>&1 \
|
||||
|| fail "cannot download $n WAL files"
|
||||
log "downloaded $n WAL files"
|
||||
|
||||
# 3. Replay to the end of the archive, then promote.
|
||||
touch "$work/data/recovery.signal"
|
||||
chown -R 70:70 "$work/data" "$work/wal"
|
||||
chmod 700 "$work/data"
|
||||
docker run -d --name "$NAME" --network none \
|
||||
-v "$work/data":/var/lib/postgresql/data -v "$work/wal":/wal:ro \
|
||||
"$IMAGE" postgres -c archive_mode=off -c max_connections=300 \
|
||||
-c "restore_command=cp /wal/%f %p" -c recovery_target_action=promote >/dev/null \
|
||||
|| fail "drill container did not start"
|
||||
|
||||
for _ in $(seq 1 180); do
|
||||
if [ "$(drill 'select pg_is_in_recovery()' 2>/dev/null || true)" = "f" ]; then break; fi
|
||||
if ! docker ps -q --filter "name=^$NAME$" | grep -q .; then
|
||||
docker logs --tail 20 "$NAME" >>"$LOG" 2>&1 || true
|
||||
fail "drill container exited during recovery"
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
[ "$(drill 'select pg_is_in_recovery()' 2>/dev/null || true)" = "f" ] \
|
||||
|| fail "recovery did not finish in 15 minutes"
|
||||
|
||||
# 4. Compare with live.
|
||||
q_tables="select count(*) from information_schema.tables where table_schema = current_schema()"
|
||||
rt=$(drill "$q_tables"); lt=$(live "$q_tables")
|
||||
[ "$rt" -gt 0 ] && [ "$rt" = "$lt" ] || fail "tables restored=$rt live=$lt"
|
||||
rd=$(drill 'select count(*) from devices'); ru=$(drill 'select count(*) from users')
|
||||
[ "$rd" -gt 0 ] && [ "$ru" -gt 0 ] || fail "restored devices=$rd users=$ru"
|
||||
rmax=$(drill 'select coalesce(max(seq), 0) from key_log')
|
||||
q_md5="select md5(coalesce(string_agg(seq::text || ':' || leaf_hash, ',' order by seq), '')) from key_log where seq <= $rmax"
|
||||
rh=$(drill "$q_md5"); lh=$(live "$q_md5")
|
||||
[ "$rh" = "$lh" ] || fail "key_log differs up to seq $rmax"
|
||||
until_ts=$(drill "select coalesce(max(created_at)::text, 'none') from key_log")
|
||||
|
||||
msg="base=$base wal=$n tables=$rt devices=$rd users=$ru key_log<=${rmax} equal (last leaf $until_ts)"
|
||||
log "PASS $msg"
|
||||
printf 'PASS %s :: %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$msg" >"$LAST"
|
||||
@@ -0,0 +1,7 @@
|
||||
[Unit]
|
||||
Description=Messenger PITR restore drill, monthly
|
||||
[Timer]
|
||||
OnCalendar=*-*-02 06:30:00 UTC
|
||||
Persistent=true
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
Reference in New Issue
Block a user