flamenet-relay/Scripts/staging-gate.sh

#!/usr/bin/env bash
# Every relay image boots in staging before prod runs it.
#
# Born from an outage: a deploy whose sweeper touched Redis before
# Vapor finished booting crash-looped production for fifteen minutes, and the
# scale test had passed because the fatal was a boot-order RACE, it only fired
# when boot took longer than the sweeper's first breath. Hence the shape of
# this gate:
#
#   * staging runs WITH Redis attached, the crash only existed on that path;
#     a gate that skips Redis certifies the wrong binary.
#   * pass requires a STABILITY WINDOW, not one green health check, a race
#     can win once; it has to win continuously for the whole window, with no
#     restart and no "Fatal error" in the log.
#   * before prod is touched, the running image is retagged :previous, the
#     outage had no instant rollback because the old image had been reclaimed.
#
# Usage: sudo ./staging-gate.sh            (from /srv/prod/messenger)
#        sudo ./staging-gate.sh --no-deploy   (gate only, leave prod alone)
set -euo pipefail

# Where the SOURCE tree is, captured before the cd below moves us out of
# it. The script lives in `<source>/Scripts/`, and the cd that follows lands
# on `/srv/prod/messenger`, a directory that holds the compose file and NOT the
# Swift sources. Anything that wants to read the source has to have remembered
# where it was.
GATE_SRC="$(cd "$(dirname "$0")/.." && pwd)"

cd "$(dirname "$0")/.."   # /srv/prod/messenger/src -> /srv/prod/messenger when installed; see below
[ -f docker-compose.yaml ] || cd /srv/prod/messenger
[ -f docker-compose.yaml ] || { echo "GATE: cannot find /srv/prod/messenger"; exit 1; }

STACK=msgr-staging
NET=${STACK}-net
PORT=18899
WINDOW=25          # seconds the candidate must stay healthy
DEPLOY=yes
[ "${1:-}" = "--no-deploy" ] && DEPLOY=no

cleanup() {
  docker rm -f ${STACK}-relay ${STACK}-db ${STACK}-redis >/dev/null 2>&1 || true
  docker network rm ${NET} >/dev/null 2>&1 || true
}
trap cleanup EXIT

fail() { echo "GATE FAILED: $1"; echo "--- candidate log tail ---"; docker logs --tail 40 ${STACK}-relay 2>&1 || true; exit 1; }

echo "GATE: building candidate image"
# FN_RELEASE (the commit sha being shipped) is baked into the image for error
# reports: `sudo FN_RELEASE=<sha> ./src/Scripts/staging-gate.sh`. ./src is a
# copy, not a checkout, so the sha cannot be read from it here.
docker build --build-arg FN_RELEASE="${FN_RELEASE:-}" -t flamenet-relay:candidate ./src >/dev/null

echo "GATE: booting staging stack (throwaway db + redis, candidate relay)"
cleanup
docker network create ${NET} >/dev/null
docker run -d --name ${STACK}-db --network ${NET} \
  -e POSTGRES_DB=staging -e POSTGRES_USER=staging -e POSTGRES_PASSWORD=staging \
  postgres:17-alpine >/dev/null
docker run -d --name ${STACK}-redis --network ${NET} redis:7-alpine >/dev/null

# 90s, not 30s. This box also runs CI, and at load 20+ a cold
# postgres:17-alpine needs well over thirty seconds to accept its first
# connection. The gate has failed here with "staging postgres never
# became ready", a green build reported as a deploy failure, purely because
# the gate ran out of patience faster than the machine ran out of work. It
# still fails if postgres is broken; it just no longer calls a busy
# machine a broken one. Waiting longer can only ever delay a failure, never
# hide one, so this loop is the safe place to be generous.
echo "GATE: waiting for staging postgres"
for i in $(seq 1 45); do
  docker exec ${STACK}-db pg_isready -U staging >/dev/null 2>&1 && break
  sleep 2
done
docker exec ${STACK}-db pg_isready -U staging >/dev/null 2>&1 || fail "staging postgres never became ready"

# Throwaway issuer keypair, minted fresh per run and never stored.
PRIV_PEM=$(openssl genpkey -algorithm ed25519 2>/dev/null)
PRIV=$(printf '%s\n' "$PRIV_PEM" | openssl pkey -outform DER 2>/dev/null | tail -c 32 | base64)
PUB=$(printf '%s\n' "$PRIV_PEM" | openssl pkey -pubout -outform DER 2>/dev/null | tail -c 32 | base64)

docker run -d --name ${STACK}-relay --network ${NET} -p 127.0.0.1:${PORT}:8080 \
  -e FN_POSTGRES_URL=postgres://staging:staging@${STACK}-db:5432/staging \
  -e FN_REDIS_URL=redis://${STACK}-redis:6379 \
  -e FN_RELAY_AUDIENCE=staging-gate \
  -e FN_ISSUER_PRIVATE_KEY="$PRIV" -e FN_ISSUER_PUBLIC_KEY="$PUB" \
  -e FN_LOG_LEVEL=notice \
  flamenet-relay:candidate >/dev/null

echo "GATE: waiting for health"
UP=""
for i in $(seq 1 30); do
  if [ "$(curl -sf -m 2 http://127.0.0.1:${PORT}/health || true)" = "ok" ]; then UP=1; break; fi
  sleep 2
done
[ -n "$UP" ] || fail "candidate never answered /health within 60s"

echo "GATE: smoke, signup, signin, landing"
SU=$(curl -s -m 5 -o /dev/null -w '%{http_code}' -X POST http://127.0.0.1:${PORT}/account/signup \
  -H 'Content-Type: application/json' \
  -d '{"username":"gatecheck","password":"gate-check-pass-1","email":"gate@staging.invalid"}')
[ "$SU" = "200" ] || fail "signup answered $SU"
SI=$(curl -s -m 5 -o /dev/null -w '%{http_code}' -X POST http://127.0.0.1:${PORT}/account/signin \
  -H 'Content-Type: application/json' \
  -d '{"username":"gatecheck","password":"gate-check-pass-1"}')
[ "$SI" = "200" ] || fail "signin answered $SI"
LP=$(curl -s -m 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:${PORT}/)
[ "$LP" = "200" ] || fail "landing answered $LP"
# The donate page once shipped 500 because a hand-copied template lost its
# world-read bit, the binary was healthy, the page was not. Render every
# public page, not just the front door.
#
# Read out of the source, not typed here. A hand-written list drifts the
# moment anybody adds a page, and a deploy that broke an unlisted page would
# pass the gate.
#
# `Entrypoint.swift` has the real table, one `("slug", "title", …)` per page.
# Deriving from it means a page added tomorrow is gated tomorrow, with nobody
# needing to remember this file.
# Matched on the table's own indentation rather than on the page titles.
# Most titles end ",  Flamenet Messenger" and /get does not, so a
# title-shaped pattern quietly skipped the install page, the one page a
# printed QR code points at.
#
# `|| true`, and an absolute path. Without both, this line KILLED the
# gate: the relative path does not exist from the deployment directory, the
# second `grep` then matched nothing and exited 1, and under `set -o pipefail`
# plus `set -e` that aborted the script, after the smoke checks had passed,
# with no failure message, leaving only a truncated log. The fallback below
# never got the chance to run. A guard that cannot fail safe is not a guard.
# The name sits on its own line, indented twelve, because the tuple opens on
# the line above:
#
#         (
#             "about", "About. Flamenet Messenger",
#
# This pattern matched EIGHT spaces and an opening paren, which is the line
# with no name on it, so it extracted nothing, every time, for as long as the
# table has been formatted this way. The fallback below then quietly became the
# whole gate: five pages out of twenty-five. Two pages shipped answering 500
# through a gate that reported PASSED, because neither was in that list of five.
PAGES=$(grep -oE '^            "[a-z-]+", "' \
          "$GATE_SRC/Sources/FlamenetRelay/Entrypoint.swift" 2>/dev/null \
        | grep -oE '"[a-z-]+"' | head -100 | tr -d '"' | sort -u || true)
# A grep that matches nothing must not silently gate nothing. Falling back
# to the old hand-list keeps the gate honest if the table is ever reshaped.
[ -n "$PAGES" ] || PAGES="donate about privacy roadmap community"
# And a floor, which is the part that was missing. Falling back is fail-safe
# only if somebody finds out; degrading from twenty-five pages to five and
# printing PASSED is fail-QUIET, which is worse than no gate because it reads
# like coverage. If the count collapses, the derivation broke: say so and stop,
# rather than deploying on a fifth of the checks and announcing success.
PAGE_COUNT=$(echo "$PAGES" | wc -w | tr -d ' ')
if [ "$PAGE_COUNT" -lt 15 ]; then
  fail "derived only $PAGE_COUNT public pages from Entrypoint.swift; expected at least 15.
       The page table has been reshaped and the pattern above no longer matches it.
       Fix the pattern, do not lower this floor."
fi
# Count what the loop ACTUALLY renders. This summed `$PAGES` alone while
# the loop below iterates `donate` and `releases` as well, so it has always
# understated by one, and the commit that added `releases` made it two. Nothing
# was ever skipped; the message was simply wrong, which makes it one more line
# that has to be verified rather than read. That is the opposite of a gate.
# The Spanish pages too. They are separate templates with their own tags,
# and the deploy that templated the contact addresses shipped /es/safety
# answering 500 through a gate that rendered only the English table.
ES_PAGES=$(ls "$GATE_SRC/Resources/Views/es/" 2>/dev/null | sed 's/\.leaf$//' | grep -v '^landing$' | sed 's|^|es/|' | tr '\n' ' ')
# `hi` is named for the same reason as `releases` below: its route is outside
# the page table, so it stays out of the sitemap, and the derived list cannot
# see it. Its Spanish twin is es/hi.leaf, which the listing above picks up.
ALL_PAGES="donate releases hi $PAGES es/ $ES_PAGES"
echo "GATE: rendering $(echo "$ALL_PAGES" | wc -w | tr -d ' ') public pages"
# `releases` is named here for the same reason `donate` is: it is NOT in
# the table above. Its route is deliberately outside `pages` because its
# content is data read from disk per request rather than a template, and the
# consequence is that the derived list cannot see it. The deploy that first
# shipped that page was, without this line, a deploy whose ONLY new page was
# the one page the gate did not render.
for PAGE in $ALL_PAGES; do
  PC=$(curl -s -m 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:${PORT}/${PAGE})
  [ "$PC" = "200" ] || fail "/${PAGE} answered $PC"
done

echo "GATE: stability window (${WINDOW}s, no restart, no fatal, health stays ok)"
STARTED=$(docker inspect -f '{{.State.StartedAt}}' ${STACK}-relay)
for i in $(seq 1 $((WINDOW / 5))); do
  sleep 5
  [ "$(curl -sf -m 2 http://127.0.0.1:${PORT}/health || true)" = "ok" ] || fail "health lapsed inside the window"
done
[ "$(docker inspect -f '{{.State.StartedAt}}' ${STACK}-relay)" = "$STARTED" ] || fail "container restarted inside the window"
[ "$(docker inspect -f '{{.RestartCount}}' ${STACK}-relay)" = "0" ] || fail "container restarted inside the window"
if docker logs ${STACK}-relay 2>&1 | grep -q "Fatal error"; then fail "'Fatal error' in the candidate log"; fi

echo "GATE PASSED"
[ "$DEPLOY" = "yes" ] || { echo "GATE: --no-deploy, stopping here"; exit 0; }

echo "DEPLOY: retagging current prod image as flamenet-relay:previous (instant rollback point)"
docker tag flamenet-relay:prod flamenet-relay:previous 2>/dev/null || echo "DEPLOY: no prod tag yet, skipping"
docker tag flamenet-relay:candidate flamenet-relay:prod

docker compose up -d --no-deps relay >/dev/null

# Wait for the edge to answer, rather than for a fixed span.
#
# This was `sleep 8` and one curl. A recreated relay takes about sixteen
# seconds to answer /health, measured on 2026-09-22: container started at
# 18:37:00.9, first healthy probe at 18:37:16.7. So the check fired while the
# relay was still coming up, read the edge's 502 as a broken deploy, and rolled
# back a build that was fine. A gate that passes its whole suite and then fails
# on the clock teaches people to stop believing it.
#
# The ceiling is generous because it costs nothing on a healthy deploy: the
# loop leaves as soon as the edge says ok, usually inside twenty seconds. Only
# a deploy that is actually broken spends the whole budget, and spending it is
# how the rollback stays correct.
EDGE=""
EDGE_DEADLINE=$(( $(date +%s) + 90 ))
while [ "$(date +%s)" -lt "$EDGE_DEADLINE" ]; do
  EDGE=$(curl -sk -m 5 --resolve flamenetmessenger.com:443:127.0.0.1 https://flamenetmessenger.com/health || true)
  [ "$EDGE" = "ok" ] && break
  sleep 2
done
if [ "$EDGE" != "ok" ]; then
  echo "DEPLOY FAILED at the edge, rolling back to :previous"
  docker tag flamenet-relay:previous flamenet-relay:prod
  docker compose up -d --no-deps relay >/dev/null
  exit 1
fi
echo "DEPLOY OK, prod healthy at the edge; rollback available: docker tag flamenet-relay:previous flamenet-relay:prod && docker compose up -d --no-deps relay"

# Carry the test fleet with production, or the live suites lie.
#
# relay1/2/3 pin `flamenet-relay:prod`, but a container keeps running the image
# it started with, so every deploy here silently leaves the fleet a build
# behind. It was ELEVEN DAYS stale when that was first noticed, answering every
# health check the whole time, while the Android live suite ran against it and
# reported green about a build production had already moved off.
#
# Best-effort on purpose. The fleet is disposable and a production deploy
# must not fail because a test relay would not come up, but it must also not
# quietly stop matching, so the outcome is printed either way and
# `messenger-privacy-check` asserts it independently.
if [ -d /srv/test/relays ]; then
  if (cd /srv/test/relays && docker compose up -d >/dev/null 2>&1); then
    echo "FLEET OK, relay1/2/3 recreated on the image production now runs"
  else
    echo "FLEET WARNING, the test fleet did not update; live suites would test the OLD build"
  fi
fi