Oier Bravo Urtasun
1638d2f6e8
Post-deploy, PostgREST keeps its pre-migration pg_proc snapshot and 404s any RPC added in the just-applied migrations until the rest container restarts. Confirmed on 2026-05-18 after migration 026 (set_item_frequency_weight + purge_item_frequency) — the "Purgar" button in /collective/manage returned "Could not find the function public.purge_item_frequency in the schema cache". `NOTIFY pgrst, 'reload schema'` triggers the same reload path the container would do on restart, but without a 2s blip in flight requests. Added at the end of the migration loop in deploy-erosi.sh, and at the end of the rebuild step in rollback-erosi.sh (rollback can change RPC shapes too). Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
199 lines
7.6 KiB
Bash
Executable File
199 lines
7.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Roll back the ambrosio production stack to a previously-deployed commit.
|
|
#
|
|
# Usage:
|
|
# infra/scripts/rollback-erosi.sh --list # print recent deploys
|
|
# infra/scripts/rollback-erosi.sh --previous # roll back to N-1
|
|
# infra/scripts/rollback-erosi.sh --to=<git-sha> # roll back to a specific deploy
|
|
# infra/scripts/rollback-erosi.sh --code-only --to=<sha> # skip DB restore
|
|
#
|
|
# Env:
|
|
# DEPLOY_HOST=ambrosio (override if needed)
|
|
# DEPLOY_PATH=/opt/colectivo
|
|
#
|
|
# How it works:
|
|
# The deploy script (deploy-erosi.sh) writes one line per successful
|
|
# deploy to /opt/colectivo/.deploys.log:
|
|
# <iso-ts> <git-sha> <backup-file>
|
|
# Rollback reads that log, picks the row matching --previous or --to,
|
|
# then:
|
|
# 1. Verifies the SHA is reachable in the local git repo.
|
|
# 2. Verifies the matching backup file still exists on ambrosio.
|
|
# 3. Asks for explicit confirmation (destructive op).
|
|
# 4. Checks out that SHA into a temp worktree on this machine.
|
|
# 5. rsync (excluding .git, etc.) → ambrosio.
|
|
# 6. Restores the DB from the matched backup.
|
|
# 7. Rebuilds the app image at the rollback SHA and brings the stack
|
|
# up. NOTE: --code-only skips step 6.
|
|
# 8. Cleans up the worktree. Does NOT write a new line to
|
|
# .deploys.log — rollback is intentionally not deploy-equivalent.
|
|
#
|
|
# What is NOT rolled back:
|
|
# * Storage volume (/var/lib/storage) — user uploads. Treated as
|
|
# append-only; backup taken pre-deploy but never restored on rollback.
|
|
# * Realtime tenant state, Kong runtime config. They are stateless or
|
|
# re-bootstrap from the DB.
|
|
|
|
set -euo pipefail
|
|
|
|
DEPLOY_HOST="${DEPLOY_HOST:-ambrosio}"
|
|
DEPLOY_PATH="${DEPLOY_PATH:-/opt/colectivo}"
|
|
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
|
|
MODE=""
|
|
TARGET_SHA=""
|
|
CODE_ONLY=0
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--list) MODE="list" ;;
|
|
--previous) MODE="previous" ;;
|
|
--to=*) MODE="to"; TARGET_SHA="${1#--to=}" ;;
|
|
--code-only) CODE_ONLY=1 ;;
|
|
-h|--help)
|
|
grep '^#' "$0" | sed 's/^# \{0,1\}//' | head -40
|
|
exit 0
|
|
;;
|
|
*) echo "Unknown arg: $1" >&2; exit 1 ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
[ -z "$MODE" ] && { echo "Specify --list, --previous, or --to=<sha>." >&2; exit 1; }
|
|
|
|
# ── 1. Read deploy log from ambrosio ──────────────────────────────────────
|
|
log=$(ssh "$DEPLOY_HOST" "cat $DEPLOY_PATH/.deploys.log 2>/dev/null || true")
|
|
if [ -z "$log" ]; then
|
|
echo "No deploy history found at $DEPLOY_HOST:$DEPLOY_PATH/.deploys.log" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [ "$MODE" = "list" ]; then
|
|
echo "Recent deploys ($DEPLOY_HOST):"
|
|
echo "------------------------------------------------------------"
|
|
printf '%s\n' "$log" | nl -ba
|
|
exit 0
|
|
fi
|
|
|
|
# ── 2. Pick the target row ────────────────────────────────────────────────
|
|
if [ "$MODE" = "previous" ]; then
|
|
# Last line = current deploy. Second-to-last = previous.
|
|
target_row=$(printf '%s\n' "$log" | tail -n 2 | head -n 1)
|
|
elif [ "$MODE" = "to" ]; then
|
|
target_row=$(printf '%s\n' "$log" | grep -F " $TARGET_SHA " | tail -n 1 || true)
|
|
fi
|
|
|
|
if [ -z "${target_row:-}" ]; then
|
|
echo "No matching deploy found." >&2
|
|
exit 1
|
|
fi
|
|
|
|
target_ts=$(printf '%s' "$target_row" | cut -f1)
|
|
target_sha=$(printf '%s' "$target_row" | cut -f2)
|
|
target_backup=$(printf '%s' "$target_row" | cut -f3)
|
|
|
|
echo "==> Rollback target"
|
|
echo " timestamp: $target_ts"
|
|
echo " git sha: $target_sha"
|
|
echo " backup: $target_backup"
|
|
|
|
# ── 3. Verify SHA + backup ───────────────────────────────────────────────
|
|
if ! git -C "$REPO_ROOT" cat-file -e "${target_sha}^{commit}" 2>/dev/null; then
|
|
echo "ERROR: commit $target_sha not present in local repo. Fetch first." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [ "$CODE_ONLY" -eq 0 ]; then
|
|
if ! ssh "$DEPLOY_HOST" "test -f $DEPLOY_PATH/$target_backup"; then
|
|
echo "ERROR: backup $target_backup missing on $DEPLOY_HOST." >&2
|
|
echo " Use --code-only to skip the DB restore." >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# ── 4. Confirm ────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "WARNING: This is DESTRUCTIVE."
|
|
if [ "$CODE_ONLY" -eq 1 ]; then
|
|
echo " Code will be rolled back to $target_sha; DB is NOT touched."
|
|
else
|
|
echo " Code AND database will be rolled back."
|
|
echo " Current DB state will be overwritten by $target_backup."
|
|
fi
|
|
echo " 5s window to abort (Ctrl-C)..."
|
|
sleep 5
|
|
|
|
# ── 5. Worktree at the target SHA ────────────────────────────────────────
|
|
worktree_dir=$(mktemp -d -t colectivo-rollback-XXXXXX)
|
|
trap 'git -C "$REPO_ROOT" worktree remove --force "$worktree_dir" 2>/dev/null || rm -rf "$worktree_dir"' EXIT
|
|
|
|
echo "--- Materialising worktree at $target_sha → $worktree_dir"
|
|
git -C "$REPO_ROOT" worktree add --detach "$worktree_dir" "$target_sha" >/dev/null
|
|
|
|
# ── 6. rsync worktree → ambrosio ─────────────────────────────────────────
|
|
echo "--- rsync code to $DEPLOY_HOST:$DEPLOY_PATH"
|
|
rsync -az --delete \
|
|
--exclude '.git' \
|
|
--exclude 'node_modules' \
|
|
--exclude '.svelte-kit' \
|
|
--exclude 'apps/web/build' \
|
|
--exclude 'apps/web/.vite' \
|
|
--exclude '**/*.log' \
|
|
--exclude '**/.env' \
|
|
--exclude '**/.env.development' \
|
|
--exclude '**/.env.local' \
|
|
--exclude 'backups' \
|
|
--exclude '.deploys.log' \
|
|
"$worktree_dir/" "$DEPLOY_HOST:$DEPLOY_PATH/"
|
|
|
|
# ── 7. Restore DB (unless --code-only) ───────────────────────────────────
|
|
if [ "$CODE_ONLY" -eq 0 ]; then
|
|
echo "--- Stopping app + auth + rest before DB restore"
|
|
ssh "$DEPLOY_HOST" bash -s << REMOTE
|
|
set -euo pipefail
|
|
cd $DEPLOY_PATH
|
|
docker compose --env-file .env -f infra/docker-compose.erosi.yml stop app auth rest realtime storage
|
|
REMOTE
|
|
|
|
echo "--- Restoring DB from $target_backup"
|
|
ssh "$DEPLOY_HOST" bash -s << REMOTE
|
|
set -euo pipefail
|
|
cd $DEPLOY_PATH
|
|
gunzip -c $target_backup \
|
|
| docker compose --env-file .env -f infra/docker-compose.erosi.yml exec -T db \
|
|
psql -U postgres -d postgres -v ON_ERROR_STOP=1
|
|
REMOTE
|
|
fi
|
|
|
|
# ── 8. Rebuild + bring the stack back up at the rollback SHA ────────────
|
|
echo "--- Rebuild app at $target_sha + bring stack up"
|
|
ssh "$DEPLOY_HOST" env GIT_SHA="$target_sha" bash -s << 'REMOTE'
|
|
set -euo pipefail
|
|
cd /opt/colectivo
|
|
: "${GIT_SHA:=dev}"
|
|
|
|
GIT_SHA="$GIT_SHA" docker compose --env-file .env -f infra/docker-compose.erosi.yml \
|
|
build --build-arg GIT_SHA="$GIT_SHA" app
|
|
GIT_SHA="$GIT_SHA" docker compose --env-file .env -f infra/docker-compose.erosi.yml up -d
|
|
|
|
# Reload PostgREST schema cache so any RPCs that changed shape between
|
|
# the rolled-back schema and the current rest container snapshot are
|
|
# resolved. Mirrors the same step at the end of deploy-erosi.sh.
|
|
docker compose --env-file .env -f infra/docker-compose.erosi.yml exec -T db \
|
|
psql -U postgres -d postgres </dev/null \
|
|
-c "NOTIFY pgrst, 'reload schema';"
|
|
REMOTE
|
|
|
|
echo ""
|
|
echo "==> Rollback complete."
|
|
echo " code: $target_sha"
|
|
if [ "$CODE_ONLY" -eq 1 ]; then
|
|
echo " db: UNCHANGED (--code-only)"
|
|
else
|
|
echo " db: restored from $target_backup"
|
|
fi
|
|
echo " App: https://erosi.limonia.net"
|
|
echo ""
|
|
echo "NOTE: .deploys.log is intentionally not updated by rollback."
|
|
echo " The next 'just deploy' will be from your current HEAD."
|