#!/usr/bin/env bash # Manual rollback to a kept release, run on the server as the deploy user: # sudo -u deploy /opt/petrolyst/infra/deploy/rollback.sh # # Code-only when the target image's Alembic head equals the metadata # database's revision. Otherwise a migration rollback: the metadata database is # dropped, recreated and restored from the backup taken for the target image, # and the restored revision is checked before anything starts. Until the # rollback completes, state/rollback-in-progress blocks deploy.sh; an # interrupted rollback is finished by running it again. Afterwards, # `git revert` and push so main matches; deploy.sh's downgrade guard stops a # plain revert from running against a newer schema. set -euo pipefail # shellcheck source=deploy/lib.sh . "$(dirname "$(readlink -f "$0")")/lib.sh" # Destructive steps below are hard-wired to service superset-db and database # superset; nothing here can be pointed at fuel-db. restore_metadata() { local backup=$1 head=$2 restored log "migration rollback: restoring $backup" fm stop superset fm exec -T superset-db psql -U superset -d postgres -v ON_ERROR_STOP=1 \ -c 'DROP DATABASE IF EXISTS superset WITH (FORCE)' \ -c 'CREATE DATABASE superset OWNER superset' fm exec -T superset-db pg_restore -U superset -d superset --exit-on-error --single-transaction <"$backup" restored=$(metadata_revision) [ "$restored" = "$head" ] || die "restored revision $restored is not the target head $head; superset stays stopped" } # metadata_db_exists: whether database superset exists, asked through the # maintenance database; a connection or authentication error aborts. metadata_db_exists() { local n n=$(fm exec -T superset-db psql -U superset -d postgres -tAc "SELECT count(*) FROM pg_database WHERE datname = 'superset'") || die "cannot reach the metadata database server" [[ $n =~ ^[01]$ ]] || die "unexpected answer from pg_database: $n" [ "$n" = 1 ] } main() { local sha=${1:-} release image revision head backup dir [[ $sha =~ ^[0-9a-f]{40}$ ]] || die "usage: rollback.sh <40-hex sha of a kept release>" acquire_lock 600 release=$RELEASES/$sha [ -s "$release/ARTIFACTS" ] || die "no finalized release with ARTIFACTS at $release" # The release's recorded identities, never recomputed. load_artifacts "$release" image=fuel-superset:$SUPERSET_IMAGE_TAG "${DOCKER[@]}" image inspect "$image" >/dev/null 2>&1 || die "image $image is no longer on this host" preflight "$release" trap on_exit EXIT fm up -d --wait superset-db redis if [ -e "$ROLLBACK_MARK" ] && ! metadata_db_exists; then # An earlier attempt stopped between DROP and CREATE. revision=fresh else revision=$(metadata_revision) fi # Empty or missing is expected only after an interrupted restore, which left the mark. [ "$revision" != fresh ] || [ -e "$ROLLBACK_MARK" ] || die "the metadata database is empty; there is nothing to roll back" head=$(alembic "$image" heads) [[ $head =~ ^[0-9a-z_]+$ ]] || die "$image has no single migration head: $head" if [ "$revision" = "$head" ]; then log "code-only rollback: the schema is already at $head" else backup=$(find "$BACKUPS" -maxdepth 1 -name "superset-meta_${SUPERSET_IMAGE_TAG}_${head}_*.dump" | sort | tail -n1) [ -n "$backup" ] || die "no backup for image $SUPERSET_IMAGE_TAG at revision $head in $BACKUPS" mkdir -p "$STATE" printf '%s\n' "$sha" >"$ROLLBACK_MARK" restore_metadata "$backup" "$head" fi promote "$release" "$sha" ensure_proxy_network compose "$LIVE" traefik up -d --wait --remove-orphans # No superset-init: the schema already matches the target image. while IFS= read -r dir; do compose "$LIVE" "$dir" up -d --wait --remove-orphans done < <(stack_dirs "$LIVE") smoke_tests record_state "$sha" rm -f "$ROLLBACK_MARK" trap - EXIT log "rolled back to $sha; now git revert and push so main matches" } main "$@"