Nasim SadatandClaude Opus 5.5
d3800ac9dd
Traefik as the server-wide edge (TLS via Let's Encrypt, dashboard on traefik.petrolyst.com behind oauth2-proxy and Keycloak). The fuel-monitoring stack: Superset on atoma.petrolyst.com with Keycloak login and Petrolyst branding, Trino reading the TimescaleDB fuel-db as catalog "fuel". Customization lives in petrolyst-insights/, deployment in the other folders. Deploys run server-side (deploy/deploy.sh) from a Gitea workflow or manually; releases are immutable, Superset metadata is backed up before migrations, and deploy/rollback.sh restores it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
92 lines
3.9 KiB
Bash
Executable File
92 lines
3.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Manual rollback to a kept release, run on the server as the deploy user:
|
|
# sudo -u deploy /opt/petrolyst/infra/deploy/rollback.sh <sha>
|
|
#
|
|
# Code-only when the target image's Alembic head equals the metadata
|
|
# database's revision. Otherwise a migration rollback: the metadata database is
|
|
# dropped, recreated and restored from the backup taken for the target image,
|
|
# and the restored revision is checked before anything starts. Until the
|
|
# rollback completes, state/rollback-in-progress blocks deploy.sh; an
|
|
# interrupted rollback is finished by running it again. Afterwards,
|
|
# `git revert` and push so main matches; deploy.sh's downgrade guard stops a
|
|
# plain revert from running against a newer schema.
|
|
set -euo pipefail
|
|
# shellcheck source=deploy/lib.sh
|
|
. "$(dirname "$(readlink -f "$0")")/lib.sh"
|
|
|
|
# Destructive steps below are hard-wired to service superset-db and database
|
|
# superset; nothing here can be pointed at fuel-db.
|
|
restore_metadata() {
|
|
local backup=$1 head=$2 restored
|
|
log "migration rollback: restoring $backup"
|
|
fm stop superset
|
|
fm exec -T superset-db psql -U superset -d postgres -v ON_ERROR_STOP=1 \
|
|
-c 'DROP DATABASE IF EXISTS superset WITH (FORCE)' \
|
|
-c 'CREATE DATABASE superset OWNER superset'
|
|
fm exec -T superset-db pg_restore -U superset -d superset --exit-on-error --single-transaction <"$backup"
|
|
restored=$(metadata_revision)
|
|
[ "$restored" = "$head" ] ||
|
|
die "restored revision $restored is not the target head $head; superset stays stopped"
|
|
}
|
|
|
|
# metadata_db_exists: whether database superset exists, asked through the
|
|
# maintenance database; a connection or authentication error aborts.
|
|
metadata_db_exists() {
|
|
local n
|
|
n=$(fm exec -T superset-db psql -U superset -d postgres -tAc "SELECT count(*) FROM pg_database WHERE datname = 'superset'") ||
|
|
die "cannot reach the metadata database server"
|
|
[[ $n =~ ^[01]$ ]] || die "unexpected answer from pg_database: $n"
|
|
[ "$n" = 1 ]
|
|
}
|
|
|
|
main() {
|
|
local sha=${1:-} release image revision head backup dir
|
|
[[ $sha =~ ^[0-9a-f]{40}$ ]] || die "usage: rollback.sh <40-hex sha of a kept release>"
|
|
acquire_lock 600
|
|
release=$RELEASES/$sha
|
|
[ -s "$release/ARTIFACTS" ] || die "no finalized release with ARTIFACTS at $release"
|
|
# The release's recorded identities, never recomputed.
|
|
load_artifacts "$release"
|
|
image=fuel-superset:$SUPERSET_IMAGE_TAG
|
|
"${DOCKER[@]}" image inspect "$image" >/dev/null 2>&1 || die "image $image is no longer on this host"
|
|
preflight "$release"
|
|
|
|
trap on_exit EXIT
|
|
fm up -d --wait superset-db redis
|
|
if [ -e "$ROLLBACK_MARK" ] && ! metadata_db_exists; then
|
|
# An earlier attempt stopped between DROP and CREATE.
|
|
revision=fresh
|
|
else
|
|
revision=$(metadata_revision)
|
|
fi
|
|
# Empty or missing is expected only after an interrupted restore, which left the mark.
|
|
[ "$revision" != fresh ] || [ -e "$ROLLBACK_MARK" ] || die "the metadata database is empty; there is nothing to roll back"
|
|
head=$(alembic "$image" heads)
|
|
[[ $head =~ ^[0-9a-z_]+$ ]] || die "$image has no single migration head: $head"
|
|
|
|
if [ "$revision" = "$head" ]; then
|
|
log "code-only rollback: the schema is already at $head"
|
|
else
|
|
backup=$(find "$BACKUPS" -maxdepth 1 -name "superset-meta_${SUPERSET_IMAGE_TAG}_${head}_*.dump" | sort | tail -n1)
|
|
[ -n "$backup" ] || die "no backup for image $SUPERSET_IMAGE_TAG at revision $head in $BACKUPS"
|
|
mkdir -p "$STATE"
|
|
printf '%s\n' "$sha" >"$ROLLBACK_MARK"
|
|
restore_metadata "$backup" "$head"
|
|
fi
|
|
|
|
promote "$release" "$sha"
|
|
ensure_proxy_network
|
|
compose "$LIVE" traefik up -d --wait --remove-orphans
|
|
# No superset-init: the schema already matches the target image.
|
|
while IFS= read -r dir; do
|
|
compose "$LIVE" "$dir" up -d --wait --remove-orphans
|
|
done < <(stack_dirs "$LIVE")
|
|
smoke_tests
|
|
record_state "$sha"
|
|
rm -f "$ROLLBACK_MARK"
|
|
trap - EXIT
|
|
log "rolled back to $sha; now git revert and push so main matches"
|
|
}
|
|
|
|
main "$@"
|