Petrolyst server infrastructure: Traefik, Trino and Superset
Traefik as the server-wide edge (TLS via Let's Encrypt, dashboard on traefik.petrolyst.com behind oauth2-proxy and Keycloak). The fuel-monitoring stack: Superset on atoma.petrolyst.com with Keycloak login and Petrolyst branding, Trino reading the TimescaleDB fuel-db as catalog "fuel". Customization lives in petrolyst-insights/, deployment in the other folders. Deploys run server-side (deploy/deploy.sh) from a Gitea workflow or manually; releases are immutable, Superset metadata is backed up before migrations, and deploy/rollback.sh restores it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
24327f9462
commit
d3800ac9dd
34 files changed
+2117
No files matched your search
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
# Manual rollback to a kept release, run on the server as the deploy user:
|
||||
# sudo -u deploy /opt/petrolyst/infra/deploy/rollback.sh <sha>
|
||||
#
|
||||
# Code-only when the target image's Alembic head equals the metadata
|
||||
# database's revision. Otherwise a migration rollback: the metadata database is
|
||||
# dropped, recreated and restored from the backup taken for the target image,
|
||||
# and the restored revision is checked before anything starts. Until the
|
||||
# rollback completes, state/rollback-in-progress blocks deploy.sh; an
|
||||
# interrupted rollback is finished by running it again. Afterwards,
|
||||
# `git revert` and push so main matches; deploy.sh's downgrade guard stops a
|
||||
# plain revert from running against a newer schema.
|
||||
set -euo pipefail
|
||||
# shellcheck source=deploy/lib.sh
|
||||
. "$(dirname "$(readlink -f "$0")")/lib.sh"
|
||||
|
||||
# Destructive steps below are hard-wired to service superset-db and database
|
||||
# superset; nothing here can be pointed at fuel-db.
|
||||
restore_metadata() {
|
||||
local backup=$1 head=$2 restored
|
||||
log "migration rollback: restoring $backup"
|
||||
fm stop superset
|
||||
fm exec -T superset-db psql -U superset -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c 'DROP DATABASE IF EXISTS superset WITH (FORCE)' \
|
||||
-c 'CREATE DATABASE superset OWNER superset'
|
||||
fm exec -T superset-db pg_restore -U superset -d superset --exit-on-error --single-transaction <"$backup"
|
||||
restored=$(metadata_revision)
|
||||
[ "$restored" = "$head" ] ||
|
||||
die "restored revision $restored is not the target head $head; superset stays stopped"
|
||||
}
|
||||
|
||||
# metadata_db_exists: whether database superset exists, asked through the
|
||||
# maintenance database; a connection or authentication error aborts.
|
||||
metadata_db_exists() {
|
||||
local n
|
||||
n=$(fm exec -T superset-db psql -U superset -d postgres -tAc "SELECT count(*) FROM pg_database WHERE datname = 'superset'") ||
|
||||
die "cannot reach the metadata database server"
|
||||
[[ $n =~ ^[01]$ ]] || die "unexpected answer from pg_database: $n"
|
||||
[ "$n" = 1 ]
|
||||
}
|
||||
|
||||
main() {
|
||||
local sha=${1:-} release image revision head backup dir
|
||||
[[ $sha =~ ^[0-9a-f]{40}$ ]] || die "usage: rollback.sh <40-hex sha of a kept release>"
|
||||
acquire_lock 600
|
||||
release=$RELEASES/$sha
|
||||
[ -s "$release/ARTIFACTS" ] || die "no finalized release with ARTIFACTS at $release"
|
||||
# The release's recorded identities, never recomputed.
|
||||
load_artifacts "$release"
|
||||
image=fuel-superset:$SUPERSET_IMAGE_TAG
|
||||
"${DOCKER[@]}" image inspect "$image" >/dev/null 2>&1 || die "image $image is no longer on this host"
|
||||
preflight "$release"
|
||||
|
||||
trap on_exit EXIT
|
||||
fm up -d --wait superset-db redis
|
||||
if [ -e "$ROLLBACK_MARK" ] && ! metadata_db_exists; then
|
||||
# An earlier attempt stopped between DROP and CREATE.
|
||||
revision=fresh
|
||||
else
|
||||
revision=$(metadata_revision)
|
||||
fi
|
||||
# Empty or missing is expected only after an interrupted restore, which left the mark.
|
||||
[ "$revision" != fresh ] || [ -e "$ROLLBACK_MARK" ] || die "the metadata database is empty; there is nothing to roll back"
|
||||
head=$(alembic "$image" heads)
|
||||
[[ $head =~ ^[0-9a-z_]+$ ]] || die "$image has no single migration head: $head"
|
||||
|
||||
if [ "$revision" = "$head" ]; then
|
||||
log "code-only rollback: the schema is already at $head"
|
||||
else
|
||||
backup=$(find "$BACKUPS" -maxdepth 1 -name "superset-meta_${SUPERSET_IMAGE_TAG}_${head}_*.dump" | sort | tail -n1)
|
||||
[ -n "$backup" ] || die "no backup for image $SUPERSET_IMAGE_TAG at revision $head in $BACKUPS"
|
||||
mkdir -p "$STATE"
|
||||
printf '%s\n' "$sha" >"$ROLLBACK_MARK"
|
||||
restore_metadata "$backup" "$head"
|
||||
fi
|
||||
|
||||
promote "$release" "$sha"
|
||||
ensure_proxy_network
|
||||
compose "$LIVE" traefik up -d --wait --remove-orphans
|
||||
# No superset-init: the schema already matches the target image.
|
||||
while IFS= read -r dir; do
|
||||
compose "$LIVE" "$dir" up -d --wait --remove-orphans
|
||||
done < <(stack_dirs "$LIVE")
|
||||
smoke_tests
|
||||
record_state "$sha"
|
||||
rm -f "$ROLLBACK_MARK"
|
||||
trap - EXIT
|
||||
log "rolled back to $sha; now git revert and push so main matches"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in new issue
Block a user