Petrolyst server infrastructure: Traefik, Trino and Superset
Traefik as the server-wide edge (TLS via Let's Encrypt, dashboard on traefik.petrolyst.com behind oauth2-proxy and Keycloak). The fuel-monitoring stack: Superset on atoma.petrolyst.com with Keycloak login and Petrolyst branding, Trino reading the TimescaleDB fuel-db as catalog "fuel". Customization lives in petrolyst-insights/, deployment in the other folders. Deploys run server-side (deploy/deploy.sh) from a Gitea workflow or manually; releases are immutable, Superset metadata is backed up before migrations, and deploy/rollback.sh restores it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Nasim SadatandClaude Opus 5.5 committed
1 parent
24327f9462
commit
d3800ac9dd
34 files changed
+2117
No files matched your search
Executable
+163
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env bash
|
||||
# Server-side deploy, run as the non-root deploy user:
|
||||
# deploy.sh <sha> <incoming-dir> CI: finalize the upload as releases/<sha>, then deploy it
|
||||
# deploy.sh <sha> re-deploy a release that is already finalized
|
||||
#
|
||||
# Order: lock, finalize, artifacts, preflight, image build and downgrade guard
|
||||
# (nothing live touched yet), then promote, Traefik, each stack, smoke tests,
|
||||
# retention. Rollback is a separate, manual script (rollback.sh) under the same
|
||||
# lock.
|
||||
set -euo pipefail
|
||||
# shellcheck source=deploy/lib.sh
|
||||
. "$(dirname "$(readlink -f "$0")")/lib.sh"
|
||||
|
||||
# finalize_release <sha> <upload>: CI never writes into releases/. A new SHA is
|
||||
# moved in atomically; a re-run of a deployed SHA must match it byte for byte
|
||||
# and leaves the finalized release, ARTIFACTS included, untouched.
|
||||
finalize_release() {
|
||||
local release=$RELEASES/$1 upload
|
||||
upload=$(readlink -f "$2") || die "upload $2 does not exist"
|
||||
[[ $upload == "$(readlink -f "$INCOMING")"/* && -d $upload ]] || die "upload $2 is not a directory under $INCOMING"
|
||||
if [ ! -e "$release" ]; then
|
||||
mv "$upload" "$release"
|
||||
log "finalized $release"
|
||||
elif diff -r --exclude=ARTIFACTS "$upload" "$release" >/dev/null; then
|
||||
rm -rf "$upload"
|
||||
log "release $1 already finalized and identical to the upload; reusing it"
|
||||
else
|
||||
die "upload differs from the finalized release $release; one SHA cannot have two contents"
|
||||
fi
|
||||
}
|
||||
|
||||
# resolve_artifacts <release>: compute the artifact identities from the
|
||||
# release's own files, record them once in ARTIFACTS, then freeze the release.
|
||||
resolve_artifacts() {
|
||||
local release=$1 computed
|
||||
computed=$(compute_artifacts "$release")
|
||||
if [ -e "$release/ARTIFACTS" ]; then
|
||||
[ "$(cat "$release/ARTIFACTS")" = "$computed" ] || die "$release/ARTIFACTS does not match the release's files"
|
||||
else
|
||||
# A crash between the move and this step leaves no ARTIFACTS; the files
|
||||
# are unchanged, so the computed values are the same.
|
||||
chmod u+w "$release"
|
||||
printf '%s\n' "$computed" >"$release/ARTIFACTS"
|
||||
fi
|
||||
chmod -R a-w "$release"
|
||||
load_artifacts "$release"
|
||||
log "artifacts: superset $SUPERSET_IMAGE_TAG, trino config $TRINO_CONFIG_HASH, traefik config $TRAEFIK_CONFIG_HASH"
|
||||
}
|
||||
|
||||
# prepare_fuel_monitoring <release>: before anything live changes, build the
|
||||
# Superset image from the release and refuse an image older than the schema of
|
||||
# a running metadata database.
|
||||
prepare_fuel_monitoring() {
|
||||
local image=fuel-superset:$SUPERSET_IMAGE_TAG revision
|
||||
if ! "${DOCKER[@]}" image inspect "$image" >/dev/null 2>&1; then
|
||||
log "building $image"
|
||||
compose "$1" stacks/fuel-monitoring build superset
|
||||
fi
|
||||
[ -e "$LIVE/stacks/fuel-monitoring/compose.yaml" ] || return 0
|
||||
[ -n "$(fm ps --status running -q superset-db 2>/dev/null)" ] || return 0
|
||||
revision=$(metadata_revision)
|
||||
[ "$revision" = fresh ] || downgrade_guard "$image" "$revision"
|
||||
}
|
||||
|
||||
downgrade_guard() {
|
||||
alembic "$1" has "$2" ||
|
||||
die "the metadata database is at revision $2, which $1 does not know: the image is older than the schema. Use deploy/rollback.sh."
|
||||
}
|
||||
|
||||
deploy_fuel_monitoring() {
|
||||
local image=fuel-superset:$SUPERSET_IMAGE_TAG revision running head
|
||||
fm up -d --wait superset-db redis
|
||||
revision=$(metadata_revision)
|
||||
if [ "$revision" = fresh ]; then
|
||||
log "metadata database is empty: fresh install, running superset-init"
|
||||
fm run --rm superset-init
|
||||
else
|
||||
# Again here: the database may not have been running during preparation.
|
||||
downgrade_guard "$image" "$revision"
|
||||
running=$(running_superset_tag)
|
||||
# An unchanged tag alone is not enough: the schema may have been changed
|
||||
# under the container (a restore, a manual downgrade).
|
||||
head=$(alembic "$image" heads)
|
||||
if [ "$running" != "$SUPERSET_IMAGE_TAG" ] || [ "$revision" != "$head" ]; then
|
||||
log "Superset image ${running:-unknown} -> $SUPERSET_IMAGE_TAG, schema $revision -> $head: stop, back up, migrate"
|
||||
fm stop superset
|
||||
backup_metadata "${running:-unknown}" "$revision"
|
||||
fm run --rm superset-init
|
||||
fi
|
||||
fi
|
||||
fm up -d --wait --remove-orphans
|
||||
}
|
||||
|
||||
# Old releases, stale uploads, old backups and unreferenced Superset images.
|
||||
retention() {
|
||||
local current=$1 dir tag referenced
|
||||
while IFS= read -r dir; do
|
||||
[ "${dir##*/}" != "$current" ] || continue
|
||||
chmod -R u+w "$dir"
|
||||
rm -rf "$dir"
|
||||
log "removed release ${dir##*/}"
|
||||
done < <(find "$RELEASES" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -rn | tail -n +6 | cut -d' ' -f2-)
|
||||
# Uploads older than a day were never finalized (interrupted CI runs). Their
|
||||
# mtime is the server's (the workflow's rsync omits directory times), and a
|
||||
# CI job lives at most 45 minutes, so an upload being sent or waiting for the
|
||||
# lock is never this old.
|
||||
while IFS= read -r -d '' dir; do
|
||||
chmod -R u+w "$dir"
|
||||
rm -rf "$dir"
|
||||
log "removed stale upload ${dir##*/}"
|
||||
done < <(find "$INCOMING" -mindepth 1 -maxdepth 1 -mmin +1440 -print0)
|
||||
find "$BACKUPS" -maxdepth 1 -name 'superset-meta_*.dump' -mtime +30 -delete
|
||||
find "$BACKUPS" -maxdepth 1 -name '.partial-*' -mmin +60 -delete
|
||||
# Keep the 3 newest images plus every image a kept release or backup names.
|
||||
referenced=$(
|
||||
sed -n 's/^SUPERSET_IMAGE_TAG=//p' "$RELEASES"/*/ARTIFACTS 2>/dev/null || true
|
||||
find "$BACKUPS" -maxdepth 1 -name 'superset-meta_*.dump' -exec basename {} \; |
|
||||
sed -n 's/^superset-meta_\([0-9a-f]\{16\}\)_.*/\1/p'
|
||||
)
|
||||
"${DOCKER[@]}" image ls fuel-superset --format '{{.Tag}}' | tail -n +4 | while IFS= read -r tag; do
|
||||
grep -qxF "$tag" <<<"$referenced" && continue
|
||||
if "${DOCKER[@]}" image rm "fuel-superset:$tag" >/dev/null; then
|
||||
log "removed image fuel-superset:$tag"
|
||||
else
|
||||
log "kept image fuel-superset:$tag (still in use)"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
main() {
|
||||
local sha=${1:-} upload=${2:-} release dir
|
||||
[[ $sha =~ ^[0-9a-f]{40}$ ]] || die "usage: deploy.sh <40-hex sha> [incoming-dir]"
|
||||
acquire_lock 900
|
||||
[ ! -e "$ROLLBACK_MARK" ] ||
|
||||
die "a rollback to $(cat "$ROLLBACK_MARK") did not finish and may have left the metadata database half-restored; run deploy/rollback.sh again first"
|
||||
release=$RELEASES/$sha
|
||||
[ -z "$upload" ] || finalize_release "$sha" "$upload"
|
||||
[ -d "$release" ] || die "no release $release"
|
||||
resolve_artifacts "$release"
|
||||
preflight "$release"
|
||||
prepare_fuel_monitoring "$release"
|
||||
|
||||
# From here on the live state changes; on failure, print the stacks' state.
|
||||
trap on_exit EXIT
|
||||
promote "$release" "$sha"
|
||||
ensure_proxy_network
|
||||
log "traefik"
|
||||
compose "$LIVE" traefik up -d --wait --remove-orphans
|
||||
while IFS= read -r dir; do
|
||||
log "$dir"
|
||||
case $dir in
|
||||
stacks/fuel-monitoring) deploy_fuel_monitoring ;;
|
||||
*) compose "$LIVE" "$dir" up -d --wait --remove-orphans ;;
|
||||
esac
|
||||
done < <(stack_dirs "$LIVE")
|
||||
smoke_tests
|
||||
record_state "$sha"
|
||||
trap - EXIT
|
||||
retention "$sha"
|
||||
log "deployed $sha"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
+271
@@ -0,0 +1,271 @@
|
||||
# shellcheck shell=bash
|
||||
# Shared by deploy.sh and rollback.sh; sourced, not executed.
|
||||
|
||||
# Server layout (README "Server paths"). PETROLYST_ROOT exists only so the
|
||||
# scripts can be exercised against a scratch directory; production never sets it.
|
||||
# shellcheck disable=SC2034 # RELEASES and INCOMING are used by the sourcing scripts
|
||||
ROOT=${PETROLYST_ROOT:-/opt/petrolyst}
|
||||
LIVE=$ROOT/infra
|
||||
RELEASES=$ROOT/releases
|
||||
INCOMING=$ROOT/incoming
|
||||
BACKUPS=$ROOT/backups
|
||||
STATE=$ROOT/state
|
||||
LOCK_FILE=$ROOT/deploy.lock
|
||||
# Exists while rollback.sh replaces the metadata database, and stays behind if
|
||||
# it is interrupted there; deploy.sh refuses to run until a rollback completes.
|
||||
ROLLBACK_MARK=$STATE/rollback-in-progress
|
||||
|
||||
ATOMA_HOST=${ATOMA_HOST:-atoma.petrolyst.com}
|
||||
TRAEFIK_HOST=${TRAEFIK_HOST:-traefik.petrolyst.com}
|
||||
KEYCLOAK_REALM_URL=${KEYCLOAK_REALM_URL:-https://iam.thedigitalbridge.io/realms/thedigitalbridge}
|
||||
PETROLYST_ROOT=$ROOT
|
||||
export ATOMA_HOST TRAEFIK_HOST KEYCLOAK_REALM_URL PETROLYST_ROOT
|
||||
|
||||
# --host pins the daemon, as in petrolyst-ui/deploy/deploy.sh: a stale
|
||||
# `docker context use` must not send this deploy to another daemon.
|
||||
DOCKER=(docker --host unix:///var/run/docker.sock)
|
||||
|
||||
# Build inputs of the Superset image, relative to a release. Everything the
|
||||
# Dockerfile copies is listed; nothing else is, so the tag changes exactly when
|
||||
# the image would.
|
||||
SUPERSET_INPUTS=(
|
||||
stacks/fuel-monitoring/superset/Dockerfile
|
||||
stacks/fuel-monitoring/superset/init.sh
|
||||
petrolyst-insights/superset_config.py
|
||||
petrolyst-insights/petrolyst_security.py
|
||||
petrolyst-insights/assets/branding
|
||||
petrolyst-insights/patches
|
||||
)
|
||||
|
||||
# Every file the stacks need from /opt/petrolyst (host/secrets-init.sh).
|
||||
REQUIRED_FILES=(
|
||||
secrets/shared/keycloak_petrolyst_client_secret
|
||||
secrets/traefik/oauth2_proxy_cookie_secret
|
||||
secrets/fuel-monitoring/superset_secret_key
|
||||
secrets/fuel-monitoring/superset_db_password
|
||||
secrets/fuel-monitoring/fuel_db_password
|
||||
secrets/fuel-monitoring/fuel_db_trino_password
|
||||
env/fuel-monitoring.env
|
||||
)
|
||||
|
||||
log() { printf '%s %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; }
|
||||
die() { log "ERROR: $*"; exit 1; }
|
||||
|
||||
[ "$(id -u)" != 0 ] || die "run as the deploy user (sudo -u deploy $0 ...), not as root"
|
||||
|
||||
# acquire_lock <seconds>: deploy.sh and rollback.sh share this lock, so a CI
|
||||
# deploy and a manual rollback never run at the same time. Held until exit.
|
||||
acquire_lock() {
|
||||
exec 9>>"$LOCK_FILE"
|
||||
flock -w "$1" 9 || die "another deploy or rollback holds $LOCK_FILE (waited $1 s)"
|
||||
}
|
||||
|
||||
sha256() {
|
||||
if command -v sha256sum >/dev/null; then sha256sum; else shasum -a 256; fi | cut -c1-64
|
||||
}
|
||||
|
||||
# hash_inputs <dir> <path>...: one short digest over the names and contents of
|
||||
# every file under the paths, independent of mtimes and listing order.
|
||||
hash_inputs() {
|
||||
local dir=$1
|
||||
shift
|
||||
(
|
||||
cd "$dir" || exit 1
|
||||
find "$@" -type f -print0 | LC_ALL=C sort -z |
|
||||
while IFS= read -r -d '' f; do printf '%s %s\n' "$f" "$(sha256 <"$f")"; done
|
||||
) | sha256 | cut -c1-16
|
||||
}
|
||||
|
||||
compute_artifacts() {
|
||||
printf 'SUPERSET_IMAGE_TAG=%s\n' "$(hash_inputs "$1" "${SUPERSET_INPUTS[@]}")"
|
||||
printf 'TRINO_CONFIG_HASH=%s\n' "$(hash_inputs "$1" stacks/fuel-monitoring/trino/etc)"
|
||||
# Only the static config: dynamic/ is watched and applies without a restart.
|
||||
printf 'TRAEFIK_CONFIG_HASH=%s\n' "$(hash_inputs "$1" traefik/static)"
|
||||
}
|
||||
|
||||
# load_artifacts <release>: export the release's recorded artifact identities.
|
||||
load_artifacts() {
|
||||
local file=$1/ARTIFACTS line key value
|
||||
[ -s "$file" ] || die "$file is missing"
|
||||
unset SUPERSET_IMAGE_TAG TRINO_CONFIG_HASH TRAEFIK_CONFIG_HASH
|
||||
while IFS= read -r line; do
|
||||
key=${line%%=*}
|
||||
value=${line#*=}
|
||||
[[ $key =~ ^(SUPERSET_IMAGE_TAG|TRINO_CONFIG_HASH|TRAEFIK_CONFIG_HASH)$ && $value =~ ^[0-9a-f]{16}$ ]] ||
|
||||
die "unexpected line in $file: $line"
|
||||
printf -v "$key" '%s' "$value"
|
||||
export "${key?}"
|
||||
done <"$file"
|
||||
[ -n "${SUPERSET_IMAGE_TAG:-}" ] && [ -n "${TRINO_CONFIG_HASH:-}" ] && [ -n "${TRAEFIK_CONFIG_HASH:-}" ] ||
|
||||
die "$file is incomplete"
|
||||
}
|
||||
|
||||
# compose <tree> <dir> <args...>: run Compose for the stack in <tree>/<dir>.
|
||||
compose() {
|
||||
local tree=$1 dir=$2
|
||||
shift 2
|
||||
"${DOCKER[@]}" compose --project-directory "$tree/$dir" -f "$tree/$dir/compose.yaml" "$@"
|
||||
}
|
||||
|
||||
fm() { compose "$LIVE" stacks/fuel-monitoring "$@"; }
|
||||
|
||||
stack_dirs() {
|
||||
local f
|
||||
for f in "$1"/stacks/*/compose.yaml; do
|
||||
[ -e "$f" ] || continue
|
||||
f=${f#"$1"/}
|
||||
printf '%s\n' "${f%/compose.yaml}"
|
||||
done
|
||||
}
|
||||
|
||||
preflight() {
|
||||
local release=$1 f bad dir
|
||||
for f in "${REQUIRED_FILES[@]}"; do
|
||||
[ -s "$ROOT/$f" ] || die "missing or empty: $ROOT/$f (run host/secrets-init.sh as root)"
|
||||
done
|
||||
# oauth2-proxy uses the client secret file byte for byte.
|
||||
f=$ROOT/secrets/shared/keycloak_petrolyst_client_secret
|
||||
[ "$(tail -c1 "$f" | wc -l)" -eq 0 ] ||
|
||||
die "$f ends with a newline; store it again with: host/secrets-init.sh keycloak-secret"
|
||||
bad=$(find "$release/petrolyst-insights" \( -name 'Dockerfile*' -o -name '*compose*.y*ml' \
|
||||
-o -name '*.env' -o -name '.env' -o -path '*/.gitea/*' -o -path '*/.github/*' \) -print)
|
||||
[ -z "$bad" ] || die "petrolyst-insights/ holds customization only; found: $bad"
|
||||
compose "$release" traefik config -q
|
||||
while IFS= read -r dir; do compose "$release" "$dir" config -q; done < <(stack_dirs "$release")
|
||||
}
|
||||
|
||||
# promote <release> <sha>: make the release the live tree. --chmod applies to
|
||||
# the destination only: the live tree stays writable, the release read-only.
|
||||
promote() {
|
||||
mkdir -p "$LIVE"
|
||||
rsync -a --delete --chmod=Du+w,Fu+w "$1/" "$LIVE/"
|
||||
printf '%s\n' "$2" >"$LIVE/REVISION"
|
||||
}
|
||||
|
||||
ensure_proxy_network() {
|
||||
"${DOCKER[@]}" network inspect proxy >/dev/null 2>&1 || "${DOCKER[@]}" network create proxy >/dev/null
|
||||
}
|
||||
|
||||
# metadata_revision: the metadata database's Alembic revision, "fresh" if it
|
||||
# has no alembic_version table yet; any other error (connection, auth) aborts.
|
||||
metadata_revision() {
|
||||
local out
|
||||
if out=$(fm exec -T superset-db psql -U superset -d superset -tAc 'SELECT version_num FROM alembic_version' 2>&1); then
|
||||
[[ $out =~ ^[0-9a-z_]+$ ]] || die "unexpected alembic_version content: $out"
|
||||
printf '%s\n' "$out"
|
||||
elif [[ $out == *'relation "alembic_version" does not exist'* ]]; then
|
||||
printf 'fresh\n'
|
||||
else
|
||||
die "cannot read the metadata database's revision: $out"
|
||||
fi
|
||||
}
|
||||
|
||||
# alembic <image> heads | has <rev>: read Superset's migration graph from the
|
||||
# image without starting the app (loading the scripts through Alembic would
|
||||
# import Superset's models, which need an initialised app).
|
||||
alembic() {
|
||||
"${DOCKER[@]}" run --rm -i --network none --entrypoint python "$1" - "${@:2}" <<'PY'
|
||||
import ast, pathlib, sys
|
||||
|
||||
revs, downs = set(), set()
|
||||
for path in pathlib.Path("/app/superset/migrations/versions").glob("*.py"):
|
||||
for node in ast.parse(path.read_text()).body:
|
||||
if isinstance(node, ast.Assign) and len(node.targets) == 1:
|
||||
target, value = node.targets[0], node.value
|
||||
elif isinstance(node, ast.AnnAssign) and node.value is not None:
|
||||
target, value = node.target, node.value
|
||||
else:
|
||||
continue
|
||||
if isinstance(target, ast.Name) and target.id in ("revision", "down_revision"):
|
||||
found = ast.literal_eval(value)
|
||||
found = found if isinstance(found, (tuple, list)) else (found,)
|
||||
(revs if target.id == "revision" else downs).update(v for v in found if v)
|
||||
if sys.argv[1] == "heads":
|
||||
print(" ".join(sorted(revs - downs)))
|
||||
else:
|
||||
sys.exit(0 if sys.argv[2] in revs else 3)
|
||||
PY
|
||||
}
|
||||
|
||||
# The image tag of the existing superset container, or of the last successful
|
||||
# deploy if the container is gone; empty if neither is known.
|
||||
running_superset_tag() {
|
||||
local id ref
|
||||
id=$(fm ps -a -q superset 2>/dev/null | head -n1)
|
||||
if [ -n "$id" ]; then
|
||||
ref=$("${DOCKER[@]}" inspect --format '{{.Config.Image}}' "$id")
|
||||
printf '%s\n' "${ref#fuel-superset:}"
|
||||
elif [ -s "$STATE/superset-image-tag" ]; then
|
||||
cat "$STATE/superset-image-tag"
|
||||
fi
|
||||
}
|
||||
|
||||
# backup_metadata <image-tag> <revision>: dump the metadata database under a
|
||||
# name that rollback.sh can match to the image whose schema it holds.
|
||||
backup_metadata() {
|
||||
local name tmp
|
||||
name=superset-meta_${1}_${2}_$(date -u +%Y%m%dT%H%M%SZ).dump
|
||||
tmp=$(mktemp "$BACKUPS/.partial-XXXXXX")
|
||||
fm exec -T superset-db pg_dump -U superset -d superset -Fc >"$tmp"
|
||||
[ -s "$tmp" ] || die "metadata backup came out empty"
|
||||
mv "$tmp" "$BACKUPS/$name"
|
||||
log "metadata backup: $BACKUPS/$name"
|
||||
}
|
||||
|
||||
record_state() {
|
||||
mkdir -p "$STATE"
|
||||
printf '%s\n' "$SUPERSET_IMAGE_TAG" >"$STATE/superset-image-tag"
|
||||
printf '%s\n' "$1" >"$STATE/deployed-sha"
|
||||
}
|
||||
|
||||
# Status code and resolved redirect target of one request through the local
|
||||
# Traefik, with certificate verification against the real hostname.
|
||||
probe() {
|
||||
local host=$1 path=$2
|
||||
curl -q --noproxy '*' -sS -o /dev/null --max-time 20 --resolve "$host:443:127.0.0.1" \
|
||||
-w '%{http_code} %{redirect_url}\n' "https://$host$path"
|
||||
}
|
||||
|
||||
smoke_tests() {
|
||||
local i body code location
|
||||
log "smoke tests"
|
||||
# First deploy: Let's Encrypt may need a minute to issue the certificate.
|
||||
for i in $(seq 1 24); do
|
||||
body=$(curl -q --noproxy '*' -fsS --max-time 10 --resolve "$ATOMA_HOST:443:127.0.0.1" \
|
||||
"https://$ATOMA_HOST/health" 2>/dev/null) && [ "$body" = OK ] && break
|
||||
[ "$i" -lt 24 ] || die "https://$ATOMA_HOST/health did not answer OK with a valid certificate"
|
||||
sleep 5
|
||||
done
|
||||
|
||||
read -r code location < <(probe "$ATOMA_HOST" /login/)
|
||||
[ "$code" = 302 ] && [[ $location == "https://$ATOMA_HOST/login/keycloak"* ]] ||
|
||||
die "/login/ should redirect to /login/keycloak, got $code $location"
|
||||
|
||||
read -r code location < <(probe "$ATOMA_HOST" /login/keycloak)
|
||||
[ "$code" = 302 ] && [[ $location == "$KEYCLOAK_REALM_URL/protocol/openid-connect/auth?"* ]] &&
|
||||
[[ $location == *client_id=petrolyst* && $location == *code_challenge_method=S256* ]] ||
|
||||
die "/login/keycloak should redirect to Keycloak with client_id=petrolyst and PKCE S256, got $code $location"
|
||||
|
||||
read -r code location < <(probe "$TRAEFIK_HOST" /)
|
||||
[ "$code" = 302 ] && [[ $location == "$KEYCLOAK_REALM_URL/"* || $location == *"/oauth2/start"* ]] ||
|
||||
die "https://$TRAEFIK_HOST/ should redirect to Keycloak, got $code $location"
|
||||
|
||||
fm exec -T trino trino --execute 'SELECT count(*) FROM fuel.information_schema.schemata' >/dev/null ||
|
||||
die "Trino cannot read the fuel catalog"
|
||||
fm exec -T superset curl -fsS --max-time 10 http://trino:8080/v1/info >/dev/null ||
|
||||
die "Superset cannot reach Trino"
|
||||
log "smoke tests passed"
|
||||
}
|
||||
|
||||
# EXIT trap for the part of a run that changes live state.
|
||||
on_exit() {
|
||||
local status=$?
|
||||
[ "$status" = 0 ] || diagnostics
|
||||
}
|
||||
|
||||
diagnostics() {
|
||||
log "state of the stacks:"
|
||||
compose "$LIVE" traefik ps -a >&2 || true
|
||||
fm ps -a >&2 || true
|
||||
fm logs --no-color --tail=100 >&2 || true
|
||||
}
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
# Manual rollback to a kept release, run on the server as the deploy user:
|
||||
# sudo -u deploy /opt/petrolyst/infra/deploy/rollback.sh <sha>
|
||||
#
|
||||
# Code-only when the target image's Alembic head equals the metadata
|
||||
# database's revision. Otherwise a migration rollback: the metadata database is
|
||||
# dropped, recreated and restored from the backup taken for the target image,
|
||||
# and the restored revision is checked before anything starts. Until the
|
||||
# rollback completes, state/rollback-in-progress blocks deploy.sh; an
|
||||
# interrupted rollback is finished by running it again. Afterwards,
|
||||
# `git revert` and push so main matches; deploy.sh's downgrade guard stops a
|
||||
# plain revert from running against a newer schema.
|
||||
set -euo pipefail
|
||||
# shellcheck source=deploy/lib.sh
|
||||
. "$(dirname "$(readlink -f "$0")")/lib.sh"
|
||||
|
||||
# Destructive steps below are hard-wired to service superset-db and database
|
||||
# superset; nothing here can be pointed at fuel-db.
|
||||
restore_metadata() {
|
||||
local backup=$1 head=$2 restored
|
||||
log "migration rollback: restoring $backup"
|
||||
fm stop superset
|
||||
fm exec -T superset-db psql -U superset -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c 'DROP DATABASE IF EXISTS superset WITH (FORCE)' \
|
||||
-c 'CREATE DATABASE superset OWNER superset'
|
||||
fm exec -T superset-db pg_restore -U superset -d superset --exit-on-error --single-transaction <"$backup"
|
||||
restored=$(metadata_revision)
|
||||
[ "$restored" = "$head" ] ||
|
||||
die "restored revision $restored is not the target head $head; superset stays stopped"
|
||||
}
|
||||
|
||||
# metadata_db_exists: whether database superset exists, asked through the
|
||||
# maintenance database; a connection or authentication error aborts.
|
||||
metadata_db_exists() {
|
||||
local n
|
||||
n=$(fm exec -T superset-db psql -U superset -d postgres -tAc "SELECT count(*) FROM pg_database WHERE datname = 'superset'") ||
|
||||
die "cannot reach the metadata database server"
|
||||
[[ $n =~ ^[01]$ ]] || die "unexpected answer from pg_database: $n"
|
||||
[ "$n" = 1 ]
|
||||
}
|
||||
|
||||
main() {
|
||||
local sha=${1:-} release image revision head backup dir
|
||||
[[ $sha =~ ^[0-9a-f]{40}$ ]] || die "usage: rollback.sh <40-hex sha of a kept release>"
|
||||
acquire_lock 600
|
||||
release=$RELEASES/$sha
|
||||
[ -s "$release/ARTIFACTS" ] || die "no finalized release with ARTIFACTS at $release"
|
||||
# The release's recorded identities, never recomputed.
|
||||
load_artifacts "$release"
|
||||
image=fuel-superset:$SUPERSET_IMAGE_TAG
|
||||
"${DOCKER[@]}" image inspect "$image" >/dev/null 2>&1 || die "image $image is no longer on this host"
|
||||
preflight "$release"
|
||||
|
||||
trap on_exit EXIT
|
||||
fm up -d --wait superset-db redis
|
||||
if [ -e "$ROLLBACK_MARK" ] && ! metadata_db_exists; then
|
||||
# An earlier attempt stopped between DROP and CREATE.
|
||||
revision=fresh
|
||||
else
|
||||
revision=$(metadata_revision)
|
||||
fi
|
||||
# Empty or missing is expected only after an interrupted restore, which left the mark.
|
||||
[ "$revision" != fresh ] || [ -e "$ROLLBACK_MARK" ] || die "the metadata database is empty; there is nothing to roll back"
|
||||
head=$(alembic "$image" heads)
|
||||
[[ $head =~ ^[0-9a-z_]+$ ]] || die "$image has no single migration head: $head"
|
||||
|
||||
if [ "$revision" = "$head" ]; then
|
||||
log "code-only rollback: the schema is already at $head"
|
||||
else
|
||||
backup=$(find "$BACKUPS" -maxdepth 1 -name "superset-meta_${SUPERSET_IMAGE_TAG}_${head}_*.dump" | sort | tail -n1)
|
||||
[ -n "$backup" ] || die "no backup for image $SUPERSET_IMAGE_TAG at revision $head in $BACKUPS"
|
||||
mkdir -p "$STATE"
|
||||
printf '%s\n' "$sha" >"$ROLLBACK_MARK"
|
||||
restore_metadata "$backup" "$head"
|
||||
fi
|
||||
|
||||
promote "$release" "$sha"
|
||||
ensure_proxy_network
|
||||
compose "$LIVE" traefik up -d --wait --remove-orphans
|
||||
# No superset-init: the schema already matches the target image.
|
||||
while IFS= read -r dir; do
|
||||
compose "$LIVE" "$dir" up -d --wait --remove-orphans
|
||||
done < <(stack_dirs "$LIVE")
|
||||
smoke_tests
|
||||
record_state "$sha"
|
||||
rm -f "$ROLLBACK_MARK"
|
||||
trap - EXIT
|
||||
log "rolled back to $sha; now git revert and push so main matches"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in new issue
Block a user