petrolyst-infra/deploy/lib.sh

# shellcheck shell=bash
# Shared by deploy.sh and rollback.sh; sourced, not executed.

# Server layout (README "Server paths"). PETROLYST_ROOT exists only so the
# scripts can be exercised against a scratch directory; production never sets it.
# shellcheck disable=SC2034 # RELEASES and INCOMING are used by the sourcing scripts
ROOT=${PETROLYST_ROOT:-/opt/petrolyst}
LIVE=$ROOT/infra
RELEASES=$ROOT/releases
INCOMING=$ROOT/incoming
BACKUPS=$ROOT/backups
STATE=$ROOT/state
LOCK_FILE=$ROOT/deploy.lock
# Exists while rollback.sh replaces the metadata database, and stays behind if
# it is interrupted there; deploy.sh refuses to run until a rollback completes.
ROLLBACK_MARK=$STATE/rollback-in-progress

ATOMA_HOST=${ATOMA_HOST:-atoma.petrolyst.com}
TRAEFIK_HOST=${TRAEFIK_HOST:-traefik.petrolyst.com}
KEYCLOAK_REALM_URL=${KEYCLOAK_REALM_URL:-https://iam.thedigitalbridge.io/realms/thedigitalbridge}
PETROLYST_ROOT=$ROOT
export ATOMA_HOST TRAEFIK_HOST KEYCLOAK_REALM_URL PETROLYST_ROOT

# --host pins the daemon, as in petrolyst-web/deploy/deploy.sh: a stale
# `docker context use` must not send this deploy to another daemon.
DOCKER=(docker --host unix:///var/run/docker.sock)

# Build inputs of the Superset image, relative to a release. Everything the
# Dockerfile copies is listed; nothing else is, so the tag changes exactly when
# the image would.
SUPERSET_INPUTS=(
  stacks/fuel-monitoring/superset/Dockerfile
  stacks/fuel-monitoring/superset/init.sh
  petrolyst-insights/superset_config.py
  petrolyst-insights/petrolyst_security.py
  petrolyst-insights/roles
  petrolyst-insights/assets/branding
  petrolyst-insights/patches
)

# Every file the stacks need from /opt/petrolyst (host/secrets-init.sh).
REQUIRED_FILES=(
  secrets/shared/keycloak_petrolyst_client_secret
  secrets/traefik/oauth2_proxy_cookie_secret
  secrets/fuel-monitoring/superset_secret_key
  secrets/fuel-monitoring/superset_db_password
  secrets/fuel-monitoring/fuel_db_password
  secrets/fuel-monitoring/fuel_db_trino_password
  env/fuel-monitoring.env
)

log() { printf '%s %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; }
die() { log "ERROR: $*"; exit 1; }

[ "$(id -u)" != 0 ] || die "run as the deploy user (sudo -u deploy $0 ...), not as root"
# `sudo -u deploy` from a root shell keeps /root as the working directory,
# which deploy cannot read and GNU find refuses to start in. Every path here
# is absolute, so nothing needs the caller's directory.
cd /

# acquire_lock <seconds>: deploy.sh and rollback.sh share this lock, so a
# deploy and a rollback never run at the same time. Held until exit.
acquire_lock() {
  exec 9>>"$LOCK_FILE"
  flock -w "$1" 9 || die "another deploy or rollback holds $LOCK_FILE (waited $1 s)"
}

sha256() {
  if command -v sha256sum >/dev/null; then sha256sum; else shasum -a 256; fi | cut -c1-64
}

# hash_inputs <dir> <path>...: one short digest over the names and contents of
# every file under the paths, independent of mtimes and listing order.
hash_inputs() {
  local dir=$1
  shift
  (
    cd "$dir" || exit 1
    find "$@" -type f -print0 | LC_ALL=C sort -z |
      while IFS= read -r -d '' f; do printf '%s %s\n' "$f" "$(sha256 <"$f")"; done
  ) | sha256 | cut -c1-16
}

compute_artifacts() {
  printf 'SUPERSET_IMAGE_TAG=%s\n' "$(hash_inputs "$1" "${SUPERSET_INPUTS[@]}")"
  printf 'TRINO_CONFIG_HASH=%s\n' "$(hash_inputs "$1" stacks/fuel-monitoring/trino/etc)"
  # Only the static config: dynamic/ is watched and applies without a restart.
  printf 'TRAEFIK_CONFIG_HASH=%s\n' "$(hash_inputs "$1" traefik/static)"
}

# load_artifacts <release>: export the release's recorded artifact identities.
load_artifacts() {
  local file=$1/ARTIFACTS line key value
  [ -s "$file" ] || die "$file is missing"
  unset SUPERSET_IMAGE_TAG TRINO_CONFIG_HASH TRAEFIK_CONFIG_HASH
  while IFS= read -r line; do
    key=${line%%=*}
    value=${line#*=}
    [[ $key =~ ^(SUPERSET_IMAGE_TAG|TRINO_CONFIG_HASH|TRAEFIK_CONFIG_HASH)$ && $value =~ ^[0-9a-f]{16}$ ]] ||
      die "unexpected line in $file: $line"
    printf -v "$key" '%s' "$value"
    export "${key?}"
  done <"$file"
  [ -n "${SUPERSET_IMAGE_TAG:-}" ] && [ -n "${TRINO_CONFIG_HASH:-}" ] && [ -n "${TRAEFIK_CONFIG_HASH:-}" ] ||
    die "$file is incomplete"
}

# compose <tree> <dir> <args...>: run Compose for the stack in <tree>/<dir>.
compose() {
  local tree=$1 dir=$2
  shift 2
  "${DOCKER[@]}" compose --project-directory "$tree/$dir" -f "$tree/$dir/compose.yaml" "$@"
}

fm() { compose "$LIVE" stacks/fuel-monitoring "$@"; }

stack_dirs() {
  local f
  for f in "$1"/stacks/*/compose.yaml; do
    [ -e "$f" ] || continue
    f=${f#"$1"/}
    printf '%s\n' "${f%/compose.yaml}"
  done
}

preflight() {
  local release=$1 f bad dir
  for f in "${REQUIRED_FILES[@]}"; do
    [ -s "$ROOT/$f" ] || die "missing or empty: $ROOT/$f (run host/secrets-init.sh as root)"
  done
  # oauth2-proxy uses the client secret file byte for byte.
  f=$ROOT/secrets/shared/keycloak_petrolyst_client_secret
  [ "$(tail -c1 "$f" | wc -l)" -eq 0 ] ||
    die "$f ends with a newline; store it again with: host/secrets-init.sh keycloak-secret"
  bad=$(find "$release/petrolyst-insights" \( -name 'Dockerfile*' -o -name '*compose*.y*ml' \
    -o -name '*.env' -o -name '.env' -o -path '*/.gitea/*' -o -path '*/.github/*' \) -print)
  [ -z "$bad" ] || die "petrolyst-insights/ holds customization only; found: $bad"
  compose "$release" traefik config -q
  while IFS= read -r dir; do compose "$release" "$dir" config -q; done < <(stack_dirs "$release")
}

# promote <release> <sha>: make the release the live tree. --chmod applies to
# the destination only: the live tree stays writable, the release read-only.
promote() {
  mkdir -p "$LIVE"
  rsync -a --delete --chmod=Du+w,Fu+w "$1/" "$LIVE/"
  printf '%s\n' "$2" >"$LIVE/REVISION"
}

ensure_proxy_network() {
  "${DOCKER[@]}" network inspect proxy >/dev/null 2>&1 || "${DOCKER[@]}" network create proxy >/dev/null
}

# metadata_revision: the metadata database's Alembic revision, "fresh" if it
# has no alembic_version table yet; any other error (connection, auth) aborts.
metadata_revision() {
  local out
  if out=$(fm exec -T superset-db psql -U superset -d superset -tAc 'SELECT version_num FROM alembic_version' 2>&1); then
    [[ $out =~ ^[0-9a-z_]+$ ]] || die "unexpected alembic_version content: $out"
    printf '%s\n' "$out"
  elif [[ $out == *'relation "alembic_version" does not exist'* ]]; then
    printf 'fresh\n'
  else
    die "cannot read the metadata database's revision: $out"
  fi
}

# alembic <image> heads | has <rev>: read Superset's migration graph from the
# image without starting the app (loading the scripts through Alembic would
# import Superset's models, which need an initialised app).
alembic() {
  "${DOCKER[@]}" run --rm -i --network none --entrypoint python "$1" - "${@:2}" <<'PY'
import ast, pathlib, sys

revs, downs = set(), set()
for path in pathlib.Path("/app/superset/migrations/versions").glob("*.py"):
    for node in ast.parse(path.read_text()).body:
        if isinstance(node, ast.Assign) and len(node.targets) == 1:
            target, value = node.targets[0], node.value
        elif isinstance(node, ast.AnnAssign) and node.value is not None:
            target, value = node.target, node.value
        else:
            continue
        if isinstance(target, ast.Name) and target.id in ("revision", "down_revision"):
            found = ast.literal_eval(value)
            found = found if isinstance(found, (tuple, list)) else (found,)
            (revs if target.id == "revision" else downs).update(v for v in found if v)
if sys.argv[1] == "heads":
    print(" ".join(sorted(revs - downs)))
else:
    sys.exit(0 if sys.argv[2] in revs else 3)
PY
}

# The image tag of the existing superset container, or of the last successful
# deploy if the container is gone; empty if neither is known.
running_superset_tag() {
  local id ref
  id=$(fm ps -a -q superset 2>/dev/null | head -n1)
  if [ -n "$id" ]; then
    ref=$("${DOCKER[@]}" inspect --format '{{.Config.Image}}' "$id")
    printf '%s\n' "${ref#fuel-superset:}"
  elif [ -s "$STATE/superset-image-tag" ]; then
    cat "$STATE/superset-image-tag"
  fi
}

# backup_metadata <image-tag> <revision>: dump the metadata database under a
# name that rollback.sh can match to the image whose schema it holds.
backup_metadata() {
  local name tmp
  name=superset-meta_${1}_${2}_$(date -u +%Y%m%dT%H%M%SZ).dump
  tmp=$(mktemp "$BACKUPS/.partial-XXXXXX")
  fm exec -T superset-db pg_dump -U superset -d superset -Fc >"$tmp"
  [ -s "$tmp" ] || die "metadata backup came out empty"
  mv "$tmp" "$BACKUPS/$name"
  log "metadata backup: $BACKUPS/$name"
}

record_state() {
  mkdir -p "$STATE"
  printf '%s\n' "$SUPERSET_IMAGE_TAG" >"$STATE/superset-image-tag"
  printf '%s\n' "$1" >"$STATE/deployed-sha"
}

# Status code and resolved redirect target of one request through the local
# Traefik, with certificate verification against the real hostname.
probe() {
  local host=$1 path=$2
  curl -q --noproxy '*' -sS -o /dev/null --max-time 20 --resolve "$host:443:127.0.0.1" \
    -w '%{http_code} %{redirect_url}\n' "https://$host$path"
}

smoke_tests() {
  local i body code location
  log "smoke tests"
  # First deploy: Let's Encrypt may need a minute to issue the certificate.
  for i in $(seq 1 24); do
    body=$(curl -q --noproxy '*' -fsS --max-time 10 --resolve "$ATOMA_HOST:443:127.0.0.1" \
      "https://$ATOMA_HOST/health" 2>/dev/null) && [ "$body" = OK ] && break
    [ "$i" -lt 24 ] || die "https://$ATOMA_HOST/health did not answer OK with a valid certificate"
    sleep 5
  done

  read -r code location < <(probe "$ATOMA_HOST" /login/)
  [ "$code" = 302 ] && [[ $location == "https://$ATOMA_HOST/login/keycloak"* ]] ||
    die "/login/ should redirect to /login/keycloak, got $code $location"

  read -r code location < <(probe "$ATOMA_HOST" /login/keycloak)
  [ "$code" = 302 ] && [[ $location == "$KEYCLOAK_REALM_URL/protocol/openid-connect/auth?"* ]] &&
    [[ $location == *client_id=petrolyst* && $location == *code_challenge_method=S256* ]] ||
    die "/login/keycloak should redirect to Keycloak with client_id=petrolyst and PKCE S256, got $code $location"

  read -r code location < <(probe "$TRAEFIK_HOST" /)
  [ "$code" = 302 ] && [[ $location == "$KEYCLOAK_REALM_URL/"* || $location == *"/oauth2/start"* ]] ||
    die "https://$TRAEFIK_HOST/ should redirect to Keycloak, got $code $location"

  fm exec -T trino trino --execute 'SELECT count(*) FROM fuel.information_schema.schemata' >/dev/null ||
    die "Trino cannot read the fuel catalog"
  fm exec -T superset curl -fsS --max-time 10 http://trino:8080/v1/info >/dev/null ||
    die "Superset cannot reach Trino"
  log "smoke tests passed"
}

# EXIT trap for the part of a run that changes live state.
on_exit() {
  local status=$?
  [ "$status" = 0 ] || diagnostics
}

diagnostics() {
  log "state of the stacks:"
  compose "$LIVE" traefik ps -a >&2 || true
  fm ps -a >&2 || true
  fm logs --no-color --tail=100 >&2 || true
}