petrolyst-infra/deploy/deploy.sh

#!/usr/bin/env bash
# Server-side deploy, run as the non-root deploy user:
#   deploy.sh <sha> <incoming-dir>   manual deploy: finalize the upload as releases/<sha>, then deploy it
#   deploy.sh <sha>                  re-deploy a release that is already finalized
#
# Order: lock, finalize, artifacts, preflight, image build and downgrade guard
# (nothing live touched yet), then promote, Traefik, each stack, smoke tests,
# retention. Rollback is a separate, manual script (rollback.sh) under the same
# lock.
set -euo pipefail
# shellcheck source=deploy/lib.sh
. "$(dirname "$(readlink -f "$0")")/lib.sh"

# finalize_release <sha> <upload>: an upload never writes into releases/. A new SHA is
# moved in atomically; a re-run of a deployed SHA must match it byte for byte
# and leaves the finalized release, ARTIFACTS included, untouched.
finalize_release() {
  local release=$RELEASES/$1 upload
  upload=$(readlink -f "$2") || die "upload $2 does not exist"
  [[ $upload == "$(readlink -f "$INCOMING")"/* && -d $upload ]] || die "upload $2 is not a directory under $INCOMING"
  if [ ! -e "$release" ]; then
    mv "$upload" "$release"
    log "finalized $release"
  elif diff -r --exclude=ARTIFACTS "$upload" "$release" >/dev/null; then
    rm -rf "$upload"
    log "release $1 already finalized and identical to the upload; reusing it"
  else
    die "upload differs from the finalized release $release; one SHA cannot have two contents"
  fi
}

# resolve_artifacts <release>: compute the artifact identities from the
# release's own files, record them once in ARTIFACTS, then freeze the release.
resolve_artifacts() {
  local release=$1 computed
  computed=$(compute_artifacts "$release")
  if [ -e "$release/ARTIFACTS" ]; then
    [ "$(cat "$release/ARTIFACTS")" = "$computed" ] || die "$release/ARTIFACTS does not match the release's files"
  else
    # A crash between the move and this step leaves no ARTIFACTS; the files
    # are unchanged, so the computed values are the same.
    chmod u+w "$release"
    printf '%s\n' "$computed" >"$release/ARTIFACTS"
  fi
  chmod -R a-w "$release"
  load_artifacts "$release"
  log "artifacts: superset $SUPERSET_IMAGE_TAG, traefik config $TRAEFIK_CONFIG_HASH"
}

# prepare_fuel_monitoring <release>: before anything live changes, build the
# Superset image from the release and refuse an image older than the schema of
# a running metadata database.
prepare_fuel_monitoring() {
  local image=fuel-superset:$SUPERSET_IMAGE_TAG revision
  if ! "${DOCKER[@]}" image inspect "$image" >/dev/null 2>&1; then
    log "building $image"
    compose "$1" stacks/fuel-monitoring build superset
  fi
  [ -e "$LIVE/stacks/fuel-monitoring/compose.yaml" ] || return 0
  [ -n "$(fm ps --status running -q superset-db 2>/dev/null)" ] || return 0
  revision=$(metadata_revision)
  [ "$revision" = fresh ] || downgrade_guard "$image" "$revision"
}

downgrade_guard() {
  alembic "$1" has "$2" ||
    die "the metadata database is at revision $2, which $1 does not know: the image is older than the schema. Use deploy/rollback.sh."
}

deploy_fuel_monitoring() {
  local image=fuel-superset:$SUPERSET_IMAGE_TAG revision running head
  fm up -d --wait superset-db redis
  revision=$(metadata_revision)
  if [ "$revision" = fresh ]; then
    log "metadata database is empty: fresh install, running superset-init"
    fm run --rm superset-init
  else
    # Again here: the database may not have been running during preparation.
    downgrade_guard "$image" "$revision"
    running=$(running_superset_tag)
    # An unchanged tag alone is not enough: the schema may have been changed
    # under the container (a restore, a manual downgrade).
    head=$(alembic "$image" heads)
    if [ "$running" != "$SUPERSET_IMAGE_TAG" ] || [ "$revision" != "$head" ]; then
      log "Superset image ${running:-unknown} -> $SUPERSET_IMAGE_TAG, schema $revision -> $head: stop, back up, migrate"
      fm stop superset
      backup_metadata "${running:-unknown}" "$revision"
      fm run --rm superset-init
    fi
  fi
  fm up -d --wait --remove-orphans
}

# Old releases, stale uploads, old backups and unreferenced Superset images.
retention() {
  local current=$1 dir tag referenced
  while IFS= read -r dir; do
    [ "${dir##*/}" != "$current" ] || continue
    chmod -R u+w "$dir"
    rm -rf "$dir"
    log "removed release ${dir##*/}"
  done < <(find "$RELEASES" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -rn | tail -n +6 | cut -d' ' -f2-)
  # Uploads older than a day were never finalized (interrupted deploys). The
  # upload directory is created on the server when the upload starts, and its
  # mtime is the server's, so an upload being sent or waiting for the lock is
  # never this old.
  while IFS= read -r -d '' dir; do
    chmod -R u+w "$dir"
    rm -rf "$dir"
    log "removed stale upload ${dir##*/}"
  done < <(find "$INCOMING" -mindepth 1 -maxdepth 1 -mmin +1440 -print0)
  find "$BACKUPS" -maxdepth 1 -name 'superset-meta_*.dump' -mtime +30 -delete
  find "$BACKUPS" -maxdepth 1 -name '.partial-*' -mmin +60 -delete
  # Keep the 3 newest images plus every image a kept release or backup names.
  referenced=$(
    sed -n 's/^SUPERSET_IMAGE_TAG=//p' "$RELEASES"/*/ARTIFACTS 2>/dev/null || true
    find "$BACKUPS" -maxdepth 1 -name 'superset-meta_*.dump' -exec basename {} \; |
      sed -n 's/^superset-meta_\([0-9a-f]\{16\}\)_.*/\1/p'
  )
  "${DOCKER[@]}" image ls fuel-superset --format '{{.Tag}}' | tail -n +4 | while IFS= read -r tag; do
    grep -qxF "$tag" <<<"$referenced" && continue
    if "${DOCKER[@]}" image rm "fuel-superset:$tag" >/dev/null; then
      log "removed image fuel-superset:$tag"
    else
      log "kept image fuel-superset:$tag (still in use)"
    fi
  done
}

main() {
  local sha=${1:-} upload=${2:-} release dir
  [[ $sha =~ ^[0-9a-f]{40}$ ]] || die "usage: deploy.sh <40-hex sha> [incoming-dir]"
  acquire_lock 900
  [ ! -e "$ROLLBACK_MARK" ] ||
    die "a rollback to $(cat "$ROLLBACK_MARK") did not finish and may have left the metadata database half-restored; run deploy/rollback.sh again first"
  release=$RELEASES/$sha
  [ -z "$upload" ] || finalize_release "$sha" "$upload"
  [ -d "$release" ] || die "no release $release"
  resolve_artifacts "$release"
  preflight "$release"
  prepare_fuel_monitoring "$release"

  # From here on the live state changes; on failure, print the stacks' state.
  trap on_exit EXIT
  promote "$release" "$sha"
  ensure_proxy_network
  log "traefik"
  compose "$LIVE" traefik up -d --wait --remove-orphans
  while IFS= read -r dir; do
    log "$dir"
    case $dir in
      stacks/fuel-monitoring) deploy_fuel_monitoring ;;
      *) compose "$LIVE" "$dir" up -d --wait --remove-orphans ;;
    esac
  done < <(stack_dirs "$LIVE")
  smoke_tests
  record_state "$sha"
  trap - EXIT
  retention "$sha"
  log "deployed $sha"
}

main "$@"