ai-resume/deploy/rehearse.sh

#!/usr/bin/env bash
# resume — rollback rehearsal (Phase 3 DoD: "Rollback rehearsed once and documented").
#
# Builds three throwaway Go services in a scratch directory and runs the real
# deploy mechanism against the real release root and the real port:
#
#   1. good       -> deploys, health-gated, serving
#   2. broken     -> does not compile: deploy must fail fast, current must not move
#   3. unhealthy  -> builds and runs, but /healthz is 500: the health gate must
#                    roll back to the previous release and exit non-zero
#   4. good again -> then `deploy.sh --rollback` must restore the earlier release
#
# It leaves nothing running and removes the release root it created, unless
# KEEP_REHEARSAL=1. Nothing outside $DEPLOY_ROOT, the scratch dir and this
# service's own container/image is touched.
#
#   ./deploy/rehearse.sh
#
# Exit 0 means every assertion held; the transcript is the evidence.

set -Eeuo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DEPLOY="$HERE/deploy.sh"
ROOT="${DEPLOY_ROOT:-/home/sam/deployment/resume}"
PORT="${APP_PORT:-8096}"
SCRATCH="${REHEARSAL_DIR:-$(mktemp -d -t resume-rehearsal-XXXXXX)}"
KEEP="${KEEP_REHEARSAL:-0}"

pass=0; fail=0
ok()   { printf '  PASS  %s\n' "$*"; pass=$((pass + 1)); }
no()   { printf '  FAIL  %s\n' "$*"; fail=$((fail + 1)); }
hdr()  { printf '\n== %s\n' "$*"; }

cleanup() {
  if docker ps -a --format '{{.Names}}' | grep -qx 'resume-app'; then
    docker rm -f resume-app >/dev/null 2>&1 || true
  fi
  docker images --format '{{.Repository}}:{{.Tag}}' | grep -q '^resume/app:' \
    && docker images --format '{{.Repository}}:{{.Tag}}' | grep '^resume/app:' \
       | xargs -r docker rmi -f >/dev/null 2>&1 || true
  if [[ "$KEEP" == "1" ]]; then
    printf '\nkept: release root %s, scratch %s, images resume/app:*\n' "$ROOT" "$SCRATCH"
  else
    rm -rf "$ROOT"
    [[ "$SCRATCH" == /tmp/resume-rehearsal-* || -n "${REHEARSAL_DIR:-}" ]] && rm -rf "$SCRATCH"
    printf '\ncleaned: release root and rehearsal sources removed\n'
  fi
}
trap cleanup EXIT

# --------------------------------------------------------------- fixtures
make_app() { # $1 dir, $2 mode: good|broken|unhealthy
  local dir="$1" mode="$2"
  mkdir -p "$dir/cmd/server" "$dir/deploy"
  cp -f "$HERE/docker-compose.yml" "$HERE/Dockerfile" "$dir/deploy/"
  printf 'module rehearsal\n\ngo 1.25\n' > "$dir/go.mod"
  cat > "$dir/cmd/server/main.go" <<'GO'
package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"time"
)

func main() {
	port := os.Getenv("PORT")
	if port == "" {
		port = "8096"
	}
	bind := os.Getenv("BIND")
	if bind == "" {
		bind = "0.0.0.0"
	}
	mux := http.NewServeMux()
	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
		BROKEN
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, `{"status":"ok","static":true,"ai":"disabled","version":%q,"time":%q}`,
			os.Getenv("APP_VERSION"), time.Now().UTC().Format(time.RFC3339))
	})
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		fmt.Fprintf(w, "<!doctype html><title>rehearsal</title><p>version %s</p>", os.Getenv("APP_VERSION"))
	})
	srv := &http.Server{Addr: bind + ":" + port, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
	log.Printf("listening on %s:%s", bind, port)
	if err := srv.ListenAndServe(); err != nil {
		log.Fatal(err)
	}
}
GO
  local patch=''
  case "$mode" in
    good)      patch='' ;;
    unhealthy) patch='w.WriteHeader(http.StatusInternalServerError)' ;;
    broken)    patch='' ;;
  esac
  python3 - "$dir/cmd/server/main.go" "$patch" <<'PY'
import sys
p, patch = sys.argv[1], sys.argv[2]
s = open(p).read().replace("BROKEN", patch, 1)
open(p, "w").write(s)
PY
  if [[ "$mode" == "broken" ]]; then
    sed -i 's/^func main() {/func main( {/' "$dir/cmd/server/main.go"
  fi
}

health() { curl -s -m 3 -o /dev/null -w '%{http_code}' "http://127.0.0.1:$PORT/healthz" || true; }
serving_version() { curl -s -m 3 "http://127.0.0.1:$PORT/healthz" | sed -n 's/.*"version":"\([^"]*\)".*/\1/p'; }
current_target() { [[ -L "$ROOT/current" ]] && readlink -f "$ROOT/current" || echo none; }

# --------------------------------------------------------------- 0. preconditions
hdr "0. preconditions"
if ss -ltn 2>/dev/null | awk '{print $4}' | grep -q ":$PORT\$"; then
  no "port $PORT already in use — nothing started"
  exit 1
fi
ok "port $PORT free"
rm -rf "$ROOT"
make_app "$SCRATCH/good" good
make_app "$SCRATCH/broken" broken
make_app "$SCRATCH/unhealthy" unhealthy
ok "three throwaway sources built in $SCRATCH"

# --------------------------------------------------------------- 1. good deploy
hdr "1. good release deploys and health-gates"
if "$DEPLOY" --source "$SCRATCH/good" > "$SCRATCH/1-deploy.log" 2>&1; then
  ok "deploy.sh exit 0"
else
  no "deploy.sh exit non-zero"; tail -20 "$SCRATCH/1-deploy.log"; exit 1
fi
[[ "$(health)" == "200" ]] && ok "/healthz 200 on 127.0.0.1:$PORT" || no "/healthz not 200"
v1=$(serving_version); [[ -n "$v1" ]] && ok "serving release $v1" || no "no version reported"
ss -ltn | grep -q "127.0.0.1:$PORT" && ok "container publishes on 127.0.0.1 only" || no "unexpected bind"
mem=$(docker inspect -f '{{.HostConfig.Memory}}' resume-app); swp=$(docker inspect -f '{{.HostConfig.MemorySwap}}' resume-app)
[[ "$mem" == "$swp" && "$mem" != "0" ]] && ok "memswap_limit == mem_limit ($mem bytes)" || no "mem limits differ ($mem vs $swp)"

# --------------------------------------------------------------- 2. broken build
hdr "2. build failure fails fast, current does not move"
before=$(current_target)
set +e; "$DEPLOY" --source "$SCRATCH/broken" > "$SCRATCH/2-broken.log" 2>&1; rc=$?; set -e
[[ $rc -ne 0 ]] && ok "deploy.sh exit $rc (non-zero, as required)" || no "broken build reported success"
[[ "$(current_target)" == "$before" ]] && ok "current unchanged ($before)" || no "current moved after a failed build"
[[ "$(health)" == "200" ]] && ok "previous release still serving" || no "site down after failed build"

# --------------------------------------------------------------- 3. health gate
hdr "3. unhealthy release is rolled back by the health gate"
before=$(current_target)
set +e; HEALTH_TIMEOUT=10 "$DEPLOY" --source "$SCRATCH/unhealthy" > "$SCRATCH/3-unhealthy.log" 2>&1; rc=$?; set -e
[[ $rc -ne 0 ]] && ok "deploy.sh exit $rc (non-zero, as required)" || no "unhealthy release reported success"
[[ "$(current_target)" == "$before" ]] && ok "current restored to $before" || no "current left on $(current_target)"
[[ "$(health)" == "200" ]] && ok "rolled-back release serving again" || no "no healthy release after rollback"
grep -q 'HEALTH GATE FAILED' "$SCRATCH/3-unhealthy.log" && ok "log records the health-gate failure" || no "no health-gate failure recorded"

# --------------------------------------------------------------- 4. --rollback
hdr "4. deploy.sh --rollback restores the previous release in one command"
"$DEPLOY" --source "$SCRATCH/good" > "$SCRATCH/4-deploy.log" 2>&1 && ok "second good release deployed" || no "second deploy failed"
v2=$(serving_version)
prev=$(readlink -f "$ROOT/previous")
set +e; "$DEPLOY" --rollback > "$SCRATCH/4-rollback.log" 2>&1; rc=$?; set -e
[[ $rc -eq 0 ]] && ok "rollback exit 0" || no "rollback exit $rc"
[[ "$(readlink -f "$ROOT/current")" == "$prev" ]] && ok "current flipped back to $prev" || no "current not restored"
[[ "$(health)" == "200" ]] && ok "rolled-back release healthy" || no "rolled-back release unhealthy"
v3=$(serving_version)
[[ -n "$v3" && "$v3" != "$v2" ]] && ok "serving $v3 again (was $v2)" || no "version did not change on rollback"

# --------------------------------------------------------------- 5. restart
hdr "5. N3: docker compose down && up restores the service"
rel=$(readlink -f "$ROOT/current")
docker compose -p resume --project-directory "$rel" -f "$rel/docker-compose.yml" down >/dev/null 2>&1
[[ "$(health)" != "200" ]] && ok "service down after compose down" || no "still serving after compose down"
docker compose -p resume --project-directory "$rel" -f "$rel/docker-compose.yml" up -d >/dev/null 2>&1
sleep 3
[[ "$(health)" == "200" ]] && ok "service restored after compose up" || no "service did not come back"

# --------------------------------------------------------------- verdict
hdr "verdict"
printf 'passed %d, failed %d\n' "$pass" "$fail"
[[ $fail -eq 0 ]] || exit 1