#!/usr/bin/env bash # resume — rollback rehearsal (Phase 3 DoD: "Rollback rehearsed once and documented"). # # Builds three throwaway Go services in a scratch directory and runs the real # deploy mechanism against the real release root and the real port: # # 1. good -> deploys, health-gated, serving # 2. broken -> does not compile: deploy must fail fast, current must not move # 3. unhealthy -> builds and runs, but /healthz is 500: the health gate must # roll back to the previous release and exit non-zero # 4. good again -> then `deploy.sh --rollback` must restore the earlier release # # It leaves nothing running and removes the release root it created, unless # KEEP_REHEARSAL=1. Nothing outside $DEPLOY_ROOT, the scratch dir and this # service's own container/image is touched. # # ./deploy/rehearse.sh # # Exit 0 means every assertion held; the transcript is the evidence. set -Eeuo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DEPLOY="$HERE/deploy.sh" ROOT="${DEPLOY_ROOT:-/home/sam/deployment/resume}" PORT="${APP_PORT:-8096}" SCRATCH="${REHEARSAL_DIR:-$(mktemp -d -t resume-rehearsal-XXXXXX)}" KEEP="${KEEP_REHEARSAL:-0}" pass=0; fail=0 ok() { printf ' PASS %s\n' "$*"; pass=$((pass + 1)); } no() { printf ' FAIL %s\n' "$*"; fail=$((fail + 1)); } hdr() { printf '\n== %s\n' "$*"; } cleanup() { if docker ps -a --format '{{.Names}}' | grep -qx 'resume-app'; then docker rm -f resume-app >/dev/null 2>&1 || true fi docker images --format '{{.Repository}}:{{.Tag}}' | grep -q '^resume/app:' \ && docker images --format '{{.Repository}}:{{.Tag}}' | grep '^resume/app:' \ | xargs -r docker rmi -f >/dev/null 2>&1 || true if [[ "$KEEP" == "1" ]]; then printf '\nkept: release root %s, scratch %s, images resume/app:*\n' "$ROOT" "$SCRATCH" else rm -rf "$ROOT" [[ "$SCRATCH" == /tmp/resume-rehearsal-* || -n "${REHEARSAL_DIR:-}" ]] && rm -rf "$SCRATCH" printf '\ncleaned: release root and rehearsal sources removed\n' fi } trap cleanup EXIT # --------------------------------------------------------------- fixtures make_app() { # $1 dir, $2 mode: good|broken|unhealthy local dir="$1" mode="$2" mkdir -p "$dir/cmd/server" "$dir/deploy" cp -f "$HERE/docker-compose.yml" "$HERE/Dockerfile" "$dir/deploy/" printf 'module rehearsal\n\ngo 1.25\n' > "$dir/go.mod" cat > "$dir/cmd/server/main.go" <<'GO' package main import ( "fmt" "log" "net/http" "os" "time" ) func main() { port := os.Getenv("PORT") if port == "" { port = "8096" } bind := os.Getenv("BIND") if bind == "" { bind = "0.0.0.0" } mux := http.NewServeMux() mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { BROKEN w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, `{"status":"ok","static":true,"ai":"disabled","version":%q,"time":%q}`, os.Getenv("APP_VERSION"), time.Now().UTC().Format(time.RFC3339)) }) mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html; charset=utf-8") fmt.Fprintf(w, "rehearsal

version %s

", os.Getenv("APP_VERSION")) }) srv := &http.Server{Addr: bind + ":" + port, Handler: mux, ReadHeaderTimeout: 5 * time.Second} log.Printf("listening on %s:%s", bind, port) if err := srv.ListenAndServe(); err != nil { log.Fatal(err) } } GO local patch='' case "$mode" in good) patch='' ;; unhealthy) patch='w.WriteHeader(http.StatusInternalServerError)' ;; broken) patch='' ;; esac python3 - "$dir/cmd/server/main.go" "$patch" <<'PY' import sys p, patch = sys.argv[1], sys.argv[2] s = open(p).read().replace("BROKEN", patch, 1) open(p, "w").write(s) PY if [[ "$mode" == "broken" ]]; then sed -i 's/^func main() {/func main( {/' "$dir/cmd/server/main.go" fi } health() { curl -s -m 3 -o /dev/null -w '%{http_code}' "http://127.0.0.1:$PORT/healthz" || true; } serving_version() { curl -s -m 3 "http://127.0.0.1:$PORT/healthz" | sed -n 's/.*"version":"\([^"]*\)".*/\1/p'; } current_target() { [[ -L "$ROOT/current" ]] && readlink -f "$ROOT/current" || echo none; } # --------------------------------------------------------------- 0. preconditions hdr "0. preconditions" if ss -ltn 2>/dev/null | awk '{print $4}' | grep -q ":$PORT\$"; then no "port $PORT already in use — nothing started" exit 1 fi ok "port $PORT free" rm -rf "$ROOT" make_app "$SCRATCH/good" good make_app "$SCRATCH/broken" broken make_app "$SCRATCH/unhealthy" unhealthy ok "three throwaway sources built in $SCRATCH" # --------------------------------------------------------------- 1. good deploy hdr "1. good release deploys and health-gates" if "$DEPLOY" --source "$SCRATCH/good" > "$SCRATCH/1-deploy.log" 2>&1; then ok "deploy.sh exit 0" else no "deploy.sh exit non-zero"; tail -20 "$SCRATCH/1-deploy.log"; exit 1 fi [[ "$(health)" == "200" ]] && ok "/healthz 200 on 127.0.0.1:$PORT" || no "/healthz not 200" v1=$(serving_version); [[ -n "$v1" ]] && ok "serving release $v1" || no "no version reported" ss -ltn | grep -q "127.0.0.1:$PORT" && ok "container publishes on 127.0.0.1 only" || no "unexpected bind" mem=$(docker inspect -f '{{.HostConfig.Memory}}' resume-app); swp=$(docker inspect -f '{{.HostConfig.MemorySwap}}' resume-app) [[ "$mem" == "$swp" && "$mem" != "0" ]] && ok "memswap_limit == mem_limit ($mem bytes)" || no "mem limits differ ($mem vs $swp)" # --------------------------------------------------------------- 2. broken build hdr "2. build failure fails fast, current does not move" before=$(current_target) set +e; "$DEPLOY" --source "$SCRATCH/broken" > "$SCRATCH/2-broken.log" 2>&1; rc=$?; set -e [[ $rc -ne 0 ]] && ok "deploy.sh exit $rc (non-zero, as required)" || no "broken build reported success" [[ "$(current_target)" == "$before" ]] && ok "current unchanged ($before)" || no "current moved after a failed build" [[ "$(health)" == "200" ]] && ok "previous release still serving" || no "site down after failed build" # --------------------------------------------------------------- 3. health gate hdr "3. unhealthy release is rolled back by the health gate" before=$(current_target) set +e; HEALTH_TIMEOUT=10 "$DEPLOY" --source "$SCRATCH/unhealthy" > "$SCRATCH/3-unhealthy.log" 2>&1; rc=$?; set -e [[ $rc -ne 0 ]] && ok "deploy.sh exit $rc (non-zero, as required)" || no "unhealthy release reported success" [[ "$(current_target)" == "$before" ]] && ok "current restored to $before" || no "current left on $(current_target)" [[ "$(health)" == "200" ]] && ok "rolled-back release serving again" || no "no healthy release after rollback" grep -q 'HEALTH GATE FAILED' "$SCRATCH/3-unhealthy.log" && ok "log records the health-gate failure" || no "no health-gate failure recorded" # --------------------------------------------------------------- 4. --rollback hdr "4. deploy.sh --rollback restores the previous release in one command" "$DEPLOY" --source "$SCRATCH/good" > "$SCRATCH/4-deploy.log" 2>&1 && ok "second good release deployed" || no "second deploy failed" v2=$(serving_version) prev=$(readlink -f "$ROOT/previous") set +e; "$DEPLOY" --rollback > "$SCRATCH/4-rollback.log" 2>&1; rc=$?; set -e [[ $rc -eq 0 ]] && ok "rollback exit 0" || no "rollback exit $rc" [[ "$(readlink -f "$ROOT/current")" == "$prev" ]] && ok "current flipped back to $prev" || no "current not restored" [[ "$(health)" == "200" ]] && ok "rolled-back release healthy" || no "rolled-back release unhealthy" v3=$(serving_version) [[ -n "$v3" && "$v3" != "$v2" ]] && ok "serving $v3 again (was $v2)" || no "version did not change on rollback" # --------------------------------------------------------------- 5. restart hdr "5. N3: docker compose down && up restores the service" rel=$(readlink -f "$ROOT/current") docker compose -p resume --project-directory "$rel" -f "$rel/docker-compose.yml" down >/dev/null 2>&1 [[ "$(health)" != "200" ]] && ok "service down after compose down" || no "still serving after compose down" docker compose -p resume --project-directory "$rel" -f "$rel/docker-compose.yml" up -d >/dev/null 2>&1 sleep 3 [[ "$(health)" == "200" ]] && ok "service restored after compose up" || no "service did not come back" # --------------------------------------------------------------- verdict hdr "verdict" printf 'passed %d, failed %d\n' "$pass" "$fail" [[ $fail -eq 0 ]] || exit 1