e6240c1e8e
Sam answered the bind question on AUD-133 (2026-10-26):
"A - dual bind: 127.0.0.1:8096 and 192.168.20.13:8096".
- docker-compose: publish 127.0.0.1 and ${APP_LAN_IP:-192.168.20.13}, never 0.0.0.0
- deploy.sh: carry APP_LAN_IP into the release .env, --lan-ip flag, health gate
now requires /healthz 200 on loopback AND the LAN address
- rehearse.sh: assert both binds, no wildcard, LAN reachability; KEEP_REHEARSAL
now really keeps the container/image; N3 waits up to 20s instead of 3s
- docs/phase-3-evidence.md: decision recorded, new measured output
DoD: 1, 3 (as amended by Sam), 4, 8 met with recorded output. 2, 5, 6 deferred
on the Phase 2 binary; 7 is Sam's Caddy apply. rehearse.sh 24/24, exit 0.
199 lines
9.1 KiB
Bash
Executable File
199 lines
9.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# resume — rollback rehearsal (Phase 3 DoD: "Rollback rehearsed once and documented").
|
|
#
|
|
# Builds three throwaway Go services in a scratch directory and runs the real
|
|
# deploy mechanism against the real release root and the real port:
|
|
#
|
|
# 1. good -> deploys, health-gated, serving on loopback and the LAN address
|
|
# 2. broken -> does not compile: deploy must fail fast, current must not move
|
|
# 3. unhealthy -> builds and runs, but /healthz is 500: the health gate must
|
|
# roll back to the previous release and exit non-zero
|
|
# 4. good again -> then `deploy.sh --rollback` must restore the earlier release
|
|
#
|
|
# It leaves nothing running and removes the release root it created, unless
|
|
# KEEP_REHEARSAL=1. Nothing outside $DEPLOY_ROOT, the scratch dir and this
|
|
# service's own container/image is touched.
|
|
#
|
|
# ./deploy/rehearse.sh
|
|
#
|
|
# Exit 0 means every assertion held; the transcript is the evidence.
|
|
|
|
set -Eeuo pipefail
|
|
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
DEPLOY="$HERE/deploy.sh"
|
|
ROOT="${DEPLOY_ROOT:-/home/sam/deployment/resume}"
|
|
PORT="${APP_PORT:-8096}"
|
|
LAN_IP="${APP_LAN_IP:-192.168.20.13}"
|
|
SCRATCH="${REHEARSAL_DIR:-$(mktemp -d -t resume-rehearsal-XXXXXX)}"
|
|
KEEP="${KEEP_REHEARSAL:-0}"
|
|
|
|
pass=0; fail=0
|
|
ok() { printf ' PASS %s\n' "$*"; pass=$((pass + 1)); }
|
|
no() { printf ' FAIL %s\n' "$*"; fail=$((fail + 1)); }
|
|
hdr() { printf '\n== %s\n' "$*"; }
|
|
|
|
cleanup() {
|
|
if [[ "$KEEP" == "1" ]]; then
|
|
printf '\nkept for inspection: container resume-app, images resume/app:*, release root %s, scratch %s\n' "$ROOT" "$SCRATCH"
|
|
return 0
|
|
fi
|
|
if docker ps -a --format '{{.Names}}' | grep -qx 'resume-app'; then
|
|
docker rm -f resume-app >/dev/null 2>&1 || true
|
|
fi
|
|
docker images --format '{{.Repository}}:{{.Tag}}' | grep -q '^resume/app:' \
|
|
&& docker images --format '{{.Repository}}:{{.Tag}}' | grep '^resume/app:' \
|
|
| xargs -r docker rmi -f >/dev/null 2>&1 || true
|
|
rm -rf "$ROOT"
|
|
[[ "$SCRATCH" == /tmp/resume-rehearsal-* || -n "${REHEARSAL_DIR:-}" ]] && rm -rf "$SCRATCH"
|
|
printf '\ncleaned: container, images, release root and rehearsal sources removed\n'
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
# --------------------------------------------------------------- fixtures
|
|
make_app() { # $1 dir, $2 mode: good|broken|unhealthy
|
|
local dir="$1" mode="$2"
|
|
mkdir -p "$dir/cmd/server" "$dir/deploy"
|
|
cp -f "$HERE/docker-compose.yml" "$HERE/Dockerfile" "$dir/deploy/"
|
|
printf 'module rehearsal\n\ngo 1.25\n' > "$dir/go.mod"
|
|
cat > "$dir/cmd/server/main.go" <<'GO'
|
|
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"time"
|
|
)
|
|
|
|
func main() {
|
|
port := os.Getenv("PORT")
|
|
if port == "" {
|
|
port = "8096"
|
|
}
|
|
bind := os.Getenv("BIND")
|
|
if bind == "" {
|
|
bind = "0.0.0.0"
|
|
}
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
|
|
BROKEN
|
|
w.Header().Set("Content-Type", "application/json")
|
|
fmt.Fprintf(w, `{"status":"ok","static":true,"ai":"disabled","version":%q,"time":%q}`,
|
|
os.Getenv("APP_VERSION"), time.Now().UTC().Format(time.RFC3339))
|
|
})
|
|
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
fmt.Fprintf(w, "<!doctype html><title>rehearsal</title><p>version %s</p>", os.Getenv("APP_VERSION"))
|
|
})
|
|
srv := &http.Server{Addr: bind + ":" + port, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
|
|
log.Printf("listening on %s:%s", bind, port)
|
|
if err := srv.ListenAndServe(); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
}
|
|
GO
|
|
local patch=''
|
|
case "$mode" in
|
|
good) patch='' ;;
|
|
unhealthy) patch='w.WriteHeader(http.StatusInternalServerError)' ;;
|
|
broken) patch='' ;;
|
|
esac
|
|
python3 - "$dir/cmd/server/main.go" "$patch" <<'PY'
|
|
import sys
|
|
p, patch = sys.argv[1], sys.argv[2]
|
|
s = open(p).read().replace("BROKEN", patch, 1)
|
|
open(p, "w").write(s)
|
|
PY
|
|
if [[ "$mode" == "broken" ]]; then
|
|
sed -i 's/^func main() {/func main( {/' "$dir/cmd/server/main.go"
|
|
fi
|
|
}
|
|
|
|
health() { curl -s -m 3 -o /dev/null -w '%{http_code}' "http://127.0.0.1:$PORT/healthz" || true; }
|
|
health_lan() { curl -s -m 3 -o /dev/null -w '%{http_code}' "http://$LAN_IP:$PORT/healthz" || true; }
|
|
listeners() { ss -ltn 2>/dev/null | awk -v p=":${PORT}\$" '$4 ~ p {print $4}' | sort -u; }
|
|
serving_version() { curl -s -m 3 "http://127.0.0.1:$PORT/healthz" | sed -n 's/.*"version":"\([^"]*\)".*/\1/p'; }
|
|
current_target() { [[ -L "$ROOT/current" ]] && readlink -f "$ROOT/current" || echo none; }
|
|
|
|
# --------------------------------------------------------------- 0. preconditions
|
|
hdr "0. preconditions"
|
|
if ss -ltn 2>/dev/null | awk '{print $4}' | grep -q ":$PORT\$"; then
|
|
no "port $PORT already in use — nothing started"
|
|
exit 1
|
|
fi
|
|
ok "port $PORT free"
|
|
rm -rf "$ROOT"
|
|
make_app "$SCRATCH/good" good
|
|
make_app "$SCRATCH/broken" broken
|
|
make_app "$SCRATCH/unhealthy" unhealthy
|
|
ok "three throwaway sources built in $SCRATCH"
|
|
|
|
# --------------------------------------------------------------- 1. good deploy
|
|
hdr "1. good release deploys and health-gates"
|
|
if "$DEPLOY" --source "$SCRATCH/good" > "$SCRATCH/1-deploy.log" 2>&1; then
|
|
ok "deploy.sh exit 0"
|
|
else
|
|
no "deploy.sh exit non-zero"; tail -20 "$SCRATCH/1-deploy.log"; exit 1
|
|
fi
|
|
[[ "$(health)" == "200" ]] && ok "/healthz 200 on 127.0.0.1:$PORT" || no "/healthz not 200"
|
|
v1=$(serving_version); [[ -n "$v1" ]] && ok "serving release $v1" || no "no version reported"
|
|
ss -ltn | grep -q "127.0.0.1:$PORT" && ok "container publishes on 127.0.0.1:$PORT (loopback, used by the deploy.sh health gate)" || no "no loopback bind"
|
|
ss -ltn | grep -q "$LAN_IP:$PORT" && ok "container publishes on $LAN_IP:$PORT (the address Caddy on .35 proxies to)" || no "no LAN bind on $LAN_IP:$PORT"
|
|
if listeners | grep -qE '^(0\.0\.0\.0|\*|\[::\]|::)'; then
|
|
no "wildcard bind on $PORT: $(listeners | tr '\n' ' ')"
|
|
else
|
|
ok "no wildcard bind on $PORT — only: $(listeners | tr '\n' ' ')"
|
|
fi
|
|
[[ "$(health_lan)" == "200" ]] && ok "/healthz 200 over the LAN address http://$LAN_IP:$PORT" || no "LAN address not answering"
|
|
mem=$(docker inspect -f '{{.HostConfig.Memory}}' resume-app); swp=$(docker inspect -f '{{.HostConfig.MemorySwap}}' resume-app)
|
|
[[ "$mem" == "$swp" && "$mem" != "0" ]] && ok "memswap_limit == mem_limit ($mem bytes)" || no "mem limits differ ($mem vs $swp)"
|
|
|
|
# --------------------------------------------------------------- 2. broken build
|
|
hdr "2. build failure fails fast, current does not move"
|
|
before=$(current_target)
|
|
set +e; "$DEPLOY" --source "$SCRATCH/broken" > "$SCRATCH/2-broken.log" 2>&1; rc=$?; set -e
|
|
[[ $rc -ne 0 ]] && ok "deploy.sh exit $rc (non-zero, as required)" || no "broken build reported success"
|
|
[[ "$(current_target)" == "$before" ]] && ok "current unchanged ($before)" || no "current moved after a failed build"
|
|
[[ "$(health)" == "200" ]] && ok "previous release still serving" || no "site down after failed build"
|
|
|
|
# --------------------------------------------------------------- 3. health gate
|
|
hdr "3. unhealthy release is rolled back by the health gate"
|
|
before=$(current_target)
|
|
set +e; HEALTH_TIMEOUT=10 "$DEPLOY" --source "$SCRATCH/unhealthy" > "$SCRATCH/3-unhealthy.log" 2>&1; rc=$?; set -e
|
|
[[ $rc -ne 0 ]] && ok "deploy.sh exit $rc (non-zero, as required)" || no "unhealthy release reported success"
|
|
[[ "$(current_target)" == "$before" ]] && ok "current restored to $before" || no "current left on $(current_target)"
|
|
[[ "$(health)" == "200" ]] && ok "rolled-back release serving again" || no "no healthy release after rollback"
|
|
grep -q 'HEALTH GATE FAILED' "$SCRATCH/3-unhealthy.log" && ok "log records the health-gate failure" || no "no health-gate failure recorded"
|
|
|
|
# --------------------------------------------------------------- 4. --rollback
|
|
hdr "4. deploy.sh --rollback restores the previous release in one command"
|
|
"$DEPLOY" --source "$SCRATCH/good" > "$SCRATCH/4-deploy.log" 2>&1 && ok "second good release deployed" || no "second deploy failed"
|
|
v2=$(serving_version)
|
|
prev=$(readlink -f "$ROOT/previous")
|
|
set +e; "$DEPLOY" --rollback > "$SCRATCH/4-rollback.log" 2>&1; rc=$?; set -e
|
|
[[ $rc -eq 0 ]] && ok "rollback exit 0" || no "rollback exit $rc"
|
|
[[ "$(readlink -f "$ROOT/current")" == "$prev" ]] && ok "current flipped back to $prev" || no "current not restored"
|
|
[[ "$(health)" == "200" ]] && ok "rolled-back release healthy" || no "rolled-back release unhealthy"
|
|
v3=$(serving_version)
|
|
[[ -n "$v3" && "$v3" != "$v2" ]] && ok "serving $v3 again (was $v2)" || no "version did not change on rollback"
|
|
|
|
# --------------------------------------------------------------- 5. restart
|
|
hdr "5. N3: docker compose down && up restores the service"
|
|
rel=$(readlink -f "$ROOT/current")
|
|
docker compose -p resume --project-directory "$rel" -f "$rel/docker-compose.yml" down >/dev/null 2>&1
|
|
[[ "$(health)" != "200" ]] && ok "service down after compose down" || no "still serving after compose down"
|
|
docker compose -p resume --project-directory "$rel" -f "$rel/docker-compose.yml" up -d >/dev/null 2>&1
|
|
restored=0
|
|
for _ in $(seq 1 20); do
|
|
if [[ "$(health)" == "200" ]]; then restored=1; break; fi
|
|
sleep 1
|
|
done
|
|
[[ $restored == 1 ]] && ok "service restored after compose up" || no "service did not come back within 20s"
|
|
|
|
# --------------------------------------------------------------- verdict
|
|
hdr "verdict"
|
|
printf 'passed %d, failed %d\n' "$pass" "$fail"
|
|
[[ $fail -eq 0 ]] || exit 1
|