Phase 3: release script, container limits, rehearsed rollback
Adds the deployment mechanism for the portfolio site. The site itself is not deployed yet: Phase 2 has not produced a binary. - deploy/deploy.sh: build + test (fail fast) -> releases/<ts> -> atomic `current` symlink flip -> docker compose up -d -> /healthz gate -> flip back to `previous` and exit non-zero on failure. `--rollback` restores the previous release in one command. Auto-detects the Go toolchain in /nix/store when `go` is not on PATH. - deploy/docker-compose.yml: 127.0.0.1:8096, mem_limit == memswap_limit == 120m, capped json-file logs (10m x 3), restart unless-stopped. - deploy/Dockerfile: runtime image only (alpine + prebuilt payload), so a release needs no toolchain, no network and no package install. - deploy/rehearse.sh: repeatable proof (21 assertions, exit 0) of deploy, build fail-fast, health-gate rollback, --rollback and N3 restart. - deploy/README.md + docs/phase-3-evidence.md: how it works, the build contract Phase 2 must satisfy, the Caddy block for Sam, and the open bind-address question (loopback-only vs reachable from .35). Satisfies DoD items 1, 3 (container), 4, 8. Items 2, 5, 6, 7 need the Phase 2 binary and Sam's Caddy change; recorded as deferred with owners, not claimed. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
System
184
deploy/rehearse.sh
Executable file
184
deploy/rehearse.sh
Executable file
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env bash
|
||||
# resume — rollback rehearsal (Phase 3 DoD: "Rollback rehearsed once and documented").
|
||||
#
|
||||
# Builds three throwaway Go services in a scratch directory and runs the real
|
||||
# deploy mechanism against the real release root and the real port:
|
||||
#
|
||||
# 1. good -> deploys, health-gated, serving
|
||||
# 2. broken -> does not compile: deploy must fail fast, current must not move
|
||||
# 3. unhealthy -> builds and runs, but /healthz is 500: the health gate must
|
||||
# roll back to the previous release and exit non-zero
|
||||
# 4. good again -> then `deploy.sh --rollback` must restore the earlier release
|
||||
#
|
||||
# It leaves nothing running and removes the release root it created, unless
|
||||
# KEEP_REHEARSAL=1. Nothing outside $DEPLOY_ROOT, the scratch dir and this
|
||||
# service's own container/image is touched.
|
||||
#
|
||||
# ./deploy/rehearse.sh
|
||||
#
|
||||
# Exit 0 means every assertion held; the transcript is the evidence.
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
DEPLOY="$HERE/deploy.sh"
|
||||
ROOT="${DEPLOY_ROOT:-/home/sam/deployment/resume}"
|
||||
PORT="${APP_PORT:-8096}"
|
||||
SCRATCH="${REHEARSAL_DIR:-$(mktemp -d -t resume-rehearsal-XXXXXX)}"
|
||||
KEEP="${KEEP_REHEARSAL:-0}"
|
||||
|
||||
pass=0; fail=0
|
||||
ok() { printf ' PASS %s\n' "$*"; pass=$((pass + 1)); }
|
||||
no() { printf ' FAIL %s\n' "$*"; fail=$((fail + 1)); }
|
||||
hdr() { printf '\n== %s\n' "$*"; }
|
||||
|
||||
cleanup() {
|
||||
if docker ps -a --format '{{.Names}}' | grep -qx 'resume-app'; then
|
||||
docker rm -f resume-app >/dev/null 2>&1 || true
|
||||
fi
|
||||
docker images --format '{{.Repository}}:{{.Tag}}' | grep -q '^resume/app:' \
|
||||
&& docker images --format '{{.Repository}}:{{.Tag}}' | grep '^resume/app:' \
|
||||
| xargs -r docker rmi -f >/dev/null 2>&1 || true
|
||||
if [[ "$KEEP" == "1" ]]; then
|
||||
printf '\nkept: release root %s, scratch %s, images resume/app:*\n' "$ROOT" "$SCRATCH"
|
||||
else
|
||||
rm -rf "$ROOT"
|
||||
[[ "$SCRATCH" == /tmp/resume-rehearsal-* || -n "${REHEARSAL_DIR:-}" ]] && rm -rf "$SCRATCH"
|
||||
printf '\ncleaned: release root and rehearsal sources removed\n'
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# --------------------------------------------------------------- fixtures
|
||||
make_app() { # $1 dir, $2 mode: good|broken|unhealthy
|
||||
local dir="$1" mode="$2"
|
||||
mkdir -p "$dir/cmd/server" "$dir/deploy"
|
||||
cp -f "$HERE/docker-compose.yml" "$HERE/Dockerfile" "$dir/deploy/"
|
||||
printf 'module rehearsal\n\ngo 1.25\n' > "$dir/go.mod"
|
||||
cat > "$dir/cmd/server/main.go" <<'GO'
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
port := os.Getenv("PORT")
|
||||
if port == "" {
|
||||
port = "8096"
|
||||
}
|
||||
bind := os.Getenv("BIND")
|
||||
if bind == "" {
|
||||
bind = "0.0.0.0"
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
|
||||
BROKEN
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprintf(w, `{"status":"ok","static":true,"ai":"disabled","version":%q,"time":%q}`,
|
||||
os.Getenv("APP_VERSION"), time.Now().UTC().Format(time.RFC3339))
|
||||
})
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
fmt.Fprintf(w, "<!doctype html><title>rehearsal</title><p>version %s</p>", os.Getenv("APP_VERSION"))
|
||||
})
|
||||
srv := &http.Server{Addr: bind + ":" + port, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
|
||||
log.Printf("listening on %s:%s", bind, port)
|
||||
if err := srv.ListenAndServe(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
GO
|
||||
local patch=''
|
||||
case "$mode" in
|
||||
good) patch='' ;;
|
||||
unhealthy) patch='w.WriteHeader(http.StatusInternalServerError)' ;;
|
||||
broken) patch='' ;;
|
||||
esac
|
||||
python3 - "$dir/cmd/server/main.go" "$patch" <<'PY'
|
||||
import sys
|
||||
p, patch = sys.argv[1], sys.argv[2]
|
||||
s = open(p).read().replace("BROKEN", patch, 1)
|
||||
open(p, "w").write(s)
|
||||
PY
|
||||
if [[ "$mode" == "broken" ]]; then
|
||||
sed -i 's/^func main() {/func main( {/' "$dir/cmd/server/main.go"
|
||||
fi
|
||||
}
|
||||
|
||||
health() { curl -s -m 3 -o /dev/null -w '%{http_code}' "http://127.0.0.1:$PORT/healthz" || true; }
|
||||
serving_version() { curl -s -m 3 "http://127.0.0.1:$PORT/healthz" | sed -n 's/.*"version":"\([^"]*\)".*/\1/p'; }
|
||||
current_target() { [[ -L "$ROOT/current" ]] && readlink -f "$ROOT/current" || echo none; }
|
||||
|
||||
# --------------------------------------------------------------- 0. preconditions
|
||||
hdr "0. preconditions"
|
||||
if ss -ltn 2>/dev/null | awk '{print $4}' | grep -q ":$PORT\$"; then
|
||||
no "port $PORT already in use — nothing started"
|
||||
exit 1
|
||||
fi
|
||||
ok "port $PORT free"
|
||||
rm -rf "$ROOT"
|
||||
make_app "$SCRATCH/good" good
|
||||
make_app "$SCRATCH/broken" broken
|
||||
make_app "$SCRATCH/unhealthy" unhealthy
|
||||
ok "three throwaway sources built in $SCRATCH"
|
||||
|
||||
# --------------------------------------------------------------- 1. good deploy
|
||||
hdr "1. good release deploys and health-gates"
|
||||
if "$DEPLOY" --source "$SCRATCH/good" > "$SCRATCH/1-deploy.log" 2>&1; then
|
||||
ok "deploy.sh exit 0"
|
||||
else
|
||||
no "deploy.sh exit non-zero"; tail -20 "$SCRATCH/1-deploy.log"; exit 1
|
||||
fi
|
||||
[[ "$(health)" == "200" ]] && ok "/healthz 200 on 127.0.0.1:$PORT" || no "/healthz not 200"
|
||||
v1=$(serving_version); [[ -n "$v1" ]] && ok "serving release $v1" || no "no version reported"
|
||||
ss -ltn | grep -q "127.0.0.1:$PORT" && ok "container publishes on 127.0.0.1 only" || no "unexpected bind"
|
||||
mem=$(docker inspect -f '{{.HostConfig.Memory}}' resume-app); swp=$(docker inspect -f '{{.HostConfig.MemorySwap}}' resume-app)
|
||||
[[ "$mem" == "$swp" && "$mem" != "0" ]] && ok "memswap_limit == mem_limit ($mem bytes)" || no "mem limits differ ($mem vs $swp)"
|
||||
|
||||
# --------------------------------------------------------------- 2. broken build
|
||||
hdr "2. build failure fails fast, current does not move"
|
||||
before=$(current_target)
|
||||
set +e; "$DEPLOY" --source "$SCRATCH/broken" > "$SCRATCH/2-broken.log" 2>&1; rc=$?; set -e
|
||||
[[ $rc -ne 0 ]] && ok "deploy.sh exit $rc (non-zero, as required)" || no "broken build reported success"
|
||||
[[ "$(current_target)" == "$before" ]] && ok "current unchanged ($before)" || no "current moved after a failed build"
|
||||
[[ "$(health)" == "200" ]] && ok "previous release still serving" || no "site down after failed build"
|
||||
|
||||
# --------------------------------------------------------------- 3. health gate
|
||||
hdr "3. unhealthy release is rolled back by the health gate"
|
||||
before=$(current_target)
|
||||
set +e; HEALTH_TIMEOUT=10 "$DEPLOY" --source "$SCRATCH/unhealthy" > "$SCRATCH/3-unhealthy.log" 2>&1; rc=$?; set -e
|
||||
[[ $rc -ne 0 ]] && ok "deploy.sh exit $rc (non-zero, as required)" || no "unhealthy release reported success"
|
||||
[[ "$(current_target)" == "$before" ]] && ok "current restored to $before" || no "current left on $(current_target)"
|
||||
[[ "$(health)" == "200" ]] && ok "rolled-back release serving again" || no "no healthy release after rollback"
|
||||
grep -q 'HEALTH GATE FAILED' "$SCRATCH/3-unhealthy.log" && ok "log records the health-gate failure" || no "no health-gate failure recorded"
|
||||
|
||||
# --------------------------------------------------------------- 4. --rollback
|
||||
hdr "4. deploy.sh --rollback restores the previous release in one command"
|
||||
"$DEPLOY" --source "$SCRATCH/good" > "$SCRATCH/4-deploy.log" 2>&1 && ok "second good release deployed" || no "second deploy failed"
|
||||
v2=$(serving_version)
|
||||
prev=$(readlink -f "$ROOT/previous")
|
||||
set +e; "$DEPLOY" --rollback > "$SCRATCH/4-rollback.log" 2>&1; rc=$?; set -e
|
||||
[[ $rc -eq 0 ]] && ok "rollback exit 0" || no "rollback exit $rc"
|
||||
[[ "$(readlink -f "$ROOT/current")" == "$prev" ]] && ok "current flipped back to $prev" || no "current not restored"
|
||||
[[ "$(health)" == "200" ]] && ok "rolled-back release healthy" || no "rolled-back release unhealthy"
|
||||
v3=$(serving_version)
|
||||
[[ -n "$v3" && "$v3" != "$v2" ]] && ok "serving $v3 again (was $v2)" || no "version did not change on rollback"
|
||||
|
||||
# --------------------------------------------------------------- 5. restart
|
||||
hdr "5. N3: docker compose down && up restores the service"
|
||||
rel=$(readlink -f "$ROOT/current")
|
||||
docker compose -p resume --project-directory "$rel" -f "$rel/docker-compose.yml" down >/dev/null 2>&1
|
||||
[[ "$(health)" != "200" ]] && ok "service down after compose down" || no "still serving after compose down"
|
||||
docker compose -p resume --project-directory "$rel" -f "$rel/docker-compose.yml" up -d >/dev/null 2>&1
|
||||
sleep 3
|
||||
[[ "$(health)" == "200" ]] && ok "service restored after compose up" || no "service did not come back"
|
||||
|
||||
# --------------------------------------------------------------- verdict
|
||||
hdr "verdict"
|
||||
printf 'passed %d, failed %d\n' "$pass" "$fail"
|
||||
[[ $fail -eq 0 ]] || exit 1
|
||||
Reference in New Issue
Block a user