e6240c1e8e
Sam answered the bind question on AUD-133 (2026-10-26):
"A - dual bind: 127.0.0.1:8096 and 192.168.20.13:8096".
- docker-compose: publish 127.0.0.1 and ${APP_LAN_IP:-192.168.20.13}, never 0.0.0.0
- deploy.sh: carry APP_LAN_IP into the release .env, --lan-ip flag, health gate
now requires /healthz 200 on loopback AND the LAN address
- rehearse.sh: assert both binds, no wildcard, LAN reachability; KEEP_REHEARSAL
now really keeps the container/image; N3 waits up to 20s instead of 3s
- docs/phase-3-evidence.md: decision recorded, new measured output
DoD: 1, 3 (as amended by Sam), 4, 8 met with recorded output. 2, 5, 6 deferred
on the Phase 2 binary; 7 is Sam's Caddy apply. rehearse.sh 24/24, exit 0.
322 lines
12 KiB
Bash
Executable File
322 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# resume — release script (Phase 3).
|
|
#
|
|
# Deploy is a script, not a model decision (AGENTS.md §4.6).
|
|
# No one hand-edits files on the production host: change this script in git.
|
|
#
|
|
# ./deploy/deploy.sh build, test, release, flip, health-gate
|
|
# ./deploy/deploy.sh --rollback restore the previous release, one command
|
|
#
|
|
# Layout on the runtime host (.13):
|
|
#
|
|
# /home/sam/deployment/resume/
|
|
# ├── releases/<UTC timestamp>/ immutable release: bin/, dist/, compose, manifest
|
|
# ├── current -> releases/<ts> the live release (atomic symlink flip)
|
|
# ├── previous -> releases/<ts> the release that was live before the last flip
|
|
# └── deploy.log append-only record of every deploy attempt
|
|
#
|
|
# A release is only "current" after /healthz answers 200 on 127.0.0.1:$PORT.
|
|
# If the health gate fails, `current` is flipped back to `previous`, the previous
|
|
# release is restarted, and this script exits non-zero.
|
|
|
|
set -Eeuo pipefail
|
|
|
|
APP_NAME="resume"
|
|
DEFAULT_PORT=8096
|
|
DEFAULT_ROOT="/home/sam/deployment/resume"
|
|
|
|
SRC_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
ROOT="${DEPLOY_ROOT:-$DEFAULT_ROOT}"
|
|
PORT="${APP_PORT:-$DEFAULT_PORT}"
|
|
# Caddy on .35 proxies to this host's LAN address (phase doc §3.3), so the
|
|
# container publishes on loopback *and* on that address. Sam's decision on
|
|
# AUD-133, 2026-10-26: "A - dual bind: 127.0.0.1:8096 and 192.168.20.13:8096".
|
|
LAN_IP="${APP_LAN_IP:-192.168.20.13}"
|
|
DO_ROLLBACK=0
|
|
HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-40}"
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage: deploy.sh [options]
|
|
|
|
--rollback restore the previous release and exit
|
|
--source DIR build from DIR instead of the repo this script lives in
|
|
--root DIR release root (default /home/sam/deployment/resume)
|
|
--port N app port (default 8096)
|
|
--lan-ip ADDR LAN address the container also publishes on (default 192.168.20.13)
|
|
--health-timeout N seconds to wait for /healthz (default 40)
|
|
-h, --help this text
|
|
|
|
Environment:
|
|
GO path to the go toolchain (auto-detected, incl. /nix/store)
|
|
DEPLOY_ROOT same as --root
|
|
APP_PORT same as --port
|
|
APP_LAN_IP same as --lan-ip
|
|
EOF
|
|
}
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--rollback) DO_ROLLBACK=1; shift ;;
|
|
--source) SRC_DIR="$(cd "$2" && pwd)"; shift 2 ;;
|
|
--root) ROOT="$2"; shift 2 ;;
|
|
--port) PORT="$2"; shift 2 ;;
|
|
--lan-ip) LAN_IP="$2"; shift 2 ;;
|
|
--health-timeout) HEALTH_TIMEOUT="$2"; shift 2 ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
RELEASES="$ROOT/releases"
|
|
CURRENT="$ROOT/current"
|
|
PREVIOUS="$ROOT/previous"
|
|
LOG="$ROOT/deploy.log"
|
|
|
|
mkdir -p "$RELEASES"
|
|
|
|
log() { printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOG" >&2; }
|
|
die() { log "FAIL: $*"; exit 1; }
|
|
|
|
# ---------------------------------------------------------------- toolchain
|
|
find_go() {
|
|
if [[ -n "${GO:-}" && -x "${GO}" ]]; then echo "$GO"; return; fi
|
|
if command -v go >/dev/null 2>&1; then command -v go; return; fi
|
|
# NixOS: the toolchain lives in the store even when it is not on PATH.
|
|
local cand
|
|
cand=$(ls -d /nix/store/*-go-1.*/bin/go 2>/dev/null | grep -v '\.drv' | sort -V | tail -1)
|
|
if [[ -n "$cand" && -x "$cand" ]]; then echo "$cand"; return; fi
|
|
return 1
|
|
}
|
|
|
|
compose() {
|
|
# $1 = release directory that holds docker-compose.yml + .env
|
|
local dir="$1"; shift
|
|
docker compose -p "$APP_NAME" --project-directory "$dir" -f "$dir/docker-compose.yml" "$@"
|
|
}
|
|
|
|
# ---------------------------------------------------------------- health gate
|
|
check_health() {
|
|
# 200 on /healthz, and if the body is JSON with a "static" field it must be true.
|
|
# AI readiness is deliberately NOT part of this gate: Mode A must work with
|
|
# every AI service stopped (acceptance test A1).
|
|
local url="http://127.0.0.1:${PORT}/healthz" body code
|
|
body=$(curl -sS --max-time 3 -w '\n%{http_code}' "$url" 2>/dev/null || true)
|
|
code=$(printf '%s' "$body" | tail -1)
|
|
body=$(printf '%s' "$body" | sed '$d')
|
|
if [[ "$code" != "200" ]]; then
|
|
log "health: $url -> HTTP ${code:-none}"
|
|
return 1
|
|
fi
|
|
if printf '%s' "$body" | head -c 1 | grep -q '{' && command -v jq >/dev/null 2>&1; then
|
|
local static_ok
|
|
static_ok=$(printf '%s' "$body" | jq -r 'if has("static") then (.static|tostring) else "absent" end' 2>/dev/null || echo absent)
|
|
if [[ "$static_ok" == "false" ]]; then
|
|
log "health: $url -> 200 but static readiness is false: $body"
|
|
return 1
|
|
fi
|
|
fi
|
|
log "health: $url -> 200"
|
|
|
|
# The container also publishes on the LAN address because that is what Caddy
|
|
# on .35 proxies to. A loopback-only success would be a site nobody can reach.
|
|
local lan_url lan_code
|
|
lan_url="http://${LAN_IP}:${PORT}/healthz"
|
|
lan_code=$(curl -s --max-time 3 -o /dev/null -w '%{http_code}' "$lan_url" 2>/dev/null || true)
|
|
if [[ "$lan_code" != "200" ]]; then
|
|
log "health: $lan_url -> HTTP ${lan_code:-none} (LAN bind missing?)"
|
|
return 1
|
|
fi
|
|
log "health: $lan_url -> 200"
|
|
return 0
|
|
}
|
|
|
|
wait_healthy() {
|
|
local i
|
|
for ((i = 1; i <= HEALTH_TIMEOUT; i++)); do
|
|
if check_health; then return 0; fi
|
|
sleep 1
|
|
done
|
|
return 1
|
|
}
|
|
|
|
container_state() {
|
|
docker inspect -f 'name={{.Name}} status={{.State.Status}} mem={{.HostConfig.Memory}} memswap={{.HostConfig.MemorySwap}} ports={{json .HostConfig.PortBindings}}' \
|
|
"${APP_NAME}-app" 2>/dev/null || echo "container ${APP_NAME}-app: absent"
|
|
}
|
|
|
|
# ---------------------------------------------------------------- build + test
|
|
discover_server() {
|
|
# Contract (deploy/README.md): the app build produces one server binary at
|
|
# bin/server or dist/server. `make build` may also emit dist/ for static assets.
|
|
local d
|
|
for d in "$1/bin/server" "$1/dist/server" "$1/server"; do
|
|
if [[ -f "$d" && -x "$d" ]]; then echo "$d"; return 0; fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
build_and_test() {
|
|
local src="$1" out="$2"
|
|
local gobin=""
|
|
log "build: source $src"
|
|
mkdir -p "$out/bin"
|
|
cd "$src"
|
|
|
|
gobin=$(find_go || true)
|
|
if [[ -z "$gobin" ]]; then
|
|
log "build: no go toolchain found (set GO=/path/to/go)"
|
|
else
|
|
export PATH="$(dirname "$gobin"):$PATH"
|
|
log "build: go $("$gobin" version)"
|
|
fi
|
|
|
|
if [[ -f Makefile ]] && grep -qE '^build:' Makefile; then
|
|
log "build: make build"
|
|
make build
|
|
else
|
|
[[ -n "$gobin" ]] || die "no Makefile 'build' target and no go toolchain"
|
|
log "build: go build ./cmd/server"
|
|
CGO_ENABLED=0 "$gobin" build -trimpath -buildvcs=false -o "$out/bin/server" ./cmd/server
|
|
fi
|
|
|
|
# Tests fail fast, before anything is copied or flipped.
|
|
if [[ -f Makefile ]] && grep -qE '^test:' Makefile; then
|
|
log "test: make test"
|
|
make test
|
|
elif [[ -n "$gobin" ]]; then
|
|
log "test: go test ./..."
|
|
"$gobin" test ./...
|
|
fi
|
|
|
|
# Collect the binary, wherever the app's build put it.
|
|
local bin
|
|
bin=$(discover_server "$src" || true)
|
|
if [[ -z "$bin" ]]; then
|
|
bin=$(discover_server "$out" || true)
|
|
fi
|
|
[[ -n "$bin" ]] || die "no server binary found (looked for bin/server, dist/server, server)"
|
|
if [[ "$(readlink -f "$bin")" != "$(readlink -f "$out/bin/server")" ]]; then
|
|
cp -f "$bin" "$out/bin/server"
|
|
fi
|
|
chmod +x "$out/bin/server"
|
|
log "build: binary $(stat -c %s "$out/bin/server") bytes, sha256 $(sha256sum "$out/bin/server" | cut -c1-16)"
|
|
|
|
# Static assets, if the build emits them.
|
|
if [[ -d "$src/dist" ]]; then
|
|
cp -a "$src/dist/." "$out/dist/" 2>/dev/null || { mkdir -p "$out/dist"; cp -a "$src/dist/." "$out/dist/"; }
|
|
log "build: copied dist/ ($(find "$out/dist" -type f | wc -l) files)"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------- deploy
|
|
do_deploy() {
|
|
local src="$1"
|
|
local ts sha branch release prev_target
|
|
ts=$(date -u +%Y%m%dT%H%M%SZ)
|
|
sha=$(git -C "$src" rev-parse --short HEAD 2>/dev/null || echo unknown)
|
|
branch=$(git -C "$src" rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown)
|
|
release="$RELEASES/$ts"
|
|
|
|
log "deploy: $APP_NAME $ts (branch $branch, $sha) -> $release"
|
|
mkdir -p "$release"
|
|
|
|
# 1. Build + test. Nothing is touched on failure.
|
|
build_and_test "$src" "$release"
|
|
|
|
# 2. Release payload: binary, compose file, runtime image, manifest.
|
|
cp -f "$src/deploy/docker-compose.yml" "$release/docker-compose.yml"
|
|
cp -f "$src/deploy/Dockerfile" "$release/Dockerfile"
|
|
{
|
|
echo "app=$APP_NAME"
|
|
echo "release=$ts"
|
|
echo "git_sha=$(git -C "$src" rev-parse HEAD 2>/dev/null || echo unknown)"
|
|
echo "git_branch=$branch"
|
|
echo "built_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
echo "built_on=$(hostname)"
|
|
echo "binary_sha256=$(sha256sum "$release/bin/server" | cut -d' ' -f1)"
|
|
} > "$release/manifest.txt"
|
|
printf 'APP_VERSION=%s\nAPP_PORT=%s\nAPP_LAN_IP=%s\n' "$ts" "$PORT" "$LAN_IP" > "$release/.env"
|
|
|
|
# 3. Runtime image, tagged with this release.
|
|
log "deploy: docker build resume/app:$ts"
|
|
docker build --quiet -t "resume/app:$ts" \
|
|
--build-arg "APP_VERSION=$ts" -f "$release/Dockerfile" "$release" >>"$LOG" 2>&1 \
|
|
|| die "image build failed"
|
|
|
|
# 4. Remember what is live now, then flip atomically.
|
|
prev_target=""
|
|
if [[ -L "$CURRENT" ]]; then prev_target=$(readlink -f "$CURRENT"); fi
|
|
ln -sfn "$release" "$CURRENT.tmp" && mv -T "$CURRENT.tmp" "$CURRENT" || die "symlink flip failed"
|
|
if [[ -n "$prev_target" && "$prev_target" != "$(readlink -f "$release")" ]]; then
|
|
ln -sfn "$prev_target" "$PREVIOUS.tmp" && mv -T "$PREVIOUS.tmp" "$PREVIOUS"
|
|
log "deploy: previous -> $prev_target"
|
|
fi
|
|
log "deploy: current -> $(readlink -f "$CURRENT")"
|
|
|
|
# 5. Bring up, then gate on health. Anything less than 200 means roll back.
|
|
compose "$release" up -d --remove-orphans >>"$LOG" 2>&1 || log "deploy: compose up returned non-zero"
|
|
if ! wait_healthy; then
|
|
log "deploy: HEALTH GATE FAILED for $ts"
|
|
rollback_internal "$ts"
|
|
exit 1
|
|
fi
|
|
|
|
log "deploy: OK $ts"
|
|
container_state | tee -a "$LOG" >&2
|
|
docker stats --no-stream --format 'rss={{.MemUsage}} pct={{.MemPerc}}' "${APP_NAME}-app" 2>/dev/null | tee -a "$LOG" >&2 || true
|
|
}
|
|
|
|
rollback_internal() {
|
|
# Used by a failed deploy: restore the previous release, restart it, verify.
|
|
local failed="$1" target
|
|
if [[ ! -L "$PREVIOUS" ]]; then
|
|
log "rollback: no previous release to restore; stopping ${APP_NAME}-app"
|
|
compose "$CURRENT" down >>"$LOG" 2>&1 || true
|
|
return 1
|
|
fi
|
|
target=$(readlink -f "$PREVIOUS")
|
|
log "rollback: $failed failed -> restoring $target"
|
|
ln -sfn "$target" "$CURRENT.tmp" && mv -T "$CURRENT.tmp" "$CURRENT"
|
|
compose "$target" up -d --remove-orphans >>"$LOG" 2>&1 || true
|
|
if wait_healthy; then
|
|
log "rollback: previous release healthy ($(readlink -f "$CURRENT"))"
|
|
else
|
|
log "rollback: previous release ALSO unhealthy — manual attention required"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
do_rollback() {
|
|
local target
|
|
if [[ ! -L "$PREVIOUS" ]]; then die "no previous release at $PREVIOUS — nothing to roll back to"; fi
|
|
target=$(readlink -f "$PREVIOUS")
|
|
[[ -d "$target" ]] || die "previous release target missing: $target"
|
|
log "rollback: current=$( [[ -L "$CURRENT" ]] && readlink -f "$CURRENT" || echo none) -> $target"
|
|
|
|
# Keep the release we are leaving, so the rollback is itself reversible.
|
|
local leaving=""
|
|
if [[ -L "$CURRENT" ]]; then leaving=$(readlink -f "$CURRENT"); fi
|
|
ln -sfn "$target" "$CURRENT.tmp" && mv -T "$CURRENT.tmp" "$CURRENT" || die "symlink flip failed"
|
|
if [[ -n "$leaving" && "$leaving" != "$target" ]]; then
|
|
ln -sfn "$leaving" "$PREVIOUS.tmp" && mv -T "$PREVIOUS.tmp" "$PREVIOUS"
|
|
fi
|
|
|
|
compose "$target" up -d --remove-orphans >>"$LOG" 2>&1 || log "rollback: compose up returned non-zero"
|
|
if wait_healthy; then
|
|
log "rollback: OK, current -> $target"
|
|
container_state | tee -a "$LOG" >&2
|
|
exit 0
|
|
fi
|
|
log "rollback: FAILED, $target is not healthy"
|
|
exit 1
|
|
}
|
|
|
|
mkdir -p "$ROOT"
|
|
log "=== ${APP_NAME} deploy.sh start (pid $$) ==="
|
|
if (( DO_ROLLBACK == 1 )); then
|
|
do_rollback
|
|
else
|
|
do_deploy "$SRC_DIR"
|
|
fi
|