diff --git a/deploy/deploy.sh b/deploy/deploy.sh new file mode 100755 index 0000000..60f75c4 --- /dev/null +++ b/deploy/deploy.sh @@ -0,0 +1,388 @@ +#!/usr/bin/env bash +# deploy.sh — release, deploy and roll back the Mode A site (docs/phase-3-deploy.md §3.1). +# +# ./deploy/deploy.sh build, test, stage a release, flip `current`, +# start the container, gate on /healthz +# ./deploy/deploy.sh --rollback put the previous release back, in one command +# ./deploy/deploy.sh --status what is live, what is previous, is it healthy +# ./deploy/deploy.sh --release-dir D deploy an already-built release directory +# (copied into releases/ first; used by the +# rollback rehearsal below) +# +# Layout on the runtime host — the release directory, not the image tag, is the +# unit that runs, so a rollback cannot publish bytes that were never tested: +# +# /home/sam/deployment/resume/ +# releases/-/{bin/resume-server,dist/,release.json} +# current -> releases/<…> what the container is running +# previous -> releases/<…> what --rollback restores +# +# A deploy is: build → test → stage → flip `current` → build the image from that +# release directory → `compose up -d` → `curl /healthz` until ready. Any failure +# after the flip puts `current` and the container back on the previous release +# and exits non-zero. No step is a model decision (AGENTS.md §4.6); this script +# is the only thing that publishes. +# +# Rollback rehearsal (Definition of Done: "a deliberate bad build proves rollback"): +# +# good=$(readlink ~/deployment/resume/current) +# bad=$(mktemp -d) && mkdir -p "$bad/bin" "$bad/dist" +# printf '#!/bin/sh\nexit 1\n' >"$bad/bin/resume-server" # binary that cannot start +# chmod +x "$bad/bin/resume-server" +# cp -a ~/deployment/resume/"$good"/dist/. "$bad/dist/" # valid pages +# ./deploy/deploy.sh --release-dir "$bad" # expect exit 1 +# readlink ~/deployment/resume/current # still $good +# +# Run it on the runtime host (.13) from a checkout of this repository. It never +# reads content/ and never calls a model. + +set -Eeuo pipefail + +APP=ai-resume +PORT=8096 +HEALTH_PATH=/healthz + +ROOT=${RESUME_DEPLOY_ROOT:-/home/sam/deployment/resume} +RELEASES=$ROOT/releases +CURRENT=$ROOT/current +PREVIOUS=$ROOT/previous + +KEEP=${RESUME_KEEP_RELEASES:-5} # release directories kept on disk +HEALTH_TRIES=${RESUME_HEALTH_TRIES:-40} # seconds to wait for /healthz + +REPO=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +COMPOSE=$REPO/deploy/compose.yml +DOCKERFILE=$REPO/deploy/Dockerfile + +MODE=deploy +RELEASE_DIR_OVERRIDE= + +log() { printf '%s %s +' "$(date -u +%H:%M:%SZ)" "$*" >&2; } +warn() { printf '%s WARN %s +' "$(date -u +%H:%M:%SZ)" "$*" >&2; } +die() { printf '%s FAIL %s +' "$(date -u +%H:%M:%SZ)" "$*" >&2; exit 1; } + +on_err() { + local rc=$? line=$1 + printf '%s FAIL line %s exited %s +' "$(date -u +%H:%M:%SZ)" "$line" "$rc" >&2 + exit "$rc" +} +trap 'on_err $LINENO' ERR + +usage() { + sed -n '2,13p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit "${1:-0}" +} + +while [ $# -gt 0 ]; do + case $1 in + --rollback) MODE=rollback ;; + --status) MODE=status ;; + --release-dir) shift; RELEASE_DIR_OVERRIDE=${1:-} ;; + --keep) shift; KEEP=${1:-5} ;; + -h|--help) usage 0 ;; + *) printf 'unknown argument: %s +' "$1" >&2; usage 1 ;; + esac + shift +done + +# ---------------------------------------------------------------- helpers --- + +compose() { docker compose -f "$COMPOSE" -p "$APP" "$@"; } + +current_target() { readlink -- "$CURRENT" 2>/dev/null || true; } +previous_target() { readlink -- "$PREVIOUS" 2>/dev/null || true; } + +# flip — replace the symlink atomically. A reader sees +# either the old release or the new one, never a missing link. +flip() { + ln -sfn "$1" "$CURRENT.tmp" + mv -Tf "$CURRENT.tmp" "$CURRENT" +} + +release_name_of() { basename "$1"; } + +# The tag deploy/compose.yml runs. Releases are distinguished by content, not by +# tag, so compose.yml keeps its fixed image name. +image_name() { printf '%s:local' "$APP"; } + +# The port must be ours, or free. 8095 belongs to offsite.lab. +check_port() { + local listening + listening=$(ss -ltnH "sport = :$PORT" 2>/dev/null || true) + if [ -n "$listening" ]; then + if [ -z "$(compose ps -q resume 2>/dev/null)" ]; then + printf '%s +' "$listening" >&2 + die "port $PORT is held by something that is not $APP; pick the next free port and record it in reference/stack.md" + fi + log "port $PORT is held by our own container (expected on a redeploy)" + fi +} + +check_tools() { + command -v docker >/dev/null || die "docker is not on PATH" + docker compose version >/dev/null 2>&1 || die "docker compose v2 is not available" + command -v curl >/dev/null || die "curl is not on PATH" + command -v ss >/dev/null || die "ss (iproute2) is not on PATH" + mkdir -p "$RELEASES" +} + +memory_report() { + local avail + avail=$(awk '/^MemAvailable:/ {print int($2/1024)}' /proc/meminfo) + log "host memory available: ${avail} MiB — the container cap is 120 MiB; .13 is memory-tight by design" + if [ "${avail:-0}" -lt 512 ]; then + warn "under 512 MiB available; the Go build may push this host into swap" + fi +} + +# --------------------------------------------------------------- build ------ + +build_and_test() { + log "build + test in $REPO" + ( + cd "$REPO" + rm -rf dist + if command -v nix >/dev/null 2>&1; then + nix develop . -c bash -c 'make validate test dist' + elif command -v templ >/dev/null 2>&1 && command -v go >/dev/null 2>&1; then + make validate test dist + else + echo "no toolchain: neither 'nix develop' nor templ+go is available" >&2 + exit 1 + fi + ) + [ -f "$REPO/bin/resume-server" ] || die "build produced no bin/resume-server" + [ -f "$REPO/dist/index.html" ] || die "build produced no dist/index.html" + log "build ok: $(jq -r '"\(.pages) pages, \(.documents) documents, \(.fragments) fragments"' "$REPO/dist/build.json" 2>/dev/null || echo 'dist/build.json unreadable')" +} + +# -------------------------------------------------------------- release ----- + +write_manifest() { + local dir=$1 sha dirty + sha=$(git -C "$REPO" rev-parse HEAD 2>/dev/null || echo unknown) + dirty=false + git -C "$REPO" diff --quiet 2>/dev/null || dirty=true + cat >"$dir/release.json" </dev/null || echo 0), + "documents": $(jq -r '.documents // 0' "$REPO/dist/build.json" 2>/dev/null || echo 0) +} +JSON +} + +stage_release() { + local name dest sha + sha=$(git -C "$REPO" rev-parse HEAD 2>/dev/null || echo unknown) + name=$(date -u +%Y%m%dT%H%M%SZ)-${sha:0:7} + dest=$RELEASES/$name + + mkdir -p "$dest/bin" "$dest/dist" + install -m 0755 "$REPO/bin/resume-server" "$dest/bin/resume-server" + cp -a "$REPO/dist/." "$dest/dist/" + write_manifest "$dest" + printf '%s +' "$dest" +} + +# Copy an externally built release into releases/ so that every deployable +# directory lives under exactly one root and `current` never dangles. +adopt_release() { + local src dest + src=$(cd "$1" && pwd) + [ -f "$src/bin/resume-server" ] || die "$src has no bin/resume-server" + [ -f "$src/dist/index.html" ] || die "$src has no dist/index.html" + dest=$RELEASES/$(date -u +%Y%m%dT%H%M%SZ)-manual + mkdir -p "$dest" + cp -a "$src/." "$dest/" + write_manifest "$dest" + printf '%s +' "$dest" +} + +# ---------------------------------------------------------------- image ----- + +build_image() { + log "docker build from $1" + docker build --file "$DOCKERFILE" --tag "$(image_name)" "$1" 1>&2 +} + +# ------------------------------------------------------------------ run ----- + +start() { + log "docker compose up -d ($1)" + compose up -d --no-build --force-recreate 1>&2 +} + +wait_healthy() { + local i body + for i in $(seq 1 "$HEALTH_TRIES"); do + if body=$(curl -fsS --max-time 3 "http://127.0.0.1:$PORT$HEALTH_PATH" 2>/dev/null); then + if printf '%s' "$body" | grep -Eq '"status"[[:space:]]*:[[:space:]]*"ok"'; then + log "health gate passed after ${i}s" + return 0 + fi + fi + sleep 1 + done + warn "health gate failed after ${HEALTH_TRIES}s; last response:" + curl -sS --max-time 3 "http://127.0.0.1:$PORT$HEALTH_PATH" >&2 || true + compose logs --tail 40 resume >&2 || true + return 1 +} + +# Definition-of-Done evidence, checked on every deploy instead of by hand: +# mem_limit == memswap_limit == the value declared in compose.yml, and the port +# published on 127.0.0.1 only. +assert_shape() { + local cid mem swap ip declared + declared=$(awk '/^[[:space:]]*mem_limit:/ {gsub(/[^0-9]/, "", $2); print $2}' "$COMPOSE") + cid=$(compose ps -q resume 2>/dev/null || true) + [ -n "$cid" ] || die "no container for service 'resume'" + mem=$(docker inspect -f '{{.HostConfig.Memory}}' "$cid") + swap=$(docker inspect -f '{{.HostConfig.MemorySwap}}' "$cid") + [ "$mem" = "$swap" ] || die "mem_limit ($mem) != memswap_limit ($swap)" + [ "$mem" = "$((declared * 1024 * 1024))" ] || die "mem_limit is $mem, expected ${declared} MiB" + ip=$(docker inspect -f '{{range $p, $b := .HostConfig.PortBindings}}{{range $b}}{{.HostIp}} {{end}}{{end}}' "$cid" | xargs) + [ "$ip" = "127.0.0.1" ] || die "published on '${ip}', expected 127.0.0.1 only" + log "shape ok: mem_limit == memswap_limit == ${declared} MiB, published on 127.0.0.1:$PORT" +} + +deploy_release() { + local dir old name + dir=$1 + name=$(release_name_of "$dir") + [ -d "$dir" ] || die "release directory does not exist: $dir" + [ -f "$dir/bin/resume-server" ] || die "$dir has no bin/resume-server" + [ -f "$dir/dist/index.html" ] || die "$dir has no dist/index.html — a build that cannot serve a landing page is not a release" + + build_image "$dir" + old=$(current_target) + + # The flip happens before the container change, so `current` always describes + # what is being started. The release it replaced becomes `previous`. + flip "releases/$name" + if [ -n "$old" ] && [ "$old" != "releases/$name" ]; then + ln -sfn "$old" "$PREVIOUS" + log "previous -> $old" + fi + + if start "$name" && wait_healthy && assert_shape; then + printf '%s +' "$(date -u +%Y-%m-%dT%H:%M:%SZ) deploy releases/$name ok" >>"$ROOT/deploy.log" + log "deployed $name" + prune + return 0 + fi + + if [ -z "$old" ] || [ "$old" = "releases/$name" ]; then + warn "first deploy failed and there is no previous release; leaving the container down" + compose down >&2 || true + rm -f "$CURRENT" + return 1 + fi + + warn "deploy of $name failed; restoring $old" + flip "$old" + build_image "$ROOT/$old" || die "could not rebuild the previous image; container state unknown" + if start "$old" && wait_healthy && assert_shape; then + printf '%s +' "$(date -u +%Y-%m-%dT%H:%M:%SZ) rolled back to $old after failed deploy of releases/$name" >>"$ROOT/deploy.log" + die "deploy of releases/$name failed; rolled back to $old (the failed release is kept for inspection)" + fi + die "deploy of releases/$name failed AND rollback to $old failed — container state is unknown; run '$0 --status' and read $ROOT/deploy.log" +} + +do_rollback() { + local cur prev + cur=$(current_target) + prev=$(previous_target) + [ -n "$prev" ] || die "no previous release recorded; nothing to roll back to" + [ "$prev" != "$cur" ] || die "previous and current are the same release ($cur); nothing to roll back" + [ -d "$ROOT/$prev" ] || die "previous release directory is missing: $ROOT/$prev" + + log "rolling back: $cur -> $prev" + build_image "$ROOT/$prev" + flip "$prev" + if start "$prev" && wait_healthy && assert_shape; then + [ -n "$cur" ] && ln -sfn "$cur" "$PREVIOUS" + printf '%s +' "$(date -u +%Y-%m-%dT%H:%M:%SZ) rollback $cur -> $prev" >>"$ROOT/deploy.log" + log "rolled back to $prev; $cur is now previous" + return 0 + fi + + warn "rollback to $prev failed; returning to $cur" + flip "$cur" + build_image "$ROOT/$cur" || die "could not rebuild $cur; container state unknown" + start "$cur" && wait_healthy || die "return to $cur failed; container state unknown" + die "rollback to $prev failed; $cur was restored" +} + +# Keep the newest KEEP releases plus current and previous, nothing else. +prune() { + local cur prev name dir + cur=$(current_target); prev=$(previous_target) + for dir in $(ls -1dt "$RELEASES"/*/ 2>/dev/null | tail -n +"$((KEEP + 1))"); do + name="releases/$(basename "$dir")" + if [ "$name" = "$cur" ] || [ "$name" = "$prev" ]; then continue; fi + rm -rf -- "$dir" + log "pruned $name" + done +} + +do_status() { + local cur prev body + cur=$(current_target); prev=$(previous_target) + body=$(curl -fsS --max-time 3 "http://127.0.0.1:$PORT$HEALTH_PATH" 2>/dev/null || true) + printf 'root %s +' "$ROOT" + printf 'current %s +' "${cur:-}" + printf 'previous %s +' "${prev:-}" + if [ -n "$body" ]; then + printf 'health %s +' "$(printf '%s' "$body" | jq -c '{status, pages: .static.pages, search: .search.ready}' 2>/dev/null || printf '%s' "$body")" + else + printf 'health unreachable +' + fi + compose ps || true +} + +# ------------------------------------------------------------------ main ---- + +case $MODE in +status) + do_status + ;; +rollback) + check_tools + do_rollback + ;; +deploy) + check_tools + check_port + memory_report + if [ -n "$RELEASE_DIR_OVERRIDE" ]; then + deploy_release "$(adopt_release "$RELEASE_DIR_OVERRIDE")" + else + build_and_test + deploy_release "$(stage_release)" + fi + log "live: $(current_target)" + ;; +esac