ai-resume/deploy/deploy.sh

#!/usr/bin/env bash
# deploy.sh — release, deploy and roll back the Mode A site (docs/phase-3-deploy.md §3.1).
#
#   ./deploy/deploy.sh                  build, test, stage a release, flip `current`,
#                                       start the container, gate on /healthz
#   ./deploy/deploy.sh --rollback       put the previous release back, in one command
#   ./deploy/deploy.sh --status         what is live, what is previous, is it healthy
#   ./deploy/deploy.sh --release-dir D  deploy an already-built release directory
#                                       (copied into releases/ first; used by the
#                                       rollback rehearsal below)
#
# Layout on the runtime host — the release directory, not the image tag, is the
# unit that runs, so a rollback cannot publish bytes that were never tested:
#
#   /home/sam/deployment/resume/
#     releases/<utc-timestamp>-<git-sha>/{bin/resume-server,dist/,release.json}
#     current   -> releases/<…>   what the container is running
#     previous  -> releases/<…>   what --rollback restores
#
# A deploy is: build → test → stage → flip `current` → build the image from that
# release directory → `compose up -d` → `curl /healthz` until ready. Any failure
# after the flip puts `current` and the container back on the previous release
# and exits non-zero. No step is a model decision (AGENTS.md §4.6); this script
# is the only thing that publishes.
#
# Rollback rehearsal (Definition of Done: "a deliberate bad build proves rollback"):
#
#   good=$(readlink ~/deployment/resume/current)
#   bad=$(mktemp -d) && mkdir -p "$bad/bin" "$bad/dist"
#   printf '#!/bin/sh\nexit 1\n' >"$bad/bin/resume-server"   # binary that cannot start
#   chmod +x "$bad/bin/resume-server"
#   cp -a ~/deployment/resume/"$good"/dist/. "$bad/dist/"   # valid pages
#   ./deploy/deploy.sh --release-dir "$bad"                 # expect exit 1
#   readlink ~/deployment/resume/current                    # still $good
#
# Run it on the runtime host (.13) from a checkout of this repository. It never
# reads content/ and never calls a model.

set -Eeuo pipefail

APP=ai-resume                  # image tag (fixed by Phase 3 §3.2)
PROJECT=ai-resume-site         # compose project; distinct from the older
                               # /home/sam/deployment/ai-resume project on .13
PORT=8096
HEALTH_PATH=/healthz

ROOT=${RESUME_DEPLOY_ROOT:-/home/sam/deployment/resume}
RELEASES=$ROOT/releases
CURRENT=$ROOT/current
PREVIOUS=$ROOT/previous

KEEP=${RESUME_KEEP_RELEASES:-5}            # release directories kept on disk
HEALTH_TRIES=${RESUME_HEALTH_TRIES:-40}    # seconds to wait for /healthz

REPO=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)
COMPOSE=$REPO/deploy/compose.yml
DOCKERFILE=$REPO/deploy/Dockerfile

MODE=deploy
RELEASE_DIR_OVERRIDE=

log()  { printf '%s  %s
' "$(date -u +%H:%M:%SZ)" "$*" >&2; }
warn() { printf '%s  WARN  %s
' "$(date -u +%H:%M:%SZ)" "$*" >&2; }
die()  { printf '%s  FAIL  %s
' "$(date -u +%H:%M:%SZ)" "$*" >&2; exit 1; }

on_err() {
	local rc=$? line=$1
	printf '%s  FAIL  line %s exited %s
' "$(date -u +%H:%M:%SZ)" "$line" "$rc" >&2
	exit "$rc"
}
trap 'on_err $LINENO' ERR

usage() {
	sed -n '2,13p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
	exit "${1:-0}"
}

while [ $# -gt 0 ]; do
	case $1 in
	--rollback)    MODE=rollback ;;
	--status)      MODE=status ;;
	--release-dir) shift; RELEASE_DIR_OVERRIDE=${1:-} ;;
	--keep)        shift; KEEP=${1:-5} ;;
	-h|--help)     usage 0 ;;
	*)             printf 'unknown argument: %s
' "$1" >&2; usage 1 ;;
	esac
	shift
done

# ---------------------------------------------------------------- helpers ---

compose() { docker compose -f "$COMPOSE" -p "$PROJECT" "$@"; }

current_target()  { readlink -- "$CURRENT" 2>/dev/null || true; }
previous_target() { readlink -- "$PREVIOUS" 2>/dev/null || true; }

# flip <relative-path-under-ROOT> — replace the symlink atomically. A reader sees
# either the old release or the new one, never a missing link.
flip() {
	ln -sfn "$1" "$CURRENT.tmp"
	mv -Tf "$CURRENT.tmp" "$CURRENT"
}

release_name_of() { basename "$1"; }

# The tag deploy/compose.yml runs. Releases are distinguished by content, not by
# tag, so compose.yml keeps its fixed image name.
image_name() { printf '%s:local' "$APP"; }

# The port must be ours, or free. 8095 belongs to offsite.lab.
check_port() {
	local listening
	listening=$(ss -ltnH "sport = :$PORT" 2>/dev/null || true)
	if [ -n "$listening" ]; then
		if [ -z "$(compose ps -q resume 2>/dev/null)" ]; then
			printf '%s
' "$listening" >&2
			die "port $PORT is held by something that is not $APP; pick the next free port and record it in reference/stack.md"
		fi
		log "port $PORT is held by our own container (expected on a redeploy)"
	fi
}

check_tools() {
	command -v docker >/dev/null || die "docker is not on PATH"
	docker compose version >/dev/null 2>&1 || die "docker compose v2 is not available"
	command -v curl >/dev/null || die "curl is not on PATH"
	command -v ss >/dev/null || die "ss (iproute2) is not on PATH"
	mkdir -p "$RELEASES"
}

memory_report() {
	local avail
	avail=$(awk '/^MemAvailable:/ {print int($2/1024)}' /proc/meminfo)
	log "host memory available: ${avail} MiB — the container cap is 120 MiB; .13 is memory-tight by design"
	if [ "${avail:-0}" -lt 512 ]; then
		warn "under 512 MiB available; the Go build may push this host into swap"
	fi
}

# --------------------------------------------------------------- build ------

build_and_test() {
	log "build + test in $REPO"
	(
		cd "$REPO"
		rm -rf dist
		if command -v nix >/dev/null 2>&1; then
			nix develop . -c bash -c 'make validate test dist'
		elif command -v templ >/dev/null 2>&1 && command -v go >/dev/null 2>&1; then
			make validate test dist
		else
			echo "no toolchain: neither 'nix develop' nor templ+go is available" >&2
			exit 1
		fi
	)
	[ -f "$REPO/bin/resume-server" ] || die "build produced no bin/resume-server"
	[ -f "$REPO/dist/index.html" ]   || die "build produced no dist/index.html"
	log "build ok: $(jq -r '"\(.pages) pages, \(.documents) documents, \(.fragments) fragments"' "$REPO/dist/build.json" 2>/dev/null || echo 'dist/build.json unreadable')"
}

# -------------------------------------------------------------- release -----

write_manifest() {
	local dir=$1 sha dirty
	sha=$(git -C "$REPO" rev-parse HEAD 2>/dev/null || echo unknown)
	dirty=false
	git -C "$REPO" diff --quiet 2>/dev/null || dirty=true
	cat >"$dir/release.json" <<JSON
{
  "release": "$(basename "$dir")",
  "image": "$(image_name)",
  "git_sha": "$sha",
  "git_dirty": $dirty,
  "built_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
  "host": "$(hostname)",
  "pages": $(jq -r '.pages // 0' "$REPO/dist/build.json" 2>/dev/null || echo 0),
  "documents": $(jq -r '.documents // 0' "$REPO/dist/build.json" 2>/dev/null || echo 0)
}
JSON
}

stage_release() {
	local name dest sha
	sha=$(git -C "$REPO" rev-parse HEAD 2>/dev/null || echo unknown)
	name=$(date -u +%Y%m%dT%H%M%SZ)-${sha:0:7}
	dest=$RELEASES/$name

	mkdir -p "$dest/bin" "$dest/dist"
	install -m 0755 "$REPO/bin/resume-server" "$dest/bin/resume-server"
	cp -a "$REPO/dist/." "$dest/dist/"
	write_manifest "$dest"
	printf '%s
' "$dest"
}

# Copy an externally built release into releases/ so that every deployable
# directory lives under exactly one root and `current` never dangles.
adopt_release() {
	local src dest
	src=$(cd "$1" && pwd)
	[ -f "$src/bin/resume-server" ] || die "$src has no bin/resume-server"
	[ -f "$src/dist/index.html" ]   || die "$src has no dist/index.html"
	dest=$RELEASES/$(date -u +%Y%m%dT%H%M%SZ)-manual
	mkdir -p "$dest"
	cp -a "$src/." "$dest/"
	write_manifest "$dest"
	printf '%s
' "$dest"
}

# ---------------------------------------------------------------- image -----

build_image() {
	log "docker build from $1"
	docker build --file "$DOCKERFILE" --tag "$(image_name)" "$1" 1>&2
}

# ------------------------------------------------------------------ run -----

start() {
	log "docker compose up -d ($1)"
	compose up -d --no-build --force-recreate 1>&2
}

wait_healthy() {
	local i body
	for i in $(seq 1 "$HEALTH_TRIES"); do
		if body=$(curl -fsS --max-time 3 "http://127.0.0.1:$PORT$HEALTH_PATH" 2>/dev/null); then
			if printf '%s' "$body" | grep -Eq '"status"[[:space:]]*:[[:space:]]*"ok"'; then
				log "health gate passed after ${i}s"
				return 0
			fi
		fi
		sleep 1
	done
	warn "health gate failed after ${HEALTH_TRIES}s; last response:"
	curl -sS --max-time 3 "http://127.0.0.1:$PORT$HEALTH_PATH" >&2 || true
	compose logs --tail 40 resume >&2 || true
	return 1
}

# Definition-of-Done evidence, checked on every deploy instead of by hand:
# mem_limit == memswap_limit == the value declared in compose.yml, and the port
# published on 127.0.0.1 only.
assert_shape() {
	local cid mem swap ip declared
	declared=$(awk '/^[[:space:]]*mem_limit:/ {gsub(/[^0-9]/, "", $2); print $2}' "$COMPOSE")
	cid=$(compose ps -q resume 2>/dev/null || true)
	[ -n "$cid" ] || die "no container for service 'resume'"
	mem=$(docker inspect -f '{{.HostConfig.Memory}}' "$cid")
	swap=$(docker inspect -f '{{.HostConfig.MemorySwap}}' "$cid")
	[ "$mem" = "$swap" ] || die "mem_limit ($mem) != memswap_limit ($swap)"
	[ "$mem" = "$((declared * 1024 * 1024))" ] || die "mem_limit is $mem, expected ${declared} MiB"
	ip=$(docker inspect -f '{{range $p, $b := .HostConfig.PortBindings}}{{range $b}}{{.HostIp}} {{end}}{{end}}' "$cid" | xargs)
	[ "$ip" = "127.0.0.1" ] || die "published on '${ip}', expected 127.0.0.1 only"
	log "shape ok: mem_limit == memswap_limit == ${declared} MiB, published on 127.0.0.1:$PORT"
}

deploy_release() {
	local dir old name
	dir=$1
	name=$(release_name_of "$dir")
	[ -d "$dir" ] || die "release directory does not exist: $dir"
	[ -f "$dir/bin/resume-server" ] || die "$dir has no bin/resume-server"
	[ -f "$dir/dist/index.html" ]   || die "$dir has no dist/index.html — a build that cannot serve a landing page is not a release"

	build_image "$dir"
	old=$(current_target)

	# The flip happens before the container change, so `current` always describes
	# what is being started. The release it replaced becomes `previous`.
	flip "releases/$name"
	if [ -n "$old" ] && [ "$old" != "releases/$name" ]; then
		ln -sfn "$old" "$PREVIOUS"
		log "previous -> $old"
	fi

	if start "$name" && wait_healthy && assert_shape; then
		printf '%s
' "$(date -u +%Y-%m-%dT%H:%M:%SZ)  deploy releases/$name ok" >>"$ROOT/deploy.log"
		log "deployed $name"
		prune
		return 0
	fi

	if [ -z "$old" ] || [ "$old" = "releases/$name" ]; then
		warn "first deploy failed and there is no previous release; leaving the container down"
		compose down >&2 || true
		rm -f "$CURRENT"
		return 1
	fi

	warn "deploy of $name failed; restoring $old"
	flip "$old"
	build_image "$ROOT/$old" || die "could not rebuild the previous image; container state unknown"
	if start "$old" && wait_healthy && assert_shape; then
		printf '%s
' "$(date -u +%Y-%m-%dT%H:%M:%SZ)  rolled back to $old after failed deploy of releases/$name" >>"$ROOT/deploy.log"
		die "deploy of releases/$name failed; rolled back to $old (the failed release is kept for inspection)"
	fi
	die "deploy of releases/$name failed AND rollback to $old failed — container state is unknown; run '$0 --status' and read $ROOT/deploy.log"
}

do_rollback() {
	local cur prev
	cur=$(current_target)
	prev=$(previous_target)
	[ -n "$prev" ] || die "no previous release recorded; nothing to roll back to"
	[ "$prev" != "$cur" ] || die "previous and current are the same release ($cur); nothing to roll back"
	[ -d "$ROOT/$prev" ] || die "previous release directory is missing: $ROOT/$prev"

	log "rolling back: $cur -> $prev"
	build_image "$ROOT/$prev"
	flip "$prev"
	if start "$prev" && wait_healthy && assert_shape; then
		[ -n "$cur" ] && ln -sfn "$cur" "$PREVIOUS"
		printf '%s
' "$(date -u +%Y-%m-%dT%H:%M:%SZ)  rollback $cur -> $prev" >>"$ROOT/deploy.log"
		log "rolled back to $prev; $cur is now previous"
		return 0
	fi

	warn "rollback to $prev failed; returning to $cur"
	flip "$cur"
	build_image "$ROOT/$cur" || die "could not rebuild $cur; container state unknown"
	start "$cur" && wait_healthy || die "return to $cur failed; container state unknown"
	die "rollback to $prev failed; $cur was restored"
}

# Keep the newest KEEP releases plus current and previous, nothing else.
prune() {
	local cur prev name dir
	cur=$(current_target); prev=$(previous_target)
	for dir in $(ls -1dt "$RELEASES"/*/ 2>/dev/null | tail -n +"$((KEEP + 1))"); do
		name="releases/$(basename "$dir")"
		if [ "$name" = "$cur" ] || [ "$name" = "$prev" ]; then continue; fi
		rm -rf -- "$dir"
		log "pruned $name"
	done
}

do_status() {
	local cur prev body
	cur=$(current_target); prev=$(previous_target)
	body=$(curl -fsS --max-time 3 "http://127.0.0.1:$PORT$HEALTH_PATH" 2>/dev/null || true)
	printf 'root      %s
' "$ROOT"
	printf 'current   %s
' "${cur:-<none>}"
	printf 'previous  %s
' "${prev:-<none>}"
	if [ -n "$body" ]; then
		printf 'health    %s
' "$(printf '%s' "$body" | jq -c '{status, pages: .static.pages, search: .search.ready}' 2>/dev/null || printf '%s' "$body")"
	else
		printf 'health    unreachable
'
	fi
	compose ps || true
}

# ------------------------------------------------------------------ main ----

case $MODE in
status)
	do_status
	;;
rollback)
	check_tools
	do_rollback
	;;
deploy)
	check_tools
	check_port
	memory_report
	if [ -n "$RELEASE_DIR_OVERRIDE" ]; then
		deploy_release "$(adopt_release "$RELEASE_DIR_OVERRIDE")"
	else
		build_and_test
		deploy_release "$(stage_release)"
	fi
	log "live: $(current_target)"
	;;
esac