Phase 3: release script, container limits, rehearsed rollback
Adds the deployment mechanism for the portfolio site. The site itself is not deployed yet: Phase 2 has not produced a binary. - deploy/deploy.sh: build + test (fail fast) -> releases/<ts> -> atomic `current` symlink flip -> docker compose up -d -> /healthz gate -> flip back to `previous` and exit non-zero on failure. `--rollback` restores the previous release in one command. Auto-detects the Go toolchain in /nix/store when `go` is not on PATH. - deploy/docker-compose.yml: 127.0.0.1:8096, mem_limit == memswap_limit == 120m, capped json-file logs (10m x 3), restart unless-stopped. - deploy/Dockerfile: runtime image only (alpine + prebuilt payload), so a release needs no toolchain, no network and no package install. - deploy/rehearse.sh: repeatable proof (21 assertions, exit 0) of deploy, build fail-fast, health-gate rollback, --rollback and N3 restart. - deploy/README.md + docs/phase-3-evidence.md: how it works, the build contract Phase 2 must satisfy, the Caddy block for Sam, and the open bind-address question (loopback-only vs reachable from .35). Satisfies DoD items 1, 3 (container), 4, 8. Items 2, 5, 6, 7 need the Phase 2 binary and Sam's Caddy change; recorded as deferred with owners, not claimed. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
System
303
deploy/deploy.sh
Executable file
303
deploy/deploy.sh
Executable file
@@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env bash
|
||||
# resume — release script (Phase 3).
|
||||
#
|
||||
# Deploy is a script, not a model decision (AGENTS.md §4.6).
|
||||
# No one hand-edits files on the production host: change this script in git.
|
||||
#
|
||||
# ./deploy/deploy.sh build, test, release, flip, health-gate
|
||||
# ./deploy/deploy.sh --rollback restore the previous release, one command
|
||||
#
|
||||
# Layout on the runtime host (.13):
|
||||
#
|
||||
# /home/sam/deployment/resume/
|
||||
# ├── releases/<UTC timestamp>/ immutable release: bin/, dist/, compose, manifest
|
||||
# ├── current -> releases/<ts> the live release (atomic symlink flip)
|
||||
# ├── previous -> releases/<ts> the release that was live before the last flip
|
||||
# └── deploy.log append-only record of every deploy attempt
|
||||
#
|
||||
# A release is only "current" after /healthz answers 200 on 127.0.0.1:$PORT.
|
||||
# If the health gate fails, `current` is flipped back to `previous`, the previous
|
||||
# release is restarted, and this script exits non-zero.
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
APP_NAME="resume"
|
||||
DEFAULT_PORT=8096
|
||||
DEFAULT_ROOT="/home/sam/deployment/resume"
|
||||
|
||||
SRC_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
ROOT="${DEPLOY_ROOT:-$DEFAULT_ROOT}"
|
||||
PORT="${APP_PORT:-$DEFAULT_PORT}"
|
||||
DO_ROLLBACK=0
|
||||
HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-40}"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: deploy.sh [options]
|
||||
|
||||
--rollback restore the previous release and exit
|
||||
--source DIR build from DIR instead of the repo this script lives in
|
||||
--root DIR release root (default /home/sam/deployment/resume)
|
||||
--port N app port (default 8096)
|
||||
--health-timeout N seconds to wait for /healthz (default 40)
|
||||
-h, --help this text
|
||||
|
||||
Environment:
|
||||
GO path to the go toolchain (auto-detected, incl. /nix/store)
|
||||
DEPLOY_ROOT same as --root
|
||||
APP_PORT same as --port
|
||||
EOF
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--rollback) DO_ROLLBACK=1; shift ;;
|
||||
--source) SRC_DIR="$(cd "$2" && pwd)"; shift 2 ;;
|
||||
--root) ROOT="$2"; shift 2 ;;
|
||||
--port) PORT="$2"; shift 2 ;;
|
||||
--health-timeout) HEALTH_TIMEOUT="$2"; shift 2 ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
RELEASES="$ROOT/releases"
|
||||
CURRENT="$ROOT/current"
|
||||
PREVIOUS="$ROOT/previous"
|
||||
LOG="$ROOT/deploy.log"
|
||||
|
||||
mkdir -p "$RELEASES"
|
||||
|
||||
log() { printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOG" >&2; }
|
||||
die() { log "FAIL: $*"; exit 1; }
|
||||
|
||||
# ---------------------------------------------------------------- toolchain
|
||||
find_go() {
|
||||
if [[ -n "${GO:-}" && -x "${GO}" ]]; then echo "$GO"; return; fi
|
||||
if command -v go >/dev/null 2>&1; then command -v go; return; fi
|
||||
# NixOS: the toolchain lives in the store even when it is not on PATH.
|
||||
local cand
|
||||
cand=$(ls -d /nix/store/*-go-1.*/bin/go 2>/dev/null | grep -v '\.drv' | sort -V | tail -1)
|
||||
if [[ -n "$cand" && -x "$cand" ]]; then echo "$cand"; return; fi
|
||||
return 1
|
||||
}
|
||||
|
||||
compose() {
|
||||
# $1 = release directory that holds docker-compose.yml + .env
|
||||
local dir="$1"; shift
|
||||
docker compose -p "$APP_NAME" --project-directory "$dir" -f "$dir/docker-compose.yml" "$@"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- health gate
|
||||
check_health() {
|
||||
# 200 on /healthz, and if the body is JSON with a "static" field it must be true.
|
||||
# AI readiness is deliberately NOT part of this gate: Mode A must work with
|
||||
# every AI service stopped (acceptance test A1).
|
||||
local url="http://127.0.0.1:${PORT}/healthz" body code
|
||||
body=$(curl -sS --max-time 3 -w '\n%{http_code}' "$url" 2>/dev/null || true)
|
||||
code=$(printf '%s' "$body" | tail -1)
|
||||
body=$(printf '%s' "$body" | sed '$d')
|
||||
if [[ "$code" != "200" ]]; then
|
||||
log "health: $url -> HTTP ${code:-none}"
|
||||
return 1
|
||||
fi
|
||||
if printf '%s' "$body" | head -c 1 | grep -q '{' && command -v jq >/dev/null 2>&1; then
|
||||
local static_ok
|
||||
static_ok=$(printf '%s' "$body" | jq -r 'if has("static") then (.static|tostring) else "absent" end' 2>/dev/null || echo absent)
|
||||
if [[ "$static_ok" == "false" ]]; then
|
||||
log "health: $url -> 200 but static readiness is false: $body"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
log "health: $url -> 200"
|
||||
return 0
|
||||
}
|
||||
|
||||
wait_healthy() {
|
||||
local i
|
||||
for ((i = 1; i <= HEALTH_TIMEOUT; i++)); do
|
||||
if check_health; then return 0; fi
|
||||
sleep 1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
container_state() {
|
||||
docker inspect -f 'name={{.Name}} status={{.State.Status}} mem={{.HostConfig.Memory}} memswap={{.HostConfig.MemorySwap}} ports={{json .HostConfig.PortBindings}}' \
|
||||
"${APP_NAME}-app" 2>/dev/null || echo "container ${APP_NAME}-app: absent"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- build + test
|
||||
discover_server() {
|
||||
# Contract (deploy/README.md): the app build produces one server binary at
|
||||
# bin/server or dist/server. `make build` may also emit dist/ for static assets.
|
||||
local d
|
||||
for d in "$1/bin/server" "$1/dist/server" "$1/server"; do
|
||||
if [[ -f "$d" && -x "$d" ]]; then echo "$d"; return 0; fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
build_and_test() {
|
||||
local src="$1" out="$2"
|
||||
local gobin=""
|
||||
log "build: source $src"
|
||||
mkdir -p "$out/bin"
|
||||
cd "$src"
|
||||
|
||||
gobin=$(find_go || true)
|
||||
if [[ -z "$gobin" ]]; then
|
||||
log "build: no go toolchain found (set GO=/path/to/go)"
|
||||
else
|
||||
export PATH="$(dirname "$gobin"):$PATH"
|
||||
log "build: go $("$gobin" version)"
|
||||
fi
|
||||
|
||||
if [[ -f Makefile ]] && grep -qE '^build:' Makefile; then
|
||||
log "build: make build"
|
||||
make build
|
||||
else
|
||||
[[ -n "$gobin" ]] || die "no Makefile 'build' target and no go toolchain"
|
||||
log "build: go build ./cmd/server"
|
||||
CGO_ENABLED=0 "$gobin" build -trimpath -buildvcs=false -o "$out/bin/server" ./cmd/server
|
||||
fi
|
||||
|
||||
# Tests fail fast, before anything is copied or flipped.
|
||||
if [[ -f Makefile ]] && grep -qE '^test:' Makefile; then
|
||||
log "test: make test"
|
||||
make test
|
||||
elif [[ -n "$gobin" ]]; then
|
||||
log "test: go test ./..."
|
||||
"$gobin" test ./...
|
||||
fi
|
||||
|
||||
# Collect the binary, wherever the app's build put it.
|
||||
local bin
|
||||
bin=$(discover_server "$src" || true)
|
||||
if [[ -z "$bin" ]]; then
|
||||
bin=$(discover_server "$out" || true)
|
||||
fi
|
||||
[[ -n "$bin" ]] || die "no server binary found (looked for bin/server, dist/server, server)"
|
||||
if [[ "$(readlink -f "$bin")" != "$(readlink -f "$out/bin/server")" ]]; then
|
||||
cp -f "$bin" "$out/bin/server"
|
||||
fi
|
||||
chmod +x "$out/bin/server"
|
||||
log "build: binary $(stat -c %s "$out/bin/server") bytes, sha256 $(sha256sum "$out/bin/server" | cut -c1-16)"
|
||||
|
||||
# Static assets, if the build emits them.
|
||||
if [[ -d "$src/dist" ]]; then
|
||||
cp -a "$src/dist/." "$out/dist/" 2>/dev/null || { mkdir -p "$out/dist"; cp -a "$src/dist/." "$out/dist/"; }
|
||||
log "build: copied dist/ ($(find "$out/dist" -type f | wc -l) files)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- deploy
|
||||
do_deploy() {
|
||||
local src="$1"
|
||||
local ts sha branch release prev_target
|
||||
ts=$(date -u +%Y%m%dT%H%M%SZ)
|
||||
sha=$(git -C "$src" rev-parse --short HEAD 2>/dev/null || echo unknown)
|
||||
branch=$(git -C "$src" rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown)
|
||||
release="$RELEASES/$ts"
|
||||
|
||||
log "deploy: $APP_NAME $ts (branch $branch, $sha) -> $release"
|
||||
mkdir -p "$release"
|
||||
|
||||
# 1. Build + test. Nothing is touched on failure.
|
||||
build_and_test "$src" "$release"
|
||||
|
||||
# 2. Release payload: binary, compose file, runtime image, manifest.
|
||||
cp -f "$src/deploy/docker-compose.yml" "$release/docker-compose.yml"
|
||||
cp -f "$src/deploy/Dockerfile" "$release/Dockerfile"
|
||||
{
|
||||
echo "app=$APP_NAME"
|
||||
echo "release=$ts"
|
||||
echo "git_sha=$(git -C "$src" rev-parse HEAD 2>/dev/null || echo unknown)"
|
||||
echo "git_branch=$branch"
|
||||
echo "built_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "built_on=$(hostname)"
|
||||
echo "binary_sha256=$(sha256sum "$release/bin/server" | cut -d' ' -f1)"
|
||||
} > "$release/manifest.txt"
|
||||
printf 'APP_VERSION=%s\nAPP_PORT=%s\n' "$ts" "$PORT" > "$release/.env"
|
||||
|
||||
# 3. Runtime image, tagged with this release.
|
||||
log "deploy: docker build resume/app:$ts"
|
||||
docker build --quiet -t "resume/app:$ts" \
|
||||
--build-arg "APP_VERSION=$ts" -f "$release/Dockerfile" "$release" >>"$LOG" 2>&1 \
|
||||
|| die "image build failed"
|
||||
|
||||
# 4. Remember what is live now, then flip atomically.
|
||||
prev_target=""
|
||||
if [[ -L "$CURRENT" ]]; then prev_target=$(readlink -f "$CURRENT"); fi
|
||||
ln -sfn "$release" "$CURRENT.tmp" && mv -T "$CURRENT.tmp" "$CURRENT" || die "symlink flip failed"
|
||||
if [[ -n "$prev_target" && "$prev_target" != "$(readlink -f "$release")" ]]; then
|
||||
ln -sfn "$prev_target" "$PREVIOUS.tmp" && mv -T "$PREVIOUS.tmp" "$PREVIOUS"
|
||||
log "deploy: previous -> $prev_target"
|
||||
fi
|
||||
log "deploy: current -> $(readlink -f "$CURRENT")"
|
||||
|
||||
# 5. Bring up, then gate on health. Anything less than 200 means roll back.
|
||||
compose "$release" up -d --remove-orphans >>"$LOG" 2>&1 || log "deploy: compose up returned non-zero"
|
||||
if ! wait_healthy; then
|
||||
log "deploy: HEALTH GATE FAILED for $ts"
|
||||
rollback_internal "$ts"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "deploy: OK $ts"
|
||||
container_state | tee -a "$LOG" >&2
|
||||
docker stats --no-stream --format 'rss={{.MemUsage}} pct={{.MemPerc}}' "${APP_NAME}-app" 2>/dev/null | tee -a "$LOG" >&2 || true
|
||||
}
|
||||
|
||||
rollback_internal() {
|
||||
# Used by a failed deploy: restore the previous release, restart it, verify.
|
||||
local failed="$1" target
|
||||
if [[ ! -L "$PREVIOUS" ]]; then
|
||||
log "rollback: no previous release to restore; stopping ${APP_NAME}-app"
|
||||
compose "$CURRENT" down >>"$LOG" 2>&1 || true
|
||||
return 1
|
||||
fi
|
||||
target=$(readlink -f "$PREVIOUS")
|
||||
log "rollback: $failed failed -> restoring $target"
|
||||
ln -sfn "$target" "$CURRENT.tmp" && mv -T "$CURRENT.tmp" "$CURRENT"
|
||||
compose "$target" up -d --remove-orphans >>"$LOG" 2>&1 || true
|
||||
if wait_healthy; then
|
||||
log "rollback: previous release healthy ($(readlink -f "$CURRENT"))"
|
||||
else
|
||||
log "rollback: previous release ALSO unhealthy — manual attention required"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
do_rollback() {
|
||||
local target
|
||||
if [[ ! -L "$PREVIOUS" ]]; then die "no previous release at $PREVIOUS — nothing to roll back to"; fi
|
||||
target=$(readlink -f "$PREVIOUS")
|
||||
[[ -d "$target" ]] || die "previous release target missing: $target"
|
||||
log "rollback: current=$( [[ -L "$CURRENT" ]] && readlink -f "$CURRENT" || echo none) -> $target"
|
||||
|
||||
# Keep the release we are leaving, so the rollback is itself reversible.
|
||||
local leaving=""
|
||||
if [[ -L "$CURRENT" ]]; then leaving=$(readlink -f "$CURRENT"); fi
|
||||
ln -sfn "$target" "$CURRENT.tmp" && mv -T "$CURRENT.tmp" "$CURRENT" || die "symlink flip failed"
|
||||
if [[ -n "$leaving" && "$leaving" != "$target" ]]; then
|
||||
ln -sfn "$leaving" "$PREVIOUS.tmp" && mv -T "$PREVIOUS.tmp" "$PREVIOUS"
|
||||
fi
|
||||
|
||||
compose "$target" up -d --remove-orphans >>"$LOG" 2>&1 || log "rollback: compose up returned non-zero"
|
||||
if wait_healthy; then
|
||||
log "rollback: OK, current -> $target"
|
||||
container_state | tee -a "$LOG" >&2
|
||||
exit 0
|
||||
fi
|
||||
log "rollback: FAILED, $target is not healthy"
|
||||
exit 1
|
||||
}
|
||||
|
||||
mkdir -p "$ROOT"
|
||||
log "=== ${APP_NAME} deploy.sh start (pid $$) ==="
|
||||
if (( DO_ROLLBACK == 1 )); then
|
||||
do_rollback
|
||||
else
|
||||
do_deploy "$SRC_DIR"
|
||||
fi
|
||||
Reference in New Issue
Block a user