compass_teams_nhs/scripts/scan-compass.sh

#!/usr/bin/env bash
# Phase 4: scan Compass for one child or both, with a per-child session.
#
# For each target child this script:
#   1. restores that child's stored browser session from the SQLite `session`
#      table (AGENTS.md §7) so a scan can target one child,
#   2. runs the patchright fetcher (read-only) to store that child's raw pages,
#   3. saves the refreshed session back into the `session` table,
#   4. parses the newest stored run into the `items` table for that child.
#
# It never restarts the running service: `parse`, `session` and `scan record`
# run in a one-off container. The fetcher runs on the host with node + patchright.
#
# Fail loudly (FR-12): a child whose fetch or parse fails is recorded in `scans`
# with ok = 0, is reported by name, and makes the script exit non-zero. A failed
# child is never shown as an empty day.
#
# Usage:
#   scripts/scan-compass.sh                      # both children
#   scripts/scan-compass.sh --child Harry        # one child
#   scripts/scan-compass.sh --child Harry --child Finn --classify none
#
# Environment overrides:
#   IMAGE                image tag                  (default compass-teams-nhs/school:0.4.0-phase6)
#   VOLUME               SQLite volume              (default compass-teams-nhs_compass-teams-nhs-data)
#   RAW_ROOT             host path to var/raw       (default <repo>/var/raw)
#   SCHOOL_TIMEZONE      family timezone            (default Australia/Melbourne)
#   OMNIROUTE_BASE_URL   OmniRoute endpoint         (default http://127.0.0.1:20129/v1)
#   DOCKER_TIMEOUT       per docker command timeout (default 300)
#   FETCH_TIMEOUT        per fetch timeout          (default 300)
#   SCHOOL_SECRETS_FILE  credential env file        (default ~/.config/environment.d/10-secrets.conf)
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"

IMAGE="${IMAGE:-compass-teams-nhs/school:0.4.0-phase6}"
VOLUME="${VOLUME:-compass-teams-nhs_compass-teams-nhs-data}"
RAW_ROOT="${RAW_ROOT:-$REPO_ROOT/var/raw}"
SCHOOL_TIMEZONE="${SCHOOL_TIMEZONE:-Australia/Melbourne}"
OMNIROUTE_BASE_URL="${OMNIROUTE_BASE_URL:-http://127.0.0.1:20129/v1}"
DOCKER_TIMEOUT="${DOCKER_TIMEOUT:-300}"
FETCH_TIMEOUT="${FETCH_TIMEOUT:-300}"
SCHOOL_SECRETS_FILE="${SCHOOL_SECRETS_FILE:-$HOME/.config/environment.d/10-secrets.conf}"
CLASSIFY="${CLASSIFY:-auto}"

CHILDREN=()
while [[ $# -gt 0 ]]; do
  case "$1" in
    --child) CHILDREN+=("$2"); shift 2 ;;
    --classify) CLASSIFY="$2"; shift 2 ;;
    --timeout) DOCKER_TIMEOUT="$2"; FETCH_TIMEOUT="$2"; shift 2 ;;
    *) echo "unknown argument: $1" >&2; exit 2 ;;
  esac
done
if [[ ${#CHILDREN[@]} -eq 0 ]]; then
  CHILDREN=(Harry Finn)
fi

# Credentials for the fetcher. They come from the environment, or from the
# user's environment.d secrets file when the environment is empty. They are
# never printed.
if [[ -z "${COMPASS_USERNAME:-}" || -z "${COMPASS_PASSWORD:-}" ]]; then
  if [[ -f "$SCHOOL_SECRETS_FILE" ]]; then
    set -a
    # shellcheck disable=SC1090
    . "$SCHOOL_SECRETS_FILE"
    set +a
  fi
fi
if [[ -z "${COMPASS_USERNAME:-}" || -z "${COMPASS_PASSWORD:-}" ]]; then
  echo "COMPASS_USERNAME and COMPASS_PASSWORD are not set (see $SCHOOL_SECRETS_FILE)" >&2
  exit 2
fi
if [[ ! -d "$RAW_ROOT" ]]; then
  echo "raw pages root not found: $RAW_ROOT" >&2
  exit 1
fi

WORK="$(mktemp -d)"
cleanup() { rm -rf "$WORK"; }
trap cleanup EXIT
chmod 0777 "$WORK" # the container user writes the session handoff file here

# run_school reads and writes the service's SQLite volume as the image user.
run_school() {
  timeout "$DOCKER_TIMEOUT" docker run --rm --network host \
    -v "$VOLUME:/data" \
    -v "$WORK:/scratch" \
    -v "$RAW_ROOT:/raw:ro" \
    -e SCHOOL_DB_PATH=/data/school.db \
    -e SCHOOL_TIMEZONE="$SCHOOL_TIMEZONE" \
    -e OMNIROUTE_API_KEY \
    -e OMNIROUTE_BASE_URL="$OMNIROUTE_BASE_URL" \
    "$IMAGE" "$@"
}

# run_school_readonly only reads the volume. It runs as the host user and mounts
# the volume read-only, so the session state it writes lands in WORK owned by
# this user and the fetcher can read it.
run_school_readonly() {
  timeout "$DOCKER_TIMEOUT" docker run --rm --network host \
    --user "$(id -u):$(id -g)" \
    -v "$VOLUME:/data:ro" \
    -v "$WORK:/scratch" \
    -e SCHOOL_DB_PATH=/data/school.db \
    -e SCHOOL_TIMEZONE="$SCHOOL_TIMEZONE" \
    "$IMAGE" "$@"
}

failed=()

for child in "${CHILDREN[@]}"; do
  slug="$(printf '%s' "$child" | tr '[:upper:]' '[:lower:]')"
  state_host="$WORK/$slug.json"
  state_ctr="/scratch/$slug.json"
  rm -f "$state_host"

  echo "== $child =="

  # 1. Restore the per-child session, if one is stored.
  run_school_readonly session get --child "$child" --out "$state_ctr" || true

  # 2. Fetch the child's raw pages (read-only against the school).
  if timeout "$FETCH_TIMEOUT" node "$REPO_ROOT/scripts/compass-fetch-patchright.mjs" \
      --child "$child" --state "$state_host"; then
    :
  else
    rc=$?
    msg="fetcher failed for $child (exit $rc)"
    echo "FAIL: $msg" >&2
    run_school scan record --child "$child" --ok=false --message "$msg" || true
    failed+=("$child: fetch failed (exit $rc)")
    continue
  fi

  # 3. Persist the refreshed session for the next scan. The container user must
  # be able to read the fetcher's file.
  if [[ -f "$state_host" ]]; then
    chmod 0644 "$state_host"
    run_school session put --child "$child" --file "$state_ctr" || true
  fi

  # 4. Parse the newest stored run into items for this child.
  if run_school parse --child "$child" --run latest --raw-root /raw --classify "$CLASSIFY"; then
    :
  else
    echo "FAIL: parse failed for $child" >&2
    failed+=("$child: parse failed")
  fi
done

echo
if [[ ${#failed[@]} -eq 0 ]]; then
  echo "scan complete: ${CHILDREN[*]}"
  exit 0
fi
echo "scan failed for:" >&2
for f in "${failed[@]}"; do echo "  - $f" >&2; done
exit 1