compass_teams_nhs/scripts/compass-fetch-patchright.mjs

#!/usr/bin/env node
/**
 * Compass fetcher for the Compass / NHS project — patchright path.
 *
 * The pi-browser-harness (headless Chrome) can load the Compass login page but
 * Cloudflare answers the login POST with a managed challenge that headless
 * Chrome does not clear. patchright launches Chrome with the automation leaks
 * patched, and the login POST goes through. This script:
 *
 *   1. reuses a stored per-child browser session when one is supplied and still
 *      valid, otherwise logs in once with the parent account
 *      (COMPASS_USERNAME / COMPASS_PASSWORD),
 *   2. resolves the child's Compass userId from the parent landing page,
 *   3. fetches that child's pages,
 *   4. stores the raw HTML + a manifest with a SHA-256 and byte count per page,
 *   5. refreshes the per-child session state (``--state``) for the next scan.
 *
 * Read-only against the school: it only navigates and reads.
 *
 * Usage:
 *   node scripts/compass-fetch-patchright.mjs --child Harry [--user-id 27038]
 *        [--state var/session/harry.json] [--pages "name=url,..."]
 *
 * Environment (never commit these):
 *   COMPASS_USERNAME / COMPASS_PASSWORD  parent/guardian account
 *   COMPASS_CHROME                        Chrome binary (default: Nix profile google-chrome)
 *   COMPASS_LOGIN_URL                     default https://nhs-vic.compass.education/login.aspx
 *   COMPASS_UA                            browser user-agent override
 *
 * Exit codes: 0 ok, 2 Cloudflare challenge not cleared, 3 second factor,
 * 4 login rejected, 1 fetch errors (a page failed or the child id is missing).
 */
import { chromium } from "patchright";
import crypto from "node:crypto";
import fs from "node:fs";
import path from "node:path";

const DEFAULT_URL = "https://nhs-vic.compass.education/login.aspx";
const DEFAULT_UA =
  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36";
const DEFAULT_CHROME = "/etc/profiles/per-user/sam/bin/google-chrome";
const CHALLENGE = /just a moment|verifying|checking your browser|attention required/i;

// Per-child pages. The calendar is deliberately NOT here: the parent calendar
// page renders the visible child's calendars only (it defaults to the eldest),
// so fetching it inside a younger child's run would leak the eldest's timetable
// into that child's items. Per-child calendar attribution is deferred; the
// calendar stays available through an explicit --pages override. The parent
// landing page (/) is always stored as "home" right after login.
const DEFAULT_PAGES = [
  { name: "child-profile", url: "/Records/User.aspx?userId={userId}" },
  { name: "learning-tasks", url: "/Records/UserNew.aspx?userId={userId}#learningTasks" },
];

// TASKS_SERVICE is the Compass JSON service behind the Learning Tasks grid. The
// grid itself is capped at LearningTask_Default_Records_Count (20) rows and, on
// the parent page, mixes children. This endpoint returns every learning task
// for one child with its submission status, so the overdue count can be checked
// against the same child's count on the home page (AUD-114, FR-12).
const TASKS_SERVICE = "/Services/LearningTasks.svc/GetAllLearningTasksByUserId";

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const sha256 = (s) => crypto.createHash("sha256").update(s, "utf8").digest("hex");

function parseArgs(argv) {
  const args = {
    child: "Harry",
    userId: null,
    out: path.resolve("var/raw"),
    state: null,
    pages: null,
  };
  for (let i = 2; i < argv.length; i++) {
    const a = argv[i];
    if (a === "--child") args.child = argv[++i];
    else if (a === "--user-id") args.userId = argv[++i];
    else if (a === "--out") args.out = argv[++i];
    else if (a === "--state") args.state = argv[++i];
    else if (a === "--no-state") args.state = null;
    else if (a === "--pages") {
      // Split each entry on the FIRST "=" so a URL may carry a query string.
      args.pages = String(argv[++i]).split(",").map((p) => {
        const eq = p.indexOf("=");
        if (eq < 0) throw new Error(`--pages entry needs name=url: ${p}`);
        return { name: p.slice(0, eq), url: p.slice(eq + 1) };
      });
    } else throw new Error(`unknown argument: ${a}`);
  }
  return args;
}

async function waitPastChallenge(page, seconds) {
  for (let i = 0; i < seconds; i++) {
    await page.waitForTimeout(1000);
    const t = await page.title().catch(() => "");
    if (t && !CHALLENGE.test(t)) return t;
  }
  return await page.title().catch(() => "");
}

/**
 * waitForFeed waits for the home news feed to render. Compass loads the feed
 * client-side after the page shell, so storing the page too early drops it and
 * the parser (correctly) fails loudly.
 */
async function waitForFeed(page, ms) {
  try {
    await page.waitForSelector('[data-testid="feed-toolbar"], .MuiCardContent-root', { timeout: ms });
  } catch {
    // The parser will fail loudly if the feed is genuinely missing.
  }
  await sleep(500);
}

async function storePage(page, outDir, slug, baseUrl) {
  const html = await page.content();
  const state = { url: page.url(), title: await page.title() };
  const bytes = Buffer.byteLength(html, "utf8");
  const hash = sha256(html);
  fs.writeFileSync(path.join(outDir, `${slug}.html`), html);
  const meta = {
    slug,
    url: state.url,
    title: state.title,
    fetchedAt: new Date().toISOString(),
    bytes,
    sha256: hash,
  };
  fs.writeFileSync(path.join(outDir, `${slug}.meta.json`), JSON.stringify(meta, null, 2) + "\n");
  return meta;
}

/**
 * fetchLearningTasksService reads the full per-child learning-task list from the
 * Compass JSON service and stores it next to the HTML pages. It is read-only:
 * one POST with the child's userId and a large page limit. The request runs
 * inside the logged-in page (`fetch`, `credentials: include`) so it carries the
 * session cookies and the Compass request-capture headers; the browser
 * context's own API client is rejected with 403. The meta file records the
 * child's userId so the Go parser can tie the list to the child and to the home
 * page count. A non-200 response or a non-JSON body is recorded as an error.
 */
async function fetchLearningTasksService(page, baseUrl, outDir, userId) {
  const url = new URL(TASKS_SERVICE, baseUrl).toString();
  const body = { userId: Number(userId), page: 1, start: 0, limit: 1000 };
  const result = await page.evaluate(
    async ({ url, body }) => {
      const r = await fetch(url, {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        credentials: "include",
        body: JSON.stringify(body),
      });
      return { status: r.status, text: await r.text() };
    },
    { url, body },
  );
  const text = result.text;
  const meta = {
    slug: "learning-tasks-all",
    url,
    title: "Learning Tasks service",
    method: "POST",
    body,
    userId: String(userId),
    fetchedAt: new Date().toISOString(),
    bytes: Buffer.byteLength(text, "utf8"),
    sha256: sha256(text),
    status: result.status,
    taskCount: null,
    total: null,
  };
  try {
    const parsed = JSON.parse(text);
    if (Array.isArray(parsed?.d?.data)) meta.taskCount = parsed.d.data.length;
    if (typeof parsed?.d?.total === "number") meta.total = parsed.d.total;
  } catch {
    // Not JSON: keep the raw body and let the caller fail loudly (FR-12).
  }
  fs.writeFileSync(path.join(outDir, "learning-tasks-all.json"), text);
  fs.writeFileSync(path.join(outDir, "learning-tasks-all.meta.json"), JSON.stringify(meta, null, 2) + "\n");
  return meta;
}

/**
 * findChildUserId resolves a child name to a Compass userId from the parent
 * landing page, for example "Harrison ROLFE" -> 27038. The parent page groups
 * each child's links under a .parentChildActions block whose <h5> holds the
 * formal name, while the family uses a short one, so compare the first four
 * letters ("Harr" / "Finn"). An explicit --user-id always wins.
 */
async function findChildUserId(page, child) {
  const want = child.trim().toLowerCase().slice(0, 4);

  const blocks = await page
    .$$eval(".parentChildActions", (els) =>
      els
        .map((el) => {
          const h5 = el.querySelector("h5");
          const a = el.querySelector('a[href*="Records/User.aspx?userId="]');
          const m = a && (a.getAttribute("href") || "").match(/userId=(\d+)/);
          return { name: (h5 && h5.textContent || "").trim(), userId: m ? m[1] : null };
        })
        .filter((c) => c.userId),
    )
    .catch(() => []);
  for (const { name, userId } of blocks) {
    const first = name.split(/\s+/)[0].toLowerCase();
    if (first.slice(0, 4) === want) return userId;
  }

  // Fallback: any link whose text starts with the child's name.
  const links = await page
    .$$eval('a[href*="Records/User.aspx?userId="]', (as) =>
      as.map((a) => ({ href: a.getAttribute("href"), text: (a.textContent || "").trim() })),
    )
    .catch(() => []);
  for (const { href, text } of links) {
    const m = href.match(/userId=(\d+)/);
    if (!m) continue;
    const first = text.split(/\s+/)[0].toLowerCase();
    if (first.slice(0, 4) === want) return m[1];
  }
  return null;
}

async function loadStorageState(file) {
  if (!file) return null;
  try {
    if (!fs.existsSync(file)) return null;
    const raw = fs.readFileSync(file, "utf8");
    JSON.parse(raw); // a truncated or empty file is not a usable session
    return JSON.parse(raw);
  } catch {
    return null;
  }
}

async function main() {
  const args = parseArgs(process.argv);
  const baseUrl = process.env.COMPASS_LOGIN_URL || DEFAULT_URL;
  const rootUrl = new URL("/", baseUrl).toString();
  const chrome = process.env.COMPASS_CHROME || DEFAULT_CHROME;
  const user = process.env.COMPASS_USERNAME;
  const pass = process.env.COMPASS_PASSWORD;
  if (!user || !pass) throw new Error("COMPASS_USERNAME and COMPASS_PASSWORD are required");

  const runId = new Date().toISOString().replace(/[:.]/g, "-");
  const outDir = path.join(args.out, args.child.toLowerCase(), runId);
  fs.mkdirSync(outDir, { recursive: true });

  const manifest = {
    child: args.child,
    runId,
    startedAt: new Date().toISOString(),
    fetcher: "patchright",
    loginUrl: baseUrl,
    loggedIn: false,
    sessionReused: false,
    sessionSaved: false,
    secondFactor: false,
    pages: [],
    errors: [],
  };

  const state = await loadStorageState(args.state);
  const browser = await chromium.launch({
    headless: true,
    executablePath: chrome,
    args: ["--no-sandbox", "--disable-dev-shm-usage", "--disable-blink-features=AutomationControlled"],
  });
  const contextOptions = {
    locale: "en-AU",
    userAgent: process.env.COMPASS_UA || DEFAULT_UA,
    viewport: { width: 1280, height: 900 },
    timezoneId: "Australia/Melbourne",
  };
  if (state) contextOptions.storageState = state;
  const ctx = await browser.newContext(contextOptions);
  const page = await ctx.newPage();

  try {
    await page.goto(rootUrl, { waitUntil: "domcontentloaded", timeout: 60000 });
    if (CHALLENGE.test(await waitPastChallenge(page, 40))) {
      manifest.errors.push("cloudflare_challenge_not_cleared");
      return 2;
    }

    // A stored session skips the login POST when Compass still accepts it.
    const needsLogin = Boolean(await page.$("#username"));
    manifest.sessionReused = state !== null && !needsLogin;

    if (needsLogin) {
      await page.goto(baseUrl, { waitUntil: "domcontentloaded", timeout: 60000 });
      if (CHALLENGE.test(await waitPastChallenge(page, 40))) {
        manifest.errors.push("cloudflare_challenge_not_cleared");
        return 2;
      }
      await page.fill("#username", user);
      await page.fill("#password", pass);
      await page.click("#button1");
      if (CHALLENGE.test(await waitPastChallenge(page, 45))) {
        manifest.errors.push("cloudflare_login_post_challenge");
        return 2;
      }

      const stillLogin = await page.$("#username");
      const otp = await page.evaluate(() =>
        [...document.querySelectorAll("input")].some((i) =>
          /otp|one-?time|passcode|verification|code/i.test((i.name || "") + " " + (i.id || "")),
        ),
      );
      if (otp && stillLogin) {
        manifest.secondFactor = true;
        manifest.pages.push(await storePage(page, outDir, "second-factor-required", baseUrl));
        return 3;
      }
      if (stillLogin) {
        manifest.errors.push("login_rejected");
        manifest.pages.push(await storePage(page, outDir, "login-rejected", baseUrl));
        return 4;
      }
    }

    manifest.loggedIn = true;
    await waitForFeed(page, 20000);
    manifest.pages.push(await storePage(page, outDir, "home", baseUrl));

    // Refresh the per-child session now that the login is known good.
    if (args.state) {
      try {
        fs.mkdirSync(path.dirname(path.resolve(args.state)), { recursive: true });
        await ctx.storageState({ path: args.state });
        manifest.sessionSaved = true;
      } catch (err) {
        manifest.errors.push(`session_save_failed:${err.message}`);
      }
    }

    let userId = args.userId;
    if (!userId) userId = await findChildUserId(page, args.child);
    manifest.userId = userId || null;

    // The authoritative per-child task list. A failure here is an explicit error
    // so a partial task list is never mistaken for "nothing overdue" (FR-12).
    if (userId) {
      const taskMeta = await fetchLearningTasksService(page, baseUrl, outDir, userId);
      manifest.pages.push(taskMeta);
      if (taskMeta.status !== 200) manifest.errors.push(`learning_tasks_all_http:${taskMeta.status}`);
      if (taskMeta.taskCount == null) manifest.errors.push("learning_tasks_all_not_json");
    } else {
      manifest.errors.push("no_user_id:learning-tasks-all");
    }

    const pages = args.pages || DEFAULT_PAGES;
    for (const p of pages) {
      if (p.url.includes("{userId}") && !userId) {
        manifest.errors.push(`no_user_id:${p.name}`);
        continue;
      }
      const url = new URL(p.url.replace("{userId}", userId || ""), baseUrl).toString();
      await page.goto(url, { waitUntil: "domcontentloaded", timeout: 60000 });
      if (CHALLENGE.test(await waitPastChallenge(page, 40))) {
        manifest.errors.push(`challenge:${p.name}`);
        continue;
      }
      // Give the profile app a moment to render the learning-tasks grid.
      await sleep(1500);
      manifest.pages.push(await storePage(page, outDir, p.name, baseUrl));
    }
  } catch (err) {
    manifest.errors.push(String(err && err.message ? err.message : err));
  } finally {
    manifest.finishedAt = new Date().toISOString();
    fs.writeFileSync(path.join(outDir, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n");
    await browser.close().catch(() => {});
  }

  console.log(`stored ${manifest.pages.length} page(s) to ${outDir}`);
  console.log(`  session reused=${manifest.sessionReused} saved=${manifest.sessionSaved}`);
  for (const p of manifest.pages) console.log(`  ${p.slug}: ${p.bytes} bytes sha256=${p.sha256} (${p.title})`);
  if (manifest.errors.length) console.error("errors: " + manifest.errors.join("; "));
  return manifest.errors.length ? 1 : 0;
}

main()
  .then((code) => process.exit(code))
  .catch((err) => {
    console.error(err);
    process.exit(1);
  });