#!/usr/bin/env node /** * Phase 2 proof-of-fetch for the Compass / NHS project. * * Drives the pi-browser-harness (its persistent Chrome session) via the harness * daemon socket, logs in to Compass once, fetches the pages we need for ONE * child, and stores the raw HTML + a manifest with a SHA-256 and byte count per * page. * * The harness daemon is the long-lived IPC bridge the pi-browser-harness * extension uses (pi -> Unix socket -> Chrome DevTools Protocol). The daemon * exposes that CDP bridge as newline-delimited JSON on its socket, so this * script can reuse the harness's already-running, cookie-persistent Chrome. * * Read-only against the school: it only navigates and reads. It never posts, * messages, or submits anything except the login form itself. * * Usage: * node scripts/compass-fetch.mjs --child Harry [--login] [--page name=path ...] * * Environment (never commit these): * COMPASS_URL login URL (default: https://nhs-vic.compass.education/login.aspx) * COMPASS_USERNAME Compass account username } required only for --login * COMPASS_PASSWORD Compass account password } * COMPASS_UA browser user-agent override (default: desktop Chrome) * COMPASS_PAGES optional comma list of name=path pairs * * Exit codes: * 0 fetch succeeded * 2 reached Compass but the Cloudflare challenge did not clear * 3 login blocked by a one-time code / second factor (STOP, do not guess) * 4 login rejected (bad credentials or unknown reason) */ import net from "node:net"; import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; const DEFAULT_URL = "https://nhs-vic.compass.education/login.aspx"; const DEFAULT_UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36"; const DAEMON_SOCKET = process.env.COMPASS_DAEMON_SOCKET || "/tmp/pi-browser-daemon.sock"; // Cloudflare's managed challenge loops forever under the default HeadlessChrome // UA. A normal desktop Chrome UA clears it. See AUD-104. const CHALLENGE_TITLE = /just a moment|verifying|checking your browser|attention required/i; const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); // --------------------------------------------------------------------------- // harness daemon client // --------------------------------------------------------------------------- function connectDaemon(clientId) { return new Promise((resolve, reject) => { const sock = net.createConnection(DAEMON_SOCKET); let buf = ""; const pending = new Map(); const listeners = []; sock.setEncoding("utf8"); let nextId = 1; let ready = false; const send = (method, params = {}, sessionId) => { const id = nextId++; const req = { type: "request", id, method, params }; if (sessionId) req.sessionId = sessionId; return new Promise((res, rej) => { pending.set(id, { res, rej }); sock.write(JSON.stringify(req) + "\n"); setTimeout(() => { if (pending.has(id)) { pending.delete(id); rej(new Error(`timeout waiting for ${method}`)); } }, 60000); }); }; const api = { send, onEvent(fn) { listeners.push(fn); }, close() { try { sock.write(JSON.stringify({ type: "control", action: "deregister", clientId }) + "\n"); } catch {} sock.destroy(); }, }; sock.on("connect", () => { sock.write(JSON.stringify({ type: "control", action: "register", clientId }) + "\n"); }); sock.on("error", reject); sock.on("data", (chunk) => { buf += chunk; let idx; while ((idx = buf.indexOf("\n")) >= 0) { const line = buf.slice(0, idx); buf = buf.slice(idx + 1); if (!line.trim()) continue; let msg; try { msg = JSON.parse(line); } catch { continue; } if (msg.type === "control" && msg.action === "registered") { if (!ready) { ready = true; resolve(api); } continue; } if (msg.type === "response" && pending.has(msg.id)) { const { res, rej } = pending.get(msg.id); pending.delete(msg.id); if (msg.error) rej(new Error(`CDP ${msg.error.code}: ${msg.error.message}`)); else res(msg.result); continue; } if (msg.type === "event") for (const l of listeners) l(msg); } }); }); } async function evalJs(api, sessionId, expression) { const r = await api.send( "Runtime.evaluate", { expression, returnByValue: true, awaitPromise: true }, sessionId, ); if (r.exceptionDetails) { throw new Error(r.exceptionDetails.exception?.description || r.exceptionDetails.text || "page JS error"); } return r.result?.value; } async function pageState(api, sessionId) { return JSON.parse( await evalJs( api, sessionId, "JSON.stringify({url:location.href,title:document.title,ready:document.readyState,bodyLen:document.body?document.body.innerHTML.length:0})", ), ); } async function waitFor(api, sessionId, predicate, { attempts = 60, everyMs = 1000 } = {}) { let state; for (let i = 0; i < attempts; i++) { await sleep(everyMs); state = await pageState(api, sessionId); if (predicate(state)) return state; } return state; } // --------------------------------------------------------------------------- // fetch + store // --------------------------------------------------------------------------- function sha256(text) { return crypto.createHash("sha256").update(text, "utf8").digest("hex"); } async function storePage(api, sessionId, outDir, slug) { const state = await pageState(api, sessionId); const html = (await evalJs(api, sessionId, "document.documentElement.outerHTML")) || ""; const bytes = Buffer.byteLength(html, "utf8"); const hash = sha256(html); fs.writeFileSync(path.join(outDir, `${slug}.html`), html); const meta = { slug, url: state.url, title: state.title, fetchedAt: new Date().toISOString(), bytes, sha256: hash, }; fs.writeFileSync(path.join(outDir, `${slug}.meta.json`), JSON.stringify(meta, null, 2) + "\n"); return meta; } async function newPage(api, clientTag) { // An isolated browser context avoids stale/conflicting Cloudflare clearance // cookies in the shared harness profile, which otherwise loop the challenge. // See AUD-104. It is in-memory only and is disposed at the end of the run. const { browserContextId } = await api.send("Target.createBrowserContext", { disposeOnDetach: false }); const { targetId } = await api.send("Target.createTarget", { url: "about:blank", browserContextId }); const { sessionId } = await api.send("Target.attachToTarget", { targetId, flatten: true }); await api.send("Page.enable", {}, sessionId); await api.send("Runtime.enable", {}, sessionId); await api.send("Network.enable", {}, sessionId); await api.send( "Network.setUserAgentOverride", { userAgent: process.env.COMPASS_UA || DEFAULT_UA, acceptLanguage: "en-AU,en;q=0.9", platform: "Linux x86_64" }, sessionId, ); await api.send( "Page.addScriptToEvaluateOnNewDocument", { source: [ "Object.defineProperty(navigator,'webdriver',{get:()=>undefined});", "Object.defineProperty(navigator,'languages',{get:()=>['en-AU','en']});", "Object.defineProperty(navigator,'plugins',{get:()=>[1,2,3,4,5]});", "window.chrome=window.chrome||{runtime:{}};", ].join(""), }, sessionId, ); return { targetId, sessionId, tag: clientTag, browserContextId }; } async function clearChallenge(api, sessionId, url) { await api.send("Page.navigate", { url }, sessionId); const state = await waitFor(api, sessionId, (s) => s.ready === "complete", { attempts: 30, everyMs: 1000 }); if (CHALLENGE_TITLE.test(state.title)) { // The challenge clears within a few seconds once the UA is not HeadlessChrome. const after = await waitFor(api, sessionId, (s) => !CHALLENGE_TITLE.test(s.title), { attempts: 40, everyMs: 1000 }); if (CHALLENGE_TITLE.test(after.title)) { console.error(`Cloudflare challenge did not clear for ${url} (title: ${after.title})`); return null; } } return state; } async function login(api, sessionId, username, password) { const form = await evalJs(api, sessionId, "!!document.querySelector('#username')"); if (!form) throw new Error("login form (#username) not present"); await evalJs( api, sessionId, `(() => { const set = (el, v) => { const proto = Object.getPrototypeOf(el); const setter = Object.getOwnPropertyDescriptor(proto, 'value').set; setter.call(el, v); el.dispatchEvent(new Event('input', { bubbles: true })); el.dispatchEvent(new Event('change', { bubbles: true })); }; set(document.querySelector('#username'), ${JSON.stringify(username)}); set(document.querySelector('#password'), ${JSON.stringify(password)}); return true; })()`, ); await evalJs(api, sessionId, "document.querySelector('#button1').click(); true"); return waitFor( api, sessionId, (s) => !/login\.aspx/i.test(s.url) || /dashboard|home|planner|messages/i.test(s.url), { attempts: 60, everyMs: 1000 }, ); } async function detectSecondFactor(api, sessionId) { return evalJs( api, sessionId, `(() => { const txt = (document.body ? document.body.innerText : '').toLowerCase(); const otpInput = [...document.querySelectorAll('input')].some( (i) => /otp|one-?time|passcode|verification|code/i.test((i.name||'') + ' ' + (i.id||'')), ); return otpInput || /verification code|one-?time code|enter the code|we sent|two-factor|2fa/.test(txt); })()`, ); } // --------------------------------------------------------------------------- // main // --------------------------------------------------------------------------- function parseArgs(argv) { const args = { child: "Harry", out: path.resolve("var/raw"), login: false, pages: [] }; for (let i = 2; i < argv.length; i++) { const a = argv[i]; if (a === "--child") args.child = argv[++i]; else if (a === "--out") args.out = argv[++i]; else if (a === "--login") args.login = true; else if (a === "--page") { const [name, p] = String(argv[++i]).split("="); args.pages.push({ name, path: p }); } else if (a === "--help") args.help = true; else throw new Error(`unknown argument: ${a}`); } if (process.env.COMPASS_PAGES) { for (const pair of process.env.COMPASS_PAGES.split(",")) { const [name, p] = pair.split("="); if (name && p) args.pages.push({ name: name.trim(), path: p.trim() }); } } return args; } async function main() { const args = parseArgs(process.argv); if (args.help) { console.log(fs.readFileSync(new URL(import.meta.url), "utf8").split("\n").slice(1, 32).join("\n")); return 0; } const baseUrl = process.env.COMPASS_URL || DEFAULT_URL; const runId = new Date().toISOString().replace(/[:.]/g, "-"); const outDir = path.join(args.out, args.child.toLowerCase(), runId); fs.mkdirSync(outDir, { recursive: true }); const api = await connectDaemon(`compass-fetch-${process.pid}`); const page = await newPage(api, args.child); const manifest = { child: args.child, runId, startedAt: new Date().toISOString(), harness: "pi-browser-harness", loginUrl: baseUrl, loggedIn: false, secondFactor: false, pages: [], errors: [], }; try { const loginState = await clearChallenge(api, page.sessionId, baseUrl); if (!loginState) { manifest.errors.push("cloudflare_challenge_not_cleared"); fs.writeFileSync(path.join(outDir, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n"); return 2; } if (args.login) { const username = process.env.COMPASS_USERNAME; const password = process.env.COMPASS_PASSWORD; if (!username || !password) throw new Error("--login requires COMPASS_USERNAME and COMPASS_PASSWORD"); const afterLogin = await login(api, page.sessionId, username, password); if (await detectSecondFactor(api, page.sessionId)) { manifest.secondFactor = true; manifest.pages.push(await storePage(api, page.sessionId, outDir, "second-factor-required")); fs.writeFileSync(path.join(outDir, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n"); console.error("Second factor / one-time code required. STOP — do not guess."); return 3; } manifest.loggedIn = true; manifest.pages.push(await storePage(api, page.sessionId, outDir, "post-login")); if (!/dashboard|home|planner|messages/i.test(afterLogin.url) && /login\.aspx/i.test(afterLogin.url)) { fs.writeFileSync(path.join(outDir, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n"); return 4; } } else { manifest.pages.push(await storePage(api, page.sessionId, outDir, "login")); } for (const p of args.pages) { const state = await clearChallenge(api, page.sessionId, new URL(p.path, baseUrl).toString()); if (!state) { manifest.errors.push(`challenge:${p.name}`); continue; } manifest.pages.push(await storePage(api, page.sessionId, outDir, p.name)); } } catch (err) { manifest.errors.push(String(err && err.message ? err.message : err)); } finally { manifest.finishedAt = new Date().toISOString(); fs.writeFileSync(path.join(outDir, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n"); try { await api.send("Target.disposeBrowserContext", { browserContextId: page.browserContextId }); } catch {} api.close(); } console.log(`stored ${manifest.pages.length} page(s) to ${outDir}`); for (const p of manifest.pages) console.log(` ${p.slug}: ${p.bytes} bytes sha256=${p.sha256} (${p.title})`); if (manifest.errors.length) console.error("errors: " + manifest.errors.join("; ")); return manifest.errors.length ? 1 : 0; } main() .then((code) => process.exit(code)) .catch((err) => { console.error(err); process.exit(1); });