6ff29df57c
- scripts/compass-fetch.mjs drives pi-browser-harness (daemon socket) to log in once and store raw Compass pages with a sha256/byte-count manifest - clear the Cloudflare managed challenge via a desktop-UA override plus an isolated browser context (the default HeadlessChrome UA loops the challenge) - docs/phase-2-harness-notes.md records the harness entrypoint, the Compass login form, and the stored-page evidence - gitignore local runtime state (var/, sqlite files) Co-Authored-By: Paperclip <noreply@paperclip.ing>
371 lines
14 KiB
JavaScript
371 lines
14 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Phase 2 proof-of-fetch for the Compass / NHS project.
|
|
*
|
|
* Drives the pi-browser-harness (its persistent Chrome session) via the harness
|
|
* daemon socket, logs in to Compass once, fetches the pages we need for ONE
|
|
* child, and stores the raw HTML + a manifest with a SHA-256 and byte count per
|
|
* page.
|
|
*
|
|
* The harness daemon is the long-lived IPC bridge the pi-browser-harness
|
|
* extension uses (pi -> Unix socket -> Chrome DevTools Protocol). The daemon
|
|
* exposes that CDP bridge as newline-delimited JSON on its socket, so this
|
|
* script can reuse the harness's already-running, cookie-persistent Chrome.
|
|
*
|
|
* Read-only against the school: it only navigates and reads. It never posts,
|
|
* messages, or submits anything except the login form itself.
|
|
*
|
|
* Usage:
|
|
* node scripts/compass-fetch.mjs --child Harry [--login] [--page name=path ...]
|
|
*
|
|
* Environment (never commit these):
|
|
* COMPASS_URL login URL (default: https://nhs-vic.compass.education/login.aspx)
|
|
* COMPASS_USERNAME Compass account username } required only for --login
|
|
* COMPASS_PASSWORD Compass account password }
|
|
* COMPASS_UA browser user-agent override (default: desktop Chrome)
|
|
* COMPASS_PAGES optional comma list of name=path pairs
|
|
*
|
|
* Exit codes:
|
|
* 0 fetch succeeded
|
|
* 2 reached Compass but the Cloudflare challenge did not clear
|
|
* 3 login blocked by a one-time code / second factor (STOP, do not guess)
|
|
* 4 login rejected (bad credentials or unknown reason)
|
|
*/
|
|
import net from "node:net";
|
|
import crypto from "node:crypto";
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
|
|
const DEFAULT_URL = "https://nhs-vic.compass.education/login.aspx";
|
|
const DEFAULT_UA =
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36";
|
|
const DAEMON_SOCKET = process.env.COMPASS_DAEMON_SOCKET || "/tmp/pi-browser-daemon.sock";
|
|
// Cloudflare's managed challenge loops forever under the default HeadlessChrome
|
|
// UA. A normal desktop Chrome UA clears it. See AUD-104.
|
|
const CHALLENGE_TITLE = /just a moment|verifying|checking your browser|attention required/i;
|
|
|
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// harness daemon client
|
|
// ---------------------------------------------------------------------------
|
|
function connectDaemon(clientId) {
|
|
return new Promise((resolve, reject) => {
|
|
const sock = net.createConnection(DAEMON_SOCKET);
|
|
let buf = "";
|
|
const pending = new Map();
|
|
const listeners = [];
|
|
sock.setEncoding("utf8");
|
|
let nextId = 1;
|
|
let ready = false;
|
|
|
|
const send = (method, params = {}, sessionId) => {
|
|
const id = nextId++;
|
|
const req = { type: "request", id, method, params };
|
|
if (sessionId) req.sessionId = sessionId;
|
|
return new Promise((res, rej) => {
|
|
pending.set(id, { res, rej });
|
|
sock.write(JSON.stringify(req) + "\n");
|
|
setTimeout(() => {
|
|
if (pending.has(id)) {
|
|
pending.delete(id);
|
|
rej(new Error(`timeout waiting for ${method}`));
|
|
}
|
|
}, 60000);
|
|
});
|
|
};
|
|
|
|
const api = {
|
|
send,
|
|
onEvent(fn) { listeners.push(fn); },
|
|
close() {
|
|
try {
|
|
sock.write(JSON.stringify({ type: "control", action: "deregister", clientId }) + "\n");
|
|
} catch {}
|
|
sock.destroy();
|
|
},
|
|
};
|
|
|
|
sock.on("connect", () => {
|
|
sock.write(JSON.stringify({ type: "control", action: "register", clientId }) + "\n");
|
|
});
|
|
sock.on("error", reject);
|
|
sock.on("data", (chunk) => {
|
|
buf += chunk;
|
|
let idx;
|
|
while ((idx = buf.indexOf("\n")) >= 0) {
|
|
const line = buf.slice(0, idx);
|
|
buf = buf.slice(idx + 1);
|
|
if (!line.trim()) continue;
|
|
let msg;
|
|
try { msg = JSON.parse(line); } catch { continue; }
|
|
if (msg.type === "control" && msg.action === "registered") {
|
|
if (!ready) { ready = true; resolve(api); }
|
|
continue;
|
|
}
|
|
if (msg.type === "response" && pending.has(msg.id)) {
|
|
const { res, rej } = pending.get(msg.id);
|
|
pending.delete(msg.id);
|
|
if (msg.error) rej(new Error(`CDP ${msg.error.code}: ${msg.error.message}`));
|
|
else res(msg.result);
|
|
continue;
|
|
}
|
|
if (msg.type === "event") for (const l of listeners) l(msg);
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
async function evalJs(api, sessionId, expression) {
|
|
const r = await api.send(
|
|
"Runtime.evaluate",
|
|
{ expression, returnByValue: true, awaitPromise: true },
|
|
sessionId,
|
|
);
|
|
if (r.exceptionDetails) {
|
|
throw new Error(r.exceptionDetails.exception?.description || r.exceptionDetails.text || "page JS error");
|
|
}
|
|
return r.result?.value;
|
|
}
|
|
|
|
async function pageState(api, sessionId) {
|
|
return JSON.parse(
|
|
await evalJs(
|
|
api,
|
|
sessionId,
|
|
"JSON.stringify({url:location.href,title:document.title,ready:document.readyState,bodyLen:document.body?document.body.innerHTML.length:0})",
|
|
),
|
|
);
|
|
}
|
|
|
|
async function waitFor(api, sessionId, predicate, { attempts = 60, everyMs = 1000 } = {}) {
|
|
let state;
|
|
for (let i = 0; i < attempts; i++) {
|
|
await sleep(everyMs);
|
|
state = await pageState(api, sessionId);
|
|
if (predicate(state)) return state;
|
|
}
|
|
return state;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// fetch + store
|
|
// ---------------------------------------------------------------------------
|
|
function sha256(text) {
|
|
return crypto.createHash("sha256").update(text, "utf8").digest("hex");
|
|
}
|
|
|
|
async function storePage(api, sessionId, outDir, slug) {
|
|
const state = await pageState(api, sessionId);
|
|
const html = (await evalJs(api, sessionId, "document.documentElement.outerHTML")) || "";
|
|
const bytes = Buffer.byteLength(html, "utf8");
|
|
const hash = sha256(html);
|
|
fs.writeFileSync(path.join(outDir, `${slug}.html`), html);
|
|
const meta = {
|
|
slug,
|
|
url: state.url,
|
|
title: state.title,
|
|
fetchedAt: new Date().toISOString(),
|
|
bytes,
|
|
sha256: hash,
|
|
};
|
|
fs.writeFileSync(path.join(outDir, `${slug}.meta.json`), JSON.stringify(meta, null, 2) + "\n");
|
|
return meta;
|
|
}
|
|
|
|
async function newPage(api, clientTag) {
|
|
// An isolated browser context avoids stale/conflicting Cloudflare clearance
|
|
// cookies in the shared harness profile, which otherwise loop the challenge.
|
|
// See AUD-104. It is in-memory only and is disposed at the end of the run.
|
|
const { browserContextId } = await api.send("Target.createBrowserContext", { disposeOnDetach: false });
|
|
const { targetId } = await api.send("Target.createTarget", { url: "about:blank", browserContextId });
|
|
const { sessionId } = await api.send("Target.attachToTarget", { targetId, flatten: true });
|
|
await api.send("Page.enable", {}, sessionId);
|
|
await api.send("Runtime.enable", {}, sessionId);
|
|
await api.send("Network.enable", {}, sessionId);
|
|
await api.send(
|
|
"Network.setUserAgentOverride",
|
|
{ userAgent: process.env.COMPASS_UA || DEFAULT_UA, acceptLanguage: "en-AU,en;q=0.9", platform: "Linux x86_64" },
|
|
sessionId,
|
|
);
|
|
await api.send(
|
|
"Page.addScriptToEvaluateOnNewDocument",
|
|
{
|
|
source: [
|
|
"Object.defineProperty(navigator,'webdriver',{get:()=>undefined});",
|
|
"Object.defineProperty(navigator,'languages',{get:()=>['en-AU','en']});",
|
|
"Object.defineProperty(navigator,'plugins',{get:()=>[1,2,3,4,5]});",
|
|
"window.chrome=window.chrome||{runtime:{}};",
|
|
].join(""),
|
|
},
|
|
sessionId,
|
|
);
|
|
return { targetId, sessionId, tag: clientTag, browserContextId };
|
|
}
|
|
|
|
async function clearChallenge(api, sessionId, url) {
|
|
await api.send("Page.navigate", { url }, sessionId);
|
|
const state = await waitFor(api, sessionId, (s) => s.ready === "complete", { attempts: 30, everyMs: 1000 });
|
|
if (CHALLENGE_TITLE.test(state.title)) {
|
|
// The challenge clears within a few seconds once the UA is not HeadlessChrome.
|
|
const after = await waitFor(api, sessionId, (s) => !CHALLENGE_TITLE.test(s.title), { attempts: 40, everyMs: 1000 });
|
|
if (CHALLENGE_TITLE.test(after.title)) {
|
|
console.error(`Cloudflare challenge did not clear for ${url} (title: ${after.title})`);
|
|
return null;
|
|
}
|
|
}
|
|
return state;
|
|
}
|
|
|
|
async function login(api, sessionId, username, password) {
|
|
const form = await evalJs(api, sessionId, "!!document.querySelector('#username')");
|
|
if (!form) throw new Error("login form (#username) not present");
|
|
await evalJs(
|
|
api,
|
|
sessionId,
|
|
`(() => {
|
|
const set = (el, v) => {
|
|
const proto = Object.getPrototypeOf(el);
|
|
const setter = Object.getOwnPropertyDescriptor(proto, 'value').set;
|
|
setter.call(el, v);
|
|
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
};
|
|
set(document.querySelector('#username'), ${JSON.stringify(username)});
|
|
set(document.querySelector('#password'), ${JSON.stringify(password)});
|
|
return true;
|
|
})()`,
|
|
);
|
|
await evalJs(api, sessionId, "document.querySelector('#button1').click(); true");
|
|
return waitFor(
|
|
api,
|
|
sessionId,
|
|
(s) => !/login\.aspx/i.test(s.url) || /dashboard|home|planner|messages/i.test(s.url),
|
|
{ attempts: 60, everyMs: 1000 },
|
|
);
|
|
}
|
|
|
|
async function detectSecondFactor(api, sessionId) {
|
|
return evalJs(
|
|
api,
|
|
sessionId,
|
|
`(() => {
|
|
const txt = (document.body ? document.body.innerText : '').toLowerCase();
|
|
const otpInput = [...document.querySelectorAll('input')].some(
|
|
(i) => /otp|one-?time|passcode|verification|code/i.test((i.name||'') + ' ' + (i.id||'')),
|
|
);
|
|
return otpInput || /verification code|one-?time code|enter the code|we sent|two-factor|2fa/.test(txt);
|
|
})()`,
|
|
);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// main
|
|
// ---------------------------------------------------------------------------
|
|
function parseArgs(argv) {
|
|
const args = { child: "Harry", out: path.resolve("var/raw"), login: false, pages: [] };
|
|
for (let i = 2; i < argv.length; i++) {
|
|
const a = argv[i];
|
|
if (a === "--child") args.child = argv[++i];
|
|
else if (a === "--out") args.out = argv[++i];
|
|
else if (a === "--login") args.login = true;
|
|
else if (a === "--page") {
|
|
const [name, p] = String(argv[++i]).split("=");
|
|
args.pages.push({ name, path: p });
|
|
} else if (a === "--help") args.help = true;
|
|
else throw new Error(`unknown argument: ${a}`);
|
|
}
|
|
if (process.env.COMPASS_PAGES) {
|
|
for (const pair of process.env.COMPASS_PAGES.split(",")) {
|
|
const [name, p] = pair.split("=");
|
|
if (name && p) args.pages.push({ name: name.trim(), path: p.trim() });
|
|
}
|
|
}
|
|
return args;
|
|
}
|
|
|
|
async function main() {
|
|
const args = parseArgs(process.argv);
|
|
if (args.help) {
|
|
console.log(fs.readFileSync(new URL(import.meta.url), "utf8").split("\n").slice(1, 32).join("\n"));
|
|
return 0;
|
|
}
|
|
|
|
const baseUrl = process.env.COMPASS_URL || DEFAULT_URL;
|
|
const runId = new Date().toISOString().replace(/[:.]/g, "-");
|
|
const outDir = path.join(args.out, args.child.toLowerCase(), runId);
|
|
fs.mkdirSync(outDir, { recursive: true });
|
|
|
|
const api = await connectDaemon(`compass-fetch-${process.pid}`);
|
|
const page = await newPage(api, args.child);
|
|
const manifest = {
|
|
child: args.child,
|
|
runId,
|
|
startedAt: new Date().toISOString(),
|
|
harness: "pi-browser-harness",
|
|
loginUrl: baseUrl,
|
|
loggedIn: false,
|
|
secondFactor: false,
|
|
pages: [],
|
|
errors: [],
|
|
};
|
|
|
|
try {
|
|
const loginState = await clearChallenge(api, page.sessionId, baseUrl);
|
|
if (!loginState) {
|
|
manifest.errors.push("cloudflare_challenge_not_cleared");
|
|
fs.writeFileSync(path.join(outDir, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n");
|
|
return 2;
|
|
}
|
|
|
|
if (args.login) {
|
|
const username = process.env.COMPASS_USERNAME;
|
|
const password = process.env.COMPASS_PASSWORD;
|
|
if (!username || !password) throw new Error("--login requires COMPASS_USERNAME and COMPASS_PASSWORD");
|
|
const afterLogin = await login(api, page.sessionId, username, password);
|
|
if (await detectSecondFactor(api, page.sessionId)) {
|
|
manifest.secondFactor = true;
|
|
manifest.pages.push(await storePage(api, page.sessionId, outDir, "second-factor-required"));
|
|
fs.writeFileSync(path.join(outDir, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n");
|
|
console.error("Second factor / one-time code required. STOP — do not guess.");
|
|
return 3;
|
|
}
|
|
manifest.loggedIn = true;
|
|
manifest.pages.push(await storePage(api, page.sessionId, outDir, "post-login"));
|
|
if (!/dashboard|home|planner|messages/i.test(afterLogin.url) && /login\.aspx/i.test(afterLogin.url)) {
|
|
fs.writeFileSync(path.join(outDir, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n");
|
|
return 4;
|
|
}
|
|
} else {
|
|
manifest.pages.push(await storePage(api, page.sessionId, outDir, "login"));
|
|
}
|
|
|
|
for (const p of args.pages) {
|
|
const state = await clearChallenge(api, page.sessionId, new URL(p.path, baseUrl).toString());
|
|
if (!state) { manifest.errors.push(`challenge:${p.name}`); continue; }
|
|
manifest.pages.push(await storePage(api, page.sessionId, outDir, p.name));
|
|
}
|
|
} catch (err) {
|
|
manifest.errors.push(String(err && err.message ? err.message : err));
|
|
} finally {
|
|
manifest.finishedAt = new Date().toISOString();
|
|
fs.writeFileSync(path.join(outDir, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n");
|
|
try {
|
|
await api.send("Target.disposeBrowserContext", { browserContextId: page.browserContextId });
|
|
} catch {}
|
|
api.close();
|
|
}
|
|
|
|
console.log(`stored ${manifest.pages.length} page(s) to ${outDir}`);
|
|
for (const p of manifest.pages) console.log(` ${p.slug}: ${p.bytes} bytes sha256=${p.sha256} (${p.title})`);
|
|
if (manifest.errors.length) console.error("errors: " + manifest.errors.join("; "));
|
|
return manifest.errors.length ? 1 : 0;
|
|
}
|
|
|
|
main()
|
|
.then((code) => process.exit(code))
|
|
.catch((err) => {
|
|
console.error(err);
|
|
process.exit(1);
|
|
});
|