Phase 2: prove the Compass login + one-child fetch via patchright
- add scripts/compass-fetch-patchright.mjs: headless patchright logs in with the parent account (no 2FA) and stores the per-child pages with a sha256 manifest - add scripts/package.json + lockfile for the patchright dependency - docs: the harness clears the Cloudflare GET but not the login POST; patchright passes it; record the real per-child URLs and the overdue counts Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Paperclip Builder
202
scripts/compass-fetch-patchright.mjs
Normal file
202
scripts/compass-fetch-patchright.mjs
Normal file
@@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Phase 2 proof-of-fetch for the Compass / NHS project — patchright path.
|
||||
*
|
||||
* The pi-browser-harness (headless Chrome) can load the Compass login page but
|
||||
* Cloudflare answers the login POST with a managed challenge that headless
|
||||
* Chrome does not clear. patchright launches Chrome with the automation leaks
|
||||
* patched, and the login POST goes through. This script:
|
||||
*
|
||||
* 1. logs in once with the parent account (COMPASS_USERNAME / COMPASS_PASSWORD),
|
||||
* 2. fetches the pages we need for ONE child,
|
||||
* 3. stores the raw HTML + a manifest with a SHA-256 and byte count per page.
|
||||
*
|
||||
* Read-only against the school: it only navigates and reads.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/compass-fetch-patchright.mjs --child Harry [--pages "name=url,..."]
|
||||
*
|
||||
* Environment (never commit these):
|
||||
* COMPASS_USERNAME / COMPASS_PASSWORD parent/guardian account
|
||||
* COMPASS_CHROME Chrome binary (default: Nix profile google-chrome)
|
||||
* COMPASS_LOGIN_URL default https://nhs-vic.compass.education/login.aspx
|
||||
* COMPASS_UA browser user-agent override
|
||||
*
|
||||
* Exit codes: 0 ok, 2 Cloudflare challenge not cleared, 3 second factor, 4 login rejected.
|
||||
*/
|
||||
import { chromium } from "patchright";
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
const DEFAULT_URL = "https://nhs-vic.compass.education/login.aspx";
|
||||
const DEFAULT_UA =
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36";
|
||||
const DEFAULT_CHROME = "/etc/profiles/per-user/sam/bin/google-chrome";
|
||||
const CHALLENGE = /just a moment|verifying|checking your browser|attention required/i;
|
||||
|
||||
// Default pages: one child's record and the calendar. The parent landing page
|
||||
// (/ ) is always stored as "home" right after login. userId is per child and is
|
||||
// read from the landing page links or passed with --user-id.
|
||||
const DEFAULT_PAGES = [
|
||||
{ name: "child-profile", url: "/Records/User.aspx?userId={userId}" },
|
||||
{ name: "calendar", url: "/Organise/Calendar/" },
|
||||
];
|
||||
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
const sha256 = (s) => crypto.createHash("sha256").update(s, "utf8").digest("hex");
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = { child: "Harry", userId: null, out: path.resolve("var/raw"), pages: null };
|
||||
for (let i = 2; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === "--child") args.child = argv[++i];
|
||||
else if (a === "--user-id") args.userId = argv[++i];
|
||||
else if (a === "--out") args.out = argv[++i];
|
||||
else if (a === "--pages") {
|
||||
args.pages = String(argv[++i]).split(",").map((p) => {
|
||||
const [name, url] = p.split("=");
|
||||
return { name, url };
|
||||
});
|
||||
} else throw new Error(`unknown argument: ${a}`);
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
async function waitPastChallenge(page, seconds) {
|
||||
for (let i = 0; i < seconds; i++) {
|
||||
await page.waitForTimeout(1000);
|
||||
const t = await page.title().catch(() => "");
|
||||
if (t && !CHALLENGE.test(t)) return t;
|
||||
}
|
||||
return await page.title().catch(() => "");
|
||||
}
|
||||
|
||||
async function storePage(page, outDir, slug, baseUrl) {
|
||||
const html = await page.content();
|
||||
const state = { url: page.url(), title: await page.title() };
|
||||
const bytes = Buffer.byteLength(html, "utf8");
|
||||
const hash = sha256(html);
|
||||
fs.writeFileSync(path.join(outDir, `${slug}.html`), html);
|
||||
const meta = {
|
||||
slug,
|
||||
url: state.url,
|
||||
title: state.title,
|
||||
fetchedAt: new Date().toISOString(),
|
||||
bytes,
|
||||
sha256: hash,
|
||||
};
|
||||
fs.writeFileSync(path.join(outDir, `${slug}.meta.json`), JSON.stringify(meta, null, 2) + "\n");
|
||||
return meta;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv);
|
||||
const baseUrl = process.env.COMPASS_LOGIN_URL || DEFAULT_URL;
|
||||
const chrome = process.env.COMPASS_CHROME || DEFAULT_CHROME;
|
||||
const user = process.env.COMPASS_USERNAME;
|
||||
const pass = process.env.COMPASS_PASSWORD;
|
||||
if (!user || !pass) throw new Error("COMPASS_USERNAME and COMPASS_PASSWORD are required");
|
||||
|
||||
const runId = new Date().toISOString().replace(/[:.]/g, "-");
|
||||
const outDir = path.join(args.out, args.child.toLowerCase(), runId);
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
|
||||
const manifest = {
|
||||
child: args.child,
|
||||
runId,
|
||||
startedAt: new Date().toISOString(),
|
||||
fetcher: "patchright",
|
||||
loginUrl: baseUrl,
|
||||
loggedIn: false,
|
||||
secondFactor: false,
|
||||
pages: [],
|
||||
errors: [],
|
||||
};
|
||||
|
||||
const browser = await chromium.launch({
|
||||
headless: true,
|
||||
executablePath: chrome,
|
||||
args: ["--no-sandbox", "--disable-dev-shm-usage", "--disable-blink-features=AutomationControlled"],
|
||||
});
|
||||
const ctx = await browser.newContext({
|
||||
locale: "en-AU",
|
||||
userAgent: process.env.COMPASS_UA || DEFAULT_UA,
|
||||
viewport: { width: 1280, height: 900 },
|
||||
timezoneId: "Australia/Melbourne",
|
||||
});
|
||||
const page = await ctx.newPage();
|
||||
|
||||
try {
|
||||
await page.goto(baseUrl, { waitUntil: "domcontentloaded", timeout: 60000 });
|
||||
if (CHALLENGE.test(await waitPastChallenge(page, 40))) {
|
||||
manifest.errors.push("cloudflare_challenge_not_cleared");
|
||||
return 2;
|
||||
}
|
||||
|
||||
await page.fill("#username", user);
|
||||
await page.fill("#password", pass);
|
||||
await page.click("#button1");
|
||||
if (CHALLENGE.test(await waitPastChallenge(page, 45))) {
|
||||
manifest.errors.push("cloudflare_login_post_challenge");
|
||||
return 2;
|
||||
}
|
||||
|
||||
const stillLogin = await page.$("#username");
|
||||
const otp = await page.evaluate(() =>
|
||||
[...document.querySelectorAll("input")].some((i) => /otp|one-?time|passcode|verification|code/i.test((i.name || "") + " " + (i.id || ""))),
|
||||
);
|
||||
if (otp && stillLogin) {
|
||||
manifest.secondFactor = true;
|
||||
manifest.pages.push(await storePage(page, outDir, "second-factor-required", baseUrl));
|
||||
return 3;
|
||||
}
|
||||
if (stillLogin) {
|
||||
manifest.errors.push("login_rejected");
|
||||
manifest.pages.push(await storePage(page, outDir, "login-rejected", baseUrl));
|
||||
return 4;
|
||||
}
|
||||
manifest.loggedIn = true;
|
||||
manifest.pages.push(await storePage(page, outDir, "home", baseUrl));
|
||||
|
||||
let userId = args.userId;
|
||||
if (!userId) {
|
||||
const match = (await page.content()).match(/Records\/User\.aspx\?userId=(\d+)/i);
|
||||
if (match) userId = match[1];
|
||||
}
|
||||
manifest.userId = userId || null;
|
||||
|
||||
const pages = args.pages || DEFAULT_PAGES;
|
||||
for (const p of pages) {
|
||||
if (p.url.includes("{userId}") && !userId) {
|
||||
manifest.errors.push(`no_user_id:${p.name}`);
|
||||
continue;
|
||||
}
|
||||
const url = new URL(p.url.replace("{userId}", userId || ""), baseUrl).toString();
|
||||
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 60000 });
|
||||
if (CHALLENGE.test(await waitPastChallenge(page, 40))) {
|
||||
manifest.errors.push(`challenge:${p.name}`);
|
||||
continue;
|
||||
}
|
||||
manifest.pages.push(await storePage(page, outDir, p.name, baseUrl));
|
||||
}
|
||||
} catch (err) {
|
||||
manifest.errors.push(String(err && err.message ? err.message : err));
|
||||
} finally {
|
||||
manifest.finishedAt = new Date().toISOString();
|
||||
fs.writeFileSync(path.join(outDir, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n");
|
||||
await browser.close().catch(() => {});
|
||||
}
|
||||
|
||||
console.log(`stored ${manifest.pages.length} page(s) to ${outDir}`);
|
||||
for (const p of manifest.pages) console.log(` ${p.slug}: ${p.bytes} bytes sha256=${p.sha256} (${p.title})`);
|
||||
if (manifest.errors.length) console.error("errors: " + manifest.errors.join("; "));
|
||||
return manifest.errors.length ? 1 : 0;
|
||||
}
|
||||
|
||||
main()
|
||||
.then((code) => process.exit(code))
|
||||
.catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user