mirror of
https://github.com/only-cli/oc.git
synced 2026-09-15 10:40:56 +02:00
feat: add authenticated browsing with per-session cookie jars
This commit is contained in:
+46
@@ -0,0 +1,46 @@
|
||||
/**
|
||||
* Detect when a fetched page is a login gate rather than the content the
|
||||
* caller expected. Runs before contentFailure so a thin login form is not
|
||||
* partially rendered as if it were real content.
|
||||
*/
|
||||
|
||||
const LOGIN_PATH = /\/(?:login|signin|sign-in|auth|oauth|sso)(?:\/|$|\?)/i;
|
||||
const LOGIN_TITLE = /\b(?:log\s*in|sign\s*in|authenticate)\b/i;
|
||||
const LOGIN_BUTTON = /\b(?:log\s*in|sign\s*in|continue|submit)\b/i;
|
||||
|
||||
/**
|
||||
* @param {string} url
|
||||
* @returns {string}
|
||||
*/
|
||||
export function sessionExpiredMessage(url) {
|
||||
return `session expired or cookies are no longer valid for ${url}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import('./distill.js').Page} page
|
||||
* @param {string} url
|
||||
* @param {{ hadAuth?: boolean }} [opts]
|
||||
* @returns {string|null}
|
||||
*/
|
||||
export function authFailure(page, url, { hadAuth = false } = {}) {
|
||||
const hasPassword = page.blocks.some((b) => b.type === 'input' && b.text === 'password');
|
||||
if (!hasPassword) return null;
|
||||
|
||||
let pathname = '';
|
||||
try {
|
||||
pathname = new URL(url).pathname;
|
||||
} catch {
|
||||
// malformed URL: rely on other signals only
|
||||
}
|
||||
|
||||
const loginUrl = LOGIN_PATH.test(pathname);
|
||||
const loginTitle = LOGIN_TITLE.test(page.title ?? '');
|
||||
const loginButton = page.blocks.some((b) =>
|
||||
(b.type === 'button' || b.type === 'input') && LOGIN_BUTTON.test(b.text ?? ''),
|
||||
);
|
||||
|
||||
if (!loginUrl && !loginTitle && !loginButton) return null;
|
||||
|
||||
if (hadAuth) return sessionExpiredMessage(url);
|
||||
return 'this page requires login; run \'oc login --cookie "..." --domain example.com\'';
|
||||
}
|
||||
+71
-7
@@ -5,7 +5,18 @@ import { distill, toMarkdown, toHTML } from './distill.js';
|
||||
import { render, estimateTokens, contentTokens, contentFailure, MIN_CONTENT } from './render.js';
|
||||
import { resolveSite, listSites } from './sites.js';
|
||||
import * as act from './act.js';
|
||||
import { DEFAULT_SESSION, loadSession, saveSession, sessionFromPage } from './session.js';
|
||||
import { DEFAULT_SESSION, assertSafeName, loadSession, saveSession, sessionFromPage } from './session.js';
|
||||
import { authFailure, sessionExpiredMessage } from './auth.js';
|
||||
import {
|
||||
loadCookieJar,
|
||||
saveCookieJar,
|
||||
clearCookieJar,
|
||||
createJarHandle,
|
||||
loginCookieJar,
|
||||
parseExpires,
|
||||
DEFAULT_EXPIRES_MS,
|
||||
JAR_EXPIRED,
|
||||
} from './cookies.js';
|
||||
|
||||
const HELP = `only-cli: the web as a compact terminal, built for AI agents.
|
||||
|
||||
@@ -23,6 +34,8 @@ usage: oc <command> [args] [flags]
|
||||
fill <n> <text> type into a numbered input (planned)
|
||||
submit [n] submit a form (planned)
|
||||
back return to the previous page (planned)
|
||||
login seed cookies for a session (--cookie, --domain)
|
||||
logout [session] clear saved cookies for a session
|
||||
session ls|rm manage saved sessions (planned)
|
||||
|
||||
flags:
|
||||
@@ -38,6 +51,11 @@ flags:
|
||||
globally. Off by default because metrics cost tokens too.
|
||||
--session <name> keep separate page state under a name (default: default)
|
||||
|
||||
Authenticated pages: run 'oc login --cookie "..." --domain example.com' to seed
|
||||
cookies for a session (default lifetime 1h, override with --expires 2h). Cookies
|
||||
live in a separate file from page state and are sent on every fetch for that
|
||||
session. 'oc logout' clears them early.
|
||||
|
||||
A page that comes back with no readable text (JavaScript-only, a consent wall,
|
||||
a bot challenge) says so in one line on stderr and exits 2, so a caller can
|
||||
tell an empty page from a page oc could not read and fall back to a browser.
|
||||
@@ -81,7 +99,7 @@ const noContent = (url, detail, hint = "; 'oc raw' has the page's markdown if th
|
||||
// Anything else in the first position is tried as a site shortcut before it is
|
||||
// called unknown, so a new clis/ definition needs no change here.
|
||||
const COMMANDS = new Set([
|
||||
'open', 'do', 'raw', 'read', 'next', 'find', 'fill', 'submit', 'back', 'session', 'sites',
|
||||
'open', 'do', 'raw', 'read', 'next', 'find', 'fill', 'submit', 'back', 'login', 'logout', 'session', 'sites',
|
||||
]);
|
||||
|
||||
async function main() {
|
||||
@@ -94,6 +112,9 @@ async function main() {
|
||||
verbose: { type: 'boolean', short: 'v', default: false },
|
||||
budget: { type: 'string' },
|
||||
session: { type: 'string' },
|
||||
cookie: { type: 'string' },
|
||||
domain: { type: 'string' },
|
||||
expires: { type: 'string' },
|
||||
help: { type: 'boolean', short: 'h', default: false },
|
||||
},
|
||||
});
|
||||
@@ -118,7 +139,7 @@ async function main() {
|
||||
command = 'open';
|
||||
}
|
||||
|
||||
const sessionName = values.session || DEFAULT_SESSION;
|
||||
const sessionName = assertSafeName(values.session || DEFAULT_SESSION);
|
||||
// Zero means "whatever this command's default is", which differs: the
|
||||
// compact view targets 500 tokens, read targets 2000.
|
||||
const asked = values.budget ? Number(values.budget) : 0;
|
||||
@@ -126,6 +147,20 @@ async function main() {
|
||||
throw new Error('--budget must be a positive number');
|
||||
}
|
||||
|
||||
if (command === 'login') {
|
||||
if (!values.cookie) throw new Error("usage: oc login --cookie \"...\" --domain example.com [--expires 1h] [--session name]");
|
||||
if (!values.domain) throw new Error('--domain is required (the site hostname your cookies belong to)');
|
||||
const expiresMs = values.expires ? parseExpires(values.expires) : DEFAULT_EXPIRES_MS;
|
||||
loginCookieJar(sessionName, values.cookie, values.domain, { expiresMs });
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === 'logout') {
|
||||
const name = args[0] ? assertSafeName(args[0]) : sessionName;
|
||||
clearCookieJar(name);
|
||||
return;
|
||||
}
|
||||
|
||||
switch (command) {
|
||||
case 'open':
|
||||
case 'do':
|
||||
@@ -148,8 +183,18 @@ async function main() {
|
||||
}
|
||||
if (!url) throw new Error(`usage: oc ${command} <url>`);
|
||||
const budget = asked || 500;
|
||||
const jarData = loadCookieJar(sessionName);
|
||||
// Clock-expired jars are wiped on load; say so in the same voice as a
|
||||
// login-page redirect rather than fetching with no cookies and guessing.
|
||||
if (jarData === JAR_EXPIRED) {
|
||||
noContent(url, sessionExpiredMessage(url));
|
||||
return;
|
||||
}
|
||||
const hadAuth = jarData != null;
|
||||
const jar = jarData ? createJarHandle(sessionName, jarData) : null;
|
||||
const t0 = performance.now();
|
||||
const { url: finalUrl, html, status, via } = await fetchPage(url);
|
||||
const { url: finalUrl, html, status, via } = await fetchPage(url, { jar: jar ?? undefined });
|
||||
if (jar) saveCookieJar(sessionName, jar.toJSON());
|
||||
const fetchMs = performance.now() - t0;
|
||||
const resources = () => {
|
||||
const processMs = performance.now() - t0 - fetchMs;
|
||||
@@ -160,16 +205,29 @@ async function main() {
|
||||
const htmlTokens = estimateTokens(html);
|
||||
if (values.json) {
|
||||
const page = distill(html, finalUrl);
|
||||
remember(page, sessionName);
|
||||
const failure = contentFailure(contentTokens(page), htmlTokens);
|
||||
const auth = authFailure(page, finalUrl, { hadAuth });
|
||||
const failure = auth ?? contentFailure(contentTokens(page), htmlTokens);
|
||||
// Always present, so a caller can branch on the field rather than on
|
||||
// whether a field it was hoping for turned up.
|
||||
console.log(JSON.stringify({ ...page, empty: failure != null }));
|
||||
if (verbose) console.error(resources());
|
||||
if (auth) {
|
||||
if (jar) clearCookieJar(sessionName);
|
||||
noContent(finalUrl, auth);
|
||||
return;
|
||||
}
|
||||
remember(page, sessionName);
|
||||
if (failure) noContent(finalUrl, failure);
|
||||
return;
|
||||
}
|
||||
if (command === 'raw') {
|
||||
const page = distill(html, finalUrl);
|
||||
const auth = authFailure(page, finalUrl, { hadAuth });
|
||||
if (auth) {
|
||||
if (jar) clearCookieJar(sessionName);
|
||||
noContent(finalUrl, auth);
|
||||
return;
|
||||
}
|
||||
const out = values.html ? toHTML(html, finalUrl) : toMarkdown(html, finalUrl);
|
||||
const outTokens = estimateTokens(out);
|
||||
console.log(out);
|
||||
@@ -187,7 +245,13 @@ async function main() {
|
||||
return;
|
||||
}
|
||||
const page = distill(html, finalUrl);
|
||||
const failure = contentFailure(contentTokens(page), htmlTokens);
|
||||
const auth = authFailure(page, finalUrl, { hadAuth });
|
||||
const failure = auth ?? contentFailure(contentTokens(page), htmlTokens);
|
||||
if (auth) {
|
||||
if (jar) clearCookieJar(sessionName);
|
||||
noContent(finalUrl, auth);
|
||||
return;
|
||||
}
|
||||
const { text, stats } = render(page, { budget });
|
||||
remember(page, sessionName, stats.next);
|
||||
console.log(text);
|
||||
|
||||
+386
@@ -0,0 +1,386 @@
|
||||
/**
|
||||
* Per-session cookie jar, stored in a sidecar file next to the page-state
|
||||
* JSON. Credentials never live in the session snapshot itself.
|
||||
*/
|
||||
|
||||
import { join } from 'node:path';
|
||||
import { mkdirSync, readFileSync, writeFileSync, unlinkSync, readdirSync, statSync, chmodSync } from 'node:fs';
|
||||
|
||||
import { sessionDir, assertSafeName } from './session.js';
|
||||
|
||||
const DEFAULT_EXPIRES_MS = 60 * 60 * 1000; // 1h
|
||||
export { DEFAULT_EXPIRES_MS };
|
||||
const FILE_MODE = 0o600;
|
||||
|
||||
/** Returned by loadCookieJar when a sidecar existed but its session ceiling had passed. */
|
||||
export const JAR_EXPIRED = Object.freeze({ expired: true });
|
||||
|
||||
let purged = false;
|
||||
|
||||
/**
|
||||
* @param {string} name
|
||||
* @returns {string}
|
||||
*/
|
||||
export function cookieJarPath(name) {
|
||||
return join(sessionDir(), `${assertSafeName(name)}.cookies.json`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse --expires values like 1h, 30m, 2d into milliseconds.
|
||||
* @param {string} value
|
||||
* @returns {number}
|
||||
*/
|
||||
export function parseExpires(value) {
|
||||
const m = String(value).trim().match(/^(\d+(?:\.\d+)?)(h|m|d|s)?$/i);
|
||||
if (!m) throw new Error(`invalid --expires '${value}', use a duration like 1h, 30m, or 2d`);
|
||||
const n = Number(m[1]);
|
||||
const unit = (m[2] ?? 'h').toLowerCase();
|
||||
const mult = unit === 'd' ? 86_400_000 : unit === 'h' ? 3_600_000 : unit === 'm' ? 60_000 : 1000;
|
||||
return n * mult;
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {Object} Cookie
|
||||
* @property {string} name
|
||||
* @property {string} value
|
||||
* @property {string} domain
|
||||
* @property {string} path
|
||||
* @property {boolean} [secure]
|
||||
* @property {boolean} [httpOnly]
|
||||
* @property {string} [expires] - ISO timestamp
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} CookieJar
|
||||
* @property {string} expiresAt - ISO session ceiling
|
||||
* @property {Cookie[]} cookies
|
||||
*/
|
||||
|
||||
/**
|
||||
* Seed a jar from a Cookie request header string.
|
||||
* @param {string} header
|
||||
* @param {string} domain
|
||||
* @param {{ expiresMs?: number }} [opts]
|
||||
* @returns {CookieJar}
|
||||
*/
|
||||
export function jarFromCookieHeader(header, domain, { expiresMs = DEFAULT_EXPIRES_MS } = {}) {
|
||||
const host = domain.toLowerCase().replace(/^\./, '');
|
||||
if (!host || host.includes('/') || host.includes(':')) {
|
||||
throw new Error('--domain must be a hostname like example.com');
|
||||
}
|
||||
/** @type {Cookie[]} */
|
||||
const cookies = [];
|
||||
for (const part of header.split(';')) {
|
||||
const trimmed = part.trim();
|
||||
if (!trimmed) continue;
|
||||
const eq = trimmed.indexOf('=');
|
||||
if (eq <= 0) continue;
|
||||
const name = trimmed.slice(0, eq).trim();
|
||||
const value = trimmed.slice(eq + 1).trim();
|
||||
if (!name) continue;
|
||||
cookies.push({ name, value, domain: host, path: '/' });
|
||||
}
|
||||
if (!cookies.length) throw new Error('no cookies found in --cookie string');
|
||||
return {
|
||||
expiresAt: new Date(Date.now() + expiresMs).toISOString(),
|
||||
cookies,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {CookieJar} jar
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isSessionExpired(jar) {
|
||||
return Date.now() >= Date.parse(jar.expiresAt);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Cookie} cookie
|
||||
* @param {string} sessionCeiling
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function isCookieExpired(cookie, sessionCeiling) {
|
||||
const ceiling = Date.parse(sessionCeiling);
|
||||
if (Date.now() >= ceiling) return true;
|
||||
if (!cookie.expires) return false;
|
||||
const exp = Date.parse(cookie.expires);
|
||||
if (Number.isNaN(exp)) return false;
|
||||
return exp <= Date.now() || exp > ceiling;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {CookieJar} jar
|
||||
* @returns {CookieJar}
|
||||
*/
|
||||
function pruneExpiredCookies(jar) {
|
||||
return {
|
||||
...jar,
|
||||
cookies: jar.cookies.filter((c) => !isCookieExpired(c, jar.expiresAt)),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Read this session's jar before the process-wide purge can erase the evidence
|
||||
* of expiry; then sweep other stale sidecars.
|
||||
* @param {string} name
|
||||
* @returns {CookieJar | typeof JAR_EXPIRED | null}
|
||||
*/
|
||||
export function loadCookieJar(name) {
|
||||
let result = null;
|
||||
try {
|
||||
const jar = /** @type {CookieJar} */ (JSON.parse(readFileSync(cookieJarPath(name), 'utf8')));
|
||||
if (!jar?.expiresAt || !Array.isArray(jar.cookies)) {
|
||||
result = null;
|
||||
} else if (isSessionExpired(jar)) {
|
||||
clearCookieJar(name);
|
||||
result = JAR_EXPIRED;
|
||||
} else {
|
||||
const pruned = pruneExpiredCookies(jar);
|
||||
if (!pruned.cookies.length) {
|
||||
clearCookieJar(name);
|
||||
result = JAR_EXPIRED;
|
||||
} else {
|
||||
result = pruned;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
result = null;
|
||||
}
|
||||
ensurePurged();
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} name
|
||||
* @param {CookieJar} jar
|
||||
*/
|
||||
export function saveCookieJar(name, jar) {
|
||||
mkdirSync(sessionDir(), { recursive: true });
|
||||
// writeFileSync only sets the mode on create, so an existing sidecar has its
|
||||
// owner-only mode reasserted on every save.
|
||||
const path = cookieJarPath(name);
|
||||
writeFileSync(path, JSON.stringify(pruneExpiredCookies(jar)), { mode: FILE_MODE });
|
||||
chmodSync(path, FILE_MODE);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} name
|
||||
*/
|
||||
export function clearCookieJar(name) {
|
||||
try {
|
||||
unlinkSync(cookieJarPath(name));
|
||||
} catch {
|
||||
// missing file is fine
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete expired sidecar jars under OC_HOME/sessions.
|
||||
*/
|
||||
export function purgeExpiredJars() {
|
||||
let dir;
|
||||
try {
|
||||
dir = sessionDir();
|
||||
readdirSync(dir);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
for (const file of readdirSync(dir)) {
|
||||
if (!file.endsWith('.cookies.json')) continue;
|
||||
const name = file.slice(0, -'.cookies.json'.length);
|
||||
try {
|
||||
const jar = /** @type {CookieJar} */ (JSON.parse(readFileSync(join(dir, file), 'utf8')));
|
||||
if (isSessionExpired(jar)) clearCookieJar(name);
|
||||
} catch {
|
||||
try { unlinkSync(join(dir, file)); } catch {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function ensurePurged() {
|
||||
if (purged) return;
|
||||
purged = true;
|
||||
purgeExpiredJars();
|
||||
}
|
||||
|
||||
/** Reset purge-once guard (tests). */
|
||||
export function _resetPurgeGuard() {
|
||||
purged = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* RFC 6265 domain matching (host-only and domain cookies).
|
||||
* @param {Cookie} cookie
|
||||
* @param {string} host
|
||||
*/
|
||||
function domainMatches(cookie, host) {
|
||||
const d = cookie.domain.toLowerCase().replace(/^\./, '');
|
||||
return host === d || host.endsWith(`.${d}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Cookie} cookie
|
||||
* @param {string} path
|
||||
*/
|
||||
function pathMatches(cookie, path) {
|
||||
const p = cookie.path || '/';
|
||||
if (path === p) return true;
|
||||
if (!path.startsWith(p)) return false;
|
||||
return p.endsWith('/') || path[p.length] === '/';
|
||||
}
|
||||
|
||||
/**
|
||||
* Cookies to send for a request URL.
|
||||
* @param {CookieJar} jar
|
||||
* @param {string} urlStr
|
||||
* @returns {string | undefined}
|
||||
*/
|
||||
export function cookieHeaderFor(jar, urlStr) {
|
||||
let url;
|
||||
try {
|
||||
url = new URL(urlStr);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
const host = url.hostname.toLowerCase();
|
||||
const path = url.pathname || '/';
|
||||
const secure = url.protocol === 'https:';
|
||||
const active = jar.cookies.filter((c) => {
|
||||
if (isCookieExpired(c, jar.expiresAt)) return false;
|
||||
if (c.secure && !secure) return false;
|
||||
return domainMatches(c, host) && pathMatches(c, path);
|
||||
});
|
||||
if (!active.length) return undefined;
|
||||
return active.map((c) => `${c.name}=${c.value}`).join('; ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one Set-Cookie header value.
|
||||
* @param {string} header
|
||||
* @param {string} requestUrl
|
||||
* @returns {Cookie | null}
|
||||
*/
|
||||
export function parseSetCookie(header, requestUrl) {
|
||||
const parts = header.split(';').map((p) => p.trim()).filter(Boolean);
|
||||
if (!parts.length) return null;
|
||||
const eq = parts[0].indexOf('=');
|
||||
if (eq <= 0) return null;
|
||||
const name = parts[0].slice(0, eq).trim();
|
||||
const value = parts[0].slice(eq + 1).trim();
|
||||
if (!name) return null;
|
||||
|
||||
const url = new URL(requestUrl);
|
||||
/** @type {Cookie} */
|
||||
const cookie = {
|
||||
name,
|
||||
value,
|
||||
domain: url.hostname.toLowerCase(),
|
||||
path: '/',
|
||||
};
|
||||
|
||||
for (const attr of parts.slice(1)) {
|
||||
const sep = attr.indexOf('=');
|
||||
const key = (sep === -1 ? attr : attr.slice(0, sep)).trim().toLowerCase();
|
||||
const val = sep === -1 ? '' : attr.slice(sep + 1).trim();
|
||||
// The Domain attribute is deliberately ignored: cookies learned from a
|
||||
// response are pinned host-only to the host that set them. Honoring Domain
|
||||
// safely needs the Public Suffix List (a site could otherwise scope a
|
||||
// cookie to '.com' and have it sent to every site under it), and a PSL is a
|
||||
// dependency this project will not take. User-seeded cookies still scope by
|
||||
// the --domain they pass, which is trusted input.
|
||||
if (key === 'path') {
|
||||
cookie.path = val || '/';
|
||||
} else if (key === 'secure') {
|
||||
cookie.secure = true;
|
||||
} else if (key === 'httponly') {
|
||||
cookie.httpOnly = true;
|
||||
} else if (key === 'max-age') {
|
||||
const age = Number(val);
|
||||
if (Number.isFinite(age)) {
|
||||
cookie.expires = new Date(Date.now() + age * 1000).toISOString();
|
||||
}
|
||||
} else if (key === 'expires') {
|
||||
const exp = Date.parse(val);
|
||||
if (!Number.isNaN(exp)) cookie.expires = new Date(exp).toISOString();
|
||||
}
|
||||
}
|
||||
return cookie;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract Set-Cookie header values from a fetch/impers response.
|
||||
* @param {any} res
|
||||
* @returns {string[]}
|
||||
*/
|
||||
export function getSetCookieHeaders(res) {
|
||||
if (typeof res.headers?.getSetCookie === 'function') {
|
||||
const list = res.headers.getSetCookie();
|
||||
if (Array.isArray(list) && list.length) return list;
|
||||
}
|
||||
const raw = res.headers?.get?.('set-cookie');
|
||||
if (!raw) return [];
|
||||
// Undici joins with ", " but Expires contains commas: split only on ", " followed by a token=
|
||||
return raw.split(/,\s(?=[\w!#$%&'*+\-.^`|~]+=)/);
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply Set-Cookie headers to the jar for a response URL.
|
||||
* @param {CookieJar} jar
|
||||
* @param {string} url
|
||||
* @param {string[]} setCookieHeaders
|
||||
* @returns {CookieJar}
|
||||
*/
|
||||
export function storeFromResponse(jar, url, setCookieHeaders) {
|
||||
if (!setCookieHeaders.length) return jar;
|
||||
let cookies = [...jar.cookies];
|
||||
for (const header of setCookieHeaders) {
|
||||
const parsed = parseSetCookie(header, url);
|
||||
if (!parsed) continue;
|
||||
// Max-Age=0 or Expires in the past deletes the cookie
|
||||
if (parsed.expires && Date.parse(parsed.expires) <= Date.now()) {
|
||||
cookies = cookies.filter((c) => !(c.name === parsed.name && domainMatches(c, parsed.domain)));
|
||||
continue;
|
||||
}
|
||||
cookies = cookies.filter((c) => !(c.name === parsed.name && domainMatches(c, parsed.domain)));
|
||||
if (parsed.expires) {
|
||||
const ceiling = Date.parse(jar.expiresAt);
|
||||
const exp = Date.parse(parsed.expires);
|
||||
if (exp > ceiling) parsed.expires = jar.expiresAt;
|
||||
}
|
||||
cookies.push(parsed);
|
||||
}
|
||||
return { ...jar, cookies };
|
||||
}
|
||||
|
||||
/**
|
||||
* Mutable jar wrapper for fetch to update in place.
|
||||
* @param {string} sessionName
|
||||
* @param {CookieJar} data
|
||||
*/
|
||||
export function createJarHandle(sessionName, data) {
|
||||
let jar = data;
|
||||
return {
|
||||
sessionName,
|
||||
cookieHeaderFor(url) {
|
||||
return cookieHeaderFor(jar, url);
|
||||
},
|
||||
storeFromResponse(url, headers) {
|
||||
jar = storeFromResponse(jar, url, headers);
|
||||
},
|
||||
toJSON() {
|
||||
return jar;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} name
|
||||
* @param {string} header
|
||||
* @param {string} domain
|
||||
* @param {{ expiresMs?: number }} [opts]
|
||||
*/
|
||||
export function loginCookieJar(name, header, domain, opts) {
|
||||
const jar = jarFromCookieHeader(header, domain, opts);
|
||||
saveCookieJar(name, jar);
|
||||
}
|
||||
+51
-13
@@ -13,6 +13,8 @@ import https from 'node:https';
|
||||
import net from 'node:net';
|
||||
import tls from 'node:tls';
|
||||
|
||||
import { getSetCookieHeaders } from './cookies.js';
|
||||
|
||||
// The fetch fallback can't fake a TLS fingerprint like impers does, but it
|
||||
// should at least send the same Chrome identity in its headers.
|
||||
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36';
|
||||
@@ -307,6 +309,14 @@ function wrapNodeResponse(res, url) {
|
||||
if (v == null) return null;
|
||||
return Array.isArray(v) ? v.join(', ') : v;
|
||||
},
|
||||
// Node keeps Set-Cookie as an array of raw values. Expose it unjoined so
|
||||
// the cookie jar reads each header intact: a comma in an Expires date
|
||||
// makes the joined form ambiguous to split back apart.
|
||||
getSetCookie() {
|
||||
const v = res.headers['set-cookie'];
|
||||
if (v == null) return [];
|
||||
return Array.isArray(v) ? v : [v];
|
||||
},
|
||||
};
|
||||
const text = () => new Promise((resolve, reject) => {
|
||||
const chunks = [];
|
||||
@@ -396,7 +406,9 @@ function httpsViaConnect(target, proxy, headers, tlsOpts = {}) {
|
||||
// tls.connect already opened the tunnel. https.request would wrap TLS
|
||||
// again, and the origin would see a second ClientHello as garbage.
|
||||
// SNI is a hostname; an IP literal is only used for the cert check.
|
||||
const hostname = target.hostname;
|
||||
// URL.hostname keeps the brackets around an IPv6 literal ("[::1]"), which
|
||||
// tls.connect would treat as a DNS name; strip them like assertSafeTarget.
|
||||
const hostname = target.hostname.replace(/^\[|\]$/g, '');
|
||||
const tlsSocket = tls.connect({
|
||||
socket,
|
||||
host: hostname,
|
||||
@@ -445,15 +457,16 @@ export function proxyGet(url, proxy, headers = {}, tlsOpts = {}) {
|
||||
/**
|
||||
* Fetch a page.
|
||||
* @param {string} url - with or without a scheme, https is assumed
|
||||
* @param {{ jar?: { cookieHeaderFor(url: string): string|undefined, storeFromResponse(url: string, headers: string[]): void } }} [opts]
|
||||
* @returns {Promise<{url: string, html: string, status: number, via: string}>}
|
||||
* final URL after redirects, the body, the HTTP status, and which client
|
||||
* identity got the page (impers:chrome, impers:firefox, or fetch)
|
||||
*/
|
||||
export async function fetchPage(url) {
|
||||
export async function fetchPage(url, { jar } = {}) {
|
||||
const target = /^https?:\/\//i.test(url) ? url : `https://${url}`;
|
||||
await assertSafeTarget(target);
|
||||
const impers = await loadImpers();
|
||||
return impers ? viaImpers(impers, target) : viaFetch(target);
|
||||
return impers ? viaImpers(impers, target, jar) : viaFetch(target, jar);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -470,11 +483,12 @@ export async function fetchPage(url) {
|
||||
* @param {string} start
|
||||
* @returns {Promise<{res: any, url: string}>} the first non-redirect response
|
||||
*/
|
||||
export async function followRedirects(get, start) {
|
||||
export async function followRedirects(get, start, { onResponse } = {}) {
|
||||
let current = start;
|
||||
for (let i = 0; ; i++) {
|
||||
if (i > MAX_REDIRECTS) throw new Error(`too many redirects for ${start}`);
|
||||
const res = await get(current);
|
||||
onResponse?.(current, res);
|
||||
const status = res.status ?? res.statusCode ?? 0;
|
||||
const location = res.headers.get('location');
|
||||
if (status >= 300 && status < 400 && location) {
|
||||
@@ -492,17 +506,38 @@ const FETCH_HEADERS = {
|
||||
'accept-language': 'en-US,en;q=0.9',
|
||||
};
|
||||
|
||||
async function viaImpers(impers, target) {
|
||||
function mergeHeaders(base, extra) {
|
||||
return extra ? { ...base, ...extra } : base;
|
||||
}
|
||||
|
||||
function jarHeaders(jar, url, base) {
|
||||
if (!jar) return base;
|
||||
const cookie = jar.cookieHeaderFor(url);
|
||||
return cookie ? mergeHeaders(base, { cookie }) : base;
|
||||
}
|
||||
|
||||
function captureSetCookie(jar, url, res) {
|
||||
if (!jar) return;
|
||||
jar.storeFromResponse(url, getSetCookieHeaders(res));
|
||||
}
|
||||
|
||||
async function viaImpers(impers, target, jar) {
|
||||
// Some sites (Reddit) 403 the chrome fingerprint but accept firefox, so a
|
||||
// blocked first attempt gets one cheap retry with a second identity.
|
||||
const asking = (impersonate) => (url) =>
|
||||
impers.get(url, { impersonate, allowRedirects: false, proxy: resolveProxy(url) ?? '' });
|
||||
impers.get(url, {
|
||||
impersonate,
|
||||
allowRedirects: false,
|
||||
proxy: resolveProxy(url) ?? '',
|
||||
headers: jarHeaders(jar, url, {}),
|
||||
});
|
||||
const onResponse = (url, res) => captureSetCookie(jar, url, res);
|
||||
let via = 'impers:chrome';
|
||||
let { res } = await followRedirects(asking('chrome'), target);
|
||||
let { res } = await followRedirects(asking('chrome'), target, { onResponse });
|
||||
let status = res.status ?? res.statusCode ?? 0;
|
||||
if (status >= 400) {
|
||||
via = 'impers:firefox';
|
||||
({ res } = await followRedirects(asking('firefox'), target));
|
||||
({ res } = await followRedirects(asking('firefox'), target, { onResponse }));
|
||||
status = res.status ?? res.statusCode ?? 0;
|
||||
}
|
||||
if (status >= 400) throw new Error(`fetch failed: ${status} for ${target}`);
|
||||
@@ -511,13 +546,16 @@ async function viaImpers(impers, target) {
|
||||
return { url: res.url ?? target, html, status, via };
|
||||
}
|
||||
|
||||
async function viaFetch(target) {
|
||||
const { res, url: current } = await followRedirects((url) => {
|
||||
async function viaFetch(target, jar) {
|
||||
const get = (url) => {
|
||||
const proxy = resolveProxy(url);
|
||||
const headers = jarHeaders(jar, url, FETCH_HEADERS);
|
||||
return proxy
|
||||
? proxyGet(url, proxy, FETCH_HEADERS)
|
||||
: fetch(url, { redirect: 'manual', headers: FETCH_HEADERS });
|
||||
}, target);
|
||||
? proxyGet(url, proxy, headers)
|
||||
: fetch(url, { redirect: 'manual', headers });
|
||||
};
|
||||
const onResponse = (url, res) => captureSetCookie(jar, url, res);
|
||||
const { res, url: current } = await followRedirects(get, target, { onResponse });
|
||||
if (!res.ok) {
|
||||
throw new Error(`fetch failed: ${res.status} ${res.statusText} for ${current}`);
|
||||
}
|
||||
|
||||
+28
-4
@@ -1,7 +1,8 @@
|
||||
/**
|
||||
* Sessions are plain JSON files on disk, one per name: the current URL, the
|
||||
* distilled blocks of the page it holds, how far the last render got through
|
||||
* them, and a short history. No daemon, no background process, no cookies yet.
|
||||
* them, and a short history. No daemon, no background process; cookies live in
|
||||
* a separate sidecar file (see cookies.js).
|
||||
*
|
||||
* The file exists so `oc do <n>` can follow a link the compact view never
|
||||
* printed the URL of. Hiding URLs is what makes `oc open` cheap; this is what
|
||||
@@ -11,18 +12,35 @@
|
||||
|
||||
import { homedir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { mkdirSync, readFileSync, writeFileSync, chmodSync } from 'node:fs';
|
||||
|
||||
export const DEFAULT_SESSION = 'default';
|
||||
|
||||
// OC_HOME relocates the whole state directory, for sandboxes, CI, and tests.
|
||||
export const sessionDir = () => join(process.env.OC_HOME ?? join(homedir(), '.only-cli'), 'sessions');
|
||||
|
||||
// A session name is interpolated straight into a filename, and the cookie
|
||||
// sidecar it names now holds real credentials, so a name that is a path
|
||||
// (absolute, or with a separator or '..') could write or delete a file outside
|
||||
// the store. Names are user-facing labels, so this charset loses nothing real.
|
||||
const SAFE_NAME = /^[A-Za-z0-9._-]+$/;
|
||||
|
||||
/**
|
||||
* @param {string} name
|
||||
* @returns {string} the same name, once it is known to be a safe filename
|
||||
*/
|
||||
export function assertSafeName(name) {
|
||||
if (typeof name !== 'string' || name === '.' || name === '..' || !SAFE_NAME.test(name)) {
|
||||
throw new Error(`invalid session name '${name}', use letters, numbers, '.', '-', or '_'`);
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} name
|
||||
* @returns {string}
|
||||
*/
|
||||
export const sessionPath = (name) => join(sessionDir(), `${name}.json`);
|
||||
export const sessionPath = (name) => join(sessionDir(), `${assertSafeName(name)}.json`);
|
||||
|
||||
// Search engines and link aggregators wrap outbound links in a tracking
|
||||
// redirector whose landing page is a script, not content, so following one
|
||||
@@ -136,7 +154,13 @@ export function handleNumbers(state) {
|
||||
*/
|
||||
export function saveSession(name, state) {
|
||||
mkdirSync(sessionDir(), { recursive: true });
|
||||
writeFileSync(sessionPath(name), JSON.stringify(state));
|
||||
// A snapshot of an authenticated page holds that page's text, so it gets the
|
||||
// same owner-only mode as the cookie sidecar. writeFileSync only sets the
|
||||
// mode on create, so a snapshot left world-readable by an older version is
|
||||
// tightened explicitly on the next save.
|
||||
const path = sessionPath(name);
|
||||
writeFileSync(path, JSON.stringify(state), { mode: 0o600 });
|
||||
chmodSync(path, 0o600);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user