mirror of
https://github.com/only-cli/oc.git
synced 2026-09-15 10:40:56 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
10c22e748a | ||
|
|
a1efa0a758 | ||
|
|
a01f497b54 | ||
|
|
24712ae19c | ||
|
|
b1cbf2302d | ||
|
|
5ed47af59b | ||
|
|
962356cb91 | ||
|
|
5b30fbfbc1 | ||
|
|
431dad3abd | ||
|
|
c344e639b6 | ||
|
|
3b44bc944b | ||
|
|
21f9955e15 | ||
|
|
e7a36a67f0 | ||
|
|
0ea9f4cbda | ||
|
|
ff5f376063 | ||
|
|
c5426c246b | ||
|
|
e046d308c5 | ||
|
|
3ea03d341c | ||
|
|
ca9f46ca91 | ||
|
|
73ac4e863a | ||
|
|
10bf7b3b82 | ||
|
|
cbf2de39de | ||
|
|
5657d60771 | ||
|
|
e1ff0f63c4 | ||
|
|
4aabae8340 | ||
|
|
227d6928c5 | ||
|
|
7f3e626a4d | ||
|
|
376851c6fa | ||
|
|
21566143d1 | ||
|
|
dfc9e25436 | ||
|
|
b7c57e6e0b | ||
|
|
b02dce55af | ||
|
|
65322bc5b6 | ||
|
|
ba49303527 | ||
|
|
7f1f109b8e | ||
|
|
2a5f97204e | ||
|
|
ccc0cf4476 | ||
|
|
c12ecf060d | ||
|
|
59964c5dca | ||
|
|
2c8b8f20c4 | ||
|
|
c3952be485 | ||
|
|
0cf453d02c | ||
|
|
8d4c4ca4eb | ||
|
|
625048a7d1 | ||
|
|
a274761a7b | ||
|
|
cad6fb4596 |
@@ -6,7 +6,7 @@
|
||||
"name": "only-cli",
|
||||
"source": { "source": "github", "repo": "only-cli/oc" },
|
||||
"description": "Browse websites from the terminal in a few hundred tokens",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"homepage": "https://github.com/only-cli/oc",
|
||||
"license": "MIT"
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"name": "only-cli",
|
||||
"description": "Browse websites from the terminal in a few hundred tokens",
|
||||
"version": "0.5.0"
|
||||
"version": "0.5.3"
|
||||
}
|
||||
|
||||
@@ -11,3 +11,8 @@ updates:
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
groups:
|
||||
# init and analyze must run the same version, so bump them in one PR.
|
||||
codeql-action:
|
||||
patterns:
|
||||
- github/codeql-action*
|
||||
|
||||
@@ -25,10 +25,10 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- uses: github/codeql-action/init@42947a340483f03ba47bb1a039b2c519aab3df85 # v3.37.8
|
||||
- uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9
|
||||
with:
|
||||
languages: javascript-typescript
|
||||
|
||||
- uses: github/codeql-action/analyze@42947a340483f03ba47bb1a039b2c519aab3df85 # v3.37.8
|
||||
- uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9
|
||||
with:
|
||||
category: "/language:javascript-typescript"
|
||||
|
||||
@@ -14,4 +14,4 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- uses: actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48 # v4.9.0
|
||||
- uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0
|
||||
|
||||
@@ -26,18 +26,18 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: ossf/scorecard-action@55891bbd73f2425e97637d96e306fc9d491d0b21 # v2.4.4
|
||||
- uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4
|
||||
with:
|
||||
results_file: results.sarif
|
||||
results_format: sarif
|
||||
publish_results: true
|
||||
|
||||
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: SARIF file
|
||||
path: results.sarif
|
||||
retention-days: 5
|
||||
|
||||
- uses: github/codeql-action/upload-sarif@42947a340483f03ba47bb1a039b2c519aab3df85 # v3.37.8
|
||||
- uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9
|
||||
with:
|
||||
sarif_file: results.sarif
|
||||
|
||||
@@ -3,6 +3,77 @@
|
||||
Notable changes per release. Releases before 0.4.0 are listed at
|
||||
[github.com/only-cli/oc/releases](https://github.com/only-cli/oc/releases).
|
||||
|
||||
## Unreleased
|
||||
|
||||
### Fixed
|
||||
|
||||
- A feed entry's title is now the link to the entry, so `oc do <n>` on a post
|
||||
in a subreddit feed opens it. The link used to sit beside the byline as an
|
||||
anchor labelled `open`, the same label on every entry, and the
|
||||
repeated-controls filter hid them all on any feed with five or more entries,
|
||||
which left nothing in a listing that led anywhere: an agent asked to open
|
||||
the first post's comments got the heading text back, refetched the feed
|
||||
looking for the link, and met Reddit's 429 (#59).
|
||||
|
||||
### Changed
|
||||
|
||||
- `oc open` on a reddit.com front page, subreddit, post, user or search URL
|
||||
fetches the matching www.reddit.com Atom feed when no login session is
|
||||
held, since the HTML page ends at a login wall for a logged-out reader.
|
||||
Following a post out of a feed lands on its comments feed instead of the
|
||||
wall. A URL that is already a feed, any other reddit.com path, and any
|
||||
request carrying reddit.com cookies are fetched as asked (#59).
|
||||
|
||||
## 0.5.3
|
||||
|
||||
### Changed
|
||||
|
||||
- Requests to reddit.com present the Firefox fingerprint first and fall back
|
||||
to Chrome, the reverse of every other site. Reddit's edge answers the Chrome
|
||||
fingerprint with a 403 or a 429 while letting Firefox through, and since it
|
||||
allows anonymous readers about ten requests a minute per address, the wasted
|
||||
Chrome attempt was costing a real share of that budget on every read (#52).
|
||||
|
||||
## 0.5.2
|
||||
|
||||
### Changed
|
||||
|
||||
- `oc reddit` reads the Atom feeds on www.reddit.com instead of old.reddit.com
|
||||
pages. Reddit has sent every logged-out old.reddit.com request to a login
|
||||
page since 30 June 2026, and the `.json` views on www.reddit.com have
|
||||
answered 403 to anything without an OAuth token since 30 May, whatever the
|
||||
User-Agent or TLS fingerprint. The feeds still answer, so `sub`, `post`,
|
||||
`user`, and `search` point at them, and `new <name>` and `top <name>` join
|
||||
the verbs. A subreddit renders in about 480 tokens and a thread with 22
|
||||
comments in about 1,000. The feeds carry no scores or comment counts, and
|
||||
anonymous reddit.com allows roughly ten requests a minute per address, so a
|
||||
burst of shortcuts ends in a 429 that takes minutes to clear. (#52)
|
||||
|
||||
## 0.5.1
|
||||
|
||||
### Added
|
||||
|
||||
- `find <query>` in every `actions:` footer, after `do <n>` and before
|
||||
`read <n>`, the order the skill's "going further, cheapest first" list
|
||||
already gives. One command lands on the block that matters, where `read`
|
||||
needs the right number first and `next` pages toward it. The entry costs 3
|
||||
or 4 tokens per render. (#46)
|
||||
|
||||
### Fixed
|
||||
|
||||
- `oc open` no longer dies with `Impersonating chrome150 is not supported` on
|
||||
machines where impers loads a system copy of libcurl-impersonate older than
|
||||
v2.1.0, which predates the fingerprint the `chrome` alias resolves to. A
|
||||
refused fingerprint now downgrades the same way a blocked response already
|
||||
did: chrome falls back to firefox, and when both identities are refused the
|
||||
plain fetch transport still gets the page. Any other impers failure
|
||||
propagates unchanged, and installs where impers works keep the newest chrome
|
||||
fingerprint. (#40)
|
||||
- The `actions:` footer no longer offers `fill <n> <text>` and `submit` on
|
||||
pages with an input. Both are still planned, and following the footer's
|
||||
own suggestion always failed. A test now keeps every footer free of
|
||||
commands that are not available yet. (#44)
|
||||
|
||||
## 0.5.0
|
||||
|
||||
### Added
|
||||
|
||||
+10
-1
@@ -23,7 +23,7 @@ Node 20+. Tests run fully offline against saved fixtures in `tests/pages/`, so a
|
||||
|
||||
## Code style
|
||||
|
||||
Plain JavaScript, ESM, JSDoc types, no build step. Match the code around you. Comments explain constraints and trade-offs, not what the next line does; if a comment restates the code, delete it. Small functions, few files: if you are adding a new file to `src/`, pause and check whether the logic belongs in one of the six that exist.
|
||||
Plain JavaScript, ESM, JSDoc types, no build step. Match the code around you. Comments explain constraints and trade-offs, not what the next line does; if a comment restates the code, delete it. Small functions, few files: if you are adding a new file to `src/`, pause and check whether the logic belongs in one of the files that already exist.
|
||||
|
||||
## Writing style
|
||||
|
||||
@@ -37,6 +37,15 @@ A definition needs no wiring: `oc <site> <verb> [args]` resolves against `clis/*
|
||||
|
||||
Per-site CLIs live in `clis/`, one JSON file per domain: the domain plus a `commands` map of name, help line, and URL template, exactly like the existing files. Keep it under 50 lines, no OpenAPI. If the site has a public JSON API, point the commands at that instead of the HTML pages. If your definition needs logic, it is trying to become an adapter, and the answer is to improve the generic engine instead.
|
||||
|
||||
A `search` verb deserves a moment's thought, because most documentation sites render search in the browser and hand oc an empty page. Check what the site actually ships, in this order:
|
||||
|
||||
- **A server-rendered results page.** Point `open` at it, the way `pkg.go.dev` and Microsoft Learn's RSS endpoint do. Nothing else is needed.
|
||||
- **A public JSON endpoint behind the results page.** Use the `api` shape: `api` is the endpoint template, `page` the human URL to remember for the session, `results` the dot path to the list, `fields` the paths to each result's `title`, `url`, and `text`, and `total` the path to the count. `clis/developer.mozilla.org.json` is the model. The site's own ranking comes back as a numbered results page for a few hundred tokens.
|
||||
- **A static search index.** Sphinx sites (docs.python.org, most Read the Docs projects) publish `searchindex.js`; RDoc sites publish `js/search_index.js`; the Node.js API docs publish `all.json`. Set `sphinx`, `rdoc`, or `nodedoc` to the docs root and oc fetches the index once a day, ranks it locally, and prints only the result list. Any Sphinx or RDoc site works with no code.
|
||||
- **None of the above.** Fall back to DuckDuckGo with a baked-in filter: `"open": "https://html.duckduckgo.com/html/?q=site%3Aexample.org+{query}"`, as `rust`, `java`, `ts`, `php`, and `cpp` do. Say so in the README row ("search via DuckDuckGo") so nobody mistakes it for the site's own search.
|
||||
|
||||
Adding a fifth search backend is a change to `src/`, not to a JSON file, and needs the same case as any other new code: a site that many agents reach for, an index or endpoint that a template cannot describe, and offline tests against a saved copy of the index.
|
||||
|
||||
## Reporting bugs
|
||||
|
||||
Open an issue with the exact command, the output you got, and the output you expected. If the page is public, include the URL. If distillation mangled a page, a saved copy of the HTML as a fixture is the most useful thing you can attach.
|
||||
|
||||
@@ -12,7 +12,7 @@ $ oc open news.ycombinator.com
|
||||
[1] Show HN: I built a tiny CSV toolkit
|
||||
[2] 312 comments
|
||||
...
|
||||
actions: do <n> | read <n> | next | raw
|
||||
actions: do <n> | find <query> | read <n> | next | raw
|
||||
|
||||
$ oc do 1
|
||||
```
|
||||
@@ -27,36 +27,7 @@ If you are an LLM reading this repository, [llms.txt](llms.txt) is the short ver
|
||||
npm install -g @only-cli/oc
|
||||
```
|
||||
|
||||
Requires Node 20+. Requests impersonate Chrome via [impers](https://github.com/lexiforest/impers); falls back to native fetch if impers is unavailable.
|
||||
|
||||
### Proxies
|
||||
|
||||
Outbound fetches honor the usual environment variables, in upper or lower case, with nothing to pass on the command line:
|
||||
|
||||
```
|
||||
HTTP_PROXY=http://proxy.example:8080 # http:// targets
|
||||
HTTPS_PROXY=http://proxy.example:8080 # https:// targets, tunneled with CONNECT
|
||||
NO_PROXY=internal.example,*.corp.example # reached directly instead
|
||||
```
|
||||
|
||||
An `https://` target prefers `HTTPS_PROXY` and falls back to `HTTP_PROXY`; an `http://` target uses `HTTP_PROXY` only. A value with no scheme is read as `http://`, so `proxy.example:8080` works. Only HTTP and HTTPS proxies are supported, and another scheme such as `socks5://` is refused by name rather than silently ignored.
|
||||
|
||||
Credentials in the proxy URL are sent as `Proxy-Authorization` to the proxy and to nothing else, including across redirects:
|
||||
|
||||
```
|
||||
HTTPS_PROXY=http://user:pass@proxy.example:8080 oc open https://example.com
|
||||
```
|
||||
|
||||
`NO_PROXY` accepts an exact host, a `.suffix` or `*.suffix` pattern, a `host:port` entry, a CIDR block, and `*` for everything.
|
||||
|
||||
An `https://` page is tunneled with CONNECT and its certificate is verified the same way it would be without a proxy, so a proxy in the path cannot read or rewrite the page.
|
||||
|
||||
Two limits are worth knowing:
|
||||
|
||||
- oc does not read `ALL_PROXY`. The impers transport is libcurl underneath and reads it on its own, so a request oc treats as direct can still leave through an `ALL_PROXY`. The same holds for the `*.suffix`, `host:port`, and CIDR forms of `NO_PROXY`, which libcurl does not parse. Set `HTTP_PROXY` and `HTTPS_PROXY` explicitly and keep `NO_PROXY` to plain host and suffix entries when the two need to agree.
|
||||
- An IPv6 literal target over HTTPS does not currently work through a proxy.
|
||||
|
||||
Private and internal addresses are refused whether or not a proxy is set. With a proxy configured, a hostname that does not resolve locally is refused too, because the proxy would otherwise resolve it on a network oc cannot see. A name that resolves publicly for oc and internally for the proxy (split horizon DNS) is not something oc can detect, so a proxy is trusted to enforce its own egress policy.
|
||||
Requires Node 20+. Requests impersonate Chrome via [impers](https://github.com/lexiforest/impers). When a site, or the local copy of libcurl-impersonate, refuses the Chrome identity, the request downgrades to Firefox, and to native fetch when impers is unavailable or refuses both.
|
||||
|
||||
### Agent skill
|
||||
|
||||
@@ -103,30 +74,6 @@ oc logout [session] forget a session: cookies and saved page
|
||||
|
||||
Flags: `--budget <tokens>` (default 500), `--json`, `--html` (raw as cleaned HTML), `--session <name>`, `--verbose`/`-v` (metrics on stderr, or export `OC_VERBOSE=1`).
|
||||
|
||||
### Authenticated sessions
|
||||
|
||||
Pages behind a login need cookies. Seed them once per session, then browse normally:
|
||||
|
||||
```bash
|
||||
printf %s "session=...; auth=..." | oc login --cookie - --domain example.com --expires 2h --session work
|
||||
oc open https://example.com/dashboard --session work
|
||||
oc logout work
|
||||
```
|
||||
|
||||
Prefer `--cookie -`, which reads the header from stdin. The flag also takes the header inline (`--cookie "session=..."`), but an argument is a live credential in `ps` for as long as `oc` runs and in your shell history afterwards.
|
||||
|
||||
Copy the `Cookie` header from your browser's devtools (Application → Cookies, or the Network tab on a request); a leading `Cookie:` is stripped for you. `--domain` is the site hostname those cookies belong to, and it has to be a real hostname: a bare TLD like `com` is refused, because the match is a suffix match and those cookies would go to every `.com` host the session ever fetched. Cookie names and values are checked at login too, so a stray control character fails there rather than deep inside the HTTP client.
|
||||
|
||||
Seeded cookies are https-only. They almost always come from an https browser session, so `oc` marks them secure and never sends them over plain `http`, including on a hop an `https` page redirects into, where you never typed the downgrade. A site that really is http-only needs `--allow-http` at login. Cookies a site sets over https are pinned the same way.
|
||||
|
||||
Cookies live in a separate sidecar file (`<session>.cookies.json`) under `~/.only-cli/sessions/`, mode `0600`, not in the page-state JSON and never in `--json` output. The default lifetime is one hour (`--expires 1h`), and a jar holds at most 50 cookies so a page cannot bloat it. When cookies expire or the site returns a login page, `oc` says so plainly (exit 2) instead of distilling the login form as content.
|
||||
|
||||
`oc logout` forgets the whole session, not just its cookies: a page saved under that name can hold the distilled text of something only the login could reach, so the snapshot goes with the jar.
|
||||
|
||||
`oc open` remembers the page it rendered in a JSON file per session under `~/.only-cli` (override with `OC_HOME`), so `oc do 3` follows `[3]` without the agent ever handling a URL. A result title on a search page is a link, so `oc do` on it opens the result rather than repeating the title. Pages longer than the budget say what they left out; `oc find`, `oc read <n>`, and `oc next` read the rest without refetching the page, and a `find` with a single match prints that region instead of the number to read it with. The budget is a target rather than a hard cap: a page that would only run a little long is printed whole rather than cut, since one extra tool call costs far more than the tokens it would have saved.
|
||||
|
||||
When a page comes back with no readable text (JavaScript-only, a consent wall, a bot challenge), `oc` says so in one line on stderr and exits 2 instead of printing a title and calling it a render. That is a different exit code from every other failure, and `--json` carries the same verdict as an `empty` field, so an agent can tell "this page has nothing on it" from "oc could not read this page" and pay for a browser only when it is worth it.
|
||||
|
||||
## Supported websites
|
||||
|
||||
Works on any mostly-static site with no per-site setup: news sites, blogs, documentation, forums, search engines. A JSON API is a page here too: `oc open` on an endpoint that answers with JSON renders one numbered item per record, keeps the fields that actually differ between items, and says once what every item shares. On top of that, `clis/` ships tuned shortcuts, so `oc hn item 4711` or `oc gh repo only-cli oc` gets there without the agent knowing how that site spells its URLs. Name the site by its short name, its bare name, or its domain (`oc hn`, `oc ycombinator`, `oc news.ycombinator.com`), and `oc sites` prints the whole list with its verbs:
|
||||
@@ -134,7 +81,7 @@ Works on any mostly-static site with no per-site setup: news sites, blogs, docum
|
||||
| website | command | shortcuts |
|
||||
| --- | --- | --- |
|
||||
| Hacker News | `oc hn` | `top`, `new`, `item <id>`, `user <name>` |
|
||||
| Reddit | `oc reddit` (via old.reddit.com) | `sub <name>`, `post <id>`, `user <name>`, `search <query>` |
|
||||
| Reddit | `oc reddit` (via the Atom feeds on www.reddit.com) | `sub <name>`, `new <name>`, `top <name>`, `post <id>`, `user <name>`, `search <query>` |
|
||||
| GitHub | `oc gh` | `repo <owner> <name>`, `user <name>`, `search <query>`, `trending`, `issues <owner> <name>` |
|
||||
| X | `oc x` | `user <name>`, `post <id>` |
|
||||
| LinkedIn | `oc linkedin` | `profile <name>`, `company <name>`, `jobs <query>` (public guest views) |
|
||||
@@ -160,40 +107,105 @@ Works on any mostly-static site with no per-site setup: news sites, blogs, docum
|
||||
|
||||
A shortcut only ever resolves to a URL and then takes the same path `oc open` does, so it changes nothing about what a page costs or how it reads. The last argument takes every word after it, so `oc ddg search claude code cli` and `oc aws search s3 lifecycle rules` need no quoting, and a path argument keeps its slashes, so `oc learn doc azure/aks/what-is-aks` reaches that page.
|
||||
|
||||
A few of these (X, Stack Overflow, YouTube, Microsoft Learn search) read pages that look login-gated or JS-only from the outside, by finding the server-rendered HTML, feed, inline data, or public API the page already ships without a login. Stack Overflow search goes through the Stack Exchange API, and each result prints its `question_id`: read one with the `question <id>` feed rather than following its link, since the question page itself answers a bot challenge instead of the question. AWS, Google Cloud, Rust, Java, TypeScript, PHP, and cppreference render docs search client-side, or as a page too bare for oc to read, so their `search` goes through DuckDuckGo with a baked-in `site:` filter instead; Go needs no such fallback, because pkg.go.dev renders its search results on the server and `oc go search` simply opens them. Python's docs are built with Sphinx, which publishes the site's full-text search index as one static file, so `oc py search` fetches that index (cached on disk for a day), ranks it locally, and prints a numbered result list; a query that names a symbol exactly, like `json.dumps`, links straight to its anchor. The same backend will work for any Sphinx site, including most Read the Docs projects. MDN also renders its search client-side, but the page gets its results from a public JSON endpoint, so `oc mdn search` asks that endpoint directly and prints the site's own ranking; that `api` shape in a site definition works for any site whose search answers as JSON. Node.js ships no search endpoint at all, but publishes its whole API reference as one static JSON file, so `oc node search` ranks that file locally the same way the Sphinx backend does, under the same day cache, and every module, class, method, property, and event heading links to its own anchor. Ruby's docs are built with RDoc, which also ships its search index as one static file, so `oc ruby search` ranks every class, method, and guide page locally the same way. PHP's manual has a lookup endpoint that sends an exact function name straight to its page, which is what `oc php fn` rides. Not supported yet: pages that only render with JavaScript and sites with hard bot challenges that expose no feed. Sites that genuinely require your account can be reached with `oc login` (bring your own cookies).
|
||||
A few of these (X, Reddit, Stack Overflow, YouTube, Microsoft Learn search) read pages that look login-gated or JS-only from the outside, by finding the server-rendered HTML, feed, inline data, or public API the page already ships without a login. Stack Overflow search goes through the Stack Exchange API, and each result prints its `question_id`: read one with the `question <id>` feed rather than following its link, since the question page itself answers a bot challenge instead of the question. Reddit goes through the Atom feeds on www.reddit.com: old.reddit.com has sent logged-out readers to a login page since June 2026 and the `.json` views answer 403 without an OAuth token, so `oc open` on a reddit.com front page, subreddit, post, user or search URL reads the matching feed instead when no login session is held, which is also how `oc do <n>` on a post in a subreddit feed reaches its comments; `oc reddit post <id>` is the same feed by hand. oc asks reddit.com with its Firefox fingerprint first, because Reddit's edge refuses the Chrome one more often than not. The feeds carry titles, authors, dates, and bodies but no scores or comment counts, and anonymous reddit.com meters each client tightly: a second request within half a minute of the first has come back 429 in testing, and a burst of Reddit shortcuts ends in refusals that take minutes to clear, so space them out. AWS, Google Cloud, Rust, Java, TypeScript, PHP, and cppreference render docs search client-side, or as a page too bare for oc to read, so their `search` goes through DuckDuckGo with a baked-in `site:` filter instead; Go needs no such fallback, because pkg.go.dev renders its search results on the server and `oc go search` simply opens them. Python's docs are built with Sphinx, which publishes the site's full-text search index as one static file, so `oc py search` fetches that index (cached on disk for a day), ranks it locally, and prints a numbered result list; a query that names a symbol exactly, like `json.dumps`, links straight to its anchor. The same backend will work for any Sphinx site, including most Read the Docs projects. MDN also renders its search client-side, but the page gets its results from a public JSON endpoint, so `oc mdn search` asks that endpoint directly and prints the site's own ranking; that `api` shape in a site definition works for any site whose search answers as JSON. Node.js ships no search endpoint at all, but publishes its whole API reference as one static JSON file, so `oc node search` ranks that file locally the same way the Sphinx backend does, under the same day cache, and every module, class, method, property, and event heading links to its own anchor. Ruby's docs are built with RDoc, which also ships its search index as one static file, so `oc ruby search` ranks every class, method, and guide page locally the same way. PHP's manual has a lookup endpoint that sends an exact function name straight to its page, which is what `oc php fn` rides. Not supported yet: pages that only render with JavaScript and sites with hard bot challenges that expose no feed. Sites that genuinely require your account can be reached with `oc login` (bring your own cookies).
|
||||
|
||||
Want a website on that list? Open a pull request, or an issue naming the site; see [CONTRIBUTING.md](CONTRIBUTING.md).
|
||||
|
||||
## Features
|
||||
|
||||
### Authenticated sessions
|
||||
|
||||
Pages behind a login need cookies. Seed them once per session, then browse normally:
|
||||
|
||||
```bash
|
||||
printf %s "session=...; auth=..." | oc login --cookie - --domain example.com --expires 2h --session work
|
||||
oc open https://example.com/dashboard --session work
|
||||
oc logout work
|
||||
```
|
||||
|
||||
Prefer `--cookie -`, which reads the header from stdin. The flag also takes the header inline (`--cookie "session=..."`), but an argument is a live credential in `ps` for as long as `oc` runs and in your shell history afterwards.
|
||||
|
||||
Copy the `Cookie` header from your browser's devtools (Application → Cookies, or the Network tab on a request); a leading `Cookie:` is stripped for you. `--domain` is the site hostname those cookies belong to, and it has to be a real hostname: a bare TLD like `com` is refused, because the match is a suffix match and those cookies would go to every `.com` host the session ever fetched. Cookie names and values are checked at login too, so a stray control character fails there rather than deep inside the HTTP client.
|
||||
|
||||
Seeded cookies are https-only. They almost always come from an https browser session, so `oc` marks them secure and never sends them over plain `http`, including on a hop an `https` page redirects into, where you never typed the downgrade. A site that really is http-only needs `--allow-http` at login. Cookies a site sets over https are pinned the same way.
|
||||
|
||||
Cookies live in a separate sidecar file (`<session>.cookies.json`) under `~/.only-cli/sessions/`, mode `0600`, not in the page-state JSON and never in `--json` output. The default lifetime is one hour (`--expires 1h`), and a jar holds at most 50 cookies so a page cannot bloat it. When cookies expire or the site returns a login page, `oc` says so plainly (exit 2) instead of distilling the login form as content.
|
||||
|
||||
`oc logout` forgets the whole session, not just its cookies: a page saved under that name can hold the distilled text of something only the login could reach, so the snapshot goes with the jar.
|
||||
|
||||
`oc open` remembers the page it rendered in a JSON file per session under `~/.only-cli` (override with `OC_HOME`), so `oc do 3` follows `[3]` without the agent ever handling a URL. A result title on a search page is a link, so `oc do` on it opens the result rather than repeating the title. Pages longer than the budget say what they left out; `oc find`, `oc read <n>`, and `oc next` read the rest without refetching the page, and a `find` with a single match prints that region instead of the number to read it with. The budget is a target rather than a hard cap: a page that would only run a little long is printed whole rather than cut, since one extra tool call costs far more than the tokens it would have saved.
|
||||
|
||||
When a page comes back with no readable text (JavaScript-only, a consent wall, a bot challenge), `oc` says so in one line on stderr and exits 2 instead of printing a title and calling it a render. That is a different exit code from every other failure, and `--json` carries the same verdict as an `empty` field, so an agent can tell "this page has nothing on it" from "oc could not read this page" and pay for a browser only when it is worth it.
|
||||
|
||||
### Proxies
|
||||
|
||||
Outbound fetches honor the usual environment variables, in upper or lower case, with nothing to pass on the command line:
|
||||
|
||||
```
|
||||
HTTP_PROXY=http://proxy.example:8080 # http:// targets
|
||||
HTTPS_PROXY=http://proxy.example:8080 # https:// targets, tunneled with CONNECT
|
||||
NO_PROXY=internal.example,*.corp.example # reached directly instead
|
||||
```
|
||||
|
||||
An `https://` target prefers `HTTPS_PROXY` and falls back to `HTTP_PROXY`; an `http://` target uses `HTTP_PROXY` only. A value with no scheme is read as `http://`, so `proxy.example:8080` works. Only HTTP and HTTPS proxies are supported, and another scheme such as `socks5://` is refused by name rather than silently ignored.
|
||||
|
||||
Credentials in the proxy URL are sent as `Proxy-Authorization` to the proxy and to nothing else, including across redirects:
|
||||
|
||||
```
|
||||
HTTPS_PROXY=http://user:pass@proxy.example:8080 oc open https://example.com
|
||||
```
|
||||
|
||||
`NO_PROXY` accepts an exact host, a `.suffix` or `*.suffix` pattern, a `host:port` entry, a CIDR block, and `*` for everything.
|
||||
|
||||
An `https://` page is tunneled with CONNECT and its certificate is verified the same way it would be without a proxy, so a proxy in the path cannot read or rewrite the page.
|
||||
|
||||
Two limits are worth knowing:
|
||||
|
||||
- oc does not read `ALL_PROXY`. The impers transport is libcurl underneath and reads it on its own, so a request oc treats as direct can still leave through an `ALL_PROXY`. The same holds for the `*.suffix`, `host:port`, and CIDR forms of `NO_PROXY`, which libcurl does not parse. Set `HTTP_PROXY` and `HTTPS_PROXY` explicitly and keep `NO_PROXY` to plain host and suffix entries when the two need to agree.
|
||||
- An IPv6 literal target over HTTPS does not currently work through a proxy.
|
||||
|
||||
Private and internal addresses are refused whether or not a proxy is set. With a proxy configured, a hostname that does not resolve locally is refused too, because the proxy would otherwise resolve it on a network oc cannot see. A name that resolves publicly for oc and internally for the proxy (split horizon DNS) is not something oc can detect, so a proxy is trusted to enforce its own egress policy.
|
||||
|
||||
## Benchmarks
|
||||
|
||||
Full methodology, per-task numbers, and other agents/models live in [only-cli/benchmarks](https://github.com/only-cli/benchmarks). The short version, measured with oc 0.4.0 on 2026-08-24 against live sites across a news front page, a Reddit discussion, a search results page, a stock quote, three cloud CLI reference pages, the Python, MDN, and Node.js references, and more:
|
||||
Full methodology, per-task rows, and the Codex runs live in [only-cli/benchmarks](https://github.com/only-cli/benchmarks). Where things stand (oc 0.5.1 to 0.5.3, September 2026, live sites):
|
||||
|
||||
| method | tokens for 15 real pages | notes |
|
||||
- **118x fewer tokens than raw HTML** across the fourteen real pages both could read: 9,466 against 1,119,003. 17x fewer than Jina Reader, 54x fewer than Playwright MCP's accessibility snapshot.
|
||||
- **Real content on all fifteen pages, Reddit included.** Reddit's pages now sit behind a login wall, so the suite reads its thread and subreddit through the Atom feeds that still answer anonymously, the same route the `oc reddit` shortcuts take: 459 and 493 tokens through oc against 10,452 and 21,155 for the feed XML, while the browser-based tools and Jina Reader got the block pages Reddit serves their fingerprints. Yahoo Finance refuses plain fetch outright and DuckDuckGo still blocks lynx; oc's Chrome impersonation read both.
|
||||
- **Half the cost of Claude Code's built-in `WebSearch`** on Wikipedia lookups: $0.23 against $0.45 for five questions, both 5/5 correct, on 25x less fresh input.
|
||||
- **21% cheaper than `WebFetch` and 34% cheaper than `WebSearch`** on eleven language docs lookups, at equal or better accuracy.
|
||||
- **10% cheaper than `WebFetch` and 49% cheaper than `WebSearch`** on twelve dependency lookups across GitHub, npm, PyPI, RubyGems, crates.io, Docker Hub, Stack Overflow and an RFC, 12/12 correct with no tuned shortcut for most of those sites.
|
||||
|
||||
The tables behind those numbers:
|
||||
|
||||
**Tokens per page, no model in the loop.** Fifteen real pages: a news front page, a Reddit thread and subreddit through their Atom feeds, search results, a stock quote, three cloud CLI references, the Python, MDN, and Node.js references, and more.
|
||||
|
||||
| method | tokens for 15 pages | notes |
|
||||
| --- | ---: | --- |
|
||||
| `oc open` | 10,973 | only method that returned real content on every page |
|
||||
| Jina Reader | 170,505 | blocked on both Reddit pages, failed the LinkedIn page |
|
||||
| raw HTML fetch | 1,535,791 | the stock quote page alone is 375,721 tokens |
|
||||
| `oc open` | 9,913 | real content on all 15 pages; the two Reddit feeds cost 459 and 493 tokens |
|
||||
| Jina Reader | 145,679 | both Reddit results are block pages; failed LinkedIn outright |
|
||||
| Playwright MCP | 535,908 | accessibility snapshots; both Reddit snapshots are block pages, since Reddit refuses the Chrome fingerprint |
|
||||
| raw HTML fetch | 1,119,003 | the two Reddit feeds are 10,452 and 21,155 tokens of XML; Yahoo Finance refused the connection |
|
||||
|
||||
Read cost is one thing, but what an agent actually spends is another, so a
|
||||
second suite runs whole tasks end to end in Claude Code and compares `oc`
|
||||
against the tools the agent already has. Five Wikipedia lookups, one tool per
|
||||
run, Sonnet driving:
|
||||
oc's budget keeps every page near 500 tokens however much it weighs: the YouTube watch page is 363,516 tokens raw and 688 through oc, Node's `fs` reference 275,425 against 479. On the fourteen pages both could read, raw HTML costs 118x what oc does: 1,119,003 against 9,466. Reddit meters anonymous feed readers, so the suite spaces its Reddit requests a minute apart; a second request from the same client inside thirty seconds came back 429 in every faster pass.
|
||||
|
||||
| tool | answered correctly | input tokens | cost | turns | avg time |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `oc wiki` | 5/5 | 5,535 | $0.27 | 22 | 11s |
|
||||
| built-in `WebFetch` | 5/5 | 128,792 | $0.37 | 25 | 14s |
|
||||
| built-in `WebSearch` | 5/5 | 160,431 | $0.52 | 27 | 22s |
|
||||
**Whole tasks against the agent's built-in web tools.** Read cost is one thing, what an agent actually spends is another, so a second set of suites runs full lookups end to end in Claude Code (`claude-sonnet-5`), one tool per run, and grades every answer. Five Wikipedia lookups, eleven language documentation lookups, and twelve lookups on the pages around a dependency, where oc has shortcuts only for GitHub and Stack Overflow and renders the rest generically:
|
||||
|
||||
All three got every answer right, so this is a cost result, not an accuracy one.
|
||||
Input tokens are the fresh context each tool put in front of the model, which is
|
||||
the number the page size drives; totals including cache reads sit closer together
|
||||
because the agent's own prompt dominates them. The spread widens with the page:
|
||||
`oc` cost 5.7x less than `WebFetch` on a short stub and 35x less on a long
|
||||
article, because the 500 token budget makes it flat at about 1,100 tokens per
|
||||
page while a full fetch pays for whatever the page weighs. `WebSearch` was given
|
||||
only the question, not the article URL, which is the honest way to use it and
|
||||
part of why it costs the most.
|
||||
| suite | tool | correct | input tokens | cost | avg time |
|
||||
| --- | --- | ---: | ---: | ---: | ---: |
|
||||
| Wikipedia | `oc wiki` | 5/5 | 5,535 | $0.23 | 8s |
|
||||
| | built-in `WebFetch` | 5/5 | 129,257 | $0.35 | 12s |
|
||||
| | built-in `WebSearch` | 5/5 | 136,982 | $0.45 | 16s |
|
||||
| Language docs | `oc docs` | 11/11 | 12,967 | $0.56 | 8s |
|
||||
| | built-in `WebFetch` | 10/11 | 203,497 | $0.71 | 12s |
|
||||
| | built-in `WebSearch` | 11/11 | 215,833 | $0.85 | 14s |
|
||||
| Dependency research | `oc open` | 12/12 | 13,261 | $0.63 | 10s |
|
||||
| | built-in `WebFetch` | 10/12 | 173,779 | $0.70 | 11s |
|
||||
| | built-in `WebSearch` | 12/12 | 326,088 | $1.22 | 19s |
|
||||
|
||||
Input tokens are the fresh context each tool put in front of the model, which is the number the page size drives; totals including cache reads sit closer together because the agent's own prompt dominates them. oc stays flat at roughly 1,100 to 1,200 tokens per task, while `WebFetch` pays for whatever the page weighs, from 5.9x more on a short Wikipedia stub to 35x more on the German Berlin article. `WebFetch`'s three misses are access results: cppreference, npm, and Stack Overflow all refuse it, while oc's Chrome impersonation reads the same pages. The dependency suite is also where oc's generic renderer pays for hard pages: the JavaScript-only crates.io entry and a support table whose row spans several blocks each cost it eight turns, and on the two tasks that start a link away the built-in tools were cheaper. `WebSearch` was given only the question, never the URL, which is the honest way to use it and part of why it costs the most.
|
||||
|
||||
The same suites through Codex (`gpt-5.6-sol`, on the 0.4.0 and 0.5.0 runs) split. On Wikipedia, `oc wiki` was cheaper and also right where Codex's own search quoted a stale Berlin population. On the docs lookups Codex's search won by 16%: those facts are already in its snippets, and it answered most tasks in two turns without opening a page.
|
||||
|
||||
## Status
|
||||
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
{
|
||||
"domain": "reddit.com",
|
||||
"commands": {
|
||||
"sub": { "open": "https://old.reddit.com/r/{name}", "args": ["name"] },
|
||||
"post": { "open": "https://old.reddit.com/comments/{id}", "args": ["id"] },
|
||||
"user": { "open": "https://old.reddit.com/user/{name}", "args": ["name"] },
|
||||
"search": { "open": "https://old.reddit.com/search?q={query}", "args": ["query"] }
|
||||
"sub": { "open": "https://www.reddit.com/r/{name}/.rss", "args": ["name"] },
|
||||
"new": { "open": "https://www.reddit.com/r/{name}/new/.rss", "args": ["name"] },
|
||||
"top": { "open": "https://www.reddit.com/r/{name}/top/.rss?t=week", "args": ["name"] },
|
||||
"post": { "open": "https://www.reddit.com/comments/{id}/.rss", "args": ["id"] },
|
||||
"user": { "open": "https://www.reddit.com/user/{name}/.rss", "args": ["name"] },
|
||||
"search": { "open": "https://www.reddit.com/search.rss?q={query}", "args": ["query"] }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,14 +9,14 @@ Key facts:
|
||||
- Default output budget is 500 tokens per page; `--budget <n>` adjusts it, and `find`, `read <n>`, or `next` collect what the budget cut without refetching the page
|
||||
- The budget is a target rather than a hard cap: a page that would finish within about four times it is printed whole, because a second command costs the agent far more than the lines the cut would have saved
|
||||
- The render leads with the page's main content and puts navigation, sidebar, and footer after it, so the budget is spent on what was asked for rather than on menus
|
||||
- Benchmarked at roughly 45x fewer tokens than reading raw HTML, with per-task numbers at https://github.com/only-cli/benchmarks
|
||||
- Works on any mostly-static website; tuned shortcuts ship for Hacker News, Reddit, GitHub, X, LinkedIn (public guest views), DuckDuckGo, Bing, Stack Overflow (via its Atom feeds and the Stack Exchange API), Yahoo Finance (quotes, history, markets), Wikipedia (articles, search, and other language editions), the AWS, Google Cloud, and Microsoft Learn documentation sites (guides, CLI reference, and search), and the language documentation for Python, JavaScript (MDN), Node.js, Ruby, Go, Rust, Java, PHP, TypeScript, C and C++ (cppreference), and .NET (the Microsoft Learn API browser)
|
||||
- Benchmarked at roughly 140x fewer tokens than reading raw HTML across fifteen live pages, with per-task numbers at https://github.com/only-cli/benchmarks
|
||||
- Works on any mostly-static website; tuned shortcuts ship for Hacker News, Reddit (via its Atom feeds, since old.reddit.com and the `.json` views need a login now), GitHub, X, LinkedIn (public guest views), DuckDuckGo, Bing, Stack Overflow (via its Atom feeds and the Stack Exchange API), Yahoo Finance (quotes, history, markets), Wikipedia (articles, search, and other language editions), the AWS, Google Cloud, and Microsoft Learn documentation sites (guides, CLI reference, and search), and the language documentation for Python, JavaScript (MDN), Node.js, Ruby, Go, Rust, Java, PHP, TypeScript, C and C++ (cppreference), and .NET (the Microsoft Learn API browser)
|
||||
- JSON APIs render like pages: an endpoint that answers with JSON becomes one numbered item per record, with the fields that differ between items kept and the ones every item shares stated once, so a search endpoint reads like a results page for a few hundred tokens
|
||||
- A page that comes back with no readable text (JavaScript-only, a consent wall, a bot challenge) prints one line on stderr and exits 2, rather than reporting an empty render as a success. `--json` carries the same verdict as an `empty` field, so a caller can tell "nothing on this page" from "oc could not read this page" and fall back to a browser only when it is worth it
|
||||
- A shortcut is `oc <site> <verb> [args]`: `oc hn top`, `oc reddit sub ClaudeAI`, `oc gh repo only-cli oc`, `oc ddg search claude code cli`, `oc learn doc azure/aks/what-is-aks`, `oc py library json`. Name the site by its short name, bare name, or domain (`oc hn`, `oc ycombinator`, `oc news.ycombinator.com`), the last argument takes every word after it so a query needs no quoting, and `oc sites` lists every site with its verbs. A shortcut resolves to a URL and then behaves exactly like `oc open <url>`
|
||||
- X profiles and individual posts read without a login (about 390 and 260 tokens); X search, explore, and hashtag pages do not, and oc reports the block instead of guessing
|
||||
- Outbound fetches honor `HTTP_PROXY`, `HTTPS_PROXY`, and `NO_PROXY` (and their lowercase forms), so oc works in a sandbox whose only route to the network is a proxy. An https target is tunneled with CONNECT and its certificate is still verified, credentials in the proxy URL reach the proxy and nothing else, and private or locally unresolvable targets stay refused. `ALL_PROXY` is not read
|
||||
- Requests impersonate Chrome, so pages that block plain scripts often still work
|
||||
- Requests impersonate Chrome, so pages that block plain scripts often still work; a Chrome identity the site or the local libcurl-impersonate refuses downgrades to Firefox, then to plain fetch
|
||||
- Agent skill included: `npx skills add https://github.com/only-cli/oc --skill web-browsing-cli` ([skills.sh](https://www.skills.sh/only-cli/oc/web-browsing-cli))
|
||||
- Authenticated pages: `printf %s "..." | oc login --cookie - --domain example.com [--expires 1h] [--session name]` seeds a timeboxed cookie jar; cookies are sent on every fetch for that session and live in a separate file from page state. `--cookie -` reads the header from stdin, which keeps the credential out of `ps` and shell history; `--domain` must be a real hostname, not a bare TLD. Seeded cookies are https-only unless `--allow-http` says the site is not, so a redirect that downgrades to `http` drops them. `oc logout` forgets that session's cookies and its saved page
|
||||
- No JavaScript rendering yet (on the roadmap)
|
||||
|
||||
Generated
+6
-5
@@ -1,14 +1,15 @@
|
||||
{
|
||||
"name": "@only-cli/oc",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@only-cli/oc",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"impers": "0.1.1",
|
||||
"linkedom": "^0.18.12",
|
||||
"turndown": "^7.2.4"
|
||||
},
|
||||
@@ -504,9 +505,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/impers": {
|
||||
"version": "0.1.0",
|
||||
"resolved": "https://registry.npmjs.org/impers/-/impers-0.1.0.tgz",
|
||||
"integrity": "sha512-yAeIrBHiFfJjvKkCyfH2ka32twKkvPcBlJVxEQdr5YyGunD5rtL5mC7jWP2rpKo6G9scCSlId0/YbBYhih6+tQ==",
|
||||
"version": "0.1.1",
|
||||
"resolved": "https://registry.npmjs.org/impers/-/impers-0.1.1.tgz",
|
||||
"integrity": "sha512-e/syicsvmH1L9FVG+QapI+cTy01MxAjHo1tLgtByGjiGHvhNQPRC704yUuktB65X+NeiZkf+1tI4LBm/YFvxfA==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@only-cli/oc",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"description": "Turn websites into a compact CLI so AI agents can browse without burning tokens.",
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -8,15 +8,15 @@ description: Token-efficient web browsing and web content extraction for AI agen
|
||||
Renders a web page as a compact, numbered terminal view instead of raw HTML. A typical page is under 500 tokens.
|
||||
|
||||
```
|
||||
npx --yes @only-cli/oc@0.5.0 open <url> compact view, numbered elements
|
||||
npx --yes @only-cli/oc@0.5.0 do <n> follow link [n], or read it if [n] is text
|
||||
npx --yes @only-cli/oc@0.5.0 find <query> where a string appears, or that place itself
|
||||
npx --yes @only-cli/oc@0.5.3 open <url> compact view, numbered elements
|
||||
npx --yes @only-cli/oc@0.5.3 do <n> follow link [n], or read it if [n] is text
|
||||
npx --yes @only-cli/oc@0.5.3 find <query> where a string appears, or that place itself
|
||||
when only one matches
|
||||
npx --yes @only-cli/oc@0.5.0 next next ~500 tokens of the page already open
|
||||
npx --yes @only-cli/oc@0.5.0 read <n> full text of region [n]
|
||||
npx --yes @only-cli/oc@0.5.0 raw [url] whole page as markdown (--html for cleaned HTML)
|
||||
npx --yes @only-cli/oc@0.5.0 login seed cookies (--cookie, --domain, --expires)
|
||||
npx --yes @only-cli/oc@0.5.0 logout [session] forget a session: cookies and saved page
|
||||
npx --yes @only-cli/oc@0.5.3 next next ~500 tokens of the page already open
|
||||
npx --yes @only-cli/oc@0.5.3 read <n> full text of region [n]
|
||||
npx --yes @only-cli/oc@0.5.3 raw [url] whole page as markdown (--html for cleaned HTML)
|
||||
npx --yes @only-cli/oc@0.5.3 login seed cookies (--cookie, --domain, --expires)
|
||||
npx --yes @only-cli/oc@0.5.3 logout [session] forget a session: cookies and saved page
|
||||
```
|
||||
|
||||
None of these except `open`/`do`/`raw <url>` fetch anything; they replay the page `open` already saved.
|
||||
@@ -32,7 +32,7 @@ oc ddg search claude code oc so question 231767 oc learn doc azur
|
||||
oc py library json oc mdn js Array/map oc node api fs
|
||||
```
|
||||
|
||||
Sites: `hn`, `reddit`, `gh`, `x`, `linkedin`, `ddg`, `bing`, `so`, `finance`, `yt`, `aws`, `gcp`, `learn`, `wiki`, `py`, `mdn`, `node`, `ruby`, `go`, `rust`, `java`, `php`, `cpp`, `ts`. Name one by short name, bare name, or domain (`oc hn`, `oc ycombinator`, `oc news.ycombinator.com`). The last argument takes every word after it, so a query or title needs no quoting. `oc sites` lists every site with its verbs, which is cheaper than guessing one.
|
||||
Sites: `hn`, `reddit`, `gh`, `x`, `linkedin`, `ddg`, `bing`, `so`, `yahoo`, `yt`, `aws`, `gcp`, `learn`, `wiki`, `py`, `mdn`, `node`, `ruby`, `go`, `rust`, `java`, `php`, `cpp`, `ts`. Name one by short name, bare name, or domain (`oc hn`, `oc ycombinator`, `oc news.ycombinator.com`). The last argument takes every word after it, so a query or title needs no quoting. `oc sites` lists every site with its verbs, which is cheaper than guessing one. Reddit reads through its Atom feeds: a reddit.com page URL given to `open` meets a login wall or a 403, so use `oc reddit post <id>` or add `/.rss` to the URL, and keep Reddit calls under about ten a minute or the site answers 429.
|
||||
|
||||
Prefer a shortcut over a hand-built URL when one exists for the site, and prefer `oc wiki article <title>` over a search when you already know the article's name.
|
||||
|
||||
|
||||
+4
-2
@@ -234,7 +234,7 @@ export function find(query, { session = DEFAULT_SESSION, budget = 500 } = {}) {
|
||||
return [
|
||||
`1 match for "${query}"${separately}, region [${only.n}]`,
|
||||
read(only.n, { session, budget: budget * FINISH }),
|
||||
`actions: ${follow}read <n> | next | raw`,
|
||||
`actions: ${follow}find <query> | read <n> | next | raw`,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
@@ -264,7 +264,9 @@ export function find(query, { session = DEFAULT_SESSION, budget = 500 } = {}) {
|
||||
if (shown < hits.length) {
|
||||
lines.push(`... ${hits.length - shown} more matches, narrow the query or raise --budget`);
|
||||
}
|
||||
lines.push(`actions: ${[hasLinks && 'do <n>', 'read <n>', 'next', 'raw'].filter(Boolean).join(' | ')}`);
|
||||
// Offering find on its own output is what makes "narrow the query" above an
|
||||
// action rather than advice.
|
||||
lines.push(`actions: ${[hasLinks && 'do <n>', 'find <query>', 'read <n>', 'next', 'raw'].filter(Boolean).join(' | ')}`);
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
|
||||
+9
-3
@@ -504,9 +504,15 @@ export function feedToHTML(text) {
|
||||
// HTML itself, ready to be embedded and parsed like any page.
|
||||
const body = (entry.querySelector('content') ?? entry.querySelector('summary') ?? entry.querySelector('description'))?.textContent ?? '';
|
||||
parts.push('<article>');
|
||||
if (title) parts.push(`<h2>${esc(title)}</h2>`);
|
||||
if (byline || href) {
|
||||
parts.push(`<p>${esc(byline)}${href ? ` <a href="${esc(href)}">open</a>` : ''}</p>`);
|
||||
// The title is the link. It used to sit beside the byline as an anchor
|
||||
// labelled "open", the same label on every entry, and the repeated-controls
|
||||
// filter hid the lot as chrome, so nothing in a subreddit feed led to a
|
||||
// post: `do <n>` on one read its heading instead of following it (#59). A
|
||||
// heading that is exactly one anchor becomes a followable heading in the
|
||||
// walk, so the number the agent already sees is the one that opens it.
|
||||
if (title) parts.push(`<h2>${href ? `<a href="${esc(href)}">${esc(title)}</a>` : esc(title)}</h2>`);
|
||||
if (byline || (href && !title)) {
|
||||
parts.push(`<p>${esc(byline)}${href && !title ? ` <a href="${esc(href)}">open</a>` : ''}</p>`);
|
||||
}
|
||||
parts.push(body, '</article>');
|
||||
}
|
||||
|
||||
+96
-11
@@ -497,7 +497,10 @@ export function proxyGet(url, proxy, headers = {}, tlsOpts = {}) {
|
||||
* identity got the page (impers:chrome, impers:firefox, or fetch)
|
||||
*/
|
||||
export async function fetchPage(url, { jar } = {}) {
|
||||
const target = /^https?:\/\//i.test(url) ? url : `https://${url}`;
|
||||
let target = /^https?:\/\//i.test(url) ? url : `https://${url}`;
|
||||
// A reader with no cookies for reddit.com gets the feed where the page
|
||||
// would be a login wall; one who logged in gets the page it asked for.
|
||||
if (!jar?.cookieHeaderFor(target)) target = redditFeedURL(target) ?? target;
|
||||
await assertSafeTarget(target);
|
||||
const impers = await loadImpers();
|
||||
return impers ? viaImpers(impers, target, jar) : viaFetch(target, jar);
|
||||
@@ -555,9 +558,81 @@ function captureSetCookie(jar, url, res) {
|
||||
jar.storeFromResponse(url, getSetCookieHeaders(res));
|
||||
}
|
||||
|
||||
async function viaImpers(impers, target, jar) {
|
||||
// Some sites (Reddit) 403 the chrome fingerprint but accept firefox, so a
|
||||
// blocked first attempt gets one cheap retry with a second identity.
|
||||
// Hosts whose edge answers the chrome fingerprint with a 403 or a 429 while
|
||||
// letting firefox through. reddit.com started doing this in 2026 (#52), so
|
||||
// starting with chrome there would turn every read into two requests against
|
||||
// a per-address rate limit of about ten a minute. Subdomains inherit the
|
||||
// entry.
|
||||
const FIREFOX_FIRST_HOSTS = ['reddit.com'];
|
||||
|
||||
// Reddit has sent logged-out readers of its HTML pages to a login page since
|
||||
// June 2026 (#52), while the Atom feed beside each of those pages still
|
||||
// answers. A feed entry links to the page, so following a post out of a
|
||||
// subreddit feed used to land on the wall. The front page, subreddit
|
||||
// listings, posts, user pages and search are mapped to their feeds here;
|
||||
// anything else on reddit.com is fetched as asked.
|
||||
const REDDIT_HOSTS = ['reddit.com', 'www.reddit.com', 'old.reddit.com', 'new.reddit.com', 'np.reddit.com'];
|
||||
const SEG = '[A-Za-z0-9_.-]+';
|
||||
const REDDIT_FEED_PATHS = new RegExp(
|
||||
`^(?:|/r/${SEG}(?:/(?:new|top|hot|rising))?|/(?:r/${SEG}/)?comments/${SEG}(?:/${SEG}){0,2}|/u(?:ser)?/${SEG})$`,
|
||||
);
|
||||
|
||||
/**
|
||||
* The www.reddit.com Atom feed for a reddit.com page URL, or null when the URL
|
||||
* is not one of the page shapes that has a feed, or is a feed already.
|
||||
* @param {string} target
|
||||
* @returns {string | null}
|
||||
*/
|
||||
export function redditFeedURL(target) {
|
||||
let u;
|
||||
try {
|
||||
u = new URL(target);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (!REDDIT_HOSTS.includes(u.hostname.toLowerCase())) return null;
|
||||
const path = u.pathname.replace(/\/+$/, '');
|
||||
if (/\.(?:rss|json|xml)$/i.test(path)) return null;
|
||||
if (path === '/search') return `https://www.reddit.com/search.rss${u.search}`;
|
||||
if (!REDDIT_FEED_PATHS.test(path)) return null;
|
||||
return `https://www.reddit.com${path.replace(/^\/u\//, '/user/')}/.rss${u.search}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The order in which impers identities are tried for a URL.
|
||||
* @param {string} target
|
||||
* @returns {['chrome', 'firefox'] | ['firefox', 'chrome']}
|
||||
*/
|
||||
export function identityOrder(target) {
|
||||
let host = '';
|
||||
try {
|
||||
host = new URL(target).hostname.toLowerCase();
|
||||
} catch {
|
||||
return ['chrome', 'firefox'];
|
||||
}
|
||||
const firefoxFirst = FIREFOX_FIRST_HOSTS.some((h) => host === h || host.endsWith(`.${h}`));
|
||||
return firefoxFirst ? ['firefox', 'chrome'] : ['chrome', 'firefox'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a page through impers, downgrading identity when one is refused.
|
||||
* Exported so the downgrade chain can be proven against a fake impers; the
|
||||
* real entry point is fetchPage.
|
||||
* @param {any} impers - the impers module (or a stand-in with a get method)
|
||||
* @param {string} target
|
||||
* @param {object} [jar]
|
||||
* @returns {Promise<{url: string, html: string, status: number, via: string}>}
|
||||
*/
|
||||
export async function viaImpers(impers, target, jar) {
|
||||
// A blocked first attempt gets one cheap retry with the other identity. An
|
||||
// ImpersonateError is the same story one layer down: impers resolves the
|
||||
// 'chrome' alias to its newest fingerprint, but the native library it loads
|
||||
// can be an older system copy of libcurl-impersonate that predates that
|
||||
// fingerprint and refuses it before any request leaves. Firefox aliases to
|
||||
// an older target that such a library usually still knows, and when both
|
||||
// identities are refused the plain fetch transport still gets the page.
|
||||
// Hosts that are known to refuse chrome outright start with firefox, so the
|
||||
// usual case there costs one request instead of a 403 and a retry.
|
||||
const asking = (impersonate) => (url) =>
|
||||
impers.get(url, {
|
||||
impersonate,
|
||||
@@ -566,14 +641,24 @@ async function viaImpers(impers, target, jar) {
|
||||
headers: jarHeaders(jar, url, {}),
|
||||
});
|
||||
const onResponse = (url, res) => captureSetCookie(jar, url, res);
|
||||
let via = 'impers:chrome';
|
||||
let { res } = await followRedirects(asking('chrome'), target, { onResponse });
|
||||
let status = res.status ?? res.statusCode ?? 0;
|
||||
if (status >= 400) {
|
||||
via = 'impers:firefox';
|
||||
({ res } = await followRedirects(asking('firefox'), target, { onResponse }));
|
||||
status = res.status ?? res.statusCode ?? 0;
|
||||
const attempt = async (impersonate) => {
|
||||
try {
|
||||
const { res } = await followRedirects(asking(impersonate), target, { onResponse });
|
||||
return { res, status: res.status ?? res.statusCode ?? 0 };
|
||||
} catch (err) {
|
||||
if (err?.name !== 'ImpersonateError') throw err;
|
||||
return null;
|
||||
}
|
||||
};
|
||||
const [first, second] = identityOrder(target);
|
||||
let via = `impers:${first}`;
|
||||
let got = await attempt(first);
|
||||
if (!got || got.status >= 400) {
|
||||
via = `impers:${second}`;
|
||||
got = (await attempt(second)) ?? got;
|
||||
}
|
||||
if (!got) return viaFetch(target, jar);
|
||||
const { res, status } = got;
|
||||
if (status >= 400) throw new Error(`fetch failed: ${status} for ${target}`);
|
||||
assertReadableType(res.headers.get('content-type'));
|
||||
assertBodySize(Number(res.headers.get('content-length')) || 0, target);
|
||||
|
||||
+10
-4
@@ -101,7 +101,6 @@ export function render(page, { budget = 500, from = 0 } = {}) {
|
||||
const lines = [...head];
|
||||
let spent = estimateTokens(lines.join('\n'));
|
||||
let hasLinks = false;
|
||||
let hasInputs = false;
|
||||
let i = Math.max(0, from);
|
||||
|
||||
// What the rest of the page would cost if it were all printed. When that is
|
||||
@@ -129,7 +128,6 @@ export function render(page, { budget = 500, from = 0 } = {}) {
|
||||
spent += cost;
|
||||
lines.push(line);
|
||||
if (block.type === 'link' || block.type === 'button') hasLinks = true;
|
||||
if (block.type === 'input') hasInputs = true;
|
||||
}
|
||||
|
||||
const rest = blocks.slice(i);
|
||||
@@ -140,10 +138,18 @@ export function render(page, { budget = 500, from = 0 } = {}) {
|
||||
if (rest.length) {
|
||||
lines.push(`... ${num(rest.length)} more blocks (~${num(leftTokens)} tokens): 'oc next' for the next ~${num(budget)}, 'oc raw' for all`);
|
||||
}
|
||||
// An input on the page adds no action. 'fill' and 'submit' are still stubs
|
||||
// that throw, and this footer is the line an agent trusts for what to run
|
||||
// next, so naming one of them costs a turn and returns nothing.
|
||||
//
|
||||
// 'find' comes before 'read' because it is the cheaper way to go deeper: one
|
||||
// command lands on the block that matters, where 'read' needs the right
|
||||
// number first and 'next' pages toward it. The entry costs about three
|
||||
// tokens on every render, and skipping one 'next' on a long page pays for
|
||||
// a hundred of them.
|
||||
const actions = [
|
||||
hasLinks && 'do <n>',
|
||||
hasInputs && 'fill <n> <text>',
|
||||
hasInputs && 'submit',
|
||||
'find <query>',
|
||||
'read <n>',
|
||||
rest.length && 'next',
|
||||
'raw',
|
||||
|
||||
@@ -244,3 +244,61 @@ test('a snippet stays one line even when the block it came from is code', () =>
|
||||
assert.match(lines[0], /^2 matches for "needle"/);
|
||||
assert.equal(lines[1], '[1] first(); needle(); third();');
|
||||
});
|
||||
|
||||
test('no footer names a command that is not available yet', async () => {
|
||||
// The footer is the line an agent reads to decide what to run next, so a
|
||||
// name in it that always throws costs a turn and returns nothing. Which
|
||||
// commands are stubs is probed here rather than listed, so the next stub to
|
||||
// land is covered without anyone remembering to come back and add it.
|
||||
const act = await import('../src/act.js');
|
||||
const stubs = Object.entries(act)
|
||||
.filter(([, value]) => typeof value === 'function')
|
||||
.filter(([, fn]) => {
|
||||
try {
|
||||
fn();
|
||||
return false;
|
||||
} catch (err) {
|
||||
return err instanceof act.NotImplemented;
|
||||
}
|
||||
})
|
||||
.map(([name]) => name);
|
||||
assert.ok(stubs.length, 'the probe found no stubs, so it is no longer testing anything');
|
||||
|
||||
open();
|
||||
const loginHTML = readFileSync(new URL('./pages/login.html', import.meta.url), 'utf8');
|
||||
// One output per place that builds a footer: a render with inputs, which is
|
||||
// what used to offer fill and submit, and both of find's paths.
|
||||
const outputs = [
|
||||
render(page(), { budget: 500 }).text,
|
||||
render(distill(loginHTML, 'https://example.test/login'), { budget: 500 }).text,
|
||||
find('postgres'),
|
||||
find('a'),
|
||||
];
|
||||
const footers = outputs.flatMap((out) => out.split('\n').filter((line) => line.startsWith('actions:')));
|
||||
assert.equal(footers.length, outputs.length, `every output should carry one footer:\n${footers.join('\n')}`);
|
||||
for (const footer of footers) {
|
||||
for (const stub of stubs) {
|
||||
assert.ok(!footer.includes(stub), `footer offers '${stub}', which throws NotImplemented:\n${footer}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('every footer offers find, the cheapest way to go deeper on a page', () => {
|
||||
// SKILL.md lists find first under "going further, cheapest first", and the
|
||||
// footer is what an agent actually reads, so the two have to agree. Same
|
||||
// three footer sites as the stub probe above: a render, and both of find's
|
||||
// paths.
|
||||
open();
|
||||
const outputs = [
|
||||
render(page(), { budget: 500 }).text,
|
||||
find('postgres'),
|
||||
find('a'),
|
||||
];
|
||||
const footers = outputs.map((out) => out.split('\n').find((line) => line.startsWith('actions:')));
|
||||
assert.equal(footers.filter(Boolean).length, outputs.length, `every output should carry a footer:\n${footers.join('\n')}`);
|
||||
for (const footer of footers) {
|
||||
assert.ok(footer.includes('find <query>'), `footer should offer find:\n${footer}`);
|
||||
// Cheapest first: find is listed ahead of read.
|
||||
assert.ok(footer.indexOf('find <query>') < footer.indexOf('read <n>'), `find should come before read:\n${footer}`);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import http from 'node:http';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, utimesSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
const { cachedFile } = await import('../src/cache.js');
|
||||
|
||||
// The cache fetches through fetchPage, which honors HTTP_PROXY, so a local
|
||||
// proxy stands in for the network: it records every request and answers with
|
||||
// whatever body the test hands it. A public IP literal keeps the target guard
|
||||
// offline (no DNS), same as the fetch tests. Nothing leaves the machine.
|
||||
const PROXY_ENV_KEYS = ['HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY', 'http_proxy', 'https_proxy', 'no_proxy'];
|
||||
const URL_JSON = 'http://1.1.1.1/docs/all.json';
|
||||
const parseJSON = (text) => JSON.parse(text);
|
||||
|
||||
function listen(server) {
|
||||
return new Promise((resolve) => {
|
||||
server.listen(0, '127.0.0.1', () => resolve(server.address().port));
|
||||
});
|
||||
}
|
||||
|
||||
// Runs fn with a fresh OC_HOME and a proxy serving `body`. Returns what the
|
||||
// proxy saw so a test can prove the network was, or was not, touched.
|
||||
async function withCache(body, fn) {
|
||||
const home = mkdtempSync(join(tmpdir(), 'oc-cache-'));
|
||||
const seen = [];
|
||||
const proxy = http.createServer((req, res) => {
|
||||
seen.push(req.url);
|
||||
res.writeHead(200, { 'content-type': 'application/json' });
|
||||
res.end(body);
|
||||
});
|
||||
const port = await listen(proxy);
|
||||
const prevHome = process.env.OC_HOME;
|
||||
const prev = Object.fromEntries(PROXY_ENV_KEYS.map((k) => [k, process.env[k]]));
|
||||
for (const k of PROXY_ENV_KEYS) delete process.env[k];
|
||||
process.env.HTTP_PROXY = `http://127.0.0.1:${port}`;
|
||||
process.env.OC_HOME = home;
|
||||
try {
|
||||
await fn({ home, seen, file: join(home, 'sphinx', '1.1.1.1.json') });
|
||||
} finally {
|
||||
proxy.close();
|
||||
for (const k of PROXY_ENV_KEYS) {
|
||||
if (prev[k] === undefined) delete process.env[k];
|
||||
else process.env[k] = prev[k];
|
||||
}
|
||||
if (prevHome === undefined) delete process.env.OC_HOME;
|
||||
else process.env.OC_HOME = prevHome;
|
||||
}
|
||||
}
|
||||
|
||||
test('a miss fetches, parses, and writes the file under host and extension', () => withCache('{"n":1}', async ({ seen, file }) => {
|
||||
const { data, via } = await cachedFile('sphinx', URL_JSON, parseJSON);
|
||||
assert.deepEqual(data, { n: 1 });
|
||||
assert.equal(via, 'network');
|
||||
assert.deepEqual(seen, [URL_JSON]);
|
||||
// One directory per backend, one file per host, the URL's own extension.
|
||||
assert.equal(readFileSync(file, 'utf8'), '{"n":1}');
|
||||
}));
|
||||
|
||||
test('a fresh file is served from disk and the network is never asked', () => withCache('{"n":"from network"}', async ({ home, seen, file }) => {
|
||||
mkdirSync(join(home, 'sphinx'), { recursive: true });
|
||||
writeFileSync(file, '{"n":"from disk"}');
|
||||
const { data, via } = await cachedFile('sphinx', URL_JSON, parseJSON);
|
||||
assert.deepEqual(data, { n: 'from disk' });
|
||||
assert.equal(via, 'cache');
|
||||
assert.equal(seen.length, 0);
|
||||
}));
|
||||
|
||||
test('a file older than a day is refetched and replaced', () => withCache('{"n":"fresh"}', async ({ home, seen, file }) => {
|
||||
mkdirSync(join(home, 'sphinx'), { recursive: true });
|
||||
writeFileSync(file, '{"n":"stale"}');
|
||||
const dayAgo = (Date.now() - 25 * 60 * 60 * 1000) / 1000;
|
||||
utimesSync(file, dayAgo, dayAgo);
|
||||
const { data, via } = await cachedFile('sphinx', URL_JSON, parseJSON);
|
||||
assert.deepEqual(data, { n: 'fresh' });
|
||||
assert.equal(via, 'network');
|
||||
assert.deepEqual(seen, [URL_JSON]);
|
||||
assert.equal(readFileSync(file, 'utf8'), '{"n":"fresh"}');
|
||||
}));
|
||||
|
||||
test('a body the parser rejects is not written, so a block page cannot poison the cache', () => withCache('<html>please verify you are human</html>', async ({ seen, file }) => {
|
||||
await assert.rejects(() => cachedFile('sphinx', URL_JSON, parseJSON), SyntaxError);
|
||||
assert.deepEqual(seen, [URL_JSON]);
|
||||
assert.ok(!existsSync(file), 'the unparseable body was written to the cache');
|
||||
}));
|
||||
|
||||
test('a stale copy survives a refetch whose body the parser rejects', () => withCache('not the index', async ({ home, file }) => {
|
||||
// The disk copy is too old to serve, but it is also the only good copy, and
|
||||
// the file is parsed before it is written, so the bad fetch leaves it alone.
|
||||
mkdirSync(join(home, 'sphinx'), { recursive: true });
|
||||
writeFileSync(file, '{"n":"stale but real"}');
|
||||
const dayAgo = (Date.now() - 25 * 60 * 60 * 1000) / 1000;
|
||||
utimesSync(file, dayAgo, dayAgo);
|
||||
await assert.rejects(() => cachedFile('sphinx', URL_JSON, parseJSON), SyntaxError);
|
||||
assert.equal(readFileSync(file, 'utf8'), '{"n":"stale but real"}');
|
||||
}));
|
||||
|
||||
test('a cache directory that cannot be created costs only the refetch', () => withCache('{"n":2}', async ({ home, seen }) => {
|
||||
// A regular file where the backend directory should be makes mkdir fail.
|
||||
// The same policy as session state: the answer still comes back, and the
|
||||
// next call pays for the network again rather than failing.
|
||||
writeFileSync(join(home, 'sphinx'), 'in the way');
|
||||
let result = await cachedFile('sphinx', URL_JSON, parseJSON);
|
||||
assert.deepEqual(result.data, { n: 2 });
|
||||
assert.equal(result.via, 'network');
|
||||
result = await cachedFile('sphinx', URL_JSON, parseJSON);
|
||||
assert.equal(result.via, 'network');
|
||||
assert.deepEqual(seen, [URL_JSON, URL_JSON]);
|
||||
}));
|
||||
|
||||
test('a URL with no extension caches under the bare host, and kinds do not share files', () => withCache('{"n":3}', async ({ home }) => {
|
||||
await cachedFile('nodedoc', 'http://1.1.1.1/api/all', parseJSON);
|
||||
assert.ok(existsSync(join(home, 'nodedoc', '1.1.1.1')));
|
||||
assert.ok(!existsSync(join(home, 'sphinx')), 'a nodedoc fetch created the sphinx directory');
|
||||
}));
|
||||
@@ -0,0 +1,222 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import http from 'node:http';
|
||||
import { mkdtempSync, readFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { spawn, spawnSync } from 'node:child_process';
|
||||
|
||||
// Dispatch tests: the first word of argv reaches the right handler with the
|
||||
// right arguments, and every wrong first word fails in one line that names the
|
||||
// way out. Each case spawns the real binary against a throwaway OC_HOME, and
|
||||
// none of them fetches: the page under test is seeded straight into a session
|
||||
// file, so 'read', 'next', 'find', and 'do' on text all have something to
|
||||
// answer with. Auth commands have their own file (cli-auth.test.js).
|
||||
const OC_HOME = mkdtempSync(join(tmpdir(), 'oc-cli-'));
|
||||
process.env.OC_HOME = OC_HOME;
|
||||
|
||||
const { distill } = await import('../src/distill.js');
|
||||
const { render } = await import('../src/render.js');
|
||||
const { saveSession, sessionFromPage } = await import('../src/session.js');
|
||||
|
||||
const bin = new URL('../src/cli.js', import.meta.url).pathname;
|
||||
const newsHtml = readFileSync(new URL('./pages/news.html', import.meta.url), 'utf8');
|
||||
|
||||
const PROXY_ENV_KEYS = ['HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY', 'http_proxy', 'https_proxy', 'no_proxy'];
|
||||
|
||||
function oc(args, envExtra = {}) {
|
||||
const env = { ...process.env, OC_HOME, ...envExtra };
|
||||
for (const k of PROXY_ENV_KEYS) if (!(k in envExtra)) delete env[k];
|
||||
return spawnSync(process.execPath, [bin, ...args], { encoding: 'utf8', env });
|
||||
}
|
||||
|
||||
// Seeds a session the way 'oc open' would have, and hands back the page so a
|
||||
// test can pick a number that means what it needs.
|
||||
function seed(name = 'default') {
|
||||
const page = distill(newsHtml, 'https://example.test/news');
|
||||
const { stats } = render(page, { budget: 500 });
|
||||
saveSession(name, sessionFromPage(page, null, { cursor: stats.next }));
|
||||
return page;
|
||||
}
|
||||
|
||||
function listen(server) {
|
||||
return new Promise((resolve) => {
|
||||
server.listen(0, '127.0.0.1', () => resolve(server.address().port));
|
||||
});
|
||||
}
|
||||
|
||||
test('no command, --help, and -h all print the usage and exit 0', () => {
|
||||
for (const args of [[], ['--help'], ['-h']]) {
|
||||
const r = oc(args);
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
assert.match(r.stdout, /^only-cli: the web as a compact terminal/);
|
||||
assert.match(r.stdout, /usage: oc <command> \[args\] \[flags\]/);
|
||||
}
|
||||
});
|
||||
|
||||
test('the usage names every dispatchable command exactly once', () => {
|
||||
// The help text and the dispatch table live a hundred lines apart. A command
|
||||
// that dispatches but is not in the help is undiscoverable; one in the help
|
||||
// that does not dispatch is a wasted turn.
|
||||
const { stdout } = oc(['--help']);
|
||||
for (const command of ['open', 'find', 'next', 'read', 'raw', 'do', 'fill', 'submit', 'back', 'login', 'logout', 'session', 'sites']) {
|
||||
const listed = stdout.split('\n').filter((line) => new RegExp(`^ ${command}( |$)`).test(line));
|
||||
assert.equal(listed.length, 1, `'${command}' should be listed once in --help, found ${listed.length}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('an unknown first word fails in one line that points at --help', () => {
|
||||
const r = oc(['frobnicate']);
|
||||
assert.equal(r.status, 1);
|
||||
assert.equal(r.stdout, '');
|
||||
assert.equal(r.stderr.trim(), "oc: unknown command 'frobnicate', run oc --help");
|
||||
});
|
||||
|
||||
test('a site name is tried as a shortcut before it is called unknown', () => {
|
||||
// The shortcut resolver owns the error here, so a wrong verb reports the
|
||||
// site's verbs, not 'unknown command'.
|
||||
const r = oc(['hn', 'frobnicate']);
|
||||
assert.equal(r.status, 1);
|
||||
assert.match(r.stderr, /^oc: 'frobnicate' is not a news\.ycombinator\.com shortcut, try: /);
|
||||
assert.doesNotMatch(r.stderr, /unknown command/);
|
||||
});
|
||||
|
||||
test('--budget must be a positive number, checked before any command runs', () => {
|
||||
// The negative case uses the '=' form: as a separate token, parseArgs reads
|
||||
// '-5' as a flag and refuses it itself before oc sees a value.
|
||||
for (const flag of [['--budget', 'abc'], ['--budget', '0'], ['--budget=-5']]) {
|
||||
const r = oc(['sites', ...flag]);
|
||||
assert.equal(r.status, 1, `${flag.join(' ')} should fail`);
|
||||
assert.equal(r.stderr.trim(), 'oc: --budget must be a positive number');
|
||||
assert.equal(r.stdout, '', `${flag.join(' ')} still ran the command`);
|
||||
}
|
||||
});
|
||||
|
||||
test('a session name that is a path is refused before anything is read or written', () => {
|
||||
for (const bad of ['../etc', 'a/b', '.', '..']) {
|
||||
const r = oc(['next', '--session', bad]);
|
||||
assert.equal(r.status, 1, `--session ${bad} should fail`);
|
||||
assert.match(r.stderr, /^oc: invalid session name/);
|
||||
}
|
||||
});
|
||||
|
||||
test('read, next, find, and do with nothing open say to run open first', () => {
|
||||
for (const args of [['read', '1'], ['next'], ['find', 'anything'], ['do', '1']]) {
|
||||
const r = oc([...args, '--session', 'never-opened']);
|
||||
assert.equal(r.status, 1, args.join(' '));
|
||||
assert.equal(r.stderr.trim(), "oc: nothing open in this session yet, run 'oc open <url>' first", args.join(' '));
|
||||
}
|
||||
});
|
||||
|
||||
test('open and raw with no URL and nothing open print a usage line', () => {
|
||||
for (const command of ['open', 'raw']) {
|
||||
const r = oc([command, '--session', 'never-opened']);
|
||||
assert.equal(r.status, 1);
|
||||
assert.equal(r.stderr.trim(), `oc: usage: oc ${command} <url>`);
|
||||
}
|
||||
});
|
||||
|
||||
test('read <n> prints the region at n from the saved page', () => {
|
||||
const page = seed('reading');
|
||||
const block = page.blocks.find((b) => b.n != null && b.type === 'heading');
|
||||
const r = oc(['read', String(block.n), '--session', 'reading']);
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
assert.ok(r.stdout.includes(block.text), `read ${block.n} should print the block text:\n${r.stdout}`);
|
||||
});
|
||||
|
||||
test('read without a valid number fails with usage rather than a stack trace', () => {
|
||||
seed('reading');
|
||||
for (const args of [['read'], ['read', 'abc'], ['read', '0']]) {
|
||||
const r = oc([...args, '--session', 'reading']);
|
||||
assert.equal(r.status, 1, args.join(' '));
|
||||
assert.match(r.stderr, /^oc: usage: oc read <n>/);
|
||||
assert.doesNotMatch(r.stderr, /\n\s+at /, 'stack trace leaked to stderr');
|
||||
}
|
||||
});
|
||||
|
||||
test('next continues the saved page and reports the end when nothing is left', () => {
|
||||
seed('paging');
|
||||
// The news fixture fits in one render, so the saved cursor is already null
|
||||
// and next has nothing more to show. Either branch of next is one line an
|
||||
// agent can act on; this fixture exercises the end.
|
||||
const r = oc(['next', '--session', 'paging']);
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
assert.match(r.stdout, /^end of https:\/\/example\.test\/news, nothing left to render/);
|
||||
});
|
||||
|
||||
test('find joins the rest of argv into one query', () => {
|
||||
seed('finding');
|
||||
// Unquoted words reach find as separate argv entries; the footer offers
|
||||
// 'find <query>' without quotes, so this is how agents type it.
|
||||
const r = oc(['find', 'Show', 'HN', '--session', 'finding']);
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
assert.match(r.stdout, /^1 match for "Show HN"|^\d+ matches for "Show HN"/);
|
||||
});
|
||||
|
||||
test('find with no query fails with usage', () => {
|
||||
seed('finding');
|
||||
const r = oc(['find', '--session', 'finding']);
|
||||
assert.equal(r.status, 1);
|
||||
assert.match(r.stderr, /^oc: usage: oc find <query>/);
|
||||
});
|
||||
|
||||
test('do on a text number reads it in place and never fetches', async () => {
|
||||
const page = seed('doing');
|
||||
const block = page.blocks.find((b) => b.n != null && b.type === 'text' && !b.href);
|
||||
assert.ok(block, 'the news fixture should have a numbered text block');
|
||||
// A proxy that records requests is the proof: if do decided to fetch, the
|
||||
// request would land here.
|
||||
const seen = [];
|
||||
const proxy = http.createServer((req, res) => {
|
||||
seen.push(req.url);
|
||||
res.writeHead(200, { 'content-type': 'text/html' });
|
||||
res.end('<html>should not be fetched</html>');
|
||||
});
|
||||
const port = await listen(proxy);
|
||||
try {
|
||||
const r = oc(['do', String(block.n), '--session', 'doing'], {
|
||||
HTTP_PROXY: `http://127.0.0.1:${port}`,
|
||||
HTTPS_PROXY: `http://127.0.0.1:${port}`,
|
||||
});
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
assert.ok(r.stdout.includes(block.text), `do ${block.n} should print the text at [${block.n}]:\n${r.stdout}`);
|
||||
assert.equal(seen.length, 0, 'do on text sent a request');
|
||||
} finally {
|
||||
proxy.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('do without a number, or with one the page does not have, fails in one line', () => {
|
||||
seed('doing');
|
||||
let r = oc(['do', '--session', 'doing']);
|
||||
assert.equal(r.status, 1);
|
||||
assert.match(r.stderr, /^oc: usage: oc do <n>/);
|
||||
r = oc(['do', '9999', '--session', 'doing']);
|
||||
assert.equal(r.status, 1);
|
||||
assert.match(r.stderr, /^oc: no \[9999\] on https:\/\/example\.test\/news \(handles 1-\d+\), run 'oc open <url>' again/);
|
||||
});
|
||||
|
||||
test('the planned commands fail with the same one-line message, naming themselves', () => {
|
||||
seed('stubs');
|
||||
for (const args of [['fill', '1', 'hello'], ['submit'], ['submit', '1'], ['back'], ['session', 'ls']]) {
|
||||
const r = oc([...args, '--session', 'stubs']);
|
||||
assert.equal(r.status, 1, args.join(' '));
|
||||
assert.equal(r.stdout, '', `${args[0]} printed to stdout`);
|
||||
assert.equal(r.stderr.trim(), `oc: 'oc ${args[0]}' is not available yet. Until then use 'oc open' and 'oc raw'.`);
|
||||
}
|
||||
});
|
||||
|
||||
test('sites lists the bundled shortcuts and exits 0', () => {
|
||||
const r = oc(['sites']);
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
assert.match(r.stdout, /news\.ycombinator\.com/);
|
||||
assert.match(r.stdout, /\bhn\b/);
|
||||
});
|
||||
|
||||
test('flags are accepted anywhere in argv, before or after the command', () => {
|
||||
seed('flags');
|
||||
const before = oc(['--session', 'flags', 'next']);
|
||||
const after = oc(['next', '--session', 'flags']);
|
||||
assert.equal(before.status, 0, before.stderr);
|
||||
assert.equal(before.stdout, after.stdout);
|
||||
});
|
||||
+49
-2
@@ -137,14 +137,61 @@ test('atom feeds render as pages: entries become headings, bodies unescape', ()
|
||||
const headings = p.blocks.filter((b) => b.type === 'heading');
|
||||
assert.equal(headings[0].text, 'Why is the sky blue?');
|
||||
assert.equal(headings[1].text, 'Answer by Tyndall for Why is the sky blue?');
|
||||
const open = p.blocks.find((b) => b.type === 'link' && b.text === 'open');
|
||||
assert.equal(open.href, 'https://example.test/questions/42/why-is-the-sky-blue');
|
||||
// The title is the entry's link, so the number the agent sees is the one
|
||||
// that follows it. It used to be a separate anchor labelled "open".
|
||||
assert.equal(headings[0].href, 'https://example.test/questions/42/why-is-the-sky-blue');
|
||||
assert.equal(p.blocks.find((b) => b.type === 'link' && b.text === 'open'), undefined, 'the open anchor is back');
|
||||
const text = p.blocks.map((b) => b.text).join(' ');
|
||||
assert.ok(text.includes('Rayleigh scattering'), 'entry body missing');
|
||||
assert.ok(text.includes('by Ray Leigh, 2026-04-08'), 'byline missing');
|
||||
assert.ok(!text.includes('<'), 'entry body left escaped');
|
||||
});
|
||||
|
||||
test('a reddit post feed renders as the post followed by its comments', () => {
|
||||
// Reddit closed old.reddit.com and its .json views to logged-out readers in
|
||||
// 2026; the Atom feeds on www.reddit.com are what oc reddit rides now. A
|
||||
// post feed is one entry for the post and one per comment, each comment
|
||||
// titled "/u/name on <post title>", so the whole thread reads as one page.
|
||||
const xml = readFileSync(new URL('./pages/reddit_post.xml', import.meta.url), 'utf8');
|
||||
const p = distill(xml, 'https://www.reddit.com/comments/1fixture/.rss');
|
||||
assert.equal(p.title, 'Why does the budget flag round up? : reddit.com');
|
||||
const headings = p.blocks.filter((b) => b.type === 'heading').map((b) => b.text);
|
||||
assert.deepEqual(headings, [
|
||||
'Why does the budget flag round up?',
|
||||
'/u/first_reply on Why does the budget flag round up?',
|
||||
'/u/second_reply on Why does the budget flag round up?',
|
||||
]);
|
||||
const post = p.blocks.find((b) => b.type === 'heading');
|
||||
assert.equal(post.href, 'https://www.reddit.com/r/FixtureSub/comments/1fixture/why_does_the_budget_flag_round_up/',
|
||||
'the post heading is not the link to the post');
|
||||
const text = p.blocks.map((b) => b.text).join(' ');
|
||||
assert.ok(text.includes('Is that on purpose?'), 'post body missing');
|
||||
assert.ok(text.includes('by /u/fixture_poster, 2026-09-01'), 'post byline missing');
|
||||
assert.ok(text.includes('One extra tool call costs more'), 'comment body missing');
|
||||
assert.ok(!text.includes('SC_OFF'), 'reddit markup comments leaked into the text');
|
||||
const rendered = render(p).text;
|
||||
assert.ok(rendered.includes('## [1] Why does the budget flag round up?'), 'post is not the first numbered heading');
|
||||
assert.ok(estimateTokens(rendered) < 500, `three-entry thread should fit the default budget, got ${estimateTokens(rendered)}`);
|
||||
});
|
||||
|
||||
test('every entry in a long feed keeps its link', () => {
|
||||
// A subreddit feed carries 25 entries. Their links used to share the label
|
||||
// "open", and five of one label is what the repeated-controls filter drops,
|
||||
// so a whole listing rendered with nothing that led to a post.
|
||||
const entries = Array.from({ length: 25 }, (_, i) =>
|
||||
`<entry><title>Post ${i}</title><link href="https://www.reddit.com/r/Fixture/comments/p${i}/post_${i}/"/>
|
||||
<author><name>/u/poster</name></author><updated>2026-09-04T00:00:00+00:00</updated>
|
||||
<content type="html"><p>Body ${i}</p></content></entry>`).join('');
|
||||
const xml = `<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"><title>Fixture</title>${entries}</feed>`;
|
||||
const p = distill(xml, 'https://www.reddit.com/r/Fixture/.rss');
|
||||
const headings = p.blocks.filter((b) => b.type === 'heading');
|
||||
assert.equal(headings.length, 25);
|
||||
assert.ok(headings.every((h, i) => h.href === `https://www.reddit.com/r/Fixture/comments/p${i}/post_${i}/`),
|
||||
'a heading lost its link');
|
||||
assert.ok(!p.blocks.some((b) => b.type === 'divider' && /repeated controls/.test(b.text)),
|
||||
'the entry links were hidden as repeated controls');
|
||||
});
|
||||
|
||||
test('feed entry code blocks survive raw markdown', () => {
|
||||
const md = toMarkdown(feed);
|
||||
assert.ok(md.startsWith('# Why is the sky blue? - Fixture Overflow'));
|
||||
|
||||
+124
-1
@@ -5,7 +5,7 @@ import https from 'node:https';
|
||||
import net from 'node:net';
|
||||
import tls from 'node:tls';
|
||||
|
||||
const { fetchPage, followRedirects, resolveProxy, proxyGet } = await import('../src/fetch.js');
|
||||
const { fetchPage, followRedirects, identityOrder, redditFeedURL, resolveProxy, proxyGet, viaImpers } = await import('../src/fetch.js');
|
||||
|
||||
const BLOCKED_MESSAGE = 'blocked: private or internal URL';
|
||||
|
||||
@@ -727,3 +727,126 @@ test('the proxy transport counts the body against the same cap', async () => {
|
||||
proxy.close();
|
||||
}
|
||||
});
|
||||
|
||||
// A stand-in for the impers module whose get() either answers with a minimal
|
||||
// 200 page or refuses the identity the way impers does when the loaded native
|
||||
// library does not know the fingerprint an alias resolves to: it throws an
|
||||
// ImpersonateError before any request leaves the process (issue #40, where a
|
||||
// stale system libcurl-impersonate predating chrome150 broke oc outright).
|
||||
const fakeImpers = (refuse, html = '<html>ok</html>') => {
|
||||
const identities = [];
|
||||
const get = (url, opts) => {
|
||||
identities.push(opts.impersonate);
|
||||
if (refuse.includes(opts.impersonate)) {
|
||||
const err = new Error(`Impersonating ${opts.impersonate} is not supported`);
|
||||
err.name = 'ImpersonateError';
|
||||
return Promise.reject(err);
|
||||
}
|
||||
return Promise.resolve({
|
||||
status: 200,
|
||||
headers: new Map([['content-type', 'text/html']]),
|
||||
text: () => Promise.resolve(html),
|
||||
url,
|
||||
});
|
||||
};
|
||||
return { get, identities };
|
||||
};
|
||||
|
||||
test('a refused chrome fingerprint falls back to the firefox identity', () => withoutProxyEnv(async () => {
|
||||
const impers = fakeImpers(['chrome']);
|
||||
const page = await viaImpers(impers, 'https://public.example/page');
|
||||
assert.deepEqual(impers.identities, ['chrome', 'firefox']);
|
||||
assert.equal(page.via, 'impers:firefox');
|
||||
assert.equal(page.status, 200);
|
||||
assert.equal(page.html, '<html>ok</html>');
|
||||
}));
|
||||
|
||||
test('when both identities are refused the page still arrives via plain fetch', async () => {
|
||||
// The proxy is only here to give viaFetch somewhere real to land without
|
||||
// leaving the machine, same shape as the HTTP_PROXY wiring test above.
|
||||
const seen = [];
|
||||
const proxy = http.createServer((req, res) => {
|
||||
seen.push(req.url);
|
||||
res.writeHead(200, { 'content-type': 'text/html' });
|
||||
res.end('<html><title>via fetch</title></html>');
|
||||
});
|
||||
const port = await listen(proxy);
|
||||
const prev = Object.fromEntries(PROXY_ENV_KEYS.map((k) => [k, process.env[k]]));
|
||||
for (const k of PROXY_ENV_KEYS) delete process.env[k];
|
||||
process.env.HTTP_PROXY = `http://127.0.0.1:${port}`;
|
||||
try {
|
||||
const impers = fakeImpers(['chrome', 'firefox']);
|
||||
const page = await viaImpers(impers, 'http://1.1.1.1/page');
|
||||
assert.deepEqual(impers.identities, ['chrome', 'firefox']);
|
||||
assert.equal(page.via, 'fetch');
|
||||
assert.equal(page.status, 200);
|
||||
assert.equal(page.html, '<html><title>via fetch</title></html>');
|
||||
assert.deepEqual(seen, ['http://1.1.1.1/page']);
|
||||
} finally {
|
||||
proxy.close();
|
||||
for (const k of PROXY_ENV_KEYS) {
|
||||
if (prev[k] === undefined) delete process.env[k];
|
||||
else process.env[k] = prev[k];
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('reddit.com is asked with the firefox fingerprint first', () => withoutProxyEnv(async () => {
|
||||
// Reddit's edge answers the chrome fingerprint with a 403 or a 429 while
|
||||
// letting firefox through (#52), and it rate-limits anonymous readers per
|
||||
// address, so a wasted chrome attempt there is a real cost, not a retry.
|
||||
assert.deepEqual(identityOrder('https://www.reddit.com/r/ClaudeAI/.rss'), ['firefox', 'chrome']);
|
||||
assert.deepEqual(identityOrder('https://old.reddit.com/r/ClaudeAI/'), ['firefox', 'chrome']);
|
||||
assert.deepEqual(identityOrder('https://reddit.com/'), ['firefox', 'chrome']);
|
||||
assert.deepEqual(identityOrder('https://notreddit.com/'), ['chrome', 'firefox']);
|
||||
assert.deepEqual(identityOrder('https://reddit.com.example/'), ['chrome', 'firefox']);
|
||||
assert.deepEqual(identityOrder('https://news.ycombinator.com/'), ['chrome', 'firefox']);
|
||||
assert.deepEqual(identityOrder('not a url'), ['chrome', 'firefox']);
|
||||
|
||||
const impers = fakeImpers([]);
|
||||
const page = await viaImpers(impers, 'https://www.reddit.com/r/ClaudeAI/.rss');
|
||||
assert.deepEqual(impers.identities, ['firefox']);
|
||||
assert.equal(page.via, 'impers:firefox');
|
||||
assert.equal(page.status, 200);
|
||||
}));
|
||||
|
||||
test('reddit.com page URLs are read as their atom feeds, feeds and everything else as asked', () => {
|
||||
// The HTML pages end at a login wall for a logged-out reader, so the page
|
||||
// shapes that have a feed beside them are fetched as that feed: this is what
|
||||
// lets `oc do <n>` on a post in a subreddit feed reach its comments (#59).
|
||||
assert.equal(redditFeedURL('https://www.reddit.com/r/ClaudeAI/comments/1w48zcr/some_title/'),
|
||||
'https://www.reddit.com/r/ClaudeAI/comments/1w48zcr/some_title/.rss');
|
||||
assert.equal(redditFeedURL('https://old.reddit.com/r/ClaudeAI/comments/1w48zcr/some_title/abc123/'),
|
||||
'https://www.reddit.com/r/ClaudeAI/comments/1w48zcr/some_title/abc123/.rss');
|
||||
assert.equal(redditFeedURL('https://reddit.com/comments/1w48zcr'), 'https://www.reddit.com/comments/1w48zcr/.rss');
|
||||
assert.equal(redditFeedURL('https://www.reddit.com/r/ClaudeAI'), 'https://www.reddit.com/r/ClaudeAI/.rss');
|
||||
assert.equal(redditFeedURL('https://www.reddit.com/r/ClaudeAI/top/?t=day'), 'https://www.reddit.com/r/ClaudeAI/top/.rss?t=day');
|
||||
assert.equal(redditFeedURL('https://www.reddit.com/u/spez'), 'https://www.reddit.com/user/spez/.rss');
|
||||
assert.equal(redditFeedURL('https://www.reddit.com/user/spez/'), 'https://www.reddit.com/user/spez/.rss');
|
||||
assert.equal(redditFeedURL('https://www.reddit.com/search?q=claude+code'), 'https://www.reddit.com/search.rss?q=claude+code');
|
||||
assert.equal(redditFeedURL('https://www.reddit.com/'), 'https://www.reddit.com/.rss');
|
||||
// Already a feed, or a shape with no feed, or not reddit at all.
|
||||
assert.equal(redditFeedURL('https://www.reddit.com/r/ClaudeAI/.rss'), null);
|
||||
assert.equal(redditFeedURL('https://www.reddit.com/comments/1w48zcr/.rss'), null);
|
||||
assert.equal(redditFeedURL('https://www.reddit.com/r/ClaudeAI/about/rules/'), null);
|
||||
assert.equal(redditFeedURL('https://www.reddit.com/r/ClaudeAI/wiki/index'), null);
|
||||
assert.equal(redditFeedURL('https://www.reddit.com/login'), null);
|
||||
assert.equal(redditFeedURL('https://www.redditmedia.com/r/ClaudeAI/'), null);
|
||||
assert.equal(redditFeedURL('https://reddit.com.example/r/ClaudeAI/'), null);
|
||||
assert.equal(redditFeedURL('not a url'), null);
|
||||
});
|
||||
|
||||
test('a refused firefox fingerprint on reddit.com falls back to chrome', () => withoutProxyEnv(async () => {
|
||||
const impers = fakeImpers(['firefox']);
|
||||
const page = await viaImpers(impers, 'https://www.reddit.com/r/ClaudeAI/.rss');
|
||||
assert.deepEqual(impers.identities, ['firefox', 'chrome']);
|
||||
assert.equal(page.via, 'impers:chrome');
|
||||
assert.equal(page.status, 200);
|
||||
}));
|
||||
|
||||
test('only an ImpersonateError downgrades; other impers failures propagate', () => withoutProxyEnv(async () => {
|
||||
const impers = {
|
||||
get: () => Promise.reject(new Error('connection reset')),
|
||||
};
|
||||
await assert.rejects(() => viaImpers(impers, 'https://public.example/page'), /connection reset/);
|
||||
}));
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?><feed xmlns="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/"><category term=" reddit.com" label="r/ reddit.com"/><updated>2026-09-04T12:39:19+00:00</updated><id>/comments/1fixture/.rss</id><link rel="self" href="https://www.reddit.com/comments/1fixture/.rss" type="application/atom+xml" /><link rel="alternate" href="https://www.reddit.com/comments/1fixture/" type="text/html" /><title>Why does the budget flag round up? : reddit.com</title><entry><author><name>/u/fixture_poster</name><uri>https://www.reddit.com/user/fixture_poster</uri></author><category term="FixtureSub" label="r/FixtureSub"/><content type="html"><!-- SC_OFF --><div class="md"><p>A page that runs a little over the budget prints whole instead of being cut. Is that on purpose?</p> </div><!-- SC_ON --> &#32; submitted by &#32; <a href="https://www.reddit.com/user/fixture_poster"> /u/fixture_poster </a> &#32; to &#32; <a href="https://www.reddit.com/r/FixtureSub/"> r/FixtureSub </a> <br/> <span><a href="https://www.reddit.com/r/FixtureSub/comments/1fixture/why_does_the_budget_flag_round_up/">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/FixtureSub/comments/1fixture/why_does_the_budget_flag_round_up/">[comments]</a></span></content><id>t3_1fixture</id><link href="https://www.reddit.com/r/FixtureSub/comments/1fixture/why_does_the_budget_flag_round_up/" /><updated>2026-09-01T11:18:08+00:00</updated><published>2026-09-01T11:18:08+00:00</published><title>Why does the budget flag round up?</title></entry><entry><author><name>/u/first_reply</name><uri>https://www.reddit.com/user/first_reply</uri></author><category term="FixtureSub" label="r/FixtureSub" /><content type="html"><!-- SC_OFF --><div class="md"><p>Yes. One extra tool call costs more than the tokens it would save.</p> </div><!-- SC_ON --></content><id>t1_c0000001</id><link href="https://www.reddit.com/r/FixtureSub/comments/1fixture/why_does_the_budget_flag_round_up/c0000001/"/><updated>2026-09-01T13:50:03+00:00</updated><title>/u/first_reply on Why does the budget flag round up?</title></entry><entry><author><name>/u/second_reply</name><uri>https://www.reddit.com/user/second_reply</uri></author><category term="FixtureSub" label="r/FixtureSub" /><content type="html"><!-- SC_OFF --><div class="md"><p>The README calls it a target rather than a hard cap.</p> </div><!-- SC_ON --></content><id>t1_c0000002</id><link href="https://www.reddit.com/r/FixtureSub/comments/1fixture/why_does_the_budget_flag_round_up/c0000002/"/><updated>2026-09-01T14:02:11+00:00</updated><title>/u/second_reply on Why does the budget flag round up?</title></entry></feed>
|
||||
@@ -42,6 +42,19 @@ test('a real site with a missing or unknown verb names the verbs it has', () =>
|
||||
/not a reddit\.com shortcut.*sub <name>/s);
|
||||
});
|
||||
|
||||
test('reddit verbs reach the www.reddit.com atom feeds, not old.reddit.com', () => {
|
||||
// old.reddit.com sends every logged-out request to a login page and the
|
||||
// .json views on www answer 403, so the feeds are the only public reading.
|
||||
assert.equal(resolveSite('reddit', ['sub', 'ClaudeAI']).url, 'https://www.reddit.com/r/ClaudeAI/.rss');
|
||||
assert.equal(resolveSite('reddit', ['post', '1w48zcr']).url, 'https://www.reddit.com/comments/1w48zcr/.rss');
|
||||
assert.equal(resolveSite('reddit', ['search', 'claude code']).url, 'https://www.reddit.com/search.rss?q=claude%20code');
|
||||
for (const verb of Object.keys(sites().get('reddit').commands)) {
|
||||
const { url } = resolveSite('reddit', [verb, 'x']);
|
||||
assert.ok(url.startsWith('https://www.reddit.com/'), `${verb} left www: ${url}`);
|
||||
assert.ok(/\.rss(\?|$)/.test(url), `${verb} is not a feed: ${url}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('a shortcut called with too few args says what it needs', () => {
|
||||
assert.throws(() => resolveSite('gh', ['repo', 'only-cli']), /usage: oc gh repo <owner> <name>/);
|
||||
});
|
||||
@@ -117,12 +130,24 @@ test('the second wave of language docs resolves the same way', () => {
|
||||
assert.equal(
|
||||
resolveSite('rust', ['std', 'vec/struct.Vec']).url,
|
||||
'https://doc.rust-lang.org/std/vec/struct.Vec.html');
|
||||
assert.equal(
|
||||
resolveSite('rust', ['search', 'Vec', 'retain']).url,
|
||||
'https://html.duckduckgo.com/html/?q=site%3Adoc.rust-lang.org+Vec%20retain');
|
||||
assert.equal(
|
||||
resolveSite('java', ['api', 'java.base/java/util/HashMap']).url,
|
||||
'https://docs.oracle.com/en/java/javase/26/docs/api/java.base/java/util/HashMap.html');
|
||||
assert.equal(
|
||||
resolveSite('java', ['search', 'HashMap', 'computeIfAbsent']).url,
|
||||
'https://html.duckduckgo.com/html/?q=site%3Adocs.oracle.com+javase+HashMap%20computeIfAbsent');
|
||||
assert.equal(
|
||||
resolveSite('ts', ['handbook', '2/everyday-types']).url,
|
||||
'https://www.typescriptlang.org/docs/handbook/2/everyday-types.html');
|
||||
assert.equal(
|
||||
resolveSite('ts', ['search', 'satisfies']).url,
|
||||
'https://html.duckduckgo.com/html/?q=site%3Atypescriptlang.org+satisfies');
|
||||
assert.equal(
|
||||
resolveSite('php', ['search', 'array', 'functions']).url,
|
||||
'https://html.duckduckgo.com/html/?q=site%3Aphp.net+array%20functions');
|
||||
assert.equal(
|
||||
resolveSite('learn', ['dotnet', 'system.string']).url,
|
||||
'https://learn.microsoft.com/en-us/dotnet/api/system.string');
|
||||
|
||||
Reference in New Issue
Block a user