mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
feat(providers): Codex CLI provider — OpenAI via ModelProvider.OPENAI (#659)
* feat(providers): Codex CLI provider — OpenAI via ModelProvider.OPENAI Mirrors the grok blueprint end to end: CodexCliProvider (RO ~/.codex mount, ANTHROPIC_* blanked), an orchestrator-side codex_auth.py refresher (JWT-exp staleness, atomic rewrite, lock-serialized single-use rotation, --check backstop; the CLI's own in-process refresh write no-ops on the RO mount by design — margins keep the orchestrator ahead of the CLI's 5-minute window), config.toml rendering with required=true gateway MCP servers, execpolicy deny rules (forbidden-only), per-role --sandbox (developer=workspace-write, review/doc roles read-only), codex exec --json with pinned ROBOCO_CODEX_CLI_MODEL (gpt-5.3-codex), usage summed from typed turn.completed events priced via the real 4-bucket split, dedicated image + entrypoint, registry/park/finalize/ compose/release wiring. V1 excludes interactive intake/secretary. Per adversarial review: migration 083 seeds the openai provider row enabled=True (without it every routing path 404'd — the whole feature was operationally dead code; grok needed the same seed in 039), the panel picker gained the OpenAI catalog group it silently lacked, and exit classification is structural — only stderr and error.message fields from error events are sniffed (word-boundaried patterns, exact auth phrases, bare 'login' dropped), so the model echoing on-topic words can never false-park the provider fleet-wide, proven by a benign-transcript test. Known open risk flagged, not claimed: whether codex's workspace-write OS sandbox excludes /app is unverified, and no hook mechanism exists to port the bash-guard defense-in-depth. * fix(providers): containment barrier on usage.json reads (code scanning) CodeQL flagged the codex usage read as path injection — correctly: os.path.basename does not neutralize '..', and the upstream segment validator isn't in CodeQL's taint model. The grok/codex reads collapse into one _read_usage_json_contained helper that resolves the built path and refuses anything outside the resolved usage root — a hostile id can never escape regardless of upstream drift. Traversal + containment regression tests added; a stray noqa in the test file replaced with a named constant per repo rule. * fix(providers): use realpath+startswith containment CodeQL recognizes The is_relative_to() guard was a real barrier but not in CodeQL's py/path-injection sanitizer model, so the alert persisted. Switch to the canonical os.path.realpath + startswith(root + os.sep) form, which CodeQL recognizes as a path-traversal barrier; behavior is identical (refuse any candidate resolving outside the usage root). * fix(providers): regexp-allowlist the usage-id segment (CodeQL barrier) Neither is_relative_to nor realpath+startswith was recognized by CodeQL's py/path-injection sanitizer model across the str->Path->open flow. Sanitize the tainted component at the source instead: the id must fullmatch a strict slug token ([A-Za-z0-9][A-Za-z0-9._-]*, no separators, no '..'), which CodeQL recognizes as a path-injection barrier; the realpath+startswith containment stays as defense-in-depth. * fix(providers): standalone regexp guard so CodeQL recognizes the barrier The sanitizer was one disjunct of a compound 'or' condition, which CodeQL's guard analysis does not trace as a barrier. Split the regexp fullmatch into its own single-condition guard (the redundant '..' check is dropped — the required alphanumeric first char already excludes it). --------- Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
Renzo F
@@ -0,0 +1,230 @@
|
||||
"""codex_auth — keep the Codex CLI credential live via the OAuth refresh grant.
|
||||
|
||||
Unlike grok's bundle (keyed by ``<issuer>::<client_id>``, carrying its own
|
||||
``expires_at``), the Codex auth.json is flat — ``{tokens: {access_token,
|
||||
refresh_token, ...}}`` — and staleness is decided purely by decoding the
|
||||
access token's JWT ``exp`` claim.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import pathlib
|
||||
import threading
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from roboco.llm.providers import codex_auth as ca
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _jwt(exp_unix: int) -> str:
|
||||
"""Build a minimal JWT (header.payload.signature) carrying an ``exp`` claim."""
|
||||
payload = (
|
||||
base64.urlsafe_b64encode(json.dumps({"exp": exp_unix}).encode())
|
||||
.rstrip(b"=")
|
||||
.decode()
|
||||
)
|
||||
header = base64.urlsafe_b64encode(b'{"alg":"RS256"}').rstrip(b"=").decode()
|
||||
return f"{header}.{payload}.sig"
|
||||
|
||||
|
||||
def _bundle(access_token: str, *, refresh_token: str = "rt") -> dict[str, Any]:
|
||||
return {
|
||||
"auth_mode": "chatgpt",
|
||||
"tokens": {
|
||||
"id_token": "id-tok",
|
||||
"access_token": access_token,
|
||||
"refresh_token": refresh_token,
|
||||
"account_id": "acct-1",
|
||||
},
|
||||
"last_refresh": "2026-01-01T00:00:00Z",
|
||||
}
|
||||
|
||||
|
||||
def _write(path: Path, bundle: dict[str, Any]) -> None:
|
||||
path.write_text(json.dumps(bundle), encoding="utf-8")
|
||||
|
||||
|
||||
def _exp(delta: timedelta) -> int:
|
||||
return int((datetime.now(UTC) + delta).timestamp())
|
||||
|
||||
|
||||
def test_seconds_until_expiry_and_is_valid(tmp_path: Path) -> None:
|
||||
path = tmp_path / "auth.json"
|
||||
token = _jwt(_exp(timedelta(minutes=54)))
|
||||
_write(path, _bundle(token))
|
||||
remaining = ca.seconds_until_expiry(path)
|
||||
assert remaining is not None
|
||||
assert 3100 < remaining < 3300 # noqa: PLR2004 — ~54 minutes
|
||||
assert ca.is_valid(path)
|
||||
assert not ca.is_valid(path, skew_seconds=3600) # <1h left, 1h skew fails
|
||||
|
||||
|
||||
def test_seconds_until_expiry_none_for_missing_or_entryless(tmp_path: Path) -> None:
|
||||
assert ca.seconds_until_expiry(tmp_path / "nope.json") is None
|
||||
path = tmp_path / "auth.json"
|
||||
_write(path, {"tokens": {"account_id": "x"}}) # no access_token
|
||||
assert ca.seconds_until_expiry(path) is None
|
||||
|
||||
|
||||
def test_seconds_until_expiry_none_for_non_jwt_access_token(tmp_path: Path) -> None:
|
||||
path = tmp_path / "auth.json"
|
||||
_write(path, _bundle("api-key-not-a-jwt"))
|
||||
assert ca.seconds_until_expiry(path) is None
|
||||
|
||||
|
||||
def test_refresh_skips_when_fresh(tmp_path: Path) -> None:
|
||||
path = tmp_path / "auth.json"
|
||||
_write(path, _bundle(_jwt(_exp(timedelta(hours=6)))))
|
||||
calls: list[str] = []
|
||||
|
||||
def _post(url: str, _form: dict[str, str]) -> dict[str, Any]:
|
||||
calls.append(url)
|
||||
return {}
|
||||
|
||||
assert ca.refresh_if_stale(path, post=_post) == "fresh"
|
||||
assert not calls # no network call when the token is still valid
|
||||
|
||||
|
||||
def test_refresh_mints_new_token_when_stale(tmp_path: Path) -> None:
|
||||
path = tmp_path / "auth.json"
|
||||
_write(path, _bundle(_jwt(_exp(timedelta(minutes=-5)))))
|
||||
new_access = _jwt(_exp(timedelta(hours=6)))
|
||||
|
||||
def _post(url: str, form: dict[str, str]) -> dict[str, Any]:
|
||||
assert url == "https://auth.openai.com/oauth/token"
|
||||
assert form["grant_type"] == "refresh_token"
|
||||
assert form["refresh_token"] == "rt"
|
||||
assert form["client_id"]
|
||||
return {
|
||||
"access_token": new_access,
|
||||
"refresh_token": "new-rt",
|
||||
"id_token": "new-id",
|
||||
}
|
||||
|
||||
assert ca.refresh_if_stale(path, post=_post) == "refreshed"
|
||||
bundle = json.loads(path.read_text())
|
||||
assert bundle["tokens"]["access_token"] == new_access
|
||||
assert bundle["tokens"]["refresh_token"] == "new-rt" # rotated
|
||||
assert bundle["tokens"]["id_token"] == "new-id"
|
||||
assert bundle["last_refresh"] != "2026-01-01T00:00:00Z"
|
||||
assert ca.is_valid(path)
|
||||
|
||||
|
||||
def test_refresh_keeps_old_refresh_token_when_response_omits_it(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
path = tmp_path / "auth.json"
|
||||
_write(path, _bundle(_jwt(_exp(timedelta(minutes=-1)))))
|
||||
|
||||
def _post(_url: str, _form: dict[str, str]) -> dict[str, Any]:
|
||||
return {"access_token": _jwt(_exp(timedelta(hours=6)))} # no refresh_token
|
||||
|
||||
assert ca.refresh_if_stale(path, post=_post) == "refreshed"
|
||||
assert json.loads(path.read_text())["tokens"]["refresh_token"] == "rt"
|
||||
|
||||
|
||||
def test_refresh_missing_file(tmp_path: Path) -> None:
|
||||
assert ca.refresh_if_stale(tmp_path / "nope.json") == "missing"
|
||||
|
||||
|
||||
def test_refresh_no_refresh_token_api_key_mode(tmp_path: Path) -> None:
|
||||
# auth_mode=apikey has no tokens/refresh_token — refresh is a graceful no-op.
|
||||
path = tmp_path / "auth.json"
|
||||
_write(path, {"auth_mode": "apikey", "OPENAI_API_KEY": "sk-x"})
|
||||
assert ca.refresh_if_stale(path) == "no_refresh_token"
|
||||
|
||||
|
||||
def test_refresh_failed_on_post_error_leaves_file_untouched(tmp_path: Path) -> None:
|
||||
path = tmp_path / "auth.json"
|
||||
stale = _jwt(_exp(timedelta(minutes=-1)))
|
||||
_write(path, _bundle(stale))
|
||||
|
||||
def _boom(_url: str, _form: dict[str, str]) -> dict[str, Any]:
|
||||
raise RuntimeError("network down")
|
||||
|
||||
assert ca.refresh_if_stale(path, post=_boom) == "failed"
|
||||
assert json.loads(path.read_text())["tokens"]["access_token"] == stale
|
||||
|
||||
|
||||
def test_refresh_failed_when_no_access_token(tmp_path: Path) -> None:
|
||||
path = tmp_path / "auth.json"
|
||||
_write(path, _bundle(_jwt(_exp(timedelta(minutes=-1)))))
|
||||
assert ca.refresh_if_stale(path, post=lambda _u, _f: {}) == "failed"
|
||||
|
||||
|
||||
def test_refresh_persists_rotated_token_when_atomic_write_fails(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""A rotated refresh_token is single-use; if the atomic write fails after
|
||||
rotation, the direct-write fallback must still land it on disk."""
|
||||
path = tmp_path / "auth.json"
|
||||
_write(path, _bundle(_jwt(_exp(timedelta(minutes=-1)))))
|
||||
new_access = _jwt(_exp(timedelta(hours=6)))
|
||||
|
||||
def _post(_url: str, _form: dict[str, str]) -> dict[str, Any]:
|
||||
return {"access_token": new_access, "refresh_token": "rotated-rt"}
|
||||
|
||||
def _boom_replace(_self: pathlib.Path, _target: pathlib.Path) -> pathlib.Path:
|
||||
raise OSError("replace failed (simulated)")
|
||||
|
||||
monkeypatch.setattr(pathlib.Path, "replace", _boom_replace)
|
||||
|
||||
assert ca.refresh_if_stale(path, post=_post) == "refreshed"
|
||||
tokens = json.loads(path.read_text())["tokens"]
|
||||
assert tokens["refresh_token"] == "rotated-rt"
|
||||
assert tokens["access_token"] == new_access
|
||||
|
||||
|
||||
def test_main_check_exit_codes(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
home = tmp_path / ".codex"
|
||||
home.mkdir()
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
_write(home / "auth.json", _bundle(_jwt(_exp(timedelta(hours=6)))))
|
||||
assert ca.main(["--check"]) == 0
|
||||
_write(home / "auth.json", _bundle(_jwt(_exp(timedelta(minutes=-1)))))
|
||||
assert ca.main(["--check"]) == 1
|
||||
|
||||
|
||||
def test_concurrent_refresh_does_not_double_rotate_single_use_token(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Two near-simultaneous ``refresh_if_stale`` calls must POST the
|
||||
refresh-token grant ONCE — a process-wide lock + re-load inside it makes
|
||||
the loser see the winner's refreshed token and return "fresh" instead of
|
||||
re-rotating (mirrors grok_auth's #94 fix)."""
|
||||
path = tmp_path / "auth.json"
|
||||
_write(path, _bundle(_jwt(_exp(timedelta(minutes=-1)))))
|
||||
posted: list[str] = []
|
||||
post_lock = threading.Lock()
|
||||
|
||||
def _post(_url: str, form: dict[str, str]) -> dict[str, Any]:
|
||||
with post_lock:
|
||||
posted.append(form["refresh_token"])
|
||||
time.sleep(0.1)
|
||||
return {
|
||||
"access_token": _jwt(_exp(timedelta(hours=6))),
|
||||
"refresh_token": "new-rt",
|
||||
}
|
||||
|
||||
results: list[str] = []
|
||||
|
||||
def _run() -> None:
|
||||
results.append(ca.refresh_if_stale(path, post=_post))
|
||||
|
||||
threads = [threading.Thread(target=_run) for _ in range(2)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
assert len(posted) == 1 # exactly one grant POST — no double rotation
|
||||
assert all(r in ("refreshed", "fresh") for r in results)
|
||||
assert "refreshed" in results
|
||||
@@ -0,0 +1,120 @@
|
||||
"""codex_cli_config — mcp-config → config.toml + execpolicy rules + combined
|
||||
prompt + per-role sandbox flag."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tomllib
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from roboco.llm.providers import codex_cli_config as cc
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
_SAMPLE_MCP = {
|
||||
"mcpServers": {
|
||||
"roboco-flow": {
|
||||
"command": "uv",
|
||||
"args": ["run", "--no-sync", "python", "-m", "roboco.mcp.flow_server"],
|
||||
"env": {"ROBOCO_AGENT_ID": "be-dev-1", "ROBOCO_AGENT_TOKEN": "tok-123"},
|
||||
},
|
||||
"roboco-do": {"command": "uv", "args": ["run", "x"]},
|
||||
"roboco-optimal": {"command": "uv", "args": ["run", "y"]},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_render_config_toml_is_valid_toml_and_injects_env() -> None:
|
||||
parsed = tomllib.loads(cc.render_config_toml(_SAMPLE_MCP))
|
||||
flow = parsed["mcp_servers"]["roboco-flow"]
|
||||
assert flow["command"] == "uv"
|
||||
assert flow["args"][:2] == ["run", "--no-sync"]
|
||||
assert flow["env"]["ROBOCO_AGENT_TOKEN"] == "tok-123"
|
||||
assert "env" not in parsed["mcp_servers"]["roboco-do"]
|
||||
|
||||
|
||||
def test_render_config_toml_marks_gateway_pair_required() -> None:
|
||||
parsed = tomllib.loads(cc.render_config_toml(_SAMPLE_MCP))
|
||||
assert parsed["mcp_servers"]["roboco-flow"]["required"] is True
|
||||
assert parsed["mcp_servers"]["roboco-do"]["required"] is True
|
||||
# Every other server is best-effort — no `required` key at all.
|
||||
assert "required" not in parsed["mcp_servers"]["roboco-optimal"]
|
||||
|
||||
|
||||
def test_render_config_toml_empty_when_no_servers() -> None:
|
||||
assert cc.render_config_toml({}) == ""
|
||||
assert cc.render_config_toml({"mcpServers": {}}) == ""
|
||||
|
||||
|
||||
def test_sandbox_level_developer_is_workspace_write() -> None:
|
||||
assert cc.sandbox_level_for_role("developer") == "workspace-write"
|
||||
|
||||
|
||||
def test_sandbox_level_other_delivery_roles_are_read_only() -> None:
|
||||
# Narrower than grok's per-role allows_write (documenter also writes there)
|
||||
# — Codex V1 restricts local sandbox writes to developer only; documenter's
|
||||
# real writes ride the roboco-docs MCP server, not a local file edit.
|
||||
for role in ("qa", "documenter", "pr_reviewer", "cell_pm", "main_pm", ""):
|
||||
assert cc.sandbox_level_for_role(role) == "read-only"
|
||||
|
||||
|
||||
def test_codex_cli_args_for_role_carries_sandbox_and_skip_git_check() -> None:
|
||||
dev_args = cc.codex_cli_args_for_role("developer")
|
||||
assert dev_args == ["--sandbox", "workspace-write", "--skip-git-repo-check"]
|
||||
qa_args = cc.codex_cli_args_for_role("qa")
|
||||
assert qa_args == ["--sandbox", "read-only", "--skip-git-repo-check"]
|
||||
|
||||
|
||||
def test_render_execpolicy_rules_covers_git_mutation_destructive_and_raw_pm() -> None:
|
||||
rules = cc.render_execpolicy_rules()
|
||||
assert 'prefix_rule(pattern = ["git", "push"], decision = "forbidden")' in rules
|
||||
assert 'prefix_rule(pattern = ["git", "tag", "-d"], decision = "forbidden")' in (
|
||||
rules
|
||||
)
|
||||
assert 'prefix_rule(pattern = ["rm", "-rf"], decision = "forbidden")' in rules
|
||||
assert 'prefix_rule(pattern = ["uv", "run"], decision = "forbidden")' in rules
|
||||
assert 'prefix_rule(pattern = ["pip", "install"], decision = "forbidden")' in rules
|
||||
# Only allow/forbidden decisions — never `prompt` (blocks headless turns).
|
||||
assert "prompt" not in rules
|
||||
|
||||
|
||||
def test_write_execpolicy_rules_writes_to_dest(tmp_path: Path) -> None:
|
||||
dest = tmp_path / "rules" / "default.rules"
|
||||
cc.write_execpolicy_rules(dest=dest)
|
||||
assert dest.exists()
|
||||
assert "git" in dest.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_render_combined_prompt_joins_system_and_task() -> None:
|
||||
combined = cc.render_combined_prompt("You are the developer.", "Fix the bug.")
|
||||
assert combined.startswith("You are the developer.")
|
||||
assert combined.endswith("Fix the bug.")
|
||||
assert "---" in combined
|
||||
|
||||
|
||||
def test_render_combined_prompt_degrades_gracefully() -> None:
|
||||
assert cc.render_combined_prompt("", "task only") == "task only"
|
||||
assert cc.render_combined_prompt("system only", "") == "system only"
|
||||
assert cc.render_combined_prompt("", "") == ""
|
||||
|
||||
|
||||
def test_write_combined_prompt_reads_source_and_writes_dest(tmp_path: Path) -> None:
|
||||
src = tmp_path / "system-prompt.md"
|
||||
src.write_text("You are the RoboCo developer.", encoding="utf-8")
|
||||
dest = tmp_path / "prompt.txt"
|
||||
found = cc.write_combined_prompt(
|
||||
task_prompt="Implement the feature.", source=src, dest=dest
|
||||
)
|
||||
assert found is True
|
||||
text = dest.read_text(encoding="utf-8")
|
||||
assert "You are the RoboCo developer." in text
|
||||
assert "Implement the feature." in text
|
||||
|
||||
|
||||
def test_write_combined_prompt_degrades_when_source_absent(tmp_path: Path) -> None:
|
||||
dest = tmp_path / "prompt.txt"
|
||||
found = cc.write_combined_prompt(
|
||||
task_prompt="Implement the feature.", source=tmp_path / "absent.md", dest=dest
|
||||
)
|
||||
assert found is False
|
||||
assert dest.read_text(encoding="utf-8") == "Implement the feature."
|
||||
@@ -0,0 +1,188 @@
|
||||
"""codex_cli_sniff — classify a Codex run from ONLY its machine-relevant text.
|
||||
|
||||
The structural guarantee under test: the model's own on-topic prose (which
|
||||
can legitimately contain the words "quota-limited", "login page", or a "429"
|
||||
substring inside a commit hash / id) must NEVER reach the classifier, because
|
||||
extraction only pulls ``error.message`` fields off error-bearing JSONL events
|
||||
plus raw stderr — never ``turn.completed`` / ``item.*`` content.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from roboco.llm.providers import codex_cli_sniff as sniff
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, lines: list[str]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def _turn_failed(message: str) -> str:
|
||||
return json.dumps({"type": "turn.failed", "error": {"message": message}})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# extract_error_text — structural isolation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_extract_error_text_pulls_only_error_message(tmp_path: Path) -> None:
|
||||
log = tmp_path / "run.jsonl"
|
||||
_write_jsonl(
|
||||
log,
|
||||
[
|
||||
json.dumps(
|
||||
{
|
||||
"type": "turn.completed",
|
||||
"usage": {"input_tokens": 1},
|
||||
"text": "the quota-limited rollout ships this sprint",
|
||||
}
|
||||
),
|
||||
_turn_failed("real error text"),
|
||||
],
|
||||
)
|
||||
assert sniff.extract_error_text(log) == "real error text"
|
||||
|
||||
|
||||
def test_extract_error_text_empty_for_missing_or_error_less_log(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
assert sniff.extract_error_text(tmp_path / "nope.jsonl") == ""
|
||||
log = tmp_path / "run.jsonl"
|
||||
_write_jsonl(log, [json.dumps({"type": "turn.completed", "usage": {}})])
|
||||
assert sniff.extract_error_text(log) == ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The false-positive class this fix exists to kill
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_benign_transcript_never_false_parks(tmp_path: Path) -> None:
|
||||
"""A transcript whose ONLY content is benign on-topic prose — mentioning
|
||||
"quota-limited" work, a "login page" bug, and a commit hash containing
|
||||
"429" — must classify as "" (no park), because none of it lives in an
|
||||
error field the extractor even looks at."""
|
||||
log = tmp_path / "run.jsonl"
|
||||
_write_jsonl(
|
||||
log,
|
||||
[
|
||||
json.dumps(
|
||||
{
|
||||
"type": "turn.completed",
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5},
|
||||
}
|
||||
),
|
||||
json.dumps(
|
||||
{
|
||||
"type": "item.completed",
|
||||
"item": {
|
||||
"type": "agent_message",
|
||||
"text": (
|
||||
"Fixed the quota-limited rollout gate and the "
|
||||
"login page redirect bug. Committed as abc4291f."
|
||||
),
|
||||
},
|
||||
}
|
||||
),
|
||||
],
|
||||
)
|
||||
err_log = tmp_path / "run.err"
|
||||
err_log.write_text("", encoding="utf-8")
|
||||
assert sniff.classify(log, err_log) == ""
|
||||
|
||||
|
||||
def test_word_boundary_prevents_429_substring_false_positive() -> None:
|
||||
# "429" embedded inside a larger digit/word run must not match — grok's
|
||||
# own \b429\b pattern, restored here after an initial cut dropped it.
|
||||
assert not sniff.is_rate_limited("commit abc14293 deployed to prod")
|
||||
assert not sniff.is_rate_limited("fix4297abc landed")
|
||||
# "quota" alone legitimately matches wherever it appears (grok's own
|
||||
# pattern, unchanged) — the false-positive class this fix kills is SCOPE
|
||||
# (which text gets scanned, i.e. never turn.completed/item.* content),
|
||||
# not the word "quota" itself. See test_benign_transcript_never_false_parks.
|
||||
|
||||
|
||||
def test_bare_login_word_does_not_classify_as_auth() -> None:
|
||||
# "login" was dropped from the auth pattern — a mention of a login PAGE
|
||||
# (this repo's own panel) must not false-park the provider.
|
||||
assert not sniff.is_auth_failure("please visit the login page to continue")
|
||||
assert not sniff.is_auth_failure("login required")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# True positives — real machine-extracted error text
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_real_429_error_message_classifies_rate_limit(tmp_path: Path) -> None:
|
||||
log = tmp_path / "run.jsonl"
|
||||
_write_jsonl(log, [_turn_failed("Rate limit exceeded: 429 Too Many Requests")])
|
||||
assert sniff.classify(log) == "rate_limit"
|
||||
|
||||
|
||||
def test_insufficient_quota_error_message_classifies_rate_limit(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
log = tmp_path / "run.jsonl"
|
||||
_write_jsonl(log, [_turn_failed("insufficient_quota: billing hard limit hit")])
|
||||
assert sniff.classify(log) == "rate_limit"
|
||||
|
||||
|
||||
def test_exact_auth_phrase_refresh_token_expired_classifies_auth(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
log = tmp_path / "run.jsonl"
|
||||
_write_jsonl(
|
||||
log, [_turn_failed("Your refresh token has expired, please re-authenticate")]
|
||||
)
|
||||
assert sniff.classify(log) == "auth"
|
||||
|
||||
|
||||
def test_exact_auth_phrase_not_signed_in_classifies_auth(tmp_path: Path) -> None:
|
||||
log = tmp_path / "run.jsonl"
|
||||
_write_jsonl(log, [_turn_failed("Error: not signed in")])
|
||||
assert sniff.classify(log) == "auth"
|
||||
|
||||
|
||||
def test_classify_reads_stderr_too(tmp_path: Path) -> None:
|
||||
log = tmp_path / "run.jsonl"
|
||||
_write_jsonl(log, [json.dumps({"type": "turn.completed", "usage": {}})])
|
||||
err_log = tmp_path / "run.err"
|
||||
err_log.write_text("fatal: 429 too many requests\n", encoding="utf-8")
|
||||
assert sniff.classify(log, err_log) == "rate_limit"
|
||||
|
||||
|
||||
def test_classify_missing_files_returns_empty(tmp_path: Path) -> None:
|
||||
assert sniff.classify(tmp_path / "nope.jsonl", tmp_path / "nope.err") == ""
|
||||
|
||||
|
||||
def test_rate_limit_checked_before_auth_when_both_present(tmp_path: Path) -> None:
|
||||
log = tmp_path / "run.jsonl"
|
||||
_write_jsonl(
|
||||
log,
|
||||
[_turn_failed("429 too many requests, and also not signed in downstream")],
|
||||
)
|
||||
assert sniff.classify(log) == "rate_limit"
|
||||
|
||||
|
||||
def test_main_cli_prints_classification(
|
||||
tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
log = tmp_path / "run.jsonl"
|
||||
_write_jsonl(log, [_turn_failed("429 too many requests")])
|
||||
assert sniff.main([str(log)]) == 0
|
||||
assert capsys.readouterr().out.strip() == "rate_limit"
|
||||
|
||||
|
||||
def test_main_cli_no_args_prints_empty(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
assert sniff.main([]) == 0
|
||||
assert capsys.readouterr().out.strip() == ""
|
||||
@@ -0,0 +1,159 @@
|
||||
"""codex_cli_usage — sum real input/output/cache usage across ``turn.completed``
|
||||
events in a captured ``codex exec --json`` JSONL log."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from roboco.llm.providers import codex_cli_usage as cu
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _turn_completed(
|
||||
*,
|
||||
input_tokens: int,
|
||||
cached_input_tokens: int = 0,
|
||||
cache_write_input_tokens: int = 0,
|
||||
output_tokens: int,
|
||||
reasoning_output_tokens: int = 0,
|
||||
) -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"type": "turn.completed",
|
||||
"usage": {
|
||||
"input_tokens": input_tokens,
|
||||
"cached_input_tokens": cached_input_tokens,
|
||||
"cache_write_input_tokens": cache_write_input_tokens,
|
||||
"output_tokens": output_tokens,
|
||||
"reasoning_output_tokens": reasoning_output_tokens,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, lines: list[str]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def test_aggregate_sums_across_multiple_turn_completed_events(tmp_path: Path) -> None:
|
||||
log = tmp_path / "run.jsonl"
|
||||
_write_jsonl(
|
||||
log,
|
||||
[
|
||||
json.dumps({"type": "thread.started"}),
|
||||
json.dumps({"type": "turn.started"}),
|
||||
_turn_completed(
|
||||
input_tokens=1000, cached_input_tokens=200, output_tokens=100
|
||||
),
|
||||
json.dumps({"type": "item.completed", "item": {"type": "command"}}),
|
||||
_turn_completed(
|
||||
input_tokens=500,
|
||||
cached_input_tokens=100,
|
||||
cache_write_input_tokens=50,
|
||||
output_tokens=80,
|
||||
reasoning_output_tokens=20,
|
||||
),
|
||||
],
|
||||
)
|
||||
agg = cu.aggregate_usage_from_jsonl(log)
|
||||
assert agg["input_tokens"] == 1500 # noqa: PLR2004
|
||||
assert agg["cached_input_tokens"] == 300 # noqa: PLR2004
|
||||
assert agg["cache_write_input_tokens"] == 50 # noqa: PLR2004
|
||||
assert agg["output_tokens"] == 180 # noqa: PLR2004
|
||||
assert agg["reasoning_output_tokens"] == 20 # noqa: PLR2004
|
||||
assert agg["turns"] == 2 # noqa: PLR2004
|
||||
|
||||
|
||||
def test_aggregate_ignores_turn_failed_and_bad_lines(tmp_path: Path) -> None:
|
||||
log = tmp_path / "run.jsonl"
|
||||
_write_jsonl(
|
||||
log,
|
||||
[
|
||||
"not json",
|
||||
json.dumps({"type": "turn.failed", "error": {"message": "boom"}}),
|
||||
_turn_completed(input_tokens=10, output_tokens=5),
|
||||
],
|
||||
)
|
||||
agg = cu.aggregate_usage_from_jsonl(log)
|
||||
assert agg["input_tokens"] == 10 # noqa: PLR2004
|
||||
assert agg["turns"] == 1
|
||||
|
||||
|
||||
def test_aggregate_zero_for_missing_or_empty_log(tmp_path: Path) -> None:
|
||||
agg = cu.aggregate_usage_from_jsonl(tmp_path / "nope.jsonl")
|
||||
assert agg["turns"] == 0
|
||||
assert all(v == 0 for k, v in agg.items() if k != "turns")
|
||||
|
||||
|
||||
def test_usage_and_cost_treats_cached_as_subset_of_input() -> None:
|
||||
# cached_input_tokens is a SUBSET of input_tokens (not additional) — the
|
||||
# "fresh" input priced at the full rate is the difference.
|
||||
agg = {
|
||||
"input_tokens": 1000,
|
||||
"cached_input_tokens": 300,
|
||||
"cache_write_input_tokens": 0,
|
||||
"output_tokens": 200,
|
||||
"reasoning_output_tokens": 50,
|
||||
}
|
||||
tin, tout, cr, cw, cost = cu.usage_and_cost("gpt-5.3-codex", agg)
|
||||
assert tin == 700 # 1000 - 300 # noqa: PLR2004
|
||||
assert tout == 250 # output + reasoning folded in # noqa: PLR2004
|
||||
assert cr == 300 # noqa: PLR2004
|
||||
assert cw == 0
|
||||
assert cost > 0.0
|
||||
|
||||
|
||||
def test_usage_and_cost_never_goes_negative_when_cached_exceeds_input() -> None:
|
||||
agg = {
|
||||
"input_tokens": 10,
|
||||
"cached_input_tokens": 50, # malformed/inconsistent upstream data
|
||||
"cache_write_input_tokens": 0,
|
||||
"output_tokens": 0,
|
||||
"reasoning_output_tokens": 0,
|
||||
}
|
||||
tin, *_rest = cu.usage_and_cost("gpt-5.3-codex", agg)
|
||||
assert tin == 0
|
||||
|
||||
|
||||
def test_capture_run_usage_writes_usage_json(tmp_path: Path) -> None:
|
||||
log = tmp_path / "run.jsonl"
|
||||
_write_jsonl(log, [_turn_completed(input_tokens=100, output_tokens=50)])
|
||||
out = tmp_path / "usage.json"
|
||||
tokens = cu.capture_run_usage(run_log=log, model="gpt-5.3-codex", out_path=out)
|
||||
assert tokens == (100, 50, 0, 0)
|
||||
data = json.loads(out.read_text())
|
||||
assert data["model"] == "gpt-5.3-codex"
|
||||
assert data["tokens_input"] == 100 # noqa: PLR2004
|
||||
assert data["tokens_output"] == 50 # noqa: PLR2004
|
||||
assert data["turns"] == 1
|
||||
assert data["cost_usd"] > 0.0
|
||||
|
||||
|
||||
def test_main_writes_usage_file(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
log = tmp_path / "run.jsonl"
|
||||
_write_jsonl(log, [_turn_completed(input_tokens=200, output_tokens=100)])
|
||||
out = tmp_path / "usage.json"
|
||||
monkeypatch.setattr(cu, "USAGE_OUT_PATH", out)
|
||||
monkeypatch.setenv("ROBOCO_CODEX_RUN_LOG", str(log))
|
||||
monkeypatch.setenv("ROBOCO_AGENT_MODEL", "gpt-5.3-codex")
|
||||
assert cu.main() == 0
|
||||
data = json.loads(out.read_text())
|
||||
assert data["tokens_input"] == 200 # noqa: PLR2004
|
||||
assert data["tokens_output"] == 100 # noqa: PLR2004
|
||||
|
||||
|
||||
def test_main_warns_when_run_log_env_missing(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
monkeypatch.delenv("ROBOCO_CODEX_RUN_LOG", raising=False)
|
||||
with caplog.at_level("WARNING", logger="roboco.llm.providers.codex_cli_usage"):
|
||||
assert cu.main() == 0
|
||||
assert any("ROBOCO_CODEX_RUN_LOG" in r.message for r in caplog.records)
|
||||
Reference in New Issue
Block a user