From f2e787c577cb67a6311962001de2d7034a75a59f Mon Sep 17 00:00:00 2001 From: Renn F Date: Fri, 19 Jun 2026 10:15:58 +0200 Subject: [PATCH] feat(grok): auto-refresh the SuperGrok token + fail fast on a dead one The grok access token has a ~6h server-set TTL (the client cannot lengthen it), the CLI has no refresh command, and headless 'grok -p' does NOT self-refresh an expired token -- it hangs forever at an interactive 'Waiting for authorization...' prompt. Live evidence: a fleet went silent within ~3 min of the token's 06:54 expiry, every agent a zombie hung at the prompt, requiring a manual 'grok login'. - grok_auth.refresh_if_stale: mint a fresh access token from the offline_access refresh token via xAI's OIDC refresh_token grant (https://auth.x.ai/oauth2/token), atomically rewriting auth.json. The orchestrator runs it once per dispatch tick (serial -> no concurrent refresh-token rotation race; throttled to 60s), keeping the host credential live so agents never mount a dead one. No more manual login. - Entrypoint --check guard: refuse to run (exit 78) on a missing/expired token instead of hanging for hours -- surfaced to _handle_stopped_container. - Orchestrator grok-dir mount flipped read-only -> read-write in all three compose files so the refresh can rewrite auth.json; the per-agent file mount stays RO. Verified: 10 unit tests; the --check guard exits 0/1/1 (valid/expired/missing) inside the real roboco-agent-grok image. Gate green (ruff/mypy/xenon). --- docker-compose.registry.yml | 6 +- docker-compose.yaml | 8 +- docker-compose.yml | 8 +- docker/scripts/grok-cli-agent-entrypoint.sh | 13 ++ roboco/llm/providers/grok.py | 7 +- roboco/llm/providers/grok_auth.py | 239 ++++++++++++++++++++ roboco/runtime/orchestrator.py | 32 +++ tests/unit/llm/providers/test_grok_auth.py | 136 +++++++++++ 8 files changed, 439 insertions(+), 10 deletions(-) create mode 100644 roboco/llm/providers/grok_auth.py create mode 100644 tests/unit/llm/providers/test_grok_auth.py diff --git a/docker-compose.registry.yml b/docker-compose.registry.yml index 9bea95d4..92272e05 100644 --- a/docker-compose.registry.yml +++ b/docker-compose.registry.yml @@ -242,8 +242,10 @@ services: - ${CLAUDE_AUTH_DIR:-${HOME}/.claude}:/root/.claude # SuperGrok auth — mount host ~/.grok at the SAME host path the orchestrator # hands each Grok agent's `-v`, so its auth.json exists() check passes here - # AND the agent bind resolves on the host. `grok login` on the host. RO. - - ${ROBOCO_HOST_GROK_DIR:-${HOME}/.grok}:${ROBOCO_HOST_GROK_DIR:-${HOME}/.grok}:ro + # AND the agent bind resolves on the host. Read-WRITE: the orchestrator + # auto-refreshes the ~6h token in place (grok_auth.refresh_if_stale) so + # agents never mount a dead credential; the agent's own mount stays RO. + - ${ROBOCO_HOST_GROK_DIR:-${HOME}/.grok}:${ROBOCO_HOST_GROK_DIR:-${HOME}/.grok} - ${ROBOCO_DATA_DIR:-./data}/mcp-configs:/app/mcp-configs - ${ROBOCO_DATA_DIR:-./data}/prompts-generated:/app/prompts-generated - ${ROBOCO_DATA_DIR:-./data}/agent-settings:/app/agent-settings diff --git a/docker-compose.yaml b/docker-compose.yaml index 60687c13..b24d2fe7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -350,9 +350,11 @@ services: # SuperGrok auth — mount the host ~/.grok at the SAME host path the # orchestrator passes to each Grok agent's `-v`, so its auth.json exists() # check passes here AND the agent bind resolves on the host. One canonical - # var for source AND target (they must be equal in docker-in-docker); `grok - # login` on the host writes auth.json (auto-refreshing). Read-only. - - ${ROBOCO_HOST_GROK_DIR:-/home/renzof/.grok}:${ROBOCO_HOST_GROK_DIR:-/home/renzof/.grok}:ro + # var for source AND target (they must be equal in docker-in-docker). + # Read-WRITE: the orchestrator auto-refreshes the ~6h token in place + # (grok_auth.refresh_if_stale) so agents never mount a dead credential; + # each agent's own auth.json mount stays read-only. + - ${ROBOCO_HOST_GROK_DIR:-/home/renzof/.grok}:${ROBOCO_HOST_GROK_DIR:-/home/renzof/.grok} # Shared config directory for MCP configs (writable) - ${ROBOCO_DATA_DIR:-./data}/mcp-configs:/app/mcp-configs # Generated prompts directory - composed at runtime from layers diff --git a/docker-compose.yml b/docker-compose.yml index 60687c13..b24d2fe7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -350,9 +350,11 @@ services: # SuperGrok auth — mount the host ~/.grok at the SAME host path the # orchestrator passes to each Grok agent's `-v`, so its auth.json exists() # check passes here AND the agent bind resolves on the host. One canonical - # var for source AND target (they must be equal in docker-in-docker); `grok - # login` on the host writes auth.json (auto-refreshing). Read-only. - - ${ROBOCO_HOST_GROK_DIR:-/home/renzof/.grok}:${ROBOCO_HOST_GROK_DIR:-/home/renzof/.grok}:ro + # var for source AND target (they must be equal in docker-in-docker). + # Read-WRITE: the orchestrator auto-refreshes the ~6h token in place + # (grok_auth.refresh_if_stale) so agents never mount a dead credential; + # each agent's own auth.json mount stays read-only. + - ${ROBOCO_HOST_GROK_DIR:-/home/renzof/.grok}:${ROBOCO_HOST_GROK_DIR:-/home/renzof/.grok} # Shared config directory for MCP configs (writable) - ${ROBOCO_DATA_DIR:-./data}/mcp-configs:/app/mcp-configs # Generated prompts directory - composed at runtime from layers diff --git a/docker/scripts/grok-cli-agent-entrypoint.sh b/docker/scripts/grok-cli-agent-entrypoint.sh index 5ee9ca3d..3a7a5423 100755 --- a/docker/scripts/grok-cli-agent-entrypoint.sh +++ b/docker/scripts/grok-cli-agent-entrypoint.sh @@ -27,6 +27,19 @@ if ! ( cd /app && python -m roboco.agent_sdk.prompt_guard "${ROBOCO_INITIAL_PROM exit 1 fi +# Auth fail-fast guard. The SuperGrok token (~/.grok/auth.json, mounted read-only) +# has a ~6h TTL; on an expired/missing token headless grok does NOT refresh — it +# hangs forever at an interactive "Waiting for authorization..." prompt, which +# reads as a silent zombie container. The orchestrator refreshes the host token +# on a loop; this is the in-container backstop: exit 78 (EX_CONFIG) immediately +# so _handle_stopped_container surfaces it, instead of hanging for hours. +if ! ( cd /app && python -m roboco.llm.providers.grok_auth --check ); then + echo "[grok] auth token missing or expired — refusing to run (would hang at" \ + "the login prompt). Refresh ~/.grok/auth.json (orchestrator auto-refresh or" \ + "'grok login' on the host)." >&2 + exit 78 +fi + # Run the agent. The prompt comes from an env var (never an untrusted argv # positional). `< /dev/null` keeps the headless run from blocking on stdin. We # do NOT `exec`: the script regains control to inspect the result + exit code. diff --git a/roboco/llm/providers/grok.py b/roboco/llm/providers/grok.py index fc4f0efc..be74fe0b 100644 --- a/roboco/llm/providers/grok.py +++ b/roboco/llm/providers/grok.py @@ -156,8 +156,11 @@ class GrokCliProvider(AgentProvider): Read-only so concurrent containers can't corrupt the shared subscription credential; grok writes its per-run state (the rendered ``config.toml``, - ``sessions/``) into the image's own ``~/.grok``. One-shot delivery runs - are short, so the token needs no mid-run refresh. + ``sessions/``) into the image's own ``~/.grok``. The ~6h token is kept + live host-side by the orchestrator (``_refresh_grok_auth`` -> + :func:`roboco.llm.providers.grok_auth.refresh_if_stale`), so a fresh + credential is always what gets mounted; the entrypoint fails fast if it + somehow still finds an expired one rather than hanging at grok's login. """ auth_json = Path(GROK_AUTH_HOST_PATH) / "auth.json" if auth_json.exists(): diff --git a/roboco/llm/providers/grok_auth.py b/roboco/llm/providers/grok_auth.py new file mode 100644 index 00000000..262322c4 --- /dev/null +++ b/roboco/llm/providers/grok_auth.py @@ -0,0 +1,239 @@ +"""Keep the SuperGrok credential live so headless grok agents never hit an +expired token. + +The grok access token in ``~/.grok/auth.json`` has a fixed ~6h TTL set by xAI's +auth server (baked into the JWT ``exp``; the client cannot lengthen it). The +grok CLI exposes only ``login`` / ``logout`` — there is no refresh command — and +headless ``grok -p`` does NOT silently refresh: on an expired token it drops to +an interactive "Waiting for authorization..." prompt that hangs forever in a +container. The bundle does carry an ``offline_access`` refresh token, and xAI's +OIDC token endpoint supports the ``refresh_token`` grant, so we mint a fresh +access token ourselves before expiry and rewrite ``auth.json`` in place. + +The orchestrator owns the refresh (the per-agent mount is read-only, so a +container can't write the credential back), calling :func:`refresh_if_stale` on +the host ``auth.json`` on a loop. The agent entrypoint calls ``--check`` as a +backstop: if the mounted token is missing/expired it exits non-zero immediately +rather than hanging at the login prompt. +""" + +from __future__ import annotations + +import contextlib +import json +import os +import re +import shutil +import sys +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import httpx +import structlog + +if TYPE_CHECKING: + from collections.abc import Callable + +logger = structlog.get_logger(__name__) + +# xAI OIDC issuer; the token endpoint is ``/oauth2/token`` (verified via +# the issuer's ``.well-known/openid-configuration``). A per-entry ``oidc_issuer`` +# overrides it. +_DEFAULT_ISSUER = "https://auth.x.ai" +# Refresh when the token expires within this window: a run that starts inside it +# could outlive the token, so refresh proactively rather than at the last second. +REFRESH_SKEW_SECONDS = int(os.environ.get("ROBOCO_GROK_AUTH_REFRESH_SKEW", "1800")) +# grok writes ``expires_at`` with nanosecond precision + ``Z``; datetime only +# parses up to microseconds, so trim the fractional part to 6 digits. +_FRACTIONAL = re.compile(r"^(?P.*\.\d{6})\d*(?P[+-]\d{2}:\d{2})?$") + + +def default_auth_path() -> Path: + """The grok ``auth.json`` for the current home (``GROK_HOME`` or ``~/.grok``).""" + home = os.environ.get("GROK_HOME") or str(Path.home() / ".grok") + return Path(home) / "auth.json" + + +def _parse_timestamp(raw: str) -> datetime | None: + """Parse grok's ISO-8601 ``expires_at`` (nanosecond ``Z`` form) to a datetime.""" + text = raw.strip() + if not text: + return None + if text.endswith("Z"): + text = text[:-1] + "+00:00" + match = _FRACTIONAL.match(text) + if match: + text = match.group("head") + (match.group("tz") or "") + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) + + +def _to_iso_z(moment: datetime) -> str: + """Serialize back to grok's ``...Z`` form.""" + return moment.astimezone(UTC).isoformat().replace("+00:00", "Z") + + +def _load(auth_path: Path) -> dict[str, Any] | None: + try: + data = json.loads(auth_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + return data if isinstance(data, dict) else None + + +def _credential_entry(bundle: dict[str, Any]) -> tuple[str, dict[str, Any]] | None: + """The single ``:: -> creds`` entry holding a refresh token.""" + for entry_key, value in bundle.items(): + if isinstance(value, dict) and value.get("refresh_token"): + return entry_key, value + return None + + +def seconds_until_expiry( + auth_path: Path, *, now: datetime | None = None +) -> float | None: + """Seconds until the access token expires, or ``None`` if unreadable/absent.""" + bundle = _load(auth_path) + if bundle is None: + return None + entry = _credential_entry(bundle) + if entry is None: + return None + expires_at = _parse_timestamp(str(entry[1].get("expires_at", ""))) + if expires_at is None: + return None + return (expires_at - (now or datetime.now(UTC))).total_seconds() + + +def is_valid( + auth_path: Path, *, skew_seconds: int = 0, now: datetime | None = None +) -> bool: + """True when a token exists and has more than ``skew_seconds`` of life left.""" + remaining = seconds_until_expiry(auth_path, now=now) + return remaining is not None and remaining > skew_seconds + + +def _post_token(url: str, form: dict[str, str]) -> dict[str, Any]: + """POST the OAuth token request; return the parsed JSON body.""" + response = httpx.post(url, data=form, timeout=30.0) + response.raise_for_status() + body = response.json() + return body if isinstance(body, dict) else {} + + +def _atomic_write(auth_path: Path, bundle: dict[str, Any]) -> None: + """Rewrite ``auth.json`` atomically, preserving the original file mode.""" + tmp = auth_path.with_name(auth_path.name + ".refresh.tmp") + tmp.write_text(json.dumps(bundle), encoding="utf-8") + with contextlib.suppress(OSError): + shutil.copymode(auth_path, tmp) + tmp.replace(auth_path) + + +def _is_stale(creds: dict[str, Any], now: datetime, skew_seconds: int) -> bool: + """True when the token is unparseable or within ``skew_seconds`` of expiry.""" + expires_at = _parse_timestamp(str(creds.get("expires_at", ""))) + return expires_at is None or (expires_at - now).total_seconds() <= skew_seconds + + +def _apply_refreshed_token( + creds: dict[str, Any], token: dict[str, Any], now: datetime +) -> None: + """Write the new access token (and rotated refresh token / expiry) into creds.""" + creds["key"] = token["access_token"] + if token.get("refresh_token"): + creds["refresh_token"] = token["refresh_token"] + expires_in = token.get("expires_in") + if isinstance(expires_in, (int, float)): + creds["expires_at"] = _to_iso_z(now + timedelta(seconds=float(expires_in))) + creds["create_time"] = _to_iso_z(now) + + +def _do_refresh( + auth_path: Path, + bundle: dict[str, Any], + entry_key: str, + now: datetime, + post: Callable[[str, dict[str, str]], dict[str, Any]], +) -> str: + """Run the refresh-token grant and persist the result; returns the status.""" + creds = bundle[entry_key] + client_id = creds.get("oidc_client_id") + refresh_token = creds.get("refresh_token") + if not (client_id and refresh_token): + return "no_refresh_token" + issuer = str(creds.get("oidc_issuer") or _DEFAULT_ISSUER).rstrip("/") + try: + token = post( + f"{issuer}/oauth2/token", + { + "grant_type": "refresh_token", + "refresh_token": str(refresh_token), + "client_id": str(client_id), + }, + ) + except Exception as exc: + logger.warning("grok auth refresh request failed", error=str(exc)) + return "failed" + if not token.get("access_token"): + logger.warning("grok auth refresh returned no access_token") + return "failed" + _apply_refreshed_token(creds, token, now) + bundle[entry_key] = creds + try: + _atomic_write(auth_path, bundle) + except OSError as exc: + logger.warning("grok auth refresh write failed", error=str(exc)) + return "failed" + logger.info("grok auth refreshed", expires_in=token.get("expires_in")) + return "refreshed" + + +def refresh_if_stale( + auth_path: Path, + *, + skew_seconds: int = REFRESH_SKEW_SECONDS, + now: datetime | None = None, + post: Callable[[str, dict[str, str]], dict[str, Any]] | None = None, +) -> str: + """Mint a fresh access token from the refresh token if expiry is near. + + Returns a status string: ``fresh`` (still valid, nothing done), ``refreshed`` + (a new token was written), ``missing`` (no auth.json), ``no_refresh_token`` + (no usable credential entry), or ``failed`` (the refresh request errored). + Best-effort: never raises. + """ + bundle = _load(auth_path) + if bundle is None: + return "missing" + entry = _credential_entry(bundle) + if entry is None: + return "no_refresh_token" + entry_key, creds = entry + now = now or datetime.now(UTC) + if not _is_stale(creds, now, skew_seconds): + return "fresh" + return _do_refresh(auth_path, bundle, entry_key, now, post or _post_token) + + +def main(argv: list[str] | None = None) -> int: + """CLI: ``--check`` for the entrypoint backstop, else refresh-if-stale. + + ``--check`` exits non-zero when the mounted token is missing or expired (so + the agent entrypoint can refuse to run instead of hanging at grok's login + prompt). With no flag it refreshes the host token if stale. + """ + args = argv if argv is not None else sys.argv[1:] + auth_path = default_auth_path() + if "--check" in args: + return 0 if is_valid(auth_path) else 1 + status = refresh_if_stale(auth_path) + return 0 if status in {"fresh", "refreshed"} else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/roboco/runtime/orchestrator.py b/roboco/runtime/orchestrator.py index 24b076e3..9fdb9a8c 100644 --- a/roboco/runtime/orchestrator.py +++ b/roboco/runtime/orchestrator.py @@ -6596,12 +6596,44 @@ Start now: evidence(task_id="{task_id}") timeout=self.dispatcher_interval, ) self._dispatch_wake.clear() + await self._refresh_grok_auth() await self._dispatch_all_work() except asyncio.CancelledError: break except Exception as e: logger.error("Dispatcher loop error", error=str(e)) + async def _refresh_grok_auth(self) -> None: + """Keep the host SuperGrok token live so grok agents never mount a dead one. + + The grok access token has a ~6h server-set TTL and headless grok cannot + self-refresh — on an expired token it hangs at an interactive login + prompt. The per-agent mount is read-only, so the orchestrator refreshes + the host ``auth.json`` itself (refresh-token grant) before expiry; agents + then mount a fresh credential. Best-effort, throttled, and serial (run + once per dispatch tick) so concurrent refreshes can't rotate the + refresh-token out from under each other. Never breaks the loop. + """ + now = datetime.now(UTC) + next_check = getattr(self, "_grok_auth_next_check", None) + if next_check is not None and now < next_check: + return + self._grok_auth_next_check = now + timedelta(seconds=60) + try: + from roboco.llm.providers import grok_auth + from roboco.llm.providers.grok import GROK_AUTH_HOST_PATH + + auth_path = Path(GROK_AUTH_HOST_PATH) / "auth.json" + status = await asyncio.to_thread(grok_auth.refresh_if_stale, auth_path) + if status == "refreshed": + logger.info("grok auth token refreshed") + elif status == "failed": + logger.warning( + "grok auth refresh failed; agents may hit an expired token" + ) + except Exception as exc: + logger.error("grok auth refresh hook error", error=str(exc)) + async def _reconcile_orphan_claims_on_startup(self) -> None: """Roll back tasks left in CLAIMED/IN_PROGRESS without a branch. diff --git a/tests/unit/llm/providers/test_grok_auth.py b/tests/unit/llm/providers/test_grok_auth.py new file mode 100644 index 00000000..1125f7f4 --- /dev/null +++ b/tests/unit/llm/providers/test_grok_auth.py @@ -0,0 +1,136 @@ +"""grok_auth — keep the SuperGrok token live via the OAuth refresh-token grant.""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any + +from roboco.llm.providers import grok_auth as ga + +if TYPE_CHECKING: + from pathlib import Path + + import pytest + +_PAST = "2020-01-01T00:00:00.000000000Z" +_FUTURE = "2099-01-01T00:00:00.000000000Z" +_CLIENT = "b1a00492-client" + + +def _bundle(expires_at: str, *, refresh_token: str = "rt") -> dict[str, Any]: + return { + f"https://auth.x.ai::{_CLIENT}": { + "key": "old-access", + "refresh_token": refresh_token, + "expires_at": expires_at, + "oidc_issuer": "https://auth.x.ai", + "oidc_client_id": _CLIENT, + } + } + + +def _write(path: Path, bundle: dict[str, Any]) -> None: + path.write_text(json.dumps(bundle), encoding="utf-8") + + +def test_parse_timestamp_handles_nanosecond_z() -> None: + parsed = ga._parse_timestamp("2026-06-19T06:54:18.840268518Z") + assert parsed is not None + assert parsed.tzinfo is not None + assert ga._parse_timestamp("not a date") is None + assert ga._parse_timestamp("") is None + + +def test_seconds_until_expiry_and_is_valid(tmp_path: Path) -> None: + path = tmp_path / "auth.json" + now = datetime(2026, 6, 19, 6, 0, tzinfo=UTC) # 54m before the 06:54 expiry + _write(path, _bundle("2026-06-19T06:54:18.840268518Z")) + remaining = ga.seconds_until_expiry(path, now=now) + assert remaining is not None + assert 3200 < remaining < 3300 # noqa: PLR2004 — ~54 minutes + assert ga.is_valid(path, now=now) + # Less than an hour left -> not valid under a 1h skew. + assert not ga.is_valid(path, skew_seconds=3600, now=now) + + +def test_seconds_until_expiry_none_for_missing_or_entryless(tmp_path: Path) -> None: + assert ga.seconds_until_expiry(tmp_path / "nope.json") is None + path = tmp_path / "auth.json" + _write(path, {"x": {"no_refresh_token": True}}) + assert ga.seconds_until_expiry(path) is None + + +def test_refresh_skips_when_fresh(tmp_path: Path) -> None: + path = tmp_path / "auth.json" + _write(path, _bundle(_FUTURE)) + calls: list[str] = [] + + def _post(url: str, _form: dict[str, str]) -> dict[str, Any]: + calls.append(url) + return {} + + assert ga.refresh_if_stale(path, post=_post) == "fresh" + assert not calls # no network call when the token is still valid + + +def test_refresh_mints_new_token_when_stale(tmp_path: Path) -> None: + path = tmp_path / "auth.json" + _write(path, _bundle(_PAST)) + + def _post(url: str, form: dict[str, str]) -> dict[str, Any]: + assert url == "https://auth.x.ai/oauth2/token" + assert form == { + "grant_type": "refresh_token", + "refresh_token": "rt", + "client_id": _CLIENT, + } + return { + "access_token": "new-access", + "refresh_token": "new-rt", + "expires_in": 21600, + } + + assert ga.refresh_if_stale(path, post=_post) == "refreshed" + creds = next(iter(json.loads(path.read_text()).values())) + assert creds["key"] == "new-access" + assert creds["refresh_token"] == "new-rt" # rotated + assert ga.is_valid(path) # fresh expires_at ~6h out, valid against real now + + +def test_refresh_missing_file(tmp_path: Path) -> None: + assert ga.refresh_if_stale(tmp_path / "nope.json") == "missing" + + +def test_refresh_no_refresh_token(tmp_path: Path) -> None: + path = tmp_path / "auth.json" + _write(path, {"https://auth.x.ai::c": {"key": "k"}}) + assert ga.refresh_if_stale(path) == "no_refresh_token" + + +def test_refresh_failed_on_post_error_leaves_file_untouched(tmp_path: Path) -> None: + path = tmp_path / "auth.json" + _write(path, _bundle(_PAST)) + + def _boom(_url: str, _form: dict[str, str]) -> dict[str, Any]: + raise RuntimeError("network down") + + assert ga.refresh_if_stale(path, post=_boom) == "failed" + creds = next(iter(json.loads(path.read_text()).values())) + assert creds["key"] == "old-access" # original credential preserved + + +def test_refresh_failed_when_no_access_token(tmp_path: Path) -> None: + path = tmp_path / "auth.json" + _write(path, _bundle(_PAST)) + assert ga.refresh_if_stale(path, post=lambda _u, _f: {"expires_in": 1}) == "failed" + + +def test_main_check_exit_codes(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + home = tmp_path / ".grok" + home.mkdir() + monkeypatch.setenv("GROK_HOME", str(home)) + _write(home / "auth.json", _bundle(_FUTURE)) + assert ga.main(["--check"]) == 0 + _write(home / "auth.json", _bundle(_PAST)) + assert ga.main(["--check"]) == 1