roboco/tests/unit/llm/providers/test_kimi_cli_sniff.py

"""kimi_cli_sniff — classify a Kimi run from ONLY its machine-relevant text.

The structural guarantee under test: the model's own on-topic prose (which
can legitimately contain the words "quota-limited" or a "429"/"401" substring
inside a commit hash / id) must NEVER reach the classifier, because
extraction only pulls a structured ``error`` field off error-bearing JSONL
events plus raw stderr — never ``role: assistant`` / ``role: tool`` content.
"""

from __future__ import annotations

import json
from typing import TYPE_CHECKING

from roboco.llm.providers import kimi_cli_sniff as sniff

if TYPE_CHECKING:
    from pathlib import Path

    import pytest


def _write_jsonl(path: Path, lines: list[str]) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text("\n".join(lines) + "\n", encoding="utf-8")


def _error_event(message: str) -> str:
    return json.dumps({"type": "error", "error": {"message": message}})


# ---------------------------------------------------------------------------
# extract_error_text — structural isolation
# ---------------------------------------------------------------------------


def test_extract_error_text_pulls_only_structured_error_field(tmp_path: Path) -> None:
    log = tmp_path / "run.jsonl"
    _write_jsonl(
        log,
        [
            json.dumps(
                {
                    "role": "assistant",
                    "content": "the quota-limited rollout ships this sprint",
                }
            ),
            _error_event("real error text"),
        ],
    )
    assert sniff.extract_error_text(log) == "real error text"


def test_extract_error_text_accepts_bare_string_error() -> None:
    assert sniff._error_text_from_event({"error": "bare string error"}) == (
        "bare string error"
    )


def test_extract_error_text_empty_for_missing_or_error_less_log(
    tmp_path: Path,
) -> None:
    assert sniff.extract_error_text(tmp_path / "nope.jsonl") == ""
    log = tmp_path / "run.jsonl"
    _write_jsonl(log, [json.dumps({"role": "assistant", "content": "hi"})])
    assert sniff.extract_error_text(log) == ""


# ---------------------------------------------------------------------------
# The false-positive class this module exists to kill
# ---------------------------------------------------------------------------


def test_benign_transcript_never_false_parks(tmp_path: Path) -> None:
    """A transcript whose ONLY content is benign on-topic prose — mentioning
    "quota-limited" work and a commit hash containing "429"/"401" — must
    classify as "" (no park), because none of it lives in a structured error
    field the extractor even looks at."""
    log = tmp_path / "run.jsonl"
    _write_jsonl(
        log,
        [
            json.dumps(
                {
                    "role": "assistant",
                    "content": (
                        "Fixed the quota-limited rollout gate. Committed as "
                        "abc4291f, also touched item 40199."
                    ),
                }
            ),
            json.dumps({"role": "tool", "tool_call_id": "1", "content": "ok"}),
        ],
    )
    err_log = tmp_path / "run.err"
    err_log.write_text("", encoding="utf-8")
    assert sniff.classify(log, err_log) == ""


def test_word_boundary_prevents_429_substring_false_positive() -> None:
    assert not sniff.is_rate_limited("commit abc14293 deployed to prod")
    assert not sniff.is_rate_limited("fix4297abc landed")


def test_word_boundary_prevents_401_substring_false_positive() -> None:
    assert not sniff.is_auth_failure("item 40199 was resolved")
    assert not sniff.is_auth_failure("ticket 14012 closed")


# ---------------------------------------------------------------------------
# True positives — the live-verified error text shapes from the spike
# ---------------------------------------------------------------------------


def test_status_code_429_classifies_rate_limit(tmp_path: Path) -> None:
    log = tmp_path / "run.jsonl"
    _write_jsonl(log, [_error_event("request failed with status code: 429")])
    assert sniff.classify(log) == "rate_limit"


def test_engine_overloaded_classifies_rate_limit(tmp_path: Path) -> None:
    log = tmp_path / "run.jsonl"
    _write_jsonl(log, [_error_event("the engine is currently overloaded")])
    assert sniff.classify(log) == "rate_limit"


def test_usage_limit_for_period_classifies_rate_limit(tmp_path: Path) -> None:
    log = tmp_path / "run.jsonl"
    _write_jsonl(log, [_error_event("usage limit for this period exceeded")])
    assert sniff.classify(log) == "rate_limit"


def test_usage_limit_for_billing_cycle_classifies_rate_limit(tmp_path: Path) -> None:
    log = tmp_path / "run.jsonl"
    _write_jsonl(log, [_error_event("usage limit for this billing cycle reached")])
    assert sniff.classify(log) == "rate_limit"


def test_api_key_invalid_classifies_auth(tmp_path: Path) -> None:
    log = tmp_path / "run.jsonl"
    _write_jsonl(log, [_error_event("API Key appears to be invalid")])
    assert sniff.classify(log) == "auth"


def test_membership_benefits_classifies_auth(tmp_path: Path) -> None:
    log = tmp_path / "run.jsonl"
    _write_jsonl(
        log,
        [_error_event("We're unable to verify your membership benefits at this time.")],
    )
    assert sniff.classify(log) == "auth"


def test_classify_reads_stderr_too(tmp_path: Path) -> None:
    log = tmp_path / "run.jsonl"
    _write_jsonl(log, [json.dumps({"role": "assistant", "content": "ok"})])
    err_log = tmp_path / "run.err"
    err_log.write_text("fatal: status code: 429\n", encoding="utf-8")
    assert sniff.classify(log, err_log) == "rate_limit"


def test_classify_missing_files_returns_empty(tmp_path: Path) -> None:
    assert sniff.classify(tmp_path / "nope.jsonl", tmp_path / "nope.err") == ""


def test_rate_limit_checked_before_auth_when_both_present(tmp_path: Path) -> None:
    log = tmp_path / "run.jsonl"
    _write_jsonl(
        log,
        [_error_event("status code: 429, and API Key appears to be invalid too")],
    )
    assert sniff.classify(log) == "rate_limit"


def test_main_cli_prints_classification(
    tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
    log = tmp_path / "run.jsonl"
    _write_jsonl(log, [_error_event("status code: 429")])
    assert sniff.main([str(log)]) == 0
    assert capsys.readouterr().out.strip() == "rate_limit"


def test_main_cli_no_args_prints_empty(capsys: pytest.CaptureFixture[str]) -> None:
    assert sniff.main([]) == 0
    assert capsys.readouterr().out.strip() == ""