mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
SnapOtterandGitHub
991c981529
* fix: make OCR portable and reliable * fix: harden OCR installation portability * fix: pin OCR partials across downloads * fix: make OCR execution reliably asynchronous * fix: harden OCR portability and docs routes * fix: preserve decoder and docs safeguards
2755 lines
110 KiB
Python
2755 lines
110 KiB
Python
"""Fault-injection tests for the v3 immutable runtime installer."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import base64
|
|
import errno
|
|
import fcntl
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
from packages.ai.python import install_runtime
|
|
|
|
|
|
TARGET = "linux-amd64-cpu-py312"
|
|
FAMILY = "ocr"
|
|
|
|
|
|
def _sha256(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
class RuntimeFixture:
|
|
def __init__(self, root: Path) -> None:
|
|
self.root = root
|
|
self.ai_data_dir = root / "ai"
|
|
self.archive = root / "runtime.tar.gz"
|
|
self.files = {
|
|
"venv/bin/python": (b"#!/bin/sh\n", 0o755),
|
|
"ocr_runner.py": (b"# immutable OCR entrypoint\n", 0o644),
|
|
"ocr_runtime.py": (b"# immutable OCR adapter\n", 0o644),
|
|
"models/pp-ocrv6-small.onnx": (b"small model bytes", 0o644),
|
|
"models/pp-ocrv6-medium.onnx": (b"medium model bytes", 0o644),
|
|
"smoke.py": (
|
|
b"import os, pathlib, sys\n"
|
|
b"assert os.environ['SNAPOTTER_NETWORK_DISABLED'] == '1'\n"
|
|
b"assert os.environ['SNAPOTTER_ALLOW_MODEL_DOWNLOAD'] == '0'\n"
|
|
b"assert os.environ['HF_HUB_OFFLINE'] == '1'\n"
|
|
b"assert os.environ['TRANSFORMERS_OFFLINE'] == '1'\n"
|
|
b"assert os.environ['PIP_NO_INDEX'] == '1'\n"
|
|
b"assert os.environ['PYTHONDONTWRITEBYTECODE'] == '1'\n"
|
|
b"assert 'PYTHONPATH' not in os.environ\n"
|
|
b"assert 'PYTHONHOME' not in os.environ\n"
|
|
b"runtime = pathlib.Path(os.environ['SNAPOTTER_RUNTIME_ROOT'])\n"
|
|
b"assert runtime.resolve() == pathlib.Path.cwd().resolve()\n"
|
|
b"log = os.environ.get('SNAPOTTER_SMOKE_LOG')\n"
|
|
b"if log:\n"
|
|
b" with open(log, 'a', encoding='utf-8') as handle:\n"
|
|
b" handle.write(str(runtime) + '\\n')\n"
|
|
b"if os.environ.get('FAIL_FINAL') == '1' and 'staging' not in str(runtime):\n"
|
|
b" sys.exit(23)\n"
|
|
b"if os.environ.get('FLOOD_SMOKE') == '1':\n"
|
|
b" os.write(2, b'x' * (2 * 1024 * 1024))\n"
|
|
b"if os.environ.get('FAIL_SMOKE_WITH_DETAIL') == '1':\n"
|
|
b" sys.stderr.write('representative smoke diagnostic')\n"
|
|
b" sys.exit(24)\n",
|
|
0o644,
|
|
),
|
|
}
|
|
self._write_archive()
|
|
|
|
def _write_archive(self, additions=(), *, mode: str = "w:gz") -> None:
|
|
with tarfile.open(self.archive, mode) as archive:
|
|
for name, (data, mode) in self.files.items():
|
|
info = tarfile.TarInfo(name)
|
|
info.size = len(data)
|
|
info.mode = mode
|
|
archive.addfile(info, io.BytesIO(data))
|
|
for info, data in additions:
|
|
archive.addfile(info, None if data is None else io.BytesIO(data))
|
|
|
|
def artifact(self) -> dict:
|
|
archive_bytes = self.archive.read_bytes()
|
|
return {
|
|
"family": FAMILY,
|
|
"target": TARGET,
|
|
"generation": "1.0.0-abc123",
|
|
"version": "1.0.0",
|
|
"platform": "linux",
|
|
"arch": "amd64",
|
|
"archive": {
|
|
"sha256": _sha256(archive_bytes),
|
|
"size": len(archive_bytes),
|
|
"expandedSize": sum(len(data) for data, _mode in self.files.values()),
|
|
},
|
|
"files": [
|
|
{
|
|
"path": name,
|
|
"sha256": _sha256(data),
|
|
"size": len(data),
|
|
"mode": mode,
|
|
}
|
|
for name, (data, mode) in sorted(self.files.items())
|
|
],
|
|
"runtime": {
|
|
"pythonPath": "venv/bin/python",
|
|
"entrypoint": "ocr_runner.py",
|
|
"adapterPath": "ocr_runtime.py",
|
|
},
|
|
"models": {
|
|
"pp-ocrv6-small": _sha256(self.files["models/pp-ocrv6-small.onnx"][0]),
|
|
"pp-ocrv6-medium": _sha256(
|
|
self.files["models/pp-ocrv6-medium.onnx"][0]
|
|
),
|
|
},
|
|
"compatibility": {
|
|
"protocolVersion": 1,
|
|
"snapotterVersion": "2.1.0",
|
|
},
|
|
"capabilities": {
|
|
"qualities": ["balanced", "best"],
|
|
"providers": ["CPUExecutionProvider"],
|
|
},
|
|
"resources": {"minimumMemoryBytes": 4 * 1024 * 1024 * 1024},
|
|
}
|
|
|
|
def smoke_command(self) -> list[str]:
|
|
return [sys.executable, "{runtime}/smoke.py"]
|
|
|
|
def authenticated_index(self, artifact: dict | None = None) -> bytes:
|
|
value = {
|
|
"schemaVersion": 1,
|
|
"artifacts": [artifact or self.artifact()],
|
|
"signature": {
|
|
"keyId": "test-key",
|
|
"algorithm": "ed25519",
|
|
"value": "authenticated-by-test-caller",
|
|
},
|
|
}
|
|
return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode(
|
|
"utf-8"
|
|
)
|
|
|
|
|
|
class InstallRuntimeTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.temporary_directory = tempfile.TemporaryDirectory(
|
|
prefix="snapotter-install-runtime-"
|
|
)
|
|
self.root = Path(self.temporary_directory.name)
|
|
self.fixture = RuntimeFixture(self.root)
|
|
self.host = install_runtime.HostInfo(platform="linux", machine="x86_64")
|
|
|
|
def tearDown(self) -> None:
|
|
self.temporary_directory.cleanup()
|
|
|
|
def _active_path(self) -> Path:
|
|
return self.fixture.ai_data_dir / "v3" / "active" / "ocr.json"
|
|
|
|
def _activation_path(self) -> Path:
|
|
return self.fixture.ai_data_dir / "v3" / "rollback" / "ocr.json"
|
|
|
|
def _generation_lock_path(self, generation: str = "1.0.0-abc123") -> Path:
|
|
return (
|
|
self.fixture.ai_data_dir
|
|
/ "v3"
|
|
/ "locks"
|
|
/ "generations"
|
|
/ FAMILY
|
|
/ f"{generation}.lock"
|
|
)
|
|
|
|
def _assert_shared_generation_probe_is_blocked(self, lock_path: Path) -> None:
|
|
descriptor = os.open(
|
|
lock_path,
|
|
os.O_RDONLY
|
|
| getattr(os, "O_CLOEXEC", 0)
|
|
| getattr(os, "O_NOFOLLOW", 0),
|
|
)
|
|
try:
|
|
with self.assertRaises(BlockingIOError):
|
|
fcntl.flock(descriptor, fcntl.LOCK_SH | fcntl.LOCK_NB)
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
def _commit(self, generation: str = "1.0.0-abc123") -> bool:
|
|
return install_runtime.commit_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
family=FAMILY,
|
|
expected_generation=generation,
|
|
)
|
|
|
|
def _write_previous_active(self) -> bytes:
|
|
artifact = self.fixture.artifact()
|
|
artifact["generation"] = "0.9.0-previous"
|
|
install_runtime.install_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
artifact=artifact,
|
|
authenticated_index=self.fixture.authenticated_index(artifact),
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
)
|
|
self._commit("0.9.0-previous")
|
|
return self._active_path().read_bytes()
|
|
|
|
def _install(self, **kwargs):
|
|
artifact = self.fixture.artifact()
|
|
return install_runtime.install_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
artifact=artifact,
|
|
authenticated_index=self.fixture.authenticated_index(artifact),
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
**kwargs,
|
|
)
|
|
|
|
@staticmethod
|
|
def _write_gc_descriptor(path: Path, family: str, generation: str) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(
|
|
install_runtime._canonical_json(
|
|
{
|
|
"schemaVersion": install_runtime.DESCRIPTOR_SCHEMA_VERSION,
|
|
"family": family,
|
|
"generation": generation,
|
|
"artifact": {
|
|
"signedIndex": {
|
|
"path": f"indexes/{'a' * 64}.json",
|
|
}
|
|
},
|
|
}
|
|
)
|
|
)
|
|
|
|
def _install_with_post_verification_action(self, action):
|
|
verified = threading.Event()
|
|
proceed = threading.Event()
|
|
outcome: dict[str, object] = {}
|
|
original_verify = install_runtime._verify_archive
|
|
|
|
def pause_after_verification(*args, **kwargs):
|
|
result = original_verify(*args, **kwargs)
|
|
verified.set()
|
|
if not proceed.wait(timeout=5):
|
|
raise RuntimeError(
|
|
"test timed out waiting to resume archive installation"
|
|
)
|
|
return result
|
|
|
|
def run_install() -> None:
|
|
try:
|
|
outcome["result"] = self._install()
|
|
except BaseException as error:
|
|
outcome["error"] = error
|
|
|
|
with mock.patch.object(
|
|
install_runtime, "_verify_archive", side_effect=pause_after_verification
|
|
):
|
|
worker = threading.Thread(target=run_install, daemon=True)
|
|
worker.start()
|
|
try:
|
|
self.assertTrue(
|
|
verified.wait(timeout=5),
|
|
"installer did not reach the post-verification barrier",
|
|
)
|
|
action()
|
|
finally:
|
|
proceed.set()
|
|
worker.join(timeout=10)
|
|
|
|
self.assertFalse(worker.is_alive(), "installer did not finish after barrier")
|
|
error = outcome.get("error")
|
|
if isinstance(error, BaseException):
|
|
raise error
|
|
return outcome["result"]
|
|
|
|
def test_success_runs_one_final_smoke_then_atomically_activates(self) -> None:
|
|
smoke_log = self.root / "smoke.log"
|
|
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{"PYTHONPATH": "/ambient/python", "PYTHONHOME": "/ambient/home"},
|
|
):
|
|
result = self._install(smoke_env={"SNAPOTTER_SMOKE_LOG": str(smoke_log)})
|
|
|
|
generation_root = (
|
|
self.fixture.ai_data_dir
|
|
/ "v3"
|
|
/ "runtimes"
|
|
/ FAMILY
|
|
/ TARGET
|
|
/ "1.0.0-abc123"
|
|
)
|
|
self.assertEqual(result.generation_root, generation_root.resolve())
|
|
self.assertEqual(len(smoke_log.read_text(encoding="utf-8").splitlines()), 1)
|
|
descriptor = json.loads(self._active_path().read_text(encoding="utf-8"))
|
|
self.assertEqual(descriptor["family"], FAMILY)
|
|
self.assertEqual(descriptor["generation"], "1.0.0-abc123")
|
|
self.assertEqual(descriptor["artifact"]["target"], TARGET)
|
|
self.assertEqual(
|
|
descriptor["artifact"]["models"], self.fixture.artifact()["models"]
|
|
)
|
|
self.assertEqual(
|
|
descriptor["artifact"]["modelFiles"]["pp-ocrv6-small"],
|
|
{
|
|
"path": f"runtimes/{FAMILY}/{TARGET}/1.0.0-abc123/models/pp-ocrv6-small.onnx",
|
|
"sha256": self.fixture.artifact()["models"]["pp-ocrv6-small"],
|
|
"size": len(self.fixture.files["models/pp-ocrv6-small.onnx"][0]),
|
|
},
|
|
)
|
|
self.assertEqual(
|
|
descriptor["runtime"]["pythonPath"],
|
|
f"runtimes/{FAMILY}/{TARGET}/1.0.0-abc123/venv/bin/python",
|
|
)
|
|
self.assertEqual(
|
|
descriptor["runtime"]["integrityFiles"]["adapter"],
|
|
{
|
|
"path": f"runtimes/{FAMILY}/{TARGET}/1.0.0-abc123/ocr_runtime.py",
|
|
"sha256": _sha256(self.fixture.files["ocr_runtime.py"][0]),
|
|
"size": len(self.fixture.files["ocr_runtime.py"][0]),
|
|
},
|
|
)
|
|
self.assertFalse(any((self.fixture.ai_data_dir / "v3" / "staging").iterdir()))
|
|
|
|
def test_first_install_stays_pending_until_commit_and_reconcile_is_idempotent(
|
|
self,
|
|
) -> None:
|
|
result = self._install()
|
|
marker = json.loads(self._activation_path().read_bytes())
|
|
self.assertEqual(marker["status"], "pending")
|
|
|
|
restored = install_runtime.reconcile_pending_activations(
|
|
ai_data_dir=self.fixture.ai_data_dir
|
|
)
|
|
|
|
self.assertEqual(restored, {FAMILY: None})
|
|
self.assertFalse(self._active_path().exists())
|
|
self.assertFalse(self._activation_path().exists())
|
|
self.assertFalse(result.generation_root.exists())
|
|
self.assertEqual(
|
|
install_runtime.reconcile_pending_activations(
|
|
ai_data_dir=self.fixture.ai_data_dir
|
|
),
|
|
{},
|
|
)
|
|
|
|
def test_reconcile_handles_crash_after_pending_marker_before_descriptor_swap(
|
|
self,
|
|
) -> None:
|
|
result = self._install()
|
|
self._active_path().unlink()
|
|
|
|
restored = install_runtime.reconcile_pending_activations(
|
|
ai_data_dir=self.fixture.ai_data_dir
|
|
)
|
|
|
|
self.assertEqual(restored, {FAMILY: None})
|
|
self.assertFalse(self._activation_path().exists())
|
|
self.assertFalse(result.generation_root.exists())
|
|
|
|
self.assertEqual(restored, {FAMILY: None})
|
|
self.assertFalse(self._active_path().exists())
|
|
self.assertFalse(result.generation_root.exists())
|
|
self.assertEqual(
|
|
install_runtime.reconcile_pending_activations(
|
|
ai_data_dir=self.fixture.ai_data_dir
|
|
),
|
|
{},
|
|
)
|
|
|
|
def test_upgrade_pending_activation_reconciles_to_exact_committed_descriptor(
|
|
self,
|
|
) -> None:
|
|
previous = self._write_previous_active()
|
|
upgraded = self._install()
|
|
|
|
self.assertEqual(
|
|
json.loads(self._activation_path().read_bytes())["status"], "pending"
|
|
)
|
|
self.assertEqual(
|
|
json.loads(self._active_path().read_bytes())["generation"],
|
|
upgraded.generation,
|
|
)
|
|
|
|
restored = install_runtime.reconcile_pending_activations(
|
|
ai_data_dir=self.fixture.ai_data_dir
|
|
)
|
|
|
|
self.assertEqual(restored, {FAMILY: "0.9.0-previous"})
|
|
self.assertEqual(self._active_path().read_bytes(), previous)
|
|
self.assertFalse(upgraded.generation_root.exists())
|
|
marker = json.loads(self._activation_path().read_bytes())
|
|
self.assertEqual(marker["status"], "committed")
|
|
self.assertEqual(marker["activatedGeneration"], "0.9.0-previous")
|
|
self.assertEqual(marker["activatedDescriptorSha256"], _sha256(previous))
|
|
self.assertIsNone(marker["previousDescriptorB64"])
|
|
|
|
def test_commit_is_idempotent_and_startup_reconciliation_preserves_it(self) -> None:
|
|
result = self._install()
|
|
|
|
self.assertTrue(self._commit(result.generation))
|
|
committed = self._activation_path().read_bytes()
|
|
self.assertEqual(json.loads(committed)["status"], "committed")
|
|
self.assertFalse(self._commit(result.generation))
|
|
self.assertEqual(self._activation_path().read_bytes(), committed)
|
|
|
|
self.assertEqual(
|
|
install_runtime.reconcile_pending_activations(
|
|
ai_data_dir=self.fixture.ai_data_dir
|
|
),
|
|
{},
|
|
)
|
|
self.assertTrue(self._active_path().exists())
|
|
self.assertTrue(result.generation_root.exists())
|
|
|
|
def test_commit_does_not_turn_unrelated_gc_failure_into_an_ambiguous_result(
|
|
self,
|
|
) -> None:
|
|
result = self._install()
|
|
|
|
with mock.patch.object(
|
|
install_runtime,
|
|
"recover_and_gc",
|
|
side_effect=install_runtime.InstallError("unrelated GC failure"),
|
|
):
|
|
self.assertTrue(self._commit(result.generation))
|
|
|
|
self.assertEqual(
|
|
json.loads(self._activation_path().read_bytes())["status"], "committed"
|
|
)
|
|
|
|
def test_commit_and_reconcile_fail_closed_on_corrupt_or_symlinked_state(self) -> None:
|
|
result = self._install()
|
|
active_before = self._active_path().read_bytes()
|
|
marker_before = self._activation_path().read_bytes()
|
|
|
|
self._activation_path().write_bytes(b"not-json\n")
|
|
with self.assertRaisesRegex(install_runtime.InstallError, "activation state"):
|
|
self._commit(result.generation)
|
|
with self.assertRaisesRegex(install_runtime.InstallError, "activation state"):
|
|
install_runtime.reconcile_pending_activations(
|
|
ai_data_dir=self.fixture.ai_data_dir
|
|
)
|
|
self.assertEqual(self._active_path().read_bytes(), active_before)
|
|
|
|
outside = self.root / "outside-activation.json"
|
|
outside.write_bytes(marker_before)
|
|
self._activation_path().unlink()
|
|
self._activation_path().symlink_to(outside)
|
|
with self.assertRaisesRegex(install_runtime.InstallError, "activation state"):
|
|
self._commit(result.generation)
|
|
with self.assertRaisesRegex(install_runtime.InstallError, "activation state"):
|
|
install_runtime.reconcile_pending_activations(
|
|
ai_data_dir=self.fixture.ai_data_dir
|
|
)
|
|
self.assertEqual(self._active_path().read_bytes(), active_before)
|
|
self.assertEqual(outside.read_bytes(), marker_before)
|
|
|
|
def test_activation_state_rejects_non_descriptor_previous_bytes(self) -> None:
|
|
result = self._install()
|
|
marker = json.loads(self._activation_path().read_bytes())
|
|
marker["previousDescriptorB64"] = "bm90LWEtZGVzY3JpcHRvcg=="
|
|
marker["previousGeneration"] = None
|
|
marker["previousIndexPath"] = None
|
|
self._activation_path().write_bytes(install_runtime._canonical_json(marker))
|
|
|
|
with self.assertRaisesRegex(install_runtime.InstallError, "activation state"):
|
|
self._commit(result.generation)
|
|
with self.assertRaisesRegex(install_runtime.InstallError, "activation state"):
|
|
install_runtime.reconcile_pending_activations(
|
|
ai_data_dir=self.fixture.ai_data_dir
|
|
)
|
|
|
|
def test_commit_and_reconcile_reject_same_generation_descriptor_replacement(
|
|
self,
|
|
) -> None:
|
|
result = self._install()
|
|
marker_before = self._activation_path().read_bytes()
|
|
replacement = json.loads(self._active_path().read_bytes())
|
|
replacement["activatedAt"] = "2026-07-13T23:59:59.000Z"
|
|
self._active_path().write_bytes(install_runtime._canonical_json(replacement))
|
|
active_before = self._active_path().read_bytes()
|
|
|
|
with self.assertRaisesRegex(install_runtime.InstallError, "active runtime changed"):
|
|
self._commit(result.generation)
|
|
with self.assertRaisesRegex(install_runtime.InstallError, "active runtime changed"):
|
|
install_runtime.reconcile_pending_activations(
|
|
ai_data_dir=self.fixture.ai_data_dir
|
|
)
|
|
|
|
self.assertEqual(self._active_path().read_bytes(), active_before)
|
|
self.assertEqual(self._activation_path().read_bytes(), marker_before)
|
|
|
|
def test_mutable_state_is_shared_gid_writable_for_arbitrary_container_uids(
|
|
self,
|
|
) -> None:
|
|
self._install()
|
|
|
|
v3_root = self.fixture.ai_data_dir / "v3"
|
|
shared_directories = (
|
|
v3_root,
|
|
v3_root / "locks",
|
|
v3_root / "locks" / "generations",
|
|
v3_root / "locks" / "generations" / FAMILY,
|
|
v3_root / "staging",
|
|
v3_root / "runtimes",
|
|
v3_root / "runtimes" / FAMILY,
|
|
v3_root / "runtimes" / FAMILY / TARGET,
|
|
v3_root / "active",
|
|
)
|
|
for path in shared_directories:
|
|
with self.subTest(path=path):
|
|
self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o2770)
|
|
|
|
self.assertEqual(
|
|
stat.S_IMODE((v3_root / "locks" / "mutation.lock").stat().st_mode),
|
|
0o660,
|
|
)
|
|
self.assertEqual(
|
|
stat.S_IMODE(self._generation_lock_path().stat().st_mode),
|
|
0o660,
|
|
)
|
|
self.assertEqual(stat.S_IMODE(self._active_path().stat().st_mode), 0o660)
|
|
|
|
generation_root = v3_root / "runtimes" / FAMILY / TARGET / "1.0.0-abc123"
|
|
self.assertEqual(stat.S_IMODE(generation_root.stat().st_mode), 0o755)
|
|
|
|
def test_target_preflight_fails_before_creating_v3_state(self) -> None:
|
|
artifact = self.fixture.artifact()
|
|
artifact["target"] = "linux-arm64-cpu-py311"
|
|
artifact["arch"] = "arm64"
|
|
|
|
with self.assertRaisesRegex(install_runtime.PreflightError, "target"):
|
|
install_runtime.install_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
artifact=artifact,
|
|
authenticated_index=self.fixture.authenticated_index(artifact),
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
)
|
|
|
|
self.assertFalse((self.fixture.ai_data_dir / "v3").exists())
|
|
|
|
def test_memory_preflight_is_signed_and_runs_before_creating_v3_state(self) -> None:
|
|
artifact = self.fixture.artifact()
|
|
del artifact["resources"]
|
|
with self.assertRaisesRegex(install_runtime.PreflightError, "memory"):
|
|
install_runtime.install_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
artifact=artifact,
|
|
authenticated_index=self.fixture.authenticated_index(artifact),
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
effective_memory_bytes=8 * 1024 * 1024 * 1024,
|
|
)
|
|
self.assertFalse((self.fixture.ai_data_dir / "v3").exists())
|
|
|
|
artifact = self.fixture.artifact()
|
|
with self.assertRaisesRegex(
|
|
install_runtime.PreflightError,
|
|
r"4294967296 bytes required, 3221225472 available",
|
|
):
|
|
install_runtime.install_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
artifact=artifact,
|
|
authenticated_index=self.fixture.authenticated_index(artifact),
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
effective_memory_bytes=3 * 1024 * 1024 * 1024,
|
|
)
|
|
self.assertFalse((self.fixture.ai_data_dir / "v3").exists())
|
|
|
|
def test_effective_memory_resolves_private_and_host_cgroup_namespaces(self) -> None:
|
|
gib = 1024 * 1024 * 1024
|
|
|
|
def read_from(files):
|
|
def read_text(path, *args, **kwargs):
|
|
value = files.get(str(path))
|
|
if value is None:
|
|
raise FileNotFoundError(path)
|
|
return value
|
|
|
|
return read_text
|
|
|
|
private_files = {
|
|
"/proc/self/cgroup": "0::/\n",
|
|
"/proc/self/mountinfo": (
|
|
"29 23 0:26 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime "
|
|
"- cgroup2 cgroup rw\n"
|
|
),
|
|
"/sys/fs/cgroup/memory.max": str(6 * gib),
|
|
}
|
|
with mock.patch.object(os, "sysconf", side_effect=[8 * gib, 1]), mock.patch.object(
|
|
Path, "read_text", new=read_from(private_files)
|
|
):
|
|
self.assertEqual(install_runtime._effective_memory_bytes(), 6 * gib)
|
|
|
|
host_files = {
|
|
"/proc/self/cgroup": "0::/system.slice/docker-deadbeef.scope\n",
|
|
"/proc/self/mountinfo": (
|
|
"29 23 0:26 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime "
|
|
"- cgroup2 cgroup rw\n"
|
|
),
|
|
"/sys/fs/cgroup/system.slice/docker-deadbeef.scope/memory.max": str(
|
|
6 * gib
|
|
),
|
|
"/sys/fs/cgroup/system.slice/memory.max": str(5 * gib),
|
|
"/sys/fs/cgroup/memory.max": "max\n",
|
|
}
|
|
with mock.patch.object(os, "sysconf", side_effect=[8 * gib, 1]), mock.patch.object(
|
|
Path, "read_text", new=read_from(host_files)
|
|
):
|
|
self.assertEqual(install_runtime._effective_memory_bytes(), 5 * gib)
|
|
|
|
def test_effective_memory_fails_closed_for_unreadable_identified_controller(
|
|
self,
|
|
) -> None:
|
|
gib = 1024 * 1024 * 1024
|
|
files = {
|
|
"/proc/self/cgroup": "0::/docker/deadbeef\n",
|
|
"/proc/self/mountinfo": (
|
|
"29 23 0:26 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime "
|
|
"- cgroup2 cgroup rw\n"
|
|
),
|
|
}
|
|
|
|
def read_text(path, *args, **kwargs):
|
|
value = files.get(str(path))
|
|
if value is None:
|
|
raise PermissionError(path)
|
|
return value
|
|
|
|
with mock.patch.object(os, "sysconf", side_effect=[8 * gib, 1]), mock.patch.object(
|
|
Path, "read_text", new=read_text
|
|
):
|
|
with self.assertRaisesRegex(install_runtime.PreflightError, "cgroup memory"):
|
|
install_runtime._effective_memory_bytes()
|
|
|
|
def test_effective_memory_fails_closed_when_mount_metadata_is_unreadable(
|
|
self,
|
|
) -> None:
|
|
gib = 1024 * 1024 * 1024
|
|
|
|
def read_text(path, *args, **kwargs):
|
|
if str(path) == "/proc/self/cgroup":
|
|
return "0::/docker/deadbeef\n"
|
|
raise PermissionError(path)
|
|
|
|
with mock.patch.object(os, "sysconf", side_effect=[8 * gib, 1]), mock.patch.object(
|
|
Path, "read_text", new=read_text
|
|
):
|
|
with self.assertRaisesRegex(install_runtime.PreflightError, "cgroup memory"):
|
|
install_runtime._effective_memory_bytes()
|
|
|
|
def test_effective_memory_fails_closed_when_linux_membership_is_unavailable(
|
|
self,
|
|
) -> None:
|
|
gib = 1024 * 1024 * 1024
|
|
for membership in (PermissionError("denied"), "malformed-membership\n"):
|
|
def read_text(path, *args, **kwargs):
|
|
if isinstance(membership, BaseException):
|
|
raise membership
|
|
return membership
|
|
|
|
with self.subTest(membership=repr(membership)), mock.patch.object(
|
|
sys, "platform", "linux"
|
|
), mock.patch.object(os, "sysconf", side_effect=[8 * gib, 1]), mock.patch.object(
|
|
Path, "read_text", new=read_text
|
|
):
|
|
with self.assertRaisesRegex(
|
|
install_runtime.PreflightError, "cgroup memory"
|
|
):
|
|
install_runtime._effective_memory_bytes()
|
|
|
|
def test_model_digests_must_bind_to_files_in_the_exact_manifest(self) -> None:
|
|
artifact = self.fixture.artifact()
|
|
artifact["models"]["pp-ocrv6-small"] = "f" * 64
|
|
|
|
with self.assertRaisesRegex(install_runtime.PreflightError, "model digest"):
|
|
install_runtime.install_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
artifact=artifact,
|
|
authenticated_index=self.fixture.authenticated_index(artifact),
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
)
|
|
|
|
self.assertFalse((self.fixture.ai_data_dir / "v3").exists())
|
|
|
|
def test_symlinked_archive_and_v3_root_are_rejected(self) -> None:
|
|
archive_link = self.root / "runtime-link.tar.gz"
|
|
archive_link.symlink_to(self.fixture.archive)
|
|
artifact = self.fixture.artifact()
|
|
with self.assertRaisesRegex(install_runtime.PreflightError, "archive.*symlink"):
|
|
install_runtime.install_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
artifact=artifact,
|
|
authenticated_index=self.fixture.authenticated_index(artifact),
|
|
archive_path=archive_link,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
)
|
|
|
|
external = self.root / "external-v3"
|
|
external.mkdir()
|
|
self.fixture.ai_data_dir.mkdir()
|
|
(self.fixture.ai_data_dir / "v3").symlink_to(external, target_is_directory=True)
|
|
with self.assertRaisesRegex(install_runtime.InstallError, "v3 root.*symlink"):
|
|
self._install()
|
|
|
|
def test_archive_with_multiple_hard_links_is_rejected_before_state_creation(
|
|
self,
|
|
) -> None:
|
|
archive_alias = self.root / "runtime-hardlink.tar.gz"
|
|
os.link(self.fixture.archive, archive_alias)
|
|
artifact = self.fixture.artifact()
|
|
|
|
with self.assertRaisesRegex(
|
|
install_runtime.PreflightError, "exactly one hard link"
|
|
):
|
|
install_runtime.install_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
artifact=artifact,
|
|
authenticated_index=self.fixture.authenticated_index(artifact),
|
|
archive_path=archive_alias,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
)
|
|
|
|
self.assertFalse((self.fixture.ai_data_dir / "v3").exists())
|
|
|
|
@unittest.skipUnless(hasattr(os, "mkfifo"), "FIFOs are POSIX-only")
|
|
def test_archive_fifo_is_rejected_without_waiting_for_a_writer(self) -> None:
|
|
fifo = self.root / "runtime.fifo"
|
|
os.mkfifo(fifo)
|
|
artifact = self.fixture.artifact()
|
|
outcome: dict[str, object] = {}
|
|
|
|
def try_install() -> None:
|
|
try:
|
|
install_runtime.install_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
artifact=artifact,
|
|
authenticated_index=self.fixture.authenticated_index(artifact),
|
|
archive_path=fifo,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
)
|
|
except BaseException as error:
|
|
outcome["error"] = error
|
|
|
|
worker = threading.Thread(target=try_install, daemon=True)
|
|
worker.start()
|
|
worker.join(timeout=2)
|
|
|
|
self.assertFalse(worker.is_alive(), "opening an archive FIFO blocked")
|
|
self.assertIsInstance(outcome.get("error"), install_runtime.PreflightError)
|
|
self.assertFalse((self.fixture.ai_data_dir / "v3").exists())
|
|
|
|
def test_archive_path_replacement_after_verification_cannot_change_extraction(
|
|
self,
|
|
) -> None:
|
|
original_runner = self.fixture.files["ocr_runner.py"][0]
|
|
replacement = self.root / "replacement.tar.gz"
|
|
replacement_files = dict(self.fixture.files)
|
|
replacement_files["ocr_runner.py"] = (
|
|
b"!" + original_runner[1:],
|
|
self.fixture.files["ocr_runner.py"][1],
|
|
)
|
|
with tarfile.open(replacement, "w:gz") as archive:
|
|
for name, (data, mode) in replacement_files.items():
|
|
info = tarfile.TarInfo(name)
|
|
info.size = len(data)
|
|
info.mode = mode
|
|
archive.addfile(info, io.BytesIO(data))
|
|
|
|
verified_archive = self.root / "verified-runtime.tar.gz"
|
|
|
|
def replace_archive_path() -> None:
|
|
self.fixture.archive.rename(verified_archive)
|
|
replacement.rename(self.fixture.archive)
|
|
|
|
result = self._install_with_post_verification_action(replace_archive_path)
|
|
|
|
self.assertEqual(
|
|
(result.generation_root / "ocr_runner.py").read_bytes(), original_runner
|
|
)
|
|
|
|
def test_in_place_archive_mutation_after_verification_is_rejected_and_cleaned(
|
|
self,
|
|
) -> None:
|
|
self.fixture.archive = self.root / "runtime.tar"
|
|
self.fixture._write_archive(mode="w")
|
|
with tarfile.open(self.fixture.archive, "r:") as archive:
|
|
runner_offset = archive.getmember("ocr_runner.py").offset_data
|
|
original_runner = self.fixture.files["ocr_runner.py"][0]
|
|
tampered_runner = b"!" + original_runner[1:]
|
|
previous = self._write_previous_active()
|
|
|
|
def mutate_open_archive() -> None:
|
|
fd = os.open(self.fixture.archive, os.O_WRONLY)
|
|
try:
|
|
self.assertEqual(
|
|
os.pwrite(fd, tampered_runner, runner_offset),
|
|
len(tampered_runner),
|
|
)
|
|
os.fsync(fd)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
with self.assertRaisesRegex(
|
|
install_runtime.IntegrityError, "archive changed while being installed"
|
|
):
|
|
self._install_with_post_verification_action(mutate_open_archive)
|
|
|
|
self.assertEqual(self._active_path().read_bytes(), previous)
|
|
staging = self.fixture.ai_data_dir / "v3" / "staging"
|
|
self.assertFalse(staging.exists() and any(staging.iterdir()))
|
|
|
|
def test_archive_descriptor_is_closed_when_verification_fails(self) -> None:
|
|
artifact = self.fixture.artifact()
|
|
artifact["archive"]["sha256"] = "0" * 64
|
|
real_open = os.open
|
|
archive_fds: list[int] = []
|
|
|
|
def tracking_open(path, flags, mode=0o777, *, dir_fd=None):
|
|
if dir_fd is None:
|
|
fd = real_open(path, flags, mode)
|
|
else:
|
|
fd = real_open(path, flags, mode, dir_fd=dir_fd)
|
|
if Path(path) == self.fixture.archive:
|
|
archive_fds.append(fd)
|
|
return fd
|
|
|
|
with mock.patch.object(install_runtime.os, "open", side_effect=tracking_open):
|
|
with self.assertRaisesRegex(install_runtime.IntegrityError, "digest"):
|
|
install_runtime.install_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
artifact=artifact,
|
|
authenticated_index=self.fixture.authenticated_index(artifact),
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
)
|
|
|
|
self.assertEqual(len(archive_fds), 1)
|
|
with self.assertRaises(OSError):
|
|
os.fstat(archive_fds[0])
|
|
|
|
def test_archive_descriptors_are_closed_when_tar_parsing_fails(self) -> None:
|
|
self.fixture.archive.write_bytes(b"this is not a tar archive")
|
|
artifact = self.fixture.artifact()
|
|
real_open = os.open
|
|
real_dup = os.dup
|
|
archive_fds: list[int] = []
|
|
|
|
def tracking_open(path, flags, mode=0o777, *, dir_fd=None):
|
|
if dir_fd is None:
|
|
fd = real_open(path, flags, mode)
|
|
else:
|
|
fd = real_open(path, flags, mode, dir_fd=dir_fd)
|
|
if Path(path) == self.fixture.archive:
|
|
archive_fds.append(fd)
|
|
return fd
|
|
|
|
def tracking_dup(fd):
|
|
duplicate = real_dup(fd)
|
|
if archive_fds and fd == archive_fds[0]:
|
|
archive_fds.append(duplicate)
|
|
return duplicate
|
|
|
|
with (
|
|
mock.patch.object(install_runtime.os, "open", side_effect=tracking_open),
|
|
mock.patch.object(install_runtime.os, "dup", side_effect=tracking_dup),
|
|
):
|
|
with self.assertRaises(install_runtime.UnsafeArchiveError):
|
|
install_runtime.install_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
artifact=artifact,
|
|
authenticated_index=self.fixture.authenticated_index(artifact),
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
)
|
|
|
|
self.assertEqual(len(archive_fds), 2)
|
|
for fd in archive_fds:
|
|
with self.subTest(fd=fd), self.assertRaises(OSError):
|
|
os.fstat(fd)
|
|
|
|
def test_disk_preflight_happens_before_staging(self) -> None:
|
|
usage = shutil.disk_usage(self.root)
|
|
tiny_usage = usage._replace(free=0)
|
|
|
|
with mock.patch.object(
|
|
install_runtime.shutil, "disk_usage", return_value=tiny_usage
|
|
):
|
|
with self.assertRaisesRegex(install_runtime.PreflightError, "disk space"):
|
|
self._install()
|
|
|
|
staging = self.fixture.ai_data_dir / "v3" / "staging"
|
|
self.assertFalse(staging.exists() and any(staging.iterdir()))
|
|
|
|
def test_existing_verified_generation_reactivates_with_descriptor_only_space(
|
|
self,
|
|
) -> None:
|
|
installed = self._install()
|
|
self._commit(installed.generation)
|
|
self._active_path().unlink()
|
|
usage = shutil.disk_usage(self.root)
|
|
descriptor_only = usage._replace(free=install_runtime.DESCRIPTOR_RESERVE_BYTES)
|
|
|
|
with mock.patch.object(
|
|
install_runtime.shutil, "disk_usage", return_value=descriptor_only
|
|
):
|
|
result = self._install()
|
|
|
|
self.assertTrue(result.activated)
|
|
self.assertTrue(self._active_path().is_file())
|
|
|
|
def test_retry_repairs_a_corrupted_preserved_authenticated_index(self) -> None:
|
|
self._install()
|
|
expected = self.fixture.authenticated_index()
|
|
descriptor = json.loads(self._active_path().read_bytes())
|
|
index_path = (
|
|
self.fixture.ai_data_dir
|
|
/ "v3"
|
|
/ descriptor["artifact"]["signedIndex"]["path"]
|
|
)
|
|
index_path.write_bytes(b"x" * len(expected))
|
|
|
|
result = self._install()
|
|
|
|
self.assertFalse(result.activated)
|
|
self.assertEqual(index_path.read_bytes(), expected)
|
|
|
|
def test_old_unreferenced_generation_is_collected_before_space_preflight(
|
|
self,
|
|
) -> None:
|
|
orphan = (
|
|
self.fixture.ai_data_dir / "v3" / "runtimes" / FAMILY / TARGET / "0.9.0-old"
|
|
)
|
|
orphan.mkdir(parents=True)
|
|
(orphan / "large.bin").write_bytes(b"old")
|
|
usage = shutil.disk_usage(self.root)
|
|
|
|
def disk_usage_after_gc(_path):
|
|
self.assertFalse(orphan.exists())
|
|
return usage
|
|
|
|
with mock.patch.object(
|
|
install_runtime.shutil, "disk_usage", side_effect=disk_usage_after_gc
|
|
):
|
|
self._install()
|
|
|
|
def test_archive_digest_size_and_exact_file_manifest_are_enforced(self) -> None:
|
|
cases = []
|
|
wrong_digest = self.fixture.artifact()
|
|
wrong_digest["archive"]["sha256"] = "0" * 64
|
|
cases.append(("archive digest", wrong_digest))
|
|
wrong_size = self.fixture.artifact()
|
|
wrong_size["archive"]["size"] += 1
|
|
cases.append(("archive size", wrong_size))
|
|
wrong_file_digest = self.fixture.artifact()
|
|
wrong_file_digest["files"][0]["sha256"] = "0" * 64
|
|
cases.append(("file digest", wrong_file_digest))
|
|
wrong_file_mode = self.fixture.artifact()
|
|
wrong_file_mode["files"][0]["mode"] = 0o600
|
|
cases.append(("file mode", wrong_file_mode))
|
|
missing_file = self.fixture.artifact()
|
|
missing_file["files"] = missing_file["files"][:-1]
|
|
cases.append(("extra file", missing_file))
|
|
|
|
for label, artifact in cases:
|
|
with self.subTest(label=label):
|
|
previous = self._write_previous_active()
|
|
with self.assertRaises(install_runtime.InstallError):
|
|
install_runtime.install_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
artifact=artifact,
|
|
authenticated_index=self.fixture.authenticated_index(artifact),
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
)
|
|
self.assertEqual(self._active_path().read_bytes(), previous)
|
|
|
|
def test_unsafe_tar_entries_are_rejected_without_changing_active(self) -> None:
|
|
unsafe_entries = []
|
|
for name in ("/absolute", "../escape", "a/../../escape"):
|
|
info = tarfile.TarInfo(name)
|
|
info.size = 1
|
|
info.mode = 0o644
|
|
unsafe_entries.append((name, info, b"x"))
|
|
|
|
link = tarfile.TarInfo("unsafe-link")
|
|
link.type = tarfile.SYMTYPE
|
|
link.linkname = "ocr_runner.py"
|
|
unsafe_entries.append(("symlink", link, None))
|
|
hardlink = tarfile.TarInfo("unsafe-hardlink")
|
|
hardlink.type = tarfile.LNKTYPE
|
|
hardlink.linkname = "ocr_runner.py"
|
|
unsafe_entries.append(("hardlink", hardlink, None))
|
|
device = tarfile.TarInfo("unsafe-device")
|
|
device.type = tarfile.CHRTYPE
|
|
unsafe_entries.append(("device", device, None))
|
|
setuid = tarfile.TarInfo("unsafe-setuid")
|
|
setuid.size = 1
|
|
setuid.mode = 0o4755
|
|
unsafe_entries.append(("setuid", setuid, b"x"))
|
|
|
|
previous = self._write_previous_active()
|
|
for label, info, data in unsafe_entries:
|
|
with self.subTest(label=label):
|
|
self.fixture._write_archive([(info, data)])
|
|
artifact = self.fixture.artifact()
|
|
# Keep the normal file manifest but acknowledge the new archive bytes.
|
|
artifact["archive"]["size"] = self.fixture.archive.stat().st_size
|
|
artifact["archive"]["sha256"] = _sha256(
|
|
self.fixture.archive.read_bytes()
|
|
)
|
|
artifact["archive"]["expandedSize"] += info.size
|
|
with self.assertRaises(install_runtime.UnsafeArchiveError):
|
|
install_runtime.install_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
artifact=artifact,
|
|
authenticated_index=self.fixture.authenticated_index(artifact),
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
)
|
|
self.assertEqual(self._active_path().read_bytes(), previous)
|
|
|
|
duplicate = tarfile.TarInfo("ocr_runner.py")
|
|
duplicate.size = 1
|
|
duplicate.mode = 0o644
|
|
self.fixture._write_archive([(duplicate, b"x")])
|
|
artifact = self.fixture.artifact()
|
|
artifact["archive"]["size"] = self.fixture.archive.stat().st_size
|
|
artifact["archive"]["sha256"] = _sha256(self.fixture.archive.read_bytes())
|
|
artifact["archive"]["expandedSize"] += 1
|
|
with self.assertRaises(install_runtime.UnsafeArchiveError):
|
|
install_runtime.install_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
artifact=artifact,
|
|
authenticated_index=self.fixture.authenticated_index(artifact),
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
)
|
|
|
|
def test_entry_and_expanded_byte_ceilings_are_enforced(self) -> None:
|
|
previous = self._write_previous_active()
|
|
with self.assertRaisesRegex(install_runtime.UnsafeArchiveError, "entries"):
|
|
self._install(
|
|
limits=install_runtime.ExtractionLimits(
|
|
max_entries=2, max_bytes=100_000
|
|
)
|
|
)
|
|
self.assertEqual(self._active_path().read_bytes(), previous)
|
|
|
|
with self.assertRaisesRegex(install_runtime.UnsafeArchiveError, "expanded"):
|
|
self._install(
|
|
limits=install_runtime.ExtractionLimits(max_entries=100, max_bytes=2)
|
|
)
|
|
self.assertEqual(self._active_path().read_bytes(), previous)
|
|
|
|
def test_final_smoke_failure_keeps_active_and_retry_reuses_verified_generation(
|
|
self,
|
|
) -> None:
|
|
previous = self._write_previous_active()
|
|
with self.assertRaisesRegex(install_runtime.SmokeError, "final"):
|
|
self._install(smoke_env={"FAIL_FINAL": "1"})
|
|
self.assertEqual(self._active_path().read_bytes(), previous)
|
|
|
|
final_root = (
|
|
self.fixture.ai_data_dir
|
|
/ "v3"
|
|
/ "runtimes"
|
|
/ FAMILY
|
|
/ TARGET
|
|
/ "1.0.0-abc123"
|
|
)
|
|
self.assertTrue(final_root.is_dir())
|
|
|
|
result = self._install()
|
|
self.assertEqual(result.generation_root, final_root.resolve())
|
|
self.assertEqual(
|
|
json.loads(self._active_path().read_bytes())["generation"], "1.0.0-abc123"
|
|
)
|
|
self._commit(result.generation)
|
|
|
|
descriptor_bytes = self._active_path().read_bytes()
|
|
second = self._install()
|
|
self.assertEqual(second.generation_root, final_root.resolve())
|
|
self.assertEqual(self._active_path().read_bytes(), descriptor_bytes)
|
|
|
|
corrupted = json.loads(descriptor_bytes)
|
|
corrupted["runtime"]["entrypoint"] = "runtimes/ocr/elsewhere.py"
|
|
self._active_path().write_text(json.dumps(corrupted), encoding="utf-8")
|
|
corrupted_bytes = self._active_path().read_bytes()
|
|
corrupted_marker = self._activation_path().read_bytes()
|
|
|
|
repaired = self._install()
|
|
self.assertEqual(repaired.generation, "1.0.0-abc123")
|
|
self.assertNotEqual(self._active_path().read_bytes(), corrupted_bytes)
|
|
pending_marker = json.loads(self._activation_path().read_bytes())
|
|
self.assertEqual(pending_marker["status"], "pending")
|
|
self.assertEqual(
|
|
(
|
|
self.fixture.ai_data_dir
|
|
/ "v3"
|
|
/ pending_marker["invalidPreviousDescriptorFile"]["path"]
|
|
).read_bytes(),
|
|
corrupted_bytes,
|
|
)
|
|
self.assertEqual(
|
|
(
|
|
self.fixture.ai_data_dir
|
|
/ "v3"
|
|
/ pending_marker["previousMarkerFile"]["path"]
|
|
).read_bytes(),
|
|
corrupted_marker,
|
|
)
|
|
|
|
resolution = install_runtime.rollback_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
family=FAMILY,
|
|
expected_generation=repaired.generation,
|
|
)
|
|
self.assertFalse(resolution.committed)
|
|
self.assertEqual(self._active_path().read_bytes(), corrupted_bytes)
|
|
self.assertEqual(self._activation_path().read_bytes(), corrupted_marker)
|
|
|
|
committed = self._install()
|
|
self._commit(committed.generation)
|
|
committed_marker = json.loads(self._activation_path().read_bytes())
|
|
self.assertIsNone(committed_marker["invalidPreviousDescriptorB64"])
|
|
self.assertIsNone(committed_marker["previousMarkerB64"])
|
|
self.assertIsNone(committed_marker["invalidPreviousDescriptorFile"])
|
|
self.assertIsNone(committed_marker["previousMarkerFile"])
|
|
|
|
def test_retry_reextracts_a_corrupted_existing_generation(self) -> None:
|
|
installed = self._install()
|
|
self._commit(installed.generation)
|
|
final_root = (
|
|
self.fixture.ai_data_dir
|
|
/ "v3"
|
|
/ "runtimes"
|
|
/ FAMILY
|
|
/ TARGET
|
|
/ "1.0.0-abc123"
|
|
)
|
|
entrypoint = final_root / "ocr_runner.py"
|
|
expected = self.fixture.files["ocr_runner.py"][0]
|
|
entrypoint.write_bytes(b"x" * len(expected))
|
|
|
|
self._install()
|
|
|
|
self.assertEqual(entrypoint.read_bytes(), expected)
|
|
|
|
def test_repair_restores_corrupt_descriptor_during_startup_reconciliation(self) -> None:
|
|
installed = self._install()
|
|
self._commit(installed.generation)
|
|
corrupt = b'{"corrupt":"descriptor"}\n'
|
|
previous_marker = self._activation_path().read_bytes()
|
|
self._active_path().write_bytes(corrupt)
|
|
|
|
repaired = self._install()
|
|
restored = install_runtime.reconcile_pending_activations(
|
|
ai_data_dir=self.fixture.ai_data_dir
|
|
)
|
|
|
|
self.assertEqual(restored, {FAMILY: None})
|
|
self.assertEqual(self._active_path().read_bytes(), corrupt)
|
|
self.assertEqual(self._activation_path().read_bytes(), previous_marker)
|
|
self.assertTrue(repaired.generation_root.is_dir())
|
|
|
|
def test_repair_quarantines_and_restores_an_invalid_activation_marker(self) -> None:
|
|
installed = self._install()
|
|
self._commit(installed.generation)
|
|
previous_descriptor = self._active_path().read_bytes()
|
|
corrupt_marker = b'{"corrupt":"marker"}\n'
|
|
self._activation_path().write_bytes(corrupt_marker)
|
|
|
|
repaired = self._install()
|
|
pending = json.loads(self._activation_path().read_bytes())
|
|
self.assertEqual(
|
|
(
|
|
self.fixture.ai_data_dir
|
|
/ "v3"
|
|
/ pending["previousMarkerFile"]["path"]
|
|
).read_bytes(),
|
|
corrupt_marker,
|
|
)
|
|
resolution = install_runtime.rollback_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
family=FAMILY,
|
|
expected_generation=repaired.generation,
|
|
)
|
|
|
|
self.assertEqual(resolution.restored_generation, installed.generation)
|
|
self.assertEqual(self._active_path().read_bytes(), previous_descriptor)
|
|
self.assertEqual(self._activation_path().read_bytes(), corrupt_marker)
|
|
|
|
def test_repair_recreates_pending_state_for_missing_or_mismatched_commit_marker(
|
|
self,
|
|
) -> None:
|
|
installed = self._install()
|
|
self._commit(installed.generation)
|
|
descriptor = self._active_path().read_bytes()
|
|
committed_marker = self._activation_path().read_bytes()
|
|
parsed_marker = json.loads(committed_marker)
|
|
wrong_generation = {
|
|
**parsed_marker,
|
|
"activatedGeneration": "wrong-generation",
|
|
}
|
|
wrong_digest = {
|
|
**parsed_marker,
|
|
"activatedDescriptorSha256": "f" * 64,
|
|
}
|
|
|
|
for label, previous_marker in (
|
|
("missing", None),
|
|
("wrong-generation", install_runtime._canonical_json(wrong_generation)),
|
|
("wrong-digest", install_runtime._canonical_json(wrong_digest)),
|
|
):
|
|
with self.subTest(label=label):
|
|
self._active_path().write_bytes(descriptor)
|
|
if previous_marker is None:
|
|
self._activation_path().unlink(missing_ok=True)
|
|
else:
|
|
self._activation_path().write_bytes(previous_marker)
|
|
|
|
repaired = self._install()
|
|
self.assertTrue(repaired.activated)
|
|
pending = json.loads(self._activation_path().read_bytes())
|
|
self.assertEqual(pending["status"], "pending")
|
|
self.assertTrue(pending["repairPreviousState"])
|
|
|
|
install_runtime.rollback_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
family=FAMILY,
|
|
expected_generation=repaired.generation,
|
|
)
|
|
self.assertEqual(self._active_path().read_bytes(), descriptor)
|
|
if previous_marker is None:
|
|
self.assertFalse(self._activation_path().exists())
|
|
else:
|
|
self.assertEqual(
|
|
self._activation_path().read_bytes(), previous_marker
|
|
)
|
|
|
|
self._active_path().write_bytes(descriptor)
|
|
self._activation_path().write_bytes(committed_marker)
|
|
|
|
def test_repair_quarantines_large_corrupt_state_without_growing_the_marker(self) -> None:
|
|
installed = self._install()
|
|
self._commit(installed.generation)
|
|
corrupt_descriptor = b"{" + b"d" * (3 * 1024 * 1024)
|
|
corrupt_marker = b"{" + b"m" * (3 * 1024 * 1024)
|
|
self._active_path().write_bytes(corrupt_descriptor)
|
|
self._activation_path().write_bytes(corrupt_marker)
|
|
|
|
repaired = self._install()
|
|
|
|
self.assertLess(
|
|
self._activation_path().stat().st_size,
|
|
install_runtime.ACTIVATION_STATE_MAX_BYTES,
|
|
)
|
|
state = install_runtime._read_rollback_marker(
|
|
self._activation_path(), FAMILY
|
|
)
|
|
self.assertEqual(state.invalid_previous_descriptor, corrupt_descriptor)
|
|
self.assertEqual(state.previous_marker, corrupt_marker)
|
|
self.assertEqual(len(state.quarantine_paths), 2)
|
|
|
|
install_runtime.rollback_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
family=FAMILY,
|
|
expected_generation=repaired.generation,
|
|
)
|
|
self.assertEqual(self._active_path().read_bytes(), corrupt_descriptor)
|
|
self.assertEqual(self._activation_path().read_bytes(), corrupt_marker)
|
|
self.assertTrue(all(not path.exists() for path in state.quarantine_paths))
|
|
|
|
def test_repair_quarantines_a_large_structurally_valid_descriptor(self) -> None:
|
|
installed = self._install()
|
|
self._commit(installed.generation)
|
|
descriptor = json.loads(self._active_path().read_bytes())
|
|
descriptor["ignoredPadding"] = "d" * (install_runtime.DESCRIPTOR_RESERVE_BYTES + 1)
|
|
oversized_descriptor = install_runtime._canonical_json(descriptor)
|
|
self.assertGreater(
|
|
len(oversized_descriptor), install_runtime.DESCRIPTOR_RESERVE_BYTES
|
|
)
|
|
self._active_path().write_bytes(oversized_descriptor)
|
|
|
|
repaired = self._install()
|
|
|
|
state = install_runtime._read_rollback_marker(
|
|
self._activation_path(), FAMILY
|
|
)
|
|
self.assertTrue(state.repair_previous_state)
|
|
self.assertEqual(state.invalid_previous_descriptor, oversized_descriptor)
|
|
self.assertEqual(len(state.quarantine_paths), 2)
|
|
self.assertLess(
|
|
self._activation_path().stat().st_size,
|
|
install_runtime.ACTIVATION_STATE_MAX_BYTES,
|
|
)
|
|
|
|
install_runtime.rollback_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
family=FAMILY,
|
|
expected_generation=repaired.generation,
|
|
)
|
|
self.assertEqual(self._active_path().read_bytes(), oversized_descriptor)
|
|
self.assertTrue(all(not path.exists() for path in state.quarantine_paths))
|
|
|
|
def test_repair_retains_referenced_quarantine_when_marker_restore_fails(self) -> None:
|
|
installed = self._install()
|
|
self._commit(installed.generation)
|
|
corrupt_descriptor = b'{"corrupt":"descriptor"}\n'
|
|
self._active_path().write_bytes(corrupt_descriptor)
|
|
original_activate = install_runtime._activate_descriptor
|
|
original_restore = install_runtime._restore_optional_state_file
|
|
|
|
def fail_active(path, raw):
|
|
if path.resolve(strict=False) == self._active_path().resolve(strict=False):
|
|
raise OSError("simulated active descriptor failure")
|
|
return original_activate(path, raw)
|
|
|
|
def fail_marker_restore(path, raw):
|
|
if path.resolve(strict=False) == self._activation_path().resolve(
|
|
strict=False
|
|
):
|
|
raise OSError("simulated marker restore failure")
|
|
return original_restore(path, raw)
|
|
|
|
with mock.patch.object(
|
|
install_runtime, "_activate_descriptor", side_effect=fail_active
|
|
), mock.patch.object(
|
|
install_runtime,
|
|
"_restore_optional_state_file",
|
|
side_effect=fail_marker_restore,
|
|
):
|
|
with self.assertRaisesRegex(
|
|
install_runtime.InstallError, "quarantine must be retained"
|
|
):
|
|
self._install()
|
|
|
|
state = install_runtime._read_rollback_marker(
|
|
self._activation_path(), FAMILY
|
|
)
|
|
self.assertTrue(all(path.is_file() for path in state.quarantine_paths))
|
|
install_runtime.reconcile_pending_activations(
|
|
ai_data_dir=self.fixture.ai_data_dir
|
|
)
|
|
self.assertEqual(self._active_path().read_bytes(), corrupt_descriptor)
|
|
self.assertTrue(all(not path.exists() for path in state.quarantine_paths))
|
|
|
|
def test_late_active_restore_failure_retains_pending_repair_state(self) -> None:
|
|
installed = self._install()
|
|
self._commit(installed.generation)
|
|
corrupt_descriptor = b'{"corrupt":"descriptor"}\n'
|
|
self._active_path().write_bytes(corrupt_descriptor)
|
|
active_path = self._active_path().resolve(strict=False)
|
|
original_atomic_write = install_runtime._atomic_write
|
|
active_write_count = 0
|
|
|
|
def fail_after_active_replace(path, raw):
|
|
nonlocal active_write_count
|
|
if path.resolve(strict=False) != active_path:
|
|
return original_atomic_write(path, raw)
|
|
active_write_count += 1
|
|
if active_write_count == 1:
|
|
original_atomic_write(path, raw)
|
|
raise OSError("simulated late descriptor fsync failure")
|
|
raise OSError("simulated active descriptor restore failure")
|
|
|
|
with mock.patch.object(
|
|
install_runtime, "_atomic_write", side_effect=fail_after_active_replace
|
|
):
|
|
with self.assertRaisesRegex(
|
|
install_runtime.InstallError, "repair state was retained"
|
|
):
|
|
self._install()
|
|
|
|
state = install_runtime._read_rollback_marker(
|
|
self._activation_path(), FAMILY
|
|
)
|
|
self.assertEqual(state.status, "pending")
|
|
self.assertTrue(state.repair_previous_state)
|
|
self.assertEqual(state.invalid_previous_descriptor, corrupt_descriptor)
|
|
self.assertTrue(all(path.is_file() for path in state.quarantine_paths))
|
|
self.assertNotEqual(self._active_path().read_bytes(), corrupt_descriptor)
|
|
|
|
install_runtime.reconcile_pending_activations(
|
|
ai_data_dir=self.fixture.ai_data_dir
|
|
)
|
|
self.assertEqual(self._active_path().read_bytes(), corrupt_descriptor)
|
|
self.assertTrue(all(not path.exists() for path in state.quarantine_paths))
|
|
|
|
def test_restore_fsync_failure_retains_pending_repair_state(self) -> None:
|
|
installed = self._install()
|
|
self._commit(installed.generation)
|
|
corrupt_descriptor = b'{"corrupt":"descriptor"}\n'
|
|
self._active_path().write_bytes(corrupt_descriptor)
|
|
active_parent = self._active_path().resolve(strict=False).parent
|
|
original_fsync_directory = install_runtime._fsync_directory
|
|
active_fsync_count = 0
|
|
|
|
def fail_active_directory_fsync(path):
|
|
nonlocal active_fsync_count
|
|
if path.resolve(strict=False) != active_parent:
|
|
return original_fsync_directory(path)
|
|
active_fsync_count += 1
|
|
raise OSError("simulated active directory fsync failure")
|
|
|
|
with mock.patch.object(
|
|
install_runtime,
|
|
"_fsync_directory",
|
|
side_effect=fail_active_directory_fsync,
|
|
):
|
|
with self.assertRaisesRegex(
|
|
install_runtime.InstallError, "durability.*repair state was retained"
|
|
):
|
|
self._install()
|
|
|
|
self.assertEqual(active_fsync_count, 3)
|
|
state = install_runtime._read_rollback_marker(
|
|
self._activation_path(), FAMILY
|
|
)
|
|
self.assertEqual(state.status, "pending")
|
|
self.assertTrue(state.repair_previous_state)
|
|
self.assertEqual(state.invalid_previous_descriptor, corrupt_descriptor)
|
|
self.assertEqual(self._active_path().read_bytes(), corrupt_descriptor)
|
|
self.assertTrue(all(path.is_file() for path in state.quarantine_paths))
|
|
|
|
def fail_reconciliation_active_fsync(path):
|
|
if path.resolve(strict=False) == active_parent:
|
|
raise OSError("simulated reconciliation fsync failure")
|
|
return original_fsync_directory(path)
|
|
|
|
with mock.patch.object(
|
|
install_runtime,
|
|
"_fsync_directory",
|
|
side_effect=fail_reconciliation_active_fsync,
|
|
):
|
|
with self.assertRaisesRegex(
|
|
install_runtime.InstallError, "durability.*repair state was retained"
|
|
):
|
|
install_runtime.reconcile_pending_activations(
|
|
ai_data_dir=self.fixture.ai_data_dir
|
|
)
|
|
retained_state = install_runtime._read_rollback_marker(
|
|
self._activation_path(), FAMILY
|
|
)
|
|
self.assertEqual(retained_state.status, "pending")
|
|
self.assertTrue(
|
|
all(path.is_file() for path in retained_state.quarantine_paths)
|
|
)
|
|
|
|
install_runtime.reconcile_pending_activations(
|
|
ai_data_dir=self.fixture.ai_data_dir
|
|
)
|
|
self.assertEqual(self._active_path().read_bytes(), corrupt_descriptor)
|
|
self.assertTrue(all(not path.exists() for path in state.quarantine_paths))
|
|
|
|
def test_pending_marker_fsync_failure_retains_referenced_quarantine(self) -> None:
|
|
installed = self._install()
|
|
self._commit(installed.generation)
|
|
corrupt_descriptor = b'{"corrupt":"descriptor"}\n'
|
|
self._active_path().write_bytes(corrupt_descriptor)
|
|
rollback_parent = self._activation_path().resolve(strict=False).parent
|
|
original_fsync_directory = install_runtime._fsync_directory
|
|
|
|
def fail_rollback_directory_fsync(path):
|
|
if path.resolve(strict=False) == rollback_parent:
|
|
raise OSError("simulated pending marker fsync failure")
|
|
return original_fsync_directory(path)
|
|
|
|
with mock.patch.object(
|
|
install_runtime,
|
|
"_fsync_directory",
|
|
side_effect=fail_rollback_directory_fsync,
|
|
):
|
|
with self.assertRaisesRegex(
|
|
install_runtime.InstallError, "activation marker.*repair state was retained"
|
|
):
|
|
self._install()
|
|
|
|
state = install_runtime._read_rollback_marker(
|
|
self._activation_path(), FAMILY
|
|
)
|
|
self.assertEqual(state.status, "pending")
|
|
self.assertTrue(state.repair_previous_state)
|
|
self.assertEqual(state.invalid_previous_descriptor, corrupt_descriptor)
|
|
self.assertEqual(self._active_path().read_bytes(), corrupt_descriptor)
|
|
self.assertTrue(all(path.is_file() for path in state.quarantine_paths))
|
|
|
|
install_runtime.reconcile_pending_activations(
|
|
ai_data_dir=self.fixture.ai_data_dir
|
|
)
|
|
self.assertEqual(self._active_path().read_bytes(), corrupt_descriptor)
|
|
self.assertTrue(all(not path.exists() for path in state.quarantine_paths))
|
|
|
|
def test_gc_removes_unreferenced_quarantine_from_a_pre_marker_crash(self) -> None:
|
|
v3_root = self.fixture.ai_data_dir / "v3"
|
|
with install_runtime.mutation_lock(v3_root):
|
|
_reference, orphan = install_runtime._quarantine_state_bytes(
|
|
v3_root, FAMILY, "orphan", b"crash-window"
|
|
)
|
|
self.assertTrue(orphan.is_file())
|
|
removed = install_runtime.recover_and_gc(v3_root, keep_unreferenced=0)
|
|
|
|
self.assertIn(orphan, removed)
|
|
self.assertFalse(orphan.exists())
|
|
|
|
def test_repair_preflight_failure_keeps_the_active_descriptor_and_generation(
|
|
self,
|
|
) -> None:
|
|
installed = self._install()
|
|
self._commit(installed.generation)
|
|
active_before = self._active_path().read_bytes()
|
|
entrypoint = installed.generation_root / "ocr_runner.py"
|
|
corrupted = b"x" * entrypoint.stat().st_size
|
|
entrypoint.write_bytes(corrupted)
|
|
usage = shutil.disk_usage(self.root)._replace(free=0)
|
|
|
|
with mock.patch.object(install_runtime.shutil, "disk_usage", return_value=usage):
|
|
with self.assertRaisesRegex(install_runtime.PreflightError, "disk space"):
|
|
self._install()
|
|
|
|
self.assertEqual(self._active_path().read_bytes(), active_before)
|
|
self.assertEqual(entrypoint.read_bytes(), corrupted)
|
|
|
|
def test_repair_smoke_failure_restores_the_prior_descriptor_and_bytes(self) -> None:
|
|
installed = self._install()
|
|
self._commit(installed.generation)
|
|
active_before = self._active_path().read_bytes()
|
|
entrypoint = installed.generation_root / "ocr_runner.py"
|
|
corrupted = b"x" * entrypoint.stat().st_size
|
|
entrypoint.write_bytes(corrupted)
|
|
|
|
with self.assertRaisesRegex(install_runtime.SmokeError, "final"):
|
|
self._install(smoke_env={"FAIL_FINAL": "1"})
|
|
|
|
self.assertEqual(self._active_path().read_bytes(), active_before)
|
|
self.assertEqual(entrypoint.read_bytes(), corrupted)
|
|
|
|
def test_repair_activation_failure_restores_the_prior_descriptor_and_bytes(
|
|
self,
|
|
) -> None:
|
|
installed = self._install()
|
|
self._commit(installed.generation)
|
|
active_before = self._active_path().read_bytes()
|
|
entrypoint = installed.generation_root / "ocr_runner.py"
|
|
corrupted = b"x" * entrypoint.stat().st_size
|
|
entrypoint.write_bytes(corrupted)
|
|
|
|
with mock.patch.object(
|
|
install_runtime,
|
|
"_activate_with_rollback",
|
|
side_effect=install_runtime.InstallError("injected activation failure"),
|
|
):
|
|
with self.assertRaisesRegex(install_runtime.InstallError, "injected"):
|
|
self._install()
|
|
|
|
self.assertEqual(self._active_path().read_bytes(), active_before)
|
|
self.assertEqual(entrypoint.read_bytes(), corrupted)
|
|
|
|
def test_successful_upgrades_retain_exactly_one_rollback_generation(self) -> None:
|
|
for generation in ("1.0.0-first", "1.0.0-second", "1.0.0-third"):
|
|
artifact = self.fixture.artifact()
|
|
artifact["generation"] = generation
|
|
result = install_runtime.install_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
artifact=artifact,
|
|
authenticated_index=self.fixture.authenticated_index(artifact),
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
)
|
|
self._commit(result.generation)
|
|
|
|
target_root = self.fixture.ai_data_dir / "v3" / "runtimes" / FAMILY / TARGET
|
|
self.assertEqual(
|
|
sorted(path.name for path in target_root.iterdir()),
|
|
["1.0.0-second", "1.0.0-third"],
|
|
)
|
|
self.assertEqual(
|
|
json.loads(self._active_path().read_bytes())["generation"],
|
|
"1.0.0-third",
|
|
)
|
|
indexes_root = self.fixture.ai_data_dir / "v3" / "indexes"
|
|
self.assertEqual(len(list(indexes_root.iterdir())), 2)
|
|
|
|
def test_rollback_restores_previous_descriptor_and_collects_failed_handoff(self) -> None:
|
|
first = self.fixture.artifact()
|
|
first["generation"] = "1.0.0-first"
|
|
first_result = install_runtime.install_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
artifact=first,
|
|
authenticated_index=self.fixture.authenticated_index(first),
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
)
|
|
self._commit(first_result.generation)
|
|
previous = self._active_path().read_bytes()
|
|
|
|
second = self.fixture.artifact()
|
|
second["generation"] = "1.0.0-second"
|
|
install_runtime.install_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
artifact=second,
|
|
authenticated_index=self.fixture.authenticated_index(second),
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
)
|
|
|
|
resolution = install_runtime.rollback_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
family=FAMILY,
|
|
expected_generation="1.0.0-second",
|
|
)
|
|
|
|
self.assertFalse(resolution.committed)
|
|
self.assertIsNone(resolution.committed_generation)
|
|
self.assertEqual(resolution.restored_generation, "1.0.0-first")
|
|
self.assertEqual(self._active_path().read_bytes(), previous)
|
|
target_root = self.fixture.ai_data_dir / "v3" / "runtimes" / FAMILY / TARGET
|
|
self.assertEqual(
|
|
sorted(path.name for path in target_root.iterdir()),
|
|
["1.0.0-first"],
|
|
)
|
|
marker = json.loads(self._activation_path().read_bytes())
|
|
self.assertEqual(marker["status"], "committed")
|
|
self.assertEqual(marker["activatedGeneration"], "1.0.0-first")
|
|
|
|
def test_rollback_resolves_an_exact_already_committed_generation_as_success(
|
|
self,
|
|
) -> None:
|
|
result = self._install()
|
|
self._commit(result.generation)
|
|
active = self._active_path().read_bytes()
|
|
marker = self._activation_path().read_bytes()
|
|
|
|
resolution = install_runtime.rollback_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
family=FAMILY,
|
|
expected_generation=result.generation,
|
|
)
|
|
|
|
self.assertTrue(resolution.committed)
|
|
self.assertEqual(resolution.committed_generation, result.generation)
|
|
self.assertIsNone(resolution.restored_generation)
|
|
self.assertEqual(self._active_path().read_bytes(), active)
|
|
self.assertEqual(self._activation_path().read_bytes(), marker)
|
|
|
|
def test_rollback_deactivates_a_failed_first_generation(self) -> None:
|
|
self._install()
|
|
|
|
resolution = install_runtime.rollback_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
family=FAMILY,
|
|
expected_generation="1.0.0-abc123",
|
|
)
|
|
|
|
self.assertFalse(resolution.committed)
|
|
self.assertIsNone(resolution.committed_generation)
|
|
self.assertIsNone(resolution.restored_generation)
|
|
self.assertFalse(self._active_path().exists())
|
|
self.assertFalse(
|
|
(
|
|
self.fixture.ai_data_dir
|
|
/ "v3"
|
|
/ "runtimes"
|
|
/ FAMILY
|
|
/ TARGET
|
|
/ "1.0.0-abc123"
|
|
).exists()
|
|
)
|
|
|
|
def test_deactivate_rejects_a_rollback_directory_symlink_before_mutation(self) -> None:
|
|
self._install()
|
|
active_before = self._active_path().read_bytes()
|
|
rollback_root = self.fixture.ai_data_dir / "v3" / "rollback"
|
|
outside = self.root / "outside-rollback"
|
|
outside.mkdir()
|
|
outside_marker = outside / f"{FAMILY}.json"
|
|
outside_marker.write_text("outside", encoding="utf-8")
|
|
shutil.rmtree(rollback_root)
|
|
rollback_root.symlink_to(outside, target_is_directory=True)
|
|
|
|
with self.assertRaisesRegex(install_runtime.InstallError, "rollback.*real directory"):
|
|
install_runtime.deactivate_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
family=FAMILY,
|
|
)
|
|
|
|
self.assertEqual(self._active_path().read_bytes(), active_before)
|
|
self.assertEqual(outside_marker.read_text(encoding="utf-8"), "outside")
|
|
|
|
def test_gc_fails_closed_when_the_active_descriptor_is_corrupted(self) -> None:
|
|
result = self._install()
|
|
self._active_path().write_bytes(b"not-json\n")
|
|
|
|
with self.assertRaisesRegex(install_runtime.InstallError, "active runtime descriptor"):
|
|
with install_runtime.mutation_lock(self.fixture.ai_data_dir / "v3"):
|
|
install_runtime.recover_and_gc(
|
|
self.fixture.ai_data_dir / "v3", keep_unreferenced=0
|
|
)
|
|
|
|
self.assertTrue(result.generation_root.is_dir())
|
|
|
|
def test_retry_does_not_replace_a_corrupted_generation_with_a_live_lease(
|
|
self,
|
|
) -> None:
|
|
installed = self._install()
|
|
self._commit(installed.generation)
|
|
active_before = self._active_path().read_bytes()
|
|
generation = "1.0.0-abc123"
|
|
final_root = (
|
|
self.fixture.ai_data_dir / "v3" / "runtimes" / FAMILY / TARGET / generation
|
|
)
|
|
entrypoint = final_root / "ocr_runner.py"
|
|
entrypoint.write_bytes(b"x" * entrypoint.stat().st_size)
|
|
now = install_runtime._utc_now()
|
|
lease = (
|
|
self.fixture.ai_data_dir
|
|
/ "v3"
|
|
/ "leases"
|
|
/ FAMILY
|
|
/ generation
|
|
/ "live.json"
|
|
)
|
|
lease.parent.mkdir(parents=True)
|
|
lease.write_text(
|
|
json.dumps(
|
|
{
|
|
"schemaVersion": 1,
|
|
"family": FAMILY,
|
|
"generation": generation,
|
|
"pid": os.getpid(),
|
|
"processNonce": "process-nonce",
|
|
"requestNonce": "request-nonce",
|
|
"createdAt": now,
|
|
"heartbeatAt": now,
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with self.assertRaisesRegex(install_runtime.IntegrityError, "lease"):
|
|
self._install()
|
|
|
|
self.assertTrue(final_root.exists())
|
|
self.assertEqual(entrypoint.read_bytes(), b"x" * entrypoint.stat().st_size)
|
|
self.assertEqual(self._active_path().read_bytes(), active_before)
|
|
|
|
def test_retry_does_not_exchange_a_corrupted_generation_while_shared_locked(
|
|
self,
|
|
) -> None:
|
|
installed = self._install()
|
|
self._commit(installed.generation)
|
|
active_before = self._active_path().read_bytes()
|
|
entrypoint = installed.generation_root / "ocr_runner.py"
|
|
corrupted = b"x" * entrypoint.stat().st_size
|
|
entrypoint.write_bytes(corrupted)
|
|
lock_path = self._generation_lock_path(installed.generation)
|
|
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
lock_path.touch(mode=0o660, exist_ok=True)
|
|
descriptor = os.open(lock_path, os.O_RDONLY)
|
|
fcntl.flock(descriptor, fcntl.LOCK_SH)
|
|
try:
|
|
with mock.patch.object(
|
|
install_runtime,
|
|
"_exchange_directories",
|
|
wraps=install_runtime._exchange_directories,
|
|
) as exchange:
|
|
with self.assertRaisesRegex(
|
|
install_runtime.IntegrityError,
|
|
"retry after active runtime requests finish",
|
|
):
|
|
self._install()
|
|
finally:
|
|
fcntl.flock(descriptor, fcntl.LOCK_UN)
|
|
os.close(descriptor)
|
|
|
|
exchange.assert_not_called()
|
|
self.assertEqual(entrypoint.read_bytes(), corrupted)
|
|
self.assertEqual(self._active_path().read_bytes(), active_before)
|
|
|
|
def test_repair_holds_exclusive_lock_through_capacity_extract_and_activation(
|
|
self,
|
|
) -> None:
|
|
installed = self._install()
|
|
self._commit(installed.generation)
|
|
entrypoint = installed.generation_root / "ocr_runner.py"
|
|
entrypoint.write_bytes(b"x" * entrypoint.stat().st_size)
|
|
lock_path = self._generation_lock_path(installed.generation)
|
|
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
lock_path.touch(mode=0o660, exist_ok=True)
|
|
events: list[str] = []
|
|
original_check_disk_space = install_runtime._check_disk_space
|
|
original_extract = install_runtime._safe_extract
|
|
original_exchange = install_runtime._exchange_directories
|
|
original_smoke = install_runtime._run_smoke
|
|
original_activate = install_runtime._activate_with_rollback
|
|
|
|
def check_disk_space(*args, **kwargs):
|
|
self._assert_shared_generation_probe_is_blocked(lock_path)
|
|
events.append("capacity")
|
|
return original_check_disk_space(*args, **kwargs)
|
|
|
|
def extract(*args, **kwargs):
|
|
self._assert_shared_generation_probe_is_blocked(lock_path)
|
|
events.append("extract")
|
|
return original_extract(*args, **kwargs)
|
|
|
|
def exchange(*args, **kwargs):
|
|
self._assert_shared_generation_probe_is_blocked(lock_path)
|
|
events.append("exchange")
|
|
return original_exchange(*args, **kwargs)
|
|
|
|
def smoke(*args, **kwargs):
|
|
self._assert_shared_generation_probe_is_blocked(lock_path)
|
|
events.append("smoke")
|
|
return original_smoke(*args, **kwargs)
|
|
|
|
def activate(*args, **kwargs):
|
|
self._assert_shared_generation_probe_is_blocked(lock_path)
|
|
events.append("activate")
|
|
return original_activate(*args, **kwargs)
|
|
|
|
with (
|
|
mock.patch.object(
|
|
install_runtime, "_check_disk_space", side_effect=check_disk_space
|
|
),
|
|
mock.patch.object(install_runtime, "_safe_extract", side_effect=extract),
|
|
mock.patch.object(
|
|
install_runtime, "_exchange_directories", side_effect=exchange
|
|
),
|
|
mock.patch.object(install_runtime, "_run_smoke", side_effect=smoke),
|
|
mock.patch.object(
|
|
install_runtime, "_activate_with_rollback", side_effect=activate
|
|
),
|
|
):
|
|
self._install()
|
|
|
|
self.assertEqual(
|
|
events,
|
|
["capacity", "extract", "exchange", "smoke", "activate"],
|
|
)
|
|
|
|
def test_repair_holds_exclusive_lock_during_restorative_exchange(self) -> None:
|
|
installed = self._install()
|
|
self._commit(installed.generation)
|
|
active_before = self._active_path().read_bytes()
|
|
entrypoint = installed.generation_root / "ocr_runner.py"
|
|
corrupted = b"x" * entrypoint.stat().st_size
|
|
entrypoint.write_bytes(corrupted)
|
|
lock_path = self._generation_lock_path(installed.generation)
|
|
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
lock_path.touch(mode=0o660, exist_ok=True)
|
|
exchanges = 0
|
|
original_exchange = install_runtime._exchange_directories
|
|
|
|
def exchange(*args, **kwargs):
|
|
nonlocal exchanges
|
|
self._assert_shared_generation_probe_is_blocked(lock_path)
|
|
exchanges += 1
|
|
return original_exchange(*args, **kwargs)
|
|
|
|
with mock.patch.object(
|
|
install_runtime,
|
|
"_exchange_directories",
|
|
side_effect=exchange,
|
|
):
|
|
with self.assertRaisesRegex(install_runtime.SmokeError, "final"):
|
|
self._install(smoke_env={"FAIL_FINAL": "1"})
|
|
|
|
self.assertEqual(exchanges, 2)
|
|
self.assertEqual(entrypoint.read_bytes(), corrupted)
|
|
self.assertEqual(self._active_path().read_bytes(), active_before)
|
|
|
|
def test_smoke_output_is_bounded_and_small_failure_details_are_retained(
|
|
self,
|
|
) -> None:
|
|
with self.assertRaisesRegex(install_runtime.SmokeError, "output exceeded"):
|
|
self._install(smoke_env={"FLOOD_SMOKE": "1"})
|
|
|
|
with self.assertRaisesRegex(
|
|
install_runtime.SmokeError,
|
|
"representative smoke diagnostic",
|
|
):
|
|
self._install(smoke_env={"FAIL_SMOKE_WITH_DETAIL": "1"})
|
|
|
|
def test_recovery_cleans_staging_but_gc_preserves_active_and_leased_generations(
|
|
self,
|
|
) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
(v3 / "staging" / "abandoned").mkdir(parents=True)
|
|
target_root = v3 / "runtimes" / FAMILY / TARGET
|
|
for generation in ("active", "leased", "orphan"):
|
|
(target_root / generation).mkdir(parents=True)
|
|
active_path = v3 / "active" / "ocr.json"
|
|
self._write_gc_descriptor(active_path, FAMILY, "active")
|
|
lease = v3 / "leases" / FAMILY / "leased" / "123-nonce.json"
|
|
lease.parent.mkdir(parents=True)
|
|
now = install_runtime._utc_now()
|
|
lease.write_text(
|
|
json.dumps(
|
|
{
|
|
"schemaVersion": 1,
|
|
"family": FAMILY,
|
|
"generation": "leased",
|
|
"pid": os.getpid(),
|
|
"processNonce": "process-nonce",
|
|
"requestNonce": "request-nonce",
|
|
"createdAt": now,
|
|
"heartbeatAt": now,
|
|
}
|
|
)
|
|
)
|
|
|
|
removed = install_runtime.recover_and_gc(v3, keep_unreferenced=0)
|
|
|
|
self.assertFalse((v3 / "staging" / "abandoned").exists())
|
|
self.assertTrue((target_root / "active").exists())
|
|
self.assertTrue((target_root / "leased").exists())
|
|
self.assertFalse((target_root / "orphan").exists())
|
|
self.assertIn(target_root / "orphan", removed)
|
|
|
|
def test_gc_removes_a_stale_lease_even_when_its_pid_was_reused(self) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
generation = "stale-pid-reuse"
|
|
generation_root = v3 / "runtimes" / FAMILY / TARGET / generation
|
|
generation_root.mkdir(parents=True)
|
|
lease = v3 / "leases" / FAMILY / generation / "live-pid-stale-heartbeat.json"
|
|
lease.parent.mkdir(parents=True)
|
|
lease.write_text(
|
|
json.dumps(
|
|
{
|
|
"schemaVersion": 1,
|
|
"family": FAMILY,
|
|
"generation": generation,
|
|
"pid": os.getpid(),
|
|
"processNonce": "old-process-nonce",
|
|
"requestNonce": "old-request-nonce",
|
|
"createdAt": "2000-01-01T00:00:00.000Z",
|
|
"heartbeatAt": "2000-01-01T00:00:00.000Z",
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
removed = install_runtime.recover_and_gc(v3, keep_unreferenced=0)
|
|
|
|
self.assertFalse(generation_root.exists())
|
|
self.assertFalse(lease.exists())
|
|
self.assertIn(generation_root, removed)
|
|
|
|
def test_gc_removes_a_stale_kernel_lock_backed_schema_two_lease(self) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
generation = "stale-schema-two"
|
|
generation_root = v3 / "runtimes" / FAMILY / TARGET / generation
|
|
generation_root.mkdir(parents=True)
|
|
lease = v3 / "leases" / FAMILY / generation / "schema-two.json"
|
|
lease.parent.mkdir(parents=True)
|
|
lease.write_text(
|
|
json.dumps(
|
|
{
|
|
"schemaVersion": 2,
|
|
"family": FAMILY,
|
|
"generation": generation,
|
|
"pid": os.getpid(),
|
|
"processNonce": "old-process-nonce",
|
|
"requestNonce": "old-request-nonce",
|
|
"createdAt": "2000-01-01T00:00:00.000Z",
|
|
"heartbeatAt": "2000-01-01T00:00:00.000Z",
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with install_runtime.mutation_lock(v3):
|
|
removed = install_runtime.recover_and_gc(v3, keep_unreferenced=0)
|
|
|
|
self.assertFalse(generation_root.exists())
|
|
self.assertFalse(lease.exists())
|
|
self.assertIn(generation_root, removed)
|
|
|
|
def test_gc_preserves_a_stale_lease_with_an_unknown_schema(self) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
generation = "unknown-lease-schema"
|
|
generation_root = v3 / "runtimes" / FAMILY / TARGET / generation
|
|
generation_root.mkdir(parents=True)
|
|
lease = v3 / "leases" / FAMILY / generation / "unknown-schema.json"
|
|
lease.parent.mkdir(parents=True)
|
|
lease.write_text(
|
|
json.dumps(
|
|
{
|
|
"schemaVersion": 3,
|
|
"family": FAMILY,
|
|
"generation": generation,
|
|
"pid": os.getpid(),
|
|
"processNonce": "old-process-nonce",
|
|
"requestNonce": "old-request-nonce",
|
|
"createdAt": "2000-01-01T00:00:00.000Z",
|
|
"heartbeatAt": "2000-01-01T00:00:00.000Z",
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with install_runtime.mutation_lock(v3):
|
|
removed = install_runtime.recover_and_gc(v3, keep_unreferenced=0)
|
|
|
|
self.assertTrue(generation_root.exists())
|
|
self.assertTrue(lease.exists())
|
|
self.assertNotIn(generation_root, removed)
|
|
|
|
def test_gc_preserves_a_heartbeat_atomically_replaced_during_stale_cleanup(
|
|
self,
|
|
) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
generation = "heartbeat-replaced-during-gc"
|
|
generation_root = v3 / "runtimes" / FAMILY / TARGET / generation
|
|
generation_root.mkdir(parents=True)
|
|
lease = v3 / "leases" / FAMILY / generation / "request.json"
|
|
lease.parent.mkdir(parents=True)
|
|
stale = {
|
|
"schemaVersion": 1,
|
|
"family": FAMILY,
|
|
"generation": generation,
|
|
"pid": os.getpid(),
|
|
"processNonce": "process-nonce",
|
|
"requestNonce": "request-nonce",
|
|
"createdAt": "2000-01-01T00:00:00.000Z",
|
|
"heartbeatAt": "2000-01-01T00:00:00.000Z",
|
|
}
|
|
lease.write_text(json.dumps(stale), encoding="utf-8")
|
|
|
|
def publish_fresh_heartbeat(_lease: object, **_kwargs: object) -> bool:
|
|
fresh = dict(stale)
|
|
fresh["requestNonce"] = "fresh-request-nonce"
|
|
fresh["createdAt"] = install_runtime._utc_now()
|
|
fresh["heartbeatAt"] = fresh["createdAt"]
|
|
replacement = lease.with_suffix(".replacement")
|
|
replacement.write_text(json.dumps(fresh), encoding="utf-8")
|
|
os.replace(replacement, lease)
|
|
return True
|
|
|
|
with mock.patch.object(
|
|
install_runtime,
|
|
"_lease_is_stale",
|
|
side_effect=publish_fresh_heartbeat,
|
|
):
|
|
removed = install_runtime.recover_and_gc(v3, keep_unreferenced=0)
|
|
|
|
self.assertTrue(generation_root.exists())
|
|
self.assertTrue(lease.exists())
|
|
self.assertEqual(json.loads(lease.read_bytes())["requestNonce"], "fresh-request-nonce")
|
|
self.assertNotIn(generation_root, removed)
|
|
|
|
def test_gc_preserves_a_fresh_lease_when_its_pid_is_not_visible(self) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
generation = "fresh-cross-namespace-lease"
|
|
generation_root = v3 / "runtimes" / FAMILY / TARGET / generation
|
|
generation_root.mkdir(parents=True)
|
|
lease = v3 / "leases" / FAMILY / generation / "hidden-pid-fresh-heartbeat.json"
|
|
lease.parent.mkdir(parents=True)
|
|
now = install_runtime._utc_now()
|
|
lease.write_text(
|
|
json.dumps(
|
|
{
|
|
"schemaVersion": 1,
|
|
"family": FAMILY,
|
|
"generation": generation,
|
|
# A live process in another PID namespace may not be visible
|
|
# to the installer even though its heartbeat is current.
|
|
"pid": 2_147_483_647,
|
|
"processNonce": "cross-namespace-process-nonce",
|
|
"requestNonce": "cross-namespace-request-nonce",
|
|
"createdAt": now,
|
|
"heartbeatAt": now,
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
removed = install_runtime.recover_and_gc(v3, keep_unreferenced=0)
|
|
|
|
self.assertTrue(generation_root.exists())
|
|
self.assertTrue(lease.exists())
|
|
self.assertNotIn(generation_root, removed)
|
|
|
|
def test_gc_conservatively_preserves_an_ambiguous_malformed_lease(self) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
generation = "ambiguous-lease"
|
|
generation_root = v3 / "runtimes" / FAMILY / TARGET / generation
|
|
generation_root.mkdir(parents=True)
|
|
lease = v3 / "leases" / FAMILY / generation / "malformed.json"
|
|
lease.parent.mkdir(parents=True)
|
|
lease.write_text("[]", encoding="utf-8")
|
|
|
|
removed = install_runtime.recover_and_gc(v3, keep_unreferenced=0)
|
|
|
|
self.assertTrue(generation_root.exists())
|
|
self.assertTrue(lease.exists())
|
|
self.assertNotIn(generation_root, removed)
|
|
|
|
def test_gc_removes_a_stale_atomic_lease_temporary_file(self) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
generation = "stale-atomic-lease-temp"
|
|
generation_root = v3 / "runtimes" / FAMILY / TARGET / generation
|
|
generation_root.mkdir(parents=True)
|
|
lease_temp = (
|
|
v3
|
|
/ "leases"
|
|
/ FAMILY
|
|
/ generation
|
|
/ (
|
|
"123-11111111-1111-4111-8111-111111111111-"
|
|
"22222222-2222-4222-8222-222222222222.json."
|
|
"33333333-3333-4333-8333-333333333333.tmp"
|
|
)
|
|
)
|
|
lease_temp.parent.mkdir(parents=True)
|
|
lease_temp.write_text("partially-written lease", encoding="utf-8")
|
|
stale_timestamp = dt.datetime(2000, 1, 1, tzinfo=dt.timezone.utc).timestamp()
|
|
os.utime(lease_temp, (stale_timestamp, stale_timestamp))
|
|
|
|
removed = install_runtime.recover_and_gc(v3, keep_unreferenced=0)
|
|
|
|
self.assertFalse(generation_root.exists())
|
|
self.assertFalse(lease_temp.exists())
|
|
self.assertIn(generation_root, removed)
|
|
|
|
def test_gc_preserves_a_recent_atomic_lease_temporary_file(self) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
generation = "recent-atomic-lease-temp"
|
|
generation_root = v3 / "runtimes" / FAMILY / TARGET / generation
|
|
generation_root.mkdir(parents=True)
|
|
lease_temp = (
|
|
v3
|
|
/ "leases"
|
|
/ FAMILY
|
|
/ generation
|
|
/ (
|
|
"123-11111111-1111-4111-8111-111111111111-"
|
|
"22222222-2222-4222-8222-222222222222.json."
|
|
"33333333-3333-4333-8333-333333333333.tmp"
|
|
)
|
|
)
|
|
lease_temp.parent.mkdir(parents=True)
|
|
lease_temp.write_text("partially-written lease", encoding="utf-8")
|
|
|
|
removed = install_runtime.recover_and_gc(v3, keep_unreferenced=0)
|
|
|
|
self.assertTrue(generation_root.exists())
|
|
self.assertTrue(lease_temp.exists())
|
|
self.assertNotIn(generation_root, removed)
|
|
|
|
def test_gc_preserves_an_unrecognized_stale_temporary_lease_file(self) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
generation = "unknown-temp-name"
|
|
generation_root = v3 / "runtimes" / FAMILY / TARGET / generation
|
|
generation_root.mkdir(parents=True)
|
|
lease_temp = (
|
|
v3 / "leases" / FAMILY / generation / "malformed.json.not-a-uuid.tmp"
|
|
)
|
|
lease_temp.parent.mkdir(parents=True)
|
|
lease_temp.write_text("partially-written lease", encoding="utf-8")
|
|
stale_timestamp = dt.datetime(2000, 1, 1, tzinfo=dt.timezone.utc).timestamp()
|
|
os.utime(lease_temp, (stale_timestamp, stale_timestamp))
|
|
|
|
removed = install_runtime.recover_and_gc(v3, keep_unreferenced=0)
|
|
|
|
self.assertTrue(generation_root.exists())
|
|
self.assertTrue(lease_temp.exists())
|
|
self.assertNotIn(generation_root, removed)
|
|
|
|
def test_gc_preserves_a_shared_locked_orphan_until_its_reader_releases(
|
|
self,
|
|
) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
generation = "shared-reader"
|
|
generation_root = v3 / "runtimes" / FAMILY / TARGET / generation
|
|
generation_root.mkdir(parents=True)
|
|
lock_path = self._generation_lock_path(generation)
|
|
lock_path.parent.mkdir(parents=True)
|
|
lock_path.touch(mode=0o660)
|
|
descriptor = os.open(lock_path, os.O_RDONLY)
|
|
fcntl.flock(descriptor, fcntl.LOCK_SH)
|
|
try:
|
|
with install_runtime.mutation_lock(v3):
|
|
removed = install_runtime.recover_and_gc(
|
|
v3, keep_unreferenced=0
|
|
)
|
|
|
|
self.assertTrue(generation_root.exists())
|
|
self.assertNotIn(generation_root, removed)
|
|
finally:
|
|
fcntl.flock(descriptor, fcntl.LOCK_UN)
|
|
os.close(descriptor)
|
|
|
|
with install_runtime.mutation_lock(v3):
|
|
removed = install_runtime.recover_and_gc(v3, keep_unreferenced=0)
|
|
|
|
self.assertFalse(generation_root.exists())
|
|
self.assertIn(generation_root, removed)
|
|
self.assertTrue(lock_path.is_file())
|
|
|
|
def test_gc_busy_generations_do_not_consume_unreferenced_retention(self) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
target_root = v3 / "runtimes" / FAMILY / TARGET
|
|
busy_root = target_root / "newer-busy"
|
|
eligible_root = target_root / "older-eligible"
|
|
busy_root.mkdir(parents=True)
|
|
eligible_root.mkdir()
|
|
os.utime(eligible_root, (1, 1))
|
|
os.utime(busy_root, (2, 2))
|
|
lock_path = self._generation_lock_path("newer-busy")
|
|
lock_path.parent.mkdir(parents=True)
|
|
lock_path.touch(mode=0o660)
|
|
descriptor = os.open(lock_path, os.O_RDONLY)
|
|
fcntl.flock(descriptor, fcntl.LOCK_SH)
|
|
try:
|
|
with install_runtime.mutation_lock(v3):
|
|
removed = install_runtime.recover_and_gc(
|
|
v3, keep_unreferenced=1
|
|
)
|
|
finally:
|
|
fcntl.flock(descriptor, fcntl.LOCK_UN)
|
|
os.close(descriptor)
|
|
|
|
self.assertTrue(busy_root.exists())
|
|
self.assertTrue(eligible_root.exists())
|
|
self.assertNotIn(busy_root, removed)
|
|
self.assertNotIn(eligible_root, removed)
|
|
|
|
def test_gc_holds_the_exclusive_generation_lock_during_actual_removal(
|
|
self,
|
|
) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
generation = "remove-under-lock"
|
|
generation_root = v3 / "runtimes" / FAMILY / TARGET / generation
|
|
generation_root.mkdir(parents=True)
|
|
lock_path = self._generation_lock_path(generation)
|
|
original_remove = install_runtime._remove_without_following
|
|
checked = False
|
|
|
|
def inspect_removal(path: Path) -> None:
|
|
nonlocal checked
|
|
if path == generation_root:
|
|
self._assert_shared_generation_probe_is_blocked(lock_path)
|
|
checked = True
|
|
original_remove(path)
|
|
|
|
with install_runtime.mutation_lock(v3):
|
|
with mock.patch.object(
|
|
install_runtime,
|
|
"_remove_without_following",
|
|
side_effect=inspect_removal,
|
|
):
|
|
install_runtime.recover_and_gc(v3, keep_unreferenced=0)
|
|
|
|
self.assertTrue(checked)
|
|
self.assertFalse(generation_root.exists())
|
|
|
|
def test_gc_rechecks_for_a_live_lease_after_reacquiring_for_deletion(
|
|
self,
|
|
) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
generation = "lease-before-delete"
|
|
generation_root = v3 / "runtimes" / FAMILY / TARGET / generation
|
|
generation_root.mkdir(parents=True)
|
|
|
|
with install_runtime.mutation_lock(v3):
|
|
with mock.patch.object(
|
|
install_runtime,
|
|
"_has_live_lease",
|
|
side_effect=(False, True),
|
|
) as has_live_lease:
|
|
removed = install_runtime.recover_and_gc(
|
|
v3, keep_unreferenced=0
|
|
)
|
|
|
|
self.assertEqual(has_live_lease.call_count, 2)
|
|
self.assertTrue(generation_root.exists())
|
|
self.assertNotIn(generation_root, removed)
|
|
|
|
def test_generation_lock_inode_is_permanent_and_has_shared_state_modes(
|
|
self,
|
|
) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
generation = "permanent-inode"
|
|
generation_root = v3 / "runtimes" / FAMILY / TARGET / generation
|
|
generation_root.mkdir(parents=True)
|
|
lock_path = self._generation_lock_path(generation)
|
|
|
|
with install_runtime.mutation_lock(v3):
|
|
install_runtime.recover_and_gc(v3, keep_unreferenced=1)
|
|
|
|
original = lock_path.stat()
|
|
for directory in (
|
|
v3 / "locks",
|
|
v3 / "locks" / "generations",
|
|
v3 / "locks" / "generations" / FAMILY,
|
|
):
|
|
with self.subTest(directory=directory):
|
|
self.assertEqual(stat.S_IMODE(directory.stat().st_mode), 0o2770)
|
|
self.assertEqual(stat.S_IMODE(original.st_mode), 0o660)
|
|
self.assertEqual(original.st_nlink, 1)
|
|
|
|
with install_runtime.mutation_lock(v3):
|
|
install_runtime.recover_and_gc(v3, keep_unreferenced=0)
|
|
|
|
self.assertFalse(generation_root.exists())
|
|
self.assertEqual(lock_path.stat().st_ino, original.st_ino)
|
|
generation_root.mkdir()
|
|
with install_runtime.mutation_lock(v3):
|
|
install_runtime.recover_and_gc(v3, keep_unreferenced=1)
|
|
self.assertEqual(lock_path.stat().st_ino, original.st_ino)
|
|
|
|
def test_gc_fails_closed_on_a_symlinked_generation_lock(self) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
generation = "symlink-lock"
|
|
generation_root = v3 / "runtimes" / FAMILY / TARGET / generation
|
|
generation_root.mkdir(parents=True)
|
|
lock_path = self._generation_lock_path(generation)
|
|
lock_path.parent.mkdir(parents=True)
|
|
sentinel = self.root / "generation-lock-sentinel"
|
|
sentinel.write_text("sentinel", encoding="utf-8")
|
|
lock_path.symlink_to(sentinel)
|
|
|
|
with self.assertRaisesRegex(install_runtime.InstallError, "generation lock"):
|
|
with install_runtime.mutation_lock(v3):
|
|
install_runtime.recover_and_gc(v3, keep_unreferenced=0)
|
|
|
|
self.assertTrue(generation_root.exists())
|
|
self.assertEqual(sentinel.read_text(encoding="utf-8"), "sentinel")
|
|
|
|
def test_gc_fails_closed_on_a_hardlinked_generation_lock(self) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
generation = "hardlink-lock"
|
|
generation_root = v3 / "runtimes" / FAMILY / TARGET / generation
|
|
generation_root.mkdir(parents=True)
|
|
lock_path = self._generation_lock_path(generation)
|
|
lock_path.parent.mkdir(parents=True)
|
|
sentinel = self.root / "hardlinked-generation-lock"
|
|
sentinel.write_text("sentinel", encoding="utf-8")
|
|
os.link(sentinel, lock_path)
|
|
|
|
with self.assertRaisesRegex(install_runtime.InstallError, "generation lock"):
|
|
with install_runtime.mutation_lock(v3):
|
|
install_runtime.recover_and_gc(v3, keep_unreferenced=0)
|
|
|
|
self.assertTrue(generation_root.exists())
|
|
self.assertEqual(sentinel.read_text(encoding="utf-8"), "sentinel")
|
|
self.assertEqual(sentinel.stat().st_nlink, 2)
|
|
|
|
def test_gc_fails_closed_when_the_generation_lock_inode_changes_during_flock(
|
|
self,
|
|
) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
generation = "raced-lock-inode"
|
|
generation_root = v3 / "runtimes" / FAMILY / TARGET / generation
|
|
generation_root.mkdir(parents=True)
|
|
lock_path = self._generation_lock_path(generation)
|
|
lock_path.parent.mkdir(parents=True)
|
|
lock_path.touch(mode=0o660)
|
|
original_inode = lock_path.stat().st_ino
|
|
original_flock = fcntl.flock
|
|
raced = False
|
|
|
|
def replace_during_flock(descriptor: int, operation: int) -> None:
|
|
nonlocal raced
|
|
if operation == fcntl.LOCK_EX | fcntl.LOCK_NB and not raced:
|
|
lock_path.unlink()
|
|
lock_path.write_bytes(b"replacement")
|
|
raced = True
|
|
original_flock(descriptor, operation)
|
|
|
|
with install_runtime.mutation_lock(v3):
|
|
with mock.patch.object(
|
|
install_runtime.fcntl,
|
|
"flock",
|
|
side_effect=replace_during_flock,
|
|
):
|
|
with self.assertRaisesRegex(
|
|
install_runtime.InstallError, "generation lock"
|
|
):
|
|
install_runtime.recover_and_gc(v3, keep_unreferenced=0)
|
|
|
|
self.assertTrue(raced)
|
|
self.assertTrue(generation_root.exists())
|
|
self.assertNotEqual(lock_path.stat().st_ino, original_inode)
|
|
|
|
def test_gc_fails_closed_when_generation_flock_is_unsupported(self) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
generation_root = v3 / "runtimes" / FAMILY / TARGET / "no-flock"
|
|
generation_root.mkdir(parents=True)
|
|
|
|
with install_runtime.mutation_lock(v3):
|
|
with mock.patch.object(
|
|
install_runtime.fcntl,
|
|
"flock",
|
|
side_effect=OSError(errno.ENOLCK, "locks unavailable"),
|
|
):
|
|
with self.assertRaisesRegex(
|
|
install_runtime.InstallError, "generation lock"
|
|
):
|
|
install_runtime.recover_and_gc(v3, keep_unreferenced=0)
|
|
|
|
self.assertTrue(generation_root.exists())
|
|
|
|
def test_gc_fails_closed_without_a_real_nofollow_open_flag(self) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
generation_root = v3 / "runtimes" / FAMILY / TARGET / "no-nofollow"
|
|
generation_root.mkdir(parents=True)
|
|
|
|
with install_runtime.mutation_lock(v3):
|
|
with mock.patch.object(install_runtime.os, "O_NOFOLLOW", 0):
|
|
with self.assertRaisesRegex(
|
|
install_runtime.InstallError, "not safely supported"
|
|
):
|
|
install_runtime.recover_and_gc(v3, keep_unreferenced=0)
|
|
|
|
self.assertTrue(generation_root.exists())
|
|
|
|
def test_generation_lock_rejects_unsafe_components(self) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
with install_runtime.mutation_lock(v3):
|
|
with self.assertRaisesRegex(
|
|
install_runtime.PreflightError, "runtime family"
|
|
):
|
|
with install_runtime._generation_lock(
|
|
v3, "../outside", "valid-generation"
|
|
):
|
|
self.fail("unsafe generation lock path was accepted")
|
|
|
|
self.assertFalse((v3 / "locks" / "outside").exists())
|
|
|
|
@unittest.skipUnless(sys.platform != "win32", "fcntl is POSIX-only")
|
|
def test_mutation_lock_is_held_by_an_open_fd_and_excludes_another_process(
|
|
self,
|
|
) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
ready = self.root / "lock-ready"
|
|
code = (
|
|
"import pathlib, sys, time; "
|
|
"from packages.ai.python.install_runtime import mutation_lock; "
|
|
"v3=pathlib.Path(sys.argv[1]); ready=pathlib.Path(sys.argv[2]); "
|
|
"cm=mutation_lock(v3); cm.__enter__(); ready.write_text('ready'); time.sleep(10)"
|
|
)
|
|
child = subprocess.Popen(
|
|
[sys.executable, "-c", code, str(v3), str(ready)],
|
|
cwd=Path(__file__).resolve().parents[4],
|
|
)
|
|
try:
|
|
for _ in range(100):
|
|
if ready.exists():
|
|
break
|
|
import time
|
|
|
|
time.sleep(0.02)
|
|
self.assertTrue(ready.exists())
|
|
with self.assertRaises(install_runtime.LockBusyError):
|
|
with install_runtime.mutation_lock(v3, blocking=False):
|
|
pass
|
|
finally:
|
|
child.terminate()
|
|
child.wait(timeout=5)
|
|
|
|
def test_mutation_lock_fsyncs_a_new_v3_entry_in_its_parent(self) -> None:
|
|
ai_data_dir = self.root / "fresh-ai-data"
|
|
ai_data_dir.mkdir()
|
|
v3 = ai_data_dir / "v3"
|
|
original_fsync_directory = install_runtime._fsync_directory
|
|
|
|
with mock.patch.object(
|
|
install_runtime,
|
|
"_fsync_directory",
|
|
wraps=original_fsync_directory,
|
|
) as fsync_directory:
|
|
with install_runtime.mutation_lock(v3):
|
|
pass
|
|
|
|
fsync_directory.assert_any_call(ai_data_dir)
|
|
|
|
def test_mutation_lock_never_follows_a_lock_file_symlink(self) -> None:
|
|
v3 = self.fixture.ai_data_dir / "v3"
|
|
lock_directory = v3 / "locks"
|
|
lock_directory.mkdir(parents=True)
|
|
target = self.root / "must-not-be-locked-or-modified"
|
|
target.write_text("sentinel", encoding="utf-8")
|
|
(lock_directory / "mutation.lock").symlink_to(target)
|
|
|
|
with self.assertRaisesRegex(install_runtime.InstallError, "lock.*safely"):
|
|
with install_runtime.mutation_lock(v3):
|
|
self.fail("symlinked lock unexpectedly acquired")
|
|
|
|
self.assertEqual(target.read_text(encoding="utf-8"), "sentinel")
|
|
|
|
def test_preverified_index_and_local_archive_use_the_same_transaction(self) -> None:
|
|
index = self.root / "index.json"
|
|
index.write_text(
|
|
json.dumps(
|
|
{
|
|
"schemaVersion": 1,
|
|
"artifacts": [self.fixture.artifact()],
|
|
"signature": {
|
|
"keyId": "test-key",
|
|
"algorithm": "ed25519",
|
|
"value": "preverified-by-release-client",
|
|
},
|
|
},
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with self.assertRaises(install_runtime.IndexVerificationError):
|
|
install_runtime.install_from_index(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
index_path=index,
|
|
family=FAMILY,
|
|
target=TARGET,
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
)
|
|
|
|
result = install_runtime.offline_import(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
index_path=index,
|
|
family=FAMILY,
|
|
target=TARGET,
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
preverified_index=True,
|
|
expected_index_sha256=_sha256(index.read_bytes()),
|
|
)
|
|
self.assertEqual(result.generation, "1.0.0-abc123")
|
|
descriptor = json.loads(self._active_path().read_bytes())
|
|
signed_index = descriptor["artifact"]["signedIndex"]
|
|
preserved_index = self.fixture.ai_data_dir / "v3" / signed_index["path"]
|
|
self.assertEqual(preserved_index.read_bytes(), index.read_bytes())
|
|
self.assertEqual(signed_index["sha256"], _sha256(index.read_bytes()))
|
|
self.assertEqual(signed_index["size"], len(index.read_bytes()))
|
|
self.assertLess(len(self._active_path().read_bytes()), 64 * 1024)
|
|
|
|
def test_preverified_index_is_bound_to_the_callers_authenticated_bytes(
|
|
self,
|
|
) -> None:
|
|
index = self.root / "index.json"
|
|
index.write_text(
|
|
json.dumps(
|
|
{
|
|
"schemaVersion": 1,
|
|
"artifacts": [self.fixture.artifact()],
|
|
"signature": {
|
|
"keyId": "test-key",
|
|
"algorithm": "ed25519",
|
|
"value": "verified-by-the-node-release-client",
|
|
},
|
|
},
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with self.assertRaisesRegex(
|
|
install_runtime.IndexVerificationError,
|
|
"authenticated index bytes changed",
|
|
):
|
|
install_runtime.install_from_index(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
index_path=index,
|
|
family=FAMILY,
|
|
target=TARGET,
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
preverified_index=True,
|
|
expected_index_sha256="0" * 64,
|
|
)
|
|
|
|
def test_local_index_reader_never_follows_a_symlink(self) -> None:
|
|
index = self.root / "index.json"
|
|
index.write_text(
|
|
json.dumps(
|
|
{
|
|
"schemaVersion": 1,
|
|
"artifacts": [self.fixture.artifact()],
|
|
"signature": {
|
|
"keyId": "test-key",
|
|
"algorithm": "ed25519",
|
|
"value": "verified-by-the-node-release-client",
|
|
},
|
|
},
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
index_link = self.root / "index-link.json"
|
|
index_link.symlink_to(index)
|
|
|
|
with self.assertRaisesRegex(
|
|
install_runtime.IndexVerificationError,
|
|
"opened safely",
|
|
):
|
|
install_runtime.install_from_index(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
index_path=index_link,
|
|
family=FAMILY,
|
|
target=TARGET,
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
preverified_index=True,
|
|
expected_index_sha256=_sha256(index.read_bytes()),
|
|
)
|
|
|
|
def test_local_index_reader_rejects_a_fifo_without_blocking(self) -> None:
|
|
index_fifo = self.root / "index.fifo"
|
|
os.mkfifo(index_fifo)
|
|
|
|
with self.assertRaisesRegex(
|
|
install_runtime.IndexVerificationError,
|
|
"regular file",
|
|
):
|
|
install_runtime.install_from_index(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
index_path=index_fifo,
|
|
family=FAMILY,
|
|
target=TARGET,
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
)
|
|
|
|
def test_local_index_reader_rejects_an_oversized_sparse_file_before_reading(
|
|
self,
|
|
) -> None:
|
|
index = self.root / "oversized-index.json"
|
|
with index.open("wb") as handle:
|
|
handle.truncate(install_runtime.INDEX_MAX_BYTES + 1)
|
|
|
|
with self.assertRaisesRegex(
|
|
install_runtime.IndexVerificationError,
|
|
"size ceiling",
|
|
):
|
|
install_runtime.install_from_index(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
index_path=index,
|
|
family=FAMILY,
|
|
target=TARGET,
|
|
archive_path=self.fixture.archive,
|
|
smoke_command=self.fixture.smoke_command(),
|
|
host=self.host,
|
|
)
|
|
|
|
def test_deactivate_removes_only_the_family_descriptor_and_collects_unleased_runtime(
|
|
self,
|
|
) -> None:
|
|
self._install()
|
|
other_active = self.fixture.ai_data_dir / "v3" / "active" / "vision-onnx.json"
|
|
self._write_gc_descriptor(other_active, "vision-onnx", "leave-me")
|
|
|
|
changed = install_runtime.deactivate_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
family=FAMILY,
|
|
)
|
|
|
|
self.assertTrue(changed)
|
|
self.assertFalse(self._active_path().exists())
|
|
self.assertTrue(other_active.exists())
|
|
target_root = self.fixture.ai_data_dir / "v3" / "runtimes" / FAMILY / TARGET
|
|
self.assertFalse((target_root / "1.0.0-abc123").exists())
|
|
|
|
def test_deactivate_preserves_a_generation_while_a_live_lease_exists(self) -> None:
|
|
self._install()
|
|
generation = "1.0.0-abc123"
|
|
lease = (
|
|
self.fixture.ai_data_dir
|
|
/ "v3"
|
|
/ "leases"
|
|
/ FAMILY
|
|
/ generation
|
|
/ "live.json"
|
|
)
|
|
lease.parent.mkdir(parents=True)
|
|
lease.write_text(json.dumps({"pid": os.getpid()}), encoding="utf-8")
|
|
|
|
install_runtime.deactivate_runtime(
|
|
ai_data_dir=self.fixture.ai_data_dir,
|
|
family=FAMILY,
|
|
)
|
|
|
|
runtime = (
|
|
self.fixture.ai_data_dir / "v3" / "runtimes" / FAMILY / TARGET / generation
|
|
)
|
|
self.assertTrue(runtime.exists())
|
|
|
|
def test_reset_deactivates_every_family_without_following_suspicious_state(
|
|
self,
|
|
) -> None:
|
|
self._install()
|
|
other_active = self.fixture.ai_data_dir / "v3" / "active" / "vision-onnx.json"
|
|
other_active.write_text(
|
|
json.dumps({"family": "vision-onnx", "generation": "leave-me"}),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
removed = install_runtime.reset_runtimes(ai_data_dir=self.fixture.ai_data_dir)
|
|
|
|
self.assertEqual(removed, 2)
|
|
self.assertEqual(list(other_active.parent.iterdir()), [])
|
|
|
|
external = self.root / "external-active"
|
|
external.mkdir()
|
|
(self.fixture.ai_data_dir / "v3" / "active").rmdir()
|
|
(self.fixture.ai_data_dir / "v3" / "active").symlink_to(
|
|
external, target_is_directory=True
|
|
)
|
|
with self.assertRaisesRegex(
|
|
install_runtime.InstallError, "active.*real directory"
|
|
):
|
|
install_runtime.reset_runtimes(ai_data_dir=self.fixture.ai_data_dir)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|