fix: release QA hardening across processing, media, security, and CI gates (#649)

A release-readiness QA pass over the whole product. The commits split into
defects a user would hit and gates that were reporting green while measuring
nothing.

## Fixes that change behaviour

Rate limiting was bypassable on every install: TRUST_PROXY defaulted to true, so
request.ip came from a client-set header and a forged X-Forwarded-For got past
the login limiter. The default is now a private-network trust list.

A transient Postgres outage stranded in-flight jobs, leaving finished output on
disk with no row pointing at it. A reconciler now resolves those rows and adopts
the bytes rather than dropping the work.

A Redis connection that moved to a new address wedged every read-blocked
consumer, so completions stopped signalling while health still answered 200.
Socket timeouts plus subscriber pings recover it.

Installing more than one AI bundle left the shared venv multi-versioned and
silently broke three tools. The installer now reconciles distributions to one
version each.

Converting an image to JXL at quality 1 through 4 returned a 500, because
libjxl 0.7 rejects the distance those values compute. The quality is floored at
what the encoder honours. A missing ffmpeg was also reported to the user as a
corrupt upload; it now says the engine is unavailable.

RAW uploads reached an unpatched LibRaw on arm64, so it is built from source at
0.22.2, and the release scan was split so it can fail on an unfixed critical
instead of hiding it behind ignore-unfixed.

## Gates that could not fail

Two mutation lanes ran zero mutants because Stryker crawled the gitignored docs
build; coverage discarded its whole report on any failing test; the lint gate
skipped root tests, scripts, and two workspaces; and several generated matrices
counted a host missing ffmpeg as a passing tool. Each now measures what it
claims.

Full evidence and the outstanding release items are tracked locally and are not
part of this branch.

packages/ai/python/tests/test_install_feature_download.py

@@ -13,6 +13,26 @@ def load_installer():
     return module
 
 
+def test_main_temporarily_lifts_hugging_face_offline_flags(monkeypatch):
+    """An explicit bundle install stays online while runtime remains fail-closed."""
+    installer = load_installer()
+    monkeypatch.setenv("HF_HUB_OFFLINE", "1")
+    monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1")
+    observed = {}
+
+    def fake_install():
+        observed["hf"] = os.environ["HF_HUB_OFFLINE"]
+        observed["transformers"] = os.environ["TRANSFORMERS_OFFLINE"]
+
+    monkeypatch.setattr(installer, "_install", fake_install)
+
+    installer.main()
+
+    assert observed == {"hf": "0", "transformers": "0"}
+    assert os.environ["HF_HUB_OFFLINE"] == "1"
+    assert os.environ["TRANSFORMERS_OFFLINE"] == "1"
+
+
 def test_download_with_hf_hub_uses_accelerated_client(monkeypatch, tmp_path):
     installer = load_installer()
     downloaded = tmp_path / "hf-cache" / "bundle.tar.gz"

packages/ai/python/tests/test_install_feature_reconcile_distributions.py

@@ -0,0 +1,376 @@
+"""Mechanics of the general distribution reconciliation in install_feature.py.
+
+The version-conflict suite covers the field failure (AI-20260726-001). This one
+covers the parts of the machinery that failure does not exercise: what a
+rollback restores, what the uninstall is allowed to touch, and that generalising
+the reconciliation did not swallow the onnxruntime flavour rule it replaced
+(#490), which is the one collision where the newer or later version is NOT the
+right winner.
+"""
+
+import importlib.util
+import os
+
+CUDA_PROVIDER_LIB = "libonnxruntime_providers_cuda.so"
+
+
+def load_installer():
+    script_path = os.path.join(os.path.dirname(__file__), "..", "install_feature.py")
+    spec = importlib.util.spec_from_file_location("install_feature_reconcile_under_test", script_path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def write_dist(sp, name, version, files, extra_record=()):
+    """Lay out a distribution: `files` maps relative path -> contents."""
+    recorded = []
+    for rel, contents in files.items():
+        target = sp / rel
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(contents)
+        recorded.append(rel)
+    dist_info = sp / f"{name}-{version}.dist-info"
+    dist_info.mkdir(parents=True, exist_ok=True)
+    (dist_info / "METADATA").write_text(f"Name: {name}\nVersion: {version}\n")
+    recorded += [
+        f"{name}-{version}.dist-info/METADATA",
+        f"{name}-{version}.dist-info/RECORD",
+    ]
+    recorded += list(extra_record)
+    (dist_info / "RECORD").write_text("\n".join(f"{rel},," for rel in recorded) + "\n")
+
+
+def merge(installer, staging, venv_sp, quarantine):
+    """The sequence _install runs, so the tests cannot drift from production."""
+    installer.reconcile_onnxruntime(str(staging), str(venv_sp))
+    plan = installer.plan_reconciliation(str(staging), str(venv_sp))
+    installer.supersede_distributions(str(venv_sp), plan, str(quarantine))
+    installer.move_tree(str(staging), str(venv_sp))
+    return plan
+
+
+def trees(tmp_path):
+    staging = tmp_path / "staging"
+    venv_sp = tmp_path / "venv-sp"
+    quarantine = tmp_path / "venv" / ".superseded"
+    staging.mkdir()
+    venv_sp.mkdir()
+    return staging, venv_sp, quarantine
+
+
+def dist_infos(sp, prefix):
+    return sorted(n for n in os.listdir(sp) if n.startswith(prefix) and n.endswith(".dist-info"))
+
+
+# -- the onnxruntime flavour rule survives generalisation (#490) --
+
+
+def test_an_incoming_cpu_onnxruntime_still_cannot_evict_the_gpu_build(tmp_path):
+    """The flavour rule drops the CPU build from staging before the general pass
+    runs, so the general pass must never see an incoming `onnxruntime` and must
+    leave `onnxruntime-gpu` installed. Newer-or-later would get this wrong: the
+    CPU wheel is a different distribution writing the same package directory."""
+    installer = load_installer()
+    staging, venv_sp, quarantine = trees(tmp_path)
+    write_dist(
+        venv_sp, "onnxruntime_gpu", "1.20.1",
+        {
+            "onnxruntime/__init__.py": "gpu",
+            f"onnxruntime/capi/{CUDA_PROVIDER_LIB}": "cuda",
+            "onnxruntime/capi/onnxruntime_pybind11_state.so": "gpu",
+        },
+    )
+    write_dist(
+        staging, "onnxruntime", "1.20.1",
+        {
+            "onnxruntime/__init__.py": "cpu",
+            "onnxruntime/capi/onnxruntime_pybind11_state.so": "cpu",
+        },
+    )
+    write_dist(staging, "faster_whisper", "1.2.1", {"faster_whisper/__init__.py": "fw"})
+
+    merge(installer, staging, venv_sp, quarantine)
+
+    assert (venv_sp / "onnxruntime" / "capi" / CUDA_PROVIDER_LIB).exists()
+    assert (venv_sp / "onnxruntime" / "capi" / "onnxruntime_pybind11_state.so").read_text() == "gpu"
+    assert dist_infos(venv_sp, "onnxruntime_gpu-") == ["onnxruntime_gpu-1.20.1.dist-info"]
+    assert dist_infos(venv_sp, "onnxruntime-") == []
+    # The rest of the bundle still installs.
+    assert (venv_sp / "faster_whisper" / "__init__.py").read_text() == "fw"
+
+
+def test_an_incoming_gpu_onnxruntime_replaces_the_cpu_build(tmp_path):
+    installer = load_installer()
+    staging, venv_sp, quarantine = trees(tmp_path)
+    write_dist(
+        venv_sp, "onnxruntime", "1.20.1",
+        {
+            "onnxruntime/__init__.py": "cpu",
+            "onnxruntime/capi/onnxruntime_pybind11_state.so": "cpu",
+        },
+    )
+    write_dist(
+        staging, "onnxruntime_gpu", "1.20.1",
+        {
+            "onnxruntime/__init__.py": "gpu",
+            f"onnxruntime/capi/{CUDA_PROVIDER_LIB}": "cuda",
+            "onnxruntime/capi/onnxruntime_pybind11_state.so": "gpu",
+        },
+    )
+
+    merge(installer, staging, venv_sp, quarantine)
+
+    assert (venv_sp / "onnxruntime" / "capi" / CUDA_PROVIDER_LIB).exists()
+    assert (venv_sp / "onnxruntime" / "capi" / "onnxruntime_pybind11_state.so").read_text() == "gpu"
+    assert dist_infos(venv_sp, "onnxruntime-") == []
+    assert dist_infos(venv_sp, "onnxruntime_gpu-") == ["onnxruntime_gpu-1.20.1.dist-info"]
+
+
+# -- what the uninstall is allowed to touch --
+
+
+def test_console_scripts_outside_site_packages_are_left_alone(tmp_path):
+    """RECORD points at `../../../bin/<name>` for entry points. Those cannot
+    shadow an import, and this installer was handed site-packages, not the venv
+    root. Laid out at the real depth so the escape actually resolves onto the
+    venv's bin directory rather than harmlessly missing it."""
+    installer = load_installer()
+    venv = tmp_path / "venv"
+    venv_sp = venv / "lib" / "python3.12" / "site-packages"
+    staging = tmp_path / "staging"
+    quarantine = venv / ".superseded"
+    venv_sp.mkdir(parents=True)
+    staging.mkdir()
+    bin_dir = venv / "bin"
+    bin_dir.mkdir()
+    (bin_dir / "tqdm").write_text("#!/bin/sh")
+    assert (venv_sp / ".." / ".." / ".." / "bin" / "tqdm").resolve() == (bin_dir / "tqdm").resolve()
+    write_dist(venv_sp, "tqdm", "4.68.3", {"tqdm/__init__.py": "old"},
+               extra_record=["../../../bin/tqdm", "/etc/hosts"])
+    write_dist(staging, "tqdm", "4.69.1", {"tqdm/__init__.py": "new"})
+
+    # Pinned at the source, because whether an escaping path survives a move
+    # depends on how deep the quarantine happens to sit: reading it is the only
+    # place the answer is unambiguous.
+    owned = installer.distribution_files(str(venv_sp), "tqdm-4.68.3.dist-info")
+    assert not [rel for rel in owned if rel.startswith("/") or ".." in rel.split("/")]
+
+    merge(installer, staging, venv_sp, quarantine)
+
+    assert (bin_dir / "tqdm").exists()
+    assert (venv_sp / "tqdm" / "__init__.py").read_text() == "new"
+
+
+def test_a_directory_another_distribution_still_uses_is_not_pruned(tmp_path):
+    """Two distributions can share a namespace directory. Emptying one of them
+    must not take the other's files with it."""
+    installer = load_installer()
+    staging, venv_sp, quarantine = trees(tmp_path)
+    write_dist(venv_sp, "google_api", "1.0.0", {"google/api/__init__.py": "api-old"})
+    write_dist(venv_sp, "google_cloud", "2.0.0", {"google/cloud/__init__.py": "cloud"})
+    write_dist(staging, "google_api", "1.1.0", {"google/api/__init__.py": "api-new"})
+
+    merge(installer, staging, venv_sp, quarantine)
+
+    assert (venv_sp / "google" / "cloud" / "__init__.py").read_text() == "cloud"
+    assert (venv_sp / "google" / "api" / "__init__.py").read_text() == "api-new"
+
+
+def test_uninstalling_one_opencv_flavour_leaves_cv2_for_its_siblings(tmp_path):
+    """The three opencv projects all own `cv2/`, the way onnxruntime's two
+    flavours own `onnxruntime/`.
+
+    Reproduced live: superseding opencv-contrib-python deleted every file its
+    RECORD listed, which is the same `cv2/` that opencv-python and
+    opencv-python-headless were still claiming. The merge only puts back the
+    files of the distribution it is placing, so the venv lost cv2 outright and
+    rembg, mediapipe, basicsr, realesrgan and gfpgan all stopped importing.
+    """
+    installer = load_installer()
+    staging, venv_sp, quarantine = trees(tmp_path)
+    shared = {"cv2/__init__.py": "shared", "cv2/cv2.abi3.so": "shared-binary"}
+    write_dist(venv_sp, "opencv_python_headless", "4.10.0.84", dict(shared))
+    write_dist(venv_sp, "opencv_python", "4.11.0.86", dict(shared))
+    write_dist(venv_sp, "opencv_contrib_python", "4.11.0.86", dict(shared))
+    write_dist(staging, "opencv_contrib_python", "4.13.0.92",
+               {"cv2/__init__.py": "contrib-new", "cv2/cv2.abi3.so": "contrib-binary"})
+
+    merge(installer, staging, venv_sp, quarantine)
+
+    assert (venv_sp / "cv2" / "__init__.py").read_text() == "contrib-new"
+    assert (venv_sp / "cv2" / "cv2.abi3.so").read_text() == "contrib-binary"
+    assert dist_infos(venv_sp, "opencv_contrib_python-") == ["opencv_contrib_python-4.13.0.92.dist-info"]
+    assert dist_infos(venv_sp, "opencv_python-") == ["opencv_python-4.11.0.86.dist-info"]
+
+
+def test_a_refused_install_does_not_take_a_siblings_shared_import_with_it(tmp_path):
+    """The rollback has the same hazard from the other side: removing what the
+    bundle placed must not remove files another installed distribution owns."""
+    installer = load_installer()
+    staging, venv_sp, quarantine = trees(tmp_path)
+    write_dist(venv_sp, "opencv_python_headless", "4.10.0.84",
+               {"cv2/__init__.py": "headless", "cv2/cv2.abi3.so": "headless-binary"})
+    write_dist(staging, "opencv_contrib_python", "4.13.0.92",
+               {"cv2/__init__.py": "contrib", "cv2/cv2.abi3.so": "contrib-binary"})
+
+    plan = merge(installer, staging, venv_sp, quarantine)
+    installer.rollback_reconciliation(str(venv_sp), plan, str(quarantine))
+
+    assert (venv_sp / "cv2" / "__init__.py").exists()
+    assert dist_infos(venv_sp, "opencv_python_headless-") == ["opencv_python_headless-4.10.0.84.dist-info"]
+    assert dist_infos(venv_sp, "opencv_contrib_python-") == []
+
+
+def test_files_the_incoming_copy_does_not_carry_are_left_where_they_are(tmp_path):
+    """Bundle archives are not always complete wheels.
+
+    Reproduced live: upscale-enhance carries setuptools 74.1.3 as `setuptools/`
+    plus its dist-info, and nothing else. The version it replaced also owned
+    `_distutils_hack/` and `distutils-precedence.pth`, the shim that gives
+    Python 3.12 a `distutils`. Removing everything the old RECORD listed deleted
+    the shim, nothing wrote it back, and basicsr, realesrgan and gfpgan all
+    stopped importing on a venv where every bundle reported installed.
+    """
+    installer = load_installer()
+    staging, venv_sp, quarantine = trees(tmp_path)
+    write_dist(
+        venv_sp, "setuptools", "78.1.1",
+        {
+            "setuptools/__init__.py": "78",
+            "setuptools/_vendor/old.py": "78-vendored",
+            "_distutils_hack/__init__.py": "shim",
+            "distutils-precedence.pth": "import _distutils_hack",
+        },
+    )
+    # The archive's RECORD is the wheel's, and claims files the archive does not
+    # actually carry. That is what made the first attempt at this guard fail on
+    # the real node: reading RECORD alone still cleared the shim.
+    write_dist(
+        staging, "setuptools", "74.1.3", {"setuptools/__init__.py": "74"},
+        extra_record=["_distutils_hack/__init__.py", "distutils-precedence.pth"],
+    )
+
+    merge(installer, staging, venv_sp, quarantine)
+
+    assert (venv_sp / "_distutils_hack" / "__init__.py").read_text() == "shim"
+    assert (venv_sp / "distutils-precedence.pth").read_text() == "import _distutils_hack"
+    # The part the incoming copy does cover is fully replaced, stale files and all.
+    assert (venv_sp / "setuptools" / "__init__.py").read_text() == "74"
+    assert not (venv_sp / "setuptools" / "_vendor" / "old.py").exists()
+    assert dist_infos(venv_sp, "setuptools-") == ["setuptools-74.1.3.dist-info"]
+
+
+def test_stale_bytecode_of_a_removed_module_is_discarded(tmp_path):
+    """A `.pyc` orphaned by the uninstall keeps the package directory alive and
+    would block the incoming version's directory from landing cleanly."""
+    installer = load_installer()
+    staging, venv_sp, quarantine = trees(tmp_path)
+    write_dist(venv_sp, "numba", "0.65.1", {"numba/dropped.py": "old"})
+    cache = venv_sp / "numba" / "__pycache__"
+    cache.mkdir(parents=True)
+    (cache / "dropped.cpython-312.pyc").write_bytes(b"stale")
+    write_dist(staging, "numba", "0.66.0", {"numba/kept.py": "new"})
+
+    merge(installer, staging, venv_sp, quarantine)
+
+    assert not (venv_sp / "numba" / "dropped.py").exists()
+    assert not (cache / "dropped.cpython-312.pyc").exists()
+    assert (venv_sp / "numba" / "kept.py").read_text() == "new"
+
+
+def test_a_distribution_with_no_record_still_loses_its_metadata(tmp_path):
+    """Without a RECORD the file list is unknowable, but leaving the old
+    dist-info behind would report two versions installed. Drop what we can."""
+    installer = load_installer()
+    staging, venv_sp, quarantine = trees(tmp_path)
+    write_dist(venv_sp, "click", "8.4.1", {"click/__init__.py": "old"})
+    (venv_sp / "click-8.4.1.dist-info" / "RECORD").unlink()
+    write_dist(staging, "click", "8.4.2", {"click/__init__.py": "new"})
+
+    merge(installer, staging, venv_sp, quarantine)
+
+    assert dist_infos(venv_sp, "click-") == ["click-8.4.2.dist-info"]
+    assert (venv_sp / "click" / "__init__.py").read_text() == "new"
+
+
+def test_name_spelling_does_not_hide_a_collision(tmp_path):
+    """`hf_xet` and `hf-xet` are the same project. Comparing raw directory names
+    would miss the collision and leave both versions installed."""
+    installer = load_installer()
+    staging, venv_sp, _q = trees(tmp_path)
+    write_dist(venv_sp, "hf_xet", "1.5.1", {"hf_xet/__init__.py": "old"})
+    write_dist(staging, "hf-xet", "1.5.2", {"hf_xet/__init__.py": "new"})
+
+    plan = installer.plan_reconciliation(str(staging), str(venv_sp))
+
+    assert [item["name"] for item in plan] == ["hf-xet"]
+    assert plan[0]["superseded"] == [("1.5.1", "hf_xet-1.5.1.dist-info")]
+
+
+# -- rollback --
+
+
+def test_a_refused_install_puts_the_displaced_version_back(tmp_path):
+    """The whole reason superseded files are quarantined rather than deleted: a
+    bundle that fails verification must not keep the versions it took from the
+    bundles that were already working."""
+    installer = load_installer()
+    staging, venv_sp, quarantine = trees(tmp_path)
+    write_dist(
+        venv_sp, "tokenizers", "0.20.3",
+        {"tokenizers/__init__.py": "0.20.3", "tokenizers/native.so": "old-binary"},
+    )
+    write_dist(venv_sp, "transformers", "4.46.3", {"transformers/__init__.py": "keep"})
+    write_dist(
+        staging, "tokenizers", "0.23.1",
+        {"tokenizers/__init__.py": "0.23.1", "tokenizers/other.so": "new-binary"},
+    )
+    write_dist(staging, "faster_whisper", "1.2.1", {"faster_whisper/__init__.py": "new"})
+
+    plan = merge(installer, staging, venv_sp, quarantine)
+    installer.rollback_reconciliation(str(venv_sp), plan, str(quarantine))
+
+    assert dist_infos(venv_sp, "tokenizers-") == ["tokenizers-0.20.3.dist-info"]
+    assert (venv_sp / "tokenizers" / "__init__.py").read_text() == "0.20.3"
+    assert (venv_sp / "tokenizers" / "native.so").read_text() == "old-binary"
+    assert not (venv_sp / "tokenizers" / "other.so").exists()
+    # The bundle's own new distribution goes away with it.
+    assert not (venv_sp / "faster_whisper").exists()
+    assert dist_infos(venv_sp, "faster_whisper-") == []
+    # An unrelated distribution is untouched either way.
+    assert (venv_sp / "transformers" / "__init__.py").read_text() == "keep"
+    assert not os.path.exists(quarantine)
+
+
+def test_a_rollback_leaves_an_unchanged_distribution_alone(tmp_path):
+    """A distribution already at the staged version never enters the plan, so a
+    rollback must not delete files that were correct before this install."""
+    installer = load_installer()
+    staging, venv_sp, quarantine = trees(tmp_path)
+    write_dist(venv_sp, "certifi", "2026.7.22", {"certifi/cacert.pem": "bundle"})
+    write_dist(staging, "certifi", "2026.7.22", {"certifi/cacert.pem": "bundle"})
+    write_dist(staging, "rembg", "2.0.62", {"rembg/__init__.py": "new"})
+
+    plan = merge(installer, staging, venv_sp, quarantine)
+    installer.rollback_reconciliation(str(venv_sp), plan, str(quarantine))
+
+    assert (venv_sp / "certifi" / "cacert.pem").read_text() == "bundle"
+    assert dist_infos(venv_sp, "certifi-") == ["certifi-2026.7.22.dist-info"]
+    assert not (venv_sp / "rembg").exists()
+
+
+def test_rollback_without_a_quarantine_is_harmless(tmp_path):
+    """Nothing collided, so there is nothing to restore and the venv is left as
+    it was before the bundle landed."""
+    installer = load_installer()
+    staging, venv_sp, quarantine = trees(tmp_path)
+    write_dist(venv_sp, "numpy", "1.26.4", {"numpy/__init__.py": "base"})
+    write_dist(staging, "mediapipe", "0.10.35", {"mediapipe/__init__.py": "new"})
+
+    plan = merge(installer, staging, venv_sp, quarantine)
+    installer.rollback_reconciliation(str(venv_sp), plan, str(quarantine))
+
+    assert (venv_sp / "numpy" / "__init__.py").read_text() == "base"
+    assert sorted(os.listdir(venv_sp)) == ["numpy", "numpy-1.26.4.dist-info"]

packages/ai/python/tests/test_install_feature_version_conflict.py

@@ -0,0 +1,180 @@
+"""Regression reproduction for the shared-venv version conflict (AI-20260726-001).
+
+`reconcile_onnxruntime` fixed exactly one package pair. Every OTHER distribution
+that two bundles both provided was merged by `move_tree` file-by-file, with no
+uninstall of the version already in the venv, so two versions ended up overlaid:
+the pure-Python modules came from whichever bundle wrote last while the compiled
+extension CPython actually imports could come from the other.
+
+Observed live on ubuntu-gpu-amd64 after Settings > AI Features > Install All:
+seventeen distributions carried two or three versions at once. Three of them
+broke a tool.
+
+  tokenizers 0.20.3 (inpaint-hq, pinned by transformers 4.46.3) and 0.23.1
+  (transcription, needed by faster-whisper) both present. The package directory
+  held tokenizers.cpython-312-x86_64-linux-gnu.so from 0.20.3 next to
+  tokenizers.abi3.so from 0.23.1; CPython prefers the platform-specific suffix,
+  so 0.23.1's `decoders/__init__.py` ran against 0.20.3's binary and raised
+  "AttributeError: module 'decoders' has no attribute 'DecodeStream'". The
+  transcription install failed its own smoke gate and could not be installed at
+  all on that host.
+
+  scipy 1.12.0 and 1.17.1 -> Real-ESRGAN cannot import, `upscale` fails.
+  huggingface_hub 0.36.2, 1.19.0 and 1.22.0 -> transformers refuses to import,
+  the high-quality Object Eraser path fails.
+
+The invariant is deliberately modest and does not presume a particular winner:
+after a bundle's site-packages has been merged into the venv, exactly ONE
+version of any given distribution remains, and no file belonging to the
+superseded version survives to shadow it.
+"""
+
+import importlib.util
+import os
+
+import pytest
+
+CASES = {
+    # distribution, older version, newer version, older extension, newer extension
+    "tokenizers": ("0.20.3", "0.23.1", "tokenizers.cpython-312-x86_64-linux-gnu.so", "tokenizers.abi3.so"),
+    "scipy": ("1.12.0", "1.17.1", "_lib/_ccallback_c.cpython-312-x86_64-linux-gnu.so", "_lib/_ccallback_c.abi3.so"),
+    "huggingface_hub": ("0.36.2", "1.22.0", None, None),
+}
+
+
+def load_installer():
+    script_path = os.path.join(os.path.dirname(__file__), "..", "install_feature.py")
+    spec = importlib.util.spec_from_file_location("install_feature_conflict_under_test", script_path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def make_distribution(site_packages, package, version, extension):
+    """Lay out one version of the package, the way a bundle archive carries it."""
+    package_dir = site_packages / package
+    (package_dir / "decoders").mkdir(parents=True, exist_ok=True)
+    (package_dir / "__init__.py").write_text(f"__version__ = {version!r}\n")
+    (package_dir / "decoders" / "__init__.py").write_text(f"# decoders for {version}\n")
+    owned = [
+        f"{package}/__init__.py",
+        f"{package}/decoders/__init__.py",
+    ]
+    if extension:
+        target = package_dir / extension
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(f"native {version}")
+        owned.append(f"{package}/{extension}")
+    dist_info = site_packages / f"{package}-{version}.dist-info"
+    dist_info.mkdir(parents=True, exist_ok=True)
+    (dist_info / "METADATA").write_text(f"Name: {package}\nVersion: {version}\n")
+    owned += [
+        f"{package}-{version}.dist-info/METADATA",
+        f"{package}-{version}.dist-info/RECORD",
+    ]
+    (dist_info / "RECORD").write_text("\n".join(f"{path},," for path in owned) + "\n")
+
+
+def merge_bundle(tmp_path, package, installed_version, incoming_version,
+                 installed_extension, incoming_extension):
+    """Run the installer's real merge sequence for a second bundle."""
+    installer = load_installer()
+    site_packages = tmp_path / "venv-site-packages"
+    staging = tmp_path / "staging-site-packages"
+    quarantine = tmp_path / "quarantine"
+    site_packages.mkdir()
+    staging.mkdir()
+
+    make_distribution(site_packages, package, installed_version, installed_extension)
+    make_distribution(staging, package, incoming_version, incoming_extension)
+
+    installer.reconcile_onnxruntime(str(staging), str(site_packages))
+    plan = installer.plan_reconciliation(str(staging), str(site_packages))
+    installer.supersede_distributions(str(site_packages), plan, str(quarantine))
+    installer.move_tree(str(staging), str(site_packages))
+    return site_packages
+
+
+def dist_info_versions(site_packages, package):
+    return sorted(
+        entry.name.removesuffix(".dist-info").split("-", 1)[1]
+        for entry in site_packages.iterdir()
+        if entry.name.startswith(f"{package}-") and entry.name.endswith(".dist-info")
+    )
+
+
+@pytest.mark.parametrize("package", sorted(CASES))
+@pytest.mark.parametrize("upgrade", [True, False], ids=["upgrade", "downgrade"])
+def test_merging_a_second_bundle_leaves_one_distribution_version(tmp_path, package, upgrade):
+    """Whichever direction the second bundle moves the version, only one remains.
+
+    Both directions matter: Install All has no fixed order, and the field
+    failure needed only that two bundles disagreed, not that the newer one
+    landed second.
+    """
+    older, newer, older_ext, newer_ext = CASES[package]
+    installed, incoming = (older, newer) if upgrade else (newer, older)
+    installed_ext, incoming_ext = (older_ext, newer_ext) if upgrade else (newer_ext, older_ext)
+
+    site_packages = merge_bundle(
+        tmp_path, package, installed, incoming, installed_ext, incoming_ext
+    )
+
+    assert dist_info_versions(site_packages, package) == [incoming]
+    assert incoming in (site_packages / package / "__init__.py").read_text()
+
+
+@pytest.mark.parametrize("package", ["tokenizers", "scipy"])
+def test_merging_a_second_bundle_removes_the_superseded_extension(tmp_path, package):
+    """The exact tokenizers mechanism: CPython prefers the platform-specific
+    suffix, so a surviving `.cpython-312-*.so` from the old version would be
+    loaded under the new version's Python modules."""
+    older, newer, older_ext, newer_ext = CASES[package]
+
+    site_packages = merge_bundle(tmp_path, package, older, newer, older_ext, newer_ext)
+
+    package_dir = site_packages / package
+    assert (package_dir / newer_ext).exists()
+    assert not (package_dir / older_ext).exists()
+
+
+def test_a_reinstall_of_the_same_version_is_left_alone(tmp_path):
+    """Placing what is already there must not churn the venv: nothing is
+    superseded, so nothing is removed and nothing can be torn."""
+    installer = load_installer()
+    site_packages = tmp_path / "venv-site-packages"
+    staging = tmp_path / "staging-site-packages"
+    site_packages.mkdir()
+    staging.mkdir()
+    make_distribution(site_packages, "tokenizers", "0.20.3", None)
+    make_distribution(staging, "tokenizers", "0.20.3", None)
+
+    plan = installer.plan_reconciliation(str(staging), str(site_packages))
+
+    assert plan == []
+
+
+def test_reinstalling_a_bundle_repairs_a_venv_already_carrying_both_versions(tmp_path):
+    """Self-heal for hosts that ran the broken installer.
+
+    Someone who already clicked Install All has a venv with both versions
+    overlaid. Reinstalling any bundle that provides the distribution has to
+    clear the other version rather than add a third.
+    """
+    installer = load_installer()
+    site_packages = tmp_path / "venv-site-packages"
+    staging = tmp_path / "staging-site-packages"
+    quarantine = tmp_path / "quarantine"
+    site_packages.mkdir()
+    staging.mkdir()
+    make_distribution(site_packages, "tokenizers", "0.20.3", "tokenizers.cpython-312-x86_64-linux-gnu.so")
+    make_distribution(site_packages, "tokenizers", "0.23.1", "tokenizers.abi3.so")
+    make_distribution(staging, "tokenizers", "0.23.1", "tokenizers.abi3.so")
+
+    plan = installer.plan_reconciliation(str(staging), str(site_packages))
+    installer.supersede_distributions(str(site_packages), plan, str(quarantine))
+    installer.move_tree(str(staging), str(site_packages))
+
+    assert dist_info_versions(site_packages, "tokenizers") == ["0.23.1"]
+    assert not (site_packages / "tokenizers" / "tokenizers.cpython-312-x86_64-linux-gnu.so").exists()

packages/ai/python/tests/test_offline_guard.py

@@ -13,19 +13,19 @@ import offline_guard  # noqa: E402
 # --- downloads_allowed ------------------------------------------------------
 
 
-def test_downloads_allowed_default_true(monkeypatch):
+def test_downloads_blocked_by_default(monkeypatch):
     monkeypatch.delenv("SNAPOTTER_ALLOW_MODEL_DOWNLOAD", raising=False)
-    assert offline_guard.downloads_allowed() is True
+    assert offline_guard.downloads_allowed() is False
 
 
-@pytest.mark.parametrize("value", ["0", "false", "FALSE", "False"])
-def test_downloads_blocked_by_explicit_off(monkeypatch, value):
+@pytest.mark.parametrize("value", ["0", "false", "FALSE", "False", "yes", "anything"])
+def test_downloads_blocked_without_explicit_opt_in(monkeypatch, value):
     monkeypatch.setenv("SNAPOTTER_ALLOW_MODEL_DOWNLOAD", value)
     assert offline_guard.downloads_allowed() is False
 
 
-@pytest.mark.parametrize("value", ["1", "true", "yes", "anything"])
-def test_downloads_allowed_for_non_off_values(monkeypatch, value):
+@pytest.mark.parametrize("value", ["1", "true", "TRUE", "True"])
+def test_downloads_allowed_for_explicit_opt_in(monkeypatch, value):
     monkeypatch.setenv("SNAPOTTER_ALLOW_MODEL_DOWNLOAD", value)
     assert offline_guard.downloads_allowed() is True
 
@@ -33,11 +33,18 @@ def test_downloads_allowed_for_non_off_values(monkeypatch, value):
 # --- ensure_download_allowed ------------------------------------------------
 
 
-def test_ensure_noop_when_allowed(monkeypatch):
-    monkeypatch.delenv("SNAPOTTER_ALLOW_MODEL_DOWNLOAD", raising=False)
+def test_ensure_noop_when_explicitly_allowed(monkeypatch):
+    monkeypatch.setenv("SNAPOTTER_ALLOW_MODEL_DOWNLOAD", "1")
     assert offline_guard.ensure_download_allowed("thing") is None
 
 
+def test_ensure_raises_actionable_error_by_default(monkeypatch):
+    monkeypatch.delenv("SNAPOTTER_ALLOW_MODEL_DOWNLOAD", raising=False)
+    with pytest.raises(RuntimeError) as exc:
+        offline_guard.ensure_download_allowed("MyModel weight")
+    assert "SNAPOTTER_ALLOW_MODEL_DOWNLOAD=1" in str(exc.value)
+
+
 def test_ensure_raises_actionable_error_when_blocked(monkeypatch):
     monkeypatch.setenv("SNAPOTTER_ALLOW_MODEL_DOWNLOAD", "0")
     with pytest.raises(RuntimeError) as exc:

packages/ai/python/tests/test_upscale_failure_message.py

@@ -0,0 +1,66 @@
+"""What Real-ESRGAN tells the user when it cannot run (AI-20260726-002).
+
+Observed on ubuntu-gpu-amd64 with `upscale-enhance` installed and working
+weights on disk: scipy was carrying 1.12.0 and 1.17.1 at once, so the import
+chain raised
+
+    cannot import name '_promote' from 'scipy.spatial.transform._rotation'
+
+and the tool answered "Install the upscale-enhance feature", which the user had
+already done. The advice has to follow the shape of the failure, not the fact
+that one occurred.
+"""
+
+import importlib.util
+import os
+
+
+def load_upscale():
+    script_path = os.path.join(os.path.dirname(__file__), "..", "upscale.py")
+    spec = importlib.util.spec_from_file_location("upscale_message_under_test", script_path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_a_broken_dependency_does_not_ask_for_an_install_that_already_happened():
+    upscale = load_upscale()
+    failure = ImportError(
+        "cannot import name '_promote' from 'scipy.spatial.transform._rotation'"
+    )
+
+    message = upscale.realesrgan_failure_message(failure)
+
+    assert "_promote" in message
+    assert "Install the upscale-enhance feature" not in message
+    assert "Reinstall" in message
+    assert "model=lanczos" in message
+
+
+def test_a_missing_module_still_asks_for_the_install():
+    upscale = load_upscale()
+
+    message = upscale.realesrgan_failure_message(ModuleNotFoundError("No module named 'realesrgan'"))
+
+    assert "Install the upscale-enhance feature" in message
+
+
+def test_missing_weights_still_ask_for_the_install():
+    """The bundle carries the .pth files, so absent weights mean absent bundle."""
+    upscale = load_upscale()
+
+    message = upscale.realesrgan_failure_message(
+        FileNotFoundError("RealESRGAN model not found: /data/ai/models/RealESRGAN_x4plus.pth")
+    )
+
+    assert "Install the upscale-enhance feature" in message
+
+
+def test_a_runtime_failure_points_at_a_repair_rather_than_an_install():
+    upscale = load_upscale()
+
+    message = upscale.realesrgan_failure_message(RuntimeError("CUDA error: no kernel image"))
+
+    assert "Install the upscale-enhance feature" not in message
+    assert "Reset AI Environment" in message