mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
fix: release QA hardening across processing, media, security, and CI gates (#649)
A release-readiness QA pass over the whole product. The commits split into defects a user would hit and gates that were reporting green while measuring nothing. ## Fixes that change behaviour Rate limiting was bypassable on every install: TRUST_PROXY defaulted to true, so request.ip came from a client-set header and a forged X-Forwarded-For got past the login limiter. The default is now a private-network trust list. A transient Postgres outage stranded in-flight jobs, leaving finished output on disk with no row pointing at it. A reconciler now resolves those rows and adopts the bytes rather than dropping the work. A Redis connection that moved to a new address wedged every read-blocked consumer, so completions stopped signalling while health still answered 200. Socket timeouts plus subscriber pings recover it. Installing more than one AI bundle left the shared venv multi-versioned and silently broke three tools. The installer now reconciles distributions to one version each. Converting an image to JXL at quality 1 through 4 returned a 500, because libjxl 0.7 rejects the distance those values compute. The quality is floored at what the encoder honours. A missing ffmpeg was also reported to the user as a corrupt upload; it now says the engine is unavailable. RAW uploads reached an unpatched LibRaw on arm64, so it is built from source at 0.22.2, and the release scan was split so it can fail on an unfixed critical instead of hiding it behind ignore-unfixed. ## Gates that could not fail Two mutation lanes ran zero mutants because Stryker crawled the gitignored docs build; coverage discarded its whole report on any failing test; the lint gate skipped root tests, scripts, and two workspaces; and several generated matrices counted a host missing ffmpeg as a passing tool. Each now measures what it claims. Full evidence and the outstanding release items are tracked locally and are not part of this branch.
This commit is contained in:
SnapOtter
@@ -12,6 +12,7 @@ Progress is reported via JSON lines on stderr (parsed by the Node bridge).
|
||||
Final result is a JSON object on stdout.
|
||||
"""
|
||||
|
||||
import csv
|
||||
import errno
|
||||
import glob
|
||||
import hashlib
|
||||
@@ -19,6 +20,7 @@ import importlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -604,6 +606,320 @@ def reconcile_onnxruntime(staging_sp: str, site_packages_dir: str) -> None:
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
# -- Distribution reconciliation (one version per distribution) --
|
||||
|
||||
DIST_INFO_SUFFIX = ".dist-info"
|
||||
|
||||
|
||||
def canonical_dist_name(name: str) -> str:
|
||||
"""Normalize a distribution name the way PEP 503 does, so the same project
|
||||
compares equal however its wheel spelled it (`hf-xet`, `hf_xet`, `HF.Xet`)."""
|
||||
return re.sub(r"[-_.]+", "-", name).lower()
|
||||
|
||||
|
||||
def list_distributions(sp_dir: str) -> dict:
|
||||
"""Map canonical distribution name -> [(version, dist-info dir name), ...].
|
||||
|
||||
A healthy site-packages holds exactly one entry per name. A venv merged by
|
||||
an installer that never uninstalls can hold several, which is the corruption
|
||||
this section exists to prevent: `move_tree` copies file over file, so the
|
||||
two versions overlay and CPython can end up loading one version's compiled
|
||||
extension underneath the other's Python modules.
|
||||
"""
|
||||
found = {}
|
||||
if not os.path.isdir(sp_dir):
|
||||
return found
|
||||
for entry in os.listdir(sp_dir):
|
||||
if not entry.endswith(DIST_INFO_SUFFIX):
|
||||
continue
|
||||
name, sep, version = entry[: -len(DIST_INFO_SUFFIX)].rpartition("-")
|
||||
if not sep or not name:
|
||||
continue
|
||||
found.setdefault(canonical_dist_name(name), []).append((version, entry))
|
||||
return found
|
||||
|
||||
|
||||
def distribution_files(sp_dir: str, dist_info: str) -> list:
|
||||
"""Relative paths a distribution owns, read from its RECORD.
|
||||
|
||||
Entries that point outside site-packages (console scripts are recorded as
|
||||
`../../bin/<name>`) are skipped: they cannot shadow an import, and deleting
|
||||
outside the directory we were handed is not this function's business.
|
||||
"""
|
||||
record = os.path.join(sp_dir, dist_info, "RECORD")
|
||||
if not os.path.exists(record):
|
||||
return []
|
||||
paths = []
|
||||
with open(record, newline="", errors="replace") as f:
|
||||
for row in csv.reader(f):
|
||||
if not row:
|
||||
continue
|
||||
rel = row[0].strip().replace("\\", "/")
|
||||
if not rel or rel.startswith("/") or ".." in rel.split("/"):
|
||||
continue
|
||||
paths.append(rel)
|
||||
return paths
|
||||
|
||||
|
||||
def paths_claimed_by_others(sp_dir: str, exclude: set) -> set:
|
||||
"""Every file some OTHER installed distribution says it owns.
|
||||
|
||||
Distributions are supposed to own disjoint files, and almost all of them do.
|
||||
The exceptions are the ones that package the same import under different
|
||||
project names: `opencv-python`, `opencv-python-headless` and
|
||||
`opencv-contrib-python` all write `cv2/`, exactly as `onnxruntime` and
|
||||
`onnxruntime-gpu` both write `onnxruntime/`. Uninstalling one of those by its
|
||||
RECORD would take the shared import away from the siblings that are still
|
||||
installed, and the merge only puts back the files of the distribution it is
|
||||
placing, so `cv2` would simply vanish.
|
||||
|
||||
Shared files are therefore left where they are and let the merge overwrite
|
||||
them, which is what pip does with these packages too. The version metadata
|
||||
still ends up single-valued, which is the invariant that matters.
|
||||
"""
|
||||
claimed = set()
|
||||
if not os.path.isdir(sp_dir):
|
||||
return claimed
|
||||
for entry in os.listdir(sp_dir):
|
||||
if entry.endswith(DIST_INFO_SUFFIX) and entry not in exclude:
|
||||
claimed.update(distribution_files(sp_dir, entry))
|
||||
return claimed
|
||||
|
||||
|
||||
def _is_inside(root: str, path: str) -> bool:
|
||||
return not os.path.relpath(os.path.abspath(path), os.path.abspath(root)).startswith("..")
|
||||
|
||||
|
||||
def _relocate(src_root: str, dst_root: str, rel_path: str) -> bool:
|
||||
"""Move one relative entry between two trees, keeping its relative position."""
|
||||
src = os.path.join(src_root, rel_path)
|
||||
if not os.path.exists(src) and not os.path.islink(src):
|
||||
return False
|
||||
dst = os.path.join(dst_root, rel_path)
|
||||
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
||||
try:
|
||||
os.replace(src, dst)
|
||||
except OSError as e:
|
||||
if getattr(e, "errno", None) != errno.EXDEV:
|
||||
raise
|
||||
shutil.move(src, dst)
|
||||
return True
|
||||
|
||||
|
||||
def _prune_empty_parents(root: str, rel_path: str) -> None:
|
||||
"""Drop directories emptied by a removal, stopping at root and at the first
|
||||
directory that still holds something (another distribution's files)."""
|
||||
directory = os.path.dirname(os.path.join(root, rel_path))
|
||||
while _is_inside(root, directory) and os.path.abspath(directory) != os.path.abspath(root):
|
||||
try:
|
||||
os.rmdir(directory)
|
||||
except OSError:
|
||||
return
|
||||
directory = os.path.dirname(directory)
|
||||
|
||||
|
||||
def _discard_stale_bytecode(root: str, rel_path: str) -> None:
|
||||
"""Remove the cached bytecode of a source file that just left the tree.
|
||||
|
||||
A `.pyc` whose `.py` is gone is not importable, but it does keep the package
|
||||
directory non-empty, which would stop `_prune_empty_parents` from clearing
|
||||
the way for the incoming version.
|
||||
"""
|
||||
if not rel_path.endswith(".py"):
|
||||
return
|
||||
directory, base = os.path.split(os.path.join(root, rel_path))
|
||||
cache = os.path.join(directory, "__pycache__")
|
||||
for stale in glob.glob(os.path.join(cache, base[:-3] + ".*.pyc")):
|
||||
try:
|
||||
os.unlink(stale)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
os.rmdir(cache)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def plan_reconciliation(staging_sp: str, sp_dir: str) -> list:
|
||||
"""Work out, per distribution the bundle is about to place, what it displaces.
|
||||
|
||||
One entry per staged distribution that will actually change the venv:
|
||||
|
||||
{"name", "version", "dist_info", "files", "superseded": [(version, dir)]}
|
||||
|
||||
`superseded` holds every copy already installed under a different version.
|
||||
A distribution already present at the staged version is left out entirely:
|
||||
placing it is a no-op, so there is nothing to remove and nothing to undo.
|
||||
"""
|
||||
staged = list_distributions(staging_sp)
|
||||
installed = list_distributions(sp_dir)
|
||||
plan = []
|
||||
for name in sorted(staged):
|
||||
version, dist_info = sorted(staged[name])[-1]
|
||||
existing = installed.get(name, [])
|
||||
superseded = [copy for copy in existing if copy[0] != version]
|
||||
if existing and not superseded:
|
||||
continue
|
||||
# RECORD describes the wheel, not the archive. A bundle's copy of
|
||||
# setuptools lists `_distutils_hack/` and `distutils-precedence.pth`
|
||||
# while carrying neither, so trusting RECORD alone would clear ground
|
||||
# that nothing is going to cover. Keep only what is really staged.
|
||||
files = [
|
||||
rel
|
||||
for rel in distribution_files(staging_sp, dist_info)
|
||||
if os.path.exists(os.path.join(staging_sp, rel))
|
||||
]
|
||||
plan.append(
|
||||
{
|
||||
"name": name,
|
||||
"version": version,
|
||||
"dist_info": dist_info,
|
||||
"files": files,
|
||||
"replaces": {rel.split("/")[0] for rel in files},
|
||||
"superseded": superseded,
|
||||
}
|
||||
)
|
||||
return plan
|
||||
|
||||
|
||||
def supersede_distributions(sp_dir: str, plan: list, quarantine_dir: str) -> int:
|
||||
"""Uninstall every superseded version, holding its files for a rollback.
|
||||
|
||||
Files move to `quarantine_dir/<name>/<version>/` rather than being deleted
|
||||
outright, so a failed verification can put the venv back exactly as it was
|
||||
instead of leaving it half-way between two bundles.
|
||||
"""
|
||||
removed = 0
|
||||
superseded_dist_infos = {info for item in plan for _v, info in item["superseded"]}
|
||||
shared = paths_claimed_by_others(sp_dir, superseded_dist_infos)
|
||||
for item in plan:
|
||||
for version, dist_info in item["superseded"]:
|
||||
destination = os.path.join(quarantine_dir, item["name"], version)
|
||||
# Only clear ground the incoming copy is going to cover. A bundle
|
||||
# archive is not always a complete wheel: upscale-enhance carries
|
||||
# setuptools 74.1.3 as `setuptools/` and its dist-info, without the
|
||||
# `_distutils_hack/` and `distutils-precedence.pth` that the version
|
||||
# it replaces owns. Removing everything the old RECORD listed took
|
||||
# the distutils shim with it and basicsr, realesrgan and gfpgan all
|
||||
# stopped importing, because nothing was going to write those files
|
||||
# back. Anything the incoming copy does not carry stays put.
|
||||
replaces = item["replaces"] | {dist_info}
|
||||
for rel in distribution_files(sp_dir, dist_info):
|
||||
if rel in shared or rel.split("/")[0] not in replaces:
|
||||
continue
|
||||
if _relocate(sp_dir, destination, rel):
|
||||
_discard_stale_bytecode(sp_dir, rel)
|
||||
_prune_empty_parents(sp_dir, rel)
|
||||
# RECORD does not always list every file pip wrote into the metadata
|
||||
# directory (INSTALLER and REQUESTED are frequently absent), so sweep
|
||||
# whatever is left before dropping the directory itself.
|
||||
leftover = os.path.join(sp_dir, dist_info)
|
||||
if os.path.isdir(leftover):
|
||||
for root, _dirs, files in os.walk(leftover):
|
||||
for name in files:
|
||||
rel = os.path.relpath(os.path.join(root, name), sp_dir)
|
||||
_relocate(sp_dir, destination, rel)
|
||||
shutil.rmtree(leftover, ignore_errors=True)
|
||||
removed += 1
|
||||
sys.stderr.write(
|
||||
f"[install] replacing {item['name']} {version} with {item['version']}: "
|
||||
f"removed the superseded version so only one stays in the venv\n"
|
||||
)
|
||||
if removed:
|
||||
sys.stderr.flush()
|
||||
return removed
|
||||
|
||||
|
||||
def restore_superseded(sp_dir: str, quarantine_dir: str) -> None:
|
||||
"""Put every quarantined version back where it came from."""
|
||||
if not os.path.isdir(quarantine_dir):
|
||||
return
|
||||
for name in os.listdir(quarantine_dir):
|
||||
holder = os.path.join(quarantine_dir, name)
|
||||
if not os.path.isdir(holder):
|
||||
continue
|
||||
for version in os.listdir(holder):
|
||||
version_root = os.path.join(holder, version)
|
||||
if not os.path.isdir(version_root):
|
||||
continue
|
||||
for root, _dirs, files in os.walk(version_root):
|
||||
for entry in files:
|
||||
rel = os.path.relpath(os.path.join(root, entry), version_root)
|
||||
_relocate(version_root, sp_dir, rel)
|
||||
shutil.rmtree(quarantine_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def discard_placed_distributions(sp_dir: str, plan: list) -> None:
|
||||
"""Remove the files this bundle placed, for the distributions it changed.
|
||||
|
||||
Only distributions the plan actually touched are removed. One already
|
||||
present at the staged version never entered the plan, so a rollback cannot
|
||||
delete a version that was there before this install and is still correct.
|
||||
"""
|
||||
placed_dist_infos = {item["dist_info"] for item in plan}
|
||||
shared = paths_claimed_by_others(sp_dir, placed_dist_infos)
|
||||
for item in plan:
|
||||
for rel in item["files"]:
|
||||
if rel in shared:
|
||||
continue
|
||||
target = os.path.join(sp_dir, rel)
|
||||
try:
|
||||
if os.path.isdir(target) and not os.path.islink(target):
|
||||
shutil.rmtree(target, ignore_errors=True)
|
||||
else:
|
||||
os.unlink(target)
|
||||
except OSError:
|
||||
pass
|
||||
_prune_empty_parents(sp_dir, rel)
|
||||
|
||||
|
||||
def rollback_reconciliation(sp_dir: str, plan: list, quarantine_dir: str) -> None:
|
||||
"""Undo a merge: drop what this bundle placed, restore what it displaced."""
|
||||
discard_placed_distributions(sp_dir, plan)
|
||||
restore_superseded(sp_dir, quarantine_dir)
|
||||
|
||||
|
||||
def mark_venv_writing(marker_path: str, bundle_id: str) -> None:
|
||||
"""Breadcrumb a destructive write to the shared venv.
|
||||
|
||||
If the process dies while the marker exists, recoverInterruptedInstalls
|
||||
reseeds the venv from the image base on next boot. That is the blunt
|
||||
fallback for a tear this installer could not undo itself.
|
||||
"""
|
||||
with open(marker_path, "w") as f:
|
||||
json.dump({"bundleId": bundle_id, "startedAt": datetime.now(timezone.utc).isoformat()}, f)
|
||||
|
||||
|
||||
def clear_venv_writing(marker_path: str) -> None:
|
||||
if os.path.exists(marker_path):
|
||||
os.unlink(marker_path)
|
||||
|
||||
|
||||
def abandon_merge(sp_dir, plan, quarantine_dir, marker_path, bundle_id) -> None:
|
||||
"""Put the venv back the way it was before this bundle was merged.
|
||||
|
||||
A refused install must leave nothing behind: without this, a bundle that
|
||||
fails verification would keep the versions it displaced from the bundles
|
||||
that were already working, so one bad install would break tools that used
|
||||
to run. Runs under the crash breadcrumb, and deliberately leaves the
|
||||
breadcrumb in place if the rollback itself fails, so a venv this installer
|
||||
could not repair still gets reseeded on next boot.
|
||||
"""
|
||||
if not sp_dir or (not plan and not os.path.isdir(quarantine_dir)):
|
||||
return
|
||||
try:
|
||||
mark_venv_writing(marker_path, bundle_id)
|
||||
rollback_reconciliation(sp_dir, plan, quarantine_dir)
|
||||
clear_venv_writing(marker_path)
|
||||
sys.stderr.write("[install] rolled the venv back to its state before this bundle\n")
|
||||
except OSError as e:
|
||||
sys.stderr.write(
|
||||
f"[install] could not roll the venv back ({e}); the AI environment will be "
|
||||
f"reseeded on next restart\n"
|
||||
)
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
# -- Fixups (NCCL wheel) --
|
||||
|
||||
def apply_fixups(staging_dir: str, venv_path: str) -> None:
|
||||
@@ -865,6 +1181,12 @@ def _install() -> None:
|
||||
emit_progress(92, "Installing packages...")
|
||||
site_packages_dir = get_site_packages_dir(venv_path)
|
||||
venv_writing_marker = os.path.join(ai_dir, "venv.writing")
|
||||
# Superseded versions wait here until the install is verified. It sits inside
|
||||
# the venv so every move is a same-filesystem rename rather than a copy of
|
||||
# several GB, and outside site-packages so a quarantined dist-info cannot be
|
||||
# picked up by importlib.metadata while it waits.
|
||||
quarantine_dir = os.path.join(venv_path, ".superseded")
|
||||
reconciliation_plan = []
|
||||
|
||||
try:
|
||||
if os.path.isdir(staging_sp) and site_packages_dir:
|
||||
@@ -875,18 +1197,19 @@ def _install() -> None:
|
||||
# reseeds the venv back to a known-good base. We clear it the instant
|
||||
# the site-packages move completes, since the venv is consistent
|
||||
# again then (a later models-move failure can't tear the venv).
|
||||
with open(venv_writing_marker, "w") as mf:
|
||||
json.dump(
|
||||
{
|
||||
"bundleId": bundle_id,
|
||||
"startedAt": datetime.now(timezone.utc).isoformat(),
|
||||
},
|
||||
mf,
|
||||
)
|
||||
mark_venv_writing(venv_writing_marker, bundle_id)
|
||||
shutil.rmtree(quarantine_dir, ignore_errors=True)
|
||||
# Flavor first: `onnxruntime` and `onnxruntime-gpu` are two different
|
||||
# distributions writing the same package directory, so which one wins
|
||||
# is not a version question and the general pass below cannot decide
|
||||
# it. Running it first also means that when the CPU build is dropped
|
||||
# from staging, the general pass never sees it and so never uninstalls
|
||||
# the GPU build the venv is keeping (#490).
|
||||
reconcile_onnxruntime(staging_sp, site_packages_dir)
|
||||
reconciliation_plan = plan_reconciliation(staging_sp, site_packages_dir)
|
||||
supersede_distributions(site_packages_dir, reconciliation_plan, quarantine_dir)
|
||||
move_tree(staging_sp, site_packages_dir)
|
||||
if os.path.exists(venv_writing_marker):
|
||||
os.unlink(venv_writing_marker)
|
||||
clear_venv_writing(venv_writing_marker)
|
||||
|
||||
# -- Move models --
|
||||
emit_progress(95, "Installing models...")
|
||||
@@ -896,6 +1219,10 @@ def _install() -> None:
|
||||
move_tree(staging_models, models_dir)
|
||||
except OSError as e:
|
||||
shutil.rmtree(staging_dir, ignore_errors=True)
|
||||
abandon_merge(
|
||||
site_packages_dir, reconciliation_plan, quarantine_dir,
|
||||
venv_writing_marker, bundle_id,
|
||||
)
|
||||
if getattr(e, "errno", None) == errno.ENOSPC:
|
||||
fail("Ran out of disk space while installing the bundle. Free up space and retry.")
|
||||
fail(f"Failed to install bundle files: {e}")
|
||||
@@ -924,9 +1251,17 @@ def _install() -> None:
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
shutil.rmtree(staging_dir, ignore_errors=True)
|
||||
abandon_merge(
|
||||
site_packages_dir, reconciliation_plan, quarantine_dir,
|
||||
venv_writing_marker, bundle_id,
|
||||
)
|
||||
fail("Installation verification timed out. Please retry the install.")
|
||||
if proc.returncode != 0:
|
||||
shutil.rmtree(staging_dir, ignore_errors=True)
|
||||
abandon_merge(
|
||||
site_packages_dir, reconciliation_plan, quarantine_dir,
|
||||
venv_writing_marker, bundle_id,
|
||||
)
|
||||
tail = "\n".join((proc.stderr or "").strip().splitlines()[-6:])
|
||||
fail(
|
||||
"Installation verification failed: the bundle installed but its "
|
||||
@@ -948,6 +1283,9 @@ def _install() -> None:
|
||||
write_installed_atomic(ai_dir, installed)
|
||||
|
||||
# -- Cleanup --
|
||||
# The install is recorded, so the versions this bundle displaced are never
|
||||
# coming back and their quarantine copy is just disk.
|
||||
shutil.rmtree(quarantine_dir, ignore_errors=True)
|
||||
if os.path.exists(staging_dir):
|
||||
shutil.rmtree(staging_dir, ignore_errors=True)
|
||||
# Clean up downloaded tar (but not if local override)
|
||||
|
||||
@@ -1,37 +1,31 @@
|
||||
"""Gate for runtime model downloads, with an optional strict offline mode.
|
||||
"""Gate runtime model downloads behind an explicit opt-in.
|
||||
|
||||
Models normally arrive through user-initiated feature bundle installs
|
||||
(install_feature.py), and the resolvers in the AI scripts always prefer those
|
||||
bundled files. When a model is missing, scripts may fetch the public model
|
||||
weights as a fallback so tools work out of the box; that fallback only ever
|
||||
downloads public model files, never user data.
|
||||
|
||||
Setting SNAPOTTER_ALLOW_MODEL_DOWNLOAD=0 enables strict offline mode for
|
||||
airgapped or locked-down deployments: every script calls
|
||||
ensure_download_allowed() immediately before any download fallback, so a
|
||||
missing file then surfaces as an actionable error instead of an outbound
|
||||
fetch.
|
||||
bundled files. If a model is missing, the runtime fails closed instead of
|
||||
fetching mutable content. Operators may explicitly opt into public model
|
||||
fallback downloads with SNAPOTTER_ALLOW_MODEL_DOWNLOAD=1.
|
||||
"""
|
||||
import os
|
||||
|
||||
|
||||
def downloads_allowed():
|
||||
"""True unless strict offline mode is explicitly enabled.
|
||||
"""Return True only when runtime downloads are explicitly enabled.
|
||||
|
||||
Runtime model downloads are allowed by default; only an explicit
|
||||
SNAPOTTER_ALLOW_MODEL_DOWNLOAD=0 (or "false") blocks them.
|
||||
Unknown values remain fail-closed to avoid enabling network access through
|
||||
a typo or an inherited environment setting.
|
||||
"""
|
||||
return os.environ.get("SNAPOTTER_ALLOW_MODEL_DOWNLOAD", "1").lower() not in ("0", "false")
|
||||
return os.environ.get("SNAPOTTER_ALLOW_MODEL_DOWNLOAD", "0").lower() in ("1", "true")
|
||||
|
||||
|
||||
def ensure_download_allowed(what):
|
||||
"""Raise a clear, actionable error when strict offline mode blocks a fetch."""
|
||||
"""Raise a clear, actionable error when runtime downloads are disabled."""
|
||||
if downloads_allowed():
|
||||
return
|
||||
raise RuntimeError(
|
||||
f"{what} is missing and automatic downloads are disabled by "
|
||||
"SNAPOTTER_ALLOW_MODEL_DOWNLOAD=0. Reinstall the feature bundle from "
|
||||
"Settings, or unset SNAPOTTER_ALLOW_MODEL_DOWNLOAD to permit downloads."
|
||||
f"{what} is missing and automatic downloads are disabled. Reinstall "
|
||||
"the feature bundle from Settings, or explicitly set "
|
||||
"SNAPOTTER_ALLOW_MODEL_DOWNLOAD=1 to permit runtime downloads."
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -13,6 +13,26 @@ def load_installer():
|
||||
return module
|
||||
|
||||
|
||||
def test_main_temporarily_lifts_hugging_face_offline_flags(monkeypatch):
|
||||
"""An explicit bundle install stays online while runtime remains fail-closed."""
|
||||
installer = load_installer()
|
||||
monkeypatch.setenv("HF_HUB_OFFLINE", "1")
|
||||
monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1")
|
||||
observed = {}
|
||||
|
||||
def fake_install():
|
||||
observed["hf"] = os.environ["HF_HUB_OFFLINE"]
|
||||
observed["transformers"] = os.environ["TRANSFORMERS_OFFLINE"]
|
||||
|
||||
monkeypatch.setattr(installer, "_install", fake_install)
|
||||
|
||||
installer.main()
|
||||
|
||||
assert observed == {"hf": "0", "transformers": "0"}
|
||||
assert os.environ["HF_HUB_OFFLINE"] == "1"
|
||||
assert os.environ["TRANSFORMERS_OFFLINE"] == "1"
|
||||
|
||||
|
||||
def test_download_with_hf_hub_uses_accelerated_client(monkeypatch, tmp_path):
|
||||
installer = load_installer()
|
||||
downloaded = tmp_path / "hf-cache" / "bundle.tar.gz"
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
"""Mechanics of the general distribution reconciliation in install_feature.py.
|
||||
|
||||
The version-conflict suite covers the field failure (AI-20260726-001). This one
|
||||
covers the parts of the machinery that failure does not exercise: what a
|
||||
rollback restores, what the uninstall is allowed to touch, and that generalising
|
||||
the reconciliation did not swallow the onnxruntime flavour rule it replaced
|
||||
(#490), which is the one collision where the newer or later version is NOT the
|
||||
right winner.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
|
||||
CUDA_PROVIDER_LIB = "libonnxruntime_providers_cuda.so"
|
||||
|
||||
|
||||
def load_installer():
|
||||
script_path = os.path.join(os.path.dirname(__file__), "..", "install_feature.py")
|
||||
spec = importlib.util.spec_from_file_location("install_feature_reconcile_under_test", script_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
assert spec.loader is not None
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def write_dist(sp, name, version, files, extra_record=()):
|
||||
"""Lay out a distribution: `files` maps relative path -> contents."""
|
||||
recorded = []
|
||||
for rel, contents in files.items():
|
||||
target = sp / rel
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(contents)
|
||||
recorded.append(rel)
|
||||
dist_info = sp / f"{name}-{version}.dist-info"
|
||||
dist_info.mkdir(parents=True, exist_ok=True)
|
||||
(dist_info / "METADATA").write_text(f"Name: {name}\nVersion: {version}\n")
|
||||
recorded += [
|
||||
f"{name}-{version}.dist-info/METADATA",
|
||||
f"{name}-{version}.dist-info/RECORD",
|
||||
]
|
||||
recorded += list(extra_record)
|
||||
(dist_info / "RECORD").write_text("\n".join(f"{rel},," for rel in recorded) + "\n")
|
||||
|
||||
|
||||
def merge(installer, staging, venv_sp, quarantine):
|
||||
"""The sequence _install runs, so the tests cannot drift from production."""
|
||||
installer.reconcile_onnxruntime(str(staging), str(venv_sp))
|
||||
plan = installer.plan_reconciliation(str(staging), str(venv_sp))
|
||||
installer.supersede_distributions(str(venv_sp), plan, str(quarantine))
|
||||
installer.move_tree(str(staging), str(venv_sp))
|
||||
return plan
|
||||
|
||||
|
||||
def trees(tmp_path):
|
||||
staging = tmp_path / "staging"
|
||||
venv_sp = tmp_path / "venv-sp"
|
||||
quarantine = tmp_path / "venv" / ".superseded"
|
||||
staging.mkdir()
|
||||
venv_sp.mkdir()
|
||||
return staging, venv_sp, quarantine
|
||||
|
||||
|
||||
def dist_infos(sp, prefix):
|
||||
return sorted(n for n in os.listdir(sp) if n.startswith(prefix) and n.endswith(".dist-info"))
|
||||
|
||||
|
||||
# -- the onnxruntime flavour rule survives generalisation (#490) --
|
||||
|
||||
|
||||
def test_an_incoming_cpu_onnxruntime_still_cannot_evict_the_gpu_build(tmp_path):
|
||||
"""The flavour rule drops the CPU build from staging before the general pass
|
||||
runs, so the general pass must never see an incoming `onnxruntime` and must
|
||||
leave `onnxruntime-gpu` installed. Newer-or-later would get this wrong: the
|
||||
CPU wheel is a different distribution writing the same package directory."""
|
||||
installer = load_installer()
|
||||
staging, venv_sp, quarantine = trees(tmp_path)
|
||||
write_dist(
|
||||
venv_sp, "onnxruntime_gpu", "1.20.1",
|
||||
{
|
||||
"onnxruntime/__init__.py": "gpu",
|
||||
f"onnxruntime/capi/{CUDA_PROVIDER_LIB}": "cuda",
|
||||
"onnxruntime/capi/onnxruntime_pybind11_state.so": "gpu",
|
||||
},
|
||||
)
|
||||
write_dist(
|
||||
staging, "onnxruntime", "1.20.1",
|
||||
{
|
||||
"onnxruntime/__init__.py": "cpu",
|
||||
"onnxruntime/capi/onnxruntime_pybind11_state.so": "cpu",
|
||||
},
|
||||
)
|
||||
write_dist(staging, "faster_whisper", "1.2.1", {"faster_whisper/__init__.py": "fw"})
|
||||
|
||||
merge(installer, staging, venv_sp, quarantine)
|
||||
|
||||
assert (venv_sp / "onnxruntime" / "capi" / CUDA_PROVIDER_LIB).exists()
|
||||
assert (venv_sp / "onnxruntime" / "capi" / "onnxruntime_pybind11_state.so").read_text() == "gpu"
|
||||
assert dist_infos(venv_sp, "onnxruntime_gpu-") == ["onnxruntime_gpu-1.20.1.dist-info"]
|
||||
assert dist_infos(venv_sp, "onnxruntime-") == []
|
||||
# The rest of the bundle still installs.
|
||||
assert (venv_sp / "faster_whisper" / "__init__.py").read_text() == "fw"
|
||||
|
||||
|
||||
def test_an_incoming_gpu_onnxruntime_replaces_the_cpu_build(tmp_path):
|
||||
installer = load_installer()
|
||||
staging, venv_sp, quarantine = trees(tmp_path)
|
||||
write_dist(
|
||||
venv_sp, "onnxruntime", "1.20.1",
|
||||
{
|
||||
"onnxruntime/__init__.py": "cpu",
|
||||
"onnxruntime/capi/onnxruntime_pybind11_state.so": "cpu",
|
||||
},
|
||||
)
|
||||
write_dist(
|
||||
staging, "onnxruntime_gpu", "1.20.1",
|
||||
{
|
||||
"onnxruntime/__init__.py": "gpu",
|
||||
f"onnxruntime/capi/{CUDA_PROVIDER_LIB}": "cuda",
|
||||
"onnxruntime/capi/onnxruntime_pybind11_state.so": "gpu",
|
||||
},
|
||||
)
|
||||
|
||||
merge(installer, staging, venv_sp, quarantine)
|
||||
|
||||
assert (venv_sp / "onnxruntime" / "capi" / CUDA_PROVIDER_LIB).exists()
|
||||
assert (venv_sp / "onnxruntime" / "capi" / "onnxruntime_pybind11_state.so").read_text() == "gpu"
|
||||
assert dist_infos(venv_sp, "onnxruntime-") == []
|
||||
assert dist_infos(venv_sp, "onnxruntime_gpu-") == ["onnxruntime_gpu-1.20.1.dist-info"]
|
||||
|
||||
|
||||
# -- what the uninstall is allowed to touch --
|
||||
|
||||
|
||||
def test_console_scripts_outside_site_packages_are_left_alone(tmp_path):
|
||||
"""RECORD points at `../../../bin/<name>` for entry points. Those cannot
|
||||
shadow an import, and this installer was handed site-packages, not the venv
|
||||
root. Laid out at the real depth so the escape actually resolves onto the
|
||||
venv's bin directory rather than harmlessly missing it."""
|
||||
installer = load_installer()
|
||||
venv = tmp_path / "venv"
|
||||
venv_sp = venv / "lib" / "python3.12" / "site-packages"
|
||||
staging = tmp_path / "staging"
|
||||
quarantine = venv / ".superseded"
|
||||
venv_sp.mkdir(parents=True)
|
||||
staging.mkdir()
|
||||
bin_dir = venv / "bin"
|
||||
bin_dir.mkdir()
|
||||
(bin_dir / "tqdm").write_text("#!/bin/sh")
|
||||
assert (venv_sp / ".." / ".." / ".." / "bin" / "tqdm").resolve() == (bin_dir / "tqdm").resolve()
|
||||
write_dist(venv_sp, "tqdm", "4.68.3", {"tqdm/__init__.py": "old"},
|
||||
extra_record=["../../../bin/tqdm", "/etc/hosts"])
|
||||
write_dist(staging, "tqdm", "4.69.1", {"tqdm/__init__.py": "new"})
|
||||
|
||||
# Pinned at the source, because whether an escaping path survives a move
|
||||
# depends on how deep the quarantine happens to sit: reading it is the only
|
||||
# place the answer is unambiguous.
|
||||
owned = installer.distribution_files(str(venv_sp), "tqdm-4.68.3.dist-info")
|
||||
assert not [rel for rel in owned if rel.startswith("/") or ".." in rel.split("/")]
|
||||
|
||||
merge(installer, staging, venv_sp, quarantine)
|
||||
|
||||
assert (bin_dir / "tqdm").exists()
|
||||
assert (venv_sp / "tqdm" / "__init__.py").read_text() == "new"
|
||||
|
||||
|
||||
def test_a_directory_another_distribution_still_uses_is_not_pruned(tmp_path):
|
||||
"""Two distributions can share a namespace directory. Emptying one of them
|
||||
must not take the other's files with it."""
|
||||
installer = load_installer()
|
||||
staging, venv_sp, quarantine = trees(tmp_path)
|
||||
write_dist(venv_sp, "google_api", "1.0.0", {"google/api/__init__.py": "api-old"})
|
||||
write_dist(venv_sp, "google_cloud", "2.0.0", {"google/cloud/__init__.py": "cloud"})
|
||||
write_dist(staging, "google_api", "1.1.0", {"google/api/__init__.py": "api-new"})
|
||||
|
||||
merge(installer, staging, venv_sp, quarantine)
|
||||
|
||||
assert (venv_sp / "google" / "cloud" / "__init__.py").read_text() == "cloud"
|
||||
assert (venv_sp / "google" / "api" / "__init__.py").read_text() == "api-new"
|
||||
|
||||
|
||||
def test_uninstalling_one_opencv_flavour_leaves_cv2_for_its_siblings(tmp_path):
|
||||
"""The three opencv projects all own `cv2/`, the way onnxruntime's two
|
||||
flavours own `onnxruntime/`.
|
||||
|
||||
Reproduced live: superseding opencv-contrib-python deleted every file its
|
||||
RECORD listed, which is the same `cv2/` that opencv-python and
|
||||
opencv-python-headless were still claiming. The merge only puts back the
|
||||
files of the distribution it is placing, so the venv lost cv2 outright and
|
||||
rembg, mediapipe, basicsr, realesrgan and gfpgan all stopped importing.
|
||||
"""
|
||||
installer = load_installer()
|
||||
staging, venv_sp, quarantine = trees(tmp_path)
|
||||
shared = {"cv2/__init__.py": "shared", "cv2/cv2.abi3.so": "shared-binary"}
|
||||
write_dist(venv_sp, "opencv_python_headless", "4.10.0.84", dict(shared))
|
||||
write_dist(venv_sp, "opencv_python", "4.11.0.86", dict(shared))
|
||||
write_dist(venv_sp, "opencv_contrib_python", "4.11.0.86", dict(shared))
|
||||
write_dist(staging, "opencv_contrib_python", "4.13.0.92",
|
||||
{"cv2/__init__.py": "contrib-new", "cv2/cv2.abi3.so": "contrib-binary"})
|
||||
|
||||
merge(installer, staging, venv_sp, quarantine)
|
||||
|
||||
assert (venv_sp / "cv2" / "__init__.py").read_text() == "contrib-new"
|
||||
assert (venv_sp / "cv2" / "cv2.abi3.so").read_text() == "contrib-binary"
|
||||
assert dist_infos(venv_sp, "opencv_contrib_python-") == ["opencv_contrib_python-4.13.0.92.dist-info"]
|
||||
assert dist_infos(venv_sp, "opencv_python-") == ["opencv_python-4.11.0.86.dist-info"]
|
||||
|
||||
|
||||
def test_a_refused_install_does_not_take_a_siblings_shared_import_with_it(tmp_path):
|
||||
"""The rollback has the same hazard from the other side: removing what the
|
||||
bundle placed must not remove files another installed distribution owns."""
|
||||
installer = load_installer()
|
||||
staging, venv_sp, quarantine = trees(tmp_path)
|
||||
write_dist(venv_sp, "opencv_python_headless", "4.10.0.84",
|
||||
{"cv2/__init__.py": "headless", "cv2/cv2.abi3.so": "headless-binary"})
|
||||
write_dist(staging, "opencv_contrib_python", "4.13.0.92",
|
||||
{"cv2/__init__.py": "contrib", "cv2/cv2.abi3.so": "contrib-binary"})
|
||||
|
||||
plan = merge(installer, staging, venv_sp, quarantine)
|
||||
installer.rollback_reconciliation(str(venv_sp), plan, str(quarantine))
|
||||
|
||||
assert (venv_sp / "cv2" / "__init__.py").exists()
|
||||
assert dist_infos(venv_sp, "opencv_python_headless-") == ["opencv_python_headless-4.10.0.84.dist-info"]
|
||||
assert dist_infos(venv_sp, "opencv_contrib_python-") == []
|
||||
|
||||
|
||||
def test_files_the_incoming_copy_does_not_carry_are_left_where_they_are(tmp_path):
|
||||
"""Bundle archives are not always complete wheels.
|
||||
|
||||
Reproduced live: upscale-enhance carries setuptools 74.1.3 as `setuptools/`
|
||||
plus its dist-info, and nothing else. The version it replaced also owned
|
||||
`_distutils_hack/` and `distutils-precedence.pth`, the shim that gives
|
||||
Python 3.12 a `distutils`. Removing everything the old RECORD listed deleted
|
||||
the shim, nothing wrote it back, and basicsr, realesrgan and gfpgan all
|
||||
stopped importing on a venv where every bundle reported installed.
|
||||
"""
|
||||
installer = load_installer()
|
||||
staging, venv_sp, quarantine = trees(tmp_path)
|
||||
write_dist(
|
||||
venv_sp, "setuptools", "78.1.1",
|
||||
{
|
||||
"setuptools/__init__.py": "78",
|
||||
"setuptools/_vendor/old.py": "78-vendored",
|
||||
"_distutils_hack/__init__.py": "shim",
|
||||
"distutils-precedence.pth": "import _distutils_hack",
|
||||
},
|
||||
)
|
||||
# The archive's RECORD is the wheel's, and claims files the archive does not
|
||||
# actually carry. That is what made the first attempt at this guard fail on
|
||||
# the real node: reading RECORD alone still cleared the shim.
|
||||
write_dist(
|
||||
staging, "setuptools", "74.1.3", {"setuptools/__init__.py": "74"},
|
||||
extra_record=["_distutils_hack/__init__.py", "distutils-precedence.pth"],
|
||||
)
|
||||
|
||||
merge(installer, staging, venv_sp, quarantine)
|
||||
|
||||
assert (venv_sp / "_distutils_hack" / "__init__.py").read_text() == "shim"
|
||||
assert (venv_sp / "distutils-precedence.pth").read_text() == "import _distutils_hack"
|
||||
# The part the incoming copy does cover is fully replaced, stale files and all.
|
||||
assert (venv_sp / "setuptools" / "__init__.py").read_text() == "74"
|
||||
assert not (venv_sp / "setuptools" / "_vendor" / "old.py").exists()
|
||||
assert dist_infos(venv_sp, "setuptools-") == ["setuptools-74.1.3.dist-info"]
|
||||
|
||||
|
||||
def test_stale_bytecode_of_a_removed_module_is_discarded(tmp_path):
|
||||
"""A `.pyc` orphaned by the uninstall keeps the package directory alive and
|
||||
would block the incoming version's directory from landing cleanly."""
|
||||
installer = load_installer()
|
||||
staging, venv_sp, quarantine = trees(tmp_path)
|
||||
write_dist(venv_sp, "numba", "0.65.1", {"numba/dropped.py": "old"})
|
||||
cache = venv_sp / "numba" / "__pycache__"
|
||||
cache.mkdir(parents=True)
|
||||
(cache / "dropped.cpython-312.pyc").write_bytes(b"stale")
|
||||
write_dist(staging, "numba", "0.66.0", {"numba/kept.py": "new"})
|
||||
|
||||
merge(installer, staging, venv_sp, quarantine)
|
||||
|
||||
assert not (venv_sp / "numba" / "dropped.py").exists()
|
||||
assert not (cache / "dropped.cpython-312.pyc").exists()
|
||||
assert (venv_sp / "numba" / "kept.py").read_text() == "new"
|
||||
|
||||
|
||||
def test_a_distribution_with_no_record_still_loses_its_metadata(tmp_path):
|
||||
"""Without a RECORD the file list is unknowable, but leaving the old
|
||||
dist-info behind would report two versions installed. Drop what we can."""
|
||||
installer = load_installer()
|
||||
staging, venv_sp, quarantine = trees(tmp_path)
|
||||
write_dist(venv_sp, "click", "8.4.1", {"click/__init__.py": "old"})
|
||||
(venv_sp / "click-8.4.1.dist-info" / "RECORD").unlink()
|
||||
write_dist(staging, "click", "8.4.2", {"click/__init__.py": "new"})
|
||||
|
||||
merge(installer, staging, venv_sp, quarantine)
|
||||
|
||||
assert dist_infos(venv_sp, "click-") == ["click-8.4.2.dist-info"]
|
||||
assert (venv_sp / "click" / "__init__.py").read_text() == "new"
|
||||
|
||||
|
||||
def test_name_spelling_does_not_hide_a_collision(tmp_path):
|
||||
"""`hf_xet` and `hf-xet` are the same project. Comparing raw directory names
|
||||
would miss the collision and leave both versions installed."""
|
||||
installer = load_installer()
|
||||
staging, venv_sp, _q = trees(tmp_path)
|
||||
write_dist(venv_sp, "hf_xet", "1.5.1", {"hf_xet/__init__.py": "old"})
|
||||
write_dist(staging, "hf-xet", "1.5.2", {"hf_xet/__init__.py": "new"})
|
||||
|
||||
plan = installer.plan_reconciliation(str(staging), str(venv_sp))
|
||||
|
||||
assert [item["name"] for item in plan] == ["hf-xet"]
|
||||
assert plan[0]["superseded"] == [("1.5.1", "hf_xet-1.5.1.dist-info")]
|
||||
|
||||
|
||||
# -- rollback --
|
||||
|
||||
|
||||
def test_a_refused_install_puts_the_displaced_version_back(tmp_path):
|
||||
"""The whole reason superseded files are quarantined rather than deleted: a
|
||||
bundle that fails verification must not keep the versions it took from the
|
||||
bundles that were already working."""
|
||||
installer = load_installer()
|
||||
staging, venv_sp, quarantine = trees(tmp_path)
|
||||
write_dist(
|
||||
venv_sp, "tokenizers", "0.20.3",
|
||||
{"tokenizers/__init__.py": "0.20.3", "tokenizers/native.so": "old-binary"},
|
||||
)
|
||||
write_dist(venv_sp, "transformers", "4.46.3", {"transformers/__init__.py": "keep"})
|
||||
write_dist(
|
||||
staging, "tokenizers", "0.23.1",
|
||||
{"tokenizers/__init__.py": "0.23.1", "tokenizers/other.so": "new-binary"},
|
||||
)
|
||||
write_dist(staging, "faster_whisper", "1.2.1", {"faster_whisper/__init__.py": "new"})
|
||||
|
||||
plan = merge(installer, staging, venv_sp, quarantine)
|
||||
installer.rollback_reconciliation(str(venv_sp), plan, str(quarantine))
|
||||
|
||||
assert dist_infos(venv_sp, "tokenizers-") == ["tokenizers-0.20.3.dist-info"]
|
||||
assert (venv_sp / "tokenizers" / "__init__.py").read_text() == "0.20.3"
|
||||
assert (venv_sp / "tokenizers" / "native.so").read_text() == "old-binary"
|
||||
assert not (venv_sp / "tokenizers" / "other.so").exists()
|
||||
# The bundle's own new distribution goes away with it.
|
||||
assert not (venv_sp / "faster_whisper").exists()
|
||||
assert dist_infos(venv_sp, "faster_whisper-") == []
|
||||
# An unrelated distribution is untouched either way.
|
||||
assert (venv_sp / "transformers" / "__init__.py").read_text() == "keep"
|
||||
assert not os.path.exists(quarantine)
|
||||
|
||||
|
||||
def test_a_rollback_leaves_an_unchanged_distribution_alone(tmp_path):
|
||||
"""A distribution already at the staged version never enters the plan, so a
|
||||
rollback must not delete files that were correct before this install."""
|
||||
installer = load_installer()
|
||||
staging, venv_sp, quarantine = trees(tmp_path)
|
||||
write_dist(venv_sp, "certifi", "2026.7.22", {"certifi/cacert.pem": "bundle"})
|
||||
write_dist(staging, "certifi", "2026.7.22", {"certifi/cacert.pem": "bundle"})
|
||||
write_dist(staging, "rembg", "2.0.62", {"rembg/__init__.py": "new"})
|
||||
|
||||
plan = merge(installer, staging, venv_sp, quarantine)
|
||||
installer.rollback_reconciliation(str(venv_sp), plan, str(quarantine))
|
||||
|
||||
assert (venv_sp / "certifi" / "cacert.pem").read_text() == "bundle"
|
||||
assert dist_infos(venv_sp, "certifi-") == ["certifi-2026.7.22.dist-info"]
|
||||
assert not (venv_sp / "rembg").exists()
|
||||
|
||||
|
||||
def test_rollback_without_a_quarantine_is_harmless(tmp_path):
|
||||
"""Nothing collided, so there is nothing to restore and the venv is left as
|
||||
it was before the bundle landed."""
|
||||
installer = load_installer()
|
||||
staging, venv_sp, quarantine = trees(tmp_path)
|
||||
write_dist(venv_sp, "numpy", "1.26.4", {"numpy/__init__.py": "base"})
|
||||
write_dist(staging, "mediapipe", "0.10.35", {"mediapipe/__init__.py": "new"})
|
||||
|
||||
plan = merge(installer, staging, venv_sp, quarantine)
|
||||
installer.rollback_reconciliation(str(venv_sp), plan, str(quarantine))
|
||||
|
||||
assert (venv_sp / "numpy" / "__init__.py").read_text() == "base"
|
||||
assert sorted(os.listdir(venv_sp)) == ["numpy", "numpy-1.26.4.dist-info"]
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Regression reproduction for the shared-venv version conflict (AI-20260726-001).
|
||||
|
||||
`reconcile_onnxruntime` fixed exactly one package pair. Every OTHER distribution
|
||||
that two bundles both provided was merged by `move_tree` file-by-file, with no
|
||||
uninstall of the version already in the venv, so two versions ended up overlaid:
|
||||
the pure-Python modules came from whichever bundle wrote last while the compiled
|
||||
extension CPython actually imports could come from the other.
|
||||
|
||||
Observed live on ubuntu-gpu-amd64 after Settings > AI Features > Install All:
|
||||
seventeen distributions carried two or three versions at once. Three of them
|
||||
broke a tool.
|
||||
|
||||
tokenizers 0.20.3 (inpaint-hq, pinned by transformers 4.46.3) and 0.23.1
|
||||
(transcription, needed by faster-whisper) both present. The package directory
|
||||
held tokenizers.cpython-312-x86_64-linux-gnu.so from 0.20.3 next to
|
||||
tokenizers.abi3.so from 0.23.1; CPython prefers the platform-specific suffix,
|
||||
so 0.23.1's `decoders/__init__.py` ran against 0.20.3's binary and raised
|
||||
"AttributeError: module 'decoders' has no attribute 'DecodeStream'". The
|
||||
transcription install failed its own smoke gate and could not be installed at
|
||||
all on that host.
|
||||
|
||||
scipy 1.12.0 and 1.17.1 -> Real-ESRGAN cannot import, `upscale` fails.
|
||||
huggingface_hub 0.36.2, 1.19.0 and 1.22.0 -> transformers refuses to import,
|
||||
the high-quality Object Eraser path fails.
|
||||
|
||||
The invariant is deliberately modest and does not presume a particular winner:
|
||||
after a bundle's site-packages has been merged into the venv, exactly ONE
|
||||
version of any given distribution remains, and no file belonging to the
|
||||
superseded version survives to shadow it.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
CASES = {
|
||||
# distribution, older version, newer version, older extension, newer extension
|
||||
"tokenizers": ("0.20.3", "0.23.1", "tokenizers.cpython-312-x86_64-linux-gnu.so", "tokenizers.abi3.so"),
|
||||
"scipy": ("1.12.0", "1.17.1", "_lib/_ccallback_c.cpython-312-x86_64-linux-gnu.so", "_lib/_ccallback_c.abi3.so"),
|
||||
"huggingface_hub": ("0.36.2", "1.22.0", None, None),
|
||||
}
|
||||
|
||||
|
||||
def load_installer():
|
||||
script_path = os.path.join(os.path.dirname(__file__), "..", "install_feature.py")
|
||||
spec = importlib.util.spec_from_file_location("install_feature_conflict_under_test", script_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
assert spec.loader is not None
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def make_distribution(site_packages, package, version, extension):
|
||||
"""Lay out one version of the package, the way a bundle archive carries it."""
|
||||
package_dir = site_packages / package
|
||||
(package_dir / "decoders").mkdir(parents=True, exist_ok=True)
|
||||
(package_dir / "__init__.py").write_text(f"__version__ = {version!r}\n")
|
||||
(package_dir / "decoders" / "__init__.py").write_text(f"# decoders for {version}\n")
|
||||
owned = [
|
||||
f"{package}/__init__.py",
|
||||
f"{package}/decoders/__init__.py",
|
||||
]
|
||||
if extension:
|
||||
target = package_dir / extension
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(f"native {version}")
|
||||
owned.append(f"{package}/{extension}")
|
||||
dist_info = site_packages / f"{package}-{version}.dist-info"
|
||||
dist_info.mkdir(parents=True, exist_ok=True)
|
||||
(dist_info / "METADATA").write_text(f"Name: {package}\nVersion: {version}\n")
|
||||
owned += [
|
||||
f"{package}-{version}.dist-info/METADATA",
|
||||
f"{package}-{version}.dist-info/RECORD",
|
||||
]
|
||||
(dist_info / "RECORD").write_text("\n".join(f"{path},," for path in owned) + "\n")
|
||||
|
||||
|
||||
def merge_bundle(tmp_path, package, installed_version, incoming_version,
|
||||
installed_extension, incoming_extension):
|
||||
"""Run the installer's real merge sequence for a second bundle."""
|
||||
installer = load_installer()
|
||||
site_packages = tmp_path / "venv-site-packages"
|
||||
staging = tmp_path / "staging-site-packages"
|
||||
quarantine = tmp_path / "quarantine"
|
||||
site_packages.mkdir()
|
||||
staging.mkdir()
|
||||
|
||||
make_distribution(site_packages, package, installed_version, installed_extension)
|
||||
make_distribution(staging, package, incoming_version, incoming_extension)
|
||||
|
||||
installer.reconcile_onnxruntime(str(staging), str(site_packages))
|
||||
plan = installer.plan_reconciliation(str(staging), str(site_packages))
|
||||
installer.supersede_distributions(str(site_packages), plan, str(quarantine))
|
||||
installer.move_tree(str(staging), str(site_packages))
|
||||
return site_packages
|
||||
|
||||
|
||||
def dist_info_versions(site_packages, package):
|
||||
return sorted(
|
||||
entry.name.removesuffix(".dist-info").split("-", 1)[1]
|
||||
for entry in site_packages.iterdir()
|
||||
if entry.name.startswith(f"{package}-") and entry.name.endswith(".dist-info")
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("package", sorted(CASES))
|
||||
@pytest.mark.parametrize("upgrade", [True, False], ids=["upgrade", "downgrade"])
|
||||
def test_merging_a_second_bundle_leaves_one_distribution_version(tmp_path, package, upgrade):
|
||||
"""Whichever direction the second bundle moves the version, only one remains.
|
||||
|
||||
Both directions matter: Install All has no fixed order, and the field
|
||||
failure needed only that two bundles disagreed, not that the newer one
|
||||
landed second.
|
||||
"""
|
||||
older, newer, older_ext, newer_ext = CASES[package]
|
||||
installed, incoming = (older, newer) if upgrade else (newer, older)
|
||||
installed_ext, incoming_ext = (older_ext, newer_ext) if upgrade else (newer_ext, older_ext)
|
||||
|
||||
site_packages = merge_bundle(
|
||||
tmp_path, package, installed, incoming, installed_ext, incoming_ext
|
||||
)
|
||||
|
||||
assert dist_info_versions(site_packages, package) == [incoming]
|
||||
assert incoming in (site_packages / package / "__init__.py").read_text()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("package", ["tokenizers", "scipy"])
|
||||
def test_merging_a_second_bundle_removes_the_superseded_extension(tmp_path, package):
|
||||
"""The exact tokenizers mechanism: CPython prefers the platform-specific
|
||||
suffix, so a surviving `.cpython-312-*.so` from the old version would be
|
||||
loaded under the new version's Python modules."""
|
||||
older, newer, older_ext, newer_ext = CASES[package]
|
||||
|
||||
site_packages = merge_bundle(tmp_path, package, older, newer, older_ext, newer_ext)
|
||||
|
||||
package_dir = site_packages / package
|
||||
assert (package_dir / newer_ext).exists()
|
||||
assert not (package_dir / older_ext).exists()
|
||||
|
||||
|
||||
def test_a_reinstall_of_the_same_version_is_left_alone(tmp_path):
|
||||
"""Placing what is already there must not churn the venv: nothing is
|
||||
superseded, so nothing is removed and nothing can be torn."""
|
||||
installer = load_installer()
|
||||
site_packages = tmp_path / "venv-site-packages"
|
||||
staging = tmp_path / "staging-site-packages"
|
||||
site_packages.mkdir()
|
||||
staging.mkdir()
|
||||
make_distribution(site_packages, "tokenizers", "0.20.3", None)
|
||||
make_distribution(staging, "tokenizers", "0.20.3", None)
|
||||
|
||||
plan = installer.plan_reconciliation(str(staging), str(site_packages))
|
||||
|
||||
assert plan == []
|
||||
|
||||
|
||||
def test_reinstalling_a_bundle_repairs_a_venv_already_carrying_both_versions(tmp_path):
|
||||
"""Self-heal for hosts that ran the broken installer.
|
||||
|
||||
Someone who already clicked Install All has a venv with both versions
|
||||
overlaid. Reinstalling any bundle that provides the distribution has to
|
||||
clear the other version rather than add a third.
|
||||
"""
|
||||
installer = load_installer()
|
||||
site_packages = tmp_path / "venv-site-packages"
|
||||
staging = tmp_path / "staging-site-packages"
|
||||
quarantine = tmp_path / "quarantine"
|
||||
site_packages.mkdir()
|
||||
staging.mkdir()
|
||||
make_distribution(site_packages, "tokenizers", "0.20.3", "tokenizers.cpython-312-x86_64-linux-gnu.so")
|
||||
make_distribution(site_packages, "tokenizers", "0.23.1", "tokenizers.abi3.so")
|
||||
make_distribution(staging, "tokenizers", "0.23.1", "tokenizers.abi3.so")
|
||||
|
||||
plan = installer.plan_reconciliation(str(staging), str(site_packages))
|
||||
installer.supersede_distributions(str(site_packages), plan, str(quarantine))
|
||||
installer.move_tree(str(staging), str(site_packages))
|
||||
|
||||
assert dist_info_versions(site_packages, "tokenizers") == ["0.23.1"]
|
||||
assert not (site_packages / "tokenizers" / "tokenizers.cpython-312-x86_64-linux-gnu.so").exists()
|
||||
@@ -13,19 +13,19 @@ import offline_guard # noqa: E402
|
||||
# --- downloads_allowed ------------------------------------------------------
|
||||
|
||||
|
||||
def test_downloads_allowed_default_true(monkeypatch):
|
||||
def test_downloads_blocked_by_default(monkeypatch):
|
||||
monkeypatch.delenv("SNAPOTTER_ALLOW_MODEL_DOWNLOAD", raising=False)
|
||||
assert offline_guard.downloads_allowed() is True
|
||||
assert offline_guard.downloads_allowed() is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["0", "false", "FALSE", "False"])
|
||||
def test_downloads_blocked_by_explicit_off(monkeypatch, value):
|
||||
@pytest.mark.parametrize("value", ["0", "false", "FALSE", "False", "yes", "anything"])
|
||||
def test_downloads_blocked_without_explicit_opt_in(monkeypatch, value):
|
||||
monkeypatch.setenv("SNAPOTTER_ALLOW_MODEL_DOWNLOAD", value)
|
||||
assert offline_guard.downloads_allowed() is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["1", "true", "yes", "anything"])
|
||||
def test_downloads_allowed_for_non_off_values(monkeypatch, value):
|
||||
@pytest.mark.parametrize("value", ["1", "true", "TRUE", "True"])
|
||||
def test_downloads_allowed_for_explicit_opt_in(monkeypatch, value):
|
||||
monkeypatch.setenv("SNAPOTTER_ALLOW_MODEL_DOWNLOAD", value)
|
||||
assert offline_guard.downloads_allowed() is True
|
||||
|
||||
@@ -33,11 +33,18 @@ def test_downloads_allowed_for_non_off_values(monkeypatch, value):
|
||||
# --- ensure_download_allowed ------------------------------------------------
|
||||
|
||||
|
||||
def test_ensure_noop_when_allowed(monkeypatch):
|
||||
monkeypatch.delenv("SNAPOTTER_ALLOW_MODEL_DOWNLOAD", raising=False)
|
||||
def test_ensure_noop_when_explicitly_allowed(monkeypatch):
|
||||
monkeypatch.setenv("SNAPOTTER_ALLOW_MODEL_DOWNLOAD", "1")
|
||||
assert offline_guard.ensure_download_allowed("thing") is None
|
||||
|
||||
|
||||
def test_ensure_raises_actionable_error_by_default(monkeypatch):
|
||||
monkeypatch.delenv("SNAPOTTER_ALLOW_MODEL_DOWNLOAD", raising=False)
|
||||
with pytest.raises(RuntimeError) as exc:
|
||||
offline_guard.ensure_download_allowed("MyModel weight")
|
||||
assert "SNAPOTTER_ALLOW_MODEL_DOWNLOAD=1" in str(exc.value)
|
||||
|
||||
|
||||
def test_ensure_raises_actionable_error_when_blocked(monkeypatch):
|
||||
monkeypatch.setenv("SNAPOTTER_ALLOW_MODEL_DOWNLOAD", "0")
|
||||
with pytest.raises(RuntimeError) as exc:
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
"""What Real-ESRGAN tells the user when it cannot run (AI-20260726-002).
|
||||
|
||||
Observed on ubuntu-gpu-amd64 with `upscale-enhance` installed and working
|
||||
weights on disk: scipy was carrying 1.12.0 and 1.17.1 at once, so the import
|
||||
chain raised
|
||||
|
||||
cannot import name '_promote' from 'scipy.spatial.transform._rotation'
|
||||
|
||||
and the tool answered "Install the upscale-enhance feature", which the user had
|
||||
already done. The advice has to follow the shape of the failure, not the fact
|
||||
that one occurred.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
|
||||
|
||||
def load_upscale():
|
||||
script_path = os.path.join(os.path.dirname(__file__), "..", "upscale.py")
|
||||
spec = importlib.util.spec_from_file_location("upscale_message_under_test", script_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
assert spec.loader is not None
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_a_broken_dependency_does_not_ask_for_an_install_that_already_happened():
|
||||
upscale = load_upscale()
|
||||
failure = ImportError(
|
||||
"cannot import name '_promote' from 'scipy.spatial.transform._rotation'"
|
||||
)
|
||||
|
||||
message = upscale.realesrgan_failure_message(failure)
|
||||
|
||||
assert "_promote" in message
|
||||
assert "Install the upscale-enhance feature" not in message
|
||||
assert "Reinstall" in message
|
||||
assert "model=lanczos" in message
|
||||
|
||||
|
||||
def test_a_missing_module_still_asks_for_the_install():
|
||||
upscale = load_upscale()
|
||||
|
||||
message = upscale.realesrgan_failure_message(ModuleNotFoundError("No module named 'realesrgan'"))
|
||||
|
||||
assert "Install the upscale-enhance feature" in message
|
||||
|
||||
|
||||
def test_missing_weights_still_ask_for_the_install():
|
||||
"""The bundle carries the .pth files, so absent weights mean absent bundle."""
|
||||
upscale = load_upscale()
|
||||
|
||||
message = upscale.realesrgan_failure_message(
|
||||
FileNotFoundError("RealESRGAN model not found: /data/ai/models/RealESRGAN_x4plus.pth")
|
||||
)
|
||||
|
||||
assert "Install the upscale-enhance feature" in message
|
||||
|
||||
|
||||
def test_a_runtime_failure_points_at_a_repair_rather_than_an_install():
|
||||
upscale = load_upscale()
|
||||
|
||||
message = upscale.realesrgan_failure_message(RuntimeError("CUDA error: no kernel image"))
|
||||
|
||||
assert "Install the upscale-enhance feature" not in message
|
||||
assert "Reset AI Environment" in message
|
||||
@@ -41,6 +41,29 @@ GFPGAN_MODEL_PATH = os.environ.get(
|
||||
)
|
||||
|
||||
|
||||
def realesrgan_failure_message(error: BaseException) -> str:
|
||||
"""Say what actually went wrong with Real-ESRGAN, and what to do about it.
|
||||
|
||||
Telling someone to install what they already installed sends them in a
|
||||
circle (AI-20260726-002): on a venv where scipy was carrying two versions
|
||||
this path fired on an ImportError raised deep inside a package that had been
|
||||
present the whole time. A missing module or missing weights is the only
|
||||
shape that really means "not installed"; anything else is an install that is
|
||||
there and broken, which needs a reinstall rather than an install.
|
||||
"""
|
||||
missing = isinstance(error, (ModuleNotFoundError, FileNotFoundError))
|
||||
remedy = (
|
||||
"Install the upscale-enhance feature"
|
||||
if missing
|
||||
else (
|
||||
"The upscale-enhance libraries are present but did not load, which usually "
|
||||
"means a partial or conflicting install. Reinstall it from Settings > AI "
|
||||
"Features, or use Reset AI Environment if that does not help"
|
||||
)
|
||||
)
|
||||
return f"Real-ESRGAN is not available: {error}. {remedy}, or use model=lanczos for basic upscaling."
|
||||
|
||||
|
||||
def apply_denoise(img, strength):
|
||||
"""Apply denoising to a PIL image. Uses OpenCV when available, falls back to PIL."""
|
||||
if strength <= 0:
|
||||
@@ -236,10 +259,7 @@ def main():
|
||||
traceback.print_exc(file=sys.stderr)
|
||||
print(json.dumps({
|
||||
"success": False,
|
||||
"error": (
|
||||
f"Real-ESRGAN is not available: {e}. "
|
||||
"Install the upscale-enhance feature or use model=lanczos for basic upscaling."
|
||||
),
|
||||
"error": realesrgan_failure_message(e),
|
||||
}))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user