mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
SnapOtterandGitHub
d10d0f544f
A release-readiness QA pass over the whole product. The commits split into defects a user would hit and gates that were reporting green while measuring nothing. ## Fixes that change behaviour Rate limiting was bypassable on every install: TRUST_PROXY defaulted to true, so request.ip came from a client-set header and a forged X-Forwarded-For got past the login limiter. The default is now a private-network trust list. A transient Postgres outage stranded in-flight jobs, leaving finished output on disk with no row pointing at it. A reconciler now resolves those rows and adopts the bytes rather than dropping the work. A Redis connection that moved to a new address wedged every read-blocked consumer, so completions stopped signalling while health still answered 200. Socket timeouts plus subscriber pings recover it. Installing more than one AI bundle left the shared venv multi-versioned and silently broke three tools. The installer now reconciles distributions to one version each. Converting an image to JXL at quality 1 through 4 returned a 500, because libjxl 0.7 rejects the distance those values compute. The quality is floored at what the encoder honours. A missing ffmpeg was also reported to the user as a corrupt upload; it now says the engine is unavailable. RAW uploads reached an unpatched LibRaw on arm64, so it is built from source at 0.22.2, and the release scan was split so it can fail on an unfixed critical instead of hiding it behind ignore-unfixed. ## Gates that could not fail Two mutation lanes ran zero mutants because Stryker crawled the gitignored docs build; coverage discarded its whole report on any failing test; the lint gate skipped root tests, scripts, and two workspaces; and several generated matrices counted a host missing ffmpeg as a passing tool. Each now measures what it claims. Full evidence and the outstanding release items are tracked locally and are not part of this branch.
2406 lines
86 KiB
TypeScript
2406 lines
86 KiB
TypeScript
import { createHash, generateKeyPairSync, sign } from "node:crypto";
|
|
import {
|
|
closeSync,
|
|
constants,
|
|
existsSync,
|
|
fstatSync,
|
|
linkSync,
|
|
mkdirSync,
|
|
mkdtempSync,
|
|
openSync,
|
|
readdirSync,
|
|
readFileSync,
|
|
renameSync,
|
|
rmSync,
|
|
statSync,
|
|
symlinkSync,
|
|
truncateSync,
|
|
writeFileSync,
|
|
} from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
import {
|
|
assertOcrRuntimeInstallDiskSpace,
|
|
buildOcrRuntimeInstallerCommand,
|
|
canonicalRuntimeJson,
|
|
cleanupDownloadedRuntimeRelease as cleanupDownloadedRuntimeReleaseWithLease,
|
|
downloadVerifiedRuntimeRelease as downloadVerifiedRuntimeReleaseWithLease,
|
|
loadOcrRuntimeTrustKeys,
|
|
OcrRuntimeImportValidationError,
|
|
OcrRuntimeNotPublishedError,
|
|
type OcrRuntimeTrustKey,
|
|
prepareOfflineRuntimeIndex,
|
|
prepareOfflineRuntimeRelease,
|
|
purgeOcrRuntimeDownloads as purgeOcrRuntimeDownloadsWithLease,
|
|
remainingInstallerTimeoutMs,
|
|
verifyRuntimeIndex,
|
|
writeBufferFully,
|
|
} from "../../../apps/api/src/lib/ocr-runtime-install.js";
|
|
|
|
const TARGET = "linux-amd64-cpu-py312" as const;
|
|
const ARCHIVE_BYTES = Buffer.from("immutable runtime archive");
|
|
|
|
const temporaryDirectories: string[] = [];
|
|
|
|
function hasOpenDescriptorFor(identity: { dev: bigint; ino: bigint }): boolean {
|
|
return readdirSync("/dev/fd").some((entry) => {
|
|
if (!/^\d+$/.test(entry)) return false;
|
|
try {
|
|
const descriptor = fstatSync(Number(entry), { bigint: true });
|
|
return descriptor.dev === identity.dev && descriptor.ino === identity.ino;
|
|
} catch (error) {
|
|
const code = (error as NodeJS.ErrnoException).code;
|
|
if (code === "EBADF" || code === "ENOENT") return false;
|
|
throw error;
|
|
}
|
|
});
|
|
}
|
|
|
|
async function withInstallLease<T>(aiDataDir: string, operation: (fd: number) => Promise<T>) {
|
|
mkdirSync(aiDataDir, { recursive: true });
|
|
const lockPath = join(aiDataDir, "install.flock");
|
|
const fd = openSync(lockPath, constants.O_RDWR | constants.O_CREAT | constants.O_NOFOLLOW, 0o660);
|
|
try {
|
|
return await operation(fd);
|
|
} finally {
|
|
closeSync(fd);
|
|
}
|
|
}
|
|
|
|
async function downloadVerifiedRuntimeRelease(
|
|
options: Omit<Parameters<typeof downloadVerifiedRuntimeReleaseWithLease>[0], "installLockFd">,
|
|
) {
|
|
return withInstallLease(options.aiDataDir, (installLockFd) =>
|
|
downloadVerifiedRuntimeReleaseWithLease({ ...options, installLockFd }),
|
|
);
|
|
}
|
|
|
|
async function cleanupDownloadedRuntimeRelease(
|
|
aiDataDir: string,
|
|
release: Parameters<typeof cleanupDownloadedRuntimeReleaseWithLease>[1],
|
|
) {
|
|
return withInstallLease(aiDataDir, (installLockFd) =>
|
|
cleanupDownloadedRuntimeReleaseWithLease(aiDataDir, release, installLockFd),
|
|
);
|
|
}
|
|
|
|
async function purgeOcrRuntimeDownloads(aiDataDir: string) {
|
|
return withInstallLease(aiDataDir, (installLockFd) =>
|
|
purgeOcrRuntimeDownloadsWithLease(aiDataDir, installLockFd),
|
|
);
|
|
}
|
|
|
|
afterEach(() => {
|
|
vi.restoreAllMocks();
|
|
vi.unstubAllEnvs();
|
|
for (const directory of temporaryDirectories.splice(0)) {
|
|
rmSync(directory, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
function signedIndex(
|
|
options: {
|
|
archiveFile?: string;
|
|
archiveSha256?: string;
|
|
archiveSize?: number;
|
|
target?: string;
|
|
minimumMemoryBytes?: number | null;
|
|
} = {},
|
|
) {
|
|
const { publicKey, privateKey } = generateKeyPairSync("ed25519");
|
|
const keyId = "snapotter-ocr-release-2026";
|
|
const artifact = {
|
|
family: "ocr",
|
|
target: options.target ?? TARGET,
|
|
generation: "2.1.0-deadbeef",
|
|
version: "2.1.0",
|
|
platform: "linux",
|
|
arch: "amd64",
|
|
archive: {
|
|
file: options.archiveFile ?? `ocr-${TARGET}.tar.gz`,
|
|
sha256:
|
|
options.archiveSha256 ?? "22553579e29b27b75c9b6375d816f02483bd2e6f76f9ffcc9ae5311422410917",
|
|
size: options.archiveSize ?? ARCHIVE_BYTES.length,
|
|
expandedSize: 1_000,
|
|
},
|
|
files: [
|
|
{
|
|
path: "venv/bin/python",
|
|
sha256: "a".repeat(64),
|
|
size: 1,
|
|
mode: 493,
|
|
},
|
|
],
|
|
runtime: { pythonPath: "venv/bin/python", entrypoint: "ocr_runner.py" },
|
|
models: { detection: "a".repeat(64) },
|
|
compatibility: { protocolVersion: 1, snapotterVersion: "2.1.0" },
|
|
capabilities: {
|
|
qualities: ["balanced", "best"],
|
|
providers: ["CPUExecutionProvider"],
|
|
},
|
|
...(options.minimumMemoryBytes === null
|
|
? {}
|
|
: { resources: { minimumMemoryBytes: options.minimumMemoryBytes ?? 4 * 1024 ** 3 } }),
|
|
};
|
|
const unsigned = { schemaVersion: 1, artifacts: [artifact] };
|
|
const signature = sign(null, Buffer.from(canonicalRuntimeJson(unsigned)), privateKey).toString(
|
|
"base64",
|
|
);
|
|
const index = {
|
|
...unsigned,
|
|
signature: { keyId, algorithm: "ed25519", value: signature },
|
|
};
|
|
const trustKey: OcrRuntimeTrustKey = {
|
|
keyId,
|
|
algorithm: "ed25519",
|
|
publicKey: publicKey.export({ type: "spki", format: "pem" }).toString(),
|
|
};
|
|
return { artifact, index, raw: Buffer.from(canonicalRuntimeJson(index)), trustKey };
|
|
}
|
|
|
|
describe("remainingInstallerTimeoutMs", () => {
|
|
it("floors a fractional remaining budget to the safe integer the installer requires", () => {
|
|
// The deadline is set at one performance.now() read and the remaining time is
|
|
// computed at a later read, so `deadline - later` is fractional in practice.
|
|
const started = 1_000.4269;
|
|
const deadline = started + 7_200_000;
|
|
const later = 1_500.8731;
|
|
// The old caller passed this raw float; runOcrRuntimeInstaller rejected it as
|
|
// a non-integer timeout, which broke every default-config install.
|
|
expect(Number.isSafeInteger(deadline - later)).toBe(false);
|
|
const remaining = remainingInstallerTimeoutMs(deadline, later);
|
|
expect(Number.isSafeInteger(remaining)).toBe(true);
|
|
expect(remaining).toBe(Math.floor(deadline - later));
|
|
});
|
|
|
|
it("returns 0 when there is no deadline and clamps a passed deadline to at least 1ms", () => {
|
|
expect(remainingInstallerTimeoutMs(undefined, 5)).toBe(0);
|
|
expect(remainingInstallerTimeoutMs(10.9, 10.1)).toBe(1);
|
|
expect(remainingInstallerTimeoutMs(5, 999.7)).toBe(1);
|
|
});
|
|
});
|
|
|
|
describe("verifyRuntimeIndex", () => {
|
|
it("loads the independently pinned release key from the official image environment", () => {
|
|
const fixture = signedIndex();
|
|
vi.stubEnv("OCR_RUNTIME_INDEX_KEY_ID", fixture.trustKey.keyId);
|
|
vi.stubEnv(
|
|
"OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64",
|
|
Buffer.from(fixture.trustKey.publicKey).toString("base64"),
|
|
);
|
|
|
|
expect(loadOcrRuntimeTrustKeys()).toEqual([fixture.trustKey]);
|
|
});
|
|
|
|
it("rejects an incomplete or non-canonical official image trust environment", () => {
|
|
vi.stubEnv("OCR_RUNTIME_INDEX_KEY_ID", "release-key");
|
|
expect(() => loadOcrRuntimeTrustKeys()).toThrow("incomplete or invalid");
|
|
|
|
vi.stubEnv("OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64", "not base64");
|
|
expect(() => loadOcrRuntimeTrustKeys()).toThrow("incomplete or invalid");
|
|
});
|
|
|
|
it("authenticates a canonical index against an app-owned Ed25519 key", () => {
|
|
const fixture = signedIndex();
|
|
const verified = verifyRuntimeIndex(fixture.raw, TARGET, [fixture.trustKey], "2.1.0");
|
|
|
|
expect(verified.artifact).toEqual(fixture.artifact);
|
|
expect(verified.archiveFile).toBe(`ocr-${TARGET}.tar.gz`);
|
|
expect(verified.archiveSha256).toBe(
|
|
"22553579e29b27b75c9b6375d816f02483bd2e6f76f9ffcc9ae5311422410917",
|
|
);
|
|
expect(verified.minimumMemoryBytes).toBe(4 * 1024 ** 3);
|
|
});
|
|
|
|
it("requires a signed positive safe minimum-memory policy", () => {
|
|
for (const minimumMemoryBytes of [null, 0, -1, Number.MAX_SAFE_INTEGER + 1]) {
|
|
const fixture = signedIndex({ minimumMemoryBytes });
|
|
expect(() => verifyRuntimeIndex(fixture.raw, TARGET, [fixture.trustKey], "2.1.0")).toThrow(
|
|
"memory",
|
|
);
|
|
}
|
|
});
|
|
|
|
it("rejects non-canonical bytes, unknown keys, tampering, and version drift", () => {
|
|
const fixture = signedIndex();
|
|
expect(() =>
|
|
verifyRuntimeIndex(
|
|
Buffer.from(JSON.stringify(fixture.index)),
|
|
TARGET,
|
|
[fixture.trustKey],
|
|
"2.1.0",
|
|
),
|
|
).toThrow("canonical");
|
|
expect(() => verifyRuntimeIndex(fixture.raw, TARGET, [], "2.1.0")).toThrow("trusted");
|
|
|
|
const tampered = structuredClone(fixture.index);
|
|
tampered.artifacts[0].archive.size += 1;
|
|
expect(() =>
|
|
verifyRuntimeIndex(
|
|
Buffer.from(canonicalRuntimeJson(tampered)),
|
|
TARGET,
|
|
[fixture.trustKey],
|
|
"2.1.0",
|
|
),
|
|
).toThrow("signature");
|
|
expect(() => verifyRuntimeIndex(fixture.raw, TARGET, [fixture.trustKey], "2.2.0")).toThrow(
|
|
"version",
|
|
);
|
|
});
|
|
|
|
it("rejects wrong targets, duplicate target objects, and unsafe release-relative paths", () => {
|
|
const wrongTarget = signedIndex({ target: "linux-arm64-cpu-py311" });
|
|
expect(() =>
|
|
verifyRuntimeIndex(wrongTarget.raw, TARGET, [wrongTarget.trustKey], "2.1.0"),
|
|
).toThrow("exactly one");
|
|
|
|
const traversal = signedIndex({ archiveFile: "../stolen.tar.gz" });
|
|
expect(() => verifyRuntimeIndex(traversal.raw, TARGET, [traversal.trustKey], "2.1.0")).toThrow(
|
|
"archive file",
|
|
);
|
|
|
|
const absolute = signedIndex({ archiveFile: "https://evil.example/runtime.tar.gz" });
|
|
expect(() => verifyRuntimeIndex(absolute.raw, TARGET, [absolute.trustKey], "2.1.0")).toThrow(
|
|
"archive file",
|
|
);
|
|
});
|
|
});
|
|
|
|
describe("downloadVerifiedRuntimeRelease", () => {
|
|
it("persists every byte when the filesystem returns short writes", async () => {
|
|
const input = Buffer.from("short writes must not truncate runtime archives");
|
|
const persisted: Buffer[] = [];
|
|
const file = {
|
|
write: vi.fn(async (buffer: Uint8Array, offset: number, length: number) => {
|
|
const bytesWritten = Math.min(3, length);
|
|
persisted.push(Buffer.from(buffer.subarray(offset, offset + bytesWritten)));
|
|
return { bytesWritten, buffer };
|
|
}),
|
|
};
|
|
|
|
await expect(writeBufferFully(file, input)).resolves.toBe(input.length);
|
|
expect(Buffer.concat(persisted)).toEqual(input);
|
|
expect(file.write).toHaveBeenCalledTimes(Math.ceil(input.length / 3));
|
|
});
|
|
|
|
it("rejects an impossible filesystem write count", async () => {
|
|
const input = Buffer.from("bounded write accounting");
|
|
const file = {
|
|
write: vi.fn(async (_buffer: Uint8Array, _offset: number, length: number) => ({
|
|
bytesWritten: length + 1,
|
|
})),
|
|
};
|
|
|
|
await expect(writeBufferFully(file, input)).rejects.toThrow("invalid byte count");
|
|
});
|
|
|
|
it("downloads only the trusted version directory and verifies archive bytes", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-download-"));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES, { status: 200 }));
|
|
const progress = vi.fn();
|
|
|
|
const result = await downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
onProgress: progress,
|
|
});
|
|
|
|
expect(fetchImpl.mock.calls.map(([url]) => String(url))).toEqual([
|
|
"https://huggingface.co/snapotter-hq/feature-bundles/resolve/main/v2.1.0/v3/ocr-runtime-index.json",
|
|
`https://huggingface.co/snapotter-hq/feature-bundles/resolve/main/v2.1.0/v3/ocr-${TARGET}.tar.gz`,
|
|
]);
|
|
expect(readFileSync(result.archivePath)).toEqual(ARCHIVE_BYTES);
|
|
expect(readFileSync(result.indexPath)).toEqual(fixture.raw);
|
|
expect(statSync(join(directory, "v3")).mode & 0o7777).toBe(0o2770);
|
|
expect(statSync(join(directory, "v3", "downloads")).mode & 0o7777).toBe(0o2770);
|
|
expect(statSync(join(directory, "v3", "downloads", ".trash")).mode & 0o7777).toBe(0o2770);
|
|
expect(statSync(result.archivePath).mode & 0o777).toBe(0o660);
|
|
expect(statSync(result.indexPath).mode & 0o777).toBe(0o660);
|
|
expect(progress).toHaveBeenCalledWith(90, "Verifying OCR runtime archive");
|
|
});
|
|
|
|
it("rejects a symlinked AI data root before creating cache children", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-root-symlink-"));
|
|
temporaryDirectories.push(directory);
|
|
const outside = join(directory, "outside");
|
|
const aiDataDir = join(directory, "ai");
|
|
mkdirSync(outside);
|
|
symlinkSync(outside, aiDataDir);
|
|
const lockFd = openSync(
|
|
join(outside, "install.flock"),
|
|
constants.O_RDWR | constants.O_CREAT | constants.O_NOFOLLOW,
|
|
0o660,
|
|
);
|
|
const fetchImpl = vi.fn<typeof fetch>();
|
|
|
|
try {
|
|
await expect(
|
|
downloadVerifiedRuntimeReleaseWithLease({
|
|
installLockFd: lockFd,
|
|
aiDataDir,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
}),
|
|
).rejects.toThrow("real directory");
|
|
} finally {
|
|
closeSync(lockFd);
|
|
}
|
|
|
|
expect(fetchImpl).not.toHaveBeenCalled();
|
|
expect(readdirSync(outside)).toEqual(["install.flock"]);
|
|
});
|
|
|
|
it("requires the shared install-lease descriptor before mutating the cache", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-no-lease-"));
|
|
temporaryDirectories.push(directory);
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeReleaseWithLease({
|
|
installLockFd: -1,
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl: vi.fn<typeof fetch>(),
|
|
}),
|
|
).rejects.toThrow("install lease");
|
|
});
|
|
|
|
it("rejects overlapping cache mutations that reuse one install lease", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-overlap-"));
|
|
temporaryDirectories.push(directory);
|
|
const lockFd = openSync(
|
|
join(directory, "install.flock"),
|
|
constants.O_RDWR | constants.O_CREAT | constants.O_NOFOLLOW,
|
|
0o660,
|
|
);
|
|
let resolveIndex!: (response: Response) => void;
|
|
const pendingIndex = new Promise<Response>((resolve) => {
|
|
resolveIndex = resolve;
|
|
});
|
|
const firstFetch = vi
|
|
.fn<typeof fetch>()
|
|
.mockImplementationOnce(() => pendingIndex)
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES));
|
|
const first = downloadVerifiedRuntimeReleaseWithLease({
|
|
installLockFd: lockFd,
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl: firstFetch,
|
|
});
|
|
|
|
try {
|
|
await vi.waitFor(() => expect(firstFetch).toHaveBeenCalledOnce());
|
|
await expect(
|
|
downloadVerifiedRuntimeReleaseWithLease({
|
|
installLockFd: lockFd,
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl: vi.fn<typeof fetch>(),
|
|
}),
|
|
).rejects.toThrow("install lease");
|
|
resolveIndex(new Response(fixture.raw));
|
|
await expect(first).resolves.toMatchObject({ archiveSize: ARCHIVE_BYTES.length });
|
|
} finally {
|
|
closeSync(lockFd);
|
|
}
|
|
});
|
|
|
|
it("retries a transient index response after Retry-After and cancels its body", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-index-retry-"));
|
|
temporaryDirectories.push(directory);
|
|
const canceled = vi.fn();
|
|
const retryBody = new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
controller.enqueue(Buffer.from("temporarily unavailable"));
|
|
},
|
|
cancel: canceled,
|
|
});
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(
|
|
new Response(retryBody, {
|
|
status: 503,
|
|
headers: { "retry-after": "1" },
|
|
}),
|
|
)
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES, { status: 200 }));
|
|
const sleep = vi.fn(async (_delayMs: number) => {});
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
retry: {
|
|
maxAttempts: 3,
|
|
baseDelayMs: 100,
|
|
maxDelayMs: 2_000,
|
|
maxTotalDelayMs: 5_000,
|
|
random: () => 0,
|
|
sleep,
|
|
},
|
|
}),
|
|
).resolves.toMatchObject({ archiveSize: ARCHIVE_BYTES.length });
|
|
expect(sleep).toHaveBeenCalledExactlyOnceWith(1_000);
|
|
expect(canceled).toHaveBeenCalledOnce();
|
|
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
|
});
|
|
|
|
it("reports a clear not-published error when the runtime index is absent", async () => {
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-index-absent-"));
|
|
temporaryDirectories.push(directory);
|
|
const canceled = vi.fn();
|
|
const missingBody = new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
controller.enqueue(Buffer.from("Entry not found"));
|
|
},
|
|
cancel: canceled,
|
|
});
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValue(new Response(missingBody, { status: 404 }));
|
|
|
|
const error = await downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [],
|
|
fetchImpl,
|
|
}).then(
|
|
() => {
|
|
throw new Error("expected the absent runtime index to reject");
|
|
},
|
|
(reason: unknown) => reason,
|
|
);
|
|
|
|
expect(error).toBeInstanceOf(OcrRuntimeNotPublishedError);
|
|
const message = (error as Error).message;
|
|
expect(message).toContain("2.1.0");
|
|
expect(message).toContain("Fast OCR");
|
|
expect(message).toContain("404");
|
|
// Absent means absent: this is not a transient failure and must not retry.
|
|
expect(fetchImpl).toHaveBeenCalledOnce();
|
|
expect(canceled).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("treats a forbidden runtime index as not published rather than a hard error", async () => {
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-index-forbidden-"));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValue(new Response("Forbidden", { status: 403 }));
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [],
|
|
fetchImpl,
|
|
}),
|
|
).rejects.toBeInstanceOf(OcrRuntimeNotPublishedError);
|
|
expect(fetchImpl).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("does not retry an oversized index and cancels its unconsumed body", async () => {
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-index-size-policy-"));
|
|
temporaryDirectories.push(directory);
|
|
const canceled = vi.fn();
|
|
const body = new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
controller.enqueue(Buffer.from("oversized by declaration"));
|
|
},
|
|
cancel: canceled,
|
|
});
|
|
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValue(
|
|
new Response(body, {
|
|
status: 200,
|
|
headers: { "content-length": String(16 * 1024 * 1024 + 1) },
|
|
}),
|
|
);
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [],
|
|
fetchImpl,
|
|
retry: {
|
|
maxAttempts: 3,
|
|
baseDelayMs: 0,
|
|
maxDelayMs: 0,
|
|
maxTotalDelayMs: 0,
|
|
sleep: async () => {},
|
|
},
|
|
}),
|
|
).rejects.toThrow("exceeds its size limit");
|
|
expect(canceled).toHaveBeenCalledOnce();
|
|
expect(fetchImpl).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("retries an index body truncated below its declared content length", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-index-truncated-"));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(
|
|
new Response(fixture.raw.subarray(0, Math.floor(fixture.raw.length / 2)), {
|
|
status: 200,
|
|
headers: { "content-length": String(fixture.raw.length) },
|
|
}),
|
|
)
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES, { status: 200 }));
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
retry: {
|
|
maxAttempts: 2,
|
|
baseDelayMs: 0,
|
|
maxDelayMs: 0,
|
|
maxTotalDelayMs: 0,
|
|
sleep: async () => {},
|
|
},
|
|
}),
|
|
).resolves.toMatchObject({ archiveSize: ARCHIVE_BYTES.length });
|
|
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
|
});
|
|
|
|
it("does not compare decoded index bytes to an encoded Content-Length", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-decoded-index-"));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(
|
|
new Response(fixture.raw, {
|
|
status: 200,
|
|
headers: {
|
|
"content-encoding": "gzip",
|
|
"content-length": String(Math.floor(fixture.raw.length / 2)),
|
|
},
|
|
}),
|
|
)
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES, { status: 200 }));
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
}),
|
|
).resolves.toMatchObject({ archiveSize: ARCHIVE_BYTES.length });
|
|
});
|
|
|
|
it("retries transient fetch failures and surfaces the final bounded-attempt error", async () => {
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-network-attempts-"));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi.fn<typeof fetch>().mockRejectedValue(
|
|
Object.assign(new Error("socket reset by peer"), {
|
|
code: "ECONNRESET",
|
|
}),
|
|
);
|
|
const sleep = vi.fn(async (_delayMs: number) => {});
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [],
|
|
fetchImpl,
|
|
retry: {
|
|
maxAttempts: 3,
|
|
baseDelayMs: 100,
|
|
maxDelayMs: 1_000,
|
|
maxTotalDelayMs: 1_000,
|
|
random: () => 0,
|
|
sleep,
|
|
},
|
|
}),
|
|
).rejects.toThrow(
|
|
"OCR runtime index download failed after 3 attempts (150ms retry delay): OCR runtime network request failed: socket reset by peer",
|
|
);
|
|
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
|
expect(sleep.mock.calls).toEqual([[50], [100]]);
|
|
});
|
|
|
|
it("does not mistake a local TypeError for a transient network failure", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-local-type-error-"));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES, { status: 200 }));
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
onProgress: (_percent, stage) => {
|
|
if (stage === "Downloading OCR runtime") throw new TypeError("progress callback failed");
|
|
},
|
|
retry: {
|
|
maxAttempts: 3,
|
|
baseDelayMs: 0,
|
|
maxDelayMs: 0,
|
|
maxTotalDelayMs: 0,
|
|
sleep: async () => {},
|
|
},
|
|
}),
|
|
).rejects.toThrow("progress callback failed");
|
|
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it("bounds Retry-After and the cumulative retry delay", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-delay-budget-"));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(
|
|
new Response(null, { status: 429, headers: { "retry-after": "999999999" } }),
|
|
)
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES, { status: 200 }));
|
|
const sleep = vi.fn(async (_delayMs: number) => {});
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
retry: {
|
|
maxAttempts: 2,
|
|
baseDelayMs: 50,
|
|
maxDelayMs: 400,
|
|
maxTotalDelayMs: 1_000,
|
|
random: () => 0.5,
|
|
sleep,
|
|
},
|
|
}),
|
|
).resolves.toMatchObject({ archiveSize: ARCHIVE_BYTES.length });
|
|
expect(sleep).toHaveBeenCalledExactlyOnceWith(400);
|
|
});
|
|
|
|
it("honors an HTTP-date Retry-After using the injected clock", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-retry-date-"));
|
|
temporaryDirectories.push(directory);
|
|
const now = Date.UTC(2026, 0, 1, 0, 0, 0);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(
|
|
new Response(null, {
|
|
status: 503,
|
|
headers: { "retry-after": new Date(now + 2_000).toUTCString() },
|
|
}),
|
|
)
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES, { status: 200 }));
|
|
const sleep = vi.fn(async (_delayMs: number) => {});
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
retry: {
|
|
maxAttempts: 2,
|
|
baseDelayMs: 100,
|
|
maxDelayMs: 5_000,
|
|
maxTotalDelayMs: 5_000,
|
|
now: () => now,
|
|
sleep,
|
|
},
|
|
}),
|
|
).resolves.toMatchObject({ archiveSize: ARCHIVE_BYTES.length });
|
|
expect(sleep).toHaveBeenCalledExactlyOnceWith(2_000);
|
|
});
|
|
|
|
it("stops when the cumulative retry-delay budget is exhausted", async () => {
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-delay-exhausted-"));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValue(new Response(null, { status: 503 }));
|
|
const sleep = vi.fn(async (_delayMs: number) => {});
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [],
|
|
fetchImpl,
|
|
retry: {
|
|
maxAttempts: 6,
|
|
baseDelayMs: 100,
|
|
maxDelayMs: 1_000,
|
|
maxTotalDelayMs: 125,
|
|
random: () => 0,
|
|
sleep,
|
|
},
|
|
}),
|
|
).rejects.toThrow("retry delay budget was exhausted after 2 attempts");
|
|
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
|
expect(sleep.mock.calls).toEqual([[50]]);
|
|
});
|
|
|
|
it("shares the total retry-delay budget across index and archive transfers", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-shared-delay-budget-"));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(null, { status: 503 }))
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(new Response(null, { status: 503 }));
|
|
const sleep = vi.fn(async (_delayMs: number) => {});
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
retry: {
|
|
maxAttempts: 3,
|
|
baseDelayMs: 100,
|
|
maxDelayMs: 1_000,
|
|
maxTotalDelayMs: 100,
|
|
random: () => 0.5,
|
|
sleep,
|
|
},
|
|
}),
|
|
).rejects.toThrow("OCR runtime archive download retry delay budget was exhausted");
|
|
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
|
expect(sleep.mock.calls).toEqual([[75]]);
|
|
});
|
|
|
|
it.each([
|
|
{ retry: { maxAttempts: 7 }, message: "retry attempts" },
|
|
{ retry: { maxDelayMs: 30_001 }, message: "retry maximum delay" },
|
|
{ retry: { maxTotalDelayMs: 120_001 }, message: "retry total delay" },
|
|
{
|
|
retry: { baseDelayMs: 1_001, maxDelayMs: 1_000 },
|
|
message: "base delay cannot exceed",
|
|
},
|
|
])("rejects an unbounded retry policy: $message", async ({ retry, message }) => {
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-retry-policy-"));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi.fn<typeof fetch>();
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [],
|
|
fetchImpl,
|
|
retry,
|
|
}),
|
|
).rejects.toThrow(message);
|
|
expect(fetchImpl).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it.each([408, 425, 429, 500, 502, 503, 504])(
|
|
"retries the explicitly allowed transient HTTP status %i",
|
|
async (status) => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), `snapotter-ocr-http-${status}-`));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(null, { status }))
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES, { status: 200 }));
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
retry: {
|
|
maxAttempts: 2,
|
|
baseDelayMs: 0,
|
|
maxDelayMs: 0,
|
|
maxTotalDelayMs: 0,
|
|
sleep: async () => {},
|
|
},
|
|
}),
|
|
).resolves.toMatchObject({ archiveSize: ARCHIVE_BYTES.length });
|
|
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
|
},
|
|
);
|
|
|
|
it.each([400, 401, 403, 404, 409, 422, 501, 505])(
|
|
"does not retry the permanent HTTP status %i",
|
|
async (status) => {
|
|
const directory = mkdtempSync(join(tmpdir(), `snapotter-ocr-http-${status}-`));
|
|
temporaryDirectories.push(directory);
|
|
const canceled = vi.fn();
|
|
const responseBody = new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
controller.enqueue(Buffer.from("permanent failure"));
|
|
},
|
|
cancel: canceled,
|
|
});
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValue(new Response(responseBody, { status }));
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [],
|
|
fetchImpl,
|
|
retry: {
|
|
maxAttempts: 3,
|
|
baseDelayMs: 0,
|
|
maxDelayMs: 0,
|
|
maxTotalDelayMs: 0,
|
|
sleep: async () => {},
|
|
},
|
|
}),
|
|
).rejects.toThrow(`HTTP ${status}`);
|
|
expect(fetchImpl).toHaveBeenCalledOnce();
|
|
expect(canceled).toHaveBeenCalledOnce();
|
|
},
|
|
);
|
|
|
|
it("accepts zero to disable both download watchdogs", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-no-watchdog-"));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES, { status: 200 }));
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
timeoutMs: 0,
|
|
stallTimeoutMs: 0,
|
|
}),
|
|
).resolves.toMatchObject({ archiveSize: ARCHIVE_BYTES.length });
|
|
});
|
|
|
|
it("rejects a fresh download before transfer when the data volume cannot stage it", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-low-space-fresh-"));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES, { status: 200 }));
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
diskFreeBytes: async () => 0n,
|
|
}),
|
|
).rejects.toThrow("insufficient disk space before OCR runtime download");
|
|
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it("rejects insufficient configured memory after authenticating the index but before archive transfer", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-low-memory-"));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES, { status: 200 }));
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
memoryOptions: { effectiveMemoryBytes: 4 * 1024 ** 3 - 1 },
|
|
}),
|
|
).rejects.toThrow("insufficient memory for accurate OCR runtime");
|
|
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it("rejects a resumed download before transfer when remaining space cannot stage it", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-low-space-resume-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
mkdirSync(downloads, { recursive: true });
|
|
const offset = 7;
|
|
writeFileSync(
|
|
join(downloads, `${fixture.artifact.archive.sha256}.part`),
|
|
ARCHIVE_BYTES.subarray(0, offset),
|
|
);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(
|
|
new Response(ARCHIVE_BYTES.subarray(offset), {
|
|
status: 206,
|
|
headers: {
|
|
"content-range": `bytes ${offset}-${ARCHIVE_BYTES.length - 1}/${ARCHIVE_BYTES.length}`,
|
|
},
|
|
}),
|
|
);
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
diskFreeBytes: async () => 0n,
|
|
}),
|
|
).rejects.toThrow("insufficient disk space before OCR runtime download");
|
|
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it("reuses a verified cached archive without requiring room for another installation", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-cached-archive-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
mkdirSync(downloads, { recursive: true });
|
|
writeFileSync(join(downloads, `${fixture.artifact.archive.sha256}.tar.gz`), ARCHIVE_BYTES);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }));
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
diskFreeBytes: async () => 0n,
|
|
}),
|
|
).resolves.toMatchObject({ archiveSize: ARCHIVE_BYTES.length });
|
|
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it("promotes a verified complete partial without requiring duplicate staging space", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-complete-partial-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
mkdirSync(downloads, { recursive: true });
|
|
writeFileSync(join(downloads, `${fixture.artifact.archive.sha256}.part`), ARCHIVE_BYTES);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }));
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
diskFreeBytes: async () => 0n,
|
|
}),
|
|
).resolves.toMatchObject({ archiveSize: ARCHIVE_BYTES.length });
|
|
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
|
expect(readFileSync(join(downloads, `${fixture.artifact.archive.sha256}.tar.gz`))).toEqual(
|
|
ARCHIVE_BYTES,
|
|
);
|
|
});
|
|
|
|
it("follows only bounded HTTPS redirects on the Hugging Face delivery hosts", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-redirect-"));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(
|
|
new Response(null, {
|
|
status: 302,
|
|
headers: { location: "https://cdn-lfs-us-1.hf.co/releases/index" },
|
|
}),
|
|
)
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(
|
|
new Response(null, {
|
|
status: 307,
|
|
headers: { location: "https://cas-bridge.xethub.com/releases/archive" },
|
|
}),
|
|
)
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES, { status: 200 }));
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
}),
|
|
).resolves.toMatchObject({ archiveSize: ARCHIVE_BYTES.length });
|
|
expect(fetchImpl).toHaveBeenCalledTimes(4);
|
|
expect(fetchImpl.mock.calls.every(([, init]) => init?.redirect === "manual")).toBe(true);
|
|
});
|
|
|
|
it("rejects a download redirect outside the trusted delivery hosts", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-evil-redirect-"));
|
|
temporaryDirectories.push(directory);
|
|
const canceled = vi.fn();
|
|
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValue(
|
|
new Response(
|
|
new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
controller.enqueue(Buffer.from("redirect"));
|
|
},
|
|
cancel: canceled,
|
|
}),
|
|
{
|
|
status: 302,
|
|
headers: { location: "https://huggingface.co.attacker.invalid/runtime" },
|
|
},
|
|
),
|
|
);
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
}),
|
|
).rejects.toThrow("redirect is not trusted");
|
|
expect(fetchImpl).toHaveBeenCalledOnce();
|
|
expect(canceled).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("does not retry a malformed redirect location", async () => {
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-malformed-redirect-"));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi.fn<typeof fetch>().mockImplementation(
|
|
async () =>
|
|
new Response(null, {
|
|
status: 302,
|
|
headers: { location: "https://[" },
|
|
}),
|
|
);
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [],
|
|
fetchImpl,
|
|
retry: {
|
|
maxAttempts: 3,
|
|
baseDelayMs: 0,
|
|
maxDelayMs: 0,
|
|
maxTotalDelayMs: 0,
|
|
sleep: async () => {},
|
|
},
|
|
}),
|
|
).rejects.toThrow("download redirect is invalid");
|
|
expect(fetchImpl).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("resumes a partial archive only when the server honors the exact range", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-resume-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
mkdirSync(downloads, { recursive: true });
|
|
const partialPath = join(
|
|
downloads,
|
|
"22553579e29b27b75c9b6375d816f02483bd2e6f76f9ffcc9ae5311422410917.part",
|
|
);
|
|
writeFileSync(partialPath, ARCHIVE_BYTES.subarray(0, 10), { flag: "wx" });
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockImplementationOnce(async (_url, init) => {
|
|
expect(new Headers(init?.headers).get("range")).toBe("bytes=10-");
|
|
return new Response(ARCHIVE_BYTES.subarray(10), {
|
|
status: 206,
|
|
headers: {
|
|
"content-range": `bytes 10-${ARCHIVE_BYTES.length - 1}/${ARCHIVE_BYTES.length}`,
|
|
},
|
|
});
|
|
});
|
|
|
|
const result = await downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
});
|
|
|
|
expect(readFileSync(result.archivePath)).toEqual(ARCHIVE_BYTES);
|
|
});
|
|
|
|
it("discards a corrupt resumed prefix and retries once from byte zero", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-corrupt-resume-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
mkdirSync(downloads, { recursive: true });
|
|
const offset = 10;
|
|
const partialPath = join(downloads, `${fixture.artifact.archive.sha256}.part`);
|
|
writeFileSync(partialPath, Buffer.alloc(offset, 0x78), { flag: "wx" });
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockImplementationOnce(async (_url, init) => {
|
|
expect(new Headers(init?.headers).get("range")).toBe(`bytes=${offset}-`);
|
|
return new Response(ARCHIVE_BYTES.subarray(offset), {
|
|
status: 206,
|
|
headers: {
|
|
"content-range": `bytes ${offset}-${ARCHIVE_BYTES.length - 1}/${ARCHIVE_BYTES.length}`,
|
|
},
|
|
});
|
|
})
|
|
.mockImplementationOnce(async (_url, init) => {
|
|
expect(new Headers(init?.headers).get("range")).toBeNull();
|
|
return new Response(ARCHIVE_BYTES, { status: 200 });
|
|
});
|
|
|
|
const result = await downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
retry: {
|
|
maxAttempts: 2,
|
|
baseDelayMs: 0,
|
|
maxDelayMs: 0,
|
|
maxTotalDelayMs: 0,
|
|
sleep: async () => {},
|
|
},
|
|
});
|
|
|
|
expect(readFileSync(result.archivePath)).toEqual(ARCHIVE_BYTES);
|
|
expect(existsSync(partialPath)).toBe(false);
|
|
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
|
});
|
|
|
|
it("rejects a non-exact resume range without retrying and cancels its body", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-range-policy-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
mkdirSync(downloads, { recursive: true });
|
|
const offset = 10;
|
|
writeFileSync(
|
|
join(downloads, `${fixture.artifact.archive.sha256}.part`),
|
|
ARCHIVE_BYTES.subarray(0, offset),
|
|
);
|
|
const canceled = vi.fn();
|
|
const body = new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
controller.enqueue(ARCHIVE_BYTES.subarray(offset));
|
|
},
|
|
cancel: canceled,
|
|
});
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(
|
|
new Response(body, {
|
|
status: 206,
|
|
headers: {
|
|
"content-range": `bytes ${offset}-${ARCHIVE_BYTES.length - 2}/${ARCHIVE_BYTES.length}`,
|
|
},
|
|
}),
|
|
);
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
stallTimeoutMs: 10,
|
|
retry: { maxAttempts: 1 },
|
|
}),
|
|
).rejects.toThrow("mismatched resume range");
|
|
expect(canceled).toHaveBeenCalledOnce();
|
|
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it("preserves and safely resumes archive bytes after a transient body failure", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-body-retry-"));
|
|
temporaryDirectories.push(directory);
|
|
const prefixLength = 9;
|
|
let firstPull = true;
|
|
const interruptedBody = new ReadableStream<Uint8Array>({
|
|
pull(controller) {
|
|
if (firstPull) {
|
|
firstPull = false;
|
|
controller.enqueue(ARCHIVE_BYTES.subarray(0, prefixLength));
|
|
return;
|
|
}
|
|
controller.error(new TypeError("terminated"));
|
|
},
|
|
});
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(new Response(interruptedBody, { status: 200 }))
|
|
.mockImplementationOnce(async (_url, init) => {
|
|
expect(new Headers(init?.headers).get("range")).toBe(`bytes=${prefixLength}-`);
|
|
return new Response(ARCHIVE_BYTES.subarray(prefixLength), {
|
|
status: 206,
|
|
headers: {
|
|
"content-range": `bytes ${prefixLength}-${ARCHIVE_BYTES.length - 1}/${ARCHIVE_BYTES.length}`,
|
|
},
|
|
});
|
|
});
|
|
|
|
const result = await downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
retry: {
|
|
maxAttempts: 3,
|
|
baseDelayMs: 0,
|
|
maxDelayMs: 0,
|
|
maxTotalDelayMs: 0,
|
|
random: () => 0,
|
|
sleep: async () => {},
|
|
},
|
|
});
|
|
|
|
expect(readFileSync(result.archivePath)).toEqual(ARCHIVE_BYTES);
|
|
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
|
});
|
|
|
|
it("treats an unexpectedly truncated archive response as resumable transport failure", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-truncated-retry-"));
|
|
temporaryDirectories.push(directory);
|
|
const prefixLength = 8;
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES.subarray(0, prefixLength), { status: 200 }))
|
|
.mockImplementationOnce(async (_url, init) => {
|
|
expect(new Headers(init?.headers).get("range")).toBe(`bytes=${prefixLength}-`);
|
|
return new Response(ARCHIVE_BYTES.subarray(prefixLength), {
|
|
status: 206,
|
|
headers: {
|
|
"content-range": `bytes ${prefixLength}-${ARCHIVE_BYTES.length - 1}/${ARCHIVE_BYTES.length}`,
|
|
},
|
|
});
|
|
});
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
retry: {
|
|
maxAttempts: 2,
|
|
baseDelayMs: 0,
|
|
maxDelayMs: 0,
|
|
maxTotalDelayMs: 0,
|
|
sleep: async () => {},
|
|
},
|
|
}),
|
|
).resolves.toMatchObject({ archiveSize: ARCHIVE_BYTES.length });
|
|
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
|
});
|
|
|
|
it("does not retry an archive size violation and cancels the abandoned body", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-size-policy-"));
|
|
temporaryDirectories.push(directory);
|
|
const canceled = vi.fn();
|
|
const oversizedBody = new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
controller.enqueue(Buffer.concat([ARCHIVE_BYTES, Buffer.from("!")]));
|
|
},
|
|
cancel: canceled,
|
|
});
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(new Response(oversizedBody, { status: 200 }));
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
retry: {
|
|
maxAttempts: 3,
|
|
baseDelayMs: 0,
|
|
maxDelayMs: 0,
|
|
maxTotalDelayMs: 0,
|
|
sleep: async () => {},
|
|
},
|
|
}),
|
|
).rejects.toThrow("exceeded its signed size");
|
|
expect(canceled).toHaveBeenCalledOnce();
|
|
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it("does not retry an archive hash failure", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-hash-policy-"));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(
|
|
new Response(Buffer.alloc(ARCHIVE_BYTES.length, 0x78), { status: 200 }),
|
|
);
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
retry: {
|
|
maxAttempts: 3,
|
|
baseDelayMs: 0,
|
|
maxDelayMs: 0,
|
|
maxTotalDelayMs: 0,
|
|
sleep: async () => {},
|
|
},
|
|
}),
|
|
).rejects.toThrow("failed SHA-256 verification");
|
|
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it("does not retry signed-index schema or signature failures", async () => {
|
|
const fixture = signedIndex();
|
|
const tampered = structuredClone(fixture.index);
|
|
tampered.artifacts[0].archive.size += 1;
|
|
const cases = [
|
|
{ label: "schema", raw: Buffer.from("{}"), message: "schema" },
|
|
{
|
|
label: "signature",
|
|
raw: Buffer.from(canonicalRuntimeJson(tampered)),
|
|
message: "signature",
|
|
},
|
|
];
|
|
|
|
for (const testCase of cases) {
|
|
const directory = mkdtempSync(join(tmpdir(), `snapotter-ocr-${testCase.label}-policy-`));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValue(new Response(testCase.raw, { status: 200 }));
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
retry: {
|
|
maxAttempts: 3,
|
|
baseDelayMs: 0,
|
|
maxDelayMs: 0,
|
|
maxTotalDelayMs: 0,
|
|
sleep: async () => {},
|
|
},
|
|
}),
|
|
).rejects.toThrow(testCase.message);
|
|
expect(fetchImpl).toHaveBeenCalledOnce();
|
|
}
|
|
});
|
|
|
|
it("does not follow a symlink planted while creating a partial archive", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-create-swap-"));
|
|
temporaryDirectories.push(directory);
|
|
const outsidePath = join(directory, "outside");
|
|
const outsideBytes = Buffer.from("must not be overwritten");
|
|
writeFileSync(outsidePath, outsideBytes);
|
|
const partialPath = join(
|
|
directory,
|
|
"v3",
|
|
"downloads",
|
|
`${fixture.artifact.archive.sha256}.part`,
|
|
);
|
|
const canceled = vi.fn();
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockImplementationOnce(async () => {
|
|
symlinkSync(outsidePath, partialPath);
|
|
return new Response(
|
|
new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
controller.enqueue(ARCHIVE_BYTES);
|
|
},
|
|
cancel: canceled,
|
|
}),
|
|
{ status: 200 },
|
|
);
|
|
});
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
}),
|
|
).rejects.toThrow();
|
|
expect(readFileSync(outsidePath)).toEqual(outsideBytes);
|
|
expect(canceled).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("does not follow a partial archive swapped to a symlink during resume", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-resume-swap-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
mkdirSync(downloads, { recursive: true });
|
|
const partialPath = join(downloads, `${fixture.artifact.archive.sha256}.part`);
|
|
const prefix = ARCHIVE_BYTES.subarray(0, 10);
|
|
writeFileSync(partialPath, prefix, { flag: "wx" });
|
|
const outsidePath = join(directory, "outside");
|
|
writeFileSync(outsidePath, prefix);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockImplementationOnce(async (_url, init) => {
|
|
expect(new Headers(init?.headers).get("range")).toBe("bytes=10-");
|
|
rmSync(partialPath);
|
|
symlinkSync(outsidePath, partialPath);
|
|
return new Response(ARCHIVE_BYTES.subarray(10), {
|
|
status: 206,
|
|
headers: {
|
|
"content-range": `bytes 10-${ARCHIVE_BYTES.length - 1}/${ARCHIVE_BYTES.length}`,
|
|
},
|
|
});
|
|
});
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
}),
|
|
).rejects.toThrow();
|
|
expect(readFileSync(outsidePath)).toEqual(prefix);
|
|
});
|
|
|
|
it("cancels a full-response body when a partial is swapped before safe restart", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-restart-swap-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
mkdirSync(downloads, { recursive: true });
|
|
const partialPath = join(downloads, `${fixture.artifact.archive.sha256}.part`);
|
|
const prefix = ARCHIVE_BYTES.subarray(0, 10);
|
|
writeFileSync(partialPath, prefix, { flag: "wx" });
|
|
const originalIdentity = statSync(partialPath, { bigint: true });
|
|
const canceled = vi.fn();
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockImplementationOnce(async (_url, init) => {
|
|
expect(new Headers(init?.headers).get("range")).toBe("bytes=10-");
|
|
expect(hasOpenDescriptorFor(originalIdentity)).toBe(true);
|
|
const replacementPath = `${partialPath}.replacement`;
|
|
writeFileSync(replacementPath, prefix, { flag: "wx" });
|
|
rmSync(partialPath);
|
|
renameSync(replacementPath, partialPath);
|
|
return new Response(
|
|
new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
controller.enqueue(ARCHIVE_BYTES);
|
|
controller.close();
|
|
},
|
|
cancel: canceled,
|
|
}),
|
|
{ status: 200 },
|
|
);
|
|
});
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
}),
|
|
).rejects.toThrow("changed while the download was in progress");
|
|
expect(canceled).toHaveBeenCalledOnce();
|
|
expect(readFileSync(partialPath)).toEqual(prefix);
|
|
expect(hasOpenDescriptorFor(originalIdentity)).toBe(false);
|
|
});
|
|
|
|
it("rejects a same-size regular-file swap during resume", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-resume-inode-swap-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
mkdirSync(downloads, { recursive: true });
|
|
const partialPath = join(downloads, `${fixture.artifact.archive.sha256}.part`);
|
|
const prefix = ARCHIVE_BYTES.subarray(0, 10);
|
|
writeFileSync(partialPath, prefix, { flag: "wx" });
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockImplementationOnce(async (_url, init) => {
|
|
expect(new Headers(init?.headers).get("range")).toBe("bytes=10-");
|
|
const replacementPath = `${partialPath}.replacement`;
|
|
writeFileSync(replacementPath, prefix, { flag: "wx" });
|
|
rmSync(partialPath);
|
|
renameSync(replacementPath, partialPath);
|
|
return new Response(ARCHIVE_BYTES.subarray(10), {
|
|
status: 206,
|
|
headers: {
|
|
"content-range": `bytes 10-${ARCHIVE_BYTES.length - 1}/${ARCHIVE_BYTES.length}`,
|
|
},
|
|
});
|
|
});
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
}),
|
|
).rejects.toThrow("changed while the download was in progress");
|
|
expect(readFileSync(partialPath)).toEqual(prefix);
|
|
});
|
|
|
|
it("repairs an oversized digest-named index left by an interrupted write", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-index-cache-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
mkdirSync(downloads, { recursive: true });
|
|
const digest = createHash("sha256").update(fixture.raw).digest("hex");
|
|
const indexPath = join(downloads, `${digest}.index.json`);
|
|
writeFileSync(indexPath, Buffer.alloc(17 * 1024 * 1024));
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl: vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES, { status: 200 })),
|
|
}),
|
|
).resolves.toMatchObject({ indexPath });
|
|
expect(readFileSync(indexPath)).toEqual(fixture.raw);
|
|
});
|
|
|
|
it("repairs a truncated digest-named index left by a prior crash", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-truncated-index-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
mkdirSync(downloads, { recursive: true });
|
|
const digest = createHash("sha256").update(fixture.raw).digest("hex");
|
|
const indexPath = join(downloads, `${digest}.index.json`);
|
|
writeFileSync(indexPath, fixture.raw.subarray(0, Math.floor(fixture.raw.length / 2)));
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES, { status: 200 }));
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
}),
|
|
).resolves.toMatchObject({ indexPath });
|
|
expect(readFileSync(indexPath)).toEqual(fixture.raw);
|
|
});
|
|
|
|
it("never replaces a digest-named index symlink while repairing the cache", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-index-symlink-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
mkdirSync(downloads, { recursive: true });
|
|
const digest = createHash("sha256").update(fixture.raw).digest("hex");
|
|
const outsidePath = join(directory, "outside-index");
|
|
const outsideBytes = Buffer.from("must not be replaced");
|
|
writeFileSync(outsidePath, outsideBytes);
|
|
symlinkSync(outsidePath, join(downloads, `${digest}.index.json`));
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl: vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 })),
|
|
}),
|
|
).rejects.toThrow("private regular file");
|
|
expect(readFileSync(outsidePath)).toEqual(outsideBytes);
|
|
});
|
|
|
|
it("cancels and retries a stalled response", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-stall-retry-"));
|
|
temporaryDirectories.push(directory);
|
|
const canceled = vi.fn();
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(
|
|
new Response(
|
|
new ReadableStream<Uint8Array>({
|
|
pull() {
|
|
return new Promise(() => {});
|
|
},
|
|
cancel: canceled,
|
|
}),
|
|
{ status: 200 },
|
|
),
|
|
)
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES, { status: 200 }));
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
timeoutMs: 1_000,
|
|
stallTimeoutMs: 10,
|
|
retry: {
|
|
maxAttempts: 2,
|
|
baseDelayMs: 0,
|
|
maxDelayMs: 0,
|
|
maxTotalDelayMs: 0,
|
|
sleep: async () => {},
|
|
},
|
|
}),
|
|
).resolves.toMatchObject({ archiveSize: ARCHIVE_BYTES.length });
|
|
expect(canceled).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("aborts a persistently stalled response instead of holding the install queue forever", async () => {
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-stall-"));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValue(
|
|
new Response(
|
|
new ReadableStream<Uint8Array>({
|
|
pull() {
|
|
return new Promise(() => {});
|
|
},
|
|
}),
|
|
{ status: 200 },
|
|
),
|
|
);
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [],
|
|
fetchImpl,
|
|
timeoutMs: 1_000,
|
|
stallTimeoutMs: 10,
|
|
retry: { maxAttempts: 1 },
|
|
}),
|
|
).rejects.toThrow("stalled");
|
|
});
|
|
|
|
it("enforces the overall deadline while a response body is stalled", async () => {
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-deadline-"));
|
|
temporaryDirectories.push(directory);
|
|
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValue(
|
|
new Response(
|
|
new ReadableStream<Uint8Array>({
|
|
pull() {
|
|
return new Promise(() => {});
|
|
},
|
|
}),
|
|
{ status: 200 },
|
|
),
|
|
);
|
|
|
|
const startedAt = Date.now();
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [],
|
|
fetchImpl,
|
|
timeoutMs: 10,
|
|
stallTimeoutMs: 1_000,
|
|
}),
|
|
).rejects.toThrow("timed out after 10ms");
|
|
expect(Date.now() - startedAt).toBeLessThan(500);
|
|
});
|
|
|
|
it("does not let an unresponsive body cancellation defeat the stall watchdog", async () => {
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-cancel-watchdog-"));
|
|
temporaryDirectories.push(directory);
|
|
const canceled = vi.fn(() => new Promise<void>(() => {}));
|
|
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValue(
|
|
new Response(
|
|
new ReadableStream<Uint8Array>({
|
|
pull() {
|
|
return new Promise(() => {});
|
|
},
|
|
cancel: canceled,
|
|
}),
|
|
{ status: 200 },
|
|
),
|
|
);
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [],
|
|
fetchImpl,
|
|
timeoutMs: 500,
|
|
stallTimeoutMs: 10,
|
|
retry: { maxAttempts: 1 },
|
|
}),
|
|
).rejects.toThrow("stalled");
|
|
expect(canceled).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("rejects a verified cached archive when hashing crosses the overall deadline", async () => {
|
|
const archiveSize = 128 * 1024 * 1024;
|
|
const archiveSha256 = "254bcc3fc4f27172636df4bf32de9f107f620d559b20d760197e452b97453917";
|
|
const fixture = signedIndex({ archiveSha256, archiveSize });
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-cached-deadline-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
mkdirSync(downloads, { recursive: true });
|
|
const archivePath = join(downloads, `${archiveSha256}.tar.gz`);
|
|
writeFileSync(archivePath, Buffer.alloc(0));
|
|
truncateSync(archivePath, archiveSize);
|
|
const nativeSetTimeout = globalThis.setTimeout;
|
|
let startDeadline: (() => void) | undefined;
|
|
vi.spyOn(globalThis, "setTimeout").mockImplementation(((handler, delay, ...args) => {
|
|
if (delay !== 20 || startDeadline) return nativeSetTimeout(handler, delay, ...args);
|
|
const placeholder = nativeSetTimeout(() => {}, 60_000);
|
|
startDeadline = () => {
|
|
clearTimeout(placeholder);
|
|
nativeSetTimeout(handler, delay, ...args);
|
|
};
|
|
return placeholder;
|
|
}) as typeof setTimeout);
|
|
const fetchImpl = vi.fn<typeof fetch>().mockImplementationOnce(async () => {
|
|
if (!startDeadline) throw new Error("overall deadline was not registered before fetch");
|
|
startDeadline();
|
|
return new Response(fixture.raw, { status: 200 });
|
|
});
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
timeoutMs: 20,
|
|
}),
|
|
).rejects.toThrow("timed out after 20ms");
|
|
expect(fetchImpl).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("clears the built-in retry timer when the overall deadline aborts", async () => {
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-retry-abort-"));
|
|
temporaryDirectories.push(directory);
|
|
const nativeSetTimeout = globalThis.setTimeout;
|
|
const retryTimers: ReturnType<typeof setTimeout>[] = [];
|
|
let startDeadline: (() => void) | undefined;
|
|
vi.spyOn(globalThis, "setTimeout").mockImplementation(((handler, delay, ...args) => {
|
|
// Defer the overall 100ms deadline until the first fetch is dispatched, so a
|
|
// loaded CI runner cannot fire it before fetch is even called (the source of
|
|
// this test's flakiness). Mirrors the deferred-deadline trick used above.
|
|
if (delay === 100 && !startDeadline) {
|
|
const placeholder = nativeSetTimeout(() => {}, 60_000);
|
|
startDeadline = () => {
|
|
clearTimeout(placeholder);
|
|
nativeSetTimeout(handler, delay, ...args);
|
|
};
|
|
return placeholder;
|
|
}
|
|
const timer = nativeSetTimeout(handler, delay, ...args);
|
|
if (delay === 500) retryTimers.push(timer);
|
|
return timer;
|
|
}) as typeof setTimeout);
|
|
const clearTimeoutSpy = vi.spyOn(globalThis, "clearTimeout");
|
|
let armed = false;
|
|
const fetchImpl = vi.fn<typeof fetch>().mockImplementation(async () => {
|
|
if (!armed) {
|
|
if (!startDeadline) throw new Error("overall deadline was not registered before fetch");
|
|
startDeadline();
|
|
armed = true;
|
|
}
|
|
return new Response("temporary outage", { status: 503 });
|
|
});
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [],
|
|
fetchImpl,
|
|
timeoutMs: 100,
|
|
retry: {
|
|
maxAttempts: 2,
|
|
baseDelayMs: 1_000,
|
|
maxDelayMs: 1_000,
|
|
random: () => 0,
|
|
},
|
|
}),
|
|
).rejects.toThrow("timed out after 100ms");
|
|
expect(fetchImpl).toHaveBeenCalledOnce();
|
|
expect(retryTimers).toHaveLength(1);
|
|
expect(clearTimeoutSpy).toHaveBeenCalledWith(retryTimers[0]);
|
|
});
|
|
|
|
it("roots serialized cache mutations in descriptors and a group-shared quarantine", () => {
|
|
const source = readFileSync(
|
|
join(process.cwd(), "apps/api/src/lib/ocr-runtime-install.ts"),
|
|
"utf8",
|
|
);
|
|
|
|
expect(source).toContain("async function quarantineAndRemove");
|
|
expect(source).toContain("/proc/self/fd/");
|
|
expect(source).toContain(".trash");
|
|
expect(source).toContain("observed.isDirectory() ? 0o5002 : 0o7002");
|
|
expect(source).not.toMatch(/await assertPathIdentity\(path, expected\);\s*await rm\(path\)/u);
|
|
expect(source).not.toContain("await rm(finalPath, { force: true });");
|
|
|
|
const entrypoint = readFileSync(join(process.cwd(), "docker/entrypoint.sh"), "utf8");
|
|
expect(entrypoint).toMatch(/^umask 0007$/mu);
|
|
});
|
|
|
|
it.skipIf(process.platform !== "linux")(
|
|
"never writes through a downloads parent swapped to an external symlink",
|
|
async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-parent-swap-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
const detachedDownloads = join(directory, "detached-downloads");
|
|
const outside = join(directory, "outside");
|
|
mkdirSync(outside);
|
|
const fetchImpl = vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockImplementationOnce(async () => {
|
|
renameSync(downloads, detachedDownloads);
|
|
symlinkSync(outside, downloads);
|
|
return new Response(ARCHIVE_BYTES, { status: 200 });
|
|
});
|
|
|
|
await expect(
|
|
downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl,
|
|
}),
|
|
).rejects.toThrow("download cache directory changed");
|
|
expect(readdirSync(outside)).toEqual([]);
|
|
expect(readdirSync(detachedDownloads)).toContain(`${fixture.artifact.archive.sha256}.tar.gz`);
|
|
},
|
|
);
|
|
|
|
it("reconciles crash-stranded quarantine links before validating cached objects", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-trash-reconcile-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
const trash = join(downloads, ".trash");
|
|
mkdirSync(trash, { recursive: true, mode: 0o700 });
|
|
const partialName = `${fixture.artifact.archive.sha256}.part`;
|
|
const temporaryTrashPath = join(trash, "pending");
|
|
writeFileSync(temporaryTrashPath, ARCHIVE_BYTES);
|
|
const identity = statSync(temporaryTrashPath, { bigint: true });
|
|
const quarantineName = [
|
|
"delete",
|
|
identity.dev,
|
|
identity.ino,
|
|
identity.size,
|
|
"00000000-0000-4000-8000-000000000000",
|
|
partialName,
|
|
].join("-");
|
|
const quarantinePath = join(trash, quarantineName);
|
|
renameSync(temporaryTrashPath, quarantinePath);
|
|
linkSync(quarantinePath, join(downloads, partialName));
|
|
expect(statSync(quarantinePath).nlink).toBe(2);
|
|
|
|
const result = await downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl: vi.fn<typeof fetch>().mockResolvedValueOnce(new Response(fixture.raw)),
|
|
});
|
|
|
|
expect(readFileSync(result.archivePath)).toEqual(ARCHIVE_BYTES);
|
|
expect(readdirSync(trash)).toEqual([]);
|
|
});
|
|
|
|
it("finishes an interrupted mismatched-object restore idempotently", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-trash-restore-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
const trash = join(downloads, ".trash");
|
|
mkdirSync(trash, { recursive: true, mode: 0o2770 });
|
|
const partialName = `${fixture.artifact.archive.sha256}.part`;
|
|
const quarantinePath = join(trash, "pending");
|
|
writeFileSync(quarantinePath, ARCHIVE_BYTES);
|
|
const actual = statSync(quarantinePath, { bigint: true });
|
|
const quarantineName = [
|
|
"delete",
|
|
actual.dev,
|
|
actual.ino + 1n,
|
|
actual.size,
|
|
"00000000-0000-4000-8000-000000000000",
|
|
partialName,
|
|
].join("-");
|
|
const encodedPath = join(trash, quarantineName);
|
|
renameSync(quarantinePath, encodedPath);
|
|
linkSync(encodedPath, join(downloads, partialName));
|
|
expect(statSync(encodedPath).nlink).toBe(2);
|
|
|
|
const result = await downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl: vi.fn<typeof fetch>().mockResolvedValueOnce(new Response(fixture.raw)),
|
|
});
|
|
|
|
expect(readFileSync(result.archivePath)).toEqual(ARCHIVE_BYTES);
|
|
expect(readdirSync(trash)).toEqual([]);
|
|
});
|
|
|
|
it("reconciles crash-stranded verified-index temporary files", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-index-temp-reconcile-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
mkdirSync(downloads, { recursive: true });
|
|
const temporaryName = `.${"b".repeat(64)}.00000000-0000-4000-8000-000000000000.tmp`;
|
|
writeFileSync(join(downloads, temporaryName), fixture.raw);
|
|
|
|
await downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl: vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw))
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES)),
|
|
});
|
|
|
|
expect(existsSync(join(downloads, temporaryName))).toBe(false);
|
|
});
|
|
|
|
it("removes verified release objects after a successful activation", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-cleanup-"));
|
|
temporaryDirectories.push(directory);
|
|
const release = await downloadVerifiedRuntimeRelease({
|
|
aiDataDir: directory,
|
|
bundleRepo: "snapotter-hq/feature-bundles",
|
|
version: "2.1.0",
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
fetchImpl: vi
|
|
.fn<typeof fetch>()
|
|
.mockResolvedValueOnce(new Response(fixture.raw, { status: 200 }))
|
|
.mockResolvedValueOnce(new Response(ARCHIVE_BYTES, { status: 200 })),
|
|
});
|
|
|
|
await cleanupDownloadedRuntimeRelease(directory, release);
|
|
|
|
expect(existsSync(release.indexPath)).toBe(false);
|
|
expect(existsSync(release.archivePath)).toBe(false);
|
|
});
|
|
|
|
it("purges completed, temporary, partial, and symlink download cache entries", async () => {
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-purge-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
const outside = join(directory, "outside");
|
|
mkdirSync(downloads, { recursive: true });
|
|
writeFileSync(outside, "keep");
|
|
for (const name of ["archive.tar.gz", "archive.part", "index.json", ".index.tmp"]) {
|
|
writeFileSync(join(downloads, name), name);
|
|
}
|
|
symlinkSync(outside, join(downloads, "untrusted-link"));
|
|
|
|
await purgeOcrRuntimeDownloads(directory);
|
|
|
|
expect(existsSync(downloads)).toBe(true);
|
|
expect(readFileSync(outside, "utf8")).toBe("keep");
|
|
expect(existsSync(join(downloads, "archive.tar.gz"))).toBe(false);
|
|
expect(existsSync(join(downloads, "archive.part"))).toBe(false);
|
|
expect(existsSync(join(downloads, "index.json"))).toBe(false);
|
|
expect(existsSync(join(downloads, ".index.tmp"))).toBe(false);
|
|
expect(existsSync(join(downloads, "untrusted-link"))).toBe(false);
|
|
});
|
|
|
|
it("refuses cleanup outside the cache and unexpected cache directories", async () => {
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-cleanup-safety-"));
|
|
temporaryDirectories.push(directory);
|
|
const downloads = join(directory, "v3", "downloads");
|
|
const outside = join(directory, "outside.tar.gz");
|
|
mkdirSync(join(downloads, "unexpected"), { recursive: true });
|
|
writeFileSync(outside, "keep");
|
|
|
|
await expect(
|
|
cleanupDownloadedRuntimeRelease(directory, {
|
|
indexPath: outside,
|
|
archivePath: join(downloads, "archive.tar.gz"),
|
|
}),
|
|
).rejects.toThrow("outside the download cache");
|
|
expect(existsSync(outside)).toBe(true);
|
|
await expect(purgeOcrRuntimeDownloads(directory)).rejects.toThrow("Unexpected entry");
|
|
});
|
|
});
|
|
|
|
describe("buildOcrRuntimeInstallerCommand", () => {
|
|
it("uses the immutable system Python, authenticated local bytes, and a functional smoke", () => {
|
|
const fixture = signedIndex();
|
|
const verified = verifyRuntimeIndex(fixture.raw, TARGET, [fixture.trustKey], "2.1.0");
|
|
vi.stubEnv("SNAPOTTER_SYSTEM_PYTHON", undefined);
|
|
const command = buildOcrRuntimeInstallerCommand({
|
|
aiDataDir: "/data/ai",
|
|
installerPath: "/app/install_runtime.py",
|
|
release: {
|
|
...verified,
|
|
indexPath: "/data/ai/v3/downloads/index.json",
|
|
archivePath: "/data/ai/v3/downloads/runtime.tar.gz",
|
|
},
|
|
});
|
|
|
|
expect(command.executable).toBe("/usr/bin/python3");
|
|
expect(command.args).toEqual([
|
|
"/app/install_runtime.py",
|
|
"install",
|
|
"--ai-data-dir",
|
|
"/data/ai",
|
|
"--index",
|
|
"/data/ai/v3/downloads/index.json",
|
|
"--archive",
|
|
"/data/ai/v3/downloads/runtime.tar.gz",
|
|
"--family",
|
|
"ocr",
|
|
"--target",
|
|
TARGET,
|
|
"--expected-index-sha256",
|
|
createHash("sha256").update(fixture.raw).digest("hex"),
|
|
"--smoke-command",
|
|
JSON.stringify(["{runtime}/venv/bin/python", "{runtime}/ocr_runner.py", "--smoke"]),
|
|
"--preverified-index",
|
|
]);
|
|
expect(command.env).not.toHaveProperty("PYTHONPATH");
|
|
expect(command.env).not.toHaveProperty("PYTHONHOME");
|
|
expect(command.env.SNAPOTTER_NETWORK_DISABLED).toBe("1");
|
|
});
|
|
|
|
it("keeps explicit and environment system-Python overrides", () => {
|
|
const fixture = signedIndex();
|
|
const verified = verifyRuntimeIndex(fixture.raw, TARGET, [fixture.trustKey], "2.1.0");
|
|
const options = {
|
|
aiDataDir: "/data/ai",
|
|
release: {
|
|
...verified,
|
|
indexPath: "/data/ai/v3/downloads/index.json",
|
|
archivePath: "/data/ai/v3/downloads/runtime.tar.gz",
|
|
},
|
|
};
|
|
vi.stubEnv("SNAPOTTER_SYSTEM_PYTHON", "/custom/system-python");
|
|
|
|
expect(buildOcrRuntimeInstallerCommand(options).executable).toBe("/custom/system-python");
|
|
expect(
|
|
buildOcrRuntimeInstallerCommand({ ...options, pythonPath: "/explicit/system-python" })
|
|
.executable,
|
|
).toBe("/explicit/system-python");
|
|
});
|
|
});
|
|
|
|
describe("prepareOfflineRuntimeRelease", () => {
|
|
it("classifies malformed signed-index bytes as import validation", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-offline-invalid-index-"));
|
|
temporaryDirectories.push(directory);
|
|
const indexPath = join(directory, "ocr-index.json");
|
|
writeFileSync(indexPath, "not canonical JSON");
|
|
|
|
await expect(
|
|
prepareOfflineRuntimeIndex({
|
|
indexPath,
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
version: "2.1.0",
|
|
}),
|
|
).rejects.toBeInstanceOf(OcrRuntimeImportValidationError);
|
|
});
|
|
|
|
it("keeps invalid trust-key configuration out of import validation", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-offline-invalid-trust-"));
|
|
temporaryDirectories.push(directory);
|
|
const indexPath = join(directory, "ocr-index.json");
|
|
writeFileSync(indexPath, fixture.raw);
|
|
|
|
const rejection = prepareOfflineRuntimeIndex({
|
|
indexPath,
|
|
target: TARGET,
|
|
trustKeys: [{ ...fixture.trustKey, publicKey: "not a public key" }],
|
|
version: "2.1.0",
|
|
});
|
|
await expect(rejection).rejects.not.toBeInstanceOf(OcrRuntimeImportValidationError);
|
|
await expect(rejection).rejects.toThrow(/trusted OCR runtime key.*invalid/i);
|
|
});
|
|
|
|
it("authenticates the signed index before any offline archive bytes are accepted", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-offline-index-"));
|
|
temporaryDirectories.push(directory);
|
|
const indexPath = join(directory, "ocr-index.json");
|
|
writeFileSync(indexPath, fixture.raw);
|
|
|
|
const verified = await prepareOfflineRuntimeIndex({
|
|
indexPath,
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
version: "2.1.0",
|
|
});
|
|
|
|
expect(verified.archiveSize).toBe(ARCHIVE_BYTES.length);
|
|
expect(verified.archiveExpandedSize).toBe(1_000);
|
|
expect(verified.canonicalIndexBytes).toBe(fixture.raw.byteLength);
|
|
});
|
|
|
|
it("reserves signed archive, expansion, metadata, and index capacity before import", async () => {
|
|
const required = ARCHIVE_BYTES.length + 1_000 + 16 * 1024 * 1024 + 1024 * 1024 + 777;
|
|
const diskFreeBytes = vi.fn(async () => BigInt(required - 1));
|
|
|
|
await expect(
|
|
assertOcrRuntimeInstallDiskSpace({
|
|
path: "/data/ai",
|
|
remainingArchiveBytes: ARCHIVE_BYTES.length,
|
|
expandedSize: 1_000,
|
|
authenticatedIndexBytes: 777,
|
|
operation: "import",
|
|
diskFreeBytes,
|
|
}),
|
|
).rejects.toThrow(`insufficient disk space before OCR runtime import: ${required} bytes`);
|
|
expect(diskFreeBytes).toHaveBeenCalledWith("/data/ai");
|
|
});
|
|
|
|
it("authenticates both local files and selects the import transaction", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-offline-"));
|
|
temporaryDirectories.push(directory);
|
|
const indexPath = join(directory, "ocr-index.json");
|
|
const archivePath = join(directory, "ocr.tar.gz");
|
|
writeFileSync(indexPath, fixture.raw);
|
|
writeFileSync(archivePath, ARCHIVE_BYTES);
|
|
|
|
const release = await prepareOfflineRuntimeRelease({
|
|
indexPath,
|
|
archivePath,
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
version: "2.1.0",
|
|
});
|
|
const command = buildOcrRuntimeInstallerCommand({
|
|
aiDataDir: directory,
|
|
release,
|
|
mode: "import",
|
|
pythonPath: "/opt/venv/bin/python3",
|
|
installerPath: "/app/install_runtime.py",
|
|
});
|
|
|
|
expect(command.args[1]).toBe("import");
|
|
writeFileSync(archivePath, Buffer.from("tampered"));
|
|
const rejection = prepareOfflineRuntimeRelease({
|
|
indexPath,
|
|
archivePath,
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
version: "2.1.0",
|
|
});
|
|
await expect(rejection).rejects.toBeInstanceOf(OcrRuntimeImportValidationError);
|
|
await expect(rejection).rejects.toThrow("signed index");
|
|
});
|
|
|
|
it("rejects insufficient memory before reading or hashing the offline archive", async () => {
|
|
const fixture = signedIndex();
|
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-offline-low-memory-"));
|
|
temporaryDirectories.push(directory);
|
|
const indexPath = join(directory, "index.json");
|
|
const archivePath = join(directory, "ocr.tar.gz");
|
|
writeFileSync(indexPath, fixture.raw);
|
|
writeFileSync(archivePath, Buffer.from("deliberately does not match the signed archive"));
|
|
|
|
await expect(
|
|
prepareOfflineRuntimeRelease({
|
|
indexPath,
|
|
archivePath,
|
|
target: TARGET,
|
|
trustKeys: [fixture.trustKey],
|
|
version: "2.1.0",
|
|
memoryOptions: { effectiveMemoryBytes: 4 * 1024 ** 3 - 1 },
|
|
}),
|
|
).rejects.toThrow("insufficient memory for accurate OCR runtime");
|
|
});
|
|
});
|