mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
fix(ocr): unblock and harden accurate-OCR install (#552)
Two OCR-install fixes surfaced while verifying the accurate-OCR (v3 runtime) path end to end: - Installer timeout must be a safe integer, not a performance.now() float. With the default INSTALL_MAX_MS this failed every accurate-OCR install via the app right after the download (masked by the unpublished runtime; CI drives install_runtime.py directly so it never surfaced). Fixed via remainingInstallerTimeoutMs(). - Classify an absent or forbidden runtime index (401/403/404/410) as OcrRuntimeNotPublishedError with a clear "Fast OCR still works" message instead of a raw HTTP 404, without retrying. Refs #552
This commit is contained in:
@@ -65,6 +65,10 @@ const HARD_MAX_RETRY_ATTEMPTS = 6;
|
|||||||
const HARD_MAX_RETRY_DELAY_MS = 30_000;
|
const HARD_MAX_RETRY_DELAY_MS = 30_000;
|
||||||
const HARD_MAX_RETRY_TOTAL_DELAY_MS = 120_000;
|
const HARD_MAX_RETRY_TOTAL_DELAY_MS = 120_000;
|
||||||
const RETRYABLE_HTTP_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]);
|
const RETRYABLE_HTTP_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]);
|
||||||
|
// The bundle host answers a runtime index that was never published for this
|
||||||
|
// version with one of these. On a public bundle repository they all mean the
|
||||||
|
// same thing to a user: the accurate runtime is not available to install yet.
|
||||||
|
const RUNTIME_NOT_PUBLISHED_STATUSES = new Set([401, 403, 404, 410]);
|
||||||
const RETRYABLE_NETWORK_CODES = new Set([
|
const RETRYABLE_NETWORK_CODES = new Set([
|
||||||
"EAI_AGAIN",
|
"EAI_AGAIN",
|
||||||
"ECONNREFUSED",
|
"ECONNREFUSED",
|
||||||
@@ -185,6 +189,22 @@ export class OcrRuntimeImportValidationError extends Error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The signed OCR runtime index is not published (or not readable) for this
|
||||||
|
* SnapOtter version. This is a definitive "nothing to install here" answer, not
|
||||||
|
* a transient failure, so it is thrown without retrying and carries a message
|
||||||
|
* the install UI can show verbatim.
|
||||||
|
*/
|
||||||
|
export class OcrRuntimeNotPublishedError extends Error {
|
||||||
|
readonly httpStatus: number;
|
||||||
|
|
||||||
|
constructor(message: string, httpStatus: number, options?: ErrorOptions) {
|
||||||
|
super(message, options);
|
||||||
|
this.name = "OcrRuntimeNotPublishedError";
|
||||||
|
this.httpStatus = httpStatus;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
interface ResolvedRetryPolicy {
|
interface ResolvedRetryPolicy {
|
||||||
maxAttempts: number;
|
maxAttempts: number;
|
||||||
baseDelayMs: number;
|
baseDelayMs: number;
|
||||||
@@ -332,6 +352,22 @@ async function assertDownloadResponse(
|
|||||||
throw new Error(message);
|
throw new Error(message);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Turn an absent (or forbidden) signed runtime index into a clear, terminal
|
||||||
|
* error instead of a raw "HTTP 404". Callers invoke this before the generic
|
||||||
|
* response check so the message reaches the install UI unretried and unwrapped.
|
||||||
|
*/
|
||||||
|
function assertOcrRuntimeIndexPublished(response: Response, version: string): void {
|
||||||
|
if (!RUNTIME_NOT_PUBLISHED_STATUSES.has(response.status)) return;
|
||||||
|
cancelResponseBody(response);
|
||||||
|
throw new OcrRuntimeNotPublishedError(
|
||||||
|
`The accurate OCR runtime for SnapOtter ${version} is not available to install yet ` +
|
||||||
|
`(its signed runtime index returned HTTP ${response.status}). Fast OCR still works; ` +
|
||||||
|
`accurate OCR becomes available once its runtime is published for this version.`,
|
||||||
|
response.status,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function retryDelayMs(error: unknown, completedAttempts: number, policy: ResolvedRetryPolicy) {
|
function retryDelayMs(error: unknown, completedAttempts: number, policy: ResolvedRetryPolicy) {
|
||||||
if (error instanceof RetryableOcrDownloadError && error.retryAfterMs !== null) {
|
if (error instanceof RetryableOcrDownloadError && error.retryAfterMs !== null) {
|
||||||
return Math.min(error.retryAfterMs, policy.maxDelayMs);
|
return Math.min(error.retryAfterMs, policy.maxDelayMs);
|
||||||
@@ -1748,6 +1784,7 @@ async function downloadVerifiedRuntimeReleaseUnderLease(
|
|||||||
signal: controller.signal,
|
signal: controller.signal,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
assertOcrRuntimeIndexPublished(indexResponse, options.version);
|
||||||
await assertDownloadResponse(indexResponse, "OCR runtime index", retryPolicy.now);
|
await assertDownloadResponse(indexResponse, "OCR runtime index", retryPolicy.now);
|
||||||
return readBoundedResponse(
|
return readBoundedResponse(
|
||||||
indexResponse,
|
indexResponse,
|
||||||
@@ -1965,6 +2002,17 @@ async function purgeOcrRuntimeDownloadsUnderLease(aiDataDir: string): Promise<vo
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Remaining installer time budget from an optional deadline. `performance.now()`
|
||||||
|
* is fractional, so a raw `deadline - now` is a float that `runOcrRuntimeInstaller`
|
||||||
|
* rejects as a non-integer timeout. Floor it to a safe integer and keep at least
|
||||||
|
* 1ms so a live install always receives a positive, valid timeout.
|
||||||
|
*/
|
||||||
|
export function remainingInstallerTimeoutMs(deadlineMs: number | undefined, nowMs: number): number {
|
||||||
|
if (deadlineMs === undefined) return 0;
|
||||||
|
return Math.max(1, Math.floor(deadlineMs - nowMs));
|
||||||
|
}
|
||||||
|
|
||||||
export function buildOcrRuntimeInstallerCommand(
|
export function buildOcrRuntimeInstallerCommand(
|
||||||
options: RunOcrRuntimeInstallerOptions,
|
options: RunOcrRuntimeInstallerOptions,
|
||||||
): OcrRuntimeInstallerCommand {
|
): OcrRuntimeInstallerCommand {
|
||||||
|
|||||||
@@ -89,6 +89,7 @@ import {
|
|||||||
prepareOfflineRuntimeIndex,
|
prepareOfflineRuntimeIndex,
|
||||||
prepareOfflineRuntimeRelease,
|
prepareOfflineRuntimeRelease,
|
||||||
purgeOcrRuntimeDownloads,
|
purgeOcrRuntimeDownloads,
|
||||||
|
remainingInstallerTimeoutMs,
|
||||||
runOcrRuntimeInstaller,
|
runOcrRuntimeInstaller,
|
||||||
runOcrRuntimeMaintenance,
|
runOcrRuntimeMaintenance,
|
||||||
waitWithOcrRuntimeHeartbeat,
|
waitWithOcrRuntimeHeartbeat,
|
||||||
@@ -227,7 +228,7 @@ function startOcrInstall(bundleId: string, jobId: string, installLockFd: number)
|
|||||||
release,
|
release,
|
||||||
aiDataDir: getAiDir(),
|
aiDataDir: getAiDir(),
|
||||||
installLockFd,
|
installLockFd,
|
||||||
timeoutMs: installDeadline ? Math.max(1, installDeadline - performance.now()) : 0,
|
timeoutMs: remainingInstallerTimeoutMs(installDeadline, performance.now()),
|
||||||
}),
|
}),
|
||||||
reportActivation,
|
reportActivation,
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -28,10 +28,12 @@ import {
|
|||||||
downloadVerifiedRuntimeRelease as downloadVerifiedRuntimeReleaseWithLease,
|
downloadVerifiedRuntimeRelease as downloadVerifiedRuntimeReleaseWithLease,
|
||||||
loadOcrRuntimeTrustKeys,
|
loadOcrRuntimeTrustKeys,
|
||||||
OcrRuntimeImportValidationError,
|
OcrRuntimeImportValidationError,
|
||||||
|
OcrRuntimeNotPublishedError,
|
||||||
type OcrRuntimeTrustKey,
|
type OcrRuntimeTrustKey,
|
||||||
prepareOfflineRuntimeIndex,
|
prepareOfflineRuntimeIndex,
|
||||||
prepareOfflineRuntimeRelease,
|
prepareOfflineRuntimeRelease,
|
||||||
purgeOcrRuntimeDownloads as purgeOcrRuntimeDownloadsWithLease,
|
purgeOcrRuntimeDownloads as purgeOcrRuntimeDownloadsWithLease,
|
||||||
|
remainingInstallerTimeoutMs,
|
||||||
verifyRuntimeIndex,
|
verifyRuntimeIndex,
|
||||||
writeBufferFully,
|
writeBufferFully,
|
||||||
} from "../../../apps/api/src/lib/ocr-runtime-install.js";
|
} from "../../../apps/api/src/lib/ocr-runtime-install.js";
|
||||||
@@ -157,6 +159,28 @@ function signedIndex(
|
|||||||
return { artifact, index, raw: Buffer.from(canonicalRuntimeJson(index)), trustKey };
|
return { artifact, index, raw: Buffer.from(canonicalRuntimeJson(index)), trustKey };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
describe("remainingInstallerTimeoutMs", () => {
|
||||||
|
it("floors a fractional remaining budget to the safe integer the installer requires", () => {
|
||||||
|
// The deadline is set at one performance.now() read and the remaining time is
|
||||||
|
// computed at a later read, so `deadline - later` is fractional in practice.
|
||||||
|
const started = 1_000.4269;
|
||||||
|
const deadline = started + 7_200_000;
|
||||||
|
const later = 1_500.8731;
|
||||||
|
// The old caller passed this raw float; runOcrRuntimeInstaller rejected it as
|
||||||
|
// a non-integer timeout, which broke every default-config install.
|
||||||
|
expect(Number.isSafeInteger(deadline - later)).toBe(false);
|
||||||
|
const remaining = remainingInstallerTimeoutMs(deadline, later);
|
||||||
|
expect(Number.isSafeInteger(remaining)).toBe(true);
|
||||||
|
expect(remaining).toBe(Math.floor(deadline - later));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns 0 when there is no deadline and clamps a passed deadline to at least 1ms", () => {
|
||||||
|
expect(remainingInstallerTimeoutMs(undefined, 5)).toBe(0);
|
||||||
|
expect(remainingInstallerTimeoutMs(10.9, 10.1)).toBe(1);
|
||||||
|
expect(remainingInstallerTimeoutMs(5, 999.7)).toBe(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("verifyRuntimeIndex", () => {
|
describe("verifyRuntimeIndex", () => {
|
||||||
it("loads the independently pinned release key from the official image environment", () => {
|
it("loads the independently pinned release key from the official image environment", () => {
|
||||||
const fixture = signedIndex();
|
const fixture = signedIndex();
|
||||||
@@ -451,6 +475,64 @@ describe("downloadVerifiedRuntimeRelease", () => {
|
|||||||
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("reports a clear not-published error when the runtime index is absent", async () => {
|
||||||
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-index-absent-"));
|
||||||
|
temporaryDirectories.push(directory);
|
||||||
|
const canceled = vi.fn();
|
||||||
|
const missingBody = new ReadableStream<Uint8Array>({
|
||||||
|
start(controller) {
|
||||||
|
controller.enqueue(Buffer.from("Entry not found"));
|
||||||
|
},
|
||||||
|
cancel: canceled,
|
||||||
|
});
|
||||||
|
const fetchImpl = vi
|
||||||
|
.fn<typeof fetch>()
|
||||||
|
.mockResolvedValue(new Response(missingBody, { status: 404 }));
|
||||||
|
|
||||||
|
const error = await downloadVerifiedRuntimeRelease({
|
||||||
|
aiDataDir: directory,
|
||||||
|
bundleRepo: "snapotter-hq/feature-bundles",
|
||||||
|
version: "2.1.0",
|
||||||
|
target: TARGET,
|
||||||
|
trustKeys: [],
|
||||||
|
fetchImpl,
|
||||||
|
}).then(
|
||||||
|
() => {
|
||||||
|
throw new Error("expected the absent runtime index to reject");
|
||||||
|
},
|
||||||
|
(reason: unknown) => reason,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(error).toBeInstanceOf(OcrRuntimeNotPublishedError);
|
||||||
|
const message = (error as Error).message;
|
||||||
|
expect(message).toContain("2.1.0");
|
||||||
|
expect(message).toContain("Fast OCR");
|
||||||
|
expect(message).toContain("404");
|
||||||
|
// Absent means absent: this is not a transient failure and must not retry.
|
||||||
|
expect(fetchImpl).toHaveBeenCalledOnce();
|
||||||
|
expect(canceled).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("treats a forbidden runtime index as not published rather than a hard error", async () => {
|
||||||
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-index-forbidden-"));
|
||||||
|
temporaryDirectories.push(directory);
|
||||||
|
const fetchImpl = vi
|
||||||
|
.fn<typeof fetch>()
|
||||||
|
.mockResolvedValue(new Response("Forbidden", { status: 403 }));
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
downloadVerifiedRuntimeRelease({
|
||||||
|
aiDataDir: directory,
|
||||||
|
bundleRepo: "snapotter-hq/feature-bundles",
|
||||||
|
version: "2.1.0",
|
||||||
|
target: TARGET,
|
||||||
|
trustKeys: [],
|
||||||
|
fetchImpl,
|
||||||
|
}),
|
||||||
|
).rejects.toBeInstanceOf(OcrRuntimeNotPublishedError);
|
||||||
|
expect(fetchImpl).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
it("does not retry an oversized index and cancels its unconsumed body", async () => {
|
it("does not retry an oversized index and cancels its unconsumed body", async () => {
|
||||||
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-index-size-policy-"));
|
const directory = mkdtempSync(join(tmpdir(), "snapotter-ocr-index-size-policy-"));
|
||||||
temporaryDirectories.push(directory);
|
temporaryDirectories.push(directory);
|
||||||
|
|||||||
Reference in New Issue
Block a user