diff --git a/apps/daemon/tests/update-lifecycle-observations.test.ts b/apps/daemon/tests/update-lifecycle-observations.test.ts index c197eea5b4..82734ef260 100644 --- a/apps/daemon/tests/update-lifecycle-observations.test.ts +++ b/apps/daemon/tests/update-lifecycle-observations.test.ts @@ -36,6 +36,21 @@ it('accepts legacy summaries without stages, then observes late desktop readines await observeUpdateLifecycleStages(options); expect(capture).toHaveBeenCalledTimes(1); }); +it('replays renderer quiescence as its own idempotent handoff stage', async () => { + const { options, capture, stage } = await fixture(); + await stage('renderer_quiesced'); + expect(await observeUpdateLifecycleStages(options)).toEqual({ queued: 1 }); + expect(capture.mock.calls[0]?.[0]).toMatchObject({ + eventName: 'update_lifecycle_observed', + insertId: 'update_lifecycle_observed:flow:renderer_quiesced', + properties: expect.objectContaining({ + stage: 'renderer_quiesced', + outcome: 'completed', + }), + }); + await observeUpdateLifecycleStages(options); + expect(capture).toHaveBeenCalledTimes(1); +}); it('retries a rejected enqueue with the same insert ID and does not conflate queue acknowledgement with ingestion', async () => { const { options, capture, stage, flow } = await fixture(); await stage('shutdown_completed', { duration_ms: 32000, repeated_quit_count: 2 }); diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index 8019fb6ba0..139c2e36f7 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -194,6 +194,8 @@ export function applyLoopbackConnectionLimitSwitch(electronApp: Electron.App): v export type DesktopMainOptions = { beforeShutdown?: (record?: (event: UpdateLifecycleObservation) => Promise) => Promise; + /** Prevent packaged renderer traffic from reaching sidecars as they retire. */ + quiesceRendererTransport?: () => void | Promise; onExternalShow?: () => void | Promise; discoverWebUrl: () => Promise; /** @@ -841,14 +843,32 @@ export async function runDesktopMain( console.info("[open-design desktop] shutdown started"); updateScheduler?.stop("shutdown"); await updater.recordLifecycle?.({ stage: "shutdown_started", outcome: "started" }); + // Stop the request-producing renderer before retiring its web/daemon + // targets. Keeping the window alive while packaged sidecars drain leaves + // polling, SSE reconnects, and ordinary fetches aimed at an origin that + // has already gone away for the entire graceful-stop window. + disposeMenu(); + removeDiagnosticsIpc(); + let rendererQuiesceFailed = false; + await desktop?.close().catch(() => { + rendererQuiesceFailed = true; + shutdownFailed = true; + }); + try { + await options.quiesceRendererTransport?.(); + } catch { + rendererQuiesceFailed = true; + shutdownFailed = true; + } + await updater.recordLifecycle?.({ + stage: "renderer_quiesced", + outcome: rendererQuiesceFailed ? "failed" : "completed", + }); await options.beforeShutdown?.((event) => updater.recordLifecycle?.(event) ?? Promise.resolve()).catch((error: unknown) => { shutdownFailed = true; console.error("desktop beforeShutdown failed", error); }); console.info("[open-design desktop] shutdown sidecars settled", { durationMs: Date.now() - startedAt }); - disposeMenu(); - removeDiagnosticsIpc(); - await desktop?.close().catch(() => { shutdownFailed = true; }); // Mark clean only after teardown; a stalled cleanup is not a clean exit. endDesktopSessionCleanly({ stateFilePath: sessionStatePath }); console.info("[open-design desktop] shutdown completed", { durationMs: Date.now() - startedAt }); diff --git a/apps/desktop/tests/main/shutdown.test.ts b/apps/desktop/tests/main/shutdown.test.ts index 1e8595ddf7..7673bdd317 100644 --- a/apps/desktop/tests/main/shutdown.test.ts +++ b/apps/desktop/tests/main/shutdown.test.ts @@ -5,7 +5,11 @@ import { describe, expect, it, vi } from "vitest"; // Execute the entry's actual shutdown declarations, substituting only Electron // and the sidecar cleanup boundary. This catches premature process.exit calls. -function shutdownHarness() { +function shutdownHarness(options: { + onRendererClosed?: () => void; + onRendererTransportQuiesced?: () => void; + onSidecarRetirementStarted?: () => void; +} = {}) { const source = readFileSync(new URL("../../src/main/index.ts", import.meta.url), "utf8"); const ast = ts.createSourceFile("index.ts", source, ts.ScriptTarget.Latest, true); const declarations: string[] = []; @@ -30,8 +34,16 @@ function shutdownHarness() { visit(ast); let finishCleanup!: () => void; const cleanup = new Promise((resolve) => { finishCleanup = resolve; }); - const beforeShutdown = vi.fn(() => cleanup); - const close = vi.fn(async () => undefined); + const beforeShutdown = vi.fn(() => { + options.onSidecarRetirementStarted?.(); + return cleanup; + }); + const close = vi.fn(async () => { + options.onRendererClosed?.(); + }); + const quiesceRendererTransport = vi.fn(async () => { + options.onRendererTransportQuiesced?.(); + }); let beforeQuit!: (event: { preventDefault(): void }) => void; const quit = vi.fn(); const on = (_event: string, listener: typeof beforeQuit) => { beforeQuit = listener; }; @@ -40,7 +52,7 @@ function shutdownHarness() { const recordLifecycle = vi.fn(async (_event: unknown) => undefined); const sandbox = { updater: { recordLifecycle }, - options: { beforeShutdown }, desktop: { close }, app: { quit, on }, process: { exit }, + options: { beforeShutdown, quiesceRendererTransport }, desktop: { close }, app: { quit, on }, process: { exit }, updateScheduler: { stop: vi.fn() }, disposeMenu: vi.fn(), removeDiagnosticsIpc: vi.fn(), endDesktopSessionCleanly: endSession, sessionStatePath: "test-session", console: { info: vi.fn(), error: vi.fn() }, }; @@ -48,7 +60,7 @@ function shutdownHarness() { const api = runInNewContext(`${code}\n({ shutdown, shutdownAndExit })`, sandbox) as { shutdown(): Promise; shutdownAndExit(): void; }; - return { ...api, beforeShutdown, close, quit, exit, endSession, finishCleanup, beforeQuit, recordLifecycle }; + return { ...api, beforeShutdown, close, quiesceRendererTransport, quit, exit, endSession, finishCleanup, beforeQuit, recordLifecycle }; } async function flushPromises() { @@ -56,6 +68,59 @@ async function flushPromises() { } describe("desktop shutdown", () => { + it("[P0] quiesces the renderer before retiring runtime sidecars", async () => { + const h = shutdownHarness(); + const pending = h.shutdown(); + + await flushPromises(); + + expect(h.close).toHaveBeenCalledTimes(1); + expect(h.beforeShutdown).toHaveBeenCalledTimes(1); + expect(h.close.mock.invocationCallOrder[0]).toBeLessThan( + h.quiesceRendererTransport.mock.invocationCallOrder[0]!, + ); + expect(h.quiesceRendererTransport.mock.invocationCallOrder[0]).toBeLessThan( + h.beforeShutdown.mock.invocationCallOrder[0]!, + ); + expect(h.recordLifecycle).toHaveBeenNthCalledWith(2, { + stage: "renderer_quiesced", + outcome: "completed", + }); + // Electron main remains alive to finish sidecar cleanup and the deferred + // payload handoff after the request-producing renderer has gone away. + expect(h.quit).not.toHaveBeenCalled(); + expect(h.exit).not.toHaveBeenCalled(); + + h.finishCleanup(); + await pending; + }); + + it("[P0] emits no renderer request after runtime retirement begins", async () => { + let rendererCanRequest = true; + let staleOriginRequestCount = 0; + const h = shutdownHarness({ + // Model a BrowserWindow.close() held by beforeunload: the transport + // barrier, not a cooperative page close, must stop stale-origin dials. + onRendererTransportQuiesced: () => { + rendererCanRequest = false; + }, + onSidecarRetirementStarted: () => { + // Deterministic stand-in for the poll/SSE/resource request that used to + // hit the retired localhost origin during the sidecar grace window. + if (rendererCanRequest) staleOriginRequestCount += 1; + }, + }); + + const pending = h.shutdown(); + await flushPromises(); + + expect(staleOriginRequestCount).toBe(0); + expect(h.exit).not.toHaveBeenCalled(); + + h.finishCleanup(); + await pending; + }); + it("does not exit early when quit is requested again during sidecar cleanup", async () => { const h = shutdownHarness(); h.shutdownAndExit(); @@ -63,7 +128,7 @@ describe("desktop shutdown", () => { await flushPromises(); expect(h.beforeShutdown).toHaveBeenCalledTimes(1); expect(h.exit).not.toHaveBeenCalled(); - expect(h.close).not.toHaveBeenCalled(); + expect(h.close).toHaveBeenCalledTimes(1); expect(h.endSession).not.toHaveBeenCalled(); h.finishCleanup(); await flushPromises(); @@ -102,6 +167,7 @@ describe("desktop shutdown", () => { expect(h.endSession).toHaveBeenCalledTimes(1); expect(h.recordLifecycle.mock.calls.map(([event]) => event)).toEqual([ { stage: "shutdown_started", outcome: "started" }, + { stage: "renderer_quiesced", outcome: "completed" }, { stage: "shutdown_completed", outcome: "completed", repeated_quit_count: 1, duration_ms: expect.any(Number) }, ]); }); diff --git a/apps/packaged/src/index.ts b/apps/packaged/src/index.ts index 9afaf62b71..2e0638ccbb 100644 --- a/apps/packaged/src/index.ts +++ b/apps/packaged/src/index.ts @@ -374,7 +374,7 @@ async function main(): Promise { // Resolve the web sidecar address per request instead of freezing it here. // The restart supervisor may bind a fresh ephemeral port, while a temporary // lack of a target should surface as the protocol layer's structured 503. - registerOdProtocol(() => sidecars.currentWebUrl()); + const odProtocol = registerOdProtocol(() => sidecars.currentWebUrl()); const { runDesktopMain } = await import("@open-design/desktop/main"); let desktopHandle: DesktopMainHandle | null = null; @@ -408,6 +408,9 @@ async function main(): Promise { await sidecars.close(record); } }, + quiesceRendererTransport() { + odProtocol.quiesce(); + }, async discoverWebUrl() { return packagedEntryUrl(); }, diff --git a/apps/packaged/src/protocol.ts b/apps/packaged/src/protocol.ts index d8478247bc..49b494ac65 100644 --- a/apps/packaged/src/protocol.ts +++ b/apps/packaged/src/protocol.ts @@ -24,6 +24,11 @@ type OdProtocolTarget = string | null; */ type OdProtocolTargetResolver = () => OdProtocolTarget; +export type OdProtocolControl = { + /** Stop forwarding renderer traffic before the current runtime retires. */ + quiesce(): void; +}; + protocol.registerSchemesAsPrivileged([ { privileges: { @@ -109,9 +114,11 @@ const OD_PROXY_RETRY_ATTEMPTS = 3; const OD_PROXY_RETRY_BACKOFF_MS = 150; // 150ms, 300ms — throw path only, ~450ms worst-case added type OdProxyRetryOptions = { + activeControllers?: Set; attempts?: number; backoffMs?: number; delay?: (ms: number) => Promise; + isQuiesced?: () => boolean; }; const defaultRetryDelay = (ms: number): Promise => @@ -177,25 +184,38 @@ async function fetchOdTargetOnce( request: Request, target: string, fetchImpl: OdProtocolFetch, + options: OdProxyRetryOptions, ): Promise { const controller = new AbortController(); + options.activeControllers?.add(controller); + const release = () => { + request.signal?.removeEventListener("abort", abortUpstream); + options.activeControllers?.delete(controller); + }; const abortUpstream = () => controller.abort(); if (request.signal != null) { if (request.signal.aborted) controller.abort(); else request.signal.addEventListener("abort", abortUpstream, { once: true }); } - const upstream = await fetchImpl( - new Request(target, { - method: request.method, - headers: request.headers, - body: request.body, - // @ts-expect-error -- duplex is required by undici/net.fetch for - // streaming request bodies and absent from the lib.dom Request typings. - duplex: "half", - signal: controller.signal, - }), - ); + let upstream: Response; + try { + upstream = await fetchImpl( + new Request(target, { + method: request.method, + headers: request.headers, + body: request.body, + // @ts-expect-error -- duplex is required by undici/net.fetch for + // streaming request bodies and absent from the lib.dom Request typings. + duplex: "half", + signal: controller.signal, + }), + ); + } catch (error) { + release(); + throw error; + } if (upstream.body == null) { + release(); return new Response(null, { status: upstream.status, statusText: upstream.statusText, @@ -208,11 +228,13 @@ async function fetchOdTargetOnce( try { const { done, value } = await reader.read(); if (done) { + release(); streamController.close(); return; } streamController.enqueue(value); } catch (error) { + release(); streamController.error(error); } }, @@ -220,6 +242,7 @@ async function fetchOdTargetOnce( // Protocol consumer went away (renderer closed the EventSource, tab // navigated, fetch aborted) — release the upstream connection NOW. abortUpstream(); + release(); return reader.cancel().catch(() => undefined); }, }); @@ -271,6 +294,23 @@ function buildClientCancelledResponse(target: string): Response { const OD_TARGET_UNAVAILABLE_STATUS = 503; // Service Unavailable +function buildRuntimeRetiringResponse(request: Request): Response { + return new Response( + JSON.stringify({ + error: "OD_PROTOCOL_RUNTIME_RETIRING", + message: "the packaged runtime is restarting", + requested: request.url, + }), + { + status: OD_TARGET_UNAVAILABLE_STATUS, + headers: { + "content-type": "application/json", + "access-control-allow-origin": "*", + }, + }, + ); +} + /** * Answer for a request that has nowhere to go: the web sidecar reported no * address, or reported one that will not parse. @@ -319,10 +359,14 @@ async function fetchOdTargetWithTransientRetry( const delay = options.delay ?? defaultRetryDelay; let lastError: unknown; for (let attempt = 1; attempt <= attempts; attempt += 1) { + if (options.isQuiesced?.()) { + throw new Error("packaged runtime is retiring"); + } try { - return await fetchOdTargetOnce(request, target, fetchImpl); + return await fetchOdTargetOnce(request, target, fetchImpl, options); } catch (error) { lastError = error; + if (options.isQuiesced?.()) break; // Retrying a request the client abandoned can only fail again, and each // extra attempt is a misleading "proxy fetch failed" line in the log. if (isClientCancelled(request)) break; @@ -408,6 +452,7 @@ export async function handleOdRequest( try { return await fetchOdTargetWithTransientRetry(request, target, fetchImpl, retryOptions); } catch (error) { + if (retryOptions.isQuiesced?.()) return buildRuntimeRetiringResponse(request); // A request the renderer cancelled is not a gateway failure — reporting it // as 502 both lies in the logs and hands error-shaped JSON to any consumer // that fails to check `res.ok`. @@ -441,13 +486,14 @@ export function packagedEntryUrl(): string { * `net.fetch` regresses a broader streaming edge on a specific Electron/OS * combo. */ -function resolveOdProxyFetch(): OdProtocolFetch { +function resolveOdProxyFetch(isQuiesced: () => boolean = () => false): OdProtocolFetch { if (process.env.OD_OD_PROXY_FETCH === "undici") return fetch; const undiciFetch = fetch; return async (request) => { try { return await net.fetch(request); } catch (error) { + if (isQuiesced()) throw error; if (!OD_PROXY_RETRYABLE_METHODS.has(request.method) || isClientCancelled(request)) { throw error; } @@ -474,9 +520,25 @@ function resolveOdProxyFetch(): OdProtocolFetch { * See `OdProtocolTargetResolver` for why this takes a provider rather than the * address itself. */ -export function registerOdProtocol(resolveWebRuntimeUrl: OdProtocolTargetResolver): void { - const fetchImpl = resolveOdProxyFetch(); +export function registerOdProtocol(resolveWebRuntimeUrl: OdProtocolTargetResolver): OdProtocolControl { + let quiesced = false; + const activeControllers = new Set(); + const fetchImpl = resolveOdProxyFetch(() => quiesced); protocol.handle(OD_SCHEME, async (request) => { - return await handleOdRequest(request, resolveWebRuntimeUrl(), fetchImpl); + // BrowserWindow.close() may be held by a renderer beforeunload guard. The + // transport barrier is therefore the authoritative boundary: once armed, + // no request can race sidecar retirement or dial the old localhost origin. + if (quiesced) return buildRuntimeRetiringResponse(request); + return await handleOdRequest(request, resolveWebRuntimeUrl(), fetchImpl, { + activeControllers, + isQuiesced: () => quiesced, + }); }); + return { + quiesce() { + quiesced = true; + for (const controller of activeControllers) controller.abort(); + activeControllers.clear(); + }, + }; } diff --git a/apps/packaged/tests/protocol.test.ts b/apps/packaged/tests/protocol.test.ts index bd72660049..64fa17f45e 100644 --- a/apps/packaged/tests/protocol.test.ts +++ b/apps/packaged/tests/protocol.test.ts @@ -393,6 +393,33 @@ describe('od:// protocol transient retry', () => { expect(waits).toEqual([150, 300]); }); + + it('does not start another retry after runtime quiescence', async () => { + let calls = 0; + let quiesced = false; + const fetchImpl: typeof fetch = async () => { + calls += 1; + throw transientSocketError(); + }; + + const response = await handleOdRequest( + new Request('od://app/api/workspace/directory'), + 'http://127.0.0.1:17579/', + fetchImpl, + { + delay: async () => { + quiesced = true; + }, + isQuiesced: () => quiesced, + }, + ); + + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ + error: 'OD_PROTOCOL_RUNTIME_RETIRING', + }); + expect(calls).toBe(1); + }); }); /** @@ -714,6 +741,51 @@ describe('od:// protocol target resolution', () => { ]); }); + it('does not dial the old origin after the renderer transport is quiesced', async () => { + vi.mocked(net.fetch).mockImplementation((async () => + new Response('unexpected', { status: 200 })) as unknown as typeof net.fetch); + + const control = registerOdProtocol(() => 'http://127.0.0.1:61424/'); + const handler = registeredHandler(); + control.quiesce(); + + const response = await handler(new Request('od://app/api/workspace/directory')); + + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ + error: 'OD_PROTOCOL_RUNTIME_RETIRING', + }); + expect(vi.mocked(net.fetch)).not.toHaveBeenCalled(); + }); + + it('aborts an in-flight old-origin request without retrying when quiesced', async () => { + const observedSignals: AbortSignal[] = []; + vi.mocked(net.fetch).mockImplementation(((request: Request) => { + observedSignals.push(request.signal); + return new Promise((_resolve, reject) => { + request.signal.addEventListener('abort', () => { + reject(new DOMException('runtime retiring', 'AbortError')); + }, { once: true }); + }); + }) as unknown as typeof net.fetch); + + const control = registerOdProtocol(() => 'http://127.0.0.1:61424/'); + const handler = registeredHandler(); + const pending = handler(new Request('od://app/api/workspace/events')); + await Promise.resolve(); + + control.quiesce(); + const response = await pending; + + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ + error: 'OD_PROTOCOL_RUNTIME_RETIRING', + }); + expect(observedSignals).toHaveLength(1); + expect(observedSignals[0]?.aborted).toBe(true); + expect(vi.mocked(net.fetch)).toHaveBeenCalledTimes(1); + }); + it('falls back to undici when net.fetch rejects a CSS-mask GET', async () => { vi.mocked(net.fetch).mockRejectedValue(new Error('net::ERR_FAILED')); const undiciFetch = vi.fn(async (_input: Request | string | URL) => diff --git a/docs/observability/update-lifecycle.md b/docs/observability/update-lifecycle.md index 9893e041e3..b6defd9e92 100644 --- a/docs/observability/update-lifecycle.md +++ b/docs/observability/update-lifecycle.md @@ -8,6 +8,7 @@ |---|---|---| | install_requested | A persisted install attempt was created | Not installation success | | shutdown_started | The desktop entered its single shared teardown | Not proof that it exited | +| renderer_quiesced | The window close was requested and the packaged transport barrier blocked new forwarding and aborted in-flight old-origin requests before runtime retirement | Failed means teardown continued but either boundary raised an error | | cleanup_daemon / cleanup_web | The owned sidecar cleanup returned | Duration, forced and remaining process counts | | shutdown_completed | Desktop teardown reached its final boundary | Duration and repeated quit count; failed if a caught teardown error occurred | | predecessor_wait_started | An updater successor started waiting for the old desktop | Only present on after-quit launches | @@ -29,7 +30,7 @@ A persisted receipt records `queued` only after the analytics client returns a l Group by `flow_id` and slice by from/to version, channel, platform and architecture. Use `occurred_at` for lifecycle ordering; a replay's ingestion time is not the original event time. - Predecessor-wait latency: duration percentiles of predecessor_wait_completed; split normal/forced/failed. -- Shutdown latency and retries: duration percentiles and repeated_quit_count of shutdown_completed, split by outcome. +- Shutdown latency and retries: duration percentiles and repeated_quit_count of shutdown_completed, split by outcome. Require renderer_quiesced to precede cleanup stages for atomic payload handoffs. - Forced cleanup fraction: forced cleanup events divided by observed cleanup completions, separately for daemon and web. - Desktop readiness coverage: distinct flows with desktop_ready divided by observed install-request flows, with a defined observation horizon and matching instrumentation versions. Report missing stages as **unknown**, not failure. - Version-apply success and desktop readiness are separate outcomes. The existing update_apply_observed event supplies the former; do not substitute one for the other. diff --git a/docs/testing/updater-lifecycle.md b/docs/testing/updater-lifecycle.md index ff7d4ad376..ef04f7fa99 100644 --- a/docs/testing/updater-lifecycle.md +++ b/docs/testing/updater-lifecycle.md @@ -48,6 +48,7 @@ spec (`e2e/specs/mac.spec.ts` / `win.spec.ts` via `release-smoke.ts`), | Node | Coverage | Owning tests | | --- | --- | --- | | activate (generation++) → after-quit takeover → confirm | U, P, F | packaged + desktop unit; specs; real-feed loop | +| Renderer quiescence before web/daemon retirement | U | desktop `shutdown.test.ts` P0 ordering/shared-cleanup cases; packaged `protocol.test.ts` no-forward/no-retry/abort cases | | Delegated pre-arm (`--od-launcher-delegated-*`) | U | launcher-proto selection; packaged delegation/launch tests; desktop activation test | | Crash rollback to lastSuccessful + self-heal on next release | P | mac/win spec `rolls back a crashing payload and self-heals on the next good update` | | Stale relaunch freeze scrub after rollback (installResult.activeVersion > running) | U, P | desktop unit stale-freeze spec; exercised by the spec's self-heal phase | diff --git a/packages/contracts/src/analytics/update-lifecycle.ts b/packages/contracts/src/analytics/update-lifecycle.ts index 540857fc10..067c5a631a 100644 --- a/packages/contracts/src/analytics/update-lifecycle.ts +++ b/packages/contracts/src/analytics/update-lifecycle.ts @@ -1,6 +1,6 @@ /** Finite, content-free stages of one persisted updater flow. */ export const UPDATE_LIFECYCLE_STAGES = [ - 'install_requested', 'shutdown_started', 'shutdown_completed', + 'install_requested', 'shutdown_started', 'renderer_quiesced', 'shutdown_completed', 'cleanup_daemon', 'cleanup_web', 'predecessor_wait_started', 'predecessor_wait_completed', 'desktop_ready', ] as const; diff --git a/packages/contracts/tests/analytics-update-lifecycle.test.ts b/packages/contracts/tests/analytics-update-lifecycle.test.ts new file mode 100644 index 0000000000..64c532d0a1 --- /dev/null +++ b/packages/contracts/tests/analytics-update-lifecycle.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest'; + +import { + UPDATE_LIFECYCLE_STAGES, + parseUpdateLifecycleObservation, +} from '../src/analytics/update-lifecycle.js'; + +describe('update lifecycle analytics contract', () => { + it('accepts the renderer quiescence boundary as a first-class handoff stage', () => { + expect(UPDATE_LIFECYCLE_STAGES).toContain('renderer_quiesced'); + expect(parseUpdateLifecycleObservation({ + stage: 'renderer_quiesced', + outcome: 'completed', + })).toEqual({ + stage: 'renderer_quiesced', + outcome: 'completed', + }); + }); +});