Cache Windows packaged build layers

2026-10-27 09:51:12 +01:00 · 2026-05-06 16:08:23 +08:00
parent 641a85c7ed
commit 9b11f03830
2 changed files with 217 additions and 71 deletions
@@ -125,6 +125,7 @@ export class ToolPackCache {
    let status: CacheAcquireReport["status"] = "hit";
    let reason: string | null = null;

+    const materialized: CacheAcquireReport["materialized"] = [];
    const manifest = await withDirectoryLock(join(this.root, "locks"), "global", async () => {
      await mkdir(dirname(entryPath), { recursive: true });
      const existingManifest = await readManifest<TMetadata>(manifestPath);
@@ -146,45 +147,48 @@ export class ToolPackCache {
              ? { reason: "key mismatch" }
              : outputInvalid ?? customInvalid;

-      if (existingManifest != null && invalidation == null) {
-        return existingManifest;
-      }
+      const manifest = existingManifest != null && invalidation == null
+        ? existingManifest
+        : null;

-      status = existingManifest == null ? "miss" : "stale";
-      reason = invalidation?.reason ?? "missing manifest";
-      const stagingPath = join(dirname(entryPath), `${basename(entryPath)}.tmp-${process.pid}-${randomUUID()}`);
-      await rm(stagingPath, { force: true, recursive: true });
-      await mkdir(stagingPath, { recursive: true });
-      try {
-        const payloadMetadata = await node.build({ entryRoot: stagingPath });
-        const missingOutput = await assertOutputsExist(stagingPath, outputs);
-        if (missingOutput != null) throw new Error(`cache node ${node.id} build did not produce ${missingOutput.reason}`);
-        const nextManifest: CacheManifest<TMetadata> = {
-          createdAt: new Date().toISOString(),
-          key: node.key,
-          nodeId: node.id,
-          outputs,
-          payloadMetadata,
-          schemaVersion: CACHE_SCHEMA_VERSION,
-        };
-        await writeManifest(join(stagingPath, "manifest.json"), nextManifest);
-        await rm(entryPath, { force: true, recursive: true });
-        await rename(stagingPath, entryPath);
-        return nextManifest;
-      } catch (error) {
+      const nextManifest = manifest ?? await (async () => {
+        status = existingManifest == null ? "miss" : "stale";
+        reason = invalidation?.reason ?? "missing manifest";
+        const stagingPath = join(dirname(entryPath), `${basename(entryPath)}.tmp-${process.pid}-${randomUUID()}`);
        await rm(stagingPath, { force: true, recursive: true });
-        throw error;
-      }
-    });
+        await mkdir(stagingPath, { recursive: true });
+        try {
+          const payloadMetadata = await node.build({ entryRoot: stagingPath });
+          const missingOutput = await assertOutputsExist(stagingPath, outputs);
+          if (missingOutput != null) throw new Error(`cache node ${node.id} build did not produce ${missingOutput.reason}`);
+          const builtManifest: CacheManifest<TMetadata> = {
+            createdAt: new Date().toISOString(),
+            key: node.key,
+            nodeId: node.id,
+            outputs,
+            payloadMetadata,
+            schemaVersion: CACHE_SCHEMA_VERSION,
+          };
+          await writeManifest(join(stagingPath, "manifest.json"), builtManifest);
+          await rm(entryPath, { force: true, recursive: true });
+          await rename(stagingPath, entryPath);
+          return builtManifest;
+        } catch (error) {
+          await rm(stagingPath, { force: true, recursive: true });
+          throw error;
+        }
+      })();

-    const materialized: CacheAcquireReport["materialized"] = [];
-    for (const target of materialize) {
-      const sourcePath = join(entryPath, target.from);
-      await rm(target.to, { force: true, recursive: true });
-      await mkdir(dirname(target.to), { recursive: true });
-      await cp(sourcePath, target.to, { recursive: true });
-      materialized.push({ from: normalizeRelativePath(target.from), to: target.to });
-    }
+      for (const target of materialize) {
+        const sourcePath = join(entryPath, target.from);
+        await rm(target.to, { force: true, recursive: true });
+        await mkdir(dirname(target.to), { recursive: true });
+        await cp(sourcePath, target.to, { recursive: true });
+        materialized.push({ from: normalizeRelativePath(target.from), to: target.to });
+      }
+
+      return nextManifest;
+    });

    this.#entries.push({
      durationMs: Date.now() - startedAt,
@@ -87,6 +87,26 @@ type PackedTarballsCacheMetadata = {
  tarballs: PackedTarballInfo[];
 };

+type PackedTarballsCacheResult = PackedTarballsCacheMetadata & {
+  key: string;
+};
+
+type AssembledAppCacheMetadata = {
+  packagedVersion: string;
+};
+
+type AssembledAppCacheResult = AssembledAppCacheMetadata & {
+  key: string;
+};
+
+type NativeRebuildCacheMetadata = {
+  modules: readonly string[];
+};
+
+type ResourceTreeCacheMetadata = {
+  resourceName: "open-design";
+};
+
 type WinPaths = {
  appBuilderConfigPath: string;
  appBuilderOutputRoot: string;
@@ -733,18 +753,18 @@ async function runNpmInstall(appRoot: string): Promise<void> {
  });
 }

-async function rebuildWinNativeDependencies(config: ToolPackConfig, paths: WinPaths): Promise<void> {
+async function runElectronRebuild(config: ToolPackConfig, appRoot: string): Promise<void> {
  const foundModules = new Set<string>();
  const rebuildResult = rebuild({
    arch: "x64",
    buildFromSource: ELECTRON_BUILDER_BUILD_DEPENDENCIES_FROM_SOURCE,
-    buildPath: paths.assembledAppRoot,
+    buildPath: appRoot,
    electronVersion: config.electronVersion,
    force: true,
    mode: ELECTRON_REBUILD_MODE,
    onlyModules: [...ELECTRON_REBUILD_NATIVE_MODULES],
    platform: "win32",
-    projectRootPath: paths.assembledAppRoot,
+    projectRootPath: appRoot,
  });
  rebuildResult.lifecycle.on("modules-found", (modules: string[]) => {
    for (const moduleName of modules) foundModules.add(moduleName);
@@ -757,6 +777,42 @@ async function rebuildWinNativeDependencies(config: ToolPackConfig, paths: WinPa
  }
 }

+function nativeRebuildOutputPath(appRoot: string): string {
+  return join(appRoot, "node_modules", "better-sqlite3", "build", "Release", "better_sqlite3.node");
+}
+
+async function rebuildWinNativeDependencies(
+  config: ToolPackConfig,
+  paths: WinPaths,
+  cache: ToolPackCache,
+  assembledApp: AssembledAppCacheResult,
+): Promise<void> {
+  const node = {
+    id: "win.native-rebuild",
+    key: hashJson({
+      arch: "x64",
+      assembledAppKey: assembledApp.key,
+      electronVersion: config.electronVersion,
+      modules: ELECTRON_REBUILD_NATIVE_MODULES,
+      platform: "win32",
+      schemaVersion: 1,
+    }),
+    outputs: ["better_sqlite3.node"],
+    invalidate: async () => null,
+    build: async ({ entryRoot }: { entryRoot: string }): Promise<NativeRebuildCacheMetadata> => {
+      const stagingAppRoot = join(entryRoot, "app");
+      await cp(paths.assembledAppRoot, stagingAppRoot, { recursive: true });
+      await runElectronRebuild(config, stagingAppRoot);
+      await cp(nativeRebuildOutputPath(stagingAppRoot), join(entryRoot, "better_sqlite3.node"));
+      return { modules: ELECTRON_REBUILD_NATIVE_MODULES };
+    },
+  };
+  await cache.acquire({
+    materialize: [{ from: "better_sqlite3.node", to: nativeRebuildOutputPath(paths.assembledAppRoot) }],
+    node,
+  });
+}
+
 async function readPackagedVersion(config: ToolPackConfig): Promise<string> {
  const packageJsonPath = join(config.workspaceRoot, "apps", "packaged", "package.json");
  const packageJson = JSON.parse(await readFile(packageJsonPath, "utf8")) as { version?: unknown };
@@ -829,12 +885,38 @@ async function buildWorkspaceArtifacts(config: ToolPackConfig): Promise<void> {
  await runPnpm(config, ["--filter", "@open-design/packaged", "build"]);
 }

-async function copyResourceTree(config: ToolPackConfig, paths: WinPaths): Promise<void> {
-  await removeTree(paths.resourceRoot);
-  await mkdir(paths.resourceRoot, { recursive: true });
-  await copyBundledResourceTrees({
-    workspaceRoot: config.workspaceRoot,
-    resourceRoot: paths.resourceRoot,
+async function createResourceTreeCacheKey(config: ToolPackConfig): Promise<string> {
+  return hashJson({
+    assetsCommunityPets: await hashPath(join(config.workspaceRoot, "assets", "community-pets")),
+    assetsFrames: await hashPath(join(config.workspaceRoot, "assets", "frames")),
+    craft: await hashPath(join(config.workspaceRoot, "craft")),
+    designSystems: await hashPath(join(config.workspaceRoot, "design-systems")),
+    node: "win.resource-tree",
+    promptTemplates: await hashPath(join(config.workspaceRoot, "prompt-templates")),
+    schemaVersion: 1,
+    skills: await hashPath(join(config.workspaceRoot, "skills")),
+  });
+}
+
+async function copyResourceTree(config: ToolPackConfig, paths: WinPaths, cache: ToolPackCache): Promise<void> {
+  const node = {
+    id: "win.resource-tree",
+    key: await createResourceTreeCacheKey(config),
+    outputs: ["open-design"],
+    invalidate: async () => null,
+    build: async ({ entryRoot }: { entryRoot: string }): Promise<ResourceTreeCacheMetadata> => {
+      const resourceRoot = join(entryRoot, "open-design");
+      await mkdir(resourceRoot, { recursive: true });
+      await copyBundledResourceTrees({
+        workspaceRoot: config.workspaceRoot,
+        resourceRoot,
+      });
+      return { resourceName: "open-design" };
+    },
+  };
+  await cache.acquire({
+    materialize: [{ from: "open-design", to: paths.resourceRoot }],
+    node,
  });
 }

@@ -867,10 +949,11 @@ async function collectWorkspaceTarballs(
  config: ToolPackConfig,
  paths: WinPaths,
  cache: ToolPackCache,
-): Promise<PackedTarballInfo[]> {
+): Promise<PackedTarballsCacheResult> {
+  const key = await createWorkspaceTarballsCacheKey(config);
  const node = {
    id: "win.workspace-tarballs",
-    key: await createWorkspaceTarballsCacheKey(config),
+    key,
    outputs: ["tarballs"],
    invalidate: async () => null,
    build: async ({ entryRoot }: { entryRoot: string }): Promise<PackedTarballsCacheMetadata> => {
@@ -893,27 +976,48 @@ async function collectWorkspaceTarballs(
    materialize: [{ from: "tarballs", to: paths.tarballsRoot }],
    node,
  });
-  return manifest.payloadMetadata.tarballs;
+  return { key, tarballs: manifest.payloadMetadata.tarballs };
 }

-async function writeAssembledApp(config: ToolPackConfig, paths: WinPaths, packedTarballs: PackedTarballInfo[]): Promise<void> {
-  const packagedVersion = await readPackagedVersion(config);
+async function writePackagedConfig(config: ToolPackConfig, paths: WinPaths, packagedVersion: string): Promise<void> {
+  await writeFile(
+    paths.packagedConfigPath,
+    `${JSON.stringify(
+      {
+        appVersion: packagedVersion,
+        namespace: config.namespace,
+        webOutputMode: config.webOutputMode,
+        ...(config.portable ? {} : { namespaceBaseRoot: config.roots.runtime.namespaceBaseRoot }),
+      },
+      null,
+      2,
+    )}\n`,
+    "utf8",
+  );
+}
+
+function createAssembledAppDependencies(paths: WinPaths, packedTarballs: PackedTarballInfo[]): Record<string, string> {
  const tarballByPackage = Object.fromEntries(packedTarballs.map((entry) => [entry.packageName, entry.fileName] as const));
-  const dependencies = Object.fromEntries(
+  return Object.fromEntries(
    INTERNAL_PACKAGES.map((packageInfo) => {
      const tarball = tarballByPackage[packageInfo.name];
      if (tarball == null) throw new Error(`missing tarball for ${packageInfo.name}`);
      return [packageInfo.name, `file:${relative(paths.assembledAppRoot, join(paths.tarballsRoot, tarball))}`];
    }),
  );
+}

-  await removeTree(join(config.roots.output.namespaceRoot, "assembled"));
+async function writeAssembledAppEntrypoints(
+  paths: WinPaths,
+  packedTarballs: PackedTarballInfo[],
+  packagedVersion: string,
+): Promise<void> {
  await mkdir(paths.assembledAppRoot, { recursive: true });
  await writeFile(
    paths.assembledPackageJsonPath,
    `${JSON.stringify(
      {
-        dependencies,
+        dependencies: createAssembledAppDependencies(paths, packedTarballs),
        description: "Open Design packaged runtime",
        main: "./main.cjs",
        name: "open-design-packaged-app",
@@ -931,21 +1035,59 @@ async function writeAssembledApp(config: ToolPackConfig, paths: WinPaths, packed
    'import("@open-design/packaged").catch((error) => {\n  console.error("packaged entry failed", error);\n  process.exit(1);\n});\n',
    "utf8",
  );
-  await writeFile(
-    paths.packagedConfigPath,
-    `${JSON.stringify(
-      {
-        appVersion: packagedVersion,
-        namespace: config.namespace,
-        webOutputMode: config.webOutputMode,
-        ...(config.portable ? {} : { namespaceBaseRoot: config.roots.runtime.namespaceBaseRoot }),
-      },
-      null,
-      2,
-    )}\n`,
-    "utf8",
-  );
-  await runNpmInstall(paths.assembledAppRoot);
+}
+
+async function createAssembledAppCacheKey(
+  config: ToolPackConfig,
+  tarballsKey: string,
+  packedTarballs: PackedTarballInfo[],
+  packagedVersion: string,
+): Promise<string> {
+  return hashJson({
+    electronVersion: config.electronVersion,
+    node: "win.assembled-app",
+    packagedVersion,
+    packedTarballs,
+    platform: "win32",
+    schemaVersion: 1,
+    tarballsKey,
+    webOutputMode: config.webOutputMode,
+  });
+}
+
+async function writeAssembledApp(
+  config: ToolPackConfig,
+  paths: WinPaths,
+  tarballs: PackedTarballsCacheResult,
+  cache: ToolPackCache,
+): Promise<AssembledAppCacheResult> {
+  const packagedVersion = await readPackagedVersion(config);
+  await removeTree(join(config.roots.output.namespaceRoot, "assembled"));
+  const packedTarballs = tarballs.tarballs;
+  const key = await createAssembledAppCacheKey(config, tarballs.key, packedTarballs, packagedVersion);
+  const node = {
+    id: "win.assembled-app",
+    key,
+    outputs: ["app"],
+    invalidate: async () => null,
+    build: async ({ entryRoot }: { entryRoot: string }): Promise<AssembledAppCacheMetadata> => {
+      const assembledAppRoot = join(entryRoot, "app");
+      await writeAssembledAppEntrypoints(
+        { ...paths, assembledAppRoot, assembledMainEntryPath: join(assembledAppRoot, "main.cjs"), assembledPackageJsonPath: join(assembledAppRoot, "package.json") },
+        packedTarballs,
+        packagedVersion,
+      );
+      await runNpmInstall(assembledAppRoot);
+      return { packagedVersion };
+    },
+  };
+  await cache.acquire({
+    materialize: [{ from: "app", to: paths.assembledAppRoot }],
+    node,
+  });
+  await writeAssembledAppEntrypoints(paths, packedTarballs, packagedVersion);
+  await writePackagedConfig(config, paths, packagedVersion);
+  return { key, packagedVersion };
 }

 function resolveWinTargets(to: ToolPackConfig["to"]): Array<"dir" | "nsis"> {
@@ -1164,11 +1306,11 @@ export async function packWin(config: ToolPackConfig): Promise<WinPackResult> {
  const paths = resolveWinPaths(config);
  const cache = new ToolPackCache(config.roots.cacheRoot);
  await buildWorkspaceArtifacts(config);
-  await copyResourceTree(config, paths);
+  await copyResourceTree(config, paths, cache);
  await copyWinIcon(paths);
  const tarballs = await collectWorkspaceTarballs(config, paths, cache);
-  await writeAssembledApp(config, paths, tarballs);
-  await rebuildWinNativeDependencies(config, paths);
+  const assembledApp = await writeAssembledApp(config, paths, tarballs, cache);
+  await rebuildWinNativeDependencies(config, paths, cache, assembledApp);
  await runElectronBuilder(config, paths);
  await writeLocalLatestYml(config, paths);
  const sizeReport = await collectWinSizeReport(config, paths);