diff --git a/tools/pack/src/cache.ts b/tools/pack/src/cache.ts index b93e694ee4..1d2b42e056 100644 --- a/tools/pack/src/cache.ts +++ b/tools/pack/src/cache.ts @@ -125,6 +125,7 @@ export class ToolPackCache { let status: CacheAcquireReport["status"] = "hit"; let reason: string | null = null; + const materialized: CacheAcquireReport["materialized"] = []; const manifest = await withDirectoryLock(join(this.root, "locks"), "global", async () => { await mkdir(dirname(entryPath), { recursive: true }); const existingManifest = await readManifest(manifestPath); @@ -146,45 +147,48 @@ export class ToolPackCache { ? { reason: "key mismatch" } : outputInvalid ?? customInvalid; - if (existingManifest != null && invalidation == null) { - return existingManifest; - } + const manifest = existingManifest != null && invalidation == null + ? existingManifest + : null; - status = existingManifest == null ? "miss" : "stale"; - reason = invalidation?.reason ?? "missing manifest"; - const stagingPath = join(dirname(entryPath), `${basename(entryPath)}.tmp-${process.pid}-${randomUUID()}`); - await rm(stagingPath, { force: true, recursive: true }); - await mkdir(stagingPath, { recursive: true }); - try { - const payloadMetadata = await node.build({ entryRoot: stagingPath }); - const missingOutput = await assertOutputsExist(stagingPath, outputs); - if (missingOutput != null) throw new Error(`cache node ${node.id} build did not produce ${missingOutput.reason}`); - const nextManifest: CacheManifest = { - createdAt: new Date().toISOString(), - key: node.key, - nodeId: node.id, - outputs, - payloadMetadata, - schemaVersion: CACHE_SCHEMA_VERSION, - }; - await writeManifest(join(stagingPath, "manifest.json"), nextManifest); - await rm(entryPath, { force: true, recursive: true }); - await rename(stagingPath, entryPath); - return nextManifest; - } catch (error) { + const nextManifest = manifest ?? await (async () => { + status = existingManifest == null ? "miss" : "stale"; + reason = invalidation?.reason ?? "missing manifest"; + const stagingPath = join(dirname(entryPath), `${basename(entryPath)}.tmp-${process.pid}-${randomUUID()}`); await rm(stagingPath, { force: true, recursive: true }); - throw error; - } - }); + await mkdir(stagingPath, { recursive: true }); + try { + const payloadMetadata = await node.build({ entryRoot: stagingPath }); + const missingOutput = await assertOutputsExist(stagingPath, outputs); + if (missingOutput != null) throw new Error(`cache node ${node.id} build did not produce ${missingOutput.reason}`); + const builtManifest: CacheManifest = { + createdAt: new Date().toISOString(), + key: node.key, + nodeId: node.id, + outputs, + payloadMetadata, + schemaVersion: CACHE_SCHEMA_VERSION, + }; + await writeManifest(join(stagingPath, "manifest.json"), builtManifest); + await rm(entryPath, { force: true, recursive: true }); + await rename(stagingPath, entryPath); + return builtManifest; + } catch (error) { + await rm(stagingPath, { force: true, recursive: true }); + throw error; + } + })(); - const materialized: CacheAcquireReport["materialized"] = []; - for (const target of materialize) { - const sourcePath = join(entryPath, target.from); - await rm(target.to, { force: true, recursive: true }); - await mkdir(dirname(target.to), { recursive: true }); - await cp(sourcePath, target.to, { recursive: true }); - materialized.push({ from: normalizeRelativePath(target.from), to: target.to }); - } + for (const target of materialize) { + const sourcePath = join(entryPath, target.from); + await rm(target.to, { force: true, recursive: true }); + await mkdir(dirname(target.to), { recursive: true }); + await cp(sourcePath, target.to, { recursive: true }); + materialized.push({ from: normalizeRelativePath(target.from), to: target.to }); + } + + return nextManifest; + }); this.#entries.push({ durationMs: Date.now() - startedAt, diff --git a/tools/pack/src/win.ts b/tools/pack/src/win.ts index d8491c8374..b9817e34f0 100644 --- a/tools/pack/src/win.ts +++ b/tools/pack/src/win.ts @@ -87,6 +87,26 @@ type PackedTarballsCacheMetadata = { tarballs: PackedTarballInfo[]; }; +type PackedTarballsCacheResult = PackedTarballsCacheMetadata & { + key: string; +}; + +type AssembledAppCacheMetadata = { + packagedVersion: string; +}; + +type AssembledAppCacheResult = AssembledAppCacheMetadata & { + key: string; +}; + +type NativeRebuildCacheMetadata = { + modules: readonly string[]; +}; + +type ResourceTreeCacheMetadata = { + resourceName: "open-design"; +}; + type WinPaths = { appBuilderConfigPath: string; appBuilderOutputRoot: string; @@ -733,18 +753,18 @@ async function runNpmInstall(appRoot: string): Promise { }); } -async function rebuildWinNativeDependencies(config: ToolPackConfig, paths: WinPaths): Promise { +async function runElectronRebuild(config: ToolPackConfig, appRoot: string): Promise { const foundModules = new Set(); const rebuildResult = rebuild({ arch: "x64", buildFromSource: ELECTRON_BUILDER_BUILD_DEPENDENCIES_FROM_SOURCE, - buildPath: paths.assembledAppRoot, + buildPath: appRoot, electronVersion: config.electronVersion, force: true, mode: ELECTRON_REBUILD_MODE, onlyModules: [...ELECTRON_REBUILD_NATIVE_MODULES], platform: "win32", - projectRootPath: paths.assembledAppRoot, + projectRootPath: appRoot, }); rebuildResult.lifecycle.on("modules-found", (modules: string[]) => { for (const moduleName of modules) foundModules.add(moduleName); @@ -757,6 +777,42 @@ async function rebuildWinNativeDependencies(config: ToolPackConfig, paths: WinPa } } +function nativeRebuildOutputPath(appRoot: string): string { + return join(appRoot, "node_modules", "better-sqlite3", "build", "Release", "better_sqlite3.node"); +} + +async function rebuildWinNativeDependencies( + config: ToolPackConfig, + paths: WinPaths, + cache: ToolPackCache, + assembledApp: AssembledAppCacheResult, +): Promise { + const node = { + id: "win.native-rebuild", + key: hashJson({ + arch: "x64", + assembledAppKey: assembledApp.key, + electronVersion: config.electronVersion, + modules: ELECTRON_REBUILD_NATIVE_MODULES, + platform: "win32", + schemaVersion: 1, + }), + outputs: ["better_sqlite3.node"], + invalidate: async () => null, + build: async ({ entryRoot }: { entryRoot: string }): Promise => { + const stagingAppRoot = join(entryRoot, "app"); + await cp(paths.assembledAppRoot, stagingAppRoot, { recursive: true }); + await runElectronRebuild(config, stagingAppRoot); + await cp(nativeRebuildOutputPath(stagingAppRoot), join(entryRoot, "better_sqlite3.node")); + return { modules: ELECTRON_REBUILD_NATIVE_MODULES }; + }, + }; + await cache.acquire({ + materialize: [{ from: "better_sqlite3.node", to: nativeRebuildOutputPath(paths.assembledAppRoot) }], + node, + }); +} + async function readPackagedVersion(config: ToolPackConfig): Promise { const packageJsonPath = join(config.workspaceRoot, "apps", "packaged", "package.json"); const packageJson = JSON.parse(await readFile(packageJsonPath, "utf8")) as { version?: unknown }; @@ -829,12 +885,38 @@ async function buildWorkspaceArtifacts(config: ToolPackConfig): Promise { await runPnpm(config, ["--filter", "@open-design/packaged", "build"]); } -async function copyResourceTree(config: ToolPackConfig, paths: WinPaths): Promise { - await removeTree(paths.resourceRoot); - await mkdir(paths.resourceRoot, { recursive: true }); - await copyBundledResourceTrees({ - workspaceRoot: config.workspaceRoot, - resourceRoot: paths.resourceRoot, +async function createResourceTreeCacheKey(config: ToolPackConfig): Promise { + return hashJson({ + assetsCommunityPets: await hashPath(join(config.workspaceRoot, "assets", "community-pets")), + assetsFrames: await hashPath(join(config.workspaceRoot, "assets", "frames")), + craft: await hashPath(join(config.workspaceRoot, "craft")), + designSystems: await hashPath(join(config.workspaceRoot, "design-systems")), + node: "win.resource-tree", + promptTemplates: await hashPath(join(config.workspaceRoot, "prompt-templates")), + schemaVersion: 1, + skills: await hashPath(join(config.workspaceRoot, "skills")), + }); +} + +async function copyResourceTree(config: ToolPackConfig, paths: WinPaths, cache: ToolPackCache): Promise { + const node = { + id: "win.resource-tree", + key: await createResourceTreeCacheKey(config), + outputs: ["open-design"], + invalidate: async () => null, + build: async ({ entryRoot }: { entryRoot: string }): Promise => { + const resourceRoot = join(entryRoot, "open-design"); + await mkdir(resourceRoot, { recursive: true }); + await copyBundledResourceTrees({ + workspaceRoot: config.workspaceRoot, + resourceRoot, + }); + return { resourceName: "open-design" }; + }, + }; + await cache.acquire({ + materialize: [{ from: "open-design", to: paths.resourceRoot }], + node, }); } @@ -867,10 +949,11 @@ async function collectWorkspaceTarballs( config: ToolPackConfig, paths: WinPaths, cache: ToolPackCache, -): Promise { +): Promise { + const key = await createWorkspaceTarballsCacheKey(config); const node = { id: "win.workspace-tarballs", - key: await createWorkspaceTarballsCacheKey(config), + key, outputs: ["tarballs"], invalidate: async () => null, build: async ({ entryRoot }: { entryRoot: string }): Promise => { @@ -893,27 +976,48 @@ async function collectWorkspaceTarballs( materialize: [{ from: "tarballs", to: paths.tarballsRoot }], node, }); - return manifest.payloadMetadata.tarballs; + return { key, tarballs: manifest.payloadMetadata.tarballs }; } -async function writeAssembledApp(config: ToolPackConfig, paths: WinPaths, packedTarballs: PackedTarballInfo[]): Promise { - const packagedVersion = await readPackagedVersion(config); +async function writePackagedConfig(config: ToolPackConfig, paths: WinPaths, packagedVersion: string): Promise { + await writeFile( + paths.packagedConfigPath, + `${JSON.stringify( + { + appVersion: packagedVersion, + namespace: config.namespace, + webOutputMode: config.webOutputMode, + ...(config.portable ? {} : { namespaceBaseRoot: config.roots.runtime.namespaceBaseRoot }), + }, + null, + 2, + )}\n`, + "utf8", + ); +} + +function createAssembledAppDependencies(paths: WinPaths, packedTarballs: PackedTarballInfo[]): Record { const tarballByPackage = Object.fromEntries(packedTarballs.map((entry) => [entry.packageName, entry.fileName] as const)); - const dependencies = Object.fromEntries( + return Object.fromEntries( INTERNAL_PACKAGES.map((packageInfo) => { const tarball = tarballByPackage[packageInfo.name]; if (tarball == null) throw new Error(`missing tarball for ${packageInfo.name}`); return [packageInfo.name, `file:${relative(paths.assembledAppRoot, join(paths.tarballsRoot, tarball))}`]; }), ); +} - await removeTree(join(config.roots.output.namespaceRoot, "assembled")); +async function writeAssembledAppEntrypoints( + paths: WinPaths, + packedTarballs: PackedTarballInfo[], + packagedVersion: string, +): Promise { await mkdir(paths.assembledAppRoot, { recursive: true }); await writeFile( paths.assembledPackageJsonPath, `${JSON.stringify( { - dependencies, + dependencies: createAssembledAppDependencies(paths, packedTarballs), description: "Open Design packaged runtime", main: "./main.cjs", name: "open-design-packaged-app", @@ -931,21 +1035,59 @@ async function writeAssembledApp(config: ToolPackConfig, paths: WinPaths, packed 'import("@open-design/packaged").catch((error) => {\n console.error("packaged entry failed", error);\n process.exit(1);\n});\n', "utf8", ); - await writeFile( - paths.packagedConfigPath, - `${JSON.stringify( - { - appVersion: packagedVersion, - namespace: config.namespace, - webOutputMode: config.webOutputMode, - ...(config.portable ? {} : { namespaceBaseRoot: config.roots.runtime.namespaceBaseRoot }), - }, - null, - 2, - )}\n`, - "utf8", - ); - await runNpmInstall(paths.assembledAppRoot); +} + +async function createAssembledAppCacheKey( + config: ToolPackConfig, + tarballsKey: string, + packedTarballs: PackedTarballInfo[], + packagedVersion: string, +): Promise { + return hashJson({ + electronVersion: config.electronVersion, + node: "win.assembled-app", + packagedVersion, + packedTarballs, + platform: "win32", + schemaVersion: 1, + tarballsKey, + webOutputMode: config.webOutputMode, + }); +} + +async function writeAssembledApp( + config: ToolPackConfig, + paths: WinPaths, + tarballs: PackedTarballsCacheResult, + cache: ToolPackCache, +): Promise { + const packagedVersion = await readPackagedVersion(config); + await removeTree(join(config.roots.output.namespaceRoot, "assembled")); + const packedTarballs = tarballs.tarballs; + const key = await createAssembledAppCacheKey(config, tarballs.key, packedTarballs, packagedVersion); + const node = { + id: "win.assembled-app", + key, + outputs: ["app"], + invalidate: async () => null, + build: async ({ entryRoot }: { entryRoot: string }): Promise => { + const assembledAppRoot = join(entryRoot, "app"); + await writeAssembledAppEntrypoints( + { ...paths, assembledAppRoot, assembledMainEntryPath: join(assembledAppRoot, "main.cjs"), assembledPackageJsonPath: join(assembledAppRoot, "package.json") }, + packedTarballs, + packagedVersion, + ); + await runNpmInstall(assembledAppRoot); + return { packagedVersion }; + }, + }; + await cache.acquire({ + materialize: [{ from: "app", to: paths.assembledAppRoot }], + node, + }); + await writeAssembledAppEntrypoints(paths, packedTarballs, packagedVersion); + await writePackagedConfig(config, paths, packagedVersion); + return { key, packagedVersion }; } function resolveWinTargets(to: ToolPackConfig["to"]): Array<"dir" | "nsis"> { @@ -1164,11 +1306,11 @@ export async function packWin(config: ToolPackConfig): Promise { const paths = resolveWinPaths(config); const cache = new ToolPackCache(config.roots.cacheRoot); await buildWorkspaceArtifacts(config); - await copyResourceTree(config, paths); + await copyResourceTree(config, paths, cache); await copyWinIcon(paths); const tarballs = await collectWorkspaceTarballs(config, paths, cache); - await writeAssembledApp(config, paths, tarballs); - await rebuildWinNativeDependencies(config, paths); + const assembledApp = await writeAssembledApp(config, paths, tarballs, cache); + await rebuildWinNativeDependencies(config, paths, cache, assembledApp); await runElectronBuilder(config, paths); await writeLocalLatestYml(config, paths); const sizeReport = await collectWinSizeReport(config, paths);