fix: build script venv handling and lint fixes

- Use /opt/venv directly when --entrypoint bash bypasses entrypoint.sh
- Use sys.executable for all pip calls (not bare pip)
- Override entrypoint in CI workflow to avoid startup banner
- Fix Biome formatting (template literals, try/catch blocks)
This commit is contained in:
SnapOtter
2026-06-13 20:48:27 +08:00
parent d0732d4a58
commit b1e94bd988
4 changed files with 69 additions and 60 deletions
+2 -2
View File
@@ -102,11 +102,11 @@ jobs:
VERSION: ${{ inputs.version }} VERSION: ${{ inputs.version }}
run: | run: |
mkdir -p /tmp/bundles mkdir -p /tmp/bundles
docker run --rm \ docker run --rm --entrypoint bash \
-v "$PWD/docker/build-bundle.sh:/build-bundle.sh:ro" \ -v "$PWD/docker/build-bundle.sh:/build-bundle.sh:ro" \
-v "/tmp/bundles:/output" \ -v "/tmp/bundles:/output" \
"ghcr.io/snapotter-hq/snapotter:${VERSION}" \ "ghcr.io/snapotter-hq/snapotter:${VERSION}" \
bash /build-bundle.sh "${BUNDLE}" "${ARCH}" /output /build-bundle.sh "${BUNDLE}" "${ARCH}" /output
- name: Upload artifact - name: Upload artifact
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+11 -5
View File
@@ -655,15 +655,21 @@ export async function importBundleArchive(
if (existsSync(stagingFixups)) { if (existsSync(stagingFixups)) {
const wheels = readdirSync(stagingFixups).filter((f) => f.endsWith(".whl")); const wheels = readdirSync(stagingFixups).filter((f) => f.endsWith(".whl"));
if (wheels.length > 0) { if (wheels.length > 0) {
const venvPython = const venvPython = `${process.env.PYTHON_VENV_PATH || join(AI_DIR, "venv")}/bin/python3`;
(process.env.PYTHON_VENV_PATH || join(AI_DIR, "venv")) + "/bin/python3";
for (const wheel of wheels) { for (const wheel of wheels) {
try { try {
execFileSync(venvPython, [ execFileSync(
"-m", "pip", "install", "--no-index", venvPython,
[
"-m",
"pip",
"install",
"--no-index",
`--find-links=${stagingFixups}`, `--find-links=${stagingFixups}`,
wheel.split("-")[0], wheel.split("-")[0],
], { stdio: "ignore", timeout: 30_000 }); ],
{ stdio: "ignore", timeout: 30_000 },
);
} catch { } catch {
// Non-fatal // Non-fatal
} }
+22 -5
View File
@@ -21,14 +21,31 @@ export ARCH="${2:?Usage: build-bundle.sh <bundleId> <arch> <outputDir>}"
OUTPUT_DIR="${3:?Usage: build-bundle.sh <bundleId> <arch> <outputDir>}" OUTPUT_DIR="${3:?Usage: build-bundle.sh <bundleId> <arch> <outputDir>}"
MANIFEST="/app/docker/feature-manifest.json" MANIFEST="/app/docker/feature-manifest.json"
SITE_PACKAGES="$(python3 -c 'import site; print(site.getsitepackages()[0])')" VENV_PATH="${PYTHON_VENV_PATH:-/opt/venv}"
export MODELS_DIR="/tmp/bundle-models" export MODELS_DIR="/tmp/bundle-models"
export BUILD_DIR="/tmp/bundle-build" export BUILD_DIR="/tmp/bundle-build"
# When running with --entrypoint bash (bypassing entrypoint.sh), the venv
# at /data/ai/venv won't exist yet. Use /opt/venv directly -- it's the base
# venv baked into the Docker image, and that's exactly what we want as the
# starting point for building bundle deltas.
if [[ ! -f "${VENV_PATH}/bin/activate" && -f "/opt/venv/bin/activate" ]]; then
VENV_PATH="/opt/venv"
fi
# Activate the venv so pip/python3 use it (not system Python)
if [[ -f "${VENV_PATH}/bin/activate" ]]; then
# shellcheck disable=SC1091
source "${VENV_PATH}/bin/activate"
fi
SITE_PACKAGES="$("${VENV_PATH}/bin/python3" -c 'import site; print(site.getsitepackages()[0])')"
# Parse platform from arch: amd64-gpu -> amd64, arm64-cpu -> arm64 # Parse platform from arch: amd64-gpu -> amd64, arm64-cpu -> arm64
export PLATFORM="${ARCH%%-*}" export PLATFORM="${ARCH%%-*}"
echo "=== Building bundle: ${BUNDLE_ID} arch=${ARCH} platform=${PLATFORM} ===" echo "=== Building bundle: ${BUNDLE_ID} arch=${ARCH} platform=${PLATFORM} ==="
echo "Venv: ${VENV_PATH}"
echo "Site-packages: ${SITE_PACKAGES}" echo "Site-packages: ${SITE_PACKAGES}"
# Validate manifest exists # Validate manifest exists
@@ -85,7 +102,7 @@ for pkg_string in packages:
break break
# pkg_string may contain embedded flags (e.g. --index-url), so pass as-is # pkg_string may contain embedded flags (e.g. --index-url), so pass as-is
cmd = f"pip install --no-cache-dir {extra_flags} {pkg_string}".strip() cmd = f"{sys.executable} -m pip install --no-cache-dir {extra_flags} {pkg_string}".strip()
print(f" > {cmd}", flush=True) print(f" > {cmd}", flush=True)
result = subprocess.run(cmd, shell=True) result = subprocess.run(cmd, shell=True)
if result.returncode != 0: if result.returncode != 0:
@@ -111,7 +128,7 @@ if not post_install:
sys.exit(0) sys.exit(0)
for pkg in post_install: for pkg in post_install:
cmd = f"pip install --no-cache-dir --force-reinstall {pkg}" cmd = f"{sys.executable} -m pip install --no-cache-dir --force-reinstall {pkg}"
print(f" > {cmd}", flush=True) print(f" > {cmd}", flush=True)
result = subprocess.run(cmd, shell=True) result = subprocess.run(cmd, shell=True)
if result.returncode != 0: if result.returncode != 0:
@@ -135,7 +152,7 @@ if not base_packages:
sys.exit(0) sys.exit(0)
pkgs = " ".join(base_packages) pkgs = " ".join(base_packages)
cmd = f"pip install --no-cache-dir --force-reinstall {pkgs}" cmd = f"{sys.executable} -m pip install --no-cache-dir --force-reinstall {pkgs}"
print(f" > {cmd}", flush=True) print(f" > {cmd}", flush=True)
result = subprocess.run(cmd, shell=True) result = subprocess.run(cmd, shell=True)
if result.returncode != 0: if result.returncode != 0:
@@ -262,7 +279,7 @@ os.makedirs(fixups_dir, exist_ok=True)
for pkg in nccl_pkgs: for pkg in nccl_pkgs:
print(f" Downloading NCCL wheel: {pkg}", flush=True) print(f" Downloading NCCL wheel: {pkg}", flush=True)
result = subprocess.run( result = subprocess.run(
["pip", "download", "--no-cache-dir", "-d", fixups_dir, pkg] [sys.executable, "-m", "pip", "download", "--no-cache-dir", "-d", fixups_dir, pkg]
) )
if result.returncode != 0: if result.returncode != 0:
print(f" WARNING: Failed to download NCCL wheel: {pkg}", file=sys.stderr) print(f" WARNING: Failed to download NCCL wheel: {pkg}", file=sys.stderr)
@@ -1,13 +1,6 @@
import { spawnSync, execFileSync } from "node:child_process"; import { execFileSync, spawnSync } from "node:child_process";
import {
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
writeFileSync,
} from "node:fs";
import { createHash } from "node:crypto"; import { createHash } from "node:crypto";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os"; import { tmpdir } from "node:os";
import { join } from "node:path"; import { join } from "node:path";
import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { afterEach, beforeEach, describe, expect, it } from "vitest";
@@ -93,10 +86,7 @@ describe("install_feature.py prebuilt mode", () => {
const { tarPath, sha256 } = createTestTar("face-detection"); const { tarPath, sha256 } = createTestTar("face-detection");
writeManifest("face-detection", tarPath, sha256); writeManifest("face-detection", tarPath, sha256);
const result = spawnSync( const result = spawnSync("python3", [scriptPath, "face-detection", manifestPath, modelsDir], {
"python3",
[scriptPath, "face-detection", manifestPath, modelsDir],
{
env: { env: {
...process.env, ...process.env,
DATA_DIR: tempDir, DATA_DIR: tempDir,
@@ -104,8 +94,7 @@ describe("install_feature.py prebuilt mode", () => {
SNAPOTTER_BUNDLE_LOCAL_PATH: tarPath, SNAPOTTER_BUNDLE_LOCAL_PATH: tarPath,
}, },
timeout: 30_000, timeout: 30_000,
}, });
);
expect(result.status, `stderr: ${result.stderr?.toString()}`).toBe(0); expect(result.status, `stderr: ${result.stderr?.toString()}`).toBe(0);
expect(existsSync(join(modelsDir, "testmodel", "weights.bin"))).toBe(true); expect(existsSync(join(modelsDir, "testmodel", "weights.bin"))).toBe(true);
@@ -120,10 +109,7 @@ describe("install_feature.py prebuilt mode", () => {
const { tarPath } = createTestTar("face-detection"); const { tarPath } = createTestTar("face-detection");
writeManifest("face-detection", tarPath, "badhash".padEnd(64, "0")); writeManifest("face-detection", tarPath, "badhash".padEnd(64, "0"));
const result = spawnSync( const result = spawnSync("python3", [scriptPath, "face-detection", manifestPath, modelsDir], {
"python3",
[scriptPath, "face-detection", manifestPath, modelsDir],
{
env: { env: {
...process.env, ...process.env,
DATA_DIR: tempDir, DATA_DIR: tempDir,
@@ -131,8 +117,7 @@ describe("install_feature.py prebuilt mode", () => {
SNAPOTTER_BUNDLE_LOCAL_PATH: tarPath, SNAPOTTER_BUNDLE_LOCAL_PATH: tarPath,
}, },
timeout: 30_000, timeout: 30_000,
}, });
);
expect(result.status).not.toBe(0); expect(result.status).not.toBe(0);
}); });
@@ -141,10 +126,7 @@ describe("install_feature.py prebuilt mode", () => {
const { tarPath, sha256 } = createTestTar("face-detection"); const { tarPath, sha256 } = createTestTar("face-detection");
writeManifest("face-detection", tarPath, sha256); writeManifest("face-detection", tarPath, sha256);
const result = spawnSync( const result = spawnSync("python3", [scriptPath, "face-detection", manifestPath, modelsDir], {
"python3",
[scriptPath, "face-detection", manifestPath, modelsDir],
{
env: { env: {
...process.env, ...process.env,
DATA_DIR: tempDir, DATA_DIR: tempDir,
@@ -152,12 +134,16 @@ describe("install_feature.py prebuilt mode", () => {
SNAPOTTER_BUNDLE_LOCAL_PATH: tarPath, SNAPOTTER_BUNDLE_LOCAL_PATH: tarPath,
}, },
timeout: 30_000, timeout: 30_000,
}, });
);
const stderr = result.stderr?.toString() ?? ""; const stderr = result.stderr?.toString() ?? "";
const progressLines = stderr.split("\n").filter((l) => { const progressLines = stderr.split("\n").filter((l) => {
try { const p = JSON.parse(l); return typeof p.progress === "number"; } catch { return false; } try {
const p = JSON.parse(l);
return typeof p.progress === "number";
} catch {
return false;
}
}); });
expect(progressLines.length).toBeGreaterThan(0); expect(progressLines.length).toBeGreaterThan(0);