mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
fix: build script venv handling and lint fixes
- Use /opt/venv directly when --entrypoint bash bypasses entrypoint.sh - Use sys.executable for all pip calls (not bare pip) - Override entrypoint in CI workflow to avoid startup banner - Fix Biome formatting (template literals, try/catch blocks)
This commit is contained in:
@@ -102,11 +102,11 @@ jobs:
|
||||
VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
mkdir -p /tmp/bundles
|
||||
docker run --rm \
|
||||
docker run --rm --entrypoint bash \
|
||||
-v "$PWD/docker/build-bundle.sh:/build-bundle.sh:ro" \
|
||||
-v "/tmp/bundles:/output" \
|
||||
"ghcr.io/snapotter-hq/snapotter:${VERSION}" \
|
||||
bash /build-bundle.sh "${BUNDLE}" "${ARCH}" /output
|
||||
/build-bundle.sh "${BUNDLE}" "${ARCH}" /output
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
|
||||
@@ -655,15 +655,21 @@ export async function importBundleArchive(
|
||||
if (existsSync(stagingFixups)) {
|
||||
const wheels = readdirSync(stagingFixups).filter((f) => f.endsWith(".whl"));
|
||||
if (wheels.length > 0) {
|
||||
const venvPython =
|
||||
(process.env.PYTHON_VENV_PATH || join(AI_DIR, "venv")) + "/bin/python3";
|
||||
const venvPython = `${process.env.PYTHON_VENV_PATH || join(AI_DIR, "venv")}/bin/python3`;
|
||||
for (const wheel of wheels) {
|
||||
try {
|
||||
execFileSync(venvPython, [
|
||||
"-m", "pip", "install", "--no-index",
|
||||
`--find-links=${stagingFixups}`,
|
||||
wheel.split("-")[0],
|
||||
], { stdio: "ignore", timeout: 30_000 });
|
||||
execFileSync(
|
||||
venvPython,
|
||||
[
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"--no-index",
|
||||
`--find-links=${stagingFixups}`,
|
||||
wheel.split("-")[0],
|
||||
],
|
||||
{ stdio: "ignore", timeout: 30_000 },
|
||||
);
|
||||
} catch {
|
||||
// Non-fatal
|
||||
}
|
||||
|
||||
+22
-5
@@ -21,14 +21,31 @@ export ARCH="${2:?Usage: build-bundle.sh <bundleId> <arch> <outputDir>}"
|
||||
OUTPUT_DIR="${3:?Usage: build-bundle.sh <bundleId> <arch> <outputDir>}"
|
||||
|
||||
MANIFEST="/app/docker/feature-manifest.json"
|
||||
SITE_PACKAGES="$(python3 -c 'import site; print(site.getsitepackages()[0])')"
|
||||
VENV_PATH="${PYTHON_VENV_PATH:-/opt/venv}"
|
||||
export MODELS_DIR="/tmp/bundle-models"
|
||||
export BUILD_DIR="/tmp/bundle-build"
|
||||
|
||||
# When running with --entrypoint bash (bypassing entrypoint.sh), the venv
|
||||
# at /data/ai/venv won't exist yet. Use /opt/venv directly -- it's the base
|
||||
# venv baked into the Docker image, and that's exactly what we want as the
|
||||
# starting point for building bundle deltas.
|
||||
if [[ ! -f "${VENV_PATH}/bin/activate" && -f "/opt/venv/bin/activate" ]]; then
|
||||
VENV_PATH="/opt/venv"
|
||||
fi
|
||||
|
||||
# Activate the venv so pip/python3 use it (not system Python)
|
||||
if [[ -f "${VENV_PATH}/bin/activate" ]]; then
|
||||
# shellcheck disable=SC1091
|
||||
source "${VENV_PATH}/bin/activate"
|
||||
fi
|
||||
|
||||
SITE_PACKAGES="$("${VENV_PATH}/bin/python3" -c 'import site; print(site.getsitepackages()[0])')"
|
||||
|
||||
# Parse platform from arch: amd64-gpu -> amd64, arm64-cpu -> arm64
|
||||
export PLATFORM="${ARCH%%-*}"
|
||||
|
||||
echo "=== Building bundle: ${BUNDLE_ID} arch=${ARCH} platform=${PLATFORM} ==="
|
||||
echo "Venv: ${VENV_PATH}"
|
||||
echo "Site-packages: ${SITE_PACKAGES}"
|
||||
|
||||
# Validate manifest exists
|
||||
@@ -85,7 +102,7 @@ for pkg_string in packages:
|
||||
break
|
||||
|
||||
# pkg_string may contain embedded flags (e.g. --index-url), so pass as-is
|
||||
cmd = f"pip install --no-cache-dir {extra_flags} {pkg_string}".strip()
|
||||
cmd = f"{sys.executable} -m pip install --no-cache-dir {extra_flags} {pkg_string}".strip()
|
||||
print(f" > {cmd}", flush=True)
|
||||
result = subprocess.run(cmd, shell=True)
|
||||
if result.returncode != 0:
|
||||
@@ -111,7 +128,7 @@ if not post_install:
|
||||
sys.exit(0)
|
||||
|
||||
for pkg in post_install:
|
||||
cmd = f"pip install --no-cache-dir --force-reinstall {pkg}"
|
||||
cmd = f"{sys.executable} -m pip install --no-cache-dir --force-reinstall {pkg}"
|
||||
print(f" > {cmd}", flush=True)
|
||||
result = subprocess.run(cmd, shell=True)
|
||||
if result.returncode != 0:
|
||||
@@ -135,7 +152,7 @@ if not base_packages:
|
||||
sys.exit(0)
|
||||
|
||||
pkgs = " ".join(base_packages)
|
||||
cmd = f"pip install --no-cache-dir --force-reinstall {pkgs}"
|
||||
cmd = f"{sys.executable} -m pip install --no-cache-dir --force-reinstall {pkgs}"
|
||||
print(f" > {cmd}", flush=True)
|
||||
result = subprocess.run(cmd, shell=True)
|
||||
if result.returncode != 0:
|
||||
@@ -262,7 +279,7 @@ os.makedirs(fixups_dir, exist_ok=True)
|
||||
for pkg in nccl_pkgs:
|
||||
print(f" Downloading NCCL wheel: {pkg}", flush=True)
|
||||
result = subprocess.run(
|
||||
["pip", "download", "--no-cache-dir", "-d", fixups_dir, pkg]
|
||||
[sys.executable, "-m", "pip", "download", "--no-cache-dir", "-d", fixups_dir, pkg]
|
||||
)
|
||||
if result.returncode != 0:
|
||||
print(f" WARNING: Failed to download NCCL wheel: {pkg}", file=sys.stderr)
|
||||
|
||||
@@ -1,13 +1,6 @@
|
||||
import { spawnSync, execFileSync } from "node:child_process";
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
@@ -93,19 +86,15 @@ describe("install_feature.py prebuilt mode", () => {
|
||||
const { tarPath, sha256 } = createTestTar("face-detection");
|
||||
writeManifest("face-detection", tarPath, sha256);
|
||||
|
||||
const result = spawnSync(
|
||||
"python3",
|
||||
[scriptPath, "face-detection", manifestPath, modelsDir],
|
||||
{
|
||||
env: {
|
||||
...process.env,
|
||||
DATA_DIR: tempDir,
|
||||
PYTHON_VENV_PATH: venvDir,
|
||||
SNAPOTTER_BUNDLE_LOCAL_PATH: tarPath,
|
||||
},
|
||||
timeout: 30_000,
|
||||
const result = spawnSync("python3", [scriptPath, "face-detection", manifestPath, modelsDir], {
|
||||
env: {
|
||||
...process.env,
|
||||
DATA_DIR: tempDir,
|
||||
PYTHON_VENV_PATH: venvDir,
|
||||
SNAPOTTER_BUNDLE_LOCAL_PATH: tarPath,
|
||||
},
|
||||
);
|
||||
timeout: 30_000,
|
||||
});
|
||||
|
||||
expect(result.status, `stderr: ${result.stderr?.toString()}`).toBe(0);
|
||||
expect(existsSync(join(modelsDir, "testmodel", "weights.bin"))).toBe(true);
|
||||
@@ -120,19 +109,15 @@ describe("install_feature.py prebuilt mode", () => {
|
||||
const { tarPath } = createTestTar("face-detection");
|
||||
writeManifest("face-detection", tarPath, "badhash".padEnd(64, "0"));
|
||||
|
||||
const result = spawnSync(
|
||||
"python3",
|
||||
[scriptPath, "face-detection", manifestPath, modelsDir],
|
||||
{
|
||||
env: {
|
||||
...process.env,
|
||||
DATA_DIR: tempDir,
|
||||
PYTHON_VENV_PATH: venvDir,
|
||||
SNAPOTTER_BUNDLE_LOCAL_PATH: tarPath,
|
||||
},
|
||||
timeout: 30_000,
|
||||
const result = spawnSync("python3", [scriptPath, "face-detection", manifestPath, modelsDir], {
|
||||
env: {
|
||||
...process.env,
|
||||
DATA_DIR: tempDir,
|
||||
PYTHON_VENV_PATH: venvDir,
|
||||
SNAPOTTER_BUNDLE_LOCAL_PATH: tarPath,
|
||||
},
|
||||
);
|
||||
timeout: 30_000,
|
||||
});
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
});
|
||||
@@ -141,23 +126,24 @@ describe("install_feature.py prebuilt mode", () => {
|
||||
const { tarPath, sha256 } = createTestTar("face-detection");
|
||||
writeManifest("face-detection", tarPath, sha256);
|
||||
|
||||
const result = spawnSync(
|
||||
"python3",
|
||||
[scriptPath, "face-detection", manifestPath, modelsDir],
|
||||
{
|
||||
env: {
|
||||
...process.env,
|
||||
DATA_DIR: tempDir,
|
||||
PYTHON_VENV_PATH: venvDir,
|
||||
SNAPOTTER_BUNDLE_LOCAL_PATH: tarPath,
|
||||
},
|
||||
timeout: 30_000,
|
||||
const result = spawnSync("python3", [scriptPath, "face-detection", manifestPath, modelsDir], {
|
||||
env: {
|
||||
...process.env,
|
||||
DATA_DIR: tempDir,
|
||||
PYTHON_VENV_PATH: venvDir,
|
||||
SNAPOTTER_BUNDLE_LOCAL_PATH: tarPath,
|
||||
},
|
||||
);
|
||||
timeout: 30_000,
|
||||
});
|
||||
|
||||
const stderr = result.stderr?.toString() ?? "";
|
||||
const progressLines = stderr.split("\n").filter((l) => {
|
||||
try { const p = JSON.parse(l); return typeof p.progress === "number"; } catch { return false; }
|
||||
try {
|
||||
const p = JSON.parse(l);
|
||||
return typeof p.progress === "number";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
expect(progressLines.length).toBeGreaterThan(0);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user