mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
fix: build script venv handling and lint fixes
- Use /opt/venv directly when --entrypoint bash bypasses entrypoint.sh - Use sys.executable for all pip calls (not bare pip) - Override entrypoint in CI workflow to avoid startup banner - Fix Biome formatting (template literals, try/catch blocks)
This commit is contained in:
@@ -102,11 +102,11 @@ jobs:
|
|||||||
VERSION: ${{ inputs.version }}
|
VERSION: ${{ inputs.version }}
|
||||||
run: |
|
run: |
|
||||||
mkdir -p /tmp/bundles
|
mkdir -p /tmp/bundles
|
||||||
docker run --rm \
|
docker run --rm --entrypoint bash \
|
||||||
-v "$PWD/docker/build-bundle.sh:/build-bundle.sh:ro" \
|
-v "$PWD/docker/build-bundle.sh:/build-bundle.sh:ro" \
|
||||||
-v "/tmp/bundles:/output" \
|
-v "/tmp/bundles:/output" \
|
||||||
"ghcr.io/snapotter-hq/snapotter:${VERSION}" \
|
"ghcr.io/snapotter-hq/snapotter:${VERSION}" \
|
||||||
bash /build-bundle.sh "${BUNDLE}" "${ARCH}" /output
|
/build-bundle.sh "${BUNDLE}" "${ARCH}" /output
|
||||||
|
|
||||||
- name: Upload artifact
|
- name: Upload artifact
|
||||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||||
|
|||||||
@@ -655,15 +655,21 @@ export async function importBundleArchive(
|
|||||||
if (existsSync(stagingFixups)) {
|
if (existsSync(stagingFixups)) {
|
||||||
const wheels = readdirSync(stagingFixups).filter((f) => f.endsWith(".whl"));
|
const wheels = readdirSync(stagingFixups).filter((f) => f.endsWith(".whl"));
|
||||||
if (wheels.length > 0) {
|
if (wheels.length > 0) {
|
||||||
const venvPython =
|
const venvPython = `${process.env.PYTHON_VENV_PATH || join(AI_DIR, "venv")}/bin/python3`;
|
||||||
(process.env.PYTHON_VENV_PATH || join(AI_DIR, "venv")) + "/bin/python3";
|
|
||||||
for (const wheel of wheels) {
|
for (const wheel of wheels) {
|
||||||
try {
|
try {
|
||||||
execFileSync(venvPython, [
|
execFileSync(
|
||||||
"-m", "pip", "install", "--no-index",
|
venvPython,
|
||||||
`--find-links=${stagingFixups}`,
|
[
|
||||||
wheel.split("-")[0],
|
"-m",
|
||||||
], { stdio: "ignore", timeout: 30_000 });
|
"pip",
|
||||||
|
"install",
|
||||||
|
"--no-index",
|
||||||
|
`--find-links=${stagingFixups}`,
|
||||||
|
wheel.split("-")[0],
|
||||||
|
],
|
||||||
|
{ stdio: "ignore", timeout: 30_000 },
|
||||||
|
);
|
||||||
} catch {
|
} catch {
|
||||||
// Non-fatal
|
// Non-fatal
|
||||||
}
|
}
|
||||||
|
|||||||
+22
-5
@@ -21,14 +21,31 @@ export ARCH="${2:?Usage: build-bundle.sh <bundleId> <arch> <outputDir>}"
|
|||||||
OUTPUT_DIR="${3:?Usage: build-bundle.sh <bundleId> <arch> <outputDir>}"
|
OUTPUT_DIR="${3:?Usage: build-bundle.sh <bundleId> <arch> <outputDir>}"
|
||||||
|
|
||||||
MANIFEST="/app/docker/feature-manifest.json"
|
MANIFEST="/app/docker/feature-manifest.json"
|
||||||
SITE_PACKAGES="$(python3 -c 'import site; print(site.getsitepackages()[0])')"
|
VENV_PATH="${PYTHON_VENV_PATH:-/opt/venv}"
|
||||||
export MODELS_DIR="/tmp/bundle-models"
|
export MODELS_DIR="/tmp/bundle-models"
|
||||||
export BUILD_DIR="/tmp/bundle-build"
|
export BUILD_DIR="/tmp/bundle-build"
|
||||||
|
|
||||||
|
# When running with --entrypoint bash (bypassing entrypoint.sh), the venv
|
||||||
|
# at /data/ai/venv won't exist yet. Use /opt/venv directly -- it's the base
|
||||||
|
# venv baked into the Docker image, and that's exactly what we want as the
|
||||||
|
# starting point for building bundle deltas.
|
||||||
|
if [[ ! -f "${VENV_PATH}/bin/activate" && -f "/opt/venv/bin/activate" ]]; then
|
||||||
|
VENV_PATH="/opt/venv"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Activate the venv so pip/python3 use it (not system Python)
|
||||||
|
if [[ -f "${VENV_PATH}/bin/activate" ]]; then
|
||||||
|
# shellcheck disable=SC1091
|
||||||
|
source "${VENV_PATH}/bin/activate"
|
||||||
|
fi
|
||||||
|
|
||||||
|
SITE_PACKAGES="$("${VENV_PATH}/bin/python3" -c 'import site; print(site.getsitepackages()[0])')"
|
||||||
|
|
||||||
# Parse platform from arch: amd64-gpu -> amd64, arm64-cpu -> arm64
|
# Parse platform from arch: amd64-gpu -> amd64, arm64-cpu -> arm64
|
||||||
export PLATFORM="${ARCH%%-*}"
|
export PLATFORM="${ARCH%%-*}"
|
||||||
|
|
||||||
echo "=== Building bundle: ${BUNDLE_ID} arch=${ARCH} platform=${PLATFORM} ==="
|
echo "=== Building bundle: ${BUNDLE_ID} arch=${ARCH} platform=${PLATFORM} ==="
|
||||||
|
echo "Venv: ${VENV_PATH}"
|
||||||
echo "Site-packages: ${SITE_PACKAGES}"
|
echo "Site-packages: ${SITE_PACKAGES}"
|
||||||
|
|
||||||
# Validate manifest exists
|
# Validate manifest exists
|
||||||
@@ -85,7 +102,7 @@ for pkg_string in packages:
|
|||||||
break
|
break
|
||||||
|
|
||||||
# pkg_string may contain embedded flags (e.g. --index-url), so pass as-is
|
# pkg_string may contain embedded flags (e.g. --index-url), so pass as-is
|
||||||
cmd = f"pip install --no-cache-dir {extra_flags} {pkg_string}".strip()
|
cmd = f"{sys.executable} -m pip install --no-cache-dir {extra_flags} {pkg_string}".strip()
|
||||||
print(f" > {cmd}", flush=True)
|
print(f" > {cmd}", flush=True)
|
||||||
result = subprocess.run(cmd, shell=True)
|
result = subprocess.run(cmd, shell=True)
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
@@ -111,7 +128,7 @@ if not post_install:
|
|||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
for pkg in post_install:
|
for pkg in post_install:
|
||||||
cmd = f"pip install --no-cache-dir --force-reinstall {pkg}"
|
cmd = f"{sys.executable} -m pip install --no-cache-dir --force-reinstall {pkg}"
|
||||||
print(f" > {cmd}", flush=True)
|
print(f" > {cmd}", flush=True)
|
||||||
result = subprocess.run(cmd, shell=True)
|
result = subprocess.run(cmd, shell=True)
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
@@ -135,7 +152,7 @@ if not base_packages:
|
|||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
pkgs = " ".join(base_packages)
|
pkgs = " ".join(base_packages)
|
||||||
cmd = f"pip install --no-cache-dir --force-reinstall {pkgs}"
|
cmd = f"{sys.executable} -m pip install --no-cache-dir --force-reinstall {pkgs}"
|
||||||
print(f" > {cmd}", flush=True)
|
print(f" > {cmd}", flush=True)
|
||||||
result = subprocess.run(cmd, shell=True)
|
result = subprocess.run(cmd, shell=True)
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
@@ -262,7 +279,7 @@ os.makedirs(fixups_dir, exist_ok=True)
|
|||||||
for pkg in nccl_pkgs:
|
for pkg in nccl_pkgs:
|
||||||
print(f" Downloading NCCL wheel: {pkg}", flush=True)
|
print(f" Downloading NCCL wheel: {pkg}", flush=True)
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["pip", "download", "--no-cache-dir", "-d", fixups_dir, pkg]
|
[sys.executable, "-m", "pip", "download", "--no-cache-dir", "-d", fixups_dir, pkg]
|
||||||
)
|
)
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
print(f" WARNING: Failed to download NCCL wheel: {pkg}", file=sys.stderr)
|
print(f" WARNING: Failed to download NCCL wheel: {pkg}", file=sys.stderr)
|
||||||
|
|||||||
@@ -1,13 +1,6 @@
|
|||||||
import { spawnSync, execFileSync } from "node:child_process";
|
import { execFileSync, spawnSync } from "node:child_process";
|
||||||
import {
|
|
||||||
existsSync,
|
|
||||||
mkdirSync,
|
|
||||||
mkdtempSync,
|
|
||||||
readFileSync,
|
|
||||||
rmSync,
|
|
||||||
writeFileSync,
|
|
||||||
} from "node:fs";
|
|
||||||
import { createHash } from "node:crypto";
|
import { createHash } from "node:crypto";
|
||||||
|
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||||
@@ -93,19 +86,15 @@ describe("install_feature.py prebuilt mode", () => {
|
|||||||
const { tarPath, sha256 } = createTestTar("face-detection");
|
const { tarPath, sha256 } = createTestTar("face-detection");
|
||||||
writeManifest("face-detection", tarPath, sha256);
|
writeManifest("face-detection", tarPath, sha256);
|
||||||
|
|
||||||
const result = spawnSync(
|
const result = spawnSync("python3", [scriptPath, "face-detection", manifestPath, modelsDir], {
|
||||||
"python3",
|
env: {
|
||||||
[scriptPath, "face-detection", manifestPath, modelsDir],
|
...process.env,
|
||||||
{
|
DATA_DIR: tempDir,
|
||||||
env: {
|
PYTHON_VENV_PATH: venvDir,
|
||||||
...process.env,
|
SNAPOTTER_BUNDLE_LOCAL_PATH: tarPath,
|
||||||
DATA_DIR: tempDir,
|
|
||||||
PYTHON_VENV_PATH: venvDir,
|
|
||||||
SNAPOTTER_BUNDLE_LOCAL_PATH: tarPath,
|
|
||||||
},
|
|
||||||
timeout: 30_000,
|
|
||||||
},
|
},
|
||||||
);
|
timeout: 30_000,
|
||||||
|
});
|
||||||
|
|
||||||
expect(result.status, `stderr: ${result.stderr?.toString()}`).toBe(0);
|
expect(result.status, `stderr: ${result.stderr?.toString()}`).toBe(0);
|
||||||
expect(existsSync(join(modelsDir, "testmodel", "weights.bin"))).toBe(true);
|
expect(existsSync(join(modelsDir, "testmodel", "weights.bin"))).toBe(true);
|
||||||
@@ -120,19 +109,15 @@ describe("install_feature.py prebuilt mode", () => {
|
|||||||
const { tarPath } = createTestTar("face-detection");
|
const { tarPath } = createTestTar("face-detection");
|
||||||
writeManifest("face-detection", tarPath, "badhash".padEnd(64, "0"));
|
writeManifest("face-detection", tarPath, "badhash".padEnd(64, "0"));
|
||||||
|
|
||||||
const result = spawnSync(
|
const result = spawnSync("python3", [scriptPath, "face-detection", manifestPath, modelsDir], {
|
||||||
"python3",
|
env: {
|
||||||
[scriptPath, "face-detection", manifestPath, modelsDir],
|
...process.env,
|
||||||
{
|
DATA_DIR: tempDir,
|
||||||
env: {
|
PYTHON_VENV_PATH: venvDir,
|
||||||
...process.env,
|
SNAPOTTER_BUNDLE_LOCAL_PATH: tarPath,
|
||||||
DATA_DIR: tempDir,
|
|
||||||
PYTHON_VENV_PATH: venvDir,
|
|
||||||
SNAPOTTER_BUNDLE_LOCAL_PATH: tarPath,
|
|
||||||
},
|
|
||||||
timeout: 30_000,
|
|
||||||
},
|
},
|
||||||
);
|
timeout: 30_000,
|
||||||
|
});
|
||||||
|
|
||||||
expect(result.status).not.toBe(0);
|
expect(result.status).not.toBe(0);
|
||||||
});
|
});
|
||||||
@@ -141,23 +126,24 @@ describe("install_feature.py prebuilt mode", () => {
|
|||||||
const { tarPath, sha256 } = createTestTar("face-detection");
|
const { tarPath, sha256 } = createTestTar("face-detection");
|
||||||
writeManifest("face-detection", tarPath, sha256);
|
writeManifest("face-detection", tarPath, sha256);
|
||||||
|
|
||||||
const result = spawnSync(
|
const result = spawnSync("python3", [scriptPath, "face-detection", manifestPath, modelsDir], {
|
||||||
"python3",
|
env: {
|
||||||
[scriptPath, "face-detection", manifestPath, modelsDir],
|
...process.env,
|
||||||
{
|
DATA_DIR: tempDir,
|
||||||
env: {
|
PYTHON_VENV_PATH: venvDir,
|
||||||
...process.env,
|
SNAPOTTER_BUNDLE_LOCAL_PATH: tarPath,
|
||||||
DATA_DIR: tempDir,
|
|
||||||
PYTHON_VENV_PATH: venvDir,
|
|
||||||
SNAPOTTER_BUNDLE_LOCAL_PATH: tarPath,
|
|
||||||
},
|
|
||||||
timeout: 30_000,
|
|
||||||
},
|
},
|
||||||
);
|
timeout: 30_000,
|
||||||
|
});
|
||||||
|
|
||||||
const stderr = result.stderr?.toString() ?? "";
|
const stderr = result.stderr?.toString() ?? "";
|
||||||
const progressLines = stderr.split("\n").filter((l) => {
|
const progressLines = stderr.split("\n").filter((l) => {
|
||||||
try { const p = JSON.parse(l); return typeof p.progress === "number"; } catch { return false; }
|
try {
|
||||||
|
const p = JSON.parse(l);
|
||||||
|
return typeof p.progress === "number";
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
});
|
});
|
||||||
expect(progressLines.length).toBeGreaterThan(0);
|
expect(progressLines.length).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user