mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
SnapOtterandGitHub
ada5727114
Build listed six test jobs in `needs` but only reads `needs.changes.outputs.code`, in its `if`. The rest were ordering, not data. That queued a 55s job behind the slowest integration shard, so it added its full runtime to the tail of every green run. Build and the test jobs are independent required checks, so a green Build still can't merge past a red shard.
450 lines
18 KiB
YAML
450 lines
18 KiB
YAML
name: CI
|
|
|
|
# Path filtering happens at the JOB level (see the `changes` job), not here.
|
|
# Workflow-level paths-ignore never creates the check runs at all, which
|
|
# deadlocks required status checks on docs-only PRs (contexts stay "expected"
|
|
# forever). Job-level skips complete with conclusion "skipped", which
|
|
# satisfies branch protection.
|
|
on:
|
|
workflow_dispatch:
|
|
push:
|
|
branches: [main]
|
|
pull_request:
|
|
branches: [main]
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
# Testcontainers' ryuk reaper is redundant here (tests/global-setup.ts explicitly
|
|
# stops its containers and CI runners are ephemeral), and its image pull from Docker
|
|
# Hub is a recurring flake source (HTTP 500 on registry-1.docker.io). Disable it so a
|
|
# registry hiccup pulling ryuk cannot fail the suite.
|
|
env:
|
|
TESTCONTAINERS_RYUK_DISABLED: "true"
|
|
|
|
jobs:
|
|
# Detects whether anything outside the docs/landing/branding surfaces
|
|
# changed. Downstream jobs skip when code is untouched; a skipped job still
|
|
# reports a conclusion, so required status checks stay mergeable.
|
|
# Fails open: any uncertainty (unknown base, force push, dispatch) runs
|
|
# the full pipeline.
|
|
changes:
|
|
name: Changes
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
code: ${{ steps.filter.outputs.code }}
|
|
landing: ${{ steps.filter.outputs.landing }}
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
- id: filter
|
|
name: Diff against the base commit
|
|
env:
|
|
EVENT_NAME: ${{ github.event_name }}
|
|
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
|
PUSH_BEFORE_SHA: ${{ github.event.before }}
|
|
run: |
|
|
if [ "$EVENT_NAME" = "pull_request" ]; then
|
|
base="$PR_BASE_SHA"
|
|
else
|
|
base="$PUSH_BEFORE_SHA"
|
|
fi
|
|
# `code`: anything outside the docs/landing surfaces changed (gates the
|
|
# main pipeline). `landing`: the landing's own surface changed (gates the
|
|
# landing e2e job). Both fail open to true when the base is unknown.
|
|
code=true
|
|
landing=true
|
|
if [ -n "$base" ] && [ "$base" != "0000000000000000000000000000000000000000" ]; then
|
|
if git fetch --no-tags --depth=1 origin "$base" && git cat-file -e "$base" 2>/dev/null; then
|
|
changed=$(git diff --name-only "$base" HEAD)
|
|
echo "Changed files:"
|
|
printf '%s\n' "$changed"
|
|
code=false
|
|
for f in $changed; do
|
|
case "$f" in
|
|
README.md|CONTRIBUTING.md|branding/*|apps/docs/*|apps/landing/*|docs/*) ;;
|
|
*) code=true; break ;;
|
|
esac
|
|
done
|
|
landing=false
|
|
for f in $changed; do
|
|
case "$f" in
|
|
apps/landing/*|tests/e2e-landing/*|playwright.landing.config.ts|packages/shared/*) landing=true; break ;;
|
|
esac
|
|
done
|
|
else
|
|
echo "Base commit unavailable; running the full pipeline"
|
|
fi
|
|
else
|
|
echo "No usable base (dispatch, force push, or new branch); running the full pipeline"
|
|
fi
|
|
echo "code=$code" >> "$GITHUB_OUTPUT"
|
|
echo "landing=$landing" >> "$GITHUB_OUTPUT"
|
|
|
|
lint:
|
|
name: Lint
|
|
runs-on: ubuntu-latest
|
|
needs: changes
|
|
if: needs.changes.outputs.code == 'true'
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
- uses: ./.github/actions/setup
|
|
- run: pnpm lint
|
|
- name: License boundary check (D15)
|
|
run: pnpm check:license-boundary
|
|
|
|
typecheck:
|
|
name: Typecheck
|
|
runs-on: ubuntu-latest
|
|
needs: changes
|
|
if: needs.changes.outputs.code == 'true'
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
- uses: ./.github/actions/setup
|
|
- run: pnpm typecheck
|
|
|
|
test-unit:
|
|
name: Unit Tests
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 8
|
|
needs: changes
|
|
if: needs.changes.outputs.code == 'true'
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
|
|
- name: Install system dependencies (HEIC + ExifTool + ImageMagick + exotic format tools)
|
|
run: |
|
|
sudo apt-get update -qq
|
|
sudo apt-get install -y --no-install-recommends libheif-examples libheif-plugin-x265 libheif-plugin-libde265 libimage-exiftool-perl libraw-bin imagemagick ghostscript libjxl-tools libopenjp2-tools
|
|
if apt-cache show libmagickcore-6.q16-7-extra >/dev/null 2>&1; then
|
|
sudo apt-get install -y --no-install-recommends libmagickcore-6.q16-7-extra
|
|
elif apt-cache show libmagickcore-6.q16-6-extra >/dev/null 2>&1; then
|
|
sudo apt-get install -y --no-install-recommends libmagickcore-6.q16-6-extra
|
|
else
|
|
echo "No supported ImageMagick EXR coder package found" >&2
|
|
exit 1
|
|
fi
|
|
convert -list format | grep -Eq '^[[:space:]]*EXR([*[:space:]]|$)'
|
|
|
|
- uses: ./.github/actions/setup
|
|
- name: Pre-pull testcontainer images (retry transient Docker Hub errors)
|
|
run: |
|
|
for img in postgres:17-alpine redis:8-alpine; do
|
|
ok=
|
|
for attempt in 1 2 3 4 5; do
|
|
if docker pull "$img"; then ok=1; break; fi
|
|
echo "::warning::docker pull $img failed (attempt $attempt/5); retrying in $((attempt * 5))s"
|
|
sleep "$((attempt * 5))"
|
|
done
|
|
[ -n "$ok" ] || { echo "::error::could not pull $img after 5 attempts"; exit 1; }
|
|
done
|
|
- run: pnpm vitest run tests/unit/ --reporter=verbose
|
|
|
|
test-integration:
|
|
name: Integration (${{ matrix.shard }}/4)
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
needs: changes
|
|
if: needs.changes.outputs.code == 'true'
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
shard: [1, 2, 3, 4]
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
|
|
- name: Install system dependencies (image formats + docs + Fast OCR)
|
|
run: |
|
|
sudo apt-get update -qq
|
|
sudo apt-get install -y --no-install-recommends \
|
|
libheif-examples libheif-plugin-x265 libheif-plugin-libde265 \
|
|
libimage-exiftool-perl libraw-bin imagemagick ghostscript \
|
|
libjxl-tools libopenjp2-tools \
|
|
qpdf libreoffice-calc libreoffice-impress libreoffice-writer \
|
|
tesseract-ocr tesseract-ocr-eng tesseract-ocr-deu \
|
|
tesseract-ocr-fra tesseract-ocr-spa \
|
|
tesseract-ocr-chi-sim tesseract-ocr-jpn
|
|
|
|
if apt-cache show libmagickcore-6.q16-7-extra >/dev/null 2>&1; then
|
|
sudo apt-get install -y --no-install-recommends libmagickcore-6.q16-7-extra
|
|
elif apt-cache show libmagickcore-6.q16-6-extra >/dev/null 2>&1; then
|
|
sudo apt-get install -y --no-install-recommends libmagickcore-6.q16-6-extra
|
|
else
|
|
echo "No supported ImageMagick EXR coder package found" >&2
|
|
exit 1
|
|
fi
|
|
convert -list format | grep -Eq '^[[:space:]]*EXR([*[:space:]]|$)'
|
|
|
|
sudo rm -f /usr/share/tesseract-ocr/5/tessdata/osd.traineddata
|
|
test ! -e /usr/share/tesseract-ocr/5/tessdata/osd.traineddata
|
|
|
|
expected=(chi_sim deu eng fra jpn spa)
|
|
mapfile -t actual < <(tesseract --list-langs 2>/dev/null | tail -n +2 | LC_ALL=C sort)
|
|
[[ "${actual[*]}" == "${expected[*]}" ]] || {
|
|
echo "::error::Fast OCR language payload changed: ${actual[*]}"
|
|
exit 1
|
|
}
|
|
|
|
- name: Install Pandoc 3.10 (sandboxed DOCX/EPUB)
|
|
run: |
|
|
curl -fsSL https://github.com/jgm/pandoc/releases/download/3.10/pandoc-3.10-1-amd64.deb -o /tmp/pandoc.deb
|
|
sudo apt-get install -y --no-install-recommends /tmp/pandoc.deb
|
|
pandoc --version
|
|
|
|
- name: Install pdfcpu (doc-engine PDF layout binary; matches docker/Dockerfile v0.13.0)
|
|
run: |
|
|
curl -fsSL https://github.com/pdfcpu/pdfcpu/releases/download/v0.13.0/pdfcpu_0.13.0_Linux_x86_64.tar.xz -o /tmp/pdfcpu.tar.xz
|
|
tar -xJf /tmp/pdfcpu.tar.xz -C /tmp
|
|
sudo install "$(find /tmp -type f -name pdfcpu | head -1)" /usr/local/bin/pdfcpu
|
|
pdfcpu version
|
|
|
|
- name: Allow ImageMagick to read EPS/PS via Ghostscript delegate
|
|
run: |
|
|
POLICY_FILE=$(find /etc/ImageMagick* -name policy.xml 2>/dev/null | head -1)
|
|
if [ -n "$POLICY_FILE" ]; then
|
|
sudo sed -i 's/<policy domain="coder" rights="none" pattern="EPS"/<policy domain="coder" rights="read" pattern="EPS"/' "$POLICY_FILE"
|
|
sudo sed -i 's/<policy domain="coder" rights="none" pattern="PS"/<policy domain="coder" rights="read" pattern="PS"/' "$POLICY_FILE"
|
|
fi
|
|
|
|
- uses: ./.github/actions/setup
|
|
- name: Pre-pull testcontainer images (retry transient Docker Hub errors)
|
|
run: |
|
|
for img in postgres:17-alpine redis:8-alpine; do
|
|
ok=
|
|
for attempt in 1 2 3 4 5; do
|
|
if docker pull "$img"; then ok=1; break; fi
|
|
echo "::warning::docker pull $img failed (attempt $attempt/5); retrying in $((attempt * 5))s"
|
|
sleep "$((attempt * 5))"
|
|
done
|
|
[ -n "$ok" ] || { echo "::error::could not pull $img after 5 attempts"; exit 1; }
|
|
done
|
|
- run: pnpm vitest run tests/integration/ --reporter=verbose --shard=${{ matrix.shard }}/4
|
|
|
|
test-e2e-smoke:
|
|
name: E2E Smoke (Chromium)
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
needs: changes
|
|
if: needs.changes.outputs.code == 'true'
|
|
services:
|
|
postgres:
|
|
image: postgres:17-alpine
|
|
env:
|
|
POSTGRES_USER: snapotter
|
|
POSTGRES_PASSWORD: snapotter
|
|
POSTGRES_DB: snapotter
|
|
ports:
|
|
- 5432:5432
|
|
options: >-
|
|
--health-cmd "pg_isready -U snapotter"
|
|
--health-interval 5s
|
|
--health-timeout 3s
|
|
--health-retries 10
|
|
redis:
|
|
image: redis:8-alpine
|
|
ports:
|
|
- 6379:6379
|
|
options: >-
|
|
--health-cmd "redis-cli ping"
|
|
--health-interval 5s
|
|
--health-timeout 3s
|
|
--health-retries 10
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
- uses: ./.github/actions/setup
|
|
- name: Get Playwright version
|
|
id: pw-version
|
|
run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
|
|
- name: Cache Playwright browsers
|
|
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
|
with:
|
|
path: ~/.cache/ms-playwright
|
|
key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
|
|
- name: Install Playwright Chromium
|
|
run: pnpm playwright install --with-deps chromium
|
|
- name: Run smoke specs
|
|
# tools-all.spec.ts is the full 240-tool per-tool render matrix (~250 tests)
|
|
# -- a comprehensive check, not a PR-gate smoke test. It runs in the nightly
|
|
# e2e-full bucket, which executes the whole suite sharded. Keep this gate to
|
|
# genuinely fast sanity specs so it finishes well inside the timeout.
|
|
run: pnpm playwright test tests/e2e/smoke.spec.ts tests/e2e/navigation.spec.ts tests/e2e/home-page.spec.ts --project=chromium
|
|
env:
|
|
PW_WORKERS: "2"
|
|
- name: Upload report on failure
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
if: failure()
|
|
with:
|
|
name: e2e-smoke-report
|
|
path: playwright-report/
|
|
retention-days: 7
|
|
|
|
test-e2e-mobile-smoke:
|
|
name: E2E Mobile Smoke (Chromium)
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
needs: changes
|
|
if: needs.changes.outputs.code == 'true'
|
|
services:
|
|
postgres:
|
|
image: postgres:17-alpine
|
|
env:
|
|
POSTGRES_USER: snapotter
|
|
POSTGRES_PASSWORD: snapotter
|
|
POSTGRES_DB: snapotter
|
|
ports:
|
|
- 5432:5432
|
|
options: >-
|
|
--health-cmd "pg_isready -U snapotter"
|
|
--health-interval 5s
|
|
--health-timeout 3s
|
|
--health-retries 10
|
|
redis:
|
|
image: redis:8-alpine
|
|
ports:
|
|
- 6379:6379
|
|
options: >-
|
|
--health-cmd "redis-cli ping"
|
|
--health-interval 5s
|
|
--health-timeout 3s
|
|
--health-retries 10
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
- uses: ./.github/actions/setup
|
|
- name: Get Playwright version
|
|
id: pw-version
|
|
run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
|
|
- name: Cache Playwright browsers
|
|
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
|
with:
|
|
path: ~/.cache/ms-playwright
|
|
key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
|
|
- name: Install Playwright Chromium
|
|
run: pnpm playwright install --with-deps chromium
|
|
- name: Run mobile device specs (Pixel 7)
|
|
run: pnpm playwright test --project=mobile-chromium --grep @mobile --grep-invert @visual
|
|
env:
|
|
PW_WORKERS: "2"
|
|
- name: Upload report on failure
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
if: failure()
|
|
with:
|
|
name: e2e-mobile-smoke-report
|
|
path: playwright-report/
|
|
retention-days: 7
|
|
|
|
test-e2e-landing:
|
|
name: E2E Landing (Chromium)
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
needs: changes
|
|
if: needs.changes.outputs.landing == 'true'
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
- uses: ./.github/actions/setup
|
|
- name: Get Playwright version
|
|
id: pw-version
|
|
run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
|
|
- name: Cache Playwright browsers
|
|
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
|
with:
|
|
path: ~/.cache/ms-playwright
|
|
key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
|
|
- name: Install Playwright Chromium
|
|
run: pnpm playwright install --with-deps chromium
|
|
- name: Run landing e2e suite
|
|
# The landing is a static Astro site with no database, so this needs no
|
|
# Postgres/Redis services. The suite's webServer starts `astro dev` on 4350.
|
|
run: pnpm test:e2e:landing
|
|
env:
|
|
# Authenticated GitHub API so the dev server's build-time star-count fetch
|
|
# isn't rate-limited on shared runner IPs. It falls back to a constant on
|
|
# failure either way, so this only keeps the run quiet, not correct.
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
- name: Upload report on failure
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
if: failure()
|
|
with:
|
|
name: e2e-landing-report
|
|
path: playwright-report/
|
|
retention-days: 7
|
|
|
|
pip-audit:
|
|
name: Python Dependency Audit
|
|
runs-on: ubuntu-latest
|
|
needs: changes
|
|
if: needs.changes.outputs.code == 'true'
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
|
|
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
|
with:
|
|
python-version: "3.11"
|
|
|
|
- run: pip install "pip-audit==2.10.0"
|
|
- name: Run pip-audit (ignoring CVEs blocked by dependency constraints)
|
|
# CVE-2025-3000: torch 2.12.0, no fixed release available as of 2026-06-11
|
|
run: >-
|
|
pip-audit -r packages/ai/python/requirements.txt
|
|
--ignore-vuln CVE-2024-27763
|
|
--ignore-vuln CVE-2025-3000
|
|
--ignore-vuln CVE-2026-40086
|
|
--ignore-vuln CVE-2026-25990
|
|
--ignore-vuln CVE-2026-40192
|
|
--ignore-vuln GHSA-55v6-g8pm-pw4c
|
|
--ignore-vuln CVE-2026-42308
|
|
--ignore-vuln CVE-2026-42310
|
|
--ignore-vuln CVE-2026-42311
|
|
--ignore-vuln PYSEC-2025-189
|
|
--ignore-vuln PYSEC-2025-190
|
|
--ignore-vuln PYSEC-2025-191
|
|
--ignore-vuln PYSEC-2025-192
|
|
--ignore-vuln PYSEC-2025-193
|
|
--ignore-vuln PYSEC-2025-194
|
|
--ignore-vuln PYSEC-2025-195
|
|
--ignore-vuln PYSEC-2025-196
|
|
--ignore-vuln PYSEC-2025-197
|
|
--ignore-vuln PYSEC-2025-210
|
|
--ignore-vuln PYSEC-2026-139
|
|
--ignore-vuln PYSEC-2024-277
|
|
--ignore-vuln PYSEC-2026-89
|
|
|
|
build:
|
|
name: Build
|
|
runs-on: ubuntu-latest
|
|
# Only `changes` is a real dependency (its output gates the `if` below).
|
|
# Gating on the test jobs used to save a runner minute on red PRs, but it
|
|
# put a 55s job behind the slowest integration shard and added that time to
|
|
# every green run. Build and the test jobs are independent required checks,
|
|
# so a green Build still can't merge past a red shard.
|
|
needs: [changes]
|
|
if: needs.changes.outputs.code == 'true'
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
- uses: ./.github/actions/setup
|
|
- run: pnpm build
|
|
|
|
# docker:
|
|
# name: Docker Build Test
|
|
# runs-on: ubuntu-latest
|
|
# steps:
|
|
# - uses: actions/checkout@v4
|
|
# - uses: docker/setup-buildx-action@v3
|
|
# - name: Log in to GHCR (for registry cache)
|
|
# uses: docker/login-action@v3
|
|
# with:
|
|
# registry: ghcr.io
|
|
# username: ${{ github.repository_owner }}
|
|
# password: ${{ secrets.GHCR_TOKEN }}
|
|
# - uses: docker/build-push-action@v6
|
|
# with:
|
|
# context: .
|
|
# file: docker/Dockerfile
|
|
# push: false
|
|
# tags: snapotter:ci
|
|
# build-args: SKIP_MODEL_DOWNLOADS=true
|
|
# cache-from: type=registry,ref=ghcr.io/snapotter-hq/snapotter:cache-linux-amd64
|
|
# cache-to: type=registry,ref=ghcr.io/snapotter-hq/snapotter:cache-ci,mode=max
|