Publish symbol-bearing debug relay images (#3250)

## Why
Native profilers such as `ddprof` need symbols to resolve optimized Buzz
relay stacks, while ordinary deployments should keep the current compact
stripped image.

## What
- Build optimized relay binaries with line-table debug information and
derive the stripped release binaries from the same ELF files
- Publish matching multi-arch `debug-*` tags while preserving existing
stripped tags and runtime behavior
- Document the debug image as an optimized symbol-bearing release, not a
debug-mode build

## Risk Assessment
Medium — this changes the relay image release workflow and adds a second
image variant, but existing tags remain stripped and use the same
runtime base, user, entrypoint, and optimized machine code.

## References
- Follows Envoy's optimized unstripped `debug-*` image pattern:
https://github.com/envoyproxy/envoy/blob/main/distribution/binary/BUILD
- Built both Docker targets locally; verified matching GNU build IDs and
`.text` hashes, with DWARF and symbol sections present only in the debug
variant
- Pre-push checks passed: branch skew, org policy, desktop checks/tests,
Rust tests, mobile tests, Tauri tests, and workspace tests

Generated with Amp

Signed-off-by: David Grochowski <dgrochowski@squareup.com>
Co-authored-by: Amp <amp@ampcode.com>

.github/workflows/docker.yml

@@ -1,6 +1,8 @@
 name: Docker image
 
-# Builds and publishes the public Buzz relay image as ghcr.io/block/buzz.
+# Builds and publishes the public Buzz relay images as ghcr.io/block/buzz.
+# Normal tags contain stripped binaries; matching debug-* tags contain the same
+# optimized binaries with line-table debug information for native profilers.
 #
 # Strategy: each architecture builds on its native runner (ubuntu-24.04 for
 # amd64, ubuntu-24.04-arm for arm64), pushes to GHCR by digest, then a final
@@ -15,8 +17,10 @@ name: Docker image
 #
 # Triggers:
 #   - push to main           → :main + :sha-<7>
+#                              + :debug-main + :debug-sha-<7>
 #   - push tags relay-v*.*.* → :{version} + :{major}.{minor} + :{major}
-#                              (+ :latest for stable, NOT for prereleases)
+#                              + matching :debug-* tags
+#                              (+ :latest/:debug-latest for stable releases)
 #   - pull_request           → build only (no push), cache stays warm
 #   - workflow_dispatch      → manual relay-tag rescue at the tag itself
 #
@@ -95,10 +99,6 @@ jobs:
             runner: ubuntu-24.04-arm
             arch: arm64
 
-    outputs:
-      # Used downstream by `merge` to stitch the manifest.
-      version: ${{ steps.meta.outputs.version }}
-
     steps:
       - name: Checkout
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6
@@ -163,12 +163,13 @@ jobs:
             org.opencontainers.image.description=WebSocket relay server for the Buzz communications platform
             org.opencontainers.image.licenses=Apache-2.0
 
-      - name: Build and push by digest
+      - name: Build and push release image by digest
-        id: build
+        id: build-release
         uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf  # v7.2.0
         with:
           context: .
           file: ./Dockerfile
+          target: runtime
           platforms: ${{ matrix.platform }}
           labels: ${{ steps.meta.outputs.labels }}
           # Push by digest, not by tag — the merge job assembles the tags
@@ -180,25 +181,49 @@ jobs:
           cache-to: |
             ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && format('type=registry,ref={0}-buildcache:{1},mode=max,compression=zstd', env.IMAGE_NAME, matrix.arch) || '' }}
 
-      - name: Export digest
+      - name: Build and push debug image by digest
+        id: build-debug
+        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf  # v7.2.0
+        with:
+          context: .
+          file: ./Dockerfile
+          target: runtime-debug
+          platforms: ${{ matrix.platform }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }}
+          cache-from: |
+            type=registry,ref=${{ env.IMAGE_NAME }}-buildcache:${{ matrix.arch }}
+
+      - name: Export release and debug digests
         if: github.event_name != 'pull_request'
         env:
-          DIGEST: ${{ steps.build.outputs.digest }}
+          RELEASE_DIGEST: ${{ steps.build-release.outputs.digest }}
+          DEBUG_DIGEST: ${{ steps.build-debug.outputs.digest }}
         run: |
-          mkdir -p /tmp/digests
+          mkdir -p /tmp/digests-release /tmp/digests-debug
-          touch "/tmp/digests/${DIGEST#sha256:}"
+          touch "/tmp/digests-release/${RELEASE_DIGEST#sha256:}"
+          touch "/tmp/digests-debug/${DEBUG_DIGEST#sha256:}"
 
-      - name: Upload digest
+      - name: Upload release digest
         if: github.event_name != 'pull_request'
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
         with:
-          name: digests-${{ matrix.arch }}
+          name: digests-release-${{ matrix.arch }}
-          path: /tmp/digests/*
+          path: /tmp/digests-release/*
+          if-no-files-found: error
+          retention-days: 1
+
+      - name: Upload debug digest
+        if: github.event_name != 'pull_request'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: digests-debug-${{ matrix.arch }}
+          path: /tmp/digests-debug/*
           if-no-files-found: error
           retention-days: 1
 
   merge:
-    name: Merge multi-arch manifest
+    name: Merge ${{ matrix.variant }} multi-arch manifest
     if: github.event_name != 'pull_request'
     runs-on: ubuntu-24.04
     needs: build
@@ -208,13 +233,21 @@ jobs:
       packages: write    # push the merged manifest
       id-token: write    # OIDC for provenance attestation on the manifest
       attestations: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - variant: release
+            tag_prefix: ""
+          - variant: debug
+            tag_prefix: debug-
 
     steps:
       - name: Download all per-arch digests
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
         with:
           path: /tmp/digests
-          pattern: digests-*
+          pattern: digests-${{ matrix.variant }}-*
           merge-multiple: true
 
       - name: Set up Docker Buildx
@@ -237,9 +270,12 @@ jobs:
           # the build job's `meta` step for why match=^relay-v(.*)$, why
           # value=${{ inputs.version }} carries the rescue-dispatch version,
           # and why :latest is left to flavor.latest=auto.
+          flavor: |
+            latest=auto
+            prefix=${{ matrix.tag_prefix }},onlatest=true
           tags: |
             type=ref,event=branch,enable=${{ github.event_name != 'workflow_dispatch' || inputs.version == '' }}
-            type=sha,prefix=sha-,format=short,enable=${{ github.event_name != 'workflow_dispatch' || inputs.version == '' }}
+            type=sha,prefix=${{ matrix.tag_prefix }}sha-,format=short,enable=${{ github.event_name != 'workflow_dispatch' || inputs.version == '' }}
             type=semver,pattern={{version}},match=^relay-v(.*)$,value=${{ inputs.version }}
             type=semver,pattern={{major}}.{{minor}},match=^relay-v(.*)$,value=${{ inputs.version }}
             type=semver,pattern={{major}},match=^relay-v(.*)$,value=${{ inputs.version }}
@@ -284,11 +320,12 @@ jobs:
       - name: Summary
         env:
           IMAGE_NAME: ${{ env.IMAGE_NAME }}
+          VARIANT: ${{ matrix.variant }}
           MERGED_DIGEST: ${{ steps.manifest.outputs.digest }}
           META_TAGS: ${{ steps.meta.outputs.tags }}
         run: |
           {
-            echo "### Published \`${IMAGE_NAME}\`"
+            echo "### Published \`${IMAGE_NAME}\` (${VARIANT})"
             echo
             echo "**Digest:** \`${MERGED_DIGEST}\`"
             echo

Dockerfile

@@ -59,6 +59,9 @@ RUN apt-get update \
         ca-certificates \
         git \
     && rm -rf /var/lib/apt/lists/*
+# Keep enough DWARF for native profilers to resolve optimized code to source
+# locations. The normal runtime strips it below; runtime-debug retains it.
+ENV CARGO_PROFILE_RELEASE_DEBUG=line-tables-only
 COPY --from=planner /build/recipe.json recipe.json
 # Cook the full workspace recipe — relay deps include workspace siblings, so
 # scoping to -p buzz-relay misses transitive deps and re-builds them later.
@@ -66,8 +69,12 @@ RUN cargo chef cook --release --recipe-path recipe.json
 COPY . .
 RUN cargo build --release --locked -p buzz-relay --bin buzz-relay \
                                    -p buzz-admin --bin buzz-admin \
-                                   -p buzz-pair-relay --bin buzz-pair-relay \
+                                   -p buzz-pair-relay --bin buzz-pair-relay
-    && strip target/release/buzz-relay \
+
+# Derive the normal release binaries from the same optimized ELF files as the
+# debug image so the two variants cannot drift at code-generation time.
+FROM builder AS stripped-binaries
+RUN strip target/release/buzz-relay \
     && strip target/release/buzz-admin \
     && strip target/release/buzz-pair-relay
 
@@ -111,8 +118,8 @@ COPY web/ web/
 COPY admin-web/ admin-web/
 RUN pnpm -C web build && pnpm -C admin-web build
 
-# ─── Stage 5: runtime ───────────────────────────────────────────────────────
+# ─── Stage 5: shared runtime ────────────────────────────────────────────────
-FROM debian:${DEBIAN_VERSION}-slim AS runtime
+FROM debian:${DEBIAN_VERSION}-slim AS runtime-base
 
 # OCI annotations: required for GHCR to auto-link the image to this repo and
 # inherit its visibility. org.opencontainers.image.source is the load-bearing
@@ -135,9 +142,6 @@ RUN apt-get update \
     && useradd  --system --uid 1000 --gid 1000 --home-dir /var/lib/buzz \
                 --create-home --shell /usr/sbin/nologin buzz
 
-COPY --from=builder    /build/target/release/buzz-relay /usr/local/bin/buzz-relay
-COPY --from=builder    /build/target/release/buzz-admin /usr/local/bin/buzz-admin
-COPY --from=builder    /build/target/release/buzz-pair-relay /usr/local/bin/buzz-pair-relay
 COPY --from=web-builder /build/web/dist                 /srv/buzz/web
 COPY --from=web-builder /build/admin-web/dist           /srv/buzz/admin-web
 
@@ -157,3 +161,18 @@ USER buzz:buzz
 WORKDIR /var/lib/buzz
 
 ENTRYPOINT ["/usr/local/bin/buzz-relay"]
+
+# Optimized binaries with line-table debug information for native profiling.
+# Published under debug-* tags; runtime behavior otherwise matches the normal
+# image exactly.
+FROM runtime-base AS runtime-debug
+COPY --from=builder /build/target/release/buzz-relay /usr/local/bin/buzz-relay
+COPY --from=builder /build/target/release/buzz-admin /usr/local/bin/buzz-admin
+COPY --from=builder /build/target/release/buzz-pair-relay /usr/local/bin/buzz-pair-relay
+
+# Keep the stripped runtime as the final/default Dockerfile target so existing
+# `docker build .` callers and release tags retain their current behavior.
+FROM runtime-base AS runtime
+COPY --from=stripped-binaries /build/target/release/buzz-relay /usr/local/bin/buzz-relay
+COPY --from=stripped-binaries /build/target/release/buzz-admin /usr/local/bin/buzz-admin
+COPY --from=stripped-binaries /build/target/release/buzz-pair-relay /usr/local/bin/buzz-pair-relay

RELEASING.md

@@ -57,10 +57,13 @@ or mobile GitHub Release.
 2. **Merge the PR.** `auto-tag-on-release-pr-merge` pushes
    `relay-v<version>`.
 3. **The tag triggers `docker.yml`.** Stable releases update the version
-   aliases and `latest`; prereleases do not.
+   aliases and `latest`; prereleases do not. Each release also publishes an
+   optimized, symbol-bearing image under matching `debug-` tags (for example,
+   `debug-0.3.0` and `debug-latest`) for native profiling. The ordinary tags
+   remain stripped and are the default for deployments that do not need it.
 
 Every push to `main` continues to publish the rolling relay `:main` and
-`:sha-<7>` tags.
+`:sha-<7>` tags, plus matching `:debug-main` and `:debug-sha-<7>` variants.
 
 ### Mobile