Merge sami/mesh-testbed: local k8s 3-replica testbed harness

Co-authored-by: Tyler Longwell <tlongwell@block.xyz>
Signed-off-by: Tyler Longwell <tlongwell@block.xyz>

* origin/sami/mesh-testbed:
  feat(deploy): local docker-desktop k8s mesh testbed (3-replica HA)
This commit is contained in:
npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d
2026-07-08 11:09:05 -04:00
co-authored by Tyler Longwell
8 changed files with 302 additions and 10 deletions
+46
View File
@@ -15,8 +15,29 @@ ARG RUST_VERSION=1.95
ARG NODE_VERSION=24
ARG DEBIAN_VERSION=bookworm
# Optional extra CA bundle for builds behind a TLS-intercepting corporate proxy
# (e.g. a Cloudflare/Zscaler gateway that re-signs TLS). Empty by default, so
# public CI builds are unaffected. Point it at a PEM file in the build context:
# docker build --build-arg EXTRA_CA_CERTS=path/to/proxy-ca.pem ...
# Consumed by the network-touching stages below (cargo + pnpm).
ARG EXTRA_CA_CERTS=
# Optional npm registry for builds where the public registry is unreachable or
# policy-blocked (e.g. a corporate mirror / Artifactory). Empty default = public
# npmjs, so public CI builds are unaffected. Consumed by the web-builder stage.
ARG NPM_REGISTRY=
# ─── Stage 1: cargo-chef base ───────────────────────────────────────────────
FROM rust:${RUST_VERSION}-${DEBIAN_VERSION} AS chef
# Trust an optional corporate-proxy CA before any network fetch (no-op if unset).
ARG EXTRA_CA_CERTS
COPY --chmod=0644 ${EXTRA_CA_CERTS:-Dockerfile} /tmp/extra-ca/src
RUN if [ -n "${EXTRA_CA_CERTS}" ]; then \
cp /tmp/extra-ca/src /usr/local/share/ca-certificates/extra-proxy-ca.crt \
&& update-ca-certificates \
&& echo "CARGO_HTTP_CAINFO=/etc/ssl/certs/ca-certificates.crt" >> /etc/environment; \
fi
ENV CARGO_HTTP_CAINFO=/etc/ssl/certs/ca-certificates.crt
RUN cargo install cargo-chef --locked --version 0.1.71
WORKDIR /build
@@ -53,6 +74,31 @@ RUN cargo build --release --locked -p buzz-relay --bin buzz-relay \
# vice versa.
FROM node:${NODE_VERSION}-${DEBIAN_VERSION}-slim AS web-builder
WORKDIR /build
# Trust an optional corporate-proxy CA so corepack + pnpm can fetch over an
# intercepting TLS gateway (no-op if EXTRA_CA_CERTS is unset).
ARG EXTRA_CA_CERTS
COPY --chmod=0644 ${EXTRA_CA_CERTS:-Dockerfile} /tmp/extra-ca/src
RUN if [ -n "${EXTRA_CA_CERTS}" ]; then \
apt-get update && apt-get install -y --no-install-recommends ca-certificates \
&& cp /tmp/extra-ca/src /usr/local/share/ca-certificates/extra-proxy-ca.crt \
&& update-ca-certificates \
&& rm -rf /var/lib/apt/lists/*; \
fi
ENV NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt
# Point npm + corepack at an optional mirror (no-op when NPM_REGISTRY is unset).
# corepack reads COREPACK_NPM_REGISTRY to fetch the pinned pnpm; pnpm/npm read
# the .npmrc registry for dependency installs.
ARG NPM_REGISTRY
ENV COREPACK_NPM_REGISTRY=${NPM_REGISTRY}
# When using a mirror, disable corepack's npmjs signature check: the mirror
# republishes tarballs without the public registry's provenance signatures, so
# strict verification fails ("No compatible signature found"). Only relaxed on
# the mirror path — public builds (NPM_REGISTRY unset) keep strict verification.
RUN if [ -n "${NPM_REGISTRY}" ]; then \
echo "registry=${NPM_REGISTRY}" > /build/.npmrc \
&& echo "COREPACK_INTEGRITY_KEYS=0" >> /etc/environment; \
fi
ENV COREPACK_INTEGRITY_KEYS=${NPM_REGISTRY:+0}
RUN corepack enable
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
COPY patches/ patches/
+1 -1
View File
@@ -30,7 +30,7 @@
⚠ QUICKSTART / EVALUATION PROFILE
{{ if .Values.postgresql.enabled }}- In-cluster Postgres subchart (CloudPirates){{ end }}
{{ if .Values.redis.enabled }}- In-cluster Redis subchart (CloudPirates){{ end }}
{{ if .Values.minio.enabled }}- In-cluster MinIO (eval-only, single replica; bucket "{{ .Values.s3.bucket }}" created by post-install Job){{ end }}
{{ if .Values.minio.enabled }}- In-cluster MinIO (eval-only, single replica; bucket "{{ .Values.s3.bucket }}" created by a bundled init Job){{ end }}
- Chart auto-generates secrets via the `lookup` pattern. This is NOT
GitOps-safe — secrets will silently rotate under ArgoCD/Flux. For
production, see examples/argocd-app.yaml or examples/flux-helmrelease.yaml.
@@ -53,6 +53,41 @@ spec:
topologySpreadConstraints:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- if .Values.minio.enabled }}
# Quickstart only: the bundled MinIO bucket is created by a concurrent
# init Job (templates/quickstart-minio-init.yaml). The relay's A3 S3
# conformance probe is startup-fatal, so without this gate the relay Pods
# CrashLoopBackOff (with growing backoff) until the bucket appears. Block
# relay start until the bucket exists — deterministic, no crash-loops.
initContainers:
- name: wait-for-bucket
image: {{ .Values.minio.mcImage | quote }}
securityContext:
{{- toYaml .Values.relay.containerSecurityContext | nindent 12 }}
env:
- name: S3_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ include "buzz.chartSecretName" . }}
key: BUZZ_S3_ACCESS_KEY
- name: S3_SECRET_KEY
valueFrom:
secretKeyRef:
name: {{ include "buzz.chartSecretName" . }}
key: BUZZ_S3_SECRET_KEY
command: ["/bin/sh", "-c"]
args:
- |
set -e
export MC_CONFIG_DIR=/tmp/.mc
until mc alias set local {{ include "buzz.minioEndpoint" . }} "$S3_ACCESS_KEY" "$S3_SECRET_KEY" >/dev/null 2>&1; do
echo "waiting for MinIO..."; sleep 2
done
until mc stat local/{{ .Values.s3.bucket }} >/dev/null 2>&1; do
echo "waiting for bucket {{ .Values.s3.bucket }}..."; sleep 2
done
echo "bucket {{ .Values.s3.bucket }} present"
{{- end }}
containers:
- name: relay
image: {{ include "buzz.image" . }}
@@ -1,22 +1,35 @@
{{- /*
Creates the media bucket in the bundled MinIO after install/upgrade. Mirrors
the docker-compose `minio-init` step. Hook-managed so it re-runs on upgrade
and is garbage-collected. Quickstart-only.
Creates the media bucket in the bundled MinIO. Mirrors the docker-compose
`minio-init` step (which depends_on minio and runs concurrently, not as a
lifecycle hook).
This is a NORMAL resource, deliberately NOT a helm hook. The relay's A3
object-store conformance probe is startup-fatal and does not create the bucket
itself, so the bucket must exist for a relay Pod to pass readiness. A
`post-install` hook cannot satisfy that: `helm install --wait` blocks on the
main resources (the relay Deployment) becoming Ready *before* post-install
hooks run, but the relay can never become Ready without the bucket — a
deadlock (relays CrashLoopBackOff forever). A `pre-install` hook deadlocks the
other way: it would run to completion before MinIO (a normal resource) is
created. Running the Job as a normal resource lets helm create MinIO, this Job,
and the relay Deployment together; the Job's `until` loop waits for MinIO, and
`--wait` waits for the Job to complete alongside the Deployment.
The name carries the release revision so `helm upgrade` creates a fresh Job
(a completed Job's spec is immutable); `ttlSecondsAfterFinished` GCs it.
Quickstart-only.
*/ -}}
{{- if .Values.minio.enabled -}}
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "buzz.minioFullname" . }}-init
name: {{ include "buzz.minioFullname" . }}-init-{{ .Release.Revision }}
labels:
{{- include "buzz.labels" . | nindent 4 }}
app.kubernetes.io/component: minio-init
annotations:
helm.sh/hook: post-install,post-upgrade
helm.sh/hook-weight: "0"
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
spec:
backoffLimit: 10
ttlSecondsAfterFinished: 120
template:
metadata:
labels:
@@ -11,6 +11,7 @@ tests:
- it: renders the in-cluster MinIO Deployment when minio.enabled
release:
name: rel
revision: 1
set:
relayUrl: wss://buzz.example.com
ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000"
@@ -24,10 +25,16 @@ tests:
name: rel-buzz-minio
template: templates/quickstart-minio.yaml
documentIndex: 0
# The bucket-init Job is a NORMAL resource (revision-suffixed name), not a
# helm hook: a post-install hook deadlocks against `--wait` because the
# relay's startup-fatal S3 probe can't pass until the bucket exists.
- containsDocument:
kind: Job
apiVersion: batch/v1
name: rel-buzz-minio-init
name: rel-buzz-minio-init-1
template: templates/quickstart-minio-init.yaml
- notExists:
path: metadata.annotations["helm.sh/hook"]
template: templates/quickstart-minio-init.yaml
- it: relay S3 endpoint resolves to the bundled MinIO Service
+3
View File
@@ -0,0 +1,3 @@
# Corporate-proxy CA is machine/environment-specific and often internal.
# Regenerate locally (see build-and-deploy.sh); never commit.
proxy-ca.pem
+144
View File
@@ -0,0 +1,144 @@
#!/usr/bin/env bash
# Local docker-desktop k8s testbed for the Buzz relay mesh.
#
# Repeatable path: build image -> helm dep build -> helm install (quickstart HA,
# 3 replicas) -> wait 3/3 Ready -> probe /_readiness on every pod. This is the
# baseline every mesh build redeploys onto (mesh lane).
#
# Prereqs: docker-desktop k8s context Ready, helm >= 3.14, kubectl, docker.
# docker-desktop shares the docker image store with k8s, so a locally-built
# tag + pullPolicy: IfNotPresent needs no registry push or `kind load`.
#
# Usage:
# deploy/local/build-and-deploy.sh # full: build + deploy
# SKIP_BUILD=1 deploy/local/build-and-deploy.sh # redeploy existing image
# IMAGE_TAG=mesh-abc1234 deploy/local/build-and-deploy.sh
set -euo pipefail
# ── config ──────────────────────────────────────────────────────────────────
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
NS="${NS:-buzz-mesh}"
RELEASE="${RELEASE:-buzz}"
IMAGE_REPO="${IMAGE_REPO:-buzz-relay}"
IMAGE_TAG="${IMAGE_TAG:-mesh-local}"
CHART="${REPO_ROOT}/deploy/charts/buzz"
VALUES="${REPO_ROOT}/deploy/local/quickstart-ha-values.yaml"
CA_PEM="${REPO_ROOT}/deploy/local/proxy-ca.pem"
EXPECT_CTX="docker-desktop"
REPLICAS=3
EVID="${EVID:-/tmp/mesh-build/deploy-evidence-$(date +%Y%m%d-%H%M%S)}"
mkdir -p "$EVID"
log() { printf '\n\033[1;36m==> %s\033[0m\n' "$*"; }
die() { printf '\033[1;31mFAIL: %s\033[0m\n' "$*" >&2; exit 1; }
# ── 0. guardrails ─────────────────────────────────────────────────────────────
CTX="$(kubectl config current-context)"
[ "$CTX" = "$EXPECT_CTX" ] || die "kube context is '$CTX', expected '$EXPECT_CTX' (refusing to touch a non-local cluster)"
log "context: $CTX"; kubectl get nodes | tee "$EVID/nodes.txt"
# ── 1. corporate-proxy CA + npm mirror (TLS-intercepting gateway) ────────────
# Two stacked blocks on Block's network: (a) the gateway re-signs TLS with
# internal CAs the build container doesn't trust; (b) public registry.npmjs.org
# is policy-blocked (Dependency Confusion mitigation), so npm/corepack must use
# the Artifactory mirror. Both no-op on a normal network (build-args stay unset).
CA_ARG=()
REG_ARG=()
# (a) Build a complete internal-CA bundle from the macOS System keychain.
if [ ! -f "$CA_PEM" ] && command -v security >/dev/null 2>&1; then
log "exporting Block internal CA bundle from System keychain"
: > "$CA_PEM"
for name in "Cloudflare Gateway CA" \
"Service To Service AWS Native CA production G0" \
"Corp Systems AWS Native CA production G0" \
"Block, Inc CA G1" \
"Square Primary Certificate Authority - G2"; do
security find-certificate -a -c "$name" -p /Library/Keychains/System.keychain >> "$CA_PEM" 2>/dev/null || true
done
fi
if [ -f "$CA_PEM" ] && grep -q 'BEGIN CERTIFICATE' "$CA_PEM"; then
CA_ARG=(--build-arg "EXTRA_CA_CERTS=deploy/local/proxy-ca.pem")
log "using proxy CA bundle ($(grep -c 'BEGIN CERTIFICATE' "$CA_PEM") certs)"
fi
# (b) Use the host's configured npm registry (Artifactory) if it isn't public.
HOST_NPM_REG="$(pnpm config get registry 2>/dev/null || echo '')"
if [ -n "$HOST_NPM_REG" ] && ! echo "$HOST_NPM_REG" | grep -q 'registry.npmjs.org'; then
REG_ARG=(--build-arg "NPM_REGISTRY=${HOST_NPM_REG}")
log "using npm registry mirror: $HOST_NPM_REG"
fi
# ── 2. build image ────────────────────────────────────────────────────────────
if [ "${SKIP_BUILD:-0}" != "1" ]; then
log "building ${IMAGE_REPO}:${IMAGE_TAG} (HEAD $(git -C "$REPO_ROOT" rev-parse --short HEAD))"
git -C "$REPO_ROOT" rev-parse HEAD > "$EVID/build-sha.txt"
docker build "${CA_ARG[@]}" "${REG_ARG[@]}" \
-t "${IMAGE_REPO}:${IMAGE_TAG}" \
-f "${REPO_ROOT}/Dockerfile" "${REPO_ROOT}" 2>&1 | tee "$EVID/build.log"
else
log "SKIP_BUILD=1 — reusing ${IMAGE_REPO}:${IMAGE_TAG}"
fi
docker image inspect "${IMAGE_REPO}:${IMAGE_TAG}" --format '{{.Id}} {{.Size}}' | tee "$EVID/image-id.txt"
# ── 3. chart deps + install ───────────────────────────────────────────────────
log "helm dependency build"
helm dependency build "$CHART" 2>&1 | tee "$EVID/helm-dep.txt"
log "helm upgrade --install $RELEASE (ns=$NS, replicas=$REPLICAS)"
# No --wait here: the relay's A3 S3 probe is startup-fatal, so relays
# CrashLoopBackOff a few times until the concurrent init Job creates the bucket.
# helm --wait races that transient and can bail early; instead we own readiness
# gating below (rollout status + per-pod probe), which tolerates the restarts.
helm upgrade --install "$RELEASE" "$CHART" \
--namespace "$NS" --create-namespace \
--values "$VALUES" \
--set image.repository="$IMAGE_REPO" \
--set image.tag="$IMAGE_TAG" \
--timeout 5m 2>&1 | tee "$EVID/helm-install.txt"
helm_rc=${PIPESTATUS[0]}
if [ "$helm_rc" != 0 ]; then
kubectl -n "$NS" get pods -o wide | tee "$EVID/pods-onfail.txt"
kubectl -n "$NS" describe pods -l app.kubernetes.io/name=buzz | tee "$EVID/describe-onfail.txt"
kubectl -n "$NS" logs -l app.kubernetes.io/name=buzz --tail=100 --all-containers | tee "$EVID/logs-onfail.txt"
die "helm install returned rc=$helm_rc"
fi
# ── 4. verify 3/3 Ready ───────────────────────────────────────────────────────
# Find the relay Deployment: everything under this release named "buzz" except
# the bundled "*-minio" Deployment. (The chart fullname collapses
# "<release>-<chart>" to "<release>" when the release name already contains the
# chart name, so the name isn't always "<release>-buzz".)
DEPLOY=""
for d in $(kubectl -n "$NS" get deploy -l "app.kubernetes.io/instance=$RELEASE" \
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
case "$d" in *-minio) continue;; esac
DEPLOY="$d"; break
done
[ -n "$DEPLOY" ] || die "could not locate the relay Deployment"
log "waiting for $REPLICAS relay pods Ready (deployment: $DEPLOY)"
kubectl -n "$NS" rollout status deployment/"$DEPLOY" --timeout=4m | tee "$EVID/rollout.txt"
kubectl -n "$NS" get pods -o wide | tee "$EVID/pods.txt"
READY=$(kubectl -n "$NS" get deploy "$DEPLOY" -o jsonpath='{.status.readyReplicas}')
[ "${READY:-0}" = "$REPLICAS" ] || die "readyReplicas=$READY, expected $REPLICAS"
log "deployment reports $READY/$REPLICAS Ready"
# ── 5. probe /_readiness on EVERY relay pod (not just the deployment aggregate)
# The bundled MinIO + init pods share app.kubernetes.io/name=buzz, so select by
# the relay Deployment's own pod-template hash to hit only relay pods.
log "probing /_readiness on each relay pod individually"
: > "$EVID/readiness.txt"
FAIL=0
RELAY_PODS=$(kubectl -n "$NS" get pods \
-l "app.kubernetes.io/name=buzz,app.kubernetes.io/instance=$RELEASE" \
-o jsonpath='{range .items[*]}{.metadata.name}{" "}{.metadata.labels.app\.kubernetes\.io/component}{"\n"}{end}' \
| awk '$2 != "minio" && $2 != "minio-init" {print $1}')
for pod in $RELAY_PODS; do
body=$(kubectl -n "$NS" exec "$pod" -- \
sh -c 'curl -sS --max-time 5 http://127.0.0.1:8080/_readiness' 2>/dev/null || echo '<curl-failed>')
echo "$pod -> $body" | tee -a "$EVID/readiness.txt"
echo "$body" | grep -q '"status":"ready"' || FAIL=1
done
[ "$FAIL" = 0 ] || die "at least one pod is not reporting ready (see $EVID/readiness.txt)"
log "ALL $REPLICAS PODS READY — baseline up. Evidence: $EVID"
echo "namespace=$NS release=$RELEASE image=${IMAGE_REPO}:${IMAGE_TAG}" | tee "$EVID/SUMMARY.txt"
+44
View File
@@ -0,0 +1,44 @@
# Local docker-desktop HA testbed: quickstart bundled services + 3 replicas.
#
# This is the baseline every mesh build redeploys onto (mesh lane, thread
# fa90bba8 / kickoff 5a760d8d). It layers the HA shape on top of the CI
# quickstart set (ci/quickstart-values.yaml): bundled Postgres/Redis/MinIO,
# chart-autogenerated secrets, membership enforcement off.
#
# Two lines carry HA:
# replicaCount: 3 — chart hard-requires Redis for buzz-pubsub
# (quickstart provides it); the git-hook HMAC
# secret autogenerates chart-side (secret-chart.yaml).
# persistence.git.enabled — false → per-pod emptyDir. The default (true) is a
# single ReadWriteOnce PVC that cannot multi-attach
# across 3 pods on one node; it would wedge 2/3.
# Object store + Postgres are the source of truth
# for git state, not local disk (values.yaml docs
# emptyDir as the correct multi-replica choice).
quickstart: true
postgresql:
enabled: true
redis:
enabled: true
minio:
enabled: true
relayUrl: wss://buzz.test.local
ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000001"
relay:
# Testbed, not business-logic validation — same rationale as CI quickstart.
requireRelayMembership: false
podDisruptionBudget:
enabled: false
# ── HA shape ──────────────────────────────────────────────────────────────
replicaCount: 3
persistence:
git:
enabled: false
# Locally-built image; docker-desktop k8s shares the docker image store, so
# IfNotPresent resolves the tag with no push / `kind load` step.
image:
repository: buzz-relay
tag: mesh-local
pullPolicy: IfNotPresent