feat(git): move repo-name registry to Postgres + relax RWM chart gate (HA relay) (#1432)

Signed-off-by: Tyler Longwell <tlongwell@block.xyz>
Co-authored-by: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@sprout-oss.stage.blox.sqprod.co>
Co-authored-by: Tyler Longwell <tlongwell@block.xyz>
This commit is contained in:
Tyler
2026-07-01 14:50:18 -04:00
committed by GitHub
co-authored by npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d Tyler Longwell
parent b32db5e0fd
commit e5aa4a2132
17 changed files with 725 additions and 154 deletions
+380
View File
@@ -0,0 +1,380 @@
//! Git repository name registry (NIP-34 kind:30617).
//!
//! The relay holds no persistent per-repo filesystem state: git reads and
//! writes hydrate an ephemeral bare repo from object storage per request, and
//! writer serialization is the object-store pointer CAS (see
//! `docs/git-on-object-storage.md`, `Inv_NoFork`). Repo-*name* uniqueness is
//! the one remaining shared-state need, and it lives here — in Postgres, not on
//! local disk — so the relay is stateless and can run multiple replicas without
//! a ReadWriteMany volume.
//!
//! Names are unique **within a community**: the primary key is
//! `(community_id, repo_id)`, matching the multi-tenant invariant that every
//! tenant-scoped key leads with `community_id`. The PK enforces uniqueness
//! atomically via `INSERT … ON CONFLICT DO NOTHING`, which replaces the old
//! filesystem `create_dir` race guard. `owner_pubkey` distinguishes an
//! idempotent re-announce (same owner) from a collision (different owner), and
//! backs the per-pubkey quota via `COUNT`.
use sqlx::{PgPool, Row as _};
use crate::error::Result;
use crate::CommunityId;
/// Outcome of a name-reservation attempt.
///
/// The caller (kind:30617 handler) uses this to decide whether to seed the
/// manifest pointer and, on seed failure, whether to release the reservation:
/// only a `Reserved` (freshly inserted) row should be rolled back — an
/// `AlreadyOwned` re-announce must leave the pre-existing reservation intact.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReserveOutcome {
/// The name was newly claimed by this owner (a fresh row was inserted).
Reserved,
/// The name was already reserved by this same owner — idempotent
/// re-announce, a no-op update. No row was inserted; the quota was not
/// re-checked (re-announcing an already-owned name never grows the count).
AlreadyOwned,
/// The name is held by a *different* owner — a collision. No row was
/// inserted.
TakenByOther,
}
/// Return the current owner pubkey of `repo_id` in `community`, or `None` if
/// the name is unreserved. Used to classify an announce (same-owner
/// re-announce vs cross-owner collision) and to gate the quota check before a
/// fresh claim.
pub async fn repo_name_owner(
pool: &PgPool,
community: CommunityId,
repo_id: &str,
) -> Result<Option<String>> {
let row = sqlx::query(
"SELECT owner_pubkey FROM git_repo_names \
WHERE community_id = $1 AND repo_id = $2",
)
.bind(community.as_uuid())
.bind(repo_id)
.fetch_optional(pool)
.await?;
row.map(|r| r.try_get("owner_pubkey"))
.transpose()
.map_err(crate::error::DbError::from)
}
/// Reserve `repo_id` for `owner_pubkey` within `community`, enforcing a
/// per-pubkey quota of `max_repos_per_pubkey`.
///
/// Semantics (mirrors the previous filesystem registry exactly):
/// - already reserved by the same owner → [`ReserveOutcome::AlreadyOwned`]
/// (idempotent, no quota check);
/// - already reserved by another owner → [`ReserveOutcome::TakenByOther`];
/// - otherwise, if the owner is under quota, atomically claim it →
/// [`ReserveOutcome::Reserved`]; if a concurrent announce wins the insert
/// race, the `ON CONFLICT` re-read resolves it to `AlreadyOwned` (same owner
/// racing itself) or `TakenByOther`.
///
/// Returns `Err` only on backend/database failure — a full quota is *not* an
/// error here; the caller enforces the limit against `Reserved` outcomes using
/// [`count_repos_for_owner`]. (Kept as a separate call so the handler owns the
/// error message and the ordering, matching the old code.)
pub async fn reserve_repo_name(
pool: &PgPool,
community: CommunityId,
repo_id: &str,
owner_pubkey: &str,
) -> Result<ReserveOutcome> {
// Atomic claim: insert only if the (community, repo) is free. RETURNING is
// non-empty exactly when *this* statement inserted the row, so it cleanly
// distinguishes "I claimed it" from "someone already holds it" without a
// separate read (TOCTOU-free, the same guarantee `create_dir` gave).
let inserted = sqlx::query(
"INSERT INTO git_repo_names (community_id, repo_id, owner_pubkey) \
VALUES ($1, $2, $3) \
ON CONFLICT (community_id, repo_id) DO NOTHING \
RETURNING owner_pubkey",
)
.bind(community.as_uuid())
.bind(repo_id)
.bind(owner_pubkey)
.fetch_optional(pool)
.await?;
if inserted.is_some() {
return Ok(ReserveOutcome::Reserved);
}
// The row already existed — read the holder to classify same-owner
// re-announce vs cross-owner collision.
let existing = sqlx::query(
"SELECT owner_pubkey FROM git_repo_names \
WHERE community_id = $1 AND repo_id = $2",
)
.bind(community.as_uuid())
.bind(repo_id)
.fetch_optional(pool)
.await?;
match existing {
Some(row) => {
let holder: String = row
.try_get("owner_pubkey")
.map_err(crate::error::DbError::from)?;
if holder == owner_pubkey {
Ok(ReserveOutcome::AlreadyOwned)
} else {
Ok(ReserveOutcome::TakenByOther)
}
}
// Extremely narrow: the conflicting row was deleted between our INSERT
// and this SELECT (e.g. a concurrent seed-failure rollback). Treat as
// taken-by-other rather than silently granting — the announcer can
// retry, and we never hand out a name we didn't atomically claim.
None => Ok(ReserveOutcome::TakenByOther),
}
}
/// Count the repos currently reserved by `owner_pubkey` in `community`.
///
/// Backs the per-pubkey quota. Called *before* [`reserve_repo_name`] for a
/// not-yet-owned name, so the handler can reject over-quota announces with its
/// own error message.
pub async fn count_repos_for_owner(
pool: &PgPool,
community: CommunityId,
owner_pubkey: &str,
) -> Result<i64> {
let row = sqlx::query(
"SELECT COUNT(*) AS n FROM git_repo_names \
WHERE community_id = $1 AND owner_pubkey = $2",
)
.bind(community.as_uuid())
.bind(owner_pubkey)
.fetch_one(pool)
.await?;
row.try_get("n").map_err(crate::error::DbError::from)
}
/// Release a reservation held by `owner_pubkey` (rollback path).
///
/// Used only when seeding the manifest pointer fails *after* a fresh
/// [`ReserveOutcome::Reserved`], so the announce is all-or-nothing. Scoped to
/// `owner_pubkey` so a rollback can never delete a name a *different* owner
/// concurrently holds. Returns the number of rows removed (0 or 1).
pub async fn release_repo_name(
pool: &PgPool,
community: CommunityId,
repo_id: &str,
owner_pubkey: &str,
) -> Result<u64> {
let result = sqlx::query(
"DELETE FROM git_repo_names \
WHERE community_id = $1 AND repo_id = $2 AND owner_pubkey = $3",
)
.bind(community.as_uuid())
.bind(repo_id)
.bind(owner_pubkey)
.execute(pool)
.await?;
Ok(result.rows_affected())
}
#[cfg(test)]
mod tests {
use super::*;
use uuid::Uuid;
const TEST_DB_URL: &str = "postgres://buzz:buzz_dev@localhost:5432/buzz";
async fn setup_pool() -> PgPool {
let database_url = std::env::var("BUZZ_TEST_DATABASE_URL")
.or_else(|_| std::env::var("DATABASE_URL"))
.unwrap_or_else(|_| TEST_DB_URL.to_owned());
PgPool::connect(&database_url)
.await
.expect("connect to test DB")
}
async fn make_test_community(pool: &PgPool) -> CommunityId {
let id = Uuid::new_v4();
let host = format!("git-repo-test-{}.example", id.simple());
sqlx::query("INSERT INTO communities (id, host) VALUES ($1, $2)")
.bind(id)
.bind(host)
.execute(pool)
.await
.expect("insert test community");
CommunityId::from_uuid(id)
}
fn pk() -> String {
format!("{:064x}", Uuid::new_v4().as_u128())
}
/// A fresh name is `Reserved`; re-announcing it as the *same* owner is
/// `AlreadyOwned` (idempotent) and never grows the owner's count; a
/// *different* owner is `TakenByOther`.
#[tokio::test]
#[ignore = "requires Postgres"]
async fn reserve_classifies_fresh_idempotent_and_collision() {
let pool = setup_pool().await;
let community = make_test_community(&pool).await;
let owner = pk();
let other = pk();
let repo = format!("repo-{}", Uuid::new_v4().simple());
assert_eq!(
reserve_repo_name(&pool, community, &repo, &owner)
.await
.expect("fresh reserve"),
ReserveOutcome::Reserved,
"first claim of a free name is Reserved"
);
assert_eq!(
reserve_repo_name(&pool, community, &repo, &owner)
.await
.expect("re-reserve same owner"),
ReserveOutcome::AlreadyOwned,
"same-owner re-announce is idempotent AlreadyOwned"
);
assert_eq!(
reserve_repo_name(&pool, community, &repo, &other)
.await
.expect("re-reserve other owner"),
ReserveOutcome::TakenByOther,
"a different owner claiming a held name is TakenByOther"
);
assert_eq!(
count_repos_for_owner(&pool, community, &owner)
.await
.expect("count owner"),
1,
"re-announce must not double-count the owner's quota"
);
assert_eq!(
count_repos_for_owner(&pool, community, &other)
.await
.expect("count other"),
0,
"a failed (TakenByOther) claim must not count toward the loser's quota"
);
}
/// `repo_name_owner` returns the holder for a reserved name and `None` for a
/// free one, so the handler can classify before claiming.
#[tokio::test]
#[ignore = "requires Postgres"]
async fn repo_name_owner_reflects_reservation() {
let pool = setup_pool().await;
let community = make_test_community(&pool).await;
let owner = pk();
let repo = format!("repo-{}", Uuid::new_v4().simple());
assert!(
repo_name_owner(&pool, community, &repo)
.await
.expect("owner of free name")
.is_none(),
"an unreserved name has no owner"
);
reserve_repo_name(&pool, community, &repo, &owner)
.await
.expect("reserve");
assert_eq!(
repo_name_owner(&pool, community, &repo)
.await
.expect("owner of reserved name"),
Some(owner),
"a reserved name resolves to its owner"
);
}
/// Release is owner-scoped: it removes the reservation only for the holder,
/// freeing the name for a subsequent claim; a release by a *non*-holder is a
/// no-op that leaves the reservation intact.
#[tokio::test]
#[ignore = "requires Postgres"]
async fn release_is_owner_scoped_and_frees_the_name() {
let pool = setup_pool().await;
let community = make_test_community(&pool).await;
let owner = pk();
let stranger = pk();
let repo = format!("repo-{}", Uuid::new_v4().simple());
reserve_repo_name(&pool, community, &repo, &owner)
.await
.expect("reserve");
// A non-holder cannot release the name.
assert_eq!(
release_repo_name(&pool, community, &repo, &stranger)
.await
.expect("stranger release"),
0,
"release by a non-holder removes nothing"
);
assert_eq!(
repo_name_owner(&pool, community, &repo)
.await
.expect("still owned"),
Some(owner.clone()),
"the reservation survives a stranger's release attempt"
);
// The holder releases it, freeing the name.
assert_eq!(
release_repo_name(&pool, community, &repo, &owner)
.await
.expect("owner release"),
1,
"the holder's release removes exactly the one row"
);
assert_eq!(
reserve_repo_name(&pool, community, &repo, &stranger)
.await
.expect("reclaim after release"),
ReserveOutcome::Reserved,
"once released, the name is free for a new owner"
);
}
/// Names are unique *within* a community, not globally: the same repo name
/// may be independently reserved by different owners in different
/// communities without collision.
#[tokio::test]
#[ignore = "requires Postgres"]
async fn names_are_scoped_per_community() {
let pool = setup_pool().await;
let community_a = make_test_community(&pool).await;
let community_b = make_test_community(&pool).await;
let owner_a = pk();
let owner_b = pk();
let repo = format!("repo-{}", Uuid::new_v4().simple());
assert_eq!(
reserve_repo_name(&pool, community_a, &repo, &owner_a)
.await
.expect("reserve in A"),
ReserveOutcome::Reserved
);
assert_eq!(
reserve_repo_name(&pool, community_b, &repo, &owner_b)
.await
.expect("reserve same name in B"),
ReserveOutcome::Reserved,
"the same name in a different community is a fresh, independent claim"
);
assert_eq!(
repo_name_owner(&pool, community_a, &repo)
.await
.expect("owner in A"),
Some(owner_a)
);
assert_eq!(
repo_name_owner(&pool, community_b, &repo)
.await
.expect("owner in B"),
Some(owner_b)
);
}
}
+46
View File
@@ -23,6 +23,8 @@ pub mod error;
pub mod event;
/// Home feed queries.
pub mod feed;
/// Git repository name registry (NIP-34 kind:30617).
pub mod git_repo;
/// Embedded database migrations.
pub mod migration;
/// Monthly table partition management.
@@ -2063,6 +2065,50 @@ impl Db {
relay_members::backfill_from_allowlist(&self.pool, community).await
}
/// Return the current owner of git repo name `repo_id` in `community`, or
/// `None` if unreserved. See [`git_repo::repo_name_owner`].
pub async fn repo_name_owner(
&self,
community: CommunityId,
repo_id: &str,
) -> Result<Option<String>> {
git_repo::repo_name_owner(&self.pool, community, repo_id).await
}
/// Reserve a git repo name for `owner_pubkey` in `community` (NIP-34).
///
/// See [`git_repo::reserve_repo_name`] for the outcome semantics. The
/// per-pubkey quota is enforced by the caller against `count_repos_for_owner`.
pub async fn reserve_repo_name(
&self,
community: CommunityId,
repo_id: &str,
owner_pubkey: &str,
) -> Result<git_repo::ReserveOutcome> {
git_repo::reserve_repo_name(&self.pool, community, repo_id, owner_pubkey).await
}
/// Count git repos reserved by `owner_pubkey` in `community` (quota check).
pub async fn count_repos_for_owner(
&self,
community: CommunityId,
owner_pubkey: &str,
) -> Result<i64> {
git_repo::count_repos_for_owner(&self.pool, community, owner_pubkey).await
}
/// Release a git repo name reservation held by `owner_pubkey` (rollback).
///
/// Returns the number of rows removed (0 or 1). See [`git_repo::release_repo_name`].
pub async fn release_repo_name(
&self,
community: CommunityId,
repo_id: &str,
owner_pubkey: &str,
) -> Result<u64> {
git_repo::release_repo_name(&self.pool, community, repo_id, owner_pubkey).await
}
/// Returns `true` if `pubkey` (64-char hex) is archived in `community_id`.
pub async fn is_archived(&self, community_id: CommunityId, pubkey: &str) -> Result<bool> {
archived_identities::is_archived(&self.pool, community_id, pubkey).await
+36 -10
View File
@@ -38,13 +38,23 @@ mod tests {
columns: Vec<String>,
}
fn migration_sql() -> &'static str {
MIGRATOR
/// Concatenated SQL of every embedded migration, in version order.
///
/// The tenant-isolation lints must cover objects introduced by *any*
/// migration, not just the consolidated `0001`. Concatenating keeps that
/// coverage honest as additive migrations (e.g. `0002_git_repo_names`) land.
fn migration_sql() -> String {
let mut migrations: Vec<_> = MIGRATOR.iter().collect();
migrations.sort_by_key(|migration| migration.version);
assert!(
!migrations.is_empty(),
"at least the initial migration must exist"
);
migrations
.iter()
.find(|migration| migration.version == 1)
.expect("initial migration must exist")
.sql
.as_str()
.map(|migration| migration.sql.as_ref())
.collect::<Vec<&str>>()
.join("\n")
}
fn strip_sql_comments(sql: &str) -> String {
@@ -458,9 +468,10 @@ mod tests {
#[test]
fn embedded_migrator_contains_consolidated_initial_schema() {
let migrations: Vec<_> = MIGRATOR.iter().collect();
let mut migrations: Vec<_> = MIGRATOR.iter().collect();
migrations.sort_by_key(|migration| migration.version);
assert_eq!(migrations.len(), 1);
assert_eq!(migrations.len(), 2);
assert_eq!(migrations[0].version, 1);
assert_eq!(&*migrations[0].description, "initial schema");
assert!(migrations[0]
@@ -484,6 +495,17 @@ mod tests {
.sql
.as_str()
.contains("search_tsv TSVECTOR GENERATED ALWAYS"));
// The git repo-name registry is an additive migration, never folded into
// 0001 — folding it would change 0001's checksum and break brownfield
// startup (sqlx VersionMismatch). It must live in its own version, and
// 0001 must not carry it.
assert_eq!(migrations[1].version, 2);
assert!(migrations[1]
.sql
.as_str()
.contains("CREATE TABLE git_repo_names"));
assert!(!migrations[0].sql.as_str().contains("git_repo_names"));
}
#[test]
@@ -575,6 +597,7 @@ mod tests {
#[test]
fn all_non_operator_global_tables_have_not_null_community_id() {
let sql = migration_sql();
let sql = sql.as_str();
let scoped = scoped_tables(sql);
let missing = create_table_definitions(sql)
.into_iter()
@@ -593,6 +616,7 @@ mod tests {
#[test]
fn scoped_primary_key_unique_and_foreign_key_constraints_lead_with_community_id() {
let sql = migration_sql();
let sql = sql.as_str();
let violations = scoped_constraint_violations(sql)
.into_iter()
.map(|constraint| {
@@ -613,6 +637,7 @@ mod tests {
#[test]
fn channels_community_id_is_immutable_after_insert() {
let sql = migration_sql();
let sql = sql.as_str();
let forbidden_mutations = forbidden_channels_community_id_mutations(sql);
assert!(
@@ -664,8 +689,9 @@ mod tests {
run_migrations(&pool).await.expect("run migrations");
assert_eq!(applied_versions(&pool).await, vec![1]);
let tables = create_tables(migration_sql());
assert_eq!(applied_versions(&pool).await, vec![1, 2]);
let sql = migration_sql();
let tables = create_tables(sql.as_str());
for table in [
"communities",
"events",
+8 -3
View File
@@ -128,9 +128,14 @@ pub struct Config {
/// 60 seconds after the last message.
pub ephemeral_ttl_override: Option<i32>,
/// Root directory for the relay's local git state. No per-repo bare repos
/// live here — runtime reads/writes hydrate ephemeral repos from object
/// storage. Holds only the name-reservation index at `{git_repo_path}/.names/`.
/// Root directory for the relay's local git scratch. No per-repo bare repos
/// or persistent git state live here — runtime reads/writes hydrate
/// ephemeral repos from object storage per request, and repo-name
/// uniqueness now lives in Postgres (`git_repo_names`), not on disk. Retained
/// for ephemeral working space and env compatibility; the relay no longer
/// depends on this path being persistent or shared across replicas, so it
/// needs no ReadWriteMany volume. (Removing the field entirely is a
/// follow-up cleanup once the deploy chart drops the git PVC mount.)
pub git_repo_path: std::path::PathBuf,
/// Maximum pack file size for git push (bytes). Default: 500 MB.
pub git_max_pack_bytes: u64,
+183 -96
View File
@@ -2149,7 +2149,7 @@ fn validate_repo_id(repo_id: &str) -> bool {
///
/// Security hardening:
/// - Repo name validated: `[a-zA-Z0-9._-]{1,64}`, no leading dots, no `..`
/// - Name reserved atomically (`.names/<repo_id>`), unique across owners
/// - Name reserved atomically in Postgres (`git_repo_names`), unique per community
/// - Per-pubkey repo count limit enforced
async fn handle_git_repo_announcement(
tenant: &TenantContext,
@@ -2173,123 +2173,165 @@ async fn handle_git_repo_announcement(
// (see `api::git::hydrate`). Announce only (1) reserves the repo name and
// (2) seeds the empty-manifest pointer that makes the repo clone-able.
//
// `.names/<community>/<repo_id>` is the relay's name registry. Each
// reservation holds an `owner` file naming the announcer. It serves three
// jobs at once inside the server-resolved community boundary:
// - uniqueness: `create_dir` is atomic, so concurrent kind:30617 events
// for the same community/name can't both claim it (TOCTOU-free);
// The `git_repo_names` table (Postgres) is the relay's name registry,
// keyed `(community_id, repo_id)`. It serves three jobs at once inside the
// server-resolved community boundary:
// - uniqueness: `INSERT … ON CONFLICT DO NOTHING` is atomic, so
// concurrent kind:30617 events for the same community/name can't both
// claim it (TOCTOU-free — the DB PK is the race guard);
// - idempotent re-announce: a reservation owned by the same pubkey is an
// update, not a collision;
// - per-pubkey quota: count the reservations whose `owner` matches.
// - per-pubkey quota: `COUNT` reservations owned by this pubkey.
//
// This is the one local-disk simplification in v1: separate relay
// instances with separate disks would each grant the name, with the CAS
// pointer (not this registry) preventing actual ref-state corruption. A
// CAS-backed name index is the multi-instance follow-up.
let git_repo_root = &state.config.git_repo_path;
let names_dir = git_repo_root
.join(".names")
.join(tenant.community().to_string());
std::fs::create_dir_all(&names_dir)
.map_err(|e| anyhow::anyhow!("failed to create name reservation index: {e}"))?;
// This replaces the v1 local-disk `.names/` index. Moving it into Postgres
// (which the relay already requires) removes the last persistent local-disk
// state, so separate replicas no longer need a shared ReadWriteMany volume
// to agree on name ownership. Actual ref-state safety remains the
// object-store pointer CAS (`api::git::cas_publish`, `Inv_NoFork`); this
// registry only governs name allocation.
let community = tenant.community();
use buzz_db::git_repo::ReserveOutcome;
let reservation = names_dir.join(&repo_id);
let owner_marker = reservation.join("owner");
// Re-announce by the same owner is a no-op update; a name held by anyone
// else is a collision (the relay signs kind:30618 with d-tag = repo_name,
// so a shared name would let one owner overwrite another's ref state).
if reservation.exists() {
match std::fs::read_to_string(&owner_marker) {
Ok(existing) if existing == owner_hex => {
info!(
repo_id = %repo_id,
owner = %owner_hex,
"kind:30617 repo announcement updated (name already reserved)"
);
return Ok(());
}
_ => {
// Classify the name first: same-owner re-announce is idempotent; a name
// held by anyone else is a collision (the relay signs kind:30618 with
// d-tag = repo_name, so a shared name would let one owner overwrite
// another's ref state). For a not-yet-owned name we must check quota
// *before* claiming, so we peek the current holder rather than inserting
// blindly.
//
// Crucially, we do NOT return early on a same-owner existing row: the row
// proves name *ownership*, not that the manifest pointer was actually
// seeded. A concurrent same-owner announce could hold the row while its
// seed is still in flight (or failed and rolled back), so trusting the row
// alone would let this handler "accept" an uncloneable repo. Instead we
// fall through to `seed_manifest_pointer`, which is idempotent under
// concurrency (create-only `put_pointer(IfNoneMatchStar)`; a `LostRace` on
// the same empty digest is success, a different non-empty pointer is a
// hard error). So re-announce *ensures* the pointer rather than assuming it.
let outcome =
if let Some(existing_owner) = state.db.repo_name_owner(community, &repo_id).await? {
if existing_owner != owner_hex {
return Err(anyhow::anyhow!(
"repo name '{repo_id}' already taken by another owner"
));
}
}
}
// Same owner: the reservation already exists (this attempt did not
// create it), so it must never be rolled back by this attempt, and the
// per-pubkey quota is unchanged (re-announce never grows the count).
ReserveOutcome::AlreadyOwned
} else {
// Not yet owned by anyone we saw: enforce the per-pubkey quota, then
// claim the name atomically. The `ON CONFLICT` guard resolves a
// concurrent announce even though the peek above missed it —
// `Reserved` means *this attempt* won the insert, `AlreadyOwned` means
// a same-owner sibling won it, `TakenByOther` is a cross-owner
// collision.
let limit = state.config.git_max_repos_per_pubkey as i64;
let owned = state
.db
.count_repos_for_owner(community, &owner_hex)
.await?;
if owned >= limit {
return Err(anyhow::anyhow!("repo limit exceeded: {owned} >= {limit}"));
}
match state
.db
.reserve_repo_name(community, &repo_id, &owner_hex)
.await?
{
outcome @ (ReserveOutcome::Reserved | ReserveOutcome::AlreadyOwned) => outcome,
ReserveOutcome::TakenByOther => {
return Err(anyhow::anyhow!(
"repo name '{repo_id}' already taken by another owner"
));
}
}
};
// Per-pubkey repo count limit: reservations owned by this pubkey.
let limit = state.config.git_max_repos_per_pubkey as usize;
let owned = std::fs::read_dir(&names_dir)
.map(|entries| {
entries
.filter_map(|e| e.ok())
.filter(|e| {
std::fs::read_to_string(e.path().join("owner"))
.map(|o| o == owner_hex)
.unwrap_or(false)
})
.count()
})
.unwrap_or(0);
if owned >= limit {
return Err(anyhow::anyhow!("repo limit exceeded: {owned} >= {limit}"));
}
// Only a genuinely fresh claim by *this* attempt may be rolled back on a
// pointer failure. An `AlreadyOwned` outcome means the row is owned by some
// other attempt (a same-owner sibling, or a prior announce that has since
// pushed), and deleting it here would strand a repo whose pointer that
// other attempt already established.
let reserved_by_this_attempt = matches!(outcome, ReserveOutcome::Reserved);
// Claim the name. `create_dir` (not `create_dir_all`) fails AlreadyExists
// if a concurrent announce won the race, closing the TOCTOU window above.
match std::fs::create_dir(&reservation) {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
return Err(anyhow::anyhow!(
"repo name '{repo_id}' already taken by another owner"
));
}
Err(e) => {
return Err(anyhow::anyhow!(
"failed to reserve repo name '{repo_id}': {e}"
));
// Establish/confirm the manifest pointer, keeping the invariant
// "repo announced ⟺ pointer exists" so the read path can rely on
// pointer-absent meaning never-announced (keeping `info_refs`'s fail-closed
// `Ok(None) → 404` unambiguous). Two distinct cases:
//
// - Fresh `Reserved` claim → `seed_manifest_pointer` (strict). This creates
// the empty pointer, and correctly *fails* if a non-empty pointer already
// exists for a name we just reserved — that would be a suspicious stale
// pointer from a prior repo lifecycle, not a legitimate re-announce.
// - Same-owner `AlreadyOwned` (re-announce) → `ensure_manifest_pointer`
// (tolerant). A non-empty pointer is the *normal* post-push state, so
// re-announce must accept it untouched; only an absent pointer is
// repaired by seeding. Using the strict seed here would wrongly reject
// every re-announce after the first push.
let pointer_result = if reserved_by_this_attempt {
seed_manifest_pointer(state, tenant, &owner_hex, &repo_id).await
} else {
ensure_manifest_pointer(state, tenant, &owner_hex, &repo_id).await
};
if let Err(pointer_err) = pointer_result {
// A reserved name without a clone-able pointer is exactly the broken
// state this step exists to prevent — but ONLY roll back the
// reservation if this attempt is the one that freshly created it. A
// genuine failure from a fresh `Reserved` attempt means the pointer
// truly could not be established, so releasing our own just-inserted
// row is safe and correct (all-or-nothing). For an `AlreadyOwned`
// attempt we release nothing: the row belongs to another attempt that
// may have seeded (or pushed) successfully.
if reserved_by_this_attempt {
if let Err(release_err) = state
.db
.release_repo_name(community, &repo_id, &owner_hex)
.await
{
warn!(
repo_id = %repo_id,
error = %release_err,
"failed to release repo name reservation after seed failure"
);
}
}
return Err(anyhow::anyhow!(
"failed to ensure manifest pointer: {pointer_err}"
));
}
if let Err(e) = std::fs::write(&owner_marker, &owner_hex) {
let _ = std::fs::remove_dir_all(&reservation);
return Err(anyhow::anyhow!("failed to record repo owner: {e}"));
}
// Seed the empty-manifest pointer in object storage. Establishes the
// invariant "repo announced ⟺ pointer exists" so the read path can rely
// on pointer-absent meaning never-announced (not just no-pushes-yet),
// keeping `info_refs`'s fail-closed `Ok(None) → 404` unambiguous.
// First push CASes the seeded pointer normally — no special-case branch.
seed_manifest_pointer(state, tenant, &owner_hex, &repo_id)
.await
.map_err(|e| {
// A reserved name without a clone-able pointer is exactly the
// broken state the seed exists to prevent. Release the reservation
// so the announce is either fully consummated or fully rolled back.
let _ = std::fs::remove_dir_all(&reservation);
anyhow::anyhow!("failed to seed manifest pointer: {e}")
})?;
info!(
repo_id = %repo_id,
owner = %owner_hex,
"kind:30617 repo announced (name reserved, manifest pointer seeded)"
reserved = reserved_by_this_attempt,
"kind:30617 repo announced (name reserved, manifest pointer ensured)"
);
// Derived after the pointer commits: kind:30618 ref-state event over the
// seeded empty manifest. Pointer is the commit; this event is the
// notification that the repo exists (with empty refs) so subscribers see
// a first signal without waiting for the first push.
if let Err(e) = emit_initial_ref_state(tenant, state, &owner_hex, &repo_id).await {
// Non-fatal: the manifest is the source of truth; this is just the
// derived notification. A failure here means subscribers miss the
// "repo now exists" event, but clone/push still works.
warn!(
repo_id = %repo_id,
owner = %owner_hex,
error = %e,
"failed to emit initial kind:30618 ref state (non-fatal)"
);
//
// Emit ONLY on a fresh `Reserved` claim. On a same-owner `AlreadyOwned`
// re-announce the pointer already exists (and, after the first push, holds
// real refs). Re-emitting the empty-refs 30618 here would publish a *newer*
// replaceable event that, under NIP-16 latest-wins ordering, shadows the
// real pushed refs — making a live repo look empty to subscribers. The
// initial empty signal is a one-time seeding notification, not something a
// re-announce should replay.
if reserved_by_this_attempt {
if let Err(e) = emit_initial_ref_state(tenant, state, &owner_hex, &repo_id).await {
// Non-fatal: the manifest is the source of truth; this is just the
// derived notification. A failure here means subscribers miss the
// "repo now exists" event, but clone/push still works.
warn!(
repo_id = %repo_id,
owner = %owner_hex,
error = %e,
"failed to emit initial kind:30618 ref state (non-fatal)"
);
}
}
Ok(())
@@ -2380,6 +2422,51 @@ async fn seed_manifest_pointer(
}
}
/// Ensure a manifest pointer exists for an already-owned repo (same-owner
/// re-announce path). Unlike [`seed_manifest_pointer`], which is strict for
/// *creation* (it refuses when a non-empty pointer already exists, since a
/// freshly-reserved name with a populated pointer is suspicious), this is the
/// tolerant *idempotent* path for a name this owner already holds:
///
/// - **pointer present** (empty *or* non-empty) → success, left untouched. A
/// non-empty pointer is the normal state after the owner has pushed; a
/// re-announce must not fail just because the repo has commits, and must
/// never overwrite real ref state.
/// - **pointer absent** → seed the empty pointer (repair the "row exists but
/// pointer missing" window, e.g. a prior announce whose seed failed after
/// the row was inserted by a sibling attempt). This restores the
/// "announced ⟺ pointer exists" invariant.
///
/// The read-then-conditional-seed is race-safe: the repair uses
/// `seed_manifest_pointer`'s create-only `put_pointer(IfNoneMatchStar)`, so a
/// concurrent seeder that wins is resolved by that function's `LostRace`
/// handling (same empty digest → Ok), and a concurrent *pusher* that populates
/// the pointer between our read and our seed loses the create race and is
/// likewise treated as an already-present pointer, not an overwrite.
async fn ensure_manifest_pointer(
state: &Arc<AppState>,
tenant: &TenantContext,
owner_hex: &str,
repo_id: &str,
) -> anyhow::Result<()> {
use crate::api::git::manifest::pointer_key;
let pkey = pointer_key(tenant.community(), owner_hex, repo_id);
let existing = state
.git_store
.get_pointer(&pkey)
.await
.map_err(|e| anyhow::anyhow!("get_pointer: {e}"))?;
match existing {
// Any existing pointer (empty or non-empty) is valid for a same-owner
// re-announce — leave it exactly as-is.
Some(_) => Ok(()),
// No pointer yet: repair by seeding the empty pointer. `LostRace` to a
// concurrent seeder/pusher is handled by `seed_manifest_pointer`.
None => seed_manifest_pointer(state, tenant, owner_hex, repo_id).await,
}
}
/// Emit the initial kind:30618 ref-state event for a freshly-announced repo.
///
/// The seeded empty manifest is the source of truth; this event is the
+4 -3
View File
@@ -54,12 +54,13 @@ The chart fails at `helm install` / `helm template` time with a clear message if
## HA (production)
`replicaCount > 1` hard-requires both:
`replicaCount > 1` hard-requires Redis:
- Redis (`redis.enabled=true`, `externalRedis.url`, or `REDIS_URL` in `existingSecret`) — for `buzz-pubsub` fan-out
- ReadWriteMany git PVC — `persistence.git.accessMode: ReadWriteMany` with a RWX storage class (e.g. `efs-sc` on AWS, `azurefile-csi` on Azure)
The chart **template-fails** if either invariant is broken. No silent degradation.
It does **not** require ReadWriteMany git storage. Git ref/object state is object-store-backed (each request hydrates an ephemeral repo from S3-compatible storage; writer serialization is the object-store pointer CAS — see `docs/git-on-object-storage.md`), and repo-name uniqueness lives in Postgres. Each replica can use its own `ReadWriteOnce` volume; no shared filesystem is needed.
The chart **template-fails** if the Redis invariant is broken at `replicaCount > 1`. No silent degradation.
## Upgrades
+2 -2
View File
@@ -46,8 +46,8 @@ spec:
persistence:
git:
enabled: true
accessMode: ReadWriteMany # required: replicaCount > 1
storageClass: efs-sc # provider-specific RWX class
accessMode: ReadWriteOnce # RWO is fine at any replicaCount (object-store-backed git)
storageClass: "" # any RWO class; no shared/RWX filesystem needed
size: 50Gi
ingress:
@@ -45,8 +45,8 @@ spec:
persistence:
git:
enabled: true
accessMode: ReadWriteMany
storageClass: efs-sc
accessMode: ReadWriteOnce # RWO is fine at any replicaCount (object-store-backed git)
storageClass: "" # any RWO class; no shared/RWX filesystem needed
size: 50Gi
ingress:
-4
View File
@@ -75,10 +75,6 @@
Kubernetes Secret and reference it via secrets.existingSecret — see
examples/secret-sample.yaml.
{{- end }}
{{- if and (gt (.Values.replicaCount | int) 1) (eq .Values.persistence.git.accessMode "ReadWriteOnce") }}
⚠ replicaCount > 1 with ReadWriteOnce git PVC will fail at template time
(this message should never appear — file a bug).
{{- end }}
{{- if not .Values.secrets.existingSecret }}
{{- if not (or .Values.postgresql.enabled .Values.redis.enabled) }}
⚠ Chart-managed Secret is in use (no secrets.existingSecret). This is fine
+14 -8
View File
@@ -17,14 +17,20 @@ surface at template time regardless of which manifest helm renders first.
{{- end -}}
{{- end -}}
{{/* replicaCount > 1 requires ReadWriteMany git storage */}}
{{- if gt (.Values.replicaCount | int) 1 -}}
{{- if and .Values.persistence.git.enabled (not .Values.persistence.git.existingClaim) -}}
{{- if ne .Values.persistence.git.accessMode "ReadWriteMany" -}}
{{- fail (printf "replicaCount=%d requires persistence.git.accessMode=ReadWriteMany (got %q). The relay's git on-disk state must be shared across replicas." (.Values.replicaCount | int) .Values.persistence.git.accessMode) -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/* replicaCount > 1 does NOT require ReadWriteMany git storage.
Git ref/object state is object-store-backed: every read and write hydrates
an ephemeral bare repo from S3-compatible storage per request, and writer
serialization is the object-store pointer CAS
(docs/git-on-object-storage.md, Inv_NoFork). No persistent git state lives
on the PVC, so replicas do not need a shared ReadWriteMany volume to agree
on refs. Repo-name uniqueness — the last shared-state need — now lives in
Postgres (git_repo_names), not on local disk.
The prior hard-fail requiring persistence.git.accessMode=ReadWriteMany was
removed here: its stated reason ("git on-disk state must be shared across
replicas") is no longer true. Redis (validated above) remains the real
multi-pod requirement for buzz-pubsub. */}}
{{/* Owner pubkey required when requireRelayMembership */}}
{{- if .Values.relay.requireRelayMembership -}}
+2 -2
View File
@@ -1,4 +1,4 @@
# HA shape: replicas=3 + Redis + RWX. Render-only check.
# HA shape: replicas=3 + Redis + RWO git (object-store-backed; no RWM). Render-only check.
relayUrl: wss://buzz.example.com
ownerPubkey: "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"
replicaCount: 3
@@ -13,7 +13,7 @@ s3:
persistence:
git:
enabled: true
accessMode: ReadWriteMany
accessMode: ReadWriteOnce
size: 50Gi
podDisruptionBudget:
enabled: true
@@ -14,7 +14,7 @@ s3:
persistence:
git:
enabled: true
accessMode: ReadWriteMany
accessMode: ReadWriteOnce
size: 50Gi
ingress:
enabled: true
+4 -4
View File
@@ -65,7 +65,7 @@ tests:
s3.accessKey: a
s3.secretKey: s
replicaCount: 3
persistence.git.accessMode: ReadWriteMany
persistence.git.accessMode: ReadWriteOnce
relay.huddleAudioAvailable: true
asserts:
- contains:
@@ -75,7 +75,7 @@ tests:
value: "true"
template: templates/deployment.yaml
- it: renders HA cleanly with replicaCount=3 + RWX + Redis
- it: renders HA cleanly with replicaCount=3 + RWO git + Redis (no RWM needed)
set:
relayUrl: wss://buzz.example.com
ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000"
@@ -85,7 +85,7 @@ tests:
s3.accessKey: a
s3.secretKey: s
replicaCount: 3
persistence.git.accessMode: ReadWriteMany
persistence.git.accessMode: ReadWriteOnce
asserts:
- equal:
path: spec.replicas
@@ -93,7 +93,7 @@ tests:
template: templates/deployment.yaml
- equal:
path: spec.accessModes[0]
value: ReadWriteMany
value: ReadWriteOnce
template: templates/pvc-git.yaml
- contains:
path: spec.template.spec.containers[0].env
@@ -39,18 +39,6 @@ tests:
- failedTemplate:
errorPattern: "replicaCount=3 requires Redis"
- it: fails when replicaCount>1 with RWO git PVC
set:
relayUrl: wss://buzz.example.com
ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000"
externalPostgresql.url: postgres://u:p@h:5432/d
externalRedis.url: redis://h:6379
replicaCount: 3
persistence.git.accessMode: ReadWriteOnce
asserts:
- failedTemplate:
errorPattern: "requires persistence.git.accessMode=ReadWriteMany"
- it: fails when ingress and httproute both enabled
set:
relayUrl: wss://buzz.example.com
+1 -1
View File
@@ -26,7 +26,7 @@
"replicaCount": {
"type": "integer",
"minimum": 1,
"description": "Replica count for the relay Deployment. replicaCount > 1 requires Redis (for buzz-pubsub) and ReadWriteMany git storage — enforced by _validate.tpl."
"description": "Replica count for the relay Deployment. replicaCount > 1 requires Redis (for buzz-pubsub) — enforced by _validate.tpl. Git storage does NOT need ReadWriteMany: git state is object-store-backed and repo names live in Postgres, so ReadWriteOnce is fine per replica."
},
"relayUrl": {
"type": "string",
+13 -6
View File
@@ -4,7 +4,8 @@
#
# PRODUCTION (default) — external Postgres/Redis/S3, existingSecret
# refs everywhere, no chart-side autogeneration, GitOps-safe (ArgoCD/Flux).
# HA-ready: replicaCount >= 2 (requires Redis and RWX storage for git).
# HA-ready: replicaCount >= 2 (requires Redis; git state is object-store-
# backed, so no ReadWriteMany volume is needed — RWO per replica is fine).
#
# QUICKSTART — bundles in-cluster Postgres + Redis + MinIO and
# auto-generates relay secrets via the `lookup` pattern (NOT GitOps-safe —
@@ -28,9 +29,11 @@ image:
pullSecrets: []
# ── Topology ────────────────────────────────────────────────────────────────
# replicaCount > 1 hard-requires:
# - Redis for buzz-pubsub (in-cluster or external)
# - ReadWriteMany storage for the git PVC (or shared FS via existingClaim)
# replicaCount > 1 hard-requires Redis for buzz-pubsub (in-cluster or external).
# It does NOT require ReadWriteMany git storage: git ref/object state is
# object-store-backed (each request hydrates an ephemeral repo from S3; writer
# serialization is the object-store pointer CAS), and repo-name uniqueness lives
# in Postgres. Each replica can use its own ReadWriteOnce volume (or none).
replicaCount: 1
# ── Public URL ───────────────────────────────────────────────────────────────
@@ -170,13 +173,17 @@ httproute:
hostnames: []
rules: [] # empty → default match-all → service
# ── Git on-disk state ───────────────────────────────────────────────────────
# ── Git scratch volume ───────────────────────────────────────────────────────
# Ephemeral working space only. No persistent git state lives here — reads/writes
# hydrate ephemeral repos from object storage per request, and repo-name
# uniqueness lives in Postgres. ReadWriteOnce is correct at any replicaCount; a
# ReadWriteMany volume is NOT required for multi-pod.
persistence:
git:
enabled: true
mountPath: /var/lib/buzz/git
storageClass: ""
accessMode: ReadWriteOnce # MUST be ReadWriteMany if replicaCount > 1
accessMode: ReadWriteOnce # RWO is fine at any replicaCount (object-store-backed git)
size: 10Gi
annotations: {}
existingClaim: ""
+29
View File
@@ -0,0 +1,29 @@
-- ── Git repo name registry (NIP-34 kind:30617) ───────────────────────────────
-- The relay holds no persistent per-repo filesystem state: git reads/writes
-- hydrate an ephemeral bare repo from object storage per request, and writer
-- serialization is the object-store pointer CAS (docs/git-on-object-storage.md,
-- Inv_NoFork). This table is the one remaining shared-state need — repo-name
-- uniqueness — moved off local disk so the relay is stateless and can run
-- multiple replicas without a ReadWriteMany volume.
--
-- Additive migration (not folded into 0001): brownfield databases that already
-- applied the pre-PR 0001 must not see its checksum change, or sqlx aborts
-- startup with a VersionMismatch. New table + index only; no edits to existing
-- objects.
--
-- Per-community, not global: a repo name is unique within a community, matching
-- the multi-tenant invariant (community_id leads the PK). The PK enforces
-- uniqueness atomically (INSERT … ON CONFLICT), replacing the old atomic
-- `create_dir`. `owner_pubkey` distinguishes idempotent re-announce (same owner)
-- from collision (different owner), and backs the per-pubkey quota via COUNT.
CREATE TABLE git_repo_names (
community_id UUID NOT NULL REFERENCES communities(id),
repo_id TEXT NOT NULL,
owner_pubkey TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (community_id, repo_id)
);
-- Backs the per-pubkey repo quota: COUNT(*) WHERE community_id = $1 AND owner_pubkey = $2.
CREATE INDEX idx_git_repo_names_owner ON git_repo_names (community_id, owner_pubkey);