diff --git a/crates/buzz-media/src/bucket_index.rs b/crates/buzz-media/src/bucket_index.rs index 2807371fd..3aa18fef0 100644 --- a/crates/buzz-media/src/bucket_index.rs +++ b/crates/buzz-media/src/bucket_index.rs @@ -13,8 +13,8 @@ //! //! | Class | Shape | //! |---|---| -//! | thumb | `{sha256}.thumb.jpg` or `media/{hh}/{hh}/{community-uuid}/{sha256}.thumb.jpg` | -//! | blob | `{sha256}.{ext}` or `media/{hh}/{hh}/{community-uuid}/{sha256}.{ext}` (ext: 1-8 mixed-case alphanumeric) | +//! | thumb | `{sha256}.thumb.jpg` or `media/{hh}/{hh}/{sha256}.thumb.jpg` | +//! | blob | `{sha256}.{ext}` or `media/{hh}/{hh}/{sha256}.{ext}` (ext: 1-8 mixed-case alphanumeric) | //! | sidecar | `_meta/{community-uuid}/{sha256}.json` | //! | auxiliary | `_uploads/{community-uuid}/{sha256}/{ulid}.json` | //! | unknown | everything else | @@ -33,16 +33,9 @@ use crate::error::MediaError; #[derive(Debug, Clone, PartialEq, Eq)] pub enum KeyClass { /// Legacy or sharded thumbnail, attributed to the blob's sha. - Thumb { - community: Option, - sha256: String, - }, - /// Legacy or sharded blob; sharded keys carry direct community attribution. - Blob { - community: Option, - sha256: String, - ext: String, - }, + Thumb { sha256: String }, + /// Legacy or sharded blob; sidecar bindings provide community attribution. + Blob { sha256: String, ext: String }, /// `_meta/{community}/{sha256}.json` — the (community, sha) binding. Sidecar { community: Uuid, sha256: String }, /// `_uploads/{community}/{sha256}/{event_id}.json` — fleet physical only. @@ -59,34 +52,20 @@ pub enum KeyClass { /// shape of the blob pattern's segment count), then blob, sidecar, /// auxiliary, and finally unknown. See module docs for the exact shapes. pub fn classify_key(key: &str) -> KeyClass { - if let Some((community, filename)) = parse_sharded_prefix(key) { - if let Some(parsed_sha) = parse_thumb_key(filename) { - return KeyClass::Thumb { - community: Some(community), - sha256: parsed_sha, - }; + if let Some(filename) = parse_sharded_prefix(key) { + if let Some(sha256) = parse_thumb_key(filename) { + return KeyClass::Thumb { sha256 }; } - if let Some((parsed_sha, ext)) = parse_blob_key(filename) { - return KeyClass::Blob { - community: Some(community), - sha256: parsed_sha, - ext, - }; + if let Some((sha256, ext)) = parse_blob_key(filename) { + return KeyClass::Blob { sha256, ext }; } return KeyClass::Unknown; } if let Some(sha256) = parse_thumb_key(key) { - return KeyClass::Thumb { - community: None, - sha256, - }; + return KeyClass::Thumb { sha256 }; } if let Some((sha256, ext)) = parse_blob_key(key) { - return KeyClass::Blob { - community: None, - sha256, - ext, - }; + return KeyClass::Blob { sha256, ext }; } if let Some((community, sha256)) = parse_sidecar_key(key) { return KeyClass::Sidecar { community, sha256 }; @@ -155,16 +134,15 @@ fn parse_canonical_uuid(s: &str) -> Option { Uuid::parse_str(s).ok() } -/// `media/{sha[0:2]}/{sha[2:4]}/{community}/{filename}`. The filename's digest -/// must agree with both shard segments; malformed migration keys stay unknown. -fn parse_sharded_prefix(key: &str) -> Option<(Uuid, &str)> { +/// `media/{sha[0:2]}/{sha[2:4]}/{filename}`. The filename's digest must +/// agree with both shard segments; malformed migration keys stay unknown. +fn parse_sharded_prefix(key: &str) -> Option<&str> { let mut segments = key.split('/'); if segments.next()? != "media" { return None; } let shard_1 = segments.next()?; let shard_2 = segments.next()?; - let community = parse_canonical_uuid(segments.next()?)?; let filename = segments.next()?; if segments.next().is_some() || shard_1.len() != 2 || shard_2.len() != 2 { return None; @@ -173,7 +151,7 @@ fn parse_sharded_prefix(key: &str) -> Option<(Uuid, &str)> { if !is_sha256(sha256) || shard_1 != &sha256[..2] || shard_2 != &sha256[2..4] { return None; } - Some((community, filename)) + Some(filename) } /// `{sha256}.thumb.jpg` @@ -286,31 +264,24 @@ pub struct BucketSnapshot { #[derive(Debug, Default)] struct LayoutCopies { legacy: Option, - sharded: HashMap, + sharded: Option, } impl LayoutCopies { - fn insert(&mut self, community: Option, size: u64) { - match community { - Some(community) => { - self.sharded.insert(community, size); - } - None => { - self.legacy = Some(size); - } - } + fn insert_legacy(&mut self, size: u64) { + self.legacy = Some(size); + } + + fn insert_sharded(&mut self, size: u64) { + self.sharded = Some(size); } fn physical_bytes(&self) -> u64 { - self.legacy.unwrap_or(0) + self.sharded.values().sum::() + self.legacy.unwrap_or(0) + self.sharded.unwrap_or(0) } - fn logical_bytes(&self, community: Uuid) -> u64 { - self.sharded - .get(&community) - .copied() - .or(self.legacy) - .unwrap_or(0) + fn logical_bytes(&self) -> u64 { + self.sharded.or(self.legacy).unwrap_or(0) } } @@ -336,21 +307,21 @@ impl BucketAggregate { self.physical_objects += 1; self.physical_bytes += size; match classify_key(key) { - KeyClass::Thumb { community, sha256 } => { - self.thumb_copies - .entry(sha256) - .or_default() - .insert(community, size); + KeyClass::Thumb { sha256 } => { + let copies = self.thumb_copies.entry(sha256).or_default(); + if key.starts_with("media/") { + copies.insert_sharded(size); + } else { + copies.insert_legacy(size); + } } - KeyClass::Blob { - community, - sha256, - ext, - } => { - self.blob_variants - .entry((sha256, ext)) - .or_default() - .insert(community, size); + KeyClass::Blob { sha256, ext } => { + let copies = self.blob_variants.entry((sha256, ext)).or_default(); + if key.starts_with("media/") { + copies.insert_sharded(size); + } else { + copies.insert_legacy(size); + } } KeyClass::Sidecar { community, sha256 } => { self.sidecar_bindings.insert((community, sha256), size); @@ -380,7 +351,7 @@ impl BucketAggregate { let mut duplicate_layout_variants = 0u64; let mut duplicate_layout_bytes = 0u64; for copies in self.blob_variants.values() { - if copies.legacy.is_some() && !copies.sharded.is_empty() { + if copies.legacy.is_some() && copies.sharded.is_some() { duplicate_layout_variants += 1; duplicate_layout_bytes += copies.physical_bytes(); } @@ -414,17 +385,12 @@ impl BucketAggregate { for (community, sha256) in self.sidecar_bindings.keys() { let blob_bytes: u64 = variants_by_sha .get(sha256.as_str()) - .map(|variants| { - variants - .iter() - .map(|copies| copies.logical_bytes(*community)) - .sum() - }) + .map(|variants| variants.iter().map(|copies| copies.logical_bytes()).sum()) .unwrap_or(0); let thumb_bytes = self .thumb_copies .get(sha256) - .map(|copies| copies.logical_bytes(*community)) + .map(|copies| copies.logical_bytes()) .unwrap_or(0); let entry = per_community.entry(*community).or_default(); entry.bytes += blob_bytes + thumb_bytes; @@ -546,10 +512,7 @@ mod tests { let s = sha(0xaa); assert_eq!( classify_key(&format!("{s}.thumb.jpg")), - KeyClass::Thumb { - community: None, - sha256: s, - } + KeyClass::Thumb { sha256: s } ); } @@ -559,7 +522,6 @@ mod tests { assert_eq!( classify_key(&format!("{s}.png")), KeyClass::Blob { - community: None, sha256: s, ext: "png".to_string() } @@ -574,7 +536,6 @@ mod tests { assert_eq!( classify_key(&format!("{s}.Z")), KeyClass::Blob { - community: None, sha256: s, ext: "Z".to_string() } @@ -582,38 +543,32 @@ mod tests { } #[test] - fn classifies_sharded_blob_and_thumb_keys_with_community() { + fn classifies_global_sharded_blob_and_thumb_keys() { let s = sha(0xab); - let c = community(10); assert_eq!( - classify_key(&format!("media/ab/ab/{c}/{s}.png")), + classify_key(&format!("media/ab/ab/{s}.png")), KeyClass::Blob { - community: Some(c), sha256: s.clone(), ext: "png".to_string(), } ); assert_eq!( - classify_key(&format!("media/ab/ab/{c}/{s}.thumb.jpg")), - KeyClass::Thumb { - community: Some(c), - sha256: s, - } + classify_key(&format!("media/ab/ab/{s}.thumb.jpg")), + KeyClass::Thumb { sha256: s } ); } #[test] fn malformed_sharded_keys_are_unknown() { let s = sha(0xab); - let c = community(11); for key in [ - format!("media/ff/ab/{c}/{s}.png"), - format!("media/ab/ff/{c}/{s}.png"), - format!("media/a/ab/{c}/{s}.png"), - format!("media/ab/ab/not-a-uuid/{s}.png"), - format!("media/ab/ab/{c}/{s}.png/extra"), - format!("media/ab/ab/{c}/{}.png", s.to_uppercase()), - format!("media/ab/ab/{c}/{s}.tar.gz"), + format!("media/ff/ab/{s}.png"), + format!("media/ab/ff/{s}.png"), + format!("media/a/ab/{s}.png"), + format!("media/ab/ab/{s}.png/extra"), + format!("media/ab/ab/{}.png", s.to_uppercase()), + format!("media/ab/ab/{s}.tar.gz"), + format!("media/ab/ab/not-a-sha.png"), ] { assert_eq!(classify_key(&key), KeyClass::Unknown, "key: {key}"); } @@ -725,9 +680,9 @@ mod tests { let c = community(12); let mut agg = BucketAggregate::default(); agg.fold(&format!("{s}.jpg"), 100); - agg.fold(&format!("media/ab/ab/{c}/{s}.jpg"), 100); + agg.fold(&format!("media/ab/ab/{s}.jpg"), 100); agg.fold(&format!("{s}.thumb.jpg"), 20); - agg.fold(&format!("media/ab/ab/{c}/{s}.thumb.jpg"), 20); + agg.fold(&format!("media/ab/ab/{s}.thumb.jpg"), 20); agg.fold(&format!("_meta/{c}/{s}.json"), 10); let snap = agg.finish(); @@ -743,18 +698,19 @@ mod tests { } #[test] - fn sharded_copy_is_attributed_only_to_its_community() { + fn global_sharded_copy_bills_every_bound_sidecar_community() { let s = sha(0xcd); - let sharded_community = community(13); - let other_community = community(14); + let first_community = community(13); + let second_community = community(14); let mut agg = BucketAggregate::default(); - agg.fold(&format!("media/cd/cd/{sharded_community}/{s}.jpg"), 200); - agg.fold(&format!("_meta/{sharded_community}/{s}.json"), 10); - agg.fold(&format!("_meta/{other_community}/{s}.json"), 10); + agg.fold(&format!("media/cd/cd/{s}.jpg"), 200); + agg.fold(&format!("_meta/{first_community}/{s}.json"), 10); + agg.fold(&format!("_meta/{second_community}/{s}.json"), 10); let snap = agg.finish(); - assert_eq!(snap.per_community[&sharded_community].bytes, 200); - assert_eq!(snap.per_community[&other_community].bytes, 0); + assert_eq!(snap.per_community[&first_community].bytes, 200); + assert_eq!(snap.per_community[&second_community].bytes, 200); + assert_eq!(snap.logical_bytes, 400); } #[test] diff --git a/crates/buzz-media/src/keys.rs b/crates/buzz-media/src/keys.rs index de604684a..a8b130716 100644 --- a/crates/buzz-media/src/keys.rs +++ b/crates/buzz-media/src/keys.rs @@ -1,10 +1,12 @@ //! Deterministic object-key derivation for media payloads. //! //! Public Blossom URLs stay flat (`/media/.`), while S3 payloads use -//! hash-leading shards so aggregate request traffic is distributed before the -//! community segment. Legacy keys remain read candidates during migration. +//! hash-leading global CAS shards. Legacy keys remain read candidates during +//! migration. -use buzz_core::tenant::{CommunityId, TenantContext}; +#[cfg(test)] +use buzz_core::tenant::CommunityId; +use buzz_core::tenant::TenantContext; /// Invalid data supplied to media object-key construction. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] @@ -23,7 +25,7 @@ pub enum MediaKeyError { /// Ordered object keys for compatibility reads. #[derive(Debug, Clone, PartialEq, Eq)] pub struct MediaReadCandidates { - /// Hash-sharded, community-scoped key tried first. + /// Hash-sharded global CAS key tried first. pub sharded: String, /// Flat pre-migration key tried only when `sharded` is not found. pub legacy: String, @@ -60,15 +62,11 @@ pub fn legacy_blob_key(sha256: &str, ext: &str) -> Result Ok(format!("{sha256}.{ext}")) } -/// Hash-leading blob key: `media/<2>/<2>//.`. -pub fn sharded_blob_key( - community: CommunityId, - sha256: &str, - ext: &str, -) -> Result { +/// Hash-leading global CAS blob key: `media/<2>/<2>/.`. +pub fn sharded_blob_key(sha256: &str, ext: &str) -> Result { let filename = legacy_blob_key(sha256, ext)?; Ok(format!( - "media/{}/{}/{community}/{filename}", + "media/{}/{}/{filename}", &sha256[..2], &sha256[2..4] )) @@ -80,11 +78,11 @@ pub fn legacy_thumb_key(sha256: &str) -> Result { Ok(format!("{sha256}.thumb.jpg")) } -/// Hash-leading thumbnail key: `media/<2>/<2>//.thumb.jpg`. -pub fn sharded_thumb_key(community: CommunityId, sha256: &str) -> Result { +/// Hash-leading global CAS thumbnail key: `media/<2>/<2>/.thumb.jpg`. +pub fn sharded_thumb_key(sha256: &str) -> Result { let filename = legacy_thumb_key(sha256)?; Ok(format!( - "media/{}/{}/{community}/{filename}", + "media/{}/{}/{filename}", &sha256[..2], &sha256[2..4] )) @@ -92,15 +90,16 @@ pub fn sharded_thumb_key(community: CommunityId, sha256: &str) -> Result Result { if let Some(sha256) = payload_name.strip_suffix(".thumb.jpg") { return Ok(MediaReadCandidates { - sharded: sharded_thumb_key(ctx.community(), sha256)?, + sharded: sharded_thumb_key(sha256)?, legacy: legacy_thumb_key(sha256)?, }); } @@ -112,7 +111,7 @@ pub fn read_candidates( return Err(MediaKeyError::InvalidPayloadName); } Ok(MediaReadCandidates { - sharded: sharded_blob_key(ctx.community(), sha256, ext)?, + sharded: sharded_blob_key(sha256, ext)?, legacy: legacy_blob_key(sha256, ext)?, }) } @@ -129,40 +128,36 @@ mod tests { } #[test] - fn derives_hash_leading_community_scoped_blob_and_thumb_keys() { - let ctx = tenant(1); - let community = ctx.community(); - + fn derives_hash_leading_global_cas_blob_and_thumb_keys() { assert_eq!( - sharded_blob_key(community, SHA, "jpg").unwrap(), - format!("media/ab/cd/{community}/{SHA}.jpg") + sharded_blob_key(SHA, "jpg").unwrap(), + format!("media/ab/cd/{SHA}.jpg") ); assert_eq!( - sharded_thumb_key(community, SHA).unwrap(), - format!("media/ab/cd/{community}/{SHA}.thumb.jpg") + sharded_thumb_key(SHA).unwrap(), + format!("media/ab/cd/{SHA}.thumb.jpg") ); - assert_ne!( - sharded_blob_key(community, SHA, "jpg").unwrap(), - sharded_blob_key(tenant(2).community(), SHA, "jpg").unwrap() + assert_eq!( + sharded_blob_key(SHA, "jpg").unwrap(), + sharded_blob_key(SHA, "jpg").unwrap() ); } #[test] fn orders_sharded_before_legacy_for_blobs_and_thumbnails() { let ctx = tenant(1); - let community = ctx.community(); assert_eq!( read_candidates(&ctx, &format!("{SHA}.png")).unwrap(), MediaReadCandidates { - sharded: format!("media/ab/cd/{community}/{SHA}.png"), + sharded: format!("media/ab/cd/{SHA}.png"), legacy: format!("{SHA}.png"), } ); assert_eq!( read_candidates(&ctx, &format!("{SHA}.thumb.jpg")).unwrap(), MediaReadCandidates { - sharded: format!("media/ab/cd/{community}/{SHA}.thumb.jpg"), + sharded: format!("media/ab/cd/{SHA}.thumb.jpg"), legacy: format!("{SHA}.thumb.jpg"), } ); diff --git a/crates/buzz-media/src/migration.rs b/crates/buzz-media/src/migration.rs index a47b8be0c..58554feb3 100644 --- a/crates/buzz-media/src/migration.rs +++ b/crates/buzz-media/src/migration.rs @@ -38,18 +38,18 @@ pub fn parse_sidecar_key(key: &str) -> Option<(CommunityId, &str)> { /// Derive payload and optional thumbnail pairs represented by a sidecar. pub fn objects_for_sidecar( - community: CommunityId, + _community: CommunityId, sha: &str, meta: &BlobMeta, ) -> Result, crate::MediaKeyError> { let mut objects = vec![MigrationObject { legacy: legacy_blob_key(sha, &meta.ext)?, - sharded: sharded_blob_key(community, sha, &meta.ext)?, + sharded: sharded_blob_key(sha, &meta.ext)?, }]; if !meta.thumb_url.is_empty() { objects.push(MigrationObject { legacy: legacy_thumb_key(sha)?, - sharded: sharded_thumb_key(community, sha)?, + sharded: sharded_thumb_key(sha)?, }); } Ok(objects) @@ -108,6 +108,8 @@ mod tests { meta.thumb_url = "https://media.example/thumb".into(); let objects = objects_for_sidecar(community, SHA, &meta).unwrap(); assert_eq!(objects.len(), 2); + assert_eq!(objects[0].sharded, format!("media/ab/cd/{SHA}.jpg")); + assert_eq!(objects[1].sharded, format!("media/ab/cd/{SHA}.thumb.jpg")); assert!(objects[1].legacy.ends_with(".thumb.jpg")); } } diff --git a/crates/buzz-media/src/storage.rs b/crates/buzz-media/src/storage.rs index f888a7931..1d05f1631 100644 --- a/crates/buzz-media/src/storage.rs +++ b/crates/buzz-media/src/storage.rs @@ -167,7 +167,7 @@ impl MediaStorage { /// copy second. Callers publish sidecars only after this returns success. pub async fn put_payload( &self, - ctx: &TenantContext, + _ctx: &TenantContext, sha256: &str, ext: &str, bytes: &[u8], @@ -175,7 +175,7 @@ impl MediaStorage { ) -> Result { let legacy = crate::keys::legacy_blob_key(sha256, ext) .map_err(|e| MediaError::StorageError(e.to_string()))?; - let sharded = crate::keys::sharded_blob_key(ctx.community(), sha256, ext) + let sharded = crate::keys::sharded_blob_key(sha256, ext) .map_err(|e| MediaError::StorageError(e.to_string()))?; if self.migration_phase.writes_sharded() { self.put(&sharded, bytes, content_type).await?; @@ -193,7 +193,7 @@ impl MediaStorage { /// Stream a media payload from disk according to the configured layout. pub async fn put_payload_file( &self, - ctx: &TenantContext, + _ctx: &TenantContext, sha256: &str, ext: &str, path: &Path, @@ -201,7 +201,7 @@ impl MediaStorage { ) -> Result { let legacy = crate::keys::legacy_blob_key(sha256, ext) .map_err(|e| MediaError::StorageError(e.to_string()))?; - let sharded = crate::keys::sharded_blob_key(ctx.community(), sha256, ext) + let sharded = crate::keys::sharded_blob_key(sha256, ext) .map_err(|e| MediaError::StorageError(e.to_string()))?; if self.migration_phase.writes_sharded() { self.put_file(&sharded, path, content_type).await?; @@ -219,13 +219,13 @@ impl MediaStorage { /// Store a thumbnail according to the configured migration layout. pub async fn put_thumbnail( &self, - ctx: &TenantContext, + _ctx: &TenantContext, sha256: &str, bytes: &[u8], ) -> Result { let legacy = crate::keys::legacy_thumb_key(sha256) .map_err(|e| MediaError::StorageError(e.to_string()))?; - let sharded = crate::keys::sharded_thumb_key(ctx.community(), sha256) + let sharded = crate::keys::sharded_thumb_key(sha256) .map_err(|e| MediaError::StorageError(e.to_string()))?; if self.migration_phase.writes_sharded() { self.put(&sharded, bytes, "image/jpeg").await?; diff --git a/crates/buzz-media/src/upload_record.rs b/crates/buzz-media/src/upload_record.rs index 7156c871b..4e8010758 100644 --- a/crates/buzz-media/src/upload_record.rs +++ b/crates/buzz-media/src/upload_record.rs @@ -35,9 +35,11 @@ //! artifacts but before the sidecar serve gate. Record existence therefore //! implies the scan inputs are readable, while record failure cannot leave //! unscanned media publicly servable. -//! - `ext`, `mime_type`, and `size` are always present so the consumer can -//! derive the blob key (`{sha256}.{ext}`) and scan eligibility without -//! extra round-trips. +//! - `ext`, `mime_type`, and `size` are always present for backward-compatible +//! consumers; new consumers should prefer `blob_key` (the exact payload key) +//! and only derive `{sha256}.{ext}` for old records that omit it. +//! - `blob_key` uses the active migration layout, so it may be flat legacy or +//! hash-sharded global CAS (`media///.`). //! - `uploader_name`, `ip`, and `port` are omitted (never `null`) when //! unknown or when collection is disabled. //! - Consumers must tolerate unknown fields; `version` bumps only on @@ -290,12 +292,13 @@ mod tests { #[test] fn record_serializes_full_shape() { + let blob_key = format!("media/bb/bb/{}.png", "b".repeat(64)); let record = UploadRecord { version: UPLOAD_RECORD_VERSION, event_id: "01J9W3TEST".into(), sha256: "b".repeat(64), ext: "png".into(), - blob_key: Some(format!("{}.png", "b".repeat(64))), + blob_key: Some(blob_key.clone()), mime_type: "image/png".into(), size: 12345, uploaded_at: 1_783_358_352, @@ -311,7 +314,7 @@ mod tests { assert_eq!(json["version"], 1); assert_eq!(json["ext"], "png"); assert_eq!(json["mime_type"], "image/png"); - assert_eq!(json["blob_key"], format!("{}.png", "b".repeat(64))); + assert_eq!(json["blob_key"], blob_key); assert_eq!(json["size"], 12345); assert_eq!(json["ip"], "203.0.113.7"); assert_eq!(json["port"], 51234); diff --git a/deploy/charts/buzz/README.md b/deploy/charts/buzz/README.md index a1ec26173..c9fa92026 100644 --- a/deploy/charts/buzz/README.md +++ b/deploy/charts/buzz/README.md @@ -317,8 +317,12 @@ For an **existing deployment**: 3. Run the backfill Job. Re-run from any logged checkpoint as needed, then run it again to a clean `copied=0` result. Reconcile storage sweep unknown keys, duplicate gauges, migration failures, and legacy fallback traffic. -4. Select `sharded-only` only after every supported rollback version understands - sharded keys and reconciliation finds no missing sharded destination. +4. Select `sharded-only` only after every supported rollback version and + external consumer understands sharded keys, and reconciliation finds no + missing sharded destination. In particular, buzz-moderation must prefer the + upload-record `blob_key` field (falling back to flat `{sha}.{ext}` for old + records) before `sharded-only` or legacy cleanup; otherwise new scans and + pending report/evidence workflows can continue to look for flat objects. 5. Run the deletion Job with its default `dry-run=true`, review the output, and retain a recovery window/S3 versions. Only then set `dry-run=false` and `BUZZ_MEDIA_DELETE_CONFIRM=delete-verified-legacy-media`. The tool checks the