fix(snapshot): send Buzz shares as PNG avatar cards (#1811)

Signed-off-by: Will Pfleger <pfleger.will@gmail.com>
Co-authored-by: npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7 <dcfd242e557282d7a1e2cf2e6877522682f1e5c6156dc92ca7d90eaedd3b0f95@sprout-oss.stage.blox.sqprod.co>
This commit is contained in:
Will Pfleger
2026-07-13 13:58:24 -04:00
committed by GitHub
co-authored by npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7
parent d41b4c3905
commit a6cfd65511
19 changed files with 558 additions and 247 deletions
@@ -281,7 +281,7 @@ async fn fetch_blob_bytes_with_cap(
pub(crate) enum SnapshotFileKind {
/// `.agent.json` — plaintext JSON; accepts memory; 5 MiB cap.
AgentJson,
/// `.agent.png` — PNG with embedded metadata; no memory; 10 MiB cap.
/// `.agent.png` — PNG with embedded metadata; 10 MiB cap.
AgentPng,
/// `.team.json` — team template; 25 MiB cap.
TeamJson,
@@ -443,9 +443,8 @@ pub async fn fetch_snapshot_bytes(
return Err("hash mismatch: fetched bytes do not match the declared SHA-256".to_string());
}
// 3. Byte magic must match the expected kind (filename → format).
// This prevents .agent.png delivering JSON bytes (including memory-bearing
// JSON) and .agent.json delivering PNG bytes.
// 3. Byte magic must match the expected kind (filename → format), so a
// filename cannot select a JSON or PNG parser for the other format.
ensure_bytes_match_kind(&bytes, kind)?;
// 4. Bytes must parse as the snapshot type selected by the filename.
@@ -173,7 +173,6 @@ fn parse_format_is_png(s: &str) -> Result<bool, String> {
///
/// **Invariants preserved (identical for both callers):**
/// - JSON/PNG format selection from the parsed `is_png` flag (magic-byte sniffing is import-only)
/// - PNG + memory hard rejection
/// - Memory-source pubkey validation
/// - Secret exclusion (env_vars never enter the manifest via `build_snapshot`)
/// - Output filename derived from the agent display name
@@ -182,18 +181,10 @@ pub(crate) async fn materialize_snapshot_bytes(
memory_source_pubkey: Option<String>,
memory_level: MemoryLevel,
is_png: bool,
avatar_png_data_url: Option<String>,
app: AppHandle,
state: State<'_, AppState>,
) -> Result<SnapshotPayload, String> {
// Eagerly reject PNG + memory — avoid an unnecessary relay round-trip.
if is_png && memory_level != MemoryLevel::None {
return Err(
"Cannot export memory to .agent.png — use JSON format for memory-bearing \
snapshots."
.to_string(),
);
}
// ── Load definition record and memory-source instance under lock ─────────
let (record, memory_pubkey) = {
let _store_guard = state
@@ -274,7 +265,9 @@ pub(crate) async fn materialize_snapshot_bytes(
let slug = crate::util::slugify(&display_name, "agent", 50);
if is_png {
let png_bytes = encode_snapshot_png(&snapshot, avatar_bytes.as_deref())
let png_body_avatar_bytes =
resolve_png_body_avatar_bytes(avatar_png_data_url.as_deref(), avatar_bytes);
let png_bytes = encode_snapshot_png(&snapshot, png_body_avatar_bytes.as_deref())
.map_err(|e| format!("Failed to encode .agent.png: {e}"))?;
validate_snapshot_encode_size(png_bytes.len(), true)?;
Ok(SnapshotPayload {
@@ -292,6 +285,17 @@ pub(crate) async fn materialize_snapshot_bytes(
}
}
/// Choose bytes for the PNG image body without changing the source avatar the
/// manifest preserves for import.
fn resolve_png_body_avatar_bytes(
avatar_png_data_url: Option<&str>,
store_avatar_bytes: Option<Vec<u8>>,
) -> Option<Vec<u8>> {
avatar_png_data_url
.and_then(crate::managed_agents::agent_snapshot::decode_avatar_data_url)
.or(store_avatar_bytes)
}
/// Export an agent definition as a `buzz-agent-snapshot v1` file.
///
/// `id` is a definition slug or a keyed-instance pubkey.
@@ -309,6 +313,7 @@ pub async fn export_agent_snapshot(
memory_source_pubkey: Option<String>,
memory_level: String,
format: String,
avatar_png_data_url: Option<String>,
app: AppHandle,
state: State<'_, AppState>,
) -> Result<bool, String> {
@@ -320,6 +325,7 @@ pub async fn export_agent_snapshot(
memory_source_pubkey,
memory_level,
is_png,
avatar_png_data_url,
app.clone(),
state,
)
@@ -370,15 +376,23 @@ pub async fn encode_agent_snapshot_for_send(
memory_source_pubkey: Option<String>,
memory_level: String,
format: String,
avatar_png_data_url: Option<String>,
app: AppHandle,
state: State<'_, AppState>,
) -> Result<EncodedSnapshotPayload, String> {
let memory_level = parse_memory_level(&memory_level)?;
let is_png = parse_format_is_png(&format)?;
let payload =
materialize_snapshot_bytes(id, memory_source_pubkey, memory_level, is_png, app, state)
.await?;
let payload = materialize_snapshot_bytes(
id,
memory_source_pubkey,
memory_level,
is_png,
avatar_png_data_url,
app,
state,
)
.await?;
Ok(EncodedSnapshotPayload {
file_bytes: payload.bytes,
@@ -388,3 +402,66 @@ pub async fn encode_agent_snapshot_for_send(
#[cfg(test)]
mod tests;
#[cfg(test)]
mod png_body_tests {
use super::*;
use base64::Engine as _;
use png::Decoder;
#[test]
fn frontend_raster_becomes_png_body_without_replacing_manifest_source() {
let avatar = image::DynamicImage::ImageRgb8(image::RgbImage::from_pixel(
3,
2,
image::Rgb([0x12, 0x34, 0x56]),
));
let mut source_png = Vec::new();
avatar
.write_to(
&mut std::io::Cursor::new(&mut source_png),
image::ImageFormat::Png,
)
.unwrap();
let avatar_data_url = format!(
"data:image/png;base64,{}",
base64::engine::general_purpose::STANDARD.encode(&source_png)
);
let snapshot = crate::managed_agents::agent_snapshot::AgentSnapshot {
format: crate::managed_agents::agent_snapshot::FORMAT_DISCRIMINATOR.to_string(),
version: crate::managed_agents::agent_snapshot::FORMAT_VERSION,
definition: crate::managed_agents::agent_snapshot::AgentSnapshotDefinition {
name: "Agent".to_string(),
system_prompt: None,
runtime: None,
model: None,
provider: None,
parallelism: None,
respond_to: None,
respond_to_allowlist: vec![],
name_pool: vec![],
idle_timeout_seconds: None,
max_turn_duration_seconds: None,
},
profile: crate::managed_agents::agent_snapshot::AgentSnapshotProfile {
display_name: "Agent".to_string(),
about: None,
avatar_data_url: None,
avatar_url: Some("https://relay.example/media/avatar.png".to_string()),
},
memory: crate::managed_agents::agent_snapshot::AgentSnapshotMemory {
level: MemoryLevel::None,
entries: vec![],
},
};
let body_bytes = resolve_png_body_avatar_bytes(Some(&avatar_data_url), None);
let png_bytes = encode_snapshot_png(&snapshot, body_bytes.as_deref()).unwrap();
let reader = Decoder::new(std::io::Cursor::new(&png_bytes))
.read_info()
.unwrap();
let decoded = import::decode_snapshot_from_bytes(&png_bytes).unwrap();
assert_eq!((reader.info().width, reader.info().height), (3, 2));
assert_eq!(decoded.profile.avatar_url, snapshot.profile.avatar_url);
}
}
@@ -197,9 +197,9 @@ const PNG_MAGIC: [u8; 4] = [0x89, 0x50, 0x4e, 0x47];
/// Fails closed on malformed content, wrong format, or unsupported version.
/// Never trusts the file extension — only the bytes.
///
/// **PNG memory policy:** any PNG manifest whose `memory.level` is not `None`,
/// or whose `memory.entries` is non-empty despite `level == None`, is rejected
/// before any write. Plaintext memory is accepted only from `.agent.json`.
/// **Memory consistency:** any manifest whose `memory.entries` is non-empty
/// despite `memory.level == None` is rejected before any write, regardless of
/// the enclosing format.
///
/// **Size cap:** PNG inputs over 10 MiB and JSON inputs over 5 MiB are rejected
/// before allocation to avoid avoidable large-input work.
@@ -214,18 +214,9 @@ pub(crate) fn decode_snapshot_from_bytes(
));
}
let snapshot = decode_snapshot_png(file_bytes)?;
// Hard reject: PNG must never carry memory.
if snapshot.memory.level != crate::managed_agents::agent_snapshot::MemoryLevel::None {
if snapshot.memory.level == MemoryLevel::None && !snapshot.memory.entries.is_empty() {
return Err(
"Cannot import a memory-bearing .agent.png — use .agent.json for \
snapshots that include memory."
.to_string(),
);
}
if !snapshot.memory.entries.is_empty() {
return Err(
"Snapshot is malformed: .agent.png carries memory entries despite \
memory.level being 'none'."
"Snapshot is malformed: memory.level is 'none' but entries are present."
.to_string(),
);
}
@@ -368,80 +368,71 @@ fn import_avatar_url_fallback_is_used_when_no_data_url() {
);
}
// ── Import: PNG memory policy ─────────────────────────────────────────────
// ── Import: PNG memory parity ─────────────────────────────────────────────
/// PNG with memory level `core` is rejected before any write.
/// PNG with core memory round-trips through the production import decoder.
#[test]
fn import_png_with_core_memory_is_rejected() {
// encode_snapshot_png refuses to encode memory-bearing snapshots —
// that guard on the EXPORT side is correct. On the IMPORT side,
// decode_snapshot_from_bytes adds a matching guard so that a
// foreign-built PNG carrying a core/everything memory level is also
// rejected. We verify the guard condition here.
let level = MemoryLevel::Core;
let entries = vec![AgentSnapshotMemoryEntry {
slug: "core".to_string(),
body: "Secret.".to_string(),
}];
// Simulate what decode_snapshot_from_bytes checks after PNG decode:
let would_reject = level != MemoryLevel::None;
assert!(
would_reject,
"PNG with core memory level must be rejected by the import guard"
);
// Also verify that the encoder itself refuses this on the export side.
fn import_png_with_core_memory_preserves_entries() {
use crate::managed_agents::agent_snapshot::encode_snapshot_png;
let snapshot = make_snapshot(MemoryLevel::Core, entries);
assert!(
encode_snapshot_png(&snapshot, None).is_err(),
"encode_snapshot_png must also refuse memory-bearing snapshots"
let snapshot = make_snapshot(
MemoryLevel::Core,
vec![AgentSnapshotMemoryEntry {
slug: "core".to_string(),
body: "Remember this.".to_string(),
}],
);
let png_bytes = encode_snapshot_png(&snapshot, None).unwrap();
let decoded = decode_snapshot_from_bytes(&png_bytes).unwrap();
assert_eq!(decoded.memory, snapshot.memory);
}
/// PNG with memory level `everything` is rejected before any write.
/// PNG with all memory round-trips through the production import decoder.
#[test]
fn import_png_with_everything_memory_is_rejected() {
// Same as core: both the encoder guard and the decoder guard fire.
let level = MemoryLevel::Everything;
let would_reject = level != MemoryLevel::None;
assert!(
would_reject,
"PNG with everything memory level must be rejected by the import guard"
);
fn import_png_with_everything_memory_preserves_entries() {
use crate::managed_agents::agent_snapshot::encode_snapshot_png;
let snapshot = make_snapshot(
MemoryLevel::Everything,
vec![AgentSnapshotMemoryEntry {
slug: "mem/research".to_string(),
body: "Notes.".to_string(),
}],
);
assert!(
encode_snapshot_png(&snapshot, None).is_err(),
"encode_snapshot_png must also refuse memory-bearing snapshots"
vec![
AgentSnapshotMemoryEntry {
slug: "core".to_string(),
body: "Remember this.".to_string(),
},
AgentSnapshotMemoryEntry {
slug: "mem/research".to_string(),
body: "Notes.".to_string(),
},
],
);
let png_bytes = encode_snapshot_png(&snapshot, None).unwrap();
let decoded = decode_snapshot_from_bytes(&png_bytes).unwrap();
assert_eq!(decoded.memory, snapshot.memory);
}
/// PNG with `none` level but non-empty entries is rejected (malformed).
/// We test this by verifying the guard condition directly, since
/// encode_snapshot_png correctly refuses to produce such a PNG.
/// PNG with `none` level but non-empty entries is rejected as malformed.
#[test]
fn import_png_with_none_level_and_entries_is_rejected() {
// This guard is enforced in decode_snapshot_from_bytes after the PNG
// path resolves. A PNG with none + entries cannot be produced by
// encode_snapshot_png (it already guards that), but could arrive from
// a foreign tool. We verify the guard condition matches the spec:
let level = MemoryLevel::None;
let entries = [AgentSnapshotMemoryEntry {
slug: "core".to_string(),
body: "Leak.".to_string(),
}];
// The guard in decode_snapshot_from_bytes:
let would_reject = !entries.is_empty() && level == MemoryLevel::None;
assert!(
would_reject,
"none level + non-empty entries must trigger the guard"
use crate::managed_agents::agent_snapshot::{
encode_snapshot_json, encode_snapshot_png, make_png_with_text, PNG_CHUNK_KEYWORD,
};
use base64::{engine::general_purpose::STANDARD, Engine as _};
let snapshot = make_snapshot(
MemoryLevel::None,
vec![AgentSnapshotMemoryEntry {
slug: "core".to_string(),
body: "Leak.".to_string(),
}],
);
let json = encode_snapshot_json(&snapshot).unwrap();
let png_bytes = make_png_with_text(PNG_CHUNK_KEYWORD, &STANDARD.encode(json)).unwrap();
assert!(encode_snapshot_png(&snapshot, None).is_err());
let error = decode_snapshot_from_bytes(&png_bytes).unwrap_err();
assert!(error.contains("'none' but entries are present"));
}
/// PNG with `none` level and no entries imports normally.
@@ -7,10 +7,12 @@
//! - **memory** — optional, owner-decrypted engrams at one of three levels
//!
//! Two encodings are supported:
//! - `.agent.json` — canonical, may carry memory at any level
//! - `.agent.json` — canonical snapshot manifest
//! - `.agent.png` — avatar image with manifest in a `buzz_agent_snapshot`
//! tEXt chunk; memory MUST be `None` (PNG files are casually shared and
//! would leak plaintext memory into chat uploads)
//! tEXt chunk
//!
//! Both formats may carry memory at any level. Memory entries are plaintext,
//! so callers must require an explicit opt-in before exporting them.
//!
//! **Zip is NOT in v1** — deferred to v2 for skills bundling.
//!
@@ -284,17 +286,13 @@ pub fn decode_snapshot_json(bytes: &[u8]) -> Result<AgentSnapshot, String> {
/// Encode a snapshot into a `.agent.png` — avatar as the image body, manifest
/// in the `buzz_agent_snapshot` tEXt chunk.
///
/// **Rejects** any snapshot whose `memory.level != None` — memory in a PNG
/// file is a security hazard (images are casually shared, pasted into chats).
pub fn encode_snapshot_png(
snapshot: &AgentSnapshot,
avatar_bytes: Option<&[u8]>,
) -> Result<Vec<u8>, String> {
if snapshot.memory.level != MemoryLevel::None || !snapshot.memory.entries.is_empty() {
if snapshot.memory.level == MemoryLevel::None && !snapshot.memory.entries.is_empty() {
return Err(
"Cannot write memory to a .agent.png file — use .agent.json for memory-bearing \
snapshots. PNG images are casually shared and would expose memory as plaintext."
"Cannot write a snapshot with memory.level 'none' and non-empty memory entries."
.to_string(),
);
}
@@ -303,18 +301,23 @@ pub fn encode_snapshot_png(
let json_bytes = encode_snapshot_json(snapshot)?;
let chunk_text = STANDARD.encode(&json_bytes);
// Use the avatar bytes as the PNG image when available; otherwise produce
// a minimal 1×1 transparent placeholder.
let png_bytes = match avatar_bytes.filter(|b| !b.is_empty()) {
Some(bytes) if bytes.starts_with(b"\x89PNG") => {
// Already a PNG — inject the tEXt chunk.
inject_text_chunk(bytes, PNG_CHUNK_KEYWORD, &chunk_text)?
}
Some(_bytes) => {
// Non-PNG avatar (JPEG, etc.) — re-encode as 1×1 placeholder and
// carry the avatar via the manifest's `profile.avatar_data_url`.
// This keeps the PNG valid while preserving the avatar in the JSON.
make_png_with_text(PNG_CHUNK_KEYWORD, &chunk_text)?
// Use the avatar as the PNG image body, transcoding decodable non-PNG
// avatars. Fall back to a minimal 1×1 transparent placeholder only when
// there is no avatar or it cannot be decoded.
let png_bytes = match avatar_bytes.filter(|bytes| !bytes.is_empty()) {
Some(bytes) => {
let encoded_avatar = if bytes.starts_with(b"\x89PNG") {
inject_text_chunk(bytes, PNG_CHUNK_KEYWORD, &chunk_text).or_else(|_| {
transcode_avatar_to_png_with_text(bytes, PNG_CHUNK_KEYWORD, &chunk_text)
})
} else {
transcode_avatar_to_png_with_text(bytes, PNG_CHUNK_KEYWORD, &chunk_text)
};
match encoded_avatar {
Ok(png_bytes) => png_bytes,
Err(_) => make_png_with_text(PNG_CHUNK_KEYWORD, &chunk_text)?,
}
}
None => make_png_with_text(PNG_CHUNK_KEYWORD, &chunk_text)?,
};
@@ -403,6 +406,21 @@ pub(crate) fn make_png_with_text(keyword: &str, text: &str) -> Result<Vec<u8>, S
Ok(buf)
}
/// Transcode a decodable avatar to PNG and add the snapshot manifest chunk.
fn transcode_avatar_to_png_with_text(
avatar_bytes: &[u8],
keyword: &str,
text: &str,
) -> Result<Vec<u8>, String> {
let image = image::load_from_memory(avatar_bytes)
.map_err(|e| format!("Failed to decode avatar image: {e}"))?;
let mut png_bytes = Vec::new();
image
.write_to(&mut Cursor::new(&mut png_bytes), image::ImageFormat::Png)
.map_err(|e| format!("Failed to encode avatar as PNG: {e}"))?;
inject_text_chunk(&png_bytes, keyword, text)
}
/// Inject a tEXt chunk into an existing PNG by re-encoding it.
///
/// Re-decodes the image data via the `png` crate and writes a fresh PNG with
@@ -576,35 +594,67 @@ mod tests {
assert_eq!(parsed.definition.name, snapshot.definition.name);
}
// ── PNG memory guard ──────────────────────────────────────────────────────
#[test]
fn png_snapshot_transcodes_jpeg_avatar_into_image_body() {
let avatar = image::DynamicImage::ImageRgb8(image::RgbImage::from_pixel(
3,
2,
image::Rgb([0x12, 0x34, 0x56]),
));
let mut jpeg_bytes = Vec::new();
avatar
.write_to(&mut Cursor::new(&mut jpeg_bytes), image::ImageFormat::Jpeg)
.unwrap();
let snapshot = build_snapshot(
&minimal_record(),
MemoryLevel::None,
vec![],
Some(&jpeg_bytes),
);
let png_bytes = encode_snapshot_png(&snapshot, Some(&jpeg_bytes)).unwrap();
let decoder = Decoder::new(Cursor::new(png_bytes));
let reader = decoder.read_info().unwrap();
assert_eq!((reader.info().width, reader.info().height), (3, 2));
}
// ── PNG memory parity ─────────────────────────────────────────────────────
#[test]
fn png_export_with_core_memory_is_rejected() {
fn png_round_trip_with_core_memory() {
let record = minimal_record();
let entries = vec![AgentSnapshotMemoryEntry {
slug: "core".to_string(),
body: "secret memory".to_string(),
body: "remember this".to_string(),
}];
let snapshot = build_snapshot(&record, MemoryLevel::Core, entries, None);
let result = encode_snapshot_png(&snapshot, None);
assert!(result.is_err(), "Expected error for PNG with memory");
let err = result.unwrap_err();
assert!(
err.contains("Cannot write memory to a .agent.png"),
"Error message should explain the PNG memory restriction, got: {err}"
);
let png_bytes = encode_snapshot_png(&snapshot, None).unwrap();
let parsed = decode_snapshot_png(&png_bytes).unwrap();
assert_eq!(parsed.memory, snapshot.memory);
}
#[test]
fn png_export_with_everything_memory_is_rejected() {
fn png_round_trip_with_everything_memory() {
let record = minimal_record();
let entries = vec![AgentSnapshotMemoryEntry {
slug: "mem/notes".to_string(),
body: "private notes".to_string(),
}];
let entries = vec![
AgentSnapshotMemoryEntry {
slug: "core".to_string(),
body: "remember this".to_string(),
},
AgentSnapshotMemoryEntry {
slug: "mem/notes".to_string(),
body: "private notes".to_string(),
},
];
let snapshot = build_snapshot(&record, MemoryLevel::Everything, entries, None);
let result = encode_snapshot_png(&snapshot, None);
assert!(result.is_err());
let png_bytes = encode_snapshot_png(&snapshot, None).unwrap();
let parsed = decode_snapshot_png(&png_bytes).unwrap();
assert_eq!(parsed.memory, snapshot.memory);
}
#[test]
@@ -632,8 +682,10 @@ mod tests {
"PNG encoder must reject level=None with non-empty entries"
);
assert!(
result.unwrap_err().contains("Cannot write memory"),
"Error must explain the PNG memory restriction"
result
.unwrap_err()
.contains("memory.level 'none' and non-empty memory entries"),
"Error must explain the malformed memory state"
);
}