fix: surface swallowed errors across crates and emit turn failures to activity log (#628)

This commit is contained in:
Will Pfleger
2026-05-20 14:55:20 -04:00
committed by GitHub
parent c9dd97ba1a
commit c8f99bbc31
8 changed files with 109 additions and 21 deletions
+41 -3
View File
@@ -474,7 +474,7 @@ async fn publish_relay_observer_event(
}
};
if let Err(error) = publisher.publish_event(signed).await {
tracing::debug!("relay observer event dropped: {error}");
tracing::warn!("relay observer event dropped: {error}");
}
}
@@ -2032,6 +2032,24 @@ fn handle_prompt_result(
};
let agent_index = result.agent.index;
let channel_id = match &result.source {
PromptSource::Channel(ch) => Some(*ch),
PromptSource::Heartbeat => None,
};
let emit_turn_error = |error_msg: &str| {
if let Some(ref observer) = observer {
observer.emit(
"turn_error",
Some(agent_index),
&observer::context_for(channel_id, None, None),
serde_json::json!({
"outcome": outcome_label,
"error": error_msg,
}),
);
}
};
match result.outcome {
// Successful prompt — return agent to pool.
PromptOutcome::Ok(_) => {
@@ -2044,11 +2062,15 @@ fn handle_prompt_result(
}
// Fatal outcomes: the agent subprocess is dead or poisoned — respawn it.
PromptOutcome::AgentExited | PromptOutcome::Timeout => {
tracing::debug!(
tracing::warn!(
agent = agent_index,
outcome = outcome_label,
"agent_returned — respawning"
);
emit_turn_error(match outcome_label {
"exited" => "Agent process exited unexpectedly",
_ => "Agent timed out",
});
let index = result.agent.index;
let slot_history = &mut crash_history[index];
if !spawn_respawn_task(
@@ -2101,8 +2123,10 @@ fn handle_prompt_result(
tracing::warn!(
agent = agent_index,
outcome = outcome_label,
error = %e,
"transport/protocol error — respawning agent"
);
emit_turn_error(&e.to_string());
let index = result.agent.index;
let slot_history = &mut crash_history[index];
if !spawn_respawn_task(
@@ -2119,11 +2143,13 @@ fn handle_prompt_result(
return LoopAction::Exit;
}
} else {
tracing::debug!(
tracing::warn!(
agent = agent_index,
outcome = outcome_label,
error = %e,
"agent_returned (application error — pipe intact)"
);
emit_turn_error(&e.to_string());
pool.return_agent(result.agent);
}
}
@@ -2178,6 +2204,18 @@ fn recover_panicked_agent(
tracing::warn!("cleared wedged heartbeat_in_flight from panicked agent {i}");
}
if let Some(ref observer) = observer {
observer.emit(
"agent_panic",
Some(i),
&observer::context_for(meta.channel_id, None, None),
serde_json::json!({
"outcome": "panic",
"error": format!("Agent task panicked: {join_error}"),
}),
);
}
// Panics count as crashes for the circuit breaker.
// The panicked task already dropped the AcpClient, so we just need to
// check the circuit and spawn a fresh agent in the background.
+4 -4
View File
@@ -1945,14 +1945,14 @@ pub(crate) async fn reaction_add(rest: &crate::relay::RestClient, event_id: &str
let builder = match sprout_sdk::build_reaction(target_id, emoji) {
Ok(b) => b,
Err(e) => {
tracing::debug!(event_id, emoji, "reaction add: build failed: {e}");
tracing::warn!(event_id, emoji, "reaction add: build failed: {e}");
return;
}
};
let event = match builder.sign_with_keys(&rest.keys) {
Ok(e) => e,
Err(e) => {
tracing::debug!(event_id, emoji, "reaction add: sign failed: {e}");
tracing::warn!(event_id, emoji, "reaction add: sign failed: {e}");
return;
}
};
@@ -2026,14 +2026,14 @@ pub(crate) async fn reaction_remove(rest: &crate::relay::RestClient, event_id: &
let builder = match sprout_sdk::build_remove_reaction(target_id) {
Ok(b) => b,
Err(e) => {
tracing::debug!(event_id, emoji, "reaction remove: build failed: {e}");
tracing::warn!(event_id, emoji, "reaction remove: build failed: {e}");
return;
}
};
let event = match builder.sign_with_keys(&rest.keys) {
Ok(e) => e,
Err(e) => {
tracing::debug!(event_id, emoji, "reaction remove: sign failed: {e}");
tracing::warn!(event_id, emoji, "reaction remove: sign failed: {e}");
return;
}
};
+1 -1
View File
@@ -82,7 +82,7 @@ pub async fn process_upload(
uploaded_at,
)),
Err(e) => {
tracing::warn!(sha256 = %sha256, "metadata generation failed; orphan blob left for GC");
tracing::warn!(sha256 = %sha256, error = %e, "metadata generation failed; orphan blob left for GC");
Err(e)
}
}
+3 -3
View File
@@ -34,7 +34,7 @@ pub(crate) fn not_found(msg: &str) -> (StatusCode, Json<serde_json::Value>) {
/// `git/transport.rs`, and `audio/handler.rs`.
pub mod relay_members {
use axum::{http::StatusCode, response::Json};
use tracing::debug;
use tracing::{debug, info};
use crate::state::AppState;
@@ -95,7 +95,7 @@ pub mod relay_members {
}
}
Err(e) => {
debug!(agent = %pubkey_hex, "NIP-OA auth tag invalid: {e}");
info!(agent = %pubkey_hex, "NIP-OA auth tag invalid: {e}");
}
}
}
@@ -126,7 +126,7 @@ pub mod relay_members {
match sprout_sdk::nip_oa::verify_auth_tag(tag_json, &agent_pubkey) {
Ok(owner) => Some(owner),
Err(e) => {
debug!("extract_nip_oa_owner: invalid auth tag: {e}");
info!("extract_nip_oa_owner: invalid auth tag: {e}");
None
}
}
+2 -2
View File
@@ -118,8 +118,8 @@ pub async fn handle_auth(event: nostr::Event, conn: Arc<ConnectionState>, state:
.await
{
Ok(owner) => owner,
Err(_) => {
warn!(conn_id = %conn_id, pubkey = %pubkey.to_hex(), "not a relay member");
Err(e) => {
warn!(conn_id = %conn_id, pubkey = %pubkey.to_hex(), error = ?e, "not a relay member");
metrics::counter!("sprout_auth_failures_total", "reason" => "not_relay_member")
.increment(1);
*conn.auth_state.write().await = AuthState::Failed;
@@ -1472,10 +1472,16 @@ async fn handle_standard_deletion_event(
0,
)
.unwrap_or_else(chrono::Utc::now);
let _ = state
if let Err(e) = state
.db
.remove_reaction(&react_target_id, react_target_ts, &actor, emoji)
.await;
.await
{
tracing::warn!(
error = %e,
"failed to remove reaction from DB during NIP-09 deletion"
);
}
}
}
}
@@ -1763,7 +1769,7 @@ async fn handle_git_repo_announcement(event: &Event, state: &Arc<AppState>) -> a
("uploadpack.allowTipSHA1InWant", "true"),
("uploadpack.allowReachableSHA1InWant", "true"),
] {
let _ = Command::new("git")
match Command::new("git")
.args(["config", "--file"])
.arg(repo_dir.join("config"))
.args([key, value])
@@ -1773,7 +1779,26 @@ async fn handle_git_repo_announcement(event: &Event, state: &Arc<AppState>) -> a
.env("GIT_CONFIG_GLOBAL", "/dev/null")
.env("HOME", "/dev/null")
.output()
.await;
.await
{
Ok(output) if !output.status.success() => {
tracing::warn!(
key,
value,
status = %output.status,
"git config failed for bare repo"
);
}
Err(e) => {
tracing::warn!(
key,
value,
error = %e,
"git config command failed for bare repo"
);
}
_ => {}
}
}
// Install pre-receive hook for permission enforcement.
@@ -1938,7 +1963,7 @@ pub async fn reconcile_channel_events(state: &Arc<AppState>) -> anyhow::Result<(
for channel in &channels {
// Check if kind:39000 event already exists for this channel.
let channel_id_str = channel.id.to_string();
let existing = state
let existing = match state
.db
.query_events(&EventQuery {
kinds: Some(vec![39000]),
@@ -1947,12 +1972,22 @@ pub async fn reconcile_channel_events(state: &Arc<AppState>) -> anyhow::Result<(
..Default::default()
})
.await
.unwrap_or_default();
{
Ok(v) => v,
Err(e) => {
tracing::warn!(
channel_id = %channel.id,
error = %e,
"reconcile: failed to query existing discovery events"
);
continue;
}
};
if existing.is_empty() {
// No discovery event — emit one.
if let Err(e) = emit_group_discovery_events(state, channel.id).await {
tracing::debug!(
tracing::warn!(
channel_id = %channel.id,
error = %e,
"reconcile: failed to emit discovery events"
+1 -1
View File
@@ -261,7 +261,7 @@ async fn main() -> anyhow::Result<()> {
{
Ok(()) => {}
Err(e) => {
tracing::debug!(error = %e, "channel reconciliation attempt failed");
tracing::warn!(error = %e, "channel reconciliation attempt failed");
}
}
}
@@ -286,6 +286,21 @@ export function processTranscriptEvent(
event.timestamp,
channelId,
);
} else if (event.kind === "turn_error" || event.kind === "agent_panic") {
const payload = asRecord(event.payload);
const outcome = asString(payload.outcome) ?? "error";
const error = asString(payload.error) ?? "Unknown error";
const title =
event.kind === "agent_panic" ? "Agent error (crash)" : "Turn error";
upsertTextItem(
d,
`${event.kind}:${ch}:${event.turnId ?? event.seq}`,
"lifecycle",
title,
`${outcome}: ${error}`,
event.timestamp,
channelId,
);
} else if (event.kind === "acp_read" || event.kind === "acp_write") {
const payload = asRecord(event.payload);
const method = asString(payload.method);