mirror of
https://github.com/block/buzz.git
synced 2026-08-18 06:50:31 +02:00
fix: channel member search via NIP-50 + Typesense indexer fix for kind:0 (#569)
Signed-off-by: Tyler Longwell <109685178+tlongwell-block@users.noreply.github.com>
This commit is contained in:
tlongwell-block
@@ -11,6 +11,10 @@ description = "WebSocket relay server for the Sprout communications platform"
|
||||
name = "sprout-relay"
|
||||
path = "src/main.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "sprout-reindex-kind0"
|
||||
path = "src/bin/reindex_kind0.rs"
|
||||
|
||||
[dependencies]
|
||||
sprout-core = { workspace = true }
|
||||
sprout-db = { workspace = true }
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
//! One-shot admin tool: re-index all kind:0 (user metadata) events in Typesense.
|
||||
//!
|
||||
//! Necessary after the indexer change that appends `display_name`/`name`/`nip05`
|
||||
//! values to the indexed content for kind:0 docs (see `sprout-search`'s
|
||||
//! `flatten_kind0_for_indexing`). Existing docs need to be rewritten with the
|
||||
//! appended tokens before they become searchable by display name.
|
||||
//!
|
||||
//! New / updated kind:0 events index correctly automatically — this tool only
|
||||
//! exists to backfill the existing population.
|
||||
//!
|
||||
//! Usage (from the repo root, with .env sourced):
|
||||
//!
|
||||
//! ```
|
||||
//! cargo run --release -p sprout-relay --bin sprout-reindex-kind0
|
||||
//! ```
|
||||
//!
|
||||
//! Idempotent — Typesense uses upsert semantics, so running twice is safe.
|
||||
//! Streams in batches so memory stays bounded regardless of relay size.
|
||||
//!
|
||||
//! ## Paging
|
||||
//!
|
||||
//! Walks `query_events` with a snapshot ceiling (`until = now()` at start) plus
|
||||
//! a keyset cursor over `(created_at, id)` matching the underlying
|
||||
//! `ORDER BY created_at DESC, id ASC` index. This guarantees:
|
||||
//!
|
||||
//! - No rows are skipped if new kind:0 events arrive during the run
|
||||
//! (they're newer than the snapshot, so they fall outside the predicate).
|
||||
//! - No rows are double-counted at page boundaries (the cursor advances
|
||||
//! strictly past the last row of each batch).
|
||||
//! - Bounded total work — won't chase its own tail under live write traffic.
|
||||
//!
|
||||
//! Newly-arrived kind:0 events that fall outside the snapshot are indexed by
|
||||
//! the relay's live write path anyway, so this backfill plus the live path
|
||||
//! together cover the full population.
|
||||
|
||||
use anyhow::Context;
|
||||
use chrono::{DateTime, Utc};
|
||||
use tracing::{info, warn};
|
||||
use tracing_subscriber::{fmt, prelude::*, EnvFilter};
|
||||
|
||||
use sprout_db::{Db, DbConfig, EventQuery};
|
||||
use sprout_relay::config::Config;
|
||||
use sprout_search::{SearchConfig, SearchService};
|
||||
|
||||
/// Page size for the SQL → Typesense pipeline. Small enough to keep DB and
|
||||
/// Typesense memory comfortable, large enough to amortise per-batch overhead.
|
||||
const BATCH: i64 = 500;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
tracing_subscriber::registry()
|
||||
.with(fmt::layer())
|
||||
.with(
|
||||
EnvFilter::from_default_env()
|
||||
.add_directive("sprout_reindex_kind0=info".parse()?)
|
||||
.add_directive("sprout_relay=info".parse()?),
|
||||
)
|
||||
.init();
|
||||
|
||||
let config = Config::from_env().context("loading relay config from environment")?;
|
||||
|
||||
let db_config = DbConfig {
|
||||
database_url: config.database_url.clone(),
|
||||
..DbConfig::default()
|
||||
};
|
||||
let db = Db::new(&db_config)
|
||||
.await
|
||||
.context("connecting to postgres")?;
|
||||
|
||||
// SearchConfig::default() reads TYPESENSE_URL / TYPESENSE_API_KEY /
|
||||
// TYPESENSE_COLLECTION from the environment, same as the relay does.
|
||||
let search = SearchService::new(SearchConfig::default());
|
||||
search
|
||||
.ensure_collection()
|
||||
.await
|
||||
.context("ensuring Typesense collection")?;
|
||||
|
||||
// Snapshot ceiling: we only reindex events that already exist at start.
|
||||
// Anything newer is handled by the relay's live indexing path.
|
||||
let snapshot: DateTime<Utc> = Utc::now();
|
||||
|
||||
// Keyset cursor over (created_at, id) — matches the underlying
|
||||
// `ORDER BY created_at DESC, id ASC` index. On the first iteration both
|
||||
// cursor fields are None and the predicate reduces to `created_at <= snapshot`.
|
||||
// Subsequent iterations advance to strictly past the last row of the prior batch.
|
||||
let mut cursor_until: DateTime<Utc> = snapshot;
|
||||
let mut cursor_before_id: Option<Vec<u8>> = None;
|
||||
|
||||
let mut total_indexed: usize = 0;
|
||||
let mut total_failed: usize = 0;
|
||||
let mut batches: usize = 0;
|
||||
|
||||
info!(?snapshot, "starting kind:0 reindex");
|
||||
|
||||
loop {
|
||||
let q = EventQuery {
|
||||
kinds: Some(vec![0]),
|
||||
limit: Some(BATCH),
|
||||
max_limit: Some(BATCH),
|
||||
until: Some(cursor_until),
|
||||
before_id: cursor_before_id.clone(),
|
||||
..EventQuery::default()
|
||||
};
|
||||
|
||||
let batch = db
|
||||
.query_events(&q)
|
||||
.await
|
||||
.context("querying kind:0 events")?;
|
||||
|
||||
if batch.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
let batch_len = batch.len();
|
||||
|
||||
// Capture the tail of the batch for cursor advance *before* the index
|
||||
// call, so we still advance even if indexing fails for this batch.
|
||||
// (We'd otherwise loop forever on a poisoned batch.)
|
||||
let tail = batch
|
||||
.last()
|
||||
.map(|ev| {
|
||||
let ts = ev.event.created_at.as_u64() as i64;
|
||||
let dt = DateTime::<Utc>::from_timestamp(ts, 0).unwrap_or(cursor_until);
|
||||
let id_bytes = ev.event.id.to_bytes().to_vec();
|
||||
(dt, id_bytes)
|
||||
})
|
||||
.expect("batch is non-empty (checked above)");
|
||||
|
||||
match search.index_batch(&batch).await {
|
||||
Ok(indexed) => {
|
||||
total_indexed += indexed;
|
||||
if indexed < batch_len {
|
||||
let failed = batch_len - indexed;
|
||||
total_failed += failed;
|
||||
warn!(failed, batch_len, "some events failed to index in batch");
|
||||
}
|
||||
info!(indexed, batch_len, batches, total_indexed, "indexed batch");
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(error = %e, batch_len, batches, "batch index failed entirely");
|
||||
total_failed += batch_len;
|
||||
}
|
||||
}
|
||||
|
||||
batches += 1;
|
||||
|
||||
// Tail of the prior batch becomes the cursor for the next page.
|
||||
// `query_events` will use the composite predicate
|
||||
// created_at < cursor_until OR (created_at = cursor_until AND id > cursor_before_id)
|
||||
// which exactly skips past the last row we just processed.
|
||||
cursor_until = tail.0;
|
||||
cursor_before_id = Some(tail.1);
|
||||
|
||||
// If we got fewer than BATCH back, we're at the tail of the table.
|
||||
if (batch_len as i64) < BATCH {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
total_indexed,
|
||||
total_failed, batches, "kind:0 reindex complete"
|
||||
);
|
||||
if total_failed > 0 {
|
||||
std::process::exit(1);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -43,9 +43,36 @@ pub fn event_to_document(event: &StoredEvent) -> Result<Value, SearchError> {
|
||||
.map(|id| id.to_string())
|
||||
.unwrap_or_else(|| "__global__".to_string());
|
||||
|
||||
// For kind:0 (user metadata) we append the parsed JSON values to `content`
|
||||
// so Typesense's word-tokenizer can index them cleanly. Without this, a
|
||||
// raw blob like `{"display_name":"alice","about":"loves cats"}` does not
|
||||
// produce a clean `alice` token — the leading `"` glues onto the next
|
||||
// word, so the doc is unreachable for the obvious `q=alice` search.
|
||||
//
|
||||
// We only flatten kind:0 (the structured-metadata kind defined by NIP-01)
|
||||
// and only the small set of fields the member-picker uses. Bio / about /
|
||||
// website are intentionally left out so they don't pollute name-prefix
|
||||
// searches with false positives. Stays consistent with `display_name >
|
||||
// nip05 > pubkey` ranking applied on the desktop side.
|
||||
//
|
||||
// The Typesense `content` field is write-only as far as the relay's read
|
||||
// paths go (the bridge fetches the canonical event from Postgres by id
|
||||
// after Typesense returns hits), so appending derived tokens here doesn't
|
||||
// affect any consumer's view of the event's actual content.
|
||||
//
|
||||
// NOTE: existing kind:0 docs indexed before this change won't have the
|
||||
// appended tokens. Running `just reindex-search` (or the
|
||||
// `sprout-relay reindex-search` admin path) repopulates them. New /
|
||||
// updated profiles get the tokens automatically.
|
||||
let content_indexed = if event_kind_i32(nostr_event) == 0 {
|
||||
flatten_kind0_for_indexing(nostr_event.content.as_str())
|
||||
} else {
|
||||
nostr_event.content.as_str().to_string()
|
||||
};
|
||||
|
||||
let doc = json!({
|
||||
"id": nostr_event.id.to_string(),
|
||||
"content": nostr_event.content.as_str(),
|
||||
"content": content_indexed,
|
||||
// Cast to i32 for Typesense schema (int32 field). nostr Kind is u16; all Sprout kinds fit in i32.
|
||||
"kind": event_kind_i32(nostr_event),
|
||||
"pubkey": nostr_event.pubkey.to_string(),
|
||||
@@ -57,6 +84,38 @@ pub fn event_to_document(event: &StoredEvent) -> Result<Value, SearchError> {
|
||||
Ok(doc)
|
||||
}
|
||||
|
||||
/// For kind:0 events, return the original content with the searchable fields
|
||||
/// (`display_name`, `name`, `nip05`) appended as space-separated plain words.
|
||||
///
|
||||
/// Tolerant of malformed input: anything that fails JSON parsing returns the
|
||||
/// original content unchanged, never an error.
|
||||
fn flatten_kind0_for_indexing(raw_content: &str) -> String {
|
||||
let Ok(parsed) = serde_json::from_str::<Value>(raw_content) else {
|
||||
return raw_content.to_string();
|
||||
};
|
||||
let Some(obj) = parsed.as_object() else {
|
||||
return raw_content.to_string();
|
||||
};
|
||||
|
||||
let mut extracted: Vec<&str> = Vec::with_capacity(3);
|
||||
for key in ["display_name", "name", "nip05"] {
|
||||
if let Some(val) = obj.get(key).and_then(Value::as_str) {
|
||||
let trimmed = val.trim();
|
||||
if !trimmed.is_empty() {
|
||||
extracted.push(trimmed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if extracted.is_empty() {
|
||||
raw_content.to_string()
|
||||
} else {
|
||||
// Single leading space ensures we don't smash the closing `}` of the
|
||||
// original JSON into the first appended token.
|
||||
format!("{} {}", raw_content, extracted.join(" "))
|
||||
}
|
||||
}
|
||||
|
||||
/// Indexes a single event via Typesense upsert.
|
||||
pub async fn index_event(
|
||||
client: &reqwest::Client,
|
||||
@@ -269,6 +328,134 @@ mod tests {
|
||||
assert_eq!(doc["channel_id"].as_str().unwrap(), "__global__");
|
||||
}
|
||||
|
||||
// ── kind:0 flattening for searchability ─────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn kind0_appends_display_name_for_tokenization() {
|
||||
let stored = make_stored_event(
|
||||
r#"{"display_name":"alice","about":"loves cats"}"#,
|
||||
Kind::Metadata,
|
||||
None,
|
||||
);
|
||||
let doc = event_to_document(&stored).unwrap();
|
||||
let content = doc["content"].as_str().unwrap();
|
||||
// Original JSON is preserved (read paths don't depend on this but it
|
||||
// costs nothing and makes debugging the index cheaper).
|
||||
assert!(content.contains(r#""display_name":"alice""#));
|
||||
// The display name is also present as a free-standing token so the
|
||||
// default Typesense tokenizer can index it without the leading-quote
|
||||
// gluing onto the next character.
|
||||
assert!(content.ends_with(" alice"), "got: {content:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kind0_appends_name_when_display_name_absent() {
|
||||
// NIP-01 allows `name` as the canonical display field too.
|
||||
let stored = make_stored_event(r#"{"name":"bob","about":"x"}"#, Kind::Metadata, None);
|
||||
let doc = event_to_document(&stored).unwrap();
|
||||
let content = doc["content"].as_str().unwrap();
|
||||
assert!(content.ends_with(" bob"), "got: {content:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kind0_includes_both_display_name_and_name_when_present() {
|
||||
let stored = make_stored_event(
|
||||
r#"{"display_name":"Alice","name":"alice"}"#,
|
||||
Kind::Metadata,
|
||||
None,
|
||||
);
|
||||
let doc = event_to_document(&stored).unwrap();
|
||||
let content = doc["content"].as_str().unwrap();
|
||||
assert!(content.ends_with(" Alice alice"), "got: {content:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kind0_includes_nip05_in_appended_tokens() {
|
||||
let stored = make_stored_event(
|
||||
r#"{"display_name":"alice","nip05":"alice@example.com"}"#,
|
||||
Kind::Metadata,
|
||||
None,
|
||||
);
|
||||
let doc = event_to_document(&stored).unwrap();
|
||||
let content = doc["content"].as_str().unwrap();
|
||||
assert!(
|
||||
content.ends_with(" alice alice@example.com"),
|
||||
"got: {content:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kind0_excludes_about_and_website_from_appended_tokens() {
|
||||
// `about` and `website` deliberately do not get appended — including
|
||||
// them would cause name-prefix searches to return false positives from
|
||||
// bios. The user's own display_name still appears.
|
||||
let stored = make_stored_event(
|
||||
r#"{"display_name":"alice","about":"I work with bob on x","website":"https://carol.example"}"#,
|
||||
Kind::Metadata,
|
||||
None,
|
||||
);
|
||||
let doc = event_to_document(&stored).unwrap();
|
||||
let content = doc["content"].as_str().unwrap();
|
||||
assert!(content.ends_with(" alice"), "got: {content:?}");
|
||||
// Sanity: the about/website are still in the doc because we preserve
|
||||
// the original JSON — they just don't appear in the trailing tokens.
|
||||
assert!(content.contains("bob"));
|
||||
assert!(content.contains("carol"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kind0_malformed_json_is_passed_through_unchanged() {
|
||||
let stored = make_stored_event("not json at all", Kind::Metadata, None);
|
||||
let doc = event_to_document(&stored).unwrap();
|
||||
assert_eq!(doc["content"].as_str().unwrap(), "not json at all");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kind0_non_object_json_is_passed_through_unchanged() {
|
||||
// Defensive: NIP-01 says content is a JSON object, but a malformed
|
||||
// client could publish e.g. a JSON array. We don't crash, we just
|
||||
// skip the flattening for that doc.
|
||||
let stored = make_stored_event(r#"["nope"]"#, Kind::Metadata, None);
|
||||
let doc = event_to_document(&stored).unwrap();
|
||||
assert_eq!(doc["content"].as_str().unwrap(), r#"["nope"]"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kind0_empty_string_values_skipped() {
|
||||
let stored = make_stored_event(
|
||||
r#"{"display_name":"","name":"alice","nip05":" "}"#,
|
||||
Kind::Metadata,
|
||||
None,
|
||||
);
|
||||
let doc = event_to_document(&stored).unwrap();
|
||||
let content = doc["content"].as_str().unwrap();
|
||||
// Only `name` is non-empty; whitespace-only `nip05` is also skipped.
|
||||
assert!(content.ends_with(" alice"), "got: {content:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kind0_no_searchable_fields_is_passed_through() {
|
||||
// Profile with only fields we don't extract.
|
||||
let stored = make_stored_event(
|
||||
r#"{"about":"just a bio","picture":"https://x"}"#,
|
||||
Kind::Metadata,
|
||||
None,
|
||||
);
|
||||
let doc = event_to_document(&stored).unwrap();
|
||||
let content = doc["content"].as_str().unwrap();
|
||||
// No trailing space-separated tokens added; original content unchanged.
|
||||
assert_eq!(content, r#"{"about":"just a bio","picture":"https://x"}"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_kind0_events_not_flattened() {
|
||||
// kind:1 (note) with a JSON-looking body must be left strictly alone.
|
||||
let json_looking = r#"{"display_name":"alice"}"#;
|
||||
let stored = make_stored_event(json_looking, Kind::TextNote, None);
|
||||
let doc = event_to_document(&stored).unwrap();
|
||||
assert_eq!(doc["content"].as_str().unwrap(), json_looking);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tag_flattening_uses_unit_separator() {
|
||||
let keys = Keys::generate();
|
||||
|
||||
@@ -59,7 +59,21 @@ impl SearchQuery {
|
||||
pub struct SearchHit {
|
||||
/// Hex event ID of the matching event.
|
||||
pub event_id: String,
|
||||
/// Event content text.
|
||||
/// Event content text **as indexed in Typesense** — not necessarily the
|
||||
/// canonical event content.
|
||||
///
|
||||
/// For kind:0 (user metadata) events, `flatten_kind0_for_indexing` in
|
||||
/// `index.rs` appends the parsed `display_name` / `name` / `nip05` values
|
||||
/// to the original JSON content (space-separated) so the default
|
||||
/// tokenizer can produce clean word tokens. That doctored string is what
|
||||
/// lands here.
|
||||
///
|
||||
/// All production read paths (`bridge.rs::handle_bridge_search`,
|
||||
/// `handlers/req.rs` WS REQ) refetch the canonical `StoredEvent` from
|
||||
/// Postgres by `event_id` and ignore this field — which is why the
|
||||
/// append-to-content trick is safe. If you're adding a new feature that
|
||||
/// reads this field directly, do the same: fetch the canonical event by
|
||||
/// id rather than trusting `content` to round-trip.
|
||||
pub content: String,
|
||||
/// Nostr kind number.
|
||||
pub kind: u16,
|
||||
|
||||
@@ -49,7 +49,7 @@ const overrides = new Map([
|
||||
["src-tauri/src/commands/media.rs", 730], // ffmpeg video transcode + poster frame extraction + run_ffmpeg_with_timeout (find_ffmpeg via resolve_command, is_video_file, transcode_to_mp4, extract_poster_frame, transcode_and_extract_poster) + spawn_blocking wrappers + tests
|
||||
["src-tauri/src/commands/agents.rs", 881], // remote agent lifecycle routing (local + provider branches) + scope enforcement + persona pack metadata wiring + mcp_toolsets field + NIP-OA auth_tag in deploy payload
|
||||
["src-tauri/src/commands/messages.rs", 510], // feed multi-query + NIP-50 search + forum thread resolution + thread ref + reactions via REQ
|
||||
["src-tauri/src/nostr_convert.rs", 870], // 12 Nostr event→model converters (channels, profiles, members, notes, search, agents, relay members) + 20 unit tests
|
||||
["src-tauri/src/nostr_convert.rs", 1150], // 12 Nostr event→model converters (channels, profiles, members, notes, search, agents, relay members) + rank_user_search_results helper for NIP-50 user search + 33 unit tests
|
||||
["src-tauri/src/managed_agents/runtime.rs", 990], // ... + respond-to gate env (SPROUT_ACP_RESPOND_TO[_ALLOWLIST]) + per-mode env builder + tests
|
||||
["src-tauri/src/managed_agents/types.rs", 700], // ManagedAgentRecord/Summary + Create/Update request structs + RespondTo enum + validate_respond_to_allowlist + tests
|
||||
["src-tauri/src/managed_agents/backend.rs", 530], // provider IPC, validation, discovery, binary resolution + tests
|
||||
|
||||
@@ -177,51 +177,38 @@ pub async fn search_users(
|
||||
limit: Option<u32>,
|
||||
state: State<'_, AppState>,
|
||||
) -> Result<SearchUsersResponse, String> {
|
||||
let q = query.trim().to_lowercase();
|
||||
let trimmed = query.trim();
|
||||
let max = limit.unwrap_or(8).min(50) as usize;
|
||||
|
||||
if q.is_empty() {
|
||||
if trimmed.is_empty() || max == 0 {
|
||||
return Ok(SearchUsersResponse { users: Vec::new() });
|
||||
}
|
||||
|
||||
// Fetch all kind:0 profiles and filter client-side. The old REST endpoint
|
||||
// used a DB ILIKE query; this is equivalent for small-to-medium relays.
|
||||
// NIP-50 search doesn't work well for user lookup because Typesense indexes
|
||||
// raw JSON content and short names don't tokenize at JSON boundaries.
|
||||
// NIP-50 full-text search on kind:0 profiles. The relay's HTTP bridge
|
||||
// intercepts the `search` field on POST /query and routes to Typesense
|
||||
// (see `crates/sprout-relay/src/api/bridge.rs::handle_bridge_search`),
|
||||
// so we get indexed, server-side search instead of fetching every kind:0
|
||||
// and scanning client-side. The old path was capped at 2000 kind:0 events
|
||||
// by the relay's HTTP bridge limit, which silently hid users on busy relays.
|
||||
//
|
||||
// We over-fetch (limit=50, which the bridge accepts up to 500) and re-rank
|
||||
// locally because Typesense scores BM25 against the whole kind:0 JSON
|
||||
// `content` blob, where a hit in `display_name` is not weighted any higher
|
||||
// than a substring hit in `about`. Re-ranking ≤50 results client-side is
|
||||
// cheap and keeps display ordering predictable for autocomplete.
|
||||
let events = query_relay(
|
||||
&state,
|
||||
&[serde_json::json!({
|
||||
"kinds": [0],
|
||||
"limit": 2000,
|
||||
"search": trimmed,
|
||||
"limit": 50,
|
||||
})],
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut users = Vec::new();
|
||||
for ev in &events {
|
||||
let pubkey_hex = ev.pubkey.to_hex();
|
||||
if let Ok(v) = serde_json::from_str::<serde_json::Value>(&ev.content) {
|
||||
let display_name = v
|
||||
.get("display_name")
|
||||
.and_then(|v| v.as_str())
|
||||
.or_else(|| v.get("name").and_then(|v| v.as_str()))
|
||||
.unwrap_or("");
|
||||
let nip05 = v.get("nip05").and_then(|v| v.as_str()).unwrap_or("");
|
||||
|
||||
let matches = display_name.to_lowercase().contains(&q)
|
||||
|| nip05.to_lowercase().contains(&q)
|
||||
|| pubkey_hex.starts_with(&q);
|
||||
|
||||
if matches {
|
||||
users.push(nostr_convert::user_search_result_from_event(ev));
|
||||
if users.len() >= max {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(SearchUsersResponse { users })
|
||||
Ok(nostr_convert::rank_user_search_results(
|
||||
&events, trimmed, max,
|
||||
))
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
|
||||
@@ -333,6 +333,123 @@ pub fn search_users_from_events(events: &[Event]) -> SearchUsersResponse {
|
||||
SearchUsersResponse { users }
|
||||
}
|
||||
|
||||
/// Rank and truncate kind:0 events from a NIP-50 search response for the
|
||||
/// member-picker / DM-recipient autocomplete.
|
||||
///
|
||||
/// The relay returns results scored by Typesense BM25 against the whole kind:0
|
||||
/// JSON content blob. That ranking is fine as a recall mechanism but not as
|
||||
/// final ordering — a user whose `display_name` *is* the query should always
|
||||
/// rank above someone whose `about` happens to mention it. We re-rank with a
|
||||
/// small deterministic scoring function:
|
||||
///
|
||||
/// - exact match (case-insensitive) > prefix match > substring match
|
||||
/// - field priority: display_name (or name) > nip05 > pubkey hex
|
||||
///
|
||||
/// `limit` clamps the output. Pubkey de-duplication keeps only the
|
||||
/// highest-scoring result per pubkey (Typesense should already return one doc
|
||||
/// per event id, and kind:0 is a NIP-16 replaceable event so stale rows are
|
||||
/// soft-deleted in the DB and filtered out before reaching us — this is
|
||||
/// defense in depth in case both somehow slip through).
|
||||
pub fn rank_user_search_results(
|
||||
events: &[Event],
|
||||
query: &str,
|
||||
limit: usize,
|
||||
) -> SearchUsersResponse {
|
||||
let q = query.trim().to_lowercase();
|
||||
if q.is_empty() || limit == 0 {
|
||||
return SearchUsersResponse { users: Vec::new() };
|
||||
}
|
||||
|
||||
// (score, input_index) — input index is a stable tiebreaker preserving
|
||||
// the relay's relevance order for ties.
|
||||
let mut scored: Vec<(u32, usize, UserSearchResultInfo)> = Vec::with_capacity(events.len());
|
||||
|
||||
for (idx, ev) in events.iter().enumerate() {
|
||||
// Defensive: NIP-50 may return kinds we didn't expect if the relay
|
||||
// doesn't honor the `kinds` filter under search. Skip non-kind:0.
|
||||
if ev.kind.as_u16() != 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let info = user_search_result_from_event(ev);
|
||||
let display = info.display_name.as_deref().unwrap_or("").to_lowercase();
|
||||
let nip05 = info.nip05_handle.as_deref().unwrap_or("").to_lowercase();
|
||||
let pubkey = info.pubkey.to_lowercase();
|
||||
|
||||
let score = match_score(&q, &display, &nip05, &pubkey);
|
||||
if score == 0 {
|
||||
// Typesense returned this as a content-blob match (e.g. `about`).
|
||||
// No useful name/nip05/pubkey hit — drop it from the autocomplete.
|
||||
continue;
|
||||
}
|
||||
scored.push((score, idx, info));
|
||||
}
|
||||
|
||||
// Sort: higher score first; on tie, earlier input index (= higher Typesense relevance) first.
|
||||
scored.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1)));
|
||||
|
||||
// Dedupe by pubkey, keeping the first occurrence (already best-scored).
|
||||
let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
|
||||
let mut users: Vec<UserSearchResultInfo> = Vec::with_capacity(limit.min(scored.len()));
|
||||
for (_, _, info) in scored {
|
||||
if !seen.insert(info.pubkey.clone()) {
|
||||
continue;
|
||||
}
|
||||
users.push(info);
|
||||
if users.len() >= limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
SearchUsersResponse { users }
|
||||
}
|
||||
|
||||
/// Score a user against a lowercased query.
|
||||
///
|
||||
/// Higher is better. Returns 0 if no field matches — caller should drop.
|
||||
fn match_score(q: &str, display_name: &str, nip05: &str, pubkey_hex: &str) -> u32 {
|
||||
// Per-field tier scores. We assign large gaps so a substring match on
|
||||
// display_name still beats an exact match on a lower-priority field —
|
||||
// "the user named alice" is the autocomplete answer to "alice", not
|
||||
// "the user with alice@something.com".
|
||||
const DISPLAY_EXACT: u32 = 1000;
|
||||
const DISPLAY_PREFIX: u32 = 900;
|
||||
const DISPLAY_CONTAINS: u32 = 800;
|
||||
const NIP05_EXACT: u32 = 700;
|
||||
const NIP05_PREFIX: u32 = 600;
|
||||
const NIP05_CONTAINS: u32 = 500;
|
||||
const PUBKEY_PREFIX: u32 = 400;
|
||||
|
||||
let score_field = |field: &str, exact: u32, prefix: u32, contains: u32| -> u32 {
|
||||
if field.is_empty() {
|
||||
0
|
||||
} else if field == q {
|
||||
exact
|
||||
} else if field.starts_with(q) {
|
||||
prefix
|
||||
} else if field.contains(q) {
|
||||
contains
|
||||
} else {
|
||||
0
|
||||
}
|
||||
};
|
||||
|
||||
let display_score = score_field(
|
||||
display_name,
|
||||
DISPLAY_EXACT,
|
||||
DISPLAY_PREFIX,
|
||||
DISPLAY_CONTAINS,
|
||||
);
|
||||
let nip05_score = score_field(nip05, NIP05_EXACT, NIP05_PREFIX, NIP05_CONTAINS);
|
||||
let pubkey_score = if !pubkey_hex.is_empty() && pubkey_hex.starts_with(q) {
|
||||
PUBKEY_PREFIX
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
display_score.max(nip05_score).max(pubkey_score)
|
||||
}
|
||||
|
||||
// ── kind:1 (notes) ──────────────────────────────────────────────────────────
|
||||
|
||||
/// Convert kind:1 events to [`UserNotesResponse`].
|
||||
@@ -738,6 +855,145 @@ mod tests {
|
||||
assert_eq!(r.users[1].display_name.as_deref(), Some("B"));
|
||||
}
|
||||
|
||||
// ── rank_user_search_results ────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn rank_empty_query_returns_empty() {
|
||||
let e = ev(0, r#"{"display_name":"alice"}"#, vec![]);
|
||||
let r = rank_user_search_results(&[e], " ", 10);
|
||||
assert!(r.users.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rank_zero_limit_returns_empty() {
|
||||
let e = ev(0, r#"{"display_name":"alice"}"#, vec![]);
|
||||
let r = rank_user_search_results(&[e], "alice", 0);
|
||||
assert!(r.users.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rank_skips_non_kind_zero_events() {
|
||||
// Even if a buggy relay returns non-kind:0 hits under a kinds:[0] filter,
|
||||
// we shouldn't surface them as users.
|
||||
let e = ev(1, "alice", vec![]);
|
||||
let r = rank_user_search_results(&[e], "alice", 10);
|
||||
assert!(r.users.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rank_skips_results_with_no_name_or_nip05_match() {
|
||||
// Typesense matched the kind:0 because "alice" appears in the `about`
|
||||
// field. We don't want that in autocomplete — there's no useful name
|
||||
// to display against the query.
|
||||
let e = ev(
|
||||
0,
|
||||
r#"{"display_name":"Bob","nip05":"bob@example.com","about":"I work with alice"}"#,
|
||||
vec![],
|
||||
);
|
||||
let r = rank_user_search_results(&[e], "alice", 10);
|
||||
assert!(r.users.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rank_exact_display_name_beats_substring_match() {
|
||||
let exact = ev(0, r#"{"display_name":"alice"}"#, vec![]);
|
||||
let sub = ev(0, r#"{"display_name":"alice-the-second"}"#, vec![]);
|
||||
// Put the substring match first to prove ordering isn't input-driven.
|
||||
let r = rank_user_search_results(&[sub, exact], "alice", 10);
|
||||
assert_eq!(r.users.len(), 2);
|
||||
assert_eq!(r.users[0].display_name.as_deref(), Some("alice"));
|
||||
assert_eq!(r.users[1].display_name.as_deref(), Some("alice-the-second"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rank_display_name_beats_nip05_match_of_same_tier() {
|
||||
// "alice" exact on display_name vs. "alice" prefix on nip05 — display wins.
|
||||
let by_name = ev(0, r#"{"display_name":"alice"}"#, vec![]);
|
||||
let by_nip05 = ev(0, r#"{"display_name":"X","nip05":"alice@x.com"}"#, vec![]);
|
||||
let r = rank_user_search_results(&[by_nip05, by_name], "alice", 10);
|
||||
assert_eq!(r.users[0].display_name.as_deref(), Some("alice"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rank_display_substring_still_beats_nip05_exact() {
|
||||
// Asserts the field-priority gap is large enough that even a display
|
||||
// substring outranks a perfect nip05 hit.
|
||||
let by_name_sub = ev(0, r#"{"display_name":"my-alice-account"}"#, vec![]);
|
||||
let by_nip05_exact = ev(0, r#"{"display_name":"Bob","nip05":"alice"}"#, vec![]);
|
||||
let r = rank_user_search_results(&[by_nip05_exact, by_name_sub], "alice", 10);
|
||||
assert_eq!(r.users[0].display_name.as_deref(), Some("my-alice-account"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rank_pubkey_prefix_match() {
|
||||
// The query is a hex prefix; the kind:0 has no name match at all.
|
||||
// Build the event then read its pubkey hex; use the first 8 chars as
|
||||
// the query so we know the prefix matches.
|
||||
let e = ev(0, r#"{"display_name":"unrelated"}"#, vec![]);
|
||||
let prefix: String = e.pubkey.to_hex().chars().take(8).collect();
|
||||
let r = rank_user_search_results(std::slice::from_ref(&e), &prefix, 10);
|
||||
assert_eq!(r.users.len(), 1);
|
||||
assert_eq!(r.users[0].pubkey, e.pubkey.to_hex());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rank_dedupes_by_pubkey() {
|
||||
// Same author publishing two kind:0s (e.g. one stale row slipping
|
||||
// through). Dedupe should keep only one entry — the higher-scored one.
|
||||
let keys = nostr::Keys::generate();
|
||||
let mk = |content: &str| {
|
||||
EventBuilder::new(Kind::from_u16(0), content)
|
||||
.sign_with_keys(&keys)
|
||||
.expect("sign")
|
||||
};
|
||||
// First event is a substring match on display_name; second is an exact match.
|
||||
let weak = mk(r#"{"display_name":"alice-old"}"#);
|
||||
let strong = mk(r#"{"display_name":"alice"}"#);
|
||||
let r = rank_user_search_results(&[weak, strong], "alice", 10);
|
||||
assert_eq!(r.users.len(), 1);
|
||||
assert_eq!(r.users[0].display_name.as_deref(), Some("alice"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rank_respects_limit() {
|
||||
let events: Vec<Event> = (0..5)
|
||||
.map(|i| ev(0, &format!(r#"{{"display_name":"alice{}"}}"#, i), vec![]))
|
||||
.collect();
|
||||
let r = rank_user_search_results(&events, "alice", 3);
|
||||
assert_eq!(r.users.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rank_case_insensitive() {
|
||||
let e = ev(0, r#"{"display_name":"AlIcE"}"#, vec![]);
|
||||
let r = rank_user_search_results(&[e], "alice", 10);
|
||||
assert_eq!(r.users.len(), 1);
|
||||
let r =
|
||||
rank_user_search_results(&[ev(0, r#"{"display_name":"alice"}"#, vec![])], "ALICE", 10);
|
||||
assert_eq!(r.users.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rank_tiebreak_preserves_relay_relevance_order() {
|
||||
// Two equally-scored matches → first one in input wins.
|
||||
let first = ev(0, r#"{"display_name":"alice"}"#, vec![]);
|
||||
let second = ev(0, r#"{"display_name":"alice"}"#, vec![]);
|
||||
let first_pk = first.pubkey.to_hex();
|
||||
let r = rank_user_search_results(&[first, second], "alice", 10);
|
||||
assert_eq!(r.users.len(), 2);
|
||||
assert_eq!(r.users[0].pubkey, first_pk);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rank_falls_back_to_name_when_display_name_absent() {
|
||||
// user_search_result_from_event already prefers display_name then name —
|
||||
// make sure the ranker sees the populated display_name and scores it.
|
||||
let e = ev(0, r#"{"name":"alice"}"#, vec![]);
|
||||
let r = rank_user_search_results(&[e], "alice", 10);
|
||||
assert_eq!(r.users.len(), 1);
|
||||
assert_eq!(r.users[0].display_name.as_deref(), Some("alice"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_notes_builds_cursor_from_last() {
|
||||
let e1 = ev(1, "first", vec![]);
|
||||
|
||||
@@ -74,7 +74,9 @@ export function ChannelMemberInviteCard({
|
||||
);
|
||||
const userSearchQuery = useUserSearchQuery(deferredInviteQuery, {
|
||||
enabled: open && deferredInviteQuery.length > 0,
|
||||
limit: 8,
|
||||
// Ask for more than we'll display so server-side ranking has room to be
|
||||
// refined client-side. The Tauri command clamps at 50.
|
||||
limit: 25,
|
||||
});
|
||||
const inviteSearchResults = React.useMemo(
|
||||
() =>
|
||||
|
||||
@@ -43,6 +43,13 @@ build:
|
||||
build-release:
|
||||
cargo build --workspace --release
|
||||
|
||||
# Rebuild Typesense docs for all kind:0 (user profile) events.
|
||||
# Required once after deploying the indexer change that flattens kind:0 content
|
||||
# for searchability; new/updated profiles are indexed correctly automatically.
|
||||
# Safe to run repeatedly — Typesense upserts.
|
||||
reindex-kind0:
|
||||
cargo run --release -p sprout-relay --bin sprout-reindex-kind0
|
||||
|
||||
# Run repo lint and formatting checks
|
||||
check: fmt-check clippy desktop-check desktop-tauri-fmt-check web-check mobile-check
|
||||
|
||||
|
||||
Reference in New Issue
Block a user