mirror of
https://github.com/block/buzz.git
synced 2026-08-18 06:50:31 +02:00
fix(search): add contains mode for mid-token substring matching
Neither FullText nor Prefix FTS can match a fragment inside a single lexeme: kind-0 profile content is tokenized as whole names, so a member search for "kurs" never finds "mattkursmark". Contains mode matches the raw content with an escaped ILIKE pattern instead of a tsquery, keeping the community fence and the search_tsv allowlist exclusion (gift wraps et al stay invisible). Accepted on the bridge as search_mode="contains". Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -341,6 +341,7 @@ fn extract_search_mode(raw: &Value) -> buzz_search::SearchMode {
|
||||
.and_then(Value::as_str)
|
||||
{
|
||||
Some("prefix") => buzz_search::SearchMode::Prefix,
|
||||
Some("contains") => buzz_search::SearchMode::Contains,
|
||||
_ => buzz_search::SearchMode::FullText,
|
||||
}
|
||||
}
|
||||
@@ -2263,6 +2264,20 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bridge_search_mode_extension_accepts_contains() {
|
||||
assert_eq!(
|
||||
extract_search_mode(
|
||||
&serde_json::json!({ "search": "kurs", "search_mode": "contains" })
|
||||
),
|
||||
buzz_search::SearchMode::Contains
|
||||
);
|
||||
assert_eq!(
|
||||
extract_search_mode(&serde_json::json!({ "search": "kurs", "searchMode": "contains" })),
|
||||
buzz_search::SearchMode::Contains
|
||||
);
|
||||
}
|
||||
|
||||
/// Attack 3 proof: two stateless relay pods sharing Redis must share one
|
||||
/// community-scoped NIP-98 seen-set. Pod A's first claim succeeds; pod B's
|
||||
/// replay of the same event id in the same community is rejected. The same
|
||||
|
||||
@@ -63,6 +63,16 @@ pub enum SearchMode {
|
||||
/// relay still refetches and re-authorizes every hit; this mode changes only
|
||||
/// the candidate tsquery, not the access boundary.
|
||||
Prefix,
|
||||
/// Case-insensitive substring match over the raw `content` (`kurs` matches
|
||||
/// `mattkursmark`).
|
||||
///
|
||||
/// Intended for user-picker typeahead over kind:0 profiles, where names are
|
||||
/// single FTS lexemes that neither `FullText` nor `Prefix` can match
|
||||
/// mid-token. Uses `ILIKE`, not tsquery, so hits carry no `ts_rank_cd`
|
||||
/// relevance (rank is 0; ordering falls back to recency) — picker surfaces
|
||||
/// re-rank client-side. Rows excluded from the search allowlist
|
||||
/// (`search_tsv IS NULL`, e.g. gift wraps) stay excluded.
|
||||
Contains,
|
||||
}
|
||||
|
||||
/// A community-scoped FTS query.
|
||||
@@ -137,8 +147,49 @@ const SEARCH_TEXT_MAX_CHARS: usize = 4096;
|
||||
/// wire untrusted input into a multi-trillion-row OFFSET.
|
||||
const PAGE_MAX: u32 = 1000;
|
||||
|
||||
fn push_tsquery(qb: &mut QueryBuilder<sqlx::Postgres>, mode: SearchMode, search_text: &str) {
|
||||
match mode {
|
||||
/// Escape SQL LIKE metacharacters (`%`, `_`, `\`) so user input is treated
|
||||
/// as literal text. Used with `ESCAPE '\'` in the query.
|
||||
///
|
||||
/// Without this, a search query of `"%"` would match every row and `"_"`
|
||||
/// would act as a single-character wildcard.
|
||||
fn escape_like(input: &str) -> String {
|
||||
input
|
||||
.replace('\\', "\\\\")
|
||||
.replace('%', "\\%")
|
||||
.replace('_', "\\_")
|
||||
}
|
||||
|
||||
/// Push the SELECT head, community fence, and mode-specific match predicate.
|
||||
///
|
||||
/// The two tsquery modes share the `search_tsv @@ query` shape with a
|
||||
/// `ts_rank_cd` relevance score. `Contains` has no tsquery: it matches the raw
|
||||
/// `content` with a LIKE-escaped `ILIKE` pattern and a constant rank, keeping
|
||||
/// `search_tsv IS NOT NULL` so kinds excluded from the search allowlist stay
|
||||
/// invisible to it.
|
||||
fn push_query_head(qb: &mut QueryBuilder<sqlx::Postgres>, query: &SearchQuery, search_text: &str) {
|
||||
if query.mode == SearchMode::Contains {
|
||||
qb.push(
|
||||
"SELECT id, kind, pubkey, channel_id, \
|
||||
EXTRACT(EPOCH FROM created_at)::bigint AS created_at_s, \
|
||||
0::real AS rank \
|
||||
FROM events WHERE community_id = ",
|
||||
);
|
||||
qb.push_bind(*query.community.as_uuid());
|
||||
qb.push(" AND deleted_at IS NULL AND search_tsv IS NOT NULL AND content ILIKE ");
|
||||
qb.push_bind(format!("%{}%", escape_like(search_text)));
|
||||
qb.push(" ESCAPE '\\'");
|
||||
return;
|
||||
}
|
||||
|
||||
qb.push(
|
||||
"SELECT id, kind, pubkey, channel_id, \
|
||||
EXTRACT(EPOCH FROM created_at)::bigint AS created_at_s, \
|
||||
ts_rank_cd(search_tsv, search_query.query) AS rank \
|
||||
FROM events CROSS JOIN LATERAL (SELECT ",
|
||||
);
|
||||
match query.mode {
|
||||
// Returned early above; this arm exists only for match totality.
|
||||
SearchMode::Contains => {}
|
||||
SearchMode::FullText => {
|
||||
qb.push("websearch_to_tsquery('simple', ");
|
||||
qb.push_bind(search_text);
|
||||
@@ -175,6 +226,9 @@ fn push_tsquery(qb: &mut QueryBuilder<sqlx::Postgres>, mode: SearchMode, search_
|
||||
);
|
||||
}
|
||||
}
|
||||
qb.push(" AS query) AS search_query WHERE community_id = ");
|
||||
qb.push_bind(*query.community.as_uuid());
|
||||
qb.push(" AND deleted_at IS NULL AND search_tsv @@ search_query.query");
|
||||
}
|
||||
fn normalized_search_text(q: &str) -> Option<String> {
|
||||
let trimmed = q.trim();
|
||||
@@ -197,7 +251,7 @@ fn normalized_search_text(q: &str) -> Option<String> {
|
||||
|
||||
/// Execute a community-scoped FTS query.
|
||||
///
|
||||
/// SQL shape (always):
|
||||
/// SQL shape (`FullText` and `Prefix`):
|
||||
/// ```sql
|
||||
/// SELECT id, kind, pubkey, channel_id, EXTRACT(EPOCH FROM created_at)::bigint AS created_at_s,
|
||||
/// ts_rank_cd(search_tsv, query) AS rank
|
||||
@@ -211,6 +265,11 @@ fn normalized_search_text(q: &str) -> Option<String> {
|
||||
/// LIMIT $per_page OFFSET (($page - 1) * $per_page)
|
||||
/// ```
|
||||
///
|
||||
/// `Contains` swaps the match arm for `search_tsv IS NOT NULL AND content
|
||||
/// ILIKE '%<escaped>%'` with a constant `0` rank (ordering falls back to
|
||||
/// recency); everything else — community fence, scopes, pagination — is the
|
||||
/// same builder tail.
|
||||
///
|
||||
/// `community_id = $ctx` is the first predicate and is non-negotiable. There
|
||||
/// is no code path through this function that omits it.
|
||||
pub async fn search(pool: &PgPool, query: &SearchQuery) -> Result<SearchResult, SearchError> {
|
||||
@@ -230,16 +289,8 @@ pub async fn search(pool: &PgPool, query: &SearchQuery) -> Result<SearchResult,
|
||||
let page = query.page.clamp(1, PAGE_MAX);
|
||||
let offset = ((page - 1) as i64) * (per_page_actual as i64);
|
||||
|
||||
let mut qb: QueryBuilder<sqlx::Postgres> = QueryBuilder::new(
|
||||
"SELECT id, kind, pubkey, channel_id, \
|
||||
EXTRACT(EPOCH FROM created_at)::bigint AS created_at_s, \
|
||||
ts_rank_cd(search_tsv, search_query.query) AS rank \
|
||||
FROM events CROSS JOIN LATERAL (SELECT ",
|
||||
);
|
||||
push_tsquery(&mut qb, query.mode, &search_text);
|
||||
qb.push(" AS query) AS search_query WHERE community_id = ");
|
||||
qb.push_bind(*query.community.as_uuid());
|
||||
qb.push(" AND deleted_at IS NULL AND search_tsv @@ search_query.query");
|
||||
let mut qb: QueryBuilder<sqlx::Postgres> = QueryBuilder::new("");
|
||||
push_query_head(&mut qb, query, &search_text);
|
||||
|
||||
// Channel scope — see `ChannelScope` doc for the four-case mapping. The
|
||||
// emitted SQL fragments are identical to the legacy 2x2 tuple for the
|
||||
@@ -349,4 +400,13 @@ mod tests {
|
||||
let cleaned = normalized_search_text(&long).expect("non-empty");
|
||||
assert_eq!(cleaned.chars().count(), SEARCH_TEXT_MAX_CHARS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn escape_like_treats_metacharacters_as_literals() {
|
||||
assert_eq!(escape_like("%"), "\\%");
|
||||
assert_eq!(escape_like("a_b"), "a\\_b");
|
||||
assert_eq!(escape_like("a\\b"), "a\\\\b");
|
||||
assert_eq!(escape_like("%_\\"), "\\%\\_\\\\");
|
||||
assert_eq!(escape_like("kurs"), "kurs");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -258,6 +258,135 @@ async fn search_does_not_return_other_community_events() {
|
||||
teardown(pool, &schema).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires Postgres"]
|
||||
async fn contains_mode_matches_mid_token_substring() {
|
||||
// Repro for "Member search should match substrings, not just prefixes":
|
||||
// a member picker query for "kurs" must find the profile whose
|
||||
// display_name is "mattkursmark". The name is a single `simple` lexeme,
|
||||
// so neither FullText nor Prefix can match it mid-token — only Contains.
|
||||
let (pool, schema) = setup().await;
|
||||
|
||||
let c = mk_community(&pool, "sub.example").await;
|
||||
let evt_id = rand_bytes32();
|
||||
insert_event(
|
||||
&pool,
|
||||
c,
|
||||
evt_id,
|
||||
rand_bytes32(),
|
||||
0,
|
||||
r#"{"name":"mattkursmark","display_name":"mattkursmark","about":"hello"}"#,
|
||||
None,
|
||||
1700000000,
|
||||
)
|
||||
.await;
|
||||
|
||||
let svc = SearchService::new(pool.clone());
|
||||
let query = |q: &str, mode: buzz_search::SearchMode| SearchQuery {
|
||||
community: c,
|
||||
q: q.into(),
|
||||
channel_scope: ChannelScope::Any,
|
||||
kinds: Some(vec![0]),
|
||||
authors: None,
|
||||
since: None,
|
||||
until: None,
|
||||
page: 1,
|
||||
per_page: 10,
|
||||
mode,
|
||||
};
|
||||
let contains = buzz_search::SearchMode::Contains;
|
||||
let prefix = buzz_search::SearchMode::Prefix;
|
||||
|
||||
let by_prefix = svc.search(&query("kurs", prefix)).await.expect("search ok");
|
||||
assert_eq!(
|
||||
by_prefix.hits.len(),
|
||||
0,
|
||||
"prefix mode cannot match mid-token — the gap contains mode closes"
|
||||
);
|
||||
|
||||
let by_substring = svc
|
||||
.search(&query("kurs", contains))
|
||||
.await
|
||||
.expect("search ok");
|
||||
assert_eq!(by_substring.hits.len(), 1, "mid-token substring matches");
|
||||
assert_eq!(by_substring.hits[0].event_id, evt_id);
|
||||
|
||||
let case_insensitive = svc
|
||||
.search(&query("KURS", contains))
|
||||
.await
|
||||
.expect("search ok");
|
||||
assert_eq!(case_insensitive.hits.len(), 1, "ILIKE is case-insensitive");
|
||||
|
||||
let wildcard = svc.search(&query("%", contains)).await.expect("search ok");
|
||||
assert_eq!(
|
||||
wildcard.hits.len(),
|
||||
0,
|
||||
"LIKE metacharacters are escaped, not wildcards"
|
||||
);
|
||||
|
||||
teardown(pool, &schema).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires Postgres"]
|
||||
async fn contains_mode_stays_community_scoped_and_respects_search_allowlist() {
|
||||
let (pool, schema) = setup().await;
|
||||
|
||||
let c_a = mk_community(&pool, "contains-a.example").await;
|
||||
let c_b = mk_community(&pool, "contains-b.example").await;
|
||||
insert_event(
|
||||
&pool,
|
||||
c_a,
|
||||
rand_bytes32(),
|
||||
rand_bytes32(),
|
||||
0,
|
||||
r#"{"display_name":"mattkursmark"}"#,
|
||||
None,
|
||||
1700000000,
|
||||
)
|
||||
.await;
|
||||
// Kind 1059 (gift wrap) is excluded from the search allowlist:
|
||||
// `search_tsv` is NULL and contains mode must not see its content.
|
||||
insert_event(
|
||||
&pool,
|
||||
c_a,
|
||||
rand_bytes32(),
|
||||
rand_bytes32(),
|
||||
1059,
|
||||
"wrapped-kursmark-payload",
|
||||
None,
|
||||
1700000000,
|
||||
)
|
||||
.await;
|
||||
|
||||
let svc = SearchService::new(pool.clone());
|
||||
let query = |community: CommunityId, kinds: Option<Vec<i32>>| SearchQuery {
|
||||
community,
|
||||
q: "kurs".into(),
|
||||
channel_scope: ChannelScope::Any,
|
||||
kinds,
|
||||
authors: None,
|
||||
since: None,
|
||||
until: None,
|
||||
page: 1,
|
||||
per_page: 10,
|
||||
mode: buzz_search::SearchMode::Contains,
|
||||
};
|
||||
|
||||
let in_a = svc.search(&query(c_a, None)).await.expect("search ok");
|
||||
assert_eq!(
|
||||
in_a.hits.len(),
|
||||
1,
|
||||
"profile matches; allowlist-excluded gift wrap does not"
|
||||
);
|
||||
assert_eq!(in_a.hits[0].kind, 0);
|
||||
|
||||
let in_b = svc.search(&query(c_b, None)).await.expect("search ok");
|
||||
assert_eq!(in_b.hits.len(), 0, "other community must see nothing");
|
||||
|
||||
teardown(pool, &schema).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires Postgres"]
|
||||
async fn kind0_search_by_display_name_works_without_flattening() {
|
||||
|
||||
Reference in New Issue
Block a user