+21

crates/trawler-core/src/index.rs

Reviewed

··· 306 306 pub fn backlinks_of(&self, target: &NodeId) -> HashSet<TreeID> { 307 307 self.backlinks.get(target).cloned().unwrap_or_default() 308 308 } 309 309 + 310 310 + /// Resolve a user-typed name (as it would appear in a query's `(ref 311 311 + /// "name")` primitive, task 4.2) to the `NodeId` it refers to, using 312 312 + /// the same page-name-first-else-tag rule as inline reference parsing. 313 313 + pub fn resolve_name(&self, name: &str) -> NodeId { 314 314 + match self.pages_by_name.get(name) { 315 315 + Some(&id) => NodeId::tree(id), 316 316 + None => NodeId::tag(name), 317 317 + } 318 318 + } 319 319 + 320 320 + /// Every block id the index currently knows about (roots plus every 321 321 + /// value reachable through `children`) — the "universe" a query's 322 322 + /// `not` primitive complements against. 323 323 + pub fn all_blocks(&self) -> HashSet<TreeID> { 324 324 + let mut all: HashSet<TreeID> = self.roots.iter().copied().collect(); 325 325 + for kids in self.children.values() { 326 326 + all.extend(kids.iter().copied()); 327 327 + } 328 328 + all 329 329 + } 309 330 } 310 331 311 332 #[cfg(test)]

+1

crates/trawler-core/src/lib.rs

Reviewed

··· 4 4 pub mod index; 5 5 pub mod outline; 6 6 pub mod properties; 7 7 + pub mod query; 7 8 pub mod query_spike; 8 9 pub mod refs; 9 10 pub mod search;

+814

crates/trawler-core/src/query.rs

Reviewed

··· 1 1 + //! The Steel (Scheme) query engine: a read-only, capability-scoped VM that 2 2 + //! evaluates query-block expressions against the graph and search index. 3 3 + //! 4 4 + //! See openspec/changes/trawler-mvp specs/steel-queries/spec.md and 5 5 + //! design.md D4 ("Scheme composes, Rust iterates"). Containment pattern is 6 6 + //! the one proven in Spike S2 (`crate::query_spike`): each query gets a 7 7 + //! fresh `Engine` on its own spawned thread, interrupted via 8 8 + //! `ThreadStateController` if it exceeds its time budget. 9 9 + //! 10 10 + //! ## Why sets are Scheme lists of id strings, not a native hash-set type 11 11 + //! 12 12 + //! Steel's `HashSet<K>: IntoSteelVal` maps to `SteelVal::HashSetV`, a 13 13 + //! distinct runtime type from lists. Using it would mean either exposing a 14 14 + //! second, hash-set-specific iteration API to query authors, or hand-rolling 15 15 + //! hash-set-walking primitives for the plain-Scheme `filter`/`map` prelude 16 16 + //! below. Representing sets as sorted `Vec<String>` (`SteelVal::ListV`) 17 17 + //! keeps exactly one collection shape in the query language: query authors 18 18 + //! get ordinary `car`/`cdr`/`filter`/`map` over query-primitive results for 19 19 + //! free, and `and`/`or`/`not` still do their real work as native Rust 20 20 + //! `HashSet` operations — the list conversion at the boundary is O(n) 21 21 + //! either way, so this loses nothing set-algebra-wise (task 4.2's "native, 22 22 + //! not per-block Scheme iteration" requirement is about how `and`/`or`/`not` 23 23 + //! themselves execute, not the wire representation of their arguments). 24 24 + 25 25 + use std::sync::mpsc; 26 26 + use std::sync::Arc; 27 27 + use std::thread; 28 28 + use std::time::Duration; 29 29 + 30 30 + use loro::TreeID; 31 31 + use steel::rvals::{FromSteelVal, IntoSteelVal}; 32 32 + use steel::steel_vm::engine::Engine; 33 33 + use steel::steel_vm::register_fn::RegisterFn; 34 34 + use steel::{SteelErr, SteelVal}; 35 35 + 36 36 + use crate::graph::{NodeId, PropertyValue}; 37 37 + use crate::index::GraphIndex; 38 38 + use crate::search::SearchIndex; 39 39 + 40 40 + /// A query's result: either an outline slice (block ids, rendered as a 41 41 + /// navigable list) or a table (property projection, rendered with sortable 42 42 + /// columns) — spec: "Result shapes". 43 43 + #[derive(Debug, Clone, PartialEq)] 44 44 + pub enum QueryValue { 45 45 + Blocks(Vec<TreeID>), 46 46 + Table { 47 47 + columns: Vec<String>, 48 48 + rows: Vec<Vec<String>>, 49 49 + }, 50 50 + } 51 51 + 52 52 + /// An evaluation error with position info where Steel provides it (spec: 53 53 + /// "inline display of evaluation errors with position information"). 54 54 + #[derive(Debug, Clone, PartialEq)] 55 55 + pub struct QueryError { 56 56 + pub message: String, 57 57 + /// Byte offsets into the query source, if Steel attached a span. 58 58 + pub span: Option<(usize, usize)>, 59 59 + } 60 60 + 61 61 + #[derive(Debug, Clone, PartialEq)] 62 62 + pub enum QueryOutcome { 63 63 + Completed(QueryValue), 64 64 + /// The query exceeded its time budget and was interrupted (spec: 65 65 + /// "Runaway query containment"). 66 66 + TimedOut, 67 67 + Errored(QueryError), 68 68 + } 69 69 + 70 70 + /// Read-only Scheme prelude available to every query: `filter`/`map`/`not` 71 71 + /// written in pure Scheme over cons lists, so per-block predicate 72 72 + /// application (task 4.3, spec: "Scheme predicates over narrowed sets") 73 73 + /// needs no native Rust<->Steel closure callback machinery — it's just 74 74 + /// ordinary recursive Scheme running inside the same sandboxed engine. 75 75 + const QUERY_PRELUDE: &str = r#" 76 76 + (define (filter pred lst) 77 77 + (if (null? lst) 78 78 + '() 79 79 + (if (pred (car lst)) 80 80 + (cons (car lst) (filter pred (cdr lst))) 81 81 + (filter pred (cdr lst))))) 82 82 + (define (map f lst) 83 83 + (if (null? lst) 84 84 + '() 85 85 + (cons (f (car lst)) (map f (cdr lst))))) 86 86 + (define (length lst) 87 87 + (if (null? lst) 0 (+ 1 (length (cdr lst))))) 88 88 + ;; Named `bool-not`, not `not`: `not` is reserved for the set-algebra 89 89 + ;; complement primitive (task 4.2) registered after this prelude runs, and 90 90 + ;; would otherwise shadow this boolean version (both bind the same 91 91 + ;; top-level identifier). 92 92 + (define (bool-not x) (if x #f #t)) 93 93 + "#; 94 94 + 95 95 + /// Evaluate `expr` against a read-only snapshot of the graph and search 96 96 + /// index, terminating it if it exceeds `budget`. 97 97 + /// 98 98 + /// Containment: the engine (not `Send`, per Spike S2) is built entirely on 99 99 + /// a spawned worker thread; only its `ThreadStateController` — a pair of 100 100 + /// `Arc`-wrapped atomics — crosses back to the caller, which uses it to 101 101 + /// request an interrupt if the result doesn't arrive in time. The worker 102 102 + /// thread is then abandoned rather than joined, so a query that somehow 103 103 + /// evades the cooperative interrupt still can't block the caller. 104 104 + pub fn eval_query( 105 105 + expr: &str, 106 106 + graph: Arc<GraphIndex>, 107 107 + search: Arc<SearchIndex>, 108 108 + budget: Duration, 109 109 + ) -> QueryOutcome { 110 110 + let (controller_tx, controller_rx) = mpsc::channel(); 111 111 + let (result_tx, result_rx) = mpsc::channel(); 112 112 + let expr = expr.to_string(); 113 113 + 114 114 + thread::spawn(move || { 115 115 + let mut engine = build_engine(graph, search); 116 116 + let _ = controller_tx.send(engine.get_thread_state_controller()); 117 117 + let outcome = run_query(&mut engine, expr); 118 118 + let _ = result_tx.send(outcome); 119 119 + }); 120 120 + 121 121 + let Ok(controller) = controller_rx.recv() else { 122 122 + return QueryOutcome::Errored(QueryError { 123 123 + message: "query VM thread panicked before starting".into(), 124 124 + span: None, 125 125 + }); 126 126 + }; 127 127 + 128 128 + match result_rx.recv_timeout(budget) { 129 129 + Ok(outcome) => outcome, 130 130 + Err(mpsc::RecvTimeoutError::Timeout) => { 131 131 + controller.interrupt(); 132 132 + QueryOutcome::TimedOut 133 133 + } 134 134 + Err(mpsc::RecvTimeoutError::Disconnected) => QueryOutcome::Errored(QueryError { 135 135 + message: "query VM thread panicked or dropped its sender".into(), 136 136 + span: None, 137 137 + }), 138 138 + } 139 139 + } 140 140 + 141 141 + fn run_query(engine: &mut Engine, expr: String) -> QueryOutcome { 142 142 + match engine.run(expr) { 143 143 + Ok(values) => match values.last() { 144 144 + Some(value) => match interpret_result(value) { 145 145 + Ok(v) => QueryOutcome::Completed(v), 146 146 + Err(e) => QueryOutcome::Errored(e), 147 147 + }, 148 148 + None => QueryOutcome::Completed(QueryValue::Blocks(Vec::new())), 149 149 + }, 150 150 + Err(e) => QueryOutcome::Errored(to_query_error(&e)), 151 151 + } 152 152 + } 153 153 + 154 154 + fn to_query_error(e: &SteelErr) -> QueryError { 155 155 + QueryError { 156 156 + message: e.to_string(), 157 157 + span: e.span().map(|s| (s.start as usize, s.end as usize)), 158 158 + } 159 159 + } 160 160 + 161 161 + /// A `table` result is a list of rows, each row a list of (already 162 162 + /// string-rendered) cells, tagged with a `'trawler-table` marker as the 163 163 + /// first list element so it's distinguishable from an ordinary block-id 164 164 + /// list without needing a custom FFI type. 165 165 + const TABLE_MARKER: &str = "trawler-table"; 166 166 + 167 167 + fn interpret_result(value: &SteelVal) -> Result<QueryValue, QueryError> { 168 168 + if let SteelVal::ListV(list) = value { 169 169 + let items: Vec<SteelVal> = list.iter().cloned().collect(); 170 170 + if items.first().and_then(steelval_as_str) == Some(TABLE_MARKER) { 171 171 + return interpret_table(&items[1..]); 172 172 + } 173 173 + } 174 174 + 175 175 + let ids: Vec<String> = Vec::from_steelval_or_err(value)?; 176 176 + let mut blocks = Vec::with_capacity(ids.len()); 177 177 + for id in ids { 178 178 + match TreeID::try_from(id.as_str()) { 179 179 + Ok(tree_id) => blocks.push(tree_id), 180 180 + Err(_) => { 181 181 + return Err(QueryError { 182 182 + message: format!("query result contained a malformed block id: {id}"), 183 183 + span: None, 184 184 + }); 185 185 + } 186 186 + } 187 187 + } 188 188 + Ok(QueryValue::Blocks(blocks)) 189 189 + } 190 190 + 191 191 + fn interpret_table(rows: &[SteelVal]) -> Result<QueryValue, QueryError> { 192 192 + let mut columns: Option<Vec<String>> = None; 193 193 + let mut out_rows = Vec::with_capacity(rows.len()); 194 194 + for row in rows { 195 195 + let SteelVal::ListV(row_list) = row else { 196 196 + return Err(QueryError { 197 197 + message: "malformed table result: expected a row list".into(), 198 198 + span: None, 199 199 + }); 200 200 + }; 201 201 + let mut header = Vec::new(); 202 202 + let mut cells = Vec::new(); 203 203 + for cell in row_list.iter() { 204 204 + let SteelVal::ListV(pair) = cell else { 205 205 + return Err(QueryError { 206 206 + message: "malformed table result: expected (column . value) cells".into(), 207 207 + span: None, 208 208 + }); 209 209 + }; 210 210 + let pair: Vec<SteelVal> = pair.iter().cloned().collect(); 211 211 + let (Some(col), Some(val)) = (pair.first(), pair.get(1)) else { 212 212 + return Err(QueryError { 213 213 + message: "malformed table result: cell missing column or value".into(), 214 214 + span: None, 215 215 + }); 216 216 + }; 217 217 + header.push(steelval_as_str(col).unwrap_or_default().to_string()); 218 218 + cells.push(steelval_as_str(val).unwrap_or_default().to_string()); 219 219 + } 220 220 + columns.get_or_insert(header); 221 221 + out_rows.push(cells); 222 222 + } 223 223 + Ok(QueryValue::Table { 224 224 + columns: columns.unwrap_or_default(), 225 225 + rows: out_rows, 226 226 + }) 227 227 + } 228 228 + 229 229 + fn steelval_as_str(value: &SteelVal) -> Option<&str> { 230 230 + match value { 231 231 + SteelVal::StringV(s) => Some(s.as_str()), 232 232 + SteelVal::SymbolV(s) => Some(s.as_str()), 233 233 + _ => None, 234 234 + } 235 235 + } 236 236 + 237 237 + /// Local helper trait so `interpret_result` can lean on the same 238 238 + /// `FromSteelVal` machinery the rest of the codebase uses, with a 239 239 + /// query-shaped error instead of a raw `SteelErr`. 240 240 + trait FromSteelValOrErr: Sized { 241 241 + fn from_steelval_or_err(value: &SteelVal) -> Result<Self, QueryError>; 242 242 + } 243 243 + 244 244 + impl FromSteelValOrErr for Vec<String> { 245 245 + fn from_steelval_or_err(value: &SteelVal) -> Result<Self, QueryError> { 246 246 + Vec::from_steelval(value).map_err(|e| QueryError { 247 247 + message: format!("query did not evaluate to a block list or table: {e}"), 248 248 + span: None, 249 249 + }) 250 250 + } 251 251 + } 252 252 + 253 253 + fn build_engine(graph: Arc<GraphIndex>, search: Arc<SearchIndex>) -> Engine { 254 254 + let mut engine = Engine::new_raw_no_kernel(); 255 255 + 256 256 + // Minimal, deliberately-curated safe stdlib. Nothing beyond this list 257 257 + // is reachable from a query — no file, process, network, or mutation 258 258 + // primitives are ever registered (spec: "Read-only capability 259 259 + // scoping" / "Mutation is unavailable" resolves via unknown-identifier, 260 260 + // simply because those names were never bound). 261 261 + register_core_stdlib(&mut engine); 262 262 + engine 263 263 + .run(QUERY_PRELUDE) 264 264 + .expect("query prelude is a fixed, tested constant"); 265 265 + register_graph_primitives(&mut engine, graph, search); 266 266 + 267 267 + engine 268 268 + } 269 269 + 270 270 + fn register_core_stdlib(engine: &mut Engine) { 271 271 + engine.register_fn("+", |a: f64, b: f64| a + b); 272 272 + engine.register_fn("-", |a: f64, b: f64| a - b); 273 273 + engine.register_fn("*", |a: f64, b: f64| a * b); 274 274 + engine.register_fn("/", |a: f64, b: f64| a / b); 275 275 + engine.register_fn("=", |a: f64, b: f64| a == b); 276 276 + engine.register_fn("<", |a: f64, b: f64| a < b); 277 277 + engine.register_fn(">", |a: f64, b: f64| a > b); 278 278 + engine.register_fn("<=", |a: f64, b: f64| a <= b); 279 279 + engine.register_fn(">=", |a: f64, b: f64| a >= b); 280 280 + engine.register_fn("string-append", |a: String, b: String| a + &b); 281 281 + engine.register_fn("string-contains?", |haystack: String, needle: String| { 282 282 + haystack.contains(&needle) 283 283 + }); 284 284 + engine.register_fn("string-equal?", |a: String, b: String| a == b); 285 285 + // Working in `Vec<SteelVal>` (not the internal `List<SteelVal>` type 286 286 + // directly) lets Steel's own `FromSteelVal`/`IntoSteelVal` impls for 287 287 + // `Vec<T>` handle the list<->Vec conversion at the FFI boundary, so 288 288 + // these never touch steel-core's private list-representation types. 289 289 + engine.register_fn("cons", |a: SteelVal, mut rest: Vec<SteelVal>| { 290 290 + rest.insert(0, a); 291 291 + rest 292 292 + }); 293 293 + engine.register_fn("car", |list: Vec<SteelVal>| -> SteelVal { 294 294 + list.into_iter().next().unwrap_or(SteelVal::Void) 295 295 + }); 296 296 + engine.register_fn("cdr", |mut list: Vec<SteelVal>| -> Vec<SteelVal> { 297 297 + if !list.is_empty() { 298 298 + list.remove(0); 299 299 + } 300 300 + list 301 301 + }); 302 302 + engine.register_fn("null?", |list: Vec<SteelVal>| list.is_empty()); 303 303 + } 304 304 + 305 305 + fn register_graph_primitives( 306 306 + engine: &mut Engine, 307 307 + graph: Arc<GraphIndex>, 308 308 + search: Arc<SearchIndex>, 309 309 + ) { 310 310 + { 311 311 + let graph = graph.clone(); 312 312 + engine.register_fn("tag", move |name: String| -> Vec<String> { 313 313 + to_sorted_ids(graph.tags.get(&name).cloned().unwrap_or_default()) 314 314 + }); 315 315 + } 316 316 + { 317 317 + let graph = graph.clone(); 318 318 + engine.register_fn("ref", move |name: String| -> Vec<String> { 319 319 + let target = graph.resolve_name(&name); 320 320 + to_sorted_ids(graph.backlinks_of(&target)) 321 321 + }); 322 322 + } 323 323 + { 324 324 + let graph = graph.clone(); 325 325 + engine.register_fn("prop", move |key: String| -> Vec<String> { 326 326 + let ids = graph 327 327 + .properties 328 328 + .get(&key) 329 329 + .map(|m| m.keys().copied().collect()) 330 330 + .unwrap_or_default(); 331 331 + to_sorted_ids(ids) 332 332 + }); 333 333 + } 334 334 + { 335 335 + let graph = graph.clone(); 336 336 + engine.register_fn( 337 337 + "prop-eq", 338 338 + move |key: String, value: String| -> Vec<String> { 339 339 + let ids = graph 340 340 + .properties 341 341 + .get(&key) 342 342 + .map(|m| { 343 343 + m.iter() 344 344 + .filter(|(_, v)| display_property_value(v) == value) 345 345 + .map(|(&id, _)| id) 346 346 + .collect() 347 347 + }) 348 348 + .unwrap_or_default(); 349 349 + to_sorted_ids(ids) 350 350 + }, 351 351 + ); 352 352 + } 353 353 + { 354 354 + let graph = graph.clone(); 355 355 + engine.register_fn( 356 356 + "date-range", 357 357 + move |from: String, to: String| -> Vec<String> { 358 358 + let (Ok(from), Ok(to)) = ( 359 359 + chrono::NaiveDate::parse_from_str(&from, "%Y-%m-%d"), 360 360 + chrono::NaiveDate::parse_from_str(&to, "%Y-%m-%d"), 361 361 + ) else { 362 362 + return Vec::new(); 363 363 + }; 364 364 + let mut ids = std::collections::HashSet::new(); 365 365 + for (&date, blocks) in &graph.dates { 366 366 + if date >= from && date <= to { 367 367 + ids.extend(blocks.iter().copied()); 368 368 + } 369 369 + } 370 370 + to_sorted_ids(ids) 371 371 + }, 372 372 + ); 373 373 + } 374 374 + { 375 375 + let graph = graph.clone(); 376 376 + engine.register_fn("descendants", move |id: String| -> Vec<String> { 377 377 + let Ok(root) = TreeID::try_from(id.as_str()) else { 378 378 + return Vec::new(); 379 379 + }; 380 380 + let mut out = std::collections::HashSet::new(); 381 381 + let mut stack = graph.children.get(&root).cloned().unwrap_or_default(); 382 382 + while let Some(id) = stack.pop() { 383 383 + out.insert(id); 384 384 + if let Some(kids) = graph.children.get(&id) { 385 385 + stack.extend(kids.iter().copied()); 386 386 + } 387 387 + } 388 388 + to_sorted_ids(out) 389 389 + }); 390 390 + } 391 391 + { 392 392 + engine.register_fn("search", move |query_text: String| -> Vec<String> { 393 393 + let hits = search.search(&query_text, 10_000).unwrap_or_default(); 394 394 + let mut sorted: Vec<String> = hits.into_iter().map(|h| h.block.to_string()).collect(); 395 395 + sorted.sort(); 396 396 + sorted.dedup(); 397 397 + sorted 398 398 + }); 399 399 + } 400 400 + { 401 401 + let graph = graph.clone(); 402 402 + engine.register_fn("not", move |set: Vec<String>| -> Vec<String> { 403 403 + let universe = graph.all_blocks(); 404 404 + let set: std::collections::HashSet<String> = set.into_iter().collect(); 405 405 + to_sorted_ids( 406 406 + universe 407 407 + .into_iter() 408 408 + .filter(|id| !set.contains(&id.to_string())) 409 409 + .collect(), 410 410 + ) 411 411 + }); 412 412 + } 413 413 + engine.register_fn("and", |a: Vec<String>, b: Vec<String>| -> Vec<String> { 414 414 + let b: std::collections::HashSet<String> = b.into_iter().collect(); 415 415 + let mut out: Vec<String> = a.into_iter().filter(|id| b.contains(id)).collect(); 416 416 + out.sort(); 417 417 + out 418 418 + }); 419 419 + engine.register_fn("or", |a: Vec<String>, b: Vec<String>| -> Vec<String> { 420 420 + let mut out: std::collections::HashSet<String> = a.into_iter().collect(); 421 421 + out.extend(b); 422 422 + let mut out: Vec<String> = out.into_iter().collect(); 423 423 + out.sort(); 424 424 + out 425 425 + }); 426 426 + { 427 427 + let graph = graph.clone(); 428 428 + engine.register_fn( 429 429 + "table", 430 430 + move |ids: Vec<String>, keys: Vec<String>| -> Vec<SteelVal> { 431 431 + let mut rows: Vec<SteelVal> = vec![string(TABLE_MARKER)]; 432 432 + let mut sorted_ids = ids; 433 433 + sorted_ids.sort(); 434 434 + for id_str in sorted_ids { 435 435 + let mut cells: Vec<SteelVal> = Vec::new(); 436 436 + if let Ok(id) = TreeID::try_from(id_str.as_str()) { 437 437 + for key in &keys { 438 438 + let value = graph 439 439 + .properties 440 440 + .get(key) 441 441 + .and_then(|m| m.get(&id)) 442 442 + .map(display_property_value) 443 443 + .unwrap_or_default(); 444 444 + cells.push(pair(string(key), string(&value))); 445 445 + } 446 446 + } 447 447 + rows.push( 448 448 + cells 449 449 + .into_steelval() 450 450 + .expect("Vec<SteelVal> always converts"), 451 451 + ); 452 452 + } 453 453 + rows 454 454 + }, 455 455 + ); 456 456 + } 457 457 + let _ = graph; 458 458 + } 459 459 + 460 460 + fn to_sorted_ids(ids: std::collections::HashSet<TreeID>) -> Vec<String> { 461 461 + let mut out: Vec<String> = ids.iter().map(TreeID::to_string).collect(); 462 462 + out.sort(); 463 463 + out 464 464 + } 465 465 + 466 466 + fn display_property_value(value: &PropertyValue) -> String { 467 467 + match value { 468 468 + PropertyValue::Text(s) => s.clone(), 469 469 + PropertyValue::Number(n) => n.to_string(), 470 470 + PropertyValue::Date(d) => d.to_string(), 471 471 + PropertyValue::Bool(b) => b.to_string(), 472 472 + PropertyValue::Ref(NodeId::Tree(s)) => s.clone(), 473 473 + PropertyValue::Ref(NodeId::Tag(s)) => s.clone(), 474 474 + PropertyValue::Ref(NodeId::Date(d)) => d.to_string(), 475 475 + } 476 476 + } 477 477 + 478 478 + fn string(s: &str) -> SteelVal { 479 479 + s.to_string() 480 480 + .into_steelval() 481 481 + .expect("String always converts to SteelVal") 482 482 + } 483 483 + 484 484 + fn pair(a: SteelVal, b: SteelVal) -> SteelVal { 485 485 + vec![a, b] 486 486 + .into_steelval() 487 487 + .expect("Vec<SteelVal> always converts to SteelVal") 488 488 + } 489 489 + 490 490 + #[cfg(test)] 491 491 + mod tests { 492 492 + use super::*; 493 493 + use crate::outline::{Outline, Position}; 494 494 + use crate::storage::OUTLINE_TREE; 495 495 + use loro::LoroDoc; 496 496 + 497 497 + fn doc_with_tree() -> LoroDoc { 498 498 + let doc = LoroDoc::new(); 499 499 + doc.get_tree(OUTLINE_TREE).enable_fractional_index(0); 500 500 + doc 501 501 + } 502 502 + 503 503 + fn empty_search() -> Arc<SearchIndex> { 504 504 + let dir = std::env::temp_dir().join(format!( 505 505 + "trawler-query-test-search-{:?}", 506 506 + std::thread::current().id() 507 507 + )); 508 508 + let _ = std::fs::remove_dir_all(&dir); 509 509 + let doc = doc_with_tree(); 510 510 + Arc::new(SearchIndex::open_or_create(&dir, &doc).unwrap()) 511 511 + } 512 512 + 513 513 + fn as_blocks(outcome: QueryOutcome) -> Vec<TreeID> { 514 514 + match outcome { 515 515 + QueryOutcome::Completed(QueryValue::Blocks(mut ids)) => { 516 516 + ids.sort(); 517 517 + ids 518 518 + } 519 519 + other => panic!("expected Completed(Blocks), got {other:?}"), 520 520 + } 521 521 + } 522 522 + 523 523 + #[test] 524 524 + fn tag_primitive_returns_tagged_blocks() { 525 525 + let doc = doc_with_tree(); 526 526 + let outline = Outline::new(&doc); 527 527 + let page = outline 528 528 + .create_block(None, Position::Index(0), "Home") 529 529 + .unwrap(); 530 530 + let block = outline 531 531 + .create_block(Some(page), Position::Index(0), "working on #project") 532 532 + .unwrap(); 533 533 + let graph = Arc::new(GraphIndex::rebuild(&doc)); 534 534 + 535 535 + let outcome = eval_query( 536 536 + "(tag 'project)", 537 537 + graph, 538 538 + empty_search(), 539 539 + Duration::from_secs(2), 540 540 + ); 541 541 + assert_eq!(as_blocks(outcome), vec![block]); 542 542 + } 543 543 + 544 544 + #[test] 545 545 + fn native_and_combines_two_sets() { 546 546 + // spec scenario: "Native combination stays fast" (correctness half; 547 547 + // task 4.5 covers the performance half separately). 548 548 + let doc = doc_with_tree(); 549 549 + let outline = Outline::new(&doc); 550 550 + let page = outline 551 551 + .create_block(None, Position::Index(0), "Home") 552 552 + .unwrap(); 553 553 + let both = outline 554 554 + .create_block(Some(page), Position::Index(0), "#project #urgent") 555 555 + .unwrap(); 556 556 + outline 557 557 + .create_block(Some(page), Position::Index(1), "#project only") 558 558 + .unwrap(); 559 559 + outline 560 560 + .create_block(Some(page), Position::Index(2), "#urgent only") 561 561 + .unwrap(); 562 562 + let graph = Arc::new(GraphIndex::rebuild(&doc)); 563 563 + 564 564 + let outcome = eval_query( 565 565 + "(and (tag 'project) (tag 'urgent))", 566 566 + graph, 567 567 + empty_search(), 568 568 + Duration::from_secs(2), 569 569 + ); 570 570 + assert_eq!(as_blocks(outcome), vec![both]); 571 571 + } 572 572 + 573 573 + #[test] 574 574 + fn not_complements_against_all_blocks() { 575 575 + let doc = doc_with_tree(); 576 576 + let outline = Outline::new(&doc); 577 577 + let page = outline 578 578 + .create_block(None, Position::Index(0), "Home") 579 579 + .unwrap(); 580 580 + let tagged = outline 581 581 + .create_block(Some(page), Position::Index(0), "#project") 582 582 + .unwrap(); 583 583 + let untagged = outline 584 584 + .create_block(Some(page), Position::Index(1), "plain") 585 585 + .unwrap(); 586 586 + let graph = Arc::new(GraphIndex::rebuild(&doc)); 587 587 + 588 588 + let outcome = eval_query( 589 589 + "(not (tag 'project))", 590 590 + graph, 591 591 + empty_search(), 592 592 + Duration::from_secs(2), 593 593 + ); 594 594 + let blocks = as_blocks(outcome); 595 595 + assert!(blocks.contains(&untagged)); 596 596 + assert!(blocks.contains(&page)); 597 597 + assert!(!blocks.contains(&tagged)); 598 598 + } 599 599 + 600 600 + #[test] 601 601 + fn scheme_predicate_filters_a_narrowed_set() { 602 602 + // spec scenario: "Scheme predicates over narrowed sets" 603 603 + let doc = doc_with_tree(); 604 604 + let outline = Outline::new(&doc); 605 605 + let page = outline 606 606 + .create_block(None, Position::Index(0), "Home") 607 607 + .unwrap(); 608 608 + let matching = outline 609 609 + .create_block(Some(page), Position::Index(0), "#project about rust") 610 610 + .unwrap(); 611 611 + outline 612 612 + .create_block(Some(page), Position::Index(1), "#project about ocaml") 613 613 + .unwrap(); 614 614 + let graph = Arc::new(GraphIndex::rebuild(&doc)); 615 615 + 616 616 + let outcome = eval_query( 617 617 + r#"(filter (lambda (id) (string-contains? id id)) (tag 'project))"#, 618 618 + graph.clone(), 619 619 + empty_search(), 620 620 + Duration::from_secs(2), 621 621 + ); 622 622 + // sanity: filter over a trivially-true predicate keeps everything 623 623 + assert_eq!(as_blocks(outcome).len(), 2); 624 624 + 625 625 + // now a predicate that actually discriminates, using the search 626 626 + // primitive's set to identify the "rust" block, then filtering the 627 627 + // tag set down to just ids present in that search result. 628 628 + let search_outcome = eval_query( 629 629 + r#"(filter (lambda (id) (bool-not (null? (filter (lambda (s) (string-equal? s id)) (search "rust"))))) (tag 'project))"#, 630 630 + graph, 631 631 + { 632 632 + let dir = std::env::temp_dir().join(format!( 633 633 + "trawler-query-test-search-with-content-{:?}", 634 634 + std::thread::current().id() 635 635 + )); 636 636 + let _ = std::fs::remove_dir_all(&dir); 637 637 + Arc::new(SearchIndex::open_or_create(&dir, &doc).unwrap()) 638 638 + }, 639 639 + Duration::from_secs(2), 640 640 + ); 641 641 + assert_eq!(as_blocks(search_outcome), vec![matching]); 642 642 + } 643 643 + 644 644 + #[test] 645 645 + fn table_result_projects_properties() { 646 646 + // spec scenario: "Projected properties become columns" 647 647 + let doc = doc_with_tree(); 648 648 + let outline = Outline::new(&doc); 649 649 + let page = outline 650 650 + .create_block(None, Position::Index(0), "Home") 651 651 + .unwrap(); 652 652 + let block = outline 653 653 + .create_block(Some(page), Position::Index(0), "a task") 654 654 + .unwrap(); 655 655 + outline 656 656 + .set_property( 657 657 + block, 658 658 + "due", 659 659 + &PropertyValue::Date(chrono::NaiveDate::from_ymd_opt(2026, 7, 10).unwrap()), 660 660 + ) 661 661 + .unwrap(); 662 662 + let graph = Arc::new(GraphIndex::rebuild(&doc)); 663 663 + 664 664 + let outcome = eval_query( 665 665 + &format!(r#"(table '("{block}") '("due"))"#), 666 666 + graph, 667 667 + empty_search(), 668 668 + Duration::from_secs(2), 669 669 + ); 670 670 + match outcome { 671 671 + QueryOutcome::Completed(QueryValue::Table { columns, rows }) => { 672 672 + assert_eq!(columns, vec!["due".to_string()]); 673 673 + assert_eq!(rows, vec![vec!["2026-07-10".to_string()]]); 674 674 + } 675 675 + other => panic!("expected Completed(Table), got {other:?}"), 676 676 + } 677 677 + } 678 678 + 679 679 + #[test] 680 680 + fn mutation_is_unavailable_as_unknown_identifier() { 681 681 + // spec scenario: "Mutation is unavailable" 682 682 + let doc = doc_with_tree(); 683 683 + let graph = Arc::new(GraphIndex::rebuild(&doc)); 684 684 + let outcome = eval_query( 685 685 + r#"(delete-block "whatever")"#, 686 686 + graph, 687 687 + empty_search(), 688 688 + Duration::from_secs(2), 689 689 + ); 690 690 + assert!(matches!(outcome, QueryOutcome::Errored(_))); 691 691 + 692 692 + // filesystem/network primitives are simply never registered either. 693 693 + let doc2 = doc_with_tree(); 694 694 + let graph2 = Arc::new(GraphIndex::rebuild(&doc2)); 695 695 + let fs_outcome = eval_query( 696 696 + r#"(open-input-file "C:/Windows/win.ini")"#, 697 697 + graph2, 698 698 + empty_search(), 699 699 + Duration::from_secs(2), 700 700 + ); 701 701 + assert!(matches!(fs_outcome, QueryOutcome::Errored(_))); 702 702 + } 703 703 + 704 704 + #[test] 705 705 + fn error_has_position_info_for_unbalanced_paren() { 706 706 + // spec scenario: "Error is shown at the block" 707 707 + let doc = doc_with_tree(); 708 708 + let graph = Arc::new(GraphIndex::rebuild(&doc)); 709 709 + let outcome = eval_query( 710 710 + "(tag 'project", 711 711 + graph, 712 712 + empty_search(), 713 713 + Duration::from_secs(2), 714 714 + ); 715 715 + assert!(matches!(outcome, QueryOutcome::Errored(_))); 716 716 + } 717 717 + 718 718 + #[test] 719 719 + fn infinite_loop_is_terminated_at_budget() { 720 720 + // spec scenario: "Infinite loop cannot freeze the app" 721 721 + let doc = doc_with_tree(); 722 722 + let graph = Arc::new(GraphIndex::rebuild(&doc)); 723 723 + let started = std::time::Instant::now(); 724 724 + let outcome = eval_query( 725 725 + "(define (spin) (spin)) (spin)", 726 726 + graph, 727 727 + empty_search(), 728 728 + Duration::from_millis(200), 729 729 + ); 730 730 + let elapsed = started.elapsed(); 731 731 + 732 732 + assert!(matches!(outcome, QueryOutcome::TimedOut)); 733 733 + assert!( 734 734 + elapsed < Duration::from_secs(3), 735 735 + "caller blocked for {elapsed:?}, expected to regain control near the 200ms budget" 736 736 + ); 737 737 + } 738 738 + 739 739 + /// spec scenario: "Native combination stays fast" — intersecting a 740 740 + /// 20k-block tag set with a 5k-block date-range set on a 100k-block 741 741 + /// graph must complete in <50ms. 742 742 + /// 743 743 + /// `#[ignore]`d by default: building a 100k-block Loro doc + rebuilding 744 744 + /// the derived index takes real wall-clock time unrelated to what this 745 745 + /// test asserts (query eval time), and would otherwise slow down every 746 746 + /// `cargo test` run. Run explicitly with: 747 747 + /// `cargo test -p trawler-core --release -- --ignored query::tests::intersecting_large_sets_stays_under_50ms` 748 748 + #[test] 749 749 + #[ignore = "expensive: builds a 100k-block graph; run with --ignored --release"] 750 750 + fn intersecting_large_sets_stays_under_50ms() { 751 751 + const BLOCK_COUNT: usize = 100_000; 752 752 + 753 753 + let doc = doc_with_tree(); 754 754 + let tree = doc.get_tree(OUTLINE_TREE); 755 755 + let page = tree.create(None).unwrap(); 756 756 + tree.get_meta(page) 757 757 + .unwrap() 758 758 + .ensure_mergeable_text("content") 759 759 + .unwrap() 760 760 + .insert(0, "Perf") 761 761 + .unwrap(); 762 762 + 763 763 + for i in 0..BLOCK_COUNT { 764 764 + let id = tree.create(page).unwrap(); 765 765 + let meta = tree.get_meta(id).unwrap(); 766 766 + let mut content = format!("block {i}"); 767 767 + // ~20,000 blocks (every 5th) carry #project. 768 768 + if i % 5 == 0 { 769 769 + content.push_str(" #project"); 770 770 + } 771 771 + // ~5,000 blocks (every 20th) reference a date inside the range 772 772 + // we'll query; the rest fall outside it. 773 773 + if i % 20 == 0 { 774 774 + content.push_str(" [[2026-07-15]]"); 775 775 + } else { 776 776 + content.push_str(" [[2020-01-01]]"); 777 777 + } 778 778 + meta.ensure_mergeable_text("content") 779 779 + .unwrap() 780 780 + .insert(0, &content) 781 781 + .unwrap(); 782 782 + } 783 783 + 784 784 + let graph = Arc::new(GraphIndex::rebuild(&doc)); 785 785 + assert!(graph.tags.get("project").map(|s| s.len()).unwrap_or(0) >= 19_000); 786 786 + assert!( 787 787 + graph 788 788 + .dates 789 789 + .get(&chrono::NaiveDate::from_ymd_opt(2026, 7, 15).unwrap()) 790 790 + .map(|s| s.len()) 791 791 + .unwrap_or(0) 792 792 + >= 4_000 793 793 + ); 794 794 + 795 795 + let search = empty_search(); 796 796 + let started = std::time::Instant::now(); 797 797 + let outcome = eval_query( 798 798 + "(and (tag 'project) (date-range \"2026-07-01\" \"2026-07-31\"))", 799 799 + graph, 800 800 + search, 801 801 + Duration::from_secs(5), 802 802 + ); 803 803 + let elapsed = started.elapsed(); 804 804 + 805 805 + assert!(matches!( 806 806 + outcome, 807 807 + QueryOutcome::Completed(QueryValue::Blocks(_)) 808 808 + )); 809 809 + assert!( 810 810 + elapsed < Duration::from_millis(50), 811 811 + "intersection took {elapsed:?}, expected < 50ms" 812 812 + ); 813 813 + } 814 814 + }

+404 -404

crates/trawler-core/src/search.rs

Reviewed

··· 1 1 - //! Full-text search over block content (tantivy), plus lexical "similar 2 2 - //! blocks" suggestions. The index directory is entirely disposable: delete 3 3 - //! it and `SearchIndex::open_or_create` transparently rebuilds from the 4 4 - //! graph on next open. 5 5 - //! 6 6 - //! See openspec/changes/trawler-mvp specs/search/spec.md and tasks.md 3.1-3.4. 7 7 - 8 8 - use std::cell::RefCell; 9 9 - use std::collections::HashSet; 10 10 - use std::fs; 11 11 - use std::path::Path; 12 12 - 13 13 - use loro::{LoroDoc, TreeID}; 14 14 - use tantivy::collector::TopDocs; 15 15 - use tantivy::query::QueryParser; 16 16 - use tantivy::schema::{Field, Schema, TantivyDocument, Value, STORED, STRING, TEXT}; 17 17 - use tantivy::snippet::SnippetGenerator; 18 18 - use tantivy::{doc, Index, IndexReader, IndexWriter, ReloadPolicy, Term}; 19 19 - 20 20 - use crate::index::GraphIndex; 21 21 - use crate::outline::Outline; 22 22 - use crate::storage::OUTLINE_TREE; 23 23 - 24 24 - /// A single search hit: the block, its relevance score, and an HTML 25 25 - /// snippet with `<b>...</b>` marking matched terms (spec: "matches 26 26 - /// highlighted"). 27 27 - #[derive(Debug, Clone)] 28 28 - pub struct SearchHit { 29 29 - pub block: TreeID, 30 30 - pub score: f32, 31 31 - pub snippet_html: String, 32 32 - } 33 33 - 34 34 - pub struct SearchIndex { 35 35 - index: Index, 36 36 - writer: RefCell<IndexWriter>, 37 37 - reader: IndexReader, 38 38 - id_field: Field, 39 39 - content_field: Field, 40 40 - } 41 41 - 42 42 - fn build_schema() -> (Schema, Field, Field) { 43 43 - let mut builder = Schema::builder(); 44 44 - let id_field = builder.add_text_field("id", STRING | STORED); 45 45 - let content_field = builder.add_text_field("content", TEXT | STORED); 46 46 - (builder.build(), id_field, content_field) 47 47 - } 48 48 - 49 49 - impl SearchIndex { 50 50 - /// Open the search index at `dir`, or build it from scratch (from 51 51 - /// `doc`) if the directory is missing or empty — the "rebuild on 52 52 - /// missing" half of "the search index is disposable". 53 53 - pub fn open_or_create(dir: impl AsRef<Path>, doc: &LoroDoc) -> tantivy::Result<Self> { 54 54 - let dir = dir.as_ref().to_path_buf(); 55 55 - fs::create_dir_all(&dir)?; 56 56 - let (schema, id_field, content_field) = build_schema(); 57 57 - 58 58 - let already_exists = dir.join("meta.json").exists(); 59 59 - let index = if already_exists { 60 60 - Index::open_in_dir(&dir)? 61 61 - } else { 62 62 - Index::create_in_dir(&dir, schema)? 63 63 - }; 64 64 - 65 65 - let writer: IndexWriter = index.writer(50_000_000)?; 66 66 - let reader = index 67 67 - .reader_builder() 68 68 - .reload_policy(ReloadPolicy::OnCommitWithDelay) 69 69 - .try_into()?; 70 70 - 71 71 - let search_index = Self { 72 72 - index, 73 73 - writer: RefCell::new(writer), 74 74 - reader, 75 75 - id_field, 76 76 - content_field, 77 77 - }; 78 78 - 79 79 - if !already_exists { 80 80 - search_index.rebuild(doc)?; 81 81 - } 82 82 - 83 83 - Ok(search_index) 84 84 - } 85 85 - 86 86 - /// Full rebuild from the graph — used for the initial build and 87 87 - /// available to callers that want to force one (e.g. after detecting 88 88 - /// drift, or a "rebuild search index" command). 89 89 - pub fn rebuild(&self, doc: &LoroDoc) -> tantivy::Result<()> { 90 90 - let outline = Outline::new(doc); 91 91 - let tree = doc.get_tree(OUTLINE_TREE); 92 92 - 93 93 - let mut writer = self.writer.borrow_mut(); 94 94 - writer.delete_all_documents()?; 95 95 - for root in tree.roots() { 96 96 - let mut stack = vec![root]; 97 97 - while let Some(id) = stack.pop() { 98 98 - if let Ok(content) = outline.content(id) { 99 99 - writer.add_document(doc!( 100 100 - self.id_field => id.to_string(), 101 101 - self.content_field => content, 102 102 - ))?; 103 103 - } 104 104 - stack.extend(tree.children(id).unwrap_or_default()); 105 105 - } 106 106 - } 107 107 - writer.commit()?; 108 108 - self.reader.reload()?; 109 109 - Ok(()) 110 110 - } 111 111 - 112 112 - /// Index (or re-index) a single block's current content. Safe to call 113 113 - /// on every content-changing edit: commit is synchronous, so the block 114 114 - /// is searchable as soon as this returns — comfortably inside the 115 115 - /// spec's 1-second freshness budget (spec: "Index freshness"). 116 116 - pub fn upsert_block(&self, id: TreeID, content: &str) -> tantivy::Result<()> { 117 117 - let mut writer = self.writer.borrow_mut(); 118 118 - writer.delete_term(Term::from_field_text(self.id_field, &id.to_string())); 119 119 - writer.add_document(doc!( 120 120 - self.id_field => id.to_string(), 121 121 - self.content_field => content, 122 122 - ))?; 123 123 - writer.commit()?; 124 124 - self.reader.reload()?; 125 125 - Ok(()) 126 126 - } 127 127 - 128 128 - /// Remove a block from the index (after a delete). 129 129 - pub fn remove_block(&self, id: TreeID) -> tantivy::Result<()> { 130 130 - let mut writer = self.writer.borrow_mut(); 131 131 - writer.delete_term(Term::from_field_text(self.id_field, &id.to_string())); 132 132 - writer.commit()?; 133 133 - self.reader.reload()?; 134 134 - Ok(()) 135 135 - } 136 136 - 137 137 - /// Ranked full-text search with HTML-highlighted snippets (spec: 138 138 - /// "Search and jump"). 139 139 - pub fn search(&self, query_text: &str, limit: usize) -> tantivy::Result<Vec<SearchHit>> { 140 140 - let searcher = self.reader.searcher(); 141 141 - let query_parser = QueryParser::for_index(&self.index, vec![self.content_field]); 142 142 - let query = query_parser.parse_query(query_text)?; 143 143 - 144 144 - let snippet_generator = SnippetGenerator::create(&searcher, &*query, self.content_field)?; 145 145 - let top_docs = searcher.search(&query, &TopDocs::with_limit(limit).order_by_score())?; 146 146 - 147 147 - let mut hits = Vec::with_capacity(top_docs.len()); 148 148 - for (score, address) in top_docs { 149 149 - let retrieved: TantivyDocument = searcher.doc(address)?; 150 150 - let Some(id_str) = retrieved.get_first(self.id_field).and_then(|v| v.as_str()) else { 151 151 - continue; 152 152 - }; 153 153 - let Ok(block) = TreeID::try_from(id_str) else { 154 154 - continue; 155 155 - }; 156 156 - let snippet = snippet_generator.snippet_from_doc(&retrieved); 157 157 - hits.push(SearchHit { 158 158 - block, 159 159 - score, 160 160 - snippet_html: snippet.to_html(), 161 161 - }); 162 162 - } 163 163 - Ok(hits) 164 164 - } 165 165 - } 166 166 - 167 167 - /// Swappable "find blocks similar to this one" provider. `SearchIndex` 168 168 - /// implements this with lexical (shared-term) similarity for MVP; a future 169 169 - /// semantic/embedding tier can implement the same trait without any 170 170 - /// consumer (UI, query engine) changing (spec: "The similarity provider 171 171 - /// SHALL be a swappable interface"). 172 172 - pub trait Similarity { 173 173 - /// Blocks similar to `block` (whose current content is `content`), 174 174 - /// excluding `block` itself and its ancestors/descendants, ranked by 175 175 - /// similarity (spec: "Similar blocks (lexical)"). 176 176 - fn similar_blocks( 177 177 - &self, 178 178 - block: TreeID, 179 179 - content: &str, 180 180 - graph: &GraphIndex, 181 181 - limit: usize, 182 182 - ) -> tantivy::Result<Vec<SearchHit>>; 183 183 - } 184 184 - 185 185 - impl Similarity for SearchIndex { 186 186 - fn similar_blocks( 187 187 - &self, 188 188 - block: TreeID, 189 189 - content: &str, 190 190 - graph: &GraphIndex, 191 191 - limit: usize, 192 192 - ) -> tantivy::Result<Vec<SearchHit>> { 193 193 - let excluded = exclusion_set(block, graph); 194 194 - 195 195 - // Lexical similarity: query using the block's own content as the 196 196 - // search text, so shared distinctive terms drive the ranking — 197 197 - // the same TF-IDF machinery as ordinary search, applied to the 198 198 - // block's own text instead of user-typed input. 199 199 - let hits = self.search(&escape_for_query(content), limit + excluded.len())?; 200 200 - Ok(hits 201 201 - .into_iter() 202 202 - .filter(|hit| !excluded.contains(&hit.block)) 203 203 - .take(limit) 204 204 - .collect()) 205 205 - } 206 206 - } 207 207 - 208 208 - fn exclusion_set(block: TreeID, graph: &GraphIndex) -> HashSet<TreeID> { 209 209 - let mut excluded = HashSet::from([block]); 210 210 - 211 211 - // Descendants: walk graph.children, which the derived index already 212 212 - // maintains — no need to touch the Loro doc here. 213 213 - let mut stack = graph.children.get(&block).cloned().unwrap_or_default(); 214 214 - while let Some(id) = stack.pop() { 215 215 - excluded.insert(id); 216 216 - if let Some(kids) = graph.children.get(&id) { 217 217 - stack.extend(kids.iter().copied()); 218 218 - } 219 219 - } 220 220 - 221 221 - // Ancestors: walk up via the same children map (no dedicated parent 222 222 - // map in GraphIndex — O(pages) since outlines aren't very deep). 223 223 - let mut current = block; 224 224 - 'outer: loop { 225 225 - for (&parent, kids) in &graph.children { 226 226 - if kids.contains(&current) { 227 227 - excluded.insert(parent); 228 228 - current = parent; 229 229 - continue 'outer; 230 230 - } 231 231 - } 232 232 - break; 233 233 - } 234 234 - 235 235 - excluded 236 236 - } 237 237 - 238 238 - /// tantivy's default query parser treats `+ - " ( ) ^ ~ * ? : \` as query 239 239 - /// syntax; escape them so a block's own Markdown content (which routinely 240 240 - /// contains e.g. `-` bullets or `:`) is treated as plain terms, not a 241 241 - /// malformed query. 242 242 - fn escape_for_query(text: &str) -> String { 243 243 - text.chars() 244 244 - .map(|c| match c { 245 245 - '+' | '-' | '"' | '(' | ')' | '^' | '~' | '*' | '?' | ':' | '\\' => ' ', 246 246 - other => other, 247 247 - }) 248 248 - .collect() 249 249 - } 250 250 - 251 251 - #[cfg(test)] 252 252 - mod tests { 253 253 - use super::*; 254 254 - use crate::outline::Position; 255 255 - use std::path::PathBuf; 256 256 - 257 257 - fn temp_dir(name: &str) -> PathBuf { 258 258 - let dir = std::env::temp_dir().join(format!("trawler-search-test-{name}")); 259 259 - let _ = fs::remove_dir_all(&dir); 260 260 - dir 261 261 - } 262 262 - 263 263 - fn doc_with_tree() -> LoroDoc { 264 264 - let doc = LoroDoc::new(); 265 265 - doc.get_tree(OUTLINE_TREE).enable_fractional_index(0); 266 266 - doc 267 267 - } 268 268 - 269 269 - #[test] 270 270 - fn search_and_jump_finds_matching_block() { 271 271 - // spec scenario: "Search and jump" 272 272 - let doc = doc_with_tree(); 273 273 - let outline = Outline::new(&doc); 274 274 - let page = outline 275 275 - .create_block(None, Position::Index(0), "Home") 276 276 - .unwrap(); 277 277 - let target = outline 278 278 - .create_block( 279 279 - Some(page), 280 280 - Position::Index(0), 281 281 - "watched an otter trawl the bay", 282 282 - ) 283 283 - .unwrap(); 284 284 - outline 285 285 - .create_block(Some(page), Position::Index(1), "unrelated grocery list") 286 286 - .unwrap(); 287 287 - 288 288 - let dir = temp_dir("search-and-jump"); 289 289 - let index = SearchIndex::open_or_create(&dir, &doc).unwrap(); 290 290 - 291 291 - let hits = index.search("otter trawl", 10).unwrap(); 292 292 - assert_eq!(hits.len(), 1); 293 293 - assert_eq!(hits[0].block, target); 294 294 - assert!(hits[0].snippet_html.contains("<b>")); 295 295 - 296 296 - fs::remove_dir_all(&dir).ok(); 297 297 - } 298 298 - 299 299 - #[test] 300 300 - fn index_freshness_finds_content_after_upsert() { 301 301 - // spec scenario: "Just-typed content is findable" 302 302 - let doc = doc_with_tree(); 303 303 - let outline = Outline::new(&doc); 304 304 - let page = outline 305 305 - .create_block(None, Position::Index(0), "Home") 306 306 - .unwrap(); 307 307 - 308 308 - let dir = temp_dir("index-freshness"); 309 309 - let index = SearchIndex::open_or_create(&dir, &doc).unwrap(); 310 310 - 311 311 - let block = outline 312 312 - .create_block( 313 313 - Some(page), 314 314 - Position::Index(0), 315 315 - "a genuinely novel zyzzyx term", 316 316 - ) 317 317 - .unwrap(); 318 318 - index 319 319 - .upsert_block(block, &outline.content(block).unwrap()) 320 320 - .unwrap(); 321 321 - 322 322 - let hits = index.search("zyzzyx", 10).unwrap(); 323 323 - assert_eq!(hits.len(), 1); 324 324 - assert_eq!(hits[0].block, block); 325 325 - 326 326 - fs::remove_dir_all(&dir).ok(); 327 327 - } 328 328 - 329 329 - #[test] 330 330 - fn rebuild_after_deletion_reproduces_same_results() { 331 331 - // spec scenario: "Rebuild after deletion" 332 332 - let doc = doc_with_tree(); 333 333 - let outline = Outline::new(&doc); 334 334 - let page = outline 335 335 - .create_block(None, Position::Index(0), "Home") 336 336 - .unwrap(); 337 337 - let target = outline 338 338 - .create_block(Some(page), Position::Index(0), "mending the cod-end net") 339 339 - .unwrap(); 340 340 - 341 341 - let dir = temp_dir("rebuild-after-deletion"); 342 342 - { 343 343 - let index = SearchIndex::open_or_create(&dir, &doc).unwrap(); 344 344 - let hits = index.search("cod-end", 10).unwrap(); 345 345 - assert_eq!(hits.len(), 1); 346 346 - } 347 347 - 348 348 - fs::remove_dir_all(&dir).unwrap(); 349 349 - 350 350 - let rebuilt = SearchIndex::open_or_create(&dir, &doc).unwrap(); 351 351 - let hits = rebuilt.search("cod-end", 10).unwrap(); 352 352 - assert_eq!(hits.len(), 1); 353 353 - assert_eq!(hits[0].block, target); 354 354 - 355 355 - fs::remove_dir_all(&dir).ok(); 356 356 - } 357 357 - 358 358 - #[test] 359 359 - fn similar_blocks_excludes_self_and_ancestors_and_finds_related() { 360 360 - // spec scenario: "Related note resurfaces" 361 361 - let doc = doc_with_tree(); 362 362 - let outline = Outline::new(&doc); 363 363 - let page = outline 364 364 - .create_block(None, Position::Index(0), "Home") 365 365 - .unwrap(); 366 366 - let subject = outline 367 367 - .create_block( 368 368 - Some(page), 369 369 - Position::Index(0), 370 370 - "mending the cod-end after the last haul", 371 371 - ) 372 372 - .unwrap(); 373 373 - let child_of_subject = outline 374 374 - .create_block(Some(subject), Position::Index(0), "cod-end thread notes") 375 375 - .unwrap(); 376 376 - let related = outline 377 377 - .create_block( 378 378 - Some(page), 379 379 - Position::Index(1), 380 380 - "another cod-end net repair from last season", 381 381 - ) 382 382 - .unwrap(); 383 383 - let unrelated = outline 384 384 - .create_block(Some(page), Position::Index(2), "grocery list: milk, eggs") 385 385 - .unwrap(); 386 386 - 387 387 - let dir = temp_dir("similar-blocks"); 388 388 - let search = SearchIndex::open_or_create(&dir, &doc).unwrap(); 389 389 - let graph = GraphIndex::rebuild(&doc); 390 390 - 391 391 - let subject_content = outline.content(subject).unwrap(); 392 392 - let hits = search 393 393 - .similar_blocks(subject, &subject_content, &graph, 10) 394 394 - .unwrap(); 395 395 - let hit_blocks: Vec<TreeID> = hits.iter().map(|h| h.block).collect(); 396 396 - 397 397 - assert!(!hit_blocks.contains(&subject)); 398 398 - assert!(!hit_blocks.contains(&child_of_subject)); 399 399 - assert!(hit_blocks.contains(&related)); 400 400 - assert!(!hit_blocks.contains(&unrelated)); 401 401 - 402 402 - fs::remove_dir_all(&dir).ok(); 403 403 - } 404 404 - } 1 1 + //! Full-text search over block content (tantivy), plus lexical "similar 2 2 + //! blocks" suggestions. The index directory is entirely disposable: delete 3 3 + //! it and `SearchIndex::open_or_create` transparently rebuilds from the 4 4 + //! graph on next open. 5 5 + //! 6 6 + //! See openspec/changes/trawler-mvp specs/search/spec.md and tasks.md 3.1-3.4. 7 7 + 8 8 + use std::collections::HashSet; 9 9 + use std::fs; 10 10 + use std::path::Path; 11 11 + use std::sync::Mutex; 12 12 + 13 13 + use loro::{LoroDoc, TreeID}; 14 14 + use tantivy::collector::TopDocs; 15 15 + use tantivy::query::QueryParser; 16 16 + use tantivy::schema::{Field, Schema, TantivyDocument, Value, STORED, STRING, TEXT}; 17 17 + use tantivy::snippet::SnippetGenerator; 18 18 + use tantivy::{doc, Index, IndexReader, IndexWriter, ReloadPolicy, Term}; 19 19 + 20 20 + use crate::index::GraphIndex; 21 21 + use crate::outline::Outline; 22 22 + use crate::storage::OUTLINE_TREE; 23 23 + 24 24 + /// A single search hit: the block, its relevance score, and an HTML 25 25 + /// snippet with `<b>...</b>` marking matched terms (spec: "matches 26 26 + /// highlighted"). 27 27 + #[derive(Debug, Clone)] 28 28 + pub struct SearchHit { 29 29 + pub block: TreeID, 30 30 + pub score: f32, 31 31 + pub snippet_html: String, 32 32 + } 33 33 + 34 34 + pub struct SearchIndex { 35 35 + index: Index, 36 36 + writer: Mutex<IndexWriter>, 37 37 + reader: IndexReader, 38 38 + id_field: Field, 39 39 + content_field: Field, 40 40 + } 41 41 + 42 42 + fn build_schema() -> (Schema, Field, Field) { 43 43 + let mut builder = Schema::builder(); 44 44 + let id_field = builder.add_text_field("id", STRING | STORED); 45 45 + let content_field = builder.add_text_field("content", TEXT | STORED); 46 46 + (builder.build(), id_field, content_field) 47 47 + } 48 48 + 49 49 + impl SearchIndex { 50 50 + /// Open the search index at `dir`, or build it from scratch (from 51 51 + /// `doc`) if the directory is missing or empty — the "rebuild on 52 52 + /// missing" half of "the search index is disposable". 53 53 + pub fn open_or_create(dir: impl AsRef<Path>, doc: &LoroDoc) -> tantivy::Result<Self> { 54 54 + let dir = dir.as_ref().to_path_buf(); 55 55 + fs::create_dir_all(&dir)?; 56 56 + let (schema, id_field, content_field) = build_schema(); 57 57 + 58 58 + let already_exists = dir.join("meta.json").exists(); 59 59 + let index = if already_exists { 60 60 + Index::open_in_dir(&dir)? 61 61 + } else { 62 62 + Index::create_in_dir(&dir, schema)? 63 63 + }; 64 64 + 65 65 + let writer: IndexWriter = index.writer(50_000_000)?; 66 66 + let reader = index 67 67 + .reader_builder() 68 68 + .reload_policy(ReloadPolicy::OnCommitWithDelay) 69 69 + .try_into()?; 70 70 + 71 71 + let search_index = Self { 72 72 + index, 73 73 + writer: Mutex::new(writer), 74 74 + reader, 75 75 + id_field, 76 76 + content_field, 77 77 + }; 78 78 + 79 79 + if !already_exists { 80 80 + search_index.rebuild(doc)?; 81 81 + } 82 82 + 83 83 + Ok(search_index) 84 84 + } 85 85 + 86 86 + /// Full rebuild from the graph — used for the initial build and 87 87 + /// available to callers that want to force one (e.g. after detecting 88 88 + /// drift, or a "rebuild search index" command). 89 89 + pub fn rebuild(&self, doc: &LoroDoc) -> tantivy::Result<()> { 90 90 + let outline = Outline::new(doc); 91 91 + let tree = doc.get_tree(OUTLINE_TREE); 92 92 + 93 93 + let mut writer = self.writer.lock().unwrap(); 94 94 + writer.delete_all_documents()?; 95 95 + for root in tree.roots() { 96 96 + let mut stack = vec![root]; 97 97 + while let Some(id) = stack.pop() { 98 98 + if let Ok(content) = outline.content(id) { 99 99 + writer.add_document(doc!( 100 100 + self.id_field => id.to_string(), 101 101 + self.content_field => content, 102 102 + ))?; 103 103 + } 104 104 + stack.extend(tree.children(id).unwrap_or_default()); 105 105 + } 106 106 + } 107 107 + writer.commit()?; 108 108 + self.reader.reload()?; 109 109 + Ok(()) 110 110 + } 111 111 + 112 112 + /// Index (or re-index) a single block's current content. Safe to call 113 113 + /// on every content-changing edit: commit is synchronous, so the block 114 114 + /// is searchable as soon as this returns — comfortably inside the 115 115 + /// spec's 1-second freshness budget (spec: "Index freshness"). 116 116 + pub fn upsert_block(&self, id: TreeID, content: &str) -> tantivy::Result<()> { 117 117 + let mut writer = self.writer.lock().unwrap(); 118 118 + writer.delete_term(Term::from_field_text(self.id_field, &id.to_string())); 119 119 + writer.add_document(doc!( 120 120 + self.id_field => id.to_string(), 121 121 + self.content_field => content, 122 122 + ))?; 123 123 + writer.commit()?; 124 124 + self.reader.reload()?; 125 125 + Ok(()) 126 126 + } 127 127 + 128 128 + /// Remove a block from the index (after a delete). 129 129 + pub fn remove_block(&self, id: TreeID) -> tantivy::Result<()> { 130 130 + let mut writer = self.writer.lock().unwrap(); 131 131 + writer.delete_term(Term::from_field_text(self.id_field, &id.to_string())); 132 132 + writer.commit()?; 133 133 + self.reader.reload()?; 134 134 + Ok(()) 135 135 + } 136 136 + 137 137 + /// Ranked full-text search with HTML-highlighted snippets (spec: 138 138 + /// "Search and jump"). 139 139 + pub fn search(&self, query_text: &str, limit: usize) -> tantivy::Result<Vec<SearchHit>> { 140 140 + let searcher = self.reader.searcher(); 141 141 + let query_parser = QueryParser::for_index(&self.index, vec![self.content_field]); 142 142 + let query = query_parser.parse_query(query_text)?; 143 143 + 144 144 + let snippet_generator = SnippetGenerator::create(&searcher, &*query, self.content_field)?; 145 145 + let top_docs = searcher.search(&query, &TopDocs::with_limit(limit).order_by_score())?; 146 146 + 147 147 + let mut hits = Vec::with_capacity(top_docs.len()); 148 148 + for (score, address) in top_docs { 149 149 + let retrieved: TantivyDocument = searcher.doc(address)?; 150 150 + let Some(id_str) = retrieved.get_first(self.id_field).and_then(|v| v.as_str()) else { 151 151 + continue; 152 152 + }; 153 153 + let Ok(block) = TreeID::try_from(id_str) else { 154 154 + continue; 155 155 + }; 156 156 + let snippet = snippet_generator.snippet_from_doc(&retrieved); 157 157 + hits.push(SearchHit { 158 158 + block, 159 159 + score, 160 160 + snippet_html: snippet.to_html(), 161 161 + }); 162 162 + } 163 163 + Ok(hits) 164 164 + } 165 165 + } 166 166 + 167 167 + /// Swappable "find blocks similar to this one" provider. `SearchIndex` 168 168 + /// implements this with lexical (shared-term) similarity for MVP; a future 169 169 + /// semantic/embedding tier can implement the same trait without any 170 170 + /// consumer (UI, query engine) changing (spec: "The similarity provider 171 171 + /// SHALL be a swappable interface"). 172 172 + pub trait Similarity { 173 173 + /// Blocks similar to `block` (whose current content is `content`), 174 174 + /// excluding `block` itself and its ancestors/descendants, ranked by 175 175 + /// similarity (spec: "Similar blocks (lexical)"). 176 176 + fn similar_blocks( 177 177 + &self, 178 178 + block: TreeID, 179 179 + content: &str, 180 180 + graph: &GraphIndex, 181 181 + limit: usize, 182 182 + ) -> tantivy::Result<Vec<SearchHit>>; 183 183 + } 184 184 + 185 185 + impl Similarity for SearchIndex { 186 186 + fn similar_blocks( 187 187 + &self, 188 188 + block: TreeID, 189 189 + content: &str, 190 190 + graph: &GraphIndex, 191 191 + limit: usize, 192 192 + ) -> tantivy::Result<Vec<SearchHit>> { 193 193 + let excluded = exclusion_set(block, graph); 194 194 + 195 195 + // Lexical similarity: query using the block's own content as the 196 196 + // search text, so shared distinctive terms drive the ranking — 197 197 + // the same TF-IDF machinery as ordinary search, applied to the 198 198 + // block's own text instead of user-typed input. 199 199 + let hits = self.search(&escape_for_query(content), limit + excluded.len())?; 200 200 + Ok(hits 201 201 + .into_iter() 202 202 + .filter(|hit| !excluded.contains(&hit.block)) 203 203 + .take(limit) 204 204 + .collect()) 205 205 + } 206 206 + } 207 207 + 208 208 + fn exclusion_set(block: TreeID, graph: &GraphIndex) -> HashSet<TreeID> { 209 209 + let mut excluded = HashSet::from([block]); 210 210 + 211 211 + // Descendants: walk graph.children, which the derived index already 212 212 + // maintains — no need to touch the Loro doc here. 213 213 + let mut stack = graph.children.get(&block).cloned().unwrap_or_default(); 214 214 + while let Some(id) = stack.pop() { 215 215 + excluded.insert(id); 216 216 + if let Some(kids) = graph.children.get(&id) { 217 217 + stack.extend(kids.iter().copied()); 218 218 + } 219 219 + } 220 220 + 221 221 + // Ancestors: walk up via the same children map (no dedicated parent 222 222 + // map in GraphIndex — O(pages) since outlines aren't very deep). 223 223 + let mut current = block; 224 224 + 'outer: loop { 225 225 + for (&parent, kids) in &graph.children { 226 226 + if kids.contains(&current) { 227 227 + excluded.insert(parent); 228 228 + current = parent; 229 229 + continue 'outer; 230 230 + } 231 231 + } 232 232 + break; 233 233 + } 234 234 + 235 235 + excluded 236 236 + } 237 237 + 238 238 + /// tantivy's default query parser treats `+ - " ( ) ^ ~ * ? : \` as query 239 239 + /// syntax; escape them so a block's own Markdown content (which routinely 240 240 + /// contains e.g. `-` bullets or `:`) is treated as plain terms, not a 241 241 + /// malformed query. 242 242 + fn escape_for_query(text: &str) -> String { 243 243 + text.chars() 244 244 + .map(|c| match c { 245 245 + '+' | '-' | '"' | '(' | ')' | '^' | '~' | '*' | '?' | ':' | '\\' => ' ', 246 246 + other => other, 247 247 + }) 248 248 + .collect() 249 249 + } 250 250 + 251 251 + #[cfg(test)] 252 252 + mod tests { 253 253 + use super::*; 254 254 + use crate::outline::Position; 255 255 + use std::path::PathBuf; 256 256 + 257 257 + fn temp_dir(name: &str) -> PathBuf { 258 258 + let dir = std::env::temp_dir().join(format!("trawler-search-test-{name}")); 259 259 + let _ = fs::remove_dir_all(&dir); 260 260 + dir 261 261 + } 262 262 + 263 263 + fn doc_with_tree() -> LoroDoc { 264 264 + let doc = LoroDoc::new(); 265 265 + doc.get_tree(OUTLINE_TREE).enable_fractional_index(0); 266 266 + doc 267 267 + } 268 268 + 269 269 + #[test] 270 270 + fn search_and_jump_finds_matching_block() { 271 271 + // spec scenario: "Search and jump" 272 272 + let doc = doc_with_tree(); 273 273 + let outline = Outline::new(&doc); 274 274 + let page = outline 275 275 + .create_block(None, Position::Index(0), "Home") 276 276 + .unwrap(); 277 277 + let target = outline 278 278 + .create_block( 279 279 + Some(page), 280 280 + Position::Index(0), 281 281 + "watched an otter trawl the bay", 282 282 + ) 283 283 + .unwrap(); 284 284 + outline 285 285 + .create_block(Some(page), Position::Index(1), "unrelated grocery list") 286 286 + .unwrap(); 287 287 + 288 288 + let dir = temp_dir("search-and-jump"); 289 289 + let index = SearchIndex::open_or_create(&dir, &doc).unwrap(); 290 290 + 291 291 + let hits = index.search("otter trawl", 10).unwrap(); 292 292 + assert_eq!(hits.len(), 1); 293 293 + assert_eq!(hits[0].block, target); 294 294 + assert!(hits[0].snippet_html.contains("<b>")); 295 295 + 296 296 + fs::remove_dir_all(&dir).ok(); 297 297 + } 298 298 + 299 299 + #[test] 300 300 + fn index_freshness_finds_content_after_upsert() { 301 301 + // spec scenario: "Just-typed content is findable" 302 302 + let doc = doc_with_tree(); 303 303 + let outline = Outline::new(&doc); 304 304 + let page = outline 305 305 + .create_block(None, Position::Index(0), "Home") 306 306 + .unwrap(); 307 307 + 308 308 + let dir = temp_dir("index-freshness"); 309 309 + let index = SearchIndex::open_or_create(&dir, &doc).unwrap(); 310 310 + 311 311 + let block = outline 312 312 + .create_block( 313 313 + Some(page), 314 314 + Position::Index(0), 315 315 + "a genuinely novel zyzzyx term", 316 316 + ) 317 317 + .unwrap(); 318 318 + index 319 319 + .upsert_block(block, &outline.content(block).unwrap()) 320 320 + .unwrap(); 321 321 + 322 322 + let hits = index.search("zyzzyx", 10).unwrap(); 323 323 + assert_eq!(hits.len(), 1); 324 324 + assert_eq!(hits[0].block, block); 325 325 + 326 326 + fs::remove_dir_all(&dir).ok(); 327 327 + } 328 328 + 329 329 + #[test] 330 330 + fn rebuild_after_deletion_reproduces_same_results() { 331 331 + // spec scenario: "Rebuild after deletion" 332 332 + let doc = doc_with_tree(); 333 333 + let outline = Outline::new(&doc); 334 334 + let page = outline 335 335 + .create_block(None, Position::Index(0), "Home") 336 336 + .unwrap(); 337 337 + let target = outline 338 338 + .create_block(Some(page), Position::Index(0), "mending the cod-end net") 339 339 + .unwrap(); 340 340 + 341 341 + let dir = temp_dir("rebuild-after-deletion"); 342 342 + { 343 343 + let index = SearchIndex::open_or_create(&dir, &doc).unwrap(); 344 344 + let hits = index.search("cod-end", 10).unwrap(); 345 345 + assert_eq!(hits.len(), 1); 346 346 + } 347 347 + 348 348 + fs::remove_dir_all(&dir).unwrap(); 349 349 + 350 350 + let rebuilt = SearchIndex::open_or_create(&dir, &doc).unwrap(); 351 351 + let hits = rebuilt.search("cod-end", 10).unwrap(); 352 352 + assert_eq!(hits.len(), 1); 353 353 + assert_eq!(hits[0].block, target); 354 354 + 355 355 + fs::remove_dir_all(&dir).ok(); 356 356 + } 357 357 + 358 358 + #[test] 359 359 + fn similar_blocks_excludes_self_and_ancestors_and_finds_related() { 360 360 + // spec scenario: "Related note resurfaces" 361 361 + let doc = doc_with_tree(); 362 362 + let outline = Outline::new(&doc); 363 363 + let page = outline 364 364 + .create_block(None, Position::Index(0), "Home") 365 365 + .unwrap(); 366 366 + let subject = outline 367 367 + .create_block( 368 368 + Some(page), 369 369 + Position::Index(0), 370 370 + "mending the cod-end after the last haul", 371 371 + ) 372 372 + .unwrap(); 373 373 + let child_of_subject = outline 374 374 + .create_block(Some(subject), Position::Index(0), "cod-end thread notes") 375 375 + .unwrap(); 376 376 + let related = outline 377 377 + .create_block( 378 378 + Some(page), 379 379 + Position::Index(1), 380 380 + "another cod-end net repair from last season", 381 381 + ) 382 382 + .unwrap(); 383 383 + let unrelated = outline 384 384 + .create_block(Some(page), Position::Index(2), "grocery list: milk, eggs") 385 385 + .unwrap(); 386 386 + 387 387 + let dir = temp_dir("similar-blocks"); 388 388 + let search = SearchIndex::open_or_create(&dir, &doc).unwrap(); 389 389 + let graph = GraphIndex::rebuild(&doc); 390 390 + 391 391 + let subject_content = outline.content(subject).unwrap(); 392 392 + let hits = search 393 393 + .similar_blocks(subject, &subject_content, &graph, 10) 394 394 + .unwrap(); 395 395 + let hit_blocks: Vec<TreeID> = hits.iter().map(|h| h.block).collect(); 396 396 + 397 397 + assert!(!hit_blocks.contains(&subject)); 398 398 + assert!(!hit_blocks.contains(&child_of_subject)); 399 399 + assert!(hit_blocks.contains(&related)); 400 400 + assert!(!hit_blocks.contains(&unrelated)); 401 401 + 402 402 + fs::remove_dir_all(&dir).ok(); 403 403 + } 404 404 + }

+6 -6

openspec/changes/trawler-mvp/tasks.md

Reviewed

··· 28 28 29 29 ## 4. Steel Query Engine (trawler-core) 30 30 31 31 - - [ ] 4.1 Embed Steel VM using the S2 containment pattern; read-only capability context with only graph primitives registered 32 32 - - [ ] 4.2 Set-algebra primitives: tag/ref/property/date-range/descendants/search sets; native and/or/not over set handles 33 33 - - [ ] 4.3 Per-block Scheme predicate application over narrowed sets 34 34 - - [ ] 4.4 Result shapes: outline slice (block IDs) and table (projected properties); error type with position info 35 35 - - [ ] 4.5 Performance test: intersect 20k-tag × 5k-date sets on 100k-block graph < 50ms (spec: steel-queries/Set-algebra query primitives) 36 36 - - [ ] 4.6 Timeout/cancellation tests: non-terminating query terminates at budget, VM context is reusable or respawned cleanly 31 31 + - [x] 4.1 Embed Steel VM using the S2 containment pattern; read-only capability context with only graph primitives registered 32 32 + - [x] 4.2 Set-algebra primitives: tag/ref/property/date-range/descendants/search sets; native and/or/not over set handles 33 33 + - [x] 4.3 Per-block Scheme predicate application over narrowed sets 34 34 + - [x] 4.4 Result shapes: outline slice (block IDs) and table (projected properties); error type with position info 35 35 + - [x] 4.5 Performance test: intersect 20k-tag × 5k-date sets on 100k-block graph < 50ms (spec: steel-queries/Set-algebra query primitives) 36 36 + - [x] 4.6 Timeout/cancellation tests: non-terminating query terminates at budget, VM context is reusable or respawned cleanly 37 37 38 38 ## 5. GPUI Shell & Outline Editor (trawler) 39 39