+44 -2

Cargo.lock

Reviewed

··· 16 16 dependencies = [ 17 17 "cfg-if", 18 18 "cipher", 19 19 - "cpufeatures", 19 19 + "cpufeatures 0.2.17", 20 20 "zeroize", 21 21 ] 22 22 ··· 905 905 ] 906 906 907 907 [[package]] 908 908 + name = "chacha20" 909 909 + version = "0.10.1" 910 910 + source = "registry+https://github.com/rust-lang/crates.io-index" 911 911 + checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" 912 912 + dependencies = [ 913 913 + "cfg-if", 914 914 + "cpufeatures 0.3.0", 915 915 + "rand_core 0.10.1", 916 916 + ] 917 917 + 918 918 + [[package]] 908 919 name = "chrono" 909 920 version = "0.4.45" 910 921 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 913 924 "iana-time-zone", 914 925 "js-sys", 915 926 "num-traits", 927 927 + "serde", 916 928 "wasm-bindgen", 917 929 "windows-link 0.2.1", 918 930 ] ··· 1285 1297 version = "0.2.17" 1286 1298 source = "registry+https://github.com/rust-lang/crates.io-index" 1287 1299 checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" 1300 1300 + dependencies = [ 1301 1301 + "libc", 1302 1302 + ] 1303 1303 + 1304 1304 + [[package]] 1305 1305 + name = "cpufeatures" 1306 1306 + version = "0.3.0" 1307 1307 + source = "registry+https://github.com/rust-lang/crates.io-index" 1308 1308 + checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" 1288 1309 dependencies = [ 1289 1310 "libc", 1290 1311 ] ··· 2289 2310 "cfg-if", 2290 2311 "libc", 2291 2312 "r-efi 6.0.0", 2313 2313 + "rand_core 0.10.1", 2292 2314 ] 2293 2315 2294 2316 [[package]] ··· 5100 5122 ] 5101 5123 5102 5124 [[package]] 5125 5125 + name = "rand" 5126 5126 + version = "0.10.2" 5127 5127 + source = "registry+https://github.com/rust-lang/crates.io-index" 5128 5128 + checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" 5129 5129 + dependencies = [ 5130 5130 + "chacha20", 5131 5131 + "getrandom 0.4.3", 5132 5132 + "rand_core 0.10.1", 5133 5133 + ] 5134 5134 + 5135 5135 + [[package]] 5103 5136 name = "rand_chacha" 5104 5137 version = "0.3.1" 5105 5138 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 5137 5170 dependencies = [ 5138 5171 "getrandom 0.3.4", 5139 5172 ] 5173 5173 + 5174 5174 + [[package]] 5175 5175 + name = "rand_core" 5176 5176 + version = "0.10.1" 5177 5177 + source = "registry+https://github.com/rust-lang/crates.io-index" 5178 5178 + checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" 5140 5179 5141 5180 [[package]] 5142 5181 name = "rand_xoshiro" ··· 5909 5948 checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" 5910 5949 dependencies = [ 5911 5950 "cfg-if", 5912 5912 - "cpufeatures", 5951 5951 + "cpufeatures 0.2.17", 5913 5952 "digest", 5914 5953 ] 5915 5954 ··· 6960 6999 name = "trawler-core" 6961 7000 version = "0.1.0" 6962 7001 dependencies = [ 7002 7002 + "chrono", 6963 7003 "loro", 7004 7004 + "rand 0.10.2", 7005 7005 + "serde", 6964 7006 "steel-core", 6965 7007 ] 6966 7008

+3

crates/trawler-core/Cargo.toml

Reviewed

··· 8 8 workspace = true 9 9 10 10 [dependencies] 11 11 + chrono = { version = "0.4.45", features = ["serde"] } 11 12 loro = "1.13.6" 13 13 + serde = { version = "1.0.228", features = ["derive"] } 12 14 steel-core = "0.8.2" 13 15 14 16 [dev-dependencies] 17 17 + rand = "0.10.2"

+3

crates/trawler-core/src/graph/mod.rs

Reviewed

··· 1 1 + mod types; 2 2 + 3 3 + pub use types::{NodeId, NodeKind, PropertyValue};

+157

crates/trawler-core/src/graph/types.rs

Reviewed

··· 1 1 + //! Core graph types: node identity, node kinds, and property values. 2 2 + //! 3 3 + //! See openspec/changes/trawler-mvp specs/block-graph/spec.md 4 4 + //! ("Everything is a node", "Stable block identity", "Lightweight properties"). 5 5 + 6 6 + use std::fmt; 7 7 + 8 8 + use chrono::NaiveDate; 9 9 + use serde::{Deserialize, Serialize}; 10 10 + 11 11 + /// Stable, globally unique identity for a node in the graph. 12 12 + /// 13 13 + /// Blocks and pages are backed by a Loro `TreeID` (`peer@counter`), which is 14 14 + /// stable across edits, moves, restarts, and future sync merges — see the 15 15 + /// "Stable block identity" requirement. Tags and dates are *not* tree nodes 16 16 + /// (design.md D8: "tags and dates SHALL be referenceable nodes with no 17 17 + /// special-cased storage"); they get a canonical id derived from their 18 18 + /// name/date instead, so referencing `#project` or `[[2026-07-10]]` never 19 19 + /// requires first creating a tree node for it. 20 20 + #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] 21 21 + pub enum NodeId { 22 22 + /// A block or page: identity is the underlying Loro tree node's id, 23 23 + /// stored as its canonical `peer@counter` string form. 24 24 + Tree(String), 25 25 + /// A tag node, identified by its normalized name (no leading `#`). 26 26 + Tag(String), 27 27 + /// A journal date node, identified by ISO-8601 date. 28 28 + Date(NaiveDate), 29 29 + } 30 30 + 31 31 + impl NodeId { 32 32 + pub fn tree(id: loro::TreeID) -> Self { 33 33 + NodeId::Tree(id.to_string()) 34 34 + } 35 35 + 36 36 + pub fn tag(name: impl Into<String>) -> Self { 37 37 + NodeId::Tag(name.into()) 38 38 + } 39 39 + 40 40 + pub fn date(date: NaiveDate) -> Self { 41 41 + NodeId::Date(date) 42 42 + } 43 43 + 44 44 + /// Recover the Loro `TreeID` for a `NodeId::Tree`, if this id is one. 45 45 + pub fn as_tree_id(&self) -> Option<loro::TreeID> { 46 46 + match self { 47 47 + NodeId::Tree(s) => loro::TreeID::try_from(s.as_str()).ok(), 48 48 + _ => None, 49 49 + } 50 50 + } 51 51 + } 52 52 + 53 53 + impl fmt::Display for NodeId { 54 54 + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 55 55 + match self { 56 56 + NodeId::Tree(s) => write!(f, "{s}"), 57 57 + NodeId::Tag(name) => write!(f, "tag:{name}"), 58 58 + NodeId::Date(date) => write!(f, "date:{date}"), 59 59 + } 60 60 + } 61 61 + } 62 62 + 63 63 + /// What kind of thing a node represents. 64 64 + /// 65 65 + /// `Page` and `Block` are both tree-backed (`NodeId::Tree`); a page is 66 66 + /// distinguished only by being a root of the outline tree (design.md D8). 67 67 + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] 68 68 + pub enum NodeKind { 69 69 + Page, 70 70 + Block, 71 71 + Tag, 72 72 + Date, 73 73 + } 74 74 + 75 75 + /// A typed property value attached to a block. 76 76 + /// 77 77 + /// See the "Lightweight properties" requirement: text, number, date, bool, 78 78 + /// or a reference to another node. Schemas/classes/inheritance are 79 79 + /// explicitly out of scope for this change. 80 80 + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 81 81 + pub enum PropertyValue { 82 82 + Text(String), 83 83 + Number(f64), 84 84 + Date(NaiveDate), 85 85 + Bool(bool), 86 86 + Ref(NodeId), 87 87 + } 88 88 + 89 89 + impl PropertyValue { 90 90 + pub fn as_text(&self) -> Option<&str> { 91 91 + match self { 92 92 + PropertyValue::Text(s) => Some(s), 93 93 + _ => None, 94 94 + } 95 95 + } 96 96 + 97 97 + pub fn as_number(&self) -> Option<f64> { 98 98 + match self { 99 99 + PropertyValue::Number(n) => Some(*n), 100 100 + _ => None, 101 101 + } 102 102 + } 103 103 + 104 104 + pub fn as_date(&self) -> Option<NaiveDate> { 105 105 + match self { 106 106 + PropertyValue::Date(d) => Some(*d), 107 107 + _ => None, 108 108 + } 109 109 + } 110 110 + 111 111 + pub fn as_bool(&self) -> Option<bool> { 112 112 + match self { 113 113 + PropertyValue::Bool(b) => Some(*b), 114 114 + _ => None, 115 115 + } 116 116 + } 117 117 + 118 118 + pub fn as_ref(&self) -> Option<&NodeId> { 119 119 + match self { 120 120 + PropertyValue::Ref(id) => Some(id), 121 121 + _ => None, 122 122 + } 123 123 + } 124 124 + } 125 125 + 126 126 + #[cfg(test)] 127 127 + mod tests { 128 128 + use super::*; 129 129 + 130 130 + #[test] 131 131 + fn tree_node_id_round_trips_through_display() { 132 132 + let tree_id = loro::TreeID::new(7, 3); 133 133 + let node_id = NodeId::tree(tree_id); 134 134 + assert_eq!(node_id.as_tree_id(), Some(tree_id)); 135 135 + } 136 136 + 137 137 + #[test] 138 138 + fn tag_and_date_ids_are_stable_without_tree_storage() { 139 139 + let a = NodeId::tag("project"); 140 140 + let b = NodeId::tag("project"); 141 141 + assert_eq!(a, b); 142 142 + assert_eq!(a.as_tree_id(), None); 143 143 + 144 144 + let d = NodeId::date(NaiveDate::from_ymd_opt(2026, 7, 10).unwrap()); 145 145 + assert_eq!(d.to_string(), "date:2026-07-10"); 146 146 + } 147 147 + 148 148 + #[test] 149 149 + fn property_value_accessors_narrow_by_variant() { 150 150 + let due = PropertyValue::Date(NaiveDate::from_ymd_opt(2026, 7, 10).unwrap()); 151 151 + assert_eq!( 152 152 + due.as_date(), 153 153 + Some(NaiveDate::from_ymd_opt(2026, 7, 10).unwrap()) 154 154 + ); 155 155 + assert_eq!(due.as_text(), None); 156 156 + } 157 157 + }

+655

crates/trawler-core/src/index.rs

Reviewed

··· 1 1 + //! The derived in-memory graph: adjacency, backlink sets, and tag/property/ 2 2 + //! date indexes, rebuilt in full from a `LoroDoc`. Every field here is 3 3 + //! disposable — delete it and call `GraphIndex::rebuild` again and querying 4 4 + //! behaves identically (spec: "Derived indexes are disposable"). 5 5 + //! 6 6 + //! See openspec/changes/trawler-mvp specs/block-graph/spec.md 7 7 + //! ("Block references and backlinks", "Lightweight properties", 8 8 + //! "Incremental index maintenance") and tasks.md 2.5/2.6. 9 9 + 10 10 + use std::collections::{HashMap, HashSet}; 11 11 + 12 12 + use chrono::NaiveDate; 13 13 + use loro::{LoroDoc, TreeID}; 14 14 + 15 15 + use crate::graph::{NodeId, PropertyValue}; 16 16 + use crate::outline::Outline; 17 17 + use crate::refs::{parse_references, ReferenceKind}; 18 18 + use crate::storage::OUTLINE_TREE; 19 19 + 20 20 + /// A referenced-but-not-yet-created page (a `[[Name]]`/`#tag` with no 21 21 + /// materialized tree node) is indexed under `NodeId::Tag(name)` rather than 22 22 + /// invented a separate identity for it — see the module doc for why: both 23 23 + /// are "named nodes with no required tree storage" until a real page is 24 24 + /// created at that name, at which point new references resolve to 25 25 + /// `NodeId::Tree` instead. This is an explicit MVP simplification, not a 26 26 + /// modeled reconciliation between the two; revisit in task 8.4 if it causes 27 27 + /// confusion once pages are actually authorable in the UI. 28 28 + #[derive(Debug, Default, Clone, PartialEq)] 29 29 + pub struct GraphIndex { 30 30 + /// parent block -> ordered children, in `NodeId` space (pages included 31 31 + /// as parent-less roots via `adjacency[&None]`... modeled instead as 32 32 + /// `roots` below, since `NodeId` has no natural "no parent" key). 33 33 + pub roots: Vec<TreeID>, 34 34 + pub children: HashMap<TreeID, Vec<TreeID>>, 35 35 + /// Every node this graph knows about, and the blocks that reference it. 36 36 + pub backlinks: HashMap<NodeId, HashSet<TreeID>>, 37 37 + /// Denormalized view of `backlinks` restricted to `NodeId::Tag` targets, 38 38 + /// for direct "blocks tagged X" lookups without filtering the whole map. 39 39 + pub tags: HashMap<String, HashSet<TreeID>>, 40 40 + /// Denormalized view of `backlinks` restricted to `NodeId::Date` targets. 41 41 + pub dates: HashMap<NaiveDate, HashSet<TreeID>>, 42 42 + /// property key -> block -> value, for "blocks with property K" queries. 43 43 + pub properties: HashMap<String, HashMap<TreeID, PropertyValue>>, 44 44 + /// page name (its root block's content) -> page id, for resolving 45 45 + /// `[[Name]]` references to an existing page when one exists. 46 46 + pub pages_by_name: HashMap<String, TreeID>, 47 47 + /// block -> the set of nodes it currently references, so incremental 48 48 + /// re-indexing of that block's content can remove exactly its old 49 49 + /// backlink entries in O(its own ref count) instead of scanning every 50 50 + /// backlink set in the graph. 51 51 + block_outgoing_refs: HashMap<TreeID, HashSet<NodeId>>, 52 52 + /// block -> the set of property keys it currently has set, for the same 53 53 + /// reason (cheap removal on re-index). 54 54 + block_property_keys: HashMap<TreeID, HashSet<String>>, 55 55 + } 56 56 + 57 57 + /// A change to feed into [`GraphIndex::apply`] after the corresponding 58 58 + /// [`Outline`] mutation, so the index stays current without a full 59 59 + /// [`GraphIndex::rebuild`]. See tasks.md 2.6 and the "Incremental index 60 60 + /// maintenance" requirement. 61 61 + /// 62 62 + /// Each variant is scoped to exactly what changed, so `apply` only touches 63 63 + /// the affected block(s)/parent, not the whole graph: 64 64 + /// - `create_block`/`move_subtree`/`reorder_sibling` change a parent's 65 65 + /// child order → `ChildrenChanged(parent)` for every parent whose 66 66 + /// children list changed (old parent *and* new parent, on a move). 67 67 + /// - `create_block`/`split_block`/`merge_block` set a block's text → 68 68 + /// `BlockContentChanged(id)` for every block whose content changed. 69 69 + /// - `set_property` → `BlockPropertiesChanged(id)`. 70 70 + /// - `delete_block`/`merge_block` (which deletes its source) → 71 71 + /// `SubtreeDeleted(id)`. 72 72 + #[derive(Debug, Clone, Copy)] 73 73 + pub enum IndexEvent { 74 74 + ChildrenChanged(Option<TreeID>), 75 75 + BlockContentChanged(TreeID), 76 76 + BlockPropertiesChanged(TreeID), 77 77 + SubtreeDeleted(TreeID), 78 78 + } 79 79 + 80 80 + impl GraphIndex { 81 81 + /// Rebuild the entire index from scratch by walking every node in 82 82 + /// `doc`'s outline tree. O(blocks); see design.md D2/D8 and the Spike 83 83 + /// S3 findings for the cost of this at realistic scale. 84 84 + pub fn rebuild(doc: &LoroDoc) -> Self { 85 85 + let outline = Outline::new(doc); 86 86 + let tree = doc.get_tree(OUTLINE_TREE); 87 87 + let mut index = GraphIndex { 88 88 + roots: tree.roots(), 89 89 + ..Default::default() 90 90 + }; 91 91 + 92 92 + for &root in &index.roots { 93 93 + if let Ok(name) = outline.content(root) { 94 94 + index.pages_by_name.insert(name, root); 95 95 + } 96 96 + } 97 97 + 98 98 + let roots = index.roots.clone(); 99 99 + for root in roots { 100 100 + index.walk_subtree(&outline, &tree, root); 101 101 + } 102 102 + 103 103 + index 104 104 + } 105 105 + 106 106 + fn walk_subtree(&mut self, outline: &Outline, tree: &loro::LoroTree, id: TreeID) { 107 107 + let children = tree.children(id).unwrap_or_default(); 108 108 + if !children.is_empty() { 109 109 + self.children.insert(id, children.clone()); 110 110 + } 111 111 + 112 112 + self.sync_content(outline, id); 113 113 + self.sync_properties(outline, id); 114 114 + 115 115 + for child in children { 116 116 + self.walk_subtree(outline, tree, child); 117 117 + } 118 118 + } 119 119 + 120 120 + /// Incrementally patch the index for a single change, without 121 121 + /// re-walking the graph. See [`IndexEvent`] for which mutation implies 122 122 + /// which event(s). 123 123 + pub fn apply(&mut self, doc: &LoroDoc, event: IndexEvent) { 124 124 + let outline = Outline::new(doc); 125 125 + match event { 126 126 + IndexEvent::ChildrenChanged(parent) => self.sync_children(&outline, parent), 127 127 + IndexEvent::BlockContentChanged(id) => self.sync_content(&outline, id), 128 128 + IndexEvent::BlockPropertiesChanged(id) => self.sync_properties(&outline, id), 129 129 + IndexEvent::SubtreeDeleted(id) => self.remove_subtree(id), 130 130 + } 131 131 + } 132 132 + 133 133 + fn sync_children(&mut self, outline: &Outline, parent: Option<TreeID>) { 134 134 + match parent { 135 135 + Some(p) => { 136 136 + let kids = outline.children(Some(p)); 137 137 + if kids.is_empty() { 138 138 + self.children.remove(&p); 139 139 + } else { 140 140 + self.children.insert(p, kids); 141 141 + } 142 142 + } 143 143 + None => { 144 144 + self.roots = outline.children(None); 145 145 + // Cheap to fully resync: bounded by page count, not block 146 146 + // count, so this stays O(pages) even for a 100k-block graph. 147 147 + self.pages_by_name.clear(); 148 148 + for &root in &self.roots { 149 149 + if let Ok(name) = outline.content(root) { 150 150 + self.pages_by_name.insert(name, root); 151 151 + } 152 152 + } 153 153 + } 154 154 + } 155 155 + } 156 156 + 157 157 + fn sync_content(&mut self, outline: &Outline, id: TreeID) { 158 158 + if let Some(old_targets) = self.block_outgoing_refs.remove(&id) { 159 159 + for target in old_targets { 160 160 + self.unlink_backlink(&target, id); 161 161 + } 162 162 + } 163 163 + 164 164 + let Ok(content) = outline.content(id) else { 165 165 + return; 166 166 + }; 167 167 + let mut new_targets = HashSet::new(); 168 168 + for reference in parse_references(&content) { 169 169 + let target = self.resolve_reference(reference.kind); 170 170 + self.link_backlink(target.clone(), id); 171 171 + new_targets.insert(target); 172 172 + } 173 173 + if !new_targets.is_empty() { 174 174 + self.block_outgoing_refs.insert(id, new_targets); 175 175 + } 176 176 + 177 177 + // A page's own content is its name; keep pages_by_name in sync on 178 178 + // rename without waiting for a `ChildrenChanged(None)` event. 179 179 + if self.roots.contains(&id) { 180 180 + self.pages_by_name.retain(|_, &mut v| v != id); 181 181 + self.pages_by_name.insert(content, id); 182 182 + } 183 183 + } 184 184 + 185 185 + fn sync_properties(&mut self, outline: &Outline, id: TreeID) { 186 186 + if let Some(old_keys) = self.block_property_keys.remove(&id) { 187 187 + for key in old_keys { 188 188 + if let Some(m) = self.properties.get_mut(&key) { 189 189 + m.remove(&id); 190 190 + if m.is_empty() { 191 191 + self.properties.remove(&key); 192 192 + } 193 193 + } 194 194 + } 195 195 + } 196 196 + 197 197 + let Ok(props) = outline.properties(id) else { 198 198 + return; 199 199 + }; 200 200 + let mut keys = HashSet::new(); 201 201 + for (key, value) in props { 202 202 + self.properties 203 203 + .entry(key.clone()) 204 204 + .or_default() 205 205 + .insert(id, value); 206 206 + keys.insert(key); 207 207 + } 208 208 + if !keys.is_empty() { 209 209 + self.block_property_keys.insert(id, keys); 210 210 + } 211 211 + } 212 212 + 213 213 + /// Remove `root` and every descendant currently known to the index (via 214 214 + /// our own `children` map, since by the time this is called the doc has 215 215 + /// already deleted them and can no longer be queried for them). 216 216 + fn remove_subtree(&mut self, root: TreeID) { 217 217 + let mut queue = vec![root]; 218 218 + while let Some(id) = queue.pop() { 219 219 + if let Some(targets) = self.block_outgoing_refs.remove(&id) { 220 220 + for target in targets { 221 221 + self.unlink_backlink(&target, id); 222 222 + } 223 223 + } 224 224 + if let Some(keys) = self.block_property_keys.remove(&id) { 225 225 + for key in keys { 226 226 + if let Some(m) = self.properties.get_mut(&key) { 227 227 + m.remove(&id); 228 228 + if m.is_empty() { 229 229 + self.properties.remove(&key); 230 230 + } 231 231 + } 232 232 + } 233 233 + } 234 234 + if let Some(children) = self.children.remove(&id) { 235 235 + queue.extend(children); 236 236 + } 237 237 + self.roots.retain(|&r| r != id); 238 238 + self.pages_by_name.retain(|_, &mut v| v != id); 239 239 + } 240 240 + } 241 241 + 242 242 + fn link_backlink(&mut self, target: NodeId, id: TreeID) { 243 243 + self.backlinks.entry(target.clone()).or_default().insert(id); 244 244 + match &target { 245 245 + NodeId::Tag(name) => { 246 246 + self.tags.entry(name.clone()).or_default().insert(id); 247 247 + } 248 248 + NodeId::Date(date) => { 249 249 + self.dates.entry(*date).or_default().insert(id); 250 250 + } 251 251 + NodeId::Tree(_) => {} 252 252 + } 253 253 + } 254 254 + 255 255 + fn unlink_backlink(&mut self, target: &NodeId, id: TreeID) { 256 256 + if let Some(set) = self.backlinks.get_mut(target) { 257 257 + set.remove(&id); 258 258 + if set.is_empty() { 259 259 + self.backlinks.remove(target); 260 260 + } 261 261 + } 262 262 + match target { 263 263 + NodeId::Tag(name) => { 264 264 + if let Some(set) = self.tags.get_mut(name) { 265 265 + set.remove(&id); 266 266 + if set.is_empty() { 267 267 + self.tags.remove(name); 268 268 + } 269 269 + } 270 270 + } 271 271 + NodeId::Date(date) => { 272 272 + if let Some(set) = self.dates.get_mut(date) { 273 273 + set.remove(&id); 274 274 + if set.is_empty() { 275 275 + self.dates.remove(date); 276 276 + } 277 277 + } 278 278 + } 279 279 + NodeId::Tree(_) => {} 280 280 + } 281 281 + } 282 282 + 283 283 + /// See the "MVP simplification" note on the struct doc: page references 284 284 + /// resolve to a real `NodeId::Tree` if that page already exists, and 285 285 + /// fall back to `NodeId::Tag(name)` otherwise so referencing an 286 286 + /// as-yet-uncreated page still produces an immediately visible, 287 287 + /// navigable backlink target (spec scenario: "Backlink appears 288 288 + /// immediately"). 289 289 + fn resolve_reference(&self, kind: ReferenceKind) -> NodeId { 290 290 + match kind { 291 291 + ReferenceKind::Page(name) => match self.pages_by_name.get(&name) { 292 292 + Some(&id) => NodeId::tree(id), 293 293 + None => NodeId::tag(name), 294 294 + }, 295 295 + ReferenceKind::Tag(name) => NodeId::tag(name), 296 296 + ReferenceKind::Date(date) => NodeId::date(date), 297 297 + ReferenceKind::Block(id_str) => match TreeID::try_from(id_str.as_str()) { 298 298 + Ok(id) => NodeId::tree(id), 299 299 + Err(_) => NodeId::tag(format!("invalid-block-ref:{id_str}")), 300 300 + }, 301 301 + } 302 302 + } 303 303 + 304 304 + /// Blocks that reference `target`, in the "Backlinks section ... every 305 305 + /// page/zoom view" sense (task 6.4 consumes this). 306 306 + pub fn backlinks_of(&self, target: &NodeId) -> HashSet<TreeID> { 307 307 + self.backlinks.get(target).cloned().unwrap_or_default() 308 308 + } 309 309 + } 310 310 + 311 311 + #[cfg(test)] 312 312 + mod tests { 313 313 + use super::*; 314 314 + use crate::outline::Position; 315 315 + 316 316 + fn doc_with_tree() -> LoroDoc { 317 317 + let doc = LoroDoc::new(); 318 318 + doc.get_tree(OUTLINE_TREE).enable_fractional_index(0); 319 319 + doc 320 320 + } 321 321 + 322 322 + #[test] 323 323 + fn tag_is_navigable_like_a_page() { 324 324 + // spec scenario: "A tag is navigable like a page" 325 325 + let doc = doc_with_tree(); 326 326 + let outline = Outline::new(&doc); 327 327 + let page = outline 328 328 + .create_block(None, Position::Index(0), "Home") 329 329 + .unwrap(); 330 330 + let block = outline 331 331 + .create_block(Some(page), Position::Index(0), "working on #project") 332 332 + .unwrap(); 333 333 + 334 334 + let index = GraphIndex::rebuild(&doc); 335 335 + 336 336 + let backlinks = index.backlinks_of(&NodeId::tag("project")); 337 337 + assert_eq!(backlinks, HashSet::from([block])); 338 338 + assert_eq!(index.tags.get("project"), Some(&HashSet::from([block]))); 339 339 + } 340 340 + 341 341 + #[test] 342 342 + fn backlink_appears_for_reference_to_not_yet_created_page() { 343 343 + // spec scenario: "Backlink appears immediately" 344 344 + let doc = doc_with_tree(); 345 345 + let outline = Outline::new(&doc); 346 346 + let page = outline 347 347 + .create_block(None, Position::Index(0), "Home") 348 348 + .unwrap(); 349 349 + let block = outline 350 350 + .create_block(Some(page), Position::Index(0), "going [[fishing]] tomorrow") 351 351 + .unwrap(); 352 352 + 353 353 + let index = GraphIndex::rebuild(&doc); 354 354 + 355 355 + assert_eq!( 356 356 + index.backlinks_of(&NodeId::tag("fishing")), 357 357 + HashSet::from([block]) 358 358 + ); 359 359 + } 360 360 + 361 361 + #[test] 362 362 + fn page_reference_resolves_to_existing_page_node() { 363 363 + let doc = doc_with_tree(); 364 364 + let outline = Outline::new(&doc); 365 365 + let target_page = outline 366 366 + .create_block(None, Position::Index(0), "Fishing") 367 367 + .unwrap(); 368 368 + let referrer_page = outline 369 369 + .create_block(None, Position::Index(1), "Home") 370 370 + .unwrap(); 371 371 + let block = outline 372 372 + .create_block(Some(referrer_page), Position::Index(0), "see [[Fishing]]") 373 373 + .unwrap(); 374 374 + 375 375 + let index = GraphIndex::rebuild(&doc); 376 376 + 377 377 + assert_eq!( 378 378 + index.backlinks_of(&NodeId::tree(target_page)), 379 379 + HashSet::from([block]) 380 380 + ); 381 381 + } 382 382 + 383 383 + #[test] 384 384 + fn date_reference_is_indexed_by_date() { 385 385 + let doc = doc_with_tree(); 386 386 + let outline = Outline::new(&doc); 387 387 + let page = outline 388 388 + .create_block(None, Position::Index(0), "Home") 389 389 + .unwrap(); 390 390 + let block = outline 391 391 + .create_block(Some(page), Position::Index(0), "due [[2026-07-10]]") 392 392 + .unwrap(); 393 393 + 394 394 + let index = GraphIndex::rebuild(&doc); 395 395 + 396 396 + let date = NaiveDate::from_ymd_opt(2026, 7, 10).unwrap(); 397 397 + assert_eq!(index.dates.get(&date), Some(&HashSet::from([block]))); 398 398 + assert_eq!( 399 399 + index.backlinks_of(&NodeId::date(date)), 400 400 + HashSet::from([block]) 401 401 + ); 402 402 + } 403 403 + 404 404 + #[test] 405 405 + fn property_round_trip_via_index() { 406 406 + // spec scenario: "Property round-trip" 407 407 + let doc = doc_with_tree(); 408 408 + let outline = Outline::new(&doc); 409 409 + let page = outline 410 410 + .create_block(None, Position::Index(0), "Home") 411 411 + .unwrap(); 412 412 + let block = outline 413 413 + .create_block(Some(page), Position::Index(0), "a task") 414 414 + .unwrap(); 415 415 + let due = PropertyValue::Date(NaiveDate::from_ymd_opt(2026, 7, 10).unwrap()); 416 416 + outline.set_property(block, "due", &due).unwrap(); 417 417 + 418 418 + let index = GraphIndex::rebuild(&doc); 419 419 + 420 420 + let by_due = index.properties.get("due").unwrap(); 421 421 + assert_eq!(by_due.get(&block), Some(&due)); 422 422 + } 423 423 + 424 424 + #[test] 425 425 + fn incremental_apply_matches_full_rebuild() { 426 426 + let doc = doc_with_tree(); 427 427 + let outline = Outline::new(&doc); 428 428 + let mut index = GraphIndex::default(); 429 429 + 430 430 + let page = outline 431 431 + .create_block(None, Position::Index(0), "Home") 432 432 + .unwrap(); 433 433 + index.apply(&doc, IndexEvent::ChildrenChanged(None)); 434 434 + index.apply(&doc, IndexEvent::BlockContentChanged(page)); 435 435 + 436 436 + let a = outline 437 437 + .create_block(Some(page), Position::Index(0), "loves #hiking") 438 438 + .unwrap(); 439 439 + index.apply(&doc, IndexEvent::ChildrenChanged(Some(page))); 440 440 + index.apply(&doc, IndexEvent::BlockContentChanged(a)); 441 441 + 442 442 + let b = outline 443 443 + .create_block(Some(page), Position::Index(1), "see [[Home]]") 444 444 + .unwrap(); 445 445 + index.apply(&doc, IndexEvent::ChildrenChanged(Some(page))); 446 446 + index.apply(&doc, IndexEvent::BlockContentChanged(b)); 447 447 + 448 448 + outline 449 449 + .set_property(b, "priority", &PropertyValue::Number(1.0)) 450 450 + .unwrap(); 451 451 + index.apply(&doc, IndexEvent::BlockPropertiesChanged(b)); 452 452 + 453 453 + // Edit `a`'s content: drop the #hiking tag, add a date instead. 454 454 + outline.delete_block(a).unwrap(); 455 455 + index.apply(&doc, IndexEvent::SubtreeDeleted(a)); 456 456 + index.apply(&doc, IndexEvent::ChildrenChanged(Some(page))); 457 457 + let a2 = outline 458 458 + .create_block(Some(page), Position::Index(0), "due [[2026-07-10]]") 459 459 + .unwrap(); 460 460 + index.apply(&doc, IndexEvent::ChildrenChanged(Some(page))); 461 461 + index.apply(&doc, IndexEvent::BlockContentChanged(a2)); 462 462 + 463 463 + let rebuilt = GraphIndex::rebuild(&doc); 464 464 + assert_eq!(index, rebuilt); 465 465 + } 466 466 + 467 467 + #[test] 468 468 + fn adjacency_preserves_outline_order() { 469 469 + // spec scenario: "Subtree move" relies on adjacency preserving 470 470 + // relative order; check it directly here too. 471 471 + let doc = doc_with_tree(); 472 472 + let outline = Outline::new(&doc); 473 473 + let page = outline 474 474 + .create_block(None, Position::Index(0), "Home") 475 475 + .unwrap(); 476 476 + let a = outline 477 477 + .create_block(Some(page), Position::Index(0), "a") 478 478 + .unwrap(); 479 479 + let b = outline 480 480 + .create_block(Some(page), Position::Index(1), "b") 481 481 + .unwrap(); 482 482 + let c = outline 483 483 + .create_block(Some(page), Position::Index(2), "c") 484 484 + .unwrap(); 485 485 + 486 486 + let index = GraphIndex::rebuild(&doc); 487 487 + 488 488 + assert_eq!(index.children.get(&page), Some(&vec![a, b, c])); 489 489 + } 490 490 + 491 491 + /// spec scenario: "Incremental equals rebuild" — a randomized sequence 492 492 + /// of at least 1,000 edit operations (create, edit, move, delete, tag, 493 493 + /// reference) must leave the incrementally-maintained index identical 494 494 + /// to one rebuilt from scratch afterward. 495 495 + #[test] 496 496 + fn randomized_edit_sequence_incremental_equals_rebuild() { 497 497 + use rand::rngs::StdRng; 498 498 + use rand::{RngExt, SeedableRng}; 499 499 + 500 500 + const OPS: usize = 2000; 501 501 + let mut rng = StdRng::seed_from_u64(0xC0FFEE); 502 502 + 503 503 + let doc = doc_with_tree(); 504 504 + let outline = Outline::new(&doc); 505 505 + let tree = doc.get_tree(OUTLINE_TREE); 506 506 + let mut index = GraphIndex::default(); 507 507 + let mut live: Vec<TreeID> = Vec::new(); 508 508 + 509 509 + // Seed with one page so most ops have something to work with. 510 510 + let seed_page = outline 511 511 + .create_block(None, Position::Index(0), "Seed") 512 512 + .unwrap(); 513 513 + index.apply(&doc, IndexEvent::ChildrenChanged(None)); 514 514 + index.apply(&doc, IndexEvent::BlockContentChanged(seed_page)); 515 515 + live.push(seed_page); 516 516 + 517 517 + let is_descendant = |tree: &loro::LoroTree, ancestor: TreeID, id: TreeID| -> bool { 518 518 + let mut stack = tree.children(ancestor).unwrap_or_default(); 519 519 + while let Some(cur) = stack.pop() { 520 520 + if cur == id { 521 521 + return true; 522 522 + } 523 523 + stack.extend(tree.children(cur).unwrap_or_default()); 524 524 + } 525 525 + false 526 526 + }; 527 527 + 528 528 + let random_content = |rng: &mut StdRng, live: &[TreeID]| -> String { 529 529 + match rng.random_range(0..6) { 530 530 + 0 => format!("plain text {}", rng.random_range(0..1000)), 531 531 + 1 => format!("tagged #topic{}", rng.random_range(0..8)), 532 532 + 2 => "references [[Seed]]".to_string(), 533 533 + 3 => format!("references [[Page{}]]", rng.random_range(0..8)), 534 534 + 4 => "due [[2026-07-10]]".to_string(), 535 535 + _ => { 536 536 + let id = live[rng.random_range(0..live.len())]; 537 537 + format!("blockref (({id}))") 538 538 + } 539 539 + } 540 540 + }; 541 541 + 542 542 + for _ in 0..OPS { 543 543 + if live.is_empty() { 544 544 + let page = outline 545 545 + .create_block(None, Position::Index(0), "Home") 546 546 + .unwrap(); 547 547 + index.apply(&doc, IndexEvent::ChildrenChanged(None)); 548 548 + index.apply(&doc, IndexEvent::BlockContentChanged(page)); 549 549 + live.push(page); 550 550 + continue; 551 551 + } 552 552 + 553 553 + match rng.random_range(0..100) { 554 554 + // Create (35%) 555 555 + 0..=34 => { 556 556 + let make_page = rng.random_range(0..10) == 0; 557 557 + let parent = if make_page { 558 558 + None 559 559 + } else { 560 560 + live.get(rng.random_range(0..live.len())).copied() 561 561 + }; 562 562 + let n_children = outline.children(parent).len(); 563 563 + let content = random_content(&mut rng, &live); 564 564 + let id = outline 565 565 + .create_block(parent, Position::Index(n_children), &content) 566 566 + .unwrap(); 567 567 + index.apply(&doc, IndexEvent::ChildrenChanged(parent)); 568 568 + index.apply(&doc, IndexEvent::BlockContentChanged(id)); 569 569 + live.push(id); 570 570 + } 571 571 + // Edit content (25%) 572 572 + 35..=59 => { 573 573 + let id = live[rng.random_range(0..live.len())]; 574 574 + let content = random_content(&mut rng, &live); 575 575 + outline.set_content(id, &content).unwrap(); 576 576 + index.apply(&doc, IndexEvent::BlockContentChanged(id)); 577 577 + } 578 578 + // Set property (10%) 579 579 + 60..=69 => { 580 580 + let id = live[rng.random_range(0..live.len())]; 581 581 + let key = ["due", "priority", "done", "note"][rng.random_range(0..4)]; 582 582 + let value = match rng.random_range(0..4) { 583 583 + 0 => PropertyValue::Text(format!("v{}", rng.random_range(0..100))), 584 584 + 1 => PropertyValue::Number(rng.random_range(0..100) as f64), 585 585 + 2 => PropertyValue::Bool(rng.random_range(0..2) == 0), 586 586 + _ => PropertyValue::Date(NaiveDate::from_ymd_opt(2026, 7, 10).unwrap()), 587 587 + }; 588 588 + outline.set_property(id, key, &value).unwrap(); 589 589 + index.apply(&doc, IndexEvent::BlockPropertiesChanged(id)); 590 590 + } 591 591 + // Move subtree (10%) 592 592 + 70..=79 => { 593 593 + let id = live[rng.random_range(0..live.len())]; 594 594 + let old_parent = outline.parent(id); 595 595 + let candidate = live[rng.random_range(0..live.len())]; 596 596 + let new_parent = if candidate == id || is_descendant(&tree, id, candidate) { 597 597 + None 598 598 + } else { 599 599 + Some(candidate) 600 600 + }; 601 601 + if new_parent == Some(id) { 602 602 + continue; 603 603 + } 604 604 + let n_children = outline.children(new_parent).len(); 605 605 + if outline 606 606 + .move_subtree(id, new_parent, Position::Index(n_children)) 607 607 + .is_ok() 608 608 + { 609 609 + index.apply(&doc, IndexEvent::ChildrenChanged(old_parent)); 610 610 + index.apply(&doc, IndexEvent::ChildrenChanged(new_parent)); 611 611 + } 612 612 + } 613 613 + // Reorder sibling (10%) 614 614 + 80..=89 => { 615 615 + let id = live[rng.random_range(0..live.len())]; 616 616 + let parent = outline.parent(id); 617 617 + let siblings = outline.children(parent); 618 618 + if siblings.len() >= 2 { 619 619 + let sibling = siblings[rng.random_range(0..siblings.len())]; 620 620 + if sibling != id 621 621 + && outline 622 622 + .reorder_sibling(id, Position::After(sibling)) 623 623 + .is_ok() 624 624 + { 625 625 + index.apply(&doc, IndexEvent::ChildrenChanged(parent)); 626 626 + } 627 627 + } 628 628 + } 629 629 + // Delete (10%) 630 630 + _ => { 631 631 + let idx = rng.random_range(0..live.len()); 632 632 + let id = live[idx]; 633 633 + let parent = outline.parent(id); 634 634 + let mut descendants = vec![id]; 635 635 + let mut stack = tree.children(id).unwrap_or_default(); 636 636 + while let Some(cur) = stack.pop() { 637 637 + descendants.push(cur); 638 638 + stack.extend(tree.children(cur).unwrap_or_default()); 639 639 + } 640 640 + if outline.delete_block(id).is_ok() { 641 641 + index.apply(&doc, IndexEvent::SubtreeDeleted(id)); 642 642 + index.apply(&doc, IndexEvent::ChildrenChanged(parent)); 643 643 + live.retain(|b| !descendants.contains(b)); 644 644 + } 645 645 + } 646 646 + } 647 647 + } 648 648 + 649 649 + let rebuilt = GraphIndex::rebuild(&doc); 650 650 + assert_eq!( 651 651 + index, rebuilt, 652 652 + "incremental index diverged from full rebuild after {OPS} randomized ops" 653 653 + ); 654 654 + } 655 655 + }

+6

crates/trawler-core/src/lib.rs

Reviewed

··· 1 1 //! Core block-graph engine: storage, indexes, queries, search. No UI dependencies. 2 2 3 3 + pub mod graph; 4 4 + pub mod index; 5 5 + pub mod outline; 6 6 + pub mod properties; 3 7 pub mod query_spike; 8 8 + pub mod refs; 9 9 + pub mod storage; 4 10 5 11 pub fn placeholder() -> &'static str { 6 12 "trawler-core"

+364

crates/trawler-core/src/outline.rs

Reviewed

··· 1 1 + //! Outline tree operations: create/delete/split/merge blocks, move 2 2 + //! subtrees, reorder siblings. Each op is one or a few calls into the Loro 3 3 + //! tree/text/map handlers, which Loro batches into a single transaction at 4 4 + //! the next `doc.commit()` (triggered implicitly by export/import calls 5 5 + //! elsewhere, e.g. `GraphStorage::persist_update`). 6 6 + //! 7 7 + //! See openspec/changes/trawler-mvp specs/block-graph/spec.md 8 8 + //! ("Outline tree operations"). 9 9 + 10 10 + use loro::{LoroDoc, LoroResult, TreeID, TreeParentId}; 11 11 + 12 12 + use crate::storage::OUTLINE_TREE; 13 13 + 14 14 + /// Where to place a newly created or moved block relative to its parent. 15 15 + #[derive(Debug, Clone, Copy)] 16 16 + pub enum Position { 17 17 + /// As the child at the given index (0 = first child). 18 18 + Index(usize), 19 19 + /// Immediately after the given sibling. 20 20 + After(TreeID), 21 21 + /// Immediately before the given sibling. 22 22 + Before(TreeID), 23 23 + } 24 24 + 25 25 + /// Thin wrapper over a `LoroDoc`'s outline tree exposing outline-editing 26 26 + /// operations in outline terms (blocks, positions, content) rather than raw 27 27 + /// tree/text/map calls. 28 28 + pub struct Outline<'a> { 29 29 + doc: &'a LoroDoc, 30 30 + } 31 31 + 32 32 + impl<'a> Outline<'a> { 33 33 + pub fn new(doc: &'a LoroDoc) -> Self { 34 34 + Self { doc } 35 35 + } 36 36 + 37 37 + fn tree(&self) -> loro::LoroTree { 38 38 + self.doc.get_tree(OUTLINE_TREE) 39 39 + } 40 40 + 41 41 + /// Create a new block with `content`, under `parent` (`None` = a new 42 42 + /// page at the root) at `position`. 43 43 + pub fn create_block( 44 44 + &self, 45 45 + parent: Option<TreeID>, 46 46 + position: Position, 47 47 + content: &str, 48 48 + ) -> LoroResult<TreeID> { 49 49 + let tree = self.tree(); 50 50 + let id = match position { 51 51 + Position::Index(ix) => tree.create_at(to_parent(parent), ix)?, 52 52 + Position::After(after) => { 53 53 + let id = tree.create(to_parent(parent))?; 54 54 + tree.mov_after(id, after)?; 55 55 + id 56 56 + } 57 57 + Position::Before(before) => { 58 58 + let id = tree.create(to_parent(parent))?; 59 59 + tree.mov_before(id, before)?; 60 60 + id 61 61 + } 62 62 + }; 63 63 + let meta = tree.get_meta(id)?; 64 64 + meta.insert( 65 65 + "block_type", 66 66 + if parent.is_none() { "page" } else { "block" }, 67 67 + )?; 68 68 + let text = meta.ensure_mergeable_text("content")?; 69 69 + if !content.is_empty() { 70 70 + text.insert(0, content)?; 71 71 + } 72 72 + Ok(id) 73 73 + } 74 74 + 75 75 + /// Delete a block and its entire subtree. 76 76 + pub fn delete_block(&self, id: TreeID) -> LoroResult<()> { 77 77 + self.tree().delete(id) 78 78 + } 79 79 + 80 80 + /// Split `id`'s content at `char_offset` (a Unicode scalar offset): the 81 81 + /// text before the offset stays on `id`, the text from the offset 82 82 + /// onward moves to a new sibling created immediately after `id`. 83 83 + /// Returns the new block's id. 84 84 + pub fn split_block(&self, id: TreeID, char_offset: usize) -> LoroResult<TreeID> { 85 85 + let tree = self.tree(); 86 86 + let meta = tree.get_meta(id)?; 87 87 + let text = meta.ensure_mergeable_text("content")?; 88 88 + let full = text.to_string(); 89 89 + let tail: String = full.chars().skip(char_offset).collect(); 90 90 + 91 91 + if !tail.is_empty() { 92 92 + text.delete(char_offset, text.len_unicode() - char_offset)?; 93 93 + } 94 94 + 95 95 + let parent = tree.parent(id).unwrap_or(TreeParentId::Root); 96 96 + let new_id = tree.create(parent)?; 97 97 + tree.mov_after(new_id, id)?; 98 98 + let new_meta = tree.get_meta(new_id)?; 99 99 + new_meta.insert("block_type", "block")?; 100 100 + let new_text = new_meta.ensure_mergeable_text("content")?; 101 101 + if !tail.is_empty() { 102 102 + new_text.insert(0, &tail)?; 103 103 + } 104 104 + Ok(new_id) 105 105 + } 106 106 + 107 107 + /// Merge `id`'s content onto the end of `into`'s content, then delete 108 108 + /// `id`. `id` must currently have no children (callers should refuse 109 109 + /// merge-with-children in the UI, same as Logseq). 110 110 + pub fn merge_block(&self, id: TreeID, into: TreeID) -> LoroResult<()> { 111 111 + let tree = self.tree(); 112 112 + let source_meta = tree.get_meta(id)?; 113 113 + let source_text = source_meta.ensure_mergeable_text("content")?; 114 114 + let source_content = source_text.to_string(); 115 115 + 116 116 + let target_meta = tree.get_meta(into)?; 117 117 + let target_text = target_meta.ensure_mergeable_text("content")?; 118 118 + if !source_content.is_empty() { 119 119 + target_text.insert(target_text.len_unicode(), &source_content)?; 120 120 + } 121 121 + 122 122 + tree.delete(id) 123 123 + } 124 124 + 125 125 + /// Move `id` (and its subtree) to be a child of `new_parent` at 126 126 + /// `position`. 127 127 + pub fn move_subtree( 128 128 + &self, 129 129 + id: TreeID, 130 130 + new_parent: Option<TreeID>, 131 131 + position: Position, 132 132 + ) -> LoroResult<()> { 133 133 + let tree = self.tree(); 134 134 + match position { 135 135 + Position::Index(ix) => tree.mov_to(id, to_parent(new_parent), ix)?, 136 136 + Position::After(after) => tree.mov_after(id, after)?, 137 137 + Position::Before(before) => tree.mov_before(id, before)?, 138 138 + } 139 139 + Ok(()) 140 140 + } 141 141 + 142 142 + /// Reorder `id` among its current siblings without changing parent. 143 143 + pub fn reorder_sibling(&self, id: TreeID, position: Position) -> LoroResult<()> { 144 144 + let tree = self.tree(); 145 145 + let parent = tree.parent(id).unwrap_or(TreeParentId::Root); 146 146 + match position { 147 147 + Position::Index(ix) => tree.mov_to(id, parent, ix)?, 148 148 + Position::After(after) => tree.mov_after(id, after)?, 149 149 + Position::Before(before) => tree.mov_before(id, before)?, 150 150 + } 151 151 + Ok(()) 152 152 + } 153 153 + 154 154 + /// Ordered children of `parent` (`None` = pages at the root). 155 155 + pub fn children(&self, parent: Option<TreeID>) -> Vec<TreeID> { 156 156 + match parent { 157 157 + Some(id) => self.tree().children(id).unwrap_or_default(), 158 158 + None => self.tree().roots(), 159 159 + } 160 160 + } 161 161 + 162 162 + /// Current text content of a block. 163 163 + pub fn content(&self, id: TreeID) -> LoroResult<String> { 164 164 + let meta = self.tree().get_meta(id)?; 165 165 + Ok(meta.ensure_mergeable_text("content")?.to_string()) 166 166 + } 167 167 + 168 168 + /// Replace a block's entire content. 169 169 + pub fn set_content(&self, id: TreeID, content: &str) -> LoroResult<()> { 170 170 + let meta = self.tree().get_meta(id)?; 171 171 + let text = meta.ensure_mergeable_text("content")?; 172 172 + let len = text.len_unicode(); 173 173 + if len > 0 { 174 174 + text.delete(0, len)?; 175 175 + } 176 176 + if !content.is_empty() { 177 177 + text.insert(0, content)?; 178 178 + } 179 179 + Ok(()) 180 180 + } 181 181 + 182 182 + /// The current parent of `id` (`None` if it's a page at the root). 183 183 + pub fn parent(&self, id: TreeID) -> Option<TreeID> { 184 184 + match self.tree().parent(id)? { 185 185 + TreeParentId::Node(p) => Some(p), 186 186 + _ => None, 187 187 + } 188 188 + } 189 189 + 190 190 + /// Set (or overwrite) a typed property on a block. 191 191 + pub fn set_property( 192 192 + &self, 193 193 + id: TreeID, 194 194 + key: &str, 195 195 + value: &crate::graph::PropertyValue, 196 196 + ) -> LoroResult<()> { 197 197 + let meta = self.tree().get_meta(id)?; 198 198 + let props = meta.ensure_mergeable_map("properties")?; 199 199 + props.insert(key, crate::properties::encode(value))?; 200 200 + Ok(()) 201 201 + } 202 202 + 203 203 + /// All properties currently set on a block. 204 204 + pub fn properties( 205 205 + &self, 206 206 + id: TreeID, 207 207 + ) -> LoroResult<std::collections::HashMap<String, crate::graph::PropertyValue>> { 208 208 + let meta = self.tree().get_meta(id)?; 209 209 + let props = meta.ensure_mergeable_map("properties")?; 210 210 + let mut out = std::collections::HashMap::new(); 211 211 + for key in props.keys() { 212 212 + if let Some(loro::ValueOrContainer::Value(loro::LoroValue::String(s))) = props.get(&key) 213 213 + { 214 214 + if let Some(value) = crate::properties::decode(&s) { 215 215 + out.insert(key.to_string(), value); 216 216 + } 217 217 + } 218 218 + } 219 219 + Ok(out) 220 220 + } 221 221 + } 222 222 + 223 223 + fn to_parent(id: Option<TreeID>) -> TreeParentId { 224 224 + match id { 225 225 + Some(id) => TreeParentId::Node(id), 226 226 + None => TreeParentId::Root, 227 227 + } 228 228 + } 229 229 + 230 230 + #[cfg(test)] 231 231 + mod tests { 232 232 + use super::*; 233 233 + 234 234 + fn doc_with_tree() -> LoroDoc { 235 235 + let doc = LoroDoc::new(); 236 236 + doc.get_tree(OUTLINE_TREE).enable_fractional_index(0); 237 237 + doc 238 238 + } 239 239 + 240 240 + #[test] 241 241 + fn create_and_read_back_content() { 242 242 + let doc = doc_with_tree(); 243 243 + let outline = Outline::new(&doc); 244 244 + let page = outline 245 245 + .create_block(None, Position::Index(0), "My Page") 246 246 + .unwrap(); 247 247 + assert_eq!(outline.content(page).unwrap(), "My Page"); 248 248 + assert_eq!(outline.children(None), vec![page]); 249 249 + } 250 250 + 251 251 + #[test] 252 252 + fn split_block_moves_tail_to_new_sibling() { 253 253 + let doc = doc_with_tree(); 254 254 + let outline = Outline::new(&doc); 255 255 + let page = outline.create_block(None, Position::Index(0), "").unwrap(); 256 256 + let block = outline 257 257 + .create_block(Some(page), Position::Index(0), "hello world") 258 258 + .unwrap(); 259 259 + 260 260 + let new_id = outline.split_block(block, 5).unwrap(); 261 261 + 262 262 + assert_eq!(outline.content(block).unwrap(), "hello"); 263 263 + assert_eq!(outline.content(new_id).unwrap(), " world"); 264 264 + assert_eq!(outline.children(Some(page)), vec![block, new_id]); 265 265 + } 266 266 + 267 267 + #[test] 268 268 + fn merge_block_appends_and_deletes_source() { 269 269 + let doc = doc_with_tree(); 270 270 + let outline = Outline::new(&doc); 271 271 + let page = outline.create_block(None, Position::Index(0), "").unwrap(); 272 272 + let a = outline 273 273 + .create_block(Some(page), Position::Index(0), "hello") 274 274 + .unwrap(); 275 275 + let b = outline 276 276 + .create_block(Some(page), Position::Index(1), " world") 277 277 + .unwrap(); 278 278 + 279 279 + outline.merge_block(b, a).unwrap(); 280 280 + 281 281 + assert_eq!(outline.content(a).unwrap(), "hello world"); 282 282 + assert_eq!(outline.children(Some(page)), vec![a]); 283 283 + } 284 284 + 285 285 + #[test] 286 286 + fn move_subtree_preserves_descendant_order() { 287 287 + let doc = doc_with_tree(); 288 288 + let outline = Outline::new(&doc); 289 289 + let page_a = outline.create_block(None, Position::Index(0), "A").unwrap(); 290 290 + let page_b = outline.create_block(None, Position::Index(1), "B").unwrap(); 291 291 + let parent = outline 292 292 + .create_block(Some(page_a), Position::Index(0), "parent") 293 293 + .unwrap(); 294 294 + let c1 = outline 295 295 + .create_block(Some(parent), Position::Index(0), "c1") 296 296 + .unwrap(); 297 297 + let c2 = outline 298 298 + .create_block(Some(parent), Position::Index(1), "c2") 299 299 + .unwrap(); 300 300 + let c3 = outline 301 301 + .create_block(Some(parent), Position::Index(2), "c3") 302 302 + .unwrap(); 303 303 + 304 304 + outline 305 305 + .move_subtree(parent, Some(page_b), Position::Index(0)) 306 306 + .unwrap(); 307 307 + 308 308 + assert_eq!(outline.children(Some(page_a)), Vec::<TreeID>::new()); 309 309 + assert_eq!(outline.children(Some(page_b)), vec![parent]); 310 310 + assert_eq!(outline.children(Some(parent)), vec![c1, c2, c3]); 311 311 + } 312 312 + 313 313 + #[test] 314 314 + fn delete_block_removes_subtree() { 315 315 + let doc = doc_with_tree(); 316 316 + let outline = Outline::new(&doc); 317 317 + let page = outline.create_block(None, Position::Index(0), "").unwrap(); 318 318 + let parent = outline 319 319 + .create_block(Some(page), Position::Index(0), "parent") 320 320 + .unwrap(); 321 321 + outline 322 322 + .create_block(Some(parent), Position::Index(0), "child") 323 323 + .unwrap(); 324 324 + 325 325 + outline.delete_block(parent).unwrap(); 326 326 + 327 327 + assert_eq!(outline.children(Some(page)), Vec::<TreeID>::new()); 328 328 + } 329 329 + 330 330 + #[test] 331 331 + fn set_property_round_trips_typed_value() { 332 332 + use crate::graph::PropertyValue; 333 333 + 334 334 + let doc = doc_with_tree(); 335 335 + let outline = Outline::new(&doc); 336 336 + let page = outline.create_block(None, Position::Index(0), "").unwrap(); 337 337 + let block = outline 338 338 + .create_block(Some(page), Position::Index(0), "task") 339 339 + .unwrap(); 340 340 + 341 341 + let due = PropertyValue::Date(chrono::NaiveDate::from_ymd_opt(2026, 7, 10).unwrap()); 342 342 + outline.set_property(block, "due", &due).unwrap(); 343 343 + 344 344 + let props = outline.properties(block).unwrap(); 345 345 + assert_eq!(props.get("due"), Some(&due)); 346 346 + } 347 347 + 348 348 + #[test] 349 349 + fn reorder_sibling_via_after() { 350 350 + let doc = doc_with_tree(); 351 351 + let outline = Outline::new(&doc); 352 352 + let page = outline.create_block(None, Position::Index(0), "").unwrap(); 353 353 + let a = outline 354 354 + .create_block(Some(page), Position::Index(0), "a") 355 355 + .unwrap(); 356 356 + let b = outline 357 357 + .create_block(Some(page), Position::Index(1), "b") 358 358 + .unwrap(); 359 359 + 360 360 + outline.reorder_sibling(a, Position::After(b)).unwrap(); 361 361 + 362 362 + assert_eq!(outline.children(Some(page)), vec![b, a]); 363 363 + } 364 364 + }

+69

crates/trawler-core/src/properties.rs

Reviewed

··· 1 1 + //! Encoding of typed [`PropertyValue`]s into Loro's scalar value space. 2 2 + //! 3 3 + //! Loro maps store `LoroValue` (null/bool/f64/string/list/map/container), 4 4 + //! not arbitrary Rust enums, so each property is encoded as a single 5 5 + //! type-tagged string (e.g. `"d:2026-07-10"` for a date). This keeps 6 6 + //! properties flat (one Loro map entry per property, no nested containers) 7 7 + //! at the cost of the value itself always going through a string. That's a 8 8 + //! fine trade for MVP property volumes; revisit if property values need to 9 9 + //! be binary or independently CRDT-mergeable. 10 10 + 11 11 + use chrono::NaiveDate; 12 12 + 13 13 + use crate::graph::{NodeId, PropertyValue}; 14 14 + 15 15 + pub fn encode(value: &PropertyValue) -> String { 16 16 + match value { 17 17 + PropertyValue::Text(s) => format!("t:{s}"), 18 18 + PropertyValue::Number(n) => format!("n:{n}"), 19 19 + PropertyValue::Date(d) => format!("d:{d}"), 20 20 + PropertyValue::Bool(b) => format!("b:{b}"), 21 21 + PropertyValue::Ref(id) => format!("r:{id}"), 22 22 + } 23 23 + } 24 24 + 25 25 + pub fn decode(encoded: &str) -> Option<PropertyValue> { 26 26 + let (tag, rest) = encoded.split_once(':')?; 27 27 + match tag { 28 28 + "t" => Some(PropertyValue::Text(rest.to_string())), 29 29 + "n" => rest.parse::<f64>().ok().map(PropertyValue::Number), 30 30 + "d" => NaiveDate::parse_from_str(rest, "%Y-%m-%d") 31 31 + .ok() 32 32 + .map(PropertyValue::Date), 33 33 + "b" => rest.parse::<bool>().ok().map(PropertyValue::Bool), 34 34 + "r" => Some(PropertyValue::Ref(decode_node_id(rest))), 35 35 + _ => None, 36 36 + } 37 37 + } 38 38 + 39 39 + fn decode_node_id(s: &str) -> NodeId { 40 40 + if let Some(name) = s.strip_prefix("tag:") { 41 41 + return NodeId::tag(name); 42 42 + } 43 43 + if let Some(date) = s.strip_prefix("date:") { 44 44 + if let Ok(d) = NaiveDate::parse_from_str(date, "%Y-%m-%d") { 45 45 + return NodeId::date(d); 46 46 + } 47 47 + } 48 48 + NodeId::Tree(s.to_string()) 49 49 + } 50 50 + 51 51 + #[cfg(test)] 52 52 + mod tests { 53 53 + use super::*; 54 54 + 55 55 + #[test] 56 56 + fn round_trips_every_property_variant() { 57 57 + let values = vec![ 58 58 + PropertyValue::Text("hello".into()), 59 59 + PropertyValue::Number(3.5), 60 60 + PropertyValue::Date(NaiveDate::from_ymd_opt(2026, 7, 10).unwrap()), 61 61 + PropertyValue::Bool(true), 62 62 + PropertyValue::Ref(NodeId::tag("project")), 63 63 + ]; 64 64 + for value in values { 65 65 + let encoded = encode(&value); 66 66 + assert_eq!(decode(&encoded), Some(value)); 67 67 + } 68 68 + } 69 69 + }

+304

crates/trawler-core/src/refs.rs

Reviewed

··· 1 1 + //! Inline reference parsing: pull `[[page]]`, `#tag`, `[[2026-07-10]]` 2 2 + //! date, and `((block-id))` references out of a block's Markdown content. 3 3 + //! 4 4 + //! See openspec/changes/trawler-mvp specs/block-graph/spec.md 5 5 + //! ("Block references and backlinks") and design.md D7: node references 6 6 + //! SHALL NOT be parsed inside code spans or fenced code blocks. 7 7 + //! 8 8 + //! This module only *extracts* reference spans from raw text; resolving a 9 9 + //! referenced name/date to a graph `NodeId` (auto-creating a page node the 10 10 + //! first time it's referenced, etc.) is index-building's job (tasks 11 11 + //! 2.5/2.6), not the parser's. 12 12 + 13 13 + use chrono::NaiveDate; 14 14 + 15 15 + /// A single inline reference found in a block's content, with its byte 16 16 + /// range in the original string (for highlighting/click-to-navigate). 17 17 + #[derive(Debug, Clone, PartialEq)] 18 18 + pub struct ReferenceSpan { 19 19 + pub range: std::ops::Range<usize>, 20 20 + pub kind: ReferenceKind, 21 21 + } 22 22 + 23 23 + #[derive(Debug, Clone, PartialEq)] 24 24 + pub enum ReferenceKind { 25 25 + /// `[[Page Name]]` where the name doesn't parse as a date. 26 26 + Page(String), 27 27 + /// `#tag` or `#[[multi word tag]]`. 28 28 + Tag(String), 29 29 + /// `[[2026-07-10]]` — a `[[...]]` reference whose name is an ISO date. 30 30 + Date(NaiveDate), 31 31 + /// `((tree-id))` — a reference to a specific block by its stable id. 32 32 + Block(String), 33 33 + } 34 34 + 35 35 + /// Extract all inline references from `text`, skipping fenced code blocks 36 36 + /// and inline code spans. 37 37 + pub fn parse_references(text: &str) -> Vec<ReferenceSpan> { 38 38 + let mut spans = Vec::new(); 39 39 + let bytes = text.as_bytes(); 40 40 + let mut i = 0usize; 41 41 + let mut in_fence = false; 42 42 + let mut fence_marker: u8 = 0; 43 43 + 44 44 + while i < bytes.len() { 45 45 + // Fenced code block toggling: a line whose first non-space chars 46 46 + // are ``` or ~~~ (3+) toggles fence state for everything after it. 47 47 + if is_line_start(bytes, i) { 48 48 + if let Some((marker, len)) = fence_open_at(bytes, i) { 49 49 + if !in_fence { 50 50 + in_fence = true; 51 51 + fence_marker = marker; 52 52 + } else if marker == fence_marker { 53 53 + in_fence = false; 54 54 + } 55 55 + i += len; 56 56 + continue; 57 57 + } 58 58 + } 59 59 + 60 60 + if in_fence { 61 61 + i += 1; 62 62 + continue; 63 63 + } 64 64 + 65 65 + // Inline code span: backtick run ... matching backtick run. 66 66 + if bytes[i] == b'`' { 67 67 + let run_len = run_length(bytes, i, b'`'); 68 68 + if let Some(close) = find_backtick_close(bytes, i + run_len, run_len) { 69 69 + i = close + run_len; 70 70 + continue; 71 71 + } 72 72 + // Unmatched backtick run: treat as plain text, keep scanning. 73 73 + i += run_len; 74 74 + continue; 75 75 + } 76 76 + 77 77 + if let Some((span, consumed)) = try_match_reference(text, bytes, i) { 78 78 + spans.push(span); 79 79 + i += consumed; 80 80 + continue; 81 81 + } 82 82 + 83 83 + i += 1; 84 84 + } 85 85 + 86 86 + spans 87 87 + } 88 88 + 89 89 + fn is_line_start(bytes: &[u8], i: usize) -> bool { 90 90 + i == 0 || bytes[i - 1] == b'\n' 91 91 + } 92 92 + 93 93 + /// If a fenced-code-block delimiter (```` ``` ```` or `~~~`, 3+ chars, 94 94 + /// possibly indented) starts at `i`, return the marker byte and how many 95 95 + /// bytes to skip (through end of that line). 96 96 + fn fence_open_at(bytes: &[u8], i: usize) -> Option<(u8, usize)> { 97 97 + let mut j = i; 98 98 + while j < bytes.len() && bytes[j] == b' ' { 99 99 + j += 1; 100 100 + } 101 101 + if j >= bytes.len() { 102 102 + return None; 103 103 + } 104 104 + let marker = bytes[j]; 105 105 + if marker != b'`' && marker != b'~' { 106 106 + return None; 107 107 + } 108 108 + let run_len = run_length(bytes, j, marker); 109 109 + if run_len < 3 { 110 110 + return None; 111 111 + } 112 112 + let line_end = bytes[j..] 113 113 + .iter() 114 114 + .position(|&b| b == b'\n') 115 115 + .map(|p| j + p + 1) 116 116 + .unwrap_or(bytes.len()); 117 117 + Some((marker, line_end - i)) 118 118 + } 119 119 + 120 120 + fn run_length(bytes: &[u8], start: usize, ch: u8) -> usize { 121 121 + let mut n = 0; 122 122 + while start + n < bytes.len() && bytes[start + n] == ch { 123 123 + n += 1; 124 124 + } 125 125 + n 126 126 + } 127 127 + 128 128 + fn find_backtick_close(bytes: &[u8], from: usize, run_len: usize) -> Option<usize> { 129 129 + let mut j = from; 130 130 + while j < bytes.len() { 131 131 + if bytes[j] == b'`' { 132 132 + let n = run_length(bytes, j, b'`'); 133 133 + if n == run_len { 134 134 + return Some(j); 135 135 + } 136 136 + j += n; 137 137 + } else { 138 138 + j += 1; 139 139 + } 140 140 + } 141 141 + None 142 142 + } 143 143 + 144 144 + /// Try to match a reference pattern starting exactly at byte offset `i`. 145 145 + /// Returns the span and how many bytes it consumed. 146 146 + fn try_match_reference(text: &str, bytes: &[u8], i: usize) -> Option<(ReferenceSpan, usize)> { 147 147 + match bytes[i] { 148 148 + b'[' if bytes.get(i + 1) == Some(&b'[') => { 149 149 + let (name, end) = find_double_bracket_close(bytes, i + 2)?; 150 150 + let kind = match NaiveDate::parse_from_str(name, "%Y-%m-%d") { 151 151 + Ok(date) => ReferenceKind::Date(date), 152 152 + Err(_) => ReferenceKind::Page(name.to_string()), 153 153 + }; 154 154 + let _ = text; 155 155 + Some(( 156 156 + ReferenceSpan { 157 157 + range: i..end, 158 158 + kind, 159 159 + }, 160 160 + end - i, 161 161 + )) 162 162 + } 163 163 + b'(' if bytes.get(i + 1) == Some(&b'(') => { 164 164 + let (id, end) = find_double_paren_close(bytes, i + 2)?; 165 165 + Some(( 166 166 + ReferenceSpan { 167 167 + range: i..end, 168 168 + kind: ReferenceKind::Block(id.to_string()), 169 169 + }, 170 170 + end - i, 171 171 + )) 172 172 + } 173 173 + b'#' => { 174 174 + if bytes.get(i + 1) == Some(&b'[') && bytes.get(i + 2) == Some(&b'[') { 175 175 + let (name, end) = find_double_bracket_close(bytes, i + 3)?; 176 176 + Some(( 177 177 + ReferenceSpan { 178 178 + range: i..end, 179 179 + kind: ReferenceKind::Tag(name.to_string()), 180 180 + }, 181 181 + end - i, 182 182 + )) 183 183 + } else { 184 184 + let prev_is_word = i > 0 && is_word_byte(bytes[i - 1]); 185 185 + if prev_is_word { 186 186 + return None; 187 187 + } 188 188 + let start = i + 1; 189 189 + let mut end = start; 190 190 + while end < bytes.len() && is_tag_byte(bytes[end]) { 191 191 + end += 1; 192 192 + } 193 193 + if end == start { 194 194 + return None; 195 195 + } 196 196 + Some(( 197 197 + ReferenceSpan { 198 198 + range: i..end, 199 199 + kind: ReferenceKind::Tag(text[start..end].to_string()), 200 200 + }, 201 201 + end - i, 202 202 + )) 203 203 + } 204 204 + } 205 205 + _ => None, 206 206 + } 207 207 + } 208 208 + 209 209 + fn is_word_byte(b: u8) -> bool { 210 210 + b.is_ascii_alphanumeric() || b == b'_' 211 211 + } 212 212 + 213 213 + fn is_tag_byte(b: u8) -> bool { 214 214 + b.is_ascii_alphanumeric() || b == b'_' || b == b'-' || b == b'/' 215 215 + } 216 216 + 217 217 + /// Given the byte index right after an opening `[[`, find the matching 218 218 + /// `]]` and return (trimmed name, byte index just past the closing `]]`). 219 219 + fn find_double_bracket_close(bytes: &[u8], start: usize) -> Option<(&str, usize)> { 220 220 + let mut j = start; 221 221 + while j + 1 < bytes.len() { 222 222 + if bytes[j] == b']' && bytes[j + 1] == b']' { 223 223 + let name = std::str::from_utf8(&bytes[start..j]).ok()?.trim(); 224 224 + return Some((name, j + 2)); 225 225 + } 226 226 + if bytes[j] == b'\n' { 227 227 + return None; 228 228 + } 229 229 + j += 1; 230 230 + } 231 231 + None 232 232 + } 233 233 + 234 234 + fn find_double_paren_close(bytes: &[u8], start: usize) -> Option<(&str, usize)> { 235 235 + let mut j = start; 236 236 + while j + 1 < bytes.len() { 237 237 + if bytes[j] == b')' && bytes[j + 1] == b')' { 238 238 + let id = std::str::from_utf8(&bytes[start..j]).ok()?.trim(); 239 239 + return Some((id, j + 2)); 240 240 + } 241 241 + if bytes[j] == b'\n' { 242 242 + return None; 243 243 + } 244 244 + j += 1; 245 245 + } 246 246 + None 247 247 + } 248 248 + 249 249 + #[cfg(test)] 250 250 + mod tests { 251 251 + use super::*; 252 252 + 253 253 + fn kinds(text: &str) -> Vec<ReferenceKind> { 254 254 + parse_references(text).into_iter().map(|s| s.kind).collect() 255 255 + } 256 256 + 257 257 + #[test] 258 258 + fn parses_page_tag_date_and_block_references() { 259 259 + let text = "See [[Project Ideas]] tagged #project, due [[2026-07-10]], ref ((abc123))"; 260 260 + let found = kinds(text); 261 261 + assert_eq!( 262 262 + found, 263 263 + vec![ 264 264 + ReferenceKind::Page("Project Ideas".into()), 265 265 + ReferenceKind::Tag("project".into()), 266 266 + ReferenceKind::Date(NaiveDate::from_ymd_opt(2026, 7, 10).unwrap()), 267 267 + ReferenceKind::Block("abc123".into()), 268 268 + ] 269 269 + ); 270 270 + } 271 271 + 272 272 + #[test] 273 273 + fn multi_word_tag_via_hash_bracket() { 274 274 + let found = kinds("#[[multi word tag]] here"); 275 275 + assert_eq!(found, vec![ReferenceKind::Tag("multi word tag".into())]); 276 276 + } 277 277 + 278 278 + #[test] 279 279 + fn does_not_parse_inside_inline_code_span() { 280 280 + let found = kinds("here is `#not-a-tag [[not a page]]` literal"); 281 281 + assert!(found.is_empty()); 282 282 + } 283 283 + 284 284 + #[test] 285 285 + fn does_not_parse_inside_fenced_code_block() { 286 286 + let text = "before\n```\n#not-a-tag [[not a page]]\n```\nafter #real-tag"; 287 287 + let found = kinds(text); 288 288 + assert_eq!(found, vec![ReferenceKind::Tag("real-tag".into())]); 289 289 + } 290 290 + 291 291 + #[test] 292 292 + fn hash_in_middle_of_word_is_not_a_tag() { 293 293 + // e.g. a C# mention shouldn't become a tag reference. 294 294 + let found = kinds("I like C#programming"); 295 295 + assert!(found.is_empty()); 296 296 + } 297 297 + 298 298 + #[test] 299 299 + fn span_ranges_point_at_the_original_bracket_text() { 300 300 + let text = "go to [[Home]] now"; 301 301 + let spans = parse_references(text); 302 302 + assert_eq!(&text[spans[0].range.clone()], "[[Home]]"); 303 303 + } 304 304 + }

+306

crates/trawler-core/src/storage.rs

Reviewed

··· 1 1 + //! Loro-backed persistence: the graph directory holds a snapshot file plus 2 2 + //! an append-only log of incremental update blobs since that snapshot. 3 3 + //! 4 4 + //! See openspec/changes/trawler-mvp specs/block-graph/spec.md 5 5 + //! ("Loro document is the source of truth", "Crash safety") and design.md 6 6 + //! D1/D8. 7 7 + 8 8 + use std::cell::RefCell; 9 9 + use std::fs::{self, File, OpenOptions}; 10 10 + use std::io::{self, Write}; 11 11 + use std::path::{Path, PathBuf}; 12 12 + 13 13 + use loro::{ExportMode, LoroDoc, VersionVector}; 14 14 + 15 15 + /// Name of the Loro movable-tree container holding the outline. Tree roots 16 16 + /// are pages; their descendants are blocks (design.md D8). 17 17 + pub const OUTLINE_TREE: &str = "outline"; 18 18 + 19 19 + const SNAPSHOT_FILE: &str = "snapshot.loro"; 20 20 + const UPDATES_FILE: &str = "updates.log"; 21 21 + 22 22 + /// Opens or creates a graph directory backed by a single Loro document. 23 23 + /// 24 24 + /// Nothing but this type touches the files in the graph directory; any 25 25 + /// derived index (in-memory graph, tantivy) lives elsewhere and can always 26 26 + /// be deleted and rebuilt from what's persisted here (spec: "Derived 27 27 + /// indexes are disposable"). 28 28 + pub struct GraphStorage { 29 29 + doc: LoroDoc, 30 30 + dir: PathBuf, 31 31 + /// Version already durably persisted (snapshot + updates log), so 32 32 + /// `persist_update` only has to export the delta since last call. 33 33 + persisted_vv: RefCell<VersionVector>, 34 34 + } 35 35 + 36 36 + impl GraphStorage { 37 37 + /// Create a brand-new, empty graph directory. 38 38 + pub fn create(dir: impl AsRef<Path>) -> io::Result<Self> { 39 39 + let dir = dir.as_ref().to_path_buf(); 40 40 + fs::create_dir_all(&dir)?; 41 41 + let doc = LoroDoc::new(); 42 42 + // Touch the outline tree so it exists in the very first snapshot, 43 43 + // and enable ordered (fractional-index) children — required by 44 44 + // every position-aware tree op (create_at/mov_to/mov_after/mov_before). 45 45 + doc.get_tree(OUTLINE_TREE).enable_fractional_index(0); 46 46 + 47 47 + let storage = Self { 48 48 + persisted_vv: RefCell::new(doc.state_vv()), 49 49 + doc, 50 50 + dir, 51 51 + }; 52 52 + storage.write_snapshot()?; 53 53 + Ok(storage) 54 54 + } 55 55 + 56 56 + /// Open an existing graph directory, replaying the snapshot and then 57 57 + /// any updates recorded after it. 58 58 + pub fn open(dir: impl AsRef<Path>) -> io::Result<Self> { 59 59 + let dir = dir.as_ref().to_path_buf(); 60 60 + let doc = LoroDoc::new(); 61 61 + 62 62 + let snapshot_path = dir.join(SNAPSHOT_FILE); 63 63 + if snapshot_path.exists() { 64 64 + let bytes = fs::read(&snapshot_path)?; 65 65 + doc.import(&bytes).map_err(to_io_err)?; 66 66 + } 67 67 + 68 68 + let updates_path = dir.join(UPDATES_FILE); 69 69 + if updates_path.exists() { 70 70 + let bytes = fs::read(&updates_path)?; 71 71 + for blob in iter_length_prefixed(&bytes) { 72 72 + doc.import(blob).map_err(to_io_err)?; 73 73 + } 74 74 + } 75 75 + 76 76 + // Idempotent: re-enabling on an already-fractional-indexed tree is a 77 77 + // no-op, and a fresh tree needs it before any position-aware op. 78 78 + doc.get_tree(OUTLINE_TREE).enable_fractional_index(0); 79 79 + 80 80 + let persisted_vv = doc.state_vv(); 81 81 + Ok(Self { 82 82 + doc, 83 83 + dir, 84 84 + persisted_vv: RefCell::new(persisted_vv), 85 85 + }) 86 86 + } 87 87 + 88 88 + /// Whether a graph directory has already been created at `dir`. 89 89 + pub fn exists(dir: impl AsRef<Path>) -> bool { 90 90 + dir.as_ref().join(SNAPSHOT_FILE).exists() 91 91 + } 92 92 + 93 93 + pub fn doc(&self) -> &LoroDoc { 94 94 + &self.doc 95 95 + } 96 96 + 97 97 + /// Durably persist every change made since the last successful 98 98 + /// `persist_update`/`create`/`open`, as an incremental update appended 99 99 + /// to the updates log. 100 100 + /// 101 101 + /// Crash-safe write ordering: the update bytes are appended and then 102 102 + /// `fsync`'d before this call returns. If the process is killed right 103 103 + /// after an edit is acknowledged (i.e. after this call returns `Ok`), 104 104 + /// the edit is guaranteed present on next `open` (spec: "Crash 105 105 + /// safety"). If it's killed *during* this call, `open` simply doesn't 106 106 + /// see the update — there's no partial/corrupt state, because we only 107 107 + /// ever append fully-formed length-prefixed blobs, and a torn trailing 108 108 + /// write is detected and ignored by `iter_length_prefixed`. 109 109 + pub fn persist_update(&self) -> io::Result<()> { 110 110 + let from = self.persisted_vv.borrow().clone(); 111 111 + let bytes = self 112 112 + .doc 113 113 + .export(ExportMode::updates(&from)) 114 114 + .map_err(to_io_err)?; 115 115 + if bytes.is_empty() { 116 116 + return Ok(()); 117 117 + } 118 118 + 119 119 + let mut file = OpenOptions::new() 120 120 + .create(true) 121 121 + .append(true) 122 122 + .open(self.dir.join(UPDATES_FILE))?; 123 123 + write_length_prefixed(&mut file, &bytes)?; 124 124 + file.sync_all()?; 125 125 + 126 126 + *self.persisted_vv.borrow_mut() = self.doc.state_vv(); 127 127 + Ok(()) 128 128 + } 129 129 + 130 130 + /// Compact the updates log into a fresh snapshot. Disposable-index 131 131 + /// spirit applies to the log too: this is purely a size/load-time 132 132 + /// optimization, never required for correctness. 133 133 + pub fn compact(&self) -> io::Result<()> { 134 134 + self.write_snapshot()?; 135 135 + let updates_path = self.dir.join(UPDATES_FILE); 136 136 + if updates_path.exists() { 137 137 + fs::remove_file(updates_path)?; 138 138 + } 139 139 + *self.persisted_vv.borrow_mut() = self.doc.state_vv(); 140 140 + Ok(()) 141 141 + } 142 142 + 143 143 + /// Atomically replace the snapshot file: write to a temp file in the 144 144 + /// same directory, `fsync` it, then rename over the target. The rename 145 145 + /// is atomic on both NTFS and POSIX filesystems, so readers never see a 146 146 + /// partially-written snapshot. 147 147 + fn write_snapshot(&self) -> io::Result<()> { 148 148 + let bytes = self.doc.export(ExportMode::Snapshot).map_err(to_io_err)?; 149 149 + let tmp_path = self.dir.join(format!("{SNAPSHOT_FILE}.tmp")); 150 150 + { 151 151 + let mut tmp = File::create(&tmp_path)?; 152 152 + tmp.write_all(&bytes)?; 153 153 + tmp.sync_all()?; 154 154 + } 155 155 + fs::rename(&tmp_path, self.dir.join(SNAPSHOT_FILE))?; 156 156 + Ok(()) 157 157 + } 158 158 + } 159 159 + 160 160 + fn to_io_err<E: std::fmt::Display>(e: E) -> io::Error { 161 161 + io::Error::other(e.to_string()) 162 162 + } 163 163 + 164 164 + fn write_length_prefixed(file: &mut File, bytes: &[u8]) -> io::Result<()> { 165 165 + file.write_all(&(bytes.len() as u64).to_le_bytes())?; 166 166 + file.write_all(bytes)?; 167 167 + Ok(()) 168 168 + } 169 169 + 170 170 + /// Iterate length-prefixed blobs in `data`, stopping (without erroring) at 171 171 + /// the first incomplete trailing entry — the crash-safety fallback for a 172 172 + /// write that was torn mid-append. 173 173 + fn iter_length_prefixed(data: &[u8]) -> impl Iterator<Item = &[u8]> { 174 174 + let mut offset = 0usize; 175 175 + std::iter::from_fn(move || { 176 176 + if offset + 8 > data.len() { 177 177 + return None; 178 178 + } 179 179 + let len = u64::from_le_bytes(data[offset..offset + 8].try_into().unwrap()) as usize; 180 180 + let start = offset + 8; 181 181 + let end = start.checked_add(len)?; 182 182 + if end > data.len() { 183 183 + return None; 184 184 + } 185 185 + offset = end; 186 186 + Some(&data[start..end]) 187 187 + }) 188 188 + } 189 189 + 190 190 + #[cfg(test)] 191 191 + mod tests { 192 192 + use super::*; 193 193 + 194 194 + fn temp_dir(name: &str) -> PathBuf { 195 195 + let dir = std::env::temp_dir().join(format!( 196 196 + "trawler-storage-test-{name}-{:?}", 197 197 + std::thread::current().id() 198 198 + )); 199 199 + let _ = fs::remove_dir_all(&dir); 200 200 + dir 201 201 + } 202 202 + 203 203 + #[test] 204 204 + fn create_then_open_round_trips_empty_graph() { 205 205 + let dir = temp_dir("create-open-empty"); 206 206 + { 207 207 + let storage = GraphStorage::create(&dir).unwrap(); 208 208 + assert!(storage.doc().get_tree(OUTLINE_TREE).roots().is_empty()); 209 209 + } 210 210 + let reopened = GraphStorage::open(&dir).unwrap(); 211 211 + assert!(reopened.doc().get_tree(OUTLINE_TREE).roots().is_empty()); 212 212 + fs::remove_dir_all(&dir).ok(); 213 213 + } 214 214 + 215 215 + #[test] 216 216 + fn persisted_update_survives_reopen() { 217 217 + let dir = temp_dir("persist-update"); 218 218 + { 219 219 + let storage = GraphStorage::create(&dir).unwrap(); 220 220 + let tree = storage.doc().get_tree(OUTLINE_TREE); 221 221 + let root = tree.create(None).unwrap(); 222 222 + let meta = tree.get_meta(root).unwrap(); 223 223 + meta.insert("block_type", "page").unwrap(); 224 224 + storage.persist_update().unwrap(); 225 225 + } 226 226 + 227 227 + let reopened = GraphStorage::open(&dir).unwrap(); 228 228 + let tree = reopened.doc().get_tree(OUTLINE_TREE); 229 229 + assert_eq!(tree.roots().len(), 1); 230 230 + fs::remove_dir_all(&dir).ok(); 231 231 + } 232 232 + 233 233 + #[test] 234 234 + fn compact_folds_updates_into_snapshot_and_still_loads() { 235 235 + let dir = temp_dir("compact"); 236 236 + { 237 237 + let storage = GraphStorage::create(&dir).unwrap(); 238 238 + let tree = storage.doc().get_tree(OUTLINE_TREE); 239 239 + for _ in 0..5 { 240 240 + tree.create(None).unwrap(); 241 241 + storage.persist_update().unwrap(); 242 242 + } 243 243 + storage.compact().unwrap(); 244 244 + assert!(!dir.join(UPDATES_FILE).exists()); 245 245 + } 246 246 + 247 247 + let reopened = GraphStorage::open(&dir).unwrap(); 248 248 + assert_eq!(reopened.doc().get_tree(OUTLINE_TREE).roots().len(), 5); 249 249 + fs::remove_dir_all(&dir).ok(); 250 250 + } 251 251 + 252 252 + /// Simulates the crash-safety scenario from the spec: kill the process 253 253 + /// right after an edit is acknowledged (i.e. after `persist_update` 254 254 + /// returns `Ok`) and confirm the edit is present on the next open — 255 255 + /// without ever constructing a second live `GraphStorage` over the same 256 256 + /// doc (which would be the real crash, just modeled as "stop touching 257 257 + /// this handle and open a fresh one instead"). 258 258 + #[test] 259 259 + fn edit_acknowledged_before_crash_is_recovered() { 260 260 + let dir = temp_dir("crash-safety"); 261 261 + { 262 262 + let storage = GraphStorage::create(&dir).unwrap(); 263 263 + let tree = storage.doc().get_tree(OUTLINE_TREE); 264 264 + let root = tree.create(None).unwrap(); 265 265 + tree.get_meta(root) 266 266 + .unwrap() 267 267 + .insert("block_type", "page") 268 268 + .unwrap(); 269 269 + storage.persist_update().unwrap(); // "acknowledged" 270 270 + // storage is dropped here without any further writes, 271 271 + // simulating the process dying immediately after. 272 272 + } 273 273 + 274 274 + let recovered = GraphStorage::open(&dir).unwrap(); 275 275 + assert_eq!(recovered.doc().get_tree(OUTLINE_TREE).roots().len(), 1); 276 276 + fs::remove_dir_all(&dir).ok(); 277 277 + } 278 278 + 279 279 + #[test] 280 280 + fn torn_trailing_write_is_ignored_not_fatal() { 281 281 + let dir = temp_dir("torn-write"); 282 282 + { 283 283 + let storage = GraphStorage::create(&dir).unwrap(); 284 284 + let tree = storage.doc().get_tree(OUTLINE_TREE); 285 285 + tree.create(None).unwrap(); 286 286 + storage.persist_update().unwrap(); 287 287 + } 288 288 + 289 289 + // Simulate a crash mid-append: truncate the last few bytes of the 290 290 + // length-prefixed update record. 291 291 + let updates_path = dir.join(UPDATES_FILE); 292 292 + let mut bytes = fs::read(&updates_path).unwrap(); 293 293 + assert!( 294 294 + !bytes.is_empty(), 295 295 + "expected a real update to have been logged" 296 296 + ); 297 297 + bytes.truncate(bytes.len() - 2); 298 298 + fs::write(&updates_path, &bytes).unwrap(); 299 299 + 300 300 + // Opening must not error, and the torn record is simply dropped — 301 301 + // we lose at most the last unacknowledged edit, never crash. 302 302 + let reopened = GraphStorage::open(&dir).unwrap(); 303 303 + assert_eq!(reopened.doc().get_tree(OUTLINE_TREE).roots().len(), 0); 304 304 + fs::remove_dir_all(&dir).ok(); 305 305 + } 306 306 + }

+131

crates/trawler-core/tests/crash_safety.rs

Reviewed

··· 1 1 + //! Integration-level crash-safety test spanning storage, outline ops, and 2 2 + //! the derived index together — the full path a real crash needs to 3 3 + //! survive, not just the raw persistence layer in isolation. 4 4 + //! 5 5 + //! See openspec/changes/trawler-mvp specs/block-graph/spec.md 6 6 + //! ("Crash safety": "the process is killed immediately after an edit is 7 7 + //! acknowledged in the UI" -> "on next startup the edit is present") and 8 8 + //! tasks.md 2.8. 9 9 + 10 10 + use std::fs; 11 11 + use std::path::PathBuf; 12 12 + 13 13 + use trawler_core::index::GraphIndex; 14 14 + use trawler_core::outline::{Outline, Position}; 15 15 + use trawler_core::storage::GraphStorage; 16 16 + 17 17 + fn temp_dir(name: &str) -> PathBuf { 18 18 + let dir = std::env::temp_dir().join(format!("trawler-crash-safety-{name}")); 19 19 + let _ = fs::remove_dir_all(&dir); 20 20 + dir 21 21 + } 22 22 + 23 23 + /// Only edits acknowledged (i.e. `persist_update` returned `Ok`) before a 24 24 + /// simulated crash survive; an edit still only in memory when the process 25 25 + /// dies is simply absent afterward — never corrupting or losing the 26 26 + /// already-acknowledged edits around it. 27 27 + #[test] 28 28 + fn only_acknowledged_edits_survive_a_simulated_crash() { 29 29 + let dir = temp_dir("only-acknowledged-survive"); 30 30 + 31 31 + let (page, acked_block, unacked_block) = { 32 32 + let storage = GraphStorage::create(&dir).unwrap(); 33 33 + let outline = Outline::new(storage.doc()); 34 34 + 35 35 + let page = outline 36 36 + .create_block(None, Position::Index(0), "Journal") 37 37 + .unwrap(); 38 38 + storage.persist_update().unwrap(); // acknowledged #1 39 39 + 40 40 + let acked_block = outline 41 41 + .create_block(Some(page), Position::Index(0), "went #fishing") 42 42 + .unwrap(); 43 43 + storage.persist_update().unwrap(); // acknowledged #2 44 44 + 45 45 + // This edit is never persisted — it models work still in flight 46 46 + // when the process is killed (e.g. a debounced/batched persist 47 47 + // that hadn't fired yet). 48 48 + let unacked_block = outline 49 49 + .create_block(Some(page), Position::Index(1), "never saved") 50 50 + .unwrap(); 51 51 + 52 52 + // No persist_update() call here. Dropping `storage` without one 53 53 + // more call is the simulated crash: whatever's only in the Loro 54 54 + // doc's memory and not yet fsync'd to updates.log is gone. 55 55 + (page, acked_block, unacked_block) 56 56 + }; 57 57 + 58 58 + let recovered = GraphStorage::open(&dir).unwrap(); 59 59 + let outline = Outline::new(recovered.doc()); 60 60 + 61 61 + let children = outline.children(Some(page)); 62 62 + assert!( 63 63 + children.contains(&acked_block), 64 64 + "acknowledged edit must survive the simulated crash" 65 65 + ); 66 66 + assert!( 67 67 + !children.contains(&unacked_block), 68 68 + "unacknowledged edit must not appear after the simulated crash" 69 69 + ); 70 70 + assert_eq!(outline.content(acked_block).unwrap(), "went #fishing"); 71 71 + 72 72 + // The recovered doc must be fully usable, not just structurally 73 73 + // present: the derived index rebuilds cleanly and reflects exactly the 74 74 + // acknowledged state (spec: "Derived indexes are disposable"). 75 75 + let index = GraphIndex::rebuild(recovered.doc()); 76 76 + let backlinks = index.backlinks_of(&trawler_core::graph::NodeId::tag("fishing")); 77 77 + assert_eq!(backlinks, std::collections::HashSet::from([acked_block])); 78 78 + 79 79 + fs::remove_dir_all(&dir).ok(); 80 80 + } 81 81 + 82 82 + /// A longer acknowledged history (multiple persisted transactions) across 83 83 + /// several storage "sessions" (create -> edit+persist -> crash -> reopen 84 84 + /// -> edit+persist -> crash -> reopen) must all be present at the end, 85 85 + /// exercising both the snapshot and the updates-log replay path together. 86 86 + #[test] 87 87 + fn edits_across_multiple_sessions_all_survive() { 88 88 + let dir = temp_dir("multi-session"); 89 89 + let mut expected_content = Vec::new(); 90 90 + 91 91 + for session in 0..4 { 92 92 + let storage = if session == 0 { 93 93 + GraphStorage::create(&dir).unwrap() 94 94 + } else { 95 95 + GraphStorage::open(&dir).unwrap() 96 96 + }; 97 97 + let outline = Outline::new(storage.doc()); 98 98 + 99 99 + let parent = if session == 0 { 100 100 + let page = outline 101 101 + .create_block(None, Position::Index(0), "Home") 102 102 + .unwrap(); 103 103 + storage.persist_update().unwrap(); 104 104 + expected_content.push("Home".to_string()); 105 105 + page 106 106 + } else { 107 107 + outline.children(None)[0] 108 108 + }; 109 109 + 110 110 + let text = format!("entry from session {session}"); 111 111 + let n = outline.children(Some(parent)).len(); 112 112 + outline 113 113 + .create_block(Some(parent), Position::Index(n), &text) 114 114 + .unwrap(); 115 115 + storage.persist_update().unwrap(); // acknowledged before "crash" 116 116 + expected_content.push(text); 117 117 + // storage drops here without a clean shutdown step — next 118 118 + // iteration's `open` is the recovery after a simulated crash. 119 119 + } 120 120 + 121 121 + let recovered = GraphStorage::open(&dir).unwrap(); 122 122 + let outline = Outline::new(recovered.doc()); 123 123 + let page = outline.children(None)[0]; 124 124 + let mut actual: Vec<String> = vec![outline.content(page).unwrap()]; 125 125 + for child in outline.children(Some(page)) { 126 126 + actual.push(outline.content(child).unwrap()); 127 127 + } 128 128 + 129 129 + assert_eq!(actual, expected_content); 130 130 + fs::remove_dir_all(&dir).ok(); 131 131 + }

+8 -8

openspec/changes/trawler-mvp/tasks.md

Reviewed

··· 10 10 11 11 ## 2. Block Graph Core (trawler-core) 12 12 13 13 - - [ ] 2.1 Define graph types: node IDs, block/page/tag/date nodes, property values (text/number/date/bool/ref) 14 14 - - [ ] 2.2 Loro storage layer: movable tree + text + metadata maps; open/create graph directory; snapshot + update-blob persistence with crash-safe write ordering 15 15 - - [ ] 2.3 Outline operations: create/delete/split/merge blocks, move subtree, reorder siblings — as transactions over the Loro doc 16 16 - - [ ] 2.4 Inline reference model: parse refs/tags/dates out of block Markdown on commit (skipping code spans/fences); represent as typed spans 17 17 - - [ ] 2.5 Derived in-memory graph: adjacency, backlink sets, tag/property/date indexes; full rebuild from Loro doc 18 18 - - [ ] 2.6 Incremental index maintenance on edit transactions 19 19 - - [ ] 2.7 Golden test: randomized 1,000+ edit sequences — incremental indexes equal full rebuild (spec: block-graph/Incremental index maintenance) 20 20 - - [ ] 2.8 Crash-safety test: kill after acknowledged edit; verify recovery (spec: block-graph/Loro document is the source of truth) 13 13 + - [x] 2.1 Define graph types: node IDs, block/page/tag/date nodes, property values (text/number/date/bool/ref) 14 14 + - [x] 2.2 Loro storage layer: movable tree + text + metadata maps; open/create graph directory; snapshot + update-blob persistence with crash-safe write ordering 15 15 + - [x] 2.3 Outline operations: create/delete/split/merge blocks, move subtree, reorder siblings — as transactions over the Loro doc 16 16 + - [x] 2.4 Inline reference model: parse refs/tags/dates out of block Markdown on commit (skipping code spans/fences); represent as typed spans 17 17 + - [x] 2.5 Derived in-memory graph: adjacency, backlink sets, tag/property/date indexes; full rebuild from Loro doc 18 18 + - [x] 2.6 Incremental index maintenance on edit transactions 19 19 + - [x] 2.7 Golden test: randomized 1,000+ edit sequences — incremental indexes equal full rebuild (spec: block-graph/Incremental index maintenance) 20 20 + - [x] 2.8 Crash-safety test: kill after acknowledged edit; verify recovery (spec: block-graph/Loro document is the source of truth) 21 21 22 22 ## 3. Search Core (trawler-core) 23 23