Personal outliner built with Rust.
phase 3: search core (tantivy full-text index, incremental upsert, ranked search + highlighting, lexical similarity trait)
graham.systems
(Jul 12, 2026, 2:52 PM -0700)
0c21e32f
68633f11
+826
-7
+416
-3
Cargo.lock
+416
-3
Cargo.lock
···
628
628
]
629
629
630
630
[[package]]
631
+
name = "bitpacking"
632
+
version = "0.9.3"
633
+
source = "registry+https://github.com/rust-lang/crates.io-index"
634
+
checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019"
635
+
dependencies = [
636
+
"crunchy",
637
+
]
638
+
639
+
[[package]]
631
640
name = "bitstream-io"
632
641
version = "4.10.0"
633
642
source = "registry+https://github.com/rust-lang/crates.io-index"
···
741
750
]
742
751
743
752
[[package]]
753
+
name = "bon"
754
+
version = "3.9.3"
755
+
source = "registry+https://github.com/rust-lang/crates.io-index"
756
+
checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561"
757
+
dependencies = [
758
+
"bon-macros",
759
+
"rustversion",
760
+
]
761
+
762
+
[[package]]
763
+
name = "bon-macros"
764
+
version = "3.9.3"
765
+
source = "registry+https://github.com/rust-lang/crates.io-index"
766
+
checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f"
767
+
dependencies = [
768
+
"darling",
769
+
"ident_case",
770
+
"prettyplease",
771
+
"proc-macro2",
772
+
"quote",
773
+
"rustversion",
774
+
"syn 2.0.118",
775
+
]
776
+
777
+
[[package]]
744
778
name = "bstr"
745
779
version = "1.12.3"
746
780
source = "registry+https://github.com/rust-lang/crates.io-index"
···
873
907
"libc",
874
908
"shlex 2.0.1",
875
909
]
910
+
911
+
[[package]]
912
+
name = "census"
913
+
version = "0.4.2"
914
+
source = "registry+https://github.com/rust-lang/crates.io-index"
915
+
checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0"
876
916
877
917
[[package]]
878
918
name = "cexpr"
···
1457
1497
checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376"
1458
1498
1459
1499
[[package]]
1500
+
name = "datasketches"
1501
+
version = "0.2.0"
1502
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1503
+
checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745"
1504
+
1505
+
[[package]]
1460
1506
name = "deflate64"
1461
1507
version = "0.1.12"
1462
1508
source = "registry+https://github.com/rust-lang/crates.io-index"
1463
1509
checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2"
1464
1510
1465
1511
[[package]]
1512
+
name = "deranged"
1513
+
version = "0.5.8"
1514
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1515
+
checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
1516
+
dependencies = [
1517
+
"serde_core",
1518
+
]
1519
+
1520
+
[[package]]
1466
1521
name = "derive_arbitrary"
1467
1522
version = "1.4.2"
1468
1523
source = "registry+https://github.com/rust-lang/crates.io-index"
···
1585
1640
version = "1.2.1"
1586
1641
source = "registry+https://github.com/rust-lang/crates.io-index"
1587
1642
checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"
1643
+
1644
+
[[package]]
1645
+
name = "downcast-rs"
1646
+
version = "2.0.2"
1647
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1648
+
checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc"
1588
1649
1589
1650
[[package]]
1590
1651
name = "dtor"
···
1870
1931
]
1871
1932
1872
1933
[[package]]
1934
+
name = "fastdivide"
1935
+
version = "0.4.2"
1936
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1937
+
checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471"
1938
+
1939
+
[[package]]
1873
1940
name = "fastrand"
1874
1941
version = "1.9.0"
1875
1942
source = "registry+https://github.com/rust-lang/crates.io-index"
···
1988
2055
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
1989
2056
1990
2057
[[package]]
2058
+
name = "foldhash"
2059
+
version = "0.2.0"
2060
+
source = "registry+https://github.com/rust-lang/crates.io-index"
2061
+
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
2062
+
2063
+
[[package]]
1991
2064
name = "font-types"
1992
2065
version = "0.11.3"
1993
2066
source = "registry+https://github.com/rust-lang/crates.io-index"
···
2081
2154
]
2082
2155
2083
2156
[[package]]
2157
+
name = "fs4"
2158
+
version = "0.13.1"
2159
+
source = "registry+https://github.com/rust-lang/crates.io-index"
2160
+
checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4"
2161
+
dependencies = [
2162
+
"rustix 1.1.4",
2163
+
"windows-sys 0.59.0",
2164
+
]
2165
+
2166
+
[[package]]
2084
2167
name = "fsevent-sys"
2085
2168
version = "4.1.0"
2086
2169
source = "registry+https://github.com/rust-lang/crates.io-index"
···
2795
2878
source = "registry+https://github.com/rust-lang/crates.io-index"
2796
2879
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
2797
2880
dependencies = [
2798
-
"foldhash",
2881
+
"foldhash 0.1.5",
2799
2882
]
2800
2883
2801
2884
[[package]]
···
2803
2886
version = "0.16.1"
2804
2887
source = "registry+https://github.com/rust-lang/crates.io-index"
2805
2888
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
2889
+
dependencies = [
2890
+
"allocator-api2",
2891
+
"equivalent",
2892
+
"foldhash 0.2.0",
2893
+
]
2806
2894
2807
2895
[[package]]
2808
2896
name = "hashbrown"
···
2927
3015
]
2928
3016
2929
3017
[[package]]
3018
+
name = "htmlescape"
3019
+
version = "0.3.1"
3020
+
source = "registry+https://github.com/rust-lang/crates.io-index"
3021
+
checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
3022
+
3023
+
[[package]]
2930
3024
name = "http"
2931
3025
version = "1.4.2"
2932
3026
source = "registry+https://github.com/rust-lang/crates.io-index"
···
3563
3657
checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8"
3564
3658
3565
3659
[[package]]
3660
+
name = "levenshtein_automata"
3661
+
version = "0.2.1"
3662
+
source = "registry+https://github.com/rust-lang/crates.io-index"
3663
+
checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25"
3664
+
3665
+
[[package]]
3566
3666
name = "libc"
3567
3667
version = "0.2.186"
3568
3668
source = "registry+https://github.com/rust-lang/crates.io-index"
···
3766
3866
"bytes",
3767
3867
"ensure-cov",
3768
3868
"loro-common",
3769
-
"lz4_flex",
3869
+
"lz4_flex 0.11.6",
3770
3870
"once_cell",
3771
3871
"quick_cache",
3772
3872
"rustc-hash 2.1.3",
···
3803
3903
]
3804
3904
3805
3905
[[package]]
3906
+
name = "lru"
3907
+
version = "0.16.4"
3908
+
source = "registry+https://github.com/rust-lang/crates.io-index"
3909
+
checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39"
3910
+
dependencies = [
3911
+
"hashbrown 0.16.1",
3912
+
]
3913
+
3914
+
[[package]]
3806
3915
name = "lru-slab"
3807
3916
version = "0.1.2"
3808
3917
source = "registry+https://github.com/rust-lang/crates.io-index"
···
3881
3990
dependencies = [
3882
3991
"twox-hash",
3883
3992
]
3993
+
3994
+
[[package]]
3995
+
name = "lz4_flex"
3996
+
version = "0.13.1"
3997
+
source = "registry+https://github.com/rust-lang/crates.io-index"
3998
+
checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
3884
3999
3885
4000
[[package]]
3886
4001
name = "mac"
···
3968
4083
checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
3969
4084
3970
4085
[[package]]
4086
+
name = "measure_time"
4087
+
version = "0.9.0"
4088
+
source = "registry+https://github.com/rust-lang/crates.io-index"
4089
+
checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e"
4090
+
dependencies = [
4091
+
"log",
4092
+
]
4093
+
4094
+
[[package]]
3971
4095
name = "memchr"
3972
4096
version = "2.8.2"
3973
4097
source = "registry+https://github.com/rust-lang/crates.io-index"
···
4067
4191
]
4068
4192
4069
4193
[[package]]
4194
+
name = "murmurhash32"
4195
+
version = "0.3.1"
4196
+
source = "registry+https://github.com/rust-lang/crates.io-index"
4197
+
checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b"
4198
+
4199
+
[[package]]
4070
4200
name = "naga"
4071
4201
version = "25.0.1"
4072
4202
source = "registry+https://github.com/rust-lang/crates.io-index"
···
4277
4407
]
4278
4408
4279
4409
[[package]]
4410
+
name = "num-conv"
4411
+
version = "0.2.2"
4412
+
source = "registry+https://github.com/rust-lang/crates.io-index"
4413
+
checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
4414
+
4415
+
[[package]]
4280
4416
name = "num-derive"
4281
4417
version = "0.4.2"
4282
4418
source = "registry+https://github.com/rust-lang/crates.io-index"
···
4481
4617
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
4482
4618
4483
4619
[[package]]
4620
+
name = "oneshot"
4621
+
version = "0.1.13"
4622
+
source = "registry+https://github.com/rust-lang/crates.io-index"
4623
+
checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107"
4624
+
4625
+
[[package]]
4484
4626
name = "oo7"
4485
4627
version = "0.5.0"
4486
4628
source = "registry+https://github.com/rust-lang/crates.io-index"
···
4556
4698
dependencies = [
4557
4699
"futures-core",
4558
4700
"pin-project-lite",
4701
+
]
4702
+
4703
+
[[package]]
4704
+
name = "ownedbytes"
4705
+
version = "0.9.0"
4706
+
source = "registry+https://github.com/rust-lang/crates.io-index"
4707
+
checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e"
4708
+
dependencies = [
4709
+
"stable_deref_trait",
4559
4710
]
4560
4711
4561
4712
[[package]]
···
4858
5009
]
4859
5010
4860
5011
[[package]]
5012
+
name = "powerfmt"
5013
+
version = "0.2.0"
5014
+
source = "registry+https://github.com/rust-lang/crates.io-index"
5015
+
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
5016
+
5017
+
[[package]]
4861
5018
name = "ppv-lite86"
4862
5019
version = "0.2.21"
4863
5020
source = "registry+https://github.com/rust-lang/crates.io-index"
···
5507
5664
"siphasher",
5508
5665
"toml 0.8.23",
5509
5666
"triomphe",
5667
+
]
5668
+
5669
+
[[package]]
5670
+
name = "rust-stemmers"
5671
+
version = "1.2.0"
5672
+
source = "registry+https://github.com/rust-lang/crates.io-index"
5673
+
checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54"
5674
+
dependencies = [
5675
+
"serde",
5676
+
"serde_derive",
5510
5677
]
5511
5678
5512
5679
[[package]]
···
6033
6200
]
6034
6201
6035
6202
[[package]]
6203
+
name = "sketches-ddsketch"
6204
+
version = "0.4.0"
6205
+
source = "registry+https://github.com/rust-lang/crates.io-index"
6206
+
checksum = "05e40b6cf54d988dc1a2223531b969c9a9e30906ad90ef64890c27b4bfbb46ea"
6207
+
dependencies = [
6208
+
"serde",
6209
+
]
6210
+
6211
+
[[package]]
6036
6212
name = "skrifa"
6037
6213
version = "0.42.1"
6038
6214
source = "registry+https://github.com/rust-lang/crates.io-index"
···
6580
6756
checksum = "8bdb6fa0dfa67b38c1e66b7041ba9dcf23b99d8121907cd31c807a332f7a0bbb"
6581
6757
6582
6758
[[package]]
6759
+
name = "tantivy"
6760
+
version = "0.26.1"
6761
+
source = "registry+https://github.com/rust-lang/crates.io-index"
6762
+
checksum = "edde6a10743fff00a4e1a8c9ef020bf5f3cbad301b7d2d39f2b07f123c4eac07"
6763
+
dependencies = [
6764
+
"aho-corasick",
6765
+
"arc-swap",
6766
+
"base64",
6767
+
"bitpacking",
6768
+
"bon",
6769
+
"byteorder",
6770
+
"census",
6771
+
"crc32fast",
6772
+
"crossbeam-channel",
6773
+
"datasketches",
6774
+
"downcast-rs 2.0.2",
6775
+
"fastdivide",
6776
+
"fnv",
6777
+
"fs4",
6778
+
"htmlescape",
6779
+
"itertools 0.14.0",
6780
+
"levenshtein_automata",
6781
+
"log",
6782
+
"lru",
6783
+
"lz4_flex 0.13.1",
6784
+
"measure_time",
6785
+
"memmap2",
6786
+
"once_cell",
6787
+
"oneshot",
6788
+
"rayon",
6789
+
"regex",
6790
+
"rust-stemmers",
6791
+
"rustc-hash 2.1.3",
6792
+
"serde",
6793
+
"serde_json",
6794
+
"sketches-ddsketch",
6795
+
"smallvec",
6796
+
"tantivy-bitpacker",
6797
+
"tantivy-columnar",
6798
+
"tantivy-common",
6799
+
"tantivy-fst",
6800
+
"tantivy-query-grammar",
6801
+
"tantivy-stacker",
6802
+
"tantivy-tokenizer-api",
6803
+
"tempfile",
6804
+
"thiserror 2.0.18",
6805
+
"time",
6806
+
"typetag",
6807
+
"uuid",
6808
+
"winapi",
6809
+
]
6810
+
6811
+
[[package]]
6812
+
name = "tantivy-bitpacker"
6813
+
version = "0.10.0"
6814
+
source = "registry+https://github.com/rust-lang/crates.io-index"
6815
+
checksum = "4fed3d674429bcd2de5d0a6d1aa5495fed8afd9c5ecce993019caf7615f53fa4"
6816
+
dependencies = [
6817
+
"bitpacking",
6818
+
]
6819
+
6820
+
[[package]]
6821
+
name = "tantivy-columnar"
6822
+
version = "0.7.0"
6823
+
source = "registry+https://github.com/rust-lang/crates.io-index"
6824
+
checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc"
6825
+
dependencies = [
6826
+
"downcast-rs 2.0.2",
6827
+
"fastdivide",
6828
+
"itertools 0.14.0",
6829
+
"serde",
6830
+
"tantivy-bitpacker",
6831
+
"tantivy-common",
6832
+
"tantivy-sstable",
6833
+
"tantivy-stacker",
6834
+
]
6835
+
6836
+
[[package]]
6837
+
name = "tantivy-common"
6838
+
version = "0.11.0"
6839
+
source = "registry+https://github.com/rust-lang/crates.io-index"
6840
+
checksum = "bbf10915aa75da3c3b0d58b58853d2e889efbaf32d4982a4c3715dde6bba23e5"
6841
+
dependencies = [
6842
+
"async-trait",
6843
+
"byteorder",
6844
+
"ownedbytes",
6845
+
"serde",
6846
+
"time",
6847
+
]
6848
+
6849
+
[[package]]
6850
+
name = "tantivy-fst"
6851
+
version = "0.5.0"
6852
+
source = "registry+https://github.com/rust-lang/crates.io-index"
6853
+
checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18"
6854
+
dependencies = [
6855
+
"byteorder",
6856
+
"regex-syntax",
6857
+
"utf8-ranges",
6858
+
]
6859
+
6860
+
[[package]]
6861
+
name = "tantivy-query-grammar"
6862
+
version = "0.26.0"
6863
+
source = "registry+https://github.com/rust-lang/crates.io-index"
6864
+
checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82"
6865
+
dependencies = [
6866
+
"fnv",
6867
+
"nom 7.1.3",
6868
+
"ordered-float",
6869
+
"serde",
6870
+
"serde_json",
6871
+
]
6872
+
6873
+
[[package]]
6874
+
name = "tantivy-sstable"
6875
+
version = "0.7.0"
6876
+
source = "registry+https://github.com/rust-lang/crates.io-index"
6877
+
checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606"
6878
+
dependencies = [
6879
+
"futures-util",
6880
+
"itertools 0.14.0",
6881
+
"tantivy-bitpacker",
6882
+
"tantivy-common",
6883
+
"tantivy-fst",
6884
+
"zstd",
6885
+
]
6886
+
6887
+
[[package]]
6888
+
name = "tantivy-stacker"
6889
+
version = "0.7.0"
6890
+
source = "registry+https://github.com/rust-lang/crates.io-index"
6891
+
checksum = "6cbb051742da9d53ca9e8fff43a9b10e319338b24e2c0e15d0372df19ffeb951"
6892
+
dependencies = [
6893
+
"murmurhash32",
6894
+
"tantivy-common",
6895
+
]
6896
+
6897
+
[[package]]
6898
+
name = "tantivy-tokenizer-api"
6899
+
version = "0.7.0"
6900
+
source = "registry+https://github.com/rust-lang/crates.io-index"
6901
+
checksum = "eac258c2c6390673f2685813afeeafcb8c4e0ee7de8dd3fc46838dcc37263f98"
6902
+
dependencies = [
6903
+
"serde",
6904
+
]
6905
+
6906
+
[[package]]
6583
6907
name = "tao-core-video-sys"
6584
6908
version = "0.2.0"
6585
6909
source = "registry+https://github.com/rust-lang/crates.io-index"
···
6697
7021
]
6698
7022
6699
7023
[[package]]
7024
+
name = "time"
7025
+
version = "0.3.53"
7026
+
source = "registry+https://github.com/rust-lang/crates.io-index"
7027
+
checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50"
7028
+
dependencies = [
7029
+
"deranged",
7030
+
"num-conv",
7031
+
"powerfmt",
7032
+
"serde_core",
7033
+
"time-core",
7034
+
"time-macros",
7035
+
]
7036
+
7037
+
[[package]]
7038
+
name = "time-core"
7039
+
version = "0.1.9"
7040
+
source = "registry+https://github.com/rust-lang/crates.io-index"
7041
+
checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109"
7042
+
7043
+
[[package]]
7044
+
name = "time-macros"
7045
+
version = "0.2.31"
7046
+
source = "registry+https://github.com/rust-lang/crates.io-index"
7047
+
checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f"
7048
+
dependencies = [
7049
+
"num-conv",
7050
+
"time-core",
7051
+
]
7052
+
7053
+
[[package]]
6700
7054
name = "tiny-keccak"
6701
7055
version = "2.0.2"
6702
7056
source = "registry+https://github.com/rust-lang/crates.io-index"
···
7004
7358
"rand 0.10.2",
7005
7359
"serde",
7006
7360
"steel-core",
7361
+
"tantivy",
7007
7362
]
7008
7363
7009
7364
[[package]]
···
7099
7454
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
7100
7455
7101
7456
[[package]]
7457
+
name = "typetag"
7458
+
version = "0.2.22"
7459
+
source = "registry+https://github.com/rust-lang/crates.io-index"
7460
+
checksum = "c5a897b12c6c1151ad0b138b8db50252dc301f93bc3b027db05eec82aeed298c"
7461
+
dependencies = [
7462
+
"erased-serde",
7463
+
"inventory",
7464
+
"once_cell",
7465
+
"serde",
7466
+
"typetag-impl",
7467
+
]
7468
+
7469
+
[[package]]
7470
+
name = "typetag-impl"
7471
+
version = "0.2.22"
7472
+
source = "registry+https://github.com/rust-lang/crates.io-index"
7473
+
checksum = "cf808357c6ed7e13ba0f3277ec8d8f21b2d501274895104263985330c726c1c5"
7474
+
dependencies = [
7475
+
"proc-macro2",
7476
+
"quote",
7477
+
"syn 2.0.118",
7478
+
]
7479
+
7480
+
[[package]]
7102
7481
name = "ucd-trie"
7103
7482
version = "0.1.7"
7104
7483
source = "registry+https://github.com/rust-lang/crates.io-index"
···
7262
7641
version = "0.7.6"
7263
7642
source = "registry+https://github.com/rust-lang/crates.io-index"
7264
7643
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
7644
+
7645
+
[[package]]
7646
+
name = "utf8-ranges"
7647
+
version = "1.0.5"
7648
+
source = "registry+https://github.com/rust-lang/crates.io-index"
7649
+
checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba"
7265
7650
7266
7651
[[package]]
7267
7652
name = "utf8_iter"
···
7476
7861
checksum = "2857dd20b54e916ec7253b3d6b4d5c4d7d4ca2c33c2e11c6c76a99bd8744755d"
7477
7862
dependencies = [
7478
7863
"cc",
7479
-
"downcast-rs",
7864
+
"downcast-rs 1.2.1",
7480
7865
"rustix 1.1.4",
7481
7866
"scoped-tls",
7482
7867
"smallvec",
···
8645
9030
version = "1.0.21"
8646
9031
source = "registry+https://github.com/rust-lang/crates.io-index"
8647
9032
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
9033
+
9034
+
[[package]]
9035
+
name = "zstd"
9036
+
version = "0.13.3"
9037
+
source = "registry+https://github.com/rust-lang/crates.io-index"
9038
+
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
9039
+
dependencies = [
9040
+
"zstd-safe",
9041
+
]
9042
+
9043
+
[[package]]
9044
+
name = "zstd-safe"
9045
+
version = "7.2.4"
9046
+
source = "registry+https://github.com/rust-lang/crates.io-index"
9047
+
checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
9048
+
dependencies = [
9049
+
"zstd-sys",
9050
+
]
9051
+
9052
+
[[package]]
9053
+
name = "zstd-sys"
9054
+
version = "2.0.16+zstd.1.5.7"
9055
+
source = "registry+https://github.com/rust-lang/crates.io-index"
9056
+
checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
9057
+
dependencies = [
9058
+
"cc",
9059
+
"pkg-config",
9060
+
]
8648
9061
8649
9062
[[package]]
8650
9063
name = "zune-core"
+1
crates/trawler-core/Cargo.toml
+1
crates/trawler-core/Cargo.toml
+1
crates/trawler-core/src/lib.rs
+1
crates/trawler-core/src/lib.rs
+404
crates/trawler-core/src/search.rs
+404
crates/trawler-core/src/search.rs
···
1
+
//! Full-text search over block content (tantivy), plus lexical "similar
2
+
//! blocks" suggestions. The index directory is entirely disposable: delete
3
+
//! it and `SearchIndex::open_or_create` transparently rebuilds from the
4
+
//! graph on next open.
5
+
//!
6
+
//! See openspec/changes/trawler-mvp specs/search/spec.md and tasks.md 3.1-3.4.
7
+
8
+
use std::cell::RefCell;
9
+
use std::collections::HashSet;
10
+
use std::fs;
11
+
use std::path::Path;
12
+
13
+
use loro::{LoroDoc, TreeID};
14
+
use tantivy::collector::TopDocs;
15
+
use tantivy::query::QueryParser;
16
+
use tantivy::schema::{Field, Schema, TantivyDocument, Value, STORED, STRING, TEXT};
17
+
use tantivy::snippet::SnippetGenerator;
18
+
use tantivy::{doc, Index, IndexReader, IndexWriter, ReloadPolicy, Term};
19
+
20
+
use crate::index::GraphIndex;
21
+
use crate::outline::Outline;
22
+
use crate::storage::OUTLINE_TREE;
23
+
24
+
/// A single search hit: the block, its relevance score, and an HTML
25
+
/// snippet with `<b>...</b>` marking matched terms (spec: "matches
26
+
/// highlighted").
27
+
#[derive(Debug, Clone)]
28
+
pub struct SearchHit {
29
+
pub block: TreeID,
30
+
pub score: f32,
31
+
pub snippet_html: String,
32
+
}
33
+
34
+
pub struct SearchIndex {
35
+
index: Index,
36
+
writer: RefCell<IndexWriter>,
37
+
reader: IndexReader,
38
+
id_field: Field,
39
+
content_field: Field,
40
+
}
41
+
42
+
fn build_schema() -> (Schema, Field, Field) {
43
+
let mut builder = Schema::builder();
44
+
let id_field = builder.add_text_field("id", STRING | STORED);
45
+
let content_field = builder.add_text_field("content", TEXT | STORED);
46
+
(builder.build(), id_field, content_field)
47
+
}
48
+
49
+
impl SearchIndex {
50
+
/// Open the search index at `dir`, or build it from scratch (from
51
+
/// `doc`) if the directory is missing or empty — the "rebuild on
52
+
/// missing" half of "the search index is disposable".
53
+
pub fn open_or_create(dir: impl AsRef<Path>, doc: &LoroDoc) -> tantivy::Result<Self> {
54
+
let dir = dir.as_ref().to_path_buf();
55
+
fs::create_dir_all(&dir)?;
56
+
let (schema, id_field, content_field) = build_schema();
57
+
58
+
let already_exists = dir.join("meta.json").exists();
59
+
let index = if already_exists {
60
+
Index::open_in_dir(&dir)?
61
+
} else {
62
+
Index::create_in_dir(&dir, schema)?
63
+
};
64
+
65
+
let writer: IndexWriter = index.writer(50_000_000)?;
66
+
let reader = index
67
+
.reader_builder()
68
+
.reload_policy(ReloadPolicy::OnCommitWithDelay)
69
+
.try_into()?;
70
+
71
+
let search_index = Self {
72
+
index,
73
+
writer: RefCell::new(writer),
74
+
reader,
75
+
id_field,
76
+
content_field,
77
+
};
78
+
79
+
if !already_exists {
80
+
search_index.rebuild(doc)?;
81
+
}
82
+
83
+
Ok(search_index)
84
+
}
85
+
86
+
/// Full rebuild from the graph — used for the initial build and
87
+
/// available to callers that want to force one (e.g. after detecting
88
+
/// drift, or a "rebuild search index" command).
89
+
pub fn rebuild(&self, doc: &LoroDoc) -> tantivy::Result<()> {
90
+
let outline = Outline::new(doc);
91
+
let tree = doc.get_tree(OUTLINE_TREE);
92
+
93
+
let mut writer = self.writer.borrow_mut();
94
+
writer.delete_all_documents()?;
95
+
for root in tree.roots() {
96
+
let mut stack = vec![root];
97
+
while let Some(id) = stack.pop() {
98
+
if let Ok(content) = outline.content(id) {
99
+
writer.add_document(doc!(
100
+
self.id_field => id.to_string(),
101
+
self.content_field => content,
102
+
))?;
103
+
}
104
+
stack.extend(tree.children(id).unwrap_or_default());
105
+
}
106
+
}
107
+
writer.commit()?;
108
+
self.reader.reload()?;
109
+
Ok(())
110
+
}
111
+
112
+
/// Index (or re-index) a single block's current content. Safe to call
113
+
/// on every content-changing edit: commit is synchronous, so the block
114
+
/// is searchable as soon as this returns — comfortably inside the
115
+
/// spec's 1-second freshness budget (spec: "Index freshness").
116
+
pub fn upsert_block(&self, id: TreeID, content: &str) -> tantivy::Result<()> {
117
+
let mut writer = self.writer.borrow_mut();
118
+
writer.delete_term(Term::from_field_text(self.id_field, &id.to_string()));
119
+
writer.add_document(doc!(
120
+
self.id_field => id.to_string(),
121
+
self.content_field => content,
122
+
))?;
123
+
writer.commit()?;
124
+
self.reader.reload()?;
125
+
Ok(())
126
+
}
127
+
128
+
/// Remove a block from the index (after a delete).
129
+
pub fn remove_block(&self, id: TreeID) -> tantivy::Result<()> {
130
+
let mut writer = self.writer.borrow_mut();
131
+
writer.delete_term(Term::from_field_text(self.id_field, &id.to_string()));
132
+
writer.commit()?;
133
+
self.reader.reload()?;
134
+
Ok(())
135
+
}
136
+
137
+
/// Ranked full-text search with HTML-highlighted snippets (spec:
138
+
/// "Search and jump").
139
+
pub fn search(&self, query_text: &str, limit: usize) -> tantivy::Result<Vec<SearchHit>> {
140
+
let searcher = self.reader.searcher();
141
+
let query_parser = QueryParser::for_index(&self.index, vec![self.content_field]);
142
+
let query = query_parser.parse_query(query_text)?;
143
+
144
+
let snippet_generator = SnippetGenerator::create(&searcher, &*query, self.content_field)?;
145
+
let top_docs = searcher.search(&query, &TopDocs::with_limit(limit).order_by_score())?;
146
+
147
+
let mut hits = Vec::with_capacity(top_docs.len());
148
+
for (score, address) in top_docs {
149
+
let retrieved: TantivyDocument = searcher.doc(address)?;
150
+
let Some(id_str) = retrieved.get_first(self.id_field).and_then(|v| v.as_str()) else {
151
+
continue;
152
+
};
153
+
let Ok(block) = TreeID::try_from(id_str) else {
154
+
continue;
155
+
};
156
+
let snippet = snippet_generator.snippet_from_doc(&retrieved);
157
+
hits.push(SearchHit {
158
+
block,
159
+
score,
160
+
snippet_html: snippet.to_html(),
161
+
});
162
+
}
163
+
Ok(hits)
164
+
}
165
+
}
166
+
167
+
/// Swappable "find blocks similar to this one" provider. `SearchIndex`
168
+
/// implements this with lexical (shared-term) similarity for MVP; a future
169
+
/// semantic/embedding tier can implement the same trait without any
170
+
/// consumer (UI, query engine) changing (spec: "The similarity provider
171
+
/// SHALL be a swappable interface").
172
+
pub trait Similarity {
173
+
/// Blocks similar to `block` (whose current content is `content`),
174
+
/// excluding `block` itself and its ancestors/descendants, ranked by
175
+
/// similarity (spec: "Similar blocks (lexical)").
176
+
fn similar_blocks(
177
+
&self,
178
+
block: TreeID,
179
+
content: &str,
180
+
graph: &GraphIndex,
181
+
limit: usize,
182
+
) -> tantivy::Result<Vec<SearchHit>>;
183
+
}
184
+
185
+
impl Similarity for SearchIndex {
186
+
fn similar_blocks(
187
+
&self,
188
+
block: TreeID,
189
+
content: &str,
190
+
graph: &GraphIndex,
191
+
limit: usize,
192
+
) -> tantivy::Result<Vec<SearchHit>> {
193
+
let excluded = exclusion_set(block, graph);
194
+
195
+
// Lexical similarity: query using the block's own content as the
196
+
// search text, so shared distinctive terms drive the ranking —
197
+
// the same TF-IDF machinery as ordinary search, applied to the
198
+
// block's own text instead of user-typed input.
199
+
let hits = self.search(&escape_for_query(content), limit + excluded.len())?;
200
+
Ok(hits
201
+
.into_iter()
202
+
.filter(|hit| !excluded.contains(&hit.block))
203
+
.take(limit)
204
+
.collect())
205
+
}
206
+
}
207
+
208
+
fn exclusion_set(block: TreeID, graph: &GraphIndex) -> HashSet<TreeID> {
209
+
let mut excluded = HashSet::from([block]);
210
+
211
+
// Descendants: walk graph.children, which the derived index already
212
+
// maintains — no need to touch the Loro doc here.
213
+
let mut stack = graph.children.get(&block).cloned().unwrap_or_default();
214
+
while let Some(id) = stack.pop() {
215
+
excluded.insert(id);
216
+
if let Some(kids) = graph.children.get(&id) {
217
+
stack.extend(kids.iter().copied());
218
+
}
219
+
}
220
+
221
+
// Ancestors: walk up via the same children map (no dedicated parent
222
+
// map in GraphIndex — O(pages) since outlines aren't very deep).
223
+
let mut current = block;
224
+
'outer: loop {
225
+
for (&parent, kids) in &graph.children {
226
+
if kids.contains(¤t) {
227
+
excluded.insert(parent);
228
+
current = parent;
229
+
continue 'outer;
230
+
}
231
+
}
232
+
break;
233
+
}
234
+
235
+
excluded
236
+
}
237
+
238
+
/// tantivy's default query parser treats `+ - " ( ) ^ ~ * ? : \` as query
239
+
/// syntax; escape them so a block's own Markdown content (which routinely
240
+
/// contains e.g. `-` bullets or `:`) is treated as plain terms, not a
241
+
/// malformed query.
242
+
fn escape_for_query(text: &str) -> String {
243
+
text.chars()
244
+
.map(|c| match c {
245
+
'+' | '-' | '"' | '(' | ')' | '^' | '~' | '*' | '?' | ':' | '\\' => ' ',
246
+
other => other,
247
+
})
248
+
.collect()
249
+
}
250
+
251
+
#[cfg(test)]
252
+
mod tests {
253
+
use super::*;
254
+
use crate::outline::Position;
255
+
use std::path::PathBuf;
256
+
257
+
fn temp_dir(name: &str) -> PathBuf {
258
+
let dir = std::env::temp_dir().join(format!("trawler-search-test-{name}"));
259
+
let _ = fs::remove_dir_all(&dir);
260
+
dir
261
+
}
262
+
263
+
fn doc_with_tree() -> LoroDoc {
264
+
let doc = LoroDoc::new();
265
+
doc.get_tree(OUTLINE_TREE).enable_fractional_index(0);
266
+
doc
267
+
}
268
+
269
+
#[test]
270
+
fn search_and_jump_finds_matching_block() {
271
+
// spec scenario: "Search and jump"
272
+
let doc = doc_with_tree();
273
+
let outline = Outline::new(&doc);
274
+
let page = outline
275
+
.create_block(None, Position::Index(0), "Home")
276
+
.unwrap();
277
+
let target = outline
278
+
.create_block(
279
+
Some(page),
280
+
Position::Index(0),
281
+
"watched an otter trawl the bay",
282
+
)
283
+
.unwrap();
284
+
outline
285
+
.create_block(Some(page), Position::Index(1), "unrelated grocery list")
286
+
.unwrap();
287
+
288
+
let dir = temp_dir("search-and-jump");
289
+
let index = SearchIndex::open_or_create(&dir, &doc).unwrap();
290
+
291
+
let hits = index.search("otter trawl", 10).unwrap();
292
+
assert_eq!(hits.len(), 1);
293
+
assert_eq!(hits[0].block, target);
294
+
assert!(hits[0].snippet_html.contains("<b>"));
295
+
296
+
fs::remove_dir_all(&dir).ok();
297
+
}
298
+
299
+
#[test]
300
+
fn index_freshness_finds_content_after_upsert() {
301
+
// spec scenario: "Just-typed content is findable"
302
+
let doc = doc_with_tree();
303
+
let outline = Outline::new(&doc);
304
+
let page = outline
305
+
.create_block(None, Position::Index(0), "Home")
306
+
.unwrap();
307
+
308
+
let dir = temp_dir("index-freshness");
309
+
let index = SearchIndex::open_or_create(&dir, &doc).unwrap();
310
+
311
+
let block = outline
312
+
.create_block(
313
+
Some(page),
314
+
Position::Index(0),
315
+
"a genuinely novel zyzzyx term",
316
+
)
317
+
.unwrap();
318
+
index
319
+
.upsert_block(block, &outline.content(block).unwrap())
320
+
.unwrap();
321
+
322
+
let hits = index.search("zyzzyx", 10).unwrap();
323
+
assert_eq!(hits.len(), 1);
324
+
assert_eq!(hits[0].block, block);
325
+
326
+
fs::remove_dir_all(&dir).ok();
327
+
}
328
+
329
+
#[test]
330
+
fn rebuild_after_deletion_reproduces_same_results() {
331
+
// spec scenario: "Rebuild after deletion"
332
+
let doc = doc_with_tree();
333
+
let outline = Outline::new(&doc);
334
+
let page = outline
335
+
.create_block(None, Position::Index(0), "Home")
336
+
.unwrap();
337
+
let target = outline
338
+
.create_block(Some(page), Position::Index(0), "mending the cod-end net")
339
+
.unwrap();
340
+
341
+
let dir = temp_dir("rebuild-after-deletion");
342
+
{
343
+
let index = SearchIndex::open_or_create(&dir, &doc).unwrap();
344
+
let hits = index.search("cod-end", 10).unwrap();
345
+
assert_eq!(hits.len(), 1);
346
+
}
347
+
348
+
fs::remove_dir_all(&dir).unwrap();
349
+
350
+
let rebuilt = SearchIndex::open_or_create(&dir, &doc).unwrap();
351
+
let hits = rebuilt.search("cod-end", 10).unwrap();
352
+
assert_eq!(hits.len(), 1);
353
+
assert_eq!(hits[0].block, target);
354
+
355
+
fs::remove_dir_all(&dir).ok();
356
+
}
357
+
358
+
#[test]
359
+
fn similar_blocks_excludes_self_and_ancestors_and_finds_related() {
360
+
// spec scenario: "Related note resurfaces"
361
+
let doc = doc_with_tree();
362
+
let outline = Outline::new(&doc);
363
+
let page = outline
364
+
.create_block(None, Position::Index(0), "Home")
365
+
.unwrap();
366
+
let subject = outline
367
+
.create_block(
368
+
Some(page),
369
+
Position::Index(0),
370
+
"mending the cod-end after the last haul",
371
+
)
372
+
.unwrap();
373
+
let child_of_subject = outline
374
+
.create_block(Some(subject), Position::Index(0), "cod-end thread notes")
375
+
.unwrap();
376
+
let related = outline
377
+
.create_block(
378
+
Some(page),
379
+
Position::Index(1),
380
+
"another cod-end net repair from last season",
381
+
)
382
+
.unwrap();
383
+
let unrelated = outline
384
+
.create_block(Some(page), Position::Index(2), "grocery list: milk, eggs")
385
+
.unwrap();
386
+
387
+
let dir = temp_dir("similar-blocks");
388
+
let search = SearchIndex::open_or_create(&dir, &doc).unwrap();
389
+
let graph = GraphIndex::rebuild(&doc);
390
+
391
+
let subject_content = outline.content(subject).unwrap();
392
+
let hits = search
393
+
.similar_blocks(subject, &subject_content, &graph, 10)
394
+
.unwrap();
395
+
let hit_blocks: Vec<TreeID> = hits.iter().map(|h| h.block).collect();
396
+
397
+
assert!(!hit_blocks.contains(&subject));
398
+
assert!(!hit_blocks.contains(&child_of_subject));
399
+
assert!(hit_blocks.contains(&related));
400
+
assert!(!hit_blocks.contains(&unrelated));
401
+
402
+
fs::remove_dir_all(&dir).ok();
403
+
}
404
+
}
+4
-4
openspec/changes/trawler-mvp/tasks.md
+4
-4
openspec/changes/trawler-mvp/tasks.md
···
21
21
22
22
## 3. Search Core (trawler-core)
23
23
24
-
- [ ] 3.1 Tantivy index over block content; disposable index directory with rebuild-on-missing
25
-
- [ ] 3.2 Incremental index updates within 1s of edit (spec: search/Index freshness)
26
-
- [ ] 3.3 Ranked search API with match spans for highlighting
27
-
- [ ] 3.4 Similarity trait + lexical (more-like-this) implementation excluding self/ancestors/descendants
24
+
- [x] 3.1 Tantivy index over block content; disposable index directory with rebuild-on-missing
25
+
- [x] 3.2 Incremental index updates within 1s of edit (spec: search/Index freshness)
26
+
- [x] 3.3 Ranked search API with match spans for highlighting
27
+
- [x] 3.4 Similarity trait + lexical (more-like-this) implementation excluding self/ancestors/descendants
28
28
29
29
## 4. Steel Query Engine (trawler-core)
30
30