atproto Thingiverse but good
PM-37: fix hydrant backfill collection filter
+127
-16
+1
.gitignore
+1
.gitignore
+1
Cargo.toml
+1
Cargo.toml
+54
-2
build.rs
+54
-2
build.rs
···
8
8
//! here: the server binary loads `.env` directly via `dotenvy` and reads it into
9
9
//! a typed `ServerConfig` struct (see `src/indexing/config.rs`).
10
10
11
+
use serde_json::Value;
11
12
use std::env;
12
-
use std::fs::File;
13
+
use std::fs::{self, File};
13
14
use std::io::Write;
15
+
use std::path::{Path, PathBuf};
14
16
15
17
fn main() {
16
18
println!("cargo:rerun-if-changed=.env");
···
21
23
f.write_all(
22
24
b"// This file is automatically generated by build.rs.\n\
23
25
// Frontend (WASM) compile-time configuration.\n\
24
-
// Do not edit; regenerate by rebuilding.\n\n",
26
+
// Do not edit; regenerate by rebuilding.\n",
25
27
)
26
28
.expect("write env.rs header");
27
29
···
37
39
value.replace('\\', "\\\\").replace('"', "\\\"")
38
40
);
39
41
f.write_all(line.as_bytes()).expect("write env.rs const");
42
+
}
43
+
}
44
+
generate_indexed_collections();
45
+
}
46
+
fn generate_indexed_collections() {
47
+
println!("cargo:rerun-if-changed=lexicons");
48
+
let lexicon_root = Path::new("lexicons");
49
+
let mut json_paths = Vec::new();
50
+
collect_json_paths(lexicon_root, &mut json_paths);
51
+
json_paths.sort();
52
+
let mut collections = Vec::new();
53
+
for path in json_paths {
54
+
println!("cargo:rerun-if-changed={}", path.display());
55
+
let contents = fs::read_to_string(&path)
56
+
.unwrap_or_else(|e| panic!("read authored lexicon {}: {e}", path.display()));
57
+
let lexicon: Value = serde_json::from_str(&contents)
58
+
.unwrap_or_else(|e| panic!("parse authored lexicon JSON {}: {e}", path.display()));
59
+
if lexicon.pointer("/defs/main/type").and_then(Value::as_str) != Some("record") {
60
+
continue;
61
+
}
62
+
let id = lexicon
63
+
.get("id")
64
+
.and_then(Value::as_str)
65
+
.unwrap_or_else(|| panic!("record lexicon {} has no string id", path.display()));
66
+
if id.starts_with("space.polymodel.") {
67
+
collections.push(id.to_owned());
68
+
}
69
+
}
70
+
collections.sort();
71
+
let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR set by Cargo"));
72
+
let out_path = out_dir.join("polymodel_record_collections.rs");
73
+
let mut out = File::create(&out_path)
74
+
.unwrap_or_else(|e| panic!("create generated collections {}: {e}", out_path.display()));
75
+
writeln!(out, "// Generated by build.rs from authored lexicons where defs.main.type == \"record\".\n// Do not edit.\npub const INDEXED_COLLECTIONS: &[&str] = &[").expect("write generated collections header");
76
+
for collection in collections {
77
+
writeln!(out, " {collection:?},").expect("write generated collection");
78
+
}
79
+
writeln!(out, "];").expect("write generated collections footer");
80
+
}
81
+
fn collect_json_paths(dir: &Path, paths: &mut Vec<PathBuf>) {
82
+
let entries =
83
+
fs::read_dir(dir).unwrap_or_else(|e| panic!("read lexicon dir {}: {e}", dir.display()));
84
+
for entry in entries {
85
+
let entry =
86
+
entry.unwrap_or_else(|e| panic!("read lexicon dir entry in {}: {e}", dir.display()));
87
+
let path = entry.path();
88
+
if path.is_dir() {
89
+
collect_json_paths(&path, paths);
90
+
} else if path.extension().and_then(|ext| ext.to_str()) == Some("json") {
91
+
paths.push(path);
40
92
}
41
93
}
42
94
}
+71
-14
src/indexing/setup.rs
+71
-14
src/indexing/setup.rs
···
1
1
//! Hydrant ATProto indexer setup and configuration.
2
2
//!
3
-
//! Initializes Hydrant from environment and applies a signal-mode filter so only
4
-
//! repos that publish `space.polymodel.*` records are indexed. The returned
5
-
//! handle is cheaply cloneable; the firehose/backfill driver is started later via
6
-
//! [`Hydrant::run`] in [`super::start_indexing`].
3
+
//! Initializes Hydrant from environment and applies exact Polymodel record
4
+
//! collection filters. Hydrant's by-collection crawler sends signal entries
5
+
//! directly to `listReposByCollection`, so wildcard NSIDs must not be used here.
6
+
//! The returned handle is cheaply cloneable; the firehose/backfill driver is started later via [`Hydrant::run`] in [`super::start_indexing`].
7
7
8
8
use hydrant::FilterMode;
9
9
use hydrant::control::Hydrant;
10
-
11
-
/// NSID patterns to index: the whole Polymodel namespace, used for both
12
-
/// signal-mode discovery and the collection allowlist.
13
-
const COLLECTIONS: [&str; 1] = ["space.polymodel.*"];
10
+
mod generated_collections {
11
+
include!(concat!(env!("OUT_DIR"), "/polymodel_record_collections.rs"));
12
+
}
13
+
/// Exact authored Polymodel repo record collections used for Hydrant discovery
14
+
/// and storage. Keep this exact because Hydrant sends signal entries to listReposByCollection.
15
+
const INDEXED_COLLECTIONS: &[&str] = generated_collections::INDEXED_COLLECTIONS;
14
16
15
17
/// Initialize Hydrant from environment and configure signal-mode filtering.
16
18
///
···
21
23
.await
22
24
.map_err(|e| anyhow::anyhow!("hydrant init failed: {e}"))?;
23
25
24
-
// Signal mode: only index repos that emit space.polymodel.* records, and
25
-
// only store those collections. Code takes precedence over HYDRANT_FILTER_*
26
-
// env vars, but the env vars are still documented in .env.example.
26
+
// Hydrant signal mode discovers repos publishing authored Polymodel records.
27
+
// In Hydrant's default ByCollection crawler, signals are also the exact
28
+
// values sent to listReposByCollection, so do not use wildcard patterns.
27
29
hydrant
28
30
.filter
29
31
.set_mode(FilterMode::Filter)
30
-
.set_signals(COLLECTIONS)
31
-
.set_collections(COLLECTIONS)
32
+
.set_signals(INDEXED_COLLECTIONS.iter().copied())
33
+
.set_collections(INDEXED_COLLECTIONS.iter().copied())
32
34
.apply()
33
35
.await
34
36
.map_err(|e| anyhow::anyhow!("hydrant filter apply failed: {e}"))?;
35
37
36
-
tracing::info!("hydrant initialized with signal filter for space.polymodel.*");
38
+
tracing::info!(
39
+
indexed_collections = INDEXED_COLLECTIONS.len(),
40
+
"hydrant initialized with exact authored record collection allowlist"
41
+
);
37
42
Ok(hydrant)
38
43
}
44
+
#[cfg(test)]
45
+
mod tests {
46
+
use super::*;
47
+
#[test]
48
+
fn hydrant_filters_use_exact_record_collections() {
49
+
assert_eq!(
50
+
INDEXED_COLLECTIONS,
51
+
generated_collections::INDEXED_COLLECTIONS
52
+
);
53
+
for collection in INDEXED_COLLECTIONS {
54
+
assert!(
55
+
!collection.contains('*'),
56
+
"indexed collection must be exact, got {collection}"
57
+
);
58
+
assert!(
59
+
!collection.ends_with(".*"),
60
+
"indexed collection must not be a wildcard, got {collection}"
61
+
);
62
+
}
63
+
let expected_records = [
64
+
"space.polymodel.actor.profile",
65
+
"space.polymodel.graph.like",
66
+
"space.polymodel.graph.list",
67
+
"space.polymodel.graph.listitem",
68
+
"space.polymodel.graph.save",
69
+
"space.polymodel.graph.tag",
70
+
"space.polymodel.library.model",
71
+
"space.polymodel.library.part",
72
+
"space.polymodel.library.thing",
73
+
];
74
+
for collection in expected_records {
75
+
assert!(
76
+
generated_collections::INDEXED_COLLECTIONS.contains(&collection),
77
+
"missing generated record collection {collection}"
78
+
);
79
+
}
80
+
let excluded_non_records = [
81
+
"space.polymodel.library.getFeed",
82
+
"space.polymodel.library.getThing",
83
+
"space.polymodel.library.getAuthorThings",
84
+
"space.polymodel.library.searchThings",
85
+
"space.polymodel.graph.getList",
86
+
"space.polymodel.library.defs",
87
+
];
88
+
for collection in excluded_non_records {
89
+
assert!(
90
+
!generated_collections::INDEXED_COLLECTIONS.contains(&collection),
91
+
"non-record lexicon must not be indexed as a collection: {collection}"
92
+
);
93
+
}
94
+
}
95
+
}