[READ-ONLY] Mirror of https://github.com/flo-bit/contrail. atproto backend in a bottle
flo-bit.dev/contrail/
add feed stuff (better not to use that yet ^^)
+724
-198
+19
-8
README.md
+19
-8
README.md
···
23
23
},
24
24
},
25
25
},
26
-
"community.lexicon.calendar.rsvp": {},
27
-
}
26
+
"community.lexicon.calendar.rsvp": {
27
+
references: {
28
+
event: {
29
+
collection: "community.lexicon.calendar.event",
30
+
field: "subject.uri",
31
+
},
32
+
},
33
+
},
34
+
},
28
35
};
29
36
```
30
37
···
70
77
| `relations.*.field` | `"subject.uri"` | Field in the related record to match against |
71
78
| `relations.*.match` | `"uri"` | Match against parent's `"uri"` or `"did"` |
72
79
| `relations.*.groupBy` | — | Split counts by this field's value |
73
-
| `queries` | `{}` | Custom query handlers |
74
-
| `searchable` | auto-detected | FTS5 search fields. `string[]` = explicit fields, `false` = disabled, omitted = all non-range queryable fields |
80
+
| `references` | `{}` | Forward references to other collections for hydration |
81
+
| `references.*.collection` | — | Target collection NSID |
82
+
| `references.*.field` | — | Field containing the target record's AT URI |
83
+
| `queries` | `{}` | Custom query handlers (raw Response) |
84
+
| `pipelineQueries` | `{}` | Custom query handlers that go through the standard filter/sort/hydration pipeline |
85
+
| `searchable` | disabled | FTS5 search fields. Provide `string[]` to enable, omit to disable |
75
86
76
87
### Profiles
77
88
···
100
111
|-------|---------|-------------|
101
112
| `actor` | `?actor=did:plc:...` or `?actor=alice.bsky.social` | Filter by DID or handle (triggers on-demand backfill) |
102
113
| `profiles` | `?profiles=true` | Include profile + identity info keyed by DID |
103
-
| `search` | `?search=meetup` | Full-text search across searchable fields (FTS5, ranked) |
114
+
| `search` | `?search=meetup` | Full-text search across searchable fields (FTS5, ranked). Requires `searchable` to be configured. |
104
115
| `{field}` | `?status=going` | Equality filter on queryable string field |
105
116
| `{field}Min` | `?startsAtMin=2026-03-16` | Range minimum (datetime/integer fields) |
106
117
| `{field}Max` | `?endsAtMax=2026-04-01` | Range maximum (datetime/integer fields) |
···
110
121
| `hydrate{Ref}` | `?hydrateEvent=true` | Embed the referenced record |
111
122
| `sort` | `?sort=startsAt` | Sort by a queryable field or count (see below) |
112
123
| `order` | `?order=asc` | Sort direction: `asc` or `desc` (default depends on field type) |
113
-
| `limit` | `?limit=25` | Page size (1-100, default 50) |
124
+
| `limit` | `?limit=25` | Page size (1-200, default 50) |
114
125
| `cursor` | `?cursor=...` | Pagination cursor |
115
126
116
127
**Sorting** — `sort` accepts any queryable field param name or a count field:
···
122
133
?sort=rsvpsGoingCount&order=asc # by going count ascending
123
134
```
124
135
125
-
**Search** uses SQLite FTS5 for ranked full-text search. By default, all non-range queryable fields are searchable. Results are ranked by relevance (BM25) with `time_us` as tiebreaker. Supports FTS5 syntax including prefix (`meetup*`), phrases (`"rust meetup"`), and boolean (`rust OR typescript`). Combinable with all other filters.
136
+
**Search** uses SQLite FTS5 for ranked full-text search. To enable, set `searchable: ["field1", "field2"]` on a collection. Results are ranked by relevance (BM25) with `time_us` as tiebreaker. Supports FTS5 syntax including prefix (`meetup*`), phrases (`"rust meetup"`), and boolean (`rust OR typescript`). Combinable with all other filters.
126
137
127
138
```
128
139
?search=meetup # basic search
···
150
161
# Single event with counts, RSVPs, and profiles
151
162
/xrpc/community.lexicon.calendar.event.getRecord?uri=at://did:plc:.../community.lexicon.calendar.event/...&hydrateRsvps=10&profiles=true
152
163
153
-
# Search for events by name/description
164
+
# Search for events by name/description (requires searchable config)
154
165
/xrpc/community.lexicon.calendar.event.listRecords?search=meetup&profiles=true
155
166
156
167
# RSVPs for a specific event, with the referenced event embedded
-4
lexicons-generated/community/lexicon/calendar/event/listRecords.json
-4
lexicons-generated/community/lexicon/calendar/event/listRecords.json
···
26
26
"type": "boolean",
27
27
"description": "Include profile + identity info keyed by DID"
28
28
},
29
-
"search": {
30
-
"type": "string",
31
-
"description": "Full-text search across: mode, name, status, description"
32
-
},
33
29
"mode": {
34
30
"type": "string",
35
31
"description": "Filter by mode"
-4
lexicons-generated/community/lexicon/calendar/rsvp/listRecords.json
-4
lexicons-generated/community/lexicon/calendar/rsvp/listRecords.json
···
26
26
"type": "boolean",
27
27
"description": "Include profile + identity info keyed by DID"
28
28
},
29
-
"search": {
30
-
"type": "string",
31
-
"description": "Full-text search across: status, subject.uri"
32
-
},
33
29
"status": {
34
30
"type": "string",
35
31
"description": "Filter by status"
+9
src/config.ts
+9
src/config.ts
+1
src/core/backfill.ts
+1
src/core/backfill.ts
+2
-1
src/core/db/index.ts
+2
-1
src/core/db/index.ts
···
1
1
export { initSchema } from "./schema";
2
-
export { getLastCursor, saveCursor, applyEvents, queryRecords } from "./records";
2
+
export { getLastCursor, saveCursor, applyEvents, queryRecords, pruneFeedItems } from "./records";
3
3
export type { QueryOptions, SortOption } from "./records";
4
+
export type { RecordSource } from "../types";
+128
-5
src/core/db/records.ts
+128
-5
src/core/db/records.ts
···
5
5
Statement,
6
6
IngestEvent,
7
7
RecordRow,
8
+
RecordSource,
8
9
} from "../types";
9
-
import { getNestedValue, getRelationField, countColumnName } from "../types";
10
+
import { getNestedValue, getRelationField, countColumnName, getFeedFollowCollections } from "../types";
10
11
import { resolvedRelationsMap } from "../queryable.generated";
11
12
import { getSearchableFields, ftsTableName, buildFtsContent } from "../search";
12
13
···
104
105
return stmts;
105
106
}
106
107
108
+
// --- Feeds ---
109
+
110
+
function buildFeedStatements(
111
+
db: Database,
112
+
event: IngestEvent,
113
+
config: ContrailConfig,
114
+
existingRecords: Map<string, string | null>
115
+
): Statement[] {
116
+
if (!config.feeds) return [];
117
+
118
+
const stmts: Statement[] = [];
119
+
120
+
for (const [, feedConfig] of Object.entries(config.feeds)) {
121
+
// Target collection: fan out to followers
122
+
if (feedConfig.targets.includes(event.collection)) {
123
+
if (event.operation === "create" || event.operation === "update") {
124
+
// Insert feed items for all followers of the event creator.
125
+
// Follow records have: did = follower, record.subject = followed person.
126
+
stmts.push(
127
+
db
128
+
.prepare(
129
+
`INSERT OR IGNORE INTO feed_items (actor, uri, collection, time_us)
130
+
SELECT r.did, ?, ?, ?
131
+
FROM records r
132
+
WHERE r.collection = ?
133
+
AND json_extract(r.record, '$.subject') = ?`
134
+
)
135
+
.bind(event.uri, event.collection, event.time_us, feedConfig.follow, event.did)
136
+
);
137
+
} else if (event.operation === "delete") {
138
+
stmts.push(
139
+
db.prepare("DELETE FROM feed_items WHERE uri = ?").bind(event.uri)
140
+
);
141
+
}
142
+
}
143
+
144
+
// Follow collection: handle follow/unfollow
145
+
if (event.collection === feedConfig.follow) {
146
+
if (event.operation === "create") {
147
+
const record = event.record ? JSON.parse(event.record) : null;
148
+
const subject = record?.subject;
149
+
if (subject) {
150
+
// New follow: backfill recent items from the followed user
151
+
for (const targetCol of feedConfig.targets) {
152
+
stmts.push(
153
+
db
154
+
.prepare(
155
+
`INSERT OR IGNORE INTO feed_items (actor, uri, collection, time_us)
156
+
SELECT ?, r.uri, r.collection, r.time_us
157
+
FROM records r
158
+
WHERE r.collection = ? AND r.did = ?
159
+
ORDER BY r.time_us DESC
160
+
LIMIT 100`
161
+
)
162
+
.bind(event.did, targetCol, subject)
163
+
);
164
+
}
165
+
}
166
+
} else if (event.operation === "delete") {
167
+
// Unfollow: remove feed items from the unfollowed user.
168
+
// The record field is null for deletes, so we look up the existing record.
169
+
const existingRecord = existingRecords.get(event.uri);
170
+
if (existingRecord) {
171
+
const parsed = JSON.parse(existingRecord);
172
+
const subject = parsed?.subject;
173
+
if (subject) {
174
+
const targetPlaceholders = feedConfig.targets.map(() => "?").join(",");
175
+
stmts.push(
176
+
db
177
+
.prepare(
178
+
`DELETE FROM feed_items WHERE actor = ? AND uri IN (
179
+
SELECT uri FROM records WHERE did = ? AND collection IN (${targetPlaceholders})
180
+
)`
181
+
)
182
+
.bind(event.did, subject, ...feedConfig.targets)
183
+
);
184
+
}
185
+
}
186
+
}
187
+
}
188
+
}
189
+
190
+
return stmts;
191
+
}
192
+
193
+
// --- Feed pruning ---
194
+
195
+
export async function pruneFeedItems(
196
+
db: Database,
197
+
maxItems: number
198
+
): Promise<number> {
199
+
const result = await db
200
+
.prepare(
201
+
`DELETE FROM feed_items WHERE rowid NOT IN (
202
+
SELECT rowid FROM (
203
+
SELECT rowid, ROW_NUMBER() OVER (PARTITION BY actor ORDER BY time_us DESC) as rn
204
+
FROM feed_items
205
+
) WHERE rn <= ?
206
+
)`
207
+
)
208
+
.bind(maxItems)
209
+
.run();
210
+
return (result as any)?.changes ?? 0;
211
+
}
212
+
107
213
// --- Cursor ---
108
214
109
215
export async function getLastCursor(db: Database): Promise<number | null> {
···
131
237
db: Database,
132
238
events: IngestEvent[],
133
239
config?: ContrailConfig,
134
-
options?: { skipReplayDetection?: boolean }
240
+
options?: { skipReplayDetection?: boolean; skipFeedFanout?: boolean }
135
241
): Promise<void> {
136
242
if (events.length === 0) return;
137
243
138
244
// Look up existing records so we can skip duplicate count updates on replayed events.
139
245
// A create/update with the same CID is a replay; a delete for a missing URI is a replay.
140
246
// Can be skipped during backfill where records are known to be fresh inserts.
247
+
// Also fetches record content for follow-delete events (needed for unfollow feed cleanup).
141
248
const existingCids = new Map<string, string | null>();
249
+
const existingRecords = new Map<string, string | null>();
250
+
const followCollections = config ? getFeedFollowCollections(config) : [];
251
+
const needRecordContent = followCollections.length > 0;
252
+
142
253
if (config && !options?.skipReplayDetection) {
143
254
const uris = events.map((e) => e.uri);
255
+
const selectCols = needRecordContent ? "uri, cid, record" : "uri, cid";
144
256
for (let i = 0; i < uris.length; i += 50) {
145
257
const chunk = uris.slice(i, i + 50);
146
258
const placeholders = chunk.map(() => "?").join(",");
147
259
const rows = await db
148
-
.prepare(`SELECT uri, cid FROM records WHERE uri IN (${placeholders})`)
260
+
.prepare(`SELECT ${selectCols} FROM records WHERE uri IN (${placeholders})`)
149
261
.bind(...chunk)
150
-
.all<{ uri: string; cid: string | null }>();
262
+
.all<{ uri: string; cid: string | null; record?: string | null }>();
151
263
for (const row of rows.results ?? []) {
152
264
existingCids.set(row.uri, row.cid);
265
+
if (needRecordContent && row.record) {
266
+
existingRecords.set(row.uri, row.record);
267
+
}
153
268
}
154
269
}
155
270
}
···
191
306
192
307
if (!isReplay) {
193
308
batch.push(...buildCountStatements(db, e, config));
309
+
if (!options?.skipFeedFanout) {
310
+
batch.push(...buildFeedStatements(db, e, config, existingRecords));
311
+
}
194
312
}
195
313
batch.push(...buildFtsStatements(db, e, config));
196
314
}
···
237
355
countFilters?: Record<string, number>;
238
356
sort?: SortOption;
239
357
search?: string;
358
+
source?: RecordSource;
240
359
}
241
360
242
361
export async function queryRecords(
···
254
373
countFilters = {},
255
374
sort,
256
375
search,
376
+
source,
257
377
} = options;
258
378
259
379
const limit = Math.min(Math.max(1, rawLimit ?? 50), 200);
260
380
const conditions: string[] = ["r.collection = ?"];
261
381
const bindings: (string | number)[] = [collection];
382
+
383
+
if (source?.conditions) conditions.push(...source.conditions);
384
+
if (source?.params) bindings.push(...source.params);
262
385
263
386
const countCols = getCountColumns(config, collection);
264
387
···
339
462
: "";
340
463
const select = `r.uri, r.did, r.collection, r.rkey, r.cid, r.record, r.time_us, r.indexed_at${countSelect}`;
341
464
342
-
const join = ftsJoin;
465
+
const join = [source?.joins, ftsJoin].filter(Boolean).join(" ");
343
466
344
467
let orderBy: string;
345
468
if (sort?.recordField) {
+34
-1
src/core/db/schema.ts
+34
-1
src/core/db/schema.ts
···
112
112
return stmts;
113
113
}
114
114
115
+
function buildFeedTables(config: ContrailConfig): string[] {
116
+
if (!config.feeds || Object.keys(config.feeds).length === 0) return [];
117
+
const stmts = [
118
+
`CREATE TABLE IF NOT EXISTS feed_items (
119
+
actor TEXT NOT NULL,
120
+
uri TEXT NOT NULL,
121
+
collection TEXT NOT NULL,
122
+
time_us INTEGER NOT NULL,
123
+
PRIMARY KEY (actor, uri)
124
+
)`,
125
+
`CREATE INDEX IF NOT EXISTS idx_feed_actor_coll_time ON feed_items(actor, collection, time_us DESC)`,
126
+
`CREATE INDEX IF NOT EXISTS idx_feed_actor_time ON feed_items(actor, time_us DESC)`,
127
+
`CREATE TABLE IF NOT EXISTS feed_backfills (
128
+
actor TEXT NOT NULL,
129
+
feed TEXT NOT NULL,
130
+
completed INTEGER NOT NULL DEFAULT 0,
131
+
PRIMARY KEY (actor, feed)
132
+
)`,
133
+
];
134
+
135
+
// Index follow collections on subject for efficient fan-out lookups
136
+
const followCollections = new Set(Object.values(config.feeds).map((f) => f.follow));
137
+
for (const col of followCollections) {
138
+
const safe = col.replace(/[^a-zA-Z0-9]/g, "_");
139
+
stmts.push(
140
+
`CREATE INDEX IF NOT EXISTS idx_${safe}_subject ON records(collection, json_extract(record, '$.subject')) WHERE collection = '${col}'`
141
+
);
142
+
}
143
+
144
+
return stmts;
145
+
}
146
+
115
147
function buildFtsTables(config: ContrailConfig): string[] {
116
148
const stmts: string[] = [];
117
149
for (const [collection, colConfig] of Object.entries(config.collections)) {
···
150
182
151
183
const indexStatements = buildDynamicIndexes(config);
152
184
const ftsStatements = buildFtsTables(config);
153
-
const all = [...baseStatements, ...indexStatements, ...ftsStatements];
185
+
const feedStatements = buildFeedTables(config);
186
+
const all = [...baseStatements, ...indexStatements, ...ftsStatements, ...feedStatements];
154
187
155
188
await db.batch(all.map((s) => db.prepare(s)));
156
189
await runMigrations(db);
+21
-15
src/core/jetstream.ts
+21
-15
src/core/jetstream.ts
···
1
1
import { JetstreamSubscription } from "@atcute/jetstream";
2
2
import type { ContrailConfig, IngestEvent, Database } from "./types";
3
-
import { getCollectionNames, getDependentCollections } from "./types";
4
-
import { initSchema, getLastCursor, saveCursor, applyEvents } from "./db";
3
+
import { getCollectionNames, getDependentCollections, DEFAULT_FEED_MAX_ITEMS } from "./types";
4
+
import { initSchema, getLastCursor, saveCursor, applyEvents, pruneFeedItems } from "./db";
5
5
import { refreshStaleIdentities } from "./identity";
6
6
7
7
const BATCH_SIZE = 50;
8
8
9
-
// Cache known DIDs in memory across ingest cycles (survives within the same Worker isolate)
9
+
// Cache state in memory across ingest cycles (survives within the same Worker isolate)
10
10
let cachedKnownDids: Set<string> | undefined;
11
+
let schemaInitialized = false;
12
+
let lastFeedPruneMs = 0;
13
+
const FEED_PRUNE_INTERVAL_MS = 60 * 60 * 1000; // 1 hour
11
14
12
15
export async function ingestEvents(
13
16
config: ContrailConfig,
···
92
95
config: ContrailConfig,
93
96
timeoutMs: number = 25_000
94
97
): Promise<void> {
95
-
await initSchema(db, config);
98
+
if (!schemaInitialized) {
99
+
await initSchema(db, config);
100
+
schemaInitialized = true;
101
+
}
96
102
97
103
const cursor = await getLastCursor(db);
98
104
const collections = getCollectionNames(config);
···
144
150
}
145
151
146
152
if (lastCursor !== null) {
147
-
// Use the later of the subscription cursor and the current time, so the
148
-
// cursor always reaches the present even when no events were received.
149
-
const nowUs = Date.now() * 1000;
150
-
const effectiveCursor = Math.max(lastCursor, nowUs);
153
+
await saveCursor(db, lastCursor);
154
+
console.log(`Saved cursor: ${lastCursor}`);
155
+
}
151
156
152
-
// Roll back cursor by 60s so the next cycle replays a small window.
153
-
// This guards against missed events when switching between Jetstream instances
154
-
// or out-of-order delivery. Duplicate events are handled safely in applyEvents.
155
-
const safetyMarginUs = 60_000_000;
156
-
const safeCursor = Math.max(0, effectiveCursor - safetyMarginUs);
157
-
await saveCursor(db, safeCursor);
158
-
console.log(`Saved cursor: ${safeCursor} (rolled back 60s from ${effectiveCursor})`);
157
+
// Prune feed items hourly
158
+
if (config.feeds && Date.now() - lastFeedPruneMs > FEED_PRUNE_INTERVAL_MS) {
159
+
const maxItems = Math.max(
160
+
...Object.values(config.feeds).map((f) => f.maxItems ?? DEFAULT_FEED_MAX_ITEMS)
161
+
);
162
+
const pruned = await pruneFeedItems(db, maxItems);
163
+
if (pruned > 0) console.log(`Pruned ${pruned} old feed items`);
164
+
lastFeedPruneMs = Date.now();
159
165
}
160
166
161
167
console.log(`Ingestion complete. Stored ${events.length} events.`);
+174
-137
src/core/router/collection.ts
+174
-137
src/core/router/collection.ts
···
1
1
import type { Hono } from "hono";
2
-
import type { ContrailConfig, Database, RecordRow, QueryableField } from "../types";
2
+
import type { ContrailConfig, Database, RecordRow, QueryableField, RecordSource } from "../types";
3
3
import { getCollectionNames, countColumnName } from "../types";
4
4
import { resolvedQueryable, resolvedRelationsMap } from "../queryable.generated";
5
5
import { queryRecords } from "../db";
···
11
11
import type { FormattedRecord } from "./helpers";
12
12
import { formatRecord, parseIntParam, fieldToParam } from "./helpers";
13
13
14
-
export function registerCollectionRoutes(
15
-
app: Hono,
14
+
export async function runPipeline(
16
15
db: Database,
17
-
config: ContrailConfig
18
-
): void {
19
-
for (const collection of getCollectionNames(config)) {
20
-
const colConfig = config.collections[collection];
21
-
const relations = colConfig.relations ?? {};
22
-
const references = colConfig.references ?? {};
23
-
const queryableFields: Record<string, QueryableField> =
24
-
resolvedQueryable[collection] ?? colConfig.queryable ?? {};
16
+
config: ContrailConfig,
17
+
collection: string,
18
+
params: URLSearchParams,
19
+
source?: RecordSource
20
+
): Promise<{ records: FormattedRecord[]; cursor?: string; profiles?: any[] }> {
21
+
const colConfig = config.collections[collection];
22
+
if (!colConfig) throw new Error(`Unknown collection: ${collection}`);
25
23
26
-
app.get(`/xrpc/${collection}.listRecords`, async (c) => {
27
-
const params = new URL(c.req.url).searchParams;
28
-
const limit = parseIntParam(params.get("limit"), 50);
29
-
const cursor = params.get("cursor") || undefined;
30
-
const actor = params.get("actor") || params.get("did") || undefined;
31
-
const wantProfiles = params.get("profiles") === "true";
24
+
const relations = colConfig.relations ?? {};
25
+
const references = colConfig.references ?? {};
26
+
const queryableFields: Record<string, QueryableField> =
27
+
resolvedQueryable[collection] ?? colConfig.queryable ?? {};
32
28
33
-
let did: string | undefined;
34
-
if (actor) {
35
-
const resolved = await resolveActor(db, actor);
36
-
if (!resolved) return c.json({ error: "Could not resolve actor" }, 400);
37
-
did = resolved;
38
-
await backfillUser(db, did, collection, Date.now() + 10_000, config);
29
+
const limit = parseIntParam(params.get("limit"), 50);
30
+
const cursor = params.get("cursor") || undefined;
31
+
const actor = params.get("actor") || params.get("did") || undefined;
32
+
const wantProfiles = params.get("profiles") === "true";
33
+
34
+
let did: string | undefined;
35
+
if (actor) {
36
+
const resolved = await resolveActor(db, actor);
37
+
if (!resolved) throw new Error("Could not resolve actor");
38
+
did = resolved;
39
+
await backfillUser(db, did, collection, Date.now() + 10_000, config);
40
+
}
41
+
42
+
const filters: Record<string, string> = {};
43
+
const rangeFilters: Record<string, { min?: string; max?: string }> = {};
44
+
for (const [field, fieldConfig] of Object.entries(queryableFields)) {
45
+
const param = fieldToParam(field);
46
+
if (fieldConfig.type === "range") {
47
+
const min = params.get(`${param}Min`);
48
+
const max = params.get(`${param}Max`);
49
+
if (min || max) {
50
+
rangeFilters[field] = {};
51
+
if (min) rangeFilters[field].min = min;
52
+
if (max) rangeFilters[field].max = max;
39
53
}
54
+
} else {
55
+
const value = params.get(param);
56
+
if (value) filters[field] = value;
57
+
}
58
+
}
40
59
41
-
const filters: Record<string, string> = {};
42
-
const rangeFilters: Record<string, { min?: string; max?: string }> = {};
43
-
for (const [field, fieldConfig] of Object.entries(queryableFields)) {
44
-
const param = fieldToParam(field);
45
-
if (fieldConfig.type === "range") {
46
-
const min = params.get(`${param}Min`);
47
-
const max = params.get(`${param}Max`);
48
-
if (min || max) {
49
-
rangeFilters[field] = {};
50
-
if (min) rangeFilters[field].min = min;
51
-
if (max) rangeFilters[field].max = max;
52
-
}
53
-
} else {
54
-
const value = params.get(param);
55
-
if (value) filters[field] = value;
56
-
}
60
+
const countFilters: Record<string, number> = {};
61
+
const relMap = resolvedRelationsMap[collection] ?? {};
62
+
for (const [relName, rel] of Object.entries(relations)) {
63
+
const totalMin = parseIntParam(params.get(`${relName}CountMin`));
64
+
if (totalMin != null) countFilters[rel.collection] = totalMin;
65
+
const mapping = relMap[relName];
66
+
if (mapping) {
67
+
const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1);
68
+
for (const [shortName, fullToken] of Object.entries(mapping.groups)) {
69
+
const val = parseIntParam(params.get(`${relName}${capitalize(shortName)}CountMin`));
70
+
if (val != null) countFilters[fullToken] = val;
57
71
}
72
+
}
73
+
}
58
74
59
-
const countFilters: Record<string, number> = {};
60
-
const relMap = resolvedRelationsMap[collection] ?? {};
75
+
let sort: SortOption | undefined;
76
+
const sortParam = params.get("sort");
77
+
if (sortParam) {
78
+
const orderParam = params.get("order");
79
+
80
+
const fieldEntry = Object.entries(queryableFields).find(
81
+
([field]) => fieldToParam(field) === sortParam
82
+
);
83
+
if (fieldEntry) {
84
+
const defaultDir = fieldEntry[1].type === "range" ? "desc" : "asc";
85
+
const direction = orderParam === "asc" ? "asc" as const : orderParam === "desc" ? "desc" as const : defaultDir as "asc" | "desc";
86
+
sort = { recordField: fieldEntry[0], direction };
87
+
} else {
88
+
const direction = orderParam === "asc" ? "asc" as const : "desc" as const;
89
+
const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1);
61
90
for (const [relName, rel] of Object.entries(relations)) {
62
-
const totalMin = parseIntParam(params.get(`${relName}CountMin`));
63
-
if (totalMin != null) countFilters[rel.collection] = totalMin;
91
+
if (sortParam === `${relName}Count`) {
92
+
sort = { countType: rel.collection, direction };
93
+
break;
94
+
}
64
95
const mapping = relMap[relName];
65
96
if (mapping) {
66
-
const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1);
67
97
for (const [shortName, fullToken] of Object.entries(mapping.groups)) {
68
-
const val = parseIntParam(params.get(`${relName}${capitalize(shortName)}CountMin`));
69
-
if (val != null) countFilters[fullToken] = val;
98
+
if (sortParam === `${relName}${capitalize(shortName)}Count`) {
99
+
sort = { countType: fullToken, direction };
100
+
break;
101
+
}
70
102
}
103
+
if (sort) break;
71
104
}
72
105
}
106
+
}
107
+
}
73
108
74
-
// Resolve sort option
75
-
let sort: SortOption | undefined;
76
-
const sortParam = params.get("sort");
77
-
if (sortParam) {
78
-
const orderParam = params.get("order");
109
+
const search = params.get("search") || undefined;
110
+
111
+
const result = await queryRecords(db, config, {
112
+
collection,
113
+
did,
114
+
limit,
115
+
cursor,
116
+
filters,
117
+
rangeFilters,
118
+
countFilters,
119
+
sort,
120
+
search,
121
+
source,
122
+
});
123
+
124
+
const rows = result.records;
125
+
const hydrateRequested = parseHydrateParams(params, relations, references);
126
+
const hydrates = await resolveHydrates(
127
+
db,
128
+
relations,
129
+
hydrateRequested.relations,
130
+
rows
131
+
);
132
+
const refs = await resolveReferences(
133
+
db,
134
+
references,
135
+
hydrateRequested.references,
136
+
rows
137
+
);
79
138
80
-
// Check if it's a queryable field (param name → json path)
81
-
const fieldEntry = Object.entries(queryableFields).find(
82
-
([field]) => fieldToParam(field) === sortParam
83
-
);
84
-
if (fieldEntry) {
85
-
// Default: desc for range fields (dates, numbers), asc for others
86
-
const defaultDir = fieldEntry[1].type === "range" ? "desc" : "asc";
87
-
const direction = orderParam === "asc" ? "asc" as const : orderParam === "desc" ? "desc" as const : defaultDir as "asc" | "desc";
88
-
sort = { recordField: fieldEntry[0], direction };
89
-
} else {
90
-
// Count fields default to desc (you usually want "most X first")
91
-
const direction = orderParam === "asc" ? "asc" as const : "desc" as const;
92
-
// Check if it's a count field (e.g. rsvpsCount → collection NSID, rsvpsGoingCount → full token)
93
-
const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1);
94
-
for (const [relName, rel] of Object.entries(relations)) {
95
-
if (sortParam === `${relName}Count`) {
96
-
sort = { countType: rel.collection, direction };
97
-
break;
98
-
}
99
-
const mapping = relMap[relName];
100
-
if (mapping) {
101
-
for (const [shortName, fullToken] of Object.entries(mapping.groups)) {
102
-
if (sortParam === `${relName}${capitalize(shortName)}Count`) {
103
-
sort = { countType: fullToken, direction };
104
-
break;
105
-
}
106
-
}
107
-
if (sort) break;
108
-
}
109
-
}
110
-
}
139
+
const formattedRecords: FormattedRecord[] = rows.map((row) => {
140
+
const formatted = formatRecord(row);
141
+
flattenCounts(formatted, row.counts, collection, relations);
142
+
const h = hydrates[row.uri];
143
+
if (h) {
144
+
for (const [relName, groups] of Object.entries(h)) {
145
+
formatted[relName] = groups;
146
+
}
147
+
}
148
+
const r = refs[row.uri];
149
+
if (r) {
150
+
for (const [refName, record] of Object.entries(r)) {
151
+
formatted[refName] = record;
111
152
}
153
+
}
154
+
return formatted;
155
+
});
112
156
113
-
const search = params.get("search") || undefined;
157
+
const allDids = collectDids(rows, hydrates);
158
+
const profileMap = wantProfiles
159
+
? await resolveProfiles(db, config, allDids)
160
+
: undefined;
114
161
115
-
const result = await queryRecords(db, config, {
116
-
collection,
117
-
did,
118
-
limit,
119
-
cursor,
120
-
filters,
121
-
rangeFilters,
122
-
countFilters,
123
-
sort,
124
-
search,
125
-
});
162
+
return {
163
+
records: formattedRecords,
164
+
cursor: result.cursor,
165
+
...(profileMap ? { profiles: Object.values(profileMap) } : {}),
166
+
};
167
+
}
126
168
127
-
const rows = result.records;
128
-
const hydrateRequested = parseHydrateParams(params, relations, references);
129
-
const hydrates = await resolveHydrates(
130
-
db,
131
-
relations,
132
-
hydrateRequested.relations,
133
-
rows
134
-
);
135
-
const refs = await resolveReferences(
136
-
db,
137
-
references,
138
-
hydrateRequested.references,
139
-
rows
140
-
);
169
+
export function registerCollectionRoutes(
170
+
app: Hono,
171
+
db: Database,
172
+
config: ContrailConfig
173
+
): void {
174
+
for (const collection of getCollectionNames(config)) {
175
+
const colConfig = config.collections[collection];
141
176
142
-
const formattedRecords: FormattedRecord[] = rows.map((row) => {
143
-
const formatted = formatRecord(row);
144
-
flattenCounts(formatted, row.counts, collection, relations);
145
-
const h = hydrates[row.uri];
146
-
if (h) {
147
-
for (const [relName, groups] of Object.entries(h)) {
148
-
formatted[relName] = groups;
149
-
}
177
+
app.get(`/xrpc/${collection}.listRecords`, async (c) => {
178
+
const params = new URL(c.req.url).searchParams;
179
+
try {
180
+
const result = await runPipeline(db, config, collection, params);
181
+
return c.json(result);
182
+
} catch (e: any) {
183
+
if (e.message === "Could not resolve actor") {
184
+
return c.json({ error: e.message }, 400);
150
185
}
151
-
const r = refs[row.uri];
152
-
if (r) {
153
-
for (const [refName, record] of Object.entries(r)) {
154
-
formatted[refName] = record;
155
-
}
156
-
}
157
-
return formatted;
158
-
});
159
-
160
-
const allDids = collectDids(rows, hydrates);
161
-
const profileMap = wantProfiles
162
-
? await resolveProfiles(db, config, allDids)
163
-
: undefined;
164
-
165
-
return c.json({
166
-
records: formattedRecords,
167
-
cursor: result.cursor,
168
-
...(profileMap ? { profiles: Object.values(profileMap) } : {}),
169
-
});
186
+
throw e;
187
+
}
170
188
});
171
189
172
190
app.get(`/xrpc/${collection}.getRecord`, async (c) => {
173
191
const uri = c.req.query("uri");
174
192
if (!uri) return c.json({ error: "uri parameter required" }, 400);
193
+
194
+
const relations = colConfig.relations ?? {};
195
+
const references = colConfig.references ?? {};
175
196
176
197
const row = await db
177
198
.prepare(
···
234
255
return handler(db, params, config);
235
256
});
236
257
}
258
+
259
+
for (const [queryName, handler] of Object.entries(
260
+
colConfig.pipelineQueries ?? {}
261
+
)) {
262
+
app.get(`/xrpc/${collection}.${queryName}`, async (c) => {
263
+
const params = new URL(c.req.url).searchParams;
264
+
try {
265
+
const source = await handler(db, params, config);
266
+
const result = await runPipeline(db, config, collection, params, source);
267
+
return c.json(result);
268
+
} catch (e: any) {
269
+
if (e.message === "Could not resolve actor") {
270
+
return c.json({ error: e.message }, 400);
271
+
}
272
+
throw e;
273
+
}
274
+
});
275
+
}
237
276
}
238
277
}
239
278
···
273
312
const relMap = resolvedRelationsMap[collection] ?? {};
274
313
const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1);
275
314
276
-
// Build reverse lookups: collection NSID → relName (for totals), full token → field name (for groups)
277
315
const collectionToRelName: Record<string, string> = {};
278
316
const tokenToField: Record<string, string> = {};
279
317
for (const [relName, mapping] of Object.entries(relMap)) {
···
282
320
tokenToField[fullToken] = `${relName}${capitalize(shortName)}Count`;
283
321
}
284
322
}
285
-
// Also map relations without groupBy (no entry in relMap)
286
323
for (const [relName, rel] of Object.entries(relations)) {
287
324
if (!collectionToRelName[rel.collection]) {
288
325
collectionToRelName[rel.collection] = relName;
+127
src/core/router/feed.ts
+127
src/core/router/feed.ts
···
1
+
import type { Hono } from "hono";
2
+
import type { ContrailConfig, Database, FeedConfig } from "../types";
3
+
import { DEFAULT_FEED_MAX_ITEMS } from "../types";
4
+
import { resolveActor } from "../identity";
5
+
import { backfillUser } from "../backfill";
6
+
import { runPipeline } from "./collection";
7
+
8
+
async function maybeBackfillFeed(
9
+
db: Database,
10
+
config: ContrailConfig,
11
+
actor: string,
12
+
feedName: string,
13
+
feedConfig: FeedConfig
14
+
): Promise<void> {
15
+
const status = await db
16
+
.prepare("SELECT completed FROM feed_backfills WHERE actor = ? AND feed = ?")
17
+
.bind(actor, feedName)
18
+
.first<{ completed: number }>();
19
+
20
+
if (status?.completed) return;
21
+
22
+
// Ensure the user's follow records are backfilled first
23
+
await backfillUser(db, actor, feedConfig.follow, Date.now() + 15_000, config);
24
+
25
+
// Mark as in-progress (idempotent)
26
+
await db
27
+
.prepare(
28
+
"INSERT INTO feed_backfills (actor, feed, completed) VALUES (?, ?, 0) ON CONFLICT DO NOTHING"
29
+
)
30
+
.bind(actor, feedName)
31
+
.run();
32
+
33
+
const maxItems = feedConfig.maxItems ?? DEFAULT_FEED_MAX_ITEMS;
34
+
35
+
// Populate feed from existing records by followed users
36
+
for (const targetCol of feedConfig.targets) {
37
+
await db
38
+
.prepare(
39
+
`INSERT OR IGNORE INTO feed_items (actor, uri, collection, time_us)
40
+
SELECT ?, r.uri, r.collection, r.time_us
41
+
FROM records r
42
+
WHERE r.collection = ?
43
+
AND r.did IN (
44
+
SELECT json_extract(f.record, '$.subject')
45
+
FROM records f
46
+
WHERE f.collection = ? AND f.did = ?
47
+
)
48
+
ORDER BY r.time_us DESC
49
+
LIMIT ?`
50
+
)
51
+
.bind(actor, targetCol, feedConfig.follow, actor, maxItems)
52
+
.run();
53
+
}
54
+
55
+
// Prune oldest items beyond the cap
56
+
await db
57
+
.prepare(
58
+
`DELETE FROM feed_items WHERE actor = ? AND uri NOT IN (
59
+
SELECT uri FROM feed_items WHERE actor = ? ORDER BY time_us DESC LIMIT ?
60
+
)`
61
+
)
62
+
.bind(actor, actor, maxItems)
63
+
.run();
64
+
65
+
await db
66
+
.prepare("UPDATE feed_backfills SET completed = 1 WHERE actor = ? AND feed = ?")
67
+
.bind(actor, feedName)
68
+
.run();
69
+
}
70
+
71
+
export function registerFeedRoutes(
72
+
app: Hono,
73
+
db: Database,
74
+
config: ContrailConfig
75
+
): void {
76
+
if (!config.feeds) return;
77
+
78
+
const ns = config.namespace;
79
+
80
+
app.get(`/xrpc/${ns}.getFeed`, async (c) => {
81
+
const params = new URL(c.req.url).searchParams;
82
+
const feedName = params.get("feed");
83
+
const actor = params.get("actor");
84
+
85
+
if (!feedName || !actor) {
86
+
return c.json({ error: "feed and actor parameters required" }, 400);
87
+
}
88
+
89
+
const feedConfig = config.feeds![feedName];
90
+
if (!feedConfig) {
91
+
return c.json({ error: "Unknown feed" }, 404);
92
+
}
93
+
94
+
const did = await resolveActor(db, actor);
95
+
if (!did) return c.json({ error: "Could not resolve actor" }, 400);
96
+
97
+
await maybeBackfillFeed(db, config, did, feedName, feedConfig);
98
+
99
+
const collection = params.get("collection") || feedConfig.targets[0];
100
+
if (!feedConfig.targets.includes(collection)) {
101
+
return c.json({ error: "Collection not in feed targets" }, 400);
102
+
}
103
+
104
+
// Strip feed-specific params so runPipeline doesn't misinterpret them
105
+
// (e.g. "actor" in feeds means "whose feed", not "filter by record creator")
106
+
const pipelineParams = new URLSearchParams(params);
107
+
pipelineParams.delete("feed");
108
+
pipelineParams.delete("actor");
109
+
pipelineParams.delete("collection");
110
+
111
+
const source = {
112
+
joins: "JOIN feed_items f ON r.uri = f.uri",
113
+
conditions: ["f.actor = ?"],
114
+
params: [did],
115
+
};
116
+
117
+
try {
118
+
const result = await runPipeline(db, config, collection, pipelineParams, source);
119
+
return c.json(result);
120
+
} catch (e: any) {
121
+
if (e.message === "Could not resolve actor") {
122
+
return c.json({ error: e.message }, 400);
123
+
}
124
+
throw e;
125
+
}
126
+
});
127
+
}
+2
src/core/router/index.ts
+2
src/core/router/index.ts
···
3
3
import type { Database, ContrailConfig } from "../types";
4
4
import { registerAdminRoutes } from "./admin";
5
5
import { registerCollectionRoutes } from "./collection";
6
+
import { registerFeedRoutes } from "./feed";
6
7
import { registerNotifyRoute } from "./notify";
7
8
import { resolveActor } from "../identity";
8
9
import { resolveProfiles } from "./profiles";
···
44
45
45
46
registerAdminRoutes(app, db, config, adminSecret);
46
47
registerCollectionRoutes(app, db, config);
48
+
registerFeedRoutes(app, db, config);
47
49
registerNotifyRoute(app, db, config);
48
50
49
51
return app;
+2
-11
src/core/search.ts
+2
-11
src/core/search.ts
···
1
1
import type { CollectionConfig } from "./types";
2
2
import { getNestedValue } from "./types";
3
-
import { resolvedQueryable } from "./queryable.generated";
4
3
5
4
/**
6
5
* Resolve which fields are searchable for a collection.
···
10
9
collection: string,
11
10
colConfig: CollectionConfig
12
11
): string[] | null {
13
-
if (colConfig.searchable === false) return null;
14
-
if (Array.isArray(colConfig.searchable)) {
15
-
return colConfig.searchable.length > 0 ? colConfig.searchable : null;
16
-
}
17
-
// Auto-detect: all non-range queryable fields
18
-
const queryable = resolvedQueryable[collection] ?? colConfig.queryable ?? {};
19
-
const fields = Object.entries(queryable)
20
-
.filter(([, f]) => f.type !== "range")
21
-
.map(([name]) => name);
22
-
return fields.length > 0 ? fields : null;
12
+
if (!Array.isArray(colConfig.searchable)) return null;
13
+
return colConfig.searchable.length > 0 ? colConfig.searchable : null;
23
14
}
24
15
25
16
/** Sanitized FTS table name for a collection. */
+38
-1
src/core/types.ts
+38
-1
src/core/types.ts
···
37
37
config: ContrailConfig
38
38
) => Promise<Response>;
39
39
40
+
export interface RecordSource {
41
+
joins?: string;
42
+
conditions?: string[];
43
+
params?: (string | number)[];
44
+
}
45
+
46
+
export type PipelineQueryHandler = (
47
+
db: Database,
48
+
params: URLSearchParams,
49
+
config: ContrailConfig
50
+
) => Promise<RecordSource>;
51
+
52
+
export interface FeedConfig {
53
+
follow: string;
54
+
targets: string[];
55
+
/** Max feed items per user (default: 200). Oldest items are pruned after backfill. */
56
+
maxItems?: number;
57
+
}
58
+
59
+
export const DEFAULT_FEED_MAX_ITEMS = 200;
60
+
40
61
export interface CollectionConfig {
41
62
discover?: boolean;
42
63
queryable?: Record<string, QueryableField>;
···
44
65
/** Forward references: fields on this collection's records that point at another collection. */
45
66
references?: Record<string, ReferenceConfig>;
46
67
queries?: Record<string, CustomQueryHandler>;
47
-
/** FTS5 search fields. string[] = explicit fields, false = disabled, omitted = auto-detect non-range queryable fields */
68
+
pipelineQueries?: Record<string, PipelineQueryHandler>;
69
+
/** FTS5 search fields. Provide an array of field names to enable full-text search. Omit or set to false to disable. */
48
70
searchable?: string[] | false;
49
71
}
50
72
···
67
89
profiles?: string[];
68
90
relays?: string[];
69
91
jetstreams?: string[];
92
+
feeds?: Record<string, FeedConfig>;
70
93
}
71
94
72
95
/**
···
81
104
}
82
105
}
83
106
107
+
// Auto-add follow collections from feed configs as dependent collections
108
+
if (config.feeds) {
109
+
for (const feed of Object.values(config.feeds)) {
110
+
if (!collections[feed.follow]) {
111
+
collections[feed.follow] = { discover: false };
112
+
}
113
+
}
114
+
}
115
+
84
116
return {
85
117
...config,
86
118
collections,
···
88
120
jetstreams: config.jetstreams ?? DEFAULT_JETSTREAMS,
89
121
relays: config.relays ?? DEFAULT_RELAYS,
90
122
};
123
+
}
124
+
125
+
export function getFeedFollowCollections(config: ContrailConfig): string[] {
126
+
if (!config.feeds) return [];
127
+
return [...new Set(Object.values(config.feeds).map((f) => f.follow))];
91
128
}
92
129
93
130
// Record types
+167
-11
src/generate.ts
+167
-11
src/generate.ts
···
307
307
defs: { main: { type: "procedure", description: "Notify of a record change for immediate indexing. Fetches the record from the user's PDS and indexes (or deletes) it.", input: { encoding: "application/json", schema: { type: "object", properties: { uri: { type: "string", format: "at-uri", description: "Single AT URI to fetch and index" }, uris: { type: "array", items: { type: "string", format: "at-uri" }, maxLength: 25, description: "Batch of AT URIs to fetch and index (max 25)" } } } }, output: { encoding: "application/json", schema: { type: "object", required: ["indexed", "deleted"], properties: { indexed: { type: "integer", description: "Number of records created or updated" }, deleted: { type: "integer", description: "Number of records deleted (not found on PDS)" }, errors: { type: "array", items: { type: "string" }, description: "Errors for individual URIs that could not be processed" } } } } } },
308
308
});
309
309
310
+
// --- Feeds ---
311
+
312
+
if (config.feeds && Object.keys(config.feeds).length > 0) {
313
+
log("Generating feed endpoint...");
314
+
315
+
const feedNames = Object.keys(config.feeds);
316
+
const allTargets = [...new Set(Object.values(config.feeds).flatMap((f) => f.targets))];
317
+
318
+
// Merge queryable fields, relations, and references from all target collections
319
+
const feedParams: Record<string, any> = {
320
+
feed: { type: "string", knownValues: feedNames, description: "Feed name" },
321
+
actor: { type: "string", format: "at-identifier", description: "DID or handle of the requesting user" },
322
+
collection: { type: "string", knownValues: allTargets, description: "Filter by target collection (defaults to first target)" },
323
+
limit: { type: "integer", minimum: 1, maximum: 200, default: 50 },
324
+
cursor: { type: "string" },
325
+
profiles: { type: "boolean", description: "Include profile + identity info keyed by DID" },
326
+
};
327
+
328
+
const feedSortableValues: string[] = [];
329
+
const feedHydrateDefs: Record<string, any> = {};
330
+
const feedRefDefs: Record<string, any> = {};
331
+
const cap = (s: string) => s.charAt(0).toUpperCase() + s.slice(1);
332
+
333
+
for (const targetCol of allTargets) {
334
+
const targetConfig = config.collections[targetCol];
335
+
if (!targetConfig) continue;
336
+
337
+
const autoDetected = detectQueryableFields(targetCol);
338
+
const manual = targetConfig.queryable ?? {};
339
+
const merged = { ...autoDetected, ...manual };
340
+
341
+
// Search
342
+
if (Array.isArray(targetConfig.searchable) && targetConfig.searchable.length > 0 && !feedParams["search"]) {
343
+
feedParams["search"] = { type: "string", description: "Full-text search" };
344
+
}
345
+
346
+
// Queryable fields
347
+
for (const [field, fieldConfig] of Object.entries(merged)) {
348
+
const param = fieldToParam(field);
349
+
if (fieldConfig.type === "range") {
350
+
if (!feedParams[`${param}Min`]) {
351
+
feedParams[`${param}Min`] = { type: "string", description: `Minimum value for ${field}` };
352
+
feedParams[`${param}Max`] = { type: "string", description: `Maximum value for ${field}` };
353
+
}
354
+
} else {
355
+
if (!feedParams[param]) {
356
+
feedParams[param] = { type: "string", description: `Filter by ${field}` };
357
+
}
358
+
}
359
+
feedSortableValues.push(fieldToParam(field));
360
+
}
361
+
362
+
// Relations (counts + hydration)
363
+
for (const [relName, rel] of Object.entries(targetConfig.relations ?? {})) {
364
+
if (!feedParams[`${relName}CountMin`]) {
365
+
feedParams[`${relName}CountMin`] = { type: "integer", description: `Minimum total ${relName} count` };
366
+
feedSortableValues.push(`${relName}Count`);
367
+
}
368
+
if (!feedParams[`hydrate${cap(relName)}`]) {
369
+
feedParams[`hydrate${cap(relName)}`] = { type: "integer", minimum: 1, maximum: 50, description: `Number of ${relName} records to embed per record` };
370
+
}
371
+
372
+
if (rel.groupBy) {
373
+
const knownValues = getKnownValues(rel.collection, rel.groupBy);
374
+
for (const token of knownValues) {
375
+
const shortName = tokenShortName(token);
376
+
const paramName = `${relName}${cap(shortName)}CountMin`;
377
+
if (!feedParams[paramName]) {
378
+
feedParams[paramName] = { type: "integer", description: `Minimum ${relName} count where ${rel.groupBy} = ${shortName}` };
379
+
feedSortableValues.push(`${relName}${cap(shortName)}Count`);
380
+
}
381
+
}
382
+
}
383
+
}
384
+
385
+
// References (hydration params)
386
+
for (const [refName] of Object.entries(targetConfig.references ?? {})) {
387
+
if (!feedParams[`hydrate${cap(refName)}`]) {
388
+
feedParams[`hydrate${cap(refName)}`] = { type: "boolean", description: `Embed the referenced ${refName} record` };
389
+
}
390
+
}
391
+
}
392
+
393
+
// Sort/order params
394
+
const uniqueSortable = [...new Set(feedSortableValues)];
395
+
if (uniqueSortable.length > 0) {
396
+
feedParams["sort"] = { type: "string", knownValues: uniqueSortable, description: "Field to sort by (default: time_us)" };
397
+
feedParams["order"] = { type: "string", knownValues: ["asc", "desc"], description: "Sort direction" };
398
+
}
399
+
400
+
// Build a record def per target collection for the union
401
+
const feedRecordDefs: Record<string, any> = {};
402
+
const feedRecordRefs: string[] = [];
403
+
404
+
for (const targetCol of allTargets) {
405
+
const targetConfig = config.collections[targetCol];
406
+
if (!targetConfig) continue;
407
+
408
+
const collectionRef = getCollectionLexiconRef(targetCol);
409
+
410
+
const countFields: CountField[] = [];
411
+
const relationDefs: RelationDef[] = [];
412
+
const referenceDefs: ReferenceDef[] = [];
413
+
414
+
for (const [relName, rel] of Object.entries(targetConfig.relations ?? {})) {
415
+
countFields.push({ name: `${relName}Count`, description: `Total ${relName} count` });
416
+
const groupMapping: Record<string, string> = {};
417
+
if (rel.groupBy) {
418
+
for (const token of getKnownValues(rel.collection, rel.groupBy)) {
419
+
const shortName = tokenShortName(token);
420
+
groupMapping[shortName] = token;
421
+
countFields.push({ name: `${relName}${cap(shortName)}Count`, description: `${relName} count where ${rel.groupBy} = ${shortName}` });
422
+
}
423
+
}
424
+
relationDefs.push({ relName, collection: rel.collection, groupBy: rel.groupBy, groups: groupMapping });
425
+
}
426
+
427
+
for (const [refName, ref] of Object.entries(targetConfig.references ?? {})) {
428
+
referenceDefs.push({ refName, collection: ref.collection });
429
+
}
430
+
431
+
const defName = `feedRecord_${targetCol.replace(/[^a-zA-Z0-9]/g, "_")}`;
432
+
feedRecordDefs[defName] = buildRecordDef(collectionRef, countFields, relationDefs, referenceDefs);
433
+
feedRecordRefs.push(`#${defName}`);
434
+
435
+
// Add hydrate + reference defs (includes grouped wrappers)
436
+
Object.assign(feedHydrateDefs, buildHydrateDefs(relationDefs));
437
+
Object.assign(feedRefDefs, buildReferenceDefs(referenceDefs));
438
+
}
439
+
440
+
const recordsItems = feedRecordRefs.length === 1
441
+
? { type: "ref", ref: feedRecordRefs[0] }
442
+
: { type: "union", refs: feedRecordRefs };
443
+
444
+
writeLexicon(`${ns}.getFeed`, {
445
+
lexicon: 1, id: `${ns}.getFeed`,
446
+
defs: {
447
+
main: {
448
+
type: "query",
449
+
description: "Get a personalized feed based on followed users' activity",
450
+
parameters: { type: "params", required: ["feed", "actor"], properties: feedParams },
451
+
output: {
452
+
encoding: "application/json",
453
+
schema: {
454
+
type: "object",
455
+
required: ["records"],
456
+
properties: {
457
+
records: { type: "array", items: recordsItems },
458
+
cursor: { type: "string" },
459
+
profiles: { type: "array", items: { type: "ref", ref: "#profileEntry" } },
460
+
},
461
+
},
462
+
},
463
+
},
464
+
...feedRecordDefs,
465
+
...feedHydrateDefs,
466
+
...feedRefDefs,
467
+
...profileDefs(),
468
+
},
469
+
});
470
+
}
471
+
310
472
// --- Per-collection ---
311
473
312
474
log("Generating collection endpoints...");
···
331
493
};
332
494
333
495
// Search param
334
-
if (colConfig.searchable !== false) {
335
-
const allQueryable = { ...autoDetected, ...manual };
336
-
const searchableFields = Array.isArray(colConfig.searchable)
337
-
? colConfig.searchable
338
-
: Object.entries(allQueryable).filter(([, f]) => f.type !== "range").map(([name]) => name);
339
-
if (searchableFields.length > 0) {
340
-
listParams["search"] = {
341
-
type: "string",
342
-
description: `Full-text search across: ${searchableFields.join(", ")}`,
343
-
};
344
-
}
496
+
if (Array.isArray(colConfig.searchable) && colConfig.searchable.length > 0) {
497
+
listParams["search"] = {
498
+
type: "string",
499
+
description: `Full-text search across: ${colConfig.searchable.join(", ")}`,
500
+
};
345
501
}
346
502
347
503
for (const [field, fieldConfig] of Object.entries(merged)) {