[READ-ONLY] Mirror of https://github.com/flo-bit/contrail. atproto backend in a bottle
flo-bit.dev/contrail/
improve counts
+96
-45
+1
src/config.ts
+1
src/config.ts
+3
-6
tests/generate.test.ts
+3
-6
tests/generate.test.ts
···
253
253
});
254
254
});
255
255
256
-
describe("search: auto-detect", () => {
257
-
it("includes search param with non-range fields only", () => {
256
+
describe("search: no searchable field configured", () => {
257
+
it("does not include search param when searchable is omitted", () => {
258
258
const lexicons = generate(SEARCH_AUTO_CONFIG);
259
259
const params = getParams(lexicons["com.example.post.listRecords"]);
260
-
expect(params.search).toBeDefined();
261
-
expect(params.search.description).toContain("title");
262
-
expect(params.search.description).toContain("body");
263
-
expect(params.search.description).not.toContain("score");
260
+
expect(params.search).toBeUndefined();
264
261
});
265
262
});
+2
-2
tests/search.test.ts
+2
-2
tests/search.test.ts
···
14
14
description: {},
15
15
startsAt: { type: "range" },
16
16
},
17
-
// searchable omitted → auto-detect non-range: mode, name, description
17
+
searchable: ["mode", "name", "description"],
18
18
},
19
19
"test.explicit.collection": {
20
20
queryable: {
···
40
40
await initSchema(db, SEARCH_CONFIG);
41
41
});
42
42
43
-
describe("FTS auto-detect (searchable omitted)", () => {
43
+
describe("FTS with explicit searchable fields", () => {
44
44
const collection = "community.lexicon.calendar.event";
45
45
46
46
beforeEach(async () => {
+2
src/core/types.ts
+2
src/core/types.ts
+85
-35
src/core/db/records.ts
+85
-35
src/core/db/records.ts
···
13
13
14
14
// --- Counts ---
15
15
16
+
interface InboundRelation {
17
+
parentCollection: string;
18
+
relationName: string;
19
+
rel: RelationConfig;
20
+
}
21
+
16
22
function getInboundRelations(
17
23
config: ContrailConfig,
18
24
foreignCollection: string
19
-
): RelationConfig[] {
20
-
const results: RelationConfig[] = [];
21
-
for (const [, colConfig] of Object.entries(config.collections)) {
22
-
for (const [, rel] of Object.entries(colConfig.relations ?? {})) {
25
+
): InboundRelation[] {
26
+
const results: InboundRelation[] = [];
27
+
for (const [colName, colConfig] of Object.entries(config.collections)) {
28
+
for (const [relName, rel] of Object.entries(colConfig.relations ?? {})) {
23
29
if (rel.collection === foreignCollection) {
24
-
results.push(rel);
30
+
results.push({ parentCollection: colName, relationName: relName, rel });
25
31
}
26
32
}
27
33
}
28
34
return results;
29
35
}
30
36
37
+
/**
38
+
* Build statements that fully recount child records for affected parents.
39
+
* Instead of +1/-1, we SELECT COUNT(*) so the count is always accurate.
40
+
* This runs for create, update, and delete operations.
41
+
*/
31
42
function buildCountStatements(
32
43
db: Database,
33
44
event: IngestEvent,
34
-
config: ContrailConfig
45
+
config: ContrailConfig,
46
+
existingRecordJson: string | null,
35
47
): Statement[] {
36
-
if (event.operation !== "create" && event.operation !== "delete") return [];
37
-
38
48
const inbound = getInboundRelations(config, event.collection);
39
49
if (inbound.length === 0) return [];
40
50
41
51
const record = event.record ? JSON.parse(event.record) : null;
42
-
if (!record && event.operation === "create") return [];
52
+
const existingRecord = existingRecordJson ? JSON.parse(existingRecordJson) : null;
43
53
44
-
const isCreate = event.operation === "create";
45
54
const statements: Statement[] = [];
55
+
const recountedTargets = new Set<string>();
46
56
47
-
for (const rel of inbound) {
48
-
const targetUri = getNestedValue(record, getRelationField(rel));
49
-
if (!targetUri) continue;
57
+
for (const { parentCollection, relationName, rel } of inbound) {
58
+
if (rel.count === false) continue;
50
59
51
-
const columns = [countColumnName(rel.collection)];
52
-
if (rel.groupBy) {
53
-
const groupValue = getNestedValue(record, rel.groupBy);
54
-
if (groupValue != null) columns.push(countColumnName(String(groupValue)));
60
+
const field = getRelationField(rel);
61
+
const matchColumn = rel.match === "did" ? "did" : "uri";
62
+
const childCollection = rel.collection;
63
+
64
+
// Collect target URIs/DIDs that need recounting (current + old if changed)
65
+
const targets: string[] = [];
66
+
if (record) {
67
+
const t = getNestedValue(record, field);
68
+
if (t) targets.push(t);
69
+
}
70
+
if (existingRecord) {
71
+
const t = getNestedValue(existingRecord, field);
72
+
if (t && !targets.includes(t)) targets.push(t);
55
73
}
56
74
57
-
for (const col of columns) {
58
-
statements.push(
59
-
db
60
-
.prepare(
61
-
isCreate
62
-
? `UPDATE records SET ${col} = ${col} + 1 WHERE uri = ?`
63
-
: `UPDATE records SET ${col} = MAX(${col} - 1, 0) WHERE uri = ?`
64
-
)
65
-
.bind(targetUri)
75
+
for (const targetValue of targets) {
76
+
const key = `${parentCollection}:${relationName}:${targetValue}`;
77
+
if (recountedTargets.has(key)) continue;
78
+
recountedTargets.add(key);
79
+
80
+
const setClauses: string[] = [];
81
+
const setBindings: (string | number)[] = [];
82
+
83
+
// Total count
84
+
const totalCol = countColumnName(rel.collection);
85
+
setClauses.push(
86
+
`${totalCol} = (SELECT COUNT(*) FROM records WHERE collection = ? AND json_extract(record, '$.${field}') = ?)`
66
87
);
88
+
setBindings.push(childCollection, targetValue);
89
+
90
+
// Grouped counts
91
+
if (rel.groupBy) {
92
+
const mapping = (resolvedRelationsMap as Record<string, any>)[parentCollection]?.[relationName];
93
+
if (mapping?.groups) {
94
+
for (const [groupValue, fullToken] of Object.entries(mapping.groups as Record<string, string>)) {
95
+
const groupCol = countColumnName(fullToken);
96
+
setClauses.push(
97
+
`${groupCol} = (SELECT COUNT(*) FROM records WHERE collection = ? AND json_extract(record, '$.${field}') = ? AND json_extract(record, '$.${rel.groupBy}') = ?)`
98
+
);
99
+
setBindings.push(childCollection, targetValue, groupValue);
100
+
}
101
+
}
102
+
}
103
+
104
+
if (setClauses.length > 0) {
105
+
statements.push(
106
+
db
107
+
.prepare(
108
+
`UPDATE records SET ${setClauses.join(", ")} WHERE ${matchColumn} = ?`
109
+
)
110
+
.bind(...setBindings, targetValue)
111
+
);
112
+
}
67
113
}
68
114
}
69
115
···
248
294
const existingCids = new Map<string, string | null>();
249
295
const existingRecords = new Map<string, string | null>();
250
296
const followCollections = config ? getFeedFollowCollections(config) : [];
251
-
const needRecordContent = followCollections.length > 0;
297
+
const hasCountingRelations = config ? Object.values(config.collections).some(c =>
298
+
Object.values(c.relations ?? {}).some(r => r.count !== false)
299
+
) : false;
300
+
const needRecordContent = followCollections.length > 0 || hasCountingRelations;
252
301
253
302
if (config && !options?.skipReplayDetection) {
254
303
const uris = events.map((e) => e.uri);
···
295
344
}
296
345
297
346
if (config) {
298
-
// Skip count updates for replayed events:
299
-
// - create/update where the record already exists with the same CID
300
-
// - delete where the record doesn't exist
347
+
// Recount is idempotent — always run it for create/update/delete.
348
+
// Pass existing record so deletes and updates that change the target can recount the old parent.
349
+
const existingRecordJson = existingRecords.get(e.uri) ?? null;
350
+
batch.push(...buildCountStatements(db, e, config, existingRecordJson));
351
+
352
+
// Feed fanout still needs replay detection
301
353
const existing = existingCids.get(e.uri);
302
354
const isReplay =
303
355
e.operation === "delete"
304
356
? existing === undefined
305
357
: existing === e.cid;
306
358
307
-
if (!isReplay) {
308
-
batch.push(...buildCountStatements(db, e, config));
309
-
if (!options?.skipFeedFanout) {
310
-
batch.push(...buildFeedStatements(db, e, config, existingRecords));
311
-
}
359
+
if (!isReplay && !options?.skipFeedFanout) {
360
+
batch.push(...buildFeedStatements(db, e, config, existingRecords));
312
361
}
313
362
batch.push(...buildFtsStatements(db, e, config));
314
363
}
···
326
375
const relMap = resolvedRelationsMap[collection] ?? {};
327
376
328
377
for (const [relName, rel] of Object.entries(colConfig.relations)) {
378
+
if (rel.count === false) continue;
329
379
columns.push({ type: rel.collection, column: countColumnName(rel.collection) });
330
380
const mapping = relMap[relName];
331
381
if (mapping) {
+1
src/core/db/schema.ts
+1
src/core/db/schema.ts
···
78
78
for (const [collection, colConfig] of Object.entries(config.collections)) {
79
79
const relMap = resolvedRelationsMap[collection] ?? {};
80
80
for (const [relName, rel] of Object.entries(colConfig.relations ?? {})) {
81
+
if (rel.count === false) continue;
81
82
// Total count column
82
83
const totalCol = countColumnName(rel.collection);
83
84
if (!addedColumns.has(totalCol)) {
+1
src/core/router/collection.ts
+1
src/core/router/collection.ts
···
285
285
const counts: Record<string, number> = {};
286
286
287
287
for (const [relName, rel] of Object.entries(relations)) {
288
+
if (rel.count === false) continue;
288
289
const totalCol = countColumnName(rel.collection);
289
290
const val = row[totalCol];
290
291
if (val != null && val !== 0) counts[rel.collection] = val;
+1
-2
src/core/router/notify.ts
+1
-2
src/core/router/notify.ts
···
105
105
did: parsed.did,
106
106
collection: parsed.collection,
107
107
rkey: parsed.rkey,
108
-
// "update" skips count statements, "create" increments them.
109
-
// Only use "create" if the record is genuinely new.
108
+
// Both "update" and "create" trigger a full recount of related records.
110
109
operation: existing ? "update" : "create",
111
110
cid: result.cid,
112
111
record: JSON.stringify(result.value),