[READ-ONLY] Mirror of https://github.com/flo-bit/contrail. atproto backend in a bottle
flo-bit.dev/contrail/
fix(search): make FTS sync idempotent to stop duplicate rows on backfill
buildFtsStatements only deleted the existing FTS row before inserting when the record was already in existingMap. Backfill runs with skipReplayDetection, leaving existingMap empty, so re-applied records looked new and appended duplicate FTS rows. The FTS5 virtual table has no uniqueness constraint, so duplicates accumulated and the search JOIN fanned each event out into one row per duplicate, breaking keyed lists in the appview. Delete-then-insert is now unconditional.
Tom Scanlan
(Jun 17, 2026, 6:47 AM EDT)
32ace91a
8edf4100
+40
-7
+13
.changeset/fts-no-duplicate-rows-on-backfill.md
+13
.changeset/fts-no-duplicate-rows-on-backfill.md
···
1
+
---
2
+
"@atmo-dev/contrail-appview": patch
3
+
---
4
+
5
+
Stop accumulating duplicate FTS rows when records are re-applied during backfill.
6
+
7
+
The FTS-sync path only deleted an existing FTS row before inserting when the
8
+
record was already in `existingMap`. Backfill runs with `skipReplayDetection`,
9
+
which leaves `existingMap` empty, so every re-applied record looked brand-new
10
+
and appended another FTS row. The FTS virtual table has no uniqueness
11
+
constraint, so these accumulated, and the search JOIN fanned each event out into
12
+
one result row per duplicate. Make the delete-then-insert unconditional so FTS
13
+
sync is idempotent regardless of replay detection.
+8
-7
packages/contrail-appview/src/core/db/records.ts
+8
-7
packages/contrail-appview/src/core/db/records.ts
···
161
161
function buildFtsStatements(
162
162
db: Database,
163
163
event: IngestEvent,
164
-
config: ContrailConfig,
165
-
existingMap: Map<string, ExistingRecordInfo>
164
+
config: ContrailConfig
166
165
): Statement[] {
167
166
// PostgreSQL: tsvector generated column is auto-maintained, no manual FTS sync
168
167
if (getDialect(db).ftsStrategy === "generated-column") return [];
···
187
186
const content = buildFtsContent(record, fields);
188
187
if (!content) return [];
189
188
190
-
// Only delete existing FTS row if this is an update (record already existed)
191
-
if (existingMap.has(event.uri)) {
192
-
stmts.push(db.prepare(`DELETE FROM ${table} WHERE uri = ?`).bind(event.uri));
193
-
}
189
+
// Always delete-then-insert so FTS sync is idempotent. The FTS virtual table
190
+
// has no uniqueness constraint, so a bare insert appends a duplicate row when
191
+
// one already exists. existingMap is unreliable here: backfill runs with
192
+
// skipReplayDetection, leaving it empty, so a re-applied record would look new
193
+
// and accumulate duplicate rows that fan out the search JOIN.
194
+
stmts.push(db.prepare(`DELETE FROM ${table} WHERE uri = ?`).bind(event.uri));
194
195
stmts.push(
195
196
db.prepare(`INSERT INTO ${table} (uri, content) VALUES (?, ?)`).bind(event.uri, content)
196
197
);
···
644
645
if (!isReplay && !options?.skipFeedFanout) {
645
646
batch.push(...buildFeedStatements(db, e, config, existingRecordStrings));
646
647
}
647
-
batch.push(...buildFtsStatements(db, e, config, existingMap));
648
+
batch.push(...buildFtsStatements(db, e, config));
648
649
}
649
650
}
650
651
+19
packages/contrail/tests/search.test.ts
+19
packages/contrail/tests/search.test.ts
···
159
159
], SEARCH_CONFIG);
160
160
expect((await queryRecords(db, SEARCH_CONFIG, { collection, search: "Deletable" })).records).toHaveLength(0);
161
161
});
162
+
163
+
it("does not duplicate FTS rows when the same record is re-applied during backfill", async () => {
164
+
// Backfill passes call applyEvents with skipReplayDetection: true, which leaves
165
+
// existingMap empty so every record looks brand-new. Re-backfilling the same
166
+
// record must not append a second FTS row; otherwise the search JOIN fans out
167
+
// and returns the event more than once (which crashes keyed lists downstream).
168
+
const event = makeEvent({
169
+
uri: "at://did:plc:a/community.lexicon.calendar.event/1",
170
+
collection,
171
+
rkey: "1",
172
+
record: { name: "Backfilled Meetup", mode: "online", description: "test" },
173
+
time_us: 1000,
174
+
});
175
+
await applyEvents(db, [event], SEARCH_CONFIG, { skipReplayDetection: true });
176
+
await applyEvents(db, [event], SEARCH_CONFIG, { skipReplayDetection: true });
177
+
178
+
const result = await queryRecords(db, SEARCH_CONFIG, { collection, search: "Meetup" });
179
+
expect(result.records).toHaveLength(1);
180
+
});
162
181
});
163
182
164
183
describe.skipIf(!hasFts)("explicit searchable fields", () => {