atproto pds in zig
pds.zat.dev
pds
atproto
Use lazy MST loading for repo writes
+132
-10
+22
bench/README.md
+22
bench/README.md
···
392
392
blockstore, eventlog, full repo export behavior, an official-PDS benchmark
393
393
harness, and an explicit ZDS index-only list probe that matches Tranquil's
394
394
`list_records`.
395
+
396
+
## June 18, 2026 write-path check
397
+
398
+
ZDS now uses `zat.mst.Mst.loadLazy` for repo authoring, backed by SQLite
399
+
`repo_blocks`, instead of reading every repo block into an in-memory CAR before
400
+
each write. This completes the earlier `zat` lazy-MST adoption for the hot write
401
+
path.
402
+
403
+
Fresh local 10-caller metastore-shaped rows. The ZDS write row was repeated
404
+
three times after the change and stayed in the same band: 2312.8-2525.2 ops/s,
405
+
p95 11.2-12.2 ms.
406
+
407
+
| operation | zds | tranquil |
408
+
|---|---:|---:|
409
+
| apply one record commit | 2312.8-2525.2 ops/s, p95 11.2-12.2 ms | 503 ops/s, p95 26.1 ms |
410
+
| current record CID lookup | 319.7k ops/s, p95 8 us | 294.6k ops/s, p95 54 us |
411
+
| list records, limit 50 | 21.5k ops/s, p95 53 us | 35.8k ops/s, p95 382 us |
412
+
413
+
Two `just bench write-profile 10 1000` runs after the lazy-MST change reported
414
+
2527.1-2698.4 ops/s, p95 9.3-9.8 ms. Average write time was split roughly
415
+
across lock wait 23-24%, lazy repo loading 9-10%, staging 10%, commit build 7%,
416
+
and SQLite/event persistence 50%.
+12
-7
docs/benchmarking.md
+12
-7
docs/benchmarking.md
···
31
31
10/100 caller concurrency curves. Official-PDS batch probes stay outside the
32
32
main matrix until they are measured through the same unit of work.
33
33
34
-
Current local Tranquil comparison, measured May 31, 2026:
34
+
Current local comparison. The Tranquil rows are from a fresh local metastore
35
+
run on June 18, 2026. The ZDS write row was remeasured after completing the
36
+
lazy-MST adoption in the repo write path.
35
37
36
38
| operation | callers | zds | tranquil | note |
37
39
|---|---:|---:|---:|---|
38
-
| apply one record commit | 1 | 281 ops/s, p95 8.1 ms | 289 ops/s, p95 4.9 ms | close throughput; Tranquil tighter tail |
39
-
| apply one record commit | 10 | 289 ops/s, p95 86.1 ms | 744 ops/s, p95 15.9 ms | Tranquil clearly ahead |
40
-
| apply one record commit | 100 | 1423 ops/s, p95 65.0 ms, p99 806 ms | 1051 ops/s, p95 100 ms | ZDS higher throughput; ugly ZDS tail |
41
-
| current CID lookup | 1 | 509k ops/s, p95 2 us | 111k ops/s, p95 12.6 us | ZDS ahead |
42
-
| current CID lookup | 10 | 345k ops/s, p95 18 us | 610k ops/s, p95 27 us | Tranquil ahead |
43
-
| current CID lookup | 100 | 302k ops/s, p95 273 us | 630k ops/s, p95 212 us | Tranquil ahead |
40
+
| apply one record commit | 1 | 1598 ops/s, p95 0.65 ms | 267 ops/s, p95 4.13 ms | ZDS no longer eagerly loads the whole repo CAR |
41
+
| apply one record commit | 10 | 2313-2525 ops/s, p95 11.2-12.2 ms | 503 ops/s, p95 26.1 ms | ZDS row is repeated-run range |
42
+
| current CID lookup | 1 | 523k ops/s, p95 2 us | 116k ops/s, p95 12.6 us | ZDS ahead |
43
+
| current CID lookup | 10 | 319.7k ops/s, p95 8 us | 294.6k ops/s, p95 54 us | both use record-index lookup |
44
44
45
45
Tranquil's built-in 1000-caller metastore row currently trips handler
46
46
backpressure in the benchmark harness. Keep it out of the table until the
47
47
harness handles backpressure as a measured result.
48
+
49
+
ZDS write authoring uses `zat.mst.Mst.loadLazy` with a SQLite-backed
50
+
`repo_blocks` reader. This avoids reading every stored repo block into a CAR
51
+
before each write while preserving the signed commit, MST update, record index,
52
+
and sync event behavior.
48
53
49
54
## read path
50
55
+98
-3
src/storage/store.zig
+98
-3
src/storage/store.zig
···
393
393
uri: []const u8,
394
394
};
395
395
396
+
const RepoBlockReader = struct {
397
+
allocator: std.mem.Allocator,
398
+
did: []const u8,
399
+
400
+
fn reader(self: *RepoBlockReader) zat.mst.BlockReader {
401
+
return .{
402
+
.ctx = self,
403
+
.getFn = getBlock,
404
+
};
405
+
}
406
+
407
+
fn getBlock(ctx: *anyopaque, cid_raw: []const u8) anyerror!?[]const u8 {
408
+
const self: *RepoBlockReader = @ptrCast(@alignCast(ctx));
409
+
const cid = try cidText(self.allocator, cid_raw);
410
+
411
+
db_mutex.lockUncancelable(store_io);
412
+
defer db_mutex.unlock(store_io);
413
+
try requireInitialized();
414
+
415
+
const row = try conn.row(
416
+
\\SELECT data
417
+
\\FROM repo_blocks
418
+
\\WHERE did = ? AND cid = ?
419
+
\\LIMIT 1
420
+
, .{ self.did, cid });
421
+
if (row == null) return null;
422
+
defer row.?.deinit();
423
+
424
+
const data = row.?.nullableBlob(0) orelse return null;
425
+
return try self.allocator.dupe(u8, data);
426
+
}
427
+
};
428
+
396
429
var conn: zqlite.Conn = undefined;
397
430
var initialized = false;
398
431
var store_io: Io = undefined;
···
1757
1790
break :blk .{
1758
1791
.current = current,
1759
1792
.rev = try revForSeq(allocator, seq, if (current) |root| root.rev else null),
1760
-
.repo_car = try readRepoCarLocked(allocator, account.did),
1761
1793
.keypair = try signingKeypairLocked(account.did),
1762
1794
.ops = resolved_ops,
1763
1795
.prev_record_cids = try previousRecordCidsLocked(allocator, account.did, resolved_ops),
···
1765
1797
};
1766
1798
const current = loaded.current;
1767
1799
const rev = loaded.rev;
1768
-
const repo_car = loaded.repo_car;
1769
1800
const keypair = loaded.keypair;
1770
1801
const resolved_ops = loaded.ops;
1771
1802
const prev_record_cids = loaded.prev_record_cids;
1772
1803
1804
+
var repo_block_reader = RepoBlockReader{
1805
+
.allocator = allocator,
1806
+
.did = account.did,
1807
+
};
1773
1808
var tree = if (current) |root|
1774
-
try zat.mst.Mst.loadFromBlocks(allocator, repo_car, root.data_cid_raw)
1809
+
try zat.mst.Mst.loadLazy(allocator, root.data_cid_raw, repo_block_reader.reader())
1775
1810
else
1776
1811
zat.mst.Mst.init(allocator);
1777
1812
addProfile(profile, .load_repo_ns, elapsedNs(load_start));
···
5556
5591
.value = custom.value,
5557
5592
} }}, .{ .validate = .skip });
5558
5593
try std.testing.expectEqualStrings("unknown", custom_result.records[0].validation_status);
5594
+
}
5595
+
5596
+
test "repo writes lazily load existing MST blocks for mutation" {
5597
+
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
5598
+
defer arena.deinit();
5599
+
const allocator = arena.allocator();
5600
+
5601
+
try init(std.Options.debug_io, ":memory:");
5602
+
defer close();
5603
+
5604
+
const account = try createAccount(
5605
+
allocator,
5606
+
"lazy.test",
5607
+
"lazy@test.com",
5608
+
"password",
5609
+
"did:plc:lazywritecheck",
5610
+
true,
5611
+
);
5612
+
5613
+
var create_ops: [160]WriteOp = undefined;
5614
+
var create_parsed: [160]std.json.Parsed(std.json.Value) = undefined;
5615
+
for (&create_ops, &create_parsed, 0..) |*op, *parsed, i| {
5616
+
parsed.* = try std.json.parseFromSlice(
5617
+
std.json.Value,
5618
+
allocator,
5619
+
try std.fmt.allocPrint(allocator, "{{\"$type\":\"app.bsky.feed.post\",\"text\":\"record {d}\"}}", .{i}),
5620
+
.{},
5621
+
);
5622
+
op.* = .{ .create = .{
5623
+
.collection = "app.bsky.feed.post",
5624
+
.rkey = try std.fmt.allocPrint(allocator, "3zlazy{d:0>4}", .{i}),
5625
+
.value = parsed.value,
5626
+
} };
5627
+
}
5628
+
defer for (&create_parsed) |*parsed| parsed.deinit();
5629
+
5630
+
_ = try applyWritesWithOptions(allocator, account, &create_ops, .{});
5631
+
5632
+
const update = try std.json.parseFromSlice(std.json.Value, allocator, "{\"$type\":\"app.bsky.feed.post\",\"text\":\"updated\"}", .{});
5633
+
defer update.deinit();
5634
+
const update_result = try applyWritesWithOptions(allocator, account, &.{.{ .update = .{
5635
+
.collection = "app.bsky.feed.post",
5636
+
.rkey = "3zlazy0077",
5637
+
.value = update.value,
5638
+
} }}, .{});
5639
+
5640
+
_ = try applyWritesWithOptions(allocator, account, &.{.{ .delete = .{
5641
+
.collection = "app.bsky.feed.post",
5642
+
.rkey = "3zlazy0012",
5643
+
} }}, .{});
5644
+
5645
+
const repo_car = try writeRepoCar(allocator, account.did);
5646
+
const loaded = try zat.loadCommitFromCAR(allocator, repo_car);
5647
+
var tree = try zat.mst.Mst.loadFromBlocks(allocator, loaded.repo_car, loaded.commit.data_cid);
5648
+
5649
+
const updated = tree.get("app.bsky.feed.post/3zlazy0077") orelse return error.MissingRecord;
5650
+
try std.testing.expectEqualStrings(update_result.records[0].cid, try cidText(allocator, updated.raw));
5651
+
try std.testing.expect(tree.get("app.bsky.feed.post/3zlazy0012") == null);
5652
+
const retained = tree.get("app.bsky.feed.post/3zlazy0159") orelse return error.MissingRecord;
5653
+
try std.testing.expectEqualStrings((get(account.did, "app.bsky.feed.post", "3zlazy0159") orelse return error.MissingRecord).cid, try cidText(allocator, retained.raw));
5559
5654
}
5560
5655
5561
5656
test "getRepo since filters repo blocks by revision" {