Merge pull request #2 from tompscanlan/feature/postgres-adapter · flo-bit.dev/contrail@d973eed

+14

Dockerfile

··· 1 + FROM node:22-alpine 2 + 3 + RUN corepack enable && corepack prepare pnpm@latest --activate 4 + 5 + WORKDIR /app 6 + 7 + COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ 8 + RUN pnpm install --frozen-lockfile 9 + 10 + COPY src/ src/ 11 + COPY app/ app/ 12 + COPY bin/ bin/ 13 + 14 + CMD ["npx", "tsx", "bin/ingest.ts"]

+37 -4

README.md

··· 79 79 await contrail.ingest(); 80 80 ``` 81 81 82 + ### Persistent ingestion 83 + 84 + ```ts 85 + // Long-lived Jetstream connection with automatic batching and reconnection 86 + const controller = new AbortController(); 87 + await contrail.runPersistent({ 88 + batchSize: 50, // flush every N events (default: 50) 89 + flushIntervalMs: 5000, // or every N ms (default: 5000) 90 + signal: controller.signal, 91 + }); 92 + ``` 93 + 94 + Call `controller.abort()` for graceful shutdown — the current batch is flushed and the cursor is saved. 95 + 82 96 ### Discover users and backfill 83 97 84 98 ```ts ··· 140 154 const contrail = new Contrail({ ...config, db }); 141 155 ``` 142 156 143 - > **Note:** The SQLite adapter uses Node's built-in `node:sqlite` (Node 22+). Full-text search (`searchable`) is not supported with this adapter because `node:sqlite` doesn't include the FTS5 extension. Search works on Cloudflare D1, which has FTS5 enabled. 157 + > **Note:** The SQLite adapter uses Node's built-in `node:sqlite` (Node 22+). Full-text search (`searchable`) is not supported with this adapter because `node:sqlite` doesn't include the FTS5 extension. Search works on Cloudflare D1 and PostgreSQL. 144 158 145 - ## Running the example (Cloudflare Workers) 159 + ### PostgreSQL adapter (Node.js / server) 160 + 161 + ```ts 162 + import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; 163 + import pg from "pg"; 164 + 165 + const pool = new pg.Pool({ connectionString: process.env.DATABASE_URL }); 166 + const db = createPostgresDatabase(pool); 167 + const contrail = new Contrail({ ...config, db }); 168 + ``` 169 + 170 + PostgreSQL uses JSONB for record storage, tsvector generated columns for full-text search (instead of FTS5), and `BIGINT` for timestamp columns. 171 + 172 + ## Examples 173 + 174 + ### PostgreSQL (Node.js) 175 + 176 + See [`examples/postgres/`](examples/postgres/) for a complete example with Docker Compose, persistent Jetstream ingestion, user discovery/backfill, and an HTTP API server. 177 + 178 + ### Cloudflare Workers 146 179 147 180 This repo includes a working example that indexes AT Protocol calendar events and RSVPs on Cloudflare Workers + D1. 148 181 ··· 190 223 | `references.*.field` | — | Field containing the target record's AT URI | 191 224 | `queries` | `{}` | Custom query handlers (raw Response) | 192 225 | `pipelineQueries` | `{}` | Custom query handlers that go through the standard filter/sort/hydration pipeline | 193 - | `searchable` | disabled | FTS5 search fields. Provide `string[]` to enable, omit to disable | 226 + | `searchable` | disabled | Full-text search fields. SQLite uses FTS5 virtual tables; PostgreSQL uses tsvector generated columns with GIN indexes. Provide `string[]` to enable, omit to disable | 194 227 195 228 ### Top-level options 196 229 ··· 251 284 ?sort=rsvpsGoingCount&order=asc # by going count ascending 252 285 ``` 253 286 254 - **Search** uses SQLite FTS5 for ranked full-text search. To enable, set `searchable: ["field1", "field2"]` on a collection. Supports FTS5 syntax including prefix (`meetup*`), phrases (`"rust meetup"`), and boolean (`rust OR typescript`). Combinable with all other filters. 287 + **Search** uses SQLite FTS5 or PostgreSQL tsvector for ranked full-text search. To enable, set `searchable: ["field1", "field2"]` on a collection. Supports FTS5 syntax including prefix (`meetup*`), phrases (`"rust meetup"`), and boolean (`rust OR typescript`). Combinable with all other filters. 255 288 256 289 ``` 257 290 ?search=meetup # basic search

+16 -2

package.json

··· 20 20 "types": "./dist/adapters/sqlite.d.ts", 21 21 "import": "./dist/adapters/sqlite.js" 22 22 }, 23 + "./postgres": { 24 + "types": "./dist/adapters/postgres.d.ts", 25 + "import": "./dist/adapters/postgres.js" 26 + }, 23 27 "./generate": { 24 28 "types": "./dist/generate.d.ts", 25 29 "import": "./dist/generate.js" ··· 63 67 "devDependencies": { 64 68 "@atcute/lex-cli": "^2.5.3", 65 69 "@atcute/lexicon-doc": "^2.1.2", 70 + "@changesets/cli": "^2.29.4", 66 71 "@cloudflare/workers-types": "^4.20250124.0", 67 72 "@types/node": "^25.5.0", 68 - "@changesets/cli": "^2.29.4", 73 + "@types/pg": "^8.20.0", 74 + "pg": "^8.20.0", 69 75 "tsup": "^8.5.0", 70 76 "tsx": "^4.21.0", 71 77 "typescript": "^5.7.3", 72 78 "vitest": "^4.1.0", 73 79 "wrangler": "^4.63.0" 74 80 }, 81 + "peerDependencies": { 82 + "pg": "^8.0.0" 83 + }, 84 + "peerDependenciesMeta": { 85 + "pg": { 86 + "optional": true 87 + } 88 + }, 75 89 "license": "MIT" 76 - } 90 + }

+119 -304

pnpm-lock.yaml

··· 42 42 '@types/node': 43 43 specifier: ^25.5.0 44 44 version: 25.5.0 45 + '@types/pg': 46 + specifier: ^8.20.0 47 + version: 8.20.0 48 + pg: 49 + specifier: ^8.20.0 50 + version: 8.20.0 45 51 tsup: 46 52 specifier: ^8.5.0 47 53 version: 8.5.1(postcss@8.5.8)(tsx@4.21.0)(typescript@5.9.3) ··· 54 60 vitest: 55 61 specifier: ^4.1.0 56 62 version: 4.1.0(@types/node@25.5.0)(vite@8.0.0(@types/node@25.5.0)(esbuild@0.27.0)(tsx@4.21.0)) 57 - wrangler: 58 - specifier: ^4.63.0 59 - version: 4.63.0(@cloudflare/workers-types@4.20260205.0) 60 - 61 - packages/core: 62 - dependencies: 63 - '@atcute/atproto': 64 - specifier: ^3.1.10 65 - version: 3.1.10 66 - '@atcute/client': 67 - specifier: ^4.2.1 68 - version: 4.2.1 69 - '@atcute/identity-resolver': 70 - specifier: ^1.2.2 71 - version: 1.2.2(@atcute/identity@1.1.3) 72 - '@atcute/jetstream': 73 - specifier: ^1.0.2 74 - version: 1.1.2 75 - '@atcute/lexicons': 76 - specifier: ^1.2.7 77 - version: 1.2.7 78 - devDependencies: 79 - '@types/better-sqlite3': 80 - specifier: ^7.6.0 81 - version: 7.6.13 82 - better-sqlite3: 83 - specifier: ^11.0.0 84 - version: 11.10.0 85 - typescript: 86 - specifier: ^5.7.3 87 - version: 5.9.3 88 - 89 - packages/worker: 90 - dependencies: 91 - '@jetstream/core': 92 - specifier: workspace:* 93 - version: link:../core 94 - devDependencies: 95 - '@cloudflare/workers-types': 96 - specifier: ^4.20250124.0 97 - version: 4.20260205.0 98 - typescript: 99 - specifier: ^5.7.3 100 - version: 5.9.3 101 63 wrangler: 102 64 specifier: ^4.63.0 103 65 version: 4.63.0(@cloudflare/workers-types@4.20260205.0) ··· 903 865 '@tybys/wasm-util@0.10.1': 904 866 resolution: {integrity: sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==} 905 867 906 - '@types/better-sqlite3@7.6.13': 907 - resolution: {integrity: sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==} 908 - 909 868 '@types/chai@5.2.3': 910 869 resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} 911 870 ··· 920 879 921 880 '@types/node@25.5.0': 922 881 resolution: {integrity: sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw==} 882 + 883 + '@types/pg@8.20.0': 884 + resolution: {integrity: sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==} 923 885 924 886 '@vitest/expect@4.1.0': 925 887 resolution: {integrity: sha512-EIxG7k4wlWweuCLG9Y5InKFwpMEOyrMb6ZJ1ihYu02LVj/bzUwn2VMU+13PinsjRW75XnITeFrQBMH5+dLvCDA==} ··· 980 942 resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} 981 943 engines: {node: '>=12'} 982 944 983 - base64-js@1.5.1: 984 - resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} 985 - 986 945 better-path-resolve@1.0.0: 987 946 resolution: {integrity: sha512-pbnl5XzGBdrFU/wT4jqmJVPn2B6UHPBOhzMQkY/SPUPB6QtUXtmBHBIwCbXJol93mOpGMnQyP/+BB19q04xj7g==} 988 947 engines: {node: '>=4'} 989 - 990 - better-sqlite3@11.10.0: 991 - resolution: {integrity: sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ==, tarball: https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-11.10.0.tgz} 992 - 993 - bindings@1.5.0: 994 - resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} 995 - 996 - bl@4.1.0: 997 - resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} 998 948 999 949 blake3-wasm@2.1.5: 1000 950 resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==} ··· 1002 952 braces@3.0.3: 1003 953 resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} 1004 954 engines: {node: '>=8'} 1005 - 1006 - buffer@5.7.1: 1007 - resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} 1008 955 1009 956 bundle-require@5.1.0: 1010 957 resolution: {integrity: sha512-3WrrOuZiyaaZPWiEt4G3+IffISVC9HYlWueJEBWED4ZH4aIAC2PnkdnuRrR94M+w6yGWn4AglWtJtBI8YqvgoA==} ··· 1026 973 chokidar@4.0.3: 1027 974 resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} 1028 975 engines: {node: '>= 14.16.0'} 1029 - 1030 - chownr@1.1.4: 1031 - resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} 1032 976 1033 977 commander@4.1.1: 1034 978 resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} ··· 1061 1005 supports-color: 1062 1006 optional: true 1063 1007 1064 - decompress-response@6.0.0: 1065 - resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} 1066 - engines: {node: '>=10'} 1067 - 1068 - deep-extend@0.6.0: 1069 - resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} 1070 - engines: {node: '>=4.0.0'} 1071 - 1072 1008 detect-indent@6.1.0: 1073 1009 resolution: {integrity: sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==} 1074 1010 engines: {node: '>=8'} ··· 1080 1016 dir-glob@3.0.1: 1081 1017 resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} 1082 1018 engines: {node: '>=8'} 1083 - 1084 - end-of-stream@1.4.5: 1085 - resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} 1086 1019 1087 1020 enquirer@2.4.1: 1088 1021 resolution: {integrity: sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ==} ··· 1113 1046 event-target-polyfill@0.0.4: 1114 1047 resolution: {integrity: sha512-Gs6RLjzlLRdT8X9ZipJdIZI/Y6/HhRLyq9RdDlCsnpxr/+Nn6bU2EFGuC94GjxqhM+Nmij2Vcq98yoHrU8uNFQ==} 1115 1048 1116 - expand-template@2.0.3: 1117 - resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} 1118 - engines: {node: '>=6'} 1119 - 1120 1049 expect-type@1.3.0: 1121 1050 resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} 1122 1051 engines: {node: '>=12.0.0'} ··· 1140 1069 picomatch: 1141 1070 optional: true 1142 1071 1143 - file-uri-to-path@1.0.0: 1144 - resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} 1145 - 1146 1072 fill-range@7.1.1: 1147 1073 resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} 1148 1074 engines: {node: '>=8'} ··· 1153 1079 1154 1080 fix-dts-default-cjs-exports@1.0.1: 1155 1081 resolution: {integrity: sha512-pVIECanWFC61Hzl2+oOCtoJ3F17kglZC/6N94eRWycFgBH35hHx0Li604ZIzhseh97mf2p0cv7vVrOZGoqhlEg==} 1156 - 1157 - fs-constants@1.0.0: 1158 - resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} 1159 1082 1160 1083 fs-extra@7.0.1: 1161 1084 resolution: {integrity: sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==} ··· 1172 1095 1173 1096 get-tsconfig@4.13.6: 1174 1097 resolution: {integrity: sha512-shZT/QMiSHc/YBLxxOkMtgSid5HFoauqCE3/exfsEcwg1WkeqjG+V40yBbBrsD+jW2HDXcs28xOfcbm2jI8Ddw==} 1175 - 1176 - github-from-package@0.0.0: 1177 - resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} 1178 1098 1179 1099 glob-parent@5.1.2: 1180 1100 resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} ··· 1199 1119 resolution: {integrity: sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==} 1200 1120 engines: {node: '>=0.10.0'} 1201 1121 1202 - ieee754@1.2.1: 1203 - resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} 1204 - 1205 1122 ignore@5.3.2: 1206 1123 resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} 1207 1124 engines: {node: '>= 4'} 1208 - 1209 - inherits@2.0.4: 1210 - resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} 1211 - 1212 - ini@1.3.8: 1213 - resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} 1214 1125 1215 1126 is-extglob@2.1.1: 1216 1127 resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} ··· 1353 1264 resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} 1354 1265 engines: {node: '>=8.6'} 1355 1266 1356 - mimic-response@3.1.0: 1357 - resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} 1358 - engines: {node: '>=10'} 1359 - 1360 1267 miniflare@4.20260205.0: 1361 1268 resolution: {integrity: sha512-jG1TknEDeFqcq/z5gsOm1rKeg4cNG7ruWxEuiPxl3pnQumavxo8kFpeQC6XKVpAhh2PI9ODGyIYlgd77sTHl5g==} 1362 1269 engines: {node: '>=18.0.0'} 1363 1270 hasBin: true 1364 - 1365 - minimist@1.2.8: 1366 - resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} 1367 - 1368 - mkdirp-classic@0.5.3: 1369 - resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} 1370 1271 1371 1272 mlly@1.8.2: 1372 1273 resolution: {integrity: sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==} ··· 1386 1287 engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} 1387 1288 hasBin: true 1388 1289 1389 - napi-build-utils@2.0.0: 1390 - resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} 1391 - 1392 - node-abi@3.89.0: 1393 - resolution: {integrity: sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA==} 1394 - engines: {node: '>=10'} 1395 - 1396 1290 object-assign@4.1.1: 1397 1291 resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} 1398 1292 engines: {node: '>=0.10.0'} 1399 1293 1400 1294 obug@2.1.1: 1401 1295 resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==} 1402 - 1403 - once@1.4.0: 1404 - resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} 1405 1296 1406 1297 outdent@0.5.0: 1407 1298 resolution: {integrity: sha512-/jHxFIzoMXdqPzTaCpFzAAWhpkSjZPF4Vsn6jAfNpmbH/ymsmd7Qc6VE9BGn0L6YMj6uwpQLxCECpus4ukKS9Q==} ··· 1450 1341 pathe@2.0.3: 1451 1342 resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} 1452 1343 1344 + pg-cloudflare@1.3.0: 1345 + resolution: {integrity: sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==} 1346 + 1347 + pg-connection-string@2.12.0: 1348 + resolution: {integrity: sha512-U7qg+bpswf3Cs5xLzRqbXbQl85ng0mfSV/J0nnA31MCLgvEaAo7CIhmeyrmJpOr7o+zm0rXK+hNnT5l9RHkCkQ==} 1349 + 1350 + pg-int8@1.0.1: 1351 + resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} 1352 + engines: {node: '>=4.0.0'} 1353 + 1354 + pg-pool@3.13.0: 1355 + resolution: {integrity: sha512-gB+R+Xud1gLFuRD/QgOIgGOBE2KCQPaPwkzBBGC9oG69pHTkhQeIuejVIk3/cnDyX39av2AxomQiyPT13WKHQA==} 1356 + peerDependencies: 1357 + pg: '>=8.0' 1358 + 1359 + pg-protocol@1.13.0: 1360 + resolution: {integrity: sha512-zzdvXfS6v89r6v7OcFCHfHlyG/wvry1ALxZo4LqgUoy7W9xhBDMaqOuMiF3qEV45VqsN6rdlcehHrfDtlCPc8w==} 1361 + 1362 + pg-types@2.2.0: 1363 + resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} 1364 + engines: {node: '>=4'} 1365 + 1366 + pg@8.20.0: 1367 + resolution: {integrity: sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==} 1368 + engines: {node: '>= 16.0.0'} 1369 + peerDependencies: 1370 + pg-native: '>=3.0.1' 1371 + peerDependenciesMeta: 1372 + pg-native: 1373 + optional: true 1374 + 1375 + pgpass@1.0.5: 1376 + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} 1377 + 1453 1378 picocolors@1.1.1: 1454 1379 resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} 1455 1380 ··· 1494 1419 resolution: {integrity: sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==} 1495 1420 engines: {node: ^10 || ^12 || >=14} 1496 1421 1497 - prebuild-install@7.1.3: 1498 - resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} 1499 - engines: {node: '>=10'} 1500 - deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available. 1501 - hasBin: true 1422 + postgres-array@2.0.0: 1423 + resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} 1424 + engines: {node: '>=4'} 1425 + 1426 + postgres-bytea@1.0.1: 1427 + resolution: {integrity: sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==} 1428 + engines: {node: '>=0.10.0'} 1429 + 1430 + postgres-date@1.0.7: 1431 + resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} 1432 + engines: {node: '>=0.10.0'} 1433 + 1434 + postgres-interval@1.2.0: 1435 + resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} 1436 + engines: {node: '>=0.10.0'} 1502 1437 1503 1438 prettier@2.8.8: 1504 1439 resolution: {integrity: sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==} ··· 1510 1445 engines: {node: '>=14'} 1511 1446 hasBin: true 1512 1447 1513 - pump@3.0.4: 1514 - resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==} 1515 - 1516 1448 quansync@0.2.11: 1517 1449 resolution: {integrity: sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA==} 1518 1450 1519 1451 queue-microtask@1.2.3: 1520 1452 resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} 1521 1453 1522 - rc@1.2.8: 1523 - resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} 1524 - hasBin: true 1525 - 1526 1454 read-yaml-file@1.1.0: 1527 1455 resolution: {integrity: sha512-VIMnQi/Z4HT2Fxuwg5KrY174U1VdUIASQVWXXyqtNRtxSr9IYkn1rsI6Tb6HsrHCmB7gVpNwX6JxPTHcH6IoTA==} 1528 1456 engines: {node: '>=6'} 1529 - 1530 - readable-stream@3.6.2: 1531 - resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} 1532 - engines: {node: '>= 6'} 1533 1457 1534 1458 readdirp@4.1.2: 1535 1459 resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} ··· 1559 1483 run-parallel@1.2.0: 1560 1484 resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} 1561 1485 1562 - safe-buffer@5.2.1: 1563 - resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} 1564 - 1565 1486 safer-buffer@2.1.2: 1566 1487 resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} 1567 1488 ··· 1589 1510 resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} 1590 1511 engines: {node: '>=14'} 1591 1512 1592 - simple-concat@1.0.1: 1593 - resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} 1594 - 1595 - simple-get@4.0.1: 1596 - resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} 1597 - 1598 1513 slash@3.0.0: 1599 1514 resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} 1600 1515 engines: {node: '>=8'} ··· 1610 1525 spawndamnit@3.0.1: 1611 1526 resolution: {integrity: sha512-MmnduQUuHCoFckZoWnXsTg7JaiLBJrKFj9UI2MbRPGaJeVpsLcVBu6P/IGZovziM/YBsellCmsprgNA+w0CzVg==} 1612 1527 1528 + split2@4.2.0: 1529 + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} 1530 + engines: {node: '>= 10.x'} 1531 + 1613 1532 sprintf-js@1.0.3: 1614 1533 resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} 1615 1534 ··· 1619 1538 std-env@4.0.0: 1620 1539 resolution: {integrity: sha512-zUMPtQ/HBY3/50VbpkupYHbRroTRZJPRLvreamgErJVys0ceuzMkD44J/QjqhHjOzK42GQ3QZIeFG1OYfOtKqQ==} 1621 1540 1622 - string_decoder@1.3.0: 1623 - resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} 1624 - 1625 1541 strip-ansi@6.0.1: 1626 1542 resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} 1627 1543 engines: {node: '>=8'} ··· 1629 1545 strip-bom@3.0.0: 1630 1546 resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} 1631 1547 engines: {node: '>=4'} 1632 - 1633 - strip-json-comments@2.0.1: 1634 - resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} 1635 - engines: {node: '>=0.10.0'} 1636 1548 1637 1549 sucrase@3.35.1: 1638 1550 resolution: {integrity: sha512-DhuTmvZWux4H1UOnWMB3sk0sbaCVOoQZjv8u1rDoTV0HTdGem9hkAZtl4JZy8P2z4Bg0nT+YMeOFyVr4zcG5Tw==} ··· 1642 1554 supports-color@10.2.2: 1643 1555 resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} 1644 1556 engines: {node: '>=18'} 1645 - 1646 - tar-fs@2.1.4: 1647 - resolution: {integrity: sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==} 1648 - 1649 - tar-stream@2.2.0: 1650 - resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} 1651 - engines: {node: '>=6'} 1652 1557 1653 1558 term-size@2.2.1: 1654 1559 resolution: {integrity: sha512-wK0Ri4fOGjv/XPy8SBHZChl8CM7uMc5VML7SqiQ0zG7+J5Vr+RMQDoHa2CNT6KHUnTGIXH34UDMkPzAUyapBZg==} ··· 1717 1622 engines: {node: '>=18.0.0'} 1718 1623 hasBin: true 1719 1624 1720 - tunnel-agent@0.6.0: 1721 - resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} 1722 - 1723 1625 type-fest@4.41.0: 1724 1626 resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} 1725 1627 engines: {node: '>=16'} ··· 1748 1650 universalify@0.1.2: 1749 1651 resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==} 1750 1652 engines: {node: '>= 4.0.0'} 1751 - 1752 - util-deprecate@1.0.2: 1753 - resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} 1754 1653 1755 1654 vite@8.0.0: 1756 1655 resolution: {integrity: sha512-fPGaRNj9Zytaf8LEiBhY7Z6ijnFKdzU/+mL8EFBaKr7Vw1/FWcTBAMW0wLPJAGMPX38ZPVCVgLceWiEqeoqL2Q==} ··· 1855 1754 '@cloudflare/workers-types': 1856 1755 optional: true 1857 1756 1858 - wrappy@1.0.2: 1859 - resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} 1860 - 1861 1757 ws@8.18.0: 1862 1758 resolution: {integrity: sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==} 1863 1759 engines: {node: '>=10.0.0'} ··· 1869 1765 optional: true 1870 1766 utf-8-validate: 1871 1767 optional: true 1768 + 1769 + xtend@4.0.2: 1770 + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} 1771 + engines: {node: '>=0.4'} 1872 1772 1873 1773 yocto-queue@1.2.2: 1874 1774 resolution: {integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==} ··· 2613 2513 tslib: 2.8.1 2614 2514 optional: true 2615 2515 2616 - '@types/better-sqlite3@7.6.13': 2617 - dependencies: 2618 - '@types/node': 25.5.0 2619 - 2620 2516 '@types/chai@5.2.3': 2621 2517 dependencies: 2622 2518 '@types/deep-eql': 4.0.2 ··· 2631 2527 '@types/node@25.5.0': 2632 2528 dependencies: 2633 2529 undici-types: 7.18.2 2530 + 2531 + '@types/pg@8.20.0': 2532 + dependencies: 2533 + '@types/node': 25.5.0 2534 + pg-protocol: 1.13.0 2535 + pg-types: 2.2.0 2634 2536 2635 2537 '@vitest/expect@4.1.0': 2636 2538 dependencies: ··· 2691 2593 2692 2594 assertion-error@2.0.1: {} 2693 2595 2694 - base64-js@1.5.1: {} 2695 - 2696 2596 better-path-resolve@1.0.0: 2697 2597 dependencies: 2698 2598 is-windows: 1.0.2 2699 - 2700 - better-sqlite3@11.10.0: 2701 - dependencies: 2702 - bindings: 1.5.0 2703 - prebuild-install: 7.1.3 2704 - 2705 - bindings@1.5.0: 2706 - dependencies: 2707 - file-uri-to-path: 1.0.0 2708 - 2709 - bl@4.1.0: 2710 - dependencies: 2711 - buffer: 5.7.1 2712 - inherits: 2.0.4 2713 - readable-stream: 3.6.2 2714 2599 2715 2600 blake3-wasm@2.1.5: {} 2716 2601 2717 2602 braces@3.0.3: 2718 2603 dependencies: 2719 2604 fill-range: 7.1.1 2720 - 2721 - buffer@5.7.1: 2722 - dependencies: 2723 - base64-js: 1.5.1 2724 - ieee754: 1.2.1 2725 2605 2726 2606 bundle-require@5.1.0(esbuild@0.27.0): 2727 2607 dependencies: ··· 2737 2617 chokidar@4.0.3: 2738 2618 dependencies: 2739 2619 readdirp: 4.1.2 2740 - 2741 - chownr@1.1.4: {} 2742 2620 2743 2621 commander@4.1.1: {} 2744 2622 ··· 2760 2638 dependencies: 2761 2639 ms: 2.1.3 2762 2640 2763 - decompress-response@6.0.0: 2764 - dependencies: 2765 - mimic-response: 3.1.0 2766 - 2767 - deep-extend@0.6.0: {} 2768 - 2769 2641 detect-indent@6.1.0: {} 2770 2642 2771 2643 detect-libc@2.1.2: {} ··· 2773 2645 dir-glob@3.0.1: 2774 2646 dependencies: 2775 2647 path-type: 4.0.0 2776 - 2777 - end-of-stream@1.4.5: 2778 - dependencies: 2779 - once: 1.4.0 2780 2648 2781 2649 enquirer@2.4.1: 2782 2650 dependencies: ··· 2826 2694 2827 2695 event-target-polyfill@0.0.4: {} 2828 2696 2829 - expand-template@2.0.3: {} 2830 - 2831 2697 expect-type@1.3.0: {} 2832 2698 2833 2699 extendable-error@0.1.7: {} ··· 2848 2714 optionalDependencies: 2849 2715 picomatch: 4.0.3 2850 2716 2851 - file-uri-to-path@1.0.0: {} 2852 - 2853 2717 fill-range@7.1.1: 2854 2718 dependencies: 2855 2719 to-regex-range: 5.0.1 ··· 2864 2728 magic-string: 0.30.21 2865 2729 mlly: 1.8.2 2866 2730 rollup: 4.60.1 2867 - 2868 - fs-constants@1.0.0: {} 2869 2731 2870 2732 fs-extra@7.0.1: 2871 2733 dependencies: ··· 2885 2747 get-tsconfig@4.13.6: 2886 2748 dependencies: 2887 2749 resolve-pkg-maps: 1.0.0 2888 - 2889 - github-from-package@0.0.0: {} 2890 2750 2891 2751 glob-parent@5.1.2: 2892 2752 dependencies: ··· 2911 2771 dependencies: 2912 2772 safer-buffer: 2.1.2 2913 2773 2914 - ieee754@1.2.1: {} 2915 - 2916 2774 ignore@5.3.2: {} 2917 - 2918 - inherits@2.0.4: {} 2919 - 2920 - ini@1.3.8: {} 2921 2775 2922 2776 is-extglob@2.1.1: {} 2923 2777 ··· 3024 2878 braces: 3.0.3 3025 2879 picomatch: 2.3.2 3026 2880 3027 - mimic-response@3.1.0: {} 3028 - 3029 2881 miniflare@4.20260205.0: 3030 2882 dependencies: 3031 2883 '@cspotcode/source-map-support': 0.8.1 ··· 3037 2889 transitivePeerDependencies: 3038 2890 - bufferutil 3039 2891 - utf-8-validate 3040 - 3041 - minimist@1.2.8: {} 3042 - 3043 - mkdirp-classic@0.5.3: {} 3044 2892 3045 2893 mlly@1.8.2: 3046 2894 dependencies: ··· 3061 2909 3062 2910 nanoid@3.3.11: {} 3063 2911 3064 - napi-build-utils@2.0.0: {} 3065 - 3066 - node-abi@3.89.0: 3067 - dependencies: 3068 - semver: 7.7.4 3069 - 3070 2912 object-assign@4.1.1: {} 3071 2913 3072 2914 obug@2.1.1: {} 3073 - 3074 - once@1.4.0: 3075 - dependencies: 3076 - wrappy: 1.0.2 3077 2915 3078 2916 outdent@0.5.0: {} 3079 2917 ··· 3111 2949 3112 2950 pathe@2.0.3: {} 3113 2951 2952 + pg-cloudflare@1.3.0: 2953 + optional: true 2954 + 2955 + pg-connection-string@2.12.0: {} 2956 + 2957 + pg-int8@1.0.1: {} 2958 + 2959 + pg-pool@3.13.0(pg@8.20.0): 2960 + dependencies: 2961 + pg: 8.20.0 2962 + 2963 + pg-protocol@1.13.0: {} 2964 + 2965 + pg-types@2.2.0: 2966 + dependencies: 2967 + pg-int8: 1.0.1 2968 + postgres-array: 2.0.0 2969 + postgres-bytea: 1.0.1 2970 + postgres-date: 1.0.7 2971 + postgres-interval: 1.2.0 2972 + 2973 + pg@8.20.0: 2974 + dependencies: 2975 + pg-connection-string: 2.12.0 2976 + pg-pool: 3.13.0(pg@8.20.0) 2977 + pg-protocol: 1.13.0 2978 + pg-types: 2.2.0 2979 + pgpass: 1.0.5 2980 + optionalDependencies: 2981 + pg-cloudflare: 1.3.0 2982 + 2983 + pgpass@1.0.5: 2984 + dependencies: 2985 + split2: 4.2.0 2986 + 3114 2987 picocolors@1.1.1: {} 3115 2988 3116 2989 picomatch@2.3.2: {} ··· 3140 3013 picocolors: 1.1.1 3141 3014 source-map-js: 1.2.1 3142 3015 3143 - prebuild-install@7.1.3: 3016 + postgres-array@2.0.0: {} 3017 + 3018 + postgres-bytea@1.0.1: {} 3019 + 3020 + postgres-date@1.0.7: {} 3021 + 3022 + postgres-interval@1.2.0: 3144 3023 dependencies: 3145 - detect-libc: 2.1.2 3146 - expand-template: 2.0.3 3147 - github-from-package: 0.0.0 3148 - minimist: 1.2.8 3149 - mkdirp-classic: 0.5.3 3150 - napi-build-utils: 2.0.0 3151 - node-abi: 3.89.0 3152 - pump: 3.0.4 3153 - rc: 1.2.8 3154 - simple-get: 4.0.1 3155 - tar-fs: 2.1.4 3156 - tunnel-agent: 0.6.0 3024 + xtend: 4.0.2 3157 3025 3158 3026 prettier@2.8.8: {} 3159 3027 3160 3028 prettier@3.8.1: {} 3161 3029 3162 - pump@3.0.4: 3163 - dependencies: 3164 - end-of-stream: 1.4.5 3165 - once: 1.4.0 3166 - 3167 3030 quansync@0.2.11: {} 3168 3031 3169 3032 queue-microtask@1.2.3: {} 3170 - 3171 - rc@1.2.8: 3172 - dependencies: 3173 - deep-extend: 0.6.0 3174 - ini: 1.3.8 3175 - minimist: 1.2.8 3176 - strip-json-comments: 2.0.1 3177 3033 3178 3034 read-yaml-file@1.1.0: 3179 3035 dependencies: ··· 3181 3037 js-yaml: 3.14.2 3182 3038 pify: 4.0.1 3183 3039 strip-bom: 3.0.0 3184 - 3185 - readable-stream@3.6.2: 3186 - dependencies: 3187 - inherits: 2.0.4 3188 - string_decoder: 1.3.0 3189 - util-deprecate: 1.0.2 3190 3040 3191 3041 readdirp@4.1.2: {} 3192 3042 ··· 3252 3102 dependencies: 3253 3103 queue-microtask: 1.2.3 3254 3104 3255 - safe-buffer@5.2.1: {} 3256 - 3257 3105 safer-buffer@2.1.2: {} 3258 3106 3259 3107 semver@7.7.4: {} ··· 3299 3147 3300 3148 signal-exit@4.1.0: {} 3301 3149 3302 - simple-concat@1.0.1: {} 3303 - 3304 - simple-get@4.0.1: 3305 - dependencies: 3306 - decompress-response: 6.0.0 3307 - once: 1.4.0 3308 - simple-concat: 1.0.1 3309 - 3310 3150 slash@3.0.0: {} 3311 3151 3312 3152 source-map-js@1.2.1: {} ··· 3318 3158 cross-spawn: 7.0.6 3319 3159 signal-exit: 4.1.0 3320 3160 3161 + split2@4.2.0: {} 3162 + 3321 3163 sprintf-js@1.0.3: {} 3322 3164 3323 3165 stackback@0.0.2: {} 3324 3166 3325 3167 std-env@4.0.0: {} 3326 3168 3327 - string_decoder@1.3.0: 3328 - dependencies: 3329 - safe-buffer: 5.2.1 3330 - 3331 3169 strip-ansi@6.0.1: 3332 3170 dependencies: 3333 3171 ansi-regex: 5.0.1 3334 3172 3335 3173 strip-bom@3.0.0: {} 3336 - 3337 - strip-json-comments@2.0.1: {} 3338 3174 3339 3175 sucrase@3.35.1: 3340 3176 dependencies: ··· 3347 3183 ts-interface-checker: 0.1.13 3348 3184 3349 3185 supports-color@10.2.2: {} 3350 - 3351 - tar-fs@2.1.4: 3352 - dependencies: 3353 - chownr: 1.1.4 3354 - mkdirp-classic: 0.5.3 3355 - pump: 3.0.4 3356 - tar-stream: 2.2.0 3357 - 3358 - tar-stream@2.2.0: 3359 - dependencies: 3360 - bl: 4.1.0 3361 - end-of-stream: 1.4.5 3362 - fs-constants: 1.0.0 3363 - inherits: 2.0.4 3364 - readable-stream: 3.6.2 3365 3186 3366 3187 term-size@2.2.1: {} 3367 3188 ··· 3432 3253 optionalDependencies: 3433 3254 fsevents: 2.3.3 3434 3255 3435 - tunnel-agent@0.6.0: 3436 - dependencies: 3437 - safe-buffer: 5.2.1 3438 - 3439 3256 type-fest@4.41.0: {} 3440 3257 3441 3258 typescript@5.9.3: {} ··· 3453 3270 unicode-segmenter@0.14.5: {} 3454 3271 3455 3272 universalify@0.1.2: {} 3456 - 3457 - util-deprecate@1.0.2: {} 3458 3273 3459 3274 vite@8.0.0(@types/node@25.5.0)(esbuild@0.27.0)(tsx@4.21.0): 3460 3275 dependencies: ··· 3531 3346 - bufferutil 3532 3347 - utf-8-validate 3533 3348 3534 - wrappy@1.0.2: {} 3535 - 3536 3349 ws@8.18.0: {} 3350 + 3351 + xtend@4.0.2: {} 3537 3352 3538 3353 yocto-queue@1.2.2: {} 3539 3354

+4

pnpm-workspace.yaml

··· 1 + ignoredBuiltDependencies: 2 + - esbuild 3 + - sharp 4 + - workerd

+2 -1

tsup.config.ts

··· 6 6 "src/server.ts", 7 7 "src/generate.ts", 8 8 "src/adapters/sqlite.ts", 9 + "src/adapters/postgres.ts", 9 10 ], 10 11 format: ["esm"], 11 12 dts: true, 12 13 sourcemap: true, 13 14 clean: true, 14 15 tsconfig: "tsconfig.build.json", 15 - external: ["node:sqlite"], 16 + external: ["node:sqlite", "pg"], 16 17 });

+2

vitest.config.ts

··· 3 3 export default defineConfig({ 4 4 test: { 5 5 include: ["tests/**/*.test.ts"], 6 + // PostgreSQL tests share a single database and cannot run in parallel 7 + fileParallelism: false, 6 8 }, 7 9 });

+5

.changeset/tall-seals-sink.md

··· 1 + --- 2 + "@atmo-dev/contrail": patch 3 + --- 4 + 5 + add postgres adapter and example

+94

bin/ingest.ts

··· 1 + /** 2 + * Persistent Jetstream ingestion with runtime configuration. 3 + * 4 + * Connects to an externally-provided database and subscribes to Jetstream. 5 + * The container does not own the database — it receives a connection URL. 6 + * 7 + * Environment: 8 + * JETSTREAM_URL - Jetstream WebSocket URL (required) 9 + * CONTRAIL_ADAPTER - "postgres" or "sqlite" (default: "postgres") 10 + * DATABASE_URL - PostgreSQL connection string (required when adapter=postgres) 11 + * SQLITE_PATH - SQLite file path (required when adapter=sqlite) 12 + * CONTRAIL_CONFIG - Path to config file (default: "/app/config/contrail.config.ts") 13 + * FLUSH_INTERVAL_MS - Batch flush interval in ms (default: 500) 14 + */ 15 + 16 + import { Contrail } from "../src/index.ts"; 17 + import type { ContrailConfig, Database } from "../src/index.ts"; 18 + 19 + const ADAPTER = process.env.CONTRAIL_ADAPTER ?? "postgres"; 20 + const JETSTREAM_URL = process.env.JETSTREAM_URL; 21 + const CONFIG_PATH = process.env.CONTRAIL_CONFIG ?? "/app/config/contrail.config.ts"; 22 + const FLUSH_INTERVAL_MS = parseInt(process.env.FLUSH_INTERVAL_MS ?? "500", 10); 23 + 24 + async function createDatabase(): Promise<{ db: Database; cleanup: () => Promise<void> }> { 25 + if (ADAPTER === "postgres") { 26 + const url = process.env.DATABASE_URL; 27 + if (!url) { 28 + console.error("DATABASE_URL is required when CONTRAIL_ADAPTER=postgres"); 29 + process.exit(1); 30 + } 31 + const pg = await import("pg"); 32 + const { createPostgresDatabase } = await import("../src/adapters/postgres.ts"); 33 + const pool = new pg.default.Pool({ connectionString: url }); 34 + return { db: createPostgresDatabase(pool), cleanup: () => pool.end() }; 35 + } 36 + 37 + if (ADAPTER === "sqlite") { 38 + const path = process.env.SQLITE_PATH; 39 + if (!path) { 40 + console.error("SQLITE_PATH is required when CONTRAIL_ADAPTER=sqlite"); 41 + process.exit(1); 42 + } 43 + const { createSqliteDatabase } = await import("../src/adapters/sqlite.ts"); 44 + return { db: createSqliteDatabase(path), cleanup: async () => {} }; 45 + } 46 + 47 + console.error(`Unknown adapter: ${ADAPTER}. Use "postgres" or "sqlite".`); 48 + process.exit(1); 49 + } 50 + 51 + async function loadConfig(): Promise<ContrailConfig> { 52 + const module = await import(CONFIG_PATH); 53 + return module.config ?? module.default; 54 + } 55 + 56 + async function main() { 57 + if (!JETSTREAM_URL) { 58 + console.error("JETSTREAM_URL is required"); 59 + process.exit(1); 60 + } 61 + 62 + const config = await loadConfig(); 63 + config.jetstreams = [JETSTREAM_URL]; 64 + 65 + const { db, cleanup } = await createDatabase(); 66 + const contrail = new Contrail({ ...config, db }); 67 + 68 + const controller = new AbortController(); 69 + const shutdown = () => { 70 + console.log("\nShutting down gracefully..."); 71 + controller.abort(); 72 + }; 73 + process.on("SIGTERM", shutdown); 74 + process.on("SIGINT", shutdown); 75 + 76 + console.log("Contrail ingester starting"); 77 + console.log(` Adapter: ${ADAPTER}`); 78 + console.log(` Jetstream: ${JETSTREAM_URL}`); 79 + console.log(` Config: ${CONFIG_PATH}`); 80 + console.log(` Flush: ${FLUSH_INTERVAL_MS}ms`); 81 + 82 + await contrail.runPersistent({ 83 + signal: controller.signal, 84 + flushIntervalMs: FLUSH_INTERVAL_MS, 85 + }); 86 + 87 + await cleanup(); 88 + console.log("Done."); 89 + } 90 + 91 + main().catch((err) => { 92 + console.error(err); 93 + process.exit(1); 94 + });

+7

src/contrail.ts

··· 9 9 import type { BackfillAllOptions, BackfillProgress } from "./core/backfill"; 10 10 import { processNotifyUris } from "./core/router/notify"; 11 11 import type { NotifyResult } from "./core/router/notify"; 12 + import { runPersistent as runPersistentIngestion } from "./core/persistent"; 13 + import type { PersistentIngestOptions } from "./core/persistent"; 12 14 13 15 export interface ContrailOptions extends ContrailConfig { 14 16 db?: Database; ··· 49 51 /** Run one Jetstream ingestion cycle (catches up to present, then stops). */ 50 52 async ingest(options?: { timeoutMs?: number }, db?: Database): Promise<void> { 51 53 await runIngestCycle(this.getDb(db), this.config, options?.timeoutMs, this._ingestState); 54 + } 55 + 56 + /** Run persistent Jetstream ingestion (long-lived, stays connected). */ 57 + async runPersistent(options?: Omit<PersistentIngestOptions, 'logger'>, db?: Database): Promise<void> { 58 + await runPersistentIngestion(this.getDb(db), this.config, { ...options, logger: this.config.logger }); 52 59 } 53 60 54 61 /** Discover users from relays. Returns discovered DIDs. */

+3

src/index.ts

··· 26 26 export type { QueryOptions, SortOption } from "./core/db/records"; 27 27 export type { BackfillProgress, BackfillAllOptions } from "./core/backfill"; 28 28 export type { NotifyResult } from "./core/router/notify"; 29 + 30 + export { runPersistent } from "./core/persistent"; 31 + export type { PersistentIngestOptions } from "./core/persistent";

+135

tests/dialect.test.ts

··· 1 + import { describe, it, expect } from "vitest"; 2 + import { sqliteDialect, postgresDialect, buildFtsSchema, ftsQueryClause } from "../src/core/dialect"; 3 + import { createSqliteDatabase } from "../src/adapters/sqlite"; 4 + 5 + describe("sqliteDialect", () => { 6 + it("generates json_extract for flat field", () => { 7 + expect(sqliteDialect.jsonExtract("record", "name")).toBe( 8 + "json_extract(record, '$.name')" 9 + ); 10 + }); 11 + 12 + it("generates json_extract for nested field", () => { 13 + expect(sqliteDialect.jsonExtract("record", "subject.uri")).toBe( 14 + "json_extract(record, '$.subject.uri')" 15 + ); 16 + }); 17 + 18 + it("generates json_extract for aliased column", () => { 19 + expect(sqliteDialect.jsonExtract("r.record", "mode")).toBe( 20 + "json_extract(r.record, '$.mode')" 21 + ); 22 + }); 23 + 24 + it("wraps statement with OR IGNORE", () => { 25 + const sql = "INSERT INTO feed_items (actor, uri) SELECT ?, ? FROM t WHERE x = ?"; 26 + expect(sqliteDialect.insertOrIgnore(sql)).toBe( 27 + "INSERT OR IGNORE INTO feed_items (actor, uri) SELECT ?, ? FROM t WHERE x = ?" 28 + ); 29 + }); 30 + 31 + it("reports recordColumnType as TEXT", () => { 32 + expect(sqliteDialect.recordColumnType).toBe("TEXT"); 33 + }); 34 + 35 + it("reports ftsStrategy as virtual-table", () => { 36 + expect(sqliteDialect.ftsStrategy).toBe("virtual-table"); 37 + }); 38 + 39 + }); 40 + 41 + describe("Database.dialect", () => { 42 + it("sqlite adapter exposes sqliteDialect", () => { 43 + const db = createSqliteDatabase(":memory:"); 44 + expect(db.dialect).toBeDefined(); 45 + expect(db.dialect.recordColumnType).toBe("TEXT"); 46 + }); 47 + }); 48 + 49 + describe("postgresDialect", () => { 50 + it("generates ->> for flat field", () => { 51 + expect(postgresDialect.jsonExtract("record", "name")).toBe( 52 + "record->>'name'" 53 + ); 54 + }); 55 + 56 + it("generates -> then ->> for nested field", () => { 57 + expect(postgresDialect.jsonExtract("record", "subject.uri")).toBe( 58 + "record->'subject'->>'uri'" 59 + ); 60 + }); 61 + 62 + it("handles aliased column", () => { 63 + expect(postgresDialect.jsonExtract("r.record", "mode")).toBe( 64 + "r.record->>'mode'" 65 + ); 66 + }); 67 + 68 + it("handles deeply nested field", () => { 69 + expect(postgresDialect.jsonExtract("record", "a.b.c")).toBe( 70 + "record->'a'->'b'->>'c'" 71 + ); 72 + }); 73 + 74 + it("appends ON CONFLICT DO NOTHING", () => { 75 + const sql = "INSERT INTO feed_items (actor, uri) SELECT ?, ? FROM t WHERE x = ?"; 76 + expect(postgresDialect.insertOrIgnore(sql)).toBe( 77 + "INSERT INTO feed_items (actor, uri) SELECT ?, ? FROM t WHERE x = ? ON CONFLICT DO NOTHING" 78 + ); 79 + }); 80 + 81 + it("reports recordColumnType as JSONB", () => { 82 + expect(postgresDialect.recordColumnType).toBe("JSONB"); 83 + }); 84 + 85 + it("reports ftsStrategy as generated-column", () => { 86 + expect(postgresDialect.ftsStrategy).toBe("generated-column"); 87 + }); 88 + 89 + }); 90 + 91 + describe("indexExpression", () => { 92 + it("sqlite passes through expression unchanged", () => { 93 + expect(sqliteDialect.indexExpression("json_extract(record, '$.name')")).toBe( 94 + "json_extract(record, '$.name')" 95 + ); 96 + }); 97 + 98 + it("postgres wraps expression in parens", () => { 99 + expect(postgresDialect.indexExpression("record->>'name'")).toBe( 100 + "(record->>'name')" 101 + ); 102 + }); 103 + }); 104 + 105 + describe("FTS schema generation", () => { 106 + it("sqlite generates virtual table", () => { 107 + const stmts = buildFtsSchema(sqliteDialect, "records_community_lexicon_calendar_event", ["name", "description"]); 108 + expect(stmts).toHaveLength(1); 109 + expect(stmts[0]).toContain("CREATE VIRTUAL TABLE"); 110 + expect(stmts[0]).toContain("USING fts5"); 111 + }); 112 + 113 + it("postgres generates tsvector column + GIN index", () => { 114 + const stmts = buildFtsSchema(postgresDialect, "records_community_lexicon_calendar_event", ["name", "description"]); 115 + expect(stmts.length).toBeGreaterThanOrEqual(2); 116 + expect(stmts.some(s => s.includes("tsvector"))).toBe(true); 117 + expect(stmts.some(s => s.includes("USING GIN"))).toBe(true); 118 + }); 119 + }); 120 + 121 + describe("FTS query clause", () => { 122 + it("sqlite uses FTS5 join and MATCH", () => { 123 + const clause = ftsQueryClause(sqliteDialect, "records_community_lexicon_calendar_event"); 124 + expect(clause.join).toContain("JOIN fts_"); 125 + expect(clause.condition).toContain("MATCH"); 126 + expect(clause.orderExpr).toBe("fts.rank"); 127 + }); 128 + 129 + it("postgres uses tsvector condition and ts_rank", () => { 130 + const clause = ftsQueryClause(postgresDialect, "records_community_lexicon_calendar_event"); 131 + expect(clause.join).toBe(""); 132 + expect(clause.condition).toContain("plainto_tsquery"); 133 + expect(clause.orderExpr).toContain("ts_rank"); 134 + }); 135 + });

+193

tests/persistent.test.ts

··· 1 + import { describe, it, expect, vi, beforeEach } from "vitest"; 2 + import type { Database } from "../src/core/types"; 3 + import { createTestDbWithSchema, TEST_CONFIG } from "./helpers"; 4 + import { runPersistent } from "../src/core/persistent"; 5 + import { getLastCursor, queryRecords } from "../src/core/db/records"; 6 + 7 + // Mock identity resolution to avoid network calls in tests 8 + vi.mock("../src/core/identity", () => ({ 9 + refreshStaleIdentities: vi.fn().mockResolvedValue(undefined), 10 + })); 11 + 12 + let db: Database; 13 + 14 + beforeEach(async () => { 15 + db = await createTestDbWithSchema(); 16 + }); 17 + 18 + // Helper: create a mock async iterable that yields events then hangs until aborted 19 + function mockSubscription(events: Array<{ kind: string; did: string; time_us: number; commit?: any }>) { 20 + let aborted = false; 21 + return { 22 + cursor: 0, 23 + [Symbol.asyncIterator]() { 24 + let i = 0; 25 + return { 26 + next: async () => { 27 + if (aborted) return { value: undefined, done: true as const }; 28 + if (i < events.length) { 29 + return { value: events[i++], done: false as const }; 30 + } 31 + // Hang until iterator is returned (abort) 32 + return new Promise<IteratorResult<any>>(() => {}); 33 + }, 34 + return: async () => { 35 + aborted = true; 36 + return { value: undefined, done: true as const }; 37 + }, 38 + }; 39 + }, 40 + }; 41 + } 42 + 43 + describe("runPersistent", () => { 44 + it("flushes when batch size is reached", async () => { 45 + const events = Array.from({ length: 50 }, (_, i) => ({ 46 + kind: "commit" as const, 47 + did: `did:plc:user${i}`, 48 + time_us: 1000 + i, 49 + commit: { 50 + collection: "community.lexicon.calendar.event", 51 + operation: "create", 52 + rkey: `rkey${i}`, 53 + cid: `cid${i}`, 54 + record: { name: `Event ${i}`, startsAt: "2026-04-01T10:00:00Z", mode: "online" }, 55 + }, 56 + })); 57 + 58 + const controller = new AbortController(); 59 + 60 + // After yielding 50 events, the mock hangs. Give it time to flush, then abort. 61 + const promise = runPersistent(db, TEST_CONFIG, { 62 + batchSize: 50, 63 + flushIntervalMs: 60_000, // high so only batch size triggers flush 64 + signal: controller.signal, 65 + createSubscription: () => mockSubscription(events) as any, 66 + }); 67 + 68 + // Wait for flush to complete 69 + await new Promise((r) => setTimeout(r, 200)); 70 + controller.abort(); 71 + await promise; 72 + 73 + const result = await queryRecords(db, TEST_CONFIG, { 74 + collection: "community.lexicon.calendar.event", 75 + limit: 100, 76 + }); 77 + expect(result.records.length).toBe(50); 78 + 79 + const cursor = await getLastCursor(db); 80 + expect(cursor).toBe(1049); // last event's time_us 81 + }); 82 + 83 + it("flushes on timer when buffer is not full", async () => { 84 + const events = Array.from({ length: 10 }, (_, i) => ({ 85 + kind: "commit" as const, 86 + did: `did:plc:user${i}`, 87 + time_us: 2000 + i, 88 + commit: { 89 + collection: "community.lexicon.calendar.event", 90 + operation: "create", 91 + rkey: `timer${i}`, 92 + cid: `cid${i}`, 93 + record: { name: `Timer Event ${i}`, startsAt: "2026-04-01T10:00:00Z", mode: "online" }, 94 + }, 95 + })); 96 + 97 + const controller = new AbortController(); 98 + 99 + const promise = runPersistent(db, TEST_CONFIG, { 100 + batchSize: 100, // high so timer triggers flush, not batch size 101 + flushIntervalMs: 100, // 100ms timer 102 + signal: controller.signal, 103 + createSubscription: () => mockSubscription(events) as any, 104 + }); 105 + 106 + // Wait for timer flush 107 + await new Promise((r) => setTimeout(r, 500)); 108 + controller.abort(); 109 + await promise; 110 + 111 + const result = await queryRecords(db, TEST_CONFIG, { 112 + collection: "community.lexicon.calendar.event", 113 + limit: 100, 114 + }); 115 + expect(result.records.length).toBe(10); 116 + }); 117 + 118 + it("flushes remaining buffer on abort", async () => { 119 + const events = Array.from({ length: 5 }, (_, i) => ({ 120 + kind: "commit" as const, 121 + did: `did:plc:user${i}`, 122 + time_us: 3000 + i, 123 + commit: { 124 + collection: "community.lexicon.calendar.event", 125 + operation: "create", 126 + rkey: `abort${i}`, 127 + cid: `cid${i}`, 128 + record: { name: `Abort Event ${i}`, startsAt: "2026-04-01T10:00:00Z", mode: "online" }, 129 + }, 130 + })); 131 + 132 + const controller = new AbortController(); 133 + 134 + const promise = runPersistent(db, TEST_CONFIG, { 135 + batchSize: 100, 136 + flushIntervalMs: 60_000, 137 + signal: controller.signal, 138 + createSubscription: () => mockSubscription(events) as any, 139 + }); 140 + 141 + // Let events be consumed, then abort before timer or batch triggers 142 + await new Promise((r) => setTimeout(r, 200)); 143 + controller.abort(); 144 + await promise; 145 + 146 + // The final flush in the finally block should have saved them 147 + const result = await queryRecords(db, TEST_CONFIG, { 148 + collection: "community.lexicon.calendar.event", 149 + limit: 100, 150 + }); 151 + expect(result.records.length).toBe(5); 152 + 153 + const cursor = await getLastCursor(db); 154 + expect(cursor).toBe(3004); 155 + }); 156 + 157 + it("skips non-commit events", async () => { 158 + const events = [ 159 + { kind: "identity" as const, did: "did:plc:someone", time_us: 4000 }, 160 + { 161 + kind: "commit" as const, 162 + did: "did:plc:real", 163 + time_us: 4001, 164 + commit: { 165 + collection: "community.lexicon.calendar.event", 166 + operation: "create", 167 + rkey: "only1", 168 + cid: "cidonly", 169 + record: { name: "Only Event", startsAt: "2026-04-01T10:00:00Z", mode: "online" }, 170 + }, 171 + }, 172 + ]; 173 + 174 + const controller = new AbortController(); 175 + 176 + const promise = runPersistent(db, TEST_CONFIG, { 177 + batchSize: 100, 178 + flushIntervalMs: 100, 179 + signal: controller.signal, 180 + createSubscription: () => mockSubscription(events) as any, 181 + }); 182 + 183 + await new Promise((r) => setTimeout(r, 500)); 184 + controller.abort(); 185 + await promise; 186 + 187 + const result = await queryRecords(db, TEST_CONFIG, { 188 + collection: "community.lexicon.calendar.event", 189 + limit: 100, 190 + }); 191 + expect(result.records.length).toBe(1); 192 + }); 193 + });

+505

tests/postgres-e2e.test.ts

··· 1 + /** 2 + * End-to-end test exercising the full Contrail lifecycle on PostgreSQL. 3 + * 4 + * Covers: schema init, event ingestion, upserts, deletes, relation counts, 5 + * JSON filters, range filters, pagination, sorting, cursor keyset, FTS via 6 + * tsvector, and hydration — all against a real PostgreSQL database. 7 + * 8 + * Requires TEST_DATABASE_URL env var pointing at a dedicated test database. 9 + */ 10 + import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; 11 + import pg from "pg"; 12 + import { createPostgresDatabase } from "../src/adapters/postgres"; 13 + import { initSchema } from "../src/core/db/schema"; 14 + import { 15 + applyEvents, 16 + queryRecords, 17 + getLastCursor, 18 + saveCursor, 19 + } from "../src/core/db/records"; 20 + import { resolveConfig } from "../src/core/types"; 21 + import type { Database } from "../src/core/types"; 22 + import { resolveHydrates, resolveReferences } from "../src/core/router/hydrate"; 23 + import { makeEvent } from "./helpers"; 24 + 25 + const TEST_CONFIG = resolveConfig({ 26 + namespace: "com.example", 27 + collections: { 28 + "community.lexicon.calendar.event": { 29 + queryable: { 30 + mode: {}, 31 + name: {}, 32 + startsAt: { type: "range" }, 33 + }, 34 + searchable: ["name", "description"], 35 + relations: { 36 + rsvps: { 37 + collection: "community.lexicon.calendar.rsvp", 38 + groupBy: "status", 39 + groups: { 40 + interested: "community.lexicon.calendar.rsvp#interested", 41 + going: "community.lexicon.calendar.rsvp#going", 42 + notgoing: "community.lexicon.calendar.rsvp#notgoing", 43 + }, 44 + }, 45 + }, 46 + }, 47 + "community.lexicon.calendar.rsvp": { 48 + references: { 49 + event: { 50 + collection: "community.lexicon.calendar.event", 51 + field: "subject.uri", 52 + }, 53 + }, 54 + }, 55 + }, 56 + }); 57 + 58 + const PG_URL = process.env.TEST_DATABASE_URL; 59 + if (!PG_URL) { 60 + describe.skip("PostgreSQL e2e (TEST_DATABASE_URL not set)", () => { 61 + it("skipped", () => {}); 62 + }); 63 + } else { 64 + let pool: pg.Pool; 65 + let db: Database; 66 + 67 + beforeAll(async () => { 68 + pool = new pg.Pool({ connectionString: PG_URL }); 69 + await pool.query("SELECT 1"); 70 + db = createPostgresDatabase(pool); 71 + }); 72 + 73 + afterAll(async () => { 74 + await pool?.end(); 75 + }); 76 + 77 + async function resetSchema() { 78 + const tables = await pool.query( 79 + `SELECT tablename FROM pg_tables WHERE schemaname = 'public' 80 + AND (tablename LIKE 'records_%' OR tablename LIKE 'fts_%' 81 + OR tablename IN ('backfills', 'discovery', 'cursor', 'identities', 'feed_items', 'feed_backfills'))` 82 + ); 83 + for (const { tablename } of tables.rows) { 84 + await pool.query(`DROP TABLE IF EXISTS ${tablename} CASCADE`); 85 + } 86 + await initSchema(db, TEST_CONFIG); 87 + } 88 + 89 + beforeEach(resetSchema); 90 + 91 + // --- Schema --- 92 + 93 + describe("schema", () => { 94 + it("creates collection tables", async () => { 95 + const result = await pool.query( 96 + "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename LIKE 'records_%' ORDER BY tablename" 97 + ); 98 + const names = result.rows.map((r: any) => r.tablename); 99 + expect(names).toContain("records_community_lexicon_calendar_event"); 100 + expect(names).toContain("records_community_lexicon_calendar_rsvp"); 101 + }); 102 + 103 + it("creates indexes for queryable fields", async () => { 104 + const result = await pool.query( 105 + "SELECT indexname FROM pg_indexes WHERE schemaname = 'public' AND indexname LIKE 'idx_community_lexicon_calendar_event_%'" 106 + ); 107 + const names = result.rows.map((r: any) => r.indexname); 108 + expect(names).toContain("idx_community_lexicon_calendar_event_mode"); 109 + // PostgreSQL lowercases unquoted identifiers 110 + expect(names).toContain("idx_community_lexicon_calendar_event_startsat"); 111 + }); 112 + 113 + it("creates tsvector search column", async () => { 114 + const result = await pool.query( 115 + "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = 'records_community_lexicon_calendar_event' AND column_name = 'search_vector'" 116 + ); 117 + expect(result.rows).toHaveLength(1); 118 + expect(result.rows[0].data_type).toBe("tsvector"); 119 + }); 120 + 121 + it("is idempotent", async () => { 122 + await initSchema(db, TEST_CONFIG); 123 + await initSchema(db, TEST_CONFIG); 124 + const result = await pool.query( 125 + "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename LIKE 'records_%'" 126 + ); 127 + expect(result.rows.length).toBeGreaterThanOrEqual(2); 128 + }); 129 + }); 130 + 131 + // --- CRUD & Ingestion --- 132 + 133 + describe("ingestion", () => { 134 + it("inserts create events", async () => { 135 + await applyEvents(db, [ 136 + makeEvent({ record: { name: "Event 1", mode: "online" } }), 137 + ]); 138 + const result = await queryRecords(db, TEST_CONFIG, { 139 + collection: "community.lexicon.calendar.event", 140 + }); 141 + expect(result.records).toHaveLength(1); 142 + expect(JSON.parse(result.records[0].record!).name).toBe("Event 1"); 143 + }); 144 + 145 + it("upserts on conflict", async () => { 146 + const uri = "at://did:plc:test/community.lexicon.calendar.event/1"; 147 + await applyEvents(db, [ 148 + makeEvent({ uri, rkey: "1", cid: "cid1", record: { name: "V1", mode: "online" } }), 149 + ]); 150 + await applyEvents(db, [ 151 + makeEvent({ uri, rkey: "1", cid: "cid2", record: { name: "V2", mode: "online" } }), 152 + ]); 153 + const result = await queryRecords(db, TEST_CONFIG, { 154 + collection: "community.lexicon.calendar.event", 155 + }); 156 + expect(result.records).toHaveLength(1); 157 + expect(JSON.parse(result.records[0].record!).name).toBe("V2"); 158 + }); 159 + 160 + it("deletes events", async () => { 161 + const uri = "at://did:plc:test/community.lexicon.calendar.event/1"; 162 + await applyEvents(db, [ 163 + makeEvent({ uri, rkey: "1", record: { name: "Gone", mode: "online" } }), 164 + ]); 165 + await applyEvents(db, [ 166 + makeEvent({ uri, rkey: "1", operation: "delete", record: null, cid: null }), 167 + ]); 168 + const result = await queryRecords(db, TEST_CONFIG, { 169 + collection: "community.lexicon.calendar.event", 170 + }); 171 + expect(result.records).toHaveLength(0); 172 + }); 173 + 174 + it("does nothing for empty events", async () => { 175 + await applyEvents(db, []); 176 + const cursor = await getLastCursor(db); 177 + expect(cursor).toBeNull(); 178 + }); 179 + }); 180 + 181 + // --- Counts --- 182 + 183 + describe("relation counts", () => { 184 + const eventUri = "at://did:plc:host/community.lexicon.calendar.event/evt1"; 185 + 186 + beforeEach(async () => { 187 + await applyEvents(db, [ 188 + makeEvent({ uri: eventUri, rkey: "evt1", record: { name: "Counted Event", mode: "online" } }), 189 + ]); 190 + }); 191 + 192 + it("increments counts on RSVP create", async () => { 193 + await applyEvents(db, [ 194 + makeEvent({ 195 + uri: "at://did:plc:u1/community.lexicon.calendar.rsvp/r1", 196 + did: "did:plc:u1", 197 + collection: "community.lexicon.calendar.rsvp", 198 + rkey: "r1", 199 + record: { subject: { uri: eventUri }, status: "community.lexicon.calendar.rsvp#going" }, 200 + time_us: 2000000, 201 + }), 202 + makeEvent({ 203 + uri: "at://did:plc:u2/community.lexicon.calendar.rsvp/r2", 204 + did: "did:plc:u2", 205 + collection: "community.lexicon.calendar.rsvp", 206 + rkey: "r2", 207 + record: { subject: { uri: eventUri }, status: "community.lexicon.calendar.rsvp#going" }, 208 + time_us: 3000000, 209 + }), 210 + makeEvent({ 211 + uri: "at://did:plc:u3/community.lexicon.calendar.rsvp/r3", 212 + did: "did:plc:u3", 213 + collection: "community.lexicon.calendar.rsvp", 214 + rkey: "r3", 215 + record: { subject: { uri: eventUri }, status: "community.lexicon.calendar.rsvp#interested" }, 216 + time_us: 4000000, 217 + }), 218 + ], TEST_CONFIG); 219 + 220 + const result = await queryRecords(db, TEST_CONFIG, { 221 + collection: "community.lexicon.calendar.event", 222 + }); 223 + expect(result.records[0].counts?.["community.lexicon.calendar.rsvp"]).toBe(3); 224 + expect(result.records[0].counts?.["community.lexicon.calendar.rsvp#going"]).toBe(2); 225 + expect(result.records[0].counts?.["community.lexicon.calendar.rsvp#interested"]).toBe(1); 226 + }); 227 + 228 + it("decrements counts on RSVP delete", async () => { 229 + const rsvpUri = "at://did:plc:u1/community.lexicon.calendar.rsvp/r1"; 230 + await applyEvents(db, [ 231 + makeEvent({ 232 + uri: rsvpUri, 233 + did: "did:plc:u1", 234 + collection: "community.lexicon.calendar.rsvp", 235 + rkey: "r1", 236 + record: { subject: { uri: eventUri }, status: "community.lexicon.calendar.rsvp#going" }, 237 + time_us: 2000000, 238 + }), 239 + ], TEST_CONFIG); 240 + 241 + // Verify count is 1 242 + let result = await queryRecords(db, TEST_CONFIG, { 243 + collection: "community.lexicon.calendar.event", 244 + }); 245 + expect(result.records[0].counts?.["community.lexicon.calendar.rsvp"]).toBe(1); 246 + 247 + // Delete the RSVP 248 + await applyEvents(db, [ 249 + makeEvent({ 250 + uri: rsvpUri, 251 + did: "did:plc:u1", 252 + collection: "community.lexicon.calendar.rsvp", 253 + rkey: "r1", 254 + operation: "delete", 255 + record: null, 256 + cid: null, 257 + time_us: 3000000, 258 + }), 259 + ], TEST_CONFIG); 260 + 261 + result = await queryRecords(db, TEST_CONFIG, { 262 + collection: "community.lexicon.calendar.event", 263 + }); 264 + // Count should be 0 (absent from counts map) 265 + expect(result.records[0].counts?.["community.lexicon.calendar.rsvp"]).toBeUndefined(); 266 + }); 267 + }); 268 + 269 + // --- Querying --- 270 + 271 + describe("queries", () => { 272 + beforeEach(async () => { 273 + await applyEvents(db, [ 274 + makeEvent({ uri: "at://a/community.lexicon.calendar.event/1", rkey: "1", did: "did:plc:alice", record: { name: "Alpha", mode: "online", startsAt: "2026-04-01T10:00:00Z" }, time_us: 1000 }), 275 + makeEvent({ uri: "at://a/community.lexicon.calendar.event/2", rkey: "2", did: "did:plc:bob", record: { name: "Beta", mode: "in-person", startsAt: "2026-05-01T10:00:00Z" }, time_us: 2000 }), 276 + makeEvent({ uri: "at://a/community.lexicon.calendar.event/3", rkey: "3", did: "did:plc:alice", record: { name: "Gamma", mode: "online", startsAt: "2026-06-01T10:00:00Z" }, time_us: 3000 }), 277 + makeEvent({ uri: "at://a/community.lexicon.calendar.event/4", rkey: "4", did: "did:plc:carol", record: { name: "Delta", mode: "hybrid", startsAt: "2026-03-01T10:00:00Z" }, time_us: 4000 }), 278 + ]); 279 + }); 280 + 281 + it("returns records ordered by time_us desc", async () => { 282 + const result = await queryRecords(db, TEST_CONFIG, { 283 + collection: "community.lexicon.calendar.event", 284 + }); 285 + expect(result.records).toHaveLength(4); 286 + expect(result.records[0].rkey).toBe("4"); // highest time_us 287 + expect(result.records[3].rkey).toBe("1"); // lowest time_us 288 + }); 289 + 290 + it("filters by did", async () => { 291 + const result = await queryRecords(db, TEST_CONFIG, { 292 + collection: "community.lexicon.calendar.event", 293 + did: "did:plc:alice", 294 + }); 295 + expect(result.records).toHaveLength(2); 296 + }); 297 + 298 + it("filters by equality", async () => { 299 + const result = await queryRecords(db, TEST_CONFIG, { 300 + collection: "community.lexicon.calendar.event", 301 + filters: { mode: "online" }, 302 + }); 303 + expect(result.records).toHaveLength(2); 304 + }); 305 + 306 + it("filters by range", async () => { 307 + const result = await queryRecords(db, TEST_CONFIG, { 308 + collection: "community.lexicon.calendar.event", 309 + rangeFilters: { 310 + startsAt: { min: "2026-04-01T00:00:00Z", max: "2026-05-31T23:59:59Z" }, 311 + }, 312 + }); 313 + expect(result.records).toHaveLength(2); 314 + }); 315 + 316 + it("respects limit", async () => { 317 + const result = await queryRecords(db, TEST_CONFIG, { 318 + collection: "community.lexicon.calendar.event", 319 + limit: 2, 320 + }); 321 + expect(result.records).toHaveLength(2); 322 + expect(result.cursor).toBeDefined(); 323 + }); 324 + 325 + it("paginates with cursor", async () => { 326 + const page1 = await queryRecords(db, TEST_CONFIG, { 327 + collection: "community.lexicon.calendar.event", 328 + limit: 2, 329 + }); 330 + expect(page1.records).toHaveLength(2); 331 + expect(page1.cursor).toBeDefined(); 332 + 333 + const page2 = await queryRecords(db, TEST_CONFIG, { 334 + collection: "community.lexicon.calendar.event", 335 + limit: 2, 336 + cursor: page1.cursor, 337 + }); 338 + expect(page2.records).toHaveLength(2); 339 + 340 + // No overlap 341 + const page1Uris = page1.records.map(r => r.uri); 342 + const page2Uris = page2.records.map(r => r.uri); 343 + expect(page1Uris.filter(u => page2Uris.includes(u))).toHaveLength(0); 344 + }); 345 + 346 + it("sorts by record field asc", async () => { 347 + const result = await queryRecords(db, TEST_CONFIG, { 348 + collection: "community.lexicon.calendar.event", 349 + sort: { recordField: "startsAt", direction: "asc" }, 350 + }); 351 + const dates = result.records.map(r => JSON.parse(r.record!).startsAt); 352 + expect(dates).toEqual([...dates].sort()); 353 + }); 354 + 355 + it("sorts by record field desc", async () => { 356 + const result = await queryRecords(db, TEST_CONFIG, { 357 + collection: "community.lexicon.calendar.event", 358 + sort: { recordField: "startsAt", direction: "desc" }, 359 + }); 360 + const dates = result.records.map(r => JSON.parse(r.record!).startsAt); 361 + expect(dates).toEqual([...dates].sort().reverse()); 362 + }); 363 + 364 + it("returns no cursor when all results fit", async () => { 365 + const result = await queryRecords(db, TEST_CONFIG, { 366 + collection: "community.lexicon.calendar.event", 367 + limit: 100, 368 + }); 369 + expect(result.cursor).toBeUndefined(); 370 + }); 371 + }); 372 + 373 + // --- FTS --- 374 + 375 + describe("full-text search", () => { 376 + beforeEach(async () => { 377 + await applyEvents(db, [ 378 + makeEvent({ uri: "at://a/community.lexicon.calendar.event/1", rkey: "1", record: { name: "Rust Meetup", description: "Systems programming", mode: "online" }, time_us: 1000 }), 379 + makeEvent({ uri: "at://a/community.lexicon.calendar.event/2", rkey: "2", record: { name: "TypeScript Workshop", description: "Learn web dev", mode: "online" }, time_us: 2000 }), 380 + makeEvent({ uri: "at://a/community.lexicon.calendar.event/3", rkey: "3", record: { name: "Go Conference", description: "Concurrency and systems", mode: "in-person" }, time_us: 3000 }), 381 + ], TEST_CONFIG); 382 + }); 383 + 384 + it("finds records matching a search term", async () => { 385 + const result = await queryRecords(db, TEST_CONFIG, { 386 + collection: "community.lexicon.calendar.event", 387 + search: "Rust", 388 + }); 389 + expect(result.records).toHaveLength(1); 390 + expect(JSON.parse(result.records[0].record!).name).toBe("Rust Meetup"); 391 + }); 392 + 393 + it("searches across multiple fields", async () => { 394 + const result = await queryRecords(db, TEST_CONFIG, { 395 + collection: "community.lexicon.calendar.event", 396 + search: "systems", 397 + }); 398 + // "systems" appears in description of Rust Meetup and Go Conference 399 + expect(result.records).toHaveLength(2); 400 + }); 401 + 402 + it("returns nothing for non-matching search", async () => { 403 + const result = await queryRecords(db, TEST_CONFIG, { 404 + collection: "community.lexicon.calendar.event", 405 + search: "python", 406 + }); 407 + expect(result.records).toHaveLength(0); 408 + }); 409 + 410 + it("combines search with filters", async () => { 411 + const result = await queryRecords(db, TEST_CONFIG, { 412 + collection: "community.lexicon.calendar.event", 413 + search: "systems", 414 + filters: { mode: "in-person" }, 415 + }); 416 + expect(result.records).toHaveLength(1); 417 + expect(JSON.parse(result.records[0].record!).name).toBe("Go Conference"); 418 + }); 419 + }); 420 + 421 + // --- Hydration --- 422 + 423 + describe("hydration", () => { 424 + const eventUri1 = "at://did:plc:host/community.lexicon.calendar.event/evt1"; 425 + const eventUri2 = "at://did:plc:host/community.lexicon.calendar.event/evt2"; 426 + 427 + beforeEach(async () => { 428 + await applyEvents(db, [ 429 + makeEvent({ uri: eventUri1, rkey: "evt1", record: { name: "Event 1", mode: "online" }, time_us: 1000 }), 430 + makeEvent({ uri: eventUri2, rkey: "evt2", record: { name: "Event 2", mode: "online" }, time_us: 2000 }), 431 + ]); 432 + await applyEvents(db, [ 433 + makeEvent({ 434 + uri: "at://did:plc:u1/community.lexicon.calendar.rsvp/r1", 435 + did: "did:plc:u1", 436 + collection: "community.lexicon.calendar.rsvp", 437 + rkey: "r1", 438 + record: { subject: { uri: eventUri1 }, status: "community.lexicon.calendar.rsvp#going" }, 439 + time_us: 3000, 440 + }), 441 + makeEvent({ 442 + uri: "at://did:plc:u2/community.lexicon.calendar.rsvp/r2", 443 + did: "did:plc:u2", 444 + collection: "community.lexicon.calendar.rsvp", 445 + rkey: "r2", 446 + record: { subject: { uri: eventUri1 }, status: "community.lexicon.calendar.rsvp#interested" }, 447 + time_us: 4000, 448 + }), 449 + ], TEST_CONFIG); 450 + }); 451 + 452 + it("hydrates related records", async () => { 453 + const events = await queryRecords(db, TEST_CONFIG, { 454 + collection: "community.lexicon.calendar.event", 455 + }); 456 + const hydrated = await resolveHydrates( 457 + db, 458 + TEST_CONFIG.collections["community.lexicon.calendar.event"].relations!, 459 + { rsvps: 10 }, 460 + events.records 461 + ); 462 + // Event 1 has 2 RSVPs in 2 groups 463 + expect(hydrated[eventUri1]).toBeDefined(); 464 + const rsvps = hydrated[eventUri1].rsvps as Record<string, any[]>; 465 + const totalRsvps = Object.values(rsvps).flat().length; 466 + expect(totalRsvps).toBe(2); 467 + }); 468 + 469 + it("resolves references (child → parent)", async () => { 470 + const rsvps = await queryRecords(db, TEST_CONFIG, { 471 + collection: "community.lexicon.calendar.rsvp", 472 + }); 473 + const refs = await resolveReferences( 474 + db, 475 + TEST_CONFIG.collections["community.lexicon.calendar.rsvp"].references!, 476 + new Set(["event"]), 477 + rsvps.records 478 + ); 479 + // Both RSVPs point at eventUri1 480 + for (const rsvp of rsvps.records) { 481 + expect(refs[rsvp.uri]?.event).toBeDefined(); 482 + expect(refs[rsvp.uri].event.uri).toBe(eventUri1); 483 + } 484 + }); 485 + }); 486 + 487 + // --- Cursor --- 488 + 489 + describe("cursor persistence", () => { 490 + it("saves and retrieves cursor", async () => { 491 + await saveCursor(db, 12345); 492 + expect(await getLastCursor(db)).toBe(12345); 493 + }); 494 + 495 + it("updates existing cursor", async () => { 496 + await saveCursor(db, 100); 497 + await saveCursor(db, 200); 498 + expect(await getLastCursor(db)).toBe(200); 499 + }); 500 + 501 + it("returns null when no cursor", async () => { 502 + expect(await getLastCursor(db)).toBeNull(); 503 + }); 504 + }); 505 + }

+145

tests/postgres.test.ts

··· 1 + import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; 2 + import pg from "pg"; 3 + import { createPostgresDatabase } from "../src/adapters/postgres"; 4 + import { initSchema } from "../src/core/db/schema"; 5 + import { applyEvents, queryRecords, getLastCursor, saveCursor } from "../src/core/db/records"; 6 + import { resolveConfig } from "../src/core/types"; 7 + import { makeEvent } from "./helpers"; 8 + 9 + const TEST_CONFIG = resolveConfig({ 10 + namespace: "com.example", 11 + collections: { 12 + "community.lexicon.calendar.event": { 13 + queryable: { mode: {}, name: {}, startsAt: { type: "range" } }, 14 + searchable: ["name", "description"], 15 + relations: { 16 + rsvps: { 17 + collection: "community.lexicon.calendar.rsvp", 18 + groupBy: "status", 19 + groups: { 20 + going: "community.lexicon.calendar.rsvp#going", 21 + }, 22 + }, 23 + }, 24 + }, 25 + "community.lexicon.calendar.rsvp": { 26 + references: { 27 + event: { 28 + collection: "community.lexicon.calendar.event", 29 + field: "subject.uri", 30 + }, 31 + }, 32 + }, 33 + }, 34 + }); 35 + 36 + const PG_URL = process.env.TEST_DATABASE_URL; 37 + if (!PG_URL) { 38 + describe.skip("PostgreSQL adapter (TEST_DATABASE_URL not set)", () => { 39 + it("skipped", () => {}); 40 + }); 41 + } else { 42 + let pool: pg.Pool; 43 + let db: ReturnType<typeof createPostgresDatabase>; 44 + 45 + beforeAll(async () => { 46 + pool = new pg.Pool({ connectionString: PG_URL }); 47 + await pool.query("SELECT 1"); 48 + db = createPostgresDatabase(pool); 49 + }); 50 + 51 + afterAll(async () => { 52 + await pool?.end(); 53 + }); 54 + 55 + beforeEach(async () => { 56 + // Drop all contrail tables 57 + const tables = await pool.query( 58 + `SELECT tablename FROM pg_tables WHERE schemaname = 'public' 59 + AND (tablename LIKE 'records_%' OR tablename LIKE 'fts_%' 60 + OR tablename IN ('backfills', 'discovery', 'cursor', 'identities', 'feed_items', 'feed_backfills'))` 61 + ); 62 + for (const { tablename } of tables.rows) { 63 + await pool.query(`DROP TABLE IF EXISTS ${tablename} CASCADE`); 64 + } 65 + await initSchema(db, TEST_CONFIG); 66 + }); 67 + 68 + describe("PostgreSQL adapter", () => { 69 + it("initializes schema", async () => { 70 + const result = await pool.query( 71 + "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename LIKE 'records_%'" 72 + ); 73 + expect(result.rows.length).toBeGreaterThanOrEqual(2); 74 + }); 75 + 76 + it("saves and retrieves cursor", async () => { 77 + await saveCursor(db, 12345); 78 + expect(await getLastCursor(db)).toBe(12345); 79 + }); 80 + 81 + it("inserts and queries records", async () => { 82 + await applyEvents(db, [makeEvent({ record: { name: "PG Test", mode: "online" } })]); 83 + const result = await queryRecords(db, TEST_CONFIG, { 84 + collection: "community.lexicon.calendar.event", 85 + }); 86 + expect(result.records).toHaveLength(1); 87 + }); 88 + 89 + it("filters by json field", async () => { 90 + await applyEvents(db, [ 91 + makeEvent({ uri: "at://a/community.lexicon.calendar.event/1", rkey: "1", record: { name: "A", mode: "online" }, time_us: 2000 }), 92 + makeEvent({ uri: "at://a/community.lexicon.calendar.event/2", rkey: "2", record: { name: "B", mode: "in-person" }, time_us: 1000 }), 93 + ]); 94 + const result = await queryRecords(db, TEST_CONFIG, { 95 + collection: "community.lexicon.calendar.event", 96 + filters: { mode: "online" }, 97 + }); 98 + expect(result.records).toHaveLength(1); 99 + }); 100 + 101 + it("counts relations", async () => { 102 + const eventUri = "at://did:plc:test/community.lexicon.calendar.event/evt1"; 103 + await applyEvents(db, [makeEvent({ uri: eventUri, rkey: "evt1" })]); 104 + await applyEvents(db, [ 105 + makeEvent({ 106 + uri: "at://did:plc:user1/community.lexicon.calendar.rsvp/r1", 107 + did: "did:plc:user1", 108 + collection: "community.lexicon.calendar.rsvp", 109 + rkey: "r1", 110 + record: { subject: { uri: eventUri }, status: "community.lexicon.calendar.rsvp#going" }, 111 + time_us: 2000000, 112 + }), 113 + ], TEST_CONFIG); 114 + 115 + const result = await queryRecords(db, TEST_CONFIG, { 116 + collection: "community.lexicon.calendar.event", 117 + }); 118 + expect(result.records[0].counts?.["community.lexicon.calendar.rsvp"]).toBe(1); 119 + }); 120 + 121 + it("full-text search works via tsvector", async () => { 122 + await applyEvents(db, [ 123 + makeEvent({ 124 + uri: "at://a/community.lexicon.calendar.event/1", 125 + rkey: "1", 126 + record: { name: "Rust Meetup", description: "A gathering of Rustaceans", mode: "online" }, 127 + time_us: 2000, 128 + }), 129 + makeEvent({ 130 + uri: "at://a/community.lexicon.calendar.event/2", 131 + rkey: "2", 132 + record: { name: "TypeScript Workshop", description: "Learn TS", mode: "online" }, 133 + time_us: 1000, 134 + }), 135 + ], TEST_CONFIG); 136 + 137 + const result = await queryRecords(db, TEST_CONFIG, { 138 + collection: "community.lexicon.calendar.event", 139 + search: "Rust", 140 + }); 141 + expect(result.records).toHaveLength(1); 142 + expect(JSON.parse(result.records[0].record!).name).toBe("Rust Meetup"); 143 + }); 144 + }); 145 + }

+112

examples/postgres/README.md

··· 1 + # Contrail — PostgreSQL Example 2 + 3 + A complete example of using Contrail to index AT Protocol calendar events and RSVPs with PostgreSQL. Includes persistent Jetstream ingestion (long-running listener) and user discovery/backfill. 4 + 5 + ## Setup 6 + 7 + ```bash 8 + # Copy this folder to a new project 9 + cp -r examples/postgres my-contrail-app 10 + cd my-contrail-app 11 + 12 + # Install dependencies 13 + npm install 14 + ``` 15 + 16 + > **Note:** The `contrail` dependency in `package.json` points at `github:flo-bit/contrail`. 17 + > If you're using a fork with PostgreSQL support that hasn't been merged yet, update the 18 + > dependency to point at your fork's branch: 19 + > 20 + > ```bash 21 + > npm install github:your-username/contrail#your-branch 22 + > ``` 23 + > 24 + > Or install from a local checkout: 25 + > 26 + > ```bash 27 + > npm install /path/to/your/contrail 28 + > ``` 29 + 30 + ### Start PostgreSQL 31 + 32 + **Option A: Docker (recommended)** 33 + 34 + ```bash 35 + docker compose up -d 36 + ``` 37 + 38 + This starts PostgreSQL on port 5433 (to avoid conflicts with a local instance) with a `contrail` database ready to use. Override the port with `PG_PORT=5432 docker compose up -d`. 39 + 40 + **Option B: Native PostgreSQL** 41 + 42 + ```bash 43 + createdb contrail 44 + ``` 45 + 46 + Set `DATABASE_URL` to point at your local instance: 47 + 48 + ```bash 49 + export DATABASE_URL="postgresql://user:password@localhost:5432/contrail" 50 + ``` 51 + 52 + ## Configure 53 + 54 + Edit `config.ts` to define your collections, queryable fields, relations, and references. See the [Contrail README](https://github.com/flo-bit/contrail) for all options. 55 + 56 + ## Run 57 + 58 + ### 1. Discover users and backfill records 59 + 60 + ```bash 61 + npm run sync 62 + ``` 63 + 64 + This finds users from ATProto relays and backfills their existing records from PDS. Safe to interrupt and restart — progress is saved per-DID in the database. 65 + 66 + ### 2. Start persistent ingestion 67 + 68 + ```bash 69 + npm run ingest 70 + ``` 71 + 72 + This opens a long-lived Jetstream connection and continuously indexes new records as they appear on the network. Events are batched and flushed every 5 seconds (or every 50 events, whichever comes first). Handles reconnection automatically. 73 + 74 + Press `Ctrl+C` for graceful shutdown — the current batch is flushed and the cursor is saved so the next run picks up where it left off. 75 + 76 + ### 3. Serve the XRPC API 77 + 78 + ```bash 79 + npm run serve 80 + ``` 81 + 82 + Your XRPC API is now available at `http://localhost:3000`: 83 + 84 + ``` 85 + # List events sorted by RSVP count 86 + /xrpc/community.lexicon.calendar.event.listRecords?sort=rsvpsCount 87 + 88 + # Upcoming events with 10+ going RSVPs 89 + /xrpc/community.lexicon.calendar.event.listRecords?startsAtMin=2026-03-16&rsvpsGoingCountMin=10 90 + 91 + # Single event with hydrated RSVPs and profiles 92 + /xrpc/community.lexicon.calendar.event.getRecord?uri=at://...&hydrateRsvps=10&profiles=true 93 + 94 + # Search events 95 + /xrpc/community.lexicon.calendar.event.listRecords?search=meetup 96 + 97 + # RSVPs for a specific event 98 + /xrpc/community.lexicon.calendar.rsvp.listRecords?subjectUri=at://... 99 + ``` 100 + 101 + ## Running everything together 102 + 103 + In production you'd typically run sync once (or periodically), then keep `ingest` and `serve` running as separate processes: 104 + 105 + ```bash 106 + # Initial sync (run once, or periodically to discover new users) 107 + npm run sync 108 + 109 + # In separate terminals (or use a process manager) 110 + npm run ingest 111 + npm run serve 112 + ```

+42

examples/postgres/config.ts

··· 1 + import type { ContrailConfig } from "@atmo-dev/contrail"; 2 + 3 + export const config: ContrailConfig = { 4 + namespace: "rsvp.atmo", 5 + collections: { 6 + "community.lexicon.calendar.event": { 7 + queryable: { 8 + mode: {}, 9 + name: {}, 10 + status: {}, 11 + startsAt: { type: "range" }, 12 + endsAt: { type: "range" }, 13 + createdAt: { type: "range" }, 14 + }, 15 + searchable: ["name", "description"], 16 + relations: { 17 + rsvps: { 18 + collection: "community.lexicon.calendar.rsvp", 19 + groupBy: "status", 20 + count: true, 21 + groups: { 22 + interested: "community.lexicon.calendar.rsvp#interested", 23 + going: "community.lexicon.calendar.rsvp#going", 24 + notgoing: "community.lexicon.calendar.rsvp#notgoing", 25 + }, 26 + }, 27 + }, 28 + }, 29 + "community.lexicon.calendar.rsvp": { 30 + queryable: { 31 + status: {}, 32 + "subject.uri": {}, 33 + }, 34 + references: { 35 + event: { 36 + collection: "community.lexicon.calendar.event", 37 + field: "subject.uri", 38 + }, 39 + }, 40 + }, 41 + }, 42 + };

+14

examples/postgres/docker-compose.yml

··· 1 + services: 2 + postgres: 3 + image: postgres:17 4 + environment: 5 + POSTGRES_DB: contrail 6 + POSTGRES_USER: contrail 7 + POSTGRES_PASSWORD: contrail 8 + ports: 9 + - "${PG_PORT:-5433}:5432" 10 + volumes: 11 + - pgdata:/var/lib/postgresql/data 12 + 13 + volumes: 14 + pgdata:

+42

examples/postgres/ingest.ts

··· 1 + /** 2 + * Persistent Jetstream ingestion against PostgreSQL. 3 + * 4 + * Opens a long-lived Jetstream connection and continuously indexes new records. 5 + * Events are batched and flushed periodically. Handles reconnection automatically. 6 + * 7 + * Usage: 8 + * DATABASE_URL="postgresql://contrail:contrail@localhost:5432/contrail" npx tsx ingest.ts 9 + */ 10 + import pg from "pg"; 11 + import { Contrail } from "@atmo-dev/contrail"; 12 + import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; 13 + import { config } from "./config"; 14 + 15 + const DATABASE_URL = 16 + process.env.DATABASE_URL ?? 17 + "postgresql://contrail:contrail@localhost:5433/contrail"; 18 + 19 + async function main() { 20 + const pool = new pg.Pool({ connectionString: DATABASE_URL }); 21 + const db = createPostgresDatabase(pool); 22 + const contrail = new Contrail({ ...config, db }); 23 + 24 + const controller = new AbortController(); 25 + const shutdown = () => { 26 + console.log("\nShutting down gracefully..."); 27 + controller.abort(); 28 + }; 29 + process.on("SIGTERM", shutdown); 30 + process.on("SIGINT", shutdown); 31 + 32 + console.log("Starting persistent ingestion..."); 33 + await contrail.runPersistent({ signal: controller.signal }); 34 + 35 + await pool.end(); 36 + console.log("Done."); 37 + } 38 + 39 + main().catch((err) => { 40 + console.error(err); 41 + process.exit(1); 42 + });

+20

examples/postgres/package.json

··· 1 + { 2 + "name": "contrail-postgres-example", 3 + "version": "0.0.1", 4 + "private": true, 5 + "type": "module", 6 + "scripts": { 7 + "sync": "tsx sync.ts", 8 + "ingest": "tsx ingest.ts", 9 + "serve": "tsx serve.ts" 10 + }, 11 + "dependencies": { 12 + "@atmo-dev/contrail": "^0.0.5", 13 + "pg": "^8.20.0" 14 + }, 15 + "devDependencies": { 16 + "@types/pg": "^8.20.0", 17 + "tsx": "^4.21.0", 18 + "typescript": "^5.7.3" 19 + } 20 + }

+61

examples/postgres/serve.ts

··· 1 + /** 2 + * Serve the Contrail XRPC API over HTTP using PostgreSQL. 3 + * 4 + * Usage: 5 + * DATABASE_URL="postgresql://contrail:contrail@localhost:5432/contrail" npx tsx serve.ts 6 + */ 7 + import pg from "pg"; 8 + import { createServer } from "node:http"; 9 + import { Contrail } from "@atmo-dev/contrail"; 10 + import { createHandler } from "@atmo-dev/contrail/server"; 11 + import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; 12 + import { config } from "./config"; 13 + 14 + const DATABASE_URL = 15 + process.env.DATABASE_URL ?? 16 + "postgresql://contrail:contrail@localhost:5433/contrail"; 17 + const PORT = parseInt(process.env.PORT ?? "3000", 10); 18 + 19 + async function main() { 20 + const pool = new pg.Pool({ connectionString: DATABASE_URL }); 21 + const db = createPostgresDatabase(pool); 22 + const contrail = new Contrail({ ...config, db }); 23 + 24 + await contrail.init(); 25 + 26 + const handle = createHandler(contrail); 27 + 28 + const server = createServer(async (req, res) => { 29 + const url = new URL(req.url!, `http://localhost:${PORT}`); 30 + const request = new Request(url.toString(), { 31 + method: req.method, 32 + headers: Object.entries(req.headers).reduce( 33 + (h, [k, v]) => { 34 + if (v) h[k] = Array.isArray(v) ? v.join(", ") : v; 35 + return h; 36 + }, 37 + {} as Record<string, string>, 38 + ), 39 + }); 40 + 41 + const response = await handle(request); 42 + res.writeHead(response.status, Object.fromEntries(response.headers)); 43 + res.end(await response.text()); 44 + }); 45 + 46 + server.listen(PORT, () => { 47 + console.log(`Contrail XRPC API listening on http://localhost:${PORT}`); 48 + }); 49 + 50 + const shutdown = () => { 51 + console.log("\nShutting down..."); 52 + server.close(() => pool.end().then(() => process.exit(0))); 53 + }; 54 + process.on("SIGTERM", shutdown); 55 + process.on("SIGINT", shutdown); 56 + } 57 + 58 + main().catch((err) => { 59 + console.error(err); 60 + process.exit(1); 61 + });

+71

examples/postgres/sync.ts

··· 1 + /** 2 + * Discover users and backfill their records into PostgreSQL. 3 + * 4 + * Safe to kill at any point — discovery and backfill cursors are saved 5 + * per-DID in the database. Restarting resumes from where it left off. 6 + * 7 + * Usage: 8 + * DATABASE_URL="postgresql://contrail:contrail@localhost:5432/contrail" npx tsx sync.ts 9 + */ 10 + import pg from "pg"; 11 + import { Contrail } from "@atmo-dev/contrail"; 12 + import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; 13 + import { config } from "./config"; 14 + 15 + const DATABASE_URL = 16 + process.env.DATABASE_URL ?? 17 + "postgresql://contrail:contrail@localhost:5433/contrail"; 18 + 19 + function elapsed(start: number): string { 20 + const ms = Date.now() - start; 21 + if (ms < 1000) return `${ms}ms`; 22 + if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`; 23 + const mins = Math.floor(ms / 60_000); 24 + const secs = ((ms % 60_000) / 1000).toFixed(0); 25 + return `${mins}m ${secs}s`; 26 + } 27 + 28 + async function main() { 29 + const pool = new pg.Pool({ connectionString: DATABASE_URL }); 30 + const db = createPostgresDatabase(pool); 31 + const contrail = new Contrail({ ...config, db }); 32 + const syncStart = Date.now(); 33 + 34 + console.log(`=== Sync (PostgreSQL) ===\n`); 35 + 36 + await contrail.init(); 37 + 38 + console.log("--- Discovery ---"); 39 + const discoveryStart = Date.now(); 40 + const discovered = await contrail.discover(); 41 + console.log( 42 + ` Done: ${discovered.length} users in ${elapsed(discoveryStart)}\n`, 43 + ); 44 + 45 + console.log("--- Backfill ---"); 46 + const backfillStart = Date.now(); 47 + const total = await contrail.backfill({ 48 + concurrency: 10, 49 + onProgress: ({ records, usersComplete, usersTotal, usersFailed }) => { 50 + const secs = (Date.now() - backfillStart) / 1000; 51 + const rate = secs > 0 ? Math.round(records / secs) : 0; 52 + const failStr = usersFailed > 0 ? ` | ${usersFailed} failed` : ""; 53 + process.stdout.write( 54 + `\r ${records} records | ${usersComplete}/${usersTotal} users | ${rate}/s | ${elapsed(backfillStart)}${failStr} `, 55 + ); 56 + }, 57 + }); 58 + process.stdout.write("\n"); 59 + console.log(` Done: ${total} records in ${elapsed(backfillStart)}\n`); 60 + 61 + console.log(`=== Finished in ${elapsed(syncStart)} ===`); 62 + console.log(` Discovered: ${discovered.length} users`); 63 + console.log(` Backfilled: ${total} records`); 64 + 65 + await pool.end(); 66 + } 67 + 68 + main().catch((err) => { 69 + console.error(err); 70 + process.exit(1); 71 + });

+13

examples/postgres/tsconfig.json

··· 1 + { 2 + "compilerOptions": { 3 + "target": "ES2022", 4 + "module": "ES2022", 5 + "moduleResolution": "bundler", 6 + "lib": ["ES2022"], 7 + "strict": true, 8 + "noEmit": true, 9 + "skipLibCheck": true, 10 + "isolatedModules": true 11 + }, 12 + "include": ["."] 13 + }

+93

src/adapters/postgres.ts

··· 1 + import pg from "pg"; 2 + import type { Database, Statement } from "../core/types"; 3 + import { postgresDialect } from "../core/dialect"; 4 + 5 + /** Internal interface for statements that can run on a specific client */ 6 + interface PgStatement extends Statement { 7 + /** Execute on a specific client (used by batch for transaction isolation) */ 8 + _runOn(client: pg.PoolClient): Promise<any>; 9 + } 10 + 11 + /** Column names known to be BIGINT — PostgreSQL returns these as strings */ 12 + const BIGINT_COLUMNS = new Set(["time_us", "indexed_at", "resolved_at"]); 13 + 14 + function normalizeRow(row: any): any { 15 + if (!row) return row; 16 + if (typeof row.record === "object" && row.record !== null) { 17 + row.record = JSON.stringify(row.record); 18 + } 19 + for (const col of BIGINT_COLUMNS) { 20 + if (typeof row[col] === "string") row[col] = Number(row[col]); 21 + } 22 + return row; 23 + } 24 + 25 + export function createPostgresDatabase(pool: pg.Pool): Database { 26 + function rewritePlaceholders(sql: string): string { 27 + let idx = 0; 28 + let inString = false; 29 + let result = ""; 30 + for (let i = 0; i < sql.length; i++) { 31 + const ch = sql[i]; 32 + if (ch === "'" && sql[i - 1] !== "\\") { 33 + inString = !inString; 34 + result += ch; 35 + } else if (ch === "?" && !inString) { 36 + result += `$${++idx}`; 37 + } else { 38 + result += ch; 39 + } 40 + } 41 + return result; 42 + } 43 + 44 + function wrapStatement(sql: string, boundValues: any[] = []): PgStatement { 45 + const pgSql = rewritePlaceholders(sql); 46 + 47 + return { 48 + bind(...values: any[]): PgStatement { 49 + return wrapStatement(sql, values); 50 + }, 51 + async run() { 52 + const result = await pool.query(pgSql, boundValues); 53 + return { changes: result.rowCount }; 54 + }, 55 + async _runOn(client: pg.PoolClient) { 56 + const result = await client.query(pgSql, boundValues); 57 + return { changes: result.rowCount }; 58 + }, 59 + async all<T>() { 60 + const result = await pool.query(pgSql, boundValues); 61 + return { results: result.rows.map(normalizeRow) as T[] }; 62 + }, 63 + async first<T>() { 64 + const result = await pool.query(pgSql, boundValues); 65 + return result.rows[0] ? (normalizeRow(result.rows[0]) as T) : null; 66 + }, 67 + }; 68 + } 69 + 70 + return { 71 + prepare(sql: string): Statement { 72 + return wrapStatement(sql); 73 + }, 74 + async batch(stmts: Statement[]): Promise<any[]> { 75 + const client = await pool.connect(); 76 + try { 77 + await client.query("BEGIN"); 78 + const results: any[] = []; 79 + for (const stmt of stmts) { 80 + results.push(await (stmt as PgStatement)._runOn(client)); 81 + } 82 + await client.query("COMMIT"); 83 + return results; 84 + } catch (e) { 85 + await client.query("ROLLBACK"); 86 + throw e; 87 + } finally { 88 + client.release(); 89 + } 90 + }, 91 + dialect: postgresDialect, 92 + }; 93 + }

+2

src/adapters/sqlite.ts

··· 1 1 import { DatabaseSync } from "node:sqlite"; 2 2 import type { Database, Statement } from "../core/types"; 3 + import { sqliteDialect } from "../core/dialect"; 3 4 4 5 export function createSqliteDatabase(path: string): Database { 5 6 const raw = new DatabaseSync(path); ··· 33 34 } 34 35 return results; 35 36 }, 37 + dialect: sqliteDialect, 36 38 }; 37 39 }

+127

src/core/dialect.ts

··· 1 + /** Get the dialect from a Database, defaulting to SQLite (for D1 compatibility) */ 2 + export function getDialect(db: { dialect?: SqlDialect }): SqlDialect { 3 + return db.dialect ?? sqliteDialect; 4 + } 5 + 6 + const SAFE_FIELD = /^[a-zA-Z0-9_.]+$/; 7 + 8 + function assertSafeField(field: string): void { 9 + if (!SAFE_FIELD.test(field)) { 10 + throw new Error(`Invalid field name: ${field}`); 11 + } 12 + } 13 + 14 + export interface SqlDialect { 15 + /** json_extract(col, '$.field') or col->>'field' */ 16 + jsonExtract(column: string, field: string): string; 17 + 18 + /** Convert INSERT INTO to ignore-duplicates form. 19 + * SQLite: INSERT INTO → INSERT OR IGNORE INTO 20 + * PG: appends ON CONFLICT DO NOTHING 21 + * Accepts full SQL starting with "INSERT INTO" (works with both VALUES and SELECT). */ 22 + insertOrIgnore(sql: string): string; 23 + 24 + /** Column type for the record column: TEXT (SQLite) or JSONB (PostgreSQL) */ 25 + readonly recordColumnType: string; 26 + 27 + /** FTS strategy: 'virtual-table' (SQLite FTS5) or 'generated-column' (PG tsvector) */ 28 + readonly ftsStrategy: "virtual-table" | "generated-column"; 29 + 30 + /** INTEGER type name — same on both, but PostgreSQL may want BIGINT for time_us */ 31 + readonly integerType: string; 32 + 33 + /** BIGINT type name for timestamps */ 34 + readonly bigintType: string; 35 + 36 + /** Wrap an expression for use in CREATE INDEX — PostgreSQL requires parens around expressions */ 37 + indexExpression(expr: string): string; 38 + } 39 + 40 + export const sqliteDialect: SqlDialect = { 41 + jsonExtract(column: string, field: string): string { 42 + assertSafeField(field); 43 + return `json_extract(${column}, '$.${field}')`; 44 + }, 45 + 46 + insertOrIgnore(sql: string): string { 47 + return sql.replace(/^INSERT INTO/, "INSERT OR IGNORE INTO"); 48 + }, 49 + 50 + recordColumnType: "TEXT", 51 + ftsStrategy: "virtual-table", 52 + integerType: "INTEGER", 53 + bigintType: "INTEGER", 54 + 55 + indexExpression(expr: string): string { 56 + return expr; 57 + }, 58 + }; 59 + 60 + export const postgresDialect: SqlDialect = { 61 + jsonExtract(column: string, field: string): string { 62 + assertSafeField(field); 63 + const parts = field.split("."); 64 + if (parts.length === 1) { 65 + return `${column}->>'${parts[0]}'`; 66 + } 67 + // a.b.c → col->'a'->'b'->>'c' 68 + const intermediate = parts.slice(0, -1).map((p) => `->'${p}'`).join(""); 69 + return `${column}${intermediate}->>'${parts[parts.length - 1]}'`; 70 + }, 71 + 72 + insertOrIgnore(sql: string): string { 73 + return `${sql} ON CONFLICT DO NOTHING`; 74 + }, 75 + 76 + recordColumnType: "JSONB", 77 + ftsStrategy: "generated-column", 78 + integerType: "INTEGER", 79 + bigintType: "BIGINT", 80 + 81 + indexExpression(expr: string): string { 82 + return `(${expr})`; 83 + }, 84 + }; 85 + 86 + /** Generate FTS schema statements based on dialect */ 87 + export function buildFtsSchema( 88 + dialect: SqlDialect, 89 + recordsTable: string, 90 + fields: string[] 91 + ): string[] { 92 + if (dialect.ftsStrategy === "virtual-table") { 93 + const ftsTable = recordsTable.replace("records_", "fts_"); 94 + return [ 95 + `CREATE VIRTUAL TABLE IF NOT EXISTS ${ftsTable} USING fts5(uri UNINDEXED, content)` 96 + ]; 97 + } else { 98 + const concatExpr = fields 99 + .map((f) => `COALESCE(${dialect.jsonExtract("record", f)}, '')`) 100 + .join(" || ' ' || "); 101 + return [ 102 + `ALTER TABLE ${recordsTable} ADD COLUMN IF NOT EXISTS search_vector TSVECTOR GENERATED ALWAYS AS (to_tsvector('english', ${concatExpr})) STORED`, 103 + `CREATE INDEX IF NOT EXISTS idx_${recordsTable}_search ON ${recordsTable} USING GIN (search_vector)`, 104 + ]; 105 + } 106 + } 107 + 108 + /** Generate FTS query clause based on dialect */ 109 + export function ftsQueryClause( 110 + dialect: SqlDialect, 111 + recordsTable: string 112 + ): { join: string; condition: string; orderExpr: string } { 113 + if (dialect.ftsStrategy === "virtual-table") { 114 + const ftsTable = recordsTable.replace("records_", "fts_"); 115 + return { 116 + join: `JOIN ${ftsTable} fts ON fts.uri = r.uri`, 117 + condition: "fts.content MATCH ?", 118 + orderExpr: "fts.rank", 119 + }; 120 + } else { 121 + return { 122 + join: "", 123 + condition: "r.search_vector @@ plainto_tsquery('english', ?)", 124 + orderExpr: "ts_rank(r.search_vector, plainto_tsquery('english', ?))", 125 + }; 126 + } 127 + }

+235

src/core/persistent.ts

··· 1 + import type { JetstreamSubscription } from "@atcute/jetstream"; 2 + import type { ContrailConfig, IngestEvent, Database, Logger } from "./types"; 3 + import { getCollectionNames, getDependentCollections, DEFAULT_FEED_MAX_ITEMS } from "./types"; 4 + import { initSchema, getLastCursor, saveCursor, applyEvents, pruneFeedItems } from "./db"; 5 + import { refreshStaleIdentities } from "./identity"; 6 + import { createIngestState } from "./jetstream"; 7 + import type { IngestState } from "./jetstream"; 8 + 9 + const FEED_PRUNE_INTERVAL_MS = 60 * 60 * 1000; 10 + 11 + export interface PersistentIngestOptions { 12 + batchSize?: number; 13 + flushIntervalMs?: number; 14 + signal?: AbortSignal; 15 + /** Override subscription creation for testing */ 16 + createSubscription?: (cursor: number | null) => JetstreamSubscription; 17 + logger?: Logger; 18 + } 19 + 20 + function getLogger(config: ContrailConfig, options?: PersistentIngestOptions): Logger { 21 + return options?.logger ?? config.logger ?? console; 22 + } 23 + 24 + export async function runPersistent( 25 + db: Database, 26 + config: ContrailConfig, 27 + options?: PersistentIngestOptions, 28 + ): Promise<void> { 29 + const log = getLogger(config, options); 30 + const batchSize = options?.batchSize ?? 50; 31 + const flushIntervalMs = options?.flushIntervalMs ?? 5_000; 32 + const signal = options?.signal; 33 + const state = createIngestState(); 34 + 35 + // Init schema once 36 + if (!state.schemaInitialized) { 37 + await initSchema(db, config); 38 + state.schemaInitialized = true; 39 + } 40 + 41 + // Load known DIDs for dependent collection filtering 42 + const dependentCollections = new Set(getDependentCollections(config)); 43 + let knownDids: Set<string> | undefined; 44 + if (dependentCollections.size > 0) { 45 + const result = await db 46 + .prepare("SELECT did FROM identities") 47 + .all<{ did: string }>(); 48 + knownDids = new Set((result.results ?? []).map((r) => r.did)); 49 + state.cachedKnownDids = knownDids; 50 + log.log(`Loaded ${knownDids.size} known DIDs from database`); 51 + } 52 + 53 + const collections = getCollectionNames(config); 54 + let reconnectAttempts = 0; 55 + 56 + while (!signal?.aborted) { 57 + const cursor = await getLastCursor(db); 58 + log.log(`Starting persistent ingestion. Cursor: ${cursor ?? "none"}, Collections: ${collections.join(", ")}`); 59 + 60 + try { 61 + await streamAndFlush(db, config, cursor, { 62 + batchSize, 63 + flushIntervalMs, 64 + signal, 65 + collections, 66 + dependentCollections, 67 + knownDids, 68 + state, 69 + log, 70 + createSubscription: options?.createSubscription, 71 + }); 72 + reconnectAttempts = 0; 73 + } catch (err) { 74 + if (signal?.aborted) break; 75 + log.error(`Jetstream connection error: ${err}`); 76 + const delay = Math.min(1000 * Math.pow(2, reconnectAttempts), 30_000); 77 + reconnectAttempts++; 78 + log.log(`Reconnecting in ${delay}ms (attempt ${reconnectAttempts})...`); 79 + await new Promise((r) => setTimeout(r, delay)); 80 + } 81 + } 82 + 83 + log.log("Persistent ingestion stopped"); 84 + } 85 + 86 + interface StreamOptions { 87 + batchSize: number; 88 + flushIntervalMs: number; 89 + signal?: AbortSignal; 90 + collections: string[]; 91 + dependentCollections: Set<string>; 92 + knownDids?: Set<string>; 93 + state: IngestState; 94 + log: Logger; 95 + createSubscription?: (cursor: number | null) => any; 96 + } 97 + 98 + async function streamAndFlush( 99 + db: Database, 100 + config: ContrailConfig, 101 + cursor: number | null, 102 + opts: StreamOptions, 103 + ): Promise<void> { 104 + const { batchSize, flushIntervalMs, signal, collections, dependentCollections, knownDids, state, log } = opts; 105 + 106 + const subscription = opts.createSubscription 107 + ? opts.createSubscription(cursor) 108 + : new (await import("@atcute/jetstream")).JetstreamSubscription({ 109 + url: config.jetstreams ?? [], 110 + wantedCollections: collections, 111 + ...(cursor !== null ? { cursor } : {}), 112 + onConnectionOpen() { log.log("Connected to Jetstream"); }, 113 + onConnectionClose(event: any) { log.log(`Disconnected: ${event.code} ${event.reason}`); }, 114 + onConnectionError(event: any) { log.error("Jetstream error:", event.error); }, 115 + }); 116 + 117 + const buffer: IngestEvent[] = []; 118 + let flushDue = false; 119 + let flushTimer: ReturnType<typeof setTimeout> | null = null; 120 + 121 + const resetFlushTimer = () => { 122 + if (flushTimer) clearTimeout(flushTimer); 123 + flushTimer = setTimeout(() => { flushDue = true; }, flushIntervalMs); 124 + }; 125 + 126 + const flush = async () => { 127 + if (buffer.length === 0) return; 128 + const batch = buffer.splice(0); 129 + flushDue = false; 130 + resetFlushTimer(); 131 + 132 + await applyEvents(db, batch, config); 133 + 134 + const lastTimeUs = Math.max(...batch.map((e) => e.time_us)); 135 + await saveCursor(db, lastTimeUs); 136 + 137 + // Identity refresh 138 + const uniqueDids = [...new Set(batch.map((e) => e.did))]; 139 + if (uniqueDids.length > 0) { 140 + try { 141 + await refreshStaleIdentities(db, uniqueDids); 142 + } catch (err) { 143 + log.warn(`Identity refresh failed: ${err}`); 144 + } 145 + } 146 + 147 + // Feed pruning 148 + if (config.feeds && Date.now() - state.lastFeedPruneMs > FEED_PRUNE_INTERVAL_MS) { 149 + const maxItems = Math.max( 150 + ...Object.values(config.feeds).map((f) => f.maxItems ?? DEFAULT_FEED_MAX_ITEMS) 151 + ); 152 + const pruned = await pruneFeedItems(db, maxItems); 153 + if (pruned > 0) log.log(`Pruned ${pruned} old feed items`); 154 + state.lastFeedPruneMs = Date.now(); 155 + } 156 + 157 + log.log(`Flushed ${batch.length} events. Cursor: ${lastTimeUs}`); 158 + }; 159 + 160 + // Handle abort 161 + const onAbort = () => { 162 + if (flushTimer) clearTimeout(flushTimer); 163 + }; 164 + signal?.addEventListener("abort", onAbort, { once: true }); 165 + 166 + resetFlushTimer(); 167 + 168 + // Use manual iterator so we can race next() against abort signal 169 + const iterator = subscription[Symbol.asyncIterator](); 170 + 171 + try { 172 + while (true) { 173 + if (signal?.aborted) break; 174 + 175 + // Race the next event against abort signal 176 + let result: IteratorResult<any>; 177 + if (signal) { 178 + const nextPromise = iterator.next(); 179 + if (signal.aborted) { 180 + result = { value: undefined, done: true }; 181 + } else { 182 + let abortHandler: () => void; 183 + const abortPromise = new Promise<IteratorResult<any>>((resolve) => { 184 + abortHandler = () => resolve({ value: undefined, done: true }); 185 + signal.addEventListener("abort", abortHandler, { once: true }); 186 + }); 187 + result = await Promise.race([nextPromise, abortPromise]); 188 + signal.removeEventListener("abort", abortHandler!); 189 + } 190 + } else { 191 + result = await iterator.next(); 192 + } 193 + 194 + if (result.done) break; 195 + const event = result.value; 196 + 197 + if (event.kind === "commit") { 198 + const { commit } = event; 199 + 200 + if (dependentCollections.has(commit.collection) && knownDids) { 201 + if (!knownDids.has(event.did)) continue; 202 + } 203 + 204 + const now = Date.now(); 205 + const uri = `at://${event.did}/${commit.collection}/${commit.rkey}`; 206 + 207 + buffer.push({ 208 + uri, 209 + did: event.did, 210 + time_us: event.time_us, 211 + collection: commit.collection, 212 + operation: commit.operation as "create" | "update" | "delete", 213 + rkey: commit.rkey, 214 + cid: commit.operation === "delete" ? null : commit.cid, 215 + record: commit.operation === "delete" ? null : JSON.stringify(commit.record), 216 + indexed_at: now * 1000, 217 + }); 218 + 219 + if (knownDids && !dependentCollections.has(commit.collection)) { 220 + knownDids.add(event.did); 221 + } 222 + } 223 + 224 + if (buffer.length >= batchSize || flushDue) { 225 + await flush(); 226 + } 227 + } 228 + } finally { 229 + // Clean up iterator 230 + await iterator.return?.({ value: undefined, done: true }); 231 + // Final flush on exit 232 + await flush(); 233 + signal?.removeEventListener("abort", onAbort); 234 + } 235 + }

+3

src/core/types.ts

··· 1 + import type { SqlDialect } from "./dialect"; 2 + 1 3 // Database interface — D1 implements this natively 2 4 export interface Database { 3 5 prepare(sql: string): Statement; 4 6 batch(stmts: Statement[]): Promise<any[]>; 7 + dialect?: SqlDialect; 5 8 } 6 9 7 10 export interface Statement {

+34 -21

src/core/db/records.ts

··· 10 10 } from "../types"; 11 11 import { getNestedValue, getRelationField, countColumnName, getFeedFollowCollections, recordsTableName } from "../types"; 12 12 import { getSearchableFields, ftsTableName, buildFtsContent } from "../search"; 13 + import { ftsQueryClause, getDialect } from "../dialect"; 13 14 14 15 // --- Counts --- 15 16 ··· 103 104 // Total count 104 105 const totalCol = countColumnName(rel.collection); 105 106 setClauses.push( 106 - `${totalCol} = (SELECT ${countExpr} FROM ${childTable} WHERE json_extract(record, '$.${field}') = ?)` 107 + `${totalCol} = (SELECT ${countExpr} FROM ${childTable} WHERE ${getDialect(db).jsonExtract('record', field)} = ?)` 107 108 ); 108 109 setBindings.push(targetValue); 109 110 ··· 114 115 for (const [, fullToken] of Object.entries(mapping.groups)) { 115 116 const groupCol = countColumnName(fullToken); 116 117 setClauses.push( 117 - `${groupCol} = (SELECT ${countExpr} FROM ${childTable} WHERE json_extract(record, '$.${field}') = ? AND json_extract(record, '$.${rel.groupBy}') = ?)` 118 + `${groupCol} = (SELECT ${countExpr} FROM ${childTable} WHERE ${getDialect(db).jsonExtract('record', field)} = ? AND ${getDialect(db).jsonExtract('record', rel.groupBy)} = ?)` 118 119 ); 119 120 setBindings.push(targetValue, fullToken); 120 121 } ··· 143 144 config: ContrailConfig, 144 145 existingMap: Map<string, ExistingRecordInfo> 145 146 ): Statement[] { 147 + // PostgreSQL: tsvector generated column is auto-maintained, no manual FTS sync 148 + if (getDialect(db).ftsStrategy === "generated-column") return []; 149 + 146 150 const colConfig = config.collections[event.collection]; 147 151 if (!colConfig) return []; 148 152 ··· 194 198 stmts.push( 195 199 db 196 200 .prepare( 197 - `INSERT OR IGNORE INTO feed_items (actor, uri, collection, time_us) 201 + getDialect(db).insertOrIgnore( 202 + `INSERT INTO feed_items (actor, uri, collection, time_us) 198 203 SELECT r.did, ?, ?, ? 199 204 FROM ${followTable} r 200 - WHERE json_extract(r.record, '$.subject') = ?` 205 + WHERE ${getDialect(db).jsonExtract('r.record', 'subject')} = ?` 206 + ) 201 207 ) 202 208 .bind(event.uri, event.collection, event.time_us, event.did) 203 209 ); ··· 219 225 stmts.push( 220 226 db 221 227 .prepare( 222 - `INSERT OR IGNORE INTO feed_items (actor, uri, collection, time_us) 228 + getDialect(db).insertOrIgnore( 229 + `INSERT INTO feed_items (actor, uri, collection, time_us) 223 230 SELECT ?, r.uri, ?, r.time_us 224 231 FROM ${targetTable} r 225 232 WHERE r.did = ? 226 233 ORDER BY r.time_us DESC 227 234 LIMIT 100` 235 + ) 228 236 ) 229 237 .bind(event.did, targetCol, subject) 230 238 ); ··· 265 273 ): Promise<number> { 266 274 const result = await db 267 275 .prepare( 268 - `DELETE FROM feed_items WHERE rowid NOT IN ( 269 - SELECT rowid FROM ( 270 - SELECT rowid, ROW_NUMBER() OVER (PARTITION BY actor ORDER BY time_us DESC) as rn 276 + `DELETE FROM feed_items WHERE (actor, uri) NOT IN ( 277 + SELECT actor, uri FROM ( 278 + SELECT actor, uri, ROW_NUMBER() OVER (PARTITION BY actor ORDER BY time_us DESC) as rn 271 279 FROM feed_items 272 - ) WHERE rn <= ? 280 + ) sub WHERE rn <= ? 273 281 )` 274 282 ) 275 283 .bind(maxItems) ··· 517 525 // Only select the columns needed for cursor pagination 518 526 let cursorSelect: string; 519 527 if (sort?.recordField) { 520 - cursorSelect = `time_us, json_extract(record, '$.${sort.recordField}') as sort_value`; 528 + cursorSelect = `time_us, ${getDialect(db).jsonExtract('record', sort.recordField)} as sort_value`; 521 529 } else if (sort?.countType) { 522 530 const sortCol = countColumnName(sort.countType); 523 531 cursorSelect = `time_us, ${sortCol}`; ··· 533 541 if (cursorRow) { 534 542 if (sort?.recordField) { 535 543 const sortValue = cursorRow.sort_value; 536 - const field = `json_extract(r.record, '$.${sort.recordField}')`; 544 + const sortExpr = getDialect(db).jsonExtract('r.record', sort.recordField); 537 545 const cmp = sort.direction === "desc" ? "<" : ">"; 538 - conditions.push(`(${field} ${cmp} ? OR (${field} = ? AND r.time_us < ?))`); 546 + conditions.push(`(${sortExpr} ${cmp} ? OR (${sortExpr} = ? AND r.time_us < ?))`); 539 547 bindings.push(sortValue ?? "", sortValue ?? "", cursorRow.time_us); 540 548 } else if (sort?.countType) { 541 549 const sortCol = countColumnName(sort.countType); ··· 551 559 } 552 560 553 561 for (const [field, value] of Object.entries(filters)) { 554 - conditions.push(`json_extract(r.record, '$.${field}') = ?`); 562 + conditions.push(`${getDialect(db).jsonExtract('r.record', field)} = ?`); 555 563 bindings.push(value); 556 564 } 557 565 558 566 for (const [field, range] of Object.entries(rangeFilters)) { 559 567 if (range.min != null) { 560 - conditions.push(`json_extract(r.record, '$.${field}') >= ?`); 568 + conditions.push(`${getDialect(db).jsonExtract('r.record', field)} >= ?`); 561 569 bindings.push(range.min); 562 570 } 563 571 if (range.max != null) { 564 - conditions.push(`json_extract(r.record, '$.${field}') <= ?`); 572 + conditions.push(`${getDialect(db).jsonExtract('r.record', field)} <= ?`); 565 573 bindings.push(range.max); 566 574 } 567 575 } ··· 574 582 575 583 // FTS search 576 584 let ftsJoin = ""; 585 + let ftsClause: ReturnType<typeof ftsQueryClause> | null = null; 577 586 if (search) { 578 587 const colConfig2 = config.collections[collection]; 579 588 const fields = colConfig2 ? getSearchableFields(collection, colConfig2) : null; 580 589 if (fields && fields.length > 0) { 581 - const ftsTable = ftsTableName(collection); 582 - ftsJoin = `JOIN ${ftsTable} fts ON fts.uri = r.uri`; 583 - conditions.push("fts.content MATCH ?"); 590 + ftsClause = ftsQueryClause(getDialect(db), recordsTableName(collection)); 591 + ftsJoin = ftsClause.join; 592 + conditions.push(ftsClause.condition); 584 593 bindings.push(search); 585 594 } 586 595 } ··· 597 606 let orderBy: string; 598 607 if (sort?.recordField) { 599 608 const dir = sort.direction === "desc" ? "DESC" : "ASC"; 600 - orderBy = `json_extract(r.record, '$.${sort.recordField}') ${dir}, r.time_us DESC`; 609 + orderBy = `${getDialect(db).jsonExtract('r.record', sort.recordField)} ${dir}, r.time_us DESC`; 601 610 } else if (sort?.countType) { 602 611 const dir = sort.direction === "desc" ? "DESC" : "ASC"; 603 612 const sortCol = countColumnName(sort.countType); 604 613 orderBy = `r.${sortCol} ${dir}, r.time_us DESC`; 605 - } else if (ftsJoin) { 606 - orderBy = "fts.rank, r.time_us DESC"; 614 + } else if (ftsClause) { 615 + orderBy = `${ftsClause.orderExpr}, r.time_us DESC`; 616 + // PG ts_rank needs the search term bound again for ORDER BY 617 + if (getDialect(db).ftsStrategy === "generated-column" && search) { 618 + bindings.push(search); 619 + } 607 620 } else { 608 621 orderBy = "r.time_us DESC"; 609 622 }

+27 -25

src/core/db/schema.ts

··· 1 1 import type { ContrailConfig, Database, ResolvedContrailConfig, ResolvedMaps } from "../types"; 2 + import type { SqlDialect } from "../dialect"; 3 + import { buildFtsSchema, getDialect } from "../dialect"; 2 4 import { getRelationField, countColumnName, recordsTableName, resolveConfig } from "../types"; 3 - import { getSearchableFields, ftsTableName } from "../search"; 5 + import { getSearchableFields } from "../search"; 4 6 5 7 function getResolved(config: ContrailConfig): ResolvedMaps { 6 8 return (config as ResolvedContrailConfig)._resolved ?? resolveConfig(config)._resolved; 7 9 } 8 10 9 - const BASE_SCHEMA = ` 11 + function buildBaseSchema(dialect: SqlDialect): string { 12 + return ` 10 13 CREATE TABLE IF NOT EXISTS backfills ( 11 14 did TEXT NOT NULL, 12 15 collection TEXT NOT NULL, ··· 25 28 ); 26 29 CREATE TABLE IF NOT EXISTS cursor ( 27 30 id INTEGER PRIMARY KEY CHECK (id = 1), 28 - time_us INTEGER NOT NULL 31 + time_us ${dialect.bigintType} NOT NULL 29 32 ); 30 33 CREATE TABLE IF NOT EXISTS identities ( 31 34 did TEXT PRIMARY KEY, 32 35 handle TEXT, 33 36 pds TEXT, 34 - resolved_at INTEGER NOT NULL 37 + resolved_at ${dialect.bigintType} NOT NULL 35 38 ); 36 39 CREATE INDEX IF NOT EXISTS idx_identities_handle ON identities(handle); 37 40 `; 41 + } 38 42 39 43 function sanitizeName(name: string): string { 40 44 return name.replace(/[^a-zA-Z0-9]/g, "_"); 41 45 } 42 46 43 - function buildCollectionTables(config: ContrailConfig): string[] { 47 + function buildCollectionTables(config: ContrailConfig, dialect: SqlDialect): string[] { 44 48 const stmts: string[] = []; 45 49 for (const collection of Object.keys(config.collections)) { 46 50 const table = recordsTableName(collection); ··· 50 54 did TEXT NOT NULL, 51 55 rkey TEXT NOT NULL, 52 56 cid TEXT, 53 - record TEXT, 54 - time_us INTEGER NOT NULL, 55 - indexed_at INTEGER NOT NULL 57 + record ${dialect.recordColumnType}, 58 + time_us ${dialect.bigintType} NOT NULL, 59 + indexed_at ${dialect.bigintType} NOT NULL 56 60 )` 57 61 ); 58 62 stmts.push(`CREATE INDEX IF NOT EXISTS idx_${sanitizeName(collection)}_did ON ${table}(did)`); ··· 61 65 return stmts; 62 66 } 63 67 64 - function buildDynamicIndexes(config: ContrailConfig): string[] { 68 + function buildDynamicIndexes(config: ContrailConfig, dialect: SqlDialect): string[] { 65 69 const resolved = getResolved(config); 66 70 const indexes: string[] = []; 67 71 for (const [collection, colConfig] of Object.entries(config.collections)) { ··· 70 74 for (const field of Object.keys(queryable)) { 71 75 const idxName = `idx_${sanitizeName(collection)}_${sanitizeName(field)}`; 72 76 indexes.push( 73 - `CREATE INDEX IF NOT EXISTS ${idxName} ON ${table}(json_extract(record, '$.${field}'))` 77 + `CREATE INDEX IF NOT EXISTS ${idxName} ON ${table}(${dialect.indexExpression(dialect.jsonExtract('record', field))})` 74 78 ); 75 79 } 76 80 ··· 80 84 const childTable = recordsTableName(rel.collection); 81 85 const idxName = `idx_${sanitizeName(rel.collection)}_${sanitizeName(on)}`; 82 86 indexes.push( 83 - `CREATE INDEX IF NOT EXISTS ${idxName} ON ${childTable}(json_extract(record, '$.${on}'))` 87 + `CREATE INDEX IF NOT EXISTS ${idxName} ON ${childTable}(${dialect.indexExpression(dialect.jsonExtract('record', on))})` 84 88 ); 85 89 } 86 90 } ··· 134 138 return stmts; 135 139 } 136 140 137 - function buildFeedTables(config: ContrailConfig): string[] { 141 + function buildFeedTables(config: ContrailConfig, dialect: SqlDialect): string[] { 138 142 if (!config.feeds || Object.keys(config.feeds).length === 0) return []; 139 143 const stmts = [ 140 144 `CREATE TABLE IF NOT EXISTS feed_items ( 141 145 actor TEXT NOT NULL, 142 146 uri TEXT NOT NULL, 143 147 collection TEXT NOT NULL, 144 - time_us INTEGER NOT NULL, 148 + time_us ${dialect.bigintType} NOT NULL, 145 149 PRIMARY KEY (actor, uri) 146 150 )`, 147 151 `CREATE INDEX IF NOT EXISTS idx_feed_actor_coll_time ON feed_items(actor, collection, time_us DESC)`, ··· 160 164 const table = recordsTableName(col); 161 165 const safe = sanitizeName(col); 162 166 stmts.push( 163 - `CREATE INDEX IF NOT EXISTS idx_${safe}_subject ON ${table}(json_extract(record, '$.subject'))` 167 + `CREATE INDEX IF NOT EXISTS idx_${safe}_subject ON ${table}(${dialect.indexExpression(dialect.jsonExtract('record', 'subject'))})` 164 168 ); 165 169 } 166 170 167 171 return stmts; 168 172 } 169 173 170 - function buildFtsTables(config: ContrailConfig): string[] { 174 + function buildFtsTables(config: ContrailConfig, dialect: SqlDialect): string[] { 171 175 const stmts: string[] = []; 172 176 for (const [collection, colConfig] of Object.entries(config.collections)) { 173 177 const fields = getSearchableFields(collection, colConfig); 174 178 if (!fields || fields.length === 0) continue; 175 - const table = ftsTableName(collection); 176 - stmts.push( 177 - `CREATE VIRTUAL TABLE IF NOT EXISTS ${table} USING fts5(uri UNINDEXED, content)` 178 - ); 179 + const table = recordsTableName(collection); 180 + stmts.push(...buildFtsSchema(dialect, table, fields)); 179 181 } 180 182 return stmts; 181 183 } ··· 199 201 db: Database, 200 202 config: ContrailConfig 201 203 ): Promise<void> { 202 - const baseStatements = BASE_SCHEMA.split(";") 204 + const dialect = getDialect(db); 205 + const baseStatements = buildBaseSchema(dialect).split(";") 203 206 .map((s) => s.trim()) 204 207 .filter((s) => s.length > 0); 205 - 206 - const collectionStatements = buildCollectionTables(config); 207 - const indexStatements = buildDynamicIndexes(config); 208 - const ftsStatements = buildFtsTables(config); 209 - const feedStatements = buildFeedTables(config); 208 + const collectionStatements = buildCollectionTables(config, dialect); 209 + const indexStatements = buildDynamicIndexes(config, dialect); 210 + const ftsStatements = buildFtsTables(config, dialect); 211 + const feedStatements = buildFeedTables(config, dialect); 210 212 const all = [...baseStatements, ...collectionStatements, ...indexStatements, ...feedStatements]; 211 213 212 214 await db.batch(all.map((s) => db.prepare(s)));

+5 -2

src/core/router/feed.ts

··· 1 1 import type { Hono } from "hono"; 2 2 import type { ContrailConfig, Database, FeedConfig } from "../types"; 3 + import { getDialect } from "../dialect"; 3 4 import { DEFAULT_FEED_MAX_ITEMS, recordsTableName } from "../types"; 4 5 import { resolveActor } from "../identity"; 5 6 import { backfillUser } from "../backfill"; ··· 38 39 const targetTable = recordsTableName(targetCol); 39 40 await db 40 41 .prepare( 41 - `INSERT OR IGNORE INTO feed_items (actor, uri, collection, time_us) 42 + getDialect(db).insertOrIgnore( 43 + `INSERT INTO feed_items (actor, uri, collection, time_us) 42 44 SELECT ?, r.uri, ?, r.time_us 43 45 FROM ${targetTable} r 44 46 WHERE r.did IN ( 45 - SELECT json_extract(f.record, '$.subject') 47 + SELECT ${getDialect(db).jsonExtract('f.record', 'subject')} 46 48 FROM ${followTable} f 47 49 WHERE f.did = ? 48 50 ) 49 51 ORDER BY r.time_us DESC 50 52 LIMIT ?` 53 + ) 51 54 ) 52 55 .bind(actor, targetCol, actor, maxItems) 53 56 .run();

+2 -1

src/core/router/hydrate.ts

··· 1 1 import type { RelationConfig, ReferenceConfig, RecordRow, Database } from "../types"; 2 + import { getDialect } from "../dialect"; 2 3 import { getNestedValue, getRelationField, recordsTableName } from "../types"; 3 4 import { batchedInQuery, formatRecord } from "./helpers"; 4 5 ··· 63 64 const relatedRows = await batchedInQuery<Omit<RecordRow, "collection">>( 64 65 db, 65 66 `SELECT uri, did, rkey, record, time_us FROM ${table} 66 - WHERE json_extract(record, '$.${field}') IN (__IN__) 67 + WHERE ${getDialect(db).jsonExtract('record', field)} IN (__IN__) 67 68 ORDER BY time_us DESC 68 69 LIMIT ${maxRows}`, 69 70 [],

Configure Feed

Configure Feed