Low-latency AT Protocol interaction indexer.
add fully configurable CLI with kong
Aly Raffauf
(Jul 3, 2026, 4:54 PM EDT)
37343725
dfbf789a
+81
-19
+33
-5
README.md
+33
-5
README.md
···
10
10
11
11
## What it does
12
12
13
-
Asterism connects directly to the relay Firehose (`com.atproto.sync.subscribeRepos`), decodes each repo commit's CAR-framed CBOR blocks itself, and recursively walks each record for link references (strong refs, AT-URIs, DIDs, URLs). Links are stored keyed by target, source collection, and field path. On startup it backfills existing repos for your configured collections so the index is useful immediately.
13
+
Asterism connects directly to the relay Firehose (`com.atproto.sync.subscribeRepos`), decodes each repo commit's CAR-framed CBOR blocks itself, and recursively walks each record for link references (strong refs, AT-URIs, DIDs, URLs). Links are stored keyed by target, source collection, and field path. It can optionally backfill existing repos for your configured collections on startup so the index is useful immediately.
14
14
15
15
This matters for two reasons:
16
16
···
30
30
The typical deployment indexes only the collections your app queries:
31
31
32
32
```bash
33
-
go run ./cmd/asterism/ -collections sh.tangled.graph.follow,sh.tangled.repo.issue,sh.tangled.feed.comment
33
+
go run ./cmd/asterism/ --collections sh.tangled.graph.follow,sh.tangled.repo.issue,sh.tangled.feed.comment
34
34
```
35
35
36
-
This connects to the relay Firehose, backfills existing repos for those collections in the background, stores links in an sqlite database at `asterism.db`, and serves the query API on `:8081`.
36
+
This connects to the relay Firehose, stores links in an sqlite database at `asterism.db`, and serves the query API on `:8081`. Firehose events that are too large to include inline are fetched through the repo API automatically.
37
37
38
-
To index all collections (Constellation-equivalent scope, not recommended for sovereign deployments):
38
+
To also backfill existing repos for your configured collections on startup:
39
39
40
40
```bash
41
+
go run ./cmd/asterism/ --backfill --collections sh.tangled.graph.follow,sh.tangled.repo.issue,sh.tangled.feed.comment
42
+
```
43
+
44
+
To live-index all collections (Constellation-equivalent scope, not recommended for sovereign deployments):
45
+
46
+
```bash
47
+
go run ./cmd/asterism/
48
+
```
49
+
50
+
### Configuration
51
+
52
+
Every flag can also be set with an environment variable.
53
+
54
+
| Flag | Environment variable | Default | Description |
55
+
|---|---|---|---|
56
+
| `--collections` | `ASTERISM_COLLECTIONS` | empty | Comma-separated collection NSIDs to index. Empty means all collections. |
57
+
| `--backfill` | `ASTERISM_BACKFILL` | false | Backfill existing repos for configured collections on startup. |
58
+
| `--database` | `ASTERISM_DATABASE` | `asterism.db` | SQLite database path. |
59
+
| `--listen` | `ASTERISM_LISTEN` | `:8081` | HTTP API listen address. |
60
+
| `--relay` | `ASTERISM_RELAY` | `relay1.us-east.bsky.network` | Relay host. Asterism derives the Firehose websocket and relay HTTP API URLs from this host. |
61
+
62
+
For example:
63
+
64
+
```bash
65
+
ASTERISM_DATABASE=/var/lib/asterism/asterism.db \
66
+
ASTERISM_LISTEN=:8081 \
67
+
ASTERISM_COLLECTIONS=sh.tangled.graph.follow,sh.tangled.repo.issue \
68
+
ASTERISM_BACKFILL=true \
41
69
go run ./cmd/asterism/
42
70
```
43
71
···
126
154
**Near term**
127
155
128
156
- [x] Full Constellation API parity (`getBacklinksCount`, `getBacklinkDids`, `getBacklinks`, `getManyToMany`, `getManyToManyCounts`)
129
-
- [ ] Configurable listen address, database path, and relay URL
157
+
- [x] Configurable listen address, database path, relay host, and startup backfill
130
158
- [ ] Account deletion and deactivation handling
131
159
- [x] Graceful shutdown and Firehose reconnect
132
160
+39
-14
cmd/asterism/main.go
+39
-14
cmd/asterism/main.go
···
2
2
3
3
import (
4
4
"context"
5
-
"flag"
6
5
"fmt"
7
6
"log/slog"
8
7
"strings"
9
8
9
+
"github.com/alecthomas/kong"
10
10
"github.com/bluesky-social/indigo/atproto/identity"
11
11
"github.com/bluesky-social/indigo/xrpc"
12
12
···
16
16
"github.com/alyraffauf/asterism/internal/store"
17
17
)
18
18
19
-
const relayURL = "wss://relay1.us-east.bsky.network/xrpc/com.atproto.sync.subscribeRepos"
19
+
const (
20
+
subscribeReposPath = "/xrpc/com.atproto.sync.subscribeRepos"
21
+
)
22
+
23
+
type CLI struct {
24
+
Collections string `help:"Comma-separated list of collection NSIDs to filter on. Empty means all." env:"ASTERISM_COLLECTIONS"`
25
+
Backfill bool `help:"Backfill existing repos on startup." env:"ASTERISM_BACKFILL"`
26
+
Database string `help:"SQLite database path." env:"ASTERISM_DATABASE" default:"asterism.db"`
27
+
Listen string `help:"HTTP listen address." env:"ASTERISM_LISTEN" default:":8081"`
28
+
Relay string `help:"Relay host." env:"ASTERISM_RELAY" default:"relay1.us-east.bsky.network"`
29
+
}
20
30
21
31
func parseCollections(raw string) map[string]struct{} {
22
32
if raw == "" {
···
35
45
return wanted
36
46
}
37
47
48
+
func relayHTTPHost(relayHost string) string {
49
+
return "https://" + relayHost
50
+
}
51
+
52
+
func subscribeReposURL(relayHost string) string {
53
+
return "wss://" + relayHost + subscribeReposPath
54
+
}
55
+
38
56
func main() {
39
-
collectionsFlag := flag.String("collections", "", "comma-separated list of collection NSIDs to filter on (empty means all)")
40
-
flag.Parse()
57
+
var cli CLI
58
+
kong.Parse(&cli,
59
+
kong.Name("asterism"),
60
+
kong.Description("AT Protocol link index."),
61
+
)
41
62
42
63
ctx := context.Background()
43
64
logger := slog.Default()
44
65
45
-
linkStore, err := store.Open("asterism.db")
66
+
linkStore, err := store.Open(cli.Database)
46
67
if err != nil {
47
68
panic(err)
48
69
}
49
70
50
71
server := &api.Server{Store: linkStore}
51
72
go func() {
52
-
if err := server.Run(":8081"); err != nil {
73
+
if err := server.Run(cli.Listen); err != nil {
53
74
panic(err)
54
75
}
55
76
}()
56
77
57
-
wantedCollections := parseCollections(*collectionsFlag)
78
+
wantedCollections := parseCollections(cli.Collections)
58
79
59
80
var collections []string
60
81
for collection := range wantedCollections {
61
82
collections = append(collections, collection)
62
83
}
84
+
85
+
relayURL := subscribeReposURL(cli.Relay)
63
86
64
87
bf := &backfill.Backfill{
65
-
Client: &xrpc.Client{Host: "https://relay1.us-east.bsky.network"},
88
+
Client: &xrpc.Client{Host: relayHTTPHost(cli.Relay)},
66
89
Directory: identity.DefaultDirectory(),
67
90
Store: linkStore,
68
91
}
69
92
70
-
if len(collections) > 0 {
71
-
go func() {
72
-
if err := bf.Run(ctx, collections); err != nil {
73
-
fmt.Println("backfill error:", err)
74
-
}
75
-
}()
93
+
if cli.Backfill {
94
+
if len(collections) > 0 {
95
+
go func() {
96
+
if err := bf.Run(ctx, collections); err != nil {
97
+
fmt.Println("backfill error:", err)
98
+
}
99
+
}()
100
+
}
76
101
}
77
102
78
103
consumer := &firehose.Consumer{
+1
go.mod
+1
go.mod
+8
go.sum
+8
go.sum
···
1
1
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
2
2
github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b h1:5/++qT1/z812ZqBvqQt6ToRswSuPZ/B33m6xVHRzADU=
3
3
github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b/go.mod h1:4+EPqMRApwwE/6yo6CxiHoSnBzjRr3jsqer7frxP8y4=
4
+
github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
5
+
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
6
+
github.com/alecthomas/kong v1.15.0 h1:BVJstKbpO73zKpmIu+m/aLRrNmWwxXPIGTNin9VmLVI=
7
+
github.com/alecthomas/kong v1.15.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I=
8
+
github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
9
+
github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
4
10
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
5
11
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
6
12
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
···
59
65
github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
60
66
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
61
67
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
68
+
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
69
+
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
62
70
github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ=
63
71
github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y=
64
72
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=