Low-latency AT Protocol interaction indexer.
backfill: don't DDoS the PDS
Aly Raffauf
(Jul 3, 2026, 1:15 AM EDT)
be9c010a
b9822e22
+28
+28
internal/backfill/backfill.go
+28
internal/backfill/backfill.go
···
19
19
indigorepo "github.com/bluesky-social/indigo/repo"
20
20
)
21
21
22
+
const hostPacingDelay = 2 * time.Second
23
+
22
24
type Backfill struct {
23
25
Client *xrpc.Client
24
26
Directory identity.Directory
25
27
Store *store.Store
28
+
29
+
lastRequest map[string]time.Time
30
+
}
31
+
32
+
// don't DDoS the PDS
33
+
func (b *Backfill) waitForHost(ctx context.Context, host string) error {
34
+
if b.lastRequest == nil {
35
+
b.lastRequest = make(map[string]time.Time)
36
+
}
37
+
38
+
if last, ok := b.lastRequest[host]; ok {
39
+
if wait := hostPacingDelay - time.Since(last); wait > 0 {
40
+
select {
41
+
case <-time.After(wait):
42
+
case <-ctx.Done():
43
+
return ctx.Err()
44
+
}
45
+
}
46
+
}
47
+
48
+
b.lastRequest[host] = time.Now()
49
+
return nil
26
50
}
27
51
28
52
func (b *Backfill) Run(ctx context.Context, collections []string) error {
···
77
101
pdsClient := &xrpc.Client{
78
102
Host: identity.PDSEndpoint(),
79
103
Client: &http.Client{Timeout: 5 * time.Minute},
104
+
}
105
+
106
+
if err := b.waitForHost(ctx, pdsClient.Host); err != nil {
107
+
return fmt.Errorf("wait for host: %w", err)
80
108
}
81
109
82
110
fetchCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)