Monorepo for Tangled
nix,spindle: sync workflow files on `sh.tangled.git.refUpdate`
Spindle will sync git repo when new repo is registered
Spindle will listen to `sh.tangled.git.refUpdate` event from knot
stream and sync its local git repo instead. Spindle's git repo will
sparse-checkout only `/.tangled/workflows` directory.
Spindle now requires git version >=2.49 for `--revision` flag in `git
clone` command.
References:
- <https://stackoverflow.com/q/47541033/13150270>
- <https://stackoverflow.com/q/600079/13150270>
Signed-off-by: Seongmin Lee <git@boltless.me>
Seongmin Lee
(May 16, 2026, 11:57 PM +0900)
0a9a35bd
78ec407c
+121
-3
+1
go.mod
+1
go.mod
···
36
36
github.com/gorilla/feeds v1.2.0
37
37
github.com/gorilla/sessions v1.4.0
38
38
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674
39
+
github.com/hashicorp/go-version v1.9.0
39
40
github.com/hashicorp/golang-lru/v2 v2.0.7
40
41
github.com/hiddeco/sshsig v0.2.0
41
42
github.com/hpcloud/tail v1.0.0
+2
go.sum
+2
go.sum
···
343
343
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
344
344
github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw=
345
345
github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw=
346
+
github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA=
347
+
github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
346
348
github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
347
349
github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
348
350
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+73
spindle/git/git.go
+73
spindle/git/git.go
···
1
+
package git
2
+
3
+
import (
4
+
"bytes"
5
+
"context"
6
+
"fmt"
7
+
"os"
8
+
"os/exec"
9
+
"strings"
10
+
11
+
"github.com/hashicorp/go-version"
12
+
)
13
+
14
+
func Version() (*version.Version, error) {
15
+
var buf bytes.Buffer
16
+
cmd := exec.Command("git", "version")
17
+
cmd.Stdout = &buf
18
+
cmd.Stderr = os.Stderr
19
+
err := cmd.Run()
20
+
if err != nil {
21
+
return nil, err
22
+
}
23
+
fields := strings.Fields(buf.String())
24
+
if len(fields) < 3 {
25
+
return nil, fmt.Errorf("invalid git version: %s", buf.String())
26
+
}
27
+
28
+
// version string is like: "git version 2.29.3" or "git version 2.29.3.windows.1"
29
+
versionString := fields[2]
30
+
if pos := strings.Index(versionString, "windows"); pos >= 1 {
31
+
versionString = versionString[:pos-1]
32
+
}
33
+
return version.NewVersion(versionString)
34
+
}
35
+
36
+
const WorkflowDir = `/.tangled/workflows`
37
+
38
+
func SparseSyncGitRepo(ctx context.Context, cloneUri, path, rev string) error {
39
+
exist, err := isDir(path)
40
+
if err != nil {
41
+
return err
42
+
}
43
+
if rev == "" {
44
+
rev = "HEAD"
45
+
}
46
+
if !exist {
47
+
if err := exec.Command("git", "clone", "--no-checkout", "--depth=1", "--filter=tree:0", "--revision="+rev, cloneUri, path).Run(); err != nil {
48
+
return fmt.Errorf("git clone: %w", err)
49
+
}
50
+
if err := exec.Command("git", "-C", path, "sparse-checkout", "set", "--no-cone", WorkflowDir).Run(); err != nil {
51
+
return fmt.Errorf("git sparse-checkout set: %w", err)
52
+
}
53
+
} else {
54
+
if err := exec.Command("git", "-C", path, "fetch", "--depth=1", "--filter=tree:0", "origin", rev).Run(); err != nil {
55
+
return fmt.Errorf("git pull: %w", err)
56
+
}
57
+
}
58
+
if err := exec.Command("git", "-C", path, "checkout", rev).Run(); err != nil {
59
+
return fmt.Errorf("git checkout: %w", err)
60
+
}
61
+
return nil
62
+
}
63
+
64
+
func isDir(path string) (bool, error) {
65
+
info, err := os.Stat(path)
66
+
if err == nil && info.IsDir() {
67
+
return true, nil
68
+
}
69
+
if os.IsNotExist(err) {
70
+
return false, nil
71
+
}
72
+
return false, err
73
+
}
+32
-1
spindle/server.go
+32
-1
spindle/server.go
···
8
8
"log/slog"
9
9
"maps"
10
10
"net/http"
11
+
"path/filepath"
11
12
"sync"
12
13
13
14
"github.com/bluesky-social/indigo/atproto/syntax"
···
24
25
"tangled.org/core/spindle/db"
25
26
"tangled.org/core/spindle/engine"
26
27
"tangled.org/core/spindle/engines/nixery"
28
+
"tangled.org/core/spindle/git"
27
29
"tangled.org/core/spindle/models"
28
30
"tangled.org/core/spindle/queue"
29
31
"tangled.org/core/spindle/secrets"
···
353
355
}
354
356
355
357
func (s *Spindle) processPipeline(ctx context.Context, src eventconsumer.Source, msg eventconsumer.Message) error {
356
-
if msg.Nsid == tangled.PipelineNSID {
358
+
l := log.FromContext(ctx).With("handler", "processKnotStream")
359
+
l = l.With("src", src.Key(), "msg.Nsid", msg.Nsid, "msg.Rkey", msg.Rkey)
360
+
switch msg.Nsid {
361
+
case tangled.PipelineNSID:
362
+
return nil
357
363
tpl := tangled.Pipeline{}
358
364
err := json.Unmarshal(msg.EventJson, &tpl)
359
365
if err != nil {
···
457
463
} else {
458
464
s.l.Error("failed to enqueue pipeline: queue is full")
459
465
}
466
+
case tangled.GitRefUpdateNSID:
467
+
var event tangled.GitRefUpdate
468
+
if err := json.Unmarshal(msg.EventJson, &event); err != nil {
469
+
return err
470
+
}
471
+
repoDid, err := syntax.ParseDID(event.Repo)
472
+
if err != nil {
473
+
return fmt.Errorf("repo should be valid did: %w", err)
474
+
}
475
+
476
+
l = l.With("repo", repoDid, "ref", event.Ref, "newSha", event.NewSha)
477
+
478
+
repo, err := s.db.GetRepoByDid(repoDid)
479
+
if err != nil {
480
+
return fmt.Errorf("failed to query repo from db: %w", err)
481
+
}
482
+
483
+
repoCloneUri := "" // {knot}/{did}
484
+
repoPath := filepath.Join("<cfg.RepoDir>", string(repo.RepoDid))
485
+
if err := git.SparseSyncGitRepo(ctx, repoCloneUri, repoPath, event.NewSha); err != nil {
486
+
return fmt.Errorf("syncing git repo: %w", err)
487
+
}
488
+
l.Info("synced git repo")
489
+
490
+
// TODO: plan the pipeline
460
491
}
461
492
462
493
return nil
+13
-2
spindle/tapclient.go
+13
-2
spindle/tapclient.go
···
7
7
"errors"
8
8
"fmt"
9
9
"log/slog"
10
+
"path/filepath"
10
11
"sync"
11
12
"time"
12
13
···
16
17
"tangled.org/core/log"
17
18
"tangled.org/core/rbac"
18
19
"tangled.org/core/spindle/db"
20
+
"tangled.org/core/spindle/git"
19
21
"tangled.org/core/tapc"
20
22
)
21
23
···
122
124
src := eventconsumer.NewKnotSource(record.Knot)
123
125
t.spindle.ks.AddSource(t.spindle.rootCtx, src)
124
126
125
-
if err := t.spindle.db.AddRepo(db.Repo{
127
+
repo := db.Repo{
126
128
Knot: record.Knot,
127
129
Owner: ownerDid,
128
130
Rkey: rkey,
129
131
RepoDid: repoDid,
130
132
CreatedAt: record.CreatedAt,
131
-
}); err != nil {
133
+
}
134
+
135
+
if err := t.spindle.db.AddRepo(repo); err != nil {
132
136
l.Error("failed to add repo row", "err", err)
133
137
return fmt.Errorf("add repo: %w", err)
138
+
}
139
+
140
+
// setup sparse sync
141
+
repoCloneUri := "" // {knot}/{did}
142
+
repoPath := filepath.Join("<cfg.RepoDir>", string(repo.RepoDid))
143
+
if err := git.SparseSyncGitRepo(ctx, repoCloneUri, repoPath, ""); err != nil {
144
+
return fmt.Errorf("setting up sparse-clone git repo: %w", err)
134
145
}
135
146
136
147
legacyName := ""