Monorepo for Tangled
wip: appview,knotmirror repo indexer
- store the language index based on tree, not commit
- index commits with author/committer/co-author emails
- store the language index in knotmirror rather than appview
Signed-off-by: Seongmin Lee <git@boltless.me>
Seongmin Lee
(May 17, 2026, 3:11 AM +0900)
39e79509
050f5a5a
+261
-2
+1
appview/state/knotstream.go
+1
appview/state/knotstream.go
···
98
98
}
99
99
}
100
100
101
+
// TODO(boltless): remove this. knotmirror should do all sort of indexing
101
102
func ingestRefUpdate(ctx context.Context, d *db.DB, enforcer *rbac.Enforcer, pc posthog.Client, notifier notify.Notifier, dev bool, c *config.Config, cfClient *cloudflare.Client, source ec.Source, msg ec.Message) error {
102
103
logger := log.FromContext(ctx)
103
104
+38
knotmirror/db/db.go
+38
knotmirror/db/db.go
···
69
69
constraint hosts_pkey primary key (hostname)
70
70
);
71
71
72
+
-- repo indices
73
+
create table if not exists repo_languages (
74
+
repo_did text not null, -- repo identifier
75
+
commit text not null, -- commit hash
76
+
language text not null,
77
+
size integer not null check (size >= 0),
78
+
79
+
constraint repo_languages_pkey
80
+
primary key (repo_did, commit, language)
81
+
);
82
+
create table if not exists commits (
83
+
repo_did text not null, -- repo identifier
84
+
commit text not null, -- commit hash
85
+
date timestamptz not null, -- commit date
86
+
87
+
constraint commits_pkey
88
+
primary key (repo_did, commit)
89
+
);
90
+
create table if not exists commit_signatures (
91
+
repo_did text not null, -- repo identifier
92
+
commit text not null, -- commit hash
93
+
name text not null, -- user name
94
+
email text not null, -- user email
95
+
reason text not null, -- 'author'/'commiter'/'co-author'/'signed-off-by'
96
+
97
+
constraint commits_emails_pkey
98
+
primary key (repo_did, commit, email),
99
+
constraint commits_emails_fk_commit
100
+
foreign key (repo_did, commit)
101
+
references commits (repo_did, commit)
102
+
on delete cascade
103
+
);
104
+
72
105
create index if not exists idx_repos_aturi on repos (at_uri);
73
106
create index if not exists idx_repos_db_updated_at on repos (db_updated_at desc);
74
107
create index if not exists idx_hosts_db_updated_at on hosts (db_updated_at desc);
108
+
109
+
create index if not exists idx_repo_languages_repo_commit
110
+
on repo_languages (repo_did, commit);
111
+
create index if not exists idx_commits_emails_email_repo_commit
112
+
on commits_emails (email, repo_did, commit);
75
113
76
114
create or replace function set_updated_at()
77
115
returns trigger as $$
+33
knotmirror/db/repo_index.go
+33
knotmirror/db/repo_index.go
···
1
+
package db
2
+
3
+
import (
4
+
"context"
5
+
6
+
"github.com/bluesky-social/indigo/atproto/syntax"
7
+
"github.com/go-git/go-git/v5/plumbing"
8
+
"tangled.org/core/api/tangled"
9
+
)
10
+
11
+
func IsLanguageIndexed(ctx context.Context, repoDid syntax.DID, commit plumbing.Hash) (bool, error) {
12
+
panic("unimplemented")
13
+
}
14
+
15
+
func InsertLanguages(ctx context.Context, repoDid syntax.DID, commit plumbing.Hash, langs []tangled.GitTempListLanguages_Language) error {
16
+
panic("unimplemented")
17
+
}
18
+
19
+
func ListLanguages(ctx context.Context, repoDid syntax.DID, commit plumbing.Hash) ([]*tangled.GitTempListLanguages_Language, error) {
20
+
panic("unimplemented")
21
+
}
22
+
23
+
// TODO: use struct for an email. to represent as (email, reason) pair.
24
+
// where reason can be "committer", "author", "co-author", "signed-off"
25
+
26
+
func IsCommitIndexed(ctx context.Context, repoDid syntax.DID, commit plumbing.Hash) (bool, error) {
27
+
panic("unimplemented")
28
+
}
29
+
30
+
// TODO: include authored/committed dates alongside with author/committer emails
31
+
func InsertCommit(ctx context.Context, repoDid syntax.DID, commit plumbing.Hash, emails []string) error {
32
+
panic("unimplemented")
33
+
}
+163
knotmirror/indexer.go
+163
knotmirror/indexer.go
···
1
1
package knotmirror
2
2
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"io"
7
+
"strings"
8
+
9
+
"github.com/bluesky-social/indigo/atproto/syntax"
10
+
"github.com/go-enry/go-enry/v2"
11
+
"github.com/go-git/go-git/v5"
12
+
"github.com/go-git/go-git/v5/plumbing"
13
+
"github.com/go-git/go-git/v5/plumbing/object"
14
+
"tangled.org/core/knotmirror/db"
15
+
)
16
+
17
+
const LangIndexCap = 16*1024 // 16KB
18
+
19
+
func repoPath(uri syntax.DID) string {
20
+
panic("unimplemented")
21
+
}
3
22
4
23
// HEAD -> store to db
5
24
// other -> on-demand calculation, cache
···
24
43
// 1. resolve current HEAD
25
44
// 2. index language stats in current HEAD
26
45
// 3. store db (ofc, cache it too)
46
+
47
+
func IndexLanguages(ctx context.Context, repo syntax.DID, commit plumbing.Hash) (map[string]int64, error) {
48
+
gr, err := git.PlainOpen(repoPath(repo))
49
+
if err != nil {
50
+
panic("unimplemented")
51
+
}
52
+
c, err := gr.CommitObject(commit)
53
+
if err != nil {
54
+
panic("unimplemented")
55
+
}
56
+
57
+
// check if (uri,tree) is already indexed
58
+
indexed, err := db.IsLanguageIndexed(ctx, repo, c.TreeHash)
59
+
if err != nil {
60
+
panic("unimplemented")
61
+
}
62
+
if indexed {
63
+
return nil, nil
64
+
}
65
+
66
+
tree, err := c.Tree()
67
+
if err != nil {
68
+
panic("unimplemented")
69
+
}
70
+
71
+
sizes := make(map[string]int64)
72
+
73
+
err = tree.Files().ForEach(func(file *object.File) error {
74
+
// skip binary file
75
+
if isbin, _ := file.IsBinary(); isbin {
76
+
return nil
77
+
}
78
+
79
+
// skip large file
80
+
if file.Size > LangIndexCap {
81
+
return nil
82
+
}
83
+
84
+
reader, err := file.Reader()
85
+
if err != nil {
86
+
return err
87
+
}
88
+
defer reader.Close()
89
+
content, err := io.ReadAll(reader)
90
+
if err != nil {
91
+
return nil
92
+
}
93
+
94
+
// skip generated file
95
+
// TODO: follow gitattributes
96
+
if enry.IsGenerated(file.Name, content) ||
97
+
strings.HasSuffix(file.Name, "bun.lock") {
98
+
return nil
99
+
}
100
+
101
+
language := analyzeLanguage(file.Name, content)
102
+
if group := enry.GetLanguageGroup(language); group != "" {
103
+
language = group
104
+
}
105
+
106
+
langType := enry.GetLanguageType(language)
107
+
if langType != enry.Programming && langType != enry.Markup && langType != enry.Unknown {
108
+
return nil
109
+
}
110
+
111
+
sizes[language] += file.Size
112
+
113
+
return nil
114
+
})
115
+
if err != nil {
116
+
panic("unimplemented")
117
+
}
118
+
119
+
panic("unimplemented")
120
+
}
121
+
122
+
func analyzeLanguage(path string, content []byte) string {
123
+
language, ok := enry.GetLanguageByExtension(path)
124
+
if ok {
125
+
return language
126
+
}
127
+
128
+
language, ok = enry.GetLanguageByFilename(path)
129
+
if ok {
130
+
return language
131
+
}
132
+
133
+
if len(content) == 0 {
134
+
return enry.OtherLanguage
135
+
}
136
+
137
+
return enry.GetLanguage(path, content)
138
+
}
139
+
140
+
func IndexCommits(ctx context.Context, repo syntax.DID, commit plumbing.Hash) error {
141
+
// TODO: index (uri,commit-hash)->{date,[]emails}
142
+
// start from given commit, walk through DAG
143
+
// for each commits, check if we've indexed it & perform index
144
+
// 1. has (uri,commit) been indexed?
145
+
// 2. index it
146
+
gr, err := git.PlainOpen(repoPath(repo))
147
+
if err != nil {
148
+
panic("unimplemented")
149
+
}
150
+
151
+
_, err = indexCommits(ctx, gr, repo, commit)
152
+
if err != nil {
153
+
panic("unimplemented")
154
+
}
155
+
156
+
panic("unimplemented")
157
+
}
158
+
159
+
// indexCommits indexes a commit and all of its ancestors
160
+
func indexCommits(ctx context.Context, gitrepo *git.Repository, repo syntax.DID, commit plumbing.Hash) (int, error) {
161
+
isIndexed, err := db.IsCommitIndexed(ctx, repo, commit)
162
+
if err != nil {
163
+
panic("unimplemented")
164
+
}
165
+
if isIndexed {
166
+
return 0, nil
167
+
}
168
+
169
+
c, err := gitrepo.CommitObject(commit)
170
+
if err != nil {
171
+
return 0, fmt.Errorf("commit '%s' not found", commit)
172
+
}
173
+
174
+
emails := []string{c.Committer.Email}
175
+
176
+
if err := db.InsertCommit(ctx, repo, commit, emails); err != nil {
177
+
return 0, fmt.Errorf("db: failed to insert commit '%s'", commit)
178
+
}
179
+
180
+
var count int
181
+
for _, pc := range c.ParentHashes {
182
+
count, err = indexCommits(ctx, gitrepo, repo, pc)
183
+
if err != nil {
184
+
return 1, err
185
+
}
186
+
}
187
+
188
+
return 1 + count, nil
189
+
}
+26
-2
knotmirror/xrpc/git_list_languages.go
+26
-2
knotmirror/xrpc/git_list_languages.go
···
11
11
"github.com/bluesky-social/indigo/atproto/atclient"
12
12
"github.com/bluesky-social/indigo/atproto/syntax"
13
13
"tangled.org/core/api/tangled"
14
+
"tangled.org/core/knotmirror/db"
15
+
"tangled.org/core/knotmirror/xrpc/gitea"
14
16
"tangled.org/core/knotserver/git"
15
17
)
16
18
···
34
36
return
35
37
}
36
38
37
-
if val, err := x.rdb.Get(r.Context(), fmt.Sprintf(RepoLanguagesByDid, repo, ref)).Result(); err == nil {
39
+
ctx := r.Context()
40
+
41
+
// TODO: wrap everything below with x.indexer.ListLanguages (include db fetch & cache handling)
42
+
43
+
repoPath, err := x.makeRepoPath(ctx, repo)
44
+
if err != nil {
45
+
panic("unimplemented")
46
+
}
47
+
commit, err := gitea.GetCommit(ctx, repoPath, ref)
48
+
if err != nil {
49
+
panic("unimplemented")
50
+
}
51
+
52
+
// serve from pre-indexed stats
53
+
if langs, err := db.ListLanguages(ctx, repo, commit.Hash); err == nil {
54
+
writeJson(w, http.StatusOK, &tangled.GitTempListLanguages_Output{
55
+
Ref: "",
56
+
Total: 0, // do we even need a total?
57
+
Languages: langs,
58
+
})
59
+
}
60
+
61
+
if val, err := x.rdb.Get(ctx, fmt.Sprintf(RepoLanguagesByDid, repo, ref)).Result(); err == nil {
38
62
l.Debug("served from cache")
39
63
var langs []*tangled.GitTempListLanguages_Language
40
64
err = json.Unmarshal([]byte(val), &langs)
···
48
72
}
49
73
50
74
var out *tangled.GitTempListLanguages_Output
51
-
out, err = x.listLanguages(r.Context(), repo, ref)
75
+
out, err = x.listLanguages(ctx, repo, ref)
52
76
if err != nil {
53
77
l.Warn("local mirror failed, trying proxy", "err", err)
54
78
if x.proxyToKnot(w, r, repo) {