Monorepo for Tangled
wip: appview/pages/markup: markdown normalizer
Issue: goldmark doesn't distinguish link and reflink...
we can't distinguish `[foo](at://bar)` and `[foo][at://bar]`
Signed-off-by: Seongmin Lee <git@boltless.me>
Seongmin Lee
(May 23, 2026, 5:38 PM +0900)
c171080d
3bc3fd40
+233
+5
appview/db/reference.go
+5
appview/db/reference.go
···
11
11
"tangled.org/core/orm"
12
12
)
13
13
14
+
func ValidateAppviewLinks(e Execer, appviewLinks []models.AppviewLink) (map[string]syntax.ATURI, error) {
15
+
// TODO: return url->ATURI map
16
+
panic("unimplemented")
17
+
}
18
+
14
19
// ValidateReferenceLinks resolves refLinks to Issue/PR/Comment ATURIs.
15
20
// It will ignore missing refLinks.
16
21
func ValidateReferenceLinks(e Execer, refLinks []models.ReferenceLink) ([]syntax.ATURI, error) {
+10
appview/models/reference.go
+10
appview/models/reference.go
···
27
27
CommentRkey *syntax.RecordKey
28
28
}
29
29
30
+
type AppviewLink struct {
31
+
Url string
32
+
Owner syntax.AtIdentifier
33
+
OwnerDid syntax.DID // optional resolved DID
34
+
RepoSlug string
35
+
Kind RefKind
36
+
SubjectId int
37
+
CommentRkey *syntax.RecordKey
38
+
}
39
+
30
40
type RichReferenceLink struct {
31
41
ReferenceLink
32
42
Title string
+216
appview/pages/markup/markdown_normalizer.go
+216
appview/pages/markup/markdown_normalizer.go
···
1
+
package markup
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"maps"
7
+
"slices"
8
+
"strings"
9
+
10
+
"github.com/bluesky-social/indigo/atproto/identity"
11
+
"github.com/bluesky-social/indigo/atproto/syntax"
12
+
"github.com/yuin/goldmark"
13
+
"github.com/yuin/goldmark/ast"
14
+
"github.com/yuin/goldmark/extension"
15
+
"github.com/yuin/goldmark/parser"
16
+
"github.com/yuin/goldmark/text"
17
+
"github.com/yuin/goldmark/util"
18
+
"tangled.org/core/appview/db"
19
+
"tangled.org/core/appview/models"
20
+
textension "tangled.org/core/appview/pages/markup/extension"
21
+
"tangled.org/core/sets"
22
+
)
23
+
24
+
// normalize(body string) -> (string, []syntax.DID, []syntax.ATURI, []syntax.CID)
25
+
// @boltless.me -> [did:...]
26
+
// https://tangled.org/tangled.org/core/issues/1 -> [at://...]
27
+
// (additionally return list of blobs referenced in markdown)
28
+
//
29
+
// parse_markdown(string) -> AST -> transform_markdown(body AST) -> (AST)
30
+
// 1. parse markdown (string->AST). reflinks are parsed at this phase
31
+
// 2. transform markdown (AST->AST). we convert blob links here. (pds/porxie + camo)
32
+
// blob links are handled from HTML sanitizer. (to handle both img tag and markdown)
33
+
//
34
+
// NOTE: can we do blob link transformation on render? don't want to touch AST. Markdown should not have an AST.
35
+
// [did:...] -> "@boltless.me"
36
+
// [at://...] -> "#123 my issue"
37
+
//  -> <img>
38
+
//
39
+
// we check if number of referenced blobs match the number of uploaded blobs
40
+
41
+
// custom goldmark parse context to accept any at-uri DID references
42
+
type parseContext struct {
43
+
parser.Context // inner
44
+
}
45
+
46
+
func (p *parseContext) Reference(label string) (parser.Reference, bool) {
47
+
if _, err := syntax.ParseDID(label); err == nil {
48
+
return parser.NewReference([]byte(label), []byte(label), nil), true
49
+
}
50
+
if _, err := syntax.ParseATURI(label); err == nil {
51
+
return parser.NewReference([]byte(label), []byte(label), nil), true
52
+
}
53
+
return p.Context.Reference(label)
54
+
}
55
+
56
+
var _ parser.Context = new(parseContext)
57
+
58
+
var smallParser parser.Parser
59
+
60
+
func init() {
61
+
smallParser = goldmark.DefaultParser()
62
+
smallParser.AddOptions(
63
+
// GFM - TaskList
64
+
parser.WithInlineParsers(util.Prioritized(extension.NewTaskCheckBoxParser(), 0)),
65
+
// GFM - Linkify
66
+
parser.WithInlineParsers(util.Prioritized(extension.NewLinkifyParser(), 999)),
67
+
// GFM - Strikethrough
68
+
parser.WithInlineParsers(util.Prioritized(extension.NewStrikethroughParser(), 500)),
69
+
// GFM - Table
70
+
parser.WithParagraphTransformers(util.Prioritized(extension.NewTableParagraphTransformer(), 200)),
71
+
parser.WithASTTransformers(util.Prioritized(extension.NewTableASTTransformer(), 0)),
72
+
// AtExt
73
+
parser.WithInlineParsers(util.Prioritized(textension.NewAtParser(), 500)),
74
+
)
75
+
}
76
+
77
+
78
+
// @boltless.me -> [did:...]
79
+
// https://tangled.org/tangled.org/core/issues/1 -> [at://...]
80
+
//  -> no-op (additionally return list of blobs referenced in markdown)
81
+
func Normalize(body string) (string, []syntax.DID, []syntax.ATURI, []syntax.CID, error) {
82
+
// TODO: fill these
83
+
var dir identity.Directory
84
+
var ctx context.Context
85
+
var ddb db.Execer
86
+
87
+
sourceBytes := []byte(body)
88
+
89
+
// 1. parse string -> AST (with handles and appviewLinks)
90
+
// 2. walk AST (collect handles, appviewLinks, blob cids)
91
+
// 3. resolve handles & appviewLinks
92
+
// 4. re-walk AST, normalize it.
93
+
94
+
root := smallParser.Parse(
95
+
text.NewReader(sourceBytes),
96
+
parser.WithContext(&parseContext{parser.NewContext()}),
97
+
)
98
+
99
+
var (
100
+
rawMentions = sets.New[syntax.Handle]()
101
+
rawAppviewLinks = sets.New[models.AppviewLink]()
102
+
blobCIDs = sets.New[syntax.CID]()
103
+
)
104
+
if err := ast.Walk(root, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
105
+
if !entering {
106
+
return ast.WalkContinue, nil
107
+
}
108
+
switch n.Kind() {
109
+
case textension.KindAt:
110
+
rawHandle := n.(*textension.AtNode).Handle
111
+
if handle, err := syntax.ParseHandle(rawHandle); err == nil {
112
+
rawMentions.Insert(handle)
113
+
}
114
+
return ast.WalkSkipChildren, nil
115
+
case ast.KindLink:
116
+
panic("unimplemented")
117
+
case ast.KindAutoLink:
118
+
panic("unimplemented")
119
+
case ast.KindImage:
120
+
n := n.(*ast.Image)
121
+
if strings.HasPrefix(string(n.Destination), "blob://") {
122
+
if cid, err := syntax.ParseCID(string(n.Destination[7:])); err != nil {
123
+
blobCIDs.Insert(cid)
124
+
}
125
+
}
126
+
return ast.WalkSkipChildren, nil
127
+
default:
128
+
return ast.WalkContinue, nil
129
+
}
130
+
}); err != nil {
131
+
return body, nil, nil, nil, fmt.Errorf("walking ast: %w", err)
132
+
}
133
+
134
+
// resolve handles & appviewlinks to DID & AT-URI
135
+
var (
136
+
normalized = ""
137
+
mentions = make(map[syntax.Handle]syntax.DID)
138
+
appviewLinks = sets.New[models.AppviewLink]()
139
+
)
140
+
141
+
// resolve all at-identifiers
142
+
for handle := range rawMentions.All() {
143
+
ident, err := dir.LookupHandle(ctx, handle)
144
+
if err != nil || ident == nil || ident.Handle.IsInvalidHandle() {
145
+
continue
146
+
}
147
+
mentions[handle] = ident.DID
148
+
}
149
+
for link := range rawAppviewLinks.All() {
150
+
ident, err := dir.Lookup(ctx, link.Owner)
151
+
if err != nil || ident == nil || ident.Handle.IsInvalidHandle() {
152
+
continue
153
+
}
154
+
link.OwnerDid = ident.DID
155
+
appviewLinks.Insert(link)
156
+
}
157
+
references, err := db.ValidateAppviewLinks(ddb, slices.Collect(appviewLinks.All()))
158
+
if err != nil {
159
+
// no-op
160
+
}
161
+
162
+
// normalize markdown body
163
+
// collect mentions, references and blobs while walking
164
+
// this time, we collect [did:...] and [at://...] as valid mentions and references
165
+
if err := ast.Walk(root, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
166
+
if !entering {
167
+
return ast.WalkContinue, nil
168
+
}
169
+
// TODO: AtExt -> [did:...]
170
+
// TODO: Link -> [at://...]
171
+
// TODO: AutoLink -> [at://...]
172
+
// TODO: RefLink -> collect <- shit, AST doesn't hold reflink???
173
+
// welp, doesn't matter. we collect ANY link which destination is AT-URI
174
+
// for mentions... we only treat links without title.
175
+
switch n.Kind() {
176
+
case textension.KindAt:
177
+
panic("unimplemented")
178
+
case ast.KindLink:
179
+
// if destination is in at-uri map, switch destination to at-uri
180
+
181
+
// but shit, we don't have a way to distinguish link and reflink
182
+
// as we cannot distinguish them, we cannot normalize them.
183
+
184
+
// Link -> "[foo](bar)"? "[foo][bar]"? We don't know
185
+
186
+
187
+
// - [](https://tangled.org/...)
188
+
// - [whatever](https://tangled.org/...)
189
+
// if destination is in appviewLink
190
+
panic("unimplemented")
191
+
case ast.KindAutoLink:
192
+
panic("unimplemented")
193
+
default:
194
+
return ast.WalkContinue, nil
195
+
}
196
+
}); err != nil {
197
+
return body, nil, nil, nil, fmt.Errorf("walking ast: %w", err)
198
+
}
199
+
200
+
201
+
202
+
203
+
// NOTE:
204
+
// it's bad idea to return mentions & references from normalizer
205
+
// because body might be already normalized
206
+
207
+
// But we also walk quite a lot here...
208
+
// so we can just collect mentions & references while re-walking
209
+
210
+
211
+
212
+
213
+
214
+
215
+
return normalized, slices.Collect(maps.Values(mentions)), slices.Collect(maps.Values(references)), slices.Collect(blobCIDs.All()), nil
216
+
}