Monorepo for Tangled
wip: appview/pages: markdown normalizer
markdown normalizer will _normalize_ mentions & references into
persistent decentralized identifiers.
```
@alice.com -> [did:plc:alice]
https://tangled.org/tangled.org/core/pulls/123 -> [at://...]
```
normalized content will be stored under `.text` in
`sh.tangled.markup.markdown` type while orininal content will be kept
under `.original`.
This will allow understanding references while ingesting from
third-party appviews
Signed-off-by: Seongmin Lee <git@boltless.me>
Seongmin Lee
(May 10, 2026, 12:45 AM +0900)
67efa863
040c9272
+467
-7
+6
-5
appview/pages/markup/extension/atlink.go
+6
-5
appview/pages/markup/extension/atlink.go
···
16
16
17
17
// An AtNode struct represents an AtNode
18
18
type AtNode struct {
19
+
ast.BaseInline
20
+
Segment text.Segment
19
21
Handle string
20
-
ast.BaseInline
21
22
}
22
23
23
24
var _ ast.Node = &AtNode{}
···
67
68
68
69
atSegment := text.NewSegment(segment.Start, segment.Start+m[1])
69
70
block.Advance(m[1])
70
-
node := &AtNode{}
71
-
node.AppendChild(node, ast.NewTextSegment(atSegment))
72
-
node.Handle = string(atSegment.Value(block.Source())[1:])
73
-
return node
71
+
return &AtNode{
72
+
Segment: atSegment,
73
+
Handle: string(atSegment.Value(block.Source())[1:]),
74
+
}
74
75
}
75
76
76
77
// atHtmlRenderer is a renderer.NodeRenderer implementation that
+75
appview/pages/markup/extension/reflink.go
+75
appview/pages/markup/extension/reflink.go
···
1
+
package extension
2
+
3
+
import (
4
+
"github.com/bluesky-social/indigo/atproto/syntax"
5
+
"github.com/yuin/goldmark/ast"
6
+
"github.com/yuin/goldmark/parser"
7
+
"github.com/yuin/goldmark/text"
8
+
"tangled.org/core/appview/models"
9
+
)
10
+
11
+
// KindCustomLink is a NodeKind of the [CustomLinkNode].
12
+
var KindCustomLink = ast.NewNodeKind("CustomLink")
13
+
14
+
type CustomLinkNode struct {
15
+
ast.BaseInline
16
+
17
+
// union type
18
+
Commit *TangledCommitLink
19
+
Issue *TangledIssueLink
20
+
Pull *TangledPullLink
21
+
}
22
+
23
+
type TangledIssueLink struct {
24
+
Id int
25
+
Title string
26
+
IsOpen bool
27
+
}
28
+
29
+
type TangledPullLink struct {
30
+
Id int
31
+
Title string
32
+
State models.PullState
33
+
}
34
+
35
+
var _ ast.Node = (*CustomLinkNode)(nil)
36
+
37
+
// Dump implements [ast.Node].
38
+
func (n *CustomLinkNode) Dump(source []byte, level int) {
39
+
ast.DumpHelper(n, source, level, nil, nil)
40
+
}
41
+
42
+
// Kind implements [ast.Node].
43
+
func (n *CustomLinkNode) Kind() ast.NodeKind {
44
+
return KindTangledLink
45
+
}
46
+
47
+
type refLinkTransformer struct{}
48
+
49
+
var _ parser.ASTTransformer = (*refLinkTransformer)(nil)
50
+
51
+
// Transform implements [parser.ASTTransformer].
52
+
func (t *refLinkTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) {
53
+
ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
54
+
if !entering {
55
+
return ast.WalkContinue, nil
56
+
}
57
+
58
+
switch n.Kind() {
59
+
case ast.KindLink:
60
+
node := n.(*ast.Link)
61
+
dest := node.Destination
62
+
if did, err := syntax.ParseDID(string(dest)); err != nil {
63
+
// pc.Get(
64
+
}
65
+
}
66
+
67
+
// 1. find links that has DID
68
+
// 2. check if pc has the resolved object
69
+
// 3. replace to renderable AtUriLink nodes (PR id, title, state etc)
70
+
71
+
return ast.WalkContinue, nil
72
+
})
73
+
}
74
+
75
+
// TODO: renderer
-1
appview/pages/markup/markdown.go
-1
appview/pages/markup/markdown.go
+224
appview/pages/markup/normalizer/normalizer.go
+224
appview/pages/markup/normalizer/normalizer.go
···
1
+
package normalizer
2
+
3
+
import (
4
+
"context"
5
+
"net/url"
6
+
"path"
7
+
"slices"
8
+
"strconv"
9
+
"strings"
10
+
11
+
"github.com/bluesky-social/indigo/atproto/identity"
12
+
"github.com/bluesky-social/indigo/atproto/syntax"
13
+
"github.com/yuin/goldmark"
14
+
"github.com/yuin/goldmark/ast"
15
+
"github.com/yuin/goldmark/extension"
16
+
"github.com/yuin/goldmark/parser"
17
+
"github.com/yuin/goldmark/text"
18
+
"tangled.org/core/appview/db"
19
+
"tangled.org/core/appview/models"
20
+
textension "tangled.org/core/appview/pages/markup/extension"
21
+
"tangled.org/core/sets"
22
+
)
23
+
24
+
type MarkdownNormalizer struct {
25
+
hostname string
26
+
dir identity.Directory
27
+
db db.Execer
28
+
}
29
+
30
+
func NewMarkdownNormalizer(hostname string, dir identity.Directory) *MarkdownNormalizer {
31
+
return &MarkdownNormalizer{
32
+
hostname: hostname,
33
+
dir: dir,
34
+
}
35
+
}
36
+
37
+
var markdownParser parser.Parser
38
+
39
+
func init() {
40
+
markdownParser = goldmark.New(
41
+
goldmark.WithExtensions(
42
+
extension.GFM,
43
+
extension.NewFootnote(
44
+
extension.WithFootnoteIDPrefix([]byte("footnote")),
45
+
),
46
+
textension.AtExt,
47
+
),
48
+
).Parser()
49
+
}
50
+
51
+
// 1. parse markdown with AtExt
52
+
// 2. walk AST
53
+
// - normalize AtExt segments -> [did:...] (when resolvable)
54
+
// @alice.com -> [did:plc:alice]
55
+
// - normalize Tangled-Link segments
56
+
// <https://...> -> `[at://...]`
57
+
58
+
// from render step:
59
+
// 1. parse normalized version with full-parser & custom parser.Context
60
+
// custom parser.Context will resolve custom ref-links and collect them into mentions & references
61
+
// 2. with resolved links, start render phase. we should know all at-uris at this phase.
62
+
63
+
// Normalize normalizes markdown body
64
+
//
65
+
// @alice.com -> [did:plc:alice]
66
+
// #123 -> [at://...]
67
+
// [link](https://tangled.org/...) -> [link](at://...)
68
+
//
69
+
// NOTE: last two cases are not implemented yet.
70
+
func (m *MarkdownNormalizer) Normalize(ctx context.Context, source []byte) ([]byte, []syntax.DID, []syntax.ATURI) {
71
+
var (
72
+
normalizedSource []byte
73
+
mentions = sets.New[syntax.DID]()
74
+
rawRefLinks = sets.New[models.ReferenceLink]()
75
+
)
76
+
77
+
root := markdownParser.Parse(text.NewReader(source))
78
+
cursor := 0
79
+
ast.Walk(root, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
80
+
if !entering {
81
+
return ast.WalkContinue, nil
82
+
}
83
+
84
+
switch n.Kind() {
85
+
case textension.KindAt:
86
+
node := n.(*textension.AtNode)
87
+
88
+
start := node.Segment.Start
89
+
stop := node.Segment.Stop
90
+
91
+
if cursor > start {
92
+
break
93
+
}
94
+
95
+
ident, err := m.dir.LookupHandle(ctx, syntax.Handle(node.Handle))
96
+
if err != nil {
97
+
return ast.WalkContinue, nil
98
+
}
99
+
100
+
mentions.Insert(ident.DID)
101
+
102
+
normalizedSource = append(normalizedSource, source[cursor:start]...)
103
+
normalizedSource = append(normalizedSource, byte('['))
104
+
normalizedSource = append(normalizedSource, []byte(ident.DID.String())...)
105
+
normalizedSource = append(normalizedSource, byte(']'))
106
+
107
+
cursor = stop
108
+
return ast.WalkSkipChildren, nil
109
+
110
+
// case ast.HashRef:
111
+
// node := n.(*textension.HashRefNode)
112
+
// // TODO: normalize to at-uri
113
+
// // example: [at://...]
114
+
// return ast.WalkSkipChildren, nil
115
+
116
+
case ast.KindLink:
117
+
node := n.(*ast.Link)
118
+
dest := string(node.Destination)
119
+
if ref := parseTangledLink(m.hostname, dest); ref != nil {
120
+
rawRefLinks.Insert(*ref)
121
+
}
122
+
123
+
// TODO: convert dest to at-uri
124
+
// example: [link](at://...)
125
+
126
+
return ast.WalkSkipChildren, nil
127
+
128
+
case ast.KindAutoLink:
129
+
node := n.(*ast.AutoLink)
130
+
if node.AutoLinkType == ast.AutoLinkURL {
131
+
dest := string(node.URL(source))
132
+
if ref := parseTangledLink(m.hostname, dest); ref != nil {
133
+
rawRefLinks.Insert(*ref)
134
+
}
135
+
}
136
+
137
+
// TODO: convert dest to at-uri
138
+
// example: <at://...>
139
+
140
+
return ast.WalkSkipChildren, nil
141
+
}
142
+
143
+
return ast.WalkContinue, nil
144
+
})
145
+
if cursor < len(source) {
146
+
normalizedSource = append(normalizedSource, source[cursor:]...)
147
+
}
148
+
149
+
var resolvedRefs []models.ReferenceLink
150
+
for rawRef := range rawRefLinks.All() {
151
+
ident, err := m.dir.Lookup(ctx, syntax.AtIdentifier(rawRef.Handle))
152
+
if err != nil || ident == nil {
153
+
continue
154
+
}
155
+
rawRef.Handle = string(ident.DID)
156
+
resolvedRefs = append(resolvedRefs, rawRef)
157
+
}
158
+
159
+
// TODO: ideally tangled-uri should be universally resolvable and should not require DB query.
160
+
// We are querying DB here because issue/pull-ids are not decentralized yet.
161
+
references, err := db.ValidateReferenceLinks(m.db, resolvedRefs)
162
+
if err != nil {
163
+
}
164
+
165
+
return normalizedSource, slices.Collect(mentions.All()), references
166
+
}
167
+
168
+
func parseTangledLink(baseHost string, urlStr string) *models.ReferenceLink {
169
+
u, err := url.Parse(urlStr)
170
+
if err != nil {
171
+
return nil
172
+
}
173
+
if u.Host != "" && !strings.EqualFold(u.Host, baseHost) {
174
+
return nil
175
+
}
176
+
177
+
p := path.Clean(u.Path)
178
+
parts := strings.FieldsFunc(p, func(r rune) bool { return r == '/' })
179
+
if len(parts) < 4 {
180
+
// need at least: atIdentifier / repoName / kind / id
181
+
return nil
182
+
}
183
+
184
+
var (
185
+
atIdentifier = parts[0]
186
+
repoName = parts[1]
187
+
kindSeg = parts[2]
188
+
subjectSeg = parts[3]
189
+
)
190
+
191
+
atIdentifier = strings.TrimPrefix(atIdentifier, "@")
192
+
193
+
var kind models.RefKind
194
+
switch kindSeg {
195
+
case "issues":
196
+
kind = models.RefKindIssue
197
+
case "pulls":
198
+
kind = models.RefKindPull
199
+
default:
200
+
return nil
201
+
}
202
+
203
+
subjectId, err := strconv.Atoi(subjectSeg)
204
+
if err != nil {
205
+
return nil
206
+
}
207
+
var commentId *int
208
+
if u.Fragment != "" {
209
+
if strings.HasPrefix(u.Fragment, "comment-") {
210
+
commentIdStr := u.Fragment[len("comment-"):]
211
+
if id, err := strconv.Atoi(commentIdStr); err == nil {
212
+
commentId = &id
213
+
}
214
+
}
215
+
}
216
+
217
+
return &models.ReferenceLink{
218
+
Handle: atIdentifier,
219
+
Repo: repoName,
220
+
Kind: kind,
221
+
SubjectId: subjectId,
222
+
CommentId: commentId,
223
+
}
224
+
}
+33
appview/pages/markup/normalizer/normalizer_test.go
+33
appview/pages/markup/normalizer/normalizer_test.go
···
1
+
package normalizer
2
+
3
+
import (
4
+
"context"
5
+
"testing"
6
+
7
+
"github.com/bluesky-social/indigo/atproto/identity"
8
+
"github.com/bluesky-social/indigo/atproto/syntax"
9
+
"github.com/stretchr/testify/assert"
10
+
)
11
+
12
+
func TestNormalize(t *testing.T) {
13
+
mockDir := identity.NewMockDirectory()
14
+
mockDir.Insert(identity.Identity{DID: "did:plc:alice", Handle: "alice.tngl.sh"})
15
+
16
+
norm := NewMarkdownNormalizer("hostname", mockDir)
17
+
18
+
ctx := context.Background()
19
+
20
+
t.Run("normalize at-mention", func(t *testing.T) {
21
+
source := "hello @alice.tngl.sh!"
22
+
result, mentions, _ := norm.Normalize(ctx, []byte(source))
23
+
assert.Equal(t, "hello [did:plc:alice]!", string(result))
24
+
assert.Equal(t, []syntax.DID{"did:plc:alice"}, mentions)
25
+
})
26
+
27
+
t.Run("don't normalize invalid at-mention", func(t *testing.T) {
28
+
source := "hello @alice!"
29
+
result, mentions, _ := norm.Normalize(ctx, []byte(source))
30
+
assert.Equal(t, "hello @alice!", string(result))
31
+
assert.Equal(t, 0, len(mentions))
32
+
})
33
+
}
+45
appview/pages/markup/parser/parser.go
+45
appview/pages/markup/parser/parser.go
···
1
+
package parser
2
+
3
+
import (
4
+
"github.com/bluesky-social/indigo/atproto/syntax"
5
+
gparser "github.com/yuin/goldmark/parser"
6
+
)
7
+
8
+
// parseContext is custom gomark [gparser.Context] that resolves DID or AT-URI references
9
+
type parseContext struct {
10
+
gparser.Context
11
+
}
12
+
13
+
// Reference implements [gparser.Context]
14
+
func (p *parseContext) Reference(label string) (gparser.Reference, bool) {
15
+
if ref, ok := p.Context.Reference(label); ok {
16
+
return ref, ok
17
+
}
18
+
if _, err := syntax.ParseDID(label); err != nil {
19
+
ref := gparser.NewReference([]byte(label), []byte(label), nil)
20
+
p.Context.AddReference(ref)
21
+
return ref, true
22
+
}
23
+
if _, err := syntax.ParseATURI(label); err != nil {
24
+
ref := gparser.NewReference([]byte(label), []byte(label), nil)
25
+
p.Context.AddReference(ref)
26
+
return ref, true
27
+
}
28
+
return nil, false
29
+
}
30
+
31
+
func (p *parseContext) Mentions() []syntax.DID {
32
+
return p.Get(mentionListKey).([]syntax.DID)
33
+
}
34
+
35
+
var _ gparser.Context = (*parseContext)(nil)
36
+
37
+
var mentionListKey = gparser.NewContextKey()
38
+
var referenceListKey = gparser.NewContextKey()
39
+
40
+
func NewParseContext(inner gparser.Context) gparser.Context {
41
+
pc := &parseContext{Context: inner}
42
+
pc.Set(mentionListKey, []syntax.DID{})
43
+
pc.Set(referenceListKey, []syntax.ATURI{})
44
+
return pc
45
+
}
+84
appview/pages/markup/parser/reflink.go
+84
appview/pages/markup/parser/reflink.go
···
1
+
package parser
2
+
3
+
import (
4
+
"github.com/bluesky-social/indigo/atproto/syntax"
5
+
"github.com/yuin/goldmark/ast"
6
+
"github.com/yuin/goldmark/parser"
7
+
"github.com/yuin/goldmark/text"
8
+
"tangled.org/core/appview/models"
9
+
)
10
+
11
+
// KindCustomLink is a NodeKind of the [CustomLinkNode].
12
+
var KindCustomLink = ast.NewNodeKind("CustomLink")
13
+
14
+
type CustomLinkNode struct {
15
+
ast.BaseInline
16
+
17
+
// union type
18
+
Mention *TangledMentionLink
19
+
Commit *TangledCommitLink
20
+
Issue *TangledIssueLink
21
+
Pull *TangledPullLink
22
+
}
23
+
24
+
type TangledMentionLink struct {
25
+
Handle syntax.Handle
26
+
}
27
+
28
+
type TangledCommitLink struct {
29
+
// TODO:
30
+
}
31
+
32
+
type TangledIssueLink struct {
33
+
Id int
34
+
Title string
35
+
IsOpen bool
36
+
}
37
+
38
+
type TangledPullLink struct {
39
+
Id int
40
+
Title string
41
+
State models.PullState
42
+
}
43
+
44
+
var _ ast.Node = (*CustomLinkNode)(nil)
45
+
46
+
// Dump implements [ast.Node].
47
+
func (n *CustomLinkNode) Dump(source []byte, level int) {
48
+
ast.DumpHelper(n, source, level, nil, nil)
49
+
}
50
+
51
+
// Kind implements [ast.Node].
52
+
func (n *CustomLinkNode) Kind() ast.NodeKind {
53
+
return KindCustomLink
54
+
}
55
+
56
+
type refLinkTransformer struct{}
57
+
58
+
var _ parser.ASTTransformer = (*refLinkTransformer)(nil)
59
+
60
+
// Transform implements [parser.ASTTransformer].
61
+
func (t *refLinkTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) {
62
+
ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
63
+
if !entering {
64
+
return ast.WalkContinue, nil
65
+
}
66
+
67
+
switch n.Kind() {
68
+
case ast.KindLink:
69
+
node := n.(*ast.Link)
70
+
dest := node.Destination
71
+
if did, err := syntax.ParseDID(string(dest)); err != nil {
72
+
list := pc.Get(mentionListKey).([]syntax.DID)
73
+
list = append(list, did)
74
+
}
75
+
return ast.WalkSkipChildren, nil
76
+
}
77
+
78
+
// 1. find links that has DID
79
+
// 2. check if pc has the resolved object
80
+
// 3. replace to renderable AtUriLink nodes (PR id, title, state etc)
81
+
82
+
return ast.WalkContinue, nil
83
+
})
84
+
}