Decode the protobuf wire format in Gleam!
Support packed values + parsers return `Parsed`
Gavin Morrow
(Sep 22, 2025, 8:57 PM EDT)
853fa452
06b4df7d
+82
-16
+58
-2
src/decoders.gleam
+58
-2
src/decoders.gleam
···
1
1
import gleam/bit_array
2
+
import gleam/dynamic
2
3
import gleam/dynamic/decode.{type Decoder}
4
+
import gleam/int
3
5
import gleam/list
6
+
import gleam/result
4
7
5
8
import internal/util
6
-
import protobuf_decode_gleam.{parse}
9
+
import protobuf_decode_gleam.{
10
+
type BytePos, type DecodeResult, type ValueParser, Parsed, UnableToDecode,
11
+
parse,
12
+
}
13
+
14
+
/// Decode a repeated field that may be either packed or expanded.
15
+
/// If the field is known ahead of time to be expanded, use the stdlib's
16
+
/// `decode.list()`.
17
+
pub fn multiple(
18
+
of decoder: Decoder(t),
19
+
using parser: ValueParser,
20
+
) -> Decoder(List(t)) {
21
+
use values <- decode.then(
22
+
decode.one_of(decode.list(of: decoder), or: [
23
+
packed_values(of: decoder, using: parser),
24
+
]),
25
+
)
26
+
values |> decode.success
27
+
}
28
+
29
+
fn packed_values(
30
+
of decoder: Decoder(t),
31
+
using parser: ValueParser,
32
+
) -> Decoder(List(t)) {
33
+
use bits <- decode.then(decode.bit_array)
34
+
35
+
let values = {
36
+
use values <- result.try(unpack_bits(bits, [], at: 0, using: parser))
37
+
values
38
+
|> list.map(dynamic.bit_array)
39
+
|> list.try_map(fn(value) { decode.run(value, decoder) })
40
+
|> result.map_error(UnableToDecode)
41
+
}
42
+
43
+
case values {
44
+
Ok(values) -> decode.success(values)
45
+
Error(_) -> decode.failure([], "Packed values")
46
+
}
47
+
}
48
+
49
+
fn unpack_bits(
50
+
bits: BitArray,
51
+
acc: List(BitArray),
52
+
at pos: BytePos,
53
+
using parser: ValueParser,
54
+
) -> DecodeResult(List(BitArray)) {
55
+
case bits {
56
+
<<>> -> acc |> list.reverse |> Ok
57
+
bits -> {
58
+
use Parsed(value:, rest:, pos:) <- result.try(parser(bits, pos))
59
+
unpack_bits(rest, [value, ..acc], at: pos, using: parser)
60
+
}
61
+
}
62
+
}
7
63
8
64
fn single(of decoder: Decoder(t)) -> Decoder(t) {
9
65
use values <- decode.then(decode.list(of: decoder))
···
32
88
33
89
let value = parse(from: bits, using: decoder())
34
90
case value {
35
-
Ok(value) -> decode.success(value)
91
+
Ok(Parsed(value:, ..)) -> decode.success(value)
36
92
Error(_) -> decode.failure(default, name)
37
93
}
38
94
}
+17
-11
src/protobuf_decode_gleam.gleam
+17
-11
src/protobuf_decode_gleam.gleam
···
13
13
pub fn parse(
14
14
from bits: BitArray,
15
15
using decoder: Decoder(t),
16
-
) -> Result(t, ParseError) {
17
-
use data <- result.try(read_fields(bits, [], 0))
18
-
decode.run(data, decoder) |> result.map_error(UnableToDecode)
16
+
) -> DecodeResult(Parsed(t)) {
17
+
use Parsed(value: data, rest:, pos:) <- result.try(read_fields(bits, [], 0))
18
+
decode.run(data, decoder)
19
+
|> result.map(fn(value) { Parsed(value:, rest:, pos:) })
20
+
|> result.map_error(UnableToDecode)
19
21
}
20
22
21
23
pub type ParseError {
···
30
32
bits: BitArray,
31
33
acc: List(Field),
32
34
pos: BytePos,
33
-
) -> Result(Dynamic, ParseError) {
35
+
) -> DecodeResult(Parsed(Dynamic)) {
34
36
case bits {
35
37
<<>> ->
36
38
acc
37
39
|> repeated_to_list
38
40
|> dict.to_list
39
41
|> dynamic.properties
42
+
|> Parsed(value: _, rest: bits, pos:)
40
43
|> Ok
41
44
bits -> {
42
45
use Parsed(value: prop, rest: bits, pos:) <- result.try(read_field(
···
70
73
})
71
74
}
72
75
73
-
type DecodeResult(t) =
76
+
pub type DecodeResult(t) =
74
77
Result(t, ParseError)
75
78
76
-
type BytePos =
79
+
pub type BytePos =
77
80
Int
78
81
79
-
type Parsed(t) {
82
+
pub type Parsed(t) {
80
83
Parsed(value: t, rest: BitArray, pos: BytePos)
81
84
}
82
85
···
89
92
Field(key: Dynamic, value: Dynamic)
90
93
}
91
94
92
-
fn wire_type_read_fn(ty: WireType) -> fn(BitArray, BytePos) -> ValueResult {
95
+
fn wire_type_read_fn(ty: WireType) -> ValueParser {
93
96
case ty {
94
97
wire_type.VarInt -> read_varint
95
98
wire_type.I64 -> read_fixed(64)
···
132
135
Ok(field)
133
136
}
134
137
135
-
type ValueResult =
138
+
pub type ValueResult =
136
139
DecodeResult(Parsed(BitArray))
137
140
138
-
fn read_varint(bits: BitArray, pos: BytePos) -> ValueResult {
141
+
pub type ValueParser =
142
+
fn(BitArray, BytePos) -> ValueResult
143
+
144
+
pub fn read_varint(bits: BitArray, pos: BytePos) -> ValueResult {
139
145
read_varint_acc(bits, <<>>, pos)
140
146
}
141
147
···
152
158
}
153
159
154
160
/// Size must be a multiple of 8.
155
-
fn read_fixed(size: Int) -> fn(BitArray, BytePos) -> ValueResult {
161
+
pub fn read_fixed(size: Int) -> ValueParser {
156
162
assert size % 8 == 0
157
163
fn(bits: BitArray, pos: BytePos) -> ValueResult {
158
164
case bits {
+7
-3
test/protobuf_decode_gleam_test.gleam
+7
-3
test/protobuf_decode_gleam_test.gleam
···
4
4
import simplifile as file
5
5
6
6
import decoders
7
-
import protobuf_decode_gleam.{parse}
7
+
import protobuf_decode_gleam.{Parsed, parse}
8
8
9
9
pub fn main() -> Nil {
10
10
gleeunit.main()
···
45
45
46
46
pub fn person_pb_test() {
47
47
let path = "./test/person.pb"
48
+
48
49
let assert Ok(bits) = file.read_bits(from: path)
49
-
let assert Ok(person) = parse(from: bits, using: person_decoder())
50
+
let assert Ok(Parsed(value: person, rest: <<>>, pos: _)) =
51
+
parse(from: bits, using: person_decoder())
52
+
50
53
assert person
51
54
== Person(
52
55
id: 42,
···
87
90
0x04,
88
91
>>
89
92
90
-
let assert Ok(data) = parse(from: bits, using: two_ints_decoder())
93
+
let assert Ok(Parsed(value: data, rest: <<>>, pos: 7)) =
94
+
parse(from: bits, using: two_ints_decoder())
91
95
assert data == Test(id: 150, age: 80_150)
92
96
}
93
97