Decode the protobuf wire format in Gleam!
packed fields
Gavin Morrow
(Sep 28, 2025, 9:04 PM EDT)
65cc35ed
dec19769
+93
-27
+16
-12
src/decoders.gleam
+16
-12
src/decoders.gleam
···
7
7
8
8
import internal/util
9
9
import protobuf_decode_gleam.{
10
-
type BytePos, type DecodeResult, type ValueParser, Parsed, UnableToDecode,
11
-
parse,
10
+
type BytePos, type DecodeResult, type ValueParser, Parsed, parse, read_varint,
12
11
}
13
12
14
13
/// Decode a repeated field that may be either packed or expanded.
15
-
/// If the field is known ahead of time to be expanded, use the stdlib's
16
-
/// `decode.list()`.
14
+
///
15
+
/// If it is impossible for a field to be expanded (ie because it is encoded as
16
+
/// a `LEN`) and therefore there is no `ValueParser` for it, then use the
17
+
/// stdlib's `decode.list()` instead.
17
18
pub fn multiple(
18
19
of decoder: Decoder(t),
19
20
using parser: ValueParser,
20
21
) -> Decoder(List(t)) {
21
-
use values <- decode.then(
22
-
decode.one_of(decode.list(of: decoder), or: [
23
-
packed_values(of: decoder, using: parser),
24
-
]),
25
-
)
22
+
use values <- decode.then(packed_values(of: decoder, using: parser))
26
23
values |> decode.success
27
24
}
28
25
···
30
27
of decoder: Decoder(t),
31
28
using parser: ValueParser,
32
29
) -> Decoder(List(t)) {
33
-
use bits <- decode.then(decode.bit_array)
30
+
use bits <- decode.then(decode.list(of: decode.bit_array))
31
+
let bits =
32
+
list.fold(bits, <<>>, fn(acc, elem) { bit_array.concat([acc, elem]) })
34
33
35
34
let values = {
36
35
use values <- result.try(unpack_bits(bits, [], at: 0, using: parser))
37
36
values
38
37
|> list.map(dynamic.bit_array)
39
38
|> list.try_map(fn(value) { decode.run(value, decoder) })
40
-
|> result.map_error(UnableToDecode)
39
+
|> result.map_error(protobuf_decode_gleam.UnableToDecode)
41
40
}
42
41
43
42
case values {
···
61
60
}
62
61
}
63
62
63
+
/// Expects a single value from a list. Panics if the list is empty.
64
64
fn single(of decoder: Decoder(t)) -> Decoder(t) {
65
65
use values <- decode.then(decode.list(of: decoder))
66
66
···
95
95
96
96
pub fn uint() -> Decoder(Int) {
97
97
use bits <- decode.then(single_or_raw(decode.bit_array))
98
-
util.bit_array_to_uint(bits) |> decode.success
98
+
use bits <- decode.then(case read_varint(bits, 0) {
99
+
Ok(Parsed(value:, rest: <<>>, pos: _)) -> decode.success(value)
100
+
_ -> decode.failure(<<>>, "uint")
101
+
})
102
+
bits |> util.bit_array_to_uint |> decode.success
99
103
}
100
104
101
105
pub fn fixed(size: Int) -> Decoder(Int) {
+39
-14
src/protobuf_decode_gleam.gleam
+39
-14
src/protobuf_decode_gleam.gleam
···
51
51
}
52
52
}
53
53
54
-
fn repeated_to_list(fields: List(Field)) -> dict.Dict(Dynamic, Dynamic) {
54
+
fn repeated_to_list(reversed_fields: List(Field)) -> dict.Dict(Dynamic, Dynamic) {
55
55
let fields = {
56
56
// Every field is a list of values for two reasons:
57
57
// a) expanded repeated values are encoded as repeated fields
···
60
60
// fields are repeating, it parses all fields as a list and then the
61
61
// decoders will handle choosing which value(s) to keep.
62
62
let acc: dict.Dict(Dynamic, List(Dynamic)) = dict.new()
63
-
use fields, Field(key:, value:) <- list.fold(over: fields, from: acc)
63
+
use fields, Field(key:, value:) <- list.fold(
64
+
over: reversed_fields,
65
+
from: acc,
66
+
)
64
67
65
68
use existing_values <- dict.upsert(in: fields, update: key)
66
69
let existing_values = option.unwrap(existing_values, or: [])
70
+
// The repeated values were previously reversed, now being reversed again
67
71
[value, ..existing_values]
68
72
}
69
73
70
-
// The repeated values must be in order, so un-reverse them here
71
-
dict.map_values(in: fields, with: fn(_key, field) {
72
-
field |> list.reverse |> dynamic.list
73
-
})
74
+
dict.map_values(in: fields, with: fn(_key, field) { field |> dynamic.list })
74
75
}
75
76
76
77
pub type DecodeResult(t) =
···
92
93
Field(key: Dynamic, value: Dynamic)
93
94
}
94
95
95
-
fn wire_type_read_fn(ty: WireType) -> ValueParser {
96
+
fn wire_type_consume_fn(ty: WireType) -> ValueParser {
96
97
case ty {
97
-
wire_type.VarInt -> read_varint
98
+
wire_type.VarInt -> consume_varint
98
99
wire_type.I64 -> read_fixed(64)
99
100
wire_type.Len -> read_len
100
101
wire_type.I32 -> read_fixed(32)
···
123
124
UnknownWireType(wire_type, pos: tag_pos),
124
125
))
125
126
126
-
let read_fn = wire_type_read_fn(wire_type)
127
-
use value: Parsed(Dynamic) <- result.try({
128
-
use value <- result.map(read_fn(bits, pos))
129
-
parsed_map(value, dynamic.bit_array)
130
-
})
127
+
let consume = wire_type_consume_fn(wire_type)
128
+
use value: Parsed(Dynamic) <- result.try(
129
+
consume(bits, pos) |> result.map(parsed_map(_, dynamic.bit_array)),
130
+
)
131
131
132
132
let field =
133
133
parsed_map(value, fn(value) { Field(key: dynamic.int(field_id), value:) })
···
141
141
pub type ValueParser =
142
142
fn(BitArray, BytePos) -> ValueResult
143
143
144
+
pub fn consume_varint(bits: BitArray, pos: BytePos) -> ValueResult {
145
+
consume_varint_acc(bits, <<>>, pos)
146
+
}
147
+
148
+
fn consume_varint_acc(
149
+
bits: BitArray,
150
+
acc: BitArray,
151
+
pos: BytePos,
152
+
) -> ValueResult {
153
+
case bits {
154
+
<<0:size(1), n:bits-size(7), rest:bytes>> -> {
155
+
let bit = <<0:size(1), n:bits>>
156
+
let acc = bit_array.concat([acc, bit])
157
+
Ok(Parsed(value: acc, rest:, pos: pos + 1))
158
+
}
159
+
<<1:size(1), n:bits-size(7), rest:bytes>> -> {
160
+
let bit = <<1:size(1), n:bits>>
161
+
consume_varint_acc(rest, bit_array.concat([acc, bit]), pos + 1)
162
+
}
163
+
bits -> Error(InvalidVarInt(leftover_bits: bits, acc:, pos:))
164
+
}
165
+
}
166
+
144
167
pub fn read_varint(bits: BitArray, pos: BytePos) -> ValueResult {
145
168
read_varint_acc(bits, <<>>, pos)
146
169
}
···
153
176
}
154
177
<<1:size(1), n:bits-size(7), rest:bytes>> ->
155
178
read_varint_acc(rest, bit_array.concat([n, acc]), pos + 1)
156
-
bits -> Error(InvalidVarInt(leftover_bits: bits, acc:, pos:))
179
+
bits -> {
180
+
Error(InvalidVarInt(leftover_bits: bits, acc:, pos:))
181
+
}
157
182
}
158
183
}
159
184
+1
test/packed-fields.pb
+1
test/packed-fields.pb
···
1
+
+6
test/packed-fields.protoscope.txt
+6
test/packed-fields.protoscope.txt
+31
-1
test/protobuf_decode_gleam_test.gleam
+31
-1
test/protobuf_decode_gleam_test.gleam
···
4
4
import simplifile as file
5
5
6
6
import decoders
7
-
import protobuf_decode_gleam.{Parsed, parse}
7
+
import protobuf_decode_gleam.{Parsed, consume_varint, parse}
8
8
9
9
pub fn main() -> Nil {
10
10
gleeunit.main()
···
189
189
102,
190
190
)
191
191
}
192
+
193
+
type PackedFields {
194
+
PackedFields(packed: List(Int), expanded: List(Int))
195
+
}
196
+
197
+
fn packed_fields_decoder() -> Decoder(PackedFields) {
198
+
use packed <- decode.field(
199
+
3,
200
+
decoders.multiple(of: decoders.uint(), using: consume_varint),
201
+
)
202
+
use expanded <- decode.field(
203
+
4,
204
+
decoders.multiple(of: decoders.uint(), using: consume_varint),
205
+
)
206
+
PackedFields(packed:, expanded:) |> decode.success
207
+
}
208
+
209
+
pub fn packed_fields_test() {
210
+
let path = "./test/packed-fields.pb"
211
+
let assert Ok(bits) = file.read_bits(from: path)
212
+
let assert Ok(packed_fields) =
213
+
parse(from: bits, using: packed_fields_decoder())
214
+
215
+
assert packed_fields
216
+
== Parsed(
217
+
value: PackedFields(packed: [1, 2, 3], expanded: [4, 5, 6]),
218
+
rest: <<>>,
219
+
pos: 9,
220
+
)
221
+
}