Decode the protobuf wire format in Gleam!
Move decoders module into main
Gavin Morrow
(Sep 29, 2025, 10:20 AM EDT)
d3748328
2edd4371
+131
-140
-121
src/decoders.gleam
-121
src/decoders.gleam
···
1
-
import gleam/bit_array
2
-
import gleam/dynamic
3
-
import gleam/dynamic/decode.{type Decoder}
4
-
import gleam/int
5
-
import gleam/list
6
-
import gleam/result
7
-
8
-
import protobin.{
9
-
type BytePos, type DecodeResult, type ValueParser, Parsed, parse, parse_varint,
10
-
}
11
-
import protobin/internal/util
12
-
13
-
/// Decode a repeated field that may be either packed or expanded.
14
-
///
15
-
/// If it is impossible for a field to be packed (ie because it is encoded as
16
-
/// a `LEN`) and therefore there is no `ValueParser` for it, then use the
17
-
/// stdlib's `decode.list()` instead.
18
-
pub fn multiple(
19
-
of decoder: Decoder(t),
20
-
using parser: ValueParser,
21
-
) -> Decoder(List(t)) {
22
-
use values <- decode.then(packed_values(of: decoder, using: parser))
23
-
values |> decode.success
24
-
}
25
-
26
-
fn packed_values(
27
-
of decoder: Decoder(t),
28
-
using parser: ValueParser,
29
-
) -> Decoder(List(t)) {
30
-
use bits <- decode.then(decode.list(of: decode.bit_array))
31
-
let bits =
32
-
list.fold(bits, <<>>, fn(acc, elem) { bit_array.concat([acc, elem]) })
33
-
34
-
let values = {
35
-
use values <- result.try(unpack_bits(bits, [], at: 0, using: parser))
36
-
values
37
-
|> list.map(dynamic.bit_array)
38
-
|> list.try_map(fn(value) { decode.run(value, decoder) })
39
-
|> result.map_error(protobin.UnableToDecode)
40
-
}
41
-
42
-
case values {
43
-
Ok(values) -> decode.success(values)
44
-
Error(_) -> decode.failure([], "Packed values")
45
-
}
46
-
}
47
-
48
-
fn unpack_bits(
49
-
bits: BitArray,
50
-
acc: List(BitArray),
51
-
at pos: BytePos,
52
-
using parser: ValueParser,
53
-
) -> DecodeResult(List(BitArray)) {
54
-
case bits {
55
-
<<>> -> acc |> list.reverse |> Ok
56
-
bits -> {
57
-
use Parsed(value:, rest:, pos:) <- result.try(parser(bits, pos))
58
-
unpack_bits(rest, [value, ..acc], at: pos, using: parser)
59
-
}
60
-
}
61
-
}
62
-
63
-
/// Expects a single value from a list. Panics if the list is empty.
64
-
fn single(of decoder: Decoder(t)) -> Decoder(t) {
65
-
use values <- decode.then(decode.list(of: decoder))
66
-
67
-
// I choose to use assert instead of `decode.failure` because the decode api
68
-
// requires passing a name and default value, which is clunky and would've
69
-
// added two parameters to this function.
70
-
// The only times this could fail are programmer error on my part or using
71
-
// this decoder on data not produced in a compatible way with this library.
72
-
let assert Ok(value) = list.last(values)
73
-
value |> decode.success
74
-
}
75
-
76
-
// Allows the decoders to be used for either single or repeated fields
77
-
fn single_or_raw(decoder: Decoder(t)) -> Decoder(t) {
78
-
decode.one_of(decoder, or: [single(of: decoder)])
79
-
}
80
-
81
-
pub fn protobuf(
82
-
// Passed as a function so recursive decoders are easier
83
-
using decoder: fn() -> Decoder(t),
84
-
named name: String,
85
-
default default: t,
86
-
) -> Decoder(t) {
87
-
use bits <- decode.then(single_or_raw(decode.bit_array))
88
-
89
-
let value = parse(from: bits, using: decoder())
90
-
case value {
91
-
Ok(Parsed(value:, ..)) -> decode.success(value)
92
-
Error(_) -> decode.failure(default, name)
93
-
}
94
-
}
95
-
96
-
pub fn uint() -> Decoder(Int) {
97
-
use bits <- decode.then(single_or_raw(decode.bit_array))
98
-
use bits <- decode.then(case parse_varint(bits, 0) {
99
-
Ok(Parsed(value:, rest: <<>>, pos: _)) -> decode.success(value)
100
-
_ -> decode.failure(<<>>, "uint")
101
-
})
102
-
bits |> util.bit_array_to_uint |> decode.success
103
-
}
104
-
105
-
pub fn fixed(size: Int) -> Decoder(Int) {
106
-
use bits <- decode.then(single_or_raw(decode.bit_array))
107
-
case bits {
108
-
<<num:unsigned-little-size(size)>> -> decode.success(num)
109
-
_ -> decode.failure(0, "Fixed(" <> int.to_string(size) <> ")")
110
-
}
111
-
}
112
-
113
-
pub fn string() -> Decoder(String) {
114
-
use bits <- decode.then(single_or_raw(decode.bit_array))
115
-
116
-
let str = bit_array.to_string(bits)
117
-
case str {
118
-
Ok(str) -> decode.success(str)
119
-
Error(_) -> decode.failure("", "String")
120
-
}
121
-
}
+110
src/protobin.gleam
+110
src/protobin.gleam
···
226
226
227
227
Ok(Parsed(value:, rest:, pos: pos + len))
228
228
}
229
+
230
+
/// Decode a repeated field that may be either packed or expanded.
231
+
///
232
+
/// If it is impossible for a field to be packed (ie because it is encoded as
233
+
/// a `LEN`) and therefore there is no `ValueParser` for it, then use the
234
+
/// stdlib's `decode.list()` instead.
235
+
pub fn decode_multiple(
236
+
of decoder: Decoder(t),
237
+
using parser: ValueParser,
238
+
) -> Decoder(List(t)) {
239
+
use values <- decode.then(packed_values(of: decoder, using: parser))
240
+
values |> decode.success
241
+
}
242
+
243
+
fn packed_values(
244
+
of decoder: Decoder(t),
245
+
using parser: ValueParser,
246
+
) -> Decoder(List(t)) {
247
+
use bits <- decode.then(decode.list(of: decode.bit_array))
248
+
let bits =
249
+
list.fold(bits, <<>>, fn(acc, elem) { bit_array.concat([acc, elem]) })
250
+
251
+
let values = {
252
+
use values <- result.try(unpack_bits(bits, [], at: 0, using: parser))
253
+
values
254
+
|> list.map(dynamic.bit_array)
255
+
|> list.try_map(fn(value) { decode.run(value, decoder) })
256
+
|> result.map_error(UnableToDecode)
257
+
}
258
+
259
+
case values {
260
+
Ok(values) -> decode.success(values)
261
+
Error(_) -> decode.failure([], "Packed values")
262
+
}
263
+
}
264
+
265
+
fn unpack_bits(
266
+
bits: BitArray,
267
+
acc: List(BitArray),
268
+
at pos: BytePos,
269
+
using parser: ValueParser,
270
+
) -> DecodeResult(List(BitArray)) {
271
+
case bits {
272
+
<<>> -> acc |> list.reverse |> Ok
273
+
bits -> {
274
+
use Parsed(value:, rest:, pos:) <- result.try(parser(bits, pos))
275
+
unpack_bits(rest, [value, ..acc], at: pos, using: parser)
276
+
}
277
+
}
278
+
}
279
+
280
+
/// Expects a single value from a list. Panics if the list is empty.
281
+
fn single(of decoder: Decoder(t)) -> Decoder(t) {
282
+
use values <- decode.then(decode.list(of: decoder))
283
+
284
+
// I choose to use assert instead of `decode.failure` because the decode api
285
+
// requires passing a name and default value, which is clunky and would've
286
+
// added two parameters to this function.
287
+
// The only times this could fail are programmer error on my part or using
288
+
// this decoder on data not produced in a compatible way with this library.
289
+
let assert Ok(value) = list.last(values)
290
+
value |> decode.success
291
+
}
292
+
293
+
// Allows the decoders to be used for either single or repeated fields
294
+
fn single_or_raw(decoder: Decoder(t)) -> Decoder(t) {
295
+
decode.one_of(decoder, or: [single(of: decoder)])
296
+
}
297
+
298
+
pub fn decode_protobuf(
299
+
// Passed as a function so recursive decoders are easier
300
+
using decoder: fn() -> Decoder(t),
301
+
named name: String,
302
+
default default: t,
303
+
) -> Decoder(t) {
304
+
use bits <- decode.then(single_or_raw(decode.bit_array))
305
+
306
+
let value = parse(from: bits, using: decoder())
307
+
case value {
308
+
Ok(Parsed(value:, ..)) -> decode.success(value)
309
+
Error(_) -> decode.failure(default, name)
310
+
}
311
+
}
312
+
313
+
pub fn decode_uint() -> Decoder(Int) {
314
+
use bits <- decode.then(single_or_raw(decode.bit_array))
315
+
use bits <- decode.then(case parse_varint(bits, 0) {
316
+
Ok(Parsed(value:, rest: <<>>, pos: _)) -> decode.success(value)
317
+
_ -> decode.failure(<<>>, "uint")
318
+
})
319
+
bits |> util.bit_array_to_uint |> decode.success
320
+
}
321
+
322
+
pub fn decode_fixed(size: Int) -> Decoder(Int) {
323
+
use bits <- decode.then(single_or_raw(decode.bit_array))
324
+
case bits {
325
+
<<num:unsigned-little-size(size)>> -> decode.success(num)
326
+
_ -> decode.failure(0, "Fixed(" <> int.to_string(size) <> ")")
327
+
}
328
+
}
329
+
330
+
pub fn decode_string() -> Decoder(String) {
331
+
use bits <- decode.then(single_or_raw(decode.bit_array))
332
+
333
+
let str = bit_array.to_string(bits)
334
+
case str {
335
+
Ok(str) -> decode.success(str)
336
+
Error(_) -> decode.failure("", "String")
337
+
}
338
+
}
+21
-19
test/protobin_test.gleam
+21
-19
test/protobin_test.gleam
···
3
3
import gleeunit
4
4
import simplifile as file
5
5
6
-
import decoders
7
6
import protobin.{Parsed, parse, read_fixed, read_varint}
8
7
9
8
pub fn main() -> Nil {
···
24
23
25
24
fn person_decoder() -> Decoder(Person) {
26
25
let person_inner_decoder =
27
-
decoders.protobuf(
26
+
protobin.decode_protobuf(
28
27
using: person_decoder,
29
28
named: "Person",
30
29
default: default_person,
31
30
)
32
31
33
-
use id <- decode.field(3, decoders.fixed(64))
34
-
use age <- decode.field(1, decoders.uint())
35
-
use score <- decode.field(2, decoders.uint())
32
+
use id <- decode.field(3, protobin.decode_fixed(64))
33
+
use age <- decode.field(1, protobin.decode_uint())
34
+
use score <- decode.field(2, protobin.decode_uint())
36
35
use person <- decode.optional_field(
37
36
4,
38
37
option.None,
39
38
decode.optional(person_inner_decoder),
40
39
)
41
-
use day <- decode.field(5, decoders.fixed(32))
40
+
use day <- decode.field(5, protobin.decode_fixed(32))
42
41
43
42
Person(id:, age:, score:, self: person, day:) |> decode.success
44
43
}
···
71
70
}
72
71
73
72
fn two_ints_decoder() -> Decoder(TwoInts) {
74
-
use id <- decode.field(1, decoders.uint())
75
-
use age <- decode.field(2, decoders.uint())
73
+
use id <- decode.field(1, protobin.decode_uint())
74
+
use age <- decode.field(2, protobin.decode_uint())
76
75
77
76
Test(id:, age:) |> decode.success
78
77
}
···
121
120
)
122
121
123
122
fn feed_header_decoder() -> Decoder(FeedHeader) {
124
-
use gtfs_realtime_version <- decode.field(1, decoders.string())
125
-
use timestamp <- decode.field(3, decoders.fixed(64))
123
+
use gtfs_realtime_version <- decode.field(1, protobin.decode_string())
124
+
use timestamp <- decode.field(3, protobin.decode_fixed(64))
126
125
use nyct <- decode.field(
127
126
1001,
128
-
decoders.protobuf(
127
+
protobin.decode_protobuf(
129
128
using: nyct_header_decoder,
130
129
named: "NyctHeader",
131
130
default: nyct_header_default,
···
135
134
}
136
135
137
136
fn nyct_header_decoder() -> Decoder(NyctHeader) {
138
-
use version <- decode.field(1, decoders.string())
137
+
use version <- decode.field(1, protobin.decode_string())
139
138
140
139
use trip_replacement_periods <- decode.field(
141
140
2,
142
-
decode.list(of: decoders.protobuf(
141
+
decode.list(of: protobin.decode_protobuf(
143
142
using: trip_replacement_period_decoder,
144
143
named: "TripReplacementPeriod",
145
144
default: trip_replacement_period_default,
···
150
149
}
151
150
152
151
fn trip_replacement_period_decoder() -> Decoder(TripReplacementPeriod) {
153
-
use route_id <- decode.field(1, decoders.string())
152
+
use route_id <- decode.field(1, protobin.decode_string())
154
153
use replacement_period <- decode.field(
155
154
2,
156
-
decoders.protobuf(
155
+
protobin.decode_protobuf(
157
156
using: trip_replacement_period_time_range_decoder,
158
157
named: "TripReplacePeriod.TimeRange",
159
158
default: 0,
···
163
162
}
164
163
165
164
fn trip_replacement_period_time_range_decoder() -> Decoder(Int) {
166
-
use time <- decode.field(2, decoders.fixed(64))
165
+
use time <- decode.field(2, protobin.decode_fixed(64))
167
166
time |> decode.success
168
167
}
169
168
···
201
200
fn packed_fields_decoder() -> Decoder(PackedFields) {
202
201
use packed <- decode.field(
203
202
3,
204
-
decoders.multiple(of: decoders.uint(), using: read_varint),
203
+
protobin.decode_multiple(of: protobin.decode_uint(), using: read_varint),
205
204
)
206
205
use expanded <- decode.field(
207
206
4,
208
-
decoders.multiple(of: decoders.uint(), using: read_varint),
207
+
protobin.decode_multiple(of: protobin.decode_uint(), using: read_varint),
209
208
)
210
209
use mixed_packed_and_expanded <- decode.field(
211
210
11,
212
-
decoders.multiple(of: decoders.fixed(32), using: read_fixed(32)),
211
+
protobin.decode_multiple(
212
+
of: protobin.decode_fixed(32),
213
+
using: read_fixed(32),
214
+
),
213
215
)
214
216
PackedFields(packed:, expanded:, mixed_packed_and_expanded:) |> decode.success
215
217
}