Decode the protobuf wire format in Gleam!
protobin.hexdocs.pm/
gleam
protobuf
Clean up names and types, add a few helper funs
Gavin Morrow
(Sep 19, 2025, 8:57 PM EDT)
c0fa6050
7031a26c
+65
-36
+55
-30
src/protobuf_decode_gleam.gleam
+55
-30
src/protobuf_decode_gleam.gleam
···
2
2
import gleam/dynamic.{type Dynamic}
3
3
import gleam/dynamic/decode.{type Decoder}
4
4
import gleam/int
5
+
import gleam/list
5
6
import gleam/option.{type Option, None, Some}
6
-
import gleam/pair
7
7
import gleam/result
8
8
9
-
pub fn decode(
9
+
pub fn parse(
10
10
from bits: BitArray,
11
11
using decoder: Decoder(t),
12
12
) -> Result(t, DecodeError) {
···
14
14
decode.run(data, decoder) |> result.map_error(UnableToDecode)
15
15
}
16
16
17
-
pub fn decoder(
17
+
pub fn decode_protobuf(
18
18
using decoder: fn() -> Decoder(t),
19
19
named name: String,
20
20
default default: t,
21
21
) -> Decoder(t) {
22
22
use bits <- decode.then(decode.bit_array)
23
23
24
-
let value = decode(from: bits, using: decoder())
24
+
let value = parse(from: bits, using: decoder())
25
25
case value {
26
26
Ok(value) -> decode.success(value)
27
27
Error(_) -> decode.failure(default, name)
···
45
45
}
46
46
}
47
47
48
+
fn wire_type_read_fn(ty: WireType) -> fn(BitArray) -> ValueResult {
49
+
case ty {
50
+
VarInt -> read_varint
51
+
I64 -> read_fixed(64)
52
+
Len -> read_len
53
+
I32 -> read_fixed(32)
54
+
}
55
+
}
56
+
48
57
pub type DecodeError {
49
58
UnknownWireType(Int)
50
59
InvalidVarInt(leftover_bits: BitArray, acc: BitArray)
51
60
InvalidFixed(size: Int, bits: BitArray)
52
-
UnableToDecode(List(decode.DecodeError))
53
61
InvalidLen(len: Int, value_bits: BitArray)
62
+
UnableToDecode(List(decode.DecodeError))
54
63
}
55
64
56
-
fn read_fields(
57
-
bits: BitArray,
58
-
acc: List(#(Dynamic, Dynamic)),
59
-
) -> Result(Dynamic, DecodeError) {
65
+
fn read_fields(bits: BitArray, acc: List(Field)) -> Result(Dynamic, DecodeError) {
60
66
case bits {
61
-
<<>> -> Ok(dynamic.properties(acc))
67
+
<<>> ->
68
+
acc
69
+
|> list.map(field_as_pair)
70
+
|> dynamic.properties
71
+
|> Ok
62
72
bits -> {
63
-
use #(prop, bits) <- result.try(read_field(bits))
73
+
use Parsed(value: prop, rest: bits) <- result.try(read_field(bits))
64
74
read_fields(bits, [prop, ..acc])
65
75
}
66
76
}
67
77
}
68
78
69
-
fn read_field(
70
-
bits: BitArray,
71
-
) -> Result(#(#(Dynamic, Dynamic), BitArray), DecodeError) {
72
-
use #(tag, bits) <- result.try(read_varint(bits))
79
+
type DecodeResult(t) =
80
+
Result(t, DecodeError)
81
+
82
+
type Parsed(t) {
83
+
Parsed(value: t, rest: BitArray)
84
+
}
85
+
86
+
fn parsed_map(of parsed: Parsed(t), with fun: fn(t) -> u) -> Parsed(u) {
87
+
let Parsed(value:, rest:) = parsed
88
+
Parsed(value: fun(value), rest:)
89
+
}
90
+
91
+
type Field {
92
+
Field(key: Dynamic, value: Dynamic)
93
+
}
94
+
95
+
fn field_as_pair(field: Field) {
96
+
#(field.key, field.value)
97
+
}
98
+
99
+
fn read_field(bits: BitArray) -> DecodeResult(Parsed(Field)) {
100
+
use Parsed(value: tag, rest: bits) <- result.try(read_varint(bits))
73
101
let tag = bit_array_to_uint(tag)
74
102
75
103
let field_id = tag |> int.bitwise_shift_right(3)
···
80
108
UnknownWireType(wire_type),
81
109
))
82
110
83
-
let read_fn: fn(BitArray) -> ValueResult = case wire_type {
84
-
VarInt -> read_varint
85
-
I64 -> read_fixed(64)
86
-
Len -> read_len
87
-
I32 -> read_fixed(32)
88
-
}
89
-
use #(value, rest): #(Dynamic, BitArray) <- result.try({
111
+
let read_fn = wire_type_read_fn(wire_type)
112
+
use value: Parsed(Dynamic) <- result.try({
90
113
use value <- result.map(read_fn(bits))
91
-
use value <- pair.map_first(value)
92
-
dynamic.bit_array(value)
114
+
parsed_map(value, dynamic.bit_array)
93
115
})
94
116
95
-
Ok(#(#(dynamic.int(field_id), value), rest))
117
+
let field =
118
+
parsed_map(value, fn(value) { Field(key: dynamic.int(field_id), value:) })
119
+
120
+
Ok(field)
96
121
}
97
122
98
123
type ValueResult =
99
-
Result(#(BitArray, BitArray), DecodeError)
124
+
DecodeResult(Parsed(BitArray))
100
125
101
126
fn read_varint(bits: BitArray) -> ValueResult {
102
127
read_varint_acc(bits, <<>>)
···
106
131
case bits {
107
132
<<0:size(1), n:bits-size(7), rest:bytes>> -> {
108
133
let acc = bit_array.concat([n, acc])
109
-
Ok(#(acc, rest))
134
+
Ok(Parsed(value: acc, rest:))
110
135
}
111
136
<<1:size(1), n:bits-size(7), rest:bytes>> ->
112
137
read_varint_acc(rest, bit_array.concat([n, acc]))
···
117
142
fn read_fixed(size: Int) -> fn(BitArray) -> ValueResult {
118
143
fn(bits: BitArray) -> ValueResult {
119
144
case bits {
120
-
<<num:bits-size(size), rest:bytes>> -> Ok(#(num, rest))
145
+
<<num:bits-size(size), rest:bytes>> -> Ok(Parsed(value: num, rest:))
121
146
bits -> Error(InvalidFixed(size:, bits:))
122
147
}
123
148
}
···
126
151
fn read_len(bits: BitArray) -> ValueResult {
127
152
// First, read the length of the value
128
153
// It is encoded as a varint immediately after the tag
129
-
use #(len, bits) <- result.try(read_varint(bits))
154
+
use Parsed(value: len, rest: bits) <- result.try(read_varint(bits))
130
155
let len = bit_array_to_uint(len)
131
156
132
157
// Just decoded a uint, so should be safe
···
140
165
let assert Ok(rest) =
141
166
bit_array.slice(from: bits, at: len, take: bit_array.byte_size(bits) - len)
142
167
143
-
Ok(#(value, rest))
168
+
Ok(Parsed(value:, rest:))
144
169
}
145
170
146
171
pub fn decode_uint() -> Decoder(Int) {
+10
-6
test/protobuf_decode_gleam_test.gleam
+10
-6
test/protobuf_decode_gleam_test.gleam
···
2
2
import gleam/option
3
3
import gleeunit
4
4
import protobuf_decode_gleam.{
5
-
decode, decode_fixed, decode_string, decode_uint, decoder,
5
+
decode_fixed, decode_protobuf, decode_string, decode_uint, parse,
6
6
}
7
7
import simplifile as file
8
8
···
24
24
25
25
fn person_decoder() -> Decoder(Person) {
26
26
let person_inner_decoder =
27
-
decoder(using: person_decoder, named: "Person", default: default_person)
27
+
decode_protobuf(
28
+
using: person_decoder,
29
+
named: "Person",
30
+
default: default_person,
31
+
)
28
32
29
33
use id <- decode.field(3, decode_fixed(64))
30
34
use age <- decode.field(1, decode_uint())
···
42
46
pub fn person_pb_test() {
43
47
let path = "./test/person.pb"
44
48
let assert Ok(bits) = file.read_bits(from: path)
45
-
let assert Ok(person) = decode(from: bits, using: person_decoder())
49
+
let assert Ok(person) = parse(from: bits, using: person_decoder())
46
50
assert person
47
51
== Person(
48
52
id: 42,
···
83
87
0x04,
84
88
>>
85
89
86
-
let assert Ok(data) = decode(from: bits, using: two_ints_decoder())
90
+
let assert Ok(data) = parse(from: bits, using: two_ints_decoder())
87
91
assert data == Test(id: 150, age: 80_150)
88
92
}
89
93
···
94
98
fn gtfs_decoder() -> Decoder(Gtfs) {
95
99
use header <- decode.field(
96
100
1,
97
-
decoder(
101
+
decode_protobuf(
98
102
using: feed_header_decoder,
99
103
named: "FeedHeader",
100
104
default: feed_header_default,
···
117
121
pub fn gtfs_test() {
118
122
let path = "./test/gtfs-short.pb"
119
123
let assert Ok(bits) = file.read_bits(from: path)
120
-
let assert Ok(gtfs) = decode(from: bits, using: gtfs_decoder())
124
+
let assert Ok(gtfs) = parse(from: bits, using: gtfs_decoder())
121
125
122
126
echo gtfs
123
127