Decode the protobuf wire format in Gleam!
Rename to protobin
Gavin Morrow
(Sep 29, 2025, 9:57 AM EDT)
c0f36a76
4fcda1e3
+474
-474
+8
-8
README.md
+8
-8
README.md
···
1
-
# protobuf_decode_gleam
1
+
# protobin
2
2
3
-
Decode the protobuf wire format!
3
+
Decode the protobuf wire format using gleam `Decoder`s!
4
4
5
5
This is just a little toy I made mostly for myself. I would be happy to
6
6
polish it and make it work better, but for now this should not be used at all.
···
9
9
10
10
Also, there is currently no support for encoding the protobuf wire format.
11
11
<!--
12
-
[](https://hex.pm/packages/protobuf_decode_gleam)
13
-
[](https://hexdocs.pm/protobuf_decode_gleam/)
12
+
[](https://hex.pm/packages/protobin)
13
+
[](https://hexdocs.pm/protobin/)
14
14
15
15
16
16
```sh
17
-
gleam add protobuf_decode_gleam@1
17
+
gleam add protobin@1
18
18
```
19
19
-->
20
20
```gleam
21
-
import protobuf_decode_gleam.{parse, read_varint}
22
-
import protobuf_decode_gleam/decoders
21
+
import protobin.{parse, read_varint}
22
+
import protobin/decoders
23
23
24
24
import gleam/option
25
25
···
65
65
```
66
66
67
67
<!--
68
-
Further documentation can be found at <https://hexdocs.pm/protobuf_decode_gleam>.
68
+
Further documentation can be found at <https://hexdocs.pm/protobin>.
69
69
-->
70
70
71
71
## Development
+2
-2
gleam.toml
+2
-2
gleam.toml
···
1
-
name = "protobuf_decode_gleam"
1
+
name = "protobin"
2
2
version = "1.0.0"
3
3
4
4
# Fill out these fields if you intend to generate HTML documentation or publish
···
6
6
#
7
7
description = "Decode the protobuf wire format."
8
8
licences = ["Apache-2.0"]
9
-
repository = { type = "github", user = "gavinmorrow", repo = "protobuf-decode-gleam" }
9
+
repository = { type = "github", user = "gavinmorrow", repo = "gleam-protobin" }
10
10
# links = [{ title = "Website", href = "" }]
11
11
#
12
12
# For a full reference of all the available options, you can have a look at
+3
-3
src/decoders.gleam
+3
-3
src/decoders.gleam
···
5
5
import gleam/list
6
6
import gleam/result
7
7
8
-
import internal/util
9
-
import protobuf_decode_gleam.{
8
+
import protobin.{
10
9
type BytePos, type DecodeResult, type ValueParser, Parsed, parse, parse_varint,
11
10
}
11
+
import internal/util
12
12
13
13
/// Decode a repeated field that may be either packed or expanded.
14
14
///
···
36
36
values
37
37
|> list.map(dynamic.bit_array)
38
38
|> list.try_map(fn(value) { decode.run(value, decoder) })
39
-
|> result.map_error(protobuf_decode_gleam.UnableToDecode)
39
+
|> result.map_error(protobin.UnableToDecode)
40
40
}
41
41
42
42
case values {
+228
src/protobin.gleam
+228
src/protobin.gleam
···
1
+
import gleam/bit_array
2
+
import gleam/dict
3
+
import gleam/dynamic.{type Dynamic}
4
+
import gleam/dynamic/decode.{type Decoder}
5
+
import gleam/int
6
+
import gleam/list
7
+
import gleam/option
8
+
import gleam/result
9
+
10
+
import internal/util
11
+
import internal/wire_type.{type WireType}
12
+
13
+
pub fn parse(
14
+
from bits: BitArray,
15
+
using decoder: Decoder(t),
16
+
) -> DecodeResult(Parsed(t)) {
17
+
use Parsed(value: data, rest:, pos:) <- result.try(read_fields(bits, [], 0))
18
+
decode.run(data, decoder)
19
+
|> result.map(fn(value) { Parsed(value:, rest:, pos:) })
20
+
|> result.map_error(UnableToDecode)
21
+
}
22
+
23
+
pub type ParseError {
24
+
UnknownWireType(Int, pos: BytePos)
25
+
InvalidVarInt(leftover_bits: BitArray, acc: BitArray, pos: BytePos)
26
+
InvalidFixed(size: Int, bits: BitArray, pos: BytePos)
27
+
InvalidLen(len: Int, value_bits: BitArray, pos: BytePos)
28
+
UnableToDecode(List(decode.DecodeError))
29
+
}
30
+
31
+
fn read_fields(
32
+
bits: BitArray,
33
+
acc: List(Field),
34
+
pos: BytePos,
35
+
) -> DecodeResult(Parsed(Dynamic)) {
36
+
case bits {
37
+
<<>> ->
38
+
acc
39
+
|> repeated_to_list
40
+
|> dict.to_list
41
+
|> dynamic.properties
42
+
|> Parsed(value: _, rest: bits, pos:)
43
+
|> Ok
44
+
bits -> {
45
+
use Parsed(value: prop, rest: bits, pos:) <- result.try(read_field(
46
+
bits,
47
+
pos,
48
+
))
49
+
read_fields(bits, [prop, ..acc], pos)
50
+
}
51
+
}
52
+
}
53
+
54
+
fn repeated_to_list(reversed_fields: List(Field)) -> dict.Dict(Dynamic, Dynamic) {
55
+
let fields = {
56
+
// Every field is a list of values for two reasons:
57
+
// a) expanded repeated values are encoded as repeated fields
58
+
// b) if a non-repeating field is defined twice, then the last value should
59
+
// be considered the correct one. since the parser doesn't know which
60
+
// fields are repeating, it parses all fields as a list and then the
61
+
// decoders will handle choosing which value(s) to keep.
62
+
let acc: dict.Dict(Dynamic, List(Dynamic)) = dict.new()
63
+
use fields, Field(key:, value:) <- list.fold(
64
+
over: reversed_fields,
65
+
from: acc,
66
+
)
67
+
68
+
use existing_values <- dict.upsert(in: fields, update: key)
69
+
let existing_values = option.unwrap(existing_values, or: [])
70
+
// The repeated values were previously reversed, now being reversed again
71
+
[value, ..existing_values]
72
+
}
73
+
74
+
dict.map_values(in: fields, with: fn(_key, field) { field |> dynamic.list })
75
+
}
76
+
77
+
pub type DecodeResult(t) =
78
+
Result(t, ParseError)
79
+
80
+
pub type BytePos =
81
+
Int
82
+
83
+
pub type Parsed(t) {
84
+
Parsed(value: t, rest: BitArray, pos: BytePos)
85
+
}
86
+
87
+
fn parsed_map(of parsed: Parsed(t), with fun: fn(t) -> u) -> Parsed(u) {
88
+
let Parsed(value:, rest:, pos:) = parsed
89
+
Parsed(value: fun(value), rest:, pos:)
90
+
}
91
+
92
+
type Field {
93
+
Field(key: Dynamic, value: Dynamic)
94
+
}
95
+
96
+
fn wire_type_read_fn(ty: WireType) -> ValueParser {
97
+
case ty {
98
+
wire_type.VarInt -> read_varint
99
+
wire_type.I64 -> read_fixed(64)
100
+
wire_type.Len -> read_len
101
+
wire_type.I32 -> read_fixed(32)
102
+
}
103
+
}
104
+
105
+
fn read_field(bits: BitArray, tag_pos: BytePos) -> DecodeResult(Parsed(Field)) {
106
+
use Parsed(value: tag, rest: bits, pos:) <- result.try(parse_varint(
107
+
bits,
108
+
tag_pos,
109
+
))
110
+
let tag = util.bit_array_to_uint(tag)
111
+
112
+
let field_id = tag |> int.bitwise_shift_right(3)
113
+
let wire_type = tag |> int.bitwise_and(0b111)
114
+
115
+
case wire_type {
116
+
6 -> {
117
+
echo tag as "tag"
118
+
Nil
119
+
}
120
+
_ -> Nil
121
+
}
122
+
use wire_type <- result.try(option.to_result(
123
+
wire_type.parse(wire_type),
124
+
UnknownWireType(wire_type, pos: tag_pos),
125
+
))
126
+
127
+
let read = wire_type_read_fn(wire_type)
128
+
use value: Parsed(Dynamic) <- result.try(
129
+
read(bits, pos) |> result.map(parsed_map(_, dynamic.bit_array)),
130
+
)
131
+
132
+
let field =
133
+
parsed_map(value, fn(value) { Field(key: dynamic.int(field_id), value:) })
134
+
135
+
Ok(field)
136
+
}
137
+
138
+
pub type ValueResult =
139
+
DecodeResult(Parsed(BitArray))
140
+
141
+
pub type ValueParser =
142
+
fn(BitArray, BytePos) -> ValueResult
143
+
144
+
/// Reads the bits from a varint and returns *all* of them. They cannot be
145
+
/// parsed as a uint.
146
+
///
147
+
/// Specifically, the continuation bits are included, so the value's bit size
148
+
/// will be a multiple of 8.
149
+
pub fn read_varint(bits: BitArray, pos: BytePos) -> ValueResult {
150
+
read_varint_acc(bits, <<>>, pos)
151
+
}
152
+
153
+
fn read_varint_acc(bits: BitArray, acc: BitArray, pos: BytePos) -> ValueResult {
154
+
case bits {
155
+
<<0:size(1), n:bits-size(7), rest:bytes>> -> {
156
+
let bit = <<0:size(1), n:bits>>
157
+
let acc = bit_array.concat([acc, bit])
158
+
Ok(Parsed(value: acc, rest:, pos: pos + 1))
159
+
}
160
+
<<1:size(1), n:bits-size(7), rest:bytes>> -> {
161
+
let bit = <<1:size(1), n:bits>>
162
+
read_varint_acc(rest, bit_array.concat([acc, bit]), pos + 1)
163
+
}
164
+
bits -> Error(InvalidVarInt(leftover_bits: bits, acc:, pos:))
165
+
}
166
+
}
167
+
168
+
/// Reads the bits from a varint and parses them a BitArray. The returned bits
169
+
/// can be parsed as a uint.
170
+
///
171
+
/// For a decoder that does this, use `decoders.uint()`.
172
+
///
173
+
/// The continuation bits are not included, so the value's bit size will be a
174
+
/// multiple of 7.
175
+
pub fn parse_varint(bits: BitArray, pos: BytePos) -> ValueResult {
176
+
parse_varint_acc(bits, <<>>, pos)
177
+
}
178
+
179
+
fn parse_varint_acc(bits: BitArray, acc: BitArray, pos: BytePos) -> ValueResult {
180
+
case bits {
181
+
<<0:size(1), n:bits-size(7), rest:bytes>> -> {
182
+
let acc = bit_array.concat([n, acc])
183
+
Ok(Parsed(value: acc, rest:, pos: pos + 1))
184
+
}
185
+
<<1:size(1), n:bits-size(7), rest:bytes>> ->
186
+
parse_varint_acc(rest, bit_array.concat([n, acc]), pos + 1)
187
+
bits -> {
188
+
Error(InvalidVarInt(leftover_bits: bits, acc:, pos:))
189
+
}
190
+
}
191
+
}
192
+
193
+
/// Size must be a multiple of 8.
194
+
pub fn read_fixed(size: Int) -> ValueParser {
195
+
assert size % 8 == 0
196
+
fn(bits: BitArray, pos: BytePos) -> ValueResult {
197
+
case bits {
198
+
<<num:bits-size(size), rest:bytes>> ->
199
+
Ok(Parsed(value: num, rest:, pos: pos + size / 8))
200
+
bits -> Error(InvalidFixed(size:, bits:, pos:))
201
+
}
202
+
}
203
+
}
204
+
205
+
fn read_len(bits: BitArray, len_pos: BytePos) -> ValueResult {
206
+
// First, read the length of the value
207
+
// It is encoded as a varint immediately after the tag
208
+
use Parsed(value: len, rest: bits, pos:) <- result.try(parse_varint(
209
+
bits,
210
+
len_pos,
211
+
))
212
+
let len = util.bit_array_to_uint(len)
213
+
214
+
// Just decoded a uint, so should be safe
215
+
assert len > 0
216
+
217
+
use value <- result.try(
218
+
bit_array.slice(from: bits, at: 0, take: len)
219
+
|> result.map_error(fn(_) {
220
+
InvalidLen(len:, value_bits: bits, pos: len_pos)
221
+
}),
222
+
)
223
+
// Assert b/c if the len was too long, it would've errored in the prev slice
224
+
let assert Ok(rest) =
225
+
bit_array.slice(from: bits, at: len, take: bit_array.byte_size(bits) - len)
226
+
227
+
Ok(Parsed(value:, rest:, pos: pos + len))
228
+
}
-228
src/protobuf_decode_gleam.gleam
-228
src/protobuf_decode_gleam.gleam
···
1
-
import gleam/bit_array
2
-
import gleam/dict
3
-
import gleam/dynamic.{type Dynamic}
4
-
import gleam/dynamic/decode.{type Decoder}
5
-
import gleam/int
6
-
import gleam/list
7
-
import gleam/option
8
-
import gleam/result
9
-
10
-
import internal/util
11
-
import internal/wire_type.{type WireType}
12
-
13
-
pub fn parse(
14
-
from bits: BitArray,
15
-
using decoder: Decoder(t),
16
-
) -> DecodeResult(Parsed(t)) {
17
-
use Parsed(value: data, rest:, pos:) <- result.try(read_fields(bits, [], 0))
18
-
decode.run(data, decoder)
19
-
|> result.map(fn(value) { Parsed(value:, rest:, pos:) })
20
-
|> result.map_error(UnableToDecode)
21
-
}
22
-
23
-
pub type ParseError {
24
-
UnknownWireType(Int, pos: BytePos)
25
-
InvalidVarInt(leftover_bits: BitArray, acc: BitArray, pos: BytePos)
26
-
InvalidFixed(size: Int, bits: BitArray, pos: BytePos)
27
-
InvalidLen(len: Int, value_bits: BitArray, pos: BytePos)
28
-
UnableToDecode(List(decode.DecodeError))
29
-
}
30
-
31
-
fn read_fields(
32
-
bits: BitArray,
33
-
acc: List(Field),
34
-
pos: BytePos,
35
-
) -> DecodeResult(Parsed(Dynamic)) {
36
-
case bits {
37
-
<<>> ->
38
-
acc
39
-
|> repeated_to_list
40
-
|> dict.to_list
41
-
|> dynamic.properties
42
-
|> Parsed(value: _, rest: bits, pos:)
43
-
|> Ok
44
-
bits -> {
45
-
use Parsed(value: prop, rest: bits, pos:) <- result.try(read_field(
46
-
bits,
47
-
pos,
48
-
))
49
-
read_fields(bits, [prop, ..acc], pos)
50
-
}
51
-
}
52
-
}
53
-
54
-
fn repeated_to_list(reversed_fields: List(Field)) -> dict.Dict(Dynamic, Dynamic) {
55
-
let fields = {
56
-
// Every field is a list of values for two reasons:
57
-
// a) expanded repeated values are encoded as repeated fields
58
-
// b) if a non-repeating field is defined twice, then the last value should
59
-
// be considered the correct one. since the parser doesn't know which
60
-
// fields are repeating, it parses all fields as a list and then the
61
-
// decoders will handle choosing which value(s) to keep.
62
-
let acc: dict.Dict(Dynamic, List(Dynamic)) = dict.new()
63
-
use fields, Field(key:, value:) <- list.fold(
64
-
over: reversed_fields,
65
-
from: acc,
66
-
)
67
-
68
-
use existing_values <- dict.upsert(in: fields, update: key)
69
-
let existing_values = option.unwrap(existing_values, or: [])
70
-
// The repeated values were previously reversed, now being reversed again
71
-
[value, ..existing_values]
72
-
}
73
-
74
-
dict.map_values(in: fields, with: fn(_key, field) { field |> dynamic.list })
75
-
}
76
-
77
-
pub type DecodeResult(t) =
78
-
Result(t, ParseError)
79
-
80
-
pub type BytePos =
81
-
Int
82
-
83
-
pub type Parsed(t) {
84
-
Parsed(value: t, rest: BitArray, pos: BytePos)
85
-
}
86
-
87
-
fn parsed_map(of parsed: Parsed(t), with fun: fn(t) -> u) -> Parsed(u) {
88
-
let Parsed(value:, rest:, pos:) = parsed
89
-
Parsed(value: fun(value), rest:, pos:)
90
-
}
91
-
92
-
type Field {
93
-
Field(key: Dynamic, value: Dynamic)
94
-
}
95
-
96
-
fn wire_type_read_fn(ty: WireType) -> ValueParser {
97
-
case ty {
98
-
wire_type.VarInt -> read_varint
99
-
wire_type.I64 -> read_fixed(64)
100
-
wire_type.Len -> read_len
101
-
wire_type.I32 -> read_fixed(32)
102
-
}
103
-
}
104
-
105
-
fn read_field(bits: BitArray, tag_pos: BytePos) -> DecodeResult(Parsed(Field)) {
106
-
use Parsed(value: tag, rest: bits, pos:) <- result.try(parse_varint(
107
-
bits,
108
-
tag_pos,
109
-
))
110
-
let tag = util.bit_array_to_uint(tag)
111
-
112
-
let field_id = tag |> int.bitwise_shift_right(3)
113
-
let wire_type = tag |> int.bitwise_and(0b111)
114
-
115
-
case wire_type {
116
-
6 -> {
117
-
echo tag as "tag"
118
-
Nil
119
-
}
120
-
_ -> Nil
121
-
}
122
-
use wire_type <- result.try(option.to_result(
123
-
wire_type.parse(wire_type),
124
-
UnknownWireType(wire_type, pos: tag_pos),
125
-
))
126
-
127
-
let read = wire_type_read_fn(wire_type)
128
-
use value: Parsed(Dynamic) <- result.try(
129
-
read(bits, pos) |> result.map(parsed_map(_, dynamic.bit_array)),
130
-
)
131
-
132
-
let field =
133
-
parsed_map(value, fn(value) { Field(key: dynamic.int(field_id), value:) })
134
-
135
-
Ok(field)
136
-
}
137
-
138
-
pub type ValueResult =
139
-
DecodeResult(Parsed(BitArray))
140
-
141
-
pub type ValueParser =
142
-
fn(BitArray, BytePos) -> ValueResult
143
-
144
-
/// Reads the bits from a varint and returns *all* of them. They cannot be
145
-
/// parsed as a uint.
146
-
///
147
-
/// Specifically, the continuation bits are included, so the value's bit size
148
-
/// will be a multiple of 8.
149
-
pub fn read_varint(bits: BitArray, pos: BytePos) -> ValueResult {
150
-
read_varint_acc(bits, <<>>, pos)
151
-
}
152
-
153
-
fn read_varint_acc(bits: BitArray, acc: BitArray, pos: BytePos) -> ValueResult {
154
-
case bits {
155
-
<<0:size(1), n:bits-size(7), rest:bytes>> -> {
156
-
let bit = <<0:size(1), n:bits>>
157
-
let acc = bit_array.concat([acc, bit])
158
-
Ok(Parsed(value: acc, rest:, pos: pos + 1))
159
-
}
160
-
<<1:size(1), n:bits-size(7), rest:bytes>> -> {
161
-
let bit = <<1:size(1), n:bits>>
162
-
read_varint_acc(rest, bit_array.concat([acc, bit]), pos + 1)
163
-
}
164
-
bits -> Error(InvalidVarInt(leftover_bits: bits, acc:, pos:))
165
-
}
166
-
}
167
-
168
-
/// Reads the bits from a varint and parses them a BitArray. The returned bits
169
-
/// can be parsed as a uint.
170
-
///
171
-
/// For a decoder that does this, use `decoders.uint()`.
172
-
///
173
-
/// The continuation bits are not included, so the value's bit size will be a
174
-
/// multiple of 7.
175
-
pub fn parse_varint(bits: BitArray, pos: BytePos) -> ValueResult {
176
-
parse_varint_acc(bits, <<>>, pos)
177
-
}
178
-
179
-
fn parse_varint_acc(bits: BitArray, acc: BitArray, pos: BytePos) -> ValueResult {
180
-
case bits {
181
-
<<0:size(1), n:bits-size(7), rest:bytes>> -> {
182
-
let acc = bit_array.concat([n, acc])
183
-
Ok(Parsed(value: acc, rest:, pos: pos + 1))
184
-
}
185
-
<<1:size(1), n:bits-size(7), rest:bytes>> ->
186
-
parse_varint_acc(rest, bit_array.concat([n, acc]), pos + 1)
187
-
bits -> {
188
-
Error(InvalidVarInt(leftover_bits: bits, acc:, pos:))
189
-
}
190
-
}
191
-
}
192
-
193
-
/// Size must be a multiple of 8.
194
-
pub fn read_fixed(size: Int) -> ValueParser {
195
-
assert size % 8 == 0
196
-
fn(bits: BitArray, pos: BytePos) -> ValueResult {
197
-
case bits {
198
-
<<num:bits-size(size), rest:bytes>> ->
199
-
Ok(Parsed(value: num, rest:, pos: pos + size / 8))
200
-
bits -> Error(InvalidFixed(size:, bits:, pos:))
201
-
}
202
-
}
203
-
}
204
-
205
-
fn read_len(bits: BitArray, len_pos: BytePos) -> ValueResult {
206
-
// First, read the length of the value
207
-
// It is encoded as a varint immediately after the tag
208
-
use Parsed(value: len, rest: bits, pos:) <- result.try(parse_varint(
209
-
bits,
210
-
len_pos,
211
-
))
212
-
let len = util.bit_array_to_uint(len)
213
-
214
-
// Just decoded a uint, so should be safe
215
-
assert len > 0
216
-
217
-
use value <- result.try(
218
-
bit_array.slice(from: bits, at: 0, take: len)
219
-
|> result.map_error(fn(_) {
220
-
InvalidLen(len:, value_bits: bits, pos: len_pos)
221
-
}),
222
-
)
223
-
// Assert b/c if the len was too long, it would've errored in the prev slice
224
-
let assert Ok(rest) =
225
-
bit_array.slice(from: bits, at: len, take: bit_array.byte_size(bits) - len)
226
-
227
-
Ok(Parsed(value:, rest:, pos: pos + len))
228
-
}
+233
test/protobin_test.gleam
+233
test/protobin_test.gleam
···
1
+
import gleam/dynamic/decode.{type Decoder}
2
+
import gleam/option
3
+
import gleeunit
4
+
import simplifile as file
5
+
6
+
import decoders
7
+
import protobin.{Parsed, parse, read_fixed, read_varint}
8
+
9
+
pub fn main() -> Nil {
10
+
gleeunit.main()
11
+
}
12
+
13
+
type Person {
14
+
Person(id: Int, age: Int, score: Int, self: option.Option(Person), day: Int)
15
+
}
16
+
17
+
const default_person = Person(
18
+
id: 0,
19
+
age: 0,
20
+
score: 0,
21
+
self: option.None,
22
+
day: 0,
23
+
)
24
+
25
+
fn person_decoder() -> Decoder(Person) {
26
+
let person_inner_decoder =
27
+
decoders.protobuf(
28
+
using: person_decoder,
29
+
named: "Person",
30
+
default: default_person,
31
+
)
32
+
33
+
use id <- decode.field(3, decoders.fixed(64))
34
+
use age <- decode.field(1, decoders.uint())
35
+
use score <- decode.field(2, decoders.uint())
36
+
use person <- decode.optional_field(
37
+
4,
38
+
option.None,
39
+
decode.optional(person_inner_decoder),
40
+
)
41
+
use day <- decode.field(5, decoders.fixed(32))
42
+
43
+
Person(id:, age:, score:, self: person, day:) |> decode.success
44
+
}
45
+
46
+
pub fn person_pb_test() {
47
+
let path = "./test/person.pb"
48
+
49
+
let assert Ok(bits) = file.read_bits(from: path)
50
+
let assert Ok(Parsed(value: person, rest: <<>>, pos: _)) =
51
+
parse(from: bits, using: person_decoder())
52
+
53
+
assert person
54
+
== Person(
55
+
id: 42,
56
+
age: 150,
57
+
score: 81_050,
58
+
self: option.Some(Person(
59
+
id: 42,
60
+
age: 150,
61
+
score: 81_050,
62
+
self: option.None,
63
+
day: 22,
64
+
)),
65
+
day: 22,
66
+
)
67
+
}
68
+
69
+
type TwoInts {
70
+
Test(id: Int, age: Int)
71
+
}
72
+
73
+
fn two_ints_decoder() -> Decoder(TwoInts) {
74
+
use id <- decode.field(1, decoders.uint())
75
+
use age <- decode.field(2, decoders.uint())
76
+
77
+
Test(id:, age:) |> decode.success
78
+
}
79
+
80
+
pub fn two_ints_test() {
81
+
let bits = <<
82
+
// field 1
83
+
0x08,
84
+
0x96,
85
+
0x01,
86
+
// field 2
87
+
0x10,
88
+
0x96,
89
+
0xf2,
90
+
0x04,
91
+
>>
92
+
93
+
let assert Ok(Parsed(value: data, rest: <<>>, pos: 7)) =
94
+
parse(from: bits, using: two_ints_decoder())
95
+
assert data == Test(id: 150, age: 80_150)
96
+
}
97
+
98
+
pub type FeedHeader {
99
+
FeedHeader(gtfs_realtime_version: String, timestamp: Int, nyct: NyctHeader)
100
+
}
101
+
102
+
pub type NyctHeader {
103
+
NyctHeader(
104
+
version: String,
105
+
trip_replacement_periods: List(TripReplacementPeriod),
106
+
)
107
+
}
108
+
109
+
pub type TripReplacementPeriod {
110
+
TripReplacementPeriod(route_id: String, replacement_period: Int)
111
+
}
112
+
113
+
const nyct_header_default = NyctHeader(
114
+
version: "1.0",
115
+
trip_replacement_periods: [],
116
+
)
117
+
118
+
const trip_replacement_period_default = TripReplacementPeriod(
119
+
route_id: "",
120
+
replacement_period: 0,
121
+
)
122
+
123
+
fn feed_header_decoder() -> Decoder(FeedHeader) {
124
+
use gtfs_realtime_version <- decode.field(1, decoders.string())
125
+
use timestamp <- decode.field(3, decoders.fixed(64))
126
+
use nyct <- decode.field(
127
+
1001,
128
+
decoders.protobuf(
129
+
using: nyct_header_decoder,
130
+
named: "NyctHeader",
131
+
default: nyct_header_default,
132
+
),
133
+
)
134
+
FeedHeader(gtfs_realtime_version:, timestamp:, nyct:) |> decode.success
135
+
}
136
+
137
+
fn nyct_header_decoder() -> Decoder(NyctHeader) {
138
+
use version <- decode.field(1, decoders.string())
139
+
140
+
use trip_replacement_periods <- decode.field(
141
+
2,
142
+
decode.list(of: decoders.protobuf(
143
+
using: trip_replacement_period_decoder,
144
+
named: "TripReplacementPeriod",
145
+
default: trip_replacement_period_default,
146
+
)),
147
+
)
148
+
149
+
NyctHeader(version:, trip_replacement_periods:) |> decode.success
150
+
}
151
+
152
+
fn trip_replacement_period_decoder() -> Decoder(TripReplacementPeriod) {
153
+
use route_id <- decode.field(1, decoders.string())
154
+
use replacement_period <- decode.field(
155
+
2,
156
+
decoders.protobuf(
157
+
using: trip_replacement_period_time_range_decoder,
158
+
named: "TripReplacePeriod.TimeRange",
159
+
default: 0,
160
+
),
161
+
)
162
+
TripReplacementPeriod(route_id:, replacement_period:) |> decode.success
163
+
}
164
+
165
+
fn trip_replacement_period_time_range_decoder() -> Decoder(Int) {
166
+
use time <- decode.field(2, decoders.fixed(64))
167
+
time |> decode.success
168
+
}
169
+
170
+
pub fn gtfs_test() {
171
+
let path = "./test/gtfs-short.pb"
172
+
let assert Ok(bits) = file.read_bits(from: path)
173
+
let assert Ok(header) = parse(from: bits, using: feed_header_decoder())
174
+
175
+
assert header
176
+
== Parsed(
177
+
FeedHeader(
178
+
"1.0",
179
+
1_758_224_061,
180
+
NyctHeader("1.0", [
181
+
TripReplacementPeriod("1", 1_758_224_061),
182
+
TripReplacementPeriod("2", 1_758_224_061),
183
+
TripReplacementPeriod("5", 1_758_224_061),
184
+
TripReplacementPeriod("6", 1_758_224_061),
185
+
TripReplacementPeriod("7", 1_758_224_061),
186
+
]),
187
+
),
188
+
<<>>,
189
+
102,
190
+
)
191
+
}
192
+
193
+
type PackedFields {
194
+
PackedFields(
195
+
packed: List(Int),
196
+
expanded: List(Int),
197
+
mixed_packed_and_expanded: List(Int),
198
+
)
199
+
}
200
+
201
+
fn packed_fields_decoder() -> Decoder(PackedFields) {
202
+
use packed <- decode.field(
203
+
3,
204
+
decoders.multiple(of: decoders.uint(), using: read_varint),
205
+
)
206
+
use expanded <- decode.field(
207
+
4,
208
+
decoders.multiple(of: decoders.uint(), using: read_varint),
209
+
)
210
+
use mixed_packed_and_expanded <- decode.field(
211
+
11,
212
+
decoders.multiple(of: decoders.fixed(32), using: read_fixed(32)),
213
+
)
214
+
PackedFields(packed:, expanded:, mixed_packed_and_expanded:) |> decode.success
215
+
}
216
+
217
+
pub fn packed_fields_test() {
218
+
let path = "./test/packed-fields.pb"
219
+
let assert Ok(bits) = file.read_bits(from: path)
220
+
let assert Ok(packed_fields) =
221
+
parse(from: bits, using: packed_fields_decoder())
222
+
223
+
assert packed_fields
224
+
== Parsed(
225
+
value: PackedFields(
226
+
packed: [0, 1, 2, 3],
227
+
expanded: [4, 5, 6],
228
+
mixed_packed_and_expanded: [256, 22, 42, 101, 4_294_967_295, 0, 1],
229
+
),
230
+
rest: <<>>,
231
+
pos: 48,
232
+
)
233
+
}
-233
test/protobuf_decode_gleam_test.gleam
-233
test/protobuf_decode_gleam_test.gleam
···
1
-
import gleam/dynamic/decode.{type Decoder}
2
-
import gleam/option
3
-
import gleeunit
4
-
import simplifile as file
5
-
6
-
import decoders
7
-
import protobuf_decode_gleam.{Parsed, parse, read_fixed, read_varint}
8
-
9
-
pub fn main() -> Nil {
10
-
gleeunit.main()
11
-
}
12
-
13
-
type Person {
14
-
Person(id: Int, age: Int, score: Int, self: option.Option(Person), day: Int)
15
-
}
16
-
17
-
const default_person = Person(
18
-
id: 0,
19
-
age: 0,
20
-
score: 0,
21
-
self: option.None,
22
-
day: 0,
23
-
)
24
-
25
-
fn person_decoder() -> Decoder(Person) {
26
-
let person_inner_decoder =
27
-
decoders.protobuf(
28
-
using: person_decoder,
29
-
named: "Person",
30
-
default: default_person,
31
-
)
32
-
33
-
use id <- decode.field(3, decoders.fixed(64))
34
-
use age <- decode.field(1, decoders.uint())
35
-
use score <- decode.field(2, decoders.uint())
36
-
use person <- decode.optional_field(
37
-
4,
38
-
option.None,
39
-
decode.optional(person_inner_decoder),
40
-
)
41
-
use day <- decode.field(5, decoders.fixed(32))
42
-
43
-
Person(id:, age:, score:, self: person, day:) |> decode.success
44
-
}
45
-
46
-
pub fn person_pb_test() {
47
-
let path = "./test/person.pb"
48
-
49
-
let assert Ok(bits) = file.read_bits(from: path)
50
-
let assert Ok(Parsed(value: person, rest: <<>>, pos: _)) =
51
-
parse(from: bits, using: person_decoder())
52
-
53
-
assert person
54
-
== Person(
55
-
id: 42,
56
-
age: 150,
57
-
score: 81_050,
58
-
self: option.Some(Person(
59
-
id: 42,
60
-
age: 150,
61
-
score: 81_050,
62
-
self: option.None,
63
-
day: 22,
64
-
)),
65
-
day: 22,
66
-
)
67
-
}
68
-
69
-
type TwoInts {
70
-
Test(id: Int, age: Int)
71
-
}
72
-
73
-
fn two_ints_decoder() -> Decoder(TwoInts) {
74
-
use id <- decode.field(1, decoders.uint())
75
-
use age <- decode.field(2, decoders.uint())
76
-
77
-
Test(id:, age:) |> decode.success
78
-
}
79
-
80
-
pub fn two_ints_test() {
81
-
let bits = <<
82
-
// field 1
83
-
0x08,
84
-
0x96,
85
-
0x01,
86
-
// field 2
87
-
0x10,
88
-
0x96,
89
-
0xf2,
90
-
0x04,
91
-
>>
92
-
93
-
let assert Ok(Parsed(value: data, rest: <<>>, pos: 7)) =
94
-
parse(from: bits, using: two_ints_decoder())
95
-
assert data == Test(id: 150, age: 80_150)
96
-
}
97
-
98
-
pub type FeedHeader {
99
-
FeedHeader(gtfs_realtime_version: String, timestamp: Int, nyct: NyctHeader)
100
-
}
101
-
102
-
pub type NyctHeader {
103
-
NyctHeader(
104
-
version: String,
105
-
trip_replacement_periods: List(TripReplacementPeriod),
106
-
)
107
-
}
108
-
109
-
pub type TripReplacementPeriod {
110
-
TripReplacementPeriod(route_id: String, replacement_period: Int)
111
-
}
112
-
113
-
const nyct_header_default = NyctHeader(
114
-
version: "1.0",
115
-
trip_replacement_periods: [],
116
-
)
117
-
118
-
const trip_replacement_period_default = TripReplacementPeriod(
119
-
route_id: "",
120
-
replacement_period: 0,
121
-
)
122
-
123
-
fn feed_header_decoder() -> Decoder(FeedHeader) {
124
-
use gtfs_realtime_version <- decode.field(1, decoders.string())
125
-
use timestamp <- decode.field(3, decoders.fixed(64))
126
-
use nyct <- decode.field(
127
-
1001,
128
-
decoders.protobuf(
129
-
using: nyct_header_decoder,
130
-
named: "NyctHeader",
131
-
default: nyct_header_default,
132
-
),
133
-
)
134
-
FeedHeader(gtfs_realtime_version:, timestamp:, nyct:) |> decode.success
135
-
}
136
-
137
-
fn nyct_header_decoder() -> Decoder(NyctHeader) {
138
-
use version <- decode.field(1, decoders.string())
139
-
140
-
use trip_replacement_periods <- decode.field(
141
-
2,
142
-
decode.list(of: decoders.protobuf(
143
-
using: trip_replacement_period_decoder,
144
-
named: "TripReplacementPeriod",
145
-
default: trip_replacement_period_default,
146
-
)),
147
-
)
148
-
149
-
NyctHeader(version:, trip_replacement_periods:) |> decode.success
150
-
}
151
-
152
-
fn trip_replacement_period_decoder() -> Decoder(TripReplacementPeriod) {
153
-
use route_id <- decode.field(1, decoders.string())
154
-
use replacement_period <- decode.field(
155
-
2,
156
-
decoders.protobuf(
157
-
using: trip_replacement_period_time_range_decoder,
158
-
named: "TripReplacePeriod.TimeRange",
159
-
default: 0,
160
-
),
161
-
)
162
-
TripReplacementPeriod(route_id:, replacement_period:) |> decode.success
163
-
}
164
-
165
-
fn trip_replacement_period_time_range_decoder() -> Decoder(Int) {
166
-
use time <- decode.field(2, decoders.fixed(64))
167
-
time |> decode.success
168
-
}
169
-
170
-
pub fn gtfs_test() {
171
-
let path = "./test/gtfs-short.pb"
172
-
let assert Ok(bits) = file.read_bits(from: path)
173
-
let assert Ok(header) = parse(from: bits, using: feed_header_decoder())
174
-
175
-
assert header
176
-
== Parsed(
177
-
FeedHeader(
178
-
"1.0",
179
-
1_758_224_061,
180
-
NyctHeader("1.0", [
181
-
TripReplacementPeriod("1", 1_758_224_061),
182
-
TripReplacementPeriod("2", 1_758_224_061),
183
-
TripReplacementPeriod("5", 1_758_224_061),
184
-
TripReplacementPeriod("6", 1_758_224_061),
185
-
TripReplacementPeriod("7", 1_758_224_061),
186
-
]),
187
-
),
188
-
<<>>,
189
-
102,
190
-
)
191
-
}
192
-
193
-
type PackedFields {
194
-
PackedFields(
195
-
packed: List(Int),
196
-
expanded: List(Int),
197
-
mixed_packed_and_expanded: List(Int),
198
-
)
199
-
}
200
-
201
-
fn packed_fields_decoder() -> Decoder(PackedFields) {
202
-
use packed <- decode.field(
203
-
3,
204
-
decoders.multiple(of: decoders.uint(), using: read_varint),
205
-
)
206
-
use expanded <- decode.field(
207
-
4,
208
-
decoders.multiple(of: decoders.uint(), using: read_varint),
209
-
)
210
-
use mixed_packed_and_expanded <- decode.field(
211
-
11,
212
-
decoders.multiple(of: decoders.fixed(32), using: read_fixed(32)),
213
-
)
214
-
PackedFields(packed:, expanded:, mixed_packed_and_expanded:) |> decode.success
215
-
}
216
-
217
-
pub fn packed_fields_test() {
218
-
let path = "./test/packed-fields.pb"
219
-
let assert Ok(bits) = file.read_bits(from: path)
220
-
let assert Ok(packed_fields) =
221
-
parse(from: bits, using: packed_fields_decoder())
222
-
223
-
assert packed_fields
224
-
== Parsed(
225
-
value: PackedFields(
226
-
packed: [0, 1, 2, 3],
227
-
expanded: [4, 5, 6],
228
-
mixed_packed_and_expanded: [256, 22, 42, 101, 4_294_967_295, 0, 1],
229
-
),
230
-
rest: <<>>,
231
-
pos: 48,
232
-
)
233
-
}