forked from
karlseguin.tngl.sh/buffer.zig
A poolable string builder (aka string buffer) for Zig
added views
Karl Seguin
(Aug 3, 2023, 5:09 PM +0800)
58207904
dae0bcb9
+196
-70
+33
readme.md
+33
readme.md
···
25
25
26
26
You can use `writeU16Big`, `writeU32Big`, `writeU64Big` and `writeU16Little`, `writeU32Little`, `writeU64Little` to write integer values.
27
27
28
+
## Views
29
+
A common pattern is to include a 2 or 4 byte payload length prefix to messages. However, this length might not until after the message is generated. The `skip` and `view` functions exist specifically to solve the problem - but they must be used carefully:
30
+
31
+
```zig
32
+
var buf = try Buffer.init(allocator, 100);
33
+
34
+
// reserve 4 bytes for our length
35
+
var start = buf.skip(4);
36
+
try buf.write("hello world");
37
+
try buf.writeByte('!');
38
+
39
+
var view = buf.view(start);
40
+
// -4 since most protocols don't include the 4 byte length in the length itself
41
+
view.writeU32Little(@intCast(buf.len() - 4);
42
+
```
43
+
44
+
The `view` exposes most of the same methods as the Buffer, but cannot grow and does not perform bound checking. Furthermore, interlacing writes to the view and the buffer can segfault. You can safely write to the buffer after you're done writing to the view.
45
+
46
+
For example, the following can segfault:
47
+
48
+
```zig
49
+
var buf = try Buffer.init(allocator, 100);
50
+
51
+
// reserve 4 bytes for our length
52
+
var start = buf.skip(4);
53
+
try buf.write("hello world");
54
+
55
+
var view = buf.view(start);
56
+
try buf.writeByte('!');
57
+
// view might not be valid as we've since written to buf
58
+
view.writeU32Little(@intCast(buf.len() - 4);
59
+
```
60
+
28
61
## Pooling
29
62
30
63
```zig
+158
-65
src/buffer.zig
+158
-65
src/buffer.zig
···
3
3
const Allocator = std.mem.Allocator;
4
4
pub const Pool = @import("pool.zig").Pool;
5
5
6
+
pub const View = struct {
7
+
// points to either static or dynamic,
8
+
buf: []u8,
9
+
10
+
// position in buf that we're at
11
+
pos: usize,
12
+
13
+
pub fn len(self: View) usize {
14
+
return self.pos;
15
+
}
16
+
17
+
pub fn string(self: View) []u8 {
18
+
return self.buf[0..self.pos];
19
+
}
20
+
21
+
pub fn truncate(self: *View, n: usize) void {
22
+
const pos = self.pos;
23
+
if (n >= pos) {
24
+
self.pos = 0;
25
+
return;
26
+
}
27
+
self.pos = pos - n;
28
+
}
29
+
30
+
pub fn skip(self: *View, n: usize) usize {
31
+
const pos = self.pos;
32
+
const end_pos = pos + n;
33
+
self.pos = end_pos;
34
+
return pos;
35
+
}
36
+
37
+
pub fn copy(self: View, allocator: Allocator) ![]u8 {
38
+
const pos = self.pos;
39
+
var c = try allocator.alloc(u8, pos);
40
+
@memcpy(c, self.buf[0..pos]);
41
+
return c;
42
+
}
43
+
44
+
pub fn writeByte(self: *View, b: u8) void {
45
+
const pos = self.pos;
46
+
self.buf[pos] = b;
47
+
self.pos = pos + 1;
48
+
}
49
+
50
+
pub fn writeByteNTimes(self: *View, b: u8, n: usize) void {
51
+
const pos = self.pos;
52
+
const buf = self.buf;
53
+
for (0..n) |offset| {
54
+
buf[pos+offset] = b;
55
+
}
56
+
self.pos = pos + n;
57
+
}
58
+
59
+
pub fn write(self: *View, data: []const u8) void {
60
+
const pos = self.pos;
61
+
const end_pos = pos + data.len;
62
+
std.mem.copyForwards(u8, self.buf[pos..end_pos], data);
63
+
self.pos = end_pos;
64
+
}
65
+
66
+
pub fn writeU16Little(self: *View, value: u16) void {
67
+
const pos = self.pos;
68
+
const end_pos = self.pos + 2;
69
+
std.mem.writeIntLittle(u16, self.buf[pos..end_pos][0..2], value);
70
+
self.pos = end_pos;
71
+
}
72
+
73
+
pub fn writeU32Little(self: *View, value: u32) void {
74
+
const pos = self.pos;
75
+
const end_pos = self.pos + 4;
76
+
std.mem.writeIntLittle(u32, self.buf[pos..end_pos][0..4], value);
77
+
self.pos = end_pos;
78
+
}
79
+
80
+
pub fn writeU64Little(self: *View, value: u64) void {
81
+
const pos = self.pos;
82
+
const end_pos = self.pos + 8;
83
+
std.mem.writeIntLittle(u64, self.buf[pos..end_pos][0..8], value);
84
+
self.pos = end_pos;
85
+
}
86
+
87
+
pub fn writeU16Big(self: *View, value: u16) void {
88
+
const pos = self.pos;
89
+
const end_pos = self.pos + 2;
90
+
std.mem.writeIntBig(u16, self.buf[pos..end_pos][0..2], value);
91
+
self.pos = end_pos;
92
+
}
93
+
94
+
pub fn writeU32Big(self: *View, value: u32) void {
95
+
const pos = self.pos;
96
+
const end_pos = self.pos + 4;
97
+
std.mem.writeIntBig(u32, self.buf[pos..end_pos][0..4], value);
98
+
self.pos = end_pos;
99
+
}
100
+
101
+
pub fn writeU64Big(self: *View, value: u64) void {
102
+
const pos = self.pos;
103
+
const end_pos = self.pos + 8;
104
+
std.mem.writeIntBig(u64, self.buf[pos..end_pos][0..8], value);
105
+
self.pos = end_pos;
106
+
}
107
+
};
108
+
6
109
pub const Buffer = struct {
7
110
// Two allocators! This is largely a feature meant to be used with the Pool.
8
111
// Imagine you have a pool of 100 StringBuilders. Each one has a static buffer
···
18
121
19
122
_da: ?Allocator,
20
123
21
-
// where in buf we currently are
22
-
pos: usize,
124
+
_view: View,
23
125
24
126
// fixed size, created on startup
25
127
static: []u8,
···
27
129
// created when we try to write more than static.len
28
130
dynamic: ?[]u8,
29
131
30
-
// points to either static or dynamic,
31
-
buf: []u8,
32
132
33
133
pub fn init(allocator: Allocator, size: usize) !Buffer {
34
134
const static = try allocator.alloc(u8, size);
35
135
return .{
36
136
._a = allocator,
37
137
._da = null,
38
-
.pos = 0,
39
138
.dynamic = null,
40
-
.buf = static,
41
139
.static = static,
140
+
._view = .{
141
+
.pos = 0,
142
+
.buf = static,
143
+
},
42
144
};
43
145
}
44
146
···
51
153
}
52
154
53
155
pub fn reset(self: *Buffer, clear_dynamic: bool) void {
54
-
self.pos = 0;
156
+
self._view.pos = 0;
55
157
if (clear_dynamic) {
56
158
if (self.dynamic) |dyn| {
57
159
(self._da orelse self._a).free(dyn);
58
160
self.dynamic = null;
59
-
self.buf = self.static;
161
+
self._view.buf = self.static;
60
162
}
61
163
self._da = null;
62
164
}
63
165
}
64
166
65
167
pub fn len(self: Buffer) usize {
66
-
return self.pos;
168
+
return self._view.pos;
67
169
}
68
170
69
171
pub fn string(self: Buffer) []const u8 {
70
-
return self.buf[0..self.pos];
172
+
return self._view.string();
71
173
}
72
174
73
175
pub fn truncate(self: *Buffer, n: usize) void {
74
-
const pos = self.pos;
75
-
if (n >= pos) {
76
-
self.pos = 0;
77
-
return;
78
-
}
79
-
self.pos = pos - n;
176
+
self._view.truncate(n);
80
177
}
81
178
82
179
pub fn writeByte(self: *Buffer, b: u8) !void {
83
180
try self.ensureUnusedCapacity(1);
84
-
self.writeByteAssumeCapacity(b);
181
+
self._view.writeByte(b);
85
182
}
86
183
87
184
pub fn writeByteAssumeCapacity(self: *Buffer, b: u8) void {
88
-
const pos = self.pos;
89
-
self.buf[pos] = b;
90
-
self.pos = pos + 1;
185
+
self._view.writeByte(b);
91
186
}
187
+
92
188
93
189
pub fn writeByteNTimes(self: *Buffer, b: u8, n: usize) !void {
94
190
try self.ensureUnusedCapacity(n);
95
-
const pos = self.pos;
96
-
const buf = self.buf;
97
-
for (0..n) |offset| {
98
-
buf[pos+offset] = b;
99
-
}
100
-
self.pos = pos + n;
191
+
self._view.writeByteNTimes(b, n);
101
192
}
102
193
103
194
pub fn write(self: *Buffer, data: []const u8) !void {
104
195
try self.ensureUnusedCapacity(data.len);
105
-
return self.writeAssumeCapacity(data);
196
+
self._view.write(data);
197
+
}
198
+
199
+
pub fn writeAssumeCapacity(self: *Buffer, data: []const u8) void {
200
+
self._view.write(data);
106
201
}
107
202
108
203
pub fn writeU16Little(self: *Buffer, value: u16) !void {
109
204
try self.ensureUnusedCapacity(2);
110
-
const pos = self.pos;
111
-
const end_pos = self.pos + 2;
112
-
std.mem.writeIntLittle(u16, self.buf[pos..end_pos][0..2], value);
113
-
self.pos = end_pos;
205
+
self._view.writeU16Little(value);
114
206
}
115
207
116
208
pub fn writeU32Little(self: *Buffer, value: u32) !void {
117
209
try self.ensureUnusedCapacity(4);
118
-
const pos = self.pos;
119
-
const end_pos = self.pos + 4;
120
-
std.mem.writeIntLittle(u32, self.buf[pos..end_pos][0..4], value);
121
-
self.pos = end_pos;
210
+
self._view.writeU32Little(value);
122
211
}
123
212
124
213
pub fn writeU64Little(self: *Buffer, value: u64) !void {
125
214
try self.ensureUnusedCapacity(8);
126
-
const pos = self.pos;
127
-
const end_pos = self.pos + 8;
128
-
std.mem.writeIntLittle(u64, self.buf[pos..end_pos][0..8], value);
129
-
self.pos = end_pos;
215
+
self._view.writeU64Little(value);
130
216
}
131
217
132
218
pub fn writeU16Big(self: *Buffer, value: u16) !void {
133
219
try self.ensureUnusedCapacity(2);
134
-
const pos = self.pos;
135
-
const end_pos = self.pos + 2;
136
-
std.mem.writeIntBig(u16, self.buf[pos..end_pos][0..2], value);
137
-
self.pos = end_pos;
220
+
self._view.writeU16Big(value);
138
221
}
139
222
140
223
pub fn writeU32Big(self: *Buffer, value: u32) !void {
141
224
try self.ensureUnusedCapacity(4);
142
-
const pos = self.pos;
143
-
const end_pos = self.pos + 4;
144
-
std.mem.writeIntBig(u32, self.buf[pos..end_pos][0..4], value);
145
-
self.pos = end_pos;
225
+
self._view.writeU32Big(value);
146
226
}
147
227
148
228
pub fn writeU64Big(self: *Buffer, value: u64) !void {
149
229
try self.ensureUnusedCapacity(8);
150
-
const pos = self.pos;
151
-
const end_pos = self.pos + 8;
152
-
std.mem.writeIntBig(u64, self.buf[pos..end_pos][0..8], value);
153
-
self.pos = end_pos;
230
+
self._view.writeU64Big(value);
154
231
}
155
232
156
-
pub fn writeAssumeCapacity(self: *Buffer, data: []const u8) void {
157
-
const pos = self.pos;
158
-
const end_pos = pos + data.len;
159
-
std.mem.copyForwards(u8, self.buf[pos..end_pos], data);
160
-
self.pos = end_pos;
233
+
pub fn skip(self: *Buffer, n: usize) !usize {
234
+
try self.ensureUnusedCapacity(n);
235
+
return self._view.skip(n);
236
+
}
237
+
238
+
pub fn view(self: *Buffer, pos: usize) View {
239
+
return .{
240
+
.pos = 0,
241
+
.buf = self._view.buf[pos..],
242
+
};
161
243
}
162
244
163
245
pub fn ensureUnusedCapacity(self: *Buffer, n: usize) !void {
164
-
return self.ensureTotalCapacity(self.pos + n);
246
+
return self.ensureTotalCapacity(self._view.pos + n);
165
247
}
166
248
167
249
pub fn ensureTotalCapacity(self: *Buffer, required_capacity: usize) !void {
168
-
const buf = self.buf;
250
+
const buf = self._view.buf;
169
251
if (required_capacity <= buf.len) {
170
252
return;
171
253
}
172
254
173
255
// from std.ArrayList
174
-
var new_capacity = self.buf.len;
256
+
var new_capacity = buf.len;
175
257
while (true) {
176
258
new_capacity +|= new_capacity / 2 + 8;
177
259
if (new_capacity >= required_capacity) break;
···
186
268
allocator.free(dyn);
187
269
}
188
270
189
-
self.buf = new_buffer;
271
+
self._view.buf = new_buffer;
190
272
self.dynamic = new_buffer;
191
273
} else {
192
274
const new_buffer = buf.ptr[0..new_capacity];
193
-
self.buf = new_buffer;
275
+
self._view.buf = new_buffer;
194
276
self.dynamic = new_buffer;
195
277
}
196
278
}
197
279
198
280
pub fn copy(self: Buffer, allocator: Allocator) ![]const u8 {
199
-
const pos = self.pos;
200
-
var c = try allocator.alloc(u8, pos);
201
-
@memcpy(c, self.buf[0..pos]);
202
-
return c;
281
+
return self._view.copy(allocator);
203
282
}
204
283
205
284
pub fn writer(self: *Buffer) Writer.IOWriter {
···
370
449
371
450
try sb.writeU16Big(15000);
372
451
try t.exectSlice(u8, &[_]u8{155, 233, 51, 249, 7, 209, 129, 21, 195, 187, 171, 56, 58, 152}, sb.string());
452
+
}
453
+
454
+
test "skip & view" {
455
+
var sb = try Buffer.init(t.allocator, 10);
456
+
defer sb.deinit();
457
+
458
+
const start = try sb.skip(4);
459
+
try sb.write("hello world!!");
460
+
461
+
var v = sb.view(start);
462
+
v.writeU32Big(@intCast(sb.len() - 4));
463
+
464
+
try sb.writeByte('\n');
465
+
try t.exectSlice(u8, &[_]u8{0, 0, 0, 13, 'h', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', '!', '\n'}, sb.string());
373
466
}
374
467
375
468
fn testString(allocator: Allocator, random: std.rand.Random) []const u8 {
+5
-5
src/pool.zig
+5
-5
src/pool.zig
···
56
56
57
57
const sb = try allocator.create(Buffer);
58
58
sb.* = try Buffer.init(allocator, self.buffer_size);
59
-
if (comptime builtin.is_test) sb.buf[0] = 0;
59
+
if (comptime builtin.is_test) sb._view.buf[0] = 0;
60
60
sb._da = dyn_allocator;
61
61
return sb;
62
62
}
···
130
130
131
131
// initialize this to 0 since we're asserting that it's 0
132
132
for (p.buffers) |sb| {
133
-
sb.buf[0] = 0;
133
+
sb._view.buf[0] = 0;
134
134
}
135
135
136
136
const t1 = try std.Thread.spawn(.{}, testPool, .{&p});
···
149
149
for (0..5000) |_| {
150
150
var sb = p.acquire() catch unreachable;
151
151
// no other thread should have set this to 255
152
-
std.debug.assert(sb.buf[0] == 0);
152
+
std.debug.assert(sb._view.buf[0] == 0);
153
153
154
-
sb.buf[0] = 255;
154
+
sb._view.buf[0] = 255;
155
155
std.time.sleep(random.uintAtMost(u32, 100000));
156
-
sb.buf[0] = 0;
156
+
sb._view.buf[0] = 0;
157
157
p.release(sb);
158
158
}
159
159
}