authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-31 22:38:38-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-07 10:04:52-07:00
log366884ab067f6b2f075cabe372ce4851417579e7
tree27b664e62e7e2d307c0186671a113f628b6240fd
parent3f8ed5504ef1968611b59282490262450f99165d

remove std.fifo

I never liked how this data structure took its API as a parameter. This use case is now served by std.Io buffering.

2 files changed, 0 insertions(+), 549 deletions(-)

lib/std/fifo.zig deleted-548
......@@ -1,548 +0,0 @@
1// FIFO of fixed size items
2// Usually used for e.g. byte buffers
3
4const std = @import("std");
5const math = std.math;
6const mem = std.mem;
7const Allocator = mem.Allocator;
8const assert = std.debug.assert;
9const testing = std.testing;
10
11pub const LinearFifoBufferType = union(enum) {
12 /// The buffer is internal to the fifo; it is of the specified size.
13 Static: usize,
14
15 /// The buffer is passed as a slice to the initialiser.
16 Slice,
17
18 /// The buffer is managed dynamically using a `mem.Allocator`.
19 Dynamic,
20};
21
22pub fn LinearFifo(
23 comptime T: type,
24 comptime buffer_type: LinearFifoBufferType,
25) type {
26 const autoalign = false;
27
28 const powers_of_two = switch (buffer_type) {
29 .Static => std.math.isPowerOfTwo(buffer_type.Static),
30 .Slice => false, // Any size slice could be passed in
31 .Dynamic => true, // This could be configurable in future
32 };
33
34 return struct {
35 allocator: if (buffer_type == .Dynamic) Allocator else void,
36 buf: if (buffer_type == .Static) [buffer_type.Static]T else []T,
37 head: usize,
38 count: usize,
39
40 const Self = @This();
41 pub const Reader = std.io.GenericReader(*Self, error{}, readFn);
42 pub const Writer = std.io.GenericWriter(*Self, error{OutOfMemory}, appendWrite);
43
44 // Type of Self argument for slice operations.
45 // If buffer is inline (Static) then we need to ensure we haven't
46 // returned a slice into a copy on the stack
47 const SliceSelfArg = if (buffer_type == .Static) *Self else Self;
48
49 pub const init = switch (buffer_type) {
50 .Static => initStatic,
51 .Slice => initSlice,
52 .Dynamic => initDynamic,
53 };
54
55 fn initStatic() Self {
56 comptime assert(buffer_type == .Static);
57 return .{
58 .allocator = {},
59 .buf = undefined,
60 .head = 0,
61 .count = 0,
62 };
63 }
64
65 fn initSlice(buf: []T) Self {
66 comptime assert(buffer_type == .Slice);
67 return .{
68 .allocator = {},
69 .buf = buf,
70 .head = 0,
71 .count = 0,
72 };
73 }
74
75 fn initDynamic(allocator: Allocator) Self {
76 comptime assert(buffer_type == .Dynamic);
77 return .{
78 .allocator = allocator,
79 .buf = &.{},
80 .head = 0,
81 .count = 0,
82 };
83 }
84
85 pub fn deinit(self: Self) void {
86 if (buffer_type == .Dynamic) self.allocator.free(self.buf);
87 }
88
89 pub fn realign(self: *Self) void {
90 if (self.buf.len - self.head >= self.count) {
91 mem.copyForwards(T, self.buf[0..self.count], self.buf[self.head..][0..self.count]);
92 self.head = 0;
93 } else {
94 var tmp: [4096 / 2 / @sizeOf(T)]T = undefined;
95
96 while (self.head != 0) {
97 const n = @min(self.head, tmp.len);
98 const m = self.buf.len - n;
99 @memcpy(tmp[0..n], self.buf[0..n]);
100 mem.copyForwards(T, self.buf[0..m], self.buf[n..][0..m]);
101 @memcpy(self.buf[m..][0..n], tmp[0..n]);
102 self.head -= n;
103 }
104 }
105 { // set unused area to undefined
106 const unused = mem.sliceAsBytes(self.buf[self.count..]);
107 @memset(unused, undefined);
108 }
109 }
110
111 /// Reduce allocated capacity to `size`.
112 pub fn shrink(self: *Self, size: usize) void {
113 assert(size >= self.count);
114 if (buffer_type == .Dynamic) {
115 self.realign();
116 self.buf = self.allocator.realloc(self.buf, size) catch |e| switch (e) {
117 error.OutOfMemory => return, // no problem, capacity is still correct then.
118 };
119 }
120 }
121
122 /// Ensure that the buffer can fit at least `size` items
123 pub fn ensureTotalCapacity(self: *Self, size: usize) !void {
124 if (self.buf.len >= size) return;
125 if (buffer_type == .Dynamic) {
126 self.realign();
127 const new_size = if (powers_of_two) math.ceilPowerOfTwo(usize, size) catch return error.OutOfMemory else size;
128 self.buf = try self.allocator.realloc(self.buf, new_size);
129 } else {
130 return error.OutOfMemory;
131 }
132 }
133
134 /// Makes sure at least `size` items are unused
135 pub fn ensureUnusedCapacity(self: *Self, size: usize) error{OutOfMemory}!void {
136 if (self.writableLength() >= size) return;
137
138 return try self.ensureTotalCapacity(math.add(usize, self.count, size) catch return error.OutOfMemory);
139 }
140
141 /// Returns number of items currently in fifo
142 pub fn readableLength(self: Self) usize {
143 return self.count;
144 }
145
146 /// Returns a writable slice from the 'read' end of the fifo
147 fn readableSliceMut(self: SliceSelfArg, offset: usize) []T {
148 if (offset > self.count) return &[_]T{};
149
150 var start = self.head + offset;
151 if (start >= self.buf.len) {
152 start -= self.buf.len;
153 return self.buf[start .. start + (self.count - offset)];
154 } else {
155 const end = @min(self.head + self.count, self.buf.len);
156 return self.buf[start..end];
157 }
158 }
159
160 /// Returns a readable slice from `offset`
161 pub fn readableSlice(self: SliceSelfArg, offset: usize) []const T {
162 return self.readableSliceMut(offset);
163 }
164
165 pub fn readableSliceOfLen(self: *Self, len: usize) []const T {
166 assert(len <= self.count);
167 const buf = self.readableSlice(0);
168 if (buf.len >= len) {
169 return buf[0..len];
170 } else {
171 self.realign();
172 return self.readableSlice(0)[0..len];
173 }
174 }
175
176 /// Discard first `count` items in the fifo
177 pub fn discard(self: *Self, count: usize) void {
178 assert(count <= self.count);
179 { // set old range to undefined. Note: may be wrapped around
180 const slice = self.readableSliceMut(0);
181 if (slice.len >= count) {
182 const unused = mem.sliceAsBytes(slice[0..count]);
183 @memset(unused, undefined);
184 } else {
185 const unused = mem.sliceAsBytes(slice[0..]);
186 @memset(unused, undefined);
187 const unused2 = mem.sliceAsBytes(self.readableSliceMut(slice.len)[0 .. count - slice.len]);
188 @memset(unused2, undefined);
189 }
190 }
191 if (autoalign and self.count == count) {
192 self.head = 0;
193 self.count = 0;
194 } else {
195 var head = self.head + count;
196 if (powers_of_two) {
197 // Note it is safe to do a wrapping subtract as
198 // bitwise & with all 1s is a noop
199 head &= self.buf.len -% 1;
200 } else {
201 head %= self.buf.len;
202 }
203 self.head = head;
204 self.count -= count;
205 }
206 }
207
208 /// Read the next item from the fifo
209 pub fn readItem(self: *Self) ?T {
210 if (self.count == 0) return null;
211
212 const c = self.buf[self.head];
213 self.discard(1);
214 return c;
215 }
216
217 /// Read data from the fifo into `dst`, returns number of items copied.
218 pub fn read(self: *Self, dst: []T) usize {
219 var dst_left = dst;
220
221 while (dst_left.len > 0) {
222 const slice = self.readableSlice(0);
223 if (slice.len == 0) break;
224 const n = @min(slice.len, dst_left.len);
225 @memcpy(dst_left[0..n], slice[0..n]);
226 self.discard(n);
227 dst_left = dst_left[n..];
228 }
229
230 return dst.len - dst_left.len;
231 }
232
233 /// Same as `read` except it returns an error union
234 /// The purpose of this function existing is to match `std.io.GenericReader` API.
235 fn readFn(self: *Self, dest: []u8) error{}!usize {
236 return self.read(dest);
237 }
238
239 pub fn reader(self: *Self) Reader {
240 return .{ .context = self };
241 }
242
243 /// Returns number of items available in fifo
244 pub fn writableLength(self: Self) usize {
245 return self.buf.len - self.count;
246 }
247
248 /// Returns the first section of writable buffer.
249 /// Note that this may be of length 0
250 pub fn writableSlice(self: SliceSelfArg, offset: usize) []T {
251 if (offset > self.buf.len) return &[_]T{};
252
253 const tail = self.head + offset + self.count;
254 if (tail < self.buf.len) {
255 return self.buf[tail..];
256 } else {
257 return self.buf[tail - self.buf.len ..][0 .. self.writableLength() - offset];
258 }
259 }
260
261 /// Returns a writable buffer of at least `size` items, allocating memory as needed.
262 /// Use `fifo.update` once you've written data to it.
263 pub fn writableWithSize(self: *Self, size: usize) ![]T {
264 try self.ensureUnusedCapacity(size);
265
266 // try to avoid realigning buffer
267 var slice = self.writableSlice(0);
268 if (slice.len < size) {
269 self.realign();
270 slice = self.writableSlice(0);
271 }
272 return slice;
273 }
274
275 /// Update the tail location of the buffer (usually follows use of writable/writableWithSize)
276 pub fn update(self: *Self, count: usize) void {
277 assert(self.count + count <= self.buf.len);
278 self.count += count;
279 }
280
281 /// Appends the data in `src` to the fifo.
282 /// You must have ensured there is enough space.
283 pub fn writeAssumeCapacity(self: *Self, src: []const T) void {
284 assert(self.writableLength() >= src.len);
285
286 var src_left = src;
287 while (src_left.len > 0) {
288 const writable_slice = self.writableSlice(0);
289 assert(writable_slice.len != 0);
290 const n = @min(writable_slice.len, src_left.len);
291 @memcpy(writable_slice[0..n], src_left[0..n]);
292 self.update(n);
293 src_left = src_left[n..];
294 }
295 }
296
297 /// Write a single item to the fifo
298 pub fn writeItem(self: *Self, item: T) !void {
299 try self.ensureUnusedCapacity(1);
300 return self.writeItemAssumeCapacity(item);
301 }
302
303 pub fn writeItemAssumeCapacity(self: *Self, item: T) void {
304 var tail = self.head + self.count;
305 if (powers_of_two) {
306 tail &= self.buf.len - 1;
307 } else {
308 tail %= self.buf.len;
309 }
310 self.buf[tail] = item;
311 self.update(1);
312 }
313
314 /// Appends the data in `src` to the fifo.
315 /// Allocates more memory as necessary
316 pub fn write(self: *Self, src: []const T) !void {
317 try self.ensureUnusedCapacity(src.len);
318
319 return self.writeAssumeCapacity(src);
320 }
321
322 /// Same as `write` except it returns the number of bytes written, which is always the same
323 /// as `bytes.len`. The purpose of this function existing is to match `std.io.GenericWriter` API.
324 fn appendWrite(self: *Self, bytes: []const u8) error{OutOfMemory}!usize {
325 try self.write(bytes);
326 return bytes.len;
327 }
328
329 pub fn writer(self: *Self) Writer {
330 return .{ .context = self };
331 }
332
333 /// Make `count` items available before the current read location
334 fn rewind(self: *Self, count: usize) void {
335 assert(self.writableLength() >= count);
336
337 var head = self.head + (self.buf.len - count);
338 if (powers_of_two) {
339 head &= self.buf.len - 1;
340 } else {
341 head %= self.buf.len;
342 }
343 self.head = head;
344 self.count += count;
345 }
346
347 /// Place data back into the read stream
348 pub fn unget(self: *Self, src: []const T) !void {
349 try self.ensureUnusedCapacity(src.len);
350
351 self.rewind(src.len);
352
353 const slice = self.readableSliceMut(0);
354 if (src.len < slice.len) {
355 @memcpy(slice[0..src.len], src);
356 } else {
357 @memcpy(slice, src[0..slice.len]);
358 const slice2 = self.readableSliceMut(slice.len);
359 @memcpy(slice2[0 .. src.len - slice.len], src[slice.len..]);
360 }
361 }
362
363 /// Returns the item at `offset`.
364 /// Asserts offset is within bounds.
365 pub fn peekItem(self: Self, offset: usize) T {
366 assert(offset < self.count);
367
368 var index = self.head + offset;
369 if (powers_of_two) {
370 index &= self.buf.len - 1;
371 } else {
372 index %= self.buf.len;
373 }
374 return self.buf[index];
375 }
376
377 /// Pump data from a reader into a writer.
378 /// Stops when reader returns 0 bytes (EOF).
379 /// Buffer size must be set before calling; a buffer length of 0 is invalid.
380 pub fn pump(self: *Self, src_reader: anytype, dest_writer: anytype) !void {
381 assert(self.buf.len > 0);
382 while (true) {
383 if (self.writableLength() > 0) {
384 const n = try src_reader.read(self.writableSlice(0));
385 if (n == 0) break; // EOF
386 self.update(n);
387 }
388 self.discard(try dest_writer.write(self.readableSlice(0)));
389 }
390 // flush remaining data
391 while (self.readableLength() > 0) {
392 self.discard(try dest_writer.write(self.readableSlice(0)));
393 }
394 }
395
396 pub fn toOwnedSlice(self: *Self) Allocator.Error![]T {
397 if (self.head != 0) self.realign();
398 assert(self.head == 0);
399 assert(self.count <= self.buf.len);
400 const allocator = self.allocator;
401 if (allocator.resize(self.buf, self.count)) {
402 const result = self.buf[0..self.count];
403 self.* = Self.init(allocator);
404 return result;
405 }
406 const new_memory = try allocator.dupe(T, self.buf[0..self.count]);
407 allocator.free(self.buf);
408 self.* = Self.init(allocator);
409 return new_memory;
410 }
411 };
412}
413
414test "LinearFifo(u8, .Dynamic) discard(0) from empty buffer should not error on overflow" {
415 var fifo = LinearFifo(u8, .Dynamic).init(testing.allocator);
416 defer fifo.deinit();
417
418 // If overflow is not explicitly allowed this will crash in debug / safe mode
419 fifo.discard(0);
420}
421
422test "LinearFifo(u8, .Dynamic)" {
423 var fifo = LinearFifo(u8, .Dynamic).init(testing.allocator);
424 defer fifo.deinit();
425
426 try fifo.write("HELLO");
427 try testing.expectEqual(@as(usize, 5), fifo.readableLength());
428 try testing.expectEqualSlices(u8, "HELLO", fifo.readableSlice(0));
429
430 {
431 var i: usize = 0;
432 while (i < 5) : (i += 1) {
433 try fifo.write(&[_]u8{fifo.peekItem(i)});
434 }
435 try testing.expectEqual(@as(usize, 10), fifo.readableLength());
436 try testing.expectEqualSlices(u8, "HELLOHELLO", fifo.readableSlice(0));
437 }
438
439 {
440 try testing.expectEqual(@as(u8, 'H'), fifo.readItem().?);
441 try testing.expectEqual(@as(u8, 'E'), fifo.readItem().?);
442 try testing.expectEqual(@as(u8, 'L'), fifo.readItem().?);
443 try testing.expectEqual(@as(u8, 'L'), fifo.readItem().?);
444 try testing.expectEqual(@as(u8, 'O'), fifo.readItem().?);
445 }
446 try testing.expectEqual(@as(usize, 5), fifo.readableLength());
447
448 { // Writes that wrap around
449 try testing.expectEqual(@as(usize, 11), fifo.writableLength());
450 try testing.expectEqual(@as(usize, 6), fifo.writableSlice(0).len);
451 fifo.writeAssumeCapacity("6<chars<11");
452 try testing.expectEqualSlices(u8, "HELLO6<char", fifo.readableSlice(0));
453 try testing.expectEqualSlices(u8, "s<11", fifo.readableSlice(11));
454 try testing.expectEqualSlices(u8, "11", fifo.readableSlice(13));
455 try testing.expectEqualSlices(u8, "", fifo.readableSlice(15));
456 fifo.discard(11);
457 try testing.expectEqualSlices(u8, "s<11", fifo.readableSlice(0));
458 fifo.discard(4);
459 try testing.expectEqual(@as(usize, 0), fifo.readableLength());
460 }
461
462 {
463 const buf = try fifo.writableWithSize(12);
464 try testing.expectEqual(@as(usize, 12), buf.len);
465 var i: u8 = 0;
466 while (i < 10) : (i += 1) {
467 buf[i] = i + 'a';
468 }
469 fifo.update(10);
470 try testing.expectEqualSlices(u8, "abcdefghij", fifo.readableSlice(0));
471 }
472
473 {
474 try fifo.unget("prependedstring");
475 var result: [30]u8 = undefined;
476 try testing.expectEqualSlices(u8, "prependedstringabcdefghij", result[0..fifo.read(&result)]);
477 try fifo.unget("b");
478 try fifo.unget("a");
479 try testing.expectEqualSlices(u8, "ab", result[0..fifo.read(&result)]);
480 }
481
482 fifo.shrink(0);
483
484 {
485 try fifo.writer().print("{s}, {s}!", .{ "Hello", "World" });
486 var result: [30]u8 = undefined;
487 try testing.expectEqualSlices(u8, "Hello, World!", result[0..fifo.read(&result)]);
488 try testing.expectEqual(@as(usize, 0), fifo.readableLength());
489 }
490
491 {
492 try fifo.writer().writeAll("This is a test");
493 var result: [30]u8 = undefined;
494 try testing.expectEqualSlices(u8, "This", (try fifo.reader().readUntilDelimiterOrEof(&result, ' ')).?);
495 try testing.expectEqualSlices(u8, "is", (try fifo.reader().readUntilDelimiterOrEof(&result, ' ')).?);
496 try testing.expectEqualSlices(u8, "a", (try fifo.reader().readUntilDelimiterOrEof(&result, ' ')).?);
497 try testing.expectEqualSlices(u8, "test", (try fifo.reader().readUntilDelimiterOrEof(&result, ' ')).?);
498 }
499
500 {
501 try fifo.ensureTotalCapacity(1);
502 var in_fbs = std.io.fixedBufferStream("pump test");
503 var out_buf: [50]u8 = undefined;
504 var out_fbs = std.io.fixedBufferStream(&out_buf);
505 try fifo.pump(in_fbs.reader(), out_fbs.writer());
506 try testing.expectEqualSlices(u8, in_fbs.buffer, out_fbs.getWritten());
507 }
508}
509
510test LinearFifo {
511 inline for ([_]type{ u1, u8, u16, u64 }) |T| {
512 inline for ([_]LinearFifoBufferType{ LinearFifoBufferType{ .Static = 32 }, .Slice, .Dynamic }) |bt| {
513 const FifoType = LinearFifo(T, bt);
514 var buf: if (bt == .Slice) [32]T else void = undefined;
515 var fifo = switch (bt) {
516 .Static => FifoType.init(),
517 .Slice => FifoType.init(buf[0..]),
518 .Dynamic => FifoType.init(testing.allocator),
519 };
520 defer fifo.deinit();
521
522 try fifo.write(&[_]T{ 0, 1, 1, 0, 1 });
523 try testing.expectEqual(@as(usize, 5), fifo.readableLength());
524
525 {
526 try testing.expectEqual(@as(T, 0), fifo.readItem().?);
527 try testing.expectEqual(@as(T, 1), fifo.readItem().?);
528 try testing.expectEqual(@as(T, 1), fifo.readItem().?);
529 try testing.expectEqual(@as(T, 0), fifo.readItem().?);
530 try testing.expectEqual(@as(T, 1), fifo.readItem().?);
531 try testing.expectEqual(@as(usize, 0), fifo.readableLength());
532 }
533
534 {
535 try fifo.writeItem(1);
536 try fifo.writeItem(1);
537 try fifo.writeItem(1);
538 try testing.expectEqual(@as(usize, 3), fifo.readableLength());
539 }
540
541 {
542 var readBuf: [3]T = undefined;
543 const n = fifo.read(&readBuf);
544 try testing.expectEqual(@as(usize, 3), n); // NOTE: It should be the number of items.
545 }
546 }
547 }
548}
lib/std/std.zig-1
......@@ -57,7 +57,6 @@ pub const debug = @import("debug.zig");
5757pub const dwarf = @import("dwarf.zig");
5858pub const elf = @import("elf.zig");
5959pub const enums = @import("enums.zig");
60pub const fifo = @import("fifo.zig");
6160pub const fmt = @import("fmt.zig");
6261pub const fs = @import("fs.zig");
6362pub const gpu = @import("gpu.zig");