authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-03-10 16:31:04-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-03-10 16:31:04-04:00
logb6fbd524f122449e6e2bb4d73ce3f59b01286f50
tree4d3154062969c26a12a9548d3b852f0e3041330a
parentba0e3be5cfa2f60f2f9d2a4eb319408f972796c2
signaturelock-open Commit is signed but in an unrecognized format.

(breaking) improve and simplify fixed buffer streams API

9 files changed, 223 insertions(+), 236 deletions(-)

lib/std/io.zig+6-150
......@@ -95,18 +95,18 @@ pub fn getStdIn() File {
9595pub const SeekableStream = @import("io/seekable_stream.zig").SeekableStream;
9696pub const InStream = @import("io/in_stream.zig").InStream;
9797pub const OutStream = @import("io/out_stream.zig").OutStream;
98pub const FixedBufferInStream = @import("io/fixed_buffer_stream.zig").FixedBufferInStream;
9998pub const BufferedAtomicFile = @import("io/buffered_atomic_file.zig").BufferedAtomicFile;
10099
101100pub const BufferedOutStream = @import("io/buffered_out_stream.zig").BufferedOutStream;
102pub const BufferedOutStreamCustom = @import("io/buffered_out_stream.zig").BufferedOutStreamCustom;
103101pub const bufferedOutStream = @import("io/buffered_out_stream.zig").bufferedOutStream;
104102
105pub const CountingOutStream = @import("io/counting_out_stream.zig").CountingOutStream;
103pub const BufferedInStream = @import("io/buffered_in_stream.zig").BufferedInStream;
104pub const bufferedInStream = @import("io/buffered_in_stream.zig").bufferedInStream;
106105
107pub fn fixedBufferStream(bytes: []const u8) FixedBufferInStream {
108 return (FixedBufferInStream{ .bytes = bytes, .pos = 0 });
109}
106pub const FixedBufferStream = @import("io/fixed_buffer_stream.zig").FixedBufferStream;
107pub const fixedBufferStream = @import("io/fixed_buffer_stream.zig").fixedBufferStream;
108
109pub const CountingOutStream = @import("io/counting_out_stream.zig").CountingOutStream;
110110
111111pub fn cOutStream(c_file: *std.c.FILE) COutStream {
112112 return .{ .context = c_file };
......@@ -144,92 +144,6 @@ pub fn readFileAlloc(allocator: *mem.Allocator, path: []const u8) ![]u8 {
144144 return fs.cwd().readFileAlloc(allocator, path, math.maxInt(usize));
145145}
146146
147pub fn BufferedInStream(comptime Error: type) type {
148 return BufferedInStreamCustom(mem.page_size, Error);
149}
150
151pub fn BufferedInStreamCustom(comptime buffer_size: usize, comptime Error: type) type {
152 return struct {
153 const Self = @This();
154 const Stream = InStream(Error);
155
156 stream: Stream,
157
158 unbuffered_in_stream: *Stream,
159
160 const FifoType = std.fifo.LinearFifo(u8, std.fifo.LinearFifoBufferType{ .Static = buffer_size });
161 fifo: FifoType,
162
163 pub fn init(unbuffered_in_stream: *Stream) Self {
164 return Self{
165 .unbuffered_in_stream = unbuffered_in_stream,
166 .fifo = FifoType.init(),
167 .stream = Stream{ .readFn = readFn },
168 };
169 }
170
171 fn readFn(in_stream: *Stream, dest: []u8) !usize {
172 const self = @fieldParentPtr(Self, "stream", in_stream);
173 var dest_index: usize = 0;
174 while (dest_index < dest.len) {
175 const written = self.fifo.read(dest[dest_index..]);
176 if (written == 0) {
177 // fifo empty, fill it
178 const writable = self.fifo.writableSlice(0);
179 assert(writable.len > 0);
180 const n = try self.unbuffered_in_stream.read(writable);
181 if (n == 0) {
182 // reading from the unbuffered stream returned nothing
183 // so we have nothing left to read.
184 return dest_index;
185 }
186 self.fifo.update(n);
187 }
188 dest_index += written;
189 }
190 return dest.len;
191 }
192 };
193}
194
195test "io.BufferedInStream" {
196 const OneByteReadInStream = struct {
197 const Error = error{NoError};
198 const Stream = InStream(Error);
199
200 stream: Stream,
201 str: []const u8,
202 curr: usize,
203
204 fn init(str: []const u8) @This() {
205 return @This(){
206 .stream = Stream{ .readFn = readFn },
207 .str = str,
208 .curr = 0,
209 };
210 }
211
212 fn readFn(in_stream: *Stream, dest: []u8) Error!usize {
213 const self = @fieldParentPtr(@This(), "stream", in_stream);
214 if (self.str.len <= self.curr or dest.len == 0)
215 return 0;
216
217 dest[0] = self.str[self.curr];
218 self.curr += 1;
219 return 1;
220 }
221 };
222
223 const str = "This is a test";
224 var one_byte_stream = OneByteReadInStream.init(str);
225 var buf_in_stream = BufferedInStream(OneByteReadInStream.Error).init(&one_byte_stream.stream);
226 const stream = &buf_in_stream.stream;
227
228 const res = try stream.readAllAlloc(testing.allocator, str.len + 1);
229 defer testing.allocator.free(res);
230 testing.expectEqualSlices(u8, str, res);
231}
232
233147/// Creates a stream which supports 'un-reading' data, so that it can be read again.
234148/// This makes look-ahead style parsing much easier.
235149pub fn PeekStream(comptime buffer_type: std.fifo.LinearFifoBufferType, comptime InStreamError: type) type {
......@@ -473,64 +387,6 @@ pub fn BitInStream(endian: builtin.Endian, comptime Error: type) type {
473387 };
474388}
475389
476/// This is a simple OutStream that writes to a fixed buffer. If the returned number
477/// of bytes written is less than requested, the buffer is full.
478/// Returns error.OutOfMemory when no bytes would be written.
479pub const SliceOutStream = struct {
480 pub const Error = error{OutOfMemory};
481 pub const Stream = OutStream(Error);
482
483 stream: Stream,
484
485 pos: usize,
486 slice: []u8,
487
488 pub fn init(slice: []u8) SliceOutStream {
489 return SliceOutStream{
490 .slice = slice,
491 .pos = 0,
492 .stream = Stream{ .writeFn = writeFn },
493 };
494 }
495
496 pub fn getWritten(self: *const SliceOutStream) []const u8 {
497 return self.slice[0..self.pos];
498 }
499
500 pub fn reset(self: *SliceOutStream) void {
501 self.pos = 0;
502 }
503
504 fn writeFn(out_stream: *Stream, bytes: []const u8) Error!usize {
505 const self = @fieldParentPtr(SliceOutStream, "stream", out_stream);
506
507 if (bytes.len == 0) return 0;
508
509 assert(self.pos <= self.slice.len);
510
511 const n = if (self.pos + bytes.len <= self.slice.len)
512 bytes.len
513 else
514 self.slice.len - self.pos;
515
516 std.mem.copy(u8, self.slice[self.pos .. self.pos + n], bytes[0..n]);
517 self.pos += n;
518
519 if (n == 0) return error.OutOfMemory;
520
521 return n;
522 }
523};
524
525test "io.SliceOutStream" {
526 var buf: [255]u8 = undefined;
527 var slice_stream = SliceOutStream.init(buf[0..]);
528 const stream = &slice_stream.stream;
529
530 try stream.print("{}{}!", .{ "Hello", "World" });
531 testing.expectEqualSlices(u8, "HelloWorld!", slice_stream.getWritten());
532}
533
534390/// An OutStream that doesn't write to anything.
535391pub const null_out_stream = @as(NullOutStream, .{ .context = {} });
536392
lib/std/io/buffered_atomic_file.zig+2-2
......@@ -10,7 +10,7 @@ pub const BufferedAtomicFile = struct {
1010 allocator: *mem.Allocator,
1111
1212 pub const buffer_size = 4096;
13 pub const BufferedOutStream = std.io.BufferedOutStreamCustom(buffer_size, File.OutStream);
13 pub const BufferedOutStream = std.io.BufferedOutStream(buffer_size, File.OutStream);
1414 pub const OutStream = std.io.OutStream(*BufferedOutStream, BufferedOutStream.Error, BufferedOutStream.write);
1515
1616 /// TODO when https://github.com/ziglang/zig/issues/2761 is solved
......@@ -29,7 +29,7 @@ pub const BufferedAtomicFile = struct {
2929 errdefer self.atomic_file.deinit();
3030
3131 self.file_stream = self.atomic_file.file.outStream();
32 self.buffered_stream = std.io.bufferedOutStream(buffer_size, self.file_stream);
32 self.buffered_stream = .{ .unbuffered_out_stream = self.file_stream };
3333 return self;
3434 }
3535
lib/std/io/buffered_in_stream.zig created+84
......@@ -0,0 +1,84 @@
1const std = @import("../std.zig");
2const io = std.io;
3
4pub fn BufferedInStream(comptime buffer_size: usize, comptime InStreamType) type {
5 return struct {
6 unbuffered_in_stream: InStreamType,
7 fifo: FifoType = FifoType.init(),
8
9 pub const Error = InStreamType.Error;
10 pub const InStream = io.InStream(*Self, Error, read);
11
12 const Self = @This();
13 const FifoType = std.fifo.LinearFifo(u8, std.fifo.LinearFifoBufferType{ .Static = buffer_size });
14
15 pub fn read(self: *Self, dest: []u8) Error!usize {
16 var dest_index: usize = 0;
17 while (dest_index < dest.len) {
18 const written = self.fifo.read(dest[dest_index..]);
19 if (written == 0) {
20 // fifo empty, fill it
21 const writable = self.fifo.writableSlice(0);
22 assert(writable.len > 0);
23 const n = try self.unbuffered_in_stream.read(writable);
24 if (n == 0) {
25 // reading from the unbuffered stream returned nothing
26 // so we have nothing left to read.
27 return dest_index;
28 }
29 self.fifo.update(n);
30 }
31 dest_index += written;
32 }
33 return dest.len;
34 }
35
36 pub fn inStream(self: *Self) InStream {
37 return .{ .context = self };
38 }
39 };
40}
41
42pub fn bufferedInStream(underlying_stream: var) BufferedInStream(4096, @TypeOf(underlying_stream)) {
43 return .{ .unbuffered_in_stream = underlying_stream };
44}
45
46test "io.BufferedInStream" {
47 const OneByteReadInStream = struct {
48 str: []const u8,
49 curr: usize,
50
51 const Error = error{NoError};
52 const Self = @This();
53 const InStream = io.InStream(*Self, Error, read);
54
55 fn init(str: []const u8) Self {
56 return Self{
57 .str = str,
58 .curr = 0,
59 };
60 }
61
62 fn read(self: *Self, dest: []u8) Error!usize {
63 if (self.str.len <= self.curr or dest.len == 0)
64 return 0;
65
66 dest[0] = self.str[self.curr];
67 self.curr += 1;
68 return 1;
69 }
70
71 fn inStream(self: *Self) InStream {
72 return .{ .context = self };
73 }
74 };
75
76 const str = "This is a test";
77 var one_byte_stream = OneByteReadInStream.init(str);
78 var buf_in_stream = bufferedInStream(one_byte_stream.inStream());
79 const stream = buf_in_stream.inStream();
80
81 const res = try stream.readAllAlloc(testing.allocator, str.len + 1);
82 defer testing.allocator.free(res);
83 testing.expectEqualSlices(u8, str, res);
84}
lib/std/io/buffered_out_stream.zig+4-19
......@@ -1,14 +1,10 @@
11const std = @import("../std.zig");
22const io = std.io;
33
4pub fn BufferedOutStream(comptime OutStreamType: type) type {
5 return BufferedOutStreamCustom(4096, OutStreamType);
6}
7
8pub fn BufferedOutStreamCustom(comptime buffer_size: usize, comptime OutStreamType: type) type {
4pub fn BufferedOutStream(comptime buffer_size: usize, comptime OutStreamType: type) type {
95 return struct {
106 unbuffered_out_stream: OutStreamType,
11 fifo: FifoType,
7 fifo: FifoType = FifoType.init(),
128
139 pub const Error = OutStreamType.Error;
1410 pub const OutStream = io.OutStream(*Self, Error, write);
......@@ -16,13 +12,6 @@ pub fn BufferedOutStreamCustom(comptime buffer_size: usize, comptime OutStreamTy
1612 const Self = @This();
1713 const FifoType = std.fifo.LinearFifo(u8, std.fifo.LinearFifoBufferType{ .Static = buffer_size });
1814
19 pub fn init(unbuffered_out_stream: OutStreamType) Self {
20 return Self{
21 .unbuffered_out_stream = unbuffered_out_stream,
22 .fifo = FifoType.init(),
23 };
24 }
25
2615 pub fn flush(self: *Self) !void {
2716 while (true) {
2817 const slice = self.fifo.readableSlice(0);
......@@ -47,10 +36,6 @@ pub fn BufferedOutStreamCustom(comptime buffer_size: usize, comptime OutStreamTy
4736 };
4837}
4938
50pub fn bufferedOutStream(
51 comptime buffer_size: usize,
52 underlying_stream: var,
53) BufferedOutStreamCustom(buffer_size, @TypeOf(underlying_stream)) {
54 return BufferedOutStreamCustom(buffer_size, @TypeOf(underlying_stream)).init(underlying_stream);
39pub fn bufferedOutStream(underlying_stream: var) BufferedOutStream(4096, @TypeOf(underlying_stream)) {
40 return .{ .unbuffered_out_stream = underlying_stream };
5541}
56
lib/std/io/fixed_buffer_stream.zig+123-60
......@@ -1,66 +1,129 @@
11const std = @import("../std.zig");
22const io = std.io;
3const testing = std.testing;
34
4pub const FixedBufferInStream = struct {
5 bytes: []const u8,
6 pos: usize,
7
8 pub const SeekError = error{EndOfStream};
9 pub const GetSeekPosError = error{};
10
11 pub const InStream = io.InStream(*FixedBufferInStream, error{}, read);
12
13 pub fn inStream(self: *FixedBufferInStream) InStream {
14 return .{ .context = self };
15 }
16
17 pub const SeekableStream = io.SeekableStream(
18 *FixedBufferInStream,
19 SeekError,
20 GetSeekPosError,
21 seekTo,
22 seekBy,
23 getPos,
24 getEndPos,
25 );
26
27 pub fn seekableStream(self: *FixedBufferInStream) SeekableStream {
28 return .{ .context = self };
29 }
30
31 pub fn read(self: *FixedBufferInStream, dest: []u8) error{}!usize {
32 const size = std.math.min(dest.len, self.bytes.len - self.pos);
33 const end = self.pos + size;
34
35 std.mem.copy(u8, dest[0..size], self.bytes[self.pos..end]);
36 self.pos = end;
37
38 return size;
39 }
40
41 pub fn seekTo(self: *FixedBufferInStream, pos: u64) SeekError!void {
42 const usize_pos = std.math.cast(usize, pos) catch return error.EndOfStream;
43 if (usize_pos > self.bytes.len) return error.EndOfStream;
44 self.pos = usize_pos;
45 }
46
47 pub fn seekBy(self: *FixedBufferInStream, amt: i64) SeekError!void {
48 if (amt < 0) {
49 const abs_amt = std.math.cast(usize, -amt) catch return error.EndOfStream;
50 if (abs_amt > self.pos) return error.EndOfStream;
51 self.pos -= abs_amt;
52 } else {
53 const usize_amt = std.math.cast(usize, amt) catch return error.EndOfStream;
54 if (self.pos + usize_amt > self.bytes.len) return error.EndOfStream;
55 self.pos += usize_amt;
5/// This turns a slice into an `io.OutStream`, `io.InStream`, or `io.SeekableStream`.
6/// If the supplied slice is const, then `io.OutStream` is not available.
7pub fn FixedBufferStream(comptime Buffer: type) type {
8 return struct {
9 /// `Buffer` is either a `[]u8` or `[]const u8`.
10 buffer: Buffer,
11 pos: usize,
12
13 pub const ReadError = error{EndOfStream};
14 pub const WriteError = error{OutOfMemory};
15 pub const SeekError = error{EndOfStream};
16 pub const GetSeekPosError = error{};
17
18 pub const InStream = io.InStream(*Self, ReadError, read);
19 pub const OutStream = io.OutStream(*Self, WriteError, write);
20
21 pub const SeekableStream = io.SeekableStream(
22 *Self,
23 SeekError,
24 GetSeekPosError,
25 seekTo,
26 seekBy,
27 getPos,
28 getEndPos,
29 );
30
31 const Self = @This();
32
33 pub fn inStream(self: *Self) InStream {
34 return .{ .context = self };
35 }
36
37 pub fn outStream(self: *Self) OutStream {
38 return .{ .context = self };
39 }
40
41 pub fn seekableStream(self: *Self) SeekableStream {
42 return .{ .context = self };
43 }
44
45 pub fn read(self: *Self, dest: []u8) ReadError!usize {
46 const size = std.math.min(dest.len, self.buffer.len - self.pos);
47 const end = self.pos + size;
48
49 std.mem.copy(u8, dest[0..size], self.buffer[self.pos..end]);
50 self.pos = end;
51
52 if (size == 0) return error.EndOfStream;
53 return size;
54 }
55
56 /// If the returned number of bytes written is less than requested, the
57 /// buffer is full. Returns `error.OutOfMemory` when no bytes would be written.
58 pub fn write(self: *Self, bytes: []const u8) WriteError!usize {
59 if (bytes.len == 0) return 0;
60
61 assert(self.pos <= self.buffer.len);
62
63 const n = if (self.pos + bytes.len <= self.buffer.len)
64 bytes.len
65 else
66 self.buffer.len - self.pos;
67
68 std.mem.copy(u8, self.buffer[self.pos .. self.pos + n], bytes[0..n]);
69 self.pos += n;
70
71 if (n == 0) return error.OutOfMemory;
72
73 return n;
74 }
75
76 pub fn seekTo(self: *Self, pos: u64) SeekError!void {
77 const usize_pos = std.math.cast(usize, pos) catch return error.EndOfStream;
78 if (usize_pos > self.buffer.len) return error.EndOfStream;
79 self.pos = usize_pos;
80 }
81
82 pub fn seekBy(self: *Self, amt: i64) SeekError!void {
83 if (amt < 0) {
84 const abs_amt = std.math.cast(usize, -amt) catch return error.EndOfStream;
85 if (abs_amt > self.pos) return error.EndOfStream;
86 self.pos -= abs_amt;
87 } else {
88 const usize_amt = std.math.cast(usize, amt) catch return error.EndOfStream;
89 if (self.pos + usize_amt > self.buffer.len) return error.EndOfStream;
90 self.pos += usize_amt;
91 }
92 }
93
94 pub fn getEndPos(self: *Self) GetSeekPosError!u64 {
95 return self.buffer.len;
5696 }
57 }
5897
59 pub fn getEndPos(self: *FixedBufferInStream) GetSeekPosError!u64 {
60 return self.bytes.len;
61 }
98 pub fn getPos(self: *Self) GetSeekPosError!u64 {
99 return self.pos;
100 }
101
102 pub fn getWritten(self: Self) []const u8 {
103 return self.slice[0..self.pos];
104 }
105
106 pub fn reset(self: *Self) void {
107 self.pos = 0;
108 }
109 };
110}
111
112pub fn fixedBufferStream(buffer: var) FixedBufferStream(NonSentinelSpan(@TypeOf(buffer))) {
113 return .{ .buffer = std.mem.span(buffer), .pos = 0 };
114}
115
116fn NonSentinelSpan(comptime T: type) type {
117 var ptr_info = @typeInfo(std.mem.Span(T)).Pointer;
118 ptr_info.sentinel = null;
119 return @Type(std.builtin.TypeInfo{ .Pointer = ptr_info });
120}
121
122test "FixedBufferStream" {
123 var buf: [255]u8 = undefined;
124 var fbs = fixedBufferStream(&buf);
125 const stream = fbs.outStream();
62126
63 pub fn getPos(self: *FixedBufferInStream) GetSeekPosError!u64 {
64 return self.pos;
65 }
66};
127 try stream.print("{}{}!", .{ "Hello", "World" });
128 testing.expectEqualSlices(u8, "HelloWorld!", fbs.getWritten());
129}
lib/std/json.zig+2-2
......@@ -2107,8 +2107,8 @@ test "import more json tests" {
21072107test "write json then parse it" {
21082108 var out_buffer: [1000]u8 = undefined;
21092109
2110 var slice_out_stream = std.io.SliceOutStream.init(&out_buffer);
2111 const out_stream = &slice_out_stream.stream;
2110 var fixed_buffer_stream = std.io.fixedBufferStream(&out_buffer);
2111 const out_stream = fixed_buffer_stream.outStream();
21122112 var jw = WriteStream(@TypeOf(out_stream).Child, 4).init(out_stream);
21132113
21142114 try jw.beginObject();
lib/std/progress.zig+1-1
......@@ -177,7 +177,7 @@ pub const Progress = struct {
177177 pub fn log(self: *Progress, comptime format: []const u8, args: var) void {
178178 const file = self.terminal orelse return;
179179 self.refresh();
180 file.outStream().stream.print(format, args) catch {
180 file.outStream().print(format, args) catch {
181181 self.terminal = null;
182182 return;
183183 };
lib/std/std.zig-1
......@@ -5,7 +5,6 @@ pub const BloomFilter = @import("bloom_filter.zig").BloomFilter;
55pub const BufMap = @import("buf_map.zig").BufMap;
66pub const BufSet = @import("buf_set.zig").BufSet;
77pub const Buffer = @import("buffer.zig").Buffer;
8pub const BufferOutStream = @import("io.zig").BufferOutStream;
98pub const ChildProcess = @import("child_process.zig").ChildProcess;
109pub const DynLib = @import("dynamic_library.zig").DynLib;
1110pub const HashMap = @import("hash_map.zig").HashMap;
src-self-hosted/print_targets.zig+1-1
......@@ -93,7 +93,7 @@ pub fn cmdTargets(
9393 };
9494 defer allocator.free(available_glibcs);
9595
96 var bos = io.bufferedOutStream(4096, stdout);
96 var bos = io.bufferedOutStream(stdout);
9797 const bos_stream = bos.outStream();
9898 var jws = std.json.WriteStream(@TypeOf(bos_stream), 6).init(bos_stream);
9999