authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-15 18:45:21-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-01 16:35:25-07:00
logb26aceba7d5a4e0facb6589b3a6d659ae301d70b
tree1f3f3b776a813d252cdd451e31e021e9d0f02648
parent5356f3a30748ba3504767b68e99d215d6aac839b

thinking about splat being the only function

6 files changed, 196 insertions(+), 113 deletions(-)

lib/std/fs/File.zig+23
...@@ -1592,6 +1592,7 @@ pub fn writer(file: File) std.io.Writer {...@@ -1592,6 +1592,7 @@ pub fn writer(file: File) std.io.Writer {
1592 .context = interface.handleToOpaque(file.handle),1592 .context = interface.handleToOpaque(file.handle),
1593 .vtable = &.{1593 .vtable = &.{
1594 .writev = interface.writev,1594 .writev = interface.writev,
1595 .splat = interface.splat,
1595 .writeFile = interface.writeFile,1596 .writeFile = interface.writeFile,
1596 },1597 },
1597 };1598 };
...@@ -1628,6 +1629,28 @@ const interface = struct {...@@ -1628,6 +1629,28 @@ const interface = struct {
1628 return std.posix.writev(file, iovecs);1629 return std.posix.writev(file, iovecs);
1629 }1630 }
16301631
1632 fn splat(context: *anyopaque, headers: []const []const u8, pattern: []const u8, n: usize) anyerror!usize {
1633 const file = opaqueToHandle(context);
1634
1635 if (is_windows) {
1636 // TODO improve this to use WriteFileScatter
1637 if (headers.len > 0) {
1638 const first = headers[0];
1639 return windows.WriteFile(file, first, null);
1640 }
1641 if (n > 0) return windows.WriteFile(file, pattern, null);
1642 return 0;
1643 }
1644
1645 var iovecs_buffer: [max_buffers_len]std.posix.iovec_const = undefined;
1646 const iovecs = iovecs_buffer[0..@min(iovecs_buffer.len, headers.len)];
1647 for (iovecs, headers[0..iovecs.len]) |*v, d| v.* = .{
1648 .base = if (d.len == 0) "" else d.ptr, // OS sadly checks ptr addr before length.
1649 .len = d.len,
1650 };
1651 return std.posix.writev(file, iovecs);
1652 }
1653
1631 fn writeFile(1654 fn writeFile(
1632 context: *anyopaque,1655 context: *anyopaque,
1633 in_file: std.fs.File,1656 in_file: std.fs.File,
lib/std/io.zig+11-3
...@@ -345,15 +345,23 @@ pub const null_writer: Writer = .{...@@ -345,15 +345,23 @@ pub const null_writer: Writer = .{
345 .context = undefined,345 .context = undefined,
346 .vtable = &.{346 .vtable = &.{
347 .writev = null_writev,347 .writev = null_writev,
348 .splat = null_splat,
348 .writeFile = null_writeFile,349 .writeFile = null_writeFile,
349 },350 },
350};351};
351352
352fn null_writev(context: *anyopaque, data: []const []const u8) anyerror!usize {353fn null_writev(context: *anyopaque, data: []const []const u8) anyerror!usize {
353 _ = context;354 _ = context;
354 var n: usize = 0;355 var written: usize = 0;
355 for (data) |bytes| n += bytes.len;356 for (data) |bytes| written += bytes.len;
356 return n;357 return written;
358}
359
360fn null_splat(context: *anyopaque, headers: []const []const u8, pattern: []const u8, n: usize) anyerror!usize {
361 _ = context;
362 var written: usize = pattern.len * n;
363 for (headers) |bytes| written += bytes.len;
364 return written;
357}365}
358366
359fn null_writeFile(367fn null_writeFile(
lib/std/io/AllocatingWriter.zig+13-4
...@@ -99,20 +99,29 @@ pub fn clearRetainingCapacity(aw: *AllocatingWriter) void {...@@ -99,20 +99,29 @@ pub fn clearRetainingCapacity(aw: *AllocatingWriter) void {
99}99}
100100
101fn writev(context: *anyopaque, data: []const []const u8) anyerror!usize {101fn writev(context: *anyopaque, data: []const []const u8) anyerror!usize {
102 return splat(context, data, &.{}, 0);
103}
104
105fn splat(context: *anyopaque, headers: []const []const u8, pattern: []const u8, n: usize) anyerror!usize {
102 const aw: *AllocatingWriter = @alignCast(@ptrCast(context));106 const aw: *AllocatingWriter = @alignCast(@ptrCast(context));
103 const start_len = aw.written.len;107 const start_len = aw.written.len;
104 const bw = &aw.buffered_writer;108 const bw = &aw.buffered_writer;
105 assert(data[0].ptr == aw.written.ptr + start_len);109 assert(headers[0].ptr == aw.written.ptr + start_len);
106 var list: std.ArrayListUnmanaged(u8) = .{110 var list: std.ArrayListUnmanaged(u8) = .{
107 .items = aw.written.ptr[0 .. start_len + data[0].len],111 .items = aw.written.ptr[0 .. start_len + headers[0].len],
108 .capacity = start_len + bw.buffer.len,112 .capacity = start_len + bw.buffer.len,
109 };113 };
110 defer setArrayList(aw, list);114 defer setArrayList(aw, list);
111 const rest = data[1..];115 const rest = headers[1..];
112 var new_capacity: usize = list.capacity;116 var new_capacity: usize = list.capacity + pattern.len * n;
113 for (rest) |bytes| new_capacity += bytes.len;117 for (rest) |bytes| new_capacity += bytes.len;
114 try list.ensureTotalCapacity(aw.allocator, new_capacity + 1);118 try list.ensureTotalCapacity(aw.allocator, new_capacity + 1);
115 for (rest) |bytes| list.appendSliceAssumeCapacity(bytes);119 for (rest) |bytes| list.appendSliceAssumeCapacity(bytes);
120 if (pattern.len == 1) {
121 list.appendNTimesAssumeCapacity(pattern[0], n);
122 } else {
123 for (0..n) |_| list.appendSliceAssumeCapacity(pattern);
124 }
116 aw.written = list.items;125 aw.written = list.items;
117 bw.buffer = list.unusedCapacitySlice();126 bw.buffer = list.unusedCapacitySlice();
118 return list.items.len - start_len;127 return list.items.len - start_len;
lib/std/io/BufferedWriter.zig+122-100
...@@ -6,6 +6,13 @@ const Writer = std.io.Writer;...@@ -6,6 +6,13 @@ const Writer = std.io.Writer;
6const testing = std.testing;6const testing = std.testing;
77
8/// Underlying stream to send bytes to.8/// Underlying stream to send bytes to.
9///
10/// A write will only be sent here if it could not fit into `buffer`, or if it
11/// is a `writeFile`.
12///
13/// `unbuffered_writer` may modify `buffer` if the number of bytes returned
14/// equals number of bytes provided. This property is exploited by
15/// `std.io.AllocatingWriter` for example.
9unbuffered_writer: Writer,16unbuffered_writer: Writer,
10/// User-provided storage that must outlive this `BufferedWriter`.17/// User-provided storage that must outlive this `BufferedWriter`.
11///18///
...@@ -27,6 +34,7 @@ pub fn writer(bw: *BufferedWriter) Writer {...@@ -27,6 +34,7 @@ pub fn writer(bw: *BufferedWriter) Writer {
27 .context = bw,34 .context = bw,
28 .vtable = &.{35 .vtable = &.{
29 .writev = passthru_writev,36 .writev = passthru_writev,
37 .splat = passthru_splat,
30 .writeFile = passthru_writeFile,38 .writeFile = passthru_writeFile,
31 },39 },
32 };40 };
...@@ -34,6 +42,7 @@ pub fn writer(bw: *BufferedWriter) Writer {...@@ -34,6 +42,7 @@ pub fn writer(bw: *BufferedWriter) Writer {
3442
35const fixed_vtable: Writer.VTable = .{43const fixed_vtable: Writer.VTable = .{
36 .writev = fixed_writev,44 .writev = fixed_writev,
45 .splat = fixed_splat,
37 .writeFile = fixed_writeFile,46 .writeFile = fixed_writeFile,
38};47};
3948
...@@ -62,7 +71,8 @@ pub fn reset(bw: *BufferedWriter) void {...@@ -62,7 +71,8 @@ pub fn reset(bw: *BufferedWriter) void {
62}71}
6372
64pub fn flush(bw: *BufferedWriter) anyerror!void {73pub fn flush(bw: *BufferedWriter) anyerror!void {
65 try bw.unbuffered_writer.writeAll(bw.buffer[0..bw.end]);74 const send_buffer = bw.buffer[0..bw.end];
75 try bw.unbuffered_writer.writeAll(send_buffer);
66 bw.end = 0;76 bw.end = 0;
67}77}
6878
...@@ -120,6 +130,102 @@ fn passthru_writev(context: *anyopaque, data: []const []const u8) anyerror!usize...@@ -120,6 +130,102 @@ fn passthru_writev(context: *anyopaque, data: []const []const u8) anyerror!usize
120 return end - start_end;130 return end - start_end;
121}131}
122132
133fn passthru_splat(context: *anyopaque, headers: []const []const u8, pattern: []const u8, n: usize) anyerror!usize {
134 const bw: *BufferedWriter = @alignCast(@ptrCast(context));
135 const buffer = bw.buffer;
136 const start_end = bw.end;
137
138 var end = bw.end;
139 for (headers, 0..) |bytes, i| {
140 const new_end = end + bytes.len;
141 if (new_end <= buffer.len) {
142 @branchHint(.likely);
143 @memcpy(buffer[end..new_end], bytes);
144 end = new_end;
145 continue;
146 }
147 if (end == 0) return bw.unbuffered_writer.splat(headers, pattern, n);
148 var buffers: [max_buffers_len][]const u8 = undefined;
149 buffers[0] = buffer[0..end];
150 const remaining_headers = headers[i..];
151 const remaining_buffers = buffers[1..];
152 const len: usize = @min(remaining_headers.len, remaining_buffers.len);
153 @memcpy(remaining_buffers[0..len], remaining_headers[0..len]);
154 const send_buffers = buffers[0 .. len + 1];
155 if (len >= remaining_headers.len) {
156 @branchHint(.likely);
157 // Made it past the headers, so we can call `splat`.
158 const written = try bw.unbuffered_writer.splat(send_buffers, pattern, n);
159 if (written < end) {
160 @branchHint(.unlikely);
161 const remainder = buffer[written..end];
162 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
163 bw.end = remainder.len;
164 return end - start_end;
165 }
166 bw.end = 0;
167 return written - start_end;
168 }
169 const written = try bw.unbuffered_writer.writev(send_buffers);
170 if (written < end) {
171 @branchHint(.unlikely);
172 const remainder = buffer[written..end];
173 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
174 bw.end = remainder.len;
175 return end - start_end;
176 }
177 bw.end = 0;
178 return written - start_end;
179 }
180
181 switch (pattern.len) {
182 0 => {
183 bw.end = end;
184 return end - start_end;
185 },
186 1 => {
187 const new_end = end + n;
188 if (new_end <= buffer.len) {
189 @branchHint(.likely);
190 @memset(buffer[end..new_end], pattern[0]);
191 bw.end = new_end;
192 return end - start_end;
193 }
194 const written = try bw.unbuffered_writer.splat(buffer[0..end], pattern, n);
195 if (written < end) {
196 @branchHint(.unlikely);
197 const remainder = buffer[written..end];
198 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
199 bw.end = remainder.len;
200 return end - start_end;
201 }
202 bw.end = 0;
203 return written - start_end;
204 },
205 else => {
206 const new_end = end + pattern.len * n;
207 if (new_end <= buffer.len) {
208 @branchHint(.likely);
209 while (end < new_end) : (end += pattern.len) {
210 @memcpy(buffer[end..][0..pattern.len], pattern);
211 }
212 bw.end = end;
213 return end - start_end;
214 }
215 const written = try bw.unbuffered_writer.splat(buffer[0..end], pattern, n);
216 if (written < end) {
217 @branchHint(.unlikely);
218 const remainder = buffer[written..end];
219 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
220 bw.end = remainder.len;
221 return end - start_end;
222 }
223 bw.end = 0;
224 return written - start_end;
225 },
226 }
227}
228
123fn fixed_writev(context: *anyopaque, data: []const []const u8) anyerror!usize {229fn fixed_writev(context: *anyopaque, data: []const []const u8) anyerror!usize {
124 const bw: *BufferedWriter = @alignCast(@ptrCast(context));230 const bw: *BufferedWriter = @alignCast(@ptrCast(context));
125 // When this function is called it means the buffer got full, so it's time231 // When this function is called it means the buffer got full, so it's time
...@@ -131,6 +237,19 @@ fn fixed_writev(context: *anyopaque, data: []const []const u8) anyerror!usize {...@@ -131,6 +237,19 @@ fn fixed_writev(context: *anyopaque, data: []const []const u8) anyerror!usize {
131 return error.NoSpaceLeft;237 return error.NoSpaceLeft;
132}238}
133239
240fn fixed_splat(context: *anyopaque, headers: []const []const u8, pattern: []const u8, n: usize) anyerror!usize {
241 const bw: *BufferedWriter = @alignCast(@ptrCast(context));
242 const dest = bw.buffer[bw.end..];
243 if (headers.len > 0) {
244 @memcpy(dest, headers[0][0..dest.len]);
245 } else switch (pattern.len) {
246 0 => unreachable,
247 1 => @memset(dest, pattern[0]),
248 else => for (0..n) |i| @memcpy(dest[i * pattern.len ..][0..pattern.len], pattern),
249 }
250 return error.NoSpaceLeft;
251}
252
134pub fn write(bw: *BufferedWriter, bytes: []const u8) anyerror!usize {253pub fn write(bw: *BufferedWriter, bytes: []const u8) anyerror!usize {
135 const buffer = bw.buffer;254 const buffer = bw.buffer;
136 const end = bw.end;255 const end = bw.end;
...@@ -210,71 +329,7 @@ pub fn splatByteAll(bw: *BufferedWriter, byte: u8, n: usize) anyerror!void {...@@ -210,71 +329,7 @@ pub fn splatByteAll(bw: *BufferedWriter, byte: u8, n: usize) anyerror!void {
210///329///
211/// Does maximum of one underlying `Writer.VTable.writev`.330/// Does maximum of one underlying `Writer.VTable.writev`.
212pub fn splatByte(bw: *BufferedWriter, byte: u8, n: usize) anyerror!usize {331pub fn splatByte(bw: *BufferedWriter, byte: u8, n: usize) anyerror!usize {
213 const buffer = bw.buffer;332 return passthru_splat(bw, &.{}, &.{byte}, n);
214 const end = bw.end;
215
216 const new_end = end + n;
217 if (new_end <= buffer.len) {
218 @branchHint(.likely);
219 @memset(buffer[end..][0..n], byte);
220 bw.end = new_end;
221 return n;
222 }
223
224 if (n <= buffer.len) {
225 const written = try bw.unbuffered_writer.write(buffer[0..end]);
226 if (written < end) {
227 @branchHint(.unlikely);
228 const remainder = buffer[written..end];
229 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
230 bw.end = remainder.len;
231 return 0;
232 }
233 assert(bw.buffer.ptr == buffer.ptr); // TODO this is not a valid assertion
234 @memset(buffer[0..n], byte);
235 bw.end = n;
236 return n;
237 }
238
239 // First try to use only the unused buffer region, to make an attempt for a
240 // single `writev`.
241 const free_space = buffer[end..];
242 var remaining = n - free_space.len;
243 @memset(free_space, byte);
244 var buffers: [max_buffers_len][]const u8 = undefined;
245 buffers[0] = buffer;
246 var buffer_i: usize = 1;
247 while (remaining > free_space.len and buffer_i < buffers.len) {
248 buffers[buffer_i] = free_space;
249 buffer_i += 1;
250 remaining -= free_space.len;
251 }
252 if (remaining > 0 and buffer_i < buffers.len) {
253 buffers[buffer_i] = free_space[0..remaining];
254 buffer_i += 1;
255 const written = try bw.unbuffered_writer.writev(buffers[0..buffer_i]);
256 if (written < end) {
257 @branchHint(.unlikely);
258 const remainder = buffer[written..end];
259 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
260 bw.end = remainder.len;
261 return 0;
262 }
263 bw.end = 0;
264 return written - end;
265 }
266
267 const written = try bw.unbuffered_writer.writev(buffers[0..buffer_i]);
268 if (written < end) {
269 @branchHint(.unlikely);
270 const remainder = buffer[written..end];
271 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
272 bw.end = remainder.len;
273 return 0;
274 }
275
276 bw.end = 0;
277 return written - end;
278}333}
279334
280/// Writes the same slice many times, performing the underlying write call as335/// Writes the same slice many times, performing the underlying write call as
...@@ -288,40 +343,7 @@ pub fn splatBytesAll(bw: *BufferedWriter, bytes: []const u8, n: usize) anyerror!...@@ -288,40 +343,7 @@ pub fn splatBytesAll(bw: *BufferedWriter, bytes: []const u8, n: usize) anyerror!
288///343///
289/// Does maximum of one underlying `Writer.VTable.writev`.344/// Does maximum of one underlying `Writer.VTable.writev`.
290pub fn splatBytes(bw: *BufferedWriter, bytes: []const u8, n: usize) anyerror!usize {345pub fn splatBytes(bw: *BufferedWriter, bytes: []const u8, n: usize) anyerror!usize {
291 const buffer = bw.buffer;346 return passthru_splat(bw, &.{}, bytes, n);
292 const start_end = bw.end;
293 var end = start_end;
294 var remaining = n;
295 while (remaining > 0 and end + bytes.len <= buffer.len) {
296 @memcpy(buffer[end..][0..bytes.len], bytes);
297 end += bytes.len;
298 remaining -= 1;
299 }
300
301 if (remaining == 0) {
302 bw.end = end;
303 return end - start_end;
304 }
305
306 var buffers: [max_buffers_len][]const u8 = undefined;
307 var buffer_i: usize = 1;
308 buffers[0] = buffer[0..end];
309 const remaining_buffers = buffers[1..];
310 const buffers_len: usize = @min(remaining, remaining_buffers.len);
311 @memset(remaining_buffers[0..buffers_len], bytes);
312 remaining -= buffers_len;
313 buffer_i += buffers_len;
314
315 const written = try bw.unbuffered_writer.writev(buffers[0..buffer_i]);
316 if (written < end) {
317 @branchHint(.unlikely);
318 const remainder = buffer[written..end];
319 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
320 bw.end = remainder.len;
321 return end - start_end;
322 }
323 bw.end = 0;
324 return written - start_end;
325}347}
326348
327/// Asserts the `buffer` was initialized with a capacity of at least `@sizeOf(T)` bytes.349/// Asserts the `buffer` was initialized with a capacity of at least `@sizeOf(T)` bytes.
lib/std/io/CountingWriter.zig+14-6
...@@ -14,6 +14,7 @@ pub fn writer(cw: *CountingWriter) Writer {...@@ -14,6 +14,7 @@ pub fn writer(cw: *CountingWriter) Writer {
14 .context = cw,14 .context = cw,
15 .vtable = &.{15 .vtable = &.{
16 .writev = passthru_writev,16 .writev = passthru_writev,
17 .splat = passthru_splat,
17 .writeFile = passthru_writeFile,18 .writeFile = passthru_writeFile,
18 },19 },
19 };20 };
...@@ -28,9 +29,16 @@ pub fn unbufferedWriter(cw: *CountingWriter) std.io.BufferedWriter {...@@ -28,9 +29,16 @@ pub fn unbufferedWriter(cw: *CountingWriter) std.io.BufferedWriter {
2829
29fn passthru_writev(context: *anyopaque, data: []const []const u8) anyerror!usize {30fn passthru_writev(context: *anyopaque, data: []const []const u8) anyerror!usize {
30 const cw: *CountingWriter = @alignCast(@ptrCast(context));31 const cw: *CountingWriter = @alignCast(@ptrCast(context));
31 const n = try cw.child_writer.writev(data);32 const written = try cw.child_writer.writev(data);
32 cw.bytes_written += n;33 cw.bytes_written += written;
33 return n;34 return written;
35}
36
37fn passthru_splat(context: *anyopaque, header: []const u8, pattern: []const u8, n: usize) anyerror!usize {
38 const cw: *CountingWriter = @alignCast(@ptrCast(context));
39 const written = try cw.child_writer.splat(header, pattern, n);
40 cw.bytes_written += written;
41 return written;
34}42}
3543
36fn passthru_writeFile(44fn passthru_writeFile(
...@@ -42,9 +50,9 @@ fn passthru_writeFile(...@@ -42,9 +50,9 @@ fn passthru_writeFile(
42 headers_len: usize,50 headers_len: usize,
43) anyerror!usize {51) anyerror!usize {
44 const cw: *CountingWriter = @alignCast(@ptrCast(context));52 const cw: *CountingWriter = @alignCast(@ptrCast(context));
45 const n = try cw.child_writer.writeFile(file, offset, len, headers_and_trailers, headers_len);53 const written = try cw.child_writer.writeFile(file, offset, len, headers_and_trailers, headers_len);
46 cw.bytes_written += n;54 cw.bytes_written += written;
47 return n;55 return written;
48}56}
4957
50test CountingWriter {58test CountingWriter {
lib/std/io/Writer.zig+13
...@@ -15,6 +15,19 @@ pub const VTable = struct {...@@ -15,6 +15,19 @@ pub const VTable = struct {
15 /// of stream via an error.15 /// of stream via an error.
16 writev: *const fn (context: *anyopaque, data: []const []const u8) anyerror!usize,16 writev: *const fn (context: *anyopaque, data: []const []const u8) anyerror!usize,
1717
18 /// `headers_and_pattern` must have length of at least one. The last slice
19 /// is `pattern` which is the byte sequence to repeat `n` times. The rest
20 /// of the slices are headers to write before the pattern.
21 ///
22 /// When `n == 1`, this is equivalent to `writev`.
23 ///
24 /// Number of bytes actually written is returned.
25 ///
26 /// Number of bytes returned may be zero, which does not mean
27 /// end-of-stream. A subsequent call may return nonzero, or may signal end
28 /// of stream via an error.
29 splat: *const fn (context: *anyopaque, headers_and_pattern: []const []const u8, n: usize) anyerror!usize,
30
18 /// Writes contents from an open file. `headers` are written first, then `len`31 /// Writes contents from an open file. `headers` are written first, then `len`
19 /// bytes of `file` starting from `offset`, then `trailers`.32 /// bytes of `file` starting from `offset`, then `trailers`.
20 ///33 ///