authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-01 07:56:09+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-01 07:56:09+01:00
loge5454ff780ae4571cfa71a2edb6f4287eb8cf4de
tree49441358c6025c991572759183002e7c11abe7d8
parent3abc96a601d2349cc1743774f0cebb2eb0ea0c61
parentcc442d24ab172a6a2e5ee5210eca3e2f822629f8

Merge pull request 'std.Io: move fileWriteStreaming to Operation' (#31065) from more-poll into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/31065

5 files changed, 219 insertions(+), 73 deletions(-)

lib/std/Io.zig+37-3
......@@ -184,7 +184,6 @@ pub const VTable = struct {
184184 fileStat: *const fn (?*anyopaque, File) File.StatError!File.Stat,
185185 fileLength: *const fn (?*anyopaque, File) File.LengthError!u64,
186186 fileClose: *const fn (?*anyopaque, []const File) void,
187 fileWriteStreaming: *const fn (?*anyopaque, File, header: []const u8, data: []const []const u8, splat: usize) File.Writer.Error!usize,
188187 fileWritePositional: *const fn (?*anyopaque, File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) File.WritePositionalError!usize,
189188 fileWriteFileStreaming: *const fn (?*anyopaque, File, header: []const u8, *Io.File.Reader, Io.Limit) File.Writer.WriteFileError!usize,
190189 fileWriteFilePositional: *const fn (?*anyopaque, File, header: []const u8, *Io.File.Reader, Io.Limit, offset: u64) File.WriteFilePositionalError!usize,
......@@ -257,6 +256,7 @@ pub const VTable = struct {
257256
258257pub const Operation = union(enum) {
259258 file_read_streaming: FileReadStreaming,
259 file_write_streaming: FileWriteStreaming,
260260
261261 pub const Tag = @typeInfo(Operation).@"union".tag_type.?;
262262
......@@ -287,7 +287,41 @@ pub const Operation = union(enum) {
287287 LockViolation,
288288 } || Io.UnexpectedError;
289289
290 pub const Result = usize;
290 pub const Result = Error!usize;
291 };
292
293 pub const FileWriteStreaming = struct {
294 file: File,
295 header: []const u8 = &.{},
296 data: []const []const u8,
297 splat: usize = 1,
298
299 pub const Error = error{
300 DiskQuota,
301 FileTooBig,
302 InputOutput,
303 NoSpaceLeft,
304 DeviceBusy,
305 /// File descriptor does not hold the required rights to write to it.
306 AccessDenied,
307 PermissionDenied,
308 /// File is an unconnected socket, or closed its read end.
309 BrokenPipe,
310 /// Insufficient kernel memory to read from in_fd.
311 SystemResources,
312 NotOpenForWriting,
313 /// The process cannot access the file because another process has locked
314 /// a portion of the file. Windows-only.
315 LockViolation,
316 /// Non-blocking has been enabled and this operation would block.
317 WouldBlock,
318 /// This error occurs when a device gets disconnected before or mid-flush
319 /// while it's being written to - errno(6): No such device or address.
320 NoDevice,
321 FileBusy,
322 } || Io.UnexpectedError;
323
324 pub const Result = Error!usize;
291325 };
292326
293327 pub const Result = Result: {
......@@ -296,7 +330,7 @@ pub const Operation = union(enum) {
296330 var field_types: [operation_fields.len]type = undefined;
297331 for (operation_fields, &field_names, &field_types) |field, *field_name, *field_type| {
298332 field_name.* = field.name;
299 field_type.* = field.type.Error!field.type.Result;
333 field_type.* = field.type.Result;
300334 }
301335 break :Result @Union(.auto, Tag, &field_names, &field_types, &@splat(.{}));
302336 };
lib/std/Io/File.zig+16-5
......@@ -572,16 +572,16 @@ pub fn setTimestampsNow(file: File, io: Io) SetTimestampsError!void {
572572
573573pub const ReadStreamingError = error{EndOfStream} || Reader.Error;
574574
575/// Returns 0 on stream end or if `buffer` has no space available for data.
575/// May return fewer bytes than buffer space available, including 0.
576/// End-of-stream is indicated by `error.EndOfStream`.
576577///
577578/// See also:
578579/// * `reader`
579580pub fn readStreaming(file: File, io: Io, buffer: []const []u8) ReadStreamingError!usize {
580 const result = try io.operate(.{ .file_read_streaming = .{
581 return (try io.operate(.{ .file_read_streaming = .{
581582 .file = file,
582583 .data = buffer,
583 } });
584 return result.file_read_streaming;
584 } })).file_read_streaming;
585585}
586586
587587pub const ReadPositionalError = error{
......@@ -714,11 +714,22 @@ pub fn writerStreaming(file: File, io: Io, buffer: []u8) Writer {
714714 return .initStreaming(file, io, buffer);
715715}
716716
717/// This is a low-level API that calls the `Io` interface function directly.
718/// For a higher level API, see `writerStreaming`.
719pub fn writeStreaming(file: File, io: Io, header: []const u8, data: []const []const u8, splat: usize) Writer.Error!usize {
720 return (try io.operate(.{ .file_write_streaming = .{
721 .file = file,
722 .header = header,
723 .data = data,
724 .splat = splat,
725 } })).file_write_streaming;
726}
727
717728/// Equivalent to creating a streaming writer, writing `bytes`, and then flushing.
718729pub fn writeStreamingAll(file: File, io: Io, bytes: []const u8) Writer.Error!void {
719730 var index: usize = 0;
720731 while (index < bytes.len) {
721 index += try io.vtable.fileWriteStreaming(io.userdata, file, &.{}, &.{bytes[index..]}, 1);
732 index += try writeStreaming(file, io, &.{}, &.{bytes[index..]}, 1);
722733 }
723734}
724735
lib/std/Io/File/Writer.zig+2-25
......@@ -20,30 +20,7 @@ interface: Io.Writer,
2020
2121pub const Mode = File.Reader.Mode;
2222
23pub const Error = error{
24 DiskQuota,
25 FileTooBig,
26 InputOutput,
27 NoSpaceLeft,
28 DeviceBusy,
29 /// File descriptor does not hold the required rights to write to it.
30 AccessDenied,
31 PermissionDenied,
32 /// File is an unconnected socket, or closed its read end.
33 BrokenPipe,
34 /// Insufficient kernel memory to read from in_fd.
35 SystemResources,
36 NotOpenForWriting,
37 /// The process cannot access the file because another process has locked
38 /// a portion of the file. Windows-only.
39 LockViolation,
40 /// Non-blocking has been enabled and this operation would block.
41 WouldBlock,
42 /// This error occurs when a device gets disconnected before or mid-flush
43 /// while it's being written to - errno(6): No such device or address.
44 NoDevice,
45 FileBusy,
46} || Io.Cancelable || Io.UnexpectedError;
23pub const Error = Io.Operation.FileWriteStreaming.Error || Io.Cancelable;
4724
4825pub const WriteFileError = Error || error{
4926 /// Descriptor is not valid or locked, or an mmap(2)-like operation is not available for in_fd.
......@@ -146,7 +123,7 @@ fn drainPositional(w: *Writer, data: []const []const u8, splat: usize) Io.Writer
146123fn drainStreaming(w: *Writer, data: []const []const u8, splat: usize) Io.Writer.Error!usize {
147124 const io = w.io;
148125 const header = w.interface.buffered();
149 const n = io.vtable.fileWriteStreaming(io.userdata, w.file, header, data, splat) catch |err| {
126 const n = w.file.writeStreaming(io, header, data, splat) catch |err| {
150127 w.err = err;
151128 return error.WriteFailed;
152129 };
lib/std/Io/Threaded.zig+162-38
......@@ -1649,7 +1649,6 @@ pub fn io(t: *Threaded) Io {
16491649 .fileStat = fileStat,
16501650 .fileLength = fileLength,
16511651 .fileClose = fileClose,
1652 .fileWriteStreaming = fileWriteStreaming,
16531652 .fileWritePositional = fileWritePositional,
16541653 .fileWriteFileStreaming = fileWriteFileStreaming,
16551654 .fileWriteFilePositional = fileWriteFilePositional,
......@@ -1813,7 +1812,6 @@ pub fn ioBasic(t: *Threaded) Io {
18131812 .fileStat = fileStat,
18141813 .fileLength = fileLength,
18151814 .fileClose = fileClose,
1816 .fileWriteStreaming = fileWriteStreaming,
18171815 .fileWritePositional = fileWritePositional,
18181816 .fileWriteFileStreaming = fileWriteFileStreaming,
18191817 .fileWriteFilePositional = fileWriteFilePositional,
......@@ -2496,6 +2494,12 @@ fn operate(userdata: ?*anyopaque, operation: Io.Operation) Io.Cancelable!Io.Oper
24962494 else => |e| e,
24972495 },
24982496 },
2497 .file_write_streaming => |o| return .{
2498 .file_write_streaming = fileWriteStreaming(t, o.file, o.header, o.data, o.splat) catch |err| switch (err) {
2499 error.Canceled => |e| return e,
2500 else => |e| e,
2501 },
2502 },
24992503 }
25002504}
25012505
......@@ -2523,6 +2527,10 @@ fn batchAwaitAsync(userdata: ?*anyopaque, b: *Io.Batch) Io.Cancelable!void {
25232527 poll_buffer[poll_len] = .{ .fd = o.file.handle, .events = posix.POLL.IN, .revents = 0 };
25242528 poll_len += 1;
25252529 },
2530 .file_write_streaming => |o| {
2531 poll_buffer[poll_len] = .{ .fd = o.file.handle, .events = posix.POLL.OUT, .revents = 0 };
2532 poll_len += 1;
2533 },
25262534 }
25272535 index = submission.node.next;
25282536 }
......@@ -2687,6 +2695,7 @@ fn batchAwaitConcurrent(userdata: ?*anyopaque, b: *Io.Batch, timeout: Io.Timeout
26872695 const submission = &b.storage[index.toIndex()].submission;
26882696 switch (submission.operation) {
26892697 .file_read_streaming => |o| try poll_storage.add(o.file, posix.POLL.IN),
2698 .file_write_streaming => |o| try poll_storage.add(o.file, posix.POLL.OUT),
26902699 }
26912700 index = submission.node.next;
26922701 }
......@@ -2864,6 +2873,7 @@ fn batchApc(apc_context: ?*anyopaque, iosb: *windows.IO_STATUS_BLOCK, _: windows
28642873 b.completions.tail = .fromIndex(index);
28652874 const result: Io.Operation.Result = switch (pending.tag) {
28662875 .file_read_streaming => .{ .file_read_streaming = ntReadFileResult(iosb) },
2876 .file_write_streaming => .{ .file_write_streaming = ntWriteFileResult(iosb) },
28672877 };
28682878 storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
28692879 },
......@@ -2950,6 +2960,66 @@ fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, Concurren
29502960 else => |status| {
29512961 syscall.finish();
29522962
2963 context.iosb.u.Status = status;
2964 batchApc(b, &context.iosb, 0);
2965 break;
2966 },
2967 };
2968 }
2969 },
2970 .file_write_streaming => |o| o: {
2971 const buffer = windowsWriteBuffer(o.header, o.data, o.splat);
2972 if (buffer.len == 0) {
2973 context.iosb = .{
2974 .u = .{ .Status = .SUCCESS },
2975 .Information = 0,
2976 };
2977 batchApc(b, &context.iosb, 0);
2978 break :o;
2979 }
2980 if (o.file.flags.nonblocking) {
2981 context.file = o.file.handle;
2982 switch (windows.ntdll.NtWriteFile(
2983 o.file.handle,
2984 null, // event
2985 &batchApc,
2986 b,
2987 &context.iosb,
2988 buffer.ptr,
2989 @intCast(buffer.len),
2990 null, // byte offset
2991 null, // key
2992 )) {
2993 .PENDING, .SUCCESS => {},
2994 .CANCELLED => unreachable,
2995 else => |status| {
2996 context.iosb.u.Status = status;
2997 batchApc(b, &context.iosb, 0);
2998 },
2999 }
3000 } else {
3001 if (concurrency) return error.ConcurrencyUnavailable;
3002
3003 const syscall: Syscall = try .start();
3004 while (true) switch (windows.ntdll.NtWriteFile(
3005 o.file.handle,
3006 null, // event
3007 null, // APC routine
3008 null, // APC context
3009 &context.iosb,
3010 buffer.ptr,
3011 @intCast(buffer.len),
3012 null, // byte offset
3013 null, // key
3014 )) {
3015 .PENDING => unreachable, // unrecoverable: wrong File nonblocking flag
3016 .CANCELLED => {
3017 try syscall.checkCancel();
3018 continue;
3019 },
3020 else => |status| {
3021 syscall.finish();
3022
29533023 context.iosb.u.Status = status;
29543024 batchApc(b, &context.iosb, 0);
29553025 break;
......@@ -2963,6 +3033,21 @@ fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, Concurren
29633033 b.submissions = .{ .head = .none, .tail = .none };
29643034}
29653035
3036/// Since Windows only supports writing one contiguous buffer, returns the
3037/// first one, while also limiting it to a length representable by 32-bit
3038/// unsigned integer.
3039fn windowsWriteBuffer(header: []const u8, data: []const []const u8, splat: usize) []const u8 {
3040 const buffer = b: {
3041 if (header.len != 0) break :b header;
3042 for (data[0 .. data.len - 1]) |buffer| {
3043 if (buffer.len != 0) break :b buffer;
3044 }
3045 if (splat == 0) return &.{};
3046 break :b data[data.len - 1];
3047 };
3048 return buffer[0..@min(buffer.len, std.math.maxInt(u32))];
3049}
3050
29663051fn submitComplete(ring: []u32, complete_tail: *Io.Batch.RingIndex, op: u32) void {
29673052 const ct = complete_tail.*;
29683053 const len: u31 = @intCast(ring.len);
......@@ -9005,6 +9090,24 @@ fn ntReadFileResult(io_status_block: *const windows.IO_STATUS_BLOCK) !usize {
90059090 }
90069091}
90079092
9093fn ntWriteFileResult(io_status_block: *const windows.IO_STATUS_BLOCK) !usize {
9094 switch (io_status_block.u.Status) {
9095 .PENDING => unreachable,
9096 .CANCELLED => unreachable,
9097 .SUCCESS => return io_status_block.Information,
9098 .INVALID_USER_BUFFER => return error.SystemResources,
9099 .NO_MEMORY => return error.SystemResources,
9100 .QUOTA_EXCEEDED => return error.SystemResources,
9101 .PIPE_BROKEN => return error.BrokenPipe,
9102 .INVALID_HANDLE => return error.NotOpenForWriting,
9103 .LOCK_NOT_GRANTED => return error.LockViolation,
9104 .ACCESS_DENIED => return error.AccessDenied,
9105 .WORKING_SET_QUOTA => return error.SystemResources,
9106 .DISK_FULL => return error.NoSpaceLeft,
9107 else => |status| return windows.unexpectedStatus(status),
9108 }
9109}
9110
90089111fn fileReadPositionalPosix(file: File, data: []const []u8, offset: u64) File.ReadPositionalError!usize {
90099112 if (!have_preadv) @compileError("TODO implement fileReadPositionalPosix for cursed operating systems that don't support preadv (it's only Haiku)");
90109113
......@@ -9837,16 +9940,9 @@ fn fileWriteStreaming(
98379940 _ = t;
98389941
98399942 if (is_windows) {
9840 if (header.len != 0) {
9841 return writeFileStreamingWindows(file.handle, header);
9842 }
9843 for (data[0 .. data.len - 1]) |buf| {
9844 if (buf.len == 0) continue;
9845 return writeFileStreamingWindows(file.handle, buf);
9846 }
9847 const pattern = data[data.len - 1];
9848 if (pattern.len == 0 or splat == 0) return 0;
9849 return writeFileStreamingWindows(file.handle, pattern);
9943 const buffer = windowsWriteBuffer(header, data, splat);
9944 if (buffer.len == 0) return 0;
9945 return fileWriteStreamingWindows(file, buffer);
98509946 }
98519947
98529948 var iovecs: [max_iovecs_len]posix.iovec_const = undefined;
......@@ -9953,38 +10049,66 @@ fn fileWriteStreaming(
995310049 }
995410050}
995510051
9956fn writeFileStreamingWindows(
9957 handle: windows.HANDLE,
9958 bytes: []const u8,
9959) File.Writer.Error!usize {
9960 assert(bytes.len != 0);
9961 var bytes_written: windows.DWORD = undefined;
9962 const adjusted_len = std.math.lossyCast(u32, bytes.len);
9963 const syscall: Syscall = try .start();
9964 while (true) {
9965 if (windows.kernel32.WriteFile(handle, bytes.ptr, adjusted_len, &bytes_written, null) != 0) {
9966 syscall.finish();
9967 return bytes_written;
10052fn fileWriteStreamingWindows(file: File, buffer: []const u8) File.Writer.Error!usize {
10053 assert(buffer.len != 0);
10054
10055 var iosb: windows.IO_STATUS_BLOCK = undefined;
10056
10057 if (file.flags.nonblocking) {
10058 var done: bool = false;
10059 switch (windows.ntdll.NtWriteFile(
10060 file.handle,
10061 null, // event
10062 flagApc,
10063 &done, // APC context
10064 &iosb,
10065 buffer.ptr,
10066 @intCast(buffer.len),
10067 null, // byte offset
10068 null, // key
10069 )) {
10070 // We must wait for the APC routine.
10071 .PENDING, .SUCCESS => while (!done) {
10072 // Once we get here we must not return from the function until the
10073 // operation completes, thereby releasing reference to io_status_block.
10074 const alertable_syscall = AlertableSyscall.start() catch |err| switch (err) {
10075 error.Canceled => |e| {
10076 var cancel_iosb: windows.IO_STATUS_BLOCK = undefined;
10077 _ = windows.ntdll.NtCancelIoFileEx(file.handle, &iosb, &cancel_iosb);
10078 while (!done) waitForApcOrAlert();
10079 return e;
10080 },
10081 };
10082 waitForApcOrAlert();
10083 alertable_syscall.finish();
10084 },
10085 else => |status| iosb.u.Status = status,
996810086 }
9969 switch (windows.GetLastError()) {
9970 .OPERATION_ABORTED => {
10087 return ntWriteFileResult(&iosb);
10088 } else {
10089 const syscall: Syscall = try .start();
10090 while (true) switch (windows.ntdll.NtWriteFile(
10091 file.handle,
10092 null, // event
10093 null, // APC routine
10094 null, // APC context
10095 &iosb,
10096 buffer.ptr,
10097 @intCast(buffer.len),
10098 null, // byte offset
10099 null, // key
10100 )) {
10101 .PENDING => unreachable, // unrecoverable: wrong File nonblocking flag
10102 .CANCELLED => {
997110103 try syscall.checkCancel();
997210104 continue;
997310105 },
9974 .INVALID_USER_BUFFER => return syscall.fail(error.SystemResources),
9975 .NOT_ENOUGH_MEMORY => return syscall.fail(error.SystemResources),
9976 .NOT_ENOUGH_QUOTA => return syscall.fail(error.SystemResources),
9977 .NO_DATA => return syscall.fail(error.BrokenPipe),
9978 .INVALID_HANDLE => return syscall.fail(error.NotOpenForWriting),
9979 .LOCK_VIOLATION => return syscall.fail(error.LockViolation),
9980 .ACCESS_DENIED => return syscall.fail(error.AccessDenied),
9981 .WORKING_SET_QUOTA => return syscall.fail(error.SystemResources),
9982 .DISK_FULL => return syscall.fail(error.NoSpaceLeft),
9983 else => |err| {
10106 else => |status| {
998410107 syscall.finish();
9985 return windows.unexpectedError(err);
10108 iosb.u.Status = status;
10109 return ntWriteFileResult(&iosb);
998610110 },
9987 }
10111 };
998810112 }
998910113}
999010114
lib/std/Progress.zig+2-2
......@@ -1437,7 +1437,7 @@ fn writeIpc(io: Io, file: Io.File, serialized: Serialized) error{BrokenPipe}!voi
14371437 // We do this in a separate write call to give a better chance for the
14381438 // writev below to be in a single packet.
14391439 const n = @min(parents.len, remaining_write_trash_bytes);
1440 if (io.vtable.fileWriteStreaming(io.userdata, file, &.{}, &.{parents[0..n]}, 1)) |written| {
1440 if (file.writeStreaming(io, &.{}, &.{parents[0..n]}, 1)) |written| {
14411441 remaining_write_trash_bytes -= written;
14421442 continue;
14431443 } else |err| switch (err) {
......@@ -1478,7 +1478,7 @@ fn writevNonblock(io: Io, file: Io.File, iov: [][]const u8) Io.File.Writer.Error
14781478 return total_written) : (iov_index += 1) written -= iov[iov_index].len;
14791479 iov[iov_index].ptr += written;
14801480 iov[iov_index].len -= written;
1481 written = try io.vtable.fileWriteStreaming(io.userdata, file, &.{}, iov, 1);
1481 written = try file.writeStreaming(io, &.{}, iov, 1);
14821482 if (written == 0) return total_written;
14831483 total_written += written;
14841484 }