authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2026-01-30 01:44:07-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-01-30 12:10:05-08:00
log3d3f22a14d7639f1a7c607da98926da6e60c3b01
treed7144601a752d50da1c9cea235db650ffb76f8ea
parent10bec043f52c08eef73b758042139635e535c0d3

Io.Batch: implement alternate API

4 files changed, 508 insertions(+), 367 deletions(-)

lib/std/Io.zig+138-109
...@@ -149,8 +149,9 @@ pub const VTable = struct {...@@ -149,8 +149,9 @@ pub const VTable = struct {
149 futexWaitUncancelable: *const fn (?*anyopaque, ptr: *const u32, expected: u32) void,149 futexWaitUncancelable: *const fn (?*anyopaque, ptr: *const u32, expected: u32) void,
150 futexWake: *const fn (?*anyopaque, ptr: *const u32, max_waiters: u32) void,150 futexWake: *const fn (?*anyopaque, ptr: *const u32, max_waiters: u32) void,
151151
152 operate: *const fn (?*anyopaque, *Operation) Cancelable!void,152 operate: *const fn (?*anyopaque, Operation) Cancelable!Operation.Result,
153 batchWait: *const fn (?*anyopaque, *Batch, Timeout) Batch.WaitError!void,153 batchAwaitAsync: *const fn (?*anyopaque, *Batch) Batch.AwaitAsyncError!void,
154 batchAwaitConcurrent: *const fn (?*anyopaque, *Batch, Timeout) Batch.AwaitConcurrentError!void,
154 batchCancel: *const fn (?*anyopaque, *Batch) void,155 batchCancel: *const fn (?*anyopaque, *Batch) void,
155156
156 dirCreateDir: *const fn (?*anyopaque, Dir, []const u8, Dir.Permissions) Dir.CreateDirError!void,157 dirCreateDir: *const fn (?*anyopaque, Dir, []const u8, Dir.Permissions) Dir.CreateDirError!void,
...@@ -255,19 +256,14 @@ pub const VTable = struct {...@@ -255,19 +256,14 @@ pub const VTable = struct {
255};256};
256257
257pub const Operation = union(enum) {258pub const Operation = union(enum) {
258 noop: Noop,
259 file_read_streaming: FileReadStreaming,259 file_read_streaming: FileReadStreaming,
260260
261 pub const Noop = struct {261 pub const Tag = @typeInfo(Operation).@"union".tag_type.?;
262 reserved: [2]usize = .{ 0, 0 },
263 status: Status(void) = .{ .unstarted = {} },
264 };
265262
266 /// May return 0 reads which is different than `error.EndOfStream`.263 /// May return 0 reads which is different than `error.EndOfStream`.
267 pub const FileReadStreaming = struct {264 pub const FileReadStreaming = struct {
268 file: File,265 file: File,
269 data: []const []u8,266 data: []const []u8,
270 status: Status(Error!usize) = .{ .unstarted = {} },
271267
272 pub const Error = UnendingError || error{EndOfStream};268 pub const Error = UnendingError || error{EndOfStream};
273 pub const UnendingError = error{269 pub const UnendingError = error{
...@@ -290,19 +286,72 @@ pub const Operation = union(enum) {...@@ -290,19 +286,72 @@ pub const Operation = union(enum) {
290 /// lock.286 /// lock.
291 LockViolation,287 LockViolation,
292 } || Io.UnexpectedError;288 } || Io.UnexpectedError;
289
290 pub const Result = usize;
291 };
292
293 pub const Result = Result: {
294 const operation_fields = @typeInfo(Operation).@"union".fields;
295 var field_names: [operation_fields.len][]const u8 = undefined;
296 var field_types: [operation_fields.len]type = undefined;
297 for (operation_fields, &field_names, &field_types) |field, *field_name, *field_type| {
298 field_name.* = field.name;
299 field_type.* = field.type.Error!field.type.Result;
300 }
301 break :Result @Union(.auto, Tag, &field_names, &field_types, &@splat(.{}));
293 };302 };
294303
295 pub fn Status(Result: type) type {304 pub const Storage = union {
296 return union {305 unused: List.DoubleNode,
297 unstarted: void,306 submission: Submission,
298 pending: *Batch,307 pending: Pending,
308 completion: Completion,
309
310 pub const Submission = struct {
311 node: List.SingleNode,
312 operation: Operation,
313 };
314
315 pub const Pending = struct {
316 node: List.DoubleNode,
317 tag: Tag,
318 context: [3]usize,
319 };
320
321 pub const Completion = struct {
322 node: List.SingleNode,
299 result: Result,323 result: Result,
300 };324 };
301 }325 };
326
327 pub const OptionalIndex = enum(u32) {
328 none = std.math.maxInt(u32),
329 _,
330
331 pub fn fromIndex(i: usize) OptionalIndex {
332 const oi: OptionalIndex = @enumFromInt(i);
333 assert(oi != .none);
334 return oi;
335 }
336
337 pub fn toIndex(oi: OptionalIndex) u32 {
338 assert(oi != .none);
339 return @intFromEnum(oi);
340 }
341 };
342 pub const List = struct {
343 head: OptionalIndex,
344 tail: OptionalIndex,
345
346 pub const empty: List = .{ .head = .none, .tail = .none };
347
348 pub const SingleNode = struct { next: OptionalIndex };
349 pub const DoubleNode = struct { prev: OptionalIndex, next: OptionalIndex };
350 };
302};351};
303352
304/// Performs one `Operation`.353/// Performs one `Operation`.
305pub fn operate(io: Io, operation: *Operation) Cancelable!void {354pub fn operate(io: Io, operation: Operation) Cancelable!Operation.Result {
306 return io.vtable.operate(io.userdata, operation);355 return io.vtable.operate(io.userdata, operation);
307}356}
308357
...@@ -312,116 +361,96 @@ pub fn operate(io: Io, operation: *Operation) Cancelable!void {...@@ -312,116 +361,96 @@ pub fn operate(io: Io, operation: *Operation) Cancelable!void {
312/// This is a low-level abstraction based on `Operation`. For a higher361/// This is a low-level abstraction based on `Operation`. For a higher
313/// level API that operates on `Future`, see `Select`.362/// level API that operates on `Future`, see `Select`.
314pub const Batch = struct {363pub const Batch = struct {
315 operations: []Operation,364 storage: []Operation.Storage,
316 ring: [*]u32,365 unused: Operation.List,
317 user: struct {366 submissions: Operation.List,
318 submit_tail: RingIndex,367 pending: Operation.List,
319 complete_head: RingIndex,368 completions: Operation.List,
320 complete_tail: RingIndex,369 context: ?*anyopaque,
321 },
322 impl: struct {
323 submit_head: RingIndex,
324 submit_tail: RingIndex,
325 complete_tail: RingIndex,
326 reserved: ?*anyopaque,
327 },
328
329 pub const RingIndex = enum(u32) {
330 _,
331
332 pub fn index(ri: RingIndex, len: u31) u31 {
333 const i = @intFromEnum(ri);
334 assert(i < @as(u32, len) * 2);
335 return @intCast(if (i < len) i else i - len);
336 }
337
338 pub fn prev(ri: RingIndex, len: u31) RingIndex {
339 const i = @intFromEnum(ri);
340 const double_len = @as(u32, len) * 2;
341 assert(i <= double_len);
342 return @enumFromInt((if (i > 0) i else double_len) - 1);
343 }
344
345 pub fn next(ri: RingIndex, len: u31) RingIndex {
346 const i = @intFromEnum(ri) + 1;
347 const double_len = @as(u32, len) * 2;
348 assert(i <= double_len);
349 return @enumFromInt(if (i < double_len) i else 0);
350 }
351 };
352370
353 /// After calling this, it is safe to unconditionally defer a call to371 /// After calling this, it is safe to unconditionally defer a call to
354 /// `cancel`.372 /// `cancel`.
355 pub fn init(operations: []Operation, ring: []u32) Batch {373 pub fn init(storage: []Operation.Storage) Batch {
356 const len: u31 = @intCast(operations.len);374 var prev: Operation.OptionalIndex = .none;
357 assert(ring.len == len);375 for (storage, 0..) |*operation, index| {
376 operation.* = .{ .unused = .{ .prev = prev, .next = .fromIndex(index + 1) } };
377 prev = .fromIndex(index);
378 }
379 storage[storage.len - 1].unused.next = .none;
358 return .{380 return .{
359 .operations = operations,381 .storage = storage,
360 .ring = ring.ptr,382 .unused = .{
361 .user = .{383 .head = .fromIndex(0),
362 .submit_tail = @enumFromInt(0),384 .tail = .fromIndex(storage.len - 1),
363 .complete_head = @enumFromInt(0),
364 .complete_tail = @enumFromInt(0),
365 },
366 .impl = .{
367 .submit_head = @enumFromInt(0),
368 .submit_tail = @enumFromInt(0),
369 .complete_tail = @enumFromInt(0),
370 .reserved = null,
371 },385 },
386 .submissions = .empty,
387 .pending = .empty,
388 .completions = .empty,
389 .context = null,
372 };390 };
373 }391 }
374392
375 /// Adds `b.operations[operation]` to the list of submitted operations393 /// Adds an operation to be performed at the next await call.
376 /// that will be performed when `wait` is called.394 /// Returns the index that will be returned by `next` after the operation completes.
377 pub fn add(b: *Batch, operation: usize) void {395 /// Asserts that no more than `storage.len` operations are active at a time.
378 const tail = b.user.submit_tail;396 pub fn add(b: *Batch, operation: Operation) u32 {
379 const len: u31 = @intCast(b.operations.len);397 const index = b.unused.next;
380 b.user.submit_tail = tail.next(len);398 b.addAt(index.toIndex(), operation);
381 b.ring[0..len][tail.index(len)] = @intCast(operation);399 return index;
382 }400 }
383401
384 fn flush(b: *Batch) void {402 /// Adds an operation to be performed at the next await call.
385 @atomicStore(RingIndex, &b.impl.submit_tail, b.user.submit_tail, .release);403 /// After the operation completes, `next` will return `index`.
386 }404 /// Asserts that the operation at `index` is not active.
405 pub fn addAt(b: *Batch, index: u32, operation: Operation) void {
406 const storage = &b.storage[index];
407 const unused = storage.unused;
408 switch (unused.prev) {
409 .none => b.unused.head = .none,
410 else => |prev_index| b.storage[prev_index.toIndex()].unused.next = unused.next,
411 }
412 switch (unused.next) {
413 .none => b.unused.tail = .none,
414 else => |next_index| b.storage[next_index.toIndex()].unused.prev = unused.prev,
415 }
387416
388 /// Returns `operation` such that `b.operations[operation]` has completed.417 switch (b.submissions.tail) {
389 /// Returns `null` when `wait` should be called.418 .none => b.submissions.head = .fromIndex(index),
390 pub fn next(b: *Batch) ?u32 {419 else => |tail_index| b.storage[tail_index.toIndex()].submission.node.next = .fromIndex(index),
391 const head = b.user.complete_head;420 }
392 if (head == b.user.complete_tail) {421 storage.* = .{ .submission = .{ .node = .{ .next = .none }, .operation = operation } };
393 @branchHint(.unlikely);422 b.submissions.tail = .fromIndex(index);
394 b.flush();423 }
395 const tail = @atomicLoad(RingIndex, &b.impl.complete_tail, .acquire);424
396 if (head == tail) {425 pub fn next(b: *Batch) ?struct { index: u32, result: Operation.Result } {
397 @branchHint(.unlikely);426 const index = b.completions.head;
398 return null;427 if (index == .none) return null;
399 }428 const storage = &b.storage[index.toIndex()];
400 assert(head != tail);429 const completion = storage.completion;
401 b.user.complete_tail = tail;430 const next_index = completion.node.next;
431 b.completions.head = next_index;
432 if (next_index == .none) b.completions.tail = .none;
433
434 const tail_index = b.unused.tail;
435 switch (tail_index) {
436 .none => b.unused.head = index,
437 else => b.storage[tail_index.toIndex()].unused.next = index,
402 }438 }
403 const len: u31 = @intCast(b.operations.len);439 storage.* = .{ .unused = .{ .prev = tail_index, .next = .none } };
404 b.user.complete_head = head.next(len);440 b.unused.tail = index;
405 return b.ring[0..len][head.index(len)];441 return .{ .index = index.toIndex(), .result = completion.result };
406 }442 }
407443
408 pub const WaitError = ConcurrentError || Cancelable || Timeout.Error;444 pub const AwaitAsyncError = Cancelable;
445 pub fn awaitAsync(b: *Batch, io: Io) AwaitAsyncError!void {
446 return io.vtable.batchAwaitAsync(io.userdata, b);
447 }
409448
410 /// Starts work on any submitted operations and returns when at least one has completeed.449 pub const AwaitConcurrentError = ConcurrentError || Cancelable || Timeout.Error;
411 ///450 pub fn awaitConcurrent(b: *Batch, io: Io, timeout: Timeout) AwaitConcurrentError!void {
412 /// Returns `error.Timeout` if `timeout` expires first.451 return io.vtable.batchAwaitConcurrent(io.userdata, b, timeout);
413 ///
414 /// Depending on the `Io` implementation, may allocate resources that are
415 /// freed with `cancel`, even if an error is returned.
416 pub fn wait(b: *Batch, io: Io, timeout: Timeout) WaitError!void {
417 return io.vtable.batchWait(io.userdata, b, timeout);
418 }452 }
419453
420 /// Returns after all `operations` have completed. Operations which have not completed
421 /// after this function returns were successfully dropped and had no side effects.
422 ///
423 /// This function is idempotent with respect to itself and `wait`. It is
424 /// safe to unconditionally `defer` a call to this function after `init`.
425 pub fn cancel(b: *Batch, io: Io) void {454 pub fn cancel(b: *Batch, io: Io) void {
426 return io.vtable.batchCancel(io.userdata, b);455 return io.vtable.batchCancel(io.userdata, b);
427 }456 }
lib/std/Io/File.zig+3-4
...@@ -559,12 +559,11 @@ pub const ReadStreamingError = error{EndOfStream} || Reader.Error;...@@ -559,12 +559,11 @@ pub const ReadStreamingError = error{EndOfStream} || Reader.Error;
559/// See also:559/// See also:
560/// * `reader`560/// * `reader`
561pub fn readStreaming(file: File, io: Io, buffer: []const []u8) ReadStreamingError!usize {561pub fn readStreaming(file: File, io: Io, buffer: []const []u8) ReadStreamingError!usize {
562 var operation: Io.Operation = .{ .file_read_streaming = .{562 const result = try io.operate(.{ .file_read_streaming = .{
563 .file = file,563 .file = file,
564 .data = buffer,564 .data = buffer,
565 } };565 } });
566 try io.operate(&operation);566 return result.file_read_streaming;
567 return operation.file_read_streaming.status.result;
568}567}
569568
570pub const ReadPositionalError = error{569pub const ReadPositionalError = error{
lib/std/Io/File/MultiReader.zig+20-32
...@@ -22,8 +22,7 @@ pub const UnendingError = Allocator.Error || File.Reader.Error || Io.ConcurrentE...@@ -22,8 +22,7 @@ pub const UnendingError = Allocator.Error || File.Reader.Error || Io.ConcurrentE
2222
23/// Trailing:23/// Trailing:
24/// * `contexts: [len]Context`24/// * `contexts: [len]Context`
25/// * `ring: [len]u32`25/// * `storage: [len]Io.Operation.Storage`
26/// * `operations: [len]Io.Operation`
27pub const Streams = extern struct {26pub const Streams = extern struct {
28 len: u32,27 len: u32,
2928
...@@ -33,17 +32,10 @@ pub const Streams = extern struct {...@@ -33,17 +32,10 @@ pub const Streams = extern struct {
33 return ptr[0..s.len];32 return ptr[0..s.len];
34 }33 }
3534
36 pub fn ring(s: *Streams) []u32 {35 pub fn storage(s: *Streams) []Io.Operation.Storage {
37 const prev = contexts(s);36 const prev = contexts(s);
38 const end = prev.ptr + prev.len;37 const end = prev.ptr + prev.len;
39 const ptr: [*]u32 = @ptrFromInt(std.mem.alignForward(usize, @intFromPtr(end), @alignOf(u32)));38 const ptr: [*]Io.Operation.Storage = @ptrFromInt(std.mem.alignForward(usize, @intFromPtr(end), @alignOf(Io.Operation.Storage)));
40 return ptr[0..s.len];
41 }
42
43 pub fn operations(s: *Streams) []Io.Operation {
44 const prev = ring(s);
45 const end = prev.ptr + prev.len;
46 const ptr: [*]Io.Operation = @ptrFromInt(std.mem.alignForward(usize, @intFromPtr(end), @alignOf(Io.Operation)));
47 return ptr[0..s.len];39 return ptr[0..s.len];
48 }40 }
49};41};
...@@ -52,8 +44,7 @@ pub fn Buffer(comptime n: usize) type {...@@ -52,8 +44,7 @@ pub fn Buffer(comptime n: usize) type {
52 return extern struct {44 return extern struct {
53 len: u32,45 len: u32,
54 contexts: [n][@sizeOf(Context)]u8 align(@alignOf(Context)),46 contexts: [n][@sizeOf(Context)]u8 align(@alignOf(Context)),
55 ring: [n]u32,47 storage: [n][@sizeOf(Io.Operation.Storage)]u8 align(@alignOf(Io.Operation.Storage)),
56 operations: [n][@sizeOf(Io.Operation)]u8 align(@alignOf(Io.Operation)),
5748
58 pub fn toStreams(b: *@This()) *Streams {49 pub fn toStreams(b: *@This()) *Streams {
59 b.len = n;50 b.len = n;
...@@ -86,25 +77,22 @@ pub fn init(mr: *MultiReader, gpa: Allocator, io: Io, streams: *Streams, files:...@@ -86,25 +77,22 @@ pub fn init(mr: *MultiReader, gpa: Allocator, io: Io, streams: *Streams, files:
86 .vec = .{&.{}},77 .vec = .{&.{}},
87 .err = null,78 .err = null,
88 };79 };
89 const operations = streams.operations();
90 const ring = streams.ring();
91 mr.* = .{80 mr.* = .{
92 .gpa = gpa,81 .gpa = gpa,
93 .streams = streams,82 .streams = streams,
94 .batch = .init(operations, ring),83 .batch = .init(streams.storage()),
95 };84 };
96 for (operations, contexts, files, 0..) |*op, *context, file, i| {85 for (contexts, 0..) |*context, i| {
97 const r = &context.fr.interface;86 const r = &context.fr.interface;
98 op.* = .{ .file_read_streaming = .{
99 .file = file,
100 .data = &context.vec,
101 } };
102 rebaseGrowing(mr, context, 1) catch |err| {87 rebaseGrowing(mr, context, 1) catch |err| {
103 context.err = err;88 context.err = err;
104 continue;89 continue;
105 };90 };
106 context.vec[0] = r.buffer;91 context.vec[0] = r.buffer;
107 mr.batch.add(i);92 mr.batch.addAt(@intCast(i), .{ .file_read_streaming = .{
93 .file = context.fr.file,
94 .data = &context.vec,
95 } });
108 }96 }
109}97}
11098
...@@ -204,7 +192,7 @@ fn fillUntimed(context: *Context, capacity: usize) Io.Reader.Error!void {...@@ -204,7 +192,7 @@ fn fillUntimed(context: *Context, capacity: usize) Io.Reader.Error!void {
204 };192 };
205}193}
206194
207pub const FillError = Io.Batch.WaitError || error{195pub const FillError = Io.Batch.AwaitConcurrentError || error{
208 /// `fill` was called when all streams already have failed or reached the196 /// `fill` was called when all streams already have failed or reached the
209 /// end.197 /// end.
210 EndOfStream,198 EndOfStream,
...@@ -213,17 +201,15 @@ pub const FillError = Io.Batch.WaitError || error{...@@ -213,17 +201,15 @@ pub const FillError = Io.Batch.WaitError || error{
213/// Wait until at least one stream receives more data.201/// Wait until at least one stream receives more data.
214pub fn fill(mr: *MultiReader, unused_capacity: usize, timeout: Io.Timeout) FillError!void {202pub fn fill(mr: *MultiReader, unused_capacity: usize, timeout: Io.Timeout) FillError!void {
215 const contexts = mr.streams.contexts();203 const contexts = mr.streams.contexts();
216 const operations = mr.streams.operations();
217 const io = contexts[0].fr.io;204 const io = contexts[0].fr.io;
218 var any_completed = false;205 var any_completed = false;
219206
220 try mr.batch.wait(io, timeout);207 try mr.batch.awaitConcurrent(io, timeout);
221208
222 while (mr.batch.next()) |i| {209 while (mr.batch.next()) |operation| {
223 any_completed = true;210 any_completed = true;
224 const context = &contexts[i];211 const context = &contexts[operation.index];
225 const operation = &operations[i];212 const n = operation.result.file_read_streaming catch |err| {
226 const n = operation.file_read_streaming.status.result catch |err| {
227 context.err = err;213 context.err = err;
228 continue;214 continue;
229 };215 };
...@@ -237,15 +223,17 @@ pub fn fill(mr: *MultiReader, unused_capacity: usize, timeout: Io.Timeout) FillE...@@ -237,15 +223,17 @@ pub fn fill(mr: *MultiReader, unused_capacity: usize, timeout: Io.Timeout) FillE
237 assert(r.seek == 0);223 assert(r.seek == 0);
238 }224 }
239 context.vec[0] = r.buffer[r.end..];225 context.vec[0] = r.buffer[r.end..];
240 operation.file_read_streaming.status = .{ .unstarted = {} };226 mr.batch.addAt(operation.index, .{ .file_read_streaming = .{
241 mr.batch.add(i);227 .file = context.fr.file,
228 .data = &context.vec,
229 } });
242 }230 }
243231
244 if (!any_completed) return error.EndOfStream;232 if (!any_completed) return error.EndOfStream;
245}233}
246234
247/// Wait until all streams fail or reach the end.235/// Wait until all streams fail or reach the end.
248pub fn fillRemaining(mr: *MultiReader, timeout: Io.Timeout) Io.Batch.WaitError!void {236pub fn fillRemaining(mr: *MultiReader, timeout: Io.Timeout) Io.Batch.AwaitConcurrentError!void {
249 while (fill(mr, 1, timeout)) |_| {} else |err| switch (err) {237 while (fill(mr, 1, timeout)) |_| {} else |err| switch (err) {
250 error.EndOfStream => return,238 error.EndOfStream => return,
251 else => |e| return e,239 else => |e| return e,
lib/std/Io/Threaded.zig+347-222
...@@ -1617,7 +1617,8 @@ pub fn io(t: *Threaded) Io {...@@ -1617,7 +1617,8 @@ pub fn io(t: *Threaded) Io {
1617 .futexWake = futexWake,1617 .futexWake = futexWake,
16181618
1619 .operate = operate,1619 .operate = operate,
1620 .batchWait = batchWait,1620 .batchAwaitAsync = batchAwaitAsync,
1621 .batchAwaitConcurrent = batchAwaitConcurrent,
1621 .batchCancel = batchCancel,1622 .batchCancel = batchCancel,
16221623
1623 .dirCreateDir = dirCreateDir,1624 .dirCreateDir = dirCreateDir,
...@@ -1780,7 +1781,8 @@ pub fn ioBasic(t: *Threaded) Io {...@@ -1780,7 +1781,8 @@ pub fn ioBasic(t: *Threaded) Io {
1780 .futexWake = futexWake,1781 .futexWake = futexWake,
17811782
1782 .operate = operate,1783 .operate = operate,
1783 .batchWait = batchWait,1784 .batchAwaitAsync = batchAwaitAsync,
1785 .batchAwaitConcurrent = batchAwaitConcurrent,
1784 .batchCancel = batchCancel,1786 .batchCancel = batchCancel,
17851787
1786 .dirCreateDir = dirCreateDir,1788 .dirCreateDir = dirCreateDir,
...@@ -2483,85 +2485,227 @@ fn futexWake(userdata: ?*anyopaque, ptr: *const u32, max_waiters: u32) void {...@@ -2483,85 +2485,227 @@ fn futexWake(userdata: ?*anyopaque, ptr: *const u32, max_waiters: u32) void {
2483 Thread.futexWake(ptr, max_waiters);2485 Thread.futexWake(ptr, max_waiters);
2484}2486}
24852487
2486fn operate(userdata: ?*anyopaque, op: *Io.Operation) Io.Cancelable!void {2488fn operate(userdata: ?*anyopaque, operation: Io.Operation) Io.Cancelable!Io.Operation.Result {
2487 const t: *Threaded = @ptrCast(@alignCast(userdata));2489 const t: *Threaded = @ptrCast(@alignCast(userdata));
2488 switch (op.*) {2490 switch (operation) {
2489 .noop => |*o| {2491 .file_read_streaming => |o| return .{
2490 _ = o.status.unstarted;2492 .file_read_streaming = fileReadStreaming(t, o.file, o.data) catch |err| switch (err) {
2491 o.status = .{ .result = {} };
2492 },
2493 .file_read_streaming => |*o| {
2494 _ = o.status.unstarted;
2495 o.status = .{ .result = fileReadStreaming(t, o.file, o.data) catch |err| switch (err) {
2496 error.Canceled => |e| return e,2493 error.Canceled => |e| return e,
2497 else => |e| e,2494 else => |e| e,
2498 } };2495 },
2499 },2496 },
2500 }2497 }
2501}2498}
25022499
2503fn batchWait(userdata: ?*anyopaque, b: *Io.Batch, timeout: Io.Timeout) Io.Batch.WaitError!void {2500fn batchAwaitAsync(userdata: ?*anyopaque, b: *Io.Batch) Io.Batch.AwaitAsyncError!void {
2504 const t: *Threaded = @ptrCast(@alignCast(userdata));2501 const t: *Threaded = @ptrCast(@alignCast(userdata));
2505 if (is_windows) return batchWaitWindows(t, b, timeout);2502 if (is_windows) {
2503 try batchAwaitWindows(b);
2504 const alertable_syscall = try AlertableSyscall.start();
2505 while (b.pending.head != .none and b.completions.head == .none) waitForApcOrAlert();
2506 alertable_syscall.finish();
2507 return;
2508 }
2506 if (native_os == .wasi and !builtin.link_libc) @panic("TODO");2509 if (native_os == .wasi and !builtin.link_libc) @panic("TODO");
2507 const operations = b.operations;2510 var poll_buffer: [poll_buffer_len]posix.pollfd = undefined;
2508 const len: u31 = @intCast(operations.len);2511 var poll_len: u32 = 0;
2509 const ring = b.ring[0..len];2512 {
2510 var submit_head = b.impl.submit_head;2513 var index = b.submissions.head;
2511 const submit_tail = b.user.submit_tail;2514 while (index != .none and poll_len < poll_buffer_len) {
2512 b.impl.submit_tail = submit_tail;2515 const submission = &b.storage[index.toIndex()].submission;
2513 var complete_tail = b.impl.complete_tail;2516 switch (submission.operation) {
2514 var map_buffer: [poll_buffer_len]u8 = undefined; // poll_buffer index to operations index2517 .file_read_streaming => |o| {
2515 var poll_i: u8 = 0;2518 poll_buffer[poll_len] = .{ .fd = o.file.handle, .events = posix.POLL.IN, .revents = 0 };
2516 defer {2519 poll_len += 1;
2517 for (map_buffer[0..poll_i]) |op| {2520 },
2518 submit_head = submit_head.prev(len);2521 }
2519 ring[submit_head.index(len)] = op;2522 index = submission.node.next;
2520 }2523 }
2521 b.impl.submit_head = submit_head;2524 }
2522 b.impl.complete_tail = complete_tail;2525 switch (poll_len) {
2523 b.user.complete_tail = complete_tail;2526 0 => return,
2527 1 => {},
2528 else => while (true) {
2529 const timeout_ms: i32 = t: {
2530 if (b.completions.head != .none) {
2531 // It is legal to call batchWait with already completed
2532 // operations in the ring. In such case, we need to avoid
2533 // blocking in the poll syscall, but we can still take this
2534 // opportunity to find additional ready operations.
2535 break :t 0;
2536 }
2537 const max_poll_ms = std.math.maxInt(i32);
2538 break :t max_poll_ms;
2539 };
2540 const syscall = try Syscall.start();
2541 const rc = posix.system.poll(&poll_buffer, poll_len, timeout_ms);
2542 syscall.finish();
2543 switch (posix.errno(rc)) {
2544 .SUCCESS => {
2545 if (rc == 0) {
2546 if (b.completions.head != .none) {
2547 // Since there are already completions available in the
2548 // queue, this is neither a timeout nor a case for
2549 // retrying.
2550 return;
2551 }
2552 continue;
2553 }
2554 var prev_index: Io.Operation.OptionalIndex = .none;
2555 var index = b.submissions.head;
2556 for (poll_buffer[0..poll_len]) |poll_entry| {
2557 const storage = &b.storage[index.toIndex()];
2558 const submission = &storage.submission;
2559 const next_index = submission.node.next;
2560 if (poll_entry.revents != 0) {
2561 const result = try operate(t, submission.operation);
2562
2563 switch (prev_index) {
2564 .none => b.submissions.head = next_index,
2565 else => b.storage[prev_index.toIndex()].submission.node.next = next_index,
2566 }
2567 if (next_index == .none) b.submissions.tail = prev_index;
2568
2569 switch (b.completions.tail) {
2570 .none => b.completions.head = index,
2571 else => |tail_index| b.storage[tail_index.toIndex()].completion.node.next = index,
2572 }
2573 storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2574 b.completions.tail = index;
2575 } else prev_index = index;
2576 index = next_index;
2577 }
2578 assert(index == .none);
2579 return;
2580 },
2581 .INTR => continue,
2582 else => break,
2583 }
2584 },
2585 }
2586 {
2587 var tail_index = b.completions.tail;
2588 defer b.completions.tail = tail_index;
2589 var index = b.submissions.head;
2590 errdefer b.submissions.head = index;
2591 while (index != .none) {
2592 const storage = &b.storage[index.toIndex()];
2593 const submission = &storage.submission;
2594 const next_index = submission.node.next;
2595 const result = try operate(t, submission.operation);
2596
2597 switch (tail_index) {
2598 .none => b.completions.head = index,
2599 else => b.storage[tail_index.toIndex()].completion.node.next = index,
2600 }
2601 storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2602 tail_index = index;
2603 index = next_index;
2604 }
2605 b.submissions = .{ .head = .none, .tail = .none };
2524 }2606 }
2607}
2608
2609fn batchAwaitConcurrent(userdata: ?*anyopaque, b: *Io.Batch, timeout: Io.Timeout) Io.Batch.AwaitConcurrentError!void {
2610 const t: *Threaded = @ptrCast(@alignCast(userdata));
2611 if (is_windows) {
2612 const deadline: ?Io.Clock.Timestamp = timeout.toDeadline(ioBasic(t)) catch |err| switch (err) {
2613 error.Unexpected => deadline: {
2614 recoverableOsBugDetected();
2615 break :deadline .{ .raw = .{ .nanoseconds = 0 }, .clock = .awake };
2616 },
2617 error.UnsupportedClock => |e| return e,
2618 };
2619 try batchAwaitWindows(b);
2620 while (b.pending.head != .none and b.completions.head == .none) {
2621 var delay_interval: windows.LARGE_INTEGER = interval: {
2622 const d = deadline orelse break :interval std.math.minInt(windows.LARGE_INTEGER);
2623 break :interval t.deadlineToWindowsInterval(d) catch |err| switch (err) {
2624 error.UnsupportedClock => |e| return e,
2625 error.Unexpected => {
2626 recoverableOsBugDetected();
2627 break :interval -1;
2628 },
2629 };
2630 };
2631 const alertable_syscall = try AlertableSyscall.start();
2632 const delay_rc = windows.ntdll.NtDelayExecution(windows.TRUE, &delay_interval);
2633 alertable_syscall.finish();
2634 switch (delay_rc) {
2635 .SUCCESS, .TIMEOUT => {
2636 // The thread woke due to the timeout. Although spurious
2637 // timeouts are OK, when no deadline is passed we must not
2638 // return `error.Timeout`.
2639 if (timeout != .none and b.completions.head == .none) return error.Timeout;
2640 },
2641 else => {},
2642 }
2643 }
2644 return;
2645 }
2646 if (native_os == .wasi and !builtin.link_libc) @panic("TODO");
2525 var poll_buffer: [poll_buffer_len]posix.pollfd = undefined;2647 var poll_buffer: [poll_buffer_len]posix.pollfd = undefined;
2526 while (submit_head != submit_tail) : (submit_head = submit_head.next(len)) {2648 var poll_storage: struct {
2527 const op = ring[submit_head.index(len)];2649 gpa: std.mem.Allocator,
2528 const operation = &operations[op];2650 b: *Io.Batch,
2529 switch (operation.*) {2651 slice: []posix.pollfd,
2530 .noop => |*o| {2652 len: u32,
2531 _ = o.status.unstarted;2653
2532 o.status = .{ .result = {} };2654 fn add(storage: *@This(), file: Io.File, events: @FieldType(posix.pollfd, "events")) Io.ConcurrentError!void {
2533 submitComplete(ring, &complete_tail, op);2655 const len = storage.len;
2534 },2656 if (len == poll_buffer_len) {
2535 .file_read_streaming => |*o| {2657 const slice: []posix.pollfd = if (storage.b.context) |context|
2536 _ = o.status.unstarted;2658 @as([*]posix.pollfd, @ptrCast(@alignCast(context)))[0..storage.b.storage.len]
2537 if (poll_buffer.len - poll_i == 0) return error.ConcurrencyUnavailable;2659 else allocation: {
2538 poll_buffer[poll_i] = .{2660 const allocation = storage.gpa.alloc(posix.pollfd, storage.b.storage.len) catch
2539 .fd = o.file.handle,2661 return error.ConcurrencyUnavailable;
2540 .events = posix.POLL.IN,2662 storage.b.context = allocation.ptr;
2541 .revents = 0,2663 break :allocation allocation;
2542 };2664 };
2543 map_buffer[poll_i] = @intCast(op);2665 @memcpy(slice[0..poll_buffer_len], storage.slice);
2544 poll_i += 1;2666 }
2545 },2667 storage.slice[len] = .{
2668 .fd = file.handle,
2669 .events = events,
2670 .revents = 0,
2671 };
2672 storage.len = len + 1;
2673 }
2674 } = .{ .gpa = t.allocator, .b = b, .slice = &poll_buffer, .len = 0 };
2675 {
2676 var index = b.submissions.head;
2677 while (index != .none) {
2678 const submission = &b.storage[index.toIndex()].submission;
2679 switch (submission.operation) {
2680 .file_read_streaming => |o| try poll_storage.add(o.file, posix.POLL.IN),
2681 }
2682 index = submission.node.next;
2546 }2683 }
2547 }2684 }
2548 switch (poll_i) {2685 switch (poll_storage.len) {
2549 0 => return,2686 0 => return,
2550 1 => if (timeout == .none) {2687 1 => if (timeout == .none) {
2551 const op = map_buffer[0];2688 const index = b.submissions.head;
2552 try operate(t, &operations[op]);2689 const storage = &b.storage[index.toIndex()];
2553 submitComplete(ring, &complete_tail, op);2690 const result = try operate(t, storage.submission.operation);
2554 poll_i = 0;2691
2692 b.submissions = .{ .head = .none, .tail = .none };
2693
2694 switch (b.completions.tail) {
2695 .none => b.completions.head = index,
2696 else => |tail_index| b.storage[tail_index.toIndex()].completion.node.next = index,
2697 }
2698 storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2699 b.completions.tail = index;
2555 return;2700 return;
2556 },2701 },
2557 else => {},2702 else => {},
2558 }2703 }
2559 const t_io = ioBasic(t);2704 const t_io = ioBasic(t);
2560 const deadline = timeout.toDeadline(t_io) catch return error.UnsupportedClock;2705 const deadline = timeout.toDeadline(t_io) catch return error.UnsupportedClock;
2561 const max_poll_ms = std.math.maxInt(i32);
2562 while (true) {2706 while (true) {
2563 const timeout_ms: i32 = t: {2707 const timeout_ms: i32 = t: {
2564 if (b.user.complete_head != complete_tail) {2708 if (b.completions.head != .none) {
2565 // It is legal to call batchWait with already completed2709 // It is legal to call batchWait with already completed
2566 // operations in the ring. In such case, we need to avoid2710 // operations in the ring. In such case, we need to avoid
2567 // blocking in the poll syscall, but we can still take this2711 // blocking in the poll syscall, but we can still take this
...@@ -2571,15 +2715,16 @@ fn batchWait(userdata: ?*anyopaque, b: *Io.Batch, timeout: Io.Timeout) Io.Batch....@@ -2571,15 +2715,16 @@ fn batchWait(userdata: ?*anyopaque, b: *Io.Batch, timeout: Io.Timeout) Io.Batch.
2571 const d = deadline orelse break :t -1;2715 const d = deadline orelse break :t -1;
2572 const duration = d.durationFromNow(t_io) catch return error.UnsupportedClock;2716 const duration = d.durationFromNow(t_io) catch return error.UnsupportedClock;
2573 if (duration.raw.nanoseconds <= 0) return error.Timeout;2717 if (duration.raw.nanoseconds <= 0) return error.Timeout;
2718 const max_poll_ms = std.math.maxInt(i32);
2574 break :t @intCast(@min(max_poll_ms, duration.raw.toMilliseconds()));2719 break :t @intCast(@min(max_poll_ms, duration.raw.toMilliseconds()));
2575 };2720 };
2576 const syscall = try Syscall.start();2721 const syscall = try Syscall.start();
2577 const rc = posix.system.poll(&poll_buffer, poll_i, timeout_ms);2722 const rc = posix.system.poll(&poll_buffer, poll_storage.len, timeout_ms);
2578 syscall.finish();2723 syscall.finish();
2579 switch (posix.errno(rc)) {2724 switch (posix.errno(rc)) {
2580 .SUCCESS => {2725 .SUCCESS => {
2581 if (rc == 0) {2726 if (rc == 0) {
2582 if (b.user.complete_head != complete_tail) {2727 if (b.completions.head != .none) {
2583 // Since there are already completions available in the2728 // Since there are already completions available in the
2584 // queue, this is neither a timeout nor a case for2729 // queue, this is neither a timeout nor a case for
2585 // retrying.2730 // retrying.
...@@ -2590,18 +2735,30 @@ fn batchWait(userdata: ?*anyopaque, b: *Io.Batch, timeout: Io.Timeout) Io.Batch....@@ -2590,18 +2735,30 @@ fn batchWait(userdata: ?*anyopaque, b: *Io.Batch, timeout: Io.Timeout) Io.Batch.
2590 if (deadline == null) continue;2735 if (deadline == null) continue;
2591 return error.Timeout;2736 return error.Timeout;
2592 }2737 }
2593 while (poll_i != 0) {2738 var prev_index: Io.Operation.OptionalIndex = .none;
2594 poll_i -= 1;2739 var index = b.submissions.head;
2595 const poll_fd = &poll_buffer[poll_i];2740 for (poll_storage.slice[0..poll_storage.len]) |poll_entry| {
2596 const op = map_buffer[poll_i];2741 const submission = &b.storage[index.toIndex()].submission;
2597 if (poll_fd.revents == 0) {2742 const next_index = submission.node.next;
2598 submit_head = submit_head.prev(len);2743 if (poll_entry.revents != 0) {
2599 ring[submit_head.index(len)] = op;2744 const result = try operate(t, submission.operation);
2600 } else {2745
2601 try operate(t, &operations[op]);2746 switch (prev_index) {
2602 submitComplete(ring, &complete_tail, op);2747 .none => b.submissions.head = next_index,
2603 }2748 else => b.storage[prev_index.toIndex()].submission.node.next = next_index,
2749 }
2750 if (next_index == .none) b.submissions.tail = prev_index;
2751
2752 switch (b.completions.tail) {
2753 .none => b.completions.head = index,
2754 else => |tail_index| b.storage[tail_index.toIndex()].completion.node.next = index,
2755 }
2756 b.completions.tail = index;
2757 b.storage[index.toIndex()] = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2758 } else prev_index = index;
2759 index = next_index;
2604 }2760 }
2761 assert(index == .none);
2605 return;2762 return;
2606 },2763 },
2607 .INTR => continue,2764 .INTR => continue,
...@@ -2610,166 +2767,126 @@ fn batchWait(userdata: ?*anyopaque, b: *Io.Batch, timeout: Io.Timeout) Io.Batch....@@ -2610,166 +2767,126 @@ fn batchWait(userdata: ?*anyopaque, b: *Io.Batch, timeout: Io.Timeout) Io.Batch.
2610 }2767 }
2611}2768}
26122769
2770const WindowsBatchPendingOperationContext = extern struct {
2771 file: windows.HANDLE,
2772 iosb: windows.IO_STATUS_BLOCK,
2773
2774 const Erased = [3]usize;
2775
2776 comptime {
2777 assert(@sizeOf(Erased) <= @sizeOf(WindowsBatchPendingOperationContext));
2778 }
2779
2780 fn toErased(context: *WindowsBatchPendingOperationContext) *Erased {
2781 return @ptrCast(context);
2782 }
2783
2784 fn fromErased(erased: *Erased) *WindowsBatchPendingOperationContext {
2785 return @ptrCast(erased);
2786 }
2787};
2788
2613fn batchCancel(userdata: ?*anyopaque, b: *Io.Batch) void {2789fn batchCancel(userdata: ?*anyopaque, b: *Io.Batch) void {
2614 const t: *Threaded = @ptrCast(@alignCast(userdata));2790 const t: *Threaded = @ptrCast(@alignCast(userdata));
2615 const operations = b.operations;2791 {
2616 const len: u31 = @intCast(operations.len);2792 var tail_index = b.unused.tail;
2617 const ring = b.ring[0..len];2793 defer b.unused.tail = tail_index;
2618 var submit_head = b.impl.submit_head;2794 var index = b.submissions.head;
2619 const submit_tail = b.user.submit_tail;2795 errdefer b.submissions.head = index;
2620 b.impl.submit_tail = submit_tail;2796 while (index != .none) {
2621 var complete_tail = b.impl.complete_tail;2797 const next_index = b.storage[index.toIndex()].submission.node.next;
2622 while (submit_head != submit_tail) : (submit_head = submit_head.next(len)) {2798 switch (tail_index) {
2623 const op = ring[submit_head.index(len)];2799 .none => b.unused.head = index,
2624 switch (operations[op]) {2800 else => b.storage[tail_index.toIndex()].unused.next = index,
2625 .noop => |*o| {2801 }
2626 _ = o.status.unstarted;2802 b.storage[index.toIndex()] = .{ .unused = .{ .prev = tail_index, .next = .none } };
2627 o.status = .{ .result = {} };2803 tail_index = index;
2628 submitComplete(ring, &complete_tail, op);2804 index = next_index;
2629 },
2630 .file_read_streaming => |*o| _ = o.status.unstarted,
2631 }2805 }
2806 b.submissions = .{ .head = .none, .tail = .none };
2632 }2807 }
2633 if (is_windows) {2808 if (is_windows) {
2634 // Iterate over pending and issue cancelations, then free the allocation for IO_STATUS_BLOCK2809 var index = b.pending.head;
2635 if (b.impl.reserved) |reserved| {2810 while (index != .none) {
2636 const gpa = t.allocator;2811 const pending = &b.storage[index.toIndex()].pending;
2637 const metadatas_ptr: [*]WinOpMetadata = @ptrCast(@alignCast(reserved));2812 const context: *WindowsBatchPendingOperationContext = .fromErased(&pending.context);
2638 const metadatas = metadatas_ptr[0..b.operations.len];2813 _ = windows.ntdll.NtCancelIoFile(context.file, &context.iosb);
2639 for (metadatas, 0..) |*metadata, op| {2814 index = pending.node.next;
2640 if (!metadata.pending) continue;2815 }
2641 const done = @atomicLoad(windows.NTSTATUS, &metadata.iosb.u.Status, .acquire) != .PENDING;2816 while (b.pending.head != .none) waitForApcOrAlert();
2642 if (done) continue;2817 } else if (b.context) |context| {
2643 switch (operations[op]) {2818 t.allocator.free(@as([*]posix.pollfd, @ptrCast(@alignCast(context)))[0..b.storage.len]);
2644 .noop => unreachable,2819 b.context = null;
2645 .file_read_streaming => |*o| {2820 }
2646 _ = windows.ntdll.NtCancelIoFile(o.file.handle, &metadata.iosb);2821 assert(b.pending.head == .none);
2647 },2822}
2648 }2823
2824fn batchApc(apc_context: ?*anyopaque, iosb: *windows.IO_STATUS_BLOCK, _: windows.ULONG) callconv(.winapi) void {
2825 const b: *Io.Batch = @ptrCast(@alignCast(apc_context));
2826 const context: *WindowsBatchPendingOperationContext = @fieldParentPtr("iosb", iosb);
2827 const erased_context = context.toErased();
2828 const pending: *Io.Operation.Storage.Pending = @fieldParentPtr("context", erased_context);
2829 switch (pending.node.prev) {
2830 .none => b.pending.head = pending.node.next,
2831 else => |prev_index| b.storage[prev_index.toIndex()].pending.node.next = pending.node.next,
2832 }
2833 switch (pending.node.next) {
2834 .none => b.pending.tail = pending.node.prev,
2835 else => |next_index| b.storage[next_index.toIndex()].pending.node.prev = pending.node.prev,
2836 }
2837 const storage: *Io.Operation.Storage = @fieldParentPtr("pending", pending);
2838 const index = storage - b.storage.ptr;
2839 switch (iosb.u.Status) {
2840 .CANCELLED => {
2841 const tail_index = b.unused.tail;
2842 switch (tail_index) {
2843 .none => b.unused.head = .fromIndex(index),
2844 else => b.storage[tail_index.toIndex()].unused.next = .fromIndex(index),
2649 }2845 }
2650 for (metadatas) |*metadata| {2846 storage.* = .{ .unused = .{ .prev = tail_index, .next = .none } };
2651 if (!metadata.pending) continue;2847 b.unused.tail = .fromIndex(index);
2652 while (@atomicLoad(windows.NTSTATUS, &metadata.iosb.u.Status, .acquire) == .PENDING) {2848 },
2653 waitForApcOrAlert();2849 else => {
2654 }2850 switch (b.completions.tail) {
2851 .none => b.completions.head = .fromIndex(index),
2852 else => |tail_index| b.storage[tail_index.toIndex()].completion.node.next = .fromIndex(index),
2655 }2853 }
2656 gpa.free(metadatas);2854 b.completions.tail = .fromIndex(index);
2657 b.impl.reserved = null;2855 const result: Io.Operation.Result = switch (pending.tag) {
2658 }2856 .file_read_streaming => .{ .file_read_streaming = ntReadFileResult(iosb) },
2857 };
2858 storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2859 },
2659 }2860 }
2660 b.impl.submit_head = submit_tail;
2661 b.impl.complete_tail = complete_tail;
2662 b.user.complete_tail = complete_tail;
2663}2861}
26642862
2665const WinOpMetadata = struct {2863fn batchAwaitWindows(b: *Io.Batch) Io.Cancelable!void {
2666 iosb: windows.IO_STATUS_BLOCK,2864 var index = b.submissions.head;
2667 pending: bool,2865 errdefer b.submissions.head = index;
2668};2866 while (index != .none) {
26692867 const storage = &b.storage[index.toIndex()];
2670fn batchWaitWindows(t: *Threaded, b: *Io.Batch, timeout: Io.Timeout) Io.Batch.WaitError!void {2868 const submission = storage.submission;
2671 const operations = b.operations;2869 errdefer storage.* = .{ .submission = submission };
2672 const len: u31 = @intCast(operations.len);2870 storage.* = .{ .pending = .{
2673 const ring = b.ring[0..len];2871 .node = .{ .prev = b.pending.tail, .next = .none },
2674 var submit_head = b.impl.submit_head;2872 .tag = submission.operation,
2675 const submit_tail = b.user.submit_tail;2873 .context = undefined,
2676 b.impl.submit_tail = submit_tail;2874 } };
2677 var complete_tail = b.impl.complete_tail;2875 const context: *WindowsBatchPendingOperationContext = .fromErased(&storage.pending.context);
26782876 switch (submission.operation) {
2679 const metadatas_ptr: [*]WinOpMetadata = if (b.impl.reserved) |reserved| @ptrCast(@alignCast(reserved)) else a: {2877 .file_read_streaming => |o| {
2680 const gpa = t.allocator;2878 context.file = o.file.handle;
2681 const metadatas = gpa.alloc(WinOpMetadata, operations.len) catch return error.ConcurrencyUnavailable;2879 try ntReadFile(o.file.handle, o.data, &batchApc, b, &context.iosb);
2682 b.impl.reserved = metadatas.ptr;
2683 @memset(metadatas, .{ .iosb = undefined, .pending = false });
2684 break :a metadatas.ptr;
2685 };
2686 const metadatas = metadatas_ptr[0..operations.len];
2687
2688 defer {
2689 b.impl.submit_head = submit_head;
2690 b.impl.complete_tail = complete_tail;
2691 b.user.complete_tail = complete_tail;
2692 }
2693
2694 while (submit_head != submit_tail) : (submit_head = submit_head.next(len)) {
2695 const op = ring[submit_head.index(len)];
2696 const operation = &operations[op];
2697 const metadata = &metadatas[op];
2698 metadata.* = .{ .iosb = .{
2699 .u = .{ .Status = .PENDING },
2700 .Information = 0,
2701 }, .pending = false };
2702 switch (operation.*) {
2703 .noop => |*o| {
2704 _ = o.status.unstarted;
2705 o.status = .{ .result = {} };
2706 submitComplete(ring, &complete_tail, op);
2707 },
2708 .file_read_streaming => |*o| {
2709 _ = o.status.unstarted;
2710 try ntReadFile(o.file.handle, o.data, &metadata.iosb);
2711 if (@atomicLoad(windows.NTSTATUS, &metadata.iosb.u.Status, .acquire) == .PENDING) {
2712 o.status = .{ .pending = b };
2713 metadata.pending = true;
2714 } else {
2715 o.status = .{ .result = ntReadFileResult(&metadata.iosb) };
2716 submitComplete(ring, &complete_tail, op);
2717 }
2718 },2880 },
2719 }2881 }
2720 }2882 switch (b.pending.tail) {
27212883 .none => b.pending.head = index,
2722 const deadline: ?Io.Clock.Timestamp = timeout.toDeadline(ioBasic(t)) catch |err| switch (err) {2884 else => |tail_index| b.storage[tail_index.toIndex()].pending.node.next = index,
2723 error.Unexpected => deadline: {
2724 recoverableOsBugDetected();
2725 break :deadline .{ .raw = .{ .nanoseconds = 0 }, .clock = .awake };
2726 },
2727 error.UnsupportedClock => |e| return e,
2728 };
2729
2730 while (true) {
2731 var any_pending = false;
2732 for (metadatas, 0..) |*metadata, op_usize| {
2733 if (!metadata.pending) continue;
2734 any_pending = true;
2735 const op: u31 = @intCast(op_usize);
2736 const done = @atomicLoad(windows.NTSTATUS, &metadata.iosb.u.Status, .acquire) != .PENDING;
2737 switch (operations[op]) {
2738 .noop => unreachable,
2739 .file_read_streaming => |*o| {
2740 assert(o.status.pending == b);
2741 if (!done) continue;
2742 o.status = .{ .result = ntReadFileResult(&metadata.iosb) };
2743 },
2744 }
2745 metadata.pending = false;
2746 submitComplete(ring, &complete_tail, op);
2747 }
2748 if (b.user.complete_head != complete_tail) return;
2749 if (!any_pending) return;
2750 var delay_interval: windows.LARGE_INTEGER = interval: {
2751 const d = deadline orelse break :interval std.math.minInt(windows.LARGE_INTEGER);
2752 break :interval t.deadlineToWindowsInterval(d) catch |err| switch (err) {
2753 error.UnsupportedClock => |e| return e,
2754 error.Unexpected => {
2755 recoverableOsBugDetected();
2756 break :interval -1;
2757 },
2758 };
2759 };
2760 const alertable_syscall = try AlertableSyscall.start();
2761 const delay_rc = windows.ntdll.NtDelayExecution(windows.TRUE, &delay_interval);
2762 alertable_syscall.finish();
2763 switch (delay_rc) {
2764 .SUCCESS, .TIMEOUT => {
2765 // The thread woke due to the timeout. Although spurious
2766 // timeouts are OK, when no deadline is passed we must not
2767 // return `error.Timeout`.
2768 if (timeout != .none) return error.Timeout;
2769 },
2770 else => {},
2771 }2885 }
2886 b.pending.tail = index;
2887 index = submission.node.next;
2772 }2888 }
2889 b.submissions = .{ .head = .none, .tail = .none };
2773}2890}
27742891
2775fn submitComplete(ring: []u32, complete_tail: *Io.Batch.RingIndex, op: u32) void {2892fn submitComplete(ring: []u32, complete_tail: *Io.Batch.RingIndex, op: u32) void {
...@@ -8701,7 +8818,7 @@ fn fileReadStreamingWindows(file: File, data: []const []u8) File.ReadStreamingEr...@@ -8701,7 +8818,7 @@ fn fileReadStreamingWindows(file: File, data: []const []u8) File.ReadStreamingEr
8701 .u = .{ .Status = .PENDING },8818 .u = .{ .Status = .PENDING },
8702 .Information = 0,8819 .Information = 0,
8703 };8820 };
8704 try ntReadFile(file.handle, data, &io_status_block);8821 try ntReadFile(file.handle, data, &noopApc, null, &io_status_block);
8705 while (@atomicLoad(windows.NTSTATUS, &io_status_block.u.Status, .acquire) == .PENDING) {8822 while (@atomicLoad(windows.NTSTATUS, &io_status_block.u.Status, .acquire) == .PENDING) {
8706 // Once we get here we must not return from the function until the8823 // Once we get here we must not return from the function until the
8707 // operation completes, thereby releasing reference to io_status_block.8824 // operation completes, thereby releasing reference to io_status_block.
...@@ -8736,12 +8853,20 @@ fn ntReadFileResult(io_status_block: *const windows.IO_STATUS_BLOCK) !usize {...@@ -8736,12 +8853,20 @@ fn ntReadFileResult(io_status_block: *const windows.IO_STATUS_BLOCK) !usize {
8736 }8853 }
8737}8854}
87388855
8739fn ntReadFile(handle: windows.HANDLE, data: []const []u8, iosb: *windows.IO_STATUS_BLOCK) Io.Cancelable!void {8856fn ntReadFile(
8857 handle: windows.HANDLE,
8858 data: []const []u8,
8859 apcRoutine: ?*const windows.IO_APC_ROUTINE,
8860 apc_context: ?*anyopaque,
8861 iosb: *windows.IO_STATUS_BLOCK,
8862) Io.Cancelable!void {
8740 var index: usize = 0;8863 var index: usize = 0;
8741 while (index < data.len and data[index].len == 0) index += 1;8864 while (index < data.len and data[index].len == 0) index += 1;
8742 if (index == data.len) {8865 if (index == data.len) {
8743 iosb.u.Status = .SUCCESS;8866 iosb.* = .{ .u = .{ .Status = .SUCCESS }, .Information = 0 };
8744 iosb.Information = 0;8867 if (apcRoutine) |routine| if (routine != &noopApc) {
8868 _ = windows.ntdll.NtQueueApcThread(windows.current_process, routine, apc_context, iosb, null);
8869 };
8745 return;8870 return;
8746 }8871 }
8747 const buffer = data[index];8872 const buffer = data[index];
...@@ -8750,8 +8875,8 @@ fn ntReadFile(handle: windows.HANDLE, data: []const []u8, iosb: *windows.IO_STAT...@@ -8750,8 +8875,8 @@ fn ntReadFile(handle: windows.HANDLE, data: []const []u8, iosb: *windows.IO_STAT
8750 while (true) switch (windows.ntdll.NtReadFile(8875 while (true) switch (windows.ntdll.NtReadFile(
8751 handle,8876 handle,
8752 null, // event8877 null, // event
8753 noopApc, // apc callback8878 apcRoutine,
8754 null, // apc context8879 apc_context,
8755 iosb,8880 iosb,
8756 buffer.ptr,8881 buffer.ptr,
8757 @min(std.math.maxInt(u32), buffer.len),8882 @min(std.math.maxInt(u32), buffer.len),