| ... | @@ -2662,7 +2662,7 @@ fn batchAwaitConcurrent(userdata: ?*anyopaque, b: *Io.Batch, timeout: Io.Timeout | ... | @@ -2662,7 +2662,7 @@ fn batchAwaitConcurrent(userdata: ?*anyopaque, b: *Io.Batch, timeout: Io.Timeout |
| 2662 | while (b.pending.head != .none and b.completions.head == .none) { | 2662 | while (b.pending.head != .none and b.completions.head == .none) { |
| 2663 | var delay_interval: windows.LARGE_INTEGER = interval: { | 2663 | var delay_interval: windows.LARGE_INTEGER = interval: { |
| 2664 | const d = deadline orelse break :interval std.math.minInt(windows.LARGE_INTEGER); | 2664 | const d = deadline orelse break :interval std.math.minInt(windows.LARGE_INTEGER); |
| 2665 | break :interval t.deadlineToWindowsInterval(d); | 2665 | break :interval timeoutToWindowsInterval(.{ .deadline = d }).?; |
| 2666 | }; | 2666 | }; |
| 2667 | const alertable_syscall = try AlertableSyscall.start(); | 2667 | const alertable_syscall = try AlertableSyscall.start(); |
| 2668 | const delay_rc = windows.ntdll.NtDelayExecution(windows.TRUE, &delay_interval); | 2668 | const delay_rc = windows.ntdll.NtDelayExecution(windows.TRUE, &delay_interval); |
| ... | @@ -4339,7 +4339,10 @@ fn dirCreateFileWindows( | ... | @@ -4339,7 +4339,10 @@ fn dirCreateFileWindows( |
| 4339 | // kernel bug with retry attempts. | 4339 | // kernel bug with retry attempts. |
| 4340 | syscall.finish(); | 4340 | syscall.finish(); |
| 4341 | if (max_attempts - attempt == 0) return error.FileBusy; | 4341 | if (max_attempts - attempt == 0) return error.FileBusy; |
| 4342 | try parking_sleep.windowsRetrySleep((@as(u32, 1) << attempt) >> 1); | 4342 | try parking_sleep.sleep(.{ .duration = .{ |
| | 4343 | .raw = .fromMilliseconds((@as(u32, 1) << attempt) >> 1), |
| | 4344 | .clock = .awake, |
| | 4345 | } }); |
| 4343 | attempt += 1; | 4346 | attempt += 1; |
| 4344 | syscall = try .start(); | 4347 | syscall = try .start(); |
| 4345 | continue; | 4348 | continue; |
| ... | @@ -4352,7 +4355,10 @@ fn dirCreateFileWindows( | ... | @@ -4352,7 +4355,10 @@ fn dirCreateFileWindows( |
| 4352 | // fixed by sleeping and retrying until the error goes away. | 4355 | // fixed by sleeping and retrying until the error goes away. |
| 4353 | syscall.finish(); | 4356 | syscall.finish(); |
| 4354 | if (max_attempts - attempt == 0) return error.FileBusy; | 4357 | if (max_attempts - attempt == 0) return error.FileBusy; |
| 4355 | try parking_sleep.windowsRetrySleep((@as(u32, 1) << attempt) >> 1); | 4358 | try parking_sleep.sleep(.{ .duration = .{ |
| | 4359 | .raw = .fromMilliseconds((@as(u32, 1) << attempt) >> 1), |
| | 4360 | .clock = .awake, |
| | 4361 | } }); |
| 4356 | attempt += 1; | 4362 | attempt += 1; |
| 4357 | syscall = try .start(); | 4363 | syscall = try .start(); |
| 4358 | continue; | 4364 | continue; |
| ... | @@ -7382,7 +7388,10 @@ fn dirReadLinkWindows(dir: Dir, sub_path: []const u8, buffer: []u8) Dir.ReadLink | ... | @@ -7382,7 +7388,10 @@ fn dirReadLinkWindows(dir: Dir, sub_path: []const u8, buffer: []u8) Dir.ReadLink |
| 7382 | // kernel bug with retry attempts. | 7388 | // kernel bug with retry attempts. |
| 7383 | syscall.finish(); | 7389 | syscall.finish(); |
| 7384 | if (max_attempts - attempt == 0) return error.FileBusy; | 7390 | if (max_attempts - attempt == 0) return error.FileBusy; |
| 7385 | try parking_sleep.windowsRetrySleep((@as(u32, 1) << attempt) >> 1); | 7391 | try parking_sleep.sleep(.{ .duration = .{ |
| | 7392 | .raw = .fromMilliseconds((@as(u32, 1) << attempt) >> 1), |
| | 7393 | .clock = .awake, |
| | 7394 | } }); |
| 7386 | attempt += 1; | 7395 | attempt += 1; |
| 7387 | syscall = try .start(); | 7396 | syscall = try .start(); |
| 7388 | continue; | 7397 | continue; |
| ... | @@ -7395,7 +7404,10 @@ fn dirReadLinkWindows(dir: Dir, sub_path: []const u8, buffer: []u8) Dir.ReadLink | ... | @@ -7395,7 +7404,10 @@ fn dirReadLinkWindows(dir: Dir, sub_path: []const u8, buffer: []u8) Dir.ReadLink |
| 7395 | // fixed by sleeping and retrying until the error goes away. | 7404 | // fixed by sleeping and retrying until the error goes away. |
| 7396 | syscall.finish(); | 7405 | syscall.finish(); |
| 7397 | if (max_attempts - attempt == 0) return error.FileBusy; | 7406 | if (max_attempts - attempt == 0) return error.FileBusy; |
| 7398 | try parking_sleep.windowsRetrySleep((@as(u32, 1) << attempt) >> 1); | 7407 | try parking_sleep.sleep(.{ .duration = .{ |
| | 7408 | .raw = .fromMilliseconds((@as(u32, 1) << attempt) >> 1), |
| | 7409 | .clock = .awake, |
| | 7410 | } }); |
| 7399 | attempt += 1; | 7411 | attempt += 1; |
| 7400 | syscall = try .start(); | 7412 | syscall = try .start(); |
| 7401 | continue; | 7413 | continue; |
| ... | @@ -10956,7 +10968,7 @@ fn nowWasi(clock: Io.Clock) Io.Timestamp { | ... | @@ -10956,7 +10968,7 @@ fn nowWasi(clock: Io.Clock) Io.Timestamp { |
| 10956 | fn sleep(userdata: ?*anyopaque, timeout: Io.Timeout) Io.Cancelable!void { | 10968 | fn sleep(userdata: ?*anyopaque, timeout: Io.Timeout) Io.Cancelable!void { |
| 10957 | const t: *Threaded = @ptrCast(@alignCast(userdata)); | 10969 | const t: *Threaded = @ptrCast(@alignCast(userdata)); |
| 10958 | if (timeout == .none) return; | 10970 | if (timeout == .none) return; |
| 10959 | if (use_parking_sleep) return parking_sleep.sleep(timeout.toTimestamp(ioBasic(t))); | 10971 | if (use_parking_sleep) return parking_sleep.sleep(timeout); |
| 10960 | if (native_os == .wasi) return sleepWasi(t, timeout); | 10972 | if (native_os == .wasi) return sleepWasi(t, timeout); |
| 10961 | if (@TypeOf(posix.system.clock_nanosleep) != void) return sleepPosix(timeout); | 10973 | if (@TypeOf(posix.system.clock_nanosleep) != void) return sleepPosix(timeout); |
| 10962 | return sleepNanosleep(t, timeout); | 10974 | return sleepNanosleep(t, timeout); |
| ... | @@ -14363,7 +14375,7 @@ const Wsa = struct { | ... | @@ -14363,7 +14375,7 @@ const Wsa = struct { |
| 14363 | | 14375 | |
| 14364 | fn initializeWsa(t: *Threaded) error{ NetworkDown, Canceled }!void { | 14376 | fn initializeWsa(t: *Threaded) error{ NetworkDown, Canceled }!void { |
| 14365 | const wsa = &t.wsa; | 14377 | const wsa = &t.wsa; |
| 14366 | try mutexLock(&wsa.mutex); | 14378 | mutexLock(&wsa.mutex); |
| 14367 | defer mutexUnlock(&wsa.mutex); | 14379 | defer mutexUnlock(&wsa.mutex); |
| 14368 | switch (wsa.status) { | 14380 | switch (wsa.status) { |
| 14369 | .uninitialized => { | 14381 | .uninitialized => { |
| ... | @@ -16943,7 +16955,7 @@ const parking_futex = struct { | ... | @@ -16943,7 +16955,7 @@ const parking_futex = struct { |
| 16943 | /// avoid a race. | 16955 | /// avoid a race. |
| 16944 | num_waiters: std.atomic.Value(u32), | 16956 | num_waiters: std.atomic.Value(u32), |
| 16945 | /// Protects `waiters`. | 16957 | /// Protects `waiters`. |
| 16946 | mutex: Io.Mutex, | 16958 | mutex: ParkingMutex, |
| 16947 | waiters: std.DoublyLinkedList, | 16959 | waiters: std.DoublyLinkedList, |
| 16948 | | 16960 | |
| 16949 | /// Prevent false sharing between buckets. | 16961 | /// Prevent false sharing between buckets. |
| ... | @@ -17007,8 +17019,8 @@ const parking_futex = struct { | ... | @@ -17007,8 +17019,8 @@ const parking_futex = struct { |
| 17007 | var status_buf: std.atomic.Value(Thread.Status) = undefined; | 17019 | var status_buf: std.atomic.Value(Thread.Status) = undefined; |
| 17008 | | 17020 | |
| 17009 | { | 17021 | { |
| 17010 | mutexLock(&bucket.mutex); | 17022 | bucket.mutex.lock(); |
| 17011 | defer mutexUnlock(&bucket.mutex); | 17023 | defer bucket.mutex.unlock(); |
| 17012 | | 17024 | |
| 17013 | _ = bucket.num_waiters.fetchAdd(1, .acquire); | 17025 | _ = bucket.num_waiters.fetchAdd(1, .acquire); |
| 17014 | | 17026 | |
| ... | @@ -17059,7 +17071,7 @@ const parking_futex = struct { | ... | @@ -17059,7 +17071,7 @@ const parking_futex = struct { |
| 17059 | bucket.waiters.append(&waiter.node); | 17071 | bucket.waiters.append(&waiter.node); |
| 17060 | } | 17072 | } |
| 17061 | | 17073 | |
| 17062 | if (park(timeout, ptr)) { | 17074 | if (park(timeout, ptr, waiter.thread_status)) { |
| 17063 | // We were unparked by either `wake` or cancelation, so our current status is either | 17075 | // We were unparked by either `wake` or cancelation, so our current status is either |
| 17064 | // `.none` or `.canceling`. In either case, they've already removed `waiter` from | 17076 | // `.none` or `.canceling`. In either case, they've already removed `waiter` from |
| 17065 | // `bucket`, so we have nothing more to do! | 17077 | // `bucket`, so we have nothing more to do! |
| ... | @@ -17084,7 +17096,7 @@ const parking_futex = struct { | ... | @@ -17084,7 +17096,7 @@ const parking_futex = struct { |
| 17084 | // to unpark us. Whoever did that will remove us from `bucket`. Wait for | 17096 | // to unpark us. Whoever did that will remove us from `bucket`. Wait for |
| 17085 | // that (and drop the unpark request in doing so). | 17097 | // that (and drop the unpark request in doing so). |
| 17086 | // New status is `.none` or `.canceling` respectively. | 17098 | // New status is `.none` or `.canceling` respectively. |
| 17087 | park(.none, ptr) catch |e| switch (e) { | 17099 | park(.none, ptr, waiter.thread_status) catch |e| switch (e) { |
| 17088 | error.Timeout => unreachable, | 17100 | error.Timeout => unreachable, |
| 17089 | }; | 17101 | }; |
| 17090 | }, | 17102 | }, |
| ... | @@ -17114,8 +17126,8 @@ const parking_futex = struct { | ... | @@ -17114,8 +17126,8 @@ const parking_futex = struct { |
| 17114 | // of the critical section. This forms a singly-linked list of waiters using `Waiter.node.next`. | 17126 | // of the critical section. This forms a singly-linked list of waiters using `Waiter.node.next`. |
| 17115 | var waking_head: ?*std.DoublyLinkedList.Node = null; | 17127 | var waking_head: ?*std.DoublyLinkedList.Node = null; |
| 17116 | { | 17128 | { |
| 17117 | mutexLock(&bucket.mutex); | 17129 | bucket.mutex.lock(); |
| 17118 | defer mutexUnlock(&bucket.mutex); | 17130 | defer bucket.mutex.unlock(); |
| 17119 | | 17131 | |
| 17120 | var num_removed: u32 = 0; | 17132 | var num_removed: u32 = 0; |
| 17121 | var it = bucket.waiters.first; | 17133 | var it = bucket.waiters.first; |
| ... | @@ -17171,8 +17183,8 @@ const parking_futex = struct { | ... | @@ -17171,8 +17183,8 @@ const parking_futex = struct { |
| 17171 | | 17183 | |
| 17172 | fn removeCanceledWaiter(waiter: *Waiter) void { | 17184 | fn removeCanceledWaiter(waiter: *Waiter) void { |
| 17173 | const bucket = bucketForAddress(waiter.address); | 17185 | const bucket = bucketForAddress(waiter.address); |
| 17174 | mutexLock(&bucket.mutex); | 17186 | bucket.mutex.lock(); |
| 17175 | defer mutexUnlock(&bucket.mutex); | 17187 | defer bucket.mutex.unlock(); |
| 17176 | bucket.waiters.remove(&waiter.node); | 17188 | bucket.waiters.remove(&waiter.node); |
| 17177 | assert(bucket.num_waiters.fetchSub(1, .monotonic) > 0); | 17189 | assert(bucket.num_waiters.fetchSub(1, .monotonic) > 0); |
| 17178 | } | 17190 | } |
| ... | @@ -17206,7 +17218,7 @@ const parking_sleep = struct { | ... | @@ -17206,7 +17218,7 @@ const parking_sleep = struct { |
| 17206 | .blocked_canceling => unreachable, | 17218 | .blocked_canceling => unreachable, |
| 17207 | } | 17219 | } |
| 17208 | } | 17220 | } |
| 17209 | if (park(timeout, null)) { | 17221 | if (park(timeout, null, &thread.status)) { |
| 17210 | // The only reason this could possibly happen is cancelation. | 17222 | // The only reason this could possibly happen is cancelation. |
| 17211 | const old_status = thread.status.load(.monotonic); | 17223 | const old_status = thread.status.load(.monotonic); |
| 17212 | assert(old_status.cancelation == .canceling); | 17224 | assert(old_status.cancelation == .canceling); |
| ... | @@ -17229,7 +17241,7 @@ const parking_sleep = struct { | ... | @@ -17229,7 +17241,7 @@ const parking_sleep = struct { |
| 17229 | // us for a cancelation. Whoever did that will have called `unpark`, so | 17241 | // us for a cancelation. Whoever did that will have called `unpark`, so |
| 17230 | // drop that unpark request by waiting for it. | 17242 | // drop that unpark request by waiting for it. |
| 17231 | // Status is still `.canceling`. | 17243 | // Status is still `.canceling`. |
| 17232 | park(.none, null) catch |e| switch (e) { | 17244 | park(.none, null, &thread.status) catch |e| switch (e) { |
| 17233 | error.Timeout => unreachable, | 17245 | error.Timeout => unreachable, |
| 17234 | }; | 17246 | }; |
| 17235 | return; | 17247 | return; |
| ... | @@ -17245,32 +17257,183 @@ const parking_sleep = struct { | ... | @@ -17245,32 +17257,183 @@ const parking_sleep = struct { |
| 17245 | } | 17257 | } |
| 17246 | } | 17258 | } |
| 17247 | // Uncancelable sleep; we expect not to be manually unparked. | 17259 | // Uncancelable sleep; we expect not to be manually unparked. |
| 17248 | if (park(timeout, null)) { | 17260 | var dummy_status: std.atomic.Value(Thread.Status) = .init(.{ .cancelation = .parked, .awaitable = .null }); |
| | 17261 | if (park(timeout, null, &dummy_status)) { |
| 17249 | unreachable; // unexpected unpark | 17262 | unreachable; // unexpected unpark |
| 17250 | } else |err| switch (err) { | 17263 | } else |err| switch (err) { |
| 17251 | error.Timeout => return, | 17264 | error.Timeout => return, |
| 17252 | } | 17265 | } |
| 17253 | } | 17266 | } |
| 17254 | }; | 17267 | }; |
| | 17268 | const ParkingMutex = struct { |
| | 17269 | state: std.atomic.Value(State), |
| 17255 | | 17270 | |
| 17256 | /// `addr_hint` has no semantic effect, but may allow the OS to optimize this operation. | 17271 | const init: ParkingMutex = .{ .state = .init(.unlocked) }; |
| 17257 | fn park(timeout: Io.Timeout, addr_hint: ?*const anyopaque) error{Timeout}!void { | 17272 | |
| | 17273 | comptime { |
| | 17274 | assert(use_parking_futex); |
| | 17275 | } |
| | 17276 | |
| | 17277 | const State = enum(usize) { |
| | 17278 | unlocked = 1, |
| | 17279 | /// This value is intentionally 0 so that `waiter` returns `null`. |
| | 17280 | locked_once = 0, |
| | 17281 | /// Contended; value is a `*Waiter`. |
| | 17282 | _, |
| | 17283 | /// Returns the head of the waiter list. Illegal to call if `s == .unlocked`. |
| | 17284 | fn waiter(s: State) ?*Waiter { |
| | 17285 | return @ptrFromInt(@intFromEnum(s)); |
| | 17286 | } |
| | 17287 | /// Returns a locked state where `w` is contending the lock. |
| | 17288 | /// If `w` is `null`, returns `.locked_once`. |
| | 17289 | fn fromWaiter(w: ?*Waiter) State { |
| | 17290 | return @enumFromInt(@intFromPtr(w)); |
| | 17291 | } |
| | 17292 | }; |
| | 17293 | const Waiter = struct { |
| | 17294 | status: std.atomic.Value(Thread.Status), |
| | 17295 | /// Never modified once the `Waiter` is in the linked list. |
| | 17296 | next: ?*Waiter, |
| | 17297 | /// Never modified once the `Waiter` is in the linked list. |
| | 17298 | tid: std.Thread.Id, |
| | 17299 | }; |
| | 17300 | fn lock(m: *ParkingMutex) void { |
| | 17301 | state: switch (State.unlocked) { // assume 'unlocked' to optimize for uncontended case |
| | 17302 | .unlocked => continue :state m.state.cmpxchgWeak( |
| | 17303 | .unlocked, |
| | 17304 | .locked_once, |
| | 17305 | .acquire, // acquire lock |
| | 17306 | .monotonic, |
| | 17307 | ) orelse { |
| | 17308 | @branchHint(.likely); |
| | 17309 | return; |
| | 17310 | }, |
| | 17311 | |
| | 17312 | .locked_once, _ => |last_state| { |
| | 17313 | const old_waiter = last_state.waiter(); |
| | 17314 | const self_tid = if (Thread.current) |t| t.id else std.Thread.getCurrentId(); |
| | 17315 | var waiter: Waiter = .{ |
| | 17316 | .next = old_waiter, |
| | 17317 | .status = .init(.{ .cancelation = .parked, .awaitable = .null }), |
| | 17318 | .tid = self_tid, |
| | 17319 | }; |
| | 17320 | if (m.state.cmpxchgWeak( |
| | 17321 | .fromWaiter(old_waiter), |
| | 17322 | .fromWaiter(&waiter), |
| | 17323 | .release, // release `waiter` |
| | 17324 | .monotonic, |
| | 17325 | )) |new_state| { |
| | 17326 | continue :state new_state; |
| | 17327 | } |
| | 17328 | // We're now in the list of waiters---park until we're given the lock. |
| | 17329 | park(.none, m, &waiter.status) catch |err| switch (err) { |
| | 17330 | error.Timeout => unreachable, |
| | 17331 | }; |
| | 17332 | // We now hold the lock. |
| | 17333 | assert(waiter.status.load(.monotonic).cancelation == .none); |
| | 17334 | return; |
| | 17335 | }, |
| | 17336 | } |
| | 17337 | } |
| | 17338 | fn unlock(m: *ParkingMutex) void { |
| | 17339 | state: switch (State.locked_once) { // assume 'locked_once' to optimize for uncontended case |
| | 17340 | .unlocked => unreachable, // we hold the lock |
| | 17341 | |
| | 17342 | .locked_once => continue :state m.state.cmpxchgWeak( |
| | 17343 | .locked_once, |
| | 17344 | .unlocked, |
| | 17345 | .release, // release lock |
| | 17346 | .acquire, // acquire any `Waiter` memory |
| | 17347 | ) orelse { |
| | 17348 | @branchHint(.likely); |
| | 17349 | return; |
| | 17350 | }, |
| | 17351 | |
| | 17352 | _ => |last_state| { |
| | 17353 | // The logic here does not have ABA problems, and does some accesses non-atomically, |
| | 17354 | // because `Waiter.next` is owned by the lock holder (that's us!) once the waiter is |
| | 17355 | // in the linked list, up until we set `Waiter.status` to `.none`. |
| | 17356 | |
| | 17357 | // Run through the waiter list to the end to ensure fairness. This is obviously not |
| | 17358 | // ideal, but it shouldn't be a big deal in practice provided the critical section |
| | 17359 | // is fairly small (so we won't get too many threads contending the mutex at once). |
| | 17360 | // There's a *chance* we could get away with a LIFO queue for our use case, but I |
| | 17361 | // don't wanna risk that. |
| | 17362 | var parent: ?*Waiter = null; |
| | 17363 | var waiter: *Waiter = last_state.waiter().?; |
| | 17364 | while (waiter.next) |next| { |
| | 17365 | parent = waiter; |
| | 17366 | waiter = next; |
| | 17367 | } |
| | 17368 | // `waiter` is next in line for the lock. Remove them from the list. |
| | 17369 | if (parent) |p| { |
| | 17370 | assert(p.next == waiter); |
| | 17371 | p.next = null; |
| | 17372 | } else { |
| | 17373 | // We're waking the last waiter, so clear the list head. |
| | 17374 | if (m.state.cmpxchgWeak( |
| | 17375 | .fromWaiter(last_state.waiter().?), |
| | 17376 | .locked_once, |
| | 17377 | .acquire, |
| | 17378 | .acquire, // acquire any new `Waiter` memory |
| | 17379 | )) |new_state| { |
| | 17380 | continue :state new_state; |
| | 17381 | } |
| | 17382 | } |
| | 17383 | // Now we're ready to actually hand the lock over to them. |
| | 17384 | const tid = waiter.tid; // load this before the store below potentially invalidates `waiter` |
| | 17385 | waiter.status.store(.{ .cancelation = .none, .awaitable = .null }, .release); // release lock |
| | 17386 | unpark(&.{tid}, m); |
| | 17387 | return; |
| | 17388 | }, |
| | 17389 | } |
| | 17390 | } |
| | 17391 | }; |
| | 17392 | |
| | 17393 | fn timeoutToWindowsInterval(timeout: Io.Timeout) ?windows.LARGE_INTEGER { |
| | 17394 | // ntdll only supports two combinations: |
| | 17395 | // * real-time (`.real`) sleeps with absolute deadlines |
| | 17396 | // * monotonic (`.awake`/`.boot`) sleeps with relative durations |
| | 17397 | const clock = switch (timeout) { |
| | 17398 | .none => return null, |
| | 17399 | .duration => |d| d.clock, |
| | 17400 | .deadline => |d| d.clock, |
| | 17401 | }; |
| | 17402 | switch (clock) { |
| | 17403 | .cpu_process, .cpu_thread => unreachable, // cannot sleep for CPU time |
| | 17404 | .real => { |
| | 17405 | const deadline = switch (timeout) { |
| | 17406 | .none => unreachable, |
| | 17407 | .duration => |d| nowWindows(clock).addDuration(d.raw), |
| | 17408 | .deadline => |d| d.raw, |
| | 17409 | }; |
| | 17410 | return @intCast(@max(@divTrunc(deadline.nanoseconds, 100), 0)); |
| | 17411 | }, |
| | 17412 | .awake, .boot => { |
| | 17413 | const duration = switch (timeout) { |
| | 17414 | .none => unreachable, |
| | 17415 | .duration => |d| d.raw, |
| | 17416 | .deadline => |d| nowWindows(clock).durationTo(d.raw), |
| | 17417 | }; |
| | 17418 | return @intCast(@min(@divTrunc(-duration.nanoseconds, 100), -1)); |
| | 17419 | }, |
| | 17420 | } |
| | 17421 | } |
| | 17422 | |
| | 17423 | fn park( |
| | 17424 | timeout: Io.Timeout, |
| | 17425 | /// This value has no semantic effect, but may allow the OS to optimize the operation. |
| | 17426 | addr_hint: ?*const anyopaque, |
| | 17427 | /// The API on NetBSD and Illumos sucks and can unpark spuriously (well, it *can't*, but signals |
| | 17428 | /// cause an indistinguishable unblock, and libpthread really likes to leave unparks pending). |
| | 17429 | /// As such, on these targets only, this `status` is checked to determine if an unpark is real. |
| | 17430 | /// no way to differentiate |
| | 17431 | status: *std.atomic.Value(Thread.Status), |
| | 17432 | ) error{Timeout}!void { |
| 17258 | comptime assert(use_parking_futex or use_parking_sleep); | 17433 | comptime assert(use_parking_futex or use_parking_sleep); |
| 17259 | switch (native_os) { | 17434 | switch (native_os) { |
| 17260 | .windows => { | 17435 | .windows => { |
| 17261 | var timeout_buf: windows.LARGE_INTEGER = undefined; | 17436 | const raw_timeout = timeoutToWindowsInterval(timeout); |
| 17262 | const raw_timeout: ?*windows.LARGE_INTEGER = timeout: switch (timeout) { | | |
| 17263 | .none => null, | | |
| 17264 | .deadline => |timestamp| continue :timeout .{ .duration = .{ | | |
| 17265 | .clock = timestamp.clock, | | |
| 17266 | .raw = (nowWindows(timestamp.clock) catch unreachable).durationTo(timestamp.raw), | | |
| 17267 | } }, | | |
| 17268 | .duration => |duration| { | | |
| 17269 | _ = duration.clock; // Windows only supports monotonic | | |
| 17270 | timeout_buf = @intCast(@divTrunc(-duration.raw.nanoseconds, 100)); | | |
| 17271 | break :timeout &timeout_buf; | | |
| 17272 | }, | | |
| 17273 | }; | | |
| 17274 | // `RtlWaitOnAddress` passes the futex address in as the first argument to this call, | 17437 | // `RtlWaitOnAddress` passes the futex address in as the first argument to this call, |
| 17275 | // but it's unclear what that actually does, especially since `NtAlertThreadByThreadId` | 17438 | // but it's unclear what that actually does, especially since `NtAlertThreadByThreadId` |
| 17276 | // does *not* accept the address so the kernel can't really be using it as a hint. An | 17439 | // does *not* accept the address so the kernel can't really be using it as a hint. An |
| ... | @@ -17284,7 +17447,10 @@ fn park(timeout: Io.Timeout, addr_hint: ?*const anyopaque) error{Timeout}!void { | ... | @@ -17284,7 +17447,10 @@ fn park(timeout: Io.Timeout, addr_hint: ?*const anyopaque) error{Timeout}!void { |
| 17284 | // this parameter). However, to err on the side of caution, let's match the behavior of | 17447 | // this parameter). However, to err on the side of caution, let's match the behavior of |
| 17285 | // `RtlWaitOnAddress` and pass the pointer, in case the kernel ever does something | 17448 | // `RtlWaitOnAddress` and pass the pointer, in case the kernel ever does something |
| 17286 | // stupid such as trying to dereference it. | 17449 | // stupid such as trying to dereference it. |
| 17287 | switch (windows.ntdll.NtWaitForAlertByThreadId(addr_hint, raw_timeout)) { | 17450 | switch (windows.ntdll.NtWaitForAlertByThreadId( |
| | 17451 | addr_hint, |
| | 17452 | if (raw_timeout) |*t| t else null, |
| | 17453 | )) { |
| 17288 | .ALERTED => return, | 17454 | .ALERTED => return, |
| 17289 | .TIMEOUT => return error.Timeout, | 17455 | .TIMEOUT => return error.Timeout, |
| 17290 | else => unreachable, | 17456 | else => unreachable, |
| ... | @@ -17303,19 +17469,23 @@ fn park(timeout: Io.Timeout, addr_hint: ?*const anyopaque) error{Timeout}!void { | ... | @@ -17303,19 +17469,23 @@ fn park(timeout: Io.Timeout, addr_hint: ?*const anyopaque) error{Timeout}!void { |
| 17303 | break :timeout .{ &ts_buf, false, duration.clock == .real }; | 17469 | break :timeout .{ &ts_buf, false, duration.clock == .real }; |
| 17304 | }, | 17470 | }, |
| 17305 | }; | 17471 | }; |
| 17306 | switch (posix.errno(std.c._lwp_park( | 17472 | // It's okay to pass the same timeout in a loop. If it's a duration, the OS actually |
| 17307 | if (clock_real) .REALTIME else .MONOTONIC, | 17473 | // writes the remaining time into the buffer when the syscall returns. |
| 17308 | .{ .ABSTIME = abstime }, | 17474 | while (status.load(.monotonic).cancelation == .parked) { |
| 17309 | ts, | 17475 | switch (posix.errno(std.c._lwp_park( |
| 17310 | 0, | 17476 | if (clock_real) .REALTIME else .MONOTONIC, |
| 17311 | addr_hint, | 17477 | .{ .ABSTIME = abstime }, |
| 17312 | null, | 17478 | ts, |
| 17313 | ))) { | 17479 | 0, |
| 17314 | .SUCCESS, .ALREADY, .INTR => return, | 17480 | addr_hint, |
| 17315 | .TIMEDOUT => return error.Timeout, | 17481 | null, |
| 17316 | .INVAL => unreachable, | 17482 | ))) { |
| 17317 | .SRCH => unreachable, | 17483 | .SUCCESS, .ALREADY, .INTR => {}, |
| 17318 | else => unreachable, | 17484 | .TIMEDOUT => return error.Timeout, |
| | 17485 | .INVAL => unreachable, |
| | 17486 | .SRCH => unreachable, |
| | 17487 | else => unreachable, |
| | 17488 | } |
| 17319 | } | 17489 | } |
| 17320 | }, | 17490 | }, |
| 17321 | .illumos => @panic("TODO: illumos lwp_park"), | 17491 | .illumos => @panic("TODO: illumos lwp_park"), |
| ... | @@ -17323,24 +17493,8 @@ fn park(timeout: Io.Timeout, addr_hint: ?*const anyopaque) error{Timeout}!void { | ... | @@ -17323,24 +17493,8 @@ fn park(timeout: Io.Timeout, addr_hint: ?*const anyopaque) error{Timeout}!void { |
| 17323 | } | 17493 | } |
| 17324 | } | 17494 | } |
| 17325 | | 17495 | |
| 17326 | fn deadlineToWindowsInterval(t: *Io.Threaded, deadline: Io.Clock.Timestamp) windows.LARGE_INTEGER { | | |
| 17327 | // ntdll only supports two combinations: | | |
| 17328 | // * real-time (`.real`) sleeps with absolute deadlines | | |
| 17329 | // * monotonic (`.awake`/`.boot`) sleeps with relative durations | | |
| 17330 | switch (deadline.clock) { | | |
| 17331 | .cpu_process, .cpu_thread => return 0, | | |
| 17332 | .real => { | | |
| 17333 | return @intCast(@max(@divTrunc(deadline.raw.nanoseconds, 100), 0)); | | |
| 17334 | }, | | |
| 17335 | .awake, .boot => { | | |
| 17336 | const duration = deadline.durationFromNow(ioBasic(t)); | | |
| 17337 | return @intCast(@min(@divTrunc(-duration.raw.nanoseconds, 100), -1)); | | |
| 17338 | }, | | |
| 17339 | } | | |
| 17340 | } | | |
| 17341 | | | |
| 17342 | const UnparkTid = switch (native_os) { | 17496 | const UnparkTid = switch (native_os) { |
| 17343 | // `NtAlertMultipleThreadByThreadId` is weird and wants 64-bit thread handles? | 17497 | // `NtAlertMultipleThreadByThreadId` is weird and wants 64-bit thread IDs? |
| 17344 | .windows => usize, | 17498 | .windows => usize, |
| 17345 | else => std.Thread.Id, | 17499 | else => std.Thread.Id, |
| 17346 | }; | 17500 | }; |