| ... | ... | @@ -5047,7 +5047,7 @@ fn statFromLinux(stx: *const std.os.linux.Statx) Io.File.Stat { |
| 5047 | 5047 | }; |
| 5048 | 5048 | } |
| 5049 | 5049 | |
| 5050 | | fn statFromPosix(st: *const std.posix.Stat) Io.File.Stat { |
| 5050 | fn statFromPosix(st: *const posix.Stat) Io.File.Stat { |
| 5051 | 5051 | const atime = st.atime(); |
| 5052 | 5052 | const mtime = st.mtime(); |
| 5053 | 5053 | const ctime = st.ctime(); |
| ... | ... | @@ -5056,20 +5056,20 @@ fn statFromPosix(st: *const std.posix.Stat) Io.File.Stat { |
| 5056 | 5056 | .size = @bitCast(st.size), |
| 5057 | 5057 | .mode = st.mode, |
| 5058 | 5058 | .kind = k: { |
| 5059 | | const m = st.mode & std.posix.S.IFMT; |
| 5059 | const m = st.mode & posix.S.IFMT; |
| 5060 | 5060 | switch (m) { |
| 5061 | | std.posix.S.IFBLK => break :k .block_device, |
| 5062 | | std.posix.S.IFCHR => break :k .character_device, |
| 5063 | | std.posix.S.IFDIR => break :k .directory, |
| 5064 | | std.posix.S.IFIFO => break :k .named_pipe, |
| 5065 | | std.posix.S.IFLNK => break :k .sym_link, |
| 5066 | | std.posix.S.IFREG => break :k .file, |
| 5067 | | std.posix.S.IFSOCK => break :k .unix_domain_socket, |
| 5061 | posix.S.IFBLK => break :k .block_device, |
| 5062 | posix.S.IFCHR => break :k .character_device, |
| 5063 | posix.S.IFDIR => break :k .directory, |
| 5064 | posix.S.IFIFO => break :k .named_pipe, |
| 5065 | posix.S.IFLNK => break :k .sym_link, |
| 5066 | posix.S.IFREG => break :k .file, |
| 5067 | posix.S.IFSOCK => break :k .unix_domain_socket, |
| 5068 | 5068 | else => {}, |
| 5069 | 5069 | } |
| 5070 | 5070 | if (native_os == .illumos) switch (m) { |
| 5071 | | std.posix.S.IFDOOR => break :k .door, |
| 5072 | | std.posix.S.IFPORT => break :k .event_port, |
| 5071 | posix.S.IFDOOR => break :k .door, |
| 5072 | posix.S.IFPORT => break :k .event_port, |
| 5073 | 5073 | else => {}, |
| 5074 | 5074 | }; |
| 5075 | 5075 | |
| ... | ... | @@ -5101,11 +5101,11 @@ fn statFromWasi(st: *const std.os.wasi.filestat_t) Io.File.Stat { |
| 5101 | 5101 | }; |
| 5102 | 5102 | } |
| 5103 | 5103 | |
| 5104 | | fn timestampFromPosix(timespec: *const std.posix.timespec) Io.Timestamp { |
| 5104 | fn timestampFromPosix(timespec: *const posix.timespec) Io.Timestamp { |
| 5105 | 5105 | return .{ .nanoseconds = @intCast(@as(i128, timespec.sec) * std.time.ns_per_s + timespec.nsec) }; |
| 5106 | 5106 | } |
| 5107 | 5107 | |
| 5108 | | fn timestampToPosix(nanoseconds: i96) std.posix.timespec { |
| 5108 | fn timestampToPosix(nanoseconds: i96) posix.timespec { |
| 5109 | 5109 | return .{ |
| 5110 | 5110 | .sec = @intCast(@divFloor(nanoseconds, std.time.ns_per_s)), |
| 5111 | 5111 | .nsec = @intCast(@mod(nanoseconds, std.time.ns_per_s)), |
| ... | ... | @@ -5503,44 +5503,7 @@ const darwin_supports_ulock_wait2 = builtin.os.version_range.semver.min.major >= |
| 5503 | 5503 | fn futexWait(t: *Threaded, ptr: *const std.atomic.Value(u32), expect: u32) Io.Cancelable!void { |
| 5504 | 5504 | @branchHint(.cold); |
| 5505 | 5505 | |
| 5506 | | if (native_os == .linux) { |
| 5507 | | const linux = std.os.linux; |
| 5508 | | try t.checkCancel(); |
| 5509 | | const rc = linux.futex_4arg(ptr, .{ .cmd = .WAIT, .private = true }, expect, null); |
| 5510 | | if (is_debug) switch (linux.E.init(rc)) { |
| 5511 | | .SUCCESS => {}, // notified by `wake()` |
| 5512 | | .INTR => {}, // gives caller a chance to check cancellation |
| 5513 | | .AGAIN => {}, // ptr.* != expect |
| 5514 | | .INVAL => {}, // possibly timeout overflow |
| 5515 | | .TIMEDOUT => unreachable, |
| 5516 | | .FAULT => unreachable, // ptr was invalid |
| 5517 | | else => unreachable, |
| 5518 | | }; |
| 5519 | | } else if (native_os.isDarwin()) { |
| 5520 | | const c = std.c; |
| 5521 | | const flags: c.UL = .{ |
| 5522 | | .op = .COMPARE_AND_WAIT, |
| 5523 | | .NO_ERRNO = true, |
| 5524 | | }; |
| 5525 | | try t.checkCancel(); |
| 5526 | | const status = if (darwin_supports_ulock_wait2) |
| 5527 | | c.__ulock_wait2(flags, ptr, expect, 0, 0) |
| 5528 | | else |
| 5529 | | c.__ulock_wait(flags, ptr, expect, 0); |
| 5530 | | |
| 5531 | | if (status >= 0) return; |
| 5532 | | |
| 5533 | | if (is_debug) switch (@as(c.E, @enumFromInt(-status))) { |
| 5534 | | // Wait was interrupted by the OS or other spurious signalling. |
| 5535 | | .INTR => {}, |
| 5536 | | // Address of the futex was paged out. This is unlikely, but possible in theory, and |
| 5537 | | // pthread/libdispatch on darwin bother to handle it. In this case we'll return |
| 5538 | | // without waiting, but the caller should retry anyway. |
| 5539 | | .FAULT => {}, |
| 5540 | | .TIMEDOUT => unreachable, |
| 5541 | | else => unreachable, |
| 5542 | | }; |
| 5543 | | } else if (builtin.cpu.arch.isWasm()) { |
| 5506 | if (builtin.cpu.arch.isWasm()) { |
| 5544 | 5507 | comptime assert(builtin.cpu.has(.wasm, .atomics)); |
| 5545 | 5508 | try t.checkCancel(); |
| 5546 | 5509 | const timeout: i64 = -1; |
| ... | ... | @@ -5562,57 +5525,74 @@ fn futexWait(t: *Threaded, ptr: *const std.atomic.Value(u32), expect: u32) Io.Ca |
| 5562 | 5525 | 2 => assert(!is_debug), // timeout |
| 5563 | 5526 | else => assert(!is_debug), |
| 5564 | 5527 | } |
| 5565 | | } else if (is_windows) { |
| 5566 | | try t.checkCancel(); |
| 5567 | | switch (windows.ntdll.RtlWaitOnAddress(ptr, &expect, @sizeOf(@TypeOf(expect)), null)) { |
| 5568 | | .SUCCESS => {}, |
| 5569 | | .CANCELLED => return error.Canceled, |
| 5570 | | else => recoverableOsBugDetected(), |
| 5571 | | } |
| 5572 | | } else { |
| 5573 | | @compileError("TODO"); |
| 5528 | } else switch (native_os) { |
| 5529 | .linux => { |
| 5530 | const linux = std.os.linux; |
| 5531 | try t.checkCancel(); |
| 5532 | const rc = linux.futex_4arg(ptr, .{ .cmd = .WAIT, .private = true }, expect, null); |
| 5533 | if (is_debug) switch (linux.E.init(rc)) { |
| 5534 | .SUCCESS => {}, // notified by `wake()` |
| 5535 | .INTR => {}, // gives caller a chance to check cancellation |
| 5536 | .AGAIN => {}, // ptr.* != expect |
| 5537 | .INVAL => {}, // possibly timeout overflow |
| 5538 | .TIMEDOUT => unreachable, |
| 5539 | .FAULT => unreachable, // ptr was invalid |
| 5540 | else => unreachable, |
| 5541 | }; |
| 5542 | }, |
| 5543 | .driverkit, .ios, .macos, .tvos, .visionos, .watchos => { |
| 5544 | const c = std.c; |
| 5545 | const flags: c.UL = .{ |
| 5546 | .op = .COMPARE_AND_WAIT, |
| 5547 | .NO_ERRNO = true, |
| 5548 | }; |
| 5549 | try t.checkCancel(); |
| 5550 | const status = if (darwin_supports_ulock_wait2) |
| 5551 | c.__ulock_wait2(flags, ptr, expect, 0, 0) |
| 5552 | else |
| 5553 | c.__ulock_wait(flags, ptr, expect, 0); |
| 5554 | |
| 5555 | if (status >= 0) return; |
| 5556 | |
| 5557 | if (is_debug) switch (@as(c.E, @enumFromInt(-status))) { |
| 5558 | .INTR => {}, // spurious wake |
| 5559 | // Address of the futex was paged out. This is unlikely, but possible in theory, and |
| 5560 | // pthread/libdispatch on darwin bother to handle it. In this case we'll return |
| 5561 | // without waiting, but the caller should retry anyway. |
| 5562 | .FAULT => {}, |
| 5563 | .TIMEDOUT => unreachable, |
| 5564 | else => unreachable, |
| 5565 | }; |
| 5566 | }, |
| 5567 | .windows => { |
| 5568 | try t.checkCancel(); |
| 5569 | switch (windows.ntdll.RtlWaitOnAddress(ptr, &expect, @sizeOf(@TypeOf(expect)), null)) { |
| 5570 | .SUCCESS => {}, |
| 5571 | .CANCELLED => return error.Canceled, |
| 5572 | else => recoverableOsBugDetected(), |
| 5573 | } |
| 5574 | }, |
| 5575 | .freebsd => { |
| 5576 | const flags = @intFromEnum(std.c.UMTX_OP.WAIT_UINT_PRIVATE); |
| 5577 | try t.checkCancel(); |
| 5578 | const rc = std.c._umtx_op(@intFromPtr(&ptr.raw), flags, @as(c_ulong, expect), 0, 0); |
| 5579 | if (is_debug) switch (posix.errno(rc)) { |
| 5580 | .SUCCESS => {}, |
| 5581 | .FAULT => unreachable, // one of the args points to invalid memory |
| 5582 | .INVAL => unreachable, // arguments should be correct |
| 5583 | .TIMEDOUT => unreachable, // no timeout provided |
| 5584 | .INTR => {}, // spurious wake |
| 5585 | else => unreachable, |
| 5586 | }; |
| 5587 | }, |
| 5588 | else => @compileError("unimplemented: futexWait"), |
| 5574 | 5589 | } |
| 5575 | 5590 | } |
| 5576 | 5591 | |
| 5577 | 5592 | pub fn futexWaitUncancelable(ptr: *const std.atomic.Value(u32), expect: u32) void { |
| 5578 | 5593 | @branchHint(.cold); |
| 5579 | 5594 | |
| 5580 | | if (native_os == .linux) { |
| 5581 | | const linux = std.os.linux; |
| 5582 | | const rc = linux.futex_4arg(ptr, .{ .cmd = .WAIT, .private = true }, expect, null); |
| 5583 | | if (is_debug) switch (linux.E.init(rc)) { |
| 5584 | | .SUCCESS => {}, // notified by `wake()` |
| 5585 | | .INTR => {}, // gives caller a chance to check cancellation |
| 5586 | | .AGAIN => {}, // ptr.* != expect |
| 5587 | | .INVAL => {}, // possibly timeout overflow |
| 5588 | | .TIMEDOUT => unreachable, |
| 5589 | | .FAULT => unreachable, // ptr was invalid |
| 5590 | | else => unreachable, |
| 5591 | | }; |
| 5592 | | } else if (native_os.isDarwin()) { |
| 5593 | | const c = std.c; |
| 5594 | | const flags: c.UL = .{ |
| 5595 | | .op = .COMPARE_AND_WAIT, |
| 5596 | | .NO_ERRNO = true, |
| 5597 | | }; |
| 5598 | | const status = if (darwin_supports_ulock_wait2) |
| 5599 | | c.__ulock_wait2(flags, ptr, expect, 0, 0) |
| 5600 | | else |
| 5601 | | c.__ulock_wait(flags, ptr, expect, 0); |
| 5602 | | |
| 5603 | | if (status >= 0) return; |
| 5604 | | |
| 5605 | | if (is_debug) switch (@as(c.E, @enumFromInt(-status))) { |
| 5606 | | // Wait was interrupted by the OS or other spurious signalling. |
| 5607 | | .INTR => {}, |
| 5608 | | // Address of the futex was paged out. This is unlikely, but possible in theory, and |
| 5609 | | // pthread/libdispatch on darwin bother to handle it. In this case we'll return |
| 5610 | | // without waiting, but the caller should retry anyway. |
| 5611 | | .FAULT => {}, |
| 5612 | | .TIMEDOUT => unreachable, |
| 5613 | | else => unreachable, |
| 5614 | | }; |
| 5615 | | } else if (builtin.cpu.arch.isWasm()) { |
| 5595 | if (builtin.cpu.arch.isWasm()) { |
| 5616 | 5596 | comptime assert(builtin.cpu.has(.wasm, .atomics)); |
| 5617 | 5597 | const timeout: i64 = -1; |
| 5618 | 5598 | const signed_expect: i32 = @bitCast(expect); |
| ... | ... | @@ -5630,16 +5610,66 @@ pub fn futexWaitUncancelable(ptr: *const std.atomic.Value(u32), expect: u32) voi |
| 5630 | 5610 | switch (result) { |
| 5631 | 5611 | 0 => {}, // ok |
| 5632 | 5612 | 1 => {}, // expected != loaded |
| 5633 | | 2 => assert(!is_debug), // timeout |
| 5634 | | else => assert(!is_debug), |
| 5635 | | } |
| 5636 | | } else if (is_windows) { |
| 5637 | | switch (windows.ntdll.RtlWaitOnAddress(ptr, &expect, @sizeOf(@TypeOf(expect)), null)) { |
| 5638 | | .SUCCESS, .CANCELLED => {}, |
| 5613 | 2 => recoverableOsBugDetected(), // timeout |
| 5639 | 5614 | else => recoverableOsBugDetected(), |
| 5640 | 5615 | } |
| 5641 | | } else { |
| 5642 | | @compileError("TODO"); |
| 5616 | } else switch (native_os) { |
| 5617 | .linux => { |
| 5618 | const linux = std.os.linux; |
| 5619 | const rc = linux.futex_4arg(ptr, .{ .cmd = .WAIT, .private = true }, expect, null); |
| 5620 | switch (linux.E.init(rc)) { |
| 5621 | .SUCCESS => {}, // notified by `wake()` |
| 5622 | .INTR => {}, // gives caller a chance to check cancellation |
| 5623 | .AGAIN => {}, // ptr.* != expect |
| 5624 | .INVAL => {}, // possibly timeout overflow |
| 5625 | .TIMEDOUT => recoverableOsBugDetected(), |
| 5626 | .FAULT => recoverableOsBugDetected(), // ptr was invalid |
| 5627 | else => recoverableOsBugDetected(), |
| 5628 | } |
| 5629 | }, |
| 5630 | .driverkit, .ios, .macos, .tvos, .visionos, .watchos => { |
| 5631 | const c = std.c; |
| 5632 | const flags: c.UL = .{ |
| 5633 | .op = .COMPARE_AND_WAIT, |
| 5634 | .NO_ERRNO = true, |
| 5635 | }; |
| 5636 | const status = if (darwin_supports_ulock_wait2) |
| 5637 | c.__ulock_wait2(flags, ptr, expect, 0, 0) |
| 5638 | else |
| 5639 | c.__ulock_wait(flags, ptr, expect, 0); |
| 5640 | |
| 5641 | if (status >= 0) return; |
| 5642 | |
| 5643 | switch (@as(c.E, @enumFromInt(-status))) { |
| 5644 | // Wait was interrupted by the OS or other spurious signalling. |
| 5645 | .INTR => {}, |
| 5646 | // Address of the futex was paged out. This is unlikely, but possible in theory, and |
| 5647 | // pthread/libdispatch on darwin bother to handle it. In this case we'll return |
| 5648 | // without waiting, but the caller should retry anyway. |
| 5649 | .FAULT => {}, |
| 5650 | .TIMEDOUT => recoverableOsBugDetected(), |
| 5651 | else => recoverableOsBugDetected(), |
| 5652 | } |
| 5653 | }, |
| 5654 | .windows => { |
| 5655 | switch (windows.ntdll.RtlWaitOnAddress(ptr, &expect, @sizeOf(@TypeOf(expect)), null)) { |
| 5656 | .SUCCESS, .CANCELLED => {}, |
| 5657 | else => recoverableOsBugDetected(), |
| 5658 | } |
| 5659 | }, |
| 5660 | .freebsd => { |
| 5661 | const flags = @intFromEnum(std.c.UMTX_OP.WAIT_UINT_PRIVATE); |
| 5662 | const rc = std.c._umtx_op(@intFromPtr(&ptr.raw), flags, @as(c_ulong, expect), 0, 0); |
| 5663 | switch (posix.errno(rc)) { |
| 5664 | .SUCCESS => {}, |
| 5665 | .INTR => {}, // spurious wake |
| 5666 | .FAULT => recoverableOsBugDetected(), // one of the args points to invalid memory |
| 5667 | .INVAL => recoverableOsBugDetected(), // arguments should be correct |
| 5668 | .TIMEDOUT => recoverableOsBugDetected(), // no timeout provided |
| 5669 | else => recoverableOsBugDetected(), |
| 5670 | } |
| 5671 | }, |
| 5672 | else => @compileError("unimplemented: futexWaitUncancelable"), |
| 5643 | 5673 | } |
| 5644 | 5674 | } |
| 5645 | 5675 | |
| ... | ... | @@ -5668,38 +5698,7 @@ pub fn futexWaitDurationUncancelable(ptr: *const std.atomic.Value(u32), expect: |
| 5668 | 5698 | pub fn futexWake(ptr: *const std.atomic.Value(u32), max_waiters: u32) void { |
| 5669 | 5699 | @branchHint(.cold); |
| 5670 | 5700 | |
| 5671 | | if (native_os == .linux) { |
| 5672 | | const linux = std.os.linux; |
| 5673 | | const rc = linux.futex_3arg( |
| 5674 | | &ptr.raw, |
| 5675 | | .{ .cmd = .WAKE, .private = true }, |
| 5676 | | @min(max_waiters, std.math.maxInt(i32)), |
| 5677 | | ); |
| 5678 | | if (is_debug) switch (linux.E.init(rc)) { |
| 5679 | | .SUCCESS => {}, // successful wake up |
| 5680 | | .INVAL => {}, // invalid futex_wait() on ptr done elsewhere |
| 5681 | | .FAULT => {}, // pointer became invalid while doing the wake |
| 5682 | | else => unreachable, |
| 5683 | | }; |
| 5684 | | } else if (native_os.isDarwin()) { |
| 5685 | | const c = std.c; |
| 5686 | | const flags: c.UL = .{ |
| 5687 | | .op = .COMPARE_AND_WAIT, |
| 5688 | | .NO_ERRNO = true, |
| 5689 | | .WAKE_ALL = max_waiters > 1, |
| 5690 | | }; |
| 5691 | | while (true) { |
| 5692 | | const status = c.__ulock_wake(flags, ptr, 0); |
| 5693 | | if (status >= 0) return; |
| 5694 | | switch (@as(c.E, @enumFromInt(-status))) { |
| 5695 | | .INTR, .CANCELED => continue, // spurious wake() |
| 5696 | | .FAULT => assert(!is_debug), // __ulock_wake doesn't generate EFAULT according to darwin pthread_cond_t |
| 5697 | | .NOENT => return, // nothing was woken up |
| 5698 | | .ALREADY => assert(!is_debug), // only for UL.Op.WAKE_THREAD |
| 5699 | | else => assert(!is_debug), |
| 5700 | | } |
| 5701 | | } |
| 5702 | | } else if (builtin.cpu.arch.isWasm()) { |
| 5701 | if (builtin.cpu.arch.isWasm()) { |
| 5703 | 5702 | comptime assert(builtin.cpu.has(.wasm, .atomics)); |
| 5704 | 5703 | assert(max_waiters != 0); |
| 5705 | 5704 | const woken_count = asm volatile ( |
| ... | ... | @@ -5712,14 +5711,63 @@ pub fn futexWake(ptr: *const std.atomic.Value(u32), max_waiters: u32) void { |
| 5712 | 5711 | [waiters] "r" (max_waiters), |
| 5713 | 5712 | ); |
| 5714 | 5713 | _ = woken_count; // can be 0 when linker flag 'shared-memory' is not enabled |
| 5715 | | } else if (is_windows) { |
| 5716 | | assert(max_waiters != 0); |
| 5717 | | switch (max_waiters) { |
| 5718 | | 1 => windows.ntdll.RtlWakeAddressSingle(ptr), |
| 5719 | | else => windows.ntdll.RtlWakeAddressAll(ptr), |
| 5720 | | } |
| 5721 | | } else { |
| 5722 | | @compileError("TODO"); |
| 5714 | } else switch (native_os) { |
| 5715 | .linux => { |
| 5716 | const linux = std.os.linux; |
| 5717 | const rc = linux.futex_3arg( |
| 5718 | &ptr.raw, |
| 5719 | .{ .cmd = .WAKE, .private = true }, |
| 5720 | @min(max_waiters, std.math.maxInt(i32)), |
| 5721 | ); |
| 5722 | if (is_debug) switch (linux.E.init(rc)) { |
| 5723 | .SUCCESS => {}, // successful wake up |
| 5724 | .INVAL => {}, // invalid futex_wait() on ptr done elsewhere |
| 5725 | .FAULT => {}, // pointer became invalid while doing the wake |
| 5726 | else => unreachable, // deadlock due to operating system bug |
| 5727 | }; |
| 5728 | }, |
| 5729 | .driverkit, .ios, .macos, .tvos, .visionos, .watchos => { |
| 5730 | const c = std.c; |
| 5731 | const flags: c.UL = .{ |
| 5732 | .op = .COMPARE_AND_WAIT, |
| 5733 | .NO_ERRNO = true, |
| 5734 | .WAKE_ALL = max_waiters > 1, |
| 5735 | }; |
| 5736 | while (true) { |
| 5737 | const status = c.__ulock_wake(flags, ptr, 0); |
| 5738 | if (status >= 0) return; |
| 5739 | switch (@as(c.E, @enumFromInt(-status))) { |
| 5740 | .INTR, .CANCELED => continue, // spurious wake() |
| 5741 | .FAULT => unreachable, // __ulock_wake doesn't generate EFAULT according to darwin pthread_cond_t |
| 5742 | .NOENT => return, // nothing was woken up |
| 5743 | .ALREADY => unreachable, // only for UL.Op.WAKE_THREAD |
| 5744 | else => unreachable, // deadlock due to operating system bug |
| 5745 | } |
| 5746 | } |
| 5747 | }, |
| 5748 | .windows => { |
| 5749 | assert(max_waiters != 0); |
| 5750 | switch (max_waiters) { |
| 5751 | 1 => windows.ntdll.RtlWakeAddressSingle(ptr), |
| 5752 | else => windows.ntdll.RtlWakeAddressAll(ptr), |
| 5753 | } |
| 5754 | }, |
| 5755 | .freebsd => { |
| 5756 | const rc = std.c._umtx_op( |
| 5757 | @intFromPtr(&ptr.raw), |
| 5758 | @intFromEnum(std.c.UMTX_OP.WAKE_PRIVATE), |
| 5759 | @as(c_ulong, max_waiters), |
| 5760 | 0, // there is no timeout struct |
| 5761 | 0, // there is no timeout struct pointer |
| 5762 | ); |
| 5763 | switch (posix.errno(rc)) { |
| 5764 | .SUCCESS => {}, |
| 5765 | .FAULT => {}, // it's ok if the ptr doesn't point to valid memory |
| 5766 | .INVAL => unreachable, // arguments should be correct |
| 5767 | else => unreachable, // deadlock due to operating system bug |
| 5768 | } |
| 5769 | }, |
| 5770 | else => @compileError("unimplemented: futexWake"), |
| 5723 | 5771 | } |
| 5724 | 5772 | } |
| 5725 | 5773 | |
Andrew Kelley <andrew@ziglang.org> 2025-10-27 14:34:42-07:00
lib