| ... | ... | @@ -19,6 +19,28 @@ pub const vtable: Allocator.VTable = .{ |
| 19 | 19 | .free = free, |
| 20 | 20 | }; |
| 21 | 21 | |
| 22 | /// Hhinting is disabled on operating systems that make an effort to not reuse |
| 23 | /// mappings. For example, OpenBSD aggressively randomizes addresses of mappings |
| 24 | /// that don't provide a hint (for security reasons, but it serves our needs |
| 25 | /// too). |
| 26 | const enable_hints = switch (builtin.target.os.tag) { |
| 27 | .openbsd => false, |
| 28 | else => true, |
| 29 | }; |
| 30 | |
| 31 | /// On operating systems that don't immediately map in the whole stack, we need |
| 32 | /// to be careful to not hint into the pages after the stack guard gap, which |
| 33 | /// the stack will expand into. The easiest way to avoid that is to hint in the |
| 34 | /// same direction as stack growth. |
| 35 | const stack_direction = builtin.target.stackGrowth(); |
| 36 | |
| 37 | /// When hinting upwards, this points to the next page that we hope to allocate |
| 38 | /// at; when hinting downwards, this points to the beginning of the last |
| 39 | /// successful allocation. |
| 40 | /// |
| 41 | /// TODO: Utilize this on Windows. |
| 42 | var addr_hint: ?[*]align(page_size_min) u8 = null; |
| 43 | |
| 22 | 44 | pub fn map(n: usize, alignment: Alignment) ?[*]u8 { |
| 23 | 45 | const page_size = std.heap.pageSize(); |
| 24 | 46 | if (n >= maxInt(usize) - page_size) return null; |
| ... | ... | @@ -41,7 +63,7 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 { |
| 41 | 63 | } |
| 42 | 64 | |
| 43 | 65 | const overalloc_len = n + alignment_bytes - page_size; |
| 44 | | const aligned_len = mem.alignForward(usize, n, page_size); |
| 66 | const page_aligned_len = mem.alignForward(usize, n, page_size); |
| 45 | 67 | |
| 46 | 68 | base_addr = null; |
| 47 | 69 | size = overalloc_len; |
| ... | ... | @@ -60,7 +82,7 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 { |
| 60 | 82 | _ = ntdll.NtFreeVirtualMemory(current_process, @ptrCast(&prefix_base), &prefix_size_param, .{ .RELEASE = true, .PRESERVE_PLACEHOLDER = true }); |
| 61 | 83 | } |
| 62 | 84 | |
| 63 | | const suffix_start = aligned_addr + aligned_len; |
| 85 | const suffix_start = aligned_addr + page_aligned_len; |
| 64 | 86 | const suffix_size = (placeholder_addr + overalloc_len) - suffix_start; |
| 65 | 87 | if (suffix_size > 0) { |
| 66 | 88 | var suffix_base = @as(?*anyopaque, @ptrFromInt(suffix_start)); |
| ... | ... | @@ -69,7 +91,7 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 { |
| 69 | 91 | } |
| 70 | 92 | |
| 71 | 93 | base_addr = @ptrFromInt(aligned_addr); |
| 72 | | size = aligned_len; |
| 94 | size = page_aligned_len; |
| 73 | 95 | |
| 74 | 96 | status = ntdll.NtAllocateVirtualMemory(current_process, @ptrCast(&base_addr), 0, &size, .{ .COMMIT = true }, .{ .READWRITE = true }); |
| 75 | 97 | |
| ... | ... | @@ -78,20 +100,34 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 { |
| 78 | 100 | } |
| 79 | 101 | |
| 80 | 102 | base_addr = @as(?*anyopaque, @ptrFromInt(aligned_addr)); |
| 81 | | size = aligned_len; |
| 103 | size = page_aligned_len; |
| 82 | 104 | _ = ntdll.NtFreeVirtualMemory(current_process, @ptrCast(&base_addr), &size, .{ .RELEASE = true }); |
| 83 | 105 | |
| 84 | 106 | return null; |
| 85 | 107 | } |
| 86 | 108 | |
| 87 | | const aligned_len = mem.alignForward(usize, n, page_size); |
| 109 | const page_aligned_len = mem.alignForward(usize, n, page_size); |
| 88 | 110 | const max_drop_len = alignment_bytes -| page_size; |
| 89 | | const overalloc_len = aligned_len + max_drop_len; |
| 90 | | const maybe_unaligned_hint = @atomicLoad(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, .unordered); |
| 111 | const overalloc_len = page_aligned_len + max_drop_len; |
| 112 | |
| 113 | const maybe_unaligned_hint, const hint = blk: { |
| 114 | if (!enable_hints) break :blk .{ null, null }; |
| 115 | |
| 116 | const maybe_unaligned_hint = @atomicLoad(@TypeOf(addr_hint), &addr_hint, .unordered); |
| 91 | 117 | |
| 92 | | // Aligning hint does not use mem.alignPointer, because it is slow. |
| 93 | | // Aligning hint does not use mem.alignForward, because it asserts that there will be no overflow. |
| 94 | | const hint: ?[*]align(page_size_min) u8 = @ptrFromInt(((@intFromPtr(maybe_unaligned_hint)) +% (alignment_bytes - 1)) & ~(alignment_bytes - 1)); |
| 118 | // For the very first mmap, let the kernel pick a good starting address; |
| 119 | // we'll begin doing our hinting from there. |
| 120 | if (maybe_unaligned_hint == null) break :blk .{ null, null }; |
| 121 | |
| 122 | // Aligning hint does not use mem.alignPointer, because it is slow. |
| 123 | // Aligning hint does not use mem.alignForward, because it asserts that there will be no overflow. |
| 124 | const hint: ?[*]align(page_size_min) u8 = @ptrFromInt(switch (stack_direction) { |
| 125 | .down => ((@intFromPtr(maybe_unaligned_hint) -% page_aligned_len) & ~(alignment_bytes - 1)) -% max_drop_len, |
| 126 | .up => (@intFromPtr(maybe_unaligned_hint) +% (alignment_bytes - 1)) & ~(alignment_bytes - 1), |
| 127 | }); |
| 128 | |
| 129 | break :blk .{ maybe_unaligned_hint, hint }; |
| 130 | }; |
| 95 | 131 | |
| 96 | 132 | const slice = posix.mmap( |
| 97 | 133 | hint, |
| ... | ... | @@ -101,16 +137,24 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 { |
| 101 | 137 | -1, |
| 102 | 138 | 0, |
| 103 | 139 | ) catch return null; |
| 104 | | const result_ptr = mem.alignPointer(slice.ptr, alignment_bytes) orelse return null; |
| 140 | const result_ptr = mem.alignPointer(slice.ptr, alignment_bytes).?; |
| 141 | |
| 105 | 142 | // Unmap the extra bytes that were only requested in order to guarantee |
| 106 | 143 | // that the range of memory we were provided had a proper alignment in it |
| 107 | 144 | // somewhere. The extra bytes could be at the beginning, or end, or both. |
| 108 | 145 | const drop_len = result_ptr - slice.ptr; |
| 109 | 146 | if (drop_len != 0) posix.munmap(slice[0..drop_len]); |
| 110 | 147 | const remaining_len = overalloc_len - drop_len; |
| 111 | | if (remaining_len > aligned_len) posix.munmap(@alignCast(result_ptr[aligned_len..remaining_len])); |
| 112 | | const new_hint: [*]align(page_size_min) u8 = @alignCast(result_ptr + aligned_len); |
| 113 | | _ = @cmpxchgStrong(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, maybe_unaligned_hint, new_hint, .monotonic, .monotonic); |
| 148 | if (remaining_len > page_aligned_len) posix.munmap(@alignCast(result_ptr[page_aligned_len..remaining_len])); |
| 149 | |
| 150 | if (enable_hints) { |
| 151 | const new_hint: [*]align(page_size_min) u8 = @alignCast(result_ptr + switch (stack_direction) { |
| 152 | .up => page_aligned_len, |
| 153 | .down => 0, |
| 154 | }); |
| 155 | _ = @cmpxchgStrong(@TypeOf(addr_hint), &addr_hint, maybe_unaligned_hint, new_hint, .monotonic, .monotonic); |
| 156 | } |
| 157 | |
| 114 | 158 | return result_ptr; |
| 115 | 159 | } |
| 116 | 160 | |
| ... | ... | @@ -181,7 +225,10 @@ pub fn realloc(uncasted_memory: []u8, alignment: Alignment, new_len: usize, may_ |
| 181 | 225 | if (new_size_aligned == page_aligned_len) |
| 182 | 226 | return memory.ptr; |
| 183 | 227 | |
| 184 | | if (posix.MREMAP != void) { |
| 228 | // When the stack grows down, only use `mremap` if the allocation may move. |
| 229 | // Otherwise, we might grow the allocation and intrude on virtual address |
| 230 | // space which we want to keep available to the stack. |
| 231 | if (posix.MREMAP != void and (stack_direction == .up or may_move)) { |
| 185 | 232 | // TODO: if the next_mmap_addr_hint is within the remapped range, update it |
| 186 | 233 | const new_memory = posix.mremap(memory.ptr, page_aligned_len, new_size_aligned, .{ .MAYMOVE = may_move }, null) catch return null; |
| 187 | 234 | return new_memory.ptr; |
Andrew Kelley <andrew@ziglang.org> 2026-02-24 21:24:50+01:00
.forgejo