| ... | @@ -19,6 +19,28 @@ pub const vtable: Allocator.VTable = .{ | ... | @@ -19,6 +19,28 @@ pub const vtable: Allocator.VTable = .{ |
| 19 | .free = free, | 19 | .free = free, |
| 20 | }; | 20 | }; |
| 21 | | 21 | |
| | 22 | /// Hhinting is disabled on operating systems that make an effort to not reuse |
| | 23 | /// mappings. For example, OpenBSD aggressively randomizes addresses of mappings |
| | 24 | /// that don't provide a hint (for security reasons, but it serves our needs |
| | 25 | /// too). |
| | 26 | const enable_hints = switch (builtin.target.os.tag) { |
| | 27 | .openbsd => false, |
| | 28 | else => true, |
| | 29 | }; |
| | 30 | |
| | 31 | /// On operating systems that don't immediately map in the whole stack, we need |
| | 32 | /// to be careful to not hint into the pages after the stack guard gap, which |
| | 33 | /// the stack will expand into. The easiest way to avoid that is to hint in the |
| | 34 | /// same direction as stack growth. |
| | 35 | const stack_direction = builtin.target.stackGrowth(); |
| | 36 | |
| | 37 | /// When hinting upwards, this points to the next page that we hope to allocate |
| | 38 | /// at; when hinting downwards, this points to the beginning of the last |
| | 39 | /// successful allocation. |
| | 40 | /// |
| | 41 | /// TODO: Utilize this on Windows. |
| | 42 | var addr_hint: ?[*]align(page_size_min) u8 = null; |
| | 43 | |
| 22 | pub fn map(n: usize, alignment: Alignment) ?[*]u8 { | 44 | pub fn map(n: usize, alignment: Alignment) ?[*]u8 { |
| 23 | const page_size = std.heap.pageSize(); | 45 | const page_size = std.heap.pageSize(); |
| 24 | if (n >= maxInt(usize) - page_size) return null; | 46 | if (n >= maxInt(usize) - page_size) return null; |
| ... | @@ -41,7 +63,7 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 { | ... | @@ -41,7 +63,7 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 { |
| 41 | } | 63 | } |
| 42 | | 64 | |
| 43 | const overalloc_len = n + alignment_bytes - page_size; | 65 | const overalloc_len = n + alignment_bytes - page_size; |
| 44 | const aligned_len = mem.alignForward(usize, n, page_size); | 66 | const page_aligned_len = mem.alignForward(usize, n, page_size); |
| 45 | | 67 | |
| 46 | base_addr = null; | 68 | base_addr = null; |
| 47 | size = overalloc_len; | 69 | size = overalloc_len; |
| ... | @@ -60,7 +82,7 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 { | ... | @@ -60,7 +82,7 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 { |
| 60 | _ = ntdll.NtFreeVirtualMemory(current_process, @ptrCast(&prefix_base), &prefix_size_param, .{ .RELEASE = true, .PRESERVE_PLACEHOLDER = true }); | 82 | _ = ntdll.NtFreeVirtualMemory(current_process, @ptrCast(&prefix_base), &prefix_size_param, .{ .RELEASE = true, .PRESERVE_PLACEHOLDER = true }); |
| 61 | } | 83 | } |
| 62 | | 84 | |
| 63 | const suffix_start = aligned_addr + aligned_len; | 85 | const suffix_start = aligned_addr + page_aligned_len; |
| 64 | const suffix_size = (placeholder_addr + overalloc_len) - suffix_start; | 86 | const suffix_size = (placeholder_addr + overalloc_len) - suffix_start; |
| 65 | if (suffix_size > 0) { | 87 | if (suffix_size > 0) { |
| 66 | var suffix_base = @as(?*anyopaque, @ptrFromInt(suffix_start)); | 88 | var suffix_base = @as(?*anyopaque, @ptrFromInt(suffix_start)); |
| ... | @@ -69,7 +91,7 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 { | ... | @@ -69,7 +91,7 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 { |
| 69 | } | 91 | } |
| 70 | | 92 | |
| 71 | base_addr = @ptrFromInt(aligned_addr); | 93 | base_addr = @ptrFromInt(aligned_addr); |
| 72 | size = aligned_len; | 94 | size = page_aligned_len; |
| 73 | | 95 | |
| 74 | status = ntdll.NtAllocateVirtualMemory(current_process, @ptrCast(&base_addr), 0, &size, .{ .COMMIT = true }, .{ .READWRITE = true }); | 96 | status = ntdll.NtAllocateVirtualMemory(current_process, @ptrCast(&base_addr), 0, &size, .{ .COMMIT = true }, .{ .READWRITE = true }); |
| 75 | | 97 | |
| ... | @@ -78,20 +100,34 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 { | ... | @@ -78,20 +100,34 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 { |
| 78 | } | 100 | } |
| 79 | | 101 | |
| 80 | base_addr = @as(?*anyopaque, @ptrFromInt(aligned_addr)); | 102 | base_addr = @as(?*anyopaque, @ptrFromInt(aligned_addr)); |
| 81 | size = aligned_len; | 103 | size = page_aligned_len; |
| 82 | _ = ntdll.NtFreeVirtualMemory(current_process, @ptrCast(&base_addr), &size, .{ .RELEASE = true }); | 104 | _ = ntdll.NtFreeVirtualMemory(current_process, @ptrCast(&base_addr), &size, .{ .RELEASE = true }); |
| 83 | | 105 | |
| 84 | return null; | 106 | return null; |
| 85 | } | 107 | } |
| 86 | | 108 | |
| 87 | const aligned_len = mem.alignForward(usize, n, page_size); | 109 | const page_aligned_len = mem.alignForward(usize, n, page_size); |
| 88 | const max_drop_len = alignment_bytes -| page_size; | 110 | const max_drop_len = alignment_bytes -| page_size; |
| 89 | const overalloc_len = aligned_len + max_drop_len; | 111 | const overalloc_len = page_aligned_len + max_drop_len; |
| 90 | const maybe_unaligned_hint = @atomicLoad(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, .unordered); | 112 | |
| | 113 | const maybe_unaligned_hint, const hint = blk: { |
| | 114 | if (!enable_hints) break :blk .{ null, null }; |
| | 115 | |
| | 116 | const maybe_unaligned_hint = @atomicLoad(@TypeOf(addr_hint), &addr_hint, .unordered); |
| 91 | | 117 | |
| 92 | // Aligning hint does not use mem.alignPointer, because it is slow. | 118 | // For the very first mmap, let the kernel pick a good starting address; |
| 93 | // Aligning hint does not use mem.alignForward, because it asserts that there will be no overflow. | 119 | // we'll begin doing our hinting from there. |
| 94 | const hint: ?[*]align(page_size_min) u8 = @ptrFromInt(((@intFromPtr(maybe_unaligned_hint)) +% (alignment_bytes - 1)) & ~(alignment_bytes - 1)); | 120 | if (maybe_unaligned_hint == null) break :blk .{ null, null }; |
| | 121 | |
| | 122 | // Aligning hint does not use mem.alignPointer, because it is slow. |
| | 123 | // Aligning hint does not use mem.alignForward, because it asserts that there will be no overflow. |
| | 124 | const hint: ?[*]align(page_size_min) u8 = @ptrFromInt(switch (stack_direction) { |
| | 125 | .down => ((@intFromPtr(maybe_unaligned_hint) -% page_aligned_len) & ~(alignment_bytes - 1)) -% max_drop_len, |
| | 126 | .up => (@intFromPtr(maybe_unaligned_hint) +% (alignment_bytes - 1)) & ~(alignment_bytes - 1), |
| | 127 | }); |
| | 128 | |
| | 129 | break :blk .{ maybe_unaligned_hint, hint }; |
| | 130 | }; |
| 95 | | 131 | |
| 96 | const slice = posix.mmap( | 132 | const slice = posix.mmap( |
| 97 | hint, | 133 | hint, |
| ... | @@ -101,16 +137,24 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 { | ... | @@ -101,16 +137,24 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 { |
| 101 | -1, | 137 | -1, |
| 102 | 0, | 138 | 0, |
| 103 | ) catch return null; | 139 | ) catch return null; |
| 104 | const result_ptr = mem.alignPointer(slice.ptr, alignment_bytes) orelse return null; | 140 | const result_ptr = mem.alignPointer(slice.ptr, alignment_bytes).?; |
| | 141 | |
| 105 | // Unmap the extra bytes that were only requested in order to guarantee | 142 | // Unmap the extra bytes that were only requested in order to guarantee |
| 106 | // that the range of memory we were provided had a proper alignment in it | 143 | // that the range of memory we were provided had a proper alignment in it |
| 107 | // somewhere. The extra bytes could be at the beginning, or end, or both. | 144 | // somewhere. The extra bytes could be at the beginning, or end, or both. |
| 108 | const drop_len = result_ptr - slice.ptr; | 145 | const drop_len = result_ptr - slice.ptr; |
| 109 | if (drop_len != 0) posix.munmap(slice[0..drop_len]); | 146 | if (drop_len != 0) posix.munmap(slice[0..drop_len]); |
| 110 | const remaining_len = overalloc_len - drop_len; | 147 | const remaining_len = overalloc_len - drop_len; |
| 111 | if (remaining_len > aligned_len) posix.munmap(@alignCast(result_ptr[aligned_len..remaining_len])); | 148 | if (remaining_len > page_aligned_len) posix.munmap(@alignCast(result_ptr[page_aligned_len..remaining_len])); |
| 112 | const new_hint: [*]align(page_size_min) u8 = @alignCast(result_ptr + aligned_len); | 149 | |
| 113 | _ = @cmpxchgStrong(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, maybe_unaligned_hint, new_hint, .monotonic, .monotonic); | 150 | if (enable_hints) { |
| | 151 | const new_hint: [*]align(page_size_min) u8 = @alignCast(result_ptr + switch (stack_direction) { |
| | 152 | .up => page_aligned_len, |
| | 153 | .down => 0, |
| | 154 | }); |
| | 155 | _ = @cmpxchgStrong(@TypeOf(addr_hint), &addr_hint, maybe_unaligned_hint, new_hint, .monotonic, .monotonic); |
| | 156 | } |
| | 157 | |
| 114 | return result_ptr; | 158 | return result_ptr; |
| 115 | } | 159 | } |
| 116 | | 160 | |
| ... | @@ -181,7 +225,10 @@ pub fn realloc(uncasted_memory: []u8, alignment: Alignment, new_len: usize, may_ | ... | @@ -181,7 +225,10 @@ pub fn realloc(uncasted_memory: []u8, alignment: Alignment, new_len: usize, may_ |
| 181 | if (new_size_aligned == page_aligned_len) | 225 | if (new_size_aligned == page_aligned_len) |
| 182 | return memory.ptr; | 226 | return memory.ptr; |
| 183 | | 227 | |
| 184 | if (posix.MREMAP != void) { | 228 | // When the stack grows down, only use `mremap` if the allocation may move. |
| | 229 | // Otherwise, we might grow the allocation and intrude on virtual address |
| | 230 | // space which we want to keep available to the stack. |
| | 231 | if (posix.MREMAP != void and (stack_direction == .up or may_move)) { |
| 185 | // TODO: if the next_mmap_addr_hint is within the remapped range, update it | 232 | // TODO: if the next_mmap_addr_hint is within the remapped range, update it |
| 186 | const new_memory = posix.mremap(memory.ptr, page_aligned_len, new_size_aligned, .{ .MAYMOVE = may_move }, null) catch return null; | 233 | const new_memory = posix.mremap(memory.ptr, page_aligned_len, new_size_aligned, .{ .MAYMOVE = may_move }, null) catch return null; |
| 187 | return new_memory.ptr; | 234 | return new_memory.ptr; |
Andrew Kelley <andrew@ziglang.org> 2026-02-24 21:24:50+01:00
.forgejo