authorgravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2026-02-18 04:54:04+01:00
committergravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2026-02-21 23:39:20+01:00
logc8dd05030519fc52dd457531f6b2654e1ad129b2
tree0dbb9592316d8d361bed029519b05f9acdc34ee7
parent5ac6ff43d41f23d7d215c3164848bb4ffcf00d59
signaturebadge-check Signed by SSH key SHA256:7B/LJ7bpR1eX8aCXSr4mtd5M45VMPKcx9zY8e95b5QM

std.heap.PageAllocator: hint mmaps in the same direction as stack growth

The old logic was fine for targets where the stack grows up (so, literally just hppa), but problematic on targets where it grows down, because we could hint that we wanted an allocation to happen in an area of the address space that the kernel expects to be able to expand the stack into. The kernel is happy to satisfy such a hint despite the obvious problems this leads to later down the road. Co-authored-by: rpkak <rpkak@noreply.codeberg.org>

2 files changed, 58 insertions(+), 17 deletions(-)

lib/std/heap.zig-3
...@@ -41,9 +41,6 @@ pub const MemoryPoolExtra = memory_pool.Extra;...@@ -41,9 +41,6 @@ pub const MemoryPoolExtra = memory_pool.Extra;
41/// Deprecated; use `memory_pool.Options`.41/// Deprecated; use `memory_pool.Options`.
42pub const MemoryPoolOptions = memory_pool.Options;42pub const MemoryPoolOptions = memory_pool.Options;
4343
44/// TODO Utilize this on Windows.
45pub var next_mmap_addr_hint: ?[*]align(page_size_min) u8 = null;
46
47/// comptime-known minimum page size of the target.44/// comptime-known minimum page size of the target.
48///45///
49/// All pointers from `mmap` or `NtAllocateVirtualMemory` are aligned to at least46/// All pointers from `mmap` or `NtAllocateVirtualMemory` are aligned to at least
lib/std/heap/PageAllocator.zig+58-14
...@@ -19,6 +19,28 @@ pub const vtable: Allocator.VTable = .{...@@ -19,6 +19,28 @@ pub const vtable: Allocator.VTable = .{
19 .free = free,19 .free = free,
20};20};
2121
22/// Hhinting is disabled on operating systems that make an effort to not reuse
23/// mappings. For example, OpenBSD aggressively randomizes addresses of mappings
24/// that don't provide a hint (for security reasons, but it serves our needs
25/// too).
26const enable_hints = switch (builtin.target.os.tag) {
27 .openbsd => false,
28 else => true,
29};
30
31/// On operating systems that don't immediately map in the whole stack, we need
32/// to be careful to not hint into the pages after the stack guard gap, which
33/// the stack will expand into. The easiest way to avoid that is to hint in the
34/// same direction as stack growth.
35const stack_direction = builtin.target.stackGrowth();
36
37/// When hinting upwards, this points to the next page that we hope to allocate
38/// at; when hinting downwards, this points to the beginning of the last
39/// successful allocation.
40///
41/// TODO: Utilize this on Windows.
42var addr_hint: ?[*]align(page_size_min) u8 = null;
43
22pub fn map(n: usize, alignment: Alignment) ?[*]u8 {44pub fn map(n: usize, alignment: Alignment) ?[*]u8 {
23 const page_size = std.heap.pageSize();45 const page_size = std.heap.pageSize();
24 if (n >= maxInt(usize) - page_size) return null;46 if (n >= maxInt(usize) - page_size) return null;
...@@ -41,7 +63,7 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 {...@@ -41,7 +63,7 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 {
41 }63 }
4264
43 const overalloc_len = n + alignment_bytes - page_size;65 const overalloc_len = n + alignment_bytes - page_size;
44 const aligned_len = mem.alignForward(usize, n, page_size);66 const page_aligned_len = mem.alignForward(usize, n, page_size);
4567
46 base_addr = null;68 base_addr = null;
47 size = overalloc_len;69 size = overalloc_len;
...@@ -60,7 +82,7 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 {...@@ -60,7 +82,7 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 {
60 _ = ntdll.NtFreeVirtualMemory(current_process, @ptrCast(&prefix_base), &prefix_size_param, .{ .RELEASE = true, .PRESERVE_PLACEHOLDER = true });82 _ = ntdll.NtFreeVirtualMemory(current_process, @ptrCast(&prefix_base), &prefix_size_param, .{ .RELEASE = true, .PRESERVE_PLACEHOLDER = true });
61 }83 }
6284
63 const suffix_start = aligned_addr + aligned_len;85 const suffix_start = aligned_addr + page_aligned_len;
64 const suffix_size = (placeholder_addr + overalloc_len) - suffix_start;86 const suffix_size = (placeholder_addr + overalloc_len) - suffix_start;
65 if (suffix_size > 0) {87 if (suffix_size > 0) {
66 var suffix_base = @as(?*anyopaque, @ptrFromInt(suffix_start));88 var suffix_base = @as(?*anyopaque, @ptrFromInt(suffix_start));
...@@ -69,7 +91,7 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 {...@@ -69,7 +91,7 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 {
69 }91 }
7092
71 base_addr = @ptrFromInt(aligned_addr);93 base_addr = @ptrFromInt(aligned_addr);
72 size = aligned_len;94 size = page_aligned_len;
7395
74 status = ntdll.NtAllocateVirtualMemory(current_process, @ptrCast(&base_addr), 0, &size, .{ .COMMIT = true }, .{ .READWRITE = true });96 status = ntdll.NtAllocateVirtualMemory(current_process, @ptrCast(&base_addr), 0, &size, .{ .COMMIT = true }, .{ .READWRITE = true });
7597
...@@ -78,20 +100,34 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 {...@@ -78,20 +100,34 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 {
78 }100 }
79101
80 base_addr = @as(?*anyopaque, @ptrFromInt(aligned_addr));102 base_addr = @as(?*anyopaque, @ptrFromInt(aligned_addr));
81 size = aligned_len;103 size = page_aligned_len;
82 _ = ntdll.NtFreeVirtualMemory(current_process, @ptrCast(&base_addr), &size, .{ .RELEASE = true });104 _ = ntdll.NtFreeVirtualMemory(current_process, @ptrCast(&base_addr), &size, .{ .RELEASE = true });
83105
84 return null;106 return null;
85 }107 }
86108
87 const aligned_len = mem.alignForward(usize, n, page_size);109 const page_aligned_len = mem.alignForward(usize, n, page_size);
88 const max_drop_len = alignment_bytes -| page_size;110 const max_drop_len = alignment_bytes -| page_size;
89 const overalloc_len = aligned_len + max_drop_len;111 const overalloc_len = page_aligned_len + max_drop_len;
90 const maybe_unaligned_hint = @atomicLoad(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, .unordered);112
113 const maybe_unaligned_hint, const hint = blk: {
114 if (!enable_hints) break :blk .{ null, null };
115
116 const maybe_unaligned_hint = @atomicLoad(@TypeOf(addr_hint), &addr_hint, .unordered);
91117
92 // Aligning hint does not use mem.alignPointer, because it is slow.118 // For the very first mmap, let the kernel pick a good starting address;
93 // Aligning hint does not use mem.alignForward, because it asserts that there will be no overflow.119 // we'll begin doing our hinting from there.
94 const hint: ?[*]align(page_size_min) u8 = @ptrFromInt(((@intFromPtr(maybe_unaligned_hint)) +% (alignment_bytes - 1)) & ~(alignment_bytes - 1));120 if (maybe_unaligned_hint == null) break :blk .{ null, null };
121
122 // Aligning hint does not use mem.alignPointer, because it is slow.
123 // Aligning hint does not use mem.alignForward, because it asserts that there will be no overflow.
124 const hint: ?[*]align(page_size_min) u8 = @ptrFromInt(switch (stack_direction) {
125 .down => ((@intFromPtr(maybe_unaligned_hint) -% page_aligned_len) & ~(alignment_bytes - 1)) -% max_drop_len,
126 .up => (@intFromPtr(maybe_unaligned_hint) +% (alignment_bytes - 1)) & ~(alignment_bytes - 1),
127 });
128
129 break :blk .{ maybe_unaligned_hint, hint };
130 };
95131
96 const slice = posix.mmap(132 const slice = posix.mmap(
97 hint,133 hint,
...@@ -101,16 +137,24 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 {...@@ -101,16 +137,24 @@ pub fn map(n: usize, alignment: Alignment) ?[*]u8 {
101 -1,137 -1,
102 0,138 0,
103 ) catch return null;139 ) catch return null;
104 const result_ptr = mem.alignPointer(slice.ptr, alignment_bytes) orelse return null;140 const result_ptr = mem.alignPointer(slice.ptr, alignment_bytes).?;
141
105 // Unmap the extra bytes that were only requested in order to guarantee142 // Unmap the extra bytes that were only requested in order to guarantee
106 // that the range of memory we were provided had a proper alignment in it143 // that the range of memory we were provided had a proper alignment in it
107 // somewhere. The extra bytes could be at the beginning, or end, or both.144 // somewhere. The extra bytes could be at the beginning, or end, or both.
108 const drop_len = result_ptr - slice.ptr;145 const drop_len = result_ptr - slice.ptr;
109 if (drop_len != 0) posix.munmap(slice[0..drop_len]);146 if (drop_len != 0) posix.munmap(slice[0..drop_len]);
110 const remaining_len = overalloc_len - drop_len;147 const remaining_len = overalloc_len - drop_len;
111 if (remaining_len > aligned_len) posix.munmap(@alignCast(result_ptr[aligned_len..remaining_len]));148 if (remaining_len > page_aligned_len) posix.munmap(@alignCast(result_ptr[page_aligned_len..remaining_len]));
112 const new_hint: [*]align(page_size_min) u8 = @alignCast(result_ptr + aligned_len);149
113 _ = @cmpxchgStrong(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, maybe_unaligned_hint, new_hint, .monotonic, .monotonic);150 if (enable_hints) {
151 const new_hint: [*]align(page_size_min) u8 = @alignCast(result_ptr + switch (stack_direction) {
152 .up => page_aligned_len,
153 .down => 0,
154 });
155 _ = @cmpxchgStrong(@TypeOf(addr_hint), &addr_hint, maybe_unaligned_hint, new_hint, .monotonic, .monotonic);
156 }
157
114 return result_ptr;158 return result_ptr;
115}159}
116160