1const builtin = @import("builtin");
2const native_os = builtin.os.tag;
3
4const std = @import("../std.zig");
5const Allocator = std.mem.Allocator;
6const Alignment = std.mem.Alignment;
7const mem = std.mem;
8const maxInt = std.math.maxInt;
9const assert = std.debug.assert;
10const windows = std.os.windows;
11const ntdll = std.os.windows.ntdll;
12const posix = std.posix;
13const page_size_min = std.heap.page_size_min;
14
15pub const vtable: Allocator.VTable = .{
16 .alloc = alloc,
17 .resize = resize,
18 .remap = remap,
19 .free = free,
20};
21
22/// Hhinting is disabled on operating systems that make an effort to not reuse
23/// mappings. For example, OpenBSD aggressively randomizes addresses of mappings
24/// that don't provide a hint (for security reasons, but it serves our needs
25/// too).
26const enable_hints = switch (builtin.target.os.tag) {
27 .linux => !builtin.target.cpu.arch.isSPARC(), // https://bugzilla.kernel.org/show_bug.cgi?id=221820
28 .openbsd => false,
29 else => true,
30};
31
32/// On operating systems that don't immediately map in the whole stack, we need
33/// to be careful to not hint into the pages after the stack guard gap, which
34/// the stack will expand into. The easiest way to avoid that is to hint in the
35/// same direction as stack growth.
36const stack_direction = builtin.target.stackGrowth();
37
38/// When hinting upwards, this points to the next page that we hope to allocate
39/// at; when hinting downwards, this points to the beginning of the last
40/// successful allocation.
41///
42/// TODO: Utilize this on Windows.
43var addr_hint: ?[*]align(page_size_min) u8 = null;
44
45pub fn map(n: usize, alignment: Alignment) ?[*]u8 {
46 const page_size = std.heap.pageSize();
47 if (n >= maxInt(usize) - page_size) return null;
48 const alignment_bytes = alignment.toByteUnits();
49
50 if (native_os == .windows) {
51 var base_addr: ?*anyopaque = null;
52 var size: windows.SIZE_T = n;
53
54 const current_process = windows.GetCurrentProcess();
55 var status = ntdll.NtAllocateVirtualMemory(current_process, @ptrCast(&base_addr), 0, &size, .{ .COMMIT = true, .RESERVE = true }, .{ .READWRITE = true });
56
57 if (status == .SUCCESS and mem.isAligned(@intFromPtr(base_addr), alignment_bytes)) {
58 return @ptrCast(base_addr);
59 }
60
61 if (status == .SUCCESS) {
62 var region_size: windows.SIZE_T = 0;
63 _ = ntdll.NtFreeVirtualMemory(current_process, @ptrCast(&base_addr), &region_size, .{ .RELEASE = true });
64 }
65
66 const overalloc_len = n + alignment_bytes - page_size;
67 const page_aligned_len = mem.alignForward(usize, n, page_size);
68
69 base_addr = null;
70 size = overalloc_len;
71
72 status = ntdll.NtAllocateVirtualMemory(current_process, @ptrCast(&base_addr), 0, &size, .{ .RESERVE = true, .RESERVE_PLACEHOLDER = true }, .{ .NOACCESS = true });
73
74 if (status != .SUCCESS) return null;
75
76 const placeholder_addr = @intFromPtr(base_addr);
77 const aligned_addr = mem.alignForward(usize, placeholder_addr, alignment_bytes);
78 const prefix_size = aligned_addr - placeholder_addr;
79
80 if (prefix_size > 0) {
81 var prefix_base = base_addr;
82 var prefix_size_param: windows.SIZE_T = prefix_size;
83 _ = ntdll.NtFreeVirtualMemory(current_process, @ptrCast(&prefix_base), &prefix_size_param, .{ .RELEASE = true, .PRESERVE_PLACEHOLDER = true });
84 }
85
86 const suffix_start = aligned_addr + page_aligned_len;
87 const suffix_size = (placeholder_addr + overalloc_len) - suffix_start;
88 if (suffix_size > 0) {
89 var suffix_base = @as(?*anyopaque, @ptrFromInt(suffix_start));
90 var suffix_size_param: windows.SIZE_T = suffix_size;
91 _ = ntdll.NtFreeVirtualMemory(current_process, @ptrCast(&suffix_base), &suffix_size_param, .{ .RELEASE = true, .PRESERVE_PLACEHOLDER = true });
92 }
93
94 base_addr = @ptrFromInt(aligned_addr);
95 size = page_aligned_len;
96
97 status = ntdll.NtAllocateVirtualMemory(current_process, @ptrCast(&base_addr), 0, &size, .{ .COMMIT = true }, .{ .READWRITE = true });
98
99 if (status == .SUCCESS) {
100 return @ptrCast(base_addr);
101 }
102
103 base_addr = @as(?*anyopaque, @ptrFromInt(aligned_addr));
104 size = page_aligned_len;
105 _ = ntdll.NtFreeVirtualMemory(current_process, @ptrCast(&base_addr), &size, .{ .RELEASE = true });
106
107 return null;
108 }
109
110 const page_aligned_len = mem.alignForward(usize, n, page_size);
111 const max_drop_len = alignment_bytes -| page_size;
112 const overalloc_len = page_aligned_len + max_drop_len;
113
114 const maybe_unaligned_hint, const hint = blk: {
115 if (!enable_hints) break :blk .{ null, null };
116
117 const maybe_unaligned_hint = @atomicLoad(@TypeOf(addr_hint), &addr_hint, .unordered);
118
119 // For the very first mmap, let the kernel pick a good starting address;
120 // we'll begin doing our hinting from there.
121 if (maybe_unaligned_hint == null) break :blk .{ null, null };
122
123 // Aligning hint does not use mem.alignPointer, because it is slow.
124 // Aligning hint does not use mem.alignForward, because it asserts that there will be no overflow.
125 const hint: ?[*]align(page_size_min) u8 = @ptrFromInt(switch (stack_direction) {
126 .down => ((@intFromPtr(maybe_unaligned_hint) -% page_aligned_len) & ~(alignment_bytes - 1)) -% max_drop_len,
127 .up => (@intFromPtr(maybe_unaligned_hint) +% (alignment_bytes - 1)) & ~(alignment_bytes - 1),
128 });
129
130 break :blk .{ maybe_unaligned_hint, hint };
131 };
132
133 const slice = posix.mmap(
134 hint,
135 overalloc_len,
136 .{ .READ = true, .WRITE = true },
137 .{ .TYPE = .PRIVATE, .ANONYMOUS = true },
138 -1,
139 0,
140 ) catch return null;
141 const result_ptr = mem.alignPointer(slice.ptr, alignment_bytes).?;
142
143 // Unmap the extra bytes that were only requested in order to guarantee
144 // that the range of memory we were provided had a proper alignment in it
145 // somewhere. The extra bytes could be at the beginning, or end, or both.
146 const drop_len = result_ptr - slice.ptr;
147 if (drop_len != 0) posix.munmap(slice[0..drop_len]);
148 const remaining_len = overalloc_len - drop_len;
149 if (remaining_len > page_aligned_len) posix.munmap(@alignCast(result_ptr[page_aligned_len..remaining_len]));
150
151 if (enable_hints) {
152 const new_hint: [*]align(page_size_min) u8 = @alignCast(result_ptr + switch (stack_direction) {
153 .up => page_aligned_len,
154 .down => 0,
155 });
156 _ = @cmpxchgStrong(@TypeOf(addr_hint), &addr_hint, maybe_unaligned_hint, new_hint, .monotonic, .monotonic);
157 }
158
159 return result_ptr;
160}
161
162fn alloc(context: *anyopaque, n: usize, alignment: Alignment, ra: usize) ?[*]u8 {
163 _ = context;
164 _ = ra;
165 assert(n > 0);
166 return map(n, alignment);
167}
168
169fn resize(context: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, return_address: usize) bool {
170 _ = context;
171 _ = return_address;
172 return realloc(memory, alignment, new_len, false) != null;
173}
174
175fn remap(context: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, return_address: usize) ?[*]u8 {
176 _ = context;
177 _ = return_address;
178 return realloc(memory, alignment, new_len, true);
179}
180
181fn free(context: *anyopaque, memory: []u8, alignment: Alignment, return_address: usize) void {
182 _ = context;
183 _ = return_address;
184 _ = alignment;
185 return unmap(@alignCast(memory));
186}
187
188pub fn unmap(memory: []align(page_size_min) u8) void {
189 if (native_os == .windows) {
190 var base_addr: ?*anyopaque = memory.ptr;
191 var region_size: windows.SIZE_T = 0;
192 _ = ntdll.NtFreeVirtualMemory(windows.GetCurrentProcess(), @ptrCast(&base_addr), &region_size, .{ .RELEASE = true });
193 } else {
194 const page_aligned_len = mem.alignForward(usize, memory.len, std.heap.pageSize());
195 posix.munmap(memory.ptr[0..page_aligned_len]);
196 }
197}
198
199pub fn realloc(uncasted_memory: []u8, alignment: Alignment, new_len: usize, may_move: bool) ?[*]u8 {
200 const memory: []align(page_size_min) u8 = @alignCast(uncasted_memory);
201 const page_size = std.heap.pageSize();
202 if (alignment.toByteUnits() > page_size) return null;
203 const new_size_aligned = mem.alignForward(usize, new_len, page_size);
204
205 if (native_os == .windows) {
206 if (new_len <= memory.len) {
207 const base_addr = @intFromPtr(memory.ptr);
208 const old_addr_end = base_addr + memory.len;
209 const new_addr_end = mem.alignForward(usize, base_addr + new_len, page_size);
210 if (old_addr_end > new_addr_end) {
211 var decommit_addr: ?*anyopaque = @ptrFromInt(new_addr_end);
212 var decommit_size: windows.SIZE_T = old_addr_end - new_addr_end;
213
214 _ = ntdll.NtAllocateVirtualMemory(windows.GetCurrentProcess(), @ptrCast(&decommit_addr), 0, &decommit_size, .{ .RESET = true }, .{ .NOACCESS = true });
215 }
216 return memory.ptr;
217 }
218 const old_size_aligned = mem.alignForward(usize, memory.len, page_size);
219 if (new_size_aligned <= old_size_aligned) {
220 return memory.ptr;
221 }
222 return null;
223 }
224
225 const page_aligned_len = mem.alignForward(usize, memory.len, page_size);
226 if (new_size_aligned == page_aligned_len)
227 return memory.ptr;
228
229 // When the stack grows down, only use `mremap` if the allocation may move.
230 // Otherwise, we might grow the allocation and intrude on virtual address
231 // space which we want to keep available to the stack.
232 if (posix.MREMAP != void and (stack_direction == .up or may_move)) {
233 // TODO: if the next_mmap_addr_hint is within the remapped range, update it
234 const new_memory = posix.mremap(memory.ptr, page_aligned_len, new_size_aligned, .{ .MAYMOVE = may_move }, null) catch return null;
235 return new_memory.ptr;
236 }
237
238 if (new_size_aligned < page_aligned_len) {
239 const ptr = memory.ptr + new_size_aligned;
240 // TODO: if the next_mmap_addr_hint is within the unmapped range, update it
241 posix.munmap(@alignCast(ptr[0 .. page_aligned_len - new_size_aligned]));
242 return memory.ptr;
243 }
244
245 return null;
246}