authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-12-04 12:11:09+00:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-12-06 00:16:33+01:00
log4ce7b57e86ca1d5e71100c6cab5c75aad929a430
tree0c60b0db7bb89b8e5d0e7c6dae038adb001f0be2
parentea94ac52c531dd7e2ac2128f097fa036f5075403

std.heap: rework `c_allocator`

The main goal here was to avoid allocating padding and header space if `malloc` already guarantees the alignment we need via `max_align_t`. Previously, the compiler was using `std.heap.raw_c_allocator` as its GPA in some cases depending on `std.c.max_align_t`, but that's pretty fragile (it meant we had to encode our alignment requirements into `src/main.zig`!). Perhaps more importantly, that solution is unnecessarily restrictive: since Zig's `Allocator` API passes the `Alignment` not only to `alloc`, but also to `free` etc, we are able to use a different strategy depending on its value. So `c_allocator` can simply compare the requested align to `Alignment.of(std.c.max_align_t)`, and use a raw `malloc` call (no header needed!) if it will guarantee a suitable alignment (which, in practice, will be true the vast majority of the time). So in short, this makes `std.heap.c_allocator` more memory efficient, and probably removes any incentive to use `std.heap.raw_c_allocator`. I also refactored the `c_allocator` implementation while doing this, just to neaten things up a little.

2 files changed, 158 insertions(+), 104 deletions(-)

lib/std/heap.zig+157-96
...@@ -141,7 +141,19 @@ test defaultQueryPageSize {...@@ -141,7 +141,19 @@ test defaultQueryPageSize {
141 assert(std.math.isPowerOfTwo(defaultQueryPageSize()));141 assert(std.math.isPowerOfTwo(defaultQueryPageSize()));
142}142}
143143
144const CAllocator = struct {144/// A wrapper around the C memory allocation API which supports the full `Allocator`
145/// interface, including arbitrary alignment. Simple `malloc` calls are used when
146/// possible, but large requested alignments may require larger buffers in order to
147/// satisfy the request. As well as `malloc`, `realloc`, and `free`, the extension
148/// functions `malloc_usable_size` and `posix_memalign` are used when available.
149///
150/// For an allocator that directly calls `malloc`/`realloc`/`free`, with no padding
151/// or special handling, see `raw_c_allocator`.
152pub const c_allocator: Allocator = .{
153 .ptr = undefined,
154 .vtable = &c_allocator_impl.vtable,
155};
156const c_allocator_impl = struct {
145 comptime {157 comptime {
146 if (!builtin.link_libc) {158 if (!builtin.link_libc) {
147 @compileError("C allocator is only available when linking against libc");159 @compileError("C allocator is only available when linking against libc");
...@@ -155,67 +167,55 @@ const CAllocator = struct {...@@ -155,67 +167,55 @@ const CAllocator = struct {
155 .free = free,167 .free = free,
156 };168 };
157169
158 pub const supports_malloc_size = @TypeOf(malloc_size) != void;170 const have_posix_memalign = switch (builtin.os.tag) {
159 pub const malloc_size = if (@TypeOf(c.malloc_size) != void)171 .dragonfly,
160 c.malloc_size172 .netbsd,
161 else if (@TypeOf(c.malloc_usable_size) != void)173 .freebsd,
162 c.malloc_usable_size174 .illumos,
163 else if (@TypeOf(c._msize) != void)175 .openbsd,
164 c._msize176 .linux,
165 else {};177 .driverkit,
166178 .ios,
167 pub const supports_posix_memalign = switch (builtin.os.tag) {179 .maccatalyst,
168 .dragonfly, .netbsd, .freebsd, .illumos, .openbsd, .linux, .driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos, .serenity => true,180 .macos,
181 .tvos,
182 .visionos,
183 .watchos,
184 .serenity,
185 => true,
169 else => false,186 else => false,
170 };187 };
171188
172 fn getHeader(ptr: [*]u8) *[*]u8 {189 fn allocStrat(need_align: Alignment) union(enum) {
173 return @ptrCast(@alignCast(ptr - @sizeOf(usize)));190 raw,
174 }191 posix_memalign: if (have_posix_memalign) void else noreturn,
175192 manual_align: if (have_posix_memalign) noreturn else void,
176 fn alignedAlloc(len: usize, alignment: Alignment) ?[*]u8 {193 } {
177 const alignment_bytes = alignment.toByteUnits();194 // If `malloc` guarantees `need_align`, always prefer a raw allocation.
178 if (supports_posix_memalign) {195 if (Alignment.compare(need_align, .lte, .of(c.max_align_t))) {
179 // The posix_memalign only accepts alignment values that are a196 return .raw;
180 // multiple of the pointer size
181 const effective_alignment = @max(alignment_bytes, @sizeOf(usize));
182
183 var aligned_ptr: ?*anyopaque = undefined;
184 if (c.posix_memalign(&aligned_ptr, effective_alignment, len) != 0)
185 return null;
186
187 return @ptrCast(aligned_ptr);
188 }
189
190 // Thin wrapper around regular malloc, overallocate to account for
191 // alignment padding and store the original malloc()'ed pointer before
192 // the aligned address.
193 const unaligned_ptr = @as([*]u8, @ptrCast(c.malloc(len + alignment_bytes - 1 + @sizeOf(usize)) orelse return null));
194 const unaligned_addr = @intFromPtr(unaligned_ptr);
195 const aligned_addr = mem.alignForward(usize, unaligned_addr + @sizeOf(usize), alignment_bytes);
196 const aligned_ptr = unaligned_ptr + (aligned_addr - unaligned_addr);
197 getHeader(aligned_ptr).* = unaligned_ptr;
198
199 return aligned_ptr;
200 }
201
202 fn alignedFree(ptr: [*]u8) void {
203 if (supports_posix_memalign) {
204 return c.free(ptr);
205 }197 }
206198 // Use `posix_memalign` if available. Otherwise, we must manually align the allocation.
207 const unaligned_ptr = getHeader(ptr).*;199 return if (have_posix_memalign) .posix_memalign else .manual_align;
208 c.free(unaligned_ptr);
209 }200 }
210201
211 fn alignedAllocSize(ptr: [*]u8) usize {202 /// If `allocStrat(a) == .manual_align`, an allocation looks like this:
212 if (supports_posix_memalign) {203 ///
213 return CAllocator.malloc_size(ptr);204 /// unaligned_ptr hdr_ptr aligned_ptr
214 }205 /// v v v
215206 /// +---------------+--------+--------------+
216 const unaligned_ptr = getHeader(ptr).*;207 /// | padding | header | usable bytes |
217 const delta = @intFromPtr(ptr) - @intFromPtr(unaligned_ptr);208 /// +---------------+--------+--------------+
218 return CAllocator.malloc_size(unaligned_ptr) - delta;209 ///
210 /// * `unaligned_ptr` is the raw return value of `malloc`.
211 /// * `aligned_ptr` is computed by aligning `unaligned_ptr` forward; it is what `alloc` returns.
212 /// * `hdr_ptr` points to a pointer-sized header directly before the usable space. This header
213 /// contains the value `unaligned_ptr`, so that we can pass it to `free` later. This is
214 /// necessary because the width of the padding is unknown.
215 ///
216 /// This function accepts `aligned_ptr` and offsets it backwards to return `hdr_ptr`.
217 fn manualAlignHeader(aligned_ptr: [*]u8) *[*]u8 {
218 return @ptrCast(@alignCast(aligned_ptr - @sizeOf(usize)));
219 }219 }
220220
221 fn alloc(221 fn alloc(
...@@ -226,67 +226,120 @@ const CAllocator = struct {...@@ -226,67 +226,120 @@ const CAllocator = struct {
226 ) ?[*]u8 {226 ) ?[*]u8 {
227 _ = return_address;227 _ = return_address;
228 assert(len > 0);228 assert(len > 0);
229 return alignedAlloc(len, alignment);229 switch (allocStrat(alignment)) {
230 .raw => {
231 // C only needs to respect `max_align_t` up to the allocation size due to object
232 // alignment rules. If necessary, extend the allocation size.
233 const actual_len = @max(len, @alignOf(std.c.max_align_t));
234 const ptr = c.malloc(actual_len) orelse return null;
235 assert(alignment.check(@intFromPtr(ptr)));
236 return @ptrCast(ptr);
237 },
238 .posix_memalign => {
239 // The posix_memalign only accepts alignment values that are a
240 // multiple of the pointer size
241 const effective_alignment = @max(alignment.toByteUnits(), @sizeOf(usize));
242 var aligned_ptr: ?*anyopaque = undefined;
243 if (c.posix_memalign(&aligned_ptr, effective_alignment, len) != 0) {
244 return null;
245 }
246 assert(alignment.check(@intFromPtr(aligned_ptr)));
247 return @ptrCast(aligned_ptr);
248 },
249 .manual_align => {
250 // Overallocate to account for alignment padding and store the original pointer
251 // returned by `malloc` before the aligned address.
252 const padded_len = len + @sizeOf(usize) + alignment.toByteUnits() - 1;
253 const unaligned_ptr: [*]u8 = @ptrCast(c.malloc(padded_len) orelse return null);
254 const unaligned_addr = @intFromPtr(unaligned_ptr);
255 const aligned_addr = alignment.forward(unaligned_addr + @sizeOf(usize));
256 const aligned_ptr = unaligned_ptr + (aligned_addr - unaligned_addr);
257 manualAlignHeader(aligned_ptr).* = unaligned_ptr;
258 return aligned_ptr;
259 },
260 }
230 }261 }
231262
232 fn resize(263 fn resize(
233 _: *anyopaque,264 _: *anyopaque,
234 buf: []u8,265 memory: []u8,
235 alignment: Alignment,266 alignment: Alignment,
236 new_len: usize,267 new_len: usize,
237 return_address: usize,268 return_address: usize,
238 ) bool {269 ) bool {
239 _ = alignment;
240 _ = return_address;270 _ = return_address;
241 if (new_len <= buf.len) {271 assert(new_len > 0);
242 return true;272 if (new_len <= memory.len) {
243 }273 return true; // in-place shrink always works
244 if (CAllocator.supports_malloc_size) {
245 const full_len = alignedAllocSize(buf.ptr);
246 if (new_len <= full_len) {
247 return true;
248 }
249 }274 }
250 return false;275 const mallocSize = func: {
276 if (@TypeOf(c.malloc_size) != void) break :func c.malloc_size;
277 if (@TypeOf(c.malloc_usable_size) != void) break :func c.malloc_usable_size;
278 if (@TypeOf(c._msize) != void) break :func c._msize;
279 return false; // we don't know how much space is actually available
280 };
281 const usable_len: usize = switch (allocStrat(alignment)) {
282 .raw, .posix_memalign => mallocSize(memory.ptr),
283 .manual_align => usable_len: {
284 const unaligned_ptr = manualAlignHeader(memory.ptr).*;
285 const full_len = mallocSize(unaligned_ptr);
286 const padding = @intFromPtr(memory.ptr) - @intFromPtr(unaligned_ptr);
287 break :usable_len full_len - padding;
288 },
289 };
290 return new_len <= usable_len;
251 }291 }
252292
253 fn remap(293 fn remap(
254 context: *anyopaque,294 ctx: *anyopaque,
255 memory: []u8,295 memory: []u8,
256 alignment: Alignment,296 alignment: Alignment,
257 new_len: usize,297 new_len: usize,
258 return_address: usize,298 return_address: usize,
259 ) ?[*]u8 {299 ) ?[*]u8 {
260 // realloc would potentially return a new allocation that does not300 assert(new_len > 0);
261 // respect the original alignment.301 // Prefer resizing in-place if possible, since `realloc` could be expensive even if legal.
262 return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;302 if (resize(ctx, memory, alignment, new_len, return_address)) {
303 return memory.ptr;
304 }
305 switch (allocStrat(alignment)) {
306 .raw => {
307 // `malloc` and friends guarantee the required alignment, so we can try `realloc`.
308 // C only needs to respect `max_align_t` up to the allocation size due to object
309 // alignment rules. If necessary, extend the allocation size.
310 const actual_len = @max(new_len, @alignOf(std.c.max_align_t));
311 const new_ptr = c.realloc(memory.ptr, actual_len) orelse return null;
312 assert(alignment.check(@intFromPtr(new_ptr)));
313 return @ptrCast(new_ptr);
314 },
315 .posix_memalign, .manual_align => {
316 // `realloc` would potentially return a new allocation which does not respect
317 // the original alignment, so we can't do anything more.
318 return null;
319 },
320 }
263 }321 }
264322
265 fn free(323 fn free(
266 _: *anyopaque,324 _: *anyopaque,
267 buf: []u8,325 memory: []u8,
268 alignment: Alignment,326 alignment: Alignment,
269 return_address: usize,327 return_address: usize,
270 ) void {328 ) void {
271 _ = alignment;
272 _ = return_address;329 _ = return_address;
273 alignedFree(buf.ptr);330 switch (allocStrat(alignment)) {
331 .raw, .posix_memalign => c.free(memory.ptr),
332 .manual_align => c.free(manualAlignHeader(memory.ptr).*),
333 }
274 }334 }
275};335};
276336
277/// Supports the full Allocator interface, including alignment, and exploiting337/// Asserts that allocations have alignments which `malloc` can satisfy. This means that
278/// `malloc_usable_size` if available. For an allocator that directly calls338/// the requested alignment is no greater than `@min(@alignOf(std.c.max_align_t), size)`.
279/// `malloc`/`free`, see `raw_c_allocator`.339///
280pub const c_allocator: Allocator = .{340/// This allocator is rarely appropriate to use. In general, prefer `c_allocator`, which
281 .ptr = undefined,341/// does not have any special requirements of its input, but is still highly efficient for
282 .vtable = &CAllocator.vtable,342/// allocation requests which obey `malloc` alignment rules.
283};
284
285/// Asserts allocations are within `@alignOf(std.c.max_align_t)` and directly
286/// calls `malloc`/`free`. Does not attempt to utilize `malloc_usable_size`.
287/// This allocator is safe to use as the backing allocator with
288/// `ArenaAllocator` for example and is more optimal in such a case than
289/// `c_allocator`.
290pub const raw_c_allocator: Allocator = .{343pub const raw_c_allocator: Allocator = .{
291 .ptr = undefined,344 .ptr = undefined,
292 .vtable = &raw_c_allocator_vtable,345 .vtable = &raw_c_allocator_vtable,
...@@ -306,13 +359,20 @@ fn rawCAlloc(...@@ -306,13 +359,20 @@ fn rawCAlloc(
306) ?[*]u8 {359) ?[*]u8 {
307 _ = context;360 _ = context;
308 _ = return_address;361 _ = return_address;
309 assert(alignment.compare(.lte, .of(std.c.max_align_t)));362 // `std.c.max_align_t` isn't the whole story, because if `len` is smaller than
310 // Note that this pointer cannot be aligncasted to max_align_t because if363 // every C type with alignment `max_align_t`, the allocation can be less-aligned.
311 // len is < max_align_t then the alignment can be smaller. For example, if364 // The implementation need only guarantee that any type of length `len` would be
312 // max_align_t is 16, but the user requests 8 bytes, there is no built-in365 // suitably aligned.
313 // type in C that is size 8 and has 16 byte alignment, so the alignment may366 //
314 // be 8 bytes rather than 16. Similarly if only 1 byte is requested, malloc367 // For instance, if `len == 8` and `alignment == .@"16"`, then `malloc` may not
315 // is allowed to return a 1-byte aligned pointer.368 // fulfil this request, because there is necessarily no C type with 8-byte size
369 // but 16-byte alignment.
370 //
371 // In theory, the resulting rule here would be target-specific, but in practice,
372 // the smallest type with an alignment of `max_align_t` has the same size (it's
373 // usually `c_longdouble`), so we can just check that `alignment <= len`.
374 assert(alignment.toByteUnits() <= len);
375 assert(Alignment.compare(alignment, .lte, .of(std.c.max_align_t)));
316 return @ptrCast(c.malloc(len));376 return @ptrCast(c.malloc(len));
317}377}
318378
...@@ -339,8 +399,9 @@ fn rawCRemap(...@@ -339,8 +399,9 @@ fn rawCRemap(
339 return_address: usize,399 return_address: usize,
340) ?[*]u8 {400) ?[*]u8 {
341 _ = context;401 _ = context;
342 _ = alignment;
343 _ = return_address;402 _ = return_address;
403 // See `rawCMalloc` for an explanation of this `assert` call.
404 assert(alignment.toByteUnits() <= new_len);
344 return @ptrCast(c.realloc(memory.ptr, new_len));405 return @ptrCast(c.realloc(memory.ptr, new_len));
345}406}
346407
src/main.zig+1-8
...@@ -167,14 +167,7 @@ pub fn main() anyerror!void {...@@ -167,14 +167,7 @@ pub fn main() anyerror!void {
167 const gpa, const is_debug = gpa: {167 const gpa, const is_debug = gpa: {
168 if (build_options.debug_gpa) break :gpa .{ debug_allocator.allocator(), true };168 if (build_options.debug_gpa) break :gpa .{ debug_allocator.allocator(), true };
169 if (native_os == .wasi) break :gpa .{ std.heap.wasm_allocator, false };169 if (native_os == .wasi) break :gpa .{ std.heap.wasm_allocator, false };
170 if (builtin.link_libc) {170 if (builtin.link_libc) break :gpa .{ std.heap.c_allocator, false };
171 // We would prefer to use raw libc allocator here, but cannot use
172 // it if it won't support the alignment we need.
173 if (@alignOf(std.c.max_align_t) < @max(@alignOf(i128), std.atomic.cache_line)) {
174 break :gpa .{ std.heap.c_allocator, false };
175 }
176 break :gpa .{ std.heap.raw_c_allocator, false };
177 }
178 break :gpa switch (builtin.mode) {171 break :gpa switch (builtin.mode) {
179 .Debug, .ReleaseSafe => .{ debug_allocator.allocator(), true },172 .Debug, .ReleaseSafe => .{ debug_allocator.allocator(), true },
180 .ReleaseFast, .ReleaseSmall => .{ std.heap.smp_allocator, false },173 .ReleaseFast, .ReleaseSmall => .{ std.heap.smp_allocator, false },