authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-12-04 12:11:09+00:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-12-06 00:16:33+01:00
log4ce7b57e86ca1d5e71100c6cab5c75aad929a430
tree0c60b0db7bb89b8e5d0e7c6dae038adb001f0be2
parentea94ac52c531dd7e2ac2128f097fa036f5075403

std.heap: rework `c_allocator`

The main goal here was to avoid allocating padding and header space if `malloc` already guarantees the alignment we need via `max_align_t`. Previously, the compiler was using `std.heap.raw_c_allocator` as its GPA in some cases depending on `std.c.max_align_t`, but that's pretty fragile (it meant we had to encode our alignment requirements into `src/main.zig`!). Perhaps more importantly, that solution is unnecessarily restrictive: since Zig's `Allocator` API passes the `Alignment` not only to `alloc`, but also to `free` etc, we are able to use a different strategy depending on its value. So `c_allocator` can simply compare the requested align to `Alignment.of(std.c.max_align_t)`, and use a raw `malloc` call (no header needed!) if it will guarantee a suitable alignment (which, in practice, will be true the vast majority of the time). So in short, this makes `std.heap.c_allocator` more memory efficient, and probably removes any incentive to use `std.heap.raw_c_allocator`. I also refactored the `c_allocator` implementation while doing this, just to neaten things up a little.

2 files changed, 158 insertions(+), 104 deletions(-)

lib/std/heap.zig+157-96
......@@ -141,7 +141,19 @@ test defaultQueryPageSize {
141141 assert(std.math.isPowerOfTwo(defaultQueryPageSize()));
142142}
143143
144const CAllocator = struct {
144/// A wrapper around the C memory allocation API which supports the full `Allocator`
145/// interface, including arbitrary alignment. Simple `malloc` calls are used when
146/// possible, but large requested alignments may require larger buffers in order to
147/// satisfy the request. As well as `malloc`, `realloc`, and `free`, the extension
148/// functions `malloc_usable_size` and `posix_memalign` are used when available.
149///
150/// For an allocator that directly calls `malloc`/`realloc`/`free`, with no padding
151/// or special handling, see `raw_c_allocator`.
152pub const c_allocator: Allocator = .{
153 .ptr = undefined,
154 .vtable = &c_allocator_impl.vtable,
155};
156const c_allocator_impl = struct {
145157 comptime {
146158 if (!builtin.link_libc) {
147159 @compileError("C allocator is only available when linking against libc");
......@@ -155,67 +167,55 @@ const CAllocator = struct {
155167 .free = free,
156168 };
157169
158 pub const supports_malloc_size = @TypeOf(malloc_size) != void;
159 pub const malloc_size = if (@TypeOf(c.malloc_size) != void)
160 c.malloc_size
161 else if (@TypeOf(c.malloc_usable_size) != void)
162 c.malloc_usable_size
163 else if (@TypeOf(c._msize) != void)
164 c._msize
165 else {};
166
167 pub const supports_posix_memalign = switch (builtin.os.tag) {
168 .dragonfly, .netbsd, .freebsd, .illumos, .openbsd, .linux, .driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos, .serenity => true,
170 const have_posix_memalign = switch (builtin.os.tag) {
171 .dragonfly,
172 .netbsd,
173 .freebsd,
174 .illumos,
175 .openbsd,
176 .linux,
177 .driverkit,
178 .ios,
179 .maccatalyst,
180 .macos,
181 .tvos,
182 .visionos,
183 .watchos,
184 .serenity,
185 => true,
169186 else => false,
170187 };
171188
172 fn getHeader(ptr: [*]u8) *[*]u8 {
173 return @ptrCast(@alignCast(ptr - @sizeOf(usize)));
174 }
175
176 fn alignedAlloc(len: usize, alignment: Alignment) ?[*]u8 {
177 const alignment_bytes = alignment.toByteUnits();
178 if (supports_posix_memalign) {
179 // The posix_memalign only accepts alignment values that are a
180 // multiple of the pointer size
181 const effective_alignment = @max(alignment_bytes, @sizeOf(usize));
182
183 var aligned_ptr: ?*anyopaque = undefined;
184 if (c.posix_memalign(&aligned_ptr, effective_alignment, len) != 0)
185 return null;
186
187 return @ptrCast(aligned_ptr);
188 }
189
190 // Thin wrapper around regular malloc, overallocate to account for
191 // alignment padding and store the original malloc()'ed pointer before
192 // the aligned address.
193 const unaligned_ptr = @as([*]u8, @ptrCast(c.malloc(len + alignment_bytes - 1 + @sizeOf(usize)) orelse return null));
194 const unaligned_addr = @intFromPtr(unaligned_ptr);
195 const aligned_addr = mem.alignForward(usize, unaligned_addr + @sizeOf(usize), alignment_bytes);
196 const aligned_ptr = unaligned_ptr + (aligned_addr - unaligned_addr);
197 getHeader(aligned_ptr).* = unaligned_ptr;
198
199 return aligned_ptr;
200 }
201
202 fn alignedFree(ptr: [*]u8) void {
203 if (supports_posix_memalign) {
204 return c.free(ptr);
189 fn allocStrat(need_align: Alignment) union(enum) {
190 raw,
191 posix_memalign: if (have_posix_memalign) void else noreturn,
192 manual_align: if (have_posix_memalign) noreturn else void,
193 } {
194 // If `malloc` guarantees `need_align`, always prefer a raw allocation.
195 if (Alignment.compare(need_align, .lte, .of(c.max_align_t))) {
196 return .raw;
205197 }
206
207 const unaligned_ptr = getHeader(ptr).*;
208 c.free(unaligned_ptr);
198 // Use `posix_memalign` if available. Otherwise, we must manually align the allocation.
199 return if (have_posix_memalign) .posix_memalign else .manual_align;
209200 }
210201
211 fn alignedAllocSize(ptr: [*]u8) usize {
212 if (supports_posix_memalign) {
213 return CAllocator.malloc_size(ptr);
214 }
215
216 const unaligned_ptr = getHeader(ptr).*;
217 const delta = @intFromPtr(ptr) - @intFromPtr(unaligned_ptr);
218 return CAllocator.malloc_size(unaligned_ptr) - delta;
202 /// If `allocStrat(a) == .manual_align`, an allocation looks like this:
203 ///
204 /// unaligned_ptr hdr_ptr aligned_ptr
205 /// v v v
206 /// +---------------+--------+--------------+
207 /// | padding | header | usable bytes |
208 /// +---------------+--------+--------------+
209 ///
210 /// * `unaligned_ptr` is the raw return value of `malloc`.
211 /// * `aligned_ptr` is computed by aligning `unaligned_ptr` forward; it is what `alloc` returns.
212 /// * `hdr_ptr` points to a pointer-sized header directly before the usable space. This header
213 /// contains the value `unaligned_ptr`, so that we can pass it to `free` later. This is
214 /// necessary because the width of the padding is unknown.
215 ///
216 /// This function accepts `aligned_ptr` and offsets it backwards to return `hdr_ptr`.
217 fn manualAlignHeader(aligned_ptr: [*]u8) *[*]u8 {
218 return @ptrCast(@alignCast(aligned_ptr - @sizeOf(usize)));
219219 }
220220
221221 fn alloc(
......@@ -226,67 +226,120 @@ const CAllocator = struct {
226226 ) ?[*]u8 {
227227 _ = return_address;
228228 assert(len > 0);
229 return alignedAlloc(len, alignment);
229 switch (allocStrat(alignment)) {
230 .raw => {
231 // C only needs to respect `max_align_t` up to the allocation size due to object
232 // alignment rules. If necessary, extend the allocation size.
233 const actual_len = @max(len, @alignOf(std.c.max_align_t));
234 const ptr = c.malloc(actual_len) orelse return null;
235 assert(alignment.check(@intFromPtr(ptr)));
236 return @ptrCast(ptr);
237 },
238 .posix_memalign => {
239 // The posix_memalign only accepts alignment values that are a
240 // multiple of the pointer size
241 const effective_alignment = @max(alignment.toByteUnits(), @sizeOf(usize));
242 var aligned_ptr: ?*anyopaque = undefined;
243 if (c.posix_memalign(&aligned_ptr, effective_alignment, len) != 0) {
244 return null;
245 }
246 assert(alignment.check(@intFromPtr(aligned_ptr)));
247 return @ptrCast(aligned_ptr);
248 },
249 .manual_align => {
250 // Overallocate to account for alignment padding and store the original pointer
251 // returned by `malloc` before the aligned address.
252 const padded_len = len + @sizeOf(usize) + alignment.toByteUnits() - 1;
253 const unaligned_ptr: [*]u8 = @ptrCast(c.malloc(padded_len) orelse return null);
254 const unaligned_addr = @intFromPtr(unaligned_ptr);
255 const aligned_addr = alignment.forward(unaligned_addr + @sizeOf(usize));
256 const aligned_ptr = unaligned_ptr + (aligned_addr - unaligned_addr);
257 manualAlignHeader(aligned_ptr).* = unaligned_ptr;
258 return aligned_ptr;
259 },
260 }
230261 }
231262
232263 fn resize(
233264 _: *anyopaque,
234 buf: []u8,
265 memory: []u8,
235266 alignment: Alignment,
236267 new_len: usize,
237268 return_address: usize,
238269 ) bool {
239 _ = alignment;
240270 _ = return_address;
241 if (new_len <= buf.len) {
242 return true;
243 }
244 if (CAllocator.supports_malloc_size) {
245 const full_len = alignedAllocSize(buf.ptr);
246 if (new_len <= full_len) {
247 return true;
248 }
271 assert(new_len > 0);
272 if (new_len <= memory.len) {
273 return true; // in-place shrink always works
249274 }
250 return false;
275 const mallocSize = func: {
276 if (@TypeOf(c.malloc_size) != void) break :func c.malloc_size;
277 if (@TypeOf(c.malloc_usable_size) != void) break :func c.malloc_usable_size;
278 if (@TypeOf(c._msize) != void) break :func c._msize;
279 return false; // we don't know how much space is actually available
280 };
281 const usable_len: usize = switch (allocStrat(alignment)) {
282 .raw, .posix_memalign => mallocSize(memory.ptr),
283 .manual_align => usable_len: {
284 const unaligned_ptr = manualAlignHeader(memory.ptr).*;
285 const full_len = mallocSize(unaligned_ptr);
286 const padding = @intFromPtr(memory.ptr) - @intFromPtr(unaligned_ptr);
287 break :usable_len full_len - padding;
288 },
289 };
290 return new_len <= usable_len;
251291 }
252292
253293 fn remap(
254 context: *anyopaque,
294 ctx: *anyopaque,
255295 memory: []u8,
256296 alignment: Alignment,
257297 new_len: usize,
258298 return_address: usize,
259299 ) ?[*]u8 {
260 // realloc would potentially return a new allocation that does not
261 // respect the original alignment.
262 return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
300 assert(new_len > 0);
301 // Prefer resizing in-place if possible, since `realloc` could be expensive even if legal.
302 if (resize(ctx, memory, alignment, new_len, return_address)) {
303 return memory.ptr;
304 }
305 switch (allocStrat(alignment)) {
306 .raw => {
307 // `malloc` and friends guarantee the required alignment, so we can try `realloc`.
308 // C only needs to respect `max_align_t` up to the allocation size due to object
309 // alignment rules. If necessary, extend the allocation size.
310 const actual_len = @max(new_len, @alignOf(std.c.max_align_t));
311 const new_ptr = c.realloc(memory.ptr, actual_len) orelse return null;
312 assert(alignment.check(@intFromPtr(new_ptr)));
313 return @ptrCast(new_ptr);
314 },
315 .posix_memalign, .manual_align => {
316 // `realloc` would potentially return a new allocation which does not respect
317 // the original alignment, so we can't do anything more.
318 return null;
319 },
320 }
263321 }
264322
265323 fn free(
266324 _: *anyopaque,
267 buf: []u8,
325 memory: []u8,
268326 alignment: Alignment,
269327 return_address: usize,
270328 ) void {
271 _ = alignment;
272329 _ = return_address;
273 alignedFree(buf.ptr);
330 switch (allocStrat(alignment)) {
331 .raw, .posix_memalign => c.free(memory.ptr),
332 .manual_align => c.free(manualAlignHeader(memory.ptr).*),
333 }
274334 }
275335};
276336
277/// Supports the full Allocator interface, including alignment, and exploiting
278/// `malloc_usable_size` if available. For an allocator that directly calls
279/// `malloc`/`free`, see `raw_c_allocator`.
280pub const c_allocator: Allocator = .{
281 .ptr = undefined,
282 .vtable = &CAllocator.vtable,
283};
284
285/// Asserts allocations are within `@alignOf(std.c.max_align_t)` and directly
286/// calls `malloc`/`free`. Does not attempt to utilize `malloc_usable_size`.
287/// This allocator is safe to use as the backing allocator with
288/// `ArenaAllocator` for example and is more optimal in such a case than
289/// `c_allocator`.
337/// Asserts that allocations have alignments which `malloc` can satisfy. This means that
338/// the requested alignment is no greater than `@min(@alignOf(std.c.max_align_t), size)`.
339///
340/// This allocator is rarely appropriate to use. In general, prefer `c_allocator`, which
341/// does not have any special requirements of its input, but is still highly efficient for
342/// allocation requests which obey `malloc` alignment rules.
290343pub const raw_c_allocator: Allocator = .{
291344 .ptr = undefined,
292345 .vtable = &raw_c_allocator_vtable,
......@@ -306,13 +359,20 @@ fn rawCAlloc(
306359) ?[*]u8 {
307360 _ = context;
308361 _ = return_address;
309 assert(alignment.compare(.lte, .of(std.c.max_align_t)));
310 // Note that this pointer cannot be aligncasted to max_align_t because if
311 // len is < max_align_t then the alignment can be smaller. For example, if
312 // max_align_t is 16, but the user requests 8 bytes, there is no built-in
313 // type in C that is size 8 and has 16 byte alignment, so the alignment may
314 // be 8 bytes rather than 16. Similarly if only 1 byte is requested, malloc
315 // is allowed to return a 1-byte aligned pointer.
362 // `std.c.max_align_t` isn't the whole story, because if `len` is smaller than
363 // every C type with alignment `max_align_t`, the allocation can be less-aligned.
364 // The implementation need only guarantee that any type of length `len` would be
365 // suitably aligned.
366 //
367 // For instance, if `len == 8` and `alignment == .@"16"`, then `malloc` may not
368 // fulfil this request, because there is necessarily no C type with 8-byte size
369 // but 16-byte alignment.
370 //
371 // In theory, the resulting rule here would be target-specific, but in practice,
372 // the smallest type with an alignment of `max_align_t` has the same size (it's
373 // usually `c_longdouble`), so we can just check that `alignment <= len`.
374 assert(alignment.toByteUnits() <= len);
375 assert(Alignment.compare(alignment, .lte, .of(std.c.max_align_t)));
316376 return @ptrCast(c.malloc(len));
317377}
318378
......@@ -339,8 +399,9 @@ fn rawCRemap(
339399 return_address: usize,
340400) ?[*]u8 {
341401 _ = context;
342 _ = alignment;
343402 _ = return_address;
403 // See `rawCMalloc` for an explanation of this `assert` call.
404 assert(alignment.toByteUnits() <= new_len);
344405 return @ptrCast(c.realloc(memory.ptr, new_len));
345406}
346407
src/main.zig+1-8
......@@ -167,14 +167,7 @@ pub fn main() anyerror!void {
167167 const gpa, const is_debug = gpa: {
168168 if (build_options.debug_gpa) break :gpa .{ debug_allocator.allocator(), true };
169169 if (native_os == .wasi) break :gpa .{ std.heap.wasm_allocator, false };
170 if (builtin.link_libc) {
171 // We would prefer to use raw libc allocator here, but cannot use
172 // it if it won't support the alignment we need.
173 if (@alignOf(std.c.max_align_t) < @max(@alignOf(i128), std.atomic.cache_line)) {
174 break :gpa .{ std.heap.c_allocator, false };
175 }
176 break :gpa .{ std.heap.raw_c_allocator, false };
177 }
170 if (builtin.link_libc) break :gpa .{ std.heap.c_allocator, false };
178171 break :gpa switch (builtin.mode) {
179172 .Debug, .ReleaseSafe => .{ debug_allocator.allocator(), true },
180173 .ReleaseFast, .ReleaseSmall => .{ std.heap.smp_allocator, false },