From 6ccabbd4e558dd0d56c471408c4cf29e0992dc1c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Feb 2026 19:28:43 -0800 Subject: [PATCH] std: brk allocator for single-threaded mode --- lib/c/malloc.zig | 6 +- lib/std/heap.zig | 29 +-- .../{WasmAllocator.zig => BrkAllocator.zig} | 96 ++++++---- lib/std/heap/sbrk_allocator.zig | 180 ------------------ lib/std/os/linux.zig | 4 + 5 files changed, 82 insertions(+), 233 deletions(-) rename lib/std/heap/{WasmAllocator.zig => BrkAllocator.zig} (79%) delete mode 100644 lib/std/heap/sbrk_allocator.zig diff --git a/lib/c/malloc.zig b/lib/c/malloc.zig index 72e7621b61057fdb1c31737c0af3c0b0262d53c0..4b8ba77b5ecef1d569c54f362285f0a647b2b596 100644 --- a/lib/c/malloc.zig +++ b/lib/c/malloc.zig @@ -1,7 +1,7 @@ -//! Based on wrapping a stateless Zig Allocator implementation, apropriate for: +//! Based on wrapping a stateless Zig Allocator implementation, appropriate for: //! - ReleaseFast and ReleaseSmall optimization modes, with multi-threading //! enabled. -//! - WebAssembly in single-threaded mode. +//! - WebAssembly or Linux in single-threaded release modes. //! //! Because the libc APIs don't have client alignment and size tracking, in //! order to take advantage of Zig allocator implementations, additional @@ -40,7 +40,7 @@ const no_context: *anyopaque = undefined; const no_ra: usize = undefined; const vtable = switch (builtin.cpu.arch) { .wasm32, .wasm64 => std.heap.WasmAllocator.vtable, - else => std.heap.SmpAllocator.vtable, + else => if (builtin.single_threaded) std.heap.BrkAllocator.vtable else std.heap.SmpAllocator.vtable, }; /// Needed because libc memory allocators don't provide old alignment and size diff --git a/lib/std/heap.zig b/lib/std/heap.zig index ba7ef0aabf4c77ef905aa14836ab0e91102b0ba1..562d0846efde69bf0d58b926896b6fc8100b3db2 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -13,9 +13,9 @@ pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator; pub const SmpAllocator = @import("heap/SmpAllocator.zig"); pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig"); pub const PageAllocator = @import("heap/PageAllocator.zig"); -pub const SbrkAllocator = @import("heap/sbrk_allocator.zig").SbrkAllocator; pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig"); -pub const WasmAllocator = @import("heap/WasmAllocator.zig"); +pub const WasmAllocator = if (builtin.single_threaded) BrkAllocator else @compileError("unimplemented"); +pub const BrkAllocator = @import("heap/BrkAllocator.zig"); pub const DebugAllocatorConfig = @import("heap/debug_allocator.zig").Config; pub const DebugAllocator = @import("heap/debug_allocator.zig").DebugAllocator; @@ -356,9 +356,6 @@ pub const page_allocator: Allocator = if (@hasDecl(root, "os") and else if (builtin.target.cpu.arch.isWasm()) .{ .ptr = undefined, .vtable = &WasmAllocator.vtable, -} else if (builtin.target.os.tag == .plan9) .{ - .ptr = undefined, - .vtable = &SbrkAllocator(std.os.plan9.sbrk).vtable, } else .{ .ptr = undefined, .vtable = &PageAllocator.vtable, @@ -369,16 +366,18 @@ pub const smp_allocator: Allocator = .{ .vtable = &SmpAllocator.vtable, }; -/// This allocator is fast, small, and specific to WebAssembly. In the future, -/// this will be the implementation automatically selected by -/// `GeneralPurposeAllocator` when compiling in `ReleaseSmall` mode for wasm32 -/// and wasm64 architectures. -/// Until then, it is available here to play with. +/// This allocator is fast, small, and specific to WebAssembly. pub const wasm_allocator: Allocator = .{ .ptr = undefined, .vtable = &WasmAllocator.vtable, }; +/// Supports single-threaded WebAssembly and Linux. +pub const brk_allocator: Allocator = .{ + .ptr = undefined, + .vtable = &BrkAllocator.vtable, +}; + /// Returns a `StackFallbackAllocator` allocating using either a /// `FixedBufferAllocator` on an array of size `size` and falling back to /// `fallback_allocator` if that fails. @@ -1014,9 +1013,11 @@ test { _ = GeneralPurposeAllocator; _ = FixedBufferAllocator; _ = ThreadSafeAllocator; - _ = SbrkAllocator; - if (builtin.target.cpu.arch.isWasm()) { - _ = WasmAllocator; + if (builtin.single_threaded) { + if (builtin.cpu.arch.isWasm() or (builtin.os.tag == .linux and !builtin.link_libc)) { + _ = brk_allocator; + } + } else { + _ = smp_allocator; } - if (!builtin.single_threaded) _ = smp_allocator; } diff --git a/lib/std/heap/WasmAllocator.zig b/lib/std/heap/BrkAllocator.zig similarity index 79% rename from lib/std/heap/WasmAllocator.zig rename to lib/std/heap/BrkAllocator.zig index 2045d7c30373742e15afe2bb800a97f86be0d847..91b7230c9d30423d53afb3b6946d9c940990716e 100644 --- a/lib/std/heap/WasmAllocator.zig +++ b/lib/std/heap/BrkAllocator.zig @@ -1,20 +1,30 @@ -const std = @import("../std.zig"); +//! Supports single-threaded targets that have a sbrk-like primitive which includes +//! Linux and WebAssembly. +//! +//! On Linux, assumes exclusive access to the brk syscall. +const BrkAllocator = @This(); const builtin = @import("builtin"); + +const std = @import("../std.zig"); const Allocator = std.mem.Allocator; -const mem = std.mem; +const Alignment = std.mem.Alignment; const assert = std.debug.assert; -const wasm = std.wasm; const math = std.math; +const page_size_max = std.heap.page_size_max; comptime { - if (!builtin.target.cpu.arch.isWasm()) { - @compileError("only available for wasm32 arch"); - } - if (!builtin.single_threaded) { - @compileError("TODO implement support for multi-threaded wasm"); - } + if (!builtin.single_threaded) @compileError("unsupported"); } +next_addrs: [size_class_count]usize = @splat(0), +/// For each size class, points to the freed pointer. +frees: [size_class_count]usize = @splat(0), +/// For each big size class, points to the freed pointer. +big_frees: [big_size_class_count]usize = @splat(0), +prev_brk: usize = 0, + +var global: BrkAllocator = .{}; + pub const vtable: Allocator.VTable = .{ .alloc = alloc, .resize = resize, @@ -27,9 +37,13 @@ pub const Error = Allocator.Error; const max_usize = math.maxInt(usize); const ushift = math.Log2Int(usize); const bigpage_size = 64 * 1024; -const pages_per_bigpage = bigpage_size / wasm.page_size; +const pages_per_bigpage = bigpage_size / page_size_max; const bigpage_count = max_usize / bigpage_size; +comptime { + assert(bigpage_size >= page_size_max); +} + /// Because of storing free list pointers, the minimum size class is 3. const min_class = math.log2(math.ceilPowerOfTwoAssert(usize, 1 + @sizeOf(usize))); const size_class_count = math.log2(bigpage_size) - min_class; @@ -39,13 +53,7 @@ const size_class_count = math.log2(bigpage_size) - min_class; /// etc. const big_size_class_count = math.log2(bigpage_count); -var next_addrs: [size_class_count]usize = @splat(0); -/// For each size class, points to the freed pointer. -var frees: [size_class_count]usize = @splat(0); -/// For each big size class, points to the freed pointer. -var big_frees: [big_size_class_count]usize = @splat(0); - -fn alloc(ctx: *anyopaque, len: usize, alignment: mem.Alignment, return_address: usize) ?[*]u8 { +fn alloc(ctx: *anyopaque, len: usize, alignment: Alignment, return_address: usize) ?[*]u8 { _ = ctx; _ = return_address; // Make room for the freelist next pointer. @@ -54,24 +62,24 @@ fn alloc(ctx: *anyopaque, len: usize, alignment: mem.Alignment, return_address: const class = math.log2(slot_size) - min_class; if (class < size_class_count) { const addr = a: { - const top_free_ptr = frees[class]; + const top_free_ptr = global.frees[class]; if (top_free_ptr != 0) { const node: *usize = @ptrFromInt(top_free_ptr + (slot_size - @sizeOf(usize))); - frees[class] = node.*; + global.frees[class] = node.*; break :a top_free_ptr; } - const next_addr = next_addrs[class]; - if (next_addr % wasm.page_size == 0) { + const next_addr = global.next_addrs[class]; + if (next_addr % page_size_max == 0) { const addr = allocBigPages(1); if (addr == 0) return null; //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{ // slot_size, class, addr, //}); - next_addrs[class] = addr + slot_size; + global.next_addrs[class] = addr + slot_size; break :a addr; } else { - next_addrs[class] = next_addr + slot_size; + global.next_addrs[class] = next_addr + slot_size; break :a next_addr; } }; @@ -84,7 +92,7 @@ fn alloc(ctx: *anyopaque, len: usize, alignment: mem.Alignment, return_address: fn resize( ctx: *anyopaque, buf: []u8, - alignment: mem.Alignment, + alignment: Alignment, new_len: usize, return_address: usize, ) bool { @@ -112,7 +120,7 @@ fn resize( fn remap( context: *anyopaque, memory: []u8, - alignment: mem.Alignment, + alignment: Alignment, new_len: usize, return_address: usize, ) ?[*]u8 { @@ -122,7 +130,7 @@ fn remap( fn free( ctx: *anyopaque, buf: []u8, - alignment: mem.Alignment, + alignment: Alignment, return_address: usize, ) void { _ = ctx; @@ -134,16 +142,16 @@ fn free( const addr = @intFromPtr(buf.ptr); if (class < size_class_count) { const node: *usize = @ptrFromInt(addr + (slot_size - @sizeOf(usize))); - node.* = frees[class]; - frees[class] = addr; + node.* = global.frees[class]; + global.frees[class] = addr; } else { const bigpages_needed = bigPagesNeeded(actual_len); const pow2_pages = math.ceilPowerOfTwoAssert(usize, bigpages_needed); const big_slot_size_bytes = pow2_pages * bigpage_size; const node: *usize = @ptrFromInt(addr + (big_slot_size_bytes - @sizeOf(usize))); const big_class = math.log2(pow2_pages); - node.* = big_frees[big_class]; - big_frees[big_class] = addr; + node.* = global.big_frees[big_class]; + global.big_frees[big_class] = addr; } } @@ -156,16 +164,30 @@ fn allocBigPages(n: usize) usize { const slot_size_bytes = pow2_pages * bigpage_size; const class = math.log2(pow2_pages); - const top_free_ptr = big_frees[class]; + const top_free_ptr = global.big_frees[class]; if (top_free_ptr != 0) { const node: *usize = @ptrFromInt(top_free_ptr + (slot_size_bytes - @sizeOf(usize))); - big_frees[class] = node.*; + global.big_frees[class] = node.*; return top_free_ptr; } - const page_index = @wasmMemoryGrow(0, pow2_pages * pages_per_bigpage); - if (page_index == -1) return 0; - return @as(usize, @intCast(page_index)) * wasm.page_size; + if (builtin.cpu.arch.isWasm()) { + const page_index = @wasmMemoryGrow(0, pow2_pages * pages_per_bigpage); + if (page_index == -1) return 0; + return @as(usize, @intCast(page_index)) * page_size_max; + } else if (builtin.os.tag == .linux) { + const start_brk = s: { + const start_brk = global.prev_brk; + break :s if (start_brk == 0) std.os.linux.brk(0) else start_brk; + }; + const end_brk = start_brk + pow2_pages * pages_per_bigpage * page_size_max; + const new_prev_brk = std.os.linux.brk(end_brk); + global.prev_brk = new_prev_brk; + if (new_prev_brk != end_brk) return 0; + return start_brk; + } else { + @compileError("no sbrk-like OS primitive available"); + } } const test_ally: Allocator = .{ @@ -257,12 +279,14 @@ test "shrink" { } test "large object - grow" { + if (builtin.os.tag == .linux) return error.SkipZigTest; + var slice1 = try test_ally.alloc(u8, bigpage_size * 2 - 20); defer test_ally.free(slice1); const old = slice1; slice1 = try test_ally.realloc(slice1, bigpage_size * 2 - 10); - try std.testing.expect(slice1.ptr == old.ptr); + try std.testing.expectEqual(slice1.ptr, old.ptr); slice1 = try test_ally.realloc(slice1, bigpage_size * 2); slice1 = try test_ally.realloc(slice1, bigpage_size * 2 + 1); diff --git a/lib/std/heap/sbrk_allocator.zig b/lib/std/heap/sbrk_allocator.zig deleted file mode 100644 index c4006d332239c38bc9fbeb7f770f81fd5f77bb22..0000000000000000000000000000000000000000 --- a/lib/std/heap/sbrk_allocator.zig +++ /dev/null @@ -1,180 +0,0 @@ -const builtin = @import("builtin"); - -const std = @import("../std.zig"); -const Io = std.Io; -const math = std.math; -const Allocator = std.mem.Allocator; -const mem = std.mem; -const heap = std.heap; -const assert = std.debug.assert; - -pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type { - return struct { - pub const vtable: Allocator.VTable = .{ - .alloc = alloc, - .resize = resize, - .remap = remap, - .free = free, - }; - - pub const Error = Allocator.Error; - - const max_usize = math.maxInt(usize); - const ushift = math.Log2Int(usize); - const bigpage_size = 64 * 1024; - const pages_per_bigpage = bigpage_size / heap.pageSize(); - const bigpage_count = max_usize / bigpage_size; - - /// Because of storing free list pointers, the minimum size class is 3. - const min_class = math.log2(math.ceilPowerOfTwoAssert(usize, 1 + @sizeOf(usize))); - const size_class_count = math.log2(bigpage_size) - min_class; - /// 0 - 1 bigpage - /// 1 - 2 bigpages - /// 2 - 4 bigpages - /// etc. - const big_size_class_count = math.log2(bigpage_count); - - var next_addrs = [1]usize{0} ** size_class_count; - /// For each size class, points to the freed pointer. - var frees = [1]usize{0} ** size_class_count; - /// For each big size class, points to the freed pointer. - var big_frees = [1]usize{0} ** big_size_class_count; - - // TODO don't do the naive locking strategy - var mutex: Io.Mutex = .{}; - fn alloc(ctx: *anyopaque, len: usize, alignment: mem.Alignment, return_address: usize) ?[*]u8 { - _ = ctx; - _ = return_address; - Io.Threaded.mutexLock(&mutex); - defer Io.Threaded.mutexUnlock(&mutex); - // Make room for the freelist next pointer. - const actual_len = @max(len +| @sizeOf(usize), alignment.toByteUnits()); - const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return null; - const class = math.log2(slot_size) - min_class; - if (class < size_class_count) { - const addr = a: { - const top_free_ptr = frees[class]; - if (top_free_ptr != 0) { - const node = @as(*usize, @ptrFromInt(top_free_ptr + (slot_size - @sizeOf(usize)))); - frees[class] = node.*; - break :a top_free_ptr; - } - - const next_addr = next_addrs[class]; - if (next_addr % heap.pageSize() == 0) { - const addr = allocBigPages(1); - if (addr == 0) return null; - //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{ - // slot_size, class, addr, - //}); - next_addrs[class] = addr + slot_size; - break :a addr; - } else { - next_addrs[class] = next_addr + slot_size; - break :a next_addr; - } - }; - return @as([*]u8, @ptrFromInt(addr)); - } - const bigpages_needed = bigPagesNeeded(actual_len); - const addr = allocBigPages(bigpages_needed); - return @as([*]u8, @ptrFromInt(addr)); - } - - fn resize( - ctx: *anyopaque, - buf: []u8, - alignment: mem.Alignment, - new_len: usize, - return_address: usize, - ) bool { - _ = ctx; - _ = return_address; - Io.Threaded.mutexLock(&mutex); - defer Io.Threaded.mutexUnlock(&mutex); - // We don't want to move anything from one size class to another, but we - // can recover bytes in between powers of two. - const buf_align = alignment.toByteUnits(); - const old_actual_len = @max(buf.len + @sizeOf(usize), buf_align); - const new_actual_len = @max(new_len +| @sizeOf(usize), buf_align); - const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_actual_len); - const old_small_class = math.log2(old_small_slot_size) - min_class; - if (old_small_class < size_class_count) { - const new_small_slot_size = math.ceilPowerOfTwo(usize, new_actual_len) catch return false; - return old_small_slot_size == new_small_slot_size; - } else { - const old_bigpages_needed = bigPagesNeeded(old_actual_len); - const old_big_slot_pages = math.ceilPowerOfTwoAssert(usize, old_bigpages_needed); - const new_bigpages_needed = bigPagesNeeded(new_actual_len); - const new_big_slot_pages = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return false; - return old_big_slot_pages == new_big_slot_pages; - } - } - - fn remap( - context: *anyopaque, - memory: []u8, - alignment: mem.Alignment, - new_len: usize, - return_address: usize, - ) ?[*]u8 { - return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null; - } - - fn free( - ctx: *anyopaque, - buf: []u8, - alignment: mem.Alignment, - return_address: usize, - ) void { - _ = ctx; - _ = return_address; - Io.Threaded.mutexLock(&mutex); - defer Io.Threaded.mutexUnlock(&mutex); - const buf_align = alignment.toByteUnits(); - const actual_len = @max(buf.len + @sizeOf(usize), buf_align); - const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len); - const class = math.log2(slot_size) - min_class; - const addr = @intFromPtr(buf.ptr); - if (class < size_class_count) { - const node = @as(*usize, @ptrFromInt(addr + (slot_size - @sizeOf(usize)))); - node.* = frees[class]; - frees[class] = addr; - } else { - const bigpages_needed = bigPagesNeeded(actual_len); - const pow2_pages = math.ceilPowerOfTwoAssert(usize, bigpages_needed); - const big_slot_size_bytes = pow2_pages * bigpage_size; - const node = @as(*usize, @ptrFromInt(addr + (big_slot_size_bytes - @sizeOf(usize)))); - const big_class = math.log2(pow2_pages); - node.* = big_frees[big_class]; - big_frees[big_class] = addr; - } - } - - inline fn bigPagesNeeded(byte_count: usize) usize { - return (byte_count + (bigpage_size + (@sizeOf(usize) - 1))) / bigpage_size; - } - - fn allocBigPages(n: usize) usize { - const pow2_pages = math.ceilPowerOfTwoAssert(usize, n); - const slot_size_bytes = pow2_pages * bigpage_size; - const class = math.log2(pow2_pages); - - const top_free_ptr = big_frees[class]; - if (top_free_ptr != 0) { - const node = @as(*usize, @ptrFromInt(top_free_ptr + (slot_size_bytes - @sizeOf(usize)))); - big_frees[class] = node.*; - return top_free_ptr; - } - return sbrk(pow2_pages * pages_per_bigpage * heap.pageSize()); - } - }; -} - -test SbrkAllocator { - _ = SbrkAllocator(struct { - fn sbrk(_: usize) usize { - return 0; - } - }.sbrk); -} diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index 32dd627e09ebdb29e7871b13f4d0ac3a1fe94fb2..8cce6c1dae8d06646450d865a8d6b932f6254710 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -594,6 +594,10 @@ pub fn errno(r: usize) E { return @enumFromInt(int); } +pub fn brk(addr: usize) usize { + return syscall1(.brk, addr); +} + pub fn dup(old: i32) usize { return syscall1(.dup, @as(usize, @bitCast(@as(isize, old)))); } -- 2.54.0