From f09386cce9ad99d77978cee0d15ae7dd422ea50c Mon Sep 17 00:00:00 2001 From: Justus Klausecker Date: Wed, 4 Mar 2026 15:27:54 +0100 Subject: [PATCH 1/9] std.heap.ArenaAllocator: optimize aligned index calculation The `alignedIndex` function is very hot (literally every single `alloc` call invokes it at least once) and `std.mem.alignPointerOffset` seems to be very slow, so this commit replaces this functions with a custom implementation that doesn't do any unnecessary validation and doesn't have any branches as a result of that. The validation `std.mem.alignPointerOffset` does isn't necessary anyway, we're not actually calculating an offset that we plan to apply to a pointer directly, but an offset into a valid buffer that we only apply to a pointer if the result is inside of that buffer. This leads to a ~4% speedup in a synthetic benchmark that just puts a lot of concurrent load on an `ArenaAllocator`. --- lib/std/heap/ArenaAllocator.zig | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/std/heap/ArenaAllocator.zig b/lib/std/heap/ArenaAllocator.zig index a9fea912841df99be85feae311cde1340b9d3088..3f3df624fad3e327ed7572129caa904c8f9d1929 100644 --- a/lib/std/heap/ArenaAllocator.zig +++ b/lib/std/heap/ArenaAllocator.zig @@ -315,8 +315,10 @@ fn pushFreeList(arena: *ArenaAllocator, first: *Node, last: *Node) void { } fn alignedIndex(buf_ptr: [*]u8, end_index: usize, alignment: Alignment) usize { - return end_index + - mem.alignPointerOffset(buf_ptr + end_index, alignment.toByteUnits()).?; + // Wrapping arithmetic to avoid overflows since `end_index` isn't bounded by + // `size`. This is always ok since the max alignment in byte units is also + // the max value of `usize` so wrapped values are correctly aligned anyway. + return alignment.forward(@intFromPtr(buf_ptr) +% end_index) -% @intFromPtr(buf_ptr); } fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u8 { -- 2.54.0 From 46c72ed970850af6ba0933b4bcd38b5764a3528e Mon Sep 17 00:00:00 2001 From: Justus Klausecker Date: Wed, 4 Mar 2026 15:35:51 +0100 Subject: [PATCH 2/9] std.heap.ArenaAllocator: do not retry failed CAS in `resize`/`free` If we use `@cmpxchgStrong` instead of `@cmpxchgWeak` to adjust the `end_index` in `resize` and `free`, the only reason the CAS can fail is that another thread has changed `end_index` in the meantime. If that's happened, the allocation we were trying to resize/free isn't the most recent allocation anymore and there's no point in retrying, so we can get rid of the loop. --- lib/std/heap/ArenaAllocator.zig | 116 ++++++++++++++------------------ 1 file changed, 52 insertions(+), 64 deletions(-) diff --git a/lib/std/heap/ArenaAllocator.zig b/lib/std/heap/ArenaAllocator.zig index 3f3df624fad3e327ed7572129caa904c8f9d1929..cc36610a3b5add68fa43437baed815d0567e308d 100644 --- a/lib/std/heap/ArenaAllocator.zig +++ b/lib/std/heap/ArenaAllocator.zig @@ -539,91 +539,79 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u } } -fn resize(ctx: *anyopaque, buf: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool { +fn resize(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool { const arena: *ArenaAllocator = @ptrCast(@alignCast(ctx)); _ = alignment; _ = ret_addr; - assert(buf.len > 0); + assert(memory.len > 0); assert(new_len > 0); - if (buf.len == new_len) return true; const node = arena.loadFirstNode().?; - const cur_buf_ptr = @as([*]u8, @ptrCast(node)) + @sizeOf(Node); + const buf_ptr = @as([*]u8, @ptrCast(node)) + @sizeOf(Node); - var cur_end_index = @atomicLoad(usize, &node.end_index, .monotonic); - while (true) { - if (cur_buf_ptr + cur_end_index != buf.ptr + buf.len) { - // It's not the most recent allocation, so it cannot be expanded, - // but it's fine if they want to make it smaller. - return new_len <= buf.len; - } - - const new_end_index: usize = new_end_index: { - if (buf.len >= new_len) { - break :new_end_index cur_end_index - (buf.len - new_len); - } - const cur_buf_len: usize = node.loadBuf().len; - // Saturating arithmetic because `end_index` and `size` are not - // guaranteed to be in sync. - if (cur_buf_len -| cur_end_index >= new_len - buf.len) { - break :new_end_index cur_end_index + (new_len - buf.len); - } - return false; - }; - - cur_end_index = @cmpxchgWeak( - usize, - &node.end_index, - cur_end_index, - new_end_index, - .monotonic, - .monotonic, - ) orelse { - return true; - }; + const cur_end_index = @atomicLoad(usize, &node.end_index, .monotonic); + if (buf_ptr + cur_end_index != memory.ptr + memory.len) { + // It's not the most recent allocation, so it cannot be expanded, + // but it's fine if they want to make it smaller. + return new_len <= memory.len; } + + const new_end_index: usize = new_end_index: { + if (memory.len >= new_len) { + break :new_end_index cur_end_index - (memory.len - new_len); + } + const cur_buf_len: usize = node.loadBuf().len; + // Saturating arithmetic because `end_index` and `size` are not + // guaranteed to be in sync. + if (cur_buf_len -| cur_end_index >= new_len - memory.len) { + break :new_end_index cur_end_index + (new_len - memory.len); + } + return false; + }; + assert(buf_ptr + new_end_index == memory.ptr + new_len); + + return null == @cmpxchgStrong( + usize, + &node.end_index, + cur_end_index, + new_end_index, + .monotonic, + .monotonic, + ); } -fn remap( - context: *anyopaque, - memory: []u8, - alignment: Alignment, - new_len: usize, - return_address: usize, -) ?[*]u8 { - return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null; +fn remap(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { + return if (resize(ctx, memory, alignment, new_len, ret_addr)) memory.ptr else null; } -fn free(ctx: *anyopaque, buf: []u8, alignment: Alignment, ret_addr: usize) void { +fn free(ctx: *anyopaque, memory: []u8, alignment: Alignment, ret_addr: usize) void { const arena: *ArenaAllocator = @ptrCast(@alignCast(ctx)); _ = alignment; _ = ret_addr; - assert(buf.len > 0); + assert(memory.len > 0); const node = arena.loadFirstNode().?; - const cur_buf_ptr: [*]u8 = @as([*]u8, @ptrCast(node)) + @sizeOf(Node); + const buf_ptr = @as([*]u8, @ptrCast(node)) + @sizeOf(Node); - var cur_end_index = @atomicLoad(usize, &node.end_index, .monotonic); - while (true) { - if (cur_buf_ptr + cur_end_index != buf.ptr + buf.len) { - // Not the most recent allocation; we cannot free it. - return; - } - const new_end_index = cur_end_index - buf.len; - - cur_end_index = @cmpxchgWeak( - usize, - &node.end_index, - cur_end_index, - new_end_index, - .monotonic, - .monotonic, - ) orelse { - return; - }; + const cur_end_index = @atomicLoad(usize, &node.end_index, .monotonic); + if (buf_ptr + cur_end_index != memory.ptr + memory.len) { + // Not the most recent allocation; we cannot free it. + return; } + + const new_end_index = cur_end_index - memory.len; + assert(buf_ptr + new_end_index == memory.ptr); + + _ = @cmpxchgStrong( + usize, + &node.end_index, + cur_end_index, + new_end_index, + .monotonic, + .monotonic, + ); } const std = @import("std"); -- 2.54.0 From 7b9865b046993dc282435d89c8da93992e84888b Mon Sep 17 00:00:00 2001 From: Justus Klausecker Date: Wed, 4 Mar 2026 15:45:09 +0100 Subject: [PATCH 3/9] std.heap.FixedBufferAllocator: complete thread-safe implementation `FixedBufferAllocator.threadSafeAllocator()` already provided a thread-safe `alloc` implementation, but all other functions were nops. This commit implements the remaining `Allocator` functions and tightens up the memory orderings in `alloc` a bit, `monotonic` is good enough here. --- lib/std/heap/FixedBufferAllocator.zig | 81 ++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 9 deletions(-) diff --git a/lib/std/heap/FixedBufferAllocator.zig b/lib/std/heap/FixedBufferAllocator.zig index 0951dd3bcc888d09b9655c3cf5267d7853d7ca72..1303bd35abbfcd378eed42553f046772dd40cb9d 100644 --- a/lib/std/heap/FixedBufferAllocator.zig +++ b/lib/std/heap/FixedBufferAllocator.zig @@ -36,9 +36,9 @@ pub fn threadSafeAllocator(self: *FixedBufferAllocator) Allocator { .ptr = self, .vtable = &.{ .alloc = threadSafeAlloc, - .resize = Allocator.noResize, - .remap = Allocator.noRemap, - .free = Allocator.noFree, + .resize = threadSafeResize, + .remap = threadSafeRemap, + .free = threadSafeFree, }, }; } @@ -127,21 +127,84 @@ pub fn free( } } -fn threadSafeAlloc(ctx: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 { +fn threadSafeAlloc(ctx: *anyopaque, n: usize, alignment: mem.Alignment, ret_addr: usize) ?[*]u8 { const self: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); - _ = ra; + _ = ret_addr; const ptr_align = alignment.toByteUnits(); - var end_index = @atomicLoad(usize, &self.end_index, .seq_cst); + var cur_end_index = @atomicLoad(usize, &self.end_index, .monotonic); while (true) { - const adjust_off = mem.alignPointerOffset(self.buffer.ptr + end_index, ptr_align) orelse return null; - const adjusted_index = end_index + adjust_off; + const adjust_off = mem.alignPointerOffset(self.buffer.ptr + cur_end_index, ptr_align) orelse return null; + const adjusted_index = cur_end_index + adjust_off; const new_end_index = adjusted_index + n; if (new_end_index > self.buffer.len) return null; - end_index = @cmpxchgWeak(usize, &self.end_index, end_index, new_end_index, .seq_cst, .seq_cst) orelse + cur_end_index = @cmpxchgWeak(usize, &self.end_index, cur_end_index, new_end_index, .monotonic, .monotonic) orelse return self.buffer[adjusted_index..new_end_index].ptr; } } +fn threadSafeResize(ctx: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ret_addr: usize) bool { + const fba: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); + _ = alignment; + _ = ret_addr; + + const cur_end_index = @atomicLoad(usize, &fba.end_index, .monotonic); + if (fba.buffer.ptr + cur_end_index != memory.ptr + memory.len) { + // It's not the most recent allocation, so it cannot be expanded, + // but it's fine if they want to make it smaller. + return new_len <= memory.len; + } + + const new_end_index: usize = new_end_index: { + if (memory.len >= new_len) { + break :new_end_index cur_end_index - (memory.len - new_len); + } + if (fba.buffer.len - cur_end_index >= new_len - memory.len) { + break :new_end_index cur_end_index + (new_len - memory.len); + } + return false; + }; + assert(fba.buffer.ptr + new_end_index == memory.ptr + new_len); + + return null == @cmpxchgStrong( + usize, + &fba.end_index, + cur_end_index, + new_end_index, + .monotonic, + .monotonic, + ); +} + +fn threadSafeRemap(ctx: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { + return if (threadSafeResize(ctx, memory, alignment, new_len, ret_addr)) memory.ptr else null; +} + +fn threadSafeFree(ctx: *anyopaque, memory: []u8, alignment: mem.Alignment, ret_addr: usize) void { + const fba: *FixedBufferAllocator = @ptrCast(@alignCast(ctx)); + _ = alignment; + _ = ret_addr; + + assert(memory.len > 0); + + const cur_end_index = @atomicLoad(usize, &fba.end_index, .monotonic); + if (fba.buffer.ptr + cur_end_index != memory.ptr + memory.len) { + // Not the most recent allocation; we cannot free it. + return; + } + + const new_end_index = cur_end_index - memory.len; + assert(fba.buffer.ptr + new_end_index == memory.ptr); + + _ = @cmpxchgStrong( + usize, + &fba.end_index, + cur_end_index, + new_end_index, + .monotonic, + .monotonic, + ); +} + pub fn reset(self: *FixedBufferAllocator) void { self.end_index = 0; } -- 2.54.0 From 0e348d415f9039b44313232304aeea69c3228c0c Mon Sep 17 00:00:00 2001 From: Justus Klausecker Date: Thu, 5 Mar 2026 16:26:01 +0100 Subject: [PATCH 4/9] std.heap.ArenaAllocator: clean up some yucky bits and add a bunch of asserts. No functional changes. --- lib/std/heap/ArenaAllocator.zig | 110 +++++++++++++++----------------- 1 file changed, 52 insertions(+), 58 deletions(-) diff --git a/lib/std/heap/ArenaAllocator.zig b/lib/std/heap/ArenaAllocator.zig index cc36610a3b5add68fa43437baed815d0567e308d..d798bfcb682b4c9016bfcfbb70bee212cf9b178e 100644 --- a/lib/std/heap/ArenaAllocator.zig +++ b/lib/std/heap/ArenaAllocator.zig @@ -261,7 +261,9 @@ const Node = struct { return @as([*]u8, @ptrCast(node))[0..size.toInt()]; } - fn endResize(node: *Node, size: usize) void { + fn endResize(node: *Node, size: usize, prev_size: usize) void { + assert(size >= prev_size); // nodes must not shrink + assert(@atomicLoad(Size, &node.size, .unordered).toInt() == prev_size); return @atomicStore(Size, &node.size, .fromInt(size), .release); // syncs with acquire in `beginResize` } @@ -302,6 +304,8 @@ fn stealFreeList(arena: *ArenaAllocator) ?*Node { fn pushFreeList(arena: *ArenaAllocator, first: *Node, last: *Node) void { assert(first != last.next); + assert(first != first.next); + assert(last != last.next); while (@cmpxchgWeak( ?*Node, &arena.state.free_list, @@ -364,7 +368,7 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u const node = first_node orelse break :resize; const allocated_slice = node.beginResize() orelse break :resize; var size = allocated_slice.len; - defer node.endResize(size); + defer node.endResize(size, allocated_slice.len); const buf = allocated_slice[@sizeOf(Node)..]; const end_index = @atomicLoad(usize, &node.end_index, .monotonic); @@ -406,92 +410,81 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u // Also this avoids the ABA problem; stealing the list with an atomic // swap doesn't introduce any potentially stale `next` pointers. - const free_list = arena.stealFreeList(); - var first_free: ?*Node = free_list; - var last_free: ?*Node = free_list; - defer { - // Push remaining stolen free list back onto `arena.state.free_list`. - if (first_free) |first| { - const last = last_free.?; - assert(last.next == null); // optimize for no new nodes added during steal - arena.pushFreeList(first, last); - } - } + const free_list = arena.stealFreeList() orelse break :from_free_list; - const candidate: ?*Node, const prev: ?*Node = candidate: { + const first_free: *Node, const last_free: *Node, const node: *Node, const prev: ?*Node = find: { var best_fit_prev: ?*Node = null; var best_fit: ?*Node = null; var best_fit_diff: usize = std.math.maxInt(usize); var it_prev: ?*Node = null; - var it = free_list; + var it: ?*Node = free_list; while (it) |node| : ({ - it_prev = it; + it_prev = node; it = node.next; }) { - last_free = node; assert(!node.size.resizing); const buf = node.allocatedSliceUnsafe()[@sizeOf(Node)..]; const aligned_index = alignedIndex(buf.ptr, 0, alignment); - if (aligned_index + n <= buf.len) { - break :candidate .{ node, it_prev }; - } - - const diff = aligned_index + n - buf.len; - if (diff <= best_fit_diff) { + const diff = aligned_index + n -| buf.len; + if (diff < best_fit_diff) { best_fit_prev = it_prev; best_fit = node; best_fit_diff = diff; } + } + + break :find .{ free_list, it_prev.?, best_fit.?, best_fit_prev }; + }; + + const aligned_index, const need_resize = aligned_index: { + const buf = node.allocatedSliceUnsafe()[@sizeOf(Node)..]; + const aligned_index = alignedIndex(buf.ptr, 0, alignment); + break :aligned_index .{ aligned_index, aligned_index + n > buf.len }; + }; + + if (need_resize) { + // Ideally we want to use all nodes in `free_list` eventually, + // so even if none fit we'll try to resize the one that was the + // closest to being large enough. + const new_size = mem.alignForward(usize, @sizeOf(Node) + aligned_index + n, 2); + if (arena.child_allocator.rawResize(node.allocatedSliceUnsafe(), .of(Node), new_size, @returnAddress())) { + node.size = .fromInt(new_size); } else { - // Ideally we want to use all nodes in `free_list` eventually, - // so even if none fit we'll try to resize the one that was the - // closest to being large enough. - if (best_fit) |node| { - const allocated_slice = node.allocatedSliceUnsafe(); - const buf = allocated_slice[@sizeOf(Node)..]; - const aligned_index = alignedIndex(buf.ptr, 0, alignment); - const new_size = mem.alignForward(usize, @sizeOf(Node) + aligned_index + n, 2); - - if (arena.child_allocator.rawResize(allocated_slice, .of(Node), new_size, @returnAddress())) { - node.size = .fromInt(new_size); - break :candidate .{ node, best_fit_prev }; - } - } - break :from_free_list; - } - }; - - { - var it = last_free; - while (it) |node| : (it = node.next) { - last_free = node; + arena.pushFreeList(first_free, last_free); + break :from_free_list; // we couldn't find a fitting free node } } - const node = candidate orelse break :from_free_list; - const old_next = node.next; - const buf = node.allocatedSliceUnsafe()[@sizeOf(Node)..]; - const aligned_index = alignedIndex(buf.ptr, 0, alignment); + const old_next = node.next; node.end_index = aligned_index + n; node.next = first_node; switch (arena.tryPushNode(node)) { .success => { - // finish removing node from free list + // Finish removing node from free list. if (prev) |p| p.next = old_next; - if (node == first_free) first_free = old_next; - if (node == last_free) last_free = prev; + + // Push remaining stolen free list back onto `arena.state.free_list`. + const new_first_free = if (node == first_free) old_next else first_free; + const new_last_free = if (node == last_free) prev else last_free; + if (new_first_free) |first| { + const last = new_last_free.?; + arena.pushFreeList(first, last); + } + return buf[aligned_index..][0..n].ptr; }, .failure => |old_first_node| { - cur_first_node = old_first_node; // restore free list to as we found it node.next = old_next; - continue :retry; + arena.pushFreeList(first_free, last_free); + + cur_first_node = old_first_node; + continue :retry; // there's a new first node; retry! }, } } @@ -503,16 +496,17 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u @branchHint(.cold); } - const size: usize = size: { + const size: Node.Size = size: { const min_size = @sizeOf(Node) + alignment.toByteUnits() + n; const big_enough_size = prev_size + min_size + 16; - break :size mem.alignForward(usize, big_enough_size + big_enough_size / 2, 2); + const size = mem.alignForward(usize, big_enough_size + big_enough_size / 2, 2); + break :size .fromInt(size); }; - const ptr = arena.child_allocator.rawAlloc(size, .of(Node), @returnAddress()) orelse + const ptr = arena.child_allocator.rawAlloc(size.toInt(), .of(Node), @returnAddress()) orelse return null; const new_node: *Node = @ptrCast(@alignCast(ptr)); new_node.* = .{ - .size = .fromInt(size), + .size = size, .end_index = undefined, // set below .next = undefined, // set below }; -- 2.54.0 From 823f9039f1cad8093cd276cf003a42996d39b74a Mon Sep 17 00:00:00 2001 From: Justus Klausecker Date: Fri, 6 Mar 2026 00:06:20 +0100 Subject: [PATCH 5/9] llvm: use atomic rmw to increment fuzzer pc counters This change makes the fuzzer instrumentation emitted by Zig thread-safe. --- src/codegen/llvm.zig | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index 4ef72e8ab7fed1786845aa5b8e08d1ddfe6091fa..2335c349297c888d4e1a9547f7fe8ca9ee4fe07c 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -4872,10 +4872,8 @@ pub const FuncGen = struct { const ptr = if (poi_index == 0) base_ptr else try self.wip.gep(.inbounds, .i8, base_ptr, &.{ try o.builder.intValue(.i32, poi_index), }, ""); - const counter = try self.wip.load(.normal, .i8, ptr, .default, ""); const one = try o.builder.intValue(.i8, 1); - const counter_incremented = try self.wip.bin(.add, counter, one, ""); - _ = try self.wip.store(.normal, counter_incremented, ptr, .default); + _ = try self.wip.atomicrmw(.normal, .add, ptr, one, self.sync_scope, .monotonic, .default, ""); // LLVM does not allow blockaddress on the entry block. const pc = if (self.wip.cursor.block == .entry) -- 2.54.0 From 933bfd4282f99e7f59ab025bbff53454c0ae2f5c Mon Sep 17 00:00:00 2001 From: Justus Klausecker Date: Fri, 6 Mar 2026 00:08:13 +0100 Subject: [PATCH 6/9] tests: pass on `-Dsanitize-thread` to unit tests --- build.zig | 5 +++++ test/tests.zig | 2 ++ 2 files changed, 7 insertions(+) diff --git a/build.zig b/build.zig index 06ba1aab65bb127957fb1abb3ba8dafc3d40db9d..0fa924e7cd3e05c0bf36b0157d8ded0b8af0f37f 100644 --- a/build.zig +++ b/build.zig @@ -478,6 +478,7 @@ pub fn build(b: *std.Build) !void { .desc = "Run the behavior tests", .optimize_modes = optimization_modes, .include_paths = &.{}, + .sanitize_thread = sanitize_thread, .skip_single_threaded = skip_single_threaded, .skip_non_native = skip_non_native, .test_only = test_only, @@ -503,6 +504,7 @@ pub fn build(b: *std.Build) !void { .desc = "Run the compiler_rt tests", .optimize_modes = optimization_modes, .include_paths = &.{}, + .sanitize_thread = sanitize_thread, .skip_single_threaded = true, .skip_non_native = skip_non_native, .test_only = test_only, @@ -529,6 +531,7 @@ pub fn build(b: *std.Build) !void { .desc = "Run the zig libc implementation unit tests", .optimize_modes = optimization_modes, .include_paths = &.{}, + .sanitize_thread = sanitize_thread, .skip_single_threaded = true, .skip_non_native = skip_non_native, .test_only = test_only, @@ -555,6 +558,7 @@ pub fn build(b: *std.Build) !void { .desc = "Run the standard library tests", .optimize_modes = optimization_modes, .include_paths = &.{}, + .sanitize_thread = sanitize_thread, .skip_single_threaded = skip_single_threaded, .skip_non_native = skip_non_native, .test_only = test_only, @@ -578,6 +582,7 @@ pub fn build(b: *std.Build) !void { .root_module = addCompilerMod(b, .{ .optimize = optimize, .target = target, + .sanitize_thread = sanitize_thread, .single_threaded = single_threaded, }), .filters = test_filters, diff --git a/test/tests.zig b/test/tests.zig index 0eaf0b1eaf745195811ae62662731a60a6ef9b00..5e0d22682e56dbd81477f501a45f625eb2a17367 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -2334,6 +2334,7 @@ pub const ModuleTestOptions = struct { skip_libc: bool, max_rss: usize = 0, no_builtin: bool = false, + sanitize_thread: ?bool = null, build_options: ?*Step.Options = null, pub const TestOnly = union(enum) { @@ -2462,6 +2463,7 @@ fn addOneModuleTest( .link_libc = test_target.link_libc, .pic = test_target.pic, .strip = test_target.strip, + .sanitize_thread = options.sanitize_thread, .single_threaded = test_target.single_threaded, }), .max_rss = max_rss, -- 2.54.0 From 73743ddbff5a7a0ffc8f7ecc780369cf5509795b Mon Sep 17 00:00:00 2001 From: Justus Klausecker Date: Fri, 6 Mar 2026 00:08:30 +0100 Subject: [PATCH 7/9] std.atomic.Mutex: use unordered atomic load on assert This silences a tsan race warning and gets optimized away in `ReleaseFast`. --- lib/std/atomic.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/atomic.zig b/lib/std/atomic.zig index 0040dbf735eeaabad5c5d532663757bfb5de5491..b564c0c67a443e264b57ba14f3fde35633c8bb05 100644 --- a/lib/std/atomic.zig +++ b/lib/std/atomic.zig @@ -513,7 +513,7 @@ pub const Mutex = enum(u8) { } pub fn unlock(m: *Mutex) void { - assert(m.* == .locked); + assert(@atomicLoad(Mutex, m, .unordered) == .locked); @atomicStore(Mutex, m, .unlocked, .release); } }; -- 2.54.0 From 2ba8c94df6b9e60d73c35bf3d1f80f143fc80cab Mon Sep 17 00:00:00 2001 From: Justus Klausecker Date: Fri, 6 Mar 2026 00:09:55 +0100 Subject: [PATCH 8/9] std.heap.ArenaAllocator: add fuzz test The fuzz test consists of a planning phase where the fuzzing smith is used to generate a list of actions to be executed and an execution phase where the actions are all executed by multiple threads at the same time. Each action is only executed exactly once and is performed on an `ArenaAllocator` and on a `FixedBufferAllocator` (for reference). The arena is backed by a special allocator that purposely introduces spurious allocation failures. After all actions are executed, the contents of all allocation pairs are compared to each other. --- lib/std/heap/ArenaAllocator.zig | 403 ++++++++++++++++++++++++++++++++ 1 file changed, 403 insertions(+) diff --git a/lib/std/heap/ArenaAllocator.zig b/lib/std/heap/ArenaAllocator.zig index d798bfcb682b4c9016bfcfbb70bee212cf9b178e..3532bd07af0daec1bf166529bbb5562c5f3b1028 100644 --- a/lib/std/heap/ArenaAllocator.zig +++ b/lib/std/heap/ArenaAllocator.zig @@ -656,3 +656,406 @@ test "reset while retaining a buffer" { try std.testing.expect(arena_allocator.state.used_list.?.next == null); try std.testing.expectEqual(2, arena_allocator.queryCapacity()); } + +test "fuzz" { + @disableInstrumentation(); + if (@import("builtin").single_threaded) return error.SkipZigTest; + + const gpa = std.heap.smp_allocator; + + var arena_state: ArenaAllocator.State = .init; + // No need to deinit arena_state, all allocations are in `sample_buffer`! + + const control_buffer = try gpa.alloc(u8, 64 << 10 << 10); + defer gpa.free(control_buffer); + var control_instance: std.heap.FixedBufferAllocator = .init(control_buffer); + + const sample_buffer = try gpa.alloc(u8, 64 << 10 << 10); + defer gpa.free(sample_buffer); + var sample_instance: FuzzAllocator = .init(sample_buffer); + + var allocs: FuzzContext.Allocs = try .initCapacity(gpa, FuzzContext.max_alloc_count); + defer allocs.deinit(gpa); + + try std.testing.fuzz(FuzzContext.Init{ + .gpa = gpa, + .allocs = &allocs, + .arena_state = &arena_state, + .control_instance = &control_instance, + .sample_instance = &sample_instance, + }, fuzzArenaAllocator, .{}); +} + +fn fuzzArenaAllocator(fuzz_init: FuzzContext.Init, smith: *std.testing.Smith) anyerror!void { + @disableInstrumentation(); + const testing = std.testing; + + // We use a 'fresh' `Threaded` instance every time to reset threadlocals to + // their default values. + + var io_instance: std.Io.Threaded = .init(fuzz_init.gpa, .{}); + defer io_instance.deinit(); + const io = io_instance.io(); + + fuzz_init.sample_instance.prepareFailures(smith); + + const control_allocator = fuzz_init.control_instance.threadSafeAllocator(); + const sample_child_allocator = fuzz_init.sample_instance.allocator(); + + var arena_instance = fuzz_init.arena_state.*.promote(sample_child_allocator); + defer fuzz_init.arena_state.* = arena_instance.state; + + var ctx: FuzzContext = .init( + io, + control_allocator, + arena_instance.allocator(), + fuzz_init.allocs, + ); + defer ctx.deinit(); + + ctx.rwl.lockUncancelable(io); + + var group: std.Io.Group = .init; + defer group.cancel(io); + + var n_actions: usize = 0; + while (!smith.eosWeightedSimple(99, 1) and n_actions < FuzzContext.max_action_count) { + errdefer comptime unreachable; + + const ActionTag = @typeInfo(FuzzContext.Action).@"union".tag_type.?; + const weights: []const testing.Smith.Weight = weights: { + if (ctx.allocs.len == ctx.allocs.capacity) + break :weights &.{ + .value(ActionTag, .resize, 1), + .value(ActionTag, .remap, 1), + .value(ActionTag, .free, 1), + }; + break :weights testing.Smith.baselineWeights(ActionTag) ++ + .{testing.Smith.Weight.value(ActionTag, .alloc, 2)}; + }; + const action: FuzzContext.Action = switch (smith.valueWeighted(ActionTag, weights)) { + .alloc => action: { + const alloc_index = ctx.allocs.addOneBounded() catch continue; + ctx.allocs.items(.len)[alloc_index] = .free; + break :action .{ .alloc = .{ + .len = nextLen(smith), + .alignment = smith.valueRangeAtMost( + Alignment, + .@"1", + .fromByteUnits(2 * std.heap.page_size_max), + ), + .index = alloc_index, + } }; + }, + .resize => .{ .resize = .{ .new_len = nextLen(smith) } }, + .remap => .{ .remap = .{ .new_len = nextLen(smith) } }, + .free => .free, + }; + group.concurrent(io, FuzzContext.doOneAction, .{ &ctx, action }) catch break; + n_actions += 1; + } + + ctx.rwl.unlock(io); + + try group.await(io); + try ctx.check(); + + // This also covers the `deinit` logic since `free_all` uses it internally. + + const old_capacity = arena_instance.queryCapacity(); + const reset_mode: ResetMode = switch (smith.value(@typeInfo(ResetMode).@"union".tag_type.?)) { + .free_all => .free_all, + .retain_capacity => .retain_capacity, + .retain_with_limit => .{ .retain_with_limit = smith.value(usize) }, + }; + const ok = arena_instance.reset(reset_mode); + const new_capacity = arena_instance.queryCapacity(); + switch (reset_mode) { + .free_all => { + try testing.expect(ok); + try testing.expectEqual(0, new_capacity); + fuzz_init.sample_instance.reset(); + }, + .retain_with_limit => |limit| if (ok) try testing.expect(new_capacity <= limit), + .retain_capacity => if (ok) try testing.expectEqual(old_capacity, new_capacity), + } + + fuzz_init.control_instance.reset(); + fuzz_init.allocs.clearRetainingCapacity(); +} +fn nextLen(smith: *std.testing.Smith) usize { + @disableInstrumentation(); + return usizeRange(smith, 1, 16 << 10 << 10); +} +fn usizeRange(smith: *std.testing.Smith, at_least: usize, at_most: usize) usize { + @disableInstrumentation(); + const Int = @Int(.unsigned, @min(64, @bitSizeOf(usize))); + return smith.valueRangeAtMost(Int, @intCast(at_least), @intCast(at_most)); +} + +const FuzzContext = struct { + io: std.Io, + rwl: std.Io.RwLock, + + control_allocator: Allocator, + sample_allocator: Allocator, + + allocs: *Allocs, + + const max_alloc_count = 4096; + const max_action_count = 2 * max_alloc_count; + + const Allocs = std.MultiArrayList(struct { + control_ptr: [*]u8, + sample_ptr: [*]u8, + len: Len, + alignment: Alignment, + }); + + const Len = enum(usize) { + free = std.math.maxInt(usize), + _, + }; + + const Action = union(enum(u8)) { + alloc: struct { len: usize, alignment: Alignment, index: usize }, + resize: struct { new_len: usize }, + remap: struct { new_len: usize }, + free, + }; + + threadlocal var tls_next: u8 = 0; + threadlocal var tls_last_index: ?usize = null; + + const Init = struct { + gpa: Allocator, + allocs: *FuzzContext.Allocs, + arena_state: *ArenaAllocator.State, + control_instance: *std.heap.FixedBufferAllocator, + sample_instance: *FuzzAllocator, + }; + + fn init( + io: std.Io, + control_allocator: Allocator, + sample_allocator: Allocator, + allocs: *Allocs, + ) FuzzContext { + @disableInstrumentation(); + return .{ + .io = io, + .rwl = .init, + .control_allocator = control_allocator, + .sample_allocator = sample_allocator, + .allocs = allocs, + }; + } + + fn deinit(ctx: *FuzzContext) void { + @disableInstrumentation(); + ctx.* = undefined; + } + + fn check(ctx: *const FuzzContext) !void { + @disableInstrumentation(); + for (0..ctx.allocs.len) |index| { + const len: usize = switch (ctx.allocs.items(.len)[index]) { + .free => continue, + _ => |len| @intFromEnum(len), + }; + const control = ctx.allocs.items(.control_ptr)[index][0..len]; + const sample = ctx.allocs.items(.sample_ptr)[index][0..len]; + try std.testing.expectEqualSlices(u8, control, sample); + } + } + + fn doOneAction(ctx: *FuzzContext, action: Action) std.Io.Cancelable!void { + @disableInstrumentation(); + ctx.rwl.lockSharedUncancelable(ctx.io); + defer ctx.rwl.unlockShared(ctx.io); + + switch (action) { + .alloc => |act| ctx.doOneAlloc(act.len, act.alignment, act.index), + .resize => |act| ctx.doOneResize(act.new_len), + .remap => |act| ctx.doOneRemap(act.new_len), + .free => ctx.doOneFree(), + } + } + + fn doOneAlloc(ctx: *FuzzContext, len: usize, alignment: Alignment, index: usize) void { + @disableInstrumentation(); + assert(ctx.allocs.items(.len)[index] == .free); + + const control_ptr = ctx.control_allocator.rawAlloc(len, alignment, @returnAddress()) orelse + return; + const sample_ptr = ctx.sample_allocator.rawAlloc(len, alignment, @returnAddress()) orelse { + ctx.control_allocator.rawFree(control_ptr[0..len], alignment, @returnAddress()); + return; + }; + + ctx.allocs.set(index, .{ + .control_ptr = control_ptr, + .sample_ptr = sample_ptr, + .len = @enumFromInt(len), + .alignment = alignment, + }); + + for (control_ptr[0..len], sample_ptr[0..len]) |*control, *sample| { + control.* = tls_next; + sample.* = tls_next; + tls_next +%= 1; + } + + tls_last_index = index; + } + fn doOneResize(ctx: *FuzzContext, new_len: usize) void { + @disableInstrumentation(); + const index = tls_last_index orelse return; + const len = ctx.allocs.items(.len)[index]; + assert(len != .free); + const memory = ctx.allocs.items(.sample_ptr)[index][0..@intFromEnum(len)]; + const alignment = ctx.allocs.items(.alignment)[index]; + + assert(alignment.check(@intFromPtr(ctx.allocs.items(.control_ptr)[index]))); + assert(alignment.check(@intFromPtr(ctx.allocs.items(.sample_ptr)[index]))); + + // Since `resize` is fallible, we have to ensure that `control_allocator` + // is always successful by reserving the memory we need beforehand. + const new_control_ptr = ctx.control_allocator.rawAlloc(new_len, alignment, @returnAddress()) orelse + return; + if (ctx.sample_allocator.rawResize(memory, alignment, new_len, @returnAddress())) { + const old_control = ctx.allocs.items(.control_ptr)[index][0..memory.len]; + const overlap = @min(memory.len, new_len); + @memcpy(new_control_ptr[0..overlap], old_control[0..overlap]); + ctx.control_allocator.rawFree(old_control, alignment, @returnAddress()); + } else { + ctx.control_allocator.rawFree(new_control_ptr[0..new_len], alignment, @returnAddress()); + return; + } + + ctx.allocs.set(index, .{ + .control_ptr = new_control_ptr, + .sample_ptr = memory.ptr, + .len = @enumFromInt(new_len), + .alignment = alignment, + }); + + if (new_len > memory.len) { + for ( + ctx.allocs.items(.control_ptr)[index][memory.len..new_len], + ctx.allocs.items(.sample_ptr)[index][memory.len..new_len], + ) |*control, *sample| { + control.* = tls_next; + sample.* = tls_next; + tls_next +%= 1; + } + } + } + fn doOneRemap(ctx: *FuzzContext, new_len: usize) void { + @disableInstrumentation(); + return doOneResize(ctx, new_len); + } + fn doOneFree(ctx: *FuzzContext) void { + @disableInstrumentation(); + const index = tls_last_index orelse return; + const len = ctx.allocs.items(.len)[index]; + assert(len != .free); + const memory = ctx.allocs.items(.sample_ptr)[index][0..@intFromEnum(len)]; + const alignment = ctx.allocs.items(.alignment)[index]; + + assert(alignment.check(@intFromPtr(ctx.allocs.items(.control_ptr)[index]))); + assert(alignment.check(@intFromPtr(ctx.allocs.items(.sample_ptr)[index]))); + + ctx.control_allocator.rawFree(ctx.allocs.items(.control_ptr)[index][0..memory.len], alignment, @returnAddress()); + ctx.sample_allocator.rawFree(ctx.allocs.items(.sample_ptr)[index][0..memory.len], alignment, @returnAddress()); + + ctx.allocs.set(index, .{ + .control_ptr = undefined, + .sample_ptr = undefined, + .len = .free, + .alignment = undefined, + }); + + tls_last_index = null; + } +}; + +const FuzzAllocator = struct { + fba: std.heap.FixedBufferAllocator, + spurious_failures: [256]u8, + index: u8, + + fn init(buffer: []u8) FuzzAllocator { + @disableInstrumentation(); + return .{ + .fba = .init(buffer), + .spurious_failures = undefined, // set with `preprepareFailures` + .index = 0, + }; + } + + fn prepareFailures(fa: *FuzzAllocator, smith: *std.testing.Smith) void { + @disableInstrumentation(); + const bool_weights: []const std.testing.Smith.Weight = &.{ + .value(u8, 0, 10), + .value(u8, 1, 1), + }; + smith.bytesWeighted(&fa.spurious_failures, bool_weights); + fa.index = 0; + } + + fn reset(fa: *FuzzAllocator) void { + @disableInstrumentation(); + fa.fba.reset(); + } + + fn allocator(fa: *FuzzAllocator) Allocator { + @disableInstrumentation(); + return .{ + .ptr = fa, + .vtable = &.{ + .alloc = FuzzAllocator.alloc, + .resize = FuzzAllocator.resize, + .remap = FuzzAllocator.remap, + .free = FuzzAllocator.free, + }, + }; + } + + fn alloc(ctx: *anyopaque, len: usize, alignment: Alignment, ret_addr: usize) ?[*]u8 { + @disableInstrumentation(); + const fa: *FuzzAllocator = @ptrCast(@alignCast(ctx)); + _ = ret_addr; + + const index = @atomicRmw(u8, &fa.index, .Add, 1, .monotonic); + if (fa.spurious_failures[index] != 0) return null; + return fa.fba.threadSafeAllocator().rawAlloc(len, alignment, @returnAddress()); + } + + fn resize(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool { + @disableInstrumentation(); + const fa: *FuzzAllocator = @ptrCast(@alignCast(ctx)); + _ = ret_addr; + + const index = @atomicRmw(u8, &fa.index, .Add, 1, .monotonic); + if (fa.spurious_failures[index] != 0) return false; + return fa.fba.threadSafeAllocator().rawResize(memory, alignment, new_len, @returnAddress()); + } + + fn remap(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { + @disableInstrumentation(); + const fa: *FuzzAllocator = @ptrCast(@alignCast(ctx)); + _ = ret_addr; + + const index = @atomicRmw(u8, &fa.index, .Add, 1, .monotonic); + if (fa.spurious_failures[index] != 0) return null; + return fa.fba.threadSafeAllocator().rawRemap(memory, alignment, new_len, @returnAddress()); + } + + fn free(ctx: *anyopaque, memory: []u8, alignment: Alignment, ret_addr: usize) void { + @disableInstrumentation(); + const fa: *FuzzAllocator = @ptrCast(@alignCast(ctx)); + _ = ret_addr; + return fa.fba.threadSafeAllocator().rawFree(memory, alignment, @returnAddress()); + } +}; -- 2.54.0 From 76498686639d01050eb917b016a61c58503510e5 Mon Sep 17 00:00:00 2001 From: Justus Klausecker Date: Fri, 6 Mar 2026 13:05:06 +0100 Subject: [PATCH 9/9] std.heap.ArenaAllocator/std.heap.FixedBufferAllocator: make shrinking always succeed Shrinking allocations should always succeed with these allocators, even if the allocation in question is the most recent one and `resize` didn't manage to decrement the end index of its buffer successfully. --- lib/std/heap/ArenaAllocator.zig | 3 ++- lib/std/heap/FixedBufferAllocator.zig | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/std/heap/ArenaAllocator.zig b/lib/std/heap/ArenaAllocator.zig index 3532bd07af0daec1bf166529bbb5562c5f3b1028..8888ebdbee3573b997e018f007d38693819b0a9d 100644 --- a/lib/std/heap/ArenaAllocator.zig +++ b/lib/std/heap/ArenaAllocator.zig @@ -572,7 +572,8 @@ fn resize(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, r new_end_index, .monotonic, .monotonic, - ); + ) or + new_len <= memory.len; // Shrinking allocations should always succeed. } fn remap(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { diff --git a/lib/std/heap/FixedBufferAllocator.zig b/lib/std/heap/FixedBufferAllocator.zig index 1303bd35abbfcd378eed42553f046772dd40cb9d..e15875948a977b6dff2193c4f44ec3399cc48aa1 100644 --- a/lib/std/heap/FixedBufferAllocator.zig +++ b/lib/std/heap/FixedBufferAllocator.zig @@ -172,7 +172,8 @@ fn threadSafeResize(ctx: *anyopaque, memory: []u8, alignment: mem.Alignment, new new_end_index, .monotonic, .monotonic, - ); + ) or + new_len <= memory.len; // Shrinking allocations should always succeed. } fn threadSafeRemap(ctx: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { -- 2.54.0