| ... | @@ -261,7 +261,9 @@ const Node = struct { | ... | @@ -261,7 +261,9 @@ const Node = struct { |
| 261 | return @as([*]u8, @ptrCast(node))[0..size.toInt()]; | 261 | return @as([*]u8, @ptrCast(node))[0..size.toInt()]; |
| 262 | } | 262 | } |
| 263 | | 263 | |
| 264 | fn endResize(node: *Node, size: usize) void { | 264 | fn endResize(node: *Node, size: usize, prev_size: usize) void { |
| | 265 | assert(size >= prev_size); // nodes must not shrink |
| | 266 | assert(@atomicLoad(Size, &node.size, .unordered).toInt() == prev_size); |
| 265 | return @atomicStore(Size, &node.size, .fromInt(size), .release); // syncs with acquire in `beginResize` | 267 | return @atomicStore(Size, &node.size, .fromInt(size), .release); // syncs with acquire in `beginResize` |
| 266 | } | 268 | } |
| 267 | | 269 | |
| ... | @@ -302,6 +304,8 @@ fn stealFreeList(arena: *ArenaAllocator) ?*Node { | ... | @@ -302,6 +304,8 @@ fn stealFreeList(arena: *ArenaAllocator) ?*Node { |
| 302 | | 304 | |
| 303 | fn pushFreeList(arena: *ArenaAllocator, first: *Node, last: *Node) void { | 305 | fn pushFreeList(arena: *ArenaAllocator, first: *Node, last: *Node) void { |
| 304 | assert(first != last.next); | 306 | assert(first != last.next); |
| | 307 | assert(first != first.next); |
| | 308 | assert(last != last.next); |
| 305 | while (@cmpxchgWeak( | 309 | while (@cmpxchgWeak( |
| 306 | ?*Node, | 310 | ?*Node, |
| 307 | &arena.state.free_list, | 311 | &arena.state.free_list, |
| ... | @@ -315,8 +319,10 @@ fn pushFreeList(arena: *ArenaAllocator, first: *Node, last: *Node) void { | ... | @@ -315,8 +319,10 @@ fn pushFreeList(arena: *ArenaAllocator, first: *Node, last: *Node) void { |
| 315 | } | 319 | } |
| 316 | | 320 | |
| 317 | fn alignedIndex(buf_ptr: [*]u8, end_index: usize, alignment: Alignment) usize { | 321 | fn alignedIndex(buf_ptr: [*]u8, end_index: usize, alignment: Alignment) usize { |
| 318 | return end_index + | 322 | // Wrapping arithmetic to avoid overflows since `end_index` isn't bounded by |
| 319 | mem.alignPointerOffset(buf_ptr + end_index, alignment.toByteUnits()).?; | 323 | // `size`. This is always ok since the max alignment in byte units is also |
| | 324 | // the max value of `usize` so wrapped values are correctly aligned anyway. |
| | 325 | return alignment.forward(@intFromPtr(buf_ptr) +% end_index) -% @intFromPtr(buf_ptr); |
| 320 | } | 326 | } |
| 321 | | 327 | |
| 322 | fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u8 { | 328 | fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u8 { |
| ... | @@ -362,7 +368,7 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u | ... | @@ -362,7 +368,7 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u |
| 362 | const node = first_node orelse break :resize; | 368 | const node = first_node orelse break :resize; |
| 363 | const allocated_slice = node.beginResize() orelse break :resize; | 369 | const allocated_slice = node.beginResize() orelse break :resize; |
| 364 | var size = allocated_slice.len; | 370 | var size = allocated_slice.len; |
| 365 | defer node.endResize(size); | 371 | defer node.endResize(size, allocated_slice.len); |
| 366 | | 372 | |
| 367 | const buf = allocated_slice[@sizeOf(Node)..]; | 373 | const buf = allocated_slice[@sizeOf(Node)..]; |
| 368 | const end_index = @atomicLoad(usize, &node.end_index, .monotonic); | 374 | const end_index = @atomicLoad(usize, &node.end_index, .monotonic); |
| ... | @@ -404,92 +410,81 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u | ... | @@ -404,92 +410,81 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u |
| 404 | // Also this avoids the ABA problem; stealing the list with an atomic | 410 | // Also this avoids the ABA problem; stealing the list with an atomic |
| 405 | // swap doesn't introduce any potentially stale `next` pointers. | 411 | // swap doesn't introduce any potentially stale `next` pointers. |
| 406 | | 412 | |
| 407 | const free_list = arena.stealFreeList(); | 413 | const free_list = arena.stealFreeList() orelse break :from_free_list; |
| 408 | var first_free: ?*Node = free_list; | | |
| 409 | var last_free: ?*Node = free_list; | | |
| 410 | defer { | | |
| 411 | // Push remaining stolen free list back onto `arena.state.free_list`. | | |
| 412 | if (first_free) |first| { | | |
| 413 | const last = last_free.?; | | |
| 414 | assert(last.next == null); // optimize for no new nodes added during steal | | |
| 415 | arena.pushFreeList(first, last); | | |
| 416 | } | | |
| 417 | } | | |
| 418 | | 414 | |
| 419 | const candidate: ?*Node, const prev: ?*Node = candidate: { | 415 | const first_free: *Node, const last_free: *Node, const node: *Node, const prev: ?*Node = find: { |
| 420 | var best_fit_prev: ?*Node = null; | 416 | var best_fit_prev: ?*Node = null; |
| 421 | var best_fit: ?*Node = null; | 417 | var best_fit: ?*Node = null; |
| 422 | var best_fit_diff: usize = std.math.maxInt(usize); | 418 | var best_fit_diff: usize = std.math.maxInt(usize); |
| 423 | | 419 | |
| 424 | var it_prev: ?*Node = null; | 420 | var it_prev: ?*Node = null; |
| 425 | var it = free_list; | 421 | var it: ?*Node = free_list; |
| 426 | while (it) |node| : ({ | 422 | while (it) |node| : ({ |
| 427 | it_prev = it; | 423 | it_prev = node; |
| 428 | it = node.next; | 424 | it = node.next; |
| 429 | }) { | 425 | }) { |
| 430 | last_free = node; | | |
| 431 | assert(!node.size.resizing); | 426 | assert(!node.size.resizing); |
| 432 | const buf = node.allocatedSliceUnsafe()[@sizeOf(Node)..]; | 427 | const buf = node.allocatedSliceUnsafe()[@sizeOf(Node)..]; |
| 433 | const aligned_index = alignedIndex(buf.ptr, 0, alignment); | 428 | const aligned_index = alignedIndex(buf.ptr, 0, alignment); |
| 434 | | 429 | |
| 435 | if (aligned_index + n <= buf.len) { | 430 | const diff = aligned_index + n -| buf.len; |
| 436 | break :candidate .{ node, it_prev }; | 431 | if (diff < best_fit_diff) { |
| 437 | } | | |
| 438 | | | |
| 439 | const diff = aligned_index + n - buf.len; | | |
| 440 | if (diff <= best_fit_diff) { | | |
| 441 | best_fit_prev = it_prev; | 432 | best_fit_prev = it_prev; |
| 442 | best_fit = node; | 433 | best_fit = node; |
| 443 | best_fit_diff = diff; | 434 | best_fit_diff = diff; |
| 444 | } | 435 | } |
| 445 | } else { | | |
| 446 | // Ideally we want to use all nodes in `free_list` eventually, | | |
| 447 | // so even if none fit we'll try to resize the one that was the | | |
| 448 | // closest to being large enough. | | |
| 449 | if (best_fit) |node| { | | |
| 450 | const allocated_slice = node.allocatedSliceUnsafe(); | | |
| 451 | const buf = allocated_slice[@sizeOf(Node)..]; | | |
| 452 | const aligned_index = alignedIndex(buf.ptr, 0, alignment); | | |
| 453 | const new_size = mem.alignForward(usize, @sizeOf(Node) + aligned_index + n, 2); | | |
| 454 | | | |
| 455 | if (arena.child_allocator.rawResize(allocated_slice, .of(Node), new_size, @returnAddress())) { | | |
| 456 | node.size = .fromInt(new_size); | | |
| 457 | break :candidate .{ node, best_fit_prev }; | | |
| 458 | } | | |
| 459 | } | | |
| 460 | break :from_free_list; | | |
| 461 | } | 436 | } |
| | 437 | |
| | 438 | break :find .{ free_list, it_prev.?, best_fit.?, best_fit_prev }; |
| | 439 | }; |
| | 440 | |
| | 441 | const aligned_index, const need_resize = aligned_index: { |
| | 442 | const buf = node.allocatedSliceUnsafe()[@sizeOf(Node)..]; |
| | 443 | const aligned_index = alignedIndex(buf.ptr, 0, alignment); |
| | 444 | break :aligned_index .{ aligned_index, aligned_index + n > buf.len }; |
| 462 | }; | 445 | }; |
| 463 | | 446 | |
| 464 | { | 447 | if (need_resize) { |
| 465 | var it = last_free; | 448 | // Ideally we want to use all nodes in `free_list` eventually, |
| 466 | while (it) |node| : (it = node.next) { | 449 | // so even if none fit we'll try to resize the one that was the |
| 467 | last_free = node; | 450 | // closest to being large enough. |
| | 451 | const new_size = mem.alignForward(usize, @sizeOf(Node) + aligned_index + n, 2); |
| | 452 | if (arena.child_allocator.rawResize(node.allocatedSliceUnsafe(), .of(Node), new_size, @returnAddress())) { |
| | 453 | node.size = .fromInt(new_size); |
| | 454 | } else { |
| | 455 | arena.pushFreeList(first_free, last_free); |
| | 456 | break :from_free_list; // we couldn't find a fitting free node |
| 468 | } | 457 | } |
| 469 | } | 458 | } |
| 470 | | 459 | |
| 471 | const node = candidate orelse break :from_free_list; | | |
| 472 | const old_next = node.next; | | |
| 473 | | | |
| 474 | const buf = node.allocatedSliceUnsafe()[@sizeOf(Node)..]; | 460 | const buf = node.allocatedSliceUnsafe()[@sizeOf(Node)..]; |
| 475 | const aligned_index = alignedIndex(buf.ptr, 0, alignment); | 461 | const old_next = node.next; |
| 476 | | 462 | |
| 477 | node.end_index = aligned_index + n; | 463 | node.end_index = aligned_index + n; |
| 478 | node.next = first_node; | 464 | node.next = first_node; |
| 479 | | 465 | |
| 480 | switch (arena.tryPushNode(node)) { | 466 | switch (arena.tryPushNode(node)) { |
| 481 | .success => { | 467 | .success => { |
| 482 | // finish removing node from free list | 468 | // Finish removing node from free list. |
| 483 | if (prev) |p| p.next = old_next; | 469 | if (prev) |p| p.next = old_next; |
| 484 | if (node == first_free) first_free = old_next; | 470 | |
| 485 | if (node == last_free) last_free = prev; | 471 | // Push remaining stolen free list back onto `arena.state.free_list`. |
| | 472 | const new_first_free = if (node == first_free) old_next else first_free; |
| | 473 | const new_last_free = if (node == last_free) prev else last_free; |
| | 474 | if (new_first_free) |first| { |
| | 475 | const last = new_last_free.?; |
| | 476 | arena.pushFreeList(first, last); |
| | 477 | } |
| | 478 | |
| 486 | return buf[aligned_index..][0..n].ptr; | 479 | return buf[aligned_index..][0..n].ptr; |
| 487 | }, | 480 | }, |
| 488 | .failure => |old_first_node| { | 481 | .failure => |old_first_node| { |
| 489 | cur_first_node = old_first_node; | | |
| 490 | // restore free list to as we found it | 482 | // restore free list to as we found it |
| 491 | node.next = old_next; | 483 | node.next = old_next; |
| 492 | continue :retry; | 484 | arena.pushFreeList(first_free, last_free); |
| | 485 | |
| | 486 | cur_first_node = old_first_node; |
| | 487 | continue :retry; // there's a new first node; retry! |
| 493 | }, | 488 | }, |
| 494 | } | 489 | } |
| 495 | } | 490 | } |
| ... | @@ -501,16 +496,17 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u | ... | @@ -501,16 +496,17 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u |
| 501 | @branchHint(.cold); | 496 | @branchHint(.cold); |
| 502 | } | 497 | } |
| 503 | | 498 | |
| 504 | const size: usize = size: { | 499 | const size: Node.Size = size: { |
| 505 | const min_size = @sizeOf(Node) + alignment.toByteUnits() + n; | 500 | const min_size = @sizeOf(Node) + alignment.toByteUnits() + n; |
| 506 | const big_enough_size = prev_size + min_size + 16; | 501 | const big_enough_size = prev_size + min_size + 16; |
| 507 | break :size mem.alignForward(usize, big_enough_size + big_enough_size / 2, 2); | 502 | const size = mem.alignForward(usize, big_enough_size + big_enough_size / 2, 2); |
| | 503 | break :size .fromInt(size); |
| 508 | }; | 504 | }; |
| 509 | const ptr = arena.child_allocator.rawAlloc(size, .of(Node), @returnAddress()) orelse | 505 | const ptr = arena.child_allocator.rawAlloc(size.toInt(), .of(Node), @returnAddress()) orelse |
| 510 | return null; | 506 | return null; |
| 511 | const new_node: *Node = @ptrCast(@alignCast(ptr)); | 507 | const new_node: *Node = @ptrCast(@alignCast(ptr)); |
| 512 | new_node.* = .{ | 508 | new_node.* = .{ |
| 513 | .size = .fromInt(size), | 509 | .size = size, |
| 514 | .end_index = undefined, // set below | 510 | .end_index = undefined, // set below |
| 515 | .next = undefined, // set below | 511 | .next = undefined, // set below |
| 516 | }; | 512 | }; |
| ... | @@ -537,91 +533,80 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u | ... | @@ -537,91 +533,80 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u |
| 537 | } | 533 | } |
| 538 | } | 534 | } |
| 539 | | 535 | |
| 540 | fn resize(ctx: *anyopaque, buf: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool { | 536 | fn resize(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool { |
| 541 | const arena: *ArenaAllocator = @ptrCast(@alignCast(ctx)); | 537 | const arena: *ArenaAllocator = @ptrCast(@alignCast(ctx)); |
| 542 | _ = alignment; | 538 | _ = alignment; |
| 543 | _ = ret_addr; | 539 | _ = ret_addr; |
| 544 | | 540 | |
| 545 | assert(buf.len > 0); | 541 | assert(memory.len > 0); |
| 546 | assert(new_len > 0); | 542 | assert(new_len > 0); |
| 547 | if (buf.len == new_len) return true; | | |
| 548 | | 543 | |
| 549 | const node = arena.loadFirstNode().?; | 544 | const node = arena.loadFirstNode().?; |
| 550 | const cur_buf_ptr = @as([*]u8, @ptrCast(node)) + @sizeOf(Node); | 545 | const buf_ptr = @as([*]u8, @ptrCast(node)) + @sizeOf(Node); |
| 551 | | | |
| 552 | var cur_end_index = @atomicLoad(usize, &node.end_index, .monotonic); | | |
| 553 | while (true) { | | |
| 554 | if (cur_buf_ptr + cur_end_index != buf.ptr + buf.len) { | | |
| 555 | // It's not the most recent allocation, so it cannot be expanded, | | |
| 556 | // but it's fine if they want to make it smaller. | | |
| 557 | return new_len <= buf.len; | | |
| 558 | } | | |
| 559 | | 546 | |
| 560 | const new_end_index: usize = new_end_index: { | 547 | const cur_end_index = @atomicLoad(usize, &node.end_index, .monotonic); |
| 561 | if (buf.len >= new_len) { | 548 | if (buf_ptr + cur_end_index != memory.ptr + memory.len) { |
| 562 | break :new_end_index cur_end_index - (buf.len - new_len); | 549 | // It's not the most recent allocation, so it cannot be expanded, |
| 563 | } | 550 | // but it's fine if they want to make it smaller. |
| 564 | const cur_buf_len: usize = node.loadBuf().len; | 551 | return new_len <= memory.len; |
| 565 | // Saturating arithmetic because `end_index` and `size` are not | | |
| 566 | // guaranteed to be in sync. | | |
| 567 | if (cur_buf_len -| cur_end_index >= new_len - buf.len) { | | |
| 568 | break :new_end_index cur_end_index + (new_len - buf.len); | | |
| 569 | } | | |
| 570 | return false; | | |
| 571 | }; | | |
| 572 | | | |
| 573 | cur_end_index = @cmpxchgWeak( | | |
| 574 | usize, | | |
| 575 | &node.end_index, | | |
| 576 | cur_end_index, | | |
| 577 | new_end_index, | | |
| 578 | .monotonic, | | |
| 579 | .monotonic, | | |
| 580 | ) orelse { | | |
| 581 | return true; | | |
| 582 | }; | | |
| 583 | } | 552 | } |
| | 553 | |
| | 554 | const new_end_index: usize = new_end_index: { |
| | 555 | if (memory.len >= new_len) { |
| | 556 | break :new_end_index cur_end_index - (memory.len - new_len); |
| | 557 | } |
| | 558 | const cur_buf_len: usize = node.loadBuf().len; |
| | 559 | // Saturating arithmetic because `end_index` and `size` are not |
| | 560 | // guaranteed to be in sync. |
| | 561 | if (cur_buf_len -| cur_end_index >= new_len - memory.len) { |
| | 562 | break :new_end_index cur_end_index + (new_len - memory.len); |
| | 563 | } |
| | 564 | return false; |
| | 565 | }; |
| | 566 | assert(buf_ptr + new_end_index == memory.ptr + new_len); |
| | 567 | |
| | 568 | return null == @cmpxchgStrong( |
| | 569 | usize, |
| | 570 | &node.end_index, |
| | 571 | cur_end_index, |
| | 572 | new_end_index, |
| | 573 | .monotonic, |
| | 574 | .monotonic, |
| | 575 | ) or |
| | 576 | new_len <= memory.len; // Shrinking allocations should always succeed. |
| 584 | } | 577 | } |
| 585 | | 578 | |
| 586 | fn remap( | 579 | fn remap(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { |
| 587 | context: *anyopaque, | 580 | return if (resize(ctx, memory, alignment, new_len, ret_addr)) memory.ptr else null; |
| 588 | memory: []u8, | | |
| 589 | alignment: Alignment, | | |
| 590 | new_len: usize, | | |
| 591 | return_address: usize, | | |
| 592 | ) ?[*]u8 { | | |
| 593 | return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null; | | |
| 594 | } | 581 | } |
| 595 | | 582 | |
| 596 | fn free(ctx: *anyopaque, buf: []u8, alignment: Alignment, ret_addr: usize) void { | 583 | fn free(ctx: *anyopaque, memory: []u8, alignment: Alignment, ret_addr: usize) void { |
| 597 | const arena: *ArenaAllocator = @ptrCast(@alignCast(ctx)); | 584 | const arena: *ArenaAllocator = @ptrCast(@alignCast(ctx)); |
| 598 | _ = alignment; | 585 | _ = alignment; |
| 599 | _ = ret_addr; | 586 | _ = ret_addr; |
| 600 | | 587 | |
| 601 | assert(buf.len > 0); | 588 | assert(memory.len > 0); |
| 602 | | 589 | |
| 603 | const node = arena.loadFirstNode().?; | 590 | const node = arena.loadFirstNode().?; |
| 604 | const cur_buf_ptr: [*]u8 = @as([*]u8, @ptrCast(node)) + @sizeOf(Node); | 591 | const buf_ptr = @as([*]u8, @ptrCast(node)) + @sizeOf(Node); |
| 605 | | 592 | |
| 606 | var cur_end_index = @atomicLoad(usize, &node.end_index, .monotonic); | 593 | const cur_end_index = @atomicLoad(usize, &node.end_index, .monotonic); |
| 607 | while (true) { | 594 | if (buf_ptr + cur_end_index != memory.ptr + memory.len) { |
| 608 | if (cur_buf_ptr + cur_end_index != buf.ptr + buf.len) { | 595 | // Not the most recent allocation; we cannot free it. |
| 609 | // Not the most recent allocation; we cannot free it. | 596 | return; |
| 610 | return; | | |
| 611 | } | | |
| 612 | const new_end_index = cur_end_index - buf.len; | | |
| 613 | | | |
| 614 | cur_end_index = @cmpxchgWeak( | | |
| 615 | usize, | | |
| 616 | &node.end_index, | | |
| 617 | cur_end_index, | | |
| 618 | new_end_index, | | |
| 619 | .monotonic, | | |
| 620 | .monotonic, | | |
| 621 | ) orelse { | | |
| 622 | return; | | |
| 623 | }; | | |
| 624 | } | 597 | } |
| | 598 | |
| | 599 | const new_end_index = cur_end_index - memory.len; |
| | 600 | assert(buf_ptr + new_end_index == memory.ptr); |
| | 601 | |
| | 602 | _ = @cmpxchgStrong( |
| | 603 | usize, |
| | 604 | &node.end_index, |
| | 605 | cur_end_index, |
| | 606 | new_end_index, |
| | 607 | .monotonic, |
| | 608 | .monotonic, |
| | 609 | ); |
| 625 | } | 610 | } |
| 626 | | 611 | |
| 627 | const std = @import("std"); | 612 | const std = @import("std"); |
| ... | @@ -672,3 +657,406 @@ test "reset while retaining a buffer" { | ... | @@ -672,3 +657,406 @@ test "reset while retaining a buffer" { |
| 672 | try std.testing.expect(arena_allocator.state.used_list.?.next == null); | 657 | try std.testing.expect(arena_allocator.state.used_list.?.next == null); |
| 673 | try std.testing.expectEqual(2, arena_allocator.queryCapacity()); | 658 | try std.testing.expectEqual(2, arena_allocator.queryCapacity()); |
| 674 | } | 659 | } |
| | 660 | |
| | 661 | test "fuzz" { |
| | 662 | @disableInstrumentation(); |
| | 663 | if (@import("builtin").single_threaded) return error.SkipZigTest; |
| | 664 | |
| | 665 | const gpa = std.heap.smp_allocator; |
| | 666 | |
| | 667 | var arena_state: ArenaAllocator.State = .init; |
| | 668 | // No need to deinit arena_state, all allocations are in `sample_buffer`! |
| | 669 | |
| | 670 | const control_buffer = try gpa.alloc(u8, 64 << 10 << 10); |
| | 671 | defer gpa.free(control_buffer); |
| | 672 | var control_instance: std.heap.FixedBufferAllocator = .init(control_buffer); |
| | 673 | |
| | 674 | const sample_buffer = try gpa.alloc(u8, 64 << 10 << 10); |
| | 675 | defer gpa.free(sample_buffer); |
| | 676 | var sample_instance: FuzzAllocator = .init(sample_buffer); |
| | 677 | |
| | 678 | var allocs: FuzzContext.Allocs = try .initCapacity(gpa, FuzzContext.max_alloc_count); |
| | 679 | defer allocs.deinit(gpa); |
| | 680 | |
| | 681 | try std.testing.fuzz(FuzzContext.Init{ |
| | 682 | .gpa = gpa, |
| | 683 | .allocs = &allocs, |
| | 684 | .arena_state = &arena_state, |
| | 685 | .control_instance = &control_instance, |
| | 686 | .sample_instance = &sample_instance, |
| | 687 | }, fuzzArenaAllocator, .{}); |
| | 688 | } |
| | 689 | |
| | 690 | fn fuzzArenaAllocator(fuzz_init: FuzzContext.Init, smith: *std.testing.Smith) anyerror!void { |
| | 691 | @disableInstrumentation(); |
| | 692 | const testing = std.testing; |
| | 693 | |
| | 694 | // We use a 'fresh' `Threaded` instance every time to reset threadlocals to |
| | 695 | // their default values. |
| | 696 | |
| | 697 | var io_instance: std.Io.Threaded = .init(fuzz_init.gpa, .{}); |
| | 698 | defer io_instance.deinit(); |
| | 699 | const io = io_instance.io(); |
| | 700 | |
| | 701 | fuzz_init.sample_instance.prepareFailures(smith); |
| | 702 | |
| | 703 | const control_allocator = fuzz_init.control_instance.threadSafeAllocator(); |
| | 704 | const sample_child_allocator = fuzz_init.sample_instance.allocator(); |
| | 705 | |
| | 706 | var arena_instance = fuzz_init.arena_state.*.promote(sample_child_allocator); |
| | 707 | defer fuzz_init.arena_state.* = arena_instance.state; |
| | 708 | |
| | 709 | var ctx: FuzzContext = .init( |
| | 710 | io, |
| | 711 | control_allocator, |
| | 712 | arena_instance.allocator(), |
| | 713 | fuzz_init.allocs, |
| | 714 | ); |
| | 715 | defer ctx.deinit(); |
| | 716 | |
| | 717 | ctx.rwl.lockUncancelable(io); |
| | 718 | |
| | 719 | var group: std.Io.Group = .init; |
| | 720 | defer group.cancel(io); |
| | 721 | |
| | 722 | var n_actions: usize = 0; |
| | 723 | while (!smith.eosWeightedSimple(99, 1) and n_actions < FuzzContext.max_action_count) { |
| | 724 | errdefer comptime unreachable; |
| | 725 | |
| | 726 | const ActionTag = @typeInfo(FuzzContext.Action).@"union".tag_type.?; |
| | 727 | const weights: []const testing.Smith.Weight = weights: { |
| | 728 | if (ctx.allocs.len == ctx.allocs.capacity) |
| | 729 | break :weights &.{ |
| | 730 | .value(ActionTag, .resize, 1), |
| | 731 | .value(ActionTag, .remap, 1), |
| | 732 | .value(ActionTag, .free, 1), |
| | 733 | }; |
| | 734 | break :weights testing.Smith.baselineWeights(ActionTag) ++ |
| | 735 | .{testing.Smith.Weight.value(ActionTag, .alloc, 2)}; |
| | 736 | }; |
| | 737 | const action: FuzzContext.Action = switch (smith.valueWeighted(ActionTag, weights)) { |
| | 738 | .alloc => action: { |
| | 739 | const alloc_index = ctx.allocs.addOneBounded() catch continue; |
| | 740 | ctx.allocs.items(.len)[alloc_index] = .free; |
| | 741 | break :action .{ .alloc = .{ |
| | 742 | .len = nextLen(smith), |
| | 743 | .alignment = smith.valueRangeAtMost( |
| | 744 | Alignment, |
| | 745 | .@"1", |
| | 746 | .fromByteUnits(2 * std.heap.page_size_max), |
| | 747 | ), |
| | 748 | .index = alloc_index, |
| | 749 | } }; |
| | 750 | }, |
| | 751 | .resize => .{ .resize = .{ .new_len = nextLen(smith) } }, |
| | 752 | .remap => .{ .remap = .{ .new_len = nextLen(smith) } }, |
| | 753 | .free => .free, |
| | 754 | }; |
| | 755 | group.concurrent(io, FuzzContext.doOneAction, .{ &ctx, action }) catch break; |
| | 756 | n_actions += 1; |
| | 757 | } |
| | 758 | |
| | 759 | ctx.rwl.unlock(io); |
| | 760 | |
| | 761 | try group.await(io); |
| | 762 | try ctx.check(); |
| | 763 | |
| | 764 | // This also covers the `deinit` logic since `free_all` uses it internally. |
| | 765 | |
| | 766 | const old_capacity = arena_instance.queryCapacity(); |
| | 767 | const reset_mode: ResetMode = switch (smith.value(@typeInfo(ResetMode).@"union".tag_type.?)) { |
| | 768 | .free_all => .free_all, |
| | 769 | .retain_capacity => .retain_capacity, |
| | 770 | .retain_with_limit => .{ .retain_with_limit = smith.value(usize) }, |
| | 771 | }; |
| | 772 | const ok = arena_instance.reset(reset_mode); |
| | 773 | const new_capacity = arena_instance.queryCapacity(); |
| | 774 | switch (reset_mode) { |
| | 775 | .free_all => { |
| | 776 | try testing.expect(ok); |
| | 777 | try testing.expectEqual(0, new_capacity); |
| | 778 | fuzz_init.sample_instance.reset(); |
| | 779 | }, |
| | 780 | .retain_with_limit => |limit| if (ok) try testing.expect(new_capacity <= limit), |
| | 781 | .retain_capacity => if (ok) try testing.expectEqual(old_capacity, new_capacity), |
| | 782 | } |
| | 783 | |
| | 784 | fuzz_init.control_instance.reset(); |
| | 785 | fuzz_init.allocs.clearRetainingCapacity(); |
| | 786 | } |
| | 787 | fn nextLen(smith: *std.testing.Smith) usize { |
| | 788 | @disableInstrumentation(); |
| | 789 | return usizeRange(smith, 1, 16 << 10 << 10); |
| | 790 | } |
| | 791 | fn usizeRange(smith: *std.testing.Smith, at_least: usize, at_most: usize) usize { |
| | 792 | @disableInstrumentation(); |
| | 793 | const Int = @Int(.unsigned, @min(64, @bitSizeOf(usize))); |
| | 794 | return smith.valueRangeAtMost(Int, @intCast(at_least), @intCast(at_most)); |
| | 795 | } |
| | 796 | |
| | 797 | const FuzzContext = struct { |
| | 798 | io: std.Io, |
| | 799 | rwl: std.Io.RwLock, |
| | 800 | |
| | 801 | control_allocator: Allocator, |
| | 802 | sample_allocator: Allocator, |
| | 803 | |
| | 804 | allocs: *Allocs, |
| | 805 | |
| | 806 | const max_alloc_count = 4096; |
| | 807 | const max_action_count = 2 * max_alloc_count; |
| | 808 | |
| | 809 | const Allocs = std.MultiArrayList(struct { |
| | 810 | control_ptr: [*]u8, |
| | 811 | sample_ptr: [*]u8, |
| | 812 | len: Len, |
| | 813 | alignment: Alignment, |
| | 814 | }); |
| | 815 | |
| | 816 | const Len = enum(usize) { |
| | 817 | free = std.math.maxInt(usize), |
| | 818 | _, |
| | 819 | }; |
| | 820 | |
| | 821 | const Action = union(enum(u8)) { |
| | 822 | alloc: struct { len: usize, alignment: Alignment, index: usize }, |
| | 823 | resize: struct { new_len: usize }, |
| | 824 | remap: struct { new_len: usize }, |
| | 825 | free, |
| | 826 | }; |
| | 827 | |
| | 828 | threadlocal var tls_next: u8 = 0; |
| | 829 | threadlocal var tls_last_index: ?usize = null; |
| | 830 | |
| | 831 | const Init = struct { |
| | 832 | gpa: Allocator, |
| | 833 | allocs: *FuzzContext.Allocs, |
| | 834 | arena_state: *ArenaAllocator.State, |
| | 835 | control_instance: *std.heap.FixedBufferAllocator, |
| | 836 | sample_instance: *FuzzAllocator, |
| | 837 | }; |
| | 838 | |
| | 839 | fn init( |
| | 840 | io: std.Io, |
| | 841 | control_allocator: Allocator, |
| | 842 | sample_allocator: Allocator, |
| | 843 | allocs: *Allocs, |
| | 844 | ) FuzzContext { |
| | 845 | @disableInstrumentation(); |
| | 846 | return .{ |
| | 847 | .io = io, |
| | 848 | .rwl = .init, |
| | 849 | .control_allocator = control_allocator, |
| | 850 | .sample_allocator = sample_allocator, |
| | 851 | .allocs = allocs, |
| | 852 | }; |
| | 853 | } |
| | 854 | |
| | 855 | fn deinit(ctx: *FuzzContext) void { |
| | 856 | @disableInstrumentation(); |
| | 857 | ctx.* = undefined; |
| | 858 | } |
| | 859 | |
| | 860 | fn check(ctx: *const FuzzContext) !void { |
| | 861 | @disableInstrumentation(); |
| | 862 | for (0..ctx.allocs.len) |index| { |
| | 863 | const len: usize = switch (ctx.allocs.items(.len)[index]) { |
| | 864 | .free => continue, |
| | 865 | _ => |len| @intFromEnum(len), |
| | 866 | }; |
| | 867 | const control = ctx.allocs.items(.control_ptr)[index][0..len]; |
| | 868 | const sample = ctx.allocs.items(.sample_ptr)[index][0..len]; |
| | 869 | try std.testing.expectEqualSlices(u8, control, sample); |
| | 870 | } |
| | 871 | } |
| | 872 | |
| | 873 | fn doOneAction(ctx: *FuzzContext, action: Action) std.Io.Cancelable!void { |
| | 874 | @disableInstrumentation(); |
| | 875 | ctx.rwl.lockSharedUncancelable(ctx.io); |
| | 876 | defer ctx.rwl.unlockShared(ctx.io); |
| | 877 | |
| | 878 | switch (action) { |
| | 879 | .alloc => |act| ctx.doOneAlloc(act.len, act.alignment, act.index), |
| | 880 | .resize => |act| ctx.doOneResize(act.new_len), |
| | 881 | .remap => |act| ctx.doOneRemap(act.new_len), |
| | 882 | .free => ctx.doOneFree(), |
| | 883 | } |
| | 884 | } |
| | 885 | |
| | 886 | fn doOneAlloc(ctx: *FuzzContext, len: usize, alignment: Alignment, index: usize) void { |
| | 887 | @disableInstrumentation(); |
| | 888 | assert(ctx.allocs.items(.len)[index] == .free); |
| | 889 | |
| | 890 | const control_ptr = ctx.control_allocator.rawAlloc(len, alignment, @returnAddress()) orelse |
| | 891 | return; |
| | 892 | const sample_ptr = ctx.sample_allocator.rawAlloc(len, alignment, @returnAddress()) orelse { |
| | 893 | ctx.control_allocator.rawFree(control_ptr[0..len], alignment, @returnAddress()); |
| | 894 | return; |
| | 895 | }; |
| | 896 | |
| | 897 | ctx.allocs.set(index, .{ |
| | 898 | .control_ptr = control_ptr, |
| | 899 | .sample_ptr = sample_ptr, |
| | 900 | .len = @enumFromInt(len), |
| | 901 | .alignment = alignment, |
| | 902 | }); |
| | 903 | |
| | 904 | for (control_ptr[0..len], sample_ptr[0..len]) |*control, *sample| { |
| | 905 | control.* = tls_next; |
| | 906 | sample.* = tls_next; |
| | 907 | tls_next +%= 1; |
| | 908 | } |
| | 909 | |
| | 910 | tls_last_index = index; |
| | 911 | } |
| | 912 | fn doOneResize(ctx: *FuzzContext, new_len: usize) void { |
| | 913 | @disableInstrumentation(); |
| | 914 | const index = tls_last_index orelse return; |
| | 915 | const len = ctx.allocs.items(.len)[index]; |
| | 916 | assert(len != .free); |
| | 917 | const memory = ctx.allocs.items(.sample_ptr)[index][0..@intFromEnum(len)]; |
| | 918 | const alignment = ctx.allocs.items(.alignment)[index]; |
| | 919 | |
| | 920 | assert(alignment.check(@intFromPtr(ctx.allocs.items(.control_ptr)[index]))); |
| | 921 | assert(alignment.check(@intFromPtr(ctx.allocs.items(.sample_ptr)[index]))); |
| | 922 | |
| | 923 | // Since `resize` is fallible, we have to ensure that `control_allocator` |
| | 924 | // is always successful by reserving the memory we need beforehand. |
| | 925 | const new_control_ptr = ctx.control_allocator.rawAlloc(new_len, alignment, @returnAddress()) orelse |
| | 926 | return; |
| | 927 | if (ctx.sample_allocator.rawResize(memory, alignment, new_len, @returnAddress())) { |
| | 928 | const old_control = ctx.allocs.items(.control_ptr)[index][0..memory.len]; |
| | 929 | const overlap = @min(memory.len, new_len); |
| | 930 | @memcpy(new_control_ptr[0..overlap], old_control[0..overlap]); |
| | 931 | ctx.control_allocator.rawFree(old_control, alignment, @returnAddress()); |
| | 932 | } else { |
| | 933 | ctx.control_allocator.rawFree(new_control_ptr[0..new_len], alignment, @returnAddress()); |
| | 934 | return; |
| | 935 | } |
| | 936 | |
| | 937 | ctx.allocs.set(index, .{ |
| | 938 | .control_ptr = new_control_ptr, |
| | 939 | .sample_ptr = memory.ptr, |
| | 940 | .len = @enumFromInt(new_len), |
| | 941 | .alignment = alignment, |
| | 942 | }); |
| | 943 | |
| | 944 | if (new_len > memory.len) { |
| | 945 | for ( |
| | 946 | ctx.allocs.items(.control_ptr)[index][memory.len..new_len], |
| | 947 | ctx.allocs.items(.sample_ptr)[index][memory.len..new_len], |
| | 948 | ) |*control, *sample| { |
| | 949 | control.* = tls_next; |
| | 950 | sample.* = tls_next; |
| | 951 | tls_next +%= 1; |
| | 952 | } |
| | 953 | } |
| | 954 | } |
| | 955 | fn doOneRemap(ctx: *FuzzContext, new_len: usize) void { |
| | 956 | @disableInstrumentation(); |
| | 957 | return doOneResize(ctx, new_len); |
| | 958 | } |
| | 959 | fn doOneFree(ctx: *FuzzContext) void { |
| | 960 | @disableInstrumentation(); |
| | 961 | const index = tls_last_index orelse return; |
| | 962 | const len = ctx.allocs.items(.len)[index]; |
| | 963 | assert(len != .free); |
| | 964 | const memory = ctx.allocs.items(.sample_ptr)[index][0..@intFromEnum(len)]; |
| | 965 | const alignment = ctx.allocs.items(.alignment)[index]; |
| | 966 | |
| | 967 | assert(alignment.check(@intFromPtr(ctx.allocs.items(.control_ptr)[index]))); |
| | 968 | assert(alignment.check(@intFromPtr(ctx.allocs.items(.sample_ptr)[index]))); |
| | 969 | |
| | 970 | ctx.control_allocator.rawFree(ctx.allocs.items(.control_ptr)[index][0..memory.len], alignment, @returnAddress()); |
| | 971 | ctx.sample_allocator.rawFree(ctx.allocs.items(.sample_ptr)[index][0..memory.len], alignment, @returnAddress()); |
| | 972 | |
| | 973 | ctx.allocs.set(index, .{ |
| | 974 | .control_ptr = undefined, |
| | 975 | .sample_ptr = undefined, |
| | 976 | .len = .free, |
| | 977 | .alignment = undefined, |
| | 978 | }); |
| | 979 | |
| | 980 | tls_last_index = null; |
| | 981 | } |
| | 982 | }; |
| | 983 | |
| | 984 | const FuzzAllocator = struct { |
| | 985 | fba: std.heap.FixedBufferAllocator, |
| | 986 | spurious_failures: [256]u8, |
| | 987 | index: u8, |
| | 988 | |
| | 989 | fn init(buffer: []u8) FuzzAllocator { |
| | 990 | @disableInstrumentation(); |
| | 991 | return .{ |
| | 992 | .fba = .init(buffer), |
| | 993 | .spurious_failures = undefined, // set with `preprepareFailures` |
| | 994 | .index = 0, |
| | 995 | }; |
| | 996 | } |
| | 997 | |
| | 998 | fn prepareFailures(fa: *FuzzAllocator, smith: *std.testing.Smith) void { |
| | 999 | @disableInstrumentation(); |
| | 1000 | const bool_weights: []const std.testing.Smith.Weight = &.{ |
| | 1001 | .value(u8, 0, 10), |
| | 1002 | .value(u8, 1, 1), |
| | 1003 | }; |
| | 1004 | smith.bytesWeighted(&fa.spurious_failures, bool_weights); |
| | 1005 | fa.index = 0; |
| | 1006 | } |
| | 1007 | |
| | 1008 | fn reset(fa: *FuzzAllocator) void { |
| | 1009 | @disableInstrumentation(); |
| | 1010 | fa.fba.reset(); |
| | 1011 | } |
| | 1012 | |
| | 1013 | fn allocator(fa: *FuzzAllocator) Allocator { |
| | 1014 | @disableInstrumentation(); |
| | 1015 | return .{ |
| | 1016 | .ptr = fa, |
| | 1017 | .vtable = &.{ |
| | 1018 | .alloc = FuzzAllocator.alloc, |
| | 1019 | .resize = FuzzAllocator.resize, |
| | 1020 | .remap = FuzzAllocator.remap, |
| | 1021 | .free = FuzzAllocator.free, |
| | 1022 | }, |
| | 1023 | }; |
| | 1024 | } |
| | 1025 | |
| | 1026 | fn alloc(ctx: *anyopaque, len: usize, alignment: Alignment, ret_addr: usize) ?[*]u8 { |
| | 1027 | @disableInstrumentation(); |
| | 1028 | const fa: *FuzzAllocator = @ptrCast(@alignCast(ctx)); |
| | 1029 | _ = ret_addr; |
| | 1030 | |
| | 1031 | const index = @atomicRmw(u8, &fa.index, .Add, 1, .monotonic); |
| | 1032 | if (fa.spurious_failures[index] != 0) return null; |
| | 1033 | return fa.fba.threadSafeAllocator().rawAlloc(len, alignment, @returnAddress()); |
| | 1034 | } |
| | 1035 | |
| | 1036 | fn resize(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool { |
| | 1037 | @disableInstrumentation(); |
| | 1038 | const fa: *FuzzAllocator = @ptrCast(@alignCast(ctx)); |
| | 1039 | _ = ret_addr; |
| | 1040 | |
| | 1041 | const index = @atomicRmw(u8, &fa.index, .Add, 1, .monotonic); |
| | 1042 | if (fa.spurious_failures[index] != 0) return false; |
| | 1043 | return fa.fba.threadSafeAllocator().rawResize(memory, alignment, new_len, @returnAddress()); |
| | 1044 | } |
| | 1045 | |
| | 1046 | fn remap(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { |
| | 1047 | @disableInstrumentation(); |
| | 1048 | const fa: *FuzzAllocator = @ptrCast(@alignCast(ctx)); |
| | 1049 | _ = ret_addr; |
| | 1050 | |
| | 1051 | const index = @atomicRmw(u8, &fa.index, .Add, 1, .monotonic); |
| | 1052 | if (fa.spurious_failures[index] != 0) return null; |
| | 1053 | return fa.fba.threadSafeAllocator().rawRemap(memory, alignment, new_len, @returnAddress()); |
| | 1054 | } |
| | 1055 | |
| | 1056 | fn free(ctx: *anyopaque, memory: []u8, alignment: Alignment, ret_addr: usize) void { |
| | 1057 | @disableInstrumentation(); |
| | 1058 | const fa: *FuzzAllocator = @ptrCast(@alignCast(ctx)); |
| | 1059 | _ = ret_addr; |
| | 1060 | return fa.fba.threadSafeAllocator().rawFree(memory, alignment, @returnAddress()); |
| | 1061 | } |
| | 1062 | }; |
Andrew Kelley <andrew@ziglang.org> 2026-03-11 03:00:07+01:00