| ... | @@ -261,7 +261,9 @@ const Node = struct { | ... | @@ -261,7 +261,9 @@ const Node = struct { |
| 261 | return @as([*]u8, @ptrCast(node))[0..size.toInt()]; | 261 | return @as([*]u8, @ptrCast(node))[0..size.toInt()]; |
| 262 | } | 262 | } |
| 263 | | 263 | |
| 264 | fn endResize(node: *Node, size: usize) void { | 264 | fn endResize(node: *Node, size: usize, prev_size: usize) void { |
| | 265 | assert(size >= prev_size); // nodes must not shrink |
| | 266 | assert(@atomicLoad(Size, &node.size, .unordered).toInt() == prev_size); |
| 265 | return @atomicStore(Size, &node.size, .fromInt(size), .release); // syncs with acquire in `beginResize` | 267 | return @atomicStore(Size, &node.size, .fromInt(size), .release); // syncs with acquire in `beginResize` |
| 266 | } | 268 | } |
| 267 | | 269 | |
| ... | @@ -302,6 +304,8 @@ fn stealFreeList(arena: *ArenaAllocator) ?*Node { | ... | @@ -302,6 +304,8 @@ fn stealFreeList(arena: *ArenaAllocator) ?*Node { |
| 302 | | 304 | |
| 303 | fn pushFreeList(arena: *ArenaAllocator, first: *Node, last: *Node) void { | 305 | fn pushFreeList(arena: *ArenaAllocator, first: *Node, last: *Node) void { |
| 304 | assert(first != last.next); | 306 | assert(first != last.next); |
| | 307 | assert(first != first.next); |
| | 308 | assert(last != last.next); |
| 305 | while (@cmpxchgWeak( | 309 | while (@cmpxchgWeak( |
| 306 | ?*Node, | 310 | ?*Node, |
| 307 | &arena.state.free_list, | 311 | &arena.state.free_list, |
| ... | @@ -364,7 +368,7 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u | ... | @@ -364,7 +368,7 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u |
| 364 | const node = first_node orelse break :resize; | 368 | const node = first_node orelse break :resize; |
| 365 | const allocated_slice = node.beginResize() orelse break :resize; | 369 | const allocated_slice = node.beginResize() orelse break :resize; |
| 366 | var size = allocated_slice.len; | 370 | var size = allocated_slice.len; |
| 367 | defer node.endResize(size); | 371 | defer node.endResize(size, allocated_slice.len); |
| 368 | | 372 | |
| 369 | const buf = allocated_slice[@sizeOf(Node)..]; | 373 | const buf = allocated_slice[@sizeOf(Node)..]; |
| 370 | const end_index = @atomicLoad(usize, &node.end_index, .monotonic); | 374 | const end_index = @atomicLoad(usize, &node.end_index, .monotonic); |
| ... | @@ -406,92 +410,81 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u | ... | @@ -406,92 +410,81 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u |
| 406 | // Also this avoids the ABA problem; stealing the list with an atomic | 410 | // Also this avoids the ABA problem; stealing the list with an atomic |
| 407 | // swap doesn't introduce any potentially stale `next` pointers. | 411 | // swap doesn't introduce any potentially stale `next` pointers. |
| 408 | | 412 | |
| 409 | const free_list = arena.stealFreeList(); | 413 | const free_list = arena.stealFreeList() orelse break :from_free_list; |
| 410 | var first_free: ?*Node = free_list; | | |
| 411 | var last_free: ?*Node = free_list; | | |
| 412 | defer { | | |
| 413 | // Push remaining stolen free list back onto `arena.state.free_list`. | | |
| 414 | if (first_free) |first| { | | |
| 415 | const last = last_free.?; | | |
| 416 | assert(last.next == null); // optimize for no new nodes added during steal | | |
| 417 | arena.pushFreeList(first, last); | | |
| 418 | } | | |
| 419 | } | | |
| 420 | | 414 | |
| 421 | const candidate: ?*Node, const prev: ?*Node = candidate: { | 415 | const first_free: *Node, const last_free: *Node, const node: *Node, const prev: ?*Node = find: { |
| 422 | var best_fit_prev: ?*Node = null; | 416 | var best_fit_prev: ?*Node = null; |
| 423 | var best_fit: ?*Node = null; | 417 | var best_fit: ?*Node = null; |
| 424 | var best_fit_diff: usize = std.math.maxInt(usize); | 418 | var best_fit_diff: usize = std.math.maxInt(usize); |
| 425 | | 419 | |
| 426 | var it_prev: ?*Node = null; | 420 | var it_prev: ?*Node = null; |
| 427 | var it = free_list; | 421 | var it: ?*Node = free_list; |
| 428 | while (it) |node| : ({ | 422 | while (it) |node| : ({ |
| 429 | it_prev = it; | 423 | it_prev = node; |
| 430 | it = node.next; | 424 | it = node.next; |
| 431 | }) { | 425 | }) { |
| 432 | last_free = node; | | |
| 433 | assert(!node.size.resizing); | 426 | assert(!node.size.resizing); |
| 434 | const buf = node.allocatedSliceUnsafe()[@sizeOf(Node)..]; | 427 | const buf = node.allocatedSliceUnsafe()[@sizeOf(Node)..]; |
| 435 | const aligned_index = alignedIndex(buf.ptr, 0, alignment); | 428 | const aligned_index = alignedIndex(buf.ptr, 0, alignment); |
| 436 | | 429 | |
| 437 | if (aligned_index + n <= buf.len) { | 430 | const diff = aligned_index + n -| buf.len; |
| 438 | break :candidate .{ node, it_prev }; | 431 | if (diff < best_fit_diff) { |
| 439 | } | | |
| 440 | | | |
| 441 | const diff = aligned_index + n - buf.len; | | |
| 442 | if (diff <= best_fit_diff) { | | |
| 443 | best_fit_prev = it_prev; | 432 | best_fit_prev = it_prev; |
| 444 | best_fit = node; | 433 | best_fit = node; |
| 445 | best_fit_diff = diff; | 434 | best_fit_diff = diff; |
| 446 | } | 435 | } |
| 447 | } else { | | |
| 448 | // Ideally we want to use all nodes in `free_list` eventually, | | |
| 449 | // so even if none fit we'll try to resize the one that was the | | |
| 450 | // closest to being large enough. | | |
| 451 | if (best_fit) |node| { | | |
| 452 | const allocated_slice = node.allocatedSliceUnsafe(); | | |
| 453 | const buf = allocated_slice[@sizeOf(Node)..]; | | |
| 454 | const aligned_index = alignedIndex(buf.ptr, 0, alignment); | | |
| 455 | const new_size = mem.alignForward(usize, @sizeOf(Node) + aligned_index + n, 2); | | |
| 456 | | | |
| 457 | if (arena.child_allocator.rawResize(allocated_slice, .of(Node), new_size, @returnAddress())) { | | |
| 458 | node.size = .fromInt(new_size); | | |
| 459 | break :candidate .{ node, best_fit_prev }; | | |
| 460 | } | | |
| 461 | } | | |
| 462 | break :from_free_list; | | |
| 463 | } | 436 | } |
| | 437 | |
| | 438 | break :find .{ free_list, it_prev.?, best_fit.?, best_fit_prev }; |
| 464 | }; | 439 | }; |
| 465 | | 440 | |
| 466 | { | 441 | const aligned_index, const need_resize = aligned_index: { |
| 467 | var it = last_free; | 442 | const buf = node.allocatedSliceUnsafe()[@sizeOf(Node)..]; |
| 468 | while (it) |node| : (it = node.next) { | 443 | const aligned_index = alignedIndex(buf.ptr, 0, alignment); |
| 469 | last_free = node; | 444 | break :aligned_index .{ aligned_index, aligned_index + n > buf.len }; |
| | 445 | }; |
| | 446 | |
| | 447 | if (need_resize) { |
| | 448 | // Ideally we want to use all nodes in `free_list` eventually, |
| | 449 | // so even if none fit we'll try to resize the one that was the |
| | 450 | // closest to being large enough. |
| | 451 | const new_size = mem.alignForward(usize, @sizeOf(Node) + aligned_index + n, 2); |
| | 452 | if (arena.child_allocator.rawResize(node.allocatedSliceUnsafe(), .of(Node), new_size, @returnAddress())) { |
| | 453 | node.size = .fromInt(new_size); |
| | 454 | } else { |
| | 455 | arena.pushFreeList(first_free, last_free); |
| | 456 | break :from_free_list; // we couldn't find a fitting free node |
| 470 | } | 457 | } |
| 471 | } | 458 | } |
| 472 | | 459 | |
| 473 | const node = candidate orelse break :from_free_list; | | |
| 474 | const old_next = node.next; | | |
| 475 | | | |
| 476 | const buf = node.allocatedSliceUnsafe()[@sizeOf(Node)..]; | 460 | const buf = node.allocatedSliceUnsafe()[@sizeOf(Node)..]; |
| 477 | const aligned_index = alignedIndex(buf.ptr, 0, alignment); | 461 | const old_next = node.next; |
| 478 | | 462 | |
| 479 | node.end_index = aligned_index + n; | 463 | node.end_index = aligned_index + n; |
| 480 | node.next = first_node; | 464 | node.next = first_node; |
| 481 | | 465 | |
| 482 | switch (arena.tryPushNode(node)) { | 466 | switch (arena.tryPushNode(node)) { |
| 483 | .success => { | 467 | .success => { |
| 484 | // finish removing node from free list | 468 | // Finish removing node from free list. |
| 485 | if (prev) |p| p.next = old_next; | 469 | if (prev) |p| p.next = old_next; |
| 486 | if (node == first_free) first_free = old_next; | 470 | |
| 487 | if (node == last_free) last_free = prev; | 471 | // Push remaining stolen free list back onto `arena.state.free_list`. |
| | 472 | const new_first_free = if (node == first_free) old_next else first_free; |
| | 473 | const new_last_free = if (node == last_free) prev else last_free; |
| | 474 | if (new_first_free) |first| { |
| | 475 | const last = new_last_free.?; |
| | 476 | arena.pushFreeList(first, last); |
| | 477 | } |
| | 478 | |
| 488 | return buf[aligned_index..][0..n].ptr; | 479 | return buf[aligned_index..][0..n].ptr; |
| 489 | }, | 480 | }, |
| 490 | .failure => |old_first_node| { | 481 | .failure => |old_first_node| { |
| 491 | cur_first_node = old_first_node; | | |
| 492 | // restore free list to as we found it | 482 | // restore free list to as we found it |
| 493 | node.next = old_next; | 483 | node.next = old_next; |
| 494 | continue :retry; | 484 | arena.pushFreeList(first_free, last_free); |
| | 485 | |
| | 486 | cur_first_node = old_first_node; |
| | 487 | continue :retry; // there's a new first node; retry! |
| 495 | }, | 488 | }, |
| 496 | } | 489 | } |
| 497 | } | 490 | } |
| ... | @@ -503,16 +496,17 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u | ... | @@ -503,16 +496,17 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u |
| 503 | @branchHint(.cold); | 496 | @branchHint(.cold); |
| 504 | } | 497 | } |
| 505 | | 498 | |
| 506 | const size: usize = size: { | 499 | const size: Node.Size = size: { |
| 507 | const min_size = @sizeOf(Node) + alignment.toByteUnits() + n; | 500 | const min_size = @sizeOf(Node) + alignment.toByteUnits() + n; |
| 508 | const big_enough_size = prev_size + min_size + 16; | 501 | const big_enough_size = prev_size + min_size + 16; |
| 509 | break :size mem.alignForward(usize, big_enough_size + big_enough_size / 2, 2); | 502 | const size = mem.alignForward(usize, big_enough_size + big_enough_size / 2, 2); |
| | 503 | break :size .fromInt(size); |
| 510 | }; | 504 | }; |
| 511 | const ptr = arena.child_allocator.rawAlloc(size, .of(Node), @returnAddress()) orelse | 505 | const ptr = arena.child_allocator.rawAlloc(size.toInt(), .of(Node), @returnAddress()) orelse |
| 512 | return null; | 506 | return null; |
| 513 | const new_node: *Node = @ptrCast(@alignCast(ptr)); | 507 | const new_node: *Node = @ptrCast(@alignCast(ptr)); |
| 514 | new_node.* = .{ | 508 | new_node.* = .{ |
| 515 | .size = .fromInt(size), | 509 | .size = size, |
| 516 | .end_index = undefined, // set below | 510 | .end_index = undefined, // set below |
| 517 | .next = undefined, // set below | 511 | .next = undefined, // set below |
| 518 | }; | 512 | }; |
Justus Klausecker <justus@klausecker.de> 2026-03-05 16:26:01+01:00
lib