| ... | @@ -207,17 +207,20 @@ pub fn reset(arena: *ArenaAllocator, mode: ResetMode) bool { | ... | @@ -207,17 +207,20 @@ pub fn reset(arena: *ArenaAllocator, mode: ResetMode) bool { |
| 207 | | 207 | |
| 208 | /// Concurrent accesses to node pointers generally have to have acquire/release | 208 | /// Concurrent accesses to node pointers generally have to have acquire/release |
| 209 | /// semantics to guarantee that newly allocated notes are in a valid state when | 209 | /// semantics to guarantee that newly allocated notes are in a valid state when |
| 210 | /// being inserted into a list. Exceptions are possible, e.g. a CAS loop that | 210 | /// being inserted into a list. Exceptions are possible, e.g. a cmpxchg loop that |
| 211 | /// never accesses the node returned on failure can use monotonic semantics on | 211 | /// never accesses the node returned on failure can use monotonic semantics on |
| 212 | /// failure, but must still use release semantics on success to protect the node | 212 | /// failure, but must still use release semantics on success to protect the node |
| 213 | /// it's trying to push. | 213 | /// it's trying to push. |
| 214 | const Node = struct { | 214 | const Node = struct { |
| 215 | /// Only meant to be accessed indirectly via the methods supplied by this type, | 215 | /// Only meant to be accessed indirectly via the methods supplied by this type, |
| 216 | /// except if the node is owned by the thread accessing it. | 216 | /// except if the node is owned by the thread accessing it. |
| 217 | /// Must always be an even number to accomodate `resize` bit. | 217 | /// Must always be an even number to accommodate `resize` bit. |
| 218 | size: Size, | 218 | size: Size, |
| 219 | /// Concurrent accesses to `end_index` can be monotonic as long as its value | 219 | /// Any increase of `end_index` has to use acquire semantics; |
| 220 | /// is compared to a version of `size` before using it to access memory. | 220 | /// any decrease of `end_index` that invalidates (formerly) active allocations |
| | 221 | /// has to use release semantics. |
| | 222 | /// This guarantees that all accesses to memory that's about to be freed |
| | 223 | /// happen-before the free is published. |
| 221 | /// Since `size` can only grow and never shrink, memory access depending on | 224 | /// Since `size` can only grow and never shrink, memory access depending on |
| 222 | /// any `end_index` <= any `size` can never be OOB. | 225 | /// any `end_index` <= any `size` can never be OOB. |
| 223 | end_index: usize, | 226 | end_index: usize, |
| ... | @@ -352,10 +355,17 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u | ... | @@ -352,10 +355,17 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u |
| 352 | // with a single cmpxchg afterwards, which may fail. | 355 | // with a single cmpxchg afterwards, which may fail. |
| 353 | | 356 | |
| 354 | const alignable = n + alignment.toByteUnits() - 1; | 357 | const alignable = n + alignment.toByteUnits() - 1; |
| 355 | const end_index = @atomicRmw(usize, &node.end_index, .Add, alignable, .monotonic); | 358 | const end_index = @atomicRmw(usize, &node.end_index, .Add, alignable, .acquire); // acquire any memory that may have been freed |
| 356 | const aligned_index = alignedIndex(buf.ptr, end_index, alignment); | 359 | const aligned_index = alignedIndex(buf.ptr, end_index, alignment); |
| 357 | assert(end_index + alignable >= aligned_index + n); | 360 | assert(end_index + alignable >= aligned_index + n); |
| 358 | _ = @cmpxchgStrong(usize, &node.end_index, end_index + alignable, aligned_index + n, .monotonic, .monotonic); | 361 | _ = @cmpxchgStrong( |
| | 362 | usize, |
| | 363 | &node.end_index, |
| | 364 | end_index + alignable, |
| | 365 | aligned_index + n, |
| | 366 | .monotonic, // no need to release alignment padding; there's no one accessing it! |
| | 367 | .monotonic, |
| | 368 | ); |
| 359 | | 369 | |
| 360 | if (aligned_index + n > buf.len) break :first_node .{ node, buf.len }; | 370 | if (aligned_index + n > buf.len) break :first_node .{ node, buf.len }; |
| 361 | return buf[aligned_index..][0..n].ptr; | 371 | return buf[aligned_index..][0..n].ptr; |
| ... | @@ -377,7 +387,7 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u | ... | @@ -377,7 +387,7 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u |
| 377 | const new_size = mem.alignForward(usize, @sizeOf(Node) + aligned_index + n, 2); | 387 | const new_size = mem.alignForward(usize, @sizeOf(Node) + aligned_index + n, 2); |
| 378 | | 388 | |
| 379 | if (new_size <= allocated_slice.len) { | 389 | if (new_size <= allocated_slice.len) { |
| 380 | // a `resize` or `free` call managed to sneak in and we need to | 390 | // A `resize` or `free` call managed to sneak in and we need to |
| 381 | // guarantee that `size` is only ever increased; retry! | 391 | // guarantee that `size` is only ever increased; retry! |
| 382 | continue :retry; | 392 | continue :retry; |
| 383 | } | 393 | } |
| ... | @@ -385,14 +395,16 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u | ... | @@ -385,14 +395,16 @@ fn alloc(ctx: *anyopaque, n: usize, alignment: Alignment, ret_addr: usize) ?[*]u |
| 385 | if (arena.child_allocator.rawResize(allocated_slice, .of(Node), new_size, @returnAddress())) { | 395 | if (arena.child_allocator.rawResize(allocated_slice, .of(Node), new_size, @returnAddress())) { |
| 386 | size = new_size; | 396 | size = new_size; |
| 387 | | 397 | |
| 388 | if (@cmpxchgStrong( // strong because a spurious failure could result in suboptimal usage of this node | 398 | // strong because a spurious failure could result in suboptimal |
| | 399 | // usage of this node |
| | 400 | if (null == @cmpxchgStrong( |
| 389 | usize, | 401 | usize, |
| 390 | &node.end_index, | 402 | &node.end_index, |
| 391 | end_index, | 403 | end_index, |
| 392 | aligned_index + n, | 404 | aligned_index + n, |
| | 405 | .acquire, // acquire any memory that may have been freed |
| 393 | .monotonic, | 406 | .monotonic, |
| 394 | .monotonic, | 407 | )) { |
| 395 | ) == null) { | | |
| 396 | const new_buf = allocated_slice.ptr[0..new_size][@sizeOf(Node)..]; | 408 | const new_buf = allocated_slice.ptr[0..new_size][@sizeOf(Node)..]; |
| 397 | return new_buf[aligned_index..][0..n].ptr; | 409 | return new_buf[aligned_index..][0..n].ptr; |
| 398 | } | 410 | } |
| ... | @@ -546,35 +558,45 @@ fn resize(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, r | ... | @@ -546,35 +558,45 @@ fn resize(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, r |
| 546 | const buf_ptr = @as([*]u8, @ptrCast(node)) + @sizeOf(Node); | 558 | const buf_ptr = @as([*]u8, @ptrCast(node)) + @sizeOf(Node); |
| 547 | | 559 | |
| 548 | const cur_end_index = @atomicLoad(usize, &node.end_index, .monotonic); | 560 | const cur_end_index = @atomicLoad(usize, &node.end_index, .monotonic); |
| | 561 | |
| 549 | if (buf_ptr + cur_end_index != memory.ptr + memory.len) { | 562 | if (buf_ptr + cur_end_index != memory.ptr + memory.len) { |
| 550 | // It's not the most recent allocation, so it cannot be expanded, | 563 | // It's not the most recent allocation, so it cannot be expanded, |
| 551 | // but it's fine if they want to make it smaller. | 564 | // but it's fine if they want to make it smaller. |
| 552 | return new_len <= memory.len; | 565 | return new_len <= memory.len; |
| 553 | } | 566 | } |
| 554 | | 567 | |
| 555 | const new_end_index: usize = new_end_index: { | 568 | if (new_len <= memory.len) { |
| 556 | if (memory.len >= new_len) { | 569 | const new_end_index = cur_end_index - (memory.len - new_len); |
| 557 | break :new_end_index cur_end_index - (memory.len - new_len); | 570 | assert(buf_ptr + new_end_index == memory.ptr + new_len); |
| 558 | } | 571 | |
| 559 | const cur_buf_len: usize = node.loadBuf().len; | 572 | _ = @cmpxchgStrong( |
| 560 | // Saturating arithmetic because `end_index` and `size` are not | 573 | usize, |
| 561 | // guaranteed to be in sync. | 574 | &node.end_index, |
| 562 | if (cur_buf_len -| cur_end_index >= new_len - memory.len) { | 575 | cur_end_index, |
| 563 | break :new_end_index cur_end_index + (new_len - memory.len); | 576 | new_end_index, |
| 564 | } | 577 | .release, // release freed memory |
| 565 | return false; | 578 | .monotonic, |
| 566 | }; | 579 | ); |
| 567 | assert(buf_ptr + new_end_index == memory.ptr + new_len); | 580 | return true; // Shrinking allocations should always succeed. |
| | 581 | } |
| 568 | | 582 | |
| 569 | return null == @cmpxchgStrong( | 583 | // Saturating arithmetic because `end_index` is not guaranteed to be `<= size`. |
| 570 | usize, | 584 | // The allocation we're trying to resize *could* belong to a different node! |
| 571 | &node.end_index, | 585 | if (node.loadBuf().len -| cur_end_index >= new_len - memory.len) { |
| 572 | cur_end_index, | 586 | const new_end_index = cur_end_index + (new_len - memory.len); |
| 573 | new_end_index, | 587 | assert(buf_ptr + new_end_index == memory.ptr + new_len); |
| 574 | .monotonic, | 588 | |
| 575 | .monotonic, | 589 | return null == @cmpxchgStrong( |
| 576 | ) or | 590 | usize, |
| 577 | new_len <= memory.len; // Shrinking allocations should always succeed. | 591 | &node.end_index, |
| | 592 | cur_end_index, |
| | 593 | new_end_index, |
| | 594 | .acquire, // acquire any memory that may have been freed |
| | 595 | .monotonic, |
| | 596 | ); |
| | 597 | } |
| | 598 | |
| | 599 | return false; |
| 578 | } | 600 | } |
| 579 | | 601 | |
| 580 | fn remap(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { | 602 | fn remap(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { |
| ... | @@ -592,6 +614,7 @@ fn free(ctx: *anyopaque, memory: []u8, alignment: Alignment, ret_addr: usize) vo | ... | @@ -592,6 +614,7 @@ fn free(ctx: *anyopaque, memory: []u8, alignment: Alignment, ret_addr: usize) vo |
| 592 | const buf_ptr = @as([*]u8, @ptrCast(node)) + @sizeOf(Node); | 614 | const buf_ptr = @as([*]u8, @ptrCast(node)) + @sizeOf(Node); |
| 593 | | 615 | |
| 594 | const cur_end_index = @atomicLoad(usize, &node.end_index, .monotonic); | 616 | const cur_end_index = @atomicLoad(usize, &node.end_index, .monotonic); |
| | 617 | |
| 595 | if (buf_ptr + cur_end_index != memory.ptr + memory.len) { | 618 | if (buf_ptr + cur_end_index != memory.ptr + memory.len) { |
| 596 | // Not the most recent allocation; we cannot free it. | 619 | // Not the most recent allocation; we cannot free it. |
| 597 | return; | 620 | return; |
| ... | @@ -605,7 +628,7 @@ fn free(ctx: *anyopaque, memory: []u8, alignment: Alignment, ret_addr: usize) vo | ... | @@ -605,7 +628,7 @@ fn free(ctx: *anyopaque, memory: []u8, alignment: Alignment, ret_addr: usize) vo |
| 605 | &node.end_index, | 628 | &node.end_index, |
| 606 | cur_end_index, | 629 | cur_end_index, |
| 607 | new_end_index, | 630 | new_end_index, |
| 608 | .monotonic, | 631 | .release, // release freed memory |
| 609 | .monotonic, | 632 | .monotonic, |
| 610 | ); | 633 | ); |
| 611 | } | 634 | } |
Justus Klausecker <justus@klausecker.de> 2026-03-24 16:32:10+01:00