authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-08-08 00:34:13-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-08-08 00:34:13-07:00
logcd6cdd0a752ba3e134ba371246cc02b1d7faaa88
treee4b6aaa5a1d47466addf16fc1b60a9b40cb98bf8
parent0347df82e8c821906ef0d07ec65fe4b3884c0212

std.mem.Allocator: add return_address to the interface

The high level Allocator interface API functions will now do a `@returnAddress()` so that stack traces captured by allocator implementations have a return address that does not include the Allocator overhead functions. This makes `4` a more reasonable default for how many stack frames to capture.

7 files changed, 227 insertions(+), 88 deletions(-)

lib/std/heap.zig+54-13
...@@ -100,7 +100,7 @@ pub fn alignPageAllocLen(full_len: usize, len: usize, len_align: u29) usize {...@@ -100,7 +100,7 @@ pub fn alignPageAllocLen(full_len: usize, len: usize, len_align: u29) usize {
100pub var next_mmap_addr_hint: ?[*]align(mem.page_size) u8 = null;100pub var next_mmap_addr_hint: ?[*]align(mem.page_size) u8 = null;
101101
102const PageAllocator = struct {102const PageAllocator = struct {
103 fn alloc(allocator: *Allocator, n: usize, alignment: u29, len_align: u29) error{OutOfMemory}![]u8 {103 fn alloc(allocator: *Allocator, n: usize, alignment: u29, len_align: u29, ra: usize) error{OutOfMemory}![]u8 {
104 assert(n > 0);104 assert(n > 0);
105 const aligned_len = mem.alignForward(n, mem.page_size);105 const aligned_len = mem.alignForward(n, mem.page_size);
106106
...@@ -196,7 +196,14 @@ const PageAllocator = struct {...@@ -196,7 +196,14 @@ const PageAllocator = struct {
196 return result_ptr[0..alignPageAllocLen(aligned_len, n, len_align)];196 return result_ptr[0..alignPageAllocLen(aligned_len, n, len_align)];
197 }197 }
198198
199 fn resize(allocator: *Allocator, buf_unaligned: []u8, buf_align: u29, new_size: usize, len_align: u29) Allocator.Error!usize {199 fn resize(
200 allocator: *Allocator,
201 buf_unaligned: []u8,
202 buf_align: u29,
203 new_size: usize,
204 len_align: u29,
205 return_address: usize,
206 ) Allocator.Error!usize {
200 const new_size_aligned = mem.alignForward(new_size, mem.page_size);207 const new_size_aligned = mem.alignForward(new_size, mem.page_size);
201208
202 if (builtin.os.tag == .windows) {209 if (builtin.os.tag == .windows) {
...@@ -344,7 +351,7 @@ const WasmPageAllocator = struct {...@@ -344,7 +351,7 @@ const WasmPageAllocator = struct {
344 return mem.alignForward(memsize, mem.page_size) / mem.page_size;351 return mem.alignForward(memsize, mem.page_size) / mem.page_size;
345 }352 }
346353
347 fn alloc(allocator: *Allocator, len: usize, alignment: u29, len_align: u29) error{OutOfMemory}![]u8 {354 fn alloc(allocator: *Allocator, len: usize, alignment: u29, len_align: u29, ra: usize) error{OutOfMemory}![]u8 {
348 const page_count = nPages(len);355 const page_count = nPages(len);
349 const page_idx = try allocPages(page_count, alignment);356 const page_idx = try allocPages(page_count, alignment);
350 return @intToPtr([*]u8, page_idx * mem.page_size)[0..alignPageAllocLen(page_count * mem.page_size, len, len_align)];357 return @intToPtr([*]u8, page_idx * mem.page_size)[0..alignPageAllocLen(page_count * mem.page_size, len, len_align)];
...@@ -397,7 +404,14 @@ const WasmPageAllocator = struct {...@@ -397,7 +404,14 @@ const WasmPageAllocator = struct {
397 }404 }
398 }405 }
399406
400 fn resize(allocator: *Allocator, buf: []u8, new_len: usize, len_align: u29) error{OutOfMemory}!usize {407 fn resize(
408 allocator: *Allocator,
409 buf: []u8,
410 buf_align: u29,
411 new_len: usize,
412 len_align: u29,
413 return_address: usize,
414 ) error{OutOfMemory}!usize {
401 const aligned_len = mem.alignForward(buf.len, mem.page_size);415 const aligned_len = mem.alignForward(buf.len, mem.page_size);
402 if (new_len > aligned_len) return error.OutOfMemory;416 if (new_len > aligned_len) return error.OutOfMemory;
403 const current_n = nPages(aligned_len);417 const current_n = nPages(aligned_len);
...@@ -437,7 +451,13 @@ pub const HeapAllocator = switch (builtin.os.tag) {...@@ -437,7 +451,13 @@ pub const HeapAllocator = switch (builtin.os.tag) {
437 return @intToPtr(*align(1) usize, @ptrToInt(buf.ptr) + buf.len);451 return @intToPtr(*align(1) usize, @ptrToInt(buf.ptr) + buf.len);
438 }452 }
439453
440 fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {454 fn alloc(
455 allocator: *Allocator,
456 n: usize,
457 ptr_align: u29,
458 len_align: u29,
459 return_address: usize,
460 ) error{OutOfMemory}![]u8 {
441 const self = @fieldParentPtr(HeapAllocator, "allocator", allocator);461 const self = @fieldParentPtr(HeapAllocator, "allocator", allocator);
442462
443 const amt = n + ptr_align - 1 + @sizeOf(usize);463 const amt = n + ptr_align - 1 + @sizeOf(usize);
...@@ -470,6 +490,7 @@ pub const HeapAllocator = switch (builtin.os.tag) {...@@ -470,6 +490,7 @@ pub const HeapAllocator = switch (builtin.os.tag) {
470 buf_align: u29,490 buf_align: u29,
471 new_size: usize,491 new_size: usize,
472 len_align: u29,492 len_align: u29,
493 return_address: usize,
473 ) error{OutOfMemory}!usize {494 ) error{OutOfMemory}!usize {
474 const self = @fieldParentPtr(HeapAllocator, "allocator", allocator);495 const self = @fieldParentPtr(HeapAllocator, "allocator", allocator);
475 if (new_size == 0) {496 if (new_size == 0) {
...@@ -542,7 +563,7 @@ pub const FixedBufferAllocator = struct {...@@ -542,7 +563,7 @@ pub const FixedBufferAllocator = struct {
542 return buf.ptr + buf.len == self.buffer.ptr + self.end_index;563 return buf.ptr + buf.len == self.buffer.ptr + self.end_index;
543 }564 }
544565
545 fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) ![]u8 {566 fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29, ra: usize) ![]u8 {
546 const self = @fieldParentPtr(FixedBufferAllocator, "allocator", allocator);567 const self = @fieldParentPtr(FixedBufferAllocator, "allocator", allocator);
547 const aligned_addr = mem.alignForward(@ptrToInt(self.buffer.ptr) + self.end_index, ptr_align);568 const aligned_addr = mem.alignForward(@ptrToInt(self.buffer.ptr) + self.end_index, ptr_align);
548 const adjusted_index = aligned_addr - @ptrToInt(self.buffer.ptr);569 const adjusted_index = aligned_addr - @ptrToInt(self.buffer.ptr);
...@@ -556,7 +577,14 @@ pub const FixedBufferAllocator = struct {...@@ -556,7 +577,14 @@ pub const FixedBufferAllocator = struct {
556 return result;577 return result;
557 }578 }
558579
559 fn resize(allocator: *Allocator, buf: []u8, buf_align: u29, new_size: usize, len_align: u29) Allocator.Error!usize {580 fn resize(
581 allocator: *Allocator,
582 buf: []u8,
583 buf_align: u29,
584 new_size: usize,
585 len_align: u29,
586 return_address: usize,
587 ) Allocator.Error!usize {
560 const self = @fieldParentPtr(FixedBufferAllocator, "allocator", allocator);588 const self = @fieldParentPtr(FixedBufferAllocator, "allocator", allocator);
561 assert(self.ownsSlice(buf)); // sanity check589 assert(self.ownsSlice(buf)); // sanity check
562590
...@@ -606,7 +634,7 @@ pub const ThreadSafeFixedBufferAllocator = blk: {...@@ -606,7 +634,7 @@ pub const ThreadSafeFixedBufferAllocator = blk: {
606 };634 };
607 }635 }
608636
609 fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) ![]u8 {637 fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29, ra: usize) ![]u8 {
610 const self = @fieldParentPtr(ThreadSafeFixedBufferAllocator, "allocator", allocator);638 const self = @fieldParentPtr(ThreadSafeFixedBufferAllocator, "allocator", allocator);
611 var end_index = @atomicLoad(usize, &self.end_index, builtin.AtomicOrder.SeqCst);639 var end_index = @atomicLoad(usize, &self.end_index, builtin.AtomicOrder.SeqCst);
612 while (true) {640 while (true) {
...@@ -654,18 +682,31 @@ pub fn StackFallbackAllocator(comptime size: usize) type {...@@ -654,18 +682,31 @@ pub fn StackFallbackAllocator(comptime size: usize) type {
654 return &self.allocator;682 return &self.allocator;
655 }683 }
656684
657 fn alloc(allocator: *Allocator, len: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![*]u8 {685 fn alloc(
686 allocator: *Allocator,
687 len: usize,
688 ptr_align: u29,
689 len_align: u29,
690 return_address: usize,
691 ) error{OutOfMemory}![*]u8 {
658 const self = @fieldParentPtr(Self, "allocator", allocator);692 const self = @fieldParentPtr(Self, "allocator", allocator);
659 return FixedBufferAllocator.alloc(&self.fixed_buffer_allocator, len, ptr_align) catch693 return FixedBufferAllocator.alloc(&self.fixed_buffer_allocator, len, ptr_align) catch
660 return fallback_allocator.alloc(len, ptr_align);694 return fallback_allocator.alloc(len, ptr_align);
661 }695 }
662696
663 fn resize(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) error{OutOfMemory}!void {697 fn resize(
698 self: *Allocator,
699 buf: []u8,
700 buf_align: u29,
701 new_len: usize,
702 len_align: u29,
703 return_address: usize,
704 ) error{OutOfMemory}!void {
664 const self = @fieldParentPtr(Self, "allocator", allocator);705 const self = @fieldParentPtr(Self, "allocator", allocator);
665 if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) {706 if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) {
666 try self.fixed_buffer_allocator.callResizeFn(buf, new_len);707 try self.fixed_buffer_allocator.resize(buf, new_len);
667 } else {708 } else {
668 try self.fallback_allocator.callResizeFn(buf, new_len);709 try self.fallback_allocator.resize(buf, new_len);
669 }710 }
670 }711 }
671 };712 };
...@@ -950,7 +991,7 @@ pub fn testAllocatorAlignedShrink(base_allocator: *mem.Allocator) mem.Allocator....@@ -950,7 +991,7 @@ pub fn testAllocatorAlignedShrink(base_allocator: *mem.Allocator) mem.Allocator.
950 slice[60] = 0x34;991 slice[60] = 0x34;
951992
952 // realloc to a smaller size but with a larger alignment993 // realloc to a smaller size but with a larger alignment
953 slice = try allocator.alignedRealloc(slice, mem.page_size * 32, alloc_size / 2);994 slice = try allocator.reallocAdvanced(slice, mem.page_size * 32, alloc_size / 2, .exact);
954 testing.expect(slice[0] == 0x12);995 testing.expect(slice[0] == 0x12);
955 testing.expect(slice[60] == 0x34);996 testing.expect(slice[60] == 0x34);
956}997}
lib/std/heap/arena_allocator.zig+2-2
...@@ -49,7 +49,7 @@ pub const ArenaAllocator = struct {...@@ -49,7 +49,7 @@ pub const ArenaAllocator = struct {
49 const actual_min_size = minimum_size + (@sizeOf(BufNode) + 16);49 const actual_min_size = minimum_size + (@sizeOf(BufNode) + 16);
50 const big_enough_len = prev_len + actual_min_size;50 const big_enough_len = prev_len + actual_min_size;
51 const len = big_enough_len + big_enough_len / 2;51 const len = big_enough_len + big_enough_len / 2;
52 const buf = try self.child_allocator.allocFn(self.child_allocator, len, @alignOf(BufNode), 1);52 const buf = try self.child_allocator.allocFn(self.child_allocator, len, @alignOf(BufNode), 1, @returnAddress());
53 const buf_node = @ptrCast(*BufNode, @alignCast(@alignOf(BufNode), buf.ptr));53 const buf_node = @ptrCast(*BufNode, @alignCast(@alignOf(BufNode), buf.ptr));
54 buf_node.* = BufNode{54 buf_node.* = BufNode{
55 .data = buf,55 .data = buf,
...@@ -60,7 +60,7 @@ pub const ArenaAllocator = struct {...@@ -60,7 +60,7 @@ pub const ArenaAllocator = struct {
60 return buf_node;60 return buf_node;
61 }61 }
6262
63 fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) ![]u8 {63 fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29, ra: usize) ![]u8 {
64 const self = @fieldParentPtr(ArenaAllocator, "allocator", allocator);64 const self = @fieldParentPtr(ArenaAllocator, "allocator", allocator);
6565
66 var cur_node = if (self.state.buffer_list.first) |first_node| first_node else try self.createNode(0, n + ptr_align);66 var cur_node = if (self.state.buffer_list.first) |first_node| first_node else try self.createNode(0, n + ptr_align);
lib/std/heap/general_purpose_allocator.zig+20-19
...@@ -104,7 +104,7 @@ const SlotIndex = std.meta.Int(false, math.log2(page_size) + 1);...@@ -104,7 +104,7 @@ const SlotIndex = std.meta.Int(false, math.log2(page_size) + 1);
104104
105pub const Config = struct {105pub const Config = struct {
106 /// Number of stack frames to capture.106 /// Number of stack frames to capture.
107 stack_trace_frames: usize = if (std.debug.runtime_safety) @as(usize, 6) else @as(usize, 0),107 stack_trace_frames: usize = if (std.debug.runtime_safety) @as(usize, 4) else @as(usize, 0),
108108
109 /// If true, the allocator will have two fields:109 /// If true, the allocator will have two fields:
110 /// * `total_requested_bytes` which tracks the total allocated bytes of memory requested.110 /// * `total_requested_bytes` which tracks the total allocated bytes of memory requested.
...@@ -199,7 +199,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -199,7 +199,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
199199
200 fn captureStackTrace(200 fn captureStackTrace(
201 bucket: *BucketHeader,201 bucket: *BucketHeader,
202 return_address: usize,202 ret_addr: usize,
203 size_class: usize,203 size_class: usize,
204 slot_index: SlotIndex,204 slot_index: SlotIndex,
205 trace_kind: TraceKind,205 trace_kind: TraceKind,
...@@ -207,7 +207,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -207,7 +207,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
207 // Initialize them to 0. When determining the count we must look207 // Initialize them to 0. When determining the count we must look
208 // for non zero addresses.208 // for non zero addresses.
209 const stack_addresses = bucket.stackTracePtr(size_class, slot_index, trace_kind);209 const stack_addresses = bucket.stackTracePtr(size_class, slot_index, trace_kind);
210 collectStackTrace(return_address, stack_addresses);210 collectStackTrace(ret_addr, stack_addresses);
211 }211 }
212 };212 };
213213
...@@ -284,7 +284,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -284,7 +284,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
284 var leaks = false;284 var leaks = false;
285 for (self.buckets) |optional_bucket, bucket_i| {285 for (self.buckets) |optional_bucket, bucket_i| {
286 const first_bucket = optional_bucket orelse continue;286 const first_bucket = optional_bucket orelse continue;
287 const size_class = @as(usize, 1) << @intCast(u6, bucket_i);287 const size_class = @as(usize, 1) << @intCast(math.Log2Int(usize), bucket_i);
288 const used_bits_count = usedBitsCount(size_class);288 const used_bits_count = usedBitsCount(size_class);
289 var bucket = first_bucket;289 var bucket = first_bucket;
290 while (true) {290 while (true) {
...@@ -377,7 +377,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -377,7 +377,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
377 trace_addr: usize,377 trace_addr: usize,
378 ) void {378 ) void {
379 // Capture stack trace to be the "first free", in case a double free happens.379 // Capture stack trace to be the "first free", in case a double free happens.
380 bucket.captureStackTrace(@returnAddress(), size_class, slot_index, .free);380 bucket.captureStackTrace(trace_addr, size_class, slot_index, .free);
381381
382 used_byte.* &= ~(@as(u8, 1) << used_bit_index);382 used_byte.* &= ~(@as(u8, 1) << used_bit_index);
383 bucket.used_count -= 1;383 bucket.used_count -= 1;
...@@ -408,7 +408,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -408,7 +408,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
408 old_align: u29,408 old_align: u29,
409 new_size: usize,409 new_size: usize,
410 len_align: u29,410 len_align: u29,
411 return_addr: usize,411 ret_addr: usize,
412 ) Error!usize {412 ) Error!usize {
413 const entry = self.large_allocations.getEntry(@ptrToInt(old_mem.ptr)) orelse {413 const entry = self.large_allocations.getEntry(@ptrToInt(old_mem.ptr)) orelse {
414 if (config.safety) {414 if (config.safety) {
...@@ -428,7 +428,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -428,7 +428,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
428 @panic("\nFree here:");428 @panic("\nFree here:");
429 }429 }
430430
431 const result_len = try self.backing_allocator.resizeFn(self.backing_allocator, old_mem, old_align, new_size, len_align);431 const result_len = try self.backing_allocator.resizeFn(self.backing_allocator, old_mem, old_align, new_size, len_align, ret_addr);
432432
433 if (result_len == 0) {433 if (result_len == 0) {
434 self.large_allocations.removeAssertDiscard(@ptrToInt(old_mem.ptr));434 self.large_allocations.removeAssertDiscard(@ptrToInt(old_mem.ptr));
...@@ -436,7 +436,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -436,7 +436,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
436 }436 }
437437
438 entry.value.bytes = old_mem.ptr[0..result_len];438 entry.value.bytes = old_mem.ptr[0..result_len];
439 collectStackTrace(return_addr, &entry.value.stack_addresses);439 collectStackTrace(ret_addr, &entry.value.stack_addresses);
440 return result_len;440 return result_len;
441 }441 }
442442
...@@ -450,6 +450,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -450,6 +450,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
450 old_align: u29,450 old_align: u29,
451 new_size: usize,451 new_size: usize,
452 len_align: u29,452 len_align: u29,
453 ret_addr: usize,
453 ) Error!usize {454 ) Error!usize {
454 const self = @fieldParentPtr(Self, "allocator", allocator);455 const self = @fieldParentPtr(Self, "allocator", allocator);
455456
...@@ -472,7 +473,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -472,7 +473,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
472473
473 const aligned_size = math.max(old_mem.len, old_align);474 const aligned_size = math.max(old_mem.len, old_align);
474 if (aligned_size > largest_bucket_object_size) {475 if (aligned_size > largest_bucket_object_size) {
475 return self.resizeLarge(old_mem, old_align, new_size, len_align, @returnAddress());476 return self.resizeLarge(old_mem, old_align, new_size, len_align, ret_addr);
476 }477 }
477 const size_class_hint = up_to_nearest_power_of_2(usize, aligned_size);478 const size_class_hint = up_to_nearest_power_of_2(usize, aligned_size);
478479
...@@ -484,7 +485,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -484,7 +485,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
484 }485 }
485 size_class *= 2;486 size_class *= 2;
486 } else {487 } else {
487 return self.resizeLarge(old_mem, old_align, new_size, len_align, @returnAddress());488 return self.resizeLarge(old_mem, old_align, new_size, len_align, ret_addr);
488 };489 };
489 const byte_offset = @ptrToInt(old_mem.ptr) - @ptrToInt(bucket.page);490 const byte_offset = @ptrToInt(old_mem.ptr) - @ptrToInt(bucket.page);
490 const slot_index = @intCast(SlotIndex, byte_offset / size_class);491 const slot_index = @intCast(SlotIndex, byte_offset / size_class);
...@@ -507,7 +508,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -507,7 +508,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
507 }508 }
508 }509 }
509 if (new_size == 0) {510 if (new_size == 0) {
510 self.freeSlot(bucket, bucket_index, size_class, slot_index, used_byte, used_bit_index, @returnAddress());511 self.freeSlot(bucket, bucket_index, size_class, slot_index, used_byte, used_bit_index, ret_addr);
511 return @as(usize, 0);512 return @as(usize, 0);
512 }513 }
513 const new_aligned_size = math.max(new_size, old_align);514 const new_aligned_size = math.max(new_size, old_align);
...@@ -518,7 +519,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -518,7 +519,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
518 return error.OutOfMemory;519 return error.OutOfMemory;
519 }520 }
520521
521 fn alloc(allocator: *Allocator, len: usize, ptr_align: u29, len_align: u29) Error![]u8 {522 fn alloc(allocator: *Allocator, len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Error![]u8 {
522 const self = @fieldParentPtr(Self, "allocator", allocator);523 const self = @fieldParentPtr(Self, "allocator", allocator);
523524
524 const held = self.mutex.acquire();525 const held = self.mutex.acquire();
...@@ -543,17 +544,17 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {...@@ -543,17 +544,17 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
543 self.large_allocations.entries.items.len + 1,544 self.large_allocations.entries.items.len + 1,
544 );545 );
545546
546 const slice = try self.backing_allocator.allocFn(self.backing_allocator, len, ptr_align, len_align);547 const slice = try self.backing_allocator.allocFn(self.backing_allocator, len, ptr_align, len_align, ret_addr);
547548
548 const gop = self.large_allocations.getOrPutAssumeCapacity(@ptrToInt(slice.ptr));549 const gop = self.large_allocations.getOrPutAssumeCapacity(@ptrToInt(slice.ptr));
549 assert(!gop.found_existing); // This would mean the kernel double-mapped pages.550 assert(!gop.found_existing); // This would mean the kernel double-mapped pages.
550 gop.entry.value.bytes = slice;551 gop.entry.value.bytes = slice;
551 collectStackTrace(@returnAddress(), &gop.entry.value.stack_addresses);552 collectStackTrace(ret_addr, &gop.entry.value.stack_addresses);
552553
553 return slice;554 return slice;
554 } else {555 } else {
555 const new_size_class = up_to_nearest_power_of_2(usize, new_aligned_size);556 const new_size_class = up_to_nearest_power_of_2(usize, new_aligned_size);
556 const ptr = try self.allocSlot(new_size_class, @returnAddress());557 const ptr = try self.allocSlot(new_size_class, ret_addr);
557 return ptr[0..len];558 return ptr[0..len];
558 }559 }
559 }560 }
...@@ -782,7 +783,7 @@ test "shrink large object to large object with larger alignment" {...@@ -782,7 +783,7 @@ test "shrink large object to large object with larger alignment" {
782 slice[0] = 0x12;783 slice[0] = 0x12;
783 slice[60] = 0x34;784 slice[60] = 0x34;
784785
785 slice = try allocator.alignedRealloc(slice, page_size * 2, alloc_size / 2);786 slice = try allocator.reallocAdvanced(slice, page_size * 2, alloc_size / 2, .exact);
786 assert(slice[0] == 0x12);787 assert(slice[0] == 0x12);
787 assert(slice[60] == 0x34);788 assert(slice[60] == 0x34);
788}789}
...@@ -833,15 +834,15 @@ test "realloc large object to larger alignment" {...@@ -833,15 +834,15 @@ test "realloc large object to larger alignment" {
833 slice[0] = 0x12;834 slice[0] = 0x12;
834 slice[16] = 0x34;835 slice[16] = 0x34;
835836
836 slice = try allocator.alignedRealloc(slice, 32, page_size * 2 + 100);837 slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 100, .exact);
837 assert(slice[0] == 0x12);838 assert(slice[0] == 0x12);
838 assert(slice[16] == 0x34);839 assert(slice[16] == 0x34);
839840
840 slice = try allocator.alignedRealloc(slice, 32, page_size * 2 + 25);841 slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 25, .exact);
841 assert(slice[0] == 0x12);842 assert(slice[0] == 0x12);
842 assert(slice[16] == 0x34);843 assert(slice[16] == 0x34);
843844
844 slice = try allocator.alignedRealloc(slice, page_size * 2, page_size * 2 + 100);845 slice = try allocator.reallocAdvanced(slice, page_size * 2, page_size * 2 + 100, .exact);
845 assert(slice[0] == 0x12);846 assert(slice[0] == 0x12);
846 assert(slice[16] == 0x34);847 assert(slice[16] == 0x34);
847}848}
lib/std/heap/logging_allocator.zig+12-5
...@@ -23,10 +23,16 @@ pub fn LoggingAllocator(comptime OutStreamType: type) type {...@@ -23,10 +23,16 @@ pub fn LoggingAllocator(comptime OutStreamType: type) type {
23 };23 };
24 }24 }
2525
26 fn alloc(allocator: *Allocator, len: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {26 fn alloc(
27 allocator: *Allocator,
28 len: usize,
29 ptr_align: u29,
30 len_align: u29,
31 ra: usize,
32 ) error{OutOfMemory}![]u8 {
27 const self = @fieldParentPtr(Self, "allocator", allocator);33 const self = @fieldParentPtr(Self, "allocator", allocator);
28 self.out_stream.print("alloc : {}", .{len}) catch {};34 self.out_stream.print("alloc : {}", .{len}) catch {};
29 const result = self.parent_allocator.allocFn(self.parent_allocator, len, ptr_align, len_align);35 const result = self.parent_allocator.allocFn(self.parent_allocator, len, ptr_align, len_align, ra);
30 if (result) |buff| {36 if (result) |buff| {
31 self.out_stream.print(" success!\n", .{}) catch {};37 self.out_stream.print(" success!\n", .{}) catch {};
32 } else |err| {38 } else |err| {
...@@ -41,6 +47,7 @@ pub fn LoggingAllocator(comptime OutStreamType: type) type {...@@ -41,6 +47,7 @@ pub fn LoggingAllocator(comptime OutStreamType: type) type {
41 buf_align: u29,47 buf_align: u29,
42 new_len: usize,48 new_len: usize,
43 len_align: u29,49 len_align: u29,
50 ra: usize,
44 ) error{OutOfMemory}!usize {51 ) error{OutOfMemory}!usize {
45 const self = @fieldParentPtr(Self, "allocator", allocator);52 const self = @fieldParentPtr(Self, "allocator", allocator);
46 if (new_len == 0) {53 if (new_len == 0) {
...@@ -50,7 +57,7 @@ pub fn LoggingAllocator(comptime OutStreamType: type) type {...@@ -50,7 +57,7 @@ pub fn LoggingAllocator(comptime OutStreamType: type) type {
50 } else {57 } else {
51 self.out_stream.print("expand: {} to {}", .{ buf.len, new_len }) catch {};58 self.out_stream.print("expand: {} to {}", .{ buf.len, new_len }) catch {};
52 }59 }
53 if (self.parent_allocator.resizeFn(self.parent_allocator, buf, buf_align, new_len, len_align)) |resized_len| {60 if (self.parent_allocator.resizeFn(self.parent_allocator, buf, buf_align, new_len, len_align, ra)) |resized_len| {
54 if (new_len > buf.len) {61 if (new_len > buf.len) {
55 self.out_stream.print(" success!\n", .{}) catch {};62 self.out_stream.print(" success!\n", .{}) catch {};
56 }63 }
...@@ -80,9 +87,9 @@ test "LoggingAllocator" {...@@ -80,9 +87,9 @@ test "LoggingAllocator" {
80 const allocator = &loggingAllocator(&fixedBufferAllocator.allocator, fbs.outStream()).allocator;87 const allocator = &loggingAllocator(&fixedBufferAllocator.allocator, fbs.outStream()).allocator;
8188
82 var a = try allocator.alloc(u8, 10);89 var a = try allocator.alloc(u8, 10);
83 a.len = allocator.shrinkBytes(a, 1, 5, 0);90 a = allocator.shrink(a, 5);
84 std.debug.assert(a.len == 5);91 std.debug.assert(a.len == 5);
85 std.testing.expectError(error.OutOfMemory, allocator.resizeFn(allocator, a, 1, 20, 0));92 std.testing.expectError(error.OutOfMemory, allocator.resize(a, 20));
86 allocator.free(a);93 allocator.free(a);
8794
88 std.testing.expectEqualSlices(u8,95 std.testing.expectEqualSlices(u8,
lib/std/mem.zig+11-4
...@@ -37,7 +37,13 @@ pub fn ValidationAllocator(comptime T: type) type {...@@ -37,7 +37,13 @@ pub fn ValidationAllocator(comptime T: type) type {
37 if (*T == *Allocator) return &self.underlying_allocator;37 if (*T == *Allocator) return &self.underlying_allocator;
38 return &self.underlying_allocator.allocator;38 return &self.underlying_allocator.allocator;
39 }39 }
40 pub fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) Allocator.Error![]u8 {40 pub fn alloc(
41 allocator: *Allocator,
42 n: usize,
43 ptr_align: u29,
44 len_align: u29,
45 ret_addr: usize,
46 ) Allocator.Error![]u8 {
41 assert(n > 0);47 assert(n > 0);
42 assert(mem.isValidAlign(ptr_align));48 assert(mem.isValidAlign(ptr_align));
43 if (len_align != 0) {49 if (len_align != 0) {
...@@ -47,7 +53,7 @@ pub fn ValidationAllocator(comptime T: type) type {...@@ -47,7 +53,7 @@ pub fn ValidationAllocator(comptime T: type) type {
4753
48 const self = @fieldParentPtr(@This(), "allocator", allocator);54 const self = @fieldParentPtr(@This(), "allocator", allocator);
49 const underlying = self.getUnderlyingAllocatorPtr();55 const underlying = self.getUnderlyingAllocatorPtr();
50 const result = try underlying.allocFn(underlying, n, ptr_align, len_align);56 const result = try underlying.allocFn(underlying, n, ptr_align, len_align, ret_addr);
51 assert(mem.isAligned(@ptrToInt(result.ptr), ptr_align));57 assert(mem.isAligned(@ptrToInt(result.ptr), ptr_align));
52 if (len_align == 0) {58 if (len_align == 0) {
53 assert(result.len == n);59 assert(result.len == n);
...@@ -63,6 +69,7 @@ pub fn ValidationAllocator(comptime T: type) type {...@@ -63,6 +69,7 @@ pub fn ValidationAllocator(comptime T: type) type {
63 buf_align: u29,69 buf_align: u29,
64 new_len: usize,70 new_len: usize,
65 len_align: u29,71 len_align: u29,
72 ret_addr: usize,
66 ) Allocator.Error!usize {73 ) Allocator.Error!usize {
67 assert(buf.len > 0);74 assert(buf.len > 0);
68 if (len_align != 0) {75 if (len_align != 0) {
...@@ -71,7 +78,7 @@ pub fn ValidationAllocator(comptime T: type) type {...@@ -71,7 +78,7 @@ pub fn ValidationAllocator(comptime T: type) type {
71 }78 }
72 const self = @fieldParentPtr(@This(), "allocator", allocator);79 const self = @fieldParentPtr(@This(), "allocator", allocator);
73 const underlying = self.getUnderlyingAllocatorPtr();80 const underlying = self.getUnderlyingAllocatorPtr();
74 const result = try underlying.resizeFn(underlying, buf, buf_align, new_len, len_align);81 const result = try underlying.resizeFn(underlying, buf, buf_align, new_len, len_align, ret_addr);
75 if (len_align == 0) {82 if (len_align == 0) {
76 assert(result == new_len);83 assert(result == new_len);
77 } else {84 } else {
...@@ -111,7 +118,7 @@ var failAllocator = Allocator{...@@ -111,7 +118,7 @@ var failAllocator = Allocator{
111 .allocFn = failAllocatorAlloc,118 .allocFn = failAllocatorAlloc,
112 .resizeFn = Allocator.noResize,119 .resizeFn = Allocator.noResize,
113};120};
114fn failAllocatorAlloc(self: *Allocator, n: usize, alignment: u29, len_align: u29) Allocator.Error![]u8 {121fn failAllocatorAlloc(self: *Allocator, n: usize, alignment: u29, len_align: u29, ra: usize) Allocator.Error![]u8 {
115 return error.OutOfMemory;122 return error.OutOfMemory;
116}123}
117124
lib/std/mem/Allocator.zig+118-42
...@@ -14,7 +14,10 @@ pub const Error = error{OutOfMemory};...@@ -14,7 +14,10 @@ pub const Error = error{OutOfMemory};
14/// otherwise, the length must be aligned to `len_align`.14/// otherwise, the length must be aligned to `len_align`.
15///15///
16/// `len` must be greater than or equal to `len_align` and must be aligned by `len_align`.16/// `len` must be greater than or equal to `len_align` and must be aligned by `len_align`.
17allocFn: fn (self: *Allocator, len: usize, ptr_align: u29, len_align: u29) Error![]u8,17///
18/// `ret_addr` is optionally provided as the first return address of the allocation call stack.
19/// If the value is `0` it means no return address has been provided.
20allocFn: fn (self: *Allocator, len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Error![]u8,
1821
19/// Attempt to expand or shrink memory in place. `buf.len` must equal the most recent22/// Attempt to expand or shrink memory in place. `buf.len` must equal the most recent
20/// length returned by `allocFn` or `resizeFn`. `buf_align` must equal the same value23/// length returned by `allocFn` or `resizeFn`. `buf_align` must equal the same value
...@@ -33,22 +36,25 @@ allocFn: fn (self: *Allocator, len: usize, ptr_align: u29, len_align: u29) Error...@@ -33,22 +36,25 @@ allocFn: fn (self: *Allocator, len: usize, ptr_align: u29, len_align: u29) Error
33/// accepting more bytes of memory from the allocator than requested.36/// accepting more bytes of memory from the allocator than requested.
34///37///
35/// `new_len` must be greater than or equal to `len_align` and must be aligned by `len_align`.38/// `new_len` must be greater than or equal to `len_align` and must be aligned by `len_align`.
36resizeFn: fn (self: *Allocator, buf: []u8, buf_align: u29, new_len: usize, len_align: u29) Error!usize,39///
40/// `ret_addr` is optionally provided as the first return address of the allocation call stack.
41/// If the value is `0` it means no return address has been provided.
42resizeFn: fn (self: *Allocator, buf: []u8, buf_align: u29, new_len: usize, len_align: u29, ret_addr: usize) Error!usize,
3743
38/// Set to resizeFn if in-place resize is not supported.44/// Set to resizeFn if in-place resize is not supported.
39pub fn noResize(self: *Allocator, buf: []u8, buf_align: u29, new_len: usize, len_align: u29) Error!usize {45pub fn noResize(
46 self: *Allocator,
47 buf: []u8,
48 buf_align: u29,
49 new_len: usize,
50 len_align: u29,
51 ret_addr: usize,
52) Error!usize {
40 if (new_len > buf.len)53 if (new_len > buf.len)
41 return error.OutOfMemory;54 return error.OutOfMemory;
42 return new_len;55 return new_len;
43}56}
4457
45/// Call `resizeFn`, but caller guarantees that `new_len` <= `buf.len` meaning
46/// error.OutOfMemory should be impossible.
47pub fn shrinkBytes(self: *Allocator, buf: []u8, buf_align: u29, new_len: usize, len_align: u29) usize {
48 assert(new_len <= buf.len);
49 return self.resizeFn(self, buf, buf_align, new_len, len_align) catch unreachable;
50}
51
52/// Realloc is used to modify the size or alignment of an existing allocation,58/// Realloc is used to modify the size or alignment of an existing allocation,
53/// as well as to provide the allocator with an opportunity to move an allocation59/// as well as to provide the allocator with an opportunity to move an allocation
54/// to a better location.60/// to a better location.
...@@ -93,9 +99,10 @@ fn reallocBytes(...@@ -93,9 +99,10 @@ fn reallocBytes(
93 /// non-zero means the length of the returned slice must be aligned by `len_align`99 /// non-zero means the length of the returned slice must be aligned by `len_align`
94 /// `new_len` must be aligned by `len_align`100 /// `new_len` must be aligned by `len_align`
95 len_align: u29,101 len_align: u29,
102 return_address: usize,
96) Error![]u8 {103) Error![]u8 {
97 if (old_mem.len == 0) {104 if (old_mem.len == 0) {
98 const new_mem = try self.allocFn(self, new_byte_count, new_alignment, len_align);105 const new_mem = try self.allocFn(self, new_byte_count, new_alignment, len_align, return_address);
99 // TODO: https://github.com/ziglang/zig/issues/4298106 // TODO: https://github.com/ziglang/zig/issues/4298
100 @memset(new_mem.ptr, undefined, new_byte_count);107 @memset(new_mem.ptr, undefined, new_byte_count);
101 return new_mem;108 return new_mem;
...@@ -103,10 +110,10 @@ fn reallocBytes(...@@ -103,10 +110,10 @@ fn reallocBytes(
103110
104 if (mem.isAligned(@ptrToInt(old_mem.ptr), new_alignment)) {111 if (mem.isAligned(@ptrToInt(old_mem.ptr), new_alignment)) {
105 if (new_byte_count <= old_mem.len) {112 if (new_byte_count <= old_mem.len) {
106 const shrunk_len = self.shrinkBytes(old_mem, old_alignment, new_byte_count, len_align);113 const shrunk_len = self.shrinkBytes(old_mem, old_alignment, new_byte_count, len_align, return_address);
107 return old_mem.ptr[0..shrunk_len];114 return old_mem.ptr[0..shrunk_len];
108 }115 }
109 if (self.resizeFn(self, old_mem, old_alignment, new_byte_count, len_align)) |resized_len| {116 if (self.resizeFn(self, old_mem, old_alignment, new_byte_count, len_align, return_address)) |resized_len| {
110 assert(resized_len >= new_byte_count);117 assert(resized_len >= new_byte_count);
111 // TODO: https://github.com/ziglang/zig/issues/4298118 // TODO: https://github.com/ziglang/zig/issues/4298
112 @memset(old_mem.ptr + new_byte_count, undefined, resized_len - new_byte_count);119 @memset(old_mem.ptr + new_byte_count, undefined, resized_len - new_byte_count);
...@@ -116,7 +123,7 @@ fn reallocBytes(...@@ -116,7 +123,7 @@ fn reallocBytes(
116 if (new_byte_count <= old_mem.len and new_alignment <= old_alignment) {123 if (new_byte_count <= old_mem.len and new_alignment <= old_alignment) {
117 return error.OutOfMemory;124 return error.OutOfMemory;
118 }125 }
119 return self.moveBytes(old_mem, old_alignment, new_byte_count, new_alignment, len_align);126 return self.moveBytes(old_mem, old_alignment, new_byte_count, new_alignment, len_align, return_address);
120}127}
121128
122/// Move the given memory to a new location in the given allocator to accomodate a new129/// Move the given memory to a new location in the given allocator to accomodate a new
...@@ -128,10 +135,11 @@ fn moveBytes(...@@ -128,10 +135,11 @@ fn moveBytes(
128 new_len: usize,135 new_len: usize,
129 new_alignment: u29,136 new_alignment: u29,
130 len_align: u29,137 len_align: u29,
138 return_address: usize,
131) Error![]u8 {139) Error![]u8 {
132 assert(old_mem.len > 0);140 assert(old_mem.len > 0);
133 assert(new_len > 0);141 assert(new_len > 0);
134 const new_mem = try self.allocFn(self, new_len, new_alignment, len_align);142 const new_mem = try self.allocFn(self, new_len, new_alignment, len_align, return_address);
135 @memcpy(new_mem.ptr, old_mem.ptr, math.min(new_len, old_mem.len));143 @memcpy(new_mem.ptr, old_mem.ptr, math.min(new_len, old_mem.len));
136 // TODO DISABLED TO AVOID BUGS IN TRANSLATE C144 // TODO DISABLED TO AVOID BUGS IN TRANSLATE C
137 // TODO see also https://github.com/ziglang/zig/issues/4298145 // TODO see also https://github.com/ziglang/zig/issues/4298
...@@ -139,7 +147,7 @@ fn moveBytes(...@@ -139,7 +147,7 @@ fn moveBytes(
139 // generated C code will be a sequence of 0xaa (the undefined value), meaning147 // generated C code will be a sequence of 0xaa (the undefined value), meaning
140 // it is printing data that has been freed148 // it is printing data that has been freed
141 //@memset(old_mem.ptr, undefined, old_mem.len);149 //@memset(old_mem.ptr, undefined, old_mem.len);
142 _ = self.shrinkBytes(old_mem, old_align, 0, 0);150 _ = self.shrinkBytes(old_mem, old_align, 0, 0, return_address);
143 return new_mem;151 return new_mem;
144}152}
145153
...@@ -147,7 +155,7 @@ fn moveBytes(...@@ -147,7 +155,7 @@ fn moveBytes(
147/// Call `destroy` with the result to free the memory.155/// Call `destroy` with the result to free the memory.
148pub fn create(self: *Allocator, comptime T: type) Error!*T {156pub fn create(self: *Allocator, comptime T: type) Error!*T {
149 if (@sizeOf(T) == 0) return &(T{});157 if (@sizeOf(T) == 0) return &(T{});
150 const slice = try self.alloc(T, 1);158 const slice = try self.allocAdvancedWithRetAddr(T, null, 1, .exact, @returnAddress());
151 return &slice[0];159 return &slice[0];
152}160}
153161
...@@ -158,7 +166,7 @@ pub fn destroy(self: *Allocator, ptr: anytype) void {...@@ -158,7 +166,7 @@ pub fn destroy(self: *Allocator, ptr: anytype) void {
158 if (@sizeOf(T) == 0) return;166 if (@sizeOf(T) == 0) return;
159 const non_const_ptr = @intToPtr([*]u8, @ptrToInt(ptr));167 const non_const_ptr = @intToPtr([*]u8, @ptrToInt(ptr));
160 const ptr_align = @typeInfo(@TypeOf(ptr)).Pointer.alignment;168 const ptr_align = @typeInfo(@TypeOf(ptr)).Pointer.alignment;
161 _ = self.shrinkBytes(non_const_ptr[0..@sizeOf(T)], ptr_align, 0, 0);169 _ = self.shrinkBytes(non_const_ptr[0..@sizeOf(T)], ptr_align, 0, 0, @returnAddress());
162}170}
163171
164/// Allocates an array of `n` items of type `T` and sets all the172/// Allocates an array of `n` items of type `T` and sets all the
...@@ -170,7 +178,7 @@ pub fn destroy(self: *Allocator, ptr: anytype) void {...@@ -170,7 +178,7 @@ pub fn destroy(self: *Allocator, ptr: anytype) void {
170///178///
171/// For allocating a single item, see `create`.179/// For allocating a single item, see `create`.
172pub fn alloc(self: *Allocator, comptime T: type, n: usize) Error![]T {180pub fn alloc(self: *Allocator, comptime T: type, n: usize) Error![]T {
173 return self.alignedAlloc(T, null, n);181 return self.allocAdvancedWithRetAddr(T, null, n, .exact, @returnAddress());
174}182}
175183
176pub fn allocWithOptions(184pub fn allocWithOptions(
...@@ -180,13 +188,25 @@ pub fn allocWithOptions(...@@ -180,13 +188,25 @@ pub fn allocWithOptions(
180 /// null means naturally aligned188 /// null means naturally aligned
181 comptime optional_alignment: ?u29,189 comptime optional_alignment: ?u29,
182 comptime optional_sentinel: ?Elem,190 comptime optional_sentinel: ?Elem,
191) Error!AllocWithOptionsPayload(Elem, optional_alignment, optional_sentinel) {
192 return self.allocWithOptionsRetAddr(Elem, n, optional_alignment, optional_sentinel, @returnAddress());
193}
194
195pub fn allocWithOptionsRetAddr(
196 self: *Allocator,
197 comptime Elem: type,
198 n: usize,
199 /// null means naturally aligned
200 comptime optional_alignment: ?u29,
201 comptime optional_sentinel: ?Elem,
202 return_address: usize,
183) Error!AllocWithOptionsPayload(Elem, optional_alignment, optional_sentinel) {203) Error!AllocWithOptionsPayload(Elem, optional_alignment, optional_sentinel) {
184 if (optional_sentinel) |sentinel| {204 if (optional_sentinel) |sentinel| {
185 const ptr = try self.alignedAlloc(Elem, optional_alignment, n + 1);205 const ptr = try self.allocAdvancedWithRetAddr(Elem, optional_alignment, n + 1, .exact, return_address);
186 ptr[n] = sentinel;206 ptr[n] = sentinel;
187 return ptr[0..n :sentinel];207 return ptr[0..n :sentinel];
188 } else {208 } else {
189 return self.alignedAlloc(Elem, optional_alignment, n);209 return self.allocAdvancedWithRetAddr(Elem, optional_alignment, n, .exact, return_address);
190 }210 }
191}211}
192212
...@@ -208,8 +228,13 @@ fn AllocWithOptionsPayload(comptime Elem: type, comptime alignment: ?u29, compti...@@ -208,8 +228,13 @@ fn AllocWithOptionsPayload(comptime Elem: type, comptime alignment: ?u29, compti
208/// For allocating a single item, see `create`.228/// For allocating a single item, see `create`.
209///229///
210/// Deprecated; use `allocWithOptions`.230/// Deprecated; use `allocWithOptions`.
211pub fn allocSentinel(self: *Allocator, comptime Elem: type, n: usize, comptime sentinel: Elem) Error![:sentinel]Elem {231pub fn allocSentinel(
212 return self.allocWithOptions(Elem, n, null, sentinel);232 self: *Allocator,
233 comptime Elem: type,
234 n: usize,
235 comptime sentinel: Elem,
236) Error![:sentinel]Elem {
237 return self.allocWithOptionsRetAddr(Elem, n, null, sentinel, @returnAddress());
213}238}
214239
215/// Deprecated: use `allocAdvanced`240/// Deprecated: use `allocAdvanced`
...@@ -220,10 +245,9 @@ pub fn alignedAlloc(...@@ -220,10 +245,9 @@ pub fn alignedAlloc(
220 comptime alignment: ?u29,245 comptime alignment: ?u29,
221 n: usize,246 n: usize,
222) Error![]align(alignment orelse @alignOf(T)) T {247) Error![]align(alignment orelse @alignOf(T)) T {
223 return self.allocAdvanced(T, alignment, n, .exact);248 return self.allocAdvancedWithRetAddr(T, alignment, n, .exact, @returnAddress());
224}249}
225250
226const Exact = enum { exact, at_least };
227pub fn allocAdvanced(251pub fn allocAdvanced(
228 self: *Allocator,252 self: *Allocator,
229 comptime T: type,253 comptime T: type,
...@@ -231,9 +255,23 @@ pub fn allocAdvanced(...@@ -231,9 +255,23 @@ pub fn allocAdvanced(
231 comptime alignment: ?u29,255 comptime alignment: ?u29,
232 n: usize,256 n: usize,
233 exact: Exact,257 exact: Exact,
258) Error![]align(alignment orelse @alignOf(T)) T {
259 return self.allocAdvancedWithRetAddr(T, alignment, n, exact, @returnAddress());
260}
261
262pub const Exact = enum { exact, at_least };
263
264pub fn allocAdvancedWithRetAddr(
265 self: *Allocator,
266 comptime T: type,
267 /// null means naturally aligned
268 comptime alignment: ?u29,
269 n: usize,
270 exact: Exact,
271 return_address: usize,
234) Error![]align(alignment orelse @alignOf(T)) T {272) Error![]align(alignment orelse @alignOf(T)) T {
235 const a = if (alignment) |a| blk: {273 const a = if (alignment) |a| blk: {
236 if (a == @alignOf(T)) return allocAdvanced(self, T, null, n, exact);274 if (a == @alignOf(T)) return allocAdvancedWithRetAddr(self, T, null, n, exact, return_address);
237 break :blk a;275 break :blk a;
238 } else @alignOf(T);276 } else @alignOf(T);
239277
...@@ -245,8 +283,12 @@ pub fn allocAdvanced(...@@ -245,8 +283,12 @@ pub fn allocAdvanced(
245 // TODO The `if (alignment == null)` blocks are workarounds for zig not being able to283 // TODO The `if (alignment == null)` blocks are workarounds for zig not being able to
246 // access certain type information about T without creating a circular dependency in async284 // access certain type information about T without creating a circular dependency in async
247 // functions that heap-allocate their own frame with @Frame(func).285 // functions that heap-allocate their own frame with @Frame(func).
248 const sizeOfT = if (alignment == null) @intCast(u29, @divExact(byte_count, n)) else @sizeOf(T);286 const size_of_T = if (alignment == null) @intCast(u29, @divExact(byte_count, n)) else @sizeOf(T);
249 const byte_slice = try self.allocFn(self, byte_count, a, if (exact == .exact) @as(u29, 0) else sizeOfT);287 const len_align: u29 = switch (exact) {
288 .exact => 0,
289 .at_least => size_of_T,
290 };
291 const byte_slice = try self.allocFn(self, byte_count, a, len_align, return_address);
250 switch (exact) {292 switch (exact) {
251 .exact => assert(byte_slice.len == byte_count),293 .exact => assert(byte_slice.len == byte_count),
252 .at_least => assert(byte_slice.len >= byte_count),294 .at_least => assert(byte_slice.len >= byte_count),
...@@ -271,7 +313,7 @@ pub fn resize(self: *Allocator, old_mem: anytype, new_n: usize) Error!@TypeOf(ol...@@ -271,7 +313,7 @@ pub fn resize(self: *Allocator, old_mem: anytype, new_n: usize) Error!@TypeOf(ol
271 }313 }
272 const old_byte_slice = mem.sliceAsBytes(old_mem);314 const old_byte_slice = mem.sliceAsBytes(old_mem);
273 const new_byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;315 const new_byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;
274 const rc = try self.resizeFn(self, old_byte_slice, Slice.alignment, new_byte_count, 0);316 const rc = try self.resizeFn(self, old_byte_slice, Slice.alignment, new_byte_count, 0, @returnAddress());
275 assert(rc == new_byte_count);317 assert(rc == new_byte_count);
276 const new_byte_slice = old_mem.ptr[0..new_byte_count];318 const new_byte_slice = old_mem.ptr[0..new_byte_count];
277 return mem.bytesAsSlice(T, new_byte_slice);319 return mem.bytesAsSlice(T, new_byte_slice);
...@@ -292,7 +334,7 @@ pub fn realloc(self: *Allocator, old_mem: anytype, new_n: usize) t: {...@@ -292,7 +334,7 @@ pub fn realloc(self: *Allocator, old_mem: anytype, new_n: usize) t: {
292 break :t Error![]align(Slice.alignment) Slice.child;334 break :t Error![]align(Slice.alignment) Slice.child;
293} {335} {
294 const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;336 const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
295 return self.reallocAdvanced(old_mem, old_alignment, new_n, .exact);337 return self.reallocAdvancedWithRetAddr(old_mem, old_alignment, new_n, .exact, @returnAddress());
296}338}
297339
298pub fn reallocAtLeast(self: *Allocator, old_mem: anytype, new_n: usize) t: {340pub fn reallocAtLeast(self: *Allocator, old_mem: anytype, new_n: usize) t: {
...@@ -300,28 +342,29 @@ pub fn reallocAtLeast(self: *Allocator, old_mem: anytype, new_n: usize) t: {...@@ -300,28 +342,29 @@ pub fn reallocAtLeast(self: *Allocator, old_mem: anytype, new_n: usize) t: {
300 break :t Error![]align(Slice.alignment) Slice.child;342 break :t Error![]align(Slice.alignment) Slice.child;
301} {343} {
302 const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;344 const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
303 return self.reallocAdvanced(old_mem, old_alignment, new_n, .at_least);345 return self.reallocAdvancedWithRetAddr(old_mem, old_alignment, new_n, .at_least, @returnAddress());
304}346}
305347
306// Deprecated: use `reallocAdvanced`348/// This is the same as `realloc`, except caller may additionally request
307pub fn alignedRealloc(349/// a new alignment, which can be larger, smaller, or the same as the old
350/// allocation.
351pub fn reallocAdvanced(
308 self: *Allocator,352 self: *Allocator,
309 old_mem: anytype,353 old_mem: anytype,
310 comptime new_alignment: u29,354 comptime new_alignment: u29,
311 new_n: usize,355 new_n: usize,
356 exact: Exact,
312) Error![]align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {357) Error![]align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
313 return self.reallocAdvanced(old_mem, new_alignment, new_n, .exact);358 return self.reallocAdvancedWithRetAddr(old_mem, new_alignment, new_n, exact, @returnAddress());
314}359}
315360
316/// This is the same as `realloc`, except caller may additionally request361pub fn reallocAdvancedWithRetAddr(
317/// a new alignment, which can be larger, smaller, or the same as the old
318/// allocation.
319pub fn reallocAdvanced(
320 self: *Allocator,362 self: *Allocator,
321 old_mem: anytype,363 old_mem: anytype,
322 comptime new_alignment: u29,364 comptime new_alignment: u29,
323 new_n: usize,365 new_n: usize,
324 exact: Exact,366 exact: Exact,
367 return_address: usize,
325) Error![]align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {368) Error![]align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
326 const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;369 const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
327 const T = Slice.child;370 const T = Slice.child;
...@@ -336,7 +379,11 @@ pub fn reallocAdvanced(...@@ -336,7 +379,11 @@ pub fn reallocAdvanced(
336 const old_byte_slice = mem.sliceAsBytes(old_mem);379 const old_byte_slice = mem.sliceAsBytes(old_mem);
337 const byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;380 const byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;
338 // Note: can't set shrunk memory to undefined as memory shouldn't be modified on realloc failure381 // Note: can't set shrunk memory to undefined as memory shouldn't be modified on realloc failure
339 const new_byte_slice = try self.reallocBytes(old_byte_slice, Slice.alignment, byte_count, new_alignment, if (exact == .exact) @as(u29, 0) else @sizeOf(T));382 const len_align: u29 = switch (exact) {
383 .exact => 0,
384 .at_least => @sizeOf(T),
385 };
386 const new_byte_slice = try self.reallocBytes(old_byte_slice, Slice.alignment, byte_count, new_alignment, len_align, return_address);
340 return mem.bytesAsSlice(T, @alignCast(new_alignment, new_byte_slice));387 return mem.bytesAsSlice(T, @alignCast(new_alignment, new_byte_slice));
341}388}
342389
...@@ -350,7 +397,7 @@ pub fn shrink(self: *Allocator, old_mem: anytype, new_n: usize) t: {...@@ -350,7 +397,7 @@ pub fn shrink(self: *Allocator, old_mem: anytype, new_n: usize) t: {
350 break :t []align(Slice.alignment) Slice.child;397 break :t []align(Slice.alignment) Slice.child;
351} {398} {
352 const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;399 const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
353 return self.alignedShrink(old_mem, old_alignment, new_n);400 return self.alignedShrinkWithRetAddr(old_mem, old_alignment, new_n, @returnAddress());
354}401}
355402
356/// This is the same as `shrink`, except caller may additionally request403/// This is the same as `shrink`, except caller may additionally request
...@@ -361,6 +408,19 @@ pub fn alignedShrink(...@@ -361,6 +408,19 @@ pub fn alignedShrink(
361 old_mem: anytype,408 old_mem: anytype,
362 comptime new_alignment: u29,409 comptime new_alignment: u29,
363 new_n: usize,410 new_n: usize,
411) []align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
412 return self.alignedShrinkWithRetAddr(old_mem, new_alignment, new_n, @returnAddress());
413}
414
415/// This is the same as `alignedShrink`, except caller may additionally pass
416/// the return address of the first stack frame, which may be relevant for
417/// allocators which collect stack traces.
418pub fn alignedShrinkWithRetAddr(
419 self: *Allocator,
420 old_mem: anytype,
421 comptime new_alignment: u29,
422 new_n: usize,
423 return_address: usize,
364) []align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {424) []align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
365 const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;425 const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
366 const T = Slice.child;426 const T = Slice.child;
...@@ -377,7 +437,7 @@ pub fn alignedShrink(...@@ -377,7 +437,7 @@ pub fn alignedShrink(
377 const old_byte_slice = mem.sliceAsBytes(old_mem);437 const old_byte_slice = mem.sliceAsBytes(old_mem);
378 // TODO: https://github.com/ziglang/zig/issues/4298438 // TODO: https://github.com/ziglang/zig/issues/4298
379 @memset(old_byte_slice.ptr + byte_count, undefined, old_byte_slice.len - byte_count);439 @memset(old_byte_slice.ptr + byte_count, undefined, old_byte_slice.len - byte_count);
380 _ = self.shrinkBytes(old_byte_slice, Slice.alignment, byte_count, 0);440 _ = self.shrinkBytes(old_byte_slice, Slice.alignment, byte_count, 0, return_address);
381 return old_mem[0..new_n];441 return old_mem[0..new_n];
382}442}
383443
...@@ -391,7 +451,7 @@ pub fn free(self: *Allocator, memory: anytype) void {...@@ -391,7 +451,7 @@ pub fn free(self: *Allocator, memory: anytype) void {
391 const non_const_ptr = @intToPtr([*]u8, @ptrToInt(bytes.ptr));451 const non_const_ptr = @intToPtr([*]u8, @ptrToInt(bytes.ptr));
392 // TODO: https://github.com/ziglang/zig/issues/4298452 // TODO: https://github.com/ziglang/zig/issues/4298
393 @memset(non_const_ptr, undefined, bytes_len);453 @memset(non_const_ptr, undefined, bytes_len);
394 _ = self.shrinkBytes(non_const_ptr[0..bytes_len], Slice.alignment, 0, 0);454 _ = self.shrinkBytes(non_const_ptr[0..bytes_len], Slice.alignment, 0, 0, @returnAddress());
395}455}
396456
397/// Copies `m` to newly allocated memory. Caller owns the memory.457/// Copies `m` to newly allocated memory. Caller owns the memory.
...@@ -408,3 +468,19 @@ pub fn dupeZ(allocator: *Allocator, comptime T: type, m: []const T) ![:0]T {...@@ -408,3 +468,19 @@ pub fn dupeZ(allocator: *Allocator, comptime T: type, m: []const T) ![:0]T {
408 new_buf[m.len] = 0;468 new_buf[m.len] = 0;
409 return new_buf[0..m.len :0];469 return new_buf[0..m.len :0];
410}470}
471
472/// Call `resizeFn`, but caller guarantees that `new_len` <= `buf.len` meaning
473/// error.OutOfMemory should be impossible.
474/// This function allows a runtime `buf_align` value. Callers should generally prefer
475/// to call `shrink` directly.
476pub fn shrinkBytes(
477 self: *Allocator,
478 buf: []u8,
479 buf_align: u29,
480 new_len: usize,
481 len_align: u29,
482 return_address: usize,
483) usize {
484 assert(new_len <= buf.len);
485 return self.resizeFn(self, buf, buf_align, new_len, len_align, return_address) catch unreachable;
486}
lib/std/testing/failing_allocator.zig+10-3
...@@ -45,12 +45,18 @@ pub const FailingAllocator = struct {...@@ -45,12 +45,18 @@ pub const FailingAllocator = struct {
45 };45 };
46 }46 }
4747
48 fn alloc(allocator: *std.mem.Allocator, len: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {48 fn alloc(
49 allocator: *std.mem.Allocator,
50 len: usize,
51 ptr_align: u29,
52 len_align: u29,
53 return_address: usize,
54 ) error{OutOfMemory}![]u8 {
49 const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);55 const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);
50 if (self.index == self.fail_index) {56 if (self.index == self.fail_index) {
51 return error.OutOfMemory;57 return error.OutOfMemory;
52 }58 }
53 const result = try self.internal_allocator.allocFn(self.internal_allocator, len, ptr_align, len_align);59 const result = try self.internal_allocator.allocFn(self.internal_allocator, len, ptr_align, len_align, return_address);
54 self.allocated_bytes += result.len;60 self.allocated_bytes += result.len;
55 self.allocations += 1;61 self.allocations += 1;
56 self.index += 1;62 self.index += 1;
...@@ -63,9 +69,10 @@ pub const FailingAllocator = struct {...@@ -63,9 +69,10 @@ pub const FailingAllocator = struct {
63 old_align: u29,69 old_align: u29,
64 new_len: usize,70 new_len: usize,
65 len_align: u29,71 len_align: u29,
72 ra: usize,
66 ) error{OutOfMemory}!usize {73 ) error{OutOfMemory}!usize {
67 const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);74 const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);
68 const r = self.internal_allocator.resizeFn(self.internal_allocator, old_mem, old_align, new_len, len_align) catch |e| {75 const r = self.internal_allocator.resizeFn(self.internal_allocator, old_mem, old_align, new_len, len_align, ra) catch |e| {
69 std.debug.assert(new_len > old_mem.len);76 std.debug.assert(new_len > old_mem.len);
70 return e;77 return e;
71 };78 };