Commit 069a6f2432be5fd5e9a3eabd13faa43143001bbd | zig

author	Andrew Kelley <andrew@ziglang.org> 2020-08-08 20:20:15-04:00
committer	GitHub <noreply@github.com> 2020-08-08 20:20:15-04:00
log	069a6f2432be5fd5e9a3eabd13faa43143001bbd
tree	f68e0ce1a1a7e62665be0db0fc701f79c25189aa
parent	ab483281d31c852e77fe0afc6a623fedd1574546
parent	f98cffc615bb7ea73eee3bd24ae8a957fe8ebb82
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #5998 from ziglang/general-purpose-allocator

std: introduce GeneralPurposeAllocator

23 files changed, 1791 insertions(+), 691 deletions(-)

doc
- langref.html.in
lib
- std
  - array_list.zig
  - atomic
    queue.zig
  - debug.zig
  - heap.zig
  - heap
    arena_allocator.zig
    general_purpose_allocator.zig
    logging_allocator.zig
  - math.zig
  - mem.zig
  - mem
    Allocator.zig
  - mutex.zig
  - once.zig
  - special
    test_runner.zig
  - std.zig
  - testing.zig
  - testing
    failing_allocator.zig
    leak_count_allocator.zig
src-self-hosted
src
- codegen.cpp
- ir.cpp

doc/langref.html.in+11-3

...	...	@@ -9357,9 +9357,17 @@ pub fn main() !void {
9357	9357	is handled correctly? In this case, use {#syntax#}std.testing.FailingAllocator{#endsyntax#}.
9358	9358	</li>
9359	9359	<li>
9360		Finally, if none of the above apply, you need a general purpose allocator. Zig does not
9361		yet have a general purpose allocator in the standard library,
9362		<a href="https://github.com/andrewrk/zig-general-purpose-allocator/">but one is being actively developed</a>.
	9360	Are you writing a test? In this case, use {#syntax#}std.testing.allocator{#endsyntax#}.
	9361	</li>
	9362	<li>
	9363	Finally, if none of the above apply, you need a general purpose allocator.
	9364	Zig's general purpose allocator is available as a function that takes a {#link\|comptime#}
	9365	{#link\|struct#} of configuration options and returns a type.
	9366	Generally, you will set up one {#syntax#}std.heap.GeneralPurposeAllocator{#endsyntax#} in
	9367	your main function, and then pass it or sub-allocators around to various parts of your
	9368	application.
	9369	</li>
	9370	<li>
9363	9371	You can also consider {#link\|Implementing an Allocator#}.
9364	9372	</li>
9365	9373	</ol>

lib/std/array_list.zig+1

...	...	@@ -263,6 +263,7 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
263	263	if (better_capacity >= new_capacity) break;
264	264	}
265	265
	266	// TODO This can be optimized to avoid needlessly copying undefined memory.
266	267	const new_memory = try self.allocator.reallocAtLeast(self.allocatedSlice(), better_capacity);
267	268	self.items.ptr = new_memory.ptr;
268	269	self.capacity = new_memory.len;

lib/std/atomic/queue.zig+1-1

...	...	@@ -22,7 +22,7 @@ pub fn Queue(comptime T: type) type {
22	22	return Self{
23	23	.head = null,
24	24	.tail = null,
25		.mutex = std.Mutex.init(),
	25	.mutex = std.Mutex{},
26	26	};
27	27	}
28	28

lib/std/debug.zig+2-5

...	...	@@ -19,9 +19,6 @@ const windows = std.os.windows;
19	19
20	20	pub const leb = @import("debug/leb128.zig");
21	21
22		pub const global_allocator = @compileError("Please switch to std.testing.allocator.");
23		pub const failing_allocator = @compileError("Please switch to std.testing.failing_allocator.");
24
25	22	pub const runtime_safety = switch (builtin.mode) {
26	23	.Debug, .ReleaseSafe => true,
27	24	.ReleaseFast, .ReleaseSmall => false,
...	...	@@ -50,7 +47,7 @@ pub const LineInfo = struct {
50	47	}
51	48	};
52	49
53		var stderr_mutex = std.Mutex.init();
	50	var stderr_mutex = std.Mutex{};
54	51
55	52	/// Deprecated. Use `std.log` functions for logging or `std.debug.print` for
56	53	/// "printf debugging".
...	...	@@ -235,7 +232,7 @@ pub fn panic(comptime format: []const u8, args: anytype) noreturn {
235	232	var panicking: u8 = 0;
236	233
237	234	// Locked to avoid interleaving panic messages from multiple threads.
238		var panic_mutex = std.Mutex.init();
	235	var panic_mutex = std.Mutex{};
239	236
240	237	/// Counts how many times the panic handler is invoked by this thread.
241	238	/// This is used to catch and handle panics triggered by the panic handler.

lib/std/heap.zig+103-37

...	...	@@ -12,6 +12,7 @@ const maxInt = std.math.maxInt;
12	12	pub const LoggingAllocator = @import("heap/logging_allocator.zig").LoggingAllocator;
13	13	pub const loggingAllocator = @import("heap/logging_allocator.zig").loggingAllocator;
14	14	pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator;
	15	pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig").GeneralPurposeAllocator;
15	16
16	17	const Allocator = mem.Allocator;
17	18
...	...	@@ -36,7 +37,7 @@ var c_allocator_state = Allocator{
36	37	.resizeFn = cResize,
37	38	};
38	39
39		fn cAlloc(self: *Allocator, len: usize, ptr_align: u29, len_align: u29) Allocator.Error![]u8 {
	40	fn cAlloc(self: *Allocator, len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Allocator.Error![]u8 {
40	41	assert(ptr_align <= @alignOf(c_longdouble));
41	42	const ptr = @ptrCast([*]u8, c.malloc(len) orelse return error.OutOfMemory);
42	43	if (len_align == 0) {
...	...	@@ -53,7 +54,14 @@ fn cAlloc(self: *Allocator, len: usize, ptr_align: u29, len_align: u29) Allocato
53	54	return ptr[0..mem.alignBackwardAnyAlign(full_len, len_align)];
54	55	}
55	56
56		fn cResize(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) Allocator.Error!usize {
	57	fn cResize(
	58	self: *Allocator,
	59	buf: []u8,
	60	old_align: u29,
	61	new_len: usize,
	62	len_align: u29,
	63	ret_addr: usize,
	64	) Allocator.Error!usize {
57	65	if (new_len == 0) {
58	66	c.free(buf.ptr);
59	67	return 0;
...	...	@@ -88,8 +96,6 @@ var wasm_page_allocator_state = Allocator{
88	96	.resizeFn = WasmPageAllocator.resize,
89	97	};
90	98
91		pub const direct_allocator = @compileError("deprecated; use std.heap.page_allocator");
92
93	99	/// Verifies that the adjusted length will still map to the full length
94	100	pub fn alignPageAllocLen(full_len: usize, len: usize, len_align: u29) usize {
95	101	const aligned_len = mem.alignAllocLen(full_len, len, len_align);
...	...	@@ -97,10 +103,13 @@ pub fn alignPageAllocLen(full_len: usize, len: usize, len_align: u29) usize {
97	103	return aligned_len;
98	104	}
99	105
	106	/// TODO Utilize this on Windows.
	107	pub var next_mmap_addr_hint: ?[*]align(mem.page_size) u8 = null;
	108
100	109	const PageAllocator = struct {
101		fn alloc(allocator: *Allocator, n: usize, alignment: u29, len_align: u29) error{OutOfMemory}![]u8 {
	110	fn alloc(allocator: *Allocator, n: usize, alignment: u29, len_align: u29, ra: usize) error{OutOfMemory}![]u8 {
102	111	assert(n > 0);
103		const alignedLen = mem.alignForward(n, mem.page_size);
	112	const aligned_len = mem.alignForward(n, mem.page_size);
104	113
105	114	if (builtin.os.tag == .windows) {
106	115	const w = os.windows;
...	...	@@ -112,14 +121,14 @@ const PageAllocator = struct {
112	121	// see https://devblogs.microsoft.com/oldnewthing/?p=42223
113	122	const addr = w.VirtualAlloc(
114	123	null,
115		alignedLen,
	124	aligned_len,
116	125	w.MEM_COMMIT \| w.MEM_RESERVE,
117	126	w.PAGE_READWRITE,
118	127	) catch return error.OutOfMemory;
119	128
120	129	// If the allocation is sufficiently aligned, use it.
121	130	if (@ptrToInt(addr) & (alignment - 1) == 0) {
122		return @ptrCast([*]u8, addr)[0..alignPageAllocLen(alignedLen, n, len_align)];
	131	return @ptrCast([*]u8, addr)[0..alignPageAllocLen(aligned_len, n, len_align)];
123	132	}
124	133
125	134	// If it wasn't, actually do an explicitely aligned allocation.
...	...	@@ -146,20 +155,24 @@ const PageAllocator = struct {
146	155	// until it succeeds.
147	156	const ptr = w.VirtualAlloc(
148	157	@intToPtr(*c_void, aligned_addr),
149		alignedLen,
	158	aligned_len,
150	159	w.MEM_COMMIT \| w.MEM_RESERVE,
151	160	w.PAGE_READWRITE,
152	161	) catch continue;
153	162
154		return @ptrCast([*]u8, ptr)[0..alignPageAllocLen(alignedLen, n, len_align)];
	163	return @ptrCast([*]u8, ptr)[0..alignPageAllocLen(aligned_len, n, len_align)];
155	164	}
156	165	}
157	166
158		const maxDropLen = alignment - std.math.min(alignment, mem.page_size);
159		const allocLen = if (maxDropLen <= alignedLen - n) alignedLen else mem.alignForward(alignedLen + maxDropLen, mem.page_size);
	167	const max_drop_len = alignment - std.math.min(alignment, mem.page_size);
	168	const alloc_len = if (max_drop_len <= aligned_len - n)
	169	aligned_len
	170	else
	171	mem.alignForward(aligned_len + max_drop_len, mem.page_size);
	172	const hint = @atomicLoad(@TypeOf(next_mmap_addr_hint), &next_mmap_addr_hint, .Unordered);
160	173	const slice = os.mmap(
161		null,
162		allocLen,
	174	hint,
	175	alloc_len,
163	176	os.PROT_READ \| os.PROT_WRITE,
164	177	os.MAP_PRIVATE \| os.MAP_ANONYMOUS,
165	178	-1,
...	...	@@ -168,25 +181,36 @@ const PageAllocator = struct {
168	181	assert(mem.isAligned(@ptrToInt(slice.ptr), mem.page_size));
169	182
170	183	const aligned_addr = mem.alignForward(@ptrToInt(slice.ptr), alignment);
	184	const result_ptr = @alignCast(mem.page_size, @intToPtr([*]u8, aligned_addr));
171	185
172	186	// Unmap the extra bytes that were only requested in order to guarantee
173	187	// that the range of memory we were provided had a proper alignment in
174	188	// it somewhere. The extra bytes could be at the beginning, or end, or both.
175		const dropLen = aligned_addr - @ptrToInt(slice.ptr);
176		if (dropLen != 0) {
177		os.munmap(slice[0..dropLen]);
	189	const drop_len = aligned_addr - @ptrToInt(slice.ptr);
	190	if (drop_len != 0) {
	191	os.munmap(slice[0..drop_len]);
178	192	}
179	193
180	194	// Unmap extra pages
181		const alignedBufferLen = allocLen - dropLen;
182		if (alignedBufferLen > alignedLen) {
183		os.munmap(@alignCast(mem.page_size, @intToPtr([*]u8, aligned_addr))[alignedLen..alignedBufferLen]);
	195	const aligned_buffer_len = alloc_len - drop_len;
	196	if (aligned_buffer_len > aligned_len) {
	197	os.munmap(result_ptr[aligned_len..aligned_buffer_len]);
184	198	}
185	199
186		return @intToPtr([*]u8, aligned_addr)[0..alignPageAllocLen(alignedLen, n, len_align)];
	200	const new_hint = @alignCast(mem.page_size, result_ptr + aligned_len);
	201	_ = @cmpxchgStrong(@TypeOf(next_mmap_addr_hint), &next_mmap_addr_hint, hint, new_hint, .Monotonic, .Monotonic);
	202
	203	return result_ptr[0..alignPageAllocLen(aligned_len, n, len_align)];
187	204	}
188	205
189		fn resize(allocator: *Allocator, buf_unaligned: []u8, new_size: usize, len_align: u29) Allocator.Error!usize {
	206	fn resize(
	207	allocator: *Allocator,
	208	buf_unaligned: []u8,
	209	buf_align: u29,
	210	new_size: usize,
	211	len_align: u29,
	212	return_address: usize,
	213	) Allocator.Error!usize {
190	214	const new_size_aligned = mem.alignForward(new_size, mem.page_size);
191	215
192	216	if (builtin.os.tag == .windows) {
...	...	@@ -201,7 +225,7 @@ const PageAllocator = struct {
201	225	w.VirtualFree(buf_unaligned.ptr, 0, w.MEM_RELEASE);
202	226	return 0;
203	227	}
204		if (new_size < buf_unaligned.len) {
	228	if (new_size <= buf_unaligned.len) {
205	229	const base_addr = @ptrToInt(buf_unaligned.ptr);
206	230	const old_addr_end = base_addr + buf_unaligned.len;
207	231	const new_addr_end = mem.alignForward(base_addr + new_size, mem.page_size);
...	...	@@ -216,10 +240,10 @@ const PageAllocator = struct {
216	240	}
217	241	return alignPageAllocLen(new_size_aligned, new_size, len_align);
218	242	}
219		if (new_size == buf_unaligned.len) {
	243	const old_size_aligned = mem.alignForward(buf_unaligned.len, mem.page_size);
	244	if (new_size_aligned <= old_size_aligned) {
220	245	return alignPageAllocLen(new_size_aligned, new_size, len_align);
221	246	}
222		// new_size > buf_unaligned.len not implemented
223	247	return error.OutOfMemory;
224	248	}
225	249
...	...	@@ -229,6 +253,7 @@ const PageAllocator = struct {
229	253
230	254	if (new_size_aligned < buf_aligned_len) {
231	255	const ptr = @intToPtr([*]align(mem.page_size) u8, @ptrToInt(buf_unaligned.ptr) + new_size_aligned);
	256	// TODO: if the next_mmap_addr_hint is within the unmapped range, update it
232	257	os.munmap(ptr[0 .. buf_aligned_len - new_size_aligned]);
233	258	if (new_size_aligned == 0)
234	259	return 0;
...	...	@@ -236,6 +261,7 @@ const PageAllocator = struct {
236	261	}
237	262
238	263	// TODO: call mremap
	264	// TODO: if the next_mmap_addr_hint is within the remapped range, update it
239	265	return error.OutOfMemory;
240	266	}
241	267	};
...	...	@@ -332,7 +358,7 @@ const WasmPageAllocator = struct {
332	358	return mem.alignForward(memsize, mem.page_size) / mem.page_size;
333	359	}
334	360
335		fn alloc(allocator: *Allocator, len: usize, alignment: u29, len_align: u29) error{OutOfMemory}![]u8 {
	361	fn alloc(allocator: *Allocator, len: usize, alignment: u29, len_align: u29, ra: usize) error{OutOfMemory}![]u8 {
336	362	const page_count = nPages(len);
337	363	const page_idx = try allocPages(page_count, alignment);
338	364	return @intToPtr([]u8, page_idx mem.page_size)[0..alignPageAllocLen(page_count * mem.page_size, len, len_align)];
...	...	@@ -385,7 +411,14 @@ const WasmPageAllocator = struct {
385	411	}
386	412	}
387	413
388		fn resize(allocator: *Allocator, buf: []u8, new_len: usize, len_align: u29) error{OutOfMemory}!usize {
	414	fn resize(
	415	allocator: *Allocator,
	416	buf: []u8,
	417	buf_align: u29,
	418	new_len: usize,
	419	len_align: u29,
	420	return_address: usize,
	421	) error{OutOfMemory}!usize {
389	422	const aligned_len = mem.alignForward(buf.len, mem.page_size);
390	423	if (new_len > aligned_len) return error.OutOfMemory;
391	424	const current_n = nPages(aligned_len);
...	...	@@ -425,7 +458,13 @@ pub const HeapAllocator = switch (builtin.os.tag) {
425	458	return @intToPtr(*align(1) usize, @ptrToInt(buf.ptr) + buf.len);
426	459	}
427	460
428		fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {
	461	fn alloc(
	462	allocator: *Allocator,
	463	n: usize,
	464	ptr_align: u29,
	465	len_align: u29,
	466	return_address: usize,
	467	) error{OutOfMemory}![]u8 {
429	468	const self = @fieldParentPtr(HeapAllocator, "allocator", allocator);
430	469
431	470	const amt = n + ptr_align - 1 + @sizeOf(usize);
...	...	@@ -452,7 +491,14 @@ pub const HeapAllocator = switch (builtin.os.tag) {
452	491	return buf;
453	492	}
454	493
455		fn resize(allocator: *Allocator, buf: []u8, new_size: usize, len_align: u29) error{OutOfMemory}!usize {
	494	fn resize(
	495	allocator: *Allocator,
	496	buf: []u8,
	497	buf_align: u29,
	498	new_size: usize,
	499	len_align: u29,
	500	return_address: usize,
	501	) error{OutOfMemory}!usize {
456	502	const self = @fieldParentPtr(HeapAllocator, "allocator", allocator);
457	503	if (new_size == 0) {
458	504	os.windows.HeapFree(self.heap_handle.?, 0, @intToPtr(c_void, getRecordPtr(buf).));
...	...	@@ -524,7 +570,7 @@ pub const FixedBufferAllocator = struct {
524	570	return buf.ptr + buf.len == self.buffer.ptr + self.end_index;
525	571	}
526	572
527		fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) ![]u8 {
	573	fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29, ra: usize) ![]u8 {
528	574	const self = @fieldParentPtr(FixedBufferAllocator, "allocator", allocator);
529	575	const aligned_addr = mem.alignForward(@ptrToInt(self.buffer.ptr) + self.end_index, ptr_align);
530	576	const adjusted_index = aligned_addr - @ptrToInt(self.buffer.ptr);
...	...	@@ -538,7 +584,14 @@ pub const FixedBufferAllocator = struct {
538	584	return result;
539	585	}
540	586
541		fn resize(allocator: *Allocator, buf: []u8, new_size: usize, len_align: u29) Allocator.Error!usize {
	587	fn resize(
	588	allocator: *Allocator,
	589	buf: []u8,
	590	buf_align: u29,
	591	new_size: usize,
	592	len_align: u29,
	593	return_address: usize,
	594	) Allocator.Error!usize {
542	595	const self = @fieldParentPtr(FixedBufferAllocator, "allocator", allocator);
543	596	assert(self.ownsSlice(buf)); // sanity check
544	597
...	...	@@ -588,7 +641,7 @@ pub const ThreadSafeFixedBufferAllocator = blk: {
588	641	};
589	642	}
590	643
591		fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) ![]u8 {
	644	fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29, ra: usize) ![]u8 {
592	645	const self = @fieldParentPtr(ThreadSafeFixedBufferAllocator, "allocator", allocator);
593	646	var end_index = @atomicLoad(usize, &self.end_index, builtin.AtomicOrder.SeqCst);
594	647	while (true) {
...	...	@@ -636,18 +689,31 @@ pub fn StackFallbackAllocator(comptime size: usize) type {
636	689	return &self.allocator;
637	690	}
638	691
639		fn alloc(allocator: Allocator, len: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {
	692	fn alloc(
	693	allocator: *Allocator,
	694	len: usize,
	695	ptr_align: u29,
	696	len_align: u29,
	697	return_address: usize,
	698	) error{OutOfMemory}![*]u8 {
640	699	const self = @fieldParentPtr(Self, "allocator", allocator);
641	700	return FixedBufferAllocator.alloc(&self.fixed_buffer_allocator, len, ptr_align) catch
642	701	return fallback_allocator.alloc(len, ptr_align);
643	702	}
644	703
645		fn resize(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) error{OutOfMemory}!void {
	704	fn resize(
	705	self: *Allocator,
	706	buf: []u8,
	707	buf_align: u29,
	708	new_len: usize,
	709	len_align: u29,
	710	return_address: usize,
	711	) error{OutOfMemory}!void {
646	712	const self = @fieldParentPtr(Self, "allocator", allocator);
647	713	if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) {
648		try self.fixed_buffer_allocator.callResizeFn(buf, new_len);
	714	try self.fixed_buffer_allocator.resize(buf, new_len);
649	715	} else {
650		try self.fallback_allocator.callResizeFn(buf, new_len);
	716	try self.fallback_allocator.resize(buf, new_len);
651	717	}
652	718	}
653	719	};
...	...	@@ -932,7 +998,7 @@ pub fn testAllocatorAlignedShrink(base_allocator: *mem.Allocator) mem.Allocator.
932	998	slice[60] = 0x34;
933	999
934	1000	// realloc to a smaller size but with a larger alignment
935		slice = try allocator.alignedRealloc(slice, mem.page_size * 32, alloc_size / 2);
	1001	slice = try allocator.reallocAdvanced(slice, mem.page_size * 32, alloc_size / 2, .exact);
936	1002	testing.expect(slice[0] == 0x12);
937	1003	testing.expect(slice[60] == 0x34);
938	1004	}

lib/std/heap/arena_allocator.zig+2-2

...	...	@@ -49,7 +49,7 @@ pub const ArenaAllocator = struct {
49	49	const actual_min_size = minimum_size + (@sizeOf(BufNode) + 16);
50	50	const big_enough_len = prev_len + actual_min_size;
51	51	const len = big_enough_len + big_enough_len / 2;
52		const buf = try self.child_allocator.callAllocFn(len, @alignOf(BufNode), 1);
	52	const buf = try self.child_allocator.allocFn(self.child_allocator, len, @alignOf(BufNode), 1, @returnAddress());
53	53	const buf_node = @ptrCast(*BufNode, @alignCast(@alignOf(BufNode), buf.ptr));
54	54	buf_node.* = BufNode{
55	55	.data = buf,
...	...	@@ -60,7 +60,7 @@ pub const ArenaAllocator = struct {
60	60	return buf_node;
61	61	}
62	62
63		fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) ![]u8 {
	63	fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29, ra: usize) ![]u8 {
64	64	const self = @fieldParentPtr(ArenaAllocator, "allocator", allocator);
65	65
66	66	var cur_node = if (self.state.buffer_list.first) \|first_node\| first_node else try self.createNode(0, n + ptr_align);

lib/std/heap/general_purpose_allocator.zig created+921

...	...	@@ -0,0 +1,921 @@
	1	//! # General Purpose Allocator
	2	//!
	3	//! ## Design Priorities
	4	//!
	5	//! ### `OptimizationMode.debug` and `OptimizationMode.release_safe`:
	6	//!
	7	//! * Detect double free, and print stack trace of:
	8	//! - Where it was first allocated
	9	//! - Where it was freed the first time
	10	//! - Where it was freed the second time
	11	//!
	12	//! * Detect leaks and print stack trace of:
	13	//! - Where it was allocated
	14	//!
	15	//! * When a page of memory is no longer needed, give it back to resident memory
	16	//! as soon as possible, so that it causes page faults when used.
	17	//!
	18	//! * Do not re-use memory slots, so that memory safety is upheld. For small
	19	//! allocations, this is handled here; for larger ones it is handled in the
	20	//! backing allocator (by default `std.heap.page_allocator`).
	21	//!
	22	//! * Make pointer math errors unlikely to harm memory from
	23	//! unrelated allocations.
	24	//!
	25	//! * It's OK for these mechanisms to cost some extra overhead bytes.
	26	//!
	27	//! * It's OK for performance cost for these mechanisms.
	28	//!
	29	//! * Rogue memory writes should not harm the allocator's state.
	30	//!
	31	//! * Cross platform. Operates based on a backing allocator which makes it work
	32	//! everywhere, even freestanding.
	33	//!
	34	//! * Compile-time configuration.
	35	//!
	36	//! ### `OptimizationMode.release_fast` (note: not much work has gone into this use case yet):
	37	//!
	38	//! * Low fragmentation is primary concern
	39	//! * Performance of worst-case latency is secondary concern
	40	//! * Performance of average-case latency is next
	41	//! * Finally, having freed memory unmapped, and pointer math errors unlikely to
	42	//! harm memory from unrelated allocations are nice-to-haves.
	43	//!
	44	//! ### `OptimizationMode.release_small` (note: not much work has gone into this use case yet):
	45	//!
	46	//! * Small binary code size of the executable is the primary concern.
	47	//! * Next, defer to the `.release_fast` priority list.
	48	//!
	49	//! ## Basic Design:
	50	//!
	51	//! Small allocations are divided into buckets:
	52	//!
	53	//! ```
	54	//! index obj_size
	55	//! 0 1
	56	//! 1 2
	57	//! 2 4
	58	//! 3 8
	59	//! 4 16
	60	//! 5 32
	61	//! 6 64
	62	//! 7 128
	63	//! 8 256
	64	//! 9 512
	65	//! 10 1024
	66	//! 11 2048
	67	//! ```
	68	//!
	69	//! The main allocator state has an array of all the "current" buckets for each
	70	//! size class. Each slot in the array can be null, meaning the bucket for that
	71	//! size class is not allocated. When the first object is allocated for a given
	72	//! size class, it allocates 1 page of memory from the OS. This page is
	73	//! divided into "slots" - one per allocated object. Along with the page of memory
	74	//! for object slots, as many pages as necessary are allocated to store the
	75	//! BucketHeader, followed by "used bits", and two stack traces for each slot
	76	//! (allocation trace and free trace).
	77	//!
	78	//! The "used bits" are 1 bit per slot representing whether the slot is used.
	79	//! Allocations use the data to iterate to find a free slot. Frees assert that the
	80	//! corresponding bit is 1 and set it to 0.
	81	//!
	82	//! Buckets have prev and next pointers. When there is only one bucket for a given
	83	//! size class, both prev and next point to itself. When all slots of a bucket are
	84	//! used, a new bucket is allocated, and enters the doubly linked list. The main
	85	//! allocator state tracks the "current" bucket for each size class. Leak detection
	86	//! currently only checks the current bucket.
	87	//!
	88	//! Resizing detects if the size class is unchanged or smaller, in which case the same
	89	//! pointer is returned unmodified. If a larger size class is required,
	90	//! `error.OutOfMemory` is returned.
	91	//!
	92	//! Large objects are allocated directly using the backing allocator and their metadata is stored
	93	//! in a `std.HashMap` using the backing allocator.
	94
	95	const std = @import("std");
	96	const math = std.math;
	97	const assert = std.debug.assert;
	98	const mem = std.mem;
	99	const Allocator = std.mem.Allocator;
	100	const page_size = std.mem.page_size;
	101	const StackTrace = std.builtin.StackTrace;
	102
	103	/// Integer type for pointing to slots in a small allocation
	104	const SlotIndex = std.meta.Int(false, math.log2(page_size) + 1);
	105
	106	const sys_can_stack_trace = switch (std.Target.current.cpu.arch) {
	107	// Observed to go into an infinite loop.
	108	// TODO: Make this work.
	109	.mips,
	110	.mipsel,
	111	=> false,
	112
	113	// `@returnAddress()` in LLVM 10 gives
	114	// "Non-Emscripten WebAssembly hasn't implemented __builtin_return_address".
	115	.wasm32,
	116	.wasm64,
	117	=> std.Target.current.os.tag == .emscripten,
	118
	119	else => true,
	120	};
	121	const default_sys_stack_trace_frames: usize = if (sys_can_stack_trace) 4 else 0;
	122	const default_stack_trace_frames: usize = switch (std.builtin.mode) {
	123	.Debug => default_sys_stack_trace_frames,
	124	else => 0,
	125	};
	126
	127	pub const Config = struct {
	128	/// Number of stack frames to capture.
	129	stack_trace_frames: usize = default_stack_trace_frames,
	130
	131	/// If true, the allocator will have two fields:
	132	/// * `total_requested_bytes` which tracks the total allocated bytes of memory requested.
	133	/// * `requested_memory_limit` which causes allocations to return `error.OutOfMemory`
	134	/// when the `total_requested_bytes` exceeds this limit.
	135	/// If false, these fields will be `void`.
	136	enable_memory_limit: bool = false,
	137
	138	/// Whether to enable safety checks.
	139	safety: bool = std.debug.runtime_safety,
	140
	141	/// Whether the allocator may be used simultaneously from multiple threads.
	142	thread_safe: bool = !std.builtin.single_threaded,
	143	};
	144
	145	pub fn GeneralPurposeAllocator(comptime config: Config) type {
	146	return struct {
	147	allocator: Allocator = Allocator{
	148	.allocFn = alloc,
	149	.resizeFn = resize,
	150	},
	151	backing_allocator: *Allocator = std.heap.page_allocator,
	152	buckets: [small_bucket_count]?BucketHeader = [1]?BucketHeader{null} ** small_bucket_count,
	153	large_allocations: LargeAllocTable = .{},
	154
	155	total_requested_bytes: @TypeOf(total_requested_bytes_init) = total_requested_bytes_init,
	156	requested_memory_limit: @TypeOf(requested_memory_limit_init) = requested_memory_limit_init,
	157
	158	mutex: @TypeOf(mutex_init) = mutex_init,
	159
	160	const Self = @This();
	161
	162	const total_requested_bytes_init = if (config.enable_memory_limit) @as(usize, 0) else {};
	163	const requested_memory_limit_init = if (config.enable_memory_limit) @as(usize, math.maxInt(usize)) else {};
	164
	165	const mutex_init = if (config.thread_safe) std.Mutex{} else std.mutex.Dummy{};
	166
	167	const stack_n = config.stack_trace_frames;
	168	const one_trace_size = @sizeOf(usize) * stack_n;
	169	const traces_per_slot = 2;
	170
	171	pub const Error = mem.Allocator.Error;
	172
	173	const small_bucket_count = math.log2(page_size);
	174	const largest_bucket_object_size = 1 << (small_bucket_count - 1);
	175
	176	const LargeAlloc = struct {
	177	bytes: []u8,
	178	stack_addresses: [stack_n]usize,
	179
	180	fn dumpStackTrace(self: *LargeAlloc) void {
	181	var len: usize = 0;
	182	while (len < stack_n and self.stack_addresses[len] != 0) {
	183	len += 1;
	184	}
	185	const stack_trace = StackTrace{
	186	.instruction_addresses = &self.stack_addresses,
	187	.index = len,
	188	};
	189	std.debug.dumpStackTrace(stack_trace);
	190	}
	191	};
	192	const LargeAllocTable = std.AutoHashMapUnmanaged(usize, LargeAlloc);
	193
	194	// Bucket: In memory, in order:
	195	// * BucketHeader
	196	// * bucket_used_bits: [N]u8, // 1 bit for every slot; 1 byte for every 8 slots
	197	// * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation
	198
	199	const BucketHeader = struct {
	200	prev: *BucketHeader,
	201	next: *BucketHeader,
	202	page: [*]align(page_size) u8,
	203	alloc_cursor: SlotIndex,
	204	used_count: SlotIndex,
	205
	206	fn usedBits(bucket: BucketHeader, index: usize) u8 {
	207	return @intToPtr(*u8, @ptrToInt(bucket) + @sizeOf(BucketHeader) + index);
	208	}
	209
	210	fn stackTracePtr(
	211	bucket: *BucketHeader,
	212	size_class: usize,
	213	slot_index: SlotIndex,
	214	trace_kind: TraceKind,
	215	) *[stack_n]usize {
	216	const start_ptr = @ptrCast([*]u8, bucket) + bucketStackFramesStart(size_class);
	217	const addr = start_ptr + one_trace_size * traces_per_slot * slot_index +
	218	@enumToInt(trace_kind) * @as(usize, one_trace_size);
	219	return @ptrCast(*[stack_n]usize, @alignCast(@alignOf(usize), addr));
	220	}
	221
	222	fn captureStackTrace(
	223	bucket: *BucketHeader,
	224	ret_addr: usize,
	225	size_class: usize,
	226	slot_index: SlotIndex,
	227	trace_kind: TraceKind,
	228	) void {
	229	// Initialize them to 0. When determining the count we must look
	230	// for non zero addresses.
	231	const stack_addresses = bucket.stackTracePtr(size_class, slot_index, trace_kind);
	232	collectStackTrace(ret_addr, stack_addresses);
	233	}
	234	};
	235
	236	fn bucketStackTrace(
	237	bucket: *BucketHeader,
	238	size_class: usize,
	239	slot_index: SlotIndex,
	240	trace_kind: TraceKind,
	241	) StackTrace {
	242	const stack_addresses = bucket.stackTracePtr(size_class, slot_index, trace_kind);
	243	var len: usize = 0;
	244	while (len < stack_n and stack_addresses[len] != 0) {
	245	len += 1;
	246	}
	247	return StackTrace{
	248	.instruction_addresses = stack_addresses,
	249	.index = len,
	250	};
	251	}
	252
	253	fn bucketStackFramesStart(size_class: usize) usize {
	254	return mem.alignForward(
	255	@sizeOf(BucketHeader) + usedBitsCount(size_class),
	256	@alignOf(usize),
	257	);
	258	}
	259
	260	fn bucketSize(size_class: usize) usize {
	261	const slot_count = @divExact(page_size, size_class);
	262	return bucketStackFramesStart(size_class) + one_trace_size * traces_per_slot * slot_count;
	263	}
	264
	265	fn usedBitsCount(size_class: usize) usize {
	266	const slot_count = @divExact(page_size, size_class);
	267	if (slot_count < 8) return 1;
	268	return @divExact(slot_count, 8);
	269	}
	270
	271	fn detectLeaksInBucket(
	272	bucket: *BucketHeader,
	273	size_class: usize,
	274	used_bits_count: usize,
	275	) bool {
	276	var leaks = false;
	277	var used_bits_byte: usize = 0;
	278	while (used_bits_byte < used_bits_count) : (used_bits_byte += 1) {
	279	const used_byte = bucket.usedBits(used_bits_byte).*;
	280	if (used_byte != 0) {
	281	var bit_index: u3 = 0;
	282	while (true) : (bit_index += 1) {
	283	const is_used = @truncate(u1, used_byte >> bit_index) != 0;
	284	if (is_used) {
	285	std.debug.print("\nMemory leak detected:\n", .{});
	286	const slot_index = @intCast(SlotIndex, used_bits_byte * 8 + bit_index);
	287	const stack_trace = bucketStackTrace(
	288	bucket,
	289	size_class,
	290	slot_index,
	291	.alloc,
	292	);
	293	std.debug.dumpStackTrace(stack_trace);
	294	leaks = true;
	295	}
	296	if (bit_index == math.maxInt(u3))
	297	break;
	298	}
	299	}
	300	}
	301	return leaks;
	302	}
	303
	304	/// Returns whether there were leaks.
	305	pub fn deinit(self: *Self) bool {
	306	var leaks = false;
	307	for (self.buckets) \|optional_bucket, bucket_i\| {
	308	const first_bucket = optional_bucket orelse continue;
	309	const size_class = @as(usize, 1) << @intCast(math.Log2Int(usize), bucket_i);
	310	const used_bits_count = usedBitsCount(size_class);
	311	var bucket = first_bucket;
	312	while (true) {
	313	leaks = detectLeaksInBucket(bucket, size_class, used_bits_count) or leaks;
	314	bucket = bucket.next;
	315	if (bucket == first_bucket)
	316	break;
	317	}
	318	}
	319	for (self.large_allocations.items()) \|*large_alloc\| {
	320	std.debug.print("\nMemory leak detected (0x{x}):\n", .{@ptrToInt(large_alloc.value.bytes.ptr)});
	321	large_alloc.value.dumpStackTrace();
	322	leaks = true;
	323	}
	324	self.large_allocations.deinit(self.backing_allocator);
	325	self.* = undefined;
	326	return leaks;
	327	}
	328
	329	fn collectStackTrace(first_trace_addr: usize, addresses: *[stack_n]usize) void {
	330	if (stack_n == 0) return;
	331	mem.set(usize, addresses, 0);
	332	var stack_trace = StackTrace{
	333	.instruction_addresses = addresses,
	334	.index = 0,
	335	};
	336	std.debug.captureStackTrace(first_trace_addr, &stack_trace);
	337	}
	338
	339	fn allocSlot(self: Self, size_class: usize, trace_addr: usize) Error![]u8 {
	340	const bucket_index = math.log2(size_class);
	341	const first_bucket = self.buckets[bucket_index] orelse try self.createBucket(
	342	size_class,
	343	bucket_index,
	344	);
	345	var bucket = first_bucket;
	346	const slot_count = @divExact(page_size, size_class);
	347	while (bucket.alloc_cursor == slot_count) {
	348	const prev_bucket = bucket;
	349	bucket = prev_bucket.next;
	350	if (bucket == first_bucket) {
	351	// make a new one
	352	bucket = try self.createBucket(size_class, bucket_index);
	353	bucket.prev = prev_bucket;
	354	bucket.next = prev_bucket.next;
	355	prev_bucket.next = bucket;
	356	bucket.next.prev = bucket;
	357	}
	358	}
	359	// change the allocator's current bucket to be this one
	360	self.buckets[bucket_index] = bucket;
	361
	362	const slot_index = bucket.alloc_cursor;
	363	bucket.alloc_cursor += 1;
	364
	365	var used_bits_byte = bucket.usedBits(slot_index / 8);
	366	const used_bit_index: u3 = @intCast(u3, slot_index % 8); // TODO cast should be unnecessary
	367	used_bits_byte.* \|= (@as(u8, 1) << used_bit_index);
	368	bucket.used_count += 1;
	369	bucket.captureStackTrace(trace_addr, size_class, slot_index, .alloc);
	370	return bucket.page + slot_index * size_class;
	371	}
	372
	373	fn searchBucket(
	374	self: *Self,
	375	bucket_index: usize,
	376	addr: usize,
	377	) ?*BucketHeader {
	378	const first_bucket = self.buckets[bucket_index] orelse return null;
	379	var bucket = first_bucket;
	380	while (true) {
	381	const in_bucket_range = (addr >= @ptrToInt(bucket.page) and
	382	addr < @ptrToInt(bucket.page) + page_size);
	383	if (in_bucket_range) return bucket;
	384	bucket = bucket.prev;
	385	if (bucket == first_bucket) {
	386	return null;
	387	}
	388	self.buckets[bucket_index] = bucket;
	389	}
	390	}
	391
	392	fn freeSlot(
	393	self: *Self,
	394	bucket: *BucketHeader,
	395	bucket_index: usize,
	396	size_class: usize,
	397	slot_index: SlotIndex,
	398	used_byte: *u8,
	399	used_bit_index: u3,
	400	trace_addr: usize,
	401	) void {
	402	// Capture stack trace to be the "first free", in case a double free happens.
	403	bucket.captureStackTrace(trace_addr, size_class, slot_index, .free);
	404
	405	used_byte.* &= ~(@as(u8, 1) << used_bit_index);
	406	bucket.used_count -= 1;
	407	if (bucket.used_count == 0) {
	408	if (bucket.next == bucket) {
	409	// it's the only bucket and therefore the current one
	410	self.buckets[bucket_index] = null;
	411	} else {
	412	bucket.next.prev = bucket.prev;
	413	bucket.prev.next = bucket.next;
	414	self.buckets[bucket_index] = bucket.prev;
	415	}
	416	self.backing_allocator.free(bucket.page[0..page_size]);
	417	const bucket_size = bucketSize(size_class);
	418	const bucket_slice = @ptrCast([*]align(@alignOf(BucketHeader)) u8, bucket)[0..bucket_size];
	419	self.backing_allocator.free(bucket_slice);
	420	} else {
	421	// TODO Set the slot data to undefined.
	422	// Related: https://github.com/ziglang/zig/issues/4298
	423	}
	424	}
	425
	426	/// This function assumes the object is in the large object storage regardless
	427	/// of the parameters.
	428	fn resizeLarge(
	429	self: *Self,
	430	old_mem: []u8,
	431	old_align: u29,
	432	new_size: usize,
	433	len_align: u29,
	434	ret_addr: usize,
	435	) Error!usize {
	436	const entry = self.large_allocations.getEntry(@ptrToInt(old_mem.ptr)) orelse {
	437	if (config.safety) {
	438	@panic("Invalid free");
	439	} else {
	440	unreachable;
	441	}
	442	};
	443
	444	if (config.safety and old_mem.len != entry.value.bytes.len) {
	445	std.debug.print("\nAllocation size {} bytes does not match free size {}. Allocated here:\n", .{
	446	entry.value.bytes.len,
	447	old_mem.len,
	448	});
	449	entry.value.dumpStackTrace();
	450
	451	@panic("\nFree here:");
	452	}
	453
	454	const result_len = try self.backing_allocator.resizeFn(self.backing_allocator, old_mem, old_align, new_size, len_align, ret_addr);
	455
	456	if (result_len == 0) {
	457	self.large_allocations.removeAssertDiscard(@ptrToInt(old_mem.ptr));
	458	return 0;
	459	}
	460
	461	entry.value.bytes = old_mem.ptr[0..result_len];
	462	collectStackTrace(ret_addr, &entry.value.stack_addresses);
	463	return result_len;
	464	}
	465
	466	pub fn setRequestedMemoryLimit(self: *Self, limit: usize) void {
	467	self.requested_memory_limit = limit;
	468	}
	469
	470	fn resize(
	471	allocator: *Allocator,
	472	old_mem: []u8,
	473	old_align: u29,
	474	new_size: usize,
	475	len_align: u29,
	476	ret_addr: usize,
	477	) Error!usize {
	478	const self = @fieldParentPtr(Self, "allocator", allocator);
	479
	480	const held = self.mutex.acquire();
	481	defer held.release();
	482
	483	const prev_req_bytes = self.total_requested_bytes;
	484	if (config.enable_memory_limit) {
	485	const new_req_bytes = prev_req_bytes + new_size - old_mem.len;
	486	if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) {
	487	return error.OutOfMemory;
	488	}
	489	self.total_requested_bytes = new_req_bytes;
	490	}
	491	errdefer if (config.enable_memory_limit) {
	492	self.total_requested_bytes = prev_req_bytes;
	493	};
	494
	495	assert(old_mem.len != 0);
	496
	497	const aligned_size = math.max(old_mem.len, old_align);
	498	if (aligned_size > largest_bucket_object_size) {
	499	return self.resizeLarge(old_mem, old_align, new_size, len_align, ret_addr);
	500	}
	501	const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size);
	502
	503	var bucket_index = math.log2(size_class_hint);
	504	var size_class: usize = size_class_hint;
	505	const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) {
	506	if (self.searchBucket(bucket_index, @ptrToInt(old_mem.ptr))) \|bucket\| {
	507	break bucket;
	508	}
	509	size_class *= 2;
	510	} else {
	511	return self.resizeLarge(old_mem, old_align, new_size, len_align, ret_addr);
	512	};
	513	const byte_offset = @ptrToInt(old_mem.ptr) - @ptrToInt(bucket.page);
	514	const slot_index = @intCast(SlotIndex, byte_offset / size_class);
	515	const used_byte_index = slot_index / 8;
	516	const used_bit_index = @intCast(u3, slot_index % 8);
	517	const used_byte = bucket.usedBits(used_byte_index);
	518	const is_used = @truncate(u1, used_byte.* >> used_bit_index) != 0;
	519	if (!is_used) {
	520	if (config.safety) {
	521	// print allocation stack trace
	522	std.debug.print("\nDouble free detected, allocated here:\n", .{});
	523	const alloc_stack_trace = bucketStackTrace(bucket, size_class, slot_index, .alloc);
	524	std.debug.dumpStackTrace(alloc_stack_trace);
	525	std.debug.print("\nFirst free here:\n", .{});
	526	const free_stack_trace = bucketStackTrace(bucket, size_class, slot_index, .free);
	527	std.debug.dumpStackTrace(free_stack_trace);
	528	@panic("\nSecond free here:");
	529	} else {
	530	unreachable;
	531	}
	532	}
	533	if (new_size == 0) {
	534	self.freeSlot(bucket, bucket_index, size_class, slot_index, used_byte, used_bit_index, ret_addr);
	535	return @as(usize, 0);
	536	}
	537	const new_aligned_size = math.max(new_size, old_align);
	538	const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size);
	539	if (new_size_class <= size_class) {
	540	return new_size;
	541	}
	542	return error.OutOfMemory;
	543	}
	544
	545	fn alloc(allocator: *Allocator, len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Error![]u8 {
	546	const self = @fieldParentPtr(Self, "allocator", allocator);
	547
	548	const held = self.mutex.acquire();
	549	defer held.release();
	550
	551	const prev_req_bytes = self.total_requested_bytes;
	552	if (config.enable_memory_limit) {
	553	const new_req_bytes = prev_req_bytes + len;
	554	if (new_req_bytes > self.requested_memory_limit) {
	555	return error.OutOfMemory;
	556	}
	557	self.total_requested_bytes = new_req_bytes;
	558	}
	559	errdefer if (config.enable_memory_limit) {
	560	self.total_requested_bytes = prev_req_bytes;
	561	};
	562
	563	const new_aligned_size = math.max(len, ptr_align);
	564	if (new_aligned_size > largest_bucket_object_size) {
	565	try self.large_allocations.ensureCapacity(
	566	self.backing_allocator,
	567	self.large_allocations.entries.items.len + 1,
	568	);
	569
	570	const slice = try self.backing_allocator.allocFn(self.backing_allocator, len, ptr_align, len_align, ret_addr);
	571
	572	const gop = self.large_allocations.getOrPutAssumeCapacity(@ptrToInt(slice.ptr));
	573	assert(!gop.found_existing); // This would mean the kernel double-mapped pages.
	574	gop.entry.value.bytes = slice;
	575	collectStackTrace(ret_addr, &gop.entry.value.stack_addresses);
	576
	577	return slice;
	578	} else {
	579	const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size);
	580	const ptr = try self.allocSlot(new_size_class, ret_addr);
	581	return ptr[0..len];
	582	}
	583	}
	584
	585	fn createBucket(self: Self, size_class: usize, bucket_index: usize) Error!BucketHeader {
	586	const page = try self.backing_allocator.allocAdvanced(u8, page_size, page_size, .exact);
	587	errdefer self.backing_allocator.free(page);
	588
	589	const bucket_size = bucketSize(size_class);
	590	const bucket_bytes = try self.backing_allocator.allocAdvanced(u8, @alignOf(BucketHeader), bucket_size, .exact);
	591	const ptr = @ptrCast(*BucketHeader, bucket_bytes.ptr);
	592	ptr.* = BucketHeader{
	593	.prev = ptr,
	594	.next = ptr,
	595	.page = page.ptr,
	596	.alloc_cursor = 0,
	597	.used_count = 0,
	598	};
	599	self.buckets[bucket_index] = ptr;
	600	// Set the used bits to all zeroes
	601	@memset(@as(*[1]u8, ptr.usedBits(0)), 0, usedBitsCount(size_class));
	602	return ptr;
	603	}
	604	};
	605	}
	606
	607	const TraceKind = enum {
	608	alloc,
	609	free,
	610	};
	611
	612	const test_config = Config{};
	613
	614	test "small allocations - free in same order" {
	615	var gpa = GeneralPurposeAllocator(test_config){};
	616	defer std.testing.expect(!gpa.deinit());
	617	const allocator = &gpa.allocator;
	618
	619	var list = std.ArrayList(*u64).init(std.testing.allocator);
	620	defer list.deinit();
	621
	622	var i: usize = 0;
	623	while (i < 513) : (i += 1) {
	624	const ptr = try allocator.create(u64);
	625	try list.append(ptr);
	626	}
	627
	628	for (list.items) \|ptr\| {
	629	allocator.destroy(ptr);
	630	}
	631	}
	632
	633	test "small allocations - free in reverse order" {
	634	var gpa = GeneralPurposeAllocator(test_config){};
	635	defer std.testing.expect(!gpa.deinit());
	636	const allocator = &gpa.allocator;
	637
	638	var list = std.ArrayList(*u64).init(std.testing.allocator);
	639	defer list.deinit();
	640
	641	var i: usize = 0;
	642	while (i < 513) : (i += 1) {
	643	const ptr = try allocator.create(u64);
	644	try list.append(ptr);
	645	}
	646
	647	while (list.popOrNull()) \|ptr\| {
	648	allocator.destroy(ptr);
	649	}
	650	}
	651
	652	test "large allocations" {
	653	var gpa = GeneralPurposeAllocator(test_config){};
	654	defer std.testing.expect(!gpa.deinit());
	655	const allocator = &gpa.allocator;
	656
	657	const ptr1 = try allocator.alloc(u64, 42768);
	658	const ptr2 = try allocator.alloc(u64, 52768);
	659	allocator.free(ptr1);
	660	const ptr3 = try allocator.alloc(u64, 62768);
	661	allocator.free(ptr3);
	662	allocator.free(ptr2);
	663	}
	664
	665	test "realloc" {
	666	var gpa = GeneralPurposeAllocator(test_config){};
	667	defer std.testing.expect(!gpa.deinit());
	668	const allocator = &gpa.allocator;
	669
	670	var slice = try allocator.alignedAlloc(u8, @alignOf(u32), 1);
	671	defer allocator.free(slice);
	672	slice[0] = 0x12;
	673
	674	// This reallocation should keep its pointer address.
	675	const old_slice = slice;
	676	slice = try allocator.realloc(slice, 2);
	677	std.testing.expect(old_slice.ptr == slice.ptr);
	678	std.testing.expect(slice[0] == 0x12);
	679	slice[1] = 0x34;
	680
	681	// This requires upgrading to a larger size class
	682	slice = try allocator.realloc(slice, 17);
	683	std.testing.expect(slice[0] == 0x12);
	684	std.testing.expect(slice[1] == 0x34);
	685	}
	686
	687	test "shrink" {
	688	var gpa = GeneralPurposeAllocator(test_config){};
	689	defer std.testing.expect(!gpa.deinit());
	690	const allocator = &gpa.allocator;
	691
	692	var slice = try allocator.alloc(u8, 20);
	693	defer allocator.free(slice);
	694
	695	mem.set(u8, slice, 0x11);
	696
	697	slice = allocator.shrink(slice, 17);
	698
	699	for (slice) \|b\| {
	700	std.testing.expect(b == 0x11);
	701	}
	702
	703	slice = allocator.shrink(slice, 16);
	704
	705	for (slice) \|b\| {
	706	std.testing.expect(b == 0x11);
	707	}
	708	}
	709
	710	test "large object - grow" {
	711	var gpa = GeneralPurposeAllocator(test_config){};
	712	defer std.testing.expect(!gpa.deinit());
	713	const allocator = &gpa.allocator;
	714
	715	var slice1 = try allocator.alloc(u8, page_size * 2 - 20);
	716	defer allocator.free(slice1);
	717
	718	const old = slice1;
	719	slice1 = try allocator.realloc(slice1, page_size * 2 - 10);
	720	std.testing.expect(slice1.ptr == old.ptr);
	721
	722	slice1 = try allocator.realloc(slice1, page_size * 2);
	723	std.testing.expect(slice1.ptr == old.ptr);
	724
	725	slice1 = try allocator.realloc(slice1, page_size * 2 + 1);
	726	}
	727
	728	test "realloc small object to large object" {
	729	var gpa = GeneralPurposeAllocator(test_config){};
	730	defer std.testing.expect(!gpa.deinit());
	731	const allocator = &gpa.allocator;
	732
	733	var slice = try allocator.alloc(u8, 70);
	734	defer allocator.free(slice);
	735	slice[0] = 0x12;
	736	slice[60] = 0x34;
	737
	738	// This requires upgrading to a large object
	739	const large_object_size = page_size * 2 + 50;
	740	slice = try allocator.realloc(slice, large_object_size);
	741	std.testing.expect(slice[0] == 0x12);
	742	std.testing.expect(slice[60] == 0x34);
	743	}
	744
	745	test "shrink large object to large object" {
	746	var gpa = GeneralPurposeAllocator(test_config){};
	747	defer std.testing.expect(!gpa.deinit());
	748	const allocator = &gpa.allocator;
	749
	750	var slice = try allocator.alloc(u8, page_size * 2 + 50);
	751	defer allocator.free(slice);
	752	slice[0] = 0x12;
	753	slice[60] = 0x34;
	754
	755	slice = try allocator.resize(slice, page_size * 2 + 1);
	756	std.testing.expect(slice[0] == 0x12);
	757	std.testing.expect(slice[60] == 0x34);
	758
	759	slice = allocator.shrink(slice, page_size * 2 + 1);
	760	std.testing.expect(slice[0] == 0x12);
	761	std.testing.expect(slice[60] == 0x34);
	762
	763	slice = try allocator.realloc(slice, page_size * 2);
	764	std.testing.expect(slice[0] == 0x12);
	765	std.testing.expect(slice[60] == 0x34);
	766	}
	767
	768	test "shrink large object to large object with larger alignment" {
	769	var gpa = GeneralPurposeAllocator(test_config){};
	770	defer std.testing.expect(!gpa.deinit());
	771	const allocator = &gpa.allocator;
	772
	773	var debug_buffer: [1000]u8 = undefined;
	774	const debug_allocator = &std.heap.FixedBufferAllocator.init(&debug_buffer).allocator;
	775
	776	const alloc_size = page_size * 2 + 50;
	777	var slice = try allocator.alignedAlloc(u8, 16, alloc_size);
	778	defer allocator.free(slice);
	779
	780	const big_alignment: usize = switch (std.Target.current.os.tag) {
	781	.windows => page_size * 32, // Windows aligns to 64K.
	782	else => page_size * 2,
	783	};
	784	// This loop allocates until we find a page that is not aligned to the big
	785	// alignment. Then we shrink the allocation after the loop, but increase the
	786	// alignment to the higher one, that we know will force it to realloc.
	787	var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator);
	788	while (mem.isAligned(@ptrToInt(slice.ptr), big_alignment)) {
	789	try stuff_to_free.append(slice);
	790	slice = try allocator.alignedAlloc(u8, 16, alloc_size);
	791	}
	792	while (stuff_to_free.popOrNull()) \|item\| {
	793	allocator.free(item);
	794	}
	795	slice[0] = 0x12;
	796	slice[60] = 0x34;
	797
	798	slice = try allocator.reallocAdvanced(slice, big_alignment, alloc_size / 2, .exact);
	799	std.testing.expect(slice[0] == 0x12);
	800	std.testing.expect(slice[60] == 0x34);
	801	}
	802
	803	test "realloc large object to small object" {
	804	var gpa = GeneralPurposeAllocator(test_config){};
	805	defer std.testing.expect(!gpa.deinit());
	806	const allocator = &gpa.allocator;
	807
	808	var slice = try allocator.alloc(u8, page_size * 2 + 50);
	809	defer allocator.free(slice);
	810	slice[0] = 0x12;
	811	slice[16] = 0x34;
	812
	813	slice = try allocator.realloc(slice, 19);
	814	std.testing.expect(slice[0] == 0x12);
	815	std.testing.expect(slice[16] == 0x34);
	816	}
	817
	818	test "non-page-allocator backing allocator" {
	819	var gpa = GeneralPurposeAllocator(.{}){ .backing_allocator = std.testing.allocator };
	820	defer std.testing.expect(!gpa.deinit());
	821	const allocator = &gpa.allocator;
	822
	823	const ptr = try allocator.create(i32);
	824	defer allocator.destroy(ptr);
	825	}
	826
	827	test "realloc large object to larger alignment" {
	828	var gpa = GeneralPurposeAllocator(test_config){};
	829	defer std.testing.expect(!gpa.deinit());
	830	const allocator = &gpa.allocator;
	831
	832	var debug_buffer: [1000]u8 = undefined;
	833	const debug_allocator = &std.heap.FixedBufferAllocator.init(&debug_buffer).allocator;
	834
	835	var slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50);
	836	defer allocator.free(slice);
	837
	838	const big_alignment: usize = switch (std.Target.current.os.tag) {
	839	.windows => page_size * 32, // Windows aligns to 64K.
	840	else => page_size * 2,
	841	};
	842	// This loop allocates until we find a page that is not aligned to the big alignment.
	843	var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator);
	844	while (mem.isAligned(@ptrToInt(slice.ptr), big_alignment)) {
	845	try stuff_to_free.append(slice);
	846	slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50);
	847	}
	848	while (stuff_to_free.popOrNull()) \|item\| {
	849	allocator.free(item);
	850	}
	851	slice[0] = 0x12;
	852	slice[16] = 0x34;
	853
	854	slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 100, .exact);
	855	std.testing.expect(slice[0] == 0x12);
	856	std.testing.expect(slice[16] == 0x34);
	857
	858	slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 25, .exact);
	859	std.testing.expect(slice[0] == 0x12);
	860	std.testing.expect(slice[16] == 0x34);
	861
	862	slice = try allocator.reallocAdvanced(slice, big_alignment, page_size * 2 + 100, .exact);
	863	std.testing.expect(slice[0] == 0x12);
	864	std.testing.expect(slice[16] == 0x34);
	865	}
	866
	867	test "large object shrinks to small but allocation fails during shrink" {
	868	var failing_allocator = std.testing.FailingAllocator.init(std.heap.page_allocator, 3);
	869	var gpa = GeneralPurposeAllocator(.{}){ .backing_allocator = &failing_allocator.allocator };
	870	defer std.testing.expect(!gpa.deinit());
	871	const allocator = &gpa.allocator;
	872
	873	var slice = try allocator.alloc(u8, page_size * 2 + 50);
	874	defer allocator.free(slice);
	875	slice[0] = 0x12;
	876	slice[3] = 0x34;
	877
	878	// Next allocation will fail in the backing allocator of the GeneralPurposeAllocator
	879
	880	slice = allocator.shrink(slice, 4);
	881	std.testing.expect(slice[0] == 0x12);
	882	std.testing.expect(slice[3] == 0x34);
	883	}
	884
	885	test "objects of size 1024 and 2048" {
	886	var gpa = GeneralPurposeAllocator(test_config){};
	887	defer std.testing.expect(!gpa.deinit());
	888	const allocator = &gpa.allocator;
	889
	890	const slice = try allocator.alloc(u8, 1025);
	891	const slice2 = try allocator.alloc(u8, 3000);
	892
	893	allocator.free(slice);
	894	allocator.free(slice2);
	895	}
	896
	897	test "setting a memory cap" {
	898	var gpa = GeneralPurposeAllocator(.{ .enable_memory_limit = true }){};
	899	defer std.testing.expect(!gpa.deinit());
	900	const allocator = &gpa.allocator;
	901
	902	gpa.setRequestedMemoryLimit(1010);
	903
	904	const small = try allocator.create(i32);
	905	std.testing.expect(gpa.total_requested_bytes == 4);
	906
	907	const big = try allocator.alloc(u8, 1000);
	908	std.testing.expect(gpa.total_requested_bytes == 1004);
	909
	910	std.testing.expectError(error.OutOfMemory, allocator.create(u64));
	911
	912	allocator.destroy(small);
	913	std.testing.expect(gpa.total_requested_bytes == 1000);
	914
	915	allocator.free(big);
	916	std.testing.expect(gpa.total_requested_bytes == 0);
	917
	918	const exact = try allocator.alloc(u8, 1010);
	919	std.testing.expect(gpa.total_requested_bytes == 1010);
	920	allocator.free(exact);
	921	}

lib/std/heap/logging_allocator.zig+19-6

...	...	@@ -23,10 +23,16 @@ pub fn LoggingAllocator(comptime OutStreamType: type) type {
23	23	};
24	24	}
25	25
26		fn alloc(allocator: *Allocator, len: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {
	26	fn alloc(
	27	allocator: *Allocator,
	28	len: usize,
	29	ptr_align: u29,
	30	len_align: u29,
	31	ra: usize,
	32	) error{OutOfMemory}![]u8 {
27	33	const self = @fieldParentPtr(Self, "allocator", allocator);
28	34	self.out_stream.print("alloc : {}", .{len}) catch {};
29		const result = self.parent_allocator.callAllocFn(len, ptr_align, len_align);
	35	const result = self.parent_allocator.allocFn(self.parent_allocator, len, ptr_align, len_align, ra);
30	36	if (result) \|buff\| {
31	37	self.out_stream.print(" success!\n", .{}) catch {};
32	38	} else \|err\| {
...	...	@@ -35,7 +41,14 @@ pub fn LoggingAllocator(comptime OutStreamType: type) type {
35	41	return result;
36	42	}
37	43
38		fn resize(allocator: *Allocator, buf: []u8, new_len: usize, len_align: u29) error{OutOfMemory}!usize {
	44	fn resize(
	45	allocator: *Allocator,
	46	buf: []u8,
	47	buf_align: u29,
	48	new_len: usize,
	49	len_align: u29,
	50	ra: usize,
	51	) error{OutOfMemory}!usize {
39	52	const self = @fieldParentPtr(Self, "allocator", allocator);
40	53	if (new_len == 0) {
41	54	self.out_stream.print("free : {}\n", .{buf.len}) catch {};
...	...	@@ -44,7 +57,7 @@ pub fn LoggingAllocator(comptime OutStreamType: type) type {
44	57	} else {
45	58	self.out_stream.print("expand: {} to {}", .{ buf.len, new_len }) catch {};
46	59	}
47		if (self.parent_allocator.callResizeFn(buf, new_len, len_align)) \|resized_len\| {
	60	if (self.parent_allocator.resizeFn(self.parent_allocator, buf, buf_align, new_len, len_align, ra)) \|resized_len\| {
48	61	if (new_len > buf.len) {
49	62	self.out_stream.print(" success!\n", .{}) catch {};
50	63	}
...	...	@@ -74,9 +87,9 @@ test "LoggingAllocator" {
74	87	const allocator = &loggingAllocator(&fixedBufferAllocator.allocator, fbs.outStream()).allocator;
75	88
76	89	var a = try allocator.alloc(u8, 10);
77		a.len = allocator.shrinkBytes(a, 5, 0);
	90	a = allocator.shrink(a, 5);
78	91	std.debug.assert(a.len == 5);
79		std.testing.expectError(error.OutOfMemory, allocator.callResizeFn(a, 20, 0));
	92	std.testing.expectError(error.OutOfMemory, allocator.resize(a, 20));
80	93	allocator.free(a);
81	94
82	95	std.testing.expectEqualSlices(u8,

lib/std/math.zig+4

...	...	@@ -837,6 +837,10 @@ pub fn ceilPowerOfTwo(comptime T: type, value: T) (error{Overflow}!T) {
837	837	return @intCast(T, x);
838	838	}
839	839
	840	pub fn ceilPowerOfTwoAssert(comptime T: type, value: T) T {
	841	return ceilPowerOfTwo(T, value) catch unreachable;
	842	}
	843
840	844	test "math.ceilPowerOfTwoPromote" {
841	845	testCeilPowerOfTwoPromote();
842	846	comptime testCeilPowerOfTwoPromote();

lib/std/mem.zig+22-385

...	...	@@ -8,391 +8,13 @@ const meta = std.meta;
8	8	const trait = meta.trait;
9	9	const testing = std.testing;
10	10
11		// https://github.com/ziglang/zig/issues/2564
	11	/// https://github.com/ziglang/zig/issues/2564
12	12	pub const page_size = switch (builtin.arch) {
13	13	.wasm32, .wasm64 => 64 * 1024,
14	14	else => 4 * 1024,
15	15	};
16	16
17		pub const Allocator = struct {
18		pub const Error = error{OutOfMemory};
19
20		/// Attempt to allocate at least `len` bytes aligned to `ptr_align`.
21		///
22		/// If `len_align` is `0`, then the length returned MUST be exactly `len` bytes,
23		/// otherwise, the length must be aligned to `len_align`.
24		///
25		/// `len` must be greater than or equal to `len_align` and must be aligned by `len_align`.
26		allocFn: fn (self: *Allocator, len: usize, ptr_align: u29, len_align: u29) Error![]u8,
27
28		/// Attempt to expand or shrink memory in place. `buf.len` must equal the most recent
29		/// length returned by `allocFn` or `resizeFn`.
30		///
31		/// Passing a `new_len` of 0 frees and invalidates the buffer such that it can no
32		/// longer be passed to `resizeFn`.
33		///
34		/// error.OutOfMemory can only be returned if `new_len` is greater than `buf.len`.
35		/// If `buf` cannot be expanded to accomodate `new_len`, then the allocation MUST be
36		/// unmodified and error.OutOfMemory MUST be returned.
37		///
38		/// If `len_align` is `0`, then the length returned MUST be exactly `len` bytes,
39		/// otherwise, the length must be aligned to `len_align`.
40		///
41		/// `new_len` must be greater than or equal to `len_align` and must be aligned by `len_align`.
42		resizeFn: fn (self: *Allocator, buf: []u8, new_len: usize, len_align: u29) Error!usize,
43
44		pub fn callAllocFn(self: *Allocator, new_len: usize, alignment: u29, len_align: u29) Error![]u8 {
45		return self.allocFn(self, new_len, alignment, len_align);
46		}
47
48		pub fn callResizeFn(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) Error!usize {
49		return self.resizeFn(self, buf, new_len, len_align);
50		}
51
52		/// Set to resizeFn if in-place resize is not supported.
53		pub fn noResize(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) Error!usize {
54		if (new_len > buf.len)
55		return error.OutOfMemory;
56		return new_len;
57		}
58
59		/// Call `resizeFn`, but caller guarantees that `new_len` <= `buf.len` meaning
60		/// error.OutOfMemory should be impossible.
61		pub fn shrinkBytes(self: *Allocator, buf: []u8, new_len: usize, len_align: u29) usize {
62		assert(new_len <= buf.len);
63		return self.callResizeFn(buf, new_len, len_align) catch unreachable;
64		}
65
66		/// Realloc is used to modify the size or alignment of an existing allocation,
67		/// as well as to provide the allocator with an opportunity to move an allocation
68		/// to a better location.
69		/// When the size/alignment is greater than the previous allocation, this function
70		/// returns `error.OutOfMemory` when the requested new allocation could not be granted.
71		/// When the size/alignment is less than or equal to the previous allocation,
72		/// this function returns `error.OutOfMemory` when the allocator decides the client
73		/// would be better off keeping the extra alignment/size. Clients will call
74		/// `callResizeFn` when they require the allocator to track a new alignment/size,
75		/// and so this function should only return success when the allocator considers
76		/// the reallocation desirable from the allocator's perspective.
77		/// As an example, `std.ArrayList` tracks a "capacity", and therefore can handle
78		/// reallocation failure, even when `new_n` <= `old_mem.len`. A `FixedBufferAllocator`
79		/// would always return `error.OutOfMemory` for `reallocFn` when the size/alignment
80		/// is less than or equal to the old allocation, because it cannot reclaim the memory,
81		/// and thus the `std.ArrayList` would be better off retaining its capacity.
82		/// When `reallocFn` returns,
83		/// `return_value[0..min(old_mem.len, new_byte_count)]` must be the same
84		/// as `old_mem` was when `reallocFn` is called. The bytes of
85		/// `return_value[old_mem.len..]` have undefined values.
86		/// The returned slice must have its pointer aligned at least to `new_alignment` bytes.
87		fn reallocBytes(
88		self: *Allocator,
89		/// Guaranteed to be the same as what was returned from most recent call to
90		/// `allocFn` or `resizeFn`.
91		/// If `old_mem.len == 0` then this is a new allocation and `new_byte_count`
92		/// is guaranteed to be >= 1.
93		old_mem: []u8,
94		/// If `old_mem.len == 0` then this is `undefined`, otherwise:
95		/// Guaranteed to be the same as what was passed to `allocFn`.
96		/// Guaranteed to be >= 1.
97		/// Guaranteed to be a power of 2.
98		old_alignment: u29,
99		/// If `new_byte_count` is 0 then this is a free and it is guaranteed that
100		/// `old_mem.len != 0`.
101		new_byte_count: usize,
102		/// Guaranteed to be >= 1.
103		/// Guaranteed to be a power of 2.
104		/// Returned slice's pointer must have this alignment.
105		new_alignment: u29,
106		/// 0 indicates the length of the slice returned MUST match `new_byte_count` exactly
107		/// non-zero means the length of the returned slice must be aligned by `len_align`
108		/// `new_len` must be aligned by `len_align`
109		len_align: u29,
110		) Error![]u8 {
111		if (old_mem.len == 0) {
112		const new_mem = try self.callAllocFn(new_byte_count, new_alignment, len_align);
113		@memset(new_mem.ptr, undefined, new_byte_count);
114		return new_mem;
115		}
116
117		if (isAligned(@ptrToInt(old_mem.ptr), new_alignment)) {
118		if (new_byte_count <= old_mem.len) {
119		const shrunk_len = self.shrinkBytes(old_mem, new_byte_count, len_align);
120		return old_mem.ptr[0..shrunk_len];
121		}
122		if (self.callResizeFn(old_mem, new_byte_count, len_align)) \|resized_len\| {
123		assert(resized_len >= new_byte_count);
124		@memset(old_mem.ptr + new_byte_count, undefined, resized_len - new_byte_count);
125		return old_mem.ptr[0..resized_len];
126		} else \|_\| {}
127		}
128		if (new_byte_count <= old_mem.len and new_alignment <= old_alignment) {
129		return error.OutOfMemory;
130		}
131		return self.moveBytes(old_mem, new_byte_count, new_alignment, len_align);
132		}
133
134		/// Move the given memory to a new location in the given allocator to accomodate a new
135		/// size and alignment.
136		fn moveBytes(self: *Allocator, old_mem: []u8, new_len: usize, new_alignment: u29, len_align: u29) Error![]u8 {
137		assert(old_mem.len > 0);
138		assert(new_len > 0);
139		const new_mem = try self.callAllocFn(new_len, new_alignment, len_align);
140		@memcpy(new_mem.ptr, old_mem.ptr, std.math.min(new_len, old_mem.len));
141		// DISABLED TO AVOID BUGS IN TRANSLATE C
142		// use './zig build test-translate-c' to reproduce, some of the symbols in the
143		// generated C code will be a sequence of 0xaa (the undefined value), meaning
144		// it is printing data that has been freed
145		//@memset(old_mem.ptr, undefined, old_mem.len);
146		_ = self.shrinkBytes(old_mem, 0, 0);
147		return new_mem;
148		}
149
150		/// Returns a pointer to undefined memory.
151		/// Call `destroy` with the result to free the memory.
152		pub fn create(self: Allocator, comptime T: type) Error!T {
153		if (@sizeOf(T) == 0) return &(T{});
154		const slice = try self.alloc(T, 1);
155		return &slice[0];
156		}
157
158		/// `ptr` should be the return value of `create`, or otherwise
159		/// have the same address and alignment property.
160		pub fn destroy(self: *Allocator, ptr: anytype) void {
161		const T = @TypeOf(ptr).Child;
162		if (@sizeOf(T) == 0) return;
163		const non_const_ptr = @intToPtr([*]u8, @ptrToInt(ptr));
164		_ = self.shrinkBytes(non_const_ptr[0..@sizeOf(T)], 0, 0);
165		}
166
167		/// Allocates an array of `n` items of type `T` and sets all the
168		/// items to `undefined`. Depending on the Allocator
169		/// implementation, it may be required to call `free` once the
170		/// memory is no longer needed, to avoid a resource leak. If the
171		/// `Allocator` implementation is unknown, then correct code will
172		/// call `free` when done.
173		///
174		/// For allocating a single item, see `create`.
175		pub fn alloc(self: *Allocator, comptime T: type, n: usize) Error![]T {
176		return self.alignedAlloc(T, null, n);
177		}
178
179		pub fn allocWithOptions(
180		self: *Allocator,
181		comptime Elem: type,
182		n: usize,
183		/// null means naturally aligned
184		comptime optional_alignment: ?u29,
185		comptime optional_sentinel: ?Elem,
186		) Error!AllocWithOptionsPayload(Elem, optional_alignment, optional_sentinel) {
187		if (optional_sentinel) \|sentinel\| {
188		const ptr = try self.alignedAlloc(Elem, optional_alignment, n + 1);
189		ptr[n] = sentinel;
190		return ptr[0..n :sentinel];
191		} else {
192		return self.alignedAlloc(Elem, optional_alignment, n);
193		}
194		}
195
196		fn AllocWithOptionsPayload(comptime Elem: type, comptime alignment: ?u29, comptime sentinel: ?Elem) type {
197		if (sentinel) \|s\| {
198		return [:s]align(alignment orelse @alignOf(Elem)) Elem;
199		} else {
200		return []align(alignment orelse @alignOf(Elem)) Elem;
201		}
202		}
203
204		/// Allocates an array of `n + 1` items of type `T` and sets the first `n`
205		/// items to `undefined` and the last item to `sentinel`. Depending on the
206		/// Allocator implementation, it may be required to call `free` once the
207		/// memory is no longer needed, to avoid a resource leak. If the
208		/// `Allocator` implementation is unknown, then correct code will
209		/// call `free` when done.
210		///
211		/// For allocating a single item, see `create`.
212		///
213		/// Deprecated; use `allocWithOptions`.
214		pub fn allocSentinel(self: *Allocator, comptime Elem: type, n: usize, comptime sentinel: Elem) Error![:sentinel]Elem {
215		return self.allocWithOptions(Elem, n, null, sentinel);
216		}
217
218		/// Deprecated: use `allocAdvanced`
219		pub fn alignedAlloc(
220		self: *Allocator,
221		comptime T: type,
222		/// null means naturally aligned
223		comptime alignment: ?u29,
224		n: usize,
225		) Error![]align(alignment orelse @alignOf(T)) T {
226		return self.allocAdvanced(T, alignment, n, .exact);
227		}
228
229		const Exact = enum { exact, at_least };
230		pub fn allocAdvanced(
231		self: *Allocator,
232		comptime T: type,
233		/// null means naturally aligned
234		comptime alignment: ?u29,
235		n: usize,
236		exact: Exact,
237		) Error![]align(alignment orelse @alignOf(T)) T {
238		const a = if (alignment) \|a\| blk: {
239		if (a == @alignOf(T)) return allocAdvanced(self, T, null, n, exact);
240		break :blk a;
241		} else @alignOf(T);
242
243		if (n == 0) {
244		return @as([*]align(a) T, undefined)[0..0];
245		}
246
247		const byte_count = math.mul(usize, @sizeOf(T), n) catch return Error.OutOfMemory;
248		// TODO The `if (alignment == null)` blocks are workarounds for zig not being able to
249		// access certain type information about T without creating a circular dependency in async
250		// functions that heap-allocate their own frame with @Frame(func).
251		const sizeOfT = if (alignment == null) @intCast(u29, @divExact(byte_count, n)) else @sizeOf(T);
252		const byte_slice = try self.callAllocFn(byte_count, a, if (exact == .exact) @as(u29, 0) else sizeOfT);
253		switch (exact) {
254		.exact => assert(byte_slice.len == byte_count),
255		.at_least => assert(byte_slice.len >= byte_count),
256		}
257		@memset(byte_slice.ptr, undefined, byte_slice.len);
258		if (alignment == null) {
259		// This if block is a workaround (see comment above)
260		return @intToPtr([*]T, @ptrToInt(byte_slice.ptr))[0..@divExact(byte_slice.len, @sizeOf(T))];
261		} else {
262		return mem.bytesAsSlice(T, @alignCast(a, byte_slice));
263		}
264		}
265
266		/// This function requests a new byte size for an existing allocation,
267		/// which can be larger, smaller, or the same size as the old memory
268		/// allocation.
269		/// This function is preferred over `shrink`, because it can fail, even
270		/// when shrinking. This gives the allocator a chance to perform a
271		/// cheap shrink operation if possible, or otherwise return OutOfMemory,
272		/// indicating that the caller should keep their capacity, for example
273		/// in `std.ArrayList.shrink`.
274		/// If you need guaranteed success, call `shrink`.
275		/// If `new_n` is 0, this is the same as `free` and it always succeeds.
276		pub fn realloc(self: *Allocator, old_mem: anytype, new_n: usize) t: {
277		const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
278		break :t Error![]align(Slice.alignment) Slice.child;
279		} {
280		const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
281		return self.reallocAdvanced(old_mem, old_alignment, new_n, .exact);
282		}
283
284		pub fn reallocAtLeast(self: *Allocator, old_mem: anytype, new_n: usize) t: {
285		const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
286		break :t Error![]align(Slice.alignment) Slice.child;
287		} {
288		const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
289		return self.reallocAdvanced(old_mem, old_alignment, new_n, .at_least);
290		}
291
292		// Deprecated: use `reallocAdvanced`
293		pub fn alignedRealloc(
294		self: *Allocator,
295		old_mem: anytype,
296		comptime new_alignment: u29,
297		new_n: usize,
298		) Error![]align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
299		return self.reallocAdvanced(old_mem, new_alignment, new_n, .exact);
300		}
301
302		/// This is the same as `realloc`, except caller may additionally request
303		/// a new alignment, which can be larger, smaller, or the same as the old
304		/// allocation.
305		pub fn reallocAdvanced(
306		self: *Allocator,
307		old_mem: anytype,
308		comptime new_alignment: u29,
309		new_n: usize,
310		exact: Exact,
311		) Error![]align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
312		const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
313		const T = Slice.child;
314		if (old_mem.len == 0) {
315		return self.allocAdvanced(T, new_alignment, new_n, exact);
316		}
317		if (new_n == 0) {
318		self.free(old_mem);
319		return @as([*]align(new_alignment) T, undefined)[0..0];
320		}
321
322		const old_byte_slice = mem.sliceAsBytes(old_mem);
323		const byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;
324		// Note: can't set shrunk memory to undefined as memory shouldn't be modified on realloc failure
325		const new_byte_slice = try self.reallocBytes(old_byte_slice, Slice.alignment, byte_count, new_alignment, if (exact == .exact) @as(u29, 0) else @sizeOf(T));
326		return mem.bytesAsSlice(T, @alignCast(new_alignment, new_byte_slice));
327		}
328
329		/// Prefer calling realloc to shrink if you can tolerate failure, such as
330		/// in an ArrayList data structure with a storage capacity.
331		/// Shrink always succeeds, and `new_n` must be <= `old_mem.len`.
332		/// Returned slice has same alignment as old_mem.
333		/// Shrinking to 0 is the same as calling `free`.
334		pub fn shrink(self: *Allocator, old_mem: anytype, new_n: usize) t: {
335		const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
336		break :t []align(Slice.alignment) Slice.child;
337		} {
338		const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
339		return self.alignedShrink(old_mem, old_alignment, new_n);
340		}
341
342		/// This is the same as `shrink`, except caller may additionally request
343		/// a new alignment, which must be smaller or the same as the old
344		/// allocation.
345		pub fn alignedShrink(
346		self: *Allocator,
347		old_mem: anytype,
348		comptime new_alignment: u29,
349		new_n: usize,
350		) []align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
351		const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
352		const T = Slice.child;
353
354		if (new_n == old_mem.len)
355		return old_mem;
356		assert(new_n < old_mem.len);
357		assert(new_alignment <= Slice.alignment);
358
359		// Here we skip the overflow checking on the multiplication because
360		// new_n <= old_mem.len and the multiplication didn't overflow for that operation.
361		const byte_count = @sizeOf(T) * new_n;
362
363		const old_byte_slice = mem.sliceAsBytes(old_mem);
364		@memset(old_byte_slice.ptr + byte_count, undefined, old_byte_slice.len - byte_count);
365		_ = self.shrinkBytes(old_byte_slice, byte_count, 0);
366		return old_mem[0..new_n];
367		}
368
369		/// Free an array allocated with `alloc`. To free a single item,
370		/// see `destroy`.
371		pub fn free(self: *Allocator, memory: anytype) void {
372		const Slice = @typeInfo(@TypeOf(memory)).Pointer;
373		const bytes = mem.sliceAsBytes(memory);
374		const bytes_len = bytes.len + if (Slice.sentinel != null) @sizeOf(Slice.child) else 0;
375		if (bytes_len == 0) return;
376		const non_const_ptr = @intToPtr([*]u8, @ptrToInt(bytes.ptr));
377		@memset(non_const_ptr, undefined, bytes_len);
378		_ = self.shrinkBytes(non_const_ptr[0..bytes_len], 0, 0);
379		}
380
381		/// Copies `m` to newly allocated memory. Caller owns the memory.
382		pub fn dupe(allocator: *Allocator, comptime T: type, m: []const T) ![]T {
383		const new_buf = try allocator.alloc(T, m.len);
384		copy(T, new_buf, m);
385		return new_buf;
386		}
387
388		/// Copies `m` to newly allocated memory, with a null-terminated element. Caller owns the memory.
389		pub fn dupeZ(allocator: *Allocator, comptime T: type, m: []const T) ![:0]T {
390		const new_buf = try allocator.alloc(T, m.len + 1);
391		copy(T, new_buf, m);
392		new_buf[m.len] = 0;
393		return new_buf[0..m.len :0];
394		}
395		};
	17	pub const Allocator = @import("mem/Allocator.zig");
396	18
397	19	/// Detects and asserts if the std.mem.Allocator interface is violated by the caller
398	20	/// or the allocator.
...	...	@@ -415,7 +37,13 @@ pub fn ValidationAllocator(comptime T: type) type {
415	37	if (T == Allocator) return &self.underlying_allocator;
416	38	return &self.underlying_allocator.allocator;
417	39	}
418		pub fn alloc(allocator: *Allocator, n: usize, ptr_align: u29, len_align: u29) Allocator.Error![]u8 {
	40	pub fn alloc(
	41	allocator: *Allocator,
	42	n: usize,
	43	ptr_align: u29,
	44	len_align: u29,
	45	ret_addr: usize,
	46	) Allocator.Error![]u8 {
419	47	assert(n > 0);
420	48	assert(mem.isValidAlign(ptr_align));
421	49	if (len_align != 0) {
...	...	@@ -424,7 +52,8 @@ pub fn ValidationAllocator(comptime T: type) type {
424	52	}
425	53
426	54	const self = @fieldParentPtr(@This(), "allocator", allocator);
427		const result = try self.getUnderlyingAllocatorPtr().callAllocFn(n, ptr_align, len_align);
	55	const underlying = self.getUnderlyingAllocatorPtr();
	56	const result = try underlying.allocFn(underlying, n, ptr_align, len_align, ret_addr);
428	57	assert(mem.isAligned(@ptrToInt(result.ptr), ptr_align));
429	58	if (len_align == 0) {
430	59	assert(result.len == n);
...	...	@@ -434,14 +63,22 @@ pub fn ValidationAllocator(comptime T: type) type {
434	63	}
435	64	return result;
436	65	}
437		pub fn resize(allocator: *Allocator, buf: []u8, new_len: usize, len_align: u29) Allocator.Error!usize {
	66	pub fn resize(
	67	allocator: *Allocator,
	68	buf: []u8,
	69	buf_align: u29,
	70	new_len: usize,
	71	len_align: u29,
	72	ret_addr: usize,
	73	) Allocator.Error!usize {
438	74	assert(buf.len > 0);
439	75	if (len_align != 0) {
440	76	assert(mem.isAlignedAnyAlign(new_len, len_align));
441	77	assert(new_len >= len_align);
442	78	}
443	79	const self = @fieldParentPtr(@This(), "allocator", allocator);
444		const result = try self.getUnderlyingAllocatorPtr().callResizeFn(buf, new_len, len_align);
	80	const underlying = self.getUnderlyingAllocatorPtr();
	81	const result = try underlying.resizeFn(underlying, buf, buf_align, new_len, len_align, ret_addr);
445	82	if (len_align == 0) {
446	83	assert(result == new_len);
447	84	} else {
...	...	@@ -481,7 +118,7 @@ var failAllocator = Allocator{
481	118	.allocFn = failAllocatorAlloc,
482	119	.resizeFn = Allocator.noResize,
483	120	};
484		fn failAllocatorAlloc(self: *Allocator, n: usize, alignment: u29, len_align: u29) Allocator.Error![]u8 {
	121	fn failAllocatorAlloc(self: *Allocator, n: usize, alignment: u29, len_align: u29, ra: usize) Allocator.Error![]u8 {
485	122	return error.OutOfMemory;
486	123	}
487	124

lib/std/mem/Allocator.zig created+486

...	...	@@ -0,0 +1,486 @@
	1	//! The standard memory allocation interface.
	2
	3	const std = @import("../std.zig");
	4	const assert = std.debug.assert;
	5	const math = std.math;
	6	const mem = std.mem;
	7	const Allocator = @This();
	8
	9	pub const Error = error{OutOfMemory};
	10
	11	/// Attempt to allocate at least `len` bytes aligned to `ptr_align`.
	12	///
	13	/// If `len_align` is `0`, then the length returned MUST be exactly `len` bytes,
	14	/// otherwise, the length must be aligned to `len_align`.
	15	///
	16	/// `len` must be greater than or equal to `len_align` and must be aligned by `len_align`.
	17	///
	18	/// `ret_addr` is optionally provided as the first return address of the allocation call stack.
	19	/// If the value is `0` it means no return address has been provided.
	20	allocFn: fn (self: *Allocator, len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Error![]u8,
	21
	22	/// Attempt to expand or shrink memory in place. `buf.len` must equal the most recent
	23	/// length returned by `allocFn` or `resizeFn`. `buf_align` must equal the same value
	24	/// that was passed as the `ptr_align` parameter to the original `allocFn` call.
	25	///
	26	/// Passing a `new_len` of 0 frees and invalidates the buffer such that it can no
	27	/// longer be passed to `resizeFn`.
	28	///
	29	/// error.OutOfMemory can only be returned if `new_len` is greater than `buf.len`.
	30	/// If `buf` cannot be expanded to accomodate `new_len`, then the allocation MUST be
	31	/// unmodified and error.OutOfMemory MUST be returned.
	32	///
	33	/// If `len_align` is `0`, then the length returned MUST be exactly `len` bytes,
	34	/// otherwise, the length must be aligned to `len_align`. Note that `len_align` does not
	35	/// provide a way to modify the alignment of a pointer. Rather it provides an API for
	36	/// accepting more bytes of memory from the allocator than requested.
	37	///
	38	/// `new_len` must be greater than or equal to `len_align` and must be aligned by `len_align`.
	39	///
	40	/// `ret_addr` is optionally provided as the first return address of the allocation call stack.
	41	/// If the value is `0` it means no return address has been provided.
	42	resizeFn: fn (self: *Allocator, buf: []u8, buf_align: u29, new_len: usize, len_align: u29, ret_addr: usize) Error!usize,
	43
	44	/// Set to resizeFn if in-place resize is not supported.
	45	pub fn noResize(
	46	self: *Allocator,
	47	buf: []u8,
	48	buf_align: u29,
	49	new_len: usize,
	50	len_align: u29,
	51	ret_addr: usize,
	52	) Error!usize {
	53	if (new_len > buf.len)
	54	return error.OutOfMemory;
	55	return new_len;
	56	}
	57
	58	/// Realloc is used to modify the size or alignment of an existing allocation,
	59	/// as well as to provide the allocator with an opportunity to move an allocation
	60	/// to a better location.
	61	/// When the size/alignment is greater than the previous allocation, this function
	62	/// returns `error.OutOfMemory` when the requested new allocation could not be granted.
	63	/// When the size/alignment is less than or equal to the previous allocation,
	64	/// this function returns `error.OutOfMemory` when the allocator decides the client
	65	/// would be better off keeping the extra alignment/size. Clients will call
	66	/// `resizeFn` when they require the allocator to track a new alignment/size,
	67	/// and so this function should only return success when the allocator considers
	68	/// the reallocation desirable from the allocator's perspective.
	69	/// As an example, `std.ArrayList` tracks a "capacity", and therefore can handle
	70	/// reallocation failure, even when `new_n` <= `old_mem.len`. A `FixedBufferAllocator`
	71	/// would always return `error.OutOfMemory` for `reallocFn` when the size/alignment
	72	/// is less than or equal to the old allocation, because it cannot reclaim the memory,
	73	/// and thus the `std.ArrayList` would be better off retaining its capacity.
	74	/// When `reallocFn` returns,
	75	/// `return_value[0..min(old_mem.len, new_byte_count)]` must be the same
	76	/// as `old_mem` was when `reallocFn` is called. The bytes of
	77	/// `return_value[old_mem.len..]` have undefined values.
	78	/// The returned slice must have its pointer aligned at least to `new_alignment` bytes.
	79	fn reallocBytes(
	80	self: *Allocator,
	81	/// Guaranteed to be the same as what was returned from most recent call to
	82	/// `allocFn` or `resizeFn`.
	83	/// If `old_mem.len == 0` then this is a new allocation and `new_byte_count`
	84	/// is guaranteed to be >= 1.
	85	old_mem: []u8,
	86	/// If `old_mem.len == 0` then this is `undefined`, otherwise:
	87	/// Guaranteed to be the same as what was passed to `allocFn`.
	88	/// Guaranteed to be >= 1.
	89	/// Guaranteed to be a power of 2.
	90	old_alignment: u29,
	91	/// If `new_byte_count` is 0 then this is a free and it is guaranteed that
	92	/// `old_mem.len != 0`.
	93	new_byte_count: usize,
	94	/// Guaranteed to be >= 1.
	95	/// Guaranteed to be a power of 2.
	96	/// Returned slice's pointer must have this alignment.
	97	new_alignment: u29,
	98	/// 0 indicates the length of the slice returned MUST match `new_byte_count` exactly
	99	/// non-zero means the length of the returned slice must be aligned by `len_align`
	100	/// `new_len` must be aligned by `len_align`
	101	len_align: u29,
	102	return_address: usize,
	103	) Error![]u8 {
	104	if (old_mem.len == 0) {
	105	const new_mem = try self.allocFn(self, new_byte_count, new_alignment, len_align, return_address);
	106	// TODO: https://github.com/ziglang/zig/issues/4298
	107	@memset(new_mem.ptr, undefined, new_byte_count);
	108	return new_mem;
	109	}
	110
	111	if (mem.isAligned(@ptrToInt(old_mem.ptr), new_alignment)) {
	112	if (new_byte_count <= old_mem.len) {
	113	const shrunk_len = self.shrinkBytes(old_mem, old_alignment, new_byte_count, len_align, return_address);
	114	return old_mem.ptr[0..shrunk_len];
	115	}
	116	if (self.resizeFn(self, old_mem, old_alignment, new_byte_count, len_align, return_address)) \|resized_len\| {
	117	assert(resized_len >= new_byte_count);
	118	// TODO: https://github.com/ziglang/zig/issues/4298
	119	@memset(old_mem.ptr + new_byte_count, undefined, resized_len - new_byte_count);
	120	return old_mem.ptr[0..resized_len];
	121	} else \|_\| {}
	122	}
	123	if (new_byte_count <= old_mem.len and new_alignment <= old_alignment) {
	124	return error.OutOfMemory;
	125	}
	126	return self.moveBytes(old_mem, old_alignment, new_byte_count, new_alignment, len_align, return_address);
	127	}
	128
	129	/// Move the given memory to a new location in the given allocator to accomodate a new
	130	/// size and alignment.
	131	fn moveBytes(
	132	self: *Allocator,
	133	old_mem: []u8,
	134	old_align: u29,
	135	new_len: usize,
	136	new_alignment: u29,
	137	len_align: u29,
	138	return_address: usize,
	139	) Error![]u8 {
	140	assert(old_mem.len > 0);
	141	assert(new_len > 0);
	142	const new_mem = try self.allocFn(self, new_len, new_alignment, len_align, return_address);
	143	@memcpy(new_mem.ptr, old_mem.ptr, math.min(new_len, old_mem.len));
	144	// TODO DISABLED TO AVOID BUGS IN TRANSLATE C
	145	// TODO see also https://github.com/ziglang/zig/issues/4298
	146	// use './zig build test-translate-c' to reproduce, some of the symbols in the
	147	// generated C code will be a sequence of 0xaa (the undefined value), meaning
	148	// it is printing data that has been freed
	149	//@memset(old_mem.ptr, undefined, old_mem.len);
	150	_ = self.shrinkBytes(old_mem, old_align, 0, 0, return_address);
	151	return new_mem;
	152	}
	153
	154	/// Returns a pointer to undefined memory.
	155	/// Call `destroy` with the result to free the memory.
	156	pub fn create(self: Allocator, comptime T: type) Error!T {
	157	if (@sizeOf(T) == 0) return &(T{});
	158	const slice = try self.allocAdvancedWithRetAddr(T, null, 1, .exact, @returnAddress());
	159	return &slice[0];
	160	}
	161
	162	/// `ptr` should be the return value of `create`, or otherwise
	163	/// have the same address and alignment property.
	164	pub fn destroy(self: *Allocator, ptr: anytype) void {
	165	const T = @TypeOf(ptr).Child;
	166	if (@sizeOf(T) == 0) return;
	167	const non_const_ptr = @intToPtr([*]u8, @ptrToInt(ptr));
	168	const ptr_align = @typeInfo(@TypeOf(ptr)).Pointer.alignment;
	169	_ = self.shrinkBytes(non_const_ptr[0..@sizeOf(T)], ptr_align, 0, 0, @returnAddress());
	170	}
	171
	172	/// Allocates an array of `n` items of type `T` and sets all the
	173	/// items to `undefined`. Depending on the Allocator
	174	/// implementation, it may be required to call `free` once the
	175	/// memory is no longer needed, to avoid a resource leak. If the
	176	/// `Allocator` implementation is unknown, then correct code will
	177	/// call `free` when done.
	178	///
	179	/// For allocating a single item, see `create`.
	180	pub fn alloc(self: *Allocator, comptime T: type, n: usize) Error![]T {
	181	return self.allocAdvancedWithRetAddr(T, null, n, .exact, @returnAddress());
	182	}
	183
	184	pub fn allocWithOptions(
	185	self: *Allocator,
	186	comptime Elem: type,
	187	n: usize,
	188	/// null means naturally aligned
	189	comptime optional_alignment: ?u29,
	190	comptime optional_sentinel: ?Elem,
	191	) Error!AllocWithOptionsPayload(Elem, optional_alignment, optional_sentinel) {
	192	return self.allocWithOptionsRetAddr(Elem, n, optional_alignment, optional_sentinel, @returnAddress());
	193	}
	194
	195	pub fn allocWithOptionsRetAddr(
	196	self: *Allocator,
	197	comptime Elem: type,
	198	n: usize,
	199	/// null means naturally aligned
	200	comptime optional_alignment: ?u29,
	201	comptime optional_sentinel: ?Elem,
	202	return_address: usize,
	203	) Error!AllocWithOptionsPayload(Elem, optional_alignment, optional_sentinel) {
	204	if (optional_sentinel) \|sentinel\| {
	205	const ptr = try self.allocAdvancedWithRetAddr(Elem, optional_alignment, n + 1, .exact, return_address);
	206	ptr[n] = sentinel;
	207	return ptr[0..n :sentinel];
	208	} else {
	209	return self.allocAdvancedWithRetAddr(Elem, optional_alignment, n, .exact, return_address);
	210	}
	211	}
	212
	213	fn AllocWithOptionsPayload(comptime Elem: type, comptime alignment: ?u29, comptime sentinel: ?Elem) type {
	214	if (sentinel) \|s\| {
	215	return [:s]align(alignment orelse @alignOf(Elem)) Elem;
	216	} else {
	217	return []align(alignment orelse @alignOf(Elem)) Elem;
	218	}
	219	}
	220
	221	/// Allocates an array of `n + 1` items of type `T` and sets the first `n`
	222	/// items to `undefined` and the last item to `sentinel`. Depending on the
	223	/// Allocator implementation, it may be required to call `free` once the
	224	/// memory is no longer needed, to avoid a resource leak. If the
	225	/// `Allocator` implementation is unknown, then correct code will
	226	/// call `free` when done.
	227	///
	228	/// For allocating a single item, see `create`.
	229	///
	230	/// Deprecated; use `allocWithOptions`.
	231	pub fn allocSentinel(
	232	self: *Allocator,
	233	comptime Elem: type,
	234	n: usize,
	235	comptime sentinel: Elem,
	236	) Error![:sentinel]Elem {
	237	return self.allocWithOptionsRetAddr(Elem, n, null, sentinel, @returnAddress());
	238	}
	239
	240	/// Deprecated: use `allocAdvanced`
	241	pub fn alignedAlloc(
	242	self: *Allocator,
	243	comptime T: type,
	244	/// null means naturally aligned
	245	comptime alignment: ?u29,
	246	n: usize,
	247	) Error![]align(alignment orelse @alignOf(T)) T {
	248	return self.allocAdvancedWithRetAddr(T, alignment, n, .exact, @returnAddress());
	249	}
	250
	251	pub fn allocAdvanced(
	252	self: *Allocator,
	253	comptime T: type,
	254	/// null means naturally aligned
	255	comptime alignment: ?u29,
	256	n: usize,
	257	exact: Exact,
	258	) Error![]align(alignment orelse @alignOf(T)) T {
	259	return self.allocAdvancedWithRetAddr(T, alignment, n, exact, @returnAddress());
	260	}
	261
	262	pub const Exact = enum { exact, at_least };
	263
	264	pub fn allocAdvancedWithRetAddr(
	265	self: *Allocator,
	266	comptime T: type,
	267	/// null means naturally aligned
	268	comptime alignment: ?u29,
	269	n: usize,
	270	exact: Exact,
	271	return_address: usize,
	272	) Error![]align(alignment orelse @alignOf(T)) T {
	273	const a = if (alignment) \|a\| blk: {
	274	if (a == @alignOf(T)) return allocAdvancedWithRetAddr(self, T, null, n, exact, return_address);
	275	break :blk a;
	276	} else @alignOf(T);
	277
	278	if (n == 0) {
	279	return @as([*]align(a) T, undefined)[0..0];
	280	}
	281
	282	const byte_count = math.mul(usize, @sizeOf(T), n) catch return Error.OutOfMemory;
	283	// TODO The `if (alignment == null)` blocks are workarounds for zig not being able to
	284	// access certain type information about T without creating a circular dependency in async
	285	// functions that heap-allocate their own frame with @Frame(func).
	286	const size_of_T = if (alignment == null) @intCast(u29, @divExact(byte_count, n)) else @sizeOf(T);
	287	const len_align: u29 = switch (exact) {
	288	.exact => 0,
	289	.at_least => size_of_T,
	290	};
	291	const byte_slice = try self.allocFn(self, byte_count, a, len_align, return_address);
	292	switch (exact) {
	293	.exact => assert(byte_slice.len == byte_count),
	294	.at_least => assert(byte_slice.len >= byte_count),
	295	}
	296	// TODO: https://github.com/ziglang/zig/issues/4298
	297	@memset(byte_slice.ptr, undefined, byte_slice.len);
	298	if (alignment == null) {
	299	// This if block is a workaround (see comment above)
	300	return @intToPtr([*]T, @ptrToInt(byte_slice.ptr))[0..@divExact(byte_slice.len, @sizeOf(T))];
	301	} else {
	302	return mem.bytesAsSlice(T, @alignCast(a, byte_slice));
	303	}
	304	}
	305
	306	/// Increases or decreases the size of an allocation. It is guaranteed to not move the pointer.
	307	pub fn resize(self: *Allocator, old_mem: anytype, new_n: usize) Error!@TypeOf(old_mem) {
	308	const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
	309	const T = Slice.child;
	310	if (new_n == 0) {
	311	self.free(old_mem);
	312	return &[0]T{};
	313	}
	314	const old_byte_slice = mem.sliceAsBytes(old_mem);
	315	const new_byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;
	316	const rc = try self.resizeFn(self, old_byte_slice, Slice.alignment, new_byte_count, 0, @returnAddress());
	317	assert(rc == new_byte_count);
	318	const new_byte_slice = old_mem.ptr[0..new_byte_count];
	319	return mem.bytesAsSlice(T, new_byte_slice);
	320	}
	321
	322	/// This function requests a new byte size for an existing allocation,
	323	/// which can be larger, smaller, or the same size as the old memory
	324	/// allocation.
	325	/// This function is preferred over `shrink`, because it can fail, even
	326	/// when shrinking. This gives the allocator a chance to perform a
	327	/// cheap shrink operation if possible, or otherwise return OutOfMemory,
	328	/// indicating that the caller should keep their capacity, for example
	329	/// in `std.ArrayList.shrink`.
	330	/// If you need guaranteed success, call `shrink`.
	331	/// If `new_n` is 0, this is the same as `free` and it always succeeds.
	332	pub fn realloc(self: *Allocator, old_mem: anytype, new_n: usize) t: {
	333	const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
	334	break :t Error![]align(Slice.alignment) Slice.child;
	335	} {
	336	const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
	337	return self.reallocAdvancedWithRetAddr(old_mem, old_alignment, new_n, .exact, @returnAddress());
	338	}
	339
	340	pub fn reallocAtLeast(self: *Allocator, old_mem: anytype, new_n: usize) t: {
	341	const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
	342	break :t Error![]align(Slice.alignment) Slice.child;
	343	} {
	344	const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
	345	return self.reallocAdvancedWithRetAddr(old_mem, old_alignment, new_n, .at_least, @returnAddress());
	346	}
	347
	348	/// This is the same as `realloc`, except caller may additionally request
	349	/// a new alignment, which can be larger, smaller, or the same as the old
	350	/// allocation.
	351	pub fn reallocAdvanced(
	352	self: *Allocator,
	353	old_mem: anytype,
	354	comptime new_alignment: u29,
	355	new_n: usize,
	356	exact: Exact,
	357	) Error![]align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
	358	return self.reallocAdvancedWithRetAddr(old_mem, new_alignment, new_n, exact, @returnAddress());
	359	}
	360
	361	pub fn reallocAdvancedWithRetAddr(
	362	self: *Allocator,
	363	old_mem: anytype,
	364	comptime new_alignment: u29,
	365	new_n: usize,
	366	exact: Exact,
	367	return_address: usize,
	368	) Error![]align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
	369	const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
	370	const T = Slice.child;
	371	if (old_mem.len == 0) {
	372	return self.allocAdvanced(T, new_alignment, new_n, exact);
	373	}
	374	if (new_n == 0) {
	375	self.free(old_mem);
	376	return @as([*]align(new_alignment) T, undefined)[0..0];
	377	}
	378
	379	const old_byte_slice = mem.sliceAsBytes(old_mem);
	380	const byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;
	381	// Note: can't set shrunk memory to undefined as memory shouldn't be modified on realloc failure
	382	const len_align: u29 = switch (exact) {
	383	.exact => 0,
	384	.at_least => @sizeOf(T),
	385	};
	386	const new_byte_slice = try self.reallocBytes(old_byte_slice, Slice.alignment, byte_count, new_alignment, len_align, return_address);
	387	return mem.bytesAsSlice(T, @alignCast(new_alignment, new_byte_slice));
	388	}
	389
	390	/// Prefer calling realloc to shrink if you can tolerate failure, such as
	391	/// in an ArrayList data structure with a storage capacity.
	392	/// Shrink always succeeds, and `new_n` must be <= `old_mem.len`.
	393	/// Returned slice has same alignment as old_mem.
	394	/// Shrinking to 0 is the same as calling `free`.
	395	pub fn shrink(self: *Allocator, old_mem: anytype, new_n: usize) t: {
	396	const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
	397	break :t []align(Slice.alignment) Slice.child;
	398	} {
	399	const old_alignment = @typeInfo(@TypeOf(old_mem)).Pointer.alignment;
	400	return self.alignedShrinkWithRetAddr(old_mem, old_alignment, new_n, @returnAddress());
	401	}
	402
	403	/// This is the same as `shrink`, except caller may additionally request
	404	/// a new alignment, which must be smaller or the same as the old
	405	/// allocation.
	406	pub fn alignedShrink(
	407	self: *Allocator,
	408	old_mem: anytype,
	409	comptime new_alignment: u29,
	410	new_n: usize,
	411	) []align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
	412	return self.alignedShrinkWithRetAddr(old_mem, new_alignment, new_n, @returnAddress());
	413	}
	414
	415	/// This is the same as `alignedShrink`, except caller may additionally pass
	416	/// the return address of the first stack frame, which may be relevant for
	417	/// allocators which collect stack traces.
	418	pub fn alignedShrinkWithRetAddr(
	419	self: *Allocator,
	420	old_mem: anytype,
	421	comptime new_alignment: u29,
	422	new_n: usize,
	423	return_address: usize,
	424	) []align(new_alignment) @typeInfo(@TypeOf(old_mem)).Pointer.child {
	425	const Slice = @typeInfo(@TypeOf(old_mem)).Pointer;
	426	const T = Slice.child;
	427
	428	if (new_n == old_mem.len)
	429	return old_mem;
	430	assert(new_n < old_mem.len);
	431	assert(new_alignment <= Slice.alignment);
	432
	433	// Here we skip the overflow checking on the multiplication because
	434	// new_n <= old_mem.len and the multiplication didn't overflow for that operation.
	435	const byte_count = @sizeOf(T) * new_n;
	436
	437	const old_byte_slice = mem.sliceAsBytes(old_mem);
	438	// TODO: https://github.com/ziglang/zig/issues/4298
	439	@memset(old_byte_slice.ptr + byte_count, undefined, old_byte_slice.len - byte_count);
	440	_ = self.shrinkBytes(old_byte_slice, Slice.alignment, byte_count, 0, return_address);
	441	return old_mem[0..new_n];
	442	}
	443
	444	/// Free an array allocated with `alloc`. To free a single item,
	445	/// see `destroy`.
	446	pub fn free(self: *Allocator, memory: anytype) void {
	447	const Slice = @typeInfo(@TypeOf(memory)).Pointer;
	448	const bytes = mem.sliceAsBytes(memory);
	449	const bytes_len = bytes.len + if (Slice.sentinel != null) @sizeOf(Slice.child) else 0;
	450	if (bytes_len == 0) return;
	451	const non_const_ptr = @intToPtr([*]u8, @ptrToInt(bytes.ptr));
	452	// TODO: https://github.com/ziglang/zig/issues/4298
	453	@memset(non_const_ptr, undefined, bytes_len);
	454	_ = self.shrinkBytes(non_const_ptr[0..bytes_len], Slice.alignment, 0, 0, @returnAddress());
	455	}
	456
	457	/// Copies `m` to newly allocated memory. Caller owns the memory.
	458	pub fn dupe(allocator: *Allocator, comptime T: type, m: []const T) ![]T {
	459	const new_buf = try allocator.alloc(T, m.len);
	460	mem.copy(T, new_buf, m);
	461	return new_buf;
	462	}
	463
	464	/// Copies `m` to newly allocated memory, with a null-terminated element. Caller owns the memory.
	465	pub fn dupeZ(allocator: *Allocator, comptime T: type, m: []const T) ![:0]T {
	466	const new_buf = try allocator.alloc(T, m.len + 1);
	467	mem.copy(T, new_buf, m);
	468	new_buf[m.len] = 0;
	469	return new_buf[0..m.len :0];
	470	}
	471
	472	/// Call `resizeFn`, but caller guarantees that `new_len` <= `buf.len` meaning
	473	/// error.OutOfMemory should be impossible.
	474	/// This function allows a runtime `buf_align` value. Callers should generally prefer
	475	/// to call `shrink` directly.
	476	pub fn shrinkBytes(
	477	self: *Allocator,
	478	buf: []u8,
	479	buf_align: u29,
	480	new_len: usize,
	481	len_align: u29,
	482	return_address: usize,
	483	) usize {
	484	assert(new_len <= buf.len);
	485	return self.resizeFn(self, buf, buf_align, new_len, len_align, return_address) catch unreachable;
	486	}

lib/std/mutex.zig+127-143

...	...	@@ -15,8 +15,7 @@ const ResetEvent = std.ResetEvent;
15	15	/// deadlock detection.
16	16	///
17	17	/// Example usage:
18		/// var m = Mutex.init();
19		/// defer m.deinit();
	18	/// var m = Mutex{};
20	19	///
21	20	/// const lock = m.acquire();
22	21	/// defer lock.release();
...	...	@@ -30,141 +29,13 @@ const ResetEvent = std.ResetEvent;
30	29	/// // ... lock not acquired
31	30	/// }
32	31	pub const Mutex = if (builtin.single_threaded)
33		struct {
34		lock: @TypeOf(lock_init),
35
36		const lock_init = if (std.debug.runtime_safety) false else {};
37
38		pub const Held = struct {
39		mutex: *Mutex,
40
41		pub fn release(self: Held) void {
42		if (std.debug.runtime_safety) {
43		self.mutex.lock = false;
44		}
45		}
46		};
47
48		/// Create a new mutex in unlocked state.
49		pub fn init() Mutex {
50		return Mutex{ .lock = lock_init };
51		}
52
53		/// Free a mutex created with init. Calling this while the
54		/// mutex is held is illegal behavior.
55		pub fn deinit(self: *Mutex) void {
56		self.* = undefined;
57		}
58
59		/// Try to acquire the mutex without blocking. Returns null if
60		/// the mutex is unavailable. Otherwise returns Held. Call
61		/// release on Held.
62		pub fn tryAcquire(self: *Mutex) ?Held {
63		if (std.debug.runtime_safety) {
64		if (self.lock) return null;
65		self.lock = true;
66		}
67		return Held{ .mutex = self };
68		}
69
70		/// Acquire the mutex. Will deadlock if the mutex is already
71		/// held by the calling thread.
72		pub fn acquire(self: *Mutex) Held {
73		return self.tryAcquire() orelse @panic("deadlock detected");
74		}
75		}
	32	Dummy
76	33	else if (builtin.os.tag == .windows)
77		// https://locklessinc.com/articles/keyed_events/
78		extern union {
79		locked: u8,
80		waiters: u32,
81
82		const WAKE = 1 << 8;
83		const WAIT = 1 << 9;
84
85		pub fn init() Mutex {
86		return Mutex{ .waiters = 0 };
87		}
88
89		pub fn deinit(self: *Mutex) void {
90		self.* = undefined;
91		}
92
93		pub fn tryAcquire(self: *Mutex) ?Held {
94		if (@atomicRmw(u8, &self.locked, .Xchg, 1, .Acquire) != 0)
95		return null;
96		return Held{ .mutex = self };
97		}
98
99		pub fn acquire(self: *Mutex) Held {
100		return self.tryAcquire() orelse self.acquireSlow();
101		}
102
103		fn acquireSpinning(self: *Mutex) Held {
104		@setCold(true);
105		while (true) : (SpinLock.yield()) {
106		return self.tryAcquire() orelse continue;
107		}
108		}
109
110		fn acquireSlow(self: *Mutex) Held {
111		// try to use NT keyed events for blocking, falling back to spinlock if unavailable
112		@setCold(true);
113		const handle = ResetEvent.OsEvent.Futex.getEventHandle() orelse return self.acquireSpinning();
114		const key = @ptrCast(*const c_void, &self.waiters);
115
116		while (true) : (SpinLock.loopHint(1)) {
117		const waiters = @atomicLoad(u32, &self.waiters, .Monotonic);
118
119		// try and take lock if unlocked
120		if ((waiters & 1) == 0) {
121		if (@atomicRmw(u8, &self.locked, .Xchg, 1, .Acquire) == 0) {
122		return Held{ .mutex = self };
123		}
124
125		// otherwise, try and update the waiting count.
126		// then unset the WAKE bit so that another unlocker can wake up a thread.
127		} else if (@cmpxchgWeak(u32, &self.waiters, waiters, (waiters + WAIT) \| 1, .Monotonic, .Monotonic) == null) {
128		const rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, null);
129		assert(rc == .SUCCESS);
130		_ = @atomicRmw(u32, &self.waiters, .Sub, WAKE, .Monotonic);
131		}
132		}
133		}
134
135		pub const Held = struct {
136		mutex: *Mutex,
137
138		pub fn release(self: Held) void {
139		// unlock without a rmw/cmpxchg instruction
140		@atomicStore(u8, @ptrCast(*u8, &self.mutex.locked), 0, .Release);
141		const handle = ResetEvent.OsEvent.Futex.getEventHandle() orelse return;
142		const key = @ptrCast(*const c_void, &self.mutex.waiters);
143
144		while (true) : (SpinLock.loopHint(1)) {
145		const waiters = @atomicLoad(u32, &self.mutex.waiters, .Monotonic);
146
147		// no one is waiting
148		if (waiters < WAIT) return;
149		// someone grabbed the lock and will do the wake instead
150		if (waiters & 1 != 0) return;
151		// someone else is currently waking up
152		if (waiters & WAKE != 0) return;
153
154		// try to decrease the waiter count & set the WAKE bit meaning a thread is waking up
155		if (@cmpxchgWeak(u32, &self.mutex.waiters, waiters, waiters - WAIT + WAKE, .Release, .Monotonic) == null) {
156		const rc = windows.ntdll.NtReleaseKeyedEvent(handle, key, windows.FALSE, null);
157		assert(rc == .SUCCESS);
158		return;
159		}
160		}
161		}
162		};
163		}
	34	WindowsMutex
164	35	else if (builtin.link_libc or builtin.os.tag == .linux)
165	36	// stack-based version of https://github.com/Amanieu/parking_lot/blob/master/core/src/word_lock.rs
166	37	struct {
167		state: usize,
	38	state: usize = 0,
168	39
169	40	/// number of times to spin trying to acquire the lock.
170	41	/// https://webkit.org/blog/6161/locking-in-webkit/
...	...	@@ -179,14 +50,6 @@ else if (builtin.link_libc or builtin.os.tag == .linux)
179	50	event: ResetEvent,
180	51	};
181	52
182		pub fn init() Mutex {
183		return Mutex{ .state = 0 };
184		}
185
186		pub fn deinit(self: *Mutex) void {
187		self.* = undefined;
188		}
189
190	53	pub fn tryAcquire(self: *Mutex) ?Held {
191	54	if (@cmpxchgWeak(usize, &self.state, 0, MUTEX_LOCK, .Acquire, .Monotonic) != null)
192	55	return null;
...	...	@@ -298,6 +161,128 @@ else if (builtin.link_libc or builtin.os.tag == .linux)
298	161	else
299	162	SpinLock;
300	163
	164	/// This has the sematics as `Mutex`, however it does not actually do any
	165	/// synchronization. Operations are safety-checked no-ops.
	166	pub const Dummy = struct {
	167	lock: @TypeOf(lock_init) = lock_init,
	168
	169	const lock_init = if (std.debug.runtime_safety) false else {};
	170
	171	pub const Held = struct {
	172	mutex: *Dummy,
	173
	174	pub fn release(self: Held) void {
	175	if (std.debug.runtime_safety) {
	176	self.mutex.lock = false;
	177	}
	178	}
	179	};
	180
	181	/// Create a new mutex in unlocked state.
	182	pub const init = Dummy{};
	183
	184	/// Try to acquire the mutex without blocking. Returns null if
	185	/// the mutex is unavailable. Otherwise returns Held. Call
	186	/// release on Held.
	187	pub fn tryAcquire(self: *Dummy) ?Held {
	188	if (std.debug.runtime_safety) {
	189	if (self.lock) return null;
	190	self.lock = true;
	191	}
	192	return Held{ .mutex = self };
	193	}
	194
	195	/// Acquire the mutex. Will deadlock if the mutex is already
	196	/// held by the calling thread.
	197	pub fn acquire(self: *Dummy) Held {
	198	return self.tryAcquire() orelse @panic("deadlock detected");
	199	}
	200	};
	201
	202	// https://locklessinc.com/articles/keyed_events/
	203	const WindowsMutex = struct {
	204	state: State = State{ .waiters = 0 },
	205
	206	const State = extern union {
	207	locked: u8,
	208	waiters: u32,
	209	};
	210
	211	const WAKE = 1 << 8;
	212	const WAIT = 1 << 9;
	213
	214	pub fn tryAcquire(self: *WindowsMutex) ?Held {
	215	if (@atomicRmw(u8, &self.state.locked, .Xchg, 1, .Acquire) != 0)
	216	return null;
	217	return Held{ .mutex = self };
	218	}
	219
	220	pub fn acquire(self: *WindowsMutex) Held {
	221	return self.tryAcquire() orelse self.acquireSlow();
	222	}
	223
	224	fn acquireSpinning(self: *WindowsMutex) Held {
	225	@setCold(true);
	226	while (true) : (SpinLock.yield()) {
	227	return self.tryAcquire() orelse continue;
	228	}
	229	}
	230
	231	fn acquireSlow(self: *WindowsMutex) Held {
	232	// try to use NT keyed events for blocking, falling back to spinlock if unavailable
	233	@setCold(true);
	234	const handle = ResetEvent.OsEvent.Futex.getEventHandle() orelse return self.acquireSpinning();
	235	const key = @ptrCast(*const c_void, &self.state.waiters);
	236
	237	while (true) : (SpinLock.loopHint(1)) {
	238	const waiters = @atomicLoad(u32, &self.state.waiters, .Monotonic);
	239
	240	// try and take lock if unlocked
	241	if ((waiters & 1) == 0) {
	242	if (@atomicRmw(u8, &self.state.locked, .Xchg, 1, .Acquire) == 0) {
	243	return Held{ .mutex = self };
	244	}
	245
	246	// otherwise, try and update the waiting count.
	247	// then unset the WAKE bit so that another unlocker can wake up a thread.
	248	} else if (@cmpxchgWeak(u32, &self.state.waiters, waiters, (waiters + WAIT) \| 1, .Monotonic, .Monotonic) == null) {
	249	const rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, null);
	250	assert(rc == .SUCCESS);
	251	_ = @atomicRmw(u32, &self.state.waiters, .Sub, WAKE, .Monotonic);
	252	}
	253	}
	254	}
	255
	256	pub const Held = struct {
	257	mutex: *WindowsMutex,
	258
	259	pub fn release(self: Held) void {
	260	// unlock without a rmw/cmpxchg instruction
	261	@atomicStore(u8, @ptrCast(*u8, &self.mutex.state.locked), 0, .Release);
	262	const handle = ResetEvent.OsEvent.Futex.getEventHandle() orelse return;
	263	const key = @ptrCast(*const c_void, &self.mutex.state.waiters);
	264
	265	while (true) : (SpinLock.loopHint(1)) {
	266	const waiters = @atomicLoad(u32, &self.mutex.state.waiters, .Monotonic);
	267
	268	// no one is waiting
	269	if (waiters < WAIT) return;
	270	// someone grabbed the lock and will do the wake instead
	271	if (waiters & 1 != 0) return;
	272	// someone else is currently waking up
	273	if (waiters & WAKE != 0) return;
	274
	275	// try to decrease the waiter count & set the WAKE bit meaning a thread is waking up
	276	if (@cmpxchgWeak(u32, &self.mutex.state.waiters, waiters, waiters - WAIT + WAKE, .Release, .Monotonic) == null) {
	277	const rc = windows.ntdll.NtReleaseKeyedEvent(handle, key, windows.FALSE, null);
	278	assert(rc == .SUCCESS);
	279	return;
	280	}
	281	}
	282	}
	283	};
	284	};
	285
301	286	const TestContext = struct {
302	287	mutex: *Mutex,
303	288	data: i128,
...	...	@@ -306,8 +291,7 @@ const TestContext = struct {
306	291	};
307	292
308	293	test "std.Mutex" {
309		var mutex = Mutex.init();
310		defer mutex.deinit();
	294	var mutex = Mutex{};
311	295
312	296	var context = TestContext{
313	297	.mutex = &mutex,

lib/std/once.zig+1-1

...	...	@@ -10,7 +10,7 @@ pub fn once(comptime f: fn () void) Once(f) {
10	10	pub fn Once(comptime f: fn () void) type {
11	11	return struct {
12	12	done: bool = false,
13		mutex: std.Mutex = std.Mutex.init(),
	13	mutex: std.Mutex = std.Mutex{},
14	14
15	15	/// Call the function `f`.
16	16	/// If `call` is invoked multiple times `f` will be executed only the

lib/std/special/test_runner.zig+17-11

...	...	@@ -19,15 +19,21 @@ pub fn main() anyerror!void {
19	19	// ignores the alignment of the slice.
20	20	async_frame_buffer = &[_]u8{};
21	21
	22	var leaks: usize = 0;
22	23	for (test_fn_list) \|test_fn, i\| {
23		std.testing.base_allocator_instance.reset();
	24	std.testing.allocator_instance = std.heap.GeneralPurposeAllocator(.{}){};
	25	defer {
	26	if (std.testing.allocator_instance.deinit()) {
	27	leaks += 1;
	28	}
	29	}
24	30	std.testing.log_level = .warn;
25	31
26	32	var test_node = root_node.start(test_fn.name, null);
27	33	test_node.activate();
28	34	progress.refresh();
29	35	if (progress.terminal == null) {
30		std.debug.warn("{}/{} {}...", .{ i + 1, test_fn_list.len, test_fn.name });
	36	std.debug.print("{}/{} {}...", .{ i + 1, test_fn_list.len, test_fn.name });
31	37	}
32	38	const result = if (test_fn.async_frame_size) \|size\| switch (io_mode) {
33	39	.evented => blk: {
...	...	@@ -42,24 +48,20 @@ pub fn main() anyerror!void {
42	48	skip_count += 1;
43	49	test_node.end();
44	50	progress.log("{}...SKIP (async test)\n", .{test_fn.name});
45		if (progress.terminal == null) std.debug.warn("SKIP (async test)\n", .{});
	51	if (progress.terminal == null) std.debug.print("SKIP (async test)\n", .{});
46	52	continue;
47	53	},
48	54	} else test_fn.func();
49	55	if (result) \|_\| {
50	56	ok_count += 1;
51	57	test_node.end();
52		std.testing.allocator_instance.validate() catch \|err\| switch (err) {
53		error.Leak => std.debug.panic("", .{}),
54		else => std.debug.panic("error.{}", .{@errorName(err)}),
55		};
56		if (progress.terminal == null) std.debug.warn("OK\n", .{});
	58	if (progress.terminal == null) std.debug.print("OK\n", .{});
57	59	} else \|err\| switch (err) {
58	60	error.SkipZigTest => {
59	61	skip_count += 1;
60	62	test_node.end();
61	63	progress.log("{}...SKIP\n", .{test_fn.name});
62		if (progress.terminal == null) std.debug.warn("SKIP\n", .{});
	64	if (progress.terminal == null) std.debug.print("SKIP\n", .{});
63	65	},
64	66	else => {
65	67	progress.log("", .{});
...	...	@@ -69,9 +71,13 @@ pub fn main() anyerror!void {
69	71	}
70	72	root_node.end();
71	73	if (ok_count == test_fn_list.len) {
72		std.debug.warn("All {} tests passed.\n", .{ok_count});
	74	std.debug.print("All {} tests passed.\n", .{ok_count});
73	75	} else {
74		std.debug.warn("{} passed; {} skipped.\n", .{ ok_count, skip_count });
	76	std.debug.print("{} passed; {} skipped.\n", .{ ok_count, skip_count });
	77	}
	78	if (leaks != 0) {
	79	std.debug.print("{} tests leaked memory\n", .{ok_count});
	80	std.process.exit(1);
75	81	}
76	82	}
77	83

lib/std/std.zig+2-1

...	...	@@ -13,7 +13,8 @@ pub const ComptimeStringMap = @import("comptime_string_map.zig").ComptimeStringM
13	13	pub const DynLib = @import("dynamic_library.zig").DynLib;
14	14	pub const HashMap = hash_map.HashMap;
15	15	pub const HashMapUnmanaged = hash_map.HashMapUnmanaged;
16		pub const Mutex = @import("mutex.zig").Mutex;
	16	pub const mutex = @import("mutex.zig");
	17	pub const Mutex = mutex.Mutex;
17	18	pub const PackedIntArray = @import("packed_int_array.zig").PackedIntArray;
18	19	pub const PackedIntArrayEndian = @import("packed_int_array.zig").PackedIntArrayEndian;
19	20	pub const PackedIntSlice = @import("packed_int_array.zig").PackedIntSlice;

lib/std/testing.zig+14-16

...	...	@@ -1,18 +1,16 @@
1	1	const std = @import("std.zig");
2		const warn = std.debug.warn;
	2	const print = std.debug.print;
3	3
4		pub const LeakCountAllocator = @import("testing/leak_count_allocator.zig").LeakCountAllocator;
5	4	pub const FailingAllocator = @import("testing/failing_allocator.zig").FailingAllocator;
6	5
7	6	/// This should only be used in temporary test programs.
8	7	pub const allocator = &allocator_instance.allocator;
9		pub var allocator_instance = LeakCountAllocator.init(&base_allocator_instance.allocator);
	8	pub var allocator_instance = std.heap.GeneralPurposeAllocator(.{}){};
10	9
11	10	pub const failing_allocator = &failing_allocator_instance.allocator;
12	11	pub var failing_allocator_instance = FailingAllocator.init(&base_allocator_instance.allocator, 0);
13	12
14		pub var base_allocator_instance = std.mem.validationWrap(std.heap.ThreadSafeFixedBufferAllocator.init(allocator_mem[0..]));
15		var allocator_mem: [2 * 1024 * 1024]u8 = undefined;
	13	pub var base_allocator_instance = std.heap.FixedBufferAllocator.init("");
16	14
17	15	/// TODO https://github.com/ziglang/zig/issues/5738
18	16	pub var log_level = std.log.Level.warn;
...	...	@@ -326,22 +324,22 @@ test "expectEqual vector" {
326	324
327	325	pub fn expectEqualStrings(expected: []const u8, actual: []const u8) void {
328	326	if (std.mem.indexOfDiff(u8, actual, expected)) \|diff_index\| {
329		warn("\n====== expected this output: =========\n", .{});
	327	print("\n====== expected this output: =========\n", .{});
330	328	printWithVisibleNewlines(expected);
331		warn("\n======== instead found this: =========\n", .{});
	329	print("\n======== instead found this: =========\n", .{});
332	330	printWithVisibleNewlines(actual);
333		warn("\n======================================\n", .{});
	331	print("\n======================================\n", .{});
334	332
335	333	var diff_line_number: usize = 1;
336	334	for (expected[0..diff_index]) \|value\| {
337	335	if (value == '\n') diff_line_number += 1;
338	336	}
339		warn("First difference occurs on line {}:\n", .{diff_line_number});
	337	print("First difference occurs on line {}:\n", .{diff_line_number});
340	338
341		warn("expected:\n", .{});
	339	print("expected:\n", .{});
342	340	printIndicatorLine(expected, diff_index);
343	341
344		warn("found:\n", .{});
	342	print("found:\n", .{});
345	343	printIndicatorLine(actual, diff_index);
346	344
347	345	@panic("test failure");
...	...	@@ -362,9 +360,9 @@ fn printIndicatorLine(source: []const u8, indicator_index: usize) void {
362	360	{
363	361	var i: usize = line_begin_index;
364	362	while (i < indicator_index) : (i += 1)
365		warn(" ", .{});
	363	print(" ", .{});
366	364	}
367		warn("^\n", .{});
	365	print("^\n", .{});
368	366	}
369	367
370	368	fn printWithVisibleNewlines(source: []const u8) void {
...	...	@@ -372,15 +370,15 @@ fn printWithVisibleNewlines(source: []const u8) void {
372	370	while (std.mem.indexOf(u8, source[i..], "\n")) \|nl\| : (i += nl + 1) {
373	371	printLine(source[i .. i + nl]);
374	372	}
375		warn("{}␃\n", .{source[i..]}); // End of Text symbol (ETX)
	373	print("{}␃\n", .{source[i..]}); // End of Text symbol (ETX)
376	374	}
377	375
378	376	fn printLine(line: []const u8) void {
379	377	if (line.len != 0) switch (line[line.len - 1]) {
380		' ', '\t' => warn("{}⏎\n", .{line}), // Carriage return symbol,
	378	' ', '\t' => print("{}⏎\n", .{line}), // Carriage return symbol,
381	379	else => {},
382	380	};
383		warn("{}\n", .{line});
	381	print("{}\n", .{line});
384	382	}
385	383
386	384	test "" {

lib/std/testing/failing_allocator.zig+17-4

...	...	@@ -45,21 +45,34 @@ pub const FailingAllocator = struct {
45	45	};
46	46	}
47	47
48		fn alloc(allocator: *std.mem.Allocator, len: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {
	48	fn alloc(
	49	allocator: *std.mem.Allocator,
	50	len: usize,
	51	ptr_align: u29,
	52	len_align: u29,
	53	return_address: usize,
	54	) error{OutOfMemory}![]u8 {
49	55	const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);
50	56	if (self.index == self.fail_index) {
51	57	return error.OutOfMemory;
52	58	}
53		const result = try self.internal_allocator.callAllocFn(len, ptr_align, len_align);
	59	const result = try self.internal_allocator.allocFn(self.internal_allocator, len, ptr_align, len_align, return_address);
54	60	self.allocated_bytes += result.len;
55	61	self.allocations += 1;
56	62	self.index += 1;
57	63	return result;
58	64	}
59	65
60		fn resize(allocator: *std.mem.Allocator, old_mem: []u8, new_len: usize, len_align: u29) error{OutOfMemory}!usize {
	66	fn resize(
	67	allocator: *std.mem.Allocator,
	68	old_mem: []u8,
	69	old_align: u29,
	70	new_len: usize,
	71	len_align: u29,
	72	ra: usize,
	73	) error{OutOfMemory}!usize {
61	74	const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);
62		const r = self.internal_allocator.callResizeFn(old_mem, new_len, len_align) catch \|e\| {
	75	const r = self.internal_allocator.resizeFn(self.internal_allocator, old_mem, old_align, new_len, len_align, ra) catch \|e\| {
63	76	std.debug.assert(new_len > old_mem.len);
64	77	return e;
65	78	};

lib/std/testing/leak_count_allocator.zig deleted-51

...	...	@@ -1,51 +0,0 @@
1		const std = @import("../std.zig");
2
3		/// This allocator is used in front of another allocator and counts the numbers of allocs and frees.
4		/// The test runner asserts every alloc has a corresponding free at the end of each test.
5		///
6		/// The detection algorithm is incredibly primitive and only accounts for number of calls.
7		/// This should be replaced by the general purpose debug allocator.
8		pub const LeakCountAllocator = struct {
9		count: usize,
10		allocator: std.mem.Allocator,
11		internal_allocator: *std.mem.Allocator,
12
13		pub fn init(allocator: *std.mem.Allocator) LeakCountAllocator {
14		return .{
15		.count = 0,
16		.allocator = .{
17		.allocFn = alloc,
18		.resizeFn = resize,
19		},
20		.internal_allocator = allocator,
21		};
22		}
23
24		fn alloc(allocator: *std.mem.Allocator, len: usize, ptr_align: u29, len_align: u29) error{OutOfMemory}![]u8 {
25		const self = @fieldParentPtr(LeakCountAllocator, "allocator", allocator);
26		const ptr = try self.internal_allocator.callAllocFn(len, ptr_align, len_align);
27		self.count += 1;
28		return ptr;
29		}
30
31		fn resize(allocator: *std.mem.Allocator, old_mem: []u8, new_size: usize, len_align: u29) error{OutOfMemory}!usize {
32		const self = @fieldParentPtr(LeakCountAllocator, "allocator", allocator);
33		if (new_size == 0) {
34		if (self.count == 0) {
35		std.debug.panic("error - too many calls to free, most likely double free", .{});
36		}
37		self.count -= 1;
38		}
39		return self.internal_allocator.callResizeFn(old_mem, new_size, len_align) catch \|e\| {
40		std.debug.assert(new_size > old_mem.len);
41		return e;
42		};
43		}
44
45		pub fn validate(self: LeakCountAllocator) !void {
46		if (self.count > 0) {
47		std.debug.warn("error - detected leaked allocations without matching free: {}\n", .{self.count});
48		return error.Leak;
49		}
50		}
51		};

src-self-hosted/link.zig+6-6

...	...	@@ -325,17 +325,17 @@ pub const File = struct {
325	325	/// local symbols, they cannot be mixed. So we must buffer all the global symbols and
326	326	/// write them at the end. These are only the local symbols. The length of this array
327	327	/// is the value used for sh_info in the .symtab section.
328		local_symbols: std.ArrayListUnmanaged(elf.Elf64_Sym) = std.ArrayListUnmanaged(elf.Elf64_Sym){},
329		global_symbols: std.ArrayListUnmanaged(elf.Elf64_Sym) = std.ArrayListUnmanaged(elf.Elf64_Sym){},
	328	local_symbols: std.ArrayListUnmanaged(elf.Elf64_Sym) = .{},
	329	global_symbols: std.ArrayListUnmanaged(elf.Elf64_Sym) = .{},
330	330
331		local_symbol_free_list: std.ArrayListUnmanaged(u32) = std.ArrayListUnmanaged(u32){},
332		global_symbol_free_list: std.ArrayListUnmanaged(u32) = std.ArrayListUnmanaged(u32){},
333		offset_table_free_list: std.ArrayListUnmanaged(u32) = std.ArrayListUnmanaged(u32){},
	331	local_symbol_free_list: std.ArrayListUnmanaged(u32) = .{},
	332	global_symbol_free_list: std.ArrayListUnmanaged(u32) = .{},
	333	offset_table_free_list: std.ArrayListUnmanaged(u32) = .{},
334	334
335	335	/// Same order as in the file. The value is the absolute vaddr value.
336	336	/// If the vaddr of the executable program header changes, the entire
337	337	/// offset table needs to be rewritten.
338		offset_table: std.ArrayListUnmanaged(u64) = std.ArrayListUnmanaged(u64){},
	338	offset_table: std.ArrayListUnmanaged(u64) = .{},
339	339
340	340	phdr_table_dirty: bool = false,
341	341	shdr_table_dirty: bool = false,

src-self-hosted/main.zig+3-2

...	...	@@ -60,9 +60,10 @@ pub fn log(
60	60	std.debug.print(prefix ++ format, args);
61	61	}
62	62
	63	var general_purpose_allocator = std.heap.GeneralPurposeAllocator(.{}){};
	64
63	65	pub fn main() !void {
64		// TODO general purpose allocator in the zig std lib
65		const gpa = if (std.builtin.link_libc) std.heap.c_allocator else std.heap.page_allocator;
	66	const gpa = if (std.builtin.link_libc) std.heap.c_allocator else &general_purpose_allocator.allocator;
66	67	var arena_instance = std.heap.ArenaAllocator.init(gpa);
67	68	defer arena_instance.deinit();
68	69	const arena = &arena_instance.allocator;

src-self-hosted/test.zig-3

...	...	@@ -407,8 +407,6 @@ pub const TestContext = struct {
407	407	defer root_node.end();
408	408
409	409	for (self.cases.items) \|case\| {
410		std.testing.base_allocator_instance.reset();
411
412	410	var prg_node = root_node.start(case.name, case.updates.items.len);
413	411	prg_node.activate();
414	412	defer prg_node.end();
...	...	@@ -419,7 +417,6 @@ pub const TestContext = struct {
419	417	progress.refresh_rate_ns = 0;
420	418
421	419	try self.runOneCase(std.testing.allocator, &prg_node, case);
422		try std.testing.allocator_instance.validate();
423	420	}
424	421	}
425	422

src/codegen.cpp+6

...	...	@@ -5886,6 +5886,12 @@ static LLVMValueRef ir_render_breakpoint(CodeGen g, IrExecutableGen executable
5886	5886	static LLVMValueRef ir_render_return_address(CodeGen g, IrExecutableGen executable,
5887	5887	IrInstGenReturnAddress *instruction)
5888	5888	{
	5889	if (target_is_wasm(g->zig_target) && g->zig_target->os != OsEmscripten) {
	5890	// I got this error from LLVM 10:
	5891	// "Non-Emscripten WebAssembly hasn't implemented __builtin_return_address"
	5892	return LLVMConstNull(get_llvm_type(g, instruction->base.value->type));
	5893	}
	5894
5889	5895	LLVMValueRef zero = LLVMConstNull(g->builtin_types.entry_i32->llvm_type);
5890	5896	LLVMValueRef ptr_val = LLVMBuildCall(g->builder, get_return_address_fn_val(g), &zero, 1, "");
5891	5897	return LLVMBuildPtrToInt(g->builder, ptr_val, g->builtin_types.entry_usize->llvm_type, "");

src/ir.cpp+26-14

...	...	@@ -25067,12 +25067,12 @@ static PtrLen size_enum_index_to_ptr_len(BuiltinPtrSize size_enum_index) {
25067	25067	zig_unreachable();
25068	25068	}
25069	25069
25070		static ZigValue create_ptr_like_type_info(IrAnalyze ira, ZigType *ptr_type_entry) {
25071		Error err;
	25070	static ZigValue create_ptr_like_type_info(IrAnalyze ira, IrInst source_instr, ZigType ptr_type_entry) {
25072	25071	ZigType *attrs_type;
25073	25072	BuiltinPtrSize size_enum_index;
25074	25073	if (is_slice(ptr_type_entry)) {
25075		attrs_type = ptr_type_entry->data.structure.fields[slice_ptr_index]->type_entry;
	25074	TypeStructField *ptr_field = ptr_type_entry->data.structure.fields[slice_ptr_index];
	25075	attrs_type = resolve_struct_field_type(ira->codegen, ptr_field);
25076	25076	size_enum_index = BuiltinPtrSizeSlice;
25077	25077	} else if (ptr_type_entry->id == ZigTypeIdPointer) {
25078	25078	attrs_type = ptr_type_entry;
...	...	@@ -25081,9 +25081,6 @@ static ZigValue create_ptr_like_type_info(IrAnalyze ira, ZigType *ptr_type_ent
25081	25081	zig_unreachable();
25082	25082	}
25083	25083
25084		if ((err = type_resolve(ira->codegen, attrs_type->data.pointer.child_type, ResolveStatusSizeKnown)))
25085		return nullptr;
25086
25087	25084	ZigType *type_info_pointer_type = ir_type_info_get_type(ira, "Pointer", nullptr);
25088	25085	assertNoError(type_resolve(ira->codegen, type_info_pointer_type, ResolveStatusSizeKnown));
25089	25086
...	...	@@ -25114,9 +25111,18 @@ static ZigValue create_ptr_like_type_info(IrAnalyze ira, ZigType *ptr_type_ent
25114	25111	fields[2]->data.x_bool = attrs_type->data.pointer.is_volatile;
25115	25112	// alignment: u32
25116	25113	ensure_field_index(result->type, "alignment", 3);
25117		fields[3]->special = ConstValSpecialStatic;
25118	25114	fields[3]->type = ira->codegen->builtin_types.entry_num_lit_int;
25119		bigint_init_unsigned(&fields[3]->data.x_bigint, get_ptr_align(ira->codegen, attrs_type));
	25115	if (attrs_type->data.pointer.explicit_alignment != 0) {
	25116	fields[3]->special = ConstValSpecialStatic;
	25117	bigint_init_unsigned(&fields[3]->data.x_bigint, attrs_type->data.pointer.explicit_alignment);
	25118	} else {
	25119	LazyValueAlignOf *lazy_align_of = heap::c_allocator.create<LazyValueAlignOf>();
	25120	lazy_align_of->ira = ira; ira_ref(ira);
	25121	fields[3]->special = ConstValSpecialLazy;
	25122	fields[3]->data.x_lazy = &lazy_align_of->base;
	25123	lazy_align_of->base.id = LazyValueIdAlignOf;
	25124	lazy_align_of->target_type = ir_const_type(ira, source_instr, attrs_type->data.pointer.child_type);
	25125	}
25120	25126	// child: type
25121	25127	ensure_field_index(result->type, "child", 4);
25122	25128	fields[4]->special = ConstValSpecialStatic;
...	...	@@ -25130,7 +25136,7 @@ static ZigValue create_ptr_like_type_info(IrAnalyze ira, ZigType *ptr_type_ent
25130	25136	// sentinel: anytype
25131	25137	ensure_field_index(result->type, "sentinel", 6);
25132	25138	fields[6]->special = ConstValSpecialStatic;
25133		if (attrs_type->data.pointer.child_type->id != ZigTypeIdOpaque) {
	25139	if (attrs_type->data.pointer.sentinel != nullptr) {
25134	25140	fields[6]->type = get_optional_type(ira->codegen, attrs_type->data.pointer.child_type);
25135	25141	set_optional_payload(fields[6], attrs_type->data.pointer.sentinel);
25136	25142	} else {
...	...	@@ -25165,9 +25171,6 @@ static Error ir_make_type_info_value(IrAnalyze ira, IrInst source_instr, ZigTy
25165	25171	assert(type_entry != nullptr);
25166	25172	assert(!type_is_invalid(type_entry));
25167	25173
25168		if ((err = type_resolve(ira->codegen, type_entry, ResolveStatusSizeKnown)))
25169		return err;
25170
25171	25174	auto entry = ira->codegen->type_info_cache.maybe_get(type_entry);
25172	25175	if (entry != nullptr) {
25173	25176	*out = entry->value;
...	...	@@ -25231,7 +25234,7 @@ static Error ir_make_type_info_value(IrAnalyze ira, IrInst source_instr, ZigTy
25231	25234	}
25232	25235	case ZigTypeIdPointer:
25233	25236	{
25234		result = create_ptr_like_type_info(ira, type_entry);
	25237	result = create_ptr_like_type_info(ira, source_instr, type_entry);
25235	25238	if (result == nullptr)
25236	25239	return ErrorSemanticAnalyzeFail;
25237	25240	break;
...	...	@@ -25317,6 +25320,9 @@ static Error ir_make_type_info_value(IrAnalyze ira, IrInst source_instr, ZigTy
25317	25320	}
25318	25321	case ZigTypeIdEnum:
25319	25322	{
	25323	if ((err = type_resolve(ira->codegen, type_entry, ResolveStatusSizeKnown)))
	25324	return err;
	25325
25320	25326	result = ira->codegen->pass1_arena->create<ZigValue>();
25321	25327	result->special = ConstValSpecialStatic;
25322	25328	result->type = ir_type_info_get_type(ira, "Enum", nullptr);
...	...	@@ -25455,6 +25461,9 @@ static Error ir_make_type_info_value(IrAnalyze ira, IrInst source_instr, ZigTy
25455	25461	}
25456	25462	case ZigTypeIdUnion:
25457	25463	{
	25464	if ((err = type_resolve(ira->codegen, type_entry, ResolveStatusSizeKnown)))
	25465	return err;
	25466
25458	25467	result = ira->codegen->pass1_arena->create<ZigValue>();
25459	25468	result->special = ConstValSpecialStatic;
25460	25469	result->type = ir_type_info_get_type(ira, "Union", nullptr);
...	...	@@ -25545,12 +25554,15 @@ static Error ir_make_type_info_value(IrAnalyze ira, IrInst source_instr, ZigTy
25545	25554	case ZigTypeIdStruct:
25546	25555	{
25547	25556	if (type_entry->data.structure.special == StructSpecialSlice) {
25548		result = create_ptr_like_type_info(ira, type_entry);
	25557	result = create_ptr_like_type_info(ira, source_instr, type_entry);
25549	25558	if (result == nullptr)
25550	25559	return ErrorSemanticAnalyzeFail;
25551	25560	break;
25552	25561	}
25553	25562
	25563	if ((err = type_resolve(ira->codegen, type_entry, ResolveStatusSizeKnown)))
	25564	return err;
	25565
25554	25566	result = ira->codegen->pass1_arena->create<ZigValue>();
25555	25567	result->special = ConstValSpecialStatic;
25556	25568	result->type = ir_type_info_get_type(ira, "Struct", nullptr);