authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-08 21:27:22-08:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-02-08 21:27:22-08:00
log604ed5281c7eef6105a75bd8819399c73570b88a
treed5fd4b29214f00c411ceadc82f553213e5c8e9d2
parentb3a11018ae1fe99190fb6fb7ae82a486c40f6f15
parent42dbd35d3e16247ee68d7e3ace0da3778a1f5d37
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #22823 from ziglang/SmpAllocator-freelist-accounting

std.heap.SmpAllocator: back to simple free implementation

1 files changed, 4 insertions(+), 42 deletions(-)

lib/std/heap/SmpAllocator.zig+4-42
...@@ -51,10 +51,6 @@ const slab_len: usize = @max(std.heap.page_size_max, 64 * 1024);...@@ -51,10 +51,6 @@ const slab_len: usize = @max(std.heap.page_size_max, 64 * 1024);
51/// Because of storing free list pointers, the minimum size class is 3.51/// Because of storing free list pointers, the minimum size class is 3.
52const min_class = math.log2(@sizeOf(usize));52const min_class = math.log2(@sizeOf(usize));
53const size_class_count = math.log2(slab_len) - min_class;53const size_class_count = math.log2(slab_len) - min_class;
54/// When a freelist length exceeds this number, a `free` will rotate up to
55/// `max_free_search` times before pushing.
56const max_freelist_len: u8 = 16;
57const max_free_search = 1;
58/// Before mapping a fresh page, `alloc` will rotate this many times.54/// Before mapping a fresh page, `alloc` will rotate this many times.
59const max_alloc_search = 1;55const max_alloc_search = 1;
6056
...@@ -73,8 +69,6 @@ const Thread = struct {...@@ -73,8 +69,6 @@ const Thread = struct {
73 next_addrs: [size_class_count]usize = @splat(0),69 next_addrs: [size_class_count]usize = @splat(0),
74 /// For each size class, points to the freed pointer.70 /// For each size class, points to the freed pointer.
75 frees: [size_class_count]usize = @splat(0),71 frees: [size_class_count]usize = @splat(0),
76 /// For each size class, tracks the number of items in the freelist.
77 freelist_lens: [size_class_count]u8 = @splat(0),
7872
79 fn lock() *Thread {73 fn lock() *Thread {
80 var index = thread_index;74 var index = thread_index;
...@@ -142,7 +136,6 @@ fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ?...@@ -142,7 +136,6 @@ fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ?
142 defer t.unlock();136 defer t.unlock();
143 const node: *usize = @ptrFromInt(top_free_ptr);137 const node: *usize = @ptrFromInt(top_free_ptr);
144 t.frees[class] = node.*;138 t.frees[class] = node.*;
145 t.freelist_lens[class] -|= 1;
146 return @ptrFromInt(top_free_ptr);139 return @ptrFromInt(top_free_ptr);
147 }140 }
148141
...@@ -160,7 +153,6 @@ fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ?...@@ -160,7 +153,6 @@ fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ?
160 // slab alignment here ensures the % slab len earlier catches the end of slots.153 // slab alignment here ensures the % slab len earlier catches the end of slots.
161 const slab = PageAllocator.map(slab_len, .fromByteUnits(slab_len)) orelse return null;154 const slab = PageAllocator.map(slab_len, .fromByteUnits(slab_len)) orelse return null;
162 t.next_addrs[class] = @intFromPtr(slab) + slot_size;155 t.next_addrs[class] = @intFromPtr(slab) + slot_size;
163 t.freelist_lens[class] = 0;
164 return slab;156 return slab;
165 }157 }
166158
...@@ -214,42 +206,12 @@ fn free(context: *anyopaque, memory: []u8, alignment: mem.Alignment, ra: usize)...@@ -214,42 +206,12 @@ fn free(context: *anyopaque, memory: []u8, alignment: mem.Alignment, ra: usize)
214 }206 }
215207
216 const node: *usize = @alignCast(@ptrCast(memory.ptr));208 const node: *usize = @alignCast(@ptrCast(memory.ptr));
217 var search_count: u8 = 0;
218
219 var t = Thread.lock();
220
221 outer: while (true) {
222 const freelist_len = t.freelist_lens[class];
223 if (freelist_len < max_freelist_len) {
224 @branchHint(.likely);
225 defer t.unlock();
226 node.* = t.frees[class];
227 t.frees[class] = @intFromPtr(node);
228 return;
229 }
230209
231 if (search_count >= max_free_search) {210 const t = Thread.lock();
232 defer t.unlock();211 defer t.unlock();
233 t.freelist_lens[class] = freelist_len +| 1;
234 node.* = t.frees[class];
235 t.frees[class] = @intFromPtr(node);
236 return;
237 }
238212
239 t.unlock();213 node.* = t.frees[class];
240 const cpu_count = getCpuCount();214 t.frees[class] = @intFromPtr(node);
241 assert(cpu_count != 0);
242 var index = thread_index;
243 while (true) {
244 index = (index + 1) % cpu_count;
245 t = &global.threads[index];
246 if (t.mutex.tryLock()) {
247 thread_index = index;
248 search_count += 1;
249 continue :outer;
250 }
251 }
252 }
253}215}
254216
255fn sizeClassIndex(len: usize, alignment: mem.Alignment) usize {217fn sizeClassIndex(len: usize, alignment: mem.Alignment) usize {