authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-11-30 01:46:37-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-11-30 01:46:37-05:00
log71038c42f554da86ee23c9c448d39e457d5818eb
tree5ea8feb7f21d19a618a5699dd1380f7b7a396de6
parente35f297aeb993ec956ae80379ddf7f86069e109b
parent7f063b2c52b5daf55b8b1184502a94d0def4cd22
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #13513 from ziglang/faster-wasm-gpa

WebAssembly-only fast allocator

4 files changed, 679 insertions(+), 340 deletions(-)

lib/std/heap.zig+17-340
......@@ -1,13 +1,13 @@
11const std = @import("std.zig");
22const builtin = @import("builtin");
33const root = @import("root");
4const debug = std.debug;
5const assert = debug.assert;
4const assert = std.debug.assert;
65const testing = std.testing;
76const mem = std.mem;
87const os = std.os;
98const c = std.c;
109const maxInt = std.math.maxInt;
10const Allocator = std.mem.Allocator;
1111
1212pub const LoggingAllocator = @import("heap/logging_allocator.zig").LoggingAllocator;
1313pub const loggingAllocator = @import("heap/logging_allocator.zig").loggingAllocator;
......@@ -16,8 +16,12 @@ pub const LogToWriterAllocator = @import("heap/log_to_writer_allocator.zig").Log
1616pub const logToWriterAllocator = @import("heap/log_to_writer_allocator.zig").logToWriterAllocator;
1717pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator;
1818pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig").GeneralPurposeAllocator;
19pub const WasmAllocator = @import("heap/WasmAllocator.zig");
20pub const WasmPageAllocator = @import("heap/WasmPageAllocator.zig");
21pub const PageAllocator = @import("heap/PageAllocator.zig");
1922
20const Allocator = mem.Allocator;
23/// TODO Utilize this on Windows.
24pub var next_mmap_addr_hint: ?[*]align(mem.page_size) u8 = null;
2125
2226const CAllocator = struct {
2327 comptime {
......@@ -227,303 +231,6 @@ pub fn alignPageAllocLen(full_len: usize, len: usize) usize {
227231 return aligned_len;
228232}
229233
230/// TODO Utilize this on Windows.
231pub var next_mmap_addr_hint: ?[*]align(mem.page_size) u8 = null;
232
233const PageAllocator = struct {
234 const vtable = Allocator.VTable{
235 .alloc = alloc,
236 .resize = resize,
237 .free = free,
238 };
239
240 fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 {
241 _ = ra;
242 _ = log2_align;
243 assert(n > 0);
244 if (n > maxInt(usize) - (mem.page_size - 1)) return null;
245 const aligned_len = mem.alignForward(n, mem.page_size);
246
247 if (builtin.os.tag == .windows) {
248 const w = os.windows;
249 const addr = w.VirtualAlloc(
250 null,
251 aligned_len,
252 w.MEM_COMMIT | w.MEM_RESERVE,
253 w.PAGE_READWRITE,
254 ) catch return null;
255 return @ptrCast([*]align(mem.page_size) u8, @alignCast(mem.page_size, addr));
256 }
257
258 const hint = @atomicLoad(@TypeOf(next_mmap_addr_hint), &next_mmap_addr_hint, .Unordered);
259 const slice = os.mmap(
260 hint,
261 aligned_len,
262 os.PROT.READ | os.PROT.WRITE,
263 os.MAP.PRIVATE | os.MAP.ANONYMOUS,
264 -1,
265 0,
266 ) catch return null;
267 assert(mem.isAligned(@ptrToInt(slice.ptr), mem.page_size));
268 const new_hint = @alignCast(mem.page_size, slice.ptr + aligned_len);
269 _ = @cmpxchgStrong(@TypeOf(next_mmap_addr_hint), &next_mmap_addr_hint, hint, new_hint, .Monotonic, .Monotonic);
270 return slice.ptr;
271 }
272
273 fn resize(
274 _: *anyopaque,
275 buf_unaligned: []u8,
276 log2_buf_align: u8,
277 new_size: usize,
278 return_address: usize,
279 ) bool {
280 _ = log2_buf_align;
281 _ = return_address;
282 const new_size_aligned = mem.alignForward(new_size, mem.page_size);
283
284 if (builtin.os.tag == .windows) {
285 const w = os.windows;
286 if (new_size <= buf_unaligned.len) {
287 const base_addr = @ptrToInt(buf_unaligned.ptr);
288 const old_addr_end = base_addr + buf_unaligned.len;
289 const new_addr_end = mem.alignForward(base_addr + new_size, mem.page_size);
290 if (old_addr_end > new_addr_end) {
291 // For shrinking that is not releasing, we will only
292 // decommit the pages not needed anymore.
293 w.VirtualFree(
294 @intToPtr(*anyopaque, new_addr_end),
295 old_addr_end - new_addr_end,
296 w.MEM_DECOMMIT,
297 );
298 }
299 return true;
300 }
301 const old_size_aligned = mem.alignForward(buf_unaligned.len, mem.page_size);
302 if (new_size_aligned <= old_size_aligned) {
303 return true;
304 }
305 return false;
306 }
307
308 const buf_aligned_len = mem.alignForward(buf_unaligned.len, mem.page_size);
309 if (new_size_aligned == buf_aligned_len)
310 return true;
311
312 if (new_size_aligned < buf_aligned_len) {
313 const ptr = @alignCast(mem.page_size, buf_unaligned.ptr + new_size_aligned);
314 // TODO: if the next_mmap_addr_hint is within the unmapped range, update it
315 os.munmap(ptr[0 .. buf_aligned_len - new_size_aligned]);
316 return true;
317 }
318
319 // TODO: call mremap
320 // TODO: if the next_mmap_addr_hint is within the remapped range, update it
321 return false;
322 }
323
324 fn free(_: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) void {
325 _ = log2_buf_align;
326 _ = return_address;
327
328 if (builtin.os.tag == .windows) {
329 os.windows.VirtualFree(slice.ptr, 0, os.windows.MEM_RELEASE);
330 } else {
331 const buf_aligned_len = mem.alignForward(slice.len, mem.page_size);
332 const ptr = @alignCast(mem.page_size, slice.ptr);
333 os.munmap(ptr[0..buf_aligned_len]);
334 }
335 }
336};
337
338const WasmPageAllocator = struct {
339 comptime {
340 if (!builtin.target.isWasm()) {
341 @compileError("WasmPageAllocator is only available for wasm32 arch");
342 }
343 }
344
345 const vtable = Allocator.VTable{
346 .alloc = alloc,
347 .resize = resize,
348 .free = free,
349 };
350
351 const PageStatus = enum(u1) {
352 used = 0,
353 free = 1,
354
355 pub const none_free: u8 = 0;
356 };
357
358 const FreeBlock = struct {
359 data: []u128,
360
361 const Io = std.packed_int_array.PackedIntIo(u1, .Little);
362
363 fn totalPages(self: FreeBlock) usize {
364 return self.data.len * 128;
365 }
366
367 fn isInitialized(self: FreeBlock) bool {
368 return self.data.len > 0;
369 }
370
371 fn getBit(self: FreeBlock, idx: usize) PageStatus {
372 const bit_offset = 0;
373 return @intToEnum(PageStatus, Io.get(mem.sliceAsBytes(self.data), idx, bit_offset));
374 }
375
376 fn setBits(self: FreeBlock, start_idx: usize, len: usize, val: PageStatus) void {
377 const bit_offset = 0;
378 var i: usize = 0;
379 while (i < len) : (i += 1) {
380 Io.set(mem.sliceAsBytes(self.data), start_idx + i, bit_offset, @enumToInt(val));
381 }
382 }
383
384 // Use '0xFFFFFFFF' as a _missing_ sentinel
385 // This saves ~50 bytes compared to returning a nullable
386
387 // We can guarantee that conventional memory never gets this big,
388 // and wasm32 would not be able to address this memory (32 GB > usize).
389
390 // Revisit if this is settled: https://github.com/ziglang/zig/issues/3806
391 const not_found = std.math.maxInt(usize);
392
393 fn useRecycled(self: FreeBlock, num_pages: usize, log2_align: u8) usize {
394 @setCold(true);
395 for (self.data) |segment, i| {
396 const spills_into_next = @bitCast(i128, segment) < 0;
397 const has_enough_bits = @popCount(segment) >= num_pages;
398
399 if (!spills_into_next and !has_enough_bits) continue;
400
401 var j: usize = i * 128;
402 while (j < (i + 1) * 128) : (j += 1) {
403 var count: usize = 0;
404 while (j + count < self.totalPages() and self.getBit(j + count) == .free) {
405 count += 1;
406 const addr = j * mem.page_size;
407 if (count >= num_pages and mem.isAlignedLog2(addr, log2_align)) {
408 self.setBits(j, num_pages, .used);
409 return j;
410 }
411 }
412 j += count;
413 }
414 }
415 return not_found;
416 }
417
418 fn recycle(self: FreeBlock, start_idx: usize, len: usize) void {
419 self.setBits(start_idx, len, .free);
420 }
421 };
422
423 var _conventional_data = [_]u128{0} ** 16;
424 // Marking `conventional` as const saves ~40 bytes
425 const conventional = FreeBlock{ .data = &_conventional_data };
426 var extended = FreeBlock{ .data = &[_]u128{} };
427
428 fn extendedOffset() usize {
429 return conventional.totalPages();
430 }
431
432 fn nPages(memsize: usize) usize {
433 return mem.alignForward(memsize, mem.page_size) / mem.page_size;
434 }
435
436 fn alloc(_: *anyopaque, len: usize, log2_align: u8, ra: usize) ?[*]u8 {
437 _ = ra;
438 if (len > maxInt(usize) - (mem.page_size - 1)) return null;
439 const page_count = nPages(len);
440 const page_idx = allocPages(page_count, log2_align) catch return null;
441 return @intToPtr([*]u8, page_idx * mem.page_size);
442 }
443
444 fn allocPages(page_count: usize, log2_align: u8) !usize {
445 {
446 const idx = conventional.useRecycled(page_count, log2_align);
447 if (idx != FreeBlock.not_found) {
448 return idx;
449 }
450 }
451
452 const idx = extended.useRecycled(page_count, log2_align);
453 if (idx != FreeBlock.not_found) {
454 return idx + extendedOffset();
455 }
456
457 const next_page_idx = @wasmMemorySize(0);
458 const next_page_addr = next_page_idx * mem.page_size;
459 const aligned_addr = mem.alignForwardLog2(next_page_addr, log2_align);
460 const drop_page_count = @divExact(aligned_addr - next_page_addr, mem.page_size);
461 const result = @wasmMemoryGrow(0, @intCast(u32, drop_page_count + page_count));
462 if (result <= 0)
463 return error.OutOfMemory;
464 assert(result == next_page_idx);
465 const aligned_page_idx = next_page_idx + drop_page_count;
466 if (drop_page_count > 0) {
467 freePages(next_page_idx, aligned_page_idx);
468 }
469 return @intCast(usize, aligned_page_idx);
470 }
471
472 fn freePages(start: usize, end: usize) void {
473 if (start < extendedOffset()) {
474 conventional.recycle(start, @min(extendedOffset(), end) - start);
475 }
476 if (end > extendedOffset()) {
477 var new_end = end;
478 if (!extended.isInitialized()) {
479 // Steal the last page from the memory currently being recycled
480 // TODO: would it be better if we use the first page instead?
481 new_end -= 1;
482
483 extended.data = @intToPtr([*]u128, new_end * mem.page_size)[0 .. mem.page_size / @sizeOf(u128)];
484 // Since this is the first page being freed and we consume it, assume *nothing* is free.
485 mem.set(u128, extended.data, PageStatus.none_free);
486 }
487 const clamped_start = @max(extendedOffset(), start);
488 extended.recycle(clamped_start - extendedOffset(), new_end - clamped_start);
489 }
490 }
491
492 fn resize(
493 _: *anyopaque,
494 buf: []u8,
495 log2_buf_align: u8,
496 new_len: usize,
497 return_address: usize,
498 ) bool {
499 _ = log2_buf_align;
500 _ = return_address;
501 const aligned_len = mem.alignForward(buf.len, mem.page_size);
502 if (new_len > aligned_len) return false;
503 const current_n = nPages(aligned_len);
504 const new_n = nPages(new_len);
505 if (new_n != current_n) {
506 const base = nPages(@ptrToInt(buf.ptr));
507 freePages(base + new_n, base + current_n);
508 }
509 return true;
510 }
511
512 fn free(
513 _: *anyopaque,
514 buf: []u8,
515 log2_buf_align: u8,
516 return_address: usize,
517 ) void {
518 _ = log2_buf_align;
519 _ = return_address;
520 const aligned_len = mem.alignForward(buf.len, mem.page_size);
521 const current_n = nPages(aligned_len);
522 const base = nPages(@ptrToInt(buf.ptr));
523 freePages(base, base + current_n);
524 }
525};
526
527234pub const HeapAllocator = switch (builtin.os.tag) {
528235 .windows => struct {
529236 heap_handle: ?HeapHandle,
......@@ -859,43 +566,6 @@ test "raw_c_allocator" {
859566 }
860567}
861568
862test "WasmPageAllocator internals" {
863 if (comptime builtin.target.isWasm()) {
864 const conventional_memsize = WasmPageAllocator.conventional.totalPages() * mem.page_size;
865 const initial = try page_allocator.alloc(u8, mem.page_size);
866 try testing.expect(@ptrToInt(initial.ptr) < conventional_memsize); // If this isn't conventional, the rest of these tests don't make sense. Also we have a serious memory leak in the test suite.
867
868 var inplace = try page_allocator.realloc(initial, 1);
869 try testing.expectEqual(initial.ptr, inplace.ptr);
870 inplace = try page_allocator.realloc(inplace, 4);
871 try testing.expectEqual(initial.ptr, inplace.ptr);
872 page_allocator.free(inplace);
873
874 const reuse = try page_allocator.alloc(u8, 1);
875 try testing.expectEqual(initial.ptr, reuse.ptr);
876 page_allocator.free(reuse);
877
878 // This segment may span conventional and extended which has really complex rules so we're just ignoring it for now.
879 const padding = try page_allocator.alloc(u8, conventional_memsize);
880 page_allocator.free(padding);
881
882 const extended = try page_allocator.alloc(u8, conventional_memsize);
883 try testing.expect(@ptrToInt(extended.ptr) >= conventional_memsize);
884
885 const use_small = try page_allocator.alloc(u8, 1);
886 try testing.expectEqual(initial.ptr, use_small.ptr);
887 page_allocator.free(use_small);
888
889 inplace = try page_allocator.realloc(extended, 1);
890 try testing.expectEqual(extended.ptr, inplace.ptr);
891 page_allocator.free(inplace);
892
893 const reuse_extended = try page_allocator.alloc(u8, conventional_memsize);
894 try testing.expectEqual(extended.ptr, reuse_extended.ptr);
895 page_allocator.free(reuse_extended);
896 }
897}
898
899569test "PageAllocator" {
900570 const allocator = page_allocator;
901571 try testAllocator(allocator);
......@@ -1163,7 +833,14 @@ pub fn testAllocatorAlignedShrink(base_allocator: mem.Allocator) !void {
1163833 try testing.expect(slice[60] == 0x34);
1164834}
1165835
1166test "heap" {
1167 _ = @import("heap/logging_allocator.zig");
1168 _ = @import("heap/log_to_writer_allocator.zig");
836test {
837 _ = LoggingAllocator;
838 _ = LogToWriterAllocator;
839 _ = ScopedLoggingAllocator;
840 _ = ArenaAllocator;
841 _ = GeneralPurposeAllocator;
842 if (comptime builtin.target.isWasm()) {
843 _ = WasmAllocator;
844 _ = WasmPageAllocator;
845 }
1169846}
lib/std/heap/PageAllocator.zig created+110
......@@ -0,0 +1,110 @@
1const std = @import("../std.zig");
2const builtin = @import("builtin");
3const Allocator = std.mem.Allocator;
4const mem = std.mem;
5const os = std.os;
6const maxInt = std.math.maxInt;
7const assert = std.debug.assert;
8
9pub const vtable = Allocator.VTable{
10 .alloc = alloc,
11 .resize = resize,
12 .free = free,
13};
14
15fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 {
16 _ = ra;
17 _ = log2_align;
18 assert(n > 0);
19 if (n > maxInt(usize) - (mem.page_size - 1)) return null;
20 const aligned_len = mem.alignForward(n, mem.page_size);
21
22 if (builtin.os.tag == .windows) {
23 const w = os.windows;
24 const addr = w.VirtualAlloc(
25 null,
26 aligned_len,
27 w.MEM_COMMIT | w.MEM_RESERVE,
28 w.PAGE_READWRITE,
29 ) catch return null;
30 return @ptrCast([*]align(mem.page_size) u8, @alignCast(mem.page_size, addr));
31 }
32
33 const hint = @atomicLoad(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, .Unordered);
34 const slice = os.mmap(
35 hint,
36 aligned_len,
37 os.PROT.READ | os.PROT.WRITE,
38 os.MAP.PRIVATE | os.MAP.ANONYMOUS,
39 -1,
40 0,
41 ) catch return null;
42 assert(mem.isAligned(@ptrToInt(slice.ptr), mem.page_size));
43 const new_hint = @alignCast(mem.page_size, slice.ptr + aligned_len);
44 _ = @cmpxchgStrong(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, hint, new_hint, .Monotonic, .Monotonic);
45 return slice.ptr;
46}
47
48fn resize(
49 _: *anyopaque,
50 buf_unaligned: []u8,
51 log2_buf_align: u8,
52 new_size: usize,
53 return_address: usize,
54) bool {
55 _ = log2_buf_align;
56 _ = return_address;
57 const new_size_aligned = mem.alignForward(new_size, mem.page_size);
58
59 if (builtin.os.tag == .windows) {
60 const w = os.windows;
61 if (new_size <= buf_unaligned.len) {
62 const base_addr = @ptrToInt(buf_unaligned.ptr);
63 const old_addr_end = base_addr + buf_unaligned.len;
64 const new_addr_end = mem.alignForward(base_addr + new_size, mem.page_size);
65 if (old_addr_end > new_addr_end) {
66 // For shrinking that is not releasing, we will only
67 // decommit the pages not needed anymore.
68 w.VirtualFree(
69 @intToPtr(*anyopaque, new_addr_end),
70 old_addr_end - new_addr_end,
71 w.MEM_DECOMMIT,
72 );
73 }
74 return true;
75 }
76 const old_size_aligned = mem.alignForward(buf_unaligned.len, mem.page_size);
77 if (new_size_aligned <= old_size_aligned) {
78 return true;
79 }
80 return false;
81 }
82
83 const buf_aligned_len = mem.alignForward(buf_unaligned.len, mem.page_size);
84 if (new_size_aligned == buf_aligned_len)
85 return true;
86
87 if (new_size_aligned < buf_aligned_len) {
88 const ptr = @alignCast(mem.page_size, buf_unaligned.ptr + new_size_aligned);
89 // TODO: if the next_mmap_addr_hint is within the unmapped range, update it
90 os.munmap(ptr[0 .. buf_aligned_len - new_size_aligned]);
91 return true;
92 }
93
94 // TODO: call mremap
95 // TODO: if the next_mmap_addr_hint is within the remapped range, update it
96 return false;
97}
98
99fn free(_: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) void {
100 _ = log2_buf_align;
101 _ = return_address;
102
103 if (builtin.os.tag == .windows) {
104 os.windows.VirtualFree(slice.ptr, 0, os.windows.MEM_RELEASE);
105 } else {
106 const buf_aligned_len = mem.alignForward(slice.len, mem.page_size);
107 const ptr = @alignCast(mem.page_size, slice.ptr);
108 os.munmap(ptr[0..buf_aligned_len]);
109 }
110}
lib/std/heap/WasmAllocator.zig created+317
......@@ -0,0 +1,317 @@
1//! This is intended to be merged into GeneralPurposeAllocator at some point.
2
3const std = @import("../std.zig");
4const builtin = @import("builtin");
5const Allocator = std.mem.Allocator;
6const mem = std.mem;
7const assert = std.debug.assert;
8const wasm = std.wasm;
9const math = std.math;
10
11comptime {
12 if (!builtin.target.isWasm()) {
13 @compileError("WasmPageAllocator is only available for wasm32 arch");
14 }
15}
16
17pub const vtable = Allocator.VTable{
18 .alloc = alloc,
19 .resize = resize,
20 .free = free,
21};
22
23pub const Error = Allocator.Error;
24
25const max_usize = math.maxInt(usize);
26const ushift = math.Log2Int(usize);
27const bigpage_size = 64 * 1024;
28const pages_per_bigpage = bigpage_size / wasm.page_size;
29const bigpage_count = max_usize / bigpage_size;
30
31/// Because of storing free list pointers, the minimum size class is 3.
32const min_class = math.log2(math.ceilPowerOfTwoAssert(usize, 1 + @sizeOf(usize)));
33const size_class_count = math.log2(bigpage_size) - min_class;
34/// 0 - 1 bigpage
35/// 1 - 2 bigpages
36/// 2 - 4 bigpages
37/// etc.
38const big_size_class_count = math.log2(bigpage_count);
39
40var next_addrs = [1]usize{0} ** size_class_count;
41/// For each size class, points to the freed pointer.
42var frees = [1]usize{0} ** size_class_count;
43/// For each big size class, points to the freed pointer.
44var big_frees = [1]usize{0} ** big_size_class_count;
45
46fn alloc(ctx: *anyopaque, len: usize, log2_align: u8, return_address: usize) ?[*]u8 {
47 _ = ctx;
48 _ = return_address;
49 // Make room for the freelist next pointer.
50 const alignment = @as(usize, 1) << @intCast(Allocator.Log2Align, log2_align);
51 const actual_len = @max(len +| @sizeOf(usize), alignment);
52 const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return null;
53 const class = math.log2(slot_size) - min_class;
54 if (class < size_class_count) {
55 const addr = a: {
56 const top_free_ptr = frees[class];
57 if (top_free_ptr != 0) {
58 const node = @intToPtr(*usize, top_free_ptr + (slot_size - @sizeOf(usize)));
59 frees[class] = node.*;
60 break :a top_free_ptr;
61 }
62
63 const next_addr = next_addrs[class];
64 if (next_addr % wasm.page_size == 0) {
65 const addr = allocBigPages(1);
66 if (addr == 0) return null;
67 //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{
68 // slot_size, class, addr,
69 //});
70 next_addrs[class] = addr + slot_size;
71 break :a addr;
72 } else {
73 next_addrs[class] = next_addr + slot_size;
74 break :a next_addr;
75 }
76 };
77 return @intToPtr([*]u8, addr);
78 }
79 const bigpages_needed = bigPagesNeeded(actual_len);
80 const addr = allocBigPages(bigpages_needed);
81 return @intToPtr([*]u8, addr);
82}
83
84fn resize(
85 ctx: *anyopaque,
86 buf: []u8,
87 log2_buf_align: u8,
88 new_len: usize,
89 return_address: usize,
90) bool {
91 _ = ctx;
92 _ = return_address;
93 // We don't want to move anything from one size class to another, but we
94 // can recover bytes in between powers of two.
95 const buf_align = @as(usize, 1) << @intCast(Allocator.Log2Align, log2_buf_align);
96 const old_actual_len = @max(buf.len + @sizeOf(usize), buf_align);
97 const new_actual_len = @max(new_len +| @sizeOf(usize), buf_align);
98 const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_actual_len);
99 const old_small_class = math.log2(old_small_slot_size) - min_class;
100 if (old_small_class < size_class_count) {
101 const new_small_slot_size = math.ceilPowerOfTwo(usize, new_actual_len) catch return false;
102 return old_small_slot_size == new_small_slot_size;
103 } else {
104 const old_bigpages_needed = bigPagesNeeded(old_actual_len);
105 const old_big_slot_pages = math.ceilPowerOfTwoAssert(usize, old_bigpages_needed);
106 const new_bigpages_needed = bigPagesNeeded(new_actual_len);
107 const new_big_slot_pages = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return false;
108 return old_big_slot_pages == new_big_slot_pages;
109 }
110}
111
112fn free(
113 ctx: *anyopaque,
114 buf: []u8,
115 log2_buf_align: u8,
116 return_address: usize,
117) void {
118 _ = ctx;
119 _ = return_address;
120 const buf_align = @as(usize, 1) << @intCast(Allocator.Log2Align, log2_buf_align);
121 const actual_len = @max(buf.len + @sizeOf(usize), buf_align);
122 const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len);
123 const class = math.log2(slot_size) - min_class;
124 const addr = @ptrToInt(buf.ptr);
125 if (class < size_class_count) {
126 const node = @intToPtr(*usize, addr + (slot_size - @sizeOf(usize)));
127 node.* = frees[class];
128 frees[class] = addr;
129 } else {
130 const bigpages_needed = bigPagesNeeded(actual_len);
131 const pow2_pages = math.ceilPowerOfTwoAssert(usize, bigpages_needed);
132 const big_slot_size_bytes = pow2_pages * bigpage_size;
133 const node = @intToPtr(*usize, addr + (big_slot_size_bytes - @sizeOf(usize)));
134 const big_class = math.log2(pow2_pages);
135 node.* = big_frees[big_class];
136 big_frees[big_class] = addr;
137 }
138}
139
140inline fn bigPagesNeeded(byte_count: usize) usize {
141 return (byte_count + (bigpage_size + (@sizeOf(usize) - 1))) / bigpage_size;
142}
143
144fn allocBigPages(n: usize) usize {
145 const pow2_pages = math.ceilPowerOfTwoAssert(usize, n);
146 const slot_size_bytes = pow2_pages * bigpage_size;
147 const class = math.log2(pow2_pages);
148
149 const top_free_ptr = big_frees[class];
150 if (top_free_ptr != 0) {
151 const node = @intToPtr(*usize, top_free_ptr + (slot_size_bytes - @sizeOf(usize)));
152 big_frees[class] = node.*;
153 return top_free_ptr;
154 }
155
156 const page_index = @wasmMemoryGrow(0, pow2_pages * pages_per_bigpage);
157 if (page_index <= 0) return 0;
158 const addr = @intCast(u32, page_index) * wasm.page_size;
159 return addr;
160}
161
162const test_ally = Allocator{
163 .ptr = undefined,
164 .vtable = &vtable,
165};
166
167test "small allocations - free in same order" {
168 var list: [513]*u64 = undefined;
169
170 var i: usize = 0;
171 while (i < 513) : (i += 1) {
172 const ptr = try test_ally.create(u64);
173 list[i] = ptr;
174 }
175
176 for (list) |ptr| {
177 test_ally.destroy(ptr);
178 }
179}
180
181test "small allocations - free in reverse order" {
182 var list: [513]*u64 = undefined;
183
184 var i: usize = 0;
185 while (i < 513) : (i += 1) {
186 const ptr = try test_ally.create(u64);
187 list[i] = ptr;
188 }
189
190 i = list.len;
191 while (i > 0) {
192 i -= 1;
193 const ptr = list[i];
194 test_ally.destroy(ptr);
195 }
196}
197
198test "large allocations" {
199 const ptr1 = try test_ally.alloc(u64, 42768);
200 const ptr2 = try test_ally.alloc(u64, 52768);
201 test_ally.free(ptr1);
202 const ptr3 = try test_ally.alloc(u64, 62768);
203 test_ally.free(ptr3);
204 test_ally.free(ptr2);
205}
206
207test "very large allocation" {
208 try std.testing.expectError(error.OutOfMemory, test_ally.alloc(u8, math.maxInt(usize)));
209}
210
211test "realloc" {
212 var slice = try test_ally.alignedAlloc(u8, @alignOf(u32), 1);
213 defer test_ally.free(slice);
214 slice[0] = 0x12;
215
216 // This reallocation should keep its pointer address.
217 const old_slice = slice;
218 slice = try test_ally.realloc(slice, 2);
219 try std.testing.expect(old_slice.ptr == slice.ptr);
220 try std.testing.expect(slice[0] == 0x12);
221 slice[1] = 0x34;
222
223 // This requires upgrading to a larger size class
224 slice = try test_ally.realloc(slice, 17);
225 try std.testing.expect(slice[0] == 0x12);
226 try std.testing.expect(slice[1] == 0x34);
227}
228
229test "shrink" {
230 var slice = try test_ally.alloc(u8, 20);
231 defer test_ally.free(slice);
232
233 mem.set(u8, slice, 0x11);
234
235 try std.testing.expect(test_ally.resize(slice, 17));
236 slice = slice[0..17];
237
238 for (slice) |b| {
239 try std.testing.expect(b == 0x11);
240 }
241
242 try std.testing.expect(test_ally.resize(slice, 16));
243 slice = slice[0..16];
244
245 for (slice) |b| {
246 try std.testing.expect(b == 0x11);
247 }
248}
249
250test "large object - grow" {
251 var slice1 = try test_ally.alloc(u8, bigpage_size * 2 - 20);
252 defer test_ally.free(slice1);
253
254 const old = slice1;
255 slice1 = try test_ally.realloc(slice1, bigpage_size * 2 - 10);
256 try std.testing.expect(slice1.ptr == old.ptr);
257
258 slice1 = try test_ally.realloc(slice1, bigpage_size * 2);
259 slice1 = try test_ally.realloc(slice1, bigpage_size * 2 + 1);
260}
261
262test "realloc small object to large object" {
263 var slice = try test_ally.alloc(u8, 70);
264 defer test_ally.free(slice);
265 slice[0] = 0x12;
266 slice[60] = 0x34;
267
268 // This requires upgrading to a large object
269 const large_object_size = bigpage_size * 2 + 50;
270 slice = try test_ally.realloc(slice, large_object_size);
271 try std.testing.expect(slice[0] == 0x12);
272 try std.testing.expect(slice[60] == 0x34);
273}
274
275test "shrink large object to large object" {
276 var slice = try test_ally.alloc(u8, bigpage_size * 2 + 50);
277 defer test_ally.free(slice);
278 slice[0] = 0x12;
279 slice[60] = 0x34;
280
281 try std.testing.expect(test_ally.resize(slice, bigpage_size * 2 + 1));
282 slice = slice[0 .. bigpage_size * 2 + 1];
283 try std.testing.expect(slice[0] == 0x12);
284 try std.testing.expect(slice[60] == 0x34);
285
286 try std.testing.expect(test_ally.resize(slice, bigpage_size * 2 + 1));
287 try std.testing.expect(slice[0] == 0x12);
288 try std.testing.expect(slice[60] == 0x34);
289
290 slice = try test_ally.realloc(slice, bigpage_size * 2);
291 try std.testing.expect(slice[0] == 0x12);
292 try std.testing.expect(slice[60] == 0x34);
293}
294
295test "realloc large object to small object" {
296 var slice = try test_ally.alloc(u8, bigpage_size * 2 + 50);
297 defer test_ally.free(slice);
298 slice[0] = 0x12;
299 slice[16] = 0x34;
300
301 slice = try test_ally.realloc(slice, 19);
302 try std.testing.expect(slice[0] == 0x12);
303 try std.testing.expect(slice[16] == 0x34);
304}
305
306test "objects of size 1024 and 2048" {
307 const slice = try test_ally.alloc(u8, 1025);
308 const slice2 = try test_ally.alloc(u8, 3000);
309
310 test_ally.free(slice);
311 test_ally.free(slice2);
312}
313
314test "standard allocator tests" {
315 try std.heap.testAllocator(test_ally);
316 try std.heap.testAllocatorAligned(test_ally);
317}
lib/std/heap/WasmPageAllocator.zig created+235
......@@ -0,0 +1,235 @@
1const WasmPageAllocator = @This();
2const std = @import("../std.zig");
3const builtin = @import("builtin");
4const Allocator = std.mem.Allocator;
5const mem = std.mem;
6const maxInt = std.math.maxInt;
7const assert = std.debug.assert;
8
9comptime {
10 if (!builtin.target.isWasm()) {
11 @compileError("WasmPageAllocator is only available for wasm32 arch");
12 }
13}
14
15pub const vtable = Allocator.VTable{
16 .alloc = alloc,
17 .resize = resize,
18 .free = free,
19};
20
21const PageStatus = enum(u1) {
22 used = 0,
23 free = 1,
24
25 pub const none_free: u8 = 0;
26};
27
28const FreeBlock = struct {
29 data: []u128,
30
31 const Io = std.packed_int_array.PackedIntIo(u1, .Little);
32
33 fn totalPages(self: FreeBlock) usize {
34 return self.data.len * 128;
35 }
36
37 fn isInitialized(self: FreeBlock) bool {
38 return self.data.len > 0;
39 }
40
41 fn getBit(self: FreeBlock, idx: usize) PageStatus {
42 const bit_offset = 0;
43 return @intToEnum(PageStatus, Io.get(mem.sliceAsBytes(self.data), idx, bit_offset));
44 }
45
46 fn setBits(self: FreeBlock, start_idx: usize, len: usize, val: PageStatus) void {
47 const bit_offset = 0;
48 var i: usize = 0;
49 while (i < len) : (i += 1) {
50 Io.set(mem.sliceAsBytes(self.data), start_idx + i, bit_offset, @enumToInt(val));
51 }
52 }
53
54 // Use '0xFFFFFFFF' as a _missing_ sentinel
55 // This saves ~50 bytes compared to returning a nullable
56
57 // We can guarantee that conventional memory never gets this big,
58 // and wasm32 would not be able to address this memory (32 GB > usize).
59
60 // Revisit if this is settled: https://github.com/ziglang/zig/issues/3806
61 const not_found = maxInt(usize);
62
63 fn useRecycled(self: FreeBlock, num_pages: usize, log2_align: u8) usize {
64 @setCold(true);
65 for (self.data) |segment, i| {
66 const spills_into_next = @bitCast(i128, segment) < 0;
67 const has_enough_bits = @popCount(segment) >= num_pages;
68
69 if (!spills_into_next and !has_enough_bits) continue;
70
71 var j: usize = i * 128;
72 while (j < (i + 1) * 128) : (j += 1) {
73 var count: usize = 0;
74 while (j + count < self.totalPages() and self.getBit(j + count) == .free) {
75 count += 1;
76 const addr = j * mem.page_size;
77 if (count >= num_pages and mem.isAlignedLog2(addr, log2_align)) {
78 self.setBits(j, num_pages, .used);
79 return j;
80 }
81 }
82 j += count;
83 }
84 }
85 return not_found;
86 }
87
88 fn recycle(self: FreeBlock, start_idx: usize, len: usize) void {
89 self.setBits(start_idx, len, .free);
90 }
91};
92
93var _conventional_data = [_]u128{0} ** 16;
94// Marking `conventional` as const saves ~40 bytes
95const conventional = FreeBlock{ .data = &_conventional_data };
96var extended = FreeBlock{ .data = &[_]u128{} };
97
98fn extendedOffset() usize {
99 return conventional.totalPages();
100}
101
102fn nPages(memsize: usize) usize {
103 return mem.alignForward(memsize, mem.page_size) / mem.page_size;
104}
105
106fn alloc(ctx: *anyopaque, len: usize, log2_align: u8, ra: usize) ?[*]u8 {
107 _ = ctx;
108 _ = ra;
109 if (len > maxInt(usize) - (mem.page_size - 1)) return null;
110 const page_count = nPages(len);
111 const page_idx = allocPages(page_count, log2_align) catch return null;
112 return @intToPtr([*]u8, page_idx * mem.page_size);
113}
114
115fn allocPages(page_count: usize, log2_align: u8) !usize {
116 {
117 const idx = conventional.useRecycled(page_count, log2_align);
118 if (idx != FreeBlock.not_found) {
119 return idx;
120 }
121 }
122
123 const idx = extended.useRecycled(page_count, log2_align);
124 if (idx != FreeBlock.not_found) {
125 return idx + extendedOffset();
126 }
127
128 const next_page_idx = @wasmMemorySize(0);
129 const next_page_addr = next_page_idx * mem.page_size;
130 const aligned_addr = mem.alignForwardLog2(next_page_addr, log2_align);
131 const drop_page_count = @divExact(aligned_addr - next_page_addr, mem.page_size);
132 const result = @wasmMemoryGrow(0, @intCast(u32, drop_page_count + page_count));
133 if (result <= 0)
134 return error.OutOfMemory;
135 assert(result == next_page_idx);
136 const aligned_page_idx = next_page_idx + drop_page_count;
137 if (drop_page_count > 0) {
138 freePages(next_page_idx, aligned_page_idx);
139 }
140 return @intCast(usize, aligned_page_idx);
141}
142
143fn freePages(start: usize, end: usize) void {
144 if (start < extendedOffset()) {
145 conventional.recycle(start, @min(extendedOffset(), end) - start);
146 }
147 if (end > extendedOffset()) {
148 var new_end = end;
149 if (!extended.isInitialized()) {
150 // Steal the last page from the memory currently being recycled
151 // TODO: would it be better if we use the first page instead?
152 new_end -= 1;
153
154 extended.data = @intToPtr([*]u128, new_end * mem.page_size)[0 .. mem.page_size / @sizeOf(u128)];
155 // Since this is the first page being freed and we consume it, assume *nothing* is free.
156 mem.set(u128, extended.data, PageStatus.none_free);
157 }
158 const clamped_start = @max(extendedOffset(), start);
159 extended.recycle(clamped_start - extendedOffset(), new_end - clamped_start);
160 }
161}
162
163fn resize(
164 ctx: *anyopaque,
165 buf: []u8,
166 log2_buf_align: u8,
167 new_len: usize,
168 return_address: usize,
169) bool {
170 _ = ctx;
171 _ = log2_buf_align;
172 _ = return_address;
173 const aligned_len = mem.alignForward(buf.len, mem.page_size);
174 if (new_len > aligned_len) return false;
175 const current_n = nPages(aligned_len);
176 const new_n = nPages(new_len);
177 if (new_n != current_n) {
178 const base = nPages(@ptrToInt(buf.ptr));
179 freePages(base + new_n, base + current_n);
180 }
181 return true;
182}
183
184fn free(
185 ctx: *anyopaque,
186 buf: []u8,
187 log2_buf_align: u8,
188 return_address: usize,
189) void {
190 _ = ctx;
191 _ = log2_buf_align;
192 _ = return_address;
193 const aligned_len = mem.alignForward(buf.len, mem.page_size);
194 const current_n = nPages(aligned_len);
195 const base = nPages(@ptrToInt(buf.ptr));
196 freePages(base, base + current_n);
197}
198
199test "internals" {
200 const page_allocator = std.heap.page_allocator;
201 const testing = std.testing;
202
203 const conventional_memsize = WasmPageAllocator.conventional.totalPages() * mem.page_size;
204 const initial = try page_allocator.alloc(u8, mem.page_size);
205 try testing.expect(@ptrToInt(initial.ptr) < conventional_memsize); // If this isn't conventional, the rest of these tests don't make sense. Also we have a serious memory leak in the test suite.
206
207 var inplace = try page_allocator.realloc(initial, 1);
208 try testing.expectEqual(initial.ptr, inplace.ptr);
209 inplace = try page_allocator.realloc(inplace, 4);
210 try testing.expectEqual(initial.ptr, inplace.ptr);
211 page_allocator.free(inplace);
212
213 const reuse = try page_allocator.alloc(u8, 1);
214 try testing.expectEqual(initial.ptr, reuse.ptr);
215 page_allocator.free(reuse);
216
217 // This segment may span conventional and extended which has really complex rules so we're just ignoring it for now.
218 const padding = try page_allocator.alloc(u8, conventional_memsize);
219 page_allocator.free(padding);
220
221 const ext = try page_allocator.alloc(u8, conventional_memsize);
222 try testing.expect(@ptrToInt(ext.ptr) >= conventional_memsize);
223
224 const use_small = try page_allocator.alloc(u8, 1);
225 try testing.expectEqual(initial.ptr, use_small.ptr);
226 page_allocator.free(use_small);
227
228 inplace = try page_allocator.realloc(ext, 1);
229 try testing.expectEqual(ext.ptr, inplace.ptr);
230 page_allocator.free(inplace);
231
232 const reuse_extended = try page_allocator.alloc(u8, conventional_memsize);
233 try testing.expectEqual(ext.ptr, reuse_extended.ptr);
234 page_allocator.free(reuse_extended);
235}