| author | |
| committer | |
| log | ea1ce2df9b20b5c91278eb0ed99a9cd0b0949e1a |
| tree | a3456166f71ca3a8dc56c99da9452ef3ffb4aa25 |
| parent | 3fe981e1ad746f9e3dfa2006fc69c907c92ddce6 |
| parent | 975cd9fc4ff8c12ae1f54e470b72be04d26e0837 |
| signature |
introduce std.heap.SmpAllocator11 files changed, 326 insertions(+), 43 deletions(-)
bootstrap.c+1-1| ... | @@ -139,7 +139,7 @@ int main(int argc, char **argv) { | ... | @@ -139,7 +139,7 @@ int main(int argc, char **argv) { |
| 139 | "pub const enable_tracy = false;\n" | 139 | "pub const enable_tracy = false;\n" |
| 140 | "pub const value_tracing = false;\n" | 140 | "pub const value_tracing = false;\n" |
| 141 | "pub const skip_non_native = false;\n" | 141 | "pub const skip_non_native = false;\n" |
| 142 | "pub const force_gpa = false;\n" | 142 | "pub const debug_gpa = false;\n" |
| 143 | "pub const dev = .core;\n" | 143 | "pub const dev = .core;\n" |
| 144 | "pub const value_interpret_mode = .direct;\n" | 144 | "pub const value_interpret_mode = .direct;\n" |
| 145 | , zig_version); | 145 | , zig_version); |
build.zig+3-3| ... | @@ -171,7 +171,7 @@ pub fn build(b: *std.Build) !void { | ... | @@ -171,7 +171,7 @@ pub fn build(b: *std.Build) !void { |
| 171 | const tracy_callstack = b.option(bool, "tracy-callstack", "Include callstack information with Tracy data. Does nothing if -Dtracy is not provided") orelse (tracy != null); | 171 | const tracy_callstack = b.option(bool, "tracy-callstack", "Include callstack information with Tracy data. Does nothing if -Dtracy is not provided") orelse (tracy != null); |
| 172 | const tracy_allocation = b.option(bool, "tracy-allocation", "Include allocation information with Tracy data. Does nothing if -Dtracy is not provided") orelse (tracy != null); | 172 | const tracy_allocation = b.option(bool, "tracy-allocation", "Include allocation information with Tracy data. Does nothing if -Dtracy is not provided") orelse (tracy != null); |
| 173 | const tracy_callstack_depth: u32 = b.option(u32, "tracy-callstack-depth", "Declare callstack depth for Tracy data. Does nothing if -Dtracy_callstack is not provided") orelse 10; | 173 | const tracy_callstack_depth: u32 = b.option(u32, "tracy-callstack-depth", "Declare callstack depth for Tracy data. Does nothing if -Dtracy_callstack is not provided") orelse 10; |
| 174 | const force_gpa = b.option(bool, "force-gpa", "Force the compiler to use GeneralPurposeAllocator") orelse false; | 174 | const debug_gpa = b.option(bool, "debug-allocator", "Force the compiler to use DebugAllocator") orelse false; |
| 175 | const link_libc = b.option(bool, "force-link-libc", "Force self-hosted compiler to link libc") orelse (enable_llvm or only_c); | 175 | const link_libc = b.option(bool, "force-link-libc", "Force self-hosted compiler to link libc") orelse (enable_llvm or only_c); |
| 176 | const sanitize_thread = b.option(bool, "sanitize-thread", "Enable thread-sanitization") orelse false; | 176 | const sanitize_thread = b.option(bool, "sanitize-thread", "Enable thread-sanitization") orelse false; |
| 177 | const strip = b.option(bool, "strip", "Omit debug information"); | 177 | const strip = b.option(bool, "strip", "Omit debug information"); |
| ... | @@ -233,7 +233,7 @@ pub fn build(b: *std.Build) !void { | ... | @@ -233,7 +233,7 @@ pub fn build(b: *std.Build) !void { |
| 233 | exe_options.addOption(bool, "llvm_has_csky", llvm_has_csky); | 233 | exe_options.addOption(bool, "llvm_has_csky", llvm_has_csky); |
| 234 | exe_options.addOption(bool, "llvm_has_arc", llvm_has_arc); | 234 | exe_options.addOption(bool, "llvm_has_arc", llvm_has_arc); |
| 235 | exe_options.addOption(bool, "llvm_has_xtensa", llvm_has_xtensa); | 235 | exe_options.addOption(bool, "llvm_has_xtensa", llvm_has_xtensa); |
| 236 | exe_options.addOption(bool, "force_gpa", force_gpa); | 236 | exe_options.addOption(bool, "debug_gpa", debug_gpa); |
| 237 | exe_options.addOption(DevEnv, "dev", b.option(DevEnv, "dev", "Build a compiler with a reduced feature set for development of specific features") orelse if (only_c) .bootstrap else .full); | 237 | exe_options.addOption(DevEnv, "dev", b.option(DevEnv, "dev", "Build a compiler with a reduced feature set for development of specific features") orelse if (only_c) .bootstrap else .full); |
| 238 | exe_options.addOption(ValueInterpretMode, "value_interpret_mode", value_interpret_mode); | 238 | exe_options.addOption(ValueInterpretMode, "value_interpret_mode", value_interpret_mode); |
| 239 | 239 | ||
| ... | @@ -608,7 +608,7 @@ fn addWasiUpdateStep(b: *std.Build, version: [:0]const u8) !void { | ... | @@ -608,7 +608,7 @@ fn addWasiUpdateStep(b: *std.Build, version: [:0]const u8) !void { |
| 608 | 608 | ||
| 609 | exe_options.addOption(u32, "mem_leak_frames", 0); | 609 | exe_options.addOption(u32, "mem_leak_frames", 0); |
| 610 | exe_options.addOption(bool, "have_llvm", false); | 610 | exe_options.addOption(bool, "have_llvm", false); |
| 611 | exe_options.addOption(bool, "force_gpa", false); | 611 | exe_options.addOption(bool, "debug_gpa", false); |
| 612 | exe_options.addOption([:0]const u8, "version", version); | 612 | exe_options.addOption([:0]const u8, "version", version); |
| 613 | exe_options.addOption(std.SemanticVersion, "semver", semver); | 613 | exe_options.addOption(std.SemanticVersion, "semver", semver); |
| 614 | exe_options.addOption(bool, "enable_debug_extensions", false); | 614 | exe_options.addOption(bool, "enable_debug_extensions", false); |
lib/libc/musl/src/thread/riscv32/clone.s+2| ... | @@ -7,6 +7,8 @@ | ... | @@ -7,6 +7,8 @@ |
| 7 | .global __clone | 7 | .global __clone |
| 8 | .type __clone, %function | 8 | .type __clone, %function |
| 9 | __clone: | 9 | __clone: |
| 10 | 	andi a1, a1, -16 | ||
| 11 | |||
| 10 | 	# Save func and arg to stack | 12 | 	# Save func and arg to stack |
| 11 | 	addi a1, a1, -16 | 13 | 	addi a1, a1, -16 |
| 12 | 	sw a0, 0(a1) | 14 | 	sw a0, 0(a1) |
lib/libc/musl/src/thread/riscv64/clone.s+2| ... | @@ -7,6 +7,8 @@ | ... | @@ -7,6 +7,8 @@ |
| 7 | .global __clone | 7 | .global __clone |
| 8 | .type __clone, %function | 8 | .type __clone, %function |
| 9 | __clone: | 9 | __clone: |
| 10 | 	andi a1, a1, -16 | ||
| 11 | |||
| 10 | 	# Save func and arg to stack | 12 | 	# Save func and arg to stack |
| 11 | 	addi a1, a1, -16 | 13 | 	addi a1, a1, -16 |
| 12 | 	sd a0, 0(a1) | 14 | 	sd a0, 0(a1) |
lib/std/heap.zig+20-5| ... | @@ -9,11 +9,12 @@ const Allocator = std.mem.Allocator; | ... | @@ -9,11 +9,12 @@ const Allocator = std.mem.Allocator; |
| 9 | const windows = std.os.windows; | 9 | const windows = std.os.windows; |
| 10 | 10 | ||
| 11 | pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator; | 11 | pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator; |
| 12 | pub const WasmAllocator = @import("heap/WasmAllocator.zig"); | 12 | pub const SmpAllocator = @import("heap/SmpAllocator.zig"); |
| 13 | pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig"); | ||
| 13 | pub const PageAllocator = @import("heap/PageAllocator.zig"); | 14 | pub const PageAllocator = @import("heap/PageAllocator.zig"); |
| 14 | pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig"); | ||
| 15 | pub const SbrkAllocator = @import("heap/sbrk_allocator.zig").SbrkAllocator; | 15 | pub const SbrkAllocator = @import("heap/sbrk_allocator.zig").SbrkAllocator; |
| 16 | pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig"); | 16 | pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig"); |
| 17 | pub const WasmAllocator = @import("heap/WasmAllocator.zig"); | ||
| 17 | 18 | ||
| 18 | pub const DebugAllocatorConfig = @import("heap/debug_allocator.zig").Config; | 19 | pub const DebugAllocatorConfig = @import("heap/debug_allocator.zig").Config; |
| 19 | pub const DebugAllocator = @import("heap/debug_allocator.zig").DebugAllocator; | 20 | pub const DebugAllocator = @import("heap/debug_allocator.zig").DebugAllocator; |
| ... | @@ -358,6 +359,11 @@ else if (builtin.target.isWasm()) .{ | ... | @@ -358,6 +359,11 @@ else if (builtin.target.isWasm()) .{ |
| 358 | .vtable = &PageAllocator.vtable, | 359 | .vtable = &PageAllocator.vtable, |
| 359 | }; | 360 | }; |
| 360 | 361 | ||
| 362 | pub const smp_allocator: Allocator = .{ | ||
| 363 | .ptr = undefined, | ||
| 364 | .vtable = &SmpAllocator.vtable, | ||
| 365 | }; | ||
| 366 | |||
| 361 | /// This allocator is fast, small, and specific to WebAssembly. In the future, | 367 | /// This allocator is fast, small, and specific to WebAssembly. In the future, |
| 362 | /// this will be the implementation automatically selected by | 368 | /// this will be the implementation automatically selected by |
| 363 | /// `GeneralPurposeAllocator` when compiling in `ReleaseSmall` mode for wasm32 | 369 | /// `GeneralPurposeAllocator` when compiling in `ReleaseSmall` mode for wasm32 |
| ... | @@ -475,7 +481,7 @@ pub fn StackFallbackAllocator(comptime size: usize) type { | ... | @@ -475,7 +481,7 @@ pub fn StackFallbackAllocator(comptime size: usize) type { |
| 475 | }; | 481 | }; |
| 476 | } | 482 | } |
| 477 | 483 | ||
| 478 | test "c_allocator" { | 484 | test c_allocator { |
| 479 | if (builtin.link_libc) { | 485 | if (builtin.link_libc) { |
| 480 | try testAllocator(c_allocator); | 486 | try testAllocator(c_allocator); |
| 481 | try testAllocatorAligned(c_allocator); | 487 | try testAllocatorAligned(c_allocator); |
| ... | @@ -484,12 +490,20 @@ test "c_allocator" { | ... | @@ -484,12 +490,20 @@ test "c_allocator" { |
| 484 | } | 490 | } |
| 485 | } | 491 | } |
| 486 | 492 | ||
| 487 | test "raw_c_allocator" { | 493 | test raw_c_allocator { |
| 488 | if (builtin.link_libc) { | 494 | if (builtin.link_libc) { |
| 489 | try testAllocator(raw_c_allocator); | 495 | try testAllocator(raw_c_allocator); |
| 490 | } | 496 | } |
| 491 | } | 497 | } |
| 492 | 498 | ||
| 499 | test smp_allocator { | ||
| 500 | if (builtin.single_threaded) return; | ||
| 501 | try testAllocator(smp_allocator); | ||
| 502 | try testAllocatorAligned(smp_allocator); | ||
| 503 | try testAllocatorLargeAlignment(smp_allocator); | ||
| 504 | try testAllocatorAlignedShrink(smp_allocator); | ||
| 505 | } | ||
| 506 | |||
| 493 | test PageAllocator { | 507 | test PageAllocator { |
| 494 | const allocator = page_allocator; | 508 | const allocator = page_allocator; |
| 495 | try testAllocator(allocator); | 509 | try testAllocator(allocator); |
| ... | @@ -978,4 +992,5 @@ test { | ... | @@ -978,4 +992,5 @@ test { |
| 978 | if (builtin.target.isWasm()) { | 992 | if (builtin.target.isWasm()) { |
| 979 | _ = WasmAllocator; | 993 | _ = WasmAllocator; |
| 980 | } | 994 | } |
| 995 | if (!builtin.single_threaded) _ = smp_allocator; | ||
| 981 | } | 996 | } |
lib/std/heap/PageAllocator.zig+18-12| ... | @@ -16,11 +16,7 @@ pub const vtable: Allocator.VTable = .{ | ... | @@ -16,11 +16,7 @@ pub const vtable: Allocator.VTable = .{ |
| 16 | .free = free, | 16 | .free = free, |
| 17 | }; | 17 | }; |
| 18 | 18 | ||
| 19 | fn alloc(context: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 { | 19 | pub fn map(n: usize, alignment: mem.Alignment) ?[*]u8 { |
| 20 | _ = context; | ||
| 21 | _ = ra; | ||
| 22 | assert(n > 0); | ||
| 23 | |||
| 24 | const page_size = std.heap.pageSize(); | 20 | const page_size = std.heap.pageSize(); |
| 25 | if (n >= maxInt(usize) - page_size) return null; | 21 | if (n >= maxInt(usize) - page_size) return null; |
| 26 | const alignment_bytes = alignment.toByteUnits(); | 22 | const alignment_bytes = alignment.toByteUnits(); |
| ... | @@ -101,6 +97,13 @@ fn alloc(context: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[* | ... | @@ -101,6 +97,13 @@ fn alloc(context: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[* |
| 101 | return result_ptr; | 97 | return result_ptr; |
| 102 | } | 98 | } |
| 103 | 99 | ||
| 100 | fn alloc(context: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 { | ||
| 101 | _ = context; | ||
| 102 | _ = ra; | ||
| 103 | assert(n > 0); | ||
| 104 | return map(n, alignment); | ||
| 105 | } | ||
| 106 | |||
| 104 | fn resize( | 107 | fn resize( |
| 105 | context: *anyopaque, | 108 | context: *anyopaque, |
| 106 | memory: []u8, | 109 | memory: []u8, |
| ... | @@ -114,7 +117,7 @@ fn resize( | ... | @@ -114,7 +117,7 @@ fn resize( |
| 114 | return realloc(memory, new_len, false) != null; | 117 | return realloc(memory, new_len, false) != null; |
| 115 | } | 118 | } |
| 116 | 119 | ||
| 117 | pub fn remap( | 120 | fn remap( |
| 118 | context: *anyopaque, | 121 | context: *anyopaque, |
| 119 | memory: []u8, | 122 | memory: []u8, |
| 120 | alignment: mem.Alignment, | 123 | alignment: mem.Alignment, |
| ... | @@ -127,21 +130,24 @@ pub fn remap( | ... | @@ -127,21 +130,24 @@ pub fn remap( |
| 127 | return realloc(memory, new_len, true); | 130 | return realloc(memory, new_len, true); |
| 128 | } | 131 | } |
| 129 | 132 | ||
| 130 | fn free(context: *anyopaque, slice: []u8, alignment: mem.Alignment, return_address: usize) void { | 133 | fn free(context: *anyopaque, memory: []u8, alignment: mem.Alignment, return_address: usize) void { |
| 131 | _ = context; | 134 | _ = context; |
| 132 | _ = alignment; | 135 | _ = alignment; |
| 133 | _ = return_address; | 136 | _ = return_address; |
| 137 | return unmap(@alignCast(memory)); | ||
| 138 | } | ||
| 134 | 139 | ||
| 140 | pub fn unmap(memory: []align(page_size_min) u8) void { | ||
| 135 | if (native_os == .windows) { | 141 | if (native_os == .windows) { |
| 136 | windows.VirtualFree(slice.ptr, 0, windows.MEM_RELEASE); | 142 | windows.VirtualFree(memory.ptr, 0, windows.MEM_RELEASE); |
| 137 | } else { | 143 | } else { |
| 138 | const buf_aligned_len = mem.alignForward(usize, slice.len, std.heap.pageSize()); | 144 | const page_aligned_len = mem.alignForward(usize, memory.len, std.heap.pageSize()); |
| 139 | posix.munmap(@alignCast(slice.ptr[0..buf_aligned_len])); | 145 | posix.munmap(memory.ptr[0..page_aligned_len]); |
| 140 | } | 146 | } |
| 141 | } | 147 | } |
| 142 | 148 | ||
| 143 | fn realloc(uncasted_memory: []u8, new_len: usize, may_move: bool) ?[*]u8 { | 149 | pub fn realloc(uncasted_memory: []u8, new_len: usize, may_move: bool) ?[*]u8 { |
| 144 | const memory: []align(std.heap.page_size_min) u8 = @alignCast(uncasted_memory); | 150 | const memory: []align(page_size_min) u8 = @alignCast(uncasted_memory); |
| 145 | const page_size = std.heap.pageSize(); | 151 | const page_size = std.heap.pageSize(); |
| 146 | const new_size_aligned = mem.alignForward(usize, new_len, page_size); | 152 | const new_size_aligned = mem.alignForward(usize, new_len, page_size); |
| 147 | 153 |
lib/std/heap/SmpAllocator.zig created+261| ... | @@ -0,0 +1,261 @@ | ||
| 1 | //! An allocator that is designed for ReleaseFast optimization mode, with | ||
| 2 | //! multi-threading enabled. | ||
| 3 | //! | ||
| 4 | //! This allocator is a singleton; it uses global state and only one should be | ||
| 5 | //! instantiated for the entire process. | ||
| 6 | //! | ||
| 7 | //! ## Basic Design | ||
| 8 | //! | ||
| 9 | //! Each thread gets a separate freelist, however, the data must be recoverable | ||
| 10 | //! when the thread exits. We do not directly learn when a thread exits, so | ||
| 11 | //! occasionally, one thread must attempt to reclaim another thread's | ||
| 12 | //! resources. | ||
| 13 | //! | ||
| 14 | //! Above a certain size, those allocations are memory mapped directly, with no | ||
| 15 | //! storage of allocation metadata. This works because the implementation | ||
| 16 | //! refuses resizes that would move an allocation from small category to large | ||
| 17 | //! category or vice versa. | ||
| 18 | //! | ||
| 19 | //! Each allocator operation checks the thread identifier from a threadlocal | ||
| 20 | //! variable to find out which metadata in the global state to access, and | ||
| 21 | //! attempts to grab its lock. This will usually succeed without contention, | ||
| 22 | //! unless another thread has been assigned the same id. In the case of such | ||
| 23 | //! contention, the thread moves on to the next thread metadata slot and | ||
| 24 | //! repeats the process of attempting to obtain the lock. | ||
| 25 | //! | ||
| 26 | //! By limiting the thread-local metadata array to the same number as the CPU | ||
| 27 | //! count, ensures that as threads are created and destroyed, they cycle | ||
| 28 | //! through the full set of freelists. | ||
| 29 | |||
| 30 | const builtin = @import("builtin"); | ||
| 31 | |||
| 32 | const std = @import("../std.zig"); | ||
| 33 | const assert = std.debug.assert; | ||
| 34 | const mem = std.mem; | ||
| 35 | const math = std.math; | ||
| 36 | const Allocator = std.mem.Allocator; | ||
| 37 | const SmpAllocator = @This(); | ||
| 38 | const PageAllocator = std.heap.PageAllocator; | ||
| 39 | |||
| 40 | cpu_count: u32, | ||
| 41 | threads: [max_thread_count]Thread, | ||
| 42 | |||
| 43 | var global: SmpAllocator = .{ | ||
| 44 | .threads = @splat(.{}), | ||
| 45 | .cpu_count = 0, | ||
| 46 | }; | ||
| 47 | threadlocal var thread_index: u32 = 0; | ||
| 48 | |||
| 49 | const max_thread_count = 128; | ||
| 50 | const slab_len: usize = @max(std.heap.page_size_max, 64 * 1024); | ||
| 51 | /// Because of storing free list pointers, the minimum size class is 3. | ||
| 52 | const min_class = math.log2(@sizeOf(usize)); | ||
| 53 | const size_class_count = math.log2(slab_len) - min_class; | ||
| 54 | /// When a freelist length exceeds this number, a `free` will rotate up to | ||
| 55 | /// `max_free_search` times before pushing. | ||
| 56 | const max_freelist_len: u8 = 16; | ||
| 57 | const max_free_search = 1; | ||
| 58 | /// Before mapping a fresh page, `alloc` will rotate this many times. | ||
| 59 | const max_alloc_search = 1; | ||
| 60 | |||
| 61 | const Thread = struct { | ||
| 62 | /// Avoid false sharing. | ||
| 63 | _: void align(std.atomic.cache_line) = {}, | ||
| 64 | |||
| 65 | /// Protects the state in this struct (per-thread state). | ||
| 66 | /// | ||
| 67 | /// Threads lock this before accessing their own state in order | ||
| 68 | /// to support freelist reclamation. | ||
| 69 | mutex: std.Thread.Mutex = .{}, | ||
| 70 | |||
| 71 | /// For each size class, tracks the next address to be returned from | ||
| 72 | /// `alloc` when the freelist is empty. | ||
| 73 | next_addrs: [size_class_count]usize = @splat(0), | ||
| 74 | /// For each size class, points to the freed pointer. | ||
| 75 | frees: [size_class_count]usize = @splat(0), | ||
| 76 | /// For each size class, tracks the number of items in the freelist. | ||
| 77 | freelist_lens: [size_class_count]u8 = @splat(0), | ||
| 78 | |||
| 79 | fn lock() *Thread { | ||
| 80 | var index = thread_index; | ||
| 81 | { | ||
| 82 | const t = &global.threads[index]; | ||
| 83 | if (t.mutex.tryLock()) { | ||
| 84 | @branchHint(.likely); | ||
| 85 | return t; | ||
| 86 | } | ||
| 87 | } | ||
| 88 | const cpu_count = getCpuCount(); | ||
| 89 | assert(cpu_count != 0); | ||
| 90 | while (true) { | ||
| 91 | index = (index + 1) % cpu_count; | ||
| 92 | const t = &global.threads[index]; | ||
| 93 | if (t.mutex.tryLock()) { | ||
| 94 | thread_index = index; | ||
| 95 | return t; | ||
| 96 | } | ||
| 97 | } | ||
| 98 | } | ||
| 99 | |||
| 100 | fn unlock(t: *Thread) void { | ||
| 101 | t.mutex.unlock(); | ||
| 102 | } | ||
| 103 | }; | ||
| 104 | |||
| 105 | fn getCpuCount() u32 { | ||
| 106 | const cpu_count = @atomicLoad(u32, &global.cpu_count, .unordered); | ||
| 107 | if (cpu_count != 0) return cpu_count; | ||
| 108 | const n: u32 = @min(std.Thread.getCpuCount() catch max_thread_count, max_thread_count); | ||
| 109 | return if (@cmpxchgStrong(u32, &global.cpu_count, 0, n, .monotonic, .monotonic)) |other| other else n; | ||
| 110 | } | ||
| 111 | |||
| 112 | pub const vtable: Allocator.VTable = .{ | ||
| 113 | .alloc = alloc, | ||
| 114 | .resize = resize, | ||
| 115 | .remap = remap, | ||
| 116 | .free = free, | ||
| 117 | }; | ||
| 118 | |||
| 119 | comptime { | ||
| 120 | assert(!builtin.single_threaded); // you're holding it wrong | ||
| 121 | } | ||
| 122 | |||
| 123 | fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 { | ||
| 124 | _ = context; | ||
| 125 | _ = ra; | ||
| 126 | const class = sizeClassIndex(len, alignment); | ||
| 127 | if (class >= size_class_count) { | ||
| 128 | @branchHint(.unlikely); | ||
| 129 | return PageAllocator.map(len, alignment); | ||
| 130 | } | ||
| 131 | |||
| 132 | const slot_size = slotSize(class); | ||
| 133 | assert(slab_len % slot_size == 0); | ||
| 134 | var search_count: u8 = 0; | ||
| 135 | |||
| 136 | var t = Thread.lock(); | ||
| 137 | |||
| 138 | outer: while (true) { | ||
| 139 | const top_free_ptr = t.frees[class]; | ||
| 140 | if (top_free_ptr != 0) { | ||
| 141 | @branchHint(.likely); | ||
| 142 | defer t.unlock(); | ||
| 143 | const node: *usize = @ptrFromInt(top_free_ptr); | ||
| 144 | t.frees[class] = node.*; | ||
| 145 | t.freelist_lens[class] -|= 1; | ||
| 146 | return @ptrFromInt(top_free_ptr); | ||
| 147 | } | ||
| 148 | |||
| 149 | const next_addr = t.next_addrs[class]; | ||
| 150 | if ((next_addr % slab_len) != 0) { | ||
| 151 | @branchHint(.likely); | ||
| 152 | defer t.unlock(); | ||
| 153 | t.next_addrs[class] = next_addr + slot_size; | ||
| 154 | return @ptrFromInt(next_addr); | ||
| 155 | } | ||
| 156 | |||
| 157 | if (search_count >= max_alloc_search) { | ||
| 158 | @branchHint(.likely); | ||
| 159 | defer t.unlock(); | ||
| 160 | // slab alignment here ensures the % slab len earlier catches the end of slots. | ||
| 161 | const slab = PageAllocator.map(slab_len, .fromByteUnits(slab_len)) orelse return null; | ||
| 162 | t.next_addrs[class] = @intFromPtr(slab) + slot_size; | ||
| 163 | t.freelist_lens[class] = 0; | ||
| 164 | return slab; | ||
| 165 | } | ||
| 166 | |||
| 167 | t.unlock(); | ||
| 168 | const cpu_count = getCpuCount(); | ||
| 169 | assert(cpu_count != 0); | ||
| 170 | var index = thread_index; | ||
| 171 | while (true) { | ||
| 172 | index = (index + 1) % cpu_count; | ||
| 173 | t = &global.threads[index]; | ||
| 174 | if (t.mutex.tryLock()) { | ||
| 175 | thread_index = index; | ||
| 176 | search_count += 1; | ||
| 177 | continue :outer; | ||
| 178 | } | ||
| 179 | } | ||
| 180 | } | ||
| 181 | } | ||
| 182 | |||
| 183 | fn resize(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ra: usize) bool { | ||
| 184 | _ = context; | ||
| 185 | _ = ra; | ||
| 186 | const class = sizeClassIndex(memory.len, alignment); | ||
| 187 | const new_class = sizeClassIndex(new_len, alignment); | ||
| 188 | if (class >= size_class_count) { | ||
| 189 | if (new_class < size_class_count) return false; | ||
| 190 | return PageAllocator.realloc(memory, new_len, false) != null; | ||
| 191 | } | ||
| 192 | return new_class == class; | ||
| 193 | } | ||
| 194 | |||
| 195 | fn remap(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ra: usize) ?[*]u8 { | ||
| 196 | _ = context; | ||
| 197 | _ = ra; | ||
| 198 | const class = sizeClassIndex(memory.len, alignment); | ||
| 199 | const new_class = sizeClassIndex(new_len, alignment); | ||
| 200 | if (class >= size_class_count) { | ||
| 201 | if (new_class < size_class_count) return null; | ||
| 202 | return PageAllocator.realloc(memory, new_len, true); | ||
| 203 | } | ||
| 204 | return if (new_class == class) memory.ptr else null; | ||
| 205 | } | ||
| 206 | |||
| 207 | fn free(context: *anyopaque, memory: []u8, alignment: mem.Alignment, ra: usize) void { | ||
| 208 | _ = context; | ||
| 209 | _ = ra; | ||
| 210 | const class = sizeClassIndex(memory.len, alignment); | ||
| 211 | if (class >= size_class_count) { | ||
| 212 | @branchHint(.unlikely); | ||
| 213 | return PageAllocator.unmap(@alignCast(memory)); | ||
| 214 | } | ||
| 215 | |||
| 216 | const node: *usize = @alignCast(@ptrCast(memory.ptr)); | ||
| 217 | var search_count: u8 = 0; | ||
| 218 | |||
| 219 | var t = Thread.lock(); | ||
| 220 | |||
| 221 | outer: while (true) { | ||
| 222 | const freelist_len = t.freelist_lens[class]; | ||
| 223 | if (freelist_len < max_freelist_len) { | ||
| 224 | @branchHint(.likely); | ||
| 225 | defer t.unlock(); | ||
| 226 | node.* = t.frees[class]; | ||
| 227 | t.frees[class] = @intFromPtr(node); | ||
| 228 | return; | ||
| 229 | } | ||
| 230 | |||
| 231 | if (search_count >= max_free_search) { | ||
| 232 | defer t.unlock(); | ||
| 233 | t.freelist_lens[class] = freelist_len +| 1; | ||
| 234 | node.* = t.frees[class]; | ||
| 235 | t.frees[class] = @intFromPtr(node); | ||
| 236 | return; | ||
| 237 | } | ||
| 238 | |||
| 239 | t.unlock(); | ||
| 240 | const cpu_count = getCpuCount(); | ||
| 241 | assert(cpu_count != 0); | ||
| 242 | var index = thread_index; | ||
| 243 | while (true) { | ||
| 244 | index = (index + 1) % cpu_count; | ||
| 245 | t = &global.threads[index]; | ||
| 246 | if (t.mutex.tryLock()) { | ||
| 247 | thread_index = index; | ||
| 248 | search_count += 1; | ||
| 249 | continue :outer; | ||
| 250 | } | ||
| 251 | } | ||
| 252 | } | ||
| 253 | } | ||
| 254 | |||
| 255 | fn sizeClassIndex(len: usize, alignment: mem.Alignment) usize { | ||
| 256 | return @max(@bitSizeOf(usize) - @clz(len - 1), @intFromEnum(alignment), min_class) - min_class; | ||
| 257 | } | ||
| 258 | |||
| 259 | fn slotSize(class: usize) usize { | ||
| 260 | return @as(usize, 1) << @intCast(class + min_class); | ||
| 261 | } | ||
lib/std/heap/WasmAllocator.zig-2| ... | @@ -1,5 +1,3 @@ | ... | @@ -1,5 +1,3 @@ |
| 1 | //! This is intended to be merged into GeneralPurposeAllocator at some point. | ||
| 2 | |||
| 3 | const std = @import("../std.zig"); | 1 | const std = @import("../std.zig"); |
| 4 | const builtin = @import("builtin"); | 2 | const builtin = @import("builtin"); |
| 5 | const Allocator = std.mem.Allocator; | 3 | const Allocator = std.mem.Allocator; |
lib/std/heap/debug_allocator.zig-2| ... | @@ -851,8 +851,6 @@ pub fn DebugAllocator(comptime config: Config) type { | ... | @@ -851,8 +851,6 @@ pub fn DebugAllocator(comptime config: Config) type { |
| 851 | self.mutex.lock(); | 851 | self.mutex.lock(); |
| 852 | defer self.mutex.unlock(); | 852 | defer self.mutex.unlock(); |
| 853 | 853 | ||
| 854 | assert(old_memory.len != 0); | ||
| 855 | |||
| 856 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(old_memory.len - 1), @intFromEnum(alignment)); | 854 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(old_memory.len - 1), @intFromEnum(alignment)); |
| 857 | if (size_class_index >= self.buckets.len) { | 855 | if (size_class_index >= self.buckets.len) { |
| 858 | @branchHint(.unlikely); | 856 | @branchHint(.unlikely); |
src/main.zig+18-17| ... | @@ -171,30 +171,31 @@ pub fn log( | ... | @@ -171,30 +171,31 @@ pub fn log( |
| 171 | std.debug.print(prefix1 ++ prefix2 ++ format ++ "\n", args); | 171 | std.debug.print(prefix1 ++ prefix2 ++ format ++ "\n", args); |
| 172 | } | 172 | } |
| 173 | 173 | ||
| 174 | var general_purpose_allocator = std.heap.GeneralPurposeAllocator(.{ | 174 | var debug_allocator: std.heap.DebugAllocator(.{ |
| 175 | .stack_trace_frames = build_options.mem_leak_frames, | 175 | .stack_trace_frames = build_options.mem_leak_frames, |
| 176 | }){}; | 176 | }) = .init; |
| 177 | 177 | ||
| 178 | pub fn main() anyerror!void { | 178 | pub fn main() anyerror!void { |
| 179 | crash_report.initialize(); | 179 | crash_report.initialize(); |
| 180 | 180 | ||
| 181 | const use_gpa = (build_options.force_gpa or !builtin.link_libc) and native_os != .wasi; | 181 | const gpa, const is_debug = gpa: { |
| 182 | const gpa = gpa: { | 182 | if (build_options.debug_gpa) break :gpa .{ debug_allocator.allocator(), true }; |
| 183 | if (native_os == .wasi) { | 183 | if (native_os == .wasi) break :gpa .{ std.heap.wasm_allocator, false }; |
| 184 | break :gpa std.heap.wasm_allocator; | 184 | if (builtin.link_libc) { |
| 185 | } | 185 | // We would prefer to use raw libc allocator here, but cannot use |
| 186 | if (use_gpa) { | 186 | // it if it won't support the alignment we need. |
| 187 | break :gpa general_purpose_allocator.allocator(); | 187 | if (@alignOf(std.c.max_align_t) < @max(@alignOf(i128), std.atomic.cache_line)) { |
| 188 | } | 188 | break :gpa .{ std.heap.c_allocator, false }; |
| 189 | // We would prefer to use raw libc allocator here, but cannot | 189 | } |
| 190 | // use it if it won't support the alignment we need. | 190 | break :gpa .{ std.heap.raw_c_allocator, false }; |
| 191 | if (@alignOf(std.c.max_align_t) < @max(@alignOf(i128), std.atomic.cache_line)) { | ||
| 192 | break :gpa std.heap.c_allocator; | ||
| 193 | } | 191 | } |
| 194 | break :gpa std.heap.raw_c_allocator; | 192 | break :gpa switch (builtin.mode) { |
| 193 | .Debug, .ReleaseSafe => .{ debug_allocator.allocator(), true }, | ||
| 194 | .ReleaseFast, .ReleaseSmall => .{ std.heap.smp_allocator, false }, | ||
| 195 | }; | ||
| 195 | }; | 196 | }; |
| 196 | defer if (use_gpa) { | 197 | defer if (is_debug) { |
| 197 | _ = general_purpose_allocator.deinit(); | 198 | _ = debug_allocator.deinit(); |
| 198 | }; | 199 | }; |
| 199 | var arena_instance = std.heap.ArenaAllocator.init(gpa); | 200 | var arena_instance = std.heap.ArenaAllocator.init(gpa); |
| 200 | defer arena_instance.deinit(); | 201 | defer arena_instance.deinit(); |
stage1/config.zig.in+1-1| ... | @@ -11,6 +11,6 @@ pub const enable_link_snapshots = false; | ... | @@ -11,6 +11,6 @@ pub const enable_link_snapshots = false; |
| 11 | pub const enable_tracy = false; | 11 | pub const enable_tracy = false; |
| 12 | pub const value_tracing = false; | 12 | pub const value_tracing = false; |
| 13 | pub const skip_non_native = false; | 13 | pub const skip_non_native = false; |
| 14 | pub const force_gpa = false; | 14 | pub const debug_gpa = false; |
| 15 | pub const dev = .core; | 15 | pub const dev = .core; |
| 16 | pub const value_interpret_mode = .direct; | 16 | pub const value_interpret_mode = .direct; |