| author |  Andrew Kelley <andrew@ziglang.org> 2025-02-05 19:18:22-08:00 |
| committer | Andrew Kelley <andrew@ziglang.org> 2025-02-06 14:23:23-08:00 |
| log | cd99ab32294a3c22f09615c93d611593a2887cc3 |
| tree | 7251e1608aeae310cf6b249077c4d790b3e6794e |
| parent | 5e9b8c38d360a254bf951674f4b39ea7a602c515 |
std.heap: rename GeneralPurposeAllocator to DebugAllocator
3 files changed, 1412 insertions(+), 1414 deletions(-)
lib/std/heap.zig+8-3
| ... | ... | @@ -9,15 +9,20 @@ const Allocator = std.mem.Allocator; |
| 9 | 9 | const windows = std.os.windows; |
| 10 | 10 | |
| 11 | 11 | pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator; |
| 12 | pub const GeneralPurposeAllocatorConfig = @import("heap/general_purpose_allocator.zig").Config; | |
| 13 | pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig").GeneralPurposeAllocator; | |
| 14 | pub const Check = @import("heap/general_purpose_allocator.zig").Check; | |
| 15 | 12 | pub const WasmAllocator = @import("heap/WasmAllocator.zig"); |
| 16 | 13 | pub const PageAllocator = @import("heap/PageAllocator.zig"); |
| 17 | 14 | pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig"); |
| 18 | 15 | pub const SbrkAllocator = @import("heap/sbrk_allocator.zig").SbrkAllocator; |
| 19 | 16 | pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig"); |
| 20 | 17 | |
| 18 | pub const DebugAllocatorConfig = @import("heap/debug_allocator.zig").Config; | |
| 19 | pub const DebugAllocator = @import("heap/debug_allocator.zig").DebugAllocator; | |
| 20 | pub const Check = enum { ok, leak }; | |
| 21 | /// Deprecated; to be removed after 0.15.0 is tagged. | |
| 22 | pub const GeneralPurposeAllocatorConfig = DebugAllocatorConfig; | |
| 23 | /// Deprecated; to be removed after 0.15.0 is tagged. | |
| 24 | pub const GeneralPurposeAllocator = DebugAllocator; | |
| 25 | ||
| 21 | 26 | const memory_pool = @import("heap/memory_pool.zig"); |
| 22 | 27 | pub const MemoryPool = memory_pool.MemoryPool; |
| 23 | 28 | pub const MemoryPoolAligned = memory_pool.MemoryPoolAligned; |
lib/std/heap/debug_allocator.zig created+1404
| ... | ... | @@ -0,0 +1,1404 @@ |
| 1 | //! An allocator that is intended to be used in Debug mode. | |
| 2 | //! | |
| 3 | //! ## Features | |
| 4 | //! | |
| 5 | //! * Captures stack traces on allocation, free, and optionally resize. | |
| 6 | //! * Double free detection, which prints all three traces (first alloc, first | |
| 7 | //! free, second free). | |
| 8 | //! * Leak detection, with stack traces. | |
| 9 | //! * Never reuses memory addresses, making it easier for Zig to detect branch | |
| 10 | //! on undefined values in case of dangling pointers. This relies on | |
| 11 | //! the backing allocator to also not reuse addresses. | |
| 12 | //! * Uses a minimum backing allocation size to avoid operating system errors | |
| 13 | //! from having too many active memory mappings. | |
| 14 | //! * When a page of memory is no longer needed, give it back to resident | |
| 15 | //! memory as soon as possible, so that it causes page faults when used. | |
| 16 | //! * Cross platform. Operates based on a backing allocator which makes it work | |
| 17 | //! everywhere, even freestanding. | |
| 18 | //! * Compile-time configuration. | |
| 19 | //! | |
| 20 | //! These features require the allocator to be quite slow and wasteful. For | |
| 21 | //! example, when allocating a single byte, the efficiency is less than 1%; | |
| 22 | //! it requires more than 100 bytes of overhead to manage the allocation for | |
| 23 | //! one byte. The efficiency gets better with larger allocations. | |
| 24 | //! | |
| 25 | //! ## Basic Design | |
| 26 | //! | |
| 27 | //! Allocations are divided into two categories, small and large. | |
| 28 | //! | |
| 29 | //! Small allocations are divided into buckets based on `page_size`: | |
| 30 | //! | |
| 31 | //! ``` | |
| 32 | //! index obj_size | |
| 33 | //! 0 1 | |
| 34 | //! 1 2 | |
| 35 | //! 2 4 | |
| 36 | //! 3 8 | |
| 37 | //! 4 16 | |
| 38 | //! 5 32 | |
| 39 | //! 6 64 | |
| 40 | //! 7 128 | |
| 41 | //! 8 256 | |
| 42 | //! 9 512 | |
| 43 | //! 10 1024 | |
| 44 | //! 11 2048 | |
| 45 | //! ... | |
| 46 | //! ``` | |
| 47 | //! | |
| 48 | //! This goes on for `small_bucket_count` indexes. | |
| 49 | //! | |
| 50 | //! Allocations are grouped into an object size based on max(len, alignment), | |
| 51 | //! rounded up to the next power of two. | |
| 52 | //! | |
| 53 | //! The main allocator state has an array of all the "current" buckets for each | |
| 54 | //! size class. Each slot in the array can be null, meaning the bucket for that | |
| 55 | //! size class is not allocated. When the first object is allocated for a given | |
| 56 | //! size class, it makes one `page_size` allocation from the backing allocator. | |
| 57 | //! This allocation is divided into "slots" - one per allocated object, leaving | |
| 58 | //! room for the allocation metadata (starting with `BucketHeader`), which is | |
| 59 | //! located at the very end of the "page". | |
| 60 | //! | |
| 61 | //! The allocation metadata includes "used bits" - 1 bit per slot representing | |
| 62 | //! whether the slot is used. Allocations always take the next available slot | |
| 63 | //! from the current bucket, setting the corresponding used bit, as well as | |
| 64 | //! incrementing `allocated_count`. | |
| 65 | //! | |
| 66 | //! Frees recover the allocation metadata based on the address, length, and | |
| 67 | //! alignment, relying on the backing allocation's large alignment, combined | |
| 68 | //! with the fact that allocations are never moved from small to large, or vice | |
| 69 | //! versa. | |
| 70 | //! | |
| 71 | //! When a bucket is full, a new one is allocated, containing a pointer to the | |
| 72 | //! previous one. This singly-linked list is iterated during leak detection. | |
| 73 | //! | |
| 74 | //! Resizing and remapping work the same on small allocations: if the size | |
| 75 | //! class would not change, then the operation succeeds, and the address is | |
| 76 | //! unchanged. Otherwise, the request is rejected. | |
| 77 | //! | |
| 78 | //! Large objects are allocated directly using the backing allocator. Metadata | |
| 79 | //! is stored separately in a `std.HashMap` using the backing allocator. | |
| 80 | //! | |
| 81 | //! Resizing and remapping are forwarded directly to the backing allocator, | |
| 82 | //! except where such operations would change the category from large to small. | |
| 83 | ||
| 84 | const std = @import("std"); | |
| 85 | const builtin = @import("builtin"); | |
| 86 | const log = std.log.scoped(.gpa); | |
| 87 | const math = std.math; | |
| 88 | const assert = std.debug.assert; | |
| 89 | const mem = std.mem; | |
| 90 | const Allocator = std.mem.Allocator; | |
| 91 | const StackTrace = std.builtin.StackTrace; | |
| 92 | ||
| 93 | const page_size: usize = @max(std.heap.page_size_max, switch (builtin.os.tag) { | |
| 94 | .windows => 64 * 1024, // Makes `std.heap.PageAllocator` take the happy path. | |
| 95 | .wasi => 64 * 1024, // Max alignment supported by `std.heap.WasmAllocator`. | |
| 96 | else => 128 * 1024, // Avoids too many active mappings when `page_size_max` is low. | |
| 97 | }); | |
| 98 | const page_align: mem.Alignment = .fromByteUnits(page_size); | |
| 99 | ||
| 100 | /// Integer type for pointing to slots in a small allocation | |
| 101 | const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size) + 1); | |
| 102 | const Log2USize = std.math.Log2Int(usize); | |
| 103 | ||
| 104 | const default_sys_stack_trace_frames: usize = if (std.debug.sys_can_stack_trace) 6 else 0; | |
| 105 | const default_stack_trace_frames: usize = switch (builtin.mode) { | |
| 106 | .Debug => default_sys_stack_trace_frames, | |
| 107 | else => 0, | |
| 108 | }; | |
| 109 | ||
| 110 | pub const Config = struct { | |
| 111 | /// Number of stack frames to capture. | |
| 112 | stack_trace_frames: usize = default_stack_trace_frames, | |
| 113 | ||
| 114 | /// If true, the allocator will have two fields: | |
| 115 | /// * `total_requested_bytes` which tracks the total allocated bytes of memory requested. | |
| 116 | /// * `requested_memory_limit` which causes allocations to return `error.OutOfMemory` | |
| 117 | /// when the `total_requested_bytes` exceeds this limit. | |
| 118 | /// If false, these fields will be `void`. | |
| 119 | enable_memory_limit: bool = false, | |
| 120 | ||
| 121 | /// Whether to enable safety checks. | |
| 122 | safety: bool = std.debug.runtime_safety, | |
| 123 | ||
| 124 | /// Whether the allocator may be used simultaneously from multiple threads. | |
| 125 | thread_safe: bool = !builtin.single_threaded, | |
| 126 | ||
| 127 | /// What type of mutex you'd like to use, for thread safety. | |
| 128 | /// when specified, the mutex type must have the same shape as `std.Thread.Mutex` and | |
| 129 | /// `DummyMutex`, and have no required fields. Specifying this field causes | |
| 130 | /// the `thread_safe` field to be ignored. | |
| 131 | /// | |
| 132 | /// when null (default): | |
| 133 | /// * the mutex type defaults to `std.Thread.Mutex` when thread_safe is enabled. | |
| 134 | /// * the mutex type defaults to `DummyMutex` otherwise. | |
| 135 | MutexType: ?type = null, | |
| 136 | ||
| 137 | /// This is a temporary debugging trick you can use to turn segfaults into more helpful | |
| 138 | /// logged error messages with stack trace details. The downside is that every allocation | |
| 139 | /// will be leaked, unless used with retain_metadata! | |
| 140 | never_unmap: bool = false, | |
| 141 | ||
| 142 | /// This is a temporary debugging aid that retains metadata about allocations indefinitely. | |
| 143 | /// This allows a greater range of double frees to be reported. All metadata is freed when | |
| 144 | /// deinit is called. When used with never_unmap, deliberately leaked memory is also freed | |
| 145 | /// during deinit. Currently should be used with never_unmap to avoid segfaults. | |
| 146 | /// TODO https://github.com/ziglang/zig/issues/4298 will allow use without never_unmap | |
| 147 | retain_metadata: bool = false, | |
| 148 | ||
| 149 | /// Enables emitting info messages with the size and address of every allocation. | |
| 150 | verbose_log: bool = false, | |
| 151 | ||
| 152 | /// Tell whether the backing allocator returns already-zeroed memory. | |
| 153 | backing_allocator_zeroes: bool = true, | |
| 154 | ||
| 155 | /// When resizing an allocation, refresh the stack trace with the resize | |
| 156 | /// callsite. Comes with a performance penalty. | |
| 157 | resize_stack_traces: bool = false, | |
| 158 | ||
| 159 | /// Magic value that distinguishes allocations owned by this allocator from | |
| 160 | /// other regions of memory. | |
| 161 | canary: usize = @truncate(0x9232a6ff85dff10f), | |
| 162 | }; | |
| 163 | ||
| 164 | /// Default initialization of this struct is deprecated; use `.init` instead. | |
| 165 | pub fn DebugAllocator(comptime config: Config) type { | |
| 166 | return struct { | |
| 167 | backing_allocator: Allocator = std.heap.page_allocator, | |
| 168 | /// Tracks the active bucket, which is the one that has free slots in it. | |
| 169 | buckets: [small_bucket_count]?*BucketHeader = [1]?*BucketHeader{null} ** small_bucket_count, | |
| 170 | large_allocations: LargeAllocTable = .empty, | |
| 171 | total_requested_bytes: @TypeOf(total_requested_bytes_init) = total_requested_bytes_init, | |
| 172 | requested_memory_limit: @TypeOf(requested_memory_limit_init) = requested_memory_limit_init, | |
| 173 | mutex: @TypeOf(mutex_init) = mutex_init, | |
| 174 | ||
| 175 | const Self = @This(); | |
| 176 | ||
| 177 | pub const init: Self = .{}; | |
| 178 | ||
| 179 | /// These can be derived from size_class_index but the calculation is nontrivial. | |
| 180 | const slot_counts: [small_bucket_count]SlotIndex = init: { | |
| 181 | @setEvalBranchQuota(10000); | |
| 182 | var result: [small_bucket_count]SlotIndex = undefined; | |
| 183 | for (&result, 0..) |*elem, i| elem.* = calculateSlotCount(i); | |
| 184 | break :init result; | |
| 185 | }; | |
| 186 | ||
| 187 | const total_requested_bytes_init = if (config.enable_memory_limit) @as(usize, 0) else {}; | |
| 188 | const requested_memory_limit_init = if (config.enable_memory_limit) @as(usize, math.maxInt(usize)) else {}; | |
| 189 | ||
| 190 | const mutex_init = if (config.MutexType) |T| | |
| 191 | T{} | |
| 192 | else if (config.thread_safe) | |
| 193 | std.Thread.Mutex{} | |
| 194 | else | |
| 195 | DummyMutex{}; | |
| 196 | ||
| 197 | const DummyMutex = struct { | |
| 198 | inline fn lock(_: *DummyMutex) void {} | |
| 199 | inline fn unlock(_: *DummyMutex) void {} | |
| 200 | }; | |
| 201 | ||
| 202 | const stack_n = config.stack_trace_frames; | |
| 203 | const one_trace_size = @sizeOf(usize) * stack_n; | |
| 204 | const traces_per_slot = 2; | |
| 205 | ||
| 206 | pub const Error = mem.Allocator.Error; | |
| 207 | ||
| 208 | /// Avoids creating buckets that would only be able to store a small | |
| 209 | /// number of slots. Value of 1 means 2 is the minimum slot count. | |
| 210 | const minimum_slots_per_bucket_log2 = 1; | |
| 211 | const small_bucket_count = math.log2(page_size) - minimum_slots_per_bucket_log2; | |
| 212 | const largest_bucket_object_size = 1 << (small_bucket_count - 1); | |
| 213 | const LargestSizeClassInt = std.math.IntFittingRange(0, largest_bucket_object_size); | |
| 214 | ||
| 215 | const bucketCompare = struct { | |
| 216 | fn compare(a: *BucketHeader, b: *BucketHeader) std.math.Order { | |
| 217 | return std.math.order(@intFromPtr(a.page), @intFromPtr(b.page)); | |
| 218 | } | |
| 219 | }.compare; | |
| 220 | ||
| 221 | const LargeAlloc = struct { | |
| 222 | bytes: []u8, | |
| 223 | requested_size: if (config.enable_memory_limit) usize else void, | |
| 224 | stack_addresses: [trace_n][stack_n]usize, | |
| 225 | freed: if (config.retain_metadata) bool else void, | |
| 226 | alignment: if (config.never_unmap and config.retain_metadata) mem.Alignment else void, | |
| 227 | ||
| 228 | const trace_n = if (config.retain_metadata) traces_per_slot else 1; | |
| 229 | ||
| 230 | fn dumpStackTrace(self: *LargeAlloc, trace_kind: TraceKind) void { | |
| 231 | std.debug.dumpStackTrace(self.getStackTrace(trace_kind)); | |
| 232 | } | |
| 233 | ||
| 234 | fn getStackTrace(self: *LargeAlloc, trace_kind: TraceKind) std.builtin.StackTrace { | |
| 235 | assert(@intFromEnum(trace_kind) < trace_n); | |
| 236 | const stack_addresses = &self.stack_addresses[@intFromEnum(trace_kind)]; | |
| 237 | var len: usize = 0; | |
| 238 | while (len < stack_n and stack_addresses[len] != 0) { | |
| 239 | len += 1; | |
| 240 | } | |
| 241 | return .{ | |
| 242 | .instruction_addresses = stack_addresses, | |
| 243 | .index = len, | |
| 244 | }; | |
| 245 | } | |
| 246 | ||
| 247 | fn captureStackTrace(self: *LargeAlloc, ret_addr: usize, trace_kind: TraceKind) void { | |
| 248 | assert(@intFromEnum(trace_kind) < trace_n); | |
| 249 | const stack_addresses = &self.stack_addresses[@intFromEnum(trace_kind)]; | |
| 250 | collectStackTrace(ret_addr, stack_addresses); | |
| 251 | } | |
| 252 | }; | |
| 253 | const LargeAllocTable = std.AutoHashMapUnmanaged(usize, LargeAlloc); | |
| 254 | ||
| 255 | /// Bucket: In memory, in order: | |
| 256 | /// * BucketHeader | |
| 257 | /// * bucket_used_bits: [N]usize, // 1 bit for every slot | |
| 258 | /// -- below only exists when config.safety is true -- | |
| 259 | /// * requested_sizes: [N]LargestSizeClassInt // 1 int for every slot | |
| 260 | /// * log2_ptr_aligns: [N]u8 // 1 byte for every slot | |
| 261 | /// -- above only exists when config.safety is true -- | |
| 262 | /// * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation | |
| 263 | const BucketHeader = struct { | |
| 264 | allocated_count: SlotIndex, | |
| 265 | freed_count: SlotIndex, | |
| 266 | prev: ?*BucketHeader, | |
| 267 | canary: usize = config.canary, | |
| 268 | ||
| 269 | fn fromPage(page_addr: usize, slot_count: usize) *BucketHeader { | |
| 270 | const unaligned = page_addr + page_size - bucketSize(slot_count); | |
| 271 | return @ptrFromInt(unaligned & ~(@as(usize, @alignOf(BucketHeader)) - 1)); | |
| 272 | } | |
| 273 | ||
| 274 | fn usedBits(bucket: *BucketHeader, index: usize) *usize { | |
| 275 | const ptr: [*]u8 = @ptrCast(bucket); | |
| 276 | const bits: [*]usize = @alignCast(@ptrCast(ptr + @sizeOf(BucketHeader))); | |
| 277 | return &bits[index]; | |
| 278 | } | |
| 279 | ||
| 280 | fn requestedSizes(bucket: *BucketHeader, slot_count: usize) []LargestSizeClassInt { | |
| 281 | if (!config.safety) @compileError("requested size is only stored when safety is enabled"); | |
| 282 | const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketRequestedSizesStart(slot_count); | |
| 283 | const sizes = @as([*]LargestSizeClassInt, @ptrCast(@alignCast(start_ptr))); | |
| 284 | return sizes[0..slot_count]; | |
| 285 | } | |
| 286 | ||
| 287 | fn log2PtrAligns(bucket: *BucketHeader, slot_count: usize) []mem.Alignment { | |
| 288 | if (!config.safety) @compileError("requested size is only stored when safety is enabled"); | |
| 289 | const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(slot_count); | |
| 290 | return @ptrCast(aligns_ptr[0..slot_count]); | |
| 291 | } | |
| 292 | ||
| 293 | fn stackTracePtr( | |
| 294 | bucket: *BucketHeader, | |
| 295 | slot_count: usize, | |
| 296 | slot_index: SlotIndex, | |
| 297 | trace_kind: TraceKind, | |
| 298 | ) *[stack_n]usize { | |
| 299 | const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketStackFramesStart(slot_count); | |
| 300 | const addr = start_ptr + one_trace_size * traces_per_slot * slot_index + | |
| 301 | @intFromEnum(trace_kind) * @as(usize, one_trace_size); | |
| 302 | return @ptrCast(@alignCast(addr)); | |
| 303 | } | |
| 304 | ||
| 305 | fn captureStackTrace( | |
| 306 | bucket: *BucketHeader, | |
| 307 | ret_addr: usize, | |
| 308 | slot_count: usize, | |
| 309 | slot_index: SlotIndex, | |
| 310 | trace_kind: TraceKind, | |
| 311 | ) void { | |
| 312 | // Initialize them to 0. When determining the count we must look | |
| 313 | // for non zero addresses. | |
| 314 | const stack_addresses = bucket.stackTracePtr(slot_count, slot_index, trace_kind); | |
| 315 | collectStackTrace(ret_addr, stack_addresses); | |
| 316 | } | |
| 317 | }; | |
| 318 | ||
| 319 | pub fn allocator(self: *Self) Allocator { | |
| 320 | return .{ | |
| 321 | .ptr = self, | |
| 322 | .vtable = &.{ | |
| 323 | .alloc = alloc, | |
| 324 | .resize = resize, | |
| 325 | .remap = remap, | |
| 326 | .free = free, | |
| 327 | }, | |
| 328 | }; | |
| 329 | } | |
| 330 | ||
| 331 | fn bucketStackTrace( | |
| 332 | bucket: *BucketHeader, | |
| 333 | slot_count: usize, | |
| 334 | slot_index: SlotIndex, | |
| 335 | trace_kind: TraceKind, | |
| 336 | ) StackTrace { | |
| 337 | const stack_addresses = bucket.stackTracePtr(slot_count, slot_index, trace_kind); | |
| 338 | var len: usize = 0; | |
| 339 | while (len < stack_n and stack_addresses[len] != 0) { | |
| 340 | len += 1; | |
| 341 | } | |
| 342 | return .{ | |
| 343 | .instruction_addresses = stack_addresses, | |
| 344 | .index = len, | |
| 345 | }; | |
| 346 | } | |
| 347 | ||
| 348 | fn bucketRequestedSizesStart(slot_count: usize) usize { | |
| 349 | if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled"); | |
| 350 | return mem.alignForward( | |
| 351 | usize, | |
| 352 | @sizeOf(BucketHeader) + usedBitsSize(slot_count), | |
| 353 | @alignOf(LargestSizeClassInt), | |
| 354 | ); | |
| 355 | } | |
| 356 | ||
| 357 | fn bucketAlignsStart(slot_count: usize) usize { | |
| 358 | if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled"); | |
| 359 | return bucketRequestedSizesStart(slot_count) + (@sizeOf(LargestSizeClassInt) * slot_count); | |
| 360 | } | |
| 361 | ||
| 362 | fn bucketStackFramesStart(slot_count: usize) usize { | |
| 363 | const unaligned_start = if (config.safety) | |
| 364 | bucketAlignsStart(slot_count) + slot_count | |
| 365 | else | |
| 366 | @sizeOf(BucketHeader) + usedBitsSize(slot_count); | |
| 367 | return mem.alignForward(usize, unaligned_start, @alignOf(usize)); | |
| 368 | } | |
| 369 | ||
| 370 | fn bucketSize(slot_count: usize) usize { | |
| 371 | return bucketStackFramesStart(slot_count) + one_trace_size * traces_per_slot * slot_count; | |
| 372 | } | |
| 373 | ||
| 374 | /// This is executed only at compile-time to prepopulate a lookup table. | |
| 375 | fn calculateSlotCount(size_class_index: usize) SlotIndex { | |
| 376 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | |
| 377 | var lower: usize = 1 << minimum_slots_per_bucket_log2; | |
| 378 | var upper: usize = (page_size - bucketSize(lower)) / size_class; | |
| 379 | while (upper > lower) { | |
| 380 | const proposed: usize = lower + (upper - lower) / 2; | |
| 381 | if (proposed == lower) return lower; | |
| 382 | const slots_end = proposed * size_class; | |
| 383 | const header_begin = mem.alignForward(usize, slots_end, @alignOf(BucketHeader)); | |
| 384 | const end = header_begin + bucketSize(proposed); | |
| 385 | if (end > page_size) { | |
| 386 | upper = proposed - 1; | |
| 387 | } else { | |
| 388 | lower = proposed; | |
| 389 | } | |
| 390 | } | |
| 391 | const slots_end = lower * size_class; | |
| 392 | const header_begin = mem.alignForward(usize, slots_end, @alignOf(BucketHeader)); | |
| 393 | const end = header_begin + bucketSize(lower); | |
| 394 | assert(end <= page_size); | |
| 395 | return lower; | |
| 396 | } | |
| 397 | ||
| 398 | fn usedBitsCount(slot_count: usize) usize { | |
| 399 | return (slot_count + (@bitSizeOf(usize) - 1)) / @bitSizeOf(usize); | |
| 400 | } | |
| 401 | ||
| 402 | fn usedBitsSize(slot_count: usize) usize { | |
| 403 | return usedBitsCount(slot_count) * @sizeOf(usize); | |
| 404 | } | |
| 405 | ||
| 406 | fn detectLeaksInBucket(bucket: *BucketHeader, size_class_index: usize, used_bits_count: usize) bool { | |
| 407 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | |
| 408 | const slot_count = slot_counts[size_class_index]; | |
| 409 | var leaks = false; | |
| 410 | for (0..used_bits_count) |used_bits_byte| { | |
| 411 | const used_int = bucket.usedBits(used_bits_byte).*; | |
| 412 | if (used_int != 0) { | |
| 413 | for (0..@bitSizeOf(usize)) |bit_index_usize| { | |
| 414 | const bit_index: Log2USize = @intCast(bit_index_usize); | |
| 415 | const is_used = @as(u1, @truncate(used_int >> bit_index)) != 0; | |
| 416 | if (is_used) { | |
| 417 | const slot_index: SlotIndex = @intCast(used_bits_byte * @bitSizeOf(usize) + bit_index); | |
| 418 | const stack_trace = bucketStackTrace(bucket, slot_count, slot_index, .alloc); | |
| 419 | const page_addr = @intFromPtr(bucket) & ~(page_size - 1); | |
| 420 | const addr = page_addr + slot_index * size_class; | |
| 421 | log.err("memory address 0x{x} leaked: {}", .{ addr, stack_trace }); | |
| 422 | leaks = true; | |
| 423 | } | |
| 424 | } | |
| 425 | } | |
| 426 | } | |
| 427 | return leaks; | |
| 428 | } | |
| 429 | ||
| 430 | /// Emits log messages for leaks and then returns whether there were any leaks. | |
| 431 | pub fn detectLeaks(self: *Self) bool { | |
| 432 | var leaks = false; | |
| 433 | ||
| 434 | for (self.buckets, 0..) |init_optional_bucket, size_class_index| { | |
| 435 | var optional_bucket = init_optional_bucket; | |
| 436 | const slot_count = slot_counts[size_class_index]; | |
| 437 | const used_bits_count = usedBitsCount(slot_count); | |
| 438 | while (optional_bucket) |bucket| { | |
| 439 | leaks = detectLeaksInBucket(bucket, size_class_index, used_bits_count) or leaks; | |
| 440 | optional_bucket = bucket.prev; | |
| 441 | } | |
| 442 | } | |
| 443 | ||
| 444 | var it = self.large_allocations.valueIterator(); | |
| 445 | while (it.next()) |large_alloc| { | |
| 446 | if (config.retain_metadata and large_alloc.freed) continue; | |
| 447 | const stack_trace = large_alloc.getStackTrace(.alloc); | |
| 448 | log.err("memory address 0x{x} leaked: {}", .{ | |
| 449 | @intFromPtr(large_alloc.bytes.ptr), stack_trace, | |
| 450 | }); | |
| 451 | leaks = true; | |
| 452 | } | |
| 453 | return leaks; | |
| 454 | } | |
| 455 | ||
| 456 | fn freeRetainedMetadata(self: *Self) void { | |
| 457 | comptime assert(config.retain_metadata); | |
| 458 | if (config.never_unmap) { | |
| 459 | // free large allocations that were intentionally leaked by never_unmap | |
| 460 | var it = self.large_allocations.iterator(); | |
| 461 | while (it.next()) |large| { | |
| 462 | if (large.value_ptr.freed) { | |
| 463 | self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.alignment, @returnAddress()); | |
| 464 | } | |
| 465 | } | |
| 466 | } | |
| 467 | } | |
| 468 | ||
| 469 | pub fn flushRetainedMetadata(self: *Self) void { | |
| 470 | comptime assert(config.retain_metadata); | |
| 471 | self.freeRetainedMetadata(); | |
| 472 | // also remove entries from large_allocations | |
| 473 | var it = self.large_allocations.iterator(); | |
| 474 | while (it.next()) |large| { | |
| 475 | if (large.value_ptr.freed) { | |
| 476 | _ = self.large_allocations.remove(@intFromPtr(large.value_ptr.bytes.ptr)); | |
| 477 | } | |
| 478 | } | |
| 479 | } | |
| 480 | ||
| 481 | /// Returns `std.heap.Check.leak` if there were leaks; `std.heap.Check.ok` otherwise. | |
| 482 | pub fn deinit(self: *Self) std.heap.Check { | |
| 483 | const leaks = if (config.safety) self.detectLeaks() else false; | |
| 484 | if (config.retain_metadata) self.freeRetainedMetadata(); | |
| 485 | self.large_allocations.deinit(self.backing_allocator); | |
| 486 | self.* = undefined; | |
| 487 | return if (leaks) .leak else .ok; | |
| 488 | } | |
| 489 | ||
| 490 | fn collectStackTrace(first_trace_addr: usize, addresses: *[stack_n]usize) void { | |
| 491 | if (stack_n == 0) return; | |
| 492 | @memset(addresses, 0); | |
| 493 | var stack_trace: StackTrace = .{ | |
| 494 | .instruction_addresses = addresses, | |
| 495 | .index = 0, | |
| 496 | }; | |
| 497 | std.debug.captureStackTrace(first_trace_addr, &stack_trace); | |
| 498 | } | |
| 499 | ||
| 500 | fn reportDoubleFree(ret_addr: usize, alloc_stack_trace: StackTrace, free_stack_trace: StackTrace) void { | |
| 501 | var addresses: [stack_n]usize = @splat(0); | |
| 502 | var second_free_stack_trace: StackTrace = .{ | |
| 503 | .instruction_addresses = &addresses, | |
| 504 | .index = 0, | |
| 505 | }; | |
| 506 | std.debug.captureStackTrace(ret_addr, &second_free_stack_trace); | |
| 507 | log.err("Double free detected. Allocation: {} First free: {} Second free: {}", .{ | |
| 508 | alloc_stack_trace, free_stack_trace, second_free_stack_trace, | |
| 509 | }); | |
| 510 | } | |
| 511 | ||
| 512 | /// This function assumes the object is in the large object storage regardless | |
| 513 | /// of the parameters. | |
| 514 | fn resizeLarge( | |
| 515 | self: *Self, | |
| 516 | old_mem: []u8, | |
| 517 | alignment: mem.Alignment, | |
| 518 | new_size: usize, | |
| 519 | ret_addr: usize, | |
| 520 | may_move: bool, | |
| 521 | ) ?[*]u8 { | |
| 522 | if (config.retain_metadata and may_move) { | |
| 523 | // Before looking up the entry (since this could invalidate | |
| 524 | // it), we must reserve space for the new entry in case the | |
| 525 | // allocation is relocated. | |
| 526 | self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1) catch return null; | |
| 527 | } | |
| 528 | ||
| 529 | const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse { | |
| 530 | if (config.safety) { | |
| 531 | @panic("Invalid free"); | |
| 532 | } else { | |
| 533 | unreachable; | |
| 534 | } | |
| 535 | }; | |
| 536 | ||
| 537 | if (config.retain_metadata and entry.value_ptr.freed) { | |
| 538 | if (config.safety) { | |
| 539 | reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free)); | |
| 540 | @panic("Unrecoverable double free"); | |
| 541 | } else { | |
| 542 | unreachable; | |
| 543 | } | |
| 544 | } | |
| 545 | ||
| 546 | if (config.safety and old_mem.len != entry.value_ptr.bytes.len) { | |
| 547 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; | |
| 548 | var free_stack_trace: StackTrace = .{ | |
| 549 | .instruction_addresses = &addresses, | |
| 550 | .index = 0, | |
| 551 | }; | |
| 552 | std.debug.captureStackTrace(ret_addr, &free_stack_trace); | |
| 553 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ | |
| 554 | entry.value_ptr.bytes.len, | |
| 555 | old_mem.len, | |
| 556 | entry.value_ptr.getStackTrace(.alloc), | |
| 557 | free_stack_trace, | |
| 558 | }); | |
| 559 | } | |
| 560 | ||
| 561 | // If this would move the allocation into a small size class, | |
| 562 | // refuse the request, because it would require creating small | |
| 563 | // allocation metadata. | |
| 564 | const new_size_class_index: usize = @max(@bitSizeOf(usize) - @clz(new_size - 1), @intFromEnum(alignment)); | |
| 565 | if (new_size_class_index < self.buckets.len) return null; | |
| 566 | ||
| 567 | // Do memory limit accounting with requested sizes rather than what | |
| 568 | // backing_allocator returns because if we want to return | |
| 569 | // error.OutOfMemory, we have to leave allocation untouched, and | |
| 570 | // that is impossible to guarantee after calling | |
| 571 | // backing_allocator.rawResize. | |
| 572 | const prev_req_bytes = self.total_requested_bytes; | |
| 573 | if (config.enable_memory_limit) { | |
| 574 | const new_req_bytes = prev_req_bytes + new_size - entry.value_ptr.requested_size; | |
| 575 | if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) { | |
| 576 | return null; | |
| 577 | } | |
| 578 | self.total_requested_bytes = new_req_bytes; | |
| 579 | } | |
| 580 | ||
| 581 | const opt_resized_ptr = if (may_move) | |
| 582 | self.backing_allocator.rawRemap(old_mem, alignment, new_size, ret_addr) | |
| 583 | else if (self.backing_allocator.rawResize(old_mem, alignment, new_size, ret_addr)) | |
| 584 | old_mem.ptr | |
| 585 | else | |
| 586 | null; | |
| 587 | ||
| 588 | const resized_ptr = opt_resized_ptr orelse { | |
| 589 | if (config.enable_memory_limit) { | |
| 590 | self.total_requested_bytes = prev_req_bytes; | |
| 591 | } | |
| 592 | return null; | |
| 593 | }; | |
| 594 | ||
| 595 | if (config.enable_memory_limit) { | |
| 596 | entry.value_ptr.requested_size = new_size; | |
| 597 | } | |
| 598 | ||
| 599 | if (config.verbose_log) { | |
| 600 | log.info("large resize {d} bytes at {*} to {d} at {*}", .{ | |
| 601 | old_mem.len, old_mem.ptr, new_size, resized_ptr, | |
| 602 | }); | |
| 603 | } | |
| 604 | entry.value_ptr.bytes = resized_ptr[0..new_size]; | |
| 605 | if (config.resize_stack_traces) | |
| 606 | entry.value_ptr.captureStackTrace(ret_addr, .alloc); | |
| 607 | ||
| 608 | // Update the key of the hash map if the memory was relocated. | |
| 609 | if (resized_ptr != old_mem.ptr) { | |
| 610 | const large_alloc = entry.value_ptr.*; | |
| 611 | if (config.retain_metadata) { | |
| 612 | entry.value_ptr.freed = true; | |
| 613 | entry.value_ptr.captureStackTrace(ret_addr, .free); | |
| 614 | } else { | |
| 615 | self.large_allocations.removeByPtr(entry.key_ptr); | |
| 616 | } | |
| 617 | ||
| 618 | const gop = self.large_allocations.getOrPutAssumeCapacity(@intFromPtr(resized_ptr)); | |
| 619 | if (config.retain_metadata and !config.never_unmap) { | |
| 620 | // Backing allocator may be reusing memory that we're retaining metadata for | |
| 621 | assert(!gop.found_existing or gop.value_ptr.freed); | |
| 622 | } else { | |
| 623 | assert(!gop.found_existing); // This would mean the kernel double-mapped pages. | |
| 624 | } | |
| 625 | gop.value_ptr.* = large_alloc; | |
| 626 | } | |
| 627 | ||
| 628 | return resized_ptr; | |
| 629 | } | |
| 630 | ||
| 631 | /// This function assumes the object is in the large object storage regardless | |
| 632 | /// of the parameters. | |
| 633 | fn freeLarge( | |
| 634 | self: *Self, | |
| 635 | old_mem: []u8, | |
| 636 | alignment: mem.Alignment, | |
| 637 | ret_addr: usize, | |
| 638 | ) void { | |
| 639 | const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse { | |
| 640 | if (config.safety) { | |
| 641 | @panic("Invalid free"); | |
| 642 | } else { | |
| 643 | unreachable; | |
| 644 | } | |
| 645 | }; | |
| 646 | ||
| 647 | if (config.retain_metadata and entry.value_ptr.freed) { | |
| 648 | if (config.safety) { | |
| 649 | reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free)); | |
| 650 | return; | |
| 651 | } else { | |
| 652 | unreachable; | |
| 653 | } | |
| 654 | } | |
| 655 | ||
| 656 | if (config.safety and old_mem.len != entry.value_ptr.bytes.len) { | |
| 657 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; | |
| 658 | var free_stack_trace = StackTrace{ | |
| 659 | .instruction_addresses = &addresses, | |
| 660 | .index = 0, | |
| 661 | }; | |
| 662 | std.debug.captureStackTrace(ret_addr, &free_stack_trace); | |
| 663 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ | |
| 664 | entry.value_ptr.bytes.len, | |
| 665 | old_mem.len, | |
| 666 | entry.value_ptr.getStackTrace(.alloc), | |
| 667 | free_stack_trace, | |
| 668 | }); | |
| 669 | } | |
| 670 | ||
| 671 | if (!config.never_unmap) { | |
| 672 | self.backing_allocator.rawFree(old_mem, alignment, ret_addr); | |
| 673 | } | |
| 674 | ||
| 675 | if (config.enable_memory_limit) { | |
| 676 | self.total_requested_bytes -= entry.value_ptr.requested_size; | |
| 677 | } | |
| 678 | ||
| 679 | if (config.verbose_log) { | |
| 680 | log.info("large free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr }); | |
| 681 | } | |
| 682 | ||
| 683 | if (!config.retain_metadata) { | |
| 684 | assert(self.large_allocations.remove(@intFromPtr(old_mem.ptr))); | |
| 685 | } else { | |
| 686 | entry.value_ptr.freed = true; | |
| 687 | entry.value_ptr.captureStackTrace(ret_addr, .free); | |
| 688 | } | |
| 689 | } | |
| 690 | ||
| 691 | fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ret_addr: usize) ?[*]u8 { | |
| 692 | const self: *Self = @ptrCast(@alignCast(context)); | |
| 693 | self.mutex.lock(); | |
| 694 | defer self.mutex.unlock(); | |
| 695 | ||
| 696 | if (config.enable_memory_limit) { | |
| 697 | const new_req_bytes = self.total_requested_bytes + len; | |
| 698 | if (new_req_bytes > self.requested_memory_limit) return null; | |
| 699 | self.total_requested_bytes = new_req_bytes; | |
| 700 | } | |
| 701 | ||
| 702 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(len - 1), @intFromEnum(alignment)); | |
| 703 | if (size_class_index >= self.buckets.len) { | |
| 704 | @branchHint(.unlikely); | |
| 705 | self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1) catch return null; | |
| 706 | const ptr = self.backing_allocator.rawAlloc(len, alignment, ret_addr) orelse return null; | |
| 707 | const slice = ptr[0..len]; | |
| 708 | ||
| 709 | const gop = self.large_allocations.getOrPutAssumeCapacity(@intFromPtr(slice.ptr)); | |
| 710 | if (config.retain_metadata and !config.never_unmap) { | |
| 711 | // Backing allocator may be reusing memory that we're retaining metadata for | |
| 712 | assert(!gop.found_existing or gop.value_ptr.freed); | |
| 713 | } else { | |
| 714 | assert(!gop.found_existing); // This would mean the kernel double-mapped pages. | |
| 715 | } | |
| 716 | gop.value_ptr.bytes = slice; | |
| 717 | if (config.enable_memory_limit) | |
| 718 | gop.value_ptr.requested_size = len; | |
| 719 | gop.value_ptr.captureStackTrace(ret_addr, .alloc); | |
| 720 | if (config.retain_metadata) { | |
| 721 | gop.value_ptr.freed = false; | |
| 722 | if (config.never_unmap) { | |
| 723 | gop.value_ptr.alignment = alignment; | |
| 724 | } | |
| 725 | } | |
| 726 | ||
| 727 | if (config.verbose_log) { | |
| 728 | log.info("large alloc {d} bytes at {*}", .{ slice.len, slice.ptr }); | |
| 729 | } | |
| 730 | return slice.ptr; | |
| 731 | } | |
| 732 | ||
| 733 | const slot_count = slot_counts[size_class_index]; | |
| 734 | ||
| 735 | if (self.buckets[size_class_index]) |bucket| { | |
| 736 | @branchHint(.likely); | |
| 737 | const slot_index = bucket.allocated_count; | |
| 738 | if (slot_index < slot_count) { | |
| 739 | @branchHint(.likely); | |
| 740 | bucket.allocated_count = slot_index + 1; | |
| 741 | const used_bits_byte = bucket.usedBits(slot_index / @bitSizeOf(usize)); | |
| 742 | const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize)); | |
| 743 | used_bits_byte.* |= (@as(usize, 1) << used_bit_index); | |
| 744 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | |
| 745 | if (config.stack_trace_frames > 0) { | |
| 746 | bucket.captureStackTrace(ret_addr, slot_count, slot_index, .alloc); | |
| 747 | } | |
| 748 | if (config.safety) { | |
| 749 | bucket.requestedSizes(slot_count)[slot_index] = @intCast(len); | |
| 750 | bucket.log2PtrAligns(slot_count)[slot_index] = alignment; | |
| 751 | } | |
| 752 | const page_addr = @intFromPtr(bucket) & ~(page_size - 1); | |
| 753 | const addr = page_addr + slot_index * size_class; | |
| 754 | if (config.verbose_log) { | |
| 755 | log.info("small alloc {d} bytes at 0x{x}", .{ len, addr }); | |
| 756 | } | |
| 757 | return @ptrFromInt(addr); | |
| 758 | } | |
| 759 | } | |
| 760 | ||
| 761 | const page = self.backing_allocator.rawAlloc(page_size, page_align, @returnAddress()) orelse | |
| 762 | return null; | |
| 763 | const bucket: *BucketHeader = .fromPage(@intFromPtr(page), slot_count); | |
| 764 | bucket.* = .{ | |
| 765 | .allocated_count = 1, | |
| 766 | .freed_count = 0, | |
| 767 | .prev = self.buckets[size_class_index], | |
| 768 | }; | |
| 769 | self.buckets[size_class_index] = bucket; | |
| 770 | ||
| 771 | if (!config.backing_allocator_zeroes) { | |
| 772 | @memset(@as([*]usize, @as(*[1]usize, bucket.usedBits(0)))[0..usedBitsCount(slot_count)], 0); | |
| 773 | if (config.safety) @memset(bucket.requestedSizes(slot_count), 0); | |
| 774 | } | |
| 775 | ||
| 776 | bucket.usedBits(0).* = 0b1; | |
| 777 | ||
| 778 | if (config.stack_trace_frames > 0) { | |
| 779 | bucket.captureStackTrace(ret_addr, slot_count, 0, .alloc); | |
| 780 | } | |
| 781 | ||
| 782 | if (config.safety) { | |
| 783 | bucket.requestedSizes(slot_count)[0] = @intCast(len); | |
| 784 | bucket.log2PtrAligns(slot_count)[0] = alignment; | |
| 785 | } | |
| 786 | ||
| 787 | if (config.verbose_log) { | |
| 788 | log.info("small alloc {d} bytes at 0x{x}", .{ len, @intFromPtr(page) }); | |
| 789 | } | |
| 790 | ||
| 791 | return page; | |
| 792 | } | |
| 793 | ||
| 794 | fn resize( | |
| 795 | context: *anyopaque, | |
| 796 | memory: []u8, | |
| 797 | alignment: mem.Alignment, | |
| 798 | new_len: usize, | |
| 799 | return_address: usize, | |
| 800 | ) bool { | |
| 801 | const self: *Self = @ptrCast(@alignCast(context)); | |
| 802 | self.mutex.lock(); | |
| 803 | defer self.mutex.unlock(); | |
| 804 | ||
| 805 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment)); | |
| 806 | if (size_class_index >= self.buckets.len) { | |
| 807 | return self.resizeLarge(memory, alignment, new_len, return_address, false) != null; | |
| 808 | } else { | |
| 809 | return resizeSmall(self, memory, alignment, new_len, return_address, size_class_index); | |
| 810 | } | |
| 811 | } | |
| 812 | ||
| 813 | fn remap( | |
| 814 | context: *anyopaque, | |
| 815 | memory: []u8, | |
| 816 | alignment: mem.Alignment, | |
| 817 | new_len: usize, | |
| 818 | return_address: usize, | |
| 819 | ) ?[*]u8 { | |
| 820 | const self: *Self = @ptrCast(@alignCast(context)); | |
| 821 | self.mutex.lock(); | |
| 822 | defer self.mutex.unlock(); | |
| 823 | ||
| 824 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment)); | |
| 825 | if (size_class_index >= self.buckets.len) { | |
| 826 | return self.resizeLarge(memory, alignment, new_len, return_address, true); | |
| 827 | } else { | |
| 828 | return if (resizeSmall(self, memory, alignment, new_len, return_address, size_class_index)) memory.ptr else null; | |
| 829 | } | |
| 830 | } | |
| 831 | ||
| 832 | fn free( | |
| 833 | context: *anyopaque, | |
| 834 | old_memory: []u8, | |
| 835 | alignment: mem.Alignment, | |
| 836 | return_address: usize, | |
| 837 | ) void { | |
| 838 | const self: *Self = @ptrCast(@alignCast(context)); | |
| 839 | self.mutex.lock(); | |
| 840 | defer self.mutex.unlock(); | |
| 841 | ||
| 842 | assert(old_memory.len != 0); | |
| 843 | ||
| 844 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(old_memory.len - 1), @intFromEnum(alignment)); | |
| 845 | if (size_class_index >= self.buckets.len) { | |
| 846 | @branchHint(.unlikely); | |
| 847 | self.freeLarge(old_memory, alignment, return_address); | |
| 848 | return; | |
| 849 | } | |
| 850 | ||
| 851 | const slot_count = slot_counts[size_class_index]; | |
| 852 | const freed_addr = @intFromPtr(old_memory.ptr); | |
| 853 | const page_addr = freed_addr & ~(page_size - 1); | |
| 854 | const bucket: *BucketHeader = .fromPage(page_addr, slot_count); | |
| 855 | if (bucket.canary != config.canary) @panic("Invalid free"); | |
| 856 | const page_offset = freed_addr - page_addr; | |
| 857 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | |
| 858 | const slot_index: SlotIndex = @intCast(page_offset / size_class); | |
| 859 | const used_byte_index = slot_index / @bitSizeOf(usize); | |
| 860 | const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize)); | |
| 861 | const used_byte = bucket.usedBits(used_byte_index); | |
| 862 | const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0; | |
| 863 | if (!is_used) { | |
| 864 | if (config.safety) { | |
| 865 | reportDoubleFree( | |
| 866 | return_address, | |
| 867 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | |
| 868 | bucketStackTrace(bucket, slot_count, slot_index, .free), | |
| 869 | ); | |
| 870 | // Recoverable since this is a free. | |
| 871 | return; | |
| 872 | } else { | |
| 873 | unreachable; | |
| 874 | } | |
| 875 | } | |
| 876 | ||
| 877 | // Definitely an in-use small alloc now. | |
| 878 | if (config.safety) { | |
| 879 | const requested_size = bucket.requestedSizes(slot_count)[slot_index]; | |
| 880 | if (requested_size == 0) @panic("Invalid free"); | |
| 881 | const slot_alignment = bucket.log2PtrAligns(slot_count)[slot_index]; | |
| 882 | if (old_memory.len != requested_size or alignment != slot_alignment) { | |
| 883 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; | |
| 884 | var free_stack_trace: StackTrace = .{ | |
| 885 | .instruction_addresses = &addresses, | |
| 886 | .index = 0, | |
| 887 | }; | |
| 888 | std.debug.captureStackTrace(return_address, &free_stack_trace); | |
| 889 | if (old_memory.len != requested_size) { | |
| 890 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ | |
| 891 | requested_size, | |
| 892 | old_memory.len, | |
| 893 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | |
| 894 | free_stack_trace, | |
| 895 | }); | |
| 896 | } | |
| 897 | if (alignment != slot_alignment) { | |
| 898 | log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{ | |
| 899 | slot_alignment.toByteUnits(), | |
| 900 | alignment.toByteUnits(), | |
| 901 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | |
| 902 | free_stack_trace, | |
| 903 | }); | |
| 904 | } | |
| 905 | } | |
| 906 | } | |
| 907 | ||
| 908 | if (config.enable_memory_limit) { | |
| 909 | self.total_requested_bytes -= old_memory.len; | |
| 910 | } | |
| 911 | ||
| 912 | if (config.stack_trace_frames > 0) { | |
| 913 | // Capture stack trace to be the "first free", in case a double free happens. | |
| 914 | bucket.captureStackTrace(return_address, slot_count, slot_index, .free); | |
| 915 | } | |
| 916 | ||
| 917 | used_byte.* &= ~(@as(usize, 1) << used_bit_index); | |
| 918 | if (config.safety) { | |
| 919 | bucket.requestedSizes(slot_count)[slot_index] = 0; | |
| 920 | } | |
| 921 | bucket.freed_count += 1; | |
| 922 | if (bucket.freed_count == bucket.allocated_count) { | |
| 923 | if (self.buckets[size_class_index] == bucket) { | |
| 924 | self.buckets[size_class_index] = null; | |
| 925 | } | |
| 926 | if (!config.never_unmap) { | |
| 927 | const page: [*]align(page_size) u8 = @ptrFromInt(page_addr); | |
| 928 | self.backing_allocator.rawFree(page[0..page_size], page_align, @returnAddress()); | |
| 929 | } | |
| 930 | } | |
| 931 | if (config.verbose_log) { | |
| 932 | log.info("small free {d} bytes at {*}", .{ old_memory.len, old_memory.ptr }); | |
| 933 | } | |
| 934 | } | |
| 935 | ||
| 936 | fn resizeSmall( | |
| 937 | self: *Self, | |
| 938 | memory: []u8, | |
| 939 | alignment: mem.Alignment, | |
| 940 | new_len: usize, | |
| 941 | return_address: usize, | |
| 942 | size_class_index: usize, | |
| 943 | ) bool { | |
| 944 | const new_size_class_index: usize = @max(@bitSizeOf(usize) - @clz(new_len - 1), @intFromEnum(alignment)); | |
| 945 | if (!config.safety) return new_size_class_index == size_class_index; | |
| 946 | const slot_count = slot_counts[size_class_index]; | |
| 947 | const memory_addr = @intFromPtr(memory.ptr); | |
| 948 | const page_addr = memory_addr & ~(page_size - 1); | |
| 949 | const bucket: *BucketHeader = .fromPage(page_addr, slot_count); | |
| 950 | if (bucket.canary != config.canary) @panic("Invalid free"); | |
| 951 | const page_offset = memory_addr - page_addr; | |
| 952 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | |
| 953 | const slot_index: SlotIndex = @intCast(page_offset / size_class); | |
| 954 | const used_byte_index = slot_index / @bitSizeOf(usize); | |
| 955 | const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize)); | |
| 956 | const used_byte = bucket.usedBits(used_byte_index); | |
| 957 | const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0; | |
| 958 | if (!is_used) { | |
| 959 | reportDoubleFree( | |
| 960 | return_address, | |
| 961 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | |
| 962 | bucketStackTrace(bucket, slot_count, slot_index, .free), | |
| 963 | ); | |
| 964 | // Recoverable since this is a free. | |
| 965 | return false; | |
| 966 | } | |
| 967 | ||
| 968 | // Definitely an in-use small alloc now. | |
| 969 | const requested_size = bucket.requestedSizes(slot_count)[slot_index]; | |
| 970 | if (requested_size == 0) @panic("Invalid free"); | |
| 971 | const slot_alignment = bucket.log2PtrAligns(slot_count)[slot_index]; | |
| 972 | if (memory.len != requested_size or alignment != slot_alignment) { | |
| 973 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; | |
| 974 | var free_stack_trace: StackTrace = .{ | |
| 975 | .instruction_addresses = &addresses, | |
| 976 | .index = 0, | |
| 977 | }; | |
| 978 | std.debug.captureStackTrace(return_address, &free_stack_trace); | |
| 979 | if (memory.len != requested_size) { | |
| 980 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ | |
| 981 | requested_size, | |
| 982 | memory.len, | |
| 983 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | |
| 984 | free_stack_trace, | |
| 985 | }); | |
| 986 | } | |
| 987 | if (alignment != slot_alignment) { | |
| 988 | log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{ | |
| 989 | slot_alignment.toByteUnits(), | |
| 990 | alignment.toByteUnits(), | |
| 991 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | |
| 992 | free_stack_trace, | |
| 993 | }); | |
| 994 | } | |
| 995 | } | |
| 996 | ||
| 997 | if (new_size_class_index != size_class_index) return false; | |
| 998 | ||
| 999 | const prev_req_bytes = self.total_requested_bytes; | |
| 1000 | if (config.enable_memory_limit) { | |
| 1001 | const new_req_bytes = prev_req_bytes - memory.len + new_len; | |
| 1002 | if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) { | |
| 1003 | return false; | |
| 1004 | } | |
| 1005 | self.total_requested_bytes = new_req_bytes; | |
| 1006 | } | |
| 1007 | ||
| 1008 | if (memory.len > new_len) @memset(memory[new_len..], undefined); | |
| 1009 | if (config.verbose_log) | |
| 1010 | log.info("small resize {d} bytes at {*} to {d}", .{ memory.len, memory.ptr, new_len }); | |
| 1011 | ||
| 1012 | if (config.safety) | |
| 1013 | bucket.requestedSizes(slot_count)[slot_index] = @intCast(new_len); | |
| 1014 | ||
| 1015 | if (config.resize_stack_traces) | |
| 1016 | bucket.captureStackTrace(return_address, slot_count, slot_index, .alloc); | |
| 1017 | ||
| 1018 | return true; | |
| 1019 | } | |
| 1020 | }; | |
| 1021 | } | |
| 1022 | ||
| 1023 | const TraceKind = enum { | |
| 1024 | alloc, | |
| 1025 | free, | |
| 1026 | }; | |
| 1027 | ||
| 1028 | const test_config = Config{}; | |
| 1029 | ||
| 1030 | test "small allocations - free in same order" { | |
| 1031 | var gpa = DebugAllocator(test_config){}; | |
| 1032 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1033 | const allocator = gpa.allocator(); | |
| 1034 | ||
| 1035 | var list = std.ArrayList(*u64).init(std.testing.allocator); | |
| 1036 | defer list.deinit(); | |
| 1037 | ||
| 1038 | var i: usize = 0; | |
| 1039 | while (i < 513) : (i += 1) { | |
| 1040 | const ptr = try allocator.create(u64); | |
| 1041 | try list.append(ptr); | |
| 1042 | } | |
| 1043 | ||
| 1044 | for (list.items) |ptr| { | |
| 1045 | allocator.destroy(ptr); | |
| 1046 | } | |
| 1047 | } | |
| 1048 | ||
| 1049 | test "small allocations - free in reverse order" { | |
| 1050 | var gpa = DebugAllocator(test_config){}; | |
| 1051 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1052 | const allocator = gpa.allocator(); | |
| 1053 | ||
| 1054 | var list = std.ArrayList(*u64).init(std.testing.allocator); | |
| 1055 | defer list.deinit(); | |
| 1056 | ||
| 1057 | var i: usize = 0; | |
| 1058 | while (i < 513) : (i += 1) { | |
| 1059 | const ptr = try allocator.create(u64); | |
| 1060 | try list.append(ptr); | |
| 1061 | } | |
| 1062 | ||
| 1063 | while (list.popOrNull()) |ptr| { | |
| 1064 | allocator.destroy(ptr); | |
| 1065 | } | |
| 1066 | } | |
| 1067 | ||
| 1068 | test "large allocations" { | |
| 1069 | var gpa = DebugAllocator(test_config){}; | |
| 1070 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1071 | const allocator = gpa.allocator(); | |
| 1072 | ||
| 1073 | const ptr1 = try allocator.alloc(u64, 42768); | |
| 1074 | const ptr2 = try allocator.alloc(u64, 52768); | |
| 1075 | allocator.free(ptr1); | |
| 1076 | const ptr3 = try allocator.alloc(u64, 62768); | |
| 1077 | allocator.free(ptr3); | |
| 1078 | allocator.free(ptr2); | |
| 1079 | } | |
| 1080 | ||
| 1081 | test "very large allocation" { | |
| 1082 | var gpa = DebugAllocator(test_config){}; | |
| 1083 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1084 | const allocator = gpa.allocator(); | |
| 1085 | ||
| 1086 | try std.testing.expectError(error.OutOfMemory, allocator.alloc(u8, math.maxInt(usize))); | |
| 1087 | } | |
| 1088 | ||
| 1089 | test "realloc" { | |
| 1090 | var gpa = DebugAllocator(test_config){}; | |
| 1091 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1092 | const allocator = gpa.allocator(); | |
| 1093 | ||
| 1094 | var slice = try allocator.alignedAlloc(u8, @alignOf(u32), 1); | |
| 1095 | defer allocator.free(slice); | |
| 1096 | slice[0] = 0x12; | |
| 1097 | ||
| 1098 | // This reallocation should keep its pointer address. | |
| 1099 | const old_slice = slice; | |
| 1100 | slice = try allocator.realloc(slice, 2); | |
| 1101 | try std.testing.expect(old_slice.ptr == slice.ptr); | |
| 1102 | try std.testing.expect(slice[0] == 0x12); | |
| 1103 | slice[1] = 0x34; | |
| 1104 | ||
| 1105 | // This requires upgrading to a larger size class | |
| 1106 | slice = try allocator.realloc(slice, 17); | |
| 1107 | try std.testing.expect(slice[0] == 0x12); | |
| 1108 | try std.testing.expect(slice[1] == 0x34); | |
| 1109 | } | |
| 1110 | ||
| 1111 | test "shrink" { | |
| 1112 | var gpa: DebugAllocator(test_config) = .{}; | |
| 1113 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1114 | const allocator = gpa.allocator(); | |
| 1115 | ||
| 1116 | var slice = try allocator.alloc(u8, 20); | |
| 1117 | defer allocator.free(slice); | |
| 1118 | ||
| 1119 | @memset(slice, 0x11); | |
| 1120 | ||
| 1121 | try std.testing.expect(allocator.resize(slice, 17)); | |
| 1122 | slice = slice[0..17]; | |
| 1123 | ||
| 1124 | for (slice) |b| { | |
| 1125 | try std.testing.expect(b == 0x11); | |
| 1126 | } | |
| 1127 | ||
| 1128 | // Does not cross size class boundaries when shrinking. | |
| 1129 | try std.testing.expect(!allocator.resize(slice, 16)); | |
| 1130 | } | |
| 1131 | ||
| 1132 | test "large object - grow" { | |
| 1133 | if (builtin.target.isWasm()) { | |
| 1134 | // Not expected to pass on targets that do not have memory mapping. | |
| 1135 | return error.SkipZigTest; | |
| 1136 | } | |
| 1137 | var gpa: DebugAllocator(test_config) = .{}; | |
| 1138 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1139 | const allocator = gpa.allocator(); | |
| 1140 | ||
| 1141 | var slice1 = try allocator.alloc(u8, page_size * 2 - 20); | |
| 1142 | defer allocator.free(slice1); | |
| 1143 | ||
| 1144 | const old = slice1; | |
| 1145 | slice1 = try allocator.realloc(slice1, page_size * 2 - 10); | |
| 1146 | try std.testing.expect(slice1.ptr == old.ptr); | |
| 1147 | ||
| 1148 | slice1 = try allocator.realloc(slice1, page_size * 2); | |
| 1149 | try std.testing.expect(slice1.ptr == old.ptr); | |
| 1150 | ||
| 1151 | slice1 = try allocator.realloc(slice1, page_size * 2 + 1); | |
| 1152 | } | |
| 1153 | ||
| 1154 | test "realloc small object to large object" { | |
| 1155 | var gpa = DebugAllocator(test_config){}; | |
| 1156 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1157 | const allocator = gpa.allocator(); | |
| 1158 | ||
| 1159 | var slice = try allocator.alloc(u8, 70); | |
| 1160 | defer allocator.free(slice); | |
| 1161 | slice[0] = 0x12; | |
| 1162 | slice[60] = 0x34; | |
| 1163 | ||
| 1164 | // This requires upgrading to a large object | |
| 1165 | const large_object_size = page_size * 2 + 50; | |
| 1166 | slice = try allocator.realloc(slice, large_object_size); | |
| 1167 | try std.testing.expect(slice[0] == 0x12); | |
| 1168 | try std.testing.expect(slice[60] == 0x34); | |
| 1169 | } | |
| 1170 | ||
| 1171 | test "shrink large object to large object" { | |
| 1172 | var gpa: DebugAllocator(test_config) = .{}; | |
| 1173 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1174 | const allocator = gpa.allocator(); | |
| 1175 | ||
| 1176 | var slice = try allocator.alloc(u8, page_size * 2 + 50); | |
| 1177 | defer allocator.free(slice); | |
| 1178 | slice[0] = 0x12; | |
| 1179 | slice[60] = 0x34; | |
| 1180 | ||
| 1181 | if (!allocator.resize(slice, page_size * 2 + 1)) return; | |
| 1182 | slice = slice.ptr[0 .. page_size * 2 + 1]; | |
| 1183 | try std.testing.expect(slice[0] == 0x12); | |
| 1184 | try std.testing.expect(slice[60] == 0x34); | |
| 1185 | ||
| 1186 | try std.testing.expect(allocator.resize(slice, page_size * 2 + 1)); | |
| 1187 | slice = slice[0 .. page_size * 2 + 1]; | |
| 1188 | try std.testing.expect(slice[0] == 0x12); | |
| 1189 | try std.testing.expect(slice[60] == 0x34); | |
| 1190 | ||
| 1191 | slice = try allocator.realloc(slice, page_size * 2); | |
| 1192 | try std.testing.expect(slice[0] == 0x12); | |
| 1193 | try std.testing.expect(slice[60] == 0x34); | |
| 1194 | } | |
| 1195 | ||
| 1196 | test "shrink large object to large object with larger alignment" { | |
| 1197 | if (!builtin.link_libc and builtin.os.tag == .wasi) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22731 | |
| 1198 | ||
| 1199 | var gpa = DebugAllocator(test_config){}; | |
| 1200 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1201 | const allocator = gpa.allocator(); | |
| 1202 | ||
| 1203 | var debug_buffer: [1000]u8 = undefined; | |
| 1204 | var fba = std.heap.FixedBufferAllocator.init(&debug_buffer); | |
| 1205 | const debug_allocator = fba.allocator(); | |
| 1206 | ||
| 1207 | const alloc_size = page_size * 2 + 50; | |
| 1208 | var slice = try allocator.alignedAlloc(u8, 16, alloc_size); | |
| 1209 | defer allocator.free(slice); | |
| 1210 | ||
| 1211 | const big_alignment: usize = switch (builtin.os.tag) { | |
| 1212 | .windows => page_size * 32, // Windows aligns to 64K. | |
| 1213 | else => page_size * 2, | |
| 1214 | }; | |
| 1215 | // This loop allocates until we find a page that is not aligned to the big | |
| 1216 | // alignment. Then we shrink the allocation after the loop, but increase the | |
| 1217 | // alignment to the higher one, that we know will force it to realloc. | |
| 1218 | var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator); | |
| 1219 | while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) { | |
| 1220 | try stuff_to_free.append(slice); | |
| 1221 | slice = try allocator.alignedAlloc(u8, 16, alloc_size); | |
| 1222 | } | |
| 1223 | while (stuff_to_free.popOrNull()) |item| { | |
| 1224 | allocator.free(item); | |
| 1225 | } | |
| 1226 | slice[0] = 0x12; | |
| 1227 | slice[60] = 0x34; | |
| 1228 | ||
| 1229 | slice = try allocator.reallocAdvanced(slice, big_alignment, alloc_size / 2); | |
| 1230 | try std.testing.expect(slice[0] == 0x12); | |
| 1231 | try std.testing.expect(slice[60] == 0x34); | |
| 1232 | } | |
| 1233 | ||
| 1234 | test "realloc large object to small object" { | |
| 1235 | var gpa = DebugAllocator(test_config){}; | |
| 1236 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1237 | const allocator = gpa.allocator(); | |
| 1238 | ||
| 1239 | var slice = try allocator.alloc(u8, page_size * 2 + 50); | |
| 1240 | defer allocator.free(slice); | |
| 1241 | slice[0] = 0x12; | |
| 1242 | slice[16] = 0x34; | |
| 1243 | ||
| 1244 | slice = try allocator.realloc(slice, 19); | |
| 1245 | try std.testing.expect(slice[0] == 0x12); | |
| 1246 | try std.testing.expect(slice[16] == 0x34); | |
| 1247 | } | |
| 1248 | ||
| 1249 | test "overridable mutexes" { | |
| 1250 | var gpa = DebugAllocator(.{ .MutexType = std.Thread.Mutex }){ | |
| 1251 | .backing_allocator = std.testing.allocator, | |
| 1252 | .mutex = std.Thread.Mutex{}, | |
| 1253 | }; | |
| 1254 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1255 | const allocator = gpa.allocator(); | |
| 1256 | ||
| 1257 | const ptr = try allocator.create(i32); | |
| 1258 | defer allocator.destroy(ptr); | |
| 1259 | } | |
| 1260 | ||
| 1261 | test "non-page-allocator backing allocator" { | |
| 1262 | var gpa: DebugAllocator(.{ | |
| 1263 | .backing_allocator_zeroes = false, | |
| 1264 | }) = .{ | |
| 1265 | .backing_allocator = std.testing.allocator, | |
| 1266 | }; | |
| 1267 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1268 | const allocator = gpa.allocator(); | |
| 1269 | ||
| 1270 | const ptr = try allocator.create(i32); | |
| 1271 | defer allocator.destroy(ptr); | |
| 1272 | } | |
| 1273 | ||
| 1274 | test "realloc large object to larger alignment" { | |
| 1275 | if (!builtin.link_libc and builtin.os.tag == .wasi) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22731 | |
| 1276 | ||
| 1277 | var gpa = DebugAllocator(test_config){}; | |
| 1278 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1279 | const allocator = gpa.allocator(); | |
| 1280 | ||
| 1281 | var debug_buffer: [1000]u8 = undefined; | |
| 1282 | var fba = std.heap.FixedBufferAllocator.init(&debug_buffer); | |
| 1283 | const debug_allocator = fba.allocator(); | |
| 1284 | ||
| 1285 | var slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50); | |
| 1286 | defer allocator.free(slice); | |
| 1287 | ||
| 1288 | const big_alignment: usize = switch (builtin.os.tag) { | |
| 1289 | .windows => page_size * 32, // Windows aligns to 64K. | |
| 1290 | else => page_size * 2, | |
| 1291 | }; | |
| 1292 | // This loop allocates until we find a page that is not aligned to the big alignment. | |
| 1293 | var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator); | |
| 1294 | while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) { | |
| 1295 | try stuff_to_free.append(slice); | |
| 1296 | slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50); | |
| 1297 | } | |
| 1298 | while (stuff_to_free.popOrNull()) |item| { | |
| 1299 | allocator.free(item); | |
| 1300 | } | |
| 1301 | slice[0] = 0x12; | |
| 1302 | slice[16] = 0x34; | |
| 1303 | ||
| 1304 | slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 100); | |
| 1305 | try std.testing.expect(slice[0] == 0x12); | |
| 1306 | try std.testing.expect(slice[16] == 0x34); | |
| 1307 | ||
| 1308 | slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 25); | |
| 1309 | try std.testing.expect(slice[0] == 0x12); | |
| 1310 | try std.testing.expect(slice[16] == 0x34); | |
| 1311 | ||
| 1312 | slice = try allocator.reallocAdvanced(slice, big_alignment, page_size * 2 + 100); | |
| 1313 | try std.testing.expect(slice[0] == 0x12); | |
| 1314 | try std.testing.expect(slice[16] == 0x34); | |
| 1315 | } | |
| 1316 | ||
| 1317 | test "large object rejects shrinking to small" { | |
| 1318 | if (builtin.target.isWasm()) { | |
| 1319 | // Not expected to pass on targets that do not have memory mapping. | |
| 1320 | return error.SkipZigTest; | |
| 1321 | } | |
| 1322 | ||
| 1323 | var failing_allocator = std.testing.FailingAllocator.init(std.heap.page_allocator, .{ .fail_index = 3 }); | |
| 1324 | var gpa: DebugAllocator(.{}) = .{ | |
| 1325 | .backing_allocator = failing_allocator.allocator(), | |
| 1326 | }; | |
| 1327 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1328 | const allocator = gpa.allocator(); | |
| 1329 | ||
| 1330 | var slice = try allocator.alloc(u8, page_size * 2 + 50); | |
| 1331 | defer allocator.free(slice); | |
| 1332 | slice[0] = 0x12; | |
| 1333 | slice[3] = 0x34; | |
| 1334 | ||
| 1335 | try std.testing.expect(!allocator.resize(slice, 4)); | |
| 1336 | try std.testing.expect(slice[0] == 0x12); | |
| 1337 | try std.testing.expect(slice[3] == 0x34); | |
| 1338 | } | |
| 1339 | ||
| 1340 | test "objects of size 1024 and 2048" { | |
| 1341 | var gpa = DebugAllocator(test_config){}; | |
| 1342 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1343 | const allocator = gpa.allocator(); | |
| 1344 | ||
| 1345 | const slice = try allocator.alloc(u8, 1025); | |
| 1346 | const slice2 = try allocator.alloc(u8, 3000); | |
| 1347 | ||
| 1348 | allocator.free(slice); | |
| 1349 | allocator.free(slice2); | |
| 1350 | } | |
| 1351 | ||
| 1352 | test "setting a memory cap" { | |
| 1353 | var gpa = DebugAllocator(.{ .enable_memory_limit = true }){}; | |
| 1354 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1355 | const allocator = gpa.allocator(); | |
| 1356 | ||
| 1357 | gpa.requested_memory_limit = 1010; | |
| 1358 | ||
| 1359 | const small = try allocator.create(i32); | |
| 1360 | try std.testing.expect(gpa.total_requested_bytes == 4); | |
| 1361 | ||
| 1362 | const big = try allocator.alloc(u8, 1000); | |
| 1363 | try std.testing.expect(gpa.total_requested_bytes == 1004); | |
| 1364 | ||
| 1365 | try std.testing.expectError(error.OutOfMemory, allocator.create(u64)); | |
| 1366 | ||
| 1367 | allocator.destroy(small); | |
| 1368 | try std.testing.expect(gpa.total_requested_bytes == 1000); | |
| 1369 | ||
| 1370 | allocator.free(big); | |
| 1371 | try std.testing.expect(gpa.total_requested_bytes == 0); | |
| 1372 | ||
| 1373 | const exact = try allocator.alloc(u8, 1010); | |
| 1374 | try std.testing.expect(gpa.total_requested_bytes == 1010); | |
| 1375 | allocator.free(exact); | |
| 1376 | } | |
| 1377 | ||
| 1378 | test "large allocations count requested size not backing size" { | |
| 1379 | var gpa: DebugAllocator(.{ .enable_memory_limit = true }) = .{}; | |
| 1380 | const allocator = gpa.allocator(); | |
| 1381 | ||
| 1382 | var buf = try allocator.alignedAlloc(u8, 1, page_size + 1); | |
| 1383 | try std.testing.expectEqual(page_size + 1, gpa.total_requested_bytes); | |
| 1384 | buf = try allocator.realloc(buf, 1); | |
| 1385 | try std.testing.expectEqual(1, gpa.total_requested_bytes); | |
| 1386 | buf = try allocator.realloc(buf, 2); | |
| 1387 | try std.testing.expectEqual(2, gpa.total_requested_bytes); | |
| 1388 | } | |
| 1389 | ||
| 1390 | test "retain metadata and never unmap" { | |
| 1391 | var gpa = std.heap.DebugAllocator(.{ | |
| 1392 | .safety = true, | |
| 1393 | .never_unmap = true, | |
| 1394 | .retain_metadata = true, | |
| 1395 | }){}; | |
| 1396 | defer std.debug.assert(gpa.deinit() == .ok); | |
| 1397 | const allocator = gpa.allocator(); | |
| 1398 | ||
| 1399 | const alloc = try allocator.alloc(u8, 8); | |
| 1400 | allocator.free(alloc); | |
| 1401 | ||
| 1402 | const alloc2 = try allocator.alloc(u8, 8); | |
| 1403 | allocator.free(alloc2); | |
| 1404 | } |
lib/std/heap/general_purpose_allocator.zig deleted-1411
| ... | ... | @@ -1,1411 +0,0 @@ |
| 1 | //! # General Purpose Allocator | |
| 2 | //! | |
| 3 | //! ## Design Priorities | |
| 4 | //! | |
| 5 | //! ### `OptimizationMode.debug` and `OptimizationMode.release_safe`: | |
| 6 | //! | |
| 7 | //! * Detect double free, and emit stack trace of: | |
| 8 | //! - Where it was first allocated | |
| 9 | //! - Where it was freed the first time | |
| 10 | //! - Where it was freed the second time | |
| 11 | //! | |
| 12 | //! * Detect leaks and emit stack trace of: | |
| 13 | //! - Where it was allocated | |
| 14 | //! | |
| 15 | //! * When a page of memory is no longer needed, give it back to resident memory | |
| 16 | //! as soon as possible, so that it causes page faults when used. | |
| 17 | //! | |
| 18 | //! * Do not re-use memory slots, so that memory safety is upheld. For small | |
| 19 | //! allocations, this is handled here; for larger ones it is handled in the | |
| 20 | //! backing allocator (by default `std.heap.page_allocator`). | |
| 21 | //! | |
| 22 | //! * Make pointer math errors unlikely to harm memory from | |
| 23 | //! unrelated allocations. | |
| 24 | //! | |
| 25 | //! * It's OK for these mechanisms to cost some extra overhead bytes. | |
| 26 | //! | |
| 27 | //! * It's OK for performance cost for these mechanisms. | |
| 28 | //! | |
| 29 | //! * Rogue memory writes should not harm the allocator's state. | |
| 30 | //! | |
| 31 | //! * Cross platform. Operates based on a backing allocator which makes it work | |
| 32 | //! everywhere, even freestanding. | |
| 33 | //! | |
| 34 | //! * Compile-time configuration. | |
| 35 | //! | |
| 36 | //! ### `OptimizationMode.release_fast` (note: not much work has gone into this use case yet): | |
| 37 | //! | |
| 38 | //! * Low fragmentation is primary concern | |
| 39 | //! * Performance of worst-case latency is secondary concern | |
| 40 | //! * Performance of average-case latency is next | |
| 41 | //! * Finally, having freed memory unmapped, and pointer math errors unlikely to | |
| 42 | //! harm memory from unrelated allocations are nice-to-haves. | |
| 43 | //! | |
| 44 | //! ### `OptimizationMode.release_small` (note: not much work has gone into this use case yet): | |
| 45 | //! | |
| 46 | //! * Small binary code size of the executable is the primary concern. | |
| 47 | //! * Next, defer to the `.release_fast` priority list. | |
| 48 | //! | |
| 49 | //! ## Basic Design: | |
| 50 | //! | |
| 51 | //! Small allocations are divided into buckets: | |
| 52 | //! | |
| 53 | //! ``` | |
| 54 | //! index obj_size | |
| 55 | //! 0 1 | |
| 56 | //! 1 2 | |
| 57 | //! 2 4 | |
| 58 | //! 3 8 | |
| 59 | //! 4 16 | |
| 60 | //! 5 32 | |
| 61 | //! 6 64 | |
| 62 | //! 7 128 | |
| 63 | //! 8 256 | |
| 64 | //! 9 512 | |
| 65 | //! 10 1024 | |
| 66 | //! 11 2048 | |
| 67 | //! ``` | |
| 68 | //! | |
| 69 | //! The main allocator state has an array of all the "current" buckets for each | |
| 70 | //! size class. Each slot in the array can be null, meaning the bucket for that | |
| 71 | //! size class is not allocated. When the first object is allocated for a given | |
| 72 | //! size class, it allocates 1 page of memory from the OS. This page is | |
| 73 | //! divided into "slots" - one per allocated object. Along with the page of memory | |
| 74 | //! for object slots, as many pages as necessary are allocated to store the | |
| 75 | //! BucketHeader, followed by "used bits", and two stack traces for each slot | |
| 76 | //! (allocation trace and free trace). | |
| 77 | //! | |
| 78 | //! The "used bits" are 1 bit per slot representing whether the slot is used. | |
| 79 | //! Allocations use the data to iterate to find a free slot. Frees assert that the | |
| 80 | //! corresponding bit is 1 and set it to 0. | |
| 81 | //! | |
| 82 | //! Buckets have prev and next pointers. When there is only one bucket for a given | |
| 83 | //! size class, both prev and next point to itself. When all slots of a bucket are | |
| 84 | //! used, a new bucket is allocated, and enters the doubly linked list. The main | |
| 85 | //! allocator state tracks the "current" bucket for each size class. Leak detection | |
| 86 | //! currently only checks the current bucket. | |
| 87 | //! | |
| 88 | //! Resizing detects if the size class is unchanged or smaller, in which case the same | |
| 89 | //! pointer is returned unmodified. If a larger size class is required, | |
| 90 | //! `error.OutOfMemory` is returned. | |
| 91 | //! | |
| 92 | //! Large objects are allocated directly using the backing allocator and their metadata is stored | |
| 93 | //! in a `std.HashMap` using the backing allocator. | |
| 94 | ||
| 95 | const std = @import("std"); | |
| 96 | const builtin = @import("builtin"); | |
| 97 | const log = std.log.scoped(.gpa); | |
| 98 | const math = std.math; | |
| 99 | const assert = std.debug.assert; | |
| 100 | const mem = std.mem; | |
| 101 | const Allocator = std.mem.Allocator; | |
| 102 | const StackTrace = std.builtin.StackTrace; | |
| 103 | ||
| 104 | const page_size: usize = @max(std.heap.page_size_max, switch (builtin.os.tag) { | |
| 105 | .windows => 64 * 1024, // Makes `std.heap.PageAllocator` take the happy path. | |
| 106 | .wasi => 64 * 1024, // Max alignment supported by `std.heap.WasmAllocator`. | |
| 107 | else => 128 * 1024, // Avoids too many active mappings when `page_size_max` is low. | |
| 108 | }); | |
| 109 | const page_align: mem.Alignment = .fromByteUnits(page_size); | |
| 110 | ||
| 111 | /// Integer type for pointing to slots in a small allocation | |
| 112 | const SlotIndex = std.meta.Int(.unsigned, math.log2(page_size) + 1); | |
| 113 | const Log2USize = std.math.Log2Int(usize); | |
| 114 | ||
| 115 | const default_sys_stack_trace_frames: usize = if (std.debug.sys_can_stack_trace) 6 else 0; | |
| 116 | const default_stack_trace_frames: usize = switch (builtin.mode) { | |
| 117 | .Debug => default_sys_stack_trace_frames, | |
| 118 | else => 0, | |
| 119 | }; | |
| 120 | ||
| 121 | pub const Config = struct { | |
| 122 | /// Number of stack frames to capture. | |
| 123 | stack_trace_frames: usize = default_stack_trace_frames, | |
| 124 | ||
| 125 | /// If true, the allocator will have two fields: | |
| 126 | /// * `total_requested_bytes` which tracks the total allocated bytes of memory requested. | |
| 127 | /// * `requested_memory_limit` which causes allocations to return `error.OutOfMemory` | |
| 128 | /// when the `total_requested_bytes` exceeds this limit. | |
| 129 | /// If false, these fields will be `void`. | |
| 130 | enable_memory_limit: bool = false, | |
| 131 | ||
| 132 | /// Whether to enable safety checks. | |
| 133 | safety: bool = std.debug.runtime_safety, | |
| 134 | ||
| 135 | /// Whether the allocator may be used simultaneously from multiple threads. | |
| 136 | thread_safe: bool = !builtin.single_threaded, | |
| 137 | ||
| 138 | /// What type of mutex you'd like to use, for thread safety. | |
| 139 | /// when specified, the mutex type must have the same shape as `std.Thread.Mutex` and | |
| 140 | /// `DummyMutex`, and have no required fields. Specifying this field causes | |
| 141 | /// the `thread_safe` field to be ignored. | |
| 142 | /// | |
| 143 | /// when null (default): | |
| 144 | /// * the mutex type defaults to `std.Thread.Mutex` when thread_safe is enabled. | |
| 145 | /// * the mutex type defaults to `DummyMutex` otherwise. | |
| 146 | MutexType: ?type = null, | |
| 147 | ||
| 148 | /// This is a temporary debugging trick you can use to turn segfaults into more helpful | |
| 149 | /// logged error messages with stack trace details. The downside is that every allocation | |
| 150 | /// will be leaked, unless used with retain_metadata! | |
| 151 | never_unmap: bool = false, | |
| 152 | ||
| 153 | /// This is a temporary debugging aid that retains metadata about allocations indefinitely. | |
| 154 | /// This allows a greater range of double frees to be reported. All metadata is freed when | |
| 155 | /// deinit is called. When used with never_unmap, deliberately leaked memory is also freed | |
| 156 | /// during deinit. Currently should be used with never_unmap to avoid segfaults. | |
| 157 | /// TODO https://github.com/ziglang/zig/issues/4298 will allow use without never_unmap | |
| 158 | retain_metadata: bool = false, | |
| 159 | ||
| 160 | /// Enables emitting info messages with the size and address of every allocation. | |
| 161 | verbose_log: bool = false, | |
| 162 | ||
| 163 | /// Tell whether the backing allocator returns already-zeroed memory. | |
| 164 | backing_allocator_zeroes: bool = true, | |
| 165 | ||
| 166 | /// When resizing an allocation, refresh the stack trace with the resize | |
| 167 | /// callsite. Comes with a performance penalty. | |
| 168 | resize_stack_traces: bool = false, | |
| 169 | ||
| 170 | /// Magic value that distinguishes allocations owned by this allocator from | |
| 171 | /// other regions of memory. | |
| 172 | canary: usize = @truncate(0x9232a6ff85dff10f), | |
| 173 | }; | |
| 174 | ||
| 175 | pub const Check = enum { ok, leak }; | |
| 176 | ||
| 177 | /// Default initialization of this struct is deprecated; use `.init` instead. | |
| 178 | pub fn GeneralPurposeAllocator(comptime config: Config) type { | |
| 179 | return struct { | |
| 180 | backing_allocator: Allocator = std.heap.page_allocator, | |
| 181 | /// Tracks the active bucket, which is the one that has free slots in it. | |
| 182 | buckets: [small_bucket_count]?*BucketHeader = [1]?*BucketHeader{null} ** small_bucket_count, | |
| 183 | large_allocations: LargeAllocTable = .empty, | |
| 184 | total_requested_bytes: @TypeOf(total_requested_bytes_init) = total_requested_bytes_init, | |
| 185 | requested_memory_limit: @TypeOf(requested_memory_limit_init) = requested_memory_limit_init, | |
| 186 | mutex: @TypeOf(mutex_init) = mutex_init, | |
| 187 | ||
| 188 | const Self = @This(); | |
| 189 | ||
| 190 | pub const init: Self = .{}; | |
| 191 | ||
| 192 | /// These can be derived from size_class_index but the calculation is nontrivial. | |
| 193 | const slot_counts: [small_bucket_count]SlotIndex = init: { | |
| 194 | @setEvalBranchQuota(10000); | |
| 195 | var result: [small_bucket_count]SlotIndex = undefined; | |
| 196 | for (&result, 0..) |*elem, i| elem.* = calculateSlotCount(i); | |
| 197 | break :init result; | |
| 198 | }; | |
| 199 | ||
| 200 | const total_requested_bytes_init = if (config.enable_memory_limit) @as(usize, 0) else {}; | |
| 201 | const requested_memory_limit_init = if (config.enable_memory_limit) @as(usize, math.maxInt(usize)) else {}; | |
| 202 | ||
| 203 | const mutex_init = if (config.MutexType) |T| | |
| 204 | T{} | |
| 205 | else if (config.thread_safe) | |
| 206 | std.Thread.Mutex{} | |
| 207 | else | |
| 208 | DummyMutex{}; | |
| 209 | ||
| 210 | const DummyMutex = struct { | |
| 211 | inline fn lock(_: *DummyMutex) void {} | |
| 212 | inline fn unlock(_: *DummyMutex) void {} | |
| 213 | }; | |
| 214 | ||
| 215 | const stack_n = config.stack_trace_frames; | |
| 216 | const one_trace_size = @sizeOf(usize) * stack_n; | |
| 217 | const traces_per_slot = 2; | |
| 218 | ||
| 219 | pub const Error = mem.Allocator.Error; | |
| 220 | ||
| 221 | /// Avoids creating buckets that would only be able to store a small | |
| 222 | /// number of slots. Value of 1 means 2 is the minimum slot count. | |
| 223 | const minimum_slots_per_bucket_log2 = 1; | |
| 224 | const small_bucket_count = math.log2(page_size) - minimum_slots_per_bucket_log2; | |
| 225 | const largest_bucket_object_size = 1 << (small_bucket_count - 1); | |
| 226 | const LargestSizeClassInt = std.math.IntFittingRange(0, largest_bucket_object_size); | |
| 227 | ||
| 228 | const bucketCompare = struct { | |
| 229 | fn compare(a: *BucketHeader, b: *BucketHeader) std.math.Order { | |
| 230 | return std.math.order(@intFromPtr(a.page), @intFromPtr(b.page)); | |
| 231 | } | |
| 232 | }.compare; | |
| 233 | ||
| 234 | const LargeAlloc = struct { | |
| 235 | bytes: []u8, | |
| 236 | requested_size: if (config.enable_memory_limit) usize else void, | |
| 237 | stack_addresses: [trace_n][stack_n]usize, | |
| 238 | freed: if (config.retain_metadata) bool else void, | |
| 239 | alignment: if (config.never_unmap and config.retain_metadata) mem.Alignment else void, | |
| 240 | ||
| 241 | const trace_n = if (config.retain_metadata) traces_per_slot else 1; | |
| 242 | ||
| 243 | fn dumpStackTrace(self: *LargeAlloc, trace_kind: TraceKind) void { | |
| 244 | std.debug.dumpStackTrace(self.getStackTrace(trace_kind)); | |
| 245 | } | |
| 246 | ||
| 247 | fn getStackTrace(self: *LargeAlloc, trace_kind: TraceKind) std.builtin.StackTrace { | |
| 248 | assert(@intFromEnum(trace_kind) < trace_n); | |
| 249 | const stack_addresses = &self.stack_addresses[@intFromEnum(trace_kind)]; | |
| 250 | var len: usize = 0; | |
| 251 | while (len < stack_n and stack_addresses[len] != 0) { | |
| 252 | len += 1; | |
| 253 | } | |
| 254 | return .{ | |
| 255 | .instruction_addresses = stack_addresses, | |
| 256 | .index = len, | |
| 257 | }; | |
| 258 | } | |
| 259 | ||
| 260 | fn captureStackTrace(self: *LargeAlloc, ret_addr: usize, trace_kind: TraceKind) void { | |
| 261 | assert(@intFromEnum(trace_kind) < trace_n); | |
| 262 | const stack_addresses = &self.stack_addresses[@intFromEnum(trace_kind)]; | |
| 263 | collectStackTrace(ret_addr, stack_addresses); | |
| 264 | } | |
| 265 | }; | |
| 266 | const LargeAllocTable = std.AutoHashMapUnmanaged(usize, LargeAlloc); | |
| 267 | ||
| 268 | /// Bucket: In memory, in order: | |
| 269 | /// * BucketHeader | |
| 270 | /// * bucket_used_bits: [N]usize, // 1 bit for every slot | |
| 271 | /// -- below only exists when config.safety is true -- | |
| 272 | /// * requested_sizes: [N]LargestSizeClassInt // 1 int for every slot | |
| 273 | /// * log2_ptr_aligns: [N]u8 // 1 byte for every slot | |
| 274 | /// -- above only exists when config.safety is true -- | |
| 275 | /// * stack_trace_addresses: [N]usize, // traces_per_slot for every allocation | |
| 276 | const BucketHeader = struct { | |
| 277 | allocated_count: SlotIndex, | |
| 278 | freed_count: SlotIndex, | |
| 279 | prev: ?*BucketHeader, | |
| 280 | canary: usize = config.canary, | |
| 281 | ||
| 282 | fn fromPage(page_addr: usize, slot_count: usize) *BucketHeader { | |
| 283 | const unaligned = page_addr + page_size - bucketSize(slot_count); | |
| 284 | return @ptrFromInt(unaligned & ~(@as(usize, @alignOf(BucketHeader)) - 1)); | |
| 285 | } | |
| 286 | ||
| 287 | fn usedBits(bucket: *BucketHeader, index: usize) *usize { | |
| 288 | const ptr: [*]u8 = @ptrCast(bucket); | |
| 289 | const bits: [*]usize = @alignCast(@ptrCast(ptr + @sizeOf(BucketHeader))); | |
| 290 | return &bits[index]; | |
| 291 | } | |
| 292 | ||
| 293 | fn requestedSizes(bucket: *BucketHeader, slot_count: usize) []LargestSizeClassInt { | |
| 294 | if (!config.safety) @compileError("requested size is only stored when safety is enabled"); | |
| 295 | const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketRequestedSizesStart(slot_count); | |
| 296 | const sizes = @as([*]LargestSizeClassInt, @ptrCast(@alignCast(start_ptr))); | |
| 297 | return sizes[0..slot_count]; | |
| 298 | } | |
| 299 | ||
| 300 | fn log2PtrAligns(bucket: *BucketHeader, slot_count: usize) []mem.Alignment { | |
| 301 | if (!config.safety) @compileError("requested size is only stored when safety is enabled"); | |
| 302 | const aligns_ptr = @as([*]u8, @ptrCast(bucket)) + bucketAlignsStart(slot_count); | |
| 303 | return @ptrCast(aligns_ptr[0..slot_count]); | |
| 304 | } | |
| 305 | ||
| 306 | fn stackTracePtr( | |
| 307 | bucket: *BucketHeader, | |
| 308 | slot_count: usize, | |
| 309 | slot_index: SlotIndex, | |
| 310 | trace_kind: TraceKind, | |
| 311 | ) *[stack_n]usize { | |
| 312 | const start_ptr = @as([*]u8, @ptrCast(bucket)) + bucketStackFramesStart(slot_count); | |
| 313 | const addr = start_ptr + one_trace_size * traces_per_slot * slot_index + | |
| 314 | @intFromEnum(trace_kind) * @as(usize, one_trace_size); | |
| 315 | return @ptrCast(@alignCast(addr)); | |
| 316 | } | |
| 317 | ||
| 318 | fn captureStackTrace( | |
| 319 | bucket: *BucketHeader, | |
| 320 | ret_addr: usize, | |
| 321 | slot_count: usize, | |
| 322 | slot_index: SlotIndex, | |
| 323 | trace_kind: TraceKind, | |
| 324 | ) void { | |
| 325 | // Initialize them to 0. When determining the count we must look | |
| 326 | // for non zero addresses. | |
| 327 | const stack_addresses = bucket.stackTracePtr(slot_count, slot_index, trace_kind); | |
| 328 | collectStackTrace(ret_addr, stack_addresses); | |
| 329 | } | |
| 330 | }; | |
| 331 | ||
| 332 | pub fn allocator(self: *Self) Allocator { | |
| 333 | return .{ | |
| 334 | .ptr = self, | |
| 335 | .vtable = &.{ | |
| 336 | .alloc = alloc, | |
| 337 | .resize = resize, | |
| 338 | .remap = remap, | |
| 339 | .free = free, | |
| 340 | }, | |
| 341 | }; | |
| 342 | } | |
| 343 | ||
| 344 | fn bucketStackTrace( | |
| 345 | bucket: *BucketHeader, | |
| 346 | slot_count: usize, | |
| 347 | slot_index: SlotIndex, | |
| 348 | trace_kind: TraceKind, | |
| 349 | ) StackTrace { | |
| 350 | const stack_addresses = bucket.stackTracePtr(slot_count, slot_index, trace_kind); | |
| 351 | var len: usize = 0; | |
| 352 | while (len < stack_n and stack_addresses[len] != 0) { | |
| 353 | len += 1; | |
| 354 | } | |
| 355 | return .{ | |
| 356 | .instruction_addresses = stack_addresses, | |
| 357 | .index = len, | |
| 358 | }; | |
| 359 | } | |
| 360 | ||
| 361 | fn bucketRequestedSizesStart(slot_count: usize) usize { | |
| 362 | if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled"); | |
| 363 | return mem.alignForward( | |
| 364 | usize, | |
| 365 | @sizeOf(BucketHeader) + usedBitsSize(slot_count), | |
| 366 | @alignOf(LargestSizeClassInt), | |
| 367 | ); | |
| 368 | } | |
| 369 | ||
| 370 | fn bucketAlignsStart(slot_count: usize) usize { | |
| 371 | if (!config.safety) @compileError("requested sizes are not stored unless safety is enabled"); | |
| 372 | return bucketRequestedSizesStart(slot_count) + (@sizeOf(LargestSizeClassInt) * slot_count); | |
| 373 | } | |
| 374 | ||
| 375 | fn bucketStackFramesStart(slot_count: usize) usize { | |
| 376 | const unaligned_start = if (config.safety) | |
| 377 | bucketAlignsStart(slot_count) + slot_count | |
| 378 | else | |
| 379 | @sizeOf(BucketHeader) + usedBitsSize(slot_count); | |
| 380 | return mem.alignForward(usize, unaligned_start, @alignOf(usize)); | |
| 381 | } | |
| 382 | ||
| 383 | fn bucketSize(slot_count: usize) usize { | |
| 384 | return bucketStackFramesStart(slot_count) + one_trace_size * traces_per_slot * slot_count; | |
| 385 | } | |
| 386 | ||
| 387 | /// This is executed only at compile-time to prepopulate a lookup table. | |
| 388 | fn calculateSlotCount(size_class_index: usize) SlotIndex { | |
| 389 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | |
| 390 | var lower: usize = 1 << minimum_slots_per_bucket_log2; | |
| 391 | var upper: usize = (page_size - bucketSize(lower)) / size_class; | |
| 392 | while (upper > lower) { | |
| 393 | const proposed: usize = lower + (upper - lower) / 2; | |
| 394 | if (proposed == lower) return lower; | |
| 395 | const slots_end = proposed * size_class; | |
| 396 | const header_begin = mem.alignForward(usize, slots_end, @alignOf(BucketHeader)); | |
| 397 | const end = header_begin + bucketSize(proposed); | |
| 398 | if (end > page_size) { | |
| 399 | upper = proposed - 1; | |
| 400 | } else { | |
| 401 | lower = proposed; | |
| 402 | } | |
| 403 | } | |
| 404 | const slots_end = lower * size_class; | |
| 405 | const header_begin = mem.alignForward(usize, slots_end, @alignOf(BucketHeader)); | |
| 406 | const end = header_begin + bucketSize(lower); | |
| 407 | assert(end <= page_size); | |
| 408 | return lower; | |
| 409 | } | |
| 410 | ||
| 411 | fn usedBitsCount(slot_count: usize) usize { | |
| 412 | return (slot_count + (@bitSizeOf(usize) - 1)) / @bitSizeOf(usize); | |
| 413 | } | |
| 414 | ||
| 415 | fn usedBitsSize(slot_count: usize) usize { | |
| 416 | return usedBitsCount(slot_count) * @sizeOf(usize); | |
| 417 | } | |
| 418 | ||
| 419 | fn detectLeaksInBucket(bucket: *BucketHeader, size_class_index: usize, used_bits_count: usize) bool { | |
| 420 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | |
| 421 | const slot_count = slot_counts[size_class_index]; | |
| 422 | var leaks = false; | |
| 423 | for (0..used_bits_count) |used_bits_byte| { | |
| 424 | const used_int = bucket.usedBits(used_bits_byte).*; | |
| 425 | if (used_int != 0) { | |
| 426 | for (0..@bitSizeOf(usize)) |bit_index_usize| { | |
| 427 | const bit_index: Log2USize = @intCast(bit_index_usize); | |
| 428 | const is_used = @as(u1, @truncate(used_int >> bit_index)) != 0; | |
| 429 | if (is_used) { | |
| 430 | const slot_index: SlotIndex = @intCast(used_bits_byte * @bitSizeOf(usize) + bit_index); | |
| 431 | const stack_trace = bucketStackTrace(bucket, slot_count, slot_index, .alloc); | |
| 432 | const page_addr = @intFromPtr(bucket) & ~(page_size - 1); | |
| 433 | const addr = page_addr + slot_index * size_class; | |
| 434 | log.err("memory address 0x{x} leaked: {}", .{ addr, stack_trace }); | |
| 435 | leaks = true; | |
| 436 | } | |
| 437 | } | |
| 438 | } | |
| 439 | } | |
| 440 | return leaks; | |
| 441 | } | |
| 442 | ||
| 443 | /// Emits log messages for leaks and then returns whether there were any leaks. | |
| 444 | pub fn detectLeaks(self: *Self) bool { | |
| 445 | var leaks = false; | |
| 446 | ||
| 447 | for (self.buckets, 0..) |init_optional_bucket, size_class_index| { | |
| 448 | var optional_bucket = init_optional_bucket; | |
| 449 | const slot_count = slot_counts[size_class_index]; | |
| 450 | const used_bits_count = usedBitsCount(slot_count); | |
| 451 | while (optional_bucket) |bucket| { | |
| 452 | leaks = detectLeaksInBucket(bucket, size_class_index, used_bits_count) or leaks; | |
| 453 | optional_bucket = bucket.prev; | |
| 454 | } | |
| 455 | } | |
| 456 | ||
| 457 | var it = self.large_allocations.valueIterator(); | |
| 458 | while (it.next()) |large_alloc| { | |
| 459 | if (config.retain_metadata and large_alloc.freed) continue; | |
| 460 | const stack_trace = large_alloc.getStackTrace(.alloc); | |
| 461 | log.err("memory address 0x{x} leaked: {}", .{ | |
| 462 | @intFromPtr(large_alloc.bytes.ptr), stack_trace, | |
| 463 | }); | |
| 464 | leaks = true; | |
| 465 | } | |
| 466 | return leaks; | |
| 467 | } | |
| 468 | ||
| 469 | fn freeRetainedMetadata(self: *Self) void { | |
| 470 | comptime assert(config.retain_metadata); | |
| 471 | if (config.never_unmap) { | |
| 472 | // free large allocations that were intentionally leaked by never_unmap | |
| 473 | var it = self.large_allocations.iterator(); | |
| 474 | while (it.next()) |large| { | |
| 475 | if (large.value_ptr.freed) { | |
| 476 | self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.alignment, @returnAddress()); | |
| 477 | } | |
| 478 | } | |
| 479 | } | |
| 480 | } | |
| 481 | ||
| 482 | pub fn flushRetainedMetadata(self: *Self) void { | |
| 483 | comptime assert(config.retain_metadata); | |
| 484 | self.freeRetainedMetadata(); | |
| 485 | // also remove entries from large_allocations | |
| 486 | var it = self.large_allocations.iterator(); | |
| 487 | while (it.next()) |large| { | |
| 488 | if (large.value_ptr.freed) { | |
| 489 | _ = self.large_allocations.remove(@intFromPtr(large.value_ptr.bytes.ptr)); | |
| 490 | } | |
| 491 | } | |
| 492 | } | |
| 493 | ||
| 494 | /// Returns `Check.leak` if there were leaks; `Check.ok` otherwise. | |
| 495 | pub fn deinit(self: *Self) Check { | |
| 496 | const leaks = if (config.safety) self.detectLeaks() else false; | |
| 497 | if (config.retain_metadata) self.freeRetainedMetadata(); | |
| 498 | self.large_allocations.deinit(self.backing_allocator); | |
| 499 | self.* = undefined; | |
| 500 | return if (leaks) .leak else .ok; | |
| 501 | } | |
| 502 | ||
| 503 | fn collectStackTrace(first_trace_addr: usize, addresses: *[stack_n]usize) void { | |
| 504 | if (stack_n == 0) return; | |
| 505 | @memset(addresses, 0); | |
| 506 | var stack_trace: StackTrace = .{ | |
| 507 | .instruction_addresses = addresses, | |
| 508 | .index = 0, | |
| 509 | }; | |
| 510 | std.debug.captureStackTrace(first_trace_addr, &stack_trace); | |
| 511 | } | |
| 512 | ||
| 513 | fn reportDoubleFree(ret_addr: usize, alloc_stack_trace: StackTrace, free_stack_trace: StackTrace) void { | |
| 514 | var addresses: [stack_n]usize = @splat(0); | |
| 515 | var second_free_stack_trace: StackTrace = .{ | |
| 516 | .instruction_addresses = &addresses, | |
| 517 | .index = 0, | |
| 518 | }; | |
| 519 | std.debug.captureStackTrace(ret_addr, &second_free_stack_trace); | |
| 520 | log.err("Double free detected. Allocation: {} First free: {} Second free: {}", .{ | |
| 521 | alloc_stack_trace, free_stack_trace, second_free_stack_trace, | |
| 522 | }); | |
| 523 | } | |
| 524 | ||
| 525 | /// This function assumes the object is in the large object storage regardless | |
| 526 | /// of the parameters. | |
| 527 | fn resizeLarge( | |
| 528 | self: *Self, | |
| 529 | old_mem: []u8, | |
| 530 | alignment: mem.Alignment, | |
| 531 | new_size: usize, | |
| 532 | ret_addr: usize, | |
| 533 | may_move: bool, | |
| 534 | ) ?[*]u8 { | |
| 535 | if (config.retain_metadata and may_move) { | |
| 536 | // Before looking up the entry (since this could invalidate | |
| 537 | // it), we must reserve space for the new entry in case the | |
| 538 | // allocation is relocated. | |
| 539 | self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1) catch return null; | |
| 540 | } | |
| 541 | ||
| 542 | const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse { | |
| 543 | if (config.safety) { | |
| 544 | @panic("Invalid free"); | |
| 545 | } else { | |
| 546 | unreachable; | |
| 547 | } | |
| 548 | }; | |
| 549 | ||
| 550 | if (config.retain_metadata and entry.value_ptr.freed) { | |
| 551 | if (config.safety) { | |
| 552 | reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free)); | |
| 553 | @panic("Unrecoverable double free"); | |
| 554 | } else { | |
| 555 | unreachable; | |
| 556 | } | |
| 557 | } | |
| 558 | ||
| 559 | if (config.safety and old_mem.len != entry.value_ptr.bytes.len) { | |
| 560 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; | |
| 561 | var free_stack_trace: StackTrace = .{ | |
| 562 | .instruction_addresses = &addresses, | |
| 563 | .index = 0, | |
| 564 | }; | |
| 565 | std.debug.captureStackTrace(ret_addr, &free_stack_trace); | |
| 566 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ | |
| 567 | entry.value_ptr.bytes.len, | |
| 568 | old_mem.len, | |
| 569 | entry.value_ptr.getStackTrace(.alloc), | |
| 570 | free_stack_trace, | |
| 571 | }); | |
| 572 | } | |
| 573 | ||
| 574 | // If this would move the allocation into a small size class, | |
| 575 | // refuse the request, because it would require creating small | |
| 576 | // allocation metadata. | |
| 577 | const new_size_class_index: usize = @max(@bitSizeOf(usize) - @clz(new_size - 1), @intFromEnum(alignment)); | |
| 578 | if (new_size_class_index < self.buckets.len) return null; | |
| 579 | ||
| 580 | // Do memory limit accounting with requested sizes rather than what | |
| 581 | // backing_allocator returns because if we want to return | |
| 582 | // error.OutOfMemory, we have to leave allocation untouched, and | |
| 583 | // that is impossible to guarantee after calling | |
| 584 | // backing_allocator.rawResize. | |
| 585 | const prev_req_bytes = self.total_requested_bytes; | |
| 586 | if (config.enable_memory_limit) { | |
| 587 | const new_req_bytes = prev_req_bytes + new_size - entry.value_ptr.requested_size; | |
| 588 | if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) { | |
| 589 | return null; | |
| 590 | } | |
| 591 | self.total_requested_bytes = new_req_bytes; | |
| 592 | } | |
| 593 | ||
| 594 | const opt_resized_ptr = if (may_move) | |
| 595 | self.backing_allocator.rawRemap(old_mem, alignment, new_size, ret_addr) | |
| 596 | else if (self.backing_allocator.rawResize(old_mem, alignment, new_size, ret_addr)) | |
| 597 | old_mem.ptr | |
| 598 | else | |
| 599 | null; | |
| 600 | ||
| 601 | const resized_ptr = opt_resized_ptr orelse { | |
| 602 | if (config.enable_memory_limit) { | |
| 603 | self.total_requested_bytes = prev_req_bytes; | |
| 604 | } | |
| 605 | return null; | |
| 606 | }; | |
| 607 | ||
| 608 | if (config.enable_memory_limit) { | |
| 609 | entry.value_ptr.requested_size = new_size; | |
| 610 | } | |
| 611 | ||
| 612 | if (config.verbose_log) { | |
| 613 | log.info("large resize {d} bytes at {*} to {d} at {*}", .{ | |
| 614 | old_mem.len, old_mem.ptr, new_size, resized_ptr, | |
| 615 | }); | |
| 616 | } | |
| 617 | entry.value_ptr.bytes = resized_ptr[0..new_size]; | |
| 618 | if (config.resize_stack_traces) | |
| 619 | entry.value_ptr.captureStackTrace(ret_addr, .alloc); | |
| 620 | ||
| 621 | // Update the key of the hash map if the memory was relocated. | |
| 622 | if (resized_ptr != old_mem.ptr) { | |
| 623 | const large_alloc = entry.value_ptr.*; | |
| 624 | if (config.retain_metadata) { | |
| 625 | entry.value_ptr.freed = true; | |
| 626 | entry.value_ptr.captureStackTrace(ret_addr, .free); | |
| 627 | } else { | |
| 628 | self.large_allocations.removeByPtr(entry.key_ptr); | |
| 629 | } | |
| 630 | ||
| 631 | const gop = self.large_allocations.getOrPutAssumeCapacity(@intFromPtr(resized_ptr)); | |
| 632 | if (config.retain_metadata and !config.never_unmap) { | |
| 633 | // Backing allocator may be reusing memory that we're retaining metadata for | |
| 634 | assert(!gop.found_existing or gop.value_ptr.freed); | |
| 635 | } else { | |
| 636 | assert(!gop.found_existing); // This would mean the kernel double-mapped pages. | |
| 637 | } | |
| 638 | gop.value_ptr.* = large_alloc; | |
| 639 | } | |
| 640 | ||
| 641 | return resized_ptr; | |
| 642 | } | |
| 643 | ||
| 644 | /// This function assumes the object is in the large object storage regardless | |
| 645 | /// of the parameters. | |
| 646 | fn freeLarge( | |
| 647 | self: *Self, | |
| 648 | old_mem: []u8, | |
| 649 | alignment: mem.Alignment, | |
| 650 | ret_addr: usize, | |
| 651 | ) void { | |
| 652 | const entry = self.large_allocations.getEntry(@intFromPtr(old_mem.ptr)) orelse { | |
| 653 | if (config.safety) { | |
| 654 | @panic("Invalid free"); | |
| 655 | } else { | |
| 656 | unreachable; | |
| 657 | } | |
| 658 | }; | |
| 659 | ||
| 660 | if (config.retain_metadata and entry.value_ptr.freed) { | |
| 661 | if (config.safety) { | |
| 662 | reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free)); | |
| 663 | return; | |
| 664 | } else { | |
| 665 | unreachable; | |
| 666 | } | |
| 667 | } | |
| 668 | ||
| 669 | if (config.safety and old_mem.len != entry.value_ptr.bytes.len) { | |
| 670 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; | |
| 671 | var free_stack_trace = StackTrace{ | |
| 672 | .instruction_addresses = &addresses, | |
| 673 | .index = 0, | |
| 674 | }; | |
| 675 | std.debug.captureStackTrace(ret_addr, &free_stack_trace); | |
| 676 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ | |
| 677 | entry.value_ptr.bytes.len, | |
| 678 | old_mem.len, | |
| 679 | entry.value_ptr.getStackTrace(.alloc), | |
| 680 | free_stack_trace, | |
| 681 | }); | |
| 682 | } | |
| 683 | ||
| 684 | if (!config.never_unmap) { | |
| 685 | self.backing_allocator.rawFree(old_mem, alignment, ret_addr); | |
| 686 | } | |
| 687 | ||
| 688 | if (config.enable_memory_limit) { | |
| 689 | self.total_requested_bytes -= entry.value_ptr.requested_size; | |
| 690 | } | |
| 691 | ||
| 692 | if (config.verbose_log) { | |
| 693 | log.info("large free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr }); | |
| 694 | } | |
| 695 | ||
| 696 | if (!config.retain_metadata) { | |
| 697 | assert(self.large_allocations.remove(@intFromPtr(old_mem.ptr))); | |
| 698 | } else { | |
| 699 | entry.value_ptr.freed = true; | |
| 700 | entry.value_ptr.captureStackTrace(ret_addr, .free); | |
| 701 | } | |
| 702 | } | |
| 703 | ||
| 704 | fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ret_addr: usize) ?[*]u8 { | |
| 705 | const self: *Self = @ptrCast(@alignCast(context)); | |
| 706 | self.mutex.lock(); | |
| 707 | defer self.mutex.unlock(); | |
| 708 | ||
| 709 | if (config.enable_memory_limit) { | |
| 710 | const new_req_bytes = self.total_requested_bytes + len; | |
| 711 | if (new_req_bytes > self.requested_memory_limit) return null; | |
| 712 | self.total_requested_bytes = new_req_bytes; | |
| 713 | } | |
| 714 | ||
| 715 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(len - 1), @intFromEnum(alignment)); | |
| 716 | if (size_class_index >= self.buckets.len) { | |
| 717 | @branchHint(.unlikely); | |
| 718 | self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1) catch return null; | |
| 719 | const ptr = self.backing_allocator.rawAlloc(len, alignment, ret_addr) orelse return null; | |
| 720 | const slice = ptr[0..len]; | |
| 721 | ||
| 722 | const gop = self.large_allocations.getOrPutAssumeCapacity(@intFromPtr(slice.ptr)); | |
| 723 | if (config.retain_metadata and !config.never_unmap) { | |
| 724 | // Backing allocator may be reusing memory that we're retaining metadata for | |
| 725 | assert(!gop.found_existing or gop.value_ptr.freed); | |
| 726 | } else { | |
| 727 | assert(!gop.found_existing); // This would mean the kernel double-mapped pages. | |
| 728 | } | |
| 729 | gop.value_ptr.bytes = slice; | |
| 730 | if (config.enable_memory_limit) | |
| 731 | gop.value_ptr.requested_size = len; | |
| 732 | gop.value_ptr.captureStackTrace(ret_addr, .alloc); | |
| 733 | if (config.retain_metadata) { | |
| 734 | gop.value_ptr.freed = false; | |
| 735 | if (config.never_unmap) { | |
| 736 | gop.value_ptr.alignment = alignment; | |
| 737 | } | |
| 738 | } | |
| 739 | ||
| 740 | if (config.verbose_log) { | |
| 741 | log.info("large alloc {d} bytes at {*}", .{ slice.len, slice.ptr }); | |
| 742 | } | |
| 743 | return slice.ptr; | |
| 744 | } | |
| 745 | ||
| 746 | const slot_count = slot_counts[size_class_index]; | |
| 747 | ||
| 748 | if (self.buckets[size_class_index]) |bucket| { | |
| 749 | @branchHint(.likely); | |
| 750 | const slot_index = bucket.allocated_count; | |
| 751 | if (slot_index < slot_count) { | |
| 752 | @branchHint(.likely); | |
| 753 | bucket.allocated_count = slot_index + 1; | |
| 754 | const used_bits_byte = bucket.usedBits(slot_index / @bitSizeOf(usize)); | |
| 755 | const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize)); | |
| 756 | used_bits_byte.* |= (@as(usize, 1) << used_bit_index); | |
| 757 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | |
| 758 | if (config.stack_trace_frames > 0) { | |
| 759 | bucket.captureStackTrace(ret_addr, slot_count, slot_index, .alloc); | |
| 760 | } | |
| 761 | if (config.safety) { | |
| 762 | bucket.requestedSizes(slot_count)[slot_index] = @intCast(len); | |
| 763 | bucket.log2PtrAligns(slot_count)[slot_index] = alignment; | |
| 764 | } | |
| 765 | const page_addr = @intFromPtr(bucket) & ~(page_size - 1); | |
| 766 | const addr = page_addr + slot_index * size_class; | |
| 767 | if (config.verbose_log) { | |
| 768 | log.info("small alloc {d} bytes at 0x{x}", .{ len, addr }); | |
| 769 | } | |
| 770 | return @ptrFromInt(addr); | |
| 771 | } | |
| 772 | } | |
| 773 | ||
| 774 | const page = self.backing_allocator.rawAlloc(page_size, page_align, @returnAddress()) orelse | |
| 775 | return null; | |
| 776 | const bucket: *BucketHeader = .fromPage(@intFromPtr(page), slot_count); | |
| 777 | bucket.* = .{ | |
| 778 | .allocated_count = 1, | |
| 779 | .freed_count = 0, | |
| 780 | .prev = self.buckets[size_class_index], | |
| 781 | }; | |
| 782 | self.buckets[size_class_index] = bucket; | |
| 783 | ||
| 784 | if (!config.backing_allocator_zeroes) { | |
| 785 | @memset(@as([*]usize, @as(*[1]usize, bucket.usedBits(0)))[0..usedBitsCount(slot_count)], 0); | |
| 786 | if (config.safety) @memset(bucket.requestedSizes(slot_count), 0); | |
| 787 | } | |
| 788 | ||
| 789 | bucket.usedBits(0).* = 0b1; | |
| 790 | ||
| 791 | if (config.stack_trace_frames > 0) { | |
| 792 | bucket.captureStackTrace(ret_addr, slot_count, 0, .alloc); | |
| 793 | } | |
| 794 | ||
| 795 | if (config.safety) { | |
| 796 | bucket.requestedSizes(slot_count)[0] = @intCast(len); | |
| 797 | bucket.log2PtrAligns(slot_count)[0] = alignment; | |
| 798 | } | |
| 799 | ||
| 800 | if (config.verbose_log) { | |
| 801 | log.info("small alloc {d} bytes at 0x{x}", .{ len, @intFromPtr(page) }); | |
| 802 | } | |
| 803 | ||
| 804 | return page; | |
| 805 | } | |
| 806 | ||
| 807 | fn resize( | |
| 808 | context: *anyopaque, | |
| 809 | memory: []u8, | |
| 810 | alignment: mem.Alignment, | |
| 811 | new_len: usize, | |
| 812 | return_address: usize, | |
| 813 | ) bool { | |
| 814 | const self: *Self = @ptrCast(@alignCast(context)); | |
| 815 | self.mutex.lock(); | |
| 816 | defer self.mutex.unlock(); | |
| 817 | ||
| 818 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment)); | |
| 819 | if (size_class_index >= self.buckets.len) { | |
| 820 | return self.resizeLarge(memory, alignment, new_len, return_address, false) != null; | |
| 821 | } else { | |
| 822 | return resizeSmall(self, memory, alignment, new_len, return_address, size_class_index); | |
| 823 | } | |
| 824 | } | |
| 825 | ||
| 826 | fn remap( | |
| 827 | context: *anyopaque, | |
| 828 | memory: []u8, | |
| 829 | alignment: mem.Alignment, | |
| 830 | new_len: usize, | |
| 831 | return_address: usize, | |
| 832 | ) ?[*]u8 { | |
| 833 | const self: *Self = @ptrCast(@alignCast(context)); | |
| 834 | self.mutex.lock(); | |
| 835 | defer self.mutex.unlock(); | |
| 836 | ||
| 837 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment)); | |
| 838 | if (size_class_index >= self.buckets.len) { | |
| 839 | return self.resizeLarge(memory, alignment, new_len, return_address, true); | |
| 840 | } else { | |
| 841 | return if (resizeSmall(self, memory, alignment, new_len, return_address, size_class_index)) memory.ptr else null; | |
| 842 | } | |
| 843 | } | |
| 844 | ||
| 845 | fn free( | |
| 846 | context: *anyopaque, | |
| 847 | old_memory: []u8, | |
| 848 | alignment: mem.Alignment, | |
| 849 | return_address: usize, | |
| 850 | ) void { | |
| 851 | const self: *Self = @ptrCast(@alignCast(context)); | |
| 852 | self.mutex.lock(); | |
| 853 | defer self.mutex.unlock(); | |
| 854 | ||
| 855 | assert(old_memory.len != 0); | |
| 856 | ||
| 857 | const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(old_memory.len - 1), @intFromEnum(alignment)); | |
| 858 | if (size_class_index >= self.buckets.len) { | |
| 859 | @branchHint(.unlikely); | |
| 860 | self.freeLarge(old_memory, alignment, return_address); | |
| 861 | return; | |
| 862 | } | |
| 863 | ||
| 864 | const slot_count = slot_counts[size_class_index]; | |
| 865 | const freed_addr = @intFromPtr(old_memory.ptr); | |
| 866 | const page_addr = freed_addr & ~(page_size - 1); | |
| 867 | const bucket: *BucketHeader = .fromPage(page_addr, slot_count); | |
| 868 | if (bucket.canary != config.canary) @panic("Invalid free"); | |
| 869 | const page_offset = freed_addr - page_addr; | |
| 870 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | |
| 871 | const slot_index: SlotIndex = @intCast(page_offset / size_class); | |
| 872 | const used_byte_index = slot_index / @bitSizeOf(usize); | |
| 873 | const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize)); | |
| 874 | const used_byte = bucket.usedBits(used_byte_index); | |
| 875 | const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0; | |
| 876 | if (!is_used) { | |
| 877 | if (config.safety) { | |
| 878 | reportDoubleFree( | |
| 879 | return_address, | |
| 880 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | |
| 881 | bucketStackTrace(bucket, slot_count, slot_index, .free), | |
| 882 | ); | |
| 883 | // Recoverable since this is a free. | |
| 884 | return; | |
| 885 | } else { | |
| 886 | unreachable; | |
| 887 | } | |
| 888 | } | |
| 889 | ||
| 890 | // Definitely an in-use small alloc now. | |
| 891 | if (config.safety) { | |
| 892 | const requested_size = bucket.requestedSizes(slot_count)[slot_index]; | |
| 893 | if (requested_size == 0) @panic("Invalid free"); | |
| 894 | const slot_alignment = bucket.log2PtrAligns(slot_count)[slot_index]; | |
| 895 | if (old_memory.len != requested_size or alignment != slot_alignment) { | |
| 896 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; | |
| 897 | var free_stack_trace: StackTrace = .{ | |
| 898 | .instruction_addresses = &addresses, | |
| 899 | .index = 0, | |
| 900 | }; | |
| 901 | std.debug.captureStackTrace(return_address, &free_stack_trace); | |
| 902 | if (old_memory.len != requested_size) { | |
| 903 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ | |
| 904 | requested_size, | |
| 905 | old_memory.len, | |
| 906 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | |
| 907 | free_stack_trace, | |
| 908 | }); | |
| 909 | } | |
| 910 | if (alignment != slot_alignment) { | |
| 911 | log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{ | |
| 912 | slot_alignment.toByteUnits(), | |
| 913 | alignment.toByteUnits(), | |
| 914 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | |
| 915 | free_stack_trace, | |
| 916 | }); | |
| 917 | } | |
| 918 | } | |
| 919 | } | |
| 920 | ||
| 921 | if (config.enable_memory_limit) { | |
| 922 | self.total_requested_bytes -= old_memory.len; | |
| 923 | } | |
| 924 | ||
| 925 | if (config.stack_trace_frames > 0) { | |
| 926 | // Capture stack trace to be the "first free", in case a double free happens. | |
| 927 | bucket.captureStackTrace(return_address, slot_count, slot_index, .free); | |
| 928 | } | |
| 929 | ||
| 930 | used_byte.* &= ~(@as(usize, 1) << used_bit_index); | |
| 931 | if (config.safety) { | |
| 932 | bucket.requestedSizes(slot_count)[slot_index] = 0; | |
| 933 | } | |
| 934 | bucket.freed_count += 1; | |
| 935 | if (bucket.freed_count == bucket.allocated_count) { | |
| 936 | if (self.buckets[size_class_index] == bucket) { | |
| 937 | self.buckets[size_class_index] = null; | |
| 938 | } | |
| 939 | if (!config.never_unmap) { | |
| 940 | const page: [*]align(page_size) u8 = @ptrFromInt(page_addr); | |
| 941 | self.backing_allocator.rawFree(page[0..page_size], page_align, @returnAddress()); | |
| 942 | } | |
| 943 | } | |
| 944 | if (config.verbose_log) { | |
| 945 | log.info("small free {d} bytes at {*}", .{ old_memory.len, old_memory.ptr }); | |
| 946 | } | |
| 947 | } | |
| 948 | ||
| 949 | fn resizeSmall( | |
| 950 | self: *Self, | |
| 951 | memory: []u8, | |
| 952 | alignment: mem.Alignment, | |
| 953 | new_len: usize, | |
| 954 | return_address: usize, | |
| 955 | size_class_index: usize, | |
| 956 | ) bool { | |
| 957 | const new_size_class_index: usize = @max(@bitSizeOf(usize) - @clz(new_len - 1), @intFromEnum(alignment)); | |
| 958 | if (!config.safety) return new_size_class_index == size_class_index; | |
| 959 | const slot_count = slot_counts[size_class_index]; | |
| 960 | const memory_addr = @intFromPtr(memory.ptr); | |
| 961 | const page_addr = memory_addr & ~(page_size - 1); | |
| 962 | const bucket: *BucketHeader = .fromPage(page_addr, slot_count); | |
| 963 | if (bucket.canary != config.canary) @panic("Invalid free"); | |
| 964 | const page_offset = memory_addr - page_addr; | |
| 965 | const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index)); | |
| 966 | const slot_index: SlotIndex = @intCast(page_offset / size_class); | |
| 967 | const used_byte_index = slot_index / @bitSizeOf(usize); | |
| 968 | const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize)); | |
| 969 | const used_byte = bucket.usedBits(used_byte_index); | |
| 970 | const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0; | |
| 971 | if (!is_used) { | |
| 972 | reportDoubleFree( | |
| 973 | return_address, | |
| 974 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | |
| 975 | bucketStackTrace(bucket, slot_count, slot_index, .free), | |
| 976 | ); | |
| 977 | // Recoverable since this is a free. | |
| 978 | return false; | |
| 979 | } | |
| 980 | ||
| 981 | // Definitely an in-use small alloc now. | |
| 982 | const requested_size = bucket.requestedSizes(slot_count)[slot_index]; | |
| 983 | if (requested_size == 0) @panic("Invalid free"); | |
| 984 | const slot_alignment = bucket.log2PtrAligns(slot_count)[slot_index]; | |
| 985 | if (memory.len != requested_size or alignment != slot_alignment) { | |
| 986 | var addresses: [stack_n]usize = [1]usize{0} ** stack_n; | |
| 987 | var free_stack_trace: StackTrace = .{ | |
| 988 | .instruction_addresses = &addresses, | |
| 989 | .index = 0, | |
| 990 | }; | |
| 991 | std.debug.captureStackTrace(return_address, &free_stack_trace); | |
| 992 | if (memory.len != requested_size) { | |
| 993 | log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {} Free: {}", .{ | |
| 994 | requested_size, | |
| 995 | memory.len, | |
| 996 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | |
| 997 | free_stack_trace, | |
| 998 | }); | |
| 999 | } | |
| 1000 | if (alignment != slot_alignment) { | |
| 1001 | log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {} Free: {}", .{ | |
| 1002 | slot_alignment.toByteUnits(), | |
| 1003 | alignment.toByteUnits(), | |
| 1004 | bucketStackTrace(bucket, slot_count, slot_index, .alloc), | |
| 1005 | free_stack_trace, | |
| 1006 | }); | |
| 1007 | } | |
| 1008 | } | |
| 1009 | ||
| 1010 | if (new_size_class_index != size_class_index) return false; | |
| 1011 | ||
| 1012 | const prev_req_bytes = self.total_requested_bytes; | |
| 1013 | if (config.enable_memory_limit) { | |
| 1014 | const new_req_bytes = prev_req_bytes - memory.len + new_len; | |
| 1015 | if (new_req_bytes > prev_req_bytes and new_req_bytes > self.requested_memory_limit) { | |
| 1016 | return false; | |
| 1017 | } | |
| 1018 | self.total_requested_bytes = new_req_bytes; | |
| 1019 | } | |
| 1020 | ||
| 1021 | if (memory.len > new_len) @memset(memory[new_len..], undefined); | |
| 1022 | if (config.verbose_log) | |
| 1023 | log.info("small resize {d} bytes at {*} to {d}", .{ memory.len, memory.ptr, new_len }); | |
| 1024 | ||
| 1025 | if (config.safety) | |
| 1026 | bucket.requestedSizes(slot_count)[slot_index] = @intCast(new_len); | |
| 1027 | ||
| 1028 | if (config.resize_stack_traces) | |
| 1029 | bucket.captureStackTrace(return_address, slot_count, slot_index, .alloc); | |
| 1030 | ||
| 1031 | return true; | |
| 1032 | } | |
| 1033 | }; | |
| 1034 | } | |
| 1035 | ||
| 1036 | const TraceKind = enum { | |
| 1037 | alloc, | |
| 1038 | free, | |
| 1039 | }; | |
| 1040 | ||
| 1041 | const test_config = Config{}; | |
| 1042 | ||
| 1043 | test "small allocations - free in same order" { | |
| 1044 | var gpa = GeneralPurposeAllocator(test_config){}; | |
| 1045 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1046 | const allocator = gpa.allocator(); | |
| 1047 | ||
| 1048 | var list = std.ArrayList(*u64).init(std.testing.allocator); | |
| 1049 | defer list.deinit(); | |
| 1050 | ||
| 1051 | var i: usize = 0; | |
| 1052 | while (i < 513) : (i += 1) { | |
| 1053 | const ptr = try allocator.create(u64); | |
| 1054 | try list.append(ptr); | |
| 1055 | } | |
| 1056 | ||
| 1057 | for (list.items) |ptr| { | |
| 1058 | allocator.destroy(ptr); | |
| 1059 | } | |
| 1060 | } | |
| 1061 | ||
| 1062 | test "small allocations - free in reverse order" { | |
| 1063 | var gpa = GeneralPurposeAllocator(test_config){}; | |
| 1064 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1065 | const allocator = gpa.allocator(); | |
| 1066 | ||
| 1067 | var list = std.ArrayList(*u64).init(std.testing.allocator); | |
| 1068 | defer list.deinit(); | |
| 1069 | ||
| 1070 | var i: usize = 0; | |
| 1071 | while (i < 513) : (i += 1) { | |
| 1072 | const ptr = try allocator.create(u64); | |
| 1073 | try list.append(ptr); | |
| 1074 | } | |
| 1075 | ||
| 1076 | while (list.popOrNull()) |ptr| { | |
| 1077 | allocator.destroy(ptr); | |
| 1078 | } | |
| 1079 | } | |
| 1080 | ||
| 1081 | test "large allocations" { | |
| 1082 | var gpa = GeneralPurposeAllocator(test_config){}; | |
| 1083 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1084 | const allocator = gpa.allocator(); | |
| 1085 | ||
| 1086 | const ptr1 = try allocator.alloc(u64, 42768); | |
| 1087 | const ptr2 = try allocator.alloc(u64, 52768); | |
| 1088 | allocator.free(ptr1); | |
| 1089 | const ptr3 = try allocator.alloc(u64, 62768); | |
| 1090 | allocator.free(ptr3); | |
| 1091 | allocator.free(ptr2); | |
| 1092 | } | |
| 1093 | ||
| 1094 | test "very large allocation" { | |
| 1095 | var gpa = GeneralPurposeAllocator(test_config){}; | |
| 1096 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1097 | const allocator = gpa.allocator(); | |
| 1098 | ||
| 1099 | try std.testing.expectError(error.OutOfMemory, allocator.alloc(u8, math.maxInt(usize))); | |
| 1100 | } | |
| 1101 | ||
| 1102 | test "realloc" { | |
| 1103 | var gpa = GeneralPurposeAllocator(test_config){}; | |
| 1104 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1105 | const allocator = gpa.allocator(); | |
| 1106 | ||
| 1107 | var slice = try allocator.alignedAlloc(u8, @alignOf(u32), 1); | |
| 1108 | defer allocator.free(slice); | |
| 1109 | slice[0] = 0x12; | |
| 1110 | ||
| 1111 | // This reallocation should keep its pointer address. | |
| 1112 | const old_slice = slice; | |
| 1113 | slice = try allocator.realloc(slice, 2); | |
| 1114 | try std.testing.expect(old_slice.ptr == slice.ptr); | |
| 1115 | try std.testing.expect(slice[0] == 0x12); | |
| 1116 | slice[1] = 0x34; | |
| 1117 | ||
| 1118 | // This requires upgrading to a larger size class | |
| 1119 | slice = try allocator.realloc(slice, 17); | |
| 1120 | try std.testing.expect(slice[0] == 0x12); | |
| 1121 | try std.testing.expect(slice[1] == 0x34); | |
| 1122 | } | |
| 1123 | ||
| 1124 | test "shrink" { | |
| 1125 | var gpa: GeneralPurposeAllocator(test_config) = .{}; | |
| 1126 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1127 | const allocator = gpa.allocator(); | |
| 1128 | ||
| 1129 | var slice = try allocator.alloc(u8, 20); | |
| 1130 | defer allocator.free(slice); | |
| 1131 | ||
| 1132 | @memset(slice, 0x11); | |
| 1133 | ||
| 1134 | try std.testing.expect(allocator.resize(slice, 17)); | |
| 1135 | slice = slice[0..17]; | |
| 1136 | ||
| 1137 | for (slice) |b| { | |
| 1138 | try std.testing.expect(b == 0x11); | |
| 1139 | } | |
| 1140 | ||
| 1141 | // Does not cross size class boundaries when shrinking. | |
| 1142 | try std.testing.expect(!allocator.resize(slice, 16)); | |
| 1143 | } | |
| 1144 | ||
| 1145 | test "large object - grow" { | |
| 1146 | if (builtin.target.isWasm()) { | |
| 1147 | // Not expected to pass on targets that do not have memory mapping. | |
| 1148 | return error.SkipZigTest; | |
| 1149 | } | |
| 1150 | var gpa: GeneralPurposeAllocator(test_config) = .{}; | |
| 1151 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1152 | const allocator = gpa.allocator(); | |
| 1153 | ||
| 1154 | var slice1 = try allocator.alloc(u8, page_size * 2 - 20); | |
| 1155 | defer allocator.free(slice1); | |
| 1156 | ||
| 1157 | const old = slice1; | |
| 1158 | slice1 = try allocator.realloc(slice1, page_size * 2 - 10); | |
| 1159 | try std.testing.expect(slice1.ptr == old.ptr); | |
| 1160 | ||
| 1161 | slice1 = try allocator.realloc(slice1, page_size * 2); | |
| 1162 | try std.testing.expect(slice1.ptr == old.ptr); | |
| 1163 | ||
| 1164 | slice1 = try allocator.realloc(slice1, page_size * 2 + 1); | |
| 1165 | } | |
| 1166 | ||
| 1167 | test "realloc small object to large object" { | |
| 1168 | var gpa = GeneralPurposeAllocator(test_config){}; | |
| 1169 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1170 | const allocator = gpa.allocator(); | |
| 1171 | ||
| 1172 | var slice = try allocator.alloc(u8, 70); | |
| 1173 | defer allocator.free(slice); | |
| 1174 | slice[0] = 0x12; | |
| 1175 | slice[60] = 0x34; | |
| 1176 | ||
| 1177 | // This requires upgrading to a large object | |
| 1178 | const large_object_size = page_size * 2 + 50; | |
| 1179 | slice = try allocator.realloc(slice, large_object_size); | |
| 1180 | try std.testing.expect(slice[0] == 0x12); | |
| 1181 | try std.testing.expect(slice[60] == 0x34); | |
| 1182 | } | |
| 1183 | ||
| 1184 | test "shrink large object to large object" { | |
| 1185 | var gpa: GeneralPurposeAllocator(test_config) = .{}; | |
| 1186 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1187 | const allocator = gpa.allocator(); | |
| 1188 | ||
| 1189 | var slice = try allocator.alloc(u8, page_size * 2 + 50); | |
| 1190 | defer allocator.free(slice); | |
| 1191 | slice[0] = 0x12; | |
| 1192 | slice[60] = 0x34; | |
| 1193 | ||
| 1194 | if (!allocator.resize(slice, page_size * 2 + 1)) return; | |
| 1195 | slice = slice.ptr[0 .. page_size * 2 + 1]; | |
| 1196 | try std.testing.expect(slice[0] == 0x12); | |
| 1197 | try std.testing.expect(slice[60] == 0x34); | |
| 1198 | ||
| 1199 | try std.testing.expect(allocator.resize(slice, page_size * 2 + 1)); | |
| 1200 | slice = slice[0 .. page_size * 2 + 1]; | |
| 1201 | try std.testing.expect(slice[0] == 0x12); | |
| 1202 | try std.testing.expect(slice[60] == 0x34); | |
| 1203 | ||
| 1204 | slice = try allocator.realloc(slice, page_size * 2); | |
| 1205 | try std.testing.expect(slice[0] == 0x12); | |
| 1206 | try std.testing.expect(slice[60] == 0x34); | |
| 1207 | } | |
| 1208 | ||
| 1209 | test "shrink large object to large object with larger alignment" { | |
| 1210 | if (!builtin.link_libc and builtin.os.tag == .wasi) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22731 | |
| 1211 | ||
| 1212 | var gpa = GeneralPurposeAllocator(test_config){}; | |
| 1213 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1214 | const allocator = gpa.allocator(); | |
| 1215 | ||
| 1216 | var debug_buffer: [1000]u8 = undefined; | |
| 1217 | var fba = std.heap.FixedBufferAllocator.init(&debug_buffer); | |
| 1218 | const debug_allocator = fba.allocator(); | |
| 1219 | ||
| 1220 | const alloc_size = page_size * 2 + 50; | |
| 1221 | var slice = try allocator.alignedAlloc(u8, 16, alloc_size); | |
| 1222 | defer allocator.free(slice); | |
| 1223 | ||
| 1224 | const big_alignment: usize = switch (builtin.os.tag) { | |
| 1225 | .windows => page_size * 32, // Windows aligns to 64K. | |
| 1226 | else => page_size * 2, | |
| 1227 | }; | |
| 1228 | // This loop allocates until we find a page that is not aligned to the big | |
| 1229 | // alignment. Then we shrink the allocation after the loop, but increase the | |
| 1230 | // alignment to the higher one, that we know will force it to realloc. | |
| 1231 | var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator); | |
| 1232 | while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) { | |
| 1233 | try stuff_to_free.append(slice); | |
| 1234 | slice = try allocator.alignedAlloc(u8, 16, alloc_size); | |
| 1235 | } | |
| 1236 | while (stuff_to_free.popOrNull()) |item| { | |
| 1237 | allocator.free(item); | |
| 1238 | } | |
| 1239 | slice[0] = 0x12; | |
| 1240 | slice[60] = 0x34; | |
| 1241 | ||
| 1242 | slice = try allocator.reallocAdvanced(slice, big_alignment, alloc_size / 2); | |
| 1243 | try std.testing.expect(slice[0] == 0x12); | |
| 1244 | try std.testing.expect(slice[60] == 0x34); | |
| 1245 | } | |
| 1246 | ||
| 1247 | test "realloc large object to small object" { | |
| 1248 | var gpa = GeneralPurposeAllocator(test_config){}; | |
| 1249 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1250 | const allocator = gpa.allocator(); | |
| 1251 | ||
| 1252 | var slice = try allocator.alloc(u8, page_size * 2 + 50); | |
| 1253 | defer allocator.free(slice); | |
| 1254 | slice[0] = 0x12; | |
| 1255 | slice[16] = 0x34; | |
| 1256 | ||
| 1257 | slice = try allocator.realloc(slice, 19); | |
| 1258 | try std.testing.expect(slice[0] == 0x12); | |
| 1259 | try std.testing.expect(slice[16] == 0x34); | |
| 1260 | } | |
| 1261 | ||
| 1262 | test "overridable mutexes" { | |
| 1263 | var gpa = GeneralPurposeAllocator(.{ .MutexType = std.Thread.Mutex }){ | |
| 1264 | .backing_allocator = std.testing.allocator, | |
| 1265 | .mutex = std.Thread.Mutex{}, | |
| 1266 | }; | |
| 1267 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1268 | const allocator = gpa.allocator(); | |
| 1269 | ||
| 1270 | const ptr = try allocator.create(i32); | |
| 1271 | defer allocator.destroy(ptr); | |
| 1272 | } | |
| 1273 | ||
| 1274 | test "non-page-allocator backing allocator" { | |
| 1275 | var gpa = GeneralPurposeAllocator(.{}){ .backing_allocator = std.testing.allocator }; | |
| 1276 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1277 | const allocator = gpa.allocator(); | |
| 1278 | ||
| 1279 | const ptr = try allocator.create(i32); | |
| 1280 | defer allocator.destroy(ptr); | |
| 1281 | } | |
| 1282 | ||
| 1283 | test "realloc large object to larger alignment" { | |
| 1284 | if (!builtin.link_libc and builtin.os.tag == .wasi) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/22731 | |
| 1285 | ||
| 1286 | var gpa = GeneralPurposeAllocator(test_config){}; | |
| 1287 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1288 | const allocator = gpa.allocator(); | |
| 1289 | ||
| 1290 | var debug_buffer: [1000]u8 = undefined; | |
| 1291 | var fba = std.heap.FixedBufferAllocator.init(&debug_buffer); | |
| 1292 | const debug_allocator = fba.allocator(); | |
| 1293 | ||
| 1294 | var slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50); | |
| 1295 | defer allocator.free(slice); | |
| 1296 | ||
| 1297 | const big_alignment: usize = switch (builtin.os.tag) { | |
| 1298 | .windows => page_size * 32, // Windows aligns to 64K. | |
| 1299 | else => page_size * 2, | |
| 1300 | }; | |
| 1301 | // This loop allocates until we find a page that is not aligned to the big alignment. | |
| 1302 | var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator); | |
| 1303 | while (mem.isAligned(@intFromPtr(slice.ptr), big_alignment)) { | |
| 1304 | try stuff_to_free.append(slice); | |
| 1305 | slice = try allocator.alignedAlloc(u8, 16, page_size * 2 + 50); | |
| 1306 | } | |
| 1307 | while (stuff_to_free.popOrNull()) |item| { | |
| 1308 | allocator.free(item); | |
| 1309 | } | |
| 1310 | slice[0] = 0x12; | |
| 1311 | slice[16] = 0x34; | |
| 1312 | ||
| 1313 | slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 100); | |
| 1314 | try std.testing.expect(slice[0] == 0x12); | |
| 1315 | try std.testing.expect(slice[16] == 0x34); | |
| 1316 | ||
| 1317 | slice = try allocator.reallocAdvanced(slice, 32, page_size * 2 + 25); | |
| 1318 | try std.testing.expect(slice[0] == 0x12); | |
| 1319 | try std.testing.expect(slice[16] == 0x34); | |
| 1320 | ||
| 1321 | slice = try allocator.reallocAdvanced(slice, big_alignment, page_size * 2 + 100); | |
| 1322 | try std.testing.expect(slice[0] == 0x12); | |
| 1323 | try std.testing.expect(slice[16] == 0x34); | |
| 1324 | } | |
| 1325 | ||
| 1326 | test "large object rejects shrinking to small" { | |
| 1327 | if (builtin.target.isWasm()) { | |
| 1328 | // Not expected to pass on targets that do not have memory mapping. | |
| 1329 | return error.SkipZigTest; | |
| 1330 | } | |
| 1331 | ||
| 1332 | var failing_allocator = std.testing.FailingAllocator.init(std.heap.page_allocator, .{ .fail_index = 3 }); | |
| 1333 | var gpa: GeneralPurposeAllocator(.{}) = .{ .backing_allocator = failing_allocator.allocator() }; | |
| 1334 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1335 | const allocator = gpa.allocator(); | |
| 1336 | ||
| 1337 | var slice = try allocator.alloc(u8, page_size * 2 + 50); | |
| 1338 | defer allocator.free(slice); | |
| 1339 | slice[0] = 0x12; | |
| 1340 | slice[3] = 0x34; | |
| 1341 | ||
| 1342 | try std.testing.expect(!allocator.resize(slice, 4)); | |
| 1343 | try std.testing.expect(slice[0] == 0x12); | |
| 1344 | try std.testing.expect(slice[3] == 0x34); | |
| 1345 | } | |
| 1346 | ||
| 1347 | test "objects of size 1024 and 2048" { | |
| 1348 | var gpa = GeneralPurposeAllocator(test_config){}; | |
| 1349 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1350 | const allocator = gpa.allocator(); | |
| 1351 | ||
| 1352 | const slice = try allocator.alloc(u8, 1025); | |
| 1353 | const slice2 = try allocator.alloc(u8, 3000); | |
| 1354 | ||
| 1355 | allocator.free(slice); | |
| 1356 | allocator.free(slice2); | |
| 1357 | } | |
| 1358 | ||
| 1359 | test "setting a memory cap" { | |
| 1360 | var gpa = GeneralPurposeAllocator(.{ .enable_memory_limit = true }){}; | |
| 1361 | defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak"); | |
| 1362 | const allocator = gpa.allocator(); | |
| 1363 | ||
| 1364 | gpa.requested_memory_limit = 1010; | |
| 1365 | ||
| 1366 | const small = try allocator.create(i32); | |
| 1367 | try std.testing.expect(gpa.total_requested_bytes == 4); | |
| 1368 | ||
| 1369 | const big = try allocator.alloc(u8, 1000); | |
| 1370 | try std.testing.expect(gpa.total_requested_bytes == 1004); | |
| 1371 | ||
| 1372 | try std.testing.expectError(error.OutOfMemory, allocator.create(u64)); | |
| 1373 | ||
| 1374 | allocator.destroy(small); | |
| 1375 | try std.testing.expect(gpa.total_requested_bytes == 1000); | |
| 1376 | ||
| 1377 | allocator.free(big); | |
| 1378 | try std.testing.expect(gpa.total_requested_bytes == 0); | |
| 1379 | ||
| 1380 | const exact = try allocator.alloc(u8, 1010); | |
| 1381 | try std.testing.expect(gpa.total_requested_bytes == 1010); | |
| 1382 | allocator.free(exact); | |
| 1383 | } | |
| 1384 | ||
| 1385 | test "large allocations count requested size not backing size" { | |
| 1386 | var gpa: GeneralPurposeAllocator(.{ .enable_memory_limit = true }) = .{}; | |
| 1387 | const allocator = gpa.allocator(); | |
| 1388 | ||
| 1389 | var buf = try allocator.alignedAlloc(u8, 1, page_size + 1); | |
| 1390 | try std.testing.expectEqual(page_size + 1, gpa.total_requested_bytes); | |
| 1391 | buf = try allocator.realloc(buf, 1); | |
| 1392 | try std.testing.expectEqual(1, gpa.total_requested_bytes); | |
| 1393 | buf = try allocator.realloc(buf, 2); | |
| 1394 | try std.testing.expectEqual(2, gpa.total_requested_bytes); | |
| 1395 | } | |
| 1396 | ||
| 1397 | test "retain metadata and never unmap" { | |
| 1398 | var gpa = std.heap.GeneralPurposeAllocator(.{ | |
| 1399 | .safety = true, | |
| 1400 | .never_unmap = true, | |
| 1401 | .retain_metadata = true, | |
| 1402 | }){}; | |
| 1403 | defer std.debug.assert(gpa.deinit() == .ok); | |
| 1404 | const allocator = gpa.allocator(); | |
| 1405 | ||
| 1406 | const alloc = try allocator.alloc(u8, 8); | |
| 1407 | allocator.free(alloc); | |
| 1408 | ||
| 1409 | const alloc2 = try allocator.alloc(u8, 8); | |
| 1410 | allocator.free(alloc2); | |
| 1411 | } |