| ... | @@ -254,79 +254,160 @@ const PageAllocator = struct { | ... | @@ -254,79 +254,160 @@ const PageAllocator = struct { |
| 254 | extern fn @"llvm.wasm.memory.size.i32"(u32) u32; | 254 | extern fn @"llvm.wasm.memory.size.i32"(u32) u32; |
| 255 | extern fn @"llvm.wasm.memory.grow.i32"(u32, u32) i32; | 255 | extern fn @"llvm.wasm.memory.grow.i32"(u32, u32) i32; |
| 256 | | 256 | |
| 257 | /// TODO: make this re-use freed pages, and cooperate with other callers of these global intrinsics | | |
| 258 | /// by better utilizing the return value of grow() | | |
| 259 | const WasmPageAllocator = struct { | 257 | const WasmPageAllocator = struct { |
| 260 | var start_ptr: [*]u8 = undefined; | | |
| 261 | var num_pages: usize = 0; | | |
| 262 | var end_index: usize = 0; | | |
| 263 | | | |
| 264 | comptime { | 258 | comptime { |
| 265 | if (builtin.arch != .wasm32) { | 259 | if (!std.Target.current.isWasm()) { |
| 266 | @compileError("WasmPageAllocator is only available for wasm32 arch"); | 260 | @compileError("WasmPageAllocator is only available for wasm32 arch"); |
| 267 | } | 261 | } |
| 268 | } | 262 | } |
| 269 | | 263 | |
| 270 | fn alloc(allocator: *Allocator, size: usize, alignment: u29) ![]u8 { | 264 | const PageStatus = enum(u1) { |
| 271 | const addr = @ptrToInt(start_ptr) + end_index; | 265 | used = 0, |
| 272 | const adjusted_addr = mem.alignForward(addr, alignment); | 266 | free = 1, |
| 273 | const adjusted_index = end_index + (adjusted_addr - addr); | 267 | |
| 274 | const new_end_index = adjusted_index + size; | 268 | pub const none_free: u8 = 0; |
| | 269 | }; |
| 275 | | 270 | |
| 276 | if (new_end_index > num_pages * mem.page_size) { | 271 | const FreeBlock = struct { |
| 277 | const required_memory = new_end_index - (num_pages * mem.page_size); | 272 | data: []u128, |
| 278 | | 273 | |
| 279 | var inner_num_pages: usize = required_memory / mem.page_size; | 274 | const Io = std.packed_int_array.PackedIntIo(u1, .Little); |
| 280 | if (required_memory % mem.page_size != 0) { | 275 | |
| 281 | inner_num_pages += 1; | 276 | fn totalPages(self: FreeBlock) usize { |
| | 277 | return self.data.len * 128; |
| | 278 | } |
| | 279 | |
| | 280 | fn isInitialized(self: FreeBlock) bool { |
| | 281 | return self.data.len > 0; |
| | 282 | } |
| | 283 | |
| | 284 | fn getBit(self: FreeBlock, idx: usize) PageStatus { |
| | 285 | const bit_offset = 0; |
| | 286 | return @intToEnum(PageStatus, Io.get(@sliceToBytes(self.data), idx, bit_offset)); |
| | 287 | } |
| | 288 | |
| | 289 | fn setBits(self: FreeBlock, start_idx: usize, len: usize, val: PageStatus) void { |
| | 290 | const bit_offset = 0; |
| | 291 | var i: usize = 0; |
| | 292 | while (i < len) : (i += 1) { |
| | 293 | Io.set(@sliceToBytes(self.data), start_idx + i, bit_offset, @enumToInt(val)); |
| 282 | } | 294 | } |
| | 295 | } |
| | 296 | |
| | 297 | // Use '0xFFFFFFFF' as a _missing_ sentinel |
| | 298 | // This saves ~50 bytes compared to returning a nullable |
| | 299 | |
| | 300 | // We can guarantee that conventional memory never gets this big, |
| | 301 | // and wasm32 would not be able to address this memory (32 GB > usize). |
| | 302 | |
| | 303 | // Revisit if this is settled: https://github.com/ziglang/zig/issues/3806 |
| | 304 | const not_found = std.math.maxInt(usize); |
| 283 | | 305 | |
| 284 | const prev_page = @"llvm.wasm.memory.grow.i32"(0, @intCast(u32, inner_num_pages)); | 306 | fn useRecycled(self: FreeBlock, num_pages: usize) usize { |
| 285 | if (prev_page == -1) { | 307 | @setCold(true); |
| 286 | return error.OutOfMemory; | 308 | for (self.data) |segment, i| { |
| | 309 | const spills_into_next = @bitCast(i128, segment) < 0; |
| | 310 | const has_enough_bits = @popCount(u128, segment) >= num_pages; |
| | 311 | |
| | 312 | if (!spills_into_next and !has_enough_bits) continue; |
| | 313 | |
| | 314 | var j: usize = i * 128; |
| | 315 | while (j < (i + 1) * 128) : (j += 1) { |
| | 316 | var count: usize = 0; |
| | 317 | while (j + count < self.totalPages() and self.getBit(j + count) == .free) { |
| | 318 | count += 1; |
| | 319 | if (count >= num_pages) { |
| | 320 | self.setBits(j, num_pages, .used); |
| | 321 | return j; |
| | 322 | } |
| | 323 | } |
| | 324 | j += count; |
| | 325 | } |
| 287 | } | 326 | } |
| | 327 | return not_found; |
| | 328 | } |
| 288 | | 329 | |
| 289 | num_pages += inner_num_pages; | 330 | fn recycle(self: FreeBlock, start_idx: usize, len: usize) void { |
| | 331 | self.setBits(start_idx, len, .free); |
| 290 | } | 332 | } |
| | 333 | }; |
| 291 | | 334 | |
| 292 | const result = start_ptr[adjusted_index..new_end_index]; | 335 | var _conventional_data = [_]u128{0} ** 16; |
| 293 | end_index = new_end_index; | 336 | // Marking `conventional` as const saves ~40 bytes |
| | 337 | const conventional = FreeBlock{ .data = &_conventional_data }; |
| | 338 | var extended = FreeBlock{ .data = &[_]u128{} }; |
| 294 | | 339 | |
| 295 | return result; | 340 | fn extendedOffset() usize { |
| | 341 | return conventional.totalPages(); |
| 296 | } | 342 | } |
| 297 | | 343 | |
| 298 | // Check if memory is the last "item" and is aligned correctly | 344 | fn nPages(memsize: usize) usize { |
| 299 | fn is_last_item(memory: []u8, alignment: u29) bool { | 345 | return std.mem.alignForward(memsize, std.mem.page_size) / std.mem.page_size; |
| 300 | return memory.ptr == start_ptr + end_index - memory.len and mem.alignForward(@ptrToInt(memory.ptr), alignment) == @ptrToInt(memory.ptr); | | |
| 301 | } | 346 | } |
| 302 | | 347 | |
| 303 | fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 { | 348 | fn alloc(allocator: *Allocator, page_count: usize, alignment: u29) error{OutOfMemory}!usize { |
| 304 | // Initialize start_ptr at the first realloc | 349 | var idx = conventional.useRecycled(page_count); |
| 305 | if (num_pages == 0) { | 350 | if (idx != FreeBlock.not_found) { |
| 306 | start_ptr = @intToPtr([*]u8, @intCast(usize, @"llvm.wasm.memory.size.i32"(0)) * mem.page_size); | 351 | return idx; |
| 307 | } | 352 | } |
| 308 | | 353 | |
| 309 | if (is_last_item(old_mem, new_align)) { | 354 | idx = extended.useRecycled(page_count); |
| 310 | const start_index = end_index - old_mem.len; | 355 | if (idx != FreeBlock.not_found) { |
| 311 | const new_end_index = start_index + new_size; | 356 | return idx + extendedOffset(); |
| | 357 | } |
| 312 | | 358 | |
| 313 | if (new_end_index > num_pages * mem.page_size) { | 359 | const prev_page_count = @"llvm.wasm.memory.grow.i32"(0, @intCast(u32, page_count)); |
| 314 | _ = try alloc(allocator, new_end_index - end_index, new_align); | 360 | if (prev_page_count <= 0) { |
| 315 | } | 361 | return error.OutOfMemory; |
| 316 | const result = start_ptr[start_index..new_end_index]; | 362 | } |
| 317 | | 363 | |
| 318 | end_index = new_end_index; | 364 | return @intCast(usize, prev_page_count); |
| 319 | return result; | 365 | } |
| 320 | } else if (new_size <= old_mem.len and new_align <= old_align) { | 366 | |
| | 367 | pub fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) Allocator.Error![]u8 { |
| | 368 | if (new_align > std.mem.page_size) { |
| 321 | return error.OutOfMemory; | 369 | return error.OutOfMemory; |
| | 370 | } |
| | 371 | |
| | 372 | if (nPages(new_size) == nPages(old_mem.len)) { |
| | 373 | return old_mem.ptr[0..new_size]; |
| | 374 | } else if (new_size < old_mem.len) { |
| | 375 | return shrink(allocator, old_mem, old_align, new_size, new_align); |
| 322 | } else { | 376 | } else { |
| 323 | const result = try alloc(allocator, new_size, new_align); | 377 | const page_idx = try alloc(allocator, nPages(new_size), new_align); |
| 324 | mem.copy(u8, result, old_mem); | 378 | const new_mem = @intToPtr([*]u8, page_idx * std.mem.page_size)[0..new_size]; |
| 325 | return result; | 379 | std.mem.copy(u8, new_mem, old_mem); |
| | 380 | _ = shrink(allocator, old_mem, old_align, 0, 0); |
| | 381 | return new_mem; |
| 326 | } | 382 | } |
| 327 | } | 383 | } |
| 328 | | 384 | |
| 329 | fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 { | 385 | pub fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 { |
| | 386 | @setCold(true); |
| | 387 | const free_start = nPages(@ptrToInt(old_mem.ptr) + new_size); |
| | 388 | var free_end = nPages(@ptrToInt(old_mem.ptr) + old_mem.len); |
| | 389 | |
| | 390 | if (free_end > free_start) { |
| | 391 | if (free_start < extendedOffset()) { |
| | 392 | const clamped_end = std.math.min(extendedOffset(), free_end); |
| | 393 | conventional.recycle(free_start, clamped_end - free_start); |
| | 394 | } |
| | 395 | |
| | 396 | if (free_end > extendedOffset()) { |
| | 397 | if (!extended.isInitialized()) { |
| | 398 | // Steal the last page from the memory currently being recycled |
| | 399 | // TODO: would it be better if we use the first page instead? |
| | 400 | free_end -= 1; |
| | 401 | |
| | 402 | extended.data = @intToPtr([*]u128, free_end * std.mem.page_size)[0 .. std.mem.page_size / @sizeOf(u128)]; |
| | 403 | // Since this is the first page being freed and we consume it, assume *nothing* is free. |
| | 404 | std.mem.set(u128, extended.data, PageStatus.none_free); |
| | 405 | } |
| | 406 | const clamped_start = std.math.max(extendedOffset(), free_start); |
| | 407 | extended.recycle(clamped_start - extendedOffset(), free_end - clamped_start); |
| | 408 | } |
| | 409 | } |
| | 410 | |
| 330 | return old_mem[0..new_size]; | 411 | return old_mem[0..new_size]; |
| 331 | } | 412 | } |
| 332 | }; | 413 | }; |
| ... | @@ -724,12 +805,51 @@ test "c_allocator" { | ... | @@ -724,12 +805,51 @@ test "c_allocator" { |
| 724 | } | 805 | } |
| 725 | } | 806 | } |
| 726 | | 807 | |
| | 808 | test "WasmPageAllocator internals" { |
| | 809 | if (comptime std.Target.current.isWasm()) { |
| | 810 | const conventional_memsize = WasmPageAllocator.conventional.totalPages() * std.mem.page_size; |
| | 811 | const initial = try page_allocator.alloc(u8, std.mem.page_size); |
| | 812 | std.debug.assert(@ptrToInt(initial.ptr) < conventional_memsize); // If this isn't conventional, the rest of these tests don't make sense. Also we have a serious memory leak in the test suite. |
| | 813 | |
| | 814 | var inplace = try page_allocator.realloc(initial, 1); |
| | 815 | testing.expectEqual(initial.ptr, inplace.ptr); |
| | 816 | inplace = try page_allocator.realloc(inplace, 4); |
| | 817 | testing.expectEqual(initial.ptr, inplace.ptr); |
| | 818 | page_allocator.free(inplace); |
| | 819 | |
| | 820 | const reuse = try page_allocator.alloc(u8, 1); |
| | 821 | testing.expectEqual(initial.ptr, reuse.ptr); |
| | 822 | page_allocator.free(reuse); |
| | 823 | |
| | 824 | // This segment may span conventional and extended which has really complex rules so we're just ignoring it for now. |
| | 825 | const padding = try page_allocator.alloc(u8, conventional_memsize); |
| | 826 | page_allocator.free(padding); |
| | 827 | |
| | 828 | const extended = try page_allocator.alloc(u8, conventional_memsize); |
| | 829 | testing.expect(@ptrToInt(extended.ptr) >= conventional_memsize); |
| | 830 | |
| | 831 | const use_small = try page_allocator.alloc(u8, 1); |
| | 832 | testing.expectEqual(initial.ptr, use_small.ptr); |
| | 833 | page_allocator.free(use_small); |
| | 834 | |
| | 835 | inplace = try page_allocator.realloc(extended, 1); |
| | 836 | testing.expectEqual(extended.ptr, inplace.ptr); |
| | 837 | page_allocator.free(inplace); |
| | 838 | |
| | 839 | const reuse_extended = try page_allocator.alloc(u8, conventional_memsize); |
| | 840 | testing.expectEqual(extended.ptr, reuse_extended.ptr); |
| | 841 | page_allocator.free(reuse_extended); |
| | 842 | } |
| | 843 | } |
| | 844 | |
| 727 | test "PageAllocator" { | 845 | test "PageAllocator" { |
| 728 | const allocator = page_allocator; | 846 | const allocator = page_allocator; |
| 729 | try testAllocator(allocator); | 847 | try testAllocator(allocator); |
| 730 | try testAllocatorAligned(allocator, 16); | 848 | try testAllocatorAligned(allocator, 16); |
| 731 | try testAllocatorLargeAlignment(allocator); | 849 | if (!std.Target.current.isWasm()) { |
| 732 | try testAllocatorAlignedShrink(allocator); | 850 | try testAllocatorLargeAlignment(allocator); |
| | 851 | try testAllocatorAlignedShrink(allocator); |
| | 852 | } |
| 733 | | 853 | |
| 734 | if (builtin.os == .windows) { | 854 | if (builtin.os == .windows) { |
| 735 | // Trying really large alignment. As mentionned in the implementation, | 855 | // Trying really large alignment. As mentionned in the implementation, |
| ... | @@ -766,7 +886,7 @@ test "ArenaAllocator" { | ... | @@ -766,7 +886,7 @@ test "ArenaAllocator" { |
| 766 | try testAllocatorAlignedShrink(&arena_allocator.allocator); | 886 | try testAllocatorAlignedShrink(&arena_allocator.allocator); |
| 767 | } | 887 | } |
| 768 | | 888 | |
| 769 | var test_fixed_buffer_allocator_memory: [80000 * @sizeOf(u64)]u8 = undefined; | 889 | var test_fixed_buffer_allocator_memory: [800000 * @sizeOf(u64)]u8 = undefined; |
| 770 | test "FixedBufferAllocator" { | 890 | test "FixedBufferAllocator" { |
| 771 | var fixed_buffer_allocator = FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]); | 891 | var fixed_buffer_allocator = FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]); |
| 772 | | 892 | |
Andrew Kelley <andrew@ziglang.org> 2019-12-10 10:58:50-05:00
GitHub <noreply@github.com> 2019-12-10 10:58:50-05:00