| ... | @@ -21,67 +21,138 @@ pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig" | ... | @@ -21,67 +21,138 @@ pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig" |
| 21 | | 21 | |
| 22 | const Allocator = mem.Allocator; | 22 | const Allocator = mem.Allocator; |
| 23 | | 23 | |
| 24 | usingnamespace if (comptime @hasDecl(c, "malloc_size")) | 24 | const CAllocator = struct { |
| 25 | struct { | 25 | comptime { |
| 26 | pub const supports_malloc_size = true; | 26 | if (!builtin.link_libc) { |
| 27 | pub const malloc_size = c.malloc_size; | 27 | @compileError("C allocator is only available when linking against libc"); |
| | 28 | } |
| 28 | } | 29 | } |
| 29 | else if (comptime @hasDecl(c, "malloc_usable_size")) | 30 | |
| 30 | struct { | 31 | usingnamespace if (comptime @hasDecl(c, "malloc_size")) |
| 31 | pub const supports_malloc_size = true; | 32 | struct { |
| 32 | pub const malloc_size = c.malloc_usable_size; | 33 | pub const supports_malloc_size = true; |
| | 34 | pub const malloc_size = c.malloc_size; |
| | 35 | } |
| | 36 | else if (comptime @hasDecl(c, "malloc_usable_size")) |
| | 37 | struct { |
| | 38 | pub const supports_malloc_size = true; |
| | 39 | pub const malloc_size = c.malloc_usable_size; |
| | 40 | } |
| | 41 | else if (comptime @hasDecl(c, "_msize")) |
| | 42 | struct { |
| | 43 | pub const supports_malloc_size = true; |
| | 44 | pub const malloc_size = c._msize; |
| | 45 | } |
| | 46 | else |
| | 47 | struct { |
| | 48 | pub const supports_malloc_size = false; |
| | 49 | }; |
| | 50 | |
| | 51 | pub const supports_posix_memalign = @hasDecl(c, "posix_memalign"); |
| | 52 | |
| | 53 | fn getHeader(ptr: [*]u8) *[*]u8 { |
| | 54 | return @intToPtr(*[*]u8, @ptrToInt(ptr) - @sizeOf(usize)); |
| 33 | } | 55 | } |
| 34 | else | | |
| 35 | struct { | | |
| 36 | pub const supports_malloc_size = false; | | |
| 37 | }; | | |
| 38 | | 56 | |
| 39 | pub const c_allocator = &c_allocator_state; | 57 | fn alignedAlloc(len: usize, alignment: usize) ?[*]u8 { |
| 40 | var c_allocator_state = Allocator{ | 58 | if (supports_posix_memalign) { |
| 41 | .allocFn = cAlloc, | 59 | // The posix_memalign only accepts alignment values that are a |
| 42 | .resizeFn = cResize, | 60 | // multiple of the pointer size |
| 43 | }; | 61 | const eff_alignment = std.math.max(alignment, @sizeOf(usize)); |
| | 62 | |
| | 63 | var aligned_ptr: ?*c_void = undefined; |
| | 64 | if (c.posix_memalign(&aligned_ptr, eff_alignment, len) != 0) |
| | 65 | return null; |
| | 66 | |
| | 67 | return @ptrCast([*]u8, aligned_ptr); |
| | 68 | } |
| 44 | | 69 | |
| 45 | fn cAlloc(self: *Allocator, len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Allocator.Error![]u8 { | 70 | // Thin wrapper around regular malloc, overallocate to account for |
| 46 | assert(ptr_align <= @alignOf(c_longdouble)); | 71 | // alignment padding and store the orignal malloc()'ed pointer before |
| 47 | const ptr = @ptrCast([*]u8, c.malloc(len) orelse return error.OutOfMemory); | 72 | // the aligned address. |
| 48 | if (len_align == 0) { | 73 | var unaligned_ptr = @ptrCast([*]u8, c.malloc(len + alignment - 1 + @sizeOf(usize)) orelse return null); |
| 49 | return ptr[0..len]; | 74 | const unaligned_addr = @ptrToInt(unaligned_ptr); |
| | 75 | const aligned_addr = mem.alignForward(unaligned_addr + @sizeOf(usize), alignment); |
| | 76 | var aligned_ptr = unaligned_ptr + (aligned_addr - unaligned_addr); |
| | 77 | getHeader(aligned_ptr).* = unaligned_ptr; |
| | 78 | |
| | 79 | return aligned_ptr; |
| 50 | } | 80 | } |
| 51 | const full_len = init: { | 81 | |
| 52 | if (supports_malloc_size) { | 82 | fn alignedFree(ptr: [*]u8) void { |
| 53 | const s = malloc_size(ptr); | 83 | if (supports_posix_memalign) { |
| 54 | assert(s >= len); | 84 | return c.free(ptr); |
| 55 | break :init s; | | |
| 56 | } | 85 | } |
| 57 | break :init len; | | |
| 58 | }; | | |
| 59 | return ptr[0..mem.alignBackwardAnyAlign(full_len, len_align)]; | | |
| 60 | } | | |
| 61 | | 86 | |
| 62 | fn cResize( | 87 | const unaligned_ptr = getHeader(ptr).*; |
| 63 | self: *Allocator, | 88 | c.free(unaligned_ptr); |
| 64 | buf: []u8, | | |
| 65 | old_align: u29, | | |
| 66 | new_len: usize, | | |
| 67 | len_align: u29, | | |
| 68 | ret_addr: usize, | | |
| 69 | ) Allocator.Error!usize { | | |
| 70 | if (new_len == 0) { | | |
| 71 | c.free(buf.ptr); | | |
| 72 | return 0; | | |
| 73 | } | 89 | } |
| 74 | if (new_len <= buf.len) { | 90 | |
| 75 | return mem.alignAllocLen(buf.len, new_len, len_align); | 91 | fn alignedAllocSize(ptr: [*]u8) usize { |
| | 92 | if (supports_posix_memalign) { |
| | 93 | return malloc_size(ptr); |
| | 94 | } |
| | 95 | |
| | 96 | const unaligned_ptr = getHeader(ptr).*; |
| | 97 | const delta = @ptrToInt(ptr) - @ptrToInt(unaligned_ptr); |
| | 98 | return malloc_size(unaligned_ptr) - delta; |
| 76 | } | 99 | } |
| 77 | if (supports_malloc_size) { | 100 | |
| 78 | const full_len = malloc_size(buf.ptr); | 101 | fn alloc( |
| 79 | if (new_len <= full_len) { | 102 | allocator: *Allocator, |
| 80 | return mem.alignAllocLen(full_len, new_len, len_align); | 103 | len: usize, |
| | 104 | alignment: u29, |
| | 105 | len_align: u29, |
| | 106 | return_address: usize, |
| | 107 | ) error{OutOfMemory}![]u8 { |
| | 108 | assert(len > 0); |
| | 109 | assert(std.math.isPowerOfTwo(alignment)); |
| | 110 | |
| | 111 | var ptr = alignedAlloc(len, alignment) orelse return error.OutOfMemory; |
| | 112 | if (len_align == 0) { |
| | 113 | return ptr[0..len]; |
| 81 | } | 114 | } |
| | 115 | const full_len = init: { |
| | 116 | if (supports_malloc_size) { |
| | 117 | const s = alignedAllocSize(ptr); |
| | 118 | assert(s >= len); |
| | 119 | break :init s; |
| | 120 | } |
| | 121 | break :init len; |
| | 122 | }; |
| | 123 | return ptr[0..mem.alignBackwardAnyAlign(full_len, len_align)]; |
| 82 | } | 124 | } |
| 83 | return error.OutOfMemory; | 125 | |
| 84 | } | 126 | fn resize( |
| | 127 | allocator: *Allocator, |
| | 128 | buf: []u8, |
| | 129 | buf_align: u29, |
| | 130 | new_len: usize, |
| | 131 | len_align: u29, |
| | 132 | return_address: usize, |
| | 133 | ) Allocator.Error!usize { |
| | 134 | if (new_len == 0) { |
| | 135 | alignedFree(buf.ptr); |
| | 136 | return 0; |
| | 137 | } |
| | 138 | if (new_len <= buf.len) { |
| | 139 | return mem.alignAllocLen(buf.len, new_len, len_align); |
| | 140 | } |
| | 141 | if (supports_malloc_size) { |
| | 142 | const full_len = alignedAllocSize(buf.ptr); |
| | 143 | if (new_len <= full_len) { |
| | 144 | return mem.alignAllocLen(full_len, new_len, len_align); |
| | 145 | } |
| | 146 | } |
| | 147 | return error.OutOfMemory; |
| | 148 | } |
| | 149 | }; |
| | 150 | |
| | 151 | pub const c_allocator = &c_allocator_state; |
| | 152 | var c_allocator_state = Allocator{ |
| | 153 | .allocFn = CAllocator.alloc, |
| | 154 | .resizeFn = CAllocator.resize, |
| | 155 | }; |
| 85 | | 156 | |
| 86 | /// This allocator makes a syscall directly for every allocation and free. | 157 | /// This allocator makes a syscall directly for every allocation and free. |
| 87 | /// Thread-safe and lock-free. | 158 | /// Thread-safe and lock-free. |
| ... | @@ -726,9 +797,10 @@ pub fn StackFallbackAllocator(comptime size: usize) type { | ... | @@ -726,9 +797,10 @@ pub fn StackFallbackAllocator(comptime size: usize) type { |
| 726 | | 797 | |
| 727 | test "c_allocator" { | 798 | test "c_allocator" { |
| 728 | if (builtin.link_libc) { | 799 | if (builtin.link_libc) { |
| 729 | var slice = try c_allocator.alloc(u8, 50); | 800 | try testAllocator(c_allocator); |
| 730 | defer c_allocator.free(slice); | 801 | try testAllocatorAligned(c_allocator); |
| 731 | slice = try c_allocator.realloc(slice, 100); | 802 | try testAllocatorLargeAlignment(c_allocator); |
| | 803 | try testAllocatorAlignedShrink(c_allocator); |
| 732 | } | 804 | } |
| 733 | } | 805 | } |
| 734 | | 806 | |
| ... | @@ -772,7 +844,7 @@ test "WasmPageAllocator internals" { | ... | @@ -772,7 +844,7 @@ test "WasmPageAllocator internals" { |
| 772 | test "PageAllocator" { | 844 | test "PageAllocator" { |
| 773 | const allocator = page_allocator; | 845 | const allocator = page_allocator; |
| 774 | try testAllocator(allocator); | 846 | try testAllocator(allocator); |
| 775 | try testAllocatorAligned(allocator, 16); | 847 | try testAllocatorAligned(allocator); |
| 776 | if (!std.Target.current.isWasm()) { | 848 | if (!std.Target.current.isWasm()) { |
| 777 | try testAllocatorLargeAlignment(allocator); | 849 | try testAllocatorLargeAlignment(allocator); |
| 778 | try testAllocatorAlignedShrink(allocator); | 850 | try testAllocatorAlignedShrink(allocator); |
| ... | @@ -802,7 +874,7 @@ test "HeapAllocator" { | ... | @@ -802,7 +874,7 @@ test "HeapAllocator" { |
| 802 | | 874 | |
| 803 | const allocator = &heap_allocator.allocator; | 875 | const allocator = &heap_allocator.allocator; |
| 804 | try testAllocator(allocator); | 876 | try testAllocator(allocator); |
| 805 | try testAllocatorAligned(allocator, 16); | 877 | try testAllocatorAligned(allocator); |
| 806 | try testAllocatorLargeAlignment(allocator); | 878 | try testAllocatorLargeAlignment(allocator); |
| 807 | try testAllocatorAlignedShrink(allocator); | 879 | try testAllocatorAlignedShrink(allocator); |
| 808 | } | 880 | } |
| ... | @@ -813,7 +885,7 @@ test "ArenaAllocator" { | ... | @@ -813,7 +885,7 @@ test "ArenaAllocator" { |
| 813 | defer arena_allocator.deinit(); | 885 | defer arena_allocator.deinit(); |
| 814 | | 886 | |
| 815 | try testAllocator(&arena_allocator.allocator); | 887 | try testAllocator(&arena_allocator.allocator); |
| 816 | try testAllocatorAligned(&arena_allocator.allocator, 16); | 888 | try testAllocatorAligned(&arena_allocator.allocator); |
| 817 | try testAllocatorLargeAlignment(&arena_allocator.allocator); | 889 | try testAllocatorLargeAlignment(&arena_allocator.allocator); |
| 818 | try testAllocatorAlignedShrink(&arena_allocator.allocator); | 890 | try testAllocatorAlignedShrink(&arena_allocator.allocator); |
| 819 | } | 891 | } |
| ... | @@ -823,7 +895,7 @@ test "FixedBufferAllocator" { | ... | @@ -823,7 +895,7 @@ test "FixedBufferAllocator" { |
| 823 | var fixed_buffer_allocator = mem.validationWrap(FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..])); | 895 | var fixed_buffer_allocator = mem.validationWrap(FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..])); |
| 824 | | 896 | |
| 825 | try testAllocator(&fixed_buffer_allocator.allocator); | 897 | try testAllocator(&fixed_buffer_allocator.allocator); |
| 826 | try testAllocatorAligned(&fixed_buffer_allocator.allocator, 16); | 898 | try testAllocatorAligned(&fixed_buffer_allocator.allocator); |
| 827 | try testAllocatorLargeAlignment(&fixed_buffer_allocator.allocator); | 899 | try testAllocatorLargeAlignment(&fixed_buffer_allocator.allocator); |
| 828 | try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator); | 900 | try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator); |
| 829 | } | 901 | } |
| ... | @@ -881,7 +953,7 @@ test "ThreadSafeFixedBufferAllocator" { | ... | @@ -881,7 +953,7 @@ test "ThreadSafeFixedBufferAllocator" { |
| 881 | var fixed_buffer_allocator = ThreadSafeFixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]); | 953 | var fixed_buffer_allocator = ThreadSafeFixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]); |
| 882 | | 954 | |
| 883 | try testAllocator(&fixed_buffer_allocator.allocator); | 955 | try testAllocator(&fixed_buffer_allocator.allocator); |
| 884 | try testAllocatorAligned(&fixed_buffer_allocator.allocator, 16); | 956 | try testAllocatorAligned(&fixed_buffer_allocator.allocator); |
| 885 | try testAllocatorLargeAlignment(&fixed_buffer_allocator.allocator); | 957 | try testAllocatorLargeAlignment(&fixed_buffer_allocator.allocator); |
| 886 | try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator); | 958 | try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator); |
| 887 | } | 959 | } |
| ... | @@ -916,6 +988,10 @@ pub fn testAllocator(base_allocator: *mem.Allocator) !void { | ... | @@ -916,6 +988,10 @@ pub fn testAllocator(base_allocator: *mem.Allocator) !void { |
| 916 | | 988 | |
| 917 | allocator.free(slice); | 989 | allocator.free(slice); |
| 918 | | 990 | |
| | 991 | // Zero-length allocation |
| | 992 | var empty = try allocator.alloc(u8, 0); |
| | 993 | allocator.free(empty); |
| | 994 | // Allocation with zero-sized types |
| 919 | const zero_bit_ptr = try allocator.create(u0); | 995 | const zero_bit_ptr = try allocator.create(u0); |
| 920 | zero_bit_ptr.* = 0; | 996 | zero_bit_ptr.* = 0; |
| 921 | allocator.destroy(zero_bit_ptr); | 997 | allocator.destroy(zero_bit_ptr); |
| ... | @@ -928,31 +1004,34 @@ pub fn testAllocator(base_allocator: *mem.Allocator) !void { | ... | @@ -928,31 +1004,34 @@ pub fn testAllocator(base_allocator: *mem.Allocator) !void { |
| 928 | allocator.free(oversize); | 1004 | allocator.free(oversize); |
| 929 | } | 1005 | } |
| 930 | | 1006 | |
| 931 | pub fn testAllocatorAligned(base_allocator: *mem.Allocator, comptime alignment: u29) !void { | 1007 | pub fn testAllocatorAligned(base_allocator: *mem.Allocator) !void { |
| 932 | var validationAllocator = mem.validationWrap(base_allocator); | 1008 | var validationAllocator = mem.validationWrap(base_allocator); |
| 933 | const allocator = &validationAllocator.allocator; | 1009 | const allocator = &validationAllocator.allocator; |
| 934 | | 1010 | |
| 935 | // initial | 1011 | // Test a few alignment values, smaller and bigger than the type's one |
| 936 | var slice = try allocator.alignedAlloc(u8, alignment, 10); | 1012 | inline for ([_]u29{ 1, 2, 4, 8, 16, 32, 64 }) |alignment| { |
| 937 | testing.expect(slice.len == 10); | 1013 | // initial |
| 938 | // grow | 1014 | var slice = try allocator.alignedAlloc(u8, alignment, 10); |
| 939 | slice = try allocator.realloc(slice, 100); | 1015 | testing.expect(slice.len == 10); |
| 940 | testing.expect(slice.len == 100); | 1016 | // grow |
| 941 | // shrink | 1017 | slice = try allocator.realloc(slice, 100); |
| 942 | slice = allocator.shrink(slice, 10); | 1018 | testing.expect(slice.len == 100); |
| 943 | testing.expect(slice.len == 10); | 1019 | // shrink |
| 944 | // go to zero | 1020 | slice = allocator.shrink(slice, 10); |
| 945 | slice = allocator.shrink(slice, 0); | 1021 | testing.expect(slice.len == 10); |
| 946 | testing.expect(slice.len == 0); | 1022 | // go to zero |
| 947 | // realloc from zero | 1023 | slice = allocator.shrink(slice, 0); |
| 948 | slice = try allocator.realloc(slice, 100); | 1024 | testing.expect(slice.len == 0); |
| 949 | testing.expect(slice.len == 100); | 1025 | // realloc from zero |
| 950 | // shrink with shrink | 1026 | slice = try allocator.realloc(slice, 100); |
| 951 | slice = allocator.shrink(slice, 10); | 1027 | testing.expect(slice.len == 100); |
| 952 | testing.expect(slice.len == 10); | 1028 | // shrink with shrink |
| 953 | // shrink to zero | 1029 | slice = allocator.shrink(slice, 10); |
| 954 | slice = allocator.shrink(slice, 0); | 1030 | testing.expect(slice.len == 10); |
| 955 | testing.expect(slice.len == 0); | 1031 | // shrink to zero |
| | 1032 | slice = allocator.shrink(slice, 0); |
| | 1033 | testing.expect(slice.len == 0); |
| | 1034 | } |
| 956 | } | 1035 | } |
| 957 | | 1036 | |
| 958 | pub fn testAllocatorLargeAlignment(base_allocator: *mem.Allocator) mem.Allocator.Error!void { | 1037 | pub fn testAllocatorLargeAlignment(base_allocator: *mem.Allocator) mem.Allocator.Error!void { |
Andrew Kelley <andrew@ziglang.org> 2020-11-18 20:16:57-08:00
GitHub <noreply@github.com> 2020-11-18 20:16:57-08:00