authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-11-18 20:16:57-08:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-11-18 20:16:57-08:00
log02a4e5a4bfc605fa8fc62a2d6ad58f0a52bc2ad0
tree98c79232e713f1cadb29d13e0fbe8574ee1acade
parent6d3c176c127b3475a2202b0edd4855374c7ead30
parente749ab1d63cfdacb3ac9e511d1ef52e77799c82b
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #6385 from LemonBoy/callocator

std: Make C allocator respect the required alignment

10 files changed, 228 insertions(+), 94 deletions(-)

lib/std/c.zig-13
......@@ -245,22 +245,9 @@ pub extern "c" fn setregid(rgid: gid_t, egid: gid_t) c_int;
245245pub extern "c" fn setresuid(ruid: uid_t, euid: uid_t, suid: uid_t) c_int;
246246pub extern "c" fn setresgid(rgid: gid_t, egid: gid_t, sgid: gid_t) c_int;
247247
248pub extern "c" fn aligned_alloc(alignment: usize, size: usize) ?*c_void;
249248pub extern "c" fn malloc(usize) ?*c_void;
250
251pub usingnamespace switch (builtin.os.tag) {
252 .linux, .freebsd, .kfreebsd, .netbsd => struct {
253 pub extern "c" fn malloc_usable_size(?*const c_void) usize;
254 },
255 .macos, .ios, .watchos, .tvos => struct {
256 pub extern "c" fn malloc_size(?*const c_void) usize;
257 },
258 else => struct {},
259};
260
261249pub extern "c" fn realloc(?*c_void, usize) ?*c_void;
262250pub extern "c" fn free(*c_void) void;
263pub extern "c" fn posix_memalign(memptr: **c_void, alignment: usize, size: usize) c_int;
264251
265252pub extern "c" fn futimes(fd: fd_t, times: *[2]timeval) c_int;
266253pub extern "c" fn utimes(path: [*:0]const u8, times: *[2]timeval) c_int;
lib/std/c/darwin.zig+3
......@@ -45,6 +45,9 @@ pub const _fstatat = if (builtin.arch == .aarch64) fstatat else @"fstatat$INODE6
4545pub extern "c" fn mach_absolute_time() u64;
4646pub extern "c" fn mach_timebase_info(tinfo: ?*mach_timebase_info_data) void;
4747
48pub extern "c" fn malloc_size(?*const c_void) usize;
49pub extern "c" fn posix_memalign(memptr: *?*c_void, alignment: usize, size: usize) c_int;
50
4851pub extern "c" fn kevent64(
4952 kq: c_int,
5053 changelist: [*]const kevent64_s,
lib/std/c/dragonfly.zig+2
......@@ -17,6 +17,8 @@ pub extern "c" fn getrandom(buf_ptr: [*]u8, buf_len: usize, flags: c_uint) isize
1717pub const dl_iterate_phdr_callback = fn (info: *dl_phdr_info, size: usize, data: ?*c_void) callconv(.C) c_int;
1818pub extern "c" fn dl_iterate_phdr(callback: dl_iterate_phdr_callback, data: ?*c_void) c_int;
1919
20pub extern "c" fn posix_memalign(memptr: *?*c_void, alignment: usize, size: usize) c_int;
21
2022pub const pthread_mutex_t = extern struct {
2123 inner: ?*c_void = null,
2224};
lib/std/c/freebsd.zig+3
......@@ -13,6 +13,9 @@ pub extern "c" fn getdents(fd: c_int, buf_ptr: [*]u8, nbytes: usize) usize;
1313pub extern "c" fn sigaltstack(ss: ?*stack_t, old_ss: ?*stack_t) c_int;
1414pub extern "c" fn getrandom(buf_ptr: [*]u8, buf_len: usize, flags: c_uint) isize;
1515
16pub extern "c" fn posix_memalign(memptr: *?*c_void, alignment: usize, size: usize) c_int;
17pub extern "c" fn malloc_usable_size(?*const c_void) usize;
18
1619pub const sf_hdtr = extern struct {
1720 headers: [*]const iovec_const,
1821 hdr_cnt: c_int,
lib/std/c/linux.zig+2
......@@ -103,6 +103,8 @@ pub extern "c" fn copy_file_range(fd_in: fd_t, off_in: ?*i64, fd_out: fd_t, off_
103103pub extern "c" fn signalfd(fd: fd_t, mask: *const sigset_t, flags: c_uint) c_int;
104104
105105pub extern "c" fn prlimit(pid: pid_t, resource: rlimit_resource, new_limit: *const rlimit, old_limit: *rlimit) c_int;
106pub extern "c" fn posix_memalign(memptr: *?*c_void, alignment: usize, size: usize) c_int;
107pub extern "c" fn malloc_usable_size(?*const c_void) usize;
106108
107109pub const pthread_attr_t = extern struct {
108110 __size: [56]u8,
lib/std/c/netbsd.zig+3
......@@ -30,6 +30,9 @@ pub extern "c" fn __getrusage50(who: c_int, usage: *rusage) c_int;
3030// libc aliases this as sched_yield
3131pub extern "c" fn __libc_thr_yield() c_int;
3232
33pub extern "c" fn posix_memalign(memptr: *?*c_void, alignment: usize, size: usize) c_int;
34pub extern "c" fn malloc_usable_size(?*const c_void) usize;
35
3336pub const pthread_mutex_t = extern struct {
3437 ptm_magic: u32 = 0x33330003,
3538 ptm_errorcheck: padded_pthread_spin_t = 0,
lib/std/c/openbsd.zig+3
......@@ -32,3 +32,6 @@ pub const pthread_spinlock_t = extern struct {
3232pub const pthread_attr_t = extern struct {
3333 inner: ?*c_void = null,
3434};
35
36pub extern "c" fn posix_memalign(memptr: *?*c_void, alignment: usize, size: usize) c_int;
37pub extern "c" fn malloc_usable_size(?*const c_void) usize;
lib/std/c/windows.zig+2
......@@ -4,3 +4,5 @@
44// The MIT license requires this copyright notice to be included in all copies
55// and substantial portions of the software.
66pub extern "c" fn _errno() *c_int;
7
8pub extern "c" fn _msize(memblock: ?*c_void) usize;
lib/std/heap.zig+159-80
......@@ -21,67 +21,138 @@ pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig"
2121
2222const Allocator = mem.Allocator;
2323
24usingnamespace if (comptime @hasDecl(c, "malloc_size"))
25 struct {
26 pub const supports_malloc_size = true;
27 pub const malloc_size = c.malloc_size;
24const CAllocator = struct {
25 comptime {
26 if (!builtin.link_libc) {
27 @compileError("C allocator is only available when linking against libc");
28 }
2829 }
29else if (comptime @hasDecl(c, "malloc_usable_size"))
30 struct {
31 pub const supports_malloc_size = true;
32 pub const malloc_size = c.malloc_usable_size;
30
31 usingnamespace if (comptime @hasDecl(c, "malloc_size"))
32 struct {
33 pub const supports_malloc_size = true;
34 pub const malloc_size = c.malloc_size;
35 }
36 else if (comptime @hasDecl(c, "malloc_usable_size"))
37 struct {
38 pub const supports_malloc_size = true;
39 pub const malloc_size = c.malloc_usable_size;
40 }
41 else if (comptime @hasDecl(c, "_msize"))
42 struct {
43 pub const supports_malloc_size = true;
44 pub const malloc_size = c._msize;
45 }
46 else
47 struct {
48 pub const supports_malloc_size = false;
49 };
50
51 pub const supports_posix_memalign = @hasDecl(c, "posix_memalign");
52
53 fn getHeader(ptr: [*]u8) *[*]u8 {
54 return @intToPtr(*[*]u8, @ptrToInt(ptr) - @sizeOf(usize));
3355 }
34else
35 struct {
36 pub const supports_malloc_size = false;
37 };
3856
39pub const c_allocator = &c_allocator_state;
40var c_allocator_state = Allocator{
41 .allocFn = cAlloc,
42 .resizeFn = cResize,
43};
57 fn alignedAlloc(len: usize, alignment: usize) ?[*]u8 {
58 if (supports_posix_memalign) {
59 // The posix_memalign only accepts alignment values that are a
60 // multiple of the pointer size
61 const eff_alignment = std.math.max(alignment, @sizeOf(usize));
62
63 var aligned_ptr: ?*c_void = undefined;
64 if (c.posix_memalign(&aligned_ptr, eff_alignment, len) != 0)
65 return null;
66
67 return @ptrCast([*]u8, aligned_ptr);
68 }
4469
45fn cAlloc(self: *Allocator, len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Allocator.Error![]u8 {
46 assert(ptr_align <= @alignOf(c_longdouble));
47 const ptr = @ptrCast([*]u8, c.malloc(len) orelse return error.OutOfMemory);
48 if (len_align == 0) {
49 return ptr[0..len];
70 // Thin wrapper around regular malloc, overallocate to account for
71 // alignment padding and store the orignal malloc()'ed pointer before
72 // the aligned address.
73 var unaligned_ptr = @ptrCast([*]u8, c.malloc(len + alignment - 1 + @sizeOf(usize)) orelse return null);
74 const unaligned_addr = @ptrToInt(unaligned_ptr);
75 const aligned_addr = mem.alignForward(unaligned_addr + @sizeOf(usize), alignment);
76 var aligned_ptr = unaligned_ptr + (aligned_addr - unaligned_addr);
77 getHeader(aligned_ptr).* = unaligned_ptr;
78
79 return aligned_ptr;
5080 }
51 const full_len = init: {
52 if (supports_malloc_size) {
53 const s = malloc_size(ptr);
54 assert(s >= len);
55 break :init s;
81
82 fn alignedFree(ptr: [*]u8) void {
83 if (supports_posix_memalign) {
84 return c.free(ptr);
5685 }
57 break :init len;
58 };
59 return ptr[0..mem.alignBackwardAnyAlign(full_len, len_align)];
60}
6186
62fn cResize(
63 self: *Allocator,
64 buf: []u8,
65 old_align: u29,
66 new_len: usize,
67 len_align: u29,
68 ret_addr: usize,
69) Allocator.Error!usize {
70 if (new_len == 0) {
71 c.free(buf.ptr);
72 return 0;
87 const unaligned_ptr = getHeader(ptr).*;
88 c.free(unaligned_ptr);
7389 }
74 if (new_len <= buf.len) {
75 return mem.alignAllocLen(buf.len, new_len, len_align);
90
91 fn alignedAllocSize(ptr: [*]u8) usize {
92 if (supports_posix_memalign) {
93 return malloc_size(ptr);
94 }
95
96 const unaligned_ptr = getHeader(ptr).*;
97 const delta = @ptrToInt(ptr) - @ptrToInt(unaligned_ptr);
98 return malloc_size(unaligned_ptr) - delta;
7699 }
77 if (supports_malloc_size) {
78 const full_len = malloc_size(buf.ptr);
79 if (new_len <= full_len) {
80 return mem.alignAllocLen(full_len, new_len, len_align);
100
101 fn alloc(
102 allocator: *Allocator,
103 len: usize,
104 alignment: u29,
105 len_align: u29,
106 return_address: usize,
107 ) error{OutOfMemory}![]u8 {
108 assert(len > 0);
109 assert(std.math.isPowerOfTwo(alignment));
110
111 var ptr = alignedAlloc(len, alignment) orelse return error.OutOfMemory;
112 if (len_align == 0) {
113 return ptr[0..len];
81114 }
115 const full_len = init: {
116 if (supports_malloc_size) {
117 const s = alignedAllocSize(ptr);
118 assert(s >= len);
119 break :init s;
120 }
121 break :init len;
122 };
123 return ptr[0..mem.alignBackwardAnyAlign(full_len, len_align)];
82124 }
83 return error.OutOfMemory;
84}
125
126 fn resize(
127 allocator: *Allocator,
128 buf: []u8,
129 buf_align: u29,
130 new_len: usize,
131 len_align: u29,
132 return_address: usize,
133 ) Allocator.Error!usize {
134 if (new_len == 0) {
135 alignedFree(buf.ptr);
136 return 0;
137 }
138 if (new_len <= buf.len) {
139 return mem.alignAllocLen(buf.len, new_len, len_align);
140 }
141 if (supports_malloc_size) {
142 const full_len = alignedAllocSize(buf.ptr);
143 if (new_len <= full_len) {
144 return mem.alignAllocLen(full_len, new_len, len_align);
145 }
146 }
147 return error.OutOfMemory;
148 }
149};
150
151pub const c_allocator = &c_allocator_state;
152var c_allocator_state = Allocator{
153 .allocFn = CAllocator.alloc,
154 .resizeFn = CAllocator.resize,
155};
85156
86157/// This allocator makes a syscall directly for every allocation and free.
87158/// Thread-safe and lock-free.
......@@ -726,9 +797,10 @@ pub fn StackFallbackAllocator(comptime size: usize) type {
726797
727798test "c_allocator" {
728799 if (builtin.link_libc) {
729 var slice = try c_allocator.alloc(u8, 50);
730 defer c_allocator.free(slice);
731 slice = try c_allocator.realloc(slice, 100);
800 try testAllocator(c_allocator);
801 try testAllocatorAligned(c_allocator);
802 try testAllocatorLargeAlignment(c_allocator);
803 try testAllocatorAlignedShrink(c_allocator);
732804 }
733805}
734806
......@@ -772,7 +844,7 @@ test "WasmPageAllocator internals" {
772844test "PageAllocator" {
773845 const allocator = page_allocator;
774846 try testAllocator(allocator);
775 try testAllocatorAligned(allocator, 16);
847 try testAllocatorAligned(allocator);
776848 if (!std.Target.current.isWasm()) {
777849 try testAllocatorLargeAlignment(allocator);
778850 try testAllocatorAlignedShrink(allocator);
......@@ -802,7 +874,7 @@ test "HeapAllocator" {
802874
803875 const allocator = &heap_allocator.allocator;
804876 try testAllocator(allocator);
805 try testAllocatorAligned(allocator, 16);
877 try testAllocatorAligned(allocator);
806878 try testAllocatorLargeAlignment(allocator);
807879 try testAllocatorAlignedShrink(allocator);
808880 }
......@@ -813,7 +885,7 @@ test "ArenaAllocator" {
813885 defer arena_allocator.deinit();
814886
815887 try testAllocator(&arena_allocator.allocator);
816 try testAllocatorAligned(&arena_allocator.allocator, 16);
888 try testAllocatorAligned(&arena_allocator.allocator);
817889 try testAllocatorLargeAlignment(&arena_allocator.allocator);
818890 try testAllocatorAlignedShrink(&arena_allocator.allocator);
819891}
......@@ -823,7 +895,7 @@ test "FixedBufferAllocator" {
823895 var fixed_buffer_allocator = mem.validationWrap(FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]));
824896
825897 try testAllocator(&fixed_buffer_allocator.allocator);
826 try testAllocatorAligned(&fixed_buffer_allocator.allocator, 16);
898 try testAllocatorAligned(&fixed_buffer_allocator.allocator);
827899 try testAllocatorLargeAlignment(&fixed_buffer_allocator.allocator);
828900 try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator);
829901}
......@@ -881,7 +953,7 @@ test "ThreadSafeFixedBufferAllocator" {
881953 var fixed_buffer_allocator = ThreadSafeFixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]);
882954
883955 try testAllocator(&fixed_buffer_allocator.allocator);
884 try testAllocatorAligned(&fixed_buffer_allocator.allocator, 16);
956 try testAllocatorAligned(&fixed_buffer_allocator.allocator);
885957 try testAllocatorLargeAlignment(&fixed_buffer_allocator.allocator);
886958 try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator);
887959}
......@@ -916,6 +988,10 @@ pub fn testAllocator(base_allocator: *mem.Allocator) !void {
916988
917989 allocator.free(slice);
918990
991 // Zero-length allocation
992 var empty = try allocator.alloc(u8, 0);
993 allocator.free(empty);
994 // Allocation with zero-sized types
919995 const zero_bit_ptr = try allocator.create(u0);
920996 zero_bit_ptr.* = 0;
921997 allocator.destroy(zero_bit_ptr);
......@@ -928,31 +1004,34 @@ pub fn testAllocator(base_allocator: *mem.Allocator) !void {
9281004 allocator.free(oversize);
9291005}
9301006
931pub fn testAllocatorAligned(base_allocator: *mem.Allocator, comptime alignment: u29) !void {
1007pub fn testAllocatorAligned(base_allocator: *mem.Allocator) !void {
9321008 var validationAllocator = mem.validationWrap(base_allocator);
9331009 const allocator = &validationAllocator.allocator;
9341010
935 // initial
936 var slice = try allocator.alignedAlloc(u8, alignment, 10);
937 testing.expect(slice.len == 10);
938 // grow
939 slice = try allocator.realloc(slice, 100);
940 testing.expect(slice.len == 100);
941 // shrink
942 slice = allocator.shrink(slice, 10);
943 testing.expect(slice.len == 10);
944 // go to zero
945 slice = allocator.shrink(slice, 0);
946 testing.expect(slice.len == 0);
947 // realloc from zero
948 slice = try allocator.realloc(slice, 100);
949 testing.expect(slice.len == 100);
950 // shrink with shrink
951 slice = allocator.shrink(slice, 10);
952 testing.expect(slice.len == 10);
953 // shrink to zero
954 slice = allocator.shrink(slice, 0);
955 testing.expect(slice.len == 0);
1011 // Test a few alignment values, smaller and bigger than the type's one
1012 inline for ([_]u29{ 1, 2, 4, 8, 16, 32, 64 }) |alignment| {
1013 // initial
1014 var slice = try allocator.alignedAlloc(u8, alignment, 10);
1015 testing.expect(slice.len == 10);
1016 // grow
1017 slice = try allocator.realloc(slice, 100);
1018 testing.expect(slice.len == 100);
1019 // shrink
1020 slice = allocator.shrink(slice, 10);
1021 testing.expect(slice.len == 10);
1022 // go to zero
1023 slice = allocator.shrink(slice, 0);
1024 testing.expect(slice.len == 0);
1025 // realloc from zero
1026 slice = try allocator.realloc(slice, 100);
1027 testing.expect(slice.len == 100);
1028 // shrink with shrink
1029 slice = allocator.shrink(slice, 10);
1030 testing.expect(slice.len == 10);
1031 // shrink to zero
1032 slice = allocator.shrink(slice, 0);
1033 testing.expect(slice.len == 0);
1034 }
9561035}
9571036
9581037pub fn testAllocatorLargeAlignment(base_allocator: *mem.Allocator) mem.Allocator.Error!void {
src/stage1/codegen.cpp+51-1
......@@ -8556,7 +8556,57 @@ static void define_builtin_types(CodeGen *g) {
85568556 add_fp_entry(g, "f32", 32, LLVMFloatType(), &g->builtin_types.entry_f32);
85578557 add_fp_entry(g, "f64", 64, LLVMDoubleType(), &g->builtin_types.entry_f64);
85588558 add_fp_entry(g, "f128", 128, LLVMFP128Type(), &g->builtin_types.entry_f128);
8559 add_fp_entry(g, "c_longdouble", 80, LLVMX86FP80Type(), &g->builtin_types.entry_c_longdouble);
8559
8560 switch (g->zig_target->arch) {
8561 case ZigLLVM_x86:
8562 case ZigLLVM_x86_64:
8563 if (g->zig_target->abi != ZigLLVM_MSVC)
8564 add_fp_entry(g, "c_longdouble", 80, LLVMX86FP80Type(), &g->builtin_types.entry_c_longdouble);
8565 else
8566 add_fp_entry(g, "c_longdouble", 64, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble);
8567 break;
8568 case ZigLLVM_arm:
8569 case ZigLLVM_armeb:
8570 case ZigLLVM_thumb:
8571 case ZigLLVM_thumbeb:
8572 add_fp_entry(g, "c_longdouble", 64, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble);
8573 break;
8574 case ZigLLVM_aarch64:
8575 case ZigLLVM_aarch64_be:
8576 if (g->zig_target->os == OsWindows || target_os_is_darwin(g->zig_target->os))
8577 add_fp_entry(g, "c_longdouble", 64, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble);
8578 else
8579 add_fp_entry(g, "c_longdouble", 128, LLVMFP128Type(), &g->builtin_types.entry_c_longdouble);
8580 break;
8581 case ZigLLVM_riscv32:
8582 case ZigLLVM_riscv64:
8583 add_fp_entry(g, "c_longdouble", 128, LLVMFP128Type(), &g->builtin_types.entry_c_longdouble);
8584 break;
8585 case ZigLLVM_wasm32:
8586 case ZigLLVM_wasm64:
8587 add_fp_entry(g, "c_longdouble", 128, LLVMFP128Type(), &g->builtin_types.entry_c_longdouble);
8588 break;
8589 case ZigLLVM_mips:
8590 case ZigLLVM_mipsel:
8591 // Assume o32 ABI
8592 add_fp_entry(g, "c_longdouble", 64, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble);
8593 break;
8594 case ZigLLVM_mips64:
8595 case ZigLLVM_mips64el:
8596 add_fp_entry(g, "c_longdouble", 128, LLVMFP128Type(), &g->builtin_types.entry_c_longdouble);
8597 break;
8598 case ZigLLVM_ppc:
8599 case ZigLLVM_ppc64:
8600 case ZigLLVM_ppc64le:
8601 add_fp_entry(g, "c_longdouble", 128, LLVMFP128Type(), &g->builtin_types.entry_c_longdouble);
8602 break;
8603 case ZigLLVM_avr:
8604 // It's either a float or a double, depending on a toolchain switch
8605 add_fp_entry(g, "c_longdouble", 64, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble);
8606 break;
8607 default:
8608 zig_panic("TODO implement mapping for c_longdouble");
8609 }
85608610
85618611 {
85628612 ZigType *entry = new_type_table_entry(ZigTypeIdVoid);