authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-05-07 12:26:26-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2019-05-07 12:26:26-04:00
log9c0596e6274323d649e12eb6c51864d9e6d087e9
tree6b0738e27a7d8e578b9868b0e4075034e54e2332
parent097a62555e39e44ed403b73350c87e20c9753532
parent043bd716210b0233f1bc0bb90a8fb063cd63d0b8
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #2427 from LemonBoy/linux-tls

Proper support for TLS on linux

6 files changed, 268 insertions(+), 73 deletions(-)

CMakeLists.txt+1
...@@ -611,6 +611,7 @@ set(ZIG_STD_FILES...@@ -611,6 +611,7 @@ set(ZIG_STD_FILES
611 "os/linux.zig"611 "os/linux.zig"
612 "os/linux/arm64.zig"612 "os/linux/arm64.zig"
613 "os/linux/errno.zig"613 "os/linux/errno.zig"
614 "os/linux/tls.zig"
614 "os/linux/vdso.zig"615 "os/linux/vdso.zig"
615 "os/linux/x86_64.zig"616 "os/linux/x86_64.zig"
616 "os/netbsd.zig"617 "os/netbsd.zig"
std/os.zig+5-12
...@@ -3126,9 +3126,6 @@ pub const SpawnThreadError = error{...@@ -3126,9 +3126,6 @@ pub const SpawnThreadError = error{
3126 Unexpected,3126 Unexpected,
3127};3127};
31283128
3129pub var linux_tls_phdr: ?*std.elf.Phdr = null;
3130pub var linux_tls_img_src: [*]const u8 = undefined; // defined if linux_tls_phdr is
3131
3132/// caller must call wait on the returned thread3129/// caller must call wait on the returned thread
3133/// fn startFn(@typeOf(context)) T3130/// fn startFn(@typeOf(context)) T
3134/// where T is u8, noreturn, void, or !void3131/// where T is u8, noreturn, void, or !void
...@@ -3238,12 +3235,10 @@ pub fn spawnThread(context: var, comptime startFn: var) SpawnThreadError!*Thread...@@ -3238,12 +3235,10 @@ pub fn spawnThread(context: var, comptime startFn: var) SpawnThreadError!*Thread
3238 }3235 }
3239 // Finally, the Thread Local Storage, if any.3236 // Finally, the Thread Local Storage, if any.
3240 if (!Thread.use_pthreads) {3237 if (!Thread.use_pthreads) {
3241 if (linux_tls_phdr) |tls_phdr| {3238 if (linux.tls.tls_image) |tls_img| {
3242 l = mem.alignForward(l, tls_phdr.p_align);3239 l = mem.alignForward(l, @alignOf(usize));
3243 tls_start_offset = l;3240 tls_start_offset = l;
3244 l += tls_phdr.p_memsz;3241 l += tls_img.alloc_size;
3245 // the fs register address
3246 l += @sizeOf(usize);
3247 }3242 }
3248 }3243 }
3249 break :blk l;3244 break :blk l;
...@@ -3284,10 +3279,8 @@ pub fn spawnThread(context: var, comptime startFn: var) SpawnThreadError!*Thread...@@ -3284,10 +3279,8 @@ pub fn spawnThread(context: var, comptime startFn: var) SpawnThreadError!*Thread
3284 posix.CLONE_THREAD | posix.CLONE_SYSVSEM | posix.CLONE_PARENT_SETTID | posix.CLONE_CHILD_CLEARTID |3279 posix.CLONE_THREAD | posix.CLONE_SYSVSEM | posix.CLONE_PARENT_SETTID | posix.CLONE_CHILD_CLEARTID |
3285 posix.CLONE_DETACHED;3280 posix.CLONE_DETACHED;
3286 var newtls: usize = undefined;3281 var newtls: usize = undefined;
3287 if (linux_tls_phdr) |tls_phdr| {3282 if (linux.tls.tls_image) |tls_img| {
3288 @memcpy(@intToPtr([*]u8, mmap_addr + tls_start_offset), linux_tls_img_src, tls_phdr.p_filesz);3283 newtls = linux.tls.copyTLS(mmap_addr + tls_start_offset);
3289 newtls = mmap_addr + mmap_len - @sizeOf(usize);
3290 @intToPtr(*usize, newtls).* = newtls;
3291 flags |= posix.CLONE_SETTLS;3284 flags |= posix.CLONE_SETTLS;
3292 }3285 }
3293 const rc = posix.clone(MainFuncs.linuxThreadMain, mmap_addr + stack_end_offset, flags, arg, &thread_ptr.data.handle, newtls, &thread_ptr.data.handle);3286 const rc = posix.clone(MainFuncs.linuxThreadMain, mmap_addr + stack_end_offset, flags, arg, &thread_ptr.data.handle, newtls, &thread_ptr.data.handle);
std/os/linux.zig+1
...@@ -3,6 +3,7 @@ const assert = std.debug.assert;...@@ -3,6 +3,7 @@ const assert = std.debug.assert;
3const builtin = @import("builtin");3const builtin = @import("builtin");
4const maxInt = std.math.maxInt;4const maxInt = std.math.maxInt;
5const elf = std.elf;5const elf = std.elf;
6pub const tls = @import("linux/tls.zig");
6const vdso = @import("linux/vdso.zig");7const vdso = @import("linux/vdso.zig");
7const dl = @import("../dynamic_library.zig");8const dl = @import("../dynamic_library.zig");
8pub use switch (builtin.arch) {9pub use switch (builtin.arch) {
std/os/linux/tls.zig created+247
...@@ -0,0 +1,247 @@
1const std = @import("std");
2const mem = std.mem;
3const posix = std.os.posix;
4const elf = std.elf;
5const builtin = @import("builtin");
6const assert = std.debug.assert;
7
8// This file implements the two TLS variants [1] used by ELF-based systems.
9//
10// The variant I has the following layout in memory:
11// -------------------------------------------------------
12// | DTV | Zig | DTV | Alignment | TLS |
13// | storage | thread data | pointer | | block |
14// ------------------------^------------------------------
15// `-- The thread pointer register points here
16//
17// In this case we allocate additional space for our control structure that's
18// placed _before_ the DTV pointer together with the DTV.
19//
20// NOTE: Some systems such as power64 or mips use this variant with a twist: the
21// alignment is not present and the tp and DTV addresses are offset by a
22// constant.
23//
24// On the other hand the variant II has the following layout in memory:
25// ---------------------------------------
26// | TLS | TCB | Zig | DTV |
27// | block | | thread data | storage |
28// --------^------------------------------
29// `-- The thread pointer register points here
30//
31// The structure of the TCB is not defined by the ABI so we reserve enough space
32// for a single pointer as some architectures such as i386 and x86_64 need a
33// pointer to the TCB block itself at the address pointed by the tp.
34//
35// In this case the control structure and DTV are placed one after another right
36// after the TLS block data.
37//
38// At the moment the DTV is very simple since we only support static TLS, all we
39// need is a two word vector to hold the number of entries (1) and the address
40// of the first TLS block.
41//
42// [1] https://www.akkadia.org/drepper/tls.pdf
43
44const TLSVariant = enum {
45 VariantI,
46 VariantII,
47};
48
49const tls_variant = switch (builtin.arch) {
50 .arm, .armeb, .aarch64, .aarch64_be => TLSVariant.VariantI,
51 .x86_64, .i386 => TLSVariant.VariantII,
52 else => @compileError("undefined tls_variant for this architecture"),
53};
54
55// Controls how many bytes are reserved for the Thread Control Block
56const tls_tcb_size = switch (builtin.arch) {
57 // ARM EABI mandates enough space for two pointers: the first one points to
58 // the DTV while the second one is unspecified but reserved
59 .arm, .armeb, .aarch64, .aarch64_be => 2 * @sizeOf(usize),
60 .i386, .x86_64 => @sizeOf(usize),
61 else => 0,
62};
63
64// Controls if the TCB should be aligned according to the TLS segment p_align
65const tls_tcb_align_size = switch (builtin.arch) {
66 .arm, .armeb, .aarch64, .aarch64_be => true,
67 else => false,
68};
69
70// Check if the architecture-specific parameters look correct
71comptime {
72 if (tls_tcb_align_size and tls_variant != TLSVariant.VariantI) {
73 @compileError("tls_tcb_align_size is only meaningful for variant I TLS");
74 }
75}
76
77// Some architectures add some offset to the tp and dtv addresses in order to
78// make the generated code more efficient
79
80const tls_tp_offset = switch (builtin.arch) {
81 else => 0,
82};
83
84const tls_dtv_offset = switch (builtin.arch) {
85 else => 0,
86};
87
88// Per-thread storage for Zig's use
89const CustomData = packed struct {
90};
91
92// Dynamic Thread Vector
93const DTV = packed struct {
94 entries: usize,
95 tls_block: [1]usize,
96};
97
98// Holds all the information about the process TLS image
99const TLSImage = struct {
100 data_src: []u8,
101 alloc_size: usize,
102 tcb_offset: usize,
103 dtv_offset: usize,
104 data_offset: usize,
105};
106
107pub var tls_image: ?TLSImage = null;
108
109pub fn setThreadPointer(addr: usize) void {
110 switch (builtin.arch) {
111 .x86_64 => {
112 const rc = std.os.linux.syscall2(std.os.linux.SYS_arch_prctl,
113 std.os.linux.ARCH_SET_FS, addr);
114 assert(rc == 0);
115 },
116 .aarch64 => {
117 asm volatile (
118 \\ msr tpidr_el0, %[addr]
119 : : [addr] "r" (addr)
120 );
121 },
122 else => @compileError("Unsupported architecture"),
123 }
124}
125
126pub fn initTLS() void {
127 var tls_phdr: ?*elf.Phdr = null;
128 var img_base: usize = 0;
129
130 const auxv = std.os.linux_elf_aux_maybe.?;
131 var at_phent: usize = undefined;
132 var at_phnum: usize = undefined;
133 var at_phdr: usize = undefined;
134
135 var i: usize = 0;
136 while (auxv[i].a_type != std.elf.AT_NULL) : (i += 1) {
137 switch (auxv[i].a_type) {
138 elf.AT_PHENT => at_phent = auxv[i].a_un.a_val,
139 elf.AT_PHNUM => at_phnum = auxv[i].a_un.a_val,
140 elf.AT_PHDR => at_phdr = auxv[i].a_un.a_val,
141 else => continue,
142 }
143 }
144
145 // Sanity check
146 assert(at_phent == @sizeOf(elf.Phdr));
147
148 // Search the TLS section
149 const phdrs = (@intToPtr([*]elf.Phdr, at_phdr))[0..at_phnum];
150
151 for (phdrs) |*phdr| {
152 switch (phdr.p_type) {
153 elf.PT_PHDR => img_base = at_phdr - phdr.p_vaddr,
154 elf.PT_TLS => tls_phdr = phdr,
155 else => continue,
156 }
157 }
158
159 if (tls_phdr) |phdr| {
160 // Offsets into the allocated TLS area
161 var tcb_offset: usize = undefined;
162 var dtv_offset: usize = undefined;
163 var data_offset: usize = undefined;
164 var thread_data_offset: usize = undefined;
165 // Compute the total size of the ABI-specific data plus our own control
166 // structures
167 const alloc_size = switch (tls_variant) {
168 .VariantI => blk: {
169 var l: usize = 0;
170 dtv_offset = l;
171 l += @sizeOf(DTV);
172 thread_data_offset = l;
173 l += @sizeOf(CustomData);
174 l = mem.alignForward(l, phdr.p_align);
175 tcb_offset = l;
176 if (tls_tcb_align_size) {
177 l += mem.alignForward(tls_tcb_size, phdr.p_align);
178 } else {
179 l += tls_tcb_size;
180 }
181 data_offset = l;
182 l += phdr.p_memsz;
183 break :blk l;
184 },
185 .VariantII => blk: {
186 var l: usize = 0;
187 data_offset = l;
188 l += phdr.p_memsz;
189 l = mem.alignForward(l, phdr.p_align);
190 tcb_offset = l;
191 l += tls_tcb_size;
192 thread_data_offset = l;
193 l += @sizeOf(CustomData);
194 dtv_offset = l;
195 l += @sizeOf(DTV);
196 break :blk l;
197 }
198 };
199
200 tls_image = TLSImage{
201 .data_src = @intToPtr([*]u8, phdr.p_vaddr + img_base)[0..phdr.p_filesz],
202 .alloc_size = alloc_size,
203 .tcb_offset = tcb_offset,
204 .dtv_offset = dtv_offset,
205 .data_offset = data_offset,
206 };
207 }
208}
209
210pub fn copyTLS(addr: usize) usize {
211 const tls_img = tls_image.?;
212
213 // Be paranoid, clear the area we're going to use
214 @memset(@intToPtr([*]u8, addr), 0, tls_img.alloc_size);
215 // Prepare the DTV
216 const dtv = @intToPtr(*DTV, addr + tls_img.dtv_offset);
217 dtv.entries = 1;
218 dtv.tls_block[0] = addr + tls_img.data_offset + tls_dtv_offset;
219 // Set-up the TCB
220 const tcb_ptr = @intToPtr(*usize, addr + tls_img.tcb_offset);
221 if (tls_variant == TLSVariant.VariantI) {
222 tcb_ptr.* = addr + tls_img.dtv_offset;
223 } else {
224 tcb_ptr.* = addr + tls_img.tcb_offset;
225 }
226 // Copy the data
227 @memcpy(@intToPtr([*]u8, addr + tls_img.data_offset), tls_img.data_src.ptr, tls_img.data_src.len);
228
229 // Return the corrected (if needed) value for the tp register
230 return addr + tls_img.tcb_offset + tls_tp_offset;
231}
232
233var main_thread_tls_buffer: [256]u8 align(32) = undefined;
234
235pub fn allocateTLS(size: usize) usize {
236 // Small TLS allocation, use our local buffer
237 if (size < main_thread_tls_buffer.len) {
238 return @ptrToInt(&main_thread_tls_buffer);
239 }
240
241 const addr = posix.mmap(null, size, posix.PROT_READ | posix.PROT_WRITE,
242 posix.MAP_PRIVATE | posix.MAP_ANONYMOUS, -1, 0);
243
244 if (posix.getErrno(addr) != 0) @panic("out of memory");
245
246 return addr;
247}
std/os/linux/x86_64.zig+5
...@@ -388,6 +388,11 @@ pub const VDSO_CGT_VER = "LINUX_2.6";...@@ -388,6 +388,11 @@ pub const VDSO_CGT_VER = "LINUX_2.6";
388pub const VDSO_GETCPU_SYM = "__vdso_getcpu";388pub const VDSO_GETCPU_SYM = "__vdso_getcpu";
389pub const VDSO_GETCPU_VER = "LINUX_2.6";389pub const VDSO_GETCPU_VER = "LINUX_2.6";
390390
391pub const ARCH_SET_GS = 0x1001;
392pub const ARCH_SET_FS = 0x1002;
393pub const ARCH_GET_FS = 0x1003;
394pub const ARCH_GET_GS = 0x1004;
395
391pub fn syscall0(number: usize) usize {396pub fn syscall0(number: usize) usize {
392 return asm volatile ("syscall"397 return asm volatile ("syscall"
393 : [ret] "={rax}" (-> usize)398 : [ret] "={rax}" (-> usize)
std/special/bootstrap.zig+9-61
...@@ -67,24 +67,19 @@ fn posixCallMainAndExit() noreturn {...@@ -67,24 +67,19 @@ fn posixCallMainAndExit() noreturn {
67 var envp_count: usize = 0;67 var envp_count: usize = 0;
68 while (envp_optional[envp_count]) |_| : (envp_count += 1) {}68 while (envp_optional[envp_count]) |_| : (envp_count += 1) {}
69 const envp = @ptrCast([*][*]u8, envp_optional)[0..envp_count];69 const envp = @ptrCast([*][*]u8, envp_optional)[0..envp_count];
70
70 if (builtin.os == builtin.Os.linux) {71 if (builtin.os == builtin.Os.linux) {
71 // Scan auxiliary vector.72 // Find the beginning of the auxiliary vector
72 const auxv = @ptrCast([*]std.elf.Auxv, envp.ptr + envp_count + 1);73 const auxv = @ptrCast([*]std.elf.Auxv, envp.ptr + envp_count + 1);
73 std.os.linux_elf_aux_maybe = auxv;74 std.os.linux_elf_aux_maybe = auxv;
74 var i: usize = 0;75 // Initialize the TLS area
75 var at_phdr: usize = 0;76 std.os.linux.tls.initTLS();
76 var at_phnum: usize = 0;77
77 var at_phent: usize = 0;78 if (std.os.linux.tls.tls_image) |tls_img| {
78 while (auxv[i].a_un.a_val != 0) : (i += 1) {79 const tls_addr = std.os.linux.tls.allocateTLS(tls_img.alloc_size);
79 switch (auxv[i].a_type) {80 const tp = std.os.linux.tls.copyTLS(tls_addr);
80 std.elf.AT_PAGESZ => assert(auxv[i].a_un.a_val == std.os.page_size),81 std.os.linux.tls.setThreadPointer(tp);
81 std.elf.AT_PHDR => at_phdr = auxv[i].a_un.a_val,
82 std.elf.AT_PHNUM => at_phnum = auxv[i].a_un.a_val,
83 std.elf.AT_PHENT => at_phent = auxv[i].a_un.a_val,
84 else => {},
85 }
86 }82 }
87 if (!builtin.single_threaded) linuxInitializeThreadLocalStorage(at_phdr, at_phnum, at_phent);
88 }83 }
8984
90 std.os.posix.exit(callMainWithArgs(argc, argv, envp));85 std.os.posix.exit(callMainWithArgs(argc, argv, envp));
...@@ -140,50 +135,3 @@ inline fn callMain() u8 {...@@ -140,50 +135,3 @@ inline fn callMain() u8 {
140135
141const main_thread_tls_align = 32;136const main_thread_tls_align = 32;
142var main_thread_tls_bytes: [64]u8 align(main_thread_tls_align) = [1]u8{0} ** 64;137var main_thread_tls_bytes: [64]u8 align(main_thread_tls_align) = [1]u8{0} ** 64;
143
144fn linuxInitializeThreadLocalStorage(at_phdr: usize, at_phnum: usize, at_phent: usize) void {
145 var phdr_addr = at_phdr;
146 var n = at_phnum;
147 var base: usize = 0;
148 while (n != 0) : ({
149 n -= 1;
150 phdr_addr += at_phent;
151 }) {
152 const phdr = @intToPtr(*std.elf.Phdr, phdr_addr);
153 // TODO look for PT_DYNAMIC when we have https://github.com/ziglang/zig/issues/1917
154 switch (phdr.p_type) {
155 std.elf.PT_PHDR => base = at_phdr - phdr.p_vaddr,
156 std.elf.PT_TLS => std.os.linux_tls_phdr = phdr,
157 else => continue,
158 }
159 }
160 const tls_phdr = std.os.linux_tls_phdr orelse return;
161 std.os.linux_tls_img_src = @intToPtr([*]const u8, base + tls_phdr.p_vaddr);
162 const end_addr = @ptrToInt(&main_thread_tls_bytes) + tls_phdr.p_memsz;
163 const max_end_addr = @ptrToInt(&main_thread_tls_bytes) + main_thread_tls_bytes.len;
164 assert(max_end_addr >= end_addr + @sizeOf(usize)); // not enough preallocated Thread Local Storage
165 assert(main_thread_tls_align >= tls_phdr.p_align); // preallocated Thread Local Storage not aligned enough
166 @memcpy(&main_thread_tls_bytes, std.os.linux_tls_img_src, tls_phdr.p_filesz);
167 const end_ptr = @intToPtr(*usize, end_addr);
168 end_ptr.* = end_addr;
169 linuxSetThreadArea(end_addr);
170}
171
172fn linuxSetThreadArea(addr: usize) void {
173 switch (builtin.arch) {
174 builtin.Arch.x86_64 => {
175 const ARCH_SET_FS = 0x1002;
176 const rc = std.os.linux.syscall2(std.os.linux.SYS_arch_prctl, ARCH_SET_FS, addr);
177 // acrh_prctl is documented to never fail
178 assert(rc == 0);
179 },
180 builtin.Arch.aarch64 => {
181 asm volatile (
182 \\ msr tpidr_el0,x0
183 \\ mov w0,#0
184 \\ ret
185 );
186 },
187 else => @compileError("Unsupported architecture"),
188 }
189}