authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-08-01 01:15:17-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-08-01 01:15:17-07:00
log91163b44dd8a7d21c76de545bca3ed584a6b6df7
tree0a92462246ac2001876dabf0c640d521176a0b58
parent16dde6d2606b52417c4b27141b2d6c66039d2ad0
parent36332a4fdc8b5bfdaa3a2b6b76772312be6f6a79
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #20857 from alexrp/tls-porting

`std.os.linux.tls`: Refactor, improve documentation, fix a bug, and port to more architectures

3 files changed, 339 insertions(+), 192 deletions(-)

lib/std/Thread.zig+4-4
...@@ -1261,9 +1261,9 @@ const LinuxThreadImpl = struct {...@@ -1261,9 +1261,9 @@ const LinuxThreadImpl = struct {
1261 bytes = std.mem.alignForward(usize, bytes, page_size);1261 bytes = std.mem.alignForward(usize, bytes, page_size);
1262 stack_offset = bytes;1262 stack_offset = bytes;
12631263
1264 bytes = std.mem.alignForward(usize, bytes, linux.tls.tls_image.alloc_align);1264 bytes = std.mem.alignForward(usize, bytes, linux.tls.area_desc.alignment);
1265 tls_offset = bytes;1265 tls_offset = bytes;
1266 bytes += linux.tls.tls_image.alloc_size;1266 bytes += linux.tls.area_desc.size;
12671267
1268 bytes = std.mem.alignForward(usize, bytes, @alignOf(Instance));1268 bytes = std.mem.alignForward(usize, bytes, @alignOf(Instance));
1269 instance_offset = bytes;1269 instance_offset = bytes;
...@@ -1304,12 +1304,12 @@ const LinuxThreadImpl = struct {...@@ -1304,12 +1304,12 @@ const LinuxThreadImpl = struct {
1304 };1304 };
13051305
1306 // Prepare the TLS segment and prepare a user_desc struct when needed on x861306 // Prepare the TLS segment and prepare a user_desc struct when needed on x86
1307 var tls_ptr = linux.tls.prepareTLS(mapped[tls_offset..]);1307 var tls_ptr = linux.tls.prepareArea(mapped[tls_offset..]);
1308 var user_desc: if (target.cpu.arch == .x86) linux.user_desc else void = undefined;1308 var user_desc: if (target.cpu.arch == .x86) linux.user_desc else void = undefined;
1309 if (target.cpu.arch == .x86) {1309 if (target.cpu.arch == .x86) {
1310 defer tls_ptr = @intFromPtr(&user_desc);1310 defer tls_ptr = @intFromPtr(&user_desc);
1311 user_desc = .{1311 user_desc = .{
1312 .entry_number = linux.tls.tls_image.gdt_entry_number,1312 .entry_number = linux.tls.area_desc.gdt_entry_number,
1313 .base_addr = tls_ptr,1313 .base_addr = tls_ptr,
1314 .limit = 0xfffff,1314 .limit = 0xfffff,
1315 .flags = .{1315 .flags = .{
lib/std/os/linux/tls.zig+334-187
...@@ -1,3 +1,14 @@...@@ -1,3 +1,14 @@
1//! This file implements the two TLS variants [1] used by ELF-based systems. Note that, in reality,
2//! Variant I has two sub-variants.
3//!
4//! It is important to understand that the term TCB (Thread Control Block) is overloaded here.
5//! Official ABI documentation uses it simply to mean the ABI TCB, i.e. a small area of ABI-defined
6//! data, usually one or two words (see the `AbiTcb` type below). People will also often use TCB to
7//! refer to the libc TCB, which can be any size and contain anything. (One could even omit it!) We
8//! refer to the latter as the Zig TCB; see the `ZigTcb` type below.
9//!
10//! [1] https://www.akkadia.org/drepper/tls.pdf
11
1const std = @import("std");12const std = @import("std");
2const mem = std.mem;13const mem = std.mem;
3const elf = std.elf;14const elf = std.elf;
...@@ -7,56 +18,63 @@ const native_arch = @import("builtin").cpu.arch;...@@ -7,56 +18,63 @@ const native_arch = @import("builtin").cpu.arch;
7const linux = std.os.linux;18const linux = std.os.linux;
8const posix = std.posix;19const posix = std.posix;
920
10// This file implements the two TLS variants [1] used by ELF-based systems.21/// Represents an ELF TLS variant.
11//22///
12// The variant I has the following layout in memory:23/// In all variants, the TP and the TLS blocks must be aligned to the `p_align` value in the
13// -------------------------------------------------------24/// `PT_TLS` ELF program header. Everything else has natural alignment.
14// | DTV | Zig | DTV | Alignment | TLS |25///
15// | storage | thread data | pointer | | block |26/// The location of the DTV does not actually matter. For simplicity, we put it in the TLS area, but
16// ------------------------^------------------------------27/// there is no actual ABI requirement that it reside there.
17// `-- The thread pointer register points here28const Variant = enum {
18//29 /// The original Variant I:
19// In this case we allocate additional space for our control structure that's30 ///
20// placed _before_ the DTV pointer together with the DTV.31 /// ----------------------------------------
21//32 /// | DTV | Zig TCB | ABI TCB | TLS Blocks |
22// NOTE: Some systems such as power64 or mips use this variant with a twist: the33 /// ----------------^-----------------------
23// alignment is not present and the tp and DTV addresses are offset by a34 /// `-- The TP register points here.
24// constant.35 ///
25//36 /// The layout in this variant necessitates separate alignment of both the TP and the TLS
26// On the other hand the variant II has the following layout in memory:37 /// blocks.
27// ---------------------------------------38 ///
28// | TLS | TCB | Zig | DTV |39 /// The first word in the ABI TCB points to the DTV. For some architectures, there may be a
29// | block | | thread data | storage |40 /// second word with an unspecified meaning.
30// --------^------------------------------41 I_original,
31// `-- The thread pointer register points here42 /// The modified Variant I:
32//43 ///
33// The structure of the TCB is not defined by the ABI so we reserve enough space44 /// ---------------------------------------------------
34// for a single pointer as some architectures such as x86 and x86_64 need a45 /// | DTV | Zig TCB | ABI TCB | [Offset] | TLS Blocks |
35// pointer to the TCB block itself at the address pointed by the tp.46 /// -------------------------------------^-------------
36//47 /// `-- The TP register points here.
37// In this case the control structure and DTV are placed one after another right48 ///
38// after the TLS block data.49 /// The offset (which can be zero) is applied to the TP only; there is never physical gap
39//50 /// between the ABI TCB and the TLS blocks. This implies that we only need to align the TP.
40// At the moment the DTV is very simple since we only support static TLS, all we51 ///
41// need is a two word vector to hold the number of entries (1) and the address52 /// The first (and only) word in the ABI TCB points to the DTV.
42// of the first TLS block.53 I_modified,
43//54 /// Variant II:
44// [1] https://www.akkadia.org/drepper/tls.pdf55 ///
4556 /// ----------------------------------------
46const TLSVariant = enum {57 /// | TLS Blocks | ABI TCB | Zig TCB | DTV |
47 VariantI,58 /// -------------^--------------------------
48 VariantII,59 /// `-- The TP register points here.
60 ///
61 /// The first (and only) word in the ABI TCB points to the ABI TCB itself.
62 II,
49};63};
5064
51const tls_variant = switch (native_arch) {65const current_variant: Variant = switch (native_arch) {
66 .arc,
52 .arm,67 .arm,
53 .armeb,68 .armeb,
54 .thumb,
55 .thumbeb,
56 .aarch64,69 .aarch64,
57 .aarch64_be,70 .aarch64_be,
58 .riscv32,71 .csky,
59 .riscv64,72 .thumb,
73 .thumbeb,
74 => .I_original,
75 .loongarch32,
76 .loongarch64,
77 .m68k,
60 .mips,78 .mips,
61 .mipsel,79 .mipsel,
62 .mips64,80 .mips64,
...@@ -65,73 +83,130 @@ const tls_variant = switch (native_arch) {...@@ -65,73 +83,130 @@ const tls_variant = switch (native_arch) {
65 .powerpcle,83 .powerpcle,
66 .powerpc64,84 .powerpc64,
67 .powerpc64le,85 .powerpc64le,
68 => TLSVariant.VariantI,86 .riscv32,
69 .x86_64, .x86, .sparc64 => TLSVariant.VariantII,87 .riscv64,
70 else => @compileError("undefined tls_variant for this architecture"),88 => .I_modified,
71};89 .hexagon,
7290 .s390x,
73// Controls how many bytes are reserved for the Thread Control Block91 .sparc64,
74const tls_tcb_size = switch (native_arch) {92 .x86,
75 // ARM EABI mandates enough space for two pointers: the first one points to93 .x86_64,
76 // the DTV while the second one is unspecified but reserved94 => .II,
77 .arm, .armeb, .thumb, .thumbeb, .aarch64, .aarch64_be => 2 * @sizeOf(usize),95 else => @compileError("undefined TLS variant for this architecture"),
78 // One pointer-sized word that points either to the DTV or the TCB itself
79 else => @sizeOf(usize),
80};96};
8197
82// Controls if the TP points to the end of the TCB instead of its beginning98/// The Offset value for the modified Variant I.
83const tls_tp_points_past_tcb = switch (native_arch) {99const current_tp_offset = switch (native_arch) {
84 .riscv32, .riscv64, .mips, .mipsel, .mips64, .mips64el, .powerpc, .powerpcle, .powerpc64, .powerpc64le => true,100 .m68k,
85 else => false,101 .mips,
102 .mipsel,
103 .mips64,
104 .mips64el,
105 .powerpc,
106 .powerpcle,
107 .powerpc64,
108 .powerpc64le,
109 => 0x7000,
110 else => 0,
86};111};
87112
88// Some architectures add some offset to the tp and dtv addresses in order to113/// Usually only used by the modified Variant I.
89// make the generated code more efficient114const current_dtv_offset = switch (native_arch) {
90115 .m68k,
91const tls_tp_offset = switch (native_arch) {116 .mips,
92 .mips, .mipsel, .mips64, .mips64el, .powerpc, .powerpcle, .powerpc64, .powerpc64le => 0x7000,117 .mipsel,
118 .mips64,
119 .mips64el,
120 .powerpc,
121 .powerpcle,
122 .powerpc64,
123 .powerpc64le,
124 => 0x8000,
125 .riscv32,
126 .riscv64,
127 => 0x800,
93 else => 0,128 else => 0,
94};129};
95130
96const tls_dtv_offset = switch (native_arch) {131/// Per-thread storage for the ELF TLS ABI.
97 .mips, .mipsel, .mips64, .mips64el, .powerpc, .powerpcle, .powerpc64, .powerpc64le => 0x8000,132const AbiTcb = switch (current_variant) {
98 .riscv32, .riscv64 => 0x800,133 .I_original, .I_modified => switch (native_arch) {
99 else => 0,134 // ARM EABI mandates enough space for two pointers: the first one points to the DTV as
135 // usual, while the second one is unspecified.
136 .aarch64,
137 .aarch64_be,
138 .arm,
139 .armeb,
140 .thumb,
141 .thumbeb,
142 => extern struct {
143 /// This is offset by `current_dtv_offset`.
144 dtv: usize,
145 reserved: ?*anyopaque,
146 },
147 else => extern struct {
148 /// This is offset by `current_dtv_offset`.
149 dtv: usize,
150 },
151 },
152 .II => extern struct {
153 /// This is self-referential.
154 self: *AbiTcb,
155 },
100};156};
101157
102// Per-thread storage for Zig's use158/// Per-thread storage for Zig's use. Currently unused.
103const CustomData = struct {159const ZigTcb = struct {
104 dummy: usize,160 dummy: usize,
105};161};
106162
107// Dynamic Thread Vector163/// Dynamic Thread Vector as specified in the ELF TLS ABI. Ordinarily, there is a block pointer per
108const DTV = extern struct {164/// dynamically-loaded module, but since we only support static TLS, we only need one block pointer.
109 entries: usize,165const Dtv = extern struct {
110 tls_block: [1][*]u8,166 len: usize = 1,
167 tls_block: [*]u8,
111};168};
112169
113// Holds all the information about the process TLS image170/// Describes a process's TLS area. The area encompasses the DTV, both TCBs, and the TLS block, with
114const TLSImage = struct {171/// the exact layout of these being dependent primarily on `current_variant`.
115 init_data: []const u8,172const AreaDesc = struct {
116 alloc_size: usize,173 size: usize,
117 alloc_align: usize,174 alignment: usize,
118 tcb_offset: usize,175
119 dtv_offset: usize,176 dtv: struct {
120 data_offset: usize,177 /// Offset into the TLS area.
121 data_size: usize,178 offset: usize,
122 // Only used on the x86 architecture179 },
180
181 abi_tcb: struct {
182 /// Offset into the TLS area.
183 offset: usize,
184 },
185
186 block: struct {
187 /// The initial data to be copied into the TLS block. Note that this may be smaller than
188 /// `size`, in which case any remaining data in the TLS block is simply left uninitialized.
189 init: []const u8,
190 /// Offset into the TLS area.
191 offset: usize,
192 /// This is the effective size of the TLS block, which may be greater than `init.len`.
193 size: usize,
194 },
195
196 /// Only used on the 32-bit x86 architecture (not x86_64, nor x32).
123 gdt_entry_number: usize,197 gdt_entry_number: usize,
124};198};
125199
126pub var tls_image: TLSImage = undefined;200pub var area_desc: AreaDesc = undefined;
127201
128pub fn setThreadPointer(addr: usize) void {202pub fn setThreadPointer(addr: usize) void {
129 @setRuntimeSafety(false);203 @setRuntimeSafety(false);
130 @disableInstrumentation();204 @disableInstrumentation();
205
131 switch (native_arch) {206 switch (native_arch) {
132 .x86 => {207 .x86 => {
133 var user_desc: linux.user_desc = .{208 var user_desc: linux.user_desc = .{
134 .entry_number = tls_image.gdt_entry_number,209 .entry_number = area_desc.gdt_entry_number,
135 .base_addr = addr,210 .base_addr = addr,
136 .limit = 0xfffff,211 .limit = 0xfffff,
137 .flags = .{212 .flags = .{
...@@ -148,7 +223,7 @@ pub fn setThreadPointer(addr: usize) void {...@@ -148,7 +223,7 @@ pub fn setThreadPointer(addr: usize) void {
148223
149 const gdt_entry_number = user_desc.entry_number;224 const gdt_entry_number = user_desc.entry_number;
150 // We have to keep track of our slot as it's also needed for clone()225 // We have to keep track of our slot as it's also needed for clone()
151 tls_image.gdt_entry_number = gdt_entry_number;226 area_desc.gdt_entry_number = gdt_entry_number;
152 // Update the %gs selector227 // Update the %gs selector
153 asm volatile ("movl %[gs_val], %%gs"228 asm volatile ("movl %[gs_val], %%gs"
154 :229 :
...@@ -166,10 +241,38 @@ pub fn setThreadPointer(addr: usize) void {...@@ -166,10 +241,38 @@ pub fn setThreadPointer(addr: usize) void {
166 : [addr] "r" (addr),241 : [addr] "r" (addr),
167 );242 );
168 },243 },
244 .arc => {
245 // We apparently need to both set r25 (TP) *and* inform the kernel...
246 asm volatile (
247 \\ mov r25, %[addr]
248 :
249 : [addr] "r" (addr),
250 );
251 const rc = @call(.always_inline, linux.syscall1, .{ .arc_settls, addr });
252 assert(rc == 0);
253 },
169 .arm, .armeb, .thumb, .thumbeb => {254 .arm, .armeb, .thumb, .thumbeb => {
170 const rc = @call(.always_inline, linux.syscall1, .{ .set_tls, addr });255 const rc = @call(.always_inline, linux.syscall1, .{ .set_tls, addr });
171 assert(rc == 0);256 assert(rc == 0);
172 },257 },
258 .m68k => {
259 const rc = linux.syscall1(.set_thread_area, addr);
260 assert(rc == 0);
261 },
262 .hexagon => {
263 asm volatile (
264 \\ ugp = %[addr]
265 :
266 : [addr] "r" (addr),
267 );
268 },
269 .loongarch32, .loongarch64 => {
270 asm volatile (
271 \\ mv tp, %[addr]
272 :
273 : [addr] "r" (addr),
274 );
275 },
173 .riscv32, .riscv64 => {276 .riscv32, .riscv64 => {
174 asm volatile (277 asm volatile (
175 \\ mv tp, %[addr]278 \\ mv tp, %[addr]
...@@ -177,7 +280,7 @@ pub fn setThreadPointer(addr: usize) void {...@@ -177,7 +280,7 @@ pub fn setThreadPointer(addr: usize) void {
177 : [addr] "r" (addr),280 : [addr] "r" (addr),
178 );281 );
179 },282 },
180 .mips, .mipsel, .mips64, .mips64el => {283 .csky, .mips, .mipsel, .mips64, .mips64el => {
181 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr });284 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr });
182 assert(rc == 0);285 assert(rc == 0);
183 },286 },
...@@ -195,6 +298,17 @@ pub fn setThreadPointer(addr: usize) void {...@@ -195,6 +298,17 @@ pub fn setThreadPointer(addr: usize) void {
195 : [addr] "r" (addr),298 : [addr] "r" (addr),
196 );299 );
197 },300 },
301 .s390x => {
302 asm volatile (
303 \\ lgr %%r0, %[addr]
304 \\ sar %%a1, %%r0
305 \\ srlg %%r0, %%r0, 32
306 \\ sar %%a0, %%r0
307 :
308 : [addr] "r" (addr),
309 : "r0"
310 );
311 },
198 .sparc64 => {312 .sparc64 => {
199 asm volatile (313 asm volatile (
200 \\ mov %[addr], %%g7314 \\ mov %[addr], %%g7
...@@ -206,7 +320,7 @@ pub fn setThreadPointer(addr: usize) void {...@@ -206,7 +320,7 @@ pub fn setThreadPointer(addr: usize) void {
206 }320 }
207}321}
208322
209fn initTLS(phdrs: []elf.Phdr) void {323fn computeAreaDesc(phdrs: []elf.Phdr) void {
210 @setRuntimeSafety(false);324 @setRuntimeSafety(false);
211 @disableInstrumentation();325 @disableInstrumentation();
212326
...@@ -221,72 +335,103 @@ fn initTLS(phdrs: []elf.Phdr) void {...@@ -221,72 +335,103 @@ fn initTLS(phdrs: []elf.Phdr) void {
221 }335 }
222 }336 }
223337
224 var tls_align_factor: usize = undefined;338 var align_factor: usize = undefined;
225 var tls_data: []const u8 = undefined;339 var block_init: []const u8 = undefined;
226 var tls_data_alloc_size: usize = undefined;340 var block_size: usize = undefined;
341
227 if (tls_phdr) |phdr| {342 if (tls_phdr) |phdr| {
228 // The effective size in memory is represented by p_memsz, the length of343 align_factor = phdr.p_align;
229 // the data stored in the PT_TLS segment is p_filesz and may be less344
230 // than the former345 // The effective size in memory is represented by `p_memsz`; the length of the data stored
231 tls_align_factor = phdr.p_align;346 // in the `PT_TLS` segment is `p_filesz` and may be less than the former.
232 tls_data = @as([*]u8, @ptrFromInt(img_base + phdr.p_vaddr))[0..phdr.p_filesz];347 block_init = @as([*]u8, @ptrFromInt(img_base + phdr.p_vaddr))[0..phdr.p_filesz];
233 tls_data_alloc_size = phdr.p_memsz;348 block_size = phdr.p_memsz;
234 } else {349 } else {
235 tls_align_factor = @alignOf(usize);350 align_factor = @alignOf(usize);
236 tls_data = &[_]u8{};351
237 tls_data_alloc_size = 0;352 block_init = &[_]u8{};
353 block_size = 0;
238 }354 }
239355
240 // Offsets into the allocated TLS area356 // Offsets into the allocated TLS area.
241 var tcb_offset: usize = undefined;
242 var dtv_offset: usize = undefined;357 var dtv_offset: usize = undefined;
243 var data_offset: usize = undefined;358 var abi_tcb_offset: usize = undefined;
244 // Compute the total size of the ABI-specific data plus our own control359 var block_offset: usize = undefined;
245 // structures. All the offset calculated here assume a well-aligned base360
246 // address.361 // Compute the total size of the ABI-specific data plus our own `ZigTcb` structure. All the
247 const alloc_size = switch (tls_variant) {362 // offsets calculated here assume a well-aligned base address.
248 .VariantI => blk: {363 const area_size = switch (current_variant) {
364 .I_original => blk: {
365 var l: usize = 0;
366 dtv_offset = l;
367 l += @sizeOf(Dtv);
368 // Add some padding here so that the TP (`abi_tcb_offset`) is aligned to `align_factor`
369 // and the `ZigTcb` structure can be found by simply subtracting `@sizeOf(ZigTcb)` from
370 // the TP.
371 const delta = (l + @sizeOf(ZigTcb)) & (align_factor - 1);
372 if (delta > 0)
373 l += align_factor - delta;
374 l += @sizeOf(ZigTcb);
375 abi_tcb_offset = l;
376 l += alignForward(@sizeOf(AbiTcb), align_factor);
377 block_offset = l;
378 l += block_size;
379 break :blk l;
380 },
381 .I_modified => blk: {
249 var l: usize = 0;382 var l: usize = 0;
250 dtv_offset = l;383 dtv_offset = l;
251 l += @sizeOf(DTV);384 l += @sizeOf(Dtv);
252 // Add some padding here so that the thread pointer (tcb_offset) is385 // In this variant, the TLS blocks must begin immediately after the end of the ABI TCB,
253 // aligned to p_align and the CustomData structure can be found by386 // with the TP pointing to the beginning of the TLS blocks. Add padding so that the TP
254 // simply subtracting its @sizeOf from the tp value387 // (`abi_tcb_offset`) is aligned to `align_factor` and the `ZigTcb` structure can be
255 const delta = (l + @sizeOf(CustomData)) & (tls_align_factor - 1);388 // found by subtracting `@sizeOf(AbiTcb) + @sizeOf(ZigTcb)` from the TP.
389 const delta = (l + @sizeOf(ZigTcb) + @sizeOf(AbiTcb)) & (align_factor - 1);
256 if (delta > 0)390 if (delta > 0)
257 l += tls_align_factor - delta;391 l += align_factor - delta;
258 l += @sizeOf(CustomData);392 l += @sizeOf(ZigTcb);
259 tcb_offset = l;393 abi_tcb_offset = l;
260 l += alignForward(tls_tcb_size, tls_align_factor);394 l += @sizeOf(AbiTcb);
261 data_offset = l;395 block_offset = l;
262 l += tls_data_alloc_size;396 l += block_size;
263 break :blk l;397 break :blk l;
264 },398 },
265 .VariantII => blk: {399 .II => blk: {
266 var l: usize = 0;400 var l: usize = 0;
267 data_offset = l;401 block_offset = l;
268 l += alignForward(tls_data_alloc_size, tls_align_factor);402 l += alignForward(block_size, align_factor);
269 // The thread pointer is aligned to p_align403 // The TP is aligned to `align_factor`.
270 tcb_offset = l;404 abi_tcb_offset = l;
271 l += tls_tcb_size;405 l += @sizeOf(AbiTcb);
272 // The CustomData structure is right after the TCB with no padding406 // The `ZigTcb` structure is right after the `AbiTcb` with no padding in between so it
273 // in between so it can be easily found407 // can be easily found.
274 l += @sizeOf(CustomData);408 l += @sizeOf(ZigTcb);
275 l = alignForward(l, @alignOf(DTV));409 // It doesn't really matter where we put the DTV, so give it natural alignment.
410 l = alignForward(l, @alignOf(Dtv));
276 dtv_offset = l;411 dtv_offset = l;
277 l += @sizeOf(DTV);412 l += @sizeOf(Dtv);
278 break :blk l;413 break :blk l;
279 },414 },
280 };415 };
281416
282 tls_image = TLSImage{417 area_desc = .{
283 .init_data = tls_data,418 .size = area_size,
284 .alloc_size = alloc_size,419 .alignment = align_factor,
285 .alloc_align = tls_align_factor,420
286 .tcb_offset = tcb_offset,421 .dtv = .{
287 .dtv_offset = dtv_offset,422 .offset = dtv_offset,
288 .data_offset = data_offset,423 },
289 .data_size = tls_data_alloc_size,424
425 .abi_tcb = .{
426 .offset = abi_tcb_offset,
427 },
428
429 .block = .{
430 .init = block_init,
431 .offset = block_offset,
432 .size = block_size,
433 },
434
290 .gdt_entry_number = @as(usize, @bitCast(@as(isize, -1))),435 .gdt_entry_number = @as(usize, @bitCast(@as(isize, -1))),
291 };436 };
292}437}
...@@ -306,78 +451,80 @@ inline fn alignPtrCast(comptime T: type, ptr: [*]u8) *T {...@@ -306,78 +451,80 @@ inline fn alignPtrCast(comptime T: type, ptr: [*]u8) *T {
306 return @ptrCast(@alignCast(ptr));451 return @ptrCast(@alignCast(ptr));
307}452}
308453
309/// Initializes all the fields of the static TLS area and returns the computed454/// Initializes all the fields of the static TLS area and returns the computed architecture-specific
310/// architecture-specific value of the thread-pointer register455/// value of the TP register.
311///456pub fn prepareArea(area: []u8) usize {
312/// This function is inline because thread local storage is not set up yet.
313pub fn prepareTLS(area: []u8) usize {
314 @setRuntimeSafety(false);457 @setRuntimeSafety(false);
315 @disableInstrumentation();458 @disableInstrumentation();
316 // Clear the area we're going to use, just to be safe459
460 // Clear the area we're going to use, just to be safe.
317 @memset(area, 0);461 @memset(area, 0);
318 // Prepare the DTV462
319 const dtv = alignPtrCast(DTV, area.ptr + tls_image.dtv_offset);463 // Prepare the ABI TCB.
320 dtv.entries = 1;464 const abi_tcb = alignPtrCast(AbiTcb, area.ptr + area_desc.abi_tcb.offset);
321 dtv.tls_block[0] = area.ptr + tls_dtv_offset + tls_image.data_offset;465 switch (current_variant) {
322 // Prepare the TCB466 .I_original, .I_modified => abi_tcb.dtv = @intFromPtr(area.ptr + area_desc.dtv.offset),
323 const tcb_ptr = alignPtrCast([*]u8, area.ptr + tls_image.tcb_offset);467 .II => abi_tcb.self = abi_tcb,
324 tcb_ptr.* = switch (tls_variant) {468 }
325 .VariantI => area.ptr + tls_image.dtv_offset,469
326 .VariantII => area.ptr + tls_image.tcb_offset,470 // Prepare the DTV.
471 const dtv = alignPtrCast(Dtv, area.ptr + area_desc.dtv.offset);
472 dtv.len = 1;
473 dtv.tls_block = area.ptr + current_dtv_offset + area_desc.block.offset;
474
475 // Copy the initial data.
476 @memcpy(area[area_desc.block.offset..][0..area_desc.block.init.len], area_desc.block.init);
477
478 // Return the corrected value (if needed) for the TP register. Overflow here is not a problem;
479 // the pointer arithmetic involving the TP is done with wrapping semantics.
480 return @intFromPtr(area.ptr) +% switch (current_variant) {
481 .I_original, .II => area_desc.abi_tcb.offset,
482 .I_modified => area_desc.block.offset +% current_tp_offset,
327 };483 };
328 // Copy the data
329 @memcpy(area[tls_image.data_offset..][0..tls_image.init_data.len], tls_image.init_data);
330
331 // Return the corrected value (if needed) for the tp register.
332 // Overflow here is not a problem, the pointer arithmetic involving the tp
333 // is done with wrapping semantics.
334 return @intFromPtr(area.ptr) +% tls_tp_offset +%
335 if (tls_tp_points_past_tcb) tls_image.data_offset else tls_image.tcb_offset;
336}484}
337485
338// The main motivation for the size chosen here is this is how much ends up being486// The main motivation for the size chosen here is that this is how much ends up being requested for
339// requested for the thread local variables of the std.crypto.random implementation.487// the thread-local variables of the `std.crypto.random` implementation. I'm not sure why it ends up
340// I'm not sure why it ends up being so much; the struct itself is only 64 bytes.488// being so much; the struct itself is only 64 bytes. I think it has to do with being page-aligned
341// I think it has to do with being page aligned and LLVM or LLD is not smart enough489// and LLVM or LLD is not smart enough to lay out the TLS data in a space-conserving way. Anyway, I
342// to lay out the TLS data in a space conserving way. Anyway I think it's fine490// think it's fine because it's less than 3 pages of memory, and putting it in the ELF like this is
343// because it's less than 3 pages of memory, and putting it in the ELF like this491// equivalent to moving the `mmap` call below into the kernel, avoiding syscall overhead.
344// is equivalent to moving the mmap call below into the kernel, avoiding syscall492var main_thread_area_buffer: [0x2100]u8 align(mem.page_size) = undefined;
345// overhead.493
346var main_thread_tls_buffer: [0x2100]u8 align(mem.page_size) = undefined;494/// Computes the layout of the static TLS area, allocates the area, initializes all of its fields,
347495/// and assigns the architecture-specific value to the TP register.
348pub fn initStaticTLS(phdrs: []elf.Phdr) void {496pub fn initStatic(phdrs: []elf.Phdr) void {
349 @setRuntimeSafety(false);497 @setRuntimeSafety(false);
350 @disableInstrumentation();498 @disableInstrumentation();
351499
352 initTLS(phdrs);500 computeAreaDesc(phdrs);
353501
354 const tls_area = blk: {502 const area = blk: {
355 // Fast path for the common case where the TLS data is really small,503 // Fast path for the common case where the TLS data is really small, avoid an allocation and
356 // avoid an allocation and use our local buffer.504 // use our local buffer.
357 if (tls_image.alloc_align <= mem.page_size and505 if (area_desc.alignment <= mem.page_size and area_desc.size <= main_thread_area_buffer.len) {
358 tls_image.alloc_size <= main_thread_tls_buffer.len)506 break :blk main_thread_area_buffer[0..area_desc.size];
359 {
360 break :blk main_thread_tls_buffer[0..tls_image.alloc_size];
361 }507 }
362508
363 const begin_addr = mmap(509 const begin_addr = mmap(
364 null,510 null,
365 tls_image.alloc_size + tls_image.alloc_align - 1,511 area_desc.size + area_desc.alignment - 1,
366 posix.PROT.READ | posix.PROT.WRITE,512 posix.PROT.READ | posix.PROT.WRITE,
367 .{ .TYPE = .PRIVATE, .ANONYMOUS = true },513 .{ .TYPE = .PRIVATE, .ANONYMOUS = true },
368 -1,514 -1,
369 0,515 0,
370 );516 );
371 if (@as(isize, @bitCast(begin_addr)) < 0) @trap();517 if (@as(isize, @bitCast(begin_addr)) < 0) @trap();
372 const alloc_tls_area: [*]align(mem.page_size) u8 = @ptrFromInt(begin_addr);518
519 const area_ptr: [*]align(mem.page_size) u8 = @ptrFromInt(begin_addr);
373520
374 // Make sure the slice is correctly aligned.521 // Make sure the slice is correctly aligned.
375 const begin_aligned_addr = alignForward(begin_addr, tls_image.alloc_align);522 const begin_aligned_addr = alignForward(begin_addr, area_desc.alignment);
376 const start = begin_aligned_addr - begin_addr;523 const start = begin_aligned_addr - begin_addr;
377 break :blk alloc_tls_area[start..][0..tls_image.alloc_size];524 break :blk area_ptr[start..][0..area_desc.size];
378 };525 };
379526
380 const tp_value = prepareTLS(tls_area);527 const tp_value = prepareArea(area);
381 setThreadPointer(tp_value);528 setThreadPointer(tp_value);
382}529}
383530
lib/std/start.zig+1-1
...@@ -469,7 +469,7 @@ fn posixCallMainAndExit(argc_argv_ptr: [*]usize) callconv(.C) noreturn {...@@ -469,7 +469,7 @@ fn posixCallMainAndExit(argc_argv_ptr: [*]usize) callconv(.C) noreturn {
469 }469 }
470470
471 // Initialize the TLS area.471 // Initialize the TLS area.
472 std.os.linux.tls.initStaticTLS(phdrs);472 std.os.linux.tls.initStatic(phdrs);
473 }473 }
474474
475 // The way Linux executables represent stack size is via the PT_GNU_STACK475 // The way Linux executables represent stack size is via the PT_GNU_STACK