1//! This file implements the two TLS variants [1] used by ELF-based systems. Note that, in reality,
2//! Variant I has two sub-variants.
3//!
4//! It is important to understand that the term TCB (Thread Control Block) is overloaded here.
5//! Official ABI documentation uses it simply to mean the ABI TCB, i.e. a small area of ABI-defined
6//! data, usually one or two words (see the `AbiTcb` type below). People will also often use TCB to
7//! refer to the libc TCB, which can be any size and contain anything. (One could even omit it!) We
8//! refer to the latter as the Zig TCB; see the `ZigTcb` type below.
9//!
10//! [1] https://www.akkadia.org/drepper/tls.pdf
11
12const std = @import("std");
13const mem = std.mem;
14const elf = std.elf;
15const math = std.math;
16const assert = std.debug.assert;
17const builtin = @import("builtin");
18const native_arch = builtin.cpu.arch;
19const linux = std.os.linux;
20const page_size_min = std.heap.page_size_min;
21
22/// Represents an ELF TLS variant.
23///
24/// In all variants, the TP and the TLS blocks must be aligned to the `p_align` value in the
25/// `PT.TLS` ELF program header. Everything else has natural alignment.
26///
27/// The location of the DTV does not actually matter. For simplicity, we put it in the TLS area, but
28/// there is no actual ABI requirement that it reside there.
29const Variant = enum {
30 /// The original Variant I:
31 ///
32 /// ----------------------------------------
33 /// | DTV | Zig TCB | ABI TCB | TLS Blocks |
34 /// ----------------^-----------------------
35 /// `-- The TP register points here.
36 ///
37 /// The layout in this variant necessitates separate alignment of both the TP and the TLS
38 /// blocks.
39 ///
40 /// The first word in the ABI TCB points to the DTV. For some architectures, there may be a
41 /// second word with an unspecified meaning.
42 I_original,
43 /// The modified Variant I:
44 ///
45 /// --------------------------------------------
46 /// | DTV | Zig TCB | ABI TCB | TLS Blocks |
47 /// ------------------------------^-------------
48 /// `-- The TP register points here (*inside* the TLS blocks).
49 ///
50 /// The offset from the start of the TLS blocks to the TP register is `current_tp_offset`. It
51 /// may be zero, in which case the TP register points to the start of the TLS blocks.
52 ///
53 /// The first (and only) word in the ABI TCB points to the DTV.
54 I_modified,
55 /// Variant II:
56 ///
57 /// ----------------------------------------
58 /// | TLS Blocks | ABI TCB | Zig TCB | DTV |
59 /// -------------^--------------------------
60 /// `-- The TP register points here.
61 ///
62 /// The first (and only) word in the ABI TCB points to the ABI TCB itself.
63 II,
64};
65
66const current_variant: Variant = switch (native_arch) {
67 .aarch64,
68 .aarch64_be,
69 .alpha,
70 .arc,
71 .arceb,
72 .arm,
73 .armeb,
74 .csky,
75 .hppa,
76 .microblaze,
77 .microblazeel,
78 .sh,
79 .sheb,
80 .thumb,
81 .thumbeb,
82 .xtensa,
83 .xtensaeb,
84 => .I_original,
85 .loongarch32,
86 .loongarch64,
87 .m68k,
88 .mips,
89 .mipsel,
90 .mips64,
91 .mips64el,
92 .or1k,
93 .powerpc,
94 .powerpcle,
95 .powerpc64,
96 .powerpc64le,
97 .riscv32,
98 .riscv64,
99 => .I_modified,
100 .hexagon,
101 .s390x,
102 .sparc,
103 .sparc64,
104 .x86,
105 .x86_64,
106 => .II,
107 else => @compileError("undefined TLS variant for this architecture"),
108};
109
110/// The offset value for the modified Variant I.
111const current_tp_offset = switch (native_arch) {
112 .m68k,
113 .mips,
114 .mipsel,
115 .mips64,
116 .mips64el,
117 .powerpc,
118 .powerpcle,
119 .powerpc64,
120 .powerpc64le,
121 => 0x7000,
122 else => 0,
123};
124
125/// Usually only used by the modified Variant I.
126const current_dtv_offset = switch (native_arch) {
127 .m68k,
128 .mips,
129 .mipsel,
130 .mips64,
131 .mips64el,
132 .powerpc,
133 .powerpcle,
134 .powerpc64,
135 .powerpc64le,
136 => 0x8000,
137 .riscv32,
138 .riscv64,
139 => 0x800,
140 else => 0,
141};
142
143/// Per-thread storage for the ELF TLS ABI.
144const AbiTcb = switch (current_variant) {
145 .I_original, .I_modified => switch (native_arch) {
146 .aarch64,
147 .aarch64_be,
148 .alpha,
149 .arm,
150 .armeb,
151 .hppa,
152 .microblaze,
153 .microblazeel,
154 .sh,
155 .sheb,
156 .thumb,
157 .thumbeb,
158 .xtensa,
159 .xtensaeb,
160 => extern struct {
161 /// This is offset by `current_dtv_offset`.
162 dtv: usize,
163 _reserved: ?*anyopaque,
164 },
165 else => extern struct {
166 /// This is offset by `current_dtv_offset`.
167 dtv: usize,
168 },
169 },
170 .II => extern struct {
171 /// This is self-referential.
172 self: *AbiTcb,
173 },
174};
175
176/// Per-thread storage for Zig's use. Currently unused.
177const ZigTcb = struct {
178 dummy: usize,
179};
180
181/// Dynamic Thread Vector as specified in the ELF TLS ABI. Ordinarily, there is a block pointer per
182/// dynamically-loaded module, but since we only support static TLS, we only need one block pointer.
183const Dtv = extern struct {
184 len: usize = 1,
185 tls_block: [*]u8,
186};
187
188/// Describes a process's TLS area. The area encompasses the DTV, both TCBs, and the TLS block, with
189/// the exact layout of these being dependent primarily on `current_variant`.
190const AreaDesc = struct {
191 size: usize,
192 alignment: usize,
193
194 dtv: struct {
195 /// Offset into the TLS area.
196 offset: usize,
197 },
198
199 abi_tcb: struct {
200 /// Offset into the TLS area.
201 offset: usize,
202 },
203
204 block: struct {
205 /// The initial data to be copied into the TLS block. Note that this may be smaller than
206 /// `size`, in which case any remaining data in the TLS block is simply left uninitialized.
207 init: []const u8,
208 /// Offset into the TLS area.
209 offset: usize,
210 /// This is the effective size of the TLS block, which may be greater than `init.len`.
211 size: usize,
212 },
213
214 /// Only used on the 32-bit x86 architecture (not x86_64, nor x32).
215 gdt_entry_number: usize,
216};
217
218pub var area_desc: AreaDesc = undefined;
219
220pub fn setThreadPointer(addr: usize) void {
221 @setRuntimeSafety(false);
222 @disableInstrumentation();
223
224 switch (native_arch) {
225 .x86 => {
226 var user_desc: linux.user_desc = .{
227 .entry_number = area_desc.gdt_entry_number,
228 .base_addr = addr,
229 .limit = 0xfffff,
230 .flags = .{
231 .seg_32bit = 1,
232 .contents = 0, // Data
233 .read_exec_only = 0,
234 .limit_in_pages = 1,
235 .seg_not_present = 0,
236 .useable = 1,
237 },
238 };
239 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, @intFromPtr(&user_desc) });
240 assert(rc == 0);
241
242 const gdt_entry_number = user_desc.entry_number;
243 // We have to keep track of our slot as it's also needed for clone()
244 area_desc.gdt_entry_number = gdt_entry_number;
245 // Update the %gs selector
246 asm volatile ("movl %[gs_val], %%gs"
247 :
248 : [gs_val] "r" (gdt_entry_number << 3 | 3),
249 );
250 },
251 .x86_64 => {
252 const rc = @call(.always_inline, linux.syscall2, .{ .arch_prctl, linux.ARCH.SET_FS, addr });
253 assert(rc == 0);
254 },
255 .aarch64, .aarch64_be => {
256 asm volatile (
257 \\ msr tpidr_el0, %[addr]
258 :
259 : [addr] "r" (addr),
260 );
261 },
262 .alpha => {
263 asm volatile (
264 \\ wruniq
265 :
266 : [addr] "{$16}" (addr),
267 );
268 },
269 .arc, .arceb => {
270 // We apparently need to both set r25 (TP) *and* inform the kernel...
271 asm volatile (
272 \\ mov r25, %[addr]
273 :
274 : [addr] "r" (addr),
275 );
276 const rc = @call(.always_inline, linux.syscall1, .{ .arc_settls, addr });
277 assert(rc == 0);
278 },
279 .arm, .armeb, .thumb, .thumbeb => {
280 const rc = @call(.always_inline, linux.syscall1, .{ .set_tls, addr });
281 assert(rc == 0);
282 },
283 .m68k => {
284 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr });
285 assert(rc == 0);
286 },
287 .hexagon => {
288 asm volatile (
289 \\ ugp = %[addr]
290 :
291 : [addr] "r" (addr),
292 );
293 },
294 .hppa => {
295 asm volatile (
296 \\ ble 0xe0(%%sr2, %%r0)
297 \\ nop
298 :
299 : [addr] "{r26}" (addr),
300 : .{ .r31 = true });
301 },
302 .loongarch32, .loongarch64 => {
303 asm volatile (
304 \\ move $tp, %[addr]
305 :
306 : [addr] "r" (addr),
307 );
308 },
309 .riscv32, .riscv64 => {
310 asm volatile (
311 \\ mv tp, %[addr]
312 :
313 : [addr] "r" (addr),
314 );
315 },
316 .csky, .mips, .mipsel, .mips64, .mips64el => {
317 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr });
318 assert(rc == 0);
319 },
320 .microblaze, .microblazeel => {
321 asm volatile (
322 \\ ori r21, %[addr], 0
323 :
324 : [addr] "r" (addr),
325 );
326 },
327 .or1k => {
328 asm volatile (
329 \\ l.ori r10, %[addr], 0
330 :
331 : [addr] "r" (addr),
332 );
333 },
334 .powerpc, .powerpcle => {
335 asm volatile (
336 \\ mr 2, %[addr]
337 :
338 : [addr] "r" (addr),
339 );
340 },
341 .powerpc64, .powerpc64le => {
342 asm volatile (
343 \\ mr 13, %[addr]
344 :
345 : [addr] "r" (addr),
346 );
347 },
348 .s390x => {
349 asm volatile (
350 \\ lgr %%r0, %[addr]
351 \\ sar %%a1, %%r0
352 \\ srlg %%r0, %%r0, 32
353 \\ sar %%a0, %%r0
354 :
355 : [addr] "r" (addr),
356 : .{ .r0 = true });
357 },
358 .sh, .sheb => {
359 asm volatile (
360 \\ ldc %[addr], gbr
361 :
362 : [addr] "r" (addr),
363 );
364 },
365 .sparc, .sparc64 => {
366 asm volatile (
367 \\ mov %[addr], %%g7
368 :
369 : [addr] "r" (addr),
370 );
371 },
372 .xtensa, .xtensaeb => {
373 asm volatile (
374 \\ wur %[addr], threadptr
375 :
376 : [addr] "a" (addr),
377 );
378 },
379 else => @compileError("Unsupported architecture"),
380 }
381}
382
383pub fn getThreadPointer() usize {
384 @setRuntimeSafety(false);
385 @disableInstrumentation();
386
387 return switch (native_arch) {
388 .aarch64, .aarch64_be => asm (
389 \\ mrs %[ret], tpidr_el0
390 : [ret] "=r" (-> usize),
391 ),
392 .alpha => asm (
393 \\ rduniq
394 : [ret] "={$0}" (-> usize),
395 ),
396 .arc, .arceb => asm (
397 \\ mov %[ret], r25
398 : [ret] "=r" (-> usize),
399 ),
400 .arm, .armeb, .thumb, .thumbeb => asm (
401 \\ mrc p15, 0, %[ret], c13, c0, 3
402 : [ret] "=r" (-> usize),
403 ),
404 .csky => asm (
405 \\ mov %[ret], r31
406 : [ret] "=r" (-> usize),
407 ),
408 .hexagon => asm (
409 \\ %[ret] = ugp
410 : [ret] "=r" (-> usize),
411 ),
412 .hppa => asm (
413 \\ mfctl %%cr27, %[ret]
414 : [ret] "=r" (-> usize),
415 ),
416 .loongarch32, .loongarch64 => asm (
417 \\ move %[ret], $tp
418 : [ret] "=r" (-> usize),
419 ),
420 .m68k => linux.syscall0(.get_thread_area),
421 .mips, .mipsel, .mips64, .mips64el => asm (
422 \\ rdhwr %[ret], $29
423 : [ret] "=r" (-> usize),
424 ),
425 .microblaze, .microblazeel => asm (
426 \\ ori %[ret], r21, 0
427 : [ret] "=r" (-> usize),
428 ),
429 .or1k => asm (
430 \\ l.ori %[ret], r10, 0
431 : [ret] "=r" (-> usize),
432 ),
433 .riscv32, .riscv64 => asm (
434 \\ mv %[ret], tp
435 : [ret] "=r" (-> usize),
436 ),
437 .powerpc, .powerpcle => asm (
438 \\ mr %[ret], 2
439 : [ret] "=r" (-> usize),
440 ),
441 .powerpc64, .powerpc64le => asm (
442 \\ mr %[ret], 13
443 : [ret] "=r" (-> usize),
444 ),
445 .s390x => asm (
446 \\ ear %[ret], %%a0
447 \\ sllg %[ret], %[ret], 32
448 \\ ear %[ret], %%a1
449 : [ret] "=r" (-> usize),
450 ),
451 .sh, .sheb => asm (
452 \\ stc gbr, %[ret]
453 : [ret] "=r" (-> usize),
454 ),
455 .sparc, .sparc64 => asm (
456 \\ mov %%g7, %[ret]
457 : [ret] "=r" (-> usize),
458 ),
459 .x86 => asm (
460 \\ movl %%gs:0, %[ret]
461 : [ret] "=r" (-> usize),
462 ),
463 .x86_64 => switch (@sizeOf(usize)) {
464 8 => asm (
465 \\ movq %%fs:0, %[ret]
466 : [ret] "=r" (-> usize),
467 ),
468 // On x32, usize is 32 bits.
469 4 => asm (
470 \\ movl %%fs:0, %[ret]
471 : [ret] "=r" (-> usize),
472 ),
473 else => comptime unreachable,
474 },
475 .xtensa, .xtensaeb => asm (
476 \\ rur %[ret], threadptr
477 : [ret] "=r" (-> usize),
478 ),
479 else => @compileError("Unsupported architecture"),
480 };
481}
482
483fn computeAreaDesc(phdrs: []elf.ElfN.Phdr) void {
484 @setRuntimeSafety(false);
485 @disableInstrumentation();
486
487 var tls_phdr: ?*elf.ElfN.Phdr = null;
488 var img_base: usize = 0;
489
490 for (phdrs) |*phdr| {
491 switch (phdr.type) {
492 .PHDR => img_base = @intFromPtr(phdrs.ptr) - phdr.vaddr,
493 .TLS => tls_phdr = phdr,
494 else => {},
495 }
496 }
497
498 var align_factor: usize = undefined;
499 var block_init: []const u8 = undefined;
500 var block_size: usize = undefined;
501
502 if (tls_phdr) |phdr| {
503 align_factor = phdr.@"align";
504
505 // The effective size in memory is represented by `memsz`; the length of the data stored
506 // in the `PT.TLS` segment is `filesz` and may be less than the former.
507 block_init = @as([*]u8, @ptrFromInt(img_base + phdr.vaddr))[0..phdr.filesz];
508 block_size = phdr.memsz;
509 } else {
510 align_factor = @alignOf(usize);
511
512 block_init = &[_]u8{};
513 block_size = 0;
514 }
515
516 // Offsets into the allocated TLS area.
517 var dtv_offset: usize = undefined;
518 var abi_tcb_offset: usize = undefined;
519 var block_offset: usize = undefined;
520
521 // Compute the total size of the ABI-specific data plus our own `ZigTcb` structure. All the
522 // offsets calculated here assume a well-aligned base address.
523 const area_size = switch (current_variant) {
524 .I_original => blk: {
525 var l: usize = 0;
526 dtv_offset = l;
527 l += @sizeOf(Dtv);
528 // Add some padding here so that the TP (`abi_tcb_offset`) is aligned to `align_factor`
529 // and the `ZigTcb` structure can be found by simply subtracting `@sizeOf(ZigTcb)` from
530 // the TP.
531 const delta = (l + @sizeOf(ZigTcb)) & (align_factor - 1);
532 if (delta > 0)
533 l += align_factor - delta;
534 l += @sizeOf(ZigTcb);
535 abi_tcb_offset = l;
536 l += alignForward(@sizeOf(AbiTcb), align_factor);
537 block_offset = l;
538 l += block_size;
539 break :blk l;
540 },
541 .I_modified => blk: {
542 var l: usize = 0;
543 dtv_offset = l;
544 l += @sizeOf(Dtv);
545 // In this variant, the TLS blocks must begin immediately after the end of the ABI TCB,
546 // with the TP pointing to the beginning of the TLS blocks. Add padding so that the TP
547 // (`abi_tcb_offset`) is aligned to `align_factor` and the `ZigTcb` structure can be
548 // found by subtracting `@sizeOf(AbiTcb) + @sizeOf(ZigTcb)` from the TP.
549 const delta = (l + @sizeOf(ZigTcb) + @sizeOf(AbiTcb)) & (align_factor - 1);
550 if (delta > 0)
551 l += align_factor - delta;
552 l += @sizeOf(ZigTcb);
553 abi_tcb_offset = l;
554 l += @sizeOf(AbiTcb);
555 block_offset = l;
556 l += block_size;
557 break :blk l;
558 },
559 .II => blk: {
560 var l: usize = 0;
561 block_offset = l;
562 l += alignForward(block_size, align_factor);
563 // The TP is aligned to `align_factor`.
564 abi_tcb_offset = l;
565 l += @sizeOf(AbiTcb);
566 // The `ZigTcb` structure is right after the `AbiTcb` with no padding in between so it
567 // can be easily found.
568 l += @sizeOf(ZigTcb);
569 // It doesn't really matter where we put the DTV, so give it natural alignment.
570 l = alignForward(l, @alignOf(Dtv));
571 dtv_offset = l;
572 l += @sizeOf(Dtv);
573 break :blk l;
574 },
575 };
576
577 area_desc = .{
578 .size = area_size,
579 .alignment = align_factor,
580
581 .dtv = .{
582 .offset = dtv_offset,
583 },
584
585 .abi_tcb = .{
586 .offset = abi_tcb_offset,
587 },
588
589 .block = .{
590 .init = block_init,
591 .offset = block_offset,
592 .size = block_size,
593 },
594
595 .gdt_entry_number = @as(usize, @bitCast(@as(isize, -1))),
596 };
597}
598
599/// Inline because TLS is not set up yet.
600inline fn alignForward(addr: usize, alignment: usize) usize {
601 return alignBackward(addr + (alignment - 1), alignment);
602}
603
604/// Inline because TLS is not set up yet.
605inline fn alignBackward(addr: usize, alignment: usize) usize {
606 return addr & ~(alignment - 1);
607}
608
609/// Inline because TLS is not set up yet.
610inline fn alignPtrCast(comptime T: type, ptr: [*]u8) *T {
611 return @ptrCast(@alignCast(ptr));
612}
613
614/// Initializes all the fields of the static TLS area and returns the computed architecture-specific
615/// value of the TP register.
616pub fn prepareArea(area: []u8) usize {
617 @setRuntimeSafety(false);
618 @disableInstrumentation();
619
620 // Clear the area we're going to use, just to be safe.
621 @memset(area, 0);
622
623 // Prepare the ABI TCB.
624 const abi_tcb = alignPtrCast(AbiTcb, area.ptr + area_desc.abi_tcb.offset);
625 switch (current_variant) {
626 .I_original, .I_modified => abi_tcb.dtv = @intFromPtr(area.ptr + area_desc.dtv.offset),
627 .II => abi_tcb.self = abi_tcb,
628 }
629
630 // Prepare the DTV.
631 const dtv = alignPtrCast(Dtv, area.ptr + area_desc.dtv.offset);
632 dtv.len = 1;
633 dtv.tls_block = area.ptr + current_dtv_offset + area_desc.block.offset;
634
635 // Copy the initial data.
636 @memcpy(area[area_desc.block.offset..][0..area_desc.block.init.len], area_desc.block.init);
637
638 // Return the corrected value (if needed) for the TP register. Overflow here is not a problem;
639 // the pointer arithmetic involving the TP is done with wrapping semantics.
640 return @intFromPtr(area.ptr) +% switch (current_variant) {
641 .I_original, .II => area_desc.abi_tcb.offset,
642 .I_modified => area_desc.block.offset +% current_tp_offset,
643 };
644}
645
646/// The main motivation for the size chosen here is to be larger than total
647/// amount of thread-local variables for most programs. Putting this allocation
648/// in the ELF like this is equivalent to moving the `mmap` call below into the
649/// kernel, avoiding syscall overhead.
650var main_thread_area_buffer: [0x1000]u8 align(page_size_min) = undefined;
651
652/// Computes the layout of the static TLS area, allocates the area, initializes all of its fields,
653/// and assigns the architecture-specific value to the TP register.
654pub fn initStatic(phdrs: []elf.ElfN.Phdr) void {
655 @setRuntimeSafety(false);
656 @disableInstrumentation();
657
658 computeAreaDesc(phdrs);
659
660 const area = blk: {
661 // Fast path for the common case where the TLS data is really small, avoid an allocation and
662 // use our local buffer.
663 if (area_desc.alignment <= page_size_min and area_desc.size <= main_thread_area_buffer.len) {
664 break :blk main_thread_area_buffer[0..area_desc.size];
665 }
666
667 const begin_addr = mmap_tls(area_desc.size + area_desc.alignment - 1);
668 if (@call(.always_inline, linux.errno, .{begin_addr}) != .SUCCESS) @trap();
669
670 const area_ptr: [*]align(page_size_min) u8 = @ptrFromInt(begin_addr);
671
672 // Make sure the slice is correctly aligned.
673 const begin_aligned_addr = alignForward(begin_addr, area_desc.alignment);
674 const start = begin_aligned_addr - begin_addr;
675 break :blk area_ptr[start..][0..area_desc.size];
676 };
677
678 const tp_value = prepareArea(area);
679 setThreadPointer(tp_value);
680}
681
682inline fn mmap_tls(length: usize) usize {
683 const prot: linux.PROT = .{ .READ = true, .WRITE = true };
684 const flags: linux.MAP = .{ .TYPE = .PRIVATE, .ANONYMOUS = true };
685
686 if (@hasField(linux.SYS, "mmap2")) {
687 return @call(.always_inline, linux.syscall6, .{
688 .mmap2,
689 0,
690 length,
691 @as(u32, @bitCast(prot)),
692 @as(u32, @bitCast(flags)),
693 @as(usize, @bitCast(@as(isize, -1))),
694 0,
695 });
696 } else {
697 // The s390x mmap() syscall existed before Linux supported syscalls with 5+ parameters, so
698 // it takes a single pointer to an array of arguments instead.
699 return if (native_arch == .s390x) @call(.always_inline, linux.syscall1, .{
700 .mmap,
701 @intFromPtr(&[_]usize{
702 0,
703 length,
704 @as(u32, @bitCast(prot)),
705 @as(u32, @bitCast(flags)),
706 @as(usize, @bitCast(@as(isize, -1))),
707 0,
708 }),
709 }) else @call(.always_inline, linux.syscall6, .{
710 .mmap,
711 0,
712 length,
713 @as(u32, @bitCast(prot)),
714 @as(u32, @bitCast(flags)),
715 @as(usize, @bitCast(@as(isize, -1))),
716 0,
717 });
718 }
719}
720
721comptime {
722 assert(!builtin.link_libc); // otherwise libc should control TLS
723
724 if (builtin.output_mode == .Exe and builtin.link_mode == .static) {
725 // This is a static executable without libc, so it is our job to provide the TLS accessor
726 // function for the GD and LD models. This function is unlikely to actually be used, since
727 // the linker should be able to relax every TLS access to the LE model and therefore
728 // eliminate all calls to this function, but that isn't guaranteed.
729 const Fns = struct {
730 const TlsIndex = switch (native_arch) {
731 .x86_64 => extern struct { module: u64, offset: u64 }, // Even for x32...
732 else => extern struct { module: usize, offset: usize }, // ...but not MIPS N32!
733 };
734 fn __tls_get_addr(ti: *const TlsIndex) callconv(.c) *anyopaque {
735 comptime assert(native_arch != .s390x);
736
737 assert(ti.module == 1); // The executable's module ID is always 1
738 const tp = getThreadPointer();
739 const block: [*]u8 = switch (current_variant) {
740 .I_original => @ptrFromInt(tp -% area_desc.abi_tcb.offset +% area_desc.block.offset),
741 .I_modified => @ptrFromInt(tp -% current_tp_offset),
742 // The `.I_original` approach would also work for `.II`, but there is an
743 // alternative strategy which is one less operation:
744 .II => @ptrFromInt(tp -% area_desc.block.size),
745 };
746 return block[@intCast(ti.offset)..];
747 }
748 fn __tls_get_offset() callconv(.naked) noreturn {
749 comptime assert(native_arch == .s390x);
750
751 // We receive the module's GOT pointer in r12 and the GOT offset in r2.
752 asm volatile (
753 \\ la %%r1, 0(%%r12, %%r2)
754 \\ lg %%r2, 8(%%r1)
755 \\ lgrl %%r0, %[block_size]
756 \\ sgr %%r2, %%r0
757 \\ br %%r14
758 :
759 : [block_size] "s" (&area_desc.block.size),
760 );
761 }
762 };
763
764 if (native_arch == .s390x)
765 @export(&Fns.__tls_get_offset, .{ .name = "__tls_get_offset" })
766 else
767 @export(&Fns.__tls_get_addr, .{ .name = "__tls_get_addr" });
768 }
769}