1/* Assembly macros for 64-bit PowerPC.
2 Copyright (C) 2002-2026 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19#include <sysdeps/powerpc/sysdep.h>
20#include <tls.h>
21
22#ifdef __ASSEMBLER__
23
24/* Stack frame offsets. */
25#define FRAME_BACKCHAIN 0
26#define FRAME_CR_SAVE 8
27#define FRAME_LR_SAVE 16
28#if _CALL_ELF != 2
29#define FRAME_MIN_SIZE 112
30#define FRAME_MIN_SIZE_PARM 112
31#define FRAME_TOC_SAVE 40
32#define FRAME_PARM_SAVE 48
33#else
34#define FRAME_ROP_SAVE -8
35#define FRAME_MIN_SIZE 48 /* Includes space for the ROP save slot */
36#define FRAME_MIN_SIZE_PARM 112 /* Includes space for the ROP save slot */
37#define FRAME_TOC_SAVE 24
38#define FRAME_PARM_SAVE 32
39#endif
40
41/* Support macros for CALL_MCOUNT. */
42 .macro SAVE_ARG NARG
43 .if \NARG
44 SAVE_ARG \NARG-1
45 std 2+\NARG,-FRAME_MIN_SIZE_PARM+FRAME_PARM_SAVE-8+8*(\NARG)(1)
46 .endif
47 .endm
48
49 .macro REST_ARG NARG
50 .if \NARG
51 REST_ARG \NARG-1
52 ld 2+\NARG,FRAME_PARM_SAVE-8+8*(\NARG)(1)
53 .endif
54 .endm
55
56 .macro CFI_SAVE_ARG NARG
57 .if \NARG
58 CFI_SAVE_ARG \NARG-1
59 cfi_offset(2+\NARG,-FRAME_MIN_SIZE_PARM+FRAME_PARM_SAVE-8+8*(\NARG))
60 .endif
61 .endm
62
63 .macro CFI_REST_ARG NARG
64 .if \NARG
65 CFI_REST_ARG \NARG-1
66 cfi_restore(2+\NARG)
67 .endif
68 .endm
69
70/* If compiled for profiling, call `_mcount' at the start of each function.
71 see ppc-mcount.S for more details. */
72 .macro CALL_MCOUNT NARG
73#ifdef PROF
74 mflr r0
75 SAVE_ARG \NARG
76 std r0,FRAME_LR_SAVE(r1)
77 stdu r1,-FRAME_MIN_SIZE_PARM(r1)
78 cfi_adjust_cfa_offset(FRAME_MIN_SIZE_PARM)
79 cfi_offset(lr,FRAME_LR_SAVE)
80 CFI_SAVE_ARG \NARG
81 bl JUMPTARGET (_mcount)
82#ifndef SHARED
83 nop
84#endif
85 ld r0,FRAME_MIN_SIZE_PARM+FRAME_LR_SAVE(r1)
86 REST_ARG \NARG
87 mtlr r0
88 addi r1,r1,FRAME_MIN_SIZE_PARM
89 cfi_adjust_cfa_offset(-FRAME_MIN_SIZE_PARM)
90 cfi_restore(lr)
91 CFI_REST_ARG \NARG
92#endif
93 .endm
94
95#if _CALL_ELF != 2
96
97/* Macro to prepare for calling via a function pointer. */
98 .macro PPC64_LOAD_FUNCPTR PTR
99 ld r12,0(\PTR)
100 ld r2,8(\PTR)
101 mtctr r12
102 ld r11,16(\PTR)
103 .endm
104
105#ifdef USE_PPC64_OVERLAPPING_OPD
106# define OPD_ENT(name) .quad BODY_LABEL (name), .TOC.@tocbase
107#else
108# define OPD_ENT(name) .quad BODY_LABEL (name), .TOC.@tocbase, 0
109#endif
110
111#define ENTRY_1(name) \
112 .type BODY_LABEL(name),@function; \
113 .globl name; \
114 .section ".opd","aw"; \
115 .p2align 3;FUNC_LABEL(name): \
116 OPD_ENT (name); \
117 .previous
118
119#define FUNC_LABEL(X) X
120#define BODY_LABEL(X) .LY##X
121#define ENTRY_2(name) \
122 .type name,@function; \
123 ENTRY_1(name)
124#define END_2(name) \
125 .size name,.-BODY_LABEL(name); \
126 .size BODY_LABEL(name),.-BODY_LABEL(name)
127#define LOCALENTRY(name)
128
129#else /* _CALL_ELF == 2 */
130
131/* Macro to prepare for calling via a function pointer. */
132 .macro PPC64_LOAD_FUNCPTR PTR
133 mr r12,\PTR
134 mtctr r12
135 .endm
136
137#define FUNC_LABEL(X) X
138#define BODY_LABEL(X) X
139#define ENTRY_2(name) \
140 .globl name; \
141 .type name,@function
142#define END_2(name) \
143 .size name,.-name
144#define LOCALENTRY(name) \
1451: addis r2,r12,.TOC.-1b@ha; \
146 addi r2,r2,.TOC.-1b@l; \
147 .localentry name,.-name
148
149#endif /* _CALL_ELF */
150
151 .macro NOPS NARG
152 .if \NARG
153 NOPS \NARG-1
154 nop
155 .endif
156 .endm
157
158 .macro ENTRY_3 name, alignp2=2, nopwords=0
159 .text
160 ENTRY_2(\name)
161 .p2align \alignp2
162 NOPS \nopwords
163BODY_LABEL(\name):
164 .endm
165
166/* Use ENTRY_TOCLESS for functions that make no use of r2 and
167 guarantee r2 is unchanged on exit. Any function that has @toc or
168 @got relocs uses r2. Functions that call other functions via the
169 PLT use r2. Use ENTRY for functions that may use or change r2.
170 The first argument is the function name.
171 The optional second argument specifies alignment of the function's
172 code, as the logarithm base two of the byte alignment. For
173 example, a value of four aligns to a sixteen byte boundary.
174 The optional third argument specifies the number of NOPs to emit
175 before the start of the function's code. */
176#ifndef PROF
177#define ENTRY_TOCLESS(name, ...) \
178 ENTRY_3 name, ## __VA_ARGS__; \
179 cfi_startproc
180
181#define ENTRY(name, ...) \
182 ENTRY_TOCLESS(name, ## __VA_ARGS__); \
183 LOCALENTRY(name)
184#else
185/* The call to _mcount is potentially via the plt, so profiling code
186 is never free of an r2 use. */
187#define ENTRY_TOCLESS(name, ...) \
188 ENTRY_3 name, ## __VA_ARGS__; \
189 cfi_startproc; \
190 LOCALENTRY(name)
191
192#define ENTRY(name, ...) \
193 ENTRY_TOCLESS(name, ## __VA_ARGS__)
194#endif
195
196/* Local labels stripped out by the linker. */
197#undef L
198#define L(x) .L##x
199
200#define tostring(s) #s
201#define stringify(s) tostring(s)
202#define XGLUE(a,b) a##b
203#define GLUE(a,b) XGLUE(a,b)
204#define LT_LABEL(name) GLUE(.LT,name)
205#define LT_LABELSUFFIX(name,suffix) GLUE(GLUE(.LT,name),suffix)
206
207/* Support Traceback tables */
208#define TB_ASM 0x000c000000000000
209#define TB_GLOBALLINK 0x0000800000000000
210#define TB_IS_EPROL 0x0000400000000000
211#define TB_HAS_TBOFF 0x0000200000000000
212#define TB_INT_PROC 0x0000100000000000
213#define TB_HAS_CTL 0x0000080000000000
214#define TB_TOCLESS 0x0000040000000000
215#define TB_FP_PRESENT 0x0000020000000000
216#define TB_LOG_ABORT 0x0000010000000000
217#define TB_INT_HANDL 0x0000008000000000
218#define TB_NAME_PRESENT 0x0000004000000000
219#define TB_USES_ALLOCA 0x0000002000000000
220#define TB_SAVES_CR 0x0000000200000000
221#define TB_SAVES_LR 0x0000000100000000
222#define TB_STORES_BC 0x0000000080000000
223#define TB_FIXUP 0x0000000040000000
224#define TB_FP_SAVED(fprs) (((fprs) & 0x3f) << 24)
225#define TB_GPR_SAVED(gprs) (((fprs) & 0x3f) << 16)
226#define TB_FIXEDPARMS(parms) (((parms) & 0xff) << 8)
227#define TB_FLOATPARMS(parms) (((parms) & 0x7f) << 1)
228#define TB_PARMSONSTK 0x0000000000000001
229
230#define PPC_HIGHER(v) (((v) >> 32) & 0xffff)
231#define TB_DEFAULT TB_ASM | TB_HAS_TBOFF | TB_NAME_PRESENT
232
233#define TRACEBACK(name) \
234LT_LABEL(name): ; \
235 .long 0 ; \
236 .quad TB_DEFAULT ; \
237 .long LT_LABEL(name)-BODY_LABEL(name) ; \
238 .short LT_LABELSUFFIX(name,_name_end)-LT_LABELSUFFIX(name,_name_start) ; \
239LT_LABELSUFFIX(name,_name_start): ;\
240 .ascii stringify(name) ; \
241LT_LABELSUFFIX(name,_name_end): ; \
242 .p2align 2
243
244#define TRACEBACK_MASK(name,mask) \
245LT_LABEL(name): ; \
246 .long 0 ; \
247 .quad TB_DEFAULT | mask ; \
248 .long LT_LABEL(name)-BODY_LABEL(name) ; \
249 .short LT_LABELSUFFIX(name,_name_end)-LT_LABELSUFFIX(name,_name_start) ; \
250LT_LABELSUFFIX(name,_name_start): ;\
251 .ascii stringify(name) ; \
252LT_LABELSUFFIX(name,_name_end): ; \
253 .p2align 2
254
255/* END generates Traceback tables */
256#undef END
257#define END(name) \
258 cfi_endproc; \
259 TRACEBACK(name); \
260 END_2(name)
261
262/* This form supports more informative traceback tables */
263#define END_GEN_TB(name,mask) \
264 cfi_endproc; \
265 TRACEBACK_MASK(name,mask); \
266 END_2(name)
267
268/* We will allocate a new frame to save LR and the non-volatile register used to
269 read the TCB when checking for scv support on syscall code. We actually just
270 need the minimum frame size plus room for 1 reg (8 bytes). But the ABI
271 mandates stack frames should be aligned at 16 Bytes, so we end up allocating
272 a bit more space then what will actually be used. */
273#define SCV_FRAME_SIZE (FRAME_MIN_SIZE+16)
274#define SCV_FRAME_NVOLREG_SAVE FRAME_MIN_SIZE
275
276/* Allocate frame and save register */
277#define NVOLREG_SAVE \
278 stdu r1,-SCV_FRAME_SIZE(r1); \
279 cfi_adjust_cfa_offset(SCV_FRAME_SIZE); \
280 std r31,SCV_FRAME_NVOLREG_SAVE(r1); \
281 cfi_rel_offset(r31,SCV_FRAME_NVOLREG_SAVE);
282
283/* Restore register and destroy frame */
284#define NVOLREG_RESTORE \
285 ld r31,SCV_FRAME_NVOLREG_SAVE(r1); \
286 cfi_restore(r31); \
287 addi r1,r1,SCV_FRAME_SIZE; \
288 cfi_adjust_cfa_offset(-SCV_FRAME_SIZE);
289
290/* Check PPC_FEATURE2_SCV bit from hwcap2 in the TCB. If it is not set, scv is
291 not available, then go to JUMPFALSE (label given by the macro's caller). We
292 save the value we read from the TCB in a non-volatile register so we can
293 reuse it later when exiting from the syscall in PSEUDO_RET. Note that for
294 the static case we need an extra check to guarantee the thread pointer has
295 already been initialized, otherwise we may try to access an invalid address
296 if a syscall is called before the TLS has been setup. */
297 .macro CHECK_SCV_SUPPORT REG JUMPFALSE
298
299#ifndef SHARED
300 /* Check if thread pointer has already been setup. */
301 cmpdi r13,0
302 beq \JUMPFALSE
303#endif
304
305 /* Read PPC_FEATURE2_SCV from TCB and store it in REG */
306 ld \REG,TCB_HWCAP(PT_THREAD_POINTER)
307 andis. \REG,\REG,PPC_FEATURE2_SCV>>16
308
309 beq \JUMPFALSE
310 .endm
311
312#if !defined(USE_PPC_SCV) || IS_IN(rtld)
313# define DO_CALL(syscall) \
314 li r0,syscall; \
315 DO_CALL_SC
316#else
317/* Before doing the syscall, check if we can use scv. scv is supported by P9
318 and later with Linux v5.9 and later. If so, use it. Otherwise, fallback to
319 sc. We use a non-volatile register to save hwcap2 from the TCB, so we need
320 to save its content beforehand. */
321# define DO_CALL(syscall) \
322 li r0,syscall; \
323 NVOLREG_SAVE; \
324 CHECK_SCV_SUPPORT r31 0f; \
325 DO_CALL_SCV; \
326 b 1f; \
3270: DO_CALL_SC; \
3281:
329#endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */
330
331/* DO_CALL_SC and DO_CALL_SCV expect the syscall number to be in r0. */
332#define DO_CALL_SC \
333 sc
334
335#define DO_CALL_SCV \
336 mflr r9; \
337 std r9,SCV_FRAME_SIZE+FRAME_LR_SAVE(r1); \
338 cfi_rel_offset(lr,SCV_FRAME_SIZE+FRAME_LR_SAVE); \
339 .machine "push"; \
340 .machine "power9"; \
341 scv 0; \
342 .machine "pop"; \
343 ld r9,SCV_FRAME_SIZE+FRAME_LR_SAVE(r1); \
344 mtlr r9; \
345 cfi_restore(lr);
346
347/* ppc64 is always PIC */
348#undef JUMPTARGET
349#define JUMPTARGET(name) FUNC_LABEL(name)
350
351#define PSEUDO(name, syscall_name, args) \
352 .section ".text"; \
353 ENTRY (name); \
354 DO_CALL (SYS_ify (syscall_name))
355
356#ifdef SHARED
357# define TAIL_CALL_NO_RETURN(__func) \
358 b JUMPTARGET (NOTOC (__func))
359#else
360# define TAIL_CALL_NO_RETURN(__func) \
361 .ifdef .Local ## __func; \
362 b .Local ## __func; \
363 .else; \
364.Local ## __func: \
365 mflr 0; \
366 std 0,FRAME_LR_SAVE(1); \
367 stdu 1,-FRAME_MIN_SIZE(1); \
368 cfi_adjust_cfa_offset(FRAME_MIN_SIZE); \
369 cfi_offset(lr,FRAME_LR_SAVE); \
370 bl JUMPTARGET(__func); \
371 nop; \
372 .endif
373#endif
374
375#ifdef SHARED
376#define TAIL_CALL_SYSCALL_ERROR \
377 b JUMPTARGET (NOTOC (__syscall_error))
378#else
379/* Static version might be linked into a large app with a toc exceeding
380 64k. We can't put a toc adjusting stub on a plain branch, so can't
381 tail call __syscall_error. */
382#define TAIL_CALL_SYSCALL_ERROR \
383 .ifdef .Local_syscall_error; \
384 b .Local_syscall_error; \
385 .else; \
386.Local_syscall_error: \
387 mflr 0; \
388 std 0,FRAME_LR_SAVE(1); \
389 stdu 1,-FRAME_MIN_SIZE(1); \
390 cfi_adjust_cfa_offset(FRAME_MIN_SIZE); \
391 cfi_offset(lr,FRAME_LR_SAVE); \
392 bl JUMPTARGET(__syscall_error); \
393 nop; \
394 ld 0,FRAME_MIN_SIZE+FRAME_LR_SAVE(1); \
395 addi 1,1,FRAME_MIN_SIZE; \
396 cfi_adjust_cfa_offset(-FRAME_MIN_SIZE); \
397 mtlr 0; \
398 cfi_restore(lr); \
399 blr; \
400 .endif
401#endif
402
403#if !defined(USE_PPC_SCV) || IS_IN(rtld)
404# define PSEUDO_RET \
405 RET_SC; \
406 TAIL_CALL_SYSCALL_ERROR
407#else
408/* This should only be called after a DO_CALL. In such cases, r31 contains the
409 value of PPC_FEATURE2_SCV read from hwcap2 by CHECK_SCV_SUPPORT. If it is
410 set, we know we have entered the kernel using scv, so handle the return code
411 accordingly. */
412# define PSEUDO_RET \
413 cmpdi cr5,r31,0; \
414 NVOLREG_RESTORE; \
415 beq cr5,0f; \
416 RET_SCV; \
417 b 1f; \
4180: RET_SC; \
4191: TAIL_CALL_SYSCALL_ERROR
420#endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */
421
422#define RET_SCV \
423 li r9,-4095; \
424 cmpld r3,r9; \
425 bltlr+; \
426 neg r3,r3;
427
428#define RET_SC \
429 bnslr+;
430
431#define ret PSEUDO_RET
432
433#undef PSEUDO_END
434#define PSEUDO_END(name) \
435 END (name)
436
437#define PSEUDO_NOERRNO(name, syscall_name, args) \
438 .section ".text"; \
439 ENTRY (name); \
440 DO_CALL (SYS_ify (syscall_name))
441
442#if !defined(USE_PPC_SCV) || IS_IN(rtld)
443# define PSEUDO_RET_NOERRNO \
444 blr
445#else
446/* This should only be called after a DO_CALL. */
447# define PSEUDO_RET_NOERRNO \
448 NVOLREG_RESTORE; \
449 blr
450#endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */
451
452#define ret_NOERRNO PSEUDO_RET_NOERRNO
453
454#undef PSEUDO_END_NOERRNO
455#define PSEUDO_END_NOERRNO(name) \
456 END (name)
457
458#define PSEUDO_ERRVAL(name, syscall_name, args) \
459 .section ".text"; \
460 ENTRY (name); \
461 DO_CALL (SYS_ify (syscall_name))
462
463#if !defined(USE_PPC_SCV) || IS_IN(rtld)
464# define PSEUDO_RET_ERRVAL \
465 blr
466#else
467/* This should only be called after a DO_CALL. */
468# define PSEUDO_RET_ERRVAL \
469 NVOLREG_RESTORE; \
470 blr
471#endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */
472
473#define ret_ERRVAL PSEUDO_RET_ERRVAL
474
475#undef PSEUDO_END_ERRVAL
476#define PSEUDO_END_ERRVAL(name) \
477 END (name)
478
479#ifdef SHARED
480# if IS_IN (rtld)
481 /* Inside ld.so we use the local alias to avoid runtime GOT
482 relocations. */
483# define __GLRO_DEF(var) \
484.LC__ ## var: \
485 .tc _rtld_local_ro[TC],_rtld_local_ro
486# else
487# define __GLRO_DEF(var) \
488.LC__ ## var: \
489 .tc _rtld_global_ro[TC],_rtld_global_ro
490# endif
491# define __GLRO(rOUT, var, offset) \
492 addis rOUT,r2,.LC__ ## var@toc@ha; \
493 ld rOUT,.LC__ ## var@toc@l(rOUT); \
494 lwz rOUT,offset(rOUT)
495#else
496# define __GLRO_DEF(var) \
497.LC__ ## var: \
498 .tc _ ## var[TC],_ ## var
499# define __GLRO(rOUT, var, offset) \
500 addis rOUT,r2,.LC__ ## var@toc@ha; \
501 ld rOUT,.LC__ ## var@toc@l(rOUT); \
502 lwz rOUT,0(rOUT)
503#endif
504
505#ifdef USE_PPC64_NOTOC
506# define NOTOC(l) l@notoc
507#else
508# define NOTOC(l) l
509#endif
510
511#else /* !__ASSEMBLER__ */
512
513#if _CALL_ELF != 2
514
515#define PPC64_LOAD_FUNCPTR(ptr) \
516 "ld 12,0(" #ptr ")\n" \
517 "ld 2,8(" #ptr ")\n" \
518 "mtctr 12\n" \
519 "ld 11,16(" #ptr ")"
520
521#ifdef USE_PPC64_OVERLAPPING_OPD
522# define OPD_ENT(name) ".quad " BODY_PREFIX #name ", .TOC.@tocbase"
523#else
524# define OPD_ENT(name) ".quad " BODY_PREFIX #name ", .TOC.@tocbase, 0"
525#endif
526
527#define ENTRY_1(name) \
528 ".type " BODY_PREFIX #name ",@function\n" \
529 ".globl " #name "\n" \
530 ".pushsection \".opd\",\"aw\"\n" \
531 ".p2align 3\n" \
532#name ":\n" \
533 OPD_ENT (name) "\n" \
534 ".popsection"
535
536#define DOT_PREFIX ""
537#define BODY_PREFIX ".LY"
538#define ENTRY_2(name) \
539 ".type " #name ",@function\n" \
540 ENTRY_1(name)
541#define END_2(name) \
542 ".size " #name ",.-" BODY_PREFIX #name "\n" \
543 ".size " BODY_PREFIX #name ",.-" BODY_PREFIX #name
544#define LOCALENTRY(name)
545
546#else /* _CALL_ELF */
547
548#define PPC64_LOAD_FUNCPTR(ptr) \
549 "mr 12," #ptr "\n" \
550 "mtctr 12"
551
552#define DOT_PREFIX ""
553#define BODY_PREFIX ""
554#define ENTRY_2(name) \
555 ".type " #name ",@function\n" \
556 ".globl " #name
557#define END_2(name) \
558 ".size " #name ",.-" #name
559#define LOCALENTRY(name) \
560 "1: addis 2,12,.TOC.-1b@ha\n" \
561 "addi 2,2,.TOC.-1b@l\n" \
562 ".localentry " #name ",.-" #name
563
564#endif /* _CALL_ELF */
565
566#endif /* __ASSEMBLER__ */