1/* $OpenBSD: cpufunc.h,v 1.47 2025/12/30 03:19:15 jsg Exp $ */
2/* $NetBSD: cpufunc.h,v 1.3 2003/05/08 10:27:43 fvdl Exp $ */
3
4/*-
5 * Copyright (c) 1998 The NetBSD Foundation, Inc.
6 * All rights reserved.
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by Charles M. Hannum.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33#ifndef _MACHINE_CPUFUNC_H_
34#define _MACHINE_CPUFUNC_H_
35
36/*
37 * Functions to provide access to i386-specific instructions.
38 */
39
40#include <sys/types.h>
41
42#include <machine/specialreg.h>
43
44#if defined(_KERNEL) && !defined (_STANDALONE)
45
46static __inline void
47invlpg(u_int64_t addr)
48{
49 __asm volatile("invlpg (%0)" : : "r" (addr) : "memory");
50}
51
52static __inline void
53sidt(void *p)
54{
55 __asm volatile("sidt (%0)" : : "r" (p) : "memory");
56}
57
58static __inline void
59lidt(void *p)
60{
61 __asm volatile("lidt (%0)" : : "r" (p) : "memory");
62}
63
64static __inline void
65sgdt(void *p)
66{
67 __asm volatile("sgdt (%0)" : : "r" (p) : "memory");
68}
69
70static __inline void
71bare_lgdt(struct region_descriptor *p)
72{
73 __asm volatile("lgdt (%0)" : : "r" (p) : "memory");
74}
75
76static __inline void
77sldt(u_short *sel)
78{
79 __asm volatile("sldt (%0)" : : "r" (sel) : "memory");
80}
81
82static __inline void
83lldt(u_short sel)
84{
85 __asm volatile("lldt %0" : : "r" (sel));
86}
87
88static __inline void
89ltr(u_short sel)
90{
91 __asm volatile("ltr %0" : : "r" (sel));
92}
93
94static __inline void
95lcr8(u_int val)
96{
97 u_int64_t val64 = val;
98 __asm volatile("movq %0,%%cr8" : : "r" (val64));
99}
100
101/*
102 * Upper 32 bits are reserved anyway, so just keep this 32bits.
103 */
104static __inline void
105lcr0(u_int val)
106{
107 u_int64_t val64 = val;
108 __asm volatile("movq %0,%%cr0" : : "r" (val64));
109}
110
111static __inline u_int
112rcr0(void)
113{
114 u_int64_t val64;
115 u_int val;
116 __asm volatile("movq %%cr0,%0" : "=r" (val64));
117 val = val64;
118 return val;
119}
120
121static __inline u_int64_t
122rcr2(void)
123{
124 u_int64_t val;
125 __asm volatile("movq %%cr2,%0" : "=r" (val));
126 return val;
127}
128
129static __inline void
130lcr3(u_int64_t val)
131{
132 __asm volatile("movq %0,%%cr3" : : "r" (val));
133}
134
135static __inline u_int64_t
136rcr3(void)
137{
138 u_int64_t val;
139 __asm volatile("movq %%cr3,%0" : "=r" (val));
140 return val;
141}
142
143/*
144 * Same as for cr0. Don't touch upper 32 bits.
145 */
146static __inline void
147lcr4(u_int val)
148{
149 u_int64_t val64 = val;
150
151 __asm volatile("movq %0,%%cr4" : : "r" (val64));
152}
153
154static __inline u_int
155rcr4(void)
156{
157 u_int64_t val64;
158 __asm volatile("movq %%cr4,%0" : "=r" (val64));
159 return (u_int) val64;
160}
161
162/*
163 * DR6 and DR7 debug registers
164 */
165static inline uint64_t
166rdr6(void)
167{
168 u_int64_t val;
169 __asm volatile("movq %%dr6,%0" : "=r" (val));
170 return val;
171}
172
173static inline uint64_t
174rdr7(void)
175{
176 u_int64_t val;
177 __asm volatile("movq %%dr7,%0" : "=r" (val));
178 return val;
179}
180
181static __inline void
182tlbflush(void)
183{
184 u_int64_t val;
185 __asm volatile("movq %%cr3,%0" : "=r" (val));
186 __asm volatile("movq %0,%%cr3" : : "r" (val));
187}
188
189static inline void
190invpcid(uint64_t type, paddr_t pcid, paddr_t addr)
191{
192 uint64_t desc[2] = { pcid, addr };
193 asm volatile("invpcid %0,%1" : : "m"(desc[0]), "r"(type));
194}
195#define INVPCID_ADDR 0
196#define INVPCID_PCID 1
197#define INVPCID_ALL 2
198#define INVPCID_NON_GLOBAL 3
199
200#ifdef notyet
201void setidt(int idx, /*XXX*/caddr_t func, int typ, int dpl);
202#endif
203
204
205/* XXXX ought to be in psl.h with spl() functions */
206
207static __inline u_long
208read_rflags(void)
209{
210 u_long ef;
211
212 __asm volatile("pushfq; popq %0" : "=r" (ef));
213 return (ef);
214}
215
216static __inline void
217write_rflags(u_long ef)
218{
219 __asm volatile("pushq %0; popfq" : : "r" (ef));
220}
221
222static __inline void
223intr_enable(void)
224{
225 __asm volatile("sti");
226}
227
228static __inline u_long
229intr_disable(void)
230{
231 u_long ef;
232
233 ef = read_rflags();
234 __asm volatile("cli");
235 return (ef);
236}
237
238static __inline void
239intr_restore(u_long ef)
240{
241 write_rflags(ef);
242}
243
244static __inline u_int64_t
245rdmsr(u_int msr)
246{
247 uint32_t hi, lo;
248 __asm volatile("rdmsr" : "=d" (hi), "=a" (lo) : "c" (msr));
249 return (((uint64_t)hi << 32) | (uint64_t) lo);
250}
251
252static __inline uint32_t
253rdpkru(u_int ecx)
254{
255 uint32_t edx, pkru;
256 asm volatile("rdpkru " : "=a" (pkru), "=d" (edx) : "c" (ecx));
257 return pkru;
258}
259
260static __inline void
261wrpkru(u_int ecx, uint32_t pkru)
262{
263 uint32_t edx = 0;
264 asm volatile("wrpkru" : : "a" (pkru), "c" (ecx), "d" (edx));
265}
266
267static __inline void
268wrmsr(u_int msr, u_int64_t newval)
269{
270 __asm volatile("wrmsr" :
271 : "a" (newval & 0xffffffff), "d" (newval >> 32), "c" (msr));
272}
273
274/*
275 * Some of the undocumented AMD64 MSRs need a 'passcode' to access.
276 *
277 * See LinuxBIOSv2: src/cpu/amd/model_fxx/model_fxx_init.c
278 */
279
280#define OPTERON_MSR_PASSCODE 0x9c5a203a
281
282static __inline u_int64_t
283rdmsr_locked(u_int msr, u_int code)
284{
285 uint32_t hi, lo;
286 __asm volatile("rdmsr"
287 : "=d" (hi), "=a" (lo)
288 : "c" (msr), "D" (code));
289 return (((uint64_t)hi << 32) | (uint64_t) lo);
290}
291
292static __inline void
293wrmsr_locked(u_int msr, u_int code, u_int64_t newval)
294{
295 __asm volatile("wrmsr" :
296 : "a" (newval & 0xffffffff), "d" (newval >> 32), "c" (msr), "D" (code));
297}
298
299static __inline void
300wbinvd(void)
301{
302 __asm volatile("wbinvd" : : : "memory");
303}
304
305#ifdef MULTIPROCESSOR
306int wbinvd_on_all_cpus(void);
307#else
308static inline int
309wbinvd_on_all_cpus(void)
310{
311 wbinvd();
312 return 0;
313}
314
315static inline int
316wbinvd_on_all_cpus_acked(void)
317{
318 wbinvd();
319 return 0;
320}
321#endif /* MULTIPROCESSOR */
322
323static __inline void
324clflush(u_int64_t addr)
325{
326 __asm volatile("clflush %0" : "+m" (*(volatile char *)addr));
327}
328
329static __inline void
330mfence(void)
331{
332 __asm volatile("mfence" : : : "memory");
333}
334
335static __inline u_int64_t
336rdtsc(void)
337{
338 uint32_t hi, lo;
339
340 __asm volatile("rdtsc" : "=d" (hi), "=a" (lo));
341 return (((uint64_t)hi << 32) | (uint64_t) lo);
342}
343
344static __inline u_int64_t
345rdtscp(void)
346{
347 uint32_t hi, lo;
348
349 __asm volatile("rdtscp" : "=d" (hi), "=a" (lo) : : "ecx");
350 return (((uint64_t)hi << 32) | (uint64_t) lo);
351}
352
353static __inline u_int64_t
354rdtsc_lfence(void)
355{
356 uint32_t hi, lo;
357
358 __asm volatile("lfence; rdtsc" : "=d" (hi), "=a" (lo));
359 return (((uint64_t)hi << 32) | (uint64_t) lo);
360}
361
362static __inline u_int64_t
363rdpmc(u_int pmc)
364{
365 uint32_t hi, lo;
366
367 __asm volatile("rdpmc" : "=d" (hi), "=a" (lo) : "c" (pmc));
368 return (((uint64_t)hi << 32) | (uint64_t) lo);
369}
370
371static __inline void
372monitor(const volatile void *addr, u_long extensions, u_int hints)
373{
374
375 __asm volatile("monitor"
376 : : "a" (addr), "c" (extensions), "d" (hints));
377}
378
379static __inline void
380mwait(u_long extensions, u_int hints)
381{
382
383 __asm volatile(
384 " mwait ;"
385 " mov $8,%%rcx ;"
386 " .align 16,0x90 ;"
387 "3: call 5f ;"
388 "4: pause ;"
389 " lfence ;"
390 " call 4b ;"
391 " .align 16,0xcc ;"
392 "5: call 7f ;"
393 "6: pause ;"
394 " lfence ;"
395 " call 6b ;"
396 " .align 16,0xcc ;"
397 "7: loop 3b ;"
398 " add $(16*8),%%rsp"
399 : "+c" (extensions) : "a" (hints));
400}
401
402static __inline void
403xsetbv(uint32_t reg, uint64_t mask)
404{
405 uint32_t lo, hi;
406
407 lo = mask;
408 hi = mask >> 32;
409 __asm volatile("xsetbv" :: "c" (reg), "a" (lo), "d" (hi) : "memory");
410}
411
412static __inline uint64_t
413xgetbv(uint32_t reg)
414{
415 uint32_t lo, hi;
416
417 __asm volatile("xgetbv" : "=a" (lo), "=d" (hi) : "c" (reg));
418
419 return (((uint64_t)hi << 32) | (uint64_t)lo);
420}
421
422static __inline void
423stgi(void)
424{
425 __asm volatile("stgi");
426}
427
428static __inline void
429clgi(void)
430{
431 __asm volatile("clgi");
432}
433
434/* Break into DDB. */
435static __inline void
436breakpoint(void)
437{
438 __asm volatile("int $3");
439}
440
441/* VMGEXIT */
442static __inline void
443vmgexit(void)
444{
445 /* rep; vmmcall encodes the vmgexit instruction */
446 __asm volatile("rep; vmmcall");
447}
448
449void amd64_errata(struct cpu_info *);
450void cpu_ucode_setup(void);
451void cpu_ucode_apply(struct cpu_info *);
452
453struct cpu_info_full;
454void cpu_enter_pages(struct cpu_info_full *);
455
456int rdmsr_safe(u_int msr, uint64_t *);
457
458#endif /* _KERNEL */
459
460#endif /* !_MACHINE_CPUFUNC_H_ */