1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1993 The Regents of the University of California.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of the University nor the names of its contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 *
31 * $FreeBSD: releng/12.0/sys/i386/include/cpufunc.h 334890 2018-06-09 18:31:19Z markj $
32 */
33
34 /*
35 * Functions to provide access to special i386 instructions.
36 * This in included in sys/systm.h, and that file should be
37 * used in preference to this.
38 */
39
40 #ifndef _MACHINE_CPUFUNC_H_
41 #define _MACHINE_CPUFUNC_H_
42
43 #ifndef _SYS_CDEFS_H_
44 #error this file needs sys/cdefs.h as a prerequisite
45 #endif
46
47 struct region_descriptor;
48
49 #define readb(va) (*(volatile uint8_t *) (va))
50 #define readw(va) (*(volatile uint16_t *) (va))
51 #define readl(va) (*(volatile uint32_t *) (va))
52
53 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d))
54 #define writew(va, d) (*(volatile uint16_t *) (va) = (d))
55 #define writel(va, d) (*(volatile uint32_t *) (va) = (d))
56
57 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE)
58
59 static __inline void
60 breakpoint(void)
61 {
62 __asm __volatile("int $3");
63 }
64
65 static __inline __pure2 u_int
66 bsfl(u_int mask)
67 {
68 u_int result;
69
70 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
71 return (result);
72 }
73
74 static __inline __pure2 u_int
75 bsrl(u_int mask)
76 {
77 u_int result;
78
79 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
80 return (result);
81 }
82
83 static __inline void
84 clflush(u_long addr)
85 {
86
87 __asm __volatile("clflush %0" : : "m" (*(char *)addr));
88 }
89
90 static __inline void
91 clflushopt(u_long addr)
92 {
93
94 __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr));
95 }
96
97 static __inline void
98 clts(void)
99 {
100
101 __asm __volatile("clts");
102 }
103
104 static __inline void
105 disable_intr(void)
106 {
107
108 __asm __volatile("cli" : : : "memory");
109 }
110
111 static __inline void
112 do_cpuid(u_int ax, u_int *p)
113 {
114 __asm __volatile("cpuid"
115 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
116 : "" (ax));
117 }
118
119 static __inline void
120 cpuid_count(u_int ax, u_int cx, u_int *p)
121 {
122 __asm __volatile("cpuid"
123 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
124 : "" (ax), "c" (cx));
125 }
126
127 static __inline void
128 enable_intr(void)
129 {
130
131 __asm __volatile("sti");
132 }
133
134 static __inline void
135 cpu_monitor(const void *addr, u_long extensions, u_int hints)
136 {
137
138 __asm __volatile("monitor"
139 : : "a" (addr), "c" (extensions), "d" (hints));
140 }
141
142 static __inline void
143 cpu_mwait(u_long extensions, u_int hints)
144 {
145
146 __asm __volatile("mwait" : : "a" (hints), "c" (extensions));
147 }
148
149 static __inline void
150 lfence(void)
151 {
152
153 __asm __volatile("lfence" : : : "memory");
154 }
155
156 static __inline void
157 mfence(void)
158 {
159
160 __asm __volatile("mfence" : : : "memory");
161 }
162
163 static __inline void
164 sfence(void)
165 {
166
167 __asm __volatile("sfence" : : : "memory");
168 }
169
170 #ifdef _KERNEL
171
172 #define HAVE_INLINE_FFS
173
174 static __inline __pure2 int
175 ffs(int mask)
176 {
177 /*
178 * Note that gcc-2's builtin ffs would be used if we didn't declare
179 * this inline or turn off the builtin. The builtin is faster but
180 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later
181 * versions.
182 */
183 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1);
184 }
185
186 #define HAVE_INLINE_FFSL
187
188 static __inline __pure2 int
189 ffsl(long mask)
190 {
191 return (ffs((int)mask));
192 }
193
194 #define HAVE_INLINE_FLS
195
196 static __inline __pure2 int
197 fls(int mask)
198 {
199 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1);
200 }
201
202 #define HAVE_INLINE_FLSL
203
204 static __inline __pure2 int
205 flsl(long mask)
206 {
207 return (fls((int)mask));
208 }
209
210 #endif /* _KERNEL */
211
212 static __inline void
213 halt(void)
214 {
215 __asm __volatile("hlt");
216 }
217
218 static __inline u_char
219 inb(u_int port)
220 {
221 u_char data;
222
223 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
224 return (data);
225 }
226
227 static __inline u_int
228 inl(u_int port)
229 {
230 u_int data;
231
232 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
233 return (data);
234 }
235
236 static __inline void
237 insb(u_int port, void *addr, size_t count)
238 {
239 __asm __volatile("cld; rep; insb"
240 : "+D" (addr), "+c" (count)
241 : "d" (port)
242 : "memory");
243 }
244
245 static __inline void
246 insw(u_int port, void *addr, size_t count)
247 {
248 __asm __volatile("cld; rep; insw"
249 : "+D" (addr), "+c" (count)
250 : "d" (port)
251 : "memory");
252 }
253
254 static __inline void
255 insl(u_int port, void *addr, size_t count)
256 {
257 __asm __volatile("cld; rep; insl"
258 : "+D" (addr), "+c" (count)
259 : "d" (port)
260 : "memory");
261 }
262
263 static __inline void
264 invd(void)
265 {
266 __asm __volatile("invd");
267 }
268
269 static __inline u_short
270 inw(u_int port)
271 {
272 u_short data;
273
274 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
275 return (data);
276 }
277
278 static __inline void
279 outb(u_int port, u_char data)
280 {
281 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
282 }
283
284 static __inline void
285 outl(u_int port, u_int data)
286 {
287 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
288 }
289
290 static __inline void
291 outsb(u_int port, const void *addr, size_t count)
292 {
293 __asm __volatile("cld; rep; outsb"
294 : "+S" (addr), "+c" (count)
295 : "d" (port));
296 }
297
298 static __inline void
299 outsw(u_int port, const void *addr, size_t count)
300 {
301 __asm __volatile("cld; rep; outsw"
302 : "+S" (addr), "+c" (count)
303 : "d" (port));
304 }
305
306 static __inline void
307 outsl(u_int port, const void *addr, size_t count)
308 {
309 __asm __volatile("cld; rep; outsl"
310 : "+S" (addr), "+c" (count)
311 : "d" (port));
312 }
313
314 static __inline void
315 outw(u_int port, u_short data)
316 {
317 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
318 }
319
320 static __inline void
321 ia32_pause(void)
322 {
323 __asm __volatile("pause");
324 }
325
326 static __inline u_int
327 read_eflags(void)
328 {
329 u_int ef;
330
331 __asm __volatile("pushfl; popl %0" : "=r" (ef));
332 return (ef);
333 }
334
335 static __inline uint64_t
336 rdmsr(u_int msr)
337 {
338 uint64_t rv;
339
340 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr));
341 return (rv);
342 }
343
344 static __inline uint32_t
345 rdmsr32(u_int msr)
346 {
347 uint32_t low;
348
349 __asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx");
350 return (low);
351 }
352
353 static __inline uint64_t
354 rdpmc(u_int pmc)
355 {
356 uint64_t rv;
357
358 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc));
359 return (rv);
360 }
361
362 static __inline uint64_t
363 rdtsc(void)
364 {
365 uint64_t rv;
366
367 __asm __volatile("rdtsc" : "=A" (rv));
368 return (rv);
369 }
370
371 static __inline uint64_t
372 rdtscp(void)
373 {
374 uint64_t rv;
375
376 __asm __volatile("rdtscp" : "=A" (rv) : : "ecx");
377 return (rv);
378 }
379
380 static __inline uint32_t
381 rdtsc32(void)
382 {
383 uint32_t rv;
384
385 __asm __volatile("rdtsc" : "=a" (rv) : : "edx");
386 return (rv);
387 }
388
389 static __inline void
390 wbinvd(void)
391 {
392 __asm __volatile("wbinvd");
393 }
394
395 static __inline void
396 write_eflags(u_int ef)
397 {
398 __asm __volatile("pushl %0; popfl" : : "r" (ef));
399 }
400
401 static __inline void
402 wrmsr(u_int msr, uint64_t newval)
403 {
404 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr));
405 }
406
407 static __inline void
408 load_cr0(u_int data)
409 {
410
411 __asm __volatile("movl %0,%%cr0" : : "r" (data));
412 }
413
414 static __inline u_int
415 rcr0(void)
416 {
417 u_int data;
418
419 __asm __volatile("movl %%cr0,%0" : "=r" (data));
420 return (data);
421 }
422
423 static __inline u_int
424 rcr2(void)
425 {
426 u_int data;
427
428 __asm __volatile("movl %%cr2,%0" : "=r" (data));
429 return (data);
430 }
431
432 static __inline void
433 load_cr3(u_int data)
434 {
435
436 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory");
437 }
438
439 static __inline u_int
440 rcr3(void)
441 {
442 u_int data;
443
444 __asm __volatile("movl %%cr3,%0" : "=r" (data));
445 return (data);
446 }
447
448 static __inline void
449 load_cr4(u_int data)
450 {
451 __asm __volatile("movl %0,%%cr4" : : "r" (data));
452 }
453
454 static __inline u_int
455 rcr4(void)
456 {
457 u_int data;
458
459 __asm __volatile("movl %%cr4,%0" : "=r" (data));
460 return (data);
461 }
462
463 static __inline uint64_t
464 rxcr(u_int reg)
465 {
466 u_int low, high;
467
468 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg));
469 return (low | ((uint64_t)high << 32));
470 }
471
472 static __inline void
473 load_xcr(u_int reg, uint64_t val)
474 {
475 u_int low, high;
476
477 low = val;
478 high = val >> 32;
479 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high));
480 }
481
482 /*
483 * Global TLB flush (except for thise for pages marked PG_G)
484 */
485 static __inline void
486 invltlb(void)
487 {
488
489 load_cr3(rcr3());
490 }
491
492 /*
493 * TLB flush for an individual page (even if it has PG_G).
494 * Only works on 486+ CPUs (i386 does not have PG_G).
495 */
496 static __inline void
497 invlpg(u_int addr)
498 {
499
500 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
501 }
502
503 static __inline u_short
504 rfs(void)
505 {
506 u_short sel;
507 __asm __volatile("movw %%fs,%0" : "=rm" (sel));
508 return (sel);
509 }
510
511 static __inline uint64_t
512 rgdt(void)
513 {
514 uint64_t gdtr;
515 __asm __volatile("sgdt %0" : "=m" (gdtr));
516 return (gdtr);
517 }
518
519 static __inline u_short
520 rgs(void)
521 {
522 u_short sel;
523 __asm __volatile("movw %%gs,%0" : "=rm" (sel));
524 return (sel);
525 }
526
527 static __inline uint64_t
528 ridt(void)
529 {
530 uint64_t idtr;
531 __asm __volatile("sidt %0" : "=m" (idtr));
532 return (idtr);
533 }
534
535 static __inline u_short
536 rldt(void)
537 {
538 u_short ldtr;
539 __asm __volatile("sldt %0" : "=g" (ldtr));
540 return (ldtr);
541 }
542
543 static __inline u_short
544 rss(void)
545 {
546 u_short sel;
547 __asm __volatile("movw %%ss,%0" : "=rm" (sel));
548 return (sel);
549 }
550
551 static __inline u_short
552 rtr(void)
553 {
554 u_short tr;
555 __asm __volatile("str %0" : "=g" (tr));
556 return (tr);
557 }
558
559 static __inline void
560 load_fs(u_short sel)
561 {
562 __asm __volatile("movw %0,%%fs" : : "rm" (sel));
563 }
564
565 static __inline void
566 load_gs(u_short sel)
567 {
568 __asm __volatile("movw %0,%%gs" : : "rm" (sel));
569 }
570
571 static __inline void
572 lidt(struct region_descriptor *addr)
573 {
574 __asm __volatile("lidt (%0)" : : "r" (addr));
575 }
576
577 static __inline void
578 lldt(u_short sel)
579 {
580 __asm __volatile("lldt %0" : : "r" (sel));
581 }
582
583 static __inline void
584 ltr(u_short sel)
585 {
586 __asm __volatile("ltr %0" : : "r" (sel));
587 }
588
589 static __inline u_int
590 rdr0(void)
591 {
592 u_int data;
593 __asm __volatile("movl %%dr0,%0" : "=r" (data));
594 return (data);
595 }
596
597 static __inline void
598 load_dr0(u_int dr0)
599 {
600 __asm __volatile("movl %0,%%dr0" : : "r" (dr0));
601 }
602
603 static __inline u_int
604 rdr1(void)
605 {
606 u_int data;
607 __asm __volatile("movl %%dr1,%0" : "=r" (data));
608 return (data);
609 }
610
611 static __inline void
612 load_dr1(u_int dr1)
613 {
614 __asm __volatile("movl %0,%%dr1" : : "r" (dr1));
615 }
616
617 static __inline u_int
618 rdr2(void)
619 {
620 u_int data;
621 __asm __volatile("movl %%dr2,%0" : "=r" (data));
622 return (data);
623 }
624
625 static __inline void
626 load_dr2(u_int dr2)
627 {
628 __asm __volatile("movl %0,%%dr2" : : "r" (dr2));
629 }
630
631 static __inline u_int
632 rdr3(void)
633 {
634 u_int data;
635 __asm __volatile("movl %%dr3,%0" : "=r" (data));
636 return (data);
637 }
638
639 static __inline void
640 load_dr3(u_int dr3)
641 {
642 __asm __volatile("movl %0,%%dr3" : : "r" (dr3));
643 }
644
645 static __inline u_int
646 rdr6(void)
647 {
648 u_int data;
649 __asm __volatile("movl %%dr6,%0" : "=r" (data));
650 return (data);
651 }
652
653 static __inline void
654 load_dr6(u_int dr6)
655 {
656 __asm __volatile("movl %0,%%dr6" : : "r" (dr6));
657 }
658
659 static __inline u_int
660 rdr7(void)
661 {
662 u_int data;
663 __asm __volatile("movl %%dr7,%0" : "=r" (data));
664 return (data);
665 }
666
667 static __inline void
668 load_dr7(u_int dr7)
669 {
670 __asm __volatile("movl %0,%%dr7" : : "r" (dr7));
671 }
672
673 static __inline u_char
674 read_cyrix_reg(u_char reg)
675 {
676 outb(0x22, reg);
677 return inb(0x23);
678 }
679
680 static __inline void
681 write_cyrix_reg(u_char reg, u_char data)
682 {
683 outb(0x22, reg);
684 outb(0x23, data);
685 }
686
687 static __inline register_t
688 intr_disable(void)
689 {
690 register_t eflags;
691
692 eflags = read_eflags();
693 disable_intr();
694 return (eflags);
695 }
696
697 static __inline void
698 intr_restore(register_t eflags)
699 {
700 write_eflags(eflags);
701 }
702
703 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */
704
705 int breakpoint(void);
706 u_int bsfl(u_int mask);
707 u_int bsrl(u_int mask);
708 void clflush(u_long addr);
709 void clts(void);
710 void cpuid_count(u_int ax, u_int cx, u_int *p);
711 void disable_intr(void);
712 void do_cpuid(u_int ax, u_int *p);
713 void enable_intr(void);
714 void halt(void);
715 void ia32_pause(void);
716 u_char inb(u_int port);
717 u_int inl(u_int port);
718 void insb(u_int port, void *addr, size_t count);
719 void insl(u_int port, void *addr, size_t count);
720 void insw(u_int port, void *addr, size_t count);
721 register_t intr_disable(void);
722 void intr_restore(register_t ef);
723 void invd(void);
724 void invlpg(u_int addr);
725 void invltlb(void);
726 u_short inw(u_int port);
727 void lidt(struct region_descriptor *addr);
728 void lldt(u_short sel);
729 void load_cr0(u_int cr0);
730 void load_cr3(u_int cr3);
731 void load_cr4(u_int cr4);
732 void load_dr0(u_int dr0);
733 void load_dr1(u_int dr1);
734 void load_dr2(u_int dr2);
735 void load_dr3(u_int dr3);
736 void load_dr6(u_int dr6);
737 void load_dr7(u_int dr7);
738 void load_fs(u_short sel);
739 void load_gs(u_short sel);
740 void ltr(u_short sel);
741 void outb(u_int port, u_char data);
742 void outl(u_int port, u_int data);
743 void outsb(u_int port, const void *addr, size_t count);
744 void outsl(u_int port, const void *addr, size_t count);
745 void outsw(u_int port, const void *addr, size_t count);
746 void outw(u_int port, u_short data);
747 u_int rcr0(void);
748 u_int rcr2(void);
749 u_int rcr3(void);
750 u_int rcr4(void);
751 uint64_t rdmsr(u_int msr);
752 uint64_t rdpmc(u_int pmc);
753 u_int rdr0(void);
754 u_int rdr1(void);
755 u_int rdr2(void);
756 u_int rdr3(void);
757 u_int rdr6(void);
758 u_int rdr7(void);
759 uint64_t rdtsc(void);
760 u_char read_cyrix_reg(u_char reg);
761 u_int read_eflags(void);
762 u_int rfs(void);
763 uint64_t rgdt(void);
764 u_int rgs(void);
765 uint64_t ridt(void);
766 u_short rldt(void);
767 u_short rtr(void);
768 void wbinvd(void);
769 void write_cyrix_reg(u_char reg, u_char data);
770 void write_eflags(u_int ef);
771 void wrmsr(u_int msr, uint64_t newval);
772
773 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */
774
775 void reset_dbregs(void);
776
777 #ifdef _KERNEL
778 int rdmsr_safe(u_int msr, uint64_t *val);
779 int wrmsr_safe(u_int msr, uint64_t newval);
780 #endif
781
782 #endif /* !_MACHINE_CPUFUNC_H_ */
Cache object: 00f20ed378afe575c62cd400ee3d860b
|