1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1993 The Regents of the University of California.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of the University nor the names of its contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 *
31 * $FreeBSD$
32 */
33
34 /*
35 * Functions to provide access to special i386 instructions.
36 * This in included in sys/systm.h, and that file should be
37 * used in preference to this.
38 */
39
40 #ifndef _MACHINE_CPUFUNC_H_
41 #define _MACHINE_CPUFUNC_H_
42
43 #ifndef _SYS_CDEFS_H_
44 #error this file needs sys/cdefs.h as a prerequisite
45 #endif
46
47 struct region_descriptor;
48
49 #define readb(va) (*(volatile uint8_t *) (va))
50 #define readw(va) (*(volatile uint16_t *) (va))
51 #define readl(va) (*(volatile uint32_t *) (va))
52
53 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d))
54 #define writew(va, d) (*(volatile uint16_t *) (va) = (d))
55 #define writel(va, d) (*(volatile uint32_t *) (va) = (d))
56
57 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE)
58
59 static __inline void
60 breakpoint(void)
61 {
62 __asm __volatile("int $3");
63 }
64
65 static __inline __pure2 u_int
66 bsfl(u_int mask)
67 {
68 u_int result;
69
70 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
71 return (result);
72 }
73
74 static __inline __pure2 u_int
75 bsrl(u_int mask)
76 {
77 u_int result;
78
79 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
80 return (result);
81 }
82
83 static __inline void
84 clflush(u_long addr)
85 {
86
87 __asm __volatile("clflush %0" : : "m" (*(char *)addr));
88 }
89
90 static __inline void
91 clflushopt(u_long addr)
92 {
93
94 __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr));
95 }
96
97 static __inline void
98 clts(void)
99 {
100
101 __asm __volatile("clts");
102 }
103
104 static __inline void
105 disable_intr(void)
106 {
107
108 __asm __volatile("cli" : : : "memory");
109 }
110
111 #ifdef _KERNEL
112 static __inline void
113 do_cpuid(u_int ax, u_int *p)
114 {
115 __asm __volatile("cpuid"
116 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
117 : "" (ax));
118 }
119
120 static __inline void
121 cpuid_count(u_int ax, u_int cx, u_int *p)
122 {
123 __asm __volatile("cpuid"
124 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
125 : "" (ax), "c" (cx));
126 }
127 #else
128 static __inline void
129 do_cpuid(u_int ax, u_int *p)
130 {
131 __asm __volatile(
132 "pushl\t%%ebx\n\t"
133 "cpuid\n\t"
134 "movl\t%%ebx,%1\n\t"
135 "popl\t%%ebx"
136 : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3])
137 : "" (ax));
138 }
139
140 static __inline void
141 cpuid_count(u_int ax, u_int cx, u_int *p)
142 {
143 __asm __volatile(
144 "pushl\t%%ebx\n\t"
145 "cpuid\n\t"
146 "movl\t%%ebx,%1\n\t"
147 "popl\t%%ebx"
148 : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3])
149 : "" (ax), "c" (cx));
150 }
151 #endif
152
153 static __inline void
154 enable_intr(void)
155 {
156
157 __asm __volatile("sti");
158 }
159
160 static __inline void
161 cpu_monitor(const void *addr, u_long extensions, u_int hints)
162 {
163
164 __asm __volatile("monitor"
165 : : "a" (addr), "c" (extensions), "d" (hints));
166 }
167
168 static __inline void
169 cpu_mwait(u_long extensions, u_int hints)
170 {
171
172 __asm __volatile("mwait" : : "a" (hints), "c" (extensions));
173 }
174
175 static __inline void
176 lfence(void)
177 {
178
179 __asm __volatile("lfence" : : : "memory");
180 }
181
182 static __inline void
183 mfence(void)
184 {
185
186 __asm __volatile("mfence" : : : "memory");
187 }
188
189 static __inline void
190 sfence(void)
191 {
192
193 __asm __volatile("sfence" : : : "memory");
194 }
195
196 #ifdef _KERNEL
197
198 #define HAVE_INLINE_FFS
199
200 static __inline __pure2 int
201 ffs(int mask)
202 {
203 /*
204 * Note that gcc-2's builtin ffs would be used if we didn't declare
205 * this inline or turn off the builtin. The builtin is faster but
206 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later
207 * versions.
208 */
209 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1);
210 }
211
212 #define HAVE_INLINE_FFSL
213
214 static __inline __pure2 int
215 ffsl(long mask)
216 {
217 return (ffs((int)mask));
218 }
219
220 #define HAVE_INLINE_FLS
221
222 static __inline __pure2 int
223 fls(int mask)
224 {
225 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1);
226 }
227
228 #define HAVE_INLINE_FLSL
229
230 static __inline __pure2 int
231 flsl(long mask)
232 {
233 return (fls((int)mask));
234 }
235
236 #endif /* _KERNEL */
237
238 static __inline void
239 halt(void)
240 {
241 __asm __volatile("hlt");
242 }
243
244 static __inline u_char
245 inb(u_int port)
246 {
247 u_char data;
248
249 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
250 return (data);
251 }
252
253 static __inline u_int
254 inl(u_int port)
255 {
256 u_int data;
257
258 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
259 return (data);
260 }
261
262 static __inline void
263 insb(u_int port, void *addr, size_t count)
264 {
265 __asm __volatile("cld; rep; insb"
266 : "+D" (addr), "+c" (count)
267 : "d" (port)
268 : "memory");
269 }
270
271 static __inline void
272 insw(u_int port, void *addr, size_t count)
273 {
274 __asm __volatile("cld; rep; insw"
275 : "+D" (addr), "+c" (count)
276 : "d" (port)
277 : "memory");
278 }
279
280 static __inline void
281 insl(u_int port, void *addr, size_t count)
282 {
283 __asm __volatile("cld; rep; insl"
284 : "+D" (addr), "+c" (count)
285 : "d" (port)
286 : "memory");
287 }
288
289 static __inline void
290 invd(void)
291 {
292 __asm __volatile("invd");
293 }
294
295 static __inline u_short
296 inw(u_int port)
297 {
298 u_short data;
299
300 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
301 return (data);
302 }
303
304 static __inline void
305 outb(u_int port, u_char data)
306 {
307 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
308 }
309
310 static __inline void
311 outl(u_int port, u_int data)
312 {
313 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
314 }
315
316 static __inline void
317 outsb(u_int port, const void *addr, size_t count)
318 {
319 __asm __volatile("cld; rep; outsb"
320 : "+S" (addr), "+c" (count)
321 : "d" (port));
322 }
323
324 static __inline void
325 outsw(u_int port, const void *addr, size_t count)
326 {
327 __asm __volatile("cld; rep; outsw"
328 : "+S" (addr), "+c" (count)
329 : "d" (port));
330 }
331
332 static __inline void
333 outsl(u_int port, const void *addr, size_t count)
334 {
335 __asm __volatile("cld; rep; outsl"
336 : "+S" (addr), "+c" (count)
337 : "d" (port));
338 }
339
340 static __inline void
341 outw(u_int port, u_short data)
342 {
343 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
344 }
345
346 static __inline void
347 ia32_pause(void)
348 {
349 __asm __volatile("pause");
350 }
351
352 static __inline u_int
353 read_eflags(void)
354 {
355 u_int ef;
356
357 __asm __volatile("pushfl; popl %0" : "=r" (ef));
358 return (ef);
359 }
360
361 static __inline uint64_t
362 rdmsr(u_int msr)
363 {
364 uint64_t rv;
365
366 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr));
367 return (rv);
368 }
369
370 static __inline uint32_t
371 rdmsr32(u_int msr)
372 {
373 uint32_t low;
374
375 __asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx");
376 return (low);
377 }
378
379 static __inline uint64_t
380 rdpmc(u_int pmc)
381 {
382 uint64_t rv;
383
384 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc));
385 return (rv);
386 }
387
388 static __inline uint64_t
389 rdtsc(void)
390 {
391 uint64_t rv;
392
393 __asm __volatile("rdtsc" : "=A" (rv));
394 return (rv);
395 }
396
397 static __inline uint64_t
398 rdtsc_ordered_lfence(void)
399 {
400 lfence();
401 return (rdtsc());
402 }
403
404 static __inline uint64_t
405 rdtsc_ordered_mfence(void)
406 {
407 mfence();
408 return (rdtsc());
409 }
410
411 static __inline uint64_t
412 rdtscp(void)
413 {
414 uint64_t rv;
415
416 __asm __volatile("rdtscp" : "=A" (rv) : : "ecx");
417 return (rv);
418 }
419
420 static __inline uint64_t
421 rdtscp_aux(uint32_t *aux)
422 {
423 uint64_t rv;
424
425 __asm __volatile("rdtscp" : "=A" (rv), "=c" (*aux));
426 return (rv);
427 }
428
429 static __inline uint32_t
430 rdtsc32(void)
431 {
432 uint32_t rv;
433
434 __asm __volatile("rdtsc" : "=a" (rv) : : "edx");
435 return (rv);
436 }
437
438 static __inline uint32_t
439 rdtscp32(void)
440 {
441 uint32_t rv;
442
443 __asm __volatile("rdtscp" : "=a" (rv) : : "ecx", "edx");
444 return (rv);
445 }
446
447 static __inline void
448 wbinvd(void)
449 {
450 __asm __volatile("wbinvd");
451 }
452
453 static __inline void
454 write_eflags(u_int ef)
455 {
456 __asm __volatile("pushl %0; popfl" : : "r" (ef));
457 }
458
459 static __inline void
460 wrmsr(u_int msr, uint64_t newval)
461 {
462 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr));
463 }
464
465 static __inline void
466 load_cr0(u_int data)
467 {
468
469 __asm __volatile("movl %0,%%cr0" : : "r" (data));
470 }
471
472 static __inline u_int
473 rcr0(void)
474 {
475 u_int data;
476
477 __asm __volatile("movl %%cr0,%0" : "=r" (data));
478 return (data);
479 }
480
481 static __inline u_int
482 rcr2(void)
483 {
484 u_int data;
485
486 __asm __volatile("movl %%cr2,%0" : "=r" (data));
487 return (data);
488 }
489
490 static __inline void
491 load_cr3(u_int data)
492 {
493
494 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory");
495 }
496
497 static __inline u_int
498 rcr3(void)
499 {
500 u_int data;
501
502 __asm __volatile("movl %%cr3,%0" : "=r" (data));
503 return (data);
504 }
505
506 static __inline void
507 load_cr4(u_int data)
508 {
509 __asm __volatile("movl %0,%%cr4" : : "r" (data));
510 }
511
512 static __inline u_int
513 rcr4(void)
514 {
515 u_int data;
516
517 __asm __volatile("movl %%cr4,%0" : "=r" (data));
518 return (data);
519 }
520
521 static __inline uint64_t
522 rxcr(u_int reg)
523 {
524 u_int low, high;
525
526 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg));
527 return (low | ((uint64_t)high << 32));
528 }
529
530 static __inline void
531 load_xcr(u_int reg, uint64_t val)
532 {
533 u_int low, high;
534
535 low = val;
536 high = val >> 32;
537 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high));
538 }
539
540 /*
541 * Global TLB flush (except for thise for pages marked PG_G)
542 */
543 static __inline void
544 invltlb(void)
545 {
546
547 load_cr3(rcr3());
548 }
549
550 /*
551 * TLB flush for an individual page (even if it has PG_G).
552 * Only works on 486+ CPUs (i386 does not have PG_G).
553 */
554 static __inline void
555 invlpg(u_int addr)
556 {
557
558 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
559 }
560
561 static __inline u_short
562 rfs(void)
563 {
564 u_short sel;
565 __asm __volatile("movw %%fs,%0" : "=rm" (sel));
566 return (sel);
567 }
568
569 static __inline uint64_t
570 rgdt(void)
571 {
572 uint64_t gdtr;
573 __asm __volatile("sgdt %0" : "=m" (gdtr));
574 return (gdtr);
575 }
576
577 static __inline u_short
578 rgs(void)
579 {
580 u_short sel;
581 __asm __volatile("movw %%gs,%0" : "=rm" (sel));
582 return (sel);
583 }
584
585 static __inline uint64_t
586 ridt(void)
587 {
588 uint64_t idtr;
589 __asm __volatile("sidt %0" : "=m" (idtr));
590 return (idtr);
591 }
592
593 static __inline u_short
594 rldt(void)
595 {
596 u_short ldtr;
597 __asm __volatile("sldt %0" : "=g" (ldtr));
598 return (ldtr);
599 }
600
601 static __inline u_short
602 rss(void)
603 {
604 u_short sel;
605 __asm __volatile("movw %%ss,%0" : "=rm" (sel));
606 return (sel);
607 }
608
609 static __inline u_short
610 rtr(void)
611 {
612 u_short tr;
613 __asm __volatile("str %0" : "=g" (tr));
614 return (tr);
615 }
616
617 static __inline void
618 load_fs(u_short sel)
619 {
620 __asm __volatile("movw %0,%%fs" : : "rm" (sel));
621 }
622
623 static __inline void
624 load_gs(u_short sel)
625 {
626 __asm __volatile("movw %0,%%gs" : : "rm" (sel));
627 }
628
629 static __inline void
630 lidt(struct region_descriptor *addr)
631 {
632 __asm __volatile("lidt (%0)" : : "r" (addr));
633 }
634
635 static __inline void
636 lldt(u_short sel)
637 {
638 __asm __volatile("lldt %0" : : "r" (sel));
639 }
640
641 static __inline void
642 ltr(u_short sel)
643 {
644 __asm __volatile("ltr %0" : : "r" (sel));
645 }
646
647 static __inline u_int
648 rdr0(void)
649 {
650 u_int data;
651 __asm __volatile("movl %%dr0,%0" : "=r" (data));
652 return (data);
653 }
654
655 static __inline void
656 load_dr0(u_int dr0)
657 {
658 __asm __volatile("movl %0,%%dr0" : : "r" (dr0));
659 }
660
661 static __inline u_int
662 rdr1(void)
663 {
664 u_int data;
665 __asm __volatile("movl %%dr1,%0" : "=r" (data));
666 return (data);
667 }
668
669 static __inline void
670 load_dr1(u_int dr1)
671 {
672 __asm __volatile("movl %0,%%dr1" : : "r" (dr1));
673 }
674
675 static __inline u_int
676 rdr2(void)
677 {
678 u_int data;
679 __asm __volatile("movl %%dr2,%0" : "=r" (data));
680 return (data);
681 }
682
683 static __inline void
684 load_dr2(u_int dr2)
685 {
686 __asm __volatile("movl %0,%%dr2" : : "r" (dr2));
687 }
688
689 static __inline u_int
690 rdr3(void)
691 {
692 u_int data;
693 __asm __volatile("movl %%dr3,%0" : "=r" (data));
694 return (data);
695 }
696
697 static __inline void
698 load_dr3(u_int dr3)
699 {
700 __asm __volatile("movl %0,%%dr3" : : "r" (dr3));
701 }
702
703 static __inline u_int
704 rdr6(void)
705 {
706 u_int data;
707 __asm __volatile("movl %%dr6,%0" : "=r" (data));
708 return (data);
709 }
710
711 static __inline void
712 load_dr6(u_int dr6)
713 {
714 __asm __volatile("movl %0,%%dr6" : : "r" (dr6));
715 }
716
717 static __inline u_int
718 rdr7(void)
719 {
720 u_int data;
721 __asm __volatile("movl %%dr7,%0" : "=r" (data));
722 return (data);
723 }
724
725 static __inline void
726 load_dr7(u_int dr7)
727 {
728 __asm __volatile("movl %0,%%dr7" : : "r" (dr7));
729 }
730
731 static __inline u_char
732 read_cyrix_reg(u_char reg)
733 {
734 outb(0x22, reg);
735 return inb(0x23);
736 }
737
738 static __inline void
739 write_cyrix_reg(u_char reg, u_char data)
740 {
741 outb(0x22, reg);
742 outb(0x23, data);
743 }
744
745 static __inline register_t
746 intr_disable(void)
747 {
748 register_t eflags;
749
750 eflags = read_eflags();
751 disable_intr();
752 return (eflags);
753 }
754
755 static __inline void
756 intr_restore(register_t eflags)
757 {
758 write_eflags(eflags);
759 }
760
761 static __inline uint32_t
762 rdpkru(void)
763 {
764 uint32_t res;
765
766 __asm __volatile("rdpkru" : "=a" (res) : "c" (0) : "edx");
767 return (res);
768 }
769
770 static __inline void
771 wrpkru(uint32_t mask)
772 {
773
774 __asm __volatile("wrpkru" : : "a" (mask), "c" (0), "d" (0));
775 }
776
777 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */
778
779 int breakpoint(void);
780 u_int bsfl(u_int mask);
781 u_int bsrl(u_int mask);
782 void clflush(u_long addr);
783 void clts(void);
784 void cpuid_count(u_int ax, u_int cx, u_int *p);
785 void disable_intr(void);
786 void do_cpuid(u_int ax, u_int *p);
787 void enable_intr(void);
788 void halt(void);
789 void ia32_pause(void);
790 u_char inb(u_int port);
791 u_int inl(u_int port);
792 void insb(u_int port, void *addr, size_t count);
793 void insl(u_int port, void *addr, size_t count);
794 void insw(u_int port, void *addr, size_t count);
795 register_t intr_disable(void);
796 void intr_restore(register_t ef);
797 void invd(void);
798 void invlpg(u_int addr);
799 void invltlb(void);
800 u_short inw(u_int port);
801 void lidt(struct region_descriptor *addr);
802 void lldt(u_short sel);
803 void load_cr0(u_int cr0);
804 void load_cr3(u_int cr3);
805 void load_cr4(u_int cr4);
806 void load_dr0(u_int dr0);
807 void load_dr1(u_int dr1);
808 void load_dr2(u_int dr2);
809 void load_dr3(u_int dr3);
810 void load_dr6(u_int dr6);
811 void load_dr7(u_int dr7);
812 void load_fs(u_short sel);
813 void load_gs(u_short sel);
814 void ltr(u_short sel);
815 void outb(u_int port, u_char data);
816 void outl(u_int port, u_int data);
817 void outsb(u_int port, const void *addr, size_t count);
818 void outsl(u_int port, const void *addr, size_t count);
819 void outsw(u_int port, const void *addr, size_t count);
820 void outw(u_int port, u_short data);
821 u_int rcr0(void);
822 u_int rcr2(void);
823 u_int rcr3(void);
824 u_int rcr4(void);
825 uint64_t rdmsr(u_int msr);
826 uint64_t rdpmc(u_int pmc);
827 u_int rdr0(void);
828 u_int rdr1(void);
829 u_int rdr2(void);
830 u_int rdr3(void);
831 u_int rdr6(void);
832 u_int rdr7(void);
833 uint64_t rdtsc(void);
834 u_char read_cyrix_reg(u_char reg);
835 u_int read_eflags(void);
836 u_int rfs(void);
837 uint64_t rgdt(void);
838 u_int rgs(void);
839 uint64_t ridt(void);
840 u_short rldt(void);
841 u_short rtr(void);
842 void wbinvd(void);
843 void write_cyrix_reg(u_char reg, u_char data);
844 void write_eflags(u_int ef);
845 void wrmsr(u_int msr, uint64_t newval);
846
847 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */
848
849 void reset_dbregs(void);
850
851 #ifdef _KERNEL
852 int rdmsr_safe(u_int msr, uint64_t *val);
853 int wrmsr_safe(u_int msr, uint64_t newval);
854 #endif
855
856 #endif /* !_MACHINE_CPUFUNC_H_ */
Cache object: 70a1d240187697be398ff4a54f219e55
|