1 /*-
2 * Copyright (c) 1993 The Regents of the University of California.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 4. Neither the name of the University nor the names of its contributors
14 * may be used to endorse or promote products derived from this software
15 * without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * $FreeBSD$
30 */
31
32 /*
33 * Functions to provide access to special i386 instructions.
34 * This in included in sys/systm.h, and that file should be
35 * used in preference to this.
36 */
37
38 #ifndef _MACHINE_CPUFUNC_H_
39 #define _MACHINE_CPUFUNC_H_
40
41 #ifndef _SYS_CDEFS_H_
42 #error this file needs sys/cdefs.h as a prerequisite
43 #endif
44
45 #ifdef XEN
46 extern void xen_cli(void);
47 extern void xen_sti(void);
48 extern u_int xen_rcr2(void);
49 extern void xen_load_cr3(u_int data);
50 extern void xen_tlb_flush(void);
51 extern void xen_invlpg(u_int addr);
52 extern void write_eflags(u_int eflags);
53 extern u_int read_eflags(void);
54 #endif
55
56 struct region_descriptor;
57
58 #define readb(va) (*(volatile uint8_t *) (va))
59 #define readw(va) (*(volatile uint16_t *) (va))
60 #define readl(va) (*(volatile uint32_t *) (va))
61
62 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d))
63 #define writew(va, d) (*(volatile uint16_t *) (va) = (d))
64 #define writel(va, d) (*(volatile uint32_t *) (va) = (d))
65
66 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE)
67
68 static __inline void
69 breakpoint(void)
70 {
71 __asm __volatile("int $3");
72 }
73
74 static __inline u_int
75 bsfl(u_int mask)
76 {
77 u_int result;
78
79 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
80 return (result);
81 }
82
83 static __inline u_int
84 bsrl(u_int mask)
85 {
86 u_int result;
87
88 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
89 return (result);
90 }
91
92 static __inline void
93 clflush(u_long addr)
94 {
95
96 __asm __volatile("clflush %0" : : "m" (*(char *)addr));
97 }
98
99 static __inline void
100 clflushopt(u_long addr)
101 {
102
103 __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr));
104 }
105
106 static __inline void
107 clts(void)
108 {
109
110 __asm __volatile("clts");
111 }
112
113 static __inline void
114 disable_intr(void)
115 {
116 #ifdef XEN
117 xen_cli();
118 #else
119 __asm __volatile("cli" : : : "memory");
120 #endif
121 }
122
123 static __inline void
124 do_cpuid(u_int ax, u_int *p)
125 {
126 __asm __volatile("cpuid"
127 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
128 : "" (ax));
129 }
130
131 static __inline void
132 cpuid_count(u_int ax, u_int cx, u_int *p)
133 {
134 __asm __volatile("cpuid"
135 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
136 : "" (ax), "c" (cx));
137 }
138
139 static __inline void
140 enable_intr(void)
141 {
142 #ifdef XEN
143 xen_sti();
144 #else
145 __asm __volatile("sti");
146 #endif
147 }
148
149 static __inline void
150 cpu_monitor(const void *addr, u_long extensions, u_int hints)
151 {
152
153 __asm __volatile("monitor"
154 : : "a" (addr), "c" (extensions), "d" (hints));
155 }
156
157 static __inline void
158 cpu_mwait(u_long extensions, u_int hints)
159 {
160
161 __asm __volatile("mwait" : : "a" (hints), "c" (extensions));
162 }
163
164 static __inline void
165 lfence(void)
166 {
167
168 __asm __volatile("lfence" : : : "memory");
169 }
170
171 static __inline void
172 mfence(void)
173 {
174
175 __asm __volatile("mfence" : : : "memory");
176 }
177
178 static __inline void
179 sfence(void)
180 {
181
182 __asm __volatile("sfence" : : : "memory");
183 }
184
185 #ifdef _KERNEL
186
187 #define HAVE_INLINE_FFS
188
189 static __inline int
190 ffs(int mask)
191 {
192 /*
193 * Note that gcc-2's builtin ffs would be used if we didn't declare
194 * this inline or turn off the builtin. The builtin is faster but
195 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later
196 * versions.
197 */
198 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1);
199 }
200
201 #define HAVE_INLINE_FFSL
202
203 static __inline int
204 ffsl(long mask)
205 {
206 return (ffs((int)mask));
207 }
208
209 #define HAVE_INLINE_FLS
210
211 static __inline int
212 fls(int mask)
213 {
214 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1);
215 }
216
217 #define HAVE_INLINE_FLSL
218
219 static __inline int
220 flsl(long mask)
221 {
222 return (fls((int)mask));
223 }
224
225 #endif /* _KERNEL */
226
227 static __inline void
228 halt(void)
229 {
230 __asm __volatile("hlt");
231 }
232
233 static __inline u_char
234 inb(u_int port)
235 {
236 u_char data;
237
238 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
239 return (data);
240 }
241
242 static __inline u_int
243 inl(u_int port)
244 {
245 u_int data;
246
247 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
248 return (data);
249 }
250
251 static __inline void
252 insb(u_int port, void *addr, size_t count)
253 {
254 __asm __volatile("cld; rep; insb"
255 : "+D" (addr), "+c" (count)
256 : "d" (port)
257 : "memory");
258 }
259
260 static __inline void
261 insw(u_int port, void *addr, size_t count)
262 {
263 __asm __volatile("cld; rep; insw"
264 : "+D" (addr), "+c" (count)
265 : "d" (port)
266 : "memory");
267 }
268
269 static __inline void
270 insl(u_int port, void *addr, size_t count)
271 {
272 __asm __volatile("cld; rep; insl"
273 : "+D" (addr), "+c" (count)
274 : "d" (port)
275 : "memory");
276 }
277
278 static __inline void
279 invd(void)
280 {
281 __asm __volatile("invd");
282 }
283
284 static __inline u_short
285 inw(u_int port)
286 {
287 u_short data;
288
289 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
290 return (data);
291 }
292
293 static __inline void
294 outb(u_int port, u_char data)
295 {
296 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
297 }
298
299 static __inline void
300 outl(u_int port, u_int data)
301 {
302 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
303 }
304
305 static __inline void
306 outsb(u_int port, const void *addr, size_t count)
307 {
308 __asm __volatile("cld; rep; outsb"
309 : "+S" (addr), "+c" (count)
310 : "d" (port));
311 }
312
313 static __inline void
314 outsw(u_int port, const void *addr, size_t count)
315 {
316 __asm __volatile("cld; rep; outsw"
317 : "+S" (addr), "+c" (count)
318 : "d" (port));
319 }
320
321 static __inline void
322 outsl(u_int port, const void *addr, size_t count)
323 {
324 __asm __volatile("cld; rep; outsl"
325 : "+S" (addr), "+c" (count)
326 : "d" (port));
327 }
328
329 static __inline void
330 outw(u_int port, u_short data)
331 {
332 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
333 }
334
335 static __inline void
336 ia32_pause(void)
337 {
338 __asm __volatile("pause");
339 }
340
341 static __inline u_int
342 #ifdef XEN
343 _read_eflags(void)
344 #else
345 read_eflags(void)
346 #endif
347 {
348 u_int ef;
349
350 __asm __volatile("pushfl; popl %0" : "=r" (ef));
351 return (ef);
352 }
353
354 static __inline uint64_t
355 rdmsr(u_int msr)
356 {
357 uint64_t rv;
358
359 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr));
360 return (rv);
361 }
362
363 static __inline uint64_t
364 rdpmc(u_int pmc)
365 {
366 uint64_t rv;
367
368 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc));
369 return (rv);
370 }
371
372 static __inline uint64_t
373 rdtsc(void)
374 {
375 uint64_t rv;
376
377 __asm __volatile("rdtsc" : "=A" (rv));
378 return (rv);
379 }
380
381 static __inline uint32_t
382 rdtsc32(void)
383 {
384 uint32_t rv;
385
386 __asm __volatile("rdtsc" : "=a" (rv) : : "edx");
387 return (rv);
388 }
389
390 static __inline void
391 wbinvd(void)
392 {
393 __asm __volatile("wbinvd");
394 }
395
396 static __inline void
397 #ifdef XEN
398 _write_eflags(u_int ef)
399 #else
400 write_eflags(u_int ef)
401 #endif
402 {
403 __asm __volatile("pushl %0; popfl" : : "r" (ef));
404 }
405
406 static __inline void
407 wrmsr(u_int msr, uint64_t newval)
408 {
409 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr));
410 }
411
412 static __inline void
413 load_cr0(u_int data)
414 {
415
416 __asm __volatile("movl %0,%%cr0" : : "r" (data));
417 }
418
419 static __inline u_int
420 rcr0(void)
421 {
422 u_int data;
423
424 __asm __volatile("movl %%cr0,%0" : "=r" (data));
425 return (data);
426 }
427
428 static __inline u_int
429 rcr2(void)
430 {
431 u_int data;
432
433 #ifdef XEN
434 return (xen_rcr2());
435 #endif
436 __asm __volatile("movl %%cr2,%0" : "=r" (data));
437 return (data);
438 }
439
440 static __inline void
441 load_cr3(u_int data)
442 {
443 #ifdef XEN
444 xen_load_cr3(data);
445 #else
446 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory");
447 #endif
448 }
449
450 static __inline u_int
451 rcr3(void)
452 {
453 u_int data;
454
455 __asm __volatile("movl %%cr3,%0" : "=r" (data));
456 return (data);
457 }
458
459 static __inline void
460 load_cr4(u_int data)
461 {
462 __asm __volatile("movl %0,%%cr4" : : "r" (data));
463 }
464
465 static __inline u_int
466 rcr4(void)
467 {
468 u_int data;
469
470 __asm __volatile("movl %%cr4,%0" : "=r" (data));
471 return (data);
472 }
473
474 static __inline uint64_t
475 rxcr(u_int reg)
476 {
477 u_int low, high;
478
479 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg));
480 return (low | ((uint64_t)high << 32));
481 }
482
483 static __inline void
484 load_xcr(u_int reg, uint64_t val)
485 {
486 u_int low, high;
487
488 low = val;
489 high = val >> 32;
490 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high));
491 }
492
493 /*
494 * Global TLB flush (except for thise for pages marked PG_G)
495 */
496 static __inline void
497 invltlb(void)
498 {
499 #ifdef XEN
500 xen_tlb_flush();
501 #else
502 load_cr3(rcr3());
503 #endif
504 }
505
506 /*
507 * TLB flush for an individual page (even if it has PG_G).
508 * Only works on 486+ CPUs (i386 does not have PG_G).
509 */
510 static __inline void
511 invlpg(u_int addr)
512 {
513
514 #ifdef XEN
515 xen_invlpg(addr);
516 #else
517 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
518 #endif
519 }
520
521 static __inline u_short
522 rfs(void)
523 {
524 u_short sel;
525 __asm __volatile("movw %%fs,%0" : "=rm" (sel));
526 return (sel);
527 }
528
529 static __inline uint64_t
530 rgdt(void)
531 {
532 uint64_t gdtr;
533 __asm __volatile("sgdt %0" : "=m" (gdtr));
534 return (gdtr);
535 }
536
537 static __inline u_short
538 rgs(void)
539 {
540 u_short sel;
541 __asm __volatile("movw %%gs,%0" : "=rm" (sel));
542 return (sel);
543 }
544
545 static __inline uint64_t
546 ridt(void)
547 {
548 uint64_t idtr;
549 __asm __volatile("sidt %0" : "=m" (idtr));
550 return (idtr);
551 }
552
553 static __inline u_short
554 rldt(void)
555 {
556 u_short ldtr;
557 __asm __volatile("sldt %0" : "=g" (ldtr));
558 return (ldtr);
559 }
560
561 static __inline u_short
562 rss(void)
563 {
564 u_short sel;
565 __asm __volatile("movw %%ss,%0" : "=rm" (sel));
566 return (sel);
567 }
568
569 static __inline u_short
570 rtr(void)
571 {
572 u_short tr;
573 __asm __volatile("str %0" : "=g" (tr));
574 return (tr);
575 }
576
577 static __inline void
578 load_fs(u_short sel)
579 {
580 __asm __volatile("movw %0,%%fs" : : "rm" (sel));
581 }
582
583 static __inline void
584 load_gs(u_short sel)
585 {
586 __asm __volatile("movw %0,%%gs" : : "rm" (sel));
587 }
588
589 static __inline void
590 lidt(struct region_descriptor *addr)
591 {
592 __asm __volatile("lidt (%0)" : : "r" (addr));
593 }
594
595 static __inline void
596 lldt(u_short sel)
597 {
598 __asm __volatile("lldt %0" : : "r" (sel));
599 }
600
601 static __inline void
602 ltr(u_short sel)
603 {
604 __asm __volatile("ltr %0" : : "r" (sel));
605 }
606
607 static __inline u_int
608 rdr0(void)
609 {
610 u_int data;
611 __asm __volatile("movl %%dr0,%0" : "=r" (data));
612 return (data);
613 }
614
615 static __inline void
616 load_dr0(u_int dr0)
617 {
618 __asm __volatile("movl %0,%%dr0" : : "r" (dr0));
619 }
620
621 static __inline u_int
622 rdr1(void)
623 {
624 u_int data;
625 __asm __volatile("movl %%dr1,%0" : "=r" (data));
626 return (data);
627 }
628
629 static __inline void
630 load_dr1(u_int dr1)
631 {
632 __asm __volatile("movl %0,%%dr1" : : "r" (dr1));
633 }
634
635 static __inline u_int
636 rdr2(void)
637 {
638 u_int data;
639 __asm __volatile("movl %%dr2,%0" : "=r" (data));
640 return (data);
641 }
642
643 static __inline void
644 load_dr2(u_int dr2)
645 {
646 __asm __volatile("movl %0,%%dr2" : : "r" (dr2));
647 }
648
649 static __inline u_int
650 rdr3(void)
651 {
652 u_int data;
653 __asm __volatile("movl %%dr3,%0" : "=r" (data));
654 return (data);
655 }
656
657 static __inline void
658 load_dr3(u_int dr3)
659 {
660 __asm __volatile("movl %0,%%dr3" : : "r" (dr3));
661 }
662
663 static __inline u_int
664 rdr4(void)
665 {
666 u_int data;
667 __asm __volatile("movl %%dr4,%0" : "=r" (data));
668 return (data);
669 }
670
671 static __inline void
672 load_dr4(u_int dr4)
673 {
674 __asm __volatile("movl %0,%%dr4" : : "r" (dr4));
675 }
676
677 static __inline u_int
678 rdr5(void)
679 {
680 u_int data;
681 __asm __volatile("movl %%dr5,%0" : "=r" (data));
682 return (data);
683 }
684
685 static __inline void
686 load_dr5(u_int dr5)
687 {
688 __asm __volatile("movl %0,%%dr5" : : "r" (dr5));
689 }
690
691 static __inline u_int
692 rdr6(void)
693 {
694 u_int data;
695 __asm __volatile("movl %%dr6,%0" : "=r" (data));
696 return (data);
697 }
698
699 static __inline void
700 load_dr6(u_int dr6)
701 {
702 __asm __volatile("movl %0,%%dr6" : : "r" (dr6));
703 }
704
705 static __inline u_int
706 rdr7(void)
707 {
708 u_int data;
709 __asm __volatile("movl %%dr7,%0" : "=r" (data));
710 return (data);
711 }
712
713 static __inline void
714 load_dr7(u_int dr7)
715 {
716 __asm __volatile("movl %0,%%dr7" : : "r" (dr7));
717 }
718
719 static __inline u_char
720 read_cyrix_reg(u_char reg)
721 {
722 outb(0x22, reg);
723 return inb(0x23);
724 }
725
726 static __inline void
727 write_cyrix_reg(u_char reg, u_char data)
728 {
729 outb(0x22, reg);
730 outb(0x23, data);
731 }
732
733 static __inline register_t
734 intr_disable(void)
735 {
736 register_t eflags;
737
738 eflags = read_eflags();
739 disable_intr();
740 return (eflags);
741 }
742
743 static __inline void
744 intr_restore(register_t eflags)
745 {
746 write_eflags(eflags);
747 }
748
749 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */
750
751 int breakpoint(void);
752 u_int bsfl(u_int mask);
753 u_int bsrl(u_int mask);
754 void clflush(u_long addr);
755 void clts(void);
756 void cpuid_count(u_int ax, u_int cx, u_int *p);
757 void disable_intr(void);
758 void do_cpuid(u_int ax, u_int *p);
759 void enable_intr(void);
760 void halt(void);
761 void ia32_pause(void);
762 u_char inb(u_int port);
763 u_int inl(u_int port);
764 void insb(u_int port, void *addr, size_t count);
765 void insl(u_int port, void *addr, size_t count);
766 void insw(u_int port, void *addr, size_t count);
767 register_t intr_disable(void);
768 void intr_restore(register_t ef);
769 void invd(void);
770 void invlpg(u_int addr);
771 void invltlb(void);
772 u_short inw(u_int port);
773 void lidt(struct region_descriptor *addr);
774 void lldt(u_short sel);
775 void load_cr0(u_int cr0);
776 void load_cr3(u_int cr3);
777 void load_cr4(u_int cr4);
778 void load_dr0(u_int dr0);
779 void load_dr1(u_int dr1);
780 void load_dr2(u_int dr2);
781 void load_dr3(u_int dr3);
782 void load_dr4(u_int dr4);
783 void load_dr5(u_int dr5);
784 void load_dr6(u_int dr6);
785 void load_dr7(u_int dr7);
786 void load_fs(u_short sel);
787 void load_gs(u_short sel);
788 void ltr(u_short sel);
789 void outb(u_int port, u_char data);
790 void outl(u_int port, u_int data);
791 void outsb(u_int port, const void *addr, size_t count);
792 void outsl(u_int port, const void *addr, size_t count);
793 void outsw(u_int port, const void *addr, size_t count);
794 void outw(u_int port, u_short data);
795 u_int rcr0(void);
796 u_int rcr2(void);
797 u_int rcr3(void);
798 u_int rcr4(void);
799 uint64_t rdmsr(u_int msr);
800 uint64_t rdpmc(u_int pmc);
801 u_int rdr0(void);
802 u_int rdr1(void);
803 u_int rdr2(void);
804 u_int rdr3(void);
805 u_int rdr4(void);
806 u_int rdr5(void);
807 u_int rdr6(void);
808 u_int rdr7(void);
809 uint64_t rdtsc(void);
810 u_char read_cyrix_reg(u_char reg);
811 u_int read_eflags(void);
812 u_int rfs(void);
813 uint64_t rgdt(void);
814 u_int rgs(void);
815 uint64_t ridt(void);
816 u_short rldt(void);
817 u_short rtr(void);
818 void wbinvd(void);
819 void write_cyrix_reg(u_char reg, u_char data);
820 void write_eflags(u_int ef);
821 void wrmsr(u_int msr, uint64_t newval);
822
823 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */
824
825 void reset_dbregs(void);
826
827 #ifdef _KERNEL
828 int rdmsr_safe(u_int msr, uint64_t *val);
829 int wrmsr_safe(u_int msr, uint64_t newval);
830 #endif
831
832 #endif /* !_MACHINE_CPUFUNC_H_ */
Cache object: b9c1fadf23d65c1053ef0ccea5a21636
|