1 /*-
2 * Copyright (c) 1993 The Regents of the University of California.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. All advertising materials mentioning features or use of this software
14 * must display the following acknowledgement:
15 * This product includes software developed by the University of
16 * California, Berkeley and its contributors.
17 * 4. Neither the name of the University nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
32 *
33 * $FreeBSD$
34 */
35
36 /*
37 * Functions to provide access to special i386 instructions.
38 */
39
40 #ifndef _MACHINE_CPUFUNC_H_
41 #define _MACHINE_CPUFUNC_H_
42
43 #include <sys/cdefs.h>
44
45 __BEGIN_DECLS
46 #define readb(va) (*(volatile u_int8_t *) (va))
47 #define readw(va) (*(volatile u_int16_t *) (va))
48 #define readl(va) (*(volatile u_int32_t *) (va))
49
50 #define writeb(va, d) (*(volatile u_int8_t *) (va) = (d))
51 #define writew(va, d) (*(volatile u_int16_t *) (va) = (d))
52 #define writel(va, d) (*(volatile u_int32_t *) (va) = (d))
53
54 #ifdef __GNUC__
55
56 #ifdef SMP
57 #include <machine/lock.h> /* XXX */
58 #endif
59
60 #ifdef SWTCH_OPTIM_STATS
61 extern int tlb_flush_count; /* XXX */
62 #endif
63
64 static __inline void
65 breakpoint(void)
66 {
67 __asm __volatile("int $3");
68 }
69
70 static __inline u_int
71 bsfl(u_int mask)
72 {
73 u_int result;
74
75 __asm __volatile("bsfl %0,%0" : "=r" (result) : "" (mask));
76 return (result);
77 }
78
79 static __inline u_int
80 bsrl(u_int mask)
81 {
82 u_int result;
83
84 __asm __volatile("bsrl %0,%0" : "=r" (result) : "" (mask));
85 return (result);
86 }
87
88 static __inline void
89 disable_intr(void)
90 {
91 __asm __volatile("cli" : : : "memory");
92 #ifdef SMP
93 MPINTR_LOCK();
94 #endif
95 }
96
97 static __inline void
98 do_cpuid(u_int ax, u_int *p)
99 {
100 __asm __volatile("cpuid"
101 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
102 : "" (ax));
103 }
104
105 static __inline void
106 cpuid_count(u_int ax, u_int cx, u_int *p)
107 {
108 __asm __volatile("cpuid"
109 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
110 : "" (ax), "c" (cx));
111 }
112
113 static __inline void
114 enable_intr(void)
115 {
116 #ifdef SMP
117 MPINTR_UNLOCK();
118 #endif
119 __asm __volatile("sti");
120 }
121
122 #define HAVE_INLINE_FFS
123
124 static __inline int
125 ffs(int mask)
126 {
127 /*
128 * Note that gcc-2's builtin ffs would be used if we didn't declare
129 * this inline or turn off the builtin. The builtin is faster but
130 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later
131 * versions.
132 */
133 return (mask == 0 ? mask : bsfl((u_int)mask) + 1);
134 }
135
136 #define HAVE_INLINE_FLS
137
138 static __inline int
139 fls(int mask)
140 {
141 return (mask == 0 ? mask : bsrl((u_int)mask) + 1);
142 }
143
144 #if __GNUC__ < 2
145
146 #define inb(port) inbv(port)
147 #define outb(port, data) outbv(port, data)
148
149 #else /* __GNUC >= 2 */
150
151 /*
152 * The following complications are to get around gcc not having a
153 * constraint letter for the range 0..255. We still put "d" in the
154 * constraint because "i" isn't a valid constraint when the port
155 * isn't constant. This only matters for -O0 because otherwise
156 * the non-working version gets optimized away.
157 *
158 * Use an expression-statement instead of a conditional expression
159 * because gcc-2.6.0 would promote the operands of the conditional
160 * and produce poor code for "if ((inb(var) & const1) == const2)".
161 *
162 * The unnecessary test `(port) < 0x10000' is to generate a warning if
163 * the `port' has type u_short or smaller. Such types are pessimal.
164 * This actually only works for signed types. The range check is
165 * careful to avoid generating warnings.
166 */
167 #define inb(port) __extension__ ({ \
168 u_char _data; \
169 if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \
170 && (port) < 0x10000) \
171 _data = inbc(port); \
172 else \
173 _data = inbv(port); \
174 _data; })
175
176 #define outb(port, data) ( \
177 __builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \
178 && (port) < 0x10000 \
179 ? outbc(port, data) : outbv(port, data))
180
181 static __inline u_char
182 inbc(u_int port)
183 {
184 u_char data;
185
186 __asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port)));
187 return (data);
188 }
189
190 static __inline void
191 outbc(u_int port, u_char data)
192 {
193 __asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port)));
194 }
195
196 #endif /* __GNUC <= 2 */
197
198 static __inline u_char
199 inbv(u_int port)
200 {
201 u_char data;
202 /*
203 * We use %%dx and not %1 here because i/o is done at %dx and not at
204 * %edx, while gcc generates inferior code (movw instead of movl)
205 * if we tell it to load (u_short) port.
206 */
207 __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port));
208 return (data);
209 }
210
211 static __inline u_int
212 inl(u_int port)
213 {
214 u_int data;
215
216 __asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port));
217 return (data);
218 }
219
220 static __inline void
221 insb(u_int port, void *addr, size_t cnt)
222 {
223 __asm __volatile("cld; rep; insb"
224 : "=D" (addr), "=c" (cnt)
225 : "" (addr), "1" (cnt), "d" (port)
226 : "memory");
227 }
228
229 static __inline void
230 insw(u_int port, void *addr, size_t cnt)
231 {
232 __asm __volatile("cld; rep; insw"
233 : "=D" (addr), "=c" (cnt)
234 : "" (addr), "1" (cnt), "d" (port)
235 : "memory");
236 }
237
238 static __inline void
239 insl(u_int port, void *addr, size_t cnt)
240 {
241 __asm __volatile("cld; rep; insl"
242 : "=D" (addr), "=c" (cnt)
243 : "" (addr), "1" (cnt), "d" (port)
244 : "memory");
245 }
246
247 static __inline void
248 invd(void)
249 {
250 __asm __volatile("invd");
251 }
252
253 #if defined(SMP) && defined(_KERNEL)
254
255 /*
256 * When using APIC IPI's, invlpg() is not simply the invlpg instruction
257 * (this is a bug) and the inlining cost is prohibitive since the call
258 * executes into the IPI transmission system.
259 */
260 void invlpg __P((u_int addr));
261 void invltlb __P((void));
262
263 static __inline void
264 cpu_invlpg(void *addr)
265 {
266 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
267 }
268
269 static __inline void
270 cpu_invltlb(void)
271 {
272 u_int temp;
273 /*
274 * This should be implemented as load_cr3(rcr3()) when load_cr3()
275 * is inlined.
276 */
277 __asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp)
278 : : "memory");
279 #if defined(SWTCH_OPTIM_STATS)
280 ++tlb_flush_count;
281 #endif
282 }
283
284 #else /* !(SMP && _KERNEL) */
285
286 static __inline void
287 invlpg(u_int addr)
288 {
289 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
290 }
291
292 static __inline void
293 invltlb(void)
294 {
295 u_int temp;
296 /*
297 * This should be implemented as load_cr3(rcr3()) when load_cr3()
298 * is inlined.
299 */
300 __asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp)
301 : : "memory");
302 #ifdef SWTCH_OPTIM_STATS
303 ++tlb_flush_count;
304 #endif
305 }
306
307 #endif /* SMP && _KERNEL */
308
309 static __inline u_short
310 inw(u_int port)
311 {
312 u_short data;
313
314 __asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port));
315 return (data);
316 }
317
318 static __inline u_int
319 loadandclear(volatile u_int *addr)
320 {
321 u_int result;
322
323 __asm __volatile("xorl %0,%0; xchgl %1,%0"
324 : "=&r" (result) : "m" (*addr));
325 return (result);
326 }
327
328 static __inline void
329 outbv(u_int port, u_char data)
330 {
331 u_char al;
332 /*
333 * Use an unnecessary assignment to help gcc's register allocator.
334 * This make a large difference for gcc-1.40 and a tiny difference
335 * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for
336 * best results. gcc-2.6.0 can't handle this.
337 */
338 al = data;
339 __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port));
340 }
341
342 static __inline void
343 outl(u_int port, u_int data)
344 {
345 /*
346 * outl() and outw() aren't used much so we haven't looked at
347 * possible micro-optimizations such as the unnecessary
348 * assignment for them.
349 */
350 __asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port));
351 }
352
353 static __inline void
354 outsb(u_int port, const void *addr, size_t cnt)
355 {
356 __asm __volatile("cld; rep; outsb"
357 : "=S" (addr), "=c" (cnt)
358 : "" (addr), "1" (cnt), "d" (port));
359 }
360
361 static __inline void
362 outsw(u_int port, const void *addr, size_t cnt)
363 {
364 __asm __volatile("cld; rep; outsw"
365 : "=S" (addr), "=c" (cnt)
366 : "" (addr), "1" (cnt), "d" (port));
367 }
368
369 static __inline void
370 outsl(u_int port, const void *addr, size_t cnt)
371 {
372 __asm __volatile("cld; rep; outsl"
373 : "=S" (addr), "=c" (cnt)
374 : "" (addr), "1" (cnt), "d" (port));
375 }
376
377 static __inline void
378 outw(u_int port, u_short data)
379 {
380 __asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port));
381 }
382
383 static __inline u_int
384 rcr2(void)
385 {
386 u_int data;
387
388 __asm __volatile("movl %%cr2,%0" : "=r" (data));
389 return (data);
390 }
391
392 static __inline u_int
393 read_eflags(void)
394 {
395 u_int ef;
396
397 __asm __volatile("pushfl; popl %0" : "=r" (ef));
398 return (ef);
399 }
400
401 static __inline u_int64_t
402 rdmsr(u_int msr)
403 {
404 u_int64_t rv;
405
406 __asm __volatile(".byte 0x0f, 0x32" : "=A" (rv) : "c" (msr));
407 return (rv);
408 }
409
410 static __inline u_int64_t
411 rdpmc(u_int pmc)
412 {
413 u_int64_t rv;
414
415 __asm __volatile(".byte 0x0f, 0x33" : "=A" (rv) : "c" (pmc));
416 return (rv);
417 }
418
419 static __inline u_int64_t
420 rdtsc(void)
421 {
422 u_int64_t rv;
423
424 __asm __volatile(".byte 0x0f, 0x31" : "=A" (rv));
425 return (rv);
426 }
427
428 static __inline void
429 wbinvd(void)
430 {
431 __asm __volatile("wbinvd");
432 }
433
434 static __inline void
435 write_eflags(u_int ef)
436 {
437 __asm __volatile("pushl %0; popfl" : : "r" (ef));
438 }
439
440 static __inline void
441 wrmsr(u_int msr, u_int64_t newval)
442 {
443 __asm __volatile(".byte 0x0f, 0x30" : : "A" (newval), "c" (msr));
444 }
445
446 static __inline u_int
447 rfs(void)
448 {
449 u_int sel;
450 __asm __volatile("movl %%fs,%0" : "=rm" (sel));
451 return (sel);
452 }
453
454 static __inline u_int
455 rgs(void)
456 {
457 u_int sel;
458 __asm __volatile("movl %%gs,%0" : "=rm" (sel));
459 return (sel);
460 }
461
462 static __inline void
463 load_fs(u_int sel)
464 {
465 __asm __volatile("movl %0,%%fs" : : "rm" (sel));
466 }
467
468 static __inline void
469 load_gs(u_int sel)
470 {
471 __asm __volatile("movl %0,%%gs" : : "rm" (sel));
472 }
473
474 static __inline u_int
475 rdr0(void)
476 {
477 u_int data;
478 __asm __volatile("movl %%dr0,%0" : "=r" (data));
479 return (data);
480 }
481
482 static __inline void
483 load_dr0(u_int sel)
484 {
485 __asm __volatile("movl %0,%%dr0" : : "r" (sel));
486 }
487
488 static __inline u_int
489 rdr1(void)
490 {
491 u_int data;
492 __asm __volatile("movl %%dr1,%0" : "=r" (data));
493 return (data);
494 }
495
496 static __inline void
497 load_dr1(u_int sel)
498 {
499 __asm __volatile("movl %0,%%dr1" : : "r" (sel));
500 }
501
502 static __inline u_int
503 rdr2(void)
504 {
505 u_int data;
506 __asm __volatile("movl %%dr2,%0" : "=r" (data));
507 return (data);
508 }
509
510 static __inline void
511 load_dr2(u_int sel)
512 {
513 __asm __volatile("movl %0,%%dr2" : : "r" (sel));
514 }
515
516 static __inline u_int
517 rdr3(void)
518 {
519 u_int data;
520 __asm __volatile("movl %%dr3,%0" : "=r" (data));
521 return (data);
522 }
523
524 static __inline void
525 load_dr3(u_int sel)
526 {
527 __asm __volatile("movl %0,%%dr3" : : "r" (sel));
528 }
529
530 static __inline u_int
531 rdr4(void)
532 {
533 u_int data;
534 __asm __volatile("movl %%dr4,%0" : "=r" (data));
535 return (data);
536 }
537
538 static __inline void
539 load_dr4(u_int sel)
540 {
541 __asm __volatile("movl %0,%%dr4" : : "r" (sel));
542 }
543
544 static __inline u_int
545 rdr5(void)
546 {
547 u_int data;
548 __asm __volatile("movl %%dr5,%0" : "=r" (data));
549 return (data);
550 }
551
552 static __inline void
553 load_dr5(u_int sel)
554 {
555 __asm __volatile("movl %0,%%dr5" : : "r" (sel));
556 }
557
558 static __inline u_int
559 rdr6(void)
560 {
561 u_int data;
562 __asm __volatile("movl %%dr6,%0" : "=r" (data));
563 return (data);
564 }
565
566 static __inline void
567 load_dr6(u_int sel)
568 {
569 __asm __volatile("movl %0,%%dr6" : : "r" (sel));
570 }
571
572 static __inline u_int
573 rdr7(void)
574 {
575 u_int data;
576 __asm __volatile("movl %%dr7,%0" : "=r" (data));
577 return (data);
578 }
579
580 static __inline void
581 load_dr7(u_int sel)
582 {
583 __asm __volatile("movl %0,%%dr7" : : "r" (sel));
584 }
585
586 #else /* !__GNUC__ */
587
588 int breakpoint __P((void));
589 u_int bsfl __P((u_int mask));
590 u_int bsrl __P((u_int mask));
591 void disable_intr __P((void));
592 void do_cpuid __P((u_int ax, u_int *p));
593 void enable_intr __P((void));
594 u_char inb __P((u_int port));
595 u_int inl __P((u_int port));
596 void insb __P((u_int port, void *addr, size_t cnt));
597 void insl __P((u_int port, void *addr, size_t cnt));
598 void insw __P((u_int port, void *addr, size_t cnt));
599 void invd __P((void));
600 void invlpg __P((u_int addr));
601 void invltlb __P((void));
602 u_short inw __P((u_int port));
603 u_int loadandclear __P((u_int *addr));
604 void outb __P((u_int port, u_char data));
605 void outl __P((u_int port, u_int data));
606 void outsb __P((u_int port, void *addr, size_t cnt));
607 void outsl __P((u_int port, void *addr, size_t cnt));
608 void outsw __P((u_int port, void *addr, size_t cnt));
609 void outw __P((u_int port, u_short data));
610 u_int rcr2 __P((void));
611 u_int64_t rdmsr __P((u_int msr));
612 u_int64_t rdpmc __P((u_int pmc));
613 u_int64_t rdtsc __P((void));
614 u_int read_eflags __P((void));
615 void wbinvd __P((void));
616 void write_eflags __P((u_int ef));
617 void wrmsr __P((u_int msr, u_int64_t newval));
618 u_int rfs __P((void));
619 u_int rgs __P((void));
620 void load_fs __P((u_int sel));
621 void load_gs __P((u_int sel));
622
623 #endif /* __GNUC__ */
624
625 void load_cr0 __P((u_int cr0));
626 void load_cr3 __P((u_int cr3));
627 void load_cr4 __P((u_int cr4));
628 void ltr __P((u_short sel));
629 u_int rcr0 __P((void));
630 u_int rcr3 __P((void));
631 u_int rcr4 __P((void));
632 void reset_dbregs __P((void));
633 __END_DECLS
634
635 #endif /* !_MACHINE_CPUFUNC_H_ */
Cache object: d5853a818fa422e1e858e67108ec0ba2
|