1 /*-
2 * Copyright (c) 2018-2019 The FreeBSD Foundation
3 * Copyright (c) 2003 Peter Wemm.
4 * Copyright (c) 1993 The Regents of the University of California.
5 * All rights reserved.
6 *
7 * Portions of this software were developed by
8 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
9 * the FreeBSD Foundation.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 4. Neither the name of the University nor the names of its contributors
20 * may be used to endorse or promote products derived from this software
21 * without specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 * SUCH DAMAGE.
34 *
35 * $FreeBSD$
36 */
37
38 #include "opt_ddb.h"
39
40 #include <machine/asmacros.h>
41 #include <machine/specialreg.h>
42 #include <machine/pmap.h>
43
44 #include "assym.s"
45
46 .text
47
48 /*
49 * bcopy family
50 * void bzero(void *buf, u_int len)
51 */
52
53 /* done */
54 ENTRY(bzero)
55 PUSH_FRAME_POINTER
56 movq %rsi,%rcx
57 xorl %eax,%eax
58 shrq $3,%rcx
59 rep
60 stosq
61 movq %rsi,%rcx
62 andq $7,%rcx
63 rep
64 stosb
65 POP_FRAME_POINTER
66 ret
67 END(bzero)
68
69 /* Address: %rdi */
70 ENTRY(pagezero)
71 PUSH_FRAME_POINTER
72 movq $PAGE_SIZE/8,%rcx
73 xorl %eax,%eax
74 rep
75 stosq
76 POP_FRAME_POINTER
77 ret
78 END(pagezero)
79
80 ENTRY(bcmp)
81 PUSH_FRAME_POINTER
82 movq %rdx,%rcx
83 shrq $3,%rcx
84 repe
85 cmpsq
86 jne 1f
87
88 movq %rdx,%rcx
89 andq $7,%rcx
90 repe
91 cmpsb
92 1:
93 setne %al
94 movsbl %al,%eax
95 POP_FRAME_POINTER
96 ret
97 END(bcmp)
98
99 /*
100 * bcopy(src, dst, cnt)
101 * rdi, rsi, rdx
102 * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800
103 */
104 ENTRY(bcopy)
105 PUSH_FRAME_POINTER
106 xchgq %rsi,%rdi
107 movq %rdx,%rcx
108
109 movq %rdi,%rax
110 subq %rsi,%rax
111 cmpq %rcx,%rax /* overlapping && src < dst? */
112 jb 1f
113
114 shrq $3,%rcx /* copy by 64-bit words */
115 rep
116 movsq
117 movq %rdx,%rcx
118 andq $7,%rcx /* any bytes left? */
119 rep
120 movsb
121 POP_FRAME_POINTER
122 ret
123
124 /* ALIGN_TEXT */
125 1:
126 addq %rcx,%rdi /* copy backwards */
127 addq %rcx,%rsi
128 decq %rdi
129 decq %rsi
130 andq $7,%rcx /* any fractional bytes? */
131 std
132 rep
133 movsb
134 movq %rdx,%rcx /* copy remainder by 32-bit words */
135 shrq $3,%rcx
136 subq $7,%rsi
137 subq $7,%rdi
138 rep
139 movsq
140 cld
141 POP_FRAME_POINTER
142 ret
143 END(bcopy)
144
145 /*
146 * Note: memcpy does not support overlapping copies
147 */
148 ENTRY(memcpy)
149 PUSH_FRAME_POINTER
150 movq %rdi,%rax
151 movq %rdx,%rcx
152 shrq $3,%rcx /* copy by 64-bit words */
153 rep
154 movsq
155 movq %rdx,%rcx
156 andq $7,%rcx /* any bytes left? */
157 rep
158 movsb
159 POP_FRAME_POINTER
160 ret
161 END(memcpy)
162
163 /*
164 * pagecopy(%rdi=from, %rsi=to)
165 */
166 ENTRY(pagecopy)
167 PUSH_FRAME_POINTER
168 movq $-PAGE_SIZE,%rax
169 movq %rax,%rdx
170 subq %rax,%rdi
171 subq %rax,%rsi
172 1:
173 prefetchnta (%rdi,%rax)
174 addq $64,%rax
175 jne 1b
176 2:
177 movq (%rdi,%rdx),%rax
178 movnti %rax,(%rsi,%rdx)
179 movq 8(%rdi,%rdx),%rax
180 movnti %rax,8(%rsi,%rdx)
181 movq 16(%rdi,%rdx),%rax
182 movnti %rax,16(%rsi,%rdx)
183 movq 24(%rdi,%rdx),%rax
184 movnti %rax,24(%rsi,%rdx)
185 addq $32,%rdx
186 jne 2b
187 sfence
188 POP_FRAME_POINTER
189 ret
190 END(pagecopy)
191
192 /* fillw(pat, base, cnt) */
193 /* %rdi,%rsi, %rdx */
194 ENTRY(fillw)
195 PUSH_FRAME_POINTER
196 movq %rdi,%rax
197 movq %rsi,%rdi
198 movq %rdx,%rcx
199 rep
200 stosw
201 POP_FRAME_POINTER
202 ret
203 END(fillw)
204
205 /*****************************************************************************/
206 /* copyout and fubyte family */
207 /*****************************************************************************/
208 /*
209 * Access user memory from inside the kernel. These routines should be
210 * the only places that do this.
211 *
212 * These routines set curpcb->pcb_onfault for the time they execute. When a
213 * protection violation occurs inside the functions, the trap handler
214 * returns to *curpcb->pcb_onfault instead of the function.
215 */
216
217 /*
218 * copyout(from_kernel, to_user, len)
219 * %rdi, %rsi, %rdx
220 */
221 ENTRY(copyout)
222 PUSH_FRAME_POINTER
223 movq PCPU(CURPCB),%rax
224 movq $copyout_fault,PCB_ONFAULT(%rax)
225 testq %rdx,%rdx /* anything to do? */
226 jz done_copyout
227
228 /*
229 * Check explicitly for non-user addresses. This check is essential
230 * because it prevents usermode from writing into the kernel. We do
231 * not verify anywhere else that the user did not specify a rogue
232 * address.
233 */
234 /*
235 * First, prevent address wrapping.
236 */
237 movq %rsi,%rax
238 addq %rdx,%rax
239 jc copyout_fault
240 /*
241 * XXX STOP USING VM_MAXUSER_ADDRESS.
242 * It is an end address, not a max, so every time it is used correctly it
243 * looks like there is an off by one error, and of course it caused an off
244 * by one error in several places.
245 */
246 movq $VM_MAXUSER_ADDRESS,%rcx
247 cmpq %rcx,%rax
248 ja copyout_fault
249
250 xchgq %rdi,%rsi
251 /* bcopy(%rsi, %rdi, %rdx) */
252 movq %rdx,%rcx
253
254 shrq $3,%rcx
255 rep
256 movsq
257 movb %dl,%cl
258 andb $7,%cl
259 rep
260 movsb
261
262 done_copyout:
263 xorl %eax,%eax
264 movq PCPU(CURPCB),%rdx
265 movq %rax,PCB_ONFAULT(%rdx)
266 POP_FRAME_POINTER
267 ret
268
269 ALIGN_TEXT
270 copyout_fault:
271 movq PCPU(CURPCB),%rdx
272 movq $0,PCB_ONFAULT(%rdx)
273 movq $EFAULT,%rax
274 POP_FRAME_POINTER
275 ret
276 END(copyout)
277
278 /*
279 * copyin(from_user, to_kernel, len)
280 * %rdi, %rsi, %rdx
281 */
282 ENTRY(copyin)
283 PUSH_FRAME_POINTER
284 movq PCPU(CURPCB),%rax
285 movq $copyin_fault,PCB_ONFAULT(%rax)
286 testq %rdx,%rdx /* anything to do? */
287 jz done_copyin
288
289 /*
290 * make sure address is valid
291 */
292 movq %rdi,%rax
293 addq %rdx,%rax
294 jc copyin_fault
295 movq $VM_MAXUSER_ADDRESS,%rcx
296 cmpq %rcx,%rax
297 ja copyin_fault
298
299 xchgq %rdi,%rsi
300 movq %rdx,%rcx
301 movb %cl,%al
302 shrq $3,%rcx /* copy longword-wise */
303 rep
304 movsq
305 movb %al,%cl
306 andb $7,%cl /* copy remaining bytes */
307 rep
308 movsb
309
310 done_copyin:
311 xorl %eax,%eax
312 movq PCPU(CURPCB),%rdx
313 movq %rax,PCB_ONFAULT(%rdx)
314 POP_FRAME_POINTER
315 ret
316
317 ALIGN_TEXT
318 copyin_fault:
319 movq PCPU(CURPCB),%rdx
320 movq $0,PCB_ONFAULT(%rdx)
321 movq $EFAULT,%rax
322 POP_FRAME_POINTER
323 ret
324 END(copyin)
325
326 /*
327 * casueword32. Compare and set user integer. Returns -1 on fault,
328 * 0 if access was successful. Old value is written to *oldp.
329 * dst = %rdi, old = %esi, oldp = %rdx, new = %ecx
330 */
331 ENTRY(casueword32)
332 PUSH_FRAME_POINTER
333 movq PCPU(CURPCB),%r8
334 movq $fusufault,PCB_ONFAULT(%r8)
335
336 movq $VM_MAXUSER_ADDRESS-4,%rax
337 cmpq %rax,%rdi /* verify address is valid */
338 ja fusufault
339
340 movl %esi,%eax /* old */
341 #ifdef SMP
342 lock
343 #endif
344 cmpxchgl %ecx,(%rdi) /* new = %ecx */
345
346 /*
347 * The old value is in %eax. If the store succeeded it will be the
348 * value we expected (old) from before the store, otherwise it will
349 * be the current value. Save %eax into %esi to prepare the return
350 * value.
351 */
352 movl %eax,%esi
353 xorl %eax,%eax
354 movq %rax,PCB_ONFAULT(%r8)
355
356 /*
357 * Access the oldp after the pcb_onfault is cleared, to correctly
358 * catch corrupted pointer.
359 */
360 movl %esi,(%rdx) /* oldp = %rdx */
361 POP_FRAME_POINTER
362 ret
363 END(casueword32)
364
365 /*
366 * casueword. Compare and set user long. Returns -1 on fault,
367 * 0 if access was successful. Old value is written to *oldp.
368 * dst = %rdi, old = %rsi, oldp = %rdx, new = %rcx
369 */
370 ENTRY(casueword)
371 PUSH_FRAME_POINTER
372 movq PCPU(CURPCB),%r8
373 movq $fusufault,PCB_ONFAULT(%r8)
374
375 movq $VM_MAXUSER_ADDRESS-4,%rax
376 cmpq %rax,%rdi /* verify address is valid */
377 ja fusufault
378
379 movq %rsi,%rax /* old */
380 #ifdef SMP
381 lock
382 #endif
383 cmpxchgq %rcx,(%rdi) /* new = %rcx */
384
385 /*
386 * The old value is in %rax. If the store succeeded it will be the
387 * value we expected (old) from before the store, otherwise it will
388 * be the current value.
389 */
390 movq %rax,%rsi
391 xorl %eax,%eax
392 movq %rax,PCB_ONFAULT(%r8)
393 movq %rsi,(%rdx)
394 POP_FRAME_POINTER
395 ret
396 END(casueword)
397
398 /*
399 * Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit
400 * byte from user memory.
401 * addr = %rdi, valp = %rsi
402 */
403
404 ALTENTRY(fueword64)
405 ENTRY(fueword)
406 PUSH_FRAME_POINTER
407 movq PCPU(CURPCB),%rcx
408 movq $fusufault,PCB_ONFAULT(%rcx)
409
410 movq $VM_MAXUSER_ADDRESS-8,%rax
411 cmpq %rax,%rdi /* verify address is valid */
412 ja fusufault
413
414 xorl %eax,%eax
415 movq (%rdi),%r11
416 movq %rax,PCB_ONFAULT(%rcx)
417 movq %r11,(%rsi)
418 POP_FRAME_POINTER
419 ret
420 END(fueword64)
421 END(fueword)
422
423 ENTRY(fueword32)
424 PUSH_FRAME_POINTER
425 movq PCPU(CURPCB),%rcx
426 movq $fusufault,PCB_ONFAULT(%rcx)
427
428 movq $VM_MAXUSER_ADDRESS-4,%rax
429 cmpq %rax,%rdi /* verify address is valid */
430 ja fusufault
431
432 xorl %eax,%eax
433 movl (%rdi),%r11d
434 movq %rax,PCB_ONFAULT(%rcx)
435 movl %r11d,(%rsi)
436 POP_FRAME_POINTER
437 ret
438 END(fueword32)
439
440 /*
441 * fuswintr() and suswintr() are specialized variants of fuword16() and
442 * suword16(), respectively. They are called from the profiling code,
443 * potentially at interrupt time. If they fail, that's okay; good things
444 * will happen later. They always fail for now, until the trap code is
445 * able to deal with this.
446 */
447 ALTENTRY(suswintr)
448 ENTRY(fuswintr)
449 movq $-1,%rax
450 ret
451 END(suswintr)
452 END(fuswintr)
453
454 ENTRY(fuword16)
455 PUSH_FRAME_POINTER
456 movq PCPU(CURPCB),%rcx
457 movq $fusufault,PCB_ONFAULT(%rcx)
458
459 movq $VM_MAXUSER_ADDRESS-2,%rax
460 cmpq %rax,%rdi
461 ja fusufault
462
463 movzwl (%rdi),%eax
464 movq $0,PCB_ONFAULT(%rcx)
465 POP_FRAME_POINTER
466 ret
467 END(fuword16)
468
469 ENTRY(fubyte)
470 PUSH_FRAME_POINTER
471 movq PCPU(CURPCB),%rcx
472 movq $fusufault,PCB_ONFAULT(%rcx)
473
474 movq $VM_MAXUSER_ADDRESS-1,%rax
475 cmpq %rax,%rdi
476 ja fusufault
477
478 movzbl (%rdi),%eax
479 movq $0,PCB_ONFAULT(%rcx)
480 POP_FRAME_POINTER
481 ret
482 END(fubyte)
483
484 ALIGN_TEXT
485 fusufault:
486 movq PCPU(CURPCB),%rcx
487 xorl %eax,%eax
488 movq %rax,PCB_ONFAULT(%rcx)
489 decq %rax
490 POP_FRAME_POINTER
491 ret
492
493 /*
494 * Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to
495 * user memory.
496 * addr = %rdi, value = %rsi
497 */
498 ALTENTRY(suword64)
499 ENTRY(suword)
500 PUSH_FRAME_POINTER
501 movq PCPU(CURPCB),%rcx
502 movq $fusufault,PCB_ONFAULT(%rcx)
503
504 movq $VM_MAXUSER_ADDRESS-8,%rax
505 cmpq %rax,%rdi /* verify address validity */
506 ja fusufault
507
508 movq %rsi,(%rdi)
509 xorl %eax,%eax
510 movq PCPU(CURPCB),%rcx
511 movq %rax,PCB_ONFAULT(%rcx)
512 POP_FRAME_POINTER
513 ret
514 END(suword64)
515 END(suword)
516
517 ENTRY(suword32)
518 PUSH_FRAME_POINTER
519 movq PCPU(CURPCB),%rcx
520 movq $fusufault,PCB_ONFAULT(%rcx)
521
522 movq $VM_MAXUSER_ADDRESS-4,%rax
523 cmpq %rax,%rdi /* verify address validity */
524 ja fusufault
525
526 movl %esi,(%rdi)
527 xorl %eax,%eax
528 movq PCPU(CURPCB),%rcx
529 movq %rax,PCB_ONFAULT(%rcx)
530 POP_FRAME_POINTER
531 ret
532 END(suword32)
533
534 ENTRY(suword16)
535 PUSH_FRAME_POINTER
536 movq PCPU(CURPCB),%rcx
537 movq $fusufault,PCB_ONFAULT(%rcx)
538
539 movq $VM_MAXUSER_ADDRESS-2,%rax
540 cmpq %rax,%rdi /* verify address validity */
541 ja fusufault
542
543 movw %si,(%rdi)
544 xorl %eax,%eax
545 movq PCPU(CURPCB),%rcx /* restore trashed register */
546 movq %rax,PCB_ONFAULT(%rcx)
547 POP_FRAME_POINTER
548 ret
549 END(suword16)
550
551 ENTRY(subyte)
552 PUSH_FRAME_POINTER
553 movq PCPU(CURPCB),%rcx
554 movq $fusufault,PCB_ONFAULT(%rcx)
555
556 movq $VM_MAXUSER_ADDRESS-1,%rax
557 cmpq %rax,%rdi /* verify address validity */
558 ja fusufault
559
560 movl %esi,%eax
561 movb %al,(%rdi)
562 xorl %eax,%eax
563 movq PCPU(CURPCB),%rcx /* restore trashed register */
564 movq %rax,PCB_ONFAULT(%rcx)
565 POP_FRAME_POINTER
566 ret
567 END(subyte)
568
569 /*
570 * copyinstr(from, to, maxlen, int *lencopied)
571 * %rdi, %rsi, %rdx, %rcx
572 *
573 * copy a string from 'from' to 'to', stop when a 0 character is reached.
574 * return ENAMETOOLONG if string is longer than maxlen, and
575 * EFAULT on protection violations. If lencopied is non-zero,
576 * return the actual length in *lencopied.
577 */
578 ENTRY(copyinstr)
579 PUSH_FRAME_POINTER
580 movq %rdx,%r8 /* %r8 = maxlen */
581 movq %rcx,%r9 /* %r9 = *len */
582 xchgq %rdi,%rsi /* %rdi = from, %rsi = to */
583 movq PCPU(CURPCB),%rcx
584 movq $cpystrflt,PCB_ONFAULT(%rcx)
585
586 movq $VM_MAXUSER_ADDRESS,%rax
587
588 /* make sure 'from' is within bounds */
589 subq %rsi,%rax
590 jbe cpystrflt
591
592 /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */
593 cmpq %rdx,%rax
594 jae 1f
595 movq %rax,%rdx
596 movq %rax,%r8
597 1:
598 incq %rdx
599
600 2:
601 decq %rdx
602 jz 3f
603
604 lodsb
605 stosb
606 orb %al,%al
607 jnz 2b
608
609 /* Success -- 0 byte reached */
610 decq %rdx
611 xorl %eax,%eax
612 jmp cpystrflt_x
613 3:
614 /* rdx is zero - return ENAMETOOLONG or EFAULT */
615 movq $VM_MAXUSER_ADDRESS,%rax
616 cmpq %rax,%rsi
617 jae cpystrflt
618 4:
619 movq $ENAMETOOLONG,%rax
620 jmp cpystrflt_x
621
622 cpystrflt:
623 movq $EFAULT,%rax
624
625 cpystrflt_x:
626 /* set *lencopied and return %eax */
627 movq PCPU(CURPCB),%rcx
628 movq $0,PCB_ONFAULT(%rcx)
629
630 testq %r9,%r9
631 jz 1f
632 subq %rdx,%r8
633 movq %r8,(%r9)
634 1:
635 POP_FRAME_POINTER
636 ret
637 END(copyinstr)
638
639 /*
640 * copystr(from, to, maxlen, int *lencopied)
641 * %rdi, %rsi, %rdx, %rcx
642 */
643 ENTRY(copystr)
644 PUSH_FRAME_POINTER
645 movq %rdx,%r8 /* %r8 = maxlen */
646
647 xchgq %rdi,%rsi
648 incq %rdx
649 1:
650 decq %rdx
651 jz 4f
652 lodsb
653 stosb
654 orb %al,%al
655 jnz 1b
656
657 /* Success -- 0 byte reached */
658 decq %rdx
659 xorl %eax,%eax
660 jmp 6f
661 4:
662 /* rdx is zero -- return ENAMETOOLONG */
663 movq $ENAMETOOLONG,%rax
664
665 6:
666
667 testq %rcx,%rcx
668 jz 7f
669 /* set *lencopied and return %rax */
670 subq %rdx,%r8
671 movq %r8,(%rcx)
672 7:
673 POP_FRAME_POINTER
674 ret
675 END(copystr)
676
677 /*
678 * Handling of special amd64 registers and descriptor tables etc
679 */
680 /* void lgdt(struct region_descriptor *rdp); */
681 ENTRY(lgdt)
682 /* reload the descriptor table */
683 lgdt (%rdi)
684
685 /* flush the prefetch q */
686 jmp 1f
687 nop
688 1:
689 movl $KDSEL,%eax
690 movl %eax,%ds
691 movl %eax,%es
692 movl %eax,%fs /* Beware, use wrmsr to set 64 bit base */
693 movl %eax,%gs
694 movl %eax,%ss
695
696 /* reload code selector by turning return into intersegmental return */
697 popq %rax
698 pushq $KCSEL
699 pushq %rax
700 MEXITCOUNT
701 lretq
702 END(lgdt)
703
704 /*****************************************************************************/
705 /* setjump, longjump */
706 /*****************************************************************************/
707
708 ENTRY(setjmp)
709 movq %rbx,0(%rdi) /* save rbx */
710 movq %rsp,8(%rdi) /* save rsp */
711 movq %rbp,16(%rdi) /* save rbp */
712 movq %r12,24(%rdi) /* save r12 */
713 movq %r13,32(%rdi) /* save r13 */
714 movq %r14,40(%rdi) /* save r14 */
715 movq %r15,48(%rdi) /* save r15 */
716 movq 0(%rsp),%rdx /* get rta */
717 movq %rdx,56(%rdi) /* save rip */
718 xorl %eax,%eax /* return(0); */
719 ret
720 END(setjmp)
721
722 ENTRY(longjmp)
723 movq 0(%rdi),%rbx /* restore rbx */
724 movq 8(%rdi),%rsp /* restore rsp */
725 movq 16(%rdi),%rbp /* restore rbp */
726 movq 24(%rdi),%r12 /* restore r12 */
727 movq 32(%rdi),%r13 /* restore r13 */
728 movq 40(%rdi),%r14 /* restore r14 */
729 movq 48(%rdi),%r15 /* restore r15 */
730 movq 56(%rdi),%rdx /* get rta */
731 movq %rdx,0(%rsp) /* put in return frame */
732 xorl %eax,%eax /* return(1); */
733 incl %eax
734 ret
735 END(longjmp)
736
737 /*
738 * Support for reading MSRs in the safe manner.
739 */
740 ENTRY(rdmsr_safe)
741 /* int rdmsr_safe(u_int msr, uint64_t *data) */
742 PUSH_FRAME_POINTER
743 movq PCPU(CURPCB),%r8
744 movq $msr_onfault,PCB_ONFAULT(%r8)
745 movl %edi,%ecx
746 rdmsr /* Read MSR pointed by %ecx. Returns
747 hi byte in edx, lo in %eax */
748 salq $32,%rdx /* sign-shift %rdx left */
749 movl %eax,%eax /* zero-extend %eax -> %rax */
750 orq %rdx,%rax
751 movq %rax,(%rsi)
752 xorq %rax,%rax
753 movq %rax,PCB_ONFAULT(%r8)
754 POP_FRAME_POINTER
755 ret
756
757 /*
758 * Support for writing MSRs in the safe manner.
759 */
760 ENTRY(wrmsr_safe)
761 /* int wrmsr_safe(u_int msr, uint64_t data) */
762 PUSH_FRAME_POINTER
763 movq PCPU(CURPCB),%r8
764 movq $msr_onfault,PCB_ONFAULT(%r8)
765 movl %edi,%ecx
766 movl %esi,%eax
767 sarq $32,%rsi
768 movl %esi,%edx
769 wrmsr /* Write MSR pointed by %ecx. Accepts
770 hi byte in edx, lo in %eax. */
771 xorq %rax,%rax
772 movq %rax,PCB_ONFAULT(%r8)
773 POP_FRAME_POINTER
774 ret
775
776 /*
777 * MSR operations fault handler
778 */
779 ALIGN_TEXT
780 msr_onfault:
781 movq $0,PCB_ONFAULT(%r8)
782 movl $EFAULT,%eax
783 POP_FRAME_POINTER
784 ret
785
786 /*
787 * void pmap_pti_pcid_invalidate(uint64_t ucr3, uint64_t kcr3);
788 * Invalidates address space addressed by ucr3, then returns to kcr3.
789 * Done in assembler to ensure no other memory accesses happen while
790 * on ucr3.
791 */
792 ALIGN_TEXT
793 ENTRY(pmap_pti_pcid_invalidate)
794 pushfq
795 cli
796 movq %rdi,%cr3 /* to user page table */
797 movq %rsi,%cr3 /* back to kernel */
798 popfq
799 retq
800
801 /*
802 * void pmap_pti_pcid_invlpg(uint64_t ucr3, uint64_t kcr3, vm_offset_t va);
803 * Invalidates virtual address va in address space ucr3, then returns to kcr3.
804 */
805 ALIGN_TEXT
806 ENTRY(pmap_pti_pcid_invlpg)
807 pushfq
808 cli
809 movq %rdi,%cr3 /* to user page table */
810 invlpg (%rdx)
811 movq %rsi,%cr3 /* back to kernel */
812 popfq
813 retq
814
815 /*
816 * void pmap_pti_pcid_invlrng(uint64_t ucr3, uint64_t kcr3, vm_offset_t sva,
817 * vm_offset_t eva);
818 * Invalidates virtual addresses between sva and eva in address space ucr3,
819 * then returns to kcr3.
820 */
821 ALIGN_TEXT
822 ENTRY(pmap_pti_pcid_invlrng)
823 pushfq
824 cli
825 movq %rdi,%cr3 /* to user page table */
826 1: invlpg (%rdx)
827 addq $PAGE_SIZE,%rdx
828 cmpq %rdx,%rcx
829 ja 1b
830 movq %rsi,%cr3 /* back to kernel */
831 popfq
832 retq
833
834 .altmacro
835 .macro rsb_seq_label l
836 rsb_seq_\l:
837 .endm
838 .macro rsb_call_label l
839 call rsb_seq_\l
840 .endm
841 .macro rsb_seq count
842 ll=1
843 .rept \count
844 rsb_call_label %(ll)
845 nop
846 rsb_seq_label %(ll)
847 addq $8,%rsp
848 ll=ll+1
849 .endr
850 .endm
851
852 ENTRY(rsb_flush)
853 rsb_seq 32
854 ret
855
856 /* all callers already saved %rax, %rdx, and %rcx */
857 ENTRY(handle_ibrs_entry)
858 cmpb $0,hw_ibrs_ibpb_active(%rip)
859 je 1f
860 movl $MSR_IA32_SPEC_CTRL,%ecx
861 rdmsr
862 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
863 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32,%edx
864 wrmsr
865 movb $1,PCPU(IBPB_SET)
866 testl $CPUID_STDEXT_SMEP,cpu_stdext_feature(%rip)
867 je rsb_flush
868 1: ret
869 END(handle_ibrs_entry)
870
871 ENTRY(handle_ibrs_exit)
872 cmpb $0,PCPU(IBPB_SET)
873 je 1f
874 movl $MSR_IA32_SPEC_CTRL,%ecx
875 rdmsr
876 andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
877 andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
878 wrmsr
879 movb $0,PCPU(IBPB_SET)
880 1: ret
881 END(handle_ibrs_exit)
882
883 /* registers-neutral version, but needs stack */
884 ENTRY(handle_ibrs_exit_rs)
885 cmpb $0,PCPU(IBPB_SET)
886 je 1f
887 pushq %rax
888 pushq %rdx
889 pushq %rcx
890 movl $MSR_IA32_SPEC_CTRL,%ecx
891 rdmsr
892 andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
893 andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
894 wrmsr
895 popq %rcx
896 popq %rdx
897 popq %rax
898 movb $0,PCPU(IBPB_SET)
899 1: ret
900 END(handle_ibrs_exit_rs)
901
902 .noaltmacro
903
904 /*
905 * Flush L1D cache. Load enough of the data from the kernel text
906 * to flush existing L1D content.
907 *
908 * N.B. The function does not follow ABI calling conventions, it corrupts %rbx.
909 * The vmm.ko caller expects that only %rax, %rdx, %rbx, %rcx, %r9, and %rflags
910 * registers are clobbered. The NMI handler caller only needs %r13 preserved.
911 */
912 ENTRY(flush_l1d_sw)
913 #define L1D_FLUSH_SIZE (64 * 1024)
914 movq $KERNBASE, %r9
915 movq $-L1D_FLUSH_SIZE, %rcx
916 /*
917 * pass 1: Preload TLB.
918 * Kernel text is mapped using superpages. TLB preload is
919 * done for the benefit of older CPUs which split 2M page
920 * into 4k TLB entries.
921 */
922 1: movb L1D_FLUSH_SIZE(%r9, %rcx), %al
923 addq $PAGE_SIZE, %rcx
924 jne 1b
925 xorl %eax, %eax
926 cpuid
927 movq $-L1D_FLUSH_SIZE, %rcx
928 /* pass 2: Read each cache line. */
929 2: movb L1D_FLUSH_SIZE(%r9, %rcx), %al
930 addq $64, %rcx
931 jne 2b
932 lfence
933 ret
934 #undef L1D_FLUSH_SIZE
935 END(flush_l1d_sw)
936
937 ENTRY(mds_handler_void)
938 retq
939 END(mds_handler_void)
940
941 ENTRY(mds_handler_verw)
942 subq $8, %rsp
943 movw %ds, (%rsp)
944 verw (%rsp)
945 addq $8, %rsp
946 retq
947 END(mds_handler_verw)
948
949 ENTRY(mds_handler_ivb)
950 pushq %rax
951 pushq %rdx
952 pushq %rcx
953
954 movq %cr0, %rax
955 testb $CR0_TS, %al
956 je 1f
957 clts
958 1: movq PCPU(MDS_BUF), %rdx
959 movdqa %xmm0, PCPU(MDS_TMP)
960 pxor %xmm0, %xmm0
961
962 lfence
963 orpd (%rdx), %xmm0
964 orpd (%rdx), %xmm0
965 mfence
966 movl $40, %ecx
967 addq $16, %rdx
968 2: movntdq %xmm0, (%rdx)
969 addq $16, %rdx
970 decl %ecx
971 jnz 2b
972 mfence
973
974 movdqa PCPU(MDS_TMP),%xmm0
975 testb $CR0_TS, %al
976 je 3f
977 movq %rax, %cr0
978 3: popq %rcx
979 popq %rdx
980 popq %rax
981 retq
982 END(mds_handler_ivb)
983
984 ENTRY(mds_handler_bdw)
985 pushq %rax
986 pushq %rbx
987 pushq %rcx
988 pushq %rdi
989 pushq %rsi
990
991 movq %cr0, %rax
992 testb $CR0_TS, %al
993 je 1f
994 clts
995 1: movq PCPU(MDS_BUF), %rbx
996 movdqa %xmm0, PCPU(MDS_TMP)
997 pxor %xmm0, %xmm0
998
999 movq %rbx, %rdi
1000 movq %rbx, %rsi
1001 movl $40, %ecx
1002 2: movntdq %xmm0, (%rbx)
1003 addq $16, %rbx
1004 decl %ecx
1005 jnz 2b
1006 mfence
1007 movl $1536, %ecx
1008 rep; movsb
1009 lfence
1010
1011 movdqa PCPU(MDS_TMP),%xmm0
1012 testb $CR0_TS, %al
1013 je 3f
1014 movq %rax, %cr0
1015 3: popq %rsi
1016 popq %rdi
1017 popq %rcx
1018 popq %rbx
1019 popq %rax
1020 retq
1021 END(mds_handler_bdw)
1022
1023 ENTRY(mds_handler_skl_sse)
1024 pushq %rax
1025 pushq %rdx
1026 pushq %rcx
1027 pushq %rdi
1028
1029 movq %cr0, %rax
1030 testb $CR0_TS, %al
1031 je 1f
1032 clts
1033 1: movq PCPU(MDS_BUF), %rdi
1034 movq PCPU(MDS_BUF64), %rdx
1035 movdqa %xmm0, PCPU(MDS_TMP)
1036 pxor %xmm0, %xmm0
1037
1038 lfence
1039 orpd (%rdx), %xmm0
1040 orpd (%rdx), %xmm0
1041 xorl %eax, %eax
1042 2: clflushopt 5376(%rdi, %rax, 8)
1043 addl $8, %eax
1044 cmpl $8 * 12, %eax
1045 jb 2b
1046 sfence
1047 movl $6144, %ecx
1048 xorl %eax, %eax
1049 rep; stosb
1050 mfence
1051
1052 movdqa PCPU(MDS_TMP), %xmm0
1053 testb $CR0_TS, %al
1054 je 3f
1055 movq %rax, %cr0
1056 3: popq %rdi
1057 popq %rcx
1058 popq %rdx
1059 popq %rax
1060 retq
1061 END(mds_handler_skl_sse)
1062
1063 ENTRY(mds_handler_skl_avx)
1064 pushq %rax
1065 pushq %rdx
1066 pushq %rcx
1067 pushq %rdi
1068
1069 movq %cr0, %rax
1070 testb $CR0_TS, %al
1071 je 1f
1072 clts
1073 1: movq PCPU(MDS_BUF), %rdi
1074 movq PCPU(MDS_BUF64), %rdx
1075 vmovdqa %ymm0, PCPU(MDS_TMP)
1076 vpxor %ymm0, %ymm0, %ymm0
1077
1078 lfence
1079 vorpd (%rdx), %ymm0, %ymm0
1080 vorpd (%rdx), %ymm0, %ymm0
1081 xorl %eax, %eax
1082 2: clflushopt 5376(%rdi, %rax, 8)
1083 addl $8, %eax
1084 cmpl $8 * 12, %eax
1085 jb 2b
1086 sfence
1087 movl $6144, %ecx
1088 xorl %eax, %eax
1089 rep; stosb
1090 mfence
1091
1092 vmovdqa PCPU(MDS_TMP), %ymm0
1093 testb $CR0_TS, %al
1094 je 3f
1095 movq %rax, %cr0
1096 3: popq %rdi
1097 popq %rcx
1098 popq %rdx
1099 popq %rax
1100 retq
1101 END(mds_handler_skl_avx)
1102
1103 ENTRY(mds_handler_skl_avx512)
1104 pushq %rax
1105 pushq %rdx
1106 pushq %rcx
1107 pushq %rdi
1108
1109 movq %cr0, %rax
1110 testb $CR0_TS, %al
1111 je 1f
1112 clts
1113 1: movq PCPU(MDS_BUF), %rdi
1114 movq PCPU(MDS_BUF64), %rdx
1115 vmovdqa64 %zmm0, PCPU(MDS_TMP)
1116 vpxor %zmm0, %zmm0, %zmm0
1117
1118 lfence
1119 vorpd (%rdx), %zmm0, %zmm0
1120 vorpd (%rdx), %zmm0, %zmm0
1121 xorl %eax, %eax
1122 2: clflushopt 5376(%rdi, %rax, 8)
1123 addl $8, %eax
1124 cmpl $8 * 12, %eax
1125 jb 2b
1126 sfence
1127 movl $6144, %ecx
1128 xorl %eax, %eax
1129 rep; stosb
1130 mfence
1131
1132 vmovdqa64 PCPU(MDS_TMP), %zmm0
1133 testb $CR0_TS, %al
1134 je 3f
1135 movq %rax, %cr0
1136 3: popq %rdi
1137 popq %rcx
1138 popq %rdx
1139 popq %rax
1140 retq
1141 END(mds_handler_skl_avx512)
1142
1143 ENTRY(mds_handler_silvermont)
1144 pushq %rax
1145 pushq %rdx
1146 pushq %rcx
1147
1148 movq %cr0, %rax
1149 testb $CR0_TS, %al
1150 je 1f
1151 clts
1152 1: movq PCPU(MDS_BUF), %rdx
1153 movdqa %xmm0, PCPU(MDS_TMP)
1154 pxor %xmm0, %xmm0
1155
1156 movl $16, %ecx
1157 2: movntdq %xmm0, (%rdx)
1158 addq $16, %rdx
1159 decl %ecx
1160 jnz 2b
1161 mfence
1162
1163 movdqa PCPU(MDS_TMP),%xmm0
1164 testb $CR0_TS, %al
1165 je 3f
1166 movq %rax, %cr0
1167 3: popq %rcx
1168 popq %rdx
1169 popq %rax
1170 retq
1171 END(mds_handler_silvermont)
Cache object: 74e449c7daf792a69abac03c8a299f3a
|