1 /*-
2 * Copyright (c) 2018-2019 The FreeBSD Foundation
3 * Copyright (c) 2003 Peter Wemm.
4 * Copyright (c) 1993 The Regents of the University of California.
5 * All rights reserved.
6 *
7 * Portions of this software were developed by
8 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
9 * the FreeBSD Foundation.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. Neither the name of the University nor the names of its contributors
20 * may be used to endorse or promote products derived from this software
21 * without specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 * SUCH DAMAGE.
34 *
35 * $FreeBSD$
36 */
37
38 #include "opt_ddb.h"
39
40 #include <machine/asmacros.h>
41 #include <machine/specialreg.h>
42 #include <machine/pmap.h>
43
44 #include "assym.inc"
45
46 .text
47
48 /* Address: %rdi */
49 ENTRY(pagezero_std)
50 PUSH_FRAME_POINTER
51 movl $PAGE_SIZE/8,%ecx
52 xorl %eax,%eax
53 rep
54 stosq
55 POP_FRAME_POINTER
56 ret
57 END(pagezero_std)
58
59 ENTRY(pagezero_erms)
60 PUSH_FRAME_POINTER
61 movl $PAGE_SIZE,%ecx
62 xorl %eax,%eax
63 rep
64 stosb
65 POP_FRAME_POINTER
66 ret
67 END(pagezero_erms)
68
69 /*
70 * pagecopy(%rdi=from, %rsi=to)
71 */
72 ENTRY(pagecopy)
73 PUSH_FRAME_POINTER
74 movl $PAGE_SIZE/8,%ecx
75 movq %rdi,%r9
76 movq %rsi,%rdi
77 movq %r9,%rsi
78 rep
79 movsq
80 POP_FRAME_POINTER
81 ret
82 END(pagecopy)
83
84 /* Address: %rdi */
85 ENTRY(sse2_pagezero)
86 PUSH_FRAME_POINTER
87 movq $-PAGE_SIZE,%rdx
88 subq %rdx,%rdi
89 xorl %eax,%eax
90 jmp 1f
91 /*
92 * The loop takes 29 bytes. Ensure that it doesn't cross a 32-byte
93 * cache line.
94 */
95 .p2align 5,0x90
96 1:
97 movnti %rax,(%rdi,%rdx)
98 movnti %rax,8(%rdi,%rdx)
99 movnti %rax,16(%rdi,%rdx)
100 movnti %rax,24(%rdi,%rdx)
101 addq $32,%rdx
102 jne 1b
103 sfence
104 POP_FRAME_POINTER
105 ret
106 END(sse2_pagezero)
107
108 /*
109 * memcmp(b1, b2, len)
110 * rdi,rsi,rdx
111 */
112 ENTRY(memcmp)
113 PUSH_FRAME_POINTER
114
115 xorl %eax,%eax
116 10:
117 cmpq $16,%rdx
118 ja 101632f
119
120 cmpb $8,%dl
121 jg 100816f
122
123 cmpb $4,%dl
124 jg 100408f
125
126 cmpb $2,%dl
127 jge 100204f
128
129 cmpb $1,%dl
130 jl 100000f
131 movzbl (%rdi),%eax
132 movzbl (%rsi),%r8d
133 subl %r8d,%eax
134 100000:
135 POP_FRAME_POINTER
136 ret
137
138 ALIGN_TEXT
139 100816:
140 movq (%rdi),%r8
141 movq (%rsi),%r9
142 cmpq %r8,%r9
143 jne 80f
144 movq -8(%rdi,%rdx),%r8
145 movq -8(%rsi,%rdx),%r9
146 cmpq %r8,%r9
147 jne 10081608f
148 POP_FRAME_POINTER
149 ret
150 ALIGN_TEXT
151 100408:
152 movl (%rdi),%r8d
153 movl (%rsi),%r9d
154 cmpl %r8d,%r9d
155 jne 80f
156 movl -4(%rdi,%rdx),%r8d
157 movl -4(%rsi,%rdx),%r9d
158 cmpl %r8d,%r9d
159 jne 10040804f
160 POP_FRAME_POINTER
161 ret
162 ALIGN_TEXT
163 100204:
164 movzwl (%rdi),%r8d
165 movzwl (%rsi),%r9d
166 cmpl %r8d,%r9d
167 jne 1f
168 movzwl -2(%rdi,%rdx),%r8d
169 movzwl -2(%rsi,%rdx),%r9d
170 cmpl %r8d,%r9d
171 jne 1f
172 POP_FRAME_POINTER
173 ret
174 ALIGN_TEXT
175 101632:
176 cmpq $32,%rdx
177 ja 103200f
178 movq (%rdi),%r8
179 movq (%rsi),%r9
180 cmpq %r8,%r9
181 jne 80f
182 movq 8(%rdi),%r8
183 movq 8(%rsi),%r9
184 cmpq %r8,%r9
185 jne 10163208f
186 movq -16(%rdi,%rdx),%r8
187 movq -16(%rsi,%rdx),%r9
188 cmpq %r8,%r9
189 jne 10163216f
190 movq -8(%rdi,%rdx),%r8
191 movq -8(%rsi,%rdx),%r9
192 cmpq %r8,%r9
193 jne 10163224f
194 POP_FRAME_POINTER
195 ret
196 ALIGN_TEXT
197 103200:
198 movq (%rdi),%r8
199 movq 8(%rdi),%r9
200 subq (%rsi),%r8
201 subq 8(%rsi),%r9
202 orq %r8,%r9
203 jnz 10320000f
204
205 movq 16(%rdi),%r8
206 movq 24(%rdi),%r9
207 subq 16(%rsi),%r8
208 subq 24(%rsi),%r9
209 orq %r8,%r9
210 jnz 10320016f
211
212 leaq 32(%rdi),%rdi
213 leaq 32(%rsi),%rsi
214 subq $32,%rdx
215 cmpq $32,%rdx
216 jae 103200b
217 cmpb $0,%dl
218 jne 10b
219 POP_FRAME_POINTER
220 ret
221
222 /*
223 * Mismatch was found.
224 *
225 * Before we compute it we narrow down the range (16 -> 8 -> 4 bytes).
226 */
227 ALIGN_TEXT
228 10320016:
229 leaq 16(%rdi),%rdi
230 leaq 16(%rsi),%rsi
231 10320000:
232 movq (%rdi),%r8
233 movq (%rsi),%r9
234 cmpq %r8,%r9
235 jne 80f
236 leaq 8(%rdi),%rdi
237 leaq 8(%rsi),%rsi
238 jmp 80f
239 ALIGN_TEXT
240 10081608:
241 10163224:
242 leaq -8(%rdi,%rdx),%rdi
243 leaq -8(%rsi,%rdx),%rsi
244 jmp 80f
245 ALIGN_TEXT
246 10163216:
247 leaq -16(%rdi,%rdx),%rdi
248 leaq -16(%rsi,%rdx),%rsi
249 jmp 80f
250 ALIGN_TEXT
251 10163208:
252 leaq 8(%rdi),%rdi
253 leaq 8(%rsi),%rsi
254 jmp 80f
255 ALIGN_TEXT
256 10040804:
257 leaq -4(%rdi,%rdx),%rdi
258 leaq -4(%rsi,%rdx),%rsi
259 jmp 1f
260
261 ALIGN_TEXT
262 80:
263 movl (%rdi),%r8d
264 movl (%rsi),%r9d
265 cmpl %r8d,%r9d
266 jne 1f
267 leaq 4(%rdi),%rdi
268 leaq 4(%rsi),%rsi
269
270 /*
271 * We have up to 4 bytes to inspect.
272 */
273 1:
274 movzbl (%rdi),%eax
275 movzbl (%rsi),%r8d
276 cmpb %r8b,%al
277 jne 2f
278
279 movzbl 1(%rdi),%eax
280 movzbl 1(%rsi),%r8d
281 cmpb %r8b,%al
282 jne 2f
283
284 movzbl 2(%rdi),%eax
285 movzbl 2(%rsi),%r8d
286 cmpb %r8b,%al
287 jne 2f
288
289 movzbl 3(%rdi),%eax
290 movzbl 3(%rsi),%r8d
291 2:
292 subl %r8d,%eax
293 POP_FRAME_POINTER
294 ret
295 END(memcmp)
296
297 /*
298 * memmove(dst, src, cnt)
299 * rdi, rsi, rdx
300 */
301
302 /*
303 * Register state at entry is supposed to be as follows:
304 * rdi - destination
305 * rsi - source
306 * rdx - count
307 *
308 * The macro possibly clobbers the above and: rcx, r8, r9, r10
309 * It does not clobber rax nor r11.
310 */
311 .macro MEMMOVE erms overlap begin end
312 \begin
313
314 /*
315 * For sizes 0..32 all data is read before it is written, so there
316 * is no correctness issue with direction of copying.
317 */
318 cmpq $32,%rcx
319 jbe 101632f
320
321 .if \overlap == 1
322 movq %rdi,%r8
323 subq %rsi,%r8
324 cmpq %rcx,%r8 /* overlapping && src < dst? */
325 jb 2f
326 .endif
327
328 cmpq $256,%rcx
329 ja 1256f
330
331 ALIGN_TEXT
332 103200:
333 movq (%rsi),%rdx
334 movq %rdx,(%rdi)
335 movq 8(%rsi),%rdx
336 movq %rdx,8(%rdi)
337 movq 16(%rsi),%rdx
338 movq %rdx,16(%rdi)
339 movq 24(%rsi),%rdx
340 movq %rdx,24(%rdi)
341 leaq 32(%rsi),%rsi
342 leaq 32(%rdi),%rdi
343 subq $32,%rcx
344 cmpq $32,%rcx
345 jae 103200b
346 cmpb $0,%cl
347 jne 101632f
348 \end
349 ret
350 ALIGN_TEXT
351 101632:
352 cmpb $16,%cl
353 jl 100816f
354 movq (%rsi),%rdx
355 movq 8(%rsi),%r8
356 movq -16(%rsi,%rcx),%r9
357 movq -8(%rsi,%rcx),%r10
358 movq %rdx,(%rdi)
359 movq %r8,8(%rdi)
360 movq %r9,-16(%rdi,%rcx)
361 movq %r10,-8(%rdi,%rcx)
362 \end
363 ret
364 ALIGN_TEXT
365 100816:
366 cmpb $8,%cl
367 jl 100408f
368 movq (%rsi),%rdx
369 movq -8(%rsi,%rcx),%r8
370 movq %rdx,(%rdi)
371 movq %r8,-8(%rdi,%rcx,)
372 \end
373 ret
374 ALIGN_TEXT
375 100408:
376 cmpb $4,%cl
377 jl 100204f
378 movl (%rsi),%edx
379 movl -4(%rsi,%rcx),%r8d
380 movl %edx,(%rdi)
381 movl %r8d,-4(%rdi,%rcx)
382 \end
383 ret
384 ALIGN_TEXT
385 100204:
386 cmpb $2,%cl
387 jl 100001f
388 movzwl (%rsi),%edx
389 movzwl -2(%rsi,%rcx),%r8d
390 movw %dx,(%rdi)
391 movw %r8w,-2(%rdi,%rcx)
392 \end
393 ret
394 ALIGN_TEXT
395 100001:
396 cmpb $1,%cl
397 jl 100000f
398 movb (%rsi),%dl
399 movb %dl,(%rdi)
400 100000:
401 \end
402 ret
403
404 ALIGN_TEXT
405 1256:
406 testb $15,%dil
407 jnz 100f
408 .if \erms == 1
409 rep
410 movsb
411 .else
412 shrq $3,%rcx /* copy by 64-bit words */
413 rep
414 movsq
415 movq %rdx,%rcx
416 andl $7,%ecx /* any bytes left? */
417 jne 100408b
418 .endif
419 \end
420 ret
421 100:
422 movq (%rsi),%r8
423 movq 8(%rsi),%r9
424 movq %rdi,%r10
425 movq %rdi,%rcx
426 andq $15,%rcx
427 leaq -16(%rdx,%rcx),%rdx
428 neg %rcx
429 leaq 16(%rdi,%rcx),%rdi
430 leaq 16(%rsi,%rcx),%rsi
431 movq %rdx,%rcx
432 .if \erms == 1
433 rep
434 movsb
435 movq %r8,(%r10)
436 movq %r9,8(%r10)
437 .else
438 shrq $3,%rcx /* copy by 64-bit words */
439 rep
440 movsq
441 movq %r8,(%r10)
442 movq %r9,8(%r10)
443 movq %rdx,%rcx
444 andl $7,%ecx /* any bytes left? */
445 jne 100408b
446 .endif
447 \end
448 ret
449
450 .if \overlap == 1
451 /*
452 * Copy backwards.
453 */
454 ALIGN_TEXT
455 2:
456 cmpq $256,%rcx
457 ja 2256f
458
459 leaq -8(%rdi,%rcx),%rdi
460 leaq -8(%rsi,%rcx),%rsi
461
462 cmpq $32,%rcx
463 jb 2016f
464
465 ALIGN_TEXT
466 2032:
467 movq (%rsi),%rdx
468 movq %rdx,(%rdi)
469 movq -8(%rsi),%rdx
470 movq %rdx,-8(%rdi)
471 movq -16(%rsi),%rdx
472 movq %rdx,-16(%rdi)
473 movq -24(%rsi),%rdx
474 movq %rdx,-24(%rdi)
475 leaq -32(%rsi),%rsi
476 leaq -32(%rdi),%rdi
477 subq $32,%rcx
478 cmpq $32,%rcx
479 jae 2032b
480 cmpb $0,%cl
481 jne 2016f
482 \end
483 ret
484 ALIGN_TEXT
485 2016:
486 cmpb $16,%cl
487 jl 2008f
488 movq (%rsi),%rdx
489 movq %rdx,(%rdi)
490 movq -8(%rsi),%rdx
491 movq %rdx,-8(%rdi)
492 subb $16,%cl
493 jz 2000f
494 leaq -16(%rsi),%rsi
495 leaq -16(%rdi),%rdi
496 2008:
497 cmpb $8,%cl
498 jl 2004f
499 movq (%rsi),%rdx
500 movq %rdx,(%rdi)
501 subb $8,%cl
502 jz 2000f
503 leaq -8(%rsi),%rsi
504 leaq -8(%rdi),%rdi
505 2004:
506 cmpb $4,%cl
507 jl 2002f
508 movl 4(%rsi),%edx
509 movl %edx,4(%rdi)
510 subb $4,%cl
511 jz 2000f
512 leaq -4(%rsi),%rsi
513 leaq -4(%rdi),%rdi
514 2002:
515 cmpb $2,%cl
516 jl 2001f
517 movw 6(%rsi),%dx
518 movw %dx,6(%rdi)
519 subb $2,%cl
520 jz 2000f
521 leaq -2(%rsi),%rsi
522 leaq -2(%rdi),%rdi
523 2001:
524 cmpb $1,%cl
525 jl 2000f
526 movb 7(%rsi),%dl
527 movb %dl,7(%rdi)
528 2000:
529 \end
530 ret
531 ALIGN_TEXT
532 2256:
533 std
534 leaq -8(%rdi,%rcx),%rdi
535 leaq -8(%rsi,%rcx),%rsi
536 shrq $3,%rcx
537 rep
538 movsq
539 cld
540 movq %rdx,%rcx
541 andb $7,%cl
542 jne 2004b
543 \end
544 ret
545 .endif
546 .endm
547
548 .macro MEMMOVE_BEGIN
549 PUSH_FRAME_POINTER
550 movq %rdi,%rax
551 movq %rdx,%rcx
552 .endm
553
554 .macro MEMMOVE_END
555 POP_FRAME_POINTER
556 .endm
557
558 ENTRY(memmove_std)
559 MEMMOVE erms=0 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END
560 END(memmove_std)
561
562 ENTRY(memmove_erms)
563 MEMMOVE erms=1 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END
564 END(memmove_erms)
565
566 /*
567 * memcpy(dst, src, len)
568 * rdi, rsi, rdx
569 *
570 * Note: memcpy does not support overlapping copies
571 */
572 ENTRY(memcpy_std)
573 MEMMOVE erms=0 overlap=0 begin=MEMMOVE_BEGIN end=MEMMOVE_END
574 END(memcpy_std)
575
576 ENTRY(memcpy_erms)
577 MEMMOVE erms=1 overlap=0 begin=MEMMOVE_BEGIN end=MEMMOVE_END
578 END(memcpy_erms)
579
580 /*
581 * memset(dst, c, len)
582 * rdi, rsi, rdx
583 */
584 .macro MEMSET erms
585 PUSH_FRAME_POINTER
586 movq %rdi,%rax
587 movq %rdx,%rcx
588 movzbq %sil,%r8
589 movabs $0x0101010101010101,%r10
590 imulq %r8,%r10
591
592 cmpq $32,%rcx
593 jbe 101632f
594
595 cmpq $256,%rcx
596 ja 1256f
597
598 ALIGN_TEXT
599 103200:
600 movq %r10,(%rdi)
601 movq %r10,8(%rdi)
602 movq %r10,16(%rdi)
603 movq %r10,24(%rdi)
604 leaq 32(%rdi),%rdi
605 subq $32,%rcx
606 cmpq $32,%rcx
607 ja 103200b
608 cmpb $16,%cl
609 ja 201632f
610 movq %r10,-16(%rdi,%rcx)
611 movq %r10,-8(%rdi,%rcx)
612 POP_FRAME_POINTER
613 ret
614 ALIGN_TEXT
615 101632:
616 cmpb $16,%cl
617 jl 100816f
618 201632:
619 movq %r10,(%rdi)
620 movq %r10,8(%rdi)
621 movq %r10,-16(%rdi,%rcx)
622 movq %r10,-8(%rdi,%rcx)
623 POP_FRAME_POINTER
624 ret
625 ALIGN_TEXT
626 100816:
627 cmpb $8,%cl
628 jl 100408f
629 movq %r10,(%rdi)
630 movq %r10,-8(%rdi,%rcx)
631 POP_FRAME_POINTER
632 ret
633 ALIGN_TEXT
634 100408:
635 cmpb $4,%cl
636 jl 100204f
637 movl %r10d,(%rdi)
638 movl %r10d,-4(%rdi,%rcx)
639 POP_FRAME_POINTER
640 ret
641 ALIGN_TEXT
642 100204:
643 cmpb $2,%cl
644 jl 100001f
645 movw %r10w,(%rdi)
646 movw %r10w,-2(%rdi,%rcx)
647 POP_FRAME_POINTER
648 ret
649 ALIGN_TEXT
650 100001:
651 cmpb $0,%cl
652 je 100000f
653 movb %r10b,(%rdi)
654 100000:
655 POP_FRAME_POINTER
656 ret
657 ALIGN_TEXT
658 1256:
659 movq %rdi,%r9
660 movq %r10,%rax
661 testl $15,%edi
662 jnz 3f
663 1:
664 .if \erms == 1
665 rep
666 stosb
667 movq %r9,%rax
668 .else
669 movq %rcx,%rdx
670 shrq $3,%rcx
671 rep
672 stosq
673 movq %r9,%rax
674 andl $7,%edx
675 jnz 2f
676 POP_FRAME_POINTER
677 ret
678 2:
679 movq %r10,-8(%rdi,%rdx)
680 .endif
681 POP_FRAME_POINTER
682 ret
683 ALIGN_TEXT
684 3:
685 movq %r10,(%rdi)
686 movq %r10,8(%rdi)
687 movq %rdi,%r8
688 andq $15,%r8
689 leaq -16(%rcx,%r8),%rcx
690 neg %r8
691 leaq 16(%rdi,%r8),%rdi
692 jmp 1b
693 .endm
694
695 ENTRY(memset_std)
696 MEMSET erms=0
697 END(memset_std)
698
699 ENTRY(memset_erms)
700 MEMSET erms=1
701 END(memset_erms)
702
703 /* fillw(pat, base, cnt) */
704 /* %rdi,%rsi, %rdx */
705 ENTRY(fillw)
706 PUSH_FRAME_POINTER
707 movq %rdi,%rax
708 movq %rsi,%rdi
709 movq %rdx,%rcx
710 rep
711 stosw
712 POP_FRAME_POINTER
713 ret
714 END(fillw)
715
716 /*****************************************************************************/
717 /* copyout and fubyte family */
718 /*****************************************************************************/
719 /*
720 * Access user memory from inside the kernel. These routines should be
721 * the only places that do this.
722 *
723 * These routines set curpcb->pcb_onfault for the time they execute. When a
724 * protection violation occurs inside the functions, the trap handler
725 * returns to *curpcb->pcb_onfault instead of the function.
726 */
727
728 .macro SMAP_DISABLE smap
729 .if \smap
730 stac
731 .endif
732 .endm
733
734
735 .macro SMAP_ENABLE smap
736 .if \smap
737 clac
738 .endif
739 .endm
740
741 .macro COPYINOUT_BEGIN
742 .endm
743
744 .macro COPYINOUT_END
745 movq %rax,PCB_ONFAULT(%r11)
746 POP_FRAME_POINTER
747 .endm
748
749 .macro COPYINOUT_SMAP_END
750 SMAP_ENABLE smap=1
751 COPYINOUT_END
752 .endm
753
754 /*
755 * copyout(from_kernel, to_user, len)
756 * %rdi, %rsi, %rdx
757 */
758 .macro COPYOUT smap erms
759 PUSH_FRAME_POINTER
760 movq PCPU(CURPCB),%r11
761 movq $copy_fault,PCB_ONFAULT(%r11)
762
763 /*
764 * Check explicitly for non-user addresses.
765 * First, prevent address wrapping.
766 */
767 movq %rsi,%rax
768 addq %rdx,%rax
769 jc copy_fault
770 /*
771 * XXX STOP USING VM_MAXUSER_ADDRESS.
772 * It is an end address, not a max, so every time it is used correctly it
773 * looks like there is an off by one error, and of course it caused an off
774 * by one error in several places.
775 */
776 movq $VM_MAXUSER_ADDRESS,%rcx
777 cmpq %rcx,%rax
778 ja copy_fault
779
780 /*
781 * Set return value to zero. Remaining failure mode goes through
782 * copy_fault.
783 */
784 xorl %eax,%eax
785
786 /*
787 * Set up arguments for MEMMOVE.
788 */
789 movq %rdi,%r8
790 movq %rsi,%rdi
791 movq %r8,%rsi
792 movq %rdx,%rcx
793
794
795 SMAP_DISABLE \smap
796 .if \smap == 1
797 MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_SMAP_END
798 .else
799 MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_END
800 .endif
801 /* NOTREACHED */
802 .endm
803
804 ENTRY(copyout_nosmap_std)
805 COPYOUT smap=0 erms=0
806 END(copyout_nosmap_std)
807
808 ENTRY(copyout_smap_std)
809 COPYOUT smap=1 erms=0
810 END(copyout_smap_std)
811
812 ENTRY(copyout_nosmap_erms)
813 COPYOUT smap=0 erms=1
814 END(copyout_nosmap_erms)
815
816 ENTRY(copyout_smap_erms)
817 COPYOUT smap=1 erms=1
818 END(copyout_smap_erms)
819
820 /*
821 * copyin(from_user, to_kernel, len)
822 * %rdi, %rsi, %rdx
823 */
824 .macro COPYIN smap erms
825 PUSH_FRAME_POINTER
826 movq PCPU(CURPCB),%r11
827 movq $copy_fault,PCB_ONFAULT(%r11)
828
829 /*
830 * make sure address is valid
831 */
832 movq %rdi,%rax
833 addq %rdx,%rax
834 jc copy_fault
835 movq $VM_MAXUSER_ADDRESS,%rcx
836 cmpq %rcx,%rax
837 ja copy_fault
838
839 xorl %eax,%eax
840
841 movq %rdi,%r8
842 movq %rsi,%rdi
843 movq %r8,%rsi
844 movq %rdx,%rcx
845
846 SMAP_DISABLE \smap
847 .if \smap == 1
848 MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_SMAP_END
849 .else
850 MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_END
851 .endif
852 /* NOTREACHED */
853 .endm
854
855 ENTRY(copyin_nosmap_std)
856 COPYIN smap=0 erms=0
857 END(copyin_nosmap_std)
858
859 ENTRY(copyin_smap_std)
860 COPYIN smap=1 erms=0
861 END(copyin_smap_std)
862
863 ENTRY(copyin_nosmap_erms)
864 COPYIN smap=0 erms=1
865 END(copyin_nosmap_erms)
866
867 ENTRY(copyin_smap_erms)
868 COPYIN smap=1 erms=1
869 END(copyin_smap_erms)
870
871 ALIGN_TEXT
872 copy_fault:
873 testl $CPUID_STDEXT_SMAP,cpu_stdext_feature(%rip)
874 je 1f
875 clac
876 1: movq $0,PCB_ONFAULT(%r11)
877 movl $EFAULT,%eax
878 POP_FRAME_POINTER
879 ret
880
881 /*
882 * casueword32. Compare and set user integer. Returns -1 on fault,
883 * 0 if access was successful. Old value is written to *oldp.
884 * dst = %rdi, old = %esi, oldp = %rdx, new = %ecx
885 */
886 ENTRY(casueword32_nosmap)
887 PUSH_FRAME_POINTER
888 movq PCPU(CURPCB),%r8
889 movq $fusufault,PCB_ONFAULT(%r8)
890
891 movq $VM_MAXUSER_ADDRESS-4,%rax
892 cmpq %rax,%rdi /* verify address is valid */
893 ja fusufault
894
895 movl %esi,%eax /* old */
896 #ifdef SMP
897 lock
898 #endif
899 cmpxchgl %ecx,(%rdi) /* new = %ecx */
900 setne %cl
901
902 /*
903 * The old value is in %eax. If the store succeeded it will be the
904 * value we expected (old) from before the store, otherwise it will
905 * be the current value. Save %eax into %esi to prepare the return
906 * value.
907 */
908 movl %eax,%esi
909 xorl %eax,%eax
910 movq %rax,PCB_ONFAULT(%r8)
911
912 /*
913 * Access the oldp after the pcb_onfault is cleared, to correctly
914 * catch corrupted pointer.
915 */
916 movl %esi,(%rdx) /* oldp = %rdx */
917 POP_FRAME_POINTER
918 movzbl %cl, %eax
919 ret
920 END(casueword32_nosmap)
921
922 ENTRY(casueword32_smap)
923 PUSH_FRAME_POINTER
924 movq PCPU(CURPCB),%r8
925 movq $fusufault,PCB_ONFAULT(%r8)
926
927 movq $VM_MAXUSER_ADDRESS-4,%rax
928 cmpq %rax,%rdi /* verify address is valid */
929 ja fusufault
930
931 movl %esi,%eax /* old */
932 stac
933 #ifdef SMP
934 lock
935 #endif
936 cmpxchgl %ecx,(%rdi) /* new = %ecx */
937 clac
938 setne %cl
939
940 /*
941 * The old value is in %eax. If the store succeeded it will be the
942 * value we expected (old) from before the store, otherwise it will
943 * be the current value. Save %eax into %esi to prepare the return
944 * value.
945 */
946 movl %eax,%esi
947 xorl %eax,%eax
948 movq %rax,PCB_ONFAULT(%r8)
949
950 /*
951 * Access the oldp after the pcb_onfault is cleared, to correctly
952 * catch corrupted pointer.
953 */
954 movl %esi,(%rdx) /* oldp = %rdx */
955 POP_FRAME_POINTER
956 movzbl %cl, %eax
957 ret
958 END(casueword32_smap)
959
960 /*
961 * casueword. Compare and set user long. Returns -1 on fault,
962 * 0 if access was successful. Old value is written to *oldp.
963 * dst = %rdi, old = %rsi, oldp = %rdx, new = %rcx
964 */
965 ENTRY(casueword_nosmap)
966 PUSH_FRAME_POINTER
967 movq PCPU(CURPCB),%r8
968 movq $fusufault,PCB_ONFAULT(%r8)
969
970 movq $VM_MAXUSER_ADDRESS-4,%rax
971 cmpq %rax,%rdi /* verify address is valid */
972 ja fusufault
973
974 movq %rsi,%rax /* old */
975 #ifdef SMP
976 lock
977 #endif
978 cmpxchgq %rcx,(%rdi) /* new = %rcx */
979 setne %cl
980
981 /*
982 * The old value is in %rax. If the store succeeded it will be the
983 * value we expected (old) from before the store, otherwise it will
984 * be the current value.
985 */
986 movq %rax,%rsi
987 xorl %eax,%eax
988 movq %rax,PCB_ONFAULT(%r8)
989 movq %rsi,(%rdx)
990 POP_FRAME_POINTER
991 movzbl %cl, %eax
992 ret
993 END(casueword_nosmap)
994
995 ENTRY(casueword_smap)
996 PUSH_FRAME_POINTER
997 movq PCPU(CURPCB),%r8
998 movq $fusufault,PCB_ONFAULT(%r8)
999
1000 movq $VM_MAXUSER_ADDRESS-4,%rax
1001 cmpq %rax,%rdi /* verify address is valid */
1002 ja fusufault
1003
1004 movq %rsi,%rax /* old */
1005 stac
1006 #ifdef SMP
1007 lock
1008 #endif
1009 cmpxchgq %rcx,(%rdi) /* new = %rcx */
1010 clac
1011 setne %cl
1012
1013 /*
1014 * The old value is in %rax. If the store succeeded it will be the
1015 * value we expected (old) from before the store, otherwise it will
1016 * be the current value.
1017 */
1018 movq %rax,%rsi
1019 xorl %eax,%eax
1020 movq %rax,PCB_ONFAULT(%r8)
1021 movq %rsi,(%rdx)
1022 POP_FRAME_POINTER
1023 movzbl %cl, %eax
1024 ret
1025 END(casueword_smap)
1026
1027 /*
1028 * Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit
1029 * byte from user memory.
1030 * addr = %rdi, valp = %rsi
1031 */
1032
1033 ENTRY(fueword_nosmap)
1034 PUSH_FRAME_POINTER
1035 movq PCPU(CURPCB),%rcx
1036 movq $fusufault,PCB_ONFAULT(%rcx)
1037
1038 movq $VM_MAXUSER_ADDRESS-8,%rax
1039 cmpq %rax,%rdi /* verify address is valid */
1040 ja fusufault
1041
1042 xorl %eax,%eax
1043 movq (%rdi),%r11
1044 movq %rax,PCB_ONFAULT(%rcx)
1045 movq %r11,(%rsi)
1046 POP_FRAME_POINTER
1047 ret
1048 END(fueword_nosmap)
1049
1050 ENTRY(fueword_smap)
1051 PUSH_FRAME_POINTER
1052 movq PCPU(CURPCB),%rcx
1053 movq $fusufault,PCB_ONFAULT(%rcx)
1054
1055 movq $VM_MAXUSER_ADDRESS-8,%rax
1056 cmpq %rax,%rdi /* verify address is valid */
1057 ja fusufault
1058
1059 xorl %eax,%eax
1060 stac
1061 movq (%rdi),%r11
1062 clac
1063 movq %rax,PCB_ONFAULT(%rcx)
1064 movq %r11,(%rsi)
1065 POP_FRAME_POINTER
1066 ret
1067 END(fueword_smap)
1068
1069 ENTRY(fueword32_nosmap)
1070 PUSH_FRAME_POINTER
1071 movq PCPU(CURPCB),%rcx
1072 movq $fusufault,PCB_ONFAULT(%rcx)
1073
1074 movq $VM_MAXUSER_ADDRESS-4,%rax
1075 cmpq %rax,%rdi /* verify address is valid */
1076 ja fusufault
1077
1078 xorl %eax,%eax
1079 movl (%rdi),%r11d
1080 movq %rax,PCB_ONFAULT(%rcx)
1081 movl %r11d,(%rsi)
1082 POP_FRAME_POINTER
1083 ret
1084 END(fueword32_nosmap)
1085
1086 ENTRY(fueword32_smap)
1087 PUSH_FRAME_POINTER
1088 movq PCPU(CURPCB),%rcx
1089 movq $fusufault,PCB_ONFAULT(%rcx)
1090
1091 movq $VM_MAXUSER_ADDRESS-4,%rax
1092 cmpq %rax,%rdi /* verify address is valid */
1093 ja fusufault
1094
1095 xorl %eax,%eax
1096 stac
1097 movl (%rdi),%r11d
1098 clac
1099 movq %rax,PCB_ONFAULT(%rcx)
1100 movl %r11d,(%rsi)
1101 POP_FRAME_POINTER
1102 ret
1103 END(fueword32_smap)
1104
1105 ENTRY(fuword16_nosmap)
1106 PUSH_FRAME_POINTER
1107 movq PCPU(CURPCB),%rcx
1108 movq $fusufault,PCB_ONFAULT(%rcx)
1109
1110 movq $VM_MAXUSER_ADDRESS-2,%rax
1111 cmpq %rax,%rdi
1112 ja fusufault
1113
1114 movzwl (%rdi),%eax
1115 movq $0,PCB_ONFAULT(%rcx)
1116 POP_FRAME_POINTER
1117 ret
1118 END(fuword16_nosmap)
1119
1120 ENTRY(fuword16_smap)
1121 PUSH_FRAME_POINTER
1122 movq PCPU(CURPCB),%rcx
1123 movq $fusufault,PCB_ONFAULT(%rcx)
1124
1125 movq $VM_MAXUSER_ADDRESS-2,%rax
1126 cmpq %rax,%rdi
1127 ja fusufault
1128
1129 stac
1130 movzwl (%rdi),%eax
1131 clac
1132 movq $0,PCB_ONFAULT(%rcx)
1133 POP_FRAME_POINTER
1134 ret
1135 END(fuword16_smap)
1136
1137 ENTRY(fubyte_nosmap)
1138 PUSH_FRAME_POINTER
1139 movq PCPU(CURPCB),%rcx
1140 movq $fusufault,PCB_ONFAULT(%rcx)
1141
1142 movq $VM_MAXUSER_ADDRESS-1,%rax
1143 cmpq %rax,%rdi
1144 ja fusufault
1145
1146 movzbl (%rdi),%eax
1147 movq $0,PCB_ONFAULT(%rcx)
1148 POP_FRAME_POINTER
1149 ret
1150 END(fubyte_nosmap)
1151
1152 ENTRY(fubyte_smap)
1153 PUSH_FRAME_POINTER
1154 movq PCPU(CURPCB),%rcx
1155 movq $fusufault,PCB_ONFAULT(%rcx)
1156
1157 movq $VM_MAXUSER_ADDRESS-1,%rax
1158 cmpq %rax,%rdi
1159 ja fusufault
1160
1161 stac
1162 movzbl (%rdi),%eax
1163 clac
1164 movq $0,PCB_ONFAULT(%rcx)
1165 POP_FRAME_POINTER
1166 ret
1167 END(fubyte_smap)
1168
1169 /*
1170 * Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to
1171 * user memory.
1172 * addr = %rdi, value = %rsi
1173 */
1174 ENTRY(suword_nosmap)
1175 PUSH_FRAME_POINTER
1176 movq PCPU(CURPCB),%rcx
1177 movq $fusufault,PCB_ONFAULT(%rcx)
1178
1179 movq $VM_MAXUSER_ADDRESS-8,%rax
1180 cmpq %rax,%rdi /* verify address validity */
1181 ja fusufault
1182
1183 movq %rsi,(%rdi)
1184 xorl %eax,%eax
1185 movq %rax,PCB_ONFAULT(%rcx)
1186 POP_FRAME_POINTER
1187 ret
1188 END(suword_nosmap)
1189
1190 ENTRY(suword_smap)
1191 PUSH_FRAME_POINTER
1192 movq PCPU(CURPCB),%rcx
1193 movq $fusufault,PCB_ONFAULT(%rcx)
1194
1195 movq $VM_MAXUSER_ADDRESS-8,%rax
1196 cmpq %rax,%rdi /* verify address validity */
1197 ja fusufault
1198
1199 stac
1200 movq %rsi,(%rdi)
1201 clac
1202 xorl %eax,%eax
1203 movq %rax,PCB_ONFAULT(%rcx)
1204 POP_FRAME_POINTER
1205 ret
1206 END(suword_smap)
1207
1208 ENTRY(suword32_nosmap)
1209 PUSH_FRAME_POINTER
1210 movq PCPU(CURPCB),%rcx
1211 movq $fusufault,PCB_ONFAULT(%rcx)
1212
1213 movq $VM_MAXUSER_ADDRESS-4,%rax
1214 cmpq %rax,%rdi /* verify address validity */
1215 ja fusufault
1216
1217 movl %esi,(%rdi)
1218 xorl %eax,%eax
1219 movq %rax,PCB_ONFAULT(%rcx)
1220 POP_FRAME_POINTER
1221 ret
1222 END(suword32_nosmap)
1223
1224 ENTRY(suword32_smap)
1225 PUSH_FRAME_POINTER
1226 movq PCPU(CURPCB),%rcx
1227 movq $fusufault,PCB_ONFAULT(%rcx)
1228
1229 movq $VM_MAXUSER_ADDRESS-4,%rax
1230 cmpq %rax,%rdi /* verify address validity */
1231 ja fusufault
1232
1233 stac
1234 movl %esi,(%rdi)
1235 clac
1236 xorl %eax,%eax
1237 movq %rax,PCB_ONFAULT(%rcx)
1238 POP_FRAME_POINTER
1239 ret
1240 END(suword32_smap)
1241
1242 ENTRY(suword16_nosmap)
1243 PUSH_FRAME_POINTER
1244 movq PCPU(CURPCB),%rcx
1245 movq $fusufault,PCB_ONFAULT(%rcx)
1246
1247 movq $VM_MAXUSER_ADDRESS-2,%rax
1248 cmpq %rax,%rdi /* verify address validity */
1249 ja fusufault
1250
1251 movw %si,(%rdi)
1252 xorl %eax,%eax
1253 movq %rax,PCB_ONFAULT(%rcx)
1254 POP_FRAME_POINTER
1255 ret
1256 END(suword16_nosmap)
1257
1258 ENTRY(suword16_smap)
1259 PUSH_FRAME_POINTER
1260 movq PCPU(CURPCB),%rcx
1261 movq $fusufault,PCB_ONFAULT(%rcx)
1262
1263 movq $VM_MAXUSER_ADDRESS-2,%rax
1264 cmpq %rax,%rdi /* verify address validity */
1265 ja fusufault
1266
1267 stac
1268 movw %si,(%rdi)
1269 clac
1270 xorl %eax,%eax
1271 movq %rax,PCB_ONFAULT(%rcx)
1272 POP_FRAME_POINTER
1273 ret
1274 END(suword16_smap)
1275
1276 ENTRY(subyte_nosmap)
1277 PUSH_FRAME_POINTER
1278 movq PCPU(CURPCB),%rcx
1279 movq $fusufault,PCB_ONFAULT(%rcx)
1280
1281 movq $VM_MAXUSER_ADDRESS-1,%rax
1282 cmpq %rax,%rdi /* verify address validity */
1283 ja fusufault
1284
1285 movl %esi,%eax
1286 movb %al,(%rdi)
1287 xorl %eax,%eax
1288 movq %rax,PCB_ONFAULT(%rcx)
1289 POP_FRAME_POINTER
1290 ret
1291 END(subyte_nosmap)
1292
1293 ENTRY(subyte_smap)
1294 PUSH_FRAME_POINTER
1295 movq PCPU(CURPCB),%rcx
1296 movq $fusufault,PCB_ONFAULT(%rcx)
1297
1298 movq $VM_MAXUSER_ADDRESS-1,%rax
1299 cmpq %rax,%rdi /* verify address validity */
1300 ja fusufault
1301
1302 movl %esi,%eax
1303 stac
1304 movb %al,(%rdi)
1305 clac
1306 xorl %eax,%eax
1307 movq %rax,PCB_ONFAULT(%rcx)
1308 POP_FRAME_POINTER
1309 ret
1310 END(subyte_smap)
1311
1312 ALIGN_TEXT
1313 fusufault:
1314 testl $CPUID_STDEXT_SMAP,cpu_stdext_feature(%rip)
1315 je 1f
1316 clac
1317 1: movq PCPU(CURPCB),%rcx
1318 xorl %eax,%eax
1319 movq %rax,PCB_ONFAULT(%rcx)
1320 decq %rax
1321 POP_FRAME_POINTER
1322 ret
1323
1324 /*
1325 * copyinstr(from, to, maxlen, int *lencopied)
1326 * %rdi, %rsi, %rdx, %rcx
1327 *
1328 * copy a string from 'from' to 'to', stop when a 0 character is reached.
1329 * return ENAMETOOLONG if string is longer than maxlen, and
1330 * EFAULT on protection violations. If lencopied is non-zero,
1331 * return the actual length in *lencopied.
1332 */
1333 .macro COPYINSTR smap
1334 PUSH_FRAME_POINTER
1335 movq %rdx,%r8 /* %r8 = maxlen */
1336 movq PCPU(CURPCB),%r9
1337 movq $cpystrflt,PCB_ONFAULT(%r9)
1338
1339 movq $VM_MAXUSER_ADDRESS,%rax
1340
1341 /* make sure 'from' is within bounds */
1342 subq %rdi,%rax
1343 jbe cpystrflt
1344
1345 SMAP_DISABLE \smap
1346
1347 /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */
1348 cmpq %rdx,%rax
1349 jb 8f
1350 1:
1351 incq %rdx
1352 2:
1353 decq %rdx
1354 .if \smap == 0
1355 jz copyinstr_toolong
1356 .else
1357 jz copyinstr_toolong_smap
1358 .endif
1359
1360 movb (%rdi),%al
1361 movb %al,(%rsi)
1362 incq %rsi
1363 incq %rdi
1364 testb %al,%al
1365 jnz 2b
1366
1367 SMAP_ENABLE \smap
1368
1369 /* Success -- 0 byte reached */
1370 decq %rdx
1371 xorl %eax,%eax
1372
1373 /* set *lencopied and return %eax */
1374 movq %rax,PCB_ONFAULT(%r9)
1375
1376 testq %rcx,%rcx
1377 jz 3f
1378 subq %rdx,%r8
1379 movq %r8,(%rcx)
1380 3:
1381 POP_FRAME_POINTER
1382 ret
1383 ALIGN_TEXT
1384 8:
1385 movq %rax,%rdx
1386 movq %rax,%r8
1387 jmp 1b
1388
1389 .endm
1390
1391 ENTRY(copyinstr_nosmap)
1392 COPYINSTR smap=0
1393 END(copyinstr_nosmap)
1394
1395 ENTRY(copyinstr_smap)
1396 COPYINSTR smap=1
1397 END(copyinstr_smap)
1398
1399 cpystrflt:
1400 testl $CPUID_STDEXT_SMAP,cpu_stdext_feature(%rip)
1401 je 1f
1402 clac
1403 1: movl $EFAULT,%eax
1404 cpystrflt_x:
1405 /* set *lencopied and return %eax */
1406 movq $0,PCB_ONFAULT(%r9)
1407
1408 testq %rcx,%rcx
1409 jz 1f
1410 subq %rdx,%r8
1411 movq %r8,(%rcx)
1412 1:
1413 POP_FRAME_POINTER
1414 ret
1415
1416 copyinstr_toolong_smap:
1417 clac
1418 copyinstr_toolong:
1419 /* rdx is zero - return ENAMETOOLONG or EFAULT */
1420 movq $VM_MAXUSER_ADDRESS,%rax
1421 cmpq %rax,%rdi
1422 jae cpystrflt
1423 movl $ENAMETOOLONG,%eax
1424 jmp cpystrflt_x
1425
1426 /*
1427 * copystr(from, to, maxlen, int *lencopied)
1428 * %rdi, %rsi, %rdx, %rcx
1429 */
1430 ENTRY(copystr)
1431 PUSH_FRAME_POINTER
1432 movq %rdx,%r8 /* %r8 = maxlen */
1433
1434 incq %rdx
1435 1:
1436 decq %rdx
1437 jz 4f
1438 movb (%rdi),%al
1439 movb %al,(%rsi)
1440 incq %rsi
1441 incq %rdi
1442 testb %al,%al
1443 jnz 1b
1444
1445 /* Success -- 0 byte reached */
1446 decq %rdx
1447 xorl %eax,%eax
1448 2:
1449 testq %rcx,%rcx
1450 jz 3f
1451 /* set *lencopied and return %rax */
1452 subq %rdx,%r8
1453 movq %r8,(%rcx)
1454 3:
1455 POP_FRAME_POINTER
1456 ret
1457 4:
1458 /* rdx is zero -- return ENAMETOOLONG */
1459 movl $ENAMETOOLONG,%eax
1460 jmp 2b
1461 END(copystr)
1462
1463 /*
1464 * Handling of special amd64 registers and descriptor tables etc
1465 */
1466 /* void lgdt(struct region_descriptor *rdp); */
1467 ENTRY(lgdt)
1468 /* reload the descriptor table */
1469 lgdt (%rdi)
1470
1471 /* flush the prefetch q */
1472 jmp 1f
1473 nop
1474 1:
1475 movl $KDSEL,%eax
1476 movl %eax,%ds
1477 movl %eax,%es
1478 movl %eax,%fs /* Beware, use wrmsr to set 64 bit base */
1479 movl %eax,%gs
1480 movl %eax,%ss
1481
1482 /* reload code selector by turning return into intersegmental return */
1483 popq %rax
1484 pushq $KCSEL
1485 pushq %rax
1486 MEXITCOUNT
1487 lretq
1488 END(lgdt)
1489
1490 /*****************************************************************************/
1491 /* setjump, longjump */
1492 /*****************************************************************************/
1493
1494 ENTRY(setjmp)
1495 movq %rbx,0(%rdi) /* save rbx */
1496 movq %rsp,8(%rdi) /* save rsp */
1497 movq %rbp,16(%rdi) /* save rbp */
1498 movq %r12,24(%rdi) /* save r12 */
1499 movq %r13,32(%rdi) /* save r13 */
1500 movq %r14,40(%rdi) /* save r14 */
1501 movq %r15,48(%rdi) /* save r15 */
1502 movq 0(%rsp),%rdx /* get rta */
1503 movq %rdx,56(%rdi) /* save rip */
1504 xorl %eax,%eax /* return(0); */
1505 ret
1506 END(setjmp)
1507
1508 ENTRY(longjmp)
1509 movq 0(%rdi),%rbx /* restore rbx */
1510 movq 8(%rdi),%rsp /* restore rsp */
1511 movq 16(%rdi),%rbp /* restore rbp */
1512 movq 24(%rdi),%r12 /* restore r12 */
1513 movq 32(%rdi),%r13 /* restore r13 */
1514 movq 40(%rdi),%r14 /* restore r14 */
1515 movq 48(%rdi),%r15 /* restore r15 */
1516 movq 56(%rdi),%rdx /* get rta */
1517 movq %rdx,0(%rsp) /* put in return frame */
1518 xorl %eax,%eax /* return(1); */
1519 incl %eax
1520 ret
1521 END(longjmp)
1522
1523 /*
1524 * Support for reading MSRs in the safe manner. (Instead of panic on #gp,
1525 * return an error.)
1526 */
1527 ENTRY(rdmsr_safe)
1528 /* int rdmsr_safe(u_int msr, uint64_t *data) */
1529 PUSH_FRAME_POINTER
1530 movq PCPU(CURPCB),%r8
1531 movq $msr_onfault,PCB_ONFAULT(%r8)
1532 movl %edi,%ecx
1533 rdmsr /* Read MSR pointed by %ecx. Returns
1534 hi byte in edx, lo in %eax */
1535 salq $32,%rdx /* sign-shift %rdx left */
1536 movl %eax,%eax /* zero-extend %eax -> %rax */
1537 orq %rdx,%rax
1538 movq %rax,(%rsi)
1539 xorq %rax,%rax
1540 movq %rax,PCB_ONFAULT(%r8)
1541 POP_FRAME_POINTER
1542 ret
1543
1544 /*
1545 * Support for writing MSRs in the safe manner. (Instead of panic on #gp,
1546 * return an error.)
1547 */
1548 ENTRY(wrmsr_safe)
1549 /* int wrmsr_safe(u_int msr, uint64_t data) */
1550 PUSH_FRAME_POINTER
1551 movq PCPU(CURPCB),%r8
1552 movq $msr_onfault,PCB_ONFAULT(%r8)
1553 movl %edi,%ecx
1554 movl %esi,%eax
1555 sarq $32,%rsi
1556 movl %esi,%edx
1557 wrmsr /* Write MSR pointed by %ecx. Accepts
1558 hi byte in edx, lo in %eax. */
1559 xorq %rax,%rax
1560 movq %rax,PCB_ONFAULT(%r8)
1561 POP_FRAME_POINTER
1562 ret
1563
1564 /*
1565 * MSR operations fault handler
1566 */
1567 ALIGN_TEXT
1568 msr_onfault:
1569 movq $0,PCB_ONFAULT(%r8)
1570 movl $EFAULT,%eax
1571 POP_FRAME_POINTER
1572 ret
1573
1574 /*
1575 * void pmap_pti_pcid_invalidate(uint64_t ucr3, uint64_t kcr3);
1576 * Invalidates address space addressed by ucr3, then returns to kcr3.
1577 * Done in assembler to ensure no other memory accesses happen while
1578 * on ucr3.
1579 */
1580 ALIGN_TEXT
1581 ENTRY(pmap_pti_pcid_invalidate)
1582 pushfq
1583 cli
1584 movq %rdi,%cr3 /* to user page table */
1585 movq %rsi,%cr3 /* back to kernel */
1586 popfq
1587 retq
1588
1589 /*
1590 * void pmap_pti_pcid_invlpg(uint64_t ucr3, uint64_t kcr3, vm_offset_t va);
1591 * Invalidates virtual address va in address space ucr3, then returns to kcr3.
1592 */
1593 ALIGN_TEXT
1594 ENTRY(pmap_pti_pcid_invlpg)
1595 pushfq
1596 cli
1597 movq %rdi,%cr3 /* to user page table */
1598 invlpg (%rdx)
1599 movq %rsi,%cr3 /* back to kernel */
1600 popfq
1601 retq
1602
1603 /*
1604 * void pmap_pti_pcid_invlrng(uint64_t ucr3, uint64_t kcr3, vm_offset_t sva,
1605 * vm_offset_t eva);
1606 * Invalidates virtual addresses between sva and eva in address space ucr3,
1607 * then returns to kcr3.
1608 */
1609 ALIGN_TEXT
1610 ENTRY(pmap_pti_pcid_invlrng)
1611 pushfq
1612 cli
1613 movq %rdi,%cr3 /* to user page table */
1614 1: invlpg (%rdx)
1615 addq $PAGE_SIZE,%rdx
1616 cmpq %rdx,%rcx
1617 ja 1b
1618 movq %rsi,%cr3 /* back to kernel */
1619 popfq
1620 retq
1621
1622 .altmacro
1623 .macro rsb_seq_label l
1624 rsb_seq_\l:
1625 .endm
1626 .macro rsb_call_label l
1627 call rsb_seq_\l
1628 .endm
1629 .macro rsb_seq count
1630 ll=1
1631 .rept \count
1632 rsb_call_label %(ll)
1633 nop
1634 rsb_seq_label %(ll)
1635 addq $8,%rsp
1636 ll=ll+1
1637 .endr
1638 .endm
1639
1640 ENTRY(rsb_flush)
1641 rsb_seq 32
1642 ret
1643
1644 /* all callers already saved %rax, %rdx, and %rcx */
1645 ENTRY(handle_ibrs_entry)
1646 cmpb $0,hw_ibrs_ibpb_active(%rip)
1647 je 1f
1648 movl $MSR_IA32_SPEC_CTRL,%ecx
1649 rdmsr
1650 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
1651 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32,%edx
1652 wrmsr
1653 movb $1,PCPU(IBPB_SET)
1654 testl $CPUID_STDEXT_SMEP,cpu_stdext_feature(%rip)
1655 je rsb_flush
1656 1: ret
1657 END(handle_ibrs_entry)
1658
1659 ENTRY(handle_ibrs_exit)
1660 cmpb $0,PCPU(IBPB_SET)
1661 je 1f
1662 movl $MSR_IA32_SPEC_CTRL,%ecx
1663 rdmsr
1664 andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
1665 andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
1666 wrmsr
1667 movb $0,PCPU(IBPB_SET)
1668 1: ret
1669 END(handle_ibrs_exit)
1670
1671 /* registers-neutral version, but needs stack */
1672 ENTRY(handle_ibrs_exit_rs)
1673 cmpb $0,PCPU(IBPB_SET)
1674 je 1f
1675 pushq %rax
1676 pushq %rdx
1677 pushq %rcx
1678 movl $MSR_IA32_SPEC_CTRL,%ecx
1679 rdmsr
1680 andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
1681 andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
1682 wrmsr
1683 popq %rcx
1684 popq %rdx
1685 popq %rax
1686 movb $0,PCPU(IBPB_SET)
1687 1: ret
1688 END(handle_ibrs_exit_rs)
1689
1690 .noaltmacro
1691
1692 /*
1693 * Flush L1D cache. Load enough of the data from the kernel text
1694 * to flush existing L1D content.
1695 *
1696 * N.B. The function does not follow ABI calling conventions, it corrupts %rbx.
1697 * The vmm.ko caller expects that only %rax, %rdx, %rbx, %rcx, %r9, and %rflags
1698 * registers are clobbered. The NMI handler caller only needs %r13 preserved.
1699 */
1700 ENTRY(flush_l1d_sw)
1701 #define L1D_FLUSH_SIZE (64 * 1024)
1702 movq $KERNBASE, %r9
1703 movq $-L1D_FLUSH_SIZE, %rcx
1704 /*
1705 * pass 1: Preload TLB.
1706 * Kernel text is mapped using superpages. TLB preload is
1707 * done for the benefit of older CPUs which split 2M page
1708 * into 4k TLB entries.
1709 */
1710 1: movb L1D_FLUSH_SIZE(%r9, %rcx), %al
1711 addq $PAGE_SIZE, %rcx
1712 jne 1b
1713 xorl %eax, %eax
1714 cpuid
1715 movq $-L1D_FLUSH_SIZE, %rcx
1716 /* pass 2: Read each cache line. */
1717 2: movb L1D_FLUSH_SIZE(%r9, %rcx), %al
1718 addq $64, %rcx
1719 jne 2b
1720 lfence
1721 ret
1722 #undef L1D_FLUSH_SIZE
1723 END(flush_l1d_sw)
1724
1725 ENTRY(flush_l1d_sw_abi)
1726 pushq %rbx
1727 call flush_l1d_sw
1728 popq %rbx
1729 ret
1730 END(flush_l1d_sw_abi)
1731
1732 ENTRY(mds_handler_void)
1733 retq
1734 END(mds_handler_void)
1735
1736 ENTRY(mds_handler_verw)
1737 subq $8, %rsp
1738 movw %ds, (%rsp)
1739 verw (%rsp)
1740 addq $8, %rsp
1741 retq
1742 END(mds_handler_verw)
1743
1744 ENTRY(mds_handler_ivb)
1745 pushq %rax
1746 pushq %rdx
1747 pushq %rcx
1748
1749 movq %cr0, %rax
1750 testb $CR0_TS, %al
1751 je 1f
1752 clts
1753 1: movq PCPU(MDS_BUF), %rdx
1754 movdqa %xmm0, PCPU(MDS_TMP)
1755 pxor %xmm0, %xmm0
1756
1757 lfence
1758 orpd (%rdx), %xmm0
1759 orpd (%rdx), %xmm0
1760 mfence
1761 movl $40, %ecx
1762 addq $16, %rdx
1763 2: movntdq %xmm0, (%rdx)
1764 addq $16, %rdx
1765 decl %ecx
1766 jnz 2b
1767 mfence
1768
1769 movdqa PCPU(MDS_TMP),%xmm0
1770 testb $CR0_TS, %al
1771 je 3f
1772 movq %rax, %cr0
1773 3: popq %rcx
1774 popq %rdx
1775 popq %rax
1776 retq
1777 END(mds_handler_ivb)
1778
1779 ENTRY(mds_handler_bdw)
1780 pushq %rax
1781 pushq %rbx
1782 pushq %rcx
1783 pushq %rdi
1784 pushq %rsi
1785
1786 movq %cr0, %rax
1787 testb $CR0_TS, %al
1788 je 1f
1789 clts
1790 1: movq PCPU(MDS_BUF), %rbx
1791 movdqa %xmm0, PCPU(MDS_TMP)
1792 pxor %xmm0, %xmm0
1793
1794 movq %rbx, %rdi
1795 movq %rbx, %rsi
1796 movl $40, %ecx
1797 2: movntdq %xmm0, (%rbx)
1798 addq $16, %rbx
1799 decl %ecx
1800 jnz 2b
1801 mfence
1802 movl $1536, %ecx
1803 rep; movsb
1804 lfence
1805
1806 movdqa PCPU(MDS_TMP),%xmm0
1807 testb $CR0_TS, %al
1808 je 3f
1809 movq %rax, %cr0
1810 3: popq %rsi
1811 popq %rdi
1812 popq %rcx
1813 popq %rbx
1814 popq %rax
1815 retq
1816 END(mds_handler_bdw)
1817
1818 ENTRY(mds_handler_skl_sse)
1819 pushq %rax
1820 pushq %rdx
1821 pushq %rcx
1822 pushq %rdi
1823
1824 movq %cr0, %rax
1825 testb $CR0_TS, %al
1826 je 1f
1827 clts
1828 1: movq PCPU(MDS_BUF), %rdi
1829 movq PCPU(MDS_BUF64), %rdx
1830 movdqa %xmm0, PCPU(MDS_TMP)
1831 pxor %xmm0, %xmm0
1832
1833 lfence
1834 orpd (%rdx), %xmm0
1835 orpd (%rdx), %xmm0
1836 xorl %eax, %eax
1837 2: clflushopt 5376(%rdi, %rax, 8)
1838 addl $8, %eax
1839 cmpl $8 * 12, %eax
1840 jb 2b
1841 sfence
1842 movl $6144, %ecx
1843 xorl %eax, %eax
1844 rep; stosb
1845 mfence
1846
1847 movdqa PCPU(MDS_TMP), %xmm0
1848 testb $CR0_TS, %al
1849 je 3f
1850 movq %rax, %cr0
1851 3: popq %rdi
1852 popq %rcx
1853 popq %rdx
1854 popq %rax
1855 retq
1856 END(mds_handler_skl_sse)
1857
1858 ENTRY(mds_handler_skl_avx)
1859 pushq %rax
1860 pushq %rdx
1861 pushq %rcx
1862 pushq %rdi
1863
1864 movq %cr0, %rax
1865 testb $CR0_TS, %al
1866 je 1f
1867 clts
1868 1: movq PCPU(MDS_BUF), %rdi
1869 movq PCPU(MDS_BUF64), %rdx
1870 vmovdqa %ymm0, PCPU(MDS_TMP)
1871 vpxor %ymm0, %ymm0, %ymm0
1872
1873 lfence
1874 vorpd (%rdx), %ymm0, %ymm0
1875 vorpd (%rdx), %ymm0, %ymm0
1876 xorl %eax, %eax
1877 2: clflushopt 5376(%rdi, %rax, 8)
1878 addl $8, %eax
1879 cmpl $8 * 12, %eax
1880 jb 2b
1881 sfence
1882 movl $6144, %ecx
1883 xorl %eax, %eax
1884 rep; stosb
1885 mfence
1886
1887 vmovdqa PCPU(MDS_TMP), %ymm0
1888 testb $CR0_TS, %al
1889 je 3f
1890 movq %rax, %cr0
1891 3: popq %rdi
1892 popq %rcx
1893 popq %rdx
1894 popq %rax
1895 retq
1896 END(mds_handler_skl_avx)
1897
1898 ENTRY(mds_handler_skl_avx512)
1899 pushq %rax
1900 pushq %rdx
1901 pushq %rcx
1902 pushq %rdi
1903
1904 movq %cr0, %rax
1905 testb $CR0_TS, %al
1906 je 1f
1907 clts
1908 1: movq PCPU(MDS_BUF), %rdi
1909 movq PCPU(MDS_BUF64), %rdx
1910 vmovdqa64 %zmm0, PCPU(MDS_TMP)
1911 vpxord %zmm0, %zmm0, %zmm0
1912
1913 lfence
1914 vorpd (%rdx), %zmm0, %zmm0
1915 vorpd (%rdx), %zmm0, %zmm0
1916 xorl %eax, %eax
1917 2: clflushopt 5376(%rdi, %rax, 8)
1918 addl $8, %eax
1919 cmpl $8 * 12, %eax
1920 jb 2b
1921 sfence
1922 movl $6144, %ecx
1923 xorl %eax, %eax
1924 rep; stosb
1925 mfence
1926
1927 vmovdqa64 PCPU(MDS_TMP), %zmm0
1928 testb $CR0_TS, %al
1929 je 3f
1930 movq %rax, %cr0
1931 3: popq %rdi
1932 popq %rcx
1933 popq %rdx
1934 popq %rax
1935 retq
1936 END(mds_handler_skl_avx512)
1937
1938 ENTRY(mds_handler_silvermont)
1939 pushq %rax
1940 pushq %rdx
1941 pushq %rcx
1942
1943 movq %cr0, %rax
1944 testb $CR0_TS, %al
1945 je 1f
1946 clts
1947 1: movq PCPU(MDS_BUF), %rdx
1948 movdqa %xmm0, PCPU(MDS_TMP)
1949 pxor %xmm0, %xmm0
1950
1951 movl $16, %ecx
1952 2: movntdq %xmm0, (%rdx)
1953 addq $16, %rdx
1954 decl %ecx
1955 jnz 2b
1956 mfence
1957
1958 movdqa PCPU(MDS_TMP),%xmm0
1959 testb $CR0_TS, %al
1960 je 3f
1961 movq %rax, %cr0
1962 3: popq %rcx
1963 popq %rdx
1964 popq %rax
1965 retq
1966 END(mds_handler_silvermont)
Cache object: 8362f0002cfdc967c1283619e96681b0
|