1 /*-
2 * Copyright (c) 2018-2019 The FreeBSD Foundation
3 * Copyright (c) 2003 Peter Wemm.
4 * Copyright (c) 1993 The Regents of the University of California.
5 * All rights reserved.
6 *
7 * Portions of this software were developed by
8 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
9 * the FreeBSD Foundation.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. Neither the name of the University nor the names of its contributors
20 * may be used to endorse or promote products derived from this software
21 * without specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 * SUCH DAMAGE.
34 *
35 * $FreeBSD: stable/12/sys/amd64/amd64/support.S 368782 2020-12-19 04:18:49Z rlibby $
36 */
37
38 #include "opt_ddb.h"
39
40 #include <machine/asmacros.h>
41 #include <machine/specialreg.h>
42 #include <machine/pmap.h>
43
44 #include "assym.inc"
45
46 .text
47
48 /* Address: %rdi */
49 ENTRY(pagezero_std)
50 PUSH_FRAME_POINTER
51 movq $PAGE_SIZE/8,%rcx
52 xorl %eax,%eax
53 rep
54 stosq
55 POP_FRAME_POINTER
56 ret
57 END(pagezero_std)
58
59 ENTRY(pagezero_erms)
60 PUSH_FRAME_POINTER
61 movq $PAGE_SIZE,%rcx
62 xorl %eax,%eax
63 rep
64 stosb
65 POP_FRAME_POINTER
66 ret
67 END(pagezero_erms)
68
69 /*
70 * pagecopy(%rdi=from, %rsi=to)
71 */
72 ENTRY(pagecopy)
73 PUSH_FRAME_POINTER
74 movq $PAGE_SIZE/8,%rcx
75 movq %rdi,%r9
76 movq %rsi,%rdi
77 movq %r9,%rsi
78 rep
79 movsq
80 POP_FRAME_POINTER
81 ret
82 END(pagecopy)
83
84 /* Address: %rdi */
85 ENTRY(sse2_pagezero)
86 PUSH_FRAME_POINTER
87 movq $-PAGE_SIZE,%rdx
88 subq %rdx,%rdi
89 xorl %eax,%eax
90 jmp 1f
91 /*
92 * The loop takes 29 bytes. Ensure that it doesn't cross a 32-byte
93 * cache line.
94 */
95 .p2align 5,0x90
96 1:
97 movnti %rax,(%rdi,%rdx)
98 movnti %rax,8(%rdi,%rdx)
99 movnti %rax,16(%rdi,%rdx)
100 movnti %rax,24(%rdi,%rdx)
101 addq $32,%rdx
102 jne 1b
103 sfence
104 POP_FRAME_POINTER
105 ret
106 END(sse2_pagezero)
107
108 /*
109 * memcmpy(b1, b2, len)
110 * rdi,rsi,rdx
111 */
112 ENTRY(memcmp)
113 PUSH_FRAME_POINTER
114
115 xorl %eax,%eax
116 10:
117 cmpq $16,%rdx
118 ja 101632f
119
120 100816:
121 cmpb $8,%dl
122 jl 100408f
123 movq (%rdi),%r8
124 movq (%rsi),%r9
125 cmpq %r8,%r9
126 jne 80f
127 movq -8(%rdi,%rdx),%r8
128 movq -8(%rsi,%rdx),%r9
129 cmpq %r8,%r9
130 jne 10081608f
131 POP_FRAME_POINTER
132 ret
133 100408:
134 cmpb $4,%dl
135 jl 100204f
136 movl (%rdi),%r8d
137 movl (%rsi),%r9d
138 cmpl %r8d,%r9d
139 jne 80f
140 movl -4(%rdi,%rdx),%r8d
141 movl -4(%rsi,%rdx),%r9d
142 cmpl %r8d,%r9d
143 jne 10040804f
144 POP_FRAME_POINTER
145 ret
146 100204:
147 cmpb $2,%dl
148 jl 100001f
149 movzwl (%rdi),%r8d
150 movzwl (%rsi),%r9d
151 cmpl %r8d,%r9d
152 jne 1f
153 movzwl -2(%rdi,%rdx),%r8d
154 movzwl -2(%rsi,%rdx),%r9d
155 cmpl %r8d,%r9d
156 jne 1f
157 POP_FRAME_POINTER
158 ret
159 100001:
160 cmpb $1,%dl
161 jl 100000f
162 movzbl (%rdi),%eax
163 movzbl (%rsi),%r8d
164 subl %r8d,%eax
165 100000:
166 POP_FRAME_POINTER
167 ret
168 ALIGN_TEXT
169 101632:
170 cmpq $32,%rdx
171 ja 103200f
172 movq (%rdi),%r8
173 movq (%rsi),%r9
174 cmpq %r8,%r9
175 jne 80f
176 movq 8(%rdi),%r8
177 movq 8(%rsi),%r9
178 cmpq %r8,%r9
179 jne 10163208f
180 movq -16(%rdi,%rdx),%r8
181 movq -16(%rsi,%rdx),%r9
182 cmpq %r8,%r9
183 jne 10163216f
184 movq -8(%rdi,%rdx),%r8
185 movq -8(%rsi,%rdx),%r9
186 cmpq %r8,%r9
187 jne 10163224f
188 POP_FRAME_POINTER
189 ret
190 ALIGN_TEXT
191 103200:
192 movq (%rdi),%r8
193 movq 8(%rdi),%r9
194 subq (%rsi),%r8
195 subq 8(%rsi),%r9
196 orq %r8,%r9
197 jnz 10320000f
198
199 movq 16(%rdi),%r8
200 movq 24(%rdi),%r9
201 subq 16(%rsi),%r8
202 subq 24(%rsi),%r9
203 orq %r8,%r9
204 jnz 10320016f
205
206 leaq 32(%rdi),%rdi
207 leaq 32(%rsi),%rsi
208 subq $32,%rdx
209 cmpq $32,%rdx
210 jae 103200b
211 cmpb $0,%dl
212 jne 10b
213 POP_FRAME_POINTER
214 ret
215
216 /*
217 * Mismatch was found.
218 *
219 * Before we compute it we narrow down the range (16 -> 8 -> 4 bytes).
220 */
221 ALIGN_TEXT
222 10320016:
223 leaq 16(%rdi),%rdi
224 leaq 16(%rsi),%rsi
225 10320000:
226 movq (%rdi),%r8
227 movq (%rsi),%r9
228 cmpq %r8,%r9
229 jne 80f
230 leaq 8(%rdi),%rdi
231 leaq 8(%rsi),%rsi
232 jmp 80f
233 ALIGN_TEXT
234 10081608:
235 10163224:
236 leaq -8(%rdi,%rdx),%rdi
237 leaq -8(%rsi,%rdx),%rsi
238 jmp 80f
239 ALIGN_TEXT
240 10163216:
241 leaq -16(%rdi,%rdx),%rdi
242 leaq -16(%rsi,%rdx),%rsi
243 jmp 80f
244 ALIGN_TEXT
245 10163208:
246 leaq 8(%rdi),%rdi
247 leaq 8(%rsi),%rsi
248 jmp 80f
249 ALIGN_TEXT
250 10040804:
251 leaq -4(%rdi,%rdx),%rdi
252 leaq -4(%rsi,%rdx),%rsi
253 jmp 1f
254
255 ALIGN_TEXT
256 80:
257 movl (%rdi),%r8d
258 movl (%rsi),%r9d
259 cmpl %r8d,%r9d
260 jne 1f
261 leaq 4(%rdi),%rdi
262 leaq 4(%rsi),%rsi
263
264 /*
265 * We have up to 4 bytes to inspect.
266 */
267 1:
268 movzbl (%rdi),%eax
269 movzbl (%rsi),%r8d
270 cmpb %r8b,%al
271 jne 2f
272
273 movzbl 1(%rdi),%eax
274 movzbl 1(%rsi),%r8d
275 cmpb %r8b,%al
276 jne 2f
277
278 movzbl 2(%rdi),%eax
279 movzbl 2(%rsi),%r8d
280 cmpb %r8b,%al
281 jne 2f
282
283 movzbl 3(%rdi),%eax
284 movzbl 3(%rsi),%r8d
285 2:
286 subl %r8d,%eax
287 POP_FRAME_POINTER
288 ret
289 END(memcmp)
290
291 /*
292 * memmove(dst, src, cnt)
293 * rdi, rsi, rdx
294 */
295
296 /*
297 * Register state at entry is supposed to be as follows:
298 * rdi - destination
299 * rsi - source
300 * rdx - count
301 *
302 * The macro possibly clobbers the above and: rcx, r8, r9, 10
303 * It does not clobber rax nor r11.
304 */
305 .macro MEMMOVE erms overlap begin end
306 \begin
307
308 /*
309 * For sizes 0..32 all data is read before it is written, so there
310 * is no correctness issue with direction of copying.
311 */
312 cmpq $32,%rcx
313 jbe 101632f
314
315 .if \overlap == 1
316 movq %rdi,%r8
317 subq %rsi,%r8
318 cmpq %rcx,%r8 /* overlapping && src < dst? */
319 jb 2f
320 .endif
321
322 cmpq $256,%rcx
323 ja 1256f
324
325 103200:
326 movq (%rsi),%rdx
327 movq %rdx,(%rdi)
328 movq 8(%rsi),%rdx
329 movq %rdx,8(%rdi)
330 movq 16(%rsi),%rdx
331 movq %rdx,16(%rdi)
332 movq 24(%rsi),%rdx
333 movq %rdx,24(%rdi)
334 leaq 32(%rsi),%rsi
335 leaq 32(%rdi),%rdi
336 subq $32,%rcx
337 cmpq $32,%rcx
338 jae 103200b
339 cmpb $0,%cl
340 jne 101632f
341 \end
342 ret
343 ALIGN_TEXT
344 101632:
345 cmpb $16,%cl
346 jl 100816f
347 movq (%rsi),%rdx
348 movq 8(%rsi),%r8
349 movq -16(%rsi,%rcx),%r9
350 movq -8(%rsi,%rcx),%r10
351 movq %rdx,(%rdi)
352 movq %r8,8(%rdi)
353 movq %r9,-16(%rdi,%rcx)
354 movq %r10,-8(%rdi,%rcx)
355 \end
356 ret
357 ALIGN_TEXT
358 100816:
359 cmpb $8,%cl
360 jl 100408f
361 movq (%rsi),%rdx
362 movq -8(%rsi,%rcx),%r8
363 movq %rdx,(%rdi)
364 movq %r8,-8(%rdi,%rcx,)
365 \end
366 ret
367 ALIGN_TEXT
368 100408:
369 cmpb $4,%cl
370 jl 100204f
371 movl (%rsi),%edx
372 movl -4(%rsi,%rcx),%r8d
373 movl %edx,(%rdi)
374 movl %r8d,-4(%rdi,%rcx)
375 \end
376 ret
377 ALIGN_TEXT
378 100204:
379 cmpb $2,%cl
380 jl 100001f
381 movzwl (%rsi),%edx
382 movzwl -2(%rsi,%rcx),%r8d
383 movw %dx,(%rdi)
384 movw %r8w,-2(%rdi,%rcx)
385 \end
386 ret
387 ALIGN_TEXT
388 100001:
389 cmpb $1,%cl
390 jl 100000f
391 movb (%rsi),%dl
392 movb %dl,(%rdi)
393 100000:
394 \end
395 ret
396
397 ALIGN_TEXT
398 1256:
399 testb $15,%dil
400 jnz 100f
401 .if \erms == 1
402 rep
403 movsb
404 .else
405 shrq $3,%rcx /* copy by 64-bit words */
406 rep
407 movsq
408 movq %rdx,%rcx
409 andl $7,%ecx /* any bytes left? */
410 jne 100408b
411 .endif
412 \end
413 ret
414 100:
415 movq (%rsi),%r8
416 movq 8(%rsi),%r9
417 movq %rdi,%r10
418 movq %rdi,%rcx
419 andq $15,%rcx
420 leaq -16(%rdx,%rcx),%rdx
421 neg %rcx
422 leaq 16(%rdi,%rcx),%rdi
423 leaq 16(%rsi,%rcx),%rsi
424 movq %rdx,%rcx
425 .if \erms == 1
426 rep
427 movsb
428 movq %r8,(%r10)
429 movq %r9,8(%r10)
430 .else
431 shrq $3,%rcx /* copy by 64-bit words */
432 rep
433 movsq
434 movq %r8,(%r10)
435 movq %r9,8(%r10)
436 movq %rdx,%rcx
437 andl $7,%ecx /* any bytes left? */
438 jne 100408b
439 .endif
440 \end
441 ret
442
443 .if \overlap == 1
444 /*
445 * Copy backwards.
446 */
447 ALIGN_TEXT
448 2:
449 cmpq $256,%rcx
450 ja 2256f
451
452 leaq -8(%rdi,%rcx),%rdi
453 leaq -8(%rsi,%rcx),%rsi
454
455 cmpq $32,%rcx
456 jb 2016f
457
458 2032:
459 movq (%rsi),%rdx
460 movq %rdx,(%rdi)
461 movq -8(%rsi),%rdx
462 movq %rdx,-8(%rdi)
463 movq -16(%rsi),%rdx
464 movq %rdx,-16(%rdi)
465 movq -24(%rsi),%rdx
466 movq %rdx,-24(%rdi)
467 leaq -32(%rsi),%rsi
468 leaq -32(%rdi),%rdi
469 subq $32,%rcx
470 cmpq $32,%rcx
471 jae 2032b
472 cmpb $0,%cl
473 jne 2016f
474 \end
475 ret
476 ALIGN_TEXT
477 2016:
478 cmpb $16,%cl
479 jl 2008f
480 movq (%rsi),%rdx
481 movq %rdx,(%rdi)
482 movq -8(%rsi),%rdx
483 movq %rdx,-8(%rdi)
484 subb $16,%cl
485 jz 2000f
486 leaq -16(%rsi),%rsi
487 leaq -16(%rdi),%rdi
488 2008:
489 cmpb $8,%cl
490 jl 2004f
491 movq (%rsi),%rdx
492 movq %rdx,(%rdi)
493 subb $8,%cl
494 jz 2000f
495 leaq -8(%rsi),%rsi
496 leaq -8(%rdi),%rdi
497 2004:
498 cmpb $4,%cl
499 jl 2002f
500 movl 4(%rsi),%edx
501 movl %edx,4(%rdi)
502 subb $4,%cl
503 jz 2000f
504 leaq -4(%rsi),%rsi
505 leaq -4(%rdi),%rdi
506 2002:
507 cmpb $2,%cl
508 jl 2001f
509 movw 6(%rsi),%dx
510 movw %dx,6(%rdi)
511 subb $2,%cl
512 jz 2000f
513 leaq -2(%rsi),%rsi
514 leaq -2(%rdi),%rdi
515 2001:
516 cmpb $1,%cl
517 jl 2000f
518 movb 7(%rsi),%dl
519 movb %dl,7(%rdi)
520 2000:
521 \end
522 ret
523 ALIGN_TEXT
524 2256:
525 std
526 .if \erms == 1
527 leaq -1(%rdi,%rcx),%rdi
528 leaq -1(%rsi,%rcx),%rsi
529 rep
530 movsb
531 cld
532 .else
533 leaq -8(%rdi,%rcx),%rdi
534 leaq -8(%rsi,%rcx),%rsi
535 shrq $3,%rcx
536 rep
537 movsq
538 cld
539 movq %rdx,%rcx
540 andb $7,%cl
541 jne 2004b
542 .endif
543 \end
544 ret
545 .endif
546 .endm
547
548 .macro MEMMOVE_BEGIN
549 PUSH_FRAME_POINTER
550 movq %rdi,%rax
551 movq %rdx,%rcx
552 .endm
553
554 .macro MEMMOVE_END
555 POP_FRAME_POINTER
556 .endm
557
558 ENTRY(memmove_std)
559 MEMMOVE erms=0 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END
560 END(memmove_std)
561
562 ENTRY(memmove_erms)
563 MEMMOVE erms=1 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END
564 END(memmove_erms)
565
566 /*
567 * memcpy(dst, src, len)
568 * rdi, rsi, rdx
569 *
570 * Note: memcpy does not support overlapping copies
571 */
572 ENTRY(memcpy_std)
573 MEMMOVE erms=0 overlap=0 begin=MEMMOVE_BEGIN end=MEMMOVE_END
574 END(memcpy_std)
575
576 ENTRY(memcpy_erms)
577 MEMMOVE erms=1 overlap=0 begin=MEMMOVE_BEGIN end=MEMMOVE_END
578 END(memcpy_erms)
579
580 /*
581 * memset(dst, c, len)
582 * rdi, rsi, rdx
583 */
584 .macro MEMSET erms
585 PUSH_FRAME_POINTER
586 movq %rdi,%rax
587 movq %rdx,%rcx
588 movzbq %sil,%r8
589 movabs $0x0101010101010101,%r10
590 imulq %r8,%r10
591
592 cmpq $32,%rcx
593 jbe 101632f
594
595 cmpq $256,%rcx
596 ja 1256f
597
598 103200:
599 movq %r10,(%rdi)
600 movq %r10,8(%rdi)
601 movq %r10,16(%rdi)
602 movq %r10,24(%rdi)
603 leaq 32(%rdi),%rdi
604 subq $32,%rcx
605 cmpq $32,%rcx
606 ja 103200b
607 cmpb $16,%cl
608 ja 201632f
609 movq %r10,-16(%rdi,%rcx)
610 movq %r10,-8(%rdi,%rcx)
611 POP_FRAME_POINTER
612 ret
613 ALIGN_TEXT
614 101632:
615 cmpb $16,%cl
616 jl 100816f
617 201632:
618 movq %r10,(%rdi)
619 movq %r10,8(%rdi)
620 movq %r10,-16(%rdi,%rcx)
621 movq %r10,-8(%rdi,%rcx)
622 POP_FRAME_POINTER
623 ret
624 ALIGN_TEXT
625 100816:
626 cmpb $8,%cl
627 jl 100408f
628 movq %r10,(%rdi)
629 movq %r10,-8(%rdi,%rcx)
630 POP_FRAME_POINTER
631 ret
632 ALIGN_TEXT
633 100408:
634 cmpb $4,%cl
635 jl 100204f
636 movl %r10d,(%rdi)
637 movl %r10d,-4(%rdi,%rcx)
638 POP_FRAME_POINTER
639 ret
640 ALIGN_TEXT
641 100204:
642 cmpb $2,%cl
643 jl 100001f
644 movw %r10w,(%rdi)
645 movw %r10w,-2(%rdi,%rcx)
646 POP_FRAME_POINTER
647 ret
648 ALIGN_TEXT
649 100001:
650 cmpb $0,%cl
651 je 100000f
652 movb %r10b,(%rdi)
653 100000:
654 POP_FRAME_POINTER
655 ret
656 ALIGN_TEXT
657 1256:
658 movq %rdi,%r9
659 movq %r10,%rax
660 testl $15,%edi
661 jnz 3f
662 1:
663 .if \erms == 1
664 rep
665 stosb
666 movq %r9,%rax
667 .else
668 movq %rcx,%rdx
669 shrq $3,%rcx
670 rep
671 stosq
672 movq %r9,%rax
673 andl $7,%edx
674 jnz 2f
675 POP_FRAME_POINTER
676 ret
677 2:
678 movq %r10,-8(%rdi,%rdx)
679 .endif
680 POP_FRAME_POINTER
681 ret
682 ALIGN_TEXT
683 3:
684 movq %r10,(%rdi)
685 movq %r10,8(%rdi)
686 movq %rdi,%r8
687 andq $15,%r8
688 leaq -16(%rcx,%r8),%rcx
689 neg %r8
690 leaq 16(%rdi,%r8),%rdi
691 jmp 1b
692 .endm
693
694 ENTRY(memset_std)
695 MEMSET erms=0
696 END(memset_std)
697
698 ENTRY(memset_erms)
699 MEMSET erms=1
700 END(memset_erms)
701
702 /* fillw(pat, base, cnt) */
703 /* %rdi,%rsi, %rdx */
704 ENTRY(fillw)
705 PUSH_FRAME_POINTER
706 movq %rdi,%rax
707 movq %rsi,%rdi
708 movq %rdx,%rcx
709 rep
710 stosw
711 POP_FRAME_POINTER
712 ret
713 END(fillw)
714
715 /*****************************************************************************/
716 /* copyout and fubyte family */
717 /*****************************************************************************/
718 /*
719 * Access user memory from inside the kernel. These routines should be
720 * the only places that do this.
721 *
722 * These routines set curpcb->pcb_onfault for the time they execute. When a
723 * protection violation occurs inside the functions, the trap handler
724 * returns to *curpcb->pcb_onfault instead of the function.
725 */
726
727 .macro SMAP_DISABLE smap
728 .if \smap
729 stac
730 .endif
731 .endm
732
733
734 .macro SMAP_ENABLE smap
735 .if \smap
736 clac
737 .endif
738 .endm
739
740 .macro COPYINOUT_BEGIN
741 .endm
742
743 .macro COPYINOUT_END
744 movq %rax,PCB_ONFAULT(%r11)
745 POP_FRAME_POINTER
746 .endm
747
748 .macro COPYINOUT_SMAP_END
749 SMAP_ENABLE smap=1
750 COPYINOUT_END
751 .endm
752
753 /*
754 * copyout(from_kernel, to_user, len)
755 * %rdi, %rsi, %rdx
756 */
757 .macro COPYOUT smap erms
758 PUSH_FRAME_POINTER
759 movq PCPU(CURPCB),%r11
760 movq $copy_fault,PCB_ONFAULT(%r11)
761
762 /*
763 * Check explicitly for non-user addresses.
764 * First, prevent address wrapping.
765 */
766 movq %rsi,%rax
767 addq %rdx,%rax
768 jc copy_fault
769 /*
770 * XXX STOP USING VM_MAXUSER_ADDRESS.
771 * It is an end address, not a max, so every time it is used correctly it
772 * looks like there is an off by one error, and of course it caused an off
773 * by one error in several places.
774 */
775 movq $VM_MAXUSER_ADDRESS,%rcx
776 cmpq %rcx,%rax
777 ja copy_fault
778
779 /*
780 * Set return value to zero. Remaining failure mode goes through
781 * copy_fault.
782 */
783 xorl %eax,%eax
784
785 /*
786 * Set up arguments for MEMMOVE.
787 */
788 movq %rdi,%r8
789 movq %rsi,%rdi
790 movq %r8,%rsi
791 movq %rdx,%rcx
792
793
794 SMAP_DISABLE \smap
795 .if \smap == 1
796 MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_SMAP_END
797 .else
798 MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_END
799 .endif
800 /* NOTREACHED */
801 .endm
802
803 ENTRY(copyout_nosmap_std)
804 COPYOUT smap=0 erms=0
805 END(copyout_nosmap_std)
806
807 ENTRY(copyout_smap_std)
808 COPYOUT smap=1 erms=0
809 END(copyout_smap_std)
810
811 ENTRY(copyout_nosmap_erms)
812 COPYOUT smap=0 erms=1
813 END(copyout_nosmap_erms)
814
815 ENTRY(copyout_smap_erms)
816 COPYOUT smap=1 erms=1
817 END(copyout_smap_erms)
818
819 /*
820 * copyin(from_user, to_kernel, len)
821 * %rdi, %rsi, %rdx
822 */
823 .macro COPYIN smap erms
824 PUSH_FRAME_POINTER
825 movq PCPU(CURPCB),%r11
826 movq $copy_fault,PCB_ONFAULT(%r11)
827
828 /*
829 * make sure address is valid
830 */
831 movq %rdi,%rax
832 addq %rdx,%rax
833 jc copy_fault
834 movq $VM_MAXUSER_ADDRESS,%rcx
835 cmpq %rcx,%rax
836 ja copy_fault
837
838 xorl %eax,%eax
839
840 movq %rdi,%r8
841 movq %rsi,%rdi
842 movq %r8,%rsi
843 movq %rdx,%rcx
844
845 SMAP_DISABLE \smap
846 .if \smap == 1
847 MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_SMAP_END
848 .else
849 MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_END
850 .endif
851 /* NOTREACHED */
852 .endm
853
854 ENTRY(copyin_nosmap_std)
855 COPYIN smap=0 erms=0
856 END(copyin_nosmap_std)
857
858 ENTRY(copyin_smap_std)
859 COPYIN smap=1 erms=0
860 END(copyin_smap_std)
861
862 ENTRY(copyin_nosmap_erms)
863 COPYIN smap=0 erms=1
864 END(copyin_nosmap_erms)
865
866 ENTRY(copyin_smap_erms)
867 COPYIN smap=1 erms=1
868 END(copyin_smap_erms)
869
870 ALIGN_TEXT
871 /* Trap entry clears PSL.AC */
872 copy_fault:
873 movq $0,PCB_ONFAULT(%r11)
874 movl $EFAULT,%eax
875 POP_FRAME_POINTER
876 ret
877
878 /*
879 * casueword32. Compare and set user integer. Returns -1 on fault,
880 * 0 if access was successful. Old value is written to *oldp.
881 * dst = %rdi, old = %esi, oldp = %rdx, new = %ecx
882 */
883 ENTRY(casueword32_nosmap)
884 PUSH_FRAME_POINTER
885 movq PCPU(CURPCB),%r8
886 movq $fusufault,PCB_ONFAULT(%r8)
887
888 movq $VM_MAXUSER_ADDRESS-4,%rax
889 cmpq %rax,%rdi /* verify address is valid */
890 ja fusufault
891
892 movl %esi,%eax /* old */
893 #ifdef SMP
894 lock
895 #endif
896 cmpxchgl %ecx,(%rdi) /* new = %ecx */
897 setne %cl
898
899 /*
900 * The old value is in %eax. If the store succeeded it will be the
901 * value we expected (old) from before the store, otherwise it will
902 * be the current value. Save %eax into %esi to prepare the return
903 * value.
904 */
905 movl %eax,%esi
906 xorl %eax,%eax
907 movq %rax,PCB_ONFAULT(%r8)
908
909 /*
910 * Access the oldp after the pcb_onfault is cleared, to correctly
911 * catch corrupted pointer.
912 */
913 movl %esi,(%rdx) /* oldp = %rdx */
914 POP_FRAME_POINTER
915 movzbl %cl, %eax
916 ret
917 END(casueword32_nosmap)
918
919 ENTRY(casueword32_smap)
920 PUSH_FRAME_POINTER
921 movq PCPU(CURPCB),%r8
922 movq $fusufault,PCB_ONFAULT(%r8)
923
924 movq $VM_MAXUSER_ADDRESS-4,%rax
925 cmpq %rax,%rdi /* verify address is valid */
926 ja fusufault
927
928 movl %esi,%eax /* old */
929 stac
930 #ifdef SMP
931 lock
932 #endif
933 cmpxchgl %ecx,(%rdi) /* new = %ecx */
934 clac
935 setne %cl
936
937 /*
938 * The old value is in %eax. If the store succeeded it will be the
939 * value we expected (old) from before the store, otherwise it will
940 * be the current value. Save %eax into %esi to prepare the return
941 * value.
942 */
943 movl %eax,%esi
944 xorl %eax,%eax
945 movq %rax,PCB_ONFAULT(%r8)
946
947 /*
948 * Access the oldp after the pcb_onfault is cleared, to correctly
949 * catch corrupted pointer.
950 */
951 movl %esi,(%rdx) /* oldp = %rdx */
952 POP_FRAME_POINTER
953 movzbl %cl, %eax
954 ret
955 END(casueword32_smap)
956
957 /*
958 * casueword. Compare and set user long. Returns -1 on fault,
959 * 0 if access was successful. Old value is written to *oldp.
960 * dst = %rdi, old = %rsi, oldp = %rdx, new = %rcx
961 */
962 ENTRY(casueword_nosmap)
963 PUSH_FRAME_POINTER
964 movq PCPU(CURPCB),%r8
965 movq $fusufault,PCB_ONFAULT(%r8)
966
967 movq $VM_MAXUSER_ADDRESS-4,%rax
968 cmpq %rax,%rdi /* verify address is valid */
969 ja fusufault
970
971 movq %rsi,%rax /* old */
972 #ifdef SMP
973 lock
974 #endif
975 cmpxchgq %rcx,(%rdi) /* new = %rcx */
976 setne %cl
977
978 /*
979 * The old value is in %rax. If the store succeeded it will be the
980 * value we expected (old) from before the store, otherwise it will
981 * be the current value.
982 */
983 movq %rax,%rsi
984 xorl %eax,%eax
985 movq %rax,PCB_ONFAULT(%r8)
986 movq %rsi,(%rdx)
987 POP_FRAME_POINTER
988 movzbl %cl, %eax
989 ret
990 END(casueword_nosmap)
991
992 ENTRY(casueword_smap)
993 PUSH_FRAME_POINTER
994 movq PCPU(CURPCB),%r8
995 movq $fusufault,PCB_ONFAULT(%r8)
996
997 movq $VM_MAXUSER_ADDRESS-4,%rax
998 cmpq %rax,%rdi /* verify address is valid */
999 ja fusufault
1000
1001 movq %rsi,%rax /* old */
1002 stac
1003 #ifdef SMP
1004 lock
1005 #endif
1006 cmpxchgq %rcx,(%rdi) /* new = %rcx */
1007 clac
1008 setne %cl
1009
1010 /*
1011 * The old value is in %rax. If the store succeeded it will be the
1012 * value we expected (old) from before the store, otherwise it will
1013 * be the current value.
1014 */
1015 movq %rax,%rsi
1016 xorl %eax,%eax
1017 movq %rax,PCB_ONFAULT(%r8)
1018 movq %rsi,(%rdx)
1019 POP_FRAME_POINTER
1020 movzbl %cl, %eax
1021 ret
1022 END(casueword_smap)
1023
1024 /*
1025 * Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit
1026 * byte from user memory.
1027 * addr = %rdi, valp = %rsi
1028 */
1029
1030 ENTRY(fueword_nosmap)
1031 PUSH_FRAME_POINTER
1032 movq PCPU(CURPCB),%rcx
1033 movq $fusufault,PCB_ONFAULT(%rcx)
1034
1035 movq $VM_MAXUSER_ADDRESS-8,%rax
1036 cmpq %rax,%rdi /* verify address is valid */
1037 ja fusufault
1038
1039 xorl %eax,%eax
1040 movq (%rdi),%r11
1041 movq %rax,PCB_ONFAULT(%rcx)
1042 movq %r11,(%rsi)
1043 POP_FRAME_POINTER
1044 ret
1045 END(fueword_nosmap)
1046
1047 ENTRY(fueword_smap)
1048 PUSH_FRAME_POINTER
1049 movq PCPU(CURPCB),%rcx
1050 movq $fusufault,PCB_ONFAULT(%rcx)
1051
1052 movq $VM_MAXUSER_ADDRESS-8,%rax
1053 cmpq %rax,%rdi /* verify address is valid */
1054 ja fusufault
1055
1056 xorl %eax,%eax
1057 stac
1058 movq (%rdi),%r11
1059 clac
1060 movq %rax,PCB_ONFAULT(%rcx)
1061 movq %r11,(%rsi)
1062 POP_FRAME_POINTER
1063 ret
1064 END(fueword_smap)
1065
1066 ENTRY(fueword32_nosmap)
1067 PUSH_FRAME_POINTER
1068 movq PCPU(CURPCB),%rcx
1069 movq $fusufault,PCB_ONFAULT(%rcx)
1070
1071 movq $VM_MAXUSER_ADDRESS-4,%rax
1072 cmpq %rax,%rdi /* verify address is valid */
1073 ja fusufault
1074
1075 xorl %eax,%eax
1076 movl (%rdi),%r11d
1077 movq %rax,PCB_ONFAULT(%rcx)
1078 movl %r11d,(%rsi)
1079 POP_FRAME_POINTER
1080 ret
1081 END(fueword32_nosmap)
1082
1083 ENTRY(fueword32_smap)
1084 PUSH_FRAME_POINTER
1085 movq PCPU(CURPCB),%rcx
1086 movq $fusufault,PCB_ONFAULT(%rcx)
1087
1088 movq $VM_MAXUSER_ADDRESS-4,%rax
1089 cmpq %rax,%rdi /* verify address is valid */
1090 ja fusufault
1091
1092 xorl %eax,%eax
1093 stac
1094 movl (%rdi),%r11d
1095 clac
1096 movq %rax,PCB_ONFAULT(%rcx)
1097 movl %r11d,(%rsi)
1098 POP_FRAME_POINTER
1099 ret
1100 END(fueword32_smap)
1101
1102 ENTRY(fuword16_nosmap)
1103 PUSH_FRAME_POINTER
1104 movq PCPU(CURPCB),%rcx
1105 movq $fusufault,PCB_ONFAULT(%rcx)
1106
1107 movq $VM_MAXUSER_ADDRESS-2,%rax
1108 cmpq %rax,%rdi
1109 ja fusufault
1110
1111 movzwl (%rdi),%eax
1112 movq $0,PCB_ONFAULT(%rcx)
1113 POP_FRAME_POINTER
1114 ret
1115 END(fuword16_nosmap)
1116
1117 ENTRY(fuword16_smap)
1118 PUSH_FRAME_POINTER
1119 movq PCPU(CURPCB),%rcx
1120 movq $fusufault,PCB_ONFAULT(%rcx)
1121
1122 movq $VM_MAXUSER_ADDRESS-2,%rax
1123 cmpq %rax,%rdi
1124 ja fusufault
1125
1126 stac
1127 movzwl (%rdi),%eax
1128 clac
1129 movq $0,PCB_ONFAULT(%rcx)
1130 POP_FRAME_POINTER
1131 ret
1132 END(fuword16_smap)
1133
1134 ENTRY(fubyte_nosmap)
1135 PUSH_FRAME_POINTER
1136 movq PCPU(CURPCB),%rcx
1137 movq $fusufault,PCB_ONFAULT(%rcx)
1138
1139 movq $VM_MAXUSER_ADDRESS-1,%rax
1140 cmpq %rax,%rdi
1141 ja fusufault
1142
1143 movzbl (%rdi),%eax
1144 movq $0,PCB_ONFAULT(%rcx)
1145 POP_FRAME_POINTER
1146 ret
1147 END(fubyte_nosmap)
1148
1149 ENTRY(fubyte_smap)
1150 PUSH_FRAME_POINTER
1151 movq PCPU(CURPCB),%rcx
1152 movq $fusufault,PCB_ONFAULT(%rcx)
1153
1154 movq $VM_MAXUSER_ADDRESS-1,%rax
1155 cmpq %rax,%rdi
1156 ja fusufault
1157
1158 stac
1159 movzbl (%rdi),%eax
1160 clac
1161 movq $0,PCB_ONFAULT(%rcx)
1162 POP_FRAME_POINTER
1163 ret
1164 END(fubyte_smap)
1165
1166 /*
1167 * Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to
1168 * user memory.
1169 * addr = %rdi, value = %rsi
1170 */
1171 ENTRY(suword_nosmap)
1172 PUSH_FRAME_POINTER
1173 movq PCPU(CURPCB),%rcx
1174 movq $fusufault,PCB_ONFAULT(%rcx)
1175
1176 movq $VM_MAXUSER_ADDRESS-8,%rax
1177 cmpq %rax,%rdi /* verify address validity */
1178 ja fusufault
1179
1180 movq %rsi,(%rdi)
1181 xorl %eax,%eax
1182 movq PCPU(CURPCB),%rcx
1183 movq %rax,PCB_ONFAULT(%rcx)
1184 POP_FRAME_POINTER
1185 ret
1186 END(suword_nosmap)
1187
1188 ENTRY(suword_smap)
1189 PUSH_FRAME_POINTER
1190 movq PCPU(CURPCB),%rcx
1191 movq $fusufault,PCB_ONFAULT(%rcx)
1192
1193 movq $VM_MAXUSER_ADDRESS-8,%rax
1194 cmpq %rax,%rdi /* verify address validity */
1195 ja fusufault
1196
1197 stac
1198 movq %rsi,(%rdi)
1199 clac
1200 xorl %eax,%eax
1201 movq PCPU(CURPCB),%rcx
1202 movq %rax,PCB_ONFAULT(%rcx)
1203 POP_FRAME_POINTER
1204 ret
1205 END(suword_smap)
1206
1207 ENTRY(suword32_nosmap)
1208 PUSH_FRAME_POINTER
1209 movq PCPU(CURPCB),%rcx
1210 movq $fusufault,PCB_ONFAULT(%rcx)
1211
1212 movq $VM_MAXUSER_ADDRESS-4,%rax
1213 cmpq %rax,%rdi /* verify address validity */
1214 ja fusufault
1215
1216 movl %esi,(%rdi)
1217 xorl %eax,%eax
1218 movq PCPU(CURPCB),%rcx
1219 movq %rax,PCB_ONFAULT(%rcx)
1220 POP_FRAME_POINTER
1221 ret
1222 END(suword32_nosmap)
1223
1224 ENTRY(suword32_smap)
1225 PUSH_FRAME_POINTER
1226 movq PCPU(CURPCB),%rcx
1227 movq $fusufault,PCB_ONFAULT(%rcx)
1228
1229 movq $VM_MAXUSER_ADDRESS-4,%rax
1230 cmpq %rax,%rdi /* verify address validity */
1231 ja fusufault
1232
1233 stac
1234 movl %esi,(%rdi)
1235 clac
1236 xorl %eax,%eax
1237 movq PCPU(CURPCB),%rcx
1238 movq %rax,PCB_ONFAULT(%rcx)
1239 POP_FRAME_POINTER
1240 ret
1241 END(suword32_smap)
1242
1243 ENTRY(suword16_nosmap)
1244 PUSH_FRAME_POINTER
1245 movq PCPU(CURPCB),%rcx
1246 movq $fusufault,PCB_ONFAULT(%rcx)
1247
1248 movq $VM_MAXUSER_ADDRESS-2,%rax
1249 cmpq %rax,%rdi /* verify address validity */
1250 ja fusufault
1251
1252 movw %si,(%rdi)
1253 xorl %eax,%eax
1254 movq %rax,PCB_ONFAULT(%rcx)
1255 POP_FRAME_POINTER
1256 ret
1257 END(suword16_nosmap)
1258
1259 ENTRY(suword16_smap)
1260 PUSH_FRAME_POINTER
1261 movq PCPU(CURPCB),%rcx
1262 movq $fusufault,PCB_ONFAULT(%rcx)
1263
1264 movq $VM_MAXUSER_ADDRESS-2,%rax
1265 cmpq %rax,%rdi /* verify address validity */
1266 ja fusufault
1267
1268 stac
1269 movw %si,(%rdi)
1270 clac
1271 xorl %eax,%eax
1272 movq %rax,PCB_ONFAULT(%rcx)
1273 POP_FRAME_POINTER
1274 ret
1275 END(suword16_smap)
1276
1277 ENTRY(subyte_nosmap)
1278 PUSH_FRAME_POINTER
1279 movq PCPU(CURPCB),%rcx
1280 movq $fusufault,PCB_ONFAULT(%rcx)
1281
1282 movq $VM_MAXUSER_ADDRESS-1,%rax
1283 cmpq %rax,%rdi /* verify address validity */
1284 ja fusufault
1285
1286 movl %esi,%eax
1287 movb %al,(%rdi)
1288 xorl %eax,%eax
1289 movq %rax,PCB_ONFAULT(%rcx)
1290 POP_FRAME_POINTER
1291 ret
1292 END(subyte_nosmap)
1293
1294 ENTRY(subyte_smap)
1295 PUSH_FRAME_POINTER
1296 movq PCPU(CURPCB),%rcx
1297 movq $fusufault,PCB_ONFAULT(%rcx)
1298
1299 movq $VM_MAXUSER_ADDRESS-1,%rax
1300 cmpq %rax,%rdi /* verify address validity */
1301 ja fusufault
1302
1303 movl %esi,%eax
1304 stac
1305 movb %al,(%rdi)
1306 clac
1307 xorl %eax,%eax
1308 movq %rax,PCB_ONFAULT(%rcx)
1309 POP_FRAME_POINTER
1310 ret
1311 END(subyte_smap)
1312
1313 ALIGN_TEXT
1314 /* Fault entry clears PSL.AC */
1315 fusufault:
1316 movq PCPU(CURPCB),%rcx
1317 xorl %eax,%eax
1318 movq %rax,PCB_ONFAULT(%rcx)
1319 decq %rax
1320 POP_FRAME_POINTER
1321 ret
1322
1323 /*
1324 * copyinstr(from, to, maxlen, int *lencopied)
1325 * %rdi, %rsi, %rdx, %rcx
1326 *
1327 * copy a string from 'from' to 'to', stop when a 0 character is reached.
1328 * return ENAMETOOLONG if string is longer than maxlen, and
1329 * EFAULT on protection violations. If lencopied is non-zero,
1330 * return the actual length in *lencopied.
1331 */
1332 .macro COPYINSTR smap
1333 PUSH_FRAME_POINTER
1334 movq %rdx,%r8 /* %r8 = maxlen */
1335 movq PCPU(CURPCB),%r9
1336 movq $cpystrflt,PCB_ONFAULT(%r9)
1337
1338 movq $VM_MAXUSER_ADDRESS,%rax
1339
1340 /* make sure 'from' is within bounds */
1341 subq %rdi,%rax
1342 jbe cpystrflt
1343
1344 SMAP_DISABLE \smap
1345
1346 /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */
1347 cmpq %rdx,%rax
1348 jb 8f
1349 1:
1350 incq %rdx
1351 2:
1352 decq %rdx
1353 .if \smap == 0
1354 jz copyinstr_toolong
1355 .else
1356 jz copyinstr_toolong_smap
1357 .endif
1358
1359 movb (%rdi),%al
1360 movb %al,(%rsi)
1361 incq %rsi
1362 incq %rdi
1363 testb %al,%al
1364 jnz 2b
1365
1366 SMAP_ENABLE \smap
1367
1368 /* Success -- 0 byte reached */
1369 decq %rdx
1370 xorl %eax,%eax
1371
1372 /* set *lencopied and return %eax */
1373 movq %rax,PCB_ONFAULT(%r9)
1374
1375 testq %rcx,%rcx
1376 jz 3f
1377 subq %rdx,%r8
1378 movq %r8,(%rcx)
1379 3:
1380 POP_FRAME_POINTER
1381 ret
1382 ALIGN_TEXT
1383 8:
1384 movq %rax,%rdx
1385 movq %rax,%r8
1386 jmp 1b
1387
1388 .endm
1389
1390 ENTRY(copyinstr_nosmap)
1391 COPYINSTR smap=0
1392 END(copyinstr_nosmap)
1393
1394 ENTRY(copyinstr_smap)
1395 COPYINSTR smap=1
1396 END(copyinstr_smap)
1397
1398 cpystrflt:
1399 /* Fault entry clears PSL.AC */
1400 movl $EFAULT,%eax
1401 cpystrflt_x:
1402 /* set *lencopied and return %eax */
1403 movq $0,PCB_ONFAULT(%r9)
1404
1405 testq %rcx,%rcx
1406 jz 1f
1407 subq %rdx,%r8
1408 movq %r8,(%rcx)
1409 1:
1410 POP_FRAME_POINTER
1411 ret
1412
1413 copyinstr_toolong_smap:
1414 clac
1415 copyinstr_toolong:
1416 /* rdx is zero - return ENAMETOOLONG or EFAULT */
1417 movq $VM_MAXUSER_ADDRESS,%rax
1418 cmpq %rax,%rdi
1419 jae cpystrflt
1420 movl $ENAMETOOLONG,%eax
1421 jmp cpystrflt_x
1422
1423 /*
1424 * copystr(from, to, maxlen, int *lencopied)
1425 * %rdi, %rsi, %rdx, %rcx
1426 */
1427 ENTRY(copystr)
1428 PUSH_FRAME_POINTER
1429 movq %rdx,%r8 /* %r8 = maxlen */
1430
1431 incq %rdx
1432 1:
1433 decq %rdx
1434 jz 4f
1435 movb (%rdi),%al
1436 movb %al,(%rsi)
1437 incq %rsi
1438 incq %rdi
1439 testb %al,%al
1440 jnz 1b
1441
1442 /* Success -- 0 byte reached */
1443 decq %rdx
1444 xorl %eax,%eax
1445 2:
1446 testq %rcx,%rcx
1447 jz 3f
1448 /* set *lencopied and return %rax */
1449 subq %rdx,%r8
1450 movq %r8,(%rcx)
1451 3:
1452 POP_FRAME_POINTER
1453 ret
1454 4:
1455 /* rdx is zero -- return ENAMETOOLONG */
1456 movl $ENAMETOOLONG,%eax
1457 jmp 2b
1458 END(copystr)
1459
1460 /*
1461 * Handling of special amd64 registers and descriptor tables etc
1462 */
1463 /* void lgdt(struct region_descriptor *rdp); */
1464 ENTRY(lgdt)
1465 /* reload the descriptor table */
1466 lgdt (%rdi)
1467
1468 /* flush the prefetch q */
1469 jmp 1f
1470 nop
1471 1:
1472 movl $KDSEL,%eax
1473 movl %eax,%ds
1474 movl %eax,%es
1475 movl %eax,%fs /* Beware, use wrmsr to set 64 bit base */
1476 movl %eax,%gs
1477 movl %eax,%ss
1478
1479 /* reload code selector by turning return into intersegmental return */
1480 popq %rax
1481 pushq $KCSEL
1482 pushq %rax
1483 MEXITCOUNT
1484 lretq
1485 END(lgdt)
1486
1487 /*****************************************************************************/
1488 /* setjump, longjump */
1489 /*****************************************************************************/
1490
1491 ENTRY(setjmp)
1492 movq %rbx,0(%rdi) /* save rbx */
1493 movq %rsp,8(%rdi) /* save rsp */
1494 movq %rbp,16(%rdi) /* save rbp */
1495 movq %r12,24(%rdi) /* save r12 */
1496 movq %r13,32(%rdi) /* save r13 */
1497 movq %r14,40(%rdi) /* save r14 */
1498 movq %r15,48(%rdi) /* save r15 */
1499 movq 0(%rsp),%rdx /* get rta */
1500 movq %rdx,56(%rdi) /* save rip */
1501 xorl %eax,%eax /* return(0); */
1502 ret
1503 END(setjmp)
1504
1505 ENTRY(longjmp)
1506 movq 0(%rdi),%rbx /* restore rbx */
1507 movq 8(%rdi),%rsp /* restore rsp */
1508 movq 16(%rdi),%rbp /* restore rbp */
1509 movq 24(%rdi),%r12 /* restore r12 */
1510 movq 32(%rdi),%r13 /* restore r13 */
1511 movq 40(%rdi),%r14 /* restore r14 */
1512 movq 48(%rdi),%r15 /* restore r15 */
1513 movq 56(%rdi),%rdx /* get rta */
1514 movq %rdx,0(%rsp) /* put in return frame */
1515 xorl %eax,%eax /* return(1); */
1516 incl %eax
1517 ret
1518 END(longjmp)
1519
1520 /*
1521 * Support for reading MSRs in the safe manner. (Instead of panic on #gp,
1522 * return an error.)
1523 */
1524 ENTRY(rdmsr_safe)
1525 /* int rdmsr_safe(u_int msr, uint64_t *data) */
1526 PUSH_FRAME_POINTER
1527 movq PCPU(CURPCB),%r8
1528 movq $msr_onfault,PCB_ONFAULT(%r8)
1529 movl %edi,%ecx
1530 rdmsr /* Read MSR pointed by %ecx. Returns
1531 hi byte in edx, lo in %eax */
1532 salq $32,%rdx /* sign-shift %rdx left */
1533 movl %eax,%eax /* zero-extend %eax -> %rax */
1534 orq %rdx,%rax
1535 movq %rax,(%rsi)
1536 xorq %rax,%rax
1537 movq %rax,PCB_ONFAULT(%r8)
1538 POP_FRAME_POINTER
1539 ret
1540
1541 /*
1542 * Support for writing MSRs in the safe manner. (Instead of panic on #gp,
1543 * return an error.)
1544 */
1545 ENTRY(wrmsr_safe)
1546 /* int wrmsr_safe(u_int msr, uint64_t data) */
1547 PUSH_FRAME_POINTER
1548 movq PCPU(CURPCB),%r8
1549 movq $msr_onfault,PCB_ONFAULT(%r8)
1550 movl %edi,%ecx
1551 movl %esi,%eax
1552 sarq $32,%rsi
1553 movl %esi,%edx
1554 wrmsr /* Write MSR pointed by %ecx. Accepts
1555 hi byte in edx, lo in %eax. */
1556 xorq %rax,%rax
1557 movq %rax,PCB_ONFAULT(%r8)
1558 POP_FRAME_POINTER
1559 ret
1560
1561 /*
1562 * MSR operations fault handler
1563 */
1564 ALIGN_TEXT
1565 msr_onfault:
1566 movq $0,PCB_ONFAULT(%r8)
1567 movl $EFAULT,%eax
1568 POP_FRAME_POINTER
1569 ret
1570
1571 /*
1572 * void pmap_pti_pcid_invalidate(uint64_t ucr3, uint64_t kcr3);
1573 * Invalidates address space addressed by ucr3, then returns to kcr3.
1574 * Done in assembler to ensure no other memory accesses happen while
1575 * on ucr3.
1576 */
1577 ALIGN_TEXT
1578 ENTRY(pmap_pti_pcid_invalidate)
1579 pushfq
1580 cli
1581 movq %rdi,%cr3 /* to user page table */
1582 movq %rsi,%cr3 /* back to kernel */
1583 popfq
1584 retq
1585
1586 /*
1587 * void pmap_pti_pcid_invlpg(uint64_t ucr3, uint64_t kcr3, vm_offset_t va);
1588 * Invalidates virtual address va in address space ucr3, then returns to kcr3.
1589 */
1590 ALIGN_TEXT
1591 ENTRY(pmap_pti_pcid_invlpg)
1592 pushfq
1593 cli
1594 movq %rdi,%cr3 /* to user page table */
1595 invlpg (%rdx)
1596 movq %rsi,%cr3 /* back to kernel */
1597 popfq
1598 retq
1599
1600 /*
1601 * void pmap_pti_pcid_invlrng(uint64_t ucr3, uint64_t kcr3, vm_offset_t sva,
1602 * vm_offset_t eva);
1603 * Invalidates virtual addresses between sva and eva in address space ucr3,
1604 * then returns to kcr3.
1605 */
1606 ALIGN_TEXT
1607 ENTRY(pmap_pti_pcid_invlrng)
1608 pushfq
1609 cli
1610 movq %rdi,%cr3 /* to user page table */
1611 1: invlpg (%rdx)
1612 addq $PAGE_SIZE,%rdx
1613 cmpq %rdx,%rcx
1614 ja 1b
1615 movq %rsi,%cr3 /* back to kernel */
1616 popfq
1617 retq
1618
1619 .altmacro
1620 .macro rsb_seq_label l
1621 rsb_seq_\l:
1622 .endm
1623 .macro rsb_call_label l
1624 call rsb_seq_\l
1625 .endm
1626 .macro rsb_seq count
1627 ll=1
1628 .rept \count
1629 rsb_call_label %(ll)
1630 nop
1631 rsb_seq_label %(ll)
1632 addq $8,%rsp
1633 ll=ll+1
1634 .endr
1635 .endm
1636
1637 ENTRY(rsb_flush)
1638 rsb_seq 32
1639 ret
1640
1641 /* all callers already saved %rax, %rdx, and %rcx */
1642 ENTRY(handle_ibrs_entry)
1643 cmpb $0,hw_ibrs_ibpb_active(%rip)
1644 je 1f
1645 movl $MSR_IA32_SPEC_CTRL,%ecx
1646 rdmsr
1647 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
1648 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32,%edx
1649 wrmsr
1650 movb $1,PCPU(IBPB_SET)
1651 testl $CPUID_STDEXT_SMEP,cpu_stdext_feature(%rip)
1652 je rsb_flush
1653 1: ret
1654 END(handle_ibrs_entry)
1655
1656 ENTRY(handle_ibrs_exit)
1657 cmpb $0,PCPU(IBPB_SET)
1658 je 1f
1659 movl $MSR_IA32_SPEC_CTRL,%ecx
1660 rdmsr
1661 andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
1662 andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
1663 wrmsr
1664 movb $0,PCPU(IBPB_SET)
1665 1: ret
1666 END(handle_ibrs_exit)
1667
1668 /* registers-neutral version, but needs stack */
1669 ENTRY(handle_ibrs_exit_rs)
1670 cmpb $0,PCPU(IBPB_SET)
1671 je 1f
1672 pushq %rax
1673 pushq %rdx
1674 pushq %rcx
1675 movl $MSR_IA32_SPEC_CTRL,%ecx
1676 rdmsr
1677 andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
1678 andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
1679 wrmsr
1680 popq %rcx
1681 popq %rdx
1682 popq %rax
1683 movb $0,PCPU(IBPB_SET)
1684 1: ret
1685 END(handle_ibrs_exit_rs)
1686
1687 .noaltmacro
1688
1689 /*
1690 * Flush L1D cache. Load enough of the data from the kernel text
1691 * to flush existing L1D content.
1692 *
1693 * N.B. The function does not follow ABI calling conventions, it corrupts %rbx.
1694 * The vmm.ko caller expects that only %rax, %rdx, %rbx, %rcx, %r9, and %rflags
1695 * registers are clobbered. The NMI handler caller only needs %r13 preserved.
1696 */
1697 ENTRY(flush_l1d_sw)
1698 #define L1D_FLUSH_SIZE (64 * 1024)
1699 movq $KERNBASE, %r9
1700 movq $-L1D_FLUSH_SIZE, %rcx
1701 /*
1702 * pass 1: Preload TLB.
1703 * Kernel text is mapped using superpages. TLB preload is
1704 * done for the benefit of older CPUs which split 2M page
1705 * into 4k TLB entries.
1706 */
1707 1: movb L1D_FLUSH_SIZE(%r9, %rcx), %al
1708 addq $PAGE_SIZE, %rcx
1709 jne 1b
1710 xorl %eax, %eax
1711 cpuid
1712 movq $-L1D_FLUSH_SIZE, %rcx
1713 /* pass 2: Read each cache line. */
1714 2: movb L1D_FLUSH_SIZE(%r9, %rcx), %al
1715 addq $64, %rcx
1716 jne 2b
1717 lfence
1718 ret
1719 #undef L1D_FLUSH_SIZE
1720 END(flush_l1d_sw)
1721
1722 ENTRY(flush_l1d_sw_abi)
1723 pushq %rbx
1724 call flush_l1d_sw
1725 popq %rbx
1726 ret
1727 END(flush_l1d_sw_abi)
1728
1729 ENTRY(mds_handler_void)
1730 retq
1731 END(mds_handler_void)
1732
1733 ENTRY(mds_handler_verw)
1734 subq $8, %rsp
1735 movw %ds, (%rsp)
1736 verw (%rsp)
1737 addq $8, %rsp
1738 retq
1739 END(mds_handler_verw)
1740
1741 ENTRY(mds_handler_ivb)
1742 pushq %rax
1743 pushq %rdx
1744 pushq %rcx
1745
1746 movq %cr0, %rax
1747 testb $CR0_TS, %al
1748 je 1f
1749 clts
1750 1: movq PCPU(MDS_BUF), %rdx
1751 movdqa %xmm0, PCPU(MDS_TMP)
1752 pxor %xmm0, %xmm0
1753
1754 lfence
1755 orpd (%rdx), %xmm0
1756 orpd (%rdx), %xmm0
1757 mfence
1758 movl $40, %ecx
1759 addq $16, %rdx
1760 2: movntdq %xmm0, (%rdx)
1761 addq $16, %rdx
1762 decl %ecx
1763 jnz 2b
1764 mfence
1765
1766 movdqa PCPU(MDS_TMP),%xmm0
1767 testb $CR0_TS, %al
1768 je 3f
1769 movq %rax, %cr0
1770 3: popq %rcx
1771 popq %rdx
1772 popq %rax
1773 retq
1774 END(mds_handler_ivb)
1775
1776 ENTRY(mds_handler_bdw)
1777 pushq %rax
1778 pushq %rbx
1779 pushq %rcx
1780 pushq %rdi
1781 pushq %rsi
1782
1783 movq %cr0, %rax
1784 testb $CR0_TS, %al
1785 je 1f
1786 clts
1787 1: movq PCPU(MDS_BUF), %rbx
1788 movdqa %xmm0, PCPU(MDS_TMP)
1789 pxor %xmm0, %xmm0
1790
1791 movq %rbx, %rdi
1792 movq %rbx, %rsi
1793 movl $40, %ecx
1794 2: movntdq %xmm0, (%rbx)
1795 addq $16, %rbx
1796 decl %ecx
1797 jnz 2b
1798 mfence
1799 movl $1536, %ecx
1800 rep; movsb
1801 lfence
1802
1803 movdqa PCPU(MDS_TMP),%xmm0
1804 testb $CR0_TS, %al
1805 je 3f
1806 movq %rax, %cr0
1807 3: popq %rsi
1808 popq %rdi
1809 popq %rcx
1810 popq %rbx
1811 popq %rax
1812 retq
1813 END(mds_handler_bdw)
1814
1815 ENTRY(mds_handler_skl_sse)
1816 pushq %rax
1817 pushq %rdx
1818 pushq %rcx
1819 pushq %rdi
1820
1821 movq %cr0, %rax
1822 testb $CR0_TS, %al
1823 je 1f
1824 clts
1825 1: movq PCPU(MDS_BUF), %rdi
1826 movq PCPU(MDS_BUF64), %rdx
1827 movdqa %xmm0, PCPU(MDS_TMP)
1828 pxor %xmm0, %xmm0
1829
1830 lfence
1831 orpd (%rdx), %xmm0
1832 orpd (%rdx), %xmm0
1833 xorl %eax, %eax
1834 2: clflushopt 5376(%rdi, %rax, 8)
1835 addl $8, %eax
1836 cmpl $8 * 12, %eax
1837 jb 2b
1838 sfence
1839 movl $6144, %ecx
1840 xorl %eax, %eax
1841 rep; stosb
1842 mfence
1843
1844 movdqa PCPU(MDS_TMP), %xmm0
1845 testb $CR0_TS, %al
1846 je 3f
1847 movq %rax, %cr0
1848 3: popq %rdi
1849 popq %rcx
1850 popq %rdx
1851 popq %rax
1852 retq
1853 END(mds_handler_skl_sse)
1854
1855 ENTRY(mds_handler_skl_avx)
1856 pushq %rax
1857 pushq %rdx
1858 pushq %rcx
1859 pushq %rdi
1860
1861 movq %cr0, %rax
1862 testb $CR0_TS, %al
1863 je 1f
1864 clts
1865 1: movq PCPU(MDS_BUF), %rdi
1866 movq PCPU(MDS_BUF64), %rdx
1867 vmovdqa %ymm0, PCPU(MDS_TMP)
1868 vpxor %ymm0, %ymm0, %ymm0
1869
1870 lfence
1871 vorpd (%rdx), %ymm0, %ymm0
1872 vorpd (%rdx), %ymm0, %ymm0
1873 xorl %eax, %eax
1874 2: clflushopt 5376(%rdi, %rax, 8)
1875 addl $8, %eax
1876 cmpl $8 * 12, %eax
1877 jb 2b
1878 sfence
1879 movl $6144, %ecx
1880 xorl %eax, %eax
1881 rep; stosb
1882 mfence
1883
1884 vmovdqa PCPU(MDS_TMP), %ymm0
1885 testb $CR0_TS, %al
1886 je 3f
1887 movq %rax, %cr0
1888 3: popq %rdi
1889 popq %rcx
1890 popq %rdx
1891 popq %rax
1892 retq
1893 END(mds_handler_skl_avx)
1894
1895 ENTRY(mds_handler_skl_avx512)
1896 pushq %rax
1897 pushq %rdx
1898 pushq %rcx
1899 pushq %rdi
1900
1901 movq %cr0, %rax
1902 testb $CR0_TS, %al
1903 je 1f
1904 clts
1905 1: movq PCPU(MDS_BUF), %rdi
1906 movq PCPU(MDS_BUF64), %rdx
1907 vmovdqa64 %zmm0, PCPU(MDS_TMP)
1908 vpxord %zmm0, %zmm0, %zmm0
1909
1910 lfence
1911 vorpd (%rdx), %zmm0, %zmm0
1912 vorpd (%rdx), %zmm0, %zmm0
1913 xorl %eax, %eax
1914 2: clflushopt 5376(%rdi, %rax, 8)
1915 addl $8, %eax
1916 cmpl $8 * 12, %eax
1917 jb 2b
1918 sfence
1919 movl $6144, %ecx
1920 xorl %eax, %eax
1921 rep; stosb
1922 mfence
1923
1924 vmovdqa64 PCPU(MDS_TMP), %zmm0
1925 testb $CR0_TS, %al
1926 je 3f
1927 movq %rax, %cr0
1928 3: popq %rdi
1929 popq %rcx
1930 popq %rdx
1931 popq %rax
1932 retq
1933 END(mds_handler_skl_avx512)
1934
1935 ENTRY(mds_handler_silvermont)
1936 pushq %rax
1937 pushq %rdx
1938 pushq %rcx
1939
1940 movq %cr0, %rax
1941 testb $CR0_TS, %al
1942 je 1f
1943 clts
1944 1: movq PCPU(MDS_BUF), %rdx
1945 movdqa %xmm0, PCPU(MDS_TMP)
1946 pxor %xmm0, %xmm0
1947
1948 movl $16, %ecx
1949 2: movntdq %xmm0, (%rdx)
1950 addq $16, %rdx
1951 decl %ecx
1952 jnz 2b
1953 mfence
1954
1955 movdqa PCPU(MDS_TMP),%xmm0
1956 testb $CR0_TS, %al
1957 je 3f
1958 movq %rax, %cr0
1959 3: popq %rcx
1960 popq %rdx
1961 popq %rax
1962 retq
1963 END(mds_handler_silvermont)
Cache object: 8362f0002cfdc967c1283619e96681b0
|