Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/support.S
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 2018-2019 The FreeBSD Foundation 3 * Copyright (c) 2003 Peter Wemm. 4 * Copyright (c) 1993 The Regents of the University of California. 5 * All rights reserved. 6 * 7 * Portions of this software were developed by 8 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from 9 * the FreeBSD Foundation. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 4. Neither the name of the University nor the names of its contributors 20 * may be used to endorse or promote products derived from this software 21 * without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 * SUCH DAMAGE. 34 * 35 * $FreeBSD$ 36 */ 37 38 #include "opt_ddb.h" 39 40 #include <machine/asmacros.h> 41 #include <machine/specialreg.h> 42 #include <machine/pmap.h> 43 44 #include "assym.s" 45 46 .text 47 48 /* 49 * bcopy family 50 * void bzero(void *buf, u_int len) 51 */ 52 53 /* done */ 54 ENTRY(bzero) 55 PUSH_FRAME_POINTER 56 movq %rsi,%rcx 57 xorl %eax,%eax 58 shrq $3,%rcx 59 rep 60 stosq 61 movq %rsi,%rcx 62 andq $7,%rcx 63 rep 64 stosb 65 POP_FRAME_POINTER 66 ret 67 END(bzero) 68 69 /* Address: %rdi */ 70 ENTRY(pagezero) 71 PUSH_FRAME_POINTER 72 movq $PAGE_SIZE/8,%rcx 73 xorl %eax,%eax 74 rep 75 stosq 76 POP_FRAME_POINTER 77 ret 78 END(pagezero) 79 80 ENTRY(bcmp) 81 PUSH_FRAME_POINTER 82 movq %rdx,%rcx 83 shrq $3,%rcx 84 repe 85 cmpsq 86 jne 1f 87 88 movq %rdx,%rcx 89 andq $7,%rcx 90 repe 91 cmpsb 92 1: 93 setne %al 94 movsbl %al,%eax 95 POP_FRAME_POINTER 96 ret 97 END(bcmp) 98 99 /* 100 * bcopy(src, dst, cnt) 101 * rdi, rsi, rdx 102 * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800 103 */ 104 ENTRY(bcopy) 105 PUSH_FRAME_POINTER 106 xchgq %rsi,%rdi 107 movq %rdx,%rcx 108 109 movq %rdi,%rax 110 subq %rsi,%rax 111 cmpq %rcx,%rax /* overlapping && src < dst? */ 112 jb 1f 113 114 shrq $3,%rcx /* copy by 64-bit words */ 115 rep 116 movsq 117 movq %rdx,%rcx 118 andq $7,%rcx /* any bytes left? */ 119 rep 120 movsb 121 POP_FRAME_POINTER 122 ret 123 124 /* ALIGN_TEXT */ 125 1: 126 addq %rcx,%rdi /* copy backwards */ 127 addq %rcx,%rsi 128 decq %rdi 129 decq %rsi 130 andq $7,%rcx /* any fractional bytes? */ 131 std 132 rep 133 movsb 134 movq %rdx,%rcx /* copy remainder by 32-bit words */ 135 shrq $3,%rcx 136 subq $7,%rsi 137 subq $7,%rdi 138 rep 139 movsq 140 cld 141 POP_FRAME_POINTER 142 ret 143 END(bcopy) 144 145 /* 146 * Note: memcpy does not support overlapping copies 147 */ 148 ENTRY(memcpy) 149 PUSH_FRAME_POINTER 150 movq %rdi,%rax 151 movq %rdx,%rcx 152 shrq $3,%rcx /* copy by 64-bit words */ 153 rep 154 movsq 155 movq %rdx,%rcx 156 andq $7,%rcx /* any bytes left? */ 157 rep 158 movsb 159 POP_FRAME_POINTER 160 ret 161 END(memcpy) 162 163 /* 164 * pagecopy(%rdi=from, %rsi=to) 165 */ 166 ENTRY(pagecopy) 167 PUSH_FRAME_POINTER 168 movq $-PAGE_SIZE,%rax 169 movq %rax,%rdx 170 subq %rax,%rdi 171 subq %rax,%rsi 172 1: 173 prefetchnta (%rdi,%rax) 174 addq $64,%rax 175 jne 1b 176 2: 177 movq (%rdi,%rdx),%rax 178 movnti %rax,(%rsi,%rdx) 179 movq 8(%rdi,%rdx),%rax 180 movnti %rax,8(%rsi,%rdx) 181 movq 16(%rdi,%rdx),%rax 182 movnti %rax,16(%rsi,%rdx) 183 movq 24(%rdi,%rdx),%rax 184 movnti %rax,24(%rsi,%rdx) 185 addq $32,%rdx 186 jne 2b 187 sfence 188 POP_FRAME_POINTER 189 ret 190 END(pagecopy) 191 192 /* fillw(pat, base, cnt) */ 193 /* %rdi,%rsi, %rdx */ 194 ENTRY(fillw) 195 PUSH_FRAME_POINTER 196 movq %rdi,%rax 197 movq %rsi,%rdi 198 movq %rdx,%rcx 199 rep 200 stosw 201 POP_FRAME_POINTER 202 ret 203 END(fillw) 204 205 /*****************************************************************************/ 206 /* copyout and fubyte family */ 207 /*****************************************************************************/ 208 /* 209 * Access user memory from inside the kernel. These routines should be 210 * the only places that do this. 211 * 212 * These routines set curpcb->pcb_onfault for the time they execute. When a 213 * protection violation occurs inside the functions, the trap handler 214 * returns to *curpcb->pcb_onfault instead of the function. 215 */ 216 217 /* 218 * copyout(from_kernel, to_user, len) 219 * %rdi, %rsi, %rdx 220 */ 221 ENTRY(copyout) 222 PUSH_FRAME_POINTER 223 movq PCPU(CURPCB),%rax 224 movq $copyout_fault,PCB_ONFAULT(%rax) 225 testq %rdx,%rdx /* anything to do? */ 226 jz done_copyout 227 228 /* 229 * Check explicitly for non-user addresses. This check is essential 230 * because it prevents usermode from writing into the kernel. We do 231 * not verify anywhere else that the user did not specify a rogue 232 * address. 233 */ 234 /* 235 * First, prevent address wrapping. 236 */ 237 movq %rsi,%rax 238 addq %rdx,%rax 239 jc copyout_fault 240 /* 241 * XXX STOP USING VM_MAXUSER_ADDRESS. 242 * It is an end address, not a max, so every time it is used correctly it 243 * looks like there is an off by one error, and of course it caused an off 244 * by one error in several places. 245 */ 246 movq $VM_MAXUSER_ADDRESS,%rcx 247 cmpq %rcx,%rax 248 ja copyout_fault 249 250 xchgq %rdi,%rsi 251 /* bcopy(%rsi, %rdi, %rdx) */ 252 movq %rdx,%rcx 253 254 shrq $3,%rcx 255 rep 256 movsq 257 movb %dl,%cl 258 andb $7,%cl 259 rep 260 movsb 261 262 done_copyout: 263 xorl %eax,%eax 264 movq PCPU(CURPCB),%rdx 265 movq %rax,PCB_ONFAULT(%rdx) 266 POP_FRAME_POINTER 267 ret 268 269 ALIGN_TEXT 270 copyout_fault: 271 movq PCPU(CURPCB),%rdx 272 movq $0,PCB_ONFAULT(%rdx) 273 movq $EFAULT,%rax 274 POP_FRAME_POINTER 275 ret 276 END(copyout) 277 278 /* 279 * copyin(from_user, to_kernel, len) 280 * %rdi, %rsi, %rdx 281 */ 282 ENTRY(copyin) 283 PUSH_FRAME_POINTER 284 movq PCPU(CURPCB),%rax 285 movq $copyin_fault,PCB_ONFAULT(%rax) 286 testq %rdx,%rdx /* anything to do? */ 287 jz done_copyin 288 289 /* 290 * make sure address is valid 291 */ 292 movq %rdi,%rax 293 addq %rdx,%rax 294 jc copyin_fault 295 movq $VM_MAXUSER_ADDRESS,%rcx 296 cmpq %rcx,%rax 297 ja copyin_fault 298 299 xchgq %rdi,%rsi 300 movq %rdx,%rcx 301 movb %cl,%al 302 shrq $3,%rcx /* copy longword-wise */ 303 rep 304 movsq 305 movb %al,%cl 306 andb $7,%cl /* copy remaining bytes */ 307 rep 308 movsb 309 310 done_copyin: 311 xorl %eax,%eax 312 movq PCPU(CURPCB),%rdx 313 movq %rax,PCB_ONFAULT(%rdx) 314 POP_FRAME_POINTER 315 ret 316 317 ALIGN_TEXT 318 copyin_fault: 319 movq PCPU(CURPCB),%rdx 320 movq $0,PCB_ONFAULT(%rdx) 321 movq $EFAULT,%rax 322 POP_FRAME_POINTER 323 ret 324 END(copyin) 325 326 /* 327 * casueword32. Compare and set user integer. Returns -1 on fault, 328 * 0 if access was successful. Old value is written to *oldp. 329 * dst = %rdi, old = %esi, oldp = %rdx, new = %ecx 330 */ 331 ENTRY(casueword32) 332 PUSH_FRAME_POINTER 333 movq PCPU(CURPCB),%r8 334 movq $fusufault,PCB_ONFAULT(%r8) 335 336 movq $VM_MAXUSER_ADDRESS-4,%rax 337 cmpq %rax,%rdi /* verify address is valid */ 338 ja fusufault 339 340 movl %esi,%eax /* old */ 341 #ifdef SMP 342 lock 343 #endif 344 cmpxchgl %ecx,(%rdi) /* new = %ecx */ 345 346 /* 347 * The old value is in %eax. If the store succeeded it will be the 348 * value we expected (old) from before the store, otherwise it will 349 * be the current value. Save %eax into %esi to prepare the return 350 * value. 351 */ 352 movl %eax,%esi 353 xorl %eax,%eax 354 movq %rax,PCB_ONFAULT(%r8) 355 356 /* 357 * Access the oldp after the pcb_onfault is cleared, to correctly 358 * catch corrupted pointer. 359 */ 360 movl %esi,(%rdx) /* oldp = %rdx */ 361 POP_FRAME_POINTER 362 ret 363 END(casueword32) 364 365 /* 366 * casueword. Compare and set user long. Returns -1 on fault, 367 * 0 if access was successful. Old value is written to *oldp. 368 * dst = %rdi, old = %rsi, oldp = %rdx, new = %rcx 369 */ 370 ENTRY(casueword) 371 PUSH_FRAME_POINTER 372 movq PCPU(CURPCB),%r8 373 movq $fusufault,PCB_ONFAULT(%r8) 374 375 movq $VM_MAXUSER_ADDRESS-4,%rax 376 cmpq %rax,%rdi /* verify address is valid */ 377 ja fusufault 378 379 movq %rsi,%rax /* old */ 380 #ifdef SMP 381 lock 382 #endif 383 cmpxchgq %rcx,(%rdi) /* new = %rcx */ 384 385 /* 386 * The old value is in %rax. If the store succeeded it will be the 387 * value we expected (old) from before the store, otherwise it will 388 * be the current value. 389 */ 390 movq %rax,%rsi 391 xorl %eax,%eax 392 movq %rax,PCB_ONFAULT(%r8) 393 movq %rsi,(%rdx) 394 POP_FRAME_POINTER 395 ret 396 END(casueword) 397 398 /* 399 * Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit 400 * byte from user memory. 401 * addr = %rdi, valp = %rsi 402 */ 403 404 ALTENTRY(fueword64) 405 ENTRY(fueword) 406 PUSH_FRAME_POINTER 407 movq PCPU(CURPCB),%rcx 408 movq $fusufault,PCB_ONFAULT(%rcx) 409 410 movq $VM_MAXUSER_ADDRESS-8,%rax 411 cmpq %rax,%rdi /* verify address is valid */ 412 ja fusufault 413 414 xorl %eax,%eax 415 movq (%rdi),%r11 416 movq %rax,PCB_ONFAULT(%rcx) 417 movq %r11,(%rsi) 418 POP_FRAME_POINTER 419 ret 420 END(fueword64) 421 END(fueword) 422 423 ENTRY(fueword32) 424 PUSH_FRAME_POINTER 425 movq PCPU(CURPCB),%rcx 426 movq $fusufault,PCB_ONFAULT(%rcx) 427 428 movq $VM_MAXUSER_ADDRESS-4,%rax 429 cmpq %rax,%rdi /* verify address is valid */ 430 ja fusufault 431 432 xorl %eax,%eax 433 movl (%rdi),%r11d 434 movq %rax,PCB_ONFAULT(%rcx) 435 movl %r11d,(%rsi) 436 POP_FRAME_POINTER 437 ret 438 END(fueword32) 439 440 /* 441 * fuswintr() and suswintr() are specialized variants of fuword16() and 442 * suword16(), respectively. They are called from the profiling code, 443 * potentially at interrupt time. If they fail, that's okay; good things 444 * will happen later. They always fail for now, until the trap code is 445 * able to deal with this. 446 */ 447 ALTENTRY(suswintr) 448 ENTRY(fuswintr) 449 movq $-1,%rax 450 ret 451 END(suswintr) 452 END(fuswintr) 453 454 ENTRY(fuword16) 455 PUSH_FRAME_POINTER 456 movq PCPU(CURPCB),%rcx 457 movq $fusufault,PCB_ONFAULT(%rcx) 458 459 movq $VM_MAXUSER_ADDRESS-2,%rax 460 cmpq %rax,%rdi 461 ja fusufault 462 463 movzwl (%rdi),%eax 464 movq $0,PCB_ONFAULT(%rcx) 465 POP_FRAME_POINTER 466 ret 467 END(fuword16) 468 469 ENTRY(fubyte) 470 PUSH_FRAME_POINTER 471 movq PCPU(CURPCB),%rcx 472 movq $fusufault,PCB_ONFAULT(%rcx) 473 474 movq $VM_MAXUSER_ADDRESS-1,%rax 475 cmpq %rax,%rdi 476 ja fusufault 477 478 movzbl (%rdi),%eax 479 movq $0,PCB_ONFAULT(%rcx) 480 POP_FRAME_POINTER 481 ret 482 END(fubyte) 483 484 ALIGN_TEXT 485 fusufault: 486 movq PCPU(CURPCB),%rcx 487 xorl %eax,%eax 488 movq %rax,PCB_ONFAULT(%rcx) 489 decq %rax 490 POP_FRAME_POINTER 491 ret 492 493 /* 494 * Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to 495 * user memory. 496 * addr = %rdi, value = %rsi 497 */ 498 ALTENTRY(suword64) 499 ENTRY(suword) 500 PUSH_FRAME_POINTER 501 movq PCPU(CURPCB),%rcx 502 movq $fusufault,PCB_ONFAULT(%rcx) 503 504 movq $VM_MAXUSER_ADDRESS-8,%rax 505 cmpq %rax,%rdi /* verify address validity */ 506 ja fusufault 507 508 movq %rsi,(%rdi) 509 xorl %eax,%eax 510 movq PCPU(CURPCB),%rcx 511 movq %rax,PCB_ONFAULT(%rcx) 512 POP_FRAME_POINTER 513 ret 514 END(suword64) 515 END(suword) 516 517 ENTRY(suword32) 518 PUSH_FRAME_POINTER 519 movq PCPU(CURPCB),%rcx 520 movq $fusufault,PCB_ONFAULT(%rcx) 521 522 movq $VM_MAXUSER_ADDRESS-4,%rax 523 cmpq %rax,%rdi /* verify address validity */ 524 ja fusufault 525 526 movl %esi,(%rdi) 527 xorl %eax,%eax 528 movq PCPU(CURPCB),%rcx 529 movq %rax,PCB_ONFAULT(%rcx) 530 POP_FRAME_POINTER 531 ret 532 END(suword32) 533 534 ENTRY(suword16) 535 PUSH_FRAME_POINTER 536 movq PCPU(CURPCB),%rcx 537 movq $fusufault,PCB_ONFAULT(%rcx) 538 539 movq $VM_MAXUSER_ADDRESS-2,%rax 540 cmpq %rax,%rdi /* verify address validity */ 541 ja fusufault 542 543 movw %si,(%rdi) 544 xorl %eax,%eax 545 movq PCPU(CURPCB),%rcx /* restore trashed register */ 546 movq %rax,PCB_ONFAULT(%rcx) 547 POP_FRAME_POINTER 548 ret 549 END(suword16) 550 551 ENTRY(subyte) 552 PUSH_FRAME_POINTER 553 movq PCPU(CURPCB),%rcx 554 movq $fusufault,PCB_ONFAULT(%rcx) 555 556 movq $VM_MAXUSER_ADDRESS-1,%rax 557 cmpq %rax,%rdi /* verify address validity */ 558 ja fusufault 559 560 movl %esi,%eax 561 movb %al,(%rdi) 562 xorl %eax,%eax 563 movq PCPU(CURPCB),%rcx /* restore trashed register */ 564 movq %rax,PCB_ONFAULT(%rcx) 565 POP_FRAME_POINTER 566 ret 567 END(subyte) 568 569 /* 570 * copyinstr(from, to, maxlen, int *lencopied) 571 * %rdi, %rsi, %rdx, %rcx 572 * 573 * copy a string from 'from' to 'to', stop when a 0 character is reached. 574 * return ENAMETOOLONG if string is longer than maxlen, and 575 * EFAULT on protection violations. If lencopied is non-zero, 576 * return the actual length in *lencopied. 577 */ 578 ENTRY(copyinstr) 579 PUSH_FRAME_POINTER 580 movq %rdx,%r8 /* %r8 = maxlen */ 581 movq %rcx,%r9 /* %r9 = *len */ 582 xchgq %rdi,%rsi /* %rdi = from, %rsi = to */ 583 movq PCPU(CURPCB),%rcx 584 movq $cpystrflt,PCB_ONFAULT(%rcx) 585 586 movq $VM_MAXUSER_ADDRESS,%rax 587 588 /* make sure 'from' is within bounds */ 589 subq %rsi,%rax 590 jbe cpystrflt 591 592 /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */ 593 cmpq %rdx,%rax 594 jae 1f 595 movq %rax,%rdx 596 movq %rax,%r8 597 1: 598 incq %rdx 599 600 2: 601 decq %rdx 602 jz 3f 603 604 lodsb 605 stosb 606 orb %al,%al 607 jnz 2b 608 609 /* Success -- 0 byte reached */ 610 decq %rdx 611 xorl %eax,%eax 612 jmp cpystrflt_x 613 3: 614 /* rdx is zero - return ENAMETOOLONG or EFAULT */ 615 movq $VM_MAXUSER_ADDRESS,%rax 616 cmpq %rax,%rsi 617 jae cpystrflt 618 4: 619 movq $ENAMETOOLONG,%rax 620 jmp cpystrflt_x 621 622 cpystrflt: 623 movq $EFAULT,%rax 624 625 cpystrflt_x: 626 /* set *lencopied and return %eax */ 627 movq PCPU(CURPCB),%rcx 628 movq $0,PCB_ONFAULT(%rcx) 629 630 testq %r9,%r9 631 jz 1f 632 subq %rdx,%r8 633 movq %r8,(%r9) 634 1: 635 POP_FRAME_POINTER 636 ret 637 END(copyinstr) 638 639 /* 640 * copystr(from, to, maxlen, int *lencopied) 641 * %rdi, %rsi, %rdx, %rcx 642 */ 643 ENTRY(copystr) 644 PUSH_FRAME_POINTER 645 movq %rdx,%r8 /* %r8 = maxlen */ 646 647 xchgq %rdi,%rsi 648 incq %rdx 649 1: 650 decq %rdx 651 jz 4f 652 lodsb 653 stosb 654 orb %al,%al 655 jnz 1b 656 657 /* Success -- 0 byte reached */ 658 decq %rdx 659 xorl %eax,%eax 660 jmp 6f 661 4: 662 /* rdx is zero -- return ENAMETOOLONG */ 663 movq $ENAMETOOLONG,%rax 664 665 6: 666 667 testq %rcx,%rcx 668 jz 7f 669 /* set *lencopied and return %rax */ 670 subq %rdx,%r8 671 movq %r8,(%rcx) 672 7: 673 POP_FRAME_POINTER 674 ret 675 END(copystr) 676 677 /* 678 * Handling of special amd64 registers and descriptor tables etc 679 */ 680 /* void lgdt(struct region_descriptor *rdp); */ 681 ENTRY(lgdt) 682 /* reload the descriptor table */ 683 lgdt (%rdi) 684 685 /* flush the prefetch q */ 686 jmp 1f 687 nop 688 1: 689 movl $KDSEL,%eax 690 movl %eax,%ds 691 movl %eax,%es 692 movl %eax,%fs /* Beware, use wrmsr to set 64 bit base */ 693 movl %eax,%gs 694 movl %eax,%ss 695 696 /* reload code selector by turning return into intersegmental return */ 697 popq %rax 698 pushq $KCSEL 699 pushq %rax 700 MEXITCOUNT 701 lretq 702 END(lgdt) 703 704 /*****************************************************************************/ 705 /* setjump, longjump */ 706 /*****************************************************************************/ 707 708 ENTRY(setjmp) 709 movq %rbx,0(%rdi) /* save rbx */ 710 movq %rsp,8(%rdi) /* save rsp */ 711 movq %rbp,16(%rdi) /* save rbp */ 712 movq %r12,24(%rdi) /* save r12 */ 713 movq %r13,32(%rdi) /* save r13 */ 714 movq %r14,40(%rdi) /* save r14 */ 715 movq %r15,48(%rdi) /* save r15 */ 716 movq 0(%rsp),%rdx /* get rta */ 717 movq %rdx,56(%rdi) /* save rip */ 718 xorl %eax,%eax /* return(0); */ 719 ret 720 END(setjmp) 721 722 ENTRY(longjmp) 723 movq 0(%rdi),%rbx /* restore rbx */ 724 movq 8(%rdi),%rsp /* restore rsp */ 725 movq 16(%rdi),%rbp /* restore rbp */ 726 movq 24(%rdi),%r12 /* restore r12 */ 727 movq 32(%rdi),%r13 /* restore r13 */ 728 movq 40(%rdi),%r14 /* restore r14 */ 729 movq 48(%rdi),%r15 /* restore r15 */ 730 movq 56(%rdi),%rdx /* get rta */ 731 movq %rdx,0(%rsp) /* put in return frame */ 732 xorl %eax,%eax /* return(1); */ 733 incl %eax 734 ret 735 END(longjmp) 736 737 /* 738 * Support for reading MSRs in the safe manner. 739 */ 740 ENTRY(rdmsr_safe) 741 /* int rdmsr_safe(u_int msr, uint64_t *data) */ 742 PUSH_FRAME_POINTER 743 movq PCPU(CURPCB),%r8 744 movq $msr_onfault,PCB_ONFAULT(%r8) 745 movl %edi,%ecx 746 rdmsr /* Read MSR pointed by %ecx. Returns 747 hi byte in edx, lo in %eax */ 748 salq $32,%rdx /* sign-shift %rdx left */ 749 movl %eax,%eax /* zero-extend %eax -> %rax */ 750 orq %rdx,%rax 751 movq %rax,(%rsi) 752 xorq %rax,%rax 753 movq %rax,PCB_ONFAULT(%r8) 754 POP_FRAME_POINTER 755 ret 756 757 /* 758 * Support for writing MSRs in the safe manner. 759 */ 760 ENTRY(wrmsr_safe) 761 /* int wrmsr_safe(u_int msr, uint64_t data) */ 762 PUSH_FRAME_POINTER 763 movq PCPU(CURPCB),%r8 764 movq $msr_onfault,PCB_ONFAULT(%r8) 765 movl %edi,%ecx 766 movl %esi,%eax 767 sarq $32,%rsi 768 movl %esi,%edx 769 wrmsr /* Write MSR pointed by %ecx. Accepts 770 hi byte in edx, lo in %eax. */ 771 xorq %rax,%rax 772 movq %rax,PCB_ONFAULT(%r8) 773 POP_FRAME_POINTER 774 ret 775 776 /* 777 * MSR operations fault handler 778 */ 779 ALIGN_TEXT 780 msr_onfault: 781 movq $0,PCB_ONFAULT(%r8) 782 movl $EFAULT,%eax 783 POP_FRAME_POINTER 784 ret 785 786 /* 787 * void pmap_pti_pcid_invalidate(uint64_t ucr3, uint64_t kcr3); 788 * Invalidates address space addressed by ucr3, then returns to kcr3. 789 * Done in assembler to ensure no other memory accesses happen while 790 * on ucr3. 791 */ 792 ALIGN_TEXT 793 ENTRY(pmap_pti_pcid_invalidate) 794 pushfq 795 cli 796 movq %rdi,%cr3 /* to user page table */ 797 movq %rsi,%cr3 /* back to kernel */ 798 popfq 799 retq 800 801 /* 802 * void pmap_pti_pcid_invlpg(uint64_t ucr3, uint64_t kcr3, vm_offset_t va); 803 * Invalidates virtual address va in address space ucr3, then returns to kcr3. 804 */ 805 ALIGN_TEXT 806 ENTRY(pmap_pti_pcid_invlpg) 807 pushfq 808 cli 809 movq %rdi,%cr3 /* to user page table */ 810 invlpg (%rdx) 811 movq %rsi,%cr3 /* back to kernel */ 812 popfq 813 retq 814 815 /* 816 * void pmap_pti_pcid_invlrng(uint64_t ucr3, uint64_t kcr3, vm_offset_t sva, 817 * vm_offset_t eva); 818 * Invalidates virtual addresses between sva and eva in address space ucr3, 819 * then returns to kcr3. 820 */ 821 ALIGN_TEXT 822 ENTRY(pmap_pti_pcid_invlrng) 823 pushfq 824 cli 825 movq %rdi,%cr3 /* to user page table */ 826 1: invlpg (%rdx) 827 addq $PAGE_SIZE,%rdx 828 cmpq %rdx,%rcx 829 ja 1b 830 movq %rsi,%cr3 /* back to kernel */ 831 popfq 832 retq 833 834 .altmacro 835 .macro rsb_seq_label l 836 rsb_seq_\l: 837 .endm 838 .macro rsb_call_label l 839 call rsb_seq_\l 840 .endm 841 .macro rsb_seq count 842 ll=1 843 .rept \count 844 rsb_call_label %(ll) 845 nop 846 rsb_seq_label %(ll) 847 addq $8,%rsp 848 ll=ll+1 849 .endr 850 .endm 851 852 ENTRY(rsb_flush) 853 rsb_seq 32 854 ret 855 856 /* all callers already saved %rax, %rdx, and %rcx */ 857 ENTRY(handle_ibrs_entry) 858 cmpb $0,hw_ibrs_ibpb_active(%rip) 859 je 1f 860 movl $MSR_IA32_SPEC_CTRL,%ecx 861 rdmsr 862 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax 863 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32,%edx 864 wrmsr 865 movb $1,PCPU(IBPB_SET) 866 testl $CPUID_STDEXT_SMEP,cpu_stdext_feature(%rip) 867 je rsb_flush 868 1: ret 869 END(handle_ibrs_entry) 870 871 ENTRY(handle_ibrs_exit) 872 cmpb $0,PCPU(IBPB_SET) 873 je 1f 874 movl $MSR_IA32_SPEC_CTRL,%ecx 875 rdmsr 876 andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax 877 andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx 878 wrmsr 879 movb $0,PCPU(IBPB_SET) 880 1: ret 881 END(handle_ibrs_exit) 882 883 /* registers-neutral version, but needs stack */ 884 ENTRY(handle_ibrs_exit_rs) 885 cmpb $0,PCPU(IBPB_SET) 886 je 1f 887 pushq %rax 888 pushq %rdx 889 pushq %rcx 890 movl $MSR_IA32_SPEC_CTRL,%ecx 891 rdmsr 892 andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax 893 andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx 894 wrmsr 895 popq %rcx 896 popq %rdx 897 popq %rax 898 movb $0,PCPU(IBPB_SET) 899 1: ret 900 END(handle_ibrs_exit_rs) 901 902 .noaltmacro 903 904 /* 905 * Flush L1D cache. Load enough of the data from the kernel text 906 * to flush existing L1D content. 907 * 908 * N.B. The function does not follow ABI calling conventions, it corrupts %rbx. 909 * The vmm.ko caller expects that only %rax, %rdx, %rbx, %rcx, %r9, and %rflags 910 * registers are clobbered. The NMI handler caller only needs %r13 preserved. 911 */ 912 ENTRY(flush_l1d_sw) 913 #define L1D_FLUSH_SIZE (64 * 1024) 914 movq $KERNBASE, %r9 915 movq $-L1D_FLUSH_SIZE, %rcx 916 /* 917 * pass 1: Preload TLB. 918 * Kernel text is mapped using superpages. TLB preload is 919 * done for the benefit of older CPUs which split 2M page 920 * into 4k TLB entries. 921 */ 922 1: movb L1D_FLUSH_SIZE(%r9, %rcx), %al 923 addq $PAGE_SIZE, %rcx 924 jne 1b 925 xorl %eax, %eax 926 cpuid 927 movq $-L1D_FLUSH_SIZE, %rcx 928 /* pass 2: Read each cache line. */ 929 2: movb L1D_FLUSH_SIZE(%r9, %rcx), %al 930 addq $64, %rcx 931 jne 2b 932 lfence 933 ret 934 #undef L1D_FLUSH_SIZE 935 END(flush_l1d_sw) 936 937 ENTRY(mds_handler_void) 938 retq 939 END(mds_handler_void) 940 941 ENTRY(mds_handler_verw) 942 subq $8, %rsp 943 movw %ds, (%rsp) 944 verw (%rsp) 945 addq $8, %rsp 946 retq 947 END(mds_handler_verw) 948 949 ENTRY(mds_handler_ivb) 950 pushq %rax 951 pushq %rdx 952 pushq %rcx 953 954 movq %cr0, %rax 955 testb $CR0_TS, %al 956 je 1f 957 clts 958 1: movq PCPU(MDS_BUF), %rdx 959 movdqa %xmm0, PCPU(MDS_TMP) 960 pxor %xmm0, %xmm0 961 962 lfence 963 orpd (%rdx), %xmm0 964 orpd (%rdx), %xmm0 965 mfence 966 movl $40, %ecx 967 addq $16, %rdx 968 2: movntdq %xmm0, (%rdx) 969 addq $16, %rdx 970 decl %ecx 971 jnz 2b 972 mfence 973 974 movdqa PCPU(MDS_TMP),%xmm0 975 testb $CR0_TS, %al 976 je 3f 977 movq %rax, %cr0 978 3: popq %rcx 979 popq %rdx 980 popq %rax 981 retq 982 END(mds_handler_ivb) 983 984 ENTRY(mds_handler_bdw) 985 pushq %rax 986 pushq %rbx 987 pushq %rcx 988 pushq %rdi 989 pushq %rsi 990 991 movq %cr0, %rax 992 testb $CR0_TS, %al 993 je 1f 994 clts 995 1: movq PCPU(MDS_BUF), %rbx 996 movdqa %xmm0, PCPU(MDS_TMP) 997 pxor %xmm0, %xmm0 998 999 movq %rbx, %rdi 1000 movq %rbx, %rsi 1001 movl $40, %ecx 1002 2: movntdq %xmm0, (%rbx) 1003 addq $16, %rbx 1004 decl %ecx 1005 jnz 2b 1006 mfence 1007 movl $1536, %ecx 1008 rep; movsb 1009 lfence 1010 1011 movdqa PCPU(MDS_TMP),%xmm0 1012 testb $CR0_TS, %al 1013 je 3f 1014 movq %rax, %cr0 1015 3: popq %rsi 1016 popq %rdi 1017 popq %rcx 1018 popq %rbx 1019 popq %rax 1020 retq 1021 END(mds_handler_bdw) 1022 1023 ENTRY(mds_handler_skl_sse) 1024 pushq %rax 1025 pushq %rdx 1026 pushq %rcx 1027 pushq %rdi 1028 1029 movq %cr0, %rax 1030 testb $CR0_TS, %al 1031 je 1f 1032 clts 1033 1: movq PCPU(MDS_BUF), %rdi 1034 movq PCPU(MDS_BUF64), %rdx 1035 movdqa %xmm0, PCPU(MDS_TMP) 1036 pxor %xmm0, %xmm0 1037 1038 lfence 1039 orpd (%rdx), %xmm0 1040 orpd (%rdx), %xmm0 1041 xorl %eax, %eax 1042 2: clflushopt 5376(%rdi, %rax, 8) 1043 addl $8, %eax 1044 cmpl $8 * 12, %eax 1045 jb 2b 1046 sfence 1047 movl $6144, %ecx 1048 xorl %eax, %eax 1049 rep; stosb 1050 mfence 1051 1052 movdqa PCPU(MDS_TMP), %xmm0 1053 testb $CR0_TS, %al 1054 je 3f 1055 movq %rax, %cr0 1056 3: popq %rdi 1057 popq %rcx 1058 popq %rdx 1059 popq %rax 1060 retq 1061 END(mds_handler_skl_sse) 1062 1063 ENTRY(mds_handler_skl_avx) 1064 pushq %rax 1065 pushq %rdx 1066 pushq %rcx 1067 pushq %rdi 1068 1069 movq %cr0, %rax 1070 testb $CR0_TS, %al 1071 je 1f 1072 clts 1073 1: movq PCPU(MDS_BUF), %rdi 1074 movq PCPU(MDS_BUF64), %rdx 1075 vmovdqa %ymm0, PCPU(MDS_TMP) 1076 vpxor %ymm0, %ymm0, %ymm0 1077 1078 lfence 1079 vorpd (%rdx), %ymm0, %ymm0 1080 vorpd (%rdx), %ymm0, %ymm0 1081 xorl %eax, %eax 1082 2: clflushopt 5376(%rdi, %rax, 8) 1083 addl $8, %eax 1084 cmpl $8 * 12, %eax 1085 jb 2b 1086 sfence 1087 movl $6144, %ecx 1088 xorl %eax, %eax 1089 rep; stosb 1090 mfence 1091 1092 vmovdqa PCPU(MDS_TMP), %ymm0 1093 testb $CR0_TS, %al 1094 je 3f 1095 movq %rax, %cr0 1096 3: popq %rdi 1097 popq %rcx 1098 popq %rdx 1099 popq %rax 1100 retq 1101 END(mds_handler_skl_avx) 1102 1103 ENTRY(mds_handler_skl_avx512) 1104 pushq %rax 1105 pushq %rdx 1106 pushq %rcx 1107 pushq %rdi 1108 1109 movq %cr0, %rax 1110 testb $CR0_TS, %al 1111 je 1f 1112 clts 1113 1: movq PCPU(MDS_BUF), %rdi 1114 movq PCPU(MDS_BUF64), %rdx 1115 vmovdqa64 %zmm0, PCPU(MDS_TMP) 1116 vpxor %zmm0, %zmm0, %zmm0 1117 1118 lfence 1119 vorpd (%rdx), %zmm0, %zmm0 1120 vorpd (%rdx), %zmm0, %zmm0 1121 xorl %eax, %eax 1122 2: clflushopt 5376(%rdi, %rax, 8) 1123 addl $8, %eax 1124 cmpl $8 * 12, %eax 1125 jb 2b 1126 sfence 1127 movl $6144, %ecx 1128 xorl %eax, %eax 1129 rep; stosb 1130 mfence 1131 1132 vmovdqa64 PCPU(MDS_TMP), %zmm0 1133 testb $CR0_TS, %al 1134 je 3f 1135 movq %rax, %cr0 1136 3: popq %rdi 1137 popq %rcx 1138 popq %rdx 1139 popq %rax 1140 retq 1141 END(mds_handler_skl_avx512) 1142 1143 ENTRY(mds_handler_silvermont) 1144 pushq %rax 1145 pushq %rdx 1146 pushq %rcx 1147 1148 movq %cr0, %rax 1149 testb $CR0_TS, %al 1150 je 1f 1151 clts 1152 1: movq PCPU(MDS_BUF), %rdx 1153 movdqa %xmm0, PCPU(MDS_TMP) 1154 pxor %xmm0, %xmm0 1155 1156 movl $16, %ecx 1157 2: movntdq %xmm0, (%rdx) 1158 addq $16, %rdx 1159 decl %ecx 1160 jnz 2b 1161 mfence 1162 1163 movdqa PCPU(MDS_TMP),%xmm0 1164 testb $CR0_TS, %al 1165 je 3f 1166 movq %rax, %cr0 1167 3: popq %rcx 1168 popq %rdx 1169 popq %rax 1170 retq 1171 END(mds_handler_silvermont)
Cache object: 74e449c7daf792a69abac03c8a299f3a
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]