Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/support.S
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 2018-2019 The FreeBSD Foundation 3 * Copyright (c) 2003 Peter Wemm. 4 * Copyright (c) 1993 The Regents of the University of California. 5 * All rights reserved. 6 * 7 * Portions of this software were developed by 8 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from 9 * the FreeBSD Foundation. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. Neither the name of the University nor the names of its contributors 20 * may be used to endorse or promote products derived from this software 21 * without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 * SUCH DAMAGE. 34 * 35 * $FreeBSD$ 36 */ 37 38 #include "opt_ddb.h" 39 40 #include <machine/asmacros.h> 41 #include <machine/specialreg.h> 42 #include <machine/pmap.h> 43 44 #include "assym.inc" 45 46 .text 47 48 /* Address: %rdi */ 49 ENTRY(pagezero_std) 50 PUSH_FRAME_POINTER 51 movl $PAGE_SIZE/8,%ecx 52 xorl %eax,%eax 53 rep 54 stosq 55 POP_FRAME_POINTER 56 ret 57 END(pagezero_std) 58 59 ENTRY(pagezero_erms) 60 PUSH_FRAME_POINTER 61 movl $PAGE_SIZE,%ecx 62 xorl %eax,%eax 63 rep 64 stosb 65 POP_FRAME_POINTER 66 ret 67 END(pagezero_erms) 68 69 /* 70 * pagecopy(%rdi=from, %rsi=to) 71 */ 72 ENTRY(pagecopy) 73 PUSH_FRAME_POINTER 74 movl $PAGE_SIZE/8,%ecx 75 movq %rdi,%r9 76 movq %rsi,%rdi 77 movq %r9,%rsi 78 rep 79 movsq 80 POP_FRAME_POINTER 81 ret 82 END(pagecopy) 83 84 /* Address: %rdi */ 85 ENTRY(sse2_pagezero) 86 PUSH_FRAME_POINTER 87 movq $-PAGE_SIZE,%rdx 88 subq %rdx,%rdi 89 xorl %eax,%eax 90 jmp 1f 91 /* 92 * The loop takes 29 bytes. Ensure that it doesn't cross a 32-byte 93 * cache line. 94 */ 95 .p2align 5,0x90 96 1: 97 movnti %rax,(%rdi,%rdx) 98 movnti %rax,8(%rdi,%rdx) 99 movnti %rax,16(%rdi,%rdx) 100 movnti %rax,24(%rdi,%rdx) 101 addq $32,%rdx 102 jne 1b 103 sfence 104 POP_FRAME_POINTER 105 ret 106 END(sse2_pagezero) 107 108 /* 109 * memcmp(b1, b2, len) 110 * rdi,rsi,rdx 111 */ 112 ENTRY(memcmp) 113 PUSH_FRAME_POINTER 114 115 xorl %eax,%eax 116 10: 117 cmpq $16,%rdx 118 ja 101632f 119 120 cmpb $8,%dl 121 jg 100816f 122 123 cmpb $4,%dl 124 jg 100408f 125 126 cmpb $2,%dl 127 jge 100204f 128 129 cmpb $1,%dl 130 jl 100000f 131 movzbl (%rdi),%eax 132 movzbl (%rsi),%r8d 133 subl %r8d,%eax 134 100000: 135 POP_FRAME_POINTER 136 ret 137 138 ALIGN_TEXT 139 100816: 140 movq (%rdi),%r8 141 movq (%rsi),%r9 142 cmpq %r8,%r9 143 jne 80f 144 movq -8(%rdi,%rdx),%r8 145 movq -8(%rsi,%rdx),%r9 146 cmpq %r8,%r9 147 jne 10081608f 148 POP_FRAME_POINTER 149 ret 150 ALIGN_TEXT 151 100408: 152 movl (%rdi),%r8d 153 movl (%rsi),%r9d 154 cmpl %r8d,%r9d 155 jne 80f 156 movl -4(%rdi,%rdx),%r8d 157 movl -4(%rsi,%rdx),%r9d 158 cmpl %r8d,%r9d 159 jne 10040804f 160 POP_FRAME_POINTER 161 ret 162 ALIGN_TEXT 163 100204: 164 movzwl (%rdi),%r8d 165 movzwl (%rsi),%r9d 166 cmpl %r8d,%r9d 167 jne 1f 168 movzwl -2(%rdi,%rdx),%r8d 169 movzwl -2(%rsi,%rdx),%r9d 170 cmpl %r8d,%r9d 171 jne 1f 172 POP_FRAME_POINTER 173 ret 174 ALIGN_TEXT 175 101632: 176 cmpq $32,%rdx 177 ja 103200f 178 movq (%rdi),%r8 179 movq (%rsi),%r9 180 cmpq %r8,%r9 181 jne 80f 182 movq 8(%rdi),%r8 183 movq 8(%rsi),%r9 184 cmpq %r8,%r9 185 jne 10163208f 186 movq -16(%rdi,%rdx),%r8 187 movq -16(%rsi,%rdx),%r9 188 cmpq %r8,%r9 189 jne 10163216f 190 movq -8(%rdi,%rdx),%r8 191 movq -8(%rsi,%rdx),%r9 192 cmpq %r8,%r9 193 jne 10163224f 194 POP_FRAME_POINTER 195 ret 196 ALIGN_TEXT 197 103200: 198 movq (%rdi),%r8 199 movq 8(%rdi),%r9 200 subq (%rsi),%r8 201 subq 8(%rsi),%r9 202 orq %r8,%r9 203 jnz 10320000f 204 205 movq 16(%rdi),%r8 206 movq 24(%rdi),%r9 207 subq 16(%rsi),%r8 208 subq 24(%rsi),%r9 209 orq %r8,%r9 210 jnz 10320016f 211 212 leaq 32(%rdi),%rdi 213 leaq 32(%rsi),%rsi 214 subq $32,%rdx 215 cmpq $32,%rdx 216 jae 103200b 217 cmpb $0,%dl 218 jne 10b 219 POP_FRAME_POINTER 220 ret 221 222 /* 223 * Mismatch was found. 224 * 225 * Before we compute it we narrow down the range (16 -> 8 -> 4 bytes). 226 */ 227 ALIGN_TEXT 228 10320016: 229 leaq 16(%rdi),%rdi 230 leaq 16(%rsi),%rsi 231 10320000: 232 movq (%rdi),%r8 233 movq (%rsi),%r9 234 cmpq %r8,%r9 235 jne 80f 236 leaq 8(%rdi),%rdi 237 leaq 8(%rsi),%rsi 238 jmp 80f 239 ALIGN_TEXT 240 10081608: 241 10163224: 242 leaq -8(%rdi,%rdx),%rdi 243 leaq -8(%rsi,%rdx),%rsi 244 jmp 80f 245 ALIGN_TEXT 246 10163216: 247 leaq -16(%rdi,%rdx),%rdi 248 leaq -16(%rsi,%rdx),%rsi 249 jmp 80f 250 ALIGN_TEXT 251 10163208: 252 leaq 8(%rdi),%rdi 253 leaq 8(%rsi),%rsi 254 jmp 80f 255 ALIGN_TEXT 256 10040804: 257 leaq -4(%rdi,%rdx),%rdi 258 leaq -4(%rsi,%rdx),%rsi 259 jmp 1f 260 261 ALIGN_TEXT 262 80: 263 movl (%rdi),%r8d 264 movl (%rsi),%r9d 265 cmpl %r8d,%r9d 266 jne 1f 267 leaq 4(%rdi),%rdi 268 leaq 4(%rsi),%rsi 269 270 /* 271 * We have up to 4 bytes to inspect. 272 */ 273 1: 274 movzbl (%rdi),%eax 275 movzbl (%rsi),%r8d 276 cmpb %r8b,%al 277 jne 2f 278 279 movzbl 1(%rdi),%eax 280 movzbl 1(%rsi),%r8d 281 cmpb %r8b,%al 282 jne 2f 283 284 movzbl 2(%rdi),%eax 285 movzbl 2(%rsi),%r8d 286 cmpb %r8b,%al 287 jne 2f 288 289 movzbl 3(%rdi),%eax 290 movzbl 3(%rsi),%r8d 291 2: 292 subl %r8d,%eax 293 POP_FRAME_POINTER 294 ret 295 END(memcmp) 296 297 /* 298 * memmove(dst, src, cnt) 299 * rdi, rsi, rdx 300 */ 301 302 /* 303 * Register state at entry is supposed to be as follows: 304 * rdi - destination 305 * rsi - source 306 * rdx - count 307 * 308 * The macro possibly clobbers the above and: rcx, r8, r9, r10 309 * It does not clobber rax nor r11. 310 */ 311 .macro MEMMOVE erms overlap begin end 312 \begin 313 314 /* 315 * For sizes 0..32 all data is read before it is written, so there 316 * is no correctness issue with direction of copying. 317 */ 318 cmpq $32,%rcx 319 jbe 101632f 320 321 .if \overlap == 1 322 movq %rdi,%r8 323 subq %rsi,%r8 324 cmpq %rcx,%r8 /* overlapping && src < dst? */ 325 jb 2f 326 .endif 327 328 cmpq $256,%rcx 329 ja 1256f 330 331 ALIGN_TEXT 332 103200: 333 movq (%rsi),%rdx 334 movq %rdx,(%rdi) 335 movq 8(%rsi),%rdx 336 movq %rdx,8(%rdi) 337 movq 16(%rsi),%rdx 338 movq %rdx,16(%rdi) 339 movq 24(%rsi),%rdx 340 movq %rdx,24(%rdi) 341 leaq 32(%rsi),%rsi 342 leaq 32(%rdi),%rdi 343 subq $32,%rcx 344 cmpq $32,%rcx 345 jae 103200b 346 cmpb $0,%cl 347 jne 101632f 348 \end 349 ret 350 ALIGN_TEXT 351 101632: 352 cmpb $16,%cl 353 jl 100816f 354 movq (%rsi),%rdx 355 movq 8(%rsi),%r8 356 movq -16(%rsi,%rcx),%r9 357 movq -8(%rsi,%rcx),%r10 358 movq %rdx,(%rdi) 359 movq %r8,8(%rdi) 360 movq %r9,-16(%rdi,%rcx) 361 movq %r10,-8(%rdi,%rcx) 362 \end 363 ret 364 ALIGN_TEXT 365 100816: 366 cmpb $8,%cl 367 jl 100408f 368 movq (%rsi),%rdx 369 movq -8(%rsi,%rcx),%r8 370 movq %rdx,(%rdi) 371 movq %r8,-8(%rdi,%rcx,) 372 \end 373 ret 374 ALIGN_TEXT 375 100408: 376 cmpb $4,%cl 377 jl 100204f 378 movl (%rsi),%edx 379 movl -4(%rsi,%rcx),%r8d 380 movl %edx,(%rdi) 381 movl %r8d,-4(%rdi,%rcx) 382 \end 383 ret 384 ALIGN_TEXT 385 100204: 386 cmpb $2,%cl 387 jl 100001f 388 movzwl (%rsi),%edx 389 movzwl -2(%rsi,%rcx),%r8d 390 movw %dx,(%rdi) 391 movw %r8w,-2(%rdi,%rcx) 392 \end 393 ret 394 ALIGN_TEXT 395 100001: 396 cmpb $1,%cl 397 jl 100000f 398 movb (%rsi),%dl 399 movb %dl,(%rdi) 400 100000: 401 \end 402 ret 403 404 ALIGN_TEXT 405 1256: 406 testb $15,%dil 407 jnz 100f 408 .if \erms == 1 409 rep 410 movsb 411 .else 412 shrq $3,%rcx /* copy by 64-bit words */ 413 rep 414 movsq 415 movq %rdx,%rcx 416 andl $7,%ecx /* any bytes left? */ 417 jne 100408b 418 .endif 419 \end 420 ret 421 100: 422 movq (%rsi),%r8 423 movq 8(%rsi),%r9 424 movq %rdi,%r10 425 movq %rdi,%rcx 426 andq $15,%rcx 427 leaq -16(%rdx,%rcx),%rdx 428 neg %rcx 429 leaq 16(%rdi,%rcx),%rdi 430 leaq 16(%rsi,%rcx),%rsi 431 movq %rdx,%rcx 432 .if \erms == 1 433 rep 434 movsb 435 movq %r8,(%r10) 436 movq %r9,8(%r10) 437 .else 438 shrq $3,%rcx /* copy by 64-bit words */ 439 rep 440 movsq 441 movq %r8,(%r10) 442 movq %r9,8(%r10) 443 movq %rdx,%rcx 444 andl $7,%ecx /* any bytes left? */ 445 jne 100408b 446 .endif 447 \end 448 ret 449 450 .if \overlap == 1 451 /* 452 * Copy backwards. 453 */ 454 ALIGN_TEXT 455 2: 456 cmpq $256,%rcx 457 ja 2256f 458 459 leaq -8(%rdi,%rcx),%rdi 460 leaq -8(%rsi,%rcx),%rsi 461 462 cmpq $32,%rcx 463 jb 2016f 464 465 ALIGN_TEXT 466 2032: 467 movq (%rsi),%rdx 468 movq %rdx,(%rdi) 469 movq -8(%rsi),%rdx 470 movq %rdx,-8(%rdi) 471 movq -16(%rsi),%rdx 472 movq %rdx,-16(%rdi) 473 movq -24(%rsi),%rdx 474 movq %rdx,-24(%rdi) 475 leaq -32(%rsi),%rsi 476 leaq -32(%rdi),%rdi 477 subq $32,%rcx 478 cmpq $32,%rcx 479 jae 2032b 480 cmpb $0,%cl 481 jne 2016f 482 \end 483 ret 484 ALIGN_TEXT 485 2016: 486 cmpb $16,%cl 487 jl 2008f 488 movq (%rsi),%rdx 489 movq %rdx,(%rdi) 490 movq -8(%rsi),%rdx 491 movq %rdx,-8(%rdi) 492 subb $16,%cl 493 jz 2000f 494 leaq -16(%rsi),%rsi 495 leaq -16(%rdi),%rdi 496 2008: 497 cmpb $8,%cl 498 jl 2004f 499 movq (%rsi),%rdx 500 movq %rdx,(%rdi) 501 subb $8,%cl 502 jz 2000f 503 leaq -8(%rsi),%rsi 504 leaq -8(%rdi),%rdi 505 2004: 506 cmpb $4,%cl 507 jl 2002f 508 movl 4(%rsi),%edx 509 movl %edx,4(%rdi) 510 subb $4,%cl 511 jz 2000f 512 leaq -4(%rsi),%rsi 513 leaq -4(%rdi),%rdi 514 2002: 515 cmpb $2,%cl 516 jl 2001f 517 movw 6(%rsi),%dx 518 movw %dx,6(%rdi) 519 subb $2,%cl 520 jz 2000f 521 leaq -2(%rsi),%rsi 522 leaq -2(%rdi),%rdi 523 2001: 524 cmpb $1,%cl 525 jl 2000f 526 movb 7(%rsi),%dl 527 movb %dl,7(%rdi) 528 2000: 529 \end 530 ret 531 ALIGN_TEXT 532 2256: 533 std 534 leaq -8(%rdi,%rcx),%rdi 535 leaq -8(%rsi,%rcx),%rsi 536 shrq $3,%rcx 537 rep 538 movsq 539 cld 540 movq %rdx,%rcx 541 andb $7,%cl 542 jne 2004b 543 \end 544 ret 545 .endif 546 .endm 547 548 .macro MEMMOVE_BEGIN 549 PUSH_FRAME_POINTER 550 movq %rdi,%rax 551 movq %rdx,%rcx 552 .endm 553 554 .macro MEMMOVE_END 555 POP_FRAME_POINTER 556 .endm 557 558 ENTRY(memmove_std) 559 MEMMOVE erms=0 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END 560 END(memmove_std) 561 562 ENTRY(memmove_erms) 563 MEMMOVE erms=1 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END 564 END(memmove_erms) 565 566 /* 567 * memcpy(dst, src, len) 568 * rdi, rsi, rdx 569 * 570 * Note: memcpy does not support overlapping copies 571 */ 572 ENTRY(memcpy_std) 573 MEMMOVE erms=0 overlap=0 begin=MEMMOVE_BEGIN end=MEMMOVE_END 574 END(memcpy_std) 575 576 ENTRY(memcpy_erms) 577 MEMMOVE erms=1 overlap=0 begin=MEMMOVE_BEGIN end=MEMMOVE_END 578 END(memcpy_erms) 579 580 /* 581 * memset(dst, c, len) 582 * rdi, rsi, rdx 583 */ 584 .macro MEMSET erms 585 PUSH_FRAME_POINTER 586 movq %rdi,%rax 587 movq %rdx,%rcx 588 movzbq %sil,%r8 589 movabs $0x0101010101010101,%r10 590 imulq %r8,%r10 591 592 cmpq $32,%rcx 593 jbe 101632f 594 595 cmpq $256,%rcx 596 ja 1256f 597 598 ALIGN_TEXT 599 103200: 600 movq %r10,(%rdi) 601 movq %r10,8(%rdi) 602 movq %r10,16(%rdi) 603 movq %r10,24(%rdi) 604 leaq 32(%rdi),%rdi 605 subq $32,%rcx 606 cmpq $32,%rcx 607 ja 103200b 608 cmpb $16,%cl 609 ja 201632f 610 movq %r10,-16(%rdi,%rcx) 611 movq %r10,-8(%rdi,%rcx) 612 POP_FRAME_POINTER 613 ret 614 ALIGN_TEXT 615 101632: 616 cmpb $16,%cl 617 jl 100816f 618 201632: 619 movq %r10,(%rdi) 620 movq %r10,8(%rdi) 621 movq %r10,-16(%rdi,%rcx) 622 movq %r10,-8(%rdi,%rcx) 623 POP_FRAME_POINTER 624 ret 625 ALIGN_TEXT 626 100816: 627 cmpb $8,%cl 628 jl 100408f 629 movq %r10,(%rdi) 630 movq %r10,-8(%rdi,%rcx) 631 POP_FRAME_POINTER 632 ret 633 ALIGN_TEXT 634 100408: 635 cmpb $4,%cl 636 jl 100204f 637 movl %r10d,(%rdi) 638 movl %r10d,-4(%rdi,%rcx) 639 POP_FRAME_POINTER 640 ret 641 ALIGN_TEXT 642 100204: 643 cmpb $2,%cl 644 jl 100001f 645 movw %r10w,(%rdi) 646 movw %r10w,-2(%rdi,%rcx) 647 POP_FRAME_POINTER 648 ret 649 ALIGN_TEXT 650 100001: 651 cmpb $0,%cl 652 je 100000f 653 movb %r10b,(%rdi) 654 100000: 655 POP_FRAME_POINTER 656 ret 657 ALIGN_TEXT 658 1256: 659 movq %rdi,%r9 660 movq %r10,%rax 661 testl $15,%edi 662 jnz 3f 663 1: 664 .if \erms == 1 665 rep 666 stosb 667 movq %r9,%rax 668 .else 669 movq %rcx,%rdx 670 shrq $3,%rcx 671 rep 672 stosq 673 movq %r9,%rax 674 andl $7,%edx 675 jnz 2f 676 POP_FRAME_POINTER 677 ret 678 2: 679 movq %r10,-8(%rdi,%rdx) 680 .endif 681 POP_FRAME_POINTER 682 ret 683 ALIGN_TEXT 684 3: 685 movq %r10,(%rdi) 686 movq %r10,8(%rdi) 687 movq %rdi,%r8 688 andq $15,%r8 689 leaq -16(%rcx,%r8),%rcx 690 neg %r8 691 leaq 16(%rdi,%r8),%rdi 692 jmp 1b 693 .endm 694 695 ENTRY(memset_std) 696 MEMSET erms=0 697 END(memset_std) 698 699 ENTRY(memset_erms) 700 MEMSET erms=1 701 END(memset_erms) 702 703 /* fillw(pat, base, cnt) */ 704 /* %rdi,%rsi, %rdx */ 705 ENTRY(fillw) 706 PUSH_FRAME_POINTER 707 movq %rdi,%rax 708 movq %rsi,%rdi 709 movq %rdx,%rcx 710 rep 711 stosw 712 POP_FRAME_POINTER 713 ret 714 END(fillw) 715 716 /*****************************************************************************/ 717 /* copyout and fubyte family */ 718 /*****************************************************************************/ 719 /* 720 * Access user memory from inside the kernel. These routines should be 721 * the only places that do this. 722 * 723 * These routines set curpcb->pcb_onfault for the time they execute. When a 724 * protection violation occurs inside the functions, the trap handler 725 * returns to *curpcb->pcb_onfault instead of the function. 726 */ 727 728 .macro SMAP_DISABLE smap 729 .if \smap 730 stac 731 .endif 732 .endm 733 734 735 .macro SMAP_ENABLE smap 736 .if \smap 737 clac 738 .endif 739 .endm 740 741 .macro COPYINOUT_BEGIN 742 .endm 743 744 .macro COPYINOUT_END 745 movq %rax,PCB_ONFAULT(%r11) 746 POP_FRAME_POINTER 747 .endm 748 749 .macro COPYINOUT_SMAP_END 750 SMAP_ENABLE smap=1 751 COPYINOUT_END 752 .endm 753 754 /* 755 * copyout(from_kernel, to_user, len) 756 * %rdi, %rsi, %rdx 757 */ 758 .macro COPYOUT smap erms 759 PUSH_FRAME_POINTER 760 movq PCPU(CURPCB),%r11 761 movq $copy_fault,PCB_ONFAULT(%r11) 762 763 /* 764 * Check explicitly for non-user addresses. 765 * First, prevent address wrapping. 766 */ 767 movq %rsi,%rax 768 addq %rdx,%rax 769 jc copy_fault 770 /* 771 * XXX STOP USING VM_MAXUSER_ADDRESS. 772 * It is an end address, not a max, so every time it is used correctly it 773 * looks like there is an off by one error, and of course it caused an off 774 * by one error in several places. 775 */ 776 movq $VM_MAXUSER_ADDRESS,%rcx 777 cmpq %rcx,%rax 778 ja copy_fault 779 780 /* 781 * Set return value to zero. Remaining failure mode goes through 782 * copy_fault. 783 */ 784 xorl %eax,%eax 785 786 /* 787 * Set up arguments for MEMMOVE. 788 */ 789 movq %rdi,%r8 790 movq %rsi,%rdi 791 movq %r8,%rsi 792 movq %rdx,%rcx 793 794 795 SMAP_DISABLE \smap 796 .if \smap == 1 797 MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_SMAP_END 798 .else 799 MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_END 800 .endif 801 /* NOTREACHED */ 802 .endm 803 804 ENTRY(copyout_nosmap_std) 805 COPYOUT smap=0 erms=0 806 END(copyout_nosmap_std) 807 808 ENTRY(copyout_smap_std) 809 COPYOUT smap=1 erms=0 810 END(copyout_smap_std) 811 812 ENTRY(copyout_nosmap_erms) 813 COPYOUT smap=0 erms=1 814 END(copyout_nosmap_erms) 815 816 ENTRY(copyout_smap_erms) 817 COPYOUT smap=1 erms=1 818 END(copyout_smap_erms) 819 820 /* 821 * copyin(from_user, to_kernel, len) 822 * %rdi, %rsi, %rdx 823 */ 824 .macro COPYIN smap erms 825 PUSH_FRAME_POINTER 826 movq PCPU(CURPCB),%r11 827 movq $copy_fault,PCB_ONFAULT(%r11) 828 829 /* 830 * make sure address is valid 831 */ 832 movq %rdi,%rax 833 addq %rdx,%rax 834 jc copy_fault 835 movq $VM_MAXUSER_ADDRESS,%rcx 836 cmpq %rcx,%rax 837 ja copy_fault 838 839 xorl %eax,%eax 840 841 movq %rdi,%r8 842 movq %rsi,%rdi 843 movq %r8,%rsi 844 movq %rdx,%rcx 845 846 SMAP_DISABLE \smap 847 .if \smap == 1 848 MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_SMAP_END 849 .else 850 MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_END 851 .endif 852 /* NOTREACHED */ 853 .endm 854 855 ENTRY(copyin_nosmap_std) 856 COPYIN smap=0 erms=0 857 END(copyin_nosmap_std) 858 859 ENTRY(copyin_smap_std) 860 COPYIN smap=1 erms=0 861 END(copyin_smap_std) 862 863 ENTRY(copyin_nosmap_erms) 864 COPYIN smap=0 erms=1 865 END(copyin_nosmap_erms) 866 867 ENTRY(copyin_smap_erms) 868 COPYIN smap=1 erms=1 869 END(copyin_smap_erms) 870 871 ALIGN_TEXT 872 copy_fault: 873 testl $CPUID_STDEXT_SMAP,cpu_stdext_feature(%rip) 874 je 1f 875 clac 876 1: movq $0,PCB_ONFAULT(%r11) 877 movl $EFAULT,%eax 878 POP_FRAME_POINTER 879 ret 880 881 /* 882 * casueword32. Compare and set user integer. Returns -1 on fault, 883 * 0 if access was successful. Old value is written to *oldp. 884 * dst = %rdi, old = %esi, oldp = %rdx, new = %ecx 885 */ 886 ENTRY(casueword32_nosmap) 887 PUSH_FRAME_POINTER 888 movq PCPU(CURPCB),%r8 889 movq $fusufault,PCB_ONFAULT(%r8) 890 891 movq $VM_MAXUSER_ADDRESS-4,%rax 892 cmpq %rax,%rdi /* verify address is valid */ 893 ja fusufault 894 895 movl %esi,%eax /* old */ 896 #ifdef SMP 897 lock 898 #endif 899 cmpxchgl %ecx,(%rdi) /* new = %ecx */ 900 setne %cl 901 902 /* 903 * The old value is in %eax. If the store succeeded it will be the 904 * value we expected (old) from before the store, otherwise it will 905 * be the current value. Save %eax into %esi to prepare the return 906 * value. 907 */ 908 movl %eax,%esi 909 xorl %eax,%eax 910 movq %rax,PCB_ONFAULT(%r8) 911 912 /* 913 * Access the oldp after the pcb_onfault is cleared, to correctly 914 * catch corrupted pointer. 915 */ 916 movl %esi,(%rdx) /* oldp = %rdx */ 917 POP_FRAME_POINTER 918 movzbl %cl, %eax 919 ret 920 END(casueword32_nosmap) 921 922 ENTRY(casueword32_smap) 923 PUSH_FRAME_POINTER 924 movq PCPU(CURPCB),%r8 925 movq $fusufault,PCB_ONFAULT(%r8) 926 927 movq $VM_MAXUSER_ADDRESS-4,%rax 928 cmpq %rax,%rdi /* verify address is valid */ 929 ja fusufault 930 931 movl %esi,%eax /* old */ 932 stac 933 #ifdef SMP 934 lock 935 #endif 936 cmpxchgl %ecx,(%rdi) /* new = %ecx */ 937 clac 938 setne %cl 939 940 /* 941 * The old value is in %eax. If the store succeeded it will be the 942 * value we expected (old) from before the store, otherwise it will 943 * be the current value. Save %eax into %esi to prepare the return 944 * value. 945 */ 946 movl %eax,%esi 947 xorl %eax,%eax 948 movq %rax,PCB_ONFAULT(%r8) 949 950 /* 951 * Access the oldp after the pcb_onfault is cleared, to correctly 952 * catch corrupted pointer. 953 */ 954 movl %esi,(%rdx) /* oldp = %rdx */ 955 POP_FRAME_POINTER 956 movzbl %cl, %eax 957 ret 958 END(casueword32_smap) 959 960 /* 961 * casueword. Compare and set user long. Returns -1 on fault, 962 * 0 if access was successful. Old value is written to *oldp. 963 * dst = %rdi, old = %rsi, oldp = %rdx, new = %rcx 964 */ 965 ENTRY(casueword_nosmap) 966 PUSH_FRAME_POINTER 967 movq PCPU(CURPCB),%r8 968 movq $fusufault,PCB_ONFAULT(%r8) 969 970 movq $VM_MAXUSER_ADDRESS-4,%rax 971 cmpq %rax,%rdi /* verify address is valid */ 972 ja fusufault 973 974 movq %rsi,%rax /* old */ 975 #ifdef SMP 976 lock 977 #endif 978 cmpxchgq %rcx,(%rdi) /* new = %rcx */ 979 setne %cl 980 981 /* 982 * The old value is in %rax. If the store succeeded it will be the 983 * value we expected (old) from before the store, otherwise it will 984 * be the current value. 985 */ 986 movq %rax,%rsi 987 xorl %eax,%eax 988 movq %rax,PCB_ONFAULT(%r8) 989 movq %rsi,(%rdx) 990 POP_FRAME_POINTER 991 movzbl %cl, %eax 992 ret 993 END(casueword_nosmap) 994 995 ENTRY(casueword_smap) 996 PUSH_FRAME_POINTER 997 movq PCPU(CURPCB),%r8 998 movq $fusufault,PCB_ONFAULT(%r8) 999 1000 movq $VM_MAXUSER_ADDRESS-4,%rax 1001 cmpq %rax,%rdi /* verify address is valid */ 1002 ja fusufault 1003 1004 movq %rsi,%rax /* old */ 1005 stac 1006 #ifdef SMP 1007 lock 1008 #endif 1009 cmpxchgq %rcx,(%rdi) /* new = %rcx */ 1010 clac 1011 setne %cl 1012 1013 /* 1014 * The old value is in %rax. If the store succeeded it will be the 1015 * value we expected (old) from before the store, otherwise it will 1016 * be the current value. 1017 */ 1018 movq %rax,%rsi 1019 xorl %eax,%eax 1020 movq %rax,PCB_ONFAULT(%r8) 1021 movq %rsi,(%rdx) 1022 POP_FRAME_POINTER 1023 movzbl %cl, %eax 1024 ret 1025 END(casueword_smap) 1026 1027 /* 1028 * Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit 1029 * byte from user memory. 1030 * addr = %rdi, valp = %rsi 1031 */ 1032 1033 ENTRY(fueword_nosmap) 1034 PUSH_FRAME_POINTER 1035 movq PCPU(CURPCB),%rcx 1036 movq $fusufault,PCB_ONFAULT(%rcx) 1037 1038 movq $VM_MAXUSER_ADDRESS-8,%rax 1039 cmpq %rax,%rdi /* verify address is valid */ 1040 ja fusufault 1041 1042 xorl %eax,%eax 1043 movq (%rdi),%r11 1044 movq %rax,PCB_ONFAULT(%rcx) 1045 movq %r11,(%rsi) 1046 POP_FRAME_POINTER 1047 ret 1048 END(fueword_nosmap) 1049 1050 ENTRY(fueword_smap) 1051 PUSH_FRAME_POINTER 1052 movq PCPU(CURPCB),%rcx 1053 movq $fusufault,PCB_ONFAULT(%rcx) 1054 1055 movq $VM_MAXUSER_ADDRESS-8,%rax 1056 cmpq %rax,%rdi /* verify address is valid */ 1057 ja fusufault 1058 1059 xorl %eax,%eax 1060 stac 1061 movq (%rdi),%r11 1062 clac 1063 movq %rax,PCB_ONFAULT(%rcx) 1064 movq %r11,(%rsi) 1065 POP_FRAME_POINTER 1066 ret 1067 END(fueword_smap) 1068 1069 ENTRY(fueword32_nosmap) 1070 PUSH_FRAME_POINTER 1071 movq PCPU(CURPCB),%rcx 1072 movq $fusufault,PCB_ONFAULT(%rcx) 1073 1074 movq $VM_MAXUSER_ADDRESS-4,%rax 1075 cmpq %rax,%rdi /* verify address is valid */ 1076 ja fusufault 1077 1078 xorl %eax,%eax 1079 movl (%rdi),%r11d 1080 movq %rax,PCB_ONFAULT(%rcx) 1081 movl %r11d,(%rsi) 1082 POP_FRAME_POINTER 1083 ret 1084 END(fueword32_nosmap) 1085 1086 ENTRY(fueword32_smap) 1087 PUSH_FRAME_POINTER 1088 movq PCPU(CURPCB),%rcx 1089 movq $fusufault,PCB_ONFAULT(%rcx) 1090 1091 movq $VM_MAXUSER_ADDRESS-4,%rax 1092 cmpq %rax,%rdi /* verify address is valid */ 1093 ja fusufault 1094 1095 xorl %eax,%eax 1096 stac 1097 movl (%rdi),%r11d 1098 clac 1099 movq %rax,PCB_ONFAULT(%rcx) 1100 movl %r11d,(%rsi) 1101 POP_FRAME_POINTER 1102 ret 1103 END(fueword32_smap) 1104 1105 ENTRY(fuword16_nosmap) 1106 PUSH_FRAME_POINTER 1107 movq PCPU(CURPCB),%rcx 1108 movq $fusufault,PCB_ONFAULT(%rcx) 1109 1110 movq $VM_MAXUSER_ADDRESS-2,%rax 1111 cmpq %rax,%rdi 1112 ja fusufault 1113 1114 movzwl (%rdi),%eax 1115 movq $0,PCB_ONFAULT(%rcx) 1116 POP_FRAME_POINTER 1117 ret 1118 END(fuword16_nosmap) 1119 1120 ENTRY(fuword16_smap) 1121 PUSH_FRAME_POINTER 1122 movq PCPU(CURPCB),%rcx 1123 movq $fusufault,PCB_ONFAULT(%rcx) 1124 1125 movq $VM_MAXUSER_ADDRESS-2,%rax 1126 cmpq %rax,%rdi 1127 ja fusufault 1128 1129 stac 1130 movzwl (%rdi),%eax 1131 clac 1132 movq $0,PCB_ONFAULT(%rcx) 1133 POP_FRAME_POINTER 1134 ret 1135 END(fuword16_smap) 1136 1137 ENTRY(fubyte_nosmap) 1138 PUSH_FRAME_POINTER 1139 movq PCPU(CURPCB),%rcx 1140 movq $fusufault,PCB_ONFAULT(%rcx) 1141 1142 movq $VM_MAXUSER_ADDRESS-1,%rax 1143 cmpq %rax,%rdi 1144 ja fusufault 1145 1146 movzbl (%rdi),%eax 1147 movq $0,PCB_ONFAULT(%rcx) 1148 POP_FRAME_POINTER 1149 ret 1150 END(fubyte_nosmap) 1151 1152 ENTRY(fubyte_smap) 1153 PUSH_FRAME_POINTER 1154 movq PCPU(CURPCB),%rcx 1155 movq $fusufault,PCB_ONFAULT(%rcx) 1156 1157 movq $VM_MAXUSER_ADDRESS-1,%rax 1158 cmpq %rax,%rdi 1159 ja fusufault 1160 1161 stac 1162 movzbl (%rdi),%eax 1163 clac 1164 movq $0,PCB_ONFAULT(%rcx) 1165 POP_FRAME_POINTER 1166 ret 1167 END(fubyte_smap) 1168 1169 /* 1170 * Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to 1171 * user memory. 1172 * addr = %rdi, value = %rsi 1173 */ 1174 ENTRY(suword_nosmap) 1175 PUSH_FRAME_POINTER 1176 movq PCPU(CURPCB),%rcx 1177 movq $fusufault,PCB_ONFAULT(%rcx) 1178 1179 movq $VM_MAXUSER_ADDRESS-8,%rax 1180 cmpq %rax,%rdi /* verify address validity */ 1181 ja fusufault 1182 1183 movq %rsi,(%rdi) 1184 xorl %eax,%eax 1185 movq %rax,PCB_ONFAULT(%rcx) 1186 POP_FRAME_POINTER 1187 ret 1188 END(suword_nosmap) 1189 1190 ENTRY(suword_smap) 1191 PUSH_FRAME_POINTER 1192 movq PCPU(CURPCB),%rcx 1193 movq $fusufault,PCB_ONFAULT(%rcx) 1194 1195 movq $VM_MAXUSER_ADDRESS-8,%rax 1196 cmpq %rax,%rdi /* verify address validity */ 1197 ja fusufault 1198 1199 stac 1200 movq %rsi,(%rdi) 1201 clac 1202 xorl %eax,%eax 1203 movq %rax,PCB_ONFAULT(%rcx) 1204 POP_FRAME_POINTER 1205 ret 1206 END(suword_smap) 1207 1208 ENTRY(suword32_nosmap) 1209 PUSH_FRAME_POINTER 1210 movq PCPU(CURPCB),%rcx 1211 movq $fusufault,PCB_ONFAULT(%rcx) 1212 1213 movq $VM_MAXUSER_ADDRESS-4,%rax 1214 cmpq %rax,%rdi /* verify address validity */ 1215 ja fusufault 1216 1217 movl %esi,(%rdi) 1218 xorl %eax,%eax 1219 movq %rax,PCB_ONFAULT(%rcx) 1220 POP_FRAME_POINTER 1221 ret 1222 END(suword32_nosmap) 1223 1224 ENTRY(suword32_smap) 1225 PUSH_FRAME_POINTER 1226 movq PCPU(CURPCB),%rcx 1227 movq $fusufault,PCB_ONFAULT(%rcx) 1228 1229 movq $VM_MAXUSER_ADDRESS-4,%rax 1230 cmpq %rax,%rdi /* verify address validity */ 1231 ja fusufault 1232 1233 stac 1234 movl %esi,(%rdi) 1235 clac 1236 xorl %eax,%eax 1237 movq %rax,PCB_ONFAULT(%rcx) 1238 POP_FRAME_POINTER 1239 ret 1240 END(suword32_smap) 1241 1242 ENTRY(suword16_nosmap) 1243 PUSH_FRAME_POINTER 1244 movq PCPU(CURPCB),%rcx 1245 movq $fusufault,PCB_ONFAULT(%rcx) 1246 1247 movq $VM_MAXUSER_ADDRESS-2,%rax 1248 cmpq %rax,%rdi /* verify address validity */ 1249 ja fusufault 1250 1251 movw %si,(%rdi) 1252 xorl %eax,%eax 1253 movq %rax,PCB_ONFAULT(%rcx) 1254 POP_FRAME_POINTER 1255 ret 1256 END(suword16_nosmap) 1257 1258 ENTRY(suword16_smap) 1259 PUSH_FRAME_POINTER 1260 movq PCPU(CURPCB),%rcx 1261 movq $fusufault,PCB_ONFAULT(%rcx) 1262 1263 movq $VM_MAXUSER_ADDRESS-2,%rax 1264 cmpq %rax,%rdi /* verify address validity */ 1265 ja fusufault 1266 1267 stac 1268 movw %si,(%rdi) 1269 clac 1270 xorl %eax,%eax 1271 movq %rax,PCB_ONFAULT(%rcx) 1272 POP_FRAME_POINTER 1273 ret 1274 END(suword16_smap) 1275 1276 ENTRY(subyte_nosmap) 1277 PUSH_FRAME_POINTER 1278 movq PCPU(CURPCB),%rcx 1279 movq $fusufault,PCB_ONFAULT(%rcx) 1280 1281 movq $VM_MAXUSER_ADDRESS-1,%rax 1282 cmpq %rax,%rdi /* verify address validity */ 1283 ja fusufault 1284 1285 movl %esi,%eax 1286 movb %al,(%rdi) 1287 xorl %eax,%eax 1288 movq %rax,PCB_ONFAULT(%rcx) 1289 POP_FRAME_POINTER 1290 ret 1291 END(subyte_nosmap) 1292 1293 ENTRY(subyte_smap) 1294 PUSH_FRAME_POINTER 1295 movq PCPU(CURPCB),%rcx 1296 movq $fusufault,PCB_ONFAULT(%rcx) 1297 1298 movq $VM_MAXUSER_ADDRESS-1,%rax 1299 cmpq %rax,%rdi /* verify address validity */ 1300 ja fusufault 1301 1302 movl %esi,%eax 1303 stac 1304 movb %al,(%rdi) 1305 clac 1306 xorl %eax,%eax 1307 movq %rax,PCB_ONFAULT(%rcx) 1308 POP_FRAME_POINTER 1309 ret 1310 END(subyte_smap) 1311 1312 ALIGN_TEXT 1313 fusufault: 1314 testl $CPUID_STDEXT_SMAP,cpu_stdext_feature(%rip) 1315 je 1f 1316 clac 1317 1: movq PCPU(CURPCB),%rcx 1318 xorl %eax,%eax 1319 movq %rax,PCB_ONFAULT(%rcx) 1320 decq %rax 1321 POP_FRAME_POINTER 1322 ret 1323 1324 /* 1325 * copyinstr(from, to, maxlen, int *lencopied) 1326 * %rdi, %rsi, %rdx, %rcx 1327 * 1328 * copy a string from 'from' to 'to', stop when a 0 character is reached. 1329 * return ENAMETOOLONG if string is longer than maxlen, and 1330 * EFAULT on protection violations. If lencopied is non-zero, 1331 * return the actual length in *lencopied. 1332 */ 1333 .macro COPYINSTR smap 1334 PUSH_FRAME_POINTER 1335 movq %rdx,%r8 /* %r8 = maxlen */ 1336 movq PCPU(CURPCB),%r9 1337 movq $cpystrflt,PCB_ONFAULT(%r9) 1338 1339 movq $VM_MAXUSER_ADDRESS,%rax 1340 1341 /* make sure 'from' is within bounds */ 1342 subq %rdi,%rax 1343 jbe cpystrflt 1344 1345 SMAP_DISABLE \smap 1346 1347 /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */ 1348 cmpq %rdx,%rax 1349 jb 8f 1350 1: 1351 incq %rdx 1352 2: 1353 decq %rdx 1354 .if \smap == 0 1355 jz copyinstr_toolong 1356 .else 1357 jz copyinstr_toolong_smap 1358 .endif 1359 1360 movb (%rdi),%al 1361 movb %al,(%rsi) 1362 incq %rsi 1363 incq %rdi 1364 testb %al,%al 1365 jnz 2b 1366 1367 SMAP_ENABLE \smap 1368 1369 /* Success -- 0 byte reached */ 1370 decq %rdx 1371 xorl %eax,%eax 1372 1373 /* set *lencopied and return %eax */ 1374 movq %rax,PCB_ONFAULT(%r9) 1375 1376 testq %rcx,%rcx 1377 jz 3f 1378 subq %rdx,%r8 1379 movq %r8,(%rcx) 1380 3: 1381 POP_FRAME_POINTER 1382 ret 1383 ALIGN_TEXT 1384 8: 1385 movq %rax,%rdx 1386 movq %rax,%r8 1387 jmp 1b 1388 1389 .endm 1390 1391 ENTRY(copyinstr_nosmap) 1392 COPYINSTR smap=0 1393 END(copyinstr_nosmap) 1394 1395 ENTRY(copyinstr_smap) 1396 COPYINSTR smap=1 1397 END(copyinstr_smap) 1398 1399 cpystrflt: 1400 testl $CPUID_STDEXT_SMAP,cpu_stdext_feature(%rip) 1401 je 1f 1402 clac 1403 1: movl $EFAULT,%eax 1404 cpystrflt_x: 1405 /* set *lencopied and return %eax */ 1406 movq $0,PCB_ONFAULT(%r9) 1407 1408 testq %rcx,%rcx 1409 jz 1f 1410 subq %rdx,%r8 1411 movq %r8,(%rcx) 1412 1: 1413 POP_FRAME_POINTER 1414 ret 1415 1416 copyinstr_toolong_smap: 1417 clac 1418 copyinstr_toolong: 1419 /* rdx is zero - return ENAMETOOLONG or EFAULT */ 1420 movq $VM_MAXUSER_ADDRESS,%rax 1421 cmpq %rax,%rdi 1422 jae cpystrflt 1423 movl $ENAMETOOLONG,%eax 1424 jmp cpystrflt_x 1425 1426 /* 1427 * copystr(from, to, maxlen, int *lencopied) 1428 * %rdi, %rsi, %rdx, %rcx 1429 */ 1430 ENTRY(copystr) 1431 PUSH_FRAME_POINTER 1432 movq %rdx,%r8 /* %r8 = maxlen */ 1433 1434 incq %rdx 1435 1: 1436 decq %rdx 1437 jz 4f 1438 movb (%rdi),%al 1439 movb %al,(%rsi) 1440 incq %rsi 1441 incq %rdi 1442 testb %al,%al 1443 jnz 1b 1444 1445 /* Success -- 0 byte reached */ 1446 decq %rdx 1447 xorl %eax,%eax 1448 2: 1449 testq %rcx,%rcx 1450 jz 3f 1451 /* set *lencopied and return %rax */ 1452 subq %rdx,%r8 1453 movq %r8,(%rcx) 1454 3: 1455 POP_FRAME_POINTER 1456 ret 1457 4: 1458 /* rdx is zero -- return ENAMETOOLONG */ 1459 movl $ENAMETOOLONG,%eax 1460 jmp 2b 1461 END(copystr) 1462 1463 /* 1464 * Handling of special amd64 registers and descriptor tables etc 1465 */ 1466 /* void lgdt(struct region_descriptor *rdp); */ 1467 ENTRY(lgdt) 1468 /* reload the descriptor table */ 1469 lgdt (%rdi) 1470 1471 /* flush the prefetch q */ 1472 jmp 1f 1473 nop 1474 1: 1475 movl $KDSEL,%eax 1476 movl %eax,%ds 1477 movl %eax,%es 1478 movl %eax,%fs /* Beware, use wrmsr to set 64 bit base */ 1479 movl %eax,%gs 1480 movl %eax,%ss 1481 1482 /* reload code selector by turning return into intersegmental return */ 1483 popq %rax 1484 pushq $KCSEL 1485 pushq %rax 1486 MEXITCOUNT 1487 lretq 1488 END(lgdt) 1489 1490 /*****************************************************************************/ 1491 /* setjump, longjump */ 1492 /*****************************************************************************/ 1493 1494 ENTRY(setjmp) 1495 movq %rbx,0(%rdi) /* save rbx */ 1496 movq %rsp,8(%rdi) /* save rsp */ 1497 movq %rbp,16(%rdi) /* save rbp */ 1498 movq %r12,24(%rdi) /* save r12 */ 1499 movq %r13,32(%rdi) /* save r13 */ 1500 movq %r14,40(%rdi) /* save r14 */ 1501 movq %r15,48(%rdi) /* save r15 */ 1502 movq 0(%rsp),%rdx /* get rta */ 1503 movq %rdx,56(%rdi) /* save rip */ 1504 xorl %eax,%eax /* return(0); */ 1505 ret 1506 END(setjmp) 1507 1508 ENTRY(longjmp) 1509 movq 0(%rdi),%rbx /* restore rbx */ 1510 movq 8(%rdi),%rsp /* restore rsp */ 1511 movq 16(%rdi),%rbp /* restore rbp */ 1512 movq 24(%rdi),%r12 /* restore r12 */ 1513 movq 32(%rdi),%r13 /* restore r13 */ 1514 movq 40(%rdi),%r14 /* restore r14 */ 1515 movq 48(%rdi),%r15 /* restore r15 */ 1516 movq 56(%rdi),%rdx /* get rta */ 1517 movq %rdx,0(%rsp) /* put in return frame */ 1518 xorl %eax,%eax /* return(1); */ 1519 incl %eax 1520 ret 1521 END(longjmp) 1522 1523 /* 1524 * Support for reading MSRs in the safe manner. (Instead of panic on #gp, 1525 * return an error.) 1526 */ 1527 ENTRY(rdmsr_safe) 1528 /* int rdmsr_safe(u_int msr, uint64_t *data) */ 1529 PUSH_FRAME_POINTER 1530 movq PCPU(CURPCB),%r8 1531 movq $msr_onfault,PCB_ONFAULT(%r8) 1532 movl %edi,%ecx 1533 rdmsr /* Read MSR pointed by %ecx. Returns 1534 hi byte in edx, lo in %eax */ 1535 salq $32,%rdx /* sign-shift %rdx left */ 1536 movl %eax,%eax /* zero-extend %eax -> %rax */ 1537 orq %rdx,%rax 1538 movq %rax,(%rsi) 1539 xorq %rax,%rax 1540 movq %rax,PCB_ONFAULT(%r8) 1541 POP_FRAME_POINTER 1542 ret 1543 1544 /* 1545 * Support for writing MSRs in the safe manner. (Instead of panic on #gp, 1546 * return an error.) 1547 */ 1548 ENTRY(wrmsr_safe) 1549 /* int wrmsr_safe(u_int msr, uint64_t data) */ 1550 PUSH_FRAME_POINTER 1551 movq PCPU(CURPCB),%r8 1552 movq $msr_onfault,PCB_ONFAULT(%r8) 1553 movl %edi,%ecx 1554 movl %esi,%eax 1555 sarq $32,%rsi 1556 movl %esi,%edx 1557 wrmsr /* Write MSR pointed by %ecx. Accepts 1558 hi byte in edx, lo in %eax. */ 1559 xorq %rax,%rax 1560 movq %rax,PCB_ONFAULT(%r8) 1561 POP_FRAME_POINTER 1562 ret 1563 1564 /* 1565 * MSR operations fault handler 1566 */ 1567 ALIGN_TEXT 1568 msr_onfault: 1569 movq $0,PCB_ONFAULT(%r8) 1570 movl $EFAULT,%eax 1571 POP_FRAME_POINTER 1572 ret 1573 1574 /* 1575 * void pmap_pti_pcid_invalidate(uint64_t ucr3, uint64_t kcr3); 1576 * Invalidates address space addressed by ucr3, then returns to kcr3. 1577 * Done in assembler to ensure no other memory accesses happen while 1578 * on ucr3. 1579 */ 1580 ALIGN_TEXT 1581 ENTRY(pmap_pti_pcid_invalidate) 1582 pushfq 1583 cli 1584 movq %rdi,%cr3 /* to user page table */ 1585 movq %rsi,%cr3 /* back to kernel */ 1586 popfq 1587 retq 1588 1589 /* 1590 * void pmap_pti_pcid_invlpg(uint64_t ucr3, uint64_t kcr3, vm_offset_t va); 1591 * Invalidates virtual address va in address space ucr3, then returns to kcr3. 1592 */ 1593 ALIGN_TEXT 1594 ENTRY(pmap_pti_pcid_invlpg) 1595 pushfq 1596 cli 1597 movq %rdi,%cr3 /* to user page table */ 1598 invlpg (%rdx) 1599 movq %rsi,%cr3 /* back to kernel */ 1600 popfq 1601 retq 1602 1603 /* 1604 * void pmap_pti_pcid_invlrng(uint64_t ucr3, uint64_t kcr3, vm_offset_t sva, 1605 * vm_offset_t eva); 1606 * Invalidates virtual addresses between sva and eva in address space ucr3, 1607 * then returns to kcr3. 1608 */ 1609 ALIGN_TEXT 1610 ENTRY(pmap_pti_pcid_invlrng) 1611 pushfq 1612 cli 1613 movq %rdi,%cr3 /* to user page table */ 1614 1: invlpg (%rdx) 1615 addq $PAGE_SIZE,%rdx 1616 cmpq %rdx,%rcx 1617 ja 1b 1618 movq %rsi,%cr3 /* back to kernel */ 1619 popfq 1620 retq 1621 1622 .altmacro 1623 .macro rsb_seq_label l 1624 rsb_seq_\l: 1625 .endm 1626 .macro rsb_call_label l 1627 call rsb_seq_\l 1628 .endm 1629 .macro rsb_seq count 1630 ll=1 1631 .rept \count 1632 rsb_call_label %(ll) 1633 nop 1634 rsb_seq_label %(ll) 1635 addq $8,%rsp 1636 ll=ll+1 1637 .endr 1638 .endm 1639 1640 ENTRY(rsb_flush) 1641 rsb_seq 32 1642 ret 1643 1644 /* all callers already saved %rax, %rdx, and %rcx */ 1645 ENTRY(handle_ibrs_entry) 1646 cmpb $0,hw_ibrs_ibpb_active(%rip) 1647 je 1f 1648 movl $MSR_IA32_SPEC_CTRL,%ecx 1649 rdmsr 1650 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax 1651 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32,%edx 1652 wrmsr 1653 movb $1,PCPU(IBPB_SET) 1654 testl $CPUID_STDEXT_SMEP,cpu_stdext_feature(%rip) 1655 je rsb_flush 1656 1: ret 1657 END(handle_ibrs_entry) 1658 1659 ENTRY(handle_ibrs_exit) 1660 cmpb $0,PCPU(IBPB_SET) 1661 je 1f 1662 movl $MSR_IA32_SPEC_CTRL,%ecx 1663 rdmsr 1664 andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax 1665 andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx 1666 wrmsr 1667 movb $0,PCPU(IBPB_SET) 1668 1: ret 1669 END(handle_ibrs_exit) 1670 1671 /* registers-neutral version, but needs stack */ 1672 ENTRY(handle_ibrs_exit_rs) 1673 cmpb $0,PCPU(IBPB_SET) 1674 je 1f 1675 pushq %rax 1676 pushq %rdx 1677 pushq %rcx 1678 movl $MSR_IA32_SPEC_CTRL,%ecx 1679 rdmsr 1680 andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax 1681 andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx 1682 wrmsr 1683 popq %rcx 1684 popq %rdx 1685 popq %rax 1686 movb $0,PCPU(IBPB_SET) 1687 1: ret 1688 END(handle_ibrs_exit_rs) 1689 1690 .noaltmacro 1691 1692 /* 1693 * Flush L1D cache. Load enough of the data from the kernel text 1694 * to flush existing L1D content. 1695 * 1696 * N.B. The function does not follow ABI calling conventions, it corrupts %rbx. 1697 * The vmm.ko caller expects that only %rax, %rdx, %rbx, %rcx, %r9, and %rflags 1698 * registers are clobbered. The NMI handler caller only needs %r13 preserved. 1699 */ 1700 ENTRY(flush_l1d_sw) 1701 #define L1D_FLUSH_SIZE (64 * 1024) 1702 movq $KERNBASE, %r9 1703 movq $-L1D_FLUSH_SIZE, %rcx 1704 /* 1705 * pass 1: Preload TLB. 1706 * Kernel text is mapped using superpages. TLB preload is 1707 * done for the benefit of older CPUs which split 2M page 1708 * into 4k TLB entries. 1709 */ 1710 1: movb L1D_FLUSH_SIZE(%r9, %rcx), %al 1711 addq $PAGE_SIZE, %rcx 1712 jne 1b 1713 xorl %eax, %eax 1714 cpuid 1715 movq $-L1D_FLUSH_SIZE, %rcx 1716 /* pass 2: Read each cache line. */ 1717 2: movb L1D_FLUSH_SIZE(%r9, %rcx), %al 1718 addq $64, %rcx 1719 jne 2b 1720 lfence 1721 ret 1722 #undef L1D_FLUSH_SIZE 1723 END(flush_l1d_sw) 1724 1725 ENTRY(flush_l1d_sw_abi) 1726 pushq %rbx 1727 call flush_l1d_sw 1728 popq %rbx 1729 ret 1730 END(flush_l1d_sw_abi) 1731 1732 ENTRY(mds_handler_void) 1733 retq 1734 END(mds_handler_void) 1735 1736 ENTRY(mds_handler_verw) 1737 subq $8, %rsp 1738 movw %ds, (%rsp) 1739 verw (%rsp) 1740 addq $8, %rsp 1741 retq 1742 END(mds_handler_verw) 1743 1744 ENTRY(mds_handler_ivb) 1745 pushq %rax 1746 pushq %rdx 1747 pushq %rcx 1748 1749 movq %cr0, %rax 1750 testb $CR0_TS, %al 1751 je 1f 1752 clts 1753 1: movq PCPU(MDS_BUF), %rdx 1754 movdqa %xmm0, PCPU(MDS_TMP) 1755 pxor %xmm0, %xmm0 1756 1757 lfence 1758 orpd (%rdx), %xmm0 1759 orpd (%rdx), %xmm0 1760 mfence 1761 movl $40, %ecx 1762 addq $16, %rdx 1763 2: movntdq %xmm0, (%rdx) 1764 addq $16, %rdx 1765 decl %ecx 1766 jnz 2b 1767 mfence 1768 1769 movdqa PCPU(MDS_TMP),%xmm0 1770 testb $CR0_TS, %al 1771 je 3f 1772 movq %rax, %cr0 1773 3: popq %rcx 1774 popq %rdx 1775 popq %rax 1776 retq 1777 END(mds_handler_ivb) 1778 1779 ENTRY(mds_handler_bdw) 1780 pushq %rax 1781 pushq %rbx 1782 pushq %rcx 1783 pushq %rdi 1784 pushq %rsi 1785 1786 movq %cr0, %rax 1787 testb $CR0_TS, %al 1788 je 1f 1789 clts 1790 1: movq PCPU(MDS_BUF), %rbx 1791 movdqa %xmm0, PCPU(MDS_TMP) 1792 pxor %xmm0, %xmm0 1793 1794 movq %rbx, %rdi 1795 movq %rbx, %rsi 1796 movl $40, %ecx 1797 2: movntdq %xmm0, (%rbx) 1798 addq $16, %rbx 1799 decl %ecx 1800 jnz 2b 1801 mfence 1802 movl $1536, %ecx 1803 rep; movsb 1804 lfence 1805 1806 movdqa PCPU(MDS_TMP),%xmm0 1807 testb $CR0_TS, %al 1808 je 3f 1809 movq %rax, %cr0 1810 3: popq %rsi 1811 popq %rdi 1812 popq %rcx 1813 popq %rbx 1814 popq %rax 1815 retq 1816 END(mds_handler_bdw) 1817 1818 ENTRY(mds_handler_skl_sse) 1819 pushq %rax 1820 pushq %rdx 1821 pushq %rcx 1822 pushq %rdi 1823 1824 movq %cr0, %rax 1825 testb $CR0_TS, %al 1826 je 1f 1827 clts 1828 1: movq PCPU(MDS_BUF), %rdi 1829 movq PCPU(MDS_BUF64), %rdx 1830 movdqa %xmm0, PCPU(MDS_TMP) 1831 pxor %xmm0, %xmm0 1832 1833 lfence 1834 orpd (%rdx), %xmm0 1835 orpd (%rdx), %xmm0 1836 xorl %eax, %eax 1837 2: clflushopt 5376(%rdi, %rax, 8) 1838 addl $8, %eax 1839 cmpl $8 * 12, %eax 1840 jb 2b 1841 sfence 1842 movl $6144, %ecx 1843 xorl %eax, %eax 1844 rep; stosb 1845 mfence 1846 1847 movdqa PCPU(MDS_TMP), %xmm0 1848 testb $CR0_TS, %al 1849 je 3f 1850 movq %rax, %cr0 1851 3: popq %rdi 1852 popq %rcx 1853 popq %rdx 1854 popq %rax 1855 retq 1856 END(mds_handler_skl_sse) 1857 1858 ENTRY(mds_handler_skl_avx) 1859 pushq %rax 1860 pushq %rdx 1861 pushq %rcx 1862 pushq %rdi 1863 1864 movq %cr0, %rax 1865 testb $CR0_TS, %al 1866 je 1f 1867 clts 1868 1: movq PCPU(MDS_BUF), %rdi 1869 movq PCPU(MDS_BUF64), %rdx 1870 vmovdqa %ymm0, PCPU(MDS_TMP) 1871 vpxor %ymm0, %ymm0, %ymm0 1872 1873 lfence 1874 vorpd (%rdx), %ymm0, %ymm0 1875 vorpd (%rdx), %ymm0, %ymm0 1876 xorl %eax, %eax 1877 2: clflushopt 5376(%rdi, %rax, 8) 1878 addl $8, %eax 1879 cmpl $8 * 12, %eax 1880 jb 2b 1881 sfence 1882 movl $6144, %ecx 1883 xorl %eax, %eax 1884 rep; stosb 1885 mfence 1886 1887 vmovdqa PCPU(MDS_TMP), %ymm0 1888 testb $CR0_TS, %al 1889 je 3f 1890 movq %rax, %cr0 1891 3: popq %rdi 1892 popq %rcx 1893 popq %rdx 1894 popq %rax 1895 retq 1896 END(mds_handler_skl_avx) 1897 1898 ENTRY(mds_handler_skl_avx512) 1899 pushq %rax 1900 pushq %rdx 1901 pushq %rcx 1902 pushq %rdi 1903 1904 movq %cr0, %rax 1905 testb $CR0_TS, %al 1906 je 1f 1907 clts 1908 1: movq PCPU(MDS_BUF), %rdi 1909 movq PCPU(MDS_BUF64), %rdx 1910 vmovdqa64 %zmm0, PCPU(MDS_TMP) 1911 vpxord %zmm0, %zmm0, %zmm0 1912 1913 lfence 1914 vorpd (%rdx), %zmm0, %zmm0 1915 vorpd (%rdx), %zmm0, %zmm0 1916 xorl %eax, %eax 1917 2: clflushopt 5376(%rdi, %rax, 8) 1918 addl $8, %eax 1919 cmpl $8 * 12, %eax 1920 jb 2b 1921 sfence 1922 movl $6144, %ecx 1923 xorl %eax, %eax 1924 rep; stosb 1925 mfence 1926 1927 vmovdqa64 PCPU(MDS_TMP), %zmm0 1928 testb $CR0_TS, %al 1929 je 3f 1930 movq %rax, %cr0 1931 3: popq %rdi 1932 popq %rcx 1933 popq %rdx 1934 popq %rax 1935 retq 1936 END(mds_handler_skl_avx512) 1937 1938 ENTRY(mds_handler_silvermont) 1939 pushq %rax 1940 pushq %rdx 1941 pushq %rcx 1942 1943 movq %cr0, %rax 1944 testb $CR0_TS, %al 1945 je 1f 1946 clts 1947 1: movq PCPU(MDS_BUF), %rdx 1948 movdqa %xmm0, PCPU(MDS_TMP) 1949 pxor %xmm0, %xmm0 1950 1951 movl $16, %ecx 1952 2: movntdq %xmm0, (%rdx) 1953 addq $16, %rdx 1954 decl %ecx 1955 jnz 2b 1956 mfence 1957 1958 movdqa PCPU(MDS_TMP),%xmm0 1959 testb $CR0_TS, %al 1960 je 3f 1961 movq %rax, %cr0 1962 3: popq %rcx 1963 popq %rdx 1964 popq %rax 1965 retq 1966 END(mds_handler_silvermont)
Cache object: 8362f0002cfdc967c1283619e96681b0
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]