Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/support.S
Version:
- FREEBSD - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-2 - FREEBSD-11-1 - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-4 - FREEBSD-10-3 - FREEBSD-10-2 - FREEBSD-10-1 - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-3 - FREEBSD-9-2 - FREEBSD-9-1 - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-4 - FREEBSD-8-3 - FREEBSD-8-2 - FREEBSD-8-1 - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-4 - FREEBSD-7-3 - FREEBSD-7-2 - FREEBSD-7-1 - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-4 - FREEBSD-6-3 - FREEBSD-6-2 - FREEBSD-6-1 - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-5 - FREEBSD-5-4 - FREEBSD-5-3 - FREEBSD-5-2 - FREEBSD-5-1 - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - linux-2.6 - linux-2.4.22 - MK83 - MK84 - PLAN9 - DFBSD - NETBSD - NETBSD5 - NETBSD4 - NETBSD3 - NETBSD20 - OPENBSD - xnu-517 - xnu-792 - xnu-792.6.70 - xnu-1228 - xnu-1456.1.26 - xnu-1699.24.8 - xnu-2050.18.24 - OPENSOLARIS - minix-3-1-1
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 2018-2019 The FreeBSD Foundation 3 * Copyright (c) 2003 Peter Wemm. 4 * Copyright (c) 1993 The Regents of the University of California. 5 * All rights reserved. 6 * 7 * Portions of this software were developed by 8 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from 9 * the FreeBSD Foundation. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. Neither the name of the University nor the names of its contributors 20 * may be used to endorse or promote products derived from this software 21 * without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 * SUCH DAMAGE. 34 * 35 * $FreeBSD: stable/12/sys/amd64/amd64/support.S 368782 2020-12-19 04:18:49Z rlibby $ 36 */ 37 38 #include "opt_ddb.h" 39 40 #include <machine/asmacros.h> 41 #include <machine/specialreg.h> 42 #include <machine/pmap.h> 43 44 #include "assym.inc" 45 46 .text 47 48 /* Address: %rdi */ 49 ENTRY(pagezero_std) 50 PUSH_FRAME_POINTER 51 movq $PAGE_SIZE/8,%rcx 52 xorl %eax,%eax 53 rep 54 stosq 55 POP_FRAME_POINTER 56 ret 57 END(pagezero_std) 58 59 ENTRY(pagezero_erms) 60 PUSH_FRAME_POINTER 61 movq $PAGE_SIZE,%rcx 62 xorl %eax,%eax 63 rep 64 stosb 65 POP_FRAME_POINTER 66 ret 67 END(pagezero_erms) 68 69 /* 70 * pagecopy(%rdi=from, %rsi=to) 71 */ 72 ENTRY(pagecopy) 73 PUSH_FRAME_POINTER 74 movq $PAGE_SIZE/8,%rcx 75 movq %rdi,%r9 76 movq %rsi,%rdi 77 movq %r9,%rsi 78 rep 79 movsq 80 POP_FRAME_POINTER 81 ret 82 END(pagecopy) 83 84 /* Address: %rdi */ 85 ENTRY(sse2_pagezero) 86 PUSH_FRAME_POINTER 87 movq $-PAGE_SIZE,%rdx 88 subq %rdx,%rdi 89 xorl %eax,%eax 90 jmp 1f 91 /* 92 * The loop takes 29 bytes. Ensure that it doesn't cross a 32-byte 93 * cache line. 94 */ 95 .p2align 5,0x90 96 1: 97 movnti %rax,(%rdi,%rdx) 98 movnti %rax,8(%rdi,%rdx) 99 movnti %rax,16(%rdi,%rdx) 100 movnti %rax,24(%rdi,%rdx) 101 addq $32,%rdx 102 jne 1b 103 sfence 104 POP_FRAME_POINTER 105 ret 106 END(sse2_pagezero) 107 108 /* 109 * memcmpy(b1, b2, len) 110 * rdi,rsi,rdx 111 */ 112 ENTRY(memcmp) 113 PUSH_FRAME_POINTER 114 115 xorl %eax,%eax 116 10: 117 cmpq $16,%rdx 118 ja 101632f 119 120 100816: 121 cmpb $8,%dl 122 jl 100408f 123 movq (%rdi),%r8 124 movq (%rsi),%r9 125 cmpq %r8,%r9 126 jne 80f 127 movq -8(%rdi,%rdx),%r8 128 movq -8(%rsi,%rdx),%r9 129 cmpq %r8,%r9 130 jne 10081608f 131 POP_FRAME_POINTER 132 ret 133 100408: 134 cmpb $4,%dl 135 jl 100204f 136 movl (%rdi),%r8d 137 movl (%rsi),%r9d 138 cmpl %r8d,%r9d 139 jne 80f 140 movl -4(%rdi,%rdx),%r8d 141 movl -4(%rsi,%rdx),%r9d 142 cmpl %r8d,%r9d 143 jne 10040804f 144 POP_FRAME_POINTER 145 ret 146 100204: 147 cmpb $2,%dl 148 jl 100001f 149 movzwl (%rdi),%r8d 150 movzwl (%rsi),%r9d 151 cmpl %r8d,%r9d 152 jne 1f 153 movzwl -2(%rdi,%rdx),%r8d 154 movzwl -2(%rsi,%rdx),%r9d 155 cmpl %r8d,%r9d 156 jne 1f 157 POP_FRAME_POINTER 158 ret 159 100001: 160 cmpb $1,%dl 161 jl 100000f 162 movzbl (%rdi),%eax 163 movzbl (%rsi),%r8d 164 subl %r8d,%eax 165 100000: 166 POP_FRAME_POINTER 167 ret 168 ALIGN_TEXT 169 101632: 170 cmpq $32,%rdx 171 ja 103200f 172 movq (%rdi),%r8 173 movq (%rsi),%r9 174 cmpq %r8,%r9 175 jne 80f 176 movq 8(%rdi),%r8 177 movq 8(%rsi),%r9 178 cmpq %r8,%r9 179 jne 10163208f 180 movq -16(%rdi,%rdx),%r8 181 movq -16(%rsi,%rdx),%r9 182 cmpq %r8,%r9 183 jne 10163216f 184 movq -8(%rdi,%rdx),%r8 185 movq -8(%rsi,%rdx),%r9 186 cmpq %r8,%r9 187 jne 10163224f 188 POP_FRAME_POINTER 189 ret 190 ALIGN_TEXT 191 103200: 192 movq (%rdi),%r8 193 movq 8(%rdi),%r9 194 subq (%rsi),%r8 195 subq 8(%rsi),%r9 196 orq %r8,%r9 197 jnz 10320000f 198 199 movq 16(%rdi),%r8 200 movq 24(%rdi),%r9 201 subq 16(%rsi),%r8 202 subq 24(%rsi),%r9 203 orq %r8,%r9 204 jnz 10320016f 205 206 leaq 32(%rdi),%rdi 207 leaq 32(%rsi),%rsi 208 subq $32,%rdx 209 cmpq $32,%rdx 210 jae 103200b 211 cmpb $0,%dl 212 jne 10b 213 POP_FRAME_POINTER 214 ret 215 216 /* 217 * Mismatch was found. 218 * 219 * Before we compute it we narrow down the range (16 -> 8 -> 4 bytes). 220 */ 221 ALIGN_TEXT 222 10320016: 223 leaq 16(%rdi),%rdi 224 leaq 16(%rsi),%rsi 225 10320000: 226 movq (%rdi),%r8 227 movq (%rsi),%r9 228 cmpq %r8,%r9 229 jne 80f 230 leaq 8(%rdi),%rdi 231 leaq 8(%rsi),%rsi 232 jmp 80f 233 ALIGN_TEXT 234 10081608: 235 10163224: 236 leaq -8(%rdi,%rdx),%rdi 237 leaq -8(%rsi,%rdx),%rsi 238 jmp 80f 239 ALIGN_TEXT 240 10163216: 241 leaq -16(%rdi,%rdx),%rdi 242 leaq -16(%rsi,%rdx),%rsi 243 jmp 80f 244 ALIGN_TEXT 245 10163208: 246 leaq 8(%rdi),%rdi 247 leaq 8(%rsi),%rsi 248 jmp 80f 249 ALIGN_TEXT 250 10040804: 251 leaq -4(%rdi,%rdx),%rdi 252 leaq -4(%rsi,%rdx),%rsi 253 jmp 1f 254 255 ALIGN_TEXT 256 80: 257 movl (%rdi),%r8d 258 movl (%rsi),%r9d 259 cmpl %r8d,%r9d 260 jne 1f 261 leaq 4(%rdi),%rdi 262 leaq 4(%rsi),%rsi 263 264 /* 265 * We have up to 4 bytes to inspect. 266 */ 267 1: 268 movzbl (%rdi),%eax 269 movzbl (%rsi),%r8d 270 cmpb %r8b,%al 271 jne 2f 272 273 movzbl 1(%rdi),%eax 274 movzbl 1(%rsi),%r8d 275 cmpb %r8b,%al 276 jne 2f 277 278 movzbl 2(%rdi),%eax 279 movzbl 2(%rsi),%r8d 280 cmpb %r8b,%al 281 jne 2f 282 283 movzbl 3(%rdi),%eax 284 movzbl 3(%rsi),%r8d 285 2: 286 subl %r8d,%eax 287 POP_FRAME_POINTER 288 ret 289 END(memcmp) 290 291 /* 292 * memmove(dst, src, cnt) 293 * rdi, rsi, rdx 294 */ 295 296 /* 297 * Register state at entry is supposed to be as follows: 298 * rdi - destination 299 * rsi - source 300 * rdx - count 301 * 302 * The macro possibly clobbers the above and: rcx, r8, r9, 10 303 * It does not clobber rax nor r11. 304 */ 305 .macro MEMMOVE erms overlap begin end 306 \begin 307 308 /* 309 * For sizes 0..32 all data is read before it is written, so there 310 * is no correctness issue with direction of copying. 311 */ 312 cmpq $32,%rcx 313 jbe 101632f 314 315 .if \overlap == 1 316 movq %rdi,%r8 317 subq %rsi,%r8 318 cmpq %rcx,%r8 /* overlapping && src < dst? */ 319 jb 2f 320 .endif 321 322 cmpq $256,%rcx 323 ja 1256f 324 325 103200: 326 movq (%rsi),%rdx 327 movq %rdx,(%rdi) 328 movq 8(%rsi),%rdx 329 movq %rdx,8(%rdi) 330 movq 16(%rsi),%rdx 331 movq %rdx,16(%rdi) 332 movq 24(%rsi),%rdx 333 movq %rdx,24(%rdi) 334 leaq 32(%rsi),%rsi 335 leaq 32(%rdi),%rdi 336 subq $32,%rcx 337 cmpq $32,%rcx 338 jae 103200b 339 cmpb $0,%cl 340 jne 101632f 341 \end 342 ret 343 ALIGN_TEXT 344 101632: 345 cmpb $16,%cl 346 jl 100816f 347 movq (%rsi),%rdx 348 movq 8(%rsi),%r8 349 movq -16(%rsi,%rcx),%r9 350 movq -8(%rsi,%rcx),%r10 351 movq %rdx,(%rdi) 352 movq %r8,8(%rdi) 353 movq %r9,-16(%rdi,%rcx) 354 movq %r10,-8(%rdi,%rcx) 355 \end 356 ret 357 ALIGN_TEXT 358 100816: 359 cmpb $8,%cl 360 jl 100408f 361 movq (%rsi),%rdx 362 movq -8(%rsi,%rcx),%r8 363 movq %rdx,(%rdi) 364 movq %r8,-8(%rdi,%rcx,) 365 \end 366 ret 367 ALIGN_TEXT 368 100408: 369 cmpb $4,%cl 370 jl 100204f 371 movl (%rsi),%edx 372 movl -4(%rsi,%rcx),%r8d 373 movl %edx,(%rdi) 374 movl %r8d,-4(%rdi,%rcx) 375 \end 376 ret 377 ALIGN_TEXT 378 100204: 379 cmpb $2,%cl 380 jl 100001f 381 movzwl (%rsi),%edx 382 movzwl -2(%rsi,%rcx),%r8d 383 movw %dx,(%rdi) 384 movw %r8w,-2(%rdi,%rcx) 385 \end 386 ret 387 ALIGN_TEXT 388 100001: 389 cmpb $1,%cl 390 jl 100000f 391 movb (%rsi),%dl 392 movb %dl,(%rdi) 393 100000: 394 \end 395 ret 396 397 ALIGN_TEXT 398 1256: 399 testb $15,%dil 400 jnz 100f 401 .if \erms == 1 402 rep 403 movsb 404 .else 405 shrq $3,%rcx /* copy by 64-bit words */ 406 rep 407 movsq 408 movq %rdx,%rcx 409 andl $7,%ecx /* any bytes left? */ 410 jne 100408b 411 .endif 412 \end 413 ret 414 100: 415 movq (%rsi),%r8 416 movq 8(%rsi),%r9 417 movq %rdi,%r10 418 movq %rdi,%rcx 419 andq $15,%rcx 420 leaq -16(%rdx,%rcx),%rdx 421 neg %rcx 422 leaq 16(%rdi,%rcx),%rdi 423 leaq 16(%rsi,%rcx),%rsi 424 movq %rdx,%rcx 425 .if \erms == 1 426 rep 427 movsb 428 movq %r8,(%r10) 429 movq %r9,8(%r10) 430 .else 431 shrq $3,%rcx /* copy by 64-bit words */ 432 rep 433 movsq 434 movq %r8,(%r10) 435 movq %r9,8(%r10) 436 movq %rdx,%rcx 437 andl $7,%ecx /* any bytes left? */ 438 jne 100408b 439 .endif 440 \end 441 ret 442 443 .if \overlap == 1 444 /* 445 * Copy backwards. 446 */ 447 ALIGN_TEXT 448 2: 449 cmpq $256,%rcx 450 ja 2256f 451 452 leaq -8(%rdi,%rcx),%rdi 453 leaq -8(%rsi,%rcx),%rsi 454 455 cmpq $32,%rcx 456 jb 2016f 457 458 2032: 459 movq (%rsi),%rdx 460 movq %rdx,(%rdi) 461 movq -8(%rsi),%rdx 462 movq %rdx,-8(%rdi) 463 movq -16(%rsi),%rdx 464 movq %rdx,-16(%rdi) 465 movq -24(%rsi),%rdx 466 movq %rdx,-24(%rdi) 467 leaq -32(%rsi),%rsi 468 leaq -32(%rdi),%rdi 469 subq $32,%rcx 470 cmpq $32,%rcx 471 jae 2032b 472 cmpb $0,%cl 473 jne 2016f 474 \end 475 ret 476 ALIGN_TEXT 477 2016: 478 cmpb $16,%cl 479 jl 2008f 480 movq (%rsi),%rdx 481 movq %rdx,(%rdi) 482 movq -8(%rsi),%rdx 483 movq %rdx,-8(%rdi) 484 subb $16,%cl 485 jz 2000f 486 leaq -16(%rsi),%rsi 487 leaq -16(%rdi),%rdi 488 2008: 489 cmpb $8,%cl 490 jl 2004f 491 movq (%rsi),%rdx 492 movq %rdx,(%rdi) 493 subb $8,%cl 494 jz 2000f 495 leaq -8(%rsi),%rsi 496 leaq -8(%rdi),%rdi 497 2004: 498 cmpb $4,%cl 499 jl 2002f 500 movl 4(%rsi),%edx 501 movl %edx,4(%rdi) 502 subb $4,%cl 503 jz 2000f 504 leaq -4(%rsi),%rsi 505 leaq -4(%rdi),%rdi 506 2002: 507 cmpb $2,%cl 508 jl 2001f 509 movw 6(%rsi),%dx 510 movw %dx,6(%rdi) 511 subb $2,%cl 512 jz 2000f 513 leaq -2(%rsi),%rsi 514 leaq -2(%rdi),%rdi 515 2001: 516 cmpb $1,%cl 517 jl 2000f 518 movb 7(%rsi),%dl 519 movb %dl,7(%rdi) 520 2000: 521 \end 522 ret 523 ALIGN_TEXT 524 2256: 525 std 526 .if \erms == 1 527 leaq -1(%rdi,%rcx),%rdi 528 leaq -1(%rsi,%rcx),%rsi 529 rep 530 movsb 531 cld 532 .else 533 leaq -8(%rdi,%rcx),%rdi 534 leaq -8(%rsi,%rcx),%rsi 535 shrq $3,%rcx 536 rep 537 movsq 538 cld 539 movq %rdx,%rcx 540 andb $7,%cl 541 jne 2004b 542 .endif 543 \end 544 ret 545 .endif 546 .endm 547 548 .macro MEMMOVE_BEGIN 549 PUSH_FRAME_POINTER 550 movq %rdi,%rax 551 movq %rdx,%rcx 552 .endm 553 554 .macro MEMMOVE_END 555 POP_FRAME_POINTER 556 .endm 557 558 ENTRY(memmove_std) 559 MEMMOVE erms=0 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END 560 END(memmove_std) 561 562 ENTRY(memmove_erms) 563 MEMMOVE erms=1 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END 564 END(memmove_erms) 565 566 /* 567 * memcpy(dst, src, len) 568 * rdi, rsi, rdx 569 * 570 * Note: memcpy does not support overlapping copies 571 */ 572 ENTRY(memcpy_std) 573 MEMMOVE erms=0 overlap=0 begin=MEMMOVE_BEGIN end=MEMMOVE_END 574 END(memcpy_std) 575 576 ENTRY(memcpy_erms) 577 MEMMOVE erms=1 overlap=0 begin=MEMMOVE_BEGIN end=MEMMOVE_END 578 END(memcpy_erms) 579 580 /* 581 * memset(dst, c, len) 582 * rdi, rsi, rdx 583 */ 584 .macro MEMSET erms 585 PUSH_FRAME_POINTER 586 movq %rdi,%rax 587 movq %rdx,%rcx 588 movzbq %sil,%r8 589 movabs $0x0101010101010101,%r10 590 imulq %r8,%r10 591 592 cmpq $32,%rcx 593 jbe 101632f 594 595 cmpq $256,%rcx 596 ja 1256f 597 598 103200: 599 movq %r10,(%rdi) 600 movq %r10,8(%rdi) 601 movq %r10,16(%rdi) 602 movq %r10,24(%rdi) 603 leaq 32(%rdi),%rdi 604 subq $32,%rcx 605 cmpq $32,%rcx 606 ja 103200b 607 cmpb $16,%cl 608 ja 201632f 609 movq %r10,-16(%rdi,%rcx) 610 movq %r10,-8(%rdi,%rcx) 611 POP_FRAME_POINTER 612 ret 613 ALIGN_TEXT 614 101632: 615 cmpb $16,%cl 616 jl 100816f 617 201632: 618 movq %r10,(%rdi) 619 movq %r10,8(%rdi) 620 movq %r10,-16(%rdi,%rcx) 621 movq %r10,-8(%rdi,%rcx) 622 POP_FRAME_POINTER 623 ret 624 ALIGN_TEXT 625 100816: 626 cmpb $8,%cl 627 jl 100408f 628 movq %r10,(%rdi) 629 movq %r10,-8(%rdi,%rcx) 630 POP_FRAME_POINTER 631 ret 632 ALIGN_TEXT 633 100408: 634 cmpb $4,%cl 635 jl 100204f 636 movl %r10d,(%rdi) 637 movl %r10d,-4(%rdi,%rcx) 638 POP_FRAME_POINTER 639 ret 640 ALIGN_TEXT 641 100204: 642 cmpb $2,%cl 643 jl 100001f 644 movw %r10w,(%rdi) 645 movw %r10w,-2(%rdi,%rcx) 646 POP_FRAME_POINTER 647 ret 648 ALIGN_TEXT 649 100001: 650 cmpb $0,%cl 651 je 100000f 652 movb %r10b,(%rdi) 653 100000: 654 POP_FRAME_POINTER 655 ret 656 ALIGN_TEXT 657 1256: 658 movq %rdi,%r9 659 movq %r10,%rax 660 testl $15,%edi 661 jnz 3f 662 1: 663 .if \erms == 1 664 rep 665 stosb 666 movq %r9,%rax 667 .else 668 movq %rcx,%rdx 669 shrq $3,%rcx 670 rep 671 stosq 672 movq %r9,%rax 673 andl $7,%edx 674 jnz 2f 675 POP_FRAME_POINTER 676 ret 677 2: 678 movq %r10,-8(%rdi,%rdx) 679 .endif 680 POP_FRAME_POINTER 681 ret 682 ALIGN_TEXT 683 3: 684 movq %r10,(%rdi) 685 movq %r10,8(%rdi) 686 movq %rdi,%r8 687 andq $15,%r8 688 leaq -16(%rcx,%r8),%rcx 689 neg %r8 690 leaq 16(%rdi,%r8),%rdi 691 jmp 1b 692 .endm 693 694 ENTRY(memset_std) 695 MEMSET erms=0 696 END(memset_std) 697 698 ENTRY(memset_erms) 699 MEMSET erms=1 700 END(memset_erms) 701 702 /* fillw(pat, base, cnt) */ 703 /* %rdi,%rsi, %rdx */ 704 ENTRY(fillw) 705 PUSH_FRAME_POINTER 706 movq %rdi,%rax 707 movq %rsi,%rdi 708 movq %rdx,%rcx 709 rep 710 stosw 711 POP_FRAME_POINTER 712 ret 713 END(fillw) 714 715 /*****************************************************************************/ 716 /* copyout and fubyte family */ 717 /*****************************************************************************/ 718 /* 719 * Access user memory from inside the kernel. These routines should be 720 * the only places that do this. 721 * 722 * These routines set curpcb->pcb_onfault for the time they execute. When a 723 * protection violation occurs inside the functions, the trap handler 724 * returns to *curpcb->pcb_onfault instead of the function. 725 */ 726 727 .macro SMAP_DISABLE smap 728 .if \smap 729 stac 730 .endif 731 .endm 732 733 734 .macro SMAP_ENABLE smap 735 .if \smap 736 clac 737 .endif 738 .endm 739 740 .macro COPYINOUT_BEGIN 741 .endm 742 743 .macro COPYINOUT_END 744 movq %rax,PCB_ONFAULT(%r11) 745 POP_FRAME_POINTER 746 .endm 747 748 .macro COPYINOUT_SMAP_END 749 SMAP_ENABLE smap=1 750 COPYINOUT_END 751 .endm 752 753 /* 754 * copyout(from_kernel, to_user, len) 755 * %rdi, %rsi, %rdx 756 */ 757 .macro COPYOUT smap erms 758 PUSH_FRAME_POINTER 759 movq PCPU(CURPCB),%r11 760 movq $copy_fault,PCB_ONFAULT(%r11) 761 762 /* 763 * Check explicitly for non-user addresses. 764 * First, prevent address wrapping. 765 */ 766 movq %rsi,%rax 767 addq %rdx,%rax 768 jc copy_fault 769 /* 770 * XXX STOP USING VM_MAXUSER_ADDRESS. 771 * It is an end address, not a max, so every time it is used correctly it 772 * looks like there is an off by one error, and of course it caused an off 773 * by one error in several places. 774 */ 775 movq $VM_MAXUSER_ADDRESS,%rcx 776 cmpq %rcx,%rax 777 ja copy_fault 778 779 /* 780 * Set return value to zero. Remaining failure mode goes through 781 * copy_fault. 782 */ 783 xorl %eax,%eax 784 785 /* 786 * Set up arguments for MEMMOVE. 787 */ 788 movq %rdi,%r8 789 movq %rsi,%rdi 790 movq %r8,%rsi 791 movq %rdx,%rcx 792 793 794 SMAP_DISABLE \smap 795 .if \smap == 1 796 MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_SMAP_END 797 .else 798 MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_END 799 .endif 800 /* NOTREACHED */ 801 .endm 802 803 ENTRY(copyout_nosmap_std) 804 COPYOUT smap=0 erms=0 805 END(copyout_nosmap_std) 806 807 ENTRY(copyout_smap_std) 808 COPYOUT smap=1 erms=0 809 END(copyout_smap_std) 810 811 ENTRY(copyout_nosmap_erms) 812 COPYOUT smap=0 erms=1 813 END(copyout_nosmap_erms) 814 815 ENTRY(copyout_smap_erms) 816 COPYOUT smap=1 erms=1 817 END(copyout_smap_erms) 818 819 /* 820 * copyin(from_user, to_kernel, len) 821 * %rdi, %rsi, %rdx 822 */ 823 .macro COPYIN smap erms 824 PUSH_FRAME_POINTER 825 movq PCPU(CURPCB),%r11 826 movq $copy_fault,PCB_ONFAULT(%r11) 827 828 /* 829 * make sure address is valid 830 */ 831 movq %rdi,%rax 832 addq %rdx,%rax 833 jc copy_fault 834 movq $VM_MAXUSER_ADDRESS,%rcx 835 cmpq %rcx,%rax 836 ja copy_fault 837 838 xorl %eax,%eax 839 840 movq %rdi,%r8 841 movq %rsi,%rdi 842 movq %r8,%rsi 843 movq %rdx,%rcx 844 845 SMAP_DISABLE \smap 846 .if \smap == 1 847 MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_SMAP_END 848 .else 849 MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_END 850 .endif 851 /* NOTREACHED */ 852 .endm 853 854 ENTRY(copyin_nosmap_std) 855 COPYIN smap=0 erms=0 856 END(copyin_nosmap_std) 857 858 ENTRY(copyin_smap_std) 859 COPYIN smap=1 erms=0 860 END(copyin_smap_std) 861 862 ENTRY(copyin_nosmap_erms) 863 COPYIN smap=0 erms=1 864 END(copyin_nosmap_erms) 865 866 ENTRY(copyin_smap_erms) 867 COPYIN smap=1 erms=1 868 END(copyin_smap_erms) 869 870 ALIGN_TEXT 871 /* Trap entry clears PSL.AC */ 872 copy_fault: 873 movq $0,PCB_ONFAULT(%r11) 874 movl $EFAULT,%eax 875 POP_FRAME_POINTER 876 ret 877 878 /* 879 * casueword32. Compare and set user integer. Returns -1 on fault, 880 * 0 if access was successful. Old value is written to *oldp. 881 * dst = %rdi, old = %esi, oldp = %rdx, new = %ecx 882 */ 883 ENTRY(casueword32_nosmap) 884 PUSH_FRAME_POINTER 885 movq PCPU(CURPCB),%r8 886 movq $fusufault,PCB_ONFAULT(%r8) 887 888 movq $VM_MAXUSER_ADDRESS-4,%rax 889 cmpq %rax,%rdi /* verify address is valid */ 890 ja fusufault 891 892 movl %esi,%eax /* old */ 893 #ifdef SMP 894 lock 895 #endif 896 cmpxchgl %ecx,(%rdi) /* new = %ecx */ 897 setne %cl 898 899 /* 900 * The old value is in %eax. If the store succeeded it will be the 901 * value we expected (old) from before the store, otherwise it will 902 * be the current value. Save %eax into %esi to prepare the return 903 * value. 904 */ 905 movl %eax,%esi 906 xorl %eax,%eax 907 movq %rax,PCB_ONFAULT(%r8) 908 909 /* 910 * Access the oldp after the pcb_onfault is cleared, to correctly 911 * catch corrupted pointer. 912 */ 913 movl %esi,(%rdx) /* oldp = %rdx */ 914 POP_FRAME_POINTER 915 movzbl %cl, %eax 916 ret 917 END(casueword32_nosmap) 918 919 ENTRY(casueword32_smap) 920 PUSH_FRAME_POINTER 921 movq PCPU(CURPCB),%r8 922 movq $fusufault,PCB_ONFAULT(%r8) 923 924 movq $VM_MAXUSER_ADDRESS-4,%rax 925 cmpq %rax,%rdi /* verify address is valid */ 926 ja fusufault 927 928 movl %esi,%eax /* old */ 929 stac 930 #ifdef SMP 931 lock 932 #endif 933 cmpxchgl %ecx,(%rdi) /* new = %ecx */ 934 clac 935 setne %cl 936 937 /* 938 * The old value is in %eax. If the store succeeded it will be the 939 * value we expected (old) from before the store, otherwise it will 940 * be the current value. Save %eax into %esi to prepare the return 941 * value. 942 */ 943 movl %eax,%esi 944 xorl %eax,%eax 945 movq %rax,PCB_ONFAULT(%r8) 946 947 /* 948 * Access the oldp after the pcb_onfault is cleared, to correctly 949 * catch corrupted pointer. 950 */ 951 movl %esi,(%rdx) /* oldp = %rdx */ 952 POP_FRAME_POINTER 953 movzbl %cl, %eax 954 ret 955 END(casueword32_smap) 956 957 /* 958 * casueword. Compare and set user long. Returns -1 on fault, 959 * 0 if access was successful. Old value is written to *oldp. 960 * dst = %rdi, old = %rsi, oldp = %rdx, new = %rcx 961 */ 962 ENTRY(casueword_nosmap) 963 PUSH_FRAME_POINTER 964 movq PCPU(CURPCB),%r8 965 movq $fusufault,PCB_ONFAULT(%r8) 966 967 movq $VM_MAXUSER_ADDRESS-4,%rax 968 cmpq %rax,%rdi /* verify address is valid */ 969 ja fusufault 970 971 movq %rsi,%rax /* old */ 972 #ifdef SMP 973 lock 974 #endif 975 cmpxchgq %rcx,(%rdi) /* new = %rcx */ 976 setne %cl 977 978 /* 979 * The old value is in %rax. If the store succeeded it will be the 980 * value we expected (old) from before the store, otherwise it will 981 * be the current value. 982 */ 983 movq %rax,%rsi 984 xorl %eax,%eax 985 movq %rax,PCB_ONFAULT(%r8) 986 movq %rsi,(%rdx) 987 POP_FRAME_POINTER 988 movzbl %cl, %eax 989 ret 990 END(casueword_nosmap) 991 992 ENTRY(casueword_smap) 993 PUSH_FRAME_POINTER 994 movq PCPU(CURPCB),%r8 995 movq $fusufault,PCB_ONFAULT(%r8) 996 997 movq $VM_MAXUSER_ADDRESS-4,%rax 998 cmpq %rax,%rdi /* verify address is valid */ 999 ja fusufault 1000 1001 movq %rsi,%rax /* old */ 1002 stac 1003 #ifdef SMP 1004 lock 1005 #endif 1006 cmpxchgq %rcx,(%rdi) /* new = %rcx */ 1007 clac 1008 setne %cl 1009 1010 /* 1011 * The old value is in %rax. If the store succeeded it will be the 1012 * value we expected (old) from before the store, otherwise it will 1013 * be the current value. 1014 */ 1015 movq %rax,%rsi 1016 xorl %eax,%eax 1017 movq %rax,PCB_ONFAULT(%r8) 1018 movq %rsi,(%rdx) 1019 POP_FRAME_POINTER 1020 movzbl %cl, %eax 1021 ret 1022 END(casueword_smap) 1023 1024 /* 1025 * Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit 1026 * byte from user memory. 1027 * addr = %rdi, valp = %rsi 1028 */ 1029 1030 ENTRY(fueword_nosmap) 1031 PUSH_FRAME_POINTER 1032 movq PCPU(CURPCB),%rcx 1033 movq $fusufault,PCB_ONFAULT(%rcx) 1034 1035 movq $VM_MAXUSER_ADDRESS-8,%rax 1036 cmpq %rax,%rdi /* verify address is valid */ 1037 ja fusufault 1038 1039 xorl %eax,%eax 1040 movq (%rdi),%r11 1041 movq %rax,PCB_ONFAULT(%rcx) 1042 movq %r11,(%rsi) 1043 POP_FRAME_POINTER 1044 ret 1045 END(fueword_nosmap) 1046 1047 ENTRY(fueword_smap) 1048 PUSH_FRAME_POINTER 1049 movq PCPU(CURPCB),%rcx 1050 movq $fusufault,PCB_ONFAULT(%rcx) 1051 1052 movq $VM_MAXUSER_ADDRESS-8,%rax 1053 cmpq %rax,%rdi /* verify address is valid */ 1054 ja fusufault 1055 1056 xorl %eax,%eax 1057 stac 1058 movq (%rdi),%r11 1059 clac 1060 movq %rax,PCB_ONFAULT(%rcx) 1061 movq %r11,(%rsi) 1062 POP_FRAME_POINTER 1063 ret 1064 END(fueword_smap) 1065 1066 ENTRY(fueword32_nosmap) 1067 PUSH_FRAME_POINTER 1068 movq PCPU(CURPCB),%rcx 1069 movq $fusufault,PCB_ONFAULT(%rcx) 1070 1071 movq $VM_MAXUSER_ADDRESS-4,%rax 1072 cmpq %rax,%rdi /* verify address is valid */ 1073 ja fusufault 1074 1075 xorl %eax,%eax 1076 movl (%rdi),%r11d 1077 movq %rax,PCB_ONFAULT(%rcx) 1078 movl %r11d,(%rsi) 1079 POP_FRAME_POINTER 1080 ret 1081 END(fueword32_nosmap) 1082 1083 ENTRY(fueword32_smap) 1084 PUSH_FRAME_POINTER 1085 movq PCPU(CURPCB),%rcx 1086 movq $fusufault,PCB_ONFAULT(%rcx) 1087 1088 movq $VM_MAXUSER_ADDRESS-4,%rax 1089 cmpq %rax,%rdi /* verify address is valid */ 1090 ja fusufault 1091 1092 xorl %eax,%eax 1093 stac 1094 movl (%rdi),%r11d 1095 clac 1096 movq %rax,PCB_ONFAULT(%rcx) 1097 movl %r11d,(%rsi) 1098 POP_FRAME_POINTER 1099 ret 1100 END(fueword32_smap) 1101 1102 ENTRY(fuword16_nosmap) 1103 PUSH_FRAME_POINTER 1104 movq PCPU(CURPCB),%rcx 1105 movq $fusufault,PCB_ONFAULT(%rcx) 1106 1107 movq $VM_MAXUSER_ADDRESS-2,%rax 1108 cmpq %rax,%rdi 1109 ja fusufault 1110 1111 movzwl (%rdi),%eax 1112 movq $0,PCB_ONFAULT(%rcx) 1113 POP_FRAME_POINTER 1114 ret 1115 END(fuword16_nosmap) 1116 1117 ENTRY(fuword16_smap) 1118 PUSH_FRAME_POINTER 1119 movq PCPU(CURPCB),%rcx 1120 movq $fusufault,PCB_ONFAULT(%rcx) 1121 1122 movq $VM_MAXUSER_ADDRESS-2,%rax 1123 cmpq %rax,%rdi 1124 ja fusufault 1125 1126 stac 1127 movzwl (%rdi),%eax 1128 clac 1129 movq $0,PCB_ONFAULT(%rcx) 1130 POP_FRAME_POINTER 1131 ret 1132 END(fuword16_smap) 1133 1134 ENTRY(fubyte_nosmap) 1135 PUSH_FRAME_POINTER 1136 movq PCPU(CURPCB),%rcx 1137 movq $fusufault,PCB_ONFAULT(%rcx) 1138 1139 movq $VM_MAXUSER_ADDRESS-1,%rax 1140 cmpq %rax,%rdi 1141 ja fusufault 1142 1143 movzbl (%rdi),%eax 1144 movq $0,PCB_ONFAULT(%rcx) 1145 POP_FRAME_POINTER 1146 ret 1147 END(fubyte_nosmap) 1148 1149 ENTRY(fubyte_smap) 1150 PUSH_FRAME_POINTER 1151 movq PCPU(CURPCB),%rcx 1152 movq $fusufault,PCB_ONFAULT(%rcx) 1153 1154 movq $VM_MAXUSER_ADDRESS-1,%rax 1155 cmpq %rax,%rdi 1156 ja fusufault 1157 1158 stac 1159 movzbl (%rdi),%eax 1160 clac 1161 movq $0,PCB_ONFAULT(%rcx) 1162 POP_FRAME_POINTER 1163 ret 1164 END(fubyte_smap) 1165 1166 /* 1167 * Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to 1168 * user memory. 1169 * addr = %rdi, value = %rsi 1170 */ 1171 ENTRY(suword_nosmap) 1172 PUSH_FRAME_POINTER 1173 movq PCPU(CURPCB),%rcx 1174 movq $fusufault,PCB_ONFAULT(%rcx) 1175 1176 movq $VM_MAXUSER_ADDRESS-8,%rax 1177 cmpq %rax,%rdi /* verify address validity */ 1178 ja fusufault 1179 1180 movq %rsi,(%rdi) 1181 xorl %eax,%eax 1182 movq PCPU(CURPCB),%rcx 1183 movq %rax,PCB_ONFAULT(%rcx) 1184 POP_FRAME_POINTER 1185 ret 1186 END(suword_nosmap) 1187 1188 ENTRY(suword_smap) 1189 PUSH_FRAME_POINTER 1190 movq PCPU(CURPCB),%rcx 1191 movq $fusufault,PCB_ONFAULT(%rcx) 1192 1193 movq $VM_MAXUSER_ADDRESS-8,%rax 1194 cmpq %rax,%rdi /* verify address validity */ 1195 ja fusufault 1196 1197 stac 1198 movq %rsi,(%rdi) 1199 clac 1200 xorl %eax,%eax 1201 movq PCPU(CURPCB),%rcx 1202 movq %rax,PCB_ONFAULT(%rcx) 1203 POP_FRAME_POINTER 1204 ret 1205 END(suword_smap) 1206 1207 ENTRY(suword32_nosmap) 1208 PUSH_FRAME_POINTER 1209 movq PCPU(CURPCB),%rcx 1210 movq $fusufault,PCB_ONFAULT(%rcx) 1211 1212 movq $VM_MAXUSER_ADDRESS-4,%rax 1213 cmpq %rax,%rdi /* verify address validity */ 1214 ja fusufault 1215 1216 movl %esi,(%rdi) 1217 xorl %eax,%eax 1218 movq PCPU(CURPCB),%rcx 1219 movq %rax,PCB_ONFAULT(%rcx) 1220 POP_FRAME_POINTER 1221 ret 1222 END(suword32_nosmap) 1223 1224 ENTRY(suword32_smap) 1225 PUSH_FRAME_POINTER 1226 movq PCPU(CURPCB),%rcx 1227 movq $fusufault,PCB_ONFAULT(%rcx) 1228 1229 movq $VM_MAXUSER_ADDRESS-4,%rax 1230 cmpq %rax,%rdi /* verify address validity */ 1231 ja fusufault 1232 1233 stac 1234 movl %esi,(%rdi) 1235 clac 1236 xorl %eax,%eax 1237 movq PCPU(CURPCB),%rcx 1238 movq %rax,PCB_ONFAULT(%rcx) 1239 POP_FRAME_POINTER 1240 ret 1241 END(suword32_smap) 1242 1243 ENTRY(suword16_nosmap) 1244 PUSH_FRAME_POINTER 1245 movq PCPU(CURPCB),%rcx 1246 movq $fusufault,PCB_ONFAULT(%rcx) 1247 1248 movq $VM_MAXUSER_ADDRESS-2,%rax 1249 cmpq %rax,%rdi /* verify address validity */ 1250 ja fusufault 1251 1252 movw %si,(%rdi) 1253 xorl %eax,%eax 1254 movq %rax,PCB_ONFAULT(%rcx) 1255 POP_FRAME_POINTER 1256 ret 1257 END(suword16_nosmap) 1258 1259 ENTRY(suword16_smap) 1260 PUSH_FRAME_POINTER 1261 movq PCPU(CURPCB),%rcx 1262 movq $fusufault,PCB_ONFAULT(%rcx) 1263 1264 movq $VM_MAXUSER_ADDRESS-2,%rax 1265 cmpq %rax,%rdi /* verify address validity */ 1266 ja fusufault 1267 1268 stac 1269 movw %si,(%rdi) 1270 clac 1271 xorl %eax,%eax 1272 movq %rax,PCB_ONFAULT(%rcx) 1273 POP_FRAME_POINTER 1274 ret 1275 END(suword16_smap) 1276 1277 ENTRY(subyte_nosmap) 1278 PUSH_FRAME_POINTER 1279 movq PCPU(CURPCB),%rcx 1280 movq $fusufault,PCB_ONFAULT(%rcx) 1281 1282 movq $VM_MAXUSER_ADDRESS-1,%rax 1283 cmpq %rax,%rdi /* verify address validity */ 1284 ja fusufault 1285 1286 movl %esi,%eax 1287 movb %al,(%rdi) 1288 xorl %eax,%eax 1289 movq %rax,PCB_ONFAULT(%rcx) 1290 POP_FRAME_POINTER 1291 ret 1292 END(subyte_nosmap) 1293 1294 ENTRY(subyte_smap) 1295 PUSH_FRAME_POINTER 1296 movq PCPU(CURPCB),%rcx 1297 movq $fusufault,PCB_ONFAULT(%rcx) 1298 1299 movq $VM_MAXUSER_ADDRESS-1,%rax 1300 cmpq %rax,%rdi /* verify address validity */ 1301 ja fusufault 1302 1303 movl %esi,%eax 1304 stac 1305 movb %al,(%rdi) 1306 clac 1307 xorl %eax,%eax 1308 movq %rax,PCB_ONFAULT(%rcx) 1309 POP_FRAME_POINTER 1310 ret 1311 END(subyte_smap) 1312 1313 ALIGN_TEXT 1314 /* Fault entry clears PSL.AC */ 1315 fusufault: 1316 movq PCPU(CURPCB),%rcx 1317 xorl %eax,%eax 1318 movq %rax,PCB_ONFAULT(%rcx) 1319 decq %rax 1320 POP_FRAME_POINTER 1321 ret 1322 1323 /* 1324 * copyinstr(from, to, maxlen, int *lencopied) 1325 * %rdi, %rsi, %rdx, %rcx 1326 * 1327 * copy a string from 'from' to 'to', stop when a 0 character is reached. 1328 * return ENAMETOOLONG if string is longer than maxlen, and 1329 * EFAULT on protection violations. If lencopied is non-zero, 1330 * return the actual length in *lencopied. 1331 */ 1332 .macro COPYINSTR smap 1333 PUSH_FRAME_POINTER 1334 movq %rdx,%r8 /* %r8 = maxlen */ 1335 movq PCPU(CURPCB),%r9 1336 movq $cpystrflt,PCB_ONFAULT(%r9) 1337 1338 movq $VM_MAXUSER_ADDRESS,%rax 1339 1340 /* make sure 'from' is within bounds */ 1341 subq %rdi,%rax 1342 jbe cpystrflt 1343 1344 SMAP_DISABLE \smap 1345 1346 /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */ 1347 cmpq %rdx,%rax 1348 jb 8f 1349 1: 1350 incq %rdx 1351 2: 1352 decq %rdx 1353 .if \smap == 0 1354 jz copyinstr_toolong 1355 .else 1356 jz copyinstr_toolong_smap 1357 .endif 1358 1359 movb (%rdi),%al 1360 movb %al,(%rsi) 1361 incq %rsi 1362 incq %rdi 1363 testb %al,%al 1364 jnz 2b 1365 1366 SMAP_ENABLE \smap 1367 1368 /* Success -- 0 byte reached */ 1369 decq %rdx 1370 xorl %eax,%eax 1371 1372 /* set *lencopied and return %eax */ 1373 movq %rax,PCB_ONFAULT(%r9) 1374 1375 testq %rcx,%rcx 1376 jz 3f 1377 subq %rdx,%r8 1378 movq %r8,(%rcx) 1379 3: 1380 POP_FRAME_POINTER 1381 ret 1382 ALIGN_TEXT 1383 8: 1384 movq %rax,%rdx 1385 movq %rax,%r8 1386 jmp 1b 1387 1388 .endm 1389 1390 ENTRY(copyinstr_nosmap) 1391 COPYINSTR smap=0 1392 END(copyinstr_nosmap) 1393 1394 ENTRY(copyinstr_smap) 1395 COPYINSTR smap=1 1396 END(copyinstr_smap) 1397 1398 cpystrflt: 1399 /* Fault entry clears PSL.AC */ 1400 movl $EFAULT,%eax 1401 cpystrflt_x: 1402 /* set *lencopied and return %eax */ 1403 movq $0,PCB_ONFAULT(%r9) 1404 1405 testq %rcx,%rcx 1406 jz 1f 1407 subq %rdx,%r8 1408 movq %r8,(%rcx) 1409 1: 1410 POP_FRAME_POINTER 1411 ret 1412 1413 copyinstr_toolong_smap: 1414 clac 1415 copyinstr_toolong: 1416 /* rdx is zero - return ENAMETOOLONG or EFAULT */ 1417 movq $VM_MAXUSER_ADDRESS,%rax 1418 cmpq %rax,%rdi 1419 jae cpystrflt 1420 movl $ENAMETOOLONG,%eax 1421 jmp cpystrflt_x 1422 1423 /* 1424 * copystr(from, to, maxlen, int *lencopied) 1425 * %rdi, %rsi, %rdx, %rcx 1426 */ 1427 ENTRY(copystr) 1428 PUSH_FRAME_POINTER 1429 movq %rdx,%r8 /* %r8 = maxlen */ 1430 1431 incq %rdx 1432 1: 1433 decq %rdx 1434 jz 4f 1435 movb (%rdi),%al 1436 movb %al,(%rsi) 1437 incq %rsi 1438 incq %rdi 1439 testb %al,%al 1440 jnz 1b 1441 1442 /* Success -- 0 byte reached */ 1443 decq %rdx 1444 xorl %eax,%eax 1445 2: 1446 testq %rcx,%rcx 1447 jz 3f 1448 /* set *lencopied and return %rax */ 1449 subq %rdx,%r8 1450 movq %r8,(%rcx) 1451 3: 1452 POP_FRAME_POINTER 1453 ret 1454 4: 1455 /* rdx is zero -- return ENAMETOOLONG */ 1456 movl $ENAMETOOLONG,%eax 1457 jmp 2b 1458 END(copystr) 1459 1460 /* 1461 * Handling of special amd64 registers and descriptor tables etc 1462 */ 1463 /* void lgdt(struct region_descriptor *rdp); */ 1464 ENTRY(lgdt) 1465 /* reload the descriptor table */ 1466 lgdt (%rdi) 1467 1468 /* flush the prefetch q */ 1469 jmp 1f 1470 nop 1471 1: 1472 movl $KDSEL,%eax 1473 movl %eax,%ds 1474 movl %eax,%es 1475 movl %eax,%fs /* Beware, use wrmsr to set 64 bit base */ 1476 movl %eax,%gs 1477 movl %eax,%ss 1478 1479 /* reload code selector by turning return into intersegmental return */ 1480 popq %rax 1481 pushq $KCSEL 1482 pushq %rax 1483 MEXITCOUNT 1484 lretq 1485 END(lgdt) 1486 1487 /*****************************************************************************/ 1488 /* setjump, longjump */ 1489 /*****************************************************************************/ 1490 1491 ENTRY(setjmp) 1492 movq %rbx,0(%rdi) /* save rbx */ 1493 movq %rsp,8(%rdi) /* save rsp */ 1494 movq %rbp,16(%rdi) /* save rbp */ 1495 movq %r12,24(%rdi) /* save r12 */ 1496 movq %r13,32(%rdi) /* save r13 */ 1497 movq %r14,40(%rdi) /* save r14 */ 1498 movq %r15,48(%rdi) /* save r15 */ 1499 movq 0(%rsp),%rdx /* get rta */ 1500 movq %rdx,56(%rdi) /* save rip */ 1501 xorl %eax,%eax /* return(0); */ 1502 ret 1503 END(setjmp) 1504 1505 ENTRY(longjmp) 1506 movq 0(%rdi),%rbx /* restore rbx */ 1507 movq 8(%rdi),%rsp /* restore rsp */ 1508 movq 16(%rdi),%rbp /* restore rbp */ 1509 movq 24(%rdi),%r12 /* restore r12 */ 1510 movq 32(%rdi),%r13 /* restore r13 */ 1511 movq 40(%rdi),%r14 /* restore r14 */ 1512 movq 48(%rdi),%r15 /* restore r15 */ 1513 movq 56(%rdi),%rdx /* get rta */ 1514 movq %rdx,0(%rsp) /* put in return frame */ 1515 xorl %eax,%eax /* return(1); */ 1516 incl %eax 1517 ret 1518 END(longjmp) 1519 1520 /* 1521 * Support for reading MSRs in the safe manner. (Instead of panic on #gp, 1522 * return an error.) 1523 */ 1524 ENTRY(rdmsr_safe) 1525 /* int rdmsr_safe(u_int msr, uint64_t *data) */ 1526 PUSH_FRAME_POINTER 1527 movq PCPU(CURPCB),%r8 1528 movq $msr_onfault,PCB_ONFAULT(%r8) 1529 movl %edi,%ecx 1530 rdmsr /* Read MSR pointed by %ecx. Returns 1531 hi byte in edx, lo in %eax */ 1532 salq $32,%rdx /* sign-shift %rdx left */ 1533 movl %eax,%eax /* zero-extend %eax -> %rax */ 1534 orq %rdx,%rax 1535 movq %rax,(%rsi) 1536 xorq %rax,%rax 1537 movq %rax,PCB_ONFAULT(%r8) 1538 POP_FRAME_POINTER 1539 ret 1540 1541 /* 1542 * Support for writing MSRs in the safe manner. (Instead of panic on #gp, 1543 * return an error.) 1544 */ 1545 ENTRY(wrmsr_safe) 1546 /* int wrmsr_safe(u_int msr, uint64_t data) */ 1547 PUSH_FRAME_POINTER 1548 movq PCPU(CURPCB),%r8 1549 movq $msr_onfault,PCB_ONFAULT(%r8) 1550 movl %edi,%ecx 1551 movl %esi,%eax 1552 sarq $32,%rsi 1553 movl %esi,%edx 1554 wrmsr /* Write MSR pointed by %ecx. Accepts 1555 hi byte in edx, lo in %eax. */ 1556 xorq %rax,%rax 1557 movq %rax,PCB_ONFAULT(%r8) 1558 POP_FRAME_POINTER 1559 ret 1560 1561 /* 1562 * MSR operations fault handler 1563 */ 1564 ALIGN_TEXT 1565 msr_onfault: 1566 movq $0,PCB_ONFAULT(%r8) 1567 movl $EFAULT,%eax 1568 POP_FRAME_POINTER 1569 ret 1570 1571 /* 1572 * void pmap_pti_pcid_invalidate(uint64_t ucr3, uint64_t kcr3); 1573 * Invalidates address space addressed by ucr3, then returns to kcr3. 1574 * Done in assembler to ensure no other memory accesses happen while 1575 * on ucr3. 1576 */ 1577 ALIGN_TEXT 1578 ENTRY(pmap_pti_pcid_invalidate) 1579 pushfq 1580 cli 1581 movq %rdi,%cr3 /* to user page table */ 1582 movq %rsi,%cr3 /* back to kernel */ 1583 popfq 1584 retq 1585 1586 /* 1587 * void pmap_pti_pcid_invlpg(uint64_t ucr3, uint64_t kcr3, vm_offset_t va); 1588 * Invalidates virtual address va in address space ucr3, then returns to kcr3. 1589 */ 1590 ALIGN_TEXT 1591 ENTRY(pmap_pti_pcid_invlpg) 1592 pushfq 1593 cli 1594 movq %rdi,%cr3 /* to user page table */ 1595 invlpg (%rdx) 1596 movq %rsi,%cr3 /* back to kernel */ 1597 popfq 1598 retq 1599 1600 /* 1601 * void pmap_pti_pcid_invlrng(uint64_t ucr3, uint64_t kcr3, vm_offset_t sva, 1602 * vm_offset_t eva); 1603 * Invalidates virtual addresses between sva and eva in address space ucr3, 1604 * then returns to kcr3. 1605 */ 1606 ALIGN_TEXT 1607 ENTRY(pmap_pti_pcid_invlrng) 1608 pushfq 1609 cli 1610 movq %rdi,%cr3 /* to user page table */ 1611 1: invlpg (%rdx) 1612 addq $PAGE_SIZE,%rdx 1613 cmpq %rdx,%rcx 1614 ja 1b 1615 movq %rsi,%cr3 /* back to kernel */ 1616 popfq 1617 retq 1618 1619 .altmacro 1620 .macro rsb_seq_label l 1621 rsb_seq_\l: 1622 .endm 1623 .macro rsb_call_label l 1624 call rsb_seq_\l 1625 .endm 1626 .macro rsb_seq count 1627 ll=1 1628 .rept \count 1629 rsb_call_label %(ll) 1630 nop 1631 rsb_seq_label %(ll) 1632 addq $8,%rsp 1633 ll=ll+1 1634 .endr 1635 .endm 1636 1637 ENTRY(rsb_flush) 1638 rsb_seq 32 1639 ret 1640 1641 /* all callers already saved %rax, %rdx, and %rcx */ 1642 ENTRY(handle_ibrs_entry) 1643 cmpb $0,hw_ibrs_ibpb_active(%rip) 1644 je 1f 1645 movl $MSR_IA32_SPEC_CTRL,%ecx 1646 rdmsr 1647 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax 1648 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32,%edx 1649 wrmsr 1650 movb $1,PCPU(IBPB_SET) 1651 testl $CPUID_STDEXT_SMEP,cpu_stdext_feature(%rip) 1652 je rsb_flush 1653 1: ret 1654 END(handle_ibrs_entry) 1655 1656 ENTRY(handle_ibrs_exit) 1657 cmpb $0,PCPU(IBPB_SET) 1658 je 1f 1659 movl $MSR_IA32_SPEC_CTRL,%ecx 1660 rdmsr 1661 andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax 1662 andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx 1663 wrmsr 1664 movb $0,PCPU(IBPB_SET) 1665 1: ret 1666 END(handle_ibrs_exit) 1667 1668 /* registers-neutral version, but needs stack */ 1669 ENTRY(handle_ibrs_exit_rs) 1670 cmpb $0,PCPU(IBPB_SET) 1671 je 1f 1672 pushq %rax 1673 pushq %rdx 1674 pushq %rcx 1675 movl $MSR_IA32_SPEC_CTRL,%ecx 1676 rdmsr 1677 andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax 1678 andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx 1679 wrmsr 1680 popq %rcx 1681 popq %rdx 1682 popq %rax 1683 movb $0,PCPU(IBPB_SET) 1684 1: ret 1685 END(handle_ibrs_exit_rs) 1686 1687 .noaltmacro 1688 1689 /* 1690 * Flush L1D cache. Load enough of the data from the kernel text 1691 * to flush existing L1D content. 1692 * 1693 * N.B. The function does not follow ABI calling conventions, it corrupts %rbx. 1694 * The vmm.ko caller expects that only %rax, %rdx, %rbx, %rcx, %r9, and %rflags 1695 * registers are clobbered. The NMI handler caller only needs %r13 preserved. 1696 */ 1697 ENTRY(flush_l1d_sw) 1698 #define L1D_FLUSH_SIZE (64 * 1024) 1699 movq $KERNBASE, %r9 1700 movq $-L1D_FLUSH_SIZE, %rcx 1701 /* 1702 * pass 1: Preload TLB. 1703 * Kernel text is mapped using superpages. TLB preload is 1704 * done for the benefit of older CPUs which split 2M page 1705 * into 4k TLB entries. 1706 */ 1707 1: movb L1D_FLUSH_SIZE(%r9, %rcx), %al 1708 addq $PAGE_SIZE, %rcx 1709 jne 1b 1710 xorl %eax, %eax 1711 cpuid 1712 movq $-L1D_FLUSH_SIZE, %rcx 1713 /* pass 2: Read each cache line. */ 1714 2: movb L1D_FLUSH_SIZE(%r9, %rcx), %al 1715 addq $64, %rcx 1716 jne 2b 1717 lfence 1718 ret 1719 #undef L1D_FLUSH_SIZE 1720 END(flush_l1d_sw) 1721 1722 ENTRY(flush_l1d_sw_abi) 1723 pushq %rbx 1724 call flush_l1d_sw 1725 popq %rbx 1726 ret 1727 END(flush_l1d_sw_abi) 1728 1729 ENTRY(mds_handler_void) 1730 retq 1731 END(mds_handler_void) 1732 1733 ENTRY(mds_handler_verw) 1734 subq $8, %rsp 1735 movw %ds, (%rsp) 1736 verw (%rsp) 1737 addq $8, %rsp 1738 retq 1739 END(mds_handler_verw) 1740 1741 ENTRY(mds_handler_ivb) 1742 pushq %rax 1743 pushq %rdx 1744 pushq %rcx 1745 1746 movq %cr0, %rax 1747 testb $CR0_TS, %al 1748 je 1f 1749 clts 1750 1: movq PCPU(MDS_BUF), %rdx 1751 movdqa %xmm0, PCPU(MDS_TMP) 1752 pxor %xmm0, %xmm0 1753 1754 lfence 1755 orpd (%rdx), %xmm0 1756 orpd (%rdx), %xmm0 1757 mfence 1758 movl $40, %ecx 1759 addq $16, %rdx 1760 2: movntdq %xmm0, (%rdx) 1761 addq $16, %rdx 1762 decl %ecx 1763 jnz 2b 1764 mfence 1765 1766 movdqa PCPU(MDS_TMP),%xmm0 1767 testb $CR0_TS, %al 1768 je 3f 1769 movq %rax, %cr0 1770 3: popq %rcx 1771 popq %rdx 1772 popq %rax 1773 retq 1774 END(mds_handler_ivb) 1775 1776 ENTRY(mds_handler_bdw) 1777 pushq %rax 1778 pushq %rbx 1779 pushq %rcx 1780 pushq %rdi 1781 pushq %rsi 1782 1783 movq %cr0, %rax 1784 testb $CR0_TS, %al 1785 je 1f 1786 clts 1787 1: movq PCPU(MDS_BUF), %rbx 1788 movdqa %xmm0, PCPU(MDS_TMP) 1789 pxor %xmm0, %xmm0 1790 1791 movq %rbx, %rdi 1792 movq %rbx, %rsi 1793 movl $40, %ecx 1794 2: movntdq %xmm0, (%rbx) 1795 addq $16, %rbx 1796 decl %ecx 1797 jnz 2b 1798 mfence 1799 movl $1536, %ecx 1800 rep; movsb 1801 lfence 1802 1803 movdqa PCPU(MDS_TMP),%xmm0 1804 testb $CR0_TS, %al 1805 je 3f 1806 movq %rax, %cr0 1807 3: popq %rsi 1808 popq %rdi 1809 popq %rcx 1810 popq %rbx 1811 popq %rax 1812 retq 1813 END(mds_handler_bdw) 1814 1815 ENTRY(mds_handler_skl_sse) 1816 pushq %rax 1817 pushq %rdx 1818 pushq %rcx 1819 pushq %rdi 1820 1821 movq %cr0, %rax 1822 testb $CR0_TS, %al 1823 je 1f 1824 clts 1825 1: movq PCPU(MDS_BUF), %rdi 1826 movq PCPU(MDS_BUF64), %rdx 1827 movdqa %xmm0, PCPU(MDS_TMP) 1828 pxor %xmm0, %xmm0 1829 1830 lfence 1831 orpd (%rdx), %xmm0 1832 orpd (%rdx), %xmm0 1833 xorl %eax, %eax 1834 2: clflushopt 5376(%rdi, %rax, 8) 1835 addl $8, %eax 1836 cmpl $8 * 12, %eax 1837 jb 2b 1838 sfence 1839 movl $6144, %ecx 1840 xorl %eax, %eax 1841 rep; stosb 1842 mfence 1843 1844 movdqa PCPU(MDS_TMP), %xmm0 1845 testb $CR0_TS, %al 1846 je 3f 1847 movq %rax, %cr0 1848 3: popq %rdi 1849 popq %rcx 1850 popq %rdx 1851 popq %rax 1852 retq 1853 END(mds_handler_skl_sse) 1854 1855 ENTRY(mds_handler_skl_avx) 1856 pushq %rax 1857 pushq %rdx 1858 pushq %rcx 1859 pushq %rdi 1860 1861 movq %cr0, %rax 1862 testb $CR0_TS, %al 1863 je 1f 1864 clts 1865 1: movq PCPU(MDS_BUF), %rdi 1866 movq PCPU(MDS_BUF64), %rdx 1867 vmovdqa %ymm0, PCPU(MDS_TMP) 1868 vpxor %ymm0, %ymm0, %ymm0 1869 1870 lfence 1871 vorpd (%rdx), %ymm0, %ymm0 1872 vorpd (%rdx), %ymm0, %ymm0 1873 xorl %eax, %eax 1874 2: clflushopt 5376(%rdi, %rax, 8) 1875 addl $8, %eax 1876 cmpl $8 * 12, %eax 1877 jb 2b 1878 sfence 1879 movl $6144, %ecx 1880 xorl %eax, %eax 1881 rep; stosb 1882 mfence 1883 1884 vmovdqa PCPU(MDS_TMP), %ymm0 1885 testb $CR0_TS, %al 1886 je 3f 1887 movq %rax, %cr0 1888 3: popq %rdi 1889 popq %rcx 1890 popq %rdx 1891 popq %rax 1892 retq 1893 END(mds_handler_skl_avx) 1894 1895 ENTRY(mds_handler_skl_avx512) 1896 pushq %rax 1897 pushq %rdx 1898 pushq %rcx 1899 pushq %rdi 1900 1901 movq %cr0, %rax 1902 testb $CR0_TS, %al 1903 je 1f 1904 clts 1905 1: movq PCPU(MDS_BUF), %rdi 1906 movq PCPU(MDS_BUF64), %rdx 1907 vmovdqa64 %zmm0, PCPU(MDS_TMP) 1908 vpxord %zmm0, %zmm0, %zmm0 1909 1910 lfence 1911 vorpd (%rdx), %zmm0, %zmm0 1912 vorpd (%rdx), %zmm0, %zmm0 1913 xorl %eax, %eax 1914 2: clflushopt 5376(%rdi, %rax, 8) 1915 addl $8, %eax 1916 cmpl $8 * 12, %eax 1917 jb 2b 1918 sfence 1919 movl $6144, %ecx 1920 xorl %eax, %eax 1921 rep; stosb 1922 mfence 1923 1924 vmovdqa64 PCPU(MDS_TMP), %zmm0 1925 testb $CR0_TS, %al 1926 je 3f 1927 movq %rax, %cr0 1928 3: popq %rdi 1929 popq %rcx 1930 popq %rdx 1931 popq %rax 1932 retq 1933 END(mds_handler_skl_avx512) 1934 1935 ENTRY(mds_handler_silvermont) 1936 pushq %rax 1937 pushq %rdx 1938 pushq %rcx 1939 1940 movq %cr0, %rax 1941 testb $CR0_TS, %al 1942 je 1f 1943 clts 1944 1: movq PCPU(MDS_BUF), %rdx 1945 movdqa %xmm0, PCPU(MDS_TMP) 1946 pxor %xmm0, %xmm0 1947 1948 movl $16, %ecx 1949 2: movntdq %xmm0, (%rdx) 1950 addq $16, %rdx 1951 decl %ecx 1952 jnz 2b 1953 mfence 1954 1955 movdqa PCPU(MDS_TMP),%xmm0 1956 testb $CR0_TS, %al 1957 je 3f 1958 movq %rax, %cr0 1959 3: popq %rcx 1960 popq %rdx 1961 popq %rax 1962 retq 1963 END(mds_handler_silvermont)
Cache object: 8362f0002cfdc967c1283619e96681b0
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]