The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/support.S

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2003 Peter Wemm.
    3  * Copyright (c) 1993 The Regents of the University of California.
    4  * All rights reserved.
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice, this list of conditions and the following disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  * 3. Neither the name of the University nor the names of its contributors
   15  *    may be used to endorse or promote products derived from this software
   16  *    without specific prior written permission.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  *
   30  * $FreeBSD: head/sys/amd64/amd64/support.S 341719 2018-12-08 04:53:08Z mjg $
   31  */
   32 
   33 #include "opt_ddb.h"
   34 
   35 #include <machine/asmacros.h>
   36 #include <machine/specialreg.h>
   37 #include <machine/pmap.h>
   38 
   39 #include "assym.inc"
   40 
   41         .text
   42 
   43 /* Address: %rdi */
   44 ENTRY(pagezero_std)
   45         PUSH_FRAME_POINTER
   46         movq    $PAGE_SIZE/8,%rcx
   47         xorl    %eax,%eax
   48         rep
   49         stosq
   50         POP_FRAME_POINTER
   51         ret
   52 END(pagezero_std)
   53 
   54 ENTRY(pagezero_erms)
   55         PUSH_FRAME_POINTER
   56         movq    $PAGE_SIZE,%rcx
   57         xorl    %eax,%eax
   58         rep
   59         stosb
   60         POP_FRAME_POINTER
   61         ret
   62 END(pagezero_erms)
   63 
   64 /*
   65  * pagecopy(%rdi=from, %rsi=to)
   66  */
   67 ENTRY(pagecopy)
   68         PUSH_FRAME_POINTER
   69         movq    $PAGE_SIZE/8,%rcx
   70         movq    %rdi,%r9
   71         movq    %rsi,%rdi
   72         movq    %r9,%rsi
   73         rep
   74         movsq
   75         POP_FRAME_POINTER
   76         ret
   77 END(pagecopy)
   78 
   79 /* Address: %rdi */
   80 ENTRY(sse2_pagezero)
   81         PUSH_FRAME_POINTER
   82         movq    $-PAGE_SIZE,%rdx
   83         subq    %rdx,%rdi
   84         xorl    %eax,%eax
   85         jmp     1f
   86         /*
   87          * The loop takes 29 bytes.  Ensure that it doesn't cross a 32-byte
   88          * cache line.
   89          */
   90         .p2align 5,0x90
   91 1:
   92         movnti  %rax,(%rdi,%rdx)
   93         movnti  %rax,8(%rdi,%rdx)
   94         movnti  %rax,16(%rdi,%rdx)
   95         movnti  %rax,24(%rdi,%rdx)
   96         addq    $32,%rdx
   97         jne     1b
   98         sfence
   99         POP_FRAME_POINTER
  100         ret
  101 END(sse2_pagezero)
  102 
  103 /*
  104  * memcmpy(b1, b2, len)
  105  *         rdi,rsi,len
  106  */
  107 ENTRY(memcmp)
  108         PUSH_FRAME_POINTER
  109         cmpq    $16,%rdx
  110         jae     5f
  111 1:
  112         testq   %rdx,%rdx
  113         je      3f
  114         xorl    %ecx,%ecx
  115 2:
  116         movzbl  (%rdi,%rcx,1),%eax
  117         movzbl  (%rsi,%rcx,1),%r8d
  118         cmpb    %r8b,%al
  119         jne     4f
  120         addq    $1,%rcx
  121         cmpq    %rcx,%rdx
  122         jz      3f
  123         movzbl  (%rdi,%rcx,1),%eax
  124         movzbl  (%rsi,%rcx,1),%r8d
  125         cmpb    %r8b,%al
  126         jne     4f
  127         addq    $1,%rcx
  128         cmpq    %rcx,%rdx
  129         jz      3f
  130         movzbl  (%rdi,%rcx,1),%eax
  131         movzbl  (%rsi,%rcx,1),%r8d
  132         cmpb    %r8b,%al
  133         jne     4f
  134         addq    $1,%rcx
  135         cmpq    %rcx,%rdx
  136         jz      3f
  137         movzbl  (%rdi,%rcx,1),%eax
  138         movzbl  (%rsi,%rcx,1),%r8d
  139         cmpb    %r8b,%al
  140         jne     4f
  141         addq    $1,%rcx
  142         cmpq    %rcx,%rdx
  143         jne     2b
  144 3:
  145         xorl    %eax,%eax
  146         POP_FRAME_POINTER
  147         ret
  148 4:
  149         subl    %r8d,%eax
  150         POP_FRAME_POINTER
  151         ret
  152 5:
  153         cmpq    $32,%rdx
  154         jae     7f
  155 6:
  156         /*
  157          * 8 bytes
  158          */
  159         movq    (%rdi),%r8
  160         movq    (%rsi),%r9
  161         cmpq    %r8,%r9
  162         jne     1b
  163         leaq    8(%rdi),%rdi
  164         leaq    8(%rsi),%rsi
  165         subq    $8,%rdx
  166         cmpq    $8,%rdx
  167         jae     6b
  168         jl      1b
  169         jmp     3b
  170 7:
  171         /*
  172          * 32 bytes
  173          */
  174         movq    (%rsi),%r8
  175         movq    8(%rsi),%r9
  176         subq    (%rdi),%r8
  177         subq    8(%rdi),%r9
  178         or      %r8,%r9
  179         jnz     1b
  180 
  181         movq    16(%rsi),%r8
  182         movq    24(%rsi),%r9
  183         subq    16(%rdi),%r8
  184         subq    24(%rdi),%r9
  185         or      %r8,%r9
  186         jnz     1b
  187 
  188         leaq    32(%rdi),%rdi
  189         leaq    32(%rsi),%rsi
  190         subq    $32,%rdx
  191         cmpq    $32,%rdx
  192         jae     7b
  193         jnz     1b
  194         jmp     3b
  195 END(memcmp)
  196 
  197 /*
  198  * memmove(dst, src, cnt)
  199  *         rdi, rsi, rdx
  200  */
  201 
  202 /*
  203  * Register state at entry is supposed to be as follows:
  204  * rdi - destination
  205  * rsi - source
  206  * rdx - count
  207  *
  208  * The macro possibly clobbers the above and: rcx, r8, r9, 10
  209  * It does not clobber rax nor r11.
  210  */
  211 .macro MEMMOVE erms overlap begin end
  212         \begin
  213 
  214         /*
  215          * For sizes 0..32 all data is read before it is written, so there
  216          * is no correctness issue with direction of copying.
  217          */
  218         cmpq    $32,%rcx
  219         jbe     101632f
  220 
  221 .if \overlap == 1
  222         movq    %rdi,%r8
  223         subq    %rsi,%r8
  224         cmpq    %rcx,%r8        /* overlapping && src < dst? */
  225         jb      2f
  226 .endif
  227 
  228         cmpq    $256,%rcx
  229         ja      1256f
  230 
  231 103200:
  232         movq    (%rsi),%rdx
  233         movq    %rdx,(%rdi)
  234         movq    8(%rsi),%rdx
  235         movq    %rdx,8(%rdi)
  236         movq    16(%rsi),%rdx
  237         movq    %rdx,16(%rdi)
  238         movq    24(%rsi),%rdx
  239         movq    %rdx,24(%rdi)
  240         leaq    32(%rsi),%rsi
  241         leaq    32(%rdi),%rdi
  242         subq    $32,%rcx
  243         cmpq    $32,%rcx
  244         jae     103200b
  245         cmpb    $0,%cl
  246         jne     101632f
  247         \end
  248         ret
  249         ALIGN_TEXT
  250 101632:
  251         cmpb    $16,%cl
  252         jl      100816f
  253         movq    (%rsi),%rdx
  254         movq    8(%rsi),%r8
  255         movq    -16(%rsi,%rcx),%r9
  256         movq    -8(%rsi,%rcx),%r10
  257         movq    %rdx,(%rdi)
  258         movq    %r8,8(%rdi)
  259         movq    %r9,-16(%rdi,%rcx)
  260         movq    %r10,-8(%rdi,%rcx)
  261         \end
  262         ret
  263         ALIGN_TEXT
  264 100816:
  265         cmpb    $8,%cl
  266         jl      100408f
  267         movq    (%rsi),%rdx
  268         movq    -8(%rsi,%rcx),%r8
  269         movq    %rdx,(%rdi)
  270         movq    %r8,-8(%rdi,%rcx,)
  271         \end
  272         ret
  273         ALIGN_TEXT
  274 100408:
  275         cmpb    $4,%cl
  276         jl      100204f
  277         movl    (%rsi),%edx
  278         movl    -4(%rsi,%rcx),%r8d
  279         movl    %edx,(%rdi)
  280         movl    %r8d,-4(%rdi,%rcx)
  281         \end
  282         ret
  283         ALIGN_TEXT
  284 100204:
  285         cmpb    $2,%cl
  286         jl      100001f
  287         movzwl  (%rsi),%edx
  288         movzwl  -2(%rsi,%rcx),%r8d
  289         movw    %dx,(%rdi)
  290         movw    %r8w,-2(%rdi,%rcx)
  291         \end
  292         ret
  293         ALIGN_TEXT
  294 100001:
  295         cmpb    $1,%cl
  296         jl      100000f
  297         movb    (%rsi),%dl
  298         movb    %dl,(%rdi)
  299 100000:
  300         \end
  301         ret
  302 
  303         ALIGN_TEXT
  304 1256:
  305         testb   $15,%dil
  306         jnz     100f
  307 .if \erms == 1
  308         rep
  309         movsb
  310 .else
  311         shrq    $3,%rcx                         /* copy by 64-bit words */
  312         rep
  313         movsq
  314         movq    %rdx,%rcx
  315         andl    $7,%ecx                         /* any bytes left? */
  316         jne     100408b
  317 .endif
  318         \end
  319         ret
  320 100:
  321         movq    (%rsi),%r8
  322         movq    8(%rsi),%r9
  323         movq    %rdi,%r10
  324         movq    %rdi,%rcx
  325         andq    $15,%rcx
  326         leaq    -16(%rdx,%rcx),%rdx
  327         neg     %rcx
  328         leaq    16(%rdi,%rcx),%rdi
  329         leaq    16(%rsi,%rcx),%rsi
  330         movq    %rdx,%rcx
  331 .if \erms == 1
  332         rep
  333         movsb
  334         movq    %r8,(%r10)
  335         movq    %r9,8(%r10)
  336 .else
  337         shrq    $3,%rcx                         /* copy by 64-bit words */
  338         rep
  339         movsq
  340         movq    %r8,(%r10)
  341         movq    %r9,8(%r10)
  342         movq    %rdx,%rcx
  343         andl    $7,%ecx                         /* any bytes left? */
  344         jne     100408b
  345 .endif
  346         \end
  347         ret
  348 
  349 .if \overlap == 1
  350         /*
  351          * Copy backwards.
  352          */
  353         ALIGN_TEXT
  354 2:
  355         cmpq    $256,%rcx
  356         ja      2256f
  357 
  358         leaq    -8(%rdi,%rcx),%rdi
  359         leaq    -8(%rsi,%rcx),%rsi
  360 
  361         cmpq    $32,%rcx
  362         jb      2016f
  363 
  364 2032:
  365         movq    (%rsi),%rdx
  366         movq    %rdx,(%rdi)
  367         movq    -8(%rsi),%rdx
  368         movq    %rdx,-8(%rdi)
  369         movq    -16(%rsi),%rdx
  370         movq    %rdx,-16(%rdi)
  371         movq    -24(%rsi),%rdx
  372         movq    %rdx,-24(%rdi)
  373         leaq    -32(%rsi),%rsi
  374         leaq    -32(%rdi),%rdi
  375         subq    $32,%rcx
  376         cmpq    $32,%rcx
  377         jae     2032b
  378         cmpb    $0,%cl
  379         jne     2016f
  380         \end
  381         ret
  382         ALIGN_TEXT
  383 2016:
  384         cmpb    $16,%cl
  385         jl      2008f
  386         movq    (%rsi),%rdx
  387         movq    %rdx,(%rdi)
  388         movq    -8(%rsi),%rdx
  389         movq    %rdx,-8(%rdi)
  390         subb    $16,%cl
  391         jz      2000f
  392         leaq    -16(%rsi),%rsi
  393         leaq    -16(%rdi),%rdi
  394 2008:
  395         cmpb    $8,%cl
  396         jl      2004f
  397         movq    (%rsi),%rdx
  398         movq    %rdx,(%rdi)
  399         subb    $8,%cl
  400         jz      2000f
  401         leaq    -8(%rsi),%rsi
  402         leaq    -8(%rdi),%rdi
  403 2004:
  404         cmpb    $4,%cl
  405         jl      2002f
  406         movl    4(%rsi),%edx
  407         movl    %edx,4(%rdi)
  408         subb    $4,%cl
  409         jz      2000f
  410         leaq    -4(%rsi),%rsi
  411         leaq    -4(%rdi),%rdi
  412 2002:
  413         cmpb    $2,%cl
  414         jl      2001f
  415         movw    6(%rsi),%dx
  416         movw    %dx,6(%rdi)
  417         subb    $2,%cl
  418         jz      2000f
  419         leaq    -2(%rsi),%rsi
  420         leaq    -2(%rdi),%rdi
  421 2001:
  422         cmpb    $1,%cl
  423         jl      2000f
  424         movb    7(%rsi),%dl
  425         movb    %dl,7(%rdi)
  426 2000:
  427         \end
  428         ret
  429         ALIGN_TEXT
  430 2256:
  431         std
  432 .if \erms == 1
  433         leaq    -1(%rdi,%rcx),%rdi
  434         leaq    -1(%rsi,%rcx),%rsi
  435         rep
  436         movsb
  437         cld
  438 .else
  439         leaq    -8(%rdi,%rcx),%rdi
  440         leaq    -8(%rsi,%rcx),%rsi
  441         shrq    $3,%rcx
  442         rep
  443         movsq
  444         cld
  445         movq    %rdx,%rcx
  446         andb    $7,%cl
  447         jne     2004b
  448 .endif
  449         \end
  450         ret
  451 .endif
  452 .endm
  453 
  454 .macro MEMMOVE_BEGIN
  455         PUSH_FRAME_POINTER
  456         movq    %rdi,%rax
  457         movq    %rdx,%rcx
  458 .endm
  459 
  460 .macro MEMMOVE_END
  461         POP_FRAME_POINTER
  462 .endm
  463 
  464 ENTRY(memmove_std)
  465         MEMMOVE erms=0 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END
  466 END(memmove_std)
  467 
  468 ENTRY(memmove_erms)
  469         MEMMOVE erms=1 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END
  470 END(memmove_erms)
  471 
  472 /*
  473  * memcpy(dst, src, len)
  474  *        rdi, rsi, rdx
  475  *
  476  * Note: memcpy does not support overlapping copies
  477  */
  478 ENTRY(memcpy_std)
  479         MEMMOVE erms=0 overlap=0 begin=MEMMOVE_BEGIN end=MEMMOVE_END
  480 END(memcpy_std)
  481 
  482 ENTRY(memcpy_erms)
  483         MEMMOVE erms=1 overlap=0 begin=MEMMOVE_BEGIN end=MEMMOVE_END
  484 END(memcpy_erms)
  485 
  486 /*
  487  * memset(dst, c,   len)
  488  *        rdi, rsi, rdx
  489  */
  490 .macro MEMSET erms
  491         PUSH_FRAME_POINTER
  492         movq    %rdi,%rax
  493         movq    %rdx,%rcx
  494         movzbq  %sil,%r8
  495         movabs  $0x0101010101010101,%r10
  496         imulq   %r8,%r10
  497 
  498         cmpq    $32,%rcx
  499         jbe     101632f
  500 
  501         cmpq    $256,%rcx
  502         ja      1256f
  503 
  504 103200:
  505         movq    %r10,(%rdi)
  506         movq    %r10,8(%rdi)
  507         movq    %r10,16(%rdi)
  508         movq    %r10,24(%rdi)
  509         leaq    32(%rdi),%rdi
  510         subq    $32,%rcx
  511         cmpq    $32,%rcx
  512         ja      103200b
  513         cmpb    $16,%cl
  514         ja      201632f
  515         movq    %r10,-16(%rdi,%rcx)
  516         movq    %r10,-8(%rdi,%rcx)
  517         POP_FRAME_POINTER
  518         ret
  519         ALIGN_TEXT
  520 101632:
  521         cmpb    $16,%cl
  522         jl      100816f
  523 201632:
  524         movq    %r10,(%rdi)
  525         movq    %r10,8(%rdi)
  526         movq    %r10,-16(%rdi,%rcx)
  527         movq    %r10,-8(%rdi,%rcx)
  528         POP_FRAME_POINTER
  529         ret
  530         ALIGN_TEXT
  531 100816:
  532         cmpb    $8,%cl
  533         jl      100408f
  534         movq    %r10,(%rdi)
  535         movq    %r10,-8(%rdi,%rcx)
  536         POP_FRAME_POINTER
  537         ret
  538         ALIGN_TEXT
  539 100408:
  540         cmpb    $4,%cl
  541         jl      100204f
  542         movl    %r10d,(%rdi)
  543         movl    %r10d,-4(%rdi,%rcx)
  544         POP_FRAME_POINTER
  545         ret
  546         ALIGN_TEXT
  547 100204:
  548         cmpb    $2,%cl
  549         jl      100001f
  550         movw    %r10w,(%rdi)
  551         movw    %r10w,-2(%rdi,%rcx)
  552         POP_FRAME_POINTER
  553         ret
  554         ALIGN_TEXT
  555 100001:
  556         cmpb    $0,%cl
  557         je      100000f
  558         movb    %r10b,(%rdi)
  559 100000:
  560         POP_FRAME_POINTER
  561         ret
  562         ALIGN_TEXT
  563 1256:
  564         movq    %rdi,%r9
  565         movq    %r10,%rax
  566         testl   $15,%edi
  567         jnz     3f
  568 1:
  569 .if \erms == 1
  570         rep
  571         stosb
  572         movq    %r9,%rax
  573 .else
  574         movq    %rcx,%rdx
  575         shrq    $3,%rcx
  576         rep
  577         stosq
  578         movq    %r9,%rax
  579         andl    $7,%edx
  580         jnz     2f
  581         POP_FRAME_POINTER
  582         ret
  583 2:
  584         movq    %r10,-8(%rdi,%rdx)
  585 .endif
  586         POP_FRAME_POINTER
  587         ret
  588         ALIGN_TEXT
  589 3:
  590         movq    %r10,(%rdi)
  591         movq    %r10,8(%rdi)
  592         movq    %rdi,%r8
  593         andq    $15,%r8
  594         leaq    -16(%rcx,%r8),%rcx
  595         neg     %r8
  596         leaq    16(%rdi,%r8),%rdi
  597         jmp     1b
  598 .endm
  599 
  600 ENTRY(memset_std)
  601         MEMSET erms=0
  602 END(memset_std)
  603 
  604 ENTRY(memset_erms)
  605         MEMSET erms=1
  606 END(memset_erms)
  607 
  608 /* fillw(pat, base, cnt) */
  609 /*       %rdi,%rsi, %rdx */
  610 ENTRY(fillw)
  611         PUSH_FRAME_POINTER
  612         movq    %rdi,%rax
  613         movq    %rsi,%rdi
  614         movq    %rdx,%rcx
  615         rep
  616         stosw
  617         POP_FRAME_POINTER
  618         ret
  619 END(fillw)
  620 
  621 /*****************************************************************************/
  622 /* copyout and fubyte family                                                 */
  623 /*****************************************************************************/
  624 /*
  625  * Access user memory from inside the kernel. These routines should be
  626  * the only places that do this.
  627  *
  628  * These routines set curpcb->pcb_onfault for the time they execute. When a
  629  * protection violation occurs inside the functions, the trap handler
  630  * returns to *curpcb->pcb_onfault instead of the function.
  631  */
  632 
  633 .macro SMAP_DISABLE smap
  634 .if     \smap
  635         stac
  636 .endif
  637 .endm
  638 
  639 
  640 .macro SMAP_ENABLE smap
  641 .if     \smap
  642         clac
  643 .endif
  644 .endm
  645 
  646 .macro COPYINOUT_BEGIN
  647 .endm
  648 
  649 .macro COPYINOUT_END
  650         movq    %rax,PCB_ONFAULT(%r11)
  651         POP_FRAME_POINTER
  652 .endm
  653 
  654 .macro COPYINOUT_SMAP_END
  655         SMAP_ENABLE smap=1
  656         COPYINOUT_END
  657 .endm
  658 
  659 /*
  660  * copyout(from_kernel, to_user, len)
  661  *         %rdi,        %rsi,    %rdx
  662  */
  663 .macro  COPYOUT smap erms
  664         PUSH_FRAME_POINTER
  665         movq    PCPU(CURPCB),%r11
  666         movq    $copy_fault,PCB_ONFAULT(%r11)
  667 
  668         /*
  669          * Check explicitly for non-user addresses.  If 486 write protection
  670          * is being used, this check is essential because we are in kernel
  671          * mode so the h/w does not provide any protection against writing
  672          * kernel addresses.
  673          */
  674 
  675         /*
  676          * First, prevent address wrapping.
  677          */
  678         movq    %rsi,%rax
  679         addq    %rdx,%rax
  680         jc      copy_fault
  681 /*
  682  * XXX STOP USING VM_MAXUSER_ADDRESS.
  683  * It is an end address, not a max, so every time it is used correctly it
  684  * looks like there is an off by one error, and of course it caused an off
  685  * by one error in several places.
  686  */
  687         movq    $VM_MAXUSER_ADDRESS,%rcx
  688         cmpq    %rcx,%rax
  689         ja      copy_fault
  690 
  691         /*
  692          * Set return value to zero. Remaining failure mode goes through
  693          * copy_fault.
  694          */
  695         xorl    %eax,%eax
  696 
  697         /*
  698          * Set up arguments for MEMMOVE.
  699          */
  700         movq    %rdi,%r8
  701         movq    %rsi,%rdi
  702         movq    %r8,%rsi
  703         movq    %rdx,%rcx
  704 
  705 
  706         SMAP_DISABLE \smap
  707 .if     \smap == 1
  708         MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_SMAP_END
  709 .else
  710         MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_END
  711 .endif
  712         /* NOTREACHED */
  713 .endm
  714 
  715 ENTRY(copyout_nosmap_std)
  716         COPYOUT smap=0 erms=0
  717 END(copyout_nosmap_std)
  718 
  719 ENTRY(copyout_smap_std)
  720         COPYOUT smap=1 erms=0
  721 END(copyout_smap_std)
  722 
  723 ENTRY(copyout_nosmap_erms)
  724         COPYOUT smap=0 erms=1
  725 END(copyout_nosmap_erms)
  726 
  727 ENTRY(copyout_smap_erms)
  728         COPYOUT smap=1 erms=1
  729 END(copyout_smap_erms)
  730 
  731 /*
  732  * copyin(from_user, to_kernel, len)
  733  *        %rdi,      %rsi,      %rdx
  734  */
  735 .macro  COPYIN smap erms
  736         PUSH_FRAME_POINTER
  737         movq    PCPU(CURPCB),%r11
  738         movq    $copy_fault,PCB_ONFAULT(%r11)
  739 
  740         /*
  741          * make sure address is valid
  742          */
  743         movq    %rdi,%rax
  744         addq    %rdx,%rax
  745         jc      copy_fault
  746         movq    $VM_MAXUSER_ADDRESS,%rcx
  747         cmpq    %rcx,%rax
  748         ja      copy_fault
  749 
  750         xorl    %eax,%eax
  751 
  752         movq    %rdi,%r8
  753         movq    %rsi,%rdi
  754         movq    %r8,%rsi
  755         movq    %rdx,%rcx
  756 
  757         SMAP_DISABLE \smap
  758 .if     \smap == 1
  759         MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_SMAP_END
  760 .else
  761         MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_END
  762 .endif
  763         /* NOTREACHED */
  764 .endm
  765 
  766 ENTRY(copyin_nosmap_std)
  767         COPYIN smap=0 erms=0
  768 END(copyin_nosmap_std)
  769 
  770 ENTRY(copyin_smap_std)
  771         COPYIN smap=1 erms=0
  772 END(copyin_smap_std)
  773 
  774 ENTRY(copyin_nosmap_erms)
  775         COPYIN smap=0 erms=1
  776 END(copyin_nosmap_erms)
  777 
  778 ENTRY(copyin_smap_erms)
  779         COPYIN smap=1 erms=1
  780 END(copyin_smap_erms)
  781 
  782         ALIGN_TEXT
  783         /* Trap entry clears PSL.AC */
  784 copy_fault:
  785         movq    $0,PCB_ONFAULT(%r11)
  786         movl    $EFAULT,%eax
  787         POP_FRAME_POINTER
  788         ret
  789 
  790 /*
  791  * casueword32.  Compare and set user integer.  Returns -1 on fault,
  792  *        0 if access was successful.  Old value is written to *oldp.
  793  *        dst = %rdi, old = %esi, oldp = %rdx, new = %ecx
  794  */
  795 ENTRY(casueword32_nosmap)
  796         PUSH_FRAME_POINTER
  797         movq    PCPU(CURPCB),%r8
  798         movq    $fusufault,PCB_ONFAULT(%r8)
  799 
  800         movq    $VM_MAXUSER_ADDRESS-4,%rax
  801         cmpq    %rax,%rdi                       /* verify address is valid */
  802         ja      fusufault
  803 
  804         movl    %esi,%eax                       /* old */
  805 #ifdef SMP
  806         lock
  807 #endif
  808         cmpxchgl %ecx,(%rdi)                    /* new = %ecx */
  809 
  810         /*
  811          * The old value is in %eax.  If the store succeeded it will be the
  812          * value we expected (old) from before the store, otherwise it will
  813          * be the current value.  Save %eax into %esi to prepare the return
  814          * value.
  815          */
  816         movl    %eax,%esi
  817         xorl    %eax,%eax
  818         movq    %rax,PCB_ONFAULT(%r8)
  819 
  820         /*
  821          * Access the oldp after the pcb_onfault is cleared, to correctly
  822          * catch corrupted pointer.
  823          */
  824         movl    %esi,(%rdx)                     /* oldp = %rdx */
  825         POP_FRAME_POINTER
  826         ret
  827 END(casueword32_nosmap)
  828 
  829 ENTRY(casueword32_smap)
  830         PUSH_FRAME_POINTER
  831         movq    PCPU(CURPCB),%r8
  832         movq    $fusufault,PCB_ONFAULT(%r8)
  833 
  834         movq    $VM_MAXUSER_ADDRESS-4,%rax
  835         cmpq    %rax,%rdi                       /* verify address is valid */
  836         ja      fusufault
  837 
  838         movl    %esi,%eax                       /* old */
  839         stac
  840 #ifdef SMP
  841         lock
  842 #endif
  843         cmpxchgl %ecx,(%rdi)                    /* new = %ecx */
  844         clac
  845 
  846         /*
  847          * The old value is in %eax.  If the store succeeded it will be the
  848          * value we expected (old) from before the store, otherwise it will
  849          * be the current value.  Save %eax into %esi to prepare the return
  850          * value.
  851          */
  852         movl    %eax,%esi
  853         xorl    %eax,%eax
  854         movq    %rax,PCB_ONFAULT(%r8)
  855 
  856         /*
  857          * Access the oldp after the pcb_onfault is cleared, to correctly
  858          * catch corrupted pointer.
  859          */
  860         movl    %esi,(%rdx)                     /* oldp = %rdx */
  861         POP_FRAME_POINTER
  862         ret
  863 END(casueword32_smap)
  864 
  865 /*
  866  * casueword.  Compare and set user long.  Returns -1 on fault,
  867  *        0 if access was successful.  Old value is written to *oldp.
  868  *        dst = %rdi, old = %rsi, oldp = %rdx, new = %rcx
  869  */
  870 ENTRY(casueword_nosmap)
  871         PUSH_FRAME_POINTER
  872         movq    PCPU(CURPCB),%r8
  873         movq    $fusufault,PCB_ONFAULT(%r8)
  874 
  875         movq    $VM_MAXUSER_ADDRESS-4,%rax
  876         cmpq    %rax,%rdi                       /* verify address is valid */
  877         ja      fusufault
  878 
  879         movq    %rsi,%rax                       /* old */
  880 #ifdef SMP
  881         lock
  882 #endif
  883         cmpxchgq %rcx,(%rdi)                    /* new = %rcx */
  884 
  885         /*
  886          * The old value is in %rax.  If the store succeeded it will be the
  887          * value we expected (old) from before the store, otherwise it will
  888          * be the current value.
  889          */
  890         movq    %rax,%rsi
  891         xorl    %eax,%eax
  892         movq    %rax,PCB_ONFAULT(%r8)
  893         movq    %rsi,(%rdx)
  894         POP_FRAME_POINTER
  895         ret
  896 END(casueword_nosmap)
  897 
  898 ENTRY(casueword_smap)
  899         PUSH_FRAME_POINTER
  900         movq    PCPU(CURPCB),%r8
  901         movq    $fusufault,PCB_ONFAULT(%r8)
  902 
  903         movq    $VM_MAXUSER_ADDRESS-4,%rax
  904         cmpq    %rax,%rdi                       /* verify address is valid */
  905         ja      fusufault
  906 
  907         movq    %rsi,%rax                       /* old */
  908         stac
  909 #ifdef SMP
  910         lock
  911 #endif
  912         cmpxchgq %rcx,(%rdi)                    /* new = %rcx */
  913         clac
  914 
  915         /*
  916          * The old value is in %rax.  If the store succeeded it will be the
  917          * value we expected (old) from before the store, otherwise it will
  918          * be the current value.
  919          */
  920         movq    %rax,%rsi
  921         xorl    %eax,%eax
  922         movq    %rax,PCB_ONFAULT(%r8)
  923         movq    %rsi,(%rdx)
  924         POP_FRAME_POINTER
  925         ret
  926 END(casueword_smap)
  927 
  928 /*
  929  * Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit
  930  * byte from user memory.
  931  * addr = %rdi, valp = %rsi
  932  */
  933 
  934 ENTRY(fueword_nosmap)
  935         PUSH_FRAME_POINTER
  936         movq    PCPU(CURPCB),%rcx
  937         movq    $fusufault,PCB_ONFAULT(%rcx)
  938 
  939         movq    $VM_MAXUSER_ADDRESS-8,%rax
  940         cmpq    %rax,%rdi                       /* verify address is valid */
  941         ja      fusufault
  942 
  943         xorl    %eax,%eax
  944         movq    (%rdi),%r11
  945         movq    %rax,PCB_ONFAULT(%rcx)
  946         movq    %r11,(%rsi)
  947         POP_FRAME_POINTER
  948         ret
  949 END(fueword_nosmap)
  950 
  951 ENTRY(fueword_smap)
  952         PUSH_FRAME_POINTER
  953         movq    PCPU(CURPCB),%rcx
  954         movq    $fusufault,PCB_ONFAULT(%rcx)
  955 
  956         movq    $VM_MAXUSER_ADDRESS-8,%rax
  957         cmpq    %rax,%rdi                       /* verify address is valid */
  958         ja      fusufault
  959 
  960         xorl    %eax,%eax
  961         stac
  962         movq    (%rdi),%r11
  963         clac
  964         movq    %rax,PCB_ONFAULT(%rcx)
  965         movq    %r11,(%rsi)
  966         POP_FRAME_POINTER
  967         ret
  968 END(fueword_smap)
  969 
  970 ENTRY(fueword32_nosmap)
  971         PUSH_FRAME_POINTER
  972         movq    PCPU(CURPCB),%rcx
  973         movq    $fusufault,PCB_ONFAULT(%rcx)
  974 
  975         movq    $VM_MAXUSER_ADDRESS-4,%rax
  976         cmpq    %rax,%rdi                       /* verify address is valid */
  977         ja      fusufault
  978 
  979         xorl    %eax,%eax
  980         movl    (%rdi),%r11d
  981         movq    %rax,PCB_ONFAULT(%rcx)
  982         movl    %r11d,(%rsi)
  983         POP_FRAME_POINTER
  984         ret
  985 END(fueword32_nosmap)
  986 
  987 ENTRY(fueword32_smap)
  988         PUSH_FRAME_POINTER
  989         movq    PCPU(CURPCB),%rcx
  990         movq    $fusufault,PCB_ONFAULT(%rcx)
  991 
  992         movq    $VM_MAXUSER_ADDRESS-4,%rax
  993         cmpq    %rax,%rdi                       /* verify address is valid */
  994         ja      fusufault
  995 
  996         xorl    %eax,%eax
  997         stac
  998         movl    (%rdi),%r11d
  999         clac
 1000         movq    %rax,PCB_ONFAULT(%rcx)
 1001         movl    %r11d,(%rsi)
 1002         POP_FRAME_POINTER
 1003         ret
 1004 END(fueword32_smap)
 1005 
 1006 ENTRY(fuword16_nosmap)
 1007         PUSH_FRAME_POINTER
 1008         movq    PCPU(CURPCB),%rcx
 1009         movq    $fusufault,PCB_ONFAULT(%rcx)
 1010 
 1011         movq    $VM_MAXUSER_ADDRESS-2,%rax
 1012         cmpq    %rax,%rdi
 1013         ja      fusufault
 1014 
 1015         movzwl  (%rdi),%eax
 1016         movq    $0,PCB_ONFAULT(%rcx)
 1017         POP_FRAME_POINTER
 1018         ret
 1019 END(fuword16_nosmap)
 1020 
 1021 ENTRY(fuword16_smap)
 1022         PUSH_FRAME_POINTER
 1023         movq    PCPU(CURPCB),%rcx
 1024         movq    $fusufault,PCB_ONFAULT(%rcx)
 1025 
 1026         movq    $VM_MAXUSER_ADDRESS-2,%rax
 1027         cmpq    %rax,%rdi
 1028         ja      fusufault
 1029 
 1030         stac
 1031         movzwl  (%rdi),%eax
 1032         clac
 1033         movq    $0,PCB_ONFAULT(%rcx)
 1034         POP_FRAME_POINTER
 1035         ret
 1036 END(fuword16_smap)
 1037 
 1038 ENTRY(fubyte_nosmap)
 1039         PUSH_FRAME_POINTER
 1040         movq    PCPU(CURPCB),%rcx
 1041         movq    $fusufault,PCB_ONFAULT(%rcx)
 1042 
 1043         movq    $VM_MAXUSER_ADDRESS-1,%rax
 1044         cmpq    %rax,%rdi
 1045         ja      fusufault
 1046 
 1047         movzbl  (%rdi),%eax
 1048         movq    $0,PCB_ONFAULT(%rcx)
 1049         POP_FRAME_POINTER
 1050         ret
 1051 END(fubyte_nosmap)
 1052 
 1053 ENTRY(fubyte_smap)
 1054         PUSH_FRAME_POINTER
 1055         movq    PCPU(CURPCB),%rcx
 1056         movq    $fusufault,PCB_ONFAULT(%rcx)
 1057 
 1058         movq    $VM_MAXUSER_ADDRESS-1,%rax
 1059         cmpq    %rax,%rdi
 1060         ja      fusufault
 1061 
 1062         stac
 1063         movzbl  (%rdi),%eax
 1064         clac
 1065         movq    $0,PCB_ONFAULT(%rcx)
 1066         POP_FRAME_POINTER
 1067         ret
 1068 END(fubyte_smap)
 1069 
 1070 /*
 1071  * Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to
 1072  * user memory.
 1073  * addr = %rdi, value = %rsi
 1074  */
 1075 ENTRY(suword_nosmap)
 1076         PUSH_FRAME_POINTER
 1077         movq    PCPU(CURPCB),%rcx
 1078         movq    $fusufault,PCB_ONFAULT(%rcx)
 1079 
 1080         movq    $VM_MAXUSER_ADDRESS-8,%rax
 1081         cmpq    %rax,%rdi                       /* verify address validity */
 1082         ja      fusufault
 1083 
 1084         movq    %rsi,(%rdi)
 1085         xorl    %eax,%eax
 1086         movq    PCPU(CURPCB),%rcx
 1087         movq    %rax,PCB_ONFAULT(%rcx)
 1088         POP_FRAME_POINTER
 1089         ret
 1090 END(suword_nosmap)
 1091 
 1092 ENTRY(suword_smap)
 1093         PUSH_FRAME_POINTER
 1094         movq    PCPU(CURPCB),%rcx
 1095         movq    $fusufault,PCB_ONFAULT(%rcx)
 1096 
 1097         movq    $VM_MAXUSER_ADDRESS-8,%rax
 1098         cmpq    %rax,%rdi                       /* verify address validity */
 1099         ja      fusufault
 1100 
 1101         stac
 1102         movq    %rsi,(%rdi)
 1103         clac
 1104         xorl    %eax,%eax
 1105         movq    PCPU(CURPCB),%rcx
 1106         movq    %rax,PCB_ONFAULT(%rcx)
 1107         POP_FRAME_POINTER
 1108         ret
 1109 END(suword_smap)
 1110 
 1111 ENTRY(suword32_nosmap)
 1112         PUSH_FRAME_POINTER
 1113         movq    PCPU(CURPCB),%rcx
 1114         movq    $fusufault,PCB_ONFAULT(%rcx)
 1115 
 1116         movq    $VM_MAXUSER_ADDRESS-4,%rax
 1117         cmpq    %rax,%rdi                       /* verify address validity */
 1118         ja      fusufault
 1119 
 1120         movl    %esi,(%rdi)
 1121         xorl    %eax,%eax
 1122         movq    PCPU(CURPCB),%rcx
 1123         movq    %rax,PCB_ONFAULT(%rcx)
 1124         POP_FRAME_POINTER
 1125         ret
 1126 END(suword32_nosmap)
 1127 
 1128 ENTRY(suword32_smap)
 1129         PUSH_FRAME_POINTER
 1130         movq    PCPU(CURPCB),%rcx
 1131         movq    $fusufault,PCB_ONFAULT(%rcx)
 1132 
 1133         movq    $VM_MAXUSER_ADDRESS-4,%rax
 1134         cmpq    %rax,%rdi                       /* verify address validity */
 1135         ja      fusufault
 1136 
 1137         stac
 1138         movl    %esi,(%rdi)
 1139         clac
 1140         xorl    %eax,%eax
 1141         movq    PCPU(CURPCB),%rcx
 1142         movq    %rax,PCB_ONFAULT(%rcx)
 1143         POP_FRAME_POINTER
 1144         ret
 1145 END(suword32_smap)
 1146 
 1147 ENTRY(suword16_nosmap)
 1148         PUSH_FRAME_POINTER
 1149         movq    PCPU(CURPCB),%rcx
 1150         movq    $fusufault,PCB_ONFAULT(%rcx)
 1151 
 1152         movq    $VM_MAXUSER_ADDRESS-2,%rax
 1153         cmpq    %rax,%rdi                       /* verify address validity */
 1154         ja      fusufault
 1155 
 1156         movw    %si,(%rdi)
 1157         xorl    %eax,%eax
 1158         movq    %rax,PCB_ONFAULT(%rcx)
 1159         POP_FRAME_POINTER
 1160         ret
 1161 END(suword16_nosmap)
 1162 
 1163 ENTRY(suword16_smap)
 1164         PUSH_FRAME_POINTER
 1165         movq    PCPU(CURPCB),%rcx
 1166         movq    $fusufault,PCB_ONFAULT(%rcx)
 1167 
 1168         movq    $VM_MAXUSER_ADDRESS-2,%rax
 1169         cmpq    %rax,%rdi                       /* verify address validity */
 1170         ja      fusufault
 1171 
 1172         stac
 1173         movw    %si,(%rdi)
 1174         clac
 1175         xorl    %eax,%eax
 1176         movq    %rax,PCB_ONFAULT(%rcx)
 1177         POP_FRAME_POINTER
 1178         ret
 1179 END(suword16_smap)
 1180 
 1181 ENTRY(subyte_nosmap)
 1182         PUSH_FRAME_POINTER
 1183         movq    PCPU(CURPCB),%rcx
 1184         movq    $fusufault,PCB_ONFAULT(%rcx)
 1185 
 1186         movq    $VM_MAXUSER_ADDRESS-1,%rax
 1187         cmpq    %rax,%rdi                       /* verify address validity */
 1188         ja      fusufault
 1189 
 1190         movl    %esi,%eax
 1191         movb    %al,(%rdi)
 1192         xorl    %eax,%eax
 1193         movq    %rax,PCB_ONFAULT(%rcx)
 1194         POP_FRAME_POINTER
 1195         ret
 1196 END(subyte_nosmap)
 1197 
 1198 ENTRY(subyte_smap)
 1199         PUSH_FRAME_POINTER
 1200         movq    PCPU(CURPCB),%rcx
 1201         movq    $fusufault,PCB_ONFAULT(%rcx)
 1202 
 1203         movq    $VM_MAXUSER_ADDRESS-1,%rax
 1204         cmpq    %rax,%rdi                       /* verify address validity */
 1205         ja      fusufault
 1206 
 1207         movl    %esi,%eax
 1208         stac
 1209         movb    %al,(%rdi)
 1210         clac
 1211         xorl    %eax,%eax
 1212         movq    %rax,PCB_ONFAULT(%rcx)
 1213         POP_FRAME_POINTER
 1214         ret
 1215 END(subyte_smap)
 1216 
 1217         ALIGN_TEXT
 1218         /* Fault entry clears PSL.AC */
 1219 fusufault:
 1220         movq    PCPU(CURPCB),%rcx
 1221         xorl    %eax,%eax
 1222         movq    %rax,PCB_ONFAULT(%rcx)
 1223         decq    %rax
 1224         POP_FRAME_POINTER
 1225         ret
 1226 
 1227 /*
 1228  * copyinstr(from, to, maxlen, int *lencopied)
 1229  *           %rdi, %rsi, %rdx, %rcx
 1230  *
 1231  *      copy a string from 'from' to 'to', stop when a 0 character is reached.
 1232  *      return ENAMETOOLONG if string is longer than maxlen, and
 1233  *      EFAULT on protection violations. If lencopied is non-zero,
 1234  *      return the actual length in *lencopied.
 1235  */
 1236 .macro COPYINSTR smap
 1237         PUSH_FRAME_POINTER
 1238         movq    %rdx,%r8                        /* %r8 = maxlen */
 1239         movq    PCPU(CURPCB),%r9
 1240         movq    $cpystrflt,PCB_ONFAULT(%r9)
 1241 
 1242         movq    $VM_MAXUSER_ADDRESS,%rax
 1243 
 1244         /* make sure 'from' is within bounds */
 1245         subq    %rdi,%rax
 1246         jbe     cpystrflt
 1247 
 1248         SMAP_DISABLE \smap
 1249 
 1250         /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */
 1251         cmpq    %rdx,%rax
 1252         jb      8f
 1253 1:
 1254         incq    %rdx
 1255 2:
 1256         decq    %rdx
 1257 .if \smap == 0
 1258         jz      copyinstr_toolong
 1259 .else
 1260         jz      copyinstr_toolong_smap
 1261 .endif
 1262 
 1263         movb    (%rdi),%al
 1264         movb    %al,(%rsi)
 1265         incq    %rsi
 1266         incq    %rdi
 1267         testb   %al,%al
 1268         jnz     2b
 1269 
 1270         SMAP_ENABLE \smap
 1271 
 1272         /* Success -- 0 byte reached */
 1273         decq    %rdx
 1274         xorl    %eax,%eax
 1275 
 1276         /* set *lencopied and return %eax */
 1277         movq    %rax,PCB_ONFAULT(%r9)
 1278 
 1279         testq   %rcx,%rcx
 1280         jz      3f
 1281         subq    %rdx,%r8
 1282         movq    %r8,(%rcx)
 1283 3:
 1284         POP_FRAME_POINTER
 1285         ret
 1286         ALIGN_TEXT
 1287 8:
 1288         movq    %rax,%rdx
 1289         movq    %rax,%r8
 1290         jmp 1b
 1291 
 1292 .endm
 1293 
 1294 ENTRY(copyinstr_nosmap)
 1295         COPYINSTR smap=0
 1296 END(copyinstr_nosmap)
 1297 
 1298 ENTRY(copyinstr_smap)
 1299         COPYINSTR smap=1
 1300 END(copyinstr_smap)
 1301 
 1302 cpystrflt:
 1303         /* Fault entry clears PSL.AC */
 1304         movl    $EFAULT,%eax
 1305 cpystrflt_x:
 1306         /* set *lencopied and return %eax */
 1307         movq    $0,PCB_ONFAULT(%r9)
 1308 
 1309         testq   %rcx,%rcx
 1310         jz      1f
 1311         subq    %rdx,%r8
 1312         movq    %r8,(%rcx)
 1313 1:
 1314         POP_FRAME_POINTER
 1315         ret
 1316 
 1317 copyinstr_toolong_smap:
 1318         clac
 1319 copyinstr_toolong:
 1320         /* rdx is zero - return ENAMETOOLONG or EFAULT */
 1321         movq    $VM_MAXUSER_ADDRESS,%rax
 1322         cmpq    %rax,%rdi
 1323         jae     cpystrflt
 1324         movl    $ENAMETOOLONG,%eax
 1325         jmp     cpystrflt_x
 1326 
 1327 /*
 1328  * copystr(from, to, maxlen, int *lencopied)
 1329  *         %rdi, %rsi, %rdx, %rcx
 1330  */
 1331 ENTRY(copystr)
 1332         PUSH_FRAME_POINTER
 1333         movq    %rdx,%r8                        /* %r8 = maxlen */
 1334 
 1335         incq    %rdx
 1336 1:
 1337         decq    %rdx
 1338         jz      4f
 1339         movb    (%rdi),%al
 1340         movb    %al,(%rsi)
 1341         incq    %rsi
 1342         incq    %rdi
 1343         testb   %al,%al
 1344         jnz     1b
 1345 
 1346         /* Success -- 0 byte reached */
 1347         decq    %rdx
 1348         xorl    %eax,%eax
 1349 2:
 1350         testq   %rcx,%rcx
 1351         jz      3f
 1352         /* set *lencopied and return %rax */
 1353         subq    %rdx,%r8
 1354         movq    %r8,(%rcx)
 1355 3:
 1356         POP_FRAME_POINTER
 1357         ret
 1358 4:
 1359         /* rdx is zero -- return ENAMETOOLONG */
 1360         movl    $ENAMETOOLONG,%eax
 1361         jmp     2b
 1362 END(copystr)
 1363 
 1364 /*
 1365  * Handling of special amd64 registers and descriptor tables etc
 1366  */
 1367 /* void lgdt(struct region_descriptor *rdp); */
 1368 ENTRY(lgdt)
 1369         /* reload the descriptor table */
 1370         lgdt    (%rdi)
 1371 
 1372         /* flush the prefetch q */
 1373         jmp     1f
 1374         nop
 1375 1:
 1376         movl    $KDSEL,%eax
 1377         movl    %eax,%ds
 1378         movl    %eax,%es
 1379         movl    %eax,%fs        /* Beware, use wrmsr to set 64 bit base */
 1380         movl    %eax,%gs
 1381         movl    %eax,%ss
 1382 
 1383         /* reload code selector by turning return into intersegmental return */
 1384         popq    %rax
 1385         pushq   $KCSEL
 1386         pushq   %rax
 1387         MEXITCOUNT
 1388         lretq
 1389 END(lgdt)
 1390 
 1391 /*****************************************************************************/
 1392 /* setjump, longjump                                                         */
 1393 /*****************************************************************************/
 1394 
 1395 ENTRY(setjmp)
 1396         movq    %rbx,0(%rdi)                    /* save rbx */
 1397         movq    %rsp,8(%rdi)                    /* save rsp */
 1398         movq    %rbp,16(%rdi)                   /* save rbp */
 1399         movq    %r12,24(%rdi)                   /* save r12 */
 1400         movq    %r13,32(%rdi)                   /* save r13 */
 1401         movq    %r14,40(%rdi)                   /* save r14 */
 1402         movq    %r15,48(%rdi)                   /* save r15 */
 1403         movq    0(%rsp),%rdx                    /* get rta */
 1404         movq    %rdx,56(%rdi)                   /* save rip */
 1405         xorl    %eax,%eax                       /* return(0); */
 1406         ret
 1407 END(setjmp)
 1408 
 1409 ENTRY(longjmp)
 1410         movq    0(%rdi),%rbx                    /* restore rbx */
 1411         movq    8(%rdi),%rsp                    /* restore rsp */
 1412         movq    16(%rdi),%rbp                   /* restore rbp */
 1413         movq    24(%rdi),%r12                   /* restore r12 */
 1414         movq    32(%rdi),%r13                   /* restore r13 */
 1415         movq    40(%rdi),%r14                   /* restore r14 */
 1416         movq    48(%rdi),%r15                   /* restore r15 */
 1417         movq    56(%rdi),%rdx                   /* get rta */
 1418         movq    %rdx,0(%rsp)                    /* put in return frame */
 1419         xorl    %eax,%eax                       /* return(1); */
 1420         incl    %eax
 1421         ret
 1422 END(longjmp)
 1423 
 1424 /*
 1425  * Support for reading MSRs in the safe manner.  (Instead of panic on #gp,
 1426  * return an error.)
 1427  */
 1428 ENTRY(rdmsr_safe)
 1429 /* int rdmsr_safe(u_int msr, uint64_t *data) */
 1430         PUSH_FRAME_POINTER
 1431         movq    PCPU(CURPCB),%r8
 1432         movq    $msr_onfault,PCB_ONFAULT(%r8)
 1433         movl    %edi,%ecx
 1434         rdmsr                   /* Read MSR pointed by %ecx. Returns
 1435                                    hi byte in edx, lo in %eax */
 1436         salq    $32,%rdx        /* sign-shift %rdx left */
 1437         movl    %eax,%eax       /* zero-extend %eax -> %rax */
 1438         orq     %rdx,%rax
 1439         movq    %rax,(%rsi)
 1440         xorq    %rax,%rax
 1441         movq    %rax,PCB_ONFAULT(%r8)
 1442         POP_FRAME_POINTER
 1443         ret
 1444 
 1445 /*
 1446  * Support for writing MSRs in the safe manner.  (Instead of panic on #gp,
 1447  * return an error.)
 1448  */
 1449 ENTRY(wrmsr_safe)
 1450 /* int wrmsr_safe(u_int msr, uint64_t data) */
 1451         PUSH_FRAME_POINTER
 1452         movq    PCPU(CURPCB),%r8
 1453         movq    $msr_onfault,PCB_ONFAULT(%r8)
 1454         movl    %edi,%ecx
 1455         movl    %esi,%eax
 1456         sarq    $32,%rsi
 1457         movl    %esi,%edx
 1458         wrmsr                   /* Write MSR pointed by %ecx. Accepts
 1459                                    hi byte in edx, lo in %eax. */
 1460         xorq    %rax,%rax
 1461         movq    %rax,PCB_ONFAULT(%r8)
 1462         POP_FRAME_POINTER
 1463         ret
 1464 
 1465 /*
 1466  * MSR operations fault handler
 1467  */
 1468         ALIGN_TEXT
 1469 msr_onfault:
 1470         movq    $0,PCB_ONFAULT(%r8)
 1471         movl    $EFAULT,%eax
 1472         POP_FRAME_POINTER
 1473         ret
 1474 
 1475 /*
 1476  * void pmap_pti_pcid_invalidate(uint64_t ucr3, uint64_t kcr3);
 1477  * Invalidates address space addressed by ucr3, then returns to kcr3.
 1478  * Done in assembler to ensure no other memory accesses happen while
 1479  * on ucr3.
 1480  */
 1481         ALIGN_TEXT
 1482 ENTRY(pmap_pti_pcid_invalidate)
 1483         pushfq
 1484         cli
 1485         movq    %rdi,%cr3       /* to user page table */
 1486         movq    %rsi,%cr3       /* back to kernel */
 1487         popfq
 1488         retq
 1489 
 1490 /*
 1491  * void pmap_pti_pcid_invlpg(uint64_t ucr3, uint64_t kcr3, vm_offset_t va);
 1492  * Invalidates virtual address va in address space ucr3, then returns to kcr3.
 1493  */
 1494         ALIGN_TEXT
 1495 ENTRY(pmap_pti_pcid_invlpg)
 1496         pushfq
 1497         cli
 1498         movq    %rdi,%cr3       /* to user page table */
 1499         invlpg  (%rdx)
 1500         movq    %rsi,%cr3       /* back to kernel */
 1501         popfq
 1502         retq
 1503 
 1504 /*
 1505  * void pmap_pti_pcid_invlrng(uint64_t ucr3, uint64_t kcr3, vm_offset_t sva,
 1506  *     vm_offset_t eva);
 1507  * Invalidates virtual addresses between sva and eva in address space ucr3,
 1508  * then returns to kcr3.
 1509  */
 1510         ALIGN_TEXT
 1511 ENTRY(pmap_pti_pcid_invlrng)
 1512         pushfq
 1513         cli
 1514         movq    %rdi,%cr3       /* to user page table */
 1515 1:      invlpg  (%rdx)
 1516         addq    $PAGE_SIZE,%rdx
 1517         cmpq    %rdx,%rcx
 1518         ja      1b
 1519         movq    %rsi,%cr3       /* back to kernel */
 1520         popfq
 1521         retq
 1522 
 1523         .altmacro
 1524         .macro  ibrs_seq_label l
 1525 handle_ibrs_\l:
 1526         .endm
 1527         .macro  ibrs_call_label l
 1528         call    handle_ibrs_\l
 1529         .endm
 1530         .macro  ibrs_seq count
 1531         ll=1
 1532         .rept   \count
 1533         ibrs_call_label %(ll)
 1534         nop
 1535         ibrs_seq_label %(ll)
 1536         addq    $8,%rsp
 1537         ll=ll+1
 1538         .endr
 1539         .endm
 1540 
 1541 /* all callers already saved %rax, %rdx, and %rcx */
 1542 ENTRY(handle_ibrs_entry)
 1543         cmpb    $0,hw_ibrs_active(%rip)
 1544         je      1f
 1545         movl    $MSR_IA32_SPEC_CTRL,%ecx
 1546         rdmsr
 1547         orl     $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
 1548         orl     $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32,%edx
 1549         wrmsr
 1550         movb    $1,PCPU(IBPB_SET)
 1551         testl   $CPUID_STDEXT_SMEP,cpu_stdext_feature(%rip)
 1552         jne     1f
 1553         ibrs_seq 32
 1554 1:      ret
 1555 END(handle_ibrs_entry)
 1556 
 1557 ENTRY(handle_ibrs_exit)
 1558         cmpb    $0,PCPU(IBPB_SET)
 1559         je      1f
 1560         movl    $MSR_IA32_SPEC_CTRL,%ecx
 1561         rdmsr
 1562         andl    $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
 1563         andl    $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
 1564         wrmsr
 1565         movb    $0,PCPU(IBPB_SET)
 1566 1:      ret
 1567 END(handle_ibrs_exit)
 1568 
 1569 /* registers-neutral version, but needs stack */
 1570 ENTRY(handle_ibrs_exit_rs)
 1571         cmpb    $0,PCPU(IBPB_SET)
 1572         je      1f
 1573         pushq   %rax
 1574         pushq   %rdx
 1575         pushq   %rcx
 1576         movl    $MSR_IA32_SPEC_CTRL,%ecx
 1577         rdmsr
 1578         andl    $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
 1579         andl    $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
 1580         wrmsr
 1581         popq    %rcx
 1582         popq    %rdx
 1583         popq    %rax
 1584         movb    $0,PCPU(IBPB_SET)
 1585 1:      ret
 1586 END(handle_ibrs_exit_rs)
 1587 
 1588         .noaltmacro
 1589 
 1590 /*
 1591  * Flush L1D cache.  Load enough of the data from the kernel text
 1592  * to flush existing L1D content.
 1593  *
 1594  * N.B. The function does not follow ABI calling conventions, it corrupts %rbx.
 1595  * The vmm.ko caller expects that only %rax, %rdx, %rbx, %rcx, %r9, and %rflags
 1596  * registers are clobbered.  The NMI handler caller only needs %r13 preserved.
 1597  */
 1598 ENTRY(flush_l1d_sw)
 1599 #define L1D_FLUSH_SIZE  (64 * 1024)
 1600         movq    $KERNBASE, %r9
 1601         movq    $-L1D_FLUSH_SIZE, %rcx
 1602         /*
 1603          * pass 1: Preload TLB.
 1604          * Kernel text is mapped using superpages.  TLB preload is
 1605          * done for the benefit of older CPUs which split 2M page
 1606          * into 4k TLB entries.
 1607          */
 1608 1:      movb    L1D_FLUSH_SIZE(%r9, %rcx), %al
 1609         addq    $PAGE_SIZE, %rcx
 1610         jne     1b
 1611         xorl    %eax, %eax
 1612         cpuid
 1613         movq    $-L1D_FLUSH_SIZE, %rcx
 1614         /* pass 2: Read each cache line. */
 1615 2:      movb    L1D_FLUSH_SIZE(%r9, %rcx), %al
 1616         addq    $64, %rcx
 1617         jne     2b
 1618         lfence
 1619         ret
 1620 #undef  L1D_FLUSH_SIZE
 1621 END(flush_l1d_sw)
 1622 
 1623 ENTRY(flush_l1d_sw_abi)
 1624         pushq   %rbx
 1625         call    flush_l1d_sw
 1626         popq    %rbx
 1627         ret
 1628 END(flush_l1d_sw_abi)

Cache object: fa78a34059161e4f7f360df3b956aa68


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.