The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/support.S

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2018-2019 The FreeBSD Foundation
    3  * Copyright (c) 2003 Peter Wemm.
    4  * Copyright (c) 1993 The Regents of the University of California.
    5  * All rights reserved.
    6  *
    7  * Portions of this software were developed by
    8  * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
    9  * the FreeBSD Foundation.
   10  *
   11  * Redistribution and use in source and binary forms, with or without
   12  * modification, are permitted provided that the following conditions
   13  * are met:
   14  * 1. Redistributions of source code must retain the above copyright
   15  *    notice, this list of conditions and the following disclaimer.
   16  * 2. Redistributions in binary form must reproduce the above copyright
   17  *    notice, this list of conditions and the following disclaimer in the
   18  *    documentation and/or other materials provided with the distribution.
   19  * 4. Neither the name of the University nor the names of its contributors
   20  *    may be used to endorse or promote products derived from this software
   21  *    without specific prior written permission.
   22  *
   23  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   24  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   26  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   27  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   28  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   29  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   31  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   32  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   33  * SUCH DAMAGE.
   34  *
   35  * $FreeBSD: stable/11/sys/amd64/amd64/support.S 361561 2020-05-27 18:55:24Z kib $
   36  */
   37 
   38 #include "opt_ddb.h"
   39 
   40 #include <machine/asmacros.h>
   41 #include <machine/specialreg.h>
   42 #include <machine/pmap.h>
   43 
   44 #include "assym.s"
   45 
   46         .text
   47 
   48 /*
   49  * bcopy family
   50  * void bzero(void *buf, u_int len)
   51  */
   52 
   53 /* done */
   54 ENTRY(bzero)
   55         PUSH_FRAME_POINTER
   56         movq    %rsi,%rcx
   57         xorl    %eax,%eax
   58         shrq    $3,%rcx
   59         rep
   60         stosq
   61         movq    %rsi,%rcx
   62         andq    $7,%rcx
   63         rep
   64         stosb
   65         POP_FRAME_POINTER
   66         ret
   67 END(bzero)
   68 
   69 /* Address: %rdi */
   70 ENTRY(pagezero)
   71         PUSH_FRAME_POINTER
   72         movq    $PAGE_SIZE/8,%rcx
   73         xorl    %eax,%eax
   74         rep
   75         stosq
   76         POP_FRAME_POINTER
   77         ret
   78 END(pagezero)
   79 
   80 ENTRY(bcmp)
   81         PUSH_FRAME_POINTER
   82         movq    %rdx,%rcx
   83         shrq    $3,%rcx
   84         repe
   85         cmpsq
   86         jne     1f
   87 
   88         movq    %rdx,%rcx
   89         andq    $7,%rcx
   90         repe
   91         cmpsb
   92 1:
   93         setne   %al
   94         movsbl  %al,%eax
   95         POP_FRAME_POINTER
   96         ret
   97 END(bcmp)
   98 
   99 /*
  100  * bcopy(src, dst, cnt)
  101  *       rdi, rsi, rdx
  102  *  ws@tools.de     (Wolfgang Solfrank, TooLs GmbH) +49-228-985800
  103  */
  104 ENTRY(bcopy)
  105         PUSH_FRAME_POINTER
  106         xchgq   %rsi,%rdi
  107         movq    %rdx,%rcx
  108 
  109         movq    %rdi,%rax
  110         subq    %rsi,%rax
  111         cmpq    %rcx,%rax                       /* overlapping && src < dst? */
  112         jb      1f
  113 
  114         shrq    $3,%rcx                         /* copy by 64-bit words */
  115         rep
  116         movsq
  117         movq    %rdx,%rcx
  118         andq    $7,%rcx                         /* any bytes left? */
  119         rep
  120         movsb
  121         POP_FRAME_POINTER
  122         ret
  123 
  124         /* ALIGN_TEXT */
  125 1:
  126         addq    %rcx,%rdi                       /* copy backwards */
  127         addq    %rcx,%rsi
  128         decq    %rdi
  129         decq    %rsi
  130         andq    $7,%rcx                         /* any fractional bytes? */
  131         std
  132         rep
  133         movsb
  134         movq    %rdx,%rcx                       /* copy remainder by 32-bit words */
  135         shrq    $3,%rcx
  136         subq    $7,%rsi
  137         subq    $7,%rdi
  138         rep
  139         movsq
  140         cld
  141         POP_FRAME_POINTER
  142         ret
  143 END(bcopy)
  144 
  145 /*
  146  * Note: memcpy does not support overlapping copies
  147  */
  148 ENTRY(memcpy)
  149         PUSH_FRAME_POINTER
  150         movq    %rdi,%rax
  151         movq    %rdx,%rcx
  152         shrq    $3,%rcx                         /* copy by 64-bit words */
  153         rep
  154         movsq
  155         movq    %rdx,%rcx
  156         andq    $7,%rcx                         /* any bytes left? */
  157         rep
  158         movsb
  159         POP_FRAME_POINTER
  160         ret
  161 END(memcpy)
  162 
  163 /*
  164  * pagecopy(%rdi=from, %rsi=to)
  165  */
  166 ENTRY(pagecopy)
  167         PUSH_FRAME_POINTER
  168         movq    $-PAGE_SIZE,%rax
  169         movq    %rax,%rdx
  170         subq    %rax,%rdi
  171         subq    %rax,%rsi
  172 1:
  173         prefetchnta (%rdi,%rax)
  174         addq    $64,%rax
  175         jne     1b
  176 2:
  177         movq    (%rdi,%rdx),%rax
  178         movnti  %rax,(%rsi,%rdx)
  179         movq    8(%rdi,%rdx),%rax
  180         movnti  %rax,8(%rsi,%rdx)
  181         movq    16(%rdi,%rdx),%rax
  182         movnti  %rax,16(%rsi,%rdx)
  183         movq    24(%rdi,%rdx),%rax
  184         movnti  %rax,24(%rsi,%rdx)
  185         addq    $32,%rdx
  186         jne     2b
  187         sfence
  188         POP_FRAME_POINTER
  189         ret
  190 END(pagecopy)
  191 
  192 /* fillw(pat, base, cnt) */
  193 /*       %rdi,%rsi, %rdx */
  194 ENTRY(fillw)
  195         PUSH_FRAME_POINTER
  196         movq    %rdi,%rax
  197         movq    %rsi,%rdi
  198         movq    %rdx,%rcx
  199         rep
  200         stosw
  201         POP_FRAME_POINTER
  202         ret
  203 END(fillw)
  204 
  205 /*****************************************************************************/
  206 /* copyout and fubyte family                                                 */
  207 /*****************************************************************************/
  208 /*
  209  * Access user memory from inside the kernel. These routines should be
  210  * the only places that do this.
  211  *
  212  * These routines set curpcb->pcb_onfault for the time they execute. When a
  213  * protection violation occurs inside the functions, the trap handler
  214  * returns to *curpcb->pcb_onfault instead of the function.
  215  */
  216 
  217 /*
  218  * copyout(from_kernel, to_user, len)
  219  *         %rdi,        %rsi,    %rdx
  220  */
  221 ENTRY(copyout)
  222         PUSH_FRAME_POINTER
  223         movq    PCPU(CURPCB),%rax
  224         movq    $copyout_fault,PCB_ONFAULT(%rax)
  225         testq   %rdx,%rdx                       /* anything to do? */
  226         jz      done_copyout
  227 
  228         /*
  229          * Check explicitly for non-user addresses.  This check is essential
  230          * because it prevents usermode from writing into the kernel.  We do
  231          * not verify anywhere else that the user did not specify a rogue
  232          * address.
  233          */
  234         /*
  235          * First, prevent address wrapping.
  236          */
  237         movq    %rsi,%rax
  238         addq    %rdx,%rax
  239         jc      copyout_fault
  240 /*
  241  * XXX STOP USING VM_MAXUSER_ADDRESS.
  242  * It is an end address, not a max, so every time it is used correctly it
  243  * looks like there is an off by one error, and of course it caused an off
  244  * by one error in several places.
  245  */
  246         movq    $VM_MAXUSER_ADDRESS,%rcx
  247         cmpq    %rcx,%rax
  248         ja      copyout_fault
  249 
  250         xchgq   %rdi,%rsi
  251         /* bcopy(%rsi, %rdi, %rdx) */
  252         movq    %rdx,%rcx
  253 
  254         shrq    $3,%rcx
  255         rep
  256         movsq
  257         movb    %dl,%cl
  258         andb    $7,%cl
  259         rep
  260         movsb
  261 
  262 done_copyout:
  263         xorl    %eax,%eax
  264         movq    PCPU(CURPCB),%rdx
  265         movq    %rax,PCB_ONFAULT(%rdx)
  266         POP_FRAME_POINTER
  267         ret
  268 
  269         ALIGN_TEXT
  270 copyout_fault:
  271         movq    PCPU(CURPCB),%rdx
  272         movq    $0,PCB_ONFAULT(%rdx)
  273         movq    $EFAULT,%rax
  274         POP_FRAME_POINTER
  275         ret
  276 END(copyout)
  277 
  278 /*
  279  * copyin(from_user, to_kernel, len)
  280  *        %rdi,      %rsi,      %rdx
  281  */
  282 ENTRY(copyin)
  283         PUSH_FRAME_POINTER
  284         movq    PCPU(CURPCB),%rax
  285         movq    $copyin_fault,PCB_ONFAULT(%rax)
  286         testq   %rdx,%rdx                       /* anything to do? */
  287         jz      done_copyin
  288 
  289         /*
  290          * make sure address is valid
  291          */
  292         movq    %rdi,%rax
  293         addq    %rdx,%rax
  294         jc      copyin_fault
  295         movq    $VM_MAXUSER_ADDRESS,%rcx
  296         cmpq    %rcx,%rax
  297         ja      copyin_fault
  298 
  299         xchgq   %rdi,%rsi
  300         movq    %rdx,%rcx
  301         movb    %cl,%al
  302         shrq    $3,%rcx                         /* copy longword-wise */
  303         rep
  304         movsq
  305         movb    %al,%cl
  306         andb    $7,%cl                          /* copy remaining bytes */
  307         rep
  308         movsb
  309 
  310 done_copyin:
  311         xorl    %eax,%eax
  312         movq    PCPU(CURPCB),%rdx
  313         movq    %rax,PCB_ONFAULT(%rdx)
  314         POP_FRAME_POINTER
  315         ret
  316 
  317         ALIGN_TEXT
  318 copyin_fault:
  319         movq    PCPU(CURPCB),%rdx
  320         movq    $0,PCB_ONFAULT(%rdx)
  321         movq    $EFAULT,%rax
  322         POP_FRAME_POINTER
  323         ret
  324 END(copyin)
  325 
  326 /*
  327  * casueword32.  Compare and set user integer.  Returns -1 on fault,
  328  *        0 if access was successful.  Old value is written to *oldp.
  329  *        dst = %rdi, old = %esi, oldp = %rdx, new = %ecx
  330  */
  331 ENTRY(casueword32)
  332         PUSH_FRAME_POINTER
  333         movq    PCPU(CURPCB),%r8
  334         movq    $fusufault,PCB_ONFAULT(%r8)
  335 
  336         movq    $VM_MAXUSER_ADDRESS-4,%rax
  337         cmpq    %rax,%rdi                       /* verify address is valid */
  338         ja      fusufault
  339 
  340         movl    %esi,%eax                       /* old */
  341 #ifdef SMP
  342         lock
  343 #endif
  344         cmpxchgl %ecx,(%rdi)                    /* new = %ecx */
  345 
  346         /*
  347          * The old value is in %eax.  If the store succeeded it will be the
  348          * value we expected (old) from before the store, otherwise it will
  349          * be the current value.  Save %eax into %esi to prepare the return
  350          * value.
  351          */
  352         movl    %eax,%esi
  353         xorl    %eax,%eax
  354         movq    %rax,PCB_ONFAULT(%r8)
  355 
  356         /*
  357          * Access the oldp after the pcb_onfault is cleared, to correctly
  358          * catch corrupted pointer.
  359          */
  360         movl    %esi,(%rdx)                     /* oldp = %rdx */
  361         POP_FRAME_POINTER
  362         ret
  363 END(casueword32)
  364 
  365 /*
  366  * casueword.  Compare and set user long.  Returns -1 on fault,
  367  *        0 if access was successful.  Old value is written to *oldp.
  368  *        dst = %rdi, old = %rsi, oldp = %rdx, new = %rcx
  369  */
  370 ENTRY(casueword)
  371         PUSH_FRAME_POINTER
  372         movq    PCPU(CURPCB),%r8
  373         movq    $fusufault,PCB_ONFAULT(%r8)
  374 
  375         movq    $VM_MAXUSER_ADDRESS-4,%rax
  376         cmpq    %rax,%rdi                       /* verify address is valid */
  377         ja      fusufault
  378 
  379         movq    %rsi,%rax                       /* old */
  380 #ifdef SMP
  381         lock
  382 #endif
  383         cmpxchgq %rcx,(%rdi)                    /* new = %rcx */
  384 
  385         /*
  386          * The old value is in %rax.  If the store succeeded it will be the
  387          * value we expected (old) from before the store, otherwise it will
  388          * be the current value.
  389          */
  390         movq    %rax,%rsi
  391         xorl    %eax,%eax
  392         movq    %rax,PCB_ONFAULT(%r8)
  393         movq    %rsi,(%rdx)
  394         POP_FRAME_POINTER
  395         ret
  396 END(casueword)
  397 
  398 /*
  399  * Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit
  400  * byte from user memory.
  401  * addr = %rdi, valp = %rsi
  402  */
  403 
  404 ALTENTRY(fueword64)
  405 ENTRY(fueword)
  406         PUSH_FRAME_POINTER
  407         movq    PCPU(CURPCB),%rcx
  408         movq    $fusufault,PCB_ONFAULT(%rcx)
  409 
  410         movq    $VM_MAXUSER_ADDRESS-8,%rax
  411         cmpq    %rax,%rdi                       /* verify address is valid */
  412         ja      fusufault
  413 
  414         xorl    %eax,%eax
  415         movq    (%rdi),%r11
  416         movq    %rax,PCB_ONFAULT(%rcx)
  417         movq    %r11,(%rsi)
  418         POP_FRAME_POINTER
  419         ret
  420 END(fueword64)
  421 END(fueword)
  422 
  423 ENTRY(fueword32)
  424         PUSH_FRAME_POINTER
  425         movq    PCPU(CURPCB),%rcx
  426         movq    $fusufault,PCB_ONFAULT(%rcx)
  427 
  428         movq    $VM_MAXUSER_ADDRESS-4,%rax
  429         cmpq    %rax,%rdi                       /* verify address is valid */
  430         ja      fusufault
  431 
  432         xorl    %eax,%eax
  433         movl    (%rdi),%r11d
  434         movq    %rax,PCB_ONFAULT(%rcx)
  435         movl    %r11d,(%rsi)
  436         POP_FRAME_POINTER
  437         ret
  438 END(fueword32)
  439 
  440 /*
  441  * fuswintr() and suswintr() are specialized variants of fuword16() and
  442  * suword16(), respectively.  They are called from the profiling code,
  443  * potentially at interrupt time.  If they fail, that's okay; good things
  444  * will happen later.  They always fail for now, until the trap code is
  445  * able to deal with this.
  446  */
  447 ALTENTRY(suswintr)
  448 ENTRY(fuswintr)
  449         movq    $-1,%rax
  450         ret
  451 END(suswintr)
  452 END(fuswintr)
  453 
  454 ENTRY(fuword16)
  455         PUSH_FRAME_POINTER
  456         movq    PCPU(CURPCB),%rcx
  457         movq    $fusufault,PCB_ONFAULT(%rcx)
  458 
  459         movq    $VM_MAXUSER_ADDRESS-2,%rax
  460         cmpq    %rax,%rdi
  461         ja      fusufault
  462 
  463         movzwl  (%rdi),%eax
  464         movq    $0,PCB_ONFAULT(%rcx)
  465         POP_FRAME_POINTER
  466         ret
  467 END(fuword16)
  468 
  469 ENTRY(fubyte)
  470         PUSH_FRAME_POINTER
  471         movq    PCPU(CURPCB),%rcx
  472         movq    $fusufault,PCB_ONFAULT(%rcx)
  473 
  474         movq    $VM_MAXUSER_ADDRESS-1,%rax
  475         cmpq    %rax,%rdi
  476         ja      fusufault
  477 
  478         movzbl  (%rdi),%eax
  479         movq    $0,PCB_ONFAULT(%rcx)
  480         POP_FRAME_POINTER
  481         ret
  482 END(fubyte)
  483 
  484         ALIGN_TEXT
  485 fusufault:
  486         movq    PCPU(CURPCB),%rcx
  487         xorl    %eax,%eax
  488         movq    %rax,PCB_ONFAULT(%rcx)
  489         decq    %rax
  490         POP_FRAME_POINTER
  491         ret
  492 
  493 /*
  494  * Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to
  495  * user memory.
  496  * addr = %rdi, value = %rsi
  497  */
  498 ALTENTRY(suword64)
  499 ENTRY(suword)
  500         PUSH_FRAME_POINTER
  501         movq    PCPU(CURPCB),%rcx
  502         movq    $fusufault,PCB_ONFAULT(%rcx)
  503 
  504         movq    $VM_MAXUSER_ADDRESS-8,%rax
  505         cmpq    %rax,%rdi                       /* verify address validity */
  506         ja      fusufault
  507 
  508         movq    %rsi,(%rdi)
  509         xorl    %eax,%eax
  510         movq    PCPU(CURPCB),%rcx
  511         movq    %rax,PCB_ONFAULT(%rcx)
  512         POP_FRAME_POINTER
  513         ret
  514 END(suword64)
  515 END(suword)
  516 
  517 ENTRY(suword32)
  518         PUSH_FRAME_POINTER
  519         movq    PCPU(CURPCB),%rcx
  520         movq    $fusufault,PCB_ONFAULT(%rcx)
  521 
  522         movq    $VM_MAXUSER_ADDRESS-4,%rax
  523         cmpq    %rax,%rdi                       /* verify address validity */
  524         ja      fusufault
  525 
  526         movl    %esi,(%rdi)
  527         xorl    %eax,%eax
  528         movq    PCPU(CURPCB),%rcx
  529         movq    %rax,PCB_ONFAULT(%rcx)
  530         POP_FRAME_POINTER
  531         ret
  532 END(suword32)
  533 
  534 ENTRY(suword16)
  535         PUSH_FRAME_POINTER
  536         movq    PCPU(CURPCB),%rcx
  537         movq    $fusufault,PCB_ONFAULT(%rcx)
  538 
  539         movq    $VM_MAXUSER_ADDRESS-2,%rax
  540         cmpq    %rax,%rdi                       /* verify address validity */
  541         ja      fusufault
  542 
  543         movw    %si,(%rdi)
  544         xorl    %eax,%eax
  545         movq    PCPU(CURPCB),%rcx               /* restore trashed register */
  546         movq    %rax,PCB_ONFAULT(%rcx)
  547         POP_FRAME_POINTER
  548         ret
  549 END(suword16)
  550 
  551 ENTRY(subyte)
  552         PUSH_FRAME_POINTER
  553         movq    PCPU(CURPCB),%rcx
  554         movq    $fusufault,PCB_ONFAULT(%rcx)
  555 
  556         movq    $VM_MAXUSER_ADDRESS-1,%rax
  557         cmpq    %rax,%rdi                       /* verify address validity */
  558         ja      fusufault
  559 
  560         movl    %esi,%eax
  561         movb    %al,(%rdi)
  562         xorl    %eax,%eax
  563         movq    PCPU(CURPCB),%rcx               /* restore trashed register */
  564         movq    %rax,PCB_ONFAULT(%rcx)
  565         POP_FRAME_POINTER
  566         ret
  567 END(subyte)
  568 
  569 /*
  570  * copyinstr(from, to, maxlen, int *lencopied)
  571  *           %rdi, %rsi, %rdx, %rcx
  572  *
  573  *      copy a string from 'from' to 'to', stop when a 0 character is reached.
  574  *      return ENAMETOOLONG if string is longer than maxlen, and
  575  *      EFAULT on protection violations. If lencopied is non-zero,
  576  *      return the actual length in *lencopied.
  577  */
  578 ENTRY(copyinstr)
  579         PUSH_FRAME_POINTER
  580         movq    %rdx,%r8                        /* %r8 = maxlen */
  581         movq    %rcx,%r9                        /* %r9 = *len */
  582         xchgq   %rdi,%rsi                       /* %rdi = from, %rsi = to */
  583         movq    PCPU(CURPCB),%rcx
  584         movq    $cpystrflt,PCB_ONFAULT(%rcx)
  585 
  586         movq    $VM_MAXUSER_ADDRESS,%rax
  587 
  588         /* make sure 'from' is within bounds */
  589         subq    %rsi,%rax
  590         jbe     cpystrflt
  591 
  592         /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */
  593         cmpq    %rdx,%rax
  594         jae     1f
  595         movq    %rax,%rdx
  596         movq    %rax,%r8
  597 1:
  598         incq    %rdx
  599 
  600 2:
  601         decq    %rdx
  602         jz      3f
  603 
  604         lodsb
  605         stosb
  606         orb     %al,%al
  607         jnz     2b
  608 
  609         /* Success -- 0 byte reached */
  610         decq    %rdx
  611         xorl    %eax,%eax
  612         jmp     cpystrflt_x
  613 3:
  614         /* rdx is zero - return ENAMETOOLONG or EFAULT */
  615         movq    $VM_MAXUSER_ADDRESS,%rax
  616         cmpq    %rax,%rsi
  617         jae     cpystrflt
  618 4:
  619         movq    $ENAMETOOLONG,%rax
  620         jmp     cpystrflt_x
  621 
  622 cpystrflt:
  623         movq    $EFAULT,%rax
  624 
  625 cpystrflt_x:
  626         /* set *lencopied and return %eax */
  627         movq    PCPU(CURPCB),%rcx
  628         movq    $0,PCB_ONFAULT(%rcx)
  629 
  630         testq   %r9,%r9
  631         jz      1f
  632         subq    %rdx,%r8
  633         movq    %r8,(%r9)
  634 1:
  635         POP_FRAME_POINTER
  636         ret
  637 END(copyinstr)
  638 
  639 /*
  640  * copystr(from, to, maxlen, int *lencopied)
  641  *         %rdi, %rsi, %rdx, %rcx
  642  */
  643 ENTRY(copystr)
  644         PUSH_FRAME_POINTER
  645         movq    %rdx,%r8                        /* %r8 = maxlen */
  646 
  647         xchgq   %rdi,%rsi
  648         incq    %rdx
  649 1:
  650         decq    %rdx
  651         jz      4f
  652         lodsb
  653         stosb
  654         orb     %al,%al
  655         jnz     1b
  656 
  657         /* Success -- 0 byte reached */
  658         decq    %rdx
  659         xorl    %eax,%eax
  660         jmp     6f
  661 4:
  662         /* rdx is zero -- return ENAMETOOLONG */
  663         movq    $ENAMETOOLONG,%rax
  664 
  665 6:
  666 
  667         testq   %rcx,%rcx
  668         jz      7f
  669         /* set *lencopied and return %rax */
  670         subq    %rdx,%r8
  671         movq    %r8,(%rcx)
  672 7:
  673         POP_FRAME_POINTER
  674         ret
  675 END(copystr)
  676 
  677 /*
  678  * Handling of special amd64 registers and descriptor tables etc
  679  */
  680 /* void lgdt(struct region_descriptor *rdp); */
  681 ENTRY(lgdt)
  682         /* reload the descriptor table */
  683         lgdt    (%rdi)
  684 
  685         /* flush the prefetch q */
  686         jmp     1f
  687         nop
  688 1:
  689         movl    $KDSEL,%eax
  690         movl    %eax,%ds
  691         movl    %eax,%es
  692         movl    %eax,%fs        /* Beware, use wrmsr to set 64 bit base */
  693         movl    %eax,%gs
  694         movl    %eax,%ss
  695 
  696         /* reload code selector by turning return into intersegmental return */
  697         popq    %rax
  698         pushq   $KCSEL
  699         pushq   %rax
  700         MEXITCOUNT
  701         lretq
  702 END(lgdt)
  703 
  704 /*****************************************************************************/
  705 /* setjump, longjump                                                         */
  706 /*****************************************************************************/
  707 
  708 ENTRY(setjmp)
  709         movq    %rbx,0(%rdi)                    /* save rbx */
  710         movq    %rsp,8(%rdi)                    /* save rsp */
  711         movq    %rbp,16(%rdi)                   /* save rbp */
  712         movq    %r12,24(%rdi)                   /* save r12 */
  713         movq    %r13,32(%rdi)                   /* save r13 */
  714         movq    %r14,40(%rdi)                   /* save r14 */
  715         movq    %r15,48(%rdi)                   /* save r15 */
  716         movq    0(%rsp),%rdx                    /* get rta */
  717         movq    %rdx,56(%rdi)                   /* save rip */
  718         xorl    %eax,%eax                       /* return(0); */
  719         ret
  720 END(setjmp)
  721 
  722 ENTRY(longjmp)
  723         movq    0(%rdi),%rbx                    /* restore rbx */
  724         movq    8(%rdi),%rsp                    /* restore rsp */
  725         movq    16(%rdi),%rbp                   /* restore rbp */
  726         movq    24(%rdi),%r12                   /* restore r12 */
  727         movq    32(%rdi),%r13                   /* restore r13 */
  728         movq    40(%rdi),%r14                   /* restore r14 */
  729         movq    48(%rdi),%r15                   /* restore r15 */
  730         movq    56(%rdi),%rdx                   /* get rta */
  731         movq    %rdx,0(%rsp)                    /* put in return frame */
  732         xorl    %eax,%eax                       /* return(1); */
  733         incl    %eax
  734         ret
  735 END(longjmp)
  736 
  737 /*
  738  * Support for reading MSRs in the safe manner.
  739  */
  740 ENTRY(rdmsr_safe)
  741 /* int rdmsr_safe(u_int msr, uint64_t *data) */
  742         PUSH_FRAME_POINTER
  743         movq    PCPU(CURPCB),%r8
  744         movq    $msr_onfault,PCB_ONFAULT(%r8)
  745         movl    %edi,%ecx
  746         rdmsr                   /* Read MSR pointed by %ecx. Returns
  747                                    hi byte in edx, lo in %eax */
  748         salq    $32,%rdx        /* sign-shift %rdx left */
  749         movl    %eax,%eax       /* zero-extend %eax -> %rax */
  750         orq     %rdx,%rax
  751         movq    %rax,(%rsi)
  752         xorq    %rax,%rax
  753         movq    %rax,PCB_ONFAULT(%r8)
  754         POP_FRAME_POINTER
  755         ret
  756 
  757 /*
  758  * Support for writing MSRs in the safe manner.
  759  */
  760 ENTRY(wrmsr_safe)
  761 /* int wrmsr_safe(u_int msr, uint64_t data) */
  762         PUSH_FRAME_POINTER
  763         movq    PCPU(CURPCB),%r8
  764         movq    $msr_onfault,PCB_ONFAULT(%r8)
  765         movl    %edi,%ecx
  766         movl    %esi,%eax
  767         sarq    $32,%rsi
  768         movl    %esi,%edx
  769         wrmsr                   /* Write MSR pointed by %ecx. Accepts
  770                                    hi byte in edx, lo in %eax. */
  771         xorq    %rax,%rax
  772         movq    %rax,PCB_ONFAULT(%r8)
  773         POP_FRAME_POINTER
  774         ret
  775 
  776 /*
  777  * MSR operations fault handler
  778  */
  779         ALIGN_TEXT
  780 msr_onfault:
  781         movq    $0,PCB_ONFAULT(%r8)
  782         movl    $EFAULT,%eax
  783         POP_FRAME_POINTER
  784         ret
  785 
  786 /*
  787  * void pmap_pti_pcid_invalidate(uint64_t ucr3, uint64_t kcr3);
  788  * Invalidates address space addressed by ucr3, then returns to kcr3.
  789  * Done in assembler to ensure no other memory accesses happen while
  790  * on ucr3.
  791  */
  792         ALIGN_TEXT
  793 ENTRY(pmap_pti_pcid_invalidate)
  794         pushfq
  795         cli
  796         movq    %rdi,%cr3       /* to user page table */
  797         movq    %rsi,%cr3       /* back to kernel */
  798         popfq
  799         retq
  800 
  801 /*
  802  * void pmap_pti_pcid_invlpg(uint64_t ucr3, uint64_t kcr3, vm_offset_t va);
  803  * Invalidates virtual address va in address space ucr3, then returns to kcr3.
  804  */
  805         ALIGN_TEXT
  806 ENTRY(pmap_pti_pcid_invlpg)
  807         pushfq
  808         cli
  809         movq    %rdi,%cr3       /* to user page table */
  810         invlpg  (%rdx)
  811         movq    %rsi,%cr3       /* back to kernel */
  812         popfq
  813         retq
  814 
  815 /*
  816  * void pmap_pti_pcid_invlrng(uint64_t ucr3, uint64_t kcr3, vm_offset_t sva,
  817  *     vm_offset_t eva);
  818  * Invalidates virtual addresses between sva and eva in address space ucr3,
  819  * then returns to kcr3.
  820  */
  821         ALIGN_TEXT
  822 ENTRY(pmap_pti_pcid_invlrng)
  823         pushfq
  824         cli
  825         movq    %rdi,%cr3       /* to user page table */
  826 1:      invlpg  (%rdx)
  827         addq    $PAGE_SIZE,%rdx
  828         cmpq    %rdx,%rcx
  829         ja      1b
  830         movq    %rsi,%cr3       /* back to kernel */
  831         popfq
  832         retq
  833 
  834         .altmacro
  835         .macro  rsb_seq_label l
  836 rsb_seq_\l:
  837         .endm
  838         .macro  rsb_call_label l
  839         call    rsb_seq_\l
  840         .endm
  841         .macro  rsb_seq count
  842         ll=1
  843         .rept   \count
  844         rsb_call_label  %(ll)
  845         nop
  846         rsb_seq_label %(ll)
  847         addq    $8,%rsp
  848         ll=ll+1
  849         .endr
  850         .endm
  851 
  852 ENTRY(rsb_flush)
  853         rsb_seq 32
  854         ret
  855 
  856 /* all callers already saved %rax, %rdx, and %rcx */
  857 ENTRY(handle_ibrs_entry)
  858         cmpb    $0,hw_ibrs_ibpb_active(%rip)
  859         je      1f
  860         movl    $MSR_IA32_SPEC_CTRL,%ecx
  861         rdmsr
  862         orl     $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
  863         orl     $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32,%edx
  864         wrmsr
  865         movb    $1,PCPU(IBPB_SET)
  866         testl   $CPUID_STDEXT_SMEP,cpu_stdext_feature(%rip)
  867         je      rsb_flush
  868 1:      ret
  869 END(handle_ibrs_entry)
  870 
  871 ENTRY(handle_ibrs_exit)
  872         cmpb    $0,PCPU(IBPB_SET)
  873         je      1f
  874         movl    $MSR_IA32_SPEC_CTRL,%ecx
  875         rdmsr
  876         andl    $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
  877         andl    $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
  878         wrmsr
  879         movb    $0,PCPU(IBPB_SET)
  880 1:      ret
  881 END(handle_ibrs_exit)
  882 
  883 /* registers-neutral version, but needs stack */
  884 ENTRY(handle_ibrs_exit_rs)
  885         cmpb    $0,PCPU(IBPB_SET)
  886         je      1f
  887         pushq   %rax
  888         pushq   %rdx
  889         pushq   %rcx
  890         movl    $MSR_IA32_SPEC_CTRL,%ecx
  891         rdmsr
  892         andl    $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
  893         andl    $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
  894         wrmsr
  895         popq    %rcx
  896         popq    %rdx
  897         popq    %rax
  898         movb    $0,PCPU(IBPB_SET)
  899 1:      ret
  900 END(handle_ibrs_exit_rs)
  901 
  902         .noaltmacro
  903 
  904 /*
  905  * Flush L1D cache.  Load enough of the data from the kernel text
  906  * to flush existing L1D content.
  907  *
  908  * N.B. The function does not follow ABI calling conventions, it corrupts %rbx.
  909  * The vmm.ko caller expects that only %rax, %rdx, %rbx, %rcx, %r9, and %rflags
  910  * registers are clobbered.  The NMI handler caller only needs %r13 preserved.
  911  */
  912 ENTRY(flush_l1d_sw)
  913 #define L1D_FLUSH_SIZE  (64 * 1024)
  914         movq    $KERNBASE, %r9
  915         movq    $-L1D_FLUSH_SIZE, %rcx
  916         /*
  917          * pass 1: Preload TLB.
  918          * Kernel text is mapped using superpages.  TLB preload is
  919          * done for the benefit of older CPUs which split 2M page
  920          * into 4k TLB entries.
  921          */
  922 1:      movb    L1D_FLUSH_SIZE(%r9, %rcx), %al
  923         addq    $PAGE_SIZE, %rcx
  924         jne     1b
  925         xorl    %eax, %eax
  926         cpuid
  927         movq    $-L1D_FLUSH_SIZE, %rcx
  928         /* pass 2: Read each cache line. */
  929 2:      movb    L1D_FLUSH_SIZE(%r9, %rcx), %al
  930         addq    $64, %rcx
  931         jne     2b
  932         lfence
  933         ret
  934 #undef  L1D_FLUSH_SIZE
  935 END(flush_l1d_sw)
  936 
  937 ENTRY(mds_handler_void)
  938         retq
  939 END(mds_handler_void)
  940 
  941 ENTRY(mds_handler_verw)
  942         subq    $8, %rsp
  943         movw    %ds, (%rsp)
  944         verw    (%rsp)
  945         addq    $8, %rsp
  946         retq
  947 END(mds_handler_verw)
  948 
  949 ENTRY(mds_handler_ivb)
  950         pushq   %rax
  951         pushq   %rdx
  952         pushq   %rcx
  953 
  954         movq    %cr0, %rax
  955         testb   $CR0_TS, %al
  956         je      1f
  957         clts
  958 1:      movq    PCPU(MDS_BUF), %rdx
  959         movdqa  %xmm0, PCPU(MDS_TMP)
  960         pxor    %xmm0, %xmm0
  961 
  962         lfence
  963         orpd    (%rdx), %xmm0
  964         orpd    (%rdx), %xmm0
  965         mfence
  966         movl    $40, %ecx
  967         addq    $16, %rdx
  968 2:      movntdq %xmm0, (%rdx)
  969         addq    $16, %rdx
  970         decl    %ecx
  971         jnz     2b
  972         mfence
  973 
  974         movdqa  PCPU(MDS_TMP),%xmm0
  975         testb   $CR0_TS, %al
  976         je      3f
  977         movq    %rax, %cr0
  978 3:      popq    %rcx
  979         popq    %rdx
  980         popq    %rax
  981         retq
  982 END(mds_handler_ivb)
  983 
  984 ENTRY(mds_handler_bdw)
  985         pushq   %rax
  986         pushq   %rbx
  987         pushq   %rcx
  988         pushq   %rdi
  989         pushq   %rsi
  990 
  991         movq    %cr0, %rax
  992         testb   $CR0_TS, %al
  993         je      1f
  994         clts
  995 1:      movq    PCPU(MDS_BUF), %rbx
  996         movdqa  %xmm0, PCPU(MDS_TMP)
  997         pxor    %xmm0, %xmm0
  998 
  999         movq    %rbx, %rdi
 1000         movq    %rbx, %rsi
 1001         movl    $40, %ecx
 1002 2:      movntdq %xmm0, (%rbx)
 1003         addq    $16, %rbx
 1004         decl    %ecx
 1005         jnz     2b
 1006         mfence
 1007         movl    $1536, %ecx
 1008         rep; movsb
 1009         lfence
 1010 
 1011         movdqa  PCPU(MDS_TMP),%xmm0
 1012         testb   $CR0_TS, %al
 1013         je      3f
 1014         movq    %rax, %cr0
 1015 3:      popq    %rsi
 1016         popq    %rdi
 1017         popq    %rcx
 1018         popq    %rbx
 1019         popq    %rax
 1020         retq
 1021 END(mds_handler_bdw)
 1022 
 1023 ENTRY(mds_handler_skl_sse)
 1024         pushq   %rax
 1025         pushq   %rdx
 1026         pushq   %rcx
 1027         pushq   %rdi
 1028 
 1029         movq    %cr0, %rax
 1030         testb   $CR0_TS, %al
 1031         je      1f
 1032         clts
 1033 1:      movq    PCPU(MDS_BUF), %rdi
 1034         movq    PCPU(MDS_BUF64), %rdx
 1035         movdqa  %xmm0, PCPU(MDS_TMP)
 1036         pxor    %xmm0, %xmm0
 1037 
 1038         lfence
 1039         orpd    (%rdx), %xmm0
 1040         orpd    (%rdx), %xmm0
 1041         xorl    %eax, %eax
 1042 2:      clflushopt      5376(%rdi, %rax, 8)
 1043         addl    $8, %eax
 1044         cmpl    $8 * 12, %eax
 1045         jb      2b
 1046         sfence
 1047         movl    $6144, %ecx
 1048         xorl    %eax, %eax
 1049         rep; stosb
 1050         mfence
 1051 
 1052         movdqa  PCPU(MDS_TMP), %xmm0
 1053         testb   $CR0_TS, %al
 1054         je      3f
 1055         movq    %rax, %cr0
 1056 3:      popq    %rdi
 1057         popq    %rcx
 1058         popq    %rdx
 1059         popq    %rax
 1060         retq
 1061 END(mds_handler_skl_sse)
 1062 
 1063 ENTRY(mds_handler_skl_avx)
 1064         pushq   %rax
 1065         pushq   %rdx
 1066         pushq   %rcx
 1067         pushq   %rdi
 1068 
 1069         movq    %cr0, %rax
 1070         testb   $CR0_TS, %al
 1071         je      1f
 1072         clts
 1073 1:      movq    PCPU(MDS_BUF), %rdi
 1074         movq    PCPU(MDS_BUF64), %rdx
 1075         vmovdqa %ymm0, PCPU(MDS_TMP)
 1076         vpxor   %ymm0, %ymm0, %ymm0
 1077 
 1078         lfence
 1079         vorpd   (%rdx), %ymm0, %ymm0
 1080         vorpd   (%rdx), %ymm0, %ymm0
 1081         xorl    %eax, %eax
 1082 2:      clflushopt      5376(%rdi, %rax, 8)
 1083         addl    $8, %eax
 1084         cmpl    $8 * 12, %eax
 1085         jb      2b
 1086         sfence
 1087         movl    $6144, %ecx
 1088         xorl    %eax, %eax
 1089         rep; stosb
 1090         mfence
 1091 
 1092         vmovdqa PCPU(MDS_TMP), %ymm0
 1093         testb   $CR0_TS, %al
 1094         je      3f
 1095         movq    %rax, %cr0
 1096 3:      popq    %rdi
 1097         popq    %rcx
 1098         popq    %rdx
 1099         popq    %rax
 1100         retq
 1101 END(mds_handler_skl_avx)
 1102 
 1103 ENTRY(mds_handler_skl_avx512)
 1104         pushq   %rax
 1105         pushq   %rdx
 1106         pushq   %rcx
 1107         pushq   %rdi
 1108 
 1109         movq    %cr0, %rax
 1110         testb   $CR0_TS, %al
 1111         je      1f
 1112         clts
 1113 1:      movq    PCPU(MDS_BUF), %rdi
 1114         movq    PCPU(MDS_BUF64), %rdx
 1115         vmovdqa64       %zmm0, PCPU(MDS_TMP)
 1116         vpxor   %zmm0, %zmm0, %zmm0
 1117 
 1118         lfence
 1119         vorpd   (%rdx), %zmm0, %zmm0
 1120         vorpd   (%rdx), %zmm0, %zmm0
 1121         xorl    %eax, %eax
 1122 2:      clflushopt      5376(%rdi, %rax, 8)
 1123         addl    $8, %eax
 1124         cmpl    $8 * 12, %eax
 1125         jb      2b
 1126         sfence
 1127         movl    $6144, %ecx
 1128         xorl    %eax, %eax
 1129         rep; stosb
 1130         mfence
 1131 
 1132         vmovdqa64       PCPU(MDS_TMP), %zmm0
 1133         testb   $CR0_TS, %al
 1134         je      3f
 1135         movq    %rax, %cr0
 1136 3:      popq    %rdi
 1137         popq    %rcx
 1138         popq    %rdx
 1139         popq    %rax
 1140         retq
 1141 END(mds_handler_skl_avx512)
 1142 
 1143 ENTRY(mds_handler_silvermont)
 1144         pushq   %rax
 1145         pushq   %rdx
 1146         pushq   %rcx
 1147 
 1148         movq    %cr0, %rax
 1149         testb   $CR0_TS, %al
 1150         je      1f
 1151         clts
 1152 1:      movq    PCPU(MDS_BUF), %rdx
 1153         movdqa  %xmm0, PCPU(MDS_TMP)
 1154         pxor    %xmm0, %xmm0
 1155 
 1156         movl    $16, %ecx
 1157 2:      movntdq %xmm0, (%rdx)
 1158         addq    $16, %rdx
 1159         decl    %ecx
 1160         jnz     2b
 1161         mfence
 1162 
 1163         movdqa  PCPU(MDS_TMP),%xmm0
 1164         testb   $CR0_TS, %al
 1165         je      3f
 1166         movq    %rax, %cr0
 1167 3:      popq    %rcx
 1168         popq    %rdx
 1169         popq    %rax
 1170         retq
 1171 END(mds_handler_silvermont)

Cache object: 74e449c7daf792a69abac03c8a299f3a


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.