The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/support.S

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2018-2019 The FreeBSD Foundation
    3  * Copyright (c) 2003 Peter Wemm.
    4  * Copyright (c) 1993 The Regents of the University of California.
    5  * All rights reserved.
    6  *
    7  * Portions of this software were developed by
    8  * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
    9  * the FreeBSD Foundation.
   10  *
   11  * Redistribution and use in source and binary forms, with or without
   12  * modification, are permitted provided that the following conditions
   13  * are met:
   14  * 1. Redistributions of source code must retain the above copyright
   15  *    notice, this list of conditions and the following disclaimer.
   16  * 2. Redistributions in binary form must reproduce the above copyright
   17  *    notice, this list of conditions and the following disclaimer in the
   18  *    documentation and/or other materials provided with the distribution.
   19  * 3. Neither the name of the University nor the names of its contributors
   20  *    may be used to endorse or promote products derived from this software
   21  *    without specific prior written permission.
   22  *
   23  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   24  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   26  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   27  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   28  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   29  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   31  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   32  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   33  * SUCH DAMAGE.
   34  *
   35  * $FreeBSD: stable/12/sys/amd64/amd64/support.S 363505 2020-07-25 00:24:11Z mjg $
   36  */
   37 
   38 #include "opt_ddb.h"
   39 
   40 #include <machine/asmacros.h>
   41 #include <machine/specialreg.h>
   42 #include <machine/pmap.h>
   43 
   44 #include "assym.inc"
   45 
   46         .text
   47 
   48 /* Address: %rdi */
   49 ENTRY(pagezero_std)
   50         PUSH_FRAME_POINTER
   51         movq    $PAGE_SIZE/8,%rcx
   52         xorl    %eax,%eax
   53         rep
   54         stosq
   55         POP_FRAME_POINTER
   56         ret
   57 END(pagezero_std)
   58 
   59 ENTRY(pagezero_erms)
   60         PUSH_FRAME_POINTER
   61         movq    $PAGE_SIZE,%rcx
   62         xorl    %eax,%eax
   63         rep
   64         stosb
   65         POP_FRAME_POINTER
   66         ret
   67 END(pagezero_erms)
   68 
   69 /*
   70  * pagecopy(%rdi=from, %rsi=to)
   71  */
   72 ENTRY(pagecopy)
   73         PUSH_FRAME_POINTER
   74         movq    $PAGE_SIZE/8,%rcx
   75         movq    %rdi,%r9
   76         movq    %rsi,%rdi
   77         movq    %r9,%rsi
   78         rep
   79         movsq
   80         POP_FRAME_POINTER
   81         ret
   82 END(pagecopy)
   83 
   84 /* Address: %rdi */
   85 ENTRY(sse2_pagezero)
   86         PUSH_FRAME_POINTER
   87         movq    $-PAGE_SIZE,%rdx
   88         subq    %rdx,%rdi
   89         xorl    %eax,%eax
   90         jmp     1f
   91         /*
   92          * The loop takes 29 bytes.  Ensure that it doesn't cross a 32-byte
   93          * cache line.
   94          */
   95         .p2align 5,0x90
   96 1:
   97         movnti  %rax,(%rdi,%rdx)
   98         movnti  %rax,8(%rdi,%rdx)
   99         movnti  %rax,16(%rdi,%rdx)
  100         movnti  %rax,24(%rdi,%rdx)
  101         addq    $32,%rdx
  102         jne     1b
  103         sfence
  104         POP_FRAME_POINTER
  105         ret
  106 END(sse2_pagezero)
  107 
  108 /*
  109  * memcmpy(b1, b2, len)
  110  *         rdi,rsi,rdx
  111  */
  112 ENTRY(memcmp)
  113         PUSH_FRAME_POINTER
  114 
  115         xorl    %eax,%eax
  116 10:
  117         cmpq    $16,%rdx
  118         ja      101632f
  119 
  120 100816:
  121         cmpb    $8,%dl
  122         jl      100408f
  123         movq    (%rdi),%r8
  124         movq    (%rsi),%r9
  125         cmpq    %r8,%r9
  126         jne     80f
  127         movq    -8(%rdi,%rdx),%r8
  128         movq    -8(%rsi,%rdx),%r9
  129         cmpq    %r8,%r9
  130         jne     10081608f
  131         POP_FRAME_POINTER
  132         ret
  133 100408:
  134         cmpb    $4,%dl
  135         jl      100204f
  136         movl    (%rdi),%r8d
  137         movl    (%rsi),%r9d
  138         cmpl    %r8d,%r9d
  139         jne     80f
  140         movl    -4(%rdi,%rdx),%r8d
  141         movl    -4(%rsi,%rdx),%r9d
  142         cmpl    %r8d,%r9d
  143         jne     10040804f
  144         POP_FRAME_POINTER
  145         ret
  146 100204:
  147         cmpb    $2,%dl
  148         jl      100001f
  149         movzwl  (%rdi),%r8d
  150         movzwl  (%rsi),%r9d
  151         cmpl    %r8d,%r9d
  152         jne     1f
  153         movzwl  -2(%rdi,%rdx),%r8d
  154         movzwl  -2(%rsi,%rdx),%r9d
  155         cmpl    %r8d,%r9d
  156         jne     1f
  157         POP_FRAME_POINTER
  158         ret
  159 100001:
  160         cmpb    $1,%dl
  161         jl      100000f
  162         movzbl  (%rdi),%eax
  163         movzbl  (%rsi),%r8d
  164         subl    %r8d,%eax
  165 100000:
  166         POP_FRAME_POINTER
  167         ret
  168 ALIGN_TEXT
  169 101632:
  170         cmpq    $32,%rdx
  171         ja      103200f
  172         movq    (%rdi),%r8
  173         movq    (%rsi),%r9
  174         cmpq    %r8,%r9
  175         jne     80f
  176         movq    8(%rdi),%r8
  177         movq    8(%rsi),%r9
  178         cmpq    %r8,%r9
  179         jne     10163208f
  180         movq    -16(%rdi,%rdx),%r8
  181         movq    -16(%rsi,%rdx),%r9
  182         cmpq    %r8,%r9
  183         jne     10163216f
  184         movq    -8(%rdi,%rdx),%r8
  185         movq    -8(%rsi,%rdx),%r9
  186         cmpq    %r8,%r9
  187         jne     10163224f
  188         POP_FRAME_POINTER
  189         ret
  190 ALIGN_TEXT
  191 103200:
  192         movq    (%rdi),%r8
  193         movq    8(%rdi),%r9
  194         subq    (%rsi),%r8
  195         subq    8(%rsi),%r9
  196         orq     %r8,%r9
  197         jnz     10320000f
  198 
  199         movq    16(%rdi),%r8
  200         movq    24(%rdi),%r9
  201         subq    16(%rsi),%r8
  202         subq    24(%rsi),%r9
  203         orq     %r8,%r9
  204         jnz     10320016f
  205 
  206         leaq    32(%rdi),%rdi
  207         leaq    32(%rsi),%rsi
  208         subq    $32,%rdx
  209         cmpq    $32,%rdx
  210         jae     103200b
  211         cmpb    $0,%dl
  212         jne     10b
  213         POP_FRAME_POINTER
  214         ret
  215 
  216 /*
  217  * Mismatch was found.
  218  *
  219  * Before we compute it we narrow down the range (16 -> 8 -> 4 bytes).
  220  */
  221 ALIGN_TEXT
  222 10320016:
  223         leaq    16(%rdi),%rdi
  224         leaq    16(%rsi),%rsi
  225 10320000:
  226         movq    (%rdi),%r8
  227         movq    (%rsi),%r9
  228         cmpq    %r8,%r9
  229         jne     80f
  230         leaq    8(%rdi),%rdi
  231         leaq    8(%rsi),%rsi
  232         jmp     80f
  233 ALIGN_TEXT
  234 10081608:
  235 10163224:
  236         leaq    -8(%rdi,%rdx),%rdi
  237         leaq    -8(%rsi,%rdx),%rsi
  238         jmp     80f
  239 ALIGN_TEXT
  240 10163216:
  241         leaq    -16(%rdi,%rdx),%rdi
  242         leaq    -16(%rsi,%rdx),%rsi
  243         jmp     80f
  244 ALIGN_TEXT
  245 10163208:
  246         leaq    8(%rdi),%rdi
  247         leaq    8(%rsi),%rsi
  248         jmp     80f
  249 ALIGN_TEXT
  250 10040804:
  251         leaq    -4(%rdi,%rdx),%rdi
  252         leaq    -4(%rsi,%rdx),%rsi
  253         jmp     1f
  254 
  255 ALIGN_TEXT
  256 80:
  257         movl    (%rdi),%r8d
  258         movl    (%rsi),%r9d
  259         cmpl    %r8d,%r9d
  260         jne     1f
  261         leaq    4(%rdi),%rdi
  262         leaq    4(%rsi),%rsi
  263 
  264 /*
  265  * We have up to 4 bytes to inspect.
  266  */
  267 1:
  268         movzbl  (%rdi),%eax
  269         movzbl  (%rsi),%r8d
  270         cmpb    %r8b,%al
  271         jne     2f
  272 
  273         movzbl  1(%rdi),%eax
  274         movzbl  1(%rsi),%r8d
  275         cmpb    %r8b,%al
  276         jne     2f
  277 
  278         movzbl  2(%rdi),%eax
  279         movzbl  2(%rsi),%r8d
  280         cmpb    %r8b,%al
  281         jne     2f
  282 
  283         movzbl  3(%rdi),%eax
  284         movzbl  3(%rsi),%r8d
  285 2:
  286         subl    %r8d,%eax
  287         POP_FRAME_POINTER
  288         ret
  289 END(memcmp)
  290 
  291 /*
  292  * memmove(dst, src, cnt)
  293  *         rdi, rsi, rdx
  294  */
  295 
  296 /*
  297  * Register state at entry is supposed to be as follows:
  298  * rdi - destination
  299  * rsi - source
  300  * rdx - count
  301  *
  302  * The macro possibly clobbers the above and: rcx, r8, r9, 10
  303  * It does not clobber rax nor r11.
  304  */
  305 .macro MEMMOVE erms overlap begin end
  306         \begin
  307 
  308         /*
  309          * For sizes 0..32 all data is read before it is written, so there
  310          * is no correctness issue with direction of copying.
  311          */
  312         cmpq    $32,%rcx
  313         jbe     101632f
  314 
  315 .if \overlap == 1
  316         movq    %rdi,%r8
  317         subq    %rsi,%r8
  318         cmpq    %rcx,%r8        /* overlapping && src < dst? */
  319         jb      2f
  320 .endif
  321 
  322         cmpq    $256,%rcx
  323         ja      1256f
  324 
  325 103200:
  326         movq    (%rsi),%rdx
  327         movq    %rdx,(%rdi)
  328         movq    8(%rsi),%rdx
  329         movq    %rdx,8(%rdi)
  330         movq    16(%rsi),%rdx
  331         movq    %rdx,16(%rdi)
  332         movq    24(%rsi),%rdx
  333         movq    %rdx,24(%rdi)
  334         leaq    32(%rsi),%rsi
  335         leaq    32(%rdi),%rdi
  336         subq    $32,%rcx
  337         cmpq    $32,%rcx
  338         jae     103200b
  339         cmpb    $0,%cl
  340         jne     101632f
  341         \end
  342         ret
  343         ALIGN_TEXT
  344 101632:
  345         cmpb    $16,%cl
  346         jl      100816f
  347         movq    (%rsi),%rdx
  348         movq    8(%rsi),%r8
  349         movq    -16(%rsi,%rcx),%r9
  350         movq    -8(%rsi,%rcx),%r10
  351         movq    %rdx,(%rdi)
  352         movq    %r8,8(%rdi)
  353         movq    %r9,-16(%rdi,%rcx)
  354         movq    %r10,-8(%rdi,%rcx)
  355         \end
  356         ret
  357         ALIGN_TEXT
  358 100816:
  359         cmpb    $8,%cl
  360         jl      100408f
  361         movq    (%rsi),%rdx
  362         movq    -8(%rsi,%rcx),%r8
  363         movq    %rdx,(%rdi)
  364         movq    %r8,-8(%rdi,%rcx,)
  365         \end
  366         ret
  367         ALIGN_TEXT
  368 100408:
  369         cmpb    $4,%cl
  370         jl      100204f
  371         movl    (%rsi),%edx
  372         movl    -4(%rsi,%rcx),%r8d
  373         movl    %edx,(%rdi)
  374         movl    %r8d,-4(%rdi,%rcx)
  375         \end
  376         ret
  377         ALIGN_TEXT
  378 100204:
  379         cmpb    $2,%cl
  380         jl      100001f
  381         movzwl  (%rsi),%edx
  382         movzwl  -2(%rsi,%rcx),%r8d
  383         movw    %dx,(%rdi)
  384         movw    %r8w,-2(%rdi,%rcx)
  385         \end
  386         ret
  387         ALIGN_TEXT
  388 100001:
  389         cmpb    $1,%cl
  390         jl      100000f
  391         movb    (%rsi),%dl
  392         movb    %dl,(%rdi)
  393 100000:
  394         \end
  395         ret
  396 
  397         ALIGN_TEXT
  398 1256:
  399         testb   $15,%dil
  400         jnz     100f
  401 .if \erms == 1
  402         rep
  403         movsb
  404 .else
  405         shrq    $3,%rcx                         /* copy by 64-bit words */
  406         rep
  407         movsq
  408         movq    %rdx,%rcx
  409         andl    $7,%ecx                         /* any bytes left? */
  410         jne     100408b
  411 .endif
  412         \end
  413         ret
  414 100:
  415         movq    (%rsi),%r8
  416         movq    8(%rsi),%r9
  417         movq    %rdi,%r10
  418         movq    %rdi,%rcx
  419         andq    $15,%rcx
  420         leaq    -16(%rdx,%rcx),%rdx
  421         neg     %rcx
  422         leaq    16(%rdi,%rcx),%rdi
  423         leaq    16(%rsi,%rcx),%rsi
  424         movq    %rdx,%rcx
  425 .if \erms == 1
  426         rep
  427         movsb
  428         movq    %r8,(%r10)
  429         movq    %r9,8(%r10)
  430 .else
  431         shrq    $3,%rcx                         /* copy by 64-bit words */
  432         rep
  433         movsq
  434         movq    %r8,(%r10)
  435         movq    %r9,8(%r10)
  436         movq    %rdx,%rcx
  437         andl    $7,%ecx                         /* any bytes left? */
  438         jne     100408b
  439 .endif
  440         \end
  441         ret
  442 
  443 .if \overlap == 1
  444         /*
  445          * Copy backwards.
  446          */
  447         ALIGN_TEXT
  448 2:
  449         cmpq    $256,%rcx
  450         ja      2256f
  451 
  452         leaq    -8(%rdi,%rcx),%rdi
  453         leaq    -8(%rsi,%rcx),%rsi
  454 
  455         cmpq    $32,%rcx
  456         jb      2016f
  457 
  458 2032:
  459         movq    (%rsi),%rdx
  460         movq    %rdx,(%rdi)
  461         movq    -8(%rsi),%rdx
  462         movq    %rdx,-8(%rdi)
  463         movq    -16(%rsi),%rdx
  464         movq    %rdx,-16(%rdi)
  465         movq    -24(%rsi),%rdx
  466         movq    %rdx,-24(%rdi)
  467         leaq    -32(%rsi),%rsi
  468         leaq    -32(%rdi),%rdi
  469         subq    $32,%rcx
  470         cmpq    $32,%rcx
  471         jae     2032b
  472         cmpb    $0,%cl
  473         jne     2016f
  474         \end
  475         ret
  476         ALIGN_TEXT
  477 2016:
  478         cmpb    $16,%cl
  479         jl      2008f
  480         movq    (%rsi),%rdx
  481         movq    %rdx,(%rdi)
  482         movq    -8(%rsi),%rdx
  483         movq    %rdx,-8(%rdi)
  484         subb    $16,%cl
  485         jz      2000f
  486         leaq    -16(%rsi),%rsi
  487         leaq    -16(%rdi),%rdi
  488 2008:
  489         cmpb    $8,%cl
  490         jl      2004f
  491         movq    (%rsi),%rdx
  492         movq    %rdx,(%rdi)
  493         subb    $8,%cl
  494         jz      2000f
  495         leaq    -8(%rsi),%rsi
  496         leaq    -8(%rdi),%rdi
  497 2004:
  498         cmpb    $4,%cl
  499         jl      2002f
  500         movl    4(%rsi),%edx
  501         movl    %edx,4(%rdi)
  502         subb    $4,%cl
  503         jz      2000f
  504         leaq    -4(%rsi),%rsi
  505         leaq    -4(%rdi),%rdi
  506 2002:
  507         cmpb    $2,%cl
  508         jl      2001f
  509         movw    6(%rsi),%dx
  510         movw    %dx,6(%rdi)
  511         subb    $2,%cl
  512         jz      2000f
  513         leaq    -2(%rsi),%rsi
  514         leaq    -2(%rdi),%rdi
  515 2001:
  516         cmpb    $1,%cl
  517         jl      2000f
  518         movb    7(%rsi),%dl
  519         movb    %dl,7(%rdi)
  520 2000:
  521         \end
  522         ret
  523         ALIGN_TEXT
  524 2256:
  525         std
  526 .if \erms == 1
  527         leaq    -1(%rdi,%rcx),%rdi
  528         leaq    -1(%rsi,%rcx),%rsi
  529         rep
  530         movsb
  531         cld
  532 .else
  533         leaq    -8(%rdi,%rcx),%rdi
  534         leaq    -8(%rsi,%rcx),%rsi
  535         shrq    $3,%rcx
  536         rep
  537         movsq
  538         cld
  539         movq    %rdx,%rcx
  540         andb    $7,%cl
  541         jne     2004b
  542 .endif
  543         \end
  544         ret
  545 .endif
  546 .endm
  547 
  548 .macro MEMMOVE_BEGIN
  549         PUSH_FRAME_POINTER
  550         movq    %rdi,%rax
  551         movq    %rdx,%rcx
  552 .endm
  553 
  554 .macro MEMMOVE_END
  555         POP_FRAME_POINTER
  556 .endm
  557 
  558 ENTRY(memmove_std)
  559         MEMMOVE erms=0 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END
  560 END(memmove_std)
  561 
  562 ENTRY(memmove_erms)
  563         MEMMOVE erms=1 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END
  564 END(memmove_erms)
  565 
  566 /*
  567  * memcpy(dst, src, len)
  568  *        rdi, rsi, rdx
  569  *
  570  * Note: memcpy does not support overlapping copies
  571  */
  572 ENTRY(memcpy_std)
  573         MEMMOVE erms=0 overlap=0 begin=MEMMOVE_BEGIN end=MEMMOVE_END
  574 END(memcpy_std)
  575 
  576 ENTRY(memcpy_erms)
  577         MEMMOVE erms=1 overlap=0 begin=MEMMOVE_BEGIN end=MEMMOVE_END
  578 END(memcpy_erms)
  579 
  580 /*
  581  * memset(dst, c,   len)
  582  *        rdi, rsi, rdx
  583  */
  584 .macro MEMSET erms
  585         PUSH_FRAME_POINTER
  586         movq    %rdi,%rax
  587         movq    %rdx,%rcx
  588         movzbq  %sil,%r8
  589         movabs  $0x0101010101010101,%r10
  590         imulq   %r8,%r10
  591 
  592         cmpq    $32,%rcx
  593         jbe     101632f
  594 
  595         cmpq    $256,%rcx
  596         ja      1256f
  597 
  598 103200:
  599         movq    %r10,(%rdi)
  600         movq    %r10,8(%rdi)
  601         movq    %r10,16(%rdi)
  602         movq    %r10,24(%rdi)
  603         leaq    32(%rdi),%rdi
  604         subq    $32,%rcx
  605         cmpq    $32,%rcx
  606         ja      103200b
  607         cmpb    $16,%cl
  608         ja      201632f
  609         movq    %r10,-16(%rdi,%rcx)
  610         movq    %r10,-8(%rdi,%rcx)
  611         POP_FRAME_POINTER
  612         ret
  613         ALIGN_TEXT
  614 101632:
  615         cmpb    $16,%cl
  616         jl      100816f
  617 201632:
  618         movq    %r10,(%rdi)
  619         movq    %r10,8(%rdi)
  620         movq    %r10,-16(%rdi,%rcx)
  621         movq    %r10,-8(%rdi,%rcx)
  622         POP_FRAME_POINTER
  623         ret
  624         ALIGN_TEXT
  625 100816:
  626         cmpb    $8,%cl
  627         jl      100408f
  628         movq    %r10,(%rdi)
  629         movq    %r10,-8(%rdi,%rcx)
  630         POP_FRAME_POINTER
  631         ret
  632         ALIGN_TEXT
  633 100408:
  634         cmpb    $4,%cl
  635         jl      100204f
  636         movl    %r10d,(%rdi)
  637         movl    %r10d,-4(%rdi,%rcx)
  638         POP_FRAME_POINTER
  639         ret
  640         ALIGN_TEXT
  641 100204:
  642         cmpb    $2,%cl
  643         jl      100001f
  644         movw    %r10w,(%rdi)
  645         movw    %r10w,-2(%rdi,%rcx)
  646         POP_FRAME_POINTER
  647         ret
  648         ALIGN_TEXT
  649 100001:
  650         cmpb    $0,%cl
  651         je      100000f
  652         movb    %r10b,(%rdi)
  653 100000:
  654         POP_FRAME_POINTER
  655         ret
  656         ALIGN_TEXT
  657 1256:
  658         movq    %rdi,%r9
  659         movq    %r10,%rax
  660         testl   $15,%edi
  661         jnz     3f
  662 1:
  663 .if \erms == 1
  664         rep
  665         stosb
  666         movq    %r9,%rax
  667 .else
  668         movq    %rcx,%rdx
  669         shrq    $3,%rcx
  670         rep
  671         stosq
  672         movq    %r9,%rax
  673         andl    $7,%edx
  674         jnz     2f
  675         POP_FRAME_POINTER
  676         ret
  677 2:
  678         movq    %r10,-8(%rdi,%rdx)
  679 .endif
  680         POP_FRAME_POINTER
  681         ret
  682         ALIGN_TEXT
  683 3:
  684         movq    %r10,(%rdi)
  685         movq    %r10,8(%rdi)
  686         movq    %rdi,%r8
  687         andq    $15,%r8
  688         leaq    -16(%rcx,%r8),%rcx
  689         neg     %r8
  690         leaq    16(%rdi,%r8),%rdi
  691         jmp     1b
  692 .endm
  693 
  694 ENTRY(memset_std)
  695         MEMSET erms=0
  696 END(memset_std)
  697 
  698 ENTRY(memset_erms)
  699         MEMSET erms=1
  700 END(memset_erms)
  701 
  702 /* fillw(pat, base, cnt) */
  703 /*       %rdi,%rsi, %rdx */
  704 ENTRY(fillw)
  705         PUSH_FRAME_POINTER
  706         movq    %rdi,%rax
  707         movq    %rsi,%rdi
  708         movq    %rdx,%rcx
  709         rep
  710         stosw
  711         POP_FRAME_POINTER
  712         ret
  713 END(fillw)
  714 
  715 /*****************************************************************************/
  716 /* copyout and fubyte family                                                 */
  717 /*****************************************************************************/
  718 /*
  719  * Access user memory from inside the kernel. These routines should be
  720  * the only places that do this.
  721  *
  722  * These routines set curpcb->pcb_onfault for the time they execute. When a
  723  * protection violation occurs inside the functions, the trap handler
  724  * returns to *curpcb->pcb_onfault instead of the function.
  725  */
  726 
  727 .macro SMAP_DISABLE smap
  728 .if     \smap
  729         stac
  730 .endif
  731 .endm
  732 
  733 
  734 .macro SMAP_ENABLE smap
  735 .if     \smap
  736         clac
  737 .endif
  738 .endm
  739 
  740 .macro COPYINOUT_BEGIN
  741 .endm
  742 
  743 .macro COPYINOUT_END
  744         movq    %rax,PCB_ONFAULT(%r11)
  745         POP_FRAME_POINTER
  746 .endm
  747 
  748 .macro COPYINOUT_SMAP_END
  749         SMAP_ENABLE smap=1
  750         COPYINOUT_END
  751 .endm
  752 
  753 /*
  754  * copyout(from_kernel, to_user, len)
  755  *         %rdi,        %rsi,    %rdx
  756  */
  757 .macro  COPYOUT smap erms
  758         PUSH_FRAME_POINTER
  759         movq    PCPU(CURPCB),%r11
  760         movq    $copy_fault,PCB_ONFAULT(%r11)
  761 
  762         /*
  763          * Check explicitly for non-user addresses.
  764          * First, prevent address wrapping.
  765          */
  766         movq    %rsi,%rax
  767         addq    %rdx,%rax
  768         jc      copy_fault
  769 /*
  770  * XXX STOP USING VM_MAXUSER_ADDRESS.
  771  * It is an end address, not a max, so every time it is used correctly it
  772  * looks like there is an off by one error, and of course it caused an off
  773  * by one error in several places.
  774  */
  775         movq    $VM_MAXUSER_ADDRESS,%rcx
  776         cmpq    %rcx,%rax
  777         ja      copy_fault
  778 
  779         /*
  780          * Set return value to zero. Remaining failure mode goes through
  781          * copy_fault.
  782          */
  783         xorl    %eax,%eax
  784 
  785         /*
  786          * Set up arguments for MEMMOVE.
  787          */
  788         movq    %rdi,%r8
  789         movq    %rsi,%rdi
  790         movq    %r8,%rsi
  791         movq    %rdx,%rcx
  792 
  793 
  794         SMAP_DISABLE \smap
  795 .if     \smap == 1
  796         MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_SMAP_END
  797 .else
  798         MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_END
  799 .endif
  800         /* NOTREACHED */
  801 .endm
  802 
  803 ENTRY(copyout_nosmap_std)
  804         COPYOUT smap=0 erms=0
  805 END(copyout_nosmap_std)
  806 
  807 ENTRY(copyout_smap_std)
  808         COPYOUT smap=1 erms=0
  809 END(copyout_smap_std)
  810 
  811 ENTRY(copyout_nosmap_erms)
  812         COPYOUT smap=0 erms=1
  813 END(copyout_nosmap_erms)
  814 
  815 ENTRY(copyout_smap_erms)
  816         COPYOUT smap=1 erms=1
  817 END(copyout_smap_erms)
  818 
  819 /*
  820  * copyin(from_user, to_kernel, len)
  821  *        %rdi,      %rsi,      %rdx
  822  */
  823 .macro  COPYIN smap erms
  824         PUSH_FRAME_POINTER
  825         movq    PCPU(CURPCB),%r11
  826         movq    $copy_fault,PCB_ONFAULT(%r11)
  827 
  828         /*
  829          * make sure address is valid
  830          */
  831         movq    %rdi,%rax
  832         addq    %rdx,%rax
  833         jc      copy_fault
  834         movq    $VM_MAXUSER_ADDRESS,%rcx
  835         cmpq    %rcx,%rax
  836         ja      copy_fault
  837 
  838         xorl    %eax,%eax
  839 
  840         movq    %rdi,%r8
  841         movq    %rsi,%rdi
  842         movq    %r8,%rsi
  843         movq    %rdx,%rcx
  844 
  845         SMAP_DISABLE \smap
  846 .if     \smap == 1
  847         MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_SMAP_END
  848 .else
  849         MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_END
  850 .endif
  851         /* NOTREACHED */
  852 .endm
  853 
  854 ENTRY(copyin_nosmap_std)
  855         COPYIN smap=0 erms=0
  856 END(copyin_nosmap_std)
  857 
  858 ENTRY(copyin_smap_std)
  859         COPYIN smap=1 erms=0
  860 END(copyin_smap_std)
  861 
  862 ENTRY(copyin_nosmap_erms)
  863         COPYIN smap=0 erms=1
  864 END(copyin_nosmap_erms)
  865 
  866 ENTRY(copyin_smap_erms)
  867         COPYIN smap=1 erms=1
  868 END(copyin_smap_erms)
  869 
  870         ALIGN_TEXT
  871         /* Trap entry clears PSL.AC */
  872 copy_fault:
  873         movq    $0,PCB_ONFAULT(%r11)
  874         movl    $EFAULT,%eax
  875         POP_FRAME_POINTER
  876         ret
  877 
  878 /*
  879  * casueword32.  Compare and set user integer.  Returns -1 on fault,
  880  *        0 if access was successful.  Old value is written to *oldp.
  881  *        dst = %rdi, old = %esi, oldp = %rdx, new = %ecx
  882  */
  883 ENTRY(casueword32_nosmap)
  884         PUSH_FRAME_POINTER
  885         movq    PCPU(CURPCB),%r8
  886         movq    $fusufault,PCB_ONFAULT(%r8)
  887 
  888         movq    $VM_MAXUSER_ADDRESS-4,%rax
  889         cmpq    %rax,%rdi                       /* verify address is valid */
  890         ja      fusufault
  891 
  892         movl    %esi,%eax                       /* old */
  893 #ifdef SMP
  894         lock
  895 #endif
  896         cmpxchgl %ecx,(%rdi)                    /* new = %ecx */
  897         setne   %cl
  898 
  899         /*
  900          * The old value is in %eax.  If the store succeeded it will be the
  901          * value we expected (old) from before the store, otherwise it will
  902          * be the current value.  Save %eax into %esi to prepare the return
  903          * value.
  904          */
  905         movl    %eax,%esi
  906         xorl    %eax,%eax
  907         movq    %rax,PCB_ONFAULT(%r8)
  908 
  909         /*
  910          * Access the oldp after the pcb_onfault is cleared, to correctly
  911          * catch corrupted pointer.
  912          */
  913         movl    %esi,(%rdx)                     /* oldp = %rdx */
  914         POP_FRAME_POINTER
  915         movzbl  %cl, %eax
  916         ret
  917 END(casueword32_nosmap)
  918 
  919 ENTRY(casueword32_smap)
  920         PUSH_FRAME_POINTER
  921         movq    PCPU(CURPCB),%r8
  922         movq    $fusufault,PCB_ONFAULT(%r8)
  923 
  924         movq    $VM_MAXUSER_ADDRESS-4,%rax
  925         cmpq    %rax,%rdi                       /* verify address is valid */
  926         ja      fusufault
  927 
  928         movl    %esi,%eax                       /* old */
  929         stac
  930 #ifdef SMP
  931         lock
  932 #endif
  933         cmpxchgl %ecx,(%rdi)                    /* new = %ecx */
  934         clac
  935         setne   %cl
  936 
  937         /*
  938          * The old value is in %eax.  If the store succeeded it will be the
  939          * value we expected (old) from before the store, otherwise it will
  940          * be the current value.  Save %eax into %esi to prepare the return
  941          * value.
  942          */
  943         movl    %eax,%esi
  944         xorl    %eax,%eax
  945         movq    %rax,PCB_ONFAULT(%r8)
  946 
  947         /*
  948          * Access the oldp after the pcb_onfault is cleared, to correctly
  949          * catch corrupted pointer.
  950          */
  951         movl    %esi,(%rdx)                     /* oldp = %rdx */
  952         POP_FRAME_POINTER
  953         movzbl  %cl, %eax
  954         ret
  955 END(casueword32_smap)
  956 
  957 /*
  958  * casueword.  Compare and set user long.  Returns -1 on fault,
  959  *        0 if access was successful.  Old value is written to *oldp.
  960  *        dst = %rdi, old = %rsi, oldp = %rdx, new = %rcx
  961  */
  962 ENTRY(casueword_nosmap)
  963         PUSH_FRAME_POINTER
  964         movq    PCPU(CURPCB),%r8
  965         movq    $fusufault,PCB_ONFAULT(%r8)
  966 
  967         movq    $VM_MAXUSER_ADDRESS-4,%rax
  968         cmpq    %rax,%rdi                       /* verify address is valid */
  969         ja      fusufault
  970 
  971         movq    %rsi,%rax                       /* old */
  972 #ifdef SMP
  973         lock
  974 #endif
  975         cmpxchgq %rcx,(%rdi)                    /* new = %rcx */
  976         setne   %cl
  977 
  978         /*
  979          * The old value is in %rax.  If the store succeeded it will be the
  980          * value we expected (old) from before the store, otherwise it will
  981          * be the current value.
  982          */
  983         movq    %rax,%rsi
  984         xorl    %eax,%eax
  985         movq    %rax,PCB_ONFAULT(%r8)
  986         movq    %rsi,(%rdx)
  987         POP_FRAME_POINTER
  988         movzbl  %cl, %eax
  989         ret
  990 END(casueword_nosmap)
  991 
  992 ENTRY(casueword_smap)
  993         PUSH_FRAME_POINTER
  994         movq    PCPU(CURPCB),%r8
  995         movq    $fusufault,PCB_ONFAULT(%r8)
  996 
  997         movq    $VM_MAXUSER_ADDRESS-4,%rax
  998         cmpq    %rax,%rdi                       /* verify address is valid */
  999         ja      fusufault
 1000 
 1001         movq    %rsi,%rax                       /* old */
 1002         stac
 1003 #ifdef SMP
 1004         lock
 1005 #endif
 1006         cmpxchgq %rcx,(%rdi)                    /* new = %rcx */
 1007         clac
 1008         setne   %cl
 1009 
 1010         /*
 1011          * The old value is in %rax.  If the store succeeded it will be the
 1012          * value we expected (old) from before the store, otherwise it will
 1013          * be the current value.
 1014          */
 1015         movq    %rax,%rsi
 1016         xorl    %eax,%eax
 1017         movq    %rax,PCB_ONFAULT(%r8)
 1018         movq    %rsi,(%rdx)
 1019         POP_FRAME_POINTER
 1020         movzbl  %cl, %eax
 1021         ret
 1022 END(casueword_smap)
 1023 
 1024 /*
 1025  * Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit
 1026  * byte from user memory.
 1027  * addr = %rdi, valp = %rsi
 1028  */
 1029 
 1030 ENTRY(fueword_nosmap)
 1031         PUSH_FRAME_POINTER
 1032         movq    PCPU(CURPCB),%rcx
 1033         movq    $fusufault,PCB_ONFAULT(%rcx)
 1034 
 1035         movq    $VM_MAXUSER_ADDRESS-8,%rax
 1036         cmpq    %rax,%rdi                       /* verify address is valid */
 1037         ja      fusufault
 1038 
 1039         xorl    %eax,%eax
 1040         movq    (%rdi),%r11
 1041         movq    %rax,PCB_ONFAULT(%rcx)
 1042         movq    %r11,(%rsi)
 1043         POP_FRAME_POINTER
 1044         ret
 1045 END(fueword_nosmap)
 1046 
 1047 ENTRY(fueword_smap)
 1048         PUSH_FRAME_POINTER
 1049         movq    PCPU(CURPCB),%rcx
 1050         movq    $fusufault,PCB_ONFAULT(%rcx)
 1051 
 1052         movq    $VM_MAXUSER_ADDRESS-8,%rax
 1053         cmpq    %rax,%rdi                       /* verify address is valid */
 1054         ja      fusufault
 1055 
 1056         xorl    %eax,%eax
 1057         stac
 1058         movq    (%rdi),%r11
 1059         clac
 1060         movq    %rax,PCB_ONFAULT(%rcx)
 1061         movq    %r11,(%rsi)
 1062         POP_FRAME_POINTER
 1063         ret
 1064 END(fueword_smap)
 1065 
 1066 ENTRY(fueword32_nosmap)
 1067         PUSH_FRAME_POINTER
 1068         movq    PCPU(CURPCB),%rcx
 1069         movq    $fusufault,PCB_ONFAULT(%rcx)
 1070 
 1071         movq    $VM_MAXUSER_ADDRESS-4,%rax
 1072         cmpq    %rax,%rdi                       /* verify address is valid */
 1073         ja      fusufault
 1074 
 1075         xorl    %eax,%eax
 1076         movl    (%rdi),%r11d
 1077         movq    %rax,PCB_ONFAULT(%rcx)
 1078         movl    %r11d,(%rsi)
 1079         POP_FRAME_POINTER
 1080         ret
 1081 END(fueword32_nosmap)
 1082 
 1083 ENTRY(fueword32_smap)
 1084         PUSH_FRAME_POINTER
 1085         movq    PCPU(CURPCB),%rcx
 1086         movq    $fusufault,PCB_ONFAULT(%rcx)
 1087 
 1088         movq    $VM_MAXUSER_ADDRESS-4,%rax
 1089         cmpq    %rax,%rdi                       /* verify address is valid */
 1090         ja      fusufault
 1091 
 1092         xorl    %eax,%eax
 1093         stac
 1094         movl    (%rdi),%r11d
 1095         clac
 1096         movq    %rax,PCB_ONFAULT(%rcx)
 1097         movl    %r11d,(%rsi)
 1098         POP_FRAME_POINTER
 1099         ret
 1100 END(fueword32_smap)
 1101 
 1102 ENTRY(fuword16_nosmap)
 1103         PUSH_FRAME_POINTER
 1104         movq    PCPU(CURPCB),%rcx
 1105         movq    $fusufault,PCB_ONFAULT(%rcx)
 1106 
 1107         movq    $VM_MAXUSER_ADDRESS-2,%rax
 1108         cmpq    %rax,%rdi
 1109         ja      fusufault
 1110 
 1111         movzwl  (%rdi),%eax
 1112         movq    $0,PCB_ONFAULT(%rcx)
 1113         POP_FRAME_POINTER
 1114         ret
 1115 END(fuword16_nosmap)
 1116 
 1117 ENTRY(fuword16_smap)
 1118         PUSH_FRAME_POINTER
 1119         movq    PCPU(CURPCB),%rcx
 1120         movq    $fusufault,PCB_ONFAULT(%rcx)
 1121 
 1122         movq    $VM_MAXUSER_ADDRESS-2,%rax
 1123         cmpq    %rax,%rdi
 1124         ja      fusufault
 1125 
 1126         stac
 1127         movzwl  (%rdi),%eax
 1128         clac
 1129         movq    $0,PCB_ONFAULT(%rcx)
 1130         POP_FRAME_POINTER
 1131         ret
 1132 END(fuword16_smap)
 1133 
 1134 ENTRY(fubyte_nosmap)
 1135         PUSH_FRAME_POINTER
 1136         movq    PCPU(CURPCB),%rcx
 1137         movq    $fusufault,PCB_ONFAULT(%rcx)
 1138 
 1139         movq    $VM_MAXUSER_ADDRESS-1,%rax
 1140         cmpq    %rax,%rdi
 1141         ja      fusufault
 1142 
 1143         movzbl  (%rdi),%eax
 1144         movq    $0,PCB_ONFAULT(%rcx)
 1145         POP_FRAME_POINTER
 1146         ret
 1147 END(fubyte_nosmap)
 1148 
 1149 ENTRY(fubyte_smap)
 1150         PUSH_FRAME_POINTER
 1151         movq    PCPU(CURPCB),%rcx
 1152         movq    $fusufault,PCB_ONFAULT(%rcx)
 1153 
 1154         movq    $VM_MAXUSER_ADDRESS-1,%rax
 1155         cmpq    %rax,%rdi
 1156         ja      fusufault
 1157 
 1158         stac
 1159         movzbl  (%rdi),%eax
 1160         clac
 1161         movq    $0,PCB_ONFAULT(%rcx)
 1162         POP_FRAME_POINTER
 1163         ret
 1164 END(fubyte_smap)
 1165 
 1166 /*
 1167  * Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to
 1168  * user memory.
 1169  * addr = %rdi, value = %rsi
 1170  */
 1171 ENTRY(suword_nosmap)
 1172         PUSH_FRAME_POINTER
 1173         movq    PCPU(CURPCB),%rcx
 1174         movq    $fusufault,PCB_ONFAULT(%rcx)
 1175 
 1176         movq    $VM_MAXUSER_ADDRESS-8,%rax
 1177         cmpq    %rax,%rdi                       /* verify address validity */
 1178         ja      fusufault
 1179 
 1180         movq    %rsi,(%rdi)
 1181         xorl    %eax,%eax
 1182         movq    PCPU(CURPCB),%rcx
 1183         movq    %rax,PCB_ONFAULT(%rcx)
 1184         POP_FRAME_POINTER
 1185         ret
 1186 END(suword_nosmap)
 1187 
 1188 ENTRY(suword_smap)
 1189         PUSH_FRAME_POINTER
 1190         movq    PCPU(CURPCB),%rcx
 1191         movq    $fusufault,PCB_ONFAULT(%rcx)
 1192 
 1193         movq    $VM_MAXUSER_ADDRESS-8,%rax
 1194         cmpq    %rax,%rdi                       /* verify address validity */
 1195         ja      fusufault
 1196 
 1197         stac
 1198         movq    %rsi,(%rdi)
 1199         clac
 1200         xorl    %eax,%eax
 1201         movq    PCPU(CURPCB),%rcx
 1202         movq    %rax,PCB_ONFAULT(%rcx)
 1203         POP_FRAME_POINTER
 1204         ret
 1205 END(suword_smap)
 1206 
 1207 ENTRY(suword32_nosmap)
 1208         PUSH_FRAME_POINTER
 1209         movq    PCPU(CURPCB),%rcx
 1210         movq    $fusufault,PCB_ONFAULT(%rcx)
 1211 
 1212         movq    $VM_MAXUSER_ADDRESS-4,%rax
 1213         cmpq    %rax,%rdi                       /* verify address validity */
 1214         ja      fusufault
 1215 
 1216         movl    %esi,(%rdi)
 1217         xorl    %eax,%eax
 1218         movq    PCPU(CURPCB),%rcx
 1219         movq    %rax,PCB_ONFAULT(%rcx)
 1220         POP_FRAME_POINTER
 1221         ret
 1222 END(suword32_nosmap)
 1223 
 1224 ENTRY(suword32_smap)
 1225         PUSH_FRAME_POINTER
 1226         movq    PCPU(CURPCB),%rcx
 1227         movq    $fusufault,PCB_ONFAULT(%rcx)
 1228 
 1229         movq    $VM_MAXUSER_ADDRESS-4,%rax
 1230         cmpq    %rax,%rdi                       /* verify address validity */
 1231         ja      fusufault
 1232 
 1233         stac
 1234         movl    %esi,(%rdi)
 1235         clac
 1236         xorl    %eax,%eax
 1237         movq    PCPU(CURPCB),%rcx
 1238         movq    %rax,PCB_ONFAULT(%rcx)
 1239         POP_FRAME_POINTER
 1240         ret
 1241 END(suword32_smap)
 1242 
 1243 ENTRY(suword16_nosmap)
 1244         PUSH_FRAME_POINTER
 1245         movq    PCPU(CURPCB),%rcx
 1246         movq    $fusufault,PCB_ONFAULT(%rcx)
 1247 
 1248         movq    $VM_MAXUSER_ADDRESS-2,%rax
 1249         cmpq    %rax,%rdi                       /* verify address validity */
 1250         ja      fusufault
 1251 
 1252         movw    %si,(%rdi)
 1253         xorl    %eax,%eax
 1254         movq    %rax,PCB_ONFAULT(%rcx)
 1255         POP_FRAME_POINTER
 1256         ret
 1257 END(suword16_nosmap)
 1258 
 1259 ENTRY(suword16_smap)
 1260         PUSH_FRAME_POINTER
 1261         movq    PCPU(CURPCB),%rcx
 1262         movq    $fusufault,PCB_ONFAULT(%rcx)
 1263 
 1264         movq    $VM_MAXUSER_ADDRESS-2,%rax
 1265         cmpq    %rax,%rdi                       /* verify address validity */
 1266         ja      fusufault
 1267 
 1268         stac
 1269         movw    %si,(%rdi)
 1270         clac
 1271         xorl    %eax,%eax
 1272         movq    %rax,PCB_ONFAULT(%rcx)
 1273         POP_FRAME_POINTER
 1274         ret
 1275 END(suword16_smap)
 1276 
 1277 ENTRY(subyte_nosmap)
 1278         PUSH_FRAME_POINTER
 1279         movq    PCPU(CURPCB),%rcx
 1280         movq    $fusufault,PCB_ONFAULT(%rcx)
 1281 
 1282         movq    $VM_MAXUSER_ADDRESS-1,%rax
 1283         cmpq    %rax,%rdi                       /* verify address validity */
 1284         ja      fusufault
 1285 
 1286         movl    %esi,%eax
 1287         movb    %al,(%rdi)
 1288         xorl    %eax,%eax
 1289         movq    %rax,PCB_ONFAULT(%rcx)
 1290         POP_FRAME_POINTER
 1291         ret
 1292 END(subyte_nosmap)
 1293 
 1294 ENTRY(subyte_smap)
 1295         PUSH_FRAME_POINTER
 1296         movq    PCPU(CURPCB),%rcx
 1297         movq    $fusufault,PCB_ONFAULT(%rcx)
 1298 
 1299         movq    $VM_MAXUSER_ADDRESS-1,%rax
 1300         cmpq    %rax,%rdi                       /* verify address validity */
 1301         ja      fusufault
 1302 
 1303         movl    %esi,%eax
 1304         stac
 1305         movb    %al,(%rdi)
 1306         clac
 1307         xorl    %eax,%eax
 1308         movq    %rax,PCB_ONFAULT(%rcx)
 1309         POP_FRAME_POINTER
 1310         ret
 1311 END(subyte_smap)
 1312 
 1313         ALIGN_TEXT
 1314         /* Fault entry clears PSL.AC */
 1315 fusufault:
 1316         movq    PCPU(CURPCB),%rcx
 1317         xorl    %eax,%eax
 1318         movq    %rax,PCB_ONFAULT(%rcx)
 1319         decq    %rax
 1320         POP_FRAME_POINTER
 1321         ret
 1322 
 1323 /*
 1324  * copyinstr(from, to, maxlen, int *lencopied)
 1325  *           %rdi, %rsi, %rdx, %rcx
 1326  *
 1327  *      copy a string from 'from' to 'to', stop when a 0 character is reached.
 1328  *      return ENAMETOOLONG if string is longer than maxlen, and
 1329  *      EFAULT on protection violations. If lencopied is non-zero,
 1330  *      return the actual length in *lencopied.
 1331  */
 1332 .macro COPYINSTR smap
 1333         PUSH_FRAME_POINTER
 1334         movq    %rdx,%r8                        /* %r8 = maxlen */
 1335         movq    PCPU(CURPCB),%r9
 1336         movq    $cpystrflt,PCB_ONFAULT(%r9)
 1337 
 1338         movq    $VM_MAXUSER_ADDRESS,%rax
 1339 
 1340         /* make sure 'from' is within bounds */
 1341         subq    %rdi,%rax
 1342         jbe     cpystrflt
 1343 
 1344         SMAP_DISABLE \smap
 1345 
 1346         /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */
 1347         cmpq    %rdx,%rax
 1348         jb      8f
 1349 1:
 1350         incq    %rdx
 1351 2:
 1352         decq    %rdx
 1353 .if \smap == 0
 1354         jz      copyinstr_toolong
 1355 .else
 1356         jz      copyinstr_toolong_smap
 1357 .endif
 1358 
 1359         movb    (%rdi),%al
 1360         movb    %al,(%rsi)
 1361         incq    %rsi
 1362         incq    %rdi
 1363         testb   %al,%al
 1364         jnz     2b
 1365 
 1366         SMAP_ENABLE \smap
 1367 
 1368         /* Success -- 0 byte reached */
 1369         decq    %rdx
 1370         xorl    %eax,%eax
 1371 
 1372         /* set *lencopied and return %eax */
 1373         movq    %rax,PCB_ONFAULT(%r9)
 1374 
 1375         testq   %rcx,%rcx
 1376         jz      3f
 1377         subq    %rdx,%r8
 1378         movq    %r8,(%rcx)
 1379 3:
 1380         POP_FRAME_POINTER
 1381         ret
 1382         ALIGN_TEXT
 1383 8:
 1384         movq    %rax,%rdx
 1385         movq    %rax,%r8
 1386         jmp 1b
 1387 
 1388 .endm
 1389 
 1390 ENTRY(copyinstr_nosmap)
 1391         COPYINSTR smap=0
 1392 END(copyinstr_nosmap)
 1393 
 1394 ENTRY(copyinstr_smap)
 1395         COPYINSTR smap=1
 1396 END(copyinstr_smap)
 1397 
 1398 cpystrflt:
 1399         /* Fault entry clears PSL.AC */
 1400         movl    $EFAULT,%eax
 1401 cpystrflt_x:
 1402         /* set *lencopied and return %eax */
 1403         movq    $0,PCB_ONFAULT(%r9)
 1404 
 1405         testq   %rcx,%rcx
 1406         jz      1f
 1407         subq    %rdx,%r8
 1408         movq    %r8,(%rcx)
 1409 1:
 1410         POP_FRAME_POINTER
 1411         ret
 1412 
 1413 copyinstr_toolong_smap:
 1414         clac
 1415 copyinstr_toolong:
 1416         /* rdx is zero - return ENAMETOOLONG or EFAULT */
 1417         movq    $VM_MAXUSER_ADDRESS,%rax
 1418         cmpq    %rax,%rdi
 1419         jae     cpystrflt
 1420         movl    $ENAMETOOLONG,%eax
 1421         jmp     cpystrflt_x
 1422 
 1423 /*
 1424  * copystr(from, to, maxlen, int *lencopied)
 1425  *         %rdi, %rsi, %rdx, %rcx
 1426  */
 1427 ENTRY(copystr)
 1428         PUSH_FRAME_POINTER
 1429         movq    %rdx,%r8                        /* %r8 = maxlen */
 1430 
 1431         incq    %rdx
 1432 1:
 1433         decq    %rdx
 1434         jz      4f
 1435         movb    (%rdi),%al
 1436         movb    %al,(%rsi)
 1437         incq    %rsi
 1438         incq    %rdi
 1439         testb   %al,%al
 1440         jnz     1b
 1441 
 1442         /* Success -- 0 byte reached */
 1443         decq    %rdx
 1444         xorl    %eax,%eax
 1445 2:
 1446         testq   %rcx,%rcx
 1447         jz      3f
 1448         /* set *lencopied and return %rax */
 1449         subq    %rdx,%r8
 1450         movq    %r8,(%rcx)
 1451 3:
 1452         POP_FRAME_POINTER
 1453         ret
 1454 4:
 1455         /* rdx is zero -- return ENAMETOOLONG */
 1456         movl    $ENAMETOOLONG,%eax
 1457         jmp     2b
 1458 END(copystr)
 1459 
 1460 /*
 1461  * Handling of special amd64 registers and descriptor tables etc
 1462  */
 1463 /* void lgdt(struct region_descriptor *rdp); */
 1464 ENTRY(lgdt)
 1465         /* reload the descriptor table */
 1466         lgdt    (%rdi)
 1467 
 1468         /* flush the prefetch q */
 1469         jmp     1f
 1470         nop
 1471 1:
 1472         movl    $KDSEL,%eax
 1473         movl    %eax,%ds
 1474         movl    %eax,%es
 1475         movl    %eax,%fs        /* Beware, use wrmsr to set 64 bit base */
 1476         movl    %eax,%gs
 1477         movl    %eax,%ss
 1478 
 1479         /* reload code selector by turning return into intersegmental return */
 1480         popq    %rax
 1481         pushq   $KCSEL
 1482         pushq   %rax
 1483         MEXITCOUNT
 1484         lretq
 1485 END(lgdt)
 1486 
 1487 /*****************************************************************************/
 1488 /* setjump, longjump                                                         */
 1489 /*****************************************************************************/
 1490 
 1491 ENTRY(setjmp)
 1492         movq    %rbx,0(%rdi)                    /* save rbx */
 1493         movq    %rsp,8(%rdi)                    /* save rsp */
 1494         movq    %rbp,16(%rdi)                   /* save rbp */
 1495         movq    %r12,24(%rdi)                   /* save r12 */
 1496         movq    %r13,32(%rdi)                   /* save r13 */
 1497         movq    %r14,40(%rdi)                   /* save r14 */
 1498         movq    %r15,48(%rdi)                   /* save r15 */
 1499         movq    0(%rsp),%rdx                    /* get rta */
 1500         movq    %rdx,56(%rdi)                   /* save rip */
 1501         xorl    %eax,%eax                       /* return(0); */
 1502         ret
 1503 END(setjmp)
 1504 
 1505 ENTRY(longjmp)
 1506         movq    0(%rdi),%rbx                    /* restore rbx */
 1507         movq    8(%rdi),%rsp                    /* restore rsp */
 1508         movq    16(%rdi),%rbp                   /* restore rbp */
 1509         movq    24(%rdi),%r12                   /* restore r12 */
 1510         movq    32(%rdi),%r13                   /* restore r13 */
 1511         movq    40(%rdi),%r14                   /* restore r14 */
 1512         movq    48(%rdi),%r15                   /* restore r15 */
 1513         movq    56(%rdi),%rdx                   /* get rta */
 1514         movq    %rdx,0(%rsp)                    /* put in return frame */
 1515         xorl    %eax,%eax                       /* return(1); */
 1516         incl    %eax
 1517         ret
 1518 END(longjmp)
 1519 
 1520 /*
 1521  * Support for reading MSRs in the safe manner.  (Instead of panic on #gp,
 1522  * return an error.)
 1523  */
 1524 ENTRY(rdmsr_safe)
 1525 /* int rdmsr_safe(u_int msr, uint64_t *data) */
 1526         PUSH_FRAME_POINTER
 1527         movq    PCPU(CURPCB),%r8
 1528         movq    $msr_onfault,PCB_ONFAULT(%r8)
 1529         movl    %edi,%ecx
 1530         rdmsr                   /* Read MSR pointed by %ecx. Returns
 1531                                    hi byte in edx, lo in %eax */
 1532         salq    $32,%rdx        /* sign-shift %rdx left */
 1533         movl    %eax,%eax       /* zero-extend %eax -> %rax */
 1534         orq     %rdx,%rax
 1535         movq    %rax,(%rsi)
 1536         xorq    %rax,%rax
 1537         movq    %rax,PCB_ONFAULT(%r8)
 1538         POP_FRAME_POINTER
 1539         ret
 1540 
 1541 /*
 1542  * Support for writing MSRs in the safe manner.  (Instead of panic on #gp,
 1543  * return an error.)
 1544  */
 1545 ENTRY(wrmsr_safe)
 1546 /* int wrmsr_safe(u_int msr, uint64_t data) */
 1547         PUSH_FRAME_POINTER
 1548         movq    PCPU(CURPCB),%r8
 1549         movq    $msr_onfault,PCB_ONFAULT(%r8)
 1550         movl    %edi,%ecx
 1551         movl    %esi,%eax
 1552         sarq    $32,%rsi
 1553         movl    %esi,%edx
 1554         wrmsr                   /* Write MSR pointed by %ecx. Accepts
 1555                                    hi byte in edx, lo in %eax. */
 1556         xorq    %rax,%rax
 1557         movq    %rax,PCB_ONFAULT(%r8)
 1558         POP_FRAME_POINTER
 1559         ret
 1560 
 1561 /*
 1562  * MSR operations fault handler
 1563  */
 1564         ALIGN_TEXT
 1565 msr_onfault:
 1566         movq    $0,PCB_ONFAULT(%r8)
 1567         movl    $EFAULT,%eax
 1568         POP_FRAME_POINTER
 1569         ret
 1570 
 1571 /*
 1572  * void pmap_pti_pcid_invalidate(uint64_t ucr3, uint64_t kcr3);
 1573  * Invalidates address space addressed by ucr3, then returns to kcr3.
 1574  * Done in assembler to ensure no other memory accesses happen while
 1575  * on ucr3.
 1576  */
 1577         ALIGN_TEXT
 1578 ENTRY(pmap_pti_pcid_invalidate)
 1579         pushfq
 1580         cli
 1581         movq    %rdi,%cr3       /* to user page table */
 1582         movq    %rsi,%cr3       /* back to kernel */
 1583         popfq
 1584         retq
 1585 
 1586 /*
 1587  * void pmap_pti_pcid_invlpg(uint64_t ucr3, uint64_t kcr3, vm_offset_t va);
 1588  * Invalidates virtual address va in address space ucr3, then returns to kcr3.
 1589  */
 1590         ALIGN_TEXT
 1591 ENTRY(pmap_pti_pcid_invlpg)
 1592         pushfq
 1593         cli
 1594         movq    %rdi,%cr3       /* to user page table */
 1595         invlpg  (%rdx)
 1596         movq    %rsi,%cr3       /* back to kernel */
 1597         popfq
 1598         retq
 1599 
 1600 /*
 1601  * void pmap_pti_pcid_invlrng(uint64_t ucr3, uint64_t kcr3, vm_offset_t sva,
 1602  *     vm_offset_t eva);
 1603  * Invalidates virtual addresses between sva and eva in address space ucr3,
 1604  * then returns to kcr3.
 1605  */
 1606         ALIGN_TEXT
 1607 ENTRY(pmap_pti_pcid_invlrng)
 1608         pushfq
 1609         cli
 1610         movq    %rdi,%cr3       /* to user page table */
 1611 1:      invlpg  (%rdx)
 1612         addq    $PAGE_SIZE,%rdx
 1613         cmpq    %rdx,%rcx
 1614         ja      1b
 1615         movq    %rsi,%cr3       /* back to kernel */
 1616         popfq
 1617         retq
 1618 
 1619         .altmacro
 1620         .macro  rsb_seq_label l
 1621 rsb_seq_\l:
 1622         .endm
 1623         .macro  rsb_call_label l
 1624         call    rsb_seq_\l
 1625         .endm
 1626         .macro  rsb_seq count
 1627         ll=1
 1628         .rept   \count
 1629         rsb_call_label  %(ll)
 1630         nop
 1631         rsb_seq_label %(ll)
 1632         addq    $8,%rsp
 1633         ll=ll+1
 1634         .endr
 1635         .endm
 1636 
 1637 ENTRY(rsb_flush)
 1638         rsb_seq 32
 1639         ret
 1640 
 1641 /* all callers already saved %rax, %rdx, and %rcx */
 1642 ENTRY(handle_ibrs_entry)
 1643         cmpb    $0,hw_ibrs_ibpb_active(%rip)
 1644         je      1f
 1645         movl    $MSR_IA32_SPEC_CTRL,%ecx
 1646         rdmsr
 1647         orl     $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
 1648         orl     $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32,%edx
 1649         wrmsr
 1650         movb    $1,PCPU(IBPB_SET)
 1651         testl   $CPUID_STDEXT_SMEP,cpu_stdext_feature(%rip)
 1652         je      rsb_flush
 1653 1:      ret
 1654 END(handle_ibrs_entry)
 1655 
 1656 ENTRY(handle_ibrs_exit)
 1657         cmpb    $0,PCPU(IBPB_SET)
 1658         je      1f
 1659         movl    $MSR_IA32_SPEC_CTRL,%ecx
 1660         rdmsr
 1661         andl    $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
 1662         andl    $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
 1663         wrmsr
 1664         movb    $0,PCPU(IBPB_SET)
 1665 1:      ret
 1666 END(handle_ibrs_exit)
 1667 
 1668 /* registers-neutral version, but needs stack */
 1669 ENTRY(handle_ibrs_exit_rs)
 1670         cmpb    $0,PCPU(IBPB_SET)
 1671         je      1f
 1672         pushq   %rax
 1673         pushq   %rdx
 1674         pushq   %rcx
 1675         movl    $MSR_IA32_SPEC_CTRL,%ecx
 1676         rdmsr
 1677         andl    $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
 1678         andl    $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
 1679         wrmsr
 1680         popq    %rcx
 1681         popq    %rdx
 1682         popq    %rax
 1683         movb    $0,PCPU(IBPB_SET)
 1684 1:      ret
 1685 END(handle_ibrs_exit_rs)
 1686 
 1687         .noaltmacro
 1688 
 1689 /*
 1690  * Flush L1D cache.  Load enough of the data from the kernel text
 1691  * to flush existing L1D content.
 1692  *
 1693  * N.B. The function does not follow ABI calling conventions, it corrupts %rbx.
 1694  * The vmm.ko caller expects that only %rax, %rdx, %rbx, %rcx, %r9, and %rflags
 1695  * registers are clobbered.  The NMI handler caller only needs %r13 preserved.
 1696  */
 1697 ENTRY(flush_l1d_sw)
 1698 #define L1D_FLUSH_SIZE  (64 * 1024)
 1699         movq    $KERNBASE, %r9
 1700         movq    $-L1D_FLUSH_SIZE, %rcx
 1701         /*
 1702          * pass 1: Preload TLB.
 1703          * Kernel text is mapped using superpages.  TLB preload is
 1704          * done for the benefit of older CPUs which split 2M page
 1705          * into 4k TLB entries.
 1706          */
 1707 1:      movb    L1D_FLUSH_SIZE(%r9, %rcx), %al
 1708         addq    $PAGE_SIZE, %rcx
 1709         jne     1b
 1710         xorl    %eax, %eax
 1711         cpuid
 1712         movq    $-L1D_FLUSH_SIZE, %rcx
 1713         /* pass 2: Read each cache line. */
 1714 2:      movb    L1D_FLUSH_SIZE(%r9, %rcx), %al
 1715         addq    $64, %rcx
 1716         jne     2b
 1717         lfence
 1718         ret
 1719 #undef  L1D_FLUSH_SIZE
 1720 END(flush_l1d_sw)
 1721 
 1722 ENTRY(flush_l1d_sw_abi)
 1723         pushq   %rbx
 1724         call    flush_l1d_sw
 1725         popq    %rbx
 1726         ret
 1727 END(flush_l1d_sw_abi)
 1728 
 1729 ENTRY(mds_handler_void)
 1730         retq
 1731 END(mds_handler_void)
 1732 
 1733 ENTRY(mds_handler_verw)
 1734         subq    $8, %rsp
 1735         movw    %ds, (%rsp)
 1736         verw    (%rsp)
 1737         addq    $8, %rsp
 1738         retq
 1739 END(mds_handler_verw)
 1740 
 1741 ENTRY(mds_handler_ivb)
 1742         pushq   %rax
 1743         pushq   %rdx
 1744         pushq   %rcx
 1745 
 1746         movq    %cr0, %rax
 1747         testb   $CR0_TS, %al
 1748         je      1f
 1749         clts
 1750 1:      movq    PCPU(MDS_BUF), %rdx
 1751         movdqa  %xmm0, PCPU(MDS_TMP)
 1752         pxor    %xmm0, %xmm0
 1753 
 1754         lfence
 1755         orpd    (%rdx), %xmm0
 1756         orpd    (%rdx), %xmm0
 1757         mfence
 1758         movl    $40, %ecx
 1759         addq    $16, %rdx
 1760 2:      movntdq %xmm0, (%rdx)
 1761         addq    $16, %rdx
 1762         decl    %ecx
 1763         jnz     2b
 1764         mfence
 1765 
 1766         movdqa  PCPU(MDS_TMP),%xmm0
 1767         testb   $CR0_TS, %al
 1768         je      3f
 1769         movq    %rax, %cr0
 1770 3:      popq    %rcx
 1771         popq    %rdx
 1772         popq    %rax
 1773         retq
 1774 END(mds_handler_ivb)
 1775 
 1776 ENTRY(mds_handler_bdw)
 1777         pushq   %rax
 1778         pushq   %rbx
 1779         pushq   %rcx
 1780         pushq   %rdi
 1781         pushq   %rsi
 1782 
 1783         movq    %cr0, %rax
 1784         testb   $CR0_TS, %al
 1785         je      1f
 1786         clts
 1787 1:      movq    PCPU(MDS_BUF), %rbx
 1788         movdqa  %xmm0, PCPU(MDS_TMP)
 1789         pxor    %xmm0, %xmm0
 1790 
 1791         movq    %rbx, %rdi
 1792         movq    %rbx, %rsi
 1793         movl    $40, %ecx
 1794 2:      movntdq %xmm0, (%rbx)
 1795         addq    $16, %rbx
 1796         decl    %ecx
 1797         jnz     2b
 1798         mfence
 1799         movl    $1536, %ecx
 1800         rep; movsb
 1801         lfence
 1802 
 1803         movdqa  PCPU(MDS_TMP),%xmm0
 1804         testb   $CR0_TS, %al
 1805         je      3f
 1806         movq    %rax, %cr0
 1807 3:      popq    %rsi
 1808         popq    %rdi
 1809         popq    %rcx
 1810         popq    %rbx
 1811         popq    %rax
 1812         retq
 1813 END(mds_handler_bdw)
 1814 
 1815 ENTRY(mds_handler_skl_sse)
 1816         pushq   %rax
 1817         pushq   %rdx
 1818         pushq   %rcx
 1819         pushq   %rdi
 1820 
 1821         movq    %cr0, %rax
 1822         testb   $CR0_TS, %al
 1823         je      1f
 1824         clts
 1825 1:      movq    PCPU(MDS_BUF), %rdi
 1826         movq    PCPU(MDS_BUF64), %rdx
 1827         movdqa  %xmm0, PCPU(MDS_TMP)
 1828         pxor    %xmm0, %xmm0
 1829 
 1830         lfence
 1831         orpd    (%rdx), %xmm0
 1832         orpd    (%rdx), %xmm0
 1833         xorl    %eax, %eax
 1834 2:      clflushopt      5376(%rdi, %rax, 8)
 1835         addl    $8, %eax
 1836         cmpl    $8 * 12, %eax
 1837         jb      2b
 1838         sfence
 1839         movl    $6144, %ecx
 1840         xorl    %eax, %eax
 1841         rep; stosb
 1842         mfence
 1843 
 1844         movdqa  PCPU(MDS_TMP), %xmm0
 1845         testb   $CR0_TS, %al
 1846         je      3f
 1847         movq    %rax, %cr0
 1848 3:      popq    %rdi
 1849         popq    %rcx
 1850         popq    %rdx
 1851         popq    %rax
 1852         retq
 1853 END(mds_handler_skl_sse)
 1854 
 1855 ENTRY(mds_handler_skl_avx)
 1856         pushq   %rax
 1857         pushq   %rdx
 1858         pushq   %rcx
 1859         pushq   %rdi
 1860 
 1861         movq    %cr0, %rax
 1862         testb   $CR0_TS, %al
 1863         je      1f
 1864         clts
 1865 1:      movq    PCPU(MDS_BUF), %rdi
 1866         movq    PCPU(MDS_BUF64), %rdx
 1867         vmovdqa %ymm0, PCPU(MDS_TMP)
 1868         vpxor   %ymm0, %ymm0, %ymm0
 1869 
 1870         lfence
 1871         vorpd   (%rdx), %ymm0, %ymm0
 1872         vorpd   (%rdx), %ymm0, %ymm0
 1873         xorl    %eax, %eax
 1874 2:      clflushopt      5376(%rdi, %rax, 8)
 1875         addl    $8, %eax
 1876         cmpl    $8 * 12, %eax
 1877         jb      2b
 1878         sfence
 1879         movl    $6144, %ecx
 1880         xorl    %eax, %eax
 1881         rep; stosb
 1882         mfence
 1883 
 1884         vmovdqa PCPU(MDS_TMP), %ymm0
 1885         testb   $CR0_TS, %al
 1886         je      3f
 1887         movq    %rax, %cr0
 1888 3:      popq    %rdi
 1889         popq    %rcx
 1890         popq    %rdx
 1891         popq    %rax
 1892         retq
 1893 END(mds_handler_skl_avx)
 1894 
 1895 ENTRY(mds_handler_skl_avx512)
 1896         pushq   %rax
 1897         pushq   %rdx
 1898         pushq   %rcx
 1899         pushq   %rdi
 1900 
 1901         movq    %cr0, %rax
 1902         testb   $CR0_TS, %al
 1903         je      1f
 1904         clts
 1905 1:      movq    PCPU(MDS_BUF), %rdi
 1906         movq    PCPU(MDS_BUF64), %rdx
 1907         vmovdqa64       %zmm0, PCPU(MDS_TMP)
 1908         vpxor   %zmm0, %zmm0, %zmm0
 1909 
 1910         lfence
 1911         vorpd   (%rdx), %zmm0, %zmm0
 1912         vorpd   (%rdx), %zmm0, %zmm0
 1913         xorl    %eax, %eax
 1914 2:      clflushopt      5376(%rdi, %rax, 8)
 1915         addl    $8, %eax
 1916         cmpl    $8 * 12, %eax
 1917         jb      2b
 1918         sfence
 1919         movl    $6144, %ecx
 1920         xorl    %eax, %eax
 1921         rep; stosb
 1922         mfence
 1923 
 1924         vmovdqa64       PCPU(MDS_TMP), %zmm0
 1925         testb   $CR0_TS, %al
 1926         je      3f
 1927         movq    %rax, %cr0
 1928 3:      popq    %rdi
 1929         popq    %rcx
 1930         popq    %rdx
 1931         popq    %rax
 1932         retq
 1933 END(mds_handler_skl_avx512)
 1934 
 1935 ENTRY(mds_handler_silvermont)
 1936         pushq   %rax
 1937         pushq   %rdx
 1938         pushq   %rcx
 1939 
 1940         movq    %cr0, %rax
 1941         testb   $CR0_TS, %al
 1942         je      1f
 1943         clts
 1944 1:      movq    PCPU(MDS_BUF), %rdx
 1945         movdqa  %xmm0, PCPU(MDS_TMP)
 1946         pxor    %xmm0, %xmm0
 1947 
 1948         movl    $16, %ecx
 1949 2:      movntdq %xmm0, (%rdx)
 1950         addq    $16, %rdx
 1951         decl    %ecx
 1952         jnz     2b
 1953         mfence
 1954 
 1955         movdqa  PCPU(MDS_TMP),%xmm0
 1956         testb   $CR0_TS, %al
 1957         je      3f
 1958         movq    %rax, %cr0
 1959 3:      popq    %rcx
 1960         popq    %rdx
 1961         popq    %rax
 1962         retq
 1963 END(mds_handler_silvermont)

Cache object: 8362f0002cfdc967c1283619e96681b0


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.