The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/support.S

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2018-2019 The FreeBSD Foundation
    3  * Copyright (c) 2003 Peter Wemm.
    4  * Copyright (c) 1993 The Regents of the University of California.
    5  * All rights reserved.
    6  *
    7  * Portions of this software were developed by
    8  * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
    9  * the FreeBSD Foundation.
   10  *
   11  * Redistribution and use in source and binary forms, with or without
   12  * modification, are permitted provided that the following conditions
   13  * are met:
   14  * 1. Redistributions of source code must retain the above copyright
   15  *    notice, this list of conditions and the following disclaimer.
   16  * 2. Redistributions in binary form must reproduce the above copyright
   17  *    notice, this list of conditions and the following disclaimer in the
   18  *    documentation and/or other materials provided with the distribution.
   19  * 3. Neither the name of the University nor the names of its contributors
   20  *    may be used to endorse or promote products derived from this software
   21  *    without specific prior written permission.
   22  *
   23  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   24  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   26  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   27  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   28  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   29  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   31  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   32  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   33  * SUCH DAMAGE.
   34  *
   35  * $FreeBSD$
   36  */
   37 
   38 #include "opt_ddb.h"
   39 
   40 #include <machine/asmacros.h>
   41 #include <machine/specialreg.h>
   42 #include <machine/pmap.h>
   43 
   44 #include "assym.inc"
   45 
   46         .text
   47 
   48 /* Address: %rdi */
   49 ENTRY(pagezero_std)
   50         PUSH_FRAME_POINTER
   51         movl    $PAGE_SIZE/8,%ecx
   52         xorl    %eax,%eax
   53         rep
   54         stosq
   55         POP_FRAME_POINTER
   56         ret
   57 END(pagezero_std)
   58 
   59 ENTRY(pagezero_erms)
   60         PUSH_FRAME_POINTER
   61         movl    $PAGE_SIZE,%ecx
   62         xorl    %eax,%eax
   63         rep
   64         stosb
   65         POP_FRAME_POINTER
   66         ret
   67 END(pagezero_erms)
   68 
   69 /*
   70  * pagecopy(%rdi=from, %rsi=to)
   71  */
   72 ENTRY(pagecopy)
   73         PUSH_FRAME_POINTER
   74         movl    $PAGE_SIZE/8,%ecx
   75         movq    %rdi,%r9
   76         movq    %rsi,%rdi
   77         movq    %r9,%rsi
   78         rep
   79         movsq
   80         POP_FRAME_POINTER
   81         ret
   82 END(pagecopy)
   83 
   84 /* Address: %rdi */
   85 ENTRY(sse2_pagezero)
   86         PUSH_FRAME_POINTER
   87         movq    $-PAGE_SIZE,%rdx
   88         subq    %rdx,%rdi
   89         xorl    %eax,%eax
   90         jmp     1f
   91         /*
   92          * The loop takes 29 bytes.  Ensure that it doesn't cross a 32-byte
   93          * cache line.
   94          */
   95         .p2align 5,0x90
   96 1:
   97         movnti  %rax,(%rdi,%rdx)
   98         movnti  %rax,8(%rdi,%rdx)
   99         movnti  %rax,16(%rdi,%rdx)
  100         movnti  %rax,24(%rdi,%rdx)
  101         addq    $32,%rdx
  102         jne     1b
  103         sfence
  104         POP_FRAME_POINTER
  105         ret
  106 END(sse2_pagezero)
  107 
  108 /*
  109  * memcmp(b1, b2, len)
  110  *         rdi,rsi,rdx
  111  */
  112 ENTRY(memcmp)
  113         PUSH_FRAME_POINTER
  114 
  115         xorl    %eax,%eax
  116 10:
  117         cmpq    $16,%rdx
  118         ja      101632f
  119 
  120         cmpb    $8,%dl
  121         jg      100816f
  122 
  123         cmpb    $4,%dl
  124         jg      100408f
  125 
  126         cmpb    $2,%dl
  127         jge     100204f
  128 
  129         cmpb    $1,%dl
  130         jl      100000f
  131         movzbl  (%rdi),%eax
  132         movzbl  (%rsi),%r8d
  133         subl    %r8d,%eax
  134 100000:
  135         POP_FRAME_POINTER
  136         ret
  137 
  138         ALIGN_TEXT
  139 100816:
  140         movq    (%rdi),%r8
  141         movq    (%rsi),%r9
  142         cmpq    %r8,%r9
  143         jne     80f
  144         movq    -8(%rdi,%rdx),%r8
  145         movq    -8(%rsi,%rdx),%r9
  146         cmpq    %r8,%r9
  147         jne     10081608f
  148         POP_FRAME_POINTER
  149         ret
  150         ALIGN_TEXT
  151 100408:
  152         movl    (%rdi),%r8d
  153         movl    (%rsi),%r9d
  154         cmpl    %r8d,%r9d
  155         jne     80f
  156         movl    -4(%rdi,%rdx),%r8d
  157         movl    -4(%rsi,%rdx),%r9d
  158         cmpl    %r8d,%r9d
  159         jne     10040804f
  160         POP_FRAME_POINTER
  161         ret
  162         ALIGN_TEXT
  163 100204:
  164         movzwl  (%rdi),%r8d
  165         movzwl  (%rsi),%r9d
  166         cmpl    %r8d,%r9d
  167         jne     1f
  168         movzwl  -2(%rdi,%rdx),%r8d
  169         movzwl  -2(%rsi,%rdx),%r9d
  170         cmpl    %r8d,%r9d
  171         jne     1f
  172         POP_FRAME_POINTER
  173         ret
  174         ALIGN_TEXT
  175 101632:
  176         cmpq    $32,%rdx
  177         ja      103200f
  178         movq    (%rdi),%r8
  179         movq    (%rsi),%r9
  180         cmpq    %r8,%r9
  181         jne     80f
  182         movq    8(%rdi),%r8
  183         movq    8(%rsi),%r9
  184         cmpq    %r8,%r9
  185         jne     10163208f
  186         movq    -16(%rdi,%rdx),%r8
  187         movq    -16(%rsi,%rdx),%r9
  188         cmpq    %r8,%r9
  189         jne     10163216f
  190         movq    -8(%rdi,%rdx),%r8
  191         movq    -8(%rsi,%rdx),%r9
  192         cmpq    %r8,%r9
  193         jne     10163224f
  194         POP_FRAME_POINTER
  195         ret
  196         ALIGN_TEXT
  197 103200:
  198         movq    (%rdi),%r8
  199         movq    8(%rdi),%r9
  200         subq    (%rsi),%r8
  201         subq    8(%rsi),%r9
  202         orq     %r8,%r9
  203         jnz     10320000f
  204 
  205         movq    16(%rdi),%r8
  206         movq    24(%rdi),%r9
  207         subq    16(%rsi),%r8
  208         subq    24(%rsi),%r9
  209         orq     %r8,%r9
  210         jnz     10320016f
  211 
  212         leaq    32(%rdi),%rdi
  213         leaq    32(%rsi),%rsi
  214         subq    $32,%rdx
  215         cmpq    $32,%rdx
  216         jae     103200b
  217         cmpb    $0,%dl
  218         jne     10b
  219         POP_FRAME_POINTER
  220         ret
  221 
  222 /*
  223  * Mismatch was found.
  224  *
  225  * Before we compute it we narrow down the range (16 -> 8 -> 4 bytes).
  226  */
  227         ALIGN_TEXT
  228 10320016:
  229         leaq    16(%rdi),%rdi
  230         leaq    16(%rsi),%rsi
  231 10320000:
  232         movq    (%rdi),%r8
  233         movq    (%rsi),%r9
  234         cmpq    %r8,%r9
  235         jne     80f
  236         leaq    8(%rdi),%rdi
  237         leaq    8(%rsi),%rsi
  238         jmp     80f
  239         ALIGN_TEXT
  240 10081608:
  241 10163224:
  242         leaq    -8(%rdi,%rdx),%rdi
  243         leaq    -8(%rsi,%rdx),%rsi
  244         jmp     80f
  245         ALIGN_TEXT
  246 10163216:
  247         leaq    -16(%rdi,%rdx),%rdi
  248         leaq    -16(%rsi,%rdx),%rsi
  249         jmp     80f
  250         ALIGN_TEXT
  251 10163208:
  252         leaq    8(%rdi),%rdi
  253         leaq    8(%rsi),%rsi
  254         jmp     80f
  255         ALIGN_TEXT
  256 10040804:
  257         leaq    -4(%rdi,%rdx),%rdi
  258         leaq    -4(%rsi,%rdx),%rsi
  259         jmp     1f
  260 
  261         ALIGN_TEXT
  262 80:
  263         movl    (%rdi),%r8d
  264         movl    (%rsi),%r9d
  265         cmpl    %r8d,%r9d
  266         jne     1f
  267         leaq    4(%rdi),%rdi
  268         leaq    4(%rsi),%rsi
  269 
  270 /*
  271  * We have up to 4 bytes to inspect.
  272  */
  273 1:
  274         movzbl  (%rdi),%eax
  275         movzbl  (%rsi),%r8d
  276         cmpb    %r8b,%al
  277         jne     2f
  278 
  279         movzbl  1(%rdi),%eax
  280         movzbl  1(%rsi),%r8d
  281         cmpb    %r8b,%al
  282         jne     2f
  283 
  284         movzbl  2(%rdi),%eax
  285         movzbl  2(%rsi),%r8d
  286         cmpb    %r8b,%al
  287         jne     2f
  288 
  289         movzbl  3(%rdi),%eax
  290         movzbl  3(%rsi),%r8d
  291 2:
  292         subl    %r8d,%eax
  293         POP_FRAME_POINTER
  294         ret
  295 END(memcmp)
  296 
  297 /*
  298  * memmove(dst, src, cnt)
  299  *         rdi, rsi, rdx
  300  */
  301 
  302 /*
  303  * Register state at entry is supposed to be as follows:
  304  * rdi - destination
  305  * rsi - source
  306  * rdx - count
  307  *
  308  * The macro possibly clobbers the above and: rcx, r8, r9, r10
  309  * It does not clobber rax nor r11.
  310  */
  311 .macro MEMMOVE erms overlap begin end
  312         \begin
  313 
  314         /*
  315          * For sizes 0..32 all data is read before it is written, so there
  316          * is no correctness issue with direction of copying.
  317          */
  318         cmpq    $32,%rcx
  319         jbe     101632f
  320 
  321 .if \overlap == 1
  322         movq    %rdi,%r8
  323         subq    %rsi,%r8
  324         cmpq    %rcx,%r8        /* overlapping && src < dst? */
  325         jb      2f
  326 .endif
  327 
  328         cmpq    $256,%rcx
  329         ja      1256f
  330 
  331         ALIGN_TEXT
  332 103200:
  333         movq    (%rsi),%rdx
  334         movq    %rdx,(%rdi)
  335         movq    8(%rsi),%rdx
  336         movq    %rdx,8(%rdi)
  337         movq    16(%rsi),%rdx
  338         movq    %rdx,16(%rdi)
  339         movq    24(%rsi),%rdx
  340         movq    %rdx,24(%rdi)
  341         leaq    32(%rsi),%rsi
  342         leaq    32(%rdi),%rdi
  343         subq    $32,%rcx
  344         cmpq    $32,%rcx
  345         jae     103200b
  346         cmpb    $0,%cl
  347         jne     101632f
  348         \end
  349         ret
  350         ALIGN_TEXT
  351 101632:
  352         cmpb    $16,%cl
  353         jl      100816f
  354         movq    (%rsi),%rdx
  355         movq    8(%rsi),%r8
  356         movq    -16(%rsi,%rcx),%r9
  357         movq    -8(%rsi,%rcx),%r10
  358         movq    %rdx,(%rdi)
  359         movq    %r8,8(%rdi)
  360         movq    %r9,-16(%rdi,%rcx)
  361         movq    %r10,-8(%rdi,%rcx)
  362         \end
  363         ret
  364         ALIGN_TEXT
  365 100816:
  366         cmpb    $8,%cl
  367         jl      100408f
  368         movq    (%rsi),%rdx
  369         movq    -8(%rsi,%rcx),%r8
  370         movq    %rdx,(%rdi)
  371         movq    %r8,-8(%rdi,%rcx,)
  372         \end
  373         ret
  374         ALIGN_TEXT
  375 100408:
  376         cmpb    $4,%cl
  377         jl      100204f
  378         movl    (%rsi),%edx
  379         movl    -4(%rsi,%rcx),%r8d
  380         movl    %edx,(%rdi)
  381         movl    %r8d,-4(%rdi,%rcx)
  382         \end
  383         ret
  384         ALIGN_TEXT
  385 100204:
  386         cmpb    $2,%cl
  387         jl      100001f
  388         movzwl  (%rsi),%edx
  389         movzwl  -2(%rsi,%rcx),%r8d
  390         movw    %dx,(%rdi)
  391         movw    %r8w,-2(%rdi,%rcx)
  392         \end
  393         ret
  394         ALIGN_TEXT
  395 100001:
  396         cmpb    $1,%cl
  397         jl      100000f
  398         movb    (%rsi),%dl
  399         movb    %dl,(%rdi)
  400 100000:
  401         \end
  402         ret
  403 
  404         ALIGN_TEXT
  405 1256:
  406         testb   $15,%dil
  407         jnz     100f
  408 .if \erms == 1
  409         rep
  410         movsb
  411 .else
  412         shrq    $3,%rcx                         /* copy by 64-bit words */
  413         rep
  414         movsq
  415         movq    %rdx,%rcx
  416         andl    $7,%ecx                         /* any bytes left? */
  417         jne     100408b
  418 .endif
  419         \end
  420         ret
  421 100:
  422         movq    (%rsi),%r8
  423         movq    8(%rsi),%r9
  424         movq    %rdi,%r10
  425         movq    %rdi,%rcx
  426         andq    $15,%rcx
  427         leaq    -16(%rdx,%rcx),%rdx
  428         neg     %rcx
  429         leaq    16(%rdi,%rcx),%rdi
  430         leaq    16(%rsi,%rcx),%rsi
  431         movq    %rdx,%rcx
  432 .if \erms == 1
  433         rep
  434         movsb
  435         movq    %r8,(%r10)
  436         movq    %r9,8(%r10)
  437 .else
  438         shrq    $3,%rcx                         /* copy by 64-bit words */
  439         rep
  440         movsq
  441         movq    %r8,(%r10)
  442         movq    %r9,8(%r10)
  443         movq    %rdx,%rcx
  444         andl    $7,%ecx                         /* any bytes left? */
  445         jne     100408b
  446 .endif
  447         \end
  448         ret
  449 
  450 .if \overlap == 1
  451         /*
  452          * Copy backwards.
  453          */
  454         ALIGN_TEXT
  455 2:
  456         cmpq    $256,%rcx
  457         ja      2256f
  458 
  459         leaq    -8(%rdi,%rcx),%rdi
  460         leaq    -8(%rsi,%rcx),%rsi
  461 
  462         cmpq    $32,%rcx
  463         jb      2016f
  464 
  465         ALIGN_TEXT
  466 2032:
  467         movq    (%rsi),%rdx
  468         movq    %rdx,(%rdi)
  469         movq    -8(%rsi),%rdx
  470         movq    %rdx,-8(%rdi)
  471         movq    -16(%rsi),%rdx
  472         movq    %rdx,-16(%rdi)
  473         movq    -24(%rsi),%rdx
  474         movq    %rdx,-24(%rdi)
  475         leaq    -32(%rsi),%rsi
  476         leaq    -32(%rdi),%rdi
  477         subq    $32,%rcx
  478         cmpq    $32,%rcx
  479         jae     2032b
  480         cmpb    $0,%cl
  481         jne     2016f
  482         \end
  483         ret
  484         ALIGN_TEXT
  485 2016:
  486         cmpb    $16,%cl
  487         jl      2008f
  488         movq    (%rsi),%rdx
  489         movq    %rdx,(%rdi)
  490         movq    -8(%rsi),%rdx
  491         movq    %rdx,-8(%rdi)
  492         subb    $16,%cl
  493         jz      2000f
  494         leaq    -16(%rsi),%rsi
  495         leaq    -16(%rdi),%rdi
  496 2008:
  497         cmpb    $8,%cl
  498         jl      2004f
  499         movq    (%rsi),%rdx
  500         movq    %rdx,(%rdi)
  501         subb    $8,%cl
  502         jz      2000f
  503         leaq    -8(%rsi),%rsi
  504         leaq    -8(%rdi),%rdi
  505 2004:
  506         cmpb    $4,%cl
  507         jl      2002f
  508         movl    4(%rsi),%edx
  509         movl    %edx,4(%rdi)
  510         subb    $4,%cl
  511         jz      2000f
  512         leaq    -4(%rsi),%rsi
  513         leaq    -4(%rdi),%rdi
  514 2002:
  515         cmpb    $2,%cl
  516         jl      2001f
  517         movw    6(%rsi),%dx
  518         movw    %dx,6(%rdi)
  519         subb    $2,%cl
  520         jz      2000f
  521         leaq    -2(%rsi),%rsi
  522         leaq    -2(%rdi),%rdi
  523 2001:
  524         cmpb    $1,%cl
  525         jl      2000f
  526         movb    7(%rsi),%dl
  527         movb    %dl,7(%rdi)
  528 2000:
  529         \end
  530         ret
  531         ALIGN_TEXT
  532 2256:
  533         std
  534         leaq    -8(%rdi,%rcx),%rdi
  535         leaq    -8(%rsi,%rcx),%rsi
  536         shrq    $3,%rcx
  537         rep
  538         movsq
  539         cld
  540         movq    %rdx,%rcx
  541         andb    $7,%cl
  542         jne     2004b
  543         \end
  544         ret
  545 .endif
  546 .endm
  547 
  548 .macro MEMMOVE_BEGIN
  549         PUSH_FRAME_POINTER
  550         movq    %rdi,%rax
  551         movq    %rdx,%rcx
  552 .endm
  553 
  554 .macro MEMMOVE_END
  555         POP_FRAME_POINTER
  556 .endm
  557 
  558 ENTRY(memmove_std)
  559         MEMMOVE erms=0 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END
  560 END(memmove_std)
  561 
  562 ENTRY(memmove_erms)
  563         MEMMOVE erms=1 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END
  564 END(memmove_erms)
  565 
  566 /*
  567  * memcpy(dst, src, len)
  568  *        rdi, rsi, rdx
  569  *
  570  * Note: memcpy does not support overlapping copies
  571  */
  572 ENTRY(memcpy_std)
  573         MEMMOVE erms=0 overlap=0 begin=MEMMOVE_BEGIN end=MEMMOVE_END
  574 END(memcpy_std)
  575 
  576 ENTRY(memcpy_erms)
  577         MEMMOVE erms=1 overlap=0 begin=MEMMOVE_BEGIN end=MEMMOVE_END
  578 END(memcpy_erms)
  579 
  580 /*
  581  * memset(dst, c,   len)
  582  *        rdi, rsi, rdx
  583  */
  584 .macro MEMSET erms
  585         PUSH_FRAME_POINTER
  586         movq    %rdi,%rax
  587         movq    %rdx,%rcx
  588         movzbq  %sil,%r8
  589         movabs  $0x0101010101010101,%r10
  590         imulq   %r8,%r10
  591 
  592         cmpq    $32,%rcx
  593         jbe     101632f
  594 
  595         cmpq    $256,%rcx
  596         ja      1256f
  597 
  598         ALIGN_TEXT
  599 103200:
  600         movq    %r10,(%rdi)
  601         movq    %r10,8(%rdi)
  602         movq    %r10,16(%rdi)
  603         movq    %r10,24(%rdi)
  604         leaq    32(%rdi),%rdi
  605         subq    $32,%rcx
  606         cmpq    $32,%rcx
  607         ja      103200b
  608         cmpb    $16,%cl
  609         ja      201632f
  610         movq    %r10,-16(%rdi,%rcx)
  611         movq    %r10,-8(%rdi,%rcx)
  612         POP_FRAME_POINTER
  613         ret
  614         ALIGN_TEXT
  615 101632:
  616         cmpb    $16,%cl
  617         jl      100816f
  618 201632:
  619         movq    %r10,(%rdi)
  620         movq    %r10,8(%rdi)
  621         movq    %r10,-16(%rdi,%rcx)
  622         movq    %r10,-8(%rdi,%rcx)
  623         POP_FRAME_POINTER
  624         ret
  625         ALIGN_TEXT
  626 100816:
  627         cmpb    $8,%cl
  628         jl      100408f
  629         movq    %r10,(%rdi)
  630         movq    %r10,-8(%rdi,%rcx)
  631         POP_FRAME_POINTER
  632         ret
  633         ALIGN_TEXT
  634 100408:
  635         cmpb    $4,%cl
  636         jl      100204f
  637         movl    %r10d,(%rdi)
  638         movl    %r10d,-4(%rdi,%rcx)
  639         POP_FRAME_POINTER
  640         ret
  641         ALIGN_TEXT
  642 100204:
  643         cmpb    $2,%cl
  644         jl      100001f
  645         movw    %r10w,(%rdi)
  646         movw    %r10w,-2(%rdi,%rcx)
  647         POP_FRAME_POINTER
  648         ret
  649         ALIGN_TEXT
  650 100001:
  651         cmpb    $0,%cl
  652         je      100000f
  653         movb    %r10b,(%rdi)
  654 100000:
  655         POP_FRAME_POINTER
  656         ret
  657         ALIGN_TEXT
  658 1256:
  659         movq    %rdi,%r9
  660         movq    %r10,%rax
  661         testl   $15,%edi
  662         jnz     3f
  663 1:
  664 .if \erms == 1
  665         rep
  666         stosb
  667         movq    %r9,%rax
  668 .else
  669         movq    %rcx,%rdx
  670         shrq    $3,%rcx
  671         rep
  672         stosq
  673         movq    %r9,%rax
  674         andl    $7,%edx
  675         jnz     2f
  676         POP_FRAME_POINTER
  677         ret
  678 2:
  679         movq    %r10,-8(%rdi,%rdx)
  680 .endif
  681         POP_FRAME_POINTER
  682         ret
  683         ALIGN_TEXT
  684 3:
  685         movq    %r10,(%rdi)
  686         movq    %r10,8(%rdi)
  687         movq    %rdi,%r8
  688         andq    $15,%r8
  689         leaq    -16(%rcx,%r8),%rcx
  690         neg     %r8
  691         leaq    16(%rdi,%r8),%rdi
  692         jmp     1b
  693 .endm
  694 
  695 ENTRY(memset_std)
  696         MEMSET erms=0
  697 END(memset_std)
  698 
  699 ENTRY(memset_erms)
  700         MEMSET erms=1
  701 END(memset_erms)
  702 
  703 /* fillw(pat, base, cnt) */
  704 /*       %rdi,%rsi, %rdx */
  705 ENTRY(fillw)
  706         PUSH_FRAME_POINTER
  707         movq    %rdi,%rax
  708         movq    %rsi,%rdi
  709         movq    %rdx,%rcx
  710         rep
  711         stosw
  712         POP_FRAME_POINTER
  713         ret
  714 END(fillw)
  715 
  716 /*****************************************************************************/
  717 /* copyout and fubyte family                                                 */
  718 /*****************************************************************************/
  719 /*
  720  * Access user memory from inside the kernel. These routines should be
  721  * the only places that do this.
  722  *
  723  * These routines set curpcb->pcb_onfault for the time they execute. When a
  724  * protection violation occurs inside the functions, the trap handler
  725  * returns to *curpcb->pcb_onfault instead of the function.
  726  */
  727 
  728 .macro SMAP_DISABLE smap
  729 .if     \smap
  730         stac
  731 .endif
  732 .endm
  733 
  734 
  735 .macro SMAP_ENABLE smap
  736 .if     \smap
  737         clac
  738 .endif
  739 .endm
  740 
  741 .macro COPYINOUT_BEGIN
  742 .endm
  743 
  744 .macro COPYINOUT_END
  745         movq    %rax,PCB_ONFAULT(%r11)
  746         POP_FRAME_POINTER
  747 .endm
  748 
  749 .macro COPYINOUT_SMAP_END
  750         SMAP_ENABLE smap=1
  751         COPYINOUT_END
  752 .endm
  753 
  754 /*
  755  * copyout(from_kernel, to_user, len)
  756  *         %rdi,        %rsi,    %rdx
  757  */
  758 .macro  COPYOUT smap erms
  759         PUSH_FRAME_POINTER
  760         movq    PCPU(CURPCB),%r11
  761         movq    $copy_fault,PCB_ONFAULT(%r11)
  762 
  763         /*
  764          * Check explicitly for non-user addresses.
  765          * First, prevent address wrapping.
  766          */
  767         movq    %rsi,%rax
  768         addq    %rdx,%rax
  769         jc      copy_fault
  770 /*
  771  * XXX STOP USING VM_MAXUSER_ADDRESS.
  772  * It is an end address, not a max, so every time it is used correctly it
  773  * looks like there is an off by one error, and of course it caused an off
  774  * by one error in several places.
  775  */
  776         movq    $VM_MAXUSER_ADDRESS,%rcx
  777         cmpq    %rcx,%rax
  778         ja      copy_fault
  779 
  780         /*
  781          * Set return value to zero. Remaining failure mode goes through
  782          * copy_fault.
  783          */
  784         xorl    %eax,%eax
  785 
  786         /*
  787          * Set up arguments for MEMMOVE.
  788          */
  789         movq    %rdi,%r8
  790         movq    %rsi,%rdi
  791         movq    %r8,%rsi
  792         movq    %rdx,%rcx
  793 
  794 
  795         SMAP_DISABLE \smap
  796 .if     \smap == 1
  797         MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_SMAP_END
  798 .else
  799         MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_END
  800 .endif
  801         /* NOTREACHED */
  802 .endm
  803 
  804 ENTRY(copyout_nosmap_std)
  805         COPYOUT smap=0 erms=0
  806 END(copyout_nosmap_std)
  807 
  808 ENTRY(copyout_smap_std)
  809         COPYOUT smap=1 erms=0
  810 END(copyout_smap_std)
  811 
  812 ENTRY(copyout_nosmap_erms)
  813         COPYOUT smap=0 erms=1
  814 END(copyout_nosmap_erms)
  815 
  816 ENTRY(copyout_smap_erms)
  817         COPYOUT smap=1 erms=1
  818 END(copyout_smap_erms)
  819 
  820 /*
  821  * copyin(from_user, to_kernel, len)
  822  *        %rdi,      %rsi,      %rdx
  823  */
  824 .macro  COPYIN smap erms
  825         PUSH_FRAME_POINTER
  826         movq    PCPU(CURPCB),%r11
  827         movq    $copy_fault,PCB_ONFAULT(%r11)
  828 
  829         /*
  830          * make sure address is valid
  831          */
  832         movq    %rdi,%rax
  833         addq    %rdx,%rax
  834         jc      copy_fault
  835         movq    $VM_MAXUSER_ADDRESS,%rcx
  836         cmpq    %rcx,%rax
  837         ja      copy_fault
  838 
  839         xorl    %eax,%eax
  840 
  841         movq    %rdi,%r8
  842         movq    %rsi,%rdi
  843         movq    %r8,%rsi
  844         movq    %rdx,%rcx
  845 
  846         SMAP_DISABLE \smap
  847 .if     \smap == 1
  848         MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_SMAP_END
  849 .else
  850         MEMMOVE erms=\erms overlap=0 begin=COPYINOUT_BEGIN end=COPYINOUT_END
  851 .endif
  852         /* NOTREACHED */
  853 .endm
  854 
  855 ENTRY(copyin_nosmap_std)
  856         COPYIN smap=0 erms=0
  857 END(copyin_nosmap_std)
  858 
  859 ENTRY(copyin_smap_std)
  860         COPYIN smap=1 erms=0
  861 END(copyin_smap_std)
  862 
  863 ENTRY(copyin_nosmap_erms)
  864         COPYIN smap=0 erms=1
  865 END(copyin_nosmap_erms)
  866 
  867 ENTRY(copyin_smap_erms)
  868         COPYIN smap=1 erms=1
  869 END(copyin_smap_erms)
  870 
  871         ALIGN_TEXT
  872 copy_fault:
  873         testl   $CPUID_STDEXT_SMAP,cpu_stdext_feature(%rip)
  874         je      1f
  875         clac
  876 1:      movq    $0,PCB_ONFAULT(%r11)
  877         movl    $EFAULT,%eax
  878         POP_FRAME_POINTER
  879         ret
  880 
  881 /*
  882  * casueword32.  Compare and set user integer.  Returns -1 on fault,
  883  *        0 if access was successful.  Old value is written to *oldp.
  884  *        dst = %rdi, old = %esi, oldp = %rdx, new = %ecx
  885  */
  886 ENTRY(casueword32_nosmap)
  887         PUSH_FRAME_POINTER
  888         movq    PCPU(CURPCB),%r8
  889         movq    $fusufault,PCB_ONFAULT(%r8)
  890 
  891         movq    $VM_MAXUSER_ADDRESS-4,%rax
  892         cmpq    %rax,%rdi                       /* verify address is valid */
  893         ja      fusufault
  894 
  895         movl    %esi,%eax                       /* old */
  896 #ifdef SMP
  897         lock
  898 #endif
  899         cmpxchgl %ecx,(%rdi)                    /* new = %ecx */
  900         setne   %cl
  901 
  902         /*
  903          * The old value is in %eax.  If the store succeeded it will be the
  904          * value we expected (old) from before the store, otherwise it will
  905          * be the current value.  Save %eax into %esi to prepare the return
  906          * value.
  907          */
  908         movl    %eax,%esi
  909         xorl    %eax,%eax
  910         movq    %rax,PCB_ONFAULT(%r8)
  911 
  912         /*
  913          * Access the oldp after the pcb_onfault is cleared, to correctly
  914          * catch corrupted pointer.
  915          */
  916         movl    %esi,(%rdx)                     /* oldp = %rdx */
  917         POP_FRAME_POINTER
  918         movzbl  %cl, %eax
  919         ret
  920 END(casueword32_nosmap)
  921 
  922 ENTRY(casueword32_smap)
  923         PUSH_FRAME_POINTER
  924         movq    PCPU(CURPCB),%r8
  925         movq    $fusufault,PCB_ONFAULT(%r8)
  926 
  927         movq    $VM_MAXUSER_ADDRESS-4,%rax
  928         cmpq    %rax,%rdi                       /* verify address is valid */
  929         ja      fusufault
  930 
  931         movl    %esi,%eax                       /* old */
  932         stac
  933 #ifdef SMP
  934         lock
  935 #endif
  936         cmpxchgl %ecx,(%rdi)                    /* new = %ecx */
  937         clac
  938         setne   %cl
  939 
  940         /*
  941          * The old value is in %eax.  If the store succeeded it will be the
  942          * value we expected (old) from before the store, otherwise it will
  943          * be the current value.  Save %eax into %esi to prepare the return
  944          * value.
  945          */
  946         movl    %eax,%esi
  947         xorl    %eax,%eax
  948         movq    %rax,PCB_ONFAULT(%r8)
  949 
  950         /*
  951          * Access the oldp after the pcb_onfault is cleared, to correctly
  952          * catch corrupted pointer.
  953          */
  954         movl    %esi,(%rdx)                     /* oldp = %rdx */
  955         POP_FRAME_POINTER
  956         movzbl  %cl, %eax
  957         ret
  958 END(casueword32_smap)
  959 
  960 /*
  961  * casueword.  Compare and set user long.  Returns -1 on fault,
  962  *        0 if access was successful.  Old value is written to *oldp.
  963  *        dst = %rdi, old = %rsi, oldp = %rdx, new = %rcx
  964  */
  965 ENTRY(casueword_nosmap)
  966         PUSH_FRAME_POINTER
  967         movq    PCPU(CURPCB),%r8
  968         movq    $fusufault,PCB_ONFAULT(%r8)
  969 
  970         movq    $VM_MAXUSER_ADDRESS-4,%rax
  971         cmpq    %rax,%rdi                       /* verify address is valid */
  972         ja      fusufault
  973 
  974         movq    %rsi,%rax                       /* old */
  975 #ifdef SMP
  976         lock
  977 #endif
  978         cmpxchgq %rcx,(%rdi)                    /* new = %rcx */
  979         setne   %cl
  980 
  981         /*
  982          * The old value is in %rax.  If the store succeeded it will be the
  983          * value we expected (old) from before the store, otherwise it will
  984          * be the current value.
  985          */
  986         movq    %rax,%rsi
  987         xorl    %eax,%eax
  988         movq    %rax,PCB_ONFAULT(%r8)
  989         movq    %rsi,(%rdx)
  990         POP_FRAME_POINTER
  991         movzbl  %cl, %eax
  992         ret
  993 END(casueword_nosmap)
  994 
  995 ENTRY(casueword_smap)
  996         PUSH_FRAME_POINTER
  997         movq    PCPU(CURPCB),%r8
  998         movq    $fusufault,PCB_ONFAULT(%r8)
  999 
 1000         movq    $VM_MAXUSER_ADDRESS-4,%rax
 1001         cmpq    %rax,%rdi                       /* verify address is valid */
 1002         ja      fusufault
 1003 
 1004         movq    %rsi,%rax                       /* old */
 1005         stac
 1006 #ifdef SMP
 1007         lock
 1008 #endif
 1009         cmpxchgq %rcx,(%rdi)                    /* new = %rcx */
 1010         clac
 1011         setne   %cl
 1012 
 1013         /*
 1014          * The old value is in %rax.  If the store succeeded it will be the
 1015          * value we expected (old) from before the store, otherwise it will
 1016          * be the current value.
 1017          */
 1018         movq    %rax,%rsi
 1019         xorl    %eax,%eax
 1020         movq    %rax,PCB_ONFAULT(%r8)
 1021         movq    %rsi,(%rdx)
 1022         POP_FRAME_POINTER
 1023         movzbl  %cl, %eax
 1024         ret
 1025 END(casueword_smap)
 1026 
 1027 /*
 1028  * Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit
 1029  * byte from user memory.
 1030  * addr = %rdi, valp = %rsi
 1031  */
 1032 
 1033 ENTRY(fueword_nosmap)
 1034         PUSH_FRAME_POINTER
 1035         movq    PCPU(CURPCB),%rcx
 1036         movq    $fusufault,PCB_ONFAULT(%rcx)
 1037 
 1038         movq    $VM_MAXUSER_ADDRESS-8,%rax
 1039         cmpq    %rax,%rdi                       /* verify address is valid */
 1040         ja      fusufault
 1041 
 1042         xorl    %eax,%eax
 1043         movq    (%rdi),%r11
 1044         movq    %rax,PCB_ONFAULT(%rcx)
 1045         movq    %r11,(%rsi)
 1046         POP_FRAME_POINTER
 1047         ret
 1048 END(fueword_nosmap)
 1049 
 1050 ENTRY(fueword_smap)
 1051         PUSH_FRAME_POINTER
 1052         movq    PCPU(CURPCB),%rcx
 1053         movq    $fusufault,PCB_ONFAULT(%rcx)
 1054 
 1055         movq    $VM_MAXUSER_ADDRESS-8,%rax
 1056         cmpq    %rax,%rdi                       /* verify address is valid */
 1057         ja      fusufault
 1058 
 1059         xorl    %eax,%eax
 1060         stac
 1061         movq    (%rdi),%r11
 1062         clac
 1063         movq    %rax,PCB_ONFAULT(%rcx)
 1064         movq    %r11,(%rsi)
 1065         POP_FRAME_POINTER
 1066         ret
 1067 END(fueword_smap)
 1068 
 1069 ENTRY(fueword32_nosmap)
 1070         PUSH_FRAME_POINTER
 1071         movq    PCPU(CURPCB),%rcx
 1072         movq    $fusufault,PCB_ONFAULT(%rcx)
 1073 
 1074         movq    $VM_MAXUSER_ADDRESS-4,%rax
 1075         cmpq    %rax,%rdi                       /* verify address is valid */
 1076         ja      fusufault
 1077 
 1078         xorl    %eax,%eax
 1079         movl    (%rdi),%r11d
 1080         movq    %rax,PCB_ONFAULT(%rcx)
 1081         movl    %r11d,(%rsi)
 1082         POP_FRAME_POINTER
 1083         ret
 1084 END(fueword32_nosmap)
 1085 
 1086 ENTRY(fueword32_smap)
 1087         PUSH_FRAME_POINTER
 1088         movq    PCPU(CURPCB),%rcx
 1089         movq    $fusufault,PCB_ONFAULT(%rcx)
 1090 
 1091         movq    $VM_MAXUSER_ADDRESS-4,%rax
 1092         cmpq    %rax,%rdi                       /* verify address is valid */
 1093         ja      fusufault
 1094 
 1095         xorl    %eax,%eax
 1096         stac
 1097         movl    (%rdi),%r11d
 1098         clac
 1099         movq    %rax,PCB_ONFAULT(%rcx)
 1100         movl    %r11d,(%rsi)
 1101         POP_FRAME_POINTER
 1102         ret
 1103 END(fueword32_smap)
 1104 
 1105 ENTRY(fuword16_nosmap)
 1106         PUSH_FRAME_POINTER
 1107         movq    PCPU(CURPCB),%rcx
 1108         movq    $fusufault,PCB_ONFAULT(%rcx)
 1109 
 1110         movq    $VM_MAXUSER_ADDRESS-2,%rax
 1111         cmpq    %rax,%rdi
 1112         ja      fusufault
 1113 
 1114         movzwl  (%rdi),%eax
 1115         movq    $0,PCB_ONFAULT(%rcx)
 1116         POP_FRAME_POINTER
 1117         ret
 1118 END(fuword16_nosmap)
 1119 
 1120 ENTRY(fuword16_smap)
 1121         PUSH_FRAME_POINTER
 1122         movq    PCPU(CURPCB),%rcx
 1123         movq    $fusufault,PCB_ONFAULT(%rcx)
 1124 
 1125         movq    $VM_MAXUSER_ADDRESS-2,%rax
 1126         cmpq    %rax,%rdi
 1127         ja      fusufault
 1128 
 1129         stac
 1130         movzwl  (%rdi),%eax
 1131         clac
 1132         movq    $0,PCB_ONFAULT(%rcx)
 1133         POP_FRAME_POINTER
 1134         ret
 1135 END(fuword16_smap)
 1136 
 1137 ENTRY(fubyte_nosmap)
 1138         PUSH_FRAME_POINTER
 1139         movq    PCPU(CURPCB),%rcx
 1140         movq    $fusufault,PCB_ONFAULT(%rcx)
 1141 
 1142         movq    $VM_MAXUSER_ADDRESS-1,%rax
 1143         cmpq    %rax,%rdi
 1144         ja      fusufault
 1145 
 1146         movzbl  (%rdi),%eax
 1147         movq    $0,PCB_ONFAULT(%rcx)
 1148         POP_FRAME_POINTER
 1149         ret
 1150 END(fubyte_nosmap)
 1151 
 1152 ENTRY(fubyte_smap)
 1153         PUSH_FRAME_POINTER
 1154         movq    PCPU(CURPCB),%rcx
 1155         movq    $fusufault,PCB_ONFAULT(%rcx)
 1156 
 1157         movq    $VM_MAXUSER_ADDRESS-1,%rax
 1158         cmpq    %rax,%rdi
 1159         ja      fusufault
 1160 
 1161         stac
 1162         movzbl  (%rdi),%eax
 1163         clac
 1164         movq    $0,PCB_ONFAULT(%rcx)
 1165         POP_FRAME_POINTER
 1166         ret
 1167 END(fubyte_smap)
 1168 
 1169 /*
 1170  * Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to
 1171  * user memory.
 1172  * addr = %rdi, value = %rsi
 1173  */
 1174 ENTRY(suword_nosmap)
 1175         PUSH_FRAME_POINTER
 1176         movq    PCPU(CURPCB),%rcx
 1177         movq    $fusufault,PCB_ONFAULT(%rcx)
 1178 
 1179         movq    $VM_MAXUSER_ADDRESS-8,%rax
 1180         cmpq    %rax,%rdi                       /* verify address validity */
 1181         ja      fusufault
 1182 
 1183         movq    %rsi,(%rdi)
 1184         xorl    %eax,%eax
 1185         movq    %rax,PCB_ONFAULT(%rcx)
 1186         POP_FRAME_POINTER
 1187         ret
 1188 END(suword_nosmap)
 1189 
 1190 ENTRY(suword_smap)
 1191         PUSH_FRAME_POINTER
 1192         movq    PCPU(CURPCB),%rcx
 1193         movq    $fusufault,PCB_ONFAULT(%rcx)
 1194 
 1195         movq    $VM_MAXUSER_ADDRESS-8,%rax
 1196         cmpq    %rax,%rdi                       /* verify address validity */
 1197         ja      fusufault
 1198 
 1199         stac
 1200         movq    %rsi,(%rdi)
 1201         clac
 1202         xorl    %eax,%eax
 1203         movq    %rax,PCB_ONFAULT(%rcx)
 1204         POP_FRAME_POINTER
 1205         ret
 1206 END(suword_smap)
 1207 
 1208 ENTRY(suword32_nosmap)
 1209         PUSH_FRAME_POINTER
 1210         movq    PCPU(CURPCB),%rcx
 1211         movq    $fusufault,PCB_ONFAULT(%rcx)
 1212 
 1213         movq    $VM_MAXUSER_ADDRESS-4,%rax
 1214         cmpq    %rax,%rdi                       /* verify address validity */
 1215         ja      fusufault
 1216 
 1217         movl    %esi,(%rdi)
 1218         xorl    %eax,%eax
 1219         movq    %rax,PCB_ONFAULT(%rcx)
 1220         POP_FRAME_POINTER
 1221         ret
 1222 END(suword32_nosmap)
 1223 
 1224 ENTRY(suword32_smap)
 1225         PUSH_FRAME_POINTER
 1226         movq    PCPU(CURPCB),%rcx
 1227         movq    $fusufault,PCB_ONFAULT(%rcx)
 1228 
 1229         movq    $VM_MAXUSER_ADDRESS-4,%rax
 1230         cmpq    %rax,%rdi                       /* verify address validity */
 1231         ja      fusufault
 1232 
 1233         stac
 1234         movl    %esi,(%rdi)
 1235         clac
 1236         xorl    %eax,%eax
 1237         movq    %rax,PCB_ONFAULT(%rcx)
 1238         POP_FRAME_POINTER
 1239         ret
 1240 END(suword32_smap)
 1241 
 1242 ENTRY(suword16_nosmap)
 1243         PUSH_FRAME_POINTER
 1244         movq    PCPU(CURPCB),%rcx
 1245         movq    $fusufault,PCB_ONFAULT(%rcx)
 1246 
 1247         movq    $VM_MAXUSER_ADDRESS-2,%rax
 1248         cmpq    %rax,%rdi                       /* verify address validity */
 1249         ja      fusufault
 1250 
 1251         movw    %si,(%rdi)
 1252         xorl    %eax,%eax
 1253         movq    %rax,PCB_ONFAULT(%rcx)
 1254         POP_FRAME_POINTER
 1255         ret
 1256 END(suword16_nosmap)
 1257 
 1258 ENTRY(suword16_smap)
 1259         PUSH_FRAME_POINTER
 1260         movq    PCPU(CURPCB),%rcx
 1261         movq    $fusufault,PCB_ONFAULT(%rcx)
 1262 
 1263         movq    $VM_MAXUSER_ADDRESS-2,%rax
 1264         cmpq    %rax,%rdi                       /* verify address validity */
 1265         ja      fusufault
 1266 
 1267         stac
 1268         movw    %si,(%rdi)
 1269         clac
 1270         xorl    %eax,%eax
 1271         movq    %rax,PCB_ONFAULT(%rcx)
 1272         POP_FRAME_POINTER
 1273         ret
 1274 END(suword16_smap)
 1275 
 1276 ENTRY(subyte_nosmap)
 1277         PUSH_FRAME_POINTER
 1278         movq    PCPU(CURPCB),%rcx
 1279         movq    $fusufault,PCB_ONFAULT(%rcx)
 1280 
 1281         movq    $VM_MAXUSER_ADDRESS-1,%rax
 1282         cmpq    %rax,%rdi                       /* verify address validity */
 1283         ja      fusufault
 1284 
 1285         movl    %esi,%eax
 1286         movb    %al,(%rdi)
 1287         xorl    %eax,%eax
 1288         movq    %rax,PCB_ONFAULT(%rcx)
 1289         POP_FRAME_POINTER
 1290         ret
 1291 END(subyte_nosmap)
 1292 
 1293 ENTRY(subyte_smap)
 1294         PUSH_FRAME_POINTER
 1295         movq    PCPU(CURPCB),%rcx
 1296         movq    $fusufault,PCB_ONFAULT(%rcx)
 1297 
 1298         movq    $VM_MAXUSER_ADDRESS-1,%rax
 1299         cmpq    %rax,%rdi                       /* verify address validity */
 1300         ja      fusufault
 1301 
 1302         movl    %esi,%eax
 1303         stac
 1304         movb    %al,(%rdi)
 1305         clac
 1306         xorl    %eax,%eax
 1307         movq    %rax,PCB_ONFAULT(%rcx)
 1308         POP_FRAME_POINTER
 1309         ret
 1310 END(subyte_smap)
 1311 
 1312         ALIGN_TEXT
 1313 fusufault:
 1314         testl   $CPUID_STDEXT_SMAP,cpu_stdext_feature(%rip)
 1315         je      1f
 1316         clac
 1317 1:      movq    PCPU(CURPCB),%rcx
 1318         xorl    %eax,%eax
 1319         movq    %rax,PCB_ONFAULT(%rcx)
 1320         decq    %rax
 1321         POP_FRAME_POINTER
 1322         ret
 1323 
 1324 /*
 1325  * copyinstr(from, to, maxlen, int *lencopied)
 1326  *           %rdi, %rsi, %rdx, %rcx
 1327  *
 1328  *      copy a string from 'from' to 'to', stop when a 0 character is reached.
 1329  *      return ENAMETOOLONG if string is longer than maxlen, and
 1330  *      EFAULT on protection violations. If lencopied is non-zero,
 1331  *      return the actual length in *lencopied.
 1332  */
 1333 .macro COPYINSTR smap
 1334         PUSH_FRAME_POINTER
 1335         movq    %rdx,%r8                        /* %r8 = maxlen */
 1336         movq    PCPU(CURPCB),%r9
 1337         movq    $cpystrflt,PCB_ONFAULT(%r9)
 1338 
 1339         movq    $VM_MAXUSER_ADDRESS,%rax
 1340 
 1341         /* make sure 'from' is within bounds */
 1342         subq    %rdi,%rax
 1343         jbe     cpystrflt
 1344 
 1345         SMAP_DISABLE \smap
 1346 
 1347         /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */
 1348         cmpq    %rdx,%rax
 1349         jb      8f
 1350 1:
 1351         incq    %rdx
 1352 2:
 1353         decq    %rdx
 1354 .if \smap == 0
 1355         jz      copyinstr_toolong
 1356 .else
 1357         jz      copyinstr_toolong_smap
 1358 .endif
 1359 
 1360         movb    (%rdi),%al
 1361         movb    %al,(%rsi)
 1362         incq    %rsi
 1363         incq    %rdi
 1364         testb   %al,%al
 1365         jnz     2b
 1366 
 1367         SMAP_ENABLE \smap
 1368 
 1369         /* Success -- 0 byte reached */
 1370         decq    %rdx
 1371         xorl    %eax,%eax
 1372 
 1373         /* set *lencopied and return %eax */
 1374         movq    %rax,PCB_ONFAULT(%r9)
 1375 
 1376         testq   %rcx,%rcx
 1377         jz      3f
 1378         subq    %rdx,%r8
 1379         movq    %r8,(%rcx)
 1380 3:
 1381         POP_FRAME_POINTER
 1382         ret
 1383         ALIGN_TEXT
 1384 8:
 1385         movq    %rax,%rdx
 1386         movq    %rax,%r8
 1387         jmp 1b
 1388 
 1389 .endm
 1390 
 1391 ENTRY(copyinstr_nosmap)
 1392         COPYINSTR smap=0
 1393 END(copyinstr_nosmap)
 1394 
 1395 ENTRY(copyinstr_smap)
 1396         COPYINSTR smap=1
 1397 END(copyinstr_smap)
 1398 
 1399 cpystrflt:
 1400         testl   $CPUID_STDEXT_SMAP,cpu_stdext_feature(%rip)
 1401         je      1f
 1402         clac
 1403 1:      movl    $EFAULT,%eax
 1404 cpystrflt_x:
 1405         /* set *lencopied and return %eax */
 1406         movq    $0,PCB_ONFAULT(%r9)
 1407 
 1408         testq   %rcx,%rcx
 1409         jz      1f
 1410         subq    %rdx,%r8
 1411         movq    %r8,(%rcx)
 1412 1:
 1413         POP_FRAME_POINTER
 1414         ret
 1415 
 1416 copyinstr_toolong_smap:
 1417         clac
 1418 copyinstr_toolong:
 1419         /* rdx is zero - return ENAMETOOLONG or EFAULT */
 1420         movq    $VM_MAXUSER_ADDRESS,%rax
 1421         cmpq    %rax,%rdi
 1422         jae     cpystrflt
 1423         movl    $ENAMETOOLONG,%eax
 1424         jmp     cpystrflt_x
 1425 
 1426 /*
 1427  * copystr(from, to, maxlen, int *lencopied)
 1428  *         %rdi, %rsi, %rdx, %rcx
 1429  */
 1430 ENTRY(copystr)
 1431         PUSH_FRAME_POINTER
 1432         movq    %rdx,%r8                        /* %r8 = maxlen */
 1433 
 1434         incq    %rdx
 1435 1:
 1436         decq    %rdx
 1437         jz      4f
 1438         movb    (%rdi),%al
 1439         movb    %al,(%rsi)
 1440         incq    %rsi
 1441         incq    %rdi
 1442         testb   %al,%al
 1443         jnz     1b
 1444 
 1445         /* Success -- 0 byte reached */
 1446         decq    %rdx
 1447         xorl    %eax,%eax
 1448 2:
 1449         testq   %rcx,%rcx
 1450         jz      3f
 1451         /* set *lencopied and return %rax */
 1452         subq    %rdx,%r8
 1453         movq    %r8,(%rcx)
 1454 3:
 1455         POP_FRAME_POINTER
 1456         ret
 1457 4:
 1458         /* rdx is zero -- return ENAMETOOLONG */
 1459         movl    $ENAMETOOLONG,%eax
 1460         jmp     2b
 1461 END(copystr)
 1462 
 1463 /*
 1464  * Handling of special amd64 registers and descriptor tables etc
 1465  */
 1466 /* void lgdt(struct region_descriptor *rdp); */
 1467 ENTRY(lgdt)
 1468         /* reload the descriptor table */
 1469         lgdt    (%rdi)
 1470 
 1471         /* flush the prefetch q */
 1472         jmp     1f
 1473         nop
 1474 1:
 1475         movl    $KDSEL,%eax
 1476         movl    %eax,%ds
 1477         movl    %eax,%es
 1478         movl    %eax,%fs        /* Beware, use wrmsr to set 64 bit base */
 1479         movl    %eax,%gs
 1480         movl    %eax,%ss
 1481 
 1482         /* reload code selector by turning return into intersegmental return */
 1483         popq    %rax
 1484         pushq   $KCSEL
 1485         pushq   %rax
 1486         MEXITCOUNT
 1487         lretq
 1488 END(lgdt)
 1489 
 1490 /*****************************************************************************/
 1491 /* setjump, longjump                                                         */
 1492 /*****************************************************************************/
 1493 
 1494 ENTRY(setjmp)
 1495         movq    %rbx,0(%rdi)                    /* save rbx */
 1496         movq    %rsp,8(%rdi)                    /* save rsp */
 1497         movq    %rbp,16(%rdi)                   /* save rbp */
 1498         movq    %r12,24(%rdi)                   /* save r12 */
 1499         movq    %r13,32(%rdi)                   /* save r13 */
 1500         movq    %r14,40(%rdi)                   /* save r14 */
 1501         movq    %r15,48(%rdi)                   /* save r15 */
 1502         movq    0(%rsp),%rdx                    /* get rta */
 1503         movq    %rdx,56(%rdi)                   /* save rip */
 1504         xorl    %eax,%eax                       /* return(0); */
 1505         ret
 1506 END(setjmp)
 1507 
 1508 ENTRY(longjmp)
 1509         movq    0(%rdi),%rbx                    /* restore rbx */
 1510         movq    8(%rdi),%rsp                    /* restore rsp */
 1511         movq    16(%rdi),%rbp                   /* restore rbp */
 1512         movq    24(%rdi),%r12                   /* restore r12 */
 1513         movq    32(%rdi),%r13                   /* restore r13 */
 1514         movq    40(%rdi),%r14                   /* restore r14 */
 1515         movq    48(%rdi),%r15                   /* restore r15 */
 1516         movq    56(%rdi),%rdx                   /* get rta */
 1517         movq    %rdx,0(%rsp)                    /* put in return frame */
 1518         xorl    %eax,%eax                       /* return(1); */
 1519         incl    %eax
 1520         ret
 1521 END(longjmp)
 1522 
 1523 /*
 1524  * Support for reading MSRs in the safe manner.  (Instead of panic on #gp,
 1525  * return an error.)
 1526  */
 1527 ENTRY(rdmsr_safe)
 1528 /* int rdmsr_safe(u_int msr, uint64_t *data) */
 1529         PUSH_FRAME_POINTER
 1530         movq    PCPU(CURPCB),%r8
 1531         movq    $msr_onfault,PCB_ONFAULT(%r8)
 1532         movl    %edi,%ecx
 1533         rdmsr                   /* Read MSR pointed by %ecx. Returns
 1534                                    hi byte in edx, lo in %eax */
 1535         salq    $32,%rdx        /* sign-shift %rdx left */
 1536         movl    %eax,%eax       /* zero-extend %eax -> %rax */
 1537         orq     %rdx,%rax
 1538         movq    %rax,(%rsi)
 1539         xorq    %rax,%rax
 1540         movq    %rax,PCB_ONFAULT(%r8)
 1541         POP_FRAME_POINTER
 1542         ret
 1543 
 1544 /*
 1545  * Support for writing MSRs in the safe manner.  (Instead of panic on #gp,
 1546  * return an error.)
 1547  */
 1548 ENTRY(wrmsr_safe)
 1549 /* int wrmsr_safe(u_int msr, uint64_t data) */
 1550         PUSH_FRAME_POINTER
 1551         movq    PCPU(CURPCB),%r8
 1552         movq    $msr_onfault,PCB_ONFAULT(%r8)
 1553         movl    %edi,%ecx
 1554         movl    %esi,%eax
 1555         sarq    $32,%rsi
 1556         movl    %esi,%edx
 1557         wrmsr                   /* Write MSR pointed by %ecx. Accepts
 1558                                    hi byte in edx, lo in %eax. */
 1559         xorq    %rax,%rax
 1560         movq    %rax,PCB_ONFAULT(%r8)
 1561         POP_FRAME_POINTER
 1562         ret
 1563 
 1564 /*
 1565  * MSR operations fault handler
 1566  */
 1567         ALIGN_TEXT
 1568 msr_onfault:
 1569         movq    $0,PCB_ONFAULT(%r8)
 1570         movl    $EFAULT,%eax
 1571         POP_FRAME_POINTER
 1572         ret
 1573 
 1574 /*
 1575  * void pmap_pti_pcid_invalidate(uint64_t ucr3, uint64_t kcr3);
 1576  * Invalidates address space addressed by ucr3, then returns to kcr3.
 1577  * Done in assembler to ensure no other memory accesses happen while
 1578  * on ucr3.
 1579  */
 1580         ALIGN_TEXT
 1581 ENTRY(pmap_pti_pcid_invalidate)
 1582         pushfq
 1583         cli
 1584         movq    %rdi,%cr3       /* to user page table */
 1585         movq    %rsi,%cr3       /* back to kernel */
 1586         popfq
 1587         retq
 1588 
 1589 /*
 1590  * void pmap_pti_pcid_invlpg(uint64_t ucr3, uint64_t kcr3, vm_offset_t va);
 1591  * Invalidates virtual address va in address space ucr3, then returns to kcr3.
 1592  */
 1593         ALIGN_TEXT
 1594 ENTRY(pmap_pti_pcid_invlpg)
 1595         pushfq
 1596         cli
 1597         movq    %rdi,%cr3       /* to user page table */
 1598         invlpg  (%rdx)
 1599         movq    %rsi,%cr3       /* back to kernel */
 1600         popfq
 1601         retq
 1602 
 1603 /*
 1604  * void pmap_pti_pcid_invlrng(uint64_t ucr3, uint64_t kcr3, vm_offset_t sva,
 1605  *     vm_offset_t eva);
 1606  * Invalidates virtual addresses between sva and eva in address space ucr3,
 1607  * then returns to kcr3.
 1608  */
 1609         ALIGN_TEXT
 1610 ENTRY(pmap_pti_pcid_invlrng)
 1611         pushfq
 1612         cli
 1613         movq    %rdi,%cr3       /* to user page table */
 1614 1:      invlpg  (%rdx)
 1615         addq    $PAGE_SIZE,%rdx
 1616         cmpq    %rdx,%rcx
 1617         ja      1b
 1618         movq    %rsi,%cr3       /* back to kernel */
 1619         popfq
 1620         retq
 1621 
 1622         .altmacro
 1623         .macro  rsb_seq_label l
 1624 rsb_seq_\l:
 1625         .endm
 1626         .macro  rsb_call_label l
 1627         call    rsb_seq_\l
 1628         .endm
 1629         .macro  rsb_seq count
 1630         ll=1
 1631         .rept   \count
 1632         rsb_call_label  %(ll)
 1633         nop
 1634         rsb_seq_label %(ll)
 1635         addq    $8,%rsp
 1636         ll=ll+1
 1637         .endr
 1638         .endm
 1639 
 1640 ENTRY(rsb_flush)
 1641         rsb_seq 32
 1642         ret
 1643 
 1644 /* all callers already saved %rax, %rdx, and %rcx */
 1645 ENTRY(handle_ibrs_entry)
 1646         cmpb    $0,hw_ibrs_ibpb_active(%rip)
 1647         je      1f
 1648         movl    $MSR_IA32_SPEC_CTRL,%ecx
 1649         rdmsr
 1650         orl     $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
 1651         orl     $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32,%edx
 1652         wrmsr
 1653         movb    $1,PCPU(IBPB_SET)
 1654         testl   $CPUID_STDEXT_SMEP,cpu_stdext_feature(%rip)
 1655         je      rsb_flush
 1656 1:      ret
 1657 END(handle_ibrs_entry)
 1658 
 1659 ENTRY(handle_ibrs_exit)
 1660         cmpb    $0,PCPU(IBPB_SET)
 1661         je      1f
 1662         movl    $MSR_IA32_SPEC_CTRL,%ecx
 1663         rdmsr
 1664         andl    $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
 1665         andl    $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
 1666         wrmsr
 1667         movb    $0,PCPU(IBPB_SET)
 1668 1:      ret
 1669 END(handle_ibrs_exit)
 1670 
 1671 /* registers-neutral version, but needs stack */
 1672 ENTRY(handle_ibrs_exit_rs)
 1673         cmpb    $0,PCPU(IBPB_SET)
 1674         je      1f
 1675         pushq   %rax
 1676         pushq   %rdx
 1677         pushq   %rcx
 1678         movl    $MSR_IA32_SPEC_CTRL,%ecx
 1679         rdmsr
 1680         andl    $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
 1681         andl    $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
 1682         wrmsr
 1683         popq    %rcx
 1684         popq    %rdx
 1685         popq    %rax
 1686         movb    $0,PCPU(IBPB_SET)
 1687 1:      ret
 1688 END(handle_ibrs_exit_rs)
 1689 
 1690         .noaltmacro
 1691 
 1692 /*
 1693  * Flush L1D cache.  Load enough of the data from the kernel text
 1694  * to flush existing L1D content.
 1695  *
 1696  * N.B. The function does not follow ABI calling conventions, it corrupts %rbx.
 1697  * The vmm.ko caller expects that only %rax, %rdx, %rbx, %rcx, %r9, and %rflags
 1698  * registers are clobbered.  The NMI handler caller only needs %r13 preserved.
 1699  */
 1700 ENTRY(flush_l1d_sw)
 1701 #define L1D_FLUSH_SIZE  (64 * 1024)
 1702         movq    $KERNBASE, %r9
 1703         movq    $-L1D_FLUSH_SIZE, %rcx
 1704         /*
 1705          * pass 1: Preload TLB.
 1706          * Kernel text is mapped using superpages.  TLB preload is
 1707          * done for the benefit of older CPUs which split 2M page
 1708          * into 4k TLB entries.
 1709          */
 1710 1:      movb    L1D_FLUSH_SIZE(%r9, %rcx), %al
 1711         addq    $PAGE_SIZE, %rcx
 1712         jne     1b
 1713         xorl    %eax, %eax
 1714         cpuid
 1715         movq    $-L1D_FLUSH_SIZE, %rcx
 1716         /* pass 2: Read each cache line. */
 1717 2:      movb    L1D_FLUSH_SIZE(%r9, %rcx), %al
 1718         addq    $64, %rcx
 1719         jne     2b
 1720         lfence
 1721         ret
 1722 #undef  L1D_FLUSH_SIZE
 1723 END(flush_l1d_sw)
 1724 
 1725 ENTRY(flush_l1d_sw_abi)
 1726         pushq   %rbx
 1727         call    flush_l1d_sw
 1728         popq    %rbx
 1729         ret
 1730 END(flush_l1d_sw_abi)
 1731 
 1732 ENTRY(mds_handler_void)
 1733         retq
 1734 END(mds_handler_void)
 1735 
 1736 ENTRY(mds_handler_verw)
 1737         subq    $8, %rsp
 1738         movw    %ds, (%rsp)
 1739         verw    (%rsp)
 1740         addq    $8, %rsp
 1741         retq
 1742 END(mds_handler_verw)
 1743 
 1744 ENTRY(mds_handler_ivb)
 1745         pushq   %rax
 1746         pushq   %rdx
 1747         pushq   %rcx
 1748 
 1749         movq    %cr0, %rax
 1750         testb   $CR0_TS, %al
 1751         je      1f
 1752         clts
 1753 1:      movq    PCPU(MDS_BUF), %rdx
 1754         movdqa  %xmm0, PCPU(MDS_TMP)
 1755         pxor    %xmm0, %xmm0
 1756 
 1757         lfence
 1758         orpd    (%rdx), %xmm0
 1759         orpd    (%rdx), %xmm0
 1760         mfence
 1761         movl    $40, %ecx
 1762         addq    $16, %rdx
 1763 2:      movntdq %xmm0, (%rdx)
 1764         addq    $16, %rdx
 1765         decl    %ecx
 1766         jnz     2b
 1767         mfence
 1768 
 1769         movdqa  PCPU(MDS_TMP),%xmm0
 1770         testb   $CR0_TS, %al
 1771         je      3f
 1772         movq    %rax, %cr0
 1773 3:      popq    %rcx
 1774         popq    %rdx
 1775         popq    %rax
 1776         retq
 1777 END(mds_handler_ivb)
 1778 
 1779 ENTRY(mds_handler_bdw)
 1780         pushq   %rax
 1781         pushq   %rbx
 1782         pushq   %rcx
 1783         pushq   %rdi
 1784         pushq   %rsi
 1785 
 1786         movq    %cr0, %rax
 1787         testb   $CR0_TS, %al
 1788         je      1f
 1789         clts
 1790 1:      movq    PCPU(MDS_BUF), %rbx
 1791         movdqa  %xmm0, PCPU(MDS_TMP)
 1792         pxor    %xmm0, %xmm0
 1793 
 1794         movq    %rbx, %rdi
 1795         movq    %rbx, %rsi
 1796         movl    $40, %ecx
 1797 2:      movntdq %xmm0, (%rbx)
 1798         addq    $16, %rbx
 1799         decl    %ecx
 1800         jnz     2b
 1801         mfence
 1802         movl    $1536, %ecx
 1803         rep; movsb
 1804         lfence
 1805 
 1806         movdqa  PCPU(MDS_TMP),%xmm0
 1807         testb   $CR0_TS, %al
 1808         je      3f
 1809         movq    %rax, %cr0
 1810 3:      popq    %rsi
 1811         popq    %rdi
 1812         popq    %rcx
 1813         popq    %rbx
 1814         popq    %rax
 1815         retq
 1816 END(mds_handler_bdw)
 1817 
 1818 ENTRY(mds_handler_skl_sse)
 1819         pushq   %rax
 1820         pushq   %rdx
 1821         pushq   %rcx
 1822         pushq   %rdi
 1823 
 1824         movq    %cr0, %rax
 1825         testb   $CR0_TS, %al
 1826         je      1f
 1827         clts
 1828 1:      movq    PCPU(MDS_BUF), %rdi
 1829         movq    PCPU(MDS_BUF64), %rdx
 1830         movdqa  %xmm0, PCPU(MDS_TMP)
 1831         pxor    %xmm0, %xmm0
 1832 
 1833         lfence
 1834         orpd    (%rdx), %xmm0
 1835         orpd    (%rdx), %xmm0
 1836         xorl    %eax, %eax
 1837 2:      clflushopt      5376(%rdi, %rax, 8)
 1838         addl    $8, %eax
 1839         cmpl    $8 * 12, %eax
 1840         jb      2b
 1841         sfence
 1842         movl    $6144, %ecx
 1843         xorl    %eax, %eax
 1844         rep; stosb
 1845         mfence
 1846 
 1847         movdqa  PCPU(MDS_TMP), %xmm0
 1848         testb   $CR0_TS, %al
 1849         je      3f
 1850         movq    %rax, %cr0
 1851 3:      popq    %rdi
 1852         popq    %rcx
 1853         popq    %rdx
 1854         popq    %rax
 1855         retq
 1856 END(mds_handler_skl_sse)
 1857 
 1858 ENTRY(mds_handler_skl_avx)
 1859         pushq   %rax
 1860         pushq   %rdx
 1861         pushq   %rcx
 1862         pushq   %rdi
 1863 
 1864         movq    %cr0, %rax
 1865         testb   $CR0_TS, %al
 1866         je      1f
 1867         clts
 1868 1:      movq    PCPU(MDS_BUF), %rdi
 1869         movq    PCPU(MDS_BUF64), %rdx
 1870         vmovdqa %ymm0, PCPU(MDS_TMP)
 1871         vpxor   %ymm0, %ymm0, %ymm0
 1872 
 1873         lfence
 1874         vorpd   (%rdx), %ymm0, %ymm0
 1875         vorpd   (%rdx), %ymm0, %ymm0
 1876         xorl    %eax, %eax
 1877 2:      clflushopt      5376(%rdi, %rax, 8)
 1878         addl    $8, %eax
 1879         cmpl    $8 * 12, %eax
 1880         jb      2b
 1881         sfence
 1882         movl    $6144, %ecx
 1883         xorl    %eax, %eax
 1884         rep; stosb
 1885         mfence
 1886 
 1887         vmovdqa PCPU(MDS_TMP), %ymm0
 1888         testb   $CR0_TS, %al
 1889         je      3f
 1890         movq    %rax, %cr0
 1891 3:      popq    %rdi
 1892         popq    %rcx
 1893         popq    %rdx
 1894         popq    %rax
 1895         retq
 1896 END(mds_handler_skl_avx)
 1897 
 1898 ENTRY(mds_handler_skl_avx512)
 1899         pushq   %rax
 1900         pushq   %rdx
 1901         pushq   %rcx
 1902         pushq   %rdi
 1903 
 1904         movq    %cr0, %rax
 1905         testb   $CR0_TS, %al
 1906         je      1f
 1907         clts
 1908 1:      movq    PCPU(MDS_BUF), %rdi
 1909         movq    PCPU(MDS_BUF64), %rdx
 1910         vmovdqa64       %zmm0, PCPU(MDS_TMP)
 1911         vpxord  %zmm0, %zmm0, %zmm0
 1912 
 1913         lfence
 1914         vorpd   (%rdx), %zmm0, %zmm0
 1915         vorpd   (%rdx), %zmm0, %zmm0
 1916         xorl    %eax, %eax
 1917 2:      clflushopt      5376(%rdi, %rax, 8)
 1918         addl    $8, %eax
 1919         cmpl    $8 * 12, %eax
 1920         jb      2b
 1921         sfence
 1922         movl    $6144, %ecx
 1923         xorl    %eax, %eax
 1924         rep; stosb
 1925         mfence
 1926 
 1927         vmovdqa64       PCPU(MDS_TMP), %zmm0
 1928         testb   $CR0_TS, %al
 1929         je      3f
 1930         movq    %rax, %cr0
 1931 3:      popq    %rdi
 1932         popq    %rcx
 1933         popq    %rdx
 1934         popq    %rax
 1935         retq
 1936 END(mds_handler_skl_avx512)
 1937 
 1938 ENTRY(mds_handler_silvermont)
 1939         pushq   %rax
 1940         pushq   %rdx
 1941         pushq   %rcx
 1942 
 1943         movq    %cr0, %rax
 1944         testb   $CR0_TS, %al
 1945         je      1f
 1946         clts
 1947 1:      movq    PCPU(MDS_BUF), %rdx
 1948         movdqa  %xmm0, PCPU(MDS_TMP)
 1949         pxor    %xmm0, %xmm0
 1950 
 1951         movl    $16, %ecx
 1952 2:      movntdq %xmm0, (%rdx)
 1953         addq    $16, %rdx
 1954         decl    %ecx
 1955         jnz     2b
 1956         mfence
 1957 
 1958         movdqa  PCPU(MDS_TMP),%xmm0
 1959         testb   $CR0_TS, %al
 1960         je      3f
 1961         movq    %rax, %cr0
 1962 3:      popq    %rcx
 1963         popq    %rdx
 1964         popq    %rax
 1965         retq
 1966 END(mds_handler_silvermont)

Cache object: 8362f0002cfdc967c1283619e96681b0


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.