The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/exception.S

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1989, 1990 William F. Jolitz.
    3  * Copyright (c) 1990 The Regents of the University of California.
    4  * Copyright (c) 2007-2018 The FreeBSD Foundation
    5  * All rights reserved.
    6  *
    7  * Portions of this software were developed by A. Joseph Koshy under
    8  * sponsorship from the FreeBSD Foundation and Google, Inc.
    9  *
   10  * Portions of this software were developed by
   11  * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
   12  * the FreeBSD Foundation.
   13  *
   14  * Redistribution and use in source and binary forms, with or without
   15  * modification, are permitted provided that the following conditions
   16  * are met:
   17  * 1. Redistributions of source code must retain the above copyright
   18  *    notice, this list of conditions and the following disclaimer.
   19  * 2. Redistributions in binary form must reproduce the above copyright
   20  *    notice, this list of conditions and the following disclaimer in the
   21  *    documentation and/or other materials provided with the distribution.
   22  * 4. Neither the name of the University nor the names of its contributors
   23  *    may be used to endorse or promote products derived from this software
   24  *    without specific prior written permission.
   25  *
   26  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   27  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   28  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   29  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   30  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   35  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   36  * SUCH DAMAGE.
   37  *
   38  * $FreeBSD: releng/11.1/sys/amd64/amd64/exception.S 333371 2018-05-08 17:12:10Z gordon $
   39  */
   40 
   41 #include "opt_atpic.h"
   42 #include "opt_compat.h"
   43 #include "opt_hwpmc_hooks.h"
   44 
   45 #include "assym.s"
   46 
   47 #include <machine/asmacros.h>
   48 #include <machine/psl.h>
   49 #include <machine/trap.h>
   50 #include <machine/specialreg.h>
   51 
   52 #ifdef KDTRACE_HOOKS
   53         .bss
   54         .globl  dtrace_invop_jump_addr
   55         .align  8
   56         .type   dtrace_invop_jump_addr,@object
   57         .size   dtrace_invop_jump_addr,8
   58 dtrace_invop_jump_addr:
   59         .zero   8
   60         .globl  dtrace_invop_calltrap_addr
   61         .align  8
   62         .type   dtrace_invop_calltrap_addr,@object
   63         .size   dtrace_invop_calltrap_addr,8
   64 dtrace_invop_calltrap_addr:
   65         .zero   8
   66 #endif
   67         .text
   68 #ifdef HWPMC_HOOKS
   69         ENTRY(start_exceptions)
   70 #endif
   71 
   72 /*****************************************************************************/
   73 /* Trap handling                                                             */
   74 /*****************************************************************************/
   75 /*
   76  * Trap and fault vector routines.
   77  *
   78  * All traps are 'interrupt gates', SDT_SYSIGT.  An interrupt gate pushes
   79  * state on the stack but also disables interrupts.  This is important for
   80  * us for the use of the swapgs instruction.  We cannot be interrupted
   81  * until the GS.base value is correct.  For most traps, we automatically
   82  * then enable interrupts if the interrupted context had them enabled.
   83  * This is equivalent to the i386 port's use of SDT_SYS386TGT.
   84  *
   85  * The cpu will push a certain amount of state onto the kernel stack for
   86  * the current process.  See amd64/include/frame.h.
   87  * This includes the current RFLAGS (status register, which includes
   88  * the interrupt disable state prior to the trap), the code segment register,
   89  * and the return instruction pointer are pushed by the cpu.  The cpu
   90  * will also push an 'error' code for certain traps.  We push a dummy
   91  * error code for those traps where the cpu doesn't in order to maintain
   92  * a consistent frame.  We also push a contrived 'trap number'.
   93  *
   94  * The CPU does not push the general registers, so we must do that, and we
   95  * must restore them prior to calling 'iret'.  The CPU adjusts %cs and %ss
   96  * but does not mess with %ds, %es, %gs or %fs.  We swap the %gs base for
   97  * for the kernel mode operation shortly, without changes to the selector
   98  * loaded.  Since superuser long mode works with any selectors loaded into
   99  * segment registers other then %cs, which makes them mostly unused in long
  100  * mode, and kernel does not reference %fs, leave them alone.  The segment
  101  * registers are reloaded on return to the usermode.
  102  */
  103 
  104 MCOUNT_LABEL(user)
  105 MCOUNT_LABEL(btrap)
  106 
  107 /* Traps that we leave interrupts disabled for. */
  108         .macro  TRAP_NOEN       l, trapno
  109         PTI_ENTRY       \l,X\l
  110         .globl  X\l
  111         .type   X\l,@function
  112 X\l:    subq $TF_RIP,%rsp
  113         movl $\trapno,TF_TRAPNO(%rsp)
  114         movq $0,TF_ADDR(%rsp)
  115         movq $0,TF_ERR(%rsp)
  116         jmp alltraps_noen
  117         .endm
  118 
  119         TRAP_NOEN       bpt, T_BPTFLT
  120 #ifdef KDTRACE_HOOKS
  121         TRAP_NOEN       dtrace_ret, T_DTRACE_RET
  122 #endif
  123 
  124 /* Regular traps; The cpu does not supply tf_err for these. */
  125         .macro  TRAP    l, trapno
  126         PTI_ENTRY       \l,X\l
  127         .globl  X\l
  128         .type   X\l,@function
  129 X\l:
  130         subq $TF_RIP,%rsp
  131         movl $\trapno,TF_TRAPNO(%rsp)
  132         movq $0,TF_ADDR(%rsp)
  133         movq $0,TF_ERR(%rsp)
  134         jmp alltraps
  135         .endm
  136 
  137         TRAP    div, T_DIVIDE
  138         TRAP    ofl, T_OFLOW
  139         TRAP    bnd, T_BOUND
  140         TRAP    ill, T_PRIVINFLT
  141         TRAP    dna, T_DNA
  142         TRAP    fpusegm, T_FPOPFLT
  143         TRAP    rsvd, T_RESERVED
  144         TRAP    fpu, T_ARITHTRAP
  145         TRAP    xmm, T_XMMFLT
  146 
  147 /* This group of traps have tf_err already pushed by the cpu. */
  148         .macro  TRAP_ERR        l, trapno
  149         PTI_ENTRY       \l,X\l,has_err=1
  150         .globl  X\l
  151         .type   X\l,@function
  152 X\l:
  153         subq $TF_ERR,%rsp
  154         movl $\trapno,TF_TRAPNO(%rsp)
  155         movq $0,TF_ADDR(%rsp)
  156         jmp alltraps
  157         .endm
  158 
  159         TRAP_ERR        tss, T_TSSFLT
  160         TRAP_ERR        align, T_ALIGNFLT
  161 
  162         /*
  163          * alltraps entry point.  Use swapgs if this is the first time in the
  164          * kernel from userland.  Reenable interrupts if they were enabled
  165          * before the trap.  This approximates SDT_SYS386TGT on the i386 port.
  166          */
  167         SUPERALIGN_TEXT
  168         .globl  alltraps
  169         .type   alltraps,@function
  170 alltraps:
  171         movq    %rdi,TF_RDI(%rsp)
  172         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  173         jz      1f              /* already running with kernel GS.base */
  174         swapgs
  175         movq    PCPU(CURPCB),%rdi
  176         andl    $~PCB_FULL_IRET,PCB_FLAGS(%rdi)
  177 1:      SAVE_SEGS
  178         movq    %rdx,TF_RDX(%rsp)
  179         movq    %rax,TF_RAX(%rsp)
  180         movq    %rcx,TF_RCX(%rsp)
  181         testb   $SEL_RPL_MASK,TF_CS(%rsp)
  182         jz      2f
  183         call    handle_ibrs_entry
  184 2:      testl   $PSL_I,TF_RFLAGS(%rsp)
  185         jz      alltraps_pushregs_no_rax
  186         sti
  187 alltraps_pushregs_no_rax:
  188         movq    %rsi,TF_RSI(%rsp)
  189         movq    %r8,TF_R8(%rsp)
  190         movq    %r9,TF_R9(%rsp)
  191         movq    %rbx,TF_RBX(%rsp)
  192         movq    %rbp,TF_RBP(%rsp)
  193         movq    %r10,TF_R10(%rsp)
  194         movq    %r11,TF_R11(%rsp)
  195         movq    %r12,TF_R12(%rsp)
  196         movq    %r13,TF_R13(%rsp)
  197         movq    %r14,TF_R14(%rsp)
  198         movq    %r15,TF_R15(%rsp)
  199         movl    $TF_HASSEGS,TF_FLAGS(%rsp)
  200         cld
  201         FAKE_MCOUNT(TF_RIP(%rsp))
  202 #ifdef KDTRACE_HOOKS
  203         /*
  204          * DTrace Function Boundary Trace (fbt) probes are triggered
  205          * by int3 (0xcc) which causes the #BP (T_BPTFLT) breakpoint
  206          * interrupt. For all other trap types, just handle them in
  207          * the usual way.
  208          */
  209         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  210         jnz     calltrap                /* ignore userland traps */
  211         cmpl    $T_BPTFLT,TF_TRAPNO(%rsp)
  212         jne     calltrap
  213 
  214         /* Check if there is no DTrace hook registered. */
  215         cmpq    $0,dtrace_invop_jump_addr
  216         je      calltrap
  217 
  218         /*
  219          * Set our jump address for the jump back in the event that
  220          * the breakpoint wasn't caused by DTrace at all.
  221          */
  222         movq    $calltrap,dtrace_invop_calltrap_addr(%rip)
  223 
  224         /* Jump to the code hooked in by DTrace. */
  225         jmpq    *dtrace_invop_jump_addr
  226 #endif
  227         .globl  calltrap
  228         .type   calltrap,@function
  229 calltrap:
  230         movq    %rsp,%rdi
  231         call    trap_check
  232         MEXITCOUNT
  233         jmp     doreti                  /* Handle any pending ASTs */
  234 
  235         /*
  236          * alltraps_noen entry point.  Unlike alltraps above, we want to
  237          * leave the interrupts disabled.  This corresponds to
  238          * SDT_SYS386IGT on the i386 port.
  239          */
  240         SUPERALIGN_TEXT
  241         .globl  alltraps_noen
  242         .type   alltraps_noen,@function
  243 alltraps_noen:
  244         movq    %rdi,TF_RDI(%rsp)
  245         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  246         jz      1f /* already running with kernel GS.base */
  247         swapgs
  248         movq    PCPU(CURPCB),%rdi
  249         andl    $~PCB_FULL_IRET,PCB_FLAGS(%rdi)
  250 1:      SAVE_SEGS
  251         movq    %rdx,TF_RDX(%rsp)
  252         movq    %rax,TF_RAX(%rsp)
  253         movq    %rcx,TF_RCX(%rsp)
  254         testb   $SEL_RPL_MASK,TF_CS(%rsp)
  255         jz      alltraps_pushregs_no_rax
  256         call    handle_ibrs_entry
  257         jmp     alltraps_pushregs_no_rax
  258 
  259 IDTVEC(dblfault)
  260         subq    $TF_ERR,%rsp
  261         movl    $T_DOUBLEFLT,TF_TRAPNO(%rsp)
  262         movq    $0,TF_ADDR(%rsp)
  263         movq    $0,TF_ERR(%rsp)
  264         movq    %rdi,TF_RDI(%rsp)
  265         movq    %rsi,TF_RSI(%rsp)
  266         movq    %rdx,TF_RDX(%rsp)
  267         movq    %rcx,TF_RCX(%rsp)
  268         movq    %r8,TF_R8(%rsp)
  269         movq    %r9,TF_R9(%rsp)
  270         movq    %rax,TF_RAX(%rsp)
  271         movq    %rbx,TF_RBX(%rsp)
  272         movq    %rbp,TF_RBP(%rsp)
  273         movq    %r10,TF_R10(%rsp)
  274         movq    %r11,TF_R11(%rsp)
  275         movq    %r12,TF_R12(%rsp)
  276         movq    %r13,TF_R13(%rsp)
  277         movq    %r14,TF_R14(%rsp)
  278         movq    %r15,TF_R15(%rsp)
  279         SAVE_SEGS
  280         movl    $TF_HASSEGS,TF_FLAGS(%rsp)
  281         cld
  282         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  283         jz      1f                      /* already running with kernel GS.base */
  284         swapgs
  285 1:
  286         movq    PCPU(KCR3),%rax
  287         cmpq    $~0,%rax
  288         je      2f
  289         movq    %rax,%cr3
  290 2:      movq    %rsp,%rdi
  291         call    dblfault_handler
  292 3:      hlt
  293         jmp     3b
  294 
  295         ALIGN_TEXT
  296 IDTVEC(page_pti)
  297         testb   $SEL_RPL_MASK,PTI_CS-2*8(%rsp)
  298         jz      Xpage
  299         swapgs
  300         pushq   %rax
  301         pushq   %rdx
  302         movq    %cr3,%rax
  303         movq    %rax,PCPU(SAVED_UCR3)
  304         PTI_UUENTRY has_err=1
  305         subq    $TF_ERR,%rsp
  306         movq    %rdi,TF_RDI(%rsp)
  307         movq    %rax,TF_RAX(%rsp)
  308         movq    %rdx,TF_RDX(%rsp)
  309         movq    %rcx,TF_RCX(%rsp)
  310         jmp     page_u
  311 IDTVEC(page)
  312         subq    $TF_ERR,%rsp
  313         movq    %rdi,TF_RDI(%rsp)       /* free up GP registers */
  314         movq    %rax,TF_RAX(%rsp)
  315         movq    %rdx,TF_RDX(%rsp)
  316         movq    %rcx,TF_RCX(%rsp)
  317         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  318         jz      page_cr2                /* already running with kernel GS.base */
  319         swapgs
  320 page_u: movq    PCPU(CURPCB),%rdi
  321         andl    $~PCB_FULL_IRET,PCB_FLAGS(%rdi)
  322         movq    PCPU(SAVED_UCR3),%rax
  323         movq    %rax,PCB_SAVED_UCR3(%rdi)
  324         call    handle_ibrs_entry
  325 page_cr2:
  326         movq    %cr2,%rdi               /* preserve %cr2 before ..  */
  327         movq    %rdi,TF_ADDR(%rsp)      /* enabling interrupts. */
  328         SAVE_SEGS
  329         movl    $T_PAGEFLT,TF_TRAPNO(%rsp)
  330         testl   $PSL_I,TF_RFLAGS(%rsp)
  331         jz      alltraps_pushregs_no_rax
  332         sti
  333         jmp     alltraps_pushregs_no_rax
  334 
  335         /*
  336          * We have to special-case this one.  If we get a trap in doreti() at
  337          * the iretq stage, we'll reenter with the wrong gs state.  We'll have
  338          * to do a special the swapgs in this case even coming from the kernel.
  339          * XXX linux has a trap handler for their equivalent of load_gs().
  340          *
  341          * On the stack, we have the hardware interrupt frame to return
  342          * to usermode (faulted) and another frame with error code, for
  343          * fault.  For PTI, copy both frames to the main thread stack.
  344          */
  345         .macro PROTF_ENTRY name,trapno
  346 \name\()_pti_doreti:
  347         pushq   %rax
  348         pushq   %rdx
  349         swapgs
  350         movq    PCPU(KCR3),%rax
  351         movq    %rax,%cr3
  352         movq    PCPU(RSP0),%rax
  353         subq    $2*PTI_SIZE-3*8,%rax /* no err, %rax, %rdx in faulted frame */
  354         MOVE_STACKS     (PTI_SIZE / 4 - 3)
  355         movq    %rax,%rsp
  356         popq    %rdx
  357         popq    %rax
  358         swapgs
  359         jmp     X\name
  360 IDTVEC(\name\()_pti)
  361         cmpq    $doreti_iret,PTI_RIP-2*8(%rsp)
  362         je      \name\()_pti_doreti
  363         testb   $SEL_RPL_MASK,PTI_CS-2*8(%rsp) /* %rax, %rdx not yet pushed */
  364         jz      X\name
  365         PTI_UENTRY has_err=1
  366         swapgs
  367 IDTVEC(\name)
  368         subq    $TF_ERR,%rsp
  369         movl    $\trapno,TF_TRAPNO(%rsp)
  370         jmp     prot_addrf
  371         .endm
  372 
  373         PROTF_ENTRY     missing, T_SEGNPFLT
  374         PROTF_ENTRY     stk, T_STKFLT
  375         PROTF_ENTRY     prot, T_PROTFLT
  376 
  377 prot_addrf:
  378         movq    $0,TF_ADDR(%rsp)
  379         movq    %rdi,TF_RDI(%rsp)       /* free up a GP register */
  380         movq    %rax,TF_RAX(%rsp)
  381         movq    %rdx,TF_RDX(%rsp)
  382         movq    %rcx,TF_RCX(%rsp)
  383         movw    %fs,TF_FS(%rsp)
  384         movw    %gs,TF_GS(%rsp)
  385         leaq    doreti_iret(%rip),%rdi
  386         cmpq    %rdi,TF_RIP(%rsp)
  387         je      5f                      /* kernel but with user gsbase!! */
  388         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  389         jz      6f                      /* already running with kernel GS.base */
  390         swapgs
  391         movq    PCPU(CURPCB),%rdi
  392 4:      call    handle_ibrs_entry
  393         orl     $PCB_FULL_IRET,PCB_FLAGS(%rdi)  /* always full iret from GPF */
  394         movw    %es,TF_ES(%rsp)
  395         movw    %ds,TF_DS(%rsp)
  396         testl   $PSL_I,TF_RFLAGS(%rsp)
  397         jz      alltraps_pushregs_no_rax
  398         sti
  399         jmp     alltraps_pushregs_no_rax
  400 
  401 5:      swapgs
  402 6:      movq    PCPU(CURPCB),%rdi
  403         jmp     4b
  404 
  405 /*
  406  * Fast syscall entry point.  We enter here with just our new %cs/%ss set,
  407  * and the new privilige level.  We are still running on the old user stack
  408  * pointer.  We have to juggle a few things around to find our stack etc.
  409  * swapgs gives us access to our PCPU space only.
  410  *
  411  * We do not support invoking this from a custom %cs or %ss (e.g. using
  412  * entries from an LDT).
  413  */
  414         SUPERALIGN_TEXT
  415 IDTVEC(fast_syscall_pti)
  416         swapgs
  417         movq    %rax,PCPU(SCRATCH_RAX)
  418         movq    PCPU(KCR3),%rax
  419         movq    %rax,%cr3
  420         jmp     fast_syscall_common
  421         SUPERALIGN_TEXT
  422 IDTVEC(fast_syscall)
  423         swapgs
  424         movq    %rax,PCPU(SCRATCH_RAX)
  425 fast_syscall_common:
  426         movq    %rsp,PCPU(SCRATCH_RSP)
  427         movq    PCPU(RSP0),%rsp
  428         /* Now emulate a trapframe. Make the 8 byte alignment odd for call. */
  429         subq    $TF_SIZE,%rsp
  430         /* defer TF_RSP till we have a spare register */
  431         movq    %r11,TF_RFLAGS(%rsp)
  432         movq    %rcx,TF_RIP(%rsp)       /* %rcx original value is in %r10 */
  433         movq    PCPU(SCRATCH_RSP),%r11  /* %r11 already saved */
  434         movq    %r11,TF_RSP(%rsp)       /* user stack pointer */
  435         movq    PCPU(SCRATCH_RAX),%rax
  436         movq    %rax,TF_RAX(%rsp)       /* syscall number */
  437         movq    %rdx,TF_RDX(%rsp)       /* arg 3 */
  438         SAVE_SEGS
  439         call    handle_ibrs_entry
  440         movq    PCPU(CURPCB),%r11
  441         andl    $~PCB_FULL_IRET,PCB_FLAGS(%r11)
  442         sti
  443         movq    $KUDSEL,TF_SS(%rsp)
  444         movq    $KUCSEL,TF_CS(%rsp)
  445         movq    $2,TF_ERR(%rsp)
  446         movq    %rdi,TF_RDI(%rsp)       /* arg 1 */
  447         movq    %rsi,TF_RSI(%rsp)       /* arg 2 */
  448         movq    %r10,TF_RCX(%rsp)       /* arg 4 */
  449         movq    %r8,TF_R8(%rsp)         /* arg 5 */
  450         movq    %r9,TF_R9(%rsp)         /* arg 6 */
  451         movq    %rbx,TF_RBX(%rsp)       /* C preserved */
  452         movq    %rbp,TF_RBP(%rsp)       /* C preserved */
  453         movq    %r12,TF_R12(%rsp)       /* C preserved */
  454         movq    %r13,TF_R13(%rsp)       /* C preserved */
  455         movq    %r14,TF_R14(%rsp)       /* C preserved */
  456         movq    %r15,TF_R15(%rsp)       /* C preserved */
  457         movl    $TF_HASSEGS,TF_FLAGS(%rsp)
  458         cld
  459         FAKE_MCOUNT(TF_RIP(%rsp))
  460         movq    PCPU(CURTHREAD),%rdi
  461         movq    %rsp,TD_FRAME(%rdi)
  462         movl    TF_RFLAGS(%rsp),%esi
  463         andl    $PSL_T,%esi
  464         call    amd64_syscall
  465 1:      movq    PCPU(CURPCB),%rax
  466         /* Disable interrupts before testing PCB_FULL_IRET. */
  467         cli
  468         testl   $PCB_FULL_IRET,PCB_FLAGS(%rax)
  469         jnz     4f
  470         /* Check for and handle AST's on return to userland. */
  471         movq    PCPU(CURTHREAD),%rax
  472         testl   $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax)
  473         jne     3f
  474         call    handle_ibrs_exit
  475         /* Restore preserved registers. */
  476         MEXITCOUNT
  477         movq    TF_RDI(%rsp),%rdi       /* bonus; preserve arg 1 */
  478         movq    TF_RSI(%rsp),%rsi       /* bonus: preserve arg 2 */
  479         movq    TF_RDX(%rsp),%rdx       /* return value 2 */
  480         movq    TF_RAX(%rsp),%rax       /* return value 1 */
  481         movq    TF_RFLAGS(%rsp),%r11    /* original %rflags */
  482         movq    TF_RIP(%rsp),%rcx       /* original %rip */
  483         movq    TF_RSP(%rsp),%rsp       /* user stack pointer */
  484         cmpb    $0,pti
  485         je      2f
  486         movq    PCPU(UCR3),%r9
  487         movq    %r9,%cr3
  488         xorl    %r9d,%r9d
  489 2:      swapgs
  490         sysretq
  491 
  492 3:      /* AST scheduled. */
  493         sti
  494         movq    %rsp,%rdi
  495         call    ast
  496         jmp     1b
  497 
  498 4:      /* Requested full context restore, use doreti for that. */
  499         MEXITCOUNT
  500         jmp     doreti
  501 
  502 /*
  503  * Here for CYA insurance, in case a "syscall" instruction gets
  504  * issued from 32 bit compatibility mode. MSR_CSTAR has to point
  505  * to *something* if EFER_SCE is enabled.
  506  */
  507 IDTVEC(fast_syscall32)
  508         sysret
  509 
  510 /*
  511  * DB# handler is very similar to NM#, because 'mov/pop %ss' delay
  512  * generation of exception until the next instruction is executed,
  513  * which might be a kernel entry.  So we must execute the handler
  514  * on IST stack and be ready for non-kernel GSBASE.
  515  */
  516 IDTVEC(dbg)
  517         subq    $TF_RIP,%rsp
  518         movl    $(T_TRCTRAP),TF_TRAPNO(%rsp)
  519         movq    $0,TF_ADDR(%rsp)
  520         movq    $0,TF_ERR(%rsp)
  521         movq    %rdi,TF_RDI(%rsp)
  522         movq    %rsi,TF_RSI(%rsp)
  523         movq    %rdx,TF_RDX(%rsp)
  524         movq    %rcx,TF_RCX(%rsp)
  525         movq    %r8,TF_R8(%rsp)
  526         movq    %r9,TF_R9(%rsp)
  527         movq    %rax,TF_RAX(%rsp)
  528         movq    %rbx,TF_RBX(%rsp)
  529         movq    %rbp,TF_RBP(%rsp)
  530         movq    %r10,TF_R10(%rsp)
  531         movq    %r11,TF_R11(%rsp)
  532         movq    %r12,TF_R12(%rsp)
  533         movq    %r13,TF_R13(%rsp)
  534         movq    %r14,TF_R14(%rsp)
  535         movq    %r15,TF_R15(%rsp)
  536         SAVE_SEGS
  537         movl    $TF_HASSEGS,TF_FLAGS(%rsp)
  538         cld
  539         testb   $SEL_RPL_MASK,TF_CS(%rsp)
  540         jnz     dbg_fromuserspace
  541         /*
  542          * We've interrupted the kernel.  Preserve GS.base in %r12,
  543          * %cr3 in %r13, and possibly lower half of MSR_IA32_SPEC_CTL in %r14d.
  544          */
  545         movl    $MSR_GSBASE,%ecx
  546         rdmsr
  547         movq    %rax,%r12
  548         shlq    $32,%rdx
  549         orq     %rdx,%r12
  550         /* Retrieve and load the canonical value for GS.base. */
  551         movq    TF_SIZE(%rsp),%rdx
  552         movl    %edx,%eax
  553         shrq    $32,%rdx
  554         wrmsr
  555         movq    %cr3,%r13
  556         movq    PCPU(KCR3),%rax
  557         cmpq    $~0,%rax
  558         je      1f
  559         movq    %rax,%cr3
  560 1:      testl   $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip)
  561         je      2f
  562         movl    $MSR_IA32_SPEC_CTRL,%ecx
  563         rdmsr
  564         movl    %eax,%r14d
  565         call    handle_ibrs_entry
  566 2:      FAKE_MCOUNT(TF_RIP(%rsp))
  567         movq    %rsp,%rdi
  568         call    trap
  569         MEXITCOUNT
  570         testl   $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip)
  571         je      3f
  572         movl    %r14d,%eax
  573         xorl    %edx,%edx
  574         movl    $MSR_IA32_SPEC_CTRL,%ecx
  575         wrmsr
  576         /*
  577          * Put back the preserved MSR_GSBASE value.
  578          */
  579 3:      movl    $MSR_GSBASE,%ecx
  580         movq    %r12,%rdx
  581         movl    %edx,%eax
  582         shrq    $32,%rdx
  583         wrmsr
  584         movq    %r13,%cr3
  585         RESTORE_REGS
  586         addq    $TF_RIP,%rsp
  587         jmp     doreti_iret
  588 dbg_fromuserspace:
  589         /*
  590          * Switch to kernel GSBASE and kernel page table, and copy frame
  591          * from the IST stack to the normal kernel stack, since trap()
  592          * re-enables interrupts, and since we might trap on DB# while
  593          * in trap().
  594          */
  595         swapgs
  596         movq    PCPU(KCR3),%rax
  597         cmpq    $~0,%rax
  598         je      1f
  599         movq    %rax,%cr3
  600 1:      movq    PCPU(RSP0),%rax
  601         movl    $TF_SIZE,%ecx
  602         subq    %rcx,%rax
  603         movq    %rax,%rdi
  604         movq    %rsp,%rsi
  605         rep;movsb
  606         movq    %rax,%rsp
  607         call    handle_ibrs_entry
  608         movq    PCPU(CURPCB),%rdi
  609         orl     $PCB_FULL_IRET,PCB_FLAGS(%rdi)
  610         testb   $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip)
  611         jz      3f
  612         cmpw    $KUF32SEL,TF_FS(%rsp)
  613         jne     2f
  614         rdfsbase %rax
  615         movq    %rax,PCB_FSBASE(%rdi)
  616 2:      cmpw    $KUG32SEL,TF_GS(%rsp)
  617         jne     3f
  618         movl    $MSR_KGSBASE,%ecx
  619         rdmsr
  620         shlq    $32,%rdx
  621         orq     %rdx,%rax
  622         movq    %rax,PCB_GSBASE(%rdi)
  623 3:      jmp     calltrap
  624 
  625 /*
  626  * NMI handling is special.
  627  *
  628  * First, NMIs do not respect the state of the processor's RFLAGS.IF
  629  * bit.  The NMI handler may be entered at any time, including when
  630  * the processor is in a critical section with RFLAGS.IF == 0.
  631  * The processor's GS.base value could be invalid on entry to the
  632  * handler.
  633  *
  634  * Second, the processor treats NMIs specially, blocking further NMIs
  635  * until an 'iretq' instruction is executed.  We thus need to execute
  636  * the NMI handler with interrupts disabled, to prevent a nested interrupt
  637  * from executing an 'iretq' instruction and inadvertently taking the
  638  * processor out of NMI mode.
  639  *
  640  * Third, the NMI handler runs on its own stack (tss_ist2). The canonical
  641  * GS.base value for the processor is stored just above the bottom of its
  642  * NMI stack.  For NMIs taken from kernel mode, the current value in
  643  * the processor's GS.base is saved at entry to C-preserved register %r12,
  644  * the canonical value for GS.base is then loaded into the processor, and
  645  * the saved value is restored at exit time.  For NMIs taken from user mode,
  646  * the cheaper 'SWAPGS' instructions are used for swapping GS.base.
  647  */
  648 
  649 IDTVEC(nmi)
  650         subq    $TF_RIP,%rsp
  651         movl    $(T_NMI),TF_TRAPNO(%rsp)
  652         movq    $0,TF_ADDR(%rsp)
  653         movq    $0,TF_ERR(%rsp)
  654         movq    %rdi,TF_RDI(%rsp)
  655         movq    %rsi,TF_RSI(%rsp)
  656         movq    %rdx,TF_RDX(%rsp)
  657         movq    %rcx,TF_RCX(%rsp)
  658         movq    %r8,TF_R8(%rsp)
  659         movq    %r9,TF_R9(%rsp)
  660         movq    %rax,TF_RAX(%rsp)
  661         movq    %rbx,TF_RBX(%rsp)
  662         movq    %rbp,TF_RBP(%rsp)
  663         movq    %r10,TF_R10(%rsp)
  664         movq    %r11,TF_R11(%rsp)
  665         movq    %r12,TF_R12(%rsp)
  666         movq    %r13,TF_R13(%rsp)
  667         movq    %r14,TF_R14(%rsp)
  668         movq    %r15,TF_R15(%rsp)
  669         SAVE_SEGS
  670         movl    $TF_HASSEGS,TF_FLAGS(%rsp)
  671         cld
  672         xorl    %ebx,%ebx
  673         testb   $SEL_RPL_MASK,TF_CS(%rsp)
  674         jnz     nmi_fromuserspace
  675         /*
  676          * We've interrupted the kernel.  Preserve GS.base in %r12,
  677          * %cr3 in %r13, and possibly lower half of MSR_IA32_SPEC_CTL in %r14d.
  678          */
  679         movl    $MSR_GSBASE,%ecx
  680         rdmsr
  681         movq    %rax,%r12
  682         shlq    $32,%rdx
  683         orq     %rdx,%r12
  684         /* Retrieve and load the canonical value for GS.base. */
  685         movq    TF_SIZE(%rsp),%rdx
  686         movl    %edx,%eax
  687         shrq    $32,%rdx
  688         wrmsr
  689         movq    %cr3,%r13
  690         movq    PCPU(KCR3),%rax
  691         cmpq    $~0,%rax
  692         je      1f
  693         movq    %rax,%cr3
  694 1:      testl   $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip)
  695         je      nmi_calltrap
  696         movl    $MSR_IA32_SPEC_CTRL,%ecx
  697         rdmsr
  698         movl    %eax,%r14d
  699         call    handle_ibrs_entry
  700         jmp     nmi_calltrap
  701 nmi_fromuserspace:
  702         incl    %ebx
  703         swapgs
  704         movq    %cr3,%r13
  705         movq    PCPU(KCR3),%rax
  706         cmpq    $~0,%rax
  707         je      1f
  708         movq    %rax,%cr3
  709 1:      call    handle_ibrs_entry
  710         movq    PCPU(CURPCB),%rdi
  711         testq   %rdi,%rdi
  712         jz      3f
  713         orl     $PCB_FULL_IRET,PCB_FLAGS(%rdi)
  714 3:
  715 /* Note: this label is also used by ddb and gdb: */
  716 nmi_calltrap:
  717         FAKE_MCOUNT(TF_RIP(%rsp))
  718         movq    %rsp,%rdi
  719         call    trap
  720         MEXITCOUNT
  721 #ifdef HWPMC_HOOKS
  722         /*
  723          * Capture a userspace callchain if needed.
  724          *
  725          * - Check if the current trap was from user mode.
  726          * - Check if the current thread is valid.
  727          * - Check if the thread requires a user call chain to be
  728          *   captured.
  729          *
  730          * We are still in NMI mode at this point.
  731          */
  732         testl   %ebx,%ebx
  733         jz      nocallchain     /* not from userspace */
  734         movq    PCPU(CURTHREAD),%rax
  735         orq     %rax,%rax       /* curthread present? */
  736         jz      nocallchain
  737         /*
  738          * Move execution to the regular kernel stack, because we
  739          * committed to return through doreti.
  740          */
  741         movq    %rsp,%rsi       /* source stack pointer */
  742         movq    $TF_SIZE,%rcx
  743         movq    PCPU(RSP0),%rdx
  744         subq    %rcx,%rdx
  745         movq    %rdx,%rdi       /* destination stack pointer */
  746         shrq    $3,%rcx         /* trap frame size in long words */
  747         cld
  748         rep
  749         movsq                   /* copy trapframe */
  750         movq    %rdx,%rsp       /* we are on the regular kstack */
  751 
  752         testl   $TDP_CALLCHAIN,TD_PFLAGS(%rax) /* flagged for capture? */
  753         jz      nocallchain
  754         /*
  755          * A user callchain is to be captured, so:
  756          * - Take the processor out of "NMI" mode by faking an "iret",
  757          *   to allow for nested NMI interrupts.
  758          * - Enable interrupts, so that copyin() can work.
  759          */
  760         movl    %ss,%eax
  761         pushq   %rax            /* tf_ss */
  762         pushq   %rdx            /* tf_rsp (on kernel stack) */
  763         pushfq                  /* tf_rflags */
  764         movl    %cs,%eax
  765         pushq   %rax            /* tf_cs */
  766         pushq   $outofnmi       /* tf_rip */
  767         iretq
  768 outofnmi:
  769         /*
  770          * At this point the processor has exited NMI mode and is running
  771          * with interrupts turned off on the normal kernel stack.
  772          *
  773          * If a pending NMI gets recognized at or after this point, it
  774          * will cause a kernel callchain to be traced.
  775          *
  776          * We turn interrupts back on, and call the user callchain capture hook.
  777          */
  778         movq    pmc_hook,%rax
  779         orq     %rax,%rax
  780         jz      nocallchain
  781         movq    PCPU(CURTHREAD),%rdi            /* thread */
  782         movq    $PMC_FN_USER_CALLCHAIN,%rsi     /* command */
  783         movq    %rsp,%rdx                       /* frame */
  784         sti
  785         call    *%rax
  786         cli
  787 nocallchain:
  788 #endif
  789         testl   %ebx,%ebx       /* %ebx == 0 => return to userland */
  790         jnz     doreti_exit
  791         /*
  792          * Restore speculation control MSR, if preserved.
  793          */
  794         testl   $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip)
  795         je      1f
  796         movl    %r14d,%eax
  797         xorl    %edx,%edx
  798         movl    $MSR_IA32_SPEC_CTRL,%ecx
  799         wrmsr
  800         /*
  801          * Put back the preserved MSR_GSBASE value.
  802          */
  803 1:      movl    $MSR_GSBASE,%ecx
  804         movq    %r12,%rdx
  805         movl    %edx,%eax
  806         shrq    $32,%rdx
  807         wrmsr
  808         movq    %r13,%cr3
  809         RESTORE_REGS
  810         addq    $TF_RIP,%rsp
  811         jmp     doreti_iret
  812 
  813 /*
  814  * MC# handling is similar to NMI.
  815  *
  816  * As with NMIs, machine check exceptions do not respect RFLAGS.IF and
  817  * can occur at any time with a GS.base value that does not correspond
  818  * to the privilege level in CS.
  819  *
  820  * Machine checks are not unblocked by iretq, but it is best to run
  821  * the handler with interrupts disabled since the exception may have
  822  * interrupted a critical section.
  823  *
  824  * The MC# handler runs on its own stack (tss_ist3).  The canonical
  825  * GS.base value for the processor is stored just above the bottom of
  826  * its MC# stack.  For exceptions taken from kernel mode, the current
  827  * value in the processor's GS.base is saved at entry to C-preserved
  828  * register %r12, the canonical value for GS.base is then loaded into
  829  * the processor, and the saved value is restored at exit time.  For
  830  * exceptions taken from user mode, the cheaper 'SWAPGS' instructions
  831  * are used for swapping GS.base.
  832  */
  833 
  834 IDTVEC(mchk)
  835         subq    $TF_RIP,%rsp
  836         movl    $(T_MCHK),TF_TRAPNO(%rsp)
  837         movq    $0,TF_ADDR(%rsp)
  838         movq    $0,TF_ERR(%rsp)
  839         movq    %rdi,TF_RDI(%rsp)
  840         movq    %rsi,TF_RSI(%rsp)
  841         movq    %rdx,TF_RDX(%rsp)
  842         movq    %rcx,TF_RCX(%rsp)
  843         movq    %r8,TF_R8(%rsp)
  844         movq    %r9,TF_R9(%rsp)
  845         movq    %rax,TF_RAX(%rsp)
  846         movq    %rbx,TF_RBX(%rsp)
  847         movq    %rbp,TF_RBP(%rsp)
  848         movq    %r10,TF_R10(%rsp)
  849         movq    %r11,TF_R11(%rsp)
  850         movq    %r12,TF_R12(%rsp)
  851         movq    %r13,TF_R13(%rsp)
  852         movq    %r14,TF_R14(%rsp)
  853         movq    %r15,TF_R15(%rsp)
  854         SAVE_SEGS
  855         movl    $TF_HASSEGS,TF_FLAGS(%rsp)
  856         cld
  857         xorl    %ebx,%ebx
  858         testb   $SEL_RPL_MASK,TF_CS(%rsp)
  859         jnz     mchk_fromuserspace
  860         /*
  861          * We've interrupted the kernel.  Preserve GS.base in %r12,
  862          * %cr3 in %r13, and possibly lower half of MSR_IA32_SPEC_CTL in %r14d.
  863          */
  864         movl    $MSR_GSBASE,%ecx
  865         rdmsr
  866         movq    %rax,%r12
  867         shlq    $32,%rdx
  868         orq     %rdx,%r12
  869         /* Retrieve and load the canonical value for GS.base. */
  870         movq    TF_SIZE(%rsp),%rdx
  871         movl    %edx,%eax
  872         shrq    $32,%rdx
  873         wrmsr
  874         movq    %cr3,%r13
  875         movq    PCPU(KCR3),%rax
  876         cmpq    $~0,%rax
  877         je      1f
  878         movq    %rax,%cr3
  879 1:      testl   $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip)
  880         je      mchk_calltrap
  881         movl    $MSR_IA32_SPEC_CTRL,%ecx
  882         rdmsr
  883         movl    %eax,%r14d
  884         call    handle_ibrs_entry
  885         jmp     mchk_calltrap
  886 mchk_fromuserspace:
  887         incl    %ebx
  888         swapgs
  889         movq    %cr3,%r13
  890         movq    PCPU(KCR3),%rax
  891         cmpq    $~0,%rax
  892         je      1f
  893         movq    %rax,%cr3
  894 1:      call    handle_ibrs_entry
  895 /* Note: this label is also used by ddb and gdb: */
  896 mchk_calltrap:
  897         FAKE_MCOUNT(TF_RIP(%rsp))
  898         movq    %rsp,%rdi
  899         call    mca_intr
  900         MEXITCOUNT
  901         testl   %ebx,%ebx       /* %ebx == 0 => return to userland */
  902         jnz     doreti_exit
  903         /*
  904          * Restore speculation control MSR, if preserved.
  905          */
  906         testl   $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip)
  907         je      1f
  908         movl    %r14d,%eax
  909         xorl    %edx,%edx
  910         movl    $MSR_IA32_SPEC_CTRL,%ecx
  911         wrmsr
  912         /*
  913          * Put back the preserved MSR_GSBASE value.
  914          */
  915 1:      movl    $MSR_GSBASE,%ecx
  916         movq    %r12,%rdx
  917         movl    %edx,%eax
  918         shrq    $32,%rdx
  919         wrmsr
  920         movq    %r13,%cr3
  921         RESTORE_REGS
  922         addq    $TF_RIP,%rsp
  923         jmp     doreti_iret
  924 
  925 ENTRY(fork_trampoline)
  926         movq    %r12,%rdi               /* function */
  927         movq    %rbx,%rsi               /* arg1 */
  928         movq    %rsp,%rdx               /* trapframe pointer */
  929         call    fork_exit
  930         MEXITCOUNT
  931         jmp     doreti                  /* Handle any ASTs */
  932 
  933 /*
  934  * To efficiently implement classification of trap and interrupt handlers
  935  * for profiling, there must be only trap handlers between the labels btrap
  936  * and bintr, and only interrupt handlers between the labels bintr and
  937  * eintr.  This is implemented (partly) by including files that contain
  938  * some of the handlers.  Before including the files, set up a normal asm
  939  * environment so that the included files doen't need to know that they are
  940  * included.
  941  */
  942 
  943 #ifdef COMPAT_FREEBSD32
  944         .data
  945         .p2align 4
  946         .text
  947         SUPERALIGN_TEXT
  948 
  949 #include <amd64/ia32/ia32_exception.S>
  950 #endif
  951 
  952         .data
  953         .p2align 4
  954         .text
  955         SUPERALIGN_TEXT
  956 MCOUNT_LABEL(bintr)
  957 
  958 #include <amd64/amd64/apic_vector.S>
  959 
  960 #ifdef DEV_ATPIC
  961         .data
  962         .p2align 4
  963         .text
  964         SUPERALIGN_TEXT
  965 
  966 #include <amd64/amd64/atpic_vector.S>
  967 #endif
  968 
  969         .text
  970 MCOUNT_LABEL(eintr)
  971 
  972 /*
  973  * void doreti(struct trapframe)
  974  *
  975  * Handle return from interrupts, traps and syscalls.
  976  */
  977         .text
  978         SUPERALIGN_TEXT
  979         .type   doreti,@function
  980         .globl  doreti
  981 doreti:
  982         FAKE_MCOUNT($bintr)             /* init "from" bintr -> doreti */
  983         /*
  984          * Check if ASTs can be handled now.
  985          */
  986         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* are we returning to user mode? */
  987         jz      doreti_exit             /* can't handle ASTs now if not */
  988 
  989 doreti_ast:
  990         /*
  991          * Check for ASTs atomically with returning.  Disabling CPU
  992          * interrupts provides sufficient locking even in the SMP case,
  993          * since we will be informed of any new ASTs by an IPI.
  994          */
  995         cli
  996         movq    PCPU(CURTHREAD),%rax
  997         testl   $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax)
  998         je      doreti_exit
  999         sti
 1000         movq    %rsp,%rdi       /* pass a pointer to the trapframe */
 1001         call    ast
 1002         jmp     doreti_ast
 1003 
 1004         /*
 1005          * doreti_exit: pop registers, iret.
 1006          *
 1007          *      The segment register pop is a special case, since it may
 1008          *      fault if (for example) a sigreturn specifies bad segment
 1009          *      registers.  The fault is handled in trap.c.
 1010          */
 1011 doreti_exit:
 1012         MEXITCOUNT
 1013         movq    PCPU(CURPCB),%r8
 1014 
 1015         /*
 1016          * Do not reload segment registers for kernel.
 1017          * Since we do not reload segments registers with sane
 1018          * values on kernel entry, descriptors referenced by
 1019          * segments registers might be not valid.  This is fatal
 1020          * for user mode, but is not a problem for the kernel.
 1021          */
 1022         testb   $SEL_RPL_MASK,TF_CS(%rsp)
 1023         jz      ld_regs
 1024         testl   $PCB_FULL_IRET,PCB_FLAGS(%r8)
 1025         jz      ld_regs
 1026         testl   $TF_HASSEGS,TF_FLAGS(%rsp)
 1027         je      set_segs
 1028 
 1029 do_segs:
 1030         /* Restore %fs and fsbase */
 1031         movw    TF_FS(%rsp),%ax
 1032         .globl  ld_fs
 1033 ld_fs:
 1034         movw    %ax,%fs
 1035         cmpw    $KUF32SEL,%ax
 1036         jne     1f
 1037         movl    $MSR_FSBASE,%ecx
 1038         movl    PCB_FSBASE(%r8),%eax
 1039         movl    PCB_FSBASE+4(%r8),%edx
 1040         .globl  ld_fsbase
 1041 ld_fsbase:
 1042         wrmsr
 1043 1:
 1044         /* Restore %gs and gsbase */
 1045         movw    TF_GS(%rsp),%si
 1046         pushfq
 1047         cli
 1048         movl    $MSR_GSBASE,%ecx
 1049         /* Save current kernel %gs base into %r12d:%r13d */
 1050         rdmsr
 1051         movl    %eax,%r12d
 1052         movl    %edx,%r13d
 1053         .globl  ld_gs
 1054 ld_gs:
 1055         movw    %si,%gs
 1056         /* Save user %gs base into %r14d:%r15d */
 1057         rdmsr
 1058         movl    %eax,%r14d
 1059         movl    %edx,%r15d
 1060         /* Restore kernel %gs base */
 1061         movl    %r12d,%eax
 1062         movl    %r13d,%edx
 1063         wrmsr
 1064         popfq
 1065         /*
 1066          * Restore user %gs base, either from PCB if used for TLS, or
 1067          * from the previously saved msr read.
 1068          */
 1069         movl    $MSR_KGSBASE,%ecx
 1070         cmpw    $KUG32SEL,%si
 1071         jne     1f
 1072         movl    PCB_GSBASE(%r8),%eax
 1073         movl    PCB_GSBASE+4(%r8),%edx
 1074         jmp     ld_gsbase
 1075 1:
 1076         movl    %r14d,%eax
 1077         movl    %r15d,%edx
 1078         .globl  ld_gsbase
 1079 ld_gsbase:
 1080         wrmsr   /* May trap if non-canonical, but only for TLS. */
 1081         .globl  ld_es
 1082 ld_es:
 1083         movw    TF_ES(%rsp),%es
 1084         .globl  ld_ds
 1085 ld_ds:
 1086         movw    TF_DS(%rsp),%ds
 1087 ld_regs:
 1088         RESTORE_REGS
 1089         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
 1090         jz      2f                      /* keep running with kernel GS.base */
 1091         cli
 1092         call    handle_ibrs_exit_rs
 1093         cmpb    $0,pti
 1094         je      1f
 1095         pushq   %rdx
 1096         movq    PCPU(PRVSPACE),%rdx
 1097         addq    $PC_PTI_STACK+PC_PTI_STACK_SZ*8-PTI_SIZE,%rdx
 1098         movq    %rax,PTI_RAX(%rdx)
 1099         popq    %rax
 1100         movq    %rax,PTI_RDX(%rdx)
 1101         movq    TF_RIP(%rsp),%rax
 1102         movq    %rax,PTI_RIP(%rdx)
 1103         movq    TF_CS(%rsp),%rax
 1104         movq    %rax,PTI_CS(%rdx)
 1105         movq    TF_RFLAGS(%rsp),%rax
 1106         movq    %rax,PTI_RFLAGS(%rdx)
 1107         movq    TF_RSP(%rsp),%rax
 1108         movq    %rax,PTI_RSP(%rdx)
 1109         movq    TF_SS(%rsp),%rax
 1110         movq    %rax,PTI_SS(%rdx)
 1111         movq    PCPU(UCR3),%rax
 1112         swapgs
 1113         movq    %rdx,%rsp
 1114         movq    %rax,%cr3
 1115         popq    %rdx
 1116         popq    %rax
 1117         addq    $8,%rsp
 1118         jmp     doreti_iret
 1119 1:      swapgs
 1120 2:      addq    $TF_RIP,%rsp
 1121         .globl  doreti_iret
 1122 doreti_iret:
 1123         iretq
 1124 
 1125 set_segs:
 1126         movw    $KUDSEL,%ax
 1127         movw    %ax,TF_DS(%rsp)
 1128         movw    %ax,TF_ES(%rsp)
 1129         movw    $KUF32SEL,TF_FS(%rsp)
 1130         movw    $KUG32SEL,TF_GS(%rsp)
 1131         jmp     do_segs
 1132 
 1133         /*
 1134          * doreti_iret_fault.  Alternative return code for
 1135          * the case where we get a fault in the doreti_exit code
 1136          * above.  trap() (amd64/amd64/trap.c) catches this specific
 1137          * case, sends the process a signal and continues in the
 1138          * corresponding place in the code below.
 1139          */
 1140         ALIGN_TEXT
 1141         .globl  doreti_iret_fault
 1142 doreti_iret_fault:
 1143         subq    $TF_RIP,%rsp            /* space including tf_err, tf_trapno */
 1144         movq    %rax,TF_RAX(%rsp)
 1145         movq    %rdx,TF_RDX(%rsp)
 1146         movq    %rcx,TF_RCX(%rsp)
 1147         call    handle_ibrs_entry
 1148         testb   $SEL_RPL_MASK,TF_CS(%rsp)
 1149         jz      1f
 1150         sti
 1151 1:
 1152         SAVE_SEGS
 1153         movl    $TF_HASSEGS,TF_FLAGS(%rsp)
 1154         movq    %rdi,TF_RDI(%rsp)
 1155         movq    %rsi,TF_RSI(%rsp)
 1156         movq    %r8,TF_R8(%rsp)
 1157         movq    %r9,TF_R9(%rsp)
 1158         movq    %rbx,TF_RBX(%rsp)
 1159         movq    %rbp,TF_RBP(%rsp)
 1160         movq    %r10,TF_R10(%rsp)
 1161         movq    %r11,TF_R11(%rsp)
 1162         movq    %r12,TF_R12(%rsp)
 1163         movq    %r13,TF_R13(%rsp)
 1164         movq    %r14,TF_R14(%rsp)
 1165         movq    %r15,TF_R15(%rsp)
 1166         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
 1167         movq    $0,TF_ERR(%rsp) /* XXX should be the error code */
 1168         movq    $0,TF_ADDR(%rsp)
 1169         FAKE_MCOUNT(TF_RIP(%rsp))
 1170         jmp     calltrap
 1171 
 1172         ALIGN_TEXT
 1173         .globl  ds_load_fault
 1174 ds_load_fault:
 1175         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
 1176         testb   $SEL_RPL_MASK,TF_CS(%rsp)
 1177         jz      1f
 1178         sti
 1179 1:
 1180         movq    %rsp,%rdi
 1181         call    trap
 1182         movw    $KUDSEL,TF_DS(%rsp)
 1183         jmp     doreti
 1184 
 1185         ALIGN_TEXT
 1186         .globl  es_load_fault
 1187 es_load_fault:
 1188         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
 1189         testl   $PSL_I,TF_RFLAGS(%rsp)
 1190         jz      1f
 1191         sti
 1192 1:
 1193         movq    %rsp,%rdi
 1194         call    trap
 1195         movw    $KUDSEL,TF_ES(%rsp)
 1196         jmp     doreti
 1197 
 1198         ALIGN_TEXT
 1199         .globl  fs_load_fault
 1200 fs_load_fault:
 1201         testl   $PSL_I,TF_RFLAGS(%rsp)
 1202         jz      1f
 1203         sti
 1204 1:
 1205         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
 1206         movq    %rsp,%rdi
 1207         call    trap
 1208         movw    $KUF32SEL,TF_FS(%rsp)
 1209         jmp     doreti
 1210 
 1211         ALIGN_TEXT
 1212         .globl  gs_load_fault
 1213 gs_load_fault:
 1214         popfq
 1215         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
 1216         testl   $PSL_I,TF_RFLAGS(%rsp)
 1217         jz      1f
 1218         sti
 1219 1:
 1220         movq    %rsp,%rdi
 1221         call    trap
 1222         movw    $KUG32SEL,TF_GS(%rsp)
 1223         jmp     doreti
 1224 
 1225         ALIGN_TEXT
 1226         .globl  fsbase_load_fault
 1227 fsbase_load_fault:
 1228         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
 1229         testl   $PSL_I,TF_RFLAGS(%rsp)
 1230         jz      1f
 1231         sti
 1232 1:
 1233         movq    %rsp,%rdi
 1234         call    trap
 1235         movq    PCPU(CURTHREAD),%r8
 1236         movq    TD_PCB(%r8),%r8
 1237         movq    $0,PCB_FSBASE(%r8)
 1238         jmp     doreti
 1239 
 1240         ALIGN_TEXT
 1241         .globl  gsbase_load_fault
 1242 gsbase_load_fault:
 1243         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
 1244         testl   $PSL_I,TF_RFLAGS(%rsp)
 1245         jz      1f
 1246         sti
 1247 1:
 1248         movq    %rsp,%rdi
 1249         call    trap
 1250         movq    PCPU(CURTHREAD),%r8
 1251         movq    TD_PCB(%r8),%r8
 1252         movq    $0,PCB_GSBASE(%r8)
 1253         jmp     doreti
 1254 
 1255 #ifdef HWPMC_HOOKS
 1256         ENTRY(end_exceptions)
 1257 #endif

Cache object: 88a629ef623e8ad8779592a8e31623ad


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.