The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/exception.S

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1989, 1990 William F. Jolitz.
    3  * Copyright (c) 1990 The Regents of the University of California.
    4  * Copyright (c) 2007 The FreeBSD Foundation
    5  * All rights reserved.
    6  *
    7  * Portions of this software were developed by A. Joseph Koshy under
    8  * sponsorship from the FreeBSD Foundation and Google, Inc.
    9  *
   10  * Redistribution and use in source and binary forms, with or without
   11  * modification, are permitted provided that the following conditions
   12  * are met:
   13  * 1. Redistributions of source code must retain the above copyright
   14  *    notice, this list of conditions and the following disclaimer.
   15  * 2. Redistributions in binary form must reproduce the above copyright
   16  *    notice, this list of conditions and the following disclaimer in the
   17  *    documentation and/or other materials provided with the distribution.
   18  * 4. Neither the name of the University nor the names of its contributors
   19  *    may be used to endorse or promote products derived from this software
   20  *    without specific prior written permission.
   21  *
   22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   32  * SUCH DAMAGE.
   33  *
   34  * $FreeBSD$
   35  */
   36 
   37 #include "opt_atpic.h"
   38 #include "opt_compat.h"
   39 #include "opt_hwpmc_hooks.h"
   40 #include "opt_kdtrace.h"
   41 
   42 #include <machine/asmacros.h>
   43 #include <machine/psl.h>
   44 #include <machine/trap.h>
   45 #include <machine/specialreg.h>
   46 
   47 #include "assym.s"
   48 
   49 #ifdef KDTRACE_HOOKS
   50         .bss
   51         .globl  dtrace_invop_jump_addr
   52         .align  8
   53         .type   dtrace_invop_jump_addr,@object
   54         .size   dtrace_invop_jump_addr,8
   55 dtrace_invop_jump_addr:
   56         .zero   8
   57         .globl  dtrace_invop_calltrap_addr
   58         .align  8
   59         .type   dtrace_invop_calltrap_addr,@object
   60         .size   dtrace_invop_calltrap_addr,8
   61 dtrace_invop_calltrap_addr:
   62         .zero   8
   63 #endif
   64         .text
   65 #ifdef HWPMC_HOOKS
   66         ENTRY(start_exceptions)
   67 #endif
   68 
   69 /*****************************************************************************/
   70 /* Trap handling                                                             */
   71 /*****************************************************************************/
   72 /*
   73  * Trap and fault vector routines.
   74  *
   75  * All traps are 'interrupt gates', SDT_SYSIGT.  An interrupt gate pushes
   76  * state on the stack but also disables interrupts.  This is important for
   77  * us for the use of the swapgs instruction.  We cannot be interrupted
   78  * until the GS.base value is correct.  For most traps, we automatically
   79  * then enable interrupts if the interrupted context had them enabled.
   80  * This is equivalent to the i386 port's use of SDT_SYS386TGT.
   81  *
   82  * The cpu will push a certain amount of state onto the kernel stack for
   83  * the current process.  See amd64/include/frame.h.
   84  * This includes the current RFLAGS (status register, which includes
   85  * the interrupt disable state prior to the trap), the code segment register,
   86  * and the return instruction pointer are pushed by the cpu.  The cpu
   87  * will also push an 'error' code for certain traps.  We push a dummy
   88  * error code for those traps where the cpu doesn't in order to maintain
   89  * a consistent frame.  We also push a contrived 'trap number'.
   90  *
   91  * The CPU does not push the general registers, so we must do that, and we
   92  * must restore them prior to calling 'iret'.  The CPU adjusts %cs and %ss
   93  * but does not mess with %ds, %es, %gs or %fs.  We swap the %gs base for
   94  * for the kernel mode operation shortly, without changes to the selector
   95  * loaded.  Since superuser long mode works with any selectors loaded into
   96  * segment registers other then %cs, which makes them mostly unused in long
   97  * mode, and kernel does not reference %fs, leave them alone.  The segment
   98  * registers are reloaded on return to the usermode.
   99  */
  100 
  101 MCOUNT_LABEL(user)
  102 MCOUNT_LABEL(btrap)
  103 
  104 /* Traps that we leave interrupts disabled for.. */
  105 #define TRAP_NOEN(a)    \
  106         subq $TF_RIP,%rsp; \
  107         movl $(a),TF_TRAPNO(%rsp) ; \
  108         movq $0,TF_ADDR(%rsp) ; \
  109         movq $0,TF_ERR(%rsp) ; \
  110         jmp alltraps_noen
  111 IDTVEC(dbg)
  112         TRAP_NOEN(T_TRCTRAP)
  113 IDTVEC(bpt)
  114         TRAP_NOEN(T_BPTFLT)
  115 #ifdef KDTRACE_HOOKS
  116 IDTVEC(dtrace_ret)
  117         TRAP_NOEN(T_DTRACE_RET)
  118 #endif
  119 
  120 /* Regular traps; The cpu does not supply tf_err for these. */
  121 #define TRAP(a)  \
  122         subq $TF_RIP,%rsp; \
  123         movl $(a),TF_TRAPNO(%rsp) ; \
  124         movq $0,TF_ADDR(%rsp) ; \
  125         movq $0,TF_ERR(%rsp) ; \
  126         jmp alltraps
  127 IDTVEC(div)
  128         TRAP(T_DIVIDE)
  129 IDTVEC(ofl)
  130         TRAP(T_OFLOW)
  131 IDTVEC(bnd)
  132         TRAP(T_BOUND)
  133 IDTVEC(ill)
  134         TRAP(T_PRIVINFLT)
  135 IDTVEC(dna)
  136         TRAP(T_DNA)
  137 IDTVEC(fpusegm)
  138         TRAP(T_FPOPFLT)
  139 IDTVEC(mchk)
  140         TRAP(T_MCHK)
  141 IDTVEC(rsvd)
  142         TRAP(T_RESERVED)
  143 IDTVEC(fpu)
  144         TRAP(T_ARITHTRAP)
  145 IDTVEC(xmm)
  146         TRAP(T_XMMFLT)
  147 
  148 /* This group of traps have tf_err already pushed by the cpu */
  149 #define TRAP_ERR(a)     \
  150         subq $TF_ERR,%rsp; \
  151         movl $(a),TF_TRAPNO(%rsp) ; \
  152         movq $0,TF_ADDR(%rsp) ; \
  153         jmp alltraps
  154 IDTVEC(tss)
  155         TRAP_ERR(T_TSSFLT)
  156 IDTVEC(missing)
  157         subq    $TF_ERR,%rsp
  158         movl    $T_SEGNPFLT,TF_TRAPNO(%rsp)
  159         jmp     prot_addrf
  160 IDTVEC(stk)
  161         subq    $TF_ERR,%rsp
  162         movl    $T_STKFLT,TF_TRAPNO(%rsp)
  163         jmp     prot_addrf
  164 IDTVEC(align)
  165         TRAP_ERR(T_ALIGNFLT)
  166 
  167         /*
  168          * alltraps entry point.  Use swapgs if this is the first time in the
  169          * kernel from userland.  Reenable interrupts if they were enabled
  170          * before the trap.  This approximates SDT_SYS386TGT on the i386 port.
  171          */
  172         SUPERALIGN_TEXT
  173         .globl  alltraps
  174         .type   alltraps,@function
  175 alltraps:
  176         movq    %rdi,TF_RDI(%rsp)
  177         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  178         jz      alltraps_testi          /* already running with kernel GS.base */
  179         swapgs
  180         movq    PCPU(CURPCB),%rdi
  181         andl    $~PCB_FULL_IRET,PCB_FLAGS(%rdi)
  182         movw    %fs,TF_FS(%rsp)
  183         movw    %gs,TF_GS(%rsp)
  184         movw    %es,TF_ES(%rsp)
  185         movw    %ds,TF_DS(%rsp)
  186 alltraps_testi:
  187         testl   $PSL_I,TF_RFLAGS(%rsp)
  188         jz      alltraps_pushregs_no_rdi
  189         sti
  190 alltraps_pushregs_no_rdi:
  191         movq    %rsi,TF_RSI(%rsp)
  192         movq    %rdx,TF_RDX(%rsp)
  193         movq    %rcx,TF_RCX(%rsp)
  194         movq    %r8,TF_R8(%rsp)
  195         movq    %r9,TF_R9(%rsp)
  196         movq    %rax,TF_RAX(%rsp)
  197         movq    %rbx,TF_RBX(%rsp)
  198         movq    %rbp,TF_RBP(%rsp)
  199         movq    %r10,TF_R10(%rsp)
  200         movq    %r11,TF_R11(%rsp)
  201         movq    %r12,TF_R12(%rsp)
  202         movq    %r13,TF_R13(%rsp)
  203         movq    %r14,TF_R14(%rsp)
  204         movq    %r15,TF_R15(%rsp)
  205         movl    $TF_HASSEGS,TF_FLAGS(%rsp)
  206         cld
  207         FAKE_MCOUNT(TF_RIP(%rsp))
  208 #ifdef KDTRACE_HOOKS
  209         /*
  210          * DTrace Function Boundary Trace (fbt) probes are triggered
  211          * by int3 (0xcc) which causes the #BP (T_BPTFLT) breakpoint
  212          * interrupt. For all other trap types, just handle them in
  213          * the usual way.
  214          */
  215         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  216         jnz     calltrap                /* ignore userland traps */
  217         cmpl    $T_BPTFLT,TF_TRAPNO(%rsp)
  218         jne     calltrap
  219 
  220         /* Check if there is no DTrace hook registered. */
  221         cmpq    $0,dtrace_invop_jump_addr
  222         je      calltrap
  223 
  224         /*
  225          * Set our jump address for the jump back in the event that
  226          * the breakpoint wasn't caused by DTrace at all.
  227          */
  228         movq    $calltrap,dtrace_invop_calltrap_addr(%rip)
  229 
  230         /* Jump to the code hooked in by DTrace. */
  231         movq    dtrace_invop_jump_addr,%rax
  232         jmpq    *dtrace_invop_jump_addr
  233 #endif
  234         .globl  calltrap
  235         .type   calltrap,@function
  236 calltrap:
  237         movq    %rsp,%rdi
  238         call    trap
  239         MEXITCOUNT
  240         jmp     doreti                  /* Handle any pending ASTs */
  241 
  242         /*
  243          * alltraps_noen entry point.  Unlike alltraps above, we want to
  244          * leave the interrupts disabled.  This corresponds to
  245          * SDT_SYS386IGT on the i386 port.
  246          */
  247         SUPERALIGN_TEXT
  248         .globl  alltraps_noen
  249         .type   alltraps_noen,@function
  250 alltraps_noen:
  251         movq    %rdi,TF_RDI(%rsp)
  252         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  253         jz      1f      /* already running with kernel GS.base */
  254         swapgs
  255         movq    PCPU(CURPCB),%rdi
  256         andl    $~PCB_FULL_IRET,PCB_FLAGS(%rdi)
  257 1:      movw    %fs,TF_FS(%rsp)
  258         movw    %gs,TF_GS(%rsp)
  259         movw    %es,TF_ES(%rsp)
  260         movw    %ds,TF_DS(%rsp)
  261         jmp     alltraps_pushregs_no_rdi
  262 
  263 IDTVEC(dblfault)
  264         subq    $TF_ERR,%rsp
  265         movl    $T_DOUBLEFLT,TF_TRAPNO(%rsp)
  266         movq    $0,TF_ADDR(%rsp)
  267         movq    $0,TF_ERR(%rsp)
  268         movq    %rdi,TF_RDI(%rsp)
  269         movq    %rsi,TF_RSI(%rsp)
  270         movq    %rdx,TF_RDX(%rsp)
  271         movq    %rcx,TF_RCX(%rsp)
  272         movq    %r8,TF_R8(%rsp)
  273         movq    %r9,TF_R9(%rsp)
  274         movq    %rax,TF_RAX(%rsp)
  275         movq    %rbx,TF_RBX(%rsp)
  276         movq    %rbp,TF_RBP(%rsp)
  277         movq    %r10,TF_R10(%rsp)
  278         movq    %r11,TF_R11(%rsp)
  279         movq    %r12,TF_R12(%rsp)
  280         movq    %r13,TF_R13(%rsp)
  281         movq    %r14,TF_R14(%rsp)
  282         movq    %r15,TF_R15(%rsp)
  283         movw    %fs,TF_FS(%rsp)
  284         movw    %gs,TF_GS(%rsp)
  285         movw    %es,TF_ES(%rsp)
  286         movw    %ds,TF_DS(%rsp)
  287         movl    $TF_HASSEGS,TF_FLAGS(%rsp)
  288         cld
  289         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  290         jz      1f                      /* already running with kernel GS.base */
  291         swapgs
  292 1:
  293         movq    %rsp,%rdi
  294         call    dblfault_handler
  295 2:
  296         hlt
  297         jmp     2b
  298 
  299 IDTVEC(page)
  300         subq    $TF_ERR,%rsp
  301         movl    $T_PAGEFLT,TF_TRAPNO(%rsp)
  302         movq    %rdi,TF_RDI(%rsp)       /* free up a GP register */
  303         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  304         jz      1f                      /* already running with kernel GS.base */
  305         swapgs
  306         movq    PCPU(CURPCB),%rdi
  307         andl    $~PCB_FULL_IRET,PCB_FLAGS(%rdi)
  308 1:      movq    %cr2,%rdi               /* preserve %cr2 before ..  */
  309         movq    %rdi,TF_ADDR(%rsp)      /* enabling interrupts. */
  310         movw    %fs,TF_FS(%rsp)
  311         movw    %gs,TF_GS(%rsp)
  312         movw    %es,TF_ES(%rsp)
  313         movw    %ds,TF_DS(%rsp)
  314         testl   $PSL_I,TF_RFLAGS(%rsp)
  315         jz      alltraps_pushregs_no_rdi
  316         sti
  317         jmp     alltraps_pushregs_no_rdi
  318 
  319         /*
  320          * We have to special-case this one.  If we get a trap in doreti() at
  321          * the iretq stage, we'll reenter with the wrong gs state.  We'll have
  322          * to do a special the swapgs in this case even coming from the kernel.
  323          * XXX linux has a trap handler for their equivalent of load_gs().
  324          */
  325 IDTVEC(prot)
  326         subq    $TF_ERR,%rsp
  327         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
  328 prot_addrf:
  329         movq    $0,TF_ADDR(%rsp)
  330         movq    %rdi,TF_RDI(%rsp)       /* free up a GP register */
  331         leaq    doreti_iret(%rip),%rdi
  332         cmpq    %rdi,TF_RIP(%rsp)
  333         je      1f                      /* kernel but with user gsbase!! */
  334         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  335         jz      2f                      /* already running with kernel GS.base */
  336 1:      swapgs
  337 2:      movq    PCPU(CURPCB),%rdi
  338         orl     $PCB_FULL_IRET,PCB_FLAGS(%rdi)  /* always full iret from GPF */
  339         movw    %fs,TF_FS(%rsp)
  340         movw    %gs,TF_GS(%rsp)
  341         movw    %es,TF_ES(%rsp)
  342         movw    %ds,TF_DS(%rsp)
  343         testl   $PSL_I,TF_RFLAGS(%rsp)
  344         jz      alltraps_pushregs_no_rdi
  345         sti
  346         jmp     alltraps_pushregs_no_rdi
  347 
  348 /*
  349  * Fast syscall entry point.  We enter here with just our new %cs/%ss set,
  350  * and the new privilige level.  We are still running on the old user stack
  351  * pointer.  We have to juggle a few things around to find our stack etc.
  352  * swapgs gives us access to our PCPU space only.
  353  *
  354  * We do not support invoking this from a custom %cs or %ss (e.g. using
  355  * entries from an LDT).
  356  */
  357 IDTVEC(fast_syscall)
  358         swapgs
  359         movq    %rsp,PCPU(SCRATCH_RSP)
  360         movq    PCPU(RSP0),%rsp
  361         /* Now emulate a trapframe. Make the 8 byte alignment odd for call. */
  362         subq    $TF_SIZE,%rsp
  363         /* defer TF_RSP till we have a spare register */
  364         movq    %r11,TF_RFLAGS(%rsp)
  365         movq    %rcx,TF_RIP(%rsp)       /* %rcx original value is in %r10 */
  366         movq    PCPU(SCRATCH_RSP),%r11  /* %r11 already saved */
  367         movq    %r11,TF_RSP(%rsp)       /* user stack pointer */
  368         movw    %fs,TF_FS(%rsp)
  369         movw    %gs,TF_GS(%rsp)
  370         movw    %es,TF_ES(%rsp)
  371         movw    %ds,TF_DS(%rsp)
  372         movq    PCPU(CURPCB),%r11
  373         andl    $~PCB_FULL_IRET,PCB_FLAGS(%r11)
  374         sti
  375         movq    $KUDSEL,TF_SS(%rsp)
  376         movq    $KUCSEL,TF_CS(%rsp)
  377         movq    $2,TF_ERR(%rsp)
  378         movq    %rdi,TF_RDI(%rsp)       /* arg 1 */
  379         movq    %rsi,TF_RSI(%rsp)       /* arg 2 */
  380         movq    %rdx,TF_RDX(%rsp)       /* arg 3 */
  381         movq    %r10,TF_RCX(%rsp)       /* arg 4 */
  382         movq    %r8,TF_R8(%rsp)         /* arg 5 */
  383         movq    %r9,TF_R9(%rsp)         /* arg 6 */
  384         movq    %rax,TF_RAX(%rsp)       /* syscall number */
  385         movq    %rbx,TF_RBX(%rsp)       /* C preserved */
  386         movq    %rbp,TF_RBP(%rsp)       /* C preserved */
  387         movq    %r12,TF_R12(%rsp)       /* C preserved */
  388         movq    %r13,TF_R13(%rsp)       /* C preserved */
  389         movq    %r14,TF_R14(%rsp)       /* C preserved */
  390         movq    %r15,TF_R15(%rsp)       /* C preserved */
  391         movl    $TF_HASSEGS,TF_FLAGS(%rsp)
  392         cld
  393         FAKE_MCOUNT(TF_RIP(%rsp))
  394         movq    PCPU(CURTHREAD),%rdi
  395         movq    %rsp,TD_FRAME(%rdi)
  396         movl    TF_RFLAGS(%rsp),%esi
  397         andl    $PSL_T,%esi
  398         call    amd64_syscall
  399 1:      movq    PCPU(CURPCB),%rax
  400         /* Disable interrupts before testing PCB_FULL_IRET. */
  401         cli
  402         testl   $PCB_FULL_IRET,PCB_FLAGS(%rax)
  403         jnz     3f
  404         /* Check for and handle AST's on return to userland. */
  405         movq    PCPU(CURTHREAD),%rax
  406         testl   $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax)
  407         jne     2f
  408         /* Restore preserved registers. */
  409         MEXITCOUNT
  410         movq    TF_RDI(%rsp),%rdi       /* bonus; preserve arg 1 */
  411         movq    TF_RSI(%rsp),%rsi       /* bonus: preserve arg 2 */
  412         movq    TF_RDX(%rsp),%rdx       /* return value 2 */
  413         movq    TF_RAX(%rsp),%rax       /* return value 1 */
  414         movq    TF_RFLAGS(%rsp),%r11    /* original %rflags */
  415         movq    TF_RIP(%rsp),%rcx       /* original %rip */
  416         movq    TF_RSP(%rsp),%rsp       /* user stack pointer */
  417         swapgs
  418         sysretq
  419 
  420 2:      /* AST scheduled. */
  421         sti
  422         movq    %rsp,%rdi
  423         call    ast
  424         jmp     1b
  425 
  426 3:      /* Requested full context restore, use doreti for that. */
  427         MEXITCOUNT
  428         jmp     doreti
  429 
  430 /*
  431  * Here for CYA insurance, in case a "syscall" instruction gets
  432  * issued from 32 bit compatability mode. MSR_CSTAR has to point
  433  * to *something* if EFER_SCE is enabled.
  434  */
  435 IDTVEC(fast_syscall32)
  436         sysret
  437 
  438 /*
  439  * NMI handling is special.
  440  *
  441  * First, NMIs do not respect the state of the processor's RFLAGS.IF
  442  * bit.  The NMI handler may be entered at any time, including when
  443  * the processor is in a critical section with RFLAGS.IF == 0.
  444  * The processor's GS.base value could be invalid on entry to the
  445  * handler.
  446  *
  447  * Second, the processor treats NMIs specially, blocking further NMIs
  448  * until an 'iretq' instruction is executed.  We thus need to execute
  449  * the NMI handler with interrupts disabled, to prevent a nested interrupt
  450  * from executing an 'iretq' instruction and inadvertently taking the
  451  * processor out of NMI mode.
  452  *
  453  * Third, the NMI handler runs on its own stack (tss_ist2). The canonical
  454  * GS.base value for the processor is stored just above the bottom of its
  455  * NMI stack.  For NMIs taken from kernel mode, the current value in
  456  * the processor's GS.base is saved at entry to C-preserved register %r12,
  457  * the canonical value for GS.base is then loaded into the processor, and
  458  * the saved value is restored at exit time.  For NMIs taken from user mode,
  459  * the cheaper 'SWAPGS' instructions are used for swapping GS.base.
  460  */
  461 
  462 IDTVEC(nmi)
  463         subq    $TF_RIP,%rsp
  464         movl    $(T_NMI),TF_TRAPNO(%rsp)
  465         movq    $0,TF_ADDR(%rsp)
  466         movq    $0,TF_ERR(%rsp)
  467         movq    %rdi,TF_RDI(%rsp)
  468         movq    %rsi,TF_RSI(%rsp)
  469         movq    %rdx,TF_RDX(%rsp)
  470         movq    %rcx,TF_RCX(%rsp)
  471         movq    %r8,TF_R8(%rsp)
  472         movq    %r9,TF_R9(%rsp)
  473         movq    %rax,TF_RAX(%rsp)
  474         movq    %rbx,TF_RBX(%rsp)
  475         movq    %rbp,TF_RBP(%rsp)
  476         movq    %r10,TF_R10(%rsp)
  477         movq    %r11,TF_R11(%rsp)
  478         movq    %r12,TF_R12(%rsp)
  479         movq    %r13,TF_R13(%rsp)
  480         movq    %r14,TF_R14(%rsp)
  481         movq    %r15,TF_R15(%rsp)
  482         movw    %fs,TF_FS(%rsp)
  483         movw    %gs,TF_GS(%rsp)
  484         movw    %es,TF_ES(%rsp)
  485         movw    %ds,TF_DS(%rsp)
  486         movl    $TF_HASSEGS,TF_FLAGS(%rsp)
  487         cld
  488         xorl    %ebx,%ebx
  489         testb   $SEL_RPL_MASK,TF_CS(%rsp)
  490         jnz     nmi_fromuserspace
  491         /*
  492          * We've interrupted the kernel.  Preserve GS.base in %r12.
  493          */
  494         movl    $MSR_GSBASE,%ecx
  495         rdmsr
  496         movq    %rax,%r12
  497         shlq    $32,%rdx
  498         orq     %rdx,%r12
  499         /* Retrieve and load the canonical value for GS.base. */
  500         movq    TF_SIZE(%rsp),%rdx
  501         movl    %edx,%eax
  502         shrq    $32,%rdx
  503         wrmsr
  504         jmp     nmi_calltrap
  505 nmi_fromuserspace:
  506         incl    %ebx
  507         swapgs
  508 /* Note: this label is also used by ddb and gdb: */
  509 nmi_calltrap:
  510         FAKE_MCOUNT(TF_RIP(%rsp))
  511         movq    %rsp,%rdi
  512         call    trap
  513         MEXITCOUNT
  514 #ifdef HWPMC_HOOKS
  515         /*
  516          * Capture a userspace callchain if needed.
  517          *
  518          * - Check if the current trap was from user mode.
  519          * - Check if the current thread is valid.
  520          * - Check if the thread requires a user call chain to be
  521          *   captured.
  522          *
  523          * We are still in NMI mode at this point.
  524          */
  525         testl   %ebx,%ebx
  526         jz      nocallchain     /* not from userspace */
  527         movq    PCPU(CURTHREAD),%rax
  528         orq     %rax,%rax       /* curthread present? */
  529         jz      nocallchain
  530         testl   $TDP_CALLCHAIN,TD_PFLAGS(%rax) /* flagged for capture? */
  531         jz      nocallchain
  532         /*
  533          * A user callchain is to be captured, so:
  534          * - Move execution to the regular kernel stack, to allow for
  535          *   nested NMI interrupts.
  536          * - Take the processor out of "NMI" mode by faking an "iret".
  537          * - Enable interrupts, so that copyin() can work.
  538          */
  539         movq    %rsp,%rsi       /* source stack pointer */
  540         movq    $TF_SIZE,%rcx
  541         movq    PCPU(RSP0),%rdx
  542         subq    %rcx,%rdx
  543         movq    %rdx,%rdi       /* destination stack pointer */
  544 
  545         shrq    $3,%rcx         /* trap frame size in long words */
  546         cld
  547         rep
  548         movsq                   /* copy trapframe */
  549 
  550         movl    %ss,%eax
  551         pushq   %rax            /* tf_ss */
  552         pushq   %rdx            /* tf_rsp (on kernel stack) */
  553         pushfq                  /* tf_rflags */
  554         movl    %cs,%eax
  555         pushq   %rax            /* tf_cs */
  556         pushq   $outofnmi       /* tf_rip */
  557         iretq
  558 outofnmi:
  559         /*
  560          * At this point the processor has exited NMI mode and is running
  561          * with interrupts turned off on the normal kernel stack.
  562          *
  563          * If a pending NMI gets recognized at or after this point, it
  564          * will cause a kernel callchain to be traced.
  565          *
  566          * We turn interrupts back on, and call the user callchain capture hook.
  567          */
  568         movq    pmc_hook,%rax
  569         orq     %rax,%rax
  570         jz      nocallchain
  571         movq    PCPU(CURTHREAD),%rdi            /* thread */
  572         movq    $PMC_FN_USER_CALLCHAIN,%rsi     /* command */
  573         movq    %rsp,%rdx                       /* frame */
  574         sti
  575         call    *%rax
  576         cli
  577 nocallchain:
  578 #endif
  579         testl   %ebx,%ebx
  580         jnz     doreti_exit
  581 nmi_kernelexit:
  582         /*
  583          * Put back the preserved MSR_GSBASE value.
  584          */
  585         movl    $MSR_GSBASE,%ecx
  586         movq    %r12,%rdx
  587         movl    %edx,%eax
  588         shrq    $32,%rdx
  589         wrmsr
  590 nmi_restoreregs:
  591         movq    TF_RDI(%rsp),%rdi
  592         movq    TF_RSI(%rsp),%rsi
  593         movq    TF_RDX(%rsp),%rdx
  594         movq    TF_RCX(%rsp),%rcx
  595         movq    TF_R8(%rsp),%r8
  596         movq    TF_R9(%rsp),%r9
  597         movq    TF_RAX(%rsp),%rax
  598         movq    TF_RBX(%rsp),%rbx
  599         movq    TF_RBP(%rsp),%rbp
  600         movq    TF_R10(%rsp),%r10
  601         movq    TF_R11(%rsp),%r11
  602         movq    TF_R12(%rsp),%r12
  603         movq    TF_R13(%rsp),%r13
  604         movq    TF_R14(%rsp),%r14
  605         movq    TF_R15(%rsp),%r15
  606         addq    $TF_RIP,%rsp
  607         jmp     doreti_iret
  608 
  609 ENTRY(fork_trampoline)
  610         movq    %r12,%rdi               /* function */
  611         movq    %rbx,%rsi               /* arg1 */
  612         movq    %rsp,%rdx               /* trapframe pointer */
  613         call    fork_exit
  614         MEXITCOUNT
  615         jmp     doreti                  /* Handle any ASTs */
  616 
  617 /*
  618  * To efficiently implement classification of trap and interrupt handlers
  619  * for profiling, there must be only trap handlers between the labels btrap
  620  * and bintr, and only interrupt handlers between the labels bintr and
  621  * eintr.  This is implemented (partly) by including files that contain
  622  * some of the handlers.  Before including the files, set up a normal asm
  623  * environment so that the included files doen't need to know that they are
  624  * included.
  625  */
  626 
  627 #ifdef COMPAT_FREEBSD32
  628         .data
  629         .p2align 4
  630         .text
  631         SUPERALIGN_TEXT
  632 
  633 #include <amd64/ia32/ia32_exception.S>
  634 #endif
  635 
  636         .data
  637         .p2align 4
  638         .text
  639         SUPERALIGN_TEXT
  640 MCOUNT_LABEL(bintr)
  641 
  642 #include <amd64/amd64/apic_vector.S>
  643 
  644 #ifdef DEV_ATPIC
  645         .data
  646         .p2align 4
  647         .text
  648         SUPERALIGN_TEXT
  649 
  650 #include <amd64/amd64/atpic_vector.S>
  651 #endif
  652 
  653         .text
  654 MCOUNT_LABEL(eintr)
  655 
  656 /*
  657  * void doreti(struct trapframe)
  658  *
  659  * Handle return from interrupts, traps and syscalls.
  660  */
  661         .text
  662         SUPERALIGN_TEXT
  663         .type   doreti,@function
  664 doreti:
  665         FAKE_MCOUNT($bintr)             /* init "from" bintr -> doreti */
  666         /*
  667          * Check if ASTs can be handled now.
  668          */
  669         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* are we returning to user mode? */
  670         jz      doreti_exit             /* can't handle ASTs now if not */
  671 
  672 doreti_ast:
  673         /*
  674          * Check for ASTs atomically with returning.  Disabling CPU
  675          * interrupts provides sufficient locking even in the SMP case,
  676          * since we will be informed of any new ASTs by an IPI.
  677          */
  678         cli
  679         movq    PCPU(CURTHREAD),%rax
  680         testl   $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax)
  681         je      doreti_exit
  682         sti
  683         movq    %rsp,%rdi       /* pass a pointer to the trapframe */
  684         call    ast
  685         jmp     doreti_ast
  686 
  687         /*
  688          * doreti_exit: pop registers, iret.
  689          *
  690          *      The segment register pop is a special case, since it may
  691          *      fault if (for example) a sigreturn specifies bad segment
  692          *      registers.  The fault is handled in trap.c.
  693          */
  694 doreti_exit:
  695         MEXITCOUNT
  696         movq    PCPU(CURPCB),%r8
  697 
  698         /*
  699          * Do not reload segment registers for kernel.
  700          * Since we do not reload segments registers with sane
  701          * values on kernel entry, descriptors referenced by
  702          * segments registers might be not valid.  This is fatal
  703          * for user mode, but is not a problem for the kernel.
  704          */
  705         testb   $SEL_RPL_MASK,TF_CS(%rsp)
  706         jz      ld_regs
  707         testl   $PCB_FULL_IRET,PCB_FLAGS(%r8)
  708         jz      ld_regs
  709         testl   $TF_HASSEGS,TF_FLAGS(%rsp)
  710         je      set_segs
  711 
  712 do_segs:
  713         /* Restore %fs and fsbase */
  714         movw    TF_FS(%rsp),%ax
  715         .globl  ld_fs
  716 ld_fs:
  717         movw    %ax,%fs
  718         cmpw    $KUF32SEL,%ax
  719         jne     1f
  720         movl    $MSR_FSBASE,%ecx
  721         movl    PCB_FSBASE(%r8),%eax
  722         movl    PCB_FSBASE+4(%r8),%edx
  723         .globl  ld_fsbase
  724 ld_fsbase:
  725         wrmsr
  726 1:
  727         /* Restore %gs and gsbase */
  728         movw    TF_GS(%rsp),%si
  729         pushfq
  730         cli
  731         movl    $MSR_GSBASE,%ecx
  732         /* Save current kernel %gs base into %r12d:%r13d */
  733         rdmsr
  734         movl    %eax,%r12d
  735         movl    %edx,%r13d
  736         .globl  ld_gs
  737 ld_gs:
  738         movw    %si,%gs
  739         /* Save user %gs base into %r14d:%r15d */
  740         rdmsr
  741         movl    %eax,%r14d
  742         movl    %edx,%r15d
  743         /* Restore kernel %gs base */
  744         movl    %r12d,%eax
  745         movl    %r13d,%edx
  746         wrmsr
  747         popfq
  748         /*
  749          * Restore user %gs base, either from PCB if used for TLS, or
  750          * from the previously saved msr read.
  751          */
  752         movl    $MSR_KGSBASE,%ecx
  753         cmpw    $KUG32SEL,%si
  754         jne     1f
  755         movl    PCB_GSBASE(%r8),%eax
  756         movl    PCB_GSBASE+4(%r8),%edx
  757         jmp     ld_gsbase
  758 1:
  759         movl    %r14d,%eax
  760         movl    %r15d,%edx
  761         .globl  ld_gsbase
  762 ld_gsbase:
  763         wrmsr   /* May trap if non-canonical, but only for TLS. */
  764         .globl  ld_es
  765 ld_es:
  766         movw    TF_ES(%rsp),%es
  767         .globl  ld_ds
  768 ld_ds:
  769         movw    TF_DS(%rsp),%ds
  770 ld_regs:
  771         movq    TF_RDI(%rsp),%rdi
  772         movq    TF_RSI(%rsp),%rsi
  773         movq    TF_RDX(%rsp),%rdx
  774         movq    TF_RCX(%rsp),%rcx
  775         movq    TF_R8(%rsp),%r8
  776         movq    TF_R9(%rsp),%r9
  777         movq    TF_RAX(%rsp),%rax
  778         movq    TF_RBX(%rsp),%rbx
  779         movq    TF_RBP(%rsp),%rbp
  780         movq    TF_R10(%rsp),%r10
  781         movq    TF_R11(%rsp),%r11
  782         movq    TF_R12(%rsp),%r12
  783         movq    TF_R13(%rsp),%r13
  784         movq    TF_R14(%rsp),%r14
  785         movq    TF_R15(%rsp),%r15
  786         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  787         jz      1f                      /* keep running with kernel GS.base */
  788         cli
  789         swapgs
  790 1:
  791         addq    $TF_RIP,%rsp            /* skip over tf_err, tf_trapno */
  792         .globl  doreti_iret
  793 doreti_iret:
  794         iretq
  795 
  796 set_segs:
  797         movw    $KUDSEL,%ax
  798         movw    %ax,TF_DS(%rsp)
  799         movw    %ax,TF_ES(%rsp)
  800         movw    $KUF32SEL,TF_FS(%rsp)
  801         movw    $KUG32SEL,TF_GS(%rsp)
  802         jmp     do_segs
  803 
  804         /*
  805          * doreti_iret_fault.  Alternative return code for
  806          * the case where we get a fault in the doreti_exit code
  807          * above.  trap() (amd64/amd64/trap.c) catches this specific
  808          * case, sends the process a signal and continues in the
  809          * corresponding place in the code below.
  810          */
  811         ALIGN_TEXT
  812         .globl  doreti_iret_fault
  813 doreti_iret_fault:
  814         subq    $TF_RIP,%rsp            /* space including tf_err, tf_trapno */
  815         testl   $PSL_I,TF_RFLAGS(%rsp)
  816         jz      1f
  817         sti
  818 1:
  819         movw    %fs,TF_FS(%rsp)
  820         movw    %gs,TF_GS(%rsp)
  821         movw    %es,TF_ES(%rsp)
  822         movw    %ds,TF_DS(%rsp)
  823         movl    $TF_HASSEGS,TF_FLAGS(%rsp)
  824         movq    %rdi,TF_RDI(%rsp)
  825         movq    %rsi,TF_RSI(%rsp)
  826         movq    %rdx,TF_RDX(%rsp)
  827         movq    %rcx,TF_RCX(%rsp)
  828         movq    %r8,TF_R8(%rsp)
  829         movq    %r9,TF_R9(%rsp)
  830         movq    %rax,TF_RAX(%rsp)
  831         movq    %rbx,TF_RBX(%rsp)
  832         movq    %rbp,TF_RBP(%rsp)
  833         movq    %r10,TF_R10(%rsp)
  834         movq    %r11,TF_R11(%rsp)
  835         movq    %r12,TF_R12(%rsp)
  836         movq    %r13,TF_R13(%rsp)
  837         movq    %r14,TF_R14(%rsp)
  838         movq    %r15,TF_R15(%rsp)
  839         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
  840         movq    $0,TF_ERR(%rsp) /* XXX should be the error code */
  841         movq    $0,TF_ADDR(%rsp)
  842         FAKE_MCOUNT(TF_RIP(%rsp))
  843         jmp     calltrap
  844 
  845         ALIGN_TEXT
  846         .globl  ds_load_fault
  847 ds_load_fault:
  848         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
  849         testl   $PSL_I,TF_RFLAGS(%rsp)
  850         jz      1f
  851         sti
  852 1:
  853         movq    %rsp,%rdi
  854         call    trap
  855         movw    $KUDSEL,TF_DS(%rsp)
  856         jmp     doreti
  857 
  858         ALIGN_TEXT
  859         .globl  es_load_fault
  860 es_load_fault:
  861         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
  862         testl   $PSL_I,TF_RFLAGS(%rsp)
  863         jz      1f
  864         sti
  865 1:
  866         movq    %rsp,%rdi
  867         call    trap
  868         movw    $KUDSEL,TF_ES(%rsp)
  869         jmp     doreti
  870 
  871         ALIGN_TEXT
  872         .globl  fs_load_fault
  873 fs_load_fault:
  874         testl   $PSL_I,TF_RFLAGS(%rsp)
  875         jz      1f
  876         sti
  877 1:
  878         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
  879         movq    %rsp,%rdi
  880         call    trap
  881         movw    $KUF32SEL,TF_FS(%rsp)
  882         jmp     doreti
  883 
  884         ALIGN_TEXT
  885         .globl  gs_load_fault
  886 gs_load_fault:
  887         popfq
  888         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
  889         testl   $PSL_I,TF_RFLAGS(%rsp)
  890         jz      1f
  891         sti
  892 1:
  893         movq    %rsp,%rdi
  894         call    trap
  895         movw    $KUG32SEL,TF_GS(%rsp)
  896         jmp     doreti
  897 
  898         ALIGN_TEXT
  899         .globl  fsbase_load_fault
  900 fsbase_load_fault:
  901         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
  902         testl   $PSL_I,TF_RFLAGS(%rsp)
  903         jz      1f
  904         sti
  905 1:
  906         movq    %rsp,%rdi
  907         call    trap
  908         movq    PCPU(CURTHREAD),%r8
  909         movq    TD_PCB(%r8),%r8
  910         movq    $0,PCB_FSBASE(%r8)
  911         jmp     doreti
  912 
  913         ALIGN_TEXT
  914         .globl  gsbase_load_fault
  915 gsbase_load_fault:
  916         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
  917         testl   $PSL_I,TF_RFLAGS(%rsp)
  918         jz      1f
  919         sti
  920 1:
  921         movq    %rsp,%rdi
  922         call    trap
  923         movq    PCPU(CURTHREAD),%r8
  924         movq    TD_PCB(%r8),%r8
  925         movq    $0,PCB_GSBASE(%r8)
  926         jmp     doreti
  927 
  928 #ifdef HWPMC_HOOKS
  929         ENTRY(end_exceptions)
  930 #endif

Cache object: 27a9e632a2239cd836d08520b90a903b


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.