The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/exception.S

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1989, 1990 William F. Jolitz.
    3  * Copyright (c) 1990 The Regents of the University of California.
    4  * Copyright (c) 2007 The FreeBSD Foundation
    5  * All rights reserved.
    6  *
    7  * Portions of this software were developed by A. Joseph Koshy under
    8  * sponsorship from the FreeBSD Foundation and Google, Inc.
    9  *
   10  * Redistribution and use in source and binary forms, with or without
   11  * modification, are permitted provided that the following conditions
   12  * are met:
   13  * 1. Redistributions of source code must retain the above copyright
   14  *    notice, this list of conditions and the following disclaimer.
   15  * 2. Redistributions in binary form must reproduce the above copyright
   16  *    notice, this list of conditions and the following disclaimer in the
   17  *    documentation and/or other materials provided with the distribution.
   18  * 4. Neither the name of the University nor the names of its contributors
   19  *    may be used to endorse or promote products derived from this software
   20  *    without specific prior written permission.
   21  *
   22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   32  * SUCH DAMAGE.
   33  *
   34  * $FreeBSD: releng/9.1/sys/amd64/amd64/exception.S 225575 2011-09-15 09:53:04Z kib $
   35  */
   36 
   37 #include "opt_atpic.h"
   38 #include "opt_compat.h"
   39 #include "opt_hwpmc_hooks.h"
   40 #include "opt_kdtrace.h"
   41 
   42 #include <machine/asmacros.h>
   43 #include <machine/psl.h>
   44 #include <machine/trap.h>
   45 #include <machine/specialreg.h>
   46 
   47 #include "assym.s"
   48 
   49 #ifdef KDTRACE_HOOKS
   50         .bss
   51         .globl  dtrace_invop_jump_addr
   52         .align  8
   53         .type   dtrace_invop_jump_addr,@object
   54         .size   dtrace_invop_jump_addr,8
   55 dtrace_invop_jump_addr:
   56         .zero   8
   57         .globl  dtrace_invop_calltrap_addr
   58         .align  8
   59         .type   dtrace_invop_calltrap_addr,@object
   60         .size   dtrace_invop_calltrap_addr,8
   61 dtrace_invop_calltrap_addr:
   62         .zero   8
   63 #endif
   64         .text
   65 #ifdef HWPMC_HOOKS
   66         ENTRY(start_exceptions)
   67 #endif
   68 
   69 /*****************************************************************************/
   70 /* Trap handling                                                             */
   71 /*****************************************************************************/
   72 /*
   73  * Trap and fault vector routines.
   74  *
   75  * All traps are 'interrupt gates', SDT_SYSIGT.  An interrupt gate pushes
   76  * state on the stack but also disables interrupts.  This is important for
   77  * us for the use of the swapgs instruction.  We cannot be interrupted
   78  * until the GS.base value is correct.  For most traps, we automatically
   79  * then enable interrupts if the interrupted context had them enabled.
   80  * This is equivalent to the i386 port's use of SDT_SYS386TGT.
   81  *
   82  * The cpu will push a certain amount of state onto the kernel stack for
   83  * the current process.  See amd64/include/frame.h.  
   84  * This includes the current RFLAGS (status register, which includes 
   85  * the interrupt disable state prior to the trap), the code segment register,
   86  * and the return instruction pointer are pushed by the cpu.  The cpu 
   87  * will also push an 'error' code for certain traps.  We push a dummy 
   88  * error code for those traps where the cpu doesn't in order to maintain 
   89  * a consistent frame.  We also push a contrived 'trap number'.
   90  *
   91  * The cpu does not push the general registers, we must do that, and we 
   92  * must restore them prior to calling 'iret'.  The cpu adjusts the %cs and
   93  * %ss segment registers, but does not mess with %ds, %es, or %fs.  Thus we
   94  * must load them with appropriate values for supervisor mode operation.
   95  */
   96 
   97 MCOUNT_LABEL(user)
   98 MCOUNT_LABEL(btrap)
   99 
  100 /* Traps that we leave interrupts disabled for.. */
  101 #define TRAP_NOEN(a)    \
  102         subq $TF_RIP,%rsp; \
  103         movl $(a),TF_TRAPNO(%rsp) ; \
  104         movq $0,TF_ADDR(%rsp) ; \
  105         movq $0,TF_ERR(%rsp) ; \
  106         jmp alltraps_noen
  107 IDTVEC(dbg)
  108         TRAP_NOEN(T_TRCTRAP)
  109 IDTVEC(bpt)
  110         TRAP_NOEN(T_BPTFLT)
  111 #ifdef KDTRACE_HOOKS
  112 IDTVEC(dtrace_ret)
  113         TRAP_NOEN(T_DTRACE_RET)
  114 #endif
  115 
  116 /* Regular traps; The cpu does not supply tf_err for these. */
  117 #define TRAP(a)  \
  118         subq $TF_RIP,%rsp; \
  119         movl $(a),TF_TRAPNO(%rsp) ; \
  120         movq $0,TF_ADDR(%rsp) ; \
  121         movq $0,TF_ERR(%rsp) ; \
  122         jmp alltraps
  123 IDTVEC(div)
  124         TRAP(T_DIVIDE)
  125 IDTVEC(ofl)
  126         TRAP(T_OFLOW)
  127 IDTVEC(bnd)
  128         TRAP(T_BOUND)
  129 IDTVEC(ill)
  130         TRAP(T_PRIVINFLT)
  131 IDTVEC(dna)
  132         TRAP(T_DNA)
  133 IDTVEC(fpusegm)
  134         TRAP(T_FPOPFLT)
  135 IDTVEC(mchk)
  136         TRAP(T_MCHK)
  137 IDTVEC(rsvd)
  138         TRAP(T_RESERVED)
  139 IDTVEC(fpu)
  140         TRAP(T_ARITHTRAP)
  141 IDTVEC(xmm)
  142         TRAP(T_XMMFLT)
  143 
  144 /* This group of traps have tf_err already pushed by the cpu */
  145 #define TRAP_ERR(a)     \
  146         subq $TF_ERR,%rsp; \
  147         movl $(a),TF_TRAPNO(%rsp) ; \
  148         movq $0,TF_ADDR(%rsp) ; \
  149         jmp alltraps
  150 IDTVEC(tss)
  151         TRAP_ERR(T_TSSFLT)
  152 IDTVEC(missing)
  153         TRAP_ERR(T_SEGNPFLT)
  154 IDTVEC(stk)
  155         TRAP_ERR(T_STKFLT)
  156 IDTVEC(align)
  157         TRAP_ERR(T_ALIGNFLT)
  158 
  159         /*
  160          * alltraps entry point.  Use swapgs if this is the first time in the
  161          * kernel from userland.  Reenable interrupts if they were enabled
  162          * before the trap.  This approximates SDT_SYS386TGT on the i386 port.
  163          */
  164         SUPERALIGN_TEXT
  165         .globl  alltraps
  166         .type   alltraps,@function
  167 alltraps:
  168         movq    %rdi,TF_RDI(%rsp)
  169         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  170         jz      alltraps_testi          /* already running with kernel GS.base */
  171         swapgs
  172         movq    PCPU(CURPCB),%rdi
  173         andl    $~PCB_FULL_IRET,PCB_FLAGS(%rdi)
  174         movw    %fs,TF_FS(%rsp)
  175         movw    %gs,TF_GS(%rsp)
  176         movw    %es,TF_ES(%rsp)
  177         movw    %ds,TF_DS(%rsp)
  178 alltraps_testi:
  179         testl   $PSL_I,TF_RFLAGS(%rsp)
  180         jz      alltraps_pushregs_no_rdi
  181         sti
  182 alltraps_pushregs_no_rdi:
  183         movq    %rsi,TF_RSI(%rsp)
  184         movq    %rdx,TF_RDX(%rsp)
  185         movq    %rcx,TF_RCX(%rsp)
  186         movq    %r8,TF_R8(%rsp)
  187         movq    %r9,TF_R9(%rsp)
  188         movq    %rax,TF_RAX(%rsp)
  189         movq    %rbx,TF_RBX(%rsp)
  190         movq    %rbp,TF_RBP(%rsp)
  191         movq    %r10,TF_R10(%rsp)
  192         movq    %r11,TF_R11(%rsp)
  193         movq    %r12,TF_R12(%rsp)
  194         movq    %r13,TF_R13(%rsp)
  195         movq    %r14,TF_R14(%rsp)
  196         movq    %r15,TF_R15(%rsp)
  197         movl    $TF_HASSEGS,TF_FLAGS(%rsp)
  198         cld
  199         FAKE_MCOUNT(TF_RIP(%rsp))
  200 #ifdef KDTRACE_HOOKS
  201         /*
  202          * DTrace Function Boundary Trace (fbt) probes are triggered
  203          * by int3 (0xcc) which causes the #BP (T_BPTFLT) breakpoint
  204          * interrupt. For all other trap types, just handle them in
  205          * the usual way.
  206          */
  207         cmpl    $T_BPTFLT,TF_TRAPNO(%rsp)
  208         jne     calltrap
  209 
  210         /* Check if there is no DTrace hook registered. */
  211         cmpq    $0,dtrace_invop_jump_addr
  212         je      calltrap
  213 
  214         /*
  215          * Set our jump address for the jump back in the event that
  216          * the breakpoint wasn't caused by DTrace at all.
  217          */
  218         movq    $calltrap,dtrace_invop_calltrap_addr(%rip)
  219 
  220         /* Jump to the code hooked in by DTrace. */
  221         movq    dtrace_invop_jump_addr,%rax
  222         jmpq    *dtrace_invop_jump_addr
  223 #endif
  224         .globl  calltrap
  225         .type   calltrap,@function
  226 calltrap:
  227         movq    %rsp,%rdi
  228         call    trap
  229         MEXITCOUNT
  230         jmp     doreti                  /* Handle any pending ASTs */
  231 
  232         /*
  233          * alltraps_noen entry point.  Unlike alltraps above, we want to
  234          * leave the interrupts disabled.  This corresponds to
  235          * SDT_SYS386IGT on the i386 port.
  236          */
  237         SUPERALIGN_TEXT
  238         .globl  alltraps_noen
  239         .type   alltraps_noen,@function
  240 alltraps_noen:
  241         movq    %rdi,TF_RDI(%rsp)
  242         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  243         jz      1f      /* already running with kernel GS.base */
  244         swapgs
  245         movq    PCPU(CURPCB),%rdi
  246         andl    $~PCB_FULL_IRET,PCB_FLAGS(%rdi)
  247 1:      movw    %fs,TF_FS(%rsp)
  248         movw    %gs,TF_GS(%rsp)
  249         movw    %es,TF_ES(%rsp)
  250         movw    %ds,TF_DS(%rsp)
  251         jmp     alltraps_pushregs_no_rdi
  252 
  253 IDTVEC(dblfault)
  254         subq    $TF_ERR,%rsp
  255         movl    $T_DOUBLEFLT,TF_TRAPNO(%rsp)
  256         movq    $0,TF_ADDR(%rsp)
  257         movq    $0,TF_ERR(%rsp)
  258         movq    %rdi,TF_RDI(%rsp)
  259         movq    %rsi,TF_RSI(%rsp)
  260         movq    %rdx,TF_RDX(%rsp)
  261         movq    %rcx,TF_RCX(%rsp)
  262         movq    %r8,TF_R8(%rsp)
  263         movq    %r9,TF_R9(%rsp)
  264         movq    %rax,TF_RAX(%rsp)
  265         movq    %rbx,TF_RBX(%rsp)
  266         movq    %rbp,TF_RBP(%rsp)
  267         movq    %r10,TF_R10(%rsp)
  268         movq    %r11,TF_R11(%rsp)
  269         movq    %r12,TF_R12(%rsp)
  270         movq    %r13,TF_R13(%rsp)
  271         movq    %r14,TF_R14(%rsp)
  272         movq    %r15,TF_R15(%rsp)
  273         movw    %fs,TF_FS(%rsp)
  274         movw    %gs,TF_GS(%rsp)
  275         movw    %es,TF_ES(%rsp)
  276         movw    %ds,TF_DS(%rsp)
  277         movl    $TF_HASSEGS,TF_FLAGS(%rsp)
  278         cld
  279         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  280         jz      1f                      /* already running with kernel GS.base */
  281         swapgs
  282 1:
  283         movq    %rsp,%rdi
  284         call    dblfault_handler
  285 2:
  286         hlt
  287         jmp     2b
  288 
  289 IDTVEC(page)
  290         subq    $TF_ERR,%rsp
  291         movl    $T_PAGEFLT,TF_TRAPNO(%rsp)
  292         movq    %rdi,TF_RDI(%rsp)       /* free up a GP register */
  293         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  294         jz      1f                      /* already running with kernel GS.base */
  295         swapgs
  296         movq    PCPU(CURPCB),%rdi
  297         andl    $~PCB_FULL_IRET,PCB_FLAGS(%rdi)
  298 1:      movq    %cr2,%rdi               /* preserve %cr2 before ..  */
  299         movq    %rdi,TF_ADDR(%rsp)      /* enabling interrupts. */
  300         movw    %fs,TF_FS(%rsp)
  301         movw    %gs,TF_GS(%rsp)
  302         movw    %es,TF_ES(%rsp)
  303         movw    %ds,TF_DS(%rsp)
  304         testl   $PSL_I,TF_RFLAGS(%rsp)
  305         jz      alltraps_pushregs_no_rdi
  306         sti
  307         jmp     alltraps_pushregs_no_rdi
  308 
  309         /*
  310          * We have to special-case this one.  If we get a trap in doreti() at
  311          * the iretq stage, we'll reenter with the wrong gs state.  We'll have
  312          * to do a special the swapgs in this case even coming from the kernel.
  313          * XXX linux has a trap handler for their equivalent of load_gs().
  314          */
  315 IDTVEC(prot)
  316         subq    $TF_ERR,%rsp
  317         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
  318         movq    $0,TF_ADDR(%rsp)
  319         movq    %rdi,TF_RDI(%rsp)       /* free up a GP register */
  320         leaq    doreti_iret(%rip),%rdi
  321         cmpq    %rdi,TF_RIP(%rsp)
  322         je      1f                      /* kernel but with user gsbase!! */
  323         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  324         jz      2f                      /* already running with kernel GS.base */
  325 1:      swapgs
  326 2:      movq    PCPU(CURPCB),%rdi
  327         orl     $PCB_FULL_IRET,PCB_FLAGS(%rdi)  /* always full iret from GPF */
  328         movw    %fs,TF_FS(%rsp)
  329         movw    %gs,TF_GS(%rsp)
  330         movw    %es,TF_ES(%rsp)
  331         movw    %ds,TF_DS(%rsp)
  332         testl   $PSL_I,TF_RFLAGS(%rsp)
  333         jz      alltraps_pushregs_no_rdi
  334         sti
  335         jmp     alltraps_pushregs_no_rdi
  336 
  337 /*
  338  * Fast syscall entry point.  We enter here with just our new %cs/%ss set,
  339  * and the new privilige level.  We are still running on the old user stack
  340  * pointer.  We have to juggle a few things around to find our stack etc.
  341  * swapgs gives us access to our PCPU space only.
  342  *
  343  * We do not support invoking this from a custom %cs or %ss (e.g. using
  344  * entries from an LDT).
  345  */
  346 IDTVEC(fast_syscall)
  347         swapgs
  348         movq    %rsp,PCPU(SCRATCH_RSP)
  349         movq    PCPU(RSP0),%rsp
  350         /* Now emulate a trapframe. Make the 8 byte alignment odd for call. */
  351         subq    $TF_SIZE,%rsp
  352         /* defer TF_RSP till we have a spare register */
  353         movq    %r11,TF_RFLAGS(%rsp)
  354         movq    %rcx,TF_RIP(%rsp)       /* %rcx original value is in %r10 */
  355         movq    PCPU(SCRATCH_RSP),%r11  /* %r11 already saved */
  356         movq    %r11,TF_RSP(%rsp)       /* user stack pointer */
  357         movw    %fs,TF_FS(%rsp)
  358         movw    %gs,TF_GS(%rsp)
  359         movw    %es,TF_ES(%rsp)
  360         movw    %ds,TF_DS(%rsp)
  361         movq    PCPU(CURPCB),%r11
  362         andl    $~PCB_FULL_IRET,PCB_FLAGS(%r11)
  363         sti
  364         movq    $KUDSEL,TF_SS(%rsp)
  365         movq    $KUCSEL,TF_CS(%rsp)
  366         movq    $2,TF_ERR(%rsp)
  367         movq    %rdi,TF_RDI(%rsp)       /* arg 1 */
  368         movq    %rsi,TF_RSI(%rsp)       /* arg 2 */
  369         movq    %rdx,TF_RDX(%rsp)       /* arg 3 */
  370         movq    %r10,TF_RCX(%rsp)       /* arg 4 */
  371         movq    %r8,TF_R8(%rsp)         /* arg 5 */
  372         movq    %r9,TF_R9(%rsp)         /* arg 6 */
  373         movq    %rax,TF_RAX(%rsp)       /* syscall number */
  374         movq    %rbx,TF_RBX(%rsp)       /* C preserved */
  375         movq    %rbp,TF_RBP(%rsp)       /* C preserved */
  376         movq    %r12,TF_R12(%rsp)       /* C preserved */
  377         movq    %r13,TF_R13(%rsp)       /* C preserved */
  378         movq    %r14,TF_R14(%rsp)       /* C preserved */
  379         movq    %r15,TF_R15(%rsp)       /* C preserved */
  380         movl    $TF_HASSEGS,TF_FLAGS(%rsp)
  381         cld
  382         FAKE_MCOUNT(TF_RIP(%rsp))
  383         movq    PCPU(CURTHREAD),%rdi
  384         movq    %rsp,TD_FRAME(%rdi)
  385         movl    TF_RFLAGS(%rsp),%esi
  386         andl    $PSL_T,%esi
  387         call    amd64_syscall
  388 1:      movq    PCPU(CURPCB),%rax
  389         /* Disable interrupts before testing PCB_FULL_IRET. */
  390         cli
  391         testl   $PCB_FULL_IRET,PCB_FLAGS(%rax)
  392         jnz     3f
  393         /* Check for and handle AST's on return to userland. */
  394         movq    PCPU(CURTHREAD),%rax
  395         testl   $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax)
  396         jne     2f
  397         /* Restore preserved registers. */
  398         MEXITCOUNT
  399         movq    TF_RDI(%rsp),%rdi       /* bonus; preserve arg 1 */
  400         movq    TF_RSI(%rsp),%rsi       /* bonus: preserve arg 2 */
  401         movq    TF_RDX(%rsp),%rdx       /* return value 2 */
  402         movq    TF_RAX(%rsp),%rax       /* return value 1 */
  403         movq    TF_RFLAGS(%rsp),%r11    /* original %rflags */
  404         movq    TF_RIP(%rsp),%rcx       /* original %rip */
  405         movq    TF_RSP(%rsp),%rsp       /* user stack pointer */
  406         swapgs
  407         sysretq
  408 
  409 2:      /* AST scheduled. */
  410         sti
  411         movq    %rsp,%rdi
  412         call    ast
  413         jmp     1b
  414 
  415 3:      /* Requested full context restore, use doreti for that. */
  416         MEXITCOUNT
  417         jmp     doreti
  418 
  419 /*
  420  * Here for CYA insurance, in case a "syscall" instruction gets
  421  * issued from 32 bit compatability mode. MSR_CSTAR has to point
  422  * to *something* if EFER_SCE is enabled.
  423  */
  424 IDTVEC(fast_syscall32)
  425         sysret
  426 
  427 /*
  428  * NMI handling is special.
  429  *
  430  * First, NMIs do not respect the state of the processor's RFLAGS.IF
  431  * bit.  The NMI handler may be entered at any time, including when
  432  * the processor is in a critical section with RFLAGS.IF == 0.
  433  * The processor's GS.base value could be invalid on entry to the
  434  * handler.
  435  *
  436  * Second, the processor treats NMIs specially, blocking further NMIs
  437  * until an 'iretq' instruction is executed.  We thus need to execute
  438  * the NMI handler with interrupts disabled, to prevent a nested interrupt
  439  * from executing an 'iretq' instruction and inadvertently taking the
  440  * processor out of NMI mode.
  441  *
  442  * Third, the NMI handler runs on its own stack (tss_ist2). The canonical
  443  * GS.base value for the processor is stored just above the bottom of its
  444  * NMI stack.  For NMIs taken from kernel mode, the current value in
  445  * the processor's GS.base is saved at entry to C-preserved register %r12,
  446  * the canonical value for GS.base is then loaded into the processor, and
  447  * the saved value is restored at exit time.  For NMIs taken from user mode,
  448  * the cheaper 'SWAPGS' instructions are used for swapping GS.base.
  449  */
  450 
  451 IDTVEC(nmi)
  452         subq    $TF_RIP,%rsp
  453         movl    $(T_NMI),TF_TRAPNO(%rsp)
  454         movq    $0,TF_ADDR(%rsp)
  455         movq    $0,TF_ERR(%rsp)
  456         movq    %rdi,TF_RDI(%rsp)
  457         movq    %rsi,TF_RSI(%rsp)
  458         movq    %rdx,TF_RDX(%rsp)
  459         movq    %rcx,TF_RCX(%rsp)
  460         movq    %r8,TF_R8(%rsp)
  461         movq    %r9,TF_R9(%rsp)
  462         movq    %rax,TF_RAX(%rsp)
  463         movq    %rbx,TF_RBX(%rsp)
  464         movq    %rbp,TF_RBP(%rsp)
  465         movq    %r10,TF_R10(%rsp)
  466         movq    %r11,TF_R11(%rsp)
  467         movq    %r12,TF_R12(%rsp)
  468         movq    %r13,TF_R13(%rsp)
  469         movq    %r14,TF_R14(%rsp)
  470         movq    %r15,TF_R15(%rsp)
  471         movw    %fs,TF_FS(%rsp)
  472         movw    %gs,TF_GS(%rsp)
  473         movw    %es,TF_ES(%rsp)
  474         movw    %ds,TF_DS(%rsp)
  475         movl    $TF_HASSEGS,TF_FLAGS(%rsp)
  476         cld
  477         xorl    %ebx,%ebx
  478         testb   $SEL_RPL_MASK,TF_CS(%rsp)
  479         jnz     nmi_fromuserspace
  480         /*
  481          * We've interrupted the kernel.  Preserve GS.base in %r12.
  482          */
  483         movl    $MSR_GSBASE,%ecx
  484         rdmsr
  485         movq    %rax,%r12
  486         shlq    $32,%rdx
  487         orq     %rdx,%r12
  488         /* Retrieve and load the canonical value for GS.base. */
  489         movq    TF_SIZE(%rsp),%rdx
  490         movl    %edx,%eax
  491         shrq    $32,%rdx
  492         wrmsr
  493         jmp     nmi_calltrap
  494 nmi_fromuserspace:
  495         incl    %ebx
  496         swapgs
  497 /* Note: this label is also used by ddb and gdb: */
  498 nmi_calltrap:
  499         FAKE_MCOUNT(TF_RIP(%rsp))
  500         movq    %rsp,%rdi
  501         call    trap
  502         MEXITCOUNT
  503 #ifdef HWPMC_HOOKS
  504         /*
  505          * Capture a userspace callchain if needed.
  506          * 
  507          * - Check if the current trap was from user mode.
  508          * - Check if the current thread is valid.
  509          * - Check if the thread requires a user call chain to be
  510          *   captured.
  511          *
  512          * We are still in NMI mode at this point.
  513          */
  514         testl   %ebx,%ebx
  515         jz      nocallchain     /* not from userspace */
  516         movq    PCPU(CURTHREAD),%rax
  517         orq     %rax,%rax       /* curthread present? */
  518         jz      nocallchain
  519         testl   $TDP_CALLCHAIN,TD_PFLAGS(%rax) /* flagged for capture? */
  520         jz      nocallchain
  521         /*
  522          * A user callchain is to be captured, so:
  523          * - Move execution to the regular kernel stack, to allow for
  524          *   nested NMI interrupts.
  525          * - Take the processor out of "NMI" mode by faking an "iret".
  526          * - Enable interrupts, so that copyin() can work.
  527          */
  528         movq    %rsp,%rsi       /* source stack pointer */
  529         movq    $TF_SIZE,%rcx
  530         movq    PCPU(RSP0),%rdx
  531         subq    %rcx,%rdx
  532         movq    %rdx,%rdi       /* destination stack pointer */
  533 
  534         shrq    $3,%rcx         /* trap frame size in long words */
  535         cld
  536         rep
  537         movsq                   /* copy trapframe */
  538 
  539         movl    %ss,%eax
  540         pushq   %rax            /* tf_ss */
  541         pushq   %rdx            /* tf_rsp (on kernel stack) */
  542         pushfq                  /* tf_rflags */
  543         movl    %cs,%eax
  544         pushq   %rax            /* tf_cs */
  545         pushq   $outofnmi       /* tf_rip */
  546         iretq
  547 outofnmi:
  548         /*
  549          * At this point the processor has exited NMI mode and is running
  550          * with interrupts turned off on the normal kernel stack.
  551          *
  552          * If a pending NMI gets recognized at or after this point, it 
  553          * will cause a kernel callchain to be traced.
  554          *
  555          * We turn interrupts back on, and call the user callchain capture hook.
  556          */
  557         movq    pmc_hook,%rax
  558         orq     %rax,%rax
  559         jz      nocallchain
  560         movq    PCPU(CURTHREAD),%rdi            /* thread */
  561         movq    $PMC_FN_USER_CALLCHAIN,%rsi     /* command */
  562         movq    %rsp,%rdx                       /* frame */
  563         sti
  564         call    *%rax
  565         cli
  566 nocallchain:
  567 #endif
  568         testl   %ebx,%ebx
  569         jnz     doreti_exit
  570 nmi_kernelexit: 
  571         /*
  572          * Put back the preserved MSR_GSBASE value.
  573          */
  574         movl    $MSR_GSBASE,%ecx
  575         movq    %r12,%rdx
  576         movl    %edx,%eax
  577         shrq    $32,%rdx
  578         wrmsr
  579 nmi_restoreregs:
  580         movq    TF_RDI(%rsp),%rdi
  581         movq    TF_RSI(%rsp),%rsi
  582         movq    TF_RDX(%rsp),%rdx
  583         movq    TF_RCX(%rsp),%rcx
  584         movq    TF_R8(%rsp),%r8
  585         movq    TF_R9(%rsp),%r9
  586         movq    TF_RAX(%rsp),%rax
  587         movq    TF_RBX(%rsp),%rbx
  588         movq    TF_RBP(%rsp),%rbp
  589         movq    TF_R10(%rsp),%r10
  590         movq    TF_R11(%rsp),%r11
  591         movq    TF_R12(%rsp),%r12
  592         movq    TF_R13(%rsp),%r13
  593         movq    TF_R14(%rsp),%r14
  594         movq    TF_R15(%rsp),%r15
  595         addq    $TF_RIP,%rsp
  596         jmp     doreti_iret
  597 
  598 ENTRY(fork_trampoline)
  599         movq    %r12,%rdi               /* function */
  600         movq    %rbx,%rsi               /* arg1 */
  601         movq    %rsp,%rdx               /* trapframe pointer */
  602         call    fork_exit
  603         MEXITCOUNT
  604         jmp     doreti                  /* Handle any ASTs */
  605 
  606 /*
  607  * To efficiently implement classification of trap and interrupt handlers
  608  * for profiling, there must be only trap handlers between the labels btrap
  609  * and bintr, and only interrupt handlers between the labels bintr and
  610  * eintr.  This is implemented (partly) by including files that contain
  611  * some of the handlers.  Before including the files, set up a normal asm
  612  * environment so that the included files doen't need to know that they are
  613  * included.
  614  */
  615 
  616 #ifdef COMPAT_FREEBSD32
  617         .data
  618         .p2align 4
  619         .text
  620         SUPERALIGN_TEXT
  621 
  622 #include <amd64/ia32/ia32_exception.S>
  623 #endif
  624 
  625         .data
  626         .p2align 4
  627         .text
  628         SUPERALIGN_TEXT
  629 MCOUNT_LABEL(bintr)
  630 
  631 #include <amd64/amd64/apic_vector.S>
  632 
  633 #ifdef DEV_ATPIC
  634         .data
  635         .p2align 4
  636         .text
  637         SUPERALIGN_TEXT
  638 
  639 #include <amd64/amd64/atpic_vector.S>
  640 #endif
  641 
  642         .text
  643 MCOUNT_LABEL(eintr)
  644 
  645 /*
  646  * void doreti(struct trapframe)
  647  *
  648  * Handle return from interrupts, traps and syscalls.
  649  */
  650         .text
  651         SUPERALIGN_TEXT
  652         .type   doreti,@function
  653 doreti:
  654         FAKE_MCOUNT($bintr)             /* init "from" bintr -> doreti */
  655         /*
  656          * Check if ASTs can be handled now.
  657          */
  658         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* are we returning to user mode? */
  659         jz      doreti_exit             /* can't handle ASTs now if not */
  660 
  661 doreti_ast:
  662         /*
  663          * Check for ASTs atomically with returning.  Disabling CPU
  664          * interrupts provides sufficient locking even in the SMP case,
  665          * since we will be informed of any new ASTs by an IPI.
  666          */
  667         cli
  668         movq    PCPU(CURTHREAD),%rax
  669         testl   $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax)
  670         je      doreti_exit
  671         sti
  672         movq    %rsp,%rdi       /* pass a pointer to the trapframe */
  673         call    ast
  674         jmp     doreti_ast
  675 
  676         /*
  677          * doreti_exit: pop registers, iret.
  678          *
  679          *      The segment register pop is a special case, since it may
  680          *      fault if (for example) a sigreturn specifies bad segment
  681          *      registers.  The fault is handled in trap.c.
  682          */
  683 doreti_exit:
  684         MEXITCOUNT
  685         movq    PCPU(CURPCB),%r8
  686 
  687         /*
  688          * Do not reload segment registers for kernel.
  689          * Since we do not reload segments registers with sane
  690          * values on kernel entry, descriptors referenced by
  691          * segments registers might be not valid.  This is fatal
  692          * for user mode, but is not a problem for the kernel.
  693          */
  694         testb   $SEL_RPL_MASK,TF_CS(%rsp)
  695         jz      ld_regs
  696         testl   $PCB_FULL_IRET,PCB_FLAGS(%r8)
  697         jz      ld_regs
  698         testl   $TF_HASSEGS,TF_FLAGS(%rsp)
  699         je      set_segs
  700 
  701 do_segs:
  702         /* Restore %fs and fsbase */
  703         movw    TF_FS(%rsp),%ax
  704         .globl  ld_fs
  705 ld_fs:
  706         movw    %ax,%fs
  707         cmpw    $KUF32SEL,%ax
  708         jne     1f
  709         movl    $MSR_FSBASE,%ecx
  710         movl    PCB_FSBASE(%r8),%eax
  711         movl    PCB_FSBASE+4(%r8),%edx
  712         .globl  ld_fsbase
  713 ld_fsbase:
  714         wrmsr
  715 1:
  716         /* Restore %gs and gsbase */
  717         movw    TF_GS(%rsp),%si
  718         pushfq
  719         cli
  720         movl    $MSR_GSBASE,%ecx
  721         rdmsr
  722         .globl  ld_gs
  723 ld_gs:
  724         movw    %si,%gs
  725         wrmsr
  726         popfq
  727         cmpw    $KUG32SEL,%si
  728         jne     1f
  729         movl    $MSR_KGSBASE,%ecx
  730         movl    PCB_GSBASE(%r8),%eax
  731         movl    PCB_GSBASE+4(%r8),%edx
  732         .globl  ld_gsbase
  733 ld_gsbase:
  734         wrmsr
  735 1:
  736         .globl  ld_es
  737 ld_es:
  738         movw    TF_ES(%rsp),%es
  739         .globl  ld_ds
  740 ld_ds:
  741         movw    TF_DS(%rsp),%ds
  742 ld_regs:
  743         movq    TF_RDI(%rsp),%rdi
  744         movq    TF_RSI(%rsp),%rsi
  745         movq    TF_RDX(%rsp),%rdx
  746         movq    TF_RCX(%rsp),%rcx
  747         movq    TF_R8(%rsp),%r8
  748         movq    TF_R9(%rsp),%r9
  749         movq    TF_RAX(%rsp),%rax
  750         movq    TF_RBX(%rsp),%rbx
  751         movq    TF_RBP(%rsp),%rbp
  752         movq    TF_R10(%rsp),%r10
  753         movq    TF_R11(%rsp),%r11
  754         movq    TF_R12(%rsp),%r12
  755         movq    TF_R13(%rsp),%r13
  756         movq    TF_R14(%rsp),%r14
  757         movq    TF_R15(%rsp),%r15
  758         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  759         jz      1f                      /* keep running with kernel GS.base */
  760         cli
  761         swapgs
  762 1:
  763         addq    $TF_RIP,%rsp            /* skip over tf_err, tf_trapno */
  764         .globl  doreti_iret
  765 doreti_iret:
  766         iretq
  767 
  768 set_segs:
  769         movw    $KUDSEL,%ax
  770         movw    %ax,TF_DS(%rsp)
  771         movw    %ax,TF_ES(%rsp)
  772         movw    $KUF32SEL,TF_FS(%rsp)
  773         movw    $KUG32SEL,TF_GS(%rsp)
  774         jmp     do_segs
  775 
  776         /*
  777          * doreti_iret_fault.  Alternative return code for
  778          * the case where we get a fault in the doreti_exit code
  779          * above.  trap() (amd64/amd64/trap.c) catches this specific
  780          * case, sends the process a signal and continues in the
  781          * corresponding place in the code below.
  782          */
  783         ALIGN_TEXT
  784         .globl  doreti_iret_fault
  785 doreti_iret_fault:
  786         subq    $TF_RIP,%rsp            /* space including tf_err, tf_trapno */
  787         testl   $PSL_I,TF_RFLAGS(%rsp)
  788         jz      1f
  789         sti
  790 1:
  791         movw    %fs,TF_FS(%rsp)
  792         movw    %gs,TF_GS(%rsp)
  793         movw    %es,TF_ES(%rsp)
  794         movw    %ds,TF_DS(%rsp)
  795         movl    $TF_HASSEGS,TF_FLAGS(%rsp)
  796         movq    %rdi,TF_RDI(%rsp)
  797         movq    %rsi,TF_RSI(%rsp)
  798         movq    %rdx,TF_RDX(%rsp)
  799         movq    %rcx,TF_RCX(%rsp)
  800         movq    %r8,TF_R8(%rsp)
  801         movq    %r9,TF_R9(%rsp)
  802         movq    %rax,TF_RAX(%rsp)
  803         movq    %rbx,TF_RBX(%rsp)
  804         movq    %rbp,TF_RBP(%rsp)
  805         movq    %r10,TF_R10(%rsp)
  806         movq    %r11,TF_R11(%rsp)
  807         movq    %r12,TF_R12(%rsp)
  808         movq    %r13,TF_R13(%rsp)
  809         movq    %r14,TF_R14(%rsp)
  810         movq    %r15,TF_R15(%rsp)
  811         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
  812         movq    $0,TF_ERR(%rsp) /* XXX should be the error code */
  813         movq    $0,TF_ADDR(%rsp)
  814         FAKE_MCOUNT(TF_RIP(%rsp))
  815         jmp     calltrap
  816 
  817         ALIGN_TEXT
  818         .globl  ds_load_fault
  819 ds_load_fault:
  820         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
  821         movq    %rsp,%rdi
  822         call    trap
  823         movw    $KUDSEL,TF_DS(%rsp)
  824         jmp     doreti
  825 
  826         ALIGN_TEXT
  827         .globl  es_load_fault
  828 es_load_fault:
  829         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
  830         movq    %rsp,%rdi
  831         call    trap
  832         movw    $KUDSEL,TF_ES(%rsp)
  833         jmp     doreti
  834 
  835         ALIGN_TEXT
  836         .globl  fs_load_fault
  837 fs_load_fault:
  838         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
  839         movq    %rsp,%rdi
  840         call    trap
  841         movw    $KUF32SEL,TF_FS(%rsp)
  842         jmp     doreti
  843 
  844         ALIGN_TEXT
  845         .globl  gs_load_fault
  846 gs_load_fault:
  847         popfq
  848         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
  849         movq    %rsp,%rdi
  850         call    trap
  851         movw    $KUG32SEL,TF_GS(%rsp)
  852         jmp     doreti
  853 
  854         ALIGN_TEXT
  855         .globl  fsbase_load_fault
  856 fsbase_load_fault:
  857         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
  858         movq    %rsp,%rdi
  859         call    trap
  860         movq    PCPU(CURTHREAD),%r8
  861         movq    TD_PCB(%r8),%r8
  862         movq    $0,PCB_FSBASE(%r8)
  863         jmp     doreti
  864 
  865         ALIGN_TEXT
  866         .globl  gsbase_load_fault
  867 gsbase_load_fault:
  868         movl    $T_PROTFLT,TF_TRAPNO(%rsp)
  869         movq    %rsp,%rdi
  870         call    trap
  871         movq    PCPU(CURTHREAD),%r8
  872         movq    TD_PCB(%r8),%r8
  873         movq    $0,PCB_GSBASE(%r8)
  874         jmp     doreti
  875 
  876 #ifdef HWPMC_HOOKS
  877         ENTRY(end_exceptions)
  878 #endif

Cache object: 5979efa6a2cea75f2ed933f484765439


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.