The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/exception.S

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1989, 1990 William F. Jolitz.
    3  * Copyright (c) 1990 The Regents of the University of California.
    4  * All rights reserved.
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice, this list of conditions and the following disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  * 4. Neither the name of the University nor the names of its contributors
   15  *    may be used to endorse or promote products derived from this software
   16  *    without specific prior written permission.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  *
   30  * $FreeBSD: src/sys/amd64/amd64/exception.S,v 1.120.2.2 2005/06/23 19:46:29 peter Exp $
   31  */
   32 
   33 #include "opt_atpic.h"
   34 #include "opt_compat.h"
   35 
   36 #include <machine/asmacros.h>
   37 #include <machine/psl.h>
   38 #include <machine/trap.h>
   39 
   40 #include "assym.s"
   41 
   42         .text
   43 
   44 /*****************************************************************************/
   45 /* Trap handling                                                             */
   46 /*****************************************************************************/
   47 /*
   48  * Trap and fault vector routines.
   49  *
   50  * All traps are 'interrupt gates', SDT_SYSIGT.  An interrupt gate pushes
   51  * state on the stack but also disables interrupts.  This is important for
   52  * us for the use of the swapgs instruction.  We cannot be interrupted
   53  * until the GS.base value is correct.  For most traps, we automatically
   54  * then enable interrupts if the interrupted context had them enabled.
   55  * This is equivalent to the i386 port's use of SDT_SYS386TGT.
   56  *
   57  * The cpu will push a certain amount of state onto the kernel stack for
   58  * the current process.  See amd64/include/frame.h.  
   59  * This includes the current RFLAGS (status register, which includes 
   60  * the interrupt disable state prior to the trap), the code segment register,
   61  * and the return instruction pointer are pushed by the cpu.  The cpu 
   62  * will also push an 'error' code for certain traps.  We push a dummy 
   63  * error code for those traps where the cpu doesn't in order to maintain 
   64  * a consistent frame.  We also push a contrived 'trap number'.
   65  *
   66  * The cpu does not push the general registers, we must do that, and we 
   67  * must restore them prior to calling 'iret'.  The cpu adjusts the %cs and
   68  * %ss segment registers, but does not mess with %ds, %es, or %fs.  Thus we
   69  * must load them with appropriate values for supervisor mode operation.
   70  */
   71 
   72 MCOUNT_LABEL(user)
   73 MCOUNT_LABEL(btrap)
   74 
   75 /* Traps that we leave interrupts disabled for.. */
   76 #define TRAP_NOEN(a)    \
   77         subq $TF_RIP,%rsp; \
   78         movq $(a),TF_TRAPNO(%rsp) ; \
   79         movq $0,TF_ADDR(%rsp) ; \
   80         movq $0,TF_ERR(%rsp) ; \
   81         jmp alltraps_noen
   82 IDTVEC(dbg)
   83         TRAP_NOEN(T_TRCTRAP)
   84 IDTVEC(bpt)
   85         TRAP_NOEN(T_BPTFLT)
   86 
   87 /* Regular traps; The cpu does not supply tf_err for these. */
   88 #define TRAP(a)  \
   89         subq $TF_RIP,%rsp; \
   90         movq $(a),TF_TRAPNO(%rsp) ; \
   91         movq $0,TF_ADDR(%rsp) ; \
   92         movq $0,TF_ERR(%rsp) ; \
   93         jmp alltraps
   94 IDTVEC(div)
   95         TRAP(T_DIVIDE)
   96 IDTVEC(nmi)
   97         TRAP(T_NMI)
   98 IDTVEC(ofl)
   99         TRAP(T_OFLOW)
  100 IDTVEC(bnd)
  101         TRAP(T_BOUND)
  102 IDTVEC(ill)
  103         TRAP(T_PRIVINFLT)
  104 IDTVEC(dna)
  105         TRAP(T_DNA)
  106 IDTVEC(fpusegm)
  107         TRAP(T_FPOPFLT)
  108 IDTVEC(mchk)
  109         TRAP(T_MCHK)
  110 IDTVEC(rsvd)
  111         TRAP(T_RESERVED)
  112 IDTVEC(fpu)
  113         TRAP(T_ARITHTRAP)
  114 IDTVEC(xmm)
  115         TRAP(T_XMMFLT)
  116 
  117 /* This group of traps have tf_err already pushed by the cpu */
  118 #define TRAP_ERR(a)     \
  119         subq $TF_ERR,%rsp; \
  120         movq $(a),TF_TRAPNO(%rsp) ; \
  121         movq $0,TF_ADDR(%rsp) ; \
  122         jmp alltraps
  123 IDTVEC(tss)
  124         TRAP_ERR(T_TSSFLT)
  125 IDTVEC(missing)
  126         TRAP_ERR(T_SEGNPFLT)
  127 IDTVEC(stk)
  128         TRAP_ERR(T_STKFLT)
  129 IDTVEC(prot)
  130         TRAP_ERR(T_PROTFLT)
  131 IDTVEC(align)
  132         TRAP_ERR(T_ALIGNFLT)
  133 
  134         /*
  135          * alltraps entry point.  Use swapgs if this is the first time in the
  136          * kernel from userland.  Reenable interrupts if they were enabled
  137          * before the trap.  This approximates SDT_SYS386TGT on the i386 port.
  138          */
  139 
  140         SUPERALIGN_TEXT
  141         .globl  alltraps
  142         .type   alltraps,@function
  143 alltraps:
  144         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  145         jz      alltraps_testi          /* already running with kernel GS.base */
  146         swapgs
  147 alltraps_testi:
  148         testl   $PSL_I,TF_RFLAGS(%rsp)
  149         jz      alltraps_pushregs
  150         sti
  151 alltraps_pushregs:
  152         movq    %rdi,TF_RDI(%rsp)
  153 alltraps_pushregs_no_rdi:
  154         movq    %rsi,TF_RSI(%rsp)
  155         movq    %rdx,TF_RDX(%rsp)
  156         movq    %rcx,TF_RCX(%rsp)
  157         movq    %r8,TF_R8(%rsp)
  158         movq    %r9,TF_R9(%rsp)
  159         movq    %rax,TF_RAX(%rsp)
  160         movq    %rbx,TF_RBX(%rsp)
  161         movq    %rbp,TF_RBP(%rsp)
  162         movq    %r10,TF_R10(%rsp)
  163         movq    %r11,TF_R11(%rsp)
  164         movq    %r12,TF_R12(%rsp)
  165         movq    %r13,TF_R13(%rsp)
  166         movq    %r14,TF_R14(%rsp)
  167         movq    %r15,TF_R15(%rsp)
  168 alltraps_with_regs_pushed:
  169         FAKE_MCOUNT(TF_RIP(%rsp))
  170 calltrap:
  171         call    trap
  172         MEXITCOUNT
  173         jmp     doreti                  /* Handle any pending ASTs */
  174 
  175         /*
  176          * alltraps_noen entry point.  Unlike alltraps above, we want to
  177          * leave the interrupts disabled.  This corresponds to
  178          * SDT_SYS386IGT on the i386 port.
  179          */
  180         SUPERALIGN_TEXT
  181         .globl  alltraps_noen
  182         .type   alltraps_noen,@function
  183 alltraps_noen:
  184         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  185         jz      alltraps_pushregs       /* already running with kernel GS.base */
  186         swapgs
  187         jmp     alltraps_pushregs
  188 
  189 IDTVEC(dblfault)
  190         subq    $TF_ERR,%rsp
  191         movq    $T_DOUBLEFLT,TF_TRAPNO(%rsp)
  192         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  193         jz      1f                      /* already running with kernel GS.base */
  194         swapgs
  195 1:      call    dblfault_handler
  196 2:      hlt
  197         jmp     2b
  198 
  199 IDTVEC(page)
  200         subq    $TF_ERR,%rsp
  201         movq    $T_PAGEFLT,TF_TRAPNO(%rsp)
  202         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  203         jz      1f                      /* already running with kernel GS.base */
  204         swapgs
  205 1:      movq    %rdi,TF_RDI(%rsp)       /* free up a GP register */
  206         movq    %cr2,%rdi               /* preserve %cr2 before ..  */
  207         movq    %rdi,TF_ADDR(%rsp)      /* enabling interrupts. */
  208         testl   $PSL_I,TF_RFLAGS(%rsp)
  209         jz      alltraps_pushregs_no_rdi
  210         sti
  211         jmp     alltraps_pushregs_no_rdi
  212 
  213 /*
  214  * Fast syscall entry point.  We enter here with just our new %cs/%ss set,
  215  * and the new privilige level.  We are still running on the old user stack
  216  * pointer.  We have to juggle a few things around to find our stack etc.
  217  * swapgs gives us access to our PCPU space only.
  218  */
  219 IDTVEC(fast_syscall)
  220         swapgs
  221         movq    %rsp,PCPU(SCRATCH_RSP)
  222         movq    PCPU(RSP0),%rsp
  223         /* Now emulate a trapframe. Make the 8 byte alignment odd for call. */
  224         subq    $TF_SIZE,%rsp
  225         /* defer TF_RSP till we have a spare register */
  226         movq    %r11,TF_RFLAGS(%rsp)
  227         movq    %rcx,TF_RIP(%rsp)       /* %rcx original value is in %r10 */
  228         movq    PCPU(SCRATCH_RSP),%r11  /* %r11 already saved */
  229         movq    %r11,TF_RSP(%rsp)       /* user stack pointer */
  230         sti
  231         movq    $KUDSEL,TF_SS(%rsp)
  232         movq    $KUCSEL,TF_CS(%rsp)
  233         movq    $2,TF_ERR(%rsp)
  234         movq    %rdi,TF_RDI(%rsp)       /* arg 1 */
  235         movq    %rsi,TF_RSI(%rsp)       /* arg 2 */
  236         movq    %rdx,TF_RDX(%rsp)       /* arg 3 */
  237         movq    %r10,TF_RCX(%rsp)       /* arg 4 */
  238         movq    %r8,TF_R8(%rsp)         /* arg 5 */
  239         movq    %r9,TF_R9(%rsp)         /* arg 6 */
  240         movq    %rax,TF_RAX(%rsp)       /* syscall number */
  241         movq    %rbx,TF_RBX(%rsp)       /* C preserved */
  242         movq    %rbp,TF_RBP(%rsp)       /* C preserved */
  243         movq    %r12,TF_R12(%rsp)       /* C preserved */
  244         movq    %r13,TF_R13(%rsp)       /* C preserved */
  245         movq    %r14,TF_R14(%rsp)       /* C preserved */
  246         movq    %r15,TF_R15(%rsp)       /* C preserved */
  247         FAKE_MCOUNT(TF_RIP(%rsp))
  248         call    syscall
  249         movq    PCPU(CURPCB),%rax
  250         testq   $PCB_FULLCTX,PCB_FLAGS(%rax)
  251         jne     3f
  252 1:      /* Check for and handle AST's on return to userland */
  253         cli
  254         movq    PCPU(CURTHREAD),%rax
  255         testl   $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax)
  256         je      2f
  257         sti
  258         movq    %rsp, %rdi
  259         call    ast
  260         jmp     1b
  261 2:      /* restore preserved registers */
  262         MEXITCOUNT
  263         movq    TF_RDI(%rsp),%rdi       /* bonus; preserve arg 1 */
  264         movq    TF_RSI(%rsp),%rsi       /* bonus: preserve arg 2 */
  265         movq    TF_RDX(%rsp),%rdx       /* return value 2 */
  266         movq    TF_RAX(%rsp),%rax       /* return value 1 */
  267         movq    TF_RBX(%rsp),%rbx       /* C preserved */
  268         movq    TF_RBP(%rsp),%rbp       /* C preserved */
  269         movq    TF_R12(%rsp),%r12       /* C preserved */
  270         movq    TF_R13(%rsp),%r13       /* C preserved */
  271         movq    TF_R14(%rsp),%r14       /* C preserved */
  272         movq    TF_R15(%rsp),%r15       /* C preserved */
  273         movq    TF_RFLAGS(%rsp),%r11    /* original %rflags */
  274         movq    TF_RIP(%rsp),%rcx       /* original %rip */
  275         movq    TF_RSP(%rsp),%r9        /* user stack pointer */
  276         movq    %r9,%rsp                /* original %rsp */
  277         swapgs
  278         sysretq
  279 3:      /* Requested full context restore, use doreti for that */
  280         andq    $~PCB_FULLCTX,PCB_FLAGS(%rax)
  281         MEXITCOUNT
  282         jmp     doreti
  283 
  284 /*
  285  * Here for CYA insurance, in case a "syscall" instruction gets
  286  * issued from 32 bit compatability mode. MSR_CSTAR has to point
  287  * to *something* if EFER_SCE is enabled.
  288  */
  289 IDTVEC(fast_syscall32)
  290         sysret
  291 
  292 ENTRY(fork_trampoline)
  293         movq    %r12, %rdi              /* function */
  294         movq    %rbx, %rsi              /* arg1 */
  295         movq    %rsp, %rdx              /* trapframe pointer */
  296         call    fork_exit
  297         MEXITCOUNT
  298         jmp     doreti                  /* Handle any ASTs */
  299 
  300 /*
  301  * To efficiently implement classification of trap and interrupt handlers
  302  * for profiling, there must be only trap handlers between the labels btrap
  303  * and bintr, and only interrupt handlers between the labels bintr and
  304  * eintr.  This is implemented (partly) by including files that contain
  305  * some of the handlers.  Before including the files, set up a normal asm
  306  * environment so that the included files doen't need to know that they are
  307  * included.
  308  */
  309 
  310 #ifdef COMPAT_IA32
  311         .data
  312         .p2align 4
  313         .text
  314         SUPERALIGN_TEXT
  315 
  316 #include <amd64/ia32/ia32_exception.S>
  317 #endif
  318 
  319         .data
  320         .p2align 4
  321         .text
  322         SUPERALIGN_TEXT
  323 MCOUNT_LABEL(bintr)
  324 
  325 #include <amd64/amd64/apic_vector.S>
  326 
  327 #ifdef DEV_ATPIC
  328         .data
  329         .p2align 4
  330         .text
  331         SUPERALIGN_TEXT
  332 
  333 #include <amd64/isa/atpic_vector.S>
  334 #endif
  335 
  336         .text
  337 MCOUNT_LABEL(eintr)
  338 
  339 /*
  340  * void doreti(struct trapframe)
  341  *
  342  * Handle return from interrupts, traps and syscalls.
  343  */
  344         .text
  345         SUPERALIGN_TEXT
  346         .type   doreti,@function
  347 doreti:
  348         FAKE_MCOUNT($bintr)             /* init "from" bintr -> doreti */
  349         /*
  350          * Check if ASTs can be handled now.
  351          */
  352         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* are we returning to user mode? */
  353         jz      doreti_exit             /* can't handle ASTs now if not */
  354 
  355 doreti_ast:
  356         /*
  357          * Check for ASTs atomically with returning.  Disabling CPU
  358          * interrupts provides sufficient locking eve in the SMP case,
  359          * since we will be informed of any new ASTs by an IPI.
  360          */
  361         cli
  362         movq    PCPU(CURTHREAD),%rax
  363         testl   $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax)
  364         je      doreti_exit
  365         sti
  366         movq    %rsp, %rdi                      /* pass a pointer to the trapframe */
  367         call    ast
  368         jmp     doreti_ast
  369 
  370         /*
  371          * doreti_exit: pop registers, iret.
  372          *
  373          *      The segment register pop is a special case, since it may
  374          *      fault if (for example) a sigreturn specifies bad segment
  375          *      registers.  The fault is handled in trap.c.
  376          */
  377 doreti_exit:
  378         MEXITCOUNT
  379         movq    TF_RDI(%rsp),%rdi
  380         movq    TF_RSI(%rsp),%rsi
  381         movq    TF_RDX(%rsp),%rdx
  382         movq    TF_RCX(%rsp),%rcx
  383         movq    TF_R8(%rsp),%r8
  384         movq    TF_R9(%rsp),%r9
  385         movq    TF_RAX(%rsp),%rax
  386         movq    TF_RBX(%rsp),%rbx
  387         movq    TF_RBP(%rsp),%rbp
  388         movq    TF_R10(%rsp),%r10
  389         movq    TF_R11(%rsp),%r11
  390         movq    TF_R12(%rsp),%r12
  391         movq    TF_R13(%rsp),%r13
  392         movq    TF_R14(%rsp),%r14
  393         movq    TF_R15(%rsp),%r15
  394         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  395         jz      1f                      /* keep running with kernel GS.base */
  396         cli
  397         swapgs
  398 1:      addq    $TF_RIP,%rsp            /* skip over tf_err, tf_trapno */
  399         .globl  doreti_iret
  400 doreti_iret:
  401         iretq
  402 
  403         /*
  404          * doreti_iret_fault and friends.  Alternative return code for
  405          * the case where we get a fault in the doreti_exit code
  406          * above.  trap() (i386/i386/trap.c) catches this specific
  407          * case, sends the process a signal and continues in the
  408          * corresponding place in the code below.
  409          */
  410         ALIGN_TEXT
  411         .globl  doreti_iret_fault
  412 doreti_iret_fault:
  413         subq    $TF_RIP,%rsp            /* space including tf_err, tf_trapno */
  414         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  415         jz      1f                      /* already running with kernel GS.base */
  416         swapgs
  417 1:      testl   $PSL_I,TF_RFLAGS(%rsp)
  418         jz      2f
  419         sti
  420 2:      movq    %rdi,TF_RDI(%rsp)
  421         movq    %rsi,TF_RSI(%rsp)
  422         movq    %rdx,TF_RDX(%rsp)
  423         movq    %rcx,TF_RCX(%rsp)
  424         movq    %r8,TF_R8(%rsp)
  425         movq    %r9,TF_R9(%rsp)
  426         movq    %rax,TF_RAX(%rsp)
  427         movq    %rbx,TF_RBX(%rsp)
  428         movq    %rbp,TF_RBP(%rsp)
  429         movq    %r10,TF_R10(%rsp)
  430         movq    %r11,TF_R11(%rsp)
  431         movq    %r12,TF_R12(%rsp)
  432         movq    %r13,TF_R13(%rsp)
  433         movq    %r14,TF_R14(%rsp)
  434         movq    %r15,TF_R15(%rsp)
  435         movq    $T_PROTFLT,TF_TRAPNO(%rsp)
  436         movq    $0,TF_ERR(%rsp) /* XXX should be the error code */
  437         jmp     alltraps_with_regs_pushed

Cache object: c77b6ea9485535c21c7d3598442e1dc2


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.