The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/exception.S

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1989, 1990 William F. Jolitz.
    3  * Copyright (c) 1990 The Regents of the University of California.
    4  * All rights reserved.
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice, this list of conditions and the following disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  * 4. Neither the name of the University nor the names of its contributors
   15  *    may be used to endorse or promote products derived from this software
   16  *    without specific prior written permission.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  *
   30  * $FreeBSD: releng/6.3/sys/amd64/amd64/exception.S 182740 2008-09-03 19:09:47Z simon $
   31  */
   32 
   33 #include "opt_atpic.h"
   34 #include "opt_compat.h"
   35 
   36 #include <machine/asmacros.h>
   37 #include <machine/psl.h>
   38 #include <machine/trap.h>
   39 
   40 #include "assym.s"
   41 
   42         .text
   43 
   44 /*****************************************************************************/
   45 /* Trap handling                                                             */
   46 /*****************************************************************************/
   47 /*
   48  * Trap and fault vector routines.
   49  *
   50  * All traps are 'interrupt gates', SDT_SYSIGT.  An interrupt gate pushes
   51  * state on the stack but also disables interrupts.  This is important for
   52  * us for the use of the swapgs instruction.  We cannot be interrupted
   53  * until the GS.base value is correct.  For most traps, we automatically
   54  * then enable interrupts if the interrupted context had them enabled.
   55  * This is equivalent to the i386 port's use of SDT_SYS386TGT.
   56  *
   57  * The cpu will push a certain amount of state onto the kernel stack for
   58  * the current process.  See amd64/include/frame.h.  
   59  * This includes the current RFLAGS (status register, which includes 
   60  * the interrupt disable state prior to the trap), the code segment register,
   61  * and the return instruction pointer are pushed by the cpu.  The cpu 
   62  * will also push an 'error' code for certain traps.  We push a dummy 
   63  * error code for those traps where the cpu doesn't in order to maintain 
   64  * a consistent frame.  We also push a contrived 'trap number'.
   65  *
   66  * The cpu does not push the general registers, we must do that, and we 
   67  * must restore them prior to calling 'iret'.  The cpu adjusts the %cs and
   68  * %ss segment registers, but does not mess with %ds, %es, or %fs.  Thus we
   69  * must load them with appropriate values for supervisor mode operation.
   70  */
   71 
   72 MCOUNT_LABEL(user)
   73 MCOUNT_LABEL(btrap)
   74 
   75 /* Traps that we leave interrupts disabled for.. */
   76 #define TRAP_NOEN(a)    \
   77         subq $TF_RIP,%rsp; \
   78         movq $(a),TF_TRAPNO(%rsp) ; \
   79         movq $0,TF_ADDR(%rsp) ; \
   80         movq $0,TF_ERR(%rsp) ; \
   81         jmp alltraps_noen
   82 IDTVEC(dbg)
   83         TRAP_NOEN(T_TRCTRAP)
   84 IDTVEC(bpt)
   85         TRAP_NOEN(T_BPTFLT)
   86 
   87 /* Regular traps; The cpu does not supply tf_err for these. */
   88 #define TRAP(a)  \
   89         subq $TF_RIP,%rsp; \
   90         movq $(a),TF_TRAPNO(%rsp) ; \
   91         movq $0,TF_ADDR(%rsp) ; \
   92         movq $0,TF_ERR(%rsp) ; \
   93         jmp alltraps
   94 IDTVEC(div)
   95         TRAP(T_DIVIDE)
   96 IDTVEC(ofl)
   97         TRAP(T_OFLOW)
   98 IDTVEC(bnd)
   99         TRAP(T_BOUND)
  100 IDTVEC(ill)
  101         TRAP(T_PRIVINFLT)
  102 IDTVEC(dna)
  103         TRAP(T_DNA)
  104 IDTVEC(fpusegm)
  105         TRAP(T_FPOPFLT)
  106 IDTVEC(mchk)
  107         TRAP(T_MCHK)
  108 IDTVEC(rsvd)
  109         TRAP(T_RESERVED)
  110 IDTVEC(fpu)
  111         TRAP(T_ARITHTRAP)
  112 IDTVEC(xmm)
  113         TRAP(T_XMMFLT)
  114 
  115 /* This group of traps have tf_err already pushed by the cpu */
  116 #define TRAP_ERR(a)     \
  117         subq $TF_ERR,%rsp; \
  118         movq $(a),TF_TRAPNO(%rsp) ; \
  119         movq $0,TF_ADDR(%rsp) ; \
  120         jmp alltraps
  121 IDTVEC(tss)
  122         TRAP_ERR(T_TSSFLT)
  123 IDTVEC(missing)
  124         TRAP_ERR(T_SEGNPFLT)
  125 IDTVEC(stk)
  126         TRAP_ERR(T_STKFLT)
  127 IDTVEC(align)
  128         TRAP_ERR(T_ALIGNFLT)
  129 
  130         /*
  131          * alltraps entry point.  Use swapgs if this is the first time in the
  132          * kernel from userland.  Reenable interrupts if they were enabled
  133          * before the trap.  This approximates SDT_SYS386TGT on the i386 port.
  134          */
  135 
  136         SUPERALIGN_TEXT
  137         .globl  alltraps
  138         .type   alltraps,@function
  139 alltraps:
  140         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  141         jz      alltraps_testi          /* already running with kernel GS.base */
  142         swapgs
  143 alltraps_testi:
  144         testl   $PSL_I,TF_RFLAGS(%rsp)
  145         jz      alltraps_pushregs
  146         sti
  147 alltraps_pushregs:
  148         movq    %rdi,TF_RDI(%rsp)
  149 alltraps_pushregs_no_rdi:
  150         movq    %rsi,TF_RSI(%rsp)
  151         movq    %rdx,TF_RDX(%rsp)
  152         movq    %rcx,TF_RCX(%rsp)
  153         movq    %r8,TF_R8(%rsp)
  154         movq    %r9,TF_R9(%rsp)
  155         movq    %rax,TF_RAX(%rsp)
  156         movq    %rbx,TF_RBX(%rsp)
  157         movq    %rbp,TF_RBP(%rsp)
  158         movq    %r10,TF_R10(%rsp)
  159         movq    %r11,TF_R11(%rsp)
  160         movq    %r12,TF_R12(%rsp)
  161         movq    %r13,TF_R13(%rsp)
  162         movq    %r14,TF_R14(%rsp)
  163         movq    %r15,TF_R15(%rsp)
  164         FAKE_MCOUNT(TF_RIP(%rsp))
  165         .globl  calltrap
  166         .type   calltrap,@function
  167 calltrap:
  168         call    trap
  169         MEXITCOUNT
  170         jmp     doreti                  /* Handle any pending ASTs */
  171 
  172         /*
  173          * alltraps_noen entry point.  Unlike alltraps above, we want to
  174          * leave the interrupts disabled.  This corresponds to
  175          * SDT_SYS386IGT on the i386 port.
  176          */
  177         SUPERALIGN_TEXT
  178         .globl  alltraps_noen
  179         .type   alltraps_noen,@function
  180 alltraps_noen:
  181         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  182         jz      alltraps_pushregs       /* already running with kernel GS.base */
  183         swapgs
  184         jmp     alltraps_pushregs
  185 
  186 IDTVEC(dblfault)
  187         subq    $TF_ERR,%rsp
  188         movq    $T_DOUBLEFLT,TF_TRAPNO(%rsp)
  189         movq    $0,TF_ADDR(%rsp)
  190         movq    $0,TF_ERR(%rsp)
  191         movq    %rdi,TF_RDI(%rsp)
  192         movq    %rsi,TF_RSI(%rsp)
  193         movq    %rdx,TF_RDX(%rsp)
  194         movq    %rcx,TF_RCX(%rsp)
  195         movq    %r8,TF_R8(%rsp)
  196         movq    %r9,TF_R9(%rsp)
  197         movq    %rax,TF_RAX(%rsp)
  198         movq    %rbx,TF_RBX(%rsp)
  199         movq    %rbp,TF_RBP(%rsp)
  200         movq    %r10,TF_R10(%rsp)
  201         movq    %r11,TF_R11(%rsp)
  202         movq    %r12,TF_R12(%rsp)
  203         movq    %r13,TF_R13(%rsp)
  204         movq    %r14,TF_R14(%rsp)
  205         movq    %r15,TF_R15(%rsp)
  206         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  207         jz      1f                      /* already running with kernel GS.base */
  208         swapgs
  209 1:      call    dblfault_handler
  210 2:      hlt
  211         jmp     2b
  212 
  213 IDTVEC(page)
  214         subq    $TF_ERR,%rsp
  215         movq    $T_PAGEFLT,TF_TRAPNO(%rsp)
  216         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  217         jz      1f                      /* already running with kernel GS.base */
  218         swapgs
  219 1:
  220         movq    %rdi,TF_RDI(%rsp)       /* free up a GP register */
  221         movq    %cr2,%rdi               /* preserve %cr2 before ..  */
  222         movq    %rdi,TF_ADDR(%rsp)      /* enabling interrupts. */
  223         testl   $PSL_I,TF_RFLAGS(%rsp)
  224         jz      alltraps_pushregs_no_rdi
  225         sti
  226         jmp     alltraps_pushregs_no_rdi
  227 
  228         /*
  229          * We have to special-case this one.  If we get a trap in doreti() at
  230          * the iretq stage, we'll reenter with the wrong gs state.  We'll have
  231          * to do a special the swapgs in this case even coming from the kernel.
  232          * XXX linux has a trap handler for their equivalent of load_gs().
  233          */
  234 IDTVEC(prot)
  235         subq    $TF_ERR,%rsp
  236         movq    $T_PROTFLT,TF_TRAPNO(%rsp)
  237         movq    $0,TF_ADDR(%rsp)
  238         movq    %rdi,TF_RDI(%rsp)       /* free up a GP register */
  239         leaq    doreti_iret(%rip),%rdi
  240         cmpq    %rdi,TF_RIP(%rsp)
  241         je      2f                      /* kernel but with user gsbase!! */
  242         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  243         jz      1f                      /* already running with kernel GS.base */
  244 2:
  245         swapgs
  246 1:
  247         testl   $PSL_I,TF_RFLAGS(%rsp)
  248         jz      alltraps_pushregs_no_rdi
  249         sti
  250         jmp     alltraps_pushregs_no_rdi
  251 
  252 /*
  253  * Fast syscall entry point.  We enter here with just our new %cs/%ss set,
  254  * and the new privilige level.  We are still running on the old user stack
  255  * pointer.  We have to juggle a few things around to find our stack etc.
  256  * swapgs gives us access to our PCPU space only.
  257  */
  258 IDTVEC(fast_syscall)
  259         swapgs
  260         movq    %rsp,PCPU(SCRATCH_RSP)
  261         movq    PCPU(RSP0),%rsp
  262         /* Now emulate a trapframe. Make the 8 byte alignment odd for call. */
  263         subq    $TF_SIZE,%rsp
  264         /* defer TF_RSP till we have a spare register */
  265         movq    %r11,TF_RFLAGS(%rsp)
  266         movq    %rcx,TF_RIP(%rsp)       /* %rcx original value is in %r10 */
  267         movq    PCPU(SCRATCH_RSP),%r11  /* %r11 already saved */
  268         movq    %r11,TF_RSP(%rsp)       /* user stack pointer */
  269         sti
  270         movq    $KUDSEL,TF_SS(%rsp)
  271         movq    $KUCSEL,TF_CS(%rsp)
  272         movq    $2,TF_ERR(%rsp)
  273         movq    %rdi,TF_RDI(%rsp)       /* arg 1 */
  274         movq    %rsi,TF_RSI(%rsp)       /* arg 2 */
  275         movq    %rdx,TF_RDX(%rsp)       /* arg 3 */
  276         movq    %r10,TF_RCX(%rsp)       /* arg 4 */
  277         movq    %r8,TF_R8(%rsp)         /* arg 5 */
  278         movq    %r9,TF_R9(%rsp)         /* arg 6 */
  279         movq    %rax,TF_RAX(%rsp)       /* syscall number */
  280         movq    %rbx,TF_RBX(%rsp)       /* C preserved */
  281         movq    %rbp,TF_RBP(%rsp)       /* C preserved */
  282         movq    %r12,TF_R12(%rsp)       /* C preserved */
  283         movq    %r13,TF_R13(%rsp)       /* C preserved */
  284         movq    %r14,TF_R14(%rsp)       /* C preserved */
  285         movq    %r15,TF_R15(%rsp)       /* C preserved */
  286         FAKE_MCOUNT(TF_RIP(%rsp))
  287         call    syscall
  288         movq    PCPU(CURPCB),%rax
  289         testq   $PCB_FULLCTX,PCB_FLAGS(%rax)
  290         jne     3f
  291 1:      /* Check for and handle AST's on return to userland */
  292         cli
  293         movq    PCPU(CURTHREAD),%rax
  294         testl   $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax)
  295         je      2f
  296         sti
  297         movq    %rsp, %rdi
  298         call    ast
  299         jmp     1b
  300 2:      /* restore preserved registers */
  301         MEXITCOUNT
  302         movq    TF_RDI(%rsp),%rdi       /* bonus; preserve arg 1 */
  303         movq    TF_RSI(%rsp),%rsi       /* bonus: preserve arg 2 */
  304         movq    TF_RDX(%rsp),%rdx       /* return value 2 */
  305         movq    TF_RAX(%rsp),%rax       /* return value 1 */
  306         movq    TF_RBX(%rsp),%rbx       /* C preserved */
  307         movq    TF_RBP(%rsp),%rbp       /* C preserved */
  308         movq    TF_R12(%rsp),%r12       /* C preserved */
  309         movq    TF_R13(%rsp),%r13       /* C preserved */
  310         movq    TF_R14(%rsp),%r14       /* C preserved */
  311         movq    TF_R15(%rsp),%r15       /* C preserved */
  312         movq    TF_RFLAGS(%rsp),%r11    /* original %rflags */
  313         movq    TF_RIP(%rsp),%rcx       /* original %rip */
  314         movq    TF_RSP(%rsp),%r9        /* user stack pointer */
  315         movq    %r9,%rsp                /* original %rsp */
  316         swapgs
  317         sysretq
  318 3:      /* Requested full context restore, use doreti for that */
  319         andq    $~PCB_FULLCTX,PCB_FLAGS(%rax)
  320         MEXITCOUNT
  321         jmp     doreti
  322 
  323 /*
  324  * Here for CYA insurance, in case a "syscall" instruction gets
  325  * issued from 32 bit compatability mode. MSR_CSTAR has to point
  326  * to *something* if EFER_SCE is enabled.
  327  */
  328 IDTVEC(fast_syscall32)
  329         sysret
  330 
  331 /*
  332  * NMI handling is special.
  333  *
  334  * First, NMIs do not respect the state of the processor's RFLAGS.IF
  335  * bit and the NMI handler may be invoked at any time, including when
  336  * the processor is in a critical section with RFLAGS.IF == 0.  In
  337  * particular, this means that the processor's GS.base values could be
  338  * inconsistent on entry to the handler, and so we need to read
  339  * MSR_GSBASE to determine if a 'swapgs' is needed.  We use '%ebx', a
  340  * C-preserved register, to remember whether to swap GS back on the
  341  * exit path.
  342  *
  343  * Second, the processor treats NMIs specially, blocking further NMIs
  344  * until an 'iretq' instruction is executed.  We therefore need to
  345  * execute the NMI handler with interrupts disabled to prevent a
  346  * nested interrupt from executing an 'iretq' instruction and
  347  * inadvertently taking the processor out of NMI mode.
  348  */
  349 
  350 IDTVEC(nmi)
  351         subq    $TF_RIP,%rsp
  352         movq    $(T_NMI),TF_TRAPNO(%rsp)
  353         movq    $0,TF_ADDR(%rsp)
  354         movq    $0,TF_ERR(%rsp)
  355         movq    %rdi,TF_RDI(%rsp)
  356         movq    %rsi,TF_RSI(%rsp)
  357         movq    %rdx,TF_RDX(%rsp)
  358         movq    %rcx,TF_RCX(%rsp)
  359         movq    %r8,TF_R8(%rsp)
  360         movq    %r9,TF_R9(%rsp)
  361         movq    %rax,TF_RAX(%rsp)
  362         movq    %rbx,TF_RBX(%rsp)
  363         movq    %rbp,TF_RBP(%rsp)
  364         movq    %r10,TF_R10(%rsp)
  365         movq    %r11,TF_R11(%rsp)
  366         movq    %r12,TF_R12(%rsp)
  367         movq    %r13,TF_R13(%rsp)
  368         movq    %r14,TF_R14(%rsp)
  369         movq    %r15,TF_R15(%rsp)
  370         xorl    %ebx,%ebx
  371         testb   $SEL_RPL_MASK,TF_CS(%rsp)
  372         jnz     nmi_needswapgs          /* we came from userland */
  373         movl    $MSR_GSBASE,%ecx
  374         rdmsr
  375         cmpl    $VM_MAXUSER_ADDRESS >> 32,%edx
  376         jae     nmi_calltrap            /* GS.base holds a kernel VA */
  377 nmi_needswapgs:
  378         incl    %ebx
  379         swapgs
  380 /* Note: this label is also used by ddb and gdb: */
  381 nmi_calltrap:
  382         FAKE_MCOUNT(TF_RIP(%rsp))
  383         call    trap
  384         MEXITCOUNT
  385         testl   %ebx,%ebx
  386         jz      nmi_restoreregs
  387         swapgs
  388 nmi_restoreregs:
  389         movq    TF_RDI(%rsp),%rdi
  390         movq    TF_RSI(%rsp),%rsi
  391         movq    TF_RDX(%rsp),%rdx
  392         movq    TF_RCX(%rsp),%rcx
  393         movq    TF_R8(%rsp),%r8
  394         movq    TF_R9(%rsp),%r9
  395         movq    TF_RAX(%rsp),%rax
  396         movq    TF_RBX(%rsp),%rbx
  397         movq    TF_RBP(%rsp),%rbp
  398         movq    TF_R10(%rsp),%r10
  399         movq    TF_R11(%rsp),%r11
  400         movq    TF_R12(%rsp),%r12
  401         movq    TF_R13(%rsp),%r13
  402         movq    TF_R14(%rsp),%r14
  403         movq    TF_R15(%rsp),%r15
  404         addq    $TF_RIP,%rsp
  405         iretq
  406 
  407 ENTRY(fork_trampoline)
  408         movq    %r12, %rdi              /* function */
  409         movq    %rbx, %rsi              /* arg1 */
  410         movq    %rsp, %rdx              /* trapframe pointer */
  411         call    fork_exit
  412         MEXITCOUNT
  413         jmp     doreti                  /* Handle any ASTs */
  414 
  415 /*
  416  * To efficiently implement classification of trap and interrupt handlers
  417  * for profiling, there must be only trap handlers between the labels btrap
  418  * and bintr, and only interrupt handlers between the labels bintr and
  419  * eintr.  This is implemented (partly) by including files that contain
  420  * some of the handlers.  Before including the files, set up a normal asm
  421  * environment so that the included files doen't need to know that they are
  422  * included.
  423  */
  424 
  425 #ifdef COMPAT_IA32
  426         .data
  427         .p2align 4
  428         .text
  429         SUPERALIGN_TEXT
  430 
  431 #include <amd64/ia32/ia32_exception.S>
  432 #endif
  433 
  434         .data
  435         .p2align 4
  436         .text
  437         SUPERALIGN_TEXT
  438 MCOUNT_LABEL(bintr)
  439 
  440 #include <amd64/amd64/apic_vector.S>
  441 
  442 #ifdef DEV_ATPIC
  443         .data
  444         .p2align 4
  445         .text
  446         SUPERALIGN_TEXT
  447 
  448 #include <amd64/isa/atpic_vector.S>
  449 #endif
  450 
  451         .text
  452 MCOUNT_LABEL(eintr)
  453 
  454 /*
  455  * void doreti(struct trapframe)
  456  *
  457  * Handle return from interrupts, traps and syscalls.
  458  */
  459         .text
  460         SUPERALIGN_TEXT
  461         .type   doreti,@function
  462 doreti:
  463         FAKE_MCOUNT($bintr)             /* init "from" bintr -> doreti */
  464         /*
  465          * Check if ASTs can be handled now.
  466          */
  467         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* are we returning to user mode? */
  468         jz      doreti_exit             /* can't handle ASTs now if not */
  469 
  470 doreti_ast:
  471         /*
  472          * Check for ASTs atomically with returning.  Disabling CPU
  473          * interrupts provides sufficient locking eve in the SMP case,
  474          * since we will be informed of any new ASTs by an IPI.
  475          */
  476         cli
  477         movq    PCPU(CURTHREAD),%rax
  478         testl   $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax)
  479         je      doreti_exit
  480         sti
  481         movq    %rsp, %rdi                      /* pass a pointer to the trapframe */
  482         call    ast
  483         jmp     doreti_ast
  484 
  485         /*
  486          * doreti_exit: pop registers, iret.
  487          *
  488          *      The segment register pop is a special case, since it may
  489          *      fault if (for example) a sigreturn specifies bad segment
  490          *      registers.  The fault is handled in trap.c.
  491          */
  492 doreti_exit:
  493         MEXITCOUNT
  494         movq    TF_RDI(%rsp),%rdi
  495         movq    TF_RSI(%rsp),%rsi
  496         movq    TF_RDX(%rsp),%rdx
  497         movq    TF_RCX(%rsp),%rcx
  498         movq    TF_R8(%rsp),%r8
  499         movq    TF_R9(%rsp),%r9
  500         movq    TF_RAX(%rsp),%rax
  501         movq    TF_RBX(%rsp),%rbx
  502         movq    TF_RBP(%rsp),%rbp
  503         movq    TF_R10(%rsp),%r10
  504         movq    TF_R11(%rsp),%r11
  505         movq    TF_R12(%rsp),%r12
  506         movq    TF_R13(%rsp),%r13
  507         movq    TF_R14(%rsp),%r14
  508         movq    TF_R15(%rsp),%r15
  509         testb   $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
  510         jz      1f                      /* keep running with kernel GS.base */
  511         cli
  512         swapgs
  513 1:      addq    $TF_RIP,%rsp            /* skip over tf_err, tf_trapno */
  514         .globl  doreti_iret
  515 doreti_iret:
  516         iretq
  517 
  518         /*
  519          * doreti_iret_fault.  Alternative return code for
  520          * the case where we get a fault in the doreti_exit code
  521          * above.  trap() (amd64/amd64/trap.c) catches this specific
  522          * case, sends the process a signal and continues in the
  523          * corresponding place in the code below.
  524          */
  525         ALIGN_TEXT
  526         .globl  doreti_iret_fault
  527 doreti_iret_fault:
  528         subq    $TF_RIP,%rsp            /* space including tf_err, tf_trapno */
  529         testl   $PSL_I,TF_RFLAGS(%rsp)
  530         jz      1f
  531         sti
  532 1:      movq    %rdi,TF_RDI(%rsp)
  533         movq    %rsi,TF_RSI(%rsp)
  534         movq    %rdx,TF_RDX(%rsp)
  535         movq    %rcx,TF_RCX(%rsp)
  536         movq    %r8,TF_R8(%rsp)
  537         movq    %r9,TF_R9(%rsp)
  538         movq    %rax,TF_RAX(%rsp)
  539         movq    %rbx,TF_RBX(%rsp)
  540         movq    %rbp,TF_RBP(%rsp)
  541         movq    %r10,TF_R10(%rsp)
  542         movq    %r11,TF_R11(%rsp)
  543         movq    %r12,TF_R12(%rsp)
  544         movq    %r13,TF_R13(%rsp)
  545         movq    %r14,TF_R14(%rsp)
  546         movq    %r15,TF_R15(%rsp)
  547         movq    $T_PROTFLT,TF_TRAPNO(%rsp)
  548         movq    $0,TF_ERR(%rsp) /* XXX should be the error code */
  549         movq    $0,TF_ADDR(%rsp)
  550         FAKE_MCOUNT(TF_RIP(%rsp))
  551         jmp     calltrap

Cache object: a3f18ac9ccf4dd08c5122fa13700f8e2


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.