[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/arch/x86/kernel/entry_32.S

Version: -  FREEBSD  -  FREEBSD10  -  FREEBSD9  -  FREEBSD92  -  FREEBSD91  -  FREEBSD90  -  FREEBSD8  -  FREEBSD82  -  FREEBSD81  -  FREEBSD80  -  FREEBSD7  -  FREEBSD74  -  FREEBSD73  -  FREEBSD72  -  FREEBSD71  -  FREEBSD70  -  FREEBSD6  -  FREEBSD64  -  FREEBSD63  -  FREEBSD62  -  FREEBSD61  -  FREEBSD60  -  FREEBSD5  -  FREEBSD55  -  FREEBSD54  -  FREEBSD53  -  FREEBSD52  -  FREEBSD51  -  FREEBSD50  -  FREEBSD4  -  FREEBSD3  -  FREEBSD22  -  cheribsd  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1  -  FREEBSD-LIBC  -  FREEBSD8-LIBC  -  FREEBSD7-LIBC  -  FREEBSD6-LIBC  -  GLIBC27 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  *
    3  *  Copyright (C) 1991, 1992  Linus Torvalds
    4  */
    5 
    6 /*
    7  * entry.S contains the system-call and fault low-level handling routines.
    8  * This also contains the timer-interrupt handler, as well as all interrupts
    9  * and faults that can result in a task-switch.
   10  *
   11  * NOTE: This code handles signal-recognition, which happens every time
   12  * after a timer-interrupt and after each system call.
   13  *
   14  * I changed all the .align's to 4 (16 byte alignment), as that's faster
   15  * on a 486.
   16  *
   17  * Stack layout in 'syscall_exit':
   18  *      ptrace needs to have all regs on the stack.
   19  *      if the order here is changed, it needs to be
   20  *      updated in fork.c:copy_process, signal.c:do_signal,
   21  *      ptrace.c and ptrace.h
   22  *
   23  *       0(%esp) - %ebx
   24  *       4(%esp) - %ecx
   25  *       8(%esp) - %edx
   26  *       C(%esp) - %esi
   27  *      10(%esp) - %edi
   28  *      14(%esp) - %ebp
   29  *      18(%esp) - %eax
   30  *      1C(%esp) - %ds
   31  *      20(%esp) - %es
   32  *      24(%esp) - %fs
   33  *      28(%esp) - %gs          saved iff !CONFIG_X86_32_LAZY_GS
   34  *      2C(%esp) - orig_eax
   35  *      30(%esp) - %eip
   36  *      34(%esp) - %cs
   37  *      38(%esp) - %eflags
   38  *      3C(%esp) - %oldesp
   39  *      40(%esp) - %oldss
   40  *
   41  * "current" is in register %ebx during any slow entries.
   42  */
   43 
   44 #include <linux/linkage.h>
   45 #include <linux/err.h>
   46 #include <asm/thread_info.h>
   47 #include <asm/irqflags.h>
   48 #include <asm/errno.h>
   49 #include <asm/segment.h>
   50 #include <asm/smp.h>
   51 #include <asm/page_types.h>
   52 #include <asm/percpu.h>
   53 #include <asm/dwarf2.h>
   54 #include <asm/processor-flags.h>
   55 #include <asm/ftrace.h>
   56 #include <asm/irq_vectors.h>
   57 #include <asm/cpufeature.h>
   58 #include <asm/alternative-asm.h>
   59 #include <asm/asm.h>
   60 #include <asm/smap.h>
   61 
   62 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
   63 #include <linux/elf-em.h>
   64 #define AUDIT_ARCH_I386         (EM_386|__AUDIT_ARCH_LE)
   65 #define __AUDIT_ARCH_LE    0x40000000
   66 
   67 #ifndef CONFIG_AUDITSYSCALL
   68 #define sysenter_audit  syscall_trace_entry
   69 #define sysexit_audit   syscall_exit_work
   70 #endif
   71 
   72         .section .entry.text, "ax"
   73 
   74 /*
   75  * We use macros for low-level operations which need to be overridden
   76  * for paravirtualization.  The following will never clobber any registers:
   77  *   INTERRUPT_RETURN (aka. "iret")
   78  *   GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
   79  *   ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
   80  *
   81  * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
   82  * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
   83  * Allowing a register to be clobbered can shrink the paravirt replacement
   84  * enough to patch inline, increasing performance.
   85  */
   86 
   87 #ifdef CONFIG_PREEMPT
   88 #define preempt_stop(clobbers)  DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
   89 #else
   90 #define preempt_stop(clobbers)
   91 #define resume_kernel           restore_all
   92 #endif
   93 
   94 .macro TRACE_IRQS_IRET
   95 #ifdef CONFIG_TRACE_IRQFLAGS
   96         testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)     # interrupts off?
   97         jz 1f
   98         TRACE_IRQS_ON
   99 1:
  100 #endif
  101 .endm
  102 
  103 /*
  104  * User gs save/restore
  105  *
  106  * %gs is used for userland TLS and kernel only uses it for stack
  107  * canary which is required to be at %gs:20 by gcc.  Read the comment
  108  * at the top of stackprotector.h for more info.
  109  *
  110  * Local labels 98 and 99 are used.
  111  */
  112 #ifdef CONFIG_X86_32_LAZY_GS
  113 
  114  /* unfortunately push/pop can't be no-op */
  115 .macro PUSH_GS
  116         pushl_cfi $0
  117 .endm
  118 .macro POP_GS pop=0
  119         addl $(4 + \pop), %esp
  120         CFI_ADJUST_CFA_OFFSET -(4 + \pop)
  121 .endm
  122 .macro POP_GS_EX
  123 .endm
  124 
  125  /* all the rest are no-op */
  126 .macro PTGS_TO_GS
  127 .endm
  128 .macro PTGS_TO_GS_EX
  129 .endm
  130 .macro GS_TO_REG reg
  131 .endm
  132 .macro REG_TO_PTGS reg
  133 .endm
  134 .macro SET_KERNEL_GS reg
  135 .endm
  136 
  137 #else   /* CONFIG_X86_32_LAZY_GS */
  138 
  139 .macro PUSH_GS
  140         pushl_cfi %gs
  141         /*CFI_REL_OFFSET gs, 0*/
  142 .endm
  143 
  144 .macro POP_GS pop=0
  145 98:     popl_cfi %gs
  146         /*CFI_RESTORE gs*/
  147   .if \pop <> 0
  148         add $\pop, %esp
  149         CFI_ADJUST_CFA_OFFSET -\pop
  150   .endif
  151 .endm
  152 .macro POP_GS_EX
  153 .pushsection .fixup, "ax"
  154 99:     movl $0, (%esp)
  155         jmp 98b
  156 .popsection
  157         _ASM_EXTABLE(98b,99b)
  158 .endm
  159 
  160 .macro PTGS_TO_GS
  161 98:     mov PT_GS(%esp), %gs
  162 .endm
  163 .macro PTGS_TO_GS_EX
  164 .pushsection .fixup, "ax"
  165 99:     movl $0, PT_GS(%esp)
  166         jmp 98b
  167 .popsection
  168         _ASM_EXTABLE(98b,99b)
  169 .endm
  170 
  171 .macro GS_TO_REG reg
  172         movl %gs, \reg
  173         /*CFI_REGISTER gs, \reg*/
  174 .endm
  175 .macro REG_TO_PTGS reg
  176         movl \reg, PT_GS(%esp)
  177         /*CFI_REL_OFFSET gs, PT_GS*/
  178 .endm
  179 .macro SET_KERNEL_GS reg
  180         movl $(__KERNEL_STACK_CANARY), \reg
  181         movl \reg, %gs
  182 .endm
  183 
  184 #endif  /* CONFIG_X86_32_LAZY_GS */
  185 
  186 .macro SAVE_ALL
  187         cld
  188         PUSH_GS
  189         pushl_cfi %fs
  190         /*CFI_REL_OFFSET fs, 0;*/
  191         pushl_cfi %es
  192         /*CFI_REL_OFFSET es, 0;*/
  193         pushl_cfi %ds
  194         /*CFI_REL_OFFSET ds, 0;*/
  195         pushl_cfi %eax
  196         CFI_REL_OFFSET eax, 0
  197         pushl_cfi %ebp
  198         CFI_REL_OFFSET ebp, 0
  199         pushl_cfi %edi
  200         CFI_REL_OFFSET edi, 0
  201         pushl_cfi %esi
  202         CFI_REL_OFFSET esi, 0
  203         pushl_cfi %edx
  204         CFI_REL_OFFSET edx, 0
  205         pushl_cfi %ecx
  206         CFI_REL_OFFSET ecx, 0
  207         pushl_cfi %ebx
  208         CFI_REL_OFFSET ebx, 0
  209         movl $(__USER_DS), %edx
  210         movl %edx, %ds
  211         movl %edx, %es
  212         movl $(__KERNEL_PERCPU), %edx
  213         movl %edx, %fs
  214         SET_KERNEL_GS %edx
  215 .endm
  216 
  217 .macro RESTORE_INT_REGS
  218         popl_cfi %ebx
  219         CFI_RESTORE ebx
  220         popl_cfi %ecx
  221         CFI_RESTORE ecx
  222         popl_cfi %edx
  223         CFI_RESTORE edx
  224         popl_cfi %esi
  225         CFI_RESTORE esi
  226         popl_cfi %edi
  227         CFI_RESTORE edi
  228         popl_cfi %ebp
  229         CFI_RESTORE ebp
  230         popl_cfi %eax
  231         CFI_RESTORE eax
  232 .endm
  233 
  234 .macro RESTORE_REGS pop=0
  235         RESTORE_INT_REGS
  236 1:      popl_cfi %ds
  237         /*CFI_RESTORE ds;*/
  238 2:      popl_cfi %es
  239         /*CFI_RESTORE es;*/
  240 3:      popl_cfi %fs
  241         /*CFI_RESTORE fs;*/
  242         POP_GS \pop
  243 .pushsection .fixup, "ax"
  244 4:      movl $0, (%esp)
  245         jmp 1b
  246 5:      movl $0, (%esp)
  247         jmp 2b
  248 6:      movl $0, (%esp)
  249         jmp 3b
  250 .popsection
  251         _ASM_EXTABLE(1b,4b)
  252         _ASM_EXTABLE(2b,5b)
  253         _ASM_EXTABLE(3b,6b)
  254         POP_GS_EX
  255 .endm
  256 
  257 .macro RING0_INT_FRAME
  258         CFI_STARTPROC simple
  259         CFI_SIGNAL_FRAME
  260         CFI_DEF_CFA esp, 3*4
  261         /*CFI_OFFSET cs, -2*4;*/
  262         CFI_OFFSET eip, -3*4
  263 .endm
  264 
  265 .macro RING0_EC_FRAME
  266         CFI_STARTPROC simple
  267         CFI_SIGNAL_FRAME
  268         CFI_DEF_CFA esp, 4*4
  269         /*CFI_OFFSET cs, -2*4;*/
  270         CFI_OFFSET eip, -3*4
  271 .endm
  272 
  273 .macro RING0_PTREGS_FRAME
  274         CFI_STARTPROC simple
  275         CFI_SIGNAL_FRAME
  276         CFI_DEF_CFA esp, PT_OLDESP-PT_EBX
  277         /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/
  278         CFI_OFFSET eip, PT_EIP-PT_OLDESP
  279         /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/
  280         /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/
  281         CFI_OFFSET eax, PT_EAX-PT_OLDESP
  282         CFI_OFFSET ebp, PT_EBP-PT_OLDESP
  283         CFI_OFFSET edi, PT_EDI-PT_OLDESP
  284         CFI_OFFSET esi, PT_ESI-PT_OLDESP
  285         CFI_OFFSET edx, PT_EDX-PT_OLDESP
  286         CFI_OFFSET ecx, PT_ECX-PT_OLDESP
  287         CFI_OFFSET ebx, PT_EBX-PT_OLDESP
  288 .endm
  289 
  290 ENTRY(ret_from_fork)
  291         CFI_STARTPROC
  292         pushl_cfi %eax
  293         call schedule_tail
  294         GET_THREAD_INFO(%ebp)
  295         popl_cfi %eax
  296         pushl_cfi $0x0202               # Reset kernel eflags
  297         popfl_cfi
  298         jmp syscall_exit
  299         CFI_ENDPROC
  300 END(ret_from_fork)
  301 
  302 ENTRY(ret_from_kernel_thread)
  303         CFI_STARTPROC
  304         pushl_cfi %eax
  305         call schedule_tail
  306         GET_THREAD_INFO(%ebp)
  307         popl_cfi %eax
  308         pushl_cfi $0x0202               # Reset kernel eflags
  309         popfl_cfi
  310         movl PT_EBP(%esp),%eax
  311         call *PT_EBX(%esp)
  312         movl $0,PT_EAX(%esp)
  313         jmp syscall_exit
  314         CFI_ENDPROC
  315 ENDPROC(ret_from_kernel_thread)
  316 
  317 /*
  318  * Interrupt exit functions should be protected against kprobes
  319  */
  320         .pushsection .kprobes.text, "ax"
  321 /*
  322  * Return to user mode is not as complex as all this looks,
  323  * but we want the default path for a system call return to
  324  * go as quickly as possible which is why some of this is
  325  * less clear than it otherwise should be.
  326  */
  327 
  328         # userspace resumption stub bypassing syscall exit tracing
  329         ALIGN
  330         RING0_PTREGS_FRAME
  331 ret_from_exception:
  332         preempt_stop(CLBR_ANY)
  333 ret_from_intr:
  334         GET_THREAD_INFO(%ebp)
  335 #ifdef CONFIG_VM86
  336         movl PT_EFLAGS(%esp), %eax      # mix EFLAGS and CS
  337         movb PT_CS(%esp), %al
  338         andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
  339 #else
  340         /*
  341          * We can be coming here from child spawned by kernel_thread().
  342          */
  343         movl PT_CS(%esp), %eax
  344         andl $SEGMENT_RPL_MASK, %eax
  345 #endif
  346         cmpl $USER_RPL, %eax
  347         jb resume_kernel                # not returning to v8086 or userspace
  348 
  349 ENTRY(resume_userspace)
  350         LOCKDEP_SYS_EXIT
  351         DISABLE_INTERRUPTS(CLBR_ANY)    # make sure we don't miss an interrupt
  352                                         # setting need_resched or sigpending
  353                                         # between sampling and the iret
  354         TRACE_IRQS_OFF
  355         movl TI_flags(%ebp), %ecx
  356         andl $_TIF_WORK_MASK, %ecx      # is there any work to be done on
  357                                         # int/exception return?
  358         jne work_pending
  359         jmp restore_all
  360 END(ret_from_exception)
  361 
  362 #ifdef CONFIG_PREEMPT
  363 ENTRY(resume_kernel)
  364         DISABLE_INTERRUPTS(CLBR_ANY)
  365         cmpl $0,TI_preempt_count(%ebp)  # non-zero preempt_count ?
  366         jnz restore_all
  367 need_resched:
  368         movl TI_flags(%ebp), %ecx       # need_resched set ?
  369         testb $_TIF_NEED_RESCHED, %cl
  370         jz restore_all
  371         testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)    # interrupts off (exception path) ?
  372         jz restore_all
  373         call preempt_schedule_irq
  374         jmp need_resched
  375 END(resume_kernel)
  376 #endif
  377         CFI_ENDPROC
  378 /*
  379  * End of kprobes section
  380  */
  381         .popsection
  382 
  383 /* SYSENTER_RETURN points to after the "sysenter" instruction in
  384    the vsyscall page.  See vsyscall-sysentry.S, which defines the symbol.  */
  385 
  386         # sysenter call handler stub
  387 ENTRY(ia32_sysenter_target)
  388         CFI_STARTPROC simple
  389         CFI_SIGNAL_FRAME
  390         CFI_DEF_CFA esp, 0
  391         CFI_REGISTER esp, ebp
  392         movl TSS_sysenter_sp0(%esp),%esp
  393 sysenter_past_esp:
  394         /*
  395          * Interrupts are disabled here, but we can't trace it until
  396          * enough kernel state to call TRACE_IRQS_OFF can be called - but
  397          * we immediately enable interrupts at that point anyway.
  398          */
  399         pushl_cfi $__USER_DS
  400         /*CFI_REL_OFFSET ss, 0*/
  401         pushl_cfi %ebp
  402         CFI_REL_OFFSET esp, 0
  403         pushfl_cfi
  404         orl $X86_EFLAGS_IF, (%esp)
  405         pushl_cfi $__USER_CS
  406         /*CFI_REL_OFFSET cs, 0*/
  407         /*
  408          * Push current_thread_info()->sysenter_return to the stack.
  409          * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
  410          * pushed above; +8 corresponds to copy_thread's esp0 setting.
  411          */
  412         pushl_cfi ((TI_sysenter_return)-THREAD_SIZE+8+4*4)(%esp)
  413         CFI_REL_OFFSET eip, 0
  414 
  415         pushl_cfi %eax
  416         SAVE_ALL
  417         ENABLE_INTERRUPTS(CLBR_NONE)
  418 
  419 /*
  420  * Load the potential sixth argument from user stack.
  421  * Careful about security.
  422  */
  423         cmpl $__PAGE_OFFSET-3,%ebp
  424         jae syscall_fault
  425         ASM_STAC
  426 1:      movl (%ebp),%ebp
  427         ASM_CLAC
  428         movl %ebp,PT_EBP(%esp)
  429         _ASM_EXTABLE(1b,syscall_fault)
  430 
  431         GET_THREAD_INFO(%ebp)
  432 
  433         testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
  434         jnz sysenter_audit
  435 sysenter_do_call:
  436         cmpl $(NR_syscalls), %eax
  437         jae syscall_badsys
  438         call *sys_call_table(,%eax,4)
  439         movl %eax,PT_EAX(%esp)
  440         LOCKDEP_SYS_EXIT
  441         DISABLE_INTERRUPTS(CLBR_ANY)
  442         TRACE_IRQS_OFF
  443         movl TI_flags(%ebp), %ecx
  444         testl $_TIF_ALLWORK_MASK, %ecx
  445         jne sysexit_audit
  446 sysenter_exit:
  447 /* if something modifies registers it must also disable sysexit */
  448         movl PT_EIP(%esp), %edx
  449         movl PT_OLDESP(%esp), %ecx
  450         xorl %ebp,%ebp
  451         TRACE_IRQS_ON
  452 1:      mov  PT_FS(%esp), %fs
  453         PTGS_TO_GS
  454         ENABLE_INTERRUPTS_SYSEXIT
  455 
  456 #ifdef CONFIG_AUDITSYSCALL
  457 sysenter_audit:
  458         testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
  459         jnz syscall_trace_entry
  460         addl $4,%esp
  461         CFI_ADJUST_CFA_OFFSET -4
  462         /* %esi already in 8(%esp)         6th arg: 4th syscall arg */
  463         /* %edx already in 4(%esp)         5th arg: 3rd syscall arg */
  464         /* %ecx already in 0(%esp)         4th arg: 2nd syscall arg */
  465         movl %ebx,%ecx                  /* 3rd arg: 1st syscall arg */
  466         movl %eax,%edx                  /* 2nd arg: syscall number */
  467         movl $AUDIT_ARCH_I386,%eax      /* 1st arg: audit arch */
  468         call __audit_syscall_entry
  469         pushl_cfi %ebx
  470         movl PT_EAX(%esp),%eax          /* reload syscall number */
  471         jmp sysenter_do_call
  472 
  473 sysexit_audit:
  474         testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx
  475         jne syscall_exit_work
  476         TRACE_IRQS_ON
  477         ENABLE_INTERRUPTS(CLBR_ANY)
  478         movl %eax,%edx          /* second arg, syscall return value */
  479         cmpl $-MAX_ERRNO,%eax   /* is it an error ? */
  480         setbe %al               /* 1 if so, 0 if not */
  481         movzbl %al,%eax         /* zero-extend that */
  482         call __audit_syscall_exit
  483         DISABLE_INTERRUPTS(CLBR_ANY)
  484         TRACE_IRQS_OFF
  485         movl TI_flags(%ebp), %ecx
  486         testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx
  487         jne syscall_exit_work
  488         movl PT_EAX(%esp),%eax  /* reload syscall return value */
  489         jmp sysenter_exit
  490 #endif
  491 
  492         CFI_ENDPROC
  493 .pushsection .fixup,"ax"
  494 2:      movl $0,PT_FS(%esp)
  495         jmp 1b
  496 .popsection
  497         _ASM_EXTABLE(1b,2b)
  498         PTGS_TO_GS_EX
  499 ENDPROC(ia32_sysenter_target)
  500 
  501 /*
  502  * syscall stub including irq exit should be protected against kprobes
  503  */
  504         .pushsection .kprobes.text, "ax"
  505         # system call handler stub
  506 ENTRY(system_call)
  507         RING0_INT_FRAME                 # can't unwind into user space anyway
  508         ASM_CLAC
  509         pushl_cfi %eax                  # save orig_eax
  510         SAVE_ALL
  511         GET_THREAD_INFO(%ebp)
  512                                         # system call tracing in operation / emulation
  513         testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
  514         jnz syscall_trace_entry
  515         cmpl $(NR_syscalls), %eax
  516         jae syscall_badsys
  517 syscall_call:
  518         call *sys_call_table(,%eax,4)
  519         movl %eax,PT_EAX(%esp)          # store the return value
  520 syscall_exit:
  521         LOCKDEP_SYS_EXIT
  522         DISABLE_INTERRUPTS(CLBR_ANY)    # make sure we don't miss an interrupt
  523                                         # setting need_resched or sigpending
  524                                         # between sampling and the iret
  525         TRACE_IRQS_OFF
  526         movl TI_flags(%ebp), %ecx
  527         testl $_TIF_ALLWORK_MASK, %ecx  # current->work
  528         jne syscall_exit_work
  529 
  530 restore_all:
  531         TRACE_IRQS_IRET
  532 restore_all_notrace:
  533         movl PT_EFLAGS(%esp), %eax      # mix EFLAGS, SS and CS
  534         # Warning: PT_OLDSS(%esp) contains the wrong/random values if we
  535         # are returning to the kernel.
  536         # See comments in process.c:copy_thread() for details.
  537         movb PT_OLDSS(%esp), %ah
  538         movb PT_CS(%esp), %al
  539         andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
  540         cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
  541         CFI_REMEMBER_STATE
  542         je ldt_ss                       # returning to user-space with LDT SS
  543 restore_nocheck:
  544         RESTORE_REGS 4                  # skip orig_eax/error_code
  545 irq_return:
  546         INTERRUPT_RETURN
  547 .section .fixup,"ax"
  548 ENTRY(iret_exc)
  549         pushl $0                        # no error code
  550         pushl $do_iret_error
  551         jmp error_code
  552 .previous
  553         _ASM_EXTABLE(irq_return,iret_exc)
  554 
  555         CFI_RESTORE_STATE
  556 ldt_ss:
  557         larl PT_OLDSS(%esp), %eax
  558         jnz restore_nocheck
  559         testl $0x00400000, %eax         # returning to 32bit stack?
  560         jnz restore_nocheck             # allright, normal return
  561 
  562 #ifdef CONFIG_PARAVIRT
  563         /*
  564          * The kernel can't run on a non-flat stack if paravirt mode
  565          * is active.  Rather than try to fixup the high bits of
  566          * ESP, bypass this code entirely.  This may break DOSemu
  567          * and/or Wine support in a paravirt VM, although the option
  568          * is still available to implement the setting of the high
  569          * 16-bits in the INTERRUPT_RETURN paravirt-op.
  570          */
  571         cmpl $0, pv_info+PARAVIRT_enabled
  572         jne restore_nocheck
  573 #endif
  574 
  575 /*
  576  * Setup and switch to ESPFIX stack
  577  *
  578  * We're returning to userspace with a 16 bit stack. The CPU will not
  579  * restore the high word of ESP for us on executing iret... This is an
  580  * "official" bug of all the x86-compatible CPUs, which we can work
  581  * around to make dosemu and wine happy. We do this by preloading the
  582  * high word of ESP with the high word of the userspace ESP while
  583  * compensating for the offset by changing to the ESPFIX segment with
  584  * a base address that matches for the difference.
  585  */
  586 #define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8)
  587         mov %esp, %edx                  /* load kernel esp */
  588         mov PT_OLDESP(%esp), %eax       /* load userspace esp */
  589         mov %dx, %ax                    /* eax: new kernel esp */
  590         sub %eax, %edx                  /* offset (low word is 0) */
  591         shr $16, %edx
  592         mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */
  593         mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */
  594         pushl_cfi $__ESPFIX_SS
  595         pushl_cfi %eax                  /* new kernel esp */
  596         /* Disable interrupts, but do not irqtrace this section: we
  597          * will soon execute iret and the tracer was already set to
  598          * the irqstate after the iret */
  599         DISABLE_INTERRUPTS(CLBR_EAX)
  600         lss (%esp), %esp                /* switch to espfix segment */
  601         CFI_ADJUST_CFA_OFFSET -8
  602         jmp restore_nocheck
  603         CFI_ENDPROC
  604 ENDPROC(system_call)
  605 
  606         # perform work that needs to be done immediately before resumption
  607         ALIGN
  608         RING0_PTREGS_FRAME              # can't unwind into user space anyway
  609 work_pending:
  610         testb $_TIF_NEED_RESCHED, %cl
  611         jz work_notifysig
  612 work_resched:
  613         call schedule
  614         LOCKDEP_SYS_EXIT
  615         DISABLE_INTERRUPTS(CLBR_ANY)    # make sure we don't miss an interrupt
  616                                         # setting need_resched or sigpending
  617                                         # between sampling and the iret
  618         TRACE_IRQS_OFF
  619         movl TI_flags(%ebp), %ecx
  620         andl $_TIF_WORK_MASK, %ecx      # is there any work to be done other
  621                                         # than syscall tracing?
  622         jz restore_all
  623         testb $_TIF_NEED_RESCHED, %cl
  624         jnz work_resched
  625 
  626 work_notifysig:                         # deal with pending signals and
  627                                         # notify-resume requests
  628 #ifdef CONFIG_VM86
  629         testl $X86_EFLAGS_VM, PT_EFLAGS(%esp)
  630         movl %esp, %eax
  631         jne work_notifysig_v86          # returning to kernel-space or
  632                                         # vm86-space
  633 1:
  634 #else
  635         movl %esp, %eax
  636 #endif
  637         TRACE_IRQS_ON
  638         ENABLE_INTERRUPTS(CLBR_NONE)
  639         movb PT_CS(%esp), %bl
  640         andb $SEGMENT_RPL_MASK, %bl
  641         cmpb $USER_RPL, %bl
  642         jb resume_kernel
  643         xorl %edx, %edx
  644         call do_notify_resume
  645         jmp resume_userspace
  646 
  647 #ifdef CONFIG_VM86
  648         ALIGN
  649 work_notifysig_v86:
  650         pushl_cfi %ecx                  # save ti_flags for do_notify_resume
  651         call save_v86_state             # %eax contains pt_regs pointer
  652         popl_cfi %ecx
  653         movl %eax, %esp
  654         jmp 1b
  655 #endif
  656 END(work_pending)
  657 
  658         # perform syscall exit tracing
  659         ALIGN
  660 syscall_trace_entry:
  661         movl $-ENOSYS,PT_EAX(%esp)
  662         movl %esp, %eax
  663         call syscall_trace_enter
  664         /* What it returned is what we'll actually use.  */
  665         cmpl $(NR_syscalls), %eax
  666         jnae syscall_call
  667         jmp syscall_exit
  668 END(syscall_trace_entry)
  669 
  670         # perform syscall exit tracing
  671         ALIGN
  672 syscall_exit_work:
  673         testl $_TIF_WORK_SYSCALL_EXIT, %ecx
  674         jz work_pending
  675         TRACE_IRQS_ON
  676         ENABLE_INTERRUPTS(CLBR_ANY)     # could let syscall_trace_leave() call
  677                                         # schedule() instead
  678         movl %esp, %eax
  679         call syscall_trace_leave
  680         jmp resume_userspace
  681 END(syscall_exit_work)
  682         CFI_ENDPROC
  683 
  684         RING0_INT_FRAME                 # can't unwind into user space anyway
  685 syscall_fault:
  686         ASM_CLAC
  687         GET_THREAD_INFO(%ebp)
  688         movl $-EFAULT,PT_EAX(%esp)
  689         jmp resume_userspace
  690 END(syscall_fault)
  691 
  692 syscall_badsys:
  693         movl $-ENOSYS,PT_EAX(%esp)
  694         jmp resume_userspace
  695 END(syscall_badsys)
  696         CFI_ENDPROC
  697 /*
  698  * End of kprobes section
  699  */
  700         .popsection
  701 
  702 /*
  703  * System calls that need a pt_regs pointer.
  704  */
  705 #define PTREGSCALL0(name) \
  706 ENTRY(ptregs_##name) ;  \
  707         leal 4(%esp),%eax; \
  708         jmp sys_##name; \
  709 ENDPROC(ptregs_##name)
  710 
  711 #define PTREGSCALL1(name) \
  712 ENTRY(ptregs_##name) ; \
  713         leal 4(%esp),%edx; \
  714         movl (PT_EBX+4)(%esp),%eax; \
  715         jmp sys_##name; \
  716 ENDPROC(ptregs_##name)
  717 
  718 #define PTREGSCALL2(name) \
  719 ENTRY(ptregs_##name) ; \
  720         leal 4(%esp),%ecx; \
  721         movl (PT_ECX+4)(%esp),%edx; \
  722         movl (PT_EBX+4)(%esp),%eax; \
  723         jmp sys_##name; \
  724 ENDPROC(ptregs_##name)
  725 
  726 #define PTREGSCALL3(name) \
  727 ENTRY(ptregs_##name) ; \
  728         CFI_STARTPROC; \
  729         leal 4(%esp),%eax; \
  730         pushl_cfi %eax; \
  731         movl PT_EDX(%eax),%ecx; \
  732         movl PT_ECX(%eax),%edx; \
  733         movl PT_EBX(%eax),%eax; \
  734         call sys_##name; \
  735         addl $4,%esp; \
  736         CFI_ADJUST_CFA_OFFSET -4; \
  737         ret; \
  738         CFI_ENDPROC; \
  739 ENDPROC(ptregs_##name)
  740 
  741 PTREGSCALL1(iopl)
  742 PTREGSCALL0(sigreturn)
  743 PTREGSCALL0(rt_sigreturn)
  744 PTREGSCALL2(vm86)
  745 PTREGSCALL1(vm86old)
  746 
  747 .macro FIXUP_ESPFIX_STACK
  748 /*
  749  * Switch back for ESPFIX stack to the normal zerobased stack
  750  *
  751  * We can't call C functions using the ESPFIX stack. This code reads
  752  * the high word of the segment base from the GDT and swiches to the
  753  * normal stack and adjusts ESP with the matching offset.
  754  */
  755         /* fixup the stack */
  756         mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */
  757         mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */
  758         shl $16, %eax
  759         addl %esp, %eax                 /* the adjusted stack pointer */
  760         pushl_cfi $__KERNEL_DS
  761         pushl_cfi %eax
  762         lss (%esp), %esp                /* switch to the normal stack segment */
  763         CFI_ADJUST_CFA_OFFSET -8
  764 .endm
  765 .macro UNWIND_ESPFIX_STACK
  766         movl %ss, %eax
  767         /* see if on espfix stack */
  768         cmpw $__ESPFIX_SS, %ax
  769         jne 27f
  770         movl $__KERNEL_DS, %eax
  771         movl %eax, %ds
  772         movl %eax, %es
  773         /* switch to normal stack */
  774         FIXUP_ESPFIX_STACK
  775 27:
  776 .endm
  777 
  778 /*
  779  * Build the entry stubs and pointer table with some assembler magic.
  780  * We pack 7 stubs into a single 32-byte chunk, which will fit in a
  781  * single cache line on all modern x86 implementations.
  782  */
  783 .section .init.rodata,"a"
  784 ENTRY(interrupt)
  785 .section .entry.text, "ax"
  786         .p2align 5
  787         .p2align CONFIG_X86_L1_CACHE_SHIFT
  788 ENTRY(irq_entries_start)
  789         RING0_INT_FRAME
  790 vector=FIRST_EXTERNAL_VECTOR
  791 .rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7
  792         .balign 32
  793   .rept 7
  794     .if vector < NR_VECTORS
  795       .if vector <> FIRST_EXTERNAL_VECTOR
  796         CFI_ADJUST_CFA_OFFSET -4
  797       .endif
  798 1:      pushl_cfi $(~vector+0x80)       /* Note: always in signed byte range */
  799       .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6
  800         jmp 2f
  801       .endif
  802       .previous
  803         .long 1b
  804       .section .entry.text, "ax"
  805 vector=vector+1
  806     .endif
  807   .endr
  808 2:      jmp common_interrupt
  809 .endr
  810 END(irq_entries_start)
  811 
  812 .previous
  813 END(interrupt)
  814 .previous
  815 
  816 /*
  817  * the CPU automatically disables interrupts when executing an IRQ vector,
  818  * so IRQ-flags tracing has to follow that:
  819  */
  820         .p2align CONFIG_X86_L1_CACHE_SHIFT
  821 common_interrupt:
  822         ASM_CLAC
  823         addl $-0x80,(%esp)      /* Adjust vector into the [-256,-1] range */
  824         SAVE_ALL
  825         TRACE_IRQS_OFF
  826         movl %esp,%eax
  827         call do_IRQ
  828         jmp ret_from_intr
  829 ENDPROC(common_interrupt)
  830         CFI_ENDPROC
  831 
  832 /*
  833  *  Irq entries should be protected against kprobes
  834  */
  835         .pushsection .kprobes.text, "ax"
  836 #define BUILD_INTERRUPT3(name, nr, fn)  \
  837 ENTRY(name)                             \
  838         RING0_INT_FRAME;                \
  839         ASM_CLAC;                       \
  840         pushl_cfi $~(nr);               \
  841         SAVE_ALL;                       \
  842         TRACE_IRQS_OFF                  \
  843         movl %esp,%eax;                 \
  844         call fn;                        \
  845         jmp ret_from_intr;              \
  846         CFI_ENDPROC;                    \
  847 ENDPROC(name)
  848 
  849 #define BUILD_INTERRUPT(name, nr)       BUILD_INTERRUPT3(name, nr, smp_##name)
  850 
  851 /* The include is where all of the SMP etc. interrupts come from */
  852 #include <asm/entry_arch.h>
  853 
  854 ENTRY(coprocessor_error)
  855         RING0_INT_FRAME
  856         ASM_CLAC
  857         pushl_cfi $0
  858         pushl_cfi $do_coprocessor_error
  859         jmp error_code
  860         CFI_ENDPROC
  861 END(coprocessor_error)
  862 
  863 ENTRY(simd_coprocessor_error)
  864         RING0_INT_FRAME
  865         ASM_CLAC
  866         pushl_cfi $0
  867 #ifdef CONFIG_X86_INVD_BUG
  868         /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */
  869 661:    pushl_cfi $do_general_protection
  870 662:
  871 .section .altinstructions,"a"
  872         altinstruction_entry 661b, 663f, X86_FEATURE_XMM, 662b-661b, 664f-663f
  873 .previous
  874 .section .altinstr_replacement,"ax"
  875 663:    pushl $do_simd_coprocessor_error
  876 664:
  877 .previous
  878 #else
  879         pushl_cfi $do_simd_coprocessor_error
  880 #endif
  881         jmp error_code
  882         CFI_ENDPROC
  883 END(simd_coprocessor_error)
  884 
  885 ENTRY(device_not_available)
  886         RING0_INT_FRAME
  887         ASM_CLAC
  888         pushl_cfi $-1                   # mark this as an int
  889         pushl_cfi $do_device_not_available
  890         jmp error_code
  891         CFI_ENDPROC
  892 END(device_not_available)
  893 
  894 #ifdef CONFIG_PARAVIRT
  895 ENTRY(native_iret)
  896         iret
  897         _ASM_EXTABLE(native_iret, iret_exc)
  898 END(native_iret)
  899 
  900 ENTRY(native_irq_enable_sysexit)
  901         sti
  902         sysexit
  903 END(native_irq_enable_sysexit)
  904 #endif
  905 
  906 ENTRY(overflow)
  907         RING0_INT_FRAME
  908         ASM_CLAC
  909         pushl_cfi $0
  910         pushl_cfi $do_overflow
  911         jmp error_code
  912         CFI_ENDPROC
  913 END(overflow)
  914 
  915 ENTRY(bounds)
  916         RING0_INT_FRAME
  917         ASM_CLAC
  918         pushl_cfi $0
  919         pushl_cfi $do_bounds
  920         jmp error_code
  921         CFI_ENDPROC
  922 END(bounds)
  923 
  924 ENTRY(invalid_op)
  925         RING0_INT_FRAME
  926         ASM_CLAC
  927         pushl_cfi $0
  928         pushl_cfi $do_invalid_op
  929         jmp error_code
  930         CFI_ENDPROC
  931 END(invalid_op)
  932 
  933 ENTRY(coprocessor_segment_overrun)
  934         RING0_INT_FRAME
  935         ASM_CLAC
  936         pushl_cfi $0
  937         pushl_cfi $do_coprocessor_segment_overrun
  938         jmp error_code
  939         CFI_ENDPROC
  940 END(coprocessor_segment_overrun)
  941 
  942 ENTRY(invalid_TSS)
  943         RING0_EC_FRAME
  944         ASM_CLAC
  945         pushl_cfi $do_invalid_TSS
  946         jmp error_code
  947         CFI_ENDPROC
  948 END(invalid_TSS)
  949 
  950 ENTRY(segment_not_present)
  951         RING0_EC_FRAME
  952         ASM_CLAC
  953         pushl_cfi $do_segment_not_present
  954         jmp error_code
  955         CFI_ENDPROC
  956 END(segment_not_present)
  957 
  958 ENTRY(stack_segment)
  959         RING0_EC_FRAME
  960         ASM_CLAC
  961         pushl_cfi $do_stack_segment
  962         jmp error_code
  963         CFI_ENDPROC
  964 END(stack_segment)
  965 
  966 ENTRY(alignment_check)
  967         RING0_EC_FRAME
  968         ASM_CLAC
  969         pushl_cfi $do_alignment_check
  970         jmp error_code
  971         CFI_ENDPROC
  972 END(alignment_check)
  973 
  974 ENTRY(divide_error)
  975         RING0_INT_FRAME
  976         ASM_CLAC
  977         pushl_cfi $0                    # no error code
  978         pushl_cfi $do_divide_error
  979         jmp error_code
  980         CFI_ENDPROC
  981 END(divide_error)
  982 
  983 #ifdef CONFIG_X86_MCE
  984 ENTRY(machine_check)
  985         RING0_INT_FRAME
  986         ASM_CLAC
  987         pushl_cfi $0
  988         pushl_cfi machine_check_vector
  989         jmp error_code
  990         CFI_ENDPROC
  991 END(machine_check)
  992 #endif
  993 
  994 ENTRY(spurious_interrupt_bug)
  995         RING0_INT_FRAME
  996         ASM_CLAC
  997         pushl_cfi $0
  998         pushl_cfi $do_spurious_interrupt_bug
  999         jmp error_code
 1000         CFI_ENDPROC
 1001 END(spurious_interrupt_bug)
 1002 /*
 1003  * End of kprobes section
 1004  */
 1005         .popsection
 1006 
 1007 #ifdef CONFIG_XEN
 1008 /* Xen doesn't set %esp to be precisely what the normal sysenter
 1009    entrypoint expects, so fix it up before using the normal path. */
 1010 ENTRY(xen_sysenter_target)
 1011         RING0_INT_FRAME
 1012         addl $5*4, %esp         /* remove xen-provided frame */
 1013         CFI_ADJUST_CFA_OFFSET -5*4
 1014         jmp sysenter_past_esp
 1015         CFI_ENDPROC
 1016 
 1017 ENTRY(xen_hypervisor_callback)
 1018         CFI_STARTPROC
 1019         pushl_cfi $-1 /* orig_ax = -1 => not a system call */
 1020         SAVE_ALL
 1021         TRACE_IRQS_OFF
 1022 
 1023         /* Check to see if we got the event in the critical
 1024            region in xen_iret_direct, after we've reenabled
 1025            events and checked for pending events.  This simulates
 1026            iret instruction's behaviour where it delivers a
 1027            pending interrupt when enabling interrupts. */
 1028         movl PT_EIP(%esp),%eax
 1029         cmpl $xen_iret_start_crit,%eax
 1030         jb   1f
 1031         cmpl $xen_iret_end_crit,%eax
 1032         jae  1f
 1033 
 1034         jmp  xen_iret_crit_fixup
 1035 
 1036 ENTRY(xen_do_upcall)
 1037 1:      mov %esp, %eax
 1038         call xen_evtchn_do_upcall
 1039         jmp  ret_from_intr
 1040         CFI_ENDPROC
 1041 ENDPROC(xen_hypervisor_callback)
 1042 
 1043 # Hypervisor uses this for application faults while it executes.
 1044 # We get here for two reasons:
 1045 #  1. Fault while reloading DS, ES, FS or GS
 1046 #  2. Fault while executing IRET
 1047 # Category 1 we fix up by reattempting the load, and zeroing the segment
 1048 # register if the load fails.
 1049 # Category 2 we fix up by jumping to do_iret_error. We cannot use the
 1050 # normal Linux return path in this case because if we use the IRET hypercall
 1051 # to pop the stack frame we end up in an infinite loop of failsafe callbacks.
 1052 # We distinguish between categories by maintaining a status value in EAX.
 1053 ENTRY(xen_failsafe_callback)
 1054         CFI_STARTPROC
 1055         pushl_cfi %eax
 1056         movl $1,%eax
 1057 1:      mov 4(%esp),%ds
 1058 2:      mov 8(%esp),%es
 1059 3:      mov 12(%esp),%fs
 1060 4:      mov 16(%esp),%gs
 1061         /* EAX == 0 => Category 1 (Bad segment)
 1062            EAX != 0 => Category 2 (Bad IRET) */
 1063         testl %eax,%eax
 1064         popl_cfi %eax
 1065         lea 16(%esp),%esp
 1066         CFI_ADJUST_CFA_OFFSET -16
 1067         jz 5f
 1068         jmp iret_exc
 1069 5:      pushl_cfi $-1 /* orig_ax = -1 => not a system call */
 1070         SAVE_ALL
 1071         jmp ret_from_exception
 1072         CFI_ENDPROC
 1073 
 1074 .section .fixup,"ax"
 1075 6:      xorl %eax,%eax
 1076         movl %eax,4(%esp)
 1077         jmp 1b
 1078 7:      xorl %eax,%eax
 1079         movl %eax,8(%esp)
 1080         jmp 2b
 1081 8:      xorl %eax,%eax
 1082         movl %eax,12(%esp)
 1083         jmp 3b
 1084 9:      xorl %eax,%eax
 1085         movl %eax,16(%esp)
 1086         jmp 4b
 1087 .previous
 1088         _ASM_EXTABLE(1b,6b)
 1089         _ASM_EXTABLE(2b,7b)
 1090         _ASM_EXTABLE(3b,8b)
 1091         _ASM_EXTABLE(4b,9b)
 1092 ENDPROC(xen_failsafe_callback)
 1093 
 1094 BUILD_INTERRUPT3(xen_hvm_callback_vector, XEN_HVM_EVTCHN_CALLBACK,
 1095                 xen_evtchn_do_upcall)
 1096 
 1097 #endif  /* CONFIG_XEN */
 1098 
 1099 #ifdef CONFIG_FUNCTION_TRACER
 1100 #ifdef CONFIG_DYNAMIC_FTRACE
 1101 
 1102 ENTRY(mcount)
 1103         ret
 1104 END(mcount)
 1105 
 1106 ENTRY(ftrace_caller)
 1107         cmpl $0, function_trace_stop
 1108         jne  ftrace_stub
 1109 
 1110         pushl %eax
 1111         pushl %ecx
 1112         pushl %edx
 1113         pushl $0        /* Pass NULL as regs pointer */
 1114         movl 4*4(%esp), %eax
 1115         movl 0x4(%ebp), %edx
 1116         leal function_trace_op, %ecx
 1117         subl $MCOUNT_INSN_SIZE, %eax
 1118 
 1119 .globl ftrace_call
 1120 ftrace_call:
 1121         call ftrace_stub
 1122 
 1123         addl $4,%esp    /* skip NULL pointer */
 1124         popl %edx
 1125         popl %ecx
 1126         popl %eax
 1127 ftrace_ret:
 1128 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 1129 .globl ftrace_graph_call
 1130 ftrace_graph_call:
 1131         jmp ftrace_stub
 1132 #endif
 1133 
 1134 .globl ftrace_stub
 1135 ftrace_stub:
 1136         ret
 1137 END(ftrace_caller)
 1138 
 1139 ENTRY(ftrace_regs_caller)
 1140         pushf   /* push flags before compare (in cs location) */
 1141         cmpl $0, function_trace_stop
 1142         jne ftrace_restore_flags
 1143 
 1144         /*
 1145          * i386 does not save SS and ESP when coming from kernel.
 1146          * Instead, to get sp, &regs->sp is used (see ptrace.h).
 1147          * Unfortunately, that means eflags must be at the same location
 1148          * as the current return ip is. We move the return ip into the
 1149          * ip location, and move flags into the return ip location.
 1150          */
 1151         pushl 4(%esp)   /* save return ip into ip slot */
 1152 
 1153         pushl $0        /* Load 0 into orig_ax */
 1154         pushl %gs
 1155         pushl %fs
 1156         pushl %es
 1157         pushl %ds
 1158         pushl %eax
 1159         pushl %ebp
 1160         pushl %edi
 1161         pushl %esi
 1162         pushl %edx
 1163         pushl %ecx
 1164         pushl %ebx
 1165 
 1166         movl 13*4(%esp), %eax   /* Get the saved flags */
 1167         movl %eax, 14*4(%esp)   /* Move saved flags into regs->flags location */
 1168                                 /* clobbering return ip */
 1169         movl $__KERNEL_CS,13*4(%esp)
 1170 
 1171         movl 12*4(%esp), %eax   /* Load ip (1st parameter) */
 1172         subl $MCOUNT_INSN_SIZE, %eax    /* Adjust ip */
 1173         movl 0x4(%ebp), %edx    /* Load parent ip (2nd parameter) */
 1174         leal function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */
 1175         pushl %esp              /* Save pt_regs as 4th parameter */
 1176 
 1177 GLOBAL(ftrace_regs_call)
 1178         call ftrace_stub
 1179 
 1180         addl $4, %esp           /* Skip pt_regs */
 1181         movl 14*4(%esp), %eax   /* Move flags back into cs */
 1182         movl %eax, 13*4(%esp)   /* Needed to keep addl from modifying flags */
 1183         movl 12*4(%esp), %eax   /* Get return ip from regs->ip */
 1184         movl %eax, 14*4(%esp)   /* Put return ip back for ret */
 1185 
 1186         popl %ebx
 1187         popl %ecx
 1188         popl %edx
 1189         popl %esi
 1190         popl %edi
 1191         popl %ebp
 1192         popl %eax
 1193         popl %ds
 1194         popl %es
 1195         popl %fs
 1196         popl %gs
 1197         addl $8, %esp           /* Skip orig_ax and ip */
 1198         popf                    /* Pop flags at end (no addl to corrupt flags) */
 1199         jmp ftrace_ret
 1200 
 1201 ftrace_restore_flags:
 1202         popf
 1203         jmp  ftrace_stub
 1204 #else /* ! CONFIG_DYNAMIC_FTRACE */
 1205 
 1206 ENTRY(mcount)
 1207         cmpl $0, function_trace_stop
 1208         jne  ftrace_stub
 1209 
 1210         cmpl $ftrace_stub, ftrace_trace_function
 1211         jnz trace
 1212 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 1213         cmpl $ftrace_stub, ftrace_graph_return
 1214         jnz ftrace_graph_caller
 1215 
 1216         cmpl $ftrace_graph_entry_stub, ftrace_graph_entry
 1217         jnz ftrace_graph_caller
 1218 #endif
 1219 .globl ftrace_stub
 1220 ftrace_stub:
 1221         ret
 1222 
 1223         /* taken from glibc */
 1224 trace:
 1225         pushl %eax
 1226         pushl %ecx
 1227         pushl %edx
 1228         movl 0xc(%esp), %eax
 1229         movl 0x4(%ebp), %edx
 1230         subl $MCOUNT_INSN_SIZE, %eax
 1231 
 1232         call *ftrace_trace_function
 1233 
 1234         popl %edx
 1235         popl %ecx
 1236         popl %eax
 1237         jmp ftrace_stub
 1238 END(mcount)
 1239 #endif /* CONFIG_DYNAMIC_FTRACE */
 1240 #endif /* CONFIG_FUNCTION_TRACER */
 1241 
 1242 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 1243 ENTRY(ftrace_graph_caller)
 1244         pushl %eax
 1245         pushl %ecx
 1246         pushl %edx
 1247         movl 0xc(%esp), %edx
 1248         lea 0x4(%ebp), %eax
 1249         movl (%ebp), %ecx
 1250         subl $MCOUNT_INSN_SIZE, %edx
 1251         call prepare_ftrace_return
 1252         popl %edx
 1253         popl %ecx
 1254         popl %eax
 1255         ret
 1256 END(ftrace_graph_caller)
 1257 
 1258 .globl return_to_handler
 1259 return_to_handler:
 1260         pushl %eax
 1261         pushl %edx
 1262         movl %ebp, %eax
 1263         call ftrace_return_to_handler
 1264         movl %eax, %ecx
 1265         popl %edx
 1266         popl %eax
 1267         jmp *%ecx
 1268 #endif
 1269 
 1270 /*
 1271  * Some functions should be protected against kprobes
 1272  */
 1273         .pushsection .kprobes.text, "ax"
 1274 
 1275 ENTRY(page_fault)
 1276         RING0_EC_FRAME
 1277         ASM_CLAC
 1278         pushl_cfi $do_page_fault
 1279         ALIGN
 1280 error_code:
 1281         /* the function address is in %gs's slot on the stack */
 1282         pushl_cfi %fs
 1283         /*CFI_REL_OFFSET fs, 0*/
 1284         pushl_cfi %es
 1285         /*CFI_REL_OFFSET es, 0*/
 1286         pushl_cfi %ds
 1287         /*CFI_REL_OFFSET ds, 0*/
 1288         pushl_cfi %eax
 1289         CFI_REL_OFFSET eax, 0
 1290         pushl_cfi %ebp
 1291         CFI_REL_OFFSET ebp, 0
 1292         pushl_cfi %edi
 1293         CFI_REL_OFFSET edi, 0
 1294         pushl_cfi %esi
 1295         CFI_REL_OFFSET esi, 0
 1296         pushl_cfi %edx
 1297         CFI_REL_OFFSET edx, 0
 1298         pushl_cfi %ecx
 1299         CFI_REL_OFFSET ecx, 0
 1300         pushl_cfi %ebx
 1301         CFI_REL_OFFSET ebx, 0
 1302         cld
 1303         movl $(__KERNEL_PERCPU), %ecx
 1304         movl %ecx, %fs
 1305         UNWIND_ESPFIX_STACK
 1306         GS_TO_REG %ecx
 1307         movl PT_GS(%esp), %edi          # get the function address
 1308         movl PT_ORIG_EAX(%esp), %edx    # get the error code
 1309         movl $-1, PT_ORIG_EAX(%esp)     # no syscall to restart
 1310         REG_TO_PTGS %ecx
 1311         SET_KERNEL_GS %ecx
 1312         movl $(__USER_DS), %ecx
 1313         movl %ecx, %ds
 1314         movl %ecx, %es
 1315         TRACE_IRQS_OFF
 1316         movl %esp,%eax                  # pt_regs pointer
 1317         call *%edi
 1318         jmp ret_from_exception
 1319         CFI_ENDPROC
 1320 END(page_fault)
 1321 
 1322 /*
 1323  * Debug traps and NMI can happen at the one SYSENTER instruction
 1324  * that sets up the real kernel stack. Check here, since we can't
 1325  * allow the wrong stack to be used.
 1326  *
 1327  * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
 1328  * already pushed 3 words if it hits on the sysenter instruction:
 1329  * eflags, cs and eip.
 1330  *
 1331  * We just load the right stack, and push the three (known) values
 1332  * by hand onto the new stack - while updating the return eip past
 1333  * the instruction that would have done it for sysenter.
 1334  */
 1335 .macro FIX_STACK offset ok label
 1336         cmpw $__KERNEL_CS, 4(%esp)
 1337         jne \ok
 1338 \label:
 1339         movl TSS_sysenter_sp0 + \offset(%esp), %esp
 1340         CFI_DEF_CFA esp, 0
 1341         CFI_UNDEFINED eip
 1342         pushfl_cfi
 1343         pushl_cfi $__KERNEL_CS
 1344         pushl_cfi $sysenter_past_esp
 1345         CFI_REL_OFFSET eip, 0
 1346 .endm
 1347 
 1348 ENTRY(debug)
 1349         RING0_INT_FRAME
 1350         ASM_CLAC
 1351         cmpl $ia32_sysenter_target,(%esp)
 1352         jne debug_stack_correct
 1353         FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn
 1354 debug_stack_correct:
 1355         pushl_cfi $-1                   # mark this as an int
 1356         SAVE_ALL
 1357         TRACE_IRQS_OFF
 1358         xorl %edx,%edx                  # error code 0
 1359         movl %esp,%eax                  # pt_regs pointer
 1360         call do_debug
 1361         jmp ret_from_exception
 1362         CFI_ENDPROC
 1363 END(debug)
 1364 
 1365 /*
 1366  * NMI is doubly nasty. It can happen _while_ we're handling
 1367  * a debug fault, and the debug fault hasn't yet been able to
 1368  * clear up the stack. So we first check whether we got  an
 1369  * NMI on the sysenter entry path, but after that we need to
 1370  * check whether we got an NMI on the debug path where the debug
 1371  * fault happened on the sysenter path.
 1372  */
 1373 ENTRY(nmi)
 1374         RING0_INT_FRAME
 1375         ASM_CLAC
 1376         pushl_cfi %eax
 1377         movl %ss, %eax
 1378         cmpw $__ESPFIX_SS, %ax
 1379         popl_cfi %eax
 1380         je nmi_espfix_stack
 1381         cmpl $ia32_sysenter_target,(%esp)
 1382         je nmi_stack_fixup
 1383         pushl_cfi %eax
 1384         movl %esp,%eax
 1385         /* Do not access memory above the end of our stack page,
 1386          * it might not exist.
 1387          */
 1388         andl $(THREAD_SIZE-1),%eax
 1389         cmpl $(THREAD_SIZE-20),%eax
 1390         popl_cfi %eax
 1391         jae nmi_stack_correct
 1392         cmpl $ia32_sysenter_target,12(%esp)
 1393         je nmi_debug_stack_check
 1394 nmi_stack_correct:
 1395         /* We have a RING0_INT_FRAME here */
 1396         pushl_cfi %eax
 1397         SAVE_ALL
 1398         xorl %edx,%edx          # zero error code
 1399         movl %esp,%eax          # pt_regs pointer
 1400         call do_nmi
 1401         jmp restore_all_notrace
 1402         CFI_ENDPROC
 1403 
 1404 nmi_stack_fixup:
 1405         RING0_INT_FRAME
 1406         FIX_STACK 12, nmi_stack_correct, 1
 1407         jmp nmi_stack_correct
 1408 
 1409 nmi_debug_stack_check:
 1410         /* We have a RING0_INT_FRAME here */
 1411         cmpw $__KERNEL_CS,16(%esp)
 1412         jne nmi_stack_correct
 1413         cmpl $debug,(%esp)
 1414         jb nmi_stack_correct
 1415         cmpl $debug_esp_fix_insn,(%esp)
 1416         ja nmi_stack_correct
 1417         FIX_STACK 24, nmi_stack_correct, 1
 1418         jmp nmi_stack_correct
 1419 
 1420 nmi_espfix_stack:
 1421         /* We have a RING0_INT_FRAME here.
 1422          *
 1423          * create the pointer to lss back
 1424          */
 1425         pushl_cfi %ss
 1426         pushl_cfi %esp
 1427         addl $4, (%esp)
 1428         /* copy the iret frame of 12 bytes */
 1429         .rept 3
 1430         pushl_cfi 16(%esp)
 1431         .endr
 1432         pushl_cfi %eax
 1433         SAVE_ALL
 1434         FIXUP_ESPFIX_STACK              # %eax == %esp
 1435         xorl %edx,%edx                  # zero error code
 1436         call do_nmi
 1437         RESTORE_REGS
 1438         lss 12+4(%esp), %esp            # back to espfix stack
 1439         CFI_ADJUST_CFA_OFFSET -24
 1440         jmp irq_return
 1441         CFI_ENDPROC
 1442 END(nmi)
 1443 
 1444 ENTRY(int3)
 1445         RING0_INT_FRAME
 1446         ASM_CLAC
 1447         pushl_cfi $-1                   # mark this as an int
 1448         SAVE_ALL
 1449         TRACE_IRQS_OFF
 1450         xorl %edx,%edx          # zero error code
 1451         movl %esp,%eax          # pt_regs pointer
 1452         call do_int3
 1453         jmp ret_from_exception
 1454         CFI_ENDPROC
 1455 END(int3)
 1456 
 1457 ENTRY(general_protection)
 1458         RING0_EC_FRAME
 1459         pushl_cfi $do_general_protection
 1460         jmp error_code
 1461         CFI_ENDPROC
 1462 END(general_protection)
 1463 
 1464 #ifdef CONFIG_KVM_GUEST
 1465 ENTRY(async_page_fault)
 1466         RING0_EC_FRAME
 1467         ASM_CLAC
 1468         pushl_cfi $do_async_page_fault
 1469         jmp error_code
 1470         CFI_ENDPROC
 1471 END(async_page_fault)
 1472 #endif
 1473 
 1474 /*
 1475  * End of kprobes section
 1476  */
 1477         .popsection

Cache object: 1312a35e8c86ab9262d852ee802e7644


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.