Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/arch/x86/kernel/entry_32.S
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* 2 * 3 * Copyright (C) 1991, 1992 Linus Torvalds 4 */ 5 6 /* 7 * entry.S contains the system-call and fault low-level handling routines. 8 * This also contains the timer-interrupt handler, as well as all interrupts 9 * and faults that can result in a task-switch. 10 * 11 * NOTE: This code handles signal-recognition, which happens every time 12 * after a timer-interrupt and after each system call. 13 * 14 * I changed all the .align's to 4 (16 byte alignment), as that's faster 15 * on a 486. 16 * 17 * Stack layout in 'syscall_exit': 18 * ptrace needs to have all regs on the stack. 19 * if the order here is changed, it needs to be 20 * updated in fork.c:copy_process, signal.c:do_signal, 21 * ptrace.c and ptrace.h 22 * 23 * 0(%esp) - %ebx 24 * 4(%esp) - %ecx 25 * 8(%esp) - %edx 26 * C(%esp) - %esi 27 * 10(%esp) - %edi 28 * 14(%esp) - %ebp 29 * 18(%esp) - %eax 30 * 1C(%esp) - %ds 31 * 20(%esp) - %es 32 * 24(%esp) - %fs 33 * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS 34 * 2C(%esp) - orig_eax 35 * 30(%esp) - %eip 36 * 34(%esp) - %cs 37 * 38(%esp) - %eflags 38 * 3C(%esp) - %oldesp 39 * 40(%esp) - %oldss 40 * 41 * "current" is in register %ebx during any slow entries. 42 */ 43 44 #include <linux/linkage.h> 45 #include <linux/err.h> 46 #include <asm/thread_info.h> 47 #include <asm/irqflags.h> 48 #include <asm/errno.h> 49 #include <asm/segment.h> 50 #include <asm/smp.h> 51 #include <asm/page_types.h> 52 #include <asm/percpu.h> 53 #include <asm/dwarf2.h> 54 #include <asm/processor-flags.h> 55 #include <asm/ftrace.h> 56 #include <asm/irq_vectors.h> 57 #include <asm/cpufeature.h> 58 #include <asm/alternative-asm.h> 59 #include <asm/asm.h> 60 #include <asm/smap.h> 61 62 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */ 63 #include <linux/elf-em.h> 64 #define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE) 65 #define __AUDIT_ARCH_LE 0x40000000 66 67 #ifndef CONFIG_AUDITSYSCALL 68 #define sysenter_audit syscall_trace_entry 69 #define sysexit_audit syscall_exit_work 70 #endif 71 72 .section .entry.text, "ax" 73 74 /* 75 * We use macros for low-level operations which need to be overridden 76 * for paravirtualization. The following will never clobber any registers: 77 * INTERRUPT_RETURN (aka. "iret") 78 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 79 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). 80 * 81 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 82 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 83 * Allowing a register to be clobbered can shrink the paravirt replacement 84 * enough to patch inline, increasing performance. 85 */ 86 87 #ifdef CONFIG_PREEMPT 88 #define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF 89 #else 90 #define preempt_stop(clobbers) 91 #define resume_kernel restore_all 92 #endif 93 94 .macro TRACE_IRQS_IRET 95 #ifdef CONFIG_TRACE_IRQFLAGS 96 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off? 97 jz 1f 98 TRACE_IRQS_ON 99 1: 100 #endif 101 .endm 102 103 /* 104 * User gs save/restore 105 * 106 * %gs is used for userland TLS and kernel only uses it for stack 107 * canary which is required to be at %gs:20 by gcc. Read the comment 108 * at the top of stackprotector.h for more info. 109 * 110 * Local labels 98 and 99 are used. 111 */ 112 #ifdef CONFIG_X86_32_LAZY_GS 113 114 /* unfortunately push/pop can't be no-op */ 115 .macro PUSH_GS 116 pushl_cfi $0 117 .endm 118 .macro POP_GS pop=0 119 addl $(4 + \pop), %esp 120 CFI_ADJUST_CFA_OFFSET -(4 + \pop) 121 .endm 122 .macro POP_GS_EX 123 .endm 124 125 /* all the rest are no-op */ 126 .macro PTGS_TO_GS 127 .endm 128 .macro PTGS_TO_GS_EX 129 .endm 130 .macro GS_TO_REG reg 131 .endm 132 .macro REG_TO_PTGS reg 133 .endm 134 .macro SET_KERNEL_GS reg 135 .endm 136 137 #else /* CONFIG_X86_32_LAZY_GS */ 138 139 .macro PUSH_GS 140 pushl_cfi %gs 141 /*CFI_REL_OFFSET gs, 0*/ 142 .endm 143 144 .macro POP_GS pop=0 145 98: popl_cfi %gs 146 /*CFI_RESTORE gs*/ 147 .if \pop <> 0 148 add $\pop, %esp 149 CFI_ADJUST_CFA_OFFSET -\pop 150 .endif 151 .endm 152 .macro POP_GS_EX 153 .pushsection .fixup, "ax" 154 99: movl $0, (%esp) 155 jmp 98b 156 .popsection 157 _ASM_EXTABLE(98b,99b) 158 .endm 159 160 .macro PTGS_TO_GS 161 98: mov PT_GS(%esp), %gs 162 .endm 163 .macro PTGS_TO_GS_EX 164 .pushsection .fixup, "ax" 165 99: movl $0, PT_GS(%esp) 166 jmp 98b 167 .popsection 168 _ASM_EXTABLE(98b,99b) 169 .endm 170 171 .macro GS_TO_REG reg 172 movl %gs, \reg 173 /*CFI_REGISTER gs, \reg*/ 174 .endm 175 .macro REG_TO_PTGS reg 176 movl \reg, PT_GS(%esp) 177 /*CFI_REL_OFFSET gs, PT_GS*/ 178 .endm 179 .macro SET_KERNEL_GS reg 180 movl $(__KERNEL_STACK_CANARY), \reg 181 movl \reg, %gs 182 .endm 183 184 #endif /* CONFIG_X86_32_LAZY_GS */ 185 186 .macro SAVE_ALL 187 cld 188 PUSH_GS 189 pushl_cfi %fs 190 /*CFI_REL_OFFSET fs, 0;*/ 191 pushl_cfi %es 192 /*CFI_REL_OFFSET es, 0;*/ 193 pushl_cfi %ds 194 /*CFI_REL_OFFSET ds, 0;*/ 195 pushl_cfi %eax 196 CFI_REL_OFFSET eax, 0 197 pushl_cfi %ebp 198 CFI_REL_OFFSET ebp, 0 199 pushl_cfi %edi 200 CFI_REL_OFFSET edi, 0 201 pushl_cfi %esi 202 CFI_REL_OFFSET esi, 0 203 pushl_cfi %edx 204 CFI_REL_OFFSET edx, 0 205 pushl_cfi %ecx 206 CFI_REL_OFFSET ecx, 0 207 pushl_cfi %ebx 208 CFI_REL_OFFSET ebx, 0 209 movl $(__USER_DS), %edx 210 movl %edx, %ds 211 movl %edx, %es 212 movl $(__KERNEL_PERCPU), %edx 213 movl %edx, %fs 214 SET_KERNEL_GS %edx 215 .endm 216 217 .macro RESTORE_INT_REGS 218 popl_cfi %ebx 219 CFI_RESTORE ebx 220 popl_cfi %ecx 221 CFI_RESTORE ecx 222 popl_cfi %edx 223 CFI_RESTORE edx 224 popl_cfi %esi 225 CFI_RESTORE esi 226 popl_cfi %edi 227 CFI_RESTORE edi 228 popl_cfi %ebp 229 CFI_RESTORE ebp 230 popl_cfi %eax 231 CFI_RESTORE eax 232 .endm 233 234 .macro RESTORE_REGS pop=0 235 RESTORE_INT_REGS 236 1: popl_cfi %ds 237 /*CFI_RESTORE ds;*/ 238 2: popl_cfi %es 239 /*CFI_RESTORE es;*/ 240 3: popl_cfi %fs 241 /*CFI_RESTORE fs;*/ 242 POP_GS \pop 243 .pushsection .fixup, "ax" 244 4: movl $0, (%esp) 245 jmp 1b 246 5: movl $0, (%esp) 247 jmp 2b 248 6: movl $0, (%esp) 249 jmp 3b 250 .popsection 251 _ASM_EXTABLE(1b,4b) 252 _ASM_EXTABLE(2b,5b) 253 _ASM_EXTABLE(3b,6b) 254 POP_GS_EX 255 .endm 256 257 .macro RING0_INT_FRAME 258 CFI_STARTPROC simple 259 CFI_SIGNAL_FRAME 260 CFI_DEF_CFA esp, 3*4 261 /*CFI_OFFSET cs, -2*4;*/ 262 CFI_OFFSET eip, -3*4 263 .endm 264 265 .macro RING0_EC_FRAME 266 CFI_STARTPROC simple 267 CFI_SIGNAL_FRAME 268 CFI_DEF_CFA esp, 4*4 269 /*CFI_OFFSET cs, -2*4;*/ 270 CFI_OFFSET eip, -3*4 271 .endm 272 273 .macro RING0_PTREGS_FRAME 274 CFI_STARTPROC simple 275 CFI_SIGNAL_FRAME 276 CFI_DEF_CFA esp, PT_OLDESP-PT_EBX 277 /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/ 278 CFI_OFFSET eip, PT_EIP-PT_OLDESP 279 /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/ 280 /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/ 281 CFI_OFFSET eax, PT_EAX-PT_OLDESP 282 CFI_OFFSET ebp, PT_EBP-PT_OLDESP 283 CFI_OFFSET edi, PT_EDI-PT_OLDESP 284 CFI_OFFSET esi, PT_ESI-PT_OLDESP 285 CFI_OFFSET edx, PT_EDX-PT_OLDESP 286 CFI_OFFSET ecx, PT_ECX-PT_OLDESP 287 CFI_OFFSET ebx, PT_EBX-PT_OLDESP 288 .endm 289 290 ENTRY(ret_from_fork) 291 CFI_STARTPROC 292 pushl_cfi %eax 293 call schedule_tail 294 GET_THREAD_INFO(%ebp) 295 popl_cfi %eax 296 pushl_cfi $0x0202 # Reset kernel eflags 297 popfl_cfi 298 jmp syscall_exit 299 CFI_ENDPROC 300 END(ret_from_fork) 301 302 ENTRY(ret_from_kernel_thread) 303 CFI_STARTPROC 304 pushl_cfi %eax 305 call schedule_tail 306 GET_THREAD_INFO(%ebp) 307 popl_cfi %eax 308 pushl_cfi $0x0202 # Reset kernel eflags 309 popfl_cfi 310 movl PT_EBP(%esp),%eax 311 call *PT_EBX(%esp) 312 movl $0,PT_EAX(%esp) 313 jmp syscall_exit 314 CFI_ENDPROC 315 ENDPROC(ret_from_kernel_thread) 316 317 /* 318 * Interrupt exit functions should be protected against kprobes 319 */ 320 .pushsection .kprobes.text, "ax" 321 /* 322 * Return to user mode is not as complex as all this looks, 323 * but we want the default path for a system call return to 324 * go as quickly as possible which is why some of this is 325 * less clear than it otherwise should be. 326 */ 327 328 # userspace resumption stub bypassing syscall exit tracing 329 ALIGN 330 RING0_PTREGS_FRAME 331 ret_from_exception: 332 preempt_stop(CLBR_ANY) 333 ret_from_intr: 334 GET_THREAD_INFO(%ebp) 335 #ifdef CONFIG_VM86 336 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS 337 movb PT_CS(%esp), %al 338 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax 339 #else 340 /* 341 * We can be coming here from child spawned by kernel_thread(). 342 */ 343 movl PT_CS(%esp), %eax 344 andl $SEGMENT_RPL_MASK, %eax 345 #endif 346 cmpl $USER_RPL, %eax 347 jb resume_kernel # not returning to v8086 or userspace 348 349 ENTRY(resume_userspace) 350 LOCKDEP_SYS_EXIT 351 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 352 # setting need_resched or sigpending 353 # between sampling and the iret 354 TRACE_IRQS_OFF 355 movl TI_flags(%ebp), %ecx 356 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on 357 # int/exception return? 358 jne work_pending 359 jmp restore_all 360 END(ret_from_exception) 361 362 #ifdef CONFIG_PREEMPT 363 ENTRY(resume_kernel) 364 DISABLE_INTERRUPTS(CLBR_ANY) 365 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ? 366 jnz restore_all 367 need_resched: 368 movl TI_flags(%ebp), %ecx # need_resched set ? 369 testb $_TIF_NEED_RESCHED, %cl 370 jz restore_all 371 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ? 372 jz restore_all 373 call preempt_schedule_irq 374 jmp need_resched 375 END(resume_kernel) 376 #endif 377 CFI_ENDPROC 378 /* 379 * End of kprobes section 380 */ 381 .popsection 382 383 /* SYSENTER_RETURN points to after the "sysenter" instruction in 384 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */ 385 386 # sysenter call handler stub 387 ENTRY(ia32_sysenter_target) 388 CFI_STARTPROC simple 389 CFI_SIGNAL_FRAME 390 CFI_DEF_CFA esp, 0 391 CFI_REGISTER esp, ebp 392 movl TSS_sysenter_sp0(%esp),%esp 393 sysenter_past_esp: 394 /* 395 * Interrupts are disabled here, but we can't trace it until 396 * enough kernel state to call TRACE_IRQS_OFF can be called - but 397 * we immediately enable interrupts at that point anyway. 398 */ 399 pushl_cfi $__USER_DS 400 /*CFI_REL_OFFSET ss, 0*/ 401 pushl_cfi %ebp 402 CFI_REL_OFFSET esp, 0 403 pushfl_cfi 404 orl $X86_EFLAGS_IF, (%esp) 405 pushl_cfi $__USER_CS 406 /*CFI_REL_OFFSET cs, 0*/ 407 /* 408 * Push current_thread_info()->sysenter_return to the stack. 409 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words 410 * pushed above; +8 corresponds to copy_thread's esp0 setting. 411 */ 412 pushl_cfi ((TI_sysenter_return)-THREAD_SIZE+8+4*4)(%esp) 413 CFI_REL_OFFSET eip, 0 414 415 pushl_cfi %eax 416 SAVE_ALL 417 ENABLE_INTERRUPTS(CLBR_NONE) 418 419 /* 420 * Load the potential sixth argument from user stack. 421 * Careful about security. 422 */ 423 cmpl $__PAGE_OFFSET-3,%ebp 424 jae syscall_fault 425 ASM_STAC 426 1: movl (%ebp),%ebp 427 ASM_CLAC 428 movl %ebp,PT_EBP(%esp) 429 _ASM_EXTABLE(1b,syscall_fault) 430 431 GET_THREAD_INFO(%ebp) 432 433 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp) 434 jnz sysenter_audit 435 sysenter_do_call: 436 cmpl $(NR_syscalls), %eax 437 jae syscall_badsys 438 call *sys_call_table(,%eax,4) 439 movl %eax,PT_EAX(%esp) 440 LOCKDEP_SYS_EXIT 441 DISABLE_INTERRUPTS(CLBR_ANY) 442 TRACE_IRQS_OFF 443 movl TI_flags(%ebp), %ecx 444 testl $_TIF_ALLWORK_MASK, %ecx 445 jne sysexit_audit 446 sysenter_exit: 447 /* if something modifies registers it must also disable sysexit */ 448 movl PT_EIP(%esp), %edx 449 movl PT_OLDESP(%esp), %ecx 450 xorl %ebp,%ebp 451 TRACE_IRQS_ON 452 1: mov PT_FS(%esp), %fs 453 PTGS_TO_GS 454 ENABLE_INTERRUPTS_SYSEXIT 455 456 #ifdef CONFIG_AUDITSYSCALL 457 sysenter_audit: 458 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%ebp) 459 jnz syscall_trace_entry 460 addl $4,%esp 461 CFI_ADJUST_CFA_OFFSET -4 462 /* %esi already in 8(%esp) 6th arg: 4th syscall arg */ 463 /* %edx already in 4(%esp) 5th arg: 3rd syscall arg */ 464 /* %ecx already in 0(%esp) 4th arg: 2nd syscall arg */ 465 movl %ebx,%ecx /* 3rd arg: 1st syscall arg */ 466 movl %eax,%edx /* 2nd arg: syscall number */ 467 movl $AUDIT_ARCH_I386,%eax /* 1st arg: audit arch */ 468 call __audit_syscall_entry 469 pushl_cfi %ebx 470 movl PT_EAX(%esp),%eax /* reload syscall number */ 471 jmp sysenter_do_call 472 473 sysexit_audit: 474 testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx 475 jne syscall_exit_work 476 TRACE_IRQS_ON 477 ENABLE_INTERRUPTS(CLBR_ANY) 478 movl %eax,%edx /* second arg, syscall return value */ 479 cmpl $-MAX_ERRNO,%eax /* is it an error ? */ 480 setbe %al /* 1 if so, 0 if not */ 481 movzbl %al,%eax /* zero-extend that */ 482 call __audit_syscall_exit 483 DISABLE_INTERRUPTS(CLBR_ANY) 484 TRACE_IRQS_OFF 485 movl TI_flags(%ebp), %ecx 486 testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx 487 jne syscall_exit_work 488 movl PT_EAX(%esp),%eax /* reload syscall return value */ 489 jmp sysenter_exit 490 #endif 491 492 CFI_ENDPROC 493 .pushsection .fixup,"ax" 494 2: movl $0,PT_FS(%esp) 495 jmp 1b 496 .popsection 497 _ASM_EXTABLE(1b,2b) 498 PTGS_TO_GS_EX 499 ENDPROC(ia32_sysenter_target) 500 501 /* 502 * syscall stub including irq exit should be protected against kprobes 503 */ 504 .pushsection .kprobes.text, "ax" 505 # system call handler stub 506 ENTRY(system_call) 507 RING0_INT_FRAME # can't unwind into user space anyway 508 ASM_CLAC 509 pushl_cfi %eax # save orig_eax 510 SAVE_ALL 511 GET_THREAD_INFO(%ebp) 512 # system call tracing in operation / emulation 513 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp) 514 jnz syscall_trace_entry 515 cmpl $(NR_syscalls), %eax 516 jae syscall_badsys 517 syscall_call: 518 call *sys_call_table(,%eax,4) 519 movl %eax,PT_EAX(%esp) # store the return value 520 syscall_exit: 521 LOCKDEP_SYS_EXIT 522 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 523 # setting need_resched or sigpending 524 # between sampling and the iret 525 TRACE_IRQS_OFF 526 movl TI_flags(%ebp), %ecx 527 testl $_TIF_ALLWORK_MASK, %ecx # current->work 528 jne syscall_exit_work 529 530 restore_all: 531 TRACE_IRQS_IRET 532 restore_all_notrace: 533 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS 534 # Warning: PT_OLDSS(%esp) contains the wrong/random values if we 535 # are returning to the kernel. 536 # See comments in process.c:copy_thread() for details. 537 movb PT_OLDSS(%esp), %ah 538 movb PT_CS(%esp), %al 539 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax 540 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax 541 CFI_REMEMBER_STATE 542 je ldt_ss # returning to user-space with LDT SS 543 restore_nocheck: 544 RESTORE_REGS 4 # skip orig_eax/error_code 545 irq_return: 546 INTERRUPT_RETURN 547 .section .fixup,"ax" 548 ENTRY(iret_exc) 549 pushl $0 # no error code 550 pushl $do_iret_error 551 jmp error_code 552 .previous 553 _ASM_EXTABLE(irq_return,iret_exc) 554 555 CFI_RESTORE_STATE 556 ldt_ss: 557 larl PT_OLDSS(%esp), %eax 558 jnz restore_nocheck 559 testl $0x00400000, %eax # returning to 32bit stack? 560 jnz restore_nocheck # allright, normal return 561 562 #ifdef CONFIG_PARAVIRT 563 /* 564 * The kernel can't run on a non-flat stack if paravirt mode 565 * is active. Rather than try to fixup the high bits of 566 * ESP, bypass this code entirely. This may break DOSemu 567 * and/or Wine support in a paravirt VM, although the option 568 * is still available to implement the setting of the high 569 * 16-bits in the INTERRUPT_RETURN paravirt-op. 570 */ 571 cmpl $0, pv_info+PARAVIRT_enabled 572 jne restore_nocheck 573 #endif 574 575 /* 576 * Setup and switch to ESPFIX stack 577 * 578 * We're returning to userspace with a 16 bit stack. The CPU will not 579 * restore the high word of ESP for us on executing iret... This is an 580 * "official" bug of all the x86-compatible CPUs, which we can work 581 * around to make dosemu and wine happy. We do this by preloading the 582 * high word of ESP with the high word of the userspace ESP while 583 * compensating for the offset by changing to the ESPFIX segment with 584 * a base address that matches for the difference. 585 */ 586 #define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8) 587 mov %esp, %edx /* load kernel esp */ 588 mov PT_OLDESP(%esp), %eax /* load userspace esp */ 589 mov %dx, %ax /* eax: new kernel esp */ 590 sub %eax, %edx /* offset (low word is 0) */ 591 shr $16, %edx 592 mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */ 593 mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */ 594 pushl_cfi $__ESPFIX_SS 595 pushl_cfi %eax /* new kernel esp */ 596 /* Disable interrupts, but do not irqtrace this section: we 597 * will soon execute iret and the tracer was already set to 598 * the irqstate after the iret */ 599 DISABLE_INTERRUPTS(CLBR_EAX) 600 lss (%esp), %esp /* switch to espfix segment */ 601 CFI_ADJUST_CFA_OFFSET -8 602 jmp restore_nocheck 603 CFI_ENDPROC 604 ENDPROC(system_call) 605 606 # perform work that needs to be done immediately before resumption 607 ALIGN 608 RING0_PTREGS_FRAME # can't unwind into user space anyway 609 work_pending: 610 testb $_TIF_NEED_RESCHED, %cl 611 jz work_notifysig 612 work_resched: 613 call schedule 614 LOCKDEP_SYS_EXIT 615 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt 616 # setting need_resched or sigpending 617 # between sampling and the iret 618 TRACE_IRQS_OFF 619 movl TI_flags(%ebp), %ecx 620 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other 621 # than syscall tracing? 622 jz restore_all 623 testb $_TIF_NEED_RESCHED, %cl 624 jnz work_resched 625 626 work_notifysig: # deal with pending signals and 627 # notify-resume requests 628 #ifdef CONFIG_VM86 629 testl $X86_EFLAGS_VM, PT_EFLAGS(%esp) 630 movl %esp, %eax 631 jne work_notifysig_v86 # returning to kernel-space or 632 # vm86-space 633 1: 634 #else 635 movl %esp, %eax 636 #endif 637 TRACE_IRQS_ON 638 ENABLE_INTERRUPTS(CLBR_NONE) 639 movb PT_CS(%esp), %bl 640 andb $SEGMENT_RPL_MASK, %bl 641 cmpb $USER_RPL, %bl 642 jb resume_kernel 643 xorl %edx, %edx 644 call do_notify_resume 645 jmp resume_userspace 646 647 #ifdef CONFIG_VM86 648 ALIGN 649 work_notifysig_v86: 650 pushl_cfi %ecx # save ti_flags for do_notify_resume 651 call save_v86_state # %eax contains pt_regs pointer 652 popl_cfi %ecx 653 movl %eax, %esp 654 jmp 1b 655 #endif 656 END(work_pending) 657 658 # perform syscall exit tracing 659 ALIGN 660 syscall_trace_entry: 661 movl $-ENOSYS,PT_EAX(%esp) 662 movl %esp, %eax 663 call syscall_trace_enter 664 /* What it returned is what we'll actually use. */ 665 cmpl $(NR_syscalls), %eax 666 jnae syscall_call 667 jmp syscall_exit 668 END(syscall_trace_entry) 669 670 # perform syscall exit tracing 671 ALIGN 672 syscall_exit_work: 673 testl $_TIF_WORK_SYSCALL_EXIT, %ecx 674 jz work_pending 675 TRACE_IRQS_ON 676 ENABLE_INTERRUPTS(CLBR_ANY) # could let syscall_trace_leave() call 677 # schedule() instead 678 movl %esp, %eax 679 call syscall_trace_leave 680 jmp resume_userspace 681 END(syscall_exit_work) 682 CFI_ENDPROC 683 684 RING0_INT_FRAME # can't unwind into user space anyway 685 syscall_fault: 686 ASM_CLAC 687 GET_THREAD_INFO(%ebp) 688 movl $-EFAULT,PT_EAX(%esp) 689 jmp resume_userspace 690 END(syscall_fault) 691 692 syscall_badsys: 693 movl $-ENOSYS,PT_EAX(%esp) 694 jmp resume_userspace 695 END(syscall_badsys) 696 CFI_ENDPROC 697 /* 698 * End of kprobes section 699 */ 700 .popsection 701 702 /* 703 * System calls that need a pt_regs pointer. 704 */ 705 #define PTREGSCALL0(name) \ 706 ENTRY(ptregs_##name) ; \ 707 leal 4(%esp),%eax; \ 708 jmp sys_##name; \ 709 ENDPROC(ptregs_##name) 710 711 #define PTREGSCALL1(name) \ 712 ENTRY(ptregs_##name) ; \ 713 leal 4(%esp),%edx; \ 714 movl (PT_EBX+4)(%esp),%eax; \ 715 jmp sys_##name; \ 716 ENDPROC(ptregs_##name) 717 718 #define PTREGSCALL2(name) \ 719 ENTRY(ptregs_##name) ; \ 720 leal 4(%esp),%ecx; \ 721 movl (PT_ECX+4)(%esp),%edx; \ 722 movl (PT_EBX+4)(%esp),%eax; \ 723 jmp sys_##name; \ 724 ENDPROC(ptregs_##name) 725 726 #define PTREGSCALL3(name) \ 727 ENTRY(ptregs_##name) ; \ 728 CFI_STARTPROC; \ 729 leal 4(%esp),%eax; \ 730 pushl_cfi %eax; \ 731 movl PT_EDX(%eax),%ecx; \ 732 movl PT_ECX(%eax),%edx; \ 733 movl PT_EBX(%eax),%eax; \ 734 call sys_##name; \ 735 addl $4,%esp; \ 736 CFI_ADJUST_CFA_OFFSET -4; \ 737 ret; \ 738 CFI_ENDPROC; \ 739 ENDPROC(ptregs_##name) 740 741 PTREGSCALL1(iopl) 742 PTREGSCALL0(sigreturn) 743 PTREGSCALL0(rt_sigreturn) 744 PTREGSCALL2(vm86) 745 PTREGSCALL1(vm86old) 746 747 .macro FIXUP_ESPFIX_STACK 748 /* 749 * Switch back for ESPFIX stack to the normal zerobased stack 750 * 751 * We can't call C functions using the ESPFIX stack. This code reads 752 * the high word of the segment base from the GDT and swiches to the 753 * normal stack and adjusts ESP with the matching offset. 754 */ 755 /* fixup the stack */ 756 mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */ 757 mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */ 758 shl $16, %eax 759 addl %esp, %eax /* the adjusted stack pointer */ 760 pushl_cfi $__KERNEL_DS 761 pushl_cfi %eax 762 lss (%esp), %esp /* switch to the normal stack segment */ 763 CFI_ADJUST_CFA_OFFSET -8 764 .endm 765 .macro UNWIND_ESPFIX_STACK 766 movl %ss, %eax 767 /* see if on espfix stack */ 768 cmpw $__ESPFIX_SS, %ax 769 jne 27f 770 movl $__KERNEL_DS, %eax 771 movl %eax, %ds 772 movl %eax, %es 773 /* switch to normal stack */ 774 FIXUP_ESPFIX_STACK 775 27: 776 .endm 777 778 /* 779 * Build the entry stubs and pointer table with some assembler magic. 780 * We pack 7 stubs into a single 32-byte chunk, which will fit in a 781 * single cache line on all modern x86 implementations. 782 */ 783 .section .init.rodata,"a" 784 ENTRY(interrupt) 785 .section .entry.text, "ax" 786 .p2align 5 787 .p2align CONFIG_X86_L1_CACHE_SHIFT 788 ENTRY(irq_entries_start) 789 RING0_INT_FRAME 790 vector=FIRST_EXTERNAL_VECTOR 791 .rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7 792 .balign 32 793 .rept 7 794 .if vector < NR_VECTORS 795 .if vector <> FIRST_EXTERNAL_VECTOR 796 CFI_ADJUST_CFA_OFFSET -4 797 .endif 798 1: pushl_cfi $(~vector+0x80) /* Note: always in signed byte range */ 799 .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6 800 jmp 2f 801 .endif 802 .previous 803 .long 1b 804 .section .entry.text, "ax" 805 vector=vector+1 806 .endif 807 .endr 808 2: jmp common_interrupt 809 .endr 810 END(irq_entries_start) 811 812 .previous 813 END(interrupt) 814 .previous 815 816 /* 817 * the CPU automatically disables interrupts when executing an IRQ vector, 818 * so IRQ-flags tracing has to follow that: 819 */ 820 .p2align CONFIG_X86_L1_CACHE_SHIFT 821 common_interrupt: 822 ASM_CLAC 823 addl $-0x80,(%esp) /* Adjust vector into the [-256,-1] range */ 824 SAVE_ALL 825 TRACE_IRQS_OFF 826 movl %esp,%eax 827 call do_IRQ 828 jmp ret_from_intr 829 ENDPROC(common_interrupt) 830 CFI_ENDPROC 831 832 /* 833 * Irq entries should be protected against kprobes 834 */ 835 .pushsection .kprobes.text, "ax" 836 #define BUILD_INTERRUPT3(name, nr, fn) \ 837 ENTRY(name) \ 838 RING0_INT_FRAME; \ 839 ASM_CLAC; \ 840 pushl_cfi $~(nr); \ 841 SAVE_ALL; \ 842 TRACE_IRQS_OFF \ 843 movl %esp,%eax; \ 844 call fn; \ 845 jmp ret_from_intr; \ 846 CFI_ENDPROC; \ 847 ENDPROC(name) 848 849 #define BUILD_INTERRUPT(name, nr) BUILD_INTERRUPT3(name, nr, smp_##name) 850 851 /* The include is where all of the SMP etc. interrupts come from */ 852 #include <asm/entry_arch.h> 853 854 ENTRY(coprocessor_error) 855 RING0_INT_FRAME 856 ASM_CLAC 857 pushl_cfi $0 858 pushl_cfi $do_coprocessor_error 859 jmp error_code 860 CFI_ENDPROC 861 END(coprocessor_error) 862 863 ENTRY(simd_coprocessor_error) 864 RING0_INT_FRAME 865 ASM_CLAC 866 pushl_cfi $0 867 #ifdef CONFIG_X86_INVD_BUG 868 /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */ 869 661: pushl_cfi $do_general_protection 870 662: 871 .section .altinstructions,"a" 872 altinstruction_entry 661b, 663f, X86_FEATURE_XMM, 662b-661b, 664f-663f 873 .previous 874 .section .altinstr_replacement,"ax" 875 663: pushl $do_simd_coprocessor_error 876 664: 877 .previous 878 #else 879 pushl_cfi $do_simd_coprocessor_error 880 #endif 881 jmp error_code 882 CFI_ENDPROC 883 END(simd_coprocessor_error) 884 885 ENTRY(device_not_available) 886 RING0_INT_FRAME 887 ASM_CLAC 888 pushl_cfi $-1 # mark this as an int 889 pushl_cfi $do_device_not_available 890 jmp error_code 891 CFI_ENDPROC 892 END(device_not_available) 893 894 #ifdef CONFIG_PARAVIRT 895 ENTRY(native_iret) 896 iret 897 _ASM_EXTABLE(native_iret, iret_exc) 898 END(native_iret) 899 900 ENTRY(native_irq_enable_sysexit) 901 sti 902 sysexit 903 END(native_irq_enable_sysexit) 904 #endif 905 906 ENTRY(overflow) 907 RING0_INT_FRAME 908 ASM_CLAC 909 pushl_cfi $0 910 pushl_cfi $do_overflow 911 jmp error_code 912 CFI_ENDPROC 913 END(overflow) 914 915 ENTRY(bounds) 916 RING0_INT_FRAME 917 ASM_CLAC 918 pushl_cfi $0 919 pushl_cfi $do_bounds 920 jmp error_code 921 CFI_ENDPROC 922 END(bounds) 923 924 ENTRY(invalid_op) 925 RING0_INT_FRAME 926 ASM_CLAC 927 pushl_cfi $0 928 pushl_cfi $do_invalid_op 929 jmp error_code 930 CFI_ENDPROC 931 END(invalid_op) 932 933 ENTRY(coprocessor_segment_overrun) 934 RING0_INT_FRAME 935 ASM_CLAC 936 pushl_cfi $0 937 pushl_cfi $do_coprocessor_segment_overrun 938 jmp error_code 939 CFI_ENDPROC 940 END(coprocessor_segment_overrun) 941 942 ENTRY(invalid_TSS) 943 RING0_EC_FRAME 944 ASM_CLAC 945 pushl_cfi $do_invalid_TSS 946 jmp error_code 947 CFI_ENDPROC 948 END(invalid_TSS) 949 950 ENTRY(segment_not_present) 951 RING0_EC_FRAME 952 ASM_CLAC 953 pushl_cfi $do_segment_not_present 954 jmp error_code 955 CFI_ENDPROC 956 END(segment_not_present) 957 958 ENTRY(stack_segment) 959 RING0_EC_FRAME 960 ASM_CLAC 961 pushl_cfi $do_stack_segment 962 jmp error_code 963 CFI_ENDPROC 964 END(stack_segment) 965 966 ENTRY(alignment_check) 967 RING0_EC_FRAME 968 ASM_CLAC 969 pushl_cfi $do_alignment_check 970 jmp error_code 971 CFI_ENDPROC 972 END(alignment_check) 973 974 ENTRY(divide_error) 975 RING0_INT_FRAME 976 ASM_CLAC 977 pushl_cfi $0 # no error code 978 pushl_cfi $do_divide_error 979 jmp error_code 980 CFI_ENDPROC 981 END(divide_error) 982 983 #ifdef CONFIG_X86_MCE 984 ENTRY(machine_check) 985 RING0_INT_FRAME 986 ASM_CLAC 987 pushl_cfi $0 988 pushl_cfi machine_check_vector 989 jmp error_code 990 CFI_ENDPROC 991 END(machine_check) 992 #endif 993 994 ENTRY(spurious_interrupt_bug) 995 RING0_INT_FRAME 996 ASM_CLAC 997 pushl_cfi $0 998 pushl_cfi $do_spurious_interrupt_bug 999 jmp error_code 1000 CFI_ENDPROC 1001 END(spurious_interrupt_bug) 1002 /* 1003 * End of kprobes section 1004 */ 1005 .popsection 1006 1007 #ifdef CONFIG_XEN 1008 /* Xen doesn't set %esp to be precisely what the normal sysenter 1009 entrypoint expects, so fix it up before using the normal path. */ 1010 ENTRY(xen_sysenter_target) 1011 RING0_INT_FRAME 1012 addl $5*4, %esp /* remove xen-provided frame */ 1013 CFI_ADJUST_CFA_OFFSET -5*4 1014 jmp sysenter_past_esp 1015 CFI_ENDPROC 1016 1017 ENTRY(xen_hypervisor_callback) 1018 CFI_STARTPROC 1019 pushl_cfi $-1 /* orig_ax = -1 => not a system call */ 1020 SAVE_ALL 1021 TRACE_IRQS_OFF 1022 1023 /* Check to see if we got the event in the critical 1024 region in xen_iret_direct, after we've reenabled 1025 events and checked for pending events. This simulates 1026 iret instruction's behaviour where it delivers a 1027 pending interrupt when enabling interrupts. */ 1028 movl PT_EIP(%esp),%eax 1029 cmpl $xen_iret_start_crit,%eax 1030 jb 1f 1031 cmpl $xen_iret_end_crit,%eax 1032 jae 1f 1033 1034 jmp xen_iret_crit_fixup 1035 1036 ENTRY(xen_do_upcall) 1037 1: mov %esp, %eax 1038 call xen_evtchn_do_upcall 1039 jmp ret_from_intr 1040 CFI_ENDPROC 1041 ENDPROC(xen_hypervisor_callback) 1042 1043 # Hypervisor uses this for application faults while it executes. 1044 # We get here for two reasons: 1045 # 1. Fault while reloading DS, ES, FS or GS 1046 # 2. Fault while executing IRET 1047 # Category 1 we fix up by reattempting the load, and zeroing the segment 1048 # register if the load fails. 1049 # Category 2 we fix up by jumping to do_iret_error. We cannot use the 1050 # normal Linux return path in this case because if we use the IRET hypercall 1051 # to pop the stack frame we end up in an infinite loop of failsafe callbacks. 1052 # We distinguish between categories by maintaining a status value in EAX. 1053 ENTRY(xen_failsafe_callback) 1054 CFI_STARTPROC 1055 pushl_cfi %eax 1056 movl $1,%eax 1057 1: mov 4(%esp),%ds 1058 2: mov 8(%esp),%es 1059 3: mov 12(%esp),%fs 1060 4: mov 16(%esp),%gs 1061 /* EAX == 0 => Category 1 (Bad segment) 1062 EAX != 0 => Category 2 (Bad IRET) */ 1063 testl %eax,%eax 1064 popl_cfi %eax 1065 lea 16(%esp),%esp 1066 CFI_ADJUST_CFA_OFFSET -16 1067 jz 5f 1068 jmp iret_exc 1069 5: pushl_cfi $-1 /* orig_ax = -1 => not a system call */ 1070 SAVE_ALL 1071 jmp ret_from_exception 1072 CFI_ENDPROC 1073 1074 .section .fixup,"ax" 1075 6: xorl %eax,%eax 1076 movl %eax,4(%esp) 1077 jmp 1b 1078 7: xorl %eax,%eax 1079 movl %eax,8(%esp) 1080 jmp 2b 1081 8: xorl %eax,%eax 1082 movl %eax,12(%esp) 1083 jmp 3b 1084 9: xorl %eax,%eax 1085 movl %eax,16(%esp) 1086 jmp 4b 1087 .previous 1088 _ASM_EXTABLE(1b,6b) 1089 _ASM_EXTABLE(2b,7b) 1090 _ASM_EXTABLE(3b,8b) 1091 _ASM_EXTABLE(4b,9b) 1092 ENDPROC(xen_failsafe_callback) 1093 1094 BUILD_INTERRUPT3(xen_hvm_callback_vector, XEN_HVM_EVTCHN_CALLBACK, 1095 xen_evtchn_do_upcall) 1096 1097 #endif /* CONFIG_XEN */ 1098 1099 #ifdef CONFIG_FUNCTION_TRACER 1100 #ifdef CONFIG_DYNAMIC_FTRACE 1101 1102 ENTRY(mcount) 1103 ret 1104 END(mcount) 1105 1106 ENTRY(ftrace_caller) 1107 cmpl $0, function_trace_stop 1108 jne ftrace_stub 1109 1110 pushl %eax 1111 pushl %ecx 1112 pushl %edx 1113 pushl $0 /* Pass NULL as regs pointer */ 1114 movl 4*4(%esp), %eax 1115 movl 0x4(%ebp), %edx 1116 leal function_trace_op, %ecx 1117 subl $MCOUNT_INSN_SIZE, %eax 1118 1119 .globl ftrace_call 1120 ftrace_call: 1121 call ftrace_stub 1122 1123 addl $4,%esp /* skip NULL pointer */ 1124 popl %edx 1125 popl %ecx 1126 popl %eax 1127 ftrace_ret: 1128 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 1129 .globl ftrace_graph_call 1130 ftrace_graph_call: 1131 jmp ftrace_stub 1132 #endif 1133 1134 .globl ftrace_stub 1135 ftrace_stub: 1136 ret 1137 END(ftrace_caller) 1138 1139 ENTRY(ftrace_regs_caller) 1140 pushf /* push flags before compare (in cs location) */ 1141 cmpl $0, function_trace_stop 1142 jne ftrace_restore_flags 1143 1144 /* 1145 * i386 does not save SS and ESP when coming from kernel. 1146 * Instead, to get sp, ®s->sp is used (see ptrace.h). 1147 * Unfortunately, that means eflags must be at the same location 1148 * as the current return ip is. We move the return ip into the 1149 * ip location, and move flags into the return ip location. 1150 */ 1151 pushl 4(%esp) /* save return ip into ip slot */ 1152 1153 pushl $0 /* Load 0 into orig_ax */ 1154 pushl %gs 1155 pushl %fs 1156 pushl %es 1157 pushl %ds 1158 pushl %eax 1159 pushl %ebp 1160 pushl %edi 1161 pushl %esi 1162 pushl %edx 1163 pushl %ecx 1164 pushl %ebx 1165 1166 movl 13*4(%esp), %eax /* Get the saved flags */ 1167 movl %eax, 14*4(%esp) /* Move saved flags into regs->flags location */ 1168 /* clobbering return ip */ 1169 movl $__KERNEL_CS,13*4(%esp) 1170 1171 movl 12*4(%esp), %eax /* Load ip (1st parameter) */ 1172 subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */ 1173 movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */ 1174 leal function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */ 1175 pushl %esp /* Save pt_regs as 4th parameter */ 1176 1177 GLOBAL(ftrace_regs_call) 1178 call ftrace_stub 1179 1180 addl $4, %esp /* Skip pt_regs */ 1181 movl 14*4(%esp), %eax /* Move flags back into cs */ 1182 movl %eax, 13*4(%esp) /* Needed to keep addl from modifying flags */ 1183 movl 12*4(%esp), %eax /* Get return ip from regs->ip */ 1184 movl %eax, 14*4(%esp) /* Put return ip back for ret */ 1185 1186 popl %ebx 1187 popl %ecx 1188 popl %edx 1189 popl %esi 1190 popl %edi 1191 popl %ebp 1192 popl %eax 1193 popl %ds 1194 popl %es 1195 popl %fs 1196 popl %gs 1197 addl $8, %esp /* Skip orig_ax and ip */ 1198 popf /* Pop flags at end (no addl to corrupt flags) */ 1199 jmp ftrace_ret 1200 1201 ftrace_restore_flags: 1202 popf 1203 jmp ftrace_stub 1204 #else /* ! CONFIG_DYNAMIC_FTRACE */ 1205 1206 ENTRY(mcount) 1207 cmpl $0, function_trace_stop 1208 jne ftrace_stub 1209 1210 cmpl $ftrace_stub, ftrace_trace_function 1211 jnz trace 1212 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 1213 cmpl $ftrace_stub, ftrace_graph_return 1214 jnz ftrace_graph_caller 1215 1216 cmpl $ftrace_graph_entry_stub, ftrace_graph_entry 1217 jnz ftrace_graph_caller 1218 #endif 1219 .globl ftrace_stub 1220 ftrace_stub: 1221 ret 1222 1223 /* taken from glibc */ 1224 trace: 1225 pushl %eax 1226 pushl %ecx 1227 pushl %edx 1228 movl 0xc(%esp), %eax 1229 movl 0x4(%ebp), %edx 1230 subl $MCOUNT_INSN_SIZE, %eax 1231 1232 call *ftrace_trace_function 1233 1234 popl %edx 1235 popl %ecx 1236 popl %eax 1237 jmp ftrace_stub 1238 END(mcount) 1239 #endif /* CONFIG_DYNAMIC_FTRACE */ 1240 #endif /* CONFIG_FUNCTION_TRACER */ 1241 1242 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 1243 ENTRY(ftrace_graph_caller) 1244 pushl %eax 1245 pushl %ecx 1246 pushl %edx 1247 movl 0xc(%esp), %edx 1248 lea 0x4(%ebp), %eax 1249 movl (%ebp), %ecx 1250 subl $MCOUNT_INSN_SIZE, %edx 1251 call prepare_ftrace_return 1252 popl %edx 1253 popl %ecx 1254 popl %eax 1255 ret 1256 END(ftrace_graph_caller) 1257 1258 .globl return_to_handler 1259 return_to_handler: 1260 pushl %eax 1261 pushl %edx 1262 movl %ebp, %eax 1263 call ftrace_return_to_handler 1264 movl %eax, %ecx 1265 popl %edx 1266 popl %eax 1267 jmp *%ecx 1268 #endif 1269 1270 /* 1271 * Some functions should be protected against kprobes 1272 */ 1273 .pushsection .kprobes.text, "ax" 1274 1275 ENTRY(page_fault) 1276 RING0_EC_FRAME 1277 ASM_CLAC 1278 pushl_cfi $do_page_fault 1279 ALIGN 1280 error_code: 1281 /* the function address is in %gs's slot on the stack */ 1282 pushl_cfi %fs 1283 /*CFI_REL_OFFSET fs, 0*/ 1284 pushl_cfi %es 1285 /*CFI_REL_OFFSET es, 0*/ 1286 pushl_cfi %ds 1287 /*CFI_REL_OFFSET ds, 0*/ 1288 pushl_cfi %eax 1289 CFI_REL_OFFSET eax, 0 1290 pushl_cfi %ebp 1291 CFI_REL_OFFSET ebp, 0 1292 pushl_cfi %edi 1293 CFI_REL_OFFSET edi, 0 1294 pushl_cfi %esi 1295 CFI_REL_OFFSET esi, 0 1296 pushl_cfi %edx 1297 CFI_REL_OFFSET edx, 0 1298 pushl_cfi %ecx 1299 CFI_REL_OFFSET ecx, 0 1300 pushl_cfi %ebx 1301 CFI_REL_OFFSET ebx, 0 1302 cld 1303 movl $(__KERNEL_PERCPU), %ecx 1304 movl %ecx, %fs 1305 UNWIND_ESPFIX_STACK 1306 GS_TO_REG %ecx 1307 movl PT_GS(%esp), %edi # get the function address 1308 movl PT_ORIG_EAX(%esp), %edx # get the error code 1309 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 1310 REG_TO_PTGS %ecx 1311 SET_KERNEL_GS %ecx 1312 movl $(__USER_DS), %ecx 1313 movl %ecx, %ds 1314 movl %ecx, %es 1315 TRACE_IRQS_OFF 1316 movl %esp,%eax # pt_regs pointer 1317 call *%edi 1318 jmp ret_from_exception 1319 CFI_ENDPROC 1320 END(page_fault) 1321 1322 /* 1323 * Debug traps and NMI can happen at the one SYSENTER instruction 1324 * that sets up the real kernel stack. Check here, since we can't 1325 * allow the wrong stack to be used. 1326 * 1327 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have 1328 * already pushed 3 words if it hits on the sysenter instruction: 1329 * eflags, cs and eip. 1330 * 1331 * We just load the right stack, and push the three (known) values 1332 * by hand onto the new stack - while updating the return eip past 1333 * the instruction that would have done it for sysenter. 1334 */ 1335 .macro FIX_STACK offset ok label 1336 cmpw $__KERNEL_CS, 4(%esp) 1337 jne \ok 1338 \label: 1339 movl TSS_sysenter_sp0 + \offset(%esp), %esp 1340 CFI_DEF_CFA esp, 0 1341 CFI_UNDEFINED eip 1342 pushfl_cfi 1343 pushl_cfi $__KERNEL_CS 1344 pushl_cfi $sysenter_past_esp 1345 CFI_REL_OFFSET eip, 0 1346 .endm 1347 1348 ENTRY(debug) 1349 RING0_INT_FRAME 1350 ASM_CLAC 1351 cmpl $ia32_sysenter_target,(%esp) 1352 jne debug_stack_correct 1353 FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn 1354 debug_stack_correct: 1355 pushl_cfi $-1 # mark this as an int 1356 SAVE_ALL 1357 TRACE_IRQS_OFF 1358 xorl %edx,%edx # error code 0 1359 movl %esp,%eax # pt_regs pointer 1360 call do_debug 1361 jmp ret_from_exception 1362 CFI_ENDPROC 1363 END(debug) 1364 1365 /* 1366 * NMI is doubly nasty. It can happen _while_ we're handling 1367 * a debug fault, and the debug fault hasn't yet been able to 1368 * clear up the stack. So we first check whether we got an 1369 * NMI on the sysenter entry path, but after that we need to 1370 * check whether we got an NMI on the debug path where the debug 1371 * fault happened on the sysenter path. 1372 */ 1373 ENTRY(nmi) 1374 RING0_INT_FRAME 1375 ASM_CLAC 1376 pushl_cfi %eax 1377 movl %ss, %eax 1378 cmpw $__ESPFIX_SS, %ax 1379 popl_cfi %eax 1380 je nmi_espfix_stack 1381 cmpl $ia32_sysenter_target,(%esp) 1382 je nmi_stack_fixup 1383 pushl_cfi %eax 1384 movl %esp,%eax 1385 /* Do not access memory above the end of our stack page, 1386 * it might not exist. 1387 */ 1388 andl $(THREAD_SIZE-1),%eax 1389 cmpl $(THREAD_SIZE-20),%eax 1390 popl_cfi %eax 1391 jae nmi_stack_correct 1392 cmpl $ia32_sysenter_target,12(%esp) 1393 je nmi_debug_stack_check 1394 nmi_stack_correct: 1395 /* We have a RING0_INT_FRAME here */ 1396 pushl_cfi %eax 1397 SAVE_ALL 1398 xorl %edx,%edx # zero error code 1399 movl %esp,%eax # pt_regs pointer 1400 call do_nmi 1401 jmp restore_all_notrace 1402 CFI_ENDPROC 1403 1404 nmi_stack_fixup: 1405 RING0_INT_FRAME 1406 FIX_STACK 12, nmi_stack_correct, 1 1407 jmp nmi_stack_correct 1408 1409 nmi_debug_stack_check: 1410 /* We have a RING0_INT_FRAME here */ 1411 cmpw $__KERNEL_CS,16(%esp) 1412 jne nmi_stack_correct 1413 cmpl $debug,(%esp) 1414 jb nmi_stack_correct 1415 cmpl $debug_esp_fix_insn,(%esp) 1416 ja nmi_stack_correct 1417 FIX_STACK 24, nmi_stack_correct, 1 1418 jmp nmi_stack_correct 1419 1420 nmi_espfix_stack: 1421 /* We have a RING0_INT_FRAME here. 1422 * 1423 * create the pointer to lss back 1424 */ 1425 pushl_cfi %ss 1426 pushl_cfi %esp 1427 addl $4, (%esp) 1428 /* copy the iret frame of 12 bytes */ 1429 .rept 3 1430 pushl_cfi 16(%esp) 1431 .endr 1432 pushl_cfi %eax 1433 SAVE_ALL 1434 FIXUP_ESPFIX_STACK # %eax == %esp 1435 xorl %edx,%edx # zero error code 1436 call do_nmi 1437 RESTORE_REGS 1438 lss 12+4(%esp), %esp # back to espfix stack 1439 CFI_ADJUST_CFA_OFFSET -24 1440 jmp irq_return 1441 CFI_ENDPROC 1442 END(nmi) 1443 1444 ENTRY(int3) 1445 RING0_INT_FRAME 1446 ASM_CLAC 1447 pushl_cfi $-1 # mark this as an int 1448 SAVE_ALL 1449 TRACE_IRQS_OFF 1450 xorl %edx,%edx # zero error code 1451 movl %esp,%eax # pt_regs pointer 1452 call do_int3 1453 jmp ret_from_exception 1454 CFI_ENDPROC 1455 END(int3) 1456 1457 ENTRY(general_protection) 1458 RING0_EC_FRAME 1459 pushl_cfi $do_general_protection 1460 jmp error_code 1461 CFI_ENDPROC 1462 END(general_protection) 1463 1464 #ifdef CONFIG_KVM_GUEST 1465 ENTRY(async_page_fault) 1466 RING0_EC_FRAME 1467 ASM_CLAC 1468 pushl_cfi $do_async_page_fault 1469 jmp error_code 1470 CFI_ENDPROC 1471 END(async_page_fault) 1472 #endif 1473 1474 /* 1475 * End of kprobes section 1476 */ 1477 .popsection
Cache object: 1312a35e8c86ab9262d852ee802e7644
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]