Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/exception.S
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 1989, 1990 William F. Jolitz. 3 * Copyright (c) 1990 The Regents of the University of California. 4 * Copyright (c) 2007 The FreeBSD Foundation 5 * All rights reserved. 6 * 7 * Portions of this software were developed by A. Joseph Koshy under 8 * sponsorship from the FreeBSD Foundation and Google, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 4. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * $FreeBSD$ 35 */ 36 37 #include "opt_atpic.h" 38 #include "opt_compat.h" 39 #include "opt_hwpmc_hooks.h" 40 #include "opt_kdtrace.h" 41 42 #include <machine/asmacros.h> 43 #include <machine/psl.h> 44 #include <machine/trap.h> 45 #include <machine/specialreg.h> 46 47 #include "assym.s" 48 49 #ifdef KDTRACE_HOOKS 50 .bss 51 .globl dtrace_invop_jump_addr 52 .align 8 53 .type dtrace_invop_jump_addr,@object 54 .size dtrace_invop_jump_addr,8 55 dtrace_invop_jump_addr: 56 .zero 8 57 .globl dtrace_invop_calltrap_addr 58 .align 8 59 .type dtrace_invop_calltrap_addr,@object 60 .size dtrace_invop_calltrap_addr,8 61 dtrace_invop_calltrap_addr: 62 .zero 8 63 #endif 64 .text 65 #ifdef HWPMC_HOOKS 66 ENTRY(start_exceptions) 67 #endif 68 69 /*****************************************************************************/ 70 /* Trap handling */ 71 /*****************************************************************************/ 72 /* 73 * Trap and fault vector routines. 74 * 75 * All traps are 'interrupt gates', SDT_SYSIGT. An interrupt gate pushes 76 * state on the stack but also disables interrupts. This is important for 77 * us for the use of the swapgs instruction. We cannot be interrupted 78 * until the GS.base value is correct. For most traps, we automatically 79 * then enable interrupts if the interrupted context had them enabled. 80 * This is equivalent to the i386 port's use of SDT_SYS386TGT. 81 * 82 * The cpu will push a certain amount of state onto the kernel stack for 83 * the current process. See amd64/include/frame.h. 84 * This includes the current RFLAGS (status register, which includes 85 * the interrupt disable state prior to the trap), the code segment register, 86 * and the return instruction pointer are pushed by the cpu. The cpu 87 * will also push an 'error' code for certain traps. We push a dummy 88 * error code for those traps where the cpu doesn't in order to maintain 89 * a consistent frame. We also push a contrived 'trap number'. 90 * 91 * The CPU does not push the general registers, so we must do that, and we 92 * must restore them prior to calling 'iret'. The CPU adjusts %cs and %ss 93 * but does not mess with %ds, %es, %gs or %fs. We swap the %gs base for 94 * for the kernel mode operation shortly, without changes to the selector 95 * loaded. Since superuser long mode works with any selectors loaded into 96 * segment registers other then %cs, which makes them mostly unused in long 97 * mode, and kernel does not reference %fs, leave them alone. The segment 98 * registers are reloaded on return to the usermode. 99 */ 100 101 MCOUNT_LABEL(user) 102 MCOUNT_LABEL(btrap) 103 104 /* Traps that we leave interrupts disabled for.. */ 105 #define TRAP_NOEN(a) \ 106 subq $TF_RIP,%rsp; \ 107 movl $(a),TF_TRAPNO(%rsp) ; \ 108 movq $0,TF_ADDR(%rsp) ; \ 109 movq $0,TF_ERR(%rsp) ; \ 110 jmp alltraps_noen 111 IDTVEC(bpt) 112 TRAP_NOEN(T_BPTFLT) 113 #ifdef KDTRACE_HOOKS 114 IDTVEC(dtrace_ret) 115 TRAP_NOEN(T_DTRACE_RET) 116 #endif 117 118 /* Regular traps; The cpu does not supply tf_err for these. */ 119 #define TRAP(a) \ 120 subq $TF_RIP,%rsp; \ 121 movl $(a),TF_TRAPNO(%rsp) ; \ 122 movq $0,TF_ADDR(%rsp) ; \ 123 movq $0,TF_ERR(%rsp) ; \ 124 jmp alltraps 125 IDTVEC(div) 126 TRAP(T_DIVIDE) 127 IDTVEC(ofl) 128 TRAP(T_OFLOW) 129 IDTVEC(bnd) 130 TRAP(T_BOUND) 131 IDTVEC(ill) 132 TRAP(T_PRIVINFLT) 133 IDTVEC(dna) 134 TRAP(T_DNA) 135 IDTVEC(fpusegm) 136 TRAP(T_FPOPFLT) 137 IDTVEC(mchk) 138 TRAP(T_MCHK) 139 IDTVEC(rsvd) 140 TRAP(T_RESERVED) 141 IDTVEC(fpu) 142 TRAP(T_ARITHTRAP) 143 IDTVEC(xmm) 144 TRAP(T_XMMFLT) 145 146 /* This group of traps have tf_err already pushed by the cpu */ 147 #define TRAP_ERR(a) \ 148 subq $TF_ERR,%rsp; \ 149 movl $(a),TF_TRAPNO(%rsp) ; \ 150 movq $0,TF_ADDR(%rsp) ; \ 151 jmp alltraps 152 IDTVEC(tss) 153 TRAP_ERR(T_TSSFLT) 154 IDTVEC(missing) 155 subq $TF_ERR,%rsp 156 movl $T_SEGNPFLT,TF_TRAPNO(%rsp) 157 jmp prot_addrf 158 IDTVEC(stk) 159 subq $TF_ERR,%rsp 160 movl $T_STKFLT,TF_TRAPNO(%rsp) 161 jmp prot_addrf 162 IDTVEC(align) 163 TRAP_ERR(T_ALIGNFLT) 164 165 /* 166 * alltraps entry point. Use swapgs if this is the first time in the 167 * kernel from userland. Reenable interrupts if they were enabled 168 * before the trap. This approximates SDT_SYS386TGT on the i386 port. 169 */ 170 SUPERALIGN_TEXT 171 .globl alltraps 172 .type alltraps,@function 173 alltraps: 174 movq %rdi,TF_RDI(%rsp) 175 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 176 jz alltraps_testi /* already running with kernel GS.base */ 177 swapgs 178 movq PCPU(CURPCB),%rdi 179 andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) 180 movw %fs,TF_FS(%rsp) 181 movw %gs,TF_GS(%rsp) 182 movw %es,TF_ES(%rsp) 183 movw %ds,TF_DS(%rsp) 184 alltraps_testi: 185 testl $PSL_I,TF_RFLAGS(%rsp) 186 jz alltraps_pushregs_no_rdi 187 sti 188 alltraps_pushregs_no_rdi: 189 movq %rsi,TF_RSI(%rsp) 190 movq %rdx,TF_RDX(%rsp) 191 movq %rcx,TF_RCX(%rsp) 192 movq %r8,TF_R8(%rsp) 193 movq %r9,TF_R9(%rsp) 194 movq %rax,TF_RAX(%rsp) 195 movq %rbx,TF_RBX(%rsp) 196 movq %rbp,TF_RBP(%rsp) 197 movq %r10,TF_R10(%rsp) 198 movq %r11,TF_R11(%rsp) 199 movq %r12,TF_R12(%rsp) 200 movq %r13,TF_R13(%rsp) 201 movq %r14,TF_R14(%rsp) 202 movq %r15,TF_R15(%rsp) 203 movl $TF_HASSEGS,TF_FLAGS(%rsp) 204 cld 205 FAKE_MCOUNT(TF_RIP(%rsp)) 206 #ifdef KDTRACE_HOOKS 207 /* 208 * DTrace Function Boundary Trace (fbt) probes are triggered 209 * by int3 (0xcc) which causes the #BP (T_BPTFLT) breakpoint 210 * interrupt. For all other trap types, just handle them in 211 * the usual way. 212 */ 213 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 214 jnz calltrap /* ignore userland traps */ 215 cmpl $T_BPTFLT,TF_TRAPNO(%rsp) 216 jne calltrap 217 218 /* Check if there is no DTrace hook registered. */ 219 cmpq $0,dtrace_invop_jump_addr 220 je calltrap 221 222 /* 223 * Set our jump address for the jump back in the event that 224 * the breakpoint wasn't caused by DTrace at all. 225 */ 226 movq $calltrap,dtrace_invop_calltrap_addr(%rip) 227 228 /* Jump to the code hooked in by DTrace. */ 229 movq dtrace_invop_jump_addr,%rax 230 jmpq *dtrace_invop_jump_addr 231 #endif 232 .globl calltrap 233 .type calltrap,@function 234 calltrap: 235 movq %rsp,%rdi 236 call trap 237 MEXITCOUNT 238 jmp doreti /* Handle any pending ASTs */ 239 240 /* 241 * alltraps_noen entry point. Unlike alltraps above, we want to 242 * leave the interrupts disabled. This corresponds to 243 * SDT_SYS386IGT on the i386 port. 244 */ 245 SUPERALIGN_TEXT 246 .globl alltraps_noen 247 .type alltraps_noen,@function 248 alltraps_noen: 249 movq %rdi,TF_RDI(%rsp) 250 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 251 jz 1f /* already running with kernel GS.base */ 252 swapgs 253 movq PCPU(CURPCB),%rdi 254 andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) 255 1: movw %fs,TF_FS(%rsp) 256 movw %gs,TF_GS(%rsp) 257 movw %es,TF_ES(%rsp) 258 movw %ds,TF_DS(%rsp) 259 jmp alltraps_pushregs_no_rdi 260 261 IDTVEC(dblfault) 262 subq $TF_ERR,%rsp 263 movl $T_DOUBLEFLT,TF_TRAPNO(%rsp) 264 movq $0,TF_ADDR(%rsp) 265 movq $0,TF_ERR(%rsp) 266 movq %rdi,TF_RDI(%rsp) 267 movq %rsi,TF_RSI(%rsp) 268 movq %rdx,TF_RDX(%rsp) 269 movq %rcx,TF_RCX(%rsp) 270 movq %r8,TF_R8(%rsp) 271 movq %r9,TF_R9(%rsp) 272 movq %rax,TF_RAX(%rsp) 273 movq %rbx,TF_RBX(%rsp) 274 movq %rbp,TF_RBP(%rsp) 275 movq %r10,TF_R10(%rsp) 276 movq %r11,TF_R11(%rsp) 277 movq %r12,TF_R12(%rsp) 278 movq %r13,TF_R13(%rsp) 279 movq %r14,TF_R14(%rsp) 280 movq %r15,TF_R15(%rsp) 281 movw %fs,TF_FS(%rsp) 282 movw %gs,TF_GS(%rsp) 283 movw %es,TF_ES(%rsp) 284 movw %ds,TF_DS(%rsp) 285 movl $TF_HASSEGS,TF_FLAGS(%rsp) 286 cld 287 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 288 jz 1f /* already running with kernel GS.base */ 289 swapgs 290 1: 291 movq %rsp,%rdi 292 call dblfault_handler 293 2: 294 hlt 295 jmp 2b 296 297 IDTVEC(page) 298 subq $TF_ERR,%rsp 299 movl $T_PAGEFLT,TF_TRAPNO(%rsp) 300 movq %rdi,TF_RDI(%rsp) /* free up a GP register */ 301 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 302 jz 1f /* already running with kernel GS.base */ 303 swapgs 304 movq PCPU(CURPCB),%rdi 305 andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) 306 1: movq %cr2,%rdi /* preserve %cr2 before .. */ 307 movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */ 308 movw %fs,TF_FS(%rsp) 309 movw %gs,TF_GS(%rsp) 310 movw %es,TF_ES(%rsp) 311 movw %ds,TF_DS(%rsp) 312 testl $PSL_I,TF_RFLAGS(%rsp) 313 jz alltraps_pushregs_no_rdi 314 sti 315 jmp alltraps_pushregs_no_rdi 316 317 /* 318 * We have to special-case this one. If we get a trap in doreti() at 319 * the iretq stage, we'll reenter with the wrong gs state. We'll have 320 * to do a special the swapgs in this case even coming from the kernel. 321 * XXX linux has a trap handler for their equivalent of load_gs(). 322 */ 323 IDTVEC(prot) 324 subq $TF_ERR,%rsp 325 movl $T_PROTFLT,TF_TRAPNO(%rsp) 326 prot_addrf: 327 movq $0,TF_ADDR(%rsp) 328 movq %rdi,TF_RDI(%rsp) /* free up a GP register */ 329 leaq doreti_iret(%rip),%rdi 330 cmpq %rdi,TF_RIP(%rsp) 331 je 1f /* kernel but with user gsbase!! */ 332 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 333 jz 2f /* already running with kernel GS.base */ 334 1: swapgs 335 2: movq PCPU(CURPCB),%rdi 336 orl $PCB_FULL_IRET,PCB_FLAGS(%rdi) /* always full iret from GPF */ 337 movw %fs,TF_FS(%rsp) 338 movw %gs,TF_GS(%rsp) 339 movw %es,TF_ES(%rsp) 340 movw %ds,TF_DS(%rsp) 341 testl $PSL_I,TF_RFLAGS(%rsp) 342 jz alltraps_pushregs_no_rdi 343 sti 344 jmp alltraps_pushregs_no_rdi 345 346 /* 347 * Fast syscall entry point. We enter here with just our new %cs/%ss set, 348 * and the new privilige level. We are still running on the old user stack 349 * pointer. We have to juggle a few things around to find our stack etc. 350 * swapgs gives us access to our PCPU space only. 351 * 352 * We do not support invoking this from a custom %cs or %ss (e.g. using 353 * entries from an LDT). 354 */ 355 IDTVEC(fast_syscall) 356 swapgs 357 movq %rsp,PCPU(SCRATCH_RSP) 358 movq PCPU(RSP0),%rsp 359 /* Now emulate a trapframe. Make the 8 byte alignment odd for call. */ 360 subq $TF_SIZE,%rsp 361 /* defer TF_RSP till we have a spare register */ 362 movq %r11,TF_RFLAGS(%rsp) 363 movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */ 364 movq PCPU(SCRATCH_RSP),%r11 /* %r11 already saved */ 365 movq %r11,TF_RSP(%rsp) /* user stack pointer */ 366 movw %fs,TF_FS(%rsp) 367 movw %gs,TF_GS(%rsp) 368 movw %es,TF_ES(%rsp) 369 movw %ds,TF_DS(%rsp) 370 movq PCPU(CURPCB),%r11 371 andl $~PCB_FULL_IRET,PCB_FLAGS(%r11) 372 sti 373 movq $KUDSEL,TF_SS(%rsp) 374 movq $KUCSEL,TF_CS(%rsp) 375 movq $2,TF_ERR(%rsp) 376 movq %rdi,TF_RDI(%rsp) /* arg 1 */ 377 movq %rsi,TF_RSI(%rsp) /* arg 2 */ 378 movq %rdx,TF_RDX(%rsp) /* arg 3 */ 379 movq %r10,TF_RCX(%rsp) /* arg 4 */ 380 movq %r8,TF_R8(%rsp) /* arg 5 */ 381 movq %r9,TF_R9(%rsp) /* arg 6 */ 382 movq %rax,TF_RAX(%rsp) /* syscall number */ 383 movq %rbx,TF_RBX(%rsp) /* C preserved */ 384 movq %rbp,TF_RBP(%rsp) /* C preserved */ 385 movq %r12,TF_R12(%rsp) /* C preserved */ 386 movq %r13,TF_R13(%rsp) /* C preserved */ 387 movq %r14,TF_R14(%rsp) /* C preserved */ 388 movq %r15,TF_R15(%rsp) /* C preserved */ 389 movl $TF_HASSEGS,TF_FLAGS(%rsp) 390 cld 391 FAKE_MCOUNT(TF_RIP(%rsp)) 392 movq PCPU(CURTHREAD),%rdi 393 movq %rsp,TD_FRAME(%rdi) 394 movl TF_RFLAGS(%rsp),%esi 395 andl $PSL_T,%esi 396 call amd64_syscall 397 1: movq PCPU(CURPCB),%rax 398 /* Disable interrupts before testing PCB_FULL_IRET. */ 399 cli 400 testl $PCB_FULL_IRET,PCB_FLAGS(%rax) 401 jnz 3f 402 /* Check for and handle AST's on return to userland. */ 403 movq PCPU(CURTHREAD),%rax 404 testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) 405 jne 2f 406 /* Restore preserved registers. */ 407 MEXITCOUNT 408 movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ 409 movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */ 410 movq TF_RDX(%rsp),%rdx /* return value 2 */ 411 movq TF_RAX(%rsp),%rax /* return value 1 */ 412 movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ 413 movq TF_RIP(%rsp),%rcx /* original %rip */ 414 movq TF_RSP(%rsp),%rsp /* user stack pointer */ 415 swapgs 416 sysretq 417 418 2: /* AST scheduled. */ 419 sti 420 movq %rsp,%rdi 421 call ast 422 jmp 1b 423 424 3: /* Requested full context restore, use doreti for that. */ 425 MEXITCOUNT 426 jmp doreti 427 428 /* 429 * Here for CYA insurance, in case a "syscall" instruction gets 430 * issued from 32 bit compatability mode. MSR_CSTAR has to point 431 * to *something* if EFER_SCE is enabled. 432 */ 433 IDTVEC(fast_syscall32) 434 sysret 435 436 /* 437 * DB# handler is very similar to NM#, because 'mov/pop %ss' delay 438 * generation of exception until the next instruction is executed, 439 * which might be a kernel entry. So we must execute the handler 440 * on IST stack and be ready for non-kernel GSBASE. 441 */ 442 IDTVEC(dbg) 443 subq $TF_RIP,%rsp 444 movl $(T_TRCTRAP),TF_TRAPNO(%rsp) 445 movq $0,TF_ADDR(%rsp) 446 movq $0,TF_ERR(%rsp) 447 movq %rdi,TF_RDI(%rsp) 448 movq %rsi,TF_RSI(%rsp) 449 movq %rdx,TF_RDX(%rsp) 450 movq %rcx,TF_RCX(%rsp) 451 movq %r8,TF_R8(%rsp) 452 movq %r9,TF_R9(%rsp) 453 movq %rax,TF_RAX(%rsp) 454 movq %rbx,TF_RBX(%rsp) 455 movq %rbp,TF_RBP(%rsp) 456 movq %r10,TF_R10(%rsp) 457 movq %r11,TF_R11(%rsp) 458 movq %r12,TF_R12(%rsp) 459 movq %r13,TF_R13(%rsp) 460 movq %r14,TF_R14(%rsp) 461 movq %r15,TF_R15(%rsp) 462 movw %fs,TF_FS(%rsp) 463 movw %gs,TF_GS(%rsp) 464 movw %es,TF_ES(%rsp) 465 movw %ds,TF_DS(%rsp) 466 movl $TF_HASSEGS,TF_FLAGS(%rsp) 467 cld 468 testb $SEL_RPL_MASK,TF_CS(%rsp) 469 jnz dbg_fromuserspace 470 /* 471 * We've interrupted the kernel. Preserve GS.base in %r12. 472 */ 473 movl $MSR_GSBASE,%ecx 474 rdmsr 475 movq %rax,%r12 476 shlq $32,%rdx 477 orq %rdx,%r12 478 /* Retrieve and load the canonical value for GS.base. */ 479 movq TF_SIZE(%rsp),%rdx 480 movl %edx,%eax 481 shrq $32,%rdx 482 wrmsr 483 FAKE_MCOUNT(TF_RIP(%rsp)) 484 movq %rsp,%rdi 485 call trap 486 MEXITCOUNT 487 /* 488 * Put back the preserved MSR_GSBASE value. 489 */ 490 movl $MSR_GSBASE,%ecx 491 movq %r12,%rdx 492 movl %edx,%eax 493 shrq $32,%rdx 494 wrmsr 495 movq TF_RDI(%rsp),%rdi 496 movq TF_RSI(%rsp),%rsi 497 movq TF_RDX(%rsp),%rdx 498 movq TF_RCX(%rsp),%rcx 499 movq TF_R8(%rsp),%r8 500 movq TF_R9(%rsp),%r9 501 movq TF_RAX(%rsp),%rax 502 movq TF_RBX(%rsp),%rbx 503 movq TF_RBP(%rsp),%rbp 504 movq TF_R10(%rsp),%r10 505 movq TF_R11(%rsp),%r11 506 movq TF_R12(%rsp),%r12 507 movq TF_R13(%rsp),%r13 508 movq TF_R14(%rsp),%r14 509 movq TF_R15(%rsp),%r15 510 addq $TF_RIP,%rsp 511 jmp doreti_iret 512 dbg_fromuserspace: 513 /* 514 * Switch to kernel GSBASE and kernel page table, and copy frame 515 * from the IST stack to the normal kernel stack, since trap() 516 * re-enables interrupts, and since we might trap on DB# while 517 * in trap(). 518 */ 519 swapgs 520 movq PCPU(RSP0),%rax 521 movl $TF_SIZE,%ecx 522 subq %rcx,%rax 523 movq %rax,%rdi 524 movq %rsp,%rsi 525 rep;movsb 526 movq %rax,%rsp 527 movq PCPU(CURPCB),%rdi 528 orl $PCB_FULL_IRET,PCB_FLAGS(%rdi) 529 jmp calltrap 530 531 /* 532 * NMI handling is special. 533 * 534 * First, NMIs do not respect the state of the processor's RFLAGS.IF 535 * bit. The NMI handler may be entered at any time, including when 536 * the processor is in a critical section with RFLAGS.IF == 0. 537 * The processor's GS.base value could be invalid on entry to the 538 * handler. 539 * 540 * Second, the processor treats NMIs specially, blocking further NMIs 541 * until an 'iretq' instruction is executed. We thus need to execute 542 * the NMI handler with interrupts disabled, to prevent a nested interrupt 543 * from executing an 'iretq' instruction and inadvertently taking the 544 * processor out of NMI mode. 545 * 546 * Third, the NMI handler runs on its own stack (tss_ist2). The canonical 547 * GS.base value for the processor is stored just above the bottom of its 548 * NMI stack. For NMIs taken from kernel mode, the current value in 549 * the processor's GS.base is saved at entry to C-preserved register %r12, 550 * the canonical value for GS.base is then loaded into the processor, and 551 * the saved value is restored at exit time. For NMIs taken from user mode, 552 * the cheaper 'SWAPGS' instructions are used for swapping GS.base. 553 */ 554 555 IDTVEC(nmi) 556 subq $TF_RIP,%rsp 557 movl $(T_NMI),TF_TRAPNO(%rsp) 558 movq $0,TF_ADDR(%rsp) 559 movq $0,TF_ERR(%rsp) 560 movq %rdi,TF_RDI(%rsp) 561 movq %rsi,TF_RSI(%rsp) 562 movq %rdx,TF_RDX(%rsp) 563 movq %rcx,TF_RCX(%rsp) 564 movq %r8,TF_R8(%rsp) 565 movq %r9,TF_R9(%rsp) 566 movq %rax,TF_RAX(%rsp) 567 movq %rbx,TF_RBX(%rsp) 568 movq %rbp,TF_RBP(%rsp) 569 movq %r10,TF_R10(%rsp) 570 movq %r11,TF_R11(%rsp) 571 movq %r12,TF_R12(%rsp) 572 movq %r13,TF_R13(%rsp) 573 movq %r14,TF_R14(%rsp) 574 movq %r15,TF_R15(%rsp) 575 movw %fs,TF_FS(%rsp) 576 movw %gs,TF_GS(%rsp) 577 movw %es,TF_ES(%rsp) 578 movw %ds,TF_DS(%rsp) 579 movl $TF_HASSEGS,TF_FLAGS(%rsp) 580 cld 581 xorl %ebx,%ebx 582 testb $SEL_RPL_MASK,TF_CS(%rsp) 583 jnz nmi_fromuserspace 584 /* 585 * We've interrupted the kernel. Preserve GS.base in %r12. 586 */ 587 movl $MSR_GSBASE,%ecx 588 rdmsr 589 movq %rax,%r12 590 shlq $32,%rdx 591 orq %rdx,%r12 592 /* Retrieve and load the canonical value for GS.base. */ 593 movq TF_SIZE(%rsp),%rdx 594 movl %edx,%eax 595 shrq $32,%rdx 596 wrmsr 597 jmp nmi_calltrap 598 nmi_fromuserspace: 599 incl %ebx 600 swapgs 601 /* Note: this label is also used by ddb and gdb: */ 602 nmi_calltrap: 603 FAKE_MCOUNT(TF_RIP(%rsp)) 604 movq %rsp,%rdi 605 call trap 606 MEXITCOUNT 607 #ifdef HWPMC_HOOKS 608 /* 609 * Capture a userspace callchain if needed. 610 * 611 * - Check if the current trap was from user mode. 612 * - Check if the current thread is valid. 613 * - Check if the thread requires a user call chain to be 614 * captured. 615 * 616 * We are still in NMI mode at this point. 617 */ 618 testl %ebx,%ebx 619 jz nocallchain /* not from userspace */ 620 movq PCPU(CURTHREAD),%rax 621 orq %rax,%rax /* curthread present? */ 622 jz nocallchain 623 testl $TDP_CALLCHAIN,TD_PFLAGS(%rax) /* flagged for capture? */ 624 jz nocallchain 625 /* 626 * A user callchain is to be captured, so: 627 * - Move execution to the regular kernel stack, to allow for 628 * nested NMI interrupts. 629 * - Take the processor out of "NMI" mode by faking an "iret". 630 * - Enable interrupts, so that copyin() can work. 631 */ 632 movq %rsp,%rsi /* source stack pointer */ 633 movq $TF_SIZE,%rcx 634 movq PCPU(RSP0),%rdx 635 subq %rcx,%rdx 636 movq %rdx,%rdi /* destination stack pointer */ 637 638 shrq $3,%rcx /* trap frame size in long words */ 639 cld 640 rep 641 movsq /* copy trapframe */ 642 643 movl %ss,%eax 644 pushq %rax /* tf_ss */ 645 pushq %rdx /* tf_rsp (on kernel stack) */ 646 pushfq /* tf_rflags */ 647 movl %cs,%eax 648 pushq %rax /* tf_cs */ 649 pushq $outofnmi /* tf_rip */ 650 iretq 651 outofnmi: 652 /* 653 * At this point the processor has exited NMI mode and is running 654 * with interrupts turned off on the normal kernel stack. 655 * 656 * If a pending NMI gets recognized at or after this point, it 657 * will cause a kernel callchain to be traced. 658 * 659 * We turn interrupts back on, and call the user callchain capture hook. 660 */ 661 movq pmc_hook,%rax 662 orq %rax,%rax 663 jz nocallchain 664 movq PCPU(CURTHREAD),%rdi /* thread */ 665 movq $PMC_FN_USER_CALLCHAIN,%rsi /* command */ 666 movq %rsp,%rdx /* frame */ 667 sti 668 call *%rax 669 cli 670 nocallchain: 671 #endif 672 testl %ebx,%ebx 673 jnz doreti_exit 674 nmi_kernelexit: 675 /* 676 * Put back the preserved MSR_GSBASE value. 677 */ 678 movl $MSR_GSBASE,%ecx 679 movq %r12,%rdx 680 movl %edx,%eax 681 shrq $32,%rdx 682 wrmsr 683 nmi_restoreregs: 684 movq TF_RDI(%rsp),%rdi 685 movq TF_RSI(%rsp),%rsi 686 movq TF_RDX(%rsp),%rdx 687 movq TF_RCX(%rsp),%rcx 688 movq TF_R8(%rsp),%r8 689 movq TF_R9(%rsp),%r9 690 movq TF_RAX(%rsp),%rax 691 movq TF_RBX(%rsp),%rbx 692 movq TF_RBP(%rsp),%rbp 693 movq TF_R10(%rsp),%r10 694 movq TF_R11(%rsp),%r11 695 movq TF_R12(%rsp),%r12 696 movq TF_R13(%rsp),%r13 697 movq TF_R14(%rsp),%r14 698 movq TF_R15(%rsp),%r15 699 addq $TF_RIP,%rsp 700 jmp doreti_iret 701 702 ENTRY(fork_trampoline) 703 movq %r12,%rdi /* function */ 704 movq %rbx,%rsi /* arg1 */ 705 movq %rsp,%rdx /* trapframe pointer */ 706 call fork_exit 707 MEXITCOUNT 708 jmp doreti /* Handle any ASTs */ 709 710 /* 711 * To efficiently implement classification of trap and interrupt handlers 712 * for profiling, there must be only trap handlers between the labels btrap 713 * and bintr, and only interrupt handlers between the labels bintr and 714 * eintr. This is implemented (partly) by including files that contain 715 * some of the handlers. Before including the files, set up a normal asm 716 * environment so that the included files doen't need to know that they are 717 * included. 718 */ 719 720 #ifdef COMPAT_FREEBSD32 721 .data 722 .p2align 4 723 .text 724 SUPERALIGN_TEXT 725 726 #include <amd64/ia32/ia32_exception.S> 727 #endif 728 729 .data 730 .p2align 4 731 .text 732 SUPERALIGN_TEXT 733 MCOUNT_LABEL(bintr) 734 735 #include <amd64/amd64/apic_vector.S> 736 737 #ifdef DEV_ATPIC 738 .data 739 .p2align 4 740 .text 741 SUPERALIGN_TEXT 742 743 #include <amd64/amd64/atpic_vector.S> 744 #endif 745 746 .text 747 MCOUNT_LABEL(eintr) 748 749 /* 750 * void doreti(struct trapframe) 751 * 752 * Handle return from interrupts, traps and syscalls. 753 */ 754 .text 755 SUPERALIGN_TEXT 756 .type doreti,@function 757 .globl doreti 758 doreti: 759 FAKE_MCOUNT($bintr) /* init "from" bintr -> doreti */ 760 /* 761 * Check if ASTs can be handled now. 762 */ 763 testb $SEL_RPL_MASK,TF_CS(%rsp) /* are we returning to user mode? */ 764 jz doreti_exit /* can't handle ASTs now if not */ 765 766 doreti_ast: 767 /* 768 * Check for ASTs atomically with returning. Disabling CPU 769 * interrupts provides sufficient locking even in the SMP case, 770 * since we will be informed of any new ASTs by an IPI. 771 */ 772 cli 773 movq PCPU(CURTHREAD),%rax 774 testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) 775 je doreti_exit 776 sti 777 movq %rsp,%rdi /* pass a pointer to the trapframe */ 778 call ast 779 jmp doreti_ast 780 781 /* 782 * doreti_exit: pop registers, iret. 783 * 784 * The segment register pop is a special case, since it may 785 * fault if (for example) a sigreturn specifies bad segment 786 * registers. The fault is handled in trap.c. 787 */ 788 doreti_exit: 789 MEXITCOUNT 790 movq PCPU(CURPCB),%r8 791 792 /* 793 * Do not reload segment registers for kernel. 794 * Since we do not reload segments registers with sane 795 * values on kernel entry, descriptors referenced by 796 * segments registers might be not valid. This is fatal 797 * for user mode, but is not a problem for the kernel. 798 */ 799 testb $SEL_RPL_MASK,TF_CS(%rsp) 800 jz ld_regs 801 testl $PCB_FULL_IRET,PCB_FLAGS(%r8) 802 jz ld_regs 803 testl $TF_HASSEGS,TF_FLAGS(%rsp) 804 je set_segs 805 806 do_segs: 807 /* Restore %fs and fsbase */ 808 movw TF_FS(%rsp),%ax 809 .globl ld_fs 810 ld_fs: 811 movw %ax,%fs 812 cmpw $KUF32SEL,%ax 813 jne 1f 814 movl $MSR_FSBASE,%ecx 815 movl PCB_FSBASE(%r8),%eax 816 movl PCB_FSBASE+4(%r8),%edx 817 .globl ld_fsbase 818 ld_fsbase: 819 wrmsr 820 1: 821 /* Restore %gs and gsbase */ 822 movw TF_GS(%rsp),%si 823 pushfq 824 cli 825 movl $MSR_GSBASE,%ecx 826 /* Save current kernel %gs base into %r12d:%r13d */ 827 rdmsr 828 movl %eax,%r12d 829 movl %edx,%r13d 830 .globl ld_gs 831 ld_gs: 832 movw %si,%gs 833 /* Save user %gs base into %r14d:%r15d */ 834 rdmsr 835 movl %eax,%r14d 836 movl %edx,%r15d 837 /* Restore kernel %gs base */ 838 movl %r12d,%eax 839 movl %r13d,%edx 840 wrmsr 841 popfq 842 /* 843 * Restore user %gs base, either from PCB if used for TLS, or 844 * from the previously saved msr read. 845 */ 846 movl $MSR_KGSBASE,%ecx 847 cmpw $KUG32SEL,%si 848 jne 1f 849 movl PCB_GSBASE(%r8),%eax 850 movl PCB_GSBASE+4(%r8),%edx 851 jmp ld_gsbase 852 1: 853 movl %r14d,%eax 854 movl %r15d,%edx 855 .globl ld_gsbase 856 ld_gsbase: 857 wrmsr /* May trap if non-canonical, but only for TLS. */ 858 .globl ld_es 859 ld_es: 860 movw TF_ES(%rsp),%es 861 .globl ld_ds 862 ld_ds: 863 movw TF_DS(%rsp),%ds 864 ld_regs: 865 movq TF_RDI(%rsp),%rdi 866 movq TF_RSI(%rsp),%rsi 867 movq TF_RDX(%rsp),%rdx 868 movq TF_RCX(%rsp),%rcx 869 movq TF_R8(%rsp),%r8 870 movq TF_R9(%rsp),%r9 871 movq TF_RAX(%rsp),%rax 872 movq TF_RBX(%rsp),%rbx 873 movq TF_RBP(%rsp),%rbp 874 movq TF_R10(%rsp),%r10 875 movq TF_R11(%rsp),%r11 876 movq TF_R12(%rsp),%r12 877 movq TF_R13(%rsp),%r13 878 movq TF_R14(%rsp),%r14 879 movq TF_R15(%rsp),%r15 880 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 881 jz 1f /* keep running with kernel GS.base */ 882 cli 883 swapgs 884 1: 885 addq $TF_RIP,%rsp /* skip over tf_err, tf_trapno */ 886 .globl doreti_iret 887 doreti_iret: 888 iretq 889 890 set_segs: 891 movw $KUDSEL,%ax 892 movw %ax,TF_DS(%rsp) 893 movw %ax,TF_ES(%rsp) 894 movw $KUF32SEL,TF_FS(%rsp) 895 movw $KUG32SEL,TF_GS(%rsp) 896 jmp do_segs 897 898 /* 899 * doreti_iret_fault. Alternative return code for 900 * the case where we get a fault in the doreti_exit code 901 * above. trap() (amd64/amd64/trap.c) catches this specific 902 * case, sends the process a signal and continues in the 903 * corresponding place in the code below. 904 */ 905 ALIGN_TEXT 906 .globl doreti_iret_fault 907 doreti_iret_fault: 908 subq $TF_RIP,%rsp /* space including tf_err, tf_trapno */ 909 testl $PSL_I,TF_RFLAGS(%rsp) 910 jz 1f 911 sti 912 1: 913 movw %fs,TF_FS(%rsp) 914 movw %gs,TF_GS(%rsp) 915 movw %es,TF_ES(%rsp) 916 movw %ds,TF_DS(%rsp) 917 movl $TF_HASSEGS,TF_FLAGS(%rsp) 918 movq %rdi,TF_RDI(%rsp) 919 movq %rsi,TF_RSI(%rsp) 920 movq %rdx,TF_RDX(%rsp) 921 movq %rcx,TF_RCX(%rsp) 922 movq %r8,TF_R8(%rsp) 923 movq %r9,TF_R9(%rsp) 924 movq %rax,TF_RAX(%rsp) 925 movq %rbx,TF_RBX(%rsp) 926 movq %rbp,TF_RBP(%rsp) 927 movq %r10,TF_R10(%rsp) 928 movq %r11,TF_R11(%rsp) 929 movq %r12,TF_R12(%rsp) 930 movq %r13,TF_R13(%rsp) 931 movq %r14,TF_R14(%rsp) 932 movq %r15,TF_R15(%rsp) 933 movl $T_PROTFLT,TF_TRAPNO(%rsp) 934 movq $0,TF_ERR(%rsp) /* XXX should be the error code */ 935 movq $0,TF_ADDR(%rsp) 936 FAKE_MCOUNT(TF_RIP(%rsp)) 937 jmp calltrap 938 939 ALIGN_TEXT 940 .globl ds_load_fault 941 ds_load_fault: 942 movl $T_PROTFLT,TF_TRAPNO(%rsp) 943 testl $PSL_I,TF_RFLAGS(%rsp) 944 jz 1f 945 sti 946 1: 947 movq %rsp,%rdi 948 call trap 949 movw $KUDSEL,TF_DS(%rsp) 950 jmp doreti 951 952 ALIGN_TEXT 953 .globl es_load_fault 954 es_load_fault: 955 movl $T_PROTFLT,TF_TRAPNO(%rsp) 956 testl $PSL_I,TF_RFLAGS(%rsp) 957 jz 1f 958 sti 959 1: 960 movq %rsp,%rdi 961 call trap 962 movw $KUDSEL,TF_ES(%rsp) 963 jmp doreti 964 965 ALIGN_TEXT 966 .globl fs_load_fault 967 fs_load_fault: 968 testl $PSL_I,TF_RFLAGS(%rsp) 969 jz 1f 970 sti 971 1: 972 movl $T_PROTFLT,TF_TRAPNO(%rsp) 973 movq %rsp,%rdi 974 call trap 975 movw $KUF32SEL,TF_FS(%rsp) 976 jmp doreti 977 978 ALIGN_TEXT 979 .globl gs_load_fault 980 gs_load_fault: 981 popfq 982 movl $T_PROTFLT,TF_TRAPNO(%rsp) 983 testl $PSL_I,TF_RFLAGS(%rsp) 984 jz 1f 985 sti 986 1: 987 movq %rsp,%rdi 988 call trap 989 movw $KUG32SEL,TF_GS(%rsp) 990 jmp doreti 991 992 ALIGN_TEXT 993 .globl fsbase_load_fault 994 fsbase_load_fault: 995 movl $T_PROTFLT,TF_TRAPNO(%rsp) 996 testl $PSL_I,TF_RFLAGS(%rsp) 997 jz 1f 998 sti 999 1: 1000 movq %rsp,%rdi 1001 call trap 1002 movq PCPU(CURTHREAD),%r8 1003 movq TD_PCB(%r8),%r8 1004 movq $0,PCB_FSBASE(%r8) 1005 jmp doreti 1006 1007 ALIGN_TEXT 1008 .globl gsbase_load_fault 1009 gsbase_load_fault: 1010 movl $T_PROTFLT,TF_TRAPNO(%rsp) 1011 testl $PSL_I,TF_RFLAGS(%rsp) 1012 jz 1f 1013 sti 1014 1: 1015 movq %rsp,%rdi 1016 call trap 1017 movq PCPU(CURTHREAD),%r8 1018 movq TD_PCB(%r8),%r8 1019 movq $0,PCB_GSBASE(%r8) 1020 jmp doreti 1021 1022 #ifdef HWPMC_HOOKS 1023 ENTRY(end_exceptions) 1024 #endif
Cache object: 20eddcf3e5e9813dc2bd5f4a218391fa
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]