FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/trap.c
1 /*-
2 * SPDX-License-Identifier: BSD-4-Clause
3 *
4 * Copyright (C) 1994, David Greenman
5 * Copyright (c) 1990, 1993
6 * The Regents of the University of California. All rights reserved.
7 *
8 * This code is derived from software contributed to Berkeley by
9 * the University of Utah, and William Jolitz.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. All advertising materials mentioning features or use of this software
20 * must display the following acknowledgement:
21 * This product includes software developed by the University of
22 * California, Berkeley and its contributors.
23 * 4. Neither the name of the University nor the names of its contributors
24 * may be used to endorse or promote products derived from this software
25 * without specific prior written permission.
26 *
27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 * SUCH DAMAGE.
38 *
39 * from: @(#)trap.c 7.4 (Berkeley) 5/13/91
40 */
41
42 #include <sys/cdefs.h>
43 __FBSDID("$FreeBSD$");
44
45 /*
46 * AMD64 Trap and System call handling
47 */
48
49 #include "opt_clock.h"
50 #include "opt_compat.h"
51 #include "opt_cpu.h"
52 #include "opt_hwpmc_hooks.h"
53 #include "opt_isa.h"
54 #include "opt_kdb.h"
55
56 #include <sys/param.h>
57 #include <sys/asan.h>
58 #include <sys/bus.h>
59 #include <sys/systm.h>
60 #include <sys/proc.h>
61 #include <sys/ptrace.h>
62 #include <sys/kdb.h>
63 #include <sys/kernel.h>
64 #include <sys/ktr.h>
65 #include <sys/lock.h>
66 #include <sys/msan.h>
67 #include <sys/mutex.h>
68 #include <sys/resourcevar.h>
69 #include <sys/signalvar.h>
70 #include <sys/syscall.h>
71 #include <sys/sysctl.h>
72 #include <sys/sysent.h>
73 #include <sys/uio.h>
74 #include <sys/vmmeter.h>
75 #ifdef HWPMC_HOOKS
76 #include <sys/pmckern.h>
77 PMC_SOFT_DEFINE( , , page_fault, all);
78 PMC_SOFT_DEFINE( , , page_fault, read);
79 PMC_SOFT_DEFINE( , , page_fault, write);
80 #endif
81
82 #include <vm/vm.h>
83 #include <vm/vm_param.h>
84 #include <vm/pmap.h>
85 #include <vm/vm_kern.h>
86 #include <vm/vm_map.h>
87 #include <vm/vm_page.h>
88 #include <vm/vm_extern.h>
89
90 #include <machine/cpu.h>
91 #include <machine/intr_machdep.h>
92 #include <x86/mca.h>
93 #include <machine/md_var.h>
94 #include <machine/pcb.h>
95 #ifdef SMP
96 #include <machine/smp.h>
97 #endif
98 #include <machine/stack.h>
99 #include <machine/trap.h>
100 #include <machine/tss.h>
101
102 #ifdef KDTRACE_HOOKS
103 #include <sys/dtrace_bsd.h>
104 #endif
105
106 extern inthand_t IDTVEC(bpt), IDTVEC(bpt_pti), IDTVEC(dbg),
107 IDTVEC(fast_syscall), IDTVEC(fast_syscall_pti), IDTVEC(fast_syscall32),
108 IDTVEC(int0x80_syscall_pti), IDTVEC(int0x80_syscall);
109
110 void __noinline trap(struct trapframe *frame);
111 void trap_check(struct trapframe *frame);
112 void dblfault_handler(struct trapframe *frame);
113
114 static int trap_pfault(struct trapframe *, bool, int *, int *);
115 static void trap_fatal(struct trapframe *, vm_offset_t);
116 #ifdef KDTRACE_HOOKS
117 static bool trap_user_dtrace(struct trapframe *,
118 int (**hook)(struct trapframe *));
119 #endif
120
121 static const char UNKNOWN[] = "unknown";
122 static const char *const trap_msg[] = {
123 [0] = UNKNOWN, /* unused */
124 [T_PRIVINFLT] = "privileged instruction fault",
125 [2] = UNKNOWN, /* unused */
126 [T_BPTFLT] = "breakpoint instruction fault",
127 [4] = UNKNOWN, /* unused */
128 [5] = UNKNOWN, /* unused */
129 [T_ARITHTRAP] = "arithmetic trap",
130 [7] = UNKNOWN, /* unused */
131 [8] = UNKNOWN, /* unused */
132 [T_PROTFLT] = "general protection fault",
133 [T_TRCTRAP] = "debug exception",
134 [11] = UNKNOWN, /* unused */
135 [T_PAGEFLT] = "page fault",
136 [13] = UNKNOWN, /* unused */
137 [T_ALIGNFLT] = "alignment fault",
138 [15] = UNKNOWN, /* unused */
139 [16] = UNKNOWN, /* unused */
140 [17] = UNKNOWN, /* unused */
141 [T_DIVIDE] = "integer divide fault",
142 [T_NMI] = "non-maskable interrupt trap",
143 [T_OFLOW] = "overflow trap",
144 [T_BOUND] = "FPU bounds check fault",
145 [T_DNA] = "FPU device not available",
146 [T_DOUBLEFLT] = "double fault",
147 [T_FPOPFLT] = "FPU operand fetch fault",
148 [T_TSSFLT] = "invalid TSS fault",
149 [T_SEGNPFLT] = "segment not present fault",
150 [T_STKFLT] = "stack fault",
151 [T_MCHK] = "machine check trap",
152 [T_XMMFLT] = "SIMD floating-point exception",
153 [T_RESERVED] = "reserved (unknown) fault",
154 [31] = UNKNOWN, /* reserved */
155 [T_DTRACE_RET] = "DTrace pid return trap",
156 };
157
158 static int uprintf_signal;
159 SYSCTL_INT(_machdep, OID_AUTO, uprintf_signal, CTLFLAG_RWTUN,
160 &uprintf_signal, 0,
161 "Print debugging information on trap signal to ctty");
162
163 /*
164 * Control L1D flush on return from NMI.
165 *
166 * Tunable can be set to the following values:
167 * 0 - only enable flush on return from NMI if required by vmm.ko (default)
168 * >1 - always flush on return from NMI.
169 *
170 * Post-boot, the sysctl indicates if flushing is currently enabled.
171 */
172 int nmi_flush_l1d_sw;
173 SYSCTL_INT(_machdep, OID_AUTO, nmi_flush_l1d_sw, CTLFLAG_RWTUN,
174 &nmi_flush_l1d_sw, 0,
175 "Flush L1 Data Cache on NMI exit, software bhyve L1TF mitigation assist");
176
177 /*
178 * Table of handlers for various segment load faults.
179 */
180 static const struct {
181 uintptr_t faddr;
182 uintptr_t fhandler;
183 } sfhandlers[] = {
184 {
185 .faddr = (uintptr_t)ld_ds,
186 .fhandler = (uintptr_t)ds_load_fault,
187 },
188 {
189 .faddr = (uintptr_t)ld_es,
190 .fhandler = (uintptr_t)es_load_fault,
191 },
192 {
193 .faddr = (uintptr_t)ld_fs,
194 .fhandler = (uintptr_t)fs_load_fault,
195 },
196 {
197 .faddr = (uintptr_t)ld_gs,
198 .fhandler = (uintptr_t)gs_load_fault,
199 },
200 {
201 .faddr = (uintptr_t)ld_gsbase,
202 .fhandler = (uintptr_t)gsbase_load_fault
203 },
204 {
205 .faddr = (uintptr_t)ld_fsbase,
206 .fhandler = (uintptr_t)fsbase_load_fault,
207 },
208 };
209
210 /*
211 * Exception, fault, and trap interface to the FreeBSD kernel.
212 * This common code is called from assembly language IDT gate entry
213 * routines that prepare a suitable stack frame, and restore this
214 * frame after the exception has been processed.
215 */
216
217 void
218 trap(struct trapframe *frame)
219 {
220 ksiginfo_t ksi;
221 struct thread *td;
222 struct proc *p;
223 register_t addr, dr6;
224 size_t i;
225 int pf, signo, ucode;
226 u_int type;
227
228 td = curthread;
229 p = td->td_proc;
230 dr6 = 0;
231
232 kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
233 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
234
235 VM_CNT_INC(v_trap);
236 type = frame->tf_trapno;
237
238 #ifdef SMP
239 /* Handler for NMI IPIs used for stopping CPUs. */
240 if (type == T_NMI && ipi_nmi_handler() == 0)
241 return;
242 #endif
243
244 #ifdef KDB
245 if (kdb_active) {
246 kdb_reenter();
247 return;
248 }
249 #endif
250
251 if (type == T_RESERVED) {
252 trap_fatal(frame, 0);
253 return;
254 }
255
256 if (type == T_NMI) {
257 #ifdef HWPMC_HOOKS
258 /*
259 * CPU PMCs interrupt using an NMI. If the PMC module is
260 * active, pass the 'rip' value to the PMC module's interrupt
261 * handler. A non-zero return value from the handler means that
262 * the NMI was consumed by it and we can return immediately.
263 */
264 if (pmc_intr != NULL &&
265 (*pmc_intr)(frame) != 0)
266 return;
267 #endif
268 }
269
270 if ((frame->tf_rflags & PSL_I) == 0) {
271 /*
272 * Buggy application or kernel code has disabled
273 * interrupts and then trapped. Enabling interrupts
274 * now is wrong, but it is better than running with
275 * interrupts disabled until they are accidentally
276 * enabled later.
277 */
278 if (TRAPF_USERMODE(frame)) {
279 uprintf(
280 "pid %ld (%s): trap %d (%s) "
281 "with interrupts disabled\n",
282 (long)curproc->p_pid, curthread->td_name, type,
283 trap_msg[type]);
284 } else {
285 switch (type) {
286 case T_NMI:
287 case T_BPTFLT:
288 case T_TRCTRAP:
289 case T_PROTFLT:
290 case T_SEGNPFLT:
291 case T_STKFLT:
292 break;
293 default:
294 printf(
295 "kernel trap %d with interrupts disabled\n",
296 type);
297
298 /*
299 * We shouldn't enable interrupts while holding a
300 * spin lock.
301 */
302 if (td->td_md.md_spinlock_count == 0)
303 enable_intr();
304 }
305 }
306 }
307
308 if (TRAPF_USERMODE(frame)) {
309 /* user trap */
310
311 td->td_pticks = 0;
312 td->td_frame = frame;
313 addr = frame->tf_rip;
314 if (td->td_cowgen != atomic_load_int(&p->p_cowgen))
315 thread_cow_update(td);
316
317 switch (type) {
318 case T_PRIVINFLT: /* privileged instruction fault */
319 signo = SIGILL;
320 ucode = ILL_PRVOPC;
321 break;
322
323 case T_BPTFLT: /* bpt instruction fault */
324 #ifdef KDTRACE_HOOKS
325 if (trap_user_dtrace(frame, &dtrace_pid_probe_ptr))
326 return;
327 #else
328 enable_intr();
329 #endif
330 signo = SIGTRAP;
331 ucode = TRAP_BRKPT;
332 break;
333
334 case T_TRCTRAP: /* debug exception */
335 enable_intr();
336 signo = SIGTRAP;
337 ucode = TRAP_TRACE;
338 dr6 = rdr6();
339 if ((dr6 & DBREG_DR6_BS) != 0) {
340 PROC_LOCK(td->td_proc);
341 if ((td->td_dbgflags & TDB_STEP) != 0) {
342 td->td_frame->tf_rflags &= ~PSL_T;
343 td->td_dbgflags &= ~TDB_STEP;
344 }
345 PROC_UNLOCK(td->td_proc);
346 }
347 break;
348
349 case T_ARITHTRAP: /* arithmetic trap */
350 ucode = fputrap_x87();
351 if (ucode == -1)
352 return;
353 signo = SIGFPE;
354 break;
355
356 case T_PROTFLT: /* general protection fault */
357 signo = SIGBUS;
358 ucode = BUS_OBJERR;
359 break;
360 case T_STKFLT: /* stack fault */
361 case T_SEGNPFLT: /* segment not present fault */
362 signo = SIGBUS;
363 ucode = BUS_ADRERR;
364 break;
365 case T_TSSFLT: /* invalid TSS fault */
366 signo = SIGBUS;
367 ucode = BUS_OBJERR;
368 break;
369 case T_ALIGNFLT:
370 signo = SIGBUS;
371 ucode = BUS_ADRALN;
372 break;
373 case T_DOUBLEFLT: /* double fault */
374 default:
375 signo = SIGBUS;
376 ucode = BUS_OBJERR;
377 break;
378
379 case T_PAGEFLT: /* page fault */
380 /*
381 * Can emulator handle this trap?
382 */
383 if (*p->p_sysent->sv_trap != NULL &&
384 (*p->p_sysent->sv_trap)(td) == 0)
385 return;
386
387 pf = trap_pfault(frame, true, &signo, &ucode);
388 if (pf == -1)
389 return;
390 if (pf == 0)
391 goto userret;
392 addr = frame->tf_addr;
393 break;
394
395 case T_DIVIDE: /* integer divide fault */
396 ucode = FPE_INTDIV;
397 signo = SIGFPE;
398 break;
399
400 case T_NMI:
401 nmi_handle_intr(type, frame);
402 return;
403
404 case T_OFLOW: /* integer overflow fault */
405 ucode = FPE_INTOVF;
406 signo = SIGFPE;
407 break;
408
409 case T_BOUND: /* bounds check fault */
410 ucode = FPE_FLTSUB;
411 signo = SIGFPE;
412 break;
413
414 case T_DNA:
415 /* transparent fault (due to context switch "late") */
416 KASSERT(PCB_USER_FPU(td->td_pcb),
417 ("kernel FPU ctx has leaked"));
418 fpudna();
419 return;
420
421 case T_FPOPFLT: /* FPU operand fetch fault */
422 ucode = ILL_COPROC;
423 signo = SIGILL;
424 break;
425
426 case T_XMMFLT: /* SIMD floating-point exception */
427 ucode = fputrap_sse();
428 if (ucode == -1)
429 return;
430 signo = SIGFPE;
431 break;
432 #ifdef KDTRACE_HOOKS
433 case T_DTRACE_RET:
434 (void)trap_user_dtrace(frame, &dtrace_return_probe_ptr);
435 return;
436 #endif
437 }
438 } else {
439 /* kernel trap */
440
441 KASSERT(cold || td->td_ucred != NULL,
442 ("kernel trap doesn't have ucred"));
443 switch (type) {
444 case T_PAGEFLT: /* page fault */
445 (void)trap_pfault(frame, false, NULL, NULL);
446 return;
447
448 case T_DNA:
449 if (PCB_USER_FPU(td->td_pcb))
450 panic("Unregistered use of FPU in kernel");
451 fpudna();
452 return;
453
454 case T_ARITHTRAP: /* arithmetic trap */
455 case T_XMMFLT: /* SIMD floating-point exception */
456 case T_FPOPFLT: /* FPU operand fetch fault */
457 /*
458 * For now, supporting kernel handler
459 * registration for FPU traps is overkill.
460 */
461 trap_fatal(frame, 0);
462 return;
463
464 case T_STKFLT: /* stack fault */
465 case T_PROTFLT: /* general protection fault */
466 case T_SEGNPFLT: /* segment not present fault */
467 if (td->td_intr_nesting_level != 0)
468 break;
469
470 /*
471 * Invalid segment selectors and out of bounds
472 * %rip's and %rsp's can be set up in user mode.
473 * This causes a fault in kernel mode when the
474 * kernel tries to return to user mode. We want
475 * to get this fault so that we can fix the
476 * problem here and not have to check all the
477 * selectors and pointers when the user changes
478 * them.
479 *
480 * In case of PTI, the IRETQ faulted while the
481 * kernel used the pti stack, and exception
482 * frame records %rsp value pointing to that
483 * stack. If we return normally to
484 * doreti_iret_fault, the trapframe is
485 * reconstructed on pti stack, and calltrap()
486 * called on it as well. Due to the very
487 * limited pti stack size, kernel does not
488 * survive for too long. Switch to the normal
489 * thread stack for the trap handling.
490 *
491 * Magic '5' is the number of qwords occupied by
492 * the hardware trap frame.
493 */
494 if (frame->tf_rip == (long)doreti_iret) {
495 KASSERT((read_rflags() & PSL_I) == 0,
496 ("interrupts enabled"));
497 frame->tf_rip = (long)doreti_iret_fault;
498 if ((PCPU_GET(curpmap)->pm_ucr3 !=
499 PMAP_NO_CR3) &&
500 (frame->tf_rsp == (uintptr_t)PCPU_GET(
501 pti_rsp0) - 5 * sizeof(register_t))) {
502 frame->tf_rsp = PCPU_GET(rsp0) - 5 *
503 sizeof(register_t);
504 }
505 return;
506 }
507
508 for (i = 0; i < nitems(sfhandlers); i++) {
509 if (frame->tf_rip == sfhandlers[i].faddr) {
510 KASSERT((read_rflags() & PSL_I) == 0,
511 ("interrupts enabled"));
512 frame->tf_rip = sfhandlers[i].fhandler;
513 return;
514 }
515 }
516
517 if (curpcb->pcb_onfault != NULL) {
518 frame->tf_rip = (long)curpcb->pcb_onfault;
519 return;
520 }
521 break;
522
523 case T_TSSFLT:
524 /*
525 * PSL_NT can be set in user mode and isn't cleared
526 * automatically when the kernel is entered. This
527 * causes a TSS fault when the kernel attempts to
528 * `iret' because the TSS link is uninitialized. We
529 * want to get this fault so that we can fix the
530 * problem here and not every time the kernel is
531 * entered.
532 */
533 if (frame->tf_rflags & PSL_NT) {
534 frame->tf_rflags &= ~PSL_NT;
535 return;
536 }
537 break;
538
539 case T_TRCTRAP: /* debug exception */
540 /* Clear any pending debug events. */
541 dr6 = rdr6();
542 load_dr6(0);
543
544 /*
545 * Ignore debug register exceptions due to
546 * accesses in the user's address space, which
547 * can happen under several conditions such as
548 * if a user sets a watchpoint on a buffer and
549 * then passes that buffer to a system call.
550 * We still want to get TRCTRAPS for addresses
551 * in kernel space because that is useful when
552 * debugging the kernel.
553 */
554 if (user_dbreg_trap(dr6))
555 return;
556
557 /*
558 * Malicious user code can configure a debug
559 * register watchpoint to trap on data access
560 * to the top of stack and then execute 'pop
561 * %ss; int 3'. Due to exception deferral for
562 * 'pop %ss', the CPU will not interrupt 'int
563 * 3' to raise the DB# exception for the debug
564 * register but will postpone the DB# until
565 * execution of the first instruction of the
566 * BP# handler (in kernel mode). Normally the
567 * previous check would ignore DB# exceptions
568 * for watchpoints on user addresses raised in
569 * kernel mode. However, some CPU errata
570 * include cases where DB# exceptions do not
571 * properly set bits in %dr6, e.g. Haswell
572 * HSD23 and Skylake-X SKZ24.
573 *
574 * A deferred DB# can also be raised on the
575 * first instructions of system call entry
576 * points or single-step traps via similar use
577 * of 'pop %ss' or 'mov xxx, %ss'.
578 */
579 if (pti) {
580 if (frame->tf_rip ==
581 (uintptr_t)IDTVEC(fast_syscall_pti) ||
582 #ifdef COMPAT_FREEBSD32
583 frame->tf_rip ==
584 (uintptr_t)IDTVEC(int0x80_syscall_pti) ||
585 #endif
586 frame->tf_rip == (uintptr_t)IDTVEC(bpt_pti))
587 return;
588 } else {
589 if (frame->tf_rip ==
590 (uintptr_t)IDTVEC(fast_syscall) ||
591 #ifdef COMPAT_FREEBSD32
592 frame->tf_rip ==
593 (uintptr_t)IDTVEC(int0x80_syscall) ||
594 #endif
595 frame->tf_rip == (uintptr_t)IDTVEC(bpt))
596 return;
597 }
598 if (frame->tf_rip == (uintptr_t)IDTVEC(dbg) ||
599 /* Needed for AMD. */
600 frame->tf_rip == (uintptr_t)IDTVEC(fast_syscall32))
601 return;
602 /*
603 * FALLTHROUGH (TRCTRAP kernel mode, kernel address)
604 */
605 case T_BPTFLT:
606 /*
607 * If KDB is enabled, let it handle the debugger trap.
608 * Otherwise, debugger traps "can't happen".
609 */
610 #ifdef KDB
611 if (kdb_trap(type, dr6, frame))
612 return;
613 #endif
614 break;
615
616 case T_NMI:
617 nmi_handle_intr(type, frame);
618 return;
619 }
620
621 trap_fatal(frame, 0);
622 return;
623 }
624
625 ksiginfo_init_trap(&ksi);
626 ksi.ksi_signo = signo;
627 ksi.ksi_code = ucode;
628 ksi.ksi_trapno = type;
629 ksi.ksi_addr = (void *)addr;
630 if (uprintf_signal) {
631 uprintf("pid %d comm %s: signal %d err %#lx code %d type %d "
632 "addr %#lx rsp %#lx rip %#lx rax %#lx"
633 "<%02x %02x %02x %02x %02x %02x %02x %02x>\n",
634 p->p_pid, p->p_comm, signo, frame->tf_err, ucode, type,
635 addr, frame->tf_rsp, frame->tf_rip, frame->tf_rax,
636 fubyte((void *)(frame->tf_rip + 0)),
637 fubyte((void *)(frame->tf_rip + 1)),
638 fubyte((void *)(frame->tf_rip + 2)),
639 fubyte((void *)(frame->tf_rip + 3)),
640 fubyte((void *)(frame->tf_rip + 4)),
641 fubyte((void *)(frame->tf_rip + 5)),
642 fubyte((void *)(frame->tf_rip + 6)),
643 fubyte((void *)(frame->tf_rip + 7)));
644 }
645 KASSERT((read_rflags() & PSL_I) != 0, ("interrupts disabled"));
646 trapsignal(td, &ksi);
647
648 userret:
649 userret(td, frame);
650 KASSERT(PCB_USER_FPU(td->td_pcb),
651 ("Return from trap with kernel FPU ctx leaked"));
652 }
653
654 /*
655 * Ensure that we ignore any DTrace-induced faults. This function cannot
656 * be instrumented, so it cannot generate such faults itself.
657 */
658 void
659 trap_check(struct trapframe *frame)
660 {
661
662 #ifdef KDTRACE_HOOKS
663 if (dtrace_trap_func != NULL &&
664 (*dtrace_trap_func)(frame, frame->tf_trapno) != 0)
665 return;
666 #endif
667 trap(frame);
668 }
669
670 static bool
671 trap_is_smap(struct trapframe *frame)
672 {
673
674 /*
675 * A page fault on a userspace address is classified as
676 * SMAP-induced if:
677 * - SMAP is supported;
678 * - kernel mode accessed present data page;
679 * - rflags.AC was cleared.
680 * Kernel must never access user space with rflags.AC cleared
681 * if SMAP is enabled.
682 */
683 return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 &&
684 (frame->tf_err & (PGEX_P | PGEX_U | PGEX_I | PGEX_RSV)) ==
685 PGEX_P && (frame->tf_rflags & PSL_AC) == 0);
686 }
687
688 static bool
689 trap_is_pti(struct trapframe *frame)
690 {
691
692 return (PCPU_GET(curpmap)->pm_ucr3 != PMAP_NO_CR3 &&
693 pg_nx != 0 && (frame->tf_err & (PGEX_P | PGEX_W |
694 PGEX_U | PGEX_I)) == (PGEX_P | PGEX_U | PGEX_I) &&
695 (curpcb->pcb_saved_ucr3 & ~CR3_PCID_MASK) ==
696 (PCPU_GET(curpmap)->pm_cr3 & ~CR3_PCID_MASK));
697 }
698
699 /*
700 * Handle all details of a page fault.
701 * Returns:
702 * -1 if this fault was fatal, typically from kernel mode
703 * (cannot happen, but we need to return something).
704 * 0 if this fault was handled by updating either the user or kernel
705 * page table, execution can continue.
706 * 1 if this fault was from usermode and it was not handled, a synchronous
707 * signal should be delivered to the thread. *signo returns the signal
708 * number, *ucode gives si_code.
709 */
710 static int
711 trap_pfault(struct trapframe *frame, bool usermode, int *signo, int *ucode)
712 {
713 struct thread *td;
714 struct proc *p;
715 vm_map_t map;
716 vm_offset_t eva;
717 int rv;
718 vm_prot_t ftype;
719
720 MPASS(!usermode || (signo != NULL && ucode != NULL));
721
722 td = curthread;
723 p = td->td_proc;
724 eva = frame->tf_addr;
725
726 if (__predict_false((td->td_pflags & TDP_NOFAULTING) != 0)) {
727 /*
728 * Due to both processor errata and lazy TLB invalidation when
729 * access restrictions are removed from virtual pages, memory
730 * accesses that are allowed by the physical mapping layer may
731 * nonetheless cause one spurious page fault per virtual page.
732 * When the thread is executing a "no faulting" section that
733 * is bracketed by vm_fault_{disable,enable}_pagefaults(),
734 * every page fault is treated as a spurious page fault,
735 * unless it accesses the same virtual address as the most
736 * recent page fault within the same "no faulting" section.
737 */
738 if (td->td_md.md_spurflt_addr != eva ||
739 (td->td_pflags & TDP_RESETSPUR) != 0) {
740 /*
741 * Do nothing to the TLB. A stale TLB entry is
742 * flushed automatically by a page fault.
743 */
744 td->td_md.md_spurflt_addr = eva;
745 td->td_pflags &= ~TDP_RESETSPUR;
746 return (0);
747 }
748 } else {
749 /*
750 * If we get a page fault while in a critical section, then
751 * it is most likely a fatal kernel page fault. The kernel
752 * is already going to panic trying to get a sleep lock to
753 * do the VM lookup, so just consider it a fatal trap so the
754 * kernel can print out a useful trap message and even get
755 * to the debugger.
756 *
757 * If we get a page fault while holding a non-sleepable
758 * lock, then it is most likely a fatal kernel page fault.
759 * If WITNESS is enabled, then it's going to whine about
760 * bogus LORs with various VM locks, so just skip to the
761 * fatal trap handling directly.
762 */
763 if (td->td_critnest != 0 ||
764 WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
765 "Kernel page fault") != 0) {
766 trap_fatal(frame, eva);
767 return (-1);
768 }
769 }
770 if (eva >= VM_MIN_KERNEL_ADDRESS) {
771 /*
772 * Don't allow user-mode faults in kernel address space.
773 */
774 if (usermode) {
775 *signo = SIGSEGV;
776 *ucode = SEGV_MAPERR;
777 return (1);
778 }
779
780 map = kernel_map;
781 } else {
782 map = &p->p_vmspace->vm_map;
783
784 /*
785 * When accessing a usermode address, kernel must be
786 * ready to accept the page fault, and provide a
787 * handling routine. Since accessing the address
788 * without the handler is a bug, do not try to handle
789 * it normally, and panic immediately.
790 *
791 * If SMAP is enabled, filter SMAP faults also,
792 * because illegal access might occur to the mapped
793 * user address, causing infinite loop.
794 */
795 if (!usermode && (td->td_intr_nesting_level != 0 ||
796 trap_is_smap(frame) || curpcb->pcb_onfault == NULL)) {
797 trap_fatal(frame, eva);
798 return (-1);
799 }
800 }
801
802 /*
803 * If the trap was caused by errant bits in the PTE then panic.
804 */
805 if (frame->tf_err & PGEX_RSV) {
806 trap_fatal(frame, eva);
807 return (-1);
808 }
809
810 /*
811 * User-mode protection key violation (PKU). May happen
812 * either from usermode or from kernel if copyin accessed
813 * key-protected mapping.
814 */
815 if ((frame->tf_err & PGEX_PK) != 0) {
816 if (eva > VM_MAXUSER_ADDRESS) {
817 trap_fatal(frame, eva);
818 return (-1);
819 }
820 if (usermode) {
821 *signo = SIGSEGV;
822 *ucode = SEGV_PKUERR;
823 return (1);
824 }
825 goto after_vmfault;
826 }
827
828 /*
829 * If nx protection of the usermode portion of kernel page
830 * tables caused trap, panic.
831 */
832 if (usermode && trap_is_pti(frame))
833 panic("PTI: pid %d comm %s tf_err %#lx", p->p_pid,
834 p->p_comm, frame->tf_err);
835
836 /*
837 * PGEX_I is defined only if the execute disable bit capability is
838 * supported and enabled.
839 */
840 if (frame->tf_err & PGEX_W)
841 ftype = VM_PROT_WRITE;
842 else if ((frame->tf_err & PGEX_I) && pg_nx != 0)
843 ftype = VM_PROT_EXECUTE;
844 else
845 ftype = VM_PROT_READ;
846
847 /* Fault in the page. */
848 rv = vm_fault_trap(map, eva, ftype, VM_FAULT_NORMAL, signo, ucode);
849 if (rv == KERN_SUCCESS) {
850 #ifdef HWPMC_HOOKS
851 if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) {
852 PMC_SOFT_CALL_TF( , , page_fault, all, frame);
853 if (ftype == VM_PROT_READ)
854 PMC_SOFT_CALL_TF( , , page_fault, read,
855 frame);
856 else
857 PMC_SOFT_CALL_TF( , , page_fault, write,
858 frame);
859 }
860 #endif
861 return (0);
862 }
863
864 if (usermode)
865 return (1);
866 after_vmfault:
867 if (td->td_intr_nesting_level == 0 &&
868 curpcb->pcb_onfault != NULL) {
869 frame->tf_rip = (long)curpcb->pcb_onfault;
870 return (0);
871 }
872 trap_fatal(frame, eva);
873 return (-1);
874 }
875
876 static void
877 trap_fatal(struct trapframe *frame, vm_offset_t eva)
878 {
879 int code, ss;
880 u_int type;
881 struct soft_segment_descriptor softseg;
882 struct user_segment_descriptor *gdt;
883 #ifdef KDB
884 bool handled;
885 #endif
886
887 code = frame->tf_err;
888 type = frame->tf_trapno;
889 gdt = *PCPU_PTR(gdt);
890 sdtossd(&gdt[IDXSEL(frame->tf_cs & 0xffff)], &softseg);
891
892 printf("\n\nFatal trap %d: %s while in %s mode\n", type,
893 type < nitems(trap_msg) ? trap_msg[type] : UNKNOWN,
894 TRAPF_USERMODE(frame) ? "user" : "kernel");
895 #ifdef SMP
896 /* two separate prints in case of a trap on an unmapped page */
897 printf("cpuid = %d; ", PCPU_GET(cpuid));
898 printf("apic id = %02x\n", PCPU_GET(apic_id));
899 #endif
900 if (type == T_PAGEFLT) {
901 printf("fault virtual address = 0x%lx\n", eva);
902 printf("fault code = %s %s %s%s%s, %s\n",
903 code & PGEX_U ? "user" : "supervisor",
904 code & PGEX_W ? "write" : "read",
905 code & PGEX_I ? "instruction" : "data",
906 code & PGEX_PK ? " prot key" : "",
907 code & PGEX_SGX ? " SGX" : "",
908 code & PGEX_RSV ? "reserved bits in PTE" :
909 code & PGEX_P ? "protection violation" : "page not present");
910 }
911 printf("instruction pointer = 0x%lx:0x%lx\n",
912 frame->tf_cs & 0xffff, frame->tf_rip);
913 ss = frame->tf_ss & 0xffff;
914 printf("stack pointer = 0x%x:0x%lx\n", ss, frame->tf_rsp);
915 printf("frame pointer = 0x%x:0x%lx\n", ss, frame->tf_rbp);
916 printf("code segment = base 0x%lx, limit 0x%lx, type 0x%x\n",
917 softseg.ssd_base, softseg.ssd_limit, softseg.ssd_type);
918 printf(" = DPL %d, pres %d, long %d, def32 %d, gran %d\n",
919 softseg.ssd_dpl, softseg.ssd_p, softseg.ssd_long, softseg.ssd_def32,
920 softseg.ssd_gran);
921 printf("processor eflags = ");
922 if (frame->tf_rflags & PSL_T)
923 printf("trace trap, ");
924 if (frame->tf_rflags & PSL_I)
925 printf("interrupt enabled, ");
926 if (frame->tf_rflags & PSL_NT)
927 printf("nested task, ");
928 if (frame->tf_rflags & PSL_RF)
929 printf("resume, ");
930 printf("IOPL = %ld\n", (frame->tf_rflags & PSL_IOPL) >> 12);
931 printf("current process = %d (%s)\n",
932 curproc->p_pid, curthread->td_name);
933
934 printf("rdi: %16lx rsi: %16lx rdx: %16lx\n", frame->tf_rdi,
935 frame->tf_rsi, frame->tf_rdx);
936 printf("rcx: %16lx r8: %16lx r9: %16lx\n", frame->tf_rcx,
937 frame->tf_r8, frame->tf_r9);
938 printf("rax: %16lx rbx: %16lx rbp: %16lx\n", frame->tf_rax,
939 frame->tf_rbx, frame->tf_rbp);
940 printf("r10: %16lx r11: %16lx r12: %16lx\n", frame->tf_r10,
941 frame->tf_r11, frame->tf_r12);
942 printf("r13: %16lx r14: %16lx r15: %16lx\n", frame->tf_r13,
943 frame->tf_r14, frame->tf_r15);
944
945 #ifdef KDB
946 if (debugger_on_trap) {
947 kdb_why = KDB_WHY_TRAP;
948 handled = kdb_trap(type, 0, frame);
949 kdb_why = KDB_WHY_UNSET;
950 if (handled)
951 return;
952 }
953 #endif
954 printf("trap number = %d\n", type);
955 panic("%s", type < nitems(trap_msg) ? trap_msg[type] :
956 "unknown/reserved trap");
957 }
958
959 #ifdef KDTRACE_HOOKS
960 /*
961 * Invoke a userspace DTrace hook. The hook pointer is cleared when no
962 * userspace probes are enabled, so we must synchronize with DTrace to ensure
963 * that a trapping thread is able to call the hook before it is cleared.
964 */
965 static bool
966 trap_user_dtrace(struct trapframe *frame, int (**hookp)(struct trapframe *))
967 {
968 int (*hook)(struct trapframe *);
969
970 hook = atomic_load_ptr(hookp);
971 enable_intr();
972 if (hook != NULL)
973 return ((hook)(frame) == 0);
974 return (false);
975 }
976 #endif
977
978 /*
979 * Double fault handler. Called when a fault occurs while writing
980 * a frame for a trap/exception onto the stack. This usually occurs
981 * when the stack overflows (such is the case with infinite recursion,
982 * for example).
983 */
984 void
985 dblfault_handler(struct trapframe *frame)
986 {
987 kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
988 #ifdef KDTRACE_HOOKS
989 if (dtrace_doubletrap_func != NULL)
990 (*dtrace_doubletrap_func)();
991 #endif
992 printf("\nFatal double fault\n"
993 "rip %#lx rsp %#lx rbp %#lx\n"
994 "rax %#lx rdx %#lx rbx %#lx\n"
995 "rcx %#lx rsi %#lx rdi %#lx\n"
996 "r8 %#lx r9 %#lx r10 %#lx\n"
997 "r11 %#lx r12 %#lx r13 %#lx\n"
998 "r14 %#lx r15 %#lx rflags %#lx\n"
999 "cs %#lx ss %#lx ds %#hx es %#hx fs %#hx gs %#hx\n"
1000 "fsbase %#lx gsbase %#lx kgsbase %#lx\n",
1001 frame->tf_rip, frame->tf_rsp, frame->tf_rbp,
1002 frame->tf_rax, frame->tf_rdx, frame->tf_rbx,
1003 frame->tf_rcx, frame->tf_rdi, frame->tf_rsi,
1004 frame->tf_r8, frame->tf_r9, frame->tf_r10,
1005 frame->tf_r11, frame->tf_r12, frame->tf_r13,
1006 frame->tf_r14, frame->tf_r15, frame->tf_rflags,
1007 frame->tf_cs, frame->tf_ss, frame->tf_ds, frame->tf_es,
1008 frame->tf_fs, frame->tf_gs,
1009 rdmsr(MSR_FSBASE), rdmsr(MSR_GSBASE), rdmsr(MSR_KGSBASE));
1010 #ifdef SMP
1011 /* two separate prints in case of a trap on an unmapped page */
1012 printf("cpuid = %d; ", PCPU_GET(cpuid));
1013 printf("apic id = %02x\n", PCPU_GET(apic_id));
1014 #endif
1015 panic("double fault");
1016 }
1017
1018 static int __noinline
1019 cpu_fetch_syscall_args_fallback(struct thread *td, struct syscall_args *sa)
1020 {
1021 struct proc *p;
1022 struct trapframe *frame;
1023 syscallarg_t *argp;
1024 caddr_t params;
1025 int reg, regcnt, error;
1026
1027 p = td->td_proc;
1028 frame = td->td_frame;
1029 reg = 0;
1030 regcnt = NARGREGS;
1031
1032 if (sa->code == SYS_syscall || sa->code == SYS___syscall) {
1033 sa->code = frame->tf_rdi;
1034 reg++;
1035 regcnt--;
1036 }
1037
1038 if (sa->code >= p->p_sysent->sv_size)
1039 sa->callp = &p->p_sysent->sv_table[0];
1040 else
1041 sa->callp = &p->p_sysent->sv_table[sa->code];
1042
1043 KASSERT(sa->callp->sy_narg <= nitems(sa->args),
1044 ("Too many syscall arguments!"));
1045 argp = &frame->tf_rdi;
1046 argp += reg;
1047 memcpy(sa->args, argp, sizeof(sa->args[0]) * NARGREGS);
1048 if (sa->callp->sy_narg > regcnt) {
1049 params = (caddr_t)frame->tf_rsp + sizeof(register_t);
1050 error = copyin(params, &sa->args[regcnt],
1051 (sa->callp->sy_narg - regcnt) * sizeof(sa->args[0]));
1052 if (__predict_false(error != 0))
1053 return (error);
1054 }
1055
1056 td->td_retval[0] = 0;
1057 td->td_retval[1] = frame->tf_rdx;
1058
1059 return (0);
1060 }
1061
1062 int
1063 cpu_fetch_syscall_args(struct thread *td)
1064 {
1065 struct proc *p;
1066 struct trapframe *frame;
1067 struct syscall_args *sa;
1068
1069 p = td->td_proc;
1070 frame = td->td_frame;
1071 sa = &td->td_sa;
1072
1073 sa->code = frame->tf_rax;
1074 sa->original_code = sa->code;
1075
1076 if (__predict_false(sa->code == SYS_syscall ||
1077 sa->code == SYS___syscall ||
1078 sa->code >= p->p_sysent->sv_size))
1079 return (cpu_fetch_syscall_args_fallback(td, sa));
1080
1081 sa->callp = &p->p_sysent->sv_table[sa->code];
1082 KASSERT(sa->callp->sy_narg <= nitems(sa->args),
1083 ("Too many syscall arguments!"));
1084
1085 if (__predict_false(sa->callp->sy_narg > NARGREGS))
1086 return (cpu_fetch_syscall_args_fallback(td, sa));
1087
1088 memcpy(sa->args, &frame->tf_rdi, sizeof(sa->args[0]) * NARGREGS);
1089
1090 td->td_retval[0] = 0;
1091 td->td_retval[1] = frame->tf_rdx;
1092
1093 return (0);
1094 }
1095
1096 #include "../../kern/subr_syscall.c"
1097
1098 static void (*syscall_ret_l1d_flush)(void);
1099 int syscall_ret_l1d_flush_mode;
1100
1101 static void
1102 flush_l1d_hw(void)
1103 {
1104
1105 wrmsr(MSR_IA32_FLUSH_CMD, IA32_FLUSH_CMD_L1D);
1106 }
1107
1108 static void __noinline
1109 amd64_syscall_ret_flush_l1d_check(int error)
1110 {
1111 void (*p)(void);
1112
1113 if (error != EEXIST && error != EAGAIN && error != EXDEV &&
1114 error != ENOENT && error != ENOTCONN && error != EINPROGRESS) {
1115 p = atomic_load_ptr(&syscall_ret_l1d_flush);
1116 if (p != NULL)
1117 p();
1118 }
1119 }
1120
1121 static void __inline
1122 amd64_syscall_ret_flush_l1d_check_inline(int error)
1123 {
1124
1125 if (__predict_false(error != 0))
1126 amd64_syscall_ret_flush_l1d_check(error);
1127 }
1128
1129 void
1130 amd64_syscall_ret_flush_l1d(int error)
1131 {
1132
1133 amd64_syscall_ret_flush_l1d_check_inline(error);
1134 }
1135
1136 void
1137 amd64_syscall_ret_flush_l1d_recalc(void)
1138 {
1139 bool l1d_hw;
1140
1141 l1d_hw = (cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) != 0;
1142 again:
1143 switch (syscall_ret_l1d_flush_mode) {
1144 case 0:
1145 syscall_ret_l1d_flush = NULL;
1146 break;
1147 case 1:
1148 syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw :
1149 flush_l1d_sw_abi;
1150 break;
1151 case 2:
1152 syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw : NULL;
1153 break;
1154 case 3:
1155 syscall_ret_l1d_flush = flush_l1d_sw_abi;
1156 break;
1157 default:
1158 syscall_ret_l1d_flush_mode = 1;
1159 goto again;
1160 }
1161 }
1162
1163 static int
1164 machdep_syscall_ret_flush_l1d(SYSCTL_HANDLER_ARGS)
1165 {
1166 int error, val;
1167
1168 val = syscall_ret_l1d_flush_mode;
1169 error = sysctl_handle_int(oidp, &val, 0, req);
1170 if (error != 0 || req->newptr == NULL)
1171 return (error);
1172 syscall_ret_l1d_flush_mode = val;
1173 amd64_syscall_ret_flush_l1d_recalc();
1174 return (0);
1175 }
1176 SYSCTL_PROC(_machdep, OID_AUTO, syscall_ret_flush_l1d, CTLTYPE_INT |
1177 CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, NULL, 0,
1178 machdep_syscall_ret_flush_l1d, "I",
1179 "Flush L1D on syscall return with error (0 - off, 1 - on, "
1180 "2 - use hw only, 3 - use sw only)");
1181
1182 /*
1183 * System call handler for native binaries. The trap frame is already
1184 * set up by the assembler trampoline and a pointer to it is saved in
1185 * td_frame.
1186 */
1187 void
1188 amd64_syscall(struct thread *td, int traced)
1189 {
1190 ksiginfo_t ksi;
1191
1192 kmsan_mark(td->td_frame, sizeof(*td->td_frame), KMSAN_STATE_INITED);
1193
1194 #ifdef DIAGNOSTIC
1195 if (!TRAPF_USERMODE(td->td_frame)) {
1196 panic("syscall");
1197 /* NOT REACHED */
1198 }
1199 #endif
1200 syscallenter(td);
1201
1202 /*
1203 * Traced syscall.
1204 */
1205 if (__predict_false(traced)) {
1206 td->td_frame->tf_rflags &= ~PSL_T;
1207 ksiginfo_init_trap(&ksi);
1208 ksi.ksi_signo = SIGTRAP;
1209 ksi.ksi_code = TRAP_TRACE;
1210 ksi.ksi_addr = (void *)td->td_frame->tf_rip;
1211 trapsignal(td, &ksi);
1212 }
1213
1214 KASSERT(PCB_USER_FPU(td->td_pcb),
1215 ("System call %s returning with kernel FPU ctx leaked",
1216 syscallname(td->td_proc, td->td_sa.code)));
1217 KASSERT(td->td_pcb->pcb_save == get_pcb_user_save_td(td),
1218 ("System call %s returning with mangled pcb_save",
1219 syscallname(td->td_proc, td->td_sa.code)));
1220 KASSERT(pmap_not_in_di(),
1221 ("System call %s returning with leaked invl_gen %lu",
1222 syscallname(td->td_proc, td->td_sa.code),
1223 td->td_md.md_invl_gen.gen));
1224
1225 syscallret(td);
1226
1227 /*
1228 * If the user-supplied value of %rip is not a canonical
1229 * address, then some CPUs will trigger a ring 0 #GP during
1230 * the sysret instruction. However, the fault handler would
1231 * execute in ring 0 with the user's %gs and %rsp which would
1232 * not be safe. Instead, use the full return path which
1233 * catches the problem safely.
1234 */
1235 if (__predict_false(td->td_frame->tf_rip >= (la57 ?
1236 VM_MAXUSER_ADDRESS_LA57 : VM_MAXUSER_ADDRESS_LA48)))
1237 set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
1238
1239 amd64_syscall_ret_flush_l1d_check_inline(td->td_errno);
1240 }
Cache object: 95b7d5f253a48e4286f8f633d9cee716
|