The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/trap.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * SPDX-License-Identifier: BSD-4-Clause
    3  *
    4  * Copyright (C) 1994, David Greenman
    5  * Copyright (c) 1990, 1993
    6  *      The Regents of the University of California.  All rights reserved.
    7  *
    8  * This code is derived from software contributed to Berkeley by
    9  * the University of Utah, and William Jolitz.
   10  *
   11  * Redistribution and use in source and binary forms, with or without
   12  * modification, are permitted provided that the following conditions
   13  * are met:
   14  * 1. Redistributions of source code must retain the above copyright
   15  *    notice, this list of conditions and the following disclaimer.
   16  * 2. Redistributions in binary form must reproduce the above copyright
   17  *    notice, this list of conditions and the following disclaimer in the
   18  *    documentation and/or other materials provided with the distribution.
   19  * 3. All advertising materials mentioning features or use of this software
   20  *    must display the following acknowledgement:
   21  *      This product includes software developed by the University of
   22  *      California, Berkeley and its contributors.
   23  * 4. Neither the name of the University nor the names of its contributors
   24  *    may be used to endorse or promote products derived from this software
   25  *    without specific prior written permission.
   26  *
   27  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   28  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   29  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   30  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   31  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   32  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   33  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   34  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   35  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   36  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   37  * SUCH DAMAGE.
   38  *
   39  *      from: @(#)trap.c        7.4 (Berkeley) 5/13/91
   40  */
   41 
   42 #include <sys/cdefs.h>
   43 __FBSDID("$FreeBSD$");
   44 
   45 /*
   46  * AMD64 Trap and System call handling
   47  */
   48 
   49 #include "opt_clock.h"
   50 #include "opt_compat.h"
   51 #include "opt_cpu.h"
   52 #include "opt_hwpmc_hooks.h"
   53 #include "opt_isa.h"
   54 #include "opt_kdb.h"
   55 #include "opt_stack.h"
   56 
   57 #include <sys/param.h>
   58 #include <sys/bus.h>
   59 #include <sys/systm.h>
   60 #include <sys/proc.h>
   61 #include <sys/pioctl.h>
   62 #include <sys/ptrace.h>
   63 #include <sys/kdb.h>
   64 #include <sys/kernel.h>
   65 #include <sys/ktr.h>
   66 #include <sys/lock.h>
   67 #include <sys/mutex.h>
   68 #include <sys/resourcevar.h>
   69 #include <sys/signalvar.h>
   70 #include <sys/syscall.h>
   71 #include <sys/sysctl.h>
   72 #include <sys/sysent.h>
   73 #include <sys/uio.h>
   74 #include <sys/vmmeter.h>
   75 #ifdef HWPMC_HOOKS
   76 #include <sys/pmckern.h>
   77 PMC_SOFT_DEFINE( , , page_fault, all);
   78 PMC_SOFT_DEFINE( , , page_fault, read);
   79 PMC_SOFT_DEFINE( , , page_fault, write);
   80 #endif
   81 
   82 #include <vm/vm.h>
   83 #include <vm/vm_param.h>
   84 #include <vm/pmap.h>
   85 #include <vm/vm_kern.h>
   86 #include <vm/vm_map.h>
   87 #include <vm/vm_page.h>
   88 #include <vm/vm_extern.h>
   89 
   90 #include <machine/cpu.h>
   91 #include <machine/intr_machdep.h>
   92 #include <x86/mca.h>
   93 #include <machine/md_var.h>
   94 #include <machine/pcb.h>
   95 #ifdef SMP
   96 #include <machine/smp.h>
   97 #endif
   98 #include <machine/stack.h>
   99 #include <machine/trap.h>
  100 #include <machine/tss.h>
  101 
  102 #ifdef KDTRACE_HOOKS
  103 #include <sys/dtrace_bsd.h>
  104 #endif
  105 
  106 extern inthand_t IDTVEC(bpt), IDTVEC(bpt_pti), IDTVEC(dbg),
  107     IDTVEC(fast_syscall), IDTVEC(fast_syscall_pti), IDTVEC(fast_syscall32),
  108     IDTVEC(int0x80_syscall_pti), IDTVEC(int0x80_syscall);
  109 
  110 void __noinline trap(struct trapframe *frame);
  111 void trap_check(struct trapframe *frame);
  112 void dblfault_handler(struct trapframe *frame);
  113 
  114 static int trap_pfault(struct trapframe *, bool, int *, int *);
  115 static void trap_fatal(struct trapframe *, vm_offset_t);
  116 #ifdef KDTRACE_HOOKS
  117 static bool trap_user_dtrace(struct trapframe *,
  118     int (**hook)(struct trapframe *));
  119 #endif
  120 
  121 static const char UNKNOWN[] = "unknown";
  122 static const char *const trap_msg[] = {
  123         [0] =                   UNKNOWN,                        /* unused */
  124         [T_PRIVINFLT] =         "privileged instruction fault",
  125         [2] =                   UNKNOWN,                        /* unused */
  126         [T_BPTFLT] =            "breakpoint instruction fault",
  127         [4] =                   UNKNOWN,                        /* unused */
  128         [5] =                   UNKNOWN,                        /* unused */
  129         [T_ARITHTRAP] =         "arithmetic trap",
  130         [7] =                   UNKNOWN,                        /* unused */
  131         [8] =                   UNKNOWN,                        /* unused */
  132         [T_PROTFLT] =           "general protection fault",
  133         [T_TRCTRAP] =           "debug exception",
  134         [11] =                  UNKNOWN,                        /* unused */
  135         [T_PAGEFLT] =           "page fault",
  136         [13] =                  UNKNOWN,                        /* unused */
  137         [T_ALIGNFLT] =          "alignment fault",
  138         [15] =                  UNKNOWN,                        /* unused */
  139         [16] =                  UNKNOWN,                        /* unused */
  140         [17] =                  UNKNOWN,                        /* unused */
  141         [T_DIVIDE] =            "integer divide fault",
  142         [T_NMI] =               "non-maskable interrupt trap",
  143         [T_OFLOW] =             "overflow trap",
  144         [T_BOUND] =             "FPU bounds check fault",
  145         [T_DNA] =               "FPU device not available",
  146         [T_DOUBLEFLT] =         "double fault",
  147         [T_FPOPFLT] =           "FPU operand fetch fault",
  148         [T_TSSFLT] =            "invalid TSS fault",
  149         [T_SEGNPFLT] =          "segment not present fault",
  150         [T_STKFLT] =            "stack fault",
  151         [T_MCHK] =              "machine check trap",
  152         [T_XMMFLT] =            "SIMD floating-point exception",
  153         [T_RESERVED] =          "reserved (unknown) fault",
  154         [31] =                  UNKNOWN,                        /* reserved */
  155         [T_DTRACE_RET] =        "DTrace pid return trap",
  156 };
  157 
  158 static int uprintf_signal;
  159 SYSCTL_INT(_machdep, OID_AUTO, uprintf_signal, CTLFLAG_RWTUN,
  160     &uprintf_signal, 0,
  161     "Print debugging information on trap signal to ctty");
  162 
  163 /*
  164  * Control L1D flush on return from NMI.
  165  *
  166  * Tunable  can be set to the following values:
  167  * 0 - only enable flush on return from NMI if required by vmm.ko (default)
  168  * >1 - always flush on return from NMI.
  169  *
  170  * Post-boot, the sysctl indicates if flushing is currently enabled.
  171  */
  172 int nmi_flush_l1d_sw;
  173 SYSCTL_INT(_machdep, OID_AUTO, nmi_flush_l1d_sw, CTLFLAG_RWTUN,
  174     &nmi_flush_l1d_sw, 0,
  175     "Flush L1 Data Cache on NMI exit, software bhyve L1TF mitigation assist");
  176 
  177 /*
  178  * Exception, fault, and trap interface to the FreeBSD kernel.
  179  * This common code is called from assembly language IDT gate entry
  180  * routines that prepare a suitable stack frame, and restore this
  181  * frame after the exception has been processed.
  182  */
  183 
  184 void
  185 trap(struct trapframe *frame)
  186 {
  187         ksiginfo_t ksi;
  188         struct thread *td;
  189         struct proc *p;
  190         register_t addr, dr6;
  191         int pf, signo, ucode;
  192         u_int type;
  193 
  194         td = curthread;
  195         p = td->td_proc;
  196         dr6 = 0;
  197 
  198         VM_CNT_INC(v_trap);
  199         type = frame->tf_trapno;
  200 
  201 #ifdef SMP
  202         /* Handler for NMI IPIs used for stopping CPUs. */
  203         if (type == T_NMI && ipi_nmi_handler() == 0)
  204                 return;
  205 #endif
  206 
  207 #ifdef KDB
  208         if (kdb_active) {
  209                 kdb_reenter();
  210                 return;
  211         }
  212 #endif
  213 
  214         if (type == T_RESERVED) {
  215                 trap_fatal(frame, 0);
  216                 return;
  217         }
  218 
  219         if (type == T_NMI) {
  220 #ifdef HWPMC_HOOKS
  221                 /*
  222                  * CPU PMCs interrupt using an NMI.  If the PMC module is
  223                  * active, pass the 'rip' value to the PMC module's interrupt
  224                  * handler.  A non-zero return value from the handler means that
  225                  * the NMI was consumed by it and we can return immediately.
  226                  */
  227                 if (pmc_intr != NULL &&
  228                     (*pmc_intr)(frame) != 0)
  229                         return;
  230 #endif
  231 
  232 #ifdef STACK
  233                 if (stack_nmi_handler(frame) != 0)
  234                         return;
  235 #endif
  236         }
  237 
  238         if ((frame->tf_rflags & PSL_I) == 0) {
  239                 /*
  240                  * Buggy application or kernel code has disabled
  241                  * interrupts and then trapped.  Enabling interrupts
  242                  * now is wrong, but it is better than running with
  243                  * interrupts disabled until they are accidentally
  244                  * enabled later.
  245                  */
  246                 if (TRAPF_USERMODE(frame))
  247                         uprintf(
  248                             "pid %ld (%s): trap %d with interrupts disabled\n",
  249                             (long)curproc->p_pid, curthread->td_name, type);
  250                 else if (type != T_NMI && type != T_BPTFLT &&
  251                     type != T_TRCTRAP) {
  252                         /*
  253                          * XXX not quite right, since this may be for a
  254                          * multiple fault in user mode.
  255                          */
  256                         printf("kernel trap %d with interrupts disabled\n",
  257                             type);
  258 
  259                         /*
  260                          * We shouldn't enable interrupts while holding a
  261                          * spin lock.
  262                          */
  263                         if (td->td_md.md_spinlock_count == 0)
  264                                 enable_intr();
  265                 }
  266         }
  267 
  268         if (TRAPF_USERMODE(frame)) {
  269                 /* user trap */
  270 
  271                 td->td_pticks = 0;
  272                 td->td_frame = frame;
  273                 addr = frame->tf_rip;
  274                 if (td->td_cowgen != p->p_cowgen)
  275                         thread_cow_update(td);
  276 
  277                 switch (type) {
  278                 case T_PRIVINFLT:       /* privileged instruction fault */
  279                         signo = SIGILL;
  280                         ucode = ILL_PRVOPC;
  281                         break;
  282 
  283                 case T_BPTFLT:          /* bpt instruction fault */
  284 #ifdef KDTRACE_HOOKS
  285                         if (trap_user_dtrace(frame, &dtrace_pid_probe_ptr))
  286                                 return;
  287 #else
  288                         enable_intr();
  289 #endif
  290                         signo = SIGTRAP;
  291                         ucode = TRAP_BRKPT;
  292                         break;
  293 
  294                 case T_TRCTRAP:         /* debug exception */
  295                         enable_intr();
  296                         signo = SIGTRAP;
  297                         ucode = TRAP_TRACE;
  298                         dr6 = rdr6();
  299                         if ((dr6 & DBREG_DR6_BS) != 0) {
  300                                 PROC_LOCK(td->td_proc);
  301                                 if ((td->td_dbgflags & TDB_STEP) != 0) {
  302                                         td->td_frame->tf_rflags &= ~PSL_T;
  303                                         td->td_dbgflags &= ~TDB_STEP;
  304                                 }
  305                                 PROC_UNLOCK(td->td_proc);
  306                         }
  307                         break;
  308 
  309                 case T_ARITHTRAP:       /* arithmetic trap */
  310                         ucode = fputrap_x87();
  311                         if (ucode == -1)
  312                                 return;
  313                         signo = SIGFPE;
  314                         break;
  315 
  316                 case T_PROTFLT:         /* general protection fault */
  317                         signo = SIGBUS;
  318                         ucode = BUS_OBJERR;
  319                         break;
  320                 case T_STKFLT:          /* stack fault */
  321                 case T_SEGNPFLT:        /* segment not present fault */
  322                         signo = SIGBUS;
  323                         ucode = BUS_ADRERR;
  324                         break;
  325                 case T_TSSFLT:          /* invalid TSS fault */
  326                         signo = SIGBUS;
  327                         ucode = BUS_OBJERR;
  328                         break;
  329                 case T_ALIGNFLT:
  330                         signo = SIGBUS;
  331                         ucode = BUS_ADRALN;
  332                         break;
  333                 case T_DOUBLEFLT:       /* double fault */
  334                 default:
  335                         signo = SIGBUS;
  336                         ucode = BUS_OBJERR;
  337                         break;
  338 
  339                 case T_PAGEFLT:         /* page fault */
  340                         /*
  341                          * Can emulator handle this trap?
  342                          */
  343                         if (*p->p_sysent->sv_trap != NULL &&
  344                             (*p->p_sysent->sv_trap)(td) == 0)
  345                                 return;
  346 
  347                         pf = trap_pfault(frame, true, &signo, &ucode);
  348                         if (pf == -1)
  349                                 return;
  350                         if (pf == 0)
  351                                 goto userret;
  352                         addr = frame->tf_addr;
  353                         break;
  354 
  355                 case T_DIVIDE:          /* integer divide fault */
  356                         ucode = FPE_INTDIV;
  357                         signo = SIGFPE;
  358                         break;
  359 
  360                 case T_NMI:
  361                         nmi_handle_intr(type, frame);
  362                         return;
  363 
  364                 case T_OFLOW:           /* integer overflow fault */
  365                         ucode = FPE_INTOVF;
  366                         signo = SIGFPE;
  367                         break;
  368 
  369                 case T_BOUND:           /* bounds check fault */
  370                         ucode = FPE_FLTSUB;
  371                         signo = SIGFPE;
  372                         break;
  373 
  374                 case T_DNA:
  375                         /* transparent fault (due to context switch "late") */
  376                         KASSERT(PCB_USER_FPU(td->td_pcb),
  377                             ("kernel FPU ctx has leaked"));
  378                         fpudna();
  379                         return;
  380 
  381                 case T_FPOPFLT:         /* FPU operand fetch fault */
  382                         ucode = ILL_COPROC;
  383                         signo = SIGILL;
  384                         break;
  385 
  386                 case T_XMMFLT:          /* SIMD floating-point exception */
  387                         ucode = fputrap_sse();
  388                         if (ucode == -1)
  389                                 return;
  390                         signo = SIGFPE;
  391                         break;
  392 #ifdef KDTRACE_HOOKS
  393                 case T_DTRACE_RET:
  394                         (void)trap_user_dtrace(frame, &dtrace_return_probe_ptr);
  395                         return;
  396 #endif
  397                 }
  398         } else {
  399                 /* kernel trap */
  400 
  401                 KASSERT(cold || td->td_ucred != NULL,
  402                     ("kernel trap doesn't have ucred"));
  403                 switch (type) {
  404                 case T_PAGEFLT:                 /* page fault */
  405                         (void)trap_pfault(frame, false, NULL, NULL);
  406                         return;
  407 
  408                 case T_DNA:
  409                         if (PCB_USER_FPU(td->td_pcb))
  410                                 panic("Unregistered use of FPU in kernel");
  411                         fpudna();
  412                         return;
  413 
  414                 case T_ARITHTRAP:       /* arithmetic trap */
  415                 case T_XMMFLT:          /* SIMD floating-point exception */
  416                 case T_FPOPFLT:         /* FPU operand fetch fault */
  417                         /*
  418                          * For now, supporting kernel handler
  419                          * registration for FPU traps is overkill.
  420                          */
  421                         trap_fatal(frame, 0);
  422                         return;
  423 
  424                 case T_STKFLT:          /* stack fault */
  425                 case T_PROTFLT:         /* general protection fault */
  426                 case T_SEGNPFLT:        /* segment not present fault */
  427                         if (td->td_intr_nesting_level != 0)
  428                                 break;
  429 
  430                         /*
  431                          * Invalid segment selectors and out of bounds
  432                          * %rip's and %rsp's can be set up in user mode.
  433                          * This causes a fault in kernel mode when the
  434                          * kernel tries to return to user mode.  We want
  435                          * to get this fault so that we can fix the
  436                          * problem here and not have to check all the
  437                          * selectors and pointers when the user changes
  438                          * them.
  439                          *
  440                          * In case of PTI, the IRETQ faulted while the
  441                          * kernel used the pti stack, and exception
  442                          * frame records %rsp value pointing to that
  443                          * stack.  If we return normally to
  444                          * doreti_iret_fault, the trapframe is
  445                          * reconstructed on pti stack, and calltrap()
  446                          * called on it as well.  Due to the very
  447                          * limited pti stack size, kernel does not
  448                          * survive for too long.  Switch to the normal
  449                          * thread stack for the trap handling.
  450                          *
  451                          * Magic '5' is the number of qwords occupied by
  452                          * the hardware trap frame.
  453                          */
  454                         if (frame->tf_rip == (long)doreti_iret) {
  455                                 frame->tf_rip = (long)doreti_iret_fault;
  456                                 if ((PCPU_GET(curpmap)->pm_ucr3 !=
  457                                     PMAP_NO_CR3) &&
  458                                     (frame->tf_rsp == (uintptr_t)PCPU_GET(
  459                                     pti_rsp0) - 5 * sizeof(register_t))) {
  460                                         frame->tf_rsp = PCPU_GET(rsp0) - 5 *
  461                                             sizeof(register_t);
  462                                 }
  463                                 return;
  464                         }
  465                         if (frame->tf_rip == (long)ld_ds) {
  466                                 frame->tf_rip = (long)ds_load_fault;
  467                                 return;
  468                         }
  469                         if (frame->tf_rip == (long)ld_es) {
  470                                 frame->tf_rip = (long)es_load_fault;
  471                                 return;
  472                         }
  473                         if (frame->tf_rip == (long)ld_fs) {
  474                                 frame->tf_rip = (long)fs_load_fault;
  475                                 return;
  476                         }
  477                         if (frame->tf_rip == (long)ld_gs) {
  478                                 frame->tf_rip = (long)gs_load_fault;
  479                                 return;
  480                         }
  481                         if (frame->tf_rip == (long)ld_gsbase) {
  482                                 frame->tf_rip = (long)gsbase_load_fault;
  483                                 return;
  484                         }
  485                         if (frame->tf_rip == (long)ld_fsbase) {
  486                                 frame->tf_rip = (long)fsbase_load_fault;
  487                                 return;
  488                         }
  489                         if (curpcb->pcb_onfault != NULL) {
  490                                 frame->tf_rip = (long)curpcb->pcb_onfault;
  491                                 return;
  492                         }
  493                         break;
  494 
  495                 case T_TSSFLT:
  496                         /*
  497                          * PSL_NT can be set in user mode and isn't cleared
  498                          * automatically when the kernel is entered.  This
  499                          * causes a TSS fault when the kernel attempts to
  500                          * `iret' because the TSS link is uninitialized.  We
  501                          * want to get this fault so that we can fix the
  502                          * problem here and not every time the kernel is
  503                          * entered.
  504                          */
  505                         if (frame->tf_rflags & PSL_NT) {
  506                                 frame->tf_rflags &= ~PSL_NT;
  507                                 return;
  508                         }
  509                         break;
  510 
  511                 case T_TRCTRAP:  /* debug exception */
  512                         /* Clear any pending debug events. */
  513                         dr6 = rdr6();
  514                         load_dr6(0);
  515 
  516                         /*
  517                          * Ignore debug register exceptions due to
  518                          * accesses in the user's address space, which
  519                          * can happen under several conditions such as
  520                          * if a user sets a watchpoint on a buffer and
  521                          * then passes that buffer to a system call.
  522                          * We still want to get TRCTRAPS for addresses
  523                          * in kernel space because that is useful when
  524                          * debugging the kernel.
  525                          */
  526                         if (user_dbreg_trap(dr6))
  527                                 return;
  528 
  529                         /*
  530                          * Malicious user code can configure a debug
  531                          * register watchpoint to trap on data access
  532                          * to the top of stack and then execute 'pop
  533                          * %ss; int 3'.  Due to exception deferral for
  534                          * 'pop %ss', the CPU will not interrupt 'int
  535                          * 3' to raise the DB# exception for the debug
  536                          * register but will postpone the DB# until
  537                          * execution of the first instruction of the
  538                          * BP# handler (in kernel mode).  Normally the
  539                          * previous check would ignore DB# exceptions
  540                          * for watchpoints on user addresses raised in
  541                          * kernel mode.  However, some CPU errata
  542                          * include cases where DB# exceptions do not
  543                          * properly set bits in %dr6, e.g. Haswell
  544                          * HSD23 and Skylake-X SKZ24.
  545                          *
  546                          * A deferred DB# can also be raised on the
  547                          * first instructions of system call entry
  548                          * points or single-step traps via similar use
  549                          * of 'pop %ss' or 'mov xxx, %ss'.
  550                          */
  551                         if (pti) {
  552                                 if (frame->tf_rip ==
  553                                     (uintptr_t)IDTVEC(fast_syscall_pti) ||
  554 #ifdef COMPAT_FREEBSD32
  555                                     frame->tf_rip ==
  556                                     (uintptr_t)IDTVEC(int0x80_syscall_pti) ||
  557 #endif
  558                                     frame->tf_rip == (uintptr_t)IDTVEC(bpt_pti))
  559                                         return;
  560                         } else {
  561                                 if (frame->tf_rip ==
  562                                     (uintptr_t)IDTVEC(fast_syscall) ||
  563 #ifdef COMPAT_FREEBSD32
  564                                     frame->tf_rip ==
  565                                     (uintptr_t)IDTVEC(int0x80_syscall) ||
  566 #endif
  567                                     frame->tf_rip == (uintptr_t)IDTVEC(bpt))
  568                                         return;
  569                         }
  570                         if (frame->tf_rip == (uintptr_t)IDTVEC(dbg) ||
  571                             /* Needed for AMD. */
  572                             frame->tf_rip == (uintptr_t)IDTVEC(fast_syscall32))
  573                                 return;
  574                         /*
  575                          * FALLTHROUGH (TRCTRAP kernel mode, kernel address)
  576                          */
  577                 case T_BPTFLT:
  578                         /*
  579                          * If KDB is enabled, let it handle the debugger trap.
  580                          * Otherwise, debugger traps "can't happen".
  581                          */
  582 #ifdef KDB
  583                         if (kdb_trap(type, dr6, frame))
  584                                 return;
  585 #endif
  586                         break;
  587 
  588                 case T_NMI:
  589                         nmi_handle_intr(type, frame);
  590                         return;
  591                 }
  592 
  593                 trap_fatal(frame, 0);
  594                 return;
  595         }
  596 
  597         /* Translate fault for emulators (e.g. Linux) */
  598         if (*p->p_sysent->sv_transtrap != NULL)
  599                 signo = (*p->p_sysent->sv_transtrap)(signo, type);
  600 
  601         ksiginfo_init_trap(&ksi);
  602         ksi.ksi_signo = signo;
  603         ksi.ksi_code = ucode;
  604         ksi.ksi_trapno = type;
  605         ksi.ksi_addr = (void *)addr;
  606         if (uprintf_signal) {
  607                 uprintf("pid %d comm %s: signal %d err %lx code %d type %d "
  608                     "addr 0x%lx rsp 0x%lx rip 0x%lx "
  609                     "<%02x %02x %02x %02x %02x %02x %02x %02x>\n",
  610                     p->p_pid, p->p_comm, signo, frame->tf_err, ucode, type,
  611                     addr, frame->tf_rsp, frame->tf_rip,
  612                     fubyte((void *)(frame->tf_rip + 0)),
  613                     fubyte((void *)(frame->tf_rip + 1)),
  614                     fubyte((void *)(frame->tf_rip + 2)),
  615                     fubyte((void *)(frame->tf_rip + 3)),
  616                     fubyte((void *)(frame->tf_rip + 4)),
  617                     fubyte((void *)(frame->tf_rip + 5)),
  618                     fubyte((void *)(frame->tf_rip + 6)),
  619                     fubyte((void *)(frame->tf_rip + 7)));
  620         }
  621         KASSERT((read_rflags() & PSL_I) != 0, ("interrupts disabled"));
  622         trapsignal(td, &ksi);
  623 
  624 userret:
  625         userret(td, frame);
  626         KASSERT(PCB_USER_FPU(td->td_pcb),
  627             ("Return from trap with kernel FPU ctx leaked"));
  628 }
  629 
  630 /*
  631  * Ensure that we ignore any DTrace-induced faults. This function cannot
  632  * be instrumented, so it cannot generate such faults itself.
  633  */
  634 void
  635 trap_check(struct trapframe *frame)
  636 {
  637 
  638 #ifdef KDTRACE_HOOKS
  639         if (dtrace_trap_func != NULL &&
  640             (*dtrace_trap_func)(frame, frame->tf_trapno) != 0)
  641                 return;
  642 #endif
  643         trap(frame);
  644 }
  645 
  646 static bool
  647 trap_is_smap(struct trapframe *frame)
  648 {
  649 
  650         /*
  651          * A page fault on a userspace address is classified as
  652          * SMAP-induced if:
  653          * - SMAP is supported;
  654          * - kernel mode accessed present data page;
  655          * - rflags.AC was cleared.
  656          * Kernel must never access user space with rflags.AC cleared
  657          * if SMAP is enabled.
  658          */
  659         return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 &&
  660             (frame->tf_err & (PGEX_P | PGEX_U | PGEX_I | PGEX_RSV)) ==
  661             PGEX_P && (frame->tf_rflags & PSL_AC) == 0);
  662 }
  663 
  664 static bool
  665 trap_is_pti(struct trapframe *frame)
  666 {
  667 
  668         return (PCPU_GET(curpmap)->pm_ucr3 != PMAP_NO_CR3 &&
  669             pg_nx != 0 && (frame->tf_err & (PGEX_P | PGEX_W |
  670             PGEX_U | PGEX_I)) == (PGEX_P | PGEX_U | PGEX_I) &&
  671             (curpcb->pcb_saved_ucr3 & ~CR3_PCID_MASK) ==
  672             (PCPU_GET(curpmap)->pm_cr3 & ~CR3_PCID_MASK));
  673 }
  674 
  675 /*
  676  * Handle all details of a page fault.
  677  * Returns:
  678  * -1 if this fault was fatal, typically from kernel mode
  679  *    (cannot happen, but we need to return something).
  680  * 0  if this fault was handled by updating either the user or kernel
  681  *    page table, execution can continue.
  682  * 1  if this fault was from usermode and it was not handled, a synchronous
  683  *    signal should be delivered to the thread.  *signo returns the signal
  684  *    number, *ucode gives si_code.
  685  */
  686 static int
  687 trap_pfault(struct trapframe *frame, bool usermode, int *signo, int *ucode)
  688 {
  689         struct thread *td;
  690         struct proc *p;
  691         vm_map_t map;
  692         vm_offset_t eva;
  693         int rv;
  694         vm_prot_t ftype;
  695 
  696         MPASS(!usermode || (signo != NULL && ucode != NULL));
  697 
  698         td = curthread;
  699         p = td->td_proc;
  700         eva = frame->tf_addr;
  701 
  702         if (__predict_false((td->td_pflags & TDP_NOFAULTING) != 0)) {
  703                 /*
  704                  * Due to both processor errata and lazy TLB invalidation when
  705                  * access restrictions are removed from virtual pages, memory
  706                  * accesses that are allowed by the physical mapping layer may
  707                  * nonetheless cause one spurious page fault per virtual page. 
  708                  * When the thread is executing a "no faulting" section that
  709                  * is bracketed by vm_fault_{disable,enable}_pagefaults(),
  710                  * every page fault is treated as a spurious page fault,
  711                  * unless it accesses the same virtual address as the most
  712                  * recent page fault within the same "no faulting" section.
  713                  */
  714                 if (td->td_md.md_spurflt_addr != eva ||
  715                     (td->td_pflags & TDP_RESETSPUR) != 0) {
  716                         /*
  717                          * Do nothing to the TLB.  A stale TLB entry is
  718                          * flushed automatically by a page fault.
  719                          */
  720                         td->td_md.md_spurflt_addr = eva;
  721                         td->td_pflags &= ~TDP_RESETSPUR;
  722                         return (0);
  723                 }
  724         } else {
  725                 /*
  726                  * If we get a page fault while in a critical section, then
  727                  * it is most likely a fatal kernel page fault.  The kernel
  728                  * is already going to panic trying to get a sleep lock to
  729                  * do the VM lookup, so just consider it a fatal trap so the
  730                  * kernel can print out a useful trap message and even get
  731                  * to the debugger.
  732                  *
  733                  * If we get a page fault while holding a non-sleepable
  734                  * lock, then it is most likely a fatal kernel page fault.
  735                  * If WITNESS is enabled, then it's going to whine about
  736                  * bogus LORs with various VM locks, so just skip to the
  737                  * fatal trap handling directly.
  738                  */
  739                 if (td->td_critnest != 0 ||
  740                     WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
  741                     "Kernel page fault") != 0) {
  742                         trap_fatal(frame, eva);
  743                         return (-1);
  744                 }
  745         }
  746         if (eva >= VM_MIN_KERNEL_ADDRESS) {
  747                 /*
  748                  * Don't allow user-mode faults in kernel address space.
  749                  */
  750                 if (usermode) {
  751                         *signo = SIGSEGV;
  752                         *ucode = SEGV_MAPERR;
  753                         return (1);
  754                 }
  755 
  756                 map = kernel_map;
  757         } else {
  758                 map = &p->p_vmspace->vm_map;
  759 
  760                 /*
  761                  * When accessing a usermode address, kernel must be
  762                  * ready to accept the page fault, and provide a
  763                  * handling routine.  Since accessing the address
  764                  * without the handler is a bug, do not try to handle
  765                  * it normally, and panic immediately.
  766                  *
  767                  * If SMAP is enabled, filter SMAP faults also,
  768                  * because illegal access might occur to the mapped
  769                  * user address, causing infinite loop.
  770                  */
  771                 if (!usermode && (td->td_intr_nesting_level != 0 ||
  772                     trap_is_smap(frame) || curpcb->pcb_onfault == NULL)) {
  773                         trap_fatal(frame, eva);
  774                         return (-1);
  775                 }
  776         }
  777 
  778         /*
  779          * If the trap was caused by errant bits in the PTE then panic.
  780          */
  781         if (frame->tf_err & PGEX_RSV) {
  782                 trap_fatal(frame, eva);
  783                 return (-1);
  784         }
  785 
  786         /*
  787          * User-mode protection key violation (PKU).  May happen
  788          * either from usermode or from kernel if copyin accessed
  789          * key-protected mapping.
  790          */
  791         if ((frame->tf_err & PGEX_PK) != 0) {
  792                 if (eva > VM_MAXUSER_ADDRESS) {
  793                         trap_fatal(frame, eva);
  794                         return (-1);
  795                 }
  796                 if (usermode) {
  797                         *signo = SIGSEGV;
  798                         *ucode = SEGV_PKUERR;
  799                         return (1);
  800                 }
  801                 goto after_vmfault;
  802         }
  803 
  804         /*
  805          * If nx protection of the usermode portion of kernel page
  806          * tables caused trap, panic.
  807          */
  808         if (usermode && trap_is_pti(frame))
  809                 panic("PTI: pid %d comm %s tf_err %#lx", p->p_pid,
  810                     p->p_comm, frame->tf_err);
  811 
  812         /*
  813          * PGEX_I is defined only if the execute disable bit capability is
  814          * supported and enabled.
  815          */
  816         if (frame->tf_err & PGEX_W)
  817                 ftype = VM_PROT_WRITE;
  818         else if ((frame->tf_err & PGEX_I) && pg_nx != 0)
  819                 ftype = VM_PROT_EXECUTE;
  820         else
  821                 ftype = VM_PROT_READ;
  822 
  823         /* Fault in the page. */
  824         rv = vm_fault_trap(map, eva, ftype, VM_FAULT_NORMAL, signo, ucode);
  825         if (rv == KERN_SUCCESS) {
  826 #ifdef HWPMC_HOOKS
  827                 if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) {
  828                         PMC_SOFT_CALL_TF( , , page_fault, all, frame);
  829                         if (ftype == VM_PROT_READ)
  830                                 PMC_SOFT_CALL_TF( , , page_fault, read,
  831                                     frame);
  832                         else
  833                                 PMC_SOFT_CALL_TF( , , page_fault, write,
  834                                     frame);
  835                 }
  836 #endif
  837                 return (0);
  838         }
  839 
  840         if (usermode)
  841                 return (1);
  842 after_vmfault:
  843         if (td->td_intr_nesting_level == 0 &&
  844             curpcb->pcb_onfault != NULL) {
  845                 frame->tf_rip = (long)curpcb->pcb_onfault;
  846                 return (0);
  847         }
  848         trap_fatal(frame, eva);
  849         return (-1);
  850 }
  851 
  852 static void
  853 trap_fatal(frame, eva)
  854         struct trapframe *frame;
  855         vm_offset_t eva;
  856 {
  857         int code, ss;
  858         u_int type;
  859         struct soft_segment_descriptor softseg;
  860 #ifdef KDB
  861         bool handled;
  862 #endif
  863 
  864         code = frame->tf_err;
  865         type = frame->tf_trapno;
  866         sdtossd(&gdt[NGDT * PCPU_GET(cpuid) + IDXSEL(frame->tf_cs & 0xffff)],
  867             &softseg);
  868 
  869         printf("\n\nFatal trap %d: %s while in %s mode\n", type,
  870             type < nitems(trap_msg) ? trap_msg[type] : UNKNOWN,
  871             TRAPF_USERMODE(frame) ? "user" : "kernel");
  872 #ifdef SMP
  873         /* two separate prints in case of a trap on an unmapped page */
  874         printf("cpuid = %d; ", PCPU_GET(cpuid));
  875         printf("apic id = %02x\n", PCPU_GET(apic_id));
  876 #endif
  877         if (type == T_PAGEFLT) {
  878                 printf("fault virtual address   = 0x%lx\n", eva);
  879                 printf("fault code              = %s %s %s%s%s, %s\n",
  880                         code & PGEX_U ? "user" : "supervisor",
  881                         code & PGEX_W ? "write" : "read",
  882                         code & PGEX_I ? "instruction" : "data",
  883                         code & PGEX_PK ? " prot key" : "",
  884                         code & PGEX_SGX ? " SGX" : "",
  885                         code & PGEX_RSV ? "reserved bits in PTE" :
  886                         code & PGEX_P ? "protection violation" : "page not present");
  887         }
  888         printf("instruction pointer     = 0x%lx:0x%lx\n",
  889                frame->tf_cs & 0xffff, frame->tf_rip);
  890         ss = frame->tf_ss & 0xffff;
  891         printf("stack pointer           = 0x%x:0x%lx\n", ss, frame->tf_rsp);
  892         printf("frame pointer           = 0x%x:0x%lx\n", ss, frame->tf_rbp);
  893         printf("code segment            = base 0x%lx, limit 0x%lx, type 0x%x\n",
  894                softseg.ssd_base, softseg.ssd_limit, softseg.ssd_type);
  895         printf("                        = DPL %d, pres %d, long %d, def32 %d, gran %d\n",
  896                softseg.ssd_dpl, softseg.ssd_p, softseg.ssd_long, softseg.ssd_def32,
  897                softseg.ssd_gran);
  898         printf("processor eflags        = ");
  899         if (frame->tf_rflags & PSL_T)
  900                 printf("trace trap, ");
  901         if (frame->tf_rflags & PSL_I)
  902                 printf("interrupt enabled, ");
  903         if (frame->tf_rflags & PSL_NT)
  904                 printf("nested task, ");
  905         if (frame->tf_rflags & PSL_RF)
  906                 printf("resume, ");
  907         printf("IOPL = %ld\n", (frame->tf_rflags & PSL_IOPL) >> 12);
  908         printf("current process         = %d (%s)\n",
  909             curproc->p_pid, curthread->td_name);
  910 
  911 #ifdef KDB
  912         if (debugger_on_trap) {
  913                 kdb_why = KDB_WHY_TRAP;
  914                 handled = kdb_trap(type, 0, frame);
  915                 kdb_why = KDB_WHY_UNSET;
  916                 if (handled)
  917                         return;
  918         }
  919 #endif
  920         printf("trap number             = %d\n", type);
  921         panic("%s", type < nitems(trap_msg) ? trap_msg[type] :
  922             "unknown/reserved trap");
  923 }
  924 
  925 #ifdef KDTRACE_HOOKS
  926 /*
  927  * Invoke a userspace DTrace hook.  The hook pointer is cleared when no
  928  * userspace probes are enabled, so we must synchronize with DTrace to ensure
  929  * that a trapping thread is able to call the hook before it is cleared.
  930  */
  931 static bool
  932 trap_user_dtrace(struct trapframe *frame, int (**hookp)(struct trapframe *))
  933 {
  934         int (*hook)(struct trapframe *);
  935 
  936         hook = (int (*)(struct trapframe *))atomic_load_ptr(hookp);
  937         enable_intr();
  938         if (hook != NULL)
  939                 return ((hook)(frame) == 0);
  940         return (false);
  941 }
  942 #endif
  943 
  944 /*
  945  * Double fault handler. Called when a fault occurs while writing
  946  * a frame for a trap/exception onto the stack. This usually occurs
  947  * when the stack overflows (such is the case with infinite recursion,
  948  * for example).
  949  */
  950 void
  951 dblfault_handler(struct trapframe *frame)
  952 {
  953 #ifdef KDTRACE_HOOKS
  954         if (dtrace_doubletrap_func != NULL)
  955                 (*dtrace_doubletrap_func)();
  956 #endif
  957         printf("\nFatal double fault\n"
  958             "rip %#lx rsp %#lx rbp %#lx\n"
  959             "rax %#lx rdx %#lx rbx %#lx\n"
  960             "rcx %#lx rsi %#lx rdi %#lx\n"
  961             "r8 %#lx r9 %#lx r10 %#lx\n"
  962             "r11 %#lx r12 %#lx r13 %#lx\n"
  963             "r14 %#lx r15 %#lx rflags %#lx\n"
  964             "cs %#lx ss %#lx ds %#hx es %#hx fs %#hx gs %#hx\n"
  965             "fsbase %#lx gsbase %#lx kgsbase %#lx\n",
  966             frame->tf_rip, frame->tf_rsp, frame->tf_rbp,
  967             frame->tf_rax, frame->tf_rdx, frame->tf_rbx,
  968             frame->tf_rcx, frame->tf_rdi, frame->tf_rsi,
  969             frame->tf_r8, frame->tf_r9, frame->tf_r10,
  970             frame->tf_r11, frame->tf_r12, frame->tf_r13,
  971             frame->tf_r14, frame->tf_r15, frame->tf_rflags,
  972             frame->tf_cs, frame->tf_ss, frame->tf_ds, frame->tf_es,
  973             frame->tf_fs, frame->tf_gs,
  974             rdmsr(MSR_FSBASE), rdmsr(MSR_GSBASE), rdmsr(MSR_KGSBASE));
  975 #ifdef SMP
  976         /* two separate prints in case of a trap on an unmapped page */
  977         printf("cpuid = %d; ", PCPU_GET(cpuid));
  978         printf("apic id = %02x\n", PCPU_GET(apic_id));
  979 #endif
  980         panic("double fault");
  981 }
  982 
  983 static int __noinline
  984 cpu_fetch_syscall_args_fallback(struct thread *td, struct syscall_args *sa)
  985 {
  986         struct proc *p;
  987         struct trapframe *frame;
  988         register_t *argp;
  989         caddr_t params;
  990         int reg, regcnt, error;
  991 
  992         p = td->td_proc;
  993         frame = td->td_frame;
  994         reg = 0;
  995         regcnt = NARGREGS;
  996 
  997         sa->code = frame->tf_rax;
  998 
  999         if (sa->code == SYS_syscall || sa->code == SYS___syscall) {
 1000                 sa->code = frame->tf_rdi;
 1001                 reg++;
 1002                 regcnt--;
 1003         }
 1004         if (p->p_sysent->sv_mask)
 1005                 sa->code &= p->p_sysent->sv_mask;
 1006 
 1007         if (sa->code >= p->p_sysent->sv_size)
 1008                 sa->callp = &p->p_sysent->sv_table[0];
 1009         else
 1010                 sa->callp = &p->p_sysent->sv_table[sa->code];
 1011 
 1012         sa->narg = sa->callp->sy_narg;
 1013         KASSERT(sa->narg <= nitems(sa->args), ("Too many syscall arguments!"));
 1014         argp = &frame->tf_rdi;
 1015         argp += reg;
 1016         memcpy(sa->args, argp, sizeof(sa->args[0]) * NARGREGS);
 1017         if (sa->narg > regcnt) {
 1018                 params = (caddr_t)frame->tf_rsp + sizeof(register_t);
 1019                 error = copyin(params, &sa->args[regcnt],
 1020                     (sa->narg - regcnt) * sizeof(sa->args[0]));
 1021                 if (__predict_false(error != 0))
 1022                         return (error);
 1023         }
 1024 
 1025         td->td_retval[0] = 0;
 1026         td->td_retval[1] = frame->tf_rdx;
 1027 
 1028         return (0);
 1029 }
 1030 
 1031 int
 1032 cpu_fetch_syscall_args(struct thread *td)
 1033 {
 1034         struct proc *p;
 1035         struct trapframe *frame;
 1036         struct syscall_args *sa;
 1037 
 1038         p = td->td_proc;
 1039         frame = td->td_frame;
 1040         sa = &td->td_sa;
 1041 
 1042         sa->code = frame->tf_rax;
 1043 
 1044         if (__predict_false(sa->code == SYS_syscall ||
 1045             sa->code == SYS___syscall ||
 1046             sa->code >= p->p_sysent->sv_size))
 1047                 return (cpu_fetch_syscall_args_fallback(td, sa));
 1048 
 1049         sa->callp = &p->p_sysent->sv_table[sa->code];
 1050         sa->narg = sa->callp->sy_narg;
 1051         KASSERT(sa->narg <= nitems(sa->args), ("Too many syscall arguments!"));
 1052 
 1053         if (p->p_sysent->sv_mask)
 1054                 sa->code &= p->p_sysent->sv_mask;
 1055 
 1056         if (__predict_false(sa->narg > NARGREGS))
 1057                 return (cpu_fetch_syscall_args_fallback(td, sa));
 1058 
 1059         memcpy(sa->args, &frame->tf_rdi, sizeof(sa->args[0]) * NARGREGS);
 1060 
 1061         td->td_retval[0] = 0;
 1062         td->td_retval[1] = frame->tf_rdx;
 1063 
 1064         return (0);
 1065 }
 1066 
 1067 #include "../../kern/subr_syscall.c"
 1068 
 1069 static void (*syscall_ret_l1d_flush)(void);
 1070 int syscall_ret_l1d_flush_mode;
 1071 
 1072 static void
 1073 flush_l1d_hw(void)
 1074 {
 1075 
 1076         wrmsr(MSR_IA32_FLUSH_CMD, IA32_FLUSH_CMD_L1D);
 1077 }
 1078 
 1079 static void __inline
 1080 amd64_syscall_ret_flush_l1d_inline(int error)
 1081 {
 1082         void (*p)(void);
 1083 
 1084         if (error != 0 && error != EEXIST && error != EAGAIN &&
 1085             error != EXDEV && error != ENOENT && error != ENOTCONN &&
 1086             error != EINPROGRESS) {
 1087                 p = syscall_ret_l1d_flush;
 1088                 if (p != NULL)
 1089                         p();
 1090         }
 1091 }
 1092 
 1093 void
 1094 amd64_syscall_ret_flush_l1d(int error)
 1095 {
 1096 
 1097         amd64_syscall_ret_flush_l1d_inline(error);
 1098 }
 1099 
 1100 void
 1101 amd64_syscall_ret_flush_l1d_recalc(void)
 1102 {
 1103         bool l1d_hw;
 1104 
 1105         l1d_hw = (cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) != 0;
 1106 again:
 1107         switch (syscall_ret_l1d_flush_mode) {
 1108         case 0:
 1109                 syscall_ret_l1d_flush = NULL;
 1110                 break;
 1111         case 1:
 1112                 syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw :
 1113                     flush_l1d_sw_abi;
 1114                 break;
 1115         case 2:
 1116                 syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw : NULL;
 1117                 break;
 1118         case 3:
 1119                 syscall_ret_l1d_flush = flush_l1d_sw_abi;
 1120                 break;
 1121         default:
 1122                 syscall_ret_l1d_flush_mode = 1;
 1123                 goto again;
 1124         }
 1125 }
 1126 
 1127 static int
 1128 machdep_syscall_ret_flush_l1d(SYSCTL_HANDLER_ARGS)
 1129 {
 1130         int error, val;
 1131 
 1132         val = syscall_ret_l1d_flush_mode;
 1133         error = sysctl_handle_int(oidp, &val, 0, req);
 1134         if (error != 0 || req->newptr == NULL)
 1135                 return (error);
 1136         syscall_ret_l1d_flush_mode = val;
 1137         amd64_syscall_ret_flush_l1d_recalc();
 1138         return (0);
 1139 }
 1140 SYSCTL_PROC(_machdep, OID_AUTO, syscall_ret_flush_l1d, CTLTYPE_INT |
 1141     CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, NULL, 0,
 1142     machdep_syscall_ret_flush_l1d, "I",
 1143     "Flush L1D on syscall return with error (0 - off, 1 - on, "
 1144     "2 - use hw only, 3 - use sw only");
 1145 
 1146 
 1147 /*
 1148  * System call handler for native binaries.  The trap frame is already
 1149  * set up by the assembler trampoline and a pointer to it is saved in
 1150  * td_frame.
 1151  */
 1152 void
 1153 amd64_syscall(struct thread *td, int traced)
 1154 {
 1155         ksiginfo_t ksi;
 1156 
 1157 #ifdef DIAGNOSTIC
 1158         if (!TRAPF_USERMODE(td->td_frame)) {
 1159                 panic("syscall");
 1160                 /* NOT REACHED */
 1161         }
 1162 #endif
 1163         syscallenter(td);
 1164 
 1165         /*
 1166          * Traced syscall.
 1167          */
 1168         if (__predict_false(traced)) {
 1169                 td->td_frame->tf_rflags &= ~PSL_T;
 1170                 ksiginfo_init_trap(&ksi);
 1171                 ksi.ksi_signo = SIGTRAP;
 1172                 ksi.ksi_code = TRAP_TRACE;
 1173                 ksi.ksi_addr = (void *)td->td_frame->tf_rip;
 1174                 trapsignal(td, &ksi);
 1175         }
 1176 
 1177         KASSERT(PCB_USER_FPU(td->td_pcb),
 1178             ("System call %s returning with kernel FPU ctx leaked",
 1179              syscallname(td->td_proc, td->td_sa.code)));
 1180         KASSERT(td->td_pcb->pcb_save == get_pcb_user_save_td(td),
 1181             ("System call %s returning with mangled pcb_save",
 1182              syscallname(td->td_proc, td->td_sa.code)));
 1183         KASSERT(pmap_not_in_di(),
 1184             ("System call %s returning with leaked invl_gen %lu",
 1185             syscallname(td->td_proc, td->td_sa.code),
 1186             td->td_md.md_invl_gen.gen));
 1187 
 1188         syscallret(td);
 1189 
 1190         /*
 1191          * If the user-supplied value of %rip is not a canonical
 1192          * address, then some CPUs will trigger a ring 0 #GP during
 1193          * the sysret instruction.  However, the fault handler would
 1194          * execute in ring 0 with the user's %gs and %rsp which would
 1195          * not be safe.  Instead, use the full return path which
 1196          * catches the problem safely.
 1197          */
 1198         if (__predict_false(td->td_frame->tf_rip >= VM_MAXUSER_ADDRESS))
 1199                 set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
 1200 
 1201         amd64_syscall_ret_flush_l1d_inline(td->td_errno);
 1202 }

Cache object: 599294639ee7fef193574026a64d20bf


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.