The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/trap.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (C) 1994, David Greenman
    3  * Copyright (c) 1990, 1993
    4  *      The Regents of the University of California.  All rights reserved.
    5  *
    6  * This code is derived from software contributed to Berkeley by
    7  * the University of Utah, and William Jolitz.
    8  *
    9  * Redistribution and use in source and binary forms, with or without
   10  * modification, are permitted provided that the following conditions
   11  * are met:
   12  * 1. Redistributions of source code must retain the above copyright
   13  *    notice, this list of conditions and the following disclaimer.
   14  * 2. Redistributions in binary form must reproduce the above copyright
   15  *    notice, this list of conditions and the following disclaimer in the
   16  *    documentation and/or other materials provided with the distribution.
   17  * 3. All advertising materials mentioning features or use of this software
   18  *    must display the following acknowledgement:
   19  *      This product includes software developed by the University of
   20  *      California, Berkeley and its contributors.
   21  * 4. Neither the name of the University nor the names of its contributors
   22  *    may be used to endorse or promote products derived from this software
   23  *    without specific prior written permission.
   24  *
   25  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   27  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   28  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   29  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   30  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   31  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   32  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   33  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   34  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   35  * SUCH DAMAGE.
   36  *
   37  *      from: @(#)trap.c        7.4 (Berkeley) 5/13/91
   38  */
   39 
   40 #include <sys/cdefs.h>
   41 __FBSDID("$FreeBSD: releng/10.3/sys/amd64/amd64/trap.c 294136 2016-01-16 07:56:49Z dchagin $");
   42 
   43 /*
   44  * AMD64 Trap and System call handling
   45  */
   46 
   47 #include "opt_clock.h"
   48 #include "opt_cpu.h"
   49 #include "opt_hwpmc_hooks.h"
   50 #include "opt_isa.h"
   51 #include "opt_kdb.h"
   52 #include "opt_kdtrace.h"
   53 
   54 #include <sys/param.h>
   55 #include <sys/bus.h>
   56 #include <sys/systm.h>
   57 #include <sys/proc.h>
   58 #include <sys/pioctl.h>
   59 #include <sys/ptrace.h>
   60 #include <sys/kdb.h>
   61 #include <sys/kernel.h>
   62 #include <sys/ktr.h>
   63 #include <sys/lock.h>
   64 #include <sys/mutex.h>
   65 #include <sys/resourcevar.h>
   66 #include <sys/signalvar.h>
   67 #include <sys/syscall.h>
   68 #include <sys/sysctl.h>
   69 #include <sys/sysent.h>
   70 #include <sys/uio.h>
   71 #include <sys/vmmeter.h>
   72 #ifdef HWPMC_HOOKS
   73 #include <sys/pmckern.h>
   74 PMC_SOFT_DEFINE( , , page_fault, all);
   75 PMC_SOFT_DEFINE( , , page_fault, read);
   76 PMC_SOFT_DEFINE( , , page_fault, write);
   77 #endif
   78 
   79 #include <vm/vm.h>
   80 #include <vm/vm_param.h>
   81 #include <vm/pmap.h>
   82 #include <vm/vm_kern.h>
   83 #include <vm/vm_map.h>
   84 #include <vm/vm_page.h>
   85 #include <vm/vm_extern.h>
   86 
   87 #include <machine/cpu.h>
   88 #include <machine/intr_machdep.h>
   89 #include <x86/mca.h>
   90 #include <machine/md_var.h>
   91 #include <machine/pcb.h>
   92 #ifdef SMP
   93 #include <machine/smp.h>
   94 #endif
   95 #include <machine/tss.h>
   96 
   97 #ifdef KDTRACE_HOOKS
   98 #include <sys/dtrace_bsd.h>
   99 #endif
  100 
  101 extern void trap(struct trapframe *frame);
  102 extern void syscall(struct trapframe *frame);
  103 void dblfault_handler(struct trapframe *frame);
  104 
  105 static int trap_pfault(struct trapframe *, int);
  106 static void trap_fatal(struct trapframe *, vm_offset_t);
  107 
  108 #define MAX_TRAP_MSG            32
  109 static char *trap_msg[] = {
  110         "",                                     /*  0 unused */
  111         "privileged instruction fault",         /*  1 T_PRIVINFLT */
  112         "",                                     /*  2 unused */
  113         "breakpoint instruction fault",         /*  3 T_BPTFLT */
  114         "",                                     /*  4 unused */
  115         "",                                     /*  5 unused */
  116         "arithmetic trap",                      /*  6 T_ARITHTRAP */
  117         "",                                     /*  7 unused */
  118         "",                                     /*  8 unused */
  119         "general protection fault",             /*  9 T_PROTFLT */
  120         "trace trap",                           /* 10 T_TRCTRAP */
  121         "",                                     /* 11 unused */
  122         "page fault",                           /* 12 T_PAGEFLT */
  123         "",                                     /* 13 unused */
  124         "alignment fault",                      /* 14 T_ALIGNFLT */
  125         "",                                     /* 15 unused */
  126         "",                                     /* 16 unused */
  127         "",                                     /* 17 unused */
  128         "integer divide fault",                 /* 18 T_DIVIDE */
  129         "non-maskable interrupt trap",          /* 19 T_NMI */
  130         "overflow trap",                        /* 20 T_OFLOW */
  131         "FPU bounds check fault",               /* 21 T_BOUND */
  132         "FPU device not available",             /* 22 T_DNA */
  133         "double fault",                         /* 23 T_DOUBLEFLT */
  134         "FPU operand fetch fault",              /* 24 T_FPOPFLT */
  135         "invalid TSS fault",                    /* 25 T_TSSFLT */
  136         "segment not present fault",            /* 26 T_SEGNPFLT */
  137         "stack fault",                          /* 27 T_STKFLT */
  138         "machine check trap",                   /* 28 T_MCHK */
  139         "SIMD floating-point exception",        /* 29 T_XMMFLT */
  140         "reserved (unknown) fault",             /* 30 T_RESERVED */
  141         "",                                     /* 31 unused (reserved) */
  142         "DTrace pid return trap",               /* 32 T_DTRACE_RET */
  143 };
  144 
  145 #ifdef KDB
  146 static int kdb_on_nmi = 1;
  147 SYSCTL_INT(_machdep, OID_AUTO, kdb_on_nmi, CTLFLAG_RW,
  148         &kdb_on_nmi, 0, "Go to KDB on NMI");
  149 TUNABLE_INT("machdep.kdb_on_nmi", &kdb_on_nmi);
  150 #endif
  151 static int panic_on_nmi = 1;
  152 SYSCTL_INT(_machdep, OID_AUTO, panic_on_nmi, CTLFLAG_RW,
  153         &panic_on_nmi, 0, "Panic on NMI");
  154 TUNABLE_INT("machdep.panic_on_nmi", &panic_on_nmi);
  155 static int prot_fault_translation;
  156 SYSCTL_INT(_machdep, OID_AUTO, prot_fault_translation, CTLFLAG_RW,
  157     &prot_fault_translation, 0,
  158     "Select signal to deliver on protection fault");
  159 static int uprintf_signal;
  160 SYSCTL_INT(_machdep, OID_AUTO, uprintf_signal, CTLFLAG_RW,
  161     &uprintf_signal, 0,
  162     "Print debugging information on trap signal to ctty");
  163 
  164 /*
  165  * Exception, fault, and trap interface to the FreeBSD kernel.
  166  * This common code is called from assembly language IDT gate entry
  167  * routines that prepare a suitable stack frame, and restore this
  168  * frame after the exception has been processed.
  169  */
  170 
  171 void
  172 trap(struct trapframe *frame)
  173 {
  174 #ifdef KDTRACE_HOOKS
  175         struct reg regs;
  176 #endif
  177         struct thread *td = curthread;
  178         struct proc *p = td->td_proc;
  179         int i = 0, ucode = 0, code;
  180         u_int type;
  181         register_t addr = 0;
  182         ksiginfo_t ksi;
  183 
  184         PCPU_INC(cnt.v_trap);
  185         type = frame->tf_trapno;
  186 
  187 #ifdef SMP
  188         /* Handler for NMI IPIs used for stopping CPUs. */
  189         if (type == T_NMI) {
  190                  if (ipi_nmi_handler() == 0)
  191                            goto out;
  192         }
  193 #endif /* SMP */
  194 
  195 #ifdef KDB
  196         if (kdb_active) {
  197                 kdb_reenter();
  198                 goto out;
  199         }
  200 #endif
  201 
  202         if (type == T_RESERVED) {
  203                 trap_fatal(frame, 0);
  204                 goto out;
  205         }
  206 
  207 #ifdef  HWPMC_HOOKS
  208         /*
  209          * CPU PMCs interrupt using an NMI.  If the PMC module is
  210          * active, pass the 'rip' value to the PMC module's interrupt
  211          * handler.  A return value of '1' from the handler means that
  212          * the NMI was handled by it and we can return immediately.
  213          */
  214         if (type == T_NMI && pmc_intr &&
  215             (*pmc_intr)(PCPU_GET(cpuid), frame))
  216                 goto out;
  217 #endif
  218 
  219         if (type == T_MCHK) {
  220                 mca_intr();
  221                 goto out;
  222         }
  223 
  224 #ifdef KDTRACE_HOOKS
  225         /*
  226          * A trap can occur while DTrace executes a probe. Before
  227          * executing the probe, DTrace blocks re-scheduling and sets
  228          * a flag in its per-cpu flags to indicate that it doesn't
  229          * want to fault. On returning from the probe, the no-fault
  230          * flag is cleared and finally re-scheduling is enabled.
  231          */
  232         if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, type))
  233                 goto out;
  234 #endif
  235 
  236         if ((frame->tf_rflags & PSL_I) == 0) {
  237                 /*
  238                  * Buggy application or kernel code has disabled
  239                  * interrupts and then trapped.  Enabling interrupts
  240                  * now is wrong, but it is better than running with
  241                  * interrupts disabled until they are accidentally
  242                  * enabled later.
  243                  */
  244                 if (ISPL(frame->tf_cs) == SEL_UPL)
  245                         uprintf(
  246                             "pid %ld (%s): trap %d with interrupts disabled\n",
  247                             (long)curproc->p_pid, curthread->td_name, type);
  248                 else if (type != T_NMI && type != T_BPTFLT &&
  249                     type != T_TRCTRAP) {
  250                         /*
  251                          * XXX not quite right, since this may be for a
  252                          * multiple fault in user mode.
  253                          */
  254                         printf("kernel trap %d with interrupts disabled\n",
  255                             type);
  256 
  257                         /*
  258                          * We shouldn't enable interrupts while holding a
  259                          * spin lock.
  260                          */
  261                         if (td->td_md.md_spinlock_count == 0)
  262                                 enable_intr();
  263                 }
  264         }
  265 
  266         code = frame->tf_err;
  267 
  268         if (ISPL(frame->tf_cs) == SEL_UPL) {
  269                 /* user trap */
  270 
  271                 td->td_pticks = 0;
  272                 td->td_frame = frame;
  273                 addr = frame->tf_rip;
  274                 if (td->td_ucred != p->p_ucred) 
  275                         cred_update_thread(td);
  276 
  277                 switch (type) {
  278                 case T_PRIVINFLT:       /* privileged instruction fault */
  279                         i = SIGILL;
  280                         ucode = ILL_PRVOPC;
  281                         break;
  282 
  283                 case T_BPTFLT:          /* bpt instruction fault */
  284                 case T_TRCTRAP:         /* trace trap */
  285                         enable_intr();
  286 #ifdef KDTRACE_HOOKS
  287                         if (type == T_BPTFLT) {
  288                                 fill_frame_regs(frame, &regs);
  289                                 if (dtrace_pid_probe_ptr != NULL &&
  290                                     dtrace_pid_probe_ptr(&regs) == 0)
  291                                         goto out;
  292                         }
  293 #endif
  294                         frame->tf_rflags &= ~PSL_T;
  295                         i = SIGTRAP;
  296                         ucode = (type == T_TRCTRAP ? TRAP_TRACE : TRAP_BRKPT);
  297                         break;
  298 
  299                 case T_ARITHTRAP:       /* arithmetic trap */
  300                         ucode = fputrap_x87();
  301                         if (ucode == -1)
  302                                 goto userout;
  303                         i = SIGFPE;
  304                         break;
  305 
  306                 case T_PROTFLT:         /* general protection fault */
  307                         i = SIGBUS;
  308                         ucode = BUS_OBJERR;
  309                         break;
  310                 case T_STKFLT:          /* stack fault */
  311                 case T_SEGNPFLT:        /* segment not present fault */
  312                         i = SIGBUS;
  313                         ucode = BUS_ADRERR;
  314                         break;
  315                 case T_TSSFLT:          /* invalid TSS fault */
  316                         i = SIGBUS;
  317                         ucode = BUS_OBJERR;
  318                         break;
  319                 case T_ALIGNFLT:
  320                         i = SIGBUS;
  321                         ucode = BUS_ADRALN;
  322                         break;
  323                 case T_DOUBLEFLT:       /* double fault */
  324                 default:
  325                         i = SIGBUS;
  326                         ucode = BUS_OBJERR;
  327                         break;
  328 
  329                 case T_PAGEFLT:         /* page fault */
  330                         /*
  331                          * Emulator can take care about this trap?
  332                          */
  333                         if (*p->p_sysent->sv_trap != NULL &&
  334                             (*p->p_sysent->sv_trap)(td) == 0)
  335                                 goto userout;
  336 
  337                         addr = frame->tf_addr;
  338                         i = trap_pfault(frame, TRUE);
  339                         if (i == -1)
  340                                 goto userout;
  341                         if (i == 0)
  342                                 goto user;
  343 
  344                         if (i == SIGSEGV)
  345                                 ucode = SEGV_MAPERR;
  346                         else {
  347                                 if (prot_fault_translation == 0) {
  348                                         /*
  349                                          * Autodetect.
  350                                          * This check also covers the images
  351                                          * without the ABI-tag ELF note.
  352                                          */
  353                                         if (SV_CURPROC_ABI() == SV_ABI_FREEBSD
  354                                             && p->p_osrel >= P_OSREL_SIGSEGV) {
  355                                                 i = SIGSEGV;
  356                                                 ucode = SEGV_ACCERR;
  357                                         } else {
  358                                                 i = SIGBUS;
  359                                                 ucode = BUS_PAGE_FAULT;
  360                                         }
  361                                 } else if (prot_fault_translation == 1) {
  362                                         /*
  363                                          * Always compat mode.
  364                                          */
  365                                         i = SIGBUS;
  366                                         ucode = BUS_PAGE_FAULT;
  367                                 } else {
  368                                         /*
  369                                          * Always SIGSEGV mode.
  370                                          */
  371                                         i = SIGSEGV;
  372                                         ucode = SEGV_ACCERR;
  373                                 }
  374                         }
  375                         break;
  376 
  377                 case T_DIVIDE:          /* integer divide fault */
  378                         ucode = FPE_INTDIV;
  379                         i = SIGFPE;
  380                         break;
  381 
  382 #ifdef DEV_ISA
  383                 case T_NMI:
  384                         /* machine/parity/power fail/"kitchen sink" faults */
  385                         if (isa_nmi(code) == 0) {
  386 #ifdef KDB
  387                                 /*
  388                                  * NMI can be hooked up to a pushbutton
  389                                  * for debugging.
  390                                  */
  391                                 if (kdb_on_nmi) {
  392                                         printf ("NMI ... going to debugger\n");
  393                                         kdb_trap(type, 0, frame);
  394                                 }
  395 #endif /* KDB */
  396                                 goto userout;
  397                         } else if (panic_on_nmi)
  398                                 panic("NMI indicates hardware failure");
  399                         break;
  400 #endif /* DEV_ISA */
  401 
  402                 case T_OFLOW:           /* integer overflow fault */
  403                         ucode = FPE_INTOVF;
  404                         i = SIGFPE;
  405                         break;
  406 
  407                 case T_BOUND:           /* bounds check fault */
  408                         ucode = FPE_FLTSUB;
  409                         i = SIGFPE;
  410                         break;
  411 
  412                 case T_DNA:
  413                         /* transparent fault (due to context switch "late") */
  414                         KASSERT(PCB_USER_FPU(td->td_pcb),
  415                             ("kernel FPU ctx has leaked"));
  416                         fpudna();
  417                         goto userout;
  418 
  419                 case T_FPOPFLT:         /* FPU operand fetch fault */
  420                         ucode = ILL_COPROC;
  421                         i = SIGILL;
  422                         break;
  423 
  424                 case T_XMMFLT:          /* SIMD floating-point exception */
  425                         ucode = fputrap_sse();
  426                         if (ucode == -1)
  427                                 goto userout;
  428                         i = SIGFPE;
  429                         break;
  430 #ifdef KDTRACE_HOOKS
  431                 case T_DTRACE_RET:
  432                         enable_intr();
  433                         fill_frame_regs(frame, &regs);
  434                         if (dtrace_return_probe_ptr != NULL &&
  435                             dtrace_return_probe_ptr(&regs) == 0)
  436                                 goto out;
  437                         break;
  438 #endif
  439                 }
  440         } else {
  441                 /* kernel trap */
  442 
  443                 KASSERT(cold || td->td_ucred != NULL,
  444                     ("kernel trap doesn't have ucred"));
  445                 switch (type) {
  446                 case T_PAGEFLT:                 /* page fault */
  447                         (void) trap_pfault(frame, FALSE);
  448                         goto out;
  449 
  450                 case T_DNA:
  451                         KASSERT(!PCB_USER_FPU(td->td_pcb),
  452                             ("Unregistered use of FPU in kernel"));
  453                         fpudna();
  454                         goto out;
  455 
  456                 case T_ARITHTRAP:       /* arithmetic trap */
  457                 case T_XMMFLT:          /* SIMD floating-point exception */
  458                 case T_FPOPFLT:         /* FPU operand fetch fault */
  459                         /*
  460                          * For now, supporting kernel handler
  461                          * registration for FPU traps is overkill.
  462                          */
  463                         trap_fatal(frame, 0);
  464                         goto out;
  465 
  466                 case T_STKFLT:          /* stack fault */
  467                 case T_PROTFLT:         /* general protection fault */
  468                 case T_SEGNPFLT:        /* segment not present fault */
  469                         if (td->td_intr_nesting_level != 0)
  470                                 break;
  471 
  472                         /*
  473                          * Invalid segment selectors and out of bounds
  474                          * %rip's and %rsp's can be set up in user mode.
  475                          * This causes a fault in kernel mode when the
  476                          * kernel tries to return to user mode.  We want
  477                          * to get this fault so that we can fix the
  478                          * problem here and not have to check all the
  479                          * selectors and pointers when the user changes
  480                          * them.
  481                          */
  482                         if (frame->tf_rip == (long)doreti_iret) {
  483                                 frame->tf_rip = (long)doreti_iret_fault;
  484                                 goto out;
  485                         }
  486                         if (frame->tf_rip == (long)ld_ds) {
  487                                 frame->tf_rip = (long)ds_load_fault;
  488                                 goto out;
  489                         }
  490                         if (frame->tf_rip == (long)ld_es) {
  491                                 frame->tf_rip = (long)es_load_fault;
  492                                 goto out;
  493                         }
  494                         if (frame->tf_rip == (long)ld_fs) {
  495                                 frame->tf_rip = (long)fs_load_fault;
  496                                 goto out;
  497                         }
  498                         if (frame->tf_rip == (long)ld_gs) {
  499                                 frame->tf_rip = (long)gs_load_fault;
  500                                 goto out;
  501                         }
  502                         if (frame->tf_rip == (long)ld_gsbase) {
  503                                 frame->tf_rip = (long)gsbase_load_fault;
  504                                 goto out;
  505                         }
  506                         if (frame->tf_rip == (long)ld_fsbase) {
  507                                 frame->tf_rip = (long)fsbase_load_fault;
  508                                 goto out;
  509                         }
  510                         if (curpcb->pcb_onfault != NULL) {
  511                                 frame->tf_rip = (long)curpcb->pcb_onfault;
  512                                 goto out;
  513                         }
  514                         break;
  515 
  516                 case T_TSSFLT:
  517                         /*
  518                          * PSL_NT can be set in user mode and isn't cleared
  519                          * automatically when the kernel is entered.  This
  520                          * causes a TSS fault when the kernel attempts to
  521                          * `iret' because the TSS link is uninitialized.  We
  522                          * want to get this fault so that we can fix the
  523                          * problem here and not every time the kernel is
  524                          * entered.
  525                          */
  526                         if (frame->tf_rflags & PSL_NT) {
  527                                 frame->tf_rflags &= ~PSL_NT;
  528                                 goto out;
  529                         }
  530                         break;
  531 
  532                 case T_TRCTRAP:  /* trace trap */
  533                         /*
  534                          * Ignore debug register trace traps due to
  535                          * accesses in the user's address space, which
  536                          * can happen under several conditions such as
  537                          * if a user sets a watchpoint on a buffer and
  538                          * then passes that buffer to a system call.
  539                          * We still want to get TRCTRAPS for addresses
  540                          * in kernel space because that is useful when
  541                          * debugging the kernel.
  542                          */
  543                         if (user_dbreg_trap()) {
  544                                 /*
  545                                  * Reset breakpoint bits because the
  546                                  * processor doesn't
  547                                  */
  548                                 /* XXX check upper bits here */
  549                                 load_dr6(rdr6() & 0xfffffff0);
  550                                 goto out;
  551                         }
  552                         /*
  553                          * FALLTHROUGH (TRCTRAP kernel mode, kernel address)
  554                          */
  555                 case T_BPTFLT:
  556                         /*
  557                          * If KDB is enabled, let it handle the debugger trap.
  558                          * Otherwise, debugger traps "can't happen".
  559                          */
  560 #ifdef KDB
  561                         if (kdb_trap(type, 0, frame))
  562                                 goto out;
  563 #endif
  564                         break;
  565 
  566 #ifdef DEV_ISA
  567                 case T_NMI:
  568                         /* machine/parity/power fail/"kitchen sink" faults */
  569                         if (isa_nmi(code) == 0) {
  570 #ifdef KDB
  571                                 /*
  572                                  * NMI can be hooked up to a pushbutton
  573                                  * for debugging.
  574                                  */
  575                                 if (kdb_on_nmi) {
  576                                         printf ("NMI ... going to debugger\n");
  577                                         kdb_trap(type, 0, frame);
  578                                 }
  579 #endif /* KDB */
  580                                 goto out;
  581                         } else if (panic_on_nmi == 0)
  582                                 goto out;
  583                         /* FALLTHROUGH */
  584 #endif /* DEV_ISA */
  585                 }
  586 
  587                 trap_fatal(frame, 0);
  588                 goto out;
  589         }
  590 
  591         /* Translate fault for emulators (e.g. Linux) */
  592         if (*p->p_sysent->sv_transtrap)
  593                 i = (*p->p_sysent->sv_transtrap)(i, type);
  594 
  595         ksiginfo_init_trap(&ksi);
  596         ksi.ksi_signo = i;
  597         ksi.ksi_code = ucode;
  598         ksi.ksi_trapno = type;
  599         ksi.ksi_addr = (void *)addr;
  600         if (uprintf_signal) {
  601                 uprintf("pid %d comm %s: signal %d err %lx code %d type %d "
  602                     "addr 0x%lx rsp 0x%lx rip 0x%lx "
  603                     "<%02x %02x %02x %02x %02x %02x %02x %02x>\n",
  604                     p->p_pid, p->p_comm, i, frame->tf_err, ucode, type, addr,
  605                     frame->tf_rsp, frame->tf_rip,
  606                     fubyte((void *)(frame->tf_rip + 0)),
  607                     fubyte((void *)(frame->tf_rip + 1)),
  608                     fubyte((void *)(frame->tf_rip + 2)),
  609                     fubyte((void *)(frame->tf_rip + 3)),
  610                     fubyte((void *)(frame->tf_rip + 4)),
  611                     fubyte((void *)(frame->tf_rip + 5)),
  612                     fubyte((void *)(frame->tf_rip + 6)),
  613                     fubyte((void *)(frame->tf_rip + 7)));
  614         }
  615         KASSERT((read_rflags() & PSL_I) != 0, ("interrupts disabled"));
  616         trapsignal(td, &ksi);
  617 
  618 user:
  619         userret(td, frame);
  620         KASSERT(PCB_USER_FPU(td->td_pcb),
  621             ("Return from trap with kernel FPU ctx leaked"));
  622 userout:
  623 out:
  624         return;
  625 }
  626 
  627 static int
  628 trap_pfault(frame, usermode)
  629         struct trapframe *frame;
  630         int usermode;
  631 {
  632         vm_offset_t va;
  633         struct vmspace *vm;
  634         vm_map_t map;
  635         int rv = 0;
  636         vm_prot_t ftype;
  637         struct thread *td = curthread;
  638         struct proc *p = td->td_proc;
  639         vm_offset_t eva = frame->tf_addr;
  640 
  641         if (__predict_false((td->td_pflags & TDP_NOFAULTING) != 0)) {
  642                 /*
  643                  * Due to both processor errata and lazy TLB invalidation when
  644                  * access restrictions are removed from virtual pages, memory
  645                  * accesses that are allowed by the physical mapping layer may
  646                  * nonetheless cause one spurious page fault per virtual page. 
  647                  * When the thread is executing a "no faulting" section that
  648                  * is bracketed by vm_fault_{disable,enable}_pagefaults(),
  649                  * every page fault is treated as a spurious page fault,
  650                  * unless it accesses the same virtual address as the most
  651                  * recent page fault within the same "no faulting" section.
  652                  */
  653                 if (td->td_md.md_spurflt_addr != eva ||
  654                     (td->td_pflags & TDP_RESETSPUR) != 0) {
  655                         /*
  656                          * Do nothing to the TLB.  A stale TLB entry is
  657                          * flushed automatically by a page fault.
  658                          */
  659                         td->td_md.md_spurflt_addr = eva;
  660                         td->td_pflags &= ~TDP_RESETSPUR;
  661                         return (0);
  662                 }
  663         } else {
  664                 /*
  665                  * If we get a page fault while in a critical section, then
  666                  * it is most likely a fatal kernel page fault.  The kernel
  667                  * is already going to panic trying to get a sleep lock to
  668                  * do the VM lookup, so just consider it a fatal trap so the
  669                  * kernel can print out a useful trap message and even get
  670                  * to the debugger.
  671                  *
  672                  * If we get a page fault while holding a non-sleepable
  673                  * lock, then it is most likely a fatal kernel page fault.
  674                  * If WITNESS is enabled, then it's going to whine about
  675                  * bogus LORs with various VM locks, so just skip to the
  676                  * fatal trap handling directly.
  677                  */
  678                 if (td->td_critnest != 0 ||
  679                     WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
  680                     "Kernel page fault") != 0) {
  681                         trap_fatal(frame, eva);
  682                         return (-1);
  683                 }
  684         }
  685         va = trunc_page(eva);
  686         if (va >= VM_MIN_KERNEL_ADDRESS) {
  687                 /*
  688                  * Don't allow user-mode faults in kernel address space.
  689                  */
  690                 if (usermode)
  691                         goto nogo;
  692 
  693                 map = kernel_map;
  694         } else {
  695                 /*
  696                  * This is a fault on non-kernel virtual memory.  If either
  697                  * p or p->p_vmspace is NULL, then the fault is fatal.
  698                  */
  699                 if (p == NULL || (vm = p->p_vmspace) == NULL)
  700                         goto nogo;
  701 
  702                 map = &vm->vm_map;
  703 
  704                 /*
  705                  * When accessing a usermode address, kernel must be
  706                  * ready to accept the page fault, and provide a
  707                  * handling routine.  Since accessing the address
  708                  * without the handler is a bug, do not try to handle
  709                  * it normally, and panic immediately.
  710                  */
  711                 if (!usermode && (td->td_intr_nesting_level != 0 ||
  712                     curpcb->pcb_onfault == NULL)) {
  713                         trap_fatal(frame, eva);
  714                         return (-1);
  715                 }
  716         }
  717 
  718         /*
  719          * If the trap was caused by errant bits in the PTE then panic.
  720          */
  721         if (frame->tf_err & PGEX_RSV) {
  722                 trap_fatal(frame, eva);
  723                 return (-1);
  724         }
  725 
  726         /*
  727          * PGEX_I is defined only if the execute disable bit capability is
  728          * supported and enabled.
  729          */
  730         if (frame->tf_err & PGEX_W)
  731                 ftype = VM_PROT_WRITE;
  732         else if ((frame->tf_err & PGEX_I) && pg_nx != 0)
  733                 ftype = VM_PROT_EXECUTE;
  734         else
  735                 ftype = VM_PROT_READ;
  736 
  737         if (map != kernel_map) {
  738                 /*
  739                  * Keep swapout from messing with us during this
  740                  *      critical time.
  741                  */
  742                 PROC_LOCK(p);
  743                 ++p->p_lock;
  744                 PROC_UNLOCK(p);
  745 
  746                 /* Fault in the user page: */
  747                 rv = vm_fault(map, va, ftype, VM_FAULT_NORMAL);
  748 
  749                 PROC_LOCK(p);
  750                 --p->p_lock;
  751                 PROC_UNLOCK(p);
  752         } else {
  753                 /*
  754                  * Don't have to worry about process locking or stacks in the
  755                  * kernel.
  756                  */
  757                 rv = vm_fault(map, va, ftype, VM_FAULT_NORMAL);
  758         }
  759         if (rv == KERN_SUCCESS) {
  760 #ifdef HWPMC_HOOKS
  761                 if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) {
  762                         PMC_SOFT_CALL_TF( , , page_fault, all, frame);
  763                         if (ftype == VM_PROT_READ)
  764                                 PMC_SOFT_CALL_TF( , , page_fault, read,
  765                                     frame);
  766                         else
  767                                 PMC_SOFT_CALL_TF( , , page_fault, write,
  768                                     frame);
  769                 }
  770 #endif
  771                 return (0);
  772         }
  773 nogo:
  774         if (!usermode) {
  775                 if (td->td_intr_nesting_level == 0 &&
  776                     curpcb->pcb_onfault != NULL) {
  777                         frame->tf_rip = (long)curpcb->pcb_onfault;
  778                         return (0);
  779                 }
  780                 trap_fatal(frame, eva);
  781                 return (-1);
  782         }
  783         return ((rv == KERN_PROTECTION_FAILURE) ? SIGBUS : SIGSEGV);
  784 }
  785 
  786 static void
  787 trap_fatal(frame, eva)
  788         struct trapframe *frame;
  789         vm_offset_t eva;
  790 {
  791         int code, ss;
  792         u_int type;
  793         long esp;
  794         struct soft_segment_descriptor softseg;
  795         char *msg;
  796 
  797         code = frame->tf_err;
  798         type = frame->tf_trapno;
  799         sdtossd(&gdt[NGDT * PCPU_GET(cpuid) + IDXSEL(frame->tf_cs & 0xffff)],
  800             &softseg);
  801 
  802         if (type <= MAX_TRAP_MSG)
  803                 msg = trap_msg[type];
  804         else
  805                 msg = "UNKNOWN";
  806         printf("\n\nFatal trap %d: %s while in %s mode\n", type, msg,
  807             ISPL(frame->tf_cs) == SEL_UPL ? "user" : "kernel");
  808 #ifdef SMP
  809         /* two separate prints in case of a trap on an unmapped page */
  810         printf("cpuid = %d; ", PCPU_GET(cpuid));
  811         printf("apic id = %02x\n", PCPU_GET(apic_id));
  812 #endif
  813         if (type == T_PAGEFLT) {
  814                 printf("fault virtual address   = 0x%lx\n", eva);
  815                 printf("fault code              = %s %s %s%s, %s\n",
  816                         code & PGEX_U ? "user" : "supervisor",
  817                         code & PGEX_W ? "write" : "read",
  818                         code & PGEX_I ? "instruction" : "data",
  819                         code & PGEX_RSV ? " rsv" : "",
  820                         code & PGEX_P ? "protection violation" : "page not present");
  821         }
  822         printf("instruction pointer     = 0x%lx:0x%lx\n",
  823                frame->tf_cs & 0xffff, frame->tf_rip);
  824         if (ISPL(frame->tf_cs) == SEL_UPL) {
  825                 ss = frame->tf_ss & 0xffff;
  826                 esp = frame->tf_rsp;
  827         } else {
  828                 ss = GSEL(GDATA_SEL, SEL_KPL);
  829                 esp = (long)&frame->tf_rsp;
  830         }
  831         printf("stack pointer           = 0x%x:0x%lx\n", ss, esp);
  832         printf("frame pointer           = 0x%x:0x%lx\n", ss, frame->tf_rbp);
  833         printf("code segment            = base 0x%lx, limit 0x%lx, type 0x%x\n",
  834                softseg.ssd_base, softseg.ssd_limit, softseg.ssd_type);
  835         printf("                        = DPL %d, pres %d, long %d, def32 %d, gran %d\n",
  836                softseg.ssd_dpl, softseg.ssd_p, softseg.ssd_long, softseg.ssd_def32,
  837                softseg.ssd_gran);
  838         printf("processor eflags        = ");
  839         if (frame->tf_rflags & PSL_T)
  840                 printf("trace trap, ");
  841         if (frame->tf_rflags & PSL_I)
  842                 printf("interrupt enabled, ");
  843         if (frame->tf_rflags & PSL_NT)
  844                 printf("nested task, ");
  845         if (frame->tf_rflags & PSL_RF)
  846                 printf("resume, ");
  847         printf("IOPL = %ld\n", (frame->tf_rflags & PSL_IOPL) >> 12);
  848         printf("current process         = %d (%s)\n",
  849             curproc->p_pid, curthread->td_name);
  850 
  851 #ifdef KDB
  852         if (debugger_on_panic || kdb_active)
  853                 if (kdb_trap(type, 0, frame))
  854                         return;
  855 #endif
  856         printf("trap number             = %d\n", type);
  857         if (type <= MAX_TRAP_MSG)
  858                 panic("%s", trap_msg[type]);
  859         else
  860                 panic("unknown/reserved trap");
  861 }
  862 
  863 /*
  864  * Double fault handler. Called when a fault occurs while writing
  865  * a frame for a trap/exception onto the stack. This usually occurs
  866  * when the stack overflows (such is the case with infinite recursion,
  867  * for example).
  868  */
  869 void
  870 dblfault_handler(struct trapframe *frame)
  871 {
  872 #ifdef KDTRACE_HOOKS
  873         if (dtrace_doubletrap_func != NULL)
  874                 (*dtrace_doubletrap_func)();
  875 #endif
  876         printf("\nFatal double fault\n");
  877         printf("rip = 0x%lx\n", frame->tf_rip);
  878         printf("rsp = 0x%lx\n", frame->tf_rsp);
  879         printf("rbp = 0x%lx\n", frame->tf_rbp);
  880 #ifdef SMP
  881         /* two separate prints in case of a trap on an unmapped page */
  882         printf("cpuid = %d; ", PCPU_GET(cpuid));
  883         printf("apic id = %02x\n", PCPU_GET(apic_id));
  884 #endif
  885         panic("double fault");
  886 }
  887 
  888 int
  889 cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa)
  890 {
  891         struct proc *p;
  892         struct trapframe *frame;
  893         register_t *argp;
  894         caddr_t params;
  895         int reg, regcnt, error;
  896 
  897         p = td->td_proc;
  898         frame = td->td_frame;
  899         reg = 0;
  900         regcnt = 6;
  901 
  902         params = (caddr_t)frame->tf_rsp + sizeof(register_t);
  903         sa->code = frame->tf_rax;
  904 
  905         if (sa->code == SYS_syscall || sa->code == SYS___syscall) {
  906                 sa->code = frame->tf_rdi;
  907                 reg++;
  908                 regcnt--;
  909         }
  910         if (p->p_sysent->sv_mask)
  911                 sa->code &= p->p_sysent->sv_mask;
  912 
  913         if (sa->code >= p->p_sysent->sv_size)
  914                 sa->callp = &p->p_sysent->sv_table[0];
  915         else
  916                 sa->callp = &p->p_sysent->sv_table[sa->code];
  917 
  918         sa->narg = sa->callp->sy_narg;
  919         KASSERT(sa->narg <= sizeof(sa->args) / sizeof(sa->args[0]),
  920             ("Too many syscall arguments!"));
  921         error = 0;
  922         argp = &frame->tf_rdi;
  923         argp += reg;
  924         bcopy(argp, sa->args, sizeof(sa->args[0]) * regcnt);
  925         if (sa->narg > regcnt) {
  926                 KASSERT(params != NULL, ("copyin args with no params!"));
  927                 error = copyin(params, &sa->args[regcnt],
  928                     (sa->narg - regcnt) * sizeof(sa->args[0]));
  929         }
  930 
  931         if (error == 0) {
  932                 td->td_retval[0] = 0;
  933                 td->td_retval[1] = frame->tf_rdx;
  934         }
  935 
  936         return (error);
  937 }
  938 
  939 #include "../../kern/subr_syscall.c"
  940 
  941 /*
  942  * System call handler for native binaries.  The trap frame is already
  943  * set up by the assembler trampoline and a pointer to it is saved in
  944  * td_frame.
  945  */
  946 void
  947 amd64_syscall(struct thread *td, int traced)
  948 {
  949         struct syscall_args sa;
  950         int error;
  951         ksiginfo_t ksi;
  952 
  953 #ifdef DIAGNOSTIC
  954         if (ISPL(td->td_frame->tf_cs) != SEL_UPL) {
  955                 panic("syscall");
  956                 /* NOT REACHED */
  957         }
  958 #endif
  959         error = syscallenter(td, &sa);
  960 
  961         /*
  962          * Traced syscall.
  963          */
  964         if (__predict_false(traced)) {
  965                 td->td_frame->tf_rflags &= ~PSL_T;
  966                 ksiginfo_init_trap(&ksi);
  967                 ksi.ksi_signo = SIGTRAP;
  968                 ksi.ksi_code = TRAP_TRACE;
  969                 ksi.ksi_addr = (void *)td->td_frame->tf_rip;
  970                 trapsignal(td, &ksi);
  971         }
  972 
  973         KASSERT(PCB_USER_FPU(td->td_pcb),
  974             ("System call %s returing with kernel FPU ctx leaked",
  975              syscallname(td->td_proc, sa.code)));
  976         KASSERT(td->td_pcb->pcb_save == get_pcb_user_save_td(td),
  977             ("System call %s returning with mangled pcb_save",
  978              syscallname(td->td_proc, sa.code)));
  979 
  980         syscallret(td, error, &sa);
  981 
  982         /*
  983          * If the user-supplied value of %rip is not a canonical
  984          * address, then some CPUs will trigger a ring 0 #GP during
  985          * the sysret instruction.  However, the fault handler would
  986          * execute in ring 0 with the user's %gs and %rsp which would
  987          * not be safe.  Instead, use the full return path which
  988          * catches the problem safely.
  989          */
  990         if (td->td_frame->tf_rip >= VM_MAXUSER_ADDRESS)
  991                 set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
  992 }

Cache object: 71e22128a304269f2f795f5583a81e34


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.