The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/fpu.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1990 William Jolitz.
    3  * Copyright (c) 1991 The Regents of the University of California.
    4  * All rights reserved.
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice, this list of conditions and the following disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  * 4. Neither the name of the University nor the names of its contributors
   15  *    may be used to endorse or promote products derived from this software
   16  *    without specific prior written permission.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  *
   30  *      from: @(#)npx.c 7.2 (Berkeley) 5/12/91
   31  */
   32 
   33 #include <sys/cdefs.h>
   34 __FBSDID("$FreeBSD: stable/10/sys/amd64/amd64/fpu.c 337245 2018-08-03 14:12:37Z kib $");
   35 
   36 #include <sys/param.h>
   37 #include <sys/systm.h>
   38 #include <sys/bus.h>
   39 #include <sys/kernel.h>
   40 #include <sys/lock.h>
   41 #include <sys/malloc.h>
   42 #include <sys/module.h>
   43 #include <sys/mutex.h>
   44 #include <sys/mutex.h>
   45 #include <sys/proc.h>
   46 #include <sys/sysctl.h>
   47 #include <machine/bus.h>
   48 #include <sys/rman.h>
   49 #include <sys/signalvar.h>
   50 #include <vm/uma.h>
   51 
   52 #include <machine/cputypes.h>
   53 #include <machine/frame.h>
   54 #include <machine/intr_machdep.h>
   55 #include <machine/md_var.h>
   56 #include <machine/pcb.h>
   57 #include <machine/psl.h>
   58 #include <machine/resource.h>
   59 #include <machine/specialreg.h>
   60 #include <machine/segments.h>
   61 #include <machine/ucontext.h>
   62 
   63 /*
   64  * Floating point support.
   65  */
   66 
   67 #if defined(__GNUCLIKE_ASM) && !defined(lint)
   68 
   69 #define fldcw(cw)               __asm __volatile("fldcw %0" : : "m" (cw))
   70 #define fnclex()                __asm __volatile("fnclex")
   71 #define fninit()                __asm __volatile("fninit")
   72 #define fnstcw(addr)            __asm __volatile("fnstcw %0" : "=m" (*(addr)))
   73 #define fnstsw(addr)            __asm __volatile("fnstsw %0" : "=am" (*(addr)))
   74 #define fxrstor(addr)           __asm __volatile("fxrstor %0" : : "m" (*(addr)))
   75 #define fxsave(addr)            __asm __volatile("fxsave %0" : "=m" (*(addr)))
   76 #define ldmxcsr(csr)            __asm __volatile("ldmxcsr %0" : : "m" (csr))
   77 #define stmxcsr(addr)           __asm __volatile("stmxcsr %0" : : "m" (*(addr)))
   78 
   79 static __inline void
   80 xrstor(char *addr, uint64_t mask)
   81 {
   82         uint32_t low, hi;
   83 
   84         low = mask;
   85         hi = mask >> 32;
   86         __asm __volatile("xrstor %0" : : "m" (*addr), "a" (low), "d" (hi));
   87 }
   88 
   89 static __inline void
   90 xsave(char *addr, uint64_t mask)
   91 {
   92         uint32_t low, hi;
   93 
   94         low = mask;
   95         hi = mask >> 32;
   96         __asm __volatile("xsave %0" : "=m" (*addr) : "a" (low), "d" (hi) :
   97             "memory");
   98 }
   99 
  100 #else   /* !(__GNUCLIKE_ASM && !lint) */
  101 
  102 void    fldcw(u_short cw);
  103 void    fnclex(void);
  104 void    fninit(void);
  105 void    fnstcw(caddr_t addr);
  106 void    fnstsw(caddr_t addr);
  107 void    fxsave(caddr_t addr);
  108 void    fxrstor(caddr_t addr);
  109 void    ldmxcsr(u_int csr);
  110 void    stmxcsr(u_int *csr);
  111 void    xrstor(char *addr, uint64_t mask);
  112 void    xsave(char *addr, uint64_t mask);
  113 
  114 #endif  /* __GNUCLIKE_ASM && !lint */
  115 
  116 #define start_emulating()       load_cr0(rcr0() | CR0_TS)
  117 #define stop_emulating()        clts()
  118 
  119 CTASSERT(sizeof(struct savefpu) == 512);
  120 CTASSERT(sizeof(struct xstate_hdr) == 64);
  121 CTASSERT(sizeof(struct savefpu_ymm) == 832);
  122 
  123 /*
  124  * This requirement is to make it easier for asm code to calculate
  125  * offset of the fpu save area from the pcb address. FPU save area
  126  * must be 64-byte aligned.
  127  */
  128 CTASSERT(sizeof(struct pcb) % XSAVE_AREA_ALIGN == 0);
  129 
  130 /*
  131  * Ensure the copy of XCR0 saved in a core is contained in the padding
  132  * area.
  133  */
  134 CTASSERT(X86_XSTATE_XCR0_OFFSET >= offsetof(struct savefpu, sv_pad) &&
  135     X86_XSTATE_XCR0_OFFSET + sizeof(uint64_t) <= sizeof(struct savefpu));
  136 
  137 static  void    fpu_clean_state(void);
  138 
  139 SYSCTL_INT(_hw, HW_FLOATINGPT, floatingpoint, CTLFLAG_RD,
  140     SYSCTL_NULL_INT_PTR, 1, "Floating point instructions executed in hardware");
  141 
  142 int lazy_fpu_switch = 0;
  143 SYSCTL_INT(_hw, OID_AUTO, lazy_fpu_switch, CTLFLAG_RWTUN | CTLFLAG_NOFETCH,
  144     &lazy_fpu_switch, 0,
  145     "Lazily load FPU context after context switch");
  146 
  147 int use_xsave;                  /* non-static for cpu_switch.S */
  148 uint64_t xsave_mask;            /* the same */
  149 static  uma_zone_t fpu_save_area_zone;
  150 static  struct savefpu *fpu_initialstate;
  151 
  152 struct xsave_area_elm_descr {
  153         u_int   offset;
  154         u_int   size;
  155 } *xsave_area_desc;
  156 
  157 void
  158 fpusave(void *addr)
  159 {
  160 
  161         if (use_xsave)
  162                 xsave((char *)addr, xsave_mask);
  163         else
  164                 fxsave((char *)addr);
  165 }
  166 
  167 void
  168 fpurestore(void *addr)
  169 {
  170 
  171         if (use_xsave)
  172                 xrstor((char *)addr, xsave_mask);
  173         else
  174                 fxrstor((char *)addr);
  175 }
  176 
  177 void
  178 fpususpend(void *addr)
  179 {
  180         u_long cr0;
  181 
  182         cr0 = rcr0();
  183         stop_emulating();
  184         fpusave(addr);
  185         load_cr0(cr0);
  186 }
  187 
  188 void
  189 fpuresume(void *addr)
  190 {
  191         u_long cr0;
  192 
  193         cr0 = rcr0();
  194         stop_emulating();
  195         fninit();
  196         if (use_xsave)
  197                 load_xcr(XCR0, xsave_mask);
  198         fpurestore(addr);
  199         load_cr0(cr0);
  200 }
  201 
  202 /*
  203  * Enable XSAVE if supported and allowed by user.
  204  * Calculate the xsave_mask.
  205  */
  206 static void
  207 fpuinit_bsp1(void)
  208 {
  209         u_int cp[4];
  210         uint64_t xsave_mask_user;
  211 
  212         TUNABLE_INT_FETCH("hw.lazy_fpu_switch", &lazy_fpu_switch);
  213         if ((cpu_feature2 & CPUID2_XSAVE) != 0) {
  214                 use_xsave = 1;
  215                 TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave);
  216         }
  217         if (!use_xsave)
  218                 return;
  219 
  220         cpuid_count(0xd, 0x0, cp);
  221         xsave_mask = XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE;
  222         if ((cp[0] & xsave_mask) != xsave_mask)
  223                 panic("CPU0 does not support X87 or SSE: %x", cp[0]);
  224         xsave_mask = ((uint64_t)cp[3] << 32) | cp[0];
  225         xsave_mask_user = xsave_mask;
  226         TUNABLE_ULONG_FETCH("hw.xsave_mask", &xsave_mask_user);
  227         xsave_mask_user |= XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE;
  228         xsave_mask &= xsave_mask_user;
  229         if ((xsave_mask & XFEATURE_AVX512) != XFEATURE_AVX512)
  230                 xsave_mask &= ~XFEATURE_AVX512;
  231         if ((xsave_mask & XFEATURE_MPX) != XFEATURE_MPX)
  232                 xsave_mask &= ~XFEATURE_MPX;
  233 
  234         cpuid_count(0xd, 0x1, cp);
  235         if ((cp[0] & CPUID_EXTSTATE_XSAVEOPT) != 0) {
  236                 /*
  237                  * Patch the XSAVE instruction in the cpu_switch code
  238                  * to XSAVEOPT.  We assume that XSAVE encoding used
  239                  * REX byte, and set the bit 4 of the r/m byte.
  240                  */
  241                 ctx_switch_xsave[3] |= 0x10;
  242         }
  243 }
  244 
  245 /*
  246  * Calculate the fpu save area size.
  247  */
  248 static void
  249 fpuinit_bsp2(void)
  250 {
  251         u_int cp[4];
  252 
  253         if (use_xsave) {
  254                 cpuid_count(0xd, 0x0, cp);
  255                 cpu_max_ext_state_size = cp[1];
  256 
  257                 /*
  258                  * Reload the cpu_feature2, since we enabled OSXSAVE.
  259                  */
  260                 do_cpuid(1, cp);
  261                 cpu_feature2 = cp[2];
  262         } else
  263                 cpu_max_ext_state_size = sizeof(struct savefpu);
  264 }
  265 
  266 /*
  267  * Initialize the floating point unit.
  268  */
  269 void
  270 fpuinit(void)
  271 {
  272         register_t saveintr;
  273         u_int mxcsr;
  274         u_short control;
  275 
  276         if (IS_BSP())
  277                 fpuinit_bsp1();
  278 
  279         if (use_xsave) {
  280                 load_cr4(rcr4() | CR4_XSAVE);
  281                 load_xcr(XCR0, xsave_mask);
  282         }
  283 
  284         /*
  285          * XCR0 shall be set up before CPU can report the save area size.
  286          */
  287         if (IS_BSP())
  288                 fpuinit_bsp2();
  289 
  290         /*
  291          * It is too early for critical_enter() to work on AP.
  292          */
  293         saveintr = intr_disable();
  294         stop_emulating();
  295         fninit();
  296         control = __INITIAL_FPUCW__;
  297         fldcw(control);
  298         mxcsr = __INITIAL_MXCSR__;
  299         ldmxcsr(mxcsr);
  300         start_emulating();
  301         intr_restore(saveintr);
  302 }
  303 
  304 /*
  305  * On the boot CPU we generate a clean state that is used to
  306  * initialize the floating point unit when it is first used by a
  307  * process.
  308  */
  309 static void
  310 fpuinitstate(void *arg __unused)
  311 {
  312         register_t saveintr;
  313         int cp[4], i, max_ext_n;
  314 
  315         fpu_initialstate = malloc(cpu_max_ext_state_size, M_DEVBUF,
  316             M_WAITOK | M_ZERO);
  317         saveintr = intr_disable();
  318         stop_emulating();
  319 
  320         fpusave(fpu_initialstate);
  321         if (fpu_initialstate->sv_env.en_mxcsr_mask)
  322                 cpu_mxcsr_mask = fpu_initialstate->sv_env.en_mxcsr_mask;
  323         else
  324                 cpu_mxcsr_mask = 0xFFBF;
  325 
  326         /*
  327          * The fninit instruction does not modify XMM registers or x87
  328          * registers (MM/ST).  The fpusave call dumped the garbage
  329          * contained in the registers after reset to the initial state
  330          * saved.  Clear XMM and x87 registers file image to make the
  331          * startup program state and signal handler XMM/x87 register
  332          * content predictable.
  333          */
  334         bzero(fpu_initialstate->sv_fp, sizeof(fpu_initialstate->sv_fp));
  335         bzero(fpu_initialstate->sv_xmm, sizeof(fpu_initialstate->sv_xmm));
  336 
  337         /*
  338          * Create a table describing the layout of the CPU Extended
  339          * Save Area.
  340          */
  341         if (use_xsave) {
  342                 max_ext_n = flsl(xsave_mask);
  343                 xsave_area_desc = malloc(max_ext_n * sizeof(struct
  344                     xsave_area_elm_descr), M_DEVBUF, M_WAITOK | M_ZERO);
  345                 /* x87 state */
  346                 xsave_area_desc[0].offset = 0;
  347                 xsave_area_desc[0].size = 160;
  348                 /* XMM */
  349                 xsave_area_desc[1].offset = 160;
  350                 xsave_area_desc[1].size = 288 - 160;
  351 
  352                 for (i = 2; i < max_ext_n; i++) {
  353                         cpuid_count(0xd, i, cp);
  354                         xsave_area_desc[i].offset = cp[1];
  355                         xsave_area_desc[i].size = cp[0];
  356                 }
  357         }
  358 
  359         fpu_save_area_zone = uma_zcreate("FPU_save_area",
  360             cpu_max_ext_state_size, NULL, NULL, NULL, NULL,
  361             XSAVE_AREA_ALIGN - 1, 0);
  362 
  363         start_emulating();
  364         intr_restore(saveintr);
  365 }
  366 SYSINIT(fpuinitstate, SI_SUB_DRIVERS, SI_ORDER_ANY, fpuinitstate, NULL);
  367 
  368 /*
  369  * Free coprocessor (if we have it).
  370  */
  371 void
  372 fpuexit(struct thread *td)
  373 {
  374 
  375         critical_enter();
  376         if (curthread == PCPU_GET(fpcurthread)) {
  377                 stop_emulating();
  378                 fpusave(curpcb->pcb_save);
  379                 start_emulating();
  380                 PCPU_SET(fpcurthread, NULL);
  381         }
  382         critical_exit();
  383 }
  384 
  385 int
  386 fpuformat(void)
  387 {
  388 
  389         return (_MC_FPFMT_XMM);
  390 }
  391 
  392 /* 
  393  * The following mechanism is used to ensure that the FPE_... value
  394  * that is passed as a trapcode to the signal handler of the user
  395  * process does not have more than one bit set.
  396  * 
  397  * Multiple bits may be set if the user process modifies the control
  398  * word while a status word bit is already set.  While this is a sign
  399  * of bad coding, we have no choise than to narrow them down to one
  400  * bit, since we must not send a trapcode that is not exactly one of
  401  * the FPE_ macros.
  402  *
  403  * The mechanism has a static table with 127 entries.  Each combination
  404  * of the 7 FPU status word exception bits directly translates to a
  405  * position in this table, where a single FPE_... value is stored.
  406  * This FPE_... value stored there is considered the "most important"
  407  * of the exception bits and will be sent as the signal code.  The
  408  * precedence of the bits is based upon Intel Document "Numerical
  409  * Applications", Chapter "Special Computational Situations".
  410  *
  411  * The macro to choose one of these values does these steps: 1) Throw
  412  * away status word bits that cannot be masked.  2) Throw away the bits
  413  * currently masked in the control word, assuming the user isn't
  414  * interested in them anymore.  3) Reinsert status word bit 7 (stack
  415  * fault) if it is set, which cannot be masked but must be presered.
  416  * 4) Use the remaining bits to point into the trapcode table.
  417  *
  418  * The 6 maskable bits in order of their preference, as stated in the
  419  * above referenced Intel manual:
  420  * 1  Invalid operation (FP_X_INV)
  421  * 1a   Stack underflow
  422  * 1b   Stack overflow
  423  * 1c   Operand of unsupported format
  424  * 1d   SNaN operand.
  425  * 2  QNaN operand (not an exception, irrelavant here)
  426  * 3  Any other invalid-operation not mentioned above or zero divide
  427  *      (FP_X_INV, FP_X_DZ)
  428  * 4  Denormal operand (FP_X_DNML)
  429  * 5  Numeric over/underflow (FP_X_OFL, FP_X_UFL)
  430  * 6  Inexact result (FP_X_IMP) 
  431  */
  432 static char fpetable[128] = {
  433         0,
  434         FPE_FLTINV,     /*  1 - INV */
  435         FPE_FLTUND,     /*  2 - DNML */
  436         FPE_FLTINV,     /*  3 - INV | DNML */
  437         FPE_FLTDIV,     /*  4 - DZ */
  438         FPE_FLTINV,     /*  5 - INV | DZ */
  439         FPE_FLTDIV,     /*  6 - DNML | DZ */
  440         FPE_FLTINV,     /*  7 - INV | DNML | DZ */
  441         FPE_FLTOVF,     /*  8 - OFL */
  442         FPE_FLTINV,     /*  9 - INV | OFL */
  443         FPE_FLTUND,     /*  A - DNML | OFL */
  444         FPE_FLTINV,     /*  B - INV | DNML | OFL */
  445         FPE_FLTDIV,     /*  C - DZ | OFL */
  446         FPE_FLTINV,     /*  D - INV | DZ | OFL */
  447         FPE_FLTDIV,     /*  E - DNML | DZ | OFL */
  448         FPE_FLTINV,     /*  F - INV | DNML | DZ | OFL */
  449         FPE_FLTUND,     /* 10 - UFL */
  450         FPE_FLTINV,     /* 11 - INV | UFL */
  451         FPE_FLTUND,     /* 12 - DNML | UFL */
  452         FPE_FLTINV,     /* 13 - INV | DNML | UFL */
  453         FPE_FLTDIV,     /* 14 - DZ | UFL */
  454         FPE_FLTINV,     /* 15 - INV | DZ | UFL */
  455         FPE_FLTDIV,     /* 16 - DNML | DZ | UFL */
  456         FPE_FLTINV,     /* 17 - INV | DNML | DZ | UFL */
  457         FPE_FLTOVF,     /* 18 - OFL | UFL */
  458         FPE_FLTINV,     /* 19 - INV | OFL | UFL */
  459         FPE_FLTUND,     /* 1A - DNML | OFL | UFL */
  460         FPE_FLTINV,     /* 1B - INV | DNML | OFL | UFL */
  461         FPE_FLTDIV,     /* 1C - DZ | OFL | UFL */
  462         FPE_FLTINV,     /* 1D - INV | DZ | OFL | UFL */
  463         FPE_FLTDIV,     /* 1E - DNML | DZ | OFL | UFL */
  464         FPE_FLTINV,     /* 1F - INV | DNML | DZ | OFL | UFL */
  465         FPE_FLTRES,     /* 20 - IMP */
  466         FPE_FLTINV,     /* 21 - INV | IMP */
  467         FPE_FLTUND,     /* 22 - DNML | IMP */
  468         FPE_FLTINV,     /* 23 - INV | DNML | IMP */
  469         FPE_FLTDIV,     /* 24 - DZ | IMP */
  470         FPE_FLTINV,     /* 25 - INV | DZ | IMP */
  471         FPE_FLTDIV,     /* 26 - DNML | DZ | IMP */
  472         FPE_FLTINV,     /* 27 - INV | DNML | DZ | IMP */
  473         FPE_FLTOVF,     /* 28 - OFL | IMP */
  474         FPE_FLTINV,     /* 29 - INV | OFL | IMP */
  475         FPE_FLTUND,     /* 2A - DNML | OFL | IMP */
  476         FPE_FLTINV,     /* 2B - INV | DNML | OFL | IMP */
  477         FPE_FLTDIV,     /* 2C - DZ | OFL | IMP */
  478         FPE_FLTINV,     /* 2D - INV | DZ | OFL | IMP */
  479         FPE_FLTDIV,     /* 2E - DNML | DZ | OFL | IMP */
  480         FPE_FLTINV,     /* 2F - INV | DNML | DZ | OFL | IMP */
  481         FPE_FLTUND,     /* 30 - UFL | IMP */
  482         FPE_FLTINV,     /* 31 - INV | UFL | IMP */
  483         FPE_FLTUND,     /* 32 - DNML | UFL | IMP */
  484         FPE_FLTINV,     /* 33 - INV | DNML | UFL | IMP */
  485         FPE_FLTDIV,     /* 34 - DZ | UFL | IMP */
  486         FPE_FLTINV,     /* 35 - INV | DZ | UFL | IMP */
  487         FPE_FLTDIV,     /* 36 - DNML | DZ | UFL | IMP */
  488         FPE_FLTINV,     /* 37 - INV | DNML | DZ | UFL | IMP */
  489         FPE_FLTOVF,     /* 38 - OFL | UFL | IMP */
  490         FPE_FLTINV,     /* 39 - INV | OFL | UFL | IMP */
  491         FPE_FLTUND,     /* 3A - DNML | OFL | UFL | IMP */
  492         FPE_FLTINV,     /* 3B - INV | DNML | OFL | UFL | IMP */
  493         FPE_FLTDIV,     /* 3C - DZ | OFL | UFL | IMP */
  494         FPE_FLTINV,     /* 3D - INV | DZ | OFL | UFL | IMP */
  495         FPE_FLTDIV,     /* 3E - DNML | DZ | OFL | UFL | IMP */
  496         FPE_FLTINV,     /* 3F - INV | DNML | DZ | OFL | UFL | IMP */
  497         FPE_FLTSUB,     /* 40 - STK */
  498         FPE_FLTSUB,     /* 41 - INV | STK */
  499         FPE_FLTUND,     /* 42 - DNML | STK */
  500         FPE_FLTSUB,     /* 43 - INV | DNML | STK */
  501         FPE_FLTDIV,     /* 44 - DZ | STK */
  502         FPE_FLTSUB,     /* 45 - INV | DZ | STK */
  503         FPE_FLTDIV,     /* 46 - DNML | DZ | STK */
  504         FPE_FLTSUB,     /* 47 - INV | DNML | DZ | STK */
  505         FPE_FLTOVF,     /* 48 - OFL | STK */
  506         FPE_FLTSUB,     /* 49 - INV | OFL | STK */
  507         FPE_FLTUND,     /* 4A - DNML | OFL | STK */
  508         FPE_FLTSUB,     /* 4B - INV | DNML | OFL | STK */
  509         FPE_FLTDIV,     /* 4C - DZ | OFL | STK */
  510         FPE_FLTSUB,     /* 4D - INV | DZ | OFL | STK */
  511         FPE_FLTDIV,     /* 4E - DNML | DZ | OFL | STK */
  512         FPE_FLTSUB,     /* 4F - INV | DNML | DZ | OFL | STK */
  513         FPE_FLTUND,     /* 50 - UFL | STK */
  514         FPE_FLTSUB,     /* 51 - INV | UFL | STK */
  515         FPE_FLTUND,     /* 52 - DNML | UFL | STK */
  516         FPE_FLTSUB,     /* 53 - INV | DNML | UFL | STK */
  517         FPE_FLTDIV,     /* 54 - DZ | UFL | STK */
  518         FPE_FLTSUB,     /* 55 - INV | DZ | UFL | STK */
  519         FPE_FLTDIV,     /* 56 - DNML | DZ | UFL | STK */
  520         FPE_FLTSUB,     /* 57 - INV | DNML | DZ | UFL | STK */
  521         FPE_FLTOVF,     /* 58 - OFL | UFL | STK */
  522         FPE_FLTSUB,     /* 59 - INV | OFL | UFL | STK */
  523         FPE_FLTUND,     /* 5A - DNML | OFL | UFL | STK */
  524         FPE_FLTSUB,     /* 5B - INV | DNML | OFL | UFL | STK */
  525         FPE_FLTDIV,     /* 5C - DZ | OFL | UFL | STK */
  526         FPE_FLTSUB,     /* 5D - INV | DZ | OFL | UFL | STK */
  527         FPE_FLTDIV,     /* 5E - DNML | DZ | OFL | UFL | STK */
  528         FPE_FLTSUB,     /* 5F - INV | DNML | DZ | OFL | UFL | STK */
  529         FPE_FLTRES,     /* 60 - IMP | STK */
  530         FPE_FLTSUB,     /* 61 - INV | IMP | STK */
  531         FPE_FLTUND,     /* 62 - DNML | IMP | STK */
  532         FPE_FLTSUB,     /* 63 - INV | DNML | IMP | STK */
  533         FPE_FLTDIV,     /* 64 - DZ | IMP | STK */
  534         FPE_FLTSUB,     /* 65 - INV | DZ | IMP | STK */
  535         FPE_FLTDIV,     /* 66 - DNML | DZ | IMP | STK */
  536         FPE_FLTSUB,     /* 67 - INV | DNML | DZ | IMP | STK */
  537         FPE_FLTOVF,     /* 68 - OFL | IMP | STK */
  538         FPE_FLTSUB,     /* 69 - INV | OFL | IMP | STK */
  539         FPE_FLTUND,     /* 6A - DNML | OFL | IMP | STK */
  540         FPE_FLTSUB,     /* 6B - INV | DNML | OFL | IMP | STK */
  541         FPE_FLTDIV,     /* 6C - DZ | OFL | IMP | STK */
  542         FPE_FLTSUB,     /* 6D - INV | DZ | OFL | IMP | STK */
  543         FPE_FLTDIV,     /* 6E - DNML | DZ | OFL | IMP | STK */
  544         FPE_FLTSUB,     /* 6F - INV | DNML | DZ | OFL | IMP | STK */
  545         FPE_FLTUND,     /* 70 - UFL | IMP | STK */
  546         FPE_FLTSUB,     /* 71 - INV | UFL | IMP | STK */
  547         FPE_FLTUND,     /* 72 - DNML | UFL | IMP | STK */
  548         FPE_FLTSUB,     /* 73 - INV | DNML | UFL | IMP | STK */
  549         FPE_FLTDIV,     /* 74 - DZ | UFL | IMP | STK */
  550         FPE_FLTSUB,     /* 75 - INV | DZ | UFL | IMP | STK */
  551         FPE_FLTDIV,     /* 76 - DNML | DZ | UFL | IMP | STK */
  552         FPE_FLTSUB,     /* 77 - INV | DNML | DZ | UFL | IMP | STK */
  553         FPE_FLTOVF,     /* 78 - OFL | UFL | IMP | STK */
  554         FPE_FLTSUB,     /* 79 - INV | OFL | UFL | IMP | STK */
  555         FPE_FLTUND,     /* 7A - DNML | OFL | UFL | IMP | STK */
  556         FPE_FLTSUB,     /* 7B - INV | DNML | OFL | UFL | IMP | STK */
  557         FPE_FLTDIV,     /* 7C - DZ | OFL | UFL | IMP | STK */
  558         FPE_FLTSUB,     /* 7D - INV | DZ | OFL | UFL | IMP | STK */
  559         FPE_FLTDIV,     /* 7E - DNML | DZ | OFL | UFL | IMP | STK */
  560         FPE_FLTSUB,     /* 7F - INV | DNML | DZ | OFL | UFL | IMP | STK */
  561 };
  562 
  563 /*
  564  * Read the FP status and control words, then generate si_code value
  565  * for SIGFPE.  The error code chosen will be one of the
  566  * FPE_... macros.  It will be sent as the second argument to old
  567  * BSD-style signal handlers and as "siginfo_t->si_code" (second
  568  * argument) to SA_SIGINFO signal handlers.
  569  *
  570  * Some time ago, we cleared the x87 exceptions with FNCLEX there.
  571  * Clearing exceptions was necessary mainly to avoid IRQ13 bugs.  The
  572  * usermode code which understands the FPU hardware enough to enable
  573  * the exceptions, can also handle clearing the exception state in the
  574  * handler.  The only consequence of not clearing the exception is the
  575  * rethrow of the SIGFPE on return from the signal handler and
  576  * reexecution of the corresponding instruction.
  577  *
  578  * For XMM traps, the exceptions were never cleared.
  579  */
  580 int
  581 fputrap_x87(void)
  582 {
  583         struct savefpu *pcb_save;
  584         u_short control, status;
  585 
  586         critical_enter();
  587 
  588         /*
  589          * Interrupt handling (for another interrupt) may have pushed the
  590          * state to memory.  Fetch the relevant parts of the state from
  591          * wherever they are.
  592          */
  593         if (PCPU_GET(fpcurthread) != curthread) {
  594                 pcb_save = curpcb->pcb_save;
  595                 control = pcb_save->sv_env.en_cw;
  596                 status = pcb_save->sv_env.en_sw;
  597         } else {
  598                 fnstcw(&control);
  599                 fnstsw(&status);
  600         }
  601 
  602         critical_exit();
  603         return (fpetable[status & ((~control & 0x3f) | 0x40)]);
  604 }
  605 
  606 int
  607 fputrap_sse(void)
  608 {
  609         u_int mxcsr;
  610 
  611         critical_enter();
  612         if (PCPU_GET(fpcurthread) != curthread)
  613                 mxcsr = curpcb->pcb_save->sv_env.en_mxcsr;
  614         else
  615                 stmxcsr(&mxcsr);
  616         critical_exit();
  617         return (fpetable[(mxcsr & (~mxcsr >> 7)) & 0x3f]);
  618 }
  619 
  620 static void
  621 restore_fpu_curthread(struct thread *td)
  622 {
  623         struct pcb *pcb;
  624 
  625         /*
  626          * Record new context early in case frstor causes a trap.
  627          */
  628         PCPU_SET(fpcurthread, td);
  629 
  630         stop_emulating();
  631         fpu_clean_state();
  632         pcb = td->td_pcb;
  633 
  634         if ((pcb->pcb_flags & PCB_FPUINITDONE) == 0) {
  635                 /*
  636                  * This is the first time this thread has used the FPU or
  637                  * the PCB doesn't contain a clean FPU state.  Explicitly
  638                  * load an initial state.
  639                  *
  640                  * We prefer to restore the state from the actual save
  641                  * area in PCB instead of directly loading from
  642                  * fpu_initialstate, to ignite the XSAVEOPT
  643                  * tracking engine.
  644                  */
  645                 bcopy(fpu_initialstate, pcb->pcb_save,
  646                     cpu_max_ext_state_size);
  647                 fpurestore(pcb->pcb_save);
  648                 if (pcb->pcb_initial_fpucw != __INITIAL_FPUCW__)
  649                         fldcw(pcb->pcb_initial_fpucw);
  650                 if (PCB_USER_FPU(pcb))
  651                         set_pcb_flags(pcb, PCB_FPUINITDONE |
  652                             PCB_USERFPUINITDONE);
  653                 else
  654                         set_pcb_flags(pcb, PCB_FPUINITDONE);
  655         } else
  656                 fpurestore(pcb->pcb_save);
  657 }
  658 
  659 /*
  660  * Device Not Available (DNA, #NM) exception handler.
  661  *
  662  * It would be better to switch FP context here (if curthread !=
  663  * fpcurthread) and not necessarily for every context switch, but it
  664  * is too hard to access foreign pcb's.
  665  */
  666 void
  667 fpudna(void)
  668 {
  669         struct thread *td;
  670 
  671         td = curthread;
  672         /*
  673          * This handler is entered with interrupts enabled, so context
  674          * switches may occur before critical_enter() is executed.  If
  675          * a context switch occurs, then when we regain control, our
  676          * state will have been completely restored.  The CPU may
  677          * change underneath us, but the only part of our context that
  678          * lives in the CPU is CR0.TS and that will be "restored" by
  679          * setting it on the new CPU.
  680          */
  681         critical_enter();
  682 
  683         if (__predict_false(PCPU_GET(fpcurthread) == td)) {
  684                 /*
  685                  * Some virtual machines seems to set %cr0.TS at
  686                  * arbitrary moments.  Silently clear the TS bit
  687                  * regardless of the eager/lazy FPU context switch
  688                  * mode.
  689                  */
  690                 stop_emulating();
  691         } else {
  692                 if (__predict_false(PCPU_GET(fpcurthread) != NULL)) {
  693                         panic(
  694                     "fpudna: fpcurthread = %p (%d), curthread = %p (%d)\n",
  695                             PCPU_GET(fpcurthread),
  696                             PCPU_GET(fpcurthread)->td_tid, td, td->td_tid);
  697                 }
  698                 restore_fpu_curthread(td);
  699         }
  700         critical_exit();
  701 }
  702 
  703 void fpu_activate_sw(struct thread *td); /* Called from the context switch */
  704 void
  705 fpu_activate_sw(struct thread *td)
  706 {
  707 
  708         if (lazy_fpu_switch || (td->td_pflags & TDP_KTHREAD) != 0 ||
  709             !PCB_USER_FPU(td->td_pcb)) {
  710                 PCPU_SET(fpcurthread, NULL);
  711                 start_emulating();
  712         } else if (PCPU_GET(fpcurthread) != td) {
  713                 restore_fpu_curthread(td);
  714         }
  715 }
  716 
  717 void
  718 fpudrop(void)
  719 {
  720         struct thread *td;
  721 
  722         td = PCPU_GET(fpcurthread);
  723         KASSERT(td == curthread, ("fpudrop: fpcurthread != curthread"));
  724         CRITICAL_ASSERT(td);
  725         PCPU_SET(fpcurthread, NULL);
  726         clear_pcb_flags(td->td_pcb, PCB_FPUINITDONE);
  727         start_emulating();
  728 }
  729 
  730 /*
  731  * Get the user state of the FPU into pcb->pcb_user_save without
  732  * dropping ownership (if possible).  It returns the FPU ownership
  733  * status.
  734  */
  735 int
  736 fpugetregs(struct thread *td)
  737 {
  738         struct pcb *pcb;
  739         uint64_t *xstate_bv, bit;
  740         char *sa;
  741         int max_ext_n, i, owned;
  742 
  743         pcb = td->td_pcb;
  744         critical_enter();
  745         if ((pcb->pcb_flags & PCB_USERFPUINITDONE) == 0) {
  746                 bcopy(fpu_initialstate, get_pcb_user_save_pcb(pcb),
  747                     cpu_max_ext_state_size);
  748                 get_pcb_user_save_pcb(pcb)->sv_env.en_cw =
  749                     pcb->pcb_initial_fpucw;
  750                 fpuuserinited(td);
  751                 critical_exit();
  752                 return (_MC_FPOWNED_PCB);
  753         }
  754         if (td == PCPU_GET(fpcurthread) && PCB_USER_FPU(pcb)) {
  755                 fpusave(get_pcb_user_save_pcb(pcb));
  756                 owned = _MC_FPOWNED_FPU;
  757         } else {
  758                 owned = _MC_FPOWNED_PCB;
  759         }
  760         if (use_xsave) {
  761                 /*
  762                  * Handle partially saved state.
  763                  */
  764                 sa = (char *)get_pcb_user_save_pcb(pcb);
  765                 xstate_bv = (uint64_t *)(sa + sizeof(struct savefpu) +
  766                     offsetof(struct xstate_hdr, xstate_bv));
  767                 max_ext_n = flsl(xsave_mask);
  768                 for (i = 0; i < max_ext_n; i++) {
  769                         bit = 1ULL << i;
  770                         if ((xsave_mask & bit) == 0 || (*xstate_bv & bit) != 0)
  771                                 continue;
  772                         bcopy((char *)fpu_initialstate +
  773                             xsave_area_desc[i].offset,
  774                             sa + xsave_area_desc[i].offset,
  775                             xsave_area_desc[i].size);
  776                         *xstate_bv |= bit;
  777                 }
  778         }
  779         critical_exit();
  780         return (owned);
  781 }
  782 
  783 void
  784 fpuuserinited(struct thread *td)
  785 {
  786         struct pcb *pcb;
  787 
  788         CRITICAL_ASSERT(td);
  789         pcb = td->td_pcb;
  790         if (PCB_USER_FPU(pcb))
  791                 set_pcb_flags(pcb,
  792                     PCB_FPUINITDONE | PCB_USERFPUINITDONE);
  793         else
  794                 set_pcb_flags(pcb, PCB_FPUINITDONE);
  795 }
  796 
  797 int
  798 fpusetxstate(struct thread *td, char *xfpustate, size_t xfpustate_size)
  799 {
  800         struct xstate_hdr *hdr, *ehdr;
  801         size_t len, max_len;
  802         uint64_t bv;
  803 
  804         /* XXXKIB should we clear all extended state in xstate_bv instead ? */
  805         if (xfpustate == NULL)
  806                 return (0);
  807         if (!use_xsave)
  808                 return (EOPNOTSUPP);
  809 
  810         len = xfpustate_size;
  811         if (len < sizeof(struct xstate_hdr))
  812                 return (EINVAL);
  813         max_len = cpu_max_ext_state_size - sizeof(struct savefpu);
  814         if (len > max_len)
  815                 return (EINVAL);
  816 
  817         ehdr = (struct xstate_hdr *)xfpustate;
  818         bv = ehdr->xstate_bv;
  819 
  820         /*
  821          * Avoid #gp.
  822          */
  823         if (bv & ~xsave_mask)
  824                 return (EINVAL);
  825 
  826         hdr = (struct xstate_hdr *)(get_pcb_user_save_td(td) + 1);
  827 
  828         hdr->xstate_bv = bv;
  829         bcopy(xfpustate + sizeof(struct xstate_hdr),
  830             (char *)(hdr + 1), len - sizeof(struct xstate_hdr));
  831 
  832         return (0);
  833 }
  834 
  835 /*
  836  * Set the state of the FPU.
  837  */
  838 int
  839 fpusetregs(struct thread *td, struct savefpu *addr, char *xfpustate,
  840     size_t xfpustate_size)
  841 {
  842         struct pcb *pcb;
  843         int error;
  844 
  845         addr->sv_env.en_mxcsr &= cpu_mxcsr_mask;
  846         pcb = td->td_pcb;
  847         error = 0;
  848         critical_enter();
  849         if (td == PCPU_GET(fpcurthread) && PCB_USER_FPU(pcb)) {
  850                 error = fpusetxstate(td, xfpustate, xfpustate_size);
  851                 if (error == 0) {
  852                         bcopy(addr, get_pcb_user_save_td(td), sizeof(*addr));
  853                         fpurestore(get_pcb_user_save_td(td));
  854                         set_pcb_flags(pcb, PCB_FPUINITDONE |
  855                             PCB_USERFPUINITDONE);
  856                 }
  857         } else {
  858                 error = fpusetxstate(td, xfpustate, xfpustate_size);
  859                 if (error == 0) {
  860                         bcopy(addr, get_pcb_user_save_td(td), sizeof(*addr));
  861                         fpuuserinited(td);
  862                 }
  863         }
  864         critical_exit();
  865         return (error);
  866 }
  867 
  868 /*
  869  * On AuthenticAMD processors, the fxrstor instruction does not restore
  870  * the x87's stored last instruction pointer, last data pointer, and last
  871  * opcode values, except in the rare case in which the exception summary
  872  * (ES) bit in the x87 status word is set to 1.
  873  *
  874  * In order to avoid leaking this information across processes, we clean
  875  * these values by performing a dummy load before executing fxrstor().
  876  */
  877 static void
  878 fpu_clean_state(void)
  879 {
  880         static float dummy_variable = 0.0;
  881         u_short status;
  882 
  883         /*
  884          * Clear the ES bit in the x87 status word if it is currently
  885          * set, in order to avoid causing a fault in the upcoming load.
  886          */
  887         fnstsw(&status);
  888         if (status & 0x80)
  889                 fnclex();
  890 
  891         /*
  892          * Load the dummy variable into the x87 stack.  This mangles
  893          * the x87 stack, but we don't care since we're about to call
  894          * fxrstor() anyway.
  895          */
  896         __asm __volatile("ffree %%st(7); flds %0" : : "m" (dummy_variable));
  897 }
  898 
  899 /*
  900  * This really sucks.  We want the acpi version only, but it requires
  901  * the isa_if.h file in order to get the definitions.
  902  */
  903 #include "opt_isa.h"
  904 #ifdef DEV_ISA
  905 #include <isa/isavar.h>
  906 /*
  907  * This sucks up the legacy ISA support assignments from PNPBIOS/ACPI.
  908  */
  909 static struct isa_pnp_id fpupnp_ids[] = {
  910         { 0x040cd041, "Legacy ISA coprocessor support" }, /* PNP0C04 */
  911         { 0 }
  912 };
  913 
  914 static int
  915 fpupnp_probe(device_t dev)
  916 {
  917         int result;
  918 
  919         result = ISA_PNP_PROBE(device_get_parent(dev), dev, fpupnp_ids);
  920         if (result <= 0)
  921                 device_quiet(dev);
  922         return (result);
  923 }
  924 
  925 static int
  926 fpupnp_attach(device_t dev)
  927 {
  928 
  929         return (0);
  930 }
  931 
  932 static device_method_t fpupnp_methods[] = {
  933         /* Device interface */
  934         DEVMETHOD(device_probe,         fpupnp_probe),
  935         DEVMETHOD(device_attach,        fpupnp_attach),
  936         DEVMETHOD(device_detach,        bus_generic_detach),
  937         DEVMETHOD(device_shutdown,      bus_generic_shutdown),
  938         DEVMETHOD(device_suspend,       bus_generic_suspend),
  939         DEVMETHOD(device_resume,        bus_generic_resume),
  940         
  941         { 0, 0 }
  942 };
  943 
  944 static driver_t fpupnp_driver = {
  945         "fpupnp",
  946         fpupnp_methods,
  947         1,                      /* no softc */
  948 };
  949 
  950 static devclass_t fpupnp_devclass;
  951 
  952 DRIVER_MODULE(fpupnp, acpi, fpupnp_driver, fpupnp_devclass, 0, 0);
  953 #endif  /* DEV_ISA */
  954 
  955 static MALLOC_DEFINE(M_FPUKERN_CTX, "fpukern_ctx",
  956     "Kernel contexts for FPU state");
  957 
  958 #define FPU_KERN_CTX_FPUINITDONE 0x01
  959 #define FPU_KERN_CTX_DUMMY       0x02   /* avoided save for the kern thread */
  960 
  961 struct fpu_kern_ctx {
  962         struct savefpu *prev;
  963         uint32_t flags;
  964         char hwstate1[];
  965 };
  966 
  967 struct fpu_kern_ctx *
  968 fpu_kern_alloc_ctx(u_int flags)
  969 {
  970         struct fpu_kern_ctx *res;
  971         size_t sz;
  972 
  973         sz = sizeof(struct fpu_kern_ctx) + XSAVE_AREA_ALIGN +
  974             cpu_max_ext_state_size;
  975         res = malloc(sz, M_FPUKERN_CTX, ((flags & FPU_KERN_NOWAIT) ?
  976             M_NOWAIT : M_WAITOK) | M_ZERO);
  977         return (res);
  978 }
  979 
  980 void
  981 fpu_kern_free_ctx(struct fpu_kern_ctx *ctx)
  982 {
  983 
  984         /* XXXKIB clear the memory ? */
  985         free(ctx, M_FPUKERN_CTX);
  986 }
  987 
  988 static struct savefpu *
  989 fpu_kern_ctx_savefpu(struct fpu_kern_ctx *ctx)
  990 {
  991         vm_offset_t p;
  992 
  993         p = (vm_offset_t)&ctx->hwstate1;
  994         p = roundup2(p, XSAVE_AREA_ALIGN);
  995         return ((struct savefpu *)p);
  996 }
  997 
  998 int
  999 fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags)
 1000 {
 1001         struct pcb *pcb;
 1002 
 1003         if ((flags & FPU_KERN_KTHR) != 0 && is_fpu_kern_thread(0)) {
 1004                 ctx->flags = FPU_KERN_CTX_DUMMY;
 1005                 return (0);
 1006         }
 1007         pcb = td->td_pcb;
 1008         critical_enter();
 1009         KASSERT(!PCB_USER_FPU(pcb) || pcb->pcb_save ==
 1010             get_pcb_user_save_pcb(pcb), ("mangled pcb_save"));
 1011         ctx->flags = 0;
 1012         if ((pcb->pcb_flags & PCB_FPUINITDONE) != 0)
 1013                 ctx->flags |= FPU_KERN_CTX_FPUINITDONE;
 1014         fpuexit(td);
 1015         ctx->prev = pcb->pcb_save;
 1016         pcb->pcb_save = fpu_kern_ctx_savefpu(ctx);
 1017         set_pcb_flags(pcb, PCB_KERNFPU);
 1018         clear_pcb_flags(pcb, PCB_FPUINITDONE);
 1019         critical_exit();
 1020         return (0);
 1021 }
 1022 
 1023 int
 1024 fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx)
 1025 {
 1026         struct pcb *pcb;
 1027 
 1028         if (is_fpu_kern_thread(0) && (ctx->flags & FPU_KERN_CTX_DUMMY) != 0)
 1029                 return (0);
 1030         KASSERT((ctx->flags & FPU_KERN_CTX_DUMMY) == 0, ("dummy ctx"));
 1031         pcb = td->td_pcb;
 1032         critical_enter();
 1033         if (curthread == PCPU_GET(fpcurthread))
 1034                 fpudrop();
 1035         pcb->pcb_save = ctx->prev;
 1036         if (pcb->pcb_save == get_pcb_user_save_pcb(pcb)) {
 1037                 if ((pcb->pcb_flags & PCB_USERFPUINITDONE) != 0) {
 1038                         set_pcb_flags(pcb, PCB_FPUINITDONE);
 1039                         clear_pcb_flags(pcb, PCB_KERNFPU);
 1040                 } else
 1041                         clear_pcb_flags(pcb, PCB_FPUINITDONE | PCB_KERNFPU);
 1042         } else {
 1043                 if ((ctx->flags & FPU_KERN_CTX_FPUINITDONE) != 0)
 1044                         set_pcb_flags(pcb, PCB_FPUINITDONE);
 1045                 else
 1046                         clear_pcb_flags(pcb, PCB_FPUINITDONE);
 1047                 KASSERT(!PCB_USER_FPU(pcb), ("unpaired fpu_kern_leave"));
 1048         }
 1049         critical_exit();
 1050         return (0);
 1051 }
 1052 
 1053 int
 1054 fpu_kern_thread(u_int flags)
 1055 {
 1056 
 1057         KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0,
 1058             ("Only kthread may use fpu_kern_thread"));
 1059         KASSERT(curpcb->pcb_save == get_pcb_user_save_pcb(curpcb),
 1060             ("mangled pcb_save"));
 1061         KASSERT(PCB_USER_FPU(curpcb), ("recursive call"));
 1062 
 1063         set_pcb_flags(curpcb, PCB_KERNFPU);
 1064         return (0);
 1065 }
 1066 
 1067 int
 1068 is_fpu_kern_thread(u_int flags)
 1069 {
 1070 
 1071         if ((curthread->td_pflags & TDP_KTHREAD) == 0)
 1072                 return (0);
 1073         return ((curpcb->pcb_flags & PCB_KERNFPU) != 0);
 1074 }
 1075 
 1076 /*
 1077  * FPU save area alloc/free/init utility routines
 1078  */
 1079 struct savefpu *
 1080 fpu_save_area_alloc(void)
 1081 {
 1082 
 1083         return (uma_zalloc(fpu_save_area_zone, 0));
 1084 }
 1085 
 1086 void
 1087 fpu_save_area_free(struct savefpu *fsa)
 1088 {
 1089 
 1090         uma_zfree(fpu_save_area_zone, fsa);
 1091 }
 1092 
 1093 void
 1094 fpu_save_area_reset(struct savefpu *fsa)
 1095 {
 1096 
 1097         bcopy(fpu_initialstate, fsa, cpu_max_ext_state_size);
 1098 }

Cache object: bcc964c197ba61f6c015368113d9b6e4


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.