The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/i386/i386/npx.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1990 William Jolitz.
    3  * Copyright (c) 1991 The Regents of the University of California.
    4  * All rights reserved.
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice, this list of conditions and the following disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  * 3. Neither the name of the University nor the names of its contributors
   15  *    may be used to endorse or promote products derived from this software
   16  *    without specific prior written permission.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  *
   30  *      from: @(#)npx.c 7.2 (Berkeley) 5/12/91
   31  */
   32 
   33 #include <sys/cdefs.h>
   34 __FBSDID("$FreeBSD$");
   35 
   36 #include "opt_cpu.h"
   37 #include "opt_isa.h"
   38 #include "opt_npx.h"
   39 
   40 #include <sys/param.h>
   41 #include <sys/systm.h>
   42 #include <sys/bus.h>
   43 #include <sys/kernel.h>
   44 #include <sys/lock.h>
   45 #include <sys/malloc.h>
   46 #include <sys/module.h>
   47 #include <sys/mutex.h>
   48 #include <sys/mutex.h>
   49 #include <sys/proc.h>
   50 #include <sys/smp.h>
   51 #include <sys/sysctl.h>
   52 #include <machine/bus.h>
   53 #include <sys/rman.h>
   54 #ifdef NPX_DEBUG
   55 #include <sys/syslog.h>
   56 #endif
   57 #include <sys/signalvar.h>
   58 #include <vm/uma.h>
   59 
   60 #include <machine/asmacros.h>
   61 #include <machine/cputypes.h>
   62 #include <machine/frame.h>
   63 #include <machine/md_var.h>
   64 #include <machine/pcb.h>
   65 #include <machine/psl.h>
   66 #include <machine/resource.h>
   67 #include <machine/specialreg.h>
   68 #include <machine/segments.h>
   69 #include <machine/ucontext.h>
   70 #include <x86/ifunc.h>
   71 
   72 #include <machine/intr_machdep.h>
   73 
   74 #ifdef DEV_ISA
   75 #include <isa/isavar.h>
   76 #endif
   77 
   78 /*
   79  * 387 and 287 Numeric Coprocessor Extension (NPX) Driver.
   80  */
   81 
   82 #if defined(__GNUCLIKE_ASM) && !defined(lint)
   83 
   84 #define fldcw(cw)               __asm __volatile("fldcw %0" : : "m" (cw))
   85 #define fnclex()                __asm __volatile("fnclex")
   86 #define fninit()                __asm __volatile("fninit")
   87 #define fnsave(addr)            __asm __volatile("fnsave %0" : "=m" (*(addr)))
   88 #define fnstcw(addr)            __asm __volatile("fnstcw %0" : "=m" (*(addr)))
   89 #define fnstsw(addr)            __asm __volatile("fnstsw %0" : "=am" (*(addr)))
   90 #define fp_divide_by_0()        __asm __volatile( \
   91                                     "fldz; fld1; fdiv %st,%st(1); fnop")
   92 #define frstor(addr)            __asm __volatile("frstor %0" : : "m" (*(addr)))
   93 #define fxrstor(addr)           __asm __volatile("fxrstor %0" : : "m" (*(addr)))
   94 #define fxsave(addr)            __asm __volatile("fxsave %0" : "=m" (*(addr)))
   95 #define ldmxcsr(csr)            __asm __volatile("ldmxcsr %0" : : "m" (csr))
   96 #define stmxcsr(addr)           __asm __volatile("stmxcsr %0" : : "m" (*(addr)))
   97 
   98 static __inline void
   99 xrstor(char *addr, uint64_t mask)
  100 {
  101         uint32_t low, hi;
  102 
  103         low = mask;
  104         hi = mask >> 32;
  105         __asm __volatile("xrstor %0" : : "m" (*addr), "a" (low), "d" (hi));
  106 }
  107 
  108 static __inline void
  109 xsave(char *addr, uint64_t mask)
  110 {
  111         uint32_t low, hi;
  112 
  113         low = mask;
  114         hi = mask >> 32;
  115         __asm __volatile("xsave %0" : "=m" (*addr) : "a" (low), "d" (hi) :
  116             "memory");
  117 }
  118 
  119 static __inline void
  120 xsaveopt(char *addr, uint64_t mask)
  121 {
  122         uint32_t low, hi;
  123 
  124         low = mask;
  125         hi = mask >> 32;
  126         __asm __volatile("xsaveopt %0" : "=m" (*addr) : "a" (low), "d" (hi) :
  127             "memory");
  128 }
  129 #else   /* !(__GNUCLIKE_ASM && !lint) */
  130 
  131 void    fldcw(u_short cw);
  132 void    fnclex(void);
  133 void    fninit(void);
  134 void    fnsave(caddr_t addr);
  135 void    fnstcw(caddr_t addr);
  136 void    fnstsw(caddr_t addr);
  137 void    fp_divide_by_0(void);
  138 void    frstor(caddr_t addr);
  139 void    fxsave(caddr_t addr);
  140 void    fxrstor(caddr_t addr);
  141 void    ldmxcsr(u_int csr);
  142 void    stmxcsr(u_int *csr);
  143 void    xrstor(char *addr, uint64_t mask);
  144 void    xsave(char *addr, uint64_t mask);
  145 void    xsaveopt(char *addr, uint64_t mask);
  146 
  147 #endif  /* __GNUCLIKE_ASM && !lint */
  148 
  149 #define start_emulating()       load_cr0(rcr0() | CR0_TS)
  150 #define stop_emulating()        clts()
  151 
  152 #define GET_FPU_CW(thread) \
  153         (cpu_fxsr ? \
  154                 (thread)->td_pcb->pcb_save->sv_xmm.sv_env.en_cw : \
  155                 (thread)->td_pcb->pcb_save->sv_87.sv_env.en_cw)
  156 #define GET_FPU_SW(thread) \
  157         (cpu_fxsr ? \
  158                 (thread)->td_pcb->pcb_save->sv_xmm.sv_env.en_sw : \
  159                 (thread)->td_pcb->pcb_save->sv_87.sv_env.en_sw)
  160 #define SET_FPU_CW(savefpu, value) do { \
  161         if (cpu_fxsr) \
  162                 (savefpu)->sv_xmm.sv_env.en_cw = (value); \
  163         else \
  164                 (savefpu)->sv_87.sv_env.en_cw = (value); \
  165 } while (0)
  166 
  167 CTASSERT(sizeof(union savefpu) == 512);
  168 CTASSERT(sizeof(struct xstate_hdr) == 64);
  169 CTASSERT(sizeof(struct savefpu_ymm) == 832);
  170 
  171 /*
  172  * This requirement is to make it easier for asm code to calculate
  173  * offset of the fpu save area from the pcb address. FPU save area
  174  * must be 64-byte aligned.
  175  */
  176 CTASSERT(sizeof(struct pcb) % XSAVE_AREA_ALIGN == 0);
  177 
  178 /*
  179  * Ensure the copy of XCR0 saved in a core is contained in the padding
  180  * area.
  181  */
  182 CTASSERT(X86_XSTATE_XCR0_OFFSET >= offsetof(struct savexmm, sv_pad) &&
  183     X86_XSTATE_XCR0_OFFSET + sizeof(uint64_t) <= sizeof(struct savexmm));
  184 
  185 static  void    fpu_clean_state(void);
  186 
  187 static  void    fpurstor(union savefpu *);
  188 
  189 int     hw_float;
  190 
  191 SYSCTL_INT(_hw, HW_FLOATINGPT, floatingpoint, CTLFLAG_RD,
  192     &hw_float, 0, "Floating point instructions executed in hardware");
  193 
  194 int lazy_fpu_switch = 0;
  195 SYSCTL_INT(_hw, OID_AUTO, lazy_fpu_switch, CTLFLAG_RWTUN | CTLFLAG_NOFETCH,
  196     &lazy_fpu_switch, 0,
  197     "Lazily load FPU context after context switch");
  198 
  199 int use_xsave;
  200 uint64_t xsave_mask;
  201 static  uma_zone_t fpu_save_area_zone;
  202 static  union savefpu *npx_initialstate;
  203 
  204 struct xsave_area_elm_descr {
  205         u_int   offset;
  206         u_int   size;
  207 } *xsave_area_desc;
  208 
  209 static  volatile u_int          npx_traps_while_probing;
  210 
  211 alias_for_inthand_t probetrap;
  212 __asm("                                                         \n\
  213         .text                                                   \n\
  214         .p2align 2,0x90                                         \n\
  215         .type   " __XSTRING(CNAME(probetrap)) ",@function       \n\
  216 " __XSTRING(CNAME(probetrap)) ":                                \n\
  217         ss                                                      \n\
  218         incl    " __XSTRING(CNAME(npx_traps_while_probing)) "   \n\
  219         fnclex                                                  \n\
  220         iret                                                    \n\
  221 ");
  222 
  223 /*
  224  * Determine if an FPU is present and how to use it.
  225  */
  226 static int
  227 npx_probe(void)
  228 {
  229         struct gate_descriptor save_idt_npxtrap;
  230         u_short control, status;
  231 
  232         /*
  233          * Modern CPUs all have an FPU that uses the INT16 interface
  234          * and provide a simple way to verify that, so handle the
  235          * common case right away.
  236          */
  237         if (cpu_feature & CPUID_FPU) {
  238                 hw_float = 1;
  239                 return (1);
  240         }
  241 
  242         save_idt_npxtrap = idt[IDT_MF];
  243         setidt(IDT_MF, probetrap, SDT_SYS386TGT, SEL_KPL,
  244             GSEL(GCODE_SEL, SEL_KPL));
  245 
  246         /*
  247          * Don't trap while we're probing.
  248          */
  249         stop_emulating();
  250 
  251         /*
  252          * Finish resetting the coprocessor, if any.  If there is an error
  253          * pending, then we may get a bogus IRQ13, but npx_intr() will handle
  254          * it OK.  Bogus halts have never been observed, but we enabled
  255          * IRQ13 and cleared the BUSY# latch early to handle them anyway.
  256          */
  257         fninit();
  258 
  259         /*
  260          * Don't use fwait here because it might hang.
  261          * Don't use fnop here because it usually hangs if there is no FPU.
  262          */
  263         DELAY(1000);            /* wait for any IRQ13 */
  264 #ifdef DIAGNOSTIC
  265         if (npx_traps_while_probing != 0)
  266                 printf("fninit caused %u bogus npx trap(s)\n",
  267                        npx_traps_while_probing);
  268 #endif
  269         /*
  270          * Check for a status of mostly zero.
  271          */
  272         status = 0x5a5a;
  273         fnstsw(&status);
  274         if ((status & 0xb8ff) == 0) {
  275                 /*
  276                  * Good, now check for a proper control word.
  277                  */
  278                 control = 0x5a5a;
  279                 fnstcw(&control);
  280                 if ((control & 0x1f3f) == 0x033f) {
  281                         /*
  282                          * We have an npx, now divide by 0 to see if exception
  283                          * 16 works.
  284                          */
  285                         control &= ~(1 << 2);   /* enable divide by 0 trap */
  286                         fldcw(control);
  287                         npx_traps_while_probing = 0;
  288                         fp_divide_by_0();
  289                         if (npx_traps_while_probing != 0) {
  290                                 /*
  291                                  * Good, exception 16 works.
  292                                  */
  293                                 hw_float = 1;
  294                                 goto cleanup;
  295                         }
  296                         printf(
  297         "FPU does not use exception 16 for error reporting\n");
  298                         goto cleanup;
  299                 }
  300         }
  301 
  302         /*
  303          * Probe failed.  Floating point simply won't work.
  304          * Notify user and disable FPU/MMX/SSE instruction execution.
  305          */
  306         printf("WARNING: no FPU!\n");
  307         __asm __volatile("smsw %%ax; orb %0,%%al; lmsw %%ax" : :
  308             "n" (CR0_EM | CR0_MP) : "ax");
  309 
  310 cleanup:
  311         idt[IDT_MF] = save_idt_npxtrap;
  312         return (hw_float);
  313 }
  314 
  315 static void
  316 fpusave_xsaveopt(union savefpu *addr)
  317 {
  318 
  319         xsaveopt((char *)addr, xsave_mask);
  320 }
  321 
  322 static void
  323 fpusave_xsave(union savefpu *addr)
  324 {
  325 
  326         xsave((char *)addr, xsave_mask);
  327 }
  328 
  329 static void
  330 fpusave_fxsave(union savefpu *addr)
  331 {
  332 
  333         fxsave((char *)addr);
  334 }
  335 
  336 static void
  337 fpusave_fnsave(union savefpu *addr)
  338 {
  339 
  340         fnsave((char *)addr);
  341 }
  342 
  343 static void
  344 init_xsave(void)
  345 {
  346 
  347         if (use_xsave)
  348                 return;
  349         if (!cpu_fxsr || (cpu_feature2 & CPUID2_XSAVE) == 0)
  350                 return;
  351         use_xsave = 1;
  352         TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave);
  353 }
  354 
  355 DEFINE_IFUNC(, void, fpusave, (union savefpu *), static)
  356 {
  357 
  358         init_xsave();
  359         if (use_xsave)
  360                 return ((cpu_stdext_feature & CPUID_EXTSTATE_XSAVEOPT) != 0 ?
  361                     fpusave_xsaveopt : fpusave_xsave);
  362         if (cpu_fxsr)
  363                 return (fpusave_fxsave);
  364         return (fpusave_fnsave);
  365 }
  366 
  367 /*
  368  * Enable XSAVE if supported and allowed by user.
  369  * Calculate the xsave_mask.
  370  */
  371 static void
  372 npxinit_bsp1(void)
  373 {
  374         u_int cp[4];
  375         uint64_t xsave_mask_user;
  376 
  377         TUNABLE_INT_FETCH("hw.lazy_fpu_switch", &lazy_fpu_switch);
  378         if (!use_xsave)
  379                 return;
  380         cpuid_count(0xd, 0x0, cp);
  381         xsave_mask = XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE;
  382         if ((cp[0] & xsave_mask) != xsave_mask)
  383                 panic("CPU0 does not support X87 or SSE: %x", cp[0]);
  384         xsave_mask = ((uint64_t)cp[3] << 32) | cp[0];
  385         xsave_mask_user = xsave_mask;
  386         TUNABLE_QUAD_FETCH("hw.xsave_mask", &xsave_mask_user);
  387         xsave_mask_user |= XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE;
  388         xsave_mask &= xsave_mask_user;
  389         if ((xsave_mask & XFEATURE_AVX512) != XFEATURE_AVX512)
  390                 xsave_mask &= ~XFEATURE_AVX512;
  391         if ((xsave_mask & XFEATURE_MPX) != XFEATURE_MPX)
  392                 xsave_mask &= ~XFEATURE_MPX;
  393 }
  394 
  395 /*
  396  * Calculate the fpu save area size.
  397  */
  398 static void
  399 npxinit_bsp2(void)
  400 {
  401         u_int cp[4];
  402 
  403         if (use_xsave) {
  404                 cpuid_count(0xd, 0x0, cp);
  405                 cpu_max_ext_state_size = cp[1];
  406 
  407                 /*
  408                  * Reload the cpu_feature2, since we enabled OSXSAVE.
  409                  */
  410                 do_cpuid(1, cp);
  411                 cpu_feature2 = cp[2];
  412         } else
  413                 cpu_max_ext_state_size = sizeof(union savefpu);
  414 }
  415 
  416 /*
  417  * Initialize floating point unit.
  418  */
  419 void
  420 npxinit(bool bsp)
  421 {
  422         static union savefpu dummy;
  423         register_t saveintr;
  424         u_int mxcsr;
  425         u_short control;
  426 
  427         if (bsp) {
  428                 if (!npx_probe())
  429                         return;
  430                 npxinit_bsp1();
  431         }
  432 
  433         if (use_xsave) {
  434                 load_cr4(rcr4() | CR4_XSAVE);
  435                 load_xcr(XCR0, xsave_mask);
  436         }
  437 
  438         /*
  439          * XCR0 shall be set up before CPU can report the save area size.
  440          */
  441         if (bsp)
  442                 npxinit_bsp2();
  443         
  444         /*
  445          * fninit has the same h/w bugs as fnsave.  Use the detoxified
  446          * fnsave to throw away any junk in the fpu.  fpusave() initializes
  447          * the fpu.
  448          *
  449          * It is too early for critical_enter() to work on AP.
  450          */
  451         saveintr = intr_disable();
  452         stop_emulating();
  453         if (cpu_fxsr)
  454                 fninit();
  455         else
  456                 fnsave(&dummy);
  457         control = __INITIAL_NPXCW__;
  458         fldcw(control);
  459         if (cpu_fxsr) {
  460                 mxcsr = __INITIAL_MXCSR__;
  461                 ldmxcsr(mxcsr);
  462         }
  463         start_emulating();
  464         intr_restore(saveintr);
  465 }
  466 
  467 /*
  468  * On the boot CPU we generate a clean state that is used to
  469  * initialize the floating point unit when it is first used by a
  470  * process.
  471  */
  472 static void
  473 npxinitstate(void *arg __unused)
  474 {
  475         uint64_t *xstate_bv;
  476         register_t saveintr;
  477         int cp[4], i, max_ext_n;
  478 
  479         if (!hw_float)
  480                 return;
  481 
  482         npx_initialstate = malloc(cpu_max_ext_state_size, M_DEVBUF,
  483             M_WAITOK | M_ZERO);
  484         saveintr = intr_disable();
  485         stop_emulating();
  486 
  487         if (cpu_fxsr)
  488                 fpusave_fxsave(npx_initialstate);
  489         else
  490                 fpusave_fnsave(npx_initialstate);
  491         if (cpu_fxsr) {
  492                 if (npx_initialstate->sv_xmm.sv_env.en_mxcsr_mask)
  493                         cpu_mxcsr_mask = 
  494                             npx_initialstate->sv_xmm.sv_env.en_mxcsr_mask;
  495                 else
  496                         cpu_mxcsr_mask = 0xFFBF;
  497 
  498                 /*
  499                  * The fninit instruction does not modify XMM
  500                  * registers or x87 registers (MM/ST).  The fpusave
  501                  * call dumped the garbage contained in the registers
  502                  * after reset to the initial state saved.  Clear XMM
  503                  * and x87 registers file image to make the startup
  504                  * program state and signal handler XMM/x87 register
  505                  * content predictable.
  506                  */
  507                 bzero(npx_initialstate->sv_xmm.sv_fp,
  508                     sizeof(npx_initialstate->sv_xmm.sv_fp));
  509                 bzero(npx_initialstate->sv_xmm.sv_xmm,
  510                     sizeof(npx_initialstate->sv_xmm.sv_xmm));
  511 
  512         } else
  513                 bzero(npx_initialstate->sv_87.sv_ac,
  514                     sizeof(npx_initialstate->sv_87.sv_ac));
  515 
  516         /*
  517          * Create a table describing the layout of the CPU Extended
  518          * Save Area.  See Intel SDM rev. 075 Vol. 1 13.4.1 "Legacy
  519          * Region of an XSAVE Area" for the source of offsets/sizes.
  520          * Note that 32bit XSAVE does not use %xmm8-%xmm15, see
  521          * 10.5.1.2 and 13.5.2 "SSE State".
  522          */
  523         if (use_xsave) {
  524                 xstate_bv = (uint64_t *)((char *)(npx_initialstate + 1) +
  525                     offsetof(struct xstate_hdr, xstate_bv));
  526                 *xstate_bv = XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE;
  527 
  528                 if (xsave_mask >> 32 != 0)
  529                         max_ext_n = fls(xsave_mask >> 32) + 32;
  530                 else
  531                         max_ext_n = fls(xsave_mask);
  532                 xsave_area_desc = malloc(max_ext_n * sizeof(struct
  533                     xsave_area_elm_descr), M_DEVBUF, M_WAITOK | M_ZERO);
  534                 /* x87 state */
  535                 xsave_area_desc[0].offset = 0;
  536                 xsave_area_desc[0].size = 160;
  537                 /* XMM */
  538                 xsave_area_desc[1].offset = 160;
  539                 xsave_area_desc[1].size = 288 - 160;
  540 
  541                 for (i = 2; i < max_ext_n; i++) {
  542                         cpuid_count(0xd, i, cp);
  543                         xsave_area_desc[i].offset = cp[1];
  544                         xsave_area_desc[i].size = cp[0];
  545                 }
  546         }
  547 
  548         fpu_save_area_zone = uma_zcreate("FPU_save_area",
  549             cpu_max_ext_state_size, NULL, NULL, NULL, NULL,
  550             XSAVE_AREA_ALIGN - 1, 0);
  551 
  552         start_emulating();
  553         intr_restore(saveintr);
  554 }
  555 SYSINIT(npxinitstate, SI_SUB_DRIVERS, SI_ORDER_ANY, npxinitstate, NULL);
  556 
  557 /*
  558  * Free coprocessor (if we have it).
  559  */
  560 void
  561 npxexit(struct thread *td)
  562 {
  563 
  564         critical_enter();
  565         if (curthread == PCPU_GET(fpcurthread)) {
  566                 stop_emulating();
  567                 fpusave(curpcb->pcb_save);
  568                 start_emulating();
  569                 PCPU_SET(fpcurthread, NULL);
  570         }
  571         critical_exit();
  572 #ifdef NPX_DEBUG
  573         if (hw_float) {
  574                 u_int   masked_exceptions;
  575 
  576                 masked_exceptions = GET_FPU_CW(td) & GET_FPU_SW(td) & 0x7f;
  577                 /*
  578                  * Log exceptions that would have trapped with the old
  579                  * control word (overflow, divide by 0, and invalid operand).
  580                  */
  581                 if (masked_exceptions & 0x0d)
  582                         log(LOG_ERR,
  583         "pid %d (%s) exited with masked floating point exceptions 0x%02x\n",
  584                             td->td_proc->p_pid, td->td_proc->p_comm,
  585                             masked_exceptions);
  586         }
  587 #endif
  588 }
  589 
  590 int
  591 npxformat(void)
  592 {
  593 
  594         if (!hw_float)
  595                 return (_MC_FPFMT_NODEV);
  596         if (cpu_fxsr)
  597                 return (_MC_FPFMT_XMM);
  598         return (_MC_FPFMT_387);
  599 }
  600 
  601 /*
  602  * The following mechanism is used to ensure that the FPE_... value
  603  * that is passed as a trapcode to the signal handler of the user
  604  * process does not have more than one bit set.
  605  *
  606  * Multiple bits may be set if the user process modifies the control
  607  * word while a status word bit is already set.  While this is a sign
  608  * of bad coding, we have no choice than to narrow them down to one
  609  * bit, since we must not send a trapcode that is not exactly one of
  610  * the FPE_ macros.
  611  *
  612  * The mechanism has a static table with 127 entries.  Each combination
  613  * of the 7 FPU status word exception bits directly translates to a
  614  * position in this table, where a single FPE_... value is stored.
  615  * This FPE_... value stored there is considered the "most important"
  616  * of the exception bits and will be sent as the signal code.  The
  617  * precedence of the bits is based upon Intel Document "Numerical
  618  * Applications", Chapter "Special Computational Situations".
  619  *
  620  * The macro to choose one of these values does these steps: 1) Throw
  621  * away status word bits that cannot be masked.  2) Throw away the bits
  622  * currently masked in the control word, assuming the user isn't
  623  * interested in them anymore.  3) Reinsert status word bit 7 (stack
  624  * fault) if it is set, which cannot be masked but must be presered.
  625  * 4) Use the remaining bits to point into the trapcode table.
  626  *
  627  * The 6 maskable bits in order of their preference, as stated in the
  628  * above referenced Intel manual:
  629  * 1  Invalid operation (FP_X_INV)
  630  * 1a   Stack underflow
  631  * 1b   Stack overflow
  632  * 1c   Operand of unsupported format
  633  * 1d   SNaN operand.
  634  * 2  QNaN operand (not an exception, irrelavant here)
  635  * 3  Any other invalid-operation not mentioned above or zero divide
  636  *      (FP_X_INV, FP_X_DZ)
  637  * 4  Denormal operand (FP_X_DNML)
  638  * 5  Numeric over/underflow (FP_X_OFL, FP_X_UFL)
  639  * 6  Inexact result (FP_X_IMP) 
  640  */
  641 static char fpetable[128] = {
  642         0,
  643         FPE_FLTINV,     /*  1 - INV */
  644         FPE_FLTUND,     /*  2 - DNML */
  645         FPE_FLTINV,     /*  3 - INV | DNML */
  646         FPE_FLTDIV,     /*  4 - DZ */
  647         FPE_FLTINV,     /*  5 - INV | DZ */
  648         FPE_FLTDIV,     /*  6 - DNML | DZ */
  649         FPE_FLTINV,     /*  7 - INV | DNML | DZ */
  650         FPE_FLTOVF,     /*  8 - OFL */
  651         FPE_FLTINV,     /*  9 - INV | OFL */
  652         FPE_FLTUND,     /*  A - DNML | OFL */
  653         FPE_FLTINV,     /*  B - INV | DNML | OFL */
  654         FPE_FLTDIV,     /*  C - DZ | OFL */
  655         FPE_FLTINV,     /*  D - INV | DZ | OFL */
  656         FPE_FLTDIV,     /*  E - DNML | DZ | OFL */
  657         FPE_FLTINV,     /*  F - INV | DNML | DZ | OFL */
  658         FPE_FLTUND,     /* 10 - UFL */
  659         FPE_FLTINV,     /* 11 - INV | UFL */
  660         FPE_FLTUND,     /* 12 - DNML | UFL */
  661         FPE_FLTINV,     /* 13 - INV | DNML | UFL */
  662         FPE_FLTDIV,     /* 14 - DZ | UFL */
  663         FPE_FLTINV,     /* 15 - INV | DZ | UFL */
  664         FPE_FLTDIV,     /* 16 - DNML | DZ | UFL */
  665         FPE_FLTINV,     /* 17 - INV | DNML | DZ | UFL */
  666         FPE_FLTOVF,     /* 18 - OFL | UFL */
  667         FPE_FLTINV,     /* 19 - INV | OFL | UFL */
  668         FPE_FLTUND,     /* 1A - DNML | OFL | UFL */
  669         FPE_FLTINV,     /* 1B - INV | DNML | OFL | UFL */
  670         FPE_FLTDIV,     /* 1C - DZ | OFL | UFL */
  671         FPE_FLTINV,     /* 1D - INV | DZ | OFL | UFL */
  672         FPE_FLTDIV,     /* 1E - DNML | DZ | OFL | UFL */
  673         FPE_FLTINV,     /* 1F - INV | DNML | DZ | OFL | UFL */
  674         FPE_FLTRES,     /* 20 - IMP */
  675         FPE_FLTINV,     /* 21 - INV | IMP */
  676         FPE_FLTUND,     /* 22 - DNML | IMP */
  677         FPE_FLTINV,     /* 23 - INV | DNML | IMP */
  678         FPE_FLTDIV,     /* 24 - DZ | IMP */
  679         FPE_FLTINV,     /* 25 - INV | DZ | IMP */
  680         FPE_FLTDIV,     /* 26 - DNML | DZ | IMP */
  681         FPE_FLTINV,     /* 27 - INV | DNML | DZ | IMP */
  682         FPE_FLTOVF,     /* 28 - OFL | IMP */
  683         FPE_FLTINV,     /* 29 - INV | OFL | IMP */
  684         FPE_FLTUND,     /* 2A - DNML | OFL | IMP */
  685         FPE_FLTINV,     /* 2B - INV | DNML | OFL | IMP */
  686         FPE_FLTDIV,     /* 2C - DZ | OFL | IMP */
  687         FPE_FLTINV,     /* 2D - INV | DZ | OFL | IMP */
  688         FPE_FLTDIV,     /* 2E - DNML | DZ | OFL | IMP */
  689         FPE_FLTINV,     /* 2F - INV | DNML | DZ | OFL | IMP */
  690         FPE_FLTUND,     /* 30 - UFL | IMP */
  691         FPE_FLTINV,     /* 31 - INV | UFL | IMP */
  692         FPE_FLTUND,     /* 32 - DNML | UFL | IMP */
  693         FPE_FLTINV,     /* 33 - INV | DNML | UFL | IMP */
  694         FPE_FLTDIV,     /* 34 - DZ | UFL | IMP */
  695         FPE_FLTINV,     /* 35 - INV | DZ | UFL | IMP */
  696         FPE_FLTDIV,     /* 36 - DNML | DZ | UFL | IMP */
  697         FPE_FLTINV,     /* 37 - INV | DNML | DZ | UFL | IMP */
  698         FPE_FLTOVF,     /* 38 - OFL | UFL | IMP */
  699         FPE_FLTINV,     /* 39 - INV | OFL | UFL | IMP */
  700         FPE_FLTUND,     /* 3A - DNML | OFL | UFL | IMP */
  701         FPE_FLTINV,     /* 3B - INV | DNML | OFL | UFL | IMP */
  702         FPE_FLTDIV,     /* 3C - DZ | OFL | UFL | IMP */
  703         FPE_FLTINV,     /* 3D - INV | DZ | OFL | UFL | IMP */
  704         FPE_FLTDIV,     /* 3E - DNML | DZ | OFL | UFL | IMP */
  705         FPE_FLTINV,     /* 3F - INV | DNML | DZ | OFL | UFL | IMP */
  706         FPE_FLTSUB,     /* 40 - STK */
  707         FPE_FLTSUB,     /* 41 - INV | STK */
  708         FPE_FLTUND,     /* 42 - DNML | STK */
  709         FPE_FLTSUB,     /* 43 - INV | DNML | STK */
  710         FPE_FLTDIV,     /* 44 - DZ | STK */
  711         FPE_FLTSUB,     /* 45 - INV | DZ | STK */
  712         FPE_FLTDIV,     /* 46 - DNML | DZ | STK */
  713         FPE_FLTSUB,     /* 47 - INV | DNML | DZ | STK */
  714         FPE_FLTOVF,     /* 48 - OFL | STK */
  715         FPE_FLTSUB,     /* 49 - INV | OFL | STK */
  716         FPE_FLTUND,     /* 4A - DNML | OFL | STK */
  717         FPE_FLTSUB,     /* 4B - INV | DNML | OFL | STK */
  718         FPE_FLTDIV,     /* 4C - DZ | OFL | STK */
  719         FPE_FLTSUB,     /* 4D - INV | DZ | OFL | STK */
  720         FPE_FLTDIV,     /* 4E - DNML | DZ | OFL | STK */
  721         FPE_FLTSUB,     /* 4F - INV | DNML | DZ | OFL | STK */
  722         FPE_FLTUND,     /* 50 - UFL | STK */
  723         FPE_FLTSUB,     /* 51 - INV | UFL | STK */
  724         FPE_FLTUND,     /* 52 - DNML | UFL | STK */
  725         FPE_FLTSUB,     /* 53 - INV | DNML | UFL | STK */
  726         FPE_FLTDIV,     /* 54 - DZ | UFL | STK */
  727         FPE_FLTSUB,     /* 55 - INV | DZ | UFL | STK */
  728         FPE_FLTDIV,     /* 56 - DNML | DZ | UFL | STK */
  729         FPE_FLTSUB,     /* 57 - INV | DNML | DZ | UFL | STK */
  730         FPE_FLTOVF,     /* 58 - OFL | UFL | STK */
  731         FPE_FLTSUB,     /* 59 - INV | OFL | UFL | STK */
  732         FPE_FLTUND,     /* 5A - DNML | OFL | UFL | STK */
  733         FPE_FLTSUB,     /* 5B - INV | DNML | OFL | UFL | STK */
  734         FPE_FLTDIV,     /* 5C - DZ | OFL | UFL | STK */
  735         FPE_FLTSUB,     /* 5D - INV | DZ | OFL | UFL | STK */
  736         FPE_FLTDIV,     /* 5E - DNML | DZ | OFL | UFL | STK */
  737         FPE_FLTSUB,     /* 5F - INV | DNML | DZ | OFL | UFL | STK */
  738         FPE_FLTRES,     /* 60 - IMP | STK */
  739         FPE_FLTSUB,     /* 61 - INV | IMP | STK */
  740         FPE_FLTUND,     /* 62 - DNML | IMP | STK */
  741         FPE_FLTSUB,     /* 63 - INV | DNML | IMP | STK */
  742         FPE_FLTDIV,     /* 64 - DZ | IMP | STK */
  743         FPE_FLTSUB,     /* 65 - INV | DZ | IMP | STK */
  744         FPE_FLTDIV,     /* 66 - DNML | DZ | IMP | STK */
  745         FPE_FLTSUB,     /* 67 - INV | DNML | DZ | IMP | STK */
  746         FPE_FLTOVF,     /* 68 - OFL | IMP | STK */
  747         FPE_FLTSUB,     /* 69 - INV | OFL | IMP | STK */
  748         FPE_FLTUND,     /* 6A - DNML | OFL | IMP | STK */
  749         FPE_FLTSUB,     /* 6B - INV | DNML | OFL | IMP | STK */
  750         FPE_FLTDIV,     /* 6C - DZ | OFL | IMP | STK */
  751         FPE_FLTSUB,     /* 6D - INV | DZ | OFL | IMP | STK */
  752         FPE_FLTDIV,     /* 6E - DNML | DZ | OFL | IMP | STK */
  753         FPE_FLTSUB,     /* 6F - INV | DNML | DZ | OFL | IMP | STK */
  754         FPE_FLTUND,     /* 70 - UFL | IMP | STK */
  755         FPE_FLTSUB,     /* 71 - INV | UFL | IMP | STK */
  756         FPE_FLTUND,     /* 72 - DNML | UFL | IMP | STK */
  757         FPE_FLTSUB,     /* 73 - INV | DNML | UFL | IMP | STK */
  758         FPE_FLTDIV,     /* 74 - DZ | UFL | IMP | STK */
  759         FPE_FLTSUB,     /* 75 - INV | DZ | UFL | IMP | STK */
  760         FPE_FLTDIV,     /* 76 - DNML | DZ | UFL | IMP | STK */
  761         FPE_FLTSUB,     /* 77 - INV | DNML | DZ | UFL | IMP | STK */
  762         FPE_FLTOVF,     /* 78 - OFL | UFL | IMP | STK */
  763         FPE_FLTSUB,     /* 79 - INV | OFL | UFL | IMP | STK */
  764         FPE_FLTUND,     /* 7A - DNML | OFL | UFL | IMP | STK */
  765         FPE_FLTSUB,     /* 7B - INV | DNML | OFL | UFL | IMP | STK */
  766         FPE_FLTDIV,     /* 7C - DZ | OFL | UFL | IMP | STK */
  767         FPE_FLTSUB,     /* 7D - INV | DZ | OFL | UFL | IMP | STK */
  768         FPE_FLTDIV,     /* 7E - DNML | DZ | OFL | UFL | IMP | STK */
  769         FPE_FLTSUB,     /* 7F - INV | DNML | DZ | OFL | UFL | IMP | STK */
  770 };
  771 
  772 /*
  773  * Read the FP status and control words, then generate si_code value
  774  * for SIGFPE.  The error code chosen will be one of the
  775  * FPE_... macros.  It will be sent as the second argument to old
  776  * BSD-style signal handlers and as "siginfo_t->si_code" (second
  777  * argument) to SA_SIGINFO signal handlers.
  778  *
  779  * Some time ago, we cleared the x87 exceptions with FNCLEX there.
  780  * Clearing exceptions was necessary mainly to avoid IRQ13 bugs.  The
  781  * usermode code which understands the FPU hardware enough to enable
  782  * the exceptions, can also handle clearing the exception state in the
  783  * handler.  The only consequence of not clearing the exception is the
  784  * rethrow of the SIGFPE on return from the signal handler and
  785  * reexecution of the corresponding instruction.
  786  *
  787  * For XMM traps, the exceptions were never cleared.
  788  */
  789 int
  790 npxtrap_x87(void)
  791 {
  792         u_short control, status;
  793 
  794         if (!hw_float) {
  795                 printf(
  796         "npxtrap_x87: fpcurthread = %p, curthread = %p, hw_float = %d\n",
  797                        PCPU_GET(fpcurthread), curthread, hw_float);
  798                 panic("npxtrap from nowhere");
  799         }
  800         critical_enter();
  801 
  802         /*
  803          * Interrupt handling (for another interrupt) may have pushed the
  804          * state to memory.  Fetch the relevant parts of the state from
  805          * wherever they are.
  806          */
  807         if (PCPU_GET(fpcurthread) != curthread) {
  808                 control = GET_FPU_CW(curthread);
  809                 status = GET_FPU_SW(curthread);
  810         } else {
  811                 fnstcw(&control);
  812                 fnstsw(&status);
  813         }
  814         critical_exit();
  815         return (fpetable[status & ((~control & 0x3f) | 0x40)]);
  816 }
  817 
  818 int
  819 npxtrap_sse(void)
  820 {
  821         u_int mxcsr;
  822 
  823         if (!hw_float) {
  824                 printf(
  825         "npxtrap_sse: fpcurthread = %p, curthread = %p, hw_float = %d\n",
  826                        PCPU_GET(fpcurthread), curthread, hw_float);
  827                 panic("npxtrap from nowhere");
  828         }
  829         critical_enter();
  830         if (PCPU_GET(fpcurthread) != curthread)
  831                 mxcsr = curthread->td_pcb->pcb_save->sv_xmm.sv_env.en_mxcsr;
  832         else
  833                 stmxcsr(&mxcsr);
  834         critical_exit();
  835         return (fpetable[(mxcsr & (~mxcsr >> 7)) & 0x3f]);
  836 }
  837 
  838 static void
  839 restore_npx_curthread(struct thread *td, struct pcb *pcb)
  840 {
  841 
  842         /*
  843          * Record new context early in case frstor causes a trap.
  844          */
  845         PCPU_SET(fpcurthread, td);
  846 
  847         stop_emulating();
  848         if (cpu_fxsr)
  849                 fpu_clean_state();
  850 
  851         if ((pcb->pcb_flags & PCB_NPXINITDONE) == 0) {
  852                 /*
  853                  * This is the first time this thread has used the FPU or
  854                  * the PCB doesn't contain a clean FPU state.  Explicitly
  855                  * load an initial state.
  856                  *
  857                  * We prefer to restore the state from the actual save
  858                  * area in PCB instead of directly loading from
  859                  * npx_initialstate, to ignite the XSAVEOPT
  860                  * tracking engine.
  861                  */
  862                 bcopy(npx_initialstate, pcb->pcb_save, cpu_max_ext_state_size);
  863                 fpurstor(pcb->pcb_save);
  864                 if (pcb->pcb_initial_npxcw != __INITIAL_NPXCW__)
  865                         fldcw(pcb->pcb_initial_npxcw);
  866                 pcb->pcb_flags |= PCB_NPXINITDONE;
  867                 if (PCB_USER_FPU(pcb))
  868                         pcb->pcb_flags |= PCB_NPXUSERINITDONE;
  869         } else {
  870                 fpurstor(pcb->pcb_save);
  871         }
  872 }
  873 
  874 /*
  875  * Implement device not available (DNA) exception
  876  *
  877  * It would be better to switch FP context here (if curthread != fpcurthread)
  878  * and not necessarily for every context switch, but it is too hard to
  879  * access foreign pcb's.
  880  */
  881 int
  882 npxdna(void)
  883 {
  884         struct thread *td;
  885 
  886         if (!hw_float)
  887                 return (0);
  888         td = curthread;
  889         critical_enter();
  890         if (__predict_false(PCPU_GET(fpcurthread) == td)) {
  891                 /*
  892                  * Some virtual machines seems to set %cr0.TS at
  893                  * arbitrary moments.  Silently clear the TS bit
  894                  * regardless of the eager/lazy FPU context switch
  895                  * mode.
  896                  */
  897                 stop_emulating();
  898         } else {
  899                 if (__predict_false(PCPU_GET(fpcurthread) != NULL)) {
  900                         printf(
  901                     "npxdna: fpcurthread = %p (%d), curthread = %p (%d)\n",
  902                             PCPU_GET(fpcurthread),
  903                             PCPU_GET(fpcurthread)->td_proc->p_pid,
  904                             td, td->td_proc->p_pid);
  905                         panic("npxdna");
  906                 }
  907                 restore_npx_curthread(td, td->td_pcb);
  908         }
  909         critical_exit();
  910         return (1);
  911 }
  912 
  913 /*
  914  * Wrapper for fpusave() called from context switch routines.
  915  *
  916  * npxsave() must be called with interrupts disabled, so that it clears
  917  * fpcurthread atomically with saving the state.  We require callers to do the
  918  * disabling, since most callers need to disable interrupts anyway to call
  919  * npxsave() atomically with checking fpcurthread.
  920  */
  921 void
  922 npxsave(union savefpu *addr)
  923 {
  924 
  925         stop_emulating();
  926         fpusave(addr);
  927 }
  928 
  929 void npxswitch(struct thread *td, struct pcb *pcb);
  930 void
  931 npxswitch(struct thread *td, struct pcb *pcb)
  932 {
  933 
  934         if (lazy_fpu_switch || (td->td_pflags & TDP_KTHREAD) != 0 ||
  935             !PCB_USER_FPU(pcb)) {
  936                 start_emulating();
  937                 PCPU_SET(fpcurthread, NULL);
  938         } else if (PCPU_GET(fpcurthread) != td) {
  939                 restore_npx_curthread(td, pcb);
  940         }
  941 }
  942 
  943 /*
  944  * Unconditionally save the current co-processor state across suspend and
  945  * resume.
  946  */
  947 void
  948 npxsuspend(union savefpu *addr)
  949 {
  950         register_t cr0;
  951 
  952         if (!hw_float)
  953                 return;
  954         if (PCPU_GET(fpcurthread) == NULL) {
  955                 bcopy(npx_initialstate, addr, cpu_max_ext_state_size);
  956                 return;
  957         }
  958         cr0 = rcr0();
  959         stop_emulating();
  960         fpusave(addr);
  961         load_cr0(cr0);
  962 }
  963 
  964 void
  965 npxresume(union savefpu *addr)
  966 {
  967         register_t cr0;
  968 
  969         if (!hw_float)
  970                 return;
  971 
  972         cr0 = rcr0();
  973         npxinit(false);
  974         stop_emulating();
  975         fpurstor(addr);
  976         load_cr0(cr0);
  977 }
  978 
  979 void
  980 npxdrop(void)
  981 {
  982         struct thread *td;
  983 
  984         /*
  985          * Discard pending exceptions in the !cpu_fxsr case so that unmasked
  986          * ones don't cause a panic on the next frstor.
  987          */
  988         if (!cpu_fxsr)
  989                 fnclex();
  990 
  991         td = PCPU_GET(fpcurthread);
  992         KASSERT(td == curthread, ("fpudrop: fpcurthread != curthread"));
  993         CRITICAL_ASSERT(td);
  994         PCPU_SET(fpcurthread, NULL);
  995         td->td_pcb->pcb_flags &= ~PCB_NPXINITDONE;
  996         start_emulating();
  997 }
  998 
  999 /*
 1000  * Get the user state of the FPU into pcb->pcb_user_save without
 1001  * dropping ownership (if possible).  It returns the FPU ownership
 1002  * status.
 1003  */
 1004 int
 1005 npxgetregs(struct thread *td)
 1006 {
 1007         struct pcb *pcb;
 1008         uint64_t *xstate_bv, bit;
 1009         char *sa;
 1010         int max_ext_n, i;
 1011         int owned;
 1012 
 1013         if (!hw_float)
 1014                 return (_MC_FPOWNED_NONE);
 1015 
 1016         pcb = td->td_pcb;
 1017         critical_enter();
 1018         if ((pcb->pcb_flags & PCB_NPXINITDONE) == 0) {
 1019                 bcopy(npx_initialstate, get_pcb_user_save_pcb(pcb),
 1020                     cpu_max_ext_state_size);
 1021                 SET_FPU_CW(get_pcb_user_save_pcb(pcb), pcb->pcb_initial_npxcw);
 1022                 npxuserinited(td);
 1023                 critical_exit();
 1024                 return (_MC_FPOWNED_PCB);
 1025         }
 1026         if (td == PCPU_GET(fpcurthread)) {
 1027                 fpusave(get_pcb_user_save_pcb(pcb));
 1028                 if (!cpu_fxsr)
 1029                         /*
 1030                          * fnsave initializes the FPU and destroys whatever
 1031                          * context it contains.  Make sure the FPU owner
 1032                          * starts with a clean state next time.
 1033                          */
 1034                         npxdrop();
 1035                 owned = _MC_FPOWNED_FPU;
 1036         } else {
 1037                 owned = _MC_FPOWNED_PCB;
 1038         }
 1039         if (use_xsave) {
 1040                 /*
 1041                  * Handle partially saved state.
 1042                  */
 1043                 sa = (char *)get_pcb_user_save_pcb(pcb);
 1044                 xstate_bv = (uint64_t *)(sa + sizeof(union savefpu) +
 1045                     offsetof(struct xstate_hdr, xstate_bv));
 1046                 if (xsave_mask >> 32 != 0)
 1047                         max_ext_n = fls(xsave_mask >> 32) + 32;
 1048                 else
 1049                         max_ext_n = fls(xsave_mask);
 1050                 for (i = 0; i < max_ext_n; i++) {
 1051                         bit = 1ULL << i;
 1052                         if ((xsave_mask & bit) == 0 || (*xstate_bv & bit) != 0)
 1053                                 continue;
 1054                         bcopy((char *)npx_initialstate +
 1055                             xsave_area_desc[i].offset,
 1056                             sa + xsave_area_desc[i].offset,
 1057                             xsave_area_desc[i].size);
 1058                         *xstate_bv |= bit;
 1059                 }
 1060         }
 1061         critical_exit();
 1062         return (owned);
 1063 }
 1064 
 1065 void
 1066 npxuserinited(struct thread *td)
 1067 {
 1068         struct pcb *pcb;
 1069 
 1070         CRITICAL_ASSERT(td);
 1071         pcb = td->td_pcb;
 1072         if (PCB_USER_FPU(pcb))
 1073                 pcb->pcb_flags |= PCB_NPXINITDONE;
 1074         pcb->pcb_flags |= PCB_NPXUSERINITDONE;
 1075 }
 1076 
 1077 int
 1078 npxsetxstate(struct thread *td, char *xfpustate, size_t xfpustate_size)
 1079 {
 1080         struct xstate_hdr *hdr, *ehdr;
 1081         size_t len, max_len;
 1082         uint64_t bv;
 1083 
 1084         /* XXXKIB should we clear all extended state in xstate_bv instead ? */
 1085         if (xfpustate == NULL)
 1086                 return (0);
 1087         if (!use_xsave)
 1088                 return (EOPNOTSUPP);
 1089 
 1090         len = xfpustate_size;
 1091         if (len < sizeof(struct xstate_hdr))
 1092                 return (EINVAL);
 1093         max_len = cpu_max_ext_state_size - sizeof(union savefpu);
 1094         if (len > max_len)
 1095                 return (EINVAL);
 1096 
 1097         ehdr = (struct xstate_hdr *)xfpustate;
 1098         bv = ehdr->xstate_bv;
 1099 
 1100         /*
 1101          * Avoid #gp.
 1102          */
 1103         if (bv & ~xsave_mask)
 1104                 return (EINVAL);
 1105 
 1106         hdr = (struct xstate_hdr *)(get_pcb_user_save_td(td) + 1);
 1107 
 1108         hdr->xstate_bv = bv;
 1109         bcopy(xfpustate + sizeof(struct xstate_hdr),
 1110             (char *)(hdr + 1), len - sizeof(struct xstate_hdr));
 1111 
 1112         return (0);
 1113 }
 1114 
 1115 int
 1116 npxsetregs(struct thread *td, union savefpu *addr, char *xfpustate,
 1117         size_t xfpustate_size)
 1118 {
 1119         struct pcb *pcb;
 1120         int error;
 1121 
 1122         if (!hw_float)
 1123                 return (ENXIO);
 1124 
 1125         if (cpu_fxsr)
 1126                 addr->sv_xmm.sv_env.en_mxcsr &= cpu_mxcsr_mask;
 1127         pcb = td->td_pcb;
 1128         error = 0;
 1129         critical_enter();
 1130         if (td == PCPU_GET(fpcurthread) && PCB_USER_FPU(pcb)) {
 1131                 error = npxsetxstate(td, xfpustate, xfpustate_size);
 1132                 if (error == 0) {
 1133                         if (!cpu_fxsr)
 1134                                 fnclex();       /* As in npxdrop(). */
 1135                         bcopy(addr, get_pcb_user_save_td(td), sizeof(*addr));
 1136                         fpurstor(get_pcb_user_save_td(td));
 1137                         pcb->pcb_flags |= PCB_NPXUSERINITDONE | PCB_NPXINITDONE;
 1138                 }
 1139         } else {
 1140                 error = npxsetxstate(td, xfpustate, xfpustate_size);
 1141                 if (error == 0) {
 1142                         bcopy(addr, get_pcb_user_save_td(td), sizeof(*addr));
 1143                         npxuserinited(td);
 1144                 }
 1145         }
 1146         critical_exit();
 1147         return (error);
 1148 }
 1149 
 1150 static void
 1151 npx_fill_fpregs_xmm1(struct savexmm *sv_xmm, struct save87 *sv_87)
 1152 {
 1153         struct env87 *penv_87;
 1154         struct envxmm *penv_xmm;
 1155         struct fpacc87 *fx_reg;
 1156         int i, st;
 1157         uint64_t mantissa;
 1158         uint16_t tw, exp;
 1159         uint8_t ab_tw;
 1160 
 1161         penv_87 = &sv_87->sv_env;
 1162         penv_xmm = &sv_xmm->sv_env;
 1163 
 1164         /* FPU control/status */
 1165         penv_87->en_cw = penv_xmm->en_cw;
 1166         penv_87->en_sw = penv_xmm->en_sw;
 1167         penv_87->en_fip = penv_xmm->en_fip;
 1168         penv_87->en_fcs = penv_xmm->en_fcs;
 1169         penv_87->en_opcode = penv_xmm->en_opcode;
 1170         penv_87->en_foo = penv_xmm->en_foo;
 1171         penv_87->en_fos = penv_xmm->en_fos;
 1172 
 1173         /*
 1174          * FPU registers and tags.
 1175          * For ST(i), i = fpu_reg - top; we start with fpu_reg=7.
 1176          */
 1177         st = 7 - ((penv_xmm->en_sw >> 11) & 7);
 1178         ab_tw = penv_xmm->en_tw;
 1179         tw = 0;
 1180         for (i = 0x80; i != 0; i >>= 1) {
 1181                 sv_87->sv_ac[st] = sv_xmm->sv_fp[st].fp_acc;
 1182                 tw <<= 2;
 1183                 if (ab_tw & i) {
 1184                         /* Non-empty - we need to check ST(i) */
 1185                         fx_reg = &sv_xmm->sv_fp[st].fp_acc;
 1186                         /* The first 64 bits contain the mantissa. */
 1187                         mantissa = *((uint64_t *)fx_reg->fp_bytes);
 1188                         /*
 1189                          * The final 16 bits contain the sign bit and the exponent.
 1190                          * Mask the sign bit since it is of no consequence to these
 1191                          * tests.
 1192                          */
 1193                         exp = *((uint16_t *)&fx_reg->fp_bytes[8]) & 0x7fff;
 1194                         if (exp == 0) {
 1195                                 if (mantissa == 0)
 1196                                         tw |= 1; /* Zero */
 1197                                 else
 1198                                         tw |= 2; /* Denormal */
 1199                         } else if (exp == 0x7fff)
 1200                                 tw |= 2; /* Infinity or NaN */
 1201                 } else
 1202                         tw |= 3; /* Empty */
 1203                 st = (st - 1) & 7;
 1204         }
 1205         penv_87->en_tw = tw;
 1206 }
 1207 
 1208 void
 1209 npx_fill_fpregs_xmm(struct savexmm *sv_xmm, struct save87 *sv_87)
 1210 {
 1211 
 1212         bzero(sv_87, sizeof(*sv_87));
 1213         npx_fill_fpregs_xmm1(sv_xmm, sv_87);
 1214 }
 1215 
 1216 void
 1217 npx_set_fpregs_xmm(struct save87 *sv_87, struct savexmm *sv_xmm)
 1218 {
 1219         struct env87 *penv_87;
 1220         struct envxmm *penv_xmm;
 1221         int i;
 1222 
 1223         penv_87 = &sv_87->sv_env;
 1224         penv_xmm = &sv_xmm->sv_env;
 1225 
 1226         /* FPU control/status */
 1227         penv_xmm->en_cw = penv_87->en_cw;
 1228         penv_xmm->en_sw = penv_87->en_sw;
 1229         penv_xmm->en_fip = penv_87->en_fip;
 1230         penv_xmm->en_fcs = penv_87->en_fcs;
 1231         penv_xmm->en_opcode = penv_87->en_opcode;
 1232         penv_xmm->en_foo = penv_87->en_foo;
 1233         penv_xmm->en_fos = penv_87->en_fos;
 1234 
 1235         /*
 1236          * FPU registers and tags.
 1237          * Abridged  /  Full translation (values in binary), see FXSAVE spec.
 1238          * 0            11
 1239          * 1            00, 01, 10
 1240          */
 1241         penv_xmm->en_tw = 0;
 1242         for (i = 0; i < 8; ++i) {
 1243                 sv_xmm->sv_fp[i].fp_acc = sv_87->sv_ac[i];
 1244                 if ((penv_87->en_tw & (3 << i * 2)) != (3 << i * 2))
 1245                         penv_xmm->en_tw |= 1 << i;
 1246         }
 1247 }
 1248 
 1249 void
 1250 npx_get_fsave(void *addr)
 1251 {
 1252         struct thread *td;
 1253         union savefpu *sv;
 1254 
 1255         td = curthread;
 1256         npxgetregs(td);
 1257         sv = get_pcb_user_save_td(td);
 1258         if (cpu_fxsr)
 1259                 npx_fill_fpregs_xmm1(&sv->sv_xmm, addr);
 1260         else
 1261                 bcopy(sv, addr, sizeof(struct env87) +
 1262                     sizeof(struct fpacc87[8]));
 1263 }
 1264 
 1265 int
 1266 npx_set_fsave(void *addr)
 1267 {
 1268         union savefpu sv;
 1269         int error;
 1270 
 1271         bzero(&sv, sizeof(sv));
 1272         if (cpu_fxsr)
 1273                 npx_set_fpregs_xmm(addr, &sv.sv_xmm);
 1274         else
 1275                 bcopy(addr, &sv, sizeof(struct env87) +
 1276                     sizeof(struct fpacc87[8]));
 1277         error = npxsetregs(curthread, &sv, NULL, 0);
 1278         return (error);
 1279 }
 1280 
 1281 /*
 1282  * On AuthenticAMD processors, the fxrstor instruction does not restore
 1283  * the x87's stored last instruction pointer, last data pointer, and last
 1284  * opcode values, except in the rare case in which the exception summary
 1285  * (ES) bit in the x87 status word is set to 1.
 1286  *
 1287  * In order to avoid leaking this information across processes, we clean
 1288  * these values by performing a dummy load before executing fxrstor().
 1289  */
 1290 static void
 1291 fpu_clean_state(void)
 1292 {
 1293         static float dummy_variable = 0.0;
 1294         u_short status;
 1295 
 1296         /*
 1297          * Clear the ES bit in the x87 status word if it is currently
 1298          * set, in order to avoid causing a fault in the upcoming load.
 1299          */
 1300         fnstsw(&status);
 1301         if (status & 0x80)
 1302                 fnclex();
 1303 
 1304         /*
 1305          * Load the dummy variable into the x87 stack.  This mangles
 1306          * the x87 stack, but we don't care since we're about to call
 1307          * fxrstor() anyway.
 1308          */
 1309         __asm __volatile("ffree %%st(7); flds %0" : : "m" (dummy_variable));
 1310 }
 1311 
 1312 static void
 1313 fpurstor(union savefpu *addr)
 1314 {
 1315 
 1316         if (use_xsave)
 1317                 xrstor((char *)addr, xsave_mask);
 1318         else if (cpu_fxsr)
 1319                 fxrstor(addr);
 1320         else
 1321                 frstor(addr);
 1322 }
 1323 
 1324 #ifdef DEV_ISA
 1325 /*
 1326  * This sucks up the legacy ISA support assignments from PNPBIOS/ACPI.
 1327  */
 1328 static struct isa_pnp_id npxisa_ids[] = {
 1329         { 0x040cd041, "Legacy ISA coprocessor support" }, /* PNP0C04 */
 1330         { 0 }
 1331 };
 1332 
 1333 static int
 1334 npxisa_probe(device_t dev)
 1335 {
 1336         int result;
 1337         if ((result = ISA_PNP_PROBE(device_get_parent(dev), dev, npxisa_ids)) <= 0) {
 1338                 device_quiet(dev);
 1339         }
 1340         return(result);
 1341 }
 1342 
 1343 static int
 1344 npxisa_attach(device_t dev)
 1345 {
 1346         return (0);
 1347 }
 1348 
 1349 static device_method_t npxisa_methods[] = {
 1350         /* Device interface */
 1351         DEVMETHOD(device_probe,         npxisa_probe),
 1352         DEVMETHOD(device_attach,        npxisa_attach),
 1353         DEVMETHOD(device_detach,        bus_generic_detach),
 1354         DEVMETHOD(device_shutdown,      bus_generic_shutdown),
 1355         DEVMETHOD(device_suspend,       bus_generic_suspend),
 1356         DEVMETHOD(device_resume,        bus_generic_resume),
 1357         
 1358         { 0, 0 }
 1359 };
 1360 
 1361 static driver_t npxisa_driver = {
 1362         "npxisa",
 1363         npxisa_methods,
 1364         1,                      /* no softc */
 1365 };
 1366 
 1367 static devclass_t npxisa_devclass;
 1368 
 1369 DRIVER_MODULE(npxisa, isa, npxisa_driver, npxisa_devclass, 0, 0);
 1370 DRIVER_MODULE(npxisa, acpi, npxisa_driver, npxisa_devclass, 0, 0);
 1371 ISA_PNP_INFO(npxisa_ids);
 1372 #endif /* DEV_ISA */
 1373 
 1374 static MALLOC_DEFINE(M_FPUKERN_CTX, "fpukern_ctx",
 1375     "Kernel contexts for FPU state");
 1376 
 1377 #define FPU_KERN_CTX_NPXINITDONE 0x01
 1378 #define FPU_KERN_CTX_DUMMY       0x02
 1379 #define FPU_KERN_CTX_INUSE       0x04
 1380 
 1381 struct fpu_kern_ctx {
 1382         union savefpu *prev;
 1383         uint32_t flags;
 1384         char hwstate1[];
 1385 };
 1386 
 1387 struct fpu_kern_ctx *
 1388 fpu_kern_alloc_ctx(u_int flags)
 1389 {
 1390         struct fpu_kern_ctx *res;
 1391         size_t sz;
 1392 
 1393         sz = sizeof(struct fpu_kern_ctx) + XSAVE_AREA_ALIGN +
 1394             cpu_max_ext_state_size;
 1395         res = malloc(sz, M_FPUKERN_CTX, ((flags & FPU_KERN_NOWAIT) ?
 1396             M_NOWAIT : M_WAITOK) | M_ZERO);
 1397         return (res);
 1398 }
 1399 
 1400 void
 1401 fpu_kern_free_ctx(struct fpu_kern_ctx *ctx)
 1402 {
 1403 
 1404         KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) == 0, ("free'ing inuse ctx"));
 1405         /* XXXKIB clear the memory ? */
 1406         free(ctx, M_FPUKERN_CTX);
 1407 }
 1408 
 1409 static union savefpu *
 1410 fpu_kern_ctx_savefpu(struct fpu_kern_ctx *ctx)
 1411 {
 1412         vm_offset_t p;
 1413 
 1414         p = (vm_offset_t)&ctx->hwstate1;
 1415         p = roundup2(p, XSAVE_AREA_ALIGN);
 1416         return ((union savefpu *)p);
 1417 }
 1418 
 1419 void
 1420 fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags)
 1421 {
 1422         struct pcb *pcb;
 1423 
 1424         KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) == 0, ("using inuse ctx"));
 1425 
 1426         if ((flags & FPU_KERN_KTHR) != 0 && is_fpu_kern_thread(0)) {
 1427                 ctx->flags = FPU_KERN_CTX_DUMMY | FPU_KERN_CTX_INUSE;
 1428                 return;
 1429         }
 1430         pcb = td->td_pcb;
 1431         critical_enter();
 1432         KASSERT(!PCB_USER_FPU(pcb) || pcb->pcb_save ==
 1433             get_pcb_user_save_pcb(pcb), ("mangled pcb_save"));
 1434         ctx->flags = FPU_KERN_CTX_INUSE;
 1435         if ((pcb->pcb_flags & PCB_NPXINITDONE) != 0)
 1436                 ctx->flags |= FPU_KERN_CTX_NPXINITDONE;
 1437         npxexit(td);
 1438         ctx->prev = pcb->pcb_save;
 1439         pcb->pcb_save = fpu_kern_ctx_savefpu(ctx);
 1440         pcb->pcb_flags |= PCB_KERNNPX;
 1441         pcb->pcb_flags &= ~PCB_NPXINITDONE;
 1442         critical_exit();
 1443 }
 1444 
 1445 int
 1446 fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx)
 1447 {
 1448         struct pcb *pcb;
 1449 
 1450         KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) != 0,
 1451             ("leaving not inuse ctx"));
 1452         ctx->flags &= ~FPU_KERN_CTX_INUSE;
 1453 
 1454         if (is_fpu_kern_thread(0) && (ctx->flags & FPU_KERN_CTX_DUMMY) != 0)
 1455                 return (0);
 1456         pcb = td->td_pcb;
 1457         critical_enter();
 1458         if (curthread == PCPU_GET(fpcurthread))
 1459                 npxdrop();
 1460         pcb->pcb_save = ctx->prev;
 1461         if (pcb->pcb_save == get_pcb_user_save_pcb(pcb)) {
 1462                 if ((pcb->pcb_flags & PCB_NPXUSERINITDONE) != 0) {
 1463                         pcb->pcb_flags |= PCB_NPXINITDONE;
 1464                         if ((pcb->pcb_flags & PCB_KERNNPX_THR) == 0)
 1465                                 pcb->pcb_flags &= ~PCB_KERNNPX;
 1466                 } else if ((pcb->pcb_flags & PCB_KERNNPX_THR) == 0)
 1467                         pcb->pcb_flags &= ~(PCB_NPXINITDONE | PCB_KERNNPX);
 1468         } else {
 1469                 if ((ctx->flags & FPU_KERN_CTX_NPXINITDONE) != 0)
 1470                         pcb->pcb_flags |= PCB_NPXINITDONE;
 1471                 else
 1472                         pcb->pcb_flags &= ~PCB_NPXINITDONE;
 1473                 KASSERT(!PCB_USER_FPU(pcb), ("unpaired fpu_kern_leave"));
 1474         }
 1475         critical_exit();
 1476         return (0);
 1477 }
 1478 
 1479 int
 1480 fpu_kern_thread(u_int flags)
 1481 {
 1482 
 1483         KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0,
 1484             ("Only kthread may use fpu_kern_thread"));
 1485         KASSERT(curpcb->pcb_save == get_pcb_user_save_pcb(curpcb),
 1486             ("mangled pcb_save"));
 1487         KASSERT(PCB_USER_FPU(curpcb), ("recursive call"));
 1488 
 1489         curpcb->pcb_flags |= PCB_KERNNPX | PCB_KERNNPX_THR;
 1490         return (0);
 1491 }
 1492 
 1493 int
 1494 is_fpu_kern_thread(u_int flags)
 1495 {
 1496 
 1497         if ((curthread->td_pflags & TDP_KTHREAD) == 0)
 1498                 return (0);
 1499         return ((curpcb->pcb_flags & PCB_KERNNPX_THR) != 0);
 1500 }
 1501 
 1502 /*
 1503  * FPU save area alloc/free/init utility routines
 1504  */
 1505 union savefpu *
 1506 fpu_save_area_alloc(void)
 1507 {
 1508 
 1509         return (uma_zalloc(fpu_save_area_zone, 0));
 1510 }
 1511 
 1512 void
 1513 fpu_save_area_free(union savefpu *fsa)
 1514 {
 1515 
 1516         uma_zfree(fpu_save_area_zone, fsa);
 1517 }
 1518 
 1519 void
 1520 fpu_save_area_reset(union savefpu *fsa)
 1521 {
 1522 
 1523         bcopy(npx_initialstate, fsa, cpu_max_ext_state_size);
 1524 }

Cache object: f920f60d8e29c233899924cc90cf9d8c


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.