The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/i386/isa/npx.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1990 William Jolitz.
    3  * Copyright (c) 1991 The Regents of the University of California.
    4  * All rights reserved.
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice, this list of conditions and the following disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  * 4. Neither the name of the University nor the names of its contributors
   15  *    may be used to endorse or promote products derived from this software
   16  *    without specific prior written permission.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  *
   30  *      from: @(#)npx.c 7.2 (Berkeley) 5/12/91
   31  */
   32 
   33 #include <sys/cdefs.h>
   34 __FBSDID("$FreeBSD: releng/10.3/sys/i386/isa/npx.c 295148 2016-02-02 14:16:07Z kib $");
   35 
   36 #include "opt_cpu.h"
   37 #include "opt_isa.h"
   38 #include "opt_npx.h"
   39 
   40 #include <sys/param.h>
   41 #include <sys/systm.h>
   42 #include <sys/bus.h>
   43 #include <sys/kernel.h>
   44 #include <sys/lock.h>
   45 #include <sys/malloc.h>
   46 #include <sys/module.h>
   47 #include <sys/mutex.h>
   48 #include <sys/mutex.h>
   49 #include <sys/proc.h>
   50 #include <sys/smp.h>
   51 #include <sys/sysctl.h>
   52 #include <machine/bus.h>
   53 #include <sys/rman.h>
   54 #ifdef NPX_DEBUG
   55 #include <sys/syslog.h>
   56 #endif
   57 #include <sys/signalvar.h>
   58 #include <vm/uma.h>
   59 
   60 #include <machine/asmacros.h>
   61 #include <machine/cputypes.h>
   62 #include <machine/frame.h>
   63 #include <machine/md_var.h>
   64 #include <machine/pcb.h>
   65 #include <machine/psl.h>
   66 #include <machine/resource.h>
   67 #include <machine/specialreg.h>
   68 #include <machine/segments.h>
   69 #include <machine/ucontext.h>
   70 
   71 #include <machine/intr_machdep.h>
   72 #ifdef XEN
   73 #include <xen/xen-os.h>
   74 #include <xen/hypervisor.h>
   75 #endif
   76 
   77 #ifdef DEV_ISA
   78 #include <isa/isavar.h>
   79 #endif
   80 
   81 #if !defined(CPU_DISABLE_SSE) && defined(I686_CPU)
   82 #define CPU_ENABLE_SSE
   83 #endif
   84 
   85 /*
   86  * 387 and 287 Numeric Coprocessor Extension (NPX) Driver.
   87  */
   88 
   89 #if defined(__GNUCLIKE_ASM) && !defined(lint)
   90 
   91 #define fldcw(cw)               __asm __volatile("fldcw %0" : : "m" (cw))
   92 #define fnclex()                __asm __volatile("fnclex")
   93 #define fninit()                __asm __volatile("fninit")
   94 #define fnsave(addr)            __asm __volatile("fnsave %0" : "=m" (*(addr)))
   95 #define fnstcw(addr)            __asm __volatile("fnstcw %0" : "=m" (*(addr)))
   96 #define fnstsw(addr)            __asm __volatile("fnstsw %0" : "=am" (*(addr)))
   97 #define fp_divide_by_0()        __asm __volatile( \
   98                                     "fldz; fld1; fdiv %st,%st(1); fnop")
   99 #define frstor(addr)            __asm __volatile("frstor %0" : : "m" (*(addr)))
  100 #ifdef CPU_ENABLE_SSE
  101 #define fxrstor(addr)           __asm __volatile("fxrstor %0" : : "m" (*(addr)))
  102 #define fxsave(addr)            __asm __volatile("fxsave %0" : "=m" (*(addr)))
  103 #define ldmxcsr(csr)            __asm __volatile("ldmxcsr %0" : : "m" (csr))
  104 #define stmxcsr(addr)           __asm __volatile("stmxcsr %0" : : "m" (*(addr)))
  105 
  106 static __inline void
  107 xrstor(char *addr, uint64_t mask)
  108 {
  109         uint32_t low, hi;
  110 
  111         low = mask;
  112         hi = mask >> 32;
  113         __asm __volatile("xrstor %0" : : "m" (*addr), "a" (low), "d" (hi));
  114 }
  115 
  116 static __inline void
  117 xsave(char *addr, uint64_t mask)
  118 {
  119         uint32_t low, hi;
  120 
  121         low = mask;
  122         hi = mask >> 32;
  123         __asm __volatile("xsave %0" : "=m" (*addr) : "a" (low), "d" (hi) :
  124             "memory");
  125 }
  126 
  127 static __inline void
  128 xsaveopt(char *addr, uint64_t mask)
  129 {
  130         uint32_t low, hi;
  131 
  132         low = mask;
  133         hi = mask >> 32;
  134         __asm __volatile("xsaveopt %0" : "=m" (*addr) : "a" (low), "d" (hi) :
  135             "memory");
  136 }
  137 #endif
  138 #else   /* !(__GNUCLIKE_ASM && !lint) */
  139 
  140 void    fldcw(u_short cw);
  141 void    fnclex(void);
  142 void    fninit(void);
  143 void    fnsave(caddr_t addr);
  144 void    fnstcw(caddr_t addr);
  145 void    fnstsw(caddr_t addr);
  146 void    fp_divide_by_0(void);
  147 void    frstor(caddr_t addr);
  148 #ifdef CPU_ENABLE_SSE
  149 void    fxsave(caddr_t addr);
  150 void    fxrstor(caddr_t addr);
  151 void    ldmxcsr(u_int csr);
  152 void    stmxcsr(u_int *csr);
  153 void    xrstor(char *addr, uint64_t mask);
  154 void    xsave(char *addr, uint64_t mask);
  155 void    xsaveopt(char *addr, uint64_t mask);
  156 #endif
  157 
  158 #endif  /* __GNUCLIKE_ASM && !lint */
  159 
  160 #ifdef XEN
  161 #define start_emulating()       (HYPERVISOR_fpu_taskswitch(1))
  162 #define stop_emulating()        (HYPERVISOR_fpu_taskswitch(0))
  163 #else
  164 #define start_emulating()       load_cr0(rcr0() | CR0_TS)
  165 #define stop_emulating()        clts()
  166 #endif
  167 
  168 #ifdef CPU_ENABLE_SSE
  169 #define GET_FPU_CW(thread) \
  170         (cpu_fxsr ? \
  171                 (thread)->td_pcb->pcb_save->sv_xmm.sv_env.en_cw : \
  172                 (thread)->td_pcb->pcb_save->sv_87.sv_env.en_cw)
  173 #define GET_FPU_SW(thread) \
  174         (cpu_fxsr ? \
  175                 (thread)->td_pcb->pcb_save->sv_xmm.sv_env.en_sw : \
  176                 (thread)->td_pcb->pcb_save->sv_87.sv_env.en_sw)
  177 #define SET_FPU_CW(savefpu, value) do { \
  178         if (cpu_fxsr) \
  179                 (savefpu)->sv_xmm.sv_env.en_cw = (value); \
  180         else \
  181                 (savefpu)->sv_87.sv_env.en_cw = (value); \
  182 } while (0)
  183 #else /* CPU_ENABLE_SSE */
  184 #define GET_FPU_CW(thread) \
  185         (thread->td_pcb->pcb_save->sv_87.sv_env.en_cw)
  186 #define GET_FPU_SW(thread) \
  187         (thread->td_pcb->pcb_save->sv_87.sv_env.en_sw)
  188 #define SET_FPU_CW(savefpu, value) \
  189         (savefpu)->sv_87.sv_env.en_cw = (value)
  190 #endif /* CPU_ENABLE_SSE */
  191 
  192 #ifdef CPU_ENABLE_SSE
  193 CTASSERT(sizeof(union savefpu) == 512);
  194 CTASSERT(sizeof(struct xstate_hdr) == 64);
  195 CTASSERT(sizeof(struct savefpu_ymm) == 832);
  196 
  197 /*
  198  * This requirement is to make it easier for asm code to calculate
  199  * offset of the fpu save area from the pcb address. FPU save area
  200  * must be 64-byte aligned.
  201  */
  202 CTASSERT(sizeof(struct pcb) % XSAVE_AREA_ALIGN == 0);
  203 
  204 /*
  205  * Ensure the copy of XCR0 saved in a core is contained in the padding
  206  * area.
  207  */
  208 CTASSERT(X86_XSTATE_XCR0_OFFSET >= offsetof(struct savexmm, sv_pad) &&
  209     X86_XSTATE_XCR0_OFFSET + sizeof(uint64_t) <= sizeof(struct savexmm));
  210 
  211 static  void    fpu_clean_state(void);
  212 #endif
  213 
  214 static  void    fpusave(union savefpu *);
  215 static  void    fpurstor(union savefpu *);
  216 
  217 int     hw_float;
  218 
  219 SYSCTL_INT(_hw, HW_FLOATINGPT, floatingpoint, CTLFLAG_RD,
  220     &hw_float, 0, "Floating point instructions executed in hardware");
  221 
  222 #ifdef CPU_ENABLE_SSE
  223 int use_xsave;
  224 uint64_t xsave_mask;
  225 #endif
  226 static  uma_zone_t fpu_save_area_zone;
  227 static  union savefpu *npx_initialstate;
  228 
  229 #ifdef CPU_ENABLE_SSE
  230 struct xsave_area_elm_descr {
  231         u_int   offset;
  232         u_int   size;
  233 } *xsave_area_desc;
  234 
  235 static int use_xsaveopt;
  236 #endif
  237 
  238 static  volatile u_int          npx_traps_while_probing;
  239 
  240 alias_for_inthand_t probetrap;
  241 __asm("                                                         \n\
  242         .text                                                   \n\
  243         .p2align 2,0x90                                         \n\
  244         .type   " __XSTRING(CNAME(probetrap)) ",@function       \n\
  245 " __XSTRING(CNAME(probetrap)) ":                                \n\
  246         ss                                                      \n\
  247         incl    " __XSTRING(CNAME(npx_traps_while_probing)) "   \n\
  248         fnclex                                                  \n\
  249         iret                                                    \n\
  250 ");
  251 
  252 /*
  253  * Determine if an FPU is present and how to use it.
  254  */
  255 static int
  256 npx_probe(void)
  257 {
  258         struct gate_descriptor save_idt_npxtrap;
  259         u_short control, status;
  260 
  261         /*
  262          * Modern CPUs all have an FPU that uses the INT16 interface
  263          * and provide a simple way to verify that, so handle the
  264          * common case right away.
  265          */
  266         if (cpu_feature & CPUID_FPU) {
  267                 hw_float = 1;
  268                 return (1);
  269         }
  270 
  271         save_idt_npxtrap = idt[IDT_MF];
  272         setidt(IDT_MF, probetrap, SDT_SYS386TGT, SEL_KPL,
  273             GSEL(GCODE_SEL, SEL_KPL));
  274 
  275         /*
  276          * Don't trap while we're probing.
  277          */
  278         stop_emulating();
  279 
  280         /*
  281          * Finish resetting the coprocessor, if any.  If there is an error
  282          * pending, then we may get a bogus IRQ13, but npx_intr() will handle
  283          * it OK.  Bogus halts have never been observed, but we enabled
  284          * IRQ13 and cleared the BUSY# latch early to handle them anyway.
  285          */
  286         fninit();
  287 
  288         /*
  289          * Don't use fwait here because it might hang.
  290          * Don't use fnop here because it usually hangs if there is no FPU.
  291          */
  292         DELAY(1000);            /* wait for any IRQ13 */
  293 #ifdef DIAGNOSTIC
  294         if (npx_traps_while_probing != 0)
  295                 printf("fninit caused %u bogus npx trap(s)\n",
  296                        npx_traps_while_probing);
  297 #endif
  298         /*
  299          * Check for a status of mostly zero.
  300          */
  301         status = 0x5a5a;
  302         fnstsw(&status);
  303         if ((status & 0xb8ff) == 0) {
  304                 /*
  305                  * Good, now check for a proper control word.
  306                  */
  307                 control = 0x5a5a;
  308                 fnstcw(&control);
  309                 if ((control & 0x1f3f) == 0x033f) {
  310                         /*
  311                          * We have an npx, now divide by 0 to see if exception
  312                          * 16 works.
  313                          */
  314                         control &= ~(1 << 2);   /* enable divide by 0 trap */
  315                         fldcw(control);
  316 #ifdef FPU_ERROR_BROKEN
  317                         /*
  318                          * FPU error signal doesn't work on some CPU
  319                          * accelerator board.
  320                          */
  321                         hw_float = 1;
  322                         return (1);
  323 #endif
  324                         npx_traps_while_probing = 0;
  325                         fp_divide_by_0();
  326                         if (npx_traps_while_probing != 0) {
  327                                 /*
  328                                  * Good, exception 16 works.
  329                                  */
  330                                 hw_float = 1;
  331                                 goto cleanup;
  332                         }
  333                         printf(
  334         "FPU does not use exception 16 for error reporting\n");
  335                         goto cleanup;
  336                 }
  337         }
  338 
  339         /*
  340          * Probe failed.  Floating point simply won't work.
  341          * Notify user and disable FPU/MMX/SSE instruction execution.
  342          */
  343         printf("WARNING: no FPU!\n");
  344         __asm __volatile("smsw %%ax; orb %0,%%al; lmsw %%ax" : :
  345             "n" (CR0_EM | CR0_MP) : "ax");
  346 
  347 cleanup:
  348         idt[IDT_MF] = save_idt_npxtrap;
  349         return (hw_float);
  350 }
  351 
  352 #ifdef CPU_ENABLE_SSE
  353 /*
  354  * Enable XSAVE if supported and allowed by user.
  355  * Calculate the xsave_mask.
  356  */
  357 static void
  358 npxinit_bsp1(void)
  359 {
  360         u_int cp[4];
  361         uint64_t xsave_mask_user;
  362 
  363         if (cpu_fxsr && (cpu_feature2 & CPUID2_XSAVE) != 0) {
  364                 use_xsave = 1;
  365                 TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave);
  366         }
  367         if (!use_xsave)
  368                 return;
  369 
  370         cpuid_count(0xd, 0x0, cp);
  371         xsave_mask = XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE;
  372         if ((cp[0] & xsave_mask) != xsave_mask)
  373                 panic("CPU0 does not support X87 or SSE: %x", cp[0]);
  374         xsave_mask = ((uint64_t)cp[3] << 32) | cp[0];
  375         xsave_mask_user = xsave_mask;
  376         TUNABLE_QUAD_FETCH("hw.xsave_mask", &xsave_mask_user);
  377         xsave_mask_user |= XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE;
  378         xsave_mask &= xsave_mask_user;
  379         if ((xsave_mask & XFEATURE_AVX512) != XFEATURE_AVX512)
  380                 xsave_mask &= ~XFEATURE_AVX512;
  381         if ((xsave_mask & XFEATURE_MPX) != XFEATURE_MPX)
  382                 xsave_mask &= ~XFEATURE_MPX;
  383 
  384         cpuid_count(0xd, 0x1, cp);
  385         if ((cp[0] & CPUID_EXTSTATE_XSAVEOPT) != 0)
  386                 use_xsaveopt = 1;
  387 }
  388 #endif
  389 /*
  390 
  391  * Calculate the fpu save area size.
  392  */
  393 static void
  394 npxinit_bsp2(void)
  395 {
  396 #ifdef CPU_ENABLE_SSE
  397         u_int cp[4];
  398 
  399         if (use_xsave) {
  400                 cpuid_count(0xd, 0x0, cp);
  401                 cpu_max_ext_state_size = cp[1];
  402 
  403                 /*
  404                  * Reload the cpu_feature2, since we enabled OSXSAVE.
  405                  */
  406                 do_cpuid(1, cp);
  407                 cpu_feature2 = cp[2];
  408         } else
  409 #endif
  410                 cpu_max_ext_state_size = sizeof(union savefpu);
  411 }
  412 
  413 /*
  414  * Initialize floating point unit.
  415  */
  416 void
  417 npxinit(bool bsp)
  418 {
  419         static union savefpu dummy;
  420         register_t saveintr;
  421 #ifdef CPU_ENABLE_SSE
  422         u_int mxcsr;
  423 #endif
  424         u_short control;
  425 
  426         if (bsp) {
  427                 if (!npx_probe())
  428                         return;
  429 #ifdef CPU_ENABLE_SSE
  430                 npxinit_bsp1();
  431 #endif
  432         }
  433 
  434 #ifdef CPU_ENABLE_SSE
  435         if (use_xsave) {
  436                 load_cr4(rcr4() | CR4_XSAVE);
  437                 load_xcr(XCR0, xsave_mask);
  438         }
  439 #endif
  440 
  441         /*
  442          * XCR0 shall be set up before CPU can report the save area size.
  443          */
  444         if (bsp)
  445                 npxinit_bsp2();
  446         
  447         /*
  448          * fninit has the same h/w bugs as fnsave.  Use the detoxified
  449          * fnsave to throw away any junk in the fpu.  fpusave() initializes
  450          * the fpu.
  451          *
  452          * It is too early for critical_enter() to work on AP.
  453          */
  454         saveintr = intr_disable();
  455         stop_emulating();
  456 #ifdef CPU_ENABLE_SSE
  457         if (cpu_fxsr)
  458                 fninit();
  459         else
  460 #endif
  461                 fnsave(&dummy);
  462         control = __INITIAL_NPXCW__;
  463         fldcw(control);
  464 #ifdef CPU_ENABLE_SSE
  465         if (cpu_fxsr) {
  466                 mxcsr = __INITIAL_MXCSR__;
  467                 ldmxcsr(mxcsr);
  468         }
  469 #endif
  470         start_emulating();
  471         intr_restore(saveintr);
  472 }
  473 
  474 /*
  475  * On the boot CPU we generate a clean state that is used to
  476  * initialize the floating point unit when it is first used by a
  477  * process.
  478  */
  479 static void
  480 npxinitstate(void *arg __unused)
  481 {
  482         register_t saveintr;
  483 #ifdef CPU_ENABLE_SSE
  484         int cp[4], i, max_ext_n;
  485 #endif
  486 
  487         if (!hw_float)
  488                 return;
  489 
  490         npx_initialstate = malloc(cpu_max_ext_state_size, M_DEVBUF,
  491             M_WAITOK | M_ZERO);
  492         saveintr = intr_disable();
  493         stop_emulating();
  494 
  495         fpusave(npx_initialstate);
  496 #ifdef CPU_ENABLE_SSE
  497         if (cpu_fxsr) {
  498                 if (npx_initialstate->sv_xmm.sv_env.en_mxcsr_mask)
  499                         cpu_mxcsr_mask = 
  500                             npx_initialstate->sv_xmm.sv_env.en_mxcsr_mask;
  501                 else
  502                         cpu_mxcsr_mask = 0xFFBF;
  503 
  504                 /*
  505                  * The fninit instruction does not modify XMM
  506                  * registers or x87 registers (MM/ST).  The fpusave
  507                  * call dumped the garbage contained in the registers
  508                  * after reset to the initial state saved.  Clear XMM
  509                  * and x87 registers file image to make the startup
  510                  * program state and signal handler XMM/x87 register
  511                  * content predictable.
  512                  */
  513                 bzero(npx_initialstate->sv_xmm.sv_fp,
  514                     sizeof(npx_initialstate->sv_xmm.sv_fp));
  515                 bzero(npx_initialstate->sv_xmm.sv_xmm,
  516                     sizeof(npx_initialstate->sv_xmm.sv_xmm));
  517         } else
  518 #endif
  519                 bzero(npx_initialstate->sv_87.sv_ac,
  520                     sizeof(npx_initialstate->sv_87.sv_ac));
  521 
  522 #ifdef CPU_ENABLE_SSE
  523         /*
  524          * Create a table describing the layout of the CPU Extended
  525          * Save Area.
  526          */
  527         if (use_xsave) {
  528                 if (xsave_mask >> 32 != 0)
  529                         max_ext_n = fls(xsave_mask >> 32) + 32;
  530                 else
  531                         max_ext_n = fls(xsave_mask);
  532                 xsave_area_desc = malloc(max_ext_n * sizeof(struct
  533                     xsave_area_elm_descr), M_DEVBUF, M_WAITOK | M_ZERO);
  534                 /* x87 state */
  535                 xsave_area_desc[0].offset = 0;
  536                 xsave_area_desc[0].size = 160;
  537                 /* XMM */
  538                 xsave_area_desc[1].offset = 160;
  539                 xsave_area_desc[1].size = 288 - 160;
  540 
  541                 for (i = 2; i < max_ext_n; i++) {
  542                         cpuid_count(0xd, i, cp);
  543                         xsave_area_desc[i].offset = cp[1];
  544                         xsave_area_desc[i].size = cp[0];
  545                 }
  546         }
  547 #endif
  548 
  549         fpu_save_area_zone = uma_zcreate("FPU_save_area",
  550             cpu_max_ext_state_size, NULL, NULL, NULL, NULL,
  551             XSAVE_AREA_ALIGN - 1, 0);
  552 
  553         start_emulating();
  554         intr_restore(saveintr);
  555 }
  556 SYSINIT(npxinitstate, SI_SUB_DRIVERS, SI_ORDER_ANY, npxinitstate, NULL);
  557 
  558 /*
  559  * Free coprocessor (if we have it).
  560  */
  561 void
  562 npxexit(td)
  563         struct thread *td;
  564 {
  565 
  566         critical_enter();
  567         if (curthread == PCPU_GET(fpcurthread)) {
  568                 stop_emulating();
  569                 fpusave(curpcb->pcb_save);
  570                 start_emulating();
  571                 PCPU_SET(fpcurthread, NULL);
  572         }
  573         critical_exit();
  574 #ifdef NPX_DEBUG
  575         if (hw_float) {
  576                 u_int   masked_exceptions;
  577 
  578                 masked_exceptions = GET_FPU_CW(td) & GET_FPU_SW(td) & 0x7f;
  579                 /*
  580                  * Log exceptions that would have trapped with the old
  581                  * control word (overflow, divide by 0, and invalid operand).
  582                  */
  583                 if (masked_exceptions & 0x0d)
  584                         log(LOG_ERR,
  585         "pid %d (%s) exited with masked floating point exceptions 0x%02x\n",
  586                             td->td_proc->p_pid, td->td_proc->p_comm,
  587                             masked_exceptions);
  588         }
  589 #endif
  590 }
  591 
  592 int
  593 npxformat()
  594 {
  595 
  596         if (!hw_float)
  597                 return (_MC_FPFMT_NODEV);
  598 #ifdef  CPU_ENABLE_SSE
  599         if (cpu_fxsr)
  600                 return (_MC_FPFMT_XMM);
  601 #endif
  602         return (_MC_FPFMT_387);
  603 }
  604 
  605 /* 
  606  * The following mechanism is used to ensure that the FPE_... value
  607  * that is passed as a trapcode to the signal handler of the user
  608  * process does not have more than one bit set.
  609  * 
  610  * Multiple bits may be set if the user process modifies the control
  611  * word while a status word bit is already set.  While this is a sign
  612  * of bad coding, we have no choise than to narrow them down to one
  613  * bit, since we must not send a trapcode that is not exactly one of
  614  * the FPE_ macros.
  615  *
  616  * The mechanism has a static table with 127 entries.  Each combination
  617  * of the 7 FPU status word exception bits directly translates to a
  618  * position in this table, where a single FPE_... value is stored.
  619  * This FPE_... value stored there is considered the "most important"
  620  * of the exception bits and will be sent as the signal code.  The
  621  * precedence of the bits is based upon Intel Document "Numerical
  622  * Applications", Chapter "Special Computational Situations".
  623  *
  624  * The macro to choose one of these values does these steps: 1) Throw
  625  * away status word bits that cannot be masked.  2) Throw away the bits
  626  * currently masked in the control word, assuming the user isn't
  627  * interested in them anymore.  3) Reinsert status word bit 7 (stack
  628  * fault) if it is set, which cannot be masked but must be presered.
  629  * 4) Use the remaining bits to point into the trapcode table.
  630  *
  631  * The 6 maskable bits in order of their preference, as stated in the
  632  * above referenced Intel manual:
  633  * 1  Invalid operation (FP_X_INV)
  634  * 1a   Stack underflow
  635  * 1b   Stack overflow
  636  * 1c   Operand of unsupported format
  637  * 1d   SNaN operand.
  638  * 2  QNaN operand (not an exception, irrelavant here)
  639  * 3  Any other invalid-operation not mentioned above or zero divide
  640  *      (FP_X_INV, FP_X_DZ)
  641  * 4  Denormal operand (FP_X_DNML)
  642  * 5  Numeric over/underflow (FP_X_OFL, FP_X_UFL)
  643  * 6  Inexact result (FP_X_IMP) 
  644  */
  645 static char fpetable[128] = {
  646         0,
  647         FPE_FLTINV,     /*  1 - INV */
  648         FPE_FLTUND,     /*  2 - DNML */
  649         FPE_FLTINV,     /*  3 - INV | DNML */
  650         FPE_FLTDIV,     /*  4 - DZ */
  651         FPE_FLTINV,     /*  5 - INV | DZ */
  652         FPE_FLTDIV,     /*  6 - DNML | DZ */
  653         FPE_FLTINV,     /*  7 - INV | DNML | DZ */
  654         FPE_FLTOVF,     /*  8 - OFL */
  655         FPE_FLTINV,     /*  9 - INV | OFL */
  656         FPE_FLTUND,     /*  A - DNML | OFL */
  657         FPE_FLTINV,     /*  B - INV | DNML | OFL */
  658         FPE_FLTDIV,     /*  C - DZ | OFL */
  659         FPE_FLTINV,     /*  D - INV | DZ | OFL */
  660         FPE_FLTDIV,     /*  E - DNML | DZ | OFL */
  661         FPE_FLTINV,     /*  F - INV | DNML | DZ | OFL */
  662         FPE_FLTUND,     /* 10 - UFL */
  663         FPE_FLTINV,     /* 11 - INV | UFL */
  664         FPE_FLTUND,     /* 12 - DNML | UFL */
  665         FPE_FLTINV,     /* 13 - INV | DNML | UFL */
  666         FPE_FLTDIV,     /* 14 - DZ | UFL */
  667         FPE_FLTINV,     /* 15 - INV | DZ | UFL */
  668         FPE_FLTDIV,     /* 16 - DNML | DZ | UFL */
  669         FPE_FLTINV,     /* 17 - INV | DNML | DZ | UFL */
  670         FPE_FLTOVF,     /* 18 - OFL | UFL */
  671         FPE_FLTINV,     /* 19 - INV | OFL | UFL */
  672         FPE_FLTUND,     /* 1A - DNML | OFL | UFL */
  673         FPE_FLTINV,     /* 1B - INV | DNML | OFL | UFL */
  674         FPE_FLTDIV,     /* 1C - DZ | OFL | UFL */
  675         FPE_FLTINV,     /* 1D - INV | DZ | OFL | UFL */
  676         FPE_FLTDIV,     /* 1E - DNML | DZ | OFL | UFL */
  677         FPE_FLTINV,     /* 1F - INV | DNML | DZ | OFL | UFL */
  678         FPE_FLTRES,     /* 20 - IMP */
  679         FPE_FLTINV,     /* 21 - INV | IMP */
  680         FPE_FLTUND,     /* 22 - DNML | IMP */
  681         FPE_FLTINV,     /* 23 - INV | DNML | IMP */
  682         FPE_FLTDIV,     /* 24 - DZ | IMP */
  683         FPE_FLTINV,     /* 25 - INV | DZ | IMP */
  684         FPE_FLTDIV,     /* 26 - DNML | DZ | IMP */
  685         FPE_FLTINV,     /* 27 - INV | DNML | DZ | IMP */
  686         FPE_FLTOVF,     /* 28 - OFL | IMP */
  687         FPE_FLTINV,     /* 29 - INV | OFL | IMP */
  688         FPE_FLTUND,     /* 2A - DNML | OFL | IMP */
  689         FPE_FLTINV,     /* 2B - INV | DNML | OFL | IMP */
  690         FPE_FLTDIV,     /* 2C - DZ | OFL | IMP */
  691         FPE_FLTINV,     /* 2D - INV | DZ | OFL | IMP */
  692         FPE_FLTDIV,     /* 2E - DNML | DZ | OFL | IMP */
  693         FPE_FLTINV,     /* 2F - INV | DNML | DZ | OFL | IMP */
  694         FPE_FLTUND,     /* 30 - UFL | IMP */
  695         FPE_FLTINV,     /* 31 - INV | UFL | IMP */
  696         FPE_FLTUND,     /* 32 - DNML | UFL | IMP */
  697         FPE_FLTINV,     /* 33 - INV | DNML | UFL | IMP */
  698         FPE_FLTDIV,     /* 34 - DZ | UFL | IMP */
  699         FPE_FLTINV,     /* 35 - INV | DZ | UFL | IMP */
  700         FPE_FLTDIV,     /* 36 - DNML | DZ | UFL | IMP */
  701         FPE_FLTINV,     /* 37 - INV | DNML | DZ | UFL | IMP */
  702         FPE_FLTOVF,     /* 38 - OFL | UFL | IMP */
  703         FPE_FLTINV,     /* 39 - INV | OFL | UFL | IMP */
  704         FPE_FLTUND,     /* 3A - DNML | OFL | UFL | IMP */
  705         FPE_FLTINV,     /* 3B - INV | DNML | OFL | UFL | IMP */
  706         FPE_FLTDIV,     /* 3C - DZ | OFL | UFL | IMP */
  707         FPE_FLTINV,     /* 3D - INV | DZ | OFL | UFL | IMP */
  708         FPE_FLTDIV,     /* 3E - DNML | DZ | OFL | UFL | IMP */
  709         FPE_FLTINV,     /* 3F - INV | DNML | DZ | OFL | UFL | IMP */
  710         FPE_FLTSUB,     /* 40 - STK */
  711         FPE_FLTSUB,     /* 41 - INV | STK */
  712         FPE_FLTUND,     /* 42 - DNML | STK */
  713         FPE_FLTSUB,     /* 43 - INV | DNML | STK */
  714         FPE_FLTDIV,     /* 44 - DZ | STK */
  715         FPE_FLTSUB,     /* 45 - INV | DZ | STK */
  716         FPE_FLTDIV,     /* 46 - DNML | DZ | STK */
  717         FPE_FLTSUB,     /* 47 - INV | DNML | DZ | STK */
  718         FPE_FLTOVF,     /* 48 - OFL | STK */
  719         FPE_FLTSUB,     /* 49 - INV | OFL | STK */
  720         FPE_FLTUND,     /* 4A - DNML | OFL | STK */
  721         FPE_FLTSUB,     /* 4B - INV | DNML | OFL | STK */
  722         FPE_FLTDIV,     /* 4C - DZ | OFL | STK */
  723         FPE_FLTSUB,     /* 4D - INV | DZ | OFL | STK */
  724         FPE_FLTDIV,     /* 4E - DNML | DZ | OFL | STK */
  725         FPE_FLTSUB,     /* 4F - INV | DNML | DZ | OFL | STK */
  726         FPE_FLTUND,     /* 50 - UFL | STK */
  727         FPE_FLTSUB,     /* 51 - INV | UFL | STK */
  728         FPE_FLTUND,     /* 52 - DNML | UFL | STK */
  729         FPE_FLTSUB,     /* 53 - INV | DNML | UFL | STK */
  730         FPE_FLTDIV,     /* 54 - DZ | UFL | STK */
  731         FPE_FLTSUB,     /* 55 - INV | DZ | UFL | STK */
  732         FPE_FLTDIV,     /* 56 - DNML | DZ | UFL | STK */
  733         FPE_FLTSUB,     /* 57 - INV | DNML | DZ | UFL | STK */
  734         FPE_FLTOVF,     /* 58 - OFL | UFL | STK */
  735         FPE_FLTSUB,     /* 59 - INV | OFL | UFL | STK */
  736         FPE_FLTUND,     /* 5A - DNML | OFL | UFL | STK */
  737         FPE_FLTSUB,     /* 5B - INV | DNML | OFL | UFL | STK */
  738         FPE_FLTDIV,     /* 5C - DZ | OFL | UFL | STK */
  739         FPE_FLTSUB,     /* 5D - INV | DZ | OFL | UFL | STK */
  740         FPE_FLTDIV,     /* 5E - DNML | DZ | OFL | UFL | STK */
  741         FPE_FLTSUB,     /* 5F - INV | DNML | DZ | OFL | UFL | STK */
  742         FPE_FLTRES,     /* 60 - IMP | STK */
  743         FPE_FLTSUB,     /* 61 - INV | IMP | STK */
  744         FPE_FLTUND,     /* 62 - DNML | IMP | STK */
  745         FPE_FLTSUB,     /* 63 - INV | DNML | IMP | STK */
  746         FPE_FLTDIV,     /* 64 - DZ | IMP | STK */
  747         FPE_FLTSUB,     /* 65 - INV | DZ | IMP | STK */
  748         FPE_FLTDIV,     /* 66 - DNML | DZ | IMP | STK */
  749         FPE_FLTSUB,     /* 67 - INV | DNML | DZ | IMP | STK */
  750         FPE_FLTOVF,     /* 68 - OFL | IMP | STK */
  751         FPE_FLTSUB,     /* 69 - INV | OFL | IMP | STK */
  752         FPE_FLTUND,     /* 6A - DNML | OFL | IMP | STK */
  753         FPE_FLTSUB,     /* 6B - INV | DNML | OFL | IMP | STK */
  754         FPE_FLTDIV,     /* 6C - DZ | OFL | IMP | STK */
  755         FPE_FLTSUB,     /* 6D - INV | DZ | OFL | IMP | STK */
  756         FPE_FLTDIV,     /* 6E - DNML | DZ | OFL | IMP | STK */
  757         FPE_FLTSUB,     /* 6F - INV | DNML | DZ | OFL | IMP | STK */
  758         FPE_FLTUND,     /* 70 - UFL | IMP | STK */
  759         FPE_FLTSUB,     /* 71 - INV | UFL | IMP | STK */
  760         FPE_FLTUND,     /* 72 - DNML | UFL | IMP | STK */
  761         FPE_FLTSUB,     /* 73 - INV | DNML | UFL | IMP | STK */
  762         FPE_FLTDIV,     /* 74 - DZ | UFL | IMP | STK */
  763         FPE_FLTSUB,     /* 75 - INV | DZ | UFL | IMP | STK */
  764         FPE_FLTDIV,     /* 76 - DNML | DZ | UFL | IMP | STK */
  765         FPE_FLTSUB,     /* 77 - INV | DNML | DZ | UFL | IMP | STK */
  766         FPE_FLTOVF,     /* 78 - OFL | UFL | IMP | STK */
  767         FPE_FLTSUB,     /* 79 - INV | OFL | UFL | IMP | STK */
  768         FPE_FLTUND,     /* 7A - DNML | OFL | UFL | IMP | STK */
  769         FPE_FLTSUB,     /* 7B - INV | DNML | OFL | UFL | IMP | STK */
  770         FPE_FLTDIV,     /* 7C - DZ | OFL | UFL | IMP | STK */
  771         FPE_FLTSUB,     /* 7D - INV | DZ | OFL | UFL | IMP | STK */
  772         FPE_FLTDIV,     /* 7E - DNML | DZ | OFL | UFL | IMP | STK */
  773         FPE_FLTSUB,     /* 7F - INV | DNML | DZ | OFL | UFL | IMP | STK */
  774 };
  775 
  776 /*
  777  * Read the FP status and control words, then generate si_code value
  778  * for SIGFPE.  The error code chosen will be one of the
  779  * FPE_... macros.  It will be sent as the second argument to old
  780  * BSD-style signal handlers and as "siginfo_t->si_code" (second
  781  * argument) to SA_SIGINFO signal handlers.
  782  *
  783  * Some time ago, we cleared the x87 exceptions with FNCLEX there.
  784  * Clearing exceptions was necessary mainly to avoid IRQ13 bugs.  The
  785  * usermode code which understands the FPU hardware enough to enable
  786  * the exceptions, can also handle clearing the exception state in the
  787  * handler.  The only consequence of not clearing the exception is the
  788  * rethrow of the SIGFPE on return from the signal handler and
  789  * reexecution of the corresponding instruction.
  790  *
  791  * For XMM traps, the exceptions were never cleared.
  792  */
  793 int
  794 npxtrap_x87(void)
  795 {
  796         u_short control, status;
  797 
  798         if (!hw_float) {
  799                 printf(
  800         "npxtrap_x87: fpcurthread = %p, curthread = %p, hw_float = %d\n",
  801                        PCPU_GET(fpcurthread), curthread, hw_float);
  802                 panic("npxtrap from nowhere");
  803         }
  804         critical_enter();
  805 
  806         /*
  807          * Interrupt handling (for another interrupt) may have pushed the
  808          * state to memory.  Fetch the relevant parts of the state from
  809          * wherever they are.
  810          */
  811         if (PCPU_GET(fpcurthread) != curthread) {
  812                 control = GET_FPU_CW(curthread);
  813                 status = GET_FPU_SW(curthread);
  814         } else {
  815                 fnstcw(&control);
  816                 fnstsw(&status);
  817         }
  818         critical_exit();
  819         return (fpetable[status & ((~control & 0x3f) | 0x40)]);
  820 }
  821 
  822 #ifdef CPU_ENABLE_SSE
  823 int
  824 npxtrap_sse(void)
  825 {
  826         u_int mxcsr;
  827 
  828         if (!hw_float) {
  829                 printf(
  830         "npxtrap_sse: fpcurthread = %p, curthread = %p, hw_float = %d\n",
  831                        PCPU_GET(fpcurthread), curthread, hw_float);
  832                 panic("npxtrap from nowhere");
  833         }
  834         critical_enter();
  835         if (PCPU_GET(fpcurthread) != curthread)
  836                 mxcsr = curthread->td_pcb->pcb_save->sv_xmm.sv_env.en_mxcsr;
  837         else
  838                 stmxcsr(&mxcsr);
  839         critical_exit();
  840         return (fpetable[(mxcsr & (~mxcsr >> 7)) & 0x3f]);
  841 }
  842 #endif
  843 
  844 /*
  845  * Implement device not available (DNA) exception
  846  *
  847  * It would be better to switch FP context here (if curthread != fpcurthread)
  848  * and not necessarily for every context switch, but it is too hard to
  849  * access foreign pcb's.
  850  */
  851 
  852 static int err_count = 0;
  853 
  854 int
  855 npxdna(void)
  856 {
  857 
  858         if (!hw_float)
  859                 return (0);
  860         critical_enter();
  861         if (PCPU_GET(fpcurthread) == curthread) {
  862                 printf("npxdna: fpcurthread == curthread %d times\n",
  863                     ++err_count);
  864                 stop_emulating();
  865                 critical_exit();
  866                 return (1);
  867         }
  868         if (PCPU_GET(fpcurthread) != NULL) {
  869                 printf("npxdna: fpcurthread = %p (%d), curthread = %p (%d)\n",
  870                        PCPU_GET(fpcurthread),
  871                        PCPU_GET(fpcurthread)->td_proc->p_pid,
  872                        curthread, curthread->td_proc->p_pid);
  873                 panic("npxdna");
  874         }
  875         stop_emulating();
  876         /*
  877          * Record new context early in case frstor causes a trap.
  878          */
  879         PCPU_SET(fpcurthread, curthread);
  880 
  881 #ifdef CPU_ENABLE_SSE
  882         if (cpu_fxsr)
  883                 fpu_clean_state();
  884 #endif
  885 
  886         if ((curpcb->pcb_flags & PCB_NPXINITDONE) == 0) {
  887                 /*
  888                  * This is the first time this thread has used the FPU or
  889                  * the PCB doesn't contain a clean FPU state.  Explicitly
  890                  * load an initial state.
  891                  *
  892                  * We prefer to restore the state from the actual save
  893                  * area in PCB instead of directly loading from
  894                  * npx_initialstate, to ignite the XSAVEOPT
  895                  * tracking engine.
  896                  */
  897                 bcopy(npx_initialstate, curpcb->pcb_save, cpu_max_ext_state_size);
  898                 fpurstor(curpcb->pcb_save);
  899                 if (curpcb->pcb_initial_npxcw != __INITIAL_NPXCW__)
  900                         fldcw(curpcb->pcb_initial_npxcw);
  901                 curpcb->pcb_flags |= PCB_NPXINITDONE;
  902                 if (PCB_USER_FPU(curpcb))
  903                         curpcb->pcb_flags |= PCB_NPXUSERINITDONE;
  904         } else {
  905                 fpurstor(curpcb->pcb_save);
  906         }
  907         critical_exit();
  908 
  909         return (1);
  910 }
  911 
  912 /*
  913  * Wrapper for fpusave() called from context switch routines.
  914  *
  915  * npxsave() must be called with interrupts disabled, so that it clears
  916  * fpcurthread atomically with saving the state.  We require callers to do the
  917  * disabling, since most callers need to disable interrupts anyway to call
  918  * npxsave() atomically with checking fpcurthread.
  919  */
  920 void
  921 npxsave(addr)
  922         union savefpu *addr;
  923 {
  924 
  925         stop_emulating();
  926 #ifdef CPU_ENABLE_SSE
  927         if (use_xsaveopt)
  928                 xsaveopt((char *)addr, xsave_mask);
  929         else
  930 #endif
  931                 fpusave(addr);
  932         start_emulating();
  933         PCPU_SET(fpcurthread, NULL);
  934 }
  935 
  936 /*
  937  * Unconditionally save the current co-processor state across suspend and
  938  * resume.
  939  */
  940 void
  941 npxsuspend(union savefpu *addr)
  942 {
  943         register_t cr0;
  944 
  945         if (!hw_float)
  946                 return;
  947         if (PCPU_GET(fpcurthread) == NULL) {
  948                 bcopy(npx_initialstate, addr, cpu_max_ext_state_size);
  949                 return;
  950         }
  951         cr0 = rcr0();
  952         stop_emulating();
  953         fpusave(addr);
  954         load_cr0(cr0);
  955 }
  956 
  957 void
  958 npxresume(union savefpu *addr)
  959 {
  960         register_t cr0;
  961 
  962         if (!hw_float)
  963                 return;
  964 
  965         cr0 = rcr0();
  966         npxinit(false);
  967         stop_emulating();
  968         fpurstor(addr);
  969         load_cr0(cr0);
  970 }
  971 
  972 void
  973 npxdrop()
  974 {
  975         struct thread *td;
  976 
  977         /*
  978          * Discard pending exceptions in the !cpu_fxsr case so that unmasked
  979          * ones don't cause a panic on the next frstor.
  980          */
  981 #ifdef CPU_ENABLE_SSE
  982         if (!cpu_fxsr)
  983 #endif
  984                 fnclex();
  985 
  986         td = PCPU_GET(fpcurthread);
  987         KASSERT(td == curthread, ("fpudrop: fpcurthread != curthread"));
  988         CRITICAL_ASSERT(td);
  989         PCPU_SET(fpcurthread, NULL);
  990         td->td_pcb->pcb_flags &= ~PCB_NPXINITDONE;
  991         start_emulating();
  992 }
  993 
  994 /*
  995  * Get the user state of the FPU into pcb->pcb_user_save without
  996  * dropping ownership (if possible).  It returns the FPU ownership
  997  * status.
  998  */
  999 int
 1000 npxgetregs(struct thread *td)
 1001 {
 1002         struct pcb *pcb;
 1003 #ifdef CPU_ENABLE_SSE
 1004         uint64_t *xstate_bv, bit;
 1005         char *sa;
 1006         int max_ext_n, i;
 1007 #endif
 1008         int owned;
 1009 
 1010         if (!hw_float)
 1011                 return (_MC_FPOWNED_NONE);
 1012 
 1013         pcb = td->td_pcb;
 1014         if ((pcb->pcb_flags & PCB_NPXINITDONE) == 0) {
 1015                 bcopy(npx_initialstate, get_pcb_user_save_pcb(pcb),
 1016                     cpu_max_ext_state_size);
 1017                 SET_FPU_CW(get_pcb_user_save_pcb(pcb), pcb->pcb_initial_npxcw);
 1018                 npxuserinited(td);
 1019                 return (_MC_FPOWNED_PCB);
 1020         }
 1021         critical_enter();
 1022         if (td == PCPU_GET(fpcurthread)) {
 1023                 fpusave(get_pcb_user_save_pcb(pcb));
 1024 #ifdef CPU_ENABLE_SSE
 1025                 if (!cpu_fxsr)
 1026 #endif
 1027                         /*
 1028                          * fnsave initializes the FPU and destroys whatever
 1029                          * context it contains.  Make sure the FPU owner
 1030                          * starts with a clean state next time.
 1031                          */
 1032                         npxdrop();
 1033                 owned = _MC_FPOWNED_FPU;
 1034         } else {
 1035                 owned = _MC_FPOWNED_PCB;
 1036         }
 1037         critical_exit();
 1038 #ifdef CPU_ENABLE_SSE
 1039         if (use_xsave) {
 1040                 /*
 1041                  * Handle partially saved state.
 1042                  */
 1043                 sa = (char *)get_pcb_user_save_pcb(pcb);
 1044                 xstate_bv = (uint64_t *)(sa + sizeof(union savefpu) +
 1045                     offsetof(struct xstate_hdr, xstate_bv));
 1046                 if (xsave_mask >> 32 != 0)
 1047                         max_ext_n = fls(xsave_mask >> 32) + 32;
 1048                 else
 1049                         max_ext_n = fls(xsave_mask);
 1050                 for (i = 0; i < max_ext_n; i++) {
 1051                         bit = 1ULL << i;
 1052                         if ((xsave_mask & bit) == 0 || (*xstate_bv & bit) != 0)
 1053                                 continue;
 1054                         bcopy((char *)npx_initialstate +
 1055                             xsave_area_desc[i].offset,
 1056                             sa + xsave_area_desc[i].offset,
 1057                             xsave_area_desc[i].size);
 1058                         *xstate_bv |= bit;
 1059                 }
 1060         }
 1061 #endif
 1062         return (owned);
 1063 }
 1064 
 1065 void
 1066 npxuserinited(struct thread *td)
 1067 {
 1068         struct pcb *pcb;
 1069 
 1070         pcb = td->td_pcb;
 1071         if (PCB_USER_FPU(pcb))
 1072                 pcb->pcb_flags |= PCB_NPXINITDONE;
 1073         pcb->pcb_flags |= PCB_NPXUSERINITDONE;
 1074 }
 1075 
 1076 #ifdef CPU_ENABLE_SSE
 1077 int
 1078 npxsetxstate(struct thread *td, char *xfpustate, size_t xfpustate_size)
 1079 {
 1080         struct xstate_hdr *hdr, *ehdr;
 1081         size_t len, max_len;
 1082         uint64_t bv;
 1083 
 1084         /* XXXKIB should we clear all extended state in xstate_bv instead ? */
 1085         if (xfpustate == NULL)
 1086                 return (0);
 1087         if (!use_xsave)
 1088                 return (EOPNOTSUPP);
 1089 
 1090         len = xfpustate_size;
 1091         if (len < sizeof(struct xstate_hdr))
 1092                 return (EINVAL);
 1093         max_len = cpu_max_ext_state_size - sizeof(union savefpu);
 1094         if (len > max_len)
 1095                 return (EINVAL);
 1096 
 1097         ehdr = (struct xstate_hdr *)xfpustate;
 1098         bv = ehdr->xstate_bv;
 1099 
 1100         /*
 1101          * Avoid #gp.
 1102          */
 1103         if (bv & ~xsave_mask)
 1104                 return (EINVAL);
 1105 
 1106         hdr = (struct xstate_hdr *)(get_pcb_user_save_td(td) + 1);
 1107 
 1108         hdr->xstate_bv = bv;
 1109         bcopy(xfpustate + sizeof(struct xstate_hdr),
 1110             (char *)(hdr + 1), len - sizeof(struct xstate_hdr));
 1111 
 1112         return (0);
 1113 }
 1114 #endif
 1115 
 1116 int
 1117 npxsetregs(struct thread *td, union savefpu *addr, char *xfpustate,
 1118         size_t xfpustate_size)
 1119 {
 1120         struct pcb *pcb;
 1121 #ifdef CPU_ENABLE_SSE
 1122         int error;
 1123 #endif
 1124 
 1125         if (!hw_float)
 1126                 return (ENXIO);
 1127 
 1128         pcb = td->td_pcb;
 1129         critical_enter();
 1130         if (td == PCPU_GET(fpcurthread) && PCB_USER_FPU(pcb)) {
 1131 #ifdef CPU_ENABLE_SSE
 1132                 error = npxsetxstate(td, xfpustate, xfpustate_size);
 1133                 if (error != 0) {
 1134                         critical_exit();
 1135                         return (error);
 1136                 }
 1137                 if (!cpu_fxsr)
 1138 #endif
 1139                         fnclex();       /* As in npxdrop(). */
 1140                 bcopy(addr, get_pcb_user_save_td(td), sizeof(*addr));
 1141                 fpurstor(get_pcb_user_save_td(td));
 1142                 critical_exit();
 1143                 pcb->pcb_flags |= PCB_NPXUSERINITDONE | PCB_NPXINITDONE;
 1144         } else {
 1145                 critical_exit();
 1146 #ifdef CPU_ENABLE_SSE
 1147                 error = npxsetxstate(td, xfpustate, xfpustate_size);
 1148                 if (error != 0)
 1149                         return (error);
 1150 #endif
 1151                 bcopy(addr, get_pcb_user_save_td(td), sizeof(*addr));
 1152                 npxuserinited(td);
 1153         }
 1154         return (0);
 1155 }
 1156 
 1157 static void
 1158 fpusave(addr)
 1159         union savefpu *addr;
 1160 {
 1161         
 1162 #ifdef CPU_ENABLE_SSE
 1163         if (use_xsave)
 1164                 xsave((char *)addr, xsave_mask);
 1165         else if (cpu_fxsr)
 1166                 fxsave(addr);
 1167         else
 1168 #endif
 1169                 fnsave(addr);
 1170 }
 1171 
 1172 #ifdef CPU_ENABLE_SSE
 1173 /*
 1174  * On AuthenticAMD processors, the fxrstor instruction does not restore
 1175  * the x87's stored last instruction pointer, last data pointer, and last
 1176  * opcode values, except in the rare case in which the exception summary
 1177  * (ES) bit in the x87 status word is set to 1.
 1178  *
 1179  * In order to avoid leaking this information across processes, we clean
 1180  * these values by performing a dummy load before executing fxrstor().
 1181  */
 1182 static void
 1183 fpu_clean_state(void)
 1184 {
 1185         static float dummy_variable = 0.0;
 1186         u_short status;
 1187 
 1188         /*
 1189          * Clear the ES bit in the x87 status word if it is currently
 1190          * set, in order to avoid causing a fault in the upcoming load.
 1191          */
 1192         fnstsw(&status);
 1193         if (status & 0x80)
 1194                 fnclex();
 1195 
 1196         /*
 1197          * Load the dummy variable into the x87 stack.  This mangles
 1198          * the x87 stack, but we don't care since we're about to call
 1199          * fxrstor() anyway.
 1200          */
 1201         __asm __volatile("ffree %%st(7); flds %0" : : "m" (dummy_variable));
 1202 }
 1203 #endif /* CPU_ENABLE_SSE */
 1204 
 1205 static void
 1206 fpurstor(addr)
 1207         union savefpu *addr;
 1208 {
 1209 
 1210 #ifdef CPU_ENABLE_SSE
 1211         if (use_xsave)
 1212                 xrstor((char *)addr, xsave_mask);
 1213         else if (cpu_fxsr)
 1214                 fxrstor(addr);
 1215         else
 1216 #endif
 1217                 frstor(addr);
 1218 }
 1219 
 1220 #ifdef DEV_ISA
 1221 /*
 1222  * This sucks up the legacy ISA support assignments from PNPBIOS/ACPI.
 1223  */
 1224 static struct isa_pnp_id npxisa_ids[] = {
 1225         { 0x040cd041, "Legacy ISA coprocessor support" }, /* PNP0C04 */
 1226         { 0 }
 1227 };
 1228 
 1229 static int
 1230 npxisa_probe(device_t dev)
 1231 {
 1232         int result;
 1233         if ((result = ISA_PNP_PROBE(device_get_parent(dev), dev, npxisa_ids)) <= 0) {
 1234                 device_quiet(dev);
 1235         }
 1236         return(result);
 1237 }
 1238 
 1239 static int
 1240 npxisa_attach(device_t dev)
 1241 {
 1242         return (0);
 1243 }
 1244 
 1245 static device_method_t npxisa_methods[] = {
 1246         /* Device interface */
 1247         DEVMETHOD(device_probe,         npxisa_probe),
 1248         DEVMETHOD(device_attach,        npxisa_attach),
 1249         DEVMETHOD(device_detach,        bus_generic_detach),
 1250         DEVMETHOD(device_shutdown,      bus_generic_shutdown),
 1251         DEVMETHOD(device_suspend,       bus_generic_suspend),
 1252         DEVMETHOD(device_resume,        bus_generic_resume),
 1253         
 1254         { 0, 0 }
 1255 };
 1256 
 1257 static driver_t npxisa_driver = {
 1258         "npxisa",
 1259         npxisa_methods,
 1260         1,                      /* no softc */
 1261 };
 1262 
 1263 static devclass_t npxisa_devclass;
 1264 
 1265 DRIVER_MODULE(npxisa, isa, npxisa_driver, npxisa_devclass, 0, 0);
 1266 #ifndef PC98
 1267 DRIVER_MODULE(npxisa, acpi, npxisa_driver, npxisa_devclass, 0, 0);
 1268 #endif
 1269 #endif /* DEV_ISA */
 1270 
 1271 static MALLOC_DEFINE(M_FPUKERN_CTX, "fpukern_ctx",
 1272     "Kernel contexts for FPU state");
 1273 
 1274 #define FPU_KERN_CTX_NPXINITDONE 0x01
 1275 #define FPU_KERN_CTX_DUMMY       0x02
 1276 
 1277 struct fpu_kern_ctx {
 1278         union savefpu *prev;
 1279         uint32_t flags;
 1280         char hwstate1[];
 1281 };
 1282 
 1283 struct fpu_kern_ctx *
 1284 fpu_kern_alloc_ctx(u_int flags)
 1285 {
 1286         struct fpu_kern_ctx *res;
 1287         size_t sz;
 1288 
 1289         sz = sizeof(struct fpu_kern_ctx) + XSAVE_AREA_ALIGN +
 1290             cpu_max_ext_state_size;
 1291         res = malloc(sz, M_FPUKERN_CTX, ((flags & FPU_KERN_NOWAIT) ?
 1292             M_NOWAIT : M_WAITOK) | M_ZERO);
 1293         return (res);
 1294 }
 1295 
 1296 void
 1297 fpu_kern_free_ctx(struct fpu_kern_ctx *ctx)
 1298 {
 1299 
 1300         /* XXXKIB clear the memory ? */
 1301         free(ctx, M_FPUKERN_CTX);
 1302 }
 1303 
 1304 static union savefpu *
 1305 fpu_kern_ctx_savefpu(struct fpu_kern_ctx *ctx)
 1306 {
 1307         vm_offset_t p;
 1308 
 1309         p = (vm_offset_t)&ctx->hwstate1;
 1310         p = roundup2(p, XSAVE_AREA_ALIGN);
 1311         return ((union savefpu *)p);
 1312 }
 1313 
 1314 int
 1315 fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags)
 1316 {
 1317         struct pcb *pcb;
 1318 
 1319         if ((flags & FPU_KERN_KTHR) != 0 && is_fpu_kern_thread(0)) {
 1320                 ctx->flags = FPU_KERN_CTX_DUMMY;
 1321                 return (0);
 1322         }
 1323         pcb = td->td_pcb;
 1324         KASSERT(!PCB_USER_FPU(pcb) || pcb->pcb_save ==
 1325             get_pcb_user_save_pcb(pcb), ("mangled pcb_save"));
 1326         ctx->flags = 0;
 1327         if ((pcb->pcb_flags & PCB_NPXINITDONE) != 0)
 1328                 ctx->flags |= FPU_KERN_CTX_NPXINITDONE;
 1329         npxexit(td);
 1330         ctx->prev = pcb->pcb_save;
 1331         pcb->pcb_save = fpu_kern_ctx_savefpu(ctx);
 1332         pcb->pcb_flags |= PCB_KERNNPX;
 1333         pcb->pcb_flags &= ~PCB_NPXINITDONE;
 1334         return (0);
 1335 }
 1336 
 1337 int
 1338 fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx)
 1339 {
 1340         struct pcb *pcb;
 1341 
 1342         if (is_fpu_kern_thread(0) && (ctx->flags & FPU_KERN_CTX_DUMMY) != 0)
 1343                 return (0);
 1344         pcb = td->td_pcb;
 1345         critical_enter();
 1346         if (curthread == PCPU_GET(fpcurthread))
 1347                 npxdrop();
 1348         critical_exit();
 1349         pcb->pcb_save = ctx->prev;
 1350         if (pcb->pcb_save == get_pcb_user_save_pcb(pcb)) {
 1351                 if ((pcb->pcb_flags & PCB_NPXUSERINITDONE) != 0)
 1352                         pcb->pcb_flags |= PCB_NPXINITDONE;
 1353                 else
 1354                         pcb->pcb_flags &= ~PCB_NPXINITDONE;
 1355                 pcb->pcb_flags &= ~PCB_KERNNPX;
 1356         } else {
 1357                 if ((ctx->flags & FPU_KERN_CTX_NPXINITDONE) != 0)
 1358                         pcb->pcb_flags |= PCB_NPXINITDONE;
 1359                 else
 1360                         pcb->pcb_flags &= ~PCB_NPXINITDONE;
 1361                 KASSERT(!PCB_USER_FPU(pcb), ("unpaired fpu_kern_leave"));
 1362         }
 1363         return (0);
 1364 }
 1365 
 1366 int
 1367 fpu_kern_thread(u_int flags)
 1368 {
 1369 
 1370         KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0,
 1371             ("Only kthread may use fpu_kern_thread"));
 1372         KASSERT(curpcb->pcb_save == get_pcb_user_save_pcb(curpcb),
 1373             ("mangled pcb_save"));
 1374         KASSERT(PCB_USER_FPU(curpcb), ("recursive call"));
 1375 
 1376         curpcb->pcb_flags |= PCB_KERNNPX;
 1377         return (0);
 1378 }
 1379 
 1380 int
 1381 is_fpu_kern_thread(u_int flags)
 1382 {
 1383 
 1384         if ((curthread->td_pflags & TDP_KTHREAD) == 0)
 1385                 return (0);
 1386         return ((curpcb->pcb_flags & PCB_KERNNPX) != 0);
 1387 }
 1388 
 1389 /*
 1390  * FPU save area alloc/free/init utility routines
 1391  */
 1392 union savefpu *
 1393 fpu_save_area_alloc(void)
 1394 {
 1395 
 1396         return (uma_zalloc(fpu_save_area_zone, 0));
 1397 }
 1398 
 1399 void
 1400 fpu_save_area_free(union savefpu *fsa)
 1401 {
 1402 
 1403         uma_zfree(fpu_save_area_zone, fsa);
 1404 }
 1405 
 1406 void
 1407 fpu_save_area_reset(union savefpu *fsa)
 1408 {
 1409 
 1410         bcopy(npx_initialstate, fsa, cpu_max_ext_state_size);
 1411 }

Cache object: c7fd2f66cba7d72c99a6cc74efa9179e


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.