FreeBSD/Linux Kernel Cross Reference
sys/arm/arm/vfp.c
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2014 Ian Lepore <ian@freebsd.org>
5 * Copyright (c) 2012 Mark Tinguely
6 *
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33
34 #ifdef VFP
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/proc.h>
38 #include <sys/imgact_elf.h>
39 #include <sys/kernel.h>
40
41 #include <machine/armreg.h>
42 #include <machine/elf.h>
43 #include <machine/frame.h>
44 #include <machine/md_var.h>
45 #include <machine/pcb.h>
46 #include <machine/undefined.h>
47 #include <machine/vfp.h>
48
49 /* function prototypes */
50 static int vfp_bounce(u_int, u_int, struct trapframe *, int);
51 static void vfp_restore(struct vfp_state *);
52
53 extern int vfp_exists;
54 static struct undefined_handler vfp10_uh, vfp11_uh;
55 /* If true the VFP unit has 32 double registers, otherwise it has 16 */
56 static int is_d32;
57
58 /*
59 * About .fpu directives in this file...
60 *
61 * We should need simply .fpu vfpv3, but clang 3.5 has a quirk where setting
62 * vfpv3 doesn't imply that vfp2 features are also available -- both have to be
63 * explicitly set to get all the features of both. This is probably a bug in
64 * clang, so it may get fixed and require changes here some day. Other changes
65 * are probably coming in clang too, because there is email and open PRs
66 * indicating they want to completely disable the ability to use .fpu and
67 * similar directives in inline asm. That would be catastrophic for us,
68 * hopefully they come to their senses. There was also some discusion of a new
69 * syntax such as .push fpu=vfpv3; ...; .pop fpu; and that would be ideal for
70 * us, better than what we have now really.
71 *
72 * For gcc, each .fpu directive completely overrides the prior directive, unlike
73 * with clang, but luckily on gcc saying v3 implies all the v2 features as well.
74 */
75
76 #define fmxr(reg, val) \
77 __asm __volatile(" .fpu vfpv2\n .fpu vfpv3\n" \
78 " vmsr " __STRING(reg) ", %0" :: "r"(val));
79
80 #define fmrx(reg) \
81 ({ u_int val = 0;\
82 __asm __volatile(" .fpu vfpv2\n .fpu vfpv3\n" \
83 " vmrs %0, " __STRING(reg) : "=r"(val)); \
84 val; \
85 })
86
87 static u_int
88 get_coprocessorACR(void)
89 {
90 u_int val;
91 __asm __volatile("mrc p15, 0, %0, c1, c0, 2" : "=r" (val) : : "cc");
92 return val;
93 }
94
95 static void
96 set_coprocessorACR(u_int val)
97 {
98 __asm __volatile("mcr p15, 0, %0, c1, c0, 2\n\t"
99 : : "r" (val) : "cc");
100 isb();
101 }
102
103 /* called for each cpu */
104 void
105 vfp_init(void)
106 {
107 u_int fpsid, fpexc, tmp;
108 u_int coproc, vfp_arch;
109
110 coproc = get_coprocessorACR();
111 coproc |= COPROC10 | COPROC11;
112 set_coprocessorACR(coproc);
113
114 fpsid = fmrx(fpsid); /* read the vfp system id */
115 fpexc = fmrx(fpexc); /* read the vfp exception reg */
116
117 if (!(fpsid & VFPSID_HARDSOFT_IMP)) {
118 vfp_exists = 1;
119 is_d32 = 0;
120 PCPU_SET(vfpsid, fpsid); /* save the fpsid */
121 elf_hwcap |= HWCAP_VFP;
122
123 vfp_arch =
124 (fpsid & VFPSID_SUBVERSION2_MASK) >> VFPSID_SUBVERSION_OFF;
125
126 if (vfp_arch >= VFP_ARCH3) {
127 tmp = fmrx(mvfr0);
128 PCPU_SET(vfpmvfr0, tmp);
129 elf_hwcap |= HWCAP_VFPv3;
130
131 if ((tmp & VMVFR0_RB_MASK) == 2) {
132 elf_hwcap |= HWCAP_VFPD32;
133 is_d32 = 1;
134 } else
135 elf_hwcap |= HWCAP_VFPv3D16;
136
137 tmp = fmrx(mvfr1);
138 PCPU_SET(vfpmvfr1, tmp);
139
140 if (PCPU_GET(cpuid) == 0) {
141 if ((tmp & VMVFR1_FZ_MASK) == 0x1) {
142 /* Denormals arithmetic support */
143 initial_fpscr &= ~VFPSCR_FZ;
144 thread0.td_pcb->pcb_vfpstate.fpscr =
145 initial_fpscr;
146 }
147 }
148
149 if ((tmp & VMVFR1_LS_MASK) >> VMVFR1_LS_OFF == 1 &&
150 (tmp & VMVFR1_I_MASK) >> VMVFR1_I_OFF == 1 &&
151 (tmp & VMVFR1_SP_MASK) >> VMVFR1_SP_OFF == 1)
152 elf_hwcap |= HWCAP_NEON;
153 if ((tmp & VMVFR1_FMAC_MASK) >> VMVFR1_FMAC_OFF == 1)
154 elf_hwcap |= HWCAP_VFPv4;
155 }
156
157 /* initialize the coprocess 10 and 11 calls
158 * These are called to restore the registers and enable
159 * the VFP hardware.
160 */
161 if (vfp10_uh.uh_handler == NULL) {
162 vfp10_uh.uh_handler = vfp_bounce;
163 vfp11_uh.uh_handler = vfp_bounce;
164 install_coproc_handler_static(10, &vfp10_uh);
165 install_coproc_handler_static(11, &vfp11_uh);
166 }
167 }
168 }
169
170 SYSINIT(vfp, SI_SUB_CPU, SI_ORDER_ANY, vfp_init, NULL);
171
172 /* start VFP unit, restore the vfp registers from the PCB and retry
173 * the instruction
174 */
175 static int
176 vfp_bounce(u_int addr, u_int insn, struct trapframe *frame, int code)
177 {
178 u_int cpu, fpexc;
179 struct pcb *curpcb;
180 ksiginfo_t ksi;
181
182 if ((code & FAULT_USER) == 0)
183 panic("undefined floating point instruction in supervisor mode");
184
185 critical_enter();
186
187 /*
188 * If the VFP is already on and we got an undefined instruction, then
189 * something tried to executate a truly invalid instruction that maps to
190 * the VFP.
191 */
192 fpexc = fmrx(fpexc);
193 if (fpexc & VFPEXC_EN) {
194 /* Clear any exceptions */
195 fmxr(fpexc, fpexc & ~(VFPEXC_EX | VFPEXC_FP2V));
196
197 /* kill the process - we do not handle emulation */
198 critical_exit();
199
200 if (fpexc & VFPEXC_EX) {
201 /* We have an exception, signal a SIGFPE */
202 ksiginfo_init_trap(&ksi);
203 ksi.ksi_signo = SIGFPE;
204 if (fpexc & VFPEXC_UFC)
205 ksi.ksi_code = FPE_FLTUND;
206 else if (fpexc & VFPEXC_OFC)
207 ksi.ksi_code = FPE_FLTOVF;
208 else if (fpexc & VFPEXC_IOC)
209 ksi.ksi_code = FPE_FLTINV;
210 ksi.ksi_addr = (void *)addr;
211 trapsignal(curthread, &ksi);
212 return 0;
213 }
214
215 return 1;
216 }
217
218 /*
219 * If the last time this thread used the VFP it was on this core, and
220 * the last thread to use the VFP on this core was this thread, then the
221 * VFP state is valid, otherwise restore this thread's state to the VFP.
222 */
223 fmxr(fpexc, fpexc | VFPEXC_EN);
224 curpcb = curthread->td_pcb;
225 cpu = PCPU_GET(cpuid);
226 if (curpcb->pcb_vfpcpu != cpu || curthread != PCPU_GET(fpcurthread)) {
227 vfp_restore(&curpcb->pcb_vfpstate);
228 curpcb->pcb_vfpcpu = cpu;
229 PCPU_SET(fpcurthread, curthread);
230 }
231
232 critical_exit();
233 return (0);
234 }
235
236 /*
237 * Restore the given state to the VFP hardware.
238 */
239 static void
240 vfp_restore(struct vfp_state *vfpsave)
241 {
242 uint32_t fpexc;
243
244 /* On vfpv3 we may need to restore FPINST and FPINST2 */
245 fpexc = vfpsave->fpexec;
246 if (fpexc & VFPEXC_EX) {
247 fmxr(fpinst, vfpsave->fpinst);
248 if (fpexc & VFPEXC_FP2V)
249 fmxr(fpinst2, vfpsave->fpinst2);
250 }
251 fmxr(fpscr, vfpsave->fpscr);
252
253 __asm __volatile(
254 " .fpu vfpv2\n"
255 " .fpu vfpv3\n"
256 " vldmia %0!, {d0-d15}\n" /* d0-d15 */
257 " cmp %1, #0\n" /* -D16 or -D32? */
258 " vldmiane %0!, {d16-d31}\n" /* d16-d31 */
259 " addeq %0, %0, #128\n" /* skip missing regs */
260 : "+&r" (vfpsave) : "r" (is_d32) : "cc"
261 );
262
263 fmxr(fpexc, fpexc);
264 }
265
266 /*
267 * If the VFP is on, save its current state and turn it off if requested to do
268 * so. If the VFP is not on, does not change the values at *vfpsave. Caller is
269 * responsible for preventing a context switch while this is running.
270 */
271 void
272 vfp_store(struct vfp_state *vfpsave, boolean_t disable_vfp)
273 {
274 uint32_t fpexc;
275
276 fpexc = fmrx(fpexc); /* Is the vfp enabled? */
277 if (fpexc & VFPEXC_EN) {
278 vfpsave->fpexec = fpexc;
279 vfpsave->fpscr = fmrx(fpscr);
280
281 /* On vfpv3 we may need to save FPINST and FPINST2 */
282 if (fpexc & VFPEXC_EX) {
283 vfpsave->fpinst = fmrx(fpinst);
284 if (fpexc & VFPEXC_FP2V)
285 vfpsave->fpinst2 = fmrx(fpinst2);
286 fpexc &= ~VFPEXC_EX;
287 }
288
289 __asm __volatile(
290 " .fpu vfpv2\n"
291 " .fpu vfpv3\n"
292 " vstmia %0!, {d0-d15}\n" /* d0-d15 */
293 " cmp %1, #0\n" /* -D16 or -D32? */
294 " vstmiane %0!, {d16-d31}\n" /* d16-d31 */
295 " addeq %0, %0, #128\n" /* skip missing regs */
296 : "+&r" (vfpsave) : "r" (is_d32) : "cc"
297 );
298
299 if (disable_vfp)
300 fmxr(fpexc , fpexc & ~VFPEXC_EN);
301 }
302 }
303
304 /*
305 * The current thread is dying. If the state currently in the hardware belongs
306 * to the current thread, set fpcurthread to NULL to indicate that the VFP
307 * hardware state does not belong to any thread. If the VFP is on, turn it off.
308 * Called only from cpu_throw(), so we don't have to worry about a context
309 * switch here.
310 */
311 void
312 vfp_discard(struct thread *td)
313 {
314 u_int tmp;
315
316 if (PCPU_GET(fpcurthread) == td)
317 PCPU_SET(fpcurthread, NULL);
318
319 tmp = fmrx(fpexc);
320 if (tmp & VFPEXC_EN)
321 fmxr(fpexc, tmp & ~VFPEXC_EN);
322 }
323
324 #endif
Cache object: c8f741ef39faffe89119ef33d3b32f4f
|