1 /*-
2 * Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com>
3 * All rights reserved.
4 *
5 * Portions of this software were developed by SRI International and the
6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract
7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
8 *
9 * Portions of this software were developed by the University of Cambridge
10 * Computer Laboratory as part of the CTSRD Project, with support from the
11 * UK Higher Education Innovation Fund (HEIF).
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35 #include "assym.inc"
36 #include "opt_sched.h"
37
38 #include <machine/param.h>
39 #include <machine/asm.h>
40 #include <machine/riscvreg.h>
41 #include <machine/pte.h>
42
43 __FBSDID("$FreeBSD$");
44
45 #ifdef FPE
46 .macro __fpe_state_save p
47 /*
48 * Enable FPE usage in supervisor mode,
49 * so we can access registers.
50 */
51 li t0, SSTATUS_FS_INITIAL
52 csrs sstatus, t0
53
54 /* Store registers */
55 frcsr t0
56 sd t0, (PCB_FCSR)(\p)
57 fsd f0, (PCB_X + 0 * 16)(\p)
58 fsd f1, (PCB_X + 1 * 16)(\p)
59 fsd f2, (PCB_X + 2 * 16)(\p)
60 fsd f3, (PCB_X + 3 * 16)(\p)
61 fsd f4, (PCB_X + 4 * 16)(\p)
62 fsd f5, (PCB_X + 5 * 16)(\p)
63 fsd f6, (PCB_X + 6 * 16)(\p)
64 fsd f7, (PCB_X + 7 * 16)(\p)
65 fsd f8, (PCB_X + 8 * 16)(\p)
66 fsd f9, (PCB_X + 9 * 16)(\p)
67 fsd f10, (PCB_X + 10 * 16)(\p)
68 fsd f11, (PCB_X + 11 * 16)(\p)
69 fsd f12, (PCB_X + 12 * 16)(\p)
70 fsd f13, (PCB_X + 13 * 16)(\p)
71 fsd f14, (PCB_X + 14 * 16)(\p)
72 fsd f15, (PCB_X + 15 * 16)(\p)
73 fsd f16, (PCB_X + 16 * 16)(\p)
74 fsd f17, (PCB_X + 17 * 16)(\p)
75 fsd f18, (PCB_X + 18 * 16)(\p)
76 fsd f19, (PCB_X + 19 * 16)(\p)
77 fsd f20, (PCB_X + 20 * 16)(\p)
78 fsd f21, (PCB_X + 21 * 16)(\p)
79 fsd f22, (PCB_X + 22 * 16)(\p)
80 fsd f23, (PCB_X + 23 * 16)(\p)
81 fsd f24, (PCB_X + 24 * 16)(\p)
82 fsd f25, (PCB_X + 25 * 16)(\p)
83 fsd f26, (PCB_X + 26 * 16)(\p)
84 fsd f27, (PCB_X + 27 * 16)(\p)
85 fsd f28, (PCB_X + 28 * 16)(\p)
86 fsd f29, (PCB_X + 29 * 16)(\p)
87 fsd f30, (PCB_X + 30 * 16)(\p)
88 fsd f31, (PCB_X + 31 * 16)(\p)
89
90 /* Disable FPE usage in supervisor mode. */
91 li t0, SSTATUS_FS_MASK
92 csrc sstatus, t0
93 .endm
94
95 .macro __fpe_state_load p
96 /*
97 * Enable FPE usage in supervisor mode,
98 * so we can access registers.
99 */
100 li t0, SSTATUS_FS_INITIAL
101 csrs sstatus, t0
102
103 /* Restore registers */
104 ld t0, (PCB_FCSR)(\p)
105 fscsr t0
106 fld f0, (PCB_X + 0 * 16)(\p)
107 fld f1, (PCB_X + 1 * 16)(\p)
108 fld f2, (PCB_X + 2 * 16)(\p)
109 fld f3, (PCB_X + 3 * 16)(\p)
110 fld f4, (PCB_X + 4 * 16)(\p)
111 fld f5, (PCB_X + 5 * 16)(\p)
112 fld f6, (PCB_X + 6 * 16)(\p)
113 fld f7, (PCB_X + 7 * 16)(\p)
114 fld f8, (PCB_X + 8 * 16)(\p)
115 fld f9, (PCB_X + 9 * 16)(\p)
116 fld f10, (PCB_X + 10 * 16)(\p)
117 fld f11, (PCB_X + 11 * 16)(\p)
118 fld f12, (PCB_X + 12 * 16)(\p)
119 fld f13, (PCB_X + 13 * 16)(\p)
120 fld f14, (PCB_X + 14 * 16)(\p)
121 fld f15, (PCB_X + 15 * 16)(\p)
122 fld f16, (PCB_X + 16 * 16)(\p)
123 fld f17, (PCB_X + 17 * 16)(\p)
124 fld f18, (PCB_X + 18 * 16)(\p)
125 fld f19, (PCB_X + 19 * 16)(\p)
126 fld f20, (PCB_X + 20 * 16)(\p)
127 fld f21, (PCB_X + 21 * 16)(\p)
128 fld f22, (PCB_X + 22 * 16)(\p)
129 fld f23, (PCB_X + 23 * 16)(\p)
130 fld f24, (PCB_X + 24 * 16)(\p)
131 fld f25, (PCB_X + 25 * 16)(\p)
132 fld f26, (PCB_X + 26 * 16)(\p)
133 fld f27, (PCB_X + 27 * 16)(\p)
134 fld f28, (PCB_X + 28 * 16)(\p)
135 fld f29, (PCB_X + 29 * 16)(\p)
136 fld f30, (PCB_X + 30 * 16)(\p)
137 fld f31, (PCB_X + 31 * 16)(\p)
138
139 /* Disable FPE usage in supervisor mode. */
140 li t0, SSTATUS_FS_MASK
141 csrc sstatus, t0
142 .endm
143
144 /*
145 * void
146 * fpe_state_save(struct thread *td)
147 */
148 ENTRY(fpe_state_save)
149 /* Get pointer to PCB */
150 ld a0, TD_PCB(a0)
151 __fpe_state_save a0
152 ret
153 END(fpe_state_save)
154 #endif /* FPE */
155
156 /*
157 * void
158 * fpe_state_clear(void)
159 */
160 ENTRY(fpe_state_clear)
161 /*
162 * Enable FPE usage in supervisor mode,
163 * so we can access registers.
164 */
165 li t0, SSTATUS_FS_INITIAL
166 csrs sstatus, t0
167
168 fscsr zero
169 fcvt.d.l f0, zero
170 fcvt.d.l f1, zero
171 fcvt.d.l f2, zero
172 fcvt.d.l f3, zero
173 fcvt.d.l f4, zero
174 fcvt.d.l f5, zero
175 fcvt.d.l f6, zero
176 fcvt.d.l f7, zero
177 fcvt.d.l f8, zero
178 fcvt.d.l f9, zero
179 fcvt.d.l f10, zero
180 fcvt.d.l f11, zero
181 fcvt.d.l f12, zero
182 fcvt.d.l f13, zero
183 fcvt.d.l f14, zero
184 fcvt.d.l f15, zero
185 fcvt.d.l f16, zero
186 fcvt.d.l f17, zero
187 fcvt.d.l f18, zero
188 fcvt.d.l f19, zero
189 fcvt.d.l f20, zero
190 fcvt.d.l f21, zero
191 fcvt.d.l f22, zero
192 fcvt.d.l f23, zero
193 fcvt.d.l f24, zero
194 fcvt.d.l f25, zero
195 fcvt.d.l f26, zero
196 fcvt.d.l f27, zero
197 fcvt.d.l f28, zero
198 fcvt.d.l f29, zero
199 fcvt.d.l f30, zero
200 fcvt.d.l f31, zero
201
202 /* Disable FPE usage in supervisor mode. */
203 li t0, SSTATUS_FS_MASK
204 csrc sstatus, t0
205
206 ret
207 END(fpe_state_clear)
208
209 /*
210 * void cpu_throw(struct thread *old __unused, struct thread *new)
211 */
212 ENTRY(cpu_throw)
213 /* Activate the new thread's pmap. */
214 mv s0, a1
215 mv a0, a1
216 call _C_LABEL(pmap_activate_sw)
217 mv a0, s0
218
219 /* Store the new curthread */
220 sd a0, PC_CURTHREAD(tp)
221 /* And the new pcb */
222 ld x13, TD_PCB(a0)
223 sd x13, PC_CURPCB(tp)
224
225 /* Load registers */
226 ld ra, (PCB_RA)(x13)
227 ld sp, (PCB_SP)(x13)
228
229 /* s[0-11] */
230 ld s0, (PCB_S + 0 * 8)(x13)
231 ld s1, (PCB_S + 1 * 8)(x13)
232 ld s2, (PCB_S + 2 * 8)(x13)
233 ld s3, (PCB_S + 3 * 8)(x13)
234 ld s4, (PCB_S + 4 * 8)(x13)
235 ld s5, (PCB_S + 5 * 8)(x13)
236 ld s6, (PCB_S + 6 * 8)(x13)
237 ld s7, (PCB_S + 7 * 8)(x13)
238 ld s8, (PCB_S + 8 * 8)(x13)
239 ld s9, (PCB_S + 9 * 8)(x13)
240 ld s10, (PCB_S + 10 * 8)(x13)
241 ld s11, (PCB_S + 11 * 8)(x13)
242
243 #ifdef FPE
244 /* Is FPE enabled for new thread? */
245 ld t0, TD_FRAME(a0)
246 ld t1, (TF_SSTATUS)(t0)
247 li t2, SSTATUS_FS_MASK
248 and t3, t1, t2
249 beqz t3, 1f /* No, skip. */
250
251 /* Restore registers. */
252 __fpe_state_load x13
253 1:
254 #endif
255
256 ret
257 END(cpu_throw)
258
259 /*
260 * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx)
261 *
262 * a0 = old
263 * a1 = new
264 * a2 = mtx
265 * x3 to x7, x16 and x17 are caller saved
266 */
267 ENTRY(cpu_switch)
268 /* Store the new curthread */
269 sd a1, PC_CURTHREAD(tp)
270 /* And the new pcb */
271 ld x13, TD_PCB(a1)
272 sd x13, PC_CURPCB(tp)
273
274 /* Save the old context. */
275 ld x13, TD_PCB(a0)
276
277 /* Store ra, sp and the callee-saved registers */
278 sd ra, (PCB_RA)(x13)
279 sd sp, (PCB_SP)(x13)
280
281 /* s[0-11] */
282 sd s0, (PCB_S + 0 * 8)(x13)
283 sd s1, (PCB_S + 1 * 8)(x13)
284 sd s2, (PCB_S + 2 * 8)(x13)
285 sd s3, (PCB_S + 3 * 8)(x13)
286 sd s4, (PCB_S + 4 * 8)(x13)
287 sd s5, (PCB_S + 5 * 8)(x13)
288 sd s6, (PCB_S + 6 * 8)(x13)
289 sd s7, (PCB_S + 7 * 8)(x13)
290 sd s8, (PCB_S + 8 * 8)(x13)
291 sd s9, (PCB_S + 9 * 8)(x13)
292 sd s10, (PCB_S + 10 * 8)(x13)
293 sd s11, (PCB_S + 11 * 8)(x13)
294
295 #ifdef FPE
296 /*
297 * Is FPE enabled and is it in dirty state
298 * for the old thread?
299 */
300 ld t0, TD_FRAME(a0)
301 ld t1, (TF_SSTATUS)(t0)
302 li t2, SSTATUS_FS_MASK
303 and t3, t1, t2
304 li t2, SSTATUS_FS_DIRTY
305 bne t3, t2, 1f /* No, skip. */
306
307 /* Yes, mark FPE state clean and save registers. */
308 li t2, ~SSTATUS_FS_MASK
309 and t3, t1, t2
310 li t2, SSTATUS_FS_CLEAN
311 or t3, t3, t2
312 sd t3, (TF_SSTATUS)(t0)
313
314 __fpe_state_save x13
315 1:
316 #endif
317
318 /* Activate the new thread's pmap */
319 mv s0, a0
320 mv s1, a1
321 mv s2, a2
322 mv a0, a1
323 call _C_LABEL(pmap_activate_sw)
324 mv a1, s1
325
326 /* Release the old thread */
327 sd s2, TD_LOCK(s0)
328 #if defined(SCHED_ULE) && defined(SMP)
329 /* Spin if TD_LOCK points to a blocked_lock */
330 la s2, _C_LABEL(blocked_lock)
331 1:
332 ld t0, TD_LOCK(a1)
333 beq t0, s2, 1b
334 #endif
335 /*
336 * Restore the saved context.
337 */
338 ld x13, TD_PCB(a1)
339
340 /* Restore the registers */
341 ld ra, (PCB_RA)(x13)
342 ld sp, (PCB_SP)(x13)
343
344 /* s[0-11] */
345 ld s0, (PCB_S + 0 * 8)(x13)
346 ld s1, (PCB_S + 1 * 8)(x13)
347 ld s2, (PCB_S + 2 * 8)(x13)
348 ld s3, (PCB_S + 3 * 8)(x13)
349 ld s4, (PCB_S + 4 * 8)(x13)
350 ld s5, (PCB_S + 5 * 8)(x13)
351 ld s6, (PCB_S + 6 * 8)(x13)
352 ld s7, (PCB_S + 7 * 8)(x13)
353 ld s8, (PCB_S + 8 * 8)(x13)
354 ld s9, (PCB_S + 9 * 8)(x13)
355 ld s10, (PCB_S + 10 * 8)(x13)
356 ld s11, (PCB_S + 11 * 8)(x13)
357
358 #ifdef FPE
359 /* Is FPE enabled for new thread? */
360 ld t0, TD_FRAME(a1)
361 ld t1, (TF_SSTATUS)(t0)
362 li t2, SSTATUS_FS_MASK
363 and t3, t1, t2
364 beqz t3, 1f /* No, skip. */
365
366 /* Restore registers. */
367 __fpe_state_load x13
368 1:
369 #endif
370
371 ret
372 .Lcpu_switch_panic_str:
373 .asciz "cpu_switch: %p\0"
374 END(cpu_switch)
375
376 /*
377 * fork_exit(void (*callout)(void *, struct trapframe *), void *arg,
378 * struct trapframe *frame)
379 */
380
381 ENTRY(fork_trampoline)
382 mv a0, s0
383 mv a1, s1
384 mv a2, sp
385 call _C_LABEL(fork_exit)
386
387 /* Restore sstatus */
388 ld t0, (TF_SSTATUS)(sp)
389 /* Ensure interrupts disabled */
390 li t1, ~SSTATUS_SIE
391 and t0, t0, t1
392 csrw sstatus, t0
393
394 /* Restore exception program counter */
395 ld t0, (TF_SEPC)(sp)
396 csrw sepc, t0
397
398 /* Restore the registers */
399 ld t0, (TF_T + 0 * 8)(sp)
400 ld t1, (TF_T + 1 * 8)(sp)
401 ld t2, (TF_T + 2 * 8)(sp)
402 ld t3, (TF_T + 3 * 8)(sp)
403 ld t4, (TF_T + 4 * 8)(sp)
404 ld t5, (TF_T + 5 * 8)(sp)
405 ld t6, (TF_T + 6 * 8)(sp)
406
407 ld s0, (TF_S + 0 * 8)(sp)
408 ld s1, (TF_S + 1 * 8)(sp)
409 ld s2, (TF_S + 2 * 8)(sp)
410 ld s3, (TF_S + 3 * 8)(sp)
411 ld s4, (TF_S + 4 * 8)(sp)
412 ld s5, (TF_S + 5 * 8)(sp)
413 ld s6, (TF_S + 6 * 8)(sp)
414 ld s7, (TF_S + 7 * 8)(sp)
415 ld s8, (TF_S + 8 * 8)(sp)
416 ld s9, (TF_S + 9 * 8)(sp)
417 ld s10, (TF_S + 10 * 8)(sp)
418 ld s11, (TF_S + 11 * 8)(sp)
419
420 ld a0, (TF_A + 0 * 8)(sp)
421 ld a1, (TF_A + 1 * 8)(sp)
422 ld a2, (TF_A + 2 * 8)(sp)
423 ld a3, (TF_A + 3 * 8)(sp)
424 ld a4, (TF_A + 4 * 8)(sp)
425 ld a5, (TF_A + 5 * 8)(sp)
426 ld a6, (TF_A + 6 * 8)(sp)
427 ld a7, (TF_A + 7 * 8)(sp)
428
429 /* Load user ra and gp */
430 ld ra, (TF_RA)(sp)
431 ld gp, (TF_GP)(sp)
432
433 /*
434 * Store our pcpup on stack, we will load it back
435 * on kernel mode trap.
436 */
437 sd tp, (TF_SIZE)(sp)
438 ld tp, (TF_TP)(sp)
439
440 /* Save kernel stack so we can use it doing a user trap */
441 addi sp, sp, TF_SIZE
442 csrw sscratch, sp
443
444 /* Load user stack */
445 ld sp, (TF_SP - TF_SIZE)(sp)
446
447 sret
448 END(fork_trampoline)
449
450 ENTRY(savectx)
451 /* Store ra, sp and the callee-saved registers */
452 sd ra, (PCB_RA)(a0)
453 sd sp, (PCB_SP)(a0)
454 sd tp, (PCB_TP)(a0)
455 sd gp, (PCB_GP)(a0)
456
457 /* s[0-11] */
458 sd s0, (PCB_S + 0 * 8)(a0)
459 sd s1, (PCB_S + 1 * 8)(a0)
460 sd s2, (PCB_S + 2 * 8)(a0)
461 sd s3, (PCB_S + 3 * 8)(a0)
462 sd s4, (PCB_S + 4 * 8)(a0)
463 sd s5, (PCB_S + 5 * 8)(a0)
464 sd s6, (PCB_S + 6 * 8)(a0)
465 sd s7, (PCB_S + 7 * 8)(a0)
466 sd s8, (PCB_S + 8 * 8)(a0)
467 sd s9, (PCB_S + 9 * 8)(a0)
468 sd s10, (PCB_S + 10 * 8)(a0)
469 sd s11, (PCB_S + 11 * 8)(a0)
470
471 #ifdef FPE
472 __fpe_state_save a0
473 #endif
474 ret
475 END(savectx)
Cache object: ac7ceafa00cf518d8118790189575b91
|