1 /*-
2 * Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com>
3 * All rights reserved.
4 *
5 * Portions of this software were developed by SRI International and the
6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract
7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
8 *
9 * Portions of this software were developed by the University of Cambridge
10 * Computer Laboratory as part of the CTSRD Project, with support from the
11 * UK Higher Education Innovation Fund (HEIF).
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35 #include "assym.inc"
36 #include "opt_sched.h"
37
38 #include <machine/param.h>
39 #include <machine/asm.h>
40 #include <machine/riscvreg.h>
41 #include <machine/pte.h>
42
43 __FBSDID("$FreeBSD: releng/12.0/sys/riscv/riscv/swtch.S 338814 2018-09-19 23:45:18Z jhb $");
44
45 #ifdef FPE
46 .macro __fpe_state_save p
47 /*
48 * Enable FPE usage in supervisor mode,
49 * so we can access registers.
50 */
51 li t0, SSTATUS_FS_INITIAL
52 csrs sstatus, t0
53
54 /* Store registers */
55 frcsr t0
56 sd t0, (PCB_FCSR)(\p)
57 fsd f0, (PCB_X + 0 * 16)(\p)
58 fsd f1, (PCB_X + 1 * 16)(\p)
59 fsd f2, (PCB_X + 2 * 16)(\p)
60 fsd f3, (PCB_X + 3 * 16)(\p)
61 fsd f4, (PCB_X + 4 * 16)(\p)
62 fsd f5, (PCB_X + 5 * 16)(\p)
63 fsd f6, (PCB_X + 6 * 16)(\p)
64 fsd f7, (PCB_X + 7 * 16)(\p)
65 fsd f8, (PCB_X + 8 * 16)(\p)
66 fsd f9, (PCB_X + 9 * 16)(\p)
67 fsd f10, (PCB_X + 10 * 16)(\p)
68 fsd f11, (PCB_X + 11 * 16)(\p)
69 fsd f12, (PCB_X + 12 * 16)(\p)
70 fsd f13, (PCB_X + 13 * 16)(\p)
71 fsd f14, (PCB_X + 14 * 16)(\p)
72 fsd f15, (PCB_X + 15 * 16)(\p)
73 fsd f16, (PCB_X + 16 * 16)(\p)
74 fsd f17, (PCB_X + 17 * 16)(\p)
75 fsd f18, (PCB_X + 18 * 16)(\p)
76 fsd f19, (PCB_X + 19 * 16)(\p)
77 fsd f20, (PCB_X + 20 * 16)(\p)
78 fsd f21, (PCB_X + 21 * 16)(\p)
79 fsd f22, (PCB_X + 22 * 16)(\p)
80 fsd f23, (PCB_X + 23 * 16)(\p)
81 fsd f24, (PCB_X + 24 * 16)(\p)
82 fsd f25, (PCB_X + 25 * 16)(\p)
83 fsd f26, (PCB_X + 26 * 16)(\p)
84 fsd f27, (PCB_X + 27 * 16)(\p)
85 fsd f28, (PCB_X + 28 * 16)(\p)
86 fsd f29, (PCB_X + 29 * 16)(\p)
87 fsd f30, (PCB_X + 30 * 16)(\p)
88 fsd f31, (PCB_X + 31 * 16)(\p)
89
90 /* Disable FPE usage in supervisor mode. */
91 li t0, SSTATUS_FS_MASK
92 csrc sstatus, t0
93 .endm
94
95 .macro __fpe_state_load p
96 /*
97 * Enable FPE usage in supervisor mode,
98 * so we can access registers.
99 */
100 li t0, SSTATUS_FS_INITIAL
101 csrs sstatus, t0
102
103 /* Restore registers */
104 ld t0, (PCB_FCSR)(\p)
105 fscsr t0
106 fld f0, (PCB_X + 0 * 16)(\p)
107 fld f1, (PCB_X + 1 * 16)(\p)
108 fld f2, (PCB_X + 2 * 16)(\p)
109 fld f3, (PCB_X + 3 * 16)(\p)
110 fld f4, (PCB_X + 4 * 16)(\p)
111 fld f5, (PCB_X + 5 * 16)(\p)
112 fld f6, (PCB_X + 6 * 16)(\p)
113 fld f7, (PCB_X + 7 * 16)(\p)
114 fld f8, (PCB_X + 8 * 16)(\p)
115 fld f9, (PCB_X + 9 * 16)(\p)
116 fld f10, (PCB_X + 10 * 16)(\p)
117 fld f11, (PCB_X + 11 * 16)(\p)
118 fld f12, (PCB_X + 12 * 16)(\p)
119 fld f13, (PCB_X + 13 * 16)(\p)
120 fld f14, (PCB_X + 14 * 16)(\p)
121 fld f15, (PCB_X + 15 * 16)(\p)
122 fld f16, (PCB_X + 16 * 16)(\p)
123 fld f17, (PCB_X + 17 * 16)(\p)
124 fld f18, (PCB_X + 18 * 16)(\p)
125 fld f19, (PCB_X + 19 * 16)(\p)
126 fld f20, (PCB_X + 20 * 16)(\p)
127 fld f21, (PCB_X + 21 * 16)(\p)
128 fld f22, (PCB_X + 22 * 16)(\p)
129 fld f23, (PCB_X + 23 * 16)(\p)
130 fld f24, (PCB_X + 24 * 16)(\p)
131 fld f25, (PCB_X + 25 * 16)(\p)
132 fld f26, (PCB_X + 26 * 16)(\p)
133 fld f27, (PCB_X + 27 * 16)(\p)
134 fld f28, (PCB_X + 28 * 16)(\p)
135 fld f29, (PCB_X + 29 * 16)(\p)
136 fld f30, (PCB_X + 30 * 16)(\p)
137 fld f31, (PCB_X + 31 * 16)(\p)
138
139 /* Disable FPE usage in supervisor mode. */
140 li t0, SSTATUS_FS_MASK
141 csrc sstatus, t0
142 .endm
143
144 /*
145 * void
146 * fpe_state_save(struct thread *td)
147 */
148 ENTRY(fpe_state_save)
149 /* Get pointer to PCB */
150 ld a0, TD_PCB(a0)
151 __fpe_state_save a0
152 ret
153 END(fpe_state_save)
154 #endif /* FPE */
155
156 /*
157 * void
158 * fpe_state_clear(void)
159 */
160 ENTRY(fpe_state_clear)
161 /*
162 * Enable FPE usage in supervisor mode,
163 * so we can access registers.
164 */
165 li t0, SSTATUS_FS_INITIAL
166 csrs sstatus, t0
167
168 fscsr zero
169 fcvt.d.l f0, zero
170 fcvt.d.l f1, zero
171 fcvt.d.l f2, zero
172 fcvt.d.l f3, zero
173 fcvt.d.l f4, zero
174 fcvt.d.l f5, zero
175 fcvt.d.l f6, zero
176 fcvt.d.l f7, zero
177 fcvt.d.l f8, zero
178 fcvt.d.l f9, zero
179 fcvt.d.l f10, zero
180 fcvt.d.l f11, zero
181 fcvt.d.l f12, zero
182 fcvt.d.l f13, zero
183 fcvt.d.l f14, zero
184 fcvt.d.l f15, zero
185 fcvt.d.l f16, zero
186 fcvt.d.l f17, zero
187 fcvt.d.l f18, zero
188 fcvt.d.l f19, zero
189 fcvt.d.l f20, zero
190 fcvt.d.l f21, zero
191 fcvt.d.l f22, zero
192 fcvt.d.l f23, zero
193 fcvt.d.l f24, zero
194 fcvt.d.l f25, zero
195 fcvt.d.l f26, zero
196 fcvt.d.l f27, zero
197 fcvt.d.l f28, zero
198 fcvt.d.l f29, zero
199 fcvt.d.l f30, zero
200 fcvt.d.l f31, zero
201
202 /* Disable FPE usage in supervisor mode. */
203 li t0, SSTATUS_FS_MASK
204 csrc sstatus, t0
205
206 ret
207 END(fpe_state_clear)
208
209 /*
210 * void cpu_throw(struct thread *old, struct thread *new)
211 */
212 ENTRY(cpu_throw)
213 /* Store the new curthread */
214 sd a1, PC_CURTHREAD(gp)
215 /* And the new pcb */
216 ld x13, TD_PCB(a1)
217 sd x13, PC_CURPCB(gp)
218
219 sfence.vma
220
221 /* Switch to the new pmap */
222 ld t0, PCB_L1ADDR(x13)
223 srli t0, t0, PAGE_SHIFT
224 li t1, SATP_MODE_SV39
225 or t0, t0, t1
226 csrw sptbr, t0
227
228 /* TODO: Invalidate the TLB */
229
230 sfence.vma
231
232 /* Load registers */
233 ld ra, (PCB_RA)(x13)
234 ld sp, (PCB_SP)(x13)
235 ld tp, (PCB_TP)(x13)
236
237 /* s[0-11] */
238 ld s0, (PCB_S + 0 * 8)(x13)
239 ld s1, (PCB_S + 1 * 8)(x13)
240 ld s2, (PCB_S + 2 * 8)(x13)
241 ld s3, (PCB_S + 3 * 8)(x13)
242 ld s4, (PCB_S + 4 * 8)(x13)
243 ld s5, (PCB_S + 5 * 8)(x13)
244 ld s6, (PCB_S + 6 * 8)(x13)
245 ld s7, (PCB_S + 7 * 8)(x13)
246 ld s8, (PCB_S + 8 * 8)(x13)
247 ld s9, (PCB_S + 9 * 8)(x13)
248 ld s10, (PCB_S + 10 * 8)(x13)
249 ld s11, (PCB_S + 11 * 8)(x13)
250
251 #ifdef FPE
252 /* Is FPE enabled for new thread? */
253 ld t0, TD_FRAME(a1)
254 ld t1, (TF_SSTATUS)(t0)
255 li t2, SSTATUS_FS_MASK
256 and t3, t1, t2
257 beqz t3, 1f /* No, skip. */
258
259 /* Restore registers. */
260 __fpe_state_load x13
261 1:
262 #endif
263
264 ret
265 END(cpu_throw)
266
267 /*
268 * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx)
269 *
270 * a0 = old
271 * a1 = new
272 * a2 = mtx
273 * x3 to x7, x16 and x17 are caller saved
274 */
275 ENTRY(cpu_switch)
276 /* Store the new curthread */
277 sd a1, PC_CURTHREAD(gp)
278 /* And the new pcb */
279 ld x13, TD_PCB(a1)
280 sd x13, PC_CURPCB(gp)
281
282 /* Save the old context. */
283 ld x13, TD_PCB(a0)
284
285 /* Store ra, sp and the callee-saved registers */
286 sd ra, (PCB_RA)(x13)
287 sd sp, (PCB_SP)(x13)
288 sd tp, (PCB_TP)(x13)
289
290 /* s[0-11] */
291 sd s0, (PCB_S + 0 * 8)(x13)
292 sd s1, (PCB_S + 1 * 8)(x13)
293 sd s2, (PCB_S + 2 * 8)(x13)
294 sd s3, (PCB_S + 3 * 8)(x13)
295 sd s4, (PCB_S + 4 * 8)(x13)
296 sd s5, (PCB_S + 5 * 8)(x13)
297 sd s6, (PCB_S + 6 * 8)(x13)
298 sd s7, (PCB_S + 7 * 8)(x13)
299 sd s8, (PCB_S + 8 * 8)(x13)
300 sd s9, (PCB_S + 9 * 8)(x13)
301 sd s10, (PCB_S + 10 * 8)(x13)
302 sd s11, (PCB_S + 11 * 8)(x13)
303
304 #ifdef FPE
305 /*
306 * Is FPE enabled and is it in dirty state
307 * for the old thread?
308 */
309 ld t0, TD_FRAME(a0)
310 ld t1, (TF_SSTATUS)(t0)
311 li t2, SSTATUS_FS_MASK
312 and t3, t1, t2
313 li t2, SSTATUS_FS_DIRTY
314 bne t3, t2, 1f /* No, skip. */
315
316 /* Yes, mark FPE state clean and save registers. */
317 li t2, ~SSTATUS_FS_MASK
318 and t3, t1, t2
319 li t2, SSTATUS_FS_CLEAN
320 or t3, t3, t2
321 sd t3, (TF_SSTATUS)(t0)
322
323 __fpe_state_save x13
324 1:
325 #endif
326
327 /*
328 * Restore the saved context.
329 */
330 ld x13, TD_PCB(a1)
331
332 /*
333 * TODO: We may need to flush the cache here if switching
334 * to a user process.
335 */
336
337 sfence.vma
338
339 /* Switch to the new pmap */
340 ld t0, PCB_L1ADDR(x13)
341 srli t0, t0, PAGE_SHIFT
342 li t1, SATP_MODE_SV39
343 or t0, t0, t1
344 csrw sptbr, t0
345
346 /* TODO: Invalidate the TLB */
347
348 sfence.vma
349
350 /* Release the old thread */
351 sd a2, TD_LOCK(a0)
352 #if defined(SCHED_ULE) && defined(SMP)
353 /* Spin if TD_LOCK points to a blocked_lock */
354 la a2, _C_LABEL(blocked_lock)
355 1:
356 ld t0, TD_LOCK(a1)
357 beq t0, a2, 1b
358 #endif
359
360 /* Restore the registers */
361 ld tp, (PCB_TP)(x13)
362 ld ra, (PCB_RA)(x13)
363 ld sp, (PCB_SP)(x13)
364
365 /* s[0-11] */
366 ld s0, (PCB_S + 0 * 8)(x13)
367 ld s1, (PCB_S + 1 * 8)(x13)
368 ld s2, (PCB_S + 2 * 8)(x13)
369 ld s3, (PCB_S + 3 * 8)(x13)
370 ld s4, (PCB_S + 4 * 8)(x13)
371 ld s5, (PCB_S + 5 * 8)(x13)
372 ld s6, (PCB_S + 6 * 8)(x13)
373 ld s7, (PCB_S + 7 * 8)(x13)
374 ld s8, (PCB_S + 8 * 8)(x13)
375 ld s9, (PCB_S + 9 * 8)(x13)
376 ld s10, (PCB_S + 10 * 8)(x13)
377 ld s11, (PCB_S + 11 * 8)(x13)
378
379 #ifdef FPE
380 /* Is FPE enabled for new thread? */
381 ld t0, TD_FRAME(a1)
382 ld t1, (TF_SSTATUS)(t0)
383 li t2, SSTATUS_FS_MASK
384 and t3, t1, t2
385 beqz t3, 1f /* No, skip. */
386
387 /* Restore registers. */
388 __fpe_state_load x13
389 1:
390 #endif
391
392 ret
393 .Lcpu_switch_panic_str:
394 .asciz "cpu_switch: %p\0"
395 END(cpu_switch)
396
397 /*
398 * fork_exit(void (*callout)(void *, struct trapframe *), void *arg,
399 * struct trapframe *frame)
400 */
401
402 ENTRY(fork_trampoline)
403 mv a0, s0
404 mv a1, s1
405 mv a2, sp
406 call _C_LABEL(fork_exit)
407
408 /* Restore sstatus */
409 ld t0, (TF_SSTATUS)(sp)
410 /* Ensure interrupts disabled */
411 li t1, ~SSTATUS_SIE
412 and t0, t0, t1
413 csrw sstatus, t0
414
415 /* Restore exception program counter */
416 ld t0, (TF_SEPC)(sp)
417 csrw sepc, t0
418
419 /* Restore the registers */
420 ld t0, (TF_T + 0 * 8)(sp)
421 ld t1, (TF_T + 1 * 8)(sp)
422 ld t2, (TF_T + 2 * 8)(sp)
423 ld t3, (TF_T + 3 * 8)(sp)
424 ld t4, (TF_T + 4 * 8)(sp)
425 ld t5, (TF_T + 5 * 8)(sp)
426 ld t6, (TF_T + 6 * 8)(sp)
427
428 ld s0, (TF_S + 0 * 8)(sp)
429 ld s1, (TF_S + 1 * 8)(sp)
430 ld s2, (TF_S + 2 * 8)(sp)
431 ld s3, (TF_S + 3 * 8)(sp)
432 ld s4, (TF_S + 4 * 8)(sp)
433 ld s5, (TF_S + 5 * 8)(sp)
434 ld s6, (TF_S + 6 * 8)(sp)
435 ld s7, (TF_S + 7 * 8)(sp)
436 ld s8, (TF_S + 8 * 8)(sp)
437 ld s9, (TF_S + 9 * 8)(sp)
438 ld s10, (TF_S + 10 * 8)(sp)
439 ld s11, (TF_S + 11 * 8)(sp)
440
441 ld a0, (TF_A + 0 * 8)(sp)
442 ld a1, (TF_A + 1 * 8)(sp)
443 ld a2, (TF_A + 2 * 8)(sp)
444 ld a3, (TF_A + 3 * 8)(sp)
445 ld a4, (TF_A + 4 * 8)(sp)
446 ld a5, (TF_A + 5 * 8)(sp)
447 ld a6, (TF_A + 6 * 8)(sp)
448 ld a7, (TF_A + 7 * 8)(sp)
449
450 /* Load user ra and sp */
451 ld ra, (TF_RA)(sp)
452
453 /*
454 * Store our pcpup on stack, we will load it back
455 * on kernel mode trap.
456 */
457 sd gp, (TF_SIZE)(sp)
458 ld gp, (TF_GP)(sp)
459
460 /* Save kernel stack so we can use it doing a user trap */
461 addi sp, sp, TF_SIZE
462 csrw sscratch, sp
463
464 /* Load user stack */
465 ld sp, (TF_SP - TF_SIZE)(sp)
466
467 sret
468 END(fork_trampoline)
469
470 ENTRY(savectx)
471 /* Store ra, sp and the callee-saved registers */
472 sd ra, (PCB_RA)(a0)
473 sd sp, (PCB_SP)(a0)
474 sd tp, (PCB_TP)(a0)
475
476 /* s[0-11] */
477 sd s0, (PCB_S + 0 * 8)(a0)
478 sd s1, (PCB_S + 1 * 8)(a0)
479 sd s2, (PCB_S + 2 * 8)(a0)
480 sd s3, (PCB_S + 3 * 8)(a0)
481 sd s4, (PCB_S + 4 * 8)(a0)
482 sd s5, (PCB_S + 5 * 8)(a0)
483 sd s6, (PCB_S + 6 * 8)(a0)
484 sd s7, (PCB_S + 7 * 8)(a0)
485 sd s8, (PCB_S + 8 * 8)(a0)
486 sd s9, (PCB_S + 9 * 8)(a0)
487 sd s10, (PCB_S + 10 * 8)(a0)
488 sd s11, (PCB_S + 11 * 8)(a0)
489
490 #ifdef FPE
491 __fpe_state_save a0
492 #endif
493 ret
494 END(savectx)
Cache object: 0d401adfc99f4af41c3ea271b3d7d0be
|