1 /*-
2 * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
3 * All rights reserved.
4 *
5 * Portions of this software were developed by SRI International and the
6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract
7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
8 *
9 * Portions of this software were developed by the University of Cambridge
10 * Computer Laboratory as part of the CTSRD Project, with support from the
11 * UK Higher Education Innovation Fund (HEIF).
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35 #include <machine/asm.h>
36 __FBSDID("$FreeBSD: releng/11.2/sys/riscv/riscv/exception.S 300614 2016-05-24 13:59:13Z br $");
37
38 #include "assym.s"
39
40 #include <machine/trap.h>
41 #include <machine/riscvreg.h>
42
43 .macro save_registers el
44 addi sp, sp, -(TF_SIZE)
45
46 sd ra, (TF_RA)(sp)
47 sd tp, (TF_TP)(sp)
48
49 .if \el == 0 /* We came from userspace. Load our pcpu */
50 sd gp, (TF_GP)(sp)
51 ld gp, (TF_SIZE)(sp)
52 .endif
53
54 sd t0, (TF_T + 0 * 8)(sp)
55 sd t1, (TF_T + 1 * 8)(sp)
56 sd t2, (TF_T + 2 * 8)(sp)
57 sd t3, (TF_T + 3 * 8)(sp)
58 sd t4, (TF_T + 4 * 8)(sp)
59 sd t5, (TF_T + 5 * 8)(sp)
60 sd t6, (TF_T + 6 * 8)(sp)
61
62 sd s0, (TF_S + 0 * 8)(sp)
63 sd s1, (TF_S + 1 * 8)(sp)
64 sd s2, (TF_S + 2 * 8)(sp)
65 sd s3, (TF_S + 3 * 8)(sp)
66 sd s4, (TF_S + 4 * 8)(sp)
67 sd s5, (TF_S + 5 * 8)(sp)
68 sd s6, (TF_S + 6 * 8)(sp)
69 sd s7, (TF_S + 7 * 8)(sp)
70 sd s8, (TF_S + 8 * 8)(sp)
71 sd s9, (TF_S + 9 * 8)(sp)
72 sd s10, (TF_S + 10 * 8)(sp)
73 sd s11, (TF_S + 11 * 8)(sp)
74
75 sd a0, (TF_A + 0 * 8)(sp)
76 sd a1, (TF_A + 1 * 8)(sp)
77 sd a2, (TF_A + 2 * 8)(sp)
78 sd a3, (TF_A + 3 * 8)(sp)
79 sd a4, (TF_A + 4 * 8)(sp)
80 sd a5, (TF_A + 5 * 8)(sp)
81 sd a6, (TF_A + 6 * 8)(sp)
82 sd a7, (TF_A + 7 * 8)(sp)
83
84 #if 0
85 /* XXX: temporary test: spin if stack is not kernel one */
86 .if \el == 1 /* kernel */
87 mv t0, sp
88 srli t0, t0, 63
89 1:
90 beqz t0, 1b
91 .endif
92 #endif
93
94 .if \el == 1
95 /* Store kernel sp */
96 li t1, TF_SIZE
97 add t0, sp, t1
98 sd t0, (TF_SP)(sp)
99 .else
100 /* Store user sp */
101 csrr t0, sscratch
102 sd t0, (TF_SP)(sp)
103 .endif
104 li t0, 0
105 csrw sscratch, t0
106 csrr t0, sepc
107 sd t0, (TF_SEPC)(sp)
108 csrr t0, sstatus
109 sd t0, (TF_SSTATUS)(sp)
110 csrr t0, sbadaddr
111 sd t0, (TF_SBADADDR)(sp)
112 csrr t0, scause
113 sd t0, (TF_SCAUSE)(sp)
114 .endm
115
116 .macro load_registers el
117 ld t0, (TF_SSTATUS)(sp)
118 .if \el == 0
119 /* Ensure user interrupts will be enabled on eret. */
120 ori t0, t0, SSTATUS_PIE
121 .else
122 /*
123 * Disable interrupts for supervisor mode exceptions.
124 * For user mode exceptions we have already done this
125 * in do_ast.
126 */
127 li t1, ~SSTATUS_IE
128 and t0, t0, t1
129 .endif
130 csrw sstatus, t0
131
132 ld t0, (TF_SEPC)(sp)
133 csrw sepc, t0
134
135 .if \el == 0
136 /* We go to userspace. Load user sp */
137 ld t0, (TF_SP)(sp)
138 csrw sscratch, t0
139
140 /* And store our pcpu */
141 sd gp, (TF_SIZE)(sp)
142 ld gp, (TF_GP)(sp)
143 .endif
144
145 ld ra, (TF_RA)(sp)
146 ld tp, (TF_TP)(sp)
147
148 ld t0, (TF_T + 0 * 8)(sp)
149 ld t1, (TF_T + 1 * 8)(sp)
150 ld t2, (TF_T + 2 * 8)(sp)
151 ld t3, (TF_T + 3 * 8)(sp)
152 ld t4, (TF_T + 4 * 8)(sp)
153 ld t5, (TF_T + 5 * 8)(sp)
154 ld t6, (TF_T + 6 * 8)(sp)
155
156 ld s0, (TF_S + 0 * 8)(sp)
157 ld s1, (TF_S + 1 * 8)(sp)
158 ld s2, (TF_S + 2 * 8)(sp)
159 ld s3, (TF_S + 3 * 8)(sp)
160 ld s4, (TF_S + 4 * 8)(sp)
161 ld s5, (TF_S + 5 * 8)(sp)
162 ld s6, (TF_S + 6 * 8)(sp)
163 ld s7, (TF_S + 7 * 8)(sp)
164 ld s8, (TF_S + 8 * 8)(sp)
165 ld s9, (TF_S + 9 * 8)(sp)
166 ld s10, (TF_S + 10 * 8)(sp)
167 ld s11, (TF_S + 11 * 8)(sp)
168
169 ld a0, (TF_A + 0 * 8)(sp)
170 ld a1, (TF_A + 1 * 8)(sp)
171 ld a2, (TF_A + 2 * 8)(sp)
172 ld a3, (TF_A + 3 * 8)(sp)
173 ld a4, (TF_A + 4 * 8)(sp)
174 ld a5, (TF_A + 5 * 8)(sp)
175 ld a6, (TF_A + 6 * 8)(sp)
176 ld a7, (TF_A + 7 * 8)(sp)
177
178 addi sp, sp, (TF_SIZE)
179 .endm
180
181 .macro do_ast
182 /* Disable interrupts */
183 csrr a4, sstatus
184 1:
185 csrci sstatus, SSTATUS_IE
186
187 ld a1, PC_CURTHREAD(gp)
188 lw a2, TD_FLAGS(a1)
189
190 li a3, (TDF_ASTPENDING|TDF_NEEDRESCHED)
191 and a2, a2, a3
192 beqz a2, 2f
193
194 /* Restore interrupts */
195 andi a4, a4, SSTATUS_IE
196 csrs sstatus, a4
197
198 /* Handle the ast */
199 mv a0, sp
200 call _C_LABEL(ast)
201
202 /* Re-check for new ast scheduled */
203 j 1b
204 2:
205 .endm
206
207 ENTRY(cpu_exception_handler_supervisor)
208 save_registers 1
209 mv a0, sp
210 call _C_LABEL(do_trap_supervisor)
211 load_registers 1
212 eret
213 END(cpu_exception_handler_supervisor)
214
215 ENTRY(cpu_exception_handler_user)
216 csrrw sp, sscratch, sp
217 save_registers 0
218 mv a0, sp
219 call _C_LABEL(do_trap_user)
220 do_ast
221 load_registers 0
222 csrrw sp, sscratch, sp
223 eret
224 END(cpu_exception_handler_user)
225
226 /*
227 * Trap handlers
228 */
229 .text
230 bad_trap:
231 j bad_trap
232
233 user_trap:
234 /* Save state */
235 csrrw sp, mscratch, sp
236 addi sp, sp, -64
237 sd t0, (8 * 0)(sp)
238 sd t1, (8 * 1)(sp)
239 sd t2, (8 * 2)(sp)
240 sd t3, (8 * 3)(sp)
241 sd t4, (8 * 4)(sp)
242 sd t5, (8 * 5)(sp)
243 sd a0, (8 * 7)(sp)
244
245 la t2, _C_LABEL(cpu_exception_handler_user)
246
247 csrr t0, mcause
248 bltz t0, machine_interrupt
249 j exit_mrts
250
251 supervisor_trap:
252 /* Save state */
253 csrrw sp, mscratch, sp
254 addi sp, sp, -64
255 sd t0, (8 * 0)(sp)
256 sd t1, (8 * 1)(sp)
257 sd t2, (8 * 2)(sp)
258 sd t3, (8 * 3)(sp)
259 sd t4, (8 * 4)(sp)
260 sd t5, (8 * 5)(sp)
261 sd a0, (8 * 7)(sp)
262
263 la t2, _C_LABEL(cpu_exception_handler_supervisor)
264
265 csrr t0, mcause
266 bltz t0, machine_interrupt
267
268 li t1, EXCP_SMODE_ENV_CALL
269 beq t0, t1, supervisor_call
270 j exit_mrts
271
272 machine_interrupt:
273 /* Type of interrupt ? */
274 csrr t0, mcause
275 andi t0, t0, EXCP_MASK
276 li t1, 0
277 beq t1, t0, software_interrupt
278 li t1, 1
279 beq t1, t0, timer_interrupt
280 li t1, 2
281 beq t1, t0, htif_interrupt
282 li t1, 4
283 beq t1, t0, io_interrupt /* lowRISC only */
284
285 /* not reached */
286 1:
287 j 1b
288
289 io_interrupt:
290 /* Disable IO interrupts so we can go to supervisor mode */
291 csrwi CSR_IO_IRQ, 0
292
293 /* Handle the trap in supervisor mode */
294 j exit_mrts
295
296 software_interrupt:
297 li t0, MIP_MSIP
298 csrc mip, t0
299 li t0, MIP_SSIP
300 csrs mip, t0
301
302 /* If PRV1 is PRV_U (user) then serve the trap */
303 csrr t0, mstatus
304 li t1, (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT)
305 and t0, t0, t1
306 beqz t0, 1f
307
308 /*
309 * If PRV1 is supervisor and interrupts were enabled,
310 * then serve the trap.
311 */
312 csrr t0, mstatus
313 li t1, (SR_IE1 | (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT))
314 and t0, t0, t1
315 li t1, (SR_IE1 | (MSTATUS_PRV_S << MSTATUS_PRV1_SHIFT))
316 beq t0, t1, 1f
317
318 j exit
319
320 1:
321 /* Handle the trap in supervisor mode */
322 j exit_mrts
323
324 timer_interrupt:
325 /* Disable machine timer interrupts */
326 li t0, MIE_MTIE
327 csrc mie, t0
328
329 /* Clear machine pending */
330 li t0, MIP_MTIP
331 csrc mip, t0
332
333 /* Post supervisor timer interrupt */
334 li t0, MIP_STIP
335 csrs mip, t0
336
337 /* If PRV1 is PRV_U (user) then serve the trap */
338 csrr t0, mstatus
339 li t1, (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT)
340 and t0, t0, t1
341 beqz t0, 1f
342
343 /*
344 * If PRV1 is supervisor and interrupts were enabled,
345 * then serve the trap.
346 */
347 csrr t0, mstatus
348 li t1, (SR_IE1 | (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT))
349 and t0, t0, t1
350 li t1, (SR_IE1 | (MSTATUS_PRV_S << MSTATUS_PRV1_SHIFT))
351 beq t0, t1, 1f
352
353 j exit
354
355 1:
356 /* Serve a trap in supervisor mode */
357 j exit_mrts
358
359 htif_interrupt:
360 1:
361 li t5, 0
362 csrrw t5, mfromhost, t5
363 beqz t5, 3f
364
365 /* Console PUT intr ? */
366 mv t1, t5
367 li t0, 0x101
368 srli t1, t1, 48
369 bne t1, t0, 2f
370 /* Yes */
371 la t0, console_intr
372 li t1, 1
373 sd t1, 0(t0)
374
375 /* Check if there is any other pending event */
376 j 1b
377
378 2:
379 /* Save entry */
380 la t0, htif_ring
381 csrr t1, mhartid
382 li t4, (HTIF_RING_SIZE + 16)
383 mulw t4, t4, t1
384 add t0, t0, t4
385 li t4, (HTIF_RING_SIZE)
386 add t0, t0, t4 /* t0 == htif_ring_cursor */
387
388 ld t1, 0(t0) /* load ptr to cursor */
389 sd t5, 0(t1) /* put entry */
390 li t4, 1
391 sd t4, 8(t1) /* mark used */
392 ld t4, 16(t1) /* take next */
393 /* Update cursor */
394 sd t4, 0(t0)
395
396 /* Post supervisor software interrupt */
397 li t0, MIP_SSIP
398 csrs mip, t0
399
400 /* Check if there is any other pending event */
401 j 1b
402
403 3:
404 j exit
405
406 supervisor_call:
407 csrr t1, mepc
408 addi t1, t1, 4 /* Next instruction in t1 */
409 li t4, ECALL_HTIF_CMD
410 beq t5, t4, htif_cmd
411 li t4, ECALL_HTIF_GET_ENTRY
412 beq t5, t4, htif_get_entry
413 li t4, ECALL_MTIMECMP
414 beq t5, t4, set_mtimecmp
415 li t4, ECALL_CLEAR_PENDING
416 beq t5, t4, clear_pending
417 li t4, ECALL_MCPUID_GET
418 beq t5, t4, mcpuid_get
419 li t4, ECALL_MIMPID_GET
420 beq t5, t4, mimpid_get
421 li t4, ECALL_SEND_IPI
422 beq t5, t4, send_ipi
423 li t4, ECALL_CLEAR_IPI
424 beq t5, t4, clear_ipi
425 li t4, ECALL_HTIF_LOWPUTC
426 beq t5, t4, htif_lowputc
427 li t4, ECALL_MIE_SET
428 beq t5, t4, mie_set
429 li t4, ECALL_IO_IRQ_MASK
430 beq t5, t4, io_irq_mask
431 j exit_next_instr
432
433 io_irq_mask:
434 csrw CSR_IO_IRQ, t6
435 j exit_next_instr
436
437 mie_set:
438 csrs mie, t6
439 j exit_next_instr
440
441 mcpuid_get:
442 csrr t6, mcpuid
443 j exit_next_instr
444
445 mimpid_get:
446 csrr t6, mimpid
447 j exit_next_instr
448
449 send_ipi:
450 /* CPU mmio base in t6 */
451 mv t0, t6
452 li t2, (CSR_IPI * XLEN)
453 add t0, t0, t2 /* t0 = CSR_IPI */
454 li t2, 1
455 sd t2, 0(t0)
456 j exit_next_instr
457
458 clear_ipi:
459 /* Do only clear if there are no new entries in HTIF ring */
460 la t0, htif_ring
461 csrr t2, mhartid
462 li t4, (HTIF_RING_SIZE + 16)
463 mulw t4, t4, t2
464 add t0, t0, t4
465 li t4, (HTIF_RING_SIZE)
466 add t0, t0, t4 /* t0 == ptr to htif_ring_cursor */
467 ld t2, 8(t0) /* load htif_ring_last */
468 ld t2, 8(t2) /* load used */
469 bnez t2, 1f
470
471 /* Clear supervisor software interrupt pending bit */
472 li t0, MIP_SSIP
473 csrc mip, t0
474
475 1:
476 j exit_next_instr
477
478 htif_get_entry:
479 /* Get a htif_ring for current core */
480 la t0, htif_ring
481 csrr t2, mhartid
482 li t4, (HTIF_RING_SIZE + 16)
483 mulw t4, t4, t2
484 add t0, t0, t4
485 li t4, (HTIF_RING_SIZE + 8)
486 add t0, t0, t4 /* t0 == htif_ring_last */
487
488 /* Check for new entries */
489 li t6, 0 /* preset return value */
490 ld t2, 0(t0) /* load ptr to last */
491 ld t4, 8(t2) /* get used */
492 beqz t4, 1f /* No new entries. Exit */
493
494 /* Get one */
495 ld t6, 0(t2) /* get entry */
496 li t4, 0
497 sd t4, 8(t2) /* mark free */
498 sd t4, 0(t2) /* free entry, just in case */
499 ld t4, 16(t2) /* take next */
500 sd t4, 0(t0) /* update ptr to last */
501 1:
502 /* Exit. Result is stored in t6 */
503 j exit_next_instr
504
505 htif_cmd:
506 1:
507 mv t0, t6
508 csrrw t0, mtohost, t0
509 bnez t0, 1b
510 j exit_next_instr
511
512 htif_lowputc:
513 1:
514 mv t0, t6
515 csrrw t0, mtohost, t0
516 bnez t0, 1b
517
518 2:
519 li t4, 0
520 csrrw t5, mfromhost, t4
521 beqz t5, 2b
522
523 /* Console PUT intr ? */
524 mv t2, t5
525 srli t2, t2, 48
526 li t3, 0x0101
527 beq t2, t3, 3f
528
529 /* Not a console PUT, so save entry */
530 la t0, htif_ring
531 csrr t2, mhartid
532 li t4, (HTIF_RING_SIZE + 16)
533 mulw t4, t4, t2
534 add t0, t0, t4
535 li t4, (HTIF_RING_SIZE)
536 add t0, t0, t4 /* t0 == htif_ring_cursor */
537
538 ld t2, 0(t0) /* load ptr to cursor */
539 sd t5, 0(t2) /* put entry */
540 li t4, 1
541 sd t4, 8(t2) /* mark used */
542 ld t4, 16(t2) /* take next */
543 /* Update cursor */
544 sd t4, 0(t0)
545
546 /* Post supervisor software interrupt */
547 li t0, MIP_SSIP
548 csrs mip, t0
549
550 /* Wait for console intr again */
551 j 2b
552
553 3:
554 j exit_next_instr
555
556 set_mtimecmp:
557 csrr t2, stime
558 add t6, t6, t2
559 csrw mtimecmp, t6
560
561 /* Enable interrupts */
562 li t0, (MIE_MTIE | MIE_STIE)
563 csrs mie, t0
564 j exit_next_instr
565
566 clear_pending:
567 li t0, MIP_STIP
568 csrc mip, t0
569 j exit_next_instr
570
571 /*
572 * Trap exit functions
573 */
574 exit_next_instr:
575 /* Next instruction is in t1 */
576 csrw mepc, t1
577 exit:
578 /* Restore state */
579 ld t0, (8 * 0)(sp)
580 ld t1, (8 * 1)(sp)
581 ld t2, (8 * 2)(sp)
582 ld t3, (8 * 3)(sp)
583 ld t4, (8 * 4)(sp)
584 ld t5, (8 * 5)(sp)
585 ld a0, (8 * 7)(sp)
586 addi sp, sp, 64
587 csrrw sp, mscratch, sp
588 eret
589
590 /*
591 * Redirect to supervisor
592 */
593 exit_mrts:
594 /* Setup exception handler */
595 li t1, KERNBASE
596 add t2, t2, t1
597 csrw stvec, t2
598
599 /* Restore state */
600 ld t0, (8 * 0)(sp)
601 ld t1, (8 * 1)(sp)
602 ld t2, (8 * 2)(sp)
603 ld t3, (8 * 3)(sp)
604 ld t4, (8 * 4)(sp)
605 ld t5, (8 * 5)(sp)
606 ld a0, (8 * 7)(sp)
607 addi sp, sp, 64
608 csrrw sp, mscratch, sp
609
610 /* Redirect to supervisor */
611 mrts
Cache object: e6346b2aa7fb1902a6e5f8e7e7d74617
|