1 /*-
2 * Copyright (C) 2006-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com>
3 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
4 * Copyright (C) 2006 Juniper Networks, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. The name of the author may not be used to endorse or promote products
16 * derived from this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
21 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
23 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
24 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
25 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
26 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 *
29 * $FreeBSD: releng/11.0/sys/powerpc/booke/trap_subr.S 295520 2016-02-11 13:15:37Z jhibbits $
30 */
31 /*-
32 * Copyright (C) 1995, 1996 Wolfgang Solfrank.
33 * Copyright (C) 1995, 1996 TooLs GmbH.
34 * All rights reserved.
35 *
36 * Redistribution and use in source and binary forms, with or without
37 * modification, are permitted provided that the following conditions
38 * are met:
39 * 1. Redistributions of source code must retain the above copyright
40 * notice, this list of conditions and the following disclaimer.
41 * 2. Redistributions in binary form must reproduce the above copyright
42 * notice, this list of conditions and the following disclaimer in the
43 * documentation and/or other materials provided with the distribution.
44 * 3. All advertising materials mentioning features or use of this software
45 * must display the following acknowledgement:
46 * This product includes software developed by TooLs GmbH.
47 * 4. The name of TooLs GmbH may not be used to endorse or promote products
48 * derived from this software without specific prior written permission.
49 *
50 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
51 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
52 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
53 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
54 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
55 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
56 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
57 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
58 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
59 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
60 *
61 * from: $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $
62 */
63
64 /*
65 * NOTICE: This is not a standalone file. to use it, #include it in
66 * your port's locore.S, like so:
67 *
68 * #include <powerpc/booke/trap_subr.S>
69 */
70
71 /*
72 * SPRG usage notes
73 *
74 * SPRG0 - pcpu pointer
75 * SPRG1 - all interrupts except TLB miss, critical, machine check
76 * SPRG2 - critical
77 * SPRG3 - machine check
78 * SPRG4-6 - scratch
79 *
80 */
81
82 /* Get the per-CPU data structure */
83 #define GET_CPUINFO(r) mfsprg0 r
84
85 #define RES_GRANULE 32
86 #define RES_LOCK 0 /* offset to the 'lock' word */
87 #define RES_RECURSE 4 /* offset to the 'recurse' word */
88
89 /*
90 * Standard interrupt prolog
91 *
92 * sprg_sp - SPRG{1-3} reg used to temporarily store the SP
93 * savearea - temp save area (pc_{tempsave, disisave, critsave, mchksave})
94 * isrr0-1 - save restore registers with CPU state at interrupt time (may be
95 * SRR0-1, CSRR0-1, MCSRR0-1
96 *
97 * 1. saves in the given savearea:
98 * - R30-31
99 * - DEAR, ESR
100 * - xSRR0-1
101 *
102 * 2. saves CR -> R30
103 *
104 * 3. switches to kstack if needed
105 *
106 * 4. notes:
107 * - R31 can be used as scratch register until a new frame is layed on
108 * the stack with FRAME_SETUP
109 *
110 * - potential TLB miss: NO. Saveareas are always acessible via TLB1
111 * permanent entries, and within this prolog we do not dereference any
112 * locations potentially not in the TLB
113 */
114 #define STANDARD_PROLOG(sprg_sp, savearea, isrr0, isrr1) \
115 mtspr sprg_sp, %r1; /* Save SP */ \
116 GET_CPUINFO(%r1); /* Per-cpu structure */ \
117 stw %r30, (savearea+CPUSAVE_R30)(%r1); \
118 stw %r31, (savearea+CPUSAVE_R31)(%r1); \
119 mfdear %r30; \
120 mfesr %r31; \
121 stw %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \
122 stw %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \
123 mfspr %r30, isrr0; \
124 mfspr %r31, isrr1; /* MSR at interrupt time */ \
125 stw %r30, (savearea+CPUSAVE_SRR0)(%r1); \
126 stw %r31, (savearea+CPUSAVE_SRR1)(%r1); \
127 isync; \
128 mfspr %r1, sprg_sp; /* Restore SP */ \
129 mfcr %r30; /* Save CR */ \
130 /* switch to per-thread kstack if intr taken in user mode */ \
131 mtcr %r31; /* MSR at interrupt time */ \
132 bf 17, 1f; \
133 GET_CPUINFO(%r1); /* Per-cpu structure */ \
134 lwz %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \
135 1:
136
137 #define STANDARD_CRIT_PROLOG(sprg_sp, savearea, isrr0, isrr1) \
138 mtspr sprg_sp, %r1; /* Save SP */ \
139 GET_CPUINFO(%r1); /* Per-cpu structure */ \
140 stw %r30, (savearea+CPUSAVE_R30)(%r1); \
141 stw %r31, (savearea+CPUSAVE_R31)(%r1); \
142 mfdear %r30; \
143 mfesr %r31; \
144 stw %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \
145 stw %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \
146 mfspr %r30, isrr0; \
147 mfspr %r31, isrr1; /* MSR at interrupt time */ \
148 stw %r30, (savearea+CPUSAVE_SRR0)(%r1); \
149 stw %r31, (savearea+CPUSAVE_SRR1)(%r1); \
150 mfspr %r30, SPR_SRR0; \
151 mfspr %r31, SPR_SRR1; /* MSR at interrupt time */ \
152 stw %r30, (savearea+CPUSAVE_SRR0+8)(%r1); \
153 stw %r31, (savearea+CPUSAVE_SRR1+8)(%r1); \
154 isync; \
155 mfspr %r1, sprg_sp; /* Restore SP */ \
156 mfcr %r30; /* Save CR */ \
157 /* switch to per-thread kstack if intr taken in user mode */ \
158 mtcr %r31; /* MSR at interrupt time */ \
159 bf 17, 1f; \
160 GET_CPUINFO(%r1); /* Per-cpu structure */ \
161 lwz %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \
162 1:
163
164 /*
165 * FRAME_SETUP assumes:
166 * SPRG{1-3} SP at the time interrupt occured
167 * savearea r30-r31, DEAR, ESR, xSRR0-1
168 * r30 CR
169 * r31 scratch
170 * r1 kernel stack
171 *
172 * sprg_sp - SPRG reg containing SP at the time interrupt occured
173 * savearea - temp save
174 * exc - exception number (EXC_xxx)
175 *
176 * 1. sets a new frame
177 * 2. saves in the frame:
178 * - R0, R1 (SP at the time of interrupt), R2, LR, CR
179 * - R3-31 (R30-31 first restored from savearea)
180 * - XER, CTR, DEAR, ESR (from savearea), xSRR0-1
181 *
182 * Notes:
183 * - potential TLB miss: YES, since we make dereferences to kstack, which
184 * can happen not covered (we can have up to two DTLB misses if fortunate
185 * enough i.e. when kstack crosses page boundary and both pages are
186 * untranslated)
187 */
188 #define FRAME_SETUP(sprg_sp, savearea, exc) \
189 mfspr %r31, sprg_sp; /* get saved SP */ \
190 /* establish a new stack frame and put everything on it */ \
191 stwu %r31, -FRAMELEN(%r1); \
192 stw %r0, FRAME_0+8(%r1); /* save r0 in the trapframe */ \
193 stw %r31, FRAME_1+8(%r1); /* save SP " " */ \
194 stw %r2, FRAME_2+8(%r1); /* save r2 " " */ \
195 mflr %r31; \
196 stw %r31, FRAME_LR+8(%r1); /* save LR " " */ \
197 stw %r30, FRAME_CR+8(%r1); /* save CR " " */ \
198 GET_CPUINFO(%r2); \
199 lwz %r30, (savearea+CPUSAVE_R30)(%r2); /* get saved r30 */ \
200 lwz %r31, (savearea+CPUSAVE_R31)(%r2); /* get saved r31 */ \
201 /* save R3-31 */ \
202 stmw %r3, FRAME_3+8(%r1) ; \
203 /* save DEAR, ESR */ \
204 lwz %r28, (savearea+CPUSAVE_BOOKE_DEAR)(%r2); \
205 lwz %r29, (savearea+CPUSAVE_BOOKE_ESR)(%r2); \
206 stw %r28, FRAME_BOOKE_DEAR+8(%r1); \
207 stw %r29, FRAME_BOOKE_ESR+8(%r1); \
208 /* save XER, CTR, exc number */ \
209 mfxer %r3; \
210 mfctr %r4; \
211 stw %r3, FRAME_XER+8(%r1); \
212 stw %r4, FRAME_CTR+8(%r1); \
213 li %r5, exc; \
214 stw %r5, FRAME_EXC+8(%r1); \
215 /* save DBCR0 */ \
216 mfspr %r3, SPR_DBCR0; \
217 stw %r3, FRAME_BOOKE_DBCR0+8(%r1); \
218 /* save xSSR0-1 */ \
219 lwz %r30, (savearea+CPUSAVE_SRR0)(%r2); \
220 lwz %r31, (savearea+CPUSAVE_SRR1)(%r2); \
221 stw %r30, FRAME_SRR0+8(%r1); \
222 stw %r31, FRAME_SRR1+8(%r1); \
223 lwz %r2,PC_CURTHREAD(%r2) /* set curthread pointer */
224
225 /*
226 *
227 * isrr0-1 - save restore registers to restore CPU state to (may be
228 * SRR0-1, CSRR0-1, MCSRR0-1
229 *
230 * Notes:
231 * - potential TLB miss: YES. The deref'd kstack may be not covered
232 */
233 #define FRAME_LEAVE(isrr0, isrr1) \
234 /* restore CTR, XER, LR, CR */ \
235 lwz %r4, FRAME_CTR+8(%r1); \
236 lwz %r5, FRAME_XER+8(%r1); \
237 lwz %r6, FRAME_LR+8(%r1); \
238 lwz %r7, FRAME_CR+8(%r1); \
239 mtctr %r4; \
240 mtxer %r5; \
241 mtlr %r6; \
242 mtcr %r7; \
243 /* restore DBCR0 */ \
244 lwz %r4, FRAME_BOOKE_DBCR0+8(%r1); \
245 mtspr SPR_DBCR0, %r4; \
246 /* restore xSRR0-1 */ \
247 lwz %r30, FRAME_SRR0+8(%r1); \
248 lwz %r31, FRAME_SRR1+8(%r1); \
249 mtspr isrr0, %r30; \
250 mtspr isrr1, %r31; \
251 /* restore R2-31, SP */ \
252 lmw %r2, FRAME_2+8(%r1) ; \
253 lwz %r0, FRAME_0+8(%r1); \
254 lwz %r1, FRAME_1+8(%r1); \
255 isync
256
257 /*
258 * TLB miss prolog
259 *
260 * saves LR, CR, SRR0-1, R20-31 in the TLBSAVE area
261 *
262 * Notes:
263 * - potential TLB miss: NO. It is crucial that we do not generate a TLB
264 * miss within the TLB prolog itself!
265 * - TLBSAVE is always translated
266 */
267 #define TLB_PROLOG \
268 mtsprg4 %r1; /* Save SP */ \
269 mtsprg5 %r28; \
270 mtsprg6 %r29; \
271 /* calculate TLB nesting level and TLBSAVE instance address */ \
272 GET_CPUINFO(%r1); /* Per-cpu structure */ \
273 lwz %r28, PC_BOOKE_TLB_LEVEL(%r1); \
274 rlwinm %r29, %r28, 6, 23, 25; /* 4 x TLBSAVE_LEN */ \
275 addi %r28, %r28, 1; \
276 stw %r28, PC_BOOKE_TLB_LEVEL(%r1); \
277 addi %r29, %r29, PC_BOOKE_TLBSAVE@l; \
278 add %r1, %r1, %r29; /* current TLBSAVE ptr */ \
279 \
280 /* save R20-31 */ \
281 mfsprg5 %r28; \
282 mfsprg6 %r29; \
283 stmw %r20, (TLBSAVE_BOOKE_R20)(%r1); \
284 /* save LR, CR */ \
285 mflr %r30; \
286 mfcr %r31; \
287 stw %r30, (TLBSAVE_BOOKE_LR)(%r1); \
288 stw %r31, (TLBSAVE_BOOKE_CR)(%r1); \
289 /* save SRR0-1 */ \
290 mfsrr0 %r30; /* execution addr at interrupt time */ \
291 mfsrr1 %r31; /* MSR at interrupt time*/ \
292 stw %r30, (TLBSAVE_BOOKE_SRR0)(%r1); /* save SRR0 */ \
293 stw %r31, (TLBSAVE_BOOKE_SRR1)(%r1); /* save SRR1 */ \
294 isync; \
295 mfsprg4 %r1
296
297 /*
298 * restores LR, CR, SRR0-1, R20-31 from the TLBSAVE area
299 *
300 * same notes as for the TLB_PROLOG
301 */
302 #define TLB_RESTORE \
303 mtsprg4 %r1; /* Save SP */ \
304 GET_CPUINFO(%r1); /* Per-cpu structure */ \
305 /* calculate TLB nesting level and TLBSAVE instance addr */ \
306 lwz %r28, PC_BOOKE_TLB_LEVEL(%r1); \
307 subi %r28, %r28, 1; \
308 stw %r28, PC_BOOKE_TLB_LEVEL(%r1); \
309 rlwinm %r29, %r28, 6, 23, 25; /* 4 x TLBSAVE_LEN */ \
310 addi %r29, %r29, PC_BOOKE_TLBSAVE@l; \
311 add %r1, %r1, %r29; \
312 \
313 /* restore LR, CR */ \
314 lwz %r30, (TLBSAVE_BOOKE_LR)(%r1); \
315 lwz %r31, (TLBSAVE_BOOKE_CR)(%r1); \
316 mtlr %r30; \
317 mtcr %r31; \
318 /* restore SRR0-1 */ \
319 lwz %r30, (TLBSAVE_BOOKE_SRR0)(%r1); \
320 lwz %r31, (TLBSAVE_BOOKE_SRR1)(%r1); \
321 mtsrr0 %r30; \
322 mtsrr1 %r31; \
323 /* restore R20-31 */ \
324 lmw %r20, (TLBSAVE_BOOKE_R20)(%r1); \
325 mfsprg4 %r1
326
327 #ifdef SMP
328 #define TLB_LOCK \
329 GET_CPUINFO(%r20); \
330 lwz %r21, PC_CURTHREAD(%r20); \
331 lwz %r22, PC_BOOKE_TLB_LOCK(%r20); \
332 \
333 1: lwarx %r23, 0, %r22; \
334 cmpwi %r23, TLB_UNLOCKED; \
335 beq 2f; \
336 \
337 /* check if this is recursion */ \
338 cmplw cr0, %r21, %r23; \
339 bne- 1b; \
340 \
341 2: /* try to acquire lock */ \
342 stwcx. %r21, 0, %r22; \
343 bne- 1b; \
344 \
345 /* got it, update recursion counter */ \
346 lwz %r21, RES_RECURSE(%r22); \
347 addi %r21, %r21, 1; \
348 stw %r21, RES_RECURSE(%r22); \
349 isync; \
350 msync
351
352 #define TLB_UNLOCK \
353 GET_CPUINFO(%r20); \
354 lwz %r21, PC_CURTHREAD(%r20); \
355 lwz %r22, PC_BOOKE_TLB_LOCK(%r20); \
356 \
357 /* update recursion counter */ \
358 lwz %r23, RES_RECURSE(%r22); \
359 subi %r23, %r23, 1; \
360 stw %r23, RES_RECURSE(%r22); \
361 \
362 cmpwi %r23, 0; \
363 bne 1f; \
364 isync; \
365 msync; \
366 \
367 /* release the lock */ \
368 li %r23, TLB_UNLOCKED; \
369 stw %r23, 0(%r22); \
370 1: isync; \
371 msync
372 #else
373 #define TLB_LOCK
374 #define TLB_UNLOCK
375 #endif /* SMP */
376
377 #define INTERRUPT(label) \
378 .globl label; \
379 .align 5; \
380 CNAME(label):
381
382 /*
383 * Interrupt handling routines in BookE can be flexibly placed and do not have
384 * to live in pre-defined vectors location. Note they need to be TLB-mapped at
385 * all times in order to be able to handle exceptions. We thus arrange for
386 * them to be part of kernel text which is always TLB-accessible.
387 *
388 * The interrupt handling routines have to be 16 bytes aligned: we align them
389 * to 32 bytes (cache line length) which supposedly performs better.
390 *
391 */
392 .text
393 .globl CNAME(interrupt_vector_base)
394 .align 5
395 interrupt_vector_base:
396 /*****************************************************************************
397 * Catch-all handler to handle uninstalled IVORs
398 ****************************************************************************/
399 INTERRUPT(int_unknown)
400 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
401 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_RSVD)
402 b trap_common
403
404 /*****************************************************************************
405 * Critical input interrupt
406 ****************************************************************************/
407 INTERRUPT(int_critical_input)
408 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
409 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_CRIT)
410 addi %r3, %r1, 8
411 bl CNAME(powerpc_interrupt)
412 FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
413 rfci
414
415
416 /*****************************************************************************
417 * Machine check interrupt
418 ****************************************************************************/
419 INTERRUPT(int_machine_check)
420 STANDARD_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1)
421 FRAME_SETUP(SPR_SPRG3, PC_BOOKE_MCHKSAVE, EXC_MCHK)
422 addi %r3, %r1, 8
423 bl CNAME(powerpc_interrupt)
424 FRAME_LEAVE(SPR_MCSRR0, SPR_MCSRR1)
425 rfmci
426
427
428 /*****************************************************************************
429 * Data storage interrupt
430 ****************************************************************************/
431 INTERRUPT(int_data_storage)
432 STANDARD_PROLOG(SPR_SPRG1, PC_DISISAVE, SPR_SRR0, SPR_SRR1)
433 FRAME_SETUP(SPR_SPRG1, PC_DISISAVE, EXC_DSI)
434 b trap_common
435
436
437 /*****************************************************************************
438 * Instruction storage interrupt
439 ****************************************************************************/
440 INTERRUPT(int_instr_storage)
441 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
442 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ISI)
443 b trap_common
444
445
446 /*****************************************************************************
447 * External input interrupt
448 ****************************************************************************/
449 INTERRUPT(int_external_input)
450 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
451 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_EXI)
452 addi %r3, %r1, 8
453 bl CNAME(powerpc_interrupt)
454 b clear_we
455
456
457 INTERRUPT(int_alignment)
458 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
459 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ALI)
460 b trap_common
461
462
463 INTERRUPT(int_program)
464 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
465 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_PGM)
466 b trap_common
467
468
469 INTERRUPT(int_fpu)
470 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
471 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FPU)
472 b trap_common
473
474
475 /*****************************************************************************
476 * System call
477 ****************************************************************************/
478 INTERRUPT(int_syscall)
479 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
480 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SC)
481 b trap_common
482
483
484 /*****************************************************************************
485 * Decrementer interrupt
486 ****************************************************************************/
487 INTERRUPT(int_decrementer)
488 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
489 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_DECR)
490 addi %r3, %r1, 8
491 bl CNAME(powerpc_interrupt)
492 b clear_we
493
494
495 /*****************************************************************************
496 * Fixed interval timer
497 ****************************************************************************/
498 INTERRUPT(int_fixed_interval_timer)
499 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
500 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FIT)
501 b trap_common
502
503
504 /*****************************************************************************
505 * Watchdog interrupt
506 ****************************************************************************/
507 INTERRUPT(int_watchdog)
508 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
509 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_WDOG)
510 b trap_common
511
512
513 /*****************************************************************************
514 * Altivec Unavailable interrupt
515 ****************************************************************************/
516 INTERRUPT(int_vec)
517 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
518 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VEC)
519 b trap_common
520
521
522 /*****************************************************************************
523 * Watchdog interrupt
524 ****************************************************************************/
525 INTERRUPT(int_vecast)
526 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
527 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VECAST_E)
528 b trap_common
529
530
531 #ifdef HWPMC_HOOKS
532 /*****************************************************************************
533 * PMC Interrupt
534 ****************************************************************************/
535 INTERRUPT(int_performance_counter)
536 STANDARD_PROLOG(SPR_SPRG3, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
537 FRAME_SETUP(SPR_SPRG3, PC_TEMPSAVE, EXC_PERF)
538 addi %r3, %r1, 8
539 bl CNAME(powerpc_interrupt)
540 b trapexit
541 #endif
542
543
544 /*****************************************************************************
545 * Data TLB miss interrupt
546 *
547 * There can be nested TLB misses - while handling a TLB miss we reference
548 * data structures that may be not covered by translations. We support up to
549 * TLB_NESTED_MAX-1 nested misses.
550 *
551 * Registers use:
552 * r31 - dear
553 * r30 - unused
554 * r29 - saved mas0
555 * r28 - saved mas1
556 * r27 - saved mas2
557 * r26 - pmap address
558 * r25 - pte address
559 *
560 * r20:r23 - scratch registers
561 ****************************************************************************/
562 INTERRUPT(int_data_tlb_error)
563 TLB_PROLOG
564 TLB_LOCK
565
566 mfdear %r31
567
568 /*
569 * Save MAS0-MAS2 registers. There might be another tlb miss during
570 * pte lookup overwriting current contents (which was hw filled).
571 */
572 mfspr %r29, SPR_MAS0
573 mfspr %r28, SPR_MAS1
574 mfspr %r27, SPR_MAS2
575
576 /* Check faulting address. */
577 lis %r21, VM_MAXUSER_ADDRESS@h
578 ori %r21, %r21, VM_MAXUSER_ADDRESS@l
579 cmplw cr0, %r31, %r21
580 blt search_user_pmap
581
582 /* If it's kernel address, allow only supervisor mode misses. */
583 mfsrr1 %r21
584 mtcr %r21
585 bt 17, search_failed /* check MSR[PR] */
586
587 search_kernel_pmap:
588 /* Load r26 with kernel_pmap address */
589 bl 1f
590 .long kernel_pmap_store-.
591 1: mflr %r21
592 lwz %r26, 0(%r21)
593 add %r26, %r21, %r26 /* kernel_pmap_store in r26 */
594
595 /* Force kernel tid, set TID to 0 in MAS1. */
596 li %r21, 0
597 rlwimi %r28, %r21, 0, 8, 15 /* clear TID bits */
598
599 tlb_miss_handle:
600 /* This may result in nested tlb miss. */
601 bl pte_lookup /* returns PTE address in R25 */
602
603 cmpwi %r25, 0 /* pte found? */
604 beq search_failed
605
606 /* Finish up, write TLB entry. */
607 bl tlb_fill_entry
608
609 tlb_miss_return:
610 TLB_UNLOCK
611 TLB_RESTORE
612 rfi
613
614 search_user_pmap:
615 /* Load r26 with current user space process pmap */
616 GET_CPUINFO(%r26)
617 lwz %r26, PC_CURPMAP(%r26)
618
619 b tlb_miss_handle
620
621 search_failed:
622 /*
623 * Whenever we don't find a TLB mapping in PT, set a TLB0 entry with
624 * the faulting virtual address anyway, but put a fake RPN and no
625 * access rights. This should cause a following {D,I}SI exception.
626 */
627 lis %r23, 0xffff0000@h /* revoke all permissions */
628
629 /* Load MAS registers. */
630 mtspr SPR_MAS0, %r29
631 isync
632 mtspr SPR_MAS1, %r28
633 isync
634 mtspr SPR_MAS2, %r27
635 isync
636 mtspr SPR_MAS3, %r23
637 isync
638
639 bl zero_mas7
640 bl zero_mas8
641
642 tlbwe
643 msync
644 isync
645 b tlb_miss_return
646
647 /*****************************************************************************
648 *
649 * Return pte address that corresponds to given pmap/va. If there is no valid
650 * entry return 0.
651 *
652 * input: r26 - pmap
653 * input: r31 - dear
654 * output: r25 - pte address
655 *
656 * scratch regs used: r21
657 *
658 ****************************************************************************/
659 pte_lookup:
660 cmpwi %r26, 0
661 beq 1f /* fail quickly if pmap is invalid */
662
663 srwi %r21, %r31, PDIR_SHIFT /* pdir offset */
664 slwi %r21, %r21, PDIR_ENTRY_SHIFT /* multiply by pdir entry size */
665
666 addi %r25, %r26, PM_PDIR /* pmap pm_dir[] address */
667 add %r25, %r25, %r21 /* offset within pm_pdir[] table */
668 /*
669 * Get ptbl address, i.e. pmap->pm_pdir[pdir_idx]
670 * This load may cause a Data TLB miss for non-kernel pmap!
671 */
672 lwz %r25, 0(%r25)
673 cmpwi %r25, 0
674 beq 2f
675
676 lis %r21, PTBL_MASK@h
677 ori %r21, %r21, PTBL_MASK@l
678 and %r21, %r21, %r31
679
680 /* ptbl offset, multiply by ptbl entry size */
681 srwi %r21, %r21, (PTBL_SHIFT - PTBL_ENTRY_SHIFT)
682
683 add %r25, %r25, %r21 /* address of pte entry */
684 /*
685 * Get pte->flags
686 * This load may cause a Data TLB miss for non-kernel pmap!
687 */
688 lwz %r21, PTE_FLAGS(%r25)
689 andi. %r21, %r21, PTE_VALID@l
690 bne 2f
691 1:
692 li %r25, 0
693 2:
694 blr
695
696 /*****************************************************************************
697 *
698 * Load MAS1-MAS3 registers with data, write TLB entry
699 *
700 * input:
701 * r29 - mas0
702 * r28 - mas1
703 * r27 - mas2
704 * r25 - pte
705 *
706 * output: none
707 *
708 * scratch regs: r21-r23
709 *
710 ****************************************************************************/
711 tlb_fill_entry:
712 /*
713 * Update PTE flags: we have to do it atomically, as pmap_protect()
714 * running on other CPUs could attempt to update the flags at the same
715 * time.
716 */
717 li %r23, PTE_FLAGS
718 1:
719 lwarx %r21, %r23, %r25 /* get pte->flags */
720 oris %r21, %r21, PTE_REFERENCED@h /* set referenced bit */
721
722 andi. %r22, %r21, (PTE_SW | PTE_UW)@l /* check if writable */
723 beq 2f
724 ori %r21, %r21, PTE_MODIFIED@l /* set modified bit */
725 2:
726 stwcx. %r21, %r23, %r25 /* write it back */
727 bne- 1b
728
729 /* Update MAS2. */
730 rlwimi %r27, %r21, 13, 27, 30 /* insert WIMG bits from pte */
731
732 /* Setup MAS3 value in r23. */
733 lwz %r23, PTE_RPN(%r25) /* get pte->rpn */
734 rlwinm %r22, %r23, 20, 0, 11 /* extract MAS3 portion of RPN */
735
736 rlwimi %r22, %r21, 30, 26, 31 /* insert protection bits from pte */
737 rlwimi %r22, %r21, 20, 12, 19 /* insert lower 8 RPN bits to MAS3 */
738 rlwinm %r23, %r23, 20, 24, 31 /* MAS7 portion of RPN */
739
740 /* Load MAS registers. */
741 mtspr SPR_MAS0, %r29
742 isync
743 mtspr SPR_MAS1, %r28
744 isync
745 mtspr SPR_MAS2, %r27
746 isync
747 mtspr SPR_MAS3, %r22
748 isync
749 mtspr SPR_MAS7, %r23
750 isync
751
752 mflr %r21
753 bl zero_mas8
754 mtlr %r21
755
756 tlbwe
757 isync
758 msync
759 blr
760
761 /*****************************************************************************
762 * Instruction TLB miss interrupt
763 *
764 * Same notes as for the Data TLB miss
765 ****************************************************************************/
766 INTERRUPT(int_inst_tlb_error)
767 TLB_PROLOG
768 TLB_LOCK
769
770 mfsrr0 %r31 /* faulting address */
771
772 /*
773 * Save MAS0-MAS2 registers. There might be another tlb miss during pte
774 * lookup overwriting current contents (which was hw filled).
775 */
776 mfspr %r29, SPR_MAS0
777 mfspr %r28, SPR_MAS1
778 mfspr %r27, SPR_MAS2
779
780 mfsrr1 %r21
781 mtcr %r21
782
783 /* check MSR[PR] */
784 bt 17, search_user_pmap
785 b search_kernel_pmap
786
787
788 .globl interrupt_vector_top
789 interrupt_vector_top:
790
791 /*****************************************************************************
792 * Debug interrupt
793 ****************************************************************************/
794 INTERRUPT(int_debug)
795 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
796 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG)
797 GET_CPUINFO(%r3)
798 lwz %r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r3)
799 bl 0f
800 .long interrupt_vector_base-.
801 .long interrupt_vector_top-.
802 0: mflr %r5
803 lwz %r4,0(%r5) /* interrupt_vector_base in r4 */
804 add %r4,%r4,%r5
805 cmplw cr0, %r3, %r4
806 blt 1f
807 lwz %r4,4(%r5) /* interrupt_vector_top in r4 */
808 add %r4,%r4,%r5
809 addi %r4,%r4,4
810 cmplw cr0, %r3, %r4
811 bge 1f
812 /* Disable single-stepping for the interrupt handlers. */
813 lwz %r3, FRAME_SRR1+8(%r1);
814 rlwinm %r3, %r3, 0, 23, 21
815 stw %r3, FRAME_SRR1+8(%r1);
816 /* Restore srr0 and srr1 as they could have been clobbered. */
817 GET_CPUINFO(%r4)
818 lwz %r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0+8)(%r4);
819 mtspr SPR_SRR0, %r3
820 lwz %r4, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR1+8)(%r4);
821 mtspr SPR_SRR1, %r4
822 b 9f
823 1:
824 addi %r3, %r1, 8
825 bl CNAME(trap)
826 /*
827 * Handle ASTs, needed for proper support of single-stepping.
828 * We actually need to return to the process with an rfi.
829 */
830 b trapexit
831 9:
832 FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
833 rfci
834
835
836 /*****************************************************************************
837 * Common trap code
838 ****************************************************************************/
839 trap_common:
840 /* Call C trap dispatcher */
841 addi %r3, %r1, 8
842 bl CNAME(trap)
843
844 .globl CNAME(trapexit) /* exported for db_backtrace use */
845 CNAME(trapexit):
846 /* disable interrupts */
847 wrteei 0
848
849 /* Test AST pending - makes sense for user process only */
850 lwz %r5, FRAME_SRR1+8(%r1)
851 mtcr %r5
852 bf 17, 1f
853
854 GET_CPUINFO(%r3)
855 lwz %r4, PC_CURTHREAD(%r3)
856 lwz %r4, TD_FLAGS(%r4)
857 lis %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@h
858 ori %r5, %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@l
859 and. %r4, %r4, %r5
860 beq 1f
861
862 /* re-enable interrupts before calling ast() */
863 wrteei 1
864
865 addi %r3, %r1, 8
866 bl CNAME(ast)
867 .globl CNAME(asttrapexit) /* db_backtrace code sentinel #2 */
868 CNAME(asttrapexit):
869 b trapexit /* test ast ret value ? */
870 1:
871 FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
872 rfi
873
874
875 #if defined(KDB)
876 /*
877 * Deliberate entry to dbtrap
878 */
879 .globl CNAME(breakpoint)
880 CNAME(breakpoint):
881 mtsprg1 %r1
882 mfmsr %r3
883 mtsrr1 %r3
884 andi. %r3, %r3, ~(PSL_EE | PSL_ME)@l
885 mtmsr %r3 /* disable interrupts */
886 isync
887 GET_CPUINFO(%r3)
888 stw %r30, (PC_DBSAVE+CPUSAVE_R30)(%r3)
889 stw %r31, (PC_DBSAVE+CPUSAVE_R31)(%r3)
890
891 mflr %r31
892 mtsrr0 %r31
893
894 mfdear %r30
895 mfesr %r31
896 stw %r30, (PC_DBSAVE+CPUSAVE_BOOKE_DEAR)(%r3)
897 stw %r31, (PC_DBSAVE+CPUSAVE_BOOKE_ESR)(%r3)
898
899 mfsrr0 %r30
900 mfsrr1 %r31
901 stw %r30, (PC_DBSAVE+CPUSAVE_SRR0)(%r3)
902 stw %r31, (PC_DBSAVE+CPUSAVE_SRR1)(%r3)
903 isync
904
905 mfcr %r30
906
907 /*
908 * Now the kdb trap catching code.
909 */
910 dbtrap:
911 FRAME_SETUP(SPR_SPRG1, PC_DBSAVE, EXC_DEBUG)
912 /* Call C trap code: */
913 addi %r3, %r1, 8
914 bl CNAME(db_trap_glue)
915 or. %r3, %r3, %r3
916 bne dbleave
917 /* This wasn't for KDB, so switch to real trap: */
918 b trap_common
919
920 dbleave:
921 FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
922 rfi
923 #endif /* KDB */
924
925 clear_we:
926 lwz %r3, (FRAME_SRR1+8)(%r1)
927 rlwinm %r3, %r3, 0, 14, 12
928 stw %r3, (FRAME_SRR1+8)(%r1)
929 b trapexit
930
931 #ifdef SMP
932 ENTRY(tlb_lock)
933 GET_CPUINFO(%r5)
934 lwz %r5, PC_CURTHREAD(%r5)
935 1: lwarx %r4, 0, %r3
936 cmpwi %r4, TLB_UNLOCKED
937 bne 1b
938 stwcx. %r5, 0, %r3
939 bne- 1b
940 isync
941 msync
942 blr
943
944 ENTRY(tlb_unlock)
945 isync
946 msync
947 li %r4, TLB_UNLOCKED
948 stw %r4, 0(%r3)
949 isync
950 msync
951 blr
952
953 /*
954 * TLB miss spin locks. For each CPU we have a reservation granule (32 bytes);
955 * only a single word from this granule will actually be used as a spin lock
956 * for mutual exclusion between TLB miss handler and pmap layer that
957 * manipulates page table contents.
958 */
959 .data
960 .align 5
961 GLOBAL(tlb0_miss_locks)
962 .space RES_GRANULE * MAXCPU
963 #endif
Cache object: 9e029344c781f5af0f98aab19a44e625
|