FreeBSD/Linux Kernel Cross Reference
sys/arm/arm/swtch-v6.S
1 /* $NetBSD: cpuswitch.S,v 1.41 2003/11/15 08:44:18 scw Exp $ */
2
3 /*-
4 * Copyright 2003 Wasabi Systems, Inc.
5 * All rights reserved.
6 *
7 * Written by Steve C. Woodford for Wasabi Systems, Inc.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 * must display the following acknowledgement:
19 * This product includes software developed for the NetBSD Project by
20 * Wasabi Systems, Inc.
21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
22 * or promote products derived from this software without specific prior
23 * written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 * POSSIBILITY OF SUCH DAMAGE.
36 */
37 /*-
38 * Copyright (c) 1994-1998 Mark Brinicombe.
39 * Copyright (c) 1994 Brini.
40 * All rights reserved.
41 *
42 * This code is derived from software written for Brini by Mark Brinicombe
43 *
44 * Redistribution and use in source and binary forms, with or without
45 * modification, are permitted provided that the following conditions
46 * are met:
47 * 1. Redistributions of source code must retain the above copyright
48 * notice, this list of conditions and the following disclaimer.
49 * 2. Redistributions in binary form must reproduce the above copyright
50 * notice, this list of conditions and the following disclaimer in the
51 * documentation and/or other materials provided with the distribution.
52 * 3. All advertising materials mentioning features or use of this software
53 * must display the following acknowledgement:
54 * This product includes software developed by Brini.
55 * 4. The name of the company nor the name of the author may be used to
56 * endorse or promote products derived from this software without specific
57 * prior written permission.
58 *
59 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED
60 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
61 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
62 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
63 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
64 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
65 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
69 * SUCH DAMAGE.
70 *
71 * RiscBSD kernel project
72 *
73 * cpuswitch.S
74 *
75 * cpu switching functions
76 *
77 * Created : 15/10/94
78 *
79 */
80
81 #include "assym.inc"
82 #include "opt_sched.h"
83
84 #include <machine/asm.h>
85 #include <machine/asmacros.h>
86 #include <machine/armreg.h>
87 #include <machine/sysreg.h>
88 #include <machine/vfp.h>
89
90 __FBSDID("$FreeBSD$");
91
92 #if defined(SMP)
93 #define GET_PCPU(tmp, tmp2) \
94 mrc CP15_MPIDR(tmp); \
95 and tmp, tmp, #0xf; \
96 ldr tmp2, .Lcurpcpu+4; \
97 mul tmp, tmp, tmp2; \
98 ldr tmp2, .Lcurpcpu; \
99 add tmp, tmp, tmp2;
100 #else
101
102 #define GET_PCPU(tmp, tmp2) \
103 ldr tmp, .Lcurpcpu
104 #endif
105
106 #ifdef VFP
107 .fpu vfp /* allow VFP instructions */
108 #endif
109
110 .Lcurpcpu:
111 .word _C_LABEL(__pcpu)
112 .word PCPU_SIZE
113 .Lblocked_lock:
114 .word _C_LABEL(blocked_lock)
115
116 ENTRY(cpu_context_switch)
117 DSB
118 /*
119 * We can directly switch between translation tables only when the
120 * size of the mapping for any given virtual address is the same
121 * in the old and new translation tables.
122 * Thus, we must switch to kernel pmap translation table as
123 * intermediate mapping because all sizes of these mappings are same
124 * (or unmapped). The same is true for switch from kernel pmap
125 * translation table to new pmap one.
126 */
127 mov r2, #(CPU_ASID_KERNEL)
128 ldr r1, =(_C_LABEL(pmap_kern_ttb))
129 ldr r1, [r1]
130 mcr CP15_TTBR0(r1) /* switch to kernel TTB */
131 ISB
132 mcr CP15_TLBIASID(r2) /* flush not global TLBs */
133 DSB
134 mcr CP15_TTBR0(r0) /* switch to new TTB */
135 ISB
136 /*
137 * We must flush not global TLBs again because PT2MAP mapping
138 * is different.
139 */
140 mcr CP15_TLBIASID(r2) /* flush not global TLBs */
141 /*
142 * Flush entire Branch Target Cache because of the branch predictor
143 * is not architecturally invisible. See ARM Architecture Reference
144 * Manual ARMv7-A and ARMv7-R edition, page B2-1264(65), Branch
145 * predictors and Requirements for branch predictor maintenance
146 * operations sections.
147 */
148 /*
149 * Additionally, to mitigate mistrained branch predictor attack
150 * we must invalidate it on affected CPUs. Unfortunately, BPIALL
151 * is effectively NOP on Cortex-A15 so it needs special treatment.
152 */
153 ldr r0, [r8, #PC_BP_HARDEN_KIND]
154 cmp r0, #PCPU_BP_HARDEN_KIND_ICIALLU
155 mcrne CP15_BPIALL /* Flush entire Branch Target Cache */
156 mcreq CP15_ICIALLU /* This is the only way how to flush */
157 /* Branch Target Cache on Cortex-A15. */
158 DSB
159 mov pc, lr
160 END(cpu_context_switch)
161
162 /*
163 * cpu_throw(oldtd, newtd)
164 *
165 * Remove current thread state, then select the next thread to run
166 * and load its state.
167 * r0 = oldtd
168 * r1 = newtd
169 */
170 ENTRY(cpu_throw)
171 mov r10, r0 /* r10 = oldtd */
172 mov r11, r1 /* r11 = newtd */
173
174 #ifdef VFP /* This thread is dying, disable */
175 bl _C_LABEL(vfp_discard) /* VFP without preserving state. */
176 #endif
177 GET_PCPU(r8, r9) /* r8 = current pcpu */
178 ldr r4, [r8, #PC_CPUID] /* r4 = current cpu id */
179
180 cmp r10, #0 /* old thread? */
181 beq 2f /* no, skip */
182
183 /* Remove this CPU from the active list. */
184 ldr r5, [r8, #PC_CURPMAP]
185 mov r0, #(PM_ACTIVE)
186 add r5, r0 /* r5 = old pm_active */
187
188 /* Compute position and mask. */
189 #if _NCPUWORDS > 1
190 lsr r0, r4, #3
191 bic r0, #3
192 add r5, r0 /* r5 = position in old pm_active */
193 mov r2, #1
194 and r0, r4, #31
195 lsl r2, r0 /* r2 = mask */
196 #else
197 mov r2, #1
198 lsl r2, r4 /* r2 = mask */
199 #endif
200 /* Clear cpu from old active list. */
201 #ifdef SMP
202 1: ldrex r0, [r5]
203 bic r0, r2
204 strex r1, r0, [r5]
205 teq r1, #0
206 bne 1b
207 #else
208 ldr r0, [r5]
209 bic r0, r2
210 str r0, [r5]
211 #endif
212
213 2:
214 #ifdef INVARIANTS
215 cmp r11, #0 /* new thread? */
216 beq badsw1 /* no, panic */
217 #endif
218 ldr r7, [r11, #(TD_PCB)] /* r7 = new PCB */
219
220 /*
221 * Registers at this point
222 * r4 = current cpu id
223 * r7 = new PCB
224 * r8 = current pcpu
225 * r11 = newtd
226 */
227
228 /* MMU switch to new thread. */
229 ldr r0, [r7, #(PCB_PAGEDIR)]
230 #ifdef INVARIANTS
231 cmp r0, #0 /* new thread? */
232 beq badsw4 /* no, panic */
233 #endif
234 bl _C_LABEL(cpu_context_switch)
235
236 /*
237 * Set new PMAP as current one.
238 * Insert cpu to new active list.
239 */
240
241 ldr r6, [r11, #(TD_PROC)] /* newtd->proc */
242 ldr r6, [r6, #(P_VMSPACE)] /* newtd->proc->vmspace */
243 add r6, #VM_PMAP /* newtd->proc->vmspace->pmap */
244 str r6, [r8, #PC_CURPMAP] /* store to curpmap */
245
246 mov r0, #PM_ACTIVE
247 add r6, r0 /* r6 = new pm_active */
248
249 /* compute position and mask */
250 #if _NCPUWORDS > 1
251 lsr r0, r4, #3
252 bic r0, #3
253 add r6, r0 /* r6 = position in new pm_active */
254 mov r2, #1
255 and r0, r4, #31
256 lsl r2, r0 /* r2 = mask */
257 #else
258 mov r2, #1
259 lsl r2, r4 /* r2 = mask */
260 #endif
261 /* Set cpu to new active list. */
262 #ifdef SMP
263 1: ldrex r0, [r6]
264 orr r0, r2
265 strex r1, r0, [r6]
266 teq r1, #0
267 bne 1b
268 #else
269 ldr r0, [r6]
270 orr r0, r2
271 str r0, [r6]
272 #endif
273 /*
274 * Registers at this point.
275 * r7 = new PCB
276 * r8 = current pcpu
277 * r11 = newtd
278 * They must match the ones in sw1 position !!!
279 */
280 DMB
281 b sw1 /* share new thread init with cpu_switch() */
282 END(cpu_throw)
283
284 /*
285 * cpu_switch(oldtd, newtd, lock)
286 *
287 * Save the current thread state, then select the next thread to run
288 * and load its state.
289 * r0 = oldtd
290 * r1 = newtd
291 * r2 = lock (new lock for old thread)
292 */
293 ENTRY(cpu_switch)
294 /* Interrupts are disabled. */
295 #ifdef INVARIANTS
296 cmp r0, #0 /* old thread? */
297 beq badsw2 /* no, panic */
298 #endif
299 /* Save all the registers in the old thread's pcb. */
300 ldr r3, [r0, #(TD_PCB)]
301 add r3, #(PCB_R4)
302 stmia r3, {r4-r12, sp, lr, pc}
303 mrc CP15_TPIDRURW(r4)
304 str r4, [r3, #(PCB_TPIDRURW - PCB_R4)]
305
306 #ifdef INVARIANTS
307 cmp r1, #0 /* new thread? */
308 beq badsw3 /* no, panic */
309 #endif
310 /*
311 * Save arguments. Note that we can now use r0-r14 until
312 * it is time to restore them for the new thread. However,
313 * some registers are not safe over function call.
314 */
315 mov r9, r2 /* r9 = lock */
316 mov r10, r0 /* r10 = oldtd */
317 mov r11, r1 /* r11 = newtd */
318
319 GET_PCPU(r8, r3) /* r8 = current PCPU */
320 ldr r7, [r11, #(TD_PCB)] /* r7 = newtd->td_pcb */
321
322
323
324 #ifdef VFP
325 ldr r3, [r10, #(TD_PCB)]
326 fmrx r0, fpexc /* If the VFP is enabled */
327 tst r0, #(VFPEXC_EN) /* the current thread has */
328 movne r1, #1 /* used it, so go save */
329 addne r0, r3, #(PCB_VFPSTATE) /* the state into the PCB */
330 blne _C_LABEL(vfp_store) /* and disable the VFP. */
331 #endif
332
333 /*
334 * MMU switch. If we're switching to a thread with the same
335 * address space as the outgoing one, we can skip the MMU switch.
336 */
337 mrc CP15_TTBR0(r1) /* r1 = old TTB */
338 ldr r0, [r7, #(PCB_PAGEDIR)] /* r0 = new TTB */
339 cmp r0, r1 /* Switching to the TTB? */
340 beq sw0 /* same TTB, skip */
341
342 #ifdef INVARIANTS
343 cmp r0, #0 /* new thread? */
344 beq badsw4 /* no, panic */
345 #endif
346
347 bl cpu_context_switch /* new TTB as argument */
348
349 /*
350 * Registers at this point
351 * r7 = new PCB
352 * r8 = current pcpu
353 * r9 = lock
354 * r10 = oldtd
355 * r11 = newtd
356 */
357
358 /*
359 * Set new PMAP as current one.
360 * Update active list on PMAPs.
361 */
362 ldr r6, [r11, #TD_PROC] /* newtd->proc */
363 ldr r6, [r6, #P_VMSPACE] /* newtd->proc->vmspace */
364 add r6, #VM_PMAP /* newtd->proc->vmspace->pmap */
365
366 ldr r5, [r8, #PC_CURPMAP] /* get old curpmap */
367 str r6, [r8, #PC_CURPMAP] /* and save new one */
368
369 mov r0, #PM_ACTIVE
370 add r5, r0 /* r5 = old pm_active */
371 add r6, r0 /* r6 = new pm_active */
372
373 /* Compute position and mask. */
374 ldr r4, [r8, #PC_CPUID]
375 #if _NCPUWORDS > 1
376 lsr r0, r4, #3
377 bic r0, #3
378 add r5, r0 /* r5 = position in old pm_active */
379 add r6, r0 /* r6 = position in new pm_active */
380 mov r2, #1
381 and r0, r4, #31
382 lsl r2, r0 /* r2 = mask */
383 #else
384 mov r2, #1
385 lsl r2, r4 /* r2 = mask */
386 #endif
387 /* Clear cpu from old active list. */
388 #ifdef SMP
389 1: ldrex r0, [r5]
390 bic r0, r2
391 strex r1, r0, [r5]
392 teq r1, #0
393 bne 1b
394 #else
395 ldr r0, [r5]
396 bic r0, r2
397 str r0, [r5]
398 #endif
399 /* Set cpu to new active list. */
400 #ifdef SMP
401 1: ldrex r0, [r6]
402 orr r0, r2
403 strex r1, r0, [r6]
404 teq r1, #0
405 bne 1b
406 #else
407 ldr r0, [r6]
408 orr r0, r2
409 str r0, [r6]
410 #endif
411
412 sw0:
413 /*
414 * Registers at this point
415 * r7 = new PCB
416 * r8 = current pcpu
417 * r9 = lock
418 * r10 = oldtd
419 * r11 = newtd
420 */
421
422 /* Change the old thread lock. */
423 add r5, r10, #TD_LOCK
424 DMB
425 1: ldrex r0, [r5]
426 strex r1, r9, [r5]
427 teq r1, #0
428 bne 1b
429 DMB
430
431 sw1:
432 clrex
433 /*
434 * Registers at this point
435 * r7 = new PCB
436 * r8 = current pcpu
437 * r11 = newtd
438 */
439
440 #if defined(SMP) && defined(SCHED_ULE)
441 /*
442 * 386 and amd64 do the blocked lock test only for SMP and SCHED_ULE
443 * QQQ: What does it mean in reality and why is it done?
444 */
445 ldr r6, =blocked_lock
446 1:
447 ldr r3, [r11, #TD_LOCK] /* atomic write regular read */
448 cmp r3, r6
449 beq 1b
450 #endif
451
452 /* We have a new curthread now so make a note it */
453 str r11, [r8, #PC_CURTHREAD]
454 mcr CP15_TPIDRPRW(r11)
455
456 /* store pcb in per cpu structure */
457 str r7, [r8, #PC_CURPCB]
458
459 /*
460 * Restore all saved registers and return. Note that some saved
461 * registers can be changed when either cpu_fork(), cpu_copy_thread(),
462 * cpu_fork_kthread_handler(), or makectx() was called.
463 *
464 * The value of TPIDRURW is also written into TPIDRURO, as
465 * userspace still uses TPIDRURO, modifying it through
466 * sysarch(ARM_SET_TP, addr).
467 */
468 ldr r3, [r7, #PCB_TPIDRURW]
469 mcr CP15_TPIDRURW(r3) /* write tls thread reg 2 */
470 mcr CP15_TPIDRURO(r3) /* write tls thread reg 3 */
471 add r3, r7, #PCB_R4
472 ldmia r3, {r4-r12, sp, pc}
473
474 #ifdef INVARIANTS
475 badsw1:
476 ldr r0, =sw1_panic_str
477 bl _C_LABEL(panic)
478 1: nop
479 b 1b
480
481 badsw2:
482 ldr r0, =sw2_panic_str
483 bl _C_LABEL(panic)
484 1: nop
485 b 1b
486
487 badsw3:
488 ldr r0, =sw3_panic_str
489 bl _C_LABEL(panic)
490 1: nop
491 b 1b
492
493 badsw4:
494 ldr r0, =sw4_panic_str
495 bl _C_LABEL(panic)
496 1: nop
497 b 1b
498
499 sw1_panic_str:
500 .asciz "cpu_throw: no newthread supplied.\n"
501 sw2_panic_str:
502 .asciz "cpu_switch: no curthread supplied.\n"
503 sw3_panic_str:
504 .asciz "cpu_switch: no newthread supplied.\n"
505 sw4_panic_str:
506 .asciz "cpu_switch: new pagedir is NULL.\n"
507 #endif
508 END(cpu_switch)
Cache object: 29e1804213be251bb3b663260126cf8f
|