Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/osfmk/ppc/savearea_asm.s
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* 2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_LICENSE_HEADER_START@ 5 * 6 * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved. 7 * 8 * This file contains Original Code and/or Modifications of Original Code 9 * as defined in and that are subject to the Apple Public Source License 10 * Version 2.0 (the 'License'). You may not use this file except in 11 * compliance with the License. Please obtain a copy of the License at 12 * http://www.opensource.apple.com/apsl/ and read it before using this 13 * file. 14 * 15 * The Original Code and all software distributed under the License are 16 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 17 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 18 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 20 * Please see the License for the specific language governing rights and 21 * limitations under the License. 22 * 23 * @APPLE_LICENSE_HEADER_END@ 24 */ 25 26 #define FPVECDBG 0 27 28 #include <assym.s> 29 #include <debug.h> 30 #include <cpus.h> 31 #include <db_machine_commands.h> 32 #include <mach_rt.h> 33 34 #include <mach_debug.h> 35 #include <ppc/asm.h> 36 #include <ppc/proc_reg.h> 37 #include <ppc/exception.h> 38 #include <ppc/Performance.h> 39 #include <ppc/exception.h> 40 #include <ppc/savearea.h> 41 #include <mach/ppc/vm_param.h> 42 43 .text 44 45 /* Register usage conventions in this code: 46 * r9 = return address 47 * r10 = per-proc ptr 48 * r11 = MSR at entry 49 * cr6 = feature flags (ie, pf64Bit) 50 * 51 * Because much of this code deals with physical addresses, 52 * there are parallel paths for 32- and 64-bit machines. 53 */ 54 55 56 /* 57 * *********************** 58 * * s a v e _ q u e u e * 59 * *********************** 60 * 61 * void save_queue(ppnum_t pagenum); 62 * 63 * This routine will add a savearea block to the free list. 64 * We also queue the block to the free pool list. This is a 65 * circular double linked list. Because this block has no free entries, 66 * it gets queued to the end of the list 67 */ 68 .align 5 69 .globl EXT(save_queue) 70 71 LEXT(save_queue) 72 mflr r9 ; get return address 73 mr r8,r3 ; move pagenum out of the way 74 bl saveSetup ; turn translation off, 64-bit on, load many regs 75 bf-- pf64Bitb,saveQueue32 ; skip if 32-bit processor 76 77 sldi r2,r8,12 ; r2 <-- phys address of page 78 li r8,sac_cnt ; Get the number of saveareas per page 79 mr r4,r2 ; Point to start of chain 80 li r0,SAVempty ; Get empty marker 81 82 saveQueue64a: 83 addic. r8,r8,-1 ; Keep track of how many we did 84 stb r0,SAVflags+2(r4) ; Set empty 85 addi r7,r4,SAVsize ; Point to the next slot 86 ble- saveQueue64b ; We are done with the chain 87 std r7,SAVprev(r4) ; Set this chain 88 mr r4,r7 ; Step to the next 89 b saveQueue64a ; Fill the whole block... 90 91 saveQueue64b: 92 bl savelock ; Go lock the save anchor 93 94 ld r7,SVfree(0) ; Get the free save area list anchor 95 lwz r6,SVfreecnt(0) ; Get the number of free saveareas 96 97 std r2,SVfree(0) ; Queue in the new one 98 addi r6,r6,sac_cnt ; Count the ones we are linking in 99 std r7,SAVprev(r4) ; Queue the old first one off of us 100 stw r6,SVfreecnt(0) ; Save the new count 101 b saveQueueExit 102 103 ; Handle 32-bit processor. 104 105 saveQueue32: 106 slwi r2,r8,12 ; r2 <-- phys address of page 107 li r8,sac_cnt ; Get the number of saveareas per page 108 mr r4,r2 ; Point to start of chain 109 li r0,SAVempty ; Get empty marker 110 111 saveQueue32a: 112 addic. r8,r8,-1 ; Keep track of how many we did 113 stb r0,SAVflags+2(r4) ; Set empty 114 addi r7,r4,SAVsize ; Point to the next slot 115 ble- saveQueue32b ; We are done with the chain 116 stw r7,SAVprev+4(r4) ; Set this chain 117 mr r4,r7 ; Step to the next 118 b saveQueue32a ; Fill the whole block... 119 120 saveQueue32b: 121 bl savelock ; Go lock the save anchor 122 123 lwz r7,SVfree+4(0) ; Get the free save area list anchor 124 lwz r6,SVfreecnt(0) ; Get the number of free saveareas 125 126 stw r2,SVfree+4(0) ; Queue in the new one 127 addi r6,r6,sac_cnt ; Count the ones we are linking in 128 stw r7,SAVprev+4(r4) ; Queue the old first one off of us 129 stw r6,SVfreecnt(0) ; Save the new count 130 131 saveQueueExit: ; join here from 64-bit path 132 bl saveunlock ; Unlock the list and set the adjust count 133 mtlr r9 ; Restore the return 134 135 #if FPVECDBG 136 mfsprg r2,1 ; (TEST/DEBUG) 137 mr. r2,r2 ; (TEST/DEBUG) 138 beq-- saveRestore ; (TEST/DEBUG) 139 lis r0,hi16(CutTrace) ; (TEST/DEBUG) 140 li r2,0x2201 ; (TEST/DEBUG) 141 oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG) 142 sc ; (TEST/DEBUG) 143 #endif 144 b saveRestore ; Restore interrupts and translation 145 146 /* 147 * ***************************** 148 * * s a v e _ g e t _ i n i t * 149 * ***************************** 150 * 151 * addr64_t save_get_init(void); 152 * 153 * Note that save_get_init is used in initial processor startup only. It 154 * is used because translation is on, but no tables exist yet and we have 155 * no V=R BAT registers that cover the entire physical memory. 156 */ 157 .align 5 158 .globl EXT(save_get_init) 159 160 LEXT(save_get_init) 161 mflr r9 ; get return address 162 bl saveSetup ; turn translation off, 64-bit on, load many regs 163 bfl-- pf64Bitb,saveGet32 ; Get r3 <- savearea, r5 <- page address (with SAC) 164 btl++ pf64Bitb,saveGet64 ; get one on a 64-bit machine 165 bl saveRestore ; restore translation etc 166 mtlr r9 167 168 ; unpack the physaddr in r3 into a long long in (r3,r4) 169 170 mr r4,r3 ; copy low word of phys address to r4 171 li r3,0 ; assume upper word was 0 172 bflr-- pf64Bitb ; if 32-bit processor, return 173 srdi r3,r4,32 ; unpack reg64_t to addr64_t on 64-bit machine 174 rlwinm r4,r4,0,0,31 175 blr 176 177 178 /* 179 * ******************* 180 * * s a v e _ g e t * 181 * ******************* 182 * 183 * savearea *save_get(void); 184 * 185 * Allocate a savearea, returning a virtual address. NOTE: we must preserve 186 * r0, r2, and r12. Our callers in cswtch.s depend on this. 187 */ 188 .align 5 189 .globl EXT(save_get) 190 191 LEXT(save_get) 192 mflr r9 ; get return address 193 mr r5,r0 ; copy regs before saveSetup nails them 194 bl saveSetup ; turn translation off, 64-bit on, load many regs 195 bf-- pf64Bitb,svgt1 ; skip if 32-bit processor 196 197 std r5,tempr0(r10) ; save r0 in per-proc across call to saveGet64 198 std r2,tempr2(r10) ; and r2 199 std r12,tempr4(r10) ; and r12 200 bl saveGet64 ; get r3 <- savearea, r5 <- page address (with SAC) 201 ld r0,tempr0(r10) ; restore callers regs 202 ld r2,tempr2(r10) 203 ld r12,tempr4(r10) 204 b svgt2 205 206 svgt1: ; handle 32-bit processor 207 stw r5,tempr0+4(r10) ; save r0 in per-proc across call to saveGet32 208 stw r2,tempr2+4(r10) ; and r2 209 stw r12,tempr4+4(r10) ; and r12 210 bl saveGet32 ; get r3 <- savearea, r5 <- page address (with SAC) 211 lwz r0,tempr0+4(r10) ; restore callers regs 212 lwz r2,tempr2+4(r10) 213 lwz r12,tempr4+4(r10) 214 215 svgt2: 216 lwz r5,SACvrswap+4(r5) ; Get the virtual to real translation (only need low word) 217 mtlr r9 ; restore return address 218 xor r3,r3,r5 ; convert physaddr to virtual 219 rlwinm r3,r3,0,0,31 ; 0 upper word if a 64-bit machine 220 221 #if FPVECDBG 222 mr r6,r0 ; (TEST/DEBUG) 223 mr r7,r2 ; (TEST/DEBUG) 224 mfsprg r2,1 ; (TEST/DEBUG) 225 mr. r2,r2 ; (TEST/DEBUG) 226 beq-- svgDBBypass ; (TEST/DEBUG) 227 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG) 228 li r2,0x2203 ; (TEST/DEBUG) 229 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG) 230 sc ; (TEST/DEBUG) 231 svgDBBypass: ; (TEST/DEBUG) 232 mr r0,r6 ; (TEST/DEBUG) 233 mr r2,r7 ; (TEST/DEBUG) 234 #endif 235 b saveRestore ; restore MSR and return to our caller 236 237 238 /* 239 * *********************************** 240 * * s a v e _ g e t _ p h y s _ 3 2 * 241 * *********************************** 242 * 243 * reg64_t save_get_phys(void); 244 * 245 * This is the entry normally called from lowmem_vectors.s with 246 * translation and interrupts already off. 247 * MUST NOT TOUCH CR7 248 */ 249 .align 5 250 .globl EXT(save_get_phys_32) 251 252 LEXT(save_get_phys_32) 253 mfsprg r10,0 ; get the per-proc ptr 254 b saveGet32 ; Get r3 <- savearea, r5 <- page address (with SAC) 255 256 257 /* 258 * *********************************** 259 * * s a v e _ g e t _ p h y s _ 6 4 * 260 * *********************************** 261 * 262 * reg64_t save_get_phys_64(void); 263 * 264 * This is the entry normally called from lowmem_vectors.s with 265 * translation and interrupts already off, and in 64-bit mode. 266 * MUST NOT TOUCH CR7 267 */ 268 .align 5 269 .globl EXT(save_get_phys_64) 270 271 LEXT(save_get_phys_64) 272 mfsprg r10,0 ; get the per-proc ptr 273 b saveGet64 ; Get r3 <- savearea, r5 <- page address (with SAC) 274 275 276 /* 277 * ********************* 278 * * s a v e G e t 6 4 * 279 * ********************* 280 * 281 * This is the internal routine to allocate a savearea on a 64-bit processor. 282 * Note that we must not take any exceptions of any kind, including PTE misses, as that 283 * would deadlock trying to reenter this routine. We pass back the 64-bit physical address. 284 * First we try the local list. If that is below a threshold, we try the global free list, 285 * which requires taking a lock, and replenish. If there are no saveareas in either list, 286 * we will install the backpocket and choke. This routine assumes that the caller has 287 * turned translation off, masked interrupts, turned on 64-bit mode, and set up: 288 * r10 = per-proc ptr 289 * 290 * We return: 291 * r3 = 64-bit physical address of the savearea 292 * r5 = 64-bit physical address of the page the savearea is in, with SAC 293 * 294 * We destroy: 295 * r2-r8. 296 * 297 * MUST NOT TOUCH CR7 298 */ 299 300 saveGet64: 301 lwz r8,lclfreecnt(r10) ; Get the count 302 ld r3,lclfree(r10) ; Get the start of local savearea list 303 cmplwi r8,LocalSaveMin ; Are we too low? 304 ble-- saveGet64GetGlobal ; We are too low and need to grow list... 305 306 ; Get it from the per-processor local list. 307 308 saveGet64GetLocal: 309 li r2,0x5555 ; get r2 <-- 0x55555555 55555555, our bugbug constant 310 ld r4,SAVprev(r3) ; Chain to the next one 311 oris r2,r2,0x5555 312 subi r8,r8,1 ; Back down count 313 rldimi r2,r2,32,0 314 315 std r2,SAVprev(r3) ; bug next ptr 316 stw r2,SAVlevel(r3) ; bug context ID 317 li r6,0 318 std r4,lclfree(r10) ; Unchain first savearea 319 stw r2,SAVact(r3) ; bug activation ptr 320 rldicr r5,r3,0,51 ; r5 <-- page ptr, where SAC is kept 321 stw r8,lclfreecnt(r10) ; Set new count 322 stw r6,SAVflags(r3) ; clear the flags 323 324 blr 325 326 ; Local list was low so replenish from global list. 327 ; r7 = return address to caller of saveGet64 328 ; r8 = lclfreecnt 329 ; r10 = per-proc ptr 330 331 saveGet64GetGlobal: 332 mflr r7 ; save return adress 333 subfic r5,r8,LocalSaveTarget ; Get the number of saveareas we need to grab to get to target 334 bl savelock ; Go lock up the anchor 335 336 lwz r2,SVfreecnt(0) ; Get the number on this list 337 ld r8,SVfree(0) ; Get the head of the save area list 338 339 sub r3,r2,r5 ; Get number left after we swipe enough for local list 340 sradi r3,r3,63 ; Get 0 if enough or -1 if not 341 andc r4,r5,r3 ; Get number to get if there are enough, 0 otherwise 342 and r5,r2,r3 ; Get 0 if there are enough, number on list otherwise 343 or. r5,r4,r5 ; r5 <- number we will move from global to local list 344 beq-- saveGet64NoFree ; There are none to get... 345 346 mtctr r5 ; Get loop count 347 mr r6,r8 ; Remember the first in the list 348 349 saveGet64c: 350 bdz saveGet64d ; Count down and branch when we hit 0... 351 ld r8,SAVprev(r8) ; Get the next 352 b saveGet64c ; Keep going... 353 354 saveGet64d: 355 ld r3,SAVprev(r8) ; Get the next one 356 lwz r4,SVinuse(0) ; Get the in use count 357 sub r2,r2,r5 ; Count down what we stole 358 std r3,SVfree(0) ; Set the new first in list 359 add r4,r4,r5 ; Count the ones we just put in the local list as "in use" 360 stw r2,SVfreecnt(0) ; Set the new count 361 stw r4,SVinuse(0) ; Set the new in use count 362 363 ld r4,lclfree(r10) ; Get the old head of list 364 lwz r3,lclfreecnt(r10) ; Get the old count 365 std r6,lclfree(r10) ; Set the new head of the list 366 add r3,r3,r5 ; Get the new count 367 std r4,SAVprev(r8) ; Point to the old head 368 stw r3,lclfreecnt(r10) ; Set the new count 369 370 bl saveunlock ; Update the adjust field and unlock 371 mtlr r7 ; restore return address 372 b saveGet64 ; Start over and finally allocate the savearea... 373 374 ; The local list is below the repopulate threshold and the global list is empty. 375 ; First we check if there are any left in the local list and if so, we allow 376 ; them to be allocated. If not, we release the backpocket list and choke. 377 ; There is nothing more that we can do at this point. Hopefully we stay alive 378 ; long enough to grab some much-needed panic information. 379 ; r7 = return address to caller of saveGet64 380 ; r10 = per-proc ptr 381 382 saveGet64NoFree: 383 lwz r8,lclfreecnt(r10) ; Get the count 384 mr. r8,r8 ; Are there any reserve to get? 385 beq-- saveGet64Choke ; No, go choke and die... 386 bl saveunlock ; Update the adjust field and unlock 387 ld r3,lclfree(r10) ; Get the start of local savearea list 388 lwz r8,lclfreecnt(r10) ; Get the count 389 mtlr r7 ; restore return address 390 b saveGet64GetLocal ; We have some left, dip on in... 391 392 ; We who are about to die salute you. The savearea chain is messed up or 393 ; empty. Add in a few so we have enough to take down the system. 394 395 saveGet64Choke: 396 lis r9,hi16(EXT(backpocket)) ; Get high order of back pocket 397 ori r9,r9,lo16(EXT(backpocket)) ; and low part 398 399 lwz r8,SVfreecnt-saveanchor(r9) ; Get the new number of free elements 400 ld r7,SVfree-saveanchor(r9) ; Get the head of the chain 401 lwz r6,SVinuse(0) ; Get total in the old list 402 403 stw r8,SVfreecnt(0) ; Set the new number of free elements 404 add r6,r6,r8 ; Add in the new ones 405 std r7,SVfree(0) ; Set the new head of the chain 406 stw r6,SVinuse(0) ; Set total in the new list 407 408 saveGetChokeJoin: ; join in the fun from 32-bit mode 409 lis r0,hi16(Choke) ; Set choke firmware call 410 li r7,0 ; Get a clear register to unlock 411 ori r0,r0,lo16(Choke) ; Set the rest of the choke call 412 li r3,failNoSavearea ; Set failure code 413 414 eieio ; Make sure all is committed 415 stw r7,SVlock(0) ; Unlock the free list 416 sc ; System ABEND 417 418 419 /* 420 * ********************* 421 * * s a v e G e t 3 2 * 422 * ********************* 423 * 424 * This is the internal routine to allocate a savearea on a 32-bit processor. 425 * Note that we must not take any exceptions of any kind, including PTE misses, as that 426 * would deadlock trying to reenter this routine. We pass back the 32-bit physical address. 427 * First we try the local list. If that is below a threshold, we try the global free list, 428 * which requires taking a lock, and replenish. If there are no saveareas in either list, 429 * we will install the backpocket and choke. This routine assumes that the caller has 430 * turned translation off, masked interrupts, and set up: 431 * r10 = per-proc ptr 432 * 433 * We return: 434 * r3 = 32-bit physical address of the savearea 435 * r5 = 32-bit physical address of the page the savearea is in, with SAC 436 * 437 * We destroy: 438 * r2-r8. 439 */ 440 441 saveGet32: 442 lwz r8,lclfreecnt(r10) ; Get the count 443 lwz r3,lclfree+4(r10) ; Get the start of local savearea list 444 cmplwi r8,LocalSaveMin ; Are we too low? 445 ble- saveGet32GetGlobal ; We are too low and need to grow list... 446 447 ; Get savearea from per-processor local list. 448 449 saveGet32GetLocal: 450 li r2,0x5555 ; get r2 <-- 0x55555555, our bugbug constant 451 lwz r4,SAVprev+4(r3) ; Chain to the next one 452 oris r2,r2,0x5555 453 subi r8,r8,1 ; Back down count 454 455 stw r2,SAVprev+4(r3) ; bug next ptr 456 stw r2,SAVlevel(r3) ; bug context ID 457 li r6,0 458 stw r4,lclfree+4(r10) ; Unchain first savearea 459 stw r2,SAVact(r3) ; bug activation ptr 460 rlwinm r5,r3,0,0,19 ; r5 <-- page ptr, where SAC is kept 461 stw r8,lclfreecnt(r10) ; Set new count 462 stw r6,SAVflags(r3) ; clear the flags 463 464 blr 465 466 ; Local list was low so replenish from global list. 467 ; r7 = return address to caller of saveGet32 468 ; r8 = lclfreecnt 469 ; r10 = per-proc ptr 470 471 saveGet32GetGlobal: 472 mflr r7 ; save return adress 473 subfic r5,r8,LocalSaveTarget ; Get the number of saveareas we need to grab to get to target 474 bl savelock ; Go lock up the anchor 475 476 lwz r2,SVfreecnt(0) ; Get the number on this list 477 lwz r8,SVfree+4(0) ; Get the head of the save area list 478 479 sub r3,r2,r5 ; Get number left after we swipe enough for local list 480 srawi r3,r3,31 ; Get 0 if enough or -1 if not 481 andc r4,r5,r3 ; Get number to get if there are enough, 0 otherwise 482 and r5,r2,r3 ; Get 0 if there are enough, number on list otherwise 483 or. r5,r4,r5 ; r5 <- number we will move from global to local list 484 beq- saveGet32NoFree ; There are none to get... 485 486 mtctr r5 ; Get loop count 487 mr r6,r8 ; Remember the first in the list 488 489 saveGet32c: 490 bdz saveGet32d ; Count down and branch when we hit 0... 491 lwz r8,SAVprev+4(r8) ; Get the next 492 b saveGet32c ; Keep going... 493 494 saveGet32d: 495 lwz r3,SAVprev+4(r8) ; Get the next one 496 lwz r4,SVinuse(0) ; Get the in use count 497 sub r2,r2,r5 ; Count down what we stole 498 stw r3,SVfree+4(0) ; Set the new first in list 499 add r4,r4,r5 ; Count the ones we just put in the local list as "in use" 500 stw r2,SVfreecnt(0) ; Set the new count 501 stw r4,SVinuse(0) ; Set the new in use count 502 503 lwz r4,lclfree+4(r10) ; Get the old head of list 504 lwz r3,lclfreecnt(r10) ; Get the old count 505 stw r6,lclfree+4(r10) ; Set the new head of the list 506 add r3,r3,r5 ; Get the new count 507 stw r4,SAVprev+4(r8) ; Point to the old head 508 stw r3,lclfreecnt(r10) ; Set the new count 509 510 bl saveunlock ; Update the adjust field and unlock 511 mtlr r7 ; restore return address 512 b saveGet32 ; Start over and finally allocate the savearea... 513 514 ; The local list is below the repopulate threshold and the global list is empty. 515 ; First we check if there are any left in the local list and if so, we allow 516 ; them to be allocated. If not, we release the backpocket list and choke. 517 ; There is nothing more that we can do at this point. Hopefully we stay alive 518 ; long enough to grab some much-needed panic information. 519 ; r7 = return address to caller of saveGet32 520 ; r10 = per-proc ptr 521 522 saveGet32NoFree: 523 lwz r8,lclfreecnt(r10) ; Get the count 524 mr. r8,r8 ; Are there any reserve to get? 525 beq- saveGet32Choke ; No, go choke and die... 526 bl saveunlock ; Update the adjust field and unlock 527 lwz r3,lclfree+4(r10) ; Get the start of local savearea list 528 lwz r8,lclfreecnt(r10) ; Get the count 529 mtlr r7 ; restore return address 530 b saveGet32GetLocal ; We have some left, dip on in... 531 532 ; We who are about to die salute you. The savearea chain is messed up or 533 ; empty. Add in a few so we have enough to take down the system. 534 535 saveGet32Choke: 536 lis r9,hi16(EXT(backpocket)) ; Get high order of back pocket 537 ori r9,r9,lo16(EXT(backpocket)) ; and low part 538 539 lwz r8,SVfreecnt-saveanchor(r9) ; Get the new number of free elements 540 lwz r7,SVfree+4-saveanchor(r9) ; Get the head of the chain 541 lwz r6,SVinuse(0) ; Get total in the old list 542 543 stw r8,SVfreecnt(0) ; Set the new number of free elements 544 add r6,r6,r8 ; Add in the new ones (why?) 545 stw r7,SVfree+4(0) ; Set the new head of the chain 546 stw r6,SVinuse(0) ; Set total in the new list 547 548 b saveGetChokeJoin 549 550 551 /* 552 * ******************* 553 * * s a v e _ r e t * 554 * ******************* 555 * 556 * void save_ret(struct savearea *); // normal call 557 * void save_ret_wMSR(struct savearea *,reg64_t); // passes MSR to restore as 2nd arg 558 * 559 * Return a savearea passed by virtual address to the free list. 560 * Note really well: we can take NO exceptions of any kind, 561 * including a PTE miss once the savearea lock is held. That's 562 * a guaranteed deadlock. That means we must disable for interrutions 563 * and turn all translation off. 564 */ 565 .globl EXT(save_ret_wMSR) ; alternate entry pt w MSR to restore in r4 566 567 LEXT(save_ret_wMSR) 568 crset 31 ; set flag for save_ret_wMSR 569 b svrt1 ; join common code 570 571 .align 5 572 .globl EXT(save_ret) 573 574 LEXT(save_ret) 575 crclr 31 ; clear flag for save_ret_wMSR 576 svrt1: ; join from save_ret_wMSR 577 mflr r9 ; get return address 578 rlwinm r7,r3,0,0,19 ; get virtual address of SAC area at start of page 579 mr r8,r3 ; save virtual address 580 lwz r5,SACvrswap+0(r7) ; get 64-bit converter from V to R 581 lwz r6,SACvrswap+4(r7) ; both halves, though only bottom used on 32-bit machine 582 #if FPVECDBG 583 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG) 584 li r2,0x2204 ; (TEST/DEBUG) 585 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG) 586 sc ; (TEST/DEBUG) 587 #endif 588 bl saveSetup ; turn translation off, 64-bit on, load many regs 589 bf++ 31,svrt3 ; skip if not save_ret_wMSR 590 mr r11,r4 ; was save_ret_wMSR, so overwrite saved MSR 591 svrt3: 592 bf-- pf64Bitb,svrt4 ; skip if a 32-bit processor 593 594 ; Handle 64-bit processor. 595 596 rldimi r6,r5,32,0 ; merge upper and lower halves of SACvrswap together 597 xor r3,r8,r6 ; get r3 <- 64-bit physical address of this savearea 598 bl saveRet64 ; return it 599 mtlr r9 ; restore return address 600 b saveRestore64 ; restore MSR 601 602 ; Handle 32-bit processor. 603 604 svrt4: 605 xor r3,r8,r6 ; get r3 <- 32-bit physical address of this savearea 606 bl saveRet32 ; return it 607 mtlr r9 ; restore return address 608 b saveRestore32 ; restore MSR 609 610 611 /* 612 * ***************************** 613 * * s a v e _ r e t _ p h y s * 614 * ***************************** 615 * 616 * void save_ret_phys(reg64_t); 617 * 618 * Called from lowmem vectors to return (ie, free) a savearea by physical address. 619 * Translation and interrupts are already off, and 64-bit mode is set if defined. 620 * We can take _no_ exceptions of any kind in this code, including PTE miss, since 621 * that would result in a deadlock. We expect: 622 * r3 = phys addr of savearea 623 * msr = IR, DR, and EE off, SF on 624 * cr6 = pf64Bit flag 625 * We destroy: 626 * r0,r2-r10. 627 */ 628 .align 5 629 .globl EXT(save_ret_phys) 630 631 LEXT(save_ret_phys) 632 mfsprg r10,0 ; get the per-proc ptr 633 bf-- pf64Bitb,saveRet32 ; handle 32-bit machine 634 b saveRet64 ; handle 64-bit machine 635 636 637 /* 638 * ********************* 639 * * s a v e R e t 6 4 * 640 * ********************* 641 * 642 * This is the internal routine to free a savearea, passed by 64-bit physical 643 * address. We assume that IR, DR, and EE are all off, that SF is on, and: 644 * r3 = phys address of the savearea 645 * r10 = per-proc ptr 646 * We destroy: 647 * r0,r2-r8. 648 */ 649 .align 5 650 saveRet64: 651 li r0,SAVempty ; Get marker for free savearea 652 lwz r7,lclfreecnt(r10) ; Get the local count 653 ld r6,lclfree(r10) ; Get the old local header 654 addi r7,r7,1 ; Pop up the free count 655 std r6,SAVprev(r3) ; Plant free chain pointer 656 cmplwi r7,LocalSaveMax ; Has the list gotten too long? 657 stb r0,SAVflags+2(r3) ; Mark savearea free 658 std r3,lclfree(r10) ; Chain us on in 659 stw r7,lclfreecnt(r10) ; Bump up the count 660 bltlr++ ; List not too long, so done 661 662 /* The local savearea chain has gotten too long. Trim it down to the target. 663 * Here's a tricky bit, and important: 664 * 665 * When we trim the list, we NEVER trim the very first one. This is because that is 666 * the very last one released and the exception exit code will release the savearea 667 * BEFORE it is done using it. Wouldn't be too good if another processor started 668 * using it, eh? So for this case, we are safe so long as the savearea stays on 669 * the local list. (Note: the exit routine needs to do this because it is in the 670 * process of restoring all context and it needs to keep it until the last second.) 671 */ 672 673 mflr r0 ; save return to caller of saveRet64 674 mr r2,r3 ; r2 <- 1st one on local list, which must not be trimmed 675 ld r3,SAVprev(r3) ; Skip over the first 676 subi r7,r7,LocalSaveTarget ; Figure out how much to trim 677 mr r6,r3 ; r6 <- first one to trim 678 mr r5,r7 ; Save the number we are trimming 679 680 saveRet64a: 681 addic. r7,r7,-1 ; Any left to do? 682 ble-- saveRet64b ; Nope... 683 ld r3,SAVprev(r3) ; Skip to the next one 684 b saveRet64a ; Keep going... 685 686 saveRet64b: ; r3 <- last one to trim 687 ld r7,SAVprev(r3) ; Point to the first one not to trim 688 li r4,LocalSaveTarget ; Set the target count 689 std r7,SAVprev(r2) ; Trim stuff leaving the one just released as first 690 stw r4,lclfreecnt(r10) ; Set the current count 691 692 bl savelock ; Lock up the anchor 693 694 ld r8,SVfree(0) ; Get the old head of the free list 695 lwz r4,SVfreecnt(0) ; Get the number of free ones 696 lwz r7,SVinuse(0) ; Get the number that are in use 697 std r6,SVfree(0) ; Point to the first trimmed savearea 698 add r4,r4,r5 ; Add number trimmed to free count 699 std r8,SAVprev(r3) ; Chain the old head to the tail of the trimmed guys 700 sub r7,r7,r5 ; Remove the trims from the in use count 701 stw r4,SVfreecnt(0) ; Set new free count 702 stw r7,SVinuse(0) ; Set new in use count 703 704 mtlr r0 ; Restore the return to our caller 705 b saveunlock ; Set adjust count, unlock the saveanchor, and return 706 707 708 /* 709 * ********************* 710 * * s a v e R e t 3 2 * 711 * ********************* 712 * 713 * This is the internal routine to free a savearea, passed by 32-bit physical 714 * address. We assume that IR, DR, and EE are all off, and: 715 * r3 = phys address of the savearea 716 * r10 = per-proc ptr 717 * We destroy: 718 * r0,r2-r8. 719 */ 720 .align 5 721 saveRet32: 722 li r0,SAVempty ; Get marker for free savearea 723 lwz r7,lclfreecnt(r10) ; Get the local count 724 lwz r6,lclfree+4(r10) ; Get the old local header 725 addi r7,r7,1 ; Pop up the free count 726 stw r6,SAVprev+4(r3) ; Plant free chain pointer 727 cmplwi r7,LocalSaveMax ; Has the list gotten too long? 728 stb r0,SAVflags+2(r3) ; Mark savearea free 729 stw r3,lclfree+4(r10) ; Chain us on in 730 stw r7,lclfreecnt(r10) ; Bump up the count 731 bltlr+ ; List not too long, so done 732 733 /* The local savearea chain has gotten too long. Trim it down to the target. 734 * Here's a tricky bit, and important: 735 * 736 * When we trim the list, we NEVER trim the very first one. This is because that is 737 * the very last one released and the exception exit code will release the savearea 738 * BEFORE it is done using it. Wouldn't be too good if another processor started 739 * using it, eh? So for this case, we are safe so long as the savearea stays on 740 * the local list. (Note: the exit routine needs to do this because it is in the 741 * process of restoring all context and it needs to keep it until the last second.) 742 */ 743 744 mflr r0 ; save return to caller of saveRet32 745 mr r2,r3 ; r2 <- 1st one on local list, which must not be trimmed 746 lwz r3,SAVprev+4(r3) ; Skip over the first 747 subi r7,r7,LocalSaveTarget ; Figure out how much to trim 748 mr r6,r3 ; r6 <- first one to trim 749 mr r5,r7 ; Save the number we are trimming 750 751 saveRet32a: 752 addic. r7,r7,-1 ; Any left to do? 753 ble- saveRet32b ; Nope... 754 lwz r3,SAVprev+4(r3) ; Skip to the next one 755 b saveRet32a ; Keep going... 756 757 saveRet32b: ; r3 <- last one to trim 758 lwz r7,SAVprev+4(r3) ; Point to the first one not to trim 759 li r4,LocalSaveTarget ; Set the target count 760 stw r7,SAVprev+4(r2) ; Trim stuff leaving the one just released as first 761 stw r4,lclfreecnt(r10) ; Set the current count 762 763 bl savelock ; Lock up the anchor 764 765 lwz r8,SVfree+4(0) ; Get the old head of the free list 766 lwz r4,SVfreecnt(0) ; Get the number of free ones 767 lwz r7,SVinuse(0) ; Get the number that are in use 768 stw r6,SVfree+4(0) ; Point to the first trimmed savearea 769 add r4,r4,r5 ; Add number trimmed to free count 770 stw r8,SAVprev+4(r3) ; Chain the old head to the tail of the trimmed guys 771 sub r7,r7,r5 ; Remove the trims from the in use count 772 stw r4,SVfreecnt(0) ; Set new free count 773 stw r7,SVinuse(0) ; Set new in use count 774 775 mtlr r0 ; Restore the return to our caller 776 b saveunlock ; Set adjust count, unlock the saveanchor, and return 777 778 779 /* 780 * ******************************* 781 * * s a v e _ t r i m _ f r e e * 782 * ******************************* 783 * 784 * struct savearea_comm *save_trim_free(void); 785 * 786 * Trim the free list down to the target count, ie by -(SVadjust) save areas. 787 * It trims the list and, if a pool page was fully allocated, puts that page on 788 * the start of the pool list. 789 * 790 * If the savearea being released is the last on a pool page (i.e., all entries 791 * are released), the page is dequeued from the pool and queued to any other 792 * found during this scan. Note that this queue is maintained virtually. 793 * 794 * When the scan is done, the saveanchor lock is released and the list of 795 * freed pool pages is returned to our caller. 796 * 797 * For latency sake we may want to revisit this code. If we are trimming a 798 * large number of saveareas, we could be disabled and holding the savearea lock 799 * for quite a while. It may be that we want to break the trim down into parts. 800 * Possibly trimming the free list, then individually pushing them into the free pool. 801 * 802 * This function expects to be called with translation on and a valid stack. 803 * It uses the standard ABI, ie we destroy r2 and r3-r11, and return the ptr in r3. 804 */ 805 .align 5 806 .globl EXT(save_trim_free) 807 808 LEXT(save_trim_free) 809 810 subi r1,r1,(FM_ALIGN(16)+FM_SIZE) ; Make space for 4 registers on stack 811 mflr r9 ; save our return address 812 stw r28,FM_SIZE+0(r1) ; Save R28 813 stw r29,FM_SIZE+4(r1) ; Save R29 814 stw r30,FM_SIZE+8(r1) ; Save R30 815 stw r31,FM_SIZE+12(r1) ; Save R31 816 817 bl saveSetup ; turn off translation and interrupts, load many regs 818 bl savelock ; Go lock up the anchor 819 820 lwz r8,SVadjust(0) ; How many do we need to clear out? 821 li r3,0 ; Get a 0 822 neg. r8,r8 ; Get the actual we need to toss (adjust is neg if too many) 823 ble- save_trim_free1 ; skip if no trimming needed anymore 824 bf-- pf64Bitb,saveTrim32 ; handle 32-bit processors 825 b saveTrim64 ; handle 64-bit processors 826 827 save_trim_free1: ; by the time we were called, no need to trim anymore 828 stw r3,SVlock(0) ; Quick unlock (no need for sync or to set adjust, nothing changed) 829 mtlr r9 ; Restore return 830 831 #if FPVECDBG 832 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG) 833 li r2,0x2206 ; (TEST/DEBUG) 834 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG) 835 sc ; (TEST/DEBUG) 836 #endif 837 addi r1,r1,(FM_ALIGN(16)+FM_SIZE); Pop stack - have not trashed register so no need to reload 838 b saveRestore ; restore translation and EE, turn SF off, return to our caller 839 840 841 /* 842 * *********************** 843 * * s a v e T r i m 3 2 * 844 * *********************** 845 * 846 * Handle "save_trim_free" on 32-bit processors. At this point, translation and interrupts 847 * are off, the savearea anchor is locked, and: 848 * r8 = #pages to trim (>0) 849 * r9 = return address 850 * r10 = per-proc ptr 851 * r11 = MSR at entry 852 */ 853 854 saveTrim32: 855 lwz r7,SVfree+4(0) ; Get the first on the free list 856 mr r6,r7 ; Save the first one 857 mr r5,r8 ; Save the number we are trimming 858 859 sttrimming: addic. r5,r5,-1 ; Any left to do? 860 ble- sttrimmed ; Nope... 861 lwz r7,SAVprev+4(r7) ; Skip to the next one 862 b sttrimming ; Keep going... 863 864 sttrimmed: lwz r5,SAVprev+4(r7) ; Get the next one (for new head of free list) 865 lwz r4,SVfreecnt(0) ; Get the free count 866 stw r5,SVfree+4(0) ; Set new head 867 sub r4,r4,r8 ; Calculate the new free count 868 li r31,0 ; Show we have no free pool blocks yet 869 crclr cr1_eq ; dont exit loop before 1st iteration 870 stw r4,SVfreecnt(0) ; Set new free count 871 lis r30,hi16(sac_empty) ; Get what empty looks like 872 873 ; NOTE: The savearea size must be 640 (0x280). We are doing a divide by shifts and stuff 874 ; here. 875 ; 876 #if SAVsize != 640 877 #error Savearea size is not 640!!!!!!!!!!!! 878 #endif 879 880 ; Loop over each savearea we are trimming. 881 ; r6 = next savearea to trim 882 ; r7 = last savearea to trim 883 ; r8 = #pages to trim (>0) 884 ; r9 = return address 885 ; r10 = per-proc ptr 886 ; r11 = MSR at entry 887 ; r30 = what SACalloc looks like when all saveareas are free 888 ; r31 = free pool block list 889 ; cr1 = beq set if we just trimmed the last, ie if we are done 890 891 sttoss: beq+ cr1,stdone ; All done now... 892 893 cmplw cr1,r6,r7 ; Have we finished the loop? 894 895 lis r0,0x0044 ; Get top of table 896 rlwinm r2,r6,0,0,19 ; Back down to the savearea control stuff 897 ori r0,r0,0x2200 ; Finish shift table 898 rlwinm r4,r6,25,27,30 ; Get (addr >> 7) & 0x1E (same as twice high nybble) 899 lwz r5,SACalloc(r2) ; Get the allocation bits 900 addi r4,r4,1 ; Shift 1 extra 901 rlwinm r3,r6,25,31,31 ; Get (addr >> 7) & 1 902 rlwnm r0,r0,r4,29,31 ; Get partial index 903 lis r4,lo16(0x8000) ; Get the bit mask 904 add r0,r0,r3 ; Make the real index 905 srw r4,r4,r0 ; Get the allocation mask 906 or r5,r5,r4 ; Free this entry 907 cmplw r5,r4 ; Is this the only free entry? 908 lwz r6,SAVprev+4(r6) ; Chain to the next trimmed savearea 909 cmplw cr7,r30,r5 ; Does this look empty? 910 stw r5,SACalloc(r2) ; Save back the allocation bits 911 beq- stputpool ; First free entry, go put it into the pool... 912 bne+ cr7,sttoss ; Not an empty block 913 914 ; 915 ; We have an empty block. Remove it from the pool list. 916 ; 917 918 lwz r29,SACflags(r2) ; Get the flags 919 cmplwi cr5,r31,0 ; Is this guy on the release list? 920 lwz r28,SACnext+4(r2) ; Get the forward chain 921 922 rlwinm. r0,r29,0,sac_permb,sac_permb ; Is this a permanently allocated area? (also sets 0 needed below) 923 bne- sttoss ; This is permanent entry, do not try to release... 924 925 lwz r29,SACprev+4(r2) ; and the previous 926 beq- cr5,stnot1st ; Not first 927 lwz r0,SACvrswap+4(r31) ; Load the previous pool page vr conversion 928 929 stnot1st: stw r28,SACnext+4(r29) ; Previous guy points to my next 930 xor r0,r0,r31 ; Make the last guy virtual 931 stw r29,SACprev+4(r28) ; Next guy points back to my previous 932 stw r0,SAVprev+4(r2) ; Store the old top virtual as my back chain 933 mr r31,r2 ; My physical is now the head of the chain 934 b sttoss ; Get the next one... 935 936 ; 937 ; A pool block that had no free entries now has one. Stick it on the pool list. 938 ; 939 940 stputpool: lwz r28,SVpoolfwd+4(0) ; Get the first guy on the list 941 li r0,saveanchor ; Point to the saveanchor 942 stw r2,SVpoolfwd+4(0) ; Put us on the top of the list 943 stw r28,SACnext+4(r2) ; We point to the old top 944 stw r2,SACprev+4(r28) ; Old top guy points back to us 945 stw r0,SACprev+4(r2) ; Our back points to the anchor 946 b sttoss ; Go on to the next one... 947 948 949 /* 950 * *********************** 951 * * s a v e T r i m 6 4 * 952 * *********************** 953 * 954 * Handle "save_trim_free" on 64-bit processors. At this point, translation and interrupts 955 * are off, SF is on, the savearea anchor is locked, and: 956 * r8 = #pages to trim (>0) 957 * r9 = return address 958 * r10 = per-proc ptr 959 * r11 = MSR at entry 960 */ 961 962 saveTrim64: 963 ld r7,SVfree(0) ; Get the first on the free list 964 mr r6,r7 ; Save the first one 965 mr r5,r8 ; Save the number we are trimming 966 967 sttrimming64: 968 addic. r5,r5,-1 ; Any left to do? 969 ble-- sttrimmed64 ; Nope... 970 ld r7,SAVprev(r7) ; Skip to the next one 971 b sttrimming64 ; Keep going... 972 973 sttrimmed64: 974 ld r5,SAVprev(r7) ; Get the next one (for new head of free list) 975 lwz r4,SVfreecnt(0) ; Get the free count 976 std r5,SVfree(0) ; Set new head 977 sub r4,r4,r8 ; Calculate the new free count 978 li r31,0 ; Show we have no free pool blocks yet 979 crclr cr1_eq ; dont exit loop before 1st iteration 980 stw r4,SVfreecnt(0) ; Set new free count 981 lis r30,hi16(sac_empty) ; Get what empty looks like 982 983 984 ; Loop over each savearea we are trimming. 985 ; r6 = next savearea to trim 986 ; r7 = last savearea to trim 987 ; r8 = #pages to trim (>0) 988 ; r9 = return address 989 ; r10 = per-proc ptr 990 ; r11 = MSR at entry 991 ; r30 = what SACalloc looks like when all saveareas are free 992 ; r31 = free pool block list 993 ; cr1 = beq set if we just trimmed the last, ie if we are done 994 ; 995 ; WARNING: as in the 32-bit path, this code is doing a divide by 640 (SAVsize). 996 997 sttoss64: 998 beq++ cr1,stdone ; All done now... 999 1000 cmpld cr1,r6,r7 ; Have we finished the loop? 1001 1002 lis r0,0x0044 ; Get top of table 1003 rldicr r2,r6,0,51 ; r2 <- phys addr of savearea block (with control area) 1004 ori r0,r0,0x2200 ; Finish shift table 1005 rlwinm r4,r6,25,27,30 ; Get (addr >> 7) & 0x1E (same as twice high nybble) 1006 lwz r5,SACalloc(r2) ; Get the allocation bits 1007 addi r4,r4,1 ; Shift 1 extra 1008 rlwinm r3,r6,25,31,31 ; Get (addr >> 7) & 1 1009 rlwnm r0,r0,r4,29,31 ; Get partial index 1010 lis r4,lo16(0x8000) ; Get the bit mask 1011 add r0,r0,r3 ; Make the real index 1012 srw r4,r4,r0 ; Get the allocation mask 1013 or r5,r5,r4 ; Free this entry 1014 cmplw r5,r4 ; Is this the only free entry? 1015 ld r6,SAVprev(r6) ; Chain to the next trimmed savearea 1016 cmplw cr7,r30,r5 ; Does this look empty? 1017 stw r5,SACalloc(r2) ; Save back the allocation bits 1018 beq-- stputpool64 ; First free entry, go put it into the pool... 1019 bne++ cr7,sttoss64 ; Not an empty block 1020 1021 ; We have an empty block. Remove it from the pool list. 1022 1023 lwz r29,SACflags(r2) ; Get the flags 1024 cmpldi cr5,r31,0 ; Is this guy on the release list? 1025 ld r28,SACnext(r2) ; Get the forward chain 1026 1027 rlwinm. r0,r29,0,sac_permb,sac_permb ; Is this a permanently allocated area? (also sets 0 needed below) 1028 bne-- sttoss64 ; This is permanent entry, do not try to release... 1029 1030 ld r29,SACprev(r2) ; and the previous 1031 beq-- cr5,stnot1st64 ; Not first 1032 ld r0,SACvrswap(r31) ; Load the previous pool page vr conversion 1033 1034 stnot1st64: 1035 std r28,SACnext(r29) ; Previous guy points to my next 1036 xor r0,r0,r31 ; Make the last guy virtual 1037 std r29,SACprev(r28) ; Next guy points back to my previous 1038 std r0,SAVprev(r2) ; Store the old top virtual as my back chain 1039 mr r31,r2 ; My physical is now the head of the chain 1040 b sttoss64 ; Get the next one... 1041 1042 ; A pool block that had no free entries now has one. Stick it on the pool list. 1043 1044 stputpool64: 1045 ld r28,SVpoolfwd(0) ; Get the first guy on the list 1046 li r0,saveanchor ; Point to the saveanchor 1047 std r2,SVpoolfwd(0) ; Put us on the top of the list 1048 std r28,SACnext(r2) ; We point to the old top 1049 std r2,SACprev(r28) ; Old top guy points back to us 1050 std r0,SACprev(r2) ; Our back points to the anchor 1051 b sttoss64 ; Go on to the next one... 1052 1053 1054 ; We are all done. Relocate pool release head, restore all, and go. This code 1055 ; is used both by the 32 and 64-bit paths. 1056 ; r9 = return address 1057 ; r10 = per-proc ptr 1058 ; r11 = MSR at entry 1059 ; r31 = free pool block list 1060 1061 stdone: bl saveunlock ; Unlock the saveanchor and set adjust field 1062 1063 mr. r3,r31 ; Move release chain and see if there are any 1064 li r5,0 ; Assume either V=R or no release chain 1065 beq- stnorel ; Nothing to release... 1066 lwz r5,SACvrswap+4(r31) ; Get the vr conversion (only need low half if 64-bit) 1067 1068 stnorel: 1069 bl saveRestore ; restore translation and exceptions, turn off SF 1070 mtlr r9 ; Restore the return 1071 1072 lwz r28,FM_SIZE+0(r1) ; Restore R28 1073 lwz r29,FM_SIZE+4(r1) ; Restore R29 1074 lwz r30,FM_SIZE+8(r1) ; Restore R30 1075 lwz r31,FM_SIZE+12(r1) ; Restore R31 1076 addi r1,r1,(FM_ALIGN(16)+FM_SIZE) ; Pop the stack 1077 xor r3,r3,r5 ; Convert release chain address to virtual 1078 rlwinm r3,r3,0,0,31 ; if 64-bit, clear upper half of virtual address 1079 1080 #if FPVECDBG 1081 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG) 1082 li r2,0x2207 ; (TEST/DEBUG) 1083 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG) 1084 sc ; (TEST/DEBUG) 1085 #endif 1086 blr ; Return... 1087 1088 1089 /* 1090 * *************************** 1091 * * s a v e _ r e c o v e r * 1092 * *************************** 1093 * 1094 * int save_recover(void); 1095 * 1096 * Returns nonzero if we can get enough saveareas to hit the target. We scan the free 1097 * pool. If we empty a pool block, we remove it from the pool list. 1098 */ 1099 1100 .align 5 1101 .globl EXT(save_recover) 1102 1103 LEXT(save_recover) 1104 mflr r9 ; save return address 1105 bl saveSetup ; turn translation and interrupts off, SF on, load many regs 1106 bl savelock ; lock the savearea anchor 1107 1108 lwz r8,SVadjust(0) ; How many do we need to clear get? 1109 li r3,0 ; Get a 0 1110 mr. r8,r8 ; Do we need any? 1111 ble-- save_recover1 ; not any more 1112 bf-- pf64Bitb,saveRecover32 ; handle 32-bit processor 1113 b saveRecover64 ; handle 64-bit processor 1114 1115 save_recover1: ; by the time we locked the anchor, no longer short 1116 mtlr r9 ; Restore return 1117 stw r3,SVlock(0) ; Quick unlock (no need for sync or to set adjust, nothing changed) 1118 #if FPVECDBG 1119 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG) 1120 li r2,0x2208 ; (TEST/DEBUG) 1121 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG) 1122 sc ; (TEST/DEBUG) 1123 #endif 1124 b saveRestore ; turn translation etc back on, return to our caller 1125 1126 1127 /* 1128 * ***************************** 1129 * * s a v e R e c o v e r 3 2 * 1130 * ***************************** 1131 * 1132 * Handle "save_recover" on 32-bit processors. At this point, translation and interrupts 1133 * are off, the savearea anchor is locked, and: 1134 * r8 = #pages to recover 1135 * r9 = return address 1136 * r10 = per-proc ptr 1137 * r11 = MSR at entry 1138 */ 1139 1140 saveRecover32: 1141 li r6,saveanchor ; Start at pool anchor 1142 crclr cr1_eq ; initialize the loop test 1143 lwz r7,SVfreecnt(0) ; Get the current free count 1144 1145 1146 ; Loop over next block in free pool. r6 is the ptr to the last block we looked at. 1147 1148 srcnpool: lwz r6,SACnext+4(r6) ; Point to the next one 1149 cmplwi r6,saveanchor ; Have we wrapped? 1150 beq- srcdone ; Yes, did not have enough... 1151 1152 lwz r5,SACalloc(r6) ; Pick up the allocation for this pool block 1153 1154 ; 1155 ; NOTE: The savearea size must be 640 (0x280). We are doing a multiply by shifts and add. 1156 ; offset = (index << 9) + (index << 7) 1157 ; 1158 #if SAVsize != 640 1159 #error Savearea size is not 640!!!!!!!!!!!! 1160 #endif 1161 1162 ; Loop over free savearea in current block. 1163 ; r5 = bitmap of free saveareas in block at r6 (ie, SACalloc) 1164 ; r6 = ptr to current free pool block 1165 ; r7 = free count 1166 ; r8 = #pages more we still need to recover 1167 ; r9 = return address 1168 ; r10 = per-proc ptr 1169 ; r11 = MSR at entry 1170 ; cr1 = beq if (r8==0) 1171 1172 srcnext: beq- cr1,srcdone ; We have no more to get... 1173 1174 lis r3,0x8000 ; Get the top bit on 1175 cntlzw r4,r5 ; Find a free slot 1176 addi r7,r7,1 ; Bump up the free count 1177 srw r3,r3,r4 ; Make a mask 1178 slwi r0,r4,7 ; First multiply by 128 1179 subi r8,r8,1 ; Decrement the need count 1180 slwi r2,r4,9 ; Then multiply by 512 1181 andc. r5,r5,r3 ; Clear out the "free" bit 1182 add r2,r2,r0 ; Sum to multiply by 640 1183 1184 stw r5,SACalloc(r6) ; Set new allocation bits 1185 1186 add r2,r2,r6 ; Get the actual address of the savearea 1187 lwz r3,SVfree+4(0) ; Get the head of the chain 1188 cmplwi cr1,r8,0 ; Do we actually need any more? 1189 stw r2,SVfree+4(0) ; Push ourselves in the front 1190 stw r3,SAVprev+4(r2) ; Chain the rest of the list behind 1191 1192 bne+ srcnext ; The pool block is not empty yet, try for another... 1193 1194 lwz r2,SACnext+4(r6) ; Get the next pointer 1195 lwz r3,SACprev+4(r6) ; Get the previous pointer 1196 stw r3,SACprev+4(r2) ; The previous of my next points to my previous 1197 stw r2,SACnext+4(r3) ; The next of my previous points to my next 1198 bne+ cr1,srcnpool ; We still have more to do... 1199 1200 1201 ; Join here from 64-bit path when we have recovered all the saveareas we need to. 1202 1203 srcdone: stw r7,SVfreecnt(0) ; Set the new free count 1204 bl saveunlock ; Unlock the save and set adjust field 1205 1206 mtlr r9 ; Restore the return 1207 #if FPVECDBG 1208 lis r0,HIGH_ADDR(CutTrace) ; (TEST/DEBUG) 1209 li r2,0x2209 ; (TEST/DEBUG) 1210 oris r0,r0,LOW_ADDR(CutTrace) ; (TEST/DEBUG) 1211 sc ; (TEST/DEBUG) 1212 #endif 1213 b saveRestore ; turn xlate and EE back on, SF off, and return to our caller 1214 1215 1216 /* 1217 * ***************************** 1218 * * s a v e R e c o v e r 6 4 * 1219 * ***************************** 1220 * 1221 * Handle "save_recover" on 64-bit processors. At this point, translation and interrupts 1222 * are off, the savearea anchor is locked, and: 1223 * r8 = #pages to recover 1224 * r9 = return address 1225 * r10 = per-proc ptr 1226 * r11 = MSR at entry 1227 */ 1228 1229 saveRecover64: 1230 li r6,saveanchor ; Start at pool anchor 1231 crclr cr1_eq ; initialize the loop test 1232 lwz r7,SVfreecnt(0) ; Get the current free count 1233 1234 1235 ; Loop over next block in free pool. r6 is the ptr to the last block we looked at. 1236 1237 srcnpool64: 1238 ld r6,SACnext(r6) ; Point to the next one 1239 cmpldi r6,saveanchor ; Have we wrapped? 1240 beq-- srcdone ; Yes, did not have enough... 1241 1242 lwz r5,SACalloc(r6) ; Pick up the allocation for this pool block 1243 1244 1245 ; Loop over free savearea in current block. 1246 ; r5 = bitmap of free saveareas in block at r6 (ie, SACalloc) 1247 ; r6 = ptr to current free pool block 1248 ; r7 = free count 1249 ; r8 = #pages more we still need to recover 1250 ; r9 = return address 1251 ; r10 = per-proc ptr 1252 ; r11 = MSR at entry 1253 ; cr1 = beq if (r8==0) 1254 ; 1255 ; WARNING: as in the 32-bit path, we depend on (SAVsize==640) 1256 1257 srcnext64: 1258 beq-- cr1,srcdone ; We have no more to get... 1259 1260 lis r3,0x8000 ; Get the top bit on 1261 cntlzw r4,r5 ; Find a free slot 1262 addi r7,r7,1 ; Bump up the free count 1263 srw r3,r3,r4 ; Make a mask 1264 slwi r0,r4,7 ; First multiply by 128 1265 subi r8,r8,1 ; Decrement the need count 1266 slwi r2,r4,9 ; Then multiply by 512 1267 andc. r5,r5,r3 ; Clear out the "free" bit 1268 add r2,r2,r0 ; Sum to multiply by 640 1269 1270 stw r5,SACalloc(r6) ; Set new allocation bits 1271 1272 add r2,r2,r6 ; Get the actual address of the savearea 1273 ld r3,SVfree(0) ; Get the head of the chain 1274 cmplwi cr1,r8,0 ; Do we actually need any more? 1275 std r2,SVfree(0) ; Push ourselves in the front 1276 std r3,SAVprev(r2) ; Chain the rest of the list behind 1277 1278 bne++ srcnext64 ; The pool block is not empty yet, try for another... 1279 1280 ld r2,SACnext(r6) ; Get the next pointer 1281 ld r3,SACprev(r6) ; Get the previous pointer 1282 std r3,SACprev(r2) ; The previous of my next points to my previous 1283 std r2,SACnext(r3) ; The next of my previous points to my next 1284 bne++ cr1,srcnpool64 ; We still have more to do... 1285 1286 b srcdone 1287 1288 1289 /* 1290 * ******************* 1291 * * s a v e l o c k * 1292 * ******************* 1293 * 1294 * Lock the savearea anchor, so we can manipulate the free list. 1295 * msr = interrupts and translation off 1296 * We destroy: 1297 * r8, r3, r12 1298 */ 1299 .align 5 1300 1301 savelock: lwz r8,SVlock(0) ; See if lock is held 1302 cmpwi r8,0 1303 li r12,saveanchor ; Point to the saveanchor 1304 bne-- savelock ; loop until lock released... 1305 1306 savelock0: lwarx r8,0,r12 ; Grab the lock value 1307 cmpwi r8,0 ; taken? 1308 li r8,1 ; get nonzero to lock it with 1309 bne-- savelock1 ; already locked, wait for it to clear... 1310 stwcx. r8,0,r12 ; Try to seize that there durn lock 1311 isync ; assume we got it 1312 beqlr++ ; reservation not lost, so we have the lock 1313 b savelock0 ; Try again... 1314 1315 savelock1: li r8,lgKillResv ; Point to killing field 1316 stwcx. r8,0,r8 ; Kill reservation 1317 b savelock ; Start over.... 1318 1319 1320 /* 1321 * *********************** 1322 * * s a v e u n l o c k * 1323 * *********************** 1324 * 1325 * 1326 * This is the common routine that sets the saveadjust field and unlocks the savearea 1327 * anchor. 1328 * msr = interrupts and translation off 1329 * We destroy: 1330 * r2, r5, r6, r8. 1331 */ 1332 .align 5 1333 saveunlock: 1334 lwz r6,SVfreecnt(0) ; and the number on the free list 1335 lwz r5,SVinuse(0) ; Pick up the in use count 1336 subic. r8,r6,FreeListMin ; do we have at least the minimum? 1337 lwz r2,SVtarget(0) ; Get the target 1338 neg r8,r8 ; assuming we are short, get r8 <- shortfall 1339 blt-- saveunlock1 ; skip if fewer than minimum on free list 1340 1341 add r6,r6,r5 ; Get the total number of saveareas 1342 addi r5,r2,-SaveLowHysteresis ; Find low end of acceptible range 1343 sub r5,r6,r5 ; Make everything below hysteresis negative 1344 sub r2,r2,r6 ; Get the distance from the target 1345 addi r5,r5,-(SaveLowHysteresis + SaveHighHysteresis + 1) ; Subtract full hysteresis range 1346 srawi r5,r5,31 ; Get 0xFFFFFFFF if outside range or 0 if inside 1347 and r8,r2,r5 ; r8 <- 0 if in range or distance to target if not 1348 1349 saveunlock1: 1350 li r5,0 ; Set a clear value 1351 stw r8,SVadjust(0) ; Set the adjustment value 1352 eieio ; Make sure everything is done 1353 stw r5,SVlock(0) ; Unlock the savearea chain 1354 blr 1355 1356 1357 /* 1358 * ******************* 1359 * * s a v e _ c p v * 1360 * ******************* 1361 * 1362 * struct savearea *save_cpv(addr64_t saveAreaPhysAddr); 1363 * 1364 * Converts a physical savearea address to virtual. Called with translation on 1365 * and in 32-bit mode. Note that the argument is passed as a long long in (r3,r4). 1366 */ 1367 1368 .align 5 1369 .globl EXT(save_cpv) 1370 1371 LEXT(save_cpv) 1372 mflr r9 ; save return address 1373 mr r8,r3 ; save upper half of phys address here 1374 bl saveSetup ; turn off translation and interrupts, turn SF on 1375 rlwinm r5,r4,0,0,19 ; Round back to the start of the physical savearea block 1376 bf-- pf64Bitb,save_cpv1 ; skip if 32-bit processor 1377 rldimi r5,r8,32,0 ; r5 <- 64-bit phys address of block 1378 save_cpv1: 1379 lwz r6,SACvrswap+4(r5) ; Get the conversion to virtual (only need low half if 64-bit) 1380 mtlr r9 ; restore return address 1381 xor r3,r4,r6 ; convert phys to virtual 1382 rlwinm r3,r3,0,0,31 ; if 64-bit, zero upper half of virtual address 1383 b saveRestore ; turn translation etc back on, SF off, and return r3 1384 1385 1386 /* 1387 * ********************* 1388 * * s a v e S e t u p * 1389 * ********************* 1390 * 1391 * This routine is called at the start of all the save-area subroutines. 1392 * It turns off translation, disabled interrupts, turns on 64-bit mode, 1393 * and sets up cr6 with the feature flags (especially pf64Bit). 1394 * 1395 * Note that most save-area routines cannot take _any_ interrupt (such as a 1396 * PTE miss) once the savearea anchor is locked, since that would result in 1397 * instant deadlock as we need a save-area to process any exception. 1398 * We set up: 1399 * r10 = per-proc ptr 1400 * r11 = old MSR 1401 * cr5 = pfNoMSRir feature flag 1402 * cr6 = pf64Bit feature flag 1403 * 1404 * We use r0, r3, r10, and r11. 1405 */ 1406 1407 saveSetup: 1408 mfmsr r11 ; get msr 1409 mfsprg r3,2 ; get feature flags 1410 li r0,0 1411 mtcrf 0x2,r3 ; copy pf64Bit to cr6 1412 ori r0,r0,lo16(MASK(MSR_IR)+MASK(MSR_DR)+MASK(MSR_EE)) 1413 mtcrf 0x4,r3 ; copy pfNoMSRir to cr5 1414 andc r3,r11,r0 ; turn off IR, DR, and EE 1415 li r0,1 ; get a 1 in case its a 64-bit machine 1416 bf-- pf64Bitb,saveSetup1 ; skip if not a 64-bit machine 1417 rldimi r3,r0,63,MSR_SF_BIT ; turn SF (bit 0) on 1418 mtmsrd r3 ; turn translation and interrupts off, 64-bit mode on 1419 isync ; wait for it to happen 1420 mfsprg r10,0 ; get per-proc ptr 1421 blr 1422 saveSetup1: ; here on 32-bit machines 1423 bt- pfNoMSRirb,saveSetup2 ; skip if cannot turn off IR with a mtmsr 1424 mtmsr r3 ; turn translation and interrupts off 1425 isync ; wait for it to happen 1426 mfsprg r10,0 ; get per-proc ptr 1427 blr 1428 saveSetup2: ; here if pfNoMSRir set for this machine 1429 li r0,loadMSR ; we will "mtmsr r3" via system call 1430 sc 1431 mfsprg r10,0 ; get per-proc ptr 1432 blr 1433 1434 1435 /* 1436 * ************************* 1437 * * s a v e R e s t o r e * 1438 * ************************* 1439 * 1440 * Undoes the effect of calling "saveSetup", ie it turns relocation and interrupts back on, 1441 * and turns 64-bit mode back off. 1442 * r11 = old MSR 1443 * cr6 = pf64Bit feature flag 1444 */ 1445 1446 saveRestore: 1447 bt++ pf64Bitb,saveRestore64 ; handle a 64-bit processor 1448 saveRestore32: 1449 mtmsr r11 ; restore MSR 1450 isync ; wait for translation to start up 1451 blr 1452 saveRestore64: ; 64-bit processor 1453 mtmsrd r11 ; restore MSR 1454 isync ; wait for changes to happen 1455 blr 1456
Cache object: 16e52750c781a2bc10377e844ca6e62e
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]