Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/osfmk/ppc/movc.s
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* 2 * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_LICENSE_HEADER_START@ 5 * 6 * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved. 7 * 8 * This file contains Original Code and/or Modifications of Original Code 9 * as defined in and that are subject to the Apple Public Source License 10 * Version 2.0 (the 'License'). You may not use this file except in 11 * compliance with the License. Please obtain a copy of the License at 12 * http://www.opensource.apple.com/apsl/ and read it before using this 13 * file. 14 * 15 * The Original Code and all software distributed under the License are 16 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 17 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 18 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 20 * Please see the License for the specific language governing rights and 21 * limitations under the License. 22 * 23 * @APPLE_LICENSE_HEADER_END@ 24 */ 25 /* 26 * @OSF_COPYRIGHT@ 27 */ 28 #include <debug.h> 29 #include <ppc/asm.h> 30 #include <ppc/proc_reg.h> 31 #include <mach/ppc/vm_param.h> 32 #include <assym.s> 33 #include <sys/errno.h> 34 35 #define INSTRUMENT 0 36 37 //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> 38 /* 39 * void pmap_zero_page(vm_offset_t pa) 40 * 41 * Zero a page of physical memory. This routine runs in 32 or 64-bit mode, 42 * and handles 32 and 128-byte cache lines. 43 */ 44 45 46 .align 5 47 .globl EXT(pmap_zero_page) 48 49 LEXT(pmap_zero_page) 50 51 mflr r12 // save return address 52 bl EXT(ml_set_physical_disabled) // turn DR and EE off, SF on, get features in r10 53 mtlr r12 // restore return address 54 andi. r9,r10,pf32Byte+pf128Byte // r9 <- cache line size 55 56 subfic r4,r9,PPC_PGBYTES // r4 <- starting offset in page 57 58 bt++ pf64Bitb,page0S4 // Go do the big guys... 59 60 slwi r3,r3,12 // get page address from page num 61 b page_zero_1 // Jump to line aligned loop... 62 63 .align 5 64 65 nop 66 nop 67 nop 68 nop 69 nop 70 nop 71 nop 72 73 page0S4: 74 sldi r3,r3,12 // get page address from page num 75 76 page_zero_1: // loop zeroing cache lines 77 sub. r5,r4,r9 // more to go? 78 dcbz128 r3,r4 // zero either 32 or 128 bytes 79 sub r4,r5,r9 // generate next offset 80 dcbz128 r3,r5 81 bne-- page_zero_1 82 83 b EXT(ml_restore) // restore MSR and do the isync 84 85 86 //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> 87 /* void 88 * phys_copy(src, dst, bytecount) 89 * addr64_t src; 90 * addr64_t dst; 91 * int bytecount 92 * 93 * This routine will copy bytecount bytes from physical address src to physical 94 * address dst. It runs in 64-bit mode if necessary, but does not handle 95 * overlap or make any attempt to be optimal. Length must be a signed word. 96 * Not performance critical. 97 */ 98 99 100 .align 5 101 .globl EXT(phys_copy) 102 103 LEXT(phys_copy) 104 105 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg 106 mflr r12 // get return address 107 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits 108 rlwinm r4,r5,0,1,0 ; Duplicate high half of long long paddr into top of reg 109 bl EXT(ml_set_physical_disabled) // turn DR and EE off, SF on, get features in r10 110 rlwimi r4,r6,0,0,31 ; Combine bottom of long long to full 64-bits 111 mtlr r12 // restore return address 112 subic. r5,r7,4 // a word to copy? 113 b phys_copy_2 114 115 .align 5 116 117 phys_copy_1: // loop copying words 118 subic. r5,r5,4 // more to go? 119 lwz r0,0(r3) 120 addi r3,r3,4 121 stw r0,0(r4) 122 addi r4,r4,4 123 phys_copy_2: 124 bge phys_copy_1 125 addic. r5,r5,4 // restore count 126 ble phys_copy_4 // no more 127 128 // Loop is aligned here 129 130 phys_copy_3: // loop copying bytes 131 subic. r5,r5,1 // more to go? 132 lbz r0,0(r3) 133 addi r3,r3,1 134 stb r0,0(r4) 135 addi r4,r4,1 136 bgt phys_copy_3 137 phys_copy_4: 138 b EXT(ml_restore) // restore MSR and do the isync 139 140 141 //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> 142 /* void 143 * pmap_copy_page(src, dst) 144 * ppnum_t src; 145 * ppnum_t dst; 146 * 147 * This routine will copy the physical page src to physical page dst 148 * 149 * This routine assumes that the src and dst are page numbers and that the 150 * destination is cached. It runs on 32 and 64 bit processors, with and 151 * without altivec, and with 32 and 128 byte cache lines. 152 * We also must assume that no-one will be executing within the destination 153 * page, and that this will be used for paging. Because this 154 * is a common routine, we have tuned loops for each processor class. 155 * 156 */ 157 #define kSFSize (FM_SIZE+160) 158 159 ENTRY(pmap_copy_page, TAG_NO_FRAME_USED) 160 161 lis r2,hi16(MASK(MSR_VEC)) ; Get the vector flag 162 mflr r0 // get return 163 ori r2,r2,lo16(MASK(MSR_FP)) ; Add the FP flag 164 stw r0,8(r1) // save 165 stwu r1,-kSFSize(r1) // set up a stack frame for VRs or FPRs 166 mfmsr r11 // save MSR at entry 167 mfsprg r10,2 // get feature flags 168 andc r11,r11,r2 // Clear out vec and fp 169 ori r2,r2,lo16(MASK(MSR_EE)) // Get EE on also 170 andc r2,r11,r2 // Clear out EE as well 171 mtcrf 0x02,r10 // we need to test pf64Bit 172 ori r2,r2,MASK(MSR_FP) // must enable FP for G3... 173 mtcrf 0x80,r10 // we need to test pfAltivec too 174 oris r2,r2,hi16(MASK(MSR_VEC)) // enable altivec for G4 (ignored if G3) 175 mtmsr r2 // turn EE off, FP and VEC on 176 isync 177 bt++ pf64Bitb,pmap_copy_64 // skip if 64-bit processor (only they take hint) 178 slwi r3,r3,12 // get page address from page num 179 slwi r4,r4,12 // get page address from page num 180 rlwinm r12,r2,0,MSR_DR_BIT+1,MSR_DR_BIT-1 // get ready to turn off DR 181 bt pfAltivecb,pmap_copy_g4 // altivec but not 64-bit means G4 182 183 184 // G3 -- copy using FPRs 185 186 stfd f0,FM_SIZE+0(r1) // save the 4 FPRs we use to copy 187 stfd f1,FM_SIZE+8(r1) 188 li r5,PPC_PGBYTES/32 // count of cache lines in a page 189 stfd f2,FM_SIZE+16(r1) 190 mtctr r5 191 stfd f3,FM_SIZE+24(r1) 192 mtmsr r12 // turn off DR after saving FPRs on stack 193 isync 194 195 pmap_g3_copy_loop: // loop over 32-byte cache lines 196 dcbz 0,r4 // avoid read of dest line 197 lfd f0,0(r3) 198 lfd f1,8(r3) 199 lfd f2,16(r3) 200 lfd f3,24(r3) 201 addi r3,r3,32 202 stfd f0,0(r4) 203 stfd f1,8(r4) 204 stfd f2,16(r4) 205 stfd f3,24(r4) 206 dcbst 0,r4 // flush dest line to RAM 207 addi r4,r4,32 208 bdnz pmap_g3_copy_loop 209 210 sync // wait for stores to take 211 subi r4,r4,PPC_PGBYTES // restore ptr to destintation page 212 li r6,PPC_PGBYTES-32 // point to last line in page 213 pmap_g3_icache_flush: 214 subic. r5,r6,32 // more to go? 215 icbi r4,r6 // flush another line in icache 216 subi r6,r5,32 // get offset to next line 217 icbi r4,r5 218 bne pmap_g3_icache_flush 219 220 sync 221 mtmsr r2 // turn DR back on 222 isync 223 lfd f0,FM_SIZE+0(r1) // restore the FPRs 224 lfd f1,FM_SIZE+8(r1) 225 lfd f2,FM_SIZE+16(r1) 226 lfd f3,FM_SIZE+24(r1) 227 228 b pmap_g4_restore // restore MSR and done 229 230 231 // G4 -- copy using VRs 232 233 pmap_copy_g4: // r2=(MSR-EE), r12=(r2-DR), r10=features, r11=old MSR 234 la r9,FM_SIZE+16(r1) // place where we save VRs to r9 235 li r5,16 // load x-form offsets into r5-r9 236 li r6,32 // another offset 237 stvx v0,0,r9 // save some VRs so we can use to copy 238 li r7,48 // another offset 239 stvx v1,r5,r9 240 li r0,PPC_PGBYTES/64 // we loop over 64-byte chunks 241 stvx v2,r6,r9 242 mtctr r0 243 li r8,96 // get look-ahead for touch 244 stvx v3,r7,r9 245 li r9,128 246 mtmsr r12 // now we've saved VRs on stack, turn off DR 247 isync // wait for it to happen 248 b pmap_g4_copy_loop 249 250 .align 5 // align inner loops 251 pmap_g4_copy_loop: // loop over 64-byte chunks 252 dcbt r3,r8 // touch 3 lines ahead 253 nop // avoid a 17-word loop... 254 dcbt r3,r9 // touch 4 lines ahead 255 nop // more padding 256 dcba 0,r4 // avoid pre-fetch of 1st dest line 257 lvx v0,0,r3 // offset 0 258 lvx v1,r5,r3 // offset 16 259 lvx v2,r6,r3 // offset 32 260 lvx v3,r7,r3 // offset 48 261 addi r3,r3,64 262 dcba r6,r4 // avoid pre-fetch of 2nd line 263 stvx v0,0,r4 // offset 0 264 stvx v1,r5,r4 // offset 16 265 stvx v2,r6,r4 // offset 32 266 stvx v3,r7,r4 // offset 48 267 dcbf 0,r4 // push line 1 268 dcbf r6,r4 // and line 2 269 addi r4,r4,64 270 bdnz pmap_g4_copy_loop 271 272 sync // wait for stores to take 273 subi r4,r4,PPC_PGBYTES // restore ptr to destintation page 274 li r8,PPC_PGBYTES-32 // point to last line in page 275 pmap_g4_icache_flush: 276 subic. r9,r8,32 // more to go? 277 icbi r4,r8 // flush from icache 278 subi r8,r9,32 // get offset to next line 279 icbi r4,r9 280 bne pmap_g4_icache_flush 281 282 sync 283 mtmsr r2 // turn DR back on 284 isync 285 la r9,FM_SIZE+16(r1) // get base of VR save area 286 lvx v0,0,r9 // restore the VRs 287 lvx v1,r5,r9 288 lvx v2,r6,r9 289 lvx v3,r7,r9 290 291 pmap_g4_restore: // r11=MSR 292 mtmsr r11 // turn EE on, VEC and FR off 293 isync // wait for it to happen 294 addi r1,r1,kSFSize // pop off our stack frame 295 lwz r0,8(r1) // restore return address 296 mtlr r0 297 blr 298 299 300 // 64-bit/128-byte processor: copy using VRs 301 302 pmap_copy_64: // r10=features, r11=old MSR 303 sldi r3,r3,12 // get page address from page num 304 sldi r4,r4,12 // get page address from page num 305 la r9,FM_SIZE+16(r1) // get base of VR save area 306 li r5,16 // load x-form offsets into r5-r9 307 li r6,32 // another offset 308 bf pfAltivecb,pmap_novmx_copy // altivec suppressed... 309 stvx v0,0,r9 // save 8 VRs so we can copy wo bubbles 310 stvx v1,r5,r9 311 li r7,48 // another offset 312 li r0,PPC_PGBYTES/128 // we loop over 128-byte chunks 313 stvx v2,r6,r9 314 stvx v3,r7,r9 315 addi r9,r9,64 // advance base ptr so we can store another 4 316 mtctr r0 317 li r0,MASK(MSR_DR) // get DR bit 318 stvx v4,0,r9 319 stvx v5,r5,r9 320 andc r12,r2,r0 // turn off DR bit 321 li r0,1 // get a 1 to slam into SF 322 stvx v6,r6,r9 323 stvx v7,r7,r9 324 rldimi r12,r0,63,MSR_SF_BIT // set SF bit (bit 0) 325 li r8,-128 // offset so we can reach back one line 326 mtmsrd r12 // now we've saved VRs, turn DR off and SF on 327 isync // wait for it to happen 328 dcbt128 0,r3,1 // start a forward stream 329 b pmap_64_copy_loop 330 331 .align 5 // align inner loops 332 pmap_64_copy_loop: // loop over 128-byte chunks 333 dcbz128 0,r4 // avoid read of destination line 334 lvx v0,0,r3 // offset 0 335 lvx v1,r5,r3 // offset 16 336 lvx v2,r6,r3 // offset 32 337 lvx v3,r7,r3 // offset 48 338 addi r3,r3,64 // don't have enough GPRs so add 64 2x 339 lvx v4,0,r3 // offset 64 340 lvx v5,r5,r3 // offset 80 341 lvx v6,r6,r3 // offset 96 342 lvx v7,r7,r3 // offset 112 343 addi r3,r3,64 344 stvx v0,0,r4 // offset 0 345 stvx v1,r5,r4 // offset 16 346 stvx v2,r6,r4 // offset 32 347 stvx v3,r7,r4 // offset 48 348 addi r4,r4,64 349 stvx v4,0,r4 // offset 64 350 stvx v5,r5,r4 // offset 80 351 stvx v6,r6,r4 // offset 96 352 stvx v7,r7,r4 // offset 112 353 addi r4,r4,64 354 dcbf r8,r4 // flush the line we just wrote 355 bdnz pmap_64_copy_loop 356 357 sync // wait for stores to take 358 subi r4,r4,PPC_PGBYTES // restore ptr to destintation page 359 li r8,PPC_PGBYTES-128 // point to last line in page 360 pmap_64_icache_flush: 361 subic. r9,r8,128 // more to go? 362 icbi r4,r8 // flush from icache 363 subi r8,r9,128 // get offset to next line 364 icbi r4,r9 365 bne pmap_64_icache_flush 366 367 sync 368 mtmsrd r2 // turn DR back on, SF off 369 isync 370 la r9,FM_SIZE+16(r1) // get base address of VR save area on stack 371 lvx v0,0,r9 // restore the VRs 372 lvx v1,r5,r9 373 lvx v2,r6,r9 374 lvx v3,r7,r9 375 addi r9,r9,64 376 lvx v4,0,r9 377 lvx v5,r5,r9 378 lvx v6,r6,r9 379 lvx v7,r7,r9 380 381 b pmap_g4_restore // restore lower half of MSR and return 382 383 // 384 // Copy on 64-bit without VMX 385 // 386 387 pmap_novmx_copy: 388 li r0,PPC_PGBYTES/128 // we loop over 128-byte chunks 389 mtctr r0 390 li r0,MASK(MSR_DR) // get DR bit 391 andc r12,r2,r0 // turn off DR bit 392 li r0,1 // get a 1 to slam into SF 393 rldimi r12,r0,63,MSR_SF_BIT // set SF bit (bit 0) 394 mtmsrd r12 // now we've saved VRs, turn DR off and SF on 395 isync // wait for it to happen 396 dcbt128 0,r3,1 // start a forward stream 397 398 pmap_novmx_copy_loop: // loop over 128-byte cache lines 399 dcbz128 0,r4 // avoid read of dest line 400 401 ld r0,0(r3) // Load half a line 402 ld r12,8(r3) 403 ld r5,16(r3) 404 ld r6,24(r3) 405 ld r7,32(r3) 406 ld r8,40(r3) 407 ld r9,48(r3) 408 ld r10,56(r3) 409 410 std r0,0(r4) // Store half a line 411 std r12,8(r4) 412 std r5,16(r4) 413 std r6,24(r4) 414 std r7,32(r4) 415 std r8,40(r4) 416 std r9,48(r4) 417 std r10,56(r4) 418 419 ld r0,64(r3) // Load half a line 420 ld r12,72(r3) 421 ld r5,80(r3) 422 ld r6,88(r3) 423 ld r7,96(r3) 424 ld r8,104(r3) 425 ld r9,112(r3) 426 ld r10,120(r3) 427 428 addi r3,r3,128 429 430 std r0,64(r4) // Store half a line 431 std r12,72(r4) 432 std r5,80(r4) 433 std r6,88(r4) 434 std r7,96(r4) 435 std r8,104(r4) 436 std r9,112(r4) 437 std r10,120(r4) 438 439 dcbf 0,r4 // flush the line we just wrote 440 addi r4,r4,128 441 bdnz pmap_novmx_copy_loop 442 443 sync // wait for stores to take 444 subi r4,r4,PPC_PGBYTES // restore ptr to destintation page 445 li r8,PPC_PGBYTES-128 // point to last line in page 446 447 pmap_novmx_icache_flush: 448 subic. r9,r8,128 // more to go? 449 icbi r4,r8 // flush from icache 450 subi r8,r9,128 // get offset to next line 451 icbi r4,r9 452 bne pmap_novmx_icache_flush 453 454 sync 455 mtmsrd r2 // turn DR back on, SF off 456 isync 457 458 b pmap_g4_restore // restore lower half of MSR and return 459 460 461 462 //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> 463 464 // Stack frame format used by copyin, copyout, copyinstr and copyoutstr. 465 // These routines all run both on 32 and 64-bit machines, though because they are called 466 // by the BSD kernel they are always in 32-bit mode when entered. The mapped ptr returned 467 // by MapUserAddressSpace will be 64 bits however on 64-bit machines. Beware to avoid 468 // using compare instructions on this ptr. This mapped ptr is kept globally in r31, so there 469 // is no need to store or load it, which are mode-dependent operations since it could be 470 // 32 or 64 bits. 471 472 #define kkFrameSize (FM_SIZE+32) 473 474 #define kkBufSize (FM_SIZE+0) 475 #define kkCR (FM_SIZE+4) 476 #define kkSource (FM_SIZE+8) 477 #define kkDest (FM_SIZE+12) 478 #define kkCountPtr (FM_SIZE+16) 479 #define kkR31Save (FM_SIZE+20) 480 481 482 // nonvolatile CR bits we use as flags in cr3 483 484 #define kk64bit 12 485 #define kkNull 13 486 #define kkIn 14 487 #define kkString 15 488 #define kkZero 15 489 490 491 //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> 492 /* 493 * int 494 * copyoutstr(src, dst, maxcount, count) 495 * vm_offset_t src; 496 * vm_offset_t dst; 497 * vm_size_t maxcount; 498 * vm_size_t* count; 499 * 500 * Set *count to the number of bytes copied. 501 */ 502 503 ENTRY(copyoutstr, TAG_NO_FRAME_USED) 504 mfcr r2 // we use nonvolatile cr3 505 li r0,0 506 crset kkString // flag as a string op 507 mr r10,r4 // for copyout, dest ptr (r4) is in user space 508 stw r0,0(r6) // initialize #bytes moved 509 crclr kkIn // flag as copyout 510 b copyJoin 511 512 513 //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> 514 /* 515 * int 516 * copyinstr(src, dst, maxcount, count) 517 * vm_offset_t src; 518 * vm_offset_t dst; 519 * vm_size_t maxcount; 520 * vm_size_t* count; 521 * 522 * Set *count to the number of bytes copied 523 * If dst == NULL, don't copy, just count bytes. 524 * Only currently called from klcopyinstr. 525 */ 526 527 ENTRY(copyinstr, TAG_NO_FRAME_USED) 528 mfcr r2 // we use nonvolatile cr3 529 cmplwi r4,0 // dst==NULL? 530 li r0,0 531 crset kkString // flag as a string op 532 mr r10,r3 // for copyin, source ptr (r3) is in user space 533 crmove kkNull,cr0_eq // remember if (dst==NULL) 534 stw r0,0(r6) // initialize #bytes moved 535 crset kkIn // flag as copyin (rather than copyout) 536 b copyJoin1 // skip over the "crclr kkNull" 537 538 539 //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> 540 /* 541 * int 542 * copyout(src, dst, count) 543 * vm_offset_t src; 544 * vm_offset_t dst; 545 * size_t count; 546 */ 547 548 .align 5 549 .globl EXT(copyout) 550 .globl EXT(copyoutmsg) 551 552 LEXT(copyout) 553 LEXT(copyoutmsg) 554 555 #if INSTRUMENT 556 mfspr r12,pmc1 ; INSTRUMENT - saveinstr[12] - Take stamp at copyout 557 stw r12,0x6100+(12*16)+0x0(0) ; INSTRUMENT - Save it 558 mfspr r12,pmc2 ; INSTRUMENT - Get stamp 559 stw r12,0x6100+(12*16)+0x4(0) ; INSTRUMENT - Save it 560 mfspr r12,pmc3 ; INSTRUMENT - Get stamp 561 stw r12,0x6100+(12*16)+0x8(0) ; INSTRUMENT - Save it 562 mfspr r12,pmc4 ; INSTRUMENT - Get stamp 563 stw r12,0x6100+(12*16)+0xC(0) ; INSTRUMENT - Save it 564 #endif 565 mfcr r2 // save caller's CR 566 crclr kkString // not a string version 567 mr r10,r4 // dest (r4) is user-space ptr 568 crclr kkIn // flag as copyout 569 b copyJoin 570 571 572 //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> 573 /* 574 * int 575 * copyin(src, dst, count) 576 * vm_offset_t src; 577 * vm_offset_t dst; 578 * size_t count; 579 */ 580 581 582 .align 5 583 .globl EXT(copyin) 584 .globl EXT(copyinmsg) 585 586 LEXT(copyin) 587 LEXT(copyinmsg) 588 589 mfcr r2 // save caller's CR 590 crclr kkString // not a string version 591 mr r10,r3 // source (r3) is user-space ptr in copyin 592 crset kkIn // flag as copyin 593 594 595 // Common code to handle setup for all the copy variants: 596 // r2 = caller's CR, since we use cr3 597 // r3-r6 = parameters 598 // r10 = user-space ptr (r3 if copyin, r4 if copyout) 599 // cr3 = kkIn, kkString, kkNull flags 600 601 copyJoin: 602 crclr kkNull // (dst==NULL) convention not used with this call 603 copyJoin1: // enter from copyinstr with kkNull set 604 mflr r0 // get return address 605 cmplwi r5,0 // buffer length 0? 606 lis r9,0x1000 // r9 <- 0x10000000 (256MB) 607 stw r0,FM_LR_SAVE(r1) // save return 608 cmplw cr1,r5,r9 // buffer length > 256MB ? 609 mfsprg r8,2 // get the features 610 beq-- copyinout_0 // 0 length is degenerate case 611 stwu r1,-kkFrameSize(r1) // set up stack frame 612 stw r2,kkCR(r1) // save caller's CR since we use cr3 613 mtcrf 0x02,r8 // move pf64Bit to cr6 614 stw r3,kkSource(r1) // save args across MapUserAddressSpace 615 stw r4,kkDest(r1) 616 stw r5,kkBufSize(r1) 617 crmove kk64bit,pf64Bitb // remember if this is a 64-bit processor 618 stw r6,kkCountPtr(r1) 619 stw r31,kkR31Save(r1) // we use r31 globally for mapped user ptr 620 li r31,0 // no mapped ptr yet 621 622 623 // Handle buffer length > 256MB. This is an error (ENAMETOOLONG) on copyin and copyout. 624 // The string ops are passed -1 lengths by some BSD callers, so for them we silently clamp 625 // the buffer length to 256MB. This isn't an issue if the string is less than 256MB 626 // (as most are!), but if they are >256MB we eventually return ENAMETOOLONG. This restriction 627 // is due to MapUserAddressSpace; we don't want to consume more than two segments for 628 // the mapping. 629 630 ble++ cr1,copyin0 // skip if buffer length <= 256MB 631 bf kkString,copyinout_too_big // error if not string op 632 mr r5,r9 // silently clamp buffer length to 256MB 633 stw r9,kkBufSize(r1) // update saved copy too 634 635 636 // Set up thread_recover in case we hit an illegal address. 637 638 copyin0: 639 mfsprg r8,1 /* Get the current act */ 640 lis r2,hi16(copyinout_error) 641 lwz r7,ACT_THREAD(r8) 642 ori r2,r2,lo16(copyinout_error) 643 lwz r3,ACT_VMMAP(r8) // r3 <- vm_map virtual address 644 stw r2,THREAD_RECOVER(r7) 645 646 647 // Map user segment into kernel map, turn on 64-bit mode. 648 // r3 = vm map 649 // r5 = buffer length 650 // r10 = user space ptr (r3 if copyin, r4 if copyout) 651 652 mr r6,r5 // Set length to map 653 li r4,0 // Note: we only do this 32-bit for now 654 mr r5,r10 // arg2 <- user space ptr 655 #if INSTRUMENT 656 mfspr r12,pmc1 ; INSTRUMENT - saveinstr[13] - Take stamp before mapuseraddressspace 657 stw r12,0x6100+(13*16)+0x0(0) ; INSTRUMENT - Save it 658 mfspr r12,pmc2 ; INSTRUMENT - Get stamp 659 stw r12,0x6100+(13*16)+0x4(0) ; INSTRUMENT - Save it 660 mfspr r12,pmc3 ; INSTRUMENT - Get stamp 661 stw r12,0x6100+(13*16)+0x8(0) ; INSTRUMENT - Save it 662 mfspr r12,pmc4 ; INSTRUMENT - Get stamp 663 stw r12,0x6100+(13*16)+0xC(0) ; INSTRUMENT - Save it 664 #endif 665 bl EXT(MapUserAddressSpace) // set r3 <- address in kernel map of user operand 666 #if INSTRUMENT 667 mfspr r12,pmc1 ; INSTRUMENT - saveinstr[14] - Take stamp after mapuseraddressspace 668 stw r12,0x6100+(14*16)+0x0(0) ; INSTRUMENT - Save it 669 mfspr r12,pmc2 ; INSTRUMENT - Get stamp 670 stw r12,0x6100+(14*16)+0x4(0) ; INSTRUMENT - Save it 671 mfspr r12,pmc3 ; INSTRUMENT - Get stamp 672 stw r12,0x6100+(14*16)+0x8(0) ; INSTRUMENT - Save it 673 mfspr r12,pmc4 ; INSTRUMENT - Get stamp 674 stw r12,0x6100+(14*16)+0xC(0) ; INSTRUMENT - Save it 675 #endif 676 or. r0,r3,r4 // Did we fail the mapping? 677 mr r31,r4 // r31 <- mapped ptr into user space (may be 64-bit) 678 beq-- copyinout_error // was 0, so there was an error making the mapping 679 bf-- kk64bit,copyin1 // skip if a 32-bit processor 680 681 rldimi r31,r3,32,0 // slam high-order bits into mapped ptr 682 mfmsr r4 // if 64-bit, turn on SF so we can use returned ptr 683 li r0,1 684 rldimi r4,r0,63,MSR_SF_BIT // light bit 0 685 mtmsrd r4 // turn on 64-bit mode 686 isync // wait for mode to change 687 688 689 // Load r3-r5, substituting mapped ptr as appropriate. 690 691 copyin1: 692 lwz r5,kkBufSize(r1) // restore length to copy 693 bf kkIn,copyin2 // skip if copyout 694 lwz r4,kkDest(r1) // copyin: source is mapped, dest is r4 at entry 695 mr r3,r31 // source is mapped ptr 696 b copyin3 697 copyin2: // handle copyout 698 lwz r3,kkSource(r1) // source is kernel buffer (r3 at entry) 699 mr r4,r31 // dest is mapped ptr into user space 700 701 702 // Finally, all set up to copy: 703 // r3 = source ptr (mapped if copyin) 704 // r4 = dest ptr (mapped if copyout) 705 // r5 = length 706 // r31 = mapped ptr returned by MapUserAddressSpace 707 // cr3 = kkIn, kkString, kk64bit, and kkNull flags 708 709 copyin3: 710 bt kkString,copyString // handle copyinstr and copyoutstr 711 bl EXT(bcopy) // copyin and copyout: let bcopy do the work 712 li r3,0 // return success 713 714 715 // Main exit point for copyin, copyout, copyinstr, and copyoutstr. Also reached 716 // from error recovery if we get a DSI accessing user space. Clear recovery ptr, 717 // and pop off frame. Note that we have kept 718 // the mapped ptr into user space in r31, as a reg64_t type (ie, a 64-bit ptr on 719 // 64-bit machines.) We must unpack r31 into an addr64_t in (r3,r4) before passing 720 // it to ReleaseUserAddressSpace. 721 // r3 = 0, EFAULT, or ENAMETOOLONG 722 723 copyinx: 724 lwz r2,kkCR(r1) // get callers cr3 725 mfsprg r6,1 // Get the current act 726 lwz r10,ACT_THREAD(r6) 727 728 bf-- kk64bit,copyinx1 // skip if 32-bit processor 729 mfmsr r12 730 rldicl r12,r12,0,MSR_SF_BIT+1 // if 64-bit processor, turn 64-bit mode off 731 mtmsrd r12 // turn SF off and EE back on 732 isync // wait for the mode to change 733 copyinx1: 734 lwz r31,kkR31Save(r1) // restore callers r31 735 addi r1,r1,kkFrameSize // pop off our stack frame 736 lwz r0,FM_LR_SAVE(r1) 737 li r4,0 738 stw r4,THREAD_RECOVER(r10) // Clear recovery 739 mtlr r0 740 mtcrf 0x10,r2 // restore cr3 741 blr 742 743 744 /* We get here via the exception handler if an illegal 745 * user memory reference was made. This error handler is used by 746 * copyin, copyout, copyinstr, and copyoutstr. Registers are as 747 * they were at point of fault, so for example cr3 flags are valid. 748 */ 749 750 copyinout_error: 751 li r3,EFAULT // return error 752 b copyinx 753 754 copyinout_0: // degenerate case: 0-length copy 755 mtcrf 0x10,r2 // restore cr3 756 li r3,0 // return success 757 blr 758 759 copyinout_too_big: // degenerate case 760 mtcrf 0x10,r2 // restore cr3 761 lwz r1,0(r1) // pop off stack frame 762 li r3,ENAMETOOLONG 763 blr 764 765 766 //<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><> 767 // Handle copyinstr and copyoutstr. At this point the stack frame is set up, 768 // the recovery ptr is set, the user's buffer is mapped, we're in 64-bit mode 769 // if necessary, and: 770 // r3 = source ptr, mapped if copyinstr 771 // r4 = dest ptr, mapped if copyoutstr 772 // r5 = buffer length 773 // r31 = mapped ptr returned by MapUserAddressSpace 774 // cr3 = kkIn, kkString, kkNull, and kk64bit flags 775 // We do word copies unless the buffer is very short, then use a byte copy loop 776 // for the leftovers if necessary. 777 778 copyString: 779 li r12,0 // Set header bytes count to zero 780 cmplwi cr1,r5,20 // is buffer very short? 781 mtctr r5 // assuming short, set up loop count for bytes 782 blt cr1,copyinstr8 // too short for word loop 783 andi. r12,r3,0x3 // is source ptr word aligned? 784 bne copyinstr11 // bytes loop 785 copyinstr1: 786 srwi r6,r5,2 // get #words in buffer 787 mtctr r6 // set up word loop count 788 lis r10,hi16(0xFEFEFEFF) // load magic constants into r10 and r11 789 lis r11,hi16(0x80808080) 790 ori r10,r10,lo16(0xFEFEFEFF) 791 ori r11,r11,lo16(0x80808080) 792 bf kkNull,copyinstr6 // enter loop that copies 793 b copyinstr5 // use loop that just counts 794 795 796 // Word loop(s). They do a word-parallel search for 0s, using the following 797 // inobvious but very efficient test: 798 // y = data + 0xFEFEFEFF 799 // z = ~data & 0x80808080 800 // If (y & z)==0, then all bytes in dataword are nonzero. We need two copies of 801 // this loop, since if we test kkNull in the loop then it becomes 9 words long. 802 803 .align 5 // align inner loops for speed 804 copyinstr5: // version that counts but does not copy 805 lwz r8,0(r3) // get next word of source 806 addi r3,r3,4 // increment source ptr 807 add r9,r10,r8 // r9 = data + 0xFEFEFEFF 808 andc r7,r11,r8 // r7 = ~data & 0x80808080 809 and. r7,r9,r7 // r7 = r9 & r7 810 bdnzt cr0_eq,copyinstr5 // if r7==0, then all bytes are nonzero 811 812 b copyinstr7 813 814 .align 5 // align inner loops for speed 815 copyinstr6: // version that counts and copies 816 lwz r8,0(r3) // get next word of source 817 addi r3,r3,4 // increment source ptr 818 addi r4,r4,4 // increment dest ptr while we wait for data 819 add r9,r10,r8 // r9 = data + 0xFEFEFEFF 820 andc r7,r11,r8 // r7 = ~data & 0x80808080 821 and. r7,r9,r7 // r7 = r9 & r7 822 stw r8,-4(r4) // pack all 4 bytes into buffer 823 bdnzt cr0_eq,copyinstr6 // if r7==0, then all bytes are nonzero 824 825 826 // Either 0 found or buffer filled. The above algorithm has mapped nonzero bytes to 0 827 // and 0 bytes to 0x80 with one exception: 0x01 bytes preceeding the first 0 are also 828 // mapped to 0x80. We must mask out these false hits before searching for an 0x80 byte. 829 830 copyinstr7: 831 crnot kkZero,cr0_eq // 0 found iff cr0_eq is off 832 mfctr r6 // get #words remaining in buffer 833 rlwinm r2,r8,7,0,31 // move 0x01 bits to 0x80 position 834 slwi r6,r6,2 // convert to #bytes remaining 835 andc r7,r7,r2 // turn off false hits from 0x0100 worst case 836 rlwimi r6,r5,0,30,31 // add in odd bytes leftover in buffer 837 srwi r7,r7,8 // we want to count the 0 as a byte xferred 838 addi r6,r6,4 // don't count last word xferred (yet) 839 cntlzw r7,r7 // now we can find the 0 byte (ie, the 0x80) 840 srwi r7,r7,3 // convert 8,16,24,32 to 1,2,3,4 841 sub. r6,r6,r7 // account for nonzero bytes in last word 842 bt++ kkZero,copyinstr10 // 0 found, so done 843 844 beq copyinstr10 // r6==0, so buffer truly full 845 mtctr r6 // 0 not found, loop over r6 bytes 846 b copyinstr8 // enter byte loop for last 1-3 leftover bytes 847 848 849 // Byte loop. This is used for very small buffers and for the odd bytes left over 850 // after searching and copying words at a time. 851 852 .align 5 // align inner loops for speed 853 copyinstr8: // loop over bytes of source 854 lbz r0,0(r3) // get next byte of source 855 addi r3,r3,1 856 addi r4,r4,1 // increment dest addr whether we store or not 857 cmpwi r0,0 // the 0? 858 bt-- kkNull,copyinstr9 // don't store (was copyinstr with NULL ptr) 859 stb r0,-1(r4) 860 copyinstr9: 861 bdnzf cr0_eq,copyinstr8 // loop if byte not 0 and more room in buffer 862 863 mfctr r6 // get #bytes left in buffer 864 crmove kkZero,cr0_eq // remember if 0 found or buffer filled 865 866 867 // Buffer filled or 0 found. Unwind and return. 868 // r5 = kkBufSize, ie buffer length 869 // r6 = untransferred bytes remaining in buffer 870 // r31 = mapped ptr returned by MapUserAddressSpace 871 // cr3 = kkZero set iff 0 found 872 873 copyinstr10: 874 lwz r9,kkCountPtr(r1) // get ptr to place to store count of bytes moved 875 sub r2,r5,r6 // get #bytes we moved, counting the 0 iff any 876 add r2,r2,r12 // add the header bytes count 877 li r3,0 // assume 0 return status 878 stw r2,0(r9) // store #bytes moved 879 bt++ kkZero,copyinx // we did find the 0 so return 0 880 li r3,ENAMETOOLONG // buffer filled 881 b copyinx // join main exit routine 882 883 // Byte loop. This is used on the header bytes for unaligned source 884 885 .align 5 // align inner loops for speed 886 copyinstr11: 887 li r10,4 // load word size 888 sub r12,r10,r12 // set the header bytes count 889 mtctr r12 // set up bytes loop count 890 copyinstr12: // loop over bytes of source 891 lbz r0,0(r3) // get next byte of source 892 addi r3,r3,1 893 addi r4,r4,1 // increment dest addr whether we store or not 894 cmpwi r0,0 // the 0? 895 bt-- kkNull,copyinstr13 // don't store (was copyinstr with NULL ptr) 896 stb r0,-1(r4) 897 copyinstr13: 898 bdnzf cr0_eq,copyinstr12 // loop if byte not 0 and more room in buffer 899 sub r5,r5,r12 // substract the bytes copied 900 bne cr0_eq,copyinstr1 // branch to word loop 901 902 mr r5,r12 // Get the header bytes count 903 li r12,0 // Clear the header bytes count 904 mfctr r6 // get #bytes left in buffer 905 crmove kkZero,cr0_eq // remember if 0 found or buffer filled 906 b copyinstr10 907
Cache object: 3e4e30bc43ae4169f97cd320e5a56cc9
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]