Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/powerpc/powerpc/support.S
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2018, Matthew Macy <mmacy@freebsd.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * $FreeBSD$ 28 */ 29 30 /* 31 * Assembly variants of various functions, for those that don't need generic C 32 * implementations. Currently this includes: 33 * 34 * - Direct-access versions of copyin/copyout methods. 35 * - These are used by Radix AIM pmap (ISA 3.0), and Book-E, to avoid 36 * unnecessary pmap_map_usr_ptr() calls. 37 */ 38 39 #include "assym.inc" 40 #include "opt_sched.h" 41 42 #include <sys/syscall.h> 43 #include <sys/errno.h> 44 45 #include <machine/param.h> 46 #include <machine/asm.h> 47 #include <machine/spr.h> 48 #include <machine/trap.h> 49 #include <machine/vmparam.h> 50 51 #ifdef _CALL_ELF 52 .abiversion _CALL_ELF 53 #endif 54 55 #ifdef __powerpc64__ 56 #define LOAD ld 57 #define STORE std 58 #define WORD 8 59 #define CMPI cmpdi 60 #define CMPLI cmpldi 61 /* log_2(8 * WORD) */ 62 #define LOOP_LOG 6 63 #define LOG_WORD 3 64 #define CURTHREAD %r13 65 #else 66 #define LOAD lwz 67 #define STORE stw 68 #define WORD 4 69 #define CMPI cmpwi 70 #define CMPLI cmplwi 71 /* log_2(8 * WORD) */ 72 #define LOOP_LOG 5 73 #define LOG_WORD 2 74 #define CURTHREAD %r2 75 #endif 76 77 #ifdef AIM 78 #define ENTRY_DIRECT(x) ENTRY(x ## _direct) 79 #define END_DIRECT(x) END(x ## _direct) 80 #else 81 #define ENTRY_DIRECT(x) ENTRY(x) 82 #define END_DIRECT(x) END(x) 83 #endif 84 85 #ifdef __powerpc64__ 86 #define PROLOGUE ;\ 87 mflr %r0 ;\ 88 std %r0, 16(%r1) ;\ 89 90 #define EPILOGUE ;\ 91 ld %r0, 16(%r1) ;\ 92 mtlr %r0 ;\ 93 blr ;\ 94 nop 95 96 #define VALIDATE_TRUNCATE_ADDR_COPY VALIDATE_ADDR_COPY 97 #define VALIDATE_ADDR_COPY(raddr, len) \ 98 srdi %r0, raddr, 52 ;\ 99 cmpwi %r0, 1 ;\ 100 bge- copy_fault ;\ 101 nop 102 103 #define VALIDATE_ADDR_FUSU(raddr) ;\ 104 srdi %r0, raddr, 52 ;\ 105 cmpwi %r0, 1 ;\ 106 bge- fusufault ;\ 107 nop 108 109 #else 110 #define PROLOGUE ;\ 111 mflr %r0 ;\ 112 stw %r0, 4(%r1) ;\ 113 114 #define EPILOGUE ;\ 115 lwz %r0, 4(%r1) ;\ 116 mtlr %r0 ;\ 117 blr ;\ 118 nop 119 120 /* %r0 is temporary */ 121 /* 122 * Validate address and length are valid. 123 * For VALIDATE_ADDR_COPY() have to account for wraparound. 124 */ 125 #define VALIDATE_ADDR_COPY(raddr, len) \ 126 lis %r0, VM_MAXUSER_ADDRESS@h ;\ 127 ori %r0, %r0, VM_MAXUSER_ADDRESS@l ;\ 128 cmplw %r0, raddr ;\ 129 blt- copy_fault ;\ 130 add %r0, raddr, len ;\ 131 cmplw 7, %r0, raddr ;\ 132 blt- 7, copy_fault ;\ 133 mtcrf 0x80, %r0 ;\ 134 bt- 0, copy_fault ;\ 135 nop 136 137 #define VALIDATE_TRUNCATE_ADDR_COPY(raddr, len) \ 138 lis %r0, VM_MAXUSER_ADDRESS@h ;\ 139 ori %r0, %r0, VM_MAXUSER_ADDRESS@l ;\ 140 cmplw %r0, raddr ;\ 141 blt- copy_fault ;\ 142 sub %r0, %r0, raddr ;\ 143 cmplw len, %r0 ;\ 144 isel len, len, %r0, 0 ;\ 145 146 #define VALIDATE_ADDR_FUSU(raddr) \ 147 lis %r0, VM_MAXUSER_ADDRESS@h ;\ 148 ori %r0, %r0, VM_MAXUSER_ADDRESS@l ;\ 149 cmplw %r0, raddr ;\ 150 ble- fusufault 151 152 #endif 153 154 #define SET_COPYFAULT(raddr, rpcb, len) \ 155 VALIDATE_ADDR_COPY(raddr, len) ;\ 156 li %r0, COPYFAULT ;\ 157 LOAD rpcb, TD_PCB(CURTHREAD) ;\ 158 STORE %r0, PCB_ONFAULT(rpcb) ;\ 159 160 #define SET_COPYFAULT_TRUNCATE(raddr, rpcb, len)\ 161 VALIDATE_TRUNCATE_ADDR_COPY(raddr, len) ;\ 162 li %r0, COPYFAULT ;\ 163 LOAD rpcb, TD_PCB(CURTHREAD) ;\ 164 STORE %r0, PCB_ONFAULT(rpcb) 165 166 #define SET_FUSUFAULT(raddr, rpcb) \ 167 VALIDATE_ADDR_FUSU(raddr) ;\ 168 li %r0, FUSUFAULT ;\ 169 LOAD rpcb, TD_PCB(CURTHREAD) ;\ 170 STORE %r0, PCB_ONFAULT(rpcb) 171 172 #define CLEAR_FAULT_NO_CLOBBER(rpcb) \ 173 LOAD rpcb, TD_PCB(CURTHREAD) ;\ 174 li %r0, 0 ;\ 175 STORE %r0, PCB_ONFAULT(rpcb) 176 177 #define CLEAR_FAULT(rpcb) \ 178 CLEAR_FAULT_NO_CLOBBER(rpcb) ;\ 179 li %r3, 0 180 181 /* 182 * bcopy(src, dst, len) 183 * %r3 %r4 %r5 184 * 185 * %r7 is the pcb pointer 186 * 187 * %r0 and %r8-%r10 are volatile 188 * %r11 and %r12 are generally volatile, used in linking and exception 189 * handling. Can be clobbered here. 190 * 191 * Does not allocate or use stack space, but clobbers all volatile registers. 192 */ 193 194 #define rs %r3 195 #define rd %r4 196 #define rl %r5 197 198 #define t1 %r6 199 #define t2 %r7 200 #define t3 %r8 201 #define t4 %r9 202 #define t5 %r10 203 #define t6 %r11 204 #define t7 %r12 205 #define t8 %r0 206 207 #define Thresh WORD * 8 208 #define W4 3 209 #define W2 2 210 #define W1 1 211 #define WORDS(n) (32 - LOG_WORD - W##n) 212 .text 213 ENTRY(bcopy_generic) 214 CMPLI 0, %r5, 0 215 beq .Lend 216 dcbtst 0, rd 217 dcbt 0, rs 218 CMPLI rl, Thresh 219 blt .Lsmall 220 b .Llarge 221 /* memcpy */ 222 /* ... */ 223 .Lsmall: /* < 8 words remaining */ 224 mtcrf 0x3, rl 225 .Lsmall_start: 226 bf WORDS(4), 0f 227 LOAD t1, 0(rs) 228 LOAD t2, WORD*1(rs) 229 LOAD t3, WORD*2(rs) 230 LOAD t4, WORD*3(rs) 231 addi rs, rs, WORD*4 232 STORE t1, 0(rd) 233 STORE t2, WORD*1(rd) 234 STORE t3, WORD*2(rd) 235 STORE t4, WORD*3(rd) 236 addi rd, rd, WORD*4 237 0: /* < 4 words remaining */ 238 bf WORDS(2), 1f 239 LOAD t1, 0(rs) 240 LOAD t2, WORD*1(rs) 241 addi rs, rs, WORD*2 242 STORE t1, 0(rd) 243 STORE t2, WORD*1(rd) 244 addi rd, rd, WORD*2 245 1: /* < 2 words remaining */ 246 bf WORDS(1), 2f 247 LOAD t1, 0(rs) 248 addi rs, rs, WORD 249 STORE t1, 0(rd) 250 addi rd, rd, WORD 251 2: /* < 1 word remaining */ 252 #ifdef __powerpc64__ 253 bf 29, 3f 254 lwz t1, 0(rs) 255 addi rs, rs, 4 256 stw t1, 0(rd) 257 addi rd, rd, 4 258 3: /* < 4 bytes remaining */ 259 #endif 260 bf 30, 4f 261 lhz t1, 0(rs) 262 addi rs, rs, 2 263 sth t1, 0(rd) 264 addi rd, rd, 2 265 4: /* < 2 bytes remaining */ 266 bf 31, .Lout 267 lbz t1, 0(rs) 268 addi rs, rs, 1 269 stb t1, 0(rd) 270 addi rd, rd, 1 271 b .Lout 272 273 .align 4 274 .Llarge: 275 neg t3, rd 276 andi. t6, t3, WORD-1 /* Align rd to word size */ 277 mtctr t6 278 sub rl, rl, t6 279 beq+ .Llargealigned 280 1: 281 lbz t1, 0(rs) 282 addi rs, rs, 1 283 stb t1, 0(rd) 284 addi rd, rd, 1 285 bdnz 1b 286 287 .Llargealigned: 288 srwi. t2, rl, LOOP_LOG /* length >> log_2(loop_size) => 8W iterations */ 289 mtcrf 0x3, rl 290 beq .Lsmall_start 291 mtctr t2 292 b 1f 293 294 .align 5 295 1: 296 LOAD t1, 0(rs) 297 LOAD t2, WORD(rs) 298 LOAD t3, WORD*2(rs) 299 LOAD t4, WORD*3(rs) 300 LOAD t5, WORD*4(rs) 301 LOAD t6, WORD*5(rs) 302 LOAD t7, WORD*6(rs) 303 LOAD t8, WORD*7(rs) 304 addi rs, rs, WORD*8 305 STORE t1, 0(rd) 306 STORE t2, WORD*1(rd) 307 STORE t3, WORD*2(rd) 308 STORE t4, WORD*3(rd) 309 STORE t5, WORD*4(rd) 310 STORE t6, WORD*5(rd) 311 STORE t7, WORD*6(rd) 312 STORE t8, WORD*7(rd) 313 addi rd, rd, WORD*8 314 bdnz 1b 315 316 b .Lsmall_start 317 .Lout: 318 /* done */ 319 .Lend: 320 blr 321 END(bcopy_generic) 322 323 /* 324 * copyout(from_kernel, to_user, len) 325 * %r3, %r4, %r5 326 */ 327 ENTRY_DIRECT(copyout) 328 PROLOGUE 329 SET_COPYFAULT(%r4, %r7, %r5) 330 bl bcopy_generic 331 nop 332 CLEAR_FAULT(%r7) 333 EPILOGUE 334 END_DIRECT(copyout) 335 336 /* 337 * copyin(from_user, to_kernel, len) 338 * %r3, %r4, %r5 339 */ 340 ENTRY_DIRECT(copyin) 341 PROLOGUE 342 SET_COPYFAULT(%r3, %r7, %r5) 343 bl bcopy_generic 344 nop 345 CLEAR_FAULT(%r7) 346 EPILOGUE 347 END_DIRECT(copyin) 348 349 /* 350 * copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done) 351 * %r3 %r4 %r5 %r6 352 */ 353 354 ENTRY_DIRECT(copyinstr) 355 PROLOGUE 356 SET_COPYFAULT_TRUNCATE(%r3, %r7, %r5) 357 addi %r9, %r5, 1 358 mtctr %r9 359 mr %r8, %r3 360 addi %r8, %r8, -1 361 addi %r4, %r4, -1 362 li %r3, ENAMETOOLONG 363 0: 364 bdz- 2f 365 lbzu %r0, 1(%r8) 366 stbu %r0, 1(%r4) 367 368 // NULL byte reached ? 369 CMPI %r0, 0 370 beq- 1f 371 b 0b 372 1: 373 li %r3, 0 374 2: 375 /* skip storing length if done is NULL */ 376 CMPI %r6, 0 377 beq- 3f 378 mfctr %r0 379 sub %r0, %r9, %r0 380 STORE %r0, 0(%r6) 381 3: 382 CLEAR_FAULT_NO_CLOBBER(%r7) 383 EPILOGUE 384 END_DIRECT(copyinstr) 385 386 ENTRY_DIRECT(subyte) 387 PROLOGUE 388 SET_FUSUFAULT(%r3, %r7) 389 stb %r4, 0(%r3) 390 CLEAR_FAULT(%r7) 391 EPILOGUE 392 END_DIRECT(subyte) 393 394 #ifndef __powerpc64__ 395 ENTRY_DIRECT(suword) 396 PROLOGUE 397 SET_FUSUFAULT(%r3, %r7) 398 stw %r4, 0(%r3) 399 CLEAR_FAULT(%r7) 400 EPILOGUE 401 END_DIRECT(suword) 402 #endif 403 404 ENTRY_DIRECT(suword32) 405 PROLOGUE 406 SET_FUSUFAULT(%r3, %r7) 407 stw %r4, 0(%r3) 408 CLEAR_FAULT(%r7) 409 EPILOGUE 410 END_DIRECT(suword32) 411 412 #ifdef __powerpc64__ 413 ENTRY_DIRECT(suword64) 414 PROLOGUE 415 SET_FUSUFAULT(%r3, %r7) 416 std %r4, 0(%r3) 417 CLEAR_FAULT(%r7) 418 EPILOGUE 419 END_DIRECT(suword64) 420 421 ENTRY_DIRECT(suword) 422 PROLOGUE 423 SET_FUSUFAULT(%r3, %r7) 424 std %r4, 0(%r3) 425 CLEAR_FAULT(%r7) 426 EPILOGUE 427 END_DIRECT(suword) 428 #endif 429 430 ENTRY_DIRECT(fubyte) 431 PROLOGUE 432 SET_FUSUFAULT(%r3, %r7) 433 lbz %r3, 0(%r3) 434 CLEAR_FAULT_NO_CLOBBER(%r7) 435 EPILOGUE 436 END_DIRECT(fubyte) 437 438 ENTRY_DIRECT(fuword16) 439 PROLOGUE 440 SET_FUSUFAULT(%r3, %r7) 441 lhz %r3, 0(%r3) 442 CLEAR_FAULT_NO_CLOBBER(%r7) 443 EPILOGUE 444 END_DIRECT(fuword16) 445 446 #ifndef __powerpc64__ 447 ENTRY_DIRECT(fueword) 448 PROLOGUE 449 SET_FUSUFAULT(%r3, %r7) 450 lwz %r0, 0(%r3) 451 stw %r0, 0(%r4) 452 CLEAR_FAULT(%r7) 453 EPILOGUE 454 END_DIRECT(fueword) 455 #endif 456 ENTRY_DIRECT(fueword32) 457 PROLOGUE 458 SET_FUSUFAULT(%r3, %r7) 459 lwz %r0, 0(%r3) 460 stw %r0, 0(%r4) 461 CLEAR_FAULT(%r7) 462 EPILOGUE 463 END_DIRECT(fueword32) 464 465 #ifdef __powerpc64__ 466 ENTRY_DIRECT(fueword) 467 PROLOGUE 468 SET_FUSUFAULT(%r3, %r7) 469 ld %r0, 0(%r3) 470 std %r0, 0(%r4) 471 CLEAR_FAULT(%r7) 472 EPILOGUE 473 END_DIRECT(fueword) 474 475 ENTRY_DIRECT(fueword64) 476 PROLOGUE 477 SET_FUSUFAULT(%r3, %r7) 478 ld %r0, 0(%r3) 479 std %r0, 0(%r4) 480 CLEAR_FAULT(%r7) 481 EPILOGUE 482 END_DIRECT(fueword64) 483 #endif 484 485 /* 486 * casueword(volatile u_long *base, u_long old, u_long *oldp, u_long new) 487 * %r3 %r4 %r5 %r6 488 */ 489 490 #define CASUEWORD32(raddr, rpcb) ;\ 491 PROLOGUE ;\ 492 SET_FUSUFAULT(raddr, rpcb) ;\ 493 li %r8, 0 ;\ 494 1: ;\ 495 lwarx %r0, 0, %r3 ;\ 496 cmplw %r4, %r0 ;\ 497 bne 2f ;\ 498 stwcx. %r6, 0, %r3 ;\ 499 bne- 3f ;\ 500 b 4f ;\ 501 2: ;\ 502 stwcx. %r0, 0, %r3 /* clear reservation (74xx) */ ;\ 503 3: ;\ 504 li %r8, 1 ;\ 505 4: ;\ 506 stw %r0, 0(%r5) ;\ 507 CLEAR_FAULT_NO_CLOBBER(rpcb) ;\ 508 mr %r3, %r8 ;\ 509 EPILOGUE 510 511 ENTRY_DIRECT(casueword32) 512 CASUEWORD32(%r3, %r7) 513 END_DIRECT(casueword32) 514 515 #ifdef __powerpc64__ 516 #define CASUEWORD64(raddr, rpcb) ;\ 517 PROLOGUE ;\ 518 SET_FUSUFAULT(raddr, rpcb) ;\ 519 li %r8, 0 ;\ 520 1: ;\ 521 ldarx %r0, 0, %r3 ;\ 522 cmpld %r4, %r0 ;\ 523 bne 2f ;\ 524 stdcx. %r6, 0, %r3 ;\ 525 bne- 3f ;\ 526 b 4f ;\ 527 2: ;\ 528 stdcx. %r0, 0, %r3 /* clear reservation (74xx) */ ;\ 529 3: ;\ 530 li %r8, 1 ;\ 531 4: ;\ 532 std %r0, 0(%r5) ;\ 533 CLEAR_FAULT_NO_CLOBBER(rpcb) ;\ 534 mr %r3, %r8 ;\ 535 EPILOGUE 536 537 ENTRY_DIRECT(casueword) 538 CASUEWORD64(%r3, %r7) 539 END_DIRECT(casueword) 540 541 ENTRY_DIRECT(casueword64) 542 CASUEWORD64(%r3, %r7) 543 END_DIRECT(casueword64) 544 #else 545 ENTRY_DIRECT(casueword) 546 CASUEWORD32(%r3, %r7) 547 END_DIRECT(casueword) 548 #endif 549 550 _NAKED_ENTRY(fusufault) 551 CLEAR_FAULT_NO_CLOBBER(%r7) 552 li %r3, -1 553 EPILOGUE 554 _END(fusufault) 555 556 _NAKED_ENTRY(copy_fault) 557 CLEAR_FAULT_NO_CLOBBER(%r7) 558 li %r3, EFAULT 559 EPILOGUE 560 _END(copy_fault)
Cache object: 8d379f200ff08cb1d718e1c7cb0f54e4
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]