Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/osfmk/ppc/cache.s
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* 2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_LICENSE_HEADER_START@ 5 * 6 * The contents of this file constitute Original Code as defined in and 7 * are subject to the Apple Public Source License Version 1.1 (the 8 * "License"). You may not use this file except in compliance with the 9 * License. Please obtain a copy of the License at 10 * http://www.apple.com/publicsource and read it before using this file. 11 * 12 * This Original Code and all software distributed under the License are 13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER 14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the 17 * License for the specific language governing rights and limitations 18 * under the License. 19 * 20 * @APPLE_LICENSE_HEADER_END@ 21 */ 22 /* 23 * @OSF_COPYRIGHT@ 24 */ 25 26 #include <ppc/asm.h> 27 #include <ppc/proc_reg.h> 28 #include <ppc/exception.h> 29 #include <assym.s> 30 31 /* These routines run in 32 or 64-bit addressing, and handle 32 * 32 and 128 byte caches. They do not use compare instructions 33 * on addresses, since compares are 32/64-bit-mode-specific. 34 */ 35 36 #define kDcbf 0x1 37 #define kDcbfb 31 38 #define kDcbi 0x2 39 #define kDcbib 30 40 #define kIcbi 0x4 41 #define kIcbib 29 42 43 44 /* 45 * extern void flush_dcache(vm_offset_t addr, unsigned count, boolean phys); 46 * extern void flush_dcache64(addr64_t addr, unsigned count, boolean phys); 47 * 48 * flush_dcache takes a virtual or physical address and count to flush 49 * and (can be called for multiple virtual pages). 50 * 51 * it flushes the data cache 52 * cache for the address range in question 53 * 54 * if 'phys' is non-zero then physical addresses will be used 55 */ 56 57 58 59 .text 60 .align 5 61 .globl _flush_dcache 62 _flush_dcache: 63 li r0,kDcbf // use DCBF instruction 64 rlwinm r3,r3,0,0,31 // truncate address in case this is a 64-bit machine 65 b cache_op_join // join common code 66 67 .align 5 68 .globl _flush_dcache64 69 _flush_dcache64: 70 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg 71 li r0,kDcbf // use DCBF instruction 72 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits 73 mr r4,r5 ; Move count 74 mr r5,r6 ; Move physical flag 75 b cache_op_join // join common code 76 77 78 /* 79 * extern void invalidate_dcache(vm_offset_t va, unsigned count, boolean phys); 80 * extern void invalidate_dcache64(addr64_t va, unsigned count, boolean phys); 81 * 82 * invalidate_dcache takes a virtual or physical address and count to 83 * invalidate and (can be called for multiple virtual pages). 84 * 85 * it invalidates the data cache for the address range in question 86 */ 87 88 .globl _invalidate_dcache 89 _invalidate_dcache: 90 li r0,kDcbi // use DCBI instruction 91 rlwinm r3,r3,0,0,31 // truncate address in case this is a 64-bit machine 92 b cache_op_join // join common code 93 94 95 .align 5 96 .globl _invalidate_dcache64 97 _invalidate_dcache64: 98 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg 99 li r0,kDcbi // use DCBI instruction 100 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits 101 mr r4,r5 ; Move count 102 mr r5,r6 ; Move physical flag 103 b cache_op_join // join common code 104 105 /* 106 * extern void invalidate_icache(vm_offset_t addr, unsigned cnt, boolean phys); 107 * extern void invalidate_icache64(addr64_t addr, unsigned cnt, boolean phys); 108 * 109 * invalidate_icache takes a virtual or physical address and 110 * count to invalidate, (can be called for multiple virtual pages). 111 * 112 * it invalidates the instruction cache for the address range in question. 113 */ 114 115 .globl _invalidate_icache 116 _invalidate_icache: 117 li r0,kIcbi // use ICBI instruction 118 rlwinm r3,r3,0,0,31 // truncate address in case this is a 64-bit machine 119 b cache_op_join // join common code 120 121 122 .align 5 123 .globl _invalidate_icache64 124 _invalidate_icache64: 125 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg 126 li r0,kIcbi // use ICBI instruction 127 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits 128 mr r4,r5 ; Move count 129 mr r5,r6 ; Move physical flag 130 b cache_op_join // join common code 131 132 /* 133 * extern void sync_ppage(ppnum_t pa); 134 * 135 * sync_ppage takes a physical page number 136 * 137 * it writes out the data cache and invalidates the instruction 138 * cache for the address range in question 139 */ 140 141 .globl _sync_ppage 142 .align 5 143 _sync_ppage: // Should be the most commonly called routine, by far 144 mfsprg r2,2 145 li r0,kDcbf+kIcbi // we need to dcbf and then icbi 146 mtcrf 0x02,r2 ; Move pf64Bit to cr6 147 li r5,1 // set flag for physical addresses 148 li r4,4096 ; Set page size 149 bt++ pf64Bitb,spp64 ; Skip if 64-bit (only they take the hint) 150 rlwinm r3,r3,12,0,19 ; Convert to physical address - 32-bit 151 b cache_op_join ; Join up.... 152 153 spp64: sldi r3,r3,12 ; Convert to physical address - 64-bit 154 b cache_op_join ; Join up.... 155 156 157 158 /* 159 * extern void sync_cache_virtual(vm_offset_t addr, unsigned count); 160 * 161 * Like "sync_cache", except it takes a virtual address and byte count. 162 * It flushes the data cache, invalidates the I cache, and sync's. 163 */ 164 165 .globl _sync_cache_virtual 166 .align 5 167 _sync_cache_virtual: 168 li r0,kDcbf+kIcbi // we need to dcbf and then icbi 169 li r5,0 // set flag for virtual addresses 170 b cache_op_join // join common code 171 172 173 /* 174 * extern void sync_cache(vm_offset_t pa, unsigned count); 175 * extern void sync_cache64(addr64_t pa, unsigned count); 176 * 177 * sync_cache takes a physical address and count to sync, thus 178 * must not be called for multiple virtual pages. 179 * 180 * it writes out the data cache and invalidates the instruction 181 * cache for the address range in question 182 */ 183 184 .globl _sync_cache 185 .align 5 186 _sync_cache: 187 li r0,kDcbf+kIcbi // we need to dcbf and then icbi 188 li r5,1 // set flag for physical addresses 189 rlwinm r3,r3,0,0,31 // truncate address in case this is a 64-bit machine 190 b cache_op_join // join common code 191 192 .globl _sync_cache64 193 .align 5 194 _sync_cache64: 195 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg 196 li r0,kDcbf+kIcbi // we need to dcbf and then icbi 197 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits 198 mr r4,r5 ; Copy over the length 199 li r5,1 // set flag for physical addresses 200 201 202 // Common code to handle the cache operations. 203 204 cache_op_join: // here with r3=addr, r4=count, r5=phys flag, r0=bits 205 mfsprg r10,2 // r10 <- processor feature flags 206 cmpwi cr5,r5,0 // using physical addresses? 207 mtcrf 0x01,r0 // move kDcbf, kDcbi, and kIcbi bits to CR7 208 andi. r9,r10,pf32Byte+pf128Byte // r9 <- cache line size 209 mtcrf 0x02,r10 // move pf64Bit bit to CR6 210 subi r8,r9,1 // r8 <- (linesize-1) 211 beq-- cr5,cache_op_2 // skip if using virtual addresses 212 213 bf-- pf64Bitb,cache_op_not64 // This is not a 64-bit machine 214 215 srdi r12,r3,31 // Slide bit 32 to bit 63 216 cmpldi r12,1 // Are we in the I/O mapped area? 217 beqlr-- // No cache ops allowed here... 218 219 cache_op_not64: 220 mflr r12 // save return address 221 bl EXT(ml_set_physical) // turn on physical addressing 222 mtlr r12 // restore return address 223 224 // get r3=first cache line, r4=first line not in set, r6=byte count 225 226 cache_op_2: 227 add r7,r3,r4 // point to 1st byte not to operate on 228 andc r3,r3,r8 // r3 <- 1st line to operate on 229 add r4,r7,r8 // round up 230 andc r4,r4,r8 // r4 <- 1st line not to operate on 231 sub. r6,r4,r3 // r6 <- byte count to operate on 232 beq-- cache_op_exit // nothing to do 233 bf-- kDcbfb,cache_op_6 // no need to dcbf 234 235 236 // DCBF loop 237 238 cache_op_5: 239 sub. r6,r6,r9 // more to go? 240 dcbf r6,r3 // flush next line to RAM 241 bne cache_op_5 // loop if more to go 242 sync // make sure the data reaches RAM 243 sub r6,r4,r3 // reset count 244 245 246 // ICBI loop 247 248 cache_op_6: 249 bf-- kIcbib,cache_op_8 // no need to icbi 250 cache_op_7: 251 sub. r6,r6,r9 // more to go? 252 icbi r6,r3 // invalidate next line 253 bne cache_op_7 254 sub r6,r4,r3 // reset count 255 isync 256 sync 257 258 259 // DCBI loop 260 261 cache_op_8: 262 bf++ kDcbib,cache_op_exit // no need to dcbi 263 cache_op_9: 264 sub. r6,r6,r9 // more to go? 265 dcbi r6,r3 // invalidate next line 266 bne cache_op_9 267 sync 268 269 270 // restore MSR iff necessary and done 271 272 cache_op_exit: 273 beqlr-- cr5 // if using virtual addresses, no need to restore MSR 274 b EXT(ml_restore) // restore MSR and return 275 276 277 //////////////////////////////////////////////////// 278 279 .align 5 280 .globl _dcache_incoherent_io_store64 281 _dcache_incoherent_io_store64: 282 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg 283 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits 284 mr r4,r5 ; Move count 285 286 // here with r3=addr, r4=count 287 mfsprg r10,2 // r10 <- processor feature flags 288 andi. r9,r10,pf32Byte+pf128Byte // r9 <- cache line size 289 mtcrf 0x02,r10 // move pf64Bit bit to CR6 290 subi r8,r9,1 // r8 <- (linesize-1) 291 292 bf-- pf64Bitb,cache_ios_not64 // This is not a 64-bit machine 293 294 srdi r12,r3,31 // Slide bit 32 to bit 63 295 cmpldi r12,1 // Are we in the I/O mapped area? 296 beqlr-- // No cache ops allowed here... 297 298 cache_ios_not64: 299 mflr r12 // save return address 300 bl EXT(ml_set_physical) // turn on physical addressing 301 mtlr r12 // restore return address 302 303 // get r3=first cache line, r4=first line not in set, r6=byte count 304 add r7,r3,r4 // point to 1st byte not to operate on 305 andc r3,r3,r8 // r3 <- 1st line to operate on 306 add r4,r7,r8 // round up 307 andc r4,r4,r8 // r4 <- 1st line not to operate on 308 sub. r6,r4,r3 // r6 <- byte count to operate on 309 beq-- cache_ios_exit // nothing to do 310 311 sub. r6,r6,r9 // >1 line? 312 beq cache_ios_last_line // use dcbst on all lines but last 313 314 // DCBST loop 315 cache_ios_5: 316 sub. r6,r6,r9 // more to go? 317 dcbst r6,r3 // store next line 318 bne cache_ios_5 // loop if more to go 319 320 cache_ios_last_line: 321 sync // flush last line 322 isync 323 dcbf r6,r3 324 sync 325 isync 326 add r6,r6,r3 327 lwz r0,0(r6) // make sure the data reaches RAM (not just the memory controller) 328 isync 329 330 // restore MSR 331 cache_ios_exit: 332 b EXT(ml_restore) // restore MSR and return 333 334 335 //////////////////////////////////////////////////// 336 337 .align 5 338 .globl _dcache_incoherent_io_flush64 339 _dcache_incoherent_io_flush64: 340 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg 341 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits 342 mr r4,r5 ; Move count 343 344 // here with r3=addr, r4=count 345 mfsprg r10,2 // r10 <- processor feature flags 346 andi. r9,r10,pf32Byte+pf128Byte // r9 <- cache line size 347 mtcrf 0x02,r10 // move pf64Bit bit to CR6 348 subi r8,r9,1 // r8 <- (linesize-1) 349 350 bf-- pf64Bitb,cache_iof_not64 // This is not a 64-bit machine 351 352 srdi r12,r3,31 // Slide bit 32 to bit 63 353 cmpldi r12,1 // Are we in the I/O mapped area? 354 beqlr-- // No cache ops allowed here... 355 356 cache_iof_not64: 357 mflr r12 // save return address 358 bl EXT(ml_set_physical) // turn on physical addressing 359 mtlr r12 // restore return address 360 361 // get r3=first cache line, r4=first line not in set, r6=byte count 362 add r7,r3,r4 // point to 1st byte not to operate on 363 andc r3,r3,r8 // r3 <- 1st line to operate on 364 add r4,r7,r8 // round up 365 andc r4,r4,r8 // r4 <- 1st line not to operate on 366 sub. r6,r4,r3 // r6 <- byte count to operate on 367 beq-- cache_iof_exit // nothing to do 368 369 // DCBF loop 370 cache_iof_5: 371 sub. r6,r6,r9 // more to go? 372 dcbf r6,r3 // store next line 373 bne cache_iof_5 // loop if more to go 374 375 cache_iof_last_line: 376 sync // flush last line 377 isync 378 379 // restore MSR 380 cache_iof_exit: 381 b EXT(ml_restore) // restore MSR and return 382 383
Cache object: 86f18ef33dae280252ea62d1947d3fa7
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]