Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/i386/i386/mplock.s
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* 2 * ---------------------------------------------------------------------------- 3 * "THE BEER-WARE LICENSE" (Revision 42): 4 * <phk@FreeBSD.org> wrote this file. As long as you retain this notice you 5 * can do whatever you want with this stuff. If we meet some day, and you think 6 * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp 7 * ---------------------------------------------------------------------------- 8 * 9 * $FreeBSD$ 10 * 11 * Functions for locking between CPUs in a SMP system. 12 * 13 * This is an "exclusive counting semaphore". This means that it can be 14 * free (0xffffffff) or be owned by a CPU (0xXXYYYYYY where XX is CPU-id 15 * and YYYYYY is the count). 16 * 17 * Contrary to most implementations around, this one is entirely atomic: 18 * The attempt to seize/release the semaphore and the increment/decrement 19 * is done in one atomic operation. This way we are safe from all kinds 20 * of weird reentrancy situations. 21 */ 22 23 #include <machine/asmacros.h> 24 #include <machine/smptests.h> /** GRAB_LOPRIO */ 25 #include <machine/apic.h> 26 27 #define GLPROFILE_NOT 28 29 #ifdef CHEAP_TPR 30 31 /* we assumme that the 'reserved bits' can be written with zeros */ 32 33 #else /* CHEAP_TPR */ 34 35 #error HEADS UP: this code needs work 36 /* 37 * The APIC doc says that reserved bits must be written with whatever 38 * value they currently contain, ie you should: read, modify, write, 39 * instead of just writing new values to the TPR register. Current 40 * silicon seems happy with just writing. If the behaviour of the 41 * silicon changes, all code that access the lapic_tpr must be modified. 42 * The last version to contain such code was: 43 * Id: mplock.s,v 1.17 1997/08/10 20:59:07 fsmp Exp 44 */ 45 46 #endif /* CHEAP_TPR */ 47 48 #ifdef GRAB_LOPRIO 49 /* 50 * Claim LOWest PRIOrity, ie. attempt to grab ALL INTerrupts. 51 */ 52 53 /* after 1st acquire of lock we grab all hardware INTs */ 54 #define GRAB_HWI movl $ALLHWI_LEVEL, lapic_tpr 55 56 /* after last release of lock give up LOW PRIO (ie, arbitrate INTerrupts) */ 57 #define ARB_HWI movl $LOPRIO_LEVEL, lapic_tpr /* CHEAP_TPR */ 58 59 #else /* GRAB_LOPRIO */ 60 61 #define GRAB_HWI /* nop */ 62 #define ARB_HWI /* nop */ 63 64 #endif /* GRAB_LOPRIO */ 65 66 67 .text 68 69 #ifdef SMP 70 71 /*********************************************************************** 72 * void MPgetlock_edx(unsigned int *lock : %edx) 73 * ---------------------------------- 74 * Destroys %eax, %ecx. %edx must hold lock argument. 75 * 76 * Grabs hardware interrupts on first aquire. 77 * 78 * NOTE: Serialization is not required if we already hold the lock, since 79 * we already hold the lock, nor do we need a locked instruction if we 80 * already hold the lock. 81 */ 82 83 NON_GPROF_ENTRY(MPgetlock_edx) 84 1: 85 movl (%edx), %eax /* Get current contents of lock */ 86 movl %eax, %ecx 87 andl $CPU_FIELD,%ecx 88 cmpl _cpu_lockid, %ecx /* Do we already own the lock? */ 89 jne 2f 90 incl %eax /* yes, just bump the count */ 91 movl %eax, (%edx) /* serialization not required */ 92 ret 93 2: 94 movl $FREE_LOCK, %eax /* lock must be free */ 95 movl _cpu_lockid, %ecx 96 incl %ecx 97 lock 98 cmpxchg %ecx, (%edx) /* attempt to replace %eax<->%ecx */ 99 #ifdef GLPROFILE 100 jne 3f 101 incl _gethits2 102 #else 103 jne 1b 104 #endif /* GLPROFILE */ 105 GRAB_HWI /* 1st acquire, grab hw INTs */ 106 ret 107 #ifdef GLPROFILE 108 3: 109 incl _gethits3 110 jmp 1b 111 #endif 112 113 /*********************************************************************** 114 * int MPtrylock(unsigned int *lock) 115 * --------------------------------- 116 * Destroys %eax, %ecx and %edx. 117 * Returns 1 if lock was successfull 118 */ 119 120 NON_GPROF_ENTRY(MPtrylock) 121 movl 4(%esp), %edx /* Get the address of the lock */ 122 123 movl $FREE_LOCK, %eax /* Assume it's free */ 124 movl _cpu_lockid, %ecx /* - get pre-shifted logical cpu id */ 125 incl %ecx /* - new count is one */ 126 lock 127 cmpxchg %ecx, (%edx) /* - try it atomically */ 128 jne 1f /* ...do not collect $200 */ 129 #ifdef GLPROFILE 130 incl _tryhits2 131 #endif /* GLPROFILE */ 132 GRAB_HWI /* 1st acquire, grab hw INTs */ 133 movl $1, %eax 134 ret 135 1: 136 movl (%edx), %eax /* Try to see if we have it already */ 137 andl $COUNT_FIELD, %eax /* - get count */ 138 movl _cpu_lockid, %ecx /* - get pre-shifted logical cpu id */ 139 orl %ecx, %eax /* - combine them */ 140 movl %eax, %ecx 141 incl %ecx /* - new count is one more */ 142 lock 143 cmpxchg %ecx, (%edx) /* - try it atomically */ 144 jne 2f /* - miss */ 145 #ifdef GLPROFILE 146 incl _tryhits 147 #endif /* GLPROFILE */ 148 movl $1, %eax 149 ret 150 2: 151 #ifdef GLPROFILE 152 incl _tryhits3 153 #endif /* GLPROFILE */ 154 movl $0, %eax 155 ret 156 157 158 /*********************************************************************** 159 * void MPrellock_edx(unsigned int *lock : %edx) 160 * ---------------------------------- 161 * Destroys %ecx, argument must be in %edx 162 * 163 * SERIALIZATION NOTE! 164 * 165 * After a lot of arguing, it turns out that there is no problem with 166 * not having a synchronizing instruction in the MP unlock code. There 167 * are two things to keep in mind: First, Intel guarentees that writes 168 * are ordered amoungst themselves. Second, the P6 is allowed to reorder 169 * reads around writes. Third, the P6 maintains cache consistency (snoops 170 * the bus). The second is not an issue since the one read we do is the 171 * basis for the conditional which determines whether the write will be 172 * made or not. 173 * 174 * Therefore, no synchronizing instruction is required on unlock. There are 175 * three performance cases: First, if a single cpu is getting and releasing 176 * the lock the removal of the synchronizing instruction saves approx 177 * 200 nS (testing w/ duel cpu PIII 450). Second, if one cpu is contending 178 * for the lock while the other holds it, the removal of the synchronizing 179 * instruction results in a 700nS LOSS in performance. Third, if two cpu's 180 * are switching off ownership of the MP lock but not contending for it (the 181 * most common case), this results in a 400nS IMPROVEMENT in performance. 182 * 183 * Since our goal is to reduce lock contention in the first place, we have 184 * decided to remove the synchronizing instruction from the unlock code. 185 */ 186 187 NON_GPROF_ENTRY(MPrellock_edx) 188 movl (%edx), %ecx /* - get the value */ 189 decl %ecx /* - new count is one less */ 190 testl $COUNT_FIELD, %ecx /* - Unless it's zero... */ 191 jnz 2f 192 ARB_HWI /* last release, arbitrate hw INTs */ 193 movl $FREE_LOCK, %ecx /* - In which case we release it */ 194 #if 0 195 lock 196 addl $0,0(%esp) /* see note above */ 197 #endif 198 2: 199 movl %ecx, (%edx) 200 ret 201 202 /*********************************************************************** 203 * void get_mplock() 204 * ----------------- 205 * All registers preserved 206 * 207 * Stack (after call to _MPgetlock): 208 * 209 * edx 4(%esp) 210 * ecx 8(%esp) 211 * eax 12(%esp) 212 * 213 * Requirements: Interrupts should be enabled on call so we can take 214 * IPI's and FAST INTs while we are waiting for the lock 215 * (else the system may not be able to halt). 216 * 217 * XXX there are still places where get_mplock() is called 218 * with interrupts disabled, so we have to temporarily reenable 219 * interrupts. 220 * 221 * Side effects: The current cpu will be given ownership of the 222 * hardware interrupts when it first aquires the lock. 223 * 224 * Costs: Initial aquisition requires the use of a costly locked 225 * instruction, but recursive aquisition is cheap. Release 226 * is very cheap. 227 */ 228 229 NON_GPROF_ENTRY(get_mplock) 230 pushl %eax 231 pushl %ecx 232 pushl %edx 233 movl $_mp_lock, %edx 234 pushfl 235 testl $(1<<9), (%esp) 236 jz 2f 237 call _MPgetlock_edx 238 addl $4,%esp 239 1: 240 popl %edx 241 popl %ecx 242 popl %eax 243 ret 244 2: 245 sti 246 call _MPgetlock_edx 247 popfl 248 jmp 1b 249 250 /* 251 * Special version of get_mplock that is used during bootstrap when we can't 252 * yet enable interrupts of any sort since the APIC isn't online yet. We 253 * do an endrun around MPgetlock_edx to avoid enabling interrupts. 254 * 255 * XXX FIXME.. - APIC should be online from the start to simplify IPI's. 256 */ 257 NON_GPROF_ENTRY(boot_get_mplock) 258 pushl %eax 259 pushl %ecx 260 pushl %edx 261 #ifdef GRAB_LOPRIO 262 pushfl 263 pushl lapic_tpr 264 cli 265 #endif 266 267 movl $_mp_lock, %edx 268 call _MPgetlock_edx 269 270 #ifdef GRAB_LOPRIO 271 popl lapic_tpr 272 popfl 273 #endif 274 popl %edx 275 popl %ecx 276 popl %eax 277 ret 278 279 /*********************************************************************** 280 * void try_mplock() 281 * ----------------- 282 * reg %eax == 1 if success 283 */ 284 285 NON_GPROF_ENTRY(try_mplock) 286 pushl %ecx 287 pushl %edx 288 pushl $_mp_lock 289 call _MPtrylock 290 add $4, %esp 291 popl %edx 292 popl %ecx 293 ret 294 295 /*********************************************************************** 296 * void rel_mplock() 297 * ----------------- 298 * All registers preserved 299 */ 300 301 NON_GPROF_ENTRY(rel_mplock) 302 pushl %ecx 303 pushl %edx 304 movl $_mp_lock,%edx 305 call _MPrellock_edx 306 popl %edx 307 popl %ecx 308 ret 309 310 #endif 311 312 /*********************************************************************** 313 * 314 */ 315 .data 316 .p2align 2 /* xx_lock aligned on int boundary */ 317 318 #ifdef SMP 319 320 .globl _mp_lock 321 _mp_lock: .long 0 322 323 #ifdef GLPROFILE 324 .globl _gethits 325 _gethits: 326 .long 0 327 _gethits2: 328 .long 0 329 _gethits3: 330 .long 0 331 332 .globl _tryhits 333 _tryhits: 334 .long 0 335 _tryhits2: 336 .long 0 337 _tryhits3: 338 .long 0 339 340 msg: 341 .asciz "lock hits: 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x\n" 342 #endif /* GLPROFILE */ 343 #endif /* SMP */
Cache object: 9c930b09ffbe9359d0d1e9cfd1964726
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]