Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/i386/i386/swtch.s
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 1990 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * William Jolitz. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by the University of 19 * California, Berkeley and its contributors. 20 * 4. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * $FreeBSD$ 37 */ 38 39 #include "npx.h" 40 #include "opt_user_ldt.h" 41 42 #include <sys/rtprio.h> 43 44 #include <machine/asmacros.h> 45 #include <machine/ipl.h> 46 47 #ifdef SMP 48 #include <machine/pmap.h> 49 #include <machine/smptests.h> /** GRAB_LOPRIO */ 50 #include <machine/apic.h> 51 #include <machine/lock.h> 52 #endif /* SMP */ 53 54 #include "assym.s" 55 56 57 /*****************************************************************************/ 58 /* Scheduling */ 59 /*****************************************************************************/ 60 61 .data 62 63 .globl _hlt_vector 64 _hlt_vector: .long _cpu_idle /* pointer to halt routine */ 65 66 .globl _panic 67 68 #if defined(SWTCH_OPTIM_STATS) 69 .globl _swtch_optim_stats, _tlb_flush_count 70 _swtch_optim_stats: .long 0 /* number of _swtch_optims */ 71 _tlb_flush_count: .long 0 72 #endif 73 74 .text 75 76 /* 77 * When no processes are on the runq, cpu_switch() branches to _idle 78 * to wait for something to come ready. 79 */ 80 ALIGN_TEXT 81 .type _idle,@function 82 _idle: 83 xorl %ebp,%ebp 84 movl %ebp,_switchtime 85 86 #ifdef SMP 87 88 /* when called, we have the mplock, intr disabled */ 89 /* use our idleproc's "context" */ 90 #ifdef PAE 91 movl $_IdlePDPT-KERNBASE, %ecx 92 #else 93 movl _IdlePTD, %ecx 94 #endif 95 movl %cr3, %eax 96 cmpl %ecx, %eax 97 je 2f 98 #if defined(SWTCH_OPTIM_STATS) 99 decl _swtch_optim_stats 100 incl _tlb_flush_count 101 #endif 102 movl %ecx, %cr3 103 2: 104 /* Keep space for nonexisting return addr, or profiling bombs */ 105 movl $gd_idlestack_top-4, %ecx 106 addl %fs:0, %ecx 107 movl %ecx, %esp 108 109 /* update common_tss.tss_esp0 pointer */ 110 movl %ecx, _common_tss + TSS_ESP0 111 112 movl _cpuid, %esi 113 btrl %esi, _private_tss 114 jae 1f 115 116 movl $gd_common_tssd, %edi 117 addl %fs:0, %edi 118 119 /* move correct tss descriptor into GDT slot, then reload tr */ 120 movl _tss_gdt, %ebx /* entry in GDT */ 121 movl 0(%edi), %eax 122 movl %eax, 0(%ebx) 123 movl 4(%edi), %eax 124 movl %eax, 4(%ebx) 125 movl $GPROC0_SEL*8, %esi /* GSEL(entry, SEL_KPL) */ 126 ltr %si 127 1: 128 129 sti 130 131 /* 132 * XXX callers of cpu_switch() do a bogus splclock(). Locking should 133 * be left to cpu_switch(). 134 * 135 * NOTE: spl*() may only be called while we hold the MP lock (which 136 * we do). 137 */ 138 call _spl0 139 140 cli 141 142 /* 143 * _REALLY_ free the lock, no matter how deep the prior nesting. 144 * We will recover the nesting on the way out when we have a new 145 * proc to load. 146 * 147 * XXX: we had damn well better be sure we had it before doing this! 148 */ 149 movl $FREE_LOCK, %eax 150 movl %eax, _mp_lock 151 152 /* do NOT have lock, intrs disabled */ 153 .globl idle_loop 154 idle_loop: 155 156 cmpl $0,_smp_active 157 jne 1f 158 cmpl $0,_cpuid 159 je 1f 160 jmp 2f 161 162 1: 163 call _procrunnable 164 testl %eax,%eax 165 jnz 3f 166 167 /* 168 * Handle page-zeroing in the idle loop. Called with interrupts 169 * disabled and the MP lock released. Inside vm_page_zero_idle 170 * we enable interrupts and grab the mplock as required. 171 */ 172 cmpl $0,_do_page_zero_idle 173 je 2f 174 175 call _vm_page_zero_idle /* internal locking */ 176 testl %eax, %eax 177 jnz idle_loop 178 2: 179 180 /* enable intrs for a halt */ 181 movl $0, lapic_tpr /* 1st candidate for an INT */ 182 call *_hlt_vector /* wait for interrupt */ 183 cli 184 jmp idle_loop 185 186 /* 187 * Note that interrupts must be enabled while obtaining the MP lock 188 * in order to be able to take IPI's while blocked. 189 */ 190 3: 191 #ifdef GRAB_LOPRIO 192 movl $LOPRIO_LEVEL, lapic_tpr /* arbitrate for INTs */ 193 #endif 194 sti 195 call _get_mplock 196 cli 197 call _procrunnable 198 testl %eax,%eax 199 CROSSJUMP(jnz, sw1a, jz) 200 call _rel_mplock 201 jmp idle_loop 202 203 #else /* !SMP */ 204 205 movl $tmpstk,%esp 206 #if defined(OVERLY_CONSERVATIVE_PTD_MGMT) 207 #if defined(SWTCH_OPTIM_STATS) 208 incl _swtch_optim_stats 209 #endif 210 #ifdef PAE 211 movl $_IdlePDPT-KERNBASE, %ecx 212 #else 213 movl _IdlePTD, %ecx 214 #endif 215 movl %cr3, %eax 216 cmpl %ecx, %eax 217 je 2f 218 #if defined(SWTCH_OPTIM_STATS) 219 decl _swtch_optim_stats 220 incl _tlb_flush_count 221 #endif 222 movl %ecx, %cr3 223 2: 224 #endif 225 226 /* update common_tss.tss_esp0 pointer */ 227 movl %esp, _common_tss + TSS_ESP0 228 229 movl $0, %esi 230 btrl %esi, _private_tss 231 jae 1f 232 233 movl $_common_tssd, %edi 234 235 /* move correct tss descriptor into GDT slot, then reload tr */ 236 movl _tss_gdt, %ebx /* entry in GDT */ 237 movl 0(%edi), %eax 238 movl %eax, 0(%ebx) 239 movl 4(%edi), %eax 240 movl %eax, 4(%ebx) 241 movl $GPROC0_SEL*8, %esi /* GSEL(entry, SEL_KPL) */ 242 ltr %si 243 1: 244 245 sti 246 247 /* 248 * XXX callers of cpu_switch() do a bogus splclock(). Locking should 249 * be left to cpu_switch(). 250 */ 251 call _spl0 252 253 ALIGN_TEXT 254 idle_loop: 255 cli 256 call _procrunnable 257 testl %eax,%eax 258 CROSSJUMP(jnz, sw1a, jz) 259 #ifdef DEVICE_POLLING 260 call _idle_poll 261 #else /* standard code */ 262 call _vm_page_zero_idle 263 #endif 264 testl %eax, %eax 265 jnz idle_loop 266 call *_hlt_vector /* wait for interrupt */ 267 jmp idle_loop 268 269 #endif /* SMP */ 270 271 CROSSJUMPTARGET(_idle) 272 273 #if 0 274 275 ENTRY(default_halt) 276 sti 277 #ifndef SMP 278 hlt /* XXX: until a wakeup IPI */ 279 #endif 280 ret 281 282 #endif 283 284 /* 285 * cpu_switch() 286 */ 287 ENTRY(cpu_switch) 288 289 /* switch to new process. first, save context as needed */ 290 movl _curproc,%ecx 291 292 /* if no process to save, don't bother */ 293 testl %ecx,%ecx 294 je sw1 295 296 #ifdef SMP 297 movb P_ONCPU(%ecx), %al /* save "last" cpu */ 298 movb %al, P_LASTCPU(%ecx) 299 movb $0xff, P_ONCPU(%ecx) /* "leave" the cpu */ 300 #endif /* SMP */ 301 movl P_VMSPACE(%ecx), %edx 302 #ifdef SMP 303 movl _cpuid, %eax 304 #else 305 xorl %eax, %eax 306 #endif /* SMP */ 307 btrl %eax, VM_PMAP+PM_ACTIVE(%edx) 308 309 movl P_ADDR(%ecx),%edx 310 311 movl (%esp),%eax /* Hardware registers */ 312 movl %eax,PCB_EIP(%edx) 313 movl %ebx,PCB_EBX(%edx) 314 movl %esp,PCB_ESP(%edx) 315 movl %ebp,PCB_EBP(%edx) 316 movl %esi,PCB_ESI(%edx) 317 movl %edi,PCB_EDI(%edx) 318 movl %gs,PCB_GS(%edx) 319 320 /* test if debug regisers should be saved */ 321 movb PCB_FLAGS(%edx),%al 322 andb $PCB_DBREGS,%al 323 jz 1f /* no, skip over */ 324 movl %dr7,%eax /* yes, do the save */ 325 movl %eax,PCB_DR7(%edx) 326 andl $0x0000fc00, %eax /* disable all watchpoints */ 327 movl %eax,%dr7 328 movl %dr6,%eax 329 movl %eax,PCB_DR6(%edx) 330 movl %dr3,%eax 331 movl %eax,PCB_DR3(%edx) 332 movl %dr2,%eax 333 movl %eax,PCB_DR2(%edx) 334 movl %dr1,%eax 335 movl %eax,PCB_DR1(%edx) 336 movl %dr0,%eax 337 movl %eax,PCB_DR0(%edx) 338 1: 339 340 #ifdef SMP 341 movl _mp_lock, %eax 342 /* XXX FIXME: we should be saving the local APIC TPR */ 343 #ifdef DIAGNOSTIC 344 cmpl $FREE_LOCK, %eax /* is it free? */ 345 je badsw4 /* yes, bad medicine! */ 346 #endif /* DIAGNOSTIC */ 347 andl $COUNT_FIELD, %eax /* clear CPU portion */ 348 movl %eax, PCB_MPNEST(%edx) /* store it */ 349 #endif /* SMP */ 350 351 #if NNPX > 0 352 /* have we used fp, and need a save? */ 353 cmpl %ecx,_npxproc 354 jne 1f 355 addl $PCB_SAVEFPU,%edx /* h/w bugs make saving complicated */ 356 pushl %edx 357 call _npxsave /* do it in a big C function */ 358 popl %eax 359 1: 360 #endif /* NNPX > 0 */ 361 362 movl $0,_curproc /* out of process */ 363 364 /* save is done, now choose a new process or idle */ 365 sw1: 366 cli 367 368 #ifdef SMP 369 /* Stop scheduling if smp_active goes zero and we are not BSP */ 370 cmpl $0,_smp_active 371 jne 1f 372 cmpl $0,_cpuid 373 CROSSJUMP(je, _idle, jne) /* wind down */ 374 1: 375 #endif 376 377 sw1a: 378 call _chooseproc /* trash ecx, edx, ret eax*/ 379 testl %eax,%eax 380 CROSSJUMP(je, _idle, jne) /* if no proc, idle */ 381 movl %eax,%ecx 382 383 xorl %eax,%eax 384 andl $~AST_RESCHED,_astpending 385 386 #ifdef DIAGNOSTIC 387 cmpl %eax,P_WCHAN(%ecx) 388 jne badsw1 389 cmpb $SRUN,P_STAT(%ecx) 390 jne badsw2 391 #endif 392 393 movl P_ADDR(%ecx),%edx 394 395 #if defined(SWTCH_OPTIM_STATS) 396 incl _swtch_optim_stats 397 #endif 398 /* switch address space */ 399 movl %cr3,%ebx 400 cmpl PCB_CR3(%edx),%ebx 401 je 4f 402 #if defined(SWTCH_OPTIM_STATS) 403 decl _swtch_optim_stats 404 incl _tlb_flush_count 405 #endif 406 movl PCB_CR3(%edx),%ebx 407 movl %ebx,%cr3 408 4: 409 410 #ifdef SMP 411 movl _cpuid, %esi 412 #else 413 xorl %esi, %esi 414 #endif 415 cmpl $0, PCB_EXT(%edx) /* has pcb extension? */ 416 je 1f 417 btsl %esi, _private_tss /* mark use of private tss */ 418 movl PCB_EXT(%edx), %edi /* new tss descriptor */ 419 jmp 2f 420 1: 421 422 /* update common_tss.tss_esp0 pointer */ 423 movl %edx, %ebx /* pcb */ 424 addl $(UPAGES * PAGE_SIZE - 16), %ebx 425 movl %ebx, _common_tss + TSS_ESP0 426 427 btrl %esi, _private_tss 428 jae 3f 429 #ifdef SMP 430 movl $gd_common_tssd, %edi 431 addl %fs:0, %edi 432 #else 433 movl $_common_tssd, %edi 434 #endif 435 2: 436 /* move correct tss descriptor into GDT slot, then reload tr */ 437 movl _tss_gdt, %ebx /* entry in GDT */ 438 movl 0(%edi), %eax 439 movl %eax, 0(%ebx) 440 movl 4(%edi), %eax 441 movl %eax, 4(%ebx) 442 movl $GPROC0_SEL*8, %esi /* GSEL(entry, SEL_KPL) */ 443 ltr %si 444 3: 445 movl P_VMSPACE(%ecx), %ebx 446 #ifdef SMP 447 movl _cpuid, %eax 448 #else 449 xorl %eax, %eax 450 #endif 451 btsl %eax, VM_PMAP+PM_ACTIVE(%ebx) 452 453 /* restore context */ 454 movl PCB_EBX(%edx),%ebx 455 movl PCB_ESP(%edx),%esp 456 movl PCB_EBP(%edx),%ebp 457 movl PCB_ESI(%edx),%esi 458 movl PCB_EDI(%edx),%edi 459 movl PCB_EIP(%edx),%eax 460 movl %eax,(%esp) 461 462 #ifdef SMP 463 #ifdef GRAB_LOPRIO /* hold LOPRIO for INTs */ 464 #ifdef CHEAP_TPR 465 movl $0, lapic_tpr 466 #else 467 andl $~APIC_TPR_PRIO, lapic_tpr 468 #endif /** CHEAP_TPR */ 469 #endif /** GRAB_LOPRIO */ 470 movl _cpuid,%eax 471 movb %al, P_ONCPU(%ecx) 472 #endif /* SMP */ 473 movl %edx, _curpcb 474 movl %ecx, _curproc /* into next process */ 475 476 #ifdef SMP 477 movl _cpu_lockid, %eax 478 orl PCB_MPNEST(%edx), %eax /* add next count from PROC */ 479 movl %eax, _mp_lock /* load the mp_lock */ 480 /* XXX FIXME: we should be restoring the local APIC TPR */ 481 #endif /* SMP */ 482 483 #ifdef USER_LDT 484 cmpl $0, PCB_USERLDT(%edx) 485 jnz 1f 486 movl __default_ldt,%eax 487 cmpl _currentldt,%eax 488 je 2f 489 lldt __default_ldt 490 movl %eax,_currentldt 491 jmp 2f 492 1: pushl %edx 493 call _set_user_ldt 494 popl %edx 495 2: 496 #endif 497 498 /* This must be done after loading the user LDT. */ 499 .globl cpu_switch_load_gs 500 cpu_switch_load_gs: 501 movl PCB_GS(%edx),%gs 502 503 /* test if debug regisers should be restored */ 504 movb PCB_FLAGS(%edx),%al 505 andb $PCB_DBREGS,%al 506 jz 1f /* no, skip over */ 507 movl PCB_DR6(%edx),%eax /* yes, do the restore */ 508 movl %eax,%dr6 509 movl PCB_DR3(%edx),%eax 510 movl %eax,%dr3 511 movl PCB_DR2(%edx),%eax 512 movl %eax,%dr2 513 movl PCB_DR1(%edx),%eax 514 movl %eax,%dr1 515 movl PCB_DR0(%edx),%eax 516 movl %eax,%dr0 517 movl %dr7,%eax /* load dr7 so as not to disturb */ 518 andl $0x0000fc00,%eax /* reserved bits */ 519 pushl %ebx 520 movl PCB_DR7(%edx),%ebx 521 andl $~0x0000fc00,%ebx 522 orl %ebx,%eax 523 popl %ebx 524 movl %eax,%dr7 525 1: 526 527 sti 528 ret 529 530 CROSSJUMPTARGET(sw1a) 531 532 #ifdef DIAGNOSTIC 533 badsw1: 534 pushl $sw0_1 535 call _panic 536 537 sw0_1: .asciz "cpu_switch: has wchan" 538 539 badsw2: 540 pushl $sw0_2 541 call _panic 542 543 sw0_2: .asciz "cpu_switch: not SRUN" 544 #endif 545 546 #if defined(SMP) && defined(DIAGNOSTIC) 547 badsw4: 548 pushl $sw0_4 549 call _panic 550 551 sw0_4: .asciz "cpu_switch: do not have lock" 552 #endif /* SMP && DIAGNOSTIC */ 553 554 /* 555 * savectx(pcb) 556 * Update pcb, saving current processor state. 557 */ 558 ENTRY(savectx) 559 /* fetch PCB */ 560 movl 4(%esp),%ecx 561 562 /* caller's return address - child won't execute this routine */ 563 movl (%esp),%eax 564 movl %eax,PCB_EIP(%ecx) 565 566 movl %cr3,%eax 567 movl %eax,PCB_CR3(%ecx) 568 569 movl %ebx,PCB_EBX(%ecx) 570 movl %esp,PCB_ESP(%ecx) 571 movl %ebp,PCB_EBP(%ecx) 572 movl %esi,PCB_ESI(%ecx) 573 movl %edi,PCB_EDI(%ecx) 574 movl %gs,PCB_GS(%ecx) 575 576 #if NNPX > 0 577 /* 578 * If npxproc == NULL, then the npx h/w state is irrelevant and the 579 * state had better already be in the pcb. This is true for forks 580 * but not for dumps (the old book-keeping with FP flags in the pcb 581 * always lost for dumps because the dump pcb has 0 flags). 582 * 583 * If npxproc != NULL, then we have to save the npx h/w state to 584 * npxproc's pcb and copy it to the requested pcb, or save to the 585 * requested pcb and reload. Copying is easier because we would 586 * have to handle h/w bugs for reloading. We used to lose the 587 * parent's npx state for forks by forgetting to reload. 588 */ 589 movl _npxproc,%eax 590 testl %eax,%eax 591 je 1f 592 593 pushl %ecx 594 movl P_ADDR(%eax),%eax 595 leal PCB_SAVEFPU(%eax),%eax 596 pushl %eax 597 pushl %eax 598 call _npxsave 599 addl $4,%esp 600 popl %eax 601 popl %ecx 602 603 pushl $PCB_SAVEFPU_SIZE 604 leal PCB_SAVEFPU(%ecx),%ecx 605 pushl %ecx 606 pushl %eax 607 call _bcopy 608 addl $12,%esp 609 #endif /* NNPX > 0 */ 610 611 1: 612 ret
Cache object: 99ceb7b1508c03fbdeb8d8ac5ecd06ab
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]