FreeBSD/Linux Kernel Cross Reference
sys/kern/thread.c
1 /*
2 * Mach Operating System
3 * Copyright (c) 1993-1987 Carnegie Mellon University
4 * All Rights Reserved.
5 *
6 * Permission to use, copy, modify and distribute this software and its
7 * documentation is hereby granted, provided that both the copyright
8 * notice and this permission notice appear in all copies of the
9 * software, derivative works or modified versions, and any portions
10 * thereof, and that both notices appear in supporting documentation.
11 *
12 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
13 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
14 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
15 *
16 * Carnegie Mellon requests users of this software to return to
17 *
18 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
19 * School of Computer Science
20 * Carnegie Mellon University
21 * Pittsburgh PA 15213-3890
22 *
23 * any improvements or extensions that they make and grant Carnegie Mellon
24 * the rights to redistribute these changes.
25 */
26 /*
27 * HISTORY
28 * $Log: thread.c,v $
29 * Revision 2.33 93/08/10 15:11:43 mrt
30 * Conditionalized atm hooks.
31 * [93/07/30 cmaeda]
32 * Included hooks for network interface.
33 * [93/06/09 15:44:22 jcb]
34 *
35 * Revision 2.32 93/05/15 18:55:33 mrt
36 * machparam.h -> machspl.h
37 *
38 * Revision 2.31 93/01/24 13:20:13 danner
39 * Add call to evc_notify_abort to thread_abort; correct call in
40 * thread_terminate.
41 * [93/01/22 danner]
42 *
43 * We must explicitly set "new_thread->pc_sample.buffer = 0;" so
44 * that we don't think we have a sampling buffer.
45 * [93/01/13 rvb]
46 *
47 * Revision 2.30 93/01/21 12:22:49 danner
48 * Add call in thread_deallocate to evc_notify_thread_destroy to
49 * deal with threads terminating while in evc_wait.
50 * [93/01/20 bershad]
51 *
52 * Revision 2.29 93/01/14 17:36:55 danner
53 * Added ANSI function prototypes.
54 * [92/12/29 dbg]
55 *
56 * Changed thread_create to hold both the pset and task locks while
57 * inserting the new thread in the lists.
58 * [92/11/20 dbg]
59 *
60 * Fixed pset lock ordering. Pset lock must be taken before task
61 * and thread locks.
62 * [92/10/28 dbg]
63 * 64bit cleanup. Proper spl typing.
64 * [92/12/01 af]
65 *
66 * Changed thread_create to hold both the pset and task locks while
67 * inserting the new thread in the lists.
68 * [92/11/20 dbg]
69 *
70 * Fixed pset lock ordering. Pset lock must be taken before task
71 * and thread locks.
72 * [92/10/28 dbg]
73 *
74 * Revision 2.28 92/08/03 17:39:56 jfriedl
75 * removed silly prototypes
76 * [92/08/02 jfriedl]
77 *
78 * Revision 2.27 92/05/21 17:16:33 jfriedl
79 * Appended 'U' to constants that would otherwise be signed.
80 * Changed name of one of the two local 'thread' variables in
81 * processor_set_stack_usage() to 'tmp_thread' for cleanness.
82 * [92/05/16 jfriedl]
83 *
84 * Revision 2.26 92/04/01 19:33:34 rpd
85 * Restored continuation in AST_TERMINATE case of thread_halt_self.
86 * [92/03/22 rpd]
87 *
88 * Revision 2.25 92/03/10 16:24:22 jsb
89 * Remove continuation from AST_TERMINATE case of thread_halt_self
90 * so that "zombie walks" is reachable.
91 * [92/02/25 dlb]
92 *
93 * Revision 2.24 92/02/19 16:07:03 elf
94 * Change calls to compute_priority.
95 * [92/01/19 rwd]
96 *
97 * Revision 2.23 92/01/03 20:18:56 dbg
98 * Make thread_wire really reserve a kernel stack.
99 * [91/12/18 dbg]
100 *
101 * Revision 2.22 91/12/13 14:54:53 jsb
102 * Removed thread_resume_from_kernel.
103 * [91/12/12 17:40:12 af]
104 *
105 * Revision 2.21 91/08/28 11:14:43 jsb
106 * Fixed thread_halt, mach_msg_interrupt interaction.
107 * Added checks for thread_exception_return, thread_bootstrap_return.
108 * [91/08/03 rpd]
109 *
110 * Revision 2.20 91/07/31 17:49:10 dbg
111 * Call pcb_module_init from thread_init.
112 * [91/07/26 dbg]
113 *
114 * When halting a thread: if it is waiting at a continuation with a
115 * known cleanup routine, call the cleanup routine instead of
116 * resuming the thread.
117 * [91/06/21 dbg]
118 *
119 * Revise scheduling state machine.
120 * [91/05/22 dbg]
121 *
122 * Add thread_wire.
123 * [91/05/14 dbg]
124 *
125 * Revision 2.19 91/06/25 10:29:48 rpd
126 * Picked up dlb's thread_doassign no-op fix.
127 * [91/06/23 rpd]
128 *
129 * Revision 2.18 91/05/18 14:34:09 rpd
130 * Fixed stack_alloc to use kmem_alloc_aligned.
131 * [91/05/14 rpd]
132 * Added argument to kernel_thread.
133 * [91/04/03 rpd]
134 *
135 * Changed thread_deallocate to reset timer and depress_timer.
136 * [91/03/31 rpd]
137 *
138 * Replaced stack_free_reserved and swap_privilege with stack_privilege.
139 * [91/03/30 rpd]
140 *
141 * Revision 2.17 91/05/14 16:48:35 mrt
142 * Correcting copyright
143 *
144 * Revision 2.16 91/05/08 12:49:14 dbg
145 * Add volatile declarations.
146 * [91/04/26 14:44:13 dbg]
147 *
148 * Revision 2.15 91/03/16 14:52:40 rpd
149 * Fixed the initialization of stack_lock_data.
150 * [91/03/11 rpd]
151 * Updated for new kmem_alloc interface.
152 * [91/03/03 rpd]
153 * Removed ith_saved.
154 * [91/02/16 rpd]
155 *
156 * Added stack_alloc_max.
157 * [91/02/10 rpd]
158 * Added active_stacks.
159 * [91/01/28 rpd]
160 * Can't use thread_dowait on the current thread now.
161 * Added reaper_thread_continue.
162 * Added stack_free_reserved.
163 * [91/01/20 rpd]
164 *
165 * Removed thread_swappable.
166 * Allow swapped threads on the run queues.
167 * Changed the AST interface.
168 * [91/01/17 rpd]
169 *
170 * Revision 2.14 91/02/05 17:30:14 mrt
171 * Changed to new Mach copyright
172 * [91/02/01 16:19:30 mrt]
173 *
174 * Revision 2.13 91/01/08 15:17:57 rpd
175 * Added KEEP_STACKS support.
176 * [91/01/06 rpd]
177 * Added consider_thread_collect, thread_collect_scan.
178 * [91/01/03 rpd]
179 *
180 * Added locking to the stack package. Added stack_collect.
181 * [90/12/31 rpd]
182 * Changed thread_dowait to discard the stacks of suspended threads.
183 * [90/12/22 rpd]
184 * Added continuation argument to thread_block.
185 * [90/12/08 rpd]
186 *
187 * Changed thread_create to make new threads be swapped.
188 * Changed kernel_thread to swapin the threads.
189 * [90/11/20 rpd]
190 *
191 * Removed stack_free/stack_alloc/etc.
192 * [90/11/12 rpd]
193 *
194 * Changed thread_create to let pcb_init handle all pcb initialization.
195 * [90/11/11 rpd]
196 *
197 * Revision 2.12 90/12/05 23:29:09 af
198 *
199 *
200 * Revision 2.11 90/12/05 20:42:19 af
201 * Added (temporarily, until we define the right new primitive with
202 * the RTMach guys) thread_resume_from_kernel() for internal kernel
203 * use only.
204 *
205 * Revision 2.7.1.1 90/09/25 13:06:04 dlb
206 * Inline thread_hold in thread_suspend and thread_release in
207 * thread_resume (while still holding the thread lock in both).
208 * This eliminates a long-standing MP bug.
209 * [90/09/20 dlb]
210 *
211 * Revision 2.10 90/11/05 14:31:46 rpd
212 * Unified untimeout and untimeout_try.
213 * [90/10/29 rpd]
214 *
215 * Revision 2.9 90/10/25 14:45:37 rwd
216 * Added host_stack_usage and processor_set_stack_usage.
217 * [90/10/22 rpd]
218 *
219 * Removed pausing code in stack_alloc/stack_free.
220 * It was broken and it isn't needed.
221 * Added code to check how much stack is actually used.
222 * [90/10/21 rpd]
223 *
224 * Revision 2.8 90/10/12 12:34:37 rpd
225 * Fixed HW_FOOTPRINT code in thread_create.
226 * Fixed a couple bugs in thread_policy.
227 * [90/10/09 rpd]
228 *
229 * Revision 2.7 90/08/27 22:04:03 dbg
230 * Fixed thread_info to return the correct count.
231 * [90/08/23 rpd]
232 *
233 * Revision 2.6 90/08/27 11:52:25 dbg
234 * Remove unneeded mode parameter to thread_start.
235 * Remove u_zone, thread_deallocate_interrupt.
236 * [90/07/17 dbg]
237 *
238 * Revision 2.5 90/08/07 17:59:04 rpd
239 * Removed tmp_address, tmp_object fields.
240 * Picked up depression abort functionality in thread_abort.
241 * Picked up initialization of max_priority in kernel_thread.
242 * Picked up revised priority computation in thread_create.
243 * Removed in-transit check from thread_max_priority.
244 * Picked up interprocessor-interrupt optimization in thread_dowait.
245 * [90/08/07 rpd]
246 *
247 * Revision 2.4 90/06/02 14:56:54 rpd
248 * Converted to new IPC and scheduling technology.
249 * [90/03/26 22:24:06 rpd]
250 *
251 * Revision 2.3 90/02/22 20:04:12 dbg
252 * Initialize per-thread global VM variables.
253 * Clean up global variables in thread_deallocate().
254 * [89/04/29 mwyoung]
255 *
256 * Revision 2.2 89/09/08 11:26:53 dbg
257 * Allocate thread kernel stack with kmem_alloc_wired to get
258 * separate vm_object for it.
259 * [89/08/18 dbg]
260 *
261 * 19-Aug-88 David Golub (dbg) at Carnegie-Mellon University
262 * Removed all non-MACH code.
263 *
264 * 11-Aug-88 David Black (dlb) at Carnegie-Mellon University
265 * Rewrote exit logic to use new ast mechanism. Includes locking
266 * bug fix to thread_terminate().
267 *
268 * 9-Aug-88 David Black (dlb) at Carnegie-Mellon University
269 * first_quantum replaces preempt_pri.
270 *
271 * Revision 2.3 88/08/06 18:26:41 rpd
272 * Changed to use ipc_thread_lock/ipc_thread_unlock macros.
273 * Eliminated use of kern/mach_ipc_defs.h.
274 * Added definitions of all_threads, all_threads_lock.
275 *
276 * 4-May-88 David Golub (dbg) and David Black (dlb) at CMU
277 * Remove vax-specific code. Add register declarations.
278 * MACH_TIME_NEW now standard. Moved thread_read_times to timer.c.
279 * SIMPLE_CLOCK: clock drift compensation in cpu_usage calculation.
280 * Initialize new fields in thread_create(). Implemented cpu usage
281 * calculation in thread_info(). Added argument to thread_setrun.
282 * Initialization changes for MACH_TIME_NEW.
283 *
284 * 13-Apr-88 David Black (dlb) at Carnegie-Mellon University
285 * Rewrite kernel stack retry code to eliminate races and handle
286 * termination correctly.
287 *
288 * 19-Feb-88 David Kirschen (kirschen) at Encore Computer Corporation
289 * Retry if kernel stacks exhausted on thread_create
290 *
291 * 12-Feb-88 David Black (dlb) at Carnegie-Mellon University
292 * Fix MACH_TIME_NEW code.
293 *
294 * 1-Feb-88 David Golub (dbg) at Carnegie-Mellon University
295 * In thread_halt: mark the victim thread suspended/runnable so that it
296 * will notify the caller when it hits the next interruptible wait.
297 * The victim may not immediately proceed to a clean point once it
298 * is awakened.
299 *
300 * 21-Jan-88 David Golub (dbg) at Carnegie-Mellon University
301 * Use new swapping state machine. Moved thread_swappable to
302 * thread_swap.c.
303 *
304 * 17-Jan-88 David Golub (dbg) at Carnegie-Mellon University
305 * Added new thread interfaces: thread_suspend, thread_resume,
306 * thread_get_state, thread_set_state, thread_abort,
307 * thread_info. Old interfaces remain (temporarily) for binary
308 * compatibility, prefixed with 'xxx_'.
309 *
310 * 29-Dec-87 David Golub (dbg) at Carnegie-Mellon University
311 * Delinted.
312 *
313 * 15-Dec-87 David Golub (dbg) at Carnegie-Mellon University
314 * Made thread_reference and thread_deallocate check for null
315 * threads. Call pcb_terminate when a thread is deallocated.
316 * Call pcb_init with thread pointer instead of pcb pointer.
317 * Add missing case to thread_dowait.
318 *
319 * 9-Dec-87 David Golub (dbg) at Carnegie-Mellon University
320 * Rewrote thread termination to have a terminating thread clean up
321 * after itself.
322 *
323 * 9-Dec-87 David Black (dlb) at Carnegie-Mellon University
324 * Moved reaper invocation to thread_terminate from
325 * thread_deallocate. [XXX temporary pending rewrite.]
326 *
327 * 8-Dec-87 David Black (dlb) at Carnegie-Mellon University
328 * Added call to ipc_thread_disable.
329 *
330 * 4-Dec-87 David Black (dlb) at Carnegie-Mellon University
331 * Set ipc_kernel in kernel_thread().
332 *
333 * 3-Dec-87 David Black (dlb) at Carnegie-Mellon University
334 * Rewrote thread_create(). thread_terminate() must throw away
335 * an extra reference if called on the current thread [ref is
336 * held by caller who will not be returned to.] Locking bug fix
337 * to thread_status.
338 *
339 * 19-Nov-87 Avadis Tevanian (avie) at Carnegie-Mellon University
340 * Eliminated TT conditionals.
341 *
342 * 30-Oct-87 David Golub (dbg) at Carnegie-Mellon University
343 * Fix race condition in thread_deallocate for thread terminating
344 * itself.
345 *
346 * 23-Oct-87 David Golub (dbg) at Carnegie-Mellon University
347 * Correctly set thread_statistics fields.
348 *
349 * 13-Oct-87 David Black (dlb) at Carnegie-Mellon University
350 * Use counts for suspend and resume primitives.
351 *
352 * 5-Oct-87 David Golub (dbg) at Carnegie-Mellon University
353 * MACH_TT: Completely replaced scheduling state machine.
354 *
355 * 30-Sep-87 Michael Young (mwyoung) at Carnegie-Mellon University
356 * Added initialization of thread->flags in thread_create().
357 * Added thread_swappable().
358 * De-linted.
359 *
360 * 30-Sep-87 David Black (dlb) at Carnegie-Mellon University
361 * Rewrote thread_dowait to more effectively stop threads.
362 *
363 * 11-Sep-87 Robert Baron (rvb) at Carnegie-Mellon University
364 * Initialize thread fields and unix_lock.
365 *
366 * 9-Sep-87 David Black (dlb) at Carnegie-Mellon University
367 * Changed thread_dowait to count a thread as stopped if it is
368 * sleeping and will stop immediately when woken up. [i.e. is
369 * sleeping interruptibly]. Corresponding change to
370 * thread_terminate().
371 *
372 * 4-Aug-87 David Golub (dbg) at Carnegie-Mellon University
373 * Moved ipc_thread_terminate to thread_terminate (from
374 * thread_deallocate), to shut out other threads that are
375 * manipulating the thread via its thread_port.
376 *
377 * 29-Jul-87 David Golub (dbg) at Carnegie-Mellon University
378 * Make sure all deallocation calls are outside of locks - they may
379 * block. Moved task_deallocate from thread_deallocate to
380 * thread_destroy, since thread may blow up if task disappears while
381 * thread is running.
382 *
383 * 26-Jun-87 David Black (dlb) at Carnegie-Mellon University
384 * Added update_priority() call to thread_release() for any thread
385 * actually released.
386 *
387 * 23-Jun-87 David Black (dlb) at Carnegie-Mellon University
388 * Initialize thread priorities in thread_create() and kernel_thread().
389 *
390 * 10-Jun-87 Karl Hauth (hauth) at Carnegie-Mellon University
391 * Added code to fill in the thread_statistics structure.
392 *
393 * 1-Jun-87 Avadis Tevanian (avie) at Carnegie-Mellon University
394 * Added thread_statistics stub.
395 *
396 * 21-May-87 Avadis Tevanian (avie) at Carnegie-Mellon University
397 * Clear the thread u-area upon creation of a thread to keep
398 * consistent.
399 *
400 * 4-May-87 Avadis Tevanian (avie) at Carnegie-Mellon University
401 * Call uarea_init to initialize u-area stuff.
402 *
403 * 29-Apr-87 Avadis Tevanian (avie) at Carnegie-Mellon University
404 * Moved call to ipc_thread_terminate into the MACH_TT only branch
405 * to prevent problems with non-TT systems.
406 *
407 * 28-Apr-87 Avadis Tevanian (avie) at Carnegie-Mellon University
408 * Support the thread status information as a MiG refarray.
409 * [NOTE: turned off since MiG is still too braindamaged.]
410 *
411 * 23-Apr-87 Rick Rashid (rfr) at Carnegie-Mellon University
412 * Moved ipc_thread_terminate to thread_deallocate from
413 * thread_destroy to eliminate having the reaper call it after
414 * the task has been freed.
415 *
416 * 18-Mar-87 Avadis Tevanian (avie) at Carnegie-Mellon University
417 * Added reaper thread for deallocating threads that cannot
418 * deallocate themselves (some time ago).
419 *
420 * 17-Mar-87 David Golub (dbg) at Carnegie-Mellon University
421 * De-linted.
422 *
423 * 14-Mar-87 Avadis Tevanian (avie) at Carnegie-Mellon University
424 * Panic if no space left in the kernel map for stacks.
425 *
426 * 6-Mar-87 Avadis Tevanian (avie) at Carnegie-Mellon University
427 * Add kernel_thread routine which starts up kernel threads.
428 *
429 * 4-Mar-87 Avadis Tevanian (avie) at Carnegie-Mellon University
430 * Make thread_terminate work.
431 *
432 * 2-Mar-87 Avadis Tevanian (avie) at Carnegie-Mellon University
433 * New kernel stack allocation mechanism.
434 *
435 * 27-Feb-87 David L. Black (dlb) at Carnegie-Mellon University
436 * MACH_TIME_NEW: Added timer inits to thread_create().
437 *
438 * 24-Feb-87 Avadis Tevanian (avie) at Carnegie-Mellon University
439 * Rewrote thread_suspend/thread_hold and added thread_wait for new
440 * user synchronization paradigm.
441 *
442 * 24-Feb-87 Avadis Tevanian (avie) at Carnegie-Mellon University
443 * Reorded locking protocol in thread_deallocate for the
444 * all_threads_lock (this allows one to reference a thread then
445 * release the all_threads_lock when scanning the thread list).
446 *
447 * 31-Jan-87 Avadis Tevanian (avie) at Carnegie-Mellon University
448 * Merged in my changes for real thread implementation.
449 *
450 * 30-Sep-86 Avadis Tevanian (avie) at Carnegie-Mellon University
451 * Make floating u-area work, maintain list of threads per task.
452 *
453 * 1-Aug-86 Michael Young (mwyoung) at Carnegie-Mellon University
454 * Added initialization for Mach-style IPC.
455 *
456 * 7-Jul-86 Rick Rashid (rfr) at Carnegie-Mellon University
457 * Added thread_in_use_chain to keep track of threads which
458 * have been created but not yet destroyed.
459 *
460 * 31-May-86 Avadis Tevanian (avie) at Carnegie-Mellon University
461 * Initialize thread state field to THREAD_WAITING. Some general
462 * cleanup.
463 *
464 */
465 /*
466 * File: kern/thread.c
467 * Author: Avadis Tevanian, Jr., Michael Wayne Young, David Golub
468 * Date: 1986
469 *
470 * Thread management primitives implementation.
471 */
472
473 #include <cpus.h>
474 #include <hw_footprint.h>
475 #include <mach_host.h>
476 #include <mach_fixpri.h>
477 #include <simple_clock.h>
478 #include <mach_debug.h>
479 #include <net_atm.h>
480
481 #include <mach/std_types.h>
482 #include <mach/policy.h>
483 #include <mach/thread_info.h>
484 #include <mach/thread_special_ports.h>
485 #include <mach/thread_status.h>
486 #include <mach/time_value.h>
487 #include <mach/vm_param.h>
488 #include <kern/ast.h>
489 #include <kern/counters.h>
490 #include <kern/ipc_tt.h>
491 #include <kern/mach_param.h>
492 #include <kern/processor.h>
493 #include <kern/queue.h>
494 #include <kern/sched.h>
495 #include <kern/sched_prim.h>
496 #include <kern/thread.h>
497 #include <kern/thread_swap.h>
498 #include <kern/host.h>
499 #include <kern/zalloc.h>
500 #include <vm/vm_kern.h>
501 #include <ipc/ipc_kmsg.h>
502 #include <ipc/ipc_port.h>
503 #include <ipc/mach_msg.h>
504 #include <machine/machspl.h> /* for splsched */
505 #include <machine/thread.h> /* for MACHINE_STACK */
506
507 #if NET_ATM
508 #include <chips/nw_mk.h>
509 #endif
510
511 thread_t active_threads[NCPUS];
512 vm_offset_t active_stacks[NCPUS];
513
514 struct zone *thread_zone;
515
516 queue_head_t reaper_queue;
517 decl_simple_lock_data(, reaper_lock)
518
519 extern int tick;
520
521 extern void pcb_module_init(void);
522
523 /* private */
524 struct thread thread_template;
525
526 #if MACH_DEBUG
527 void stack_init(vm_offset_t stack); /* forward */
528 void stack_finalize(vm_offset_t stack); /* forward */
529
530 #define STACK_MARKER 0xdeadbeefU
531 boolean_t stack_check_usage = FALSE;
532 decl_simple_lock_data(, stack_usage_lock)
533 vm_size_t stack_max_usage = 0;
534 #endif /* MACH_DEBUG */
535
536 /*
537 * Machine-dependent code must define:
538 * pcb_init
539 * pcb_terminate
540 * pcb_collect
541 *
542 * The thread->pcb field is reserved for machine-dependent code.
543 */
544
545 #ifdef MACHINE_STACK
546 /*
547 * Machine-dependent code must define:
548 * stack_alloc_try
549 * stack_alloc
550 * stack_free
551 * stack_handoff
552 * stack_collect
553 * and if MACH_DEBUG:
554 * stack_statistics
555 */
556 #else /* MACHINE_STACK */
557 /*
558 * We allocate stacks from generic kernel VM.
559 * Machine-dependent code must define:
560 * stack_attach
561 * stack_detach
562 * stack_handoff
563 *
564 * The stack_free_list can only be accessed at splsched,
565 * because stack_alloc_try/thread_invoke operate at splsched.
566 */
567
568 decl_simple_lock_data(, stack_lock_data)/* splsched only */
569 #define stack_lock() simple_lock(&stack_lock_data)
570 #define stack_unlock() simple_unlock(&stack_lock_data)
571
572 vm_offset_t stack_free_list; /* splsched only */
573 unsigned int stack_free_count = 0; /* splsched only */
574 unsigned int stack_free_limit = 1; /* patchable */
575
576 unsigned int stack_alloc_hits = 0; /* debugging */
577 unsigned int stack_alloc_misses = 0; /* debugging */
578 unsigned int stack_alloc_max = 0; /* debugging */
579
580 /*
581 * The next field is at the base of the stack,
582 * so the low end is left unsullied.
583 */
584
585 #define stack_next(stack) (*((vm_offset_t *)((stack) + KERNEL_STACK_SIZE) - 1))
586
587 /*
588 * stack_alloc_try:
589 *
590 * Non-blocking attempt to allocate a kernel stack.
591 * Called at splsched with the thread locked.
592 */
593
594 boolean_t stack_alloc_try(
595 thread_t thread,
596 void (*resume)(thread_t))
597 {
598 register vm_offset_t stack;
599
600 stack_lock();
601 stack = stack_free_list;
602 if (stack != 0) {
603 stack_free_list = stack_next(stack);
604 stack_free_count--;
605 } else {
606 stack = thread->stack_privilege;
607 }
608 stack_unlock();
609
610 if (stack != 0) {
611 stack_attach(thread, stack, resume);
612 stack_alloc_hits++;
613 return TRUE;
614 } else {
615 stack_alloc_misses++;
616 return FALSE;
617 }
618 }
619
620 /*
621 * stack_alloc:
622 *
623 * Allocate a kernel stack for a thread.
624 * May block.
625 */
626
627 void stack_alloc(
628 thread_t thread,
629 void (*resume)(thread_t))
630 {
631 vm_offset_t stack;
632 spl_t s;
633
634 /*
635 * We first try the free list. It is probably empty,
636 * or stack_alloc_try would have succeeded, but possibly
637 * a stack was freed before the swapin thread got to us.
638 */
639
640 s = splsched();
641 stack_lock();
642 stack = stack_free_list;
643 if (stack != 0) {
644 stack_free_list = stack_next(stack);
645 stack_free_count--;
646 }
647 stack_unlock();
648 (void) splx(s);
649
650 if (stack == 0) {
651 /*
652 * Kernel stacks should be naturally aligned,
653 * so that it is easy to find the starting/ending
654 * addresses of a stack given an address in the middle.
655 */
656
657 if (kmem_alloc_aligned(kernel_map, &stack, KERNEL_STACK_SIZE)
658 != KERN_SUCCESS)
659 panic("stack_alloc");
660
661 #if MACH_DEBUG
662 stack_init(stack);
663 #endif /* MACH_DEBUG */
664 }
665
666 stack_attach(thread, stack, resume);
667 }
668
669 /*
670 * stack_free:
671 *
672 * Free a thread's kernel stack.
673 * Called at splsched with the thread locked.
674 */
675
676 void stack_free(
677 thread_t thread)
678 {
679 register vm_offset_t stack;
680
681 stack = stack_detach(thread);
682
683 if (stack != thread->stack_privilege) {
684 stack_lock();
685 stack_next(stack) = stack_free_list;
686 stack_free_list = stack;
687 if (++stack_free_count > stack_alloc_max)
688 stack_alloc_max = stack_free_count;
689 stack_unlock();
690 }
691 }
692
693 /*
694 * stack_collect:
695 *
696 * Free excess kernel stacks.
697 * May block.
698 */
699
700 void stack_collect(void)
701 {
702 register vm_offset_t stack;
703 spl_t s;
704
705 s = splsched();
706 stack_lock();
707 while (stack_free_count > stack_free_limit) {
708 stack = stack_free_list;
709 stack_free_list = stack_next(stack);
710 stack_free_count--;
711 stack_unlock();
712 (void) splx(s);
713
714 #if MACH_DEBUG
715 stack_finalize(stack);
716 #endif /* MACH_DEBUG */
717 kmem_free(kernel_map, stack, KERNEL_STACK_SIZE);
718
719 s = splsched();
720 stack_lock();
721 }
722 stack_unlock();
723 (void) splx(s);
724 }
725 #endif /* MACHINE_STACK */
726
727 /*
728 * stack_privilege:
729 *
730 * stack_alloc_try on this thread must always succeed.
731 */
732
733 void stack_privilege(
734 register thread_t thread)
735 {
736 /*
737 * This implementation only works for the current thread.
738 */
739
740 if (thread != current_thread())
741 panic("stack_privilege");
742
743 if (thread->stack_privilege == 0)
744 thread->stack_privilege = current_stack();
745 }
746
747 void thread_init(void)
748 {
749 thread_zone = zinit(
750 sizeof(struct thread),
751 THREAD_MAX * sizeof(struct thread),
752 THREAD_CHUNK * sizeof(struct thread),
753 FALSE, "threads");
754
755 /*
756 * Fill in a template thread for fast initialization.
757 * [Fields that must be (or are typically) reset at
758 * time of creation are so noted.]
759 */
760
761 /* thread_template.links (none) */
762 thread_template.runq = RUN_QUEUE_NULL;
763
764 /* thread_template.task (later) */
765 /* thread_template.thread_list (later) */
766 /* thread_template.pset_threads (later) */
767
768 /* thread_template.lock (later) */
769 /* one ref for being alive; one for the guy who creates the thread */
770 thread_template.ref_count = 2;
771
772 thread_template.pcb = (pcb_t) 0; /* (reset) */
773 thread_template.kernel_stack = (vm_offset_t) 0;
774 thread_template.stack_privilege = (vm_offset_t) 0;
775
776 thread_template.wait_event = 0;
777 /* thread_template.suspend_count (later) */
778 thread_template.wait_result = KERN_SUCCESS;
779 thread_template.wake_active = FALSE;
780 thread_template.state = TH_SUSP | TH_SWAPPED;
781 thread_template.swap_func = thread_bootstrap_return;
782
783 /* thread_template.priority (later) */
784 thread_template.max_priority = BASEPRI_USER;
785 /* thread_template.sched_pri (later - compute_priority) */
786 #if MACH_FIXPRI
787 thread_template.sched_data = 0;
788 thread_template.policy = POLICY_TIMESHARE;
789 #endif /* MACH_FIXPRI */
790 thread_template.depress_priority = -1;
791 thread_template.cpu_usage = 0;
792 thread_template.sched_usage = 0;
793 /* thread_template.sched_stamp (later) */
794
795 thread_template.recover = (vm_offset_t) 0;
796 thread_template.vm_privilege = FALSE;
797
798 thread_template.user_stop_count = 1;
799
800 /* thread_template.<IPC structures> (later) */
801
802 timer_init(&(thread_template.user_timer));
803 timer_init(&(thread_template.system_timer));
804 thread_template.user_timer_save.low = 0;
805 thread_template.user_timer_save.high = 0;
806 thread_template.system_timer_save.low = 0;
807 thread_template.system_timer_save.high = 0;
808 thread_template.cpu_delta = 0;
809 thread_template.sched_delta = 0;
810
811 thread_template.active = FALSE; /* reset */
812 thread_template.ast = AST_ZILCH;
813
814 /* thread_template.processor_set (later) */
815 thread_template.bound_processor = PROCESSOR_NULL;
816 #if MACH_HOST
817 thread_template.may_assign = TRUE;
818 thread_template.assign_active = FALSE;
819 #endif /* MACH_HOST */
820
821 #if NCPUS > 1
822 /* thread_template.last_processor (later) */
823 #endif /* NCPUS > 1 */
824
825 /*
826 * Initialize other data structures used in
827 * this module.
828 */
829
830 queue_init(&reaper_queue);
831 simple_lock_init(&reaper_lock);
832
833 #ifndef MACHINE_STACK
834 simple_lock_init(&stack_lock_data);
835 #endif /* MACHINE_STACK */
836
837 #if MACH_DEBUG
838 simple_lock_init(&stack_usage_lock);
839 #endif /* MACH_DEBUG */
840
841 /*
842 * Initialize any machine-dependent
843 * per-thread structures necessary.
844 */
845
846 pcb_module_init();
847 }
848
849 kern_return_t thread_create(
850 register task_t parent_task,
851 thread_t *child_thread) /* OUT */
852 {
853 register thread_t new_thread;
854 register processor_set_t pset;
855
856 if (parent_task == TASK_NULL)
857 return KERN_INVALID_ARGUMENT;
858
859 /*
860 * Allocate a thread and initialize static fields
861 */
862
863 new_thread = (thread_t) zalloc(thread_zone);
864
865 if (new_thread == THREAD_NULL)
866 return KERN_RESOURCE_SHORTAGE;
867
868 *new_thread = thread_template;
869
870 /*
871 * Initialize runtime-dependent fields
872 */
873
874 new_thread->task = parent_task;
875 simple_lock_init(&new_thread->lock);
876 new_thread->sched_stamp = sched_tick;
877 thread_timeout_setup(new_thread);
878
879 /*
880 * Create a pcb. The kernel stack is created later,
881 * when the thread is swapped-in.
882 */
883 pcb_init(new_thread);
884
885 ipc_thread_init(new_thread);
886
887 #if NET_ATM
888 new_thread->nw_ep_waited = 0;
889 #endif
890
891 /*
892 * Find the processor set for the parent task.
893 */
894 task_lock(parent_task);
895 pset = parent_task->processor_set;
896 pset_reference(pset);
897 task_unlock(parent_task);
898
899 /*
900 * Lock both the processor set and the task,
901 * so that the thread can be added to both
902 * simultaneously. Processor set must be
903 * locked first.
904 */
905
906 Restart:
907 pset_lock(pset);
908 task_lock(parent_task);
909
910 /*
911 * If the task has changed processor sets,
912 * catch up (involves lots of lock juggling).
913 */
914 {
915 processor_set_t cur_pset;
916
917 cur_pset = parent_task->processor_set;
918 if (!cur_pset->active)
919 cur_pset = &default_pset;
920
921 if (cur_pset != pset) {
922 pset_reference(cur_pset);
923 task_unlock(parent_task);
924 pset_unlock(pset);
925 pset_deallocate(pset);
926 pset = cur_pset;
927 goto Restart;
928 }
929 }
930
931 /*
932 * Set the thread`s priority from the pset and task.
933 */
934
935 new_thread->priority = parent_task->priority;
936 if (pset->max_priority > new_thread->max_priority)
937 new_thread->max_priority = pset->max_priority;
938 if (new_thread->max_priority > new_thread->priority)
939 new_thread->priority = new_thread->max_priority;
940 /*
941 * Don't need to lock thread here because it can't
942 * possibly execute and no one else knows about it.
943 */
944 compute_priority(new_thread, TRUE);
945
946 /*
947 * Thread is suspended if the task is. Add 1 to
948 * suspend count since thread is created in suspended
949 * state.
950 */
951 new_thread->suspend_count = parent_task->suspend_count + 1;
952
953 /*
954 * Add the thread to the processor set.
955 * If the pset is empty, suspend the thread again.
956 */
957
958 pset_add_thread(pset, new_thread);
959 if (pset->empty)
960 new_thread->suspend_count++;
961
962 #if HW_FOOTPRINT
963 /*
964 * Need to set last_processor, idle processor would be best, but
965 * that requires extra locking nonsense. Go for tail of
966 * processors queue to avoid master.
967 */
968 if (!pset->empty) {
969 new_thread->last_processor =
970 (processor_t)queue_first(&pset->processors);
971 }
972 else {
973 /*
974 * Thread created in empty processor set. Pick
975 * master processor as an acceptable legal value.
976 */
977 new_thread->last_processor = master_processor;
978 }
979 #else /* HW_FOOTPRINT */
980 /*
981 * Don't need to initialize because the context switch
982 * code will set it before it can be used.
983 */
984 #endif /* HW_FOOTPRINT */
985
986 new_thread->pc_sample.buffer = 0;
987 /*
988 * Add the thread to the task`s list of threads.
989 * The new thread holds another reference to the task.
990 */
991
992 parent_task->ref_count++;
993
994 parent_task->thread_count++;
995 queue_enter(&parent_task->thread_list, new_thread, thread_t,
996 thread_list);
997
998 /*
999 * Finally, mark the thread active.
1000 */
1001
1002 new_thread->active = TRUE;
1003
1004 if (!parent_task->active) {
1005 task_unlock(parent_task);
1006 pset_unlock(pset);
1007 (void) thread_terminate(new_thread);
1008 /* release ref we would have given our caller */
1009 thread_deallocate(new_thread);
1010 return KERN_FAILURE;
1011 }
1012 task_unlock(parent_task);
1013 pset_unlock(pset);
1014
1015 ipc_thread_enable(new_thread);
1016
1017 *child_thread = new_thread;
1018 return KERN_SUCCESS;
1019 }
1020
1021 unsigned int thread_deallocate_stack = 0;
1022
1023 void thread_deallocate(
1024 register thread_t thread)
1025 {
1026 spl_t s;
1027 register task_t task;
1028 register processor_set_t pset;
1029
1030 time_value_t user_time, system_time;
1031
1032 if (thread == THREAD_NULL)
1033 return;
1034
1035 /*
1036 * First, check for new count > 0 (the common case).
1037 * Only the thread needs to be locked.
1038 */
1039 s = splsched();
1040 thread_lock(thread);
1041 if (--thread->ref_count > 0) {
1042 thread_unlock(thread);
1043 (void) splx(s);
1044 return;
1045 }
1046
1047 /*
1048 * Count is zero. However, the task's and processor set's
1049 * thread lists have implicit references to
1050 * the thread, and may make new ones. Their locks also
1051 * dominate the thread lock. To check for this, we
1052 * temporarily restore the one thread reference, unlock
1053 * the thread, and then lock the other structures in
1054 * the proper order.
1055 */
1056 thread->ref_count = 1;
1057 thread_unlock(thread);
1058 (void) splx(s);
1059
1060 pset = thread->processor_set;
1061 pset_lock(pset);
1062
1063 #if MACH_HOST
1064 /*
1065 * The thread might have moved.
1066 */
1067 while (pset != thread->processor_set) {
1068 pset_unlock(pset);
1069 pset = thread->processor_set;
1070 pset_lock(pset);
1071 }
1072 #endif /* MACH_HOST */
1073
1074 task = thread->task;
1075 task_lock(task);
1076
1077 s = splsched();
1078 thread_lock(thread);
1079
1080 if (--thread->ref_count > 0) {
1081 /*
1082 * Task or processor_set made extra reference.
1083 */
1084 thread_unlock(thread);
1085 (void) splx(s);
1086 task_unlock(task);
1087 pset_unlock(pset);
1088 return;
1089 }
1090
1091 /*
1092 * Thread has no references - we can remove it.
1093 */
1094
1095 /*
1096 * Remove pending timeouts.
1097 */
1098 reset_timeout_check(&thread->timer);
1099
1100 reset_timeout_check(&thread->depress_timer);
1101 thread->depress_priority = -1;
1102
1103 /*
1104 * Accumulate times for dead threads in task.
1105 */
1106 thread_read_times(thread, &user_time, &system_time);
1107 time_value_add(&task->total_user_time, &user_time);
1108 time_value_add(&task->total_system_time, &system_time);
1109
1110 /*
1111 * Remove thread from task list and processor_set threads list.
1112 */
1113 task->thread_count--;
1114 queue_remove(&task->thread_list, thread, thread_t, thread_list);
1115
1116 pset_remove_thread(pset, thread);
1117
1118 thread_unlock(thread); /* no more references - safe */
1119 (void) splx(s);
1120 task_unlock(task);
1121 pset_unlock(pset);
1122 pset_deallocate(pset);
1123
1124 /*
1125 * A couple of quick sanity checks
1126 */
1127
1128 if (thread == current_thread()) {
1129 panic("thread deallocating itself");
1130 }
1131 if ((thread->state & ~(TH_RUN | TH_HALTED | TH_SWAPPED)) != TH_SUSP)
1132 panic("unstopped thread destroyed!");
1133
1134 /*
1135 * Deallocate the task reference, since we know the thread
1136 * is not running.
1137 */
1138 task_deallocate(thread->task); /* may block */
1139
1140 /*
1141 * Clean up any machine-dependent resources.
1142 */
1143 if ((thread->state & TH_SWAPPED) == 0) {
1144 spl_t _s_ = splsched();
1145 stack_free(thread);
1146 (void) splx(s);
1147 thread_deallocate_stack++;
1148 }
1149 /*
1150 * Rattle the event count machinery (gag)
1151 */
1152 evc_notify_abort(thread);
1153
1154 pcb_terminate(thread);
1155 zfree(thread_zone, (vm_offset_t) thread);
1156 }
1157
1158 void thread_reference(
1159 register thread_t thread)
1160 {
1161 spl_t s;
1162
1163 if (thread == THREAD_NULL)
1164 return;
1165
1166 s = splsched();
1167 thread_lock(thread);
1168 thread->ref_count++;
1169 thread_unlock(thread);
1170 (void) splx(s);
1171 }
1172
1173 /*
1174 * thread_terminate:
1175 *
1176 * Permanently stop execution of the specified thread.
1177 *
1178 * A thread to be terminated must be allowed to clean up any state
1179 * that it has before it exits. The thread is broken out of any
1180 * wait condition that it is in, and signalled to exit. It then
1181 * cleans up its state and calls thread_halt_self on its way out of
1182 * the kernel. The caller waits for the thread to halt, terminates
1183 * its IPC state, and then deallocates it.
1184 *
1185 * If the caller is the current thread, it must still exit the kernel
1186 * to clean up any state (thread and port references, messages, etc).
1187 * When it exits the kernel, it then terminates its IPC state and
1188 * queues itself for the reaper thread, which will wait for the thread
1189 * to stop and then deallocate it. (A thread cannot deallocate itself,
1190 * since it needs a kernel stack to execute.)
1191 */
1192 kern_return_t thread_terminate(
1193 register thread_t thread)
1194 {
1195 register thread_t cur_thread = current_thread();
1196 register task_t cur_task;
1197 spl_t s;
1198
1199 if (thread == THREAD_NULL)
1200 return KERN_INVALID_ARGUMENT;
1201
1202 /*
1203 * Break IPC control over the thread.
1204 */
1205 ipc_thread_disable(thread);
1206
1207 if (thread == cur_thread) {
1208
1209 /*
1210 * Current thread will queue itself for reaper when
1211 * exiting kernel.
1212 */
1213 s = splsched();
1214 thread_lock(thread);
1215 if (thread->active) {
1216 thread->active = FALSE;
1217 thread_ast_set(thread, AST_TERMINATE);
1218 }
1219 thread_unlock(thread);
1220 ast_on(cpu_number(), AST_TERMINATE);
1221 splx(s);
1222 return KERN_SUCCESS;
1223 }
1224
1225 /*
1226 * Lock both threads and the current task
1227 * to check termination races and prevent deadlocks.
1228 */
1229 cur_task = current_task();
1230 task_lock(cur_task);
1231 s = splsched();
1232 if ((vm_offset_t)thread < (vm_offset_t)cur_thread) {
1233 thread_lock(thread);
1234 thread_lock(cur_thread);
1235 }
1236 else {
1237 thread_lock(cur_thread);
1238 thread_lock(thread);
1239 }
1240
1241 /*
1242 * If the current thread is being terminated, help out.
1243 */
1244 if ((!cur_task->active) || (!cur_thread->active)) {
1245 thread_unlock(cur_thread);
1246 thread_unlock(thread);
1247 (void) splx(s);
1248 task_unlock(cur_task);
1249 thread_terminate(cur_thread);
1250 return KERN_FAILURE;
1251 }
1252
1253 thread_unlock(cur_thread);
1254 task_unlock(cur_task);
1255
1256 /*
1257 * Terminate victim thread.
1258 */
1259 if (!thread->active) {
1260 /*
1261 * Someone else got there first.
1262 */
1263 thread_unlock(thread);
1264 (void) splx(s);
1265 return KERN_FAILURE;
1266 }
1267
1268 thread->active = FALSE;
1269
1270 thread_unlock(thread);
1271 (void) splx(s);
1272
1273 #if MACH_HOST
1274 /*
1275 * Reassign thread to default pset if needed.
1276 */
1277 thread_freeze(thread);
1278 if (thread->processor_set != &default_pset) {
1279 thread_doassign(thread, &default_pset, FALSE);
1280 }
1281 #endif /* MACH_HOST */
1282
1283 /*
1284 * Halt the victim at the clean point.
1285 */
1286 (void) thread_halt(thread, TRUE);
1287 #if MACH_HOST
1288 thread_unfreeze(thread);
1289 #endif /* MACH_HOST */
1290 /*
1291 * Shut down the victims IPC and deallocate its
1292 * reference to itself.
1293 */
1294 ipc_thread_terminate(thread);
1295 #if NET_ATM
1296 mk_waited_collect(thread);
1297 #endif
1298 thread_deallocate(thread);
1299 return KERN_SUCCESS;
1300 }
1301
1302 /*
1303 * thread_force_terminate:
1304 *
1305 * Version of thread_terminate called by task_terminate. thread is
1306 * not the current thread. task_terminate is the dominant operation,
1307 * so we can force this thread to stop.
1308 */
1309 void
1310 thread_force_terminate(
1311 register thread_t thread)
1312 {
1313 boolean_t deallocate_here = FALSE;
1314 spl_t s;
1315
1316 ipc_thread_disable(thread);
1317
1318 #if MACH_HOST
1319 /*
1320 * Reassign thread to default pset if needed.
1321 */
1322 thread_freeze(thread);
1323 if (thread->processor_set != &default_pset)
1324 thread_doassign(thread, &default_pset, FALSE);
1325 #endif /* MACH_HOST */
1326
1327 s = splsched();
1328 thread_lock(thread);
1329 deallocate_here = thread->active;
1330 thread->active = FALSE;
1331 thread_unlock(thread);
1332 (void) splx(s);
1333
1334 (void) thread_halt(thread, TRUE);
1335 ipc_thread_terminate(thread);
1336 #if NET_ATM
1337 mk_waited_collect(thread);
1338 #endif
1339
1340 #if MACH_HOST
1341 thread_unfreeze(thread);
1342 #endif /* MACH_HOST */
1343
1344 if (deallocate_here)
1345 thread_deallocate(thread);
1346 }
1347
1348
1349 /*
1350 * Halt a thread at a clean point, leaving it suspended.
1351 *
1352 * must_halt indicates whether thread must halt.
1353 *
1354 */
1355 kern_return_t thread_halt(
1356 register thread_t thread,
1357 boolean_t must_halt)
1358 {
1359 register thread_t cur_thread = current_thread();
1360 register kern_return_t ret;
1361 spl_t s;
1362
1363 if (thread == cur_thread)
1364 panic("thread_halt: trying to halt current thread.");
1365 /*
1366 * If must_halt is FALSE, then a check must be made for
1367 * a cycle of halt operations.
1368 */
1369 if (!must_halt) {
1370 /*
1371 * Grab both thread locks.
1372 */
1373 s = splsched();
1374 if ((vm_offset_t)thread < (vm_offset_t)cur_thread) {
1375 thread_lock(thread);
1376 thread_lock(cur_thread);
1377 }
1378 else {
1379 thread_lock(cur_thread);
1380 thread_lock(thread);
1381 }
1382
1383 /*
1384 * If target thread is already halted, grab a hold
1385 * on it and return.
1386 */
1387 if (thread->state & TH_HALTED) {
1388 thread->suspend_count++;
1389 thread_unlock(cur_thread);
1390 thread_unlock(thread);
1391 (void) splx(s);
1392 return KERN_SUCCESS;
1393 }
1394
1395 /*
1396 * If someone is trying to halt us, we have a potential
1397 * halt cycle. Break the cycle by interrupting anyone
1398 * who is trying to halt us, and causing this operation
1399 * to fail; retry logic will only retry operations
1400 * that cannot deadlock. (If must_halt is TRUE, this
1401 * operation can never cause a deadlock.)
1402 */
1403 if (cur_thread->ast & AST_HALT) {
1404 thread_wakeup_with_result((event_t)&cur_thread->wake_active,
1405 THREAD_INTERRUPTED);
1406 thread_unlock(thread);
1407 thread_unlock(cur_thread);
1408 (void) splx(s);
1409 return KERN_FAILURE;
1410 }
1411
1412 thread_unlock(cur_thread);
1413
1414 }
1415 else {
1416 /*
1417 * Lock thread and check whether it is already halted.
1418 */
1419 s = splsched();
1420 thread_lock(thread);
1421 if (thread->state & TH_HALTED) {
1422 thread->suspend_count++;
1423 thread_unlock(thread);
1424 (void) splx(s);
1425 return KERN_SUCCESS;
1426 }
1427 }
1428
1429 /*
1430 * Suspend thread - inline version of thread_hold() because
1431 * thread is already locked.
1432 */
1433 thread->suspend_count++;
1434 thread->state |= TH_SUSP;
1435
1436 /*
1437 * If someone else is halting it, wait for that to complete.
1438 * Fail if wait interrupted and must_halt is false.
1439 */
1440 while ((thread->ast & AST_HALT) && (!(thread->state & TH_HALTED))) {
1441 thread->wake_active = TRUE;
1442 thread_sleep((event_t) &thread->wake_active,
1443 simple_lock_addr(thread->lock), TRUE);
1444
1445 if (thread->state & TH_HALTED) {
1446 (void) splx(s);
1447 return KERN_SUCCESS;
1448 }
1449 if ((current_thread()->wait_result != THREAD_AWAKENED)
1450 && !(must_halt)) {
1451 (void) splx(s);
1452 thread_release(thread);
1453 return KERN_FAILURE;
1454 }
1455 thread_lock(thread);
1456 }
1457
1458 /*
1459 * Otherwise, have to do it ourselves.
1460 */
1461
1462 thread_ast_set(thread, AST_HALT);
1463
1464 while (TRUE) {
1465 /*
1466 * Wait for thread to stop.
1467 */
1468 thread_unlock(thread);
1469 (void) splx(s);
1470
1471 ret = thread_dowait(thread, must_halt);
1472
1473 /*
1474 * If the dowait failed, so do we. Drop AST_HALT, and
1475 * wake up anyone else who might be waiting for it.
1476 */
1477 if (ret != KERN_SUCCESS) {
1478 s = splsched();
1479 thread_lock(thread);
1480 thread_ast_clear(thread, AST_HALT);
1481 thread_wakeup_with_result((event_t)&thread->wake_active,
1482 THREAD_INTERRUPTED);
1483 thread_unlock(thread);
1484 (void) splx(s);
1485
1486 thread_release(thread);
1487 return ret;
1488 }
1489
1490 /*
1491 * Clear any interruptible wait.
1492 */
1493 clear_wait(thread, THREAD_INTERRUPTED, TRUE);
1494
1495 /*
1496 * If the thread's at a clean point, we're done.
1497 * Don't need a lock because it really is stopped.
1498 */
1499 if (thread->state & TH_HALTED) {
1500 return KERN_SUCCESS;
1501 }
1502
1503 /*
1504 * If the thread is at a nice continuation,
1505 * or a continuation with a cleanup routine,
1506 * call the cleanup routine.
1507 */
1508 if ((((thread->swap_func == mach_msg_continue) ||
1509 (thread->swap_func == mach_msg_receive_continue)) &&
1510 mach_msg_interrupt(thread)) ||
1511 (thread->swap_func == thread_exception_return) ||
1512 (thread->swap_func == thread_bootstrap_return)) {
1513 s = splsched();
1514 thread_lock(thread);
1515 thread->state |= TH_HALTED;
1516 thread_ast_clear(thread, AST_HALT);
1517 thread_unlock(thread);
1518 splx(s);
1519
1520 return KERN_SUCCESS;
1521 }
1522
1523 /*
1524 * Force the thread to stop at a clean
1525 * point, and arrange to wait for it.
1526 *
1527 * Set it running, so it can notice. Override
1528 * the suspend count. We know that the thread
1529 * is suspended and not waiting.
1530 *
1531 * Since the thread may hit an interruptible wait
1532 * before it reaches a clean point, we must force it
1533 * to wake us up when it does so. This involves some
1534 * trickery:
1535 * We mark the thread SUSPENDED so that thread_block
1536 * will suspend it and wake us up.
1537 * We mark the thread RUNNING so that it will run.
1538 * We mark the thread UN-INTERRUPTIBLE (!) so that
1539 * some other thread trying to halt or suspend it won't
1540 * take it off the run queue before it runs. Since
1541 * dispatching a thread (the tail of thread_invoke) marks
1542 * the thread interruptible, it will stop at the next
1543 * context switch or interruptible wait.
1544 */
1545
1546 s = splsched();
1547 thread_lock(thread);
1548 if ((thread->state & TH_SCHED_STATE) != TH_SUSP)
1549 panic("thread_halt");
1550 thread->state |= TH_RUN | TH_UNINT;
1551 thread_setrun(thread, FALSE);
1552
1553 /*
1554 * Continue loop and wait for thread to stop.
1555 */
1556 }
1557 }
1558
1559 void walking_zombie(void)
1560 {
1561 panic("the zombie walks!");
1562 }
1563
1564 /*
1565 * Thread calls this routine on exit from the kernel when it
1566 * notices a halt request.
1567 */
1568 void thread_halt_self(void)
1569 {
1570 register thread_t thread = current_thread();
1571 spl_t s;
1572
1573 if (thread->ast & AST_TERMINATE) {
1574 /*
1575 * Thread is terminating itself. Shut
1576 * down IPC, then queue it up for the
1577 * reaper thread.
1578 */
1579 ipc_thread_terminate(thread);
1580 #if NET_ATM
1581 mk_waited_collect(thread);
1582 #endif
1583
1584 thread_hold(thread);
1585
1586 s = splsched();
1587 simple_lock(&reaper_lock);
1588 enqueue_tail(&reaper_queue, (queue_entry_t) thread);
1589 simple_unlock(&reaper_lock);
1590
1591 thread_lock(thread);
1592 thread->state |= TH_HALTED;
1593 thread_unlock(thread);
1594 (void) splx(s);
1595
1596 thread_wakeup((event_t)&reaper_queue);
1597 counter(c_thread_halt_self_block++);
1598 thread_block(walking_zombie);
1599 /*NOTREACHED*/
1600 } else {
1601 /*
1602 * Thread was asked to halt - show that it
1603 * has done so.
1604 */
1605 s = splsched();
1606 thread_lock(thread);
1607 thread->state |= TH_HALTED;
1608 thread_ast_clear(thread, AST_HALT);
1609 thread_unlock(thread);
1610 splx(s);
1611 counter(c_thread_halt_self_block++);
1612 thread_block(thread_exception_return);
1613 /*
1614 * thread_release resets TH_HALTED.
1615 */
1616 }
1617 }
1618
1619 /*
1620 * thread_hold:
1621 *
1622 * Suspend execution of the specified thread.
1623 * This is a recursive-style suspension of the thread, a count of
1624 * suspends is maintained.
1625 */
1626 void thread_hold(
1627 register thread_t thread)
1628 {
1629 spl_t s;
1630
1631 s = splsched();
1632 thread_lock(thread);
1633 thread->suspend_count++;
1634 thread->state |= TH_SUSP;
1635 thread_unlock(thread);
1636 (void) splx(s);
1637 }
1638
1639 /*
1640 * thread_dowait:
1641 *
1642 * Wait for a thread to actually enter stopped state.
1643 *
1644 * must_halt argument indicates if this may fail on interruption.
1645 * This is FALSE only if called from thread_abort via thread_halt.
1646 */
1647 kern_return_t
1648 thread_dowait(
1649 register thread_t thread,
1650 boolean_t must_halt)
1651 {
1652 register boolean_t need_wakeup;
1653 register kern_return_t ret = KERN_SUCCESS;
1654 spl_t s;
1655
1656 if (thread == current_thread())
1657 panic("thread_dowait");
1658
1659 /*
1660 * If a thread is not interruptible, it may not be suspended
1661 * until it becomes interruptible. In this case, we wait for
1662 * the thread to stop itself, and indicate that we are waiting
1663 * for it to stop so that it can wake us up when it does stop.
1664 *
1665 * If the thread is interruptible, we may be able to suspend
1666 * it immediately. There are several cases:
1667 *
1668 * 1) The thread is already stopped (trivial)
1669 * 2) The thread is runnable (marked RUN and on a run queue).
1670 * We pull it off the run queue and mark it stopped.
1671 * 3) The thread is running. We wait for it to stop.
1672 */
1673
1674 need_wakeup = FALSE;
1675 s = splsched();
1676 thread_lock(thread);
1677
1678 for (;;) {
1679 switch (thread->state & TH_SCHED_STATE) {
1680 case TH_SUSP:
1681 case TH_WAIT | TH_SUSP:
1682 /*
1683 * Thread is already suspended, or sleeping in an
1684 * interruptible wait. We win!
1685 */
1686 break;
1687
1688 case TH_RUN | TH_SUSP:
1689 /*
1690 * The thread is interruptible. If we can pull
1691 * it off a runq, stop it here.
1692 */
1693 if (rem_runq(thread) != RUN_QUEUE_NULL) {
1694 thread->state &= ~TH_RUN;
1695 need_wakeup = thread->wake_active;
1696 thread->wake_active = FALSE;
1697 break;
1698 }
1699 #if NCPUS > 1
1700 /*
1701 * The thread must be running, so make its
1702 * processor execute ast_check(). This
1703 * should cause the thread to take an ast and
1704 * context switch to suspend for us.
1705 */
1706 cause_ast_check(thread->last_processor);
1707 #endif /* NCPUS > 1 */
1708
1709 /*
1710 * Fall through to wait for thread to stop.
1711 */
1712
1713 case TH_RUN | TH_SUSP | TH_UNINT:
1714 case TH_RUN | TH_WAIT | TH_SUSP:
1715 case TH_RUN | TH_WAIT | TH_SUSP | TH_UNINT:
1716 case TH_WAIT | TH_SUSP | TH_UNINT:
1717 /*
1718 * Wait for the thread to stop, or sleep interruptibly
1719 * (thread_block will stop it in the latter case).
1720 * Check for failure if interrupted.
1721 */
1722 thread->wake_active = TRUE;
1723 thread_sleep((event_t) &thread->wake_active,
1724 simple_lock_addr(thread->lock), TRUE);
1725 thread_lock(thread);
1726 if ((current_thread()->wait_result != THREAD_AWAKENED) &&
1727 !must_halt) {
1728 ret = KERN_FAILURE;
1729 break;
1730 }
1731
1732 /*
1733 * Repeat loop to check thread`s state.
1734 */
1735 continue;
1736 }
1737 /*
1738 * Thread is stopped at this point.
1739 */
1740 break;
1741 }
1742
1743 thread_unlock(thread);
1744 (void) splx(s);
1745
1746 if (need_wakeup)
1747 thread_wakeup((event_t) &thread->wake_active);
1748
1749 return ret;
1750 }
1751
1752 void thread_release(
1753 register thread_t thread)
1754 {
1755 spl_t s;
1756
1757 s = splsched();
1758 thread_lock(thread);
1759 if (--thread->suspend_count == 0) {
1760 thread->state &= ~(TH_SUSP | TH_HALTED);
1761 if ((thread->state & (TH_WAIT | TH_RUN)) == 0) {
1762 /* was only suspended */
1763 thread->state |= TH_RUN;
1764 thread_setrun(thread, TRUE);
1765 }
1766 }
1767 thread_unlock(thread);
1768 (void) splx(s);
1769 }
1770
1771 kern_return_t thread_suspend(
1772 register thread_t thread)
1773 {
1774 register boolean_t hold;
1775 spl_t spl;
1776
1777 if (thread == THREAD_NULL)
1778 return KERN_INVALID_ARGUMENT;
1779
1780 hold = FALSE;
1781 spl = splsched();
1782 thread_lock(thread);
1783 if (thread->user_stop_count++ == 0) {
1784 hold = TRUE;
1785 thread->suspend_count++;
1786 thread->state |= TH_SUSP;
1787 }
1788 thread_unlock(thread);
1789 (void) splx(spl);
1790
1791 /*
1792 * Now wait for the thread if necessary.
1793 */
1794 if (hold) {
1795 if (thread == current_thread()) {
1796 /*
1797 * We want to call thread_block on our way out,
1798 * to stop running.
1799 */
1800 spl = splsched();
1801 ast_on(cpu_number(), AST_BLOCK);
1802 (void) splx(spl);
1803 } else
1804 (void) thread_dowait(thread, TRUE);
1805 }
1806 return KERN_SUCCESS;
1807 }
1808
1809
1810 kern_return_t thread_resume(
1811 register thread_t thread)
1812 {
1813 register kern_return_t ret;
1814 spl_t s;
1815
1816 if (thread == THREAD_NULL)
1817 return KERN_INVALID_ARGUMENT;
1818
1819 ret = KERN_SUCCESS;
1820
1821 s = splsched();
1822 thread_lock(thread);
1823 if (thread->user_stop_count > 0) {
1824 if (--thread->user_stop_count == 0) {
1825 if (--thread->suspend_count == 0) {
1826 thread->state &= ~(TH_SUSP | TH_HALTED);
1827 if ((thread->state & (TH_WAIT | TH_RUN)) == 0) {
1828 /* was only suspended */
1829 thread->state |= TH_RUN;
1830 thread_setrun(thread, TRUE);
1831 }
1832 }
1833 }
1834 }
1835 else {
1836 ret = KERN_FAILURE;
1837 }
1838
1839 thread_unlock(thread);
1840 (void) splx(s);
1841
1842 return ret;
1843 }
1844
1845 /*
1846 * Return thread's machine-dependent state.
1847 */
1848 kern_return_t thread_get_state(
1849 register thread_t thread,
1850 int flavor,
1851 thread_state_t old_state, /* pointer to OUT array */
1852 natural_t *old_state_count) /*IN/OUT*/
1853 {
1854 kern_return_t ret;
1855
1856 if (thread == THREAD_NULL || thread == current_thread()) {
1857 return KERN_INVALID_ARGUMENT;
1858 }
1859
1860 thread_hold(thread);
1861 (void) thread_dowait(thread, TRUE);
1862
1863 ret = thread_getstatus(thread, flavor, old_state, old_state_count);
1864
1865 thread_release(thread);
1866 return ret;
1867 }
1868
1869 /*
1870 * Change thread's machine-dependent state.
1871 */
1872 kern_return_t thread_set_state(
1873 register thread_t thread,
1874 int flavor,
1875 thread_state_t new_state,
1876 natural_t new_state_count)
1877 {
1878 kern_return_t ret;
1879
1880 if (thread == THREAD_NULL || thread == current_thread()) {
1881 return KERN_INVALID_ARGUMENT;
1882 }
1883
1884 thread_hold(thread);
1885 (void) thread_dowait(thread, TRUE);
1886
1887 ret = thread_setstatus(thread, flavor, new_state, new_state_count);
1888
1889 thread_release(thread);
1890 return ret;
1891 }
1892
1893 kern_return_t thread_info(
1894 register thread_t thread,
1895 int flavor,
1896 thread_info_t thread_info_out, /* pointer to OUT array */
1897 natural_t *thread_info_count) /*IN/OUT*/
1898 {
1899 int state, flags;
1900 spl_t s;
1901
1902 if (thread == THREAD_NULL)
1903 return KERN_INVALID_ARGUMENT;
1904
1905 if (flavor == THREAD_BASIC_INFO) {
1906 register thread_basic_info_t basic_info;
1907
1908 if (*thread_info_count < THREAD_BASIC_INFO_COUNT) {
1909 return KERN_INVALID_ARGUMENT;
1910 }
1911
1912 basic_info = (thread_basic_info_t) thread_info_out;
1913
1914 s = splsched();
1915 thread_lock(thread);
1916
1917 /*
1918 * Update lazy-evaluated scheduler info because someone wants it.
1919 */
1920 if ((thread->state & TH_RUN) == 0 &&
1921 thread->sched_stamp != sched_tick)
1922 update_priority(thread);
1923
1924 /* fill in info */
1925
1926 thread_read_times(thread,
1927 &basic_info->user_time,
1928 &basic_info->system_time);
1929 basic_info->base_priority = thread->priority;
1930 basic_info->cur_priority = thread->sched_pri;
1931
1932 /*
1933 * To calculate cpu_usage, first correct for timer rate,
1934 * then for 5/8 ageing. The correction factor [3/5] is
1935 * (1/(5/8) - 1).
1936 */
1937 basic_info->cpu_usage = thread->cpu_usage /
1938 (TIMER_RATE/TH_USAGE_SCALE);
1939 basic_info->cpu_usage = (basic_info->cpu_usage * 3) / 5;
1940 #if SIMPLE_CLOCK
1941 /*
1942 * Clock drift compensation.
1943 */
1944 basic_info->cpu_usage =
1945 (basic_info->cpu_usage * 1000000)/sched_usec;
1946 #endif /* SIMPLE_CLOCK */
1947
1948 if (thread->state & TH_SWAPPED)
1949 flags = TH_FLAGS_SWAPPED;
1950 else if (thread->state & TH_IDLE)
1951 flags = TH_FLAGS_IDLE;
1952 else
1953 flags = 0;
1954
1955 if (thread->state & TH_HALTED)
1956 state = TH_STATE_HALTED;
1957 else
1958 if (thread->state & TH_RUN)
1959 state = TH_STATE_RUNNING;
1960 else
1961 if (thread->state & TH_UNINT)
1962 state = TH_STATE_UNINTERRUPTIBLE;
1963 else
1964 if (thread->state & TH_SUSP)
1965 state = TH_STATE_STOPPED;
1966 else
1967 if (thread->state & TH_WAIT)
1968 state = TH_STATE_WAITING;
1969 else
1970 state = 0; /* ? */
1971
1972 basic_info->run_state = state;
1973 basic_info->flags = flags;
1974 basic_info->suspend_count = thread->user_stop_count;
1975 if (state == TH_STATE_RUNNING)
1976 basic_info->sleep_time = 0;
1977 else
1978 basic_info->sleep_time = sched_tick - thread->sched_stamp;
1979
1980 thread_unlock(thread);
1981 splx(s);
1982
1983 *thread_info_count = THREAD_BASIC_INFO_COUNT;
1984 return KERN_SUCCESS;
1985 }
1986 else if (flavor == THREAD_SCHED_INFO) {
1987 register thread_sched_info_t sched_info;
1988
1989 if (*thread_info_count < THREAD_SCHED_INFO_COUNT) {
1990 return KERN_INVALID_ARGUMENT;
1991 }
1992
1993 sched_info = (thread_sched_info_t) thread_info_out;
1994
1995 s = splsched();
1996 thread_lock(thread);
1997
1998 #if MACH_FIXPRI
1999 sched_info->policy = thread->policy;
2000 if (thread->policy == POLICY_FIXEDPRI) {
2001 sched_info->data = (thread->sched_data * tick)/1000;
2002 }
2003 else {
2004 sched_info->data = 0;
2005 }
2006 #else /* MACH_FIXPRI */
2007 sched_info->policy = POLICY_TIMESHARE;
2008 sched_info->data = 0;
2009 #endif /* MACH_FIXPRI */
2010
2011 sched_info->base_priority = thread->priority;
2012 sched_info->max_priority = thread->max_priority;
2013 sched_info->cur_priority = thread->sched_pri;
2014
2015 sched_info->depressed = (thread->depress_priority >= 0);
2016 sched_info->depress_priority = thread->depress_priority;
2017
2018 thread_unlock(thread);
2019 splx(s);
2020
2021 *thread_info_count = THREAD_SCHED_INFO_COUNT;
2022 return KERN_SUCCESS;
2023 }
2024
2025 return KERN_INVALID_ARGUMENT;
2026 }
2027
2028 kern_return_t thread_abort(
2029 register thread_t thread)
2030 {
2031 if (thread == THREAD_NULL || thread == current_thread()) {
2032 return KERN_INVALID_ARGUMENT;
2033 }
2034
2035 /*
2036 *
2037 * clear it of an event wait
2038 */
2039 evc_notify_abort(thread);
2040
2041 /*
2042 * Try to force the thread to a clean point
2043 * If the halt operation fails return KERN_ABORTED.
2044 * ipc code will convert this to an ipc interrupted error code.
2045 */
2046 if (thread_halt(thread, FALSE) != KERN_SUCCESS)
2047 return KERN_ABORTED;
2048
2049 /*
2050 * If the thread was in an exception, abort that too.
2051 */
2052 mach_msg_abort_rpc(thread);
2053
2054 /*
2055 * Then set it going again.
2056 */
2057 thread_release(thread);
2058
2059 /*
2060 * Also abort any depression.
2061 */
2062 if (thread->depress_priority != -1)
2063 thread_depress_abort(thread);
2064
2065 return KERN_SUCCESS;
2066 }
2067
2068 /*
2069 * thread_start:
2070 *
2071 * Start a thread at the specified routine.
2072 * The thread must be in a swapped state.
2073 */
2074
2075 void
2076 thread_start(
2077 thread_t thread,
2078 continuation_t start)
2079 {
2080 thread->swap_func = start;
2081 }
2082
2083 /*
2084 * kernel_thread:
2085 *
2086 * Start up a kernel thread in the specified task.
2087 */
2088
2089 thread_t kernel_thread(
2090 task_t task,
2091 continuation_t start,
2092 void * arg)
2093 {
2094 thread_t thread;
2095
2096 (void) thread_create(task, &thread);
2097 /* release "extra" ref that thread_create gave us */
2098 thread_deallocate(thread);
2099 thread_start(thread, start);
2100 thread->ith_other = arg;
2101
2102 /*
2103 * We ensure that the kernel thread starts with a stack.
2104 * The swapin mechanism might not be operational yet.
2105 */
2106 thread_doswapin(thread);
2107 thread->max_priority = BASEPRI_SYSTEM;
2108 thread->priority = BASEPRI_SYSTEM;
2109 thread->sched_pri = BASEPRI_SYSTEM;
2110 (void) thread_resume(thread);
2111 return thread;
2112 }
2113
2114 /*
2115 * reaper_thread:
2116 *
2117 * This kernel thread runs forever looking for threads to destroy
2118 * (when they request that they be destroyed, of course).
2119 */
2120 void reaper_thread_continue(void)
2121 {
2122 for (;;) {
2123 register thread_t thread;
2124 spl_t s;
2125
2126 s = splsched();
2127 simple_lock(&reaper_lock);
2128
2129 while ((thread = (thread_t) dequeue_head(&reaper_queue))
2130 != THREAD_NULL) {
2131 simple_unlock(&reaper_lock);
2132 (void) splx(s);
2133
2134 (void) thread_dowait(thread, TRUE); /* may block */
2135 thread_deallocate(thread); /* may block */
2136
2137 s = splsched();
2138 simple_lock(&reaper_lock);
2139 }
2140
2141 assert_wait((event_t) &reaper_queue, FALSE);
2142 simple_unlock(&reaper_lock);
2143 (void) splx(s);
2144 counter(c_reaper_thread_block++);
2145 thread_block(reaper_thread_continue);
2146 }
2147 }
2148
2149 void reaper_thread(void)
2150 {
2151 reaper_thread_continue();
2152 /*NOTREACHED*/
2153 }
2154
2155 #if MACH_HOST
2156 /*
2157 * thread_assign:
2158 *
2159 * Change processor set assignment.
2160 * Caller must hold an extra reference to the thread (if this is
2161 * called directly from the ipc interface, this is an operation
2162 * in progress reference). Caller must hold no locks -- this may block.
2163 */
2164
2165 kern_return_t
2166 thread_assign(
2167 thread_t thread,
2168 processor_set_t new_pset)
2169 {
2170 if (thread == THREAD_NULL || new_pset == PROCESSOR_SET_NULL) {
2171 return KERN_INVALID_ARGUMENT;
2172 }
2173
2174 thread_freeze(thread);
2175 thread_doassign(thread, new_pset, TRUE);
2176
2177 return KERN_SUCCESS;
2178 }
2179
2180 /*
2181 * thread_freeze:
2182 *
2183 * Freeze thread's assignment. Prelude to assigning thread.
2184 * Only one freeze may be held per thread.
2185 */
2186 void
2187 thread_freeze(
2188 thread_t thread)
2189 {
2190 spl_t s;
2191 /*
2192 * Freeze the assignment, deferring to a prior freeze.
2193 */
2194 s = splsched();
2195 thread_lock(thread);
2196 while (thread->may_assign == FALSE) {
2197 thread->assign_active = TRUE;
2198 thread_sleep((event_t) &thread->assign_active,
2199 simple_lock_addr(thread->lock), FALSE);
2200 thread_lock(thread);
2201 }
2202 thread->may_assign = FALSE;
2203 thread_unlock(thread);
2204 (void) splx(s);
2205
2206 }
2207
2208 /*
2209 * thread_unfreeze: release freeze on thread's assignment.
2210 */
2211 void
2212 thread_unfreeze(
2213 thread_t thread)
2214 {
2215 spl_t s;
2216
2217 s = splsched();
2218 thread_lock(thread);
2219 thread->may_assign = TRUE;
2220 if (thread->assign_active) {
2221 thread->assign_active = FALSE;
2222 thread_wakeup((event_t)&thread->assign_active);
2223 }
2224 thread_unlock(thread);
2225 splx(s);
2226 }
2227
2228 /*
2229 * thread_doassign:
2230 *
2231 * Actually do thread assignment. thread_will_assign must have been
2232 * called on the thread. release_freeze argument indicates whether
2233 * to release freeze on thread.
2234 */
2235
2236 void
2237 thread_doassign(
2238 register thread_t thread,
2239 register processor_set_t new_pset,
2240 boolean_t release_freeze)
2241 {
2242 register processor_set_t pset;
2243 register boolean_t old_empty, new_empty;
2244 boolean_t recompute_pri = FALSE;
2245 spl_t s;
2246
2247 /*
2248 * Check for silly no-op.
2249 */
2250 pset = thread->processor_set;
2251 if (pset == new_pset) {
2252 if (release_freeze)
2253 thread_unfreeze(thread);
2254 return;
2255 }
2256 /*
2257 * Suspend the thread and stop it if it's not the current thread.
2258 */
2259 thread_hold(thread);
2260 if (thread != current_thread())
2261 (void) thread_dowait(thread, TRUE);
2262
2263 /*
2264 * Lock both psets now, use ordering to avoid deadlocks.
2265 */
2266 Restart:
2267 if ((vm_offset_t)pset < (vm_offset_t)new_pset) {
2268 pset_lock(pset);
2269 pset_lock(new_pset);
2270 }
2271 else {
2272 pset_lock(new_pset);
2273 pset_lock(pset);
2274 }
2275
2276 /*
2277 * Check if new_pset is ok to assign to. If not, reassign
2278 * to default_pset.
2279 */
2280 if (!new_pset->active) {
2281 pset_unlock(pset);
2282 pset_unlock(new_pset);
2283 new_pset = &default_pset;
2284 goto Restart;
2285 }
2286
2287 pset_reference(new_pset);
2288
2289 /*
2290 * Grab the thread lock and move the thread.
2291 * Then drop the lock on the old pset and the thread's
2292 * reference to it.
2293 */
2294 s = splsched();
2295 thread_lock(thread);
2296
2297 thread_change_psets(thread, pset, new_pset);
2298
2299 old_empty = pset->empty;
2300 new_empty = new_pset->empty;
2301
2302 pset_unlock(pset);
2303
2304 /*
2305 * Reset policy and priorities if needed.
2306 */
2307 #if MACH_FIXPRI
2308 if (thread->policy & new_pset->policies == 0) {
2309 thread->policy = POLICY_TIMESHARE;
2310 recompute_pri = TRUE;
2311 }
2312 #endif /* MACH_FIXPRI */
2313
2314 if (thread->max_priority < new_pset->max_priority) {
2315 thread->max_priority = new_pset->max_priority;
2316 if (thread->priority < thread->max_priority) {
2317 thread->priority = thread->max_priority;
2318 recompute_pri = TRUE;
2319 }
2320 else {
2321 if ((thread->depress_priority >= 0) &&
2322 (thread->depress_priority < thread->max_priority)) {
2323 thread->depress_priority = thread->max_priority;
2324 }
2325 }
2326 }
2327
2328 pset_unlock(new_pset);
2329
2330 if (recompute_pri)
2331 compute_priority(thread, TRUE);
2332
2333 if (release_freeze) {
2334 thread->may_assign = TRUE;
2335 if (thread->assign_active) {
2336 thread->assign_active = FALSE;
2337 thread_wakeup((event_t)&thread->assign_active);
2338 }
2339 }
2340
2341 thread_unlock(thread);
2342 splx(s);
2343
2344 pset_deallocate(pset);
2345
2346 /*
2347 * Figure out hold status of thread. Threads assigned to empty
2348 * psets must be held. Therefore:
2349 * If old pset was empty release its hold.
2350 * Release our hold from above unless new pset is empty.
2351 */
2352
2353 if (old_empty)
2354 thread_release(thread);
2355 if (!new_empty)
2356 thread_release(thread);
2357
2358 /*
2359 * If current_thread is assigned, context switch to force
2360 * assignment to happen. This also causes hold to take
2361 * effect if the new pset is empty.
2362 */
2363 if (thread == current_thread()) {
2364 s = splsched();
2365 ast_on(cpu_number(), AST_BLOCK);
2366 (void) splx(s);
2367 }
2368 }
2369 #else /* MACH_HOST */
2370 kern_return_t
2371 thread_assign(
2372 thread_t thread,
2373 processor_set_t new_pset)
2374 {
2375 return KERN_FAILURE;
2376 }
2377 #endif /* MACH_HOST */
2378
2379 /*
2380 * thread_assign_default:
2381 *
2382 * Special version of thread_assign for assigning threads to default
2383 * processor set.
2384 */
2385 kern_return_t
2386 thread_assign_default(
2387 thread_t thread)
2388 {
2389 return thread_assign(thread, &default_pset);
2390 }
2391
2392 /*
2393 * thread_get_assignment
2394 *
2395 * Return current assignment for this thread.
2396 */
2397 kern_return_t thread_get_assignment(
2398 thread_t thread,
2399 processor_set_t *pset)
2400 {
2401 *pset = thread->processor_set;
2402 pset_reference(*pset);
2403 return KERN_SUCCESS;
2404 }
2405
2406 /*
2407 * thread_priority:
2408 *
2409 * Set priority (and possibly max priority) for thread.
2410 */
2411 kern_return_t
2412 thread_priority(
2413 thread_t thread,
2414 int priority,
2415 boolean_t set_max)
2416 {
2417 spl_t s;
2418 kern_return_t ret = KERN_SUCCESS;
2419
2420 if ((thread == THREAD_NULL) || invalid_pri(priority))
2421 return KERN_INVALID_ARGUMENT;
2422
2423 s = splsched();
2424 thread_lock(thread);
2425
2426 /*
2427 * Check for violation of max priority
2428 */
2429 if (priority < thread->max_priority) {
2430 ret = KERN_FAILURE;
2431 }
2432 else {
2433 /*
2434 * Set priorities. If a depression is in progress,
2435 * change the priority to restore.
2436 */
2437 if (thread->depress_priority >= 0) {
2438 thread->depress_priority = priority;
2439 }
2440 else {
2441 thread->priority = priority;
2442 compute_priority(thread, TRUE);
2443 }
2444
2445 if (set_max)
2446 thread->max_priority = priority;
2447 }
2448 thread_unlock(thread);
2449 (void) splx(s);
2450
2451 return ret;
2452 }
2453
2454 /*
2455 * thread_set_own_priority:
2456 *
2457 * Internal use only; sets the priority of the calling thread.
2458 * Will adjust max_priority if necessary.
2459 */
2460 void
2461 thread_set_own_priority(
2462 int priority)
2463 {
2464 spl_t s;
2465 thread_t thread = current_thread();
2466
2467 s = splsched();
2468 thread_lock(thread);
2469
2470 if (priority < thread->max_priority)
2471 thread->max_priority = priority;
2472 thread->priority = priority;
2473 compute_priority(thread, TRUE);
2474
2475 thread_unlock(thread);
2476 (void) splx(s);
2477 }
2478
2479 /*
2480 * thread_max_priority:
2481 *
2482 * Reset the max priority for a thread.
2483 */
2484 kern_return_t
2485 thread_max_priority(
2486 thread_t thread,
2487 processor_set_t pset,
2488 int max_priority)
2489 {
2490 spl_t s;
2491 kern_return_t ret = KERN_SUCCESS;
2492
2493 if ((thread == THREAD_NULL) || (pset == PROCESSOR_SET_NULL) ||
2494 invalid_pri(max_priority))
2495 return KERN_INVALID_ARGUMENT;
2496
2497 s = splsched();
2498 thread_lock(thread);
2499
2500 #if MACH_HOST
2501 /*
2502 * Check for wrong processor set.
2503 */
2504 if (pset != thread->processor_set) {
2505 ret = KERN_FAILURE;
2506 }
2507 else {
2508 #endif /* MACH_HOST */
2509 thread->max_priority = max_priority;
2510
2511 /*
2512 * Reset priority if it violates new max priority
2513 */
2514 if (max_priority > thread->priority) {
2515 thread->priority = max_priority;
2516
2517 compute_priority(thread, TRUE);
2518 }
2519 else {
2520 if (thread->depress_priority >= 0 &&
2521 max_priority > thread->depress_priority)
2522 thread->depress_priority = max_priority;
2523 }
2524 #if MACH_HOST
2525 }
2526 #endif /* MACH_HOST */
2527
2528 thread_unlock(thread);
2529 (void) splx(s);
2530
2531 return ret;
2532 }
2533
2534 /*
2535 * thread_policy:
2536 *
2537 * Set scheduling policy for thread.
2538 */
2539 kern_return_t
2540 thread_policy(
2541 thread_t thread,
2542 int policy,
2543 int data)
2544 {
2545 #if MACH_FIXPRI
2546 register kern_return_t ret = KERN_SUCCESS;
2547 register int temp;
2548 spl_t s;
2549 #endif /* MACH_FIXPRI */
2550
2551 if ((thread == THREAD_NULL) || invalid_policy(policy))
2552 return KERN_INVALID_ARGUMENT;
2553
2554 #if MACH_FIXPRI
2555 s = splsched();
2556 thread_lock(thread);
2557
2558 /*
2559 * Check if changing policy.
2560 */
2561 if (policy == thread->policy) {
2562 /*
2563 * Just changing data. This is meaningless for
2564 * timesharing, quantum for fixed priority (but
2565 * has no effect until current quantum runs out).
2566 */
2567 if (policy == POLICY_FIXEDPRI) {
2568 temp = data * 1000;
2569 if (temp % tick)
2570 temp += tick;
2571 thread->sched_data = temp/tick;
2572 }
2573 }
2574 else {
2575 /*
2576 * Changing policy. Check if new policy is allowed.
2577 */
2578 if ((thread->processor_set->policies & policy) == 0) {
2579 ret = KERN_FAILURE;
2580 }
2581 else {
2582 /*
2583 * Changing policy. Save data and calculate new
2584 * priority.
2585 */
2586 thread->policy = policy;
2587 if (policy == POLICY_FIXEDPRI) {
2588 temp = data * 1000;
2589 if (temp % tick)
2590 temp += tick;
2591 thread->sched_data = temp/tick;
2592 }
2593 compute_priority(thread, TRUE);
2594 }
2595 }
2596 thread_unlock(thread);
2597 (void) splx(s);
2598
2599 return ret;
2600 #else /* MACH_FIXPRI */
2601 if (policy == POLICY_TIMESHARE)
2602 return KERN_SUCCESS;
2603 else
2604 return KERN_FAILURE;
2605 #endif /* MACH_FIXPRI */
2606 }
2607
2608 /*
2609 * thread_wire:
2610 *
2611 * Specify that the target thread must always be able
2612 * to run and to allocate memory.
2613 */
2614 kern_return_t
2615 thread_wire(
2616 host_t host,
2617 thread_t thread,
2618 boolean_t wired)
2619 {
2620 spl_t s;
2621
2622 if (host == HOST_NULL)
2623 return KERN_INVALID_ARGUMENT;
2624
2625 if (thread == THREAD_NULL)
2626 return KERN_INVALID_ARGUMENT;
2627
2628 /*
2629 * This implementation only works for the current thread.
2630 * See stack_privilege.
2631 */
2632 if (thread != current_thread())
2633 return KERN_INVALID_ARGUMENT;
2634
2635 s = splsched();
2636 thread_lock(thread);
2637
2638 if (wired) {
2639 thread->vm_privilege = TRUE;
2640 stack_privilege(thread);
2641 }
2642 else {
2643 thread->vm_privilege = FALSE;
2644 /*XXX stack_unprivilege(thread); */
2645 thread->stack_privilege = 0;
2646 }
2647
2648 thread_unlock(thread);
2649 splx(s);
2650
2651 return KERN_SUCCESS;
2652 }
2653
2654 /*
2655 * thread_collect_scan:
2656 *
2657 * Attempt to free resources owned by threads.
2658 * pcb_collect doesn't do anything yet.
2659 */
2660
2661 void thread_collect_scan(void)
2662 {
2663 #if 0
2664 register thread_t thread, prev_thread;
2665 processor_set_t pset, prev_pset;
2666
2667 prev_thread = THREAD_NULL;
2668 prev_pset = PROCESSOR_SET_NULL;
2669
2670 simple_lock(&all_psets_lock);
2671 queue_iterate(&all_psets, pset, processor_set_t, all_psets) {
2672 pset_lock(pset);
2673 queue_iterate(&pset->threads, thread, thread_t, pset_threads) {
2674 spl_t s = splsched();
2675 thread_lock(thread);
2676
2677 /*
2678 * Only collect threads which are
2679 * not runnable and are swapped.
2680 */
2681
2682 if ((thread->state & (TH_RUN|TH_SWAPPED))
2683 == TH_SWAPPED) {
2684 thread->ref_count++;
2685 thread_unlock(thread);
2686 (void) splx(s);
2687 pset->ref_count++;
2688 pset_unlock(pset);
2689 simple_unlock(&all_psets_lock);
2690
2691 pcb_collect(thread);
2692
2693 if (prev_thread != THREAD_NULL)
2694 thread_deallocate(prev_thread);
2695 prev_thread = thread;
2696
2697 if (prev_pset != PROCESSOR_SET_NULL)
2698 pset_deallocate(prev_pset);
2699 prev_pset = pset;
2700
2701 simple_lock(&all_psets_lock);
2702 pset_lock(pset);
2703 } else {
2704 thread_unlock(thread);
2705 (void) splx(s);
2706 }
2707 }
2708 pset_unlock(pset);
2709 }
2710 simple_unlock(&all_psets_lock);
2711
2712 if (prev_thread != THREAD_NULL)
2713 thread_deallocate(prev_thread);
2714 if (prev_pset != PROCESSOR_SET_NULL)
2715 pset_deallocate(prev_pset);
2716 #endif /* 0 */
2717 }
2718
2719 boolean_t thread_collect_allowed = TRUE;
2720 unsigned thread_collect_last_tick = 0;
2721 unsigned thread_collect_max_rate = 0; /* in ticks */
2722
2723 /*
2724 * consider_thread_collect:
2725 *
2726 * Called by the pageout daemon when the system needs more free pages.
2727 */
2728
2729 void consider_thread_collect(void)
2730 {
2731 /*
2732 * By default, don't attempt thread collection more frequently
2733 * than once a second.
2734 */
2735
2736 if (thread_collect_max_rate == 0)
2737 thread_collect_max_rate = hz;
2738
2739 if (thread_collect_allowed &&
2740 (sched_tick >
2741 (thread_collect_last_tick + thread_collect_max_rate))) {
2742 thread_collect_last_tick = sched_tick;
2743 thread_collect_scan();
2744 }
2745 }
2746
2747 #if MACH_DEBUG
2748
2749 vm_size_t stack_usage(
2750 register vm_offset_t stack)
2751 {
2752 int i;
2753
2754 for (i = 0; i < KERNEL_STACK_SIZE/sizeof(unsigned int); i++)
2755 if (((unsigned int *)stack)[i] != STACK_MARKER)
2756 break;
2757
2758 return KERNEL_STACK_SIZE - i * sizeof(unsigned int);
2759 }
2760
2761 /*
2762 * Machine-dependent code should call stack_init
2763 * before doing its own initialization of the stack.
2764 */
2765
2766 void stack_init(
2767 register vm_offset_t stack)
2768 {
2769 if (stack_check_usage) {
2770 int i;
2771
2772 for (i = 0; i < KERNEL_STACK_SIZE/sizeof(unsigned int); i++)
2773 ((unsigned int *)stack)[i] = STACK_MARKER;
2774 }
2775 }
2776
2777 /*
2778 * Machine-dependent code should call stack_finalize
2779 * before releasing the stack memory.
2780 */
2781
2782 void stack_finalize(
2783 register vm_offset_t stack)
2784 {
2785 if (stack_check_usage) {
2786 vm_size_t used = stack_usage(stack);
2787
2788 simple_lock(&stack_usage_lock);
2789 if (used > stack_max_usage)
2790 stack_max_usage = used;
2791 simple_unlock(&stack_usage_lock);
2792 }
2793 }
2794
2795 #ifndef MACHINE_STACK
2796 /*
2797 * stack_statistics:
2798 *
2799 * Return statistics on cached kernel stacks.
2800 * *maxusagep must be initialized by the caller.
2801 */
2802
2803 void stack_statistics(
2804 natural_t *totalp,
2805 vm_size_t *maxusagep)
2806 {
2807 spl_t s;
2808
2809 s = splsched();
2810 stack_lock();
2811 if (stack_check_usage) {
2812 vm_offset_t stack;
2813
2814 /*
2815 * This is pretty expensive to do at splsched,
2816 * but it only happens when someone makes
2817 * a debugging call, so it should be OK.
2818 */
2819
2820 for (stack = stack_free_list; stack != 0;
2821 stack = stack_next(stack)) {
2822 vm_size_t usage = stack_usage(stack);
2823
2824 if (usage > *maxusagep)
2825 *maxusagep = usage;
2826 }
2827 }
2828
2829 *totalp = stack_free_count;
2830 stack_unlock();
2831 (void) splx(s);
2832 }
2833 #endif /* MACHINE_STACK */
2834
2835 kern_return_t host_stack_usage(
2836 host_t host,
2837 vm_size_t *reservedp,
2838 unsigned int *totalp,
2839 vm_size_t *spacep,
2840 vm_size_t *residentp,
2841 vm_size_t *maxusagep,
2842 vm_offset_t *maxstackp)
2843 {
2844 unsigned int total;
2845 vm_size_t maxusage;
2846
2847 if (host == HOST_NULL)
2848 return KERN_INVALID_HOST;
2849
2850 simple_lock(&stack_usage_lock);
2851 maxusage = stack_max_usage;
2852 simple_unlock(&stack_usage_lock);
2853
2854 stack_statistics(&total, &maxusage);
2855
2856 *reservedp = 0;
2857 *totalp = total;
2858 *spacep = *residentp = total * round_page(KERNEL_STACK_SIZE);
2859 *maxusagep = maxusage;
2860 *maxstackp = 0;
2861 return KERN_SUCCESS;
2862 }
2863
2864 kern_return_t processor_set_stack_usage(
2865 processor_set_t pset,
2866 unsigned int *totalp,
2867 vm_size_t *spacep,
2868 vm_size_t *residentp,
2869 vm_size_t *maxusagep,
2870 vm_offset_t *maxstackp)
2871 {
2872 unsigned int total;
2873 vm_size_t maxusage;
2874 vm_offset_t maxstack;
2875
2876 register thread_t *threads;
2877 register thread_t tmp_thread;
2878
2879 unsigned int actual; /* this many things */
2880 unsigned int i;
2881
2882 vm_size_t size, size_needed;
2883 vm_offset_t addr;
2884
2885 if (pset == PROCESSOR_SET_NULL)
2886 return KERN_INVALID_ARGUMENT;
2887
2888 size = 0; addr = 0;
2889
2890 for (;;) {
2891 pset_lock(pset);
2892 if (!pset->active) {
2893 pset_unlock(pset);
2894 return KERN_INVALID_ARGUMENT;
2895 }
2896
2897 actual = pset->thread_count;
2898
2899 /* do we have the memory we need? */
2900
2901 size_needed = actual * sizeof(thread_t);
2902 if (size_needed <= size)
2903 break;
2904
2905 /* unlock the pset and allocate more memory */
2906 pset_unlock(pset);
2907
2908 if (size != 0)
2909 kfree(addr, size);
2910
2911 assert(size_needed > 0);
2912 size = size_needed;
2913
2914 addr = kalloc(size);
2915 if (addr == 0)
2916 return KERN_RESOURCE_SHORTAGE;
2917 }
2918
2919 /* OK, have memory and the processor_set is locked & active */
2920
2921 threads = (thread_t *) addr;
2922 for (i = 0, tmp_thread = (thread_t) queue_first(&pset->threads);
2923 i < actual;
2924 i++,
2925 tmp_thread = (thread_t) queue_next(&tmp_thread->pset_threads)) {
2926 thread_reference(tmp_thread);
2927 threads[i] = tmp_thread;
2928 }
2929 assert(queue_end(&pset->threads, (queue_entry_t) tmp_thread));
2930
2931 /* can unlock processor set now that we have the thread refs */
2932 pset_unlock(pset);
2933
2934 /* calculate maxusage and free thread references */
2935
2936 total = 0;
2937 maxusage = 0;
2938 maxstack = 0;
2939 for (i = 0; i < actual; i++) {
2940 thread_t thread = threads[i];
2941 vm_offset_t stack = 0;
2942
2943 /*
2944 * thread->kernel_stack is only accurate if the
2945 * thread isn't swapped and is not executing.
2946 *
2947 * Of course, we don't have the appropriate locks
2948 * for these shenanigans.
2949 */
2950
2951 if ((thread->state & TH_SWAPPED) == 0) {
2952 int cpu;
2953
2954 stack = thread->kernel_stack;
2955
2956 for (cpu = 0; cpu < NCPUS; cpu++)
2957 if (active_threads[cpu] == thread) {
2958 stack = active_stacks[cpu];
2959 break;
2960 }
2961 }
2962
2963 if (stack != 0) {
2964 total++;
2965
2966 if (stack_check_usage) {
2967 vm_size_t usage = stack_usage(stack);
2968
2969 if (usage > maxusage) {
2970 maxusage = usage;
2971 maxstack = (vm_offset_t) thread;
2972 }
2973 }
2974 }
2975
2976 thread_deallocate(thread);
2977 }
2978
2979 if (size != 0)
2980 kfree(addr, size);
2981
2982 *totalp = total;
2983 *residentp = *spacep = total * round_page(KERNEL_STACK_SIZE);
2984 *maxusagep = maxusage;
2985 *maxstackp = maxstack;
2986 return KERN_SUCCESS;
2987 }
2988
2989 /*
2990 * Useful in the debugger:
2991 */
2992 void
2993 thread_stats(void)
2994 {
2995 register thread_t thread;
2996 int total = 0, rpcreply = 0;
2997
2998 queue_iterate(&default_pset.threads, thread, thread_t, pset_threads) {
2999 total++;
3000 if (thread->ith_rpc_reply != IP_NULL)
3001 rpcreply++;
3002 }
3003
3004 printf("%d total threads.\n", total);
3005 printf("%d using rpc_reply.\n", rpcreply);
3006 }
3007 #endif /* MACH_DEBUG */
Cache object: c1345f8d906783d14f8d2b3ed7d1ee5b
|