FreeBSD/Linux Kernel Cross Reference
sys/common/os/lwp.c
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #include <sys/param.h>
28 #include <sys/types.h>
29 #include <sys/sysmacros.h>
30 #include <sys/systm.h>
31 #include <sys/thread.h>
32 #include <sys/proc.h>
33 #include <sys/task.h>
34 #include <sys/project.h>
35 #include <sys/signal.h>
36 #include <sys/errno.h>
37 #include <sys/vmparam.h>
38 #include <sys/stack.h>
39 #include <sys/procfs.h>
40 #include <sys/prsystm.h>
41 #include <sys/cpuvar.h>
42 #include <sys/kmem.h>
43 #include <sys/vtrace.h>
44 #include <sys/door.h>
45 #include <vm/seg_kp.h>
46 #include <sys/debug.h>
47 #include <sys/tnf.h>
48 #include <sys/schedctl.h>
49 #include <sys/poll.h>
50 #include <sys/copyops.h>
51 #include <sys/lwp_upimutex_impl.h>
52 #include <sys/cpupart.h>
53 #include <sys/lgrp.h>
54 #include <sys/rctl.h>
55 #include <sys/contract_impl.h>
56 #include <sys/cpc_impl.h>
57 #include <sys/sdt.h>
58 #include <sys/cmn_err.h>
59 #include <sys/brand.h>
60 #include <sys/cyclic.h>
61 #include <sys/pool.h>
62
63 /* hash function for the lwpid hash table, p->p_tidhash[] */
64 #define TIDHASH(tid, hash_sz) ((tid) & ((hash_sz) - 1))
65
66 void *segkp_lwp; /* cookie for pool of segkp resources */
67 extern void reapq_move_lq_to_tq(kthread_t *);
68 extern void freectx_ctx(struct ctxop *);
69
70 /*
71 * Create a kernel thread associated with a particular system process. Give
72 * it an LWP so that microstate accounting will be available for it.
73 */
74 kthread_t *
75 lwp_kernel_create(proc_t *p, void (*proc)(), void *arg, int state, pri_t pri)
76 {
77 klwp_t *lwp;
78
79 VERIFY((p->p_flag & SSYS) != 0);
80
81 lwp = lwp_create(proc, arg, 0, p, state, pri, &t0.t_hold, syscid, 0);
82
83 VERIFY(lwp != NULL);
84
85 return (lwptot(lwp));
86 }
87
88 /*
89 * Create a thread that appears to be stopped at sys_rtt.
90 */
91 klwp_t *
92 lwp_create(void (*proc)(), caddr_t arg, size_t len, proc_t *p,
93 int state, int pri, const k_sigset_t *smask, int cid, id_t lwpid)
94 {
95 klwp_t *lwp = NULL;
96 kthread_t *t;
97 kthread_t *tx;
98 cpupart_t *oldpart = NULL;
99 size_t stksize;
100 caddr_t lwpdata = NULL;
101 processorid_t binding;
102 int err = 0;
103 kproject_t *oldkpj, *newkpj;
104 void *bufp = NULL;
105 klwp_t *curlwp;
106 lwpent_t *lep;
107 lwpdir_t *old_dir = NULL;
108 uint_t old_dirsz = 0;
109 tidhash_t *old_hash = NULL;
110 uint_t old_hashsz = 0;
111 ret_tidhash_t *ret_tidhash = NULL;
112 int i;
113 int rctlfail = 0;
114 boolean_t branded = 0;
115 struct ctxop *ctx = NULL;
116
117 ASSERT(cid != sysdccid); /* system threads must start in SYS */
118
119 ASSERT(p != &p0); /* No new LWPs in p0. */
120
121 mutex_enter(&p->p_lock);
122 mutex_enter(&p->p_zone->zone_nlwps_lock);
123 /*
124 * don't enforce rctl limits on system processes
125 */
126 if (!CLASS_KERNEL(cid)) {
127 if (p->p_task->tk_nlwps >= p->p_task->tk_nlwps_ctl)
128 if (rctl_test(rc_task_lwps, p->p_task->tk_rctls, p,
129 1, 0) & RCT_DENY)
130 rctlfail = 1;
131 if (p->p_task->tk_proj->kpj_nlwps >=
132 p->p_task->tk_proj->kpj_nlwps_ctl)
133 if (rctl_test(rc_project_nlwps,
134 p->p_task->tk_proj->kpj_rctls, p, 1, 0)
135 & RCT_DENY)
136 rctlfail = 1;
137 if (p->p_zone->zone_nlwps >= p->p_zone->zone_nlwps_ctl)
138 if (rctl_test(rc_zone_nlwps, p->p_zone->zone_rctls, p,
139 1, 0) & RCT_DENY)
140 rctlfail = 1;
141 }
142 if (rctlfail) {
143 mutex_exit(&p->p_zone->zone_nlwps_lock);
144 mutex_exit(&p->p_lock);
145 return (NULL);
146 }
147 p->p_task->tk_nlwps++;
148 p->p_task->tk_proj->kpj_nlwps++;
149 p->p_zone->zone_nlwps++;
150 mutex_exit(&p->p_zone->zone_nlwps_lock);
151 mutex_exit(&p->p_lock);
152
153 curlwp = ttolwp(curthread);
154 if (curlwp == NULL || (stksize = curlwp->lwp_childstksz) == 0)
155 stksize = lwp_default_stksize;
156
157 if (CLASS_KERNEL(cid)) {
158 /*
159 * Since we are creating an LWP in an SSYS process, we do not
160 * inherit anything from the current thread's LWP. We set
161 * stksize and lwpdata to 0 in order to let thread_create()
162 * allocate a regular kernel thread stack for this thread.
163 */
164 curlwp = NULL;
165 stksize = 0;
166 lwpdata = NULL;
167
168 } else if (stksize == lwp_default_stksize) {
169 /*
170 * Try to reuse an <lwp,stack> from the LWP deathrow.
171 */
172 if (lwp_reapcnt > 0) {
173 mutex_enter(&reaplock);
174 if ((t = lwp_deathrow) != NULL) {
175 ASSERT(t->t_swap);
176 lwp_deathrow = t->t_forw;
177 lwp_reapcnt--;
178 lwpdata = t->t_swap;
179 lwp = t->t_lwp;
180 ctx = t->t_ctx;
181 t->t_swap = NULL;
182 t->t_lwp = NULL;
183 t->t_ctx = NULL;
184 reapq_move_lq_to_tq(t);
185 }
186 mutex_exit(&reaplock);
187 if (lwp != NULL) {
188 lwp_stk_fini(lwp);
189 }
190 if (ctx != NULL) {
191 freectx_ctx(ctx);
192 }
193 }
194 if (lwpdata == NULL &&
195 (lwpdata = (caddr_t)segkp_cache_get(segkp_lwp)) == NULL) {
196 mutex_enter(&p->p_lock);
197 mutex_enter(&p->p_zone->zone_nlwps_lock);
198 p->p_task->tk_nlwps--;
199 p->p_task->tk_proj->kpj_nlwps--;
200 p->p_zone->zone_nlwps--;
201 mutex_exit(&p->p_zone->zone_nlwps_lock);
202 mutex_exit(&p->p_lock);
203 return (NULL);
204 }
205 } else {
206 stksize = roundup(stksize, PAGESIZE);
207 if ((lwpdata = (caddr_t)segkp_get(segkp, stksize,
208 (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED))) == NULL) {
209 mutex_enter(&p->p_lock);
210 mutex_enter(&p->p_zone->zone_nlwps_lock);
211 p->p_task->tk_nlwps--;
212 p->p_task->tk_proj->kpj_nlwps--;
213 p->p_zone->zone_nlwps--;
214 mutex_exit(&p->p_zone->zone_nlwps_lock);
215 mutex_exit(&p->p_lock);
216 return (NULL);
217 }
218 }
219
220 /*
221 * Create a thread, initializing the stack pointer
222 */
223 t = thread_create(lwpdata, stksize, NULL, NULL, 0, p, TS_STOPPED, pri);
224
225 /*
226 * If a non-NULL stack base is passed in, thread_create() assumes
227 * that the stack might be statically allocated (as opposed to being
228 * allocated from segkp), and so it does not set t_swap. Since
229 * the lwpdata was allocated from segkp, we must set t_swap to point
230 * to it ourselves.
231 *
232 * This would be less confusing if t_swap had a better name; it really
233 * indicates that the stack is allocated from segkp, regardless of
234 * whether or not it is swappable.
235 */
236 if (lwpdata != NULL) {
237 ASSERT(!CLASS_KERNEL(cid));
238 ASSERT(t->t_swap == NULL);
239 t->t_swap = lwpdata; /* Start of page-able data */
240 }
241
242 /*
243 * If the stack and lwp can be reused, mark the thread as such.
244 * When we get to reapq_add() from resume_from_zombie(), these
245 * threads will go onto lwp_deathrow instead of thread_deathrow.
246 */
247 if (!CLASS_KERNEL(cid) && stksize == lwp_default_stksize)
248 t->t_flag |= T_LWPREUSE;
249
250 if (lwp == NULL)
251 lwp = kmem_cache_alloc(lwp_cache, KM_SLEEP);
252 bzero(lwp, sizeof (*lwp));
253 t->t_lwp = lwp;
254
255 t->t_hold = *smask;
256 lwp->lwp_thread = t;
257 lwp->lwp_procp = p;
258 lwp->lwp_sigaltstack.ss_flags = SS_DISABLE;
259 if (curlwp != NULL && curlwp->lwp_childstksz != 0)
260 lwp->lwp_childstksz = curlwp->lwp_childstksz;
261
262 t->t_stk = lwp_stk_init(lwp, t->t_stk);
263 thread_load(t, proc, arg, len);
264
265 /*
266 * Allocate the SIGPROF buffer if ITIMER_REALPROF is in effect.
267 */
268 if (p->p_rprof_cyclic != CYCLIC_NONE)
269 t->t_rprof = kmem_zalloc(sizeof (struct rprof), KM_SLEEP);
270
271 if (cid != NOCLASS)
272 (void) CL_ALLOC(&bufp, cid, KM_SLEEP);
273
274 /*
275 * Allocate an lwp directory entry for the new lwp.
276 */
277 lep = kmem_zalloc(sizeof (*lep), KM_SLEEP);
278
279 mutex_enter(&p->p_lock);
280 grow:
281 /*
282 * Grow the lwp (thread) directory and lwpid hash table if necessary.
283 * A note on the growth algorithm:
284 * The new lwp directory size is computed as:
285 * new = 2 * old + 2
286 * Starting with an initial size of 2 (see exec_common()),
287 * this yields numbers that are a power of two minus 2:
288 * 2, 6, 14, 30, 62, 126, 254, 510, 1022, ...
289 * The size of the lwpid hash table must be a power of two
290 * and must be commensurate in size with the lwp directory
291 * so that hash bucket chains remain short. Therefore,
292 * the lwpid hash table size is computed as:
293 * hashsz = (dirsz + 2) / 2
294 * which leads to these hash table sizes corresponding to
295 * the above directory sizes:
296 * 2, 4, 8, 16, 32, 64, 128, 256, 512, ...
297 * A note on growing the hash table:
298 * For performance reasons, code in lwp_unpark() does not
299 * acquire curproc->p_lock when searching the hash table.
300 * Rather, it calls lwp_hash_lookup_and_lock() which
301 * acquires only the individual hash bucket lock, taking
302 * care to deal with reallocation of the hash table
303 * during the time it takes to acquire the lock.
304 *
305 * This is sufficient to protect the integrity of the
306 * hash table, but it requires us to acquire all of the
307 * old hash bucket locks before growing the hash table
308 * and to release them afterwards. It also requires us
309 * not to free the old hash table because some thread
310 * in lwp_hash_lookup_and_lock() might still be trying
311 * to acquire the old bucket lock.
312 *
313 * So we adopt the tactic of keeping all of the retired
314 * hash tables on a linked list, so they can be safely
315 * freed when the process exits or execs.
316 *
317 * Because the hash table grows in powers of two, the
318 * total size of all of the hash tables will be slightly
319 * less than twice the size of the largest hash table.
320 */
321 while (p->p_lwpfree == NULL) {
322 uint_t dirsz = p->p_lwpdir_sz;
323 lwpdir_t *new_dir;
324 uint_t new_dirsz;
325 lwpdir_t *ldp;
326 tidhash_t *new_hash;
327 uint_t new_hashsz;
328
329 mutex_exit(&p->p_lock);
330
331 /*
332 * Prepare to remember the old p_tidhash for later
333 * kmem_free()ing when the process exits or execs.
334 */
335 if (ret_tidhash == NULL)
336 ret_tidhash = kmem_zalloc(sizeof (ret_tidhash_t),
337 KM_SLEEP);
338 if (old_dir != NULL)
339 kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
340 if (old_hash != NULL)
341 kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
342
343 new_dirsz = 2 * dirsz + 2;
344 new_dir = kmem_zalloc(new_dirsz * sizeof (lwpdir_t), KM_SLEEP);
345 for (ldp = new_dir, i = 1; i < new_dirsz; i++, ldp++)
346 ldp->ld_next = ldp + 1;
347 new_hashsz = (new_dirsz + 2) / 2;
348 new_hash = kmem_zalloc(new_hashsz * sizeof (tidhash_t),
349 KM_SLEEP);
350
351 mutex_enter(&p->p_lock);
352 if (p == curproc)
353 prbarrier(p);
354
355 if (dirsz != p->p_lwpdir_sz || p->p_lwpfree != NULL) {
356 /*
357 * Someone else beat us to it or some lwp exited.
358 * Set up to free our memory and take a lap.
359 */
360 old_dir = new_dir;
361 old_dirsz = new_dirsz;
362 old_hash = new_hash;
363 old_hashsz = new_hashsz;
364 } else {
365 /*
366 * For the benefit of lwp_hash_lookup_and_lock(),
367 * called from lwp_unpark(), which searches the
368 * tid hash table without acquiring p->p_lock,
369 * we must acquire all of the tid hash table
370 * locks before replacing p->p_tidhash.
371 */
372 old_hash = p->p_tidhash;
373 old_hashsz = p->p_tidhash_sz;
374 for (i = 0; i < old_hashsz; i++) {
375 mutex_enter(&old_hash[i].th_lock);
376 mutex_enter(&new_hash[i].th_lock);
377 }
378
379 /*
380 * We simply hash in all of the old directory entries.
381 * This works because the old directory has no empty
382 * slots and the new hash table starts out empty.
383 * This reproduces the original directory ordering
384 * (required for /proc directory semantics).
385 */
386 old_dir = p->p_lwpdir;
387 old_dirsz = p->p_lwpdir_sz;
388 p->p_lwpdir = new_dir;
389 p->p_lwpfree = new_dir;
390 p->p_lwpdir_sz = new_dirsz;
391 for (ldp = old_dir, i = 0; i < old_dirsz; i++, ldp++)
392 lwp_hash_in(p, ldp->ld_entry,
393 new_hash, new_hashsz, 0);
394
395 /*
396 * Remember the old hash table along with all
397 * of the previously-remembered hash tables.
398 * We will free them at process exit or exec.
399 */
400 ret_tidhash->rth_tidhash = old_hash;
401 ret_tidhash->rth_tidhash_sz = old_hashsz;
402 ret_tidhash->rth_next = p->p_ret_tidhash;
403 p->p_ret_tidhash = ret_tidhash;
404
405 /*
406 * Now establish the new tid hash table.
407 * As soon as we assign p->p_tidhash,
408 * code in lwp_unpark() can start using it.
409 */
410 membar_producer();
411 p->p_tidhash = new_hash;
412
413 /*
414 * It is necessary that p_tidhash reach global
415 * visibility before p_tidhash_sz. Otherwise,
416 * code in lwp_hash_lookup_and_lock() could
417 * index into the old p_tidhash using the new
418 * p_tidhash_sz and thereby access invalid data.
419 */
420 membar_producer();
421 p->p_tidhash_sz = new_hashsz;
422
423 /*
424 * Release the locks; allow lwp_unpark() to carry on.
425 */
426 for (i = 0; i < old_hashsz; i++) {
427 mutex_exit(&old_hash[i].th_lock);
428 mutex_exit(&new_hash[i].th_lock);
429 }
430
431 /*
432 * Avoid freeing these objects below.
433 */
434 ret_tidhash = NULL;
435 old_hash = NULL;
436 old_hashsz = 0;
437 }
438 }
439
440 /*
441 * Block the process against /proc while we manipulate p->p_tlist,
442 * unless lwp_create() was called by /proc for the PCAGENT operation.
443 * We want to do this early enough so that we don't drop p->p_lock
444 * until the thread is put on the p->p_tlist.
445 */
446 if (p == curproc) {
447 prbarrier(p);
448 /*
449 * If the current lwp has been requested to stop, do so now.
450 * Otherwise we have a race condition between /proc attempting
451 * to stop the process and this thread creating a new lwp
452 * that was not seen when the /proc PCSTOP request was issued.
453 * We rely on stop() to call prbarrier(p) before returning.
454 */
455 while ((curthread->t_proc_flag & TP_PRSTOP) &&
456 !ttolwp(curthread)->lwp_nostop) {
457 /*
458 * We called pool_barrier_enter() before calling
459 * here to lwp_create(). We have to call
460 * pool_barrier_exit() before stopping.
461 */
462 pool_barrier_exit();
463 prbarrier(p);
464 stop(PR_REQUESTED, 0);
465 /*
466 * And we have to repeat the call to
467 * pool_barrier_enter after stopping.
468 */
469 pool_barrier_enter();
470 prbarrier(p);
471 }
472
473 /*
474 * If process is exiting, there could be a race between
475 * the agent lwp creation and the new lwp currently being
476 * created. So to prevent this race lwp creation is failed
477 * if the process is exiting.
478 */
479 if (p->p_flag & (SEXITLWPS|SKILLED)) {
480 err = 1;
481 goto error;
482 }
483
484 /*
485 * Since we might have dropped p->p_lock, the
486 * lwp directory free list might have changed.
487 */
488 if (p->p_lwpfree == NULL)
489 goto grow;
490 }
491
492 kpreempt_disable(); /* can't grab cpu_lock here */
493
494 /*
495 * Inherit processor and processor set bindings from curthread.
496 *
497 * For kernel LWPs, we do not inherit processor set bindings at
498 * process creation time (i.e. when p != curproc). After the
499 * kernel process is created, any subsequent LWPs must be created
500 * by threads in the kernel process, at which point we *will*
501 * inherit processor set bindings.
502 */
503 if (CLASS_KERNEL(cid) && p != curproc) {
504 t->t_bind_cpu = binding = PBIND_NONE;
505 t->t_cpupart = oldpart = &cp_default;
506 t->t_bind_pset = PS_NONE;
507 t->t_bindflag = (uchar_t)default_binding_mode;
508 } else {
509 binding = curthread->t_bind_cpu;
510 t->t_bind_cpu = binding;
511 oldpart = t->t_cpupart;
512 t->t_cpupart = curthread->t_cpupart;
513 t->t_bind_pset = curthread->t_bind_pset;
514 t->t_bindflag = curthread->t_bindflag |
515 (uchar_t)default_binding_mode;
516 }
517
518 /*
519 * thread_create() initializes this thread's home lgroup to the root.
520 * Choose a more suitable lgroup, since this thread is associated
521 * with an lwp.
522 */
523 ASSERT(oldpart != NULL);
524 if (binding != PBIND_NONE && t->t_affinitycnt == 0) {
525 t->t_bound_cpu = cpu[binding];
526 if (t->t_lpl != t->t_bound_cpu->cpu_lpl)
527 lgrp_move_thread(t, t->t_bound_cpu->cpu_lpl, 1);
528 } else if (CLASS_KERNEL(cid)) {
529 /*
530 * Kernel threads are always in the root lgrp.
531 */
532 lgrp_move_thread(t,
533 &t->t_cpupart->cp_lgrploads[LGRP_ROOTID], 1);
534 } else {
535 lgrp_move_thread(t, lgrp_choose(t, t->t_cpupart), 1);
536 }
537
538 kpreempt_enable();
539
540 /*
541 * make sure lpl points to our own partition
542 */
543 ASSERT(t->t_lpl >= t->t_cpupart->cp_lgrploads);
544 ASSERT(t->t_lpl < t->t_cpupart->cp_lgrploads +
545 t->t_cpupart->cp_nlgrploads);
546
547 /*
548 * It is safe to point the thread to the new project without holding it
549 * since we're holding the target process' p_lock here and therefore
550 * we're guaranteed that it will not move to another project.
551 */
552 newkpj = p->p_task->tk_proj;
553 oldkpj = ttoproj(t);
554 if (newkpj != oldkpj) {
555 t->t_proj = newkpj;
556 (void) project_hold(newkpj);
557 project_rele(oldkpj);
558 }
559
560 if (cid != NOCLASS) {
561 /*
562 * If the lwp is being created in the current process
563 * and matches the current thread's scheduling class,
564 * we should propagate the current thread's scheduling
565 * parameters by calling CL_FORK. Otherwise just use
566 * the defaults by calling CL_ENTERCLASS.
567 */
568 if (p != curproc || curthread->t_cid != cid) {
569 err = CL_ENTERCLASS(t, cid, NULL, NULL, bufp);
570 t->t_pri = pri; /* CL_ENTERCLASS may have changed it */
571 /*
572 * We don't call schedctl_set_cidpri(t) here
573 * because the schedctl data is not yet set
574 * up for the newly-created lwp.
575 */
576 } else {
577 t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
578 err = CL_FORK(curthread, t, bufp);
579 t->t_cid = cid;
580 }
581 if (err)
582 goto error;
583 else
584 bufp = NULL;
585 }
586
587 /*
588 * If we were given an lwpid then use it, else allocate one.
589 */
590 if (lwpid != 0)
591 t->t_tid = lwpid;
592 else {
593 /*
594 * lwp/thread id 0 is never valid; reserved for special checks.
595 * lwp/thread id 1 is reserved for the main thread.
596 * Start again at 2 when INT_MAX has been reached
597 * (id_t is a signed 32-bit integer).
598 */
599 id_t prev_id = p->p_lwpid; /* last allocated tid */
600
601 do { /* avoid lwpid duplication */
602 if (p->p_lwpid == INT_MAX) {
603 p->p_flag |= SLWPWRAP;
604 p->p_lwpid = 1;
605 }
606 if ((t->t_tid = ++p->p_lwpid) == prev_id) {
607 /*
608 * All lwpids are allocated; fail the request.
609 */
610 err = 1;
611 goto error;
612 }
613 /*
614 * We only need to worry about colliding with an id
615 * that's already in use if this process has
616 * cycled through all available lwp ids.
617 */
618 if ((p->p_flag & SLWPWRAP) == 0)
619 break;
620 } while (lwp_hash_lookup(p, t->t_tid) != NULL);
621 }
622
623 /*
624 * If this is a branded process, let the brand do any necessary lwp
625 * initialization.
626 */
627 if (PROC_IS_BRANDED(p)) {
628 if (BROP(p)->b_initlwp(lwp)) {
629 err = 1;
630 goto error;
631 }
632 branded = 1;
633 }
634
635 if (t->t_tid == 1) {
636 kpreempt_disable();
637 ASSERT(t->t_lpl != NULL);
638 p->p_t1_lgrpid = t->t_lpl->lpl_lgrpid;
639 kpreempt_enable();
640 if (p->p_tr_lgrpid != LGRP_NONE &&
641 p->p_tr_lgrpid != p->p_t1_lgrpid) {
642 lgrp_update_trthr_migrations(1);
643 }
644 }
645
646 p->p_lwpcnt++;
647 t->t_waitfor = -1;
648
649 /*
650 * Turn microstate accounting on for thread if on for process.
651 */
652 if (p->p_flag & SMSACCT)
653 t->t_proc_flag |= TP_MSACCT;
654
655 /*
656 * If the process has watchpoints, mark the new thread as such.
657 */
658 if (pr_watch_active(p))
659 watch_enable(t);
660
661 /*
662 * The lwp is being created in the stopped state.
663 * We set all the necessary flags to indicate that fact here.
664 * We omit the TS_CREATE flag from t_schedflag so that the lwp
665 * cannot be set running until the caller is finished with it,
666 * even if lwp_continue() is called on it after we drop p->p_lock.
667 * When the caller is finished with the newly-created lwp,
668 * the caller must call lwp_create_done() to allow the lwp
669 * to be set running. If the TP_HOLDLWP is left set, the
670 * lwp will suspend itself after reaching system call exit.
671 */
672 init_mstate(t, LMS_STOPPED);
673 t->t_proc_flag |= TP_HOLDLWP;
674 t->t_schedflag |= (TS_ALLSTART & ~(TS_CSTART | TS_CREATE));
675 t->t_whystop = PR_SUSPENDED;
676 t->t_whatstop = SUSPEND_NORMAL;
677 t->t_sig_check = 1; /* ensure that TP_HOLDLWP is honored */
678
679 /*
680 * Set system call processing flags in case tracing or profiling
681 * is set. The first system call will evaluate these and turn
682 * them off if they aren't needed.
683 */
684 t->t_pre_sys = 1;
685 t->t_post_sys = 1;
686
687 /*
688 * Insert the new thread into the list of all threads.
689 */
690 if ((tx = p->p_tlist) == NULL) {
691 t->t_back = t;
692 t->t_forw = t;
693 p->p_tlist = t;
694 } else {
695 t->t_forw = tx;
696 t->t_back = tx->t_back;
697 tx->t_back->t_forw = t;
698 tx->t_back = t;
699 }
700
701 /*
702 * Insert the new lwp into an lwp directory slot position
703 * and into the lwpid hash table.
704 */
705 lep->le_thread = t;
706 lep->le_lwpid = t->t_tid;
707 lep->le_start = t->t_start;
708 lwp_hash_in(p, lep, p->p_tidhash, p->p_tidhash_sz, 1);
709
710 if (state == TS_RUN) {
711 /*
712 * We set the new lwp running immediately.
713 */
714 t->t_proc_flag &= ~TP_HOLDLWP;
715 lwp_create_done(t);
716 }
717
718 error:
719 if (err) {
720 if (CLASS_KERNEL(cid)) {
721 /*
722 * This should only happen if a system process runs
723 * out of lwpids, which shouldn't occur.
724 */
725 panic("Failed to create a system LWP");
726 }
727 /*
728 * We have failed to create an lwp, so decrement the number
729 * of lwps in the task and let the lgroup load averages know
730 * that this thread isn't going to show up.
731 */
732 kpreempt_disable();
733 lgrp_move_thread(t, NULL, 1);
734 kpreempt_enable();
735
736 ASSERT(MUTEX_HELD(&p->p_lock));
737 mutex_enter(&p->p_zone->zone_nlwps_lock);
738 p->p_task->tk_nlwps--;
739 p->p_task->tk_proj->kpj_nlwps--;
740 p->p_zone->zone_nlwps--;
741 mutex_exit(&p->p_zone->zone_nlwps_lock);
742 if (cid != NOCLASS && bufp != NULL)
743 CL_FREE(cid, bufp);
744
745 if (branded)
746 BROP(p)->b_freelwp(lwp);
747
748 mutex_exit(&p->p_lock);
749 t->t_state = TS_FREE;
750 thread_rele(t);
751
752 /*
753 * We need to remove t from the list of all threads
754 * because thread_exit()/lwp_exit() isn't called on t.
755 */
756 mutex_enter(&pidlock);
757 ASSERT(t != t->t_next); /* t0 never exits */
758 t->t_next->t_prev = t->t_prev;
759 t->t_prev->t_next = t->t_next;
760 mutex_exit(&pidlock);
761
762 thread_free(t);
763 kmem_free(lep, sizeof (*lep));
764 lwp = NULL;
765 } else {
766 mutex_exit(&p->p_lock);
767 }
768
769 if (old_dir != NULL)
770 kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
771 if (old_hash != NULL)
772 kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
773 if (ret_tidhash != NULL)
774 kmem_free(ret_tidhash, sizeof (ret_tidhash_t));
775
776 DTRACE_PROC1(lwp__create, kthread_t *, t);
777 return (lwp);
778 }
779
780 /*
781 * lwp_create_done() is called by the caller of lwp_create() to set the
782 * newly-created lwp running after the caller has finished manipulating it.
783 */
784 void
785 lwp_create_done(kthread_t *t)
786 {
787 proc_t *p = ttoproc(t);
788
789 ASSERT(MUTEX_HELD(&p->p_lock));
790
791 /*
792 * We set the TS_CREATE and TS_CSTART flags and call setrun_locked().
793 * (The absence of the TS_CREATE flag prevents the lwp from running
794 * until we are finished with it, even if lwp_continue() is called on
795 * it by some other lwp in the process or elsewhere in the kernel.)
796 */
797 thread_lock(t);
798 ASSERT(t->t_state == TS_STOPPED && !(t->t_schedflag & TS_CREATE));
799 /*
800 * If TS_CSTART is set, lwp_continue(t) has been called and
801 * has already incremented p_lwprcnt; avoid doing this twice.
802 */
803 if (!(t->t_schedflag & TS_CSTART))
804 p->p_lwprcnt++;
805 t->t_schedflag |= (TS_CSTART | TS_CREATE);
806 setrun_locked(t);
807 thread_unlock(t);
808 }
809
810 /*
811 * Copy an LWP's active templates, and clear the latest contracts.
812 */
813 void
814 lwp_ctmpl_copy(klwp_t *dst, klwp_t *src)
815 {
816 int i;
817
818 for (i = 0; i < ct_ntypes; i++) {
819 dst->lwp_ct_active[i] = ctmpl_dup(src->lwp_ct_active[i]);
820 dst->lwp_ct_latest[i] = NULL;
821 }
822 }
823
824 /*
825 * Clear an LWP's contract template state.
826 */
827 void
828 lwp_ctmpl_clear(klwp_t *lwp)
829 {
830 ct_template_t *tmpl;
831 int i;
832
833 for (i = 0; i < ct_ntypes; i++) {
834 if ((tmpl = lwp->lwp_ct_active[i]) != NULL) {
835 ctmpl_free(tmpl);
836 lwp->lwp_ct_active[i] = NULL;
837 }
838
839 if (lwp->lwp_ct_latest[i] != NULL) {
840 contract_rele(lwp->lwp_ct_latest[i]);
841 lwp->lwp_ct_latest[i] = NULL;
842 }
843 }
844 }
845
846 /*
847 * Individual lwp exit.
848 * If this is the last lwp, exit the whole process.
849 */
850 void
851 lwp_exit(void)
852 {
853 kthread_t *t = curthread;
854 klwp_t *lwp = ttolwp(t);
855 proc_t *p = ttoproc(t);
856
857 ASSERT(MUTEX_HELD(&p->p_lock));
858
859 mutex_exit(&p->p_lock);
860
861 #if defined(__sparc)
862 /*
863 * Ensure that the user stack is fully abandoned..
864 */
865 trash_user_windows();
866 #endif
867
868 tsd_exit(); /* free thread specific data */
869
870 kcpc_passivate(); /* Clean up performance counter state */
871
872 pollcleanup();
873
874 if (t->t_door)
875 door_slam();
876
877 if (t->t_schedctl != NULL)
878 schedctl_lwp_cleanup(t);
879
880 if (t->t_upimutex != NULL)
881 upimutex_cleanup();
882
883 /*
884 * Perform any brand specific exit processing, then release any
885 * brand data associated with the lwp
886 */
887 if (PROC_IS_BRANDED(p))
888 BROP(p)->b_lwpexit(lwp);
889
890 lwp_pcb_exit();
891
892 mutex_enter(&p->p_lock);
893 lwp_cleanup();
894
895 /*
896 * When this process is dumping core, its lwps are held here
897 * until the core dump is finished. Then exitlwps() is called
898 * again to release these lwps so that they can finish exiting.
899 */
900 if (p->p_flag & SCOREDUMP)
901 stop(PR_SUSPENDED, SUSPEND_NORMAL);
902
903 /*
904 * Block the process against /proc now that we have really acquired
905 * p->p_lock (to decrement p_lwpcnt and manipulate p_tlist at least).
906 */
907 prbarrier(p);
908
909 /*
910 * Call proc_exit() if this is the last non-daemon lwp in the process.
911 */
912 if (!(t->t_proc_flag & TP_DAEMON) &&
913 p->p_lwpcnt == p->p_lwpdaemon + 1) {
914 mutex_exit(&p->p_lock);
915 if (proc_exit(CLD_EXITED, 0) == 0) {
916 /* Restarting init. */
917 return;
918 }
919
920 /*
921 * proc_exit() returns a non-zero value when some other
922 * lwp got there first. We just have to continue in
923 * lwp_exit().
924 */
925 mutex_enter(&p->p_lock);
926 ASSERT(curproc->p_flag & SEXITLWPS);
927 prbarrier(p);
928 }
929
930 DTRACE_PROC(lwp__exit);
931
932 /*
933 * If the lwp is a detached lwp or if the process is exiting,
934 * remove (lwp_hash_out()) the lwp from the lwp directory.
935 * Otherwise null out the lwp's le_thread pointer in the lwp
936 * directory so that other threads will see it as a zombie lwp.
937 */
938 prlwpexit(t); /* notify /proc */
939 if (!(t->t_proc_flag & TP_TWAIT) || (p->p_flag & SEXITLWPS))
940 lwp_hash_out(p, t->t_tid);
941 else {
942 ASSERT(!(t->t_proc_flag & TP_DAEMON));
943 p->p_lwpdir[t->t_dslot].ld_entry->le_thread = NULL;
944 p->p_zombcnt++;
945 cv_broadcast(&p->p_lwpexit);
946 }
947 if (t->t_proc_flag & TP_DAEMON) {
948 p->p_lwpdaemon--;
949 t->t_proc_flag &= ~TP_DAEMON;
950 }
951 t->t_proc_flag &= ~TP_TWAIT;
952
953 /*
954 * Maintain accurate lwp count for task.max-lwps resource control.
955 */
956 mutex_enter(&p->p_zone->zone_nlwps_lock);
957 p->p_task->tk_nlwps--;
958 p->p_task->tk_proj->kpj_nlwps--;
959 p->p_zone->zone_nlwps--;
960 mutex_exit(&p->p_zone->zone_nlwps_lock);
961
962 CL_EXIT(t); /* tell the scheduler that t is exiting */
963 ASSERT(p->p_lwpcnt != 0);
964 p->p_lwpcnt--;
965
966 /*
967 * If all remaining non-daemon lwps are waiting in lwp_wait(),
968 * wake them up so someone can return EDEADLK.
969 * (See the block comment preceeding lwp_wait().)
970 */
971 if (p->p_lwpcnt == p->p_lwpdaemon + (p->p_lwpwait - p->p_lwpdwait))
972 cv_broadcast(&p->p_lwpexit);
973
974 t->t_proc_flag |= TP_LWPEXIT;
975 term_mstate(t);
976
977 #ifndef NPROBE
978 /* Kernel probe */
979 if (t->t_tnf_tpdp)
980 tnf_thread_exit();
981 #endif /* NPROBE */
982
983 t->t_forw->t_back = t->t_back;
984 t->t_back->t_forw = t->t_forw;
985 if (t == p->p_tlist)
986 p->p_tlist = t->t_forw;
987
988 /*
989 * Clean up the signal state.
990 */
991 if (t->t_sigqueue != NULL)
992 sigdelq(p, t, 0);
993 if (lwp->lwp_curinfo != NULL) {
994 siginfofree(lwp->lwp_curinfo);
995 lwp->lwp_curinfo = NULL;
996 }
997
998 thread_rele(t);
999
1000 /*
1001 * Terminated lwps are associated with process zero and are put onto
1002 * death-row by resume(). Avoid preemption after resetting t->t_procp.
1003 */
1004 t->t_preempt++;
1005
1006 if (t->t_ctx != NULL)
1007 exitctx(t);
1008 if (p->p_pctx != NULL)
1009 exitpctx(p);
1010
1011 t->t_procp = &p0;
1012
1013 /*
1014 * Notify the HAT about the change of address space
1015 */
1016 hat_thread_exit(t);
1017 /*
1018 * When this is the last running lwp in this process and some lwp is
1019 * waiting for this condition to become true, or this thread was being
1020 * suspended, then the waiting lwp is awakened.
1021 *
1022 * Also, if the process is exiting, we may have a thread waiting in
1023 * exitlwps() that needs to be notified.
1024 */
1025 if (--p->p_lwprcnt == 0 || (t->t_proc_flag & TP_HOLDLWP) ||
1026 (p->p_flag & SEXITLWPS))
1027 cv_broadcast(&p->p_holdlwps);
1028
1029 /*
1030 * Need to drop p_lock so we can reacquire pidlock.
1031 */
1032 mutex_exit(&p->p_lock);
1033 mutex_enter(&pidlock);
1034
1035 ASSERT(t != t->t_next); /* t0 never exits */
1036 t->t_next->t_prev = t->t_prev;
1037 t->t_prev->t_next = t->t_next;
1038 cv_broadcast(&t->t_joincv); /* wake up anyone in thread_join */
1039 mutex_exit(&pidlock);
1040
1041 t->t_state = TS_ZOMB;
1042 swtch_from_zombie();
1043 /* never returns */
1044 }
1045
1046
1047 /*
1048 * Cleanup function for an exiting lwp.
1049 * Called both from lwp_exit() and from proc_exit().
1050 * p->p_lock is repeatedly released and grabbed in this function.
1051 */
1052 void
1053 lwp_cleanup(void)
1054 {
1055 kthread_t *t = curthread;
1056 proc_t *p = ttoproc(t);
1057
1058 ASSERT(MUTEX_HELD(&p->p_lock));
1059
1060 /* untimeout any lwp-bound realtime timers */
1061 if (p->p_itimer != NULL)
1062 timer_lwpexit();
1063
1064 /*
1065 * If this is the /proc agent lwp that is exiting, readjust p_lwpid
1066 * so it appears that the agent never existed, and clear p_agenttp.
1067 */
1068 if (t == p->p_agenttp) {
1069 ASSERT(t->t_tid == p->p_lwpid);
1070 p->p_lwpid--;
1071 p->p_agenttp = NULL;
1072 }
1073
1074 /*
1075 * Do lgroup bookkeeping to account for thread exiting.
1076 */
1077 kpreempt_disable();
1078 lgrp_move_thread(t, NULL, 1);
1079 if (t->t_tid == 1) {
1080 p->p_t1_lgrpid = LGRP_NONE;
1081 }
1082 kpreempt_enable();
1083
1084 lwp_ctmpl_clear(ttolwp(t));
1085 }
1086
1087 int
1088 lwp_suspend(kthread_t *t)
1089 {
1090 int tid;
1091 proc_t *p = ttoproc(t);
1092
1093 ASSERT(MUTEX_HELD(&p->p_lock));
1094
1095 /*
1096 * Set the thread's TP_HOLDLWP flag so it will stop in holdlwp().
1097 * If an lwp is stopping itself, there is no need to wait.
1098 */
1099 top:
1100 t->t_proc_flag |= TP_HOLDLWP;
1101 if (t == curthread) {
1102 t->t_sig_check = 1;
1103 } else {
1104 /*
1105 * Make sure the lwp stops promptly.
1106 */
1107 thread_lock(t);
1108 t->t_sig_check = 1;
1109 /*
1110 * XXX Should use virtual stop like /proc does instead of
1111 * XXX waking the thread to get it to stop.
1112 */
1113 if (ISWAKEABLE(t) || ISWAITING(t)) {
1114 setrun_locked(t);
1115 } else if (t->t_state == TS_ONPROC && t->t_cpu != CPU) {
1116 poke_cpu(t->t_cpu->cpu_id);
1117 }
1118
1119 tid = t->t_tid; /* remember thread ID */
1120 /*
1121 * Wait for lwp to stop
1122 */
1123 while (!SUSPENDED(t)) {
1124 /*
1125 * Drop the thread lock before waiting and reacquire it
1126 * afterwards, so the thread can change its t_state
1127 * field.
1128 */
1129 thread_unlock(t);
1130
1131 /*
1132 * Check if aborted by exitlwps().
1133 */
1134 if (p->p_flag & SEXITLWPS)
1135 lwp_exit();
1136
1137 /*
1138 * Cooperate with jobcontrol signals and /proc stopping
1139 * by calling cv_wait_sig() to wait for the target
1140 * lwp to stop. Just using cv_wait() can lead to
1141 * deadlock because, if some other lwp has stopped
1142 * by either of these mechanisms, then p_lwprcnt will
1143 * never become zero if we do a cv_wait().
1144 */
1145 if (!cv_wait_sig(&p->p_holdlwps, &p->p_lock))
1146 return (EINTR);
1147
1148 /*
1149 * Check to see if thread died while we were
1150 * waiting for it to suspend.
1151 */
1152 if (idtot(p, tid) == NULL)
1153 return (ESRCH);
1154
1155 thread_lock(t);
1156 /*
1157 * If the TP_HOLDLWP flag went away, lwp_continue()
1158 * or vfork() must have been called while we were
1159 * waiting, so start over again.
1160 */
1161 if ((t->t_proc_flag & TP_HOLDLWP) == 0) {
1162 thread_unlock(t);
1163 goto top;
1164 }
1165 }
1166 thread_unlock(t);
1167 }
1168 return (0);
1169 }
1170
1171 /*
1172 * continue a lwp that's been stopped by lwp_suspend().
1173 */
1174 void
1175 lwp_continue(kthread_t *t)
1176 {
1177 proc_t *p = ttoproc(t);
1178 int was_suspended = t->t_proc_flag & TP_HOLDLWP;
1179
1180 ASSERT(MUTEX_HELD(&p->p_lock));
1181
1182 t->t_proc_flag &= ~TP_HOLDLWP;
1183 thread_lock(t);
1184 if (SUSPENDED(t) &&
1185 !(p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH))) {
1186 p->p_lwprcnt++;
1187 t->t_schedflag |= TS_CSTART;
1188 setrun_locked(t);
1189 }
1190 thread_unlock(t);
1191 /*
1192 * Wakeup anyone waiting for this thread to be suspended
1193 */
1194 if (was_suspended)
1195 cv_broadcast(&p->p_holdlwps);
1196 }
1197
1198 /*
1199 * ********************************
1200 * Miscellaneous lwp routines *
1201 * ********************************
1202 */
1203 /*
1204 * When a process is undergoing a forkall(), its p_flag is set to SHOLDFORK.
1205 * This will cause the process's lwps to stop at a hold point. A hold
1206 * point is where a kernel thread has a flat stack. This is at the
1207 * return from a system call and at the return from a user level trap.
1208 *
1209 * When a process is undergoing a fork1() or vfork(), its p_flag is set to
1210 * SHOLDFORK1. This will cause the process's lwps to stop at a modified
1211 * hold point. The lwps in the process are not being cloned, so they
1212 * are held at the usual hold points and also within issig_forreal().
1213 * This has the side-effect that their system calls do not return
1214 * showing EINTR.
1215 *
1216 * An lwp can also be held. This is identified by the TP_HOLDLWP flag on
1217 * the thread. The TP_HOLDLWP flag is set in lwp_suspend(), where the active
1218 * lwp is waiting for the target lwp to be stopped.
1219 */
1220 void
1221 holdlwp(void)
1222 {
1223 proc_t *p = curproc;
1224 kthread_t *t = curthread;
1225
1226 mutex_enter(&p->p_lock);
1227 /*
1228 * Don't terminate immediately if the process is dumping core.
1229 * Once the process has dumped core, all lwps are terminated.
1230 */
1231 if (!(p->p_flag & SCOREDUMP)) {
1232 if ((p->p_flag & SEXITLWPS) || (t->t_proc_flag & TP_EXITLWP))
1233 lwp_exit();
1234 }
1235 if (!(ISHOLD(p)) && !(p->p_flag & (SHOLDFORK1 | SHOLDWATCH))) {
1236 mutex_exit(&p->p_lock);
1237 return;
1238 }
1239 /*
1240 * stop() decrements p->p_lwprcnt and cv_signal()s &p->p_holdlwps
1241 * when p->p_lwprcnt becomes zero.
1242 */
1243 stop(PR_SUSPENDED, SUSPEND_NORMAL);
1244 if (p->p_flag & SEXITLWPS)
1245 lwp_exit();
1246 mutex_exit(&p->p_lock);
1247 }
1248
1249 /*
1250 * Have all lwps within the process hold at a point where they are
1251 * cloneable (SHOLDFORK) or just safe w.r.t. fork1 (SHOLDFORK1).
1252 */
1253 int
1254 holdlwps(int holdflag)
1255 {
1256 proc_t *p = curproc;
1257
1258 ASSERT(holdflag == SHOLDFORK || holdflag == SHOLDFORK1);
1259 mutex_enter(&p->p_lock);
1260 schedctl_finish_sigblock(curthread);
1261 again:
1262 while (p->p_flag & (SEXITLWPS | SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) {
1263 /*
1264 * If another lwp is doing a forkall() or proc_exit(), bail out.
1265 */
1266 if (p->p_flag & (SEXITLWPS | SHOLDFORK)) {
1267 mutex_exit(&p->p_lock);
1268 return (0);
1269 }
1270 /*
1271 * Another lwp is doing a fork1() or is undergoing
1272 * watchpoint activity. We hold here for it to complete.
1273 */
1274 stop(PR_SUSPENDED, SUSPEND_NORMAL);
1275 }
1276 p->p_flag |= holdflag;
1277 pokelwps(p);
1278 --p->p_lwprcnt;
1279 /*
1280 * Wait for the process to become quiescent (p->p_lwprcnt == 0).
1281 */
1282 while (p->p_lwprcnt > 0) {
1283 /*
1284 * Check if aborted by exitlwps().
1285 * Also check if SHOLDWATCH is set; it takes precedence.
1286 */
1287 if (p->p_flag & (SEXITLWPS | SHOLDWATCH)) {
1288 p->p_lwprcnt++;
1289 p->p_flag &= ~holdflag;
1290 cv_broadcast(&p->p_holdlwps);
1291 goto again;
1292 }
1293 /*
1294 * Cooperate with jobcontrol signals and /proc stopping.
1295 * If some other lwp has stopped by either of these
1296 * mechanisms, then p_lwprcnt will never become zero
1297 * and the process will appear deadlocked unless we
1298 * stop here in sympathy with the other lwp before
1299 * doing the cv_wait() below.
1300 *
1301 * If the other lwp stops after we do the cv_wait(), it
1302 * will wake us up to loop around and do the sympathy stop.
1303 *
1304 * Since stop() drops p->p_lock, we must start from
1305 * the top again on returning from stop().
1306 */
1307 if (p->p_stopsig | (curthread->t_proc_flag & TP_PRSTOP)) {
1308 int whystop = p->p_stopsig? PR_JOBCONTROL :
1309 PR_REQUESTED;
1310 p->p_lwprcnt++;
1311 p->p_flag &= ~holdflag;
1312 stop(whystop, p->p_stopsig);
1313 goto again;
1314 }
1315 cv_wait(&p->p_holdlwps, &p->p_lock);
1316 }
1317 p->p_lwprcnt++;
1318 p->p_flag &= ~holdflag;
1319 mutex_exit(&p->p_lock);
1320 return (1);
1321 }
1322
1323 /*
1324 * See comments for holdwatch(), below.
1325 */
1326 static int
1327 holdcheck(int clearflags)
1328 {
1329 proc_t *p = curproc;
1330
1331 /*
1332 * If we are trying to exit, that takes precedence over anything else.
1333 */
1334 if (p->p_flag & SEXITLWPS) {
1335 p->p_lwprcnt++;
1336 p->p_flag &= ~clearflags;
1337 lwp_exit();
1338 }
1339
1340 /*
1341 * If another thread is calling fork1(), stop the current thread so the
1342 * other can complete.
1343 */
1344 if (p->p_flag & SHOLDFORK1) {
1345 p->p_lwprcnt++;
1346 stop(PR_SUSPENDED, SUSPEND_NORMAL);
1347 if (p->p_flag & SEXITLWPS) {
1348 p->p_flag &= ~clearflags;
1349 lwp_exit();
1350 }
1351 return (-1);
1352 }
1353
1354 /*
1355 * If another thread is calling fork(), then indicate we are doing
1356 * watchpoint activity. This will cause holdlwps() above to stop the
1357 * forking thread, at which point we can continue with watchpoint
1358 * activity.
1359 */
1360 if (p->p_flag & SHOLDFORK) {
1361 p->p_lwprcnt++;
1362 while (p->p_flag & SHOLDFORK) {
1363 p->p_flag |= SHOLDWATCH;
1364 cv_broadcast(&p->p_holdlwps);
1365 cv_wait(&p->p_holdlwps, &p->p_lock);
1366 p->p_flag &= ~SHOLDWATCH;
1367 }
1368 return (-1);
1369 }
1370
1371 return (0);
1372 }
1373
1374 /*
1375 * Stop all lwps within the process, holding themselves in the kernel while the
1376 * active lwp undergoes watchpoint activity. This is more complicated than
1377 * expected because stop() relies on calling holdwatch() in order to copyin data
1378 * from the user's address space. A double barrier is used to prevent an
1379 * infinite loop.
1380 *
1381 * o The first thread into holdwatch() is the 'master' thread and does
1382 * the following:
1383 *
1384 * - Sets SHOLDWATCH on the current process
1385 * - Sets TP_WATCHSTOP on the current thread
1386 * - Waits for all threads to be either stopped or have
1387 * TP_WATCHSTOP set.
1388 * - Sets the SWATCHOK flag on the process
1389 * - Unsets TP_WATCHSTOP
1390 * - Waits for the other threads to completely stop
1391 * - Unsets SWATCHOK
1392 *
1393 * o If SHOLDWATCH is already set when we enter this function, then another
1394 * thread is already trying to stop this thread. This 'slave' thread
1395 * does the following:
1396 *
1397 * - Sets TP_WATCHSTOP on the current thread
1398 * - Waits for SWATCHOK flag to be set
1399 * - Calls stop()
1400 *
1401 * o If SWATCHOK is set on the process, then this function immediately
1402 * returns, as we must have been called via stop().
1403 *
1404 * In addition, there are other flags that take precedence over SHOLDWATCH:
1405 *
1406 * o If SEXITLWPS is set, exit immediately.
1407 *
1408 * o If SHOLDFORK1 is set, wait for fork1() to complete.
1409 *
1410 * o If SHOLDFORK is set, then watchpoint activity takes precedence In this
1411 * case, set SHOLDWATCH, signalling the forking thread to stop first.
1412 *
1413 * o If the process is being stopped via /proc (TP_PRSTOP is set), then we
1414 * stop the current thread.
1415 *
1416 * Returns 0 if all threads have been quiesced. Returns non-zero if not all
1417 * threads were stopped, or the list of watched pages has changed.
1418 */
1419 int
1420 holdwatch(void)
1421 {
1422 proc_t *p = curproc;
1423 kthread_t *t = curthread;
1424 int ret = 0;
1425
1426 mutex_enter(&p->p_lock);
1427
1428 p->p_lwprcnt--;
1429
1430 /*
1431 * Check for bail-out conditions as outlined above.
1432 */
1433 if (holdcheck(0) != 0) {
1434 mutex_exit(&p->p_lock);
1435 return (-1);
1436 }
1437
1438 if (!(p->p_flag & SHOLDWATCH)) {
1439 /*
1440 * We are the master watchpoint thread. Set SHOLDWATCH and poke
1441 * the other threads.
1442 */
1443 p->p_flag |= SHOLDWATCH;
1444 pokelwps(p);
1445
1446 /*
1447 * Wait for all threads to be stopped or have TP_WATCHSTOP set.
1448 */
1449 while (pr_allstopped(p, 1) > 0) {
1450 if (holdcheck(SHOLDWATCH) != 0) {
1451 p->p_flag &= ~SHOLDWATCH;
1452 mutex_exit(&p->p_lock);
1453 return (-1);
1454 }
1455
1456 cv_wait(&p->p_holdlwps, &p->p_lock);
1457 }
1458
1459 /*
1460 * All threads are now stopped or in the process of stopping.
1461 * Set SWATCHOK and let them stop completely.
1462 */
1463 p->p_flag |= SWATCHOK;
1464 t->t_proc_flag &= ~TP_WATCHSTOP;
1465 cv_broadcast(&p->p_holdlwps);
1466
1467 while (pr_allstopped(p, 0) > 0) {
1468 /*
1469 * At first glance, it may appear that we don't need a
1470 * call to holdcheck() here. But if the process gets a
1471 * SIGKILL signal, one of our stopped threads may have
1472 * been awakened and is waiting in exitlwps(), which
1473 * takes precedence over watchpoints.
1474 */
1475 if (holdcheck(SHOLDWATCH | SWATCHOK) != 0) {
1476 p->p_flag &= ~(SHOLDWATCH | SWATCHOK);
1477 mutex_exit(&p->p_lock);
1478 return (-1);
1479 }
1480
1481 cv_wait(&p->p_holdlwps, &p->p_lock);
1482 }
1483
1484 /*
1485 * All threads are now completely stopped.
1486 */
1487 p->p_flag &= ~SWATCHOK;
1488 p->p_flag &= ~SHOLDWATCH;
1489 p->p_lwprcnt++;
1490
1491 } else if (!(p->p_flag & SWATCHOK)) {
1492
1493 /*
1494 * SHOLDWATCH is set, so another thread is trying to do
1495 * watchpoint activity. Indicate this thread is stopping, and
1496 * wait for the OK from the master thread.
1497 */
1498 t->t_proc_flag |= TP_WATCHSTOP;
1499 cv_broadcast(&p->p_holdlwps);
1500
1501 while (!(p->p_flag & SWATCHOK)) {
1502 if (holdcheck(0) != 0) {
1503 t->t_proc_flag &= ~TP_WATCHSTOP;
1504 mutex_exit(&p->p_lock);
1505 return (-1);
1506 }
1507
1508 cv_wait(&p->p_holdlwps, &p->p_lock);
1509 }
1510
1511 /*
1512 * Once the master thread has given the OK, this thread can
1513 * actually call stop().
1514 */
1515 t->t_proc_flag &= ~TP_WATCHSTOP;
1516 p->p_lwprcnt++;
1517
1518 stop(PR_SUSPENDED, SUSPEND_NORMAL);
1519
1520 /*
1521 * It's not OK to do watchpoint activity, notify caller to
1522 * retry.
1523 */
1524 ret = -1;
1525
1526 } else {
1527
1528 /*
1529 * The only way we can hit the case where SHOLDWATCH is set and
1530 * SWATCHOK is set is if we are triggering this from within a
1531 * stop() call. Assert that this is the case.
1532 */
1533
1534 ASSERT(t->t_proc_flag & TP_STOPPING);
1535 p->p_lwprcnt++;
1536 }
1537
1538 mutex_exit(&p->p_lock);
1539
1540 return (ret);
1541 }
1542
1543 /*
1544 * force all interruptible lwps to trap into the kernel.
1545 */
1546 void
1547 pokelwps(proc_t *p)
1548 {
1549 kthread_t *t;
1550
1551 ASSERT(MUTEX_HELD(&p->p_lock));
1552
1553 t = p->p_tlist;
1554 do {
1555 if (t == curthread)
1556 continue;
1557 thread_lock(t);
1558 aston(t); /* make thread trap or do post_syscall */
1559 if (ISWAKEABLE(t) || ISWAITING(t)) {
1560 setrun_locked(t);
1561 } else if (t->t_state == TS_STOPPED) {
1562 /*
1563 * Ensure that proc_exit() is not blocked by lwps
1564 * that were stopped via jobcontrol or /proc.
1565 */
1566 if (p->p_flag & SEXITLWPS) {
1567 p->p_stopsig = 0;
1568 t->t_schedflag |= (TS_XSTART | TS_PSTART);
1569 setrun_locked(t);
1570 }
1571 /*
1572 * If we are holding lwps for a forkall(),
1573 * force lwps that have been suspended via
1574 * lwp_suspend() and are suspended inside
1575 * of a system call to proceed to their
1576 * holdlwp() points where they are clonable.
1577 */
1578 if ((p->p_flag & SHOLDFORK) && SUSPENDED(t)) {
1579 if ((t->t_schedflag & TS_CSTART) == 0) {
1580 p->p_lwprcnt++;
1581 t->t_schedflag |= TS_CSTART;
1582 setrun_locked(t);
1583 }
1584 }
1585 } else if (t->t_state == TS_ONPROC) {
1586 if (t->t_cpu != CPU)
1587 poke_cpu(t->t_cpu->cpu_id);
1588 }
1589 thread_unlock(t);
1590 } while ((t = t->t_forw) != p->p_tlist);
1591 }
1592
1593 /*
1594 * undo the effects of holdlwps() or holdwatch().
1595 */
1596 void
1597 continuelwps(proc_t *p)
1598 {
1599 kthread_t *t;
1600
1601 /*
1602 * If this flag is set, then the original holdwatch() didn't actually
1603 * stop the process. See comments for holdwatch().
1604 */
1605 if (p->p_flag & SWATCHOK) {
1606 ASSERT(curthread->t_proc_flag & TP_STOPPING);
1607 return;
1608 }
1609
1610 ASSERT(MUTEX_HELD(&p->p_lock));
1611 ASSERT((p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) == 0);
1612
1613 t = p->p_tlist;
1614 do {
1615 thread_lock(t); /* SUSPENDED looks at t_schedflag */
1616 if (SUSPENDED(t) && !(t->t_proc_flag & TP_HOLDLWP)) {
1617 p->p_lwprcnt++;
1618 t->t_schedflag |= TS_CSTART;
1619 setrun_locked(t);
1620 }
1621 thread_unlock(t);
1622 } while ((t = t->t_forw) != p->p_tlist);
1623 }
1624
1625 /*
1626 * Force all other LWPs in the current process other than the caller to exit,
1627 * and then cv_wait() on p_holdlwps for them to exit. The exitlwps() function
1628 * is typically used in these situations:
1629 *
1630 * (a) prior to an exec() system call
1631 * (b) prior to dumping a core file
1632 * (c) prior to a uadmin() shutdown
1633 *
1634 * If the 'coredump' flag is set, other LWPs are quiesced but not destroyed.
1635 * Multiple threads in the process can call this function at one time by
1636 * triggering execs or core dumps simultaneously, so the SEXITLWPS bit is used
1637 * to declare one particular thread the winner who gets to kill the others.
1638 * If a thread wins the exitlwps() dance, zero is returned; otherwise an
1639 * appropriate errno value is returned to caller for its system call to return.
1640 */
1641 int
1642 exitlwps(int coredump)
1643 {
1644 proc_t *p = curproc;
1645 int heldcnt;
1646
1647 if (curthread->t_door)
1648 door_slam();
1649 if (p->p_door_list)
1650 door_revoke_all();
1651 if (curthread->t_schedctl != NULL)
1652 schedctl_lwp_cleanup(curthread);
1653
1654 /*
1655 * Ensure that before starting to wait for other lwps to exit,
1656 * cleanup all upimutexes held by curthread. Otherwise, some other
1657 * lwp could be waiting (uninterruptibly) for a upimutex held by
1658 * curthread, and the call to pokelwps() below would deadlock.
1659 * Even if a blocked upimutex_lock is made interruptible,
1660 * curthread's upimutexes need to be unlocked: do it here.
1661 */
1662 if (curthread->t_upimutex != NULL)
1663 upimutex_cleanup();
1664
1665 /*
1666 * Grab p_lock in order to check and set SEXITLWPS to declare a winner.
1667 * We must also block any further /proc access from this point forward.
1668 */
1669 mutex_enter(&p->p_lock);
1670 prbarrier(p);
1671
1672 if (p->p_flag & SEXITLWPS) {
1673 mutex_exit(&p->p_lock);
1674 aston(curthread); /* force a trip through post_syscall */
1675 return (set_errno(EINTR));
1676 }
1677
1678 p->p_flag |= SEXITLWPS;
1679 if (coredump) /* tell other lwps to stop, not exit */
1680 p->p_flag |= SCOREDUMP;
1681
1682 /*
1683 * Give precedence to exitlwps() if a holdlwps() is
1684 * in progress. The lwp doing the holdlwps() operation
1685 * is aborted when it is awakened.
1686 */
1687 while (p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) {
1688 cv_broadcast(&p->p_holdlwps);
1689 cv_wait(&p->p_holdlwps, &p->p_lock);
1690 prbarrier(p);
1691 }
1692 p->p_flag |= SHOLDFORK;
1693 pokelwps(p);
1694
1695 /*
1696 * Wait for process to become quiescent.
1697 */
1698 --p->p_lwprcnt;
1699 while (p->p_lwprcnt > 0) {
1700 cv_wait(&p->p_holdlwps, &p->p_lock);
1701 prbarrier(p);
1702 }
1703 p->p_lwprcnt++;
1704 ASSERT(p->p_lwprcnt == 1);
1705
1706 /*
1707 * The SCOREDUMP flag puts the process into a quiescent
1708 * state. The process's lwps remain attached to this
1709 * process until exitlwps() is called again without the
1710 * 'coredump' flag set, then the lwps are terminated
1711 * and the process can exit.
1712 */
1713 if (coredump) {
1714 p->p_flag &= ~(SCOREDUMP | SHOLDFORK | SEXITLWPS);
1715 goto out;
1716 }
1717
1718 /*
1719 * Determine if there are any lwps left dangling in
1720 * the stopped state. This happens when exitlwps()
1721 * aborts a holdlwps() operation.
1722 */
1723 p->p_flag &= ~SHOLDFORK;
1724 if ((heldcnt = p->p_lwpcnt) > 1) {
1725 kthread_t *t;
1726 for (t = curthread->t_forw; --heldcnt > 0; t = t->t_forw) {
1727 t->t_proc_flag &= ~TP_TWAIT;
1728 lwp_continue(t);
1729 }
1730 }
1731
1732 /*
1733 * Wait for all other lwps to exit.
1734 */
1735 --p->p_lwprcnt;
1736 while (p->p_lwpcnt > 1) {
1737 cv_wait(&p->p_holdlwps, &p->p_lock);
1738 prbarrier(p);
1739 }
1740 ++p->p_lwprcnt;
1741 ASSERT(p->p_lwpcnt == 1 && p->p_lwprcnt == 1);
1742
1743 p->p_flag &= ~SEXITLWPS;
1744 curthread->t_proc_flag &= ~TP_TWAIT;
1745
1746 out:
1747 if (!coredump && p->p_zombcnt) { /* cleanup the zombie lwps */
1748 lwpdir_t *ldp;
1749 lwpent_t *lep;
1750 int i;
1751
1752 for (ldp = p->p_lwpdir, i = 0; i < p->p_lwpdir_sz; i++, ldp++) {
1753 lep = ldp->ld_entry;
1754 if (lep != NULL && lep->le_thread != curthread) {
1755 ASSERT(lep->le_thread == NULL);
1756 p->p_zombcnt--;
1757 lwp_hash_out(p, lep->le_lwpid);
1758 }
1759 }
1760 ASSERT(p->p_zombcnt == 0);
1761 }
1762
1763 /*
1764 * If some other LWP in the process wanted us to suspend ourself,
1765 * then we will not do it. The other LWP is now terminated and
1766 * no one will ever continue us again if we suspend ourself.
1767 */
1768 curthread->t_proc_flag &= ~TP_HOLDLWP;
1769 p->p_flag &= ~(SHOLDFORK | SHOLDFORK1 | SHOLDWATCH | SLWPWRAP);
1770 mutex_exit(&p->p_lock);
1771 return (0);
1772 }
1773
1774 /*
1775 * duplicate a lwp.
1776 */
1777 klwp_t *
1778 forklwp(klwp_t *lwp, proc_t *cp, id_t lwpid)
1779 {
1780 klwp_t *clwp;
1781 void *tregs, *tfpu;
1782 kthread_t *t = lwptot(lwp);
1783 kthread_t *ct;
1784 proc_t *p = lwptoproc(lwp);
1785 int cid;
1786 void *bufp;
1787 void *brand_data;
1788 int val;
1789
1790 ASSERT(p == curproc);
1791 ASSERT(t == curthread || (SUSPENDED(t) && lwp->lwp_asleep == 0));
1792
1793 #if defined(__sparc)
1794 if (t == curthread)
1795 (void) flush_user_windows_to_stack(NULL);
1796 #endif
1797
1798 if (t == curthread)
1799 /* copy args out of registers first */
1800 (void) save_syscall_args();
1801
1802 clwp = lwp_create(cp->p_lwpcnt == 0 ? lwp_rtt_initial : lwp_rtt,
1803 NULL, 0, cp, TS_STOPPED, t->t_pri, &t->t_hold, NOCLASS, lwpid);
1804 if (clwp == NULL)
1805 return (NULL);
1806
1807 /*
1808 * most of the parent's lwp can be copied to its duplicate,
1809 * except for the fields that are unique to each lwp, like
1810 * lwp_thread, lwp_procp, lwp_regs, and lwp_ap.
1811 */
1812 ct = clwp->lwp_thread;
1813 tregs = clwp->lwp_regs;
1814 tfpu = clwp->lwp_fpu;
1815 brand_data = clwp->lwp_brand;
1816
1817 /*
1818 * Copy parent lwp to child lwp. Hold child's p_lock to prevent
1819 * mstate_aggr_state() from reading stale mstate entries copied
1820 * from lwp to clwp.
1821 */
1822 mutex_enter(&cp->p_lock);
1823 *clwp = *lwp;
1824
1825 /* clear microstate and resource usage data in new lwp */
1826 init_mstate(ct, LMS_STOPPED);
1827 bzero(&clwp->lwp_ru, sizeof (clwp->lwp_ru));
1828 mutex_exit(&cp->p_lock);
1829
1830 /* fix up child's lwp */
1831
1832 clwp->lwp_pcb.pcb_flags = 0;
1833 #if defined(__sparc)
1834 clwp->lwp_pcb.pcb_step = STEP_NONE;
1835 #endif
1836 clwp->lwp_cursig = 0;
1837 clwp->lwp_extsig = 0;
1838 clwp->lwp_curinfo = (struct sigqueue *)0;
1839 clwp->lwp_thread = ct;
1840 ct->t_sysnum = t->t_sysnum;
1841 clwp->lwp_regs = tregs;
1842 clwp->lwp_fpu = tfpu;
1843 clwp->lwp_brand = brand_data;
1844 clwp->lwp_ap = clwp->lwp_arg;
1845 clwp->lwp_procp = cp;
1846 bzero(clwp->lwp_timer, sizeof (clwp->lwp_timer));
1847 clwp->lwp_lastfault = 0;
1848 clwp->lwp_lastfaddr = 0;
1849
1850 /* copy parent's struct regs to child. */
1851 lwp_forkregs(lwp, clwp);
1852
1853 /*
1854 * Fork thread context ops, if any.
1855 */
1856 if (t->t_ctx)
1857 forkctx(t, ct);
1858
1859 /* fix door state in the child */
1860 if (t->t_door)
1861 door_fork(t, ct);
1862
1863 /* copy current contract templates, clear latest contracts */
1864 lwp_ctmpl_copy(clwp, lwp);
1865
1866 mutex_enter(&cp->p_lock);
1867 /* lwp_create() set the TP_HOLDLWP flag */
1868 if (!(t->t_proc_flag & TP_HOLDLWP))
1869 ct->t_proc_flag &= ~TP_HOLDLWP;
1870 if (cp->p_flag & SMSACCT)
1871 ct->t_proc_flag |= TP_MSACCT;
1872 mutex_exit(&cp->p_lock);
1873
1874 /* Allow brand to propagate brand-specific state */
1875 if (PROC_IS_BRANDED(p))
1876 BROP(p)->b_forklwp(lwp, clwp);
1877
1878 retry:
1879 cid = t->t_cid;
1880
1881 val = CL_ALLOC(&bufp, cid, KM_SLEEP);
1882 ASSERT(val == 0);
1883
1884 mutex_enter(&p->p_lock);
1885 if (cid != t->t_cid) {
1886 /*
1887 * Someone just changed this thread's scheduling class,
1888 * so try pre-allocating the buffer again. Hopefully we
1889 * don't hit this often.
1890 */
1891 mutex_exit(&p->p_lock);
1892 CL_FREE(cid, bufp);
1893 goto retry;
1894 }
1895
1896 ct->t_unpark = t->t_unpark;
1897 ct->t_clfuncs = t->t_clfuncs;
1898 CL_FORK(t, ct, bufp);
1899 ct->t_cid = t->t_cid; /* after data allocated so prgetpsinfo works */
1900 mutex_exit(&p->p_lock);
1901
1902 return (clwp);
1903 }
1904
1905 /*
1906 * Add a new lwp entry to the lwp directory and to the lwpid hash table.
1907 */
1908 void
1909 lwp_hash_in(proc_t *p, lwpent_t *lep, tidhash_t *tidhash, uint_t tidhash_sz,
1910 int do_lock)
1911 {
1912 tidhash_t *thp = &tidhash[TIDHASH(lep->le_lwpid, tidhash_sz)];
1913 lwpdir_t **ldpp;
1914 lwpdir_t *ldp;
1915 kthread_t *t;
1916
1917 /*
1918 * Allocate a directory element from the free list.
1919 * Code elsewhere guarantees a free slot.
1920 */
1921 ldp = p->p_lwpfree;
1922 p->p_lwpfree = ldp->ld_next;
1923 ASSERT(ldp->ld_entry == NULL);
1924 ldp->ld_entry = lep;
1925
1926 if (do_lock)
1927 mutex_enter(&thp->th_lock);
1928
1929 /*
1930 * Insert it into the lwpid hash table.
1931 */
1932 ldpp = &thp->th_list;
1933 ldp->ld_next = *ldpp;
1934 *ldpp = ldp;
1935
1936 /*
1937 * Set the active thread's directory slot entry.
1938 */
1939 if ((t = lep->le_thread) != NULL) {
1940 ASSERT(lep->le_lwpid == t->t_tid);
1941 t->t_dslot = (int)(ldp - p->p_lwpdir);
1942 }
1943
1944 if (do_lock)
1945 mutex_exit(&thp->th_lock);
1946 }
1947
1948 /*
1949 * Remove an lwp from the lwpid hash table and free its directory entry.
1950 * This is done when a detached lwp exits in lwp_exit() or
1951 * when a non-detached lwp is waited for in lwp_wait() or
1952 * when a zombie lwp is detached in lwp_detach().
1953 */
1954 void
1955 lwp_hash_out(proc_t *p, id_t lwpid)
1956 {
1957 tidhash_t *thp = &p->p_tidhash[TIDHASH(lwpid, p->p_tidhash_sz)];
1958 lwpdir_t **ldpp;
1959 lwpdir_t *ldp;
1960 lwpent_t *lep;
1961
1962 mutex_enter(&thp->th_lock);
1963 for (ldpp = &thp->th_list;
1964 (ldp = *ldpp) != NULL; ldpp = &ldp->ld_next) {
1965 lep = ldp->ld_entry;
1966 if (lep->le_lwpid == lwpid) {
1967 prlwpfree(p, lep); /* /proc deals with le_trace */
1968 *ldpp = ldp->ld_next;
1969 ldp->ld_entry = NULL;
1970 ldp->ld_next = p->p_lwpfree;
1971 p->p_lwpfree = ldp;
1972 kmem_free(lep, sizeof (*lep));
1973 break;
1974 }
1975 }
1976 mutex_exit(&thp->th_lock);
1977 }
1978
1979 /*
1980 * Lookup an lwp in the lwpid hash table by lwpid.
1981 */
1982 lwpdir_t *
1983 lwp_hash_lookup(proc_t *p, id_t lwpid)
1984 {
1985 tidhash_t *thp;
1986 lwpdir_t *ldp;
1987
1988 /*
1989 * The process may be exiting, after p_tidhash has been set to NULL in
1990 * proc_exit() but before prfee() has been called. Return failure in
1991 * this case.
1992 */
1993 if (p->p_tidhash == NULL)
1994 return (NULL);
1995
1996 thp = &p->p_tidhash[TIDHASH(lwpid, p->p_tidhash_sz)];
1997 for (ldp = thp->th_list; ldp != NULL; ldp = ldp->ld_next) {
1998 if (ldp->ld_entry->le_lwpid == lwpid)
1999 return (ldp);
2000 }
2001
2002 return (NULL);
2003 }
2004
2005 /*
2006 * Same as lwp_hash_lookup(), but acquire and return
2007 * the tid hash table entry lock on success.
2008 */
2009 lwpdir_t *
2010 lwp_hash_lookup_and_lock(proc_t *p, id_t lwpid, kmutex_t **mpp)
2011 {
2012 tidhash_t *tidhash;
2013 uint_t tidhash_sz;
2014 tidhash_t *thp;
2015 lwpdir_t *ldp;
2016
2017 top:
2018 tidhash_sz = p->p_tidhash_sz;
2019 membar_consumer();
2020 if ((tidhash = p->p_tidhash) == NULL)
2021 return (NULL);
2022
2023 thp = &tidhash[TIDHASH(lwpid, tidhash_sz)];
2024 mutex_enter(&thp->th_lock);
2025
2026 /*
2027 * Since we are not holding p->p_lock, the tid hash table
2028 * may have changed. If so, start over. If not, then
2029 * it cannot change until after we drop &thp->th_lock;
2030 */
2031 if (tidhash != p->p_tidhash || tidhash_sz != p->p_tidhash_sz) {
2032 mutex_exit(&thp->th_lock);
2033 goto top;
2034 }
2035
2036 for (ldp = thp->th_list; ldp != NULL; ldp = ldp->ld_next) {
2037 if (ldp->ld_entry->le_lwpid == lwpid) {
2038 *mpp = &thp->th_lock;
2039 return (ldp);
2040 }
2041 }
2042
2043 mutex_exit(&thp->th_lock);
2044 return (NULL);
2045 }
2046
2047 /*
2048 * Update the indicated LWP usage statistic for the current LWP.
2049 */
2050 void
2051 lwp_stat_update(lwp_stat_id_t lwp_stat_id, long inc)
2052 {
2053 klwp_t *lwp = ttolwp(curthread);
2054
2055 if (lwp == NULL)
2056 return;
2057
2058 switch (lwp_stat_id) {
2059 case LWP_STAT_INBLK:
2060 lwp->lwp_ru.inblock += inc;
2061 break;
2062 case LWP_STAT_OUBLK:
2063 lwp->lwp_ru.oublock += inc;
2064 break;
2065 case LWP_STAT_MSGRCV:
2066 lwp->lwp_ru.msgrcv += inc;
2067 break;
2068 case LWP_STAT_MSGSND:
2069 lwp->lwp_ru.msgsnd += inc;
2070 break;
2071 default:
2072 panic("lwp_stat_update: invalid lwp_stat_id 0x%x", lwp_stat_id);
2073 }
2074 }
Cache object: 6cbd7417ca31a6ff3326b92b905acc12
|