1 /*-
2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * 3. Berkeley Software Design Inc's name may not be used to endorse or
13 * promote products derived from this software without specific prior
14 * written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $
29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
30 */
31
32 /*
33 * Implementation of the `witness' lock verifier. Originally implemented for
34 * mutexes in BSD/OS. Extended to handle generic lock objects and lock
35 * classes in FreeBSD.
36 */
37
38 /*
39 * Main Entry: witness
40 * Pronunciation: 'wit-n&s
41 * Function: noun
42 * Etymology: Middle English witnesse, from Old English witnes knowledge,
43 * testimony, witness, from 2wit
44 * Date: before 12th century
45 * 1 : attestation of a fact or event : TESTIMONY
46 * 2 : one that gives evidence; specifically : one who testifies in
47 * a cause or before a judicial tribunal
48 * 3 : one asked to be present at a transaction so as to be able to
49 * testify to its having taken place
50 * 4 : one who has personal knowledge of something
51 * 5 a : something serving as evidence or proof : SIGN
52 * b : public affirmation by word or example of usually
53 * religious faith or conviction <the heroic witness to divine
54 * life -- Pilot>
55 * 6 capitalized : a member of the Jehovah's Witnesses
56 */
57
58 /*
59 * Special rules concerning Giant and lock orders:
60 *
61 * 1) Giant must be acquired before any other mutexes. Stated another way,
62 * no other mutex may be held when Giant is acquired.
63 *
64 * 2) Giant must be released when blocking on a sleepable lock.
65 *
66 * This rule is less obvious, but is a result of Giant providing the same
67 * semantics as spl(). Basically, when a thread sleeps, it must release
68 * Giant. When a thread blocks on a sleepable lock, it sleeps. Hence rule
69 * 2).
70 *
71 * 3) Giant may be acquired before or after sleepable locks.
72 *
73 * This rule is also not quite as obvious. Giant may be acquired after
74 * a sleepable lock because it is a non-sleepable lock and non-sleepable
75 * locks may always be acquired while holding a sleepable lock. The second
76 * case, Giant before a sleepable lock, follows from rule 2) above. Suppose
77 * you have two threads T1 and T2 and a sleepable lock X. Suppose that T1
78 * acquires X and blocks on Giant. Then suppose that T2 acquires Giant and
79 * blocks on X. When T2 blocks on X, T2 will release Giant allowing T1 to
80 * execute. Thus, acquiring Giant both before and after a sleepable lock
81 * will not result in a lock order reversal.
82 */
83
84 #include <sys/cdefs.h>
85 __FBSDID("$FreeBSD$");
86
87 #include "opt_ddb.h"
88 #include "opt_witness.h"
89
90 #include <sys/param.h>
91 #include <sys/bus.h>
92 #include <sys/kdb.h>
93 #include <sys/kernel.h>
94 #include <sys/ktr.h>
95 #include <sys/lock.h>
96 #include <sys/malloc.h>
97 #include <sys/mutex.h>
98 #include <sys/proc.h>
99 #include <sys/sysctl.h>
100 #include <sys/systm.h>
101
102 #include <ddb/ddb.h>
103
104 #include <machine/stdarg.h>
105
106 /* Define this to check for blessed mutexes */
107 #undef BLESSING
108
109 #define WITNESS_COUNT 200
110 #define WITNESS_CHILDCOUNT (WITNESS_COUNT * 4)
111 /*
112 * XXX: This is somewhat bogus, as we assume here that at most 1024 threads
113 * will hold LOCK_NCHILDREN * 2 locks. We handle failure ok, and we should
114 * probably be safe for the most part, but it's still a SWAG.
115 */
116 #define LOCK_CHILDCOUNT (MAXCPU + 1024) * 2
117
118 #define WITNESS_NCHILDREN 6
119
120 struct witness_child_list_entry;
121
122 struct witness {
123 const char *w_name;
124 struct lock_class *w_class;
125 STAILQ_ENTRY(witness) w_list; /* List of all witnesses. */
126 STAILQ_ENTRY(witness) w_typelist; /* Witnesses of a type. */
127 struct witness_child_list_entry *w_children; /* Great evilness... */
128 const char *w_file;
129 int w_line;
130 u_int w_level;
131 u_int w_refcount;
132 u_char w_Giant_squawked:1;
133 u_char w_other_squawked:1;
134 u_char w_same_squawked:1;
135 u_char w_displayed:1;
136 };
137
138 struct witness_child_list_entry {
139 struct witness_child_list_entry *wcl_next;
140 struct witness *wcl_children[WITNESS_NCHILDREN];
141 u_int wcl_count;
142 };
143
144 STAILQ_HEAD(witness_list, witness);
145
146 #ifdef BLESSING
147 struct witness_blessed {
148 const char *b_lock1;
149 const char *b_lock2;
150 };
151 #endif
152
153 struct witness_order_list_entry {
154 const char *w_name;
155 struct lock_class *w_class;
156 };
157
158 #ifdef BLESSING
159 static int blessed(struct witness *, struct witness *);
160 #endif
161 static int depart(struct witness *w);
162 static struct witness *enroll(const char *description,
163 struct lock_class *lock_class);
164 static int insertchild(struct witness *parent, struct witness *child);
165 static int isitmychild(struct witness *parent, struct witness *child);
166 static int isitmydescendant(struct witness *parent, struct witness *child);
167 static int itismychild(struct witness *parent, struct witness *child);
168 static void removechild(struct witness *parent, struct witness *child);
169 static int sysctl_debug_witness_watch(SYSCTL_HANDLER_ARGS);
170 static const char *fixup_filename(const char *file);
171 static struct witness *witness_get(void);
172 static void witness_free(struct witness *m);
173 static struct witness_child_list_entry *witness_child_get(void);
174 static void witness_child_free(struct witness_child_list_entry *wcl);
175 static struct lock_list_entry *witness_lock_list_get(void);
176 static void witness_lock_list_free(struct lock_list_entry *lle);
177 static struct lock_instance *find_instance(struct lock_list_entry *lock_list,
178 struct lock_object *lock);
179 static void witness_list_lock(struct lock_instance *instance);
180 #ifdef DDB
181 static void witness_leveldescendents(struct witness *parent, int level);
182 static void witness_levelall(void);
183 static void witness_displaydescendants(void(*)(const char *fmt, ...),
184 struct witness *, int indent);
185 static void witness_display_list(void(*prnt)(const char *fmt, ...),
186 struct witness_list *list);
187 static void witness_display(void(*)(const char *fmt, ...));
188 static void witness_list(struct thread *td);
189 #endif
190
191 SYSCTL_NODE(_debug, OID_AUTO, witness, CTLFLAG_RW, 0, "Witness Locking");
192
193 /*
194 * If set to 0, witness is disabled. If set to a non-zero value, witness
195 * performs full lock order checking for all locks. At runtime, this
196 * value may be set to 0 to turn off witness. witness is not allowed be
197 * turned on once it is turned off, however.
198 */
199 static int witness_watch = 1;
200 TUNABLE_INT("debug.witness.watch", &witness_watch);
201 SYSCTL_PROC(_debug_witness, OID_AUTO, watch, CTLFLAG_RW | CTLTYPE_INT, NULL, 0,
202 sysctl_debug_witness_watch, "I", "witness is watching lock operations");
203
204 #ifdef KDB
205 /*
206 * When KDB is enabled and witness_kdb is set to 1, it will cause the system
207 * to drop into kdebug() when:
208 * - a lock heirarchy violation occurs
209 * - locks are held when going to sleep.
210 */
211 #ifdef WITNESS_KDB
212 int witness_kdb = 1;
213 #else
214 int witness_kdb = 0;
215 #endif
216 TUNABLE_INT("debug.witness.kdb", &witness_kdb);
217 SYSCTL_INT(_debug_witness, OID_AUTO, kdb, CTLFLAG_RW, &witness_kdb, 0, "");
218
219 /*
220 * When KDB is enabled and witness_trace is set to 1, it will cause the system
221 * to print a stack trace:
222 * - a lock heirarchy violation occurs
223 * - locks are held when going to sleep.
224 */
225 int witness_trace = 1;
226 TUNABLE_INT("debug.witness.trace", &witness_trace);
227 SYSCTL_INT(_debug_witness, OID_AUTO, trace, CTLFLAG_RW, &witness_trace, 0, "");
228 #endif /* KDB */
229
230 #ifdef WITNESS_SKIPSPIN
231 int witness_skipspin = 1;
232 #else
233 int witness_skipspin = 0;
234 #endif
235 TUNABLE_INT("debug.witness.skipspin", &witness_skipspin);
236 SYSCTL_INT(_debug_witness, OID_AUTO, skipspin, CTLFLAG_RDTUN,
237 &witness_skipspin, 0, "");
238
239 static struct mtx w_mtx;
240 static struct witness_list w_free = STAILQ_HEAD_INITIALIZER(w_free);
241 static struct witness_list w_all = STAILQ_HEAD_INITIALIZER(w_all);
242 static struct witness_list w_spin = STAILQ_HEAD_INITIALIZER(w_spin);
243 static struct witness_list w_sleep = STAILQ_HEAD_INITIALIZER(w_sleep);
244 static struct witness_child_list_entry *w_child_free = NULL;
245 static struct lock_list_entry *w_lock_list_free = NULL;
246
247 static int w_free_cnt, w_spin_cnt, w_sleep_cnt, w_child_free_cnt, w_child_cnt;
248 SYSCTL_INT(_debug_witness, OID_AUTO, free_cnt, CTLFLAG_RD, &w_free_cnt, 0, "");
249 SYSCTL_INT(_debug_witness, OID_AUTO, spin_cnt, CTLFLAG_RD, &w_spin_cnt, 0, "");
250 SYSCTL_INT(_debug_witness, OID_AUTO, sleep_cnt, CTLFLAG_RD, &w_sleep_cnt, 0,
251 "");
252 SYSCTL_INT(_debug_witness, OID_AUTO, child_free_cnt, CTLFLAG_RD,
253 &w_child_free_cnt, 0, "");
254 SYSCTL_INT(_debug_witness, OID_AUTO, child_cnt, CTLFLAG_RD, &w_child_cnt, 0,
255 "");
256
257 static struct witness w_data[WITNESS_COUNT];
258 static struct witness_child_list_entry w_childdata[WITNESS_CHILDCOUNT];
259 static struct lock_list_entry w_locklistdata[LOCK_CHILDCOUNT];
260
261 static struct witness_order_list_entry order_lists[] = {
262 { "proctree", &lock_class_sx },
263 { "allproc", &lock_class_sx },
264 { "Giant", &lock_class_mtx_sleep },
265 { "filedesc structure", &lock_class_mtx_sleep },
266 { "pipe mutex", &lock_class_mtx_sleep },
267 { "sigio lock", &lock_class_mtx_sleep },
268 { "process group", &lock_class_mtx_sleep },
269 { "process lock", &lock_class_mtx_sleep },
270 { "session", &lock_class_mtx_sleep },
271 { "uidinfo hash", &lock_class_mtx_sleep },
272 { "uidinfo struct", &lock_class_mtx_sleep },
273 { "allprison", &lock_class_mtx_sleep },
274 { NULL, NULL },
275 /*
276 * Sockets
277 */
278 { "filedesc structure", &lock_class_mtx_sleep },
279 { "accept", &lock_class_mtx_sleep },
280 { "so_snd", &lock_class_mtx_sleep },
281 { "so_rcv", &lock_class_mtx_sleep },
282 { "sellck", &lock_class_mtx_sleep },
283 { NULL, NULL },
284 /*
285 * Routing
286 */
287 { "so_rcv", &lock_class_mtx_sleep },
288 { "radix node head", &lock_class_mtx_sleep },
289 { "rtentry", &lock_class_mtx_sleep },
290 { "ifaddr", &lock_class_mtx_sleep },
291 { NULL, NULL },
292 /*
293 * Multicast - protocol locks before interface locks.
294 */
295 { "in_multi_mtx", &lock_class_mtx_sleep },
296 { "igmp_mtx", &lock_class_mtx_sleep },
297 { "if_addr_mtx", &lock_class_mtx_sleep },
298 { NULL, NULL },
299 /*
300 * UNIX Domain Sockets
301 */
302 { "unp", &lock_class_mtx_sleep },
303 { "so_snd", &lock_class_mtx_sleep },
304 { NULL, NULL },
305 /*
306 * UDP/IP
307 */
308 { "udp", &lock_class_mtx_sleep },
309 { "udpinp", &lock_class_mtx_sleep },
310 { "so_snd", &lock_class_mtx_sleep },
311 { NULL, NULL },
312 /*
313 * TCP/IP
314 */
315 { "tcp", &lock_class_mtx_sleep },
316 { "tcpinp", &lock_class_mtx_sleep },
317 { "so_snd", &lock_class_mtx_sleep },
318 { NULL, NULL },
319 /*
320 * SLIP
321 */
322 { "slip_mtx", &lock_class_mtx_sleep },
323 { "slip sc_mtx", &lock_class_mtx_sleep },
324 { NULL, NULL },
325 /*
326 * netatalk
327 */
328 { "ddp_list_mtx", &lock_class_mtx_sleep },
329 { "ddp_mtx", &lock_class_mtx_sleep },
330 { NULL, NULL },
331 /*
332 * BPF
333 */
334 { "bpf global lock", &lock_class_mtx_sleep },
335 { "bpf interface lock", &lock_class_mtx_sleep },
336 { "bpf cdev lock", &lock_class_mtx_sleep },
337 { NULL, NULL },
338 /*
339 * spin locks
340 */
341 #ifdef SMP
342 { "ap boot", &lock_class_mtx_spin },
343 #endif
344 { "sio", &lock_class_mtx_spin },
345 #ifdef __i386__
346 { "cy", &lock_class_mtx_spin },
347 #endif
348 { "uart_hwmtx", &lock_class_mtx_spin },
349 { "sabtty", &lock_class_mtx_spin },
350 { "zstty", &lock_class_mtx_spin },
351 { "ng_node", &lock_class_mtx_spin },
352 { "ng_worklist", &lock_class_mtx_spin },
353 { "taskqueue_fast", &lock_class_mtx_spin },
354 { "intr table", &lock_class_mtx_spin },
355 { "ithread table lock", &lock_class_mtx_spin },
356 { "sleepq chain", &lock_class_mtx_spin },
357 { "sched lock", &lock_class_mtx_spin },
358 { "turnstile chain", &lock_class_mtx_spin },
359 { "td_contested", &lock_class_mtx_spin },
360 { "callout", &lock_class_mtx_spin },
361 { "entropy harvest mutex", &lock_class_mtx_spin },
362 /*
363 * leaf locks
364 */
365 { "allpmaps", &lock_class_mtx_spin },
366 { "vm page queue free mutex", &lock_class_mtx_spin },
367 { "icu", &lock_class_mtx_spin },
368 #ifdef SMP
369 { "smp rendezvous", &lock_class_mtx_spin },
370 #if defined(__i386__) || defined(__amd64__)
371 { "tlb", &lock_class_mtx_spin },
372 #endif
373 #ifdef __sparc64__
374 { "ipi", &lock_class_mtx_spin },
375 #endif
376 #endif
377 { "clk", &lock_class_mtx_spin },
378 { "mutex profiling lock", &lock_class_mtx_spin },
379 { "kse zombie lock", &lock_class_mtx_spin },
380 { "ALD Queue", &lock_class_mtx_spin },
381 #ifdef __ia64__
382 { "MCA spin lock", &lock_class_mtx_spin },
383 #endif
384 #if defined(__i386__) || defined(__amd64__)
385 { "pcicfg", &lock_class_mtx_spin },
386 #endif
387 { "tw_osl_io_lock", &lock_class_mtx_spin },
388 { "tw_osl_q_lock", &lock_class_mtx_spin },
389 { "tw_cl_io_lock", &lock_class_mtx_spin },
390 { "tw_cl_intr_lock", &lock_class_mtx_spin },
391 { "tw_cl_gen_lock", &lock_class_mtx_spin },
392 { NULL, NULL },
393 { NULL, NULL }
394 };
395
396 #ifdef BLESSING
397 /*
398 * Pairs of locks which have been blessed
399 * Don't complain about order problems with blessed locks
400 */
401 static struct witness_blessed blessed_list[] = {
402 };
403 static int blessed_count =
404 sizeof(blessed_list) / sizeof(struct witness_blessed);
405 #endif
406
407 /*
408 * List of all locks in the system.
409 */
410 TAILQ_HEAD(, lock_object) all_locks = TAILQ_HEAD_INITIALIZER(all_locks);
411
412 static struct mtx all_mtx = {
413 { &lock_class_mtx_sleep, /* mtx_object.lo_class */
414 "All locks list", /* mtx_object.lo_name */
415 "All locks list", /* mtx_object.lo_type */
416 LO_INITIALIZED, /* mtx_object.lo_flags */
417 { NULL, NULL }, /* mtx_object.lo_list */
418 NULL }, /* mtx_object.lo_witness */
419 MTX_UNOWNED, 0 /* mtx_lock, mtx_recurse */
420 };
421
422 /*
423 * This global is set to 0 once it becomes safe to use the witness code.
424 */
425 static int witness_cold = 1;
426
427 /*
428 * Global variables for book keeping.
429 */
430 static int lock_cur_cnt;
431 static int lock_max_cnt;
432
433 /*
434 * The WITNESS-enabled diagnostic code.
435 */
436 static void
437 witness_initialize(void *dummy __unused)
438 {
439 struct lock_object *lock;
440 struct witness_order_list_entry *order;
441 struct witness *w, *w1;
442 int i;
443
444 /*
445 * We have to release Giant before initializing its witness
446 * structure so that WITNESS doesn't get confused.
447 */
448 mtx_unlock(&Giant);
449 mtx_assert(&Giant, MA_NOTOWNED);
450
451 CTR1(KTR_WITNESS, "%s: initializing witness", __func__);
452 TAILQ_INSERT_HEAD(&all_locks, &all_mtx.mtx_object, lo_list);
453 mtx_init(&w_mtx, "witness lock", NULL, MTX_SPIN | MTX_QUIET |
454 MTX_NOWITNESS);
455 for (i = 0; i < WITNESS_COUNT; i++)
456 witness_free(&w_data[i]);
457 for (i = 0; i < WITNESS_CHILDCOUNT; i++)
458 witness_child_free(&w_childdata[i]);
459 for (i = 0; i < LOCK_CHILDCOUNT; i++)
460 witness_lock_list_free(&w_locklistdata[i]);
461
462 /* First add in all the specified order lists. */
463 for (order = order_lists; order->w_name != NULL; order++) {
464 w = enroll(order->w_name, order->w_class);
465 if (w == NULL)
466 continue;
467 w->w_file = "order list";
468 for (order++; order->w_name != NULL; order++) {
469 w1 = enroll(order->w_name, order->w_class);
470 if (w1 == NULL)
471 continue;
472 w1->w_file = "order list";
473 if (!itismychild(w, w1))
474 panic("Not enough memory for static orders!");
475 w = w1;
476 }
477 }
478
479 /* Iterate through all locks and add them to witness. */
480 mtx_lock(&all_mtx);
481 TAILQ_FOREACH(lock, &all_locks, lo_list) {
482 if (lock->lo_flags & LO_WITNESS)
483 lock->lo_witness = enroll(lock->lo_type,
484 lock->lo_class);
485 else
486 lock->lo_witness = NULL;
487 }
488 mtx_unlock(&all_mtx);
489
490 /* Mark the witness code as being ready for use. */
491 atomic_store_rel_int(&witness_cold, 0);
492
493 mtx_lock(&Giant);
494 }
495 SYSINIT(witness_init, SI_SUB_WITNESS, SI_ORDER_FIRST, witness_initialize, NULL)
496
497 static int
498 sysctl_debug_witness_watch(SYSCTL_HANDLER_ARGS)
499 {
500 int error, value;
501
502 value = witness_watch;
503 error = sysctl_handle_int(oidp, &value, 0, req);
504 if (error != 0 || req->newptr == NULL)
505 return (error);
506 error = suser(req->td);
507 if (error != 0)
508 return (error);
509 if (value == witness_watch)
510 return (0);
511 if (value != 0)
512 return (EINVAL);
513 witness_watch = 0;
514 return (0);
515 }
516
517 void
518 witness_init(struct lock_object *lock)
519 {
520 struct lock_class *class;
521
522 class = lock->lo_class;
523 if (lock->lo_flags & LO_INITIALIZED)
524 panic("%s: lock (%s) %s is already initialized", __func__,
525 class->lc_name, lock->lo_name);
526 if ((lock->lo_flags & LO_RECURSABLE) != 0 &&
527 (class->lc_flags & LC_RECURSABLE) == 0)
528 panic("%s: lock (%s) %s can not be recursable", __func__,
529 class->lc_name, lock->lo_name);
530 if ((lock->lo_flags & LO_SLEEPABLE) != 0 &&
531 (class->lc_flags & LC_SLEEPABLE) == 0)
532 panic("%s: lock (%s) %s can not be sleepable", __func__,
533 class->lc_name, lock->lo_name);
534 if ((lock->lo_flags & LO_UPGRADABLE) != 0 &&
535 (class->lc_flags & LC_UPGRADABLE) == 0)
536 panic("%s: lock (%s) %s can not be upgradable", __func__,
537 class->lc_name, lock->lo_name);
538
539 mtx_lock(&all_mtx);
540 TAILQ_INSERT_TAIL(&all_locks, lock, lo_list);
541 lock->lo_flags |= LO_INITIALIZED;
542 lock_cur_cnt++;
543 if (lock_cur_cnt > lock_max_cnt)
544 lock_max_cnt = lock_cur_cnt;
545 mtx_unlock(&all_mtx);
546 if (!witness_cold && witness_watch != 0 && panicstr == NULL &&
547 (lock->lo_flags & LO_WITNESS) != 0)
548 lock->lo_witness = enroll(lock->lo_type, class);
549 else
550 lock->lo_witness = NULL;
551 }
552
553 void
554 witness_destroy(struct lock_object *lock)
555 {
556 struct witness *w;
557
558 if (witness_cold)
559 panic("lock (%s) %s destroyed while witness_cold",
560 lock->lo_class->lc_name, lock->lo_name);
561 if ((lock->lo_flags & LO_INITIALIZED) == 0)
562 panic("%s: lock (%s) %s is not initialized", __func__,
563 lock->lo_class->lc_name, lock->lo_name);
564
565 /* XXX: need to verify that no one holds the lock */
566 w = lock->lo_witness;
567 if (w != NULL) {
568 mtx_lock_spin(&w_mtx);
569 MPASS(w->w_refcount > 0);
570 w->w_refcount--;
571
572 /*
573 * Lock is already released if we have an allocation failure
574 * and depart() fails.
575 */
576 if (w->w_refcount != 0 || depart(w))
577 mtx_unlock_spin(&w_mtx);
578 }
579
580 mtx_lock(&all_mtx);
581 lock_cur_cnt--;
582 TAILQ_REMOVE(&all_locks, lock, lo_list);
583 lock->lo_flags &= ~LO_INITIALIZED;
584 mtx_unlock(&all_mtx);
585 }
586
587 #ifdef DDB
588 static void
589 witness_levelall (void)
590 {
591 struct witness_list *list;
592 struct witness *w, *w1;
593
594 /*
595 * First clear all levels.
596 */
597 STAILQ_FOREACH(w, &w_all, w_list) {
598 w->w_level = 0;
599 }
600
601 /*
602 * Look for locks with no parent and level all their descendants.
603 */
604 STAILQ_FOREACH(w, &w_all, w_list) {
605 /*
606 * This is just an optimization, technically we could get
607 * away just walking the all list each time.
608 */
609 if (w->w_class->lc_flags & LC_SLEEPLOCK)
610 list = &w_sleep;
611 else
612 list = &w_spin;
613 STAILQ_FOREACH(w1, list, w_typelist) {
614 if (isitmychild(w1, w))
615 goto skip;
616 }
617 witness_leveldescendents(w, 0);
618 skip:
619 ; /* silence GCC 3.x */
620 }
621 }
622
623 static void
624 witness_leveldescendents(struct witness *parent, int level)
625 {
626 struct witness_child_list_entry *wcl;
627 int i;
628
629 if (parent->w_level < level)
630 parent->w_level = level;
631 level++;
632 for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next)
633 for (i = 0; i < wcl->wcl_count; i++)
634 witness_leveldescendents(wcl->wcl_children[i], level);
635 }
636
637 static void
638 witness_displaydescendants(void(*prnt)(const char *fmt, ...),
639 struct witness *parent, int indent)
640 {
641 struct witness_child_list_entry *wcl;
642 int i, level;
643
644 level = parent->w_level;
645 prnt("%-2d", level);
646 for (i = 0; i < indent; i++)
647 prnt(" ");
648 if (parent->w_refcount > 0)
649 prnt("%s", parent->w_name);
650 else
651 prnt("(dead)");
652 if (parent->w_displayed) {
653 prnt(" -- (already displayed)\n");
654 return;
655 }
656 parent->w_displayed = 1;
657 if (parent->w_refcount > 0) {
658 if (parent->w_file != NULL)
659 prnt(" -- last acquired @ %s:%d", parent->w_file,
660 parent->w_line);
661 }
662 prnt("\n");
663 for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next)
664 for (i = 0; i < wcl->wcl_count; i++)
665 witness_displaydescendants(prnt,
666 wcl->wcl_children[i], indent + 1);
667 }
668
669 static void
670 witness_display_list(void(*prnt)(const char *fmt, ...),
671 struct witness_list *list)
672 {
673 struct witness *w;
674
675 STAILQ_FOREACH(w, list, w_typelist) {
676 if (w->w_file == NULL || w->w_level > 0)
677 continue;
678 /*
679 * This lock has no anscestors, display its descendants.
680 */
681 witness_displaydescendants(prnt, w, 0);
682 }
683 }
684
685 static void
686 witness_display(void(*prnt)(const char *fmt, ...))
687 {
688 struct witness *w;
689
690 KASSERT(!witness_cold, ("%s: witness_cold", __func__));
691 witness_levelall();
692
693 /* Clear all the displayed flags. */
694 STAILQ_FOREACH(w, &w_all, w_list) {
695 w->w_displayed = 0;
696 }
697
698 /*
699 * First, handle sleep locks which have been acquired at least
700 * once.
701 */
702 prnt("Sleep locks:\n");
703 witness_display_list(prnt, &w_sleep);
704
705 /*
706 * Now do spin locks which have been acquired at least once.
707 */
708 prnt("\nSpin locks:\n");
709 witness_display_list(prnt, &w_spin);
710
711 /*
712 * Finally, any locks which have not been acquired yet.
713 */
714 prnt("\nLocks which were never acquired:\n");
715 STAILQ_FOREACH(w, &w_all, w_list) {
716 if (w->w_file != NULL || w->w_refcount == 0)
717 continue;
718 prnt("%s\n", w->w_name);
719 }
720 }
721 #endif /* DDB */
722
723 /* Trim useless garbage from filenames. */
724 static const char *
725 fixup_filename(const char *file)
726 {
727
728 if (file == NULL)
729 return (NULL);
730 while (strncmp(file, "../", 3) == 0)
731 file += 3;
732 return (file);
733 }
734
735 int
736 witness_defineorder(struct lock_object *lock1, struct lock_object *lock2)
737 {
738
739 if (witness_watch == 0 || panicstr != NULL)
740 return (0);
741
742 /* Require locks that witness knows about. */
743 if (lock1 == NULL || lock1->lo_witness == NULL || lock2 == NULL ||
744 lock2->lo_witness == NULL)
745 return (EINVAL);
746
747 MPASS(!mtx_owned(&w_mtx));
748 mtx_lock_spin(&w_mtx);
749
750 /*
751 * If we already have either an explicit or implied lock order that
752 * is the other way around, then return an error.
753 */
754 if (isitmydescendant(lock2->lo_witness, lock1->lo_witness)) {
755 mtx_unlock_spin(&w_mtx);
756 return (EDOOFUS);
757 }
758
759 /* Try to add the new order. */
760 CTR3(KTR_WITNESS, "%s: adding %s as a child of %s", __func__,
761 lock2->lo_type, lock1->lo_type);
762 if (!itismychild(lock1->lo_witness, lock2->lo_witness))
763 return (ENOMEM);
764 mtx_unlock_spin(&w_mtx);
765 return (0);
766 }
767
768 void
769 witness_checkorder(struct lock_object *lock, int flags, const char *file,
770 int line)
771 {
772 struct lock_list_entry **lock_list, *lle;
773 struct lock_instance *lock1, *lock2;
774 struct lock_class *class;
775 struct witness *w, *w1;
776 struct thread *td;
777 int i, j;
778
779 if (witness_cold || witness_watch == 0 || lock->lo_witness == NULL ||
780 panicstr != NULL)
781 return;
782
783 /*
784 * Try locks do not block if they fail to acquire the lock, thus
785 * there is no danger of deadlocks or of switching while holding a
786 * spin lock if we acquire a lock via a try operation. This
787 * function shouldn't even be called for try locks, so panic if
788 * that happens.
789 */
790 if (flags & LOP_TRYLOCK)
791 panic("%s should not be called for try lock operations",
792 __func__);
793
794 w = lock->lo_witness;
795 class = lock->lo_class;
796 td = curthread;
797 file = fixup_filename(file);
798
799 if (class->lc_flags & LC_SLEEPLOCK) {
800 /*
801 * Since spin locks include a critical section, this check
802 * implicitly enforces a lock order of all sleep locks before
803 * all spin locks.
804 */
805 if (td->td_critnest != 0)
806 panic("blockable sleep lock (%s) %s @ %s:%d",
807 class->lc_name, lock->lo_name, file, line);
808
809 /*
810 * If this is the first lock acquired then just return as
811 * no order checking is needed.
812 */
813 if (td->td_sleeplocks == NULL)
814 return;
815 lock_list = &td->td_sleeplocks;
816 } else {
817 /*
818 * If this is the first lock, just return as no order
819 * checking is needed. We check this in both if clauses
820 * here as unifying the check would require us to use a
821 * critical section to ensure we don't migrate while doing
822 * the check. Note that if this is not the first lock, we
823 * are already in a critical section and are safe for the
824 * rest of the check.
825 */
826 if (PCPU_GET(spinlocks) == NULL)
827 return;
828 lock_list = PCPU_PTR(spinlocks);
829 }
830
831 /*
832 * Check to see if we are recursing on a lock we already own. If
833 * so, make sure that we don't mismatch exclusive and shared lock
834 * acquires.
835 */
836 lock1 = find_instance(*lock_list, lock);
837 if (lock1 != NULL) {
838 if ((lock1->li_flags & LI_EXCLUSIVE) != 0 &&
839 (flags & LOP_EXCLUSIVE) == 0) {
840 printf("shared lock of (%s) %s @ %s:%d\n",
841 class->lc_name, lock->lo_name, file, line);
842 printf("while exclusively locked from %s:%d\n",
843 lock1->li_file, lock1->li_line);
844 panic("share->excl");
845 }
846 if ((lock1->li_flags & LI_EXCLUSIVE) == 0 &&
847 (flags & LOP_EXCLUSIVE) != 0) {
848 printf("exclusive lock of (%s) %s @ %s:%d\n",
849 class->lc_name, lock->lo_name, file, line);
850 printf("while share locked from %s:%d\n",
851 lock1->li_file, lock1->li_line);
852 panic("excl->share");
853 }
854 return;
855 }
856
857 /*
858 * Try locks do not block if they fail to acquire the lock, thus
859 * there is no danger of deadlocks or of switching while holding a
860 * spin lock if we acquire a lock via a try operation.
861 */
862 if (flags & LOP_TRYLOCK)
863 return;
864
865 /*
866 * Check for duplicate locks of the same type. Note that we only
867 * have to check for this on the last lock we just acquired. Any
868 * other cases will be caught as lock order violations.
869 */
870 lock1 = &(*lock_list)->ll_children[(*lock_list)->ll_count - 1];
871 w1 = lock1->li_lock->lo_witness;
872 if (w1 == w) {
873 if (w->w_same_squawked || (lock->lo_flags & LO_DUPOK))
874 return;
875 w->w_same_squawked = 1;
876 printf("acquiring duplicate lock of same type: \"%s\"\n",
877 lock->lo_type);
878 printf(" 1st %s @ %s:%d\n", lock1->li_lock->lo_name,
879 lock1->li_file, lock1->li_line);
880 printf(" 2nd %s @ %s:%d\n", lock->lo_name, file, line);
881 #ifdef KDB
882 goto debugger;
883 #else
884 return;
885 #endif
886 }
887 MPASS(!mtx_owned(&w_mtx));
888 mtx_lock_spin(&w_mtx);
889 /*
890 * If we know that the the lock we are acquiring comes after
891 * the lock we most recently acquired in the lock order tree,
892 * then there is no need for any further checks.
893 */
894 if (isitmychild(w1, w)) {
895 mtx_unlock_spin(&w_mtx);
896 return;
897 }
898 for (j = 0, lle = *lock_list; lle != NULL; lle = lle->ll_next) {
899 for (i = lle->ll_count - 1; i >= 0; i--, j++) {
900
901 MPASS(j < WITNESS_COUNT);
902 lock1 = &lle->ll_children[i];
903 w1 = lock1->li_lock->lo_witness;
904
905 /*
906 * If this lock doesn't undergo witness checking,
907 * then skip it.
908 */
909 if (w1 == NULL) {
910 KASSERT((lock1->li_lock->lo_flags & LO_WITNESS) == 0,
911 ("lock missing witness structure"));
912 continue;
913 }
914 /*
915 * If we are locking Giant and this is a sleepable
916 * lock, then skip it.
917 */
918 if ((lock1->li_lock->lo_flags & LO_SLEEPABLE) != 0 &&
919 lock == &Giant.mtx_object)
920 continue;
921 /*
922 * If we are locking a sleepable lock and this lock
923 * is Giant, then skip it.
924 */
925 if ((lock->lo_flags & LO_SLEEPABLE) != 0 &&
926 lock1->li_lock == &Giant.mtx_object)
927 continue;
928 /*
929 * If we are locking a sleepable lock and this lock
930 * isn't sleepable, we want to treat it as a lock
931 * order violation to enfore a general lock order of
932 * sleepable locks before non-sleepable locks.
933 */
934 if (!((lock->lo_flags & LO_SLEEPABLE) != 0 &&
935 (lock1->li_lock->lo_flags & LO_SLEEPABLE) == 0))
936 /*
937 * Check the lock order hierarchy for a reveresal.
938 */
939 if (!isitmydescendant(w, w1))
940 continue;
941 /*
942 * We have a lock order violation, check to see if it
943 * is allowed or has already been yelled about.
944 */
945 mtx_unlock_spin(&w_mtx);
946 #ifdef BLESSING
947 /*
948 * If the lock order is blessed, just bail. We don't
949 * look for other lock order violations though, which
950 * may be a bug.
951 */
952 if (blessed(w, w1))
953 return;
954 #endif
955 if (lock1->li_lock == &Giant.mtx_object) {
956 if (w1->w_Giant_squawked)
957 return;
958 else
959 w1->w_Giant_squawked = 1;
960 } else {
961 if (w1->w_other_squawked)
962 return;
963 else
964 w1->w_other_squawked = 1;
965 }
966 /*
967 * Ok, yell about it.
968 */
969 printf("lock order reversal\n");
970 /*
971 * Try to locate an earlier lock with
972 * witness w in our list.
973 */
974 do {
975 lock2 = &lle->ll_children[i];
976 MPASS(lock2->li_lock != NULL);
977 if (lock2->li_lock->lo_witness == w)
978 break;
979 if (i == 0 && lle->ll_next != NULL) {
980 lle = lle->ll_next;
981 i = lle->ll_count - 1;
982 MPASS(i >= 0 && i < LOCK_NCHILDREN);
983 } else
984 i--;
985 } while (i >= 0);
986 if (i < 0) {
987 printf(" 1st %p %s (%s) @ %s:%d\n",
988 lock1->li_lock, lock1->li_lock->lo_name,
989 lock1->li_lock->lo_type, lock1->li_file,
990 lock1->li_line);
991 printf(" 2nd %p %s (%s) @ %s:%d\n", lock,
992 lock->lo_name, lock->lo_type, file, line);
993 } else {
994 printf(" 1st %p %s (%s) @ %s:%d\n",
995 lock2->li_lock, lock2->li_lock->lo_name,
996 lock2->li_lock->lo_type, lock2->li_file,
997 lock2->li_line);
998 printf(" 2nd %p %s (%s) @ %s:%d\n",
999 lock1->li_lock, lock1->li_lock->lo_name,
1000 lock1->li_lock->lo_type, lock1->li_file,
1001 lock1->li_line);
1002 printf(" 3rd %p %s (%s) @ %s:%d\n", lock,
1003 lock->lo_name, lock->lo_type, file, line);
1004 }
1005 #ifdef KDB
1006 goto debugger;
1007 #else
1008 return;
1009 #endif
1010 }
1011 }
1012 lock1 = &(*lock_list)->ll_children[(*lock_list)->ll_count - 1];
1013 /*
1014 * If requested, build a new lock order. However, don't build a new
1015 * relationship between a sleepable lock and Giant if it is in the
1016 * wrong direction. The correct lock order is that sleepable locks
1017 * always come before Giant.
1018 */
1019 if (flags & LOP_NEWORDER &&
1020 !(lock1->li_lock == &Giant.mtx_object &&
1021 (lock->lo_flags & LO_SLEEPABLE) != 0)) {
1022 CTR3(KTR_WITNESS, "%s: adding %s as a child of %s", __func__,
1023 lock->lo_type, lock1->li_lock->lo_type);
1024 if (!itismychild(lock1->li_lock->lo_witness, w))
1025 /* Witness is dead. */
1026 return;
1027 }
1028 mtx_unlock_spin(&w_mtx);
1029 return;
1030
1031 #ifdef KDB
1032 debugger:
1033 if (witness_trace)
1034 kdb_backtrace();
1035 if (witness_kdb)
1036 kdb_enter(__func__);
1037 #endif
1038 }
1039
1040 void
1041 witness_lock(struct lock_object *lock, int flags, const char *file, int line)
1042 {
1043 struct lock_list_entry **lock_list, *lle;
1044 struct lock_instance *instance;
1045 struct witness *w;
1046 struct thread *td;
1047
1048 if (witness_cold || witness_watch == 0 || lock->lo_witness == NULL ||
1049 panicstr != NULL)
1050 return;
1051 w = lock->lo_witness;
1052 td = curthread;
1053 file = fixup_filename(file);
1054
1055 /* Determine lock list for this lock. */
1056 if (lock->lo_class->lc_flags & LC_SLEEPLOCK)
1057 lock_list = &td->td_sleeplocks;
1058 else
1059 lock_list = PCPU_PTR(spinlocks);
1060
1061 /* Check to see if we are recursing on a lock we already own. */
1062 instance = find_instance(*lock_list, lock);
1063 if (instance != NULL) {
1064 instance->li_flags++;
1065 CTR4(KTR_WITNESS, "%s: pid %d recursed on %s r=%d", __func__,
1066 td->td_proc->p_pid, lock->lo_name,
1067 instance->li_flags & LI_RECURSEMASK);
1068 instance->li_file = file;
1069 instance->li_line = line;
1070 return;
1071 }
1072
1073 /* Update per-witness last file and line acquire. */
1074 w->w_file = file;
1075 w->w_line = line;
1076
1077 /* Find the next open lock instance in the list and fill it. */
1078 lle = *lock_list;
1079 if (lle == NULL || lle->ll_count == LOCK_NCHILDREN) {
1080 lle = witness_lock_list_get();
1081 if (lle == NULL)
1082 return;
1083 lle->ll_next = *lock_list;
1084 CTR3(KTR_WITNESS, "%s: pid %d added lle %p", __func__,
1085 td->td_proc->p_pid, lle);
1086 *lock_list = lle;
1087 }
1088 instance = &lle->ll_children[lle->ll_count++];
1089 instance->li_lock = lock;
1090 instance->li_line = line;
1091 instance->li_file = file;
1092 if ((flags & LOP_EXCLUSIVE) != 0)
1093 instance->li_flags = LI_EXCLUSIVE;
1094 else
1095 instance->li_flags = 0;
1096 CTR4(KTR_WITNESS, "%s: pid %d added %s as lle[%d]", __func__,
1097 td->td_proc->p_pid, lock->lo_name, lle->ll_count - 1);
1098 }
1099
1100 void
1101 witness_upgrade(struct lock_object *lock, int flags, const char *file, int line)
1102 {
1103 struct lock_instance *instance;
1104 struct lock_class *class;
1105
1106 KASSERT(!witness_cold, ("%s: witness_cold", __func__));
1107 if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL)
1108 return;
1109 class = lock->lo_class;
1110 file = fixup_filename(file);
1111 if ((lock->lo_flags & LO_UPGRADABLE) == 0)
1112 panic("upgrade of non-upgradable lock (%s) %s @ %s:%d",
1113 class->lc_name, lock->lo_name, file, line);
1114 if ((flags & LOP_TRYLOCK) == 0)
1115 panic("non-try upgrade of lock (%s) %s @ %s:%d", class->lc_name,
1116 lock->lo_name, file, line);
1117 if ((lock->lo_class->lc_flags & LC_SLEEPLOCK) == 0)
1118 panic("upgrade of non-sleep lock (%s) %s @ %s:%d",
1119 class->lc_name, lock->lo_name, file, line);
1120 instance = find_instance(curthread->td_sleeplocks, lock);
1121 if (instance == NULL)
1122 panic("upgrade of unlocked lock (%s) %s @ %s:%d",
1123 class->lc_name, lock->lo_name, file, line);
1124 if ((instance->li_flags & LI_EXCLUSIVE) != 0)
1125 panic("upgrade of exclusive lock (%s) %s @ %s:%d",
1126 class->lc_name, lock->lo_name, file, line);
1127 if ((instance->li_flags & LI_RECURSEMASK) != 0)
1128 panic("upgrade of recursed lock (%s) %s r=%d @ %s:%d",
1129 class->lc_name, lock->lo_name,
1130 instance->li_flags & LI_RECURSEMASK, file, line);
1131 instance->li_flags |= LI_EXCLUSIVE;
1132 }
1133
1134 void
1135 witness_downgrade(struct lock_object *lock, int flags, const char *file,
1136 int line)
1137 {
1138 struct lock_instance *instance;
1139 struct lock_class *class;
1140
1141 KASSERT(!witness_cold, ("%s: witness_cold", __func__));
1142 if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL)
1143 return;
1144 class = lock->lo_class;
1145 file = fixup_filename(file);
1146 if ((lock->lo_flags & LO_UPGRADABLE) == 0)
1147 panic("downgrade of non-upgradable lock (%s) %s @ %s:%d",
1148 class->lc_name, lock->lo_name, file, line);
1149 if ((lock->lo_class->lc_flags & LC_SLEEPLOCK) == 0)
1150 panic("downgrade of non-sleep lock (%s) %s @ %s:%d",
1151 class->lc_name, lock->lo_name, file, line);
1152 instance = find_instance(curthread->td_sleeplocks, lock);
1153 if (instance == NULL)
1154 panic("downgrade of unlocked lock (%s) %s @ %s:%d",
1155 class->lc_name, lock->lo_name, file, line);
1156 if ((instance->li_flags & LI_EXCLUSIVE) == 0)
1157 panic("downgrade of shared lock (%s) %s @ %s:%d",
1158 class->lc_name, lock->lo_name, file, line);
1159 if ((instance->li_flags & LI_RECURSEMASK) != 0)
1160 panic("downgrade of recursed lock (%s) %s r=%d @ %s:%d",
1161 class->lc_name, lock->lo_name,
1162 instance->li_flags & LI_RECURSEMASK, file, line);
1163 instance->li_flags &= ~LI_EXCLUSIVE;
1164 }
1165
1166 void
1167 witness_unlock(struct lock_object *lock, int flags, const char *file, int line)
1168 {
1169 struct lock_list_entry **lock_list, *lle;
1170 struct lock_instance *instance;
1171 struct lock_class *class;
1172 struct thread *td;
1173 register_t s;
1174 int i, j;
1175
1176 if (witness_cold || witness_watch == 0 || lock->lo_witness == NULL ||
1177 panicstr != NULL)
1178 return;
1179 td = curthread;
1180 class = lock->lo_class;
1181 file = fixup_filename(file);
1182
1183 /* Find lock instance associated with this lock. */
1184 if (class->lc_flags & LC_SLEEPLOCK)
1185 lock_list = &td->td_sleeplocks;
1186 else
1187 lock_list = PCPU_PTR(spinlocks);
1188 for (; *lock_list != NULL; lock_list = &(*lock_list)->ll_next)
1189 for (i = 0; i < (*lock_list)->ll_count; i++) {
1190 instance = &(*lock_list)->ll_children[i];
1191 if (instance->li_lock == lock)
1192 goto found;
1193 }
1194 panic("lock (%s) %s not locked @ %s:%d", class->lc_name, lock->lo_name,
1195 file, line);
1196 found:
1197
1198 /* First, check for shared/exclusive mismatches. */
1199 if ((instance->li_flags & LI_EXCLUSIVE) != 0 &&
1200 (flags & LOP_EXCLUSIVE) == 0) {
1201 printf("shared unlock of (%s) %s @ %s:%d\n", class->lc_name,
1202 lock->lo_name, file, line);
1203 printf("while exclusively locked from %s:%d\n",
1204 instance->li_file, instance->li_line);
1205 panic("excl->ushare");
1206 }
1207 if ((instance->li_flags & LI_EXCLUSIVE) == 0 &&
1208 (flags & LOP_EXCLUSIVE) != 0) {
1209 printf("exclusive unlock of (%s) %s @ %s:%d\n", class->lc_name,
1210 lock->lo_name, file, line);
1211 printf("while share locked from %s:%d\n", instance->li_file,
1212 instance->li_line);
1213 panic("share->uexcl");
1214 }
1215
1216 /* If we are recursed, unrecurse. */
1217 if ((instance->li_flags & LI_RECURSEMASK) > 0) {
1218 CTR4(KTR_WITNESS, "%s: pid %d unrecursed on %s r=%d", __func__,
1219 td->td_proc->p_pid, instance->li_lock->lo_name,
1220 instance->li_flags);
1221 instance->li_flags--;
1222 return;
1223 }
1224
1225 /* Otherwise, remove this item from the list. */
1226 s = intr_disable();
1227 CTR4(KTR_WITNESS, "%s: pid %d removed %s from lle[%d]", __func__,
1228 td->td_proc->p_pid, instance->li_lock->lo_name,
1229 (*lock_list)->ll_count - 1);
1230 for (j = i; j < (*lock_list)->ll_count - 1; j++)
1231 (*lock_list)->ll_children[j] =
1232 (*lock_list)->ll_children[j + 1];
1233 (*lock_list)->ll_count--;
1234 intr_restore(s);
1235
1236 /* If this lock list entry is now empty, free it. */
1237 if ((*lock_list)->ll_count == 0) {
1238 lle = *lock_list;
1239 *lock_list = lle->ll_next;
1240 CTR3(KTR_WITNESS, "%s: pid %d removed lle %p", __func__,
1241 td->td_proc->p_pid, lle);
1242 witness_lock_list_free(lle);
1243 }
1244 }
1245
1246 /*
1247 * Warn if any locks other than 'lock' are held. Flags can be passed in to
1248 * exempt Giant and sleepable locks from the checks as well. If any
1249 * non-exempt locks are held, then a supplied message is printed to the
1250 * console along with a list of the offending locks. If indicated in the
1251 * flags then a failure results in a panic as well.
1252 */
1253 int
1254 witness_warn(int flags, struct lock_object *lock, const char *fmt, ...)
1255 {
1256 struct lock_list_entry *lle;
1257 struct lock_instance *lock1;
1258 struct thread *td;
1259 va_list ap;
1260 int i, n;
1261
1262 if (witness_cold || witness_watch == 0 || panicstr != NULL)
1263 return (0);
1264 n = 0;
1265 td = curthread;
1266 for (lle = td->td_sleeplocks; lle != NULL; lle = lle->ll_next)
1267 for (i = lle->ll_count - 1; i >= 0; i--) {
1268 lock1 = &lle->ll_children[i];
1269 if (lock1->li_lock == lock)
1270 continue;
1271 if (flags & WARN_GIANTOK &&
1272 lock1->li_lock == &Giant.mtx_object)
1273 continue;
1274 if (flags & WARN_SLEEPOK &&
1275 (lock1->li_lock->lo_flags & LO_SLEEPABLE) != 0)
1276 continue;
1277 if (n == 0) {
1278 va_start(ap, fmt);
1279 vprintf(fmt, ap);
1280 va_end(ap);
1281 printf(" with the following");
1282 if (flags & WARN_SLEEPOK)
1283 printf(" non-sleepable");
1284 printf(" locks held:\n");
1285 }
1286 n++;
1287 witness_list_lock(lock1);
1288 }
1289 if (PCPU_GET(spinlocks) != NULL) {
1290 /*
1291 * Since we already hold a spinlock preemption is
1292 * already blocked.
1293 */
1294 if (n == 0) {
1295 va_start(ap, fmt);
1296 vprintf(fmt, ap);
1297 va_end(ap);
1298 printf(" with the following");
1299 if (flags & WARN_SLEEPOK)
1300 printf(" non-sleepable");
1301 printf(" locks held:\n");
1302 }
1303 n += witness_list_locks(PCPU_PTR(spinlocks));
1304 }
1305 if (flags & WARN_PANIC && n)
1306 panic("witness_warn");
1307 #ifdef KDB
1308 else if (witness_kdb && n)
1309 kdb_enter(__func__);
1310 else if (witness_trace && n)
1311 kdb_backtrace();
1312 #endif
1313 return (n);
1314 }
1315
1316 const char *
1317 witness_file(struct lock_object *lock)
1318 {
1319 struct witness *w;
1320
1321 if (witness_cold || witness_watch == 0 || lock->lo_witness == NULL)
1322 return ("?");
1323 w = lock->lo_witness;
1324 return (w->w_file);
1325 }
1326
1327 int
1328 witness_line(struct lock_object *lock)
1329 {
1330 struct witness *w;
1331
1332 if (witness_cold || witness_watch == 0 || lock->lo_witness == NULL)
1333 return (0);
1334 w = lock->lo_witness;
1335 return (w->w_line);
1336 }
1337
1338 static struct witness *
1339 enroll(const char *description, struct lock_class *lock_class)
1340 {
1341 struct witness *w;
1342
1343 if (witness_watch == 0 || panicstr != NULL)
1344 return (NULL);
1345 if ((lock_class->lc_flags & LC_SPINLOCK) && witness_skipspin)
1346 return (NULL);
1347 mtx_lock_spin(&w_mtx);
1348 STAILQ_FOREACH(w, &w_all, w_list) {
1349 if (w->w_name == description || (w->w_refcount > 0 &&
1350 strcmp(description, w->w_name) == 0)) {
1351 w->w_refcount++;
1352 mtx_unlock_spin(&w_mtx);
1353 if (lock_class != w->w_class)
1354 panic(
1355 "lock (%s) %s does not match earlier (%s) lock",
1356 description, lock_class->lc_name,
1357 w->w_class->lc_name);
1358 return (w);
1359 }
1360 }
1361 /*
1362 * This isn't quite right, as witness_cold is still 0 while we
1363 * enroll all the locks initialized before witness_initialize().
1364 */
1365 if ((lock_class->lc_flags & LC_SPINLOCK) && !witness_cold) {
1366 mtx_unlock_spin(&w_mtx);
1367 panic("spin lock %s not in order list", description);
1368 }
1369 if ((w = witness_get()) == NULL)
1370 return (NULL);
1371 w->w_name = description;
1372 w->w_class = lock_class;
1373 w->w_refcount = 1;
1374 STAILQ_INSERT_HEAD(&w_all, w, w_list);
1375 if (lock_class->lc_flags & LC_SPINLOCK) {
1376 STAILQ_INSERT_HEAD(&w_spin, w, w_typelist);
1377 w_spin_cnt++;
1378 } else if (lock_class->lc_flags & LC_SLEEPLOCK) {
1379 STAILQ_INSERT_HEAD(&w_sleep, w, w_typelist);
1380 w_sleep_cnt++;
1381 } else {
1382 mtx_unlock_spin(&w_mtx);
1383 panic("lock class %s is not sleep or spin",
1384 lock_class->lc_name);
1385 }
1386 mtx_unlock_spin(&w_mtx);
1387 return (w);
1388 }
1389
1390 /* Don't let the door bang you on the way out... */
1391 static int
1392 depart(struct witness *w)
1393 {
1394 struct witness_child_list_entry *wcl, *nwcl;
1395 struct witness_list *list;
1396 struct witness *parent;
1397
1398 MPASS(w->w_refcount == 0);
1399 if (w->w_class->lc_flags & LC_SLEEPLOCK) {
1400 list = &w_sleep;
1401 w_sleep_cnt--;
1402 } else {
1403 list = &w_spin;
1404 w_spin_cnt--;
1405 }
1406 /*
1407 * First, we run through the entire tree looking for any
1408 * witnesses that the outgoing witness is a child of. For
1409 * each parent that we find, we reparent all the direct
1410 * children of the outgoing witness to its parent.
1411 */
1412 STAILQ_FOREACH(parent, list, w_typelist) {
1413 if (!isitmychild(parent, w))
1414 continue;
1415 removechild(parent, w);
1416 }
1417
1418 /*
1419 * Now we go through and free up the child list of the
1420 * outgoing witness.
1421 */
1422 for (wcl = w->w_children; wcl != NULL; wcl = nwcl) {
1423 nwcl = wcl->wcl_next;
1424 w_child_cnt--;
1425 witness_child_free(wcl);
1426 }
1427
1428 /*
1429 * Detach from various lists and free.
1430 */
1431 STAILQ_REMOVE(list, w, witness, w_typelist);
1432 STAILQ_REMOVE(&w_all, w, witness, w_list);
1433 witness_free(w);
1434
1435 return (1);
1436 }
1437
1438 /*
1439 * Add "child" as a direct child of "parent". Returns false if
1440 * we fail due to out of memory.
1441 */
1442 static int
1443 insertchild(struct witness *parent, struct witness *child)
1444 {
1445 struct witness_child_list_entry **wcl;
1446
1447 MPASS(child != NULL && parent != NULL);
1448
1449 /*
1450 * Insert "child" after "parent"
1451 */
1452 wcl = &parent->w_children;
1453 while (*wcl != NULL && (*wcl)->wcl_count == WITNESS_NCHILDREN)
1454 wcl = &(*wcl)->wcl_next;
1455 if (*wcl == NULL) {
1456 *wcl = witness_child_get();
1457 if (*wcl == NULL)
1458 return (0);
1459 w_child_cnt++;
1460 }
1461 (*wcl)->wcl_children[(*wcl)->wcl_count++] = child;
1462
1463 return (1);
1464 }
1465
1466
1467 static int
1468 itismychild(struct witness *parent, struct witness *child)
1469 {
1470 struct witness_list *list;
1471
1472 MPASS(child != NULL && parent != NULL);
1473 if ((parent->w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK)) !=
1474 (child->w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK)))
1475 panic(
1476 "%s: parent (%s) and child (%s) are not the same lock type",
1477 __func__, parent->w_class->lc_name,
1478 child->w_class->lc_name);
1479
1480 if (!insertchild(parent, child))
1481 return (0);
1482
1483 if (parent->w_class->lc_flags & LC_SLEEPLOCK)
1484 list = &w_sleep;
1485 else
1486 list = &w_spin;
1487 return (1);
1488 }
1489
1490 static void
1491 removechild(struct witness *parent, struct witness *child)
1492 {
1493 struct witness_child_list_entry **wcl, *wcl1;
1494 int i;
1495
1496 for (wcl = &parent->w_children; *wcl != NULL; wcl = &(*wcl)->wcl_next)
1497 for (i = 0; i < (*wcl)->wcl_count; i++)
1498 if ((*wcl)->wcl_children[i] == child)
1499 goto found;
1500 return;
1501 found:
1502 (*wcl)->wcl_count--;
1503 if ((*wcl)->wcl_count > i)
1504 (*wcl)->wcl_children[i] =
1505 (*wcl)->wcl_children[(*wcl)->wcl_count];
1506 MPASS((*wcl)->wcl_children[i] != NULL);
1507 if ((*wcl)->wcl_count != 0)
1508 return;
1509 wcl1 = *wcl;
1510 *wcl = wcl1->wcl_next;
1511 w_child_cnt--;
1512 witness_child_free(wcl1);
1513 }
1514
1515 static int
1516 isitmychild(struct witness *parent, struct witness *child)
1517 {
1518 struct witness_child_list_entry *wcl;
1519 int i;
1520
1521 for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next) {
1522 for (i = 0; i < wcl->wcl_count; i++) {
1523 if (wcl->wcl_children[i] == child)
1524 return (1);
1525 }
1526 }
1527 return (0);
1528 }
1529
1530 static int
1531 isitmydescendant(struct witness *parent, struct witness *child)
1532 {
1533 struct witness_child_list_entry *wcl;
1534 int i, j;
1535
1536 if (isitmychild(parent, child))
1537 return (1);
1538 j = 0;
1539 for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next) {
1540 MPASS(j < 1000);
1541 for (i = 0; i < wcl->wcl_count; i++) {
1542 if (isitmydescendant(wcl->wcl_children[i], child))
1543 return (1);
1544 }
1545 j++;
1546 }
1547 return (0);
1548 }
1549
1550 #ifdef BLESSING
1551 static int
1552 blessed(struct witness *w1, struct witness *w2)
1553 {
1554 int i;
1555 struct witness_blessed *b;
1556
1557 for (i = 0; i < blessed_count; i++) {
1558 b = &blessed_list[i];
1559 if (strcmp(w1->w_name, b->b_lock1) == 0) {
1560 if (strcmp(w2->w_name, b->b_lock2) == 0)
1561 return (1);
1562 continue;
1563 }
1564 if (strcmp(w1->w_name, b->b_lock2) == 0)
1565 if (strcmp(w2->w_name, b->b_lock1) == 0)
1566 return (1);
1567 }
1568 return (0);
1569 }
1570 #endif
1571
1572 static struct witness *
1573 witness_get(void)
1574 {
1575 struct witness *w;
1576
1577 if (witness_watch == 0) {
1578 mtx_unlock_spin(&w_mtx);
1579 return (NULL);
1580 }
1581 if (STAILQ_EMPTY(&w_free)) {
1582 witness_watch = 0;
1583 mtx_unlock_spin(&w_mtx);
1584 printf("%s: witness exhausted\n", __func__);
1585 return (NULL);
1586 }
1587 w = STAILQ_FIRST(&w_free);
1588 STAILQ_REMOVE_HEAD(&w_free, w_list);
1589 w_free_cnt--;
1590 bzero(w, sizeof(*w));
1591 return (w);
1592 }
1593
1594 static void
1595 witness_free(struct witness *w)
1596 {
1597
1598 STAILQ_INSERT_HEAD(&w_free, w, w_list);
1599 w_free_cnt++;
1600 }
1601
1602 static struct witness_child_list_entry *
1603 witness_child_get(void)
1604 {
1605 struct witness_child_list_entry *wcl;
1606
1607 if (witness_watch == 0) {
1608 mtx_unlock_spin(&w_mtx);
1609 return (NULL);
1610 }
1611 wcl = w_child_free;
1612 if (wcl == NULL) {
1613 witness_watch = 0;
1614 mtx_unlock_spin(&w_mtx);
1615 printf("%s: witness exhausted\n", __func__);
1616 return (NULL);
1617 }
1618 w_child_free = wcl->wcl_next;
1619 w_child_free_cnt--;
1620 bzero(wcl, sizeof(*wcl));
1621 return (wcl);
1622 }
1623
1624 static void
1625 witness_child_free(struct witness_child_list_entry *wcl)
1626 {
1627
1628 wcl->wcl_next = w_child_free;
1629 w_child_free = wcl;
1630 w_child_free_cnt++;
1631 }
1632
1633 static struct lock_list_entry *
1634 witness_lock_list_get(void)
1635 {
1636 struct lock_list_entry *lle;
1637
1638 if (witness_watch == 0)
1639 return (NULL);
1640 mtx_lock_spin(&w_mtx);
1641 lle = w_lock_list_free;
1642 if (lle == NULL) {
1643 witness_watch = 0;
1644 mtx_unlock_spin(&w_mtx);
1645 printf("%s: witness exhausted\n", __func__);
1646 return (NULL);
1647 }
1648 w_lock_list_free = lle->ll_next;
1649 mtx_unlock_spin(&w_mtx);
1650 bzero(lle, sizeof(*lle));
1651 return (lle);
1652 }
1653
1654 static void
1655 witness_lock_list_free(struct lock_list_entry *lle)
1656 {
1657
1658 mtx_lock_spin(&w_mtx);
1659 lle->ll_next = w_lock_list_free;
1660 w_lock_list_free = lle;
1661 mtx_unlock_spin(&w_mtx);
1662 }
1663
1664 static struct lock_instance *
1665 find_instance(struct lock_list_entry *lock_list, struct lock_object *lock)
1666 {
1667 struct lock_list_entry *lle;
1668 struct lock_instance *instance;
1669 int i;
1670
1671 for (lle = lock_list; lle != NULL; lle = lle->ll_next)
1672 for (i = lle->ll_count - 1; i >= 0; i--) {
1673 instance = &lle->ll_children[i];
1674 if (instance->li_lock == lock)
1675 return (instance);
1676 }
1677 return (NULL);
1678 }
1679
1680 static void
1681 witness_list_lock(struct lock_instance *instance)
1682 {
1683 struct lock_object *lock;
1684
1685 lock = instance->li_lock;
1686 printf("%s %s %s", (instance->li_flags & LI_EXCLUSIVE) != 0 ?
1687 "exclusive" : "shared", lock->lo_class->lc_name, lock->lo_name);
1688 if (lock->lo_type != lock->lo_name)
1689 printf(" (%s)", lock->lo_type);
1690 printf(" r = %d (%p) locked @ %s:%d\n",
1691 instance->li_flags & LI_RECURSEMASK, lock, instance->li_file,
1692 instance->li_line);
1693 }
1694
1695 #ifdef DDB
1696 static int
1697 witness_thread_has_locks(struct thread *td)
1698 {
1699
1700 return (td->td_sleeplocks != NULL);
1701 }
1702
1703 static int
1704 witness_proc_has_locks(struct proc *p)
1705 {
1706 struct thread *td;
1707
1708 FOREACH_THREAD_IN_PROC(p, td) {
1709 if (witness_thread_has_locks(td))
1710 return (1);
1711 }
1712 return (0);
1713 }
1714 #endif
1715
1716 int
1717 witness_list_locks(struct lock_list_entry **lock_list)
1718 {
1719 struct lock_list_entry *lle;
1720 int i, nheld;
1721
1722 nheld = 0;
1723 for (lle = *lock_list; lle != NULL; lle = lle->ll_next)
1724 for (i = lle->ll_count - 1; i >= 0; i--) {
1725 witness_list_lock(&lle->ll_children[i]);
1726 nheld++;
1727 }
1728 return (nheld);
1729 }
1730
1731 /*
1732 * This is a bit risky at best. We call this function when we have timed
1733 * out acquiring a spin lock, and we assume that the other CPU is stuck
1734 * with this lock held. So, we go groveling around in the other CPU's
1735 * per-cpu data to try to find the lock instance for this spin lock to
1736 * see when it was last acquired.
1737 */
1738 void
1739 witness_display_spinlock(struct lock_object *lock, struct thread *owner)
1740 {
1741 struct lock_instance *instance;
1742 struct pcpu *pc;
1743
1744 if (owner->td_critnest == 0 || owner->td_oncpu == NOCPU)
1745 return;
1746 pc = pcpu_find(owner->td_oncpu);
1747 instance = find_instance(pc->pc_spinlocks, lock);
1748 if (instance != NULL)
1749 witness_list_lock(instance);
1750 }
1751
1752 void
1753 witness_save(struct lock_object *lock, const char **filep, int *linep)
1754 {
1755 struct lock_instance *instance;
1756
1757 KASSERT(!witness_cold, ("%s: witness_cold", __func__));
1758 if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL)
1759 return;
1760 if ((lock->lo_class->lc_flags & LC_SLEEPLOCK) == 0)
1761 panic("%s: lock (%s) %s is not a sleep lock", __func__,
1762 lock->lo_class->lc_name, lock->lo_name);
1763 instance = find_instance(curthread->td_sleeplocks, lock);
1764 if (instance == NULL)
1765 panic("%s: lock (%s) %s not locked", __func__,
1766 lock->lo_class->lc_name, lock->lo_name);
1767 *filep = instance->li_file;
1768 *linep = instance->li_line;
1769 }
1770
1771 void
1772 witness_restore(struct lock_object *lock, const char *file, int line)
1773 {
1774 struct lock_instance *instance;
1775
1776 KASSERT(!witness_cold, ("%s: witness_cold", __func__));
1777 if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL)
1778 return;
1779 if ((lock->lo_class->lc_flags & LC_SLEEPLOCK) == 0)
1780 panic("%s: lock (%s) %s is not a sleep lock", __func__,
1781 lock->lo_class->lc_name, lock->lo_name);
1782 instance = find_instance(curthread->td_sleeplocks, lock);
1783 if (instance == NULL)
1784 panic("%s: lock (%s) %s not locked", __func__,
1785 lock->lo_class->lc_name, lock->lo_name);
1786 lock->lo_witness->w_file = file;
1787 lock->lo_witness->w_line = line;
1788 instance->li_file = file;
1789 instance->li_line = line;
1790 }
1791
1792 void
1793 witness_assert(struct lock_object *lock, int flags, const char *file, int line)
1794 {
1795 #ifdef INVARIANT_SUPPORT
1796 struct lock_instance *instance;
1797
1798 if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL)
1799 return;
1800 if ((lock->lo_class->lc_flags & LC_SLEEPLOCK) != 0)
1801 instance = find_instance(curthread->td_sleeplocks, lock);
1802 else if ((lock->lo_class->lc_flags & LC_SPINLOCK) != 0)
1803 instance = find_instance(PCPU_GET(spinlocks), lock);
1804 else {
1805 panic("Lock (%s) %s is not sleep or spin!",
1806 lock->lo_class->lc_name, lock->lo_name);
1807 }
1808 file = fixup_filename(file);
1809 switch (flags) {
1810 case LA_UNLOCKED:
1811 if (instance != NULL)
1812 panic("Lock (%s) %s locked @ %s:%d.",
1813 lock->lo_class->lc_name, lock->lo_name, file, line);
1814 break;
1815 case LA_LOCKED:
1816 case LA_LOCKED | LA_RECURSED:
1817 case LA_LOCKED | LA_NOTRECURSED:
1818 case LA_SLOCKED:
1819 case LA_SLOCKED | LA_RECURSED:
1820 case LA_SLOCKED | LA_NOTRECURSED:
1821 case LA_XLOCKED:
1822 case LA_XLOCKED | LA_RECURSED:
1823 case LA_XLOCKED | LA_NOTRECURSED:
1824 if (instance == NULL) {
1825 panic("Lock (%s) %s not locked @ %s:%d.",
1826 lock->lo_class->lc_name, lock->lo_name, file, line);
1827 break;
1828 }
1829 if ((flags & LA_XLOCKED) != 0 &&
1830 (instance->li_flags & LI_EXCLUSIVE) == 0)
1831 panic("Lock (%s) %s not exclusively locked @ %s:%d.",
1832 lock->lo_class->lc_name, lock->lo_name, file, line);
1833 if ((flags & LA_SLOCKED) != 0 &&
1834 (instance->li_flags & LI_EXCLUSIVE) != 0)
1835 panic("Lock (%s) %s exclusively locked @ %s:%d.",
1836 lock->lo_class->lc_name, lock->lo_name, file, line);
1837 if ((flags & LA_RECURSED) != 0 &&
1838 (instance->li_flags & LI_RECURSEMASK) == 0)
1839 panic("Lock (%s) %s not recursed @ %s:%d.",
1840 lock->lo_class->lc_name, lock->lo_name, file, line);
1841 if ((flags & LA_NOTRECURSED) != 0 &&
1842 (instance->li_flags & LI_RECURSEMASK) != 0)
1843 panic("Lock (%s) %s recursed @ %s:%d.",
1844 lock->lo_class->lc_name, lock->lo_name, file, line);
1845 break;
1846 default:
1847 panic("Invalid lock assertion at %s:%d.", file, line);
1848
1849 }
1850 #endif /* INVARIANT_SUPPORT */
1851 }
1852
1853 #ifdef DDB
1854 static void
1855 witness_list(struct thread *td)
1856 {
1857
1858 KASSERT(!witness_cold, ("%s: witness_cold", __func__));
1859 KASSERT(kdb_active, ("%s: not in the debugger", __func__));
1860
1861 if (witness_watch == 0)
1862 return;
1863
1864 witness_list_locks(&td->td_sleeplocks);
1865
1866 /*
1867 * We only handle spinlocks if td == curthread. This is somewhat broken
1868 * if td is currently executing on some other CPU and holds spin locks
1869 * as we won't display those locks. If we had a MI way of getting
1870 * the per-cpu data for a given cpu then we could use
1871 * td->td_oncpu to get the list of spinlocks for this thread
1872 * and "fix" this.
1873 *
1874 * That still wouldn't really fix this unless we locked sched_lock
1875 * or stopped the other CPU to make sure it wasn't changing the list
1876 * out from under us. It is probably best to just not try to handle
1877 * threads on other CPU's for now.
1878 */
1879 if (td == curthread && PCPU_GET(spinlocks) != NULL)
1880 witness_list_locks(PCPU_PTR(spinlocks));
1881 }
1882
1883 DB_SHOW_COMMAND(locks, db_witness_list)
1884 {
1885 struct thread *td;
1886 pid_t pid;
1887 struct proc *p;
1888
1889 if (have_addr) {
1890 pid = (addr % 16) + ((addr >> 4) % 16) * 10 +
1891 ((addr >> 8) % 16) * 100 + ((addr >> 12) % 16) * 1000 +
1892 ((addr >> 16) % 16) * 10000;
1893 /* sx_slock(&allproc_lock); */
1894 FOREACH_PROC_IN_SYSTEM(p) {
1895 if (p->p_pid == pid)
1896 break;
1897 }
1898 /* sx_sunlock(&allproc_lock); */
1899 if (p == NULL) {
1900 db_printf("pid %d not found\n", pid);
1901 return;
1902 }
1903 FOREACH_THREAD_IN_PROC(p, td) {
1904 witness_list(td);
1905 }
1906 } else {
1907 td = curthread;
1908 witness_list(td);
1909 }
1910 }
1911
1912 DB_SHOW_COMMAND(alllocks, db_witness_list_all)
1913 {
1914 struct thread *td;
1915 struct proc *p;
1916
1917 /*
1918 * It would be nice to list only threads and processes that actually
1919 * held sleep locks, but that information is currently not exported
1920 * by WITNESS.
1921 */
1922 FOREACH_PROC_IN_SYSTEM(p) {
1923 if (!witness_proc_has_locks(p))
1924 continue;
1925 FOREACH_THREAD_IN_PROC(p, td) {
1926 if (!witness_thread_has_locks(td))
1927 continue;
1928 printf("Process %d (%s) thread %p (%d)\n", p->p_pid,
1929 p->p_comm, td, td->td_tid);
1930 witness_list(td);
1931 }
1932 }
1933 }
1934
1935 DB_SHOW_COMMAND(witness, db_witness_display)
1936 {
1937
1938 witness_display(db_printf);
1939 }
1940 #endif
Cache object: 05dc45a5c88209e8c7f8f22e1b7e1769
|