FreeBSD/Linux Kernel Cross Reference
sys/dev/lockstat.c
1 /* $NetBSD: lockstat.c,v 1.4 2006/11/16 01:32:45 christos Exp $ */
2
3 /*-
4 * Copyright (c) 2006 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. All advertising materials mentioning features or use of this software
19 * must display the following acknowledgement:
20 * This product includes software developed by the NetBSD
21 * Foundation, Inc. and its contributors.
22 * 4. Neither the name of The NetBSD Foundation nor the names of its
23 * contributors may be used to endorse or promote products derived
24 * from this software without specific prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36 * POSSIBILITY OF SUCH DAMAGE.
37 */
38
39 /*
40 * Lock statistics driver, providing kernel support for the lockstat(8)
41 * command.
42 */
43
44 #include <sys/cdefs.h>
45 __KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.4 2006/11/16 01:32:45 christos Exp $");
46
47 #include <sys/types.h>
48 #include <sys/param.h>
49 #include <sys/lock.h>
50 #include <sys/proc.h>
51 #include <sys/resourcevar.h>
52 #include <sys/systm.h>
53 #include <sys/kernel.h>
54 #include <sys/malloc.h>
55 #include <sys/conf.h>
56 #include <sys/syslog.h>
57
58 #include <dev/lockstat.h>
59
60 #ifndef __HAVE_CPU_COUNTER
61 #error CPU counters not available
62 #endif
63
64 #if LONG_BIT == 64
65 #define LOCKSTAT_HASH_SHIFT 3
66 #elif LONG_BIT == 32
67 #define LOCKSTAT_HASH_SHIFT 2
68 #endif
69
70 #define LOCKSTAT_MINBUFS 100
71 #define LOCKSTAT_DEFBUFS 1000
72 #define LOCKSTAT_MAXBUFS 10000
73
74 #define LOCKSTAT_HASH_SIZE 64
75 #define LOCKSTAT_HASH_MASK (LOCKSTAT_HASH_SIZE - 1)
76 #define LOCKSTAT_HASH(key) \
77 ((key >> LOCKSTAT_HASH_SHIFT) & LOCKSTAT_HASH_MASK)
78
79 typedef struct lscpu {
80 SLIST_HEAD(, lsbuf) lc_free;
81 u_int lc_overflow;
82 LIST_HEAD(lslist, lsbuf) lc_hash[LOCKSTAT_HASH_SIZE];
83 } lscpu_t;
84
85 typedef struct lslist lslist_t;
86
87 void lockstatattach(int);
88 void lockstat_start(lsenable_t *);
89 int lockstat_alloc(lsenable_t *);
90 void lockstat_init_tables(lsenable_t *);
91 int lockstat_stop(lsdisable_t *);
92 void lockstat_free(void);
93
94 dev_type_open(lockstat_open);
95 dev_type_close(lockstat_close);
96 dev_type_read(lockstat_read);
97 dev_type_ioctl(lockstat_ioctl);
98
99 /* Protected against write by lockstat_lock(). Used by lockstat_event(). */
100 volatile u_int lockstat_enabled;
101 uintptr_t lockstat_csstart;
102 uintptr_t lockstat_csend;
103 uintptr_t lockstat_csmask;
104 uintptr_t lockstat_lockaddr;
105
106 /* Protected by lockstat_lock(). */
107 struct simplelock lockstat_slock;
108 lsbuf_t *lockstat_baseb;
109 size_t lockstat_sizeb;
110 int lockstat_busy;
111 int lockstat_devopen;
112 struct timespec lockstat_stime;
113
114 const struct cdevsw lockstat_cdevsw = {
115 lockstat_open, lockstat_close, lockstat_read, nowrite, lockstat_ioctl,
116 nostop, notty, nopoll, nommap, nokqfilter, 0
117 };
118
119 MALLOC_DEFINE(M_LOCKSTAT, "lockstat", "lockstat event buffers");
120
121 /*
122 * Called when the pseudo-driver is attached.
123 */
124 void
125 lockstatattach(int nunits)
126 {
127
128 (void)nunits;
129
130 __cpu_simple_lock_init(&lockstat_slock.lock_data);
131 }
132
133 /*
134 * Grab the global lock. If busy is set, we want to block out operations on
135 * the control device.
136 */
137 static inline int
138 lockstat_lock(int busy)
139 {
140
141 if (!__cpu_simple_lock_try(&lockstat_slock.lock_data))
142 return (EBUSY);
143 if (busy) {
144 if (lockstat_busy) {
145 __cpu_simple_unlock(&lockstat_slock.lock_data);
146 return (EBUSY);
147 }
148 lockstat_busy = 1;
149 }
150 KASSERT(lockstat_busy);
151
152 return 0;
153 }
154
155 /*
156 * Release the global lock. If unbusy is set, we want to allow new
157 * operations on the control device.
158 */
159 static inline void
160 lockstat_unlock(int unbusy)
161 {
162
163 KASSERT(lockstat_busy);
164 if (unbusy)
165 lockstat_busy = 0;
166 __cpu_simple_unlock(&lockstat_slock.lock_data);
167 }
168
169 /*
170 * Prepare the per-CPU tables for use, or clear down tables when tracing is
171 * stopped.
172 */
173 void
174 lockstat_init_tables(lsenable_t *le)
175 {
176 int i, ncpu, per, slop, cpuno;
177 CPU_INFO_ITERATOR cii;
178 struct cpu_info *ci;
179 lscpu_t *lc;
180 lsbuf_t *lb;
181
182 KASSERT(!lockstat_enabled);
183
184 ncpu = 0;
185 for (CPU_INFO_FOREACH(cii, ci)) {
186 if (ci->ci_lockstat != NULL) {
187 free(ci->ci_lockstat, M_LOCKSTAT);
188 ci->ci_lockstat = NULL;
189 }
190 ncpu++;
191 }
192
193 if (le == NULL)
194 return;
195
196 lb = lockstat_baseb;
197 per = le->le_nbufs / ncpu;
198 slop = le->le_nbufs - (per * ncpu);
199 cpuno = 0;
200 for (CPU_INFO_FOREACH(cii, ci)) {
201 lc = malloc(sizeof(*lc), M_LOCKSTAT, M_WAITOK);
202 lc->lc_overflow = 0;
203 ci->ci_lockstat = lc;
204
205 SLIST_INIT(&lc->lc_free);
206 for (i = 0; i < LOCKSTAT_HASH_SIZE; i++)
207 LIST_INIT(&lc->lc_hash[i]);
208
209 for (i = per; i != 0; i--, lb++) {
210 lb->lb_cpu = (uint16_t)cpuno;
211 SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
212 }
213 if (--slop > 0) {
214 lb->lb_cpu = (uint16_t)cpuno;
215 SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
216 lb++;
217 }
218 cpuno++;
219 }
220 }
221
222 /*
223 * Start collecting lock statistics.
224 */
225 void
226 lockstat_start(lsenable_t *le)
227 {
228
229 KASSERT(!lockstat_enabled);
230
231 lockstat_init_tables(le);
232
233 if ((le->le_flags & LE_CALLSITE) != 0)
234 lockstat_csmask = (uintptr_t)-1LL;
235 else
236 lockstat_csmask = 0;
237
238 lockstat_csstart = le->le_csstart;
239 lockstat_csend = le->le_csend;
240 lockstat_lockaddr = le->le_lock;
241
242 /*
243 * Force a write barrier. XXX This may not be sufficient..
244 */
245 lockstat_unlock(0);
246 tsleep(&lockstat_start, PPAUSE, "lockstat", mstohz(10));
247 (void)lockstat_lock(0);
248
249 getnanotime(&lockstat_stime);
250 lockstat_enabled = le->le_mask;
251 lockstat_unlock(0);
252 (void)lockstat_lock(0);
253 }
254
255 /*
256 * Stop collecting lock statistics.
257 */
258 int
259 lockstat_stop(lsdisable_t *ld)
260 {
261 CPU_INFO_ITERATOR cii;
262 struct cpu_info *ci;
263 u_int cpuno, overflow;
264 struct timespec ts;
265 int error;
266
267 KASSERT(lockstat_enabled);
268
269 /*
270 * Set enabled false, force a write barrier, and wait for other CPUs
271 * to exit lockstat_event(). XXX This may not be sufficient..
272 */
273 lockstat_enabled = 0;
274 lockstat_unlock(0);
275 getnanotime(&ts);
276 tsleep(&lockstat_stop, PPAUSE, "lockstat", mstohz(10));
277 (void)lockstat_lock(0);
278
279 /*
280 * Did we run out of buffers while tracing?
281 */
282 overflow = 0;
283 for (CPU_INFO_FOREACH(cii, ci))
284 overflow += ((lscpu_t *)ci->ci_lockstat)->lc_overflow;
285
286 if (overflow != 0) {
287 error = EOVERFLOW;
288 log(LOG_NOTICE, "lockstat: %d buffer allocations failed\n",
289 overflow);
290 } else
291 error = 0;
292
293 lockstat_init_tables(NULL);
294
295 if (ld == NULL)
296 return (error);
297
298 /*
299 * Fill out the disable struct for the caller.
300 */
301 timespecsub(&ts, &lockstat_stime, &ld->ld_time);
302 ld->ld_size = lockstat_sizeb;
303
304 cpuno = 0;
305 for (CPU_INFO_FOREACH(cii, ci)) {
306 if (cpuno > sizeof(ld->ld_freq) / sizeof(ld->ld_freq[0])) {
307 log(LOG_WARNING, "lockstat: too many CPUs\n");
308 break;
309 }
310 ld->ld_freq[cpuno++] = cpu_frequency(ci);
311 }
312
313 return (error);
314 }
315
316 /*
317 * Allocate buffers for lockstat_start().
318 */
319 int
320 lockstat_alloc(lsenable_t *le)
321 {
322 lsbuf_t *lb;
323 size_t sz;
324
325 KASSERT(!lockstat_enabled);
326 lockstat_free();
327
328 sz = sizeof(*lb) * le->le_nbufs;
329
330 lockstat_unlock(0);
331 lb = malloc(sz, M_LOCKSTAT, M_WAITOK | M_ZERO);
332 (void)lockstat_lock(0);
333
334 if (lb == NULL)
335 return (ENOMEM);
336
337 KASSERT(!lockstat_enabled);
338 KASSERT(lockstat_baseb == NULL);
339 lockstat_sizeb = sz;
340 lockstat_baseb = lb;
341
342 return (0);
343 }
344
345 /*
346 * Free allocated buffers after tracing has stopped.
347 */
348 void
349 lockstat_free(void)
350 {
351
352 KASSERT(!lockstat_enabled);
353
354 if (lockstat_baseb != NULL) {
355 free(lockstat_baseb, M_LOCKSTAT);
356 lockstat_baseb = NULL;
357 }
358 }
359
360 /*
361 * Main entry point from lock primatives.
362 */
363 void
364 lockstat_event(uintptr_t lock, uintptr_t callsite, u_int flags, u_int count,
365 uint64_t time)
366 {
367 lslist_t *ll;
368 lscpu_t *lc;
369 lsbuf_t *lb;
370 u_int event;
371 int s;
372
373 if ((flags & lockstat_enabled) != flags || count == 0)
374 return;
375 if (lockstat_lockaddr != 0 && lock != lockstat_lockaddr)
376 return;
377 if (callsite < lockstat_csstart || callsite > lockstat_csend)
378 return;
379
380 callsite &= lockstat_csmask;
381
382 /*
383 * Find the table for this lock+callsite pair, and try to locate a
384 * buffer with the same key.
385 */
386 lc = curcpu()->ci_lockstat;
387 ll = &lc->lc_hash[LOCKSTAT_HASH(lock ^ callsite)];
388 event = (flags & LB_EVENT_MASK) - 1;
389 s = spllock();
390
391 LIST_FOREACH(lb, ll, lb_chain.list) {
392 if (lb->lb_lock == lock && lb->lb_callsite == callsite)
393 break;
394 }
395
396 if (lb != NULL) {
397 /*
398 * We found a record. Move it to the front of the list, as
399 * we're likely to hit it again soon.
400 */
401 if (lb != LIST_FIRST(ll)) {
402 LIST_REMOVE(lb, lb_chain.list);
403 LIST_INSERT_HEAD(ll, lb, lb_chain.list);
404 }
405 lb->lb_counts[event] += count;
406 lb->lb_times[event] += time;
407 } else if ((lb = SLIST_FIRST(&lc->lc_free)) != NULL) {
408 /*
409 * Pinch a new buffer and fill it out.
410 */
411 SLIST_REMOVE_HEAD(&lc->lc_free, lb_chain.slist);
412 LIST_INSERT_HEAD(ll, lb, lb_chain.list);
413 lb->lb_flags = (uint16_t)flags;
414 lb->lb_lock = lock;
415 lb->lb_callsite = callsite;
416 lb->lb_counts[event] = count;
417 lb->lb_times[event] = time;
418 } else {
419 /*
420 * We didn't find a buffer and there were none free.
421 * lockstat_stop() will notice later on and report the
422 * error.
423 */
424 lc->lc_overflow++;
425 }
426
427 splx(s);
428 }
429
430 /*
431 * Accept an open() on /dev/lockstat.
432 */
433 int
434 lockstat_open(dev_t dev, int flag, int mode,
435 struct lwp *l)
436 {
437 int error;
438
439 if ((error = lockstat_lock(1)) != 0)
440 return error;
441
442 if (lockstat_devopen)
443 error = EBUSY;
444 else {
445 lockstat_devopen = 1;
446 error = 0;
447 }
448
449 lockstat_unlock(1);
450
451 return error;
452 }
453
454 /*
455 * Accept the last close() on /dev/lockstat.
456 */
457 int
458 lockstat_close(dev_t dev, int flag, int mode,
459 struct lwp *l)
460 {
461 int error;
462
463 if ((error = lockstat_lock(1)) == 0) {
464 if (lockstat_enabled)
465 (void)lockstat_stop(NULL);
466 lockstat_free();
467 lockstat_devopen = 0;
468 lockstat_unlock(1);
469 }
470
471 return error;
472 }
473
474 /*
475 * Handle control operations.
476 */
477 int
478 lockstat_ioctl(dev_t dev, u_long cmd, caddr_t data,
479 int flag, struct lwp *l)
480 {
481 lsenable_t *le;
482 int error;
483
484 if ((error = lockstat_lock(1)) != 0)
485 return error;
486
487 switch (cmd) {
488 case IOC_LOCKSTAT_GVERSION:
489 *(int *)data = LS_VERSION;
490 error = 0;
491 break;
492
493 case IOC_LOCKSTAT_ENABLE:
494 le = (lsenable_t *)data;
495
496 if (!cpu_hascounter()) {
497 error = ENODEV;
498 break;
499 }
500 if (lockstat_enabled) {
501 error = EBUSY;
502 break;
503 }
504
505 /*
506 * Sanitize the arguments passed in and set up filtering.
507 */
508 if (le->le_nbufs == 0)
509 le->le_nbufs = LOCKSTAT_DEFBUFS;
510 else if (le->le_nbufs > LOCKSTAT_MAXBUFS ||
511 le->le_nbufs < LOCKSTAT_MINBUFS) {
512 error = EINVAL;
513 break;
514 }
515 if ((le->le_flags & LE_ONE_CALLSITE) == 0) {
516 le->le_csstart = 0;
517 le->le_csend = le->le_csstart - 1;
518 }
519 if ((le->le_flags & LE_ONE_LOCK) == 0)
520 le->le_lock = 0;
521 if ((le->le_mask & LB_EVENT_MASK) == 0)
522 return (EINVAL);
523 if ((le->le_mask & LB_LOCK_MASK) == 0)
524 return (EINVAL);
525
526 /*
527 * Start tracing.
528 */
529 if ((error = lockstat_alloc(le)) == 0)
530 lockstat_start(le);
531 break;
532
533 case IOC_LOCKSTAT_DISABLE:
534 if (!lockstat_enabled)
535 error = EINVAL;
536 else
537 error = lockstat_stop((lsdisable_t *)data);
538 break;
539
540 default:
541 error = ENOTTY;
542 break;
543 }
544
545 lockstat_unlock(1);
546 return error;
547 }
548
549 /*
550 * Copy buffers out to user-space.
551 */
552 int
553 lockstat_read(dev_t dev, struct uio *uio, int flag)
554 {
555 int error;
556
557 if ((error = lockstat_lock(1)) != 0)
558 return (error);
559
560 if (lockstat_enabled) {
561 lockstat_unlock(1);
562 return (EBUSY);
563 }
564
565 lockstat_unlock(0);
566 error = uiomove(lockstat_baseb, lockstat_sizeb, uio);
567 lockstat_lock(0);
568
569 lockstat_unlock(1);
570
571 return (error);
572 }
Cache object: b631142a6c72074c8786444ba973e09f
|