1 /*-
2 * Copyright (c) 1987, 1991, 1993
3 * The Regents of the University of California.
4 * Copyright (c) 2005 Robert N. M. Watson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 4. Neither the name of the University nor the names of its contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 *
31 * @(#)kern_malloc.c 8.3 (Berkeley) 1/4/94
32 */
33
34 #include <sys/cdefs.h>
35 __FBSDID("$FreeBSD: releng/6.0/sys/kern/kern_malloc.c 148559 2005-07-30 06:14:57Z pjd $");
36
37 #include "opt_vm.h"
38
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/kdb.h>
42 #include <sys/kernel.h>
43 #include <sys/lock.h>
44 #include <sys/malloc.h>
45 #include <sys/mbuf.h>
46 #include <sys/mutex.h>
47 #include <sys/vmmeter.h>
48 #include <sys/proc.h>
49 #include <sys/sbuf.h>
50 #include <sys/sysctl.h>
51 #include <sys/time.h>
52
53 #include <vm/vm.h>
54 #include <vm/pmap.h>
55 #include <vm/vm_param.h>
56 #include <vm/vm_kern.h>
57 #include <vm/vm_extern.h>
58 #include <vm/vm_map.h>
59 #include <vm/vm_page.h>
60 #include <vm/uma.h>
61 #include <vm/uma_int.h>
62 #include <vm/uma_dbg.h>
63
64 #ifdef DEBUG_MEMGUARD
65 #include <vm/memguard.h>
66 #endif
67
68 #if defined(INVARIANTS) && defined(__i386__)
69 #include <machine/cpu.h>
70 #endif
71
72 /*
73 * When realloc() is called, if the new size is sufficiently smaller than
74 * the old size, realloc() will allocate a new, smaller block to avoid
75 * wasting memory. 'Sufficiently smaller' is defined as: newsize <=
76 * oldsize / 2^n, where REALLOC_FRACTION defines the value of 'n'.
77 */
78 #ifndef REALLOC_FRACTION
79 #define REALLOC_FRACTION 1 /* new block if <= half the size */
80 #endif
81
82 MALLOC_DEFINE(M_CACHE, "cache", "Various Dynamically allocated caches");
83 MALLOC_DEFINE(M_DEVBUF, "devbuf", "device driver memory");
84 MALLOC_DEFINE(M_TEMP, "temp", "misc temporary data buffers");
85
86 MALLOC_DEFINE(M_IP6OPT, "ip6opt", "IPv6 options");
87 MALLOC_DEFINE(M_IP6NDP, "ip6ndp", "IPv6 Neighbor Discovery");
88
89 static void kmeminit(void *);
90 SYSINIT(kmem, SI_SUB_KMEM, SI_ORDER_FIRST, kmeminit, NULL)
91
92 static MALLOC_DEFINE(M_FREE, "free", "should be on free list");
93
94 static struct malloc_type *kmemstatistics;
95 static char *kmembase;
96 static char *kmemlimit;
97 static int kmemcount;
98
99 #define KMEM_ZSHIFT 4
100 #define KMEM_ZBASE 16
101 #define KMEM_ZMASK (KMEM_ZBASE - 1)
102
103 #define KMEM_ZMAX PAGE_SIZE
104 #define KMEM_ZSIZE (KMEM_ZMAX >> KMEM_ZSHIFT)
105 static u_int8_t kmemsize[KMEM_ZSIZE + 1];
106
107 /* These won't be powers of two for long */
108 struct {
109 int kz_size;
110 char *kz_name;
111 uma_zone_t kz_zone;
112 } kmemzones[] = {
113 {16, "16", NULL},
114 {32, "32", NULL},
115 {64, "64", NULL},
116 {128, "128", NULL},
117 {256, "256", NULL},
118 {512, "512", NULL},
119 {1024, "1024", NULL},
120 {2048, "2048", NULL},
121 {4096, "4096", NULL},
122 #if PAGE_SIZE > 4096
123 {8192, "8192", NULL},
124 #if PAGE_SIZE > 8192
125 {16384, "16384", NULL},
126 #if PAGE_SIZE > 16384
127 {32768, "32768", NULL},
128 #if PAGE_SIZE > 32768
129 {65536, "65536", NULL},
130 #if PAGE_SIZE > 65536
131 #error "Unsupported PAGE_SIZE"
132 #endif /* 65536 */
133 #endif /* 32768 */
134 #endif /* 16384 */
135 #endif /* 8192 */
136 #endif /* 4096 */
137 {0, NULL},
138 };
139
140 static uma_zone_t mt_zone;
141
142 #ifdef DEBUG_MEMGUARD
143 u_int vm_memguard_divisor;
144 SYSCTL_UINT(_vm, OID_AUTO, memguard_divisor, CTLFLAG_RD, &vm_memguard_divisor,
145 0, "(kmem_size/memguard_divisor) == memguard submap size");
146 #endif
147
148 u_int vm_kmem_size;
149 SYSCTL_UINT(_vm, OID_AUTO, kmem_size, CTLFLAG_RD, &vm_kmem_size, 0,
150 "Size of kernel memory");
151
152 u_int vm_kmem_size_max;
153 SYSCTL_UINT(_vm, OID_AUTO, kmem_size_max, CTLFLAG_RD, &vm_kmem_size_max, 0,
154 "Maximum size of kernel memory");
155
156 u_int vm_kmem_size_scale;
157 SYSCTL_UINT(_vm, OID_AUTO, kmem_size_scale, CTLFLAG_RD, &vm_kmem_size_scale, 0,
158 "Scale factor for kernel memory size");
159
160 /*
161 * The malloc_mtx protects the kmemstatistics linked list.
162 */
163
164 struct mtx malloc_mtx;
165
166 #ifdef MALLOC_PROFILE
167 uint64_t krequests[KMEM_ZSIZE + 1];
168
169 static int sysctl_kern_mprof(SYSCTL_HANDLER_ARGS);
170 #endif
171
172 static int sysctl_kern_malloc(SYSCTL_HANDLER_ARGS);
173 static int sysctl_kern_malloc_stats(SYSCTL_HANDLER_ARGS);
174
175 /* time_uptime of last malloc(9) failure */
176 static time_t t_malloc_fail;
177
178 #ifdef MALLOC_MAKE_FAILURES
179 /*
180 * Causes malloc failures every (n) mallocs with M_NOWAIT. If set to 0,
181 * doesn't cause failures.
182 */
183 SYSCTL_NODE(_debug, OID_AUTO, malloc, CTLFLAG_RD, 0,
184 "Kernel malloc debugging options");
185
186 static int malloc_failure_rate;
187 static int malloc_nowait_count;
188 static int malloc_failure_count;
189 SYSCTL_INT(_debug_malloc, OID_AUTO, failure_rate, CTLFLAG_RW,
190 &malloc_failure_rate, 0, "Every (n) mallocs with M_NOWAIT will fail");
191 TUNABLE_INT("debug.malloc.failure_rate", &malloc_failure_rate);
192 SYSCTL_INT(_debug_malloc, OID_AUTO, failure_count, CTLFLAG_RD,
193 &malloc_failure_count, 0, "Number of imposed M_NOWAIT malloc failures");
194 #endif
195
196 int
197 malloc_last_fail(void)
198 {
199
200 return (time_uptime - t_malloc_fail);
201 }
202
203 /*
204 * Add this to the informational malloc_type bucket.
205 */
206 static void
207 malloc_type_zone_allocated(struct malloc_type *mtp, unsigned long size,
208 int zindx)
209 {
210 struct malloc_type_internal *mtip;
211 struct malloc_type_stats *mtsp;
212
213 critical_enter();
214 mtip = mtp->ks_handle;
215 mtsp = &mtip->mti_stats[curcpu];
216 if (size > 0) {
217 mtsp->mts_memalloced += size;
218 mtsp->mts_numallocs++;
219 }
220 if (zindx != -1)
221 mtsp->mts_size |= 1 << zindx;
222 critical_exit();
223 }
224
225 void
226 malloc_type_allocated(struct malloc_type *mtp, unsigned long size)
227 {
228
229 if (size > 0)
230 malloc_type_zone_allocated(mtp, size, -1);
231 }
232
233 /*
234 * Remove this allocation from the informational malloc_type bucket.
235 */
236 void
237 malloc_type_freed(struct malloc_type *mtp, unsigned long size)
238 {
239 struct malloc_type_internal *mtip;
240 struct malloc_type_stats *mtsp;
241
242 critical_enter();
243 mtip = mtp->ks_handle;
244 mtsp = &mtip->mti_stats[curcpu];
245 mtsp->mts_memfreed += size;
246 mtsp->mts_numfrees++;
247 critical_exit();
248 }
249
250 /*
251 * malloc:
252 *
253 * Allocate a block of memory.
254 *
255 * If M_NOWAIT is set, this routine will not block and return NULL if
256 * the allocation fails.
257 */
258 void *
259 malloc(unsigned long size, struct malloc_type *mtp, int flags)
260 {
261 int indx;
262 caddr_t va;
263 uma_zone_t zone;
264 uma_keg_t keg;
265 #ifdef DIAGNOSTIC
266 unsigned long osize = size;
267 #endif
268
269 #ifdef INVARIANTS
270 /*
271 * To make sure that WAITOK or NOWAIT is set, but not more than
272 * one, and check against the API botches that are common.
273 */
274 indx = flags & (M_WAITOK | M_NOWAIT | M_DONTWAIT | M_TRYWAIT);
275 if (indx != M_NOWAIT && indx != M_WAITOK) {
276 static struct timeval lasterr;
277 static int curerr, once;
278 if (once == 0 && ppsratecheck(&lasterr, &curerr, 1)) {
279 printf("Bad malloc flags: %x\n", indx);
280 kdb_backtrace();
281 flags |= M_WAITOK;
282 once++;
283 }
284 }
285 #endif
286 #if 0
287 if (size == 0)
288 kdb_enter("zero size malloc");
289 #endif
290 #ifdef MALLOC_MAKE_FAILURES
291 if ((flags & M_NOWAIT) && (malloc_failure_rate != 0)) {
292 atomic_add_int(&malloc_nowait_count, 1);
293 if ((malloc_nowait_count % malloc_failure_rate) == 0) {
294 atomic_add_int(&malloc_failure_count, 1);
295 t_malloc_fail = time_uptime;
296 return (NULL);
297 }
298 }
299 #endif
300 if (flags & M_WAITOK)
301 KASSERT(curthread->td_intr_nesting_level == 0,
302 ("malloc(M_WAITOK) in interrupt context"));
303
304 #ifdef DEBUG_MEMGUARD
305 /* XXX CHANGEME! */
306 if (mtp == M_SUBPROC)
307 return memguard_alloc(size, flags);
308 #endif
309
310 if (size <= KMEM_ZMAX) {
311 if (size & KMEM_ZMASK)
312 size = (size & ~KMEM_ZMASK) + KMEM_ZBASE;
313 indx = kmemsize[size >> KMEM_ZSHIFT];
314 zone = kmemzones[indx].kz_zone;
315 keg = zone->uz_keg;
316 #ifdef MALLOC_PROFILE
317 krequests[size >> KMEM_ZSHIFT]++;
318 #endif
319 va = uma_zalloc(zone, flags);
320 if (va != NULL)
321 size = keg->uk_size;
322 malloc_type_zone_allocated(mtp, va == NULL ? 0 : size, indx);
323 } else {
324 size = roundup(size, PAGE_SIZE);
325 zone = NULL;
326 keg = NULL;
327 va = uma_large_malloc(size, flags);
328 malloc_type_allocated(mtp, va == NULL ? 0 : size);
329 }
330 if (flags & M_WAITOK)
331 KASSERT(va != NULL, ("malloc(M_WAITOK) returned NULL"));
332 else if (va == NULL)
333 t_malloc_fail = time_uptime;
334 #ifdef DIAGNOSTIC
335 if (va != NULL && !(flags & M_ZERO)) {
336 memset(va, 0x70, osize);
337 }
338 #endif
339 return ((void *) va);
340 }
341
342 /*
343 * free:
344 *
345 * Free a block of memory allocated by malloc.
346 *
347 * This routine may not block.
348 */
349 void
350 free(void *addr, struct malloc_type *mtp)
351 {
352 uma_slab_t slab;
353 u_long size;
354
355 /* free(NULL, ...) does nothing */
356 if (addr == NULL)
357 return;
358
359 #ifdef DEBUG_MEMGUARD
360 /* XXX CHANGEME! */
361 if (mtp == M_SUBPROC) {
362 memguard_free(addr);
363 return;
364 }
365 #endif
366
367 size = 0;
368
369 slab = vtoslab((vm_offset_t)addr & (~UMA_SLAB_MASK));
370
371 if (slab == NULL)
372 panic("free: address %p(%p) has not been allocated.\n",
373 addr, (void *)((u_long)addr & (~UMA_SLAB_MASK)));
374
375
376 if (!(slab->us_flags & UMA_SLAB_MALLOC)) {
377 #ifdef INVARIANTS
378 struct malloc_type **mtpp = addr;
379 #endif
380 size = slab->us_keg->uk_size;
381 #ifdef INVARIANTS
382 /*
383 * Cache a pointer to the malloc_type that most recently freed
384 * this memory here. This way we know who is most likely to
385 * have stepped on it later.
386 *
387 * This code assumes that size is a multiple of 8 bytes for
388 * 64 bit machines
389 */
390 mtpp = (struct malloc_type **)
391 ((unsigned long)mtpp & ~UMA_ALIGN_PTR);
392 mtpp += (size - sizeof(struct malloc_type *)) /
393 sizeof(struct malloc_type *);
394 *mtpp = mtp;
395 #endif
396 uma_zfree_arg(LIST_FIRST(&slab->us_keg->uk_zones), addr, slab);
397 } else {
398 size = slab->us_size;
399 uma_large_free(slab);
400 }
401 malloc_type_freed(mtp, size);
402 }
403
404 /*
405 * realloc: change the size of a memory block
406 */
407 void *
408 realloc(void *addr, unsigned long size, struct malloc_type *mtp, int flags)
409 {
410 uma_slab_t slab;
411 unsigned long alloc;
412 void *newaddr;
413
414 /* realloc(NULL, ...) is equivalent to malloc(...) */
415 if (addr == NULL)
416 return (malloc(size, mtp, flags));
417
418 /*
419 * XXX: Should report free of old memory and alloc of new memory to
420 * per-CPU stats.
421 */
422
423 #ifdef DEBUG_MEMGUARD
424 /* XXX: CHANGEME! */
425 if (mtp == M_SUBPROC) {
426 slab = NULL;
427 alloc = size;
428 } else {
429 #endif
430
431 slab = vtoslab((vm_offset_t)addr & ~(UMA_SLAB_MASK));
432
433 /* Sanity check */
434 KASSERT(slab != NULL,
435 ("realloc: address %p out of range", (void *)addr));
436
437 /* Get the size of the original block */
438 if (slab->us_keg)
439 alloc = slab->us_keg->uk_size;
440 else
441 alloc = slab->us_size;
442
443 /* Reuse the original block if appropriate */
444 if (size <= alloc
445 && (size > (alloc >> REALLOC_FRACTION) || alloc == MINALLOCSIZE))
446 return (addr);
447
448 #ifdef DEBUG_MEMGUARD
449 }
450 #endif
451
452 /* Allocate a new, bigger (or smaller) block */
453 if ((newaddr = malloc(size, mtp, flags)) == NULL)
454 return (NULL);
455
456 /* Copy over original contents */
457 bcopy(addr, newaddr, min(size, alloc));
458 free(addr, mtp);
459 return (newaddr);
460 }
461
462 /*
463 * reallocf: same as realloc() but free memory on failure.
464 */
465 void *
466 reallocf(void *addr, unsigned long size, struct malloc_type *mtp, int flags)
467 {
468 void *mem;
469
470 if ((mem = realloc(addr, size, mtp, flags)) == NULL)
471 free(addr, mtp);
472 return (mem);
473 }
474
475 /*
476 * Initialize the kernel memory allocator
477 */
478 /* ARGSUSED*/
479 static void
480 kmeminit(void *dummy)
481 {
482 u_int8_t indx;
483 u_long mem_size;
484 int i;
485
486 mtx_init(&malloc_mtx, "malloc", NULL, MTX_DEF);
487
488 /*
489 * Try to auto-tune the kernel memory size, so that it is
490 * more applicable for a wider range of machine sizes.
491 * On an X86, a VM_KMEM_SIZE_SCALE value of 4 is good, while
492 * a VM_KMEM_SIZE of 12MB is a fair compromise. The
493 * VM_KMEM_SIZE_MAX is dependent on the maximum KVA space
494 * available, and on an X86 with a total KVA space of 256MB,
495 * try to keep VM_KMEM_SIZE_MAX at 80MB or below.
496 *
497 * Note that the kmem_map is also used by the zone allocator,
498 * so make sure that there is enough space.
499 */
500 vm_kmem_size = VM_KMEM_SIZE + nmbclusters * PAGE_SIZE;
501 mem_size = cnt.v_page_count;
502
503 #if defined(VM_KMEM_SIZE_SCALE)
504 vm_kmem_size_scale = VM_KMEM_SIZE_SCALE;
505 #endif
506 TUNABLE_INT_FETCH("vm.kmem_size_scale", &vm_kmem_size_scale);
507 if (vm_kmem_size_scale > 0 &&
508 (mem_size / vm_kmem_size_scale) > (vm_kmem_size / PAGE_SIZE))
509 vm_kmem_size = (mem_size / vm_kmem_size_scale) * PAGE_SIZE;
510
511 #if defined(VM_KMEM_SIZE_MAX)
512 vm_kmem_size_max = VM_KMEM_SIZE_MAX;
513 #endif
514 TUNABLE_INT_FETCH("vm.kmem_size_max", &vm_kmem_size_max);
515 if (vm_kmem_size_max > 0 && vm_kmem_size >= vm_kmem_size_max)
516 vm_kmem_size = vm_kmem_size_max;
517
518 /* Allow final override from the kernel environment */
519 #ifndef BURN_BRIDGES
520 if (TUNABLE_INT_FETCH("kern.vm.kmem.size", &vm_kmem_size) != 0)
521 printf("kern.vm.kmem.size is now called vm.kmem_size!\n");
522 #endif
523 TUNABLE_INT_FETCH("vm.kmem_size", &vm_kmem_size);
524
525 /*
526 * Limit kmem virtual size to twice the physical memory.
527 * This allows for kmem map sparseness, but limits the size
528 * to something sane. Be careful to not overflow the 32bit
529 * ints while doing the check.
530 */
531 if (((vm_kmem_size / 2) / PAGE_SIZE) > cnt.v_page_count)
532 vm_kmem_size = 2 * cnt.v_page_count * PAGE_SIZE;
533
534 /*
535 * Tune settings based on the kernel map's size at this time.
536 */
537 init_param3(vm_kmem_size / PAGE_SIZE);
538
539 kmem_map = kmem_suballoc(kernel_map, (vm_offset_t *)&kmembase,
540 (vm_offset_t *)&kmemlimit, vm_kmem_size);
541 kmem_map->system_map = 1;
542
543 #ifdef DEBUG_MEMGUARD
544 /*
545 * Initialize MemGuard if support compiled in. MemGuard is a
546 * replacement allocator used for detecting tamper-after-free
547 * scenarios as they occur. It is only used for debugging.
548 */
549 vm_memguard_divisor = 10;
550 TUNABLE_INT_FETCH("vm.memguard_divisor", &vm_memguard_divisor);
551
552 /* Pick a conservative value if provided value sucks. */
553 if ((vm_memguard_divisor <= 0) ||
554 ((vm_kmem_size / vm_memguard_divisor) == 0))
555 vm_memguard_divisor = 10;
556 memguard_init(kmem_map, vm_kmem_size / vm_memguard_divisor);
557 #endif
558
559 uma_startup2();
560
561 mt_zone = uma_zcreate("mt_zone", sizeof(struct malloc_type_internal),
562 #ifdef INVARIANTS
563 mtrash_ctor, mtrash_dtor, mtrash_init, mtrash_fini,
564 #else
565 NULL, NULL, NULL, NULL,
566 #endif
567 UMA_ALIGN_PTR, UMA_ZONE_MALLOC);
568 for (i = 0, indx = 0; kmemzones[indx].kz_size != 0; indx++) {
569 int size = kmemzones[indx].kz_size;
570 char *name = kmemzones[indx].kz_name;
571
572 kmemzones[indx].kz_zone = uma_zcreate(name, size,
573 #ifdef INVARIANTS
574 mtrash_ctor, mtrash_dtor, mtrash_init, mtrash_fini,
575 #else
576 NULL, NULL, NULL, NULL,
577 #endif
578 UMA_ALIGN_PTR, UMA_ZONE_MALLOC);
579
580 for (;i <= size; i+= KMEM_ZBASE)
581 kmemsize[i >> KMEM_ZSHIFT] = indx;
582
583 }
584 }
585
586 void
587 malloc_init(void *data)
588 {
589 struct malloc_type_internal *mtip;
590 struct malloc_type *mtp;
591
592 KASSERT(cnt.v_page_count != 0, ("malloc_register before vm_init"));
593
594 mtp = data;
595 mtip = uma_zalloc(mt_zone, M_WAITOK | M_ZERO);
596 mtp->ks_handle = mtip;
597
598 mtx_lock(&malloc_mtx);
599 mtp->ks_next = kmemstatistics;
600 kmemstatistics = mtp;
601 kmemcount++;
602 mtx_unlock(&malloc_mtx);
603 }
604
605 void
606 malloc_uninit(void *data)
607 {
608 struct malloc_type_internal *mtip;
609 struct malloc_type *mtp, *temp;
610
611 mtp = data;
612 KASSERT(mtp->ks_handle != NULL, ("malloc_deregister: cookie NULL"));
613 mtx_lock(&malloc_mtx);
614 mtip = mtp->ks_handle;
615 mtp->ks_handle = NULL;
616 if (mtp != kmemstatistics) {
617 for (temp = kmemstatistics; temp != NULL;
618 temp = temp->ks_next) {
619 if (temp->ks_next == mtp)
620 temp->ks_next = mtp->ks_next;
621 }
622 } else
623 kmemstatistics = mtp->ks_next;
624 kmemcount--;
625 mtx_unlock(&malloc_mtx);
626 uma_zfree(mt_zone, mtip);
627 }
628
629 static int
630 sysctl_kern_malloc(SYSCTL_HANDLER_ARGS)
631 {
632 struct malloc_type_stats mts_local, *mtsp;
633 struct malloc_type_internal *mtip;
634 struct malloc_type *mtp;
635 struct sbuf sbuf;
636 long temp_allocs, temp_bytes;
637 int linesize = 128;
638 int bufsize;
639 int first;
640 int error;
641 char *buf;
642 int cnt;
643 int i;
644
645 cnt = 0;
646
647 /* Guess at how much room is needed. */
648 mtx_lock(&malloc_mtx);
649 cnt = kmemcount;
650 mtx_unlock(&malloc_mtx);
651
652 bufsize = linesize * (cnt + 1);
653 buf = malloc(bufsize, M_TEMP, M_WAITOK|M_ZERO);
654 sbuf_new(&sbuf, buf, bufsize, SBUF_FIXEDLEN);
655
656 mtx_lock(&malloc_mtx);
657 sbuf_printf(&sbuf,
658 "\n Type InUse MemUse HighUse Requests Size(s)\n");
659 for (mtp = kmemstatistics; cnt != 0 && mtp != NULL;
660 mtp = mtp->ks_next, cnt--) {
661 mtip = mtp->ks_handle;
662 bzero(&mts_local, sizeof(mts_local));
663 for (i = 0; i < MAXCPU; i++) {
664 mtsp = &mtip->mti_stats[i];
665 mts_local.mts_memalloced += mtsp->mts_memalloced;
666 mts_local.mts_memfreed += mtsp->mts_memfreed;
667 mts_local.mts_numallocs += mtsp->mts_numallocs;
668 mts_local.mts_numfrees += mtsp->mts_numfrees;
669 mts_local.mts_size |= mtsp->mts_size;
670 }
671 if (mts_local.mts_numallocs == 0)
672 continue;
673
674 /*
675 * Due to races in per-CPU statistics gather, it's possible to
676 * get a slightly negative number here. If we do, approximate
677 * with 0.
678 */
679 if (mts_local.mts_numallocs > mts_local.mts_numfrees)
680 temp_allocs = mts_local.mts_numallocs -
681 mts_local.mts_numfrees;
682 else
683 temp_allocs = 0;
684
685 /*
686 * Ditto for bytes allocated.
687 */
688 if (mts_local.mts_memalloced > mts_local.mts_memfreed)
689 temp_bytes = mts_local.mts_memalloced -
690 mts_local.mts_memfreed;
691 else
692 temp_bytes = 0;
693
694 /*
695 * High-waterwark is no longer easily available, so we just
696 * print '-' for that column.
697 */
698 sbuf_printf(&sbuf, "%13s%6lu%6luK -%9llu",
699 mtp->ks_shortdesc,
700 temp_allocs,
701 (temp_bytes + 1023) / 1024,
702 (unsigned long long)mts_local.mts_numallocs);
703
704 first = 1;
705 for (i = 0; i < sizeof(kmemzones) / sizeof(kmemzones[0]) - 1;
706 i++) {
707 if (mts_local.mts_size & (1 << i)) {
708 if (first)
709 sbuf_printf(&sbuf, " ");
710 else
711 sbuf_printf(&sbuf, ",");
712 sbuf_printf(&sbuf, "%s",
713 kmemzones[i].kz_name);
714 first = 0;
715 }
716 }
717 sbuf_printf(&sbuf, "\n");
718 }
719 sbuf_finish(&sbuf);
720 mtx_unlock(&malloc_mtx);
721
722 error = SYSCTL_OUT(req, sbuf_data(&sbuf), sbuf_len(&sbuf));
723
724 sbuf_delete(&sbuf);
725 free(buf, M_TEMP);
726 return (error);
727 }
728
729 SYSCTL_OID(_kern, OID_AUTO, malloc, CTLTYPE_STRING|CTLFLAG_RD,
730 NULL, 0, sysctl_kern_malloc, "A", "Malloc Stats");
731
732 static int
733 sysctl_kern_malloc_stats(SYSCTL_HANDLER_ARGS)
734 {
735 struct malloc_type_stream_header mtsh;
736 struct malloc_type_internal *mtip;
737 struct malloc_type_header mth;
738 struct malloc_type *mtp;
739 int buflen, count, error, i;
740 struct sbuf sbuf;
741 char *buffer;
742
743 mtx_lock(&malloc_mtx);
744 restart:
745 mtx_assert(&malloc_mtx, MA_OWNED);
746 count = kmemcount;
747 mtx_unlock(&malloc_mtx);
748 buflen = sizeof(mtsh) + count * (sizeof(mth) +
749 sizeof(struct malloc_type_stats) * MAXCPU) + 1;
750 buffer = malloc(buflen, M_TEMP, M_WAITOK | M_ZERO);
751 mtx_lock(&malloc_mtx);
752 if (count < kmemcount) {
753 free(buffer, M_TEMP);
754 goto restart;
755 }
756
757 sbuf_new(&sbuf, buffer, buflen, SBUF_FIXEDLEN);
758
759 /*
760 * Insert stream header.
761 */
762 bzero(&mtsh, sizeof(mtsh));
763 mtsh.mtsh_version = MALLOC_TYPE_STREAM_VERSION;
764 mtsh.mtsh_maxcpus = MAXCPU;
765 mtsh.mtsh_count = kmemcount;
766 if (sbuf_bcat(&sbuf, &mtsh, sizeof(mtsh)) < 0) {
767 mtx_unlock(&malloc_mtx);
768 error = ENOMEM;
769 goto out;
770 }
771
772 /*
773 * Insert alternating sequence of type headers and type statistics.
774 */
775 for (mtp = kmemstatistics; mtp != NULL; mtp = mtp->ks_next) {
776 mtip = (struct malloc_type_internal *)mtp->ks_handle;
777
778 /*
779 * Insert type header.
780 */
781 bzero(&mth, sizeof(mth));
782 strlcpy(mth.mth_name, mtp->ks_shortdesc, MALLOC_MAX_NAME);
783 if (sbuf_bcat(&sbuf, &mth, sizeof(mth)) < 0) {
784 mtx_unlock(&malloc_mtx);
785 error = ENOMEM;
786 goto out;
787 }
788
789 /*
790 * Insert type statistics for each CPU.
791 */
792 for (i = 0; i < MAXCPU; i++) {
793 if (sbuf_bcat(&sbuf, &mtip->mti_stats[i],
794 sizeof(mtip->mti_stats[i])) < 0) {
795 mtx_unlock(&malloc_mtx);
796 error = ENOMEM;
797 goto out;
798 }
799 }
800 }
801 mtx_unlock(&malloc_mtx);
802 sbuf_finish(&sbuf);
803 error = SYSCTL_OUT(req, sbuf_data(&sbuf), sbuf_len(&sbuf));
804 out:
805 sbuf_delete(&sbuf);
806 free(buffer, M_TEMP);
807 return (error);
808 }
809
810 SYSCTL_PROC(_kern, OID_AUTO, malloc_stats, CTLFLAG_RD|CTLTYPE_STRUCT,
811 0, 0, sysctl_kern_malloc_stats, "s,malloc_type_ustats",
812 "Return malloc types");
813
814 SYSCTL_INT(_kern, OID_AUTO, malloc_count, CTLFLAG_RD, &kmemcount, 0,
815 "Count of kernel malloc types");
816
817 #ifdef MALLOC_PROFILE
818
819 static int
820 sysctl_kern_mprof(SYSCTL_HANDLER_ARGS)
821 {
822 int linesize = 64;
823 struct sbuf sbuf;
824 uint64_t count;
825 uint64_t waste;
826 uint64_t mem;
827 int bufsize;
828 int error;
829 char *buf;
830 int rsize;
831 int size;
832 int i;
833
834 bufsize = linesize * (KMEM_ZSIZE + 1);
835 bufsize += 128; /* For the stats line */
836 bufsize += 128; /* For the banner line */
837 waste = 0;
838 mem = 0;
839
840 buf = malloc(bufsize, M_TEMP, M_WAITOK|M_ZERO);
841 sbuf_new(&sbuf, buf, bufsize, SBUF_FIXEDLEN);
842 sbuf_printf(&sbuf,
843 "\n Size Requests Real Size\n");
844 for (i = 0; i < KMEM_ZSIZE; i++) {
845 size = i << KMEM_ZSHIFT;
846 rsize = kmemzones[kmemsize[i]].kz_size;
847 count = (long long unsigned)krequests[i];
848
849 sbuf_printf(&sbuf, "%6d%28llu%11d\n", size,
850 (unsigned long long)count, rsize);
851
852 if ((rsize * count) > (size * count))
853 waste += (rsize * count) - (size * count);
854 mem += (rsize * count);
855 }
856 sbuf_printf(&sbuf,
857 "\nTotal memory used:\t%30llu\nTotal Memory wasted:\t%30llu\n",
858 (unsigned long long)mem, (unsigned long long)waste);
859 sbuf_finish(&sbuf);
860
861 error = SYSCTL_OUT(req, sbuf_data(&sbuf), sbuf_len(&sbuf));
862
863 sbuf_delete(&sbuf);
864 free(buf, M_TEMP);
865 return (error);
866 }
867
868 SYSCTL_OID(_kern, OID_AUTO, mprof, CTLTYPE_STRING|CTLFLAG_RD,
869 NULL, 0, sysctl_kern_mprof, "A", "Malloc Profiling");
870 #endif /* MALLOC_PROFILE */
Cache object: 2d04ba8f7b2341c1e8e8da5367e264e1
|