The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_malloc.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1987, 1991, 1993
    3  *      The Regents of the University of California.
    4  * Copyright (c) 2005-2009 Robert N. M. Watson
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice, this list of conditions and the following disclaimer.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice, this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  * 4. Neither the name of the University nor the names of its contributors
   16  *    may be used to endorse or promote products derived from this software
   17  *    without specific prior written permission.
   18  *
   19  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   29  * SUCH DAMAGE.
   30  *
   31  *      @(#)kern_malloc.c       8.3 (Berkeley) 1/4/94
   32  */
   33 
   34 /*
   35  * Kernel malloc(9) implementation -- general purpose kernel memory allocator
   36  * based on memory types.  Back end is implemented using the UMA(9) zone
   37  * allocator.  A set of fixed-size buckets are used for smaller allocations,
   38  * and a special UMA allocation interface is used for larger allocations.
   39  * Callers declare memory types, and statistics are maintained independently
   40  * for each memory type.  Statistics are maintained per-CPU for performance
   41  * reasons.  See malloc(9) and comments in malloc.h for a detailed
   42  * description.
   43  */
   44 
   45 #include <sys/cdefs.h>
   46 __FBSDID("$FreeBSD: releng/8.3/sys/kern/kern_malloc.c 230419 2012-01-21 07:21:44Z alc $");
   47 
   48 #include "opt_ddb.h"
   49 #include "opt_kdtrace.h"
   50 #include "opt_vm.h"
   51 
   52 #include <sys/param.h>
   53 #include <sys/systm.h>
   54 #include <sys/kdb.h>
   55 #include <sys/kernel.h>
   56 #include <sys/lock.h>
   57 #include <sys/malloc.h>
   58 #include <sys/mbuf.h>
   59 #include <sys/mutex.h>
   60 #include <sys/vmmeter.h>
   61 #include <sys/proc.h>
   62 #include <sys/sbuf.h>
   63 #include <sys/sysctl.h>
   64 #include <sys/time.h>
   65 
   66 #include <vm/vm.h>
   67 #include <vm/pmap.h>
   68 #include <vm/vm_param.h>
   69 #include <vm/vm_kern.h>
   70 #include <vm/vm_extern.h>
   71 #include <vm/vm_map.h>
   72 #include <vm/vm_page.h>
   73 #include <vm/uma.h>
   74 #include <vm/uma_int.h>
   75 #include <vm/uma_dbg.h>
   76 
   77 #ifdef DEBUG_MEMGUARD
   78 #include <vm/memguard.h>
   79 #endif
   80 #ifdef DEBUG_REDZONE
   81 #include <vm/redzone.h>
   82 #endif
   83 
   84 #if defined(INVARIANTS) && defined(__i386__)
   85 #include <machine/cpu.h>
   86 #endif
   87 
   88 #include <ddb/ddb.h>
   89 
   90 #ifdef KDTRACE_HOOKS
   91 #include <sys/dtrace_bsd.h>
   92 
   93 dtrace_malloc_probe_func_t      dtrace_malloc_probe;
   94 #endif
   95 
   96 /*
   97  * When realloc() is called, if the new size is sufficiently smaller than
   98  * the old size, realloc() will allocate a new, smaller block to avoid
   99  * wasting memory. 'Sufficiently smaller' is defined as: newsize <=
  100  * oldsize / 2^n, where REALLOC_FRACTION defines the value of 'n'.
  101  */
  102 #ifndef REALLOC_FRACTION
  103 #define REALLOC_FRACTION        1       /* new block if <= half the size */
  104 #endif
  105 
  106 /*
  107  * Centrally define some common malloc types.
  108  */
  109 MALLOC_DEFINE(M_CACHE, "cache", "Various Dynamically allocated caches");
  110 MALLOC_DEFINE(M_DEVBUF, "devbuf", "device driver memory");
  111 MALLOC_DEFINE(M_TEMP, "temp", "misc temporary data buffers");
  112 
  113 MALLOC_DEFINE(M_IP6OPT, "ip6opt", "IPv6 options");
  114 MALLOC_DEFINE(M_IP6NDP, "ip6ndp", "IPv6 Neighbor Discovery");
  115 
  116 static void kmeminit(void *);
  117 SYSINIT(kmem, SI_SUB_KMEM, SI_ORDER_FIRST, kmeminit, NULL);
  118 
  119 static MALLOC_DEFINE(M_FREE, "free", "should be on free list");
  120 
  121 static struct malloc_type *kmemstatistics;
  122 static vm_offset_t kmembase;
  123 static vm_offset_t kmemlimit;
  124 static int kmemcount;
  125 
  126 #define KMEM_ZSHIFT     4
  127 #define KMEM_ZBASE      16
  128 #define KMEM_ZMASK      (KMEM_ZBASE - 1)
  129 
  130 #define KMEM_ZMAX       PAGE_SIZE
  131 #define KMEM_ZSIZE      (KMEM_ZMAX >> KMEM_ZSHIFT)
  132 static u_int8_t kmemsize[KMEM_ZSIZE + 1];
  133 
  134 /*
  135  * Small malloc(9) memory allocations are allocated from a set of UMA buckets
  136  * of various sizes.
  137  *
  138  * XXX: The comment here used to read "These won't be powers of two for
  139  * long."  It's possible that a significant amount of wasted memory could be
  140  * recovered by tuning the sizes of these buckets.
  141  */
  142 struct {
  143         int kz_size;
  144         char *kz_name;
  145         uma_zone_t kz_zone;
  146 } kmemzones[] = {
  147         {16, "16", NULL},
  148         {32, "32", NULL},
  149         {64, "64", NULL},
  150         {128, "128", NULL},
  151         {256, "256", NULL},
  152         {512, "512", NULL},
  153         {1024, "1024", NULL},
  154         {2048, "2048", NULL},
  155         {4096, "4096", NULL},
  156 #if PAGE_SIZE > 4096
  157         {8192, "8192", NULL},
  158 #if PAGE_SIZE > 8192
  159         {16384, "16384", NULL},
  160 #if PAGE_SIZE > 16384
  161         {32768, "32768", NULL},
  162 #if PAGE_SIZE > 32768
  163         {65536, "65536", NULL},
  164 #if PAGE_SIZE > 65536
  165 #error  "Unsupported PAGE_SIZE"
  166 #endif  /* 65536 */
  167 #endif  /* 32768 */
  168 #endif  /* 16384 */
  169 #endif  /* 8192 */
  170 #endif  /* 4096 */
  171         {0, NULL},
  172 };
  173 
  174 /*
  175  * Zone to allocate malloc type descriptions from.  For ABI reasons, memory
  176  * types are described by a data structure passed by the declaring code, but
  177  * the malloc(9) implementation has its own data structure describing the
  178  * type and statistics.  This permits the malloc(9)-internal data structures
  179  * to be modified without breaking binary-compiled kernel modules that
  180  * declare malloc types.
  181  */
  182 static uma_zone_t mt_zone;
  183 
  184 u_long vm_kmem_size;
  185 SYSCTL_ULONG(_vm, OID_AUTO, kmem_size, CTLFLAG_RDTUN, &vm_kmem_size, 0,
  186     "Size of kernel memory");
  187 
  188 static u_long vm_kmem_size_min;
  189 SYSCTL_ULONG(_vm, OID_AUTO, kmem_size_min, CTLFLAG_RDTUN, &vm_kmem_size_min, 0,
  190     "Minimum size of kernel memory");
  191 
  192 static u_long vm_kmem_size_max;
  193 SYSCTL_ULONG(_vm, OID_AUTO, kmem_size_max, CTLFLAG_RDTUN, &vm_kmem_size_max, 0,
  194     "Maximum size of kernel memory");
  195 
  196 static u_int vm_kmem_size_scale;
  197 SYSCTL_UINT(_vm, OID_AUTO, kmem_size_scale, CTLFLAG_RDTUN, &vm_kmem_size_scale, 0,
  198     "Scale factor for kernel memory size");
  199 
  200 static int sysctl_kmem_map_size(SYSCTL_HANDLER_ARGS);
  201 SYSCTL_PROC(_vm, OID_AUTO, kmem_map_size,
  202     CTLFLAG_RD | CTLTYPE_ULONG | CTLFLAG_MPSAFE, NULL, 0,
  203     sysctl_kmem_map_size, "LU", "Current kmem_map allocation size");
  204 
  205 static int sysctl_kmem_map_free(SYSCTL_HANDLER_ARGS);
  206 SYSCTL_PROC(_vm, OID_AUTO, kmem_map_free,
  207     CTLFLAG_RD | CTLTYPE_ULONG | CTLFLAG_MPSAFE, NULL, 0,
  208     sysctl_kmem_map_free, "LU", "Largest contiguous free range in kmem_map");
  209 
  210 /*
  211  * The malloc_mtx protects the kmemstatistics linked list.
  212  */
  213 struct mtx malloc_mtx;
  214 
  215 #ifdef MALLOC_PROFILE
  216 uint64_t krequests[KMEM_ZSIZE + 1];
  217 
  218 static int sysctl_kern_mprof(SYSCTL_HANDLER_ARGS);
  219 #endif
  220 
  221 static int sysctl_kern_malloc_stats(SYSCTL_HANDLER_ARGS);
  222 
  223 /*
  224  * time_uptime of the last malloc(9) failure (induced or real).
  225  */
  226 static time_t t_malloc_fail;
  227 
  228 /*
  229  * malloc(9) fault injection -- cause malloc failures every (n) mallocs when
  230  * the caller specifies M_NOWAIT.  If set to 0, no failures are caused.
  231  */
  232 #ifdef MALLOC_MAKE_FAILURES
  233 SYSCTL_NODE(_debug, OID_AUTO, malloc, CTLFLAG_RD, 0,
  234     "Kernel malloc debugging options");
  235 
  236 static int malloc_failure_rate;
  237 static int malloc_nowait_count;
  238 static int malloc_failure_count;
  239 SYSCTL_INT(_debug_malloc, OID_AUTO, failure_rate, CTLFLAG_RW,
  240     &malloc_failure_rate, 0, "Every (n) mallocs with M_NOWAIT will fail");
  241 TUNABLE_INT("debug.malloc.failure_rate", &malloc_failure_rate);
  242 SYSCTL_INT(_debug_malloc, OID_AUTO, failure_count, CTLFLAG_RD,
  243     &malloc_failure_count, 0, "Number of imposed M_NOWAIT malloc failures");
  244 #endif
  245 
  246 static int
  247 sysctl_kmem_map_size(SYSCTL_HANDLER_ARGS)
  248 {
  249         u_long size;
  250 
  251         size = kmem_map->size;
  252         return (sysctl_handle_long(oidp, &size, 0, req));
  253 }
  254 
  255 static int
  256 sysctl_kmem_map_free(SYSCTL_HANDLER_ARGS)
  257 {
  258         u_long size;
  259 
  260         vm_map_lock_read(kmem_map);
  261         size = kmem_map->root != NULL ? kmem_map->root->max_free :
  262             kmem_map->max_offset - kmem_map->min_offset;
  263         vm_map_unlock_read(kmem_map);
  264         return (sysctl_handle_long(oidp, &size, 0, req));
  265 }
  266 
  267 int
  268 malloc_last_fail(void)
  269 {
  270 
  271         return (time_uptime - t_malloc_fail);
  272 }
  273 
  274 /*
  275  * An allocation has succeeded -- update malloc type statistics for the
  276  * amount of bucket size.  Occurs within a critical section so that the
  277  * thread isn't preempted and doesn't migrate while updating per-PCU
  278  * statistics.
  279  */
  280 static void
  281 malloc_type_zone_allocated(struct malloc_type *mtp, unsigned long size,
  282     int zindx)
  283 {
  284         struct malloc_type_internal *mtip;
  285         struct malloc_type_stats *mtsp;
  286 
  287         critical_enter();
  288         mtip = mtp->ks_handle;
  289         mtsp = &mtip->mti_stats[curcpu];
  290         if (size > 0) {
  291                 mtsp->mts_memalloced += size;
  292                 mtsp->mts_numallocs++;
  293         }
  294         if (zindx != -1)
  295                 mtsp->mts_size |= 1 << zindx;
  296 
  297 #ifdef KDTRACE_HOOKS
  298         if (dtrace_malloc_probe != NULL) {
  299                 uint32_t probe_id = mtip->mti_probes[DTMALLOC_PROBE_MALLOC];
  300                 if (probe_id != 0)
  301                         (dtrace_malloc_probe)(probe_id,
  302                             (uintptr_t) mtp, (uintptr_t) mtip,
  303                             (uintptr_t) mtsp, size, zindx);
  304         }
  305 #endif
  306 
  307         critical_exit();
  308 }
  309 
  310 void
  311 malloc_type_allocated(struct malloc_type *mtp, unsigned long size)
  312 {
  313 
  314         if (size > 0)
  315                 malloc_type_zone_allocated(mtp, size, -1);
  316 }
  317 
  318 /*
  319  * A free operation has occurred -- update malloc type statistics for the
  320  * amount of the bucket size.  Occurs within a critical section so that the
  321  * thread isn't preempted and doesn't migrate while updating per-CPU
  322  * statistics.
  323  */
  324 void
  325 malloc_type_freed(struct malloc_type *mtp, unsigned long size)
  326 {
  327         struct malloc_type_internal *mtip;
  328         struct malloc_type_stats *mtsp;
  329 
  330         critical_enter();
  331         mtip = mtp->ks_handle;
  332         mtsp = &mtip->mti_stats[curcpu];
  333         mtsp->mts_memfreed += size;
  334         mtsp->mts_numfrees++;
  335 
  336 #ifdef KDTRACE_HOOKS
  337         if (dtrace_malloc_probe != NULL) {
  338                 uint32_t probe_id = mtip->mti_probes[DTMALLOC_PROBE_FREE];
  339                 if (probe_id != 0)
  340                         (dtrace_malloc_probe)(probe_id,
  341                             (uintptr_t) mtp, (uintptr_t) mtip,
  342                             (uintptr_t) mtsp, size, 0);
  343         }
  344 #endif
  345 
  346         critical_exit();
  347 }
  348 
  349 /*
  350  *      malloc:
  351  *
  352  *      Allocate a block of memory.
  353  *
  354  *      If M_NOWAIT is set, this routine will not block and return NULL if
  355  *      the allocation fails.
  356  */
  357 void *
  358 malloc(unsigned long size, struct malloc_type *mtp, int flags)
  359 {
  360         int indx;
  361         caddr_t va;
  362         uma_zone_t zone;
  363 #if defined(DIAGNOSTIC) || defined(DEBUG_REDZONE)
  364         unsigned long osize = size;
  365 #endif
  366 
  367 #ifdef INVARIANTS
  368         KASSERT(mtp->ks_magic == M_MAGIC, ("malloc: bad malloc type magic"));
  369         /*
  370          * Check that exactly one of M_WAITOK or M_NOWAIT is specified.
  371          */
  372         indx = flags & (M_WAITOK | M_NOWAIT);
  373         if (indx != M_NOWAIT && indx != M_WAITOK) {
  374                 static  struct timeval lasterr;
  375                 static  int curerr, once;
  376                 if (once == 0 && ppsratecheck(&lasterr, &curerr, 1)) {
  377                         printf("Bad malloc flags: %x\n", indx);
  378                         kdb_backtrace();
  379                         flags |= M_WAITOK;
  380                         once++;
  381                 }
  382         }
  383 #endif
  384 #ifdef MALLOC_MAKE_FAILURES
  385         if ((flags & M_NOWAIT) && (malloc_failure_rate != 0)) {
  386                 atomic_add_int(&malloc_nowait_count, 1);
  387                 if ((malloc_nowait_count % malloc_failure_rate) == 0) {
  388                         atomic_add_int(&malloc_failure_count, 1);
  389                         t_malloc_fail = time_uptime;
  390                         return (NULL);
  391                 }
  392         }
  393 #endif
  394         if (flags & M_WAITOK)
  395                 KASSERT(curthread->td_intr_nesting_level == 0,
  396                    ("malloc(M_WAITOK) in interrupt context"));
  397 
  398 #ifdef DEBUG_MEMGUARD
  399         if (memguard_cmp(mtp, size)) {
  400                 va = memguard_alloc(size, flags);
  401                 if (va != NULL)
  402                         return (va);
  403                 /* This is unfortunate but should not be fatal. */
  404         }
  405 #endif
  406 
  407 #ifdef DEBUG_REDZONE
  408         size = redzone_size_ntor(size);
  409 #endif
  410 
  411         if (size <= KMEM_ZMAX) {
  412                 if (size & KMEM_ZMASK)
  413                         size = (size & ~KMEM_ZMASK) + KMEM_ZBASE;
  414                 indx = kmemsize[size >> KMEM_ZSHIFT];
  415                 zone = kmemzones[indx].kz_zone;
  416 #ifdef MALLOC_PROFILE
  417                 krequests[size >> KMEM_ZSHIFT]++;
  418 #endif
  419                 va = uma_zalloc(zone, flags);
  420                 if (va != NULL)
  421                         size = zone->uz_size;
  422                 malloc_type_zone_allocated(mtp, va == NULL ? 0 : size, indx);
  423         } else {
  424                 size = roundup(size, PAGE_SIZE);
  425                 zone = NULL;
  426                 va = uma_large_malloc(size, flags);
  427                 malloc_type_allocated(mtp, va == NULL ? 0 : size);
  428         }
  429         if (flags & M_WAITOK)
  430                 KASSERT(va != NULL, ("malloc(M_WAITOK) returned NULL"));
  431         else if (va == NULL)
  432                 t_malloc_fail = time_uptime;
  433 #ifdef DIAGNOSTIC
  434         if (va != NULL && !(flags & M_ZERO)) {
  435                 memset(va, 0x70, osize);
  436         }
  437 #endif
  438 #ifdef DEBUG_REDZONE
  439         if (va != NULL)
  440                 va = redzone_setup(va, osize);
  441 #endif
  442         return ((void *) va);
  443 }
  444 
  445 /*
  446  *      free:
  447  *
  448  *      Free a block of memory allocated by malloc.
  449  *
  450  *      This routine may not block.
  451  */
  452 void
  453 free(void *addr, struct malloc_type *mtp)
  454 {
  455         uma_slab_t slab;
  456         u_long size;
  457 
  458         KASSERT(mtp->ks_magic == M_MAGIC, ("free: bad malloc type magic"));
  459 
  460         /* free(NULL, ...) does nothing */
  461         if (addr == NULL)
  462                 return;
  463 
  464 #ifdef DEBUG_MEMGUARD
  465         if (is_memguard_addr(addr)) {
  466                 memguard_free(addr);
  467                 return;
  468         }
  469 #endif
  470 
  471 #ifdef DEBUG_REDZONE
  472         redzone_check(addr);
  473         addr = redzone_addr_ntor(addr);
  474 #endif
  475 
  476         slab = vtoslab((vm_offset_t)addr & (~UMA_SLAB_MASK));
  477 
  478         if (slab == NULL)
  479                 panic("free: address %p(%p) has not been allocated.\n",
  480                     addr, (void *)((u_long)addr & (~UMA_SLAB_MASK)));
  481 
  482 
  483         if (!(slab->us_flags & UMA_SLAB_MALLOC)) {
  484 #ifdef INVARIANTS
  485                 struct malloc_type **mtpp = addr;
  486 #endif
  487                 size = slab->us_keg->uk_size;
  488 #ifdef INVARIANTS
  489                 /*
  490                  * Cache a pointer to the malloc_type that most recently freed
  491                  * this memory here.  This way we know who is most likely to
  492                  * have stepped on it later.
  493                  *
  494                  * This code assumes that size is a multiple of 8 bytes for
  495                  * 64 bit machines
  496                  */
  497                 mtpp = (struct malloc_type **)
  498                     ((unsigned long)mtpp & ~UMA_ALIGN_PTR);
  499                 mtpp += (size - sizeof(struct malloc_type *)) /
  500                     sizeof(struct malloc_type *);
  501                 *mtpp = mtp;
  502 #endif
  503                 uma_zfree_arg(LIST_FIRST(&slab->us_keg->uk_zones), addr, slab);
  504         } else {
  505                 size = slab->us_size;
  506                 uma_large_free(slab);
  507         }
  508         malloc_type_freed(mtp, size);
  509 }
  510 
  511 /*
  512  *      realloc: change the size of a memory block
  513  */
  514 void *
  515 realloc(void *addr, unsigned long size, struct malloc_type *mtp, int flags)
  516 {
  517         uma_slab_t slab;
  518         unsigned long alloc;
  519         void *newaddr;
  520 
  521         KASSERT(mtp->ks_magic == M_MAGIC,
  522             ("realloc: bad malloc type magic"));
  523 
  524         /* realloc(NULL, ...) is equivalent to malloc(...) */
  525         if (addr == NULL)
  526                 return (malloc(size, mtp, flags));
  527 
  528         /*
  529          * XXX: Should report free of old memory and alloc of new memory to
  530          * per-CPU stats.
  531          */
  532 
  533 #ifdef DEBUG_MEMGUARD
  534         if (is_memguard_addr(addr))
  535                 return (memguard_realloc(addr, size, mtp, flags));
  536 #endif
  537 
  538 #ifdef DEBUG_REDZONE
  539         slab = NULL;
  540         alloc = redzone_get_size(addr);
  541 #else
  542         slab = vtoslab((vm_offset_t)addr & ~(UMA_SLAB_MASK));
  543 
  544         /* Sanity check */
  545         KASSERT(slab != NULL,
  546             ("realloc: address %p out of range", (void *)addr));
  547 
  548         /* Get the size of the original block */
  549         if (!(slab->us_flags & UMA_SLAB_MALLOC))
  550                 alloc = slab->us_keg->uk_size;
  551         else
  552                 alloc = slab->us_size;
  553 
  554         /* Reuse the original block if appropriate */
  555         if (size <= alloc
  556             && (size > (alloc >> REALLOC_FRACTION) || alloc == MINALLOCSIZE))
  557                 return (addr);
  558 #endif /* !DEBUG_REDZONE */
  559 
  560         /* Allocate a new, bigger (or smaller) block */
  561         if ((newaddr = malloc(size, mtp, flags)) == NULL)
  562                 return (NULL);
  563 
  564         /* Copy over original contents */
  565         bcopy(addr, newaddr, min(size, alloc));
  566         free(addr, mtp);
  567         return (newaddr);
  568 }
  569 
  570 /*
  571  *      reallocf: same as realloc() but free memory on failure.
  572  */
  573 void *
  574 reallocf(void *addr, unsigned long size, struct malloc_type *mtp, int flags)
  575 {
  576         void *mem;
  577 
  578         if ((mem = realloc(addr, size, mtp, flags)) == NULL)
  579                 free(addr, mtp);
  580         return (mem);
  581 }
  582 
  583 /*
  584  * Initialize the kernel memory allocator
  585  */
  586 /* ARGSUSED*/
  587 static void
  588 kmeminit(void *dummy)
  589 {
  590         u_int8_t indx;
  591         u_long mem_size, tmp;
  592         int i;
  593  
  594         mtx_init(&malloc_mtx, "malloc", NULL, MTX_DEF);
  595 
  596         /*
  597          * Try to auto-tune the kernel memory size, so that it is
  598          * more applicable for a wider range of machine sizes.  The
  599          * VM_KMEM_SIZE_MAX is dependent on the maximum KVA space
  600          * available.
  601          *
  602          * Note that the kmem_map is also used by the zone allocator,
  603          * so make sure that there is enough space.
  604          */
  605         vm_kmem_size = VM_KMEM_SIZE + nmbclusters * PAGE_SIZE;
  606         mem_size = cnt.v_page_count;
  607 
  608 #if defined(VM_KMEM_SIZE_SCALE)
  609         vm_kmem_size_scale = VM_KMEM_SIZE_SCALE;
  610 #endif
  611         TUNABLE_INT_FETCH("vm.kmem_size_scale", &vm_kmem_size_scale);
  612         if (vm_kmem_size_scale > 0 &&
  613             (mem_size / vm_kmem_size_scale) > (vm_kmem_size / PAGE_SIZE))
  614                 vm_kmem_size = (mem_size / vm_kmem_size_scale) * PAGE_SIZE;
  615 
  616 #if defined(VM_KMEM_SIZE_MIN)
  617         vm_kmem_size_min = VM_KMEM_SIZE_MIN;
  618 #endif
  619         TUNABLE_ULONG_FETCH("vm.kmem_size_min", &vm_kmem_size_min);
  620         if (vm_kmem_size_min > 0 && vm_kmem_size < vm_kmem_size_min) {
  621                 vm_kmem_size = vm_kmem_size_min;
  622         }
  623 
  624 #if defined(VM_KMEM_SIZE_MAX)
  625         vm_kmem_size_max = VM_KMEM_SIZE_MAX;
  626 #endif
  627         TUNABLE_ULONG_FETCH("vm.kmem_size_max", &vm_kmem_size_max);
  628         if (vm_kmem_size_max > 0 && vm_kmem_size >= vm_kmem_size_max)
  629                 vm_kmem_size = vm_kmem_size_max;
  630 
  631         /* Allow final override from the kernel environment */
  632         TUNABLE_ULONG_FETCH("vm.kmem_size", &vm_kmem_size);
  633 
  634         /*
  635          * Limit kmem virtual size to twice the physical memory.
  636          * This allows for kmem map sparseness, but limits the size
  637          * to something sane.  Be careful to not overflow the 32bit
  638          * ints while doing the check or the adjustment.
  639          */
  640         if (vm_kmem_size / 2 / PAGE_SIZE > mem_size)
  641                 vm_kmem_size = 2 * mem_size * PAGE_SIZE;
  642 
  643         /*
  644          * Tune settings based on the kmem map's size at this time.
  645          */
  646         init_param3(vm_kmem_size / PAGE_SIZE);
  647 
  648 #ifdef DEBUG_MEMGUARD
  649         tmp = memguard_fudge(vm_kmem_size, vm_kmem_size_max);
  650 #else
  651         tmp = vm_kmem_size;
  652 #endif
  653         kmem_map = kmem_suballoc(kernel_map, &kmembase, &kmemlimit,
  654             tmp, TRUE);
  655         kmem_map->system_map = 1;
  656 
  657 #ifdef DEBUG_MEMGUARD
  658         /*
  659          * Initialize MemGuard if support compiled in.  MemGuard is a
  660          * replacement allocator used for detecting tamper-after-free
  661          * scenarios as they occur.  It is only used for debugging.
  662          */
  663         memguard_init(kmem_map);
  664 #endif
  665 
  666         uma_startup2();
  667 
  668         mt_zone = uma_zcreate("mt_zone", sizeof(struct malloc_type_internal),
  669 #ifdef INVARIANTS
  670             mtrash_ctor, mtrash_dtor, mtrash_init, mtrash_fini,
  671 #else
  672             NULL, NULL, NULL, NULL,
  673 #endif
  674             UMA_ALIGN_PTR, UMA_ZONE_MALLOC);
  675         for (i = 0, indx = 0; kmemzones[indx].kz_size != 0; indx++) {
  676                 int size = kmemzones[indx].kz_size;
  677                 char *name = kmemzones[indx].kz_name;
  678 
  679                 kmemzones[indx].kz_zone = uma_zcreate(name, size,
  680 #ifdef INVARIANTS
  681                     mtrash_ctor, mtrash_dtor, mtrash_init, mtrash_fini,
  682 #else
  683                     NULL, NULL, NULL, NULL,
  684 #endif
  685                     UMA_ALIGN_PTR, UMA_ZONE_MALLOC);
  686                     
  687                 for (;i <= size; i+= KMEM_ZBASE)
  688                         kmemsize[i >> KMEM_ZSHIFT] = indx;
  689                 
  690         }
  691 }
  692 
  693 void
  694 malloc_init(void *data)
  695 {
  696         struct malloc_type_internal *mtip;
  697         struct malloc_type *mtp;
  698 
  699         KASSERT(cnt.v_page_count != 0, ("malloc_register before vm_init"));
  700 
  701         mtp = data;
  702         if (mtp->ks_magic != M_MAGIC)
  703                 panic("malloc_init: bad malloc type magic");
  704 
  705         mtip = uma_zalloc(mt_zone, M_WAITOK | M_ZERO);
  706         mtp->ks_handle = mtip;
  707 
  708         mtx_lock(&malloc_mtx);
  709         mtp->ks_next = kmemstatistics;
  710         kmemstatistics = mtp;
  711         kmemcount++;
  712         mtx_unlock(&malloc_mtx);
  713 }
  714 
  715 void
  716 malloc_uninit(void *data)
  717 {
  718         struct malloc_type_internal *mtip;
  719         struct malloc_type_stats *mtsp;
  720         struct malloc_type *mtp, *temp;
  721         uma_slab_t slab;
  722         long temp_allocs, temp_bytes;
  723         int i;
  724 
  725         mtp = data;
  726         KASSERT(mtp->ks_magic == M_MAGIC,
  727             ("malloc_uninit: bad malloc type magic"));
  728         KASSERT(mtp->ks_handle != NULL, ("malloc_deregister: cookie NULL"));
  729 
  730         mtx_lock(&malloc_mtx);
  731         mtip = mtp->ks_handle;
  732         mtp->ks_handle = NULL;
  733         if (mtp != kmemstatistics) {
  734                 for (temp = kmemstatistics; temp != NULL;
  735                     temp = temp->ks_next) {
  736                         if (temp->ks_next == mtp) {
  737                                 temp->ks_next = mtp->ks_next;
  738                                 break;
  739                         }
  740                 }
  741                 KASSERT(temp,
  742                     ("malloc_uninit: type '%s' not found", mtp->ks_shortdesc));
  743         } else
  744                 kmemstatistics = mtp->ks_next;
  745         kmemcount--;
  746         mtx_unlock(&malloc_mtx);
  747 
  748         /*
  749          * Look for memory leaks.
  750          */
  751         temp_allocs = temp_bytes = 0;
  752         for (i = 0; i < MAXCPU; i++) {
  753                 mtsp = &mtip->mti_stats[i];
  754                 temp_allocs += mtsp->mts_numallocs;
  755                 temp_allocs -= mtsp->mts_numfrees;
  756                 temp_bytes += mtsp->mts_memalloced;
  757                 temp_bytes -= mtsp->mts_memfreed;
  758         }
  759         if (temp_allocs > 0 || temp_bytes > 0) {
  760                 printf("Warning: memory type %s leaked memory on destroy "
  761                     "(%ld allocations, %ld bytes leaked).\n", mtp->ks_shortdesc,
  762                     temp_allocs, temp_bytes);
  763         }
  764 
  765         slab = vtoslab((vm_offset_t) mtip & (~UMA_SLAB_MASK));
  766         uma_zfree_arg(mt_zone, mtip, slab);
  767 }
  768 
  769 struct malloc_type *
  770 malloc_desc2type(const char *desc)
  771 {
  772         struct malloc_type *mtp;
  773 
  774         mtx_assert(&malloc_mtx, MA_OWNED);
  775         for (mtp = kmemstatistics; mtp != NULL; mtp = mtp->ks_next) {
  776                 if (strcmp(mtp->ks_shortdesc, desc) == 0)
  777                         return (mtp);
  778         }
  779         return (NULL);
  780 }
  781 
  782 static int
  783 sysctl_kern_malloc_stats(SYSCTL_HANDLER_ARGS)
  784 {
  785         struct malloc_type_stream_header mtsh;
  786         struct malloc_type_internal *mtip;
  787         struct malloc_type_header mth;
  788         struct malloc_type *mtp;
  789         int buflen, count, error, i;
  790         struct sbuf sbuf;
  791         char *buffer;
  792 
  793         mtx_lock(&malloc_mtx);
  794 restart:
  795         mtx_assert(&malloc_mtx, MA_OWNED);
  796         count = kmemcount;
  797         mtx_unlock(&malloc_mtx);
  798         buflen = sizeof(mtsh) + count * (sizeof(mth) +
  799             sizeof(struct malloc_type_stats) * MAXCPU) + 1;
  800         buffer = malloc(buflen, M_TEMP, M_WAITOK | M_ZERO);
  801         mtx_lock(&malloc_mtx);
  802         if (count < kmemcount) {
  803                 free(buffer, M_TEMP);
  804                 goto restart;
  805         }
  806 
  807         sbuf_new(&sbuf, buffer, buflen, SBUF_FIXEDLEN);
  808 
  809         /*
  810          * Insert stream header.
  811          */
  812         bzero(&mtsh, sizeof(mtsh));
  813         mtsh.mtsh_version = MALLOC_TYPE_STREAM_VERSION;
  814         mtsh.mtsh_maxcpus = MAXCPU;
  815         mtsh.mtsh_count = kmemcount;
  816         if (sbuf_bcat(&sbuf, &mtsh, sizeof(mtsh)) < 0) {
  817                 mtx_unlock(&malloc_mtx);
  818                 error = ENOMEM;
  819                 goto out;
  820         }
  821 
  822         /*
  823          * Insert alternating sequence of type headers and type statistics.
  824          */
  825         for (mtp = kmemstatistics; mtp != NULL; mtp = mtp->ks_next) {
  826                 mtip = (struct malloc_type_internal *)mtp->ks_handle;
  827 
  828                 /*
  829                  * Insert type header.
  830                  */
  831                 bzero(&mth, sizeof(mth));
  832                 strlcpy(mth.mth_name, mtp->ks_shortdesc, MALLOC_MAX_NAME);
  833                 if (sbuf_bcat(&sbuf, &mth, sizeof(mth)) < 0) {
  834                         mtx_unlock(&malloc_mtx);
  835                         error = ENOMEM;
  836                         goto out;
  837                 }
  838 
  839                 /*
  840                  * Insert type statistics for each CPU.
  841                  */
  842                 for (i = 0; i < MAXCPU; i++) {
  843                         if (sbuf_bcat(&sbuf, &mtip->mti_stats[i],
  844                             sizeof(mtip->mti_stats[i])) < 0) {
  845                                 mtx_unlock(&malloc_mtx);
  846                                 error = ENOMEM;
  847                                 goto out;
  848                         }
  849                 }
  850         }
  851         mtx_unlock(&malloc_mtx);
  852         sbuf_finish(&sbuf);
  853         error = SYSCTL_OUT(req, sbuf_data(&sbuf), sbuf_len(&sbuf));
  854 out:
  855         sbuf_delete(&sbuf);
  856         free(buffer, M_TEMP);
  857         return (error);
  858 }
  859 
  860 SYSCTL_PROC(_kern, OID_AUTO, malloc_stats, CTLFLAG_RD|CTLTYPE_STRUCT,
  861     0, 0, sysctl_kern_malloc_stats, "s,malloc_type_ustats",
  862     "Return malloc types");
  863 
  864 SYSCTL_INT(_kern, OID_AUTO, malloc_count, CTLFLAG_RD, &kmemcount, 0,
  865     "Count of kernel malloc types");
  866 
  867 void
  868 malloc_type_list(malloc_type_list_func_t *func, void *arg)
  869 {
  870         struct malloc_type *mtp, **bufmtp;
  871         int count, i;
  872         size_t buflen;
  873 
  874         mtx_lock(&malloc_mtx);
  875 restart:
  876         mtx_assert(&malloc_mtx, MA_OWNED);
  877         count = kmemcount;
  878         mtx_unlock(&malloc_mtx);
  879 
  880         buflen = sizeof(struct malloc_type *) * count;
  881         bufmtp = malloc(buflen, M_TEMP, M_WAITOK);
  882 
  883         mtx_lock(&malloc_mtx);
  884 
  885         if (count < kmemcount) {
  886                 free(bufmtp, M_TEMP);
  887                 goto restart;
  888         }
  889 
  890         for (mtp = kmemstatistics, i = 0; mtp != NULL; mtp = mtp->ks_next, i++)
  891                 bufmtp[i] = mtp;
  892 
  893         mtx_unlock(&malloc_mtx);
  894 
  895         for (i = 0; i < count; i++)
  896                 (func)(bufmtp[i], arg);
  897 
  898         free(bufmtp, M_TEMP);
  899 }
  900 
  901 #ifdef DDB
  902 DB_SHOW_COMMAND(malloc, db_show_malloc)
  903 {
  904         struct malloc_type_internal *mtip;
  905         struct malloc_type *mtp;
  906         u_int64_t allocs, frees;
  907         u_int64_t alloced, freed;
  908         int i;
  909 
  910         db_printf("%18s %12s  %12s %12s\n", "Type", "InUse", "MemUse",
  911             "Requests");
  912         for (mtp = kmemstatistics; mtp != NULL; mtp = mtp->ks_next) {
  913                 mtip = (struct malloc_type_internal *)mtp->ks_handle;
  914                 allocs = 0;
  915                 frees = 0;
  916                 alloced = 0;
  917                 freed = 0;
  918                 for (i = 0; i < MAXCPU; i++) {
  919                         allocs += mtip->mti_stats[i].mts_numallocs;
  920                         frees += mtip->mti_stats[i].mts_numfrees;
  921                         alloced += mtip->mti_stats[i].mts_memalloced;
  922                         freed += mtip->mti_stats[i].mts_memfreed;
  923                 }
  924                 db_printf("%18s %12ju %12juK %12ju\n",
  925                     mtp->ks_shortdesc, allocs - frees,
  926                     (alloced - freed + 1023) / 1024, allocs);
  927         }
  928 }
  929 #endif
  930 
  931 #ifdef MALLOC_PROFILE
  932 
  933 static int
  934 sysctl_kern_mprof(SYSCTL_HANDLER_ARGS)
  935 {
  936         int linesize = 64;
  937         struct sbuf sbuf;
  938         uint64_t count;
  939         uint64_t waste;
  940         uint64_t mem;
  941         int bufsize;
  942         int error;
  943         char *buf;
  944         int rsize;
  945         int size;
  946         int i;
  947 
  948         bufsize = linesize * (KMEM_ZSIZE + 1);
  949         bufsize += 128;         /* For the stats line */
  950         bufsize += 128;         /* For the banner line */
  951         waste = 0;
  952         mem = 0;
  953 
  954         buf = malloc(bufsize, M_TEMP, M_WAITOK|M_ZERO);
  955         sbuf_new(&sbuf, buf, bufsize, SBUF_FIXEDLEN);
  956         sbuf_printf(&sbuf, 
  957             "\n  Size                    Requests  Real Size\n");
  958         for (i = 0; i < KMEM_ZSIZE; i++) {
  959                 size = i << KMEM_ZSHIFT;
  960                 rsize = kmemzones[kmemsize[i]].kz_size;
  961                 count = (long long unsigned)krequests[i];
  962 
  963                 sbuf_printf(&sbuf, "%6d%28llu%11d\n", size,
  964                     (unsigned long long)count, rsize);
  965 
  966                 if ((rsize * count) > (size * count))
  967                         waste += (rsize * count) - (size * count);
  968                 mem += (rsize * count);
  969         }
  970         sbuf_printf(&sbuf,
  971             "\nTotal memory used:\t%30llu\nTotal Memory wasted:\t%30llu\n",
  972             (unsigned long long)mem, (unsigned long long)waste);
  973         sbuf_finish(&sbuf);
  974 
  975         error = SYSCTL_OUT(req, sbuf_data(&sbuf), sbuf_len(&sbuf));
  976 
  977         sbuf_delete(&sbuf);
  978         free(buf, M_TEMP);
  979         return (error);
  980 }
  981 
  982 SYSCTL_OID(_kern, OID_AUTO, mprof, CTLTYPE_STRING|CTLFLAG_RD,
  983     NULL, 0, sysctl_kern_mprof, "A", "Malloc Profiling");
  984 #endif /* MALLOC_PROFILE */

Cache object: e478c2619efd9d4fba3988504504705c


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.