The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/vm/uma_core.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2002-2005, 2009, 2013 Jeffrey Roberson <jeff@FreeBSD.org>
    3  * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
    4  * Copyright (c) 2004-2006 Robert N. M. Watson
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice unmodified, this list of conditions, and the following
   12  *    disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
   26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   27  */
   28 
   29 /*
   30  * uma_core.c  Implementation of the Universal Memory allocator
   31  *
   32  * This allocator is intended to replace the multitude of similar object caches
   33  * in the standard FreeBSD kernel.  The intent is to be flexible as well as
   34  * efficient.  A primary design goal is to return unused memory to the rest of
   35  * the system.  This will make the system as a whole more flexible due to the
   36  * ability to move memory to subsystems which most need it instead of leaving
   37  * pools of reserved memory unused.
   38  *
   39  * The basic ideas stem from similar slab/zone based allocators whose algorithms
   40  * are well known.
   41  *
   42  */
   43 
   44 /*
   45  * TODO:
   46  *      - Improve memory usage for large allocations
   47  *      - Investigate cache size adjustments
   48  */
   49 
   50 #include <sys/cdefs.h>
   51 __FBSDID("$FreeBSD: releng/11.1/sys/vm/uma_core.c 318169 2017-05-11 03:37:05Z jhb $");
   52 
   53 /* I should really use ktr.. */
   54 /*
   55 #define UMA_DEBUG 1
   56 #define UMA_DEBUG_ALLOC 1
   57 #define UMA_DEBUG_ALLOC_1 1
   58 */
   59 
   60 #include "opt_ddb.h"
   61 #include "opt_param.h"
   62 #include "opt_vm.h"
   63 
   64 #include <sys/param.h>
   65 #include <sys/systm.h>
   66 #include <sys/bitset.h>
   67 #include <sys/eventhandler.h>
   68 #include <sys/kernel.h>
   69 #include <sys/types.h>
   70 #include <sys/queue.h>
   71 #include <sys/malloc.h>
   72 #include <sys/ktr.h>
   73 #include <sys/lock.h>
   74 #include <sys/sysctl.h>
   75 #include <sys/mutex.h>
   76 #include <sys/proc.h>
   77 #include <sys/random.h>
   78 #include <sys/rwlock.h>
   79 #include <sys/sbuf.h>
   80 #include <sys/sched.h>
   81 #include <sys/smp.h>
   82 #include <sys/taskqueue.h>
   83 #include <sys/vmmeter.h>
   84 
   85 #include <vm/vm.h>
   86 #include <vm/vm_object.h>
   87 #include <vm/vm_page.h>
   88 #include <vm/vm_pageout.h>
   89 #include <vm/vm_param.h>
   90 #include <vm/vm_map.h>
   91 #include <vm/vm_kern.h>
   92 #include <vm/vm_extern.h>
   93 #include <vm/uma.h>
   94 #include <vm/uma_int.h>
   95 #include <vm/uma_dbg.h>
   96 
   97 #include <ddb/ddb.h>
   98 
   99 #ifdef DEBUG_MEMGUARD
  100 #include <vm/memguard.h>
  101 #endif
  102 
  103 /*
  104  * This is the zone and keg from which all zones are spawned.  The idea is that
  105  * even the zone & keg heads are allocated from the allocator, so we use the
  106  * bss section to bootstrap us.
  107  */
  108 static struct uma_keg masterkeg;
  109 static struct uma_zone masterzone_k;
  110 static struct uma_zone masterzone_z;
  111 static uma_zone_t kegs = &masterzone_k;
  112 static uma_zone_t zones = &masterzone_z;
  113 
  114 /* This is the zone from which all of uma_slab_t's are allocated. */
  115 static uma_zone_t slabzone;
  116 
  117 /*
  118  * The initial hash tables come out of this zone so they can be allocated
  119  * prior to malloc coming up.
  120  */
  121 static uma_zone_t hashzone;
  122 
  123 /* The boot-time adjusted value for cache line alignment. */
  124 int uma_align_cache = 64 - 1;
  125 
  126 static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets");
  127 
  128 /*
  129  * Are we allowed to allocate buckets?
  130  */
  131 static int bucketdisable = 1;
  132 
  133 /* Linked list of all kegs in the system */
  134 static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(uma_kegs);
  135 
  136 /* Linked list of all cache-only zones in the system */
  137 static LIST_HEAD(,uma_zone) uma_cachezones =
  138     LIST_HEAD_INITIALIZER(uma_cachezones);
  139 
  140 /* This RW lock protects the keg list */
  141 static struct rwlock_padalign uma_rwlock;
  142 
  143 /* Linked list of boot time pages */
  144 static LIST_HEAD(,uma_slab) uma_boot_pages =
  145     LIST_HEAD_INITIALIZER(uma_boot_pages);
  146 
  147 /* This mutex protects the boot time pages list */
  148 static struct mtx_padalign uma_boot_pages_mtx;
  149 
  150 static struct sx uma_drain_lock;
  151 
  152 /* Is the VM done starting up? */
  153 static int booted = 0;
  154 #define UMA_STARTUP     1
  155 #define UMA_STARTUP2    2
  156 
  157 /*
  158  * This is the handle used to schedule events that need to happen
  159  * outside of the allocation fast path.
  160  */
  161 static struct callout uma_callout;
  162 #define UMA_TIMEOUT     20              /* Seconds for callout interval. */
  163 
  164 /*
  165  * This structure is passed as the zone ctor arg so that I don't have to create
  166  * a special allocation function just for zones.
  167  */
  168 struct uma_zctor_args {
  169         const char *name;
  170         size_t size;
  171         uma_ctor ctor;
  172         uma_dtor dtor;
  173         uma_init uminit;
  174         uma_fini fini;
  175         uma_import import;
  176         uma_release release;
  177         void *arg;
  178         uma_keg_t keg;
  179         int align;
  180         uint32_t flags;
  181 };
  182 
  183 struct uma_kctor_args {
  184         uma_zone_t zone;
  185         size_t size;
  186         uma_init uminit;
  187         uma_fini fini;
  188         int align;
  189         uint32_t flags;
  190 };
  191 
  192 struct uma_bucket_zone {
  193         uma_zone_t      ubz_zone;
  194         char            *ubz_name;
  195         int             ubz_entries;    /* Number of items it can hold. */
  196         int             ubz_maxsize;    /* Maximum allocation size per-item. */
  197 };
  198 
  199 /*
  200  * Compute the actual number of bucket entries to pack them in power
  201  * of two sizes for more efficient space utilization.
  202  */
  203 #define BUCKET_SIZE(n)                                          \
  204     (((sizeof(void *) * (n)) - sizeof(struct uma_bucket)) / sizeof(void *))
  205 
  206 #define BUCKET_MAX      BUCKET_SIZE(256)
  207 
  208 struct uma_bucket_zone bucket_zones[] = {
  209         { NULL, "4 Bucket", BUCKET_SIZE(4), 4096 },
  210         { NULL, "6 Bucket", BUCKET_SIZE(6), 3072 },
  211         { NULL, "8 Bucket", BUCKET_SIZE(8), 2048 },
  212         { NULL, "12 Bucket", BUCKET_SIZE(12), 1536 },
  213         { NULL, "16 Bucket", BUCKET_SIZE(16), 1024 },
  214         { NULL, "32 Bucket", BUCKET_SIZE(32), 512 },
  215         { NULL, "64 Bucket", BUCKET_SIZE(64), 256 },
  216         { NULL, "128 Bucket", BUCKET_SIZE(128), 128 },
  217         { NULL, "256 Bucket", BUCKET_SIZE(256), 64 },
  218         { NULL, NULL, 0}
  219 };
  220 
  221 /*
  222  * Flags and enumerations to be passed to internal functions.
  223  */
  224 enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI };
  225 
  226 /* Prototypes.. */
  227 
  228 static void *noobj_alloc(uma_zone_t, vm_size_t, uint8_t *, int);
  229 static void *page_alloc(uma_zone_t, vm_size_t, uint8_t *, int);
  230 static void *startup_alloc(uma_zone_t, vm_size_t, uint8_t *, int);
  231 static void page_free(void *, vm_size_t, uint8_t);
  232 static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int);
  233 static void cache_drain(uma_zone_t);
  234 static void bucket_drain(uma_zone_t, uma_bucket_t);
  235 static void bucket_cache_drain(uma_zone_t zone);
  236 static int keg_ctor(void *, int, void *, int);
  237 static void keg_dtor(void *, int, void *);
  238 static int zone_ctor(void *, int, void *, int);
  239 static void zone_dtor(void *, int, void *);
  240 static int zero_init(void *, int, int);
  241 static void keg_small_init(uma_keg_t keg);
  242 static void keg_large_init(uma_keg_t keg);
  243 static void zone_foreach(void (*zfunc)(uma_zone_t));
  244 static void zone_timeout(uma_zone_t zone);
  245 static int hash_alloc(struct uma_hash *);
  246 static int hash_expand(struct uma_hash *, struct uma_hash *);
  247 static void hash_free(struct uma_hash *hash);
  248 static void uma_timeout(void *);
  249 static void uma_startup3(void);
  250 static void *zone_alloc_item(uma_zone_t, void *, int);
  251 static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip);
  252 static void bucket_enable(void);
  253 static void bucket_init(void);
  254 static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int);
  255 static void bucket_free(uma_zone_t zone, uma_bucket_t, void *);
  256 static void bucket_zone_drain(void);
  257 static uma_bucket_t zone_alloc_bucket(uma_zone_t zone, void *, int flags);
  258 static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags);
  259 static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags);
  260 static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
  261 static void slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item);
  262 static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
  263     uma_fini fini, int align, uint32_t flags);
  264 static int zone_import(uma_zone_t zone, void **bucket, int max, int flags);
  265 static void zone_release(uma_zone_t zone, void **bucket, int cnt);
  266 static void uma_zero_item(void *item, uma_zone_t zone);
  267 
  268 void uma_print_zone(uma_zone_t);
  269 void uma_print_stats(void);
  270 static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS);
  271 static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS);
  272 
  273 #ifdef INVARIANTS
  274 static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item);
  275 static void uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item);
  276 #endif
  277 
  278 SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL);
  279 
  280 SYSCTL_PROC(_vm, OID_AUTO, zone_count, CTLFLAG_RD|CTLTYPE_INT,
  281     0, 0, sysctl_vm_zone_count, "I", "Number of UMA zones");
  282 
  283 SYSCTL_PROC(_vm, OID_AUTO, zone_stats, CTLFLAG_RD|CTLTYPE_STRUCT,
  284     0, 0, sysctl_vm_zone_stats, "s,struct uma_type_header", "Zone Stats");
  285 
  286 static int zone_warnings = 1;
  287 SYSCTL_INT(_vm, OID_AUTO, zone_warnings, CTLFLAG_RWTUN, &zone_warnings, 0,
  288     "Warn when UMA zones becomes full");
  289 
  290 /*
  291  * This routine checks to see whether or not it's safe to enable buckets.
  292  */
  293 static void
  294 bucket_enable(void)
  295 {
  296         bucketdisable = vm_page_count_min();
  297 }
  298 
  299 /*
  300  * Initialize bucket_zones, the array of zones of buckets of various sizes.
  301  *
  302  * For each zone, calculate the memory required for each bucket, consisting
  303  * of the header and an array of pointers.
  304  */
  305 static void
  306 bucket_init(void)
  307 {
  308         struct uma_bucket_zone *ubz;
  309         int size;
  310 
  311         for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) {
  312                 size = roundup(sizeof(struct uma_bucket), sizeof(void *));
  313                 size += sizeof(void *) * ubz->ubz_entries;
  314                 ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size,
  315                     NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
  316                     UMA_ZONE_MTXCLASS | UMA_ZFLAG_BUCKET);
  317         }
  318 }
  319 
  320 /*
  321  * Given a desired number of entries for a bucket, return the zone from which
  322  * to allocate the bucket.
  323  */
  324 static struct uma_bucket_zone *
  325 bucket_zone_lookup(int entries)
  326 {
  327         struct uma_bucket_zone *ubz;
  328 
  329         for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
  330                 if (ubz->ubz_entries >= entries)
  331                         return (ubz);
  332         ubz--;
  333         return (ubz);
  334 }
  335 
  336 static int
  337 bucket_select(int size)
  338 {
  339         struct uma_bucket_zone *ubz;
  340 
  341         ubz = &bucket_zones[0];
  342         if (size > ubz->ubz_maxsize)
  343                 return MAX((ubz->ubz_maxsize * ubz->ubz_entries) / size, 1);
  344 
  345         for (; ubz->ubz_entries != 0; ubz++)
  346                 if (ubz->ubz_maxsize < size)
  347                         break;
  348         ubz--;
  349         return (ubz->ubz_entries);
  350 }
  351 
  352 static uma_bucket_t
  353 bucket_alloc(uma_zone_t zone, void *udata, int flags)
  354 {
  355         struct uma_bucket_zone *ubz;
  356         uma_bucket_t bucket;
  357 
  358         /*
  359          * This is to stop us from allocating per cpu buckets while we're
  360          * running out of vm.boot_pages.  Otherwise, we would exhaust the
  361          * boot pages.  This also prevents us from allocating buckets in
  362          * low memory situations.
  363          */
  364         if (bucketdisable)
  365                 return (NULL);
  366         /*
  367          * To limit bucket recursion we store the original zone flags
  368          * in a cookie passed via zalloc_arg/zfree_arg.  This allows the
  369          * NOVM flag to persist even through deep recursions.  We also
  370          * store ZFLAG_BUCKET once we have recursed attempting to allocate
  371          * a bucket for a bucket zone so we do not allow infinite bucket
  372          * recursion.  This cookie will even persist to frees of unused
  373          * buckets via the allocation path or bucket allocations in the
  374          * free path.
  375          */
  376         if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0)
  377                 udata = (void *)(uintptr_t)zone->uz_flags;
  378         else {
  379                 if ((uintptr_t)udata & UMA_ZFLAG_BUCKET)
  380                         return (NULL);
  381                 udata = (void *)((uintptr_t)udata | UMA_ZFLAG_BUCKET);
  382         }
  383         if ((uintptr_t)udata & UMA_ZFLAG_CACHEONLY)
  384                 flags |= M_NOVM;
  385         ubz = bucket_zone_lookup(zone->uz_count);
  386         if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0)
  387                 ubz++;
  388         bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags);
  389         if (bucket) {
  390 #ifdef INVARIANTS
  391                 bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries);
  392 #endif
  393                 bucket->ub_cnt = 0;
  394                 bucket->ub_entries = ubz->ubz_entries;
  395         }
  396 
  397         return (bucket);
  398 }
  399 
  400 static void
  401 bucket_free(uma_zone_t zone, uma_bucket_t bucket, void *udata)
  402 {
  403         struct uma_bucket_zone *ubz;
  404 
  405         KASSERT(bucket->ub_cnt == 0,
  406             ("bucket_free: Freeing a non free bucket."));
  407         if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0)
  408                 udata = (void *)(uintptr_t)zone->uz_flags;
  409         ubz = bucket_zone_lookup(bucket->ub_entries);
  410         uma_zfree_arg(ubz->ubz_zone, bucket, udata);
  411 }
  412 
  413 static void
  414 bucket_zone_drain(void)
  415 {
  416         struct uma_bucket_zone *ubz;
  417 
  418         for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
  419                 zone_drain(ubz->ubz_zone);
  420 }
  421 
  422 static void
  423 zone_log_warning(uma_zone_t zone)
  424 {
  425         static const struct timeval warninterval = { 300, 0 };
  426 
  427         if (!zone_warnings || zone->uz_warning == NULL)
  428                 return;
  429 
  430         if (ratecheck(&zone->uz_ratecheck, &warninterval))
  431                 printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning);
  432 }
  433 
  434 static inline void
  435 zone_maxaction(uma_zone_t zone)
  436 {
  437 
  438         if (zone->uz_maxaction.ta_func != NULL)
  439                 taskqueue_enqueue(taskqueue_thread, &zone->uz_maxaction);
  440 }
  441 
  442 static void
  443 zone_foreach_keg(uma_zone_t zone, void (*kegfn)(uma_keg_t))
  444 {
  445         uma_klink_t klink;
  446 
  447         LIST_FOREACH(klink, &zone->uz_kegs, kl_link)
  448                 kegfn(klink->kl_keg);
  449 }
  450 
  451 /*
  452  * Routine called by timeout which is used to fire off some time interval
  453  * based calculations.  (stats, hash size, etc.)
  454  *
  455  * Arguments:
  456  *      arg   Unused
  457  *
  458  * Returns:
  459  *      Nothing
  460  */
  461 static void
  462 uma_timeout(void *unused)
  463 {
  464         bucket_enable();
  465         zone_foreach(zone_timeout);
  466 
  467         /* Reschedule this event */
  468         callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
  469 }
  470 
  471 /*
  472  * Routine to perform timeout driven calculations.  This expands the
  473  * hashes and does per cpu statistics aggregation.
  474  *
  475  *  Returns nothing.
  476  */
  477 static void
  478 keg_timeout(uma_keg_t keg)
  479 {
  480 
  481         KEG_LOCK(keg);
  482         /*
  483          * Expand the keg hash table.
  484          *
  485          * This is done if the number of slabs is larger than the hash size.
  486          * What I'm trying to do here is completely reduce collisions.  This
  487          * may be a little aggressive.  Should I allow for two collisions max?
  488          */
  489         if (keg->uk_flags & UMA_ZONE_HASH &&
  490             keg->uk_pages / keg->uk_ppera >= keg->uk_hash.uh_hashsize) {
  491                 struct uma_hash newhash;
  492                 struct uma_hash oldhash;
  493                 int ret;
  494 
  495                 /*
  496                  * This is so involved because allocating and freeing
  497                  * while the keg lock is held will lead to deadlock.
  498                  * I have to do everything in stages and check for
  499                  * races.
  500                  */
  501                 newhash = keg->uk_hash;
  502                 KEG_UNLOCK(keg);
  503                 ret = hash_alloc(&newhash);
  504                 KEG_LOCK(keg);
  505                 if (ret) {
  506                         if (hash_expand(&keg->uk_hash, &newhash)) {
  507                                 oldhash = keg->uk_hash;
  508                                 keg->uk_hash = newhash;
  509                         } else
  510                                 oldhash = newhash;
  511 
  512                         KEG_UNLOCK(keg);
  513                         hash_free(&oldhash);
  514                         return;
  515                 }
  516         }
  517         KEG_UNLOCK(keg);
  518 }
  519 
  520 static void
  521 zone_timeout(uma_zone_t zone)
  522 {
  523 
  524         zone_foreach_keg(zone, &keg_timeout);
  525 }
  526 
  527 /*
  528  * Allocate and zero fill the next sized hash table from the appropriate
  529  * backing store.
  530  *
  531  * Arguments:
  532  *      hash  A new hash structure with the old hash size in uh_hashsize
  533  *
  534  * Returns:
  535  *      1 on success and 0 on failure.
  536  */
  537 static int
  538 hash_alloc(struct uma_hash *hash)
  539 {
  540         int oldsize;
  541         int alloc;
  542 
  543         oldsize = hash->uh_hashsize;
  544 
  545         /* We're just going to go to a power of two greater */
  546         if (oldsize)  {
  547                 hash->uh_hashsize = oldsize * 2;
  548                 alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize;
  549                 hash->uh_slab_hash = (struct slabhead *)malloc(alloc,
  550                     M_UMAHASH, M_NOWAIT);
  551         } else {
  552                 alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT;
  553                 hash->uh_slab_hash = zone_alloc_item(hashzone, NULL,
  554                     M_WAITOK);
  555                 hash->uh_hashsize = UMA_HASH_SIZE_INIT;
  556         }
  557         if (hash->uh_slab_hash) {
  558                 bzero(hash->uh_slab_hash, alloc);
  559                 hash->uh_hashmask = hash->uh_hashsize - 1;
  560                 return (1);
  561         }
  562 
  563         return (0);
  564 }
  565 
  566 /*
  567  * Expands the hash table for HASH zones.  This is done from zone_timeout
  568  * to reduce collisions.  This must not be done in the regular allocation
  569  * path, otherwise, we can recurse on the vm while allocating pages.
  570  *
  571  * Arguments:
  572  *      oldhash  The hash you want to expand
  573  *      newhash  The hash structure for the new table
  574  *
  575  * Returns:
  576  *      Nothing
  577  *
  578  * Discussion:
  579  */
  580 static int
  581 hash_expand(struct uma_hash *oldhash, struct uma_hash *newhash)
  582 {
  583         uma_slab_t slab;
  584         int hval;
  585         int i;
  586 
  587         if (!newhash->uh_slab_hash)
  588                 return (0);
  589 
  590         if (oldhash->uh_hashsize >= newhash->uh_hashsize)
  591                 return (0);
  592 
  593         /*
  594          * I need to investigate hash algorithms for resizing without a
  595          * full rehash.
  596          */
  597 
  598         for (i = 0; i < oldhash->uh_hashsize; i++)
  599                 while (!SLIST_EMPTY(&oldhash->uh_slab_hash[i])) {
  600                         slab = SLIST_FIRST(&oldhash->uh_slab_hash[i]);
  601                         SLIST_REMOVE_HEAD(&oldhash->uh_slab_hash[i], us_hlink);
  602                         hval = UMA_HASH(newhash, slab->us_data);
  603                         SLIST_INSERT_HEAD(&newhash->uh_slab_hash[hval],
  604                             slab, us_hlink);
  605                 }
  606 
  607         return (1);
  608 }
  609 
  610 /*
  611  * Free the hash bucket to the appropriate backing store.
  612  *
  613  * Arguments:
  614  *      slab_hash  The hash bucket we're freeing
  615  *      hashsize   The number of entries in that hash bucket
  616  *
  617  * Returns:
  618  *      Nothing
  619  */
  620 static void
  621 hash_free(struct uma_hash *hash)
  622 {
  623         if (hash->uh_slab_hash == NULL)
  624                 return;
  625         if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
  626                 zone_free_item(hashzone, hash->uh_slab_hash, NULL, SKIP_NONE);
  627         else
  628                 free(hash->uh_slab_hash, M_UMAHASH);
  629 }
  630 
  631 /*
  632  * Frees all outstanding items in a bucket
  633  *
  634  * Arguments:
  635  *      zone   The zone to free to, must be unlocked.
  636  *      bucket The free/alloc bucket with items, cpu queue must be locked.
  637  *
  638  * Returns:
  639  *      Nothing
  640  */
  641 
  642 static void
  643 bucket_drain(uma_zone_t zone, uma_bucket_t bucket)
  644 {
  645         int i;
  646 
  647         if (bucket == NULL)
  648                 return;
  649 
  650         if (zone->uz_fini)
  651                 for (i = 0; i < bucket->ub_cnt; i++) 
  652                         zone->uz_fini(bucket->ub_bucket[i], zone->uz_size);
  653         zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt);
  654         bucket->ub_cnt = 0;
  655 }
  656 
  657 /*
  658  * Drains the per cpu caches for a zone.
  659  *
  660  * NOTE: This may only be called while the zone is being turn down, and not
  661  * during normal operation.  This is necessary in order that we do not have
  662  * to migrate CPUs to drain the per-CPU caches.
  663  *
  664  * Arguments:
  665  *      zone     The zone to drain, must be unlocked.
  666  *
  667  * Returns:
  668  *      Nothing
  669  */
  670 static void
  671 cache_drain(uma_zone_t zone)
  672 {
  673         uma_cache_t cache;
  674         int cpu;
  675 
  676         /*
  677          * XXX: It is safe to not lock the per-CPU caches, because we're
  678          * tearing down the zone anyway.  I.e., there will be no further use
  679          * of the caches at this point.
  680          *
  681          * XXX: It would good to be able to assert that the zone is being
  682          * torn down to prevent improper use of cache_drain().
  683          *
  684          * XXX: We lock the zone before passing into bucket_cache_drain() as
  685          * it is used elsewhere.  Should the tear-down path be made special
  686          * there in some form?
  687          */
  688         CPU_FOREACH(cpu) {
  689                 cache = &zone->uz_cpu[cpu];
  690                 bucket_drain(zone, cache->uc_allocbucket);
  691                 bucket_drain(zone, cache->uc_freebucket);
  692                 if (cache->uc_allocbucket != NULL)
  693                         bucket_free(zone, cache->uc_allocbucket, NULL);
  694                 if (cache->uc_freebucket != NULL)
  695                         bucket_free(zone, cache->uc_freebucket, NULL);
  696                 cache->uc_allocbucket = cache->uc_freebucket = NULL;
  697         }
  698         ZONE_LOCK(zone);
  699         bucket_cache_drain(zone);
  700         ZONE_UNLOCK(zone);
  701 }
  702 
  703 static void
  704 cache_shrink(uma_zone_t zone)
  705 {
  706 
  707         if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
  708                 return;
  709 
  710         ZONE_LOCK(zone);
  711         zone->uz_count = (zone->uz_count_min + zone->uz_count) / 2;
  712         ZONE_UNLOCK(zone);
  713 }
  714 
  715 static void
  716 cache_drain_safe_cpu(uma_zone_t zone)
  717 {
  718         uma_cache_t cache;
  719         uma_bucket_t b1, b2;
  720 
  721         if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
  722                 return;
  723 
  724         b1 = b2 = NULL;
  725         ZONE_LOCK(zone);
  726         critical_enter();
  727         cache = &zone->uz_cpu[curcpu];
  728         if (cache->uc_allocbucket) {
  729                 if (cache->uc_allocbucket->ub_cnt != 0)
  730                         LIST_INSERT_HEAD(&zone->uz_buckets,
  731                             cache->uc_allocbucket, ub_link);
  732                 else
  733                         b1 = cache->uc_allocbucket;
  734                 cache->uc_allocbucket = NULL;
  735         }
  736         if (cache->uc_freebucket) {
  737                 if (cache->uc_freebucket->ub_cnt != 0)
  738                         LIST_INSERT_HEAD(&zone->uz_buckets,
  739                             cache->uc_freebucket, ub_link);
  740                 else
  741                         b2 = cache->uc_freebucket;
  742                 cache->uc_freebucket = NULL;
  743         }
  744         critical_exit();
  745         ZONE_UNLOCK(zone);
  746         if (b1)
  747                 bucket_free(zone, b1, NULL);
  748         if (b2)
  749                 bucket_free(zone, b2, NULL);
  750 }
  751 
  752 /*
  753  * Safely drain per-CPU caches of a zone(s) to alloc bucket.
  754  * This is an expensive call because it needs to bind to all CPUs
  755  * one by one and enter a critical section on each of them in order
  756  * to safely access their cache buckets.
  757  * Zone lock must not be held on call this function.
  758  */
  759 static void
  760 cache_drain_safe(uma_zone_t zone)
  761 {
  762         int cpu;
  763 
  764         /*
  765          * Polite bucket sizes shrinking was not enouth, shrink aggressively.
  766          */
  767         if (zone)
  768                 cache_shrink(zone);
  769         else
  770                 zone_foreach(cache_shrink);
  771 
  772         CPU_FOREACH(cpu) {
  773                 thread_lock(curthread);
  774                 sched_bind(curthread, cpu);
  775                 thread_unlock(curthread);
  776 
  777                 if (zone)
  778                         cache_drain_safe_cpu(zone);
  779                 else
  780                         zone_foreach(cache_drain_safe_cpu);
  781         }
  782         thread_lock(curthread);
  783         sched_unbind(curthread);
  784         thread_unlock(curthread);
  785 }
  786 
  787 /*
  788  * Drain the cached buckets from a zone.  Expects a locked zone on entry.
  789  */
  790 static void
  791 bucket_cache_drain(uma_zone_t zone)
  792 {
  793         uma_bucket_t bucket;
  794 
  795         /*
  796          * Drain the bucket queues and free the buckets, we just keep two per
  797          * cpu (alloc/free).
  798          */
  799         while ((bucket = LIST_FIRST(&zone->uz_buckets)) != NULL) {
  800                 LIST_REMOVE(bucket, ub_link);
  801                 ZONE_UNLOCK(zone);
  802                 bucket_drain(zone, bucket);
  803                 bucket_free(zone, bucket, NULL);
  804                 ZONE_LOCK(zone);
  805         }
  806 
  807         /*
  808          * Shrink further bucket sizes.  Price of single zone lock collision
  809          * is probably lower then price of global cache drain.
  810          */
  811         if (zone->uz_count > zone->uz_count_min)
  812                 zone->uz_count--;
  813 }
  814 
  815 static void
  816 keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start)
  817 {
  818         uint8_t *mem;
  819         int i;
  820         uint8_t flags;
  821 
  822         mem = slab->us_data;
  823         flags = slab->us_flags;
  824         i = start;
  825         if (keg->uk_fini != NULL) {
  826                 for (i--; i > -1; i--)
  827                         keg->uk_fini(slab->us_data + (keg->uk_rsize * i),
  828                             keg->uk_size);
  829         }
  830         if (keg->uk_flags & UMA_ZONE_OFFPAGE)
  831                 zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
  832 #ifdef UMA_DEBUG
  833         printf("%s: Returning %d bytes.\n", keg->uk_name,
  834             PAGE_SIZE * keg->uk_ppera);
  835 #endif
  836         keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags);
  837 }
  838 
  839 /*
  840  * Frees pages from a keg back to the system.  This is done on demand from
  841  * the pageout daemon.
  842  *
  843  * Returns nothing.
  844  */
  845 static void
  846 keg_drain(uma_keg_t keg)
  847 {
  848         struct slabhead freeslabs = { 0 };
  849         uma_slab_t slab, tmp;
  850 
  851         /*
  852          * We don't want to take pages from statically allocated kegs at this
  853          * time
  854          */
  855         if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL)
  856                 return;
  857 
  858 #ifdef UMA_DEBUG
  859         printf("%s free items: %u\n", keg->uk_name, keg->uk_free);
  860 #endif
  861         KEG_LOCK(keg);
  862         if (keg->uk_free == 0)
  863                 goto finished;
  864 
  865         LIST_FOREACH_SAFE(slab, &keg->uk_free_slab, us_link, tmp) {
  866                 /* We have nowhere to free these to. */
  867                 if (slab->us_flags & UMA_SLAB_BOOT)
  868                         continue;
  869 
  870                 LIST_REMOVE(slab, us_link);
  871                 keg->uk_pages -= keg->uk_ppera;
  872                 keg->uk_free -= keg->uk_ipers;
  873 
  874                 if (keg->uk_flags & UMA_ZONE_HASH)
  875                         UMA_HASH_REMOVE(&keg->uk_hash, slab, slab->us_data);
  876 
  877                 SLIST_INSERT_HEAD(&freeslabs, slab, us_hlink);
  878         }
  879 finished:
  880         KEG_UNLOCK(keg);
  881 
  882         while ((slab = SLIST_FIRST(&freeslabs)) != NULL) {
  883                 SLIST_REMOVE(&freeslabs, slab, uma_slab, us_hlink);
  884                 keg_free_slab(keg, slab, keg->uk_ipers);
  885         }
  886 }
  887 
  888 static void
  889 zone_drain_wait(uma_zone_t zone, int waitok)
  890 {
  891 
  892         /*
  893          * Set draining to interlock with zone_dtor() so we can release our
  894          * locks as we go.  Only dtor() should do a WAITOK call since it
  895          * is the only call that knows the structure will still be available
  896          * when it wakes up.
  897          */
  898         ZONE_LOCK(zone);
  899         while (zone->uz_flags & UMA_ZFLAG_DRAINING) {
  900                 if (waitok == M_NOWAIT)
  901                         goto out;
  902                 msleep(zone, zone->uz_lockptr, PVM, "zonedrain", 1);
  903         }
  904         zone->uz_flags |= UMA_ZFLAG_DRAINING;
  905         bucket_cache_drain(zone);
  906         ZONE_UNLOCK(zone);
  907         /*
  908          * The DRAINING flag protects us from being freed while
  909          * we're running.  Normally the uma_rwlock would protect us but we
  910          * must be able to release and acquire the right lock for each keg.
  911          */
  912         zone_foreach_keg(zone, &keg_drain);
  913         ZONE_LOCK(zone);
  914         zone->uz_flags &= ~UMA_ZFLAG_DRAINING;
  915         wakeup(zone);
  916 out:
  917         ZONE_UNLOCK(zone);
  918 }
  919 
  920 void
  921 zone_drain(uma_zone_t zone)
  922 {
  923 
  924         zone_drain_wait(zone, M_NOWAIT);
  925 }
  926 
  927 /*
  928  * Allocate a new slab for a keg.  This does not insert the slab onto a list.
  929  *
  930  * Arguments:
  931  *      wait  Shall we wait?
  932  *
  933  * Returns:
  934  *      The slab that was allocated or NULL if there is no memory and the
  935  *      caller specified M_NOWAIT.
  936  */
  937 static uma_slab_t
  938 keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait)
  939 {
  940         uma_alloc allocf;
  941         uma_slab_t slab;
  942         uint8_t *mem;
  943         uint8_t flags;
  944         int i;
  945 
  946         mtx_assert(&keg->uk_lock, MA_OWNED);
  947         slab = NULL;
  948         mem = NULL;
  949 
  950 #ifdef UMA_DEBUG
  951         printf("alloc_slab:  Allocating a new slab for %s\n", keg->uk_name);
  952 #endif
  953         allocf = keg->uk_allocf;
  954         KEG_UNLOCK(keg);
  955 
  956         if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
  957                 slab = zone_alloc_item(keg->uk_slabzone, NULL, wait);
  958                 if (slab == NULL)
  959                         goto out;
  960         }
  961 
  962         /*
  963          * This reproduces the old vm_zone behavior of zero filling pages the
  964          * first time they are added to a zone.
  965          *
  966          * Malloced items are zeroed in uma_zalloc.
  967          */
  968 
  969         if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0)
  970                 wait |= M_ZERO;
  971         else
  972                 wait &= ~M_ZERO;
  973 
  974         if (keg->uk_flags & UMA_ZONE_NODUMP)
  975                 wait |= M_NODUMP;
  976 
  977         /* zone is passed for legacy reasons. */
  978         mem = allocf(zone, keg->uk_ppera * PAGE_SIZE, &flags, wait);
  979         if (mem == NULL) {
  980                 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
  981                         zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
  982                 slab = NULL;
  983                 goto out;
  984         }
  985 
  986         /* Point the slab into the allocated memory */
  987         if (!(keg->uk_flags & UMA_ZONE_OFFPAGE))
  988                 slab = (uma_slab_t )(mem + keg->uk_pgoff);
  989 
  990         if (keg->uk_flags & UMA_ZONE_VTOSLAB)
  991                 for (i = 0; i < keg->uk_ppera; i++)
  992                         vsetslab((vm_offset_t)mem + (i * PAGE_SIZE), slab);
  993 
  994         slab->us_keg = keg;
  995         slab->us_data = mem;
  996         slab->us_freecount = keg->uk_ipers;
  997         slab->us_flags = flags;
  998         BIT_FILL(SLAB_SETSIZE, &slab->us_free);
  999 #ifdef INVARIANTS
 1000         BIT_ZERO(SLAB_SETSIZE, &slab->us_debugfree);
 1001 #endif
 1002 
 1003         if (keg->uk_init != NULL) {
 1004                 for (i = 0; i < keg->uk_ipers; i++)
 1005                         if (keg->uk_init(slab->us_data + (keg->uk_rsize * i),
 1006                             keg->uk_size, wait) != 0)
 1007                                 break;
 1008                 if (i != keg->uk_ipers) {
 1009                         keg_free_slab(keg, slab, i);
 1010                         slab = NULL;
 1011                         goto out;
 1012                 }
 1013         }
 1014 out:
 1015         KEG_LOCK(keg);
 1016 
 1017         if (slab != NULL) {
 1018                 if (keg->uk_flags & UMA_ZONE_HASH)
 1019                         UMA_HASH_INSERT(&keg->uk_hash, slab, mem);
 1020 
 1021                 keg->uk_pages += keg->uk_ppera;
 1022                 keg->uk_free += keg->uk_ipers;
 1023         }
 1024 
 1025         return (slab);
 1026 }
 1027 
 1028 /*
 1029  * This function is intended to be used early on in place of page_alloc() so
 1030  * that we may use the boot time page cache to satisfy allocations before
 1031  * the VM is ready.
 1032  */
 1033 static void *
 1034 startup_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *pflag, int wait)
 1035 {
 1036         uma_keg_t keg;
 1037         uma_slab_t tmps;
 1038         int pages, check_pages;
 1039 
 1040         keg = zone_first_keg(zone);
 1041         pages = howmany(bytes, PAGE_SIZE);
 1042         check_pages = pages - 1;
 1043         KASSERT(pages > 0, ("startup_alloc can't reserve 0 pages\n"));
 1044 
 1045         /*
 1046          * Check our small startup cache to see if it has pages remaining.
 1047          */
 1048         mtx_lock(&uma_boot_pages_mtx);
 1049 
 1050         /* First check if we have enough room. */
 1051         tmps = LIST_FIRST(&uma_boot_pages);
 1052         while (tmps != NULL && check_pages-- > 0)
 1053                 tmps = LIST_NEXT(tmps, us_link);
 1054         if (tmps != NULL) {
 1055                 /*
 1056                  * It's ok to lose tmps references.  The last one will
 1057                  * have tmps->us_data pointing to the start address of
 1058                  * "pages" contiguous pages of memory.
 1059                  */
 1060                 while (pages-- > 0) {
 1061                         tmps = LIST_FIRST(&uma_boot_pages);
 1062                         LIST_REMOVE(tmps, us_link);
 1063                 }
 1064                 mtx_unlock(&uma_boot_pages_mtx);
 1065                 *pflag = tmps->us_flags;
 1066                 return (tmps->us_data);
 1067         }
 1068         mtx_unlock(&uma_boot_pages_mtx);
 1069         if (booted < UMA_STARTUP2)
 1070                 panic("UMA: Increase vm.boot_pages");
 1071         /*
 1072          * Now that we've booted reset these users to their real allocator.
 1073          */
 1074 #ifdef UMA_MD_SMALL_ALLOC
 1075         keg->uk_allocf = (keg->uk_ppera > 1) ? page_alloc : uma_small_alloc;
 1076 #else
 1077         keg->uk_allocf = page_alloc;
 1078 #endif
 1079         return keg->uk_allocf(zone, bytes, pflag, wait);
 1080 }
 1081 
 1082 /*
 1083  * Allocates a number of pages from the system
 1084  *
 1085  * Arguments:
 1086  *      bytes  The number of bytes requested
 1087  *      wait  Shall we wait?
 1088  *
 1089  * Returns:
 1090  *      A pointer to the alloced memory or possibly
 1091  *      NULL if M_NOWAIT is set.
 1092  */
 1093 static void *
 1094 page_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *pflag, int wait)
 1095 {
 1096         void *p;        /* Returned page */
 1097 
 1098         *pflag = UMA_SLAB_KMEM;
 1099         p = (void *) kmem_malloc(kmem_arena, bytes, wait);
 1100 
 1101         return (p);
 1102 }
 1103 
 1104 /*
 1105  * Allocates a number of pages from within an object
 1106  *
 1107  * Arguments:
 1108  *      bytes  The number of bytes requested
 1109  *      wait   Shall we wait?
 1110  *
 1111  * Returns:
 1112  *      A pointer to the alloced memory or possibly
 1113  *      NULL if M_NOWAIT is set.
 1114  */
 1115 static void *
 1116 noobj_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *flags, int wait)
 1117 {
 1118         TAILQ_HEAD(, vm_page) alloctail;
 1119         u_long npages;
 1120         vm_offset_t retkva, zkva;
 1121         vm_page_t p, p_next;
 1122         uma_keg_t keg;
 1123 
 1124         TAILQ_INIT(&alloctail);
 1125         keg = zone_first_keg(zone);
 1126 
 1127         npages = howmany(bytes, PAGE_SIZE);
 1128         while (npages > 0) {
 1129                 p = vm_page_alloc(NULL, 0, VM_ALLOC_INTERRUPT |
 1130                     VM_ALLOC_WIRED | VM_ALLOC_NOOBJ);
 1131                 if (p != NULL) {
 1132                         /*
 1133                          * Since the page does not belong to an object, its
 1134                          * listq is unused.
 1135                          */
 1136                         TAILQ_INSERT_TAIL(&alloctail, p, listq);
 1137                         npages--;
 1138                         continue;
 1139                 }
 1140                 if (wait & M_WAITOK) {
 1141                         VM_WAIT;
 1142                         continue;
 1143                 }
 1144 
 1145                 /*
 1146                  * Page allocation failed, free intermediate pages and
 1147                  * exit.
 1148                  */
 1149                 TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) {
 1150                         vm_page_unwire(p, PQ_NONE);
 1151                         vm_page_free(p); 
 1152                 }
 1153                 return (NULL);
 1154         }
 1155         *flags = UMA_SLAB_PRIV;
 1156         zkva = keg->uk_kva +
 1157             atomic_fetchadd_long(&keg->uk_offset, round_page(bytes));
 1158         retkva = zkva;
 1159         TAILQ_FOREACH(p, &alloctail, listq) {
 1160                 pmap_qenter(zkva, &p, 1);
 1161                 zkva += PAGE_SIZE;
 1162         }
 1163 
 1164         return ((void *)retkva);
 1165 }
 1166 
 1167 /*
 1168  * Frees a number of pages to the system
 1169  *
 1170  * Arguments:
 1171  *      mem   A pointer to the memory to be freed
 1172  *      size  The size of the memory being freed
 1173  *      flags The original p->us_flags field
 1174  *
 1175  * Returns:
 1176  *      Nothing
 1177  */
 1178 static void
 1179 page_free(void *mem, vm_size_t size, uint8_t flags)
 1180 {
 1181         struct vmem *vmem;
 1182 
 1183         if (flags & UMA_SLAB_KMEM)
 1184                 vmem = kmem_arena;
 1185         else if (flags & UMA_SLAB_KERNEL)
 1186                 vmem = kernel_arena;
 1187         else
 1188                 panic("UMA: page_free used with invalid flags %d", flags);
 1189 
 1190         kmem_free(vmem, (vm_offset_t)mem, size);
 1191 }
 1192 
 1193 /*
 1194  * Zero fill initializer
 1195  *
 1196  * Arguments/Returns follow uma_init specifications
 1197  */
 1198 static int
 1199 zero_init(void *mem, int size, int flags)
 1200 {
 1201         bzero(mem, size);
 1202         return (0);
 1203 }
 1204 
 1205 /*
 1206  * Finish creating a small uma keg.  This calculates ipers, and the keg size.
 1207  *
 1208  * Arguments
 1209  *      keg  The zone we should initialize
 1210  *
 1211  * Returns
 1212  *      Nothing
 1213  */
 1214 static void
 1215 keg_small_init(uma_keg_t keg)
 1216 {
 1217         u_int rsize;
 1218         u_int memused;
 1219         u_int wastedspace;
 1220         u_int shsize;
 1221         u_int slabsize;
 1222 
 1223         if (keg->uk_flags & UMA_ZONE_PCPU) {
 1224                 u_int ncpus = (mp_maxid + 1) ? (mp_maxid + 1) : MAXCPU;
 1225 
 1226                 slabsize = sizeof(struct pcpu);
 1227                 keg->uk_ppera = howmany(ncpus * sizeof(struct pcpu),
 1228                     PAGE_SIZE);
 1229         } else {
 1230                 slabsize = UMA_SLAB_SIZE;
 1231                 keg->uk_ppera = 1;
 1232         }
 1233 
 1234         /*
 1235          * Calculate the size of each allocation (rsize) according to
 1236          * alignment.  If the requested size is smaller than we have
 1237          * allocation bits for we round it up.
 1238          */
 1239         rsize = keg->uk_size;
 1240         if (rsize < slabsize / SLAB_SETSIZE)
 1241                 rsize = slabsize / SLAB_SETSIZE;
 1242         if (rsize & keg->uk_align)
 1243                 rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1);
 1244         keg->uk_rsize = rsize;
 1245 
 1246         KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 ||
 1247             keg->uk_rsize < sizeof(struct pcpu),
 1248             ("%s: size %u too large", __func__, keg->uk_rsize));
 1249 
 1250         if (keg->uk_flags & UMA_ZONE_OFFPAGE)
 1251                 shsize = 0;
 1252         else 
 1253                 shsize = sizeof(struct uma_slab);
 1254 
 1255         keg->uk_ipers = (slabsize - shsize) / rsize;
 1256         KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE,
 1257             ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
 1258 
 1259         memused = keg->uk_ipers * rsize + shsize;
 1260         wastedspace = slabsize - memused;
 1261 
 1262         /*
 1263          * We can't do OFFPAGE if we're internal or if we've been
 1264          * asked to not go to the VM for buckets.  If we do this we
 1265          * may end up going to the VM  for slabs which we do not
 1266          * want to do if we're UMA_ZFLAG_CACHEONLY as a result
 1267          * of UMA_ZONE_VM, which clearly forbids it.
 1268          */
 1269         if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) ||
 1270             (keg->uk_flags & UMA_ZFLAG_CACHEONLY))
 1271                 return;
 1272 
 1273         /*
 1274          * See if using an OFFPAGE slab will limit our waste.  Only do
 1275          * this if it permits more items per-slab.
 1276          *
 1277          * XXX We could try growing slabsize to limit max waste as well.
 1278          * Historically this was not done because the VM could not
 1279          * efficiently handle contiguous allocations.
 1280          */
 1281         if ((wastedspace >= slabsize / UMA_MAX_WASTE) &&
 1282             (keg->uk_ipers < (slabsize / keg->uk_rsize))) {
 1283                 keg->uk_ipers = slabsize / keg->uk_rsize;
 1284                 KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE,
 1285                     ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
 1286 #ifdef UMA_DEBUG
 1287                 printf("UMA decided we need offpage slab headers for "
 1288                     "keg: %s, calculated wastedspace = %d, "
 1289                     "maximum wasted space allowed = %d, "
 1290                     "calculated ipers = %d, "
 1291                     "new wasted space = %d\n", keg->uk_name, wastedspace,
 1292                     slabsize / UMA_MAX_WASTE, keg->uk_ipers,
 1293                     slabsize - keg->uk_ipers * keg->uk_rsize);
 1294 #endif
 1295                 keg->uk_flags |= UMA_ZONE_OFFPAGE;
 1296         }
 1297 
 1298         if ((keg->uk_flags & UMA_ZONE_OFFPAGE) &&
 1299             (keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
 1300                 keg->uk_flags |= UMA_ZONE_HASH;
 1301 }
 1302 
 1303 /*
 1304  * Finish creating a large (> UMA_SLAB_SIZE) uma kegs.  Just give in and do
 1305  * OFFPAGE for now.  When I can allow for more dynamic slab sizes this will be
 1306  * more complicated.
 1307  *
 1308  * Arguments
 1309  *      keg  The keg we should initialize
 1310  *
 1311  * Returns
 1312  *      Nothing
 1313  */
 1314 static void
 1315 keg_large_init(uma_keg_t keg)
 1316 {
 1317         u_int shsize;
 1318 
 1319         KASSERT(keg != NULL, ("Keg is null in keg_large_init"));
 1320         KASSERT((keg->uk_flags & UMA_ZFLAG_CACHEONLY) == 0,
 1321             ("keg_large_init: Cannot large-init a UMA_ZFLAG_CACHEONLY keg"));
 1322         KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0,
 1323             ("%s: Cannot large-init a UMA_ZONE_PCPU keg", __func__));
 1324 
 1325         keg->uk_ppera = howmany(keg->uk_size, PAGE_SIZE);
 1326         keg->uk_ipers = 1;
 1327         keg->uk_rsize = keg->uk_size;
 1328 
 1329         /* We can't do OFFPAGE if we're internal, bail out here. */
 1330         if (keg->uk_flags & UMA_ZFLAG_INTERNAL)
 1331                 return;
 1332 
 1333         /* Check whether we have enough space to not do OFFPAGE. */
 1334         if ((keg->uk_flags & UMA_ZONE_OFFPAGE) == 0) {
 1335                 shsize = sizeof(struct uma_slab);
 1336                 if (shsize & UMA_ALIGN_PTR)
 1337                         shsize = (shsize & ~UMA_ALIGN_PTR) +
 1338                             (UMA_ALIGN_PTR + 1);
 1339 
 1340                 if ((PAGE_SIZE * keg->uk_ppera) - keg->uk_rsize < shsize)
 1341                         keg->uk_flags |= UMA_ZONE_OFFPAGE;
 1342         }
 1343 
 1344         if ((keg->uk_flags & UMA_ZONE_OFFPAGE) &&
 1345             (keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
 1346                 keg->uk_flags |= UMA_ZONE_HASH;
 1347 }
 1348 
 1349 static void
 1350 keg_cachespread_init(uma_keg_t keg)
 1351 {
 1352         int alignsize;
 1353         int trailer;
 1354         int pages;
 1355         int rsize;
 1356 
 1357         KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0,
 1358             ("%s: Cannot cachespread-init a UMA_ZONE_PCPU keg", __func__));
 1359 
 1360         alignsize = keg->uk_align + 1;
 1361         rsize = keg->uk_size;
 1362         /*
 1363          * We want one item to start on every align boundary in a page.  To
 1364          * do this we will span pages.  We will also extend the item by the
 1365          * size of align if it is an even multiple of align.  Otherwise, it
 1366          * would fall on the same boundary every time.
 1367          */
 1368         if (rsize & keg->uk_align)
 1369                 rsize = (rsize & ~keg->uk_align) + alignsize;
 1370         if ((rsize & alignsize) == 0)
 1371                 rsize += alignsize;
 1372         trailer = rsize - keg->uk_size;
 1373         pages = (rsize * (PAGE_SIZE / alignsize)) / PAGE_SIZE;
 1374         pages = MIN(pages, (128 * 1024) / PAGE_SIZE);
 1375         keg->uk_rsize = rsize;
 1376         keg->uk_ppera = pages;
 1377         keg->uk_ipers = ((pages * PAGE_SIZE) + trailer) / rsize;
 1378         keg->uk_flags |= UMA_ZONE_OFFPAGE | UMA_ZONE_VTOSLAB;
 1379         KASSERT(keg->uk_ipers <= SLAB_SETSIZE,
 1380             ("%s: keg->uk_ipers too high(%d) increase max_ipers", __func__,
 1381             keg->uk_ipers));
 1382 }
 1383 
 1384 /*
 1385  * Keg header ctor.  This initializes all fields, locks, etc.  And inserts
 1386  * the keg onto the global keg list.
 1387  *
 1388  * Arguments/Returns follow uma_ctor specifications
 1389  *      udata  Actually uma_kctor_args
 1390  */
 1391 static int
 1392 keg_ctor(void *mem, int size, void *udata, int flags)
 1393 {
 1394         struct uma_kctor_args *arg = udata;
 1395         uma_keg_t keg = mem;
 1396         uma_zone_t zone;
 1397 
 1398         bzero(keg, size);
 1399         keg->uk_size = arg->size;
 1400         keg->uk_init = arg->uminit;
 1401         keg->uk_fini = arg->fini;
 1402         keg->uk_align = arg->align;
 1403         keg->uk_free = 0;
 1404         keg->uk_reserve = 0;
 1405         keg->uk_pages = 0;
 1406         keg->uk_flags = arg->flags;
 1407         keg->uk_allocf = page_alloc;
 1408         keg->uk_freef = page_free;
 1409         keg->uk_slabzone = NULL;
 1410 
 1411         /*
 1412          * The master zone is passed to us at keg-creation time.
 1413          */
 1414         zone = arg->zone;
 1415         keg->uk_name = zone->uz_name;
 1416 
 1417         if (arg->flags & UMA_ZONE_VM)
 1418                 keg->uk_flags |= UMA_ZFLAG_CACHEONLY;
 1419 
 1420         if (arg->flags & UMA_ZONE_ZINIT)
 1421                 keg->uk_init = zero_init;
 1422 
 1423         if (arg->flags & UMA_ZONE_MALLOC)
 1424                 keg->uk_flags |= UMA_ZONE_VTOSLAB;
 1425 
 1426         if (arg->flags & UMA_ZONE_PCPU)
 1427 #ifdef SMP
 1428                 keg->uk_flags |= UMA_ZONE_OFFPAGE;
 1429 #else
 1430                 keg->uk_flags &= ~UMA_ZONE_PCPU;
 1431 #endif
 1432 
 1433         if (keg->uk_flags & UMA_ZONE_CACHESPREAD) {
 1434                 keg_cachespread_init(keg);
 1435         } else {
 1436                 if (keg->uk_size > (UMA_SLAB_SIZE - sizeof(struct uma_slab)))
 1437                         keg_large_init(keg);
 1438                 else
 1439                         keg_small_init(keg);
 1440         }
 1441 
 1442         if (keg->uk_flags & UMA_ZONE_OFFPAGE)
 1443                 keg->uk_slabzone = slabzone;
 1444 
 1445         /*
 1446          * If we haven't booted yet we need allocations to go through the
 1447          * startup cache until the vm is ready.
 1448          */
 1449         if (keg->uk_ppera == 1) {
 1450 #ifdef UMA_MD_SMALL_ALLOC
 1451                 keg->uk_allocf = uma_small_alloc;
 1452                 keg->uk_freef = uma_small_free;
 1453 
 1454                 if (booted < UMA_STARTUP)
 1455                         keg->uk_allocf = startup_alloc;
 1456 #else
 1457                 if (booted < UMA_STARTUP2)
 1458                         keg->uk_allocf = startup_alloc;
 1459 #endif
 1460         } else if (booted < UMA_STARTUP2 &&
 1461             (keg->uk_flags & UMA_ZFLAG_INTERNAL))
 1462                 keg->uk_allocf = startup_alloc;
 1463 
 1464         /*
 1465          * Initialize keg's lock
 1466          */
 1467         KEG_LOCK_INIT(keg, (arg->flags & UMA_ZONE_MTXCLASS));
 1468 
 1469         /*
 1470          * If we're putting the slab header in the actual page we need to
 1471          * figure out where in each page it goes.  This calculates a right
 1472          * justified offset into the memory on an ALIGN_PTR boundary.
 1473          */
 1474         if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) {
 1475                 u_int totsize;
 1476 
 1477                 /* Size of the slab struct and free list */
 1478                 totsize = sizeof(struct uma_slab);
 1479 
 1480                 if (totsize & UMA_ALIGN_PTR)
 1481                         totsize = (totsize & ~UMA_ALIGN_PTR) +
 1482                             (UMA_ALIGN_PTR + 1);
 1483                 keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - totsize;
 1484 
 1485                 /*
 1486                  * The only way the following is possible is if with our
 1487                  * UMA_ALIGN_PTR adjustments we are now bigger than
 1488                  * UMA_SLAB_SIZE.  I haven't checked whether this is
 1489                  * mathematically possible for all cases, so we make
 1490                  * sure here anyway.
 1491                  */
 1492                 totsize = keg->uk_pgoff + sizeof(struct uma_slab);
 1493                 if (totsize > PAGE_SIZE * keg->uk_ppera) {
 1494                         printf("zone %s ipers %d rsize %d size %d\n",
 1495                             zone->uz_name, keg->uk_ipers, keg->uk_rsize,
 1496                             keg->uk_size);
 1497                         panic("UMA slab won't fit.");
 1498                 }
 1499         }
 1500 
 1501         if (keg->uk_flags & UMA_ZONE_HASH)
 1502                 hash_alloc(&keg->uk_hash);
 1503 
 1504 #ifdef UMA_DEBUG
 1505         printf("UMA: %s(%p) size %d(%d) flags %#x ipers %d ppera %d out %d free %d\n",
 1506             zone->uz_name, zone, keg->uk_size, keg->uk_rsize, keg->uk_flags,
 1507             keg->uk_ipers, keg->uk_ppera,
 1508             (keg->uk_pages / keg->uk_ppera) * keg->uk_ipers - keg->uk_free,
 1509             keg->uk_free);
 1510 #endif
 1511 
 1512         LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link);
 1513 
 1514         rw_wlock(&uma_rwlock);
 1515         LIST_INSERT_HEAD(&uma_kegs, keg, uk_link);
 1516         rw_wunlock(&uma_rwlock);
 1517         return (0);
 1518 }
 1519 
 1520 /*
 1521  * Zone header ctor.  This initializes all fields, locks, etc.
 1522  *
 1523  * Arguments/Returns follow uma_ctor specifications
 1524  *      udata  Actually uma_zctor_args
 1525  */
 1526 static int
 1527 zone_ctor(void *mem, int size, void *udata, int flags)
 1528 {
 1529         struct uma_zctor_args *arg = udata;
 1530         uma_zone_t zone = mem;
 1531         uma_zone_t z;
 1532         uma_keg_t keg;
 1533 
 1534         bzero(zone, size);
 1535         zone->uz_name = arg->name;
 1536         zone->uz_ctor = arg->ctor;
 1537         zone->uz_dtor = arg->dtor;
 1538         zone->uz_slab = zone_fetch_slab;
 1539         zone->uz_init = NULL;
 1540         zone->uz_fini = NULL;
 1541         zone->uz_allocs = 0;
 1542         zone->uz_frees = 0;
 1543         zone->uz_fails = 0;
 1544         zone->uz_sleeps = 0;
 1545         zone->uz_count = 0;
 1546         zone->uz_count_min = 0;
 1547         zone->uz_flags = 0;
 1548         zone->uz_warning = NULL;
 1549         timevalclear(&zone->uz_ratecheck);
 1550         keg = arg->keg;
 1551 
 1552         ZONE_LOCK_INIT(zone, (arg->flags & UMA_ZONE_MTXCLASS));
 1553 
 1554         /*
 1555          * This is a pure cache zone, no kegs.
 1556          */
 1557         if (arg->import) {
 1558                 if (arg->flags & UMA_ZONE_VM)
 1559                         arg->flags |= UMA_ZFLAG_CACHEONLY;
 1560                 zone->uz_flags = arg->flags;
 1561                 zone->uz_size = arg->size;
 1562                 zone->uz_import = arg->import;
 1563                 zone->uz_release = arg->release;
 1564                 zone->uz_arg = arg->arg;
 1565                 zone->uz_lockptr = &zone->uz_lock;
 1566                 rw_wlock(&uma_rwlock);
 1567                 LIST_INSERT_HEAD(&uma_cachezones, zone, uz_link);
 1568                 rw_wunlock(&uma_rwlock);
 1569                 goto out;
 1570         }
 1571 
 1572         /*
 1573          * Use the regular zone/keg/slab allocator.
 1574          */
 1575         zone->uz_import = (uma_import)zone_import;
 1576         zone->uz_release = (uma_release)zone_release;
 1577         zone->uz_arg = zone; 
 1578 
 1579         if (arg->flags & UMA_ZONE_SECONDARY) {
 1580                 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
 1581                 zone->uz_init = arg->uminit;
 1582                 zone->uz_fini = arg->fini;
 1583                 zone->uz_lockptr = &keg->uk_lock;
 1584                 zone->uz_flags |= UMA_ZONE_SECONDARY;
 1585                 rw_wlock(&uma_rwlock);
 1586                 ZONE_LOCK(zone);
 1587                 LIST_FOREACH(z, &keg->uk_zones, uz_link) {
 1588                         if (LIST_NEXT(z, uz_link) == NULL) {
 1589                                 LIST_INSERT_AFTER(z, zone, uz_link);
 1590                                 break;
 1591                         }
 1592                 }
 1593                 ZONE_UNLOCK(zone);
 1594                 rw_wunlock(&uma_rwlock);
 1595         } else if (keg == NULL) {
 1596                 if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini,
 1597                     arg->align, arg->flags)) == NULL)
 1598                         return (ENOMEM);
 1599         } else {
 1600                 struct uma_kctor_args karg;
 1601                 int error;
 1602 
 1603                 /* We should only be here from uma_startup() */
 1604                 karg.size = arg->size;
 1605                 karg.uminit = arg->uminit;
 1606                 karg.fini = arg->fini;
 1607                 karg.align = arg->align;
 1608                 karg.flags = arg->flags;
 1609                 karg.zone = zone;
 1610                 error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg,
 1611                     flags);
 1612                 if (error)
 1613                         return (error);
 1614         }
 1615 
 1616         /*
 1617          * Link in the first keg.
 1618          */
 1619         zone->uz_klink.kl_keg = keg;
 1620         LIST_INSERT_HEAD(&zone->uz_kegs, &zone->uz_klink, kl_link);
 1621         zone->uz_lockptr = &keg->uk_lock;
 1622         zone->uz_size = keg->uk_size;
 1623         zone->uz_flags |= (keg->uk_flags &
 1624             (UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT));
 1625 
 1626         /*
 1627          * Some internal zones don't have room allocated for the per cpu
 1628          * caches.  If we're internal, bail out here.
 1629          */
 1630         if (keg->uk_flags & UMA_ZFLAG_INTERNAL) {
 1631                 KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
 1632                     ("Secondary zone requested UMA_ZFLAG_INTERNAL"));
 1633                 return (0);
 1634         }
 1635 
 1636 out:
 1637         if ((arg->flags & UMA_ZONE_MAXBUCKET) == 0)
 1638                 zone->uz_count = bucket_select(zone->uz_size);
 1639         else
 1640                 zone->uz_count = BUCKET_MAX;
 1641         zone->uz_count_min = zone->uz_count;
 1642 
 1643         return (0);
 1644 }
 1645 
 1646 /*
 1647  * Keg header dtor.  This frees all data, destroys locks, frees the hash
 1648  * table and removes the keg from the global list.
 1649  *
 1650  * Arguments/Returns follow uma_dtor specifications
 1651  *      udata  unused
 1652  */
 1653 static void
 1654 keg_dtor(void *arg, int size, void *udata)
 1655 {
 1656         uma_keg_t keg;
 1657 
 1658         keg = (uma_keg_t)arg;
 1659         KEG_LOCK(keg);
 1660         if (keg->uk_free != 0) {
 1661                 printf("Freed UMA keg (%s) was not empty (%d items). "
 1662                     " Lost %d pages of memory.\n",
 1663                     keg->uk_name ? keg->uk_name : "",
 1664                     keg->uk_free, keg->uk_pages);
 1665         }
 1666         KEG_UNLOCK(keg);
 1667 
 1668         hash_free(&keg->uk_hash);
 1669 
 1670         KEG_LOCK_FINI(keg);
 1671 }
 1672 
 1673 /*
 1674  * Zone header dtor.
 1675  *
 1676  * Arguments/Returns follow uma_dtor specifications
 1677  *      udata  unused
 1678  */
 1679 static void
 1680 zone_dtor(void *arg, int size, void *udata)
 1681 {
 1682         uma_klink_t klink;
 1683         uma_zone_t zone;
 1684         uma_keg_t keg;
 1685 
 1686         zone = (uma_zone_t)arg;
 1687         keg = zone_first_keg(zone);
 1688 
 1689         if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL))
 1690                 cache_drain(zone);
 1691 
 1692         rw_wlock(&uma_rwlock);
 1693         LIST_REMOVE(zone, uz_link);
 1694         rw_wunlock(&uma_rwlock);
 1695         /*
 1696          * XXX there are some races here where
 1697          * the zone can be drained but zone lock
 1698          * released and then refilled before we
 1699          * remove it... we dont care for now
 1700          */
 1701         zone_drain_wait(zone, M_WAITOK);
 1702         /*
 1703          * Unlink all of our kegs.
 1704          */
 1705         while ((klink = LIST_FIRST(&zone->uz_kegs)) != NULL) {
 1706                 klink->kl_keg = NULL;
 1707                 LIST_REMOVE(klink, kl_link);
 1708                 if (klink == &zone->uz_klink)
 1709                         continue;
 1710                 free(klink, M_TEMP);
 1711         }
 1712         /*
 1713          * We only destroy kegs from non secondary zones.
 1714          */
 1715         if (keg != NULL && (zone->uz_flags & UMA_ZONE_SECONDARY) == 0)  {
 1716                 rw_wlock(&uma_rwlock);
 1717                 LIST_REMOVE(keg, uk_link);
 1718                 rw_wunlock(&uma_rwlock);
 1719                 zone_free_item(kegs, keg, NULL, SKIP_NONE);
 1720         }
 1721         ZONE_LOCK_FINI(zone);
 1722 }
 1723 
 1724 /*
 1725  * Traverses every zone in the system and calls a callback
 1726  *
 1727  * Arguments:
 1728  *      zfunc  A pointer to a function which accepts a zone
 1729  *              as an argument.
 1730  *
 1731  * Returns:
 1732  *      Nothing
 1733  */
 1734 static void
 1735 zone_foreach(void (*zfunc)(uma_zone_t))
 1736 {
 1737         uma_keg_t keg;
 1738         uma_zone_t zone;
 1739 
 1740         rw_rlock(&uma_rwlock);
 1741         LIST_FOREACH(keg, &uma_kegs, uk_link) {
 1742                 LIST_FOREACH(zone, &keg->uk_zones, uz_link)
 1743                         zfunc(zone);
 1744         }
 1745         rw_runlock(&uma_rwlock);
 1746 }
 1747 
 1748 /* Public functions */
 1749 /* See uma.h */
 1750 void
 1751 uma_startup(void *bootmem, int boot_pages)
 1752 {
 1753         struct uma_zctor_args args;
 1754         uma_slab_t slab;
 1755         int i;
 1756 
 1757 #ifdef UMA_DEBUG
 1758         printf("Creating uma keg headers zone and keg.\n");
 1759 #endif
 1760         rw_init(&uma_rwlock, "UMA lock");
 1761 
 1762         /* "manually" create the initial zone */
 1763         memset(&args, 0, sizeof(args));
 1764         args.name = "UMA Kegs";
 1765         args.size = sizeof(struct uma_keg);
 1766         args.ctor = keg_ctor;
 1767         args.dtor = keg_dtor;
 1768         args.uminit = zero_init;
 1769         args.fini = NULL;
 1770         args.keg = &masterkeg;
 1771         args.align = 32 - 1;
 1772         args.flags = UMA_ZFLAG_INTERNAL;
 1773         /* The initial zone has no Per cpu queues so it's smaller */
 1774         zone_ctor(kegs, sizeof(struct uma_zone), &args, M_WAITOK);
 1775 
 1776 #ifdef UMA_DEBUG
 1777         printf("Filling boot free list.\n");
 1778 #endif
 1779         for (i = 0; i < boot_pages; i++) {
 1780                 slab = (uma_slab_t)((uint8_t *)bootmem + (i * UMA_SLAB_SIZE));
 1781                 slab->us_data = (uint8_t *)slab;
 1782                 slab->us_flags = UMA_SLAB_BOOT;
 1783                 LIST_INSERT_HEAD(&uma_boot_pages, slab, us_link);
 1784         }
 1785         mtx_init(&uma_boot_pages_mtx, "UMA boot pages", NULL, MTX_DEF);
 1786 
 1787 #ifdef UMA_DEBUG
 1788         printf("Creating uma zone headers zone and keg.\n");
 1789 #endif
 1790         args.name = "UMA Zones";
 1791         args.size = sizeof(struct uma_zone) +
 1792             (sizeof(struct uma_cache) * (mp_maxid + 1));
 1793         args.ctor = zone_ctor;
 1794         args.dtor = zone_dtor;
 1795         args.uminit = zero_init;
 1796         args.fini = NULL;
 1797         args.keg = NULL;
 1798         args.align = 32 - 1;
 1799         args.flags = UMA_ZFLAG_INTERNAL;
 1800         /* The initial zone has no Per cpu queues so it's smaller */
 1801         zone_ctor(zones, sizeof(struct uma_zone), &args, M_WAITOK);
 1802 
 1803 #ifdef UMA_DEBUG
 1804         printf("Creating slab and hash zones.\n");
 1805 #endif
 1806 
 1807         /* Now make a zone for slab headers */
 1808         slabzone = uma_zcreate("UMA Slabs",
 1809                                 sizeof(struct uma_slab),
 1810                                 NULL, NULL, NULL, NULL,
 1811                                 UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL);
 1812 
 1813         hashzone = uma_zcreate("UMA Hash",
 1814             sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT,
 1815             NULL, NULL, NULL, NULL,
 1816             UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL);
 1817 
 1818         bucket_init();
 1819 
 1820         booted = UMA_STARTUP;
 1821 
 1822 #ifdef UMA_DEBUG
 1823         printf("UMA startup complete.\n");
 1824 #endif
 1825 }
 1826 
 1827 /* see uma.h */
 1828 void
 1829 uma_startup2(void)
 1830 {
 1831         booted = UMA_STARTUP2;
 1832         bucket_enable();
 1833         sx_init(&uma_drain_lock, "umadrain");
 1834 #ifdef UMA_DEBUG
 1835         printf("UMA startup2 complete.\n");
 1836 #endif
 1837 }
 1838 
 1839 /*
 1840  * Initialize our callout handle
 1841  *
 1842  */
 1843 
 1844 static void
 1845 uma_startup3(void)
 1846 {
 1847 #ifdef UMA_DEBUG
 1848         printf("Starting callout.\n");
 1849 #endif
 1850         callout_init(&uma_callout, 1);
 1851         callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
 1852 #ifdef UMA_DEBUG
 1853         printf("UMA startup3 complete.\n");
 1854 #endif
 1855 }
 1856 
 1857 static uma_keg_t
 1858 uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini,
 1859                 int align, uint32_t flags)
 1860 {
 1861         struct uma_kctor_args args;
 1862 
 1863         args.size = size;
 1864         args.uminit = uminit;
 1865         args.fini = fini;
 1866         args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align;
 1867         args.flags = flags;
 1868         args.zone = zone;
 1869         return (zone_alloc_item(kegs, &args, M_WAITOK));
 1870 }
 1871 
 1872 /* See uma.h */
 1873 void
 1874 uma_set_align(int align)
 1875 {
 1876 
 1877         if (align != UMA_ALIGN_CACHE)
 1878                 uma_align_cache = align;
 1879 }
 1880 
 1881 /* See uma.h */
 1882 uma_zone_t
 1883 uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor,
 1884                 uma_init uminit, uma_fini fini, int align, uint32_t flags)
 1885 
 1886 {
 1887         struct uma_zctor_args args;
 1888         uma_zone_t res;
 1889         bool locked;
 1890 
 1891         KASSERT(powerof2(align + 1), ("invalid zone alignment %d for \"%s\"",
 1892             align, name));
 1893 
 1894         /* This stuff is essential for the zone ctor */
 1895         memset(&args, 0, sizeof(args));
 1896         args.name = name;
 1897         args.size = size;
 1898         args.ctor = ctor;
 1899         args.dtor = dtor;
 1900         args.uminit = uminit;
 1901         args.fini = fini;
 1902 #ifdef  INVARIANTS
 1903         /*
 1904          * If a zone is being created with an empty constructor and
 1905          * destructor, pass UMA constructor/destructor which checks for
 1906          * memory use after free.
 1907          */
 1908         if ((!(flags & (UMA_ZONE_ZINIT | UMA_ZONE_NOFREE))) &&
 1909             ctor == NULL && dtor == NULL && uminit == NULL && fini == NULL) {
 1910                 args.ctor = trash_ctor;
 1911                 args.dtor = trash_dtor;
 1912                 args.uminit = trash_init;
 1913                 args.fini = trash_fini;
 1914         }
 1915 #endif
 1916         args.align = align;
 1917         args.flags = flags;
 1918         args.keg = NULL;
 1919 
 1920         if (booted < UMA_STARTUP2) {
 1921                 locked = false;
 1922         } else {
 1923                 sx_slock(&uma_drain_lock);
 1924                 locked = true;
 1925         }
 1926         res = zone_alloc_item(zones, &args, M_WAITOK);
 1927         if (locked)
 1928                 sx_sunlock(&uma_drain_lock);
 1929         return (res);
 1930 }
 1931 
 1932 /* See uma.h */
 1933 uma_zone_t
 1934 uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor,
 1935                     uma_init zinit, uma_fini zfini, uma_zone_t master)
 1936 {
 1937         struct uma_zctor_args args;
 1938         uma_keg_t keg;
 1939         uma_zone_t res;
 1940         bool locked;
 1941 
 1942         keg = zone_first_keg(master);
 1943         memset(&args, 0, sizeof(args));
 1944         args.name = name;
 1945         args.size = keg->uk_size;
 1946         args.ctor = ctor;
 1947         args.dtor = dtor;
 1948         args.uminit = zinit;
 1949         args.fini = zfini;
 1950         args.align = keg->uk_align;
 1951         args.flags = keg->uk_flags | UMA_ZONE_SECONDARY;
 1952         args.keg = keg;
 1953 
 1954         if (booted < UMA_STARTUP2) {
 1955                 locked = false;
 1956         } else {
 1957                 sx_slock(&uma_drain_lock);
 1958                 locked = true;
 1959         }
 1960         /* XXX Attaches only one keg of potentially many. */
 1961         res = zone_alloc_item(zones, &args, M_WAITOK);
 1962         if (locked)
 1963                 sx_sunlock(&uma_drain_lock);
 1964         return (res);
 1965 }
 1966 
 1967 /* See uma.h */
 1968 uma_zone_t
 1969 uma_zcache_create(char *name, int size, uma_ctor ctor, uma_dtor dtor,
 1970                     uma_init zinit, uma_fini zfini, uma_import zimport,
 1971                     uma_release zrelease, void *arg, int flags)
 1972 {
 1973         struct uma_zctor_args args;
 1974 
 1975         memset(&args, 0, sizeof(args));
 1976         args.name = name;
 1977         args.size = size;
 1978         args.ctor = ctor;
 1979         args.dtor = dtor;
 1980         args.uminit = zinit;
 1981         args.fini = zfini;
 1982         args.import = zimport;
 1983         args.release = zrelease;
 1984         args.arg = arg;
 1985         args.align = 0;
 1986         args.flags = flags;
 1987 
 1988         return (zone_alloc_item(zones, &args, M_WAITOK));
 1989 }
 1990 
 1991 static void
 1992 zone_lock_pair(uma_zone_t a, uma_zone_t b)
 1993 {
 1994         if (a < b) {
 1995                 ZONE_LOCK(a);
 1996                 mtx_lock_flags(b->uz_lockptr, MTX_DUPOK);
 1997         } else {
 1998                 ZONE_LOCK(b);
 1999                 mtx_lock_flags(a->uz_lockptr, MTX_DUPOK);
 2000         }
 2001 }
 2002 
 2003 static void
 2004 zone_unlock_pair(uma_zone_t a, uma_zone_t b)
 2005 {
 2006 
 2007         ZONE_UNLOCK(a);
 2008         ZONE_UNLOCK(b);
 2009 }
 2010 
 2011 int
 2012 uma_zsecond_add(uma_zone_t zone, uma_zone_t master)
 2013 {
 2014         uma_klink_t klink;
 2015         uma_klink_t kl;
 2016         int error;
 2017 
 2018         error = 0;
 2019         klink = malloc(sizeof(*klink), M_TEMP, M_WAITOK | M_ZERO);
 2020 
 2021         zone_lock_pair(zone, master);
 2022         /*
 2023          * zone must use vtoslab() to resolve objects and must already be
 2024          * a secondary.
 2025          */
 2026         if ((zone->uz_flags & (UMA_ZONE_VTOSLAB | UMA_ZONE_SECONDARY))
 2027             != (UMA_ZONE_VTOSLAB | UMA_ZONE_SECONDARY)) {
 2028                 error = EINVAL;
 2029                 goto out;
 2030         }
 2031         /*
 2032          * The new master must also use vtoslab().
 2033          */
 2034         if ((zone->uz_flags & UMA_ZONE_VTOSLAB) != UMA_ZONE_VTOSLAB) {
 2035                 error = EINVAL;
 2036                 goto out;
 2037         }
 2038 
 2039         /*
 2040          * The underlying object must be the same size.  rsize
 2041          * may be different.
 2042          */
 2043         if (master->uz_size != zone->uz_size) {
 2044                 error = E2BIG;
 2045                 goto out;
 2046         }
 2047         /*
 2048          * Put it at the end of the list.
 2049          */
 2050         klink->kl_keg = zone_first_keg(master);
 2051         LIST_FOREACH(kl, &zone->uz_kegs, kl_link) {
 2052                 if (LIST_NEXT(kl, kl_link) == NULL) {
 2053                         LIST_INSERT_AFTER(kl, klink, kl_link);
 2054                         break;
 2055                 }
 2056         }
 2057         klink = NULL;
 2058         zone->uz_flags |= UMA_ZFLAG_MULTI;
 2059         zone->uz_slab = zone_fetch_slab_multi;
 2060 
 2061 out:
 2062         zone_unlock_pair(zone, master);
 2063         if (klink != NULL)
 2064                 free(klink, M_TEMP);
 2065 
 2066         return (error);
 2067 }
 2068 
 2069 
 2070 /* See uma.h */
 2071 void
 2072 uma_zdestroy(uma_zone_t zone)
 2073 {
 2074 
 2075         sx_slock(&uma_drain_lock);
 2076         zone_free_item(zones, zone, NULL, SKIP_NONE);
 2077         sx_sunlock(&uma_drain_lock);
 2078 }
 2079 
 2080 /* See uma.h */
 2081 void *
 2082 uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
 2083 {
 2084         void *item;
 2085         uma_cache_t cache;
 2086         uma_bucket_t bucket;
 2087         int lockfail;
 2088         int cpu;
 2089 
 2090         /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */
 2091         random_harvest_fast_uma(&zone, sizeof(zone), 1, RANDOM_UMA);
 2092 
 2093         /* This is the fast path allocation */
 2094 #ifdef UMA_DEBUG_ALLOC_1
 2095         printf("Allocating one item from %s(%p)\n", zone->uz_name, zone);
 2096 #endif
 2097         CTR3(KTR_UMA, "uma_zalloc_arg thread %x zone %s flags %d", curthread,
 2098             zone->uz_name, flags);
 2099 
 2100         if (flags & M_WAITOK) {
 2101                 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
 2102                     "uma_zalloc_arg: zone \"%s\"", zone->uz_name);
 2103         }
 2104         KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(),
 2105             ("uma_zalloc_arg: called with spinlock or critical section held"));
 2106 
 2107 #ifdef DEBUG_MEMGUARD
 2108         if (memguard_cmp_zone(zone)) {
 2109                 item = memguard_alloc(zone->uz_size, flags);
 2110                 if (item != NULL) {
 2111                         if (zone->uz_init != NULL &&
 2112                             zone->uz_init(item, zone->uz_size, flags) != 0)
 2113                                 return (NULL);
 2114                         if (zone->uz_ctor != NULL &&
 2115                             zone->uz_ctor(item, zone->uz_size, udata,
 2116                             flags) != 0) {
 2117                                 zone->uz_fini(item, zone->uz_size);
 2118                                 return (NULL);
 2119                         }
 2120                         return (item);
 2121                 }
 2122                 /* This is unfortunate but should not be fatal. */
 2123         }
 2124 #endif
 2125         /*
 2126          * If possible, allocate from the per-CPU cache.  There are two
 2127          * requirements for safe access to the per-CPU cache: (1) the thread
 2128          * accessing the cache must not be preempted or yield during access,
 2129          * and (2) the thread must not migrate CPUs without switching which
 2130          * cache it accesses.  We rely on a critical section to prevent
 2131          * preemption and migration.  We release the critical section in
 2132          * order to acquire the zone mutex if we are unable to allocate from
 2133          * the current cache; when we re-acquire the critical section, we
 2134          * must detect and handle migration if it has occurred.
 2135          */
 2136         critical_enter();
 2137         cpu = curcpu;
 2138         cache = &zone->uz_cpu[cpu];
 2139 
 2140 zalloc_start:
 2141         bucket = cache->uc_allocbucket;
 2142         if (bucket != NULL && bucket->ub_cnt > 0) {
 2143                 bucket->ub_cnt--;
 2144                 item = bucket->ub_bucket[bucket->ub_cnt];
 2145 #ifdef INVARIANTS
 2146                 bucket->ub_bucket[bucket->ub_cnt] = NULL;
 2147 #endif
 2148                 KASSERT(item != NULL, ("uma_zalloc: Bucket pointer mangled."));
 2149                 cache->uc_allocs++;
 2150                 critical_exit();
 2151                 if (zone->uz_ctor != NULL &&
 2152                     zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
 2153                         atomic_add_long(&zone->uz_fails, 1);
 2154                         zone_free_item(zone, item, udata, SKIP_DTOR);
 2155                         return (NULL);
 2156                 }
 2157 #ifdef INVARIANTS
 2158                 uma_dbg_alloc(zone, NULL, item);
 2159 #endif
 2160                 if (flags & M_ZERO)
 2161                         uma_zero_item(item, zone);
 2162                 return (item);
 2163         }
 2164 
 2165         /*
 2166          * We have run out of items in our alloc bucket.
 2167          * See if we can switch with our free bucket.
 2168          */
 2169         bucket = cache->uc_freebucket;
 2170         if (bucket != NULL && bucket->ub_cnt > 0) {
 2171 #ifdef UMA_DEBUG_ALLOC
 2172                 printf("uma_zalloc: Swapping empty with alloc.\n");
 2173 #endif
 2174                 cache->uc_freebucket = cache->uc_allocbucket;
 2175                 cache->uc_allocbucket = bucket;
 2176                 goto zalloc_start;
 2177         }
 2178 
 2179         /*
 2180          * Discard any empty allocation bucket while we hold no locks.
 2181          */
 2182         bucket = cache->uc_allocbucket;
 2183         cache->uc_allocbucket = NULL;
 2184         critical_exit();
 2185         if (bucket != NULL)
 2186                 bucket_free(zone, bucket, udata);
 2187 
 2188         /* Short-circuit for zones without buckets and low memory. */
 2189         if (zone->uz_count == 0 || bucketdisable)
 2190                 goto zalloc_item;
 2191 
 2192         /*
 2193          * Attempt to retrieve the item from the per-CPU cache has failed, so
 2194          * we must go back to the zone.  This requires the zone lock, so we
 2195          * must drop the critical section, then re-acquire it when we go back
 2196          * to the cache.  Since the critical section is released, we may be
 2197          * preempted or migrate.  As such, make sure not to maintain any
 2198          * thread-local state specific to the cache from prior to releasing
 2199          * the critical section.
 2200          */
 2201         lockfail = 0;
 2202         if (ZONE_TRYLOCK(zone) == 0) {
 2203                 /* Record contention to size the buckets. */
 2204                 ZONE_LOCK(zone);
 2205                 lockfail = 1;
 2206         }
 2207         critical_enter();
 2208         cpu = curcpu;
 2209         cache = &zone->uz_cpu[cpu];
 2210 
 2211         /*
 2212          * Since we have locked the zone we may as well send back our stats.
 2213          */
 2214         atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
 2215         atomic_add_long(&zone->uz_frees, cache->uc_frees);
 2216         cache->uc_allocs = 0;
 2217         cache->uc_frees = 0;
 2218 
 2219         /* See if we lost the race to fill the cache. */
 2220         if (cache->uc_allocbucket != NULL) {
 2221                 ZONE_UNLOCK(zone);
 2222                 goto zalloc_start;
 2223         }
 2224 
 2225         /*
 2226          * Check the zone's cache of buckets.
 2227          */
 2228         if ((bucket = LIST_FIRST(&zone->uz_buckets)) != NULL) {
 2229                 KASSERT(bucket->ub_cnt != 0,
 2230                     ("uma_zalloc_arg: Returning an empty bucket."));
 2231 
 2232                 LIST_REMOVE(bucket, ub_link);
 2233                 cache->uc_allocbucket = bucket;
 2234                 ZONE_UNLOCK(zone);
 2235                 goto zalloc_start;
 2236         }
 2237         /* We are no longer associated with this CPU. */
 2238         critical_exit();
 2239 
 2240         /*
 2241          * We bump the uz count when the cache size is insufficient to
 2242          * handle the working set.
 2243          */
 2244         if (lockfail && zone->uz_count < BUCKET_MAX)
 2245                 zone->uz_count++;
 2246         ZONE_UNLOCK(zone);
 2247 
 2248         /*
 2249          * Now lets just fill a bucket and put it on the free list.  If that
 2250          * works we'll restart the allocation from the beginning and it
 2251          * will use the just filled bucket.
 2252          */
 2253         bucket = zone_alloc_bucket(zone, udata, flags);
 2254         if (bucket != NULL) {
 2255                 ZONE_LOCK(zone);
 2256                 critical_enter();
 2257                 cpu = curcpu;
 2258                 cache = &zone->uz_cpu[cpu];
 2259                 /*
 2260                  * See if we lost the race or were migrated.  Cache the
 2261                  * initialized bucket to make this less likely or claim
 2262                  * the memory directly.
 2263                  */
 2264                 if (cache->uc_allocbucket == NULL)
 2265                         cache->uc_allocbucket = bucket;
 2266                 else
 2267                         LIST_INSERT_HEAD(&zone->uz_buckets, bucket, ub_link);
 2268                 ZONE_UNLOCK(zone);
 2269                 goto zalloc_start;
 2270         }
 2271 
 2272         /*
 2273          * We may not be able to get a bucket so return an actual item.
 2274          */
 2275 #ifdef UMA_DEBUG
 2276         printf("uma_zalloc_arg: Bucketzone returned NULL\n");
 2277 #endif
 2278 
 2279 zalloc_item:
 2280         item = zone_alloc_item(zone, udata, flags);
 2281 
 2282         return (item);
 2283 }
 2284 
 2285 static uma_slab_t
 2286 keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int flags)
 2287 {
 2288         uma_slab_t slab;
 2289         int reserve;
 2290 
 2291         mtx_assert(&keg->uk_lock, MA_OWNED);
 2292         slab = NULL;
 2293         reserve = 0;
 2294         if ((flags & M_USE_RESERVE) == 0)
 2295                 reserve = keg->uk_reserve;
 2296 
 2297         for (;;) {
 2298                 /*
 2299                  * Find a slab with some space.  Prefer slabs that are partially
 2300                  * used over those that are totally full.  This helps to reduce
 2301                  * fragmentation.
 2302                  */
 2303                 if (keg->uk_free > reserve) {
 2304                         if (!LIST_EMPTY(&keg->uk_part_slab)) {
 2305                                 slab = LIST_FIRST(&keg->uk_part_slab);
 2306                         } else {
 2307                                 slab = LIST_FIRST(&keg->uk_free_slab);
 2308                                 LIST_REMOVE(slab, us_link);
 2309                                 LIST_INSERT_HEAD(&keg->uk_part_slab, slab,
 2310                                     us_link);
 2311                         }
 2312                         MPASS(slab->us_keg == keg);
 2313                         return (slab);
 2314                 }
 2315 
 2316                 /*
 2317                  * M_NOVM means don't ask at all!
 2318                  */
 2319                 if (flags & M_NOVM)
 2320                         break;
 2321 
 2322                 if (keg->uk_maxpages && keg->uk_pages >= keg->uk_maxpages) {
 2323                         keg->uk_flags |= UMA_ZFLAG_FULL;
 2324                         /*
 2325                          * If this is not a multi-zone, set the FULL bit.
 2326                          * Otherwise slab_multi() takes care of it.
 2327                          */
 2328                         if ((zone->uz_flags & UMA_ZFLAG_MULTI) == 0) {
 2329                                 zone->uz_flags |= UMA_ZFLAG_FULL;
 2330                                 zone_log_warning(zone);
 2331                                 zone_maxaction(zone);
 2332                         }
 2333                         if (flags & M_NOWAIT)
 2334                                 break;
 2335                         zone->uz_sleeps++;
 2336                         msleep(keg, &keg->uk_lock, PVM, "keglimit", 0);
 2337                         continue;
 2338                 }
 2339                 slab = keg_alloc_slab(keg, zone, flags);
 2340                 /*
 2341                  * If we got a slab here it's safe to mark it partially used
 2342                  * and return.  We assume that the caller is going to remove
 2343                  * at least one item.
 2344                  */
 2345                 if (slab) {
 2346                         MPASS(slab->us_keg == keg);
 2347                         LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link);
 2348                         return (slab);
 2349                 }
 2350                 /*
 2351                  * We might not have been able to get a slab but another cpu
 2352                  * could have while we were unlocked.  Check again before we
 2353                  * fail.
 2354                  */
 2355                 flags |= M_NOVM;
 2356         }
 2357         return (slab);
 2358 }
 2359 
 2360 static uma_slab_t
 2361 zone_fetch_slab(uma_zone_t zone, uma_keg_t keg, int flags)
 2362 {
 2363         uma_slab_t slab;
 2364 
 2365         if (keg == NULL) {
 2366                 keg = zone_first_keg(zone);
 2367                 KEG_LOCK(keg);
 2368         }
 2369 
 2370         for (;;) {
 2371                 slab = keg_fetch_slab(keg, zone, flags);
 2372                 if (slab)
 2373                         return (slab);
 2374                 if (flags & (M_NOWAIT | M_NOVM))
 2375                         break;
 2376         }
 2377         KEG_UNLOCK(keg);
 2378         return (NULL);
 2379 }
 2380 
 2381 /*
 2382  * uma_zone_fetch_slab_multi:  Fetches a slab from one available keg.  Returns
 2383  * with the keg locked.  On NULL no lock is held.
 2384  *
 2385  * The last pointer is used to seed the search.  It is not required.
 2386  */
 2387 static uma_slab_t
 2388 zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int rflags)
 2389 {
 2390         uma_klink_t klink;
 2391         uma_slab_t slab;
 2392         uma_keg_t keg;
 2393         int flags;
 2394         int empty;
 2395         int full;
 2396 
 2397         /*
 2398          * Don't wait on the first pass.  This will skip limit tests
 2399          * as well.  We don't want to block if we can find a provider
 2400          * without blocking.
 2401          */
 2402         flags = (rflags & ~M_WAITOK) | M_NOWAIT;
 2403         /*
 2404          * Use the last slab allocated as a hint for where to start
 2405          * the search.
 2406          */
 2407         if (last != NULL) {
 2408                 slab = keg_fetch_slab(last, zone, flags);
 2409                 if (slab)
 2410                         return (slab);
 2411                 KEG_UNLOCK(last);
 2412         }
 2413         /*
 2414          * Loop until we have a slab incase of transient failures
 2415          * while M_WAITOK is specified.  I'm not sure this is 100%
 2416          * required but we've done it for so long now.
 2417          */
 2418         for (;;) {
 2419                 empty = 0;
 2420                 full = 0;
 2421                 /*
 2422                  * Search the available kegs for slabs.  Be careful to hold the
 2423                  * correct lock while calling into the keg layer.
 2424                  */
 2425                 LIST_FOREACH(klink, &zone->uz_kegs, kl_link) {
 2426                         keg = klink->kl_keg;
 2427                         KEG_LOCK(keg);
 2428                         if ((keg->uk_flags & UMA_ZFLAG_FULL) == 0) {
 2429                                 slab = keg_fetch_slab(keg, zone, flags);
 2430                                 if (slab)
 2431                                         return (slab);
 2432                         }
 2433                         if (keg->uk_flags & UMA_ZFLAG_FULL)
 2434                                 full++;
 2435                         else
 2436                                 empty++;
 2437                         KEG_UNLOCK(keg);
 2438                 }
 2439                 if (rflags & (M_NOWAIT | M_NOVM))
 2440                         break;
 2441                 flags = rflags;
 2442                 /*
 2443                  * All kegs are full.  XXX We can't atomically check all kegs
 2444                  * and sleep so just sleep for a short period and retry.
 2445                  */
 2446                 if (full && !empty) {
 2447                         ZONE_LOCK(zone);
 2448                         zone->uz_flags |= UMA_ZFLAG_FULL;
 2449                         zone->uz_sleeps++;
 2450                         zone_log_warning(zone);
 2451                         zone_maxaction(zone);
 2452                         msleep(zone, zone->uz_lockptr, PVM,
 2453                             "zonelimit", hz/100);
 2454                         zone->uz_flags &= ~UMA_ZFLAG_FULL;
 2455                         ZONE_UNLOCK(zone);
 2456                         continue;
 2457                 }
 2458         }
 2459         return (NULL);
 2460 }
 2461 
 2462 static void *
 2463 slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
 2464 {
 2465         void *item;
 2466         uint8_t freei;
 2467 
 2468         MPASS(keg == slab->us_keg);
 2469         mtx_assert(&keg->uk_lock, MA_OWNED);
 2470 
 2471         freei = BIT_FFS(SLAB_SETSIZE, &slab->us_free) - 1;
 2472         BIT_CLR(SLAB_SETSIZE, freei, &slab->us_free);
 2473         item = slab->us_data + (keg->uk_rsize * freei);
 2474         slab->us_freecount--;
 2475         keg->uk_free--;
 2476 
 2477         /* Move this slab to the full list */
 2478         if (slab->us_freecount == 0) {
 2479                 LIST_REMOVE(slab, us_link);
 2480                 LIST_INSERT_HEAD(&keg->uk_full_slab, slab, us_link);
 2481         }
 2482 
 2483         return (item);
 2484 }
 2485 
 2486 static int
 2487 zone_import(uma_zone_t zone, void **bucket, int max, int flags)
 2488 {
 2489         uma_slab_t slab;
 2490         uma_keg_t keg;
 2491         int i;
 2492 
 2493         slab = NULL;
 2494         keg = NULL;
 2495         /* Try to keep the buckets totally full */
 2496         for (i = 0; i < max; ) {
 2497                 if ((slab = zone->uz_slab(zone, keg, flags)) == NULL)
 2498                         break;
 2499                 keg = slab->us_keg;
 2500                 while (slab->us_freecount && i < max) { 
 2501                         bucket[i++] = slab_alloc_item(keg, slab);
 2502                         if (keg->uk_free <= keg->uk_reserve)
 2503                                 break;
 2504                 }
 2505                 /* Don't grab more than one slab at a time. */
 2506                 flags &= ~M_WAITOK;
 2507                 flags |= M_NOWAIT;
 2508         }
 2509         if (slab != NULL)
 2510                 KEG_UNLOCK(keg);
 2511 
 2512         return i;
 2513 }
 2514 
 2515 static uma_bucket_t
 2516 zone_alloc_bucket(uma_zone_t zone, void *udata, int flags)
 2517 {
 2518         uma_bucket_t bucket;
 2519         int max;
 2520 
 2521         /* Don't wait for buckets, preserve caller's NOVM setting. */
 2522         bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM));
 2523         if (bucket == NULL)
 2524                 return (NULL);
 2525 
 2526         max = MIN(bucket->ub_entries, zone->uz_count);
 2527         bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket,
 2528             max, flags);
 2529 
 2530         /*
 2531          * Initialize the memory if necessary.
 2532          */
 2533         if (bucket->ub_cnt != 0 && zone->uz_init != NULL) {
 2534                 int i;
 2535 
 2536                 for (i = 0; i < bucket->ub_cnt; i++)
 2537                         if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size,
 2538                             flags) != 0)
 2539                                 break;
 2540                 /*
 2541                  * If we couldn't initialize the whole bucket, put the
 2542                  * rest back onto the freelist.
 2543                  */
 2544                 if (i != bucket->ub_cnt) {
 2545                         zone->uz_release(zone->uz_arg, &bucket->ub_bucket[i],
 2546                             bucket->ub_cnt - i);
 2547 #ifdef INVARIANTS
 2548                         bzero(&bucket->ub_bucket[i],
 2549                             sizeof(void *) * (bucket->ub_cnt - i));
 2550 #endif
 2551                         bucket->ub_cnt = i;
 2552                 }
 2553         }
 2554 
 2555         if (bucket->ub_cnt == 0) {
 2556                 bucket_free(zone, bucket, udata);
 2557                 atomic_add_long(&zone->uz_fails, 1);
 2558                 return (NULL);
 2559         }
 2560 
 2561         return (bucket);
 2562 }
 2563 
 2564 /*
 2565  * Allocates a single item from a zone.
 2566  *
 2567  * Arguments
 2568  *      zone   The zone to alloc for.
 2569  *      udata  The data to be passed to the constructor.
 2570  *      flags  M_WAITOK, M_NOWAIT, M_ZERO.
 2571  *
 2572  * Returns
 2573  *      NULL if there is no memory and M_NOWAIT is set
 2574  *      An item if successful
 2575  */
 2576 
 2577 static void *
 2578 zone_alloc_item(uma_zone_t zone, void *udata, int flags)
 2579 {
 2580         void *item;
 2581 
 2582         item = NULL;
 2583 
 2584 #ifdef UMA_DEBUG_ALLOC
 2585         printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone);
 2586 #endif
 2587         if (zone->uz_import(zone->uz_arg, &item, 1, flags) != 1)
 2588                 goto fail;
 2589         atomic_add_long(&zone->uz_allocs, 1);
 2590 
 2591         /*
 2592          * We have to call both the zone's init (not the keg's init)
 2593          * and the zone's ctor.  This is because the item is going from
 2594          * a keg slab directly to the user, and the user is expecting it
 2595          * to be both zone-init'd as well as zone-ctor'd.
 2596          */
 2597         if (zone->uz_init != NULL) {
 2598                 if (zone->uz_init(item, zone->uz_size, flags) != 0) {
 2599                         zone_free_item(zone, item, udata, SKIP_FINI);
 2600                         goto fail;
 2601                 }
 2602         }
 2603         if (zone->uz_ctor != NULL) {
 2604                 if (zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
 2605                         zone_free_item(zone, item, udata, SKIP_DTOR);
 2606                         goto fail;
 2607                 }
 2608         }
 2609 #ifdef INVARIANTS
 2610         uma_dbg_alloc(zone, NULL, item);
 2611 #endif
 2612         if (flags & M_ZERO)
 2613                 uma_zero_item(item, zone);
 2614 
 2615         return (item);
 2616 
 2617 fail:
 2618         atomic_add_long(&zone->uz_fails, 1);
 2619         return (NULL);
 2620 }
 2621 
 2622 /* See uma.h */
 2623 void
 2624 uma_zfree_arg(uma_zone_t zone, void *item, void *udata)
 2625 {
 2626         uma_cache_t cache;
 2627         uma_bucket_t bucket;
 2628         int lockfail;
 2629         int cpu;
 2630 
 2631         /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */
 2632         random_harvest_fast_uma(&zone, sizeof(zone), 1, RANDOM_UMA);
 2633 
 2634 #ifdef UMA_DEBUG_ALLOC_1
 2635         printf("Freeing item %p to %s(%p)\n", item, zone->uz_name, zone);
 2636 #endif
 2637         CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread,
 2638             zone->uz_name);
 2639 
 2640         KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(),
 2641             ("uma_zfree_arg: called with spinlock or critical section held"));
 2642 
 2643         /* uma_zfree(..., NULL) does nothing, to match free(9). */
 2644         if (item == NULL)
 2645                 return;
 2646 #ifdef DEBUG_MEMGUARD
 2647         if (is_memguard_addr(item)) {
 2648                 if (zone->uz_dtor != NULL)
 2649                         zone->uz_dtor(item, zone->uz_size, udata);
 2650                 if (zone->uz_fini != NULL)
 2651                         zone->uz_fini(item, zone->uz_size);
 2652                 memguard_free(item);
 2653                 return;
 2654         }
 2655 #endif
 2656 #ifdef INVARIANTS
 2657         if (zone->uz_flags & UMA_ZONE_MALLOC)
 2658                 uma_dbg_free(zone, udata, item);
 2659         else
 2660                 uma_dbg_free(zone, NULL, item);
 2661 #endif
 2662         if (zone->uz_dtor != NULL)
 2663                 zone->uz_dtor(item, zone->uz_size, udata);
 2664 
 2665         /*
 2666          * The race here is acceptable.  If we miss it we'll just have to wait
 2667          * a little longer for the limits to be reset.
 2668          */
 2669         if (zone->uz_flags & UMA_ZFLAG_FULL)
 2670                 goto zfree_item;
 2671 
 2672         /*
 2673          * If possible, free to the per-CPU cache.  There are two
 2674          * requirements for safe access to the per-CPU cache: (1) the thread
 2675          * accessing the cache must not be preempted or yield during access,
 2676          * and (2) the thread must not migrate CPUs without switching which
 2677          * cache it accesses.  We rely on a critical section to prevent
 2678          * preemption and migration.  We release the critical section in
 2679          * order to acquire the zone mutex if we are unable to free to the
 2680          * current cache; when we re-acquire the critical section, we must
 2681          * detect and handle migration if it has occurred.
 2682          */
 2683 zfree_restart:
 2684         critical_enter();
 2685         cpu = curcpu;
 2686         cache = &zone->uz_cpu[cpu];
 2687 
 2688 zfree_start:
 2689         /*
 2690          * Try to free into the allocbucket first to give LIFO ordering
 2691          * for cache-hot datastructures.  Spill over into the freebucket
 2692          * if necessary.  Alloc will swap them if one runs dry.
 2693          */
 2694         bucket = cache->uc_allocbucket;
 2695         if (bucket == NULL || bucket->ub_cnt >= bucket->ub_entries)
 2696                 bucket = cache->uc_freebucket;
 2697         if (bucket != NULL && bucket->ub_cnt < bucket->ub_entries) {
 2698                 KASSERT(bucket->ub_bucket[bucket->ub_cnt] == NULL,
 2699                     ("uma_zfree: Freeing to non free bucket index."));
 2700                 bucket->ub_bucket[bucket->ub_cnt] = item;
 2701                 bucket->ub_cnt++;
 2702                 cache->uc_frees++;
 2703                 critical_exit();
 2704                 return;
 2705         }
 2706 
 2707         /*
 2708          * We must go back the zone, which requires acquiring the zone lock,
 2709          * which in turn means we must release and re-acquire the critical
 2710          * section.  Since the critical section is released, we may be
 2711          * preempted or migrate.  As such, make sure not to maintain any
 2712          * thread-local state specific to the cache from prior to releasing
 2713          * the critical section.
 2714          */
 2715         critical_exit();
 2716         if (zone->uz_count == 0 || bucketdisable)
 2717                 goto zfree_item;
 2718 
 2719         lockfail = 0;
 2720         if (ZONE_TRYLOCK(zone) == 0) {
 2721                 /* Record contention to size the buckets. */
 2722                 ZONE_LOCK(zone);
 2723                 lockfail = 1;
 2724         }
 2725         critical_enter();
 2726         cpu = curcpu;
 2727         cache = &zone->uz_cpu[cpu];
 2728 
 2729         /*
 2730          * Since we have locked the zone we may as well send back our stats.
 2731          */
 2732         atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
 2733         atomic_add_long(&zone->uz_frees, cache->uc_frees);
 2734         cache->uc_allocs = 0;
 2735         cache->uc_frees = 0;
 2736 
 2737         bucket = cache->uc_freebucket;
 2738         if (bucket != NULL && bucket->ub_cnt < bucket->ub_entries) {
 2739                 ZONE_UNLOCK(zone);
 2740                 goto zfree_start;
 2741         }
 2742         cache->uc_freebucket = NULL;
 2743         /* We are no longer associated with this CPU. */
 2744         critical_exit();
 2745 
 2746         /* Can we throw this on the zone full list? */
 2747         if (bucket != NULL) {
 2748 #ifdef UMA_DEBUG_ALLOC
 2749                 printf("uma_zfree: Putting old bucket on the free list.\n");
 2750 #endif
 2751                 /* ub_cnt is pointing to the last free item */
 2752                 KASSERT(bucket->ub_cnt != 0,
 2753                     ("uma_zfree: Attempting to insert an empty bucket onto the full list.\n"));
 2754                 LIST_INSERT_HEAD(&zone->uz_buckets, bucket, ub_link);
 2755         }
 2756 
 2757         /*
 2758          * We bump the uz count when the cache size is insufficient to
 2759          * handle the working set.
 2760          */
 2761         if (lockfail && zone->uz_count < BUCKET_MAX)
 2762                 zone->uz_count++;
 2763         ZONE_UNLOCK(zone);
 2764 
 2765 #ifdef UMA_DEBUG_ALLOC
 2766         printf("uma_zfree: Allocating new free bucket.\n");
 2767 #endif
 2768         bucket = bucket_alloc(zone, udata, M_NOWAIT);
 2769         if (bucket) {
 2770                 critical_enter();
 2771                 cpu = curcpu;
 2772                 cache = &zone->uz_cpu[cpu];
 2773                 if (cache->uc_freebucket == NULL) {
 2774                         cache->uc_freebucket = bucket;
 2775                         goto zfree_start;
 2776                 }
 2777                 /*
 2778                  * We lost the race, start over.  We have to drop our
 2779                  * critical section to free the bucket.
 2780                  */
 2781                 critical_exit();
 2782                 bucket_free(zone, bucket, udata);
 2783                 goto zfree_restart;
 2784         }
 2785 
 2786         /*
 2787          * If nothing else caught this, we'll just do an internal free.
 2788          */
 2789 zfree_item:
 2790         zone_free_item(zone, item, udata, SKIP_DTOR);
 2791 
 2792         return;
 2793 }
 2794 
 2795 static void
 2796 slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item)
 2797 {
 2798         uint8_t freei;
 2799 
 2800         mtx_assert(&keg->uk_lock, MA_OWNED);
 2801         MPASS(keg == slab->us_keg);
 2802 
 2803         /* Do we need to remove from any lists? */
 2804         if (slab->us_freecount+1 == keg->uk_ipers) {
 2805                 LIST_REMOVE(slab, us_link);
 2806                 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link);
 2807         } else if (slab->us_freecount == 0) {
 2808                 LIST_REMOVE(slab, us_link);
 2809                 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link);
 2810         }
 2811 
 2812         /* Slab management. */
 2813         freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
 2814         BIT_SET(SLAB_SETSIZE, freei, &slab->us_free);
 2815         slab->us_freecount++;
 2816 
 2817         /* Keg statistics. */
 2818         keg->uk_free++;
 2819 }
 2820 
 2821 static void
 2822 zone_release(uma_zone_t zone, void **bucket, int cnt)
 2823 {
 2824         void *item;
 2825         uma_slab_t slab;
 2826         uma_keg_t keg;
 2827         uint8_t *mem;
 2828         int clearfull;
 2829         int i;
 2830 
 2831         clearfull = 0;
 2832         keg = zone_first_keg(zone);
 2833         KEG_LOCK(keg);
 2834         for (i = 0; i < cnt; i++) {
 2835                 item = bucket[i];
 2836                 if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
 2837                         mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
 2838                         if (zone->uz_flags & UMA_ZONE_HASH) {
 2839                                 slab = hash_sfind(&keg->uk_hash, mem);
 2840                         } else {
 2841                                 mem += keg->uk_pgoff;
 2842                                 slab = (uma_slab_t)mem;
 2843                         }
 2844                 } else {
 2845                         slab = vtoslab((vm_offset_t)item);
 2846                         if (slab->us_keg != keg) {
 2847                                 KEG_UNLOCK(keg);
 2848                                 keg = slab->us_keg;
 2849                                 KEG_LOCK(keg);
 2850                         }
 2851                 }
 2852                 slab_free_item(keg, slab, item);
 2853                 if (keg->uk_flags & UMA_ZFLAG_FULL) {
 2854                         if (keg->uk_pages < keg->uk_maxpages) {
 2855                                 keg->uk_flags &= ~UMA_ZFLAG_FULL;
 2856                                 clearfull = 1;
 2857                         }
 2858 
 2859                         /* 
 2860                          * We can handle one more allocation. Since we're
 2861                          * clearing ZFLAG_FULL, wake up all procs blocked
 2862                          * on pages. This should be uncommon, so keeping this
 2863                          * simple for now (rather than adding count of blocked 
 2864                          * threads etc).
 2865                          */
 2866                         wakeup(keg);
 2867                 }
 2868         }
 2869         KEG_UNLOCK(keg);
 2870         if (clearfull) {
 2871                 ZONE_LOCK(zone);
 2872                 zone->uz_flags &= ~UMA_ZFLAG_FULL;
 2873                 wakeup(zone);
 2874                 ZONE_UNLOCK(zone);
 2875         }
 2876 
 2877 }
 2878 
 2879 /*
 2880  * Frees a single item to any zone.
 2881  *
 2882  * Arguments:
 2883  *      zone   The zone to free to
 2884  *      item   The item we're freeing
 2885  *      udata  User supplied data for the dtor
 2886  *      skip   Skip dtors and finis
 2887  */
 2888 static void
 2889 zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip)
 2890 {
 2891 
 2892 #ifdef INVARIANTS
 2893         if (skip == SKIP_NONE) {
 2894                 if (zone->uz_flags & UMA_ZONE_MALLOC)
 2895                         uma_dbg_free(zone, udata, item);
 2896                 else
 2897                         uma_dbg_free(zone, NULL, item);
 2898         }
 2899 #endif
 2900         if (skip < SKIP_DTOR && zone->uz_dtor)
 2901                 zone->uz_dtor(item, zone->uz_size, udata);
 2902 
 2903         if (skip < SKIP_FINI && zone->uz_fini)
 2904                 zone->uz_fini(item, zone->uz_size);
 2905 
 2906         atomic_add_long(&zone->uz_frees, 1);
 2907         zone->uz_release(zone->uz_arg, &item, 1);
 2908 }
 2909 
 2910 /* See uma.h */
 2911 int
 2912 uma_zone_set_max(uma_zone_t zone, int nitems)
 2913 {
 2914         uma_keg_t keg;
 2915 
 2916         keg = zone_first_keg(zone);
 2917         if (keg == NULL)
 2918                 return (0);
 2919         KEG_LOCK(keg);
 2920         keg->uk_maxpages = (nitems / keg->uk_ipers) * keg->uk_ppera;
 2921         if (keg->uk_maxpages * keg->uk_ipers < nitems)
 2922                 keg->uk_maxpages += keg->uk_ppera;
 2923         nitems = (keg->uk_maxpages / keg->uk_ppera) * keg->uk_ipers;
 2924         KEG_UNLOCK(keg);
 2925 
 2926         return (nitems);
 2927 }
 2928 
 2929 /* See uma.h */
 2930 int
 2931 uma_zone_get_max(uma_zone_t zone)
 2932 {
 2933         int nitems;
 2934         uma_keg_t keg;
 2935 
 2936         keg = zone_first_keg(zone);
 2937         if (keg == NULL)
 2938                 return (0);
 2939         KEG_LOCK(keg);
 2940         nitems = (keg->uk_maxpages / keg->uk_ppera) * keg->uk_ipers;
 2941         KEG_UNLOCK(keg);
 2942 
 2943         return (nitems);
 2944 }
 2945 
 2946 /* See uma.h */
 2947 void
 2948 uma_zone_set_warning(uma_zone_t zone, const char *warning)
 2949 {
 2950 
 2951         ZONE_LOCK(zone);
 2952         zone->uz_warning = warning;
 2953         ZONE_UNLOCK(zone);
 2954 }
 2955 
 2956 /* See uma.h */
 2957 void
 2958 uma_zone_set_maxaction(uma_zone_t zone, uma_maxaction_t maxaction)
 2959 {
 2960 
 2961         ZONE_LOCK(zone);
 2962         TASK_INIT(&zone->uz_maxaction, 0, (task_fn_t *)maxaction, zone);
 2963         ZONE_UNLOCK(zone);
 2964 }
 2965 
 2966 /* See uma.h */
 2967 int
 2968 uma_zone_get_cur(uma_zone_t zone)
 2969 {
 2970         int64_t nitems;
 2971         u_int i;
 2972 
 2973         ZONE_LOCK(zone);
 2974         nitems = zone->uz_allocs - zone->uz_frees;
 2975         CPU_FOREACH(i) {
 2976                 /*
 2977                  * See the comment in sysctl_vm_zone_stats() regarding the
 2978                  * safety of accessing the per-cpu caches. With the zone lock
 2979                  * held, it is safe, but can potentially result in stale data.
 2980                  */
 2981                 nitems += zone->uz_cpu[i].uc_allocs -
 2982                     zone->uz_cpu[i].uc_frees;
 2983         }
 2984         ZONE_UNLOCK(zone);
 2985 
 2986         return (nitems < 0 ? 0 : nitems);
 2987 }
 2988 
 2989 /* See uma.h */
 2990 void
 2991 uma_zone_set_init(uma_zone_t zone, uma_init uminit)
 2992 {
 2993         uma_keg_t keg;
 2994 
 2995         keg = zone_first_keg(zone);
 2996         KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
 2997         KEG_LOCK(keg);
 2998         KASSERT(keg->uk_pages == 0,
 2999             ("uma_zone_set_init on non-empty keg"));
 3000         keg->uk_init = uminit;
 3001         KEG_UNLOCK(keg);
 3002 }
 3003 
 3004 /* See uma.h */
 3005 void
 3006 uma_zone_set_fini(uma_zone_t zone, uma_fini fini)
 3007 {
 3008         uma_keg_t keg;
 3009 
 3010         keg = zone_first_keg(zone);
 3011         KASSERT(keg != NULL, ("uma_zone_set_fini: Invalid zone type"));
 3012         KEG_LOCK(keg);
 3013         KASSERT(keg->uk_pages == 0,
 3014             ("uma_zone_set_fini on non-empty keg"));
 3015         keg->uk_fini = fini;
 3016         KEG_UNLOCK(keg);
 3017 }
 3018 
 3019 /* See uma.h */
 3020 void
 3021 uma_zone_set_zinit(uma_zone_t zone, uma_init zinit)
 3022 {
 3023 
 3024         ZONE_LOCK(zone);
 3025         KASSERT(zone_first_keg(zone)->uk_pages == 0,
 3026             ("uma_zone_set_zinit on non-empty keg"));
 3027         zone->uz_init = zinit;
 3028         ZONE_UNLOCK(zone);
 3029 }
 3030 
 3031 /* See uma.h */
 3032 void
 3033 uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini)
 3034 {
 3035 
 3036         ZONE_LOCK(zone);
 3037         KASSERT(zone_first_keg(zone)->uk_pages == 0,
 3038             ("uma_zone_set_zfini on non-empty keg"));
 3039         zone->uz_fini = zfini;
 3040         ZONE_UNLOCK(zone);
 3041 }
 3042 
 3043 /* See uma.h */
 3044 /* XXX uk_freef is not actually used with the zone locked */
 3045 void
 3046 uma_zone_set_freef(uma_zone_t zone, uma_free freef)
 3047 {
 3048         uma_keg_t keg;
 3049 
 3050         keg = zone_first_keg(zone);
 3051         KASSERT(keg != NULL, ("uma_zone_set_freef: Invalid zone type"));
 3052         KEG_LOCK(keg);
 3053         keg->uk_freef = freef;
 3054         KEG_UNLOCK(keg);
 3055 }
 3056 
 3057 /* See uma.h */
 3058 /* XXX uk_allocf is not actually used with the zone locked */
 3059 void
 3060 uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf)
 3061 {
 3062         uma_keg_t keg;
 3063 
 3064         keg = zone_first_keg(zone);
 3065         KEG_LOCK(keg);
 3066         keg->uk_allocf = allocf;
 3067         KEG_UNLOCK(keg);
 3068 }
 3069 
 3070 /* See uma.h */
 3071 void
 3072 uma_zone_reserve(uma_zone_t zone, int items)
 3073 {
 3074         uma_keg_t keg;
 3075 
 3076         keg = zone_first_keg(zone);
 3077         if (keg == NULL)
 3078                 return;
 3079         KEG_LOCK(keg);
 3080         keg->uk_reserve = items;
 3081         KEG_UNLOCK(keg);
 3082 
 3083         return;
 3084 }
 3085 
 3086 /* See uma.h */
 3087 int
 3088 uma_zone_reserve_kva(uma_zone_t zone, int count)
 3089 {
 3090         uma_keg_t keg;
 3091         vm_offset_t kva;
 3092         u_int pages;
 3093 
 3094         keg = zone_first_keg(zone);
 3095         if (keg == NULL)
 3096                 return (0);
 3097         pages = count / keg->uk_ipers;
 3098 
 3099         if (pages * keg->uk_ipers < count)
 3100                 pages++;
 3101         pages *= keg->uk_ppera;
 3102 
 3103 #ifdef UMA_MD_SMALL_ALLOC
 3104         if (keg->uk_ppera > 1) {
 3105 #else
 3106         if (1) {
 3107 #endif
 3108                 kva = kva_alloc((vm_size_t)pages * PAGE_SIZE);
 3109                 if (kva == 0)
 3110                         return (0);
 3111         } else
 3112                 kva = 0;
 3113         KEG_LOCK(keg);
 3114         keg->uk_kva = kva;
 3115         keg->uk_offset = 0;
 3116         keg->uk_maxpages = pages;
 3117 #ifdef UMA_MD_SMALL_ALLOC
 3118         keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc;
 3119 #else
 3120         keg->uk_allocf = noobj_alloc;
 3121 #endif
 3122         keg->uk_flags |= UMA_ZONE_NOFREE;
 3123         KEG_UNLOCK(keg);
 3124 
 3125         return (1);
 3126 }
 3127 
 3128 /* See uma.h */
 3129 void
 3130 uma_prealloc(uma_zone_t zone, int items)
 3131 {
 3132         int slabs;
 3133         uma_slab_t slab;
 3134         uma_keg_t keg;
 3135 
 3136         keg = zone_first_keg(zone);
 3137         if (keg == NULL)
 3138                 return;
 3139         KEG_LOCK(keg);
 3140         slabs = items / keg->uk_ipers;
 3141         if (slabs * keg->uk_ipers < items)
 3142                 slabs++;
 3143         while (slabs > 0) {
 3144                 slab = keg_alloc_slab(keg, zone, M_WAITOK);
 3145                 if (slab == NULL)
 3146                         break;
 3147                 MPASS(slab->us_keg == keg);
 3148                 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link);
 3149                 slabs--;
 3150         }
 3151         KEG_UNLOCK(keg);
 3152 }
 3153 
 3154 /* See uma.h */
 3155 static void
 3156 uma_reclaim_locked(bool kmem_danger)
 3157 {
 3158 
 3159 #ifdef UMA_DEBUG
 3160         printf("UMA: vm asked us to release pages!\n");
 3161 #endif
 3162         sx_assert(&uma_drain_lock, SA_XLOCKED);
 3163         bucket_enable();
 3164         zone_foreach(zone_drain);
 3165         if (vm_page_count_min() || kmem_danger) {
 3166                 cache_drain_safe(NULL);
 3167                 zone_foreach(zone_drain);
 3168         }
 3169         /*
 3170          * Some slabs may have been freed but this zone will be visited early
 3171          * we visit again so that we can free pages that are empty once other
 3172          * zones are drained.  We have to do the same for buckets.
 3173          */
 3174         zone_drain(slabzone);
 3175         bucket_zone_drain();
 3176 }
 3177 
 3178 void
 3179 uma_reclaim(void)
 3180 {
 3181 
 3182         sx_xlock(&uma_drain_lock);
 3183         uma_reclaim_locked(false);
 3184         sx_xunlock(&uma_drain_lock);
 3185 }
 3186 
 3187 static int uma_reclaim_needed;
 3188 
 3189 void
 3190 uma_reclaim_wakeup(void)
 3191 {
 3192 
 3193         uma_reclaim_needed = 1;
 3194         wakeup(&uma_reclaim_needed);
 3195 }
 3196 
 3197 void
 3198 uma_reclaim_worker(void *arg __unused)
 3199 {
 3200 
 3201         sx_xlock(&uma_drain_lock);
 3202         for (;;) {
 3203                 sx_sleep(&uma_reclaim_needed, &uma_drain_lock, PVM,
 3204                     "umarcl", 0);
 3205                 if (uma_reclaim_needed) {
 3206                         uma_reclaim_needed = 0;
 3207                         sx_xunlock(&uma_drain_lock);
 3208                         EVENTHANDLER_INVOKE(vm_lowmem, VM_LOW_KMEM);
 3209                         sx_xlock(&uma_drain_lock);
 3210                         uma_reclaim_locked(true);
 3211                 }
 3212         }
 3213 }
 3214 
 3215 /* See uma.h */
 3216 int
 3217 uma_zone_exhausted(uma_zone_t zone)
 3218 {
 3219         int full;
 3220 
 3221         ZONE_LOCK(zone);
 3222         full = (zone->uz_flags & UMA_ZFLAG_FULL);
 3223         ZONE_UNLOCK(zone);
 3224         return (full);  
 3225 }
 3226 
 3227 int
 3228 uma_zone_exhausted_nolock(uma_zone_t zone)
 3229 {
 3230         return (zone->uz_flags & UMA_ZFLAG_FULL);
 3231 }
 3232 
 3233 void *
 3234 uma_large_malloc(vm_size_t size, int wait)
 3235 {
 3236         void *mem;
 3237         uma_slab_t slab;
 3238         uint8_t flags;
 3239 
 3240         slab = zone_alloc_item(slabzone, NULL, wait);
 3241         if (slab == NULL)
 3242                 return (NULL);
 3243         mem = page_alloc(NULL, size, &flags, wait);
 3244         if (mem) {
 3245                 vsetslab((vm_offset_t)mem, slab);
 3246                 slab->us_data = mem;
 3247                 slab->us_flags = flags | UMA_SLAB_MALLOC;
 3248                 slab->us_size = size;
 3249         } else {
 3250                 zone_free_item(slabzone, slab, NULL, SKIP_NONE);
 3251         }
 3252 
 3253         return (mem);
 3254 }
 3255 
 3256 void
 3257 uma_large_free(uma_slab_t slab)
 3258 {
 3259 
 3260         page_free(slab->us_data, slab->us_size, slab->us_flags);
 3261         zone_free_item(slabzone, slab, NULL, SKIP_NONE);
 3262 }
 3263 
 3264 static void
 3265 uma_zero_item(void *item, uma_zone_t zone)
 3266 {
 3267         int i;
 3268 
 3269         if (zone->uz_flags & UMA_ZONE_PCPU) {
 3270                 CPU_FOREACH(i)
 3271                         bzero(zpcpu_get_cpu(item, i), zone->uz_size);
 3272         } else
 3273                 bzero(item, zone->uz_size);
 3274 }
 3275 
 3276 void
 3277 uma_print_stats(void)
 3278 {
 3279         zone_foreach(uma_print_zone);
 3280 }
 3281 
 3282 static void
 3283 slab_print(uma_slab_t slab)
 3284 {
 3285         printf("slab: keg %p, data %p, freecount %d\n",
 3286                 slab->us_keg, slab->us_data, slab->us_freecount);
 3287 }
 3288 
 3289 static void
 3290 cache_print(uma_cache_t cache)
 3291 {
 3292         printf("alloc: %p(%d), free: %p(%d)\n",
 3293                 cache->uc_allocbucket,
 3294                 cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0,
 3295                 cache->uc_freebucket,
 3296                 cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0);
 3297 }
 3298 
 3299 static void
 3300 uma_print_keg(uma_keg_t keg)
 3301 {
 3302         uma_slab_t slab;
 3303 
 3304         printf("keg: %s(%p) size %d(%d) flags %#x ipers %d ppera %d "
 3305             "out %d free %d limit %d\n",
 3306             keg->uk_name, keg, keg->uk_size, keg->uk_rsize, keg->uk_flags,
 3307             keg->uk_ipers, keg->uk_ppera,
 3308             (keg->uk_pages / keg->uk_ppera) * keg->uk_ipers - keg->uk_free,
 3309             keg->uk_free, (keg->uk_maxpages / keg->uk_ppera) * keg->uk_ipers);
 3310         printf("Part slabs:\n");
 3311         LIST_FOREACH(slab, &keg->uk_part_slab, us_link)
 3312                 slab_print(slab);
 3313         printf("Free slabs:\n");
 3314         LIST_FOREACH(slab, &keg->uk_free_slab, us_link)
 3315                 slab_print(slab);
 3316         printf("Full slabs:\n");
 3317         LIST_FOREACH(slab, &keg->uk_full_slab, us_link)
 3318                 slab_print(slab);
 3319 }
 3320 
 3321 void
 3322 uma_print_zone(uma_zone_t zone)
 3323 {
 3324         uma_cache_t cache;
 3325         uma_klink_t kl;
 3326         int i;
 3327 
 3328         printf("zone: %s(%p) size %d flags %#x\n",
 3329             zone->uz_name, zone, zone->uz_size, zone->uz_flags);
 3330         LIST_FOREACH(kl, &zone->uz_kegs, kl_link)
 3331                 uma_print_keg(kl->kl_keg);
 3332         CPU_FOREACH(i) {
 3333                 cache = &zone->uz_cpu[i];
 3334                 printf("CPU %d Cache:\n", i);
 3335                 cache_print(cache);
 3336         }
 3337 }
 3338 
 3339 #ifdef DDB
 3340 /*
 3341  * Generate statistics across both the zone and its per-cpu cache's.  Return
 3342  * desired statistics if the pointer is non-NULL for that statistic.
 3343  *
 3344  * Note: does not update the zone statistics, as it can't safely clear the
 3345  * per-CPU cache statistic.
 3346  *
 3347  * XXXRW: Following the uc_allocbucket and uc_freebucket pointers here isn't
 3348  * safe from off-CPU; we should modify the caches to track this information
 3349  * directly so that we don't have to.
 3350  */
 3351 static void
 3352 uma_zone_sumstat(uma_zone_t z, int *cachefreep, uint64_t *allocsp,
 3353     uint64_t *freesp, uint64_t *sleepsp)
 3354 {
 3355         uma_cache_t cache;
 3356         uint64_t allocs, frees, sleeps;
 3357         int cachefree, cpu;
 3358 
 3359         allocs = frees = sleeps = 0;
 3360         cachefree = 0;
 3361         CPU_FOREACH(cpu) {
 3362                 cache = &z->uz_cpu[cpu];
 3363                 if (cache->uc_allocbucket != NULL)
 3364                         cachefree += cache->uc_allocbucket->ub_cnt;
 3365                 if (cache->uc_freebucket != NULL)
 3366                         cachefree += cache->uc_freebucket->ub_cnt;
 3367                 allocs += cache->uc_allocs;
 3368                 frees += cache->uc_frees;
 3369         }
 3370         allocs += z->uz_allocs;
 3371         frees += z->uz_frees;
 3372         sleeps += z->uz_sleeps;
 3373         if (cachefreep != NULL)
 3374                 *cachefreep = cachefree;
 3375         if (allocsp != NULL)
 3376                 *allocsp = allocs;
 3377         if (freesp != NULL)
 3378                 *freesp = frees;
 3379         if (sleepsp != NULL)
 3380                 *sleepsp = sleeps;
 3381 }
 3382 #endif /* DDB */
 3383 
 3384 static int
 3385 sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS)
 3386 {
 3387         uma_keg_t kz;
 3388         uma_zone_t z;
 3389         int count;
 3390 
 3391         count = 0;
 3392         rw_rlock(&uma_rwlock);
 3393         LIST_FOREACH(kz, &uma_kegs, uk_link) {
 3394                 LIST_FOREACH(z, &kz->uk_zones, uz_link)
 3395                         count++;
 3396         }
 3397         rw_runlock(&uma_rwlock);
 3398         return (sysctl_handle_int(oidp, &count, 0, req));
 3399 }
 3400 
 3401 static int
 3402 sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS)
 3403 {
 3404         struct uma_stream_header ush;
 3405         struct uma_type_header uth;
 3406         struct uma_percpu_stat ups;
 3407         uma_bucket_t bucket;
 3408         struct sbuf sbuf;
 3409         uma_cache_t cache;
 3410         uma_klink_t kl;
 3411         uma_keg_t kz;
 3412         uma_zone_t z;
 3413         uma_keg_t k;
 3414         int count, error, i;
 3415 
 3416         error = sysctl_wire_old_buffer(req, 0);
 3417         if (error != 0)
 3418                 return (error);
 3419         sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
 3420         sbuf_clear_flags(&sbuf, SBUF_INCLUDENUL);
 3421 
 3422         count = 0;
 3423         rw_rlock(&uma_rwlock);
 3424         LIST_FOREACH(kz, &uma_kegs, uk_link) {
 3425                 LIST_FOREACH(z, &kz->uk_zones, uz_link)
 3426                         count++;
 3427         }
 3428 
 3429         /*
 3430          * Insert stream header.
 3431          */
 3432         bzero(&ush, sizeof(ush));
 3433         ush.ush_version = UMA_STREAM_VERSION;
 3434         ush.ush_maxcpus = (mp_maxid + 1);
 3435         ush.ush_count = count;
 3436         (void)sbuf_bcat(&sbuf, &ush, sizeof(ush));
 3437 
 3438         LIST_FOREACH(kz, &uma_kegs, uk_link) {
 3439                 LIST_FOREACH(z, &kz->uk_zones, uz_link) {
 3440                         bzero(&uth, sizeof(uth));
 3441                         ZONE_LOCK(z);
 3442                         strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME);
 3443                         uth.uth_align = kz->uk_align;
 3444                         uth.uth_size = kz->uk_size;
 3445                         uth.uth_rsize = kz->uk_rsize;
 3446                         LIST_FOREACH(kl, &z->uz_kegs, kl_link) {
 3447                                 k = kl->kl_keg;
 3448                                 uth.uth_maxpages += k->uk_maxpages;
 3449                                 uth.uth_pages += k->uk_pages;
 3450                                 uth.uth_keg_free += k->uk_free;
 3451                                 uth.uth_limit = (k->uk_maxpages / k->uk_ppera)
 3452                                     * k->uk_ipers;
 3453                         }
 3454 
 3455                         /*
 3456                          * A zone is secondary is it is not the first entry
 3457                          * on the keg's zone list.
 3458                          */
 3459                         if ((z->uz_flags & UMA_ZONE_SECONDARY) &&
 3460                             (LIST_FIRST(&kz->uk_zones) != z))
 3461                                 uth.uth_zone_flags = UTH_ZONE_SECONDARY;
 3462 
 3463                         LIST_FOREACH(bucket, &z->uz_buckets, ub_link)
 3464                                 uth.uth_zone_free += bucket->ub_cnt;
 3465                         uth.uth_allocs = z->uz_allocs;
 3466                         uth.uth_frees = z->uz_frees;
 3467                         uth.uth_fails = z->uz_fails;
 3468                         uth.uth_sleeps = z->uz_sleeps;
 3469                         (void)sbuf_bcat(&sbuf, &uth, sizeof(uth));
 3470                         /*
 3471                          * While it is not normally safe to access the cache
 3472                          * bucket pointers while not on the CPU that owns the
 3473                          * cache, we only allow the pointers to be exchanged
 3474                          * without the zone lock held, not invalidated, so
 3475                          * accept the possible race associated with bucket
 3476                          * exchange during monitoring.
 3477                          */
 3478                         for (i = 0; i < (mp_maxid + 1); i++) {
 3479                                 bzero(&ups, sizeof(ups));
 3480                                 if (kz->uk_flags & UMA_ZFLAG_INTERNAL)
 3481                                         goto skip;
 3482                                 if (CPU_ABSENT(i))
 3483                                         goto skip;
 3484                                 cache = &z->uz_cpu[i];
 3485                                 if (cache->uc_allocbucket != NULL)
 3486                                         ups.ups_cache_free +=
 3487                                             cache->uc_allocbucket->ub_cnt;
 3488                                 if (cache->uc_freebucket != NULL)
 3489                                         ups.ups_cache_free +=
 3490                                             cache->uc_freebucket->ub_cnt;
 3491                                 ups.ups_allocs = cache->uc_allocs;
 3492                                 ups.ups_frees = cache->uc_frees;
 3493 skip:
 3494                                 (void)sbuf_bcat(&sbuf, &ups, sizeof(ups));
 3495                         }
 3496                         ZONE_UNLOCK(z);
 3497                 }
 3498         }
 3499         rw_runlock(&uma_rwlock);
 3500         error = sbuf_finish(&sbuf);
 3501         sbuf_delete(&sbuf);
 3502         return (error);
 3503 }
 3504 
 3505 int
 3506 sysctl_handle_uma_zone_max(SYSCTL_HANDLER_ARGS)
 3507 {
 3508         uma_zone_t zone = *(uma_zone_t *)arg1;
 3509         int error, max;
 3510 
 3511         max = uma_zone_get_max(zone);
 3512         error = sysctl_handle_int(oidp, &max, 0, req);
 3513         if (error || !req->newptr)
 3514                 return (error);
 3515 
 3516         uma_zone_set_max(zone, max);
 3517 
 3518         return (0);
 3519 }
 3520 
 3521 int
 3522 sysctl_handle_uma_zone_cur(SYSCTL_HANDLER_ARGS)
 3523 {
 3524         uma_zone_t zone = *(uma_zone_t *)arg1;
 3525         int cur;
 3526 
 3527         cur = uma_zone_get_cur(zone);
 3528         return (sysctl_handle_int(oidp, &cur, 0, req));
 3529 }
 3530 
 3531 #ifdef INVARIANTS
 3532 static uma_slab_t
 3533 uma_dbg_getslab(uma_zone_t zone, void *item)
 3534 {
 3535         uma_slab_t slab;
 3536         uma_keg_t keg;
 3537         uint8_t *mem;
 3538 
 3539         mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
 3540         if (zone->uz_flags & UMA_ZONE_VTOSLAB) {
 3541                 slab = vtoslab((vm_offset_t)mem);
 3542         } else {
 3543                 /*
 3544                  * It is safe to return the slab here even though the
 3545                  * zone is unlocked because the item's allocation state
 3546                  * essentially holds a reference.
 3547                  */
 3548                 ZONE_LOCK(zone);
 3549                 keg = LIST_FIRST(&zone->uz_kegs)->kl_keg;
 3550                 if (keg->uk_flags & UMA_ZONE_HASH)
 3551                         slab = hash_sfind(&keg->uk_hash, mem);
 3552                 else
 3553                         slab = (uma_slab_t)(mem + keg->uk_pgoff);
 3554                 ZONE_UNLOCK(zone);
 3555         }
 3556 
 3557         return (slab);
 3558 }
 3559 
 3560 /*
 3561  * Set up the slab's freei data such that uma_dbg_free can function.
 3562  *
 3563  */
 3564 static void
 3565 uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item)
 3566 {
 3567         uma_keg_t keg;
 3568         int freei;
 3569 
 3570         if (zone_first_keg(zone) == NULL)
 3571                 return;
 3572         if (slab == NULL) {
 3573                 slab = uma_dbg_getslab(zone, item);
 3574                 if (slab == NULL) 
 3575                         panic("uma: item %p did not belong to zone %s\n",
 3576                             item, zone->uz_name);
 3577         }
 3578         keg = slab->us_keg;
 3579         freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
 3580 
 3581         if (BIT_ISSET(SLAB_SETSIZE, freei, &slab->us_debugfree))
 3582                 panic("Duplicate alloc of %p from zone %p(%s) slab %p(%d)\n",
 3583                     item, zone, zone->uz_name, slab, freei);
 3584         BIT_SET_ATOMIC(SLAB_SETSIZE, freei, &slab->us_debugfree);
 3585 
 3586         return;
 3587 }
 3588 
 3589 /*
 3590  * Verifies freed addresses.  Checks for alignment, valid slab membership
 3591  * and duplicate frees.
 3592  *
 3593  */
 3594 static void
 3595 uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item)
 3596 {
 3597         uma_keg_t keg;
 3598         int freei;
 3599 
 3600         if (zone_first_keg(zone) == NULL)
 3601                 return;
 3602         if (slab == NULL) {
 3603                 slab = uma_dbg_getslab(zone, item);
 3604                 if (slab == NULL) 
 3605                         panic("uma: Freed item %p did not belong to zone %s\n",
 3606                             item, zone->uz_name);
 3607         }
 3608         keg = slab->us_keg;
 3609         freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
 3610 
 3611         if (freei >= keg->uk_ipers)
 3612                 panic("Invalid free of %p from zone %p(%s) slab %p(%d)\n",
 3613                     item, zone, zone->uz_name, slab, freei);
 3614 
 3615         if (((freei * keg->uk_rsize) + slab->us_data) != item) 
 3616                 panic("Unaligned free of %p from zone %p(%s) slab %p(%d)\n",
 3617                     item, zone, zone->uz_name, slab, freei);
 3618 
 3619         if (!BIT_ISSET(SLAB_SETSIZE, freei, &slab->us_debugfree))
 3620                 panic("Duplicate free of %p from zone %p(%s) slab %p(%d)\n",
 3621                     item, zone, zone->uz_name, slab, freei);
 3622 
 3623         BIT_CLR_ATOMIC(SLAB_SETSIZE, freei, &slab->us_debugfree);
 3624 }
 3625 #endif /* INVARIANTS */
 3626 
 3627 #ifdef DDB
 3628 DB_SHOW_COMMAND(uma, db_show_uma)
 3629 {
 3630         uint64_t allocs, frees, sleeps;
 3631         uma_bucket_t bucket;
 3632         uma_keg_t kz;
 3633         uma_zone_t z;
 3634         int cachefree;
 3635 
 3636         db_printf("%18s %8s %8s %8s %12s %8s %8s\n", "Zone", "Size", "Used",
 3637             "Free", "Requests", "Sleeps", "Bucket");
 3638         LIST_FOREACH(kz, &uma_kegs, uk_link) {
 3639                 LIST_FOREACH(z, &kz->uk_zones, uz_link) {
 3640                         if (kz->uk_flags & UMA_ZFLAG_INTERNAL) {
 3641                                 allocs = z->uz_allocs;
 3642                                 frees = z->uz_frees;
 3643                                 sleeps = z->uz_sleeps;
 3644                                 cachefree = 0;
 3645                         } else
 3646                                 uma_zone_sumstat(z, &cachefree, &allocs,
 3647                                     &frees, &sleeps);
 3648                         if (!((z->uz_flags & UMA_ZONE_SECONDARY) &&
 3649                             (LIST_FIRST(&kz->uk_zones) != z)))
 3650                                 cachefree += kz->uk_free;
 3651                         LIST_FOREACH(bucket, &z->uz_buckets, ub_link)
 3652                                 cachefree += bucket->ub_cnt;
 3653                         db_printf("%18s %8ju %8jd %8d %12ju %8ju %8u\n",
 3654                             z->uz_name, (uintmax_t)kz->uk_size,
 3655                             (intmax_t)(allocs - frees), cachefree,
 3656                             (uintmax_t)allocs, sleeps, z->uz_count);
 3657                         if (db_pager_quit)
 3658                                 return;
 3659                 }
 3660         }
 3661 }
 3662 
 3663 DB_SHOW_COMMAND(umacache, db_show_umacache)
 3664 {
 3665         uint64_t allocs, frees;
 3666         uma_bucket_t bucket;
 3667         uma_zone_t z;
 3668         int cachefree;
 3669 
 3670         db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free",
 3671             "Requests", "Bucket");
 3672         LIST_FOREACH(z, &uma_cachezones, uz_link) {
 3673                 uma_zone_sumstat(z, &cachefree, &allocs, &frees, NULL);
 3674                 LIST_FOREACH(bucket, &z->uz_buckets, ub_link)
 3675                         cachefree += bucket->ub_cnt;
 3676                 db_printf("%18s %8ju %8jd %8d %12ju %8u\n",
 3677                     z->uz_name, (uintmax_t)z->uz_size,
 3678                     (intmax_t)(allocs - frees), cachefree,
 3679                     (uintmax_t)allocs, z->uz_count);
 3680                 if (db_pager_quit)
 3681                         return;
 3682         }
 3683 }
 3684 #endif  /* DDB */

Cache object: 7dc8dd80a37e57362e31f1650aa3e829


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.