vm_zone.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*
    2  * Copyright (c) 1997, 1998 John S. Dyson
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *      notice immediately at the beginning of the file, without modification,
   10  *      this list of conditions, and the following disclaimer.
   11  * 2. Absolutely no warranty of function or purpose is made by the author
   12  *      John S. Dyson.
   13  *
   14  * $FreeBSD$
   15  */
   16 
   17 #include <sys/param.h>
   18 #include <sys/systm.h>
   19 #include <sys/kernel.h>
   20 #include <sys/lock.h>
   21 #include <sys/malloc.h>
   22 #include <sys/sysctl.h>
   23 
   24 #include <vm/vm.h>
   25 #include <vm/vm_object.h>
   26 #include <vm/vm_prot.h>
   27 #include <vm/vm_page.h>
   28 #include <vm/vm_map.h>
   29 #include <vm/vm_kern.h>
   30 #include <vm/vm_extern.h>
   31 #include <vm/vm_zone.h>
   32 
   33 static MALLOC_DEFINE(M_ZONE, "ZONE", "Zone header");
   34 
   35 /*
   36  * This file comprises a very simple zone allocator.  This is used
   37  * in lieu of the malloc allocator, where needed or more optimal.
   38  *
   39  * Note that the initial implementation of this had coloring, and
   40  * absolutely no improvement (actually perf degradation) occurred.
   41  *
   42  * Note also that the zones are type stable.  The only restriction is
   43  * that the first two longwords of a data structure can be changed
   44  * between allocations.  Any data that must be stable between allocations
   45  * must reside in areas after the first two longwords.
   46  *
   47  * zinitna, zinit, zbootinit are the initialization routines.
   48  * zalloc, zfree, are the interrupt/lock unsafe allocation/free routines.
   49  * zalloci, zfreei, are the interrupt/lock safe allocation/free routines.
   50  */
   51 
   52 static struct vm_zone *zlist;
   53 static int sysctl_vm_zone SYSCTL_HANDLER_ARGS;
   54 static int zone_kmem_pages, zone_kern_pages, zone_kmem_kvaspace;
   55 
   56 /*
   57  * Create a zone, but don't allocate the zone structure.  If the
   58  * zone had been previously created by the zone boot code, initialize
   59  * various parts of the zone code.
   60  *
   61  * If waits are not allowed during allocation (e.g. during interrupt
   62  * code), a-priori allocate the kernel virtual space, and allocate
   63  * only pages when needed.
   64  *
   65  * Arguments:
   66  * z            pointer to zone structure.
   67  * obj          pointer to VM object (opt).
   68  * name         name of zone.
   69  * size         size of zone entries.
   70  * nentries     number of zone entries allocated (only ZONE_INTERRUPT.)
   71  * flags        ZONE_INTERRUPT -- items can be allocated at interrupt time.
   72  * zalloc       number of pages allocated when memory is needed.
   73  *
   74  * Note that when using ZONE_INTERRUPT, the size of the zone is limited
   75  * by the nentries argument.  The size of the memory allocatable is
   76  * unlimited if ZONE_INTERRUPT is not set.
   77  *
   78  */
   79 int
   80 zinitna(vm_zone_t z, vm_object_t obj, char *name, int size,
   81         int nentries, int flags, int zalloc)
   82 {
   83         int totsize;
   84 
   85         if ((z->zflags & ZONE_BOOT) == 0) {
   86                 z->zsize = (size + ZONE_ROUNDING - 1) & ~(ZONE_ROUNDING - 1);
   87                 simple_lock_init(&z->zlock);
   88                 z->zfreecnt = 0;
   89                 z->ztotal = 0;
   90                 z->zmax = 0;
   91                 z->zname = name;
   92                 z->znalloc = 0;
   93                 z->zitems = NULL;
   94 
   95                 if (zlist == 0) {
   96                         zlist = z;
   97                 } else {
   98                         z->znext = zlist;
   99                         zlist = z;
  100                 }
  101         }
  102 
  103         z->zflags |= flags;
  104 
  105         /*
  106          * If we cannot wait, allocate KVA space up front, and we will fill
  107          * in pages as needed.
  108          */
  109         if (z->zflags & ZONE_INTERRUPT) {
  110 
  111                 totsize = round_page(z->zsize * nentries);
  112                 zone_kmem_kvaspace += totsize;
  113 
  114                 z->zkva = kmem_alloc_pageable(kernel_map, totsize);
  115                 if (z->zkva == 0)
  116                         return 0;
  117 
  118                 z->zpagemax = totsize / PAGE_SIZE;
  119                 if (obj == NULL) {
  120                         z->zobj = vm_object_allocate(OBJT_DEFAULT, z->zpagemax);
  121                 } else {
  122                         z->zobj = obj;
  123                         _vm_object_allocate(OBJT_DEFAULT, z->zpagemax, obj);
  124                 }
  125                 z->zallocflag = VM_ALLOC_INTERRUPT;
  126                 z->zmax += nentries;
  127         } else {
  128                 z->zallocflag = VM_ALLOC_SYSTEM;
  129                 z->zmax = 0;
  130         }
  131 
  132 
  133         if (z->zsize > PAGE_SIZE)
  134                 z->zfreemin = 1;
  135         else
  136                 z->zfreemin = PAGE_SIZE / z->zsize;
  137 
  138         z->zpagecount = 0;
  139         if (zalloc)
  140                 z->zalloc = zalloc;
  141         else
  142                 z->zalloc = 1;
  143 
  144         return 1;
  145 }
  146 
  147 /*
  148  * Subroutine same as zinitna, except zone data structure is allocated
  149  * automatically by malloc.  This routine should normally be used, except
  150  * in certain tricky startup conditions in the VM system -- then
  151  * zbootinit and zinitna can be used.  Zinit is the standard zone
  152  * initialization call.
  153  */
  154 vm_zone_t
  155 zinit(char *name, int size, int nentries, int flags, int zalloc)
  156 {
  157         vm_zone_t z;
  158 
  159         z = (vm_zone_t) malloc(sizeof (struct vm_zone), M_ZONE, M_NOWAIT);
  160         if (z == NULL)
  161                 return NULL;
  162 
  163         z->zflags = 0;
  164         if (zinitna(z, NULL, name, size, nentries, flags, zalloc) == 0) {
  165                 free(z, M_ZONE);
  166                 return NULL;
  167         }
  168 
  169         return z;
  170 }
  171 
  172 /*
  173  * Initialize a zone before the system is fully up.  This routine should
  174  * only be called before full VM startup.
  175  */
  176 void
  177 zbootinit(vm_zone_t z, char *name, int size, void *item, int nitems)
  178 {
  179         int i;
  180 
  181         z->zname = name;
  182         z->zsize = size;
  183         z->zpagemax = 0;
  184         z->zobj = NULL;
  185         z->zflags = ZONE_BOOT;
  186         z->zfreemin = 0;
  187         z->zallocflag = 0;
  188         z->zpagecount = 0;
  189         z->zalloc = 0;
  190         z->znalloc = 0;
  191         simple_lock_init(&z->zlock);
  192 
  193         bzero(item, nitems * z->zsize);
  194         z->zitems = NULL;
  195         for (i = 0; i < nitems; i++) {
  196                 ((void **) item)[0] = z->zitems;
  197 #ifdef INVARIANTS
  198                 ((void **) item)[1] = (void *) ZENTRY_FREE;
  199 #endif
  200                 z->zitems = item;
  201                 (char *) item += z->zsize;
  202         }
  203         z->zfreecnt = nitems;
  204         z->zmax = nitems;
  205         z->ztotal = nitems;
  206 
  207         if (zlist == 0) {
  208                 zlist = z;
  209         } else {
  210                 z->znext = zlist;
  211                 zlist = z;
  212         }
  213 }
  214 
  215 /*
  216  * Zone critical region locks.
  217  */
  218 static __inline int
  219 zlock(vm_zone_t z)
  220 {
  221         int s;
  222 
  223         s = splhigh();
  224         simple_lock(&z->zlock);
  225         return s;
  226 }
  227 
  228 static __inline void
  229 zunlock(vm_zone_t z, int s)
  230 {
  231         simple_unlock(&z->zlock);
  232         splx(s);
  233 }
  234 
  235 /*
  236  * void *zalloc(vm_zone_t zone) --
  237  *      Returns an item from a specified zone.
  238  *
  239  * void zfree(vm_zone_t zone, void *item) --
  240  *  Frees an item back to a specified zone.
  241  *
  242  * void *zalloci(vm_zone_t zone) --
  243  *      Returns an item from a specified zone, interrupt safe.
  244  *
  245  * void zfreei(vm_zone_t zone, void *item) --
  246  *  Frees an item back to a specified zone, interrupt safe.
  247  *
  248  */
  249 
  250 /*
  251  * Zone allocator/deallocator.  These are interrupt / (or potentially SMP)
  252  * safe.  The raw zalloc/zfree routines are in the vm_zone header file,
  253  * and are not interrupt safe, but are fast.
  254  */
  255 void *
  256 zalloci(vm_zone_t z)
  257 {
  258         int s;
  259         void *item;
  260 
  261         s = zlock(z);
  262         item = _zalloc(z);
  263         zunlock(z, s);
  264         return item;
  265 }
  266 
  267 void
  268 zfreei(vm_zone_t z, void *item)
  269 {
  270         int s;
  271 
  272         s = zlock(z);
  273         _zfree(z, item);
  274         zunlock(z, s);
  275         return;
  276 }
  277 
  278 /*
  279  * Internal zone routine.  Not to be called from external (non vm_zone) code.
  280  */
  281 void *
  282 _zget(vm_zone_t z)
  283 {
  284         int i;
  285         vm_page_t m;
  286         int nitems, nbytes;
  287         void *item;
  288 
  289         if (z == NULL)
  290                 panic("zget: null zone");
  291 
  292         if (z->zflags & ZONE_INTERRUPT) {
  293                 item = (char *) z->zkva + z->zpagecount * PAGE_SIZE;
  294                 for (i = 0; ((i < z->zalloc) && (z->zpagecount < z->zpagemax));
  295                      i++) {
  296                         vm_offset_t zkva;
  297 
  298                         m = vm_page_alloc(z->zobj, z->zpagecount,
  299                                           z->zallocflag);
  300                         if (m == NULL)
  301                                 break;
  302 
  303                         zkva = z->zkva + z->zpagecount * PAGE_SIZE;
  304                         pmap_kenter(zkva, VM_PAGE_TO_PHYS(m));
  305                         bzero((caddr_t) zkva, PAGE_SIZE);
  306                         z->zpagecount++;
  307                         zone_kmem_pages++;
  308                 }
  309                 nitems = (i * PAGE_SIZE) / z->zsize;
  310         } else {
  311                 nbytes = z->zalloc * PAGE_SIZE;
  312 
  313                 /*
  314                  * Check to see if the kernel map is already locked.  We could allow
  315                  * for recursive locks, but that eliminates a valuable debugging
  316                  * mechanism, and opens up the kernel map for potential corruption
  317                  * by inconsistent data structure manipulation.  We could also use
  318                  * the interrupt allocation mechanism, but that has size limitations.
  319                  * Luckily, we have kmem_map that is a submap of kernel map available
  320                  * for memory allocation, and manipulation of that map doesn't affect
  321                  * the kernel map structures themselves.
  322                  *
  323                  * We can wait, so just do normal map allocation in the appropriate
  324                  * map.
  325                  */
  326                 if (lockstatus(&kernel_map->lock)) {
  327                         int s;
  328                         s = splvm();
  329 #ifdef SMP
  330                         simple_unlock(&z->zlock);
  331 #endif
  332                         item = (void *) kmem_malloc(kmem_map, nbytes, M_WAITOK);
  333 #ifdef SMP
  334                         simple_lock(&z->zlock);
  335 #endif
  336                         zone_kmem_pages += z->zalloc;
  337                         splx(s);
  338                 } else {
  339 #ifdef SMP
  340                         simple_unlock(&z->zlock);
  341 #endif
  342                         item = (void *) kmem_alloc(kernel_map, nbytes);
  343 #ifdef SMP
  344                         simple_lock(&z->zlock);
  345 #endif
  346                         zone_kern_pages += z->zalloc;
  347                 }
  348                 bzero(item, nbytes);
  349                 nitems = nbytes / z->zsize;
  350         }
  351         z->ztotal += nitems;
  352 
  353         /*
  354          * Save one for immediate allocation
  355          */
  356         if (nitems != 0) {
  357                 nitems -= 1;
  358                 for (i = 0; i < nitems; i++) {
  359                         ((void **) item)[0] = z->zitems;
  360 #ifdef INVARIANTS
  361                         ((void **) item)[1] = (void *) ZENTRY_FREE;
  362 #endif
  363                         z->zitems = item;
  364                         (char *) item += z->zsize;
  365                 }
  366                 z->zfreecnt += nitems;
  367         } else if (z->zfreecnt > 0) {
  368                 item = z->zitems;
  369                 z->zitems = ((void **) item)[0];
  370 #ifdef INVARIANTS
  371                 if (((void **) item)[1] != (void *) ZENTRY_FREE)
  372                         zerror(ZONE_ERROR_NOTFREE);
  373                 ((void **) item)[1] = 0;
  374 #endif
  375                 z->zfreecnt--;
  376         } else {
  377                 item = NULL;
  378         }
  379 
  380         return item;
  381 }
  382 
  383 static int
  384 sysctl_vm_zone SYSCTL_HANDLER_ARGS
  385 {
  386         int error=0;
  387         vm_zone_t curzone, nextzone;
  388         char tmpbuf[128];
  389         char tmpname[14];
  390 
  391         snprintf(tmpbuf, sizeof(tmpbuf),
  392             "\nITEM            SIZE     LIMIT    USED    FREE  REQUESTS\n");
  393         error = SYSCTL_OUT(req, tmpbuf, strlen(tmpbuf));
  394         if (error)
  395                 return (error);
  396 
  397         for (curzone = zlist; curzone; curzone = nextzone) {
  398                 int i;
  399                 int len;
  400                 int offset;
  401 
  402                 nextzone = curzone->znext;
  403                 len = strlen(curzone->zname);
  404                 if (len >= (sizeof(tmpname) - 1))
  405                         len = (sizeof(tmpname) - 1);
  406                 for(i = 0; i < sizeof(tmpname) - 1; i++)
  407                         tmpname[i] = ' ';
  408                 tmpname[i] = 0;
  409                 memcpy(tmpname, curzone->zname, len);
  410                 tmpname[len] = ':';
  411                 offset = 0;
  412                 if (curzone == zlist) {
  413                         offset = 1;
  414                         tmpbuf[0] = '\n';
  415                 }
  416 
  417                 snprintf(tmpbuf + offset, sizeof(tmpbuf) - offset,
  418                         "%s %6.6u, %8.8u, %6.6u, %6.6u, %8.8u\n",
  419                         tmpname, curzone->zsize, curzone->zmax,
  420                         (curzone->ztotal - curzone->zfreecnt),
  421                         curzone->zfreecnt, curzone->znalloc);
  422 
  423                 len = strlen((char *)tmpbuf);
  424                 if (nextzone == NULL)
  425                         tmpbuf[len - 1] = 0;
  426 
  427                 error = SYSCTL_OUT(req, tmpbuf, len);
  428 
  429                 if (error)
  430                         return (error);
  431         }
  432         return (0);
  433 }
  434 
  435 #ifdef INVARIANT_SUPPORT
  436 void
  437 zerror(int error)
  438 {
  439         char *msg;
  440 
  441         switch (error) {
  442         case ZONE_ERROR_INVALID:
  443                 msg = "zone: invalid zone";
  444                 break;
  445         case ZONE_ERROR_NOTFREE:
  446                 msg = "zone: entry not free";
  447                 break;
  448         case ZONE_ERROR_ALREADYFREE:
  449                 msg = "zone: freeing free entry";
  450                 break;
  451         default:
  452                 msg = "zone: invalid error";
  453                 break;
  454         }
  455         panic(msg);
  456 }
  457 #endif
  458 
  459 SYSCTL_OID(_vm, OID_AUTO, zone, CTLTYPE_STRING|CTLFLAG_RD, \
  460         NULL, 0, sysctl_vm_zone, "A", "Zone Info");
  461 
  462 SYSCTL_INT(_vm, OID_AUTO, zone_kmem_pages,
  463         CTLFLAG_RD, &zone_kmem_pages, 0, "Number of interrupt safe pages allocated by zone");
  464 SYSCTL_INT(_vm, OID_AUTO, zone_kmem_kvaspace,
  465         CTLFLAG_RD, &zone_kmem_kvaspace, 0, "KVA space allocated by zone");
  466 SYSCTL_INT(_vm, OID_AUTO, zone_kern_pages,
  467         CTLFLAG_RD, &zone_kern_pages, 0, "Number of non-interrupt safe pages allocated by zone");
Cache object: f07c0c9bf526dde675c644ff799c9a3b
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/vm/vm_zone.c

FreeBSD/Linux Kernel Cross Reference
sys/vm/vm_zone.c
