The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_mbuf.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2004, 2005,
    3  *      Bosko Milekic <bmilekic@FreeBSD.org>.  All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice unmodified, this list of conditions and the following
   10  *    disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  *
   15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   25  * SUCH DAMAGE.
   26  */
   27 
   28 #include <sys/cdefs.h>
   29 __FBSDID("$FreeBSD: releng/7.3/sys/kern/kern_mbuf.c 196838 2009-09-04 19:59:32Z jhb $");
   30 
   31 #include "opt_mac.h"
   32 #include "opt_param.h"
   33 
   34 #include <sys/param.h>
   35 #include <sys/malloc.h>
   36 #include <sys/systm.h>
   37 #include <sys/mbuf.h>
   38 #include <sys/domain.h>
   39 #include <sys/eventhandler.h>
   40 #include <sys/kernel.h>
   41 #include <sys/protosw.h>
   42 #include <sys/smp.h>
   43 #include <sys/sysctl.h>
   44 
   45 #include <security/mac/mac_framework.h>
   46 
   47 #include <vm/vm.h>
   48 #include <vm/vm_extern.h>
   49 #include <vm/vm_kern.h>
   50 #include <vm/vm_page.h>
   51 #include <vm/uma.h>
   52 #include <vm/uma_int.h>
   53 #include <vm/uma_dbg.h>
   54 
   55 /*
   56  * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA
   57  * Zones.
   58  *
   59  * Mbuf Clusters (2K, contiguous) are allocated from the Cluster
   60  * Zone.  The Zone can be capped at kern.ipc.nmbclusters, if the
   61  * administrator so desires.
   62  *
   63  * Mbufs are allocated from a UMA Master Zone called the Mbuf
   64  * Zone.
   65  *
   66  * Additionally, FreeBSD provides a Packet Zone, which it
   67  * configures as a Secondary Zone to the Mbuf Master Zone,
   68  * thus sharing backend Slab kegs with the Mbuf Master Zone.
   69  *
   70  * Thus common-case allocations and locking are simplified:
   71  *
   72  *  m_clget()                m_getcl()
   73  *    |                         |
   74  *    |   .------------>[(Packet Cache)]    m_get(), m_gethdr()
   75  *    |   |             [     Packet   ]            |
   76  *  [(Cluster Cache)]   [    Secondary ]   [ (Mbuf Cache)     ]
   77  *  [ Cluster Zone  ]   [     Zone     ]   [ Mbuf Master Zone ]
   78  *        |                       \________         |
   79  *  [ Cluster Keg   ]                      \       /
   80  *        |                              [ Mbuf Keg   ]
   81  *  [ Cluster Slabs ]                         |
   82  *        |                              [ Mbuf Slabs ]
   83  *         \____________(VM)_________________/
   84  *
   85  *
   86  * Whenever an object is allocated with uma_zalloc() out of
   87  * one of the Zones its _ctor_ function is executed.  The same
   88  * for any deallocation through uma_zfree() the _dtor_ function
   89  * is executed.
   90  *
   91  * Caches are per-CPU and are filled from the Master Zone.
   92  *
   93  * Whenever an object is allocated from the underlying global
   94  * memory pool it gets pre-initialized with the _zinit_ functions.
   95  * When the Keg's are overfull objects get decomissioned with
   96  * _zfini_ functions and free'd back to the global memory pool.
   97  *
   98  */
   99 
  100 int nmbclusters;                /* limits number of mbuf clusters */
  101 int nmbjumbop;                  /* limits number of page size jumbo clusters */
  102 int nmbjumbo9;                  /* limits number of 9k jumbo clusters */
  103 int nmbjumbo16;                 /* limits number of 16k jumbo clusters */
  104 struct mbstat mbstat;
  105 
  106 /*
  107  * tunable_mbinit() has to be run before init_maxsockets() thus
  108  * the SYSINIT order below is SI_ORDER_MIDDLE while init_maxsockets()
  109  * runs at SI_ORDER_ANY.
  110  */
  111 static void
  112 tunable_mbinit(void *dummy)
  113 {
  114 
  115         /* This has to be done before VM init. */
  116         nmbclusters = 1024 + maxusers * 64;
  117         nmbjumbop = nmbclusters / 2;
  118         nmbjumbo9 = nmbjumbop / 2;
  119         nmbjumbo16 = nmbjumbo9 / 2;
  120         TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
  121 }
  122 SYSINIT(tunable_mbinit, SI_SUB_TUNABLES, SI_ORDER_MIDDLE, tunable_mbinit, NULL);
  123 
  124 /* XXX: These should be tuneables. Can't change UMA limits on the fly. */
  125 static int
  126 sysctl_nmbclusters(SYSCTL_HANDLER_ARGS)
  127 {
  128         int error, newnmbclusters;
  129 
  130         newnmbclusters = nmbclusters;
  131         error = sysctl_handle_int(oidp, &newnmbclusters, 0, req); 
  132         if (error == 0 && req->newptr) {
  133                 if (newnmbclusters > nmbclusters) {
  134                         nmbclusters = newnmbclusters;
  135                         uma_zone_set_max(zone_clust, nmbclusters);
  136                         EVENTHANDLER_INVOKE(nmbclusters_change);
  137                 } else
  138                         error = EINVAL;
  139         }
  140         return (error);
  141 }
  142 SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW,
  143 &nmbclusters, 0, sysctl_nmbclusters, "IU",
  144     "Maximum number of mbuf clusters allowed");
  145 
  146 static int
  147 sysctl_nmbjumbop(SYSCTL_HANDLER_ARGS)
  148 {
  149         int error, newnmbjumbop;
  150 
  151         newnmbjumbop = nmbjumbop;
  152         error = sysctl_handle_int(oidp, &newnmbjumbop, 0, req); 
  153         if (error == 0 && req->newptr) {
  154                 if (newnmbjumbop> nmbjumbop) {
  155                         nmbjumbop = newnmbjumbop;
  156                         uma_zone_set_max(zone_jumbop, nmbjumbop);
  157                 } else
  158                         error = EINVAL;
  159         }
  160         return (error);
  161 }
  162 SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbop, CTLTYPE_INT|CTLFLAG_RW,
  163 &nmbjumbop, 0, sysctl_nmbjumbop, "IU",
  164          "Maximum number of mbuf page size jumbo clusters allowed");
  165 
  166 
  167 static int
  168 sysctl_nmbjumbo9(SYSCTL_HANDLER_ARGS)
  169 {
  170         int error, newnmbjumbo9;
  171 
  172         newnmbjumbo9 = nmbjumbo9;
  173         error = sysctl_handle_int(oidp, &newnmbjumbo9, 0, req); 
  174         if (error == 0 && req->newptr) {
  175                 if (newnmbjumbo9> nmbjumbo9) {
  176                         nmbjumbo9 = newnmbjumbo9;
  177                         uma_zone_set_max(zone_jumbo9, nmbjumbo9);
  178                 } else
  179                         error = EINVAL;
  180         }
  181         return (error);
  182 }
  183 SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo9, CTLTYPE_INT|CTLFLAG_RW,
  184 &nmbjumbo9, 0, sysctl_nmbjumbo9, "IU",
  185         "Maximum number of mbuf 9k jumbo clusters allowed"); 
  186 
  187 static int
  188 sysctl_nmbjumbo16(SYSCTL_HANDLER_ARGS)
  189 {
  190         int error, newnmbjumbo16;
  191 
  192         newnmbjumbo16 = nmbjumbo16;
  193         error = sysctl_handle_int(oidp, &newnmbjumbo16, 0, req); 
  194         if (error == 0 && req->newptr) {
  195                 if (newnmbjumbo16> nmbjumbo16) {
  196                         nmbjumbo16 = newnmbjumbo16;
  197                         uma_zone_set_max(zone_jumbo16, nmbjumbo16);
  198                 } else
  199                         error = EINVAL;
  200         }
  201         return (error);
  202 }
  203 SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo16, CTLTYPE_INT|CTLFLAG_RW,
  204 &nmbjumbo16, 0, sysctl_nmbjumbo16, "IU",
  205     "Maximum number of mbuf 16k jumbo clusters allowed");
  206 
  207 
  208 
  209 SYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbstat, CTLFLAG_RD, &mbstat, mbstat,
  210     "Mbuf general information and statistics");
  211 
  212 /*
  213  * Zones from which we allocate.
  214  */
  215 uma_zone_t      zone_mbuf;
  216 uma_zone_t      zone_clust;
  217 uma_zone_t      zone_pack;
  218 uma_zone_t      zone_jumbop;
  219 uma_zone_t      zone_jumbo9;
  220 uma_zone_t      zone_jumbo16;
  221 uma_zone_t      zone_ext_refcnt;
  222 
  223 /*
  224  * Local prototypes.
  225  */
  226 static int      mb_ctor_mbuf(void *, int, void *, int);
  227 static int      mb_ctor_clust(void *, int, void *, int);
  228 static int      mb_ctor_pack(void *, int, void *, int);
  229 static void     mb_dtor_mbuf(void *, int, void *);
  230 static void     mb_dtor_clust(void *, int, void *);
  231 static void     mb_dtor_pack(void *, int, void *);
  232 static int      mb_zinit_pack(void *, int, int);
  233 static void     mb_zfini_pack(void *, int);
  234 
  235 static void     mb_reclaim(void *);
  236 static void     mbuf_init(void *);
  237 static void    *mbuf_jumbo_alloc(uma_zone_t, int, u_int8_t *, int);
  238 
  239 /* Ensure that MSIZE doesn't break dtom() - it must be a power of 2 */
  240 CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
  241 
  242 /*
  243  * Initialize FreeBSD Network buffer allocation.
  244  */
  245 SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL);
  246 static void
  247 mbuf_init(void *dummy)
  248 {
  249 
  250         /*
  251          * Configure UMA zones for Mbufs, Clusters, and Packets.
  252          */
  253         zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE,
  254             mb_ctor_mbuf, mb_dtor_mbuf,
  255 #ifdef INVARIANTS
  256             trash_init, trash_fini,
  257 #else
  258             NULL, NULL,
  259 #endif
  260             MSIZE - 1, UMA_ZONE_MAXBUCKET);
  261 
  262         zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
  263             mb_ctor_clust, mb_dtor_clust,
  264 #ifdef INVARIANTS
  265             trash_init, trash_fini,
  266 #else
  267             NULL, NULL,
  268 #endif
  269             UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
  270         if (nmbclusters > 0)
  271                 uma_zone_set_max(zone_clust, nmbclusters);
  272 
  273         zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
  274             mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
  275 
  276         /* Make jumbo frame zone too. Page size, 9k and 16k. */
  277         zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,
  278             mb_ctor_clust, mb_dtor_clust,
  279 #ifdef INVARIANTS
  280             trash_init, trash_fini,
  281 #else
  282             NULL, NULL,
  283 #endif
  284             UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
  285         if (nmbjumbop > 0)
  286                 uma_zone_set_max(zone_jumbop, nmbjumbop);
  287 
  288         zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
  289             mb_ctor_clust, mb_dtor_clust,
  290 #ifdef INVARIANTS
  291             trash_init, trash_fini,
  292 #else
  293             NULL, NULL,
  294 #endif
  295             UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
  296         if (nmbjumbo9 > 0)
  297                 uma_zone_set_max(zone_jumbo9, nmbjumbo9);
  298         uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc);
  299 
  300         zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
  301             mb_ctor_clust, mb_dtor_clust,
  302 #ifdef INVARIANTS
  303             trash_init, trash_fini,
  304 #else
  305             NULL, NULL,
  306 #endif
  307             UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
  308         if (nmbjumbo16 > 0)
  309                 uma_zone_set_max(zone_jumbo16, nmbjumbo16);
  310         uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc);
  311 
  312         zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int),
  313             NULL, NULL,
  314             NULL, NULL,
  315             UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
  316 
  317         /* uma_prealloc() goes here... */
  318 
  319         /*
  320          * Hook event handler for low-memory situation, used to
  321          * drain protocols and push data back to the caches (UMA
  322          * later pushes it back to VM).
  323          */
  324         EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
  325             EVENTHANDLER_PRI_FIRST);
  326 
  327         /*
  328          * [Re]set counters and local statistics knobs.
  329          * XXX Some of these should go and be replaced, but UMA stat
  330          * gathering needs to be revised.
  331          */
  332         mbstat.m_mbufs = 0;
  333         mbstat.m_mclusts = 0;
  334         mbstat.m_drain = 0;
  335         mbstat.m_msize = MSIZE;
  336         mbstat.m_mclbytes = MCLBYTES;
  337         mbstat.m_minclsize = MINCLSIZE;
  338         mbstat.m_mlen = MLEN;
  339         mbstat.m_mhlen = MHLEN;
  340         mbstat.m_numtypes = MT_NTYPES;
  341 
  342         mbstat.m_mcfail = mbstat.m_mpfail = 0;
  343         mbstat.sf_iocnt = 0;
  344         mbstat.sf_allocwait = mbstat.sf_allocfail = 0;
  345 }
  346 
  347 /*
  348  * UMA backend page allocator for the jumbo frame zones.
  349  *
  350  * Allocates kernel virtual memory that is backed by contiguous physical
  351  * pages.
  352  */
  353 static void *
  354 mbuf_jumbo_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
  355 {
  356 
  357         /* Inform UMA that this allocator uses kernel_map/object. */
  358         *flags = UMA_SLAB_KERNEL;
  359         return ((void *)kmem_alloc_contig(kernel_map, bytes, wait,
  360             (vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT));
  361 }
  362 
  363 /*
  364  * Constructor for Mbuf master zone.
  365  *
  366  * The 'arg' pointer points to a mb_args structure which
  367  * contains call-specific information required to support the
  368  * mbuf allocation API.  See mbuf.h.
  369  */
  370 static int
  371 mb_ctor_mbuf(void *mem, int size, void *arg, int how)
  372 {
  373         struct mbuf *m;
  374         struct mb_args *args;
  375 #ifdef MAC
  376         int error;
  377 #endif
  378         int flags;
  379         short type;
  380 
  381 #ifdef INVARIANTS
  382         trash_ctor(mem, size, arg, how);
  383 #endif
  384         m = (struct mbuf *)mem;
  385         args = (struct mb_args *)arg;
  386         flags = args->flags;
  387         type = args->type;
  388 
  389         /*
  390          * The mbuf is initialized later.  The caller has the
  391          * responsibility to set up any MAC labels too.
  392          */
  393         if (type == MT_NOINIT)
  394                 return (0);
  395 
  396         m->m_next = NULL;
  397         m->m_nextpkt = NULL;
  398         m->m_len = 0;
  399         m->m_flags = flags;
  400         m->m_type = type;
  401         if (flags & M_PKTHDR) {
  402                 m->m_data = m->m_pktdat;
  403                 m->m_pkthdr.rcvif = NULL;
  404                 m->m_pkthdr.len = 0;
  405                 m->m_pkthdr.header = NULL;
  406                 m->m_pkthdr.csum_flags = 0;
  407                 m->m_pkthdr.csum_data = 0;
  408                 m->m_pkthdr.tso_segsz = 0;
  409                 m->m_pkthdr.ether_vtag = 0;
  410                 SLIST_INIT(&m->m_pkthdr.tags);
  411 #ifdef MAC
  412                 /* If the label init fails, fail the alloc */
  413                 error = mac_init_mbuf(m, how);
  414                 if (error)
  415                         return (error);
  416 #endif
  417         } else
  418                 m->m_data = m->m_dat;
  419         return (0);
  420 }
  421 
  422 /*
  423  * The Mbuf master zone destructor.
  424  */
  425 static void
  426 mb_dtor_mbuf(void *mem, int size, void *arg)
  427 {
  428         struct mbuf *m;
  429         unsigned long flags; 
  430 
  431         m = (struct mbuf *)mem;
  432         flags = (unsigned long)arg;
  433         
  434         if ((flags & MB_NOTAGS) == 0 && (m->m_flags & M_PKTHDR) != 0)
  435                 m_tag_delete_chain(m, NULL);
  436         KASSERT((m->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__));
  437         KASSERT((m->m_flags & M_NOFREE) == 0, ("%s: M_NOFREE set", __func__));  
  438 #ifdef INVARIANTS
  439         trash_dtor(mem, size, arg);
  440 #endif
  441 }
  442 
  443 /*
  444  * The Mbuf Packet zone destructor.
  445  */
  446 static void
  447 mb_dtor_pack(void *mem, int size, void *arg)
  448 {
  449         struct mbuf *m;
  450 
  451         m = (struct mbuf *)mem;
  452         if ((m->m_flags & M_PKTHDR) != 0)
  453                 m_tag_delete_chain(m, NULL);
  454 
  455         /* Make sure we've got a clean cluster back. */
  456         KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
  457         KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__));
  458         KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__));
  459         KASSERT(m->m_ext.ext_args == NULL, ("%s: ext_args != NULL", __func__));
  460         KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__));
  461         KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__));
  462         KASSERT(*m->m_ext.ref_cnt == 1, ("%s: ref_cnt != 1", __func__));
  463 #ifdef INVARIANTS
  464         trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg);
  465 #endif
  466         /*
  467          * If there are processes blocked on zone_clust, waiting for pages to be freed up,
  468          * cause them to be woken up by draining the packet zone. We are exposed to a race here 
  469          * (in the check for the UMA_ZFLAG_FULL) where we might miss the flag set, but that is 
  470          * deliberate. We don't want to acquire the zone lock for every mbuf free.
  471          */
  472         if (uma_zone_exhausted_nolock(zone_clust))
  473                 zone_drain(zone_pack);
  474 }
  475 
  476 /*
  477  * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor.
  478  *
  479  * Here the 'arg' pointer points to the Mbuf which we
  480  * are configuring cluster storage for.  If 'arg' is
  481  * empty we allocate just the cluster without setting
  482  * the mbuf to it.  See mbuf.h.
  483  */
  484 static int
  485 mb_ctor_clust(void *mem, int size, void *arg, int how)
  486 {
  487         struct mbuf *m;
  488         u_int *refcnt;
  489         int type;
  490         uma_zone_t zone;
  491         
  492 #ifdef INVARIANTS
  493         trash_ctor(mem, size, arg, how);
  494 #endif
  495         switch (size) {
  496         case MCLBYTES:
  497                 type = EXT_CLUSTER;
  498                 zone = zone_clust;
  499                 break;
  500 #if MJUMPAGESIZE != MCLBYTES
  501         case MJUMPAGESIZE:
  502                 type = EXT_JUMBOP;
  503                 zone = zone_jumbop;
  504                 break;
  505 #endif
  506         case MJUM9BYTES:
  507                 type = EXT_JUMBO9;
  508                 zone = zone_jumbo9;
  509                 break;
  510         case MJUM16BYTES:
  511                 type = EXT_JUMBO16;
  512                 zone = zone_jumbo16;
  513                 break;
  514         default:
  515                 panic("unknown cluster size");
  516                 break;
  517         }
  518 
  519         m = (struct mbuf *)arg;
  520         refcnt = uma_find_refcnt(zone, mem);
  521         *refcnt = 1;                    
  522         if (m != NULL) {
  523                 m->m_ext.ext_buf = (caddr_t)mem;
  524                 m->m_data = m->m_ext.ext_buf;
  525                 m->m_flags |= M_EXT;
  526                 m->m_ext.ext_free = NULL;
  527                 m->m_ext.ext_args = NULL;
  528                 m->m_ext.ext_size = size;
  529                 m->m_ext.ext_type = type;
  530                 m->m_ext.ref_cnt = refcnt;
  531         }
  532 
  533         return (0);
  534 }
  535 
  536 /*
  537  * The Mbuf Cluster zone destructor.
  538  */
  539 static void
  540 mb_dtor_clust(void *mem, int size, void *arg)
  541 {
  542 #ifdef INVARIANTS
  543         uma_zone_t zone;
  544 
  545         zone = m_getzone(size);
  546         KASSERT(*(uma_find_refcnt(zone, mem)) <= 1,
  547                 ("%s: refcnt incorrect %u", __func__,
  548                  *(uma_find_refcnt(zone, mem))) );
  549 
  550         trash_dtor(mem, size, arg);
  551 #endif
  552 }
  553 
  554 /*
  555  * The Packet secondary zone's init routine, executed on the
  556  * object's transition from mbuf keg slab to zone cache.
  557  */
  558 static int
  559 mb_zinit_pack(void *mem, int size, int how)
  560 {
  561         struct mbuf *m;
  562 
  563         m = (struct mbuf *)mem;         /* m is virgin. */
  564         if (uma_zalloc_arg(zone_clust, m, how) == NULL ||
  565             m->m_ext.ext_buf == NULL)
  566                 return (ENOMEM);
  567         m->m_ext.ext_type = EXT_PACKET; /* Override. */
  568 #ifdef INVARIANTS
  569         trash_init(m->m_ext.ext_buf, MCLBYTES, how);
  570 #endif
  571         return (0);
  572 }
  573 
  574 /*
  575  * The Packet secondary zone's fini routine, executed on the
  576  * object's transition from zone cache to keg slab.
  577  */
  578 static void
  579 mb_zfini_pack(void *mem, int size)
  580 {
  581         struct mbuf *m;
  582 
  583         m = (struct mbuf *)mem;
  584 #ifdef INVARIANTS
  585         trash_fini(m->m_ext.ext_buf, MCLBYTES);
  586 #endif
  587         uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL);
  588 #ifdef INVARIANTS
  589         trash_dtor(mem, size, NULL);
  590 #endif
  591 }
  592 
  593 /*
  594  * The "packet" keg constructor.
  595  */
  596 static int
  597 mb_ctor_pack(void *mem, int size, void *arg, int how)
  598 {
  599         struct mbuf *m;
  600         struct mb_args *args;
  601 #ifdef MAC
  602         int error;
  603 #endif
  604         int flags;
  605         short type;
  606 
  607         m = (struct mbuf *)mem;
  608         args = (struct mb_args *)arg;
  609         flags = args->flags;
  610         type = args->type;
  611 
  612 #ifdef INVARIANTS
  613         trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how);
  614 #endif
  615         m->m_next = NULL;
  616         m->m_nextpkt = NULL;
  617         m->m_data = m->m_ext.ext_buf;
  618         m->m_len = 0;
  619         m->m_flags = (flags | M_EXT);
  620         m->m_type = type;
  621             
  622         if (flags & M_PKTHDR) {
  623                 m->m_pkthdr.rcvif = NULL;
  624                 m->m_pkthdr.len = 0;
  625                 m->m_pkthdr.header = NULL;
  626                 m->m_pkthdr.csum_flags = 0;
  627                 m->m_pkthdr.csum_data = 0;
  628                 m->m_pkthdr.tso_segsz = 0;
  629                 m->m_pkthdr.ether_vtag = 0;
  630                 SLIST_INIT(&m->m_pkthdr.tags);
  631 #ifdef MAC
  632                 /* If the label init fails, fail the alloc */
  633                 error = mac_init_mbuf(m, how);
  634                 if (error)
  635                         return (error);
  636 #endif
  637         }
  638         /* m_ext is already initialized. */
  639 
  640         return (0);
  641 }
  642 
  643 /*
  644  * This is the protocol drain routine.
  645  *
  646  * No locks should be held when this is called.  The drain routines have to
  647  * presently acquire some locks which raises the possibility of lock order
  648  * reversal.
  649  */
  650 static void
  651 mb_reclaim(void *junk)
  652 {
  653         struct domain *dp;
  654         struct protosw *pr;
  655 
  656         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL,
  657             "mb_reclaim()");
  658 
  659         for (dp = domains; dp != NULL; dp = dp->dom_next)
  660                 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
  661                         if (pr->pr_drain != NULL)
  662                                 (*pr->pr_drain)();
  663 }

Cache object: 2a145d428bef55257f08e35801b7211b


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.