The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_mbuf.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2004, 2005,
    3  *      Bosko Milekic <bmilekic@FreeBSD.org>.  All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice unmodified, this list of conditions and the following
   10  *    disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  *
   15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   25  * SUCH DAMAGE.
   26  */
   27 
   28 #include <sys/cdefs.h>
   29 __FBSDID("$FreeBSD: releng/9.2/sys/kern/kern_mbuf.c 241468 2012-10-11 23:02:35Z np $");
   30 
   31 #include "opt_param.h"
   32 
   33 #include <sys/param.h>
   34 #include <sys/malloc.h>
   35 #include <sys/systm.h>
   36 #include <sys/mbuf.h>
   37 #include <sys/domain.h>
   38 #include <sys/eventhandler.h>
   39 #include <sys/kernel.h>
   40 #include <sys/protosw.h>
   41 #include <sys/smp.h>
   42 #include <sys/sysctl.h>
   43 
   44 #include <security/mac/mac_framework.h>
   45 
   46 #include <vm/vm.h>
   47 #include <vm/vm_extern.h>
   48 #include <vm/vm_kern.h>
   49 #include <vm/vm_page.h>
   50 #include <vm/uma.h>
   51 #include <vm/uma_int.h>
   52 #include <vm/uma_dbg.h>
   53 
   54 /*
   55  * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA
   56  * Zones.
   57  *
   58  * Mbuf Clusters (2K, contiguous) are allocated from the Cluster
   59  * Zone.  The Zone can be capped at kern.ipc.nmbclusters, if the
   60  * administrator so desires.
   61  *
   62  * Mbufs are allocated from a UMA Master Zone called the Mbuf
   63  * Zone.
   64  *
   65  * Additionally, FreeBSD provides a Packet Zone, which it
   66  * configures as a Secondary Zone to the Mbuf Master Zone,
   67  * thus sharing backend Slab kegs with the Mbuf Master Zone.
   68  *
   69  * Thus common-case allocations and locking are simplified:
   70  *
   71  *  m_clget()                m_getcl()
   72  *    |                         |
   73  *    |   .------------>[(Packet Cache)]    m_get(), m_gethdr()
   74  *    |   |             [     Packet   ]            |
   75  *  [(Cluster Cache)]   [    Secondary ]   [ (Mbuf Cache)     ]
   76  *  [ Cluster Zone  ]   [     Zone     ]   [ Mbuf Master Zone ]
   77  *        |                       \________         |
   78  *  [ Cluster Keg   ]                      \       /
   79  *        |                              [ Mbuf Keg   ]
   80  *  [ Cluster Slabs ]                         |
   81  *        |                              [ Mbuf Slabs ]
   82  *         \____________(VM)_________________/
   83  *
   84  *
   85  * Whenever an object is allocated with uma_zalloc() out of
   86  * one of the Zones its _ctor_ function is executed.  The same
   87  * for any deallocation through uma_zfree() the _dtor_ function
   88  * is executed.
   89  *
   90  * Caches are per-CPU and are filled from the Master Zone.
   91  *
   92  * Whenever an object is allocated from the underlying global
   93  * memory pool it gets pre-initialized with the _zinit_ functions.
   94  * When the Keg's are overfull objects get decomissioned with
   95  * _zfini_ functions and free'd back to the global memory pool.
   96  *
   97  */
   98 
   99 int nmbclusters;                /* limits number of mbuf clusters */
  100 int nmbjumbop;                  /* limits number of page size jumbo clusters */
  101 int nmbjumbo9;                  /* limits number of 9k jumbo clusters */
  102 int nmbjumbo16;                 /* limits number of 16k jumbo clusters */
  103 struct mbstat mbstat;
  104 
  105 /*
  106  * tunable_mbinit() has to be run before init_maxsockets() thus
  107  * the SYSINIT order below is SI_ORDER_MIDDLE while init_maxsockets()
  108  * runs at SI_ORDER_ANY.
  109  */
  110 static void
  111 tunable_mbinit(void *dummy)
  112 {
  113 
  114         /* This has to be done before VM init. */
  115         TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
  116         if (nmbclusters == 0)
  117                 nmbclusters = 1024 + maxusers * 64;
  118 
  119         TUNABLE_INT_FETCH("kern.ipc.nmbjumbop", &nmbjumbop);
  120         if (nmbjumbop == 0)
  121                 nmbjumbop = nmbclusters / 2;
  122 
  123         TUNABLE_INT_FETCH("kern.ipc.nmbjumbo9", &nmbjumbo9);
  124         if (nmbjumbo9 == 0)
  125                 nmbjumbo9 = nmbclusters / 4;
  126 
  127         TUNABLE_INT_FETCH("kern.ipc.nmbjumbo16", &nmbjumbo16);
  128         if (nmbjumbo16 == 0)
  129                 nmbjumbo16 = nmbclusters / 8;
  130 }
  131 SYSINIT(tunable_mbinit, SI_SUB_TUNABLES, SI_ORDER_MIDDLE, tunable_mbinit, NULL);
  132 
  133 static int
  134 sysctl_nmbclusters(SYSCTL_HANDLER_ARGS)
  135 {
  136         int error, newnmbclusters;
  137 
  138         newnmbclusters = nmbclusters;
  139         error = sysctl_handle_int(oidp, &newnmbclusters, 0, req); 
  140         if (error == 0 && req->newptr) {
  141                 if (newnmbclusters > nmbclusters) {
  142                         nmbclusters = newnmbclusters;
  143                         uma_zone_set_max(zone_clust, nmbclusters);
  144                         EVENTHANDLER_INVOKE(nmbclusters_change);
  145                 } else
  146                         error = EINVAL;
  147         }
  148         return (error);
  149 }
  150 SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW,
  151 &nmbclusters, 0, sysctl_nmbclusters, "IU",
  152     "Maximum number of mbuf clusters allowed");
  153 
  154 static int
  155 sysctl_nmbjumbop(SYSCTL_HANDLER_ARGS)
  156 {
  157         int error, newnmbjumbop;
  158 
  159         newnmbjumbop = nmbjumbop;
  160         error = sysctl_handle_int(oidp, &newnmbjumbop, 0, req); 
  161         if (error == 0 && req->newptr) {
  162                 if (newnmbjumbop> nmbjumbop) {
  163                         nmbjumbop = newnmbjumbop;
  164                         uma_zone_set_max(zone_jumbop, nmbjumbop);
  165                 } else
  166                         error = EINVAL;
  167         }
  168         return (error);
  169 }
  170 SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbop, CTLTYPE_INT|CTLFLAG_RW,
  171 &nmbjumbop, 0, sysctl_nmbjumbop, "IU",
  172          "Maximum number of mbuf page size jumbo clusters allowed");
  173 
  174 
  175 static int
  176 sysctl_nmbjumbo9(SYSCTL_HANDLER_ARGS)
  177 {
  178         int error, newnmbjumbo9;
  179 
  180         newnmbjumbo9 = nmbjumbo9;
  181         error = sysctl_handle_int(oidp, &newnmbjumbo9, 0, req); 
  182         if (error == 0 && req->newptr) {
  183                 if (newnmbjumbo9> nmbjumbo9) {
  184                         nmbjumbo9 = newnmbjumbo9;
  185                         uma_zone_set_max(zone_jumbo9, nmbjumbo9);
  186                 } else
  187                         error = EINVAL;
  188         }
  189         return (error);
  190 }
  191 SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo9, CTLTYPE_INT|CTLFLAG_RW,
  192 &nmbjumbo9, 0, sysctl_nmbjumbo9, "IU",
  193         "Maximum number of mbuf 9k jumbo clusters allowed"); 
  194 
  195 static int
  196 sysctl_nmbjumbo16(SYSCTL_HANDLER_ARGS)
  197 {
  198         int error, newnmbjumbo16;
  199 
  200         newnmbjumbo16 = nmbjumbo16;
  201         error = sysctl_handle_int(oidp, &newnmbjumbo16, 0, req); 
  202         if (error == 0 && req->newptr) {
  203                 if (newnmbjumbo16> nmbjumbo16) {
  204                         nmbjumbo16 = newnmbjumbo16;
  205                         uma_zone_set_max(zone_jumbo16, nmbjumbo16);
  206                 } else
  207                         error = EINVAL;
  208         }
  209         return (error);
  210 }
  211 SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo16, CTLTYPE_INT|CTLFLAG_RW,
  212 &nmbjumbo16, 0, sysctl_nmbjumbo16, "IU",
  213     "Maximum number of mbuf 16k jumbo clusters allowed");
  214 
  215 
  216 
  217 SYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbstat, CTLFLAG_RD, &mbstat, mbstat,
  218     "Mbuf general information and statistics");
  219 
  220 /*
  221  * Zones from which we allocate.
  222  */
  223 uma_zone_t      zone_mbuf;
  224 uma_zone_t      zone_clust;
  225 uma_zone_t      zone_pack;
  226 uma_zone_t      zone_jumbop;
  227 uma_zone_t      zone_jumbo9;
  228 uma_zone_t      zone_jumbo16;
  229 uma_zone_t      zone_ext_refcnt;
  230 
  231 /*
  232  * Local prototypes.
  233  */
  234 static int      mb_ctor_mbuf(void *, int, void *, int);
  235 static int      mb_ctor_clust(void *, int, void *, int);
  236 static int      mb_ctor_pack(void *, int, void *, int);
  237 static void     mb_dtor_mbuf(void *, int, void *);
  238 static void     mb_dtor_clust(void *, int, void *);
  239 static void     mb_dtor_pack(void *, int, void *);
  240 static int      mb_zinit_pack(void *, int, int);
  241 static void     mb_zfini_pack(void *, int);
  242 
  243 static void     mb_reclaim(void *);
  244 static void     mbuf_init(void *);
  245 static void    *mbuf_jumbo_alloc(uma_zone_t, int, uint8_t *, int);
  246 
  247 /* Ensure that MSIZE doesn't break dtom() - it must be a power of 2 */
  248 CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
  249 
  250 /*
  251  * Initialize FreeBSD Network buffer allocation.
  252  */
  253 SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL);
  254 static void
  255 mbuf_init(void *dummy)
  256 {
  257 
  258         /*
  259          * Configure UMA zones for Mbufs, Clusters, and Packets.
  260          */
  261         zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE,
  262             mb_ctor_mbuf, mb_dtor_mbuf,
  263 #ifdef INVARIANTS
  264             trash_init, trash_fini,
  265 #else
  266             NULL, NULL,
  267 #endif
  268             MSIZE - 1, UMA_ZONE_MAXBUCKET);
  269 
  270         zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
  271             mb_ctor_clust, mb_dtor_clust,
  272 #ifdef INVARIANTS
  273             trash_init, trash_fini,
  274 #else
  275             NULL, NULL,
  276 #endif
  277             UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
  278         if (nmbclusters > 0)
  279                 uma_zone_set_max(zone_clust, nmbclusters);
  280 
  281         zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
  282             mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
  283 
  284         /* Make jumbo frame zone too. Page size, 9k and 16k. */
  285         zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,
  286             mb_ctor_clust, mb_dtor_clust,
  287 #ifdef INVARIANTS
  288             trash_init, trash_fini,
  289 #else
  290             NULL, NULL,
  291 #endif
  292             UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
  293         if (nmbjumbop > 0)
  294                 uma_zone_set_max(zone_jumbop, nmbjumbop);
  295 
  296         zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
  297             mb_ctor_clust, mb_dtor_clust,
  298 #ifdef INVARIANTS
  299             trash_init, trash_fini,
  300 #else
  301             NULL, NULL,
  302 #endif
  303             UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
  304         if (nmbjumbo9 > 0)
  305                 uma_zone_set_max(zone_jumbo9, nmbjumbo9);
  306         uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc);
  307 
  308         zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
  309             mb_ctor_clust, mb_dtor_clust,
  310 #ifdef INVARIANTS
  311             trash_init, trash_fini,
  312 #else
  313             NULL, NULL,
  314 #endif
  315             UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
  316         if (nmbjumbo16 > 0)
  317                 uma_zone_set_max(zone_jumbo16, nmbjumbo16);
  318         uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc);
  319 
  320         zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int),
  321             NULL, NULL,
  322             NULL, NULL,
  323             UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
  324 
  325         /* uma_prealloc() goes here... */
  326 
  327         /*
  328          * Hook event handler for low-memory situation, used to
  329          * drain protocols and push data back to the caches (UMA
  330          * later pushes it back to VM).
  331          */
  332         EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
  333             EVENTHANDLER_PRI_FIRST);
  334 
  335         /*
  336          * [Re]set counters and local statistics knobs.
  337          * XXX Some of these should go and be replaced, but UMA stat
  338          * gathering needs to be revised.
  339          */
  340         mbstat.m_mbufs = 0;
  341         mbstat.m_mclusts = 0;
  342         mbstat.m_drain = 0;
  343         mbstat.m_msize = MSIZE;
  344         mbstat.m_mclbytes = MCLBYTES;
  345         mbstat.m_minclsize = MINCLSIZE;
  346         mbstat.m_mlen = MLEN;
  347         mbstat.m_mhlen = MHLEN;
  348         mbstat.m_numtypes = MT_NTYPES;
  349 
  350         mbstat.m_mcfail = mbstat.m_mpfail = 0;
  351         mbstat.sf_iocnt = 0;
  352         mbstat.sf_allocwait = mbstat.sf_allocfail = 0;
  353 }
  354 
  355 /*
  356  * UMA backend page allocator for the jumbo frame zones.
  357  *
  358  * Allocates kernel virtual memory that is backed by contiguous physical
  359  * pages.
  360  */
  361 static void *
  362 mbuf_jumbo_alloc(uma_zone_t zone, int bytes, uint8_t *flags, int wait)
  363 {
  364 
  365         /* Inform UMA that this allocator uses kernel_map/object. */
  366         *flags = UMA_SLAB_KERNEL;
  367         return ((void *)kmem_alloc_contig(kernel_map, bytes, wait,
  368             (vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT));
  369 }
  370 
  371 /*
  372  * Constructor for Mbuf master zone.
  373  *
  374  * The 'arg' pointer points to a mb_args structure which
  375  * contains call-specific information required to support the
  376  * mbuf allocation API.  See mbuf.h.
  377  */
  378 static int
  379 mb_ctor_mbuf(void *mem, int size, void *arg, int how)
  380 {
  381         struct mbuf *m;
  382         struct mb_args *args;
  383 #ifdef MAC
  384         int error;
  385 #endif
  386         int flags;
  387         short type;
  388 
  389 #ifdef INVARIANTS
  390         trash_ctor(mem, size, arg, how);
  391 #endif
  392         m = (struct mbuf *)mem;
  393         args = (struct mb_args *)arg;
  394         flags = args->flags;
  395         type = args->type;
  396 
  397         /*
  398          * The mbuf is initialized later.  The caller has the
  399          * responsibility to set up any MAC labels too.
  400          */
  401         if (type == MT_NOINIT)
  402                 return (0);
  403 
  404         m->m_next = NULL;
  405         m->m_nextpkt = NULL;
  406         m->m_len = 0;
  407         m->m_flags = flags;
  408         m->m_type = type;
  409         if (flags & M_PKTHDR) {
  410                 m->m_data = m->m_pktdat;
  411                 m->m_pkthdr.rcvif = NULL;
  412                 m->m_pkthdr.header = NULL;
  413                 m->m_pkthdr.len = 0;
  414                 m->m_pkthdr.csum_flags = 0;
  415                 m->m_pkthdr.csum_data = 0;
  416                 m->m_pkthdr.tso_segsz = 0;
  417                 m->m_pkthdr.ether_vtag = 0;
  418                 m->m_pkthdr.flowid = 0;
  419                 SLIST_INIT(&m->m_pkthdr.tags);
  420 #ifdef MAC
  421                 /* If the label init fails, fail the alloc */
  422                 error = mac_mbuf_init(m, how);
  423                 if (error)
  424                         return (error);
  425 #endif
  426         } else
  427                 m->m_data = m->m_dat;
  428         return (0);
  429 }
  430 
  431 /*
  432  * The Mbuf master zone destructor.
  433  */
  434 static void
  435 mb_dtor_mbuf(void *mem, int size, void *arg)
  436 {
  437         struct mbuf *m;
  438         unsigned long flags; 
  439 
  440         m = (struct mbuf *)mem;
  441         flags = (unsigned long)arg;
  442 
  443         if ((flags & MB_NOTAGS) == 0 && (m->m_flags & M_PKTHDR) != 0)
  444                 m_tag_delete_chain(m, NULL);
  445         KASSERT((m->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__));
  446         KASSERT((m->m_flags & M_NOFREE) == 0, ("%s: M_NOFREE set", __func__));
  447 #ifdef INVARIANTS
  448         trash_dtor(mem, size, arg);
  449 #endif
  450 }
  451 
  452 /*
  453  * The Mbuf Packet zone destructor.
  454  */
  455 static void
  456 mb_dtor_pack(void *mem, int size, void *arg)
  457 {
  458         struct mbuf *m;
  459 
  460         m = (struct mbuf *)mem;
  461         if ((m->m_flags & M_PKTHDR) != 0)
  462                 m_tag_delete_chain(m, NULL);
  463 
  464         /* Make sure we've got a clean cluster back. */
  465         KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
  466         KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__));
  467         KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__));
  468         KASSERT(m->m_ext.ext_arg1 == NULL, ("%s: ext_arg1 != NULL", __func__));
  469         KASSERT(m->m_ext.ext_arg2 == NULL, ("%s: ext_arg2 != NULL", __func__));
  470         KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__));
  471         KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__));
  472         KASSERT(*m->m_ext.ref_cnt == 1, ("%s: ref_cnt != 1", __func__));
  473 #ifdef INVARIANTS
  474         trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg);
  475 #endif
  476         /*
  477          * If there are processes blocked on zone_clust, waiting for pages
  478          * to be freed up, * cause them to be woken up by draining the
  479          * packet zone.  We are exposed to a race here * (in the check for
  480          * the UMA_ZFLAG_FULL) where we might miss the flag set, but that
  481          * is deliberate. We don't want to acquire the zone lock for every
  482          * mbuf free.
  483          */
  484         if (uma_zone_exhausted_nolock(zone_clust))
  485                 zone_drain(zone_pack);
  486 }
  487 
  488 /*
  489  * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor.
  490  *
  491  * Here the 'arg' pointer points to the Mbuf which we
  492  * are configuring cluster storage for.  If 'arg' is
  493  * empty we allocate just the cluster without setting
  494  * the mbuf to it.  See mbuf.h.
  495  */
  496 static int
  497 mb_ctor_clust(void *mem, int size, void *arg, int how)
  498 {
  499         struct mbuf *m;
  500         u_int *refcnt;
  501         int type;
  502         uma_zone_t zone;
  503 
  504 #ifdef INVARIANTS
  505         trash_ctor(mem, size, arg, how);
  506 #endif
  507         switch (size) {
  508         case MCLBYTES:
  509                 type = EXT_CLUSTER;
  510                 zone = zone_clust;
  511                 break;
  512 #if MJUMPAGESIZE != MCLBYTES
  513         case MJUMPAGESIZE:
  514                 type = EXT_JUMBOP;
  515                 zone = zone_jumbop;
  516                 break;
  517 #endif
  518         case MJUM9BYTES:
  519                 type = EXT_JUMBO9;
  520                 zone = zone_jumbo9;
  521                 break;
  522         case MJUM16BYTES:
  523                 type = EXT_JUMBO16;
  524                 zone = zone_jumbo16;
  525                 break;
  526         default:
  527                 panic("unknown cluster size");
  528                 break;
  529         }
  530 
  531         m = (struct mbuf *)arg;
  532         refcnt = uma_find_refcnt(zone, mem);
  533         *refcnt = 1;
  534         if (m != NULL) {
  535                 m->m_ext.ext_buf = (caddr_t)mem;
  536                 m->m_data = m->m_ext.ext_buf;
  537                 m->m_flags |= M_EXT;
  538                 m->m_ext.ext_free = NULL;
  539                 m->m_ext.ext_arg1 = NULL;
  540                 m->m_ext.ext_arg2 = NULL;
  541                 m->m_ext.ext_size = size;
  542                 m->m_ext.ext_type = type;
  543                 m->m_ext.ref_cnt = refcnt;
  544         }
  545 
  546         return (0);
  547 }
  548 
  549 /*
  550  * The Mbuf Cluster zone destructor.
  551  */
  552 static void
  553 mb_dtor_clust(void *mem, int size, void *arg)
  554 {
  555 #ifdef INVARIANTS
  556         uma_zone_t zone;
  557 
  558         zone = m_getzone(size);
  559         KASSERT(*(uma_find_refcnt(zone, mem)) <= 1,
  560                 ("%s: refcnt incorrect %u", __func__,
  561                  *(uma_find_refcnt(zone, mem))) );
  562 
  563         trash_dtor(mem, size, arg);
  564 #endif
  565 }
  566 
  567 /*
  568  * The Packet secondary zone's init routine, executed on the
  569  * object's transition from mbuf keg slab to zone cache.
  570  */
  571 static int
  572 mb_zinit_pack(void *mem, int size, int how)
  573 {
  574         struct mbuf *m;
  575 
  576         m = (struct mbuf *)mem;         /* m is virgin. */
  577         if (uma_zalloc_arg(zone_clust, m, how) == NULL ||
  578             m->m_ext.ext_buf == NULL)
  579                 return (ENOMEM);
  580         m->m_ext.ext_type = EXT_PACKET; /* Override. */
  581 #ifdef INVARIANTS
  582         trash_init(m->m_ext.ext_buf, MCLBYTES, how);
  583 #endif
  584         return (0);
  585 }
  586 
  587 /*
  588  * The Packet secondary zone's fini routine, executed on the
  589  * object's transition from zone cache to keg slab.
  590  */
  591 static void
  592 mb_zfini_pack(void *mem, int size)
  593 {
  594         struct mbuf *m;
  595 
  596         m = (struct mbuf *)mem;
  597 #ifdef INVARIANTS
  598         trash_fini(m->m_ext.ext_buf, MCLBYTES);
  599 #endif
  600         uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL);
  601 #ifdef INVARIANTS
  602         trash_dtor(mem, size, NULL);
  603 #endif
  604 }
  605 
  606 /*
  607  * The "packet" keg constructor.
  608  */
  609 static int
  610 mb_ctor_pack(void *mem, int size, void *arg, int how)
  611 {
  612         struct mbuf *m;
  613         struct mb_args *args;
  614 #ifdef MAC
  615         int error;
  616 #endif
  617         int flags;
  618         short type;
  619 
  620         m = (struct mbuf *)mem;
  621         args = (struct mb_args *)arg;
  622         flags = args->flags;
  623         type = args->type;
  624 
  625 #ifdef INVARIANTS
  626         trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how);
  627 #endif
  628         m->m_next = NULL;
  629         m->m_nextpkt = NULL;
  630         m->m_data = m->m_ext.ext_buf;
  631         m->m_len = 0;
  632         m->m_flags = (flags | M_EXT);
  633         m->m_type = type;
  634 
  635         if (flags & M_PKTHDR) {
  636                 m->m_pkthdr.rcvif = NULL;
  637                 m->m_pkthdr.len = 0;
  638                 m->m_pkthdr.header = NULL;
  639                 m->m_pkthdr.csum_flags = 0;
  640                 m->m_pkthdr.csum_data = 0;
  641                 m->m_pkthdr.tso_segsz = 0;
  642                 m->m_pkthdr.ether_vtag = 0;
  643                 m->m_pkthdr.flowid = 0;
  644                 SLIST_INIT(&m->m_pkthdr.tags);
  645 #ifdef MAC
  646                 /* If the label init fails, fail the alloc */
  647                 error = mac_mbuf_init(m, how);
  648                 if (error)
  649                         return (error);
  650 #endif
  651         }
  652         /* m_ext is already initialized. */
  653 
  654         return (0);
  655 }
  656 
  657 int
  658 m_pkthdr_init(struct mbuf *m, int how)
  659 {
  660 #ifdef MAC
  661         int error;
  662 #endif
  663         m->m_data = m->m_pktdat;
  664         SLIST_INIT(&m->m_pkthdr.tags);
  665         m->m_pkthdr.rcvif = NULL;
  666         m->m_pkthdr.header = NULL;
  667         m->m_pkthdr.len = 0;
  668         m->m_pkthdr.flowid = 0;
  669         m->m_pkthdr.csum_flags = 0;
  670         m->m_pkthdr.csum_data = 0;
  671         m->m_pkthdr.tso_segsz = 0;
  672         m->m_pkthdr.ether_vtag = 0;
  673 #ifdef MAC
  674         /* If the label init fails, fail the alloc */
  675         error = mac_mbuf_init(m, how);
  676         if (error)
  677                 return (error);
  678 #endif
  679 
  680         return (0);
  681 }
  682 
  683 /*
  684  * This is the protocol drain routine.
  685  *
  686  * No locks should be held when this is called.  The drain routines have to
  687  * presently acquire some locks which raises the possibility of lock order
  688  * reversal.
  689  */
  690 static void
  691 mb_reclaim(void *junk)
  692 {
  693         struct domain *dp;
  694         struct protosw *pr;
  695 
  696         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL,
  697             "mb_reclaim()");
  698 
  699         for (dp = domains; dp != NULL; dp = dp->dom_next)
  700                 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
  701                         if (pr->pr_drain != NULL)
  702                                 (*pr->pr_drain)();
  703 }

Cache object: 84a67ffaa16a28202495872dd93797a6


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.