The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_mbuf.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2004, 2005,
    3  *      Bosko Milekic <bmilekic@FreeBSD.org>.  All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice unmodified, this list of conditions and the following
   10  *    disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  *
   15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   25  * SUCH DAMAGE.
   26  */
   27 
   28 #include <sys/cdefs.h>
   29 __FBSDID("$FreeBSD: stable/10/sys/kern/kern_mbuf.c 302234 2016-06-27 21:50:30Z bdrewery $");
   30 
   31 #include "opt_param.h"
   32 
   33 #include <sys/param.h>
   34 #include <sys/malloc.h>
   35 #include <sys/systm.h>
   36 #include <sys/mbuf.h>
   37 #include <sys/domain.h>
   38 #include <sys/eventhandler.h>
   39 #include <sys/kernel.h>
   40 #include <sys/protosw.h>
   41 #include <sys/smp.h>
   42 #include <sys/sysctl.h>
   43 
   44 #include <security/mac/mac_framework.h>
   45 
   46 #include <vm/vm.h>
   47 #include <vm/vm_extern.h>
   48 #include <vm/vm_kern.h>
   49 #include <vm/vm_page.h>
   50 #include <vm/vm_map.h>
   51 #include <vm/uma.h>
   52 #include <vm/uma_int.h>
   53 #include <vm/uma_dbg.h>
   54 
   55 /*
   56  * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA
   57  * Zones.
   58  *
   59  * Mbuf Clusters (2K, contiguous) are allocated from the Cluster
   60  * Zone.  The Zone can be capped at kern.ipc.nmbclusters, if the
   61  * administrator so desires.
   62  *
   63  * Mbufs are allocated from a UMA Master Zone called the Mbuf
   64  * Zone.
   65  *
   66  * Additionally, FreeBSD provides a Packet Zone, which it
   67  * configures as a Secondary Zone to the Mbuf Master Zone,
   68  * thus sharing backend Slab kegs with the Mbuf Master Zone.
   69  *
   70  * Thus common-case allocations and locking are simplified:
   71  *
   72  *  m_clget()                m_getcl()
   73  *    |                         |
   74  *    |   .------------>[(Packet Cache)]    m_get(), m_gethdr()
   75  *    |   |             [     Packet   ]            |
   76  *  [(Cluster Cache)]   [    Secondary ]   [ (Mbuf Cache)     ]
   77  *  [ Cluster Zone  ]   [     Zone     ]   [ Mbuf Master Zone ]
   78  *        |                       \________         |
   79  *  [ Cluster Keg   ]                      \       /
   80  *        |                              [ Mbuf Keg   ]
   81  *  [ Cluster Slabs ]                         |
   82  *        |                              [ Mbuf Slabs ]
   83  *         \____________(VM)_________________/
   84  *
   85  *
   86  * Whenever an object is allocated with uma_zalloc() out of
   87  * one of the Zones its _ctor_ function is executed.  The same
   88  * for any deallocation through uma_zfree() the _dtor_ function
   89  * is executed.
   90  *
   91  * Caches are per-CPU and are filled from the Master Zone.
   92  *
   93  * Whenever an object is allocated from the underlying global
   94  * memory pool it gets pre-initialized with the _zinit_ functions.
   95  * When the Keg's are overfull objects get decommissioned with
   96  * _zfini_ functions and free'd back to the global memory pool.
   97  *
   98  */
   99 
  100 int nmbufs;                     /* limits number of mbufs */
  101 int nmbclusters;                /* limits number of mbuf clusters */
  102 int nmbjumbop;                  /* limits number of page size jumbo clusters */
  103 int nmbjumbo9;                  /* limits number of 9k jumbo clusters */
  104 int nmbjumbo16;                 /* limits number of 16k jumbo clusters */
  105 
  106 static quad_t maxmbufmem;       /* overall real memory limit for all mbufs */
  107 
  108 SYSCTL_QUAD(_kern_ipc, OID_AUTO, maxmbufmem, CTLFLAG_RDTUN, &maxmbufmem, 0,
  109     "Maximum real memory allocatable to various mbuf types");
  110 
  111 /*
  112  * tunable_mbinit() has to be run before any mbuf allocations are done.
  113  */
  114 static void
  115 tunable_mbinit(void *dummy)
  116 {
  117         quad_t realmem;
  118 
  119         /*
  120          * The default limit for all mbuf related memory is 1/2 of all
  121          * available kernel memory (physical or kmem).
  122          * At most it can be 3/4 of available kernel memory.
  123          */
  124         realmem = qmin((quad_t)physmem * PAGE_SIZE, vm_kmem_size);
  125         maxmbufmem = realmem / 2;
  126         TUNABLE_QUAD_FETCH("kern.ipc.maxmbufmem", &maxmbufmem);
  127         if (maxmbufmem > realmem / 4 * 3)
  128                 maxmbufmem = realmem / 4 * 3;
  129 
  130         TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
  131         if (nmbclusters == 0)
  132                 nmbclusters = maxmbufmem / MCLBYTES / 4;
  133 
  134         TUNABLE_INT_FETCH("kern.ipc.nmbjumbop", &nmbjumbop);
  135         if (nmbjumbop == 0)
  136                 nmbjumbop = maxmbufmem / MJUMPAGESIZE / 4;
  137 
  138         TUNABLE_INT_FETCH("kern.ipc.nmbjumbo9", &nmbjumbo9);
  139         if (nmbjumbo9 == 0)
  140                 nmbjumbo9 = maxmbufmem / MJUM9BYTES / 6;
  141 
  142         TUNABLE_INT_FETCH("kern.ipc.nmbjumbo16", &nmbjumbo16);
  143         if (nmbjumbo16 == 0)
  144                 nmbjumbo16 = maxmbufmem / MJUM16BYTES / 6;
  145 
  146         /*
  147          * We need at least as many mbufs as we have clusters of
  148          * the various types added together.
  149          */
  150         TUNABLE_INT_FETCH("kern.ipc.nmbufs", &nmbufs);
  151         if (nmbufs < nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16)
  152                 nmbufs = lmax(maxmbufmem / MSIZE / 5,
  153                     nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16);
  154 }
  155 SYSINIT(tunable_mbinit, SI_SUB_KMEM, SI_ORDER_MIDDLE, tunable_mbinit, NULL);
  156 
  157 static int
  158 sysctl_nmbclusters(SYSCTL_HANDLER_ARGS)
  159 {
  160         int error, newnmbclusters;
  161 
  162         newnmbclusters = nmbclusters;
  163         error = sysctl_handle_int(oidp, &newnmbclusters, 0, req);
  164         if (error == 0 && req->newptr && newnmbclusters != nmbclusters) {
  165                 if (newnmbclusters > nmbclusters &&
  166                     nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
  167                         nmbclusters = newnmbclusters;
  168                         nmbclusters = uma_zone_set_max(zone_clust, nmbclusters);
  169                         EVENTHANDLER_INVOKE(nmbclusters_change);
  170                 } else
  171                         error = EINVAL;
  172         }
  173         return (error);
  174 }
  175 SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW,
  176 &nmbclusters, 0, sysctl_nmbclusters, "IU",
  177     "Maximum number of mbuf clusters allowed");
  178 
  179 static int
  180 sysctl_nmbjumbop(SYSCTL_HANDLER_ARGS)
  181 {
  182         int error, newnmbjumbop;
  183 
  184         newnmbjumbop = nmbjumbop;
  185         error = sysctl_handle_int(oidp, &newnmbjumbop, 0, req);
  186         if (error == 0 && req->newptr && newnmbjumbop != nmbjumbop) {
  187                 if (newnmbjumbop > nmbjumbop &&
  188                     nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
  189                         nmbjumbop = newnmbjumbop;
  190                         nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop);
  191                 } else
  192                         error = EINVAL;
  193         }
  194         return (error);
  195 }
  196 SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbop, CTLTYPE_INT|CTLFLAG_RW,
  197 &nmbjumbop, 0, sysctl_nmbjumbop, "IU",
  198     "Maximum number of mbuf page size jumbo clusters allowed");
  199 
  200 static int
  201 sysctl_nmbjumbo9(SYSCTL_HANDLER_ARGS)
  202 {
  203         int error, newnmbjumbo9;
  204 
  205         newnmbjumbo9 = nmbjumbo9;
  206         error = sysctl_handle_int(oidp, &newnmbjumbo9, 0, req);
  207         if (error == 0 && req->newptr && newnmbjumbo9 != nmbjumbo9) {
  208                 if (newnmbjumbo9 > nmbjumbo9 &&
  209                     nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
  210                         nmbjumbo9 = newnmbjumbo9;
  211                         nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9);
  212                 } else
  213                         error = EINVAL;
  214         }
  215         return (error);
  216 }
  217 SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo9, CTLTYPE_INT|CTLFLAG_RW,
  218 &nmbjumbo9, 0, sysctl_nmbjumbo9, "IU",
  219     "Maximum number of mbuf 9k jumbo clusters allowed");
  220 
  221 static int
  222 sysctl_nmbjumbo16(SYSCTL_HANDLER_ARGS)
  223 {
  224         int error, newnmbjumbo16;
  225 
  226         newnmbjumbo16 = nmbjumbo16;
  227         error = sysctl_handle_int(oidp, &newnmbjumbo16, 0, req);
  228         if (error == 0 && req->newptr && newnmbjumbo16 != nmbjumbo16) {
  229                 if (newnmbjumbo16 > nmbjumbo16 &&
  230                     nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
  231                         nmbjumbo16 = newnmbjumbo16;
  232                         nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16);
  233                 } else
  234                         error = EINVAL;
  235         }
  236         return (error);
  237 }
  238 SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo16, CTLTYPE_INT|CTLFLAG_RW,
  239 &nmbjumbo16, 0, sysctl_nmbjumbo16, "IU",
  240     "Maximum number of mbuf 16k jumbo clusters allowed");
  241 
  242 static int
  243 sysctl_nmbufs(SYSCTL_HANDLER_ARGS)
  244 {
  245         int error, newnmbufs;
  246 
  247         newnmbufs = nmbufs;
  248         error = sysctl_handle_int(oidp, &newnmbufs, 0, req);
  249         if (error == 0 && req->newptr && newnmbufs != nmbufs) {
  250                 if (newnmbufs > nmbufs) {
  251                         nmbufs = newnmbufs;
  252                         nmbufs = uma_zone_set_max(zone_mbuf, nmbufs);
  253                         EVENTHANDLER_INVOKE(nmbufs_change);
  254                 } else
  255                         error = EINVAL;
  256         }
  257         return (error);
  258 }
  259 SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbufs, CTLTYPE_INT|CTLFLAG_RW,
  260 &nmbufs, 0, sysctl_nmbufs, "IU",
  261     "Maximum number of mbufs allowed");
  262 
  263 /*
  264  * Zones from which we allocate.
  265  */
  266 uma_zone_t      zone_mbuf;
  267 uma_zone_t      zone_clust;
  268 uma_zone_t      zone_pack;
  269 uma_zone_t      zone_jumbop;
  270 uma_zone_t      zone_jumbo9;
  271 uma_zone_t      zone_jumbo16;
  272 uma_zone_t      zone_ext_refcnt;
  273 
  274 /*
  275  * Local prototypes.
  276  */
  277 static int      mb_ctor_mbuf(void *, int, void *, int);
  278 static int      mb_ctor_clust(void *, int, void *, int);
  279 static int      mb_ctor_pack(void *, int, void *, int);
  280 static void     mb_dtor_mbuf(void *, int, void *);
  281 static void     mb_dtor_clust(void *, int, void *);
  282 static void     mb_dtor_pack(void *, int, void *);
  283 static int      mb_zinit_pack(void *, int, int);
  284 static void     mb_zfini_pack(void *, int);
  285 
  286 static void     mb_reclaim(void *);
  287 static void    *mbuf_jumbo_alloc(uma_zone_t, vm_size_t, uint8_t *, int);
  288 
  289 /* Ensure that MSIZE is a power of 2. */
  290 CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
  291 
  292 /*
  293  * Initialize FreeBSD Network buffer allocation.
  294  */
  295 static void
  296 mbuf_init(void *dummy)
  297 {
  298 
  299         /*
  300          * Configure UMA zones for Mbufs, Clusters, and Packets.
  301          */
  302         zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE,
  303             mb_ctor_mbuf, mb_dtor_mbuf,
  304 #ifdef INVARIANTS
  305             trash_init, trash_fini,
  306 #else
  307             NULL, NULL,
  308 #endif
  309             MSIZE - 1, UMA_ZONE_MAXBUCKET);
  310         if (nmbufs > 0)
  311                 nmbufs = uma_zone_set_max(zone_mbuf, nmbufs);
  312         uma_zone_set_warning(zone_mbuf, "kern.ipc.nmbufs limit reached");
  313 
  314         zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
  315             mb_ctor_clust, mb_dtor_clust,
  316 #ifdef INVARIANTS
  317             trash_init, trash_fini,
  318 #else
  319             NULL, NULL,
  320 #endif
  321             UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
  322         if (nmbclusters > 0)
  323                 nmbclusters = uma_zone_set_max(zone_clust, nmbclusters);
  324         uma_zone_set_warning(zone_clust, "kern.ipc.nmbclusters limit reached");
  325 
  326         zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
  327             mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
  328 
  329         /* Make jumbo frame zone too. Page size, 9k and 16k. */
  330         zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,
  331             mb_ctor_clust, mb_dtor_clust,
  332 #ifdef INVARIANTS
  333             trash_init, trash_fini,
  334 #else
  335             NULL, NULL,
  336 #endif
  337             UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
  338         if (nmbjumbop > 0)
  339                 nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop);
  340         uma_zone_set_warning(zone_jumbop, "kern.ipc.nmbjumbop limit reached");
  341 
  342         zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
  343             mb_ctor_clust, mb_dtor_clust,
  344 #ifdef INVARIANTS
  345             trash_init, trash_fini,
  346 #else
  347             NULL, NULL,
  348 #endif
  349             UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
  350         uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc);
  351         if (nmbjumbo9 > 0)
  352                 nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9);
  353         uma_zone_set_warning(zone_jumbo9, "kern.ipc.nmbjumbo9 limit reached");
  354 
  355         zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
  356             mb_ctor_clust, mb_dtor_clust,
  357 #ifdef INVARIANTS
  358             trash_init, trash_fini,
  359 #else
  360             NULL, NULL,
  361 #endif
  362             UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
  363         uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc);
  364         if (nmbjumbo16 > 0)
  365                 nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16);
  366         uma_zone_set_warning(zone_jumbo16, "kern.ipc.nmbjumbo16 limit reached");
  367 
  368         zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int),
  369             NULL, NULL,
  370             NULL, NULL,
  371             UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
  372 
  373         /* uma_prealloc() goes here... */
  374 
  375         /*
  376          * Hook event handler for low-memory situation, used to
  377          * drain protocols and push data back to the caches (UMA
  378          * later pushes it back to VM).
  379          */
  380         EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
  381             EVENTHANDLER_PRI_FIRST);
  382 }
  383 SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL);
  384 
  385 /*
  386  * UMA backend page allocator for the jumbo frame zones.
  387  *
  388  * Allocates kernel virtual memory that is backed by contiguous physical
  389  * pages.
  390  */
  391 static void *
  392 mbuf_jumbo_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *flags, int wait)
  393 {
  394 
  395         /* Inform UMA that this allocator uses kernel_map/object. */
  396         *flags = UMA_SLAB_KERNEL;
  397         return ((void *)kmem_alloc_contig(kernel_arena, bytes, wait,
  398             (vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT));
  399 }
  400 
  401 /*
  402  * Constructor for Mbuf master zone.
  403  *
  404  * The 'arg' pointer points to a mb_args structure which
  405  * contains call-specific information required to support the
  406  * mbuf allocation API.  See mbuf.h.
  407  */
  408 static int
  409 mb_ctor_mbuf(void *mem, int size, void *arg, int how)
  410 {
  411         struct mbuf *m;
  412         struct mb_args *args;
  413         int error;
  414         int flags;
  415         short type;
  416 
  417 #ifdef INVARIANTS
  418         trash_ctor(mem, size, arg, how);
  419 #endif
  420         args = (struct mb_args *)arg;
  421         type = args->type;
  422 
  423         /*
  424          * The mbuf is initialized later.  The caller has the
  425          * responsibility to set up any MAC labels too.
  426          */
  427         if (type == MT_NOINIT)
  428                 return (0);
  429 
  430         m = (struct mbuf *)mem;
  431         flags = args->flags;
  432         MPASS((flags & M_NOFREE) == 0);
  433 
  434         error = m_init(m, NULL, size, how, type, flags);
  435 
  436         return (error);
  437 }
  438 
  439 /*
  440  * The Mbuf master zone destructor.
  441  */
  442 static void
  443 mb_dtor_mbuf(void *mem, int size, void *arg)
  444 {
  445         struct mbuf *m;
  446         unsigned long flags;
  447 
  448         m = (struct mbuf *)mem;
  449         flags = (unsigned long)arg;
  450 
  451         if ((m->m_flags & M_PKTHDR) && !SLIST_EMPTY(&m->m_pkthdr.tags))
  452                 m_tag_delete_chain(m, NULL);
  453         KASSERT((m->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__));
  454         KASSERT((m->m_flags & M_NOFREE) == 0, ("%s: M_NOFREE set", __func__));
  455 #ifdef INVARIANTS
  456         trash_dtor(mem, size, arg);
  457 #endif
  458 }
  459 
  460 /*
  461  * The Mbuf Packet zone destructor.
  462  */
  463 static void
  464 mb_dtor_pack(void *mem, int size, void *arg)
  465 {
  466         struct mbuf *m;
  467 
  468         m = (struct mbuf *)mem;
  469         if ((m->m_flags & M_PKTHDR) != 0)
  470                 m_tag_delete_chain(m, NULL);
  471 
  472         /* Make sure we've got a clean cluster back. */
  473         KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
  474         KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__));
  475         KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__));
  476         KASSERT(m->m_ext.ext_arg1 == NULL, ("%s: ext_arg1 != NULL", __func__));
  477         KASSERT(m->m_ext.ext_arg2 == NULL, ("%s: ext_arg2 != NULL", __func__));
  478         KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__));
  479         KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__));
  480         KASSERT(*m->m_ext.ref_cnt == 1, ("%s: ref_cnt != 1", __func__));
  481 #ifdef INVARIANTS
  482         trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg);
  483 #endif
  484         /*
  485          * If there are processes blocked on zone_clust, waiting for pages
  486          * to be freed up, * cause them to be woken up by draining the
  487          * packet zone.  We are exposed to a race here * (in the check for
  488          * the UMA_ZFLAG_FULL) where we might miss the flag set, but that
  489          * is deliberate. We don't want to acquire the zone lock for every
  490          * mbuf free.
  491          */
  492         if (uma_zone_exhausted_nolock(zone_clust))
  493                 zone_drain(zone_pack);
  494 }
  495 
  496 /*
  497  * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor.
  498  *
  499  * Here the 'arg' pointer points to the Mbuf which we
  500  * are configuring cluster storage for.  If 'arg' is
  501  * empty we allocate just the cluster without setting
  502  * the mbuf to it.  See mbuf.h.
  503  */
  504 static int
  505 mb_ctor_clust(void *mem, int size, void *arg, int how)
  506 {
  507         struct mbuf *m;
  508         u_int *refcnt;
  509         int type;
  510         uma_zone_t zone;
  511 
  512 #ifdef INVARIANTS
  513         trash_ctor(mem, size, arg, how);
  514 #endif
  515         switch (size) {
  516         case MCLBYTES:
  517                 type = EXT_CLUSTER;
  518                 zone = zone_clust;
  519                 break;
  520 #if MJUMPAGESIZE != MCLBYTES
  521         case MJUMPAGESIZE:
  522                 type = EXT_JUMBOP;
  523                 zone = zone_jumbop;
  524                 break;
  525 #endif
  526         case MJUM9BYTES:
  527                 type = EXT_JUMBO9;
  528                 zone = zone_jumbo9;
  529                 break;
  530         case MJUM16BYTES:
  531                 type = EXT_JUMBO16;
  532                 zone = zone_jumbo16;
  533                 break;
  534         default:
  535                 panic("unknown cluster size");
  536                 break;
  537         }
  538 
  539         m = (struct mbuf *)arg;
  540         refcnt = uma_find_refcnt(zone, mem);
  541         *refcnt = 1;
  542         if (m != NULL) {
  543                 m->m_ext.ext_buf = (caddr_t)mem;
  544                 m->m_data = m->m_ext.ext_buf;
  545                 m->m_flags |= M_EXT;
  546                 m->m_ext.ext_free = NULL;
  547                 m->m_ext.ext_arg1 = NULL;
  548                 m->m_ext.ext_arg2 = NULL;
  549                 m->m_ext.ext_size = size;
  550                 m->m_ext.ext_type = type;
  551                 m->m_ext.ext_flags = 0;
  552                 m->m_ext.ref_cnt = refcnt;
  553         }
  554 
  555         return (0);
  556 }
  557 
  558 /*
  559  * The Mbuf Cluster zone destructor.
  560  */
  561 static void
  562 mb_dtor_clust(void *mem, int size, void *arg)
  563 {
  564 #ifdef INVARIANTS
  565         uma_zone_t zone;
  566 
  567         zone = m_getzone(size);
  568         KASSERT(*(uma_find_refcnt(zone, mem)) <= 1,
  569                 ("%s: refcnt incorrect %u", __func__,
  570                  *(uma_find_refcnt(zone, mem))) );
  571 
  572         trash_dtor(mem, size, arg);
  573 #endif
  574 }
  575 
  576 /*
  577  * The Packet secondary zone's init routine, executed on the
  578  * object's transition from mbuf keg slab to zone cache.
  579  */
  580 static int
  581 mb_zinit_pack(void *mem, int size, int how)
  582 {
  583         struct mbuf *m;
  584 
  585         m = (struct mbuf *)mem;         /* m is virgin. */
  586         if (uma_zalloc_arg(zone_clust, m, how) == NULL ||
  587             m->m_ext.ext_buf == NULL)
  588                 return (ENOMEM);
  589         m->m_ext.ext_type = EXT_PACKET; /* Override. */
  590 #ifdef INVARIANTS
  591         trash_init(m->m_ext.ext_buf, MCLBYTES, how);
  592 #endif
  593         return (0);
  594 }
  595 
  596 /*
  597  * The Packet secondary zone's fini routine, executed on the
  598  * object's transition from zone cache to keg slab.
  599  */
  600 static void
  601 mb_zfini_pack(void *mem, int size)
  602 {
  603         struct mbuf *m;
  604 
  605         m = (struct mbuf *)mem;
  606 #ifdef INVARIANTS
  607         trash_fini(m->m_ext.ext_buf, MCLBYTES);
  608 #endif
  609         uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL);
  610 #ifdef INVARIANTS
  611         trash_dtor(mem, size, NULL);
  612 #endif
  613 }
  614 
  615 /*
  616  * The "packet" keg constructor.
  617  */
  618 static int
  619 mb_ctor_pack(void *mem, int size, void *arg, int how)
  620 {
  621         struct mbuf *m;
  622         struct mb_args *args;
  623         int error, flags;
  624         short type;
  625 
  626         m = (struct mbuf *)mem;
  627         args = (struct mb_args *)arg;
  628         flags = args->flags;
  629         type = args->type;
  630         MPASS((flags & M_NOFREE) == 0);
  631 
  632 #ifdef INVARIANTS
  633         trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how);
  634 #endif
  635 
  636         error = m_init(m, NULL, size, how, type, flags);
  637 
  638         /* m_ext is already initialized. */
  639         m->m_data = m->m_ext.ext_buf;
  640         m->m_flags = (flags | M_EXT);
  641 
  642         return (error);
  643 }
  644 
  645 int
  646 m_pkthdr_init(struct mbuf *m, int how)
  647 {
  648 #ifdef MAC
  649         int error;
  650 #endif
  651         m->m_data = m->m_pktdat;
  652         m->m_pkthdr.rcvif = NULL;
  653         SLIST_INIT(&m->m_pkthdr.tags);
  654         m->m_pkthdr.len = 0;
  655         m->m_pkthdr.flowid = 0;
  656         m->m_pkthdr.csum_flags = 0;
  657         m->m_pkthdr.fibnum = 0;
  658         m->m_pkthdr.cosqos = 0;
  659         m->m_pkthdr.rsstype = 0;
  660         m->m_pkthdr.l2hlen = 0;
  661         m->m_pkthdr.l3hlen = 0;
  662         m->m_pkthdr.l4hlen = 0;
  663         m->m_pkthdr.l5hlen = 0;
  664         m->m_pkthdr.PH_per.sixtyfour[0] = 0;
  665         m->m_pkthdr.PH_loc.sixtyfour[0] = 0;
  666 #ifdef MAC
  667         /* If the label init fails, fail the alloc */
  668         error = mac_mbuf_init(m, how);
  669         if (error)
  670                 return (error);
  671 #endif
  672 
  673         return (0);
  674 }
  675 
  676 /*
  677  * This is the protocol drain routine.
  678  *
  679  * No locks should be held when this is called.  The drain routines have to
  680  * presently acquire some locks which raises the possibility of lock order
  681  * reversal.
  682  */
  683 static void
  684 mb_reclaim(void *junk)
  685 {
  686         struct domain *dp;
  687         struct protosw *pr;
  688 
  689         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL,
  690             "mb_reclaim()");
  691 
  692         for (dp = domains; dp != NULL; dp = dp->dom_next)
  693                 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
  694                         if (pr->pr_drain != NULL)
  695                                 (*pr->pr_drain)();
  696 }

Cache object: 8508acdbac34cbaadd02284891197dde


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.