The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/uipc_mbuf.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1982, 1986, 1988, 1991, 1993
    3  *      The Regents of the University of California.  All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  * 4. Neither the name of the University nor the names of its contributors
   14  *    may be used to endorse or promote products derived from this software
   15  *    without specific prior written permission.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  *
   29  *      @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94
   30  */
   31 
   32 #include <sys/cdefs.h>
   33 __FBSDID("$FreeBSD$");
   34 
   35 #include "opt_mac.h"
   36 #include "opt_param.h"
   37 #include "opt_mbuf_stress_test.h"
   38 
   39 #include <sys/param.h>
   40 #include <sys/systm.h>
   41 #include <sys/kernel.h>
   42 #include <sys/limits.h>
   43 #include <sys/lock.h>
   44 #include <sys/malloc.h>
   45 #include <sys/mbuf.h>
   46 #include <sys/sysctl.h>
   47 #include <sys/domain.h>
   48 #include <sys/protosw.h>
   49 #include <sys/uio.h>
   50 
   51 #include <security/mac/mac_framework.h>
   52 
   53 int     max_linkhdr;
   54 int     max_protohdr;
   55 int     max_hdr;
   56 int     max_datalen;
   57 #ifdef MBUF_STRESS_TEST
   58 int     m_defragpackets;
   59 int     m_defragbytes;
   60 int     m_defraguseless;
   61 int     m_defragfailure;
   62 int     m_defragrandomfailures;
   63 #endif
   64 
   65 /*
   66  * sysctl(8) exported objects
   67  */
   68 SYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RD,
   69            &max_linkhdr, 0, "Size of largest link layer header");
   70 SYSCTL_INT(_kern_ipc, KIPC_MAX_PROTOHDR, max_protohdr, CTLFLAG_RD,
   71            &max_protohdr, 0, "Size of largest protocol layer header");
   72 SYSCTL_INT(_kern_ipc, KIPC_MAX_HDR, max_hdr, CTLFLAG_RD,
   73            &max_hdr, 0, "Size of largest link plus protocol header");
   74 SYSCTL_INT(_kern_ipc, KIPC_MAX_DATALEN, max_datalen, CTLFLAG_RD,
   75            &max_datalen, 0, "Minimum space left in mbuf after max_hdr");
   76 #ifdef MBUF_STRESS_TEST
   77 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragpackets, CTLFLAG_RD,
   78            &m_defragpackets, 0, "");
   79 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragbytes, CTLFLAG_RD,
   80            &m_defragbytes, 0, "");
   81 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defraguseless, CTLFLAG_RD,
   82            &m_defraguseless, 0, "");
   83 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragfailure, CTLFLAG_RD,
   84            &m_defragfailure, 0, "");
   85 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW,
   86            &m_defragrandomfailures, 0, "");
   87 #endif
   88 
   89 /*
   90  * Allocate a given length worth of mbufs and/or clusters (whatever fits
   91  * best) and return a pointer to the top of the allocated chain.  If an
   92  * existing mbuf chain is provided, then we will append the new chain
   93  * to the existing one but still return the top of the newly allocated
   94  * chain.
   95  */
   96 struct mbuf *
   97 m_getm2(struct mbuf *m, int len, int how, short type, int flags)
   98 {
   99         struct mbuf *mb, *nm = NULL, *mtail = NULL;
  100 
  101         KASSERT(len >= 0, ("%s: len is < 0", __func__));
  102 
  103         /* Validate flags. */
  104         flags &= (M_PKTHDR | M_EOR);
  105 
  106         /* Packet header mbuf must be first in chain. */
  107         if ((flags & M_PKTHDR) && m != NULL)
  108                 flags &= ~M_PKTHDR;
  109 
  110         /* Loop and append maximum sized mbufs to the chain tail. */
  111         while (len > 0) {
  112                 if (len > MCLBYTES)
  113                         mb = m_getjcl(how, type, (flags & M_PKTHDR),
  114                             MJUMPAGESIZE);
  115                 else if (len >= MINCLSIZE)
  116                         mb = m_getcl(how, type, (flags & M_PKTHDR));
  117                 else if (flags & M_PKTHDR)
  118                         mb = m_gethdr(how, type);
  119                 else
  120                         mb = m_get(how, type);
  121 
  122                 /* Fail the whole operation if one mbuf can't be allocated. */
  123                 if (mb == NULL) {
  124                         if (nm != NULL)
  125                                 m_freem(nm);
  126                         return (NULL);
  127                 }
  128 
  129                 /* Book keeping. */
  130                 len -= (mb->m_flags & M_EXT) ? mb->m_ext.ext_size :
  131                         ((mb->m_flags & M_PKTHDR) ? MHLEN : MLEN);
  132                 if (mtail != NULL)
  133                         mtail->m_next = mb;
  134                 else
  135                         nm = mb;
  136                 mtail = mb;
  137                 flags &= ~M_PKTHDR;     /* Only valid on the first mbuf. */
  138         }
  139         if (flags & M_EOR)
  140                 mtail->m_flags |= M_EOR;  /* Only valid on the last mbuf. */
  141 
  142         /* If mbuf was supplied, append new chain to the end of it. */
  143         if (m != NULL) {
  144                 for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next)
  145                         ;
  146                 mtail->m_next = nm;
  147                 mtail->m_flags &= ~M_EOR;
  148         } else
  149                 m = nm;
  150 
  151         return (m);
  152 }
  153 
  154 /*
  155  * Free an entire chain of mbufs and associated external buffers, if
  156  * applicable.
  157  */
  158 void
  159 m_freem(struct mbuf *mb)
  160 {
  161 
  162         while (mb != NULL)
  163                 mb = m_free(mb);
  164 }
  165 
  166 /*-
  167  * Configure a provided mbuf to refer to the provided external storage
  168  * buffer and setup a reference count for said buffer.  If the setting
  169  * up of the reference count fails, the M_EXT bit will not be set.  If
  170  * successfull, the M_EXT bit is set in the mbuf's flags.
  171  *
  172  * Arguments:
  173  *    mb     The existing mbuf to which to attach the provided buffer.
  174  *    buf    The address of the provided external storage buffer.
  175  *    size   The size of the provided buffer.
  176  *    freef  A pointer to a routine that is responsible for freeing the
  177  *           provided external storage buffer.
  178  *    args   A pointer to an argument structure (of any type) to be passed
  179  *           to the provided freef routine (may be NULL).
  180  *    flags  Any other flags to be passed to the provided mbuf.
  181  *    type   The type that the external storage buffer should be
  182  *           labeled with.
  183  *
  184  * Returns:
  185  *    Nothing.
  186  */
  187 void
  188 m_extadd(struct mbuf *mb, caddr_t buf, u_int size,
  189     void (*freef)(void *, void *), void *args, int flags, int type)
  190 {
  191         KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__));
  192 
  193         if (type != EXT_EXTREF)
  194                 mb->m_ext.ref_cnt = (u_int *)uma_zalloc(zone_ext_refcnt, M_NOWAIT);
  195         if (mb->m_ext.ref_cnt != NULL) {
  196                 *(mb->m_ext.ref_cnt) = 1;
  197                 mb->m_flags |= (M_EXT | flags);
  198                 mb->m_ext.ext_buf = buf;
  199                 mb->m_data = mb->m_ext.ext_buf;
  200                 mb->m_ext.ext_size = size;
  201                 mb->m_ext.ext_free = freef;
  202                 mb->m_ext.ext_args = args;
  203                 mb->m_ext.ext_type = type;
  204         }
  205 }
  206 
  207 /*
  208  * Non-directly-exported function to clean up after mbufs with M_EXT
  209  * storage attached to them if the reference count hits 1.
  210  */
  211 void
  212 mb_free_ext(struct mbuf *m)
  213 {
  214         int skipmbuf;
  215         
  216         KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
  217         KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__));
  218 
  219 
  220         /*
  221          * check if the header is embedded in the cluster
  222          */     
  223         skipmbuf = (m->m_flags & M_NOFREE);
  224         
  225         /* Free attached storage if this mbuf is the only reference to it. */
  226         if (*(m->m_ext.ref_cnt) == 1 ||
  227             atomic_fetchadd_int(m->m_ext.ref_cnt, -1) == 1) {
  228                 switch (m->m_ext.ext_type) {
  229                 case EXT_PACKET:        /* The packet zone is special. */
  230                         if (*(m->m_ext.ref_cnt) == 0)
  231                                 *(m->m_ext.ref_cnt) = 1;
  232                         uma_zfree(zone_pack, m);
  233                         return;         /* Job done. */
  234                 case EXT_CLUSTER:
  235                         uma_zfree(zone_clust, m->m_ext.ext_buf);
  236                         break;
  237                 case EXT_JUMBOP:
  238                         uma_zfree(zone_jumbop, m->m_ext.ext_buf);
  239                         break;
  240                 case EXT_JUMBO9:
  241                         uma_zfree(zone_jumbo9, m->m_ext.ext_buf);
  242                         break;
  243                 case EXT_JUMBO16:
  244                         uma_zfree(zone_jumbo16, m->m_ext.ext_buf);
  245                         break;
  246                 case EXT_SFBUF:
  247                 case EXT_NET_DRV:
  248                 case EXT_MOD_TYPE:
  249                 case EXT_DISPOSABLE:
  250                         *(m->m_ext.ref_cnt) = 0;
  251                         uma_zfree(zone_ext_refcnt, __DEVOLATILE(u_int *,
  252                                 m->m_ext.ref_cnt));
  253                         /* FALLTHROUGH */
  254                 case EXT_EXTREF:
  255                         KASSERT(m->m_ext.ext_free != NULL,
  256                                 ("%s: ext_free not set", __func__));
  257                         (*(m->m_ext.ext_free))(m->m_ext.ext_buf,
  258                             m->m_ext.ext_args);
  259                         break;
  260                 default:
  261                         KASSERT(m->m_ext.ext_type == 0,
  262                                 ("%s: unknown ext_type", __func__));
  263                 }
  264         }
  265         if (skipmbuf)
  266                 return;
  267         
  268         /*
  269          * Free this mbuf back to the mbuf zone with all m_ext
  270          * information purged.
  271          */
  272         m->m_ext.ext_buf = NULL;
  273         m->m_ext.ext_free = NULL;
  274         m->m_ext.ext_args = NULL;
  275         m->m_ext.ref_cnt = NULL;
  276         m->m_ext.ext_size = 0;
  277         m->m_ext.ext_type = 0;
  278         m->m_flags &= ~M_EXT;
  279         uma_zfree(zone_mbuf, m);
  280 }
  281 
  282 /*
  283  * Attach the cluster from *m to *n, set up m_ext in *n
  284  * and bump the refcount of the cluster.
  285  */
  286 static void
  287 mb_dupcl(struct mbuf *n, struct mbuf *m)
  288 {
  289         KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
  290         KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__));
  291         KASSERT((n->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__));
  292 
  293         if (*(m->m_ext.ref_cnt) == 1)
  294                 *(m->m_ext.ref_cnt) += 1;
  295         else
  296                 atomic_add_int(m->m_ext.ref_cnt, 1);
  297         n->m_ext.ext_buf = m->m_ext.ext_buf;
  298         n->m_ext.ext_free = m->m_ext.ext_free;
  299         n->m_ext.ext_args = m->m_ext.ext_args;
  300         n->m_ext.ext_size = m->m_ext.ext_size;
  301         n->m_ext.ref_cnt = m->m_ext.ref_cnt;
  302         n->m_ext.ext_type = m->m_ext.ext_type;
  303         n->m_flags |= M_EXT;
  304         n->m_flags |= m->m_flags & M_RDONLY;
  305 }
  306 
  307 /*
  308  * Clean up mbuf (chain) from any tags and packet headers.
  309  * If "all" is set then the first mbuf in the chain will be
  310  * cleaned too.
  311  */
  312 void
  313 m_demote(struct mbuf *m0, int all)
  314 {
  315         struct mbuf *m;
  316 
  317         for (m = all ? m0 : m0->m_next; m != NULL; m = m->m_next) {
  318                 if (m->m_flags & M_PKTHDR) {
  319                         m_tag_delete_chain(m, NULL);
  320                         m->m_flags &= ~M_PKTHDR;
  321                         bzero(&m->m_pkthdr, sizeof(struct pkthdr));
  322                 }
  323                 if (m->m_type == MT_HEADER)
  324                         m->m_type = MT_DATA;
  325                 if (m != m0 && m->m_nextpkt != NULL)
  326                         m->m_nextpkt = NULL;
  327                 m->m_flags = m->m_flags & (M_EXT|M_EOR|M_RDONLY|M_FREELIST);
  328         }
  329 }
  330 
  331 /*
  332  * Sanity checks on mbuf (chain) for use in KASSERT() and general
  333  * debugging.
  334  * Returns 0 or panics when bad and 1 on all tests passed.
  335  * Sanitize, 0 to run M_SANITY_ACTION, 1 to garble things so they
  336  * blow up later.
  337  */
  338 int
  339 m_sanity(struct mbuf *m0, int sanitize)
  340 {
  341         struct mbuf *m;
  342         caddr_t a, b;
  343         int pktlen = 0;
  344 
  345 #ifdef INVARIANTS
  346 #define M_SANITY_ACTION(s)      panic("mbuf %p: " s, m)
  347 #else 
  348 #define M_SANITY_ACTION(s)      printf("mbuf %p: " s, m)
  349 #endif
  350 
  351         for (m = m0; m != NULL; m = m->m_next) {
  352                 /*
  353                  * Basic pointer checks.  If any of these fails then some
  354                  * unrelated kernel memory before or after us is trashed.
  355                  * No way to recover from that.
  356                  */
  357                 a = ((m->m_flags & M_EXT) ? m->m_ext.ext_buf :
  358                         ((m->m_flags & M_PKTHDR) ? (caddr_t)(&m->m_pktdat) :
  359                          (caddr_t)(&m->m_dat)) );
  360                 b = (caddr_t)(a + (m->m_flags & M_EXT ? m->m_ext.ext_size :
  361                         ((m->m_flags & M_PKTHDR) ? MHLEN : MLEN)));
  362                 if ((caddr_t)m->m_data < a)
  363                         M_SANITY_ACTION("m_data outside mbuf data range left");
  364                 if ((caddr_t)m->m_data > b)
  365                         M_SANITY_ACTION("m_data outside mbuf data range right");
  366                 if ((caddr_t)m->m_data + m->m_len > b)
  367                         M_SANITY_ACTION("m_data + m_len exeeds mbuf space");
  368                 if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.header) {
  369                         if ((caddr_t)m->m_pkthdr.header < a ||
  370                             (caddr_t)m->m_pkthdr.header > b)
  371                                 M_SANITY_ACTION("m_pkthdr.header outside mbuf data range");
  372                 }
  373 
  374                 /* m->m_nextpkt may only be set on first mbuf in chain. */
  375                 if (m != m0 && m->m_nextpkt != NULL) {
  376                         if (sanitize) {
  377                                 m_freem(m->m_nextpkt);
  378                                 m->m_nextpkt = (struct mbuf *)0xDEADC0DE;
  379                         } else
  380                                 M_SANITY_ACTION("m->m_nextpkt on in-chain mbuf");
  381                 }
  382 
  383                 /* packet length (not mbuf length!) calculation */
  384                 if (m0->m_flags & M_PKTHDR)
  385                         pktlen += m->m_len;
  386 
  387                 /* m_tags may only be attached to first mbuf in chain. */
  388                 if (m != m0 && m->m_flags & M_PKTHDR &&
  389                     !SLIST_EMPTY(&m->m_pkthdr.tags)) {
  390                         if (sanitize) {
  391                                 m_tag_delete_chain(m, NULL);
  392                                 /* put in 0xDEADC0DE perhaps? */
  393                         } else
  394                                 M_SANITY_ACTION("m_tags on in-chain mbuf");
  395                 }
  396 
  397                 /* M_PKTHDR may only be set on first mbuf in chain */
  398                 if (m != m0 && m->m_flags & M_PKTHDR) {
  399                         if (sanitize) {
  400                                 bzero(&m->m_pkthdr, sizeof(m->m_pkthdr));
  401                                 m->m_flags &= ~M_PKTHDR;
  402                                 /* put in 0xDEADCODE and leave hdr flag in */
  403                         } else
  404                                 M_SANITY_ACTION("M_PKTHDR on in-chain mbuf");
  405                 }
  406         }
  407         m = m0;
  408         if (pktlen && pktlen != m->m_pkthdr.len) {
  409                 if (sanitize)
  410                         m->m_pkthdr.len = 0;
  411                 else
  412                         M_SANITY_ACTION("m_pkthdr.len != mbuf chain length");
  413         }
  414         return 1;
  415 
  416 #undef  M_SANITY_ACTION
  417 }
  418 
  419 
  420 /*
  421  * "Move" mbuf pkthdr from "from" to "to".
  422  * "from" must have M_PKTHDR set, and "to" must be empty.
  423  */
  424 void
  425 m_move_pkthdr(struct mbuf *to, struct mbuf *from)
  426 {
  427 
  428 #if 0
  429         /* see below for why these are not enabled */
  430         M_ASSERTPKTHDR(to);
  431         /* Note: with MAC, this may not be a good assertion. */
  432         KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags),
  433             ("m_move_pkthdr: to has tags"));
  434 #endif
  435 #ifdef MAC
  436         /*
  437          * XXXMAC: It could be this should also occur for non-MAC?
  438          */
  439         if (to->m_flags & M_PKTHDR)
  440                 m_tag_delete_chain(to, NULL);
  441 #endif
  442         to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
  443         if ((to->m_flags & M_EXT) == 0)
  444                 to->m_data = to->m_pktdat;
  445         to->m_pkthdr = from->m_pkthdr;          /* especially tags */
  446         SLIST_INIT(&from->m_pkthdr.tags);       /* purge tags from src */
  447         from->m_flags &= ~M_PKTHDR;
  448 }
  449 
  450 /*
  451  * Duplicate "from"'s mbuf pkthdr in "to".
  452  * "from" must have M_PKTHDR set, and "to" must be empty.
  453  * In particular, this does a deep copy of the packet tags.
  454  */
  455 int
  456 m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
  457 {
  458 
  459 #if 0
  460         /*
  461          * The mbuf allocator only initializes the pkthdr
  462          * when the mbuf is allocated with MGETHDR. Many users
  463          * (e.g. m_copy*, m_prepend) use MGET and then
  464          * smash the pkthdr as needed causing these
  465          * assertions to trip.  For now just disable them.
  466          */
  467         M_ASSERTPKTHDR(to);
  468         /* Note: with MAC, this may not be a good assertion. */
  469         KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), ("m_dup_pkthdr: to has tags"));
  470 #endif
  471         MBUF_CHECKSLEEP(how);
  472 #ifdef MAC
  473         if (to->m_flags & M_PKTHDR)
  474                 m_tag_delete_chain(to, NULL);
  475 #endif
  476         to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
  477         if ((to->m_flags & M_EXT) == 0)
  478                 to->m_data = to->m_pktdat;
  479         to->m_pkthdr = from->m_pkthdr;
  480         SLIST_INIT(&to->m_pkthdr.tags);
  481         return (m_tag_copy_chain(to, from, MBTOM(how)));
  482 }
  483 
  484 /*
  485  * Lesser-used path for M_PREPEND:
  486  * allocate new mbuf to prepend to chain,
  487  * copy junk along.
  488  */
  489 struct mbuf *
  490 m_prepend(struct mbuf *m, int len, int how)
  491 {
  492         struct mbuf *mn;
  493 
  494         if (m->m_flags & M_PKTHDR)
  495                 MGETHDR(mn, how, m->m_type);
  496         else
  497                 MGET(mn, how, m->m_type);
  498         if (mn == NULL) {
  499                 m_freem(m);
  500                 return (NULL);
  501         }
  502         if (m->m_flags & M_PKTHDR)
  503                 M_MOVE_PKTHDR(mn, m);
  504         mn->m_next = m;
  505         m = mn;
  506         if(m->m_flags & M_PKTHDR) {
  507                 if (len < MHLEN)
  508                         MH_ALIGN(m, len);
  509         } else {
  510                 if (len < MLEN) 
  511                         M_ALIGN(m, len);
  512         }
  513         m->m_len = len;
  514         return (m);
  515 }
  516 
  517 /*
  518  * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
  519  * continuing for "len" bytes.  If len is M_COPYALL, copy to end of mbuf.
  520  * The wait parameter is a choice of M_TRYWAIT/M_DONTWAIT from caller.
  521  * Note that the copy is read-only, because clusters are not copied,
  522  * only their reference counts are incremented.
  523  */
  524 struct mbuf *
  525 m_copym(struct mbuf *m, int off0, int len, int wait)
  526 {
  527         struct mbuf *n, **np;
  528         int off = off0;
  529         struct mbuf *top;
  530         int copyhdr = 0;
  531 
  532         KASSERT(off >= 0, ("m_copym, negative off %d", off));
  533         KASSERT(len >= 0, ("m_copym, negative len %d", len));
  534         MBUF_CHECKSLEEP(wait);
  535         if (off == 0 && m->m_flags & M_PKTHDR)
  536                 copyhdr = 1;
  537         while (off > 0) {
  538                 KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain"));
  539                 if (off < m->m_len)
  540                         break;
  541                 off -= m->m_len;
  542                 m = m->m_next;
  543         }
  544         np = &top;
  545         top = 0;
  546         while (len > 0) {
  547                 if (m == NULL) {
  548                         KASSERT(len == M_COPYALL, 
  549                             ("m_copym, length > size of mbuf chain"));
  550                         break;
  551                 }
  552                 if (copyhdr)
  553                         MGETHDR(n, wait, m->m_type);
  554                 else
  555                         MGET(n, wait, m->m_type);
  556                 *np = n;
  557                 if (n == NULL)
  558                         goto nospace;
  559                 if (copyhdr) {
  560                         if (!m_dup_pkthdr(n, m, wait))
  561                                 goto nospace;
  562                         if (len == M_COPYALL)
  563                                 n->m_pkthdr.len -= off0;
  564                         else
  565                                 n->m_pkthdr.len = len;
  566                         copyhdr = 0;
  567                 }
  568                 n->m_len = min(len, m->m_len - off);
  569                 if (m->m_flags & M_EXT) {
  570                         n->m_data = m->m_data + off;
  571                         mb_dupcl(n, m);
  572                 } else
  573                         bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
  574                             (u_int)n->m_len);
  575                 if (len != M_COPYALL)
  576                         len -= n->m_len;
  577                 off = 0;
  578                 m = m->m_next;
  579                 np = &n->m_next;
  580         }
  581         if (top == NULL)
  582                 mbstat.m_mcfail++;      /* XXX: No consistency. */
  583 
  584         return (top);
  585 nospace:
  586         m_freem(top);
  587         mbstat.m_mcfail++;      /* XXX: No consistency. */
  588         return (NULL);
  589 }
  590 
  591 /*
  592  * Returns mbuf chain with new head for the prepending case.
  593  * Copies from mbuf (chain) n from off for len to mbuf (chain) m
  594  * either prepending or appending the data.
  595  * The resulting mbuf (chain) m is fully writeable.
  596  * m is destination (is made writeable)
  597  * n is source, off is offset in source, len is len from offset
  598  * dir, 0 append, 1 prepend
  599  * how, wait or nowait
  600  */
  601 
  602 static int
  603 m_bcopyxxx(void *s, void *t, u_int len)
  604 {
  605         bcopy(s, t, (size_t)len);
  606         return 0;
  607 }
  608 
  609 struct mbuf *
  610 m_copymdata(struct mbuf *m, struct mbuf *n, int off, int len,
  611     int prep, int how)
  612 {
  613         struct mbuf *mm, *x, *z, *prev = NULL;
  614         caddr_t p;
  615         int i, nlen = 0;
  616         caddr_t buf[MLEN];
  617 
  618         KASSERT(m != NULL && n != NULL, ("m_copymdata, no target or source"));
  619         KASSERT(off >= 0, ("m_copymdata, negative off %d", off));
  620         KASSERT(len >= 0, ("m_copymdata, negative len %d", len));
  621         KASSERT(prep == 0 || prep == 1, ("m_copymdata, unknown direction %d", prep));
  622 
  623         mm = m;
  624         if (!prep) {
  625                 while(mm->m_next) {
  626                         prev = mm;
  627                         mm = mm->m_next;
  628                 }
  629         }
  630         for (z = n; z != NULL; z = z->m_next)
  631                 nlen += z->m_len;
  632         if (len == M_COPYALL)
  633                 len = nlen - off;
  634         if (off + len > nlen || len < 1)
  635                 return NULL;
  636 
  637         if (!M_WRITABLE(mm)) {
  638                 /* XXX: Use proper m_xxx function instead. */
  639                 x = m_getcl(how, MT_DATA, mm->m_flags);
  640                 if (x == NULL)
  641                         return NULL;
  642                 bcopy(mm->m_ext.ext_buf, x->m_ext.ext_buf, x->m_ext.ext_size);
  643                 p = x->m_ext.ext_buf + (mm->m_data - mm->m_ext.ext_buf);
  644                 x->m_data = p;
  645                 mm->m_next = NULL;
  646                 if (mm != m)
  647                         prev->m_next = x;
  648                 m_free(mm);
  649                 mm = x;
  650         }
  651 
  652         /*
  653          * Append/prepend the data.  Allocating mbufs as necessary.
  654          */
  655         /* Shortcut if enough free space in first/last mbuf. */
  656         if (!prep && M_TRAILINGSPACE(mm) >= len) {
  657                 m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t) +
  658                          mm->m_len);
  659                 mm->m_len += len;
  660                 mm->m_pkthdr.len += len;
  661                 return m;
  662         }
  663         if (prep && M_LEADINGSPACE(mm) >= len) {
  664                 mm->m_data = mtod(mm, caddr_t) - len;
  665                 m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t));
  666                 mm->m_len += len;
  667                 mm->m_pkthdr.len += len;
  668                 return mm;
  669         }
  670 
  671         /* Expand first/last mbuf to cluster if possible. */
  672         if (!prep && !(mm->m_flags & M_EXT) && len > M_TRAILINGSPACE(mm)) {
  673                 bcopy(mm->m_data, &buf, mm->m_len);
  674                 m_clget(mm, how);
  675                 if (!(mm->m_flags & M_EXT))
  676                         return NULL;
  677                 bcopy(&buf, mm->m_ext.ext_buf, mm->m_len);
  678                 mm->m_data = mm->m_ext.ext_buf;
  679                 mm->m_pkthdr.header = NULL;
  680         }
  681         if (prep && !(mm->m_flags & M_EXT) && len > M_LEADINGSPACE(mm)) {
  682                 bcopy(mm->m_data, &buf, mm->m_len);
  683                 m_clget(mm, how);
  684                 if (!(mm->m_flags & M_EXT))
  685                         return NULL;
  686                 bcopy(&buf, (caddr_t *)mm->m_ext.ext_buf +
  687                        mm->m_ext.ext_size - mm->m_len, mm->m_len);
  688                 mm->m_data = (caddr_t)mm->m_ext.ext_buf +
  689                               mm->m_ext.ext_size - mm->m_len;
  690                 mm->m_pkthdr.header = NULL;
  691         }
  692 
  693         /* Append/prepend as many mbuf (clusters) as necessary to fit len. */
  694         if (!prep && len > M_TRAILINGSPACE(mm)) {
  695                 if (!m_getm(mm, len - M_TRAILINGSPACE(mm), how, MT_DATA))
  696                         return NULL;
  697         }
  698         if (prep && len > M_LEADINGSPACE(mm)) {
  699                 if (!(z = m_getm(NULL, len - M_LEADINGSPACE(mm), how, MT_DATA)))
  700                         return NULL;
  701                 i = 0;
  702                 for (x = z; x != NULL; x = x->m_next) {
  703                         i += x->m_flags & M_EXT ? x->m_ext.ext_size :
  704                               (x->m_flags & M_PKTHDR ? MHLEN : MLEN);
  705                         if (!x->m_next)
  706                                 break;
  707                 }
  708                 z->m_data += i - len;
  709                 m_move_pkthdr(mm, z);
  710                 x->m_next = mm;
  711                 mm = z;
  712         }
  713 
  714         /* Seek to start position in source mbuf. Optimization for long chains. */
  715         while (off > 0) {
  716                 if (off < n->m_len)
  717                         break;
  718                 off -= n->m_len;
  719                 n = n->m_next;
  720         }
  721 
  722         /* Copy data into target mbuf. */
  723         z = mm;
  724         while (len > 0) {
  725                 KASSERT(z != NULL, ("m_copymdata, falling off target edge"));
  726                 i = M_TRAILINGSPACE(z);
  727                 m_apply(n, off, i, m_bcopyxxx, mtod(z, caddr_t) + z->m_len);
  728                 z->m_len += i;
  729                 /* fixup pkthdr.len if necessary */
  730                 if ((prep ? mm : m)->m_flags & M_PKTHDR)
  731                         (prep ? mm : m)->m_pkthdr.len += i;
  732                 off += i;
  733                 len -= i;
  734                 z = z->m_next;
  735         }
  736         return (prep ? mm : m);
  737 }
  738 
  739 /*
  740  * Copy an entire packet, including header (which must be present).
  741  * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'.
  742  * Note that the copy is read-only, because clusters are not copied,
  743  * only their reference counts are incremented.
  744  * Preserve alignment of the first mbuf so if the creator has left
  745  * some room at the beginning (e.g. for inserting protocol headers)
  746  * the copies still have the room available.
  747  */
  748 struct mbuf *
  749 m_copypacket(struct mbuf *m, int how)
  750 {
  751         struct mbuf *top, *n, *o;
  752 
  753         MBUF_CHECKSLEEP(how);
  754         MGET(n, how, m->m_type);
  755         top = n;
  756         if (n == NULL)
  757                 goto nospace;
  758 
  759         if (!m_dup_pkthdr(n, m, how))
  760                 goto nospace;
  761         n->m_len = m->m_len;
  762         if (m->m_flags & M_EXT) {
  763                 n->m_data = m->m_data;
  764                 mb_dupcl(n, m);
  765         } else {
  766                 n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat );
  767                 bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
  768         }
  769 
  770         m = m->m_next;
  771         while (m) {
  772                 MGET(o, how, m->m_type);
  773                 if (o == NULL)
  774                         goto nospace;
  775 
  776                 n->m_next = o;
  777                 n = n->m_next;
  778 
  779                 n->m_len = m->m_len;
  780                 if (m->m_flags & M_EXT) {
  781                         n->m_data = m->m_data;
  782                         mb_dupcl(n, m);
  783                 } else {
  784                         bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
  785                 }
  786 
  787                 m = m->m_next;
  788         }
  789         return top;
  790 nospace:
  791         m_freem(top);
  792         mbstat.m_mcfail++;      /* XXX: No consistency. */ 
  793         return (NULL);
  794 }
  795 
  796 /*
  797  * Copy data from an mbuf chain starting "off" bytes from the beginning,
  798  * continuing for "len" bytes, into the indicated buffer.
  799  */
  800 void
  801 m_copydata(const struct mbuf *m, int off, int len, caddr_t cp)
  802 {
  803         u_int count;
  804 
  805         KASSERT(off >= 0, ("m_copydata, negative off %d", off));
  806         KASSERT(len >= 0, ("m_copydata, negative len %d", len));
  807         while (off > 0) {
  808                 KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain"));
  809                 if (off < m->m_len)
  810                         break;
  811                 off -= m->m_len;
  812                 m = m->m_next;
  813         }
  814         while (len > 0) {
  815                 KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain"));
  816                 count = min(m->m_len - off, len);
  817                 bcopy(mtod(m, caddr_t) + off, cp, count);
  818                 len -= count;
  819                 cp += count;
  820                 off = 0;
  821                 m = m->m_next;
  822         }
  823 }
  824 
  825 /*
  826  * Copy a packet header mbuf chain into a completely new chain, including
  827  * copying any mbuf clusters.  Use this instead of m_copypacket() when
  828  * you need a writable copy of an mbuf chain.
  829  */
  830 struct mbuf *
  831 m_dup(struct mbuf *m, int how)
  832 {
  833         struct mbuf **p, *top = NULL;
  834         int remain, moff, nsize;
  835 
  836         MBUF_CHECKSLEEP(how);
  837         /* Sanity check */
  838         if (m == NULL)
  839                 return (NULL);
  840         M_ASSERTPKTHDR(m);
  841 
  842         /* While there's more data, get a new mbuf, tack it on, and fill it */
  843         remain = m->m_pkthdr.len;
  844         moff = 0;
  845         p = &top;
  846         while (remain > 0 || top == NULL) {     /* allow m->m_pkthdr.len == 0 */
  847                 struct mbuf *n;
  848 
  849                 /* Get the next new mbuf */
  850                 if (remain >= MINCLSIZE) {
  851                         n = m_getcl(how, m->m_type, 0);
  852                         nsize = MCLBYTES;
  853                 } else {
  854                         n = m_get(how, m->m_type);
  855                         nsize = MLEN;
  856                 }
  857                 if (n == NULL)
  858                         goto nospace;
  859 
  860                 if (top == NULL) {              /* First one, must be PKTHDR */
  861                         if (!m_dup_pkthdr(n, m, how)) {
  862                                 m_free(n);
  863                                 goto nospace;
  864                         }
  865                         if ((n->m_flags & M_EXT) == 0)
  866                                 nsize = MHLEN;
  867                 }
  868                 n->m_len = 0;
  869 
  870                 /* Link it into the new chain */
  871                 *p = n;
  872                 p = &n->m_next;
  873 
  874                 /* Copy data from original mbuf(s) into new mbuf */
  875                 while (n->m_len < nsize && m != NULL) {
  876                         int chunk = min(nsize - n->m_len, m->m_len - moff);
  877 
  878                         bcopy(m->m_data + moff, n->m_data + n->m_len, chunk);
  879                         moff += chunk;
  880                         n->m_len += chunk;
  881                         remain -= chunk;
  882                         if (moff == m->m_len) {
  883                                 m = m->m_next;
  884                                 moff = 0;
  885                         }
  886                 }
  887 
  888                 /* Check correct total mbuf length */
  889                 KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL),
  890                         ("%s: bogus m_pkthdr.len", __func__));
  891         }
  892         return (top);
  893 
  894 nospace:
  895         m_freem(top);
  896         mbstat.m_mcfail++;      /* XXX: No consistency. */
  897         return (NULL);
  898 }
  899 
  900 /*
  901  * Concatenate mbuf chain n to m.
  902  * Both chains must be of the same type (e.g. MT_DATA).
  903  * Any m_pkthdr is not updated.
  904  */
  905 void
  906 m_cat(struct mbuf *m, struct mbuf *n)
  907 {
  908         while (m->m_next)
  909                 m = m->m_next;
  910         while (n) {
  911                 if (m->m_flags & M_EXT ||
  912                     m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
  913                         /* just join the two chains */
  914                         m->m_next = n;
  915                         return;
  916                 }
  917                 /* splat the data from one into the other */
  918                 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
  919                     (u_int)n->m_len);
  920                 m->m_len += n->m_len;
  921                 n = m_free(n);
  922         }
  923 }
  924 
  925 void
  926 m_adj(struct mbuf *mp, int req_len)
  927 {
  928         int len = req_len;
  929         struct mbuf *m;
  930         int count;
  931 
  932         if ((m = mp) == NULL)
  933                 return;
  934         if (len >= 0) {
  935                 /*
  936                  * Trim from head.
  937                  */
  938                 while (m != NULL && len > 0) {
  939                         if (m->m_len <= len) {
  940                                 len -= m->m_len;
  941                                 m->m_len = 0;
  942                                 m = m->m_next;
  943                         } else {
  944                                 m->m_len -= len;
  945                                 m->m_data += len;
  946                                 len = 0;
  947                         }
  948                 }
  949                 m = mp;
  950                 if (mp->m_flags & M_PKTHDR)
  951                         m->m_pkthdr.len -= (req_len - len);
  952         } else {
  953                 /*
  954                  * Trim from tail.  Scan the mbuf chain,
  955                  * calculating its length and finding the last mbuf.
  956                  * If the adjustment only affects this mbuf, then just
  957                  * adjust and return.  Otherwise, rescan and truncate
  958                  * after the remaining size.
  959                  */
  960                 len = -len;
  961                 count = 0;
  962                 for (;;) {
  963                         count += m->m_len;
  964                         if (m->m_next == (struct mbuf *)0)
  965                                 break;
  966                         m = m->m_next;
  967                 }
  968                 if (m->m_len >= len) {
  969                         m->m_len -= len;
  970                         if (mp->m_flags & M_PKTHDR)
  971                                 mp->m_pkthdr.len -= len;
  972                         return;
  973                 }
  974                 count -= len;
  975                 if (count < 0)
  976                         count = 0;
  977                 /*
  978                  * Correct length for chain is "count".
  979                  * Find the mbuf with last data, adjust its length,
  980                  * and toss data from remaining mbufs on chain.
  981                  */
  982                 m = mp;
  983                 if (m->m_flags & M_PKTHDR)
  984                         m->m_pkthdr.len = count;
  985                 for (; m; m = m->m_next) {
  986                         if (m->m_len >= count) {
  987                                 m->m_len = count;
  988                                 if (m->m_next != NULL) {
  989                                         m_freem(m->m_next);
  990                                         m->m_next = NULL;
  991                                 }
  992                                 break;
  993                         }
  994                         count -= m->m_len;
  995                 }
  996         }
  997 }
  998 
  999 /*
 1000  * Rearange an mbuf chain so that len bytes are contiguous
 1001  * and in the data area of an mbuf (so that mtod and dtom
 1002  * will work for a structure of size len).  Returns the resulting
 1003  * mbuf chain on success, frees it and returns null on failure.
 1004  * If there is room, it will add up to max_protohdr-len extra bytes to the
 1005  * contiguous region in an attempt to avoid being called next time.
 1006  */
 1007 struct mbuf *
 1008 m_pullup(struct mbuf *n, int len)
 1009 {
 1010         struct mbuf *m;
 1011         int count;
 1012         int space;
 1013 
 1014         /*
 1015          * If first mbuf has no cluster, and has room for len bytes
 1016          * without shifting current data, pullup into it,
 1017          * otherwise allocate a new mbuf to prepend to the chain.
 1018          */
 1019         if ((n->m_flags & M_EXT) == 0 &&
 1020             n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
 1021                 if (n->m_len >= len)
 1022                         return (n);
 1023                 m = n;
 1024                 n = n->m_next;
 1025                 len -= m->m_len;
 1026         } else {
 1027                 if (len > MHLEN)
 1028                         goto bad;
 1029                 MGET(m, M_DONTWAIT, n->m_type);
 1030                 if (m == NULL)
 1031                         goto bad;
 1032                 m->m_len = 0;
 1033                 if (n->m_flags & M_PKTHDR)
 1034                         M_MOVE_PKTHDR(m, n);
 1035         }
 1036         space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
 1037         do {
 1038                 count = min(min(max(len, max_protohdr), space), n->m_len);
 1039                 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
 1040                   (u_int)count);
 1041                 len -= count;
 1042                 m->m_len += count;
 1043                 n->m_len -= count;
 1044                 space -= count;
 1045                 if (n->m_len)
 1046                         n->m_data += count;
 1047                 else
 1048                         n = m_free(n);
 1049         } while (len > 0 && n);
 1050         if (len > 0) {
 1051                 (void) m_free(m);
 1052                 goto bad;
 1053         }
 1054         m->m_next = n;
 1055         return (m);
 1056 bad:
 1057         m_freem(n);
 1058         mbstat.m_mpfail++;      /* XXX: No consistency. */
 1059         return (NULL);
 1060 }
 1061 
 1062 /*
 1063  * Like m_pullup(), except a new mbuf is always allocated, and we allow
 1064  * the amount of empty space before the data in the new mbuf to be specified
 1065  * (in the event that the caller expects to prepend later).
 1066  */
 1067 int MSFail;
 1068 
 1069 struct mbuf *
 1070 m_copyup(struct mbuf *n, int len, int dstoff)
 1071 {
 1072         struct mbuf *m;
 1073         int count, space;
 1074 
 1075         if (len > (MHLEN - dstoff))
 1076                 goto bad;
 1077         MGET(m, M_DONTWAIT, n->m_type);
 1078         if (m == NULL)
 1079                 goto bad;
 1080         m->m_len = 0;
 1081         if (n->m_flags & M_PKTHDR)
 1082                 M_MOVE_PKTHDR(m, n);
 1083         m->m_data += dstoff;
 1084         space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
 1085         do {
 1086                 count = min(min(max(len, max_protohdr), space), n->m_len);
 1087                 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t),
 1088                     (unsigned)count);
 1089                 len -= count;
 1090                 m->m_len += count;
 1091                 n->m_len -= count;
 1092                 space -= count;
 1093                 if (n->m_len)
 1094                         n->m_data += count;
 1095                 else
 1096                         n = m_free(n);
 1097         } while (len > 0 && n);
 1098         if (len > 0) {
 1099                 (void) m_free(m);
 1100                 goto bad;
 1101         }
 1102         m->m_next = n;
 1103         return (m);
 1104  bad:
 1105         m_freem(n);
 1106         MSFail++;
 1107         return (NULL);
 1108 }
 1109 
 1110 /*
 1111  * Partition an mbuf chain in two pieces, returning the tail --
 1112  * all but the first len0 bytes.  In case of failure, it returns NULL and
 1113  * attempts to restore the chain to its original state.
 1114  *
 1115  * Note that the resulting mbufs might be read-only, because the new
 1116  * mbuf can end up sharing an mbuf cluster with the original mbuf if
 1117  * the "breaking point" happens to lie within a cluster mbuf. Use the
 1118  * M_WRITABLE() macro to check for this case.
 1119  */
 1120 struct mbuf *
 1121 m_split(struct mbuf *m0, int len0, int wait)
 1122 {
 1123         struct mbuf *m, *n;
 1124         u_int len = len0, remain;
 1125 
 1126         MBUF_CHECKSLEEP(wait);
 1127         for (m = m0; m && len > m->m_len; m = m->m_next)
 1128                 len -= m->m_len;
 1129         if (m == NULL)
 1130                 return (NULL);
 1131         remain = m->m_len - len;
 1132         if (m0->m_flags & M_PKTHDR) {
 1133                 MGETHDR(n, wait, m0->m_type);
 1134                 if (n == NULL)
 1135                         return (NULL);
 1136                 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
 1137                 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
 1138                 m0->m_pkthdr.len = len0;
 1139                 if (m->m_flags & M_EXT)
 1140                         goto extpacket;
 1141                 if (remain > MHLEN) {
 1142                         /* m can't be the lead packet */
 1143                         MH_ALIGN(n, 0);
 1144                         n->m_next = m_split(m, len, wait);
 1145                         if (n->m_next == NULL) {
 1146                                 (void) m_free(n);
 1147                                 return (NULL);
 1148                         } else {
 1149                                 n->m_len = 0;
 1150                                 return (n);
 1151                         }
 1152                 } else
 1153                         MH_ALIGN(n, remain);
 1154         } else if (remain == 0) {
 1155                 n = m->m_next;
 1156                 m->m_next = NULL;
 1157                 return (n);
 1158         } else {
 1159                 MGET(n, wait, m->m_type);
 1160                 if (n == NULL)
 1161                         return (NULL);
 1162                 M_ALIGN(n, remain);
 1163         }
 1164 extpacket:
 1165         if (m->m_flags & M_EXT) {
 1166                 n->m_data = m->m_data + len;
 1167                 mb_dupcl(n, m);
 1168         } else {
 1169                 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
 1170         }
 1171         n->m_len = remain;
 1172         m->m_len = len;
 1173         n->m_next = m->m_next;
 1174         m->m_next = NULL;
 1175         return (n);
 1176 }
 1177 /*
 1178  * Routine to copy from device local memory into mbufs.
 1179  * Note that `off' argument is offset into first mbuf of target chain from
 1180  * which to begin copying the data to.
 1181  */
 1182 struct mbuf *
 1183 m_devget(char *buf, int totlen, int off, struct ifnet *ifp,
 1184     void (*copy)(char *from, caddr_t to, u_int len))
 1185 {
 1186         struct mbuf *m;
 1187         struct mbuf *top = NULL, **mp = &top;
 1188         int len;
 1189 
 1190         if (off < 0 || off > MHLEN)
 1191                 return (NULL);
 1192 
 1193         while (totlen > 0) {
 1194                 if (top == NULL) {      /* First one, must be PKTHDR */
 1195                         if (totlen + off >= MINCLSIZE) {
 1196                                 m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
 1197                                 len = MCLBYTES;
 1198                         } else {
 1199                                 m = m_gethdr(M_DONTWAIT, MT_DATA);
 1200                                 len = MHLEN;
 1201 
 1202                                 /* Place initial small packet/header at end of mbuf */
 1203                                 if (m && totlen + off + max_linkhdr <= MLEN) {
 1204                                         m->m_data += max_linkhdr;
 1205                                         len -= max_linkhdr;
 1206                                 }
 1207                         }
 1208                         if (m == NULL)
 1209                                 return NULL;
 1210                         m->m_pkthdr.rcvif = ifp;
 1211                         m->m_pkthdr.len = totlen;
 1212                 } else {
 1213                         if (totlen + off >= MINCLSIZE) {
 1214                                 m = m_getcl(M_DONTWAIT, MT_DATA, 0);
 1215                                 len = MCLBYTES;
 1216                         } else {
 1217                                 m = m_get(M_DONTWAIT, MT_DATA);
 1218                                 len = MLEN;
 1219                         }
 1220                         if (m == NULL) {
 1221                                 m_freem(top);
 1222                                 return NULL;
 1223                         }
 1224                 }
 1225                 if (off) {
 1226                         m->m_data += off;
 1227                         len -= off;
 1228                         off = 0;
 1229                 }
 1230                 m->m_len = len = min(totlen, len);
 1231                 if (copy)
 1232                         copy(buf, mtod(m, caddr_t), (u_int)len);
 1233                 else
 1234                         bcopy(buf, mtod(m, caddr_t), (u_int)len);
 1235                 buf += len;
 1236                 *mp = m;
 1237                 mp = &m->m_next;
 1238                 totlen -= len;
 1239         }
 1240         return (top);
 1241 }
 1242 
 1243 /*
 1244  * Copy data from a buffer back into the indicated mbuf chain,
 1245  * starting "off" bytes from the beginning, extending the mbuf
 1246  * chain if necessary.
 1247  */
 1248 void
 1249 m_copyback(struct mbuf *m0, int off, int len, c_caddr_t cp)
 1250 {
 1251         int mlen;
 1252         struct mbuf *m = m0, *n;
 1253         int totlen = 0;
 1254 
 1255         if (m0 == NULL)
 1256                 return;
 1257         while (off > (mlen = m->m_len)) {
 1258                 off -= mlen;
 1259                 totlen += mlen;
 1260                 if (m->m_next == NULL) {
 1261                         n = m_get(M_DONTWAIT, m->m_type);
 1262                         if (n == NULL)
 1263                                 goto out;
 1264                         bzero(mtod(n, caddr_t), MLEN);
 1265                         n->m_len = min(MLEN, len + off);
 1266                         m->m_next = n;
 1267                 }
 1268                 m = m->m_next;
 1269         }
 1270         while (len > 0) {
 1271                 if (m->m_next == NULL && (len > m->m_len - off)) {
 1272                         m->m_len += min(len - (m->m_len - off),
 1273                             M_TRAILINGSPACE(m));
 1274                 }
 1275                 mlen = min (m->m_len - off, len);
 1276                 bcopy(cp, off + mtod(m, caddr_t), (u_int)mlen);
 1277                 cp += mlen;
 1278                 len -= mlen;
 1279                 mlen += off;
 1280                 off = 0;
 1281                 totlen += mlen;
 1282                 if (len == 0)
 1283                         break;
 1284                 if (m->m_next == NULL) {
 1285                         n = m_get(M_DONTWAIT, m->m_type);
 1286                         if (n == NULL)
 1287                                 break;
 1288                         n->m_len = min(MLEN, len);
 1289                         m->m_next = n;
 1290                 }
 1291                 m = m->m_next;
 1292         }
 1293 out:    if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
 1294                 m->m_pkthdr.len = totlen;
 1295 }
 1296 
 1297 /*
 1298  * Append the specified data to the indicated mbuf chain,
 1299  * Extend the mbuf chain if the new data does not fit in
 1300  * existing space.
 1301  *
 1302  * Return 1 if able to complete the job; otherwise 0.
 1303  */
 1304 int
 1305 m_append(struct mbuf *m0, int len, c_caddr_t cp)
 1306 {
 1307         struct mbuf *m, *n;
 1308         int remainder, space;
 1309 
 1310         for (m = m0; m->m_next != NULL; m = m->m_next)
 1311                 ;
 1312         remainder = len;
 1313         space = M_TRAILINGSPACE(m);
 1314         if (space > 0) {
 1315                 /*
 1316                  * Copy into available space.
 1317                  */
 1318                 if (space > remainder)
 1319                         space = remainder;
 1320                 bcopy(cp, mtod(m, caddr_t) + m->m_len, space);
 1321                 m->m_len += space;
 1322                 cp += space, remainder -= space;
 1323         }
 1324         while (remainder > 0) {
 1325                 /*
 1326                  * Allocate a new mbuf; could check space
 1327                  * and allocate a cluster instead.
 1328                  */
 1329                 n = m_get(M_DONTWAIT, m->m_type);
 1330                 if (n == NULL)
 1331                         break;
 1332                 n->m_len = min(MLEN, remainder);
 1333                 bcopy(cp, mtod(n, caddr_t), n->m_len);
 1334                 cp += n->m_len, remainder -= n->m_len;
 1335                 m->m_next = n;
 1336                 m = n;
 1337         }
 1338         if (m0->m_flags & M_PKTHDR)
 1339                 m0->m_pkthdr.len += len - remainder;
 1340         return (remainder == 0);
 1341 }
 1342 
 1343 /*
 1344  * Apply function f to the data in an mbuf chain starting "off" bytes from
 1345  * the beginning, continuing for "len" bytes.
 1346  */
 1347 int
 1348 m_apply(struct mbuf *m, int off, int len,
 1349     int (*f)(void *, void *, u_int), void *arg)
 1350 {
 1351         u_int count;
 1352         int rval;
 1353 
 1354         KASSERT(off >= 0, ("m_apply, negative off %d", off));
 1355         KASSERT(len >= 0, ("m_apply, negative len %d", len));
 1356         while (off > 0) {
 1357                 KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain"));
 1358                 if (off < m->m_len)
 1359                         break;
 1360                 off -= m->m_len;
 1361                 m = m->m_next;
 1362         }
 1363         while (len > 0) {
 1364                 KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain"));
 1365                 count = min(m->m_len - off, len);
 1366                 rval = (*f)(arg, mtod(m, caddr_t) + off, count);
 1367                 if (rval)
 1368                         return (rval);
 1369                 len -= count;
 1370                 off = 0;
 1371                 m = m->m_next;
 1372         }
 1373         return (0);
 1374 }
 1375 
 1376 /*
 1377  * Return a pointer to mbuf/offset of location in mbuf chain.
 1378  */
 1379 struct mbuf *
 1380 m_getptr(struct mbuf *m, int loc, int *off)
 1381 {
 1382 
 1383         while (loc >= 0) {
 1384                 /* Normal end of search. */
 1385                 if (m->m_len > loc) {
 1386                         *off = loc;
 1387                         return (m);
 1388                 } else {
 1389                         loc -= m->m_len;
 1390                         if (m->m_next == NULL) {
 1391                                 if (loc == 0) {
 1392                                         /* Point at the end of valid data. */
 1393                                         *off = m->m_len;
 1394                                         return (m);
 1395                                 }
 1396                                 return (NULL);
 1397                         }
 1398                         m = m->m_next;
 1399                 }
 1400         }
 1401         return (NULL);
 1402 }
 1403 
 1404 void
 1405 m_print(const struct mbuf *m, int maxlen)
 1406 {
 1407         int len;
 1408         int pdata;
 1409         const struct mbuf *m2;
 1410 
 1411         if (m->m_flags & M_PKTHDR)
 1412                 len = m->m_pkthdr.len;
 1413         else
 1414                 len = -1;
 1415         m2 = m;
 1416         while (m2 != NULL && (len == -1 || len)) {
 1417                 pdata = m2->m_len;
 1418                 if (maxlen != -1 && pdata > maxlen)
 1419                         pdata = maxlen;
 1420                 printf("mbuf: %p len: %d, next: %p, %b%s", m2, m2->m_len,
 1421                     m2->m_next, m2->m_flags, "\2\20freelist\17skipfw"
 1422                     "\11proto5\10proto4\7proto3\6proto2\5proto1\4rdonly"
 1423                     "\3eor\2pkthdr\1ext", pdata ? "" : "\n");
 1424                 if (pdata)
 1425                         printf(", %*D\n", pdata, (u_char *)m2->m_data, "-");
 1426                 if (len != -1)
 1427                         len -= m2->m_len;
 1428                 m2 = m2->m_next;
 1429         }
 1430         if (len > 0)
 1431                 printf("%d bytes unaccounted for.\n", len);
 1432         return;
 1433 }
 1434 
 1435 u_int
 1436 m_fixhdr(struct mbuf *m0)
 1437 {
 1438         u_int len;
 1439 
 1440         len = m_length(m0, NULL);
 1441         m0->m_pkthdr.len = len;
 1442         return (len);
 1443 }
 1444 
 1445 u_int
 1446 m_length(struct mbuf *m0, struct mbuf **last)
 1447 {
 1448         struct mbuf *m;
 1449         u_int len;
 1450 
 1451         len = 0;
 1452         for (m = m0; m != NULL; m = m->m_next) {
 1453                 len += m->m_len;
 1454                 if (m->m_next == NULL)
 1455                         break;
 1456         }
 1457         if (last != NULL)
 1458                 *last = m;
 1459         return (len);
 1460 }
 1461 
 1462 /*
 1463  * Defragment a mbuf chain, returning the shortest possible
 1464  * chain of mbufs and clusters.  If allocation fails and
 1465  * this cannot be completed, NULL will be returned, but
 1466  * the passed in chain will be unchanged.  Upon success,
 1467  * the original chain will be freed, and the new chain
 1468  * will be returned.
 1469  *
 1470  * If a non-packet header is passed in, the original
 1471  * mbuf (chain?) will be returned unharmed.
 1472  */
 1473 struct mbuf *
 1474 m_defrag(struct mbuf *m0, int how)
 1475 {
 1476         struct mbuf *m_new = NULL, *m_final = NULL;
 1477         int progress = 0, length;
 1478 
 1479         MBUF_CHECKSLEEP(how);
 1480         if (!(m0->m_flags & M_PKTHDR))
 1481                 return (m0);
 1482 
 1483         m_fixhdr(m0); /* Needed sanity check */
 1484 
 1485 #ifdef MBUF_STRESS_TEST
 1486         if (m_defragrandomfailures) {
 1487                 int temp = arc4random() & 0xff;
 1488                 if (temp == 0xba)
 1489                         goto nospace;
 1490         }
 1491 #endif
 1492         
 1493         if (m0->m_pkthdr.len > MHLEN)
 1494                 m_final = m_getcl(how, MT_DATA, M_PKTHDR);
 1495         else
 1496                 m_final = m_gethdr(how, MT_DATA);
 1497 
 1498         if (m_final == NULL)
 1499                 goto nospace;
 1500 
 1501         if (m_dup_pkthdr(m_final, m0, how) == 0)
 1502                 goto nospace;
 1503 
 1504         m_new = m_final;
 1505 
 1506         while (progress < m0->m_pkthdr.len) {
 1507                 length = m0->m_pkthdr.len - progress;
 1508                 if (length > MCLBYTES)
 1509                         length = MCLBYTES;
 1510 
 1511                 if (m_new == NULL) {
 1512                         if (length > MLEN)
 1513                                 m_new = m_getcl(how, MT_DATA, 0);
 1514                         else
 1515                                 m_new = m_get(how, MT_DATA);
 1516                         if (m_new == NULL)
 1517                                 goto nospace;
 1518                 }
 1519 
 1520                 m_copydata(m0, progress, length, mtod(m_new, caddr_t));
 1521                 progress += length;
 1522                 m_new->m_len = length;
 1523                 if (m_new != m_final)
 1524                         m_cat(m_final, m_new);
 1525                 m_new = NULL;
 1526         }
 1527 #ifdef MBUF_STRESS_TEST
 1528         if (m0->m_next == NULL)
 1529                 m_defraguseless++;
 1530 #endif
 1531         m_freem(m0);
 1532         m0 = m_final;
 1533 #ifdef MBUF_STRESS_TEST
 1534         m_defragpackets++;
 1535         m_defragbytes += m0->m_pkthdr.len;
 1536 #endif
 1537         return (m0);
 1538 nospace:
 1539 #ifdef MBUF_STRESS_TEST
 1540         m_defragfailure++;
 1541 #endif
 1542         if (m_final)
 1543                 m_freem(m_final);
 1544         return (NULL);
 1545 }
 1546 
 1547 /*
 1548  * Defragment an mbuf chain, returning at most maxfrags separate
 1549  * mbufs+clusters.  If this is not possible NULL is returned and
 1550  * the original mbuf chain is left in it's present (potentially
 1551  * modified) state.  We use two techniques: collapsing consecutive
 1552  * mbufs and replacing consecutive mbufs by a cluster.
 1553  *
 1554  * NB: this should really be named m_defrag but that name is taken
 1555  */
 1556 struct mbuf *
 1557 m_collapse(struct mbuf *m0, int how, int maxfrags)
 1558 {
 1559         struct mbuf *m, *n, *n2, **prev;
 1560         u_int curfrags;
 1561 
 1562         /*
 1563          * Calculate the current number of frags.
 1564          */
 1565         curfrags = 0;
 1566         for (m = m0; m != NULL; m = m->m_next)
 1567                 curfrags++;
 1568         /*
 1569          * First, try to collapse mbufs.  Note that we always collapse
 1570          * towards the front so we don't need to deal with moving the
 1571          * pkthdr.  This may be suboptimal if the first mbuf has much
 1572          * less data than the following.
 1573          */
 1574         m = m0;
 1575 again:
 1576         for (;;) {
 1577                 n = m->m_next;
 1578                 if (n == NULL)
 1579                         break;
 1580                 if ((m->m_flags & M_RDONLY) == 0 &&
 1581                     n->m_len < M_TRAILINGSPACE(m)) {
 1582                         bcopy(mtod(n, void *), mtod(m, char *) + m->m_len,
 1583                                 n->m_len);
 1584                         m->m_len += n->m_len;
 1585                         m->m_next = n->m_next;
 1586                         m_free(n);
 1587                         if (--curfrags <= maxfrags)
 1588                                 return m0;
 1589                 } else
 1590                         m = n;
 1591         }
 1592         KASSERT(maxfrags > 1,
 1593                 ("maxfrags %u, but normal collapse failed", maxfrags));
 1594         /*
 1595          * Collapse consecutive mbufs to a cluster.
 1596          */
 1597         prev = &m0->m_next;             /* NB: not the first mbuf */
 1598         while ((n = *prev) != NULL) {
 1599                 if ((n2 = n->m_next) != NULL &&
 1600                     n->m_len + n2->m_len < MCLBYTES) {
 1601                         m = m_getcl(how, MT_DATA, 0);
 1602                         if (m == NULL)
 1603                                 goto bad;
 1604                         bcopy(mtod(n, void *), mtod(m, void *), n->m_len);
 1605                         bcopy(mtod(n2, void *), mtod(m, char *) + n->m_len,
 1606                                 n2->m_len);
 1607                         m->m_len = n->m_len + n2->m_len;
 1608                         m->m_next = n2->m_next;
 1609                         *prev = m;
 1610                         m_free(n);
 1611                         m_free(n2);
 1612                         if (--curfrags <= maxfrags)     /* +1 cl -2 mbufs */
 1613                                 return m0;
 1614                         /*
 1615                          * Still not there, try the normal collapse
 1616                          * again before we allocate another cluster.
 1617                          */
 1618                         goto again;
 1619                 }
 1620                 prev = &n->m_next;
 1621         }
 1622         /*
 1623          * No place where we can collapse to a cluster; punt.
 1624          * This can occur if, for example, you request 2 frags
 1625          * but the packet requires that both be clusters (we
 1626          * never reallocate the first mbuf to avoid moving the
 1627          * packet header).
 1628          */
 1629 bad:
 1630         return NULL;
 1631 }
 1632 
 1633 #ifdef MBUF_STRESS_TEST
 1634 
 1635 /*
 1636  * Fragment an mbuf chain.  There's no reason you'd ever want to do
 1637  * this in normal usage, but it's great for stress testing various
 1638  * mbuf consumers.
 1639  *
 1640  * If fragmentation is not possible, the original chain will be
 1641  * returned.
 1642  *
 1643  * Possible length values:
 1644  * 0     no fragmentation will occur
 1645  * > 0  each fragment will be of the specified length
 1646  * -1   each fragment will be the same random value in length
 1647  * -2   each fragment's length will be entirely random
 1648  * (Random values range from 1 to 256)
 1649  */
 1650 struct mbuf *
 1651 m_fragment(struct mbuf *m0, int how, int length)
 1652 {
 1653         struct mbuf *m_new = NULL, *m_final = NULL;
 1654         int progress = 0;
 1655 
 1656         if (!(m0->m_flags & M_PKTHDR))
 1657                 return (m0);
 1658         
 1659         if ((length == 0) || (length < -2))
 1660                 return (m0);
 1661 
 1662         m_fixhdr(m0); /* Needed sanity check */
 1663 
 1664         m_final = m_getcl(how, MT_DATA, M_PKTHDR);
 1665 
 1666         if (m_final == NULL)
 1667                 goto nospace;
 1668 
 1669         if (m_dup_pkthdr(m_final, m0, how) == 0)
 1670                 goto nospace;
 1671 
 1672         m_new = m_final;
 1673 
 1674         if (length == -1)
 1675                 length = 1 + (arc4random() & 255);
 1676 
 1677         while (progress < m0->m_pkthdr.len) {
 1678                 int fraglen;
 1679 
 1680                 if (length > 0)
 1681                         fraglen = length;
 1682                 else
 1683                         fraglen = 1 + (arc4random() & 255);
 1684                 if (fraglen > m0->m_pkthdr.len - progress)
 1685                         fraglen = m0->m_pkthdr.len - progress;
 1686 
 1687                 if (fraglen > MCLBYTES)
 1688                         fraglen = MCLBYTES;
 1689 
 1690                 if (m_new == NULL) {
 1691                         m_new = m_getcl(how, MT_DATA, 0);
 1692                         if (m_new == NULL)
 1693                                 goto nospace;
 1694                 }
 1695 
 1696                 m_copydata(m0, progress, fraglen, mtod(m_new, caddr_t));
 1697                 progress += fraglen;
 1698                 m_new->m_len = fraglen;
 1699                 if (m_new != m_final)
 1700                         m_cat(m_final, m_new);
 1701                 m_new = NULL;
 1702         }
 1703         m_freem(m0);
 1704         m0 = m_final;
 1705         return (m0);
 1706 nospace:
 1707         if (m_final)
 1708                 m_freem(m_final);
 1709         /* Return the original chain on failure */
 1710         return (m0);
 1711 }
 1712 
 1713 #endif
 1714 
 1715 /*
 1716  * Copy the contents of uio into a properly sized mbuf chain.
 1717  */
 1718 struct mbuf *
 1719 m_uiotombuf(struct uio *uio, int how, int len, int align, int flags)
 1720 {
 1721         struct mbuf *m, *mb;
 1722         int error, length, total;
 1723         int progress = 0;
 1724 
 1725         /*
 1726          * len can be zero or an arbitrary large value bound by
 1727          * the total data supplied by the uio.
 1728          */
 1729         if (len > 0)
 1730                 total = min(uio->uio_resid, len);
 1731         else
 1732                 total = uio->uio_resid;
 1733 
 1734         /*
 1735          * The smallest unit returned by m_getm2() is a single mbuf
 1736          * with pkthdr.  We can't align past it.
 1737          */
 1738         if (align >= MHLEN)
 1739                 return (NULL);
 1740 
 1741         /*
 1742          * Give us the full allocation or nothing.
 1743          * If len is zero return the smallest empty mbuf.
 1744          */
 1745         m = m_getm2(NULL, max(total + align, 1), how, MT_DATA, flags);
 1746         if (m == NULL)
 1747                 return (NULL);
 1748         m->m_data += align;
 1749 
 1750         /* Fill all mbufs with uio data and update header information. */
 1751         for (mb = m; mb != NULL; mb = mb->m_next) {
 1752                 length = min(M_TRAILINGSPACE(mb), total - progress);
 1753 
 1754                 error = uiomove(mtod(mb, void *), length, uio);
 1755                 if (error) {
 1756                         m_freem(m);
 1757                         return (NULL);
 1758                 }
 1759 
 1760                 mb->m_len = length;
 1761                 progress += length;
 1762                 if (flags & M_PKTHDR)
 1763                         m->m_pkthdr.len += length;
 1764         }
 1765         KASSERT(progress == total, ("%s: progress != total", __func__));
 1766 
 1767         return (m);
 1768 }
 1769 
 1770 /*
 1771  * Set the m_data pointer of a newly-allocated mbuf
 1772  * to place an object of the specified size at the
 1773  * end of the mbuf, longword aligned.
 1774  */
 1775 void
 1776 m_align(struct mbuf *m, int len)
 1777 {
 1778         int adjust;
 1779 
 1780         if (m->m_flags & M_EXT)
 1781                 adjust = m->m_ext.ext_size - len;
 1782         else if (m->m_flags & M_PKTHDR)
 1783                 adjust = MHLEN - len;
 1784         else
 1785                 adjust = MLEN - len;
 1786         m->m_data += adjust &~ (sizeof(long)-1);
 1787 }
 1788 
 1789 /*
 1790  * Create a writable copy of the mbuf chain.  While doing this
 1791  * we compact the chain with a goal of producing a chain with
 1792  * at most two mbufs.  The second mbuf in this chain is likely
 1793  * to be a cluster.  The primary purpose of this work is to create
 1794  * a writable packet for encryption, compression, etc.  The
 1795  * secondary goal is to linearize the data so the data can be
 1796  * passed to crypto hardware in the most efficient manner possible.
 1797  */
 1798 struct mbuf *
 1799 m_unshare(struct mbuf *m0, int how)
 1800 {
 1801         struct mbuf *m, *mprev;
 1802         struct mbuf *n, *mfirst, *mlast;
 1803         int len, off;
 1804 
 1805         mprev = NULL;
 1806         for (m = m0; m != NULL; m = mprev->m_next) {
 1807                 /*
 1808                  * Regular mbufs are ignored unless there's a cluster
 1809                  * in front of it that we can use to coalesce.  We do
 1810                  * the latter mainly so later clusters can be coalesced
 1811                  * also w/o having to handle them specially (i.e. convert
 1812                  * mbuf+cluster -> cluster).  This optimization is heavily
 1813                  * influenced by the assumption that we're running over
 1814                  * Ethernet where MCLBYTES is large enough that the max
 1815                  * packet size will permit lots of coalescing into a
 1816                  * single cluster.  This in turn permits efficient
 1817                  * crypto operations, especially when using hardware.
 1818                  */
 1819                 if ((m->m_flags & M_EXT) == 0) {
 1820                         if (mprev && (mprev->m_flags & M_EXT) &&
 1821                             m->m_len <= M_TRAILINGSPACE(mprev)) {
 1822                                 /* XXX: this ignores mbuf types */
 1823                                 memcpy(mtod(mprev, caddr_t) + mprev->m_len,
 1824                                        mtod(m, caddr_t), m->m_len);
 1825                                 mprev->m_len += m->m_len;
 1826                                 mprev->m_next = m->m_next;      /* unlink from chain */
 1827                                 m_free(m);                      /* reclaim mbuf */
 1828 #if 0
 1829                                 newipsecstat.ips_mbcoalesced++;
 1830 #endif
 1831                         } else {
 1832                                 mprev = m;
 1833                         }
 1834                         continue;
 1835                 }
 1836                 /*
 1837                  * Writable mbufs are left alone (for now).
 1838                  */
 1839                 if (M_WRITABLE(m)) {
 1840                         mprev = m;
 1841                         continue;
 1842                 }
 1843 
 1844                 /*
 1845                  * Not writable, replace with a copy or coalesce with
 1846                  * the previous mbuf if possible (since we have to copy
 1847                  * it anyway, we try to reduce the number of mbufs and
 1848                  * clusters so that future work is easier).
 1849                  */
 1850                 KASSERT(m->m_flags & M_EXT, ("m_flags 0x%x", m->m_flags));
 1851                 /* NB: we only coalesce into a cluster or larger */
 1852                 if (mprev != NULL && (mprev->m_flags & M_EXT) &&
 1853                     m->m_len <= M_TRAILINGSPACE(mprev)) {
 1854                         /* XXX: this ignores mbuf types */
 1855                         memcpy(mtod(mprev, caddr_t) + mprev->m_len,
 1856                                mtod(m, caddr_t), m->m_len);
 1857                         mprev->m_len += m->m_len;
 1858                         mprev->m_next = m->m_next;      /* unlink from chain */
 1859                         m_free(m);                      /* reclaim mbuf */
 1860 #if 0
 1861                         newipsecstat.ips_clcoalesced++;
 1862 #endif
 1863                         continue;
 1864                 }
 1865 
 1866                 /*
 1867                  * Allocate new space to hold the copy...
 1868                  */
 1869                 /* XXX why can M_PKTHDR be set past the first mbuf? */
 1870                 if (mprev == NULL && (m->m_flags & M_PKTHDR)) {
 1871                         /*
 1872                          * NB: if a packet header is present we must
 1873                          * allocate the mbuf separately from any cluster
 1874                          * because M_MOVE_PKTHDR will smash the data
 1875                          * pointer and drop the M_EXT marker.
 1876                          */
 1877                         MGETHDR(n, how, m->m_type);
 1878                         if (n == NULL) {
 1879                                 m_freem(m0);
 1880                                 return (NULL);
 1881                         }
 1882                         M_MOVE_PKTHDR(n, m);
 1883                         MCLGET(n, how);
 1884                         if ((n->m_flags & M_EXT) == 0) {
 1885                                 m_free(n);
 1886                                 m_freem(m0);
 1887                                 return (NULL);
 1888                         }
 1889                 } else {
 1890                         n = m_getcl(how, m->m_type, m->m_flags);
 1891                         if (n == NULL) {
 1892                                 m_freem(m0);
 1893                                 return (NULL);
 1894                         }
 1895                 }
 1896                 /*
 1897                  * ... and copy the data.  We deal with jumbo mbufs
 1898                  * (i.e. m_len > MCLBYTES) by splitting them into
 1899                  * clusters.  We could just malloc a buffer and make
 1900                  * it external but too many device drivers don't know
 1901                  * how to break up the non-contiguous memory when
 1902                  * doing DMA.
 1903                  */
 1904                 len = m->m_len;
 1905                 off = 0;
 1906                 mfirst = n;
 1907                 mlast = NULL;
 1908                 for (;;) {
 1909                         int cc = min(len, MCLBYTES);
 1910                         memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, cc);
 1911                         n->m_len = cc;
 1912                         if (mlast != NULL)
 1913                                 mlast->m_next = n;
 1914                         mlast = n;      
 1915 #if 0
 1916                         newipsecstat.ips_clcopied++;
 1917 #endif
 1918 
 1919                         len -= cc;
 1920                         if (len <= 0)
 1921                                 break;
 1922                         off += cc;
 1923 
 1924                         n = m_getcl(how, m->m_type, m->m_flags);
 1925                         if (n == NULL) {
 1926                                 m_freem(mfirst);
 1927                                 m_freem(m0);
 1928                                 return (NULL);
 1929                         }
 1930                 }
 1931                 n->m_next = m->m_next; 
 1932                 if (mprev == NULL)
 1933                         m0 = mfirst;            /* new head of chain */
 1934                 else
 1935                         mprev->m_next = mfirst; /* replace old mbuf */
 1936                 m_free(m);                      /* release old mbuf */
 1937                 mprev = mfirst;
 1938         }
 1939         return (m0);
 1940 }

Cache object: c1dfc4c3c4b64f84a44a324800016bb0


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.