Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/kern/uipc_mbuf.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* $NetBSD: uipc_mbuf.c,v 1.128.6.1 2009/04/07 23:43:16 snj Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1982, 1986, 1988, 1991, 1993 35 * The Regents of the University of California. All rights reserved. 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 1. Redistributions of source code must retain the above copyright 41 * notice, this list of conditions and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. Neither the name of the University nor the names of its contributors 46 * may be used to endorse or promote products derived from this software 47 * without specific prior written permission. 48 * 49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 59 * SUCH DAMAGE. 60 * 61 * @(#)uipc_mbuf.c 8.4 (Berkeley) 2/14/95 62 */ 63 64 #include <sys/cdefs.h> 65 __KERNEL_RCSID(0, "$NetBSD: uipc_mbuf.c,v 1.128.6.1 2009/04/07 23:43:16 snj Exp $"); 66 67 #include "opt_mbuftrace.h" 68 #include "opt_ddb.h" 69 70 #include <sys/param.h> 71 #include <sys/systm.h> 72 #include <sys/atomic.h> 73 #include <sys/cpu.h> 74 #include <sys/proc.h> 75 #include <sys/malloc.h> 76 #define MBTYPES 77 #include <sys/mbuf.h> 78 #include <sys/kernel.h> 79 #include <sys/syslog.h> 80 #include <sys/domain.h> 81 #include <sys/protosw.h> 82 #include <sys/percpu.h> 83 #include <sys/pool.h> 84 #include <sys/socket.h> 85 #include <sys/sysctl.h> 86 87 #include <net/if.h> 88 89 #include <uvm/uvm.h> 90 91 pool_cache_t mb_cache; /* mbuf cache */ 92 pool_cache_t mcl_cache; /* mbuf cluster cache */ 93 94 struct mbstat mbstat; 95 int max_linkhdr; 96 int max_protohdr; 97 int max_hdr; 98 int max_datalen; 99 100 static int mb_ctor(void *, void *, int); 101 102 static void *mclpool_alloc(struct pool *, int); 103 static void mclpool_release(struct pool *, void *); 104 105 static struct pool_allocator mclpool_allocator = { 106 .pa_alloc = mclpool_alloc, 107 .pa_free = mclpool_release, 108 }; 109 110 static struct mbuf *m_copym0(struct mbuf *, int, int, int, int); 111 static struct mbuf *m_split0(struct mbuf *, int, int, int); 112 static int m_copyback0(struct mbuf **, int, int, const void *, int, int); 113 114 /* flags for m_copyback0 */ 115 #define M_COPYBACK0_COPYBACK 0x0001 /* copyback from cp */ 116 #define M_COPYBACK0_PRESERVE 0x0002 /* preserve original data */ 117 #define M_COPYBACK0_COW 0x0004 /* do copy-on-write */ 118 #define M_COPYBACK0_EXTEND 0x0008 /* extend chain */ 119 120 static const char mclpool_warnmsg[] = 121 "WARNING: mclpool limit reached; increase NMBCLUSTERS"; 122 123 MALLOC_DEFINE(M_MBUF, "mbuf", "mbuf"); 124 125 static percpu_t *mbstat_percpu; 126 127 #ifdef MBUFTRACE 128 struct mownerhead mowners = LIST_HEAD_INITIALIZER(mowners); 129 struct mowner unknown_mowners[] = { 130 MOWNER_INIT("unknown", "free"), 131 MOWNER_INIT("unknown", "data"), 132 MOWNER_INIT("unknown", "header"), 133 MOWNER_INIT("unknown", "soname"), 134 MOWNER_INIT("unknown", "soopts"), 135 MOWNER_INIT("unknown", "ftable"), 136 MOWNER_INIT("unknown", "control"), 137 MOWNER_INIT("unknown", "oobdata"), 138 }; 139 struct mowner revoked_mowner = MOWNER_INIT("revoked", ""); 140 #endif 141 142 #define MEXT_ISEMBEDDED(m) ((m)->m_ext_ref == (m)) 143 144 #define MCLADDREFERENCE(o, n) \ 145 do { \ 146 KASSERT(((o)->m_flags & M_EXT) != 0); \ 147 KASSERT(((n)->m_flags & M_EXT) == 0); \ 148 KASSERT((o)->m_ext.ext_refcnt >= 1); \ 149 (n)->m_flags |= ((o)->m_flags & M_EXTCOPYFLAGS); \ 150 atomic_inc_uint(&(o)->m_ext.ext_refcnt); \ 151 (n)->m_ext_ref = (o)->m_ext_ref; \ 152 mowner_ref((n), (n)->m_flags); \ 153 MCLREFDEBUGN((n), __FILE__, __LINE__); \ 154 } while (/* CONSTCOND */ 0) 155 156 /* 157 * Initialize the mbuf allocator. 158 */ 159 void 160 mbinit(void) 161 { 162 163 CTASSERT(sizeof(struct _m_ext) <= MHLEN); 164 CTASSERT(sizeof(struct mbuf) == MSIZE); 165 166 mclpool_allocator.pa_backingmap = mb_map; 167 168 mb_cache = pool_cache_init(msize, 0, 0, 0, "mbpl", 169 NULL, IPL_VM, mb_ctor, NULL, NULL); 170 KASSERT(mb_cache != NULL); 171 172 mcl_cache = pool_cache_init(mclbytes, 0, 0, 0, "mclpl", 173 &mclpool_allocator, IPL_VM, NULL, NULL, NULL); 174 KASSERT(mcl_cache != NULL); 175 176 pool_cache_set_drain_hook(mb_cache, m_reclaim, NULL); 177 pool_cache_set_drain_hook(mcl_cache, m_reclaim, NULL); 178 179 /* 180 * Set the hard limit on the mclpool to the number of 181 * mbuf clusters the kernel is to support. Log the limit 182 * reached message max once a minute. 183 */ 184 pool_cache_sethardlimit(mcl_cache, nmbclusters, mclpool_warnmsg, 60); 185 186 mbstat_percpu = percpu_alloc(sizeof(struct mbstat_cpu)); 187 188 /* 189 * Set a low water mark for both mbufs and clusters. This should 190 * help ensure that they can be allocated in a memory starvation 191 * situation. This is important for e.g. diskless systems which 192 * must allocate mbufs in order for the pagedaemon to clean pages. 193 */ 194 pool_cache_setlowat(mb_cache, mblowat); 195 pool_cache_setlowat(mcl_cache, mcllowat); 196 197 #ifdef MBUFTRACE 198 { 199 /* 200 * Attach the unknown mowners. 201 */ 202 int i; 203 MOWNER_ATTACH(&revoked_mowner); 204 for (i = sizeof(unknown_mowners)/sizeof(unknown_mowners[0]); 205 i-- > 0; ) 206 MOWNER_ATTACH(&unknown_mowners[i]); 207 } 208 #endif 209 } 210 211 /* 212 * sysctl helper routine for the kern.mbuf subtree. nmbclusters may 213 * or may not be writable, and mblowat and mcllowat need range 214 * checking and pool tweaking after being reset. 215 */ 216 static int 217 sysctl_kern_mbuf(SYSCTLFN_ARGS) 218 { 219 int error, newval; 220 struct sysctlnode node; 221 222 node = *rnode; 223 node.sysctl_data = &newval; 224 switch (rnode->sysctl_num) { 225 case MBUF_NMBCLUSTERS: 226 if (mb_map != NULL) { 227 node.sysctl_flags &= ~CTLFLAG_READWRITE; 228 node.sysctl_flags |= CTLFLAG_READONLY; 229 } 230 /* FALLTHROUGH */ 231 case MBUF_MBLOWAT: 232 case MBUF_MCLLOWAT: 233 newval = *(int*)rnode->sysctl_data; 234 break; 235 default: 236 return (EOPNOTSUPP); 237 } 238 239 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 240 if (error || newp == NULL) 241 return (error); 242 if (newval < 0) 243 return (EINVAL); 244 245 switch (node.sysctl_num) { 246 case MBUF_NMBCLUSTERS: 247 if (newval < nmbclusters) 248 return (EINVAL); 249 nmbclusters = newval; 250 pool_cache_sethardlimit(mcl_cache, nmbclusters, 251 mclpool_warnmsg, 60); 252 break; 253 case MBUF_MBLOWAT: 254 mblowat = newval; 255 pool_cache_setlowat(mb_cache, mblowat); 256 break; 257 case MBUF_MCLLOWAT: 258 mcllowat = newval; 259 pool_cache_setlowat(mcl_cache, mcllowat); 260 break; 261 } 262 263 return (0); 264 } 265 266 #ifdef MBUFTRACE 267 static void 268 mowner_conver_to_user_cb(void *v1, void *v2, struct cpu_info *ci) 269 { 270 struct mowner_counter *mc = v1; 271 struct mowner_user *mo_user = v2; 272 int i; 273 274 for (i = 0; i < MOWNER_COUNTER_NCOUNTERS; i++) { 275 mo_user->mo_counter[i] += mc->mc_counter[i]; 276 } 277 } 278 279 static void 280 mowner_convert_to_user(struct mowner *mo, struct mowner_user *mo_user) 281 { 282 283 memset(mo_user, 0, sizeof(*mo_user)); 284 CTASSERT(sizeof(mo_user->mo_name) == sizeof(mo->mo_name)); 285 CTASSERT(sizeof(mo_user->mo_descr) == sizeof(mo->mo_descr)); 286 memcpy(mo_user->mo_name, mo->mo_name, sizeof(mo->mo_name)); 287 memcpy(mo_user->mo_descr, mo->mo_descr, sizeof(mo->mo_descr)); 288 percpu_foreach(mo->mo_counters, mowner_conver_to_user_cb, mo_user); 289 } 290 291 static int 292 sysctl_kern_mbuf_mowners(SYSCTLFN_ARGS) 293 { 294 struct mowner *mo; 295 size_t len = 0; 296 int error = 0; 297 298 if (namelen != 0) 299 return (EINVAL); 300 if (newp != NULL) 301 return (EPERM); 302 303 LIST_FOREACH(mo, &mowners, mo_link) { 304 struct mowner_user mo_user; 305 306 mowner_convert_to_user(mo, &mo_user); 307 308 if (oldp != NULL) { 309 if (*oldlenp - len < sizeof(mo_user)) { 310 error = ENOMEM; 311 break; 312 } 313 error = copyout(&mo_user, (char *)oldp + len, 314 sizeof(mo_user)); 315 if (error) 316 break; 317 } 318 len += sizeof(mo_user); 319 } 320 321 if (error == 0) 322 *oldlenp = len; 323 324 return (error); 325 } 326 #endif /* MBUFTRACE */ 327 328 static void 329 mbstat_conver_to_user_cb(void *v1, void *v2, struct cpu_info *ci) 330 { 331 struct mbstat_cpu *mbsc = v1; 332 struct mbstat *mbs = v2; 333 int i; 334 335 for (i = 0; i < __arraycount(mbs->m_mtypes); i++) { 336 mbs->m_mtypes[i] += mbsc->m_mtypes[i]; 337 } 338 } 339 340 static void 341 mbstat_convert_to_user(struct mbstat *mbs) 342 { 343 344 memset(mbs, 0, sizeof(*mbs)); 345 mbs->m_drain = mbstat.m_drain; 346 percpu_foreach(mbstat_percpu, mbstat_conver_to_user_cb, mbs); 347 } 348 349 static int 350 sysctl_kern_mbuf_stats(SYSCTLFN_ARGS) 351 { 352 struct sysctlnode node; 353 struct mbstat mbs; 354 355 mbstat_convert_to_user(&mbs); 356 node = *rnode; 357 node.sysctl_data = &mbs; 358 node.sysctl_size = sizeof(mbs); 359 return sysctl_lookup(SYSCTLFN_CALL(&node)); 360 } 361 362 SYSCTL_SETUP(sysctl_kern_mbuf_setup, "sysctl kern.mbuf subtree setup") 363 { 364 365 sysctl_createv(clog, 0, NULL, NULL, 366 CTLFLAG_PERMANENT, 367 CTLTYPE_NODE, "kern", NULL, 368 NULL, 0, NULL, 0, 369 CTL_KERN, CTL_EOL); 370 sysctl_createv(clog, 0, NULL, NULL, 371 CTLFLAG_PERMANENT, 372 CTLTYPE_NODE, "mbuf", 373 SYSCTL_DESCR("mbuf control variables"), 374 NULL, 0, NULL, 0, 375 CTL_KERN, KERN_MBUF, CTL_EOL); 376 377 sysctl_createv(clog, 0, NULL, NULL, 378 CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE, 379 CTLTYPE_INT, "msize", 380 SYSCTL_DESCR("mbuf base size"), 381 NULL, msize, NULL, 0, 382 CTL_KERN, KERN_MBUF, MBUF_MSIZE, CTL_EOL); 383 sysctl_createv(clog, 0, NULL, NULL, 384 CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE, 385 CTLTYPE_INT, "mclbytes", 386 SYSCTL_DESCR("mbuf cluster size"), 387 NULL, mclbytes, NULL, 0, 388 CTL_KERN, KERN_MBUF, MBUF_MCLBYTES, CTL_EOL); 389 sysctl_createv(clog, 0, NULL, NULL, 390 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 391 CTLTYPE_INT, "nmbclusters", 392 SYSCTL_DESCR("Limit on the number of mbuf clusters"), 393 sysctl_kern_mbuf, 0, &nmbclusters, 0, 394 CTL_KERN, KERN_MBUF, MBUF_NMBCLUSTERS, CTL_EOL); 395 sysctl_createv(clog, 0, NULL, NULL, 396 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 397 CTLTYPE_INT, "mblowat", 398 SYSCTL_DESCR("mbuf low water mark"), 399 sysctl_kern_mbuf, 0, &mblowat, 0, 400 CTL_KERN, KERN_MBUF, MBUF_MBLOWAT, CTL_EOL); 401 sysctl_createv(clog, 0, NULL, NULL, 402 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 403 CTLTYPE_INT, "mcllowat", 404 SYSCTL_DESCR("mbuf cluster low water mark"), 405 sysctl_kern_mbuf, 0, &mcllowat, 0, 406 CTL_KERN, KERN_MBUF, MBUF_MCLLOWAT, CTL_EOL); 407 sysctl_createv(clog, 0, NULL, NULL, 408 CTLFLAG_PERMANENT, 409 CTLTYPE_STRUCT, "stats", 410 SYSCTL_DESCR("mbuf allocation statistics"), 411 sysctl_kern_mbuf_stats, 0, NULL, 0, 412 CTL_KERN, KERN_MBUF, MBUF_STATS, CTL_EOL); 413 #ifdef MBUFTRACE 414 sysctl_createv(clog, 0, NULL, NULL, 415 CTLFLAG_PERMANENT, 416 CTLTYPE_STRUCT, "mowners", 417 SYSCTL_DESCR("Information about mbuf owners"), 418 sysctl_kern_mbuf_mowners, 0, NULL, 0, 419 CTL_KERN, KERN_MBUF, MBUF_MOWNERS, CTL_EOL); 420 #endif /* MBUFTRACE */ 421 } 422 423 static void * 424 mclpool_alloc(struct pool *pp, int flags) 425 { 426 bool waitok = (flags & PR_WAITOK) ? true : false; 427 428 return ((void *)uvm_km_alloc_poolpage(mb_map, waitok)); 429 } 430 431 static void 432 mclpool_release(struct pool *pp, void *v) 433 { 434 435 uvm_km_free_poolpage(mb_map, (vaddr_t)v); 436 } 437 438 /*ARGSUSED*/ 439 static int 440 mb_ctor(void *arg, void *object, int flags) 441 { 442 struct mbuf *m = object; 443 444 #ifdef POOL_VTOPHYS 445 m->m_paddr = POOL_VTOPHYS(m); 446 #else 447 m->m_paddr = M_PADDR_INVALID; 448 #endif 449 return (0); 450 } 451 452 void 453 m_reclaim(void *arg, int flags) 454 { 455 struct domain *dp; 456 const struct protosw *pr; 457 struct ifnet *ifp; 458 int s; 459 460 KERNEL_LOCK(1, NULL); 461 s = splvm(); 462 DOMAIN_FOREACH(dp) { 463 for (pr = dp->dom_protosw; 464 pr < dp->dom_protoswNPROTOSW; pr++) 465 if (pr->pr_drain) 466 (*pr->pr_drain)(); 467 } 468 IFNET_FOREACH(ifp) { 469 if (ifp->if_drain) 470 (*ifp->if_drain)(ifp); 471 } 472 splx(s); 473 mbstat.m_drain++; 474 KERNEL_UNLOCK_ONE(NULL); 475 } 476 477 /* 478 * Space allocation routines. 479 * These are also available as macros 480 * for critical paths. 481 */ 482 struct mbuf * 483 m_get(int nowait, int type) 484 { 485 struct mbuf *m; 486 487 m = pool_cache_get(mb_cache, 488 nowait == M_WAIT ? PR_WAITOK|PR_LIMITFAIL : 0); 489 if (m == NULL) 490 return NULL; 491 492 mbstat_type_add(type, 1); 493 mowner_init(m, type); 494 m->m_ext_ref = m; 495 m->m_type = type; 496 m->m_next = NULL; 497 m->m_nextpkt = NULL; 498 m->m_data = m->m_dat; 499 m->m_flags = 0; 500 501 return m; 502 } 503 504 struct mbuf * 505 m_gethdr(int nowait, int type) 506 { 507 struct mbuf *m; 508 509 m = m_get(nowait, type); 510 if (m == NULL) 511 return NULL; 512 513 m->m_data = m->m_pktdat; 514 m->m_flags = M_PKTHDR; 515 m->m_pkthdr.rcvif = NULL; 516 m->m_pkthdr.csum_flags = 0; 517 m->m_pkthdr.csum_data = 0; 518 SLIST_INIT(&m->m_pkthdr.tags); 519 520 return m; 521 } 522 523 struct mbuf * 524 m_getclr(int nowait, int type) 525 { 526 struct mbuf *m; 527 528 MGET(m, nowait, type); 529 if (m == 0) 530 return (NULL); 531 memset(mtod(m, void *), 0, MLEN); 532 return (m); 533 } 534 535 void 536 m_clget(struct mbuf *m, int nowait) 537 { 538 539 MCLGET(m, nowait); 540 } 541 542 struct mbuf * 543 m_free(struct mbuf *m) 544 { 545 struct mbuf *n; 546 547 MFREE(m, n); 548 return (n); 549 } 550 551 void 552 m_freem(struct mbuf *m) 553 { 554 struct mbuf *n; 555 556 if (m == NULL) 557 return; 558 do { 559 MFREE(m, n); 560 m = n; 561 } while (m); 562 } 563 564 #ifdef MBUFTRACE 565 /* 566 * Walk a chain of mbufs, claiming ownership of each mbuf in the chain. 567 */ 568 void 569 m_claimm(struct mbuf *m, struct mowner *mo) 570 { 571 572 for (; m != NULL; m = m->m_next) 573 MCLAIM(m, mo); 574 } 575 #endif 576 577 /* 578 * Mbuffer utility routines. 579 */ 580 581 /* 582 * Lesser-used path for M_PREPEND: 583 * allocate new mbuf to prepend to chain, 584 * copy junk along. 585 */ 586 struct mbuf * 587 m_prepend(struct mbuf *m, int len, int how) 588 { 589 struct mbuf *mn; 590 591 MGET(mn, how, m->m_type); 592 if (mn == (struct mbuf *)NULL) { 593 m_freem(m); 594 return ((struct mbuf *)NULL); 595 } 596 if (m->m_flags & M_PKTHDR) { 597 M_MOVE_PKTHDR(mn, m); 598 } else { 599 MCLAIM(mn, m->m_owner); 600 } 601 mn->m_next = m; 602 m = mn; 603 if (len < MHLEN) 604 MH_ALIGN(m, len); 605 m->m_len = len; 606 return (m); 607 } 608 609 /* 610 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 611 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 612 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 613 */ 614 int MCFail; 615 616 struct mbuf * 617 m_copym(struct mbuf *m, int off0, int len, int wait) 618 { 619 620 return m_copym0(m, off0, len, wait, 0); /* shallow copy on M_EXT */ 621 } 622 623 struct mbuf * 624 m_dup(struct mbuf *m, int off0, int len, int wait) 625 { 626 627 return m_copym0(m, off0, len, wait, 1); /* deep copy */ 628 } 629 630 static struct mbuf * 631 m_copym0(struct mbuf *m, int off0, int len, int wait, int deep) 632 { 633 struct mbuf *n, **np; 634 int off = off0; 635 struct mbuf *top; 636 int copyhdr = 0; 637 638 if (off < 0 || len < 0) 639 panic("m_copym: off %d, len %d", off, len); 640 if (off == 0 && m->m_flags & M_PKTHDR) 641 copyhdr = 1; 642 while (off > 0) { 643 if (m == 0) 644 panic("m_copym: m == 0, off %d", off); 645 if (off < m->m_len) 646 break; 647 off -= m->m_len; 648 m = m->m_next; 649 } 650 np = ⊤ 651 top = 0; 652 while (len > 0) { 653 if (m == 0) { 654 if (len != M_COPYALL) 655 panic("m_copym: m == 0, len %d [!COPYALL]", 656 len); 657 break; 658 } 659 MGET(n, wait, m->m_type); 660 *np = n; 661 if (n == 0) 662 goto nospace; 663 MCLAIM(n, m->m_owner); 664 if (copyhdr) { 665 M_COPY_PKTHDR(n, m); 666 if (len == M_COPYALL) 667 n->m_pkthdr.len -= off0; 668 else 669 n->m_pkthdr.len = len; 670 copyhdr = 0; 671 } 672 n->m_len = min(len, m->m_len - off); 673 if (m->m_flags & M_EXT) { 674 if (!deep) { 675 n->m_data = m->m_data + off; 676 MCLADDREFERENCE(m, n); 677 } else { 678 /* 679 * we are unsure about the way m was allocated. 680 * copy into multiple MCLBYTES cluster mbufs. 681 */ 682 MCLGET(n, wait); 683 n->m_len = 0; 684 n->m_len = M_TRAILINGSPACE(n); 685 n->m_len = min(n->m_len, len); 686 n->m_len = min(n->m_len, m->m_len - off); 687 memcpy(mtod(n, void *), mtod(m, char *) + off, 688 (unsigned)n->m_len); 689 } 690 } else 691 memcpy(mtod(n, void *), mtod(m, char *) + off, 692 (unsigned)n->m_len); 693 if (len != M_COPYALL) 694 len -= n->m_len; 695 off += n->m_len; 696 #ifdef DIAGNOSTIC 697 if (off > m->m_len) 698 panic("m_copym0 overrun"); 699 #endif 700 if (off == m->m_len) { 701 m = m->m_next; 702 off = 0; 703 } 704 np = &n->m_next; 705 } 706 if (top == 0) 707 MCFail++; 708 return (top); 709 nospace: 710 m_freem(top); 711 MCFail++; 712 return (NULL); 713 } 714 715 /* 716 * Copy an entire packet, including header (which must be present). 717 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'. 718 */ 719 struct mbuf * 720 m_copypacket(struct mbuf *m, int how) 721 { 722 struct mbuf *top, *n, *o; 723 724 MGET(n, how, m->m_type); 725 top = n; 726 if (!n) 727 goto nospace; 728 729 MCLAIM(n, m->m_owner); 730 M_COPY_PKTHDR(n, m); 731 n->m_len = m->m_len; 732 if (m->m_flags & M_EXT) { 733 n->m_data = m->m_data; 734 MCLADDREFERENCE(m, n); 735 } else { 736 memcpy(mtod(n, char *), mtod(m, char *), n->m_len); 737 } 738 739 m = m->m_next; 740 while (m) { 741 MGET(o, how, m->m_type); 742 if (!o) 743 goto nospace; 744 745 MCLAIM(o, m->m_owner); 746 n->m_next = o; 747 n = n->m_next; 748 749 n->m_len = m->m_len; 750 if (m->m_flags & M_EXT) { 751 n->m_data = m->m_data; 752 MCLADDREFERENCE(m, n); 753 } else { 754 memcpy(mtod(n, char *), mtod(m, char *), n->m_len); 755 } 756 757 m = m->m_next; 758 } 759 return top; 760 nospace: 761 m_freem(top); 762 MCFail++; 763 return NULL; 764 } 765 766 /* 767 * Copy data from an mbuf chain starting "off" bytes from the beginning, 768 * continuing for "len" bytes, into the indicated buffer. 769 */ 770 void 771 m_copydata(struct mbuf *m, int off, int len, void *vp) 772 { 773 unsigned count; 774 void * cp = vp; 775 776 if (off < 0 || len < 0) 777 panic("m_copydata: off %d, len %d", off, len); 778 while (off > 0) { 779 if (m == NULL) 780 panic("m_copydata: m == NULL, off %d", off); 781 if (off < m->m_len) 782 break; 783 off -= m->m_len; 784 m = m->m_next; 785 } 786 while (len > 0) { 787 if (m == NULL) 788 panic("m_copydata: m == NULL, len %d", len); 789 count = min(m->m_len - off, len); 790 memcpy(cp, mtod(m, char *) + off, count); 791 len -= count; 792 cp = (char *)cp + count; 793 off = 0; 794 m = m->m_next; 795 } 796 } 797 798 /* 799 * Concatenate mbuf chain n to m. 800 * n might be copied into m (when n->m_len is small), therefore data portion of 801 * n could be copied into an mbuf of different mbuf type. 802 * Any m_pkthdr is not updated. 803 */ 804 void 805 m_cat(struct mbuf *m, struct mbuf *n) 806 { 807 808 while (m->m_next) 809 m = m->m_next; 810 while (n) { 811 if (M_READONLY(m) || n->m_len > M_TRAILINGSPACE(m)) { 812 /* just join the two chains */ 813 m->m_next = n; 814 return; 815 } 816 /* splat the data from one into the other */ 817 memcpy(mtod(m, char *) + m->m_len, mtod(n, void *), 818 (u_int)n->m_len); 819 m->m_len += n->m_len; 820 n = m_free(n); 821 } 822 } 823 824 void 825 m_adj(struct mbuf *mp, int req_len) 826 { 827 int len = req_len; 828 struct mbuf *m; 829 int count; 830 831 if ((m = mp) == NULL) 832 return; 833 if (len >= 0) { 834 /* 835 * Trim from head. 836 */ 837 while (m != NULL && len > 0) { 838 if (m->m_len <= len) { 839 len -= m->m_len; 840 m->m_len = 0; 841 m = m->m_next; 842 } else { 843 m->m_len -= len; 844 m->m_data += len; 845 len = 0; 846 } 847 } 848 m = mp; 849 if (mp->m_flags & M_PKTHDR) 850 m->m_pkthdr.len -= (req_len - len); 851 } else { 852 /* 853 * Trim from tail. Scan the mbuf chain, 854 * calculating its length and finding the last mbuf. 855 * If the adjustment only affects this mbuf, then just 856 * adjust and return. Otherwise, rescan and truncate 857 * after the remaining size. 858 */ 859 len = -len; 860 count = 0; 861 for (;;) { 862 count += m->m_len; 863 if (m->m_next == (struct mbuf *)0) 864 break; 865 m = m->m_next; 866 } 867 if (m->m_len >= len) { 868 m->m_len -= len; 869 if (mp->m_flags & M_PKTHDR) 870 mp->m_pkthdr.len -= len; 871 return; 872 } 873 count -= len; 874 if (count < 0) 875 count = 0; 876 /* 877 * Correct length for chain is "count". 878 * Find the mbuf with last data, adjust its length, 879 * and toss data from remaining mbufs on chain. 880 */ 881 m = mp; 882 if (m->m_flags & M_PKTHDR) 883 m->m_pkthdr.len = count; 884 for (; m; m = m->m_next) { 885 if (m->m_len >= count) { 886 m->m_len = count; 887 break; 888 } 889 count -= m->m_len; 890 } 891 if (m) 892 while (m->m_next) 893 (m = m->m_next)->m_len = 0; 894 } 895 } 896 897 /* 898 * Rearrange an mbuf chain so that len bytes are contiguous 899 * and in the data area of an mbuf (so that mtod and dtom 900 * will work for a structure of size len). Returns the resulting 901 * mbuf chain on success, frees it and returns null on failure. 902 * If there is room, it will add up to max_protohdr-len extra bytes to the 903 * contiguous region in an attempt to avoid being called next time. 904 */ 905 int MPFail; 906 907 struct mbuf * 908 m_pullup(struct mbuf *n, int len) 909 { 910 struct mbuf *m; 911 int count; 912 int space; 913 914 /* 915 * If first mbuf has no cluster, and has room for len bytes 916 * without shifting current data, pullup into it, 917 * otherwise allocate a new mbuf to prepend to the chain. 918 */ 919 if ((n->m_flags & M_EXT) == 0 && 920 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 921 if (n->m_len >= len) 922 return (n); 923 m = n; 924 n = n->m_next; 925 len -= m->m_len; 926 } else { 927 if (len > MHLEN) 928 goto bad; 929 MGET(m, M_DONTWAIT, n->m_type); 930 if (m == 0) 931 goto bad; 932 MCLAIM(m, n->m_owner); 933 m->m_len = 0; 934 if (n->m_flags & M_PKTHDR) { 935 M_MOVE_PKTHDR(m, n); 936 } 937 } 938 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 939 do { 940 count = min(min(max(len, max_protohdr), space), n->m_len); 941 memcpy(mtod(m, char *) + m->m_len, mtod(n, void *), 942 (unsigned)count); 943 len -= count; 944 m->m_len += count; 945 n->m_len -= count; 946 space -= count; 947 if (n->m_len) 948 n->m_data += count; 949 else 950 n = m_free(n); 951 } while (len > 0 && n); 952 if (len > 0) { 953 (void) m_free(m); 954 goto bad; 955 } 956 m->m_next = n; 957 return (m); 958 bad: 959 m_freem(n); 960 MPFail++; 961 return (NULL); 962 } 963 964 /* 965 * Like m_pullup(), except a new mbuf is always allocated, and we allow 966 * the amount of empty space before the data in the new mbuf to be specified 967 * (in the event that the caller expects to prepend later). 968 */ 969 int MSFail; 970 971 struct mbuf * 972 m_copyup(struct mbuf *n, int len, int dstoff) 973 { 974 struct mbuf *m; 975 int count, space; 976 977 if (len > (MHLEN - dstoff)) 978 goto bad; 979 MGET(m, M_DONTWAIT, n->m_type); 980 if (m == NULL) 981 goto bad; 982 MCLAIM(m, n->m_owner); 983 m->m_len = 0; 984 if (n->m_flags & M_PKTHDR) { 985 M_MOVE_PKTHDR(m, n); 986 } 987 m->m_data += dstoff; 988 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 989 do { 990 count = min(min(max(len, max_protohdr), space), n->m_len); 991 memcpy(mtod(m, char *) + m->m_len, mtod(n, void *), 992 (unsigned)count); 993 len -= count; 994 m->m_len += count; 995 n->m_len -= count; 996 space -= count; 997 if (n->m_len) 998 n->m_data += count; 999 else 1000 n = m_free(n); 1001 } while (len > 0 && n); 1002 if (len > 0) { 1003 (void) m_free(m); 1004 goto bad; 1005 } 1006 m->m_next = n; 1007 return (m); 1008 bad: 1009 m_freem(n); 1010 MSFail++; 1011 return (NULL); 1012 } 1013 1014 /* 1015 * Partition an mbuf chain in two pieces, returning the tail -- 1016 * all but the first len0 bytes. In case of failure, it returns NULL and 1017 * attempts to restore the chain to its original state. 1018 */ 1019 struct mbuf * 1020 m_split(struct mbuf *m0, int len0, int wait) 1021 { 1022 1023 return m_split0(m0, len0, wait, 1); 1024 } 1025 1026 static struct mbuf * 1027 m_split0(struct mbuf *m0, int len0, int wait, int copyhdr) 1028 { 1029 struct mbuf *m, *n; 1030 unsigned len = len0, remain, len_save; 1031 1032 for (m = m0; m && len > m->m_len; m = m->m_next) 1033 len -= m->m_len; 1034 if (m == 0) 1035 return (NULL); 1036 remain = m->m_len - len; 1037 if (copyhdr && (m0->m_flags & M_PKTHDR)) { 1038 MGETHDR(n, wait, m0->m_type); 1039 if (n == 0) 1040 return (NULL); 1041 MCLAIM(n, m0->m_owner); 1042 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 1043 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 1044 len_save = m0->m_pkthdr.len; 1045 m0->m_pkthdr.len = len0; 1046 if (m->m_flags & M_EXT) 1047 goto extpacket; 1048 if (remain > MHLEN) { 1049 /* m can't be the lead packet */ 1050 MH_ALIGN(n, 0); 1051 n->m_len = 0; 1052 n->m_next = m_split(m, len, wait); 1053 if (n->m_next == 0) { 1054 (void) m_free(n); 1055 m0->m_pkthdr.len = len_save; 1056 return (NULL); 1057 } else 1058 return (n); 1059 } else 1060 MH_ALIGN(n, remain); 1061 } else if (remain == 0) { 1062 n = m->m_next; 1063 m->m_next = 0; 1064 return (n); 1065 } else { 1066 MGET(n, wait, m->m_type); 1067 if (n == 0) 1068 return (NULL); 1069 MCLAIM(n, m->m_owner); 1070 M_ALIGN(n, remain); 1071 } 1072 extpacket: 1073 if (m->m_flags & M_EXT) { 1074 n->m_data = m->m_data + len; 1075 MCLADDREFERENCE(m, n); 1076 } else { 1077 memcpy(mtod(n, void *), mtod(m, char *) + len, remain); 1078 } 1079 n->m_len = remain; 1080 m->m_len = len; 1081 n->m_next = m->m_next; 1082 m->m_next = 0; 1083 return (n); 1084 } 1085 /* 1086 * Routine to copy from device local memory into mbufs. 1087 */ 1088 struct mbuf * 1089 m_devget(char *buf, int totlen, int off0, struct ifnet *ifp, 1090 void (*copy)(const void *from, void *to, size_t len)) 1091 { 1092 struct mbuf *m; 1093 struct mbuf *top = 0, **mp = ⊤ 1094 int off = off0, len; 1095 char *cp; 1096 char *epkt; 1097 1098 cp = buf; 1099 epkt = cp + totlen; 1100 if (off) { 1101 /* 1102 * If 'off' is non-zero, packet is trailer-encapsulated, 1103 * so we have to skip the type and length fields. 1104 */ 1105 cp += off + 2 * sizeof(uint16_t); 1106 totlen -= 2 * sizeof(uint16_t); 1107 } 1108 MGETHDR(m, M_DONTWAIT, MT_DATA); 1109 if (m == 0) 1110 return (NULL); 1111 m->m_pkthdr.rcvif = ifp; 1112 m->m_pkthdr.len = totlen; 1113 m->m_len = MHLEN; 1114 1115 while (totlen > 0) { 1116 if (top) { 1117 MGET(m, M_DONTWAIT, MT_DATA); 1118 if (m == 0) { 1119 m_freem(top); 1120 return (NULL); 1121 } 1122 m->m_len = MLEN; 1123 } 1124 len = min(totlen, epkt - cp); 1125 if (len >= MINCLSIZE) { 1126 MCLGET(m, M_DONTWAIT); 1127 if ((m->m_flags & M_EXT) == 0) { 1128 m_free(m); 1129 m_freem(top); 1130 return (NULL); 1131 } 1132 m->m_len = len = min(len, MCLBYTES); 1133 } else { 1134 /* 1135 * Place initial small packet/header at end of mbuf. 1136 */ 1137 if (len < m->m_len) { 1138 if (top == 0 && len + max_linkhdr <= m->m_len) 1139 m->m_data += max_linkhdr; 1140 m->m_len = len; 1141 } else 1142 len = m->m_len; 1143 } 1144 if (copy) 1145 copy(cp, mtod(m, void *), (size_t)len); 1146 else 1147 memcpy(mtod(m, void *), cp, (size_t)len); 1148 cp += len; 1149 *mp = m; 1150 mp = &m->m_next; 1151 totlen -= len; 1152 if (cp == epkt) 1153 cp = buf; 1154 } 1155 return (top); 1156 } 1157 1158 /* 1159 * Copy data from a buffer back into the indicated mbuf chain, 1160 * starting "off" bytes from the beginning, extending the mbuf 1161 * chain if necessary. 1162 */ 1163 void 1164 m_copyback(struct mbuf *m0, int off, int len, const void *cp) 1165 { 1166 #if defined(DEBUG) 1167 struct mbuf *origm = m0; 1168 int error; 1169 #endif /* defined(DEBUG) */ 1170 1171 if (m0 == NULL) 1172 return; 1173 1174 #if defined(DEBUG) 1175 error = 1176 #endif /* defined(DEBUG) */ 1177 m_copyback0(&m0, off, len, cp, 1178 M_COPYBACK0_COPYBACK|M_COPYBACK0_EXTEND, M_DONTWAIT); 1179 1180 #if defined(DEBUG) 1181 if (error != 0 || (m0 != NULL && origm != m0)) 1182 panic("m_copyback"); 1183 #endif /* defined(DEBUG) */ 1184 } 1185 1186 struct mbuf * 1187 m_copyback_cow(struct mbuf *m0, int off, int len, const void *cp, int how) 1188 { 1189 int error; 1190 1191 /* don't support chain expansion */ 1192 KDASSERT(off + len <= m_length(m0)); 1193 1194 error = m_copyback0(&m0, off, len, cp, 1195 M_COPYBACK0_COPYBACK|M_COPYBACK0_COW, how); 1196 if (error) { 1197 /* 1198 * no way to recover from partial success. 1199 * just free the chain. 1200 */ 1201 m_freem(m0); 1202 return NULL; 1203 } 1204 return m0; 1205 } 1206 1207 /* 1208 * m_makewritable: ensure the specified range writable. 1209 */ 1210 int 1211 m_makewritable(struct mbuf **mp, int off, int len, int how) 1212 { 1213 int error; 1214 #if defined(DEBUG) 1215 struct mbuf *n; 1216 int origlen, reslen; 1217 1218 origlen = m_length(*mp); 1219 #endif /* defined(DEBUG) */ 1220 1221 #if 0 /* M_COPYALL is large enough */ 1222 if (len == M_COPYALL) 1223 len = m_length(*mp) - off; /* XXX */ 1224 #endif 1225 1226 error = m_copyback0(mp, off, len, NULL, 1227 M_COPYBACK0_PRESERVE|M_COPYBACK0_COW, how); 1228 1229 #if defined(DEBUG) 1230 reslen = 0; 1231 for (n = *mp; n; n = n->m_next) 1232 reslen += n->m_len; 1233 if (origlen != reslen) 1234 panic("m_makewritable: length changed"); 1235 if (((*mp)->m_flags & M_PKTHDR) != 0 && reslen != (*mp)->m_pkthdr.len) 1236 panic("m_makewritable: inconsist"); 1237 #endif /* defined(DEBUG) */ 1238 1239 return error; 1240 } 1241 1242 int 1243 m_copyback0(struct mbuf **mp0, int off, int len, const void *vp, int flags, 1244 int how) 1245 { 1246 int mlen; 1247 struct mbuf *m, *n; 1248 struct mbuf **mp; 1249 int totlen = 0; 1250 const char *cp = vp; 1251 1252 KASSERT(mp0 != NULL); 1253 KASSERT(*mp0 != NULL); 1254 KASSERT((flags & M_COPYBACK0_PRESERVE) == 0 || cp == NULL); 1255 KASSERT((flags & M_COPYBACK0_COPYBACK) == 0 || cp != NULL); 1256 1257 /* 1258 * we don't bother to update "totlen" in the case of M_COPYBACK0_COW, 1259 * assuming that M_COPYBACK0_EXTEND and M_COPYBACK0_COW are exclusive. 1260 */ 1261 1262 KASSERT((~flags & (M_COPYBACK0_EXTEND|M_COPYBACK0_COW)) != 0); 1263 1264 mp = mp0; 1265 m = *mp; 1266 while (off > (mlen = m->m_len)) { 1267 off -= mlen; 1268 totlen += mlen; 1269 if (m->m_next == NULL) { 1270 int tspace; 1271 extend: 1272 if ((flags & M_COPYBACK0_EXTEND) == 0) 1273 goto out; 1274 1275 /* 1276 * try to make some space at the end of "m". 1277 */ 1278 1279 mlen = m->m_len; 1280 if (off + len >= MINCLSIZE && 1281 (m->m_flags & M_EXT) == 0 && m->m_len == 0) { 1282 MCLGET(m, how); 1283 } 1284 tspace = M_TRAILINGSPACE(m); 1285 if (tspace > 0) { 1286 tspace = min(tspace, off + len); 1287 KASSERT(tspace > 0); 1288 memset(mtod(m, char *) + m->m_len, 0, 1289 min(off, tspace)); 1290 m->m_len += tspace; 1291 off += mlen; 1292 totlen -= mlen; 1293 continue; 1294 } 1295 1296 /* 1297 * need to allocate an mbuf. 1298 */ 1299 1300 if (off + len >= MINCLSIZE) { 1301 n = m_getcl(how, m->m_type, 0); 1302 } else { 1303 n = m_get(how, m->m_type); 1304 } 1305 if (n == NULL) { 1306 goto out; 1307 } 1308 n->m_len = 0; 1309 n->m_len = min(M_TRAILINGSPACE(n), off + len); 1310 memset(mtod(n, char *), 0, min(n->m_len, off)); 1311 m->m_next = n; 1312 } 1313 mp = &m->m_next; 1314 m = m->m_next; 1315 } 1316 while (len > 0) { 1317 mlen = m->m_len - off; 1318 if (mlen != 0 && M_READONLY(m)) { 1319 char *datap; 1320 int eatlen; 1321 1322 /* 1323 * this mbuf is read-only. 1324 * allocate a new writable mbuf and try again. 1325 */ 1326 1327 #if defined(DIAGNOSTIC) 1328 if ((flags & M_COPYBACK0_COW) == 0) 1329 panic("m_copyback0: read-only"); 1330 #endif /* defined(DIAGNOSTIC) */ 1331 1332 /* 1333 * if we're going to write into the middle of 1334 * a mbuf, split it first. 1335 */ 1336 if (off > 0 && len < mlen) { 1337 n = m_split0(m, off, how, 0); 1338 if (n == NULL) 1339 goto enobufs; 1340 m->m_next = n; 1341 mp = &m->m_next; 1342 m = n; 1343 off = 0; 1344 continue; 1345 } 1346 1347 /* 1348 * XXX TODO coalesce into the trailingspace of 1349 * the previous mbuf when possible. 1350 */ 1351 1352 /* 1353 * allocate a new mbuf. copy packet header if needed. 1354 */ 1355 MGET(n, how, m->m_type); 1356 if (n == NULL) 1357 goto enobufs; 1358 MCLAIM(n, m->m_owner); 1359 if (off == 0 && (m->m_flags & M_PKTHDR) != 0) { 1360 M_MOVE_PKTHDR(n, m); 1361 n->m_len = MHLEN; 1362 } else { 1363 if (len >= MINCLSIZE) 1364 MCLGET(n, M_DONTWAIT); 1365 n->m_len = 1366 (n->m_flags & M_EXT) ? MCLBYTES : MLEN; 1367 } 1368 if (n->m_len > len) 1369 n->m_len = len; 1370 1371 /* 1372 * free the region which has been overwritten. 1373 * copying data from old mbufs if requested. 1374 */ 1375 if (flags & M_COPYBACK0_PRESERVE) 1376 datap = mtod(n, char *); 1377 else 1378 datap = NULL; 1379 eatlen = n->m_len; 1380 KDASSERT(off == 0 || eatlen >= mlen); 1381 if (off > 0) { 1382 KDASSERT(len >= mlen); 1383 m->m_len = off; 1384 m->m_next = n; 1385 if (datap) { 1386 m_copydata(m, off, mlen, datap); 1387 datap += mlen; 1388 } 1389 eatlen -= mlen; 1390 mp = &m->m_next; 1391 m = m->m_next; 1392 } 1393 while (m != NULL && M_READONLY(m) && 1394 n->m_type == m->m_type && eatlen > 0) { 1395 mlen = min(eatlen, m->m_len); 1396 if (datap) { 1397 m_copydata(m, 0, mlen, datap); 1398 datap += mlen; 1399 } 1400 m->m_data += mlen; 1401 m->m_len -= mlen; 1402 eatlen -= mlen; 1403 if (m->m_len == 0) 1404 *mp = m = m_free(m); 1405 } 1406 if (eatlen > 0) 1407 n->m_len -= eatlen; 1408 n->m_next = m; 1409 *mp = m = n; 1410 continue; 1411 } 1412 mlen = min(mlen, len); 1413 if (flags & M_COPYBACK0_COPYBACK) { 1414 memcpy(mtod(m, char *) + off, cp, (unsigned)mlen); 1415 cp += mlen; 1416 } 1417 len -= mlen; 1418 mlen += off; 1419 off = 0; 1420 totlen += mlen; 1421 if (len == 0) 1422 break; 1423 if (m->m_next == NULL) { 1424 goto extend; 1425 } 1426 mp = &m->m_next; 1427 m = m->m_next; 1428 } 1429 out: if (((m = *mp0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) { 1430 KASSERT((flags & M_COPYBACK0_EXTEND) != 0); 1431 m->m_pkthdr.len = totlen; 1432 } 1433 1434 return 0; 1435 1436 enobufs: 1437 return ENOBUFS; 1438 } 1439 1440 void 1441 m_move_pkthdr(struct mbuf *to, struct mbuf *from) 1442 { 1443 1444 KASSERT((to->m_flags & M_EXT) == 0); 1445 KASSERT((to->m_flags & M_PKTHDR) == 0 || m_tag_first(to) == NULL); 1446 KASSERT((from->m_flags & M_PKTHDR) != 0); 1447 1448 to->m_pkthdr = from->m_pkthdr; 1449 to->m_flags = from->m_flags & M_COPYFLAGS; 1450 to->m_data = to->m_pktdat; 1451 1452 from->m_flags &= ~M_PKTHDR; 1453 } 1454 1455 /* 1456 * Apply function f to the data in an mbuf chain starting "off" bytes from the 1457 * beginning, continuing for "len" bytes. 1458 */ 1459 int 1460 m_apply(struct mbuf *m, int off, int len, 1461 int (*f)(void *, void *, unsigned int), void *arg) 1462 { 1463 unsigned int count; 1464 int rval; 1465 1466 KASSERT(len >= 0); 1467 KASSERT(off >= 0); 1468 1469 while (off > 0) { 1470 KASSERT(m != NULL); 1471 if (off < m->m_len) 1472 break; 1473 off -= m->m_len; 1474 m = m->m_next; 1475 } 1476 while (len > 0) { 1477 KASSERT(m != NULL); 1478 count = min(m->m_len - off, len); 1479 1480 rval = (*f)(arg, mtod(m, char *) + off, count); 1481 if (rval) 1482 return (rval); 1483 1484 len -= count; 1485 off = 0; 1486 m = m->m_next; 1487 } 1488 1489 return (0); 1490 } 1491 1492 /* 1493 * Return a pointer to mbuf/offset of location in mbuf chain. 1494 */ 1495 struct mbuf * 1496 m_getptr(struct mbuf *m, int loc, int *off) 1497 { 1498 1499 while (loc >= 0) { 1500 /* Normal end of search */ 1501 if (m->m_len > loc) { 1502 *off = loc; 1503 return (m); 1504 } else { 1505 loc -= m->m_len; 1506 1507 if (m->m_next == NULL) { 1508 if (loc == 0) { 1509 /* Point at the end of valid data */ 1510 *off = m->m_len; 1511 return (m); 1512 } else 1513 return (NULL); 1514 } else 1515 m = m->m_next; 1516 } 1517 } 1518 1519 return (NULL); 1520 } 1521 1522 /* 1523 * m_ext_free: release a reference to the mbuf external storage. 1524 * 1525 * => free the mbuf m itsself as well. 1526 */ 1527 1528 void 1529 m_ext_free(struct mbuf *m) 1530 { 1531 bool embedded = MEXT_ISEMBEDDED(m); 1532 bool dofree = true; 1533 u_int refcnt; 1534 1535 KASSERT((m->m_flags & M_EXT) != 0); 1536 KASSERT(MEXT_ISEMBEDDED(m->m_ext_ref)); 1537 KASSERT((m->m_ext_ref->m_flags & M_EXT) != 0); 1538 KASSERT((m->m_flags & M_EXT_CLUSTER) == 1539 (m->m_ext_ref->m_flags & M_EXT_CLUSTER)); 1540 1541 if (__predict_true(m->m_ext.ext_refcnt == 1)) { 1542 refcnt = m->m_ext.ext_refcnt = 0; 1543 } else { 1544 refcnt = atomic_dec_uint_nv(&m->m_ext.ext_refcnt); 1545 } 1546 if (refcnt > 0) { 1547 if (embedded) { 1548 /* 1549 * other mbuf's m_ext_ref still points to us. 1550 */ 1551 dofree = false; 1552 } else { 1553 m->m_ext_ref = m; 1554 } 1555 } else { 1556 /* 1557 * dropping the last reference 1558 */ 1559 if (!embedded) { 1560 m->m_ext.ext_refcnt++; /* XXX */ 1561 m_ext_free(m->m_ext_ref); 1562 m->m_ext_ref = m; 1563 } else if ((m->m_flags & M_EXT_CLUSTER) != 0) { 1564 pool_cache_put_paddr((struct pool_cache *) 1565 m->m_ext.ext_arg, 1566 m->m_ext.ext_buf, m->m_ext.ext_paddr); 1567 } else if (m->m_ext.ext_free) { 1568 (*m->m_ext.ext_free)(m, 1569 m->m_ext.ext_buf, m->m_ext.ext_size, 1570 m->m_ext.ext_arg); 1571 /* 1572 * 'm' is already freed by the ext_free callback. 1573 */ 1574 dofree = false; 1575 } else { 1576 free(m->m_ext.ext_buf, m->m_ext.ext_type); 1577 } 1578 } 1579 if (dofree) { 1580 pool_cache_put(mb_cache, m); 1581 } 1582 } 1583 1584 #if defined(DDB) 1585 void 1586 m_print(const struct mbuf *m, const char *modif, void (*pr)(const char *, ...)) 1587 { 1588 char ch; 1589 bool opt_c = false; 1590 char buf[512]; 1591 1592 while ((ch = *(modif++)) != '\0') { 1593 switch (ch) { 1594 case 'c': 1595 opt_c = true; 1596 break; 1597 } 1598 } 1599 1600 nextchain: 1601 (*pr)("MBUF %p\n", m); 1602 bitmask_snprintf((u_int)m->m_flags, M_FLAGS_BITS, buf, sizeof(buf)); 1603 (*pr)(" data=%p, len=%d, type=%d, flags=0x%s\n", 1604 m->m_data, m->m_len, m->m_type, buf); 1605 (*pr)(" owner=%p, next=%p, nextpkt=%p\n", m->m_owner, m->m_next, 1606 m->m_nextpkt); 1607 (*pr)(" leadingspace=%u, trailingspace=%u, readonly=%u\n", 1608 (int)M_LEADINGSPACE(m), (int)M_TRAILINGSPACE(m), 1609 (int)M_READONLY(m)); 1610 if ((m->m_flags & M_PKTHDR) != 0) { 1611 bitmask_snprintf(m->m_pkthdr.csum_flags, M_CSUM_BITS, buf, 1612 sizeof(buf)); 1613 (*pr)(" pktlen=%d, rcvif=%p, csum_flags=0x%s, csum_data=0x%" 1614 PRIx32 ", segsz=%u\n", 1615 m->m_pkthdr.len, m->m_pkthdr.rcvif, 1616 buf, m->m_pkthdr.csum_data, m->m_pkthdr.segsz); 1617 } 1618 if ((m->m_flags & M_EXT)) { 1619 (*pr)(" ext_refcnt=%u, ext_buf=%p, ext_size=%zd, " 1620 "ext_free=%p, ext_arg=%p\n", 1621 m->m_ext.ext_refcnt, 1622 m->m_ext.ext_buf, m->m_ext.ext_size, 1623 m->m_ext.ext_free, m->m_ext.ext_arg); 1624 } 1625 if ((~m->m_flags & (M_EXT|M_EXT_PAGES)) == 0) { 1626 vaddr_t sva = (vaddr_t)m->m_ext.ext_buf; 1627 vaddr_t eva = sva + m->m_ext.ext_size; 1628 int n = (round_page(eva) - trunc_page(sva)) >> PAGE_SHIFT; 1629 int i; 1630 1631 (*pr)(" pages:"); 1632 for (i = 0; i < n; i ++) { 1633 (*pr)(" %p", m->m_ext.ext_pgs[i]); 1634 } 1635 (*pr)("\n"); 1636 } 1637 1638 if (opt_c) { 1639 m = m->m_next; 1640 if (m != NULL) { 1641 goto nextchain; 1642 } 1643 } 1644 } 1645 #endif /* defined(DDB) */ 1646 1647 void 1648 mbstat_type_add(int type, int diff) 1649 { 1650 struct mbstat_cpu *mb; 1651 int s; 1652 1653 s = splvm(); 1654 mb = percpu_getref(mbstat_percpu); 1655 mb->m_mtypes[type] += diff; 1656 percpu_putref(mbstat_percpu); 1657 splx(s); 1658 } 1659 1660 #if defined(MBUFTRACE) 1661 void 1662 mowner_attach(struct mowner *mo) 1663 { 1664 1665 KASSERT(mo->mo_counters == NULL); 1666 mo->mo_counters = percpu_alloc(sizeof(struct mowner_counter)); 1667 1668 /* XXX lock */ 1669 LIST_INSERT_HEAD(&mowners, mo, mo_link); 1670 } 1671 1672 void 1673 mowner_detach(struct mowner *mo) 1674 { 1675 1676 KASSERT(mo->mo_counters != NULL); 1677 1678 /* XXX lock */ 1679 LIST_REMOVE(mo, mo_link); 1680 1681 percpu_free(mo->mo_counters, sizeof(struct mowner_counter)); 1682 mo->mo_counters = NULL; 1683 } 1684 1685 void 1686 mowner_init(struct mbuf *m, int type) 1687 { 1688 struct mowner_counter *mc; 1689 struct mowner *mo; 1690 int s; 1691 1692 m->m_owner = mo = &unknown_mowners[type]; 1693 s = splvm(); 1694 mc = percpu_getref(mo->mo_counters); 1695 mc->mc_counter[MOWNER_COUNTER_CLAIMS]++; 1696 percpu_putref(mo->mo_counters); 1697 splx(s); 1698 } 1699 1700 void 1701 mowner_ref(struct mbuf *m, int flags) 1702 { 1703 struct mowner *mo = m->m_owner; 1704 struct mowner_counter *mc; 1705 int s; 1706 1707 s = splvm(); 1708 mc = percpu_getref(mo->mo_counters); 1709 if ((flags & M_EXT) != 0) 1710 mc->mc_counter[MOWNER_COUNTER_EXT_CLAIMS]++; 1711 if ((flags & M_CLUSTER) != 0) 1712 mc->mc_counter[MOWNER_COUNTER_CLUSTER_CLAIMS]++; 1713 percpu_putref(mo->mo_counters); 1714 splx(s); 1715 } 1716 1717 void 1718 mowner_revoke(struct mbuf *m, bool all, int flags) 1719 { 1720 struct mowner *mo = m->m_owner; 1721 struct mowner_counter *mc; 1722 int s; 1723 1724 s = splvm(); 1725 mc = percpu_getref(mo->mo_counters); 1726 if ((flags & M_EXT) != 0) 1727 mc->mc_counter[MOWNER_COUNTER_EXT_RELEASES]++; 1728 if ((flags & M_CLUSTER) != 0) 1729 mc->mc_counter[MOWNER_COUNTER_CLUSTER_RELEASES]++; 1730 if (all) 1731 mc->mc_counter[MOWNER_COUNTER_RELEASES]++; 1732 percpu_putref(mo->mo_counters); 1733 splx(s); 1734 if (all) 1735 m->m_owner = &revoked_mowner; 1736 } 1737 1738 static void 1739 mowner_claim(struct mbuf *m, struct mowner *mo) 1740 { 1741 struct mowner_counter *mc; 1742 int flags = m->m_flags; 1743 int s; 1744 1745 s = splvm(); 1746 mc = percpu_getref(mo->mo_counters); 1747 mc->mc_counter[MOWNER_COUNTER_CLAIMS]++; 1748 if ((flags & M_EXT) != 0) 1749 mc->mc_counter[MOWNER_COUNTER_EXT_CLAIMS]++; 1750 if ((flags & M_CLUSTER) != 0) 1751 mc->mc_counter[MOWNER_COUNTER_CLUSTER_CLAIMS]++; 1752 percpu_putref(mo->mo_counters); 1753 splx(s); 1754 m->m_owner = mo; 1755 } 1756 1757 void 1758 m_claim(struct mbuf *m, struct mowner *mo) 1759 { 1760 1761 if (m->m_owner == mo || mo == NULL) 1762 return; 1763 1764 mowner_revoke(m, true, m->m_flags); 1765 mowner_claim(m, mo); 1766 } 1767 #endif /* defined(MBUFTRACE) */
Cache object: 5ee8af2991906e051f240f6c9d3f1014
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]