Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/kern/uipc_mbuf.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* $NetBSD: uipc_mbuf.c,v 1.92.6.3 2006/09/08 12:06:51 ghen Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the NetBSD 22 * Foundation, Inc. and its contributors. 23 * 4. Neither the name of The NetBSD Foundation nor the names of its 24 * contributors may be used to endorse or promote products derived 25 * from this software without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 29 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 30 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 31 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 37 * POSSIBILITY OF SUCH DAMAGE. 38 */ 39 40 /* 41 * Copyright (c) 1982, 1986, 1988, 1991, 1993 42 * The Regents of the University of California. All rights reserved. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. Neither the name of the University nor the names of its contributors 53 * may be used to endorse or promote products derived from this software 54 * without specific prior written permission. 55 * 56 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 57 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 58 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 59 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 60 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 61 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 62 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 63 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 64 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 65 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 66 * SUCH DAMAGE. 67 * 68 * @(#)uipc_mbuf.c 8.4 (Berkeley) 2/14/95 69 */ 70 71 #include <sys/cdefs.h> 72 __KERNEL_RCSID(0, "$NetBSD: uipc_mbuf.c,v 1.92.6.3 2006/09/08 12:06:51 ghen Exp $"); 73 74 #include "opt_mbuftrace.h" 75 76 #include <sys/param.h> 77 #include <sys/systm.h> 78 #include <sys/proc.h> 79 #include <sys/malloc.h> 80 #define MBTYPES 81 #include <sys/mbuf.h> 82 #include <sys/kernel.h> 83 #include <sys/syslog.h> 84 #include <sys/domain.h> 85 #include <sys/protosw.h> 86 #include <sys/pool.h> 87 #include <sys/socket.h> 88 #include <sys/sysctl.h> 89 90 #include <net/if.h> 91 92 #include <uvm/uvm.h> 93 94 95 struct pool mbpool; /* mbuf pool */ 96 struct pool mclpool; /* mbuf cluster pool */ 97 98 struct pool_cache mbpool_cache; 99 struct pool_cache mclpool_cache; 100 101 struct mbstat mbstat; 102 int max_linkhdr; 103 int max_protohdr; 104 int max_hdr; 105 int max_datalen; 106 107 static int mb_ctor(void *, void *, int); 108 109 void *mclpool_alloc(struct pool *, int); 110 void mclpool_release(struct pool *, void *); 111 112 struct pool_allocator mclpool_allocator = { 113 mclpool_alloc, mclpool_release, 0, 114 }; 115 116 static struct mbuf *m_copym0(struct mbuf *, int, int, int, int); 117 static struct mbuf *m_split0(struct mbuf *, int, int, int); 118 static int m_copyback0(struct mbuf **, int, int, const void *, int, int); 119 120 /* flags for m_copyback0 */ 121 #define M_COPYBACK0_COPYBACK 0x0001 /* copyback from cp */ 122 #define M_COPYBACK0_PRESERVE 0x0002 /* preserve original data */ 123 #define M_COPYBACK0_COW 0x0004 /* do copy-on-write */ 124 #define M_COPYBACK0_EXTEND 0x0008 /* extend chain */ 125 126 const char mclpool_warnmsg[] = 127 "WARNING: mclpool limit reached; increase NMBCLUSTERS"; 128 129 MALLOC_DEFINE(M_MBUF, "mbuf", "mbuf"); 130 131 #ifdef MBUFTRACE 132 struct mownerhead mowners = LIST_HEAD_INITIALIZER(mowners); 133 struct mowner unknown_mowners[] = { 134 { "unknown", "free" }, 135 { "unknown", "data" }, 136 { "unknown", "header" }, 137 { "unknown", "soname" }, 138 { "unknown", "soopts" }, 139 { "unknown", "ftable" }, 140 { "unknown", "control" }, 141 { "unknown", "oobdata" }, 142 }; 143 struct mowner revoked_mowner = { "revoked", "" }; 144 #endif 145 146 /* 147 * Initialize the mbuf allocator. 148 */ 149 void 150 mbinit(void) 151 { 152 153 KASSERT(sizeof(struct _m_ext) <= MHLEN); 154 KASSERT(sizeof(struct mbuf) == MSIZE); 155 156 pool_init(&mbpool, msize, 0, 0, 0, "mbpl", NULL); 157 pool_init(&mclpool, mclbytes, 0, 0, 0, "mclpl", &mclpool_allocator); 158 159 pool_set_drain_hook(&mbpool, m_reclaim, NULL); 160 pool_set_drain_hook(&mclpool, m_reclaim, NULL); 161 162 pool_cache_init(&mbpool_cache, &mbpool, mb_ctor, NULL, NULL); 163 pool_cache_init(&mclpool_cache, &mclpool, NULL, NULL, NULL); 164 165 /* 166 * Set the hard limit on the mclpool to the number of 167 * mbuf clusters the kernel is to support. Log the limit 168 * reached message max once a minute. 169 */ 170 pool_sethardlimit(&mclpool, nmbclusters, mclpool_warnmsg, 60); 171 172 /* 173 * Set a low water mark for both mbufs and clusters. This should 174 * help ensure that they can be allocated in a memory starvation 175 * situation. This is important for e.g. diskless systems which 176 * must allocate mbufs in order for the pagedaemon to clean pages. 177 */ 178 pool_setlowat(&mbpool, mblowat); 179 pool_setlowat(&mclpool, mcllowat); 180 181 #ifdef MBUFTRACE 182 { 183 /* 184 * Attach the unknown mowners. 185 */ 186 int i; 187 MOWNER_ATTACH(&revoked_mowner); 188 for (i = sizeof(unknown_mowners)/sizeof(unknown_mowners[0]); 189 i-- > 0; ) 190 MOWNER_ATTACH(&unknown_mowners[i]); 191 } 192 #endif 193 } 194 195 /* 196 * sysctl helper routine for the kern.mbuf subtree. nmbclusters may 197 * or may not be writable, and mblowat and mcllowat need range 198 * checking and pool tweaking after being reset. 199 */ 200 static int 201 sysctl_kern_mbuf(SYSCTLFN_ARGS) 202 { 203 int error, newval; 204 struct sysctlnode node; 205 206 node = *rnode; 207 node.sysctl_data = &newval; 208 switch (rnode->sysctl_num) { 209 case MBUF_NMBCLUSTERS: 210 if (mb_map != NULL) { 211 node.sysctl_flags &= ~CTLFLAG_READWRITE; 212 node.sysctl_flags |= CTLFLAG_READONLY; 213 } 214 /* FALLTHROUGH */ 215 case MBUF_MBLOWAT: 216 case MBUF_MCLLOWAT: 217 newval = *(int*)rnode->sysctl_data; 218 break; 219 default: 220 return (EOPNOTSUPP); 221 } 222 223 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 224 if (error || newp == NULL) 225 return (error); 226 if (newval < 0) 227 return (EINVAL); 228 229 switch (node.sysctl_num) { 230 case MBUF_NMBCLUSTERS: 231 if (newval < nmbclusters) 232 return (EINVAL); 233 nmbclusters = newval; 234 pool_sethardlimit(&mclpool, nmbclusters, mclpool_warnmsg, 60); 235 break; 236 case MBUF_MBLOWAT: 237 mblowat = newval; 238 pool_setlowat(&mbpool, mblowat); 239 break; 240 case MBUF_MCLLOWAT: 241 mcllowat = newval; 242 pool_setlowat(&mclpool, mcllowat); 243 break; 244 } 245 246 return (0); 247 } 248 249 #ifdef MBUFTRACE 250 static int 251 sysctl_kern_mbuf_mowners(SYSCTLFN_ARGS) 252 { 253 struct mowner *mo; 254 size_t len = 0; 255 int error = 0; 256 257 if (namelen != 0) 258 return (EINVAL); 259 if (newp != NULL) 260 return (EPERM); 261 262 LIST_FOREACH(mo, &mowners, mo_link) { 263 if (oldp != NULL) { 264 if (*oldlenp - len < sizeof(*mo)) { 265 error = ENOMEM; 266 break; 267 } 268 error = copyout(mo, (caddr_t) oldp + len, 269 sizeof(*mo)); 270 if (error) 271 break; 272 } 273 len += sizeof(*mo); 274 } 275 276 if (error == 0) 277 *oldlenp = len; 278 279 return (error); 280 } 281 #endif /* MBUFTRACE */ 282 283 SYSCTL_SETUP(sysctl_kern_mbuf_setup, "sysctl kern.mbuf subtree setup") 284 { 285 286 sysctl_createv(clog, 0, NULL, NULL, 287 CTLFLAG_PERMANENT, 288 CTLTYPE_NODE, "kern", NULL, 289 NULL, 0, NULL, 0, 290 CTL_KERN, CTL_EOL); 291 sysctl_createv(clog, 0, NULL, NULL, 292 CTLFLAG_PERMANENT, 293 CTLTYPE_NODE, "mbuf", 294 SYSCTL_DESCR("mbuf control variables"), 295 NULL, 0, NULL, 0, 296 CTL_KERN, KERN_MBUF, CTL_EOL); 297 298 sysctl_createv(clog, 0, NULL, NULL, 299 CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE, 300 CTLTYPE_INT, "msize", 301 SYSCTL_DESCR("mbuf base size"), 302 NULL, msize, NULL, 0, 303 CTL_KERN, KERN_MBUF, MBUF_MSIZE, CTL_EOL); 304 sysctl_createv(clog, 0, NULL, NULL, 305 CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE, 306 CTLTYPE_INT, "mclbytes", 307 SYSCTL_DESCR("mbuf cluster size"), 308 NULL, mclbytes, NULL, 0, 309 CTL_KERN, KERN_MBUF, MBUF_MCLBYTES, CTL_EOL); 310 sysctl_createv(clog, 0, NULL, NULL, 311 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 312 CTLTYPE_INT, "nmbclusters", 313 SYSCTL_DESCR("Limit on the number of mbuf clusters"), 314 sysctl_kern_mbuf, 0, &nmbclusters, 0, 315 CTL_KERN, KERN_MBUF, MBUF_NMBCLUSTERS, CTL_EOL); 316 sysctl_createv(clog, 0, NULL, NULL, 317 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 318 CTLTYPE_INT, "mblowat", 319 SYSCTL_DESCR("mbuf low water mark"), 320 sysctl_kern_mbuf, 0, &mblowat, 0, 321 CTL_KERN, KERN_MBUF, MBUF_MBLOWAT, CTL_EOL); 322 sysctl_createv(clog, 0, NULL, NULL, 323 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 324 CTLTYPE_INT, "mcllowat", 325 SYSCTL_DESCR("mbuf cluster low water mark"), 326 sysctl_kern_mbuf, 0, &mcllowat, 0, 327 CTL_KERN, KERN_MBUF, MBUF_MCLLOWAT, CTL_EOL); 328 sysctl_createv(clog, 0, NULL, NULL, 329 CTLFLAG_PERMANENT, 330 CTLTYPE_STRUCT, "stats", 331 SYSCTL_DESCR("mbuf allocation statistics"), 332 NULL, 0, &mbstat, sizeof(mbstat), 333 CTL_KERN, KERN_MBUF, MBUF_STATS, CTL_EOL); 334 #ifdef MBUFTRACE 335 sysctl_createv(clog, 0, NULL, NULL, 336 CTLFLAG_PERMANENT, 337 CTLTYPE_STRUCT, "mowners", 338 SYSCTL_DESCR("Information about mbuf owners"), 339 sysctl_kern_mbuf_mowners, 0, NULL, 0, 340 CTL_KERN, KERN_MBUF, MBUF_MOWNERS, CTL_EOL); 341 #endif /* MBUFTRACE */ 342 } 343 344 void * 345 mclpool_alloc(struct pool *pp, int flags) 346 { 347 boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; 348 349 return ((void *)uvm_km_alloc_poolpage1(mb_map, NULL, waitok)); 350 } 351 352 void 353 mclpool_release(struct pool *pp, void *v) 354 { 355 356 uvm_km_free_poolpage1(mb_map, (vaddr_t)v); 357 } 358 359 /*ARGSUSED*/ 360 static int 361 mb_ctor(void *arg, void *object, int flags) 362 { 363 struct mbuf *m = object; 364 365 #ifdef POOL_VTOPHYS 366 m->m_paddr = POOL_VTOPHYS(m); 367 #else 368 m->m_paddr = M_PADDR_INVALID; 369 #endif 370 return (0); 371 } 372 373 void 374 m_reclaim(void *arg, int flags) 375 { 376 struct domain *dp; 377 const struct protosw *pr; 378 struct ifnet *ifp; 379 int s = splvm(); 380 381 DOMAIN_FOREACH(dp) { 382 for (pr = dp->dom_protosw; 383 pr < dp->dom_protoswNPROTOSW; pr++) 384 if (pr->pr_drain) 385 (*pr->pr_drain)(); 386 } 387 IFNET_FOREACH(ifp) { 388 if (ifp->if_drain) 389 (*ifp->if_drain)(ifp); 390 } 391 splx(s); 392 mbstat.m_drain++; 393 } 394 395 /* 396 * Space allocation routines. 397 * These are also available as macros 398 * for critical paths. 399 */ 400 struct mbuf * 401 m_get(int nowait, int type) 402 { 403 struct mbuf *m; 404 405 MGET(m, nowait, type); 406 return (m); 407 } 408 409 struct mbuf * 410 m_gethdr(int nowait, int type) 411 { 412 struct mbuf *m; 413 414 MGETHDR(m, nowait, type); 415 return (m); 416 } 417 418 struct mbuf * 419 m_getclr(int nowait, int type) 420 { 421 struct mbuf *m; 422 423 MGET(m, nowait, type); 424 if (m == 0) 425 return (NULL); 426 memset(mtod(m, caddr_t), 0, MLEN); 427 return (m); 428 } 429 430 void 431 m_clget(struct mbuf *m, int nowait) 432 { 433 434 MCLGET(m, nowait); 435 } 436 437 struct mbuf * 438 m_free(struct mbuf *m) 439 { 440 struct mbuf *n; 441 442 MFREE(m, n); 443 return (n); 444 } 445 446 void 447 m_freem(struct mbuf *m) 448 { 449 struct mbuf *n; 450 451 if (m == NULL) 452 return; 453 do { 454 MFREE(m, n); 455 m = n; 456 } while (m); 457 } 458 459 #ifdef MBUFTRACE 460 /* 461 * Walk a chain of mbufs, claiming ownership of each mbuf in the chain. 462 */ 463 void 464 m_claimm(struct mbuf *m, struct mowner *mo) 465 { 466 467 for (; m != NULL; m = m->m_next) 468 MCLAIM(m, mo); 469 } 470 #endif 471 472 /* 473 * Mbuffer utility routines. 474 */ 475 476 /* 477 * Lesser-used path for M_PREPEND: 478 * allocate new mbuf to prepend to chain, 479 * copy junk along. 480 */ 481 struct mbuf * 482 m_prepend(struct mbuf *m, int len, int how) 483 { 484 struct mbuf *mn; 485 486 MGET(mn, how, m->m_type); 487 if (mn == (struct mbuf *)NULL) { 488 m_freem(m); 489 return ((struct mbuf *)NULL); 490 } 491 if (m->m_flags & M_PKTHDR) { 492 M_COPY_PKTHDR(mn, m); 493 m_tag_delete_chain(m, NULL); 494 m->m_flags &= ~M_PKTHDR; 495 } else { 496 MCLAIM(mn, m->m_owner); 497 } 498 mn->m_next = m; 499 m = mn; 500 if (len < MHLEN) 501 MH_ALIGN(m, len); 502 m->m_len = len; 503 return (m); 504 } 505 506 /* 507 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 508 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 509 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 510 */ 511 int MCFail; 512 513 struct mbuf * 514 m_copym(struct mbuf *m, int off0, int len, int wait) 515 { 516 517 return m_copym0(m, off0, len, wait, 0); /* shallow copy on M_EXT */ 518 } 519 520 struct mbuf * 521 m_dup(struct mbuf *m, int off0, int len, int wait) 522 { 523 524 return m_copym0(m, off0, len, wait, 1); /* deep copy */ 525 } 526 527 static struct mbuf * 528 m_copym0(struct mbuf *m, int off0, int len, int wait, int deep) 529 { 530 struct mbuf *n, **np; 531 int off = off0; 532 struct mbuf *top; 533 int copyhdr = 0; 534 535 if (off < 0 || len < 0) 536 panic("m_copym: off %d, len %d", off, len); 537 if (off == 0 && m->m_flags & M_PKTHDR) 538 copyhdr = 1; 539 while (off > 0) { 540 if (m == 0) 541 panic("m_copym: m == 0, off %d", off); 542 if (off < m->m_len) 543 break; 544 off -= m->m_len; 545 m = m->m_next; 546 } 547 np = ⊤ 548 top = 0; 549 while (len > 0) { 550 if (m == 0) { 551 if (len != M_COPYALL) 552 panic("m_copym: m == 0, len %d [!COPYALL]", 553 len); 554 break; 555 } 556 MGET(n, wait, m->m_type); 557 *np = n; 558 if (n == 0) 559 goto nospace; 560 MCLAIM(n, m->m_owner); 561 if (copyhdr) { 562 M_COPY_PKTHDR(n, m); 563 if (len == M_COPYALL) 564 n->m_pkthdr.len -= off0; 565 else 566 n->m_pkthdr.len = len; 567 copyhdr = 0; 568 } 569 n->m_len = min(len, m->m_len - off); 570 if (m->m_flags & M_EXT) { 571 if (!deep) { 572 n->m_data = m->m_data + off; 573 n->m_ext = m->m_ext; 574 MCLADDREFERENCE(m, n); 575 } else { 576 /* 577 * we are unsure about the way m was allocated. 578 * copy into multiple MCLBYTES cluster mbufs. 579 */ 580 MCLGET(n, wait); 581 n->m_len = 0; 582 n->m_len = M_TRAILINGSPACE(n); 583 n->m_len = min(n->m_len, len); 584 n->m_len = min(n->m_len, m->m_len - off); 585 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, 586 (unsigned)n->m_len); 587 } 588 } else 589 memcpy(mtod(n, caddr_t), mtod(m, caddr_t)+off, 590 (unsigned)n->m_len); 591 if (len != M_COPYALL) 592 len -= n->m_len; 593 off += n->m_len; 594 #ifdef DIAGNOSTIC 595 if (off > m->m_len) 596 panic("m_copym0 overrun"); 597 #endif 598 if (off == m->m_len) { 599 m = m->m_next; 600 off = 0; 601 } 602 np = &n->m_next; 603 } 604 if (top == 0) 605 MCFail++; 606 return (top); 607 nospace: 608 m_freem(top); 609 MCFail++; 610 return (NULL); 611 } 612 613 /* 614 * Copy an entire packet, including header (which must be present). 615 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'. 616 */ 617 struct mbuf * 618 m_copypacket(struct mbuf *m, int how) 619 { 620 struct mbuf *top, *n, *o; 621 622 MGET(n, how, m->m_type); 623 top = n; 624 if (!n) 625 goto nospace; 626 627 MCLAIM(n, m->m_owner); 628 M_COPY_PKTHDR(n, m); 629 n->m_len = m->m_len; 630 if (m->m_flags & M_EXT) { 631 n->m_data = m->m_data; 632 n->m_ext = m->m_ext; 633 MCLADDREFERENCE(m, n); 634 } else { 635 memcpy(mtod(n, char *), mtod(m, char *), n->m_len); 636 } 637 638 m = m->m_next; 639 while (m) { 640 MGET(o, how, m->m_type); 641 if (!o) 642 goto nospace; 643 644 MCLAIM(o, m->m_owner); 645 n->m_next = o; 646 n = n->m_next; 647 648 n->m_len = m->m_len; 649 if (m->m_flags & M_EXT) { 650 n->m_data = m->m_data; 651 n->m_ext = m->m_ext; 652 MCLADDREFERENCE(m, n); 653 } else { 654 memcpy(mtod(n, char *), mtod(m, char *), n->m_len); 655 } 656 657 m = m->m_next; 658 } 659 return top; 660 nospace: 661 m_freem(top); 662 MCFail++; 663 return NULL; 664 } 665 666 /* 667 * Copy data from an mbuf chain starting "off" bytes from the beginning, 668 * continuing for "len" bytes, into the indicated buffer. 669 */ 670 void 671 m_copydata(struct mbuf *m, int off, int len, void *vp) 672 { 673 unsigned count; 674 char *cp = vp; 675 676 if (off < 0 || len < 0) 677 panic("m_copydata: off %d, len %d", off, len); 678 while (off > 0) { 679 if (m == 0) 680 panic("m_copydata: m == 0, off %d", off); 681 if (off < m->m_len) 682 break; 683 off -= m->m_len; 684 m = m->m_next; 685 } 686 while (len > 0) { 687 if (m == 0) 688 panic("m_copydata: m == 0, len %d", len); 689 count = min(m->m_len - off, len); 690 memcpy(cp, mtod(m, caddr_t) + off, count); 691 len -= count; 692 cp += count; 693 off = 0; 694 m = m->m_next; 695 } 696 } 697 698 /* 699 * Concatenate mbuf chain n to m. 700 * n might be copied into m (when n->m_len is small), therefore data portion of 701 * n could be copied into an mbuf of different mbuf type. 702 * Any m_pkthdr is not updated. 703 */ 704 void 705 m_cat(struct mbuf *m, struct mbuf *n) 706 { 707 708 while (m->m_next) 709 m = m->m_next; 710 while (n) { 711 if (M_READONLY(m) || n->m_len > M_TRAILINGSPACE(m)) { 712 /* just join the two chains */ 713 m->m_next = n; 714 return; 715 } 716 /* splat the data from one into the other */ 717 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 718 (u_int)n->m_len); 719 m->m_len += n->m_len; 720 n = m_free(n); 721 } 722 } 723 724 void 725 m_adj(struct mbuf *mp, int req_len) 726 { 727 int len = req_len; 728 struct mbuf *m; 729 int count; 730 731 if ((m = mp) == NULL) 732 return; 733 if (len >= 0) { 734 /* 735 * Trim from head. 736 */ 737 while (m != NULL && len > 0) { 738 if (m->m_len <= len) { 739 len -= m->m_len; 740 m->m_len = 0; 741 m = m->m_next; 742 } else { 743 m->m_len -= len; 744 m->m_data += len; 745 len = 0; 746 } 747 } 748 m = mp; 749 if (mp->m_flags & M_PKTHDR) 750 m->m_pkthdr.len -= (req_len - len); 751 } else { 752 /* 753 * Trim from tail. Scan the mbuf chain, 754 * calculating its length and finding the last mbuf. 755 * If the adjustment only affects this mbuf, then just 756 * adjust and return. Otherwise, rescan and truncate 757 * after the remaining size. 758 */ 759 len = -len; 760 count = 0; 761 for (;;) { 762 count += m->m_len; 763 if (m->m_next == (struct mbuf *)0) 764 break; 765 m = m->m_next; 766 } 767 if (m->m_len >= len) { 768 m->m_len -= len; 769 if (mp->m_flags & M_PKTHDR) 770 mp->m_pkthdr.len -= len; 771 return; 772 } 773 count -= len; 774 if (count < 0) 775 count = 0; 776 /* 777 * Correct length for chain is "count". 778 * Find the mbuf with last data, adjust its length, 779 * and toss data from remaining mbufs on chain. 780 */ 781 m = mp; 782 if (m->m_flags & M_PKTHDR) 783 m->m_pkthdr.len = count; 784 for (; m; m = m->m_next) { 785 if (m->m_len >= count) { 786 m->m_len = count; 787 break; 788 } 789 count -= m->m_len; 790 } 791 while (m->m_next) 792 (m = m->m_next) ->m_len = 0; 793 } 794 } 795 796 /* 797 * Rearange an mbuf chain so that len bytes are contiguous 798 * and in the data area of an mbuf (so that mtod and dtom 799 * will work for a structure of size len). Returns the resulting 800 * mbuf chain on success, frees it and returns null on failure. 801 * If there is room, it will add up to max_protohdr-len extra bytes to the 802 * contiguous region in an attempt to avoid being called next time. 803 */ 804 int MPFail; 805 806 struct mbuf * 807 m_pullup(struct mbuf *n, int len) 808 { 809 struct mbuf *m; 810 int count; 811 int space; 812 813 /* 814 * If first mbuf has no cluster, and has room for len bytes 815 * without shifting current data, pullup into it, 816 * otherwise allocate a new mbuf to prepend to the chain. 817 */ 818 if ((n->m_flags & M_EXT) == 0 && 819 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 820 if (n->m_len >= len) 821 return (n); 822 m = n; 823 n = n->m_next; 824 len -= m->m_len; 825 } else { 826 if (len > MHLEN) 827 goto bad; 828 MGET(m, M_DONTWAIT, n->m_type); 829 if (m == 0) 830 goto bad; 831 MCLAIM(m, n->m_owner); 832 m->m_len = 0; 833 if (n->m_flags & M_PKTHDR) { 834 M_COPY_PKTHDR(m, n); 835 m_tag_delete_chain(n, NULL); 836 n->m_flags &= ~M_PKTHDR; 837 } 838 } 839 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 840 do { 841 count = min(min(max(len, max_protohdr), space), n->m_len); 842 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 843 (unsigned)count); 844 len -= count; 845 m->m_len += count; 846 n->m_len -= count; 847 space -= count; 848 if (n->m_len) 849 n->m_data += count; 850 else 851 n = m_free(n); 852 } while (len > 0 && n); 853 if (len > 0) { 854 (void) m_free(m); 855 goto bad; 856 } 857 m->m_next = n; 858 return (m); 859 bad: 860 m_freem(n); 861 MPFail++; 862 return (NULL); 863 } 864 865 /* 866 * Like m_pullup(), except a new mbuf is always allocated, and we allow 867 * the amount of empty space before the data in the new mbuf to be specified 868 * (in the event that the caller expects to prepend later). 869 */ 870 int MSFail; 871 872 struct mbuf * 873 m_copyup(struct mbuf *n, int len, int dstoff) 874 { 875 struct mbuf *m; 876 int count, space; 877 878 if (len > (MHLEN - dstoff)) 879 goto bad; 880 MGET(m, M_DONTWAIT, n->m_type); 881 if (m == NULL) 882 goto bad; 883 MCLAIM(m, n->m_owner); 884 m->m_len = 0; 885 if (n->m_flags & M_PKTHDR) { 886 M_COPY_PKTHDR(m, n); 887 m_tag_delete_chain(n, NULL); 888 n->m_flags &= ~M_PKTHDR; 889 } 890 m->m_data += dstoff; 891 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 892 do { 893 count = min(min(max(len, max_protohdr), space), n->m_len); 894 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 895 (unsigned)count); 896 len -= count; 897 m->m_len += count; 898 n->m_len -= count; 899 space -= count; 900 if (n->m_len) 901 n->m_data += count; 902 else 903 n = m_free(n); 904 } while (len > 0 && n); 905 if (len > 0) { 906 (void) m_free(m); 907 goto bad; 908 } 909 m->m_next = n; 910 return (m); 911 bad: 912 m_freem(n); 913 MSFail++; 914 return (NULL); 915 } 916 917 /* 918 * Partition an mbuf chain in two pieces, returning the tail -- 919 * all but the first len0 bytes. In case of failure, it returns NULL and 920 * attempts to restore the chain to its original state. 921 */ 922 struct mbuf * 923 m_split(struct mbuf *m0, int len0, int wait) 924 { 925 926 return m_split0(m0, len0, wait, 1); 927 } 928 929 static struct mbuf * 930 m_split0(struct mbuf *m0, int len0, int wait, int copyhdr) 931 { 932 struct mbuf *m, *n; 933 unsigned len = len0, remain, len_save; 934 935 for (m = m0; m && len > m->m_len; m = m->m_next) 936 len -= m->m_len; 937 if (m == 0) 938 return (NULL); 939 remain = m->m_len - len; 940 if (copyhdr && (m0->m_flags & M_PKTHDR)) { 941 MGETHDR(n, wait, m0->m_type); 942 if (n == 0) 943 return (NULL); 944 MCLAIM(n, m0->m_owner); 945 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 946 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 947 len_save = m0->m_pkthdr.len; 948 m0->m_pkthdr.len = len0; 949 if (m->m_flags & M_EXT) 950 goto extpacket; 951 if (remain > MHLEN) { 952 /* m can't be the lead packet */ 953 MH_ALIGN(n, 0); 954 n->m_next = m_split(m, len, wait); 955 if (n->m_next == 0) { 956 (void) m_free(n); 957 m0->m_pkthdr.len = len_save; 958 return (NULL); 959 } else 960 return (n); 961 } else 962 MH_ALIGN(n, remain); 963 } else if (remain == 0) { 964 n = m->m_next; 965 m->m_next = 0; 966 return (n); 967 } else { 968 MGET(n, wait, m->m_type); 969 if (n == 0) 970 return (NULL); 971 MCLAIM(n, m->m_owner); 972 M_ALIGN(n, remain); 973 } 974 extpacket: 975 if (m->m_flags & M_EXT) { 976 n->m_ext = m->m_ext; 977 MCLADDREFERENCE(m, n); 978 n->m_data = m->m_data + len; 979 } else { 980 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + len, remain); 981 } 982 n->m_len = remain; 983 m->m_len = len; 984 n->m_next = m->m_next; 985 m->m_next = 0; 986 return (n); 987 } 988 /* 989 * Routine to copy from device local memory into mbufs. 990 */ 991 struct mbuf * 992 m_devget(char *buf, int totlen, int off0, struct ifnet *ifp, 993 void (*copy)(const void *from, void *to, size_t len)) 994 { 995 struct mbuf *m; 996 struct mbuf *top = 0, **mp = ⊤ 997 int off = off0, len; 998 char *cp; 999 char *epkt; 1000 1001 cp = buf; 1002 epkt = cp + totlen; 1003 if (off) { 1004 /* 1005 * If 'off' is non-zero, packet is trailer-encapsulated, 1006 * so we have to skip the type and length fields. 1007 */ 1008 cp += off + 2 * sizeof(u_int16_t); 1009 totlen -= 2 * sizeof(u_int16_t); 1010 } 1011 MGETHDR(m, M_DONTWAIT, MT_DATA); 1012 if (m == 0) 1013 return (NULL); 1014 m->m_pkthdr.rcvif = ifp; 1015 m->m_pkthdr.len = totlen; 1016 m->m_len = MHLEN; 1017 1018 while (totlen > 0) { 1019 if (top) { 1020 MGET(m, M_DONTWAIT, MT_DATA); 1021 if (m == 0) { 1022 m_freem(top); 1023 return (NULL); 1024 } 1025 m->m_len = MLEN; 1026 } 1027 len = min(totlen, epkt - cp); 1028 if (len >= MINCLSIZE) { 1029 MCLGET(m, M_DONTWAIT); 1030 if ((m->m_flags & M_EXT) == 0) { 1031 m_free(m); 1032 m_freem(top); 1033 return (NULL); 1034 } 1035 m->m_len = len = min(len, MCLBYTES); 1036 } else { 1037 /* 1038 * Place initial small packet/header at end of mbuf. 1039 */ 1040 if (len < m->m_len) { 1041 if (top == 0 && len + max_linkhdr <= m->m_len) 1042 m->m_data += max_linkhdr; 1043 m->m_len = len; 1044 } else 1045 len = m->m_len; 1046 } 1047 if (copy) 1048 copy(cp, mtod(m, caddr_t), (size_t)len); 1049 else 1050 memcpy(mtod(m, caddr_t), cp, (size_t)len); 1051 cp += len; 1052 *mp = m; 1053 mp = &m->m_next; 1054 totlen -= len; 1055 if (cp == epkt) 1056 cp = buf; 1057 } 1058 return (top); 1059 } 1060 1061 /* 1062 * Copy data from a buffer back into the indicated mbuf chain, 1063 * starting "off" bytes from the beginning, extending the mbuf 1064 * chain if necessary. 1065 */ 1066 void 1067 m_copyback(struct mbuf *m0, int off, int len, const void *cp) 1068 { 1069 #if defined(DEBUG) 1070 struct mbuf *origm = m0; 1071 int error; 1072 #endif /* defined(DEBUG) */ 1073 1074 if (m0 == NULL) 1075 return; 1076 1077 #if defined(DEBUG) 1078 error = 1079 #endif /* defined(DEBUG) */ 1080 m_copyback0(&m0, off, len, cp, 1081 M_COPYBACK0_COPYBACK|M_COPYBACK0_EXTEND, M_DONTWAIT); 1082 1083 #if defined(DEBUG) 1084 if (error != 0 || (m0 != NULL && origm != m0)) 1085 panic("m_copyback"); 1086 #endif /* defined(DEBUG) */ 1087 } 1088 1089 struct mbuf * 1090 m_copyback_cow(struct mbuf *m0, int off, int len, const void *cp, int how) 1091 { 1092 int error; 1093 1094 /* don't support chain expansion */ 1095 KDASSERT(off + len <= m_length(m0)); 1096 1097 error = m_copyback0(&m0, off, len, cp, 1098 M_COPYBACK0_COPYBACK|M_COPYBACK0_COW, how); 1099 if (error) { 1100 /* 1101 * no way to recover from partial success. 1102 * just free the chain. 1103 */ 1104 m_freem(m0); 1105 return NULL; 1106 } 1107 return m0; 1108 } 1109 1110 /* 1111 * m_makewritable: ensure the specified range writable. 1112 */ 1113 int 1114 m_makewritable(struct mbuf **mp, int off, int len, int how) 1115 { 1116 int error; 1117 #if defined(DEBUG) 1118 struct mbuf *n; 1119 int origlen, reslen; 1120 1121 origlen = m_length(*mp); 1122 #endif /* defined(DEBUG) */ 1123 1124 #if 0 /* M_COPYALL is large enough */ 1125 if (len == M_COPYALL) 1126 len = m_length(*mp) - off; /* XXX */ 1127 #endif 1128 1129 error = m_copyback0(mp, off, len, NULL, 1130 M_COPYBACK0_PRESERVE|M_COPYBACK0_COW, how); 1131 1132 #if defined(DEBUG) 1133 reslen = 0; 1134 for (n = *mp; n; n = n->m_next) 1135 reslen += n->m_len; 1136 if (origlen != reslen) 1137 panic("m_makewritable: length changed"); 1138 if (((*mp)->m_flags & M_PKTHDR) != 0 && reslen != (*mp)->m_pkthdr.len) 1139 panic("m_makewritable: inconsist"); 1140 #endif /* defined(DEBUG) */ 1141 1142 return error; 1143 } 1144 1145 int 1146 m_copyback0(struct mbuf **mp0, int off, int len, const void *vp, int flags, 1147 int how) 1148 { 1149 int mlen; 1150 struct mbuf *m, *n; 1151 struct mbuf **mp; 1152 int totlen = 0; 1153 const char *cp = vp; 1154 1155 KASSERT(mp0 != NULL); 1156 KASSERT(*mp0 != NULL); 1157 KASSERT((flags & M_COPYBACK0_PRESERVE) == 0 || cp == NULL); 1158 KASSERT((flags & M_COPYBACK0_COPYBACK) == 0 || cp != NULL); 1159 1160 mp = mp0; 1161 m = *mp; 1162 while (off > (mlen = m->m_len)) { 1163 off -= mlen; 1164 totlen += mlen; 1165 if (m->m_next == 0) { 1166 if ((flags & M_COPYBACK0_EXTEND) == 0) 1167 goto out; 1168 n = m_getclr(how, m->m_type); 1169 if (n == 0) 1170 goto out; 1171 n->m_len = min(MLEN, len + off); 1172 m->m_next = n; 1173 } 1174 mp = &m->m_next; 1175 m = m->m_next; 1176 } 1177 while (len > 0) { 1178 mlen = m->m_len - off; 1179 if (mlen != 0 && M_READONLY(m)) { 1180 char *datap; 1181 int eatlen; 1182 1183 /* 1184 * this mbuf is read-only. 1185 * allocate a new writable mbuf and try again. 1186 */ 1187 1188 #if defined(DIAGNOSTIC) 1189 if ((flags & M_COPYBACK0_COW) == 0) 1190 panic("m_copyback0: read-only"); 1191 #endif /* defined(DIAGNOSTIC) */ 1192 1193 /* 1194 * if we're going to write into the middle of 1195 * a mbuf, split it first. 1196 */ 1197 if (off > 0 && len < mlen) { 1198 n = m_split0(m, off, how, 0); 1199 if (n == NULL) 1200 goto enobufs; 1201 m->m_next = n; 1202 mp = &m->m_next; 1203 m = n; 1204 off = 0; 1205 continue; 1206 } 1207 1208 /* 1209 * XXX TODO coalesce into the trailingspace of 1210 * the previous mbuf when possible. 1211 */ 1212 1213 /* 1214 * allocate a new mbuf. copy packet header if needed. 1215 */ 1216 MGET(n, how, m->m_type); 1217 if (n == NULL) 1218 goto enobufs; 1219 MCLAIM(n, m->m_owner); 1220 if (off == 0 && (m->m_flags & M_PKTHDR) != 0) { 1221 /* XXX M_MOVE_PKTHDR */ 1222 M_COPY_PKTHDR(n, m); 1223 n->m_len = MHLEN; 1224 } else { 1225 if (len >= MINCLSIZE) 1226 MCLGET(n, M_DONTWAIT); 1227 n->m_len = 1228 (n->m_flags & M_EXT) ? MCLBYTES : MLEN; 1229 } 1230 if (n->m_len > len) 1231 n->m_len = len; 1232 1233 /* 1234 * free the region which has been overwritten. 1235 * copying data from old mbufs if requested. 1236 */ 1237 if (flags & M_COPYBACK0_PRESERVE) 1238 datap = mtod(n, char *); 1239 else 1240 datap = NULL; 1241 eatlen = n->m_len; 1242 KDASSERT(off == 0 || eatlen >= mlen); 1243 if (off > 0) { 1244 KDASSERT(len >= mlen); 1245 m->m_len = off; 1246 m->m_next = n; 1247 if (datap) { 1248 m_copydata(m, off, mlen, datap); 1249 datap += mlen; 1250 } 1251 eatlen -= mlen; 1252 mp = &m->m_next; 1253 m = m->m_next; 1254 } 1255 while (m != NULL && M_READONLY(m) && 1256 n->m_type == m->m_type && eatlen > 0) { 1257 mlen = min(eatlen, m->m_len); 1258 if (datap) { 1259 m_copydata(m, 0, mlen, datap); 1260 datap += mlen; 1261 } 1262 m->m_data += mlen; 1263 m->m_len -= mlen; 1264 eatlen -= mlen; 1265 if (m->m_len == 0) 1266 *mp = m = m_free(m); 1267 } 1268 if (eatlen > 0) 1269 n->m_len -= eatlen; 1270 n->m_next = m; 1271 *mp = m = n; 1272 continue; 1273 } 1274 mlen = min(mlen, len); 1275 if (flags & M_COPYBACK0_COPYBACK) { 1276 memcpy(mtod(m, caddr_t) + off, cp, (unsigned)mlen); 1277 cp += mlen; 1278 } 1279 len -= mlen; 1280 mlen += off; 1281 off = 0; 1282 totlen += mlen; 1283 if (len == 0) 1284 break; 1285 if (m->m_next == 0) { 1286 if ((flags & M_COPYBACK0_EXTEND) == 0) 1287 goto out; 1288 n = m_get(how, m->m_type); 1289 if (n == 0) 1290 break; 1291 n->m_len = min(MLEN, len); 1292 m->m_next = n; 1293 } 1294 mp = &m->m_next; 1295 m = m->m_next; 1296 } 1297 out: if (((m = *mp0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 1298 m->m_pkthdr.len = totlen; 1299 1300 return 0; 1301 1302 enobufs: 1303 return ENOBUFS; 1304 } 1305 1306 /* 1307 * Apply function f to the data in an mbuf chain starting "off" bytes from the 1308 * beginning, continuing for "len" bytes. 1309 */ 1310 int 1311 m_apply(struct mbuf *m, int off, int len, 1312 int (*f)(void *, caddr_t, unsigned int), void *arg) 1313 { 1314 unsigned int count; 1315 int rval; 1316 1317 KASSERT(len >= 0); 1318 KASSERT(off >= 0); 1319 1320 while (off > 0) { 1321 KASSERT(m != NULL); 1322 if (off < m->m_len) 1323 break; 1324 off -= m->m_len; 1325 m = m->m_next; 1326 } 1327 while (len > 0) { 1328 KASSERT(m != NULL); 1329 count = min(m->m_len - off, len); 1330 1331 rval = (*f)(arg, mtod(m, caddr_t) + off, count); 1332 if (rval) 1333 return (rval); 1334 1335 len -= count; 1336 off = 0; 1337 m = m->m_next; 1338 } 1339 1340 return (0); 1341 } 1342 1343 /* 1344 * Return a pointer to mbuf/offset of location in mbuf chain. 1345 */ 1346 struct mbuf * 1347 m_getptr(struct mbuf *m, int loc, int *off) 1348 { 1349 1350 while (loc >= 0) { 1351 /* Normal end of search */ 1352 if (m->m_len > loc) { 1353 *off = loc; 1354 return (m); 1355 } else { 1356 loc -= m->m_len; 1357 1358 if (m->m_next == NULL) { 1359 if (loc == 0) { 1360 /* Point at the end of valid data */ 1361 *off = m->m_len; 1362 return (m); 1363 } else 1364 return (NULL); 1365 } else 1366 m = m->m_next; 1367 } 1368 } 1369 1370 return (NULL); 1371 }
Cache object: 3231edc31fdce8c185070b41bcad2153
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]