Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/kern/uipc_mbuf.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* uipc_mbuf.c,v 1.84 2004/07/21 12:06:46 yamt Exp */ 2 3 /*- 4 * Copyright (c) 1999, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the NetBSD 22 * Foundation, Inc. and its contributors. 23 * 4. Neither the name of The NetBSD Foundation nor the names of its 24 * contributors may be used to endorse or promote products derived 25 * from this software without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 29 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 30 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 31 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 37 * POSSIBILITY OF SUCH DAMAGE. 38 */ 39 40 /* 41 * Copyright (c) 1982, 1986, 1988, 1991, 1993 42 * The Regents of the University of California. All rights reserved. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. Neither the name of the University nor the names of its contributors 53 * may be used to endorse or promote products derived from this software 54 * without specific prior written permission. 55 * 56 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 57 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 58 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 59 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 60 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 61 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 62 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 63 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 64 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 65 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 66 * SUCH DAMAGE. 67 * 68 * @(#)uipc_mbuf.c 8.4 (Berkeley) 2/14/95 69 */ 70 71 #include <sys/cdefs.h> 72 __KERNEL_RCSID(0, "$NetBSD: uipc_mbuf.c,v 1.80.2.5 2004/10/08 03:05:26 jmc Exp $"); 73 74 #include "opt_mbuftrace.h" 75 76 #include <sys/param.h> 77 #include <sys/systm.h> 78 #include <sys/proc.h> 79 #include <sys/malloc.h> 80 #define MBTYPES 81 #include <sys/mbuf.h> 82 #include <sys/kernel.h> 83 #include <sys/syslog.h> 84 #include <sys/domain.h> 85 #include <sys/protosw.h> 86 #include <sys/pool.h> 87 #include <sys/socket.h> 88 #include <sys/sysctl.h> 89 90 #include <net/if.h> 91 92 #include <uvm/uvm.h> 93 94 95 struct pool mbpool; /* mbuf pool */ 96 struct pool mclpool; /* mbuf cluster pool */ 97 98 struct pool_cache mbpool_cache; 99 struct pool_cache mclpool_cache; 100 101 struct mbstat mbstat; 102 int max_linkhdr; 103 int max_protohdr; 104 int max_hdr; 105 int max_datalen; 106 107 static int mb_ctor(void *, void *, int); 108 109 void *mclpool_alloc(struct pool *, int); 110 void mclpool_release(struct pool *, void *); 111 112 struct pool_allocator mclpool_allocator = { 113 mclpool_alloc, mclpool_release, 0, 114 }; 115 116 static struct mbuf *m_copym0(struct mbuf *, int, int, int, int); 117 static struct mbuf *m_split0(struct mbuf *, int, int, int); 118 static int m_copyback0(struct mbuf **, int, int, caddr_t, int, int); 119 120 /* flags for m_copyback0 */ 121 #define M_COPYBACK0_COPYBACK 0x0001 /* copyback from cp */ 122 #define M_COPYBACK0_PRESERVE 0x0002 /* preserve original data */ 123 #define M_COPYBACK0_COW 0x0004 /* do copy-on-write */ 124 #define M_COPYBACK0_EXTEND 0x0008 /* extend chain */ 125 126 const char mclpool_warnmsg[] = 127 "WARNING: mclpool limit reached; increase NMBCLUSTERS"; 128 129 MALLOC_DEFINE(M_MBUF, "mbuf", "mbuf"); 130 131 #ifdef MBUFTRACE 132 struct mownerhead mowners = LIST_HEAD_INITIALIZER(mowners); 133 struct mowner unknown_mowners[] = { 134 { "unknown", "free" }, 135 { "unknown", "data" }, 136 { "unknown", "header" }, 137 { "unknown", "soname" }, 138 { "unknown", "soopts" }, 139 { "unknown", "ftable" }, 140 { "unknown", "control" }, 141 { "unknown", "oobdata" }, 142 }; 143 struct mowner revoked_mowner = { "revoked", "" }; 144 #endif 145 146 /* 147 * Initialize the mbuf allocator. 148 */ 149 void 150 mbinit(void) 151 { 152 153 KASSERT(sizeof(struct _m_ext) <= MHLEN); 154 KASSERT(sizeof(struct mbuf) == MSIZE); 155 156 pool_init(&mbpool, msize, 0, 0, 0, "mbpl", NULL); 157 pool_init(&mclpool, mclbytes, 0, 0, 0, "mclpl", &mclpool_allocator); 158 159 pool_set_drain_hook(&mbpool, m_reclaim, NULL); 160 pool_set_drain_hook(&mclpool, m_reclaim, NULL); 161 162 pool_cache_init(&mbpool_cache, &mbpool, mb_ctor, NULL, NULL); 163 pool_cache_init(&mclpool_cache, &mclpool, NULL, NULL, NULL); 164 165 /* 166 * Set the hard limit on the mclpool to the number of 167 * mbuf clusters the kernel is to support. Log the limit 168 * reached message max once a minute. 169 */ 170 pool_sethardlimit(&mclpool, nmbclusters, mclpool_warnmsg, 60); 171 172 /* 173 * Set a low water mark for both mbufs and clusters. This should 174 * help ensure that they can be allocated in a memory starvation 175 * situation. This is important for e.g. diskless systems which 176 * must allocate mbufs in order for the pagedaemon to clean pages. 177 */ 178 pool_setlowat(&mbpool, mblowat); 179 pool_setlowat(&mclpool, mcllowat); 180 181 #ifdef MBUFTRACE 182 { 183 /* 184 * Attach the unknown mowners. 185 */ 186 int i; 187 MOWNER_ATTACH(&revoked_mowner); 188 for (i = sizeof(unknown_mowners)/sizeof(unknown_mowners[0]); 189 i-- > 0; ) 190 MOWNER_ATTACH(&unknown_mowners[i]); 191 } 192 #endif 193 } 194 195 /* 196 * sysctl helper routine for the kern.mbuf subtree. nmbclusters may 197 * or may not be writable, and mblowat and mcllowat need range 198 * checking and pool tweaking after being reset. 199 */ 200 static int 201 sysctl_kern_mbuf(SYSCTLFN_ARGS) 202 { 203 int error, newval; 204 struct sysctlnode node; 205 206 node = *rnode; 207 node.sysctl_data = &newval; 208 switch (rnode->sysctl_num) { 209 case MBUF_NMBCLUSTERS: 210 if (mb_map != NULL) { 211 node.sysctl_flags &= ~CTLFLAG_READWRITE; 212 node.sysctl_flags |= CTLFLAG_READONLY; 213 } 214 /* FALLTHROUGH */ 215 case MBUF_MBLOWAT: 216 case MBUF_MCLLOWAT: 217 newval = *(int*)rnode->sysctl_data; 218 break; 219 default: 220 return (EOPNOTSUPP); 221 } 222 223 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 224 if (error || newp == NULL) 225 return (error); 226 if (newval < 0) 227 return (EINVAL); 228 229 switch (node.sysctl_num) { 230 case MBUF_NMBCLUSTERS: 231 if (newval < nmbclusters) 232 return (EINVAL); 233 nmbclusters = newval; 234 pool_sethardlimit(&mclpool, nmbclusters, mclpool_warnmsg, 60); 235 break; 236 case MBUF_MBLOWAT: 237 mblowat = newval; 238 pool_setlowat(&mbpool, mblowat); 239 break; 240 case MBUF_MCLLOWAT: 241 mcllowat = newval; 242 pool_setlowat(&mclpool, mcllowat); 243 break; 244 } 245 246 return (0); 247 } 248 249 #ifdef MBUFTRACE 250 static int 251 sysctl_kern_mbuf_mowners(SYSCTLFN_ARGS) 252 { 253 struct mowner *mo; 254 size_t len = 0; 255 int error = 0; 256 257 if (namelen != 0) 258 return (EINVAL); 259 if (newp != NULL) 260 return (EPERM); 261 262 LIST_FOREACH(mo, &mowners, mo_link) { 263 if (oldp != NULL) { 264 if (*oldlenp - len < sizeof(*mo)) { 265 error = ENOMEM; 266 break; 267 } 268 error = copyout(mo, (caddr_t) oldp + len, 269 sizeof(*mo)); 270 if (error) 271 break; 272 } 273 len += sizeof(*mo); 274 } 275 276 if (error == 0) 277 *oldlenp = len; 278 279 return (error); 280 } 281 #endif /* MBUFTRACE */ 282 283 SYSCTL_SETUP(sysctl_kern_mbuf_setup, "sysctl kern.mbuf subtree setup") 284 { 285 286 sysctl_createv(clog, 0, NULL, NULL, 287 CTLFLAG_PERMANENT, 288 CTLTYPE_NODE, "kern", NULL, 289 NULL, 0, NULL, 0, 290 CTL_KERN, CTL_EOL); 291 sysctl_createv(clog, 0, NULL, NULL, 292 CTLFLAG_PERMANENT, 293 CTLTYPE_NODE, "mbuf", 294 SYSCTL_DESCR("mbuf control variables"), 295 NULL, 0, NULL, 0, 296 CTL_KERN, KERN_MBUF, CTL_EOL); 297 298 sysctl_createv(clog, 0, NULL, NULL, 299 CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE, 300 CTLTYPE_INT, "msize", 301 SYSCTL_DESCR("mbuf base size"), 302 NULL, msize, NULL, 0, 303 CTL_KERN, KERN_MBUF, MBUF_MSIZE, CTL_EOL); 304 sysctl_createv(clog, 0, NULL, NULL, 305 CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE, 306 CTLTYPE_INT, "mclbytes", 307 SYSCTL_DESCR("mbuf cluster size"), 308 NULL, mclbytes, NULL, 0, 309 CTL_KERN, KERN_MBUF, MBUF_MCLBYTES, CTL_EOL); 310 sysctl_createv(clog, 0, NULL, NULL, 311 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 312 CTLTYPE_INT, "nmbclusters", 313 SYSCTL_DESCR("Limit on the number of mbuf clusters"), 314 sysctl_kern_mbuf, 0, &nmbclusters, 0, 315 CTL_KERN, KERN_MBUF, MBUF_NMBCLUSTERS, CTL_EOL); 316 sysctl_createv(clog, 0, NULL, NULL, 317 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 318 CTLTYPE_INT, "mblowat", 319 SYSCTL_DESCR("mbuf low water mark"), 320 sysctl_kern_mbuf, 0, &mblowat, 0, 321 CTL_KERN, KERN_MBUF, MBUF_MBLOWAT, CTL_EOL); 322 sysctl_createv(clog, 0, NULL, NULL, 323 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 324 CTLTYPE_INT, "mcllowat", 325 SYSCTL_DESCR("mbuf cluster low water mark"), 326 sysctl_kern_mbuf, 0, &mcllowat, 0, 327 CTL_KERN, KERN_MBUF, MBUF_MCLLOWAT, CTL_EOL); 328 sysctl_createv(clog, 0, NULL, NULL, 329 CTLFLAG_PERMANENT, 330 CTLTYPE_STRUCT, "stats", 331 SYSCTL_DESCR("mbuf allocation statistics"), 332 NULL, 0, &mbstat, sizeof(mbstat), 333 CTL_KERN, KERN_MBUF, MBUF_STATS, CTL_EOL); 334 #ifdef MBUFTRACE 335 sysctl_createv(clog, 0, NULL, NULL, 336 CTLFLAG_PERMANENT, 337 CTLTYPE_STRUCT, "mowners", 338 SYSCTL_DESCR("Information about mbuf owners"), 339 sysctl_kern_mbuf_mowners, 0, NULL, 0, 340 CTL_KERN, KERN_MBUF, MBUF_MOWNERS, CTL_EOL); 341 #endif /* MBUFTRACE */ 342 } 343 344 void * 345 mclpool_alloc(struct pool *pp, int flags) 346 { 347 boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; 348 349 return ((void *)uvm_km_alloc_poolpage1(mb_map, NULL, waitok)); 350 } 351 352 void 353 mclpool_release(struct pool *pp, void *v) 354 { 355 356 uvm_km_free_poolpage1(mb_map, (vaddr_t)v); 357 } 358 359 /*ARGSUSED*/ 360 static int 361 mb_ctor(void *arg, void *object, int flags) 362 { 363 struct mbuf *m = object; 364 365 #ifdef POOL_VTOPHYS 366 m->m_paddr = POOL_VTOPHYS(m); 367 #else 368 m->m_paddr = M_PADDR_INVALID; 369 #endif 370 return (0); 371 } 372 373 void 374 m_reclaim(void *arg, int flags) 375 { 376 struct domain *dp; 377 struct protosw *pr; 378 struct ifnet *ifp; 379 int s = splvm(); 380 381 for (dp = domains; dp; dp = dp->dom_next) 382 for (pr = dp->dom_protosw; 383 pr < dp->dom_protoswNPROTOSW; pr++) 384 if (pr->pr_drain) 385 (*pr->pr_drain)(); 386 for (ifp = TAILQ_FIRST(&ifnet); ifp; ifp = TAILQ_NEXT(ifp, if_list)) 387 if (ifp->if_drain) 388 (*ifp->if_drain)(ifp); 389 splx(s); 390 mbstat.m_drain++; 391 } 392 393 /* 394 * Space allocation routines. 395 * These are also available as macros 396 * for critical paths. 397 */ 398 struct mbuf * 399 m_get(int nowait, int type) 400 { 401 struct mbuf *m; 402 403 MGET(m, nowait, type); 404 return (m); 405 } 406 407 struct mbuf * 408 m_gethdr(int nowait, int type) 409 { 410 struct mbuf *m; 411 412 MGETHDR(m, nowait, type); 413 return (m); 414 } 415 416 struct mbuf * 417 m_getclr(int nowait, int type) 418 { 419 struct mbuf *m; 420 421 MGET(m, nowait, type); 422 if (m == 0) 423 return (NULL); 424 memset(mtod(m, caddr_t), 0, MLEN); 425 return (m); 426 } 427 428 void 429 m_clget(struct mbuf *m, int nowait) 430 { 431 432 MCLGET(m, nowait); 433 } 434 435 struct mbuf * 436 m_free(struct mbuf *m) 437 { 438 struct mbuf *n; 439 440 MFREE(m, n); 441 return (n); 442 } 443 444 void 445 m_freem(struct mbuf *m) 446 { 447 struct mbuf *n; 448 449 if (m == NULL) 450 return; 451 do { 452 MFREE(m, n); 453 m = n; 454 } while (m); 455 } 456 457 #ifdef MBUFTRACE 458 /* 459 * Walk a chain of mbufs, claiming ownership of each mbuf in the chain. 460 */ 461 void 462 m_claimm(struct mbuf *m, struct mowner *mo) 463 { 464 465 for (; m != NULL; m = m->m_next) 466 MCLAIM(m, mo); 467 } 468 #endif 469 470 /* 471 * Mbuffer utility routines. 472 */ 473 474 /* 475 * Lesser-used path for M_PREPEND: 476 * allocate new mbuf to prepend to chain, 477 * copy junk along. 478 */ 479 struct mbuf * 480 m_prepend(struct mbuf *m, int len, int how) 481 { 482 struct mbuf *mn; 483 484 MGET(mn, how, m->m_type); 485 if (mn == (struct mbuf *)NULL) { 486 m_freem(m); 487 return ((struct mbuf *)NULL); 488 } 489 if (m->m_flags & M_PKTHDR) { 490 M_COPY_PKTHDR(mn, m); 491 m_tag_delete_chain(m, NULL); 492 m->m_flags &= ~M_PKTHDR; 493 } else { 494 MCLAIM(mn, m->m_owner); 495 } 496 mn->m_next = m; 497 m = mn; 498 if (len < MHLEN) 499 MH_ALIGN(m, len); 500 m->m_len = len; 501 return (m); 502 } 503 504 /* 505 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 506 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 507 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 508 */ 509 int MCFail; 510 511 struct mbuf * 512 m_copym(struct mbuf *m, int off0, int len, int wait) 513 { 514 515 return m_copym0(m, off0, len, wait, 0); /* shallow copy on M_EXT */ 516 } 517 518 struct mbuf * 519 m_dup(struct mbuf *m, int off0, int len, int wait) 520 { 521 522 return m_copym0(m, off0, len, wait, 1); /* deep copy */ 523 } 524 525 static struct mbuf * 526 m_copym0(struct mbuf *m, int off0, int len, int wait, int deep) 527 { 528 struct mbuf *n, **np; 529 int off = off0; 530 struct mbuf *top; 531 int copyhdr = 0; 532 533 if (off < 0 || len < 0) 534 panic("m_copym: off %d, len %d", off, len); 535 if (off == 0 && m->m_flags & M_PKTHDR) 536 copyhdr = 1; 537 while (off > 0) { 538 if (m == 0) 539 panic("m_copym: m == 0"); 540 if (off < m->m_len) 541 break; 542 off -= m->m_len; 543 m = m->m_next; 544 } 545 np = ⊤ 546 top = 0; 547 while (len > 0) { 548 if (m == 0) { 549 if (len != M_COPYALL) 550 panic("m_copym: m == 0 and not COPYALL"); 551 break; 552 } 553 MGET(n, wait, m->m_type); 554 *np = n; 555 if (n == 0) 556 goto nospace; 557 MCLAIM(n, m->m_owner); 558 if (copyhdr) { 559 M_COPY_PKTHDR(n, m); 560 if (len == M_COPYALL) 561 n->m_pkthdr.len -= off0; 562 else 563 n->m_pkthdr.len = len; 564 copyhdr = 0; 565 } 566 n->m_len = min(len, m->m_len - off); 567 if (m->m_flags & M_EXT) { 568 if (!deep) { 569 n->m_data = m->m_data + off; 570 n->m_ext = m->m_ext; 571 MCLADDREFERENCE(m, n); 572 } else { 573 /* 574 * we are unsure about the way m was allocated. 575 * copy into multiple MCLBYTES cluster mbufs. 576 */ 577 MCLGET(n, wait); 578 n->m_len = 0; 579 n->m_len = M_TRAILINGSPACE(n); 580 n->m_len = min(n->m_len, len); 581 n->m_len = min(n->m_len, m->m_len - off); 582 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, 583 (unsigned)n->m_len); 584 } 585 } else 586 memcpy(mtod(n, caddr_t), mtod(m, caddr_t)+off, 587 (unsigned)n->m_len); 588 if (len != M_COPYALL) 589 len -= n->m_len; 590 off += n->m_len; 591 #ifdef DIAGNOSTIC 592 if (off > m->m_len) 593 panic("m_copym0 overrun"); 594 #endif 595 if (off == m->m_len) { 596 m = m->m_next; 597 off = 0; 598 } 599 np = &n->m_next; 600 } 601 if (top == 0) 602 MCFail++; 603 return (top); 604 nospace: 605 m_freem(top); 606 MCFail++; 607 return (NULL); 608 } 609 610 /* 611 * Copy an entire packet, including header (which must be present). 612 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'. 613 */ 614 struct mbuf * 615 m_copypacket(struct mbuf *m, int how) 616 { 617 struct mbuf *top, *n, *o; 618 619 MGET(n, how, m->m_type); 620 top = n; 621 if (!n) 622 goto nospace; 623 624 MCLAIM(n, m->m_owner); 625 M_COPY_PKTHDR(n, m); 626 n->m_len = m->m_len; 627 if (m->m_flags & M_EXT) { 628 n->m_data = m->m_data; 629 n->m_ext = m->m_ext; 630 MCLADDREFERENCE(m, n); 631 } else { 632 memcpy(mtod(n, char *), mtod(m, char *), n->m_len); 633 } 634 635 m = m->m_next; 636 while (m) { 637 MGET(o, how, m->m_type); 638 if (!o) 639 goto nospace; 640 641 MCLAIM(o, m->m_owner); 642 n->m_next = o; 643 n = n->m_next; 644 645 n->m_len = m->m_len; 646 if (m->m_flags & M_EXT) { 647 n->m_data = m->m_data; 648 n->m_ext = m->m_ext; 649 MCLADDREFERENCE(m, n); 650 } else { 651 memcpy(mtod(n, char *), mtod(m, char *), n->m_len); 652 } 653 654 m = m->m_next; 655 } 656 return top; 657 nospace: 658 m_freem(top); 659 MCFail++; 660 return NULL; 661 } 662 663 /* 664 * Copy data from an mbuf chain starting "off" bytes from the beginning, 665 * continuing for "len" bytes, into the indicated buffer. 666 */ 667 void 668 m_copydata(struct mbuf *m, int off, int len, caddr_t cp) 669 { 670 unsigned count; 671 672 if (off < 0 || len < 0) 673 panic("m_copydata"); 674 while (off > 0) { 675 if (m == 0) 676 panic("m_copydata"); 677 if (off < m->m_len) 678 break; 679 off -= m->m_len; 680 m = m->m_next; 681 } 682 while (len > 0) { 683 if (m == 0) 684 panic("m_copydata"); 685 count = min(m->m_len - off, len); 686 memcpy(cp, mtod(m, caddr_t) + off, count); 687 len -= count; 688 cp += count; 689 off = 0; 690 m = m->m_next; 691 } 692 } 693 694 /* 695 * Concatenate mbuf chain n to m. 696 * n might be copied into m (when n->m_len is small), therefore data portion of 697 * n could be copied into an mbuf of different mbuf type. 698 * Any m_pkthdr is not updated. 699 */ 700 void 701 m_cat(struct mbuf *m, struct mbuf *n) 702 { 703 704 while (m->m_next) 705 m = m->m_next; 706 while (n) { 707 if (M_READONLY(m) || n->m_len > M_TRAILINGSPACE(m)) { 708 /* just join the two chains */ 709 m->m_next = n; 710 return; 711 } 712 /* splat the data from one into the other */ 713 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 714 (u_int)n->m_len); 715 m->m_len += n->m_len; 716 n = m_free(n); 717 } 718 } 719 720 void 721 m_adj(struct mbuf *mp, int req_len) 722 { 723 int len = req_len; 724 struct mbuf *m; 725 int count; 726 727 if ((m = mp) == NULL) 728 return; 729 if (len >= 0) { 730 /* 731 * Trim from head. 732 */ 733 while (m != NULL && len > 0) { 734 if (m->m_len <= len) { 735 len -= m->m_len; 736 m->m_len = 0; 737 m = m->m_next; 738 } else { 739 m->m_len -= len; 740 m->m_data += len; 741 len = 0; 742 } 743 } 744 m = mp; 745 if (mp->m_flags & M_PKTHDR) 746 m->m_pkthdr.len -= (req_len - len); 747 } else { 748 /* 749 * Trim from tail. Scan the mbuf chain, 750 * calculating its length and finding the last mbuf. 751 * If the adjustment only affects this mbuf, then just 752 * adjust and return. Otherwise, rescan and truncate 753 * after the remaining size. 754 */ 755 len = -len; 756 count = 0; 757 for (;;) { 758 count += m->m_len; 759 if (m->m_next == (struct mbuf *)0) 760 break; 761 m = m->m_next; 762 } 763 if (m->m_len >= len) { 764 m->m_len -= len; 765 if (mp->m_flags & M_PKTHDR) 766 mp->m_pkthdr.len -= len; 767 return; 768 } 769 count -= len; 770 if (count < 0) 771 count = 0; 772 /* 773 * Correct length for chain is "count". 774 * Find the mbuf with last data, adjust its length, 775 * and toss data from remaining mbufs on chain. 776 */ 777 m = mp; 778 if (m->m_flags & M_PKTHDR) 779 m->m_pkthdr.len = count; 780 for (; m; m = m->m_next) { 781 if (m->m_len >= count) { 782 m->m_len = count; 783 break; 784 } 785 count -= m->m_len; 786 } 787 while (m->m_next) 788 (m = m->m_next) ->m_len = 0; 789 } 790 } 791 792 /* 793 * Rearange an mbuf chain so that len bytes are contiguous 794 * and in the data area of an mbuf (so that mtod and dtom 795 * will work for a structure of size len). Returns the resulting 796 * mbuf chain on success, frees it and returns null on failure. 797 * If there is room, it will add up to max_protohdr-len extra bytes to the 798 * contiguous region in an attempt to avoid being called next time. 799 */ 800 int MPFail; 801 802 struct mbuf * 803 m_pullup(struct mbuf *n, int len) 804 { 805 struct mbuf *m; 806 int count; 807 int space; 808 809 /* 810 * If first mbuf has no cluster, and has room for len bytes 811 * without shifting current data, pullup into it, 812 * otherwise allocate a new mbuf to prepend to the chain. 813 */ 814 if ((n->m_flags & M_EXT) == 0 && 815 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 816 if (n->m_len >= len) 817 return (n); 818 m = n; 819 n = n->m_next; 820 len -= m->m_len; 821 } else { 822 if (len > MHLEN) 823 goto bad; 824 MGET(m, M_DONTWAIT, n->m_type); 825 if (m == 0) 826 goto bad; 827 MCLAIM(m, n->m_owner); 828 m->m_len = 0; 829 if (n->m_flags & M_PKTHDR) { 830 M_COPY_PKTHDR(m, n); 831 m_tag_delete_chain(n, NULL); 832 n->m_flags &= ~M_PKTHDR; 833 } 834 } 835 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 836 do { 837 count = min(min(max(len, max_protohdr), space), n->m_len); 838 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 839 (unsigned)count); 840 len -= count; 841 m->m_len += count; 842 n->m_len -= count; 843 space -= count; 844 if (n->m_len) 845 n->m_data += count; 846 else 847 n = m_free(n); 848 } while (len > 0 && n); 849 if (len > 0) { 850 (void) m_free(m); 851 goto bad; 852 } 853 m->m_next = n; 854 return (m); 855 bad: 856 m_freem(n); 857 MPFail++; 858 return (NULL); 859 } 860 861 /* 862 * Like m_pullup(), except a new mbuf is always allocated, and we allow 863 * the amount of empty space before the data in the new mbuf to be specified 864 * (in the event that the caller expects to prepend later). 865 */ 866 int MSFail; 867 868 struct mbuf * 869 m_copyup(struct mbuf *n, int len, int dstoff) 870 { 871 struct mbuf *m; 872 int count, space; 873 874 if (len > (MHLEN - dstoff)) 875 goto bad; 876 MGET(m, M_DONTWAIT, n->m_type); 877 if (m == NULL) 878 goto bad; 879 MCLAIM(m, n->m_owner); 880 m->m_len = 0; 881 if (n->m_flags & M_PKTHDR) { 882 M_COPY_PKTHDR(m, n); 883 m_tag_delete_chain(m, NULL); 884 n->m_flags &= ~M_PKTHDR; 885 } 886 m->m_data += dstoff; 887 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 888 do { 889 count = min(min(max(len, max_protohdr), space), n->m_len); 890 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 891 (unsigned)count); 892 len -= count; 893 m->m_len += count; 894 n->m_len -= count; 895 space -= count; 896 if (n->m_len) 897 n->m_data += count; 898 else 899 n = m_free(n); 900 } while (len > 0 && n); 901 if (len > 0) { 902 (void) m_free(m); 903 goto bad; 904 } 905 m->m_next = n; 906 return (m); 907 bad: 908 m_freem(n); 909 MSFail++; 910 return (NULL); 911 } 912 913 /* 914 * Partition an mbuf chain in two pieces, returning the tail -- 915 * all but the first len0 bytes. In case of failure, it returns NULL and 916 * attempts to restore the chain to its original state. 917 */ 918 struct mbuf * 919 m_split(struct mbuf *m0, int len0, int wait) 920 { 921 922 return m_split0(m0, len0, wait, 1); 923 } 924 925 static struct mbuf * 926 m_split0(struct mbuf *m0, int len0, int wait, int copyhdr) 927 { 928 struct mbuf *m, *n; 929 unsigned len = len0, remain, len_save; 930 931 for (m = m0; m && len > m->m_len; m = m->m_next) 932 len -= m->m_len; 933 if (m == 0) 934 return (NULL); 935 remain = m->m_len - len; 936 if (copyhdr && (m0->m_flags & M_PKTHDR)) { 937 MGETHDR(n, wait, m0->m_type); 938 if (n == 0) 939 return (NULL); 940 MCLAIM(m, m0->m_owner); 941 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 942 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 943 len_save = m0->m_pkthdr.len; 944 m0->m_pkthdr.len = len0; 945 if (m->m_flags & M_EXT) 946 goto extpacket; 947 if (remain > MHLEN) { 948 /* m can't be the lead packet */ 949 MH_ALIGN(n, 0); 950 n->m_next = m_split(m, len, wait); 951 if (n->m_next == 0) { 952 (void) m_free(n); 953 m0->m_pkthdr.len = len_save; 954 return (NULL); 955 } else 956 return (n); 957 } else 958 MH_ALIGN(n, remain); 959 } else if (remain == 0) { 960 n = m->m_next; 961 m->m_next = 0; 962 return (n); 963 } else { 964 MGET(n, wait, m->m_type); 965 if (n == 0) 966 return (NULL); 967 MCLAIM(n, m->m_owner); 968 M_ALIGN(n, remain); 969 } 970 extpacket: 971 if (m->m_flags & M_EXT) { 972 n->m_ext = m->m_ext; 973 MCLADDREFERENCE(m, n); 974 n->m_data = m->m_data + len; 975 } else { 976 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + len, remain); 977 } 978 n->m_len = remain; 979 m->m_len = len; 980 n->m_next = m->m_next; 981 m->m_next = 0; 982 return (n); 983 } 984 /* 985 * Routine to copy from device local memory into mbufs. 986 */ 987 struct mbuf * 988 m_devget(char *buf, int totlen, int off0, struct ifnet *ifp, 989 void (*copy)(const void *from, void *to, size_t len)) 990 { 991 struct mbuf *m; 992 struct mbuf *top = 0, **mp = ⊤ 993 int off = off0, len; 994 char *cp; 995 char *epkt; 996 997 cp = buf; 998 epkt = cp + totlen; 999 if (off) { 1000 /* 1001 * If 'off' is non-zero, packet is trailer-encapsulated, 1002 * so we have to skip the type and length fields. 1003 */ 1004 cp += off + 2 * sizeof(u_int16_t); 1005 totlen -= 2 * sizeof(u_int16_t); 1006 } 1007 MGETHDR(m, M_DONTWAIT, MT_DATA); 1008 if (m == 0) 1009 return (NULL); 1010 m->m_pkthdr.rcvif = ifp; 1011 m->m_pkthdr.len = totlen; 1012 m->m_len = MHLEN; 1013 1014 while (totlen > 0) { 1015 if (top) { 1016 MGET(m, M_DONTWAIT, MT_DATA); 1017 if (m == 0) { 1018 m_freem(top); 1019 return (NULL); 1020 } 1021 m->m_len = MLEN; 1022 } 1023 len = min(totlen, epkt - cp); 1024 if (len >= MINCLSIZE) { 1025 MCLGET(m, M_DONTWAIT); 1026 if ((m->m_flags & M_EXT) == 0) { 1027 m_free(m); 1028 m_freem(top); 1029 return (NULL); 1030 } 1031 m->m_len = len = min(len, MCLBYTES); 1032 } else { 1033 /* 1034 * Place initial small packet/header at end of mbuf. 1035 */ 1036 if (len < m->m_len) { 1037 if (top == 0 && len + max_linkhdr <= m->m_len) 1038 m->m_data += max_linkhdr; 1039 m->m_len = len; 1040 } else 1041 len = m->m_len; 1042 } 1043 if (copy) 1044 copy(cp, mtod(m, caddr_t), (size_t)len); 1045 else 1046 memcpy(mtod(m, caddr_t), cp, (size_t)len); 1047 cp += len; 1048 *mp = m; 1049 mp = &m->m_next; 1050 totlen -= len; 1051 if (cp == epkt) 1052 cp = buf; 1053 } 1054 return (top); 1055 } 1056 1057 /* 1058 * Copy data from a buffer back into the indicated mbuf chain, 1059 * starting "off" bytes from the beginning, extending the mbuf 1060 * chain if necessary. 1061 */ 1062 void 1063 m_copyback(struct mbuf *m0, int off, int len, caddr_t cp) 1064 { 1065 #if defined(DEBUG) 1066 struct mbuf *origm = m0; 1067 int error; 1068 #endif /* defined(DEBUG) */ 1069 1070 if (m0 == NULL) 1071 return; 1072 1073 #if defined(DEBUG) 1074 error = 1075 #endif /* defined(DEBUG) */ 1076 m_copyback0(&m0, off, len, cp, 1077 M_COPYBACK0_COPYBACK|M_COPYBACK0_EXTEND, M_DONTWAIT); 1078 1079 #if defined(DEBUG) 1080 if (error != 0 || (m0 != NULL && origm != m0)) 1081 panic("m_copyback"); 1082 #endif /* defined(DEBUG) */ 1083 } 1084 1085 struct mbuf * 1086 m_copyback_cow(struct mbuf *m0, int off, int len, caddr_t cp, int how) 1087 { 1088 int error; 1089 1090 /* don't support chain expansion */ 1091 KDASSERT(off + len <= m_length(m0)); 1092 1093 error = m_copyback0(&m0, off, len, cp, 1094 M_COPYBACK0_COPYBACK|M_COPYBACK0_COW, how); 1095 if (error) { 1096 /* 1097 * no way to recover from partial success. 1098 * just free the chain. 1099 */ 1100 m_freem(m0); 1101 return NULL; 1102 } 1103 return m0; 1104 } 1105 1106 /* 1107 * m_makewritable: ensure the specified range writable. 1108 */ 1109 int 1110 m_makewritable(struct mbuf **mp, int off, int len, int how) 1111 { 1112 int error; 1113 #if defined(DEBUG) 1114 struct mbuf *n; 1115 int origlen, reslen; 1116 1117 origlen = m_length(*mp); 1118 #endif /* defined(DEBUG) */ 1119 1120 #if 0 /* M_COPYALL is large enough */ 1121 if (len == M_COPYALL) 1122 len = m_length(*mp) - off; /* XXX */ 1123 #endif 1124 1125 error = m_copyback0(mp, off, len, NULL, 1126 M_COPYBACK0_PRESERVE|M_COPYBACK0_COW, how); 1127 1128 #if defined(DEBUG) 1129 reslen = 0; 1130 for (n = *mp; n; n = n->m_next) 1131 reslen += n->m_len; 1132 if (origlen != reslen) 1133 panic("m_makewritable: length changed"); 1134 if (((*mp)->m_flags & M_PKTHDR) != 0 && reslen != (*mp)->m_pkthdr.len) 1135 panic("m_makewritable: inconsist"); 1136 #endif /* defined(DEBUG) */ 1137 1138 return error; 1139 } 1140 1141 int 1142 m_copyback0(struct mbuf **mp0, int off, int len, caddr_t cp, int flags, int how) 1143 { 1144 int mlen; 1145 struct mbuf *m, *n; 1146 struct mbuf **mp; 1147 int totlen = 0; 1148 1149 KASSERT(mp0 != NULL); 1150 KASSERT(*mp0 != NULL); 1151 KASSERT((flags & M_COPYBACK0_PRESERVE) == 0 || cp == NULL); 1152 KASSERT((flags & M_COPYBACK0_COPYBACK) == 0 || cp != NULL); 1153 1154 mp = mp0; 1155 m = *mp; 1156 while (off > (mlen = m->m_len)) { 1157 off -= mlen; 1158 totlen += mlen; 1159 if (m->m_next == 0) { 1160 if ((flags & M_COPYBACK0_EXTEND) == 0) 1161 goto out; 1162 n = m_getclr(how, m->m_type); 1163 if (n == 0) 1164 goto out; 1165 n->m_len = min(MLEN, len + off); 1166 m->m_next = n; 1167 } 1168 mp = &m->m_next; 1169 m = m->m_next; 1170 } 1171 while (len > 0) { 1172 mlen = m->m_len - off; 1173 if (mlen != 0 && M_READONLY(m)) { 1174 char *datap; 1175 int eatlen; 1176 1177 /* 1178 * this mbuf is read-only. 1179 * allocate a new writable mbuf and try again. 1180 */ 1181 1182 #if defined(DIAGNOSTIC) 1183 if ((flags & M_COPYBACK0_COW) == 0) 1184 panic("m_copyback0: read-only"); 1185 #endif /* defined(DIAGNOSTIC) */ 1186 1187 /* 1188 * if we're going to write into the middle of 1189 * a mbuf, split it first. 1190 */ 1191 if (off > 0 && len < mlen) { 1192 n = m_split0(m, off, how, 0); 1193 if (n == NULL) 1194 goto enobufs; 1195 m->m_next = n; 1196 mp = &m->m_next; 1197 m = n; 1198 off = 0; 1199 continue; 1200 } 1201 1202 /* 1203 * XXX TODO coalesce into the trailingspace of 1204 * the previous mbuf when possible. 1205 */ 1206 1207 /* 1208 * allocate a new mbuf. copy packet header if needed. 1209 */ 1210 MGET(n, how, m->m_type); 1211 if (n == NULL) 1212 goto enobufs; 1213 MCLAIM(n, m->m_owner); 1214 if (off == 0 && (m->m_flags & M_PKTHDR) != 0) { 1215 /* XXX M_MOVE_PKTHDR */ 1216 M_COPY_PKTHDR(n, m); 1217 n->m_len = MHLEN; 1218 } else { 1219 if (len >= MINCLSIZE) 1220 MCLGET(n, M_DONTWAIT); 1221 n->m_len = 1222 (n->m_flags & M_EXT) ? MCLBYTES : MLEN; 1223 } 1224 if (n->m_len > len) 1225 n->m_len = len; 1226 1227 /* 1228 * free the region which has been overwritten. 1229 * copying data from old mbufs if requested. 1230 */ 1231 if (flags & M_COPYBACK0_PRESERVE) 1232 datap = mtod(n, char *); 1233 else 1234 datap = NULL; 1235 eatlen = n->m_len; 1236 KDASSERT(off == 0 || eatlen >= mlen); 1237 if (off > 0) { 1238 KDASSERT(len >= mlen); 1239 m->m_len = off; 1240 m->m_next = n; 1241 if (datap) { 1242 m_copydata(m, off, mlen, datap); 1243 datap += mlen; 1244 } 1245 eatlen -= mlen; 1246 mp = &m->m_next; 1247 m = m->m_next; 1248 } 1249 while (m != NULL && M_READONLY(m) && 1250 n->m_type == m->m_type && eatlen > 0) { 1251 mlen = min(eatlen, m->m_len); 1252 if (datap) { 1253 m_copydata(m, 0, mlen, datap); 1254 datap += mlen; 1255 } 1256 m->m_data += mlen; 1257 m->m_len -= mlen; 1258 eatlen -= mlen; 1259 if (m->m_len == 0) 1260 *mp = m = m_free(m); 1261 } 1262 if (eatlen > 0) 1263 n->m_len -= eatlen; 1264 n->m_next = m; 1265 *mp = m = n; 1266 continue; 1267 } 1268 mlen = min(mlen, len); 1269 if (flags & M_COPYBACK0_COPYBACK) { 1270 memcpy(mtod(m, caddr_t) + off, cp, (unsigned)mlen); 1271 cp += mlen; 1272 } 1273 len -= mlen; 1274 mlen += off; 1275 off = 0; 1276 totlen += mlen; 1277 if (len == 0) 1278 break; 1279 if (m->m_next == 0) { 1280 if ((flags & M_COPYBACK0_EXTEND) == 0) 1281 goto out; 1282 n = m_get(how, m->m_type); 1283 if (n == 0) 1284 break; 1285 n->m_len = min(MLEN, len); 1286 m->m_next = n; 1287 } 1288 mp = &m->m_next; 1289 m = m->m_next; 1290 } 1291 out: if (((m = *mp0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 1292 m->m_pkthdr.len = totlen; 1293 1294 return 0; 1295 1296 enobufs: 1297 return ENOBUFS; 1298 } 1299 1300 /* 1301 * Apply function f to the data in an mbuf chain starting "off" bytes from the 1302 * beginning, continuing for "len" bytes. 1303 */ 1304 int 1305 m_apply(struct mbuf *m, int off, int len, 1306 int (*f)(void *, caddr_t, unsigned int), void *arg) 1307 { 1308 unsigned int count; 1309 int rval; 1310 1311 KASSERT(len >= 0); 1312 KASSERT(off >= 0); 1313 1314 while (off > 0) { 1315 KASSERT(m != NULL); 1316 if (off < m->m_len) 1317 break; 1318 off -= m->m_len; 1319 m = m->m_next; 1320 } 1321 while (len > 0) { 1322 KASSERT(m != NULL); 1323 count = min(m->m_len - off, len); 1324 1325 rval = (*f)(arg, mtod(m, caddr_t) + off, count); 1326 if (rval) 1327 return (rval); 1328 1329 len -= count; 1330 off = 0; 1331 m = m->m_next; 1332 } 1333 1334 return (0); 1335 } 1336 1337 /* 1338 * Return a pointer to mbuf/offset of location in mbuf chain. 1339 */ 1340 struct mbuf * 1341 m_getptr(struct mbuf *m, int loc, int *off) 1342 { 1343 1344 while (loc >= 0) { 1345 /* Normal end of search */ 1346 if (m->m_len > loc) { 1347 *off = loc; 1348 return (m); 1349 } else { 1350 loc -= m->m_len; 1351 1352 if (m->m_next == NULL) { 1353 if (loc == 0) { 1354 /* Point at the end of valid data */ 1355 *off = m->m_len; 1356 return (m); 1357 } else 1358 return (NULL); 1359 } else 1360 m = m->m_next; 1361 } 1362 } 1363 1364 return (NULL); 1365 }
Cache object: 678bdf1912acfeb0ac440edc30c85a0d
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]