Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/bsd/kern/uipc_mbuf.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* 2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_LICENSE_HEADER_START@ 5 * 6 * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved. 7 * 8 * This file contains Original Code and/or Modifications of Original Code 9 * as defined in and that are subject to the Apple Public Source License 10 * Version 2.0 (the 'License'). You may not use this file except in 11 * compliance with the License. Please obtain a copy of the License at 12 * http://www.opensource.apple.com/apsl/ and read it before using this 13 * file. 14 * 15 * The Original Code and all software distributed under the License are 16 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 17 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 18 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 20 * Please see the License for the specific language governing rights and 21 * limitations under the License. 22 * 23 * @APPLE_LICENSE_HEADER_END@ 24 */ 25 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */ 26 /* 27 * Copyright (c) 1982, 1986, 1988, 1991, 1993 28 * The Regents of the University of California. All rights reserved. 29 * 30 * Redistribution and use in source and binary forms, with or without 31 * modification, are permitted provided that the following conditions 32 * are met: 33 * 1. Redistributions of source code must retain the above copyright 34 * notice, this list of conditions and the following disclaimer. 35 * 2. Redistributions in binary form must reproduce the above copyright 36 * notice, this list of conditions and the following disclaimer in the 37 * documentation and/or other materials provided with the distribution. 38 * 3. All advertising materials mentioning features or use of this software 39 * must display the following acknowledgement: 40 * This product includes software developed by the University of 41 * California, Berkeley and its contributors. 42 * 4. Neither the name of the University nor the names of its contributors 43 * may be used to endorse or promote products derived from this software 44 * without specific prior written permission. 45 * 46 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 47 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 48 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 49 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 50 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 51 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 52 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 53 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 54 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 55 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 56 * SUCH DAMAGE. 57 * 58 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94 59 */ 60 /* HISTORY 61 * 62 * 10/15/97 Annette DeSchon (deschon@apple.com) 63 * Fixed bug in which all cluster mbufs were broken up 64 * into regular mbufs: Some clusters are now reserved. 65 * When a cluster is needed, regular mbufs are no longer 66 * used. (Radar 1683621) 67 * 20-May-95 Mac Gillon (mgillon) at NeXT 68 * New version based on 4.4 69 */ 70 71 #include <sys/param.h> 72 #include <sys/systm.h> 73 #include <sys/malloc.h> 74 #include <sys/mbuf.h> 75 #include <sys/kernel.h> 76 #include <sys/syslog.h> 77 #include <sys/protosw.h> 78 #include <sys/domain.h> 79 #include <net/netisr.h> 80 81 #include <kern/queue.h> 82 #include <kern/kern_types.h> 83 #include <kern/sched_prim.h> 84 85 #include <IOKit/IOMapper.h> 86 87 #define _MCLREF(p) (++mclrefcnt[mtocl(p)]) 88 #define _MCLUNREF(p) (--mclrefcnt[mtocl(p)] == 0) 89 #define _M_CLEAR_PKTHDR(mbuf_ptr) (mbuf_ptr)->m_pkthdr.rcvif = NULL; \ 90 (mbuf_ptr)->m_pkthdr.len = 0; \ 91 (mbuf_ptr)->m_pkthdr.header = NULL; \ 92 (mbuf_ptr)->m_pkthdr.csum_flags = 0; \ 93 (mbuf_ptr)->m_pkthdr.csum_data = 0; \ 94 (mbuf_ptr)->m_pkthdr.aux = (struct mbuf*)NULL; \ 95 (mbuf_ptr)->m_pkthdr.reserved1 = NULL; \ 96 (mbuf_ptr)->m_pkthdr.reserved2 = NULL; 97 98 extern pmap_t kernel_pmap; /* The kernel's pmap */ 99 /* kernel translater */ 100 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); 101 102 decl_simple_lock_data(, mbuf_slock); 103 struct mbuf *mfree; /* mbuf free list */ 104 struct mbuf *mfreelater; /* mbuf deallocation list */ 105 extern vm_map_t mb_map; /* special map */ 106 int m_want; /* sleepers on mbufs */ 107 extern int nmbclusters; /* max number of mapped clusters */ 108 short *mclrefcnt; /* mapped cluster reference counts */ 109 int *mcl_paddr; 110 static ppnum_t mcl_paddr_base; /* Handle returned by IOMapper::iovmAlloc() */ 111 union mcluster *mclfree; /* mapped cluster free list */ 112 int max_linkhdr; /* largest link-level header */ 113 int max_protohdr; /* largest protocol header */ 114 int max_hdr; /* largest link+protocol header */ 115 int max_datalen; /* MHLEN - max_hdr */ 116 struct mbstat mbstat; /* statistics */ 117 union mcluster *mbutl; /* first mapped cluster address */ 118 union mcluster *embutl; /* ending virtual address of mclusters */ 119 120 static int nclpp; /* # clusters per physical page */ 121 static char mbfail[] = "mbuf not mapped"; 122 123 static int m_howmany(); 124 125 /* The number of cluster mbufs that are allocated, to start. */ 126 #define MINCL max(16, 2) 127 128 static int mbuf_expand_thread_wakeup = 0; 129 static int mbuf_expand_mcl = 0; 130 static int mbuf_expand_thread_initialized = 0; 131 132 static void mbuf_expand_thread_init(void); 133 134 #if 0 135 static int mfree_munge = 0; 136 #if 0 137 #define _MFREE_MUNGE(m) { \ 138 if (mfree_munge) \ 139 { int i; \ 140 vm_offset_t *element = (vm_offset_t *)(m); \ 141 for (i = 0; \ 142 i < sizeof(struct mbuf)/sizeof(vm_offset_t); \ 143 i++) \ 144 (element)[i] = 0xdeadbeef; \ 145 } \ 146 } 147 #else 148 void 149 munge_mbuf(struct mbuf *m) 150 { 151 int i; 152 vm_offset_t *element = (vm_offset_t *)(m); 153 for (i = 0; 154 i < sizeof(struct mbuf)/sizeof(vm_offset_t); 155 i++) 156 (element)[i] = 0xdeadbeef; 157 } 158 #define _MFREE_MUNGE(m) { \ 159 if (mfree_munge) \ 160 munge_mbuf(m); \ 161 } 162 #endif 163 #else 164 #define _MFREE_MUNGE(m) 165 #endif 166 167 168 #define _MINTGET(m, type) { \ 169 MBUF_LOCK(); \ 170 if (((m) = mfree) != 0) { \ 171 MCHECK(m); \ 172 ++mclrefcnt[mtocl(m)]; \ 173 mbstat.m_mtypes[MT_FREE]--; \ 174 mbstat.m_mtypes[(type)]++; \ 175 mfree = (m)->m_next; \ 176 } \ 177 MBUF_UNLOCK(); \ 178 } 179 180 181 void 182 mbinit() 183 { 184 int s,m; 185 int initmcl = 32; 186 int mcl_pages; 187 188 if (nclpp) 189 return; 190 nclpp = round_page_32(MCLBYTES) / MCLBYTES; /* see mbufgc() */ 191 if (nclpp < 1) nclpp = 1; 192 MBUF_LOCKINIT(); 193 // NETISR_LOCKINIT(); 194 195 mbstat.m_msize = MSIZE; 196 mbstat.m_mclbytes = MCLBYTES; 197 mbstat.m_minclsize = MINCLSIZE; 198 mbstat.m_mlen = MLEN; 199 mbstat.m_mhlen = MHLEN; 200 201 if (nmbclusters == 0) 202 nmbclusters = NMBCLUSTERS; 203 MALLOC(mclrefcnt, short *, nmbclusters * sizeof (short), 204 M_TEMP, M_WAITOK); 205 if (mclrefcnt == 0) 206 panic("mbinit"); 207 for (m = 0; m < nmbclusters; m++) 208 mclrefcnt[m] = -1; 209 210 /* Calculate the number of pages assigned to the cluster pool */ 211 mcl_pages = nmbclusters/(PAGE_SIZE/CLBYTES); 212 MALLOC(mcl_paddr, int *, mcl_pages * sizeof(int), M_TEMP, M_WAITOK); 213 if (mcl_paddr == 0) 214 panic("mbinit1"); 215 /* Register with the I/O Bus mapper */ 216 mcl_paddr_base = IOMapperIOVMAlloc(mcl_pages); 217 bzero((char *)mcl_paddr, mcl_pages * sizeof(int)); 218 219 embutl = (union mcluster *)((unsigned char *)mbutl + (nmbclusters * MCLBYTES)); 220 221 PE_parse_boot_arg("initmcl", &initmcl); 222 223 if (m_clalloc(max(PAGE_SIZE/CLBYTES, 1) * initmcl, M_WAIT) == 0) 224 goto bad; 225 MBUF_UNLOCK(); 226 227 (void) kernel_thread(kernel_task, mbuf_expand_thread_init); 228 229 return; 230 bad: 231 panic("mbinit"); 232 } 233 234 /* 235 * Allocate some number of mbuf clusters 236 * and place on cluster free list. 237 */ 238 /* ARGSUSED */ 239 m_clalloc(ncl, nowait) 240 register int ncl; 241 int nowait; 242 { 243 register union mcluster *mcl; 244 register int i; 245 vm_size_t size; 246 static char doing_alloc; 247 248 /* 249 * Honor the caller's wish to block or not block. 250 * We have a way to grow the pool asynchronously, 251 * by kicking the dlil_input_thread. 252 */ 253 if ((i = m_howmany()) <= 0) 254 goto out; 255 256 if ((nowait == M_DONTWAIT)) 257 goto out; 258 259 if (ncl < i) 260 ncl = i; 261 size = round_page_32(ncl * MCLBYTES); 262 mcl = (union mcluster *)kmem_mb_alloc(mb_map, size); 263 264 if (mcl == 0 && ncl > 1) { 265 size = round_page_32(MCLBYTES); /* Try for 1 if failed */ 266 mcl = (union mcluster *)kmem_mb_alloc(mb_map, size); 267 } 268 269 if (mcl) { 270 MBUF_LOCK(); 271 ncl = size / MCLBYTES; 272 for (i = 0; i < ncl; i++) { 273 if (++mclrefcnt[mtocl(mcl)] != 0) 274 panic("m_clalloc already there"); 275 if (((int)mcl & PAGE_MASK) == 0) { 276 ppnum_t offset = ((char *)mcl - (char *)mbutl)/PAGE_SIZE; 277 ppnum_t new_page = pmap_find_phys(kernel_pmap, (vm_address_t) mcl); 278 279 /* 280 * In the case of no mapper being available 281 * the following code nops and returns the 282 * input page, if there is a mapper the I/O 283 * page appropriate is returned. 284 */ 285 new_page = IOMapperInsertPage(mcl_paddr_base, offset, new_page); 286 mcl_paddr[offset] = new_page << 12; 287 } 288 289 mcl->mcl_next = mclfree; 290 mclfree = mcl++; 291 } 292 mbstat.m_clfree += ncl; 293 mbstat.m_clusters += ncl; 294 return (ncl); 295 } /* else ... */ 296 out: 297 MBUF_LOCK(); 298 299 /* 300 * When non-blocking we kick the dlil thread if we havve to grow the 301 * pool or if the number of free clusters is less than requested. 302 */ 303 if ((nowait == M_DONTWAIT) && (i > 0 || ncl >= mbstat.m_clfree)) { 304 mbuf_expand_mcl = 1; 305 if (mbuf_expand_thread_initialized) 306 wakeup((caddr_t)&mbuf_expand_thread_wakeup); 307 } 308 309 if (mbstat.m_clfree >= ncl) 310 return 1; 311 312 return 0; 313 } 314 315 /* 316 * Add more free mbufs by cutting up a cluster. 317 */ 318 m_expand(canwait) 319 int canwait; 320 { 321 register caddr_t mcl; 322 323 if (mbstat.m_clfree < (mbstat.m_clusters >> 4)) 324 /* 1/16th of the total number of cluster mbufs allocated is 325 reserved for large packets. The number reserved must 326 always be < 1/2, or future allocation will be prevented. 327 */ 328 return 0; 329 330 MCLALLOC(mcl, canwait); 331 if (mcl) { 332 register struct mbuf *m = (struct mbuf *)mcl; 333 register int i = NMBPCL; 334 MBUF_LOCK(); 335 mbstat.m_mtypes[MT_FREE] += i; 336 mbstat.m_mbufs += i; 337 while (i--) { 338 _MFREE_MUNGE(m); 339 m->m_type = MT_FREE; 340 m->m_next = mfree; 341 mfree = m++; 342 } 343 i = m_want; 344 m_want = 0; 345 MBUF_UNLOCK(); 346 if (i) wakeup((caddr_t)&mfree); 347 return 1; 348 } 349 return 0; 350 } 351 352 /* 353 * When MGET failes, ask protocols to free space when short of memory, 354 * then re-attempt to allocate an mbuf. 355 */ 356 struct mbuf * 357 m_retry(canwait, type) 358 int canwait, type; 359 { 360 register struct mbuf *m; 361 int wait, s; 362 funnel_t * fnl; 363 int fnl_switch = 0; 364 boolean_t funnel_state; 365 366 for (;;) { 367 (void) m_expand(canwait); 368 _MINTGET(m, type); 369 if (m) { 370 (m)->m_next = (m)->m_nextpkt = 0; 371 (m)->m_type = (type); 372 (m)->m_data = (m)->m_dat; 373 (m)->m_flags = 0; 374 } 375 if (m || canwait == M_DONTWAIT) 376 break; 377 MBUF_LOCK(); 378 wait = m_want++; 379 mbuf_expand_mcl = 1; 380 if (wait == 0) 381 mbstat.m_drain++; 382 else 383 mbstat.m_wait++; 384 MBUF_UNLOCK(); 385 386 if (mbuf_expand_thread_initialized) 387 wakeup((caddr_t)&mbuf_expand_thread_wakeup); 388 389 /* 390 * Need to be inside network funnel for m_reclaim because it calls into the 391 * socket domains and tsleep end-up calling splhigh 392 */ 393 fnl = thread_funnel_get(); 394 if (wait == 0 && fnl == network_flock) { 395 m_reclaim(); 396 } else if (fnl != THR_FUNNEL_NULL) { 397 /* Sleep with a small timeout as insurance */ 398 (void) tsleep((caddr_t)&mfree, PZERO-1, "m_retry", hz); 399 } else { 400 /* We are called from a non-BSD context: use mach primitives */ 401 u_int64_t abstime = 0; 402 403 assert_wait((event_t)&mfree, THREAD_UNINT); 404 clock_interval_to_deadline(hz, NSEC_PER_SEC / hz, &abstime); 405 thread_set_timer_deadline(abstime); 406 if (thread_block(THREAD_CONTINUE_NULL) != THREAD_TIMED_OUT) 407 thread_cancel_timer(); 408 } 409 } 410 if (m == 0) 411 mbstat.m_drops++; 412 return (m); 413 } 414 415 /* 416 * As above; retry an MGETHDR. 417 */ 418 struct mbuf * 419 m_retryhdr(canwait, type) 420 int canwait, type; 421 { 422 register struct mbuf *m; 423 424 if (m = m_retry(canwait, type)) { 425 m->m_flags |= M_PKTHDR; 426 m->m_data = m->m_pktdat; 427 _M_CLEAR_PKTHDR(m); 428 } 429 return (m); 430 } 431 432 m_reclaim() 433 { 434 register struct domain *dp; 435 register struct protosw *pr; 436 437 for (dp = domains; dp; dp = dp->dom_next) 438 for (pr = dp->dom_protosw; pr; pr = pr->pr_next) 439 if (pr->pr_drain) 440 (*pr->pr_drain)(); 441 mbstat.m_drain++; 442 } 443 444 /* 445 * Space allocation routines. 446 * These are also available as macros 447 * for critical paths. 448 */ 449 struct mbuf * 450 m_get(nowait, type) 451 int nowait, type; 452 { 453 register struct mbuf *m; 454 455 _MINTGET(m, type); 456 if (m) { 457 m->m_next = m->m_nextpkt = 0; 458 m->m_type = type; 459 m->m_data = m->m_dat; 460 m->m_flags = 0; 461 } else 462 (m) = m_retry(nowait, type); 463 464 return (m); 465 } 466 467 struct mbuf * 468 m_gethdr(nowait, type) 469 int nowait, type; 470 { 471 register struct mbuf *m; 472 473 _MINTGET(m, type); 474 if (m) { 475 m->m_next = m->m_nextpkt = 0; 476 m->m_type = type; 477 m->m_data = m->m_pktdat; 478 m->m_flags = M_PKTHDR; 479 _M_CLEAR_PKTHDR(m) 480 } else 481 m = m_retryhdr(nowait, type); 482 483 return m; 484 } 485 486 struct mbuf * 487 m_getclr(nowait, type) 488 int nowait, type; 489 { 490 register struct mbuf *m; 491 492 MGET(m, nowait, type); 493 if (m == 0) 494 return (0); 495 bzero(mtod(m, caddr_t), MLEN); 496 return (m); 497 } 498 499 struct mbuf * 500 m_free(m) 501 struct mbuf *m; 502 { 503 struct mbuf *n = m->m_next; 504 int i, s; 505 506 if (m->m_type == MT_FREE) 507 panic("freeing free mbuf"); 508 509 /* Free the aux data if there is any */ 510 if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.aux) 511 { 512 m_freem(m->m_pkthdr.aux); 513 } 514 515 MBUF_LOCK(); 516 if ((m->m_flags & M_EXT)) 517 { 518 if (MCLHASREFERENCE(m)) { 519 remque((queue_t)&m->m_ext.ext_refs); 520 } else if (m->m_ext.ext_free == NULL) { 521 union mcluster *mcl= (union mcluster *)m->m_ext.ext_buf; 522 if (_MCLUNREF(mcl)) { 523 mcl->mcl_next = mclfree; 524 mclfree = mcl; 525 ++mbstat.m_clfree; 526 } 527 #ifdef COMMENT_OUT 528 /* *** Since m_split() increments "mclrefcnt[mtocl(m->m_ext.ext_buf)]", 529 and AppleTalk ADSP uses m_split(), this incorrect sanity check 530 caused a panic. 531 *** */ 532 else /* sanity check - not referenced this way */ 533 panic("m_free m_ext cluster not free"); 534 #endif 535 } else { 536 (*(m->m_ext.ext_free))(m->m_ext.ext_buf, 537 m->m_ext.ext_size, m->m_ext.ext_arg); 538 } 539 } 540 mbstat.m_mtypes[m->m_type]--; 541 (void) _MCLUNREF(m); 542 _MFREE_MUNGE(m); 543 m->m_type = MT_FREE; 544 mbstat.m_mtypes[m->m_type]++; 545 m->m_flags = 0; 546 m->m_next = mfree; 547 m->m_len = 0; 548 mfree = m; 549 i = m_want; 550 m_want = 0; 551 MBUF_UNLOCK(); 552 if (i) wakeup((caddr_t)&mfree); 553 return (n); 554 } 555 556 /* m_mclget() add an mbuf cluster to a normal mbuf */ 557 struct mbuf * 558 m_mclget(m, nowait) 559 struct mbuf *m; 560 int nowait; 561 { 562 MCLALLOC(m->m_ext.ext_buf, nowait); 563 if (m->m_ext.ext_buf) { 564 m->m_data = m->m_ext.ext_buf; 565 m->m_flags |= M_EXT; 566 m->m_ext.ext_size = MCLBYTES; 567 m->m_ext.ext_free = 0; 568 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward = 569 &m->m_ext.ext_refs; 570 } 571 572 return m; 573 } 574 575 /* m_mclalloc() allocate an mbuf cluster */ 576 caddr_t 577 m_mclalloc( nowait) 578 int nowait; 579 { 580 caddr_t p; 581 582 (void)m_clalloc(1, nowait); 583 if ((p = (caddr_t)mclfree)) { 584 ++mclrefcnt[mtocl(p)]; 585 mbstat.m_clfree--; 586 mclfree = ((union mcluster *)p)->mcl_next; 587 } else { 588 mbstat.m_drops++; 589 } 590 MBUF_UNLOCK(); 591 592 return p; 593 } 594 595 /* m_mclfree() releases a reference to a cluster allocated by MCLALLOC, 596 * freeing the cluster if the reference count has reached 0. */ 597 void 598 m_mclfree(p) 599 caddr_t p; 600 { 601 MBUF_LOCK(); 602 if (--mclrefcnt[mtocl(p)] == 0) { 603 ((union mcluster *)(p))->mcl_next = mclfree; 604 mclfree = (union mcluster *)(p); 605 mbstat.m_clfree++; 606 } 607 MBUF_UNLOCK(); 608 } 609 610 /* mcl_hasreference() checks if a cluster of an mbuf is referenced by another mbuf */ 611 int 612 m_mclhasreference(m) 613 struct mbuf *m; 614 { 615 return (m->m_ext.ext_refs.forward != &(m->m_ext.ext_refs)); 616 } 617 618 /* */ 619 void 620 m_copy_pkthdr(to, from) 621 struct mbuf *to, *from; 622 { 623 to->m_pkthdr = from->m_pkthdr; 624 from->m_pkthdr.aux = (struct mbuf *)NULL; 625 to->m_flags = from->m_flags & M_COPYFLAGS; 626 to->m_data = (to)->m_pktdat; 627 } 628 629 /* Best effort to get a mbuf cluster + pkthdr under one lock. 630 * If we don't have them avail, just bail out and use the regular 631 * path. 632 * Used by drivers to allocated packets on receive ring. 633 */ 634 struct mbuf * 635 m_getpacket(void) 636 { 637 struct mbuf *m; 638 m_clalloc(1, M_DONTWAIT); /* takes the MBUF_LOCK, but doesn't release it... */ 639 if ((mfree != 0) && (mclfree != 0)) { /* mbuf + cluster are available */ 640 m = mfree; 641 mfree = m->m_next; 642 MCHECK(m); 643 ++mclrefcnt[mtocl(m)]; 644 mbstat.m_mtypes[MT_FREE]--; 645 mbstat.m_mtypes[MT_DATA]++; 646 m->m_ext.ext_buf = (caddr_t)mclfree; /* get the cluster */ 647 ++mclrefcnt[mtocl(m->m_ext.ext_buf)]; 648 mbstat.m_clfree--; 649 mclfree = ((union mcluster *)(m->m_ext.ext_buf))->mcl_next; 650 651 m->m_next = m->m_nextpkt = 0; 652 m->m_type = MT_DATA; 653 m->m_data = m->m_ext.ext_buf; 654 m->m_flags = M_PKTHDR | M_EXT; 655 _M_CLEAR_PKTHDR(m) 656 m->m_ext.ext_free = 0; 657 m->m_ext.ext_size = MCLBYTES; 658 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward = 659 &m->m_ext.ext_refs; 660 MBUF_UNLOCK(); 661 } 662 else { /* slow path: either mbuf or cluster need to be allocated anyway */ 663 MBUF_UNLOCK(); 664 665 MGETHDR(m, M_WAITOK, MT_DATA ); 666 667 if ( m == 0 ) 668 return (NULL); 669 670 MCLGET( m, M_WAITOK ); 671 if ( ( m->m_flags & M_EXT ) == 0 ) 672 { 673 m_free(m); m = 0; 674 } 675 } 676 return (m); 677 } 678 679 680 /* 681 * return a list of mbuf hdrs that point to clusters... 682 * try for num_needed, if this can't be met, return whatever 683 * number were available... set up the first num_with_pkthdrs 684 * with mbuf hdrs configured as packet headers... these are 685 * chained on the m_nextpkt field... any packets requested beyond 686 * this are chained onto the last packet header's m_next field. 687 */ 688 struct mbuf * 689 m_getpackets(int num_needed, int num_with_pkthdrs, int how) 690 { 691 struct mbuf *m; 692 struct mbuf **np, *top; 693 694 top = NULL; 695 np = ⊤ 696 697 m_clalloc(num_needed, how); /* takes the MBUF_LOCK, but doesn't release it... */ 698 699 while (num_needed--) { 700 if (mfree && mclfree) { /* mbuf + cluster are available */ 701 m = mfree; 702 MCHECK(m); 703 mfree = m->m_next; 704 ++mclrefcnt[mtocl(m)]; 705 mbstat.m_mtypes[MT_FREE]--; 706 mbstat.m_mtypes[MT_DATA]++; 707 m->m_ext.ext_buf = (caddr_t)mclfree; /* get the cluster */ 708 ++mclrefcnt[mtocl(m->m_ext.ext_buf)]; 709 mbstat.m_clfree--; 710 mclfree = ((union mcluster *)(m->m_ext.ext_buf))->mcl_next; 711 712 m->m_next = m->m_nextpkt = 0; 713 m->m_type = MT_DATA; 714 m->m_data = m->m_ext.ext_buf; 715 m->m_ext.ext_free = 0; 716 m->m_ext.ext_size = MCLBYTES; 717 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward = &m->m_ext.ext_refs; 718 719 if (num_with_pkthdrs == 0) 720 m->m_flags = M_EXT; 721 else { 722 m->m_flags = M_PKTHDR | M_EXT; 723 _M_CLEAR_PKTHDR(m); 724 725 num_with_pkthdrs--; 726 } 727 728 } else { 729 730 MBUF_UNLOCK(); 731 732 if (num_with_pkthdrs == 0) { 733 MGET(m, how, MT_DATA ); 734 } else { 735 MGETHDR(m, how, MT_DATA); 736 737 num_with_pkthdrs--; 738 } 739 if (m == 0) 740 return(top); 741 742 MCLGET(m, how); 743 if ((m->m_flags & M_EXT) == 0) { 744 m_free(m); 745 return(top); 746 } 747 MBUF_LOCK(); 748 } 749 *np = m; 750 751 if (num_with_pkthdrs) 752 np = &m->m_nextpkt; 753 else 754 np = &m->m_next; 755 } 756 MBUF_UNLOCK(); 757 758 return (top); 759 } 760 761 762 /* 763 * return a list of mbuf hdrs set up as packet hdrs 764 * chained together on the m_nextpkt field 765 */ 766 struct mbuf * 767 m_getpackethdrs(int num_needed, int how) 768 { 769 struct mbuf *m; 770 struct mbuf **np, *top; 771 772 top = NULL; 773 np = ⊤ 774 775 MBUF_LOCK(); 776 777 while (num_needed--) { 778 if (m = mfree) { /* mbufs are available */ 779 MCHECK(m); 780 mfree = m->m_next; 781 ++mclrefcnt[mtocl(m)]; 782 mbstat.m_mtypes[MT_FREE]--; 783 mbstat.m_mtypes[MT_DATA]++; 784 785 m->m_next = m->m_nextpkt = 0; 786 m->m_type = MT_DATA; 787 m->m_flags = M_PKTHDR; 788 m->m_data = m->m_pktdat; 789 _M_CLEAR_PKTHDR(m); 790 791 } else { 792 793 MBUF_UNLOCK(); 794 795 m = m_retryhdr(how, MT_DATA); 796 797 if (m == 0) 798 return(top); 799 800 MBUF_LOCK(); 801 } 802 *np = m; 803 np = &m->m_nextpkt; 804 } 805 MBUF_UNLOCK(); 806 807 return (top); 808 } 809 810 811 /* free and mbuf list (m_nextpkt) while following m_next under one lock. 812 * returns the count for mbufs packets freed. Used by the drivers. 813 */ 814 int 815 m_freem_list(m) 816 struct mbuf *m; 817 { 818 struct mbuf *nextpkt; 819 int i, count=0; 820 821 MBUF_LOCK(); 822 823 while (m) { 824 if (m) 825 nextpkt = m->m_nextpkt; /* chain of linked mbufs from driver */ 826 else 827 nextpkt = 0; 828 829 count++; 830 831 while (m) { /* free the mbuf chain (like mfreem) */ 832 833 struct mbuf *n; 834 835 /* Free the aux data if there is any */ 836 if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.aux) { 837 /* 838 * Treat the current m as the nextpkt and set m 839 * to the aux data. Preserve nextpkt in m->m_nextpkt. 840 * This lets us free the aux data in this loop 841 * without having to call m_freem recursively, 842 * which wouldn't work because we've still got 843 * the lock. 844 */ 845 m->m_nextpkt = nextpkt; 846 nextpkt = m; 847 m = nextpkt->m_pkthdr.aux; 848 nextpkt->m_pkthdr.aux = NULL; 849 } 850 851 n = m->m_next; 852 853 if (n && n->m_nextpkt) 854 panic("m_freem_list: m_nextpkt of m_next != NULL"); 855 if (m->m_type == MT_FREE) 856 panic("freeing free mbuf"); 857 858 if (m->m_flags & M_EXT) { 859 if (MCLHASREFERENCE(m)) { 860 remque((queue_t)&m->m_ext.ext_refs); 861 } else if (m->m_ext.ext_free == NULL) { 862 union mcluster *mcl= (union mcluster *)m->m_ext.ext_buf; 863 if (_MCLUNREF(mcl)) { 864 mcl->mcl_next = mclfree; 865 mclfree = mcl; 866 ++mbstat.m_clfree; 867 } 868 } else { 869 (*(m->m_ext.ext_free))(m->m_ext.ext_buf, 870 m->m_ext.ext_size, m->m_ext.ext_arg); 871 } 872 } 873 mbstat.m_mtypes[m->m_type]--; 874 (void) _MCLUNREF(m); 875 _MFREE_MUNGE(m); 876 mbstat.m_mtypes[MT_FREE]++; 877 m->m_type = MT_FREE; 878 m->m_flags = 0; 879 m->m_len = 0; 880 m->m_next = mfree; 881 mfree = m; 882 m = n; 883 } 884 m = nextpkt; /* bump m with saved nextpkt if any */ 885 } 886 if (i = m_want) 887 m_want = 0; 888 889 MBUF_UNLOCK(); 890 891 if (i) 892 wakeup((caddr_t)&mfree); 893 894 return (count); 895 } 896 897 void 898 m_freem(m) 899 register struct mbuf *m; 900 { 901 while (m) 902 m = m_free(m); 903 } 904 905 /* 906 * Mbuffer utility routines. 907 */ 908 /* 909 * Compute the amount of space available 910 * before the current start of data in an mbuf. 911 */ 912 m_leadingspace(m) 913 register struct mbuf *m; 914 { 915 if (m->m_flags & M_EXT) { 916 if (MCLHASREFERENCE(m)) 917 return(0); 918 return (m->m_data - m->m_ext.ext_buf); 919 } 920 if (m->m_flags & M_PKTHDR) 921 return (m->m_data - m->m_pktdat); 922 return (m->m_data - m->m_dat); 923 } 924 925 /* 926 * Compute the amount of space available 927 * after the end of data in an mbuf. 928 */ 929 m_trailingspace(m) 930 register struct mbuf *m; 931 { 932 if (m->m_flags & M_EXT) { 933 if (MCLHASREFERENCE(m)) 934 return(0); 935 return (m->m_ext.ext_buf + m->m_ext.ext_size - 936 (m->m_data + m->m_len)); 937 } 938 return (&m->m_dat[MLEN] - (m->m_data + m->m_len)); 939 } 940 941 /* 942 * Lesser-used path for M_PREPEND: 943 * allocate new mbuf to prepend to chain, 944 * copy junk along. 945 * Does not adjust packet header length. 946 */ 947 struct mbuf * 948 m_prepend(m, len, how) 949 register struct mbuf *m; 950 int len, how; 951 { 952 struct mbuf *mn; 953 954 MGET(mn, how, m->m_type); 955 if (mn == (struct mbuf *)NULL) { 956 m_freem(m); 957 return ((struct mbuf *)NULL); 958 } 959 if (m->m_flags & M_PKTHDR) { 960 M_COPY_PKTHDR(mn, m); 961 m->m_flags &= ~M_PKTHDR; 962 } 963 mn->m_next = m; 964 m = mn; 965 if (len < MHLEN) 966 MH_ALIGN(m, len); 967 m->m_len = len; 968 return (m); 969 } 970 971 /* 972 * Replacement for old M_PREPEND macro: 973 * allocate new mbuf to prepend to chain, 974 * copy junk along, and adjust length. 975 * 976 */ 977 struct mbuf * 978 m_prepend_2(m, len, how) 979 register struct mbuf *m; 980 int len, how; 981 { 982 if (M_LEADINGSPACE(m) >= len) { 983 m->m_data -= len; 984 m->m_len += len; 985 } else { 986 m = m_prepend(m, len, how); 987 } 988 if ((m) && (m->m_flags & M_PKTHDR)) 989 m->m_pkthdr.len += len; 990 return (m); 991 } 992 993 /* 994 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 995 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 996 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 997 */ 998 int MCFail; 999 1000 struct mbuf * 1001 m_copym(m, off0, len, wait) 1002 register struct mbuf *m; 1003 int off0, wait; 1004 register int len; 1005 { 1006 register struct mbuf *n, **np; 1007 register int off = off0; 1008 struct mbuf *top; 1009 int copyhdr = 0; 1010 1011 if (off < 0 || len < 0) 1012 panic("m_copym"); 1013 if (off == 0 && m->m_flags & M_PKTHDR) 1014 copyhdr = 1; 1015 1016 while (off >= m->m_len) { 1017 if (m == 0) 1018 panic("m_copym"); 1019 off -= m->m_len; 1020 m = m->m_next; 1021 } 1022 np = ⊤ 1023 top = 0; 1024 1025 MBUF_LOCK(); 1026 1027 while (len > 0) { 1028 if (m == 0) { 1029 if (len != M_COPYALL) 1030 panic("m_copym"); 1031 break; 1032 } 1033 if (n = mfree) { 1034 MCHECK(n); 1035 ++mclrefcnt[mtocl(n)]; 1036 mbstat.m_mtypes[MT_FREE]--; 1037 mbstat.m_mtypes[m->m_type]++; 1038 mfree = n->m_next; 1039 n->m_next = n->m_nextpkt = 0; 1040 n->m_type = m->m_type; 1041 n->m_data = n->m_dat; 1042 n->m_flags = 0; 1043 } else { 1044 MBUF_UNLOCK(); 1045 n = m_retry(wait, m->m_type); 1046 MBUF_LOCK(); 1047 } 1048 *np = n; 1049 1050 if (n == 0) 1051 goto nospace; 1052 if (copyhdr) { 1053 M_COPY_PKTHDR(n, m); 1054 if (len == M_COPYALL) 1055 n->m_pkthdr.len -= off0; 1056 else 1057 n->m_pkthdr.len = len; 1058 copyhdr = 0; 1059 } 1060 if (len == M_COPYALL) { 1061 if (min(len, (m->m_len - off)) == len) { 1062 printf("m->m_len %d - off %d = %d, %d\n", 1063 m->m_len, off, m->m_len - off, 1064 min(len, (m->m_len - off))); 1065 } 1066 } 1067 n->m_len = min(len, (m->m_len - off)); 1068 if (n->m_len == M_COPYALL) { 1069 printf("n->m_len == M_COPYALL, fixing\n"); 1070 n->m_len = MHLEN; 1071 } 1072 if (m->m_flags & M_EXT) { 1073 n->m_ext = m->m_ext; 1074 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs); 1075 n->m_data = m->m_data + off; 1076 n->m_flags |= M_EXT; 1077 } else { 1078 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 1079 (unsigned)n->m_len); 1080 } 1081 if (len != M_COPYALL) 1082 len -= n->m_len; 1083 off = 0; 1084 m = m->m_next; 1085 np = &n->m_next; 1086 } 1087 MBUF_UNLOCK(); 1088 1089 if (top == 0) 1090 MCFail++; 1091 1092 return (top); 1093 nospace: 1094 MBUF_UNLOCK(); 1095 1096 m_freem(top); 1097 MCFail++; 1098 return (0); 1099 } 1100 1101 1102 /* 1103 * equivilent to m_copym except that all necessary 1104 * mbuf hdrs are allocated within this routine 1105 * also, the last mbuf and offset accessed are passed 1106 * out and can be passed back in to avoid having to 1107 * rescan the entire mbuf list (normally hung off of the socket) 1108 */ 1109 struct mbuf * 1110 m_copym_with_hdrs(m, off0, len, wait, m_last, m_off) 1111 register struct mbuf *m; 1112 int off0, wait; 1113 register int len; 1114 struct mbuf **m_last; 1115 int *m_off; 1116 { 1117 register struct mbuf *n, **np; 1118 register int off = off0; 1119 struct mbuf *top = 0; 1120 int copyhdr = 0; 1121 int type; 1122 1123 if (off == 0 && m->m_flags & M_PKTHDR) 1124 copyhdr = 1; 1125 1126 if (*m_last) { 1127 m = *m_last; 1128 off = *m_off; 1129 } else { 1130 while (off >= m->m_len) { 1131 off -= m->m_len; 1132 m = m->m_next; 1133 } 1134 } 1135 MBUF_LOCK(); 1136 1137 while (len > 0) { 1138 if (top == 0) 1139 type = MT_HEADER; 1140 else { 1141 if (m == 0) 1142 panic("m_gethdr_and_copym"); 1143 type = m->m_type; 1144 } 1145 if (n = mfree) { 1146 MCHECK(n); 1147 ++mclrefcnt[mtocl(n)]; 1148 mbstat.m_mtypes[MT_FREE]--; 1149 mbstat.m_mtypes[type]++; 1150 mfree = n->m_next; 1151 n->m_next = n->m_nextpkt = 0; 1152 n->m_type = type; 1153 1154 if (top) { 1155 n->m_data = n->m_dat; 1156 n->m_flags = 0; 1157 } else { 1158 n->m_data = n->m_pktdat; 1159 n->m_flags = M_PKTHDR; 1160 _M_CLEAR_PKTHDR(n); 1161 } 1162 } else { 1163 MBUF_UNLOCK(); 1164 if (top) 1165 n = m_retry(wait, type); 1166 else 1167 n = m_retryhdr(wait, type); 1168 MBUF_LOCK(); 1169 } 1170 if (n == 0) 1171 goto nospace; 1172 if (top == 0) { 1173 top = n; 1174 np = &top->m_next; 1175 continue; 1176 } else 1177 *np = n; 1178 1179 if (copyhdr) { 1180 M_COPY_PKTHDR(n, m); 1181 n->m_pkthdr.len = len; 1182 copyhdr = 0; 1183 } 1184 n->m_len = min(len, (m->m_len - off)); 1185 1186 if (m->m_flags & M_EXT) { 1187 n->m_ext = m->m_ext; 1188 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs); 1189 n->m_data = m->m_data + off; 1190 n->m_flags |= M_EXT; 1191 } else { 1192 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 1193 (unsigned)n->m_len); 1194 } 1195 len -= n->m_len; 1196 1197 if (len == 0) { 1198 if ((off + n->m_len) == m->m_len) { 1199 *m_last = m->m_next; 1200 *m_off = 0; 1201 } else { 1202 *m_last = m; 1203 *m_off = off + n->m_len; 1204 } 1205 break; 1206 } 1207 off = 0; 1208 m = m->m_next; 1209 np = &n->m_next; 1210 } 1211 MBUF_UNLOCK(); 1212 1213 return (top); 1214 nospace: 1215 MBUF_UNLOCK(); 1216 1217 if (top) 1218 m_freem(top); 1219 MCFail++; 1220 return (0); 1221 } 1222 1223 1224 /* 1225 * Copy data from an mbuf chain starting "off" bytes from the beginning, 1226 * continuing for "len" bytes, into the indicated buffer. 1227 */ 1228 void m_copydata(m, off, len, cp) 1229 register struct mbuf *m; 1230 register int off; 1231 register int len; 1232 caddr_t cp; 1233 { 1234 register unsigned count; 1235 1236 if (off < 0 || len < 0) 1237 panic("m_copydata"); 1238 while (off > 0) { 1239 if (m == 0) 1240 panic("m_copydata"); 1241 if (off < m->m_len) 1242 break; 1243 off -= m->m_len; 1244 m = m->m_next; 1245 } 1246 while (len > 0) { 1247 if (m == 0) 1248 panic("m_copydata"); 1249 count = min(m->m_len - off, len); 1250 bcopy(mtod(m, caddr_t) + off, cp, count); 1251 len -= count; 1252 cp += count; 1253 off = 0; 1254 m = m->m_next; 1255 } 1256 } 1257 1258 /* 1259 * Concatenate mbuf chain n to m. 1260 * Both chains must be of the same type (e.g. MT_DATA). 1261 * Any m_pkthdr is not updated. 1262 */ 1263 void m_cat(m, n) 1264 register struct mbuf *m, *n; 1265 { 1266 while (m->m_next) 1267 m = m->m_next; 1268 while (n) { 1269 if (m->m_flags & M_EXT || 1270 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 1271 /* just join the two chains */ 1272 m->m_next = n; 1273 return; 1274 } 1275 /* splat the data from one into the other */ 1276 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 1277 (u_int)n->m_len); 1278 m->m_len += n->m_len; 1279 n = m_free(n); 1280 } 1281 } 1282 1283 void 1284 m_adj(mp, req_len) 1285 struct mbuf *mp; 1286 int req_len; 1287 { 1288 register int len = req_len; 1289 register struct mbuf *m; 1290 register count; 1291 1292 if ((m = mp) == NULL) 1293 return; 1294 if (len >= 0) { 1295 /* 1296 * Trim from head. 1297 */ 1298 while (m != NULL && len > 0) { 1299 if (m->m_len <= len) { 1300 len -= m->m_len; 1301 m->m_len = 0; 1302 m = m->m_next; 1303 } else { 1304 m->m_len -= len; 1305 m->m_data += len; 1306 len = 0; 1307 } 1308 } 1309 m = mp; 1310 if (m->m_flags & M_PKTHDR) 1311 m->m_pkthdr.len -= (req_len - len); 1312 } else { 1313 /* 1314 * Trim from tail. Scan the mbuf chain, 1315 * calculating its length and finding the last mbuf. 1316 * If the adjustment only affects this mbuf, then just 1317 * adjust and return. Otherwise, rescan and truncate 1318 * after the remaining size. 1319 */ 1320 len = -len; 1321 count = 0; 1322 for (;;) { 1323 count += m->m_len; 1324 if (m->m_next == (struct mbuf *)0) 1325 break; 1326 m = m->m_next; 1327 } 1328 if (m->m_len >= len) { 1329 m->m_len -= len; 1330 m = mp; 1331 if (m->m_flags & M_PKTHDR) 1332 m->m_pkthdr.len -= len; 1333 return; 1334 } 1335 count -= len; 1336 if (count < 0) 1337 count = 0; 1338 /* 1339 * Correct length for chain is "count". 1340 * Find the mbuf with last data, adjust its length, 1341 * and toss data from remaining mbufs on chain. 1342 */ 1343 m = mp; 1344 if (m->m_flags & M_PKTHDR) 1345 m->m_pkthdr.len = count; 1346 for (; m; m = m->m_next) { 1347 if (m->m_len >= count) { 1348 m->m_len = count; 1349 break; 1350 } 1351 count -= m->m_len; 1352 } 1353 while (m = m->m_next) 1354 m->m_len = 0; 1355 } 1356 } 1357 1358 /* 1359 * Rearange an mbuf chain so that len bytes are contiguous 1360 * and in the data area of an mbuf (so that mtod and dtom 1361 * will work for a structure of size len). Returns the resulting 1362 * mbuf chain on success, frees it and returns null on failure. 1363 * If there is room, it will add up to max_protohdr-len extra bytes to the 1364 * contiguous region in an attempt to avoid being called next time. 1365 */ 1366 int MPFail; 1367 1368 struct mbuf * 1369 m_pullup(n, len) 1370 register struct mbuf *n; 1371 int len; 1372 { 1373 register struct mbuf *m; 1374 register int count; 1375 int space; 1376 1377 /* 1378 * If first mbuf has no cluster, and has room for len bytes 1379 * without shifting current data, pullup into it, 1380 * otherwise allocate a new mbuf to prepend to the chain. 1381 */ 1382 if ((n->m_flags & M_EXT) == 0 && 1383 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 1384 if (n->m_len >= len) 1385 return (n); 1386 m = n; 1387 n = n->m_next; 1388 len -= m->m_len; 1389 } else { 1390 if (len > MHLEN) 1391 goto bad; 1392 MGET(m, M_DONTWAIT, n->m_type); 1393 if (m == 0) 1394 goto bad; 1395 m->m_len = 0; 1396 if (n->m_flags & M_PKTHDR) { 1397 M_COPY_PKTHDR(m, n); 1398 n->m_flags &= ~M_PKTHDR; 1399 } 1400 } 1401 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 1402 do { 1403 count = min(min(max(len, max_protohdr), space), n->m_len); 1404 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 1405 (unsigned)count); 1406 len -= count; 1407 m->m_len += count; 1408 n->m_len -= count; 1409 space -= count; 1410 if (n->m_len) 1411 n->m_data += count; 1412 else 1413 n = m_free(n); 1414 } while (len > 0 && n); 1415 if (len > 0) { 1416 (void) m_free(m); 1417 goto bad; 1418 } 1419 m->m_next = n; 1420 return (m); 1421 bad: 1422 m_freem(n); 1423 MPFail++; 1424 return (0); 1425 } 1426 1427 /* 1428 * Partition an mbuf chain in two pieces, returning the tail -- 1429 * all but the first len0 bytes. In case of failure, it returns NULL and 1430 * attempts to restore the chain to its original state. 1431 */ 1432 struct mbuf * 1433 m_split(m0, len0, wait) 1434 register struct mbuf *m0; 1435 int len0, wait; 1436 { 1437 register struct mbuf *m, *n; 1438 unsigned len = len0, remain; 1439 1440 for (m = m0; m && len > m->m_len; m = m->m_next) 1441 len -= m->m_len; 1442 if (m == 0) 1443 return (0); 1444 remain = m->m_len - len; 1445 if (m0->m_flags & M_PKTHDR) { 1446 MGETHDR(n, wait, m0->m_type); 1447 if (n == 0) 1448 return (0); 1449 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 1450 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 1451 m0->m_pkthdr.len = len0; 1452 if (m->m_flags & M_EXT) 1453 goto extpacket; 1454 if (remain > MHLEN) { 1455 /* m can't be the lead packet */ 1456 MH_ALIGN(n, 0); 1457 n->m_next = m_split(m, len, wait); 1458 if (n->m_next == 0) { 1459 (void) m_free(n); 1460 return (0); 1461 } else 1462 return (n); 1463 } else 1464 MH_ALIGN(n, remain); 1465 } else if (remain == 0) { 1466 n = m->m_next; 1467 m->m_next = 0; 1468 return (n); 1469 } else { 1470 MGET(n, wait, m->m_type); 1471 if (n == 0) 1472 return (0); 1473 M_ALIGN(n, remain); 1474 } 1475 extpacket: 1476 if (m->m_flags & M_EXT) { 1477 n->m_flags |= M_EXT; 1478 MBUF_LOCK(); 1479 n->m_ext = m->m_ext; 1480 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs); 1481 MBUF_UNLOCK(); 1482 n->m_data = m->m_data + len; 1483 } else { 1484 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); 1485 } 1486 n->m_len = remain; 1487 m->m_len = len; 1488 n->m_next = m->m_next; 1489 m->m_next = 0; 1490 return (n); 1491 } 1492 /* 1493 * Routine to copy from device local memory into mbufs. 1494 */ 1495 struct mbuf * 1496 m_devget(buf, totlen, off0, ifp, copy) 1497 char *buf; 1498 int totlen, off0; 1499 struct ifnet *ifp; 1500 void (*copy)(); 1501 { 1502 register struct mbuf *m; 1503 struct mbuf *top = 0, **mp = ⊤ 1504 register int off = off0, len; 1505 register char *cp; 1506 char *epkt; 1507 1508 cp = buf; 1509 epkt = cp + totlen; 1510 if (off) { 1511 /* 1512 * If 'off' is non-zero, packet is trailer-encapsulated, 1513 * so we have to skip the type and length fields. 1514 */ 1515 cp += off + 2 * sizeof(u_int16_t); 1516 totlen -= 2 * sizeof(u_int16_t); 1517 } 1518 MGETHDR(m, M_DONTWAIT, MT_DATA); 1519 if (m == 0) 1520 return (0); 1521 m->m_pkthdr.rcvif = ifp; 1522 m->m_pkthdr.len = totlen; 1523 m->m_len = MHLEN; 1524 1525 while (totlen > 0) { 1526 if (top) { 1527 MGET(m, M_DONTWAIT, MT_DATA); 1528 if (m == 0) { 1529 m_freem(top); 1530 return (0); 1531 } 1532 m->m_len = MLEN; 1533 } 1534 len = min(totlen, epkt - cp); 1535 if (len >= MINCLSIZE) { 1536 MCLGET(m, M_DONTWAIT); 1537 if (m->m_flags & M_EXT) 1538 m->m_len = len = min(len, MCLBYTES); 1539 else { 1540 /* give up when it's out of cluster mbufs */ 1541 if (top) 1542 m_freem(top); 1543 m_freem(m); 1544 return (0); 1545 } 1546 } else { 1547 /* 1548 * Place initial small packet/header at end of mbuf. 1549 */ 1550 if (len < m->m_len) { 1551 if (top == 0 && len + max_linkhdr <= m->m_len) 1552 m->m_data += max_linkhdr; 1553 m->m_len = len; 1554 } else 1555 len = m->m_len; 1556 } 1557 if (copy) 1558 copy(cp, mtod(m, caddr_t), (unsigned)len); 1559 else 1560 bcopy(cp, mtod(m, caddr_t), (unsigned)len); 1561 cp += len; 1562 *mp = m; 1563 mp = &m->m_next; 1564 totlen -= len; 1565 if (cp == epkt) 1566 cp = buf; 1567 } 1568 return (top); 1569 } 1570 1571 /* 1572 * Cluster freelist allocation check. The mbuf lock must be held. 1573 * Ensure hysteresis between hi/lo. 1574 */ 1575 static int 1576 m_howmany() 1577 { 1578 register int i; 1579 1580 /* Under minimum */ 1581 if (mbstat.m_clusters < MINCL) 1582 return (MINCL - mbstat.m_clusters); 1583 /* Too few (free < 1/2 total) and not over maximum */ 1584 if (mbstat.m_clusters < nmbclusters && 1585 (i = ((mbstat.m_clusters >> 1) - mbstat.m_clfree)) > 0) 1586 return i; 1587 return 0; 1588 } 1589 1590 1591 /* 1592 * Copy data from a buffer back into the indicated mbuf chain, 1593 * starting "off" bytes from the beginning, extending the mbuf 1594 * chain if necessary. 1595 */ 1596 void 1597 m_copyback(m0, off, len, cp) 1598 struct mbuf *m0; 1599 register int off; 1600 register int len; 1601 caddr_t cp; 1602 { 1603 register int mlen; 1604 register struct mbuf *m = m0, *n; 1605 int totlen = 0; 1606 1607 if (m0 == 0) 1608 return; 1609 while (off > (mlen = m->m_len)) { 1610 off -= mlen; 1611 totlen += mlen; 1612 if (m->m_next == 0) { 1613 n = m_getclr(M_DONTWAIT, m->m_type); 1614 if (n == 0) 1615 goto out; 1616 n->m_len = min(MLEN, len + off); 1617 m->m_next = n; 1618 } 1619 m = m->m_next; 1620 } 1621 while (len > 0) { 1622 mlen = min (m->m_len - off, len); 1623 bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen); 1624 cp += mlen; 1625 len -= mlen; 1626 mlen += off; 1627 off = 0; 1628 totlen += mlen; 1629 if (len == 0) 1630 break; 1631 if (m->m_next == 0) { 1632 n = m_get(M_DONTWAIT, m->m_type); 1633 if (n == 0) 1634 break; 1635 n->m_len = min(MLEN, len); 1636 m->m_next = n; 1637 } 1638 m = m->m_next; 1639 } 1640 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 1641 m->m_pkthdr.len = totlen; 1642 } 1643 1644 1645 char *mcl_to_paddr(register char *addr) { 1646 register int base_phys; 1647 1648 if (addr < (char *)mbutl || addr >= (char *)embutl) 1649 return (0); 1650 base_phys = mcl_paddr[(addr - (char *)mbutl) >> PAGE_SHIFT]; 1651 1652 if (base_phys == 0) 1653 return (0); 1654 return ((char *)((int)base_phys | ((int)addr & PAGE_MASK))); 1655 } 1656 1657 /* 1658 * Dup the mbuf chain passed in. The whole thing. No cute additional cruft. 1659 * And really copy the thing. That way, we don't "precompute" checksums 1660 * for unsuspecting consumers. 1661 * Assumption: m->m_nextpkt == 0. 1662 * Trick: for small packets, don't dup into a cluster. That way received 1663 * packets don't take up too much room in the sockbuf (cf. sbspace()). 1664 */ 1665 int MDFail; 1666 1667 struct mbuf * 1668 m_dup(register struct mbuf *m, int how) 1669 { register struct mbuf *n, **np; 1670 struct mbuf *top; 1671 int copyhdr = 0; 1672 1673 np = ⊤ 1674 top = 0; 1675 if (m->m_flags & M_PKTHDR) 1676 copyhdr = 1; 1677 1678 /* 1679 * Quick check: if we have one mbuf and its data fits in an 1680 * mbuf with packet header, just copy and go. 1681 */ 1682 if (m->m_next == NULL) 1683 { /* Then just move the data into an mbuf and be done... */ 1684 if (copyhdr) 1685 { if (m->m_pkthdr.len <= MHLEN) 1686 { if ((n = m_gethdr(how, m->m_type)) == NULL) 1687 return(NULL); 1688 n->m_len = m->m_len; 1689 n->m_flags |= (m->m_flags & M_COPYFLAGS); 1690 n->m_pkthdr.len = m->m_pkthdr.len; 1691 n->m_pkthdr.rcvif = m->m_pkthdr.rcvif; 1692 n->m_pkthdr.header = NULL; 1693 n->m_pkthdr.csum_flags = 0; 1694 n->m_pkthdr.csum_data = 0; 1695 n->m_pkthdr.aux = NULL; 1696 n->m_pkthdr.reserved1 = 0; 1697 n->m_pkthdr.reserved2 = 0; 1698 bcopy(m->m_data, n->m_data, m->m_pkthdr.len); 1699 return(n); 1700 } 1701 } else if (m->m_len <= MLEN) 1702 { if ((n = m_get(how, m->m_type)) == NULL) 1703 return(NULL); 1704 bcopy(m->m_data, n->m_data, m->m_len); 1705 n->m_len = m->m_len; 1706 return(n); 1707 } 1708 } 1709 while (m) 1710 { 1711 #if BLUE_DEBUG 1712 kprintf("<%x: %x, %x, %x\n", m, m->m_flags, m->m_len, 1713 m->m_data); 1714 #endif 1715 if (copyhdr) 1716 n = m_gethdr(how, m->m_type); 1717 else 1718 n = m_get(how, m->m_type); 1719 if (n == 0) 1720 goto nospace; 1721 if (m->m_flags & M_EXT) 1722 { MCLGET(n, how); 1723 if ((n->m_flags & M_EXT) == 0) 1724 goto nospace; 1725 } 1726 *np = n; 1727 if (copyhdr) 1728 { /* Don't use M_COPY_PKTHDR: preserve m_data */ 1729 n->m_pkthdr = m->m_pkthdr; 1730 n->m_flags |= (m->m_flags & M_COPYFLAGS); 1731 copyhdr = 0; 1732 if ((n->m_flags & M_EXT) == 0) 1733 n->m_data = n->m_pktdat; 1734 } 1735 n->m_len = m->m_len; 1736 /* 1737 * Get the dup on the same bdry as the original 1738 * Assume that the two mbufs have the same offset to data area 1739 * (up to word bdries) 1740 */ 1741 bcopy(mtod(m, caddr_t), mtod(n, caddr_t), (unsigned)n->m_len); 1742 m = m->m_next; 1743 np = &n->m_next; 1744 #if BLUE_DEBUG 1745 kprintf(">%x: %x, %x, %x\n", n, n->m_flags, n->m_len, 1746 n->m_data); 1747 #endif 1748 } 1749 1750 if (top == 0) 1751 MDFail++; 1752 return (top); 1753 nospace: 1754 m_freem(top); 1755 MDFail++; 1756 return (0); 1757 } 1758 1759 int 1760 m_mclref(struct mbuf *p) 1761 { 1762 return (_MCLREF(p)); 1763 } 1764 1765 int 1766 m_mclunref(struct mbuf *p) 1767 { 1768 return (_MCLUNREF(p)); 1769 } 1770 1771 /* change mbuf to new type */ 1772 void 1773 m_mchtype(struct mbuf *m, int t) 1774 { 1775 MBUF_LOCK(); 1776 mbstat.m_mtypes[(m)->m_type]--; 1777 mbstat.m_mtypes[t]++; 1778 (m)->m_type = t; 1779 MBUF_UNLOCK(); 1780 } 1781 1782 void *m_mtod(struct mbuf *m) 1783 { 1784 return ((m)->m_data); 1785 } 1786 1787 struct mbuf *m_dtom(void *x) 1788 { 1789 return ((struct mbuf *)((u_long)(x) & ~(MSIZE-1))); 1790 } 1791 1792 int m_mtocl(void *x) 1793 { 1794 return (((char *)(x) - (char *)mbutl) / sizeof(union mcluster)); 1795 } 1796 1797 union mcluster *m_cltom(int x) 1798 { 1799 return ((union mcluster *)(mbutl + (x))); 1800 } 1801 1802 1803 void m_mcheck(struct mbuf *m) 1804 { 1805 if (m->m_type != MT_FREE) 1806 panic("mget MCHECK: m_type=%x m=%x", m->m_type, m); 1807 } 1808 1809 void 1810 mbuf_expand_thread(void) 1811 { 1812 while (1) { 1813 int expand_mcl; 1814 MBUF_LOCK(); 1815 expand_mcl = mbuf_expand_mcl; 1816 mbuf_expand_mcl = 0; 1817 MBUF_UNLOCK(); 1818 if (expand_mcl) { 1819 caddr_t p; 1820 MCLALLOC(p, M_WAIT); 1821 if (p) MCLFREE(p); 1822 } 1823 assert_wait(&mbuf_expand_thread_wakeup, THREAD_UNINT); 1824 (void) thread_block(mbuf_expand_thread); 1825 } 1826 } 1827 1828 void 1829 mbuf_expand_thread_init(void) 1830 { 1831 mbuf_expand_thread_initialized++; 1832 mbuf_expand_thread(); 1833 } 1834
Cache object: 618d33ddefaf344534eac4fe2a40cd40
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]