FreeBSD/Linux Kernel Cross Reference
sys/kern/uipc_mbuf.c
1 /* $OpenBSD: uipc_mbuf.c,v 1.107 2008/11/29 19:57:09 deraadt Exp $ */
2 /* $NetBSD: uipc_mbuf.c,v 1.15.4.1 1996/06/13 17:11:44 cgd Exp $ */
3
4 /*
5 * Copyright (c) 1982, 1986, 1988, 1991, 1993
6 * The Regents of the University of California. All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the name of the University nor the names of its contributors
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 *
32 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94
33 */
34
35 /*
36 * @(#)COPYRIGHT 1.1 (NRL) 17 January 1995
37 *
38 * NRL grants permission for redistribution and use in source and binary
39 * forms, with or without modification, of the software and documentation
40 * created at NRL provided that the following conditions are met:
41 *
42 * 1. Redistributions of source code must retain the above copyright
43 * notice, this list of conditions and the following disclaimer.
44 * 2. Redistributions in binary form must reproduce the above copyright
45 * notice, this list of conditions and the following disclaimer in the
46 * documentation and/or other materials provided with the distribution.
47 * 3. All advertising materials mentioning features or use of this software
48 * must display the following acknowledgements:
49 * This product includes software developed by the University of
50 * California, Berkeley and its contributors.
51 * This product includes software developed at the Information
52 * Technology Division, US Naval Research Laboratory.
53 * 4. Neither the name of the NRL nor the names of its contributors
54 * may be used to endorse or promote products derived from this software
55 * without specific prior written permission.
56 *
57 * THE SOFTWARE PROVIDED BY NRL IS PROVIDED BY NRL AND CONTRIBUTORS ``AS
58 * IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
59 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
60 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NRL OR
61 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
62 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
63 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
64 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
65 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
66 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
67 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
68 *
69 * The views and conclusions contained in the software and documentation
70 * are those of the authors and should not be interpreted as representing
71 * official policies, either expressed or implied, of the US Naval
72 * Research Laboratory (NRL).
73 */
74
75 #include <sys/param.h>
76 #include <sys/systm.h>
77 #include <sys/proc.h>
78 #include <sys/malloc.h>
79 #define MBTYPES
80 #include <sys/mbuf.h>
81 #include <sys/kernel.h>
82 #include <sys/syslog.h>
83 #include <sys/domain.h>
84 #include <sys/protosw.h>
85 #include <sys/pool.h>
86
87 #include <sys/socket.h>
88 #include <sys/socketvar.h>
89 #include <net/if.h>
90
91 #include <machine/cpu.h>
92
93 #include <uvm/uvm_extern.h>
94
95 struct mbstat mbstat; /* mbuf stats */
96 struct pool mbpool; /* mbuf pool */
97
98 /* mbuf cluster pools */
99 u_int mclsizes[] = {
100 MCLBYTES, /* must be at slot 0 */
101 4 * 1024,
102 #if 0
103 8 * 1024,
104 9 * 1024,
105 12 * 1024,
106 16 * 1024,
107 64 * 1024
108 #endif
109 };
110 static char mclnames[MCLPOOLS][8];
111 struct pool mclpools[MCLPOOLS];
112
113 int max_linkhdr; /* largest link-level header */
114 int max_protohdr; /* largest protocol header */
115 int max_hdr; /* largest link+protocol header */
116 int max_datalen; /* MHLEN - max_hdr */
117
118 void m_extfree(struct mbuf *);
119 struct mbuf *m_copym0(struct mbuf *, int, int, int, int);
120 void nmbclust_update(void);
121
122
123 const char *mclpool_warnmsg =
124 "WARNING: mclpools limit reached; increase kern.maxclusters";
125
126 /*
127 * Initialize the mbuf allocator.
128 */
129 void
130 mbinit(void)
131 {
132 int i;
133
134 pool_init(&mbpool, MSIZE, 0, 0, 0, "mbpl", NULL);
135 pool_setlowat(&mbpool, mblowat);
136
137 for (i = 0; i < nitems(mclsizes); i++) {
138 snprintf(mclnames[i], sizeof(mclnames[0]), "mcl%dk",
139 mclsizes[i] >> 10);
140 pool_init(&mclpools[i], mclsizes[i], 0, 0, 0, mclnames[i],
141 NULL);
142 pool_setlowat(&mclpools[i], mcllowat);
143 }
144
145 nmbclust_update();
146 }
147
148 void
149 nmbclust_update(void)
150 {
151 int i;
152 /*
153 * Set the hard limit on the mclpools to the number of
154 * mbuf clusters the kernel is to support. Log the limit
155 * reached message max once a minute.
156 */
157 for (i = 0; i < nitems(mclsizes); i++) {
158 (void)pool_sethardlimit(&mclpools[i], nmbclust,
159 mclpool_warnmsg, 60);
160 }
161 pool_sethiwat(&mbpool, nmbclust);
162 }
163
164 void
165 m_reclaim(void *arg, int flags)
166 {
167 struct domain *dp;
168 struct protosw *pr;
169 int s = splvm();
170
171 for (dp = domains; dp; dp = dp->dom_next)
172 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
173 if (pr->pr_drain)
174 (*pr->pr_drain)();
175 splx(s);
176 mbstat.m_drain++;
177 }
178
179 /*
180 * Space allocation routines.
181 */
182 struct mbuf *
183 m_get(int nowait, int type)
184 {
185 struct mbuf *m;
186 int s;
187
188 s = splvm();
189 m = pool_get(&mbpool, nowait == M_WAIT ? PR_WAITOK : 0);
190 splx(s);
191 if (m) {
192 m->m_type = type;
193 mbstat.m_mtypes[type]++;
194 m->m_next = (struct mbuf *)NULL;
195 m->m_nextpkt = (struct mbuf *)NULL;
196 m->m_data = m->m_dat;
197 m->m_flags = 0;
198 }
199 return (m);
200 }
201
202 /*
203 * ATTN: When changing anything here check m_inithdr() and m_defrag() those
204 * may need to change as well.
205 */
206 struct mbuf *
207 m_gethdr(int nowait, int type)
208 {
209 struct mbuf *m;
210 int s;
211
212 s = splvm();
213 m = pool_get(&mbpool, nowait == M_WAIT ? PR_WAITOK : 0);
214 splx(s);
215 if (m) {
216 m->m_type = type;
217 mbstat.m_mtypes[type]++;
218
219 /* keep in sync with m_inithdr */
220 m->m_next = (struct mbuf *)NULL;
221 m->m_nextpkt = (struct mbuf *)NULL;
222 m->m_data = m->m_pktdat;
223 m->m_flags = M_PKTHDR;
224 m->m_pkthdr.rcvif = NULL;
225 SLIST_INIT(&m->m_pkthdr.tags);
226 m->m_pkthdr.csum_flags = 0;
227 m->m_pkthdr.ether_vtag = 0;
228 m->m_pkthdr.pf.hdr = NULL;
229 m->m_pkthdr.pf.statekey = NULL;
230 m->m_pkthdr.pf.rtableid = 0;
231 m->m_pkthdr.pf.qid = 0;
232 m->m_pkthdr.pf.tag = 0;
233 m->m_pkthdr.pf.flags = 0;
234 m->m_pkthdr.pf.routed = 0;
235 }
236 return (m);
237 }
238
239 struct mbuf *
240 m_inithdr(struct mbuf *m)
241 {
242 /* keep in sync with m_gethdr */
243 m->m_next = (struct mbuf *)NULL;
244 m->m_nextpkt = (struct mbuf *)NULL;
245 m->m_data = m->m_pktdat;
246 m->m_flags = M_PKTHDR;
247 m->m_pkthdr.rcvif = NULL;
248 SLIST_INIT(&m->m_pkthdr.tags);
249 m->m_pkthdr.csum_flags = 0;
250 m->m_pkthdr.ether_vtag = 0;
251 m->m_pkthdr.pf.hdr = NULL;
252 m->m_pkthdr.pf.statekey = NULL;
253 m->m_pkthdr.pf.rtableid = 0;
254 m->m_pkthdr.pf.qid = 0;
255 m->m_pkthdr.pf.tag = 0;
256 m->m_pkthdr.pf.flags = 0;
257 m->m_pkthdr.pf.routed = 0;
258
259 return (m);
260 }
261
262 struct mbuf *
263 m_getclr(int nowait, int type)
264 {
265 struct mbuf *m;
266
267 MGET(m, nowait, type);
268 if (m == NULL)
269 return (NULL);
270 memset(mtod(m, caddr_t), 0, MLEN);
271 return (m);
272 }
273
274 void
275 m_clget(struct mbuf *m, int how, struct ifnet *ifp, u_int pktlen)
276 {
277 struct pool *mclp;
278 int pi;
279 int s;
280
281 for (pi = 0; pi < nitems(mclpools); pi++) {
282 mclp = &mclpools[pi];
283 if (pktlen <= mclp->pr_size)
284 break;
285 }
286
287 #ifdef DIAGNOSTIC
288 if (mclp == NULL)
289 panic("m_clget: request for %d sized cluster", pktlen);
290 #endif
291
292 if (ifp != NULL && m_cldrop(ifp, pi))
293 return;
294
295 s = splvm();
296 m->m_ext.ext_buf = pool_get(mclp, how == M_WAIT ? PR_WAITOK : 0);
297 splx(s);
298 if (m->m_ext.ext_buf != NULL) {
299 m->m_data = m->m_ext.ext_buf;
300 m->m_flags |= M_EXT|M_CLUSTER;
301 m->m_ext.ext_size = mclp->pr_size;
302 m->m_ext.ext_free = NULL;
303 m->m_ext.ext_arg = NULL;
304
305 m->m_ext.ext_backend = pi;
306 m->m_ext.ext_ifp = ifp;
307 if (ifp != NULL)
308 m_clcount(ifp, pi);
309
310 MCLINITREFERENCE(m);
311 }
312 }
313
314 struct mbuf *
315 m_free(struct mbuf *m)
316 {
317 struct mbuf *n;
318 int s;
319
320 s = splvm();
321 mbstat.m_mtypes[m->m_type]--;
322 if (m->m_flags & M_PKTHDR)
323 m_tag_delete_chain(m);
324 if (m->m_flags & M_EXT)
325 m_extfree(m);
326 m->m_flags = 0;
327 n = m->m_next;
328 pool_put(&mbpool, m);
329 splx(s);
330
331 return (n);
332 }
333
334 void
335 m_extfree(struct mbuf *m)
336 {
337 if (MCLISREFERENCED(m)) {
338 m->m_ext.ext_nextref->m_ext.ext_prevref =
339 m->m_ext.ext_prevref;
340 m->m_ext.ext_prevref->m_ext.ext_nextref =
341 m->m_ext.ext_nextref;
342 } else if (m->m_flags & M_CLUSTER) {
343 m_cluncount(m, 0);
344 pool_put(&mclpools[m->m_ext.ext_backend],
345 m->m_ext.ext_buf);
346 } else if (m->m_ext.ext_free)
347 (*(m->m_ext.ext_free))(m->m_ext.ext_buf,
348 m->m_ext.ext_size, m->m_ext.ext_arg);
349 else
350 panic("unknown type of extension buffer");
351 m->m_ext.ext_size = 0;
352 m->m_flags &= ~(M_EXT|M_CLUSTER);
353 }
354
355 void
356 m_freem(struct mbuf *m)
357 {
358 struct mbuf *n;
359
360 if (m == NULL)
361 return;
362 do {
363 MFREE(m, n);
364 } while ((m = n) != NULL);
365 }
366
367 /*
368 * mbuf chain defragmenter. This function uses some evil tricks to defragment
369 * an mbuf chain into a single buffer without changing the mbuf pointer.
370 * This needs to know a lot of the mbuf internals to make this work.
371 */
372 int
373 m_defrag(struct mbuf *m, int how)
374 {
375 struct mbuf *m0;
376
377 if (m->m_next == NULL)
378 return 0;
379
380 #ifdef DIAGNOSTIC
381 if (!(m->m_flags & M_PKTHDR))
382 panic("m_defrag: no packet hdr or not a chain");
383 #endif
384
385 if ((m0 = m_gethdr(how, m->m_type)) == NULL)
386 return -1;
387 if (m->m_pkthdr.len > MHLEN) {
388 MCLGETI(m0, how, NULL, m->m_pkthdr.len);
389 if (!(m0->m_flags & M_EXT)) {
390 m_free(m0);
391 return -1;
392 }
393 }
394 m_copydata(m, 0, m->m_pkthdr.len, mtod(m0, caddr_t));
395 m0->m_pkthdr.len = m0->m_len = m->m_pkthdr.len;
396
397 /* free chain behind and possible ext buf on the first mbuf */
398 m_freem(m->m_next);
399 m->m_next = NULL;
400
401 if (m->m_flags & M_EXT) {
402 int s = splvm();
403 m_extfree(m);
404 splx(s);
405 }
406
407 /*
408 * Bounce copy mbuf over to the original mbuf and set everything up.
409 * This needs to reset or clear all pointers that may go into the
410 * original mbuf chain.
411 */
412 if (m0->m_flags & M_EXT) {
413 bcopy(&m0->m_ext, &m->m_ext, sizeof(struct mbuf_ext));
414 MCLINITREFERENCE(m);
415 m->m_flags |= M_EXT|M_CLUSTER;
416 m->m_data = m->m_ext.ext_buf;
417 } else {
418 m->m_data = m->m_pktdat;
419 bcopy(&m0->m_data, &m->m_data, m0->m_len);
420 }
421 m->m_pkthdr.len = m->m_len = m0->m_len;
422 m->m_pkthdr.pf.hdr = NULL; /* altq will cope */
423
424 m0->m_flags &= ~(M_EXT|M_CLUSTER); /* cluster is gone */
425 m_free(m0);
426
427 return 0;
428 }
429
430 /*
431 * Mbuffer utility routines.
432 */
433
434 /*
435 * Lesser-used path for M_PREPEND:
436 * allocate new mbuf to prepend to chain,
437 * copy junk along.
438 */
439 struct mbuf *
440 m_prepend(struct mbuf *m, int len, int how)
441 {
442 struct mbuf *mn;
443
444 if (len > MHLEN)
445 panic("mbuf prepend length too big");
446
447 MGET(mn, how, m->m_type);
448 if (mn == NULL) {
449 m_freem(m);
450 return (NULL);
451 }
452 if (m->m_flags & M_PKTHDR)
453 M_MOVE_PKTHDR(mn, m);
454 mn->m_next = m;
455 m = mn;
456 MH_ALIGN(m, len);
457 m->m_len = len;
458 return (m);
459 }
460
461 /*
462 * Make a copy of an mbuf chain starting "off" bytes from the beginning,
463 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
464 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
465 */
466 int MCFail;
467
468 struct mbuf *
469 m_copym(struct mbuf *m, int off, int len, int wait)
470 {
471 return m_copym0(m, off, len, wait, 0); /* shallow copy on M_EXT */
472 }
473
474 /*
475 * m_copym2() is like m_copym(), except it COPIES cluster mbufs, instead
476 * of merely bumping the reference count.
477 */
478 struct mbuf *
479 m_copym2(struct mbuf *m, int off, int len, int wait)
480 {
481 return m_copym0(m, off, len, wait, 1); /* deep copy */
482 }
483
484 struct mbuf *
485 m_copym0(struct mbuf *m, int off, int len, int wait, int deep)
486 {
487 struct mbuf *n, **np;
488 struct mbuf *top;
489 int copyhdr = 0;
490
491 if (off < 0 || len < 0)
492 panic("m_copym0: off %d, len %d", off, len);
493 if (off == 0 && m->m_flags & M_PKTHDR)
494 copyhdr = 1;
495 while (off > 0) {
496 if (m == NULL)
497 panic("m_copym0: null mbuf");
498 if (off < m->m_len)
499 break;
500 off -= m->m_len;
501 m = m->m_next;
502 }
503 np = ⊤
504 top = NULL;
505 while (len > 0) {
506 if (m == NULL) {
507 if (len != M_COPYALL)
508 panic("m_copym0: m == NULL and not COPYALL");
509 break;
510 }
511 MGET(n, wait, m->m_type);
512 *np = n;
513 if (n == NULL)
514 goto nospace;
515 if (copyhdr) {
516 M_DUP_PKTHDR(n, m);
517 if (len != M_COPYALL)
518 n->m_pkthdr.len = len;
519 copyhdr = 0;
520 }
521 n->m_len = min(len, m->m_len - off);
522 if (m->m_flags & M_EXT) {
523 if (!deep) {
524 n->m_data = m->m_data + off;
525 n->m_ext = m->m_ext;
526 MCLADDREFERENCE(m, n);
527 } else {
528 /*
529 * we are unsure about the way m was allocated.
530 * copy into multiple MCLBYTES cluster mbufs.
531 */
532 MCLGET(n, wait);
533 n->m_len = 0;
534 n->m_len = M_TRAILINGSPACE(n);
535 n->m_len = min(n->m_len, len);
536 n->m_len = min(n->m_len, m->m_len - off);
537 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off,
538 (unsigned)n->m_len);
539 }
540 } else
541 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off,
542 (unsigned)n->m_len);
543 if (len != M_COPYALL)
544 len -= n->m_len;
545 off += n->m_len;
546 #ifdef DIAGNOSTIC
547 if (off > m->m_len)
548 panic("m_copym0 overrun");
549 #endif
550 if (off == m->m_len) {
551 m = m->m_next;
552 off = 0;
553 }
554 np = &n->m_next;
555 }
556 if (top == NULL)
557 MCFail++;
558 return (top);
559 nospace:
560 m_freem(top);
561 MCFail++;
562 return (NULL);
563 }
564
565 /*
566 * Copy data from an mbuf chain starting "off" bytes from the beginning,
567 * continuing for "len" bytes, into the indicated buffer.
568 */
569 void
570 m_copydata(struct mbuf *m, int off, int len, caddr_t cp)
571 {
572 unsigned count;
573
574 if (off < 0)
575 panic("m_copydata: off %d < 0", off);
576 if (len < 0)
577 panic("m_copydata: len %d < 0", len);
578 while (off > 0) {
579 if (m == NULL)
580 panic("m_copydata: null mbuf in skip");
581 if (off < m->m_len)
582 break;
583 off -= m->m_len;
584 m = m->m_next;
585 }
586 while (len > 0) {
587 if (m == NULL)
588 panic("m_copydata: null mbuf");
589 count = min(m->m_len - off, len);
590 bcopy(mtod(m, caddr_t) + off, cp, count);
591 len -= count;
592 cp += count;
593 off = 0;
594 m = m->m_next;
595 }
596 }
597
598 /*
599 * Copy data from a buffer back into the indicated mbuf chain,
600 * starting "off" bytes from the beginning, extending the mbuf
601 * chain if necessary. The mbuf needs to be properly initialized
602 * including the setting of m_len.
603 */
604 void
605 m_copyback(struct mbuf *m0, int off, int len, const void *_cp)
606 {
607 int mlen;
608 struct mbuf *m = m0, *n;
609 int totlen = 0;
610 caddr_t cp = (caddr_t)_cp;
611
612 if (m0 == NULL)
613 return;
614 while (off > (mlen = m->m_len)) {
615 off -= mlen;
616 totlen += mlen;
617 if (m->m_next == NULL) {
618 n = m_getclr(M_DONTWAIT, m->m_type);
619 if (n == NULL)
620 goto out;
621 n->m_len = min(MLEN, len + off);
622 m->m_next = n;
623 }
624 m = m->m_next;
625 }
626 while (len > 0) {
627 mlen = min (m->m_len - off, len);
628 bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen);
629 cp += mlen;
630 len -= mlen;
631 mlen += off;
632 off = 0;
633 totlen += mlen;
634 if (len == 0)
635 break;
636 if (m->m_next == NULL) {
637 n = m_get(M_DONTWAIT, m->m_type);
638 if (n == NULL)
639 break;
640 n->m_len = min(MLEN, len);
641 m->m_next = n;
642 }
643 m = m->m_next;
644 }
645 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
646 m->m_pkthdr.len = totlen;
647 }
648
649 /*
650 * Concatenate mbuf chain n to m.
651 * n might be copied into m (when n->m_len is small), therefore data portion of
652 * n could be copied into an mbuf of different mbuf type.
653 * Therefore both chains should be of the same type (e.g. MT_DATA).
654 * Any m_pkthdr is not updated.
655 */
656 void
657 m_cat(struct mbuf *m, struct mbuf *n)
658 {
659 while (m->m_next)
660 m = m->m_next;
661 while (n) {
662 if (M_READONLY(m) || n->m_len > M_TRAILINGSPACE(m)) {
663 /* just join the two chains */
664 m->m_next = n;
665 return;
666 }
667 /* splat the data from one into the other */
668 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
669 (u_int)n->m_len);
670 m->m_len += n->m_len;
671 n = m_free(n);
672 }
673 }
674
675 void
676 m_adj(struct mbuf *mp, int req_len)
677 {
678 int len = req_len;
679 struct mbuf *m;
680 int count;
681
682 if ((m = mp) == NULL)
683 return;
684 if (len >= 0) {
685 /*
686 * Trim from head.
687 */
688 while (m != NULL && len > 0) {
689 if (m->m_len <= len) {
690 len -= m->m_len;
691 m->m_len = 0;
692 m = m->m_next;
693 } else {
694 m->m_len -= len;
695 m->m_data += len;
696 len = 0;
697 }
698 }
699 m = mp;
700 if (mp->m_flags & M_PKTHDR)
701 m->m_pkthdr.len -= (req_len - len);
702 } else {
703 /*
704 * Trim from tail. Scan the mbuf chain,
705 * calculating its length and finding the last mbuf.
706 * If the adjustment only affects this mbuf, then just
707 * adjust and return. Otherwise, rescan and truncate
708 * after the remaining size.
709 */
710 len = -len;
711 count = 0;
712 for (;;) {
713 count += m->m_len;
714 if (m->m_next == NULL)
715 break;
716 m = m->m_next;
717 }
718 if (m->m_len >= len) {
719 m->m_len -= len;
720 if (mp->m_flags & M_PKTHDR)
721 mp->m_pkthdr.len -= len;
722 return;
723 }
724 count -= len;
725 if (count < 0)
726 count = 0;
727 /*
728 * Correct length for chain is "count".
729 * Find the mbuf with last data, adjust its length,
730 * and toss data from remaining mbufs on chain.
731 */
732 m = mp;
733 if (m->m_flags & M_PKTHDR)
734 m->m_pkthdr.len = count;
735 for (; m; m = m->m_next) {
736 if (m->m_len >= count) {
737 m->m_len = count;
738 break;
739 }
740 count -= m->m_len;
741 }
742 while ((m = m->m_next) != NULL)
743 m->m_len = 0;
744 }
745 }
746
747 /*
748 * Rearange an mbuf chain so that len bytes are contiguous
749 * and in the data area of an mbuf (so that mtod and dtom
750 * will work for a structure of size len). Returns the resulting
751 * mbuf chain on success, frees it and returns null on failure.
752 * If there is room, it will add up to max_protohdr-len extra bytes to the
753 * contiguous region in an attempt to avoid being called next time.
754 */
755 int MPFail;
756
757 struct mbuf *
758 m_pullup(struct mbuf *n, int len)
759 {
760 struct mbuf *m;
761 int count;
762 int space;
763
764 /*
765 * If first mbuf has no cluster, and has room for len bytes
766 * without shifting current data, pullup into it,
767 * otherwise allocate a new mbuf to prepend to the chain.
768 */
769 if ((n->m_flags & M_EXT) == 0 &&
770 n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
771 if (n->m_len >= len)
772 return (n);
773 m = n;
774 n = n->m_next;
775 len -= m->m_len;
776 } else {
777 if (len > MHLEN)
778 goto bad;
779 MGET(m, M_DONTWAIT, n->m_type);
780 if (m == NULL)
781 goto bad;
782 m->m_len = 0;
783 if (n->m_flags & M_PKTHDR)
784 M_MOVE_PKTHDR(m, n);
785 }
786 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
787 do {
788 count = min(min(max(len, max_protohdr), space), n->m_len);
789 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
790 (unsigned)count);
791 len -= count;
792 m->m_len += count;
793 n->m_len -= count;
794 space -= count;
795 if (n->m_len)
796 n->m_data += count;
797 else
798 n = m_free(n);
799 } while (len > 0 && n);
800 if (len > 0) {
801 (void)m_free(m);
802 goto bad;
803 }
804 m->m_next = n;
805 return (m);
806 bad:
807 m_freem(n);
808 MPFail++;
809 return (NULL);
810 }
811
812 /*
813 * m_pullup2() works like m_pullup, save that len can be <= MCLBYTES.
814 * m_pullup2() only works on values of len such that MHLEN < len <= MCLBYTES,
815 * it calls m_pullup() for values <= MHLEN. It also only coagulates the
816 * reqested number of bytes. (For those of us who expect unwieldly option
817 * headers.
818 *
819 * KEBE SAYS: Remember that dtom() calls with data in clusters does not work!
820 */
821 struct mbuf *
822 m_pullup2(struct mbuf *n, int len)
823 {
824 struct mbuf *m;
825 int count;
826
827 if (len <= MHLEN)
828 return m_pullup(n, len);
829 if ((n->m_flags & M_EXT) != 0 &&
830 n->m_data + len < &n->m_data[MCLBYTES] && n->m_next) {
831 if (n->m_len >= len)
832 return (n);
833 m = n;
834 n = n->m_next;
835 len -= m->m_len;
836 } else {
837 if (len > MCLBYTES)
838 goto bad;
839 MGET(m, M_DONTWAIT, n->m_type);
840 if (m == NULL)
841 goto bad;
842 MCLGET(m, M_DONTWAIT);
843 if ((m->m_flags & M_EXT) == 0) {
844 m_free(m);
845 goto bad;
846 }
847 m->m_len = 0;
848 if (n->m_flags & M_PKTHDR) {
849 /* Too many adverse side effects. */
850 /* M_MOVE_PKTHDR(m, n); */
851 m->m_flags = (n->m_flags & M_COPYFLAGS) |
852 M_EXT | M_CLUSTER;
853 M_MOVE_HDR(m, n);
854 /* n->m_data is cool. */
855 }
856 }
857
858 do {
859 count = min(len, n->m_len);
860 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
861 (unsigned)count);
862 len -= count;
863 m->m_len += count;
864 n->m_len -= count;
865 if (n->m_len)
866 n->m_data += count;
867 else
868 n = m_free(n);
869 } while (len > 0 && n);
870 if (len > 0) {
871 (void)m_free(m);
872 goto bad;
873 }
874 m->m_next = n;
875
876 return (m);
877 bad:
878 m_freem(n);
879 MPFail++;
880 return (NULL);
881 }
882
883 /*
884 * Return a pointer to mbuf/offset of location in mbuf chain.
885 */
886 struct mbuf *
887 m_getptr(struct mbuf *m, int loc, int *off)
888 {
889 while (loc >= 0) {
890 /* Normal end of search */
891 if (m->m_len > loc) {
892 *off = loc;
893 return (m);
894 }
895 else {
896 loc -= m->m_len;
897
898 if (m->m_next == NULL) {
899 if (loc == 0) {
900 /* Point at the end of valid data */
901 *off = m->m_len;
902 return (m);
903 }
904 else
905 return (NULL);
906 } else
907 m = m->m_next;
908 }
909 }
910
911 return (NULL);
912 }
913
914 /*
915 * Inject a new mbuf chain of length siz in mbuf chain m0 at
916 * position len0. Returns a pointer to the first injected mbuf, or
917 * NULL on failure (m0 is left undisturbed). Note that if there is
918 * enough space for an object of size siz in the appropriate position,
919 * no memory will be allocated. Also, there will be no data movement in
920 * the first len0 bytes (pointers to that will remain valid).
921 *
922 * XXX It is assumed that siz is less than the size of an mbuf at the moment.
923 */
924 struct mbuf *
925 m_inject(struct mbuf *m0, int len0, int siz, int wait)
926 {
927 struct mbuf *m, *n, *n2 = NULL, *n3;
928 unsigned len = len0, remain;
929
930 if ((siz >= MHLEN) || (len0 <= 0))
931 return (NULL);
932 for (m = m0; m && len > m->m_len; m = m->m_next)
933 len -= m->m_len;
934 if (m == NULL)
935 return (NULL);
936 remain = m->m_len - len;
937 if (remain == 0) {
938 if ((m->m_next) && (M_LEADINGSPACE(m->m_next) >= siz)) {
939 m->m_next->m_len += siz;
940 if (m0->m_flags & M_PKTHDR)
941 m0->m_pkthdr.len += siz;
942 m->m_next->m_data -= siz;
943 return m->m_next;
944 }
945 } else {
946 n2 = m_copym2(m, len, remain, wait);
947 if (n2 == NULL)
948 return (NULL);
949 }
950
951 MGET(n, wait, MT_DATA);
952 if (n == NULL) {
953 if (n2)
954 m_freem(n2);
955 return (NULL);
956 }
957
958 n->m_len = siz;
959 if (m0->m_flags & M_PKTHDR)
960 m0->m_pkthdr.len += siz;
961 m->m_len -= remain; /* Trim */
962 if (n2) {
963 for (n3 = n; n3->m_next != NULL; n3 = n3->m_next)
964 ;
965 n3->m_next = n2;
966 } else
967 n3 = n;
968 for (; n3->m_next != NULL; n3 = n3->m_next)
969 ;
970 n3->m_next = m->m_next;
971 m->m_next = n;
972 return n;
973 }
974
975 /*
976 * Partition an mbuf chain in two pieces, returning the tail --
977 * all but the first len0 bytes. In case of failure, it returns NULL and
978 * attempts to restore the chain to its original state.
979 */
980 struct mbuf *
981 m_split(struct mbuf *m0, int len0, int wait)
982 {
983 struct mbuf *m, *n;
984 unsigned len = len0, remain, olen;
985
986 for (m = m0; m && len > m->m_len; m = m->m_next)
987 len -= m->m_len;
988 if (m == NULL)
989 return (NULL);
990 remain = m->m_len - len;
991 if (m0->m_flags & M_PKTHDR) {
992 MGETHDR(n, wait, m0->m_type);
993 if (n == NULL)
994 return (NULL);
995 M_DUP_PKTHDR(n, m0);
996 n->m_pkthdr.len -= len0;
997 olen = m0->m_pkthdr.len;
998 m0->m_pkthdr.len = len0;
999 if (m->m_flags & M_EXT)
1000 goto extpacket;
1001 if (remain > MHLEN) {
1002 /* m can't be the lead packet */
1003 MH_ALIGN(n, 0);
1004 n->m_next = m_split(m, len, wait);
1005 if (n->m_next == NULL) {
1006 (void) m_free(n);
1007 m0->m_pkthdr.len = olen;
1008 return (NULL);
1009 } else
1010 return (n);
1011 } else
1012 MH_ALIGN(n, remain);
1013 } else if (remain == 0) {
1014 n = m->m_next;
1015 m->m_next = NULL;
1016 return (n);
1017 } else {
1018 MGET(n, wait, m->m_type);
1019 if (n == NULL)
1020 return (NULL);
1021 M_ALIGN(n, remain);
1022 }
1023 extpacket:
1024 if (m->m_flags & M_EXT) {
1025 n->m_ext = m->m_ext;
1026 MCLADDREFERENCE(m, n);
1027 n->m_data = m->m_data + len;
1028 } else {
1029 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
1030 }
1031 n->m_len = remain;
1032 m->m_len = len;
1033 n->m_next = m->m_next;
1034 m->m_next = NULL;
1035 return (n);
1036 }
1037
1038 /*
1039 * Routine to copy from device local memory into mbufs.
1040 */
1041 struct mbuf *
1042 m_devget(char *buf, int totlen, int off, struct ifnet *ifp,
1043 void (*copy)(const void *, void *, size_t))
1044 {
1045 struct mbuf *m;
1046 struct mbuf *top, **mp;
1047 int len;
1048
1049 top = NULL;
1050 mp = ⊤
1051
1052 if (off < 0 || off > MHLEN)
1053 return (NULL);
1054
1055 MGETHDR(m, M_DONTWAIT, MT_DATA);
1056 if (m == NULL)
1057 return (NULL);
1058
1059 m->m_pkthdr.rcvif = ifp;
1060 m->m_pkthdr.len = totlen;
1061
1062 len = MHLEN;
1063
1064 while (totlen > 0) {
1065 if (top != NULL) {
1066 MGET(m, M_DONTWAIT, MT_DATA);
1067 if (m == NULL) {
1068 m_freem(top);
1069 return (NULL);
1070 }
1071 len = MLEN;
1072 }
1073
1074 if (totlen + off >= MINCLSIZE) {
1075 MCLGET(m, M_DONTWAIT);
1076 if (m->m_flags & M_EXT)
1077 len = MCLBYTES;
1078 } else {
1079 /* Place initial small packet/header at end of mbuf. */
1080 if (top == NULL && totlen + off + max_linkhdr <= len) {
1081 m->m_data += max_linkhdr;
1082 len -= max_linkhdr;
1083 }
1084 }
1085
1086 if (off) {
1087 m->m_data += off;
1088 len -= off;
1089 off = 0;
1090 }
1091
1092 m->m_len = len = min(totlen, len);
1093
1094 if (copy)
1095 copy(buf, mtod(m, caddr_t), (size_t)len);
1096 else
1097 bcopy(buf, mtod(m, caddr_t), (size_t)len);
1098
1099 buf += len;
1100 *mp = m;
1101 mp = &m->m_next;
1102 totlen -= len;
1103 }
1104 return (top);
1105 }
1106
1107 void
1108 m_zero(struct mbuf *m)
1109 {
1110 while (m) {
1111 #ifdef DIAGNOSTIC
1112 if (M_READONLY(m))
1113 panic("m_zero: M_READONLY");
1114 #endif /* DIAGNOSTIC */
1115 if (m->m_flags & M_EXT)
1116 memset(m->m_ext.ext_buf, 0, m->m_ext.ext_size);
1117 else {
1118 if (m->m_flags & M_PKTHDR)
1119 memset(m->m_pktdat, 0, MHLEN);
1120 else
1121 memset(m->m_dat, 0, MLEN);
1122 }
1123 m = m->m_next;
1124 }
1125 }
1126
1127 /*
1128 * Apply function f to the data in an mbuf chain starting "off" bytes from the
1129 * beginning, continuing for "len" bytes.
1130 */
1131 int
1132 m_apply(struct mbuf *m, int off, int len,
1133 int (*f)(caddr_t, caddr_t, unsigned int), caddr_t fstate)
1134 {
1135 int rval;
1136 unsigned int count;
1137
1138 if (len < 0)
1139 panic("m_apply: len %d < 0", len);
1140 if (off < 0)
1141 panic("m_apply: off %d < 0", off);
1142 while (off > 0) {
1143 if (m == NULL)
1144 panic("m_apply: null mbuf in skip");
1145 if (off < m->m_len)
1146 break;
1147 off -= m->m_len;
1148 m = m->m_next;
1149 }
1150 while (len > 0) {
1151 if (m == NULL)
1152 panic("m_apply: null mbuf");
1153 count = min(m->m_len - off, len);
1154
1155 rval = f(fstate, mtod(m, caddr_t) + off, count);
1156 if (rval)
1157 return (rval);
1158
1159 len -= count;
1160 off = 0;
1161 m = m->m_next;
1162 }
1163
1164 return (0);
1165 }
1166
1167 int
1168 m_leadingspace(struct mbuf *m)
1169 {
1170 if (M_READONLY((m)))
1171 return 0;
1172 return ((m)->m_flags & M_EXT ? (m)->m_data - (m)->m_ext.ext_buf :
1173 (m)->m_flags & M_PKTHDR ? (m)->m_data - (m)->m_pktdat :
1174 (m)->m_data - (m)->m_dat);
1175 }
1176
1177 int
1178 m_trailingspace(struct mbuf *m)
1179 {
1180 if (M_READONLY(m))
1181 return 0;
1182 return ((m)->m_flags & M_EXT ? (m)->m_ext.ext_buf +
1183 (m)->m_ext.ext_size - ((m)->m_data + (m)->m_len) :
1184 &(m)->m_dat[MLEN] - ((m)->m_data + (m)->m_len));
1185 }
Cache object: 5f55485f2373ced379032727b12a006f
|