FreeBSD/Linux Kernel Cross Reference
sys/kern/uipc_mbuf.c
1 /* uipc_mbuf.c,v 1.84 2004/07/21 12:06:46 yamt Exp */
2
3 /*-
4 * Copyright (c) 1999, 2001 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9 * NASA Ames Research Center.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. All advertising materials mentioning features or use of this software
20 * must display the following acknowledgement:
21 * This product includes software developed by the NetBSD
22 * Foundation, Inc. and its contributors.
23 * 4. Neither the name of The NetBSD Foundation nor the names of its
24 * contributors may be used to endorse or promote products derived
25 * from this software without specific prior written permission.
26 *
27 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
29 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
31 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
37 * POSSIBILITY OF SUCH DAMAGE.
38 */
39
40 /*
41 * Copyright (c) 1982, 1986, 1988, 1991, 1993
42 * The Regents of the University of California. All rights reserved.
43 *
44 * Redistribution and use in source and binary forms, with or without
45 * modification, are permitted provided that the following conditions
46 * are met:
47 * 1. Redistributions of source code must retain the above copyright
48 * notice, this list of conditions and the following disclaimer.
49 * 2. Redistributions in binary form must reproduce the above copyright
50 * notice, this list of conditions and the following disclaimer in the
51 * documentation and/or other materials provided with the distribution.
52 * 3. Neither the name of the University nor the names of its contributors
53 * may be used to endorse or promote products derived from this software
54 * without specific prior written permission.
55 *
56 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
57 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
58 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
59 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
60 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
61 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
62 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
63 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
64 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
65 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
66 * SUCH DAMAGE.
67 *
68 * @(#)uipc_mbuf.c 8.4 (Berkeley) 2/14/95
69 */
70
71 #include <sys/cdefs.h>
72 __KERNEL_RCSID(0, "$NetBSD: uipc_mbuf.c,v 1.80.2.5 2004/10/08 03:05:26 jmc Exp $");
73
74 #include "opt_mbuftrace.h"
75
76 #include <sys/param.h>
77 #include <sys/systm.h>
78 #include <sys/proc.h>
79 #include <sys/malloc.h>
80 #define MBTYPES
81 #include <sys/mbuf.h>
82 #include <sys/kernel.h>
83 #include <sys/syslog.h>
84 #include <sys/domain.h>
85 #include <sys/protosw.h>
86 #include <sys/pool.h>
87 #include <sys/socket.h>
88 #include <sys/sysctl.h>
89
90 #include <net/if.h>
91
92 #include <uvm/uvm.h>
93
94
95 struct pool mbpool; /* mbuf pool */
96 struct pool mclpool; /* mbuf cluster pool */
97
98 struct pool_cache mbpool_cache;
99 struct pool_cache mclpool_cache;
100
101 struct mbstat mbstat;
102 int max_linkhdr;
103 int max_protohdr;
104 int max_hdr;
105 int max_datalen;
106
107 static int mb_ctor(void *, void *, int);
108
109 void *mclpool_alloc(struct pool *, int);
110 void mclpool_release(struct pool *, void *);
111
112 struct pool_allocator mclpool_allocator = {
113 mclpool_alloc, mclpool_release, 0,
114 };
115
116 static struct mbuf *m_copym0(struct mbuf *, int, int, int, int);
117 static struct mbuf *m_split0(struct mbuf *, int, int, int);
118 static int m_copyback0(struct mbuf **, int, int, caddr_t, int, int);
119
120 /* flags for m_copyback0 */
121 #define M_COPYBACK0_COPYBACK 0x0001 /* copyback from cp */
122 #define M_COPYBACK0_PRESERVE 0x0002 /* preserve original data */
123 #define M_COPYBACK0_COW 0x0004 /* do copy-on-write */
124 #define M_COPYBACK0_EXTEND 0x0008 /* extend chain */
125
126 const char mclpool_warnmsg[] =
127 "WARNING: mclpool limit reached; increase NMBCLUSTERS";
128
129 MALLOC_DEFINE(M_MBUF, "mbuf", "mbuf");
130
131 #ifdef MBUFTRACE
132 struct mownerhead mowners = LIST_HEAD_INITIALIZER(mowners);
133 struct mowner unknown_mowners[] = {
134 { "unknown", "free" },
135 { "unknown", "data" },
136 { "unknown", "header" },
137 { "unknown", "soname" },
138 { "unknown", "soopts" },
139 { "unknown", "ftable" },
140 { "unknown", "control" },
141 { "unknown", "oobdata" },
142 };
143 struct mowner revoked_mowner = { "revoked", "" };
144 #endif
145
146 /*
147 * Initialize the mbuf allocator.
148 */
149 void
150 mbinit(void)
151 {
152
153 KASSERT(sizeof(struct _m_ext) <= MHLEN);
154 KASSERT(sizeof(struct mbuf) == MSIZE);
155
156 pool_init(&mbpool, msize, 0, 0, 0, "mbpl", NULL);
157 pool_init(&mclpool, mclbytes, 0, 0, 0, "mclpl", &mclpool_allocator);
158
159 pool_set_drain_hook(&mbpool, m_reclaim, NULL);
160 pool_set_drain_hook(&mclpool, m_reclaim, NULL);
161
162 pool_cache_init(&mbpool_cache, &mbpool, mb_ctor, NULL, NULL);
163 pool_cache_init(&mclpool_cache, &mclpool, NULL, NULL, NULL);
164
165 /*
166 * Set the hard limit on the mclpool to the number of
167 * mbuf clusters the kernel is to support. Log the limit
168 * reached message max once a minute.
169 */
170 pool_sethardlimit(&mclpool, nmbclusters, mclpool_warnmsg, 60);
171
172 /*
173 * Set a low water mark for both mbufs and clusters. This should
174 * help ensure that they can be allocated in a memory starvation
175 * situation. This is important for e.g. diskless systems which
176 * must allocate mbufs in order for the pagedaemon to clean pages.
177 */
178 pool_setlowat(&mbpool, mblowat);
179 pool_setlowat(&mclpool, mcllowat);
180
181 #ifdef MBUFTRACE
182 {
183 /*
184 * Attach the unknown mowners.
185 */
186 int i;
187 MOWNER_ATTACH(&revoked_mowner);
188 for (i = sizeof(unknown_mowners)/sizeof(unknown_mowners[0]);
189 i-- > 0; )
190 MOWNER_ATTACH(&unknown_mowners[i]);
191 }
192 #endif
193 }
194
195 /*
196 * sysctl helper routine for the kern.mbuf subtree. nmbclusters may
197 * or may not be writable, and mblowat and mcllowat need range
198 * checking and pool tweaking after being reset.
199 */
200 static int
201 sysctl_kern_mbuf(SYSCTLFN_ARGS)
202 {
203 int error, newval;
204 struct sysctlnode node;
205
206 node = *rnode;
207 node.sysctl_data = &newval;
208 switch (rnode->sysctl_num) {
209 case MBUF_NMBCLUSTERS:
210 if (mb_map != NULL) {
211 node.sysctl_flags &= ~CTLFLAG_READWRITE;
212 node.sysctl_flags |= CTLFLAG_READONLY;
213 }
214 /* FALLTHROUGH */
215 case MBUF_MBLOWAT:
216 case MBUF_MCLLOWAT:
217 newval = *(int*)rnode->sysctl_data;
218 break;
219 default:
220 return (EOPNOTSUPP);
221 }
222
223 error = sysctl_lookup(SYSCTLFN_CALL(&node));
224 if (error || newp == NULL)
225 return (error);
226 if (newval < 0)
227 return (EINVAL);
228
229 switch (node.sysctl_num) {
230 case MBUF_NMBCLUSTERS:
231 if (newval < nmbclusters)
232 return (EINVAL);
233 nmbclusters = newval;
234 pool_sethardlimit(&mclpool, nmbclusters, mclpool_warnmsg, 60);
235 break;
236 case MBUF_MBLOWAT:
237 mblowat = newval;
238 pool_setlowat(&mbpool, mblowat);
239 break;
240 case MBUF_MCLLOWAT:
241 mcllowat = newval;
242 pool_setlowat(&mclpool, mcllowat);
243 break;
244 }
245
246 return (0);
247 }
248
249 #ifdef MBUFTRACE
250 static int
251 sysctl_kern_mbuf_mowners(SYSCTLFN_ARGS)
252 {
253 struct mowner *mo;
254 size_t len = 0;
255 int error = 0;
256
257 if (namelen != 0)
258 return (EINVAL);
259 if (newp != NULL)
260 return (EPERM);
261
262 LIST_FOREACH(mo, &mowners, mo_link) {
263 if (oldp != NULL) {
264 if (*oldlenp - len < sizeof(*mo)) {
265 error = ENOMEM;
266 break;
267 }
268 error = copyout(mo, (caddr_t) oldp + len,
269 sizeof(*mo));
270 if (error)
271 break;
272 }
273 len += sizeof(*mo);
274 }
275
276 if (error == 0)
277 *oldlenp = len;
278
279 return (error);
280 }
281 #endif /* MBUFTRACE */
282
283 SYSCTL_SETUP(sysctl_kern_mbuf_setup, "sysctl kern.mbuf subtree setup")
284 {
285
286 sysctl_createv(clog, 0, NULL, NULL,
287 CTLFLAG_PERMANENT,
288 CTLTYPE_NODE, "kern", NULL,
289 NULL, 0, NULL, 0,
290 CTL_KERN, CTL_EOL);
291 sysctl_createv(clog, 0, NULL, NULL,
292 CTLFLAG_PERMANENT,
293 CTLTYPE_NODE, "mbuf",
294 SYSCTL_DESCR("mbuf control variables"),
295 NULL, 0, NULL, 0,
296 CTL_KERN, KERN_MBUF, CTL_EOL);
297
298 sysctl_createv(clog, 0, NULL, NULL,
299 CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE,
300 CTLTYPE_INT, "msize",
301 SYSCTL_DESCR("mbuf base size"),
302 NULL, msize, NULL, 0,
303 CTL_KERN, KERN_MBUF, MBUF_MSIZE, CTL_EOL);
304 sysctl_createv(clog, 0, NULL, NULL,
305 CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE,
306 CTLTYPE_INT, "mclbytes",
307 SYSCTL_DESCR("mbuf cluster size"),
308 NULL, mclbytes, NULL, 0,
309 CTL_KERN, KERN_MBUF, MBUF_MCLBYTES, CTL_EOL);
310 sysctl_createv(clog, 0, NULL, NULL,
311 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
312 CTLTYPE_INT, "nmbclusters",
313 SYSCTL_DESCR("Limit on the number of mbuf clusters"),
314 sysctl_kern_mbuf, 0, &nmbclusters, 0,
315 CTL_KERN, KERN_MBUF, MBUF_NMBCLUSTERS, CTL_EOL);
316 sysctl_createv(clog, 0, NULL, NULL,
317 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
318 CTLTYPE_INT, "mblowat",
319 SYSCTL_DESCR("mbuf low water mark"),
320 sysctl_kern_mbuf, 0, &mblowat, 0,
321 CTL_KERN, KERN_MBUF, MBUF_MBLOWAT, CTL_EOL);
322 sysctl_createv(clog, 0, NULL, NULL,
323 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
324 CTLTYPE_INT, "mcllowat",
325 SYSCTL_DESCR("mbuf cluster low water mark"),
326 sysctl_kern_mbuf, 0, &mcllowat, 0,
327 CTL_KERN, KERN_MBUF, MBUF_MCLLOWAT, CTL_EOL);
328 sysctl_createv(clog, 0, NULL, NULL,
329 CTLFLAG_PERMANENT,
330 CTLTYPE_STRUCT, "stats",
331 SYSCTL_DESCR("mbuf allocation statistics"),
332 NULL, 0, &mbstat, sizeof(mbstat),
333 CTL_KERN, KERN_MBUF, MBUF_STATS, CTL_EOL);
334 #ifdef MBUFTRACE
335 sysctl_createv(clog, 0, NULL, NULL,
336 CTLFLAG_PERMANENT,
337 CTLTYPE_STRUCT, "mowners",
338 SYSCTL_DESCR("Information about mbuf owners"),
339 sysctl_kern_mbuf_mowners, 0, NULL, 0,
340 CTL_KERN, KERN_MBUF, MBUF_MOWNERS, CTL_EOL);
341 #endif /* MBUFTRACE */
342 }
343
344 void *
345 mclpool_alloc(struct pool *pp, int flags)
346 {
347 boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE;
348
349 return ((void *)uvm_km_alloc_poolpage1(mb_map, NULL, waitok));
350 }
351
352 void
353 mclpool_release(struct pool *pp, void *v)
354 {
355
356 uvm_km_free_poolpage1(mb_map, (vaddr_t)v);
357 }
358
359 /*ARGSUSED*/
360 static int
361 mb_ctor(void *arg, void *object, int flags)
362 {
363 struct mbuf *m = object;
364
365 #ifdef POOL_VTOPHYS
366 m->m_paddr = POOL_VTOPHYS(m);
367 #else
368 m->m_paddr = M_PADDR_INVALID;
369 #endif
370 return (0);
371 }
372
373 void
374 m_reclaim(void *arg, int flags)
375 {
376 struct domain *dp;
377 struct protosw *pr;
378 struct ifnet *ifp;
379 int s = splvm();
380
381 for (dp = domains; dp; dp = dp->dom_next)
382 for (pr = dp->dom_protosw;
383 pr < dp->dom_protoswNPROTOSW; pr++)
384 if (pr->pr_drain)
385 (*pr->pr_drain)();
386 for (ifp = TAILQ_FIRST(&ifnet); ifp; ifp = TAILQ_NEXT(ifp, if_list))
387 if (ifp->if_drain)
388 (*ifp->if_drain)(ifp);
389 splx(s);
390 mbstat.m_drain++;
391 }
392
393 /*
394 * Space allocation routines.
395 * These are also available as macros
396 * for critical paths.
397 */
398 struct mbuf *
399 m_get(int nowait, int type)
400 {
401 struct mbuf *m;
402
403 MGET(m, nowait, type);
404 return (m);
405 }
406
407 struct mbuf *
408 m_gethdr(int nowait, int type)
409 {
410 struct mbuf *m;
411
412 MGETHDR(m, nowait, type);
413 return (m);
414 }
415
416 struct mbuf *
417 m_getclr(int nowait, int type)
418 {
419 struct mbuf *m;
420
421 MGET(m, nowait, type);
422 if (m == 0)
423 return (NULL);
424 memset(mtod(m, caddr_t), 0, MLEN);
425 return (m);
426 }
427
428 void
429 m_clget(struct mbuf *m, int nowait)
430 {
431
432 MCLGET(m, nowait);
433 }
434
435 struct mbuf *
436 m_free(struct mbuf *m)
437 {
438 struct mbuf *n;
439
440 MFREE(m, n);
441 return (n);
442 }
443
444 void
445 m_freem(struct mbuf *m)
446 {
447 struct mbuf *n;
448
449 if (m == NULL)
450 return;
451 do {
452 MFREE(m, n);
453 m = n;
454 } while (m);
455 }
456
457 #ifdef MBUFTRACE
458 /*
459 * Walk a chain of mbufs, claiming ownership of each mbuf in the chain.
460 */
461 void
462 m_claimm(struct mbuf *m, struct mowner *mo)
463 {
464
465 for (; m != NULL; m = m->m_next)
466 MCLAIM(m, mo);
467 }
468 #endif
469
470 /*
471 * Mbuffer utility routines.
472 */
473
474 /*
475 * Lesser-used path for M_PREPEND:
476 * allocate new mbuf to prepend to chain,
477 * copy junk along.
478 */
479 struct mbuf *
480 m_prepend(struct mbuf *m, int len, int how)
481 {
482 struct mbuf *mn;
483
484 MGET(mn, how, m->m_type);
485 if (mn == (struct mbuf *)NULL) {
486 m_freem(m);
487 return ((struct mbuf *)NULL);
488 }
489 if (m->m_flags & M_PKTHDR) {
490 M_COPY_PKTHDR(mn, m);
491 m_tag_delete_chain(m, NULL);
492 m->m_flags &= ~M_PKTHDR;
493 } else {
494 MCLAIM(mn, m->m_owner);
495 }
496 mn->m_next = m;
497 m = mn;
498 if (len < MHLEN)
499 MH_ALIGN(m, len);
500 m->m_len = len;
501 return (m);
502 }
503
504 /*
505 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
506 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
507 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
508 */
509 int MCFail;
510
511 struct mbuf *
512 m_copym(struct mbuf *m, int off0, int len, int wait)
513 {
514
515 return m_copym0(m, off0, len, wait, 0); /* shallow copy on M_EXT */
516 }
517
518 struct mbuf *
519 m_dup(struct mbuf *m, int off0, int len, int wait)
520 {
521
522 return m_copym0(m, off0, len, wait, 1); /* deep copy */
523 }
524
525 static struct mbuf *
526 m_copym0(struct mbuf *m, int off0, int len, int wait, int deep)
527 {
528 struct mbuf *n, **np;
529 int off = off0;
530 struct mbuf *top;
531 int copyhdr = 0;
532
533 if (off < 0 || len < 0)
534 panic("m_copym: off %d, len %d", off, len);
535 if (off == 0 && m->m_flags & M_PKTHDR)
536 copyhdr = 1;
537 while (off > 0) {
538 if (m == 0)
539 panic("m_copym: m == 0");
540 if (off < m->m_len)
541 break;
542 off -= m->m_len;
543 m = m->m_next;
544 }
545 np = ⊤
546 top = 0;
547 while (len > 0) {
548 if (m == 0) {
549 if (len != M_COPYALL)
550 panic("m_copym: m == 0 and not COPYALL");
551 break;
552 }
553 MGET(n, wait, m->m_type);
554 *np = n;
555 if (n == 0)
556 goto nospace;
557 MCLAIM(n, m->m_owner);
558 if (copyhdr) {
559 M_COPY_PKTHDR(n, m);
560 if (len == M_COPYALL)
561 n->m_pkthdr.len -= off0;
562 else
563 n->m_pkthdr.len = len;
564 copyhdr = 0;
565 }
566 n->m_len = min(len, m->m_len - off);
567 if (m->m_flags & M_EXT) {
568 if (!deep) {
569 n->m_data = m->m_data + off;
570 n->m_ext = m->m_ext;
571 MCLADDREFERENCE(m, n);
572 } else {
573 /*
574 * we are unsure about the way m was allocated.
575 * copy into multiple MCLBYTES cluster mbufs.
576 */
577 MCLGET(n, wait);
578 n->m_len = 0;
579 n->m_len = M_TRAILINGSPACE(n);
580 n->m_len = min(n->m_len, len);
581 n->m_len = min(n->m_len, m->m_len - off);
582 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off,
583 (unsigned)n->m_len);
584 }
585 } else
586 memcpy(mtod(n, caddr_t), mtod(m, caddr_t)+off,
587 (unsigned)n->m_len);
588 if (len != M_COPYALL)
589 len -= n->m_len;
590 off += n->m_len;
591 #ifdef DIAGNOSTIC
592 if (off > m->m_len)
593 panic("m_copym0 overrun");
594 #endif
595 if (off == m->m_len) {
596 m = m->m_next;
597 off = 0;
598 }
599 np = &n->m_next;
600 }
601 if (top == 0)
602 MCFail++;
603 return (top);
604 nospace:
605 m_freem(top);
606 MCFail++;
607 return (NULL);
608 }
609
610 /*
611 * Copy an entire packet, including header (which must be present).
612 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'.
613 */
614 struct mbuf *
615 m_copypacket(struct mbuf *m, int how)
616 {
617 struct mbuf *top, *n, *o;
618
619 MGET(n, how, m->m_type);
620 top = n;
621 if (!n)
622 goto nospace;
623
624 MCLAIM(n, m->m_owner);
625 M_COPY_PKTHDR(n, m);
626 n->m_len = m->m_len;
627 if (m->m_flags & M_EXT) {
628 n->m_data = m->m_data;
629 n->m_ext = m->m_ext;
630 MCLADDREFERENCE(m, n);
631 } else {
632 memcpy(mtod(n, char *), mtod(m, char *), n->m_len);
633 }
634
635 m = m->m_next;
636 while (m) {
637 MGET(o, how, m->m_type);
638 if (!o)
639 goto nospace;
640
641 MCLAIM(o, m->m_owner);
642 n->m_next = o;
643 n = n->m_next;
644
645 n->m_len = m->m_len;
646 if (m->m_flags & M_EXT) {
647 n->m_data = m->m_data;
648 n->m_ext = m->m_ext;
649 MCLADDREFERENCE(m, n);
650 } else {
651 memcpy(mtod(n, char *), mtod(m, char *), n->m_len);
652 }
653
654 m = m->m_next;
655 }
656 return top;
657 nospace:
658 m_freem(top);
659 MCFail++;
660 return NULL;
661 }
662
663 /*
664 * Copy data from an mbuf chain starting "off" bytes from the beginning,
665 * continuing for "len" bytes, into the indicated buffer.
666 */
667 void
668 m_copydata(struct mbuf *m, int off, int len, caddr_t cp)
669 {
670 unsigned count;
671
672 if (off < 0 || len < 0)
673 panic("m_copydata");
674 while (off > 0) {
675 if (m == 0)
676 panic("m_copydata");
677 if (off < m->m_len)
678 break;
679 off -= m->m_len;
680 m = m->m_next;
681 }
682 while (len > 0) {
683 if (m == 0)
684 panic("m_copydata");
685 count = min(m->m_len - off, len);
686 memcpy(cp, mtod(m, caddr_t) + off, count);
687 len -= count;
688 cp += count;
689 off = 0;
690 m = m->m_next;
691 }
692 }
693
694 /*
695 * Concatenate mbuf chain n to m.
696 * n might be copied into m (when n->m_len is small), therefore data portion of
697 * n could be copied into an mbuf of different mbuf type.
698 * Any m_pkthdr is not updated.
699 */
700 void
701 m_cat(struct mbuf *m, struct mbuf *n)
702 {
703
704 while (m->m_next)
705 m = m->m_next;
706 while (n) {
707 if (M_READONLY(m) || n->m_len > M_TRAILINGSPACE(m)) {
708 /* just join the two chains */
709 m->m_next = n;
710 return;
711 }
712 /* splat the data from one into the other */
713 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t),
714 (u_int)n->m_len);
715 m->m_len += n->m_len;
716 n = m_free(n);
717 }
718 }
719
720 void
721 m_adj(struct mbuf *mp, int req_len)
722 {
723 int len = req_len;
724 struct mbuf *m;
725 int count;
726
727 if ((m = mp) == NULL)
728 return;
729 if (len >= 0) {
730 /*
731 * Trim from head.
732 */
733 while (m != NULL && len > 0) {
734 if (m->m_len <= len) {
735 len -= m->m_len;
736 m->m_len = 0;
737 m = m->m_next;
738 } else {
739 m->m_len -= len;
740 m->m_data += len;
741 len = 0;
742 }
743 }
744 m = mp;
745 if (mp->m_flags & M_PKTHDR)
746 m->m_pkthdr.len -= (req_len - len);
747 } else {
748 /*
749 * Trim from tail. Scan the mbuf chain,
750 * calculating its length and finding the last mbuf.
751 * If the adjustment only affects this mbuf, then just
752 * adjust and return. Otherwise, rescan and truncate
753 * after the remaining size.
754 */
755 len = -len;
756 count = 0;
757 for (;;) {
758 count += m->m_len;
759 if (m->m_next == (struct mbuf *)0)
760 break;
761 m = m->m_next;
762 }
763 if (m->m_len >= len) {
764 m->m_len -= len;
765 if (mp->m_flags & M_PKTHDR)
766 mp->m_pkthdr.len -= len;
767 return;
768 }
769 count -= len;
770 if (count < 0)
771 count = 0;
772 /*
773 * Correct length for chain is "count".
774 * Find the mbuf with last data, adjust its length,
775 * and toss data from remaining mbufs on chain.
776 */
777 m = mp;
778 if (m->m_flags & M_PKTHDR)
779 m->m_pkthdr.len = count;
780 for (; m; m = m->m_next) {
781 if (m->m_len >= count) {
782 m->m_len = count;
783 break;
784 }
785 count -= m->m_len;
786 }
787 while (m->m_next)
788 (m = m->m_next) ->m_len = 0;
789 }
790 }
791
792 /*
793 * Rearange an mbuf chain so that len bytes are contiguous
794 * and in the data area of an mbuf (so that mtod and dtom
795 * will work for a structure of size len). Returns the resulting
796 * mbuf chain on success, frees it and returns null on failure.
797 * If there is room, it will add up to max_protohdr-len extra bytes to the
798 * contiguous region in an attempt to avoid being called next time.
799 */
800 int MPFail;
801
802 struct mbuf *
803 m_pullup(struct mbuf *n, int len)
804 {
805 struct mbuf *m;
806 int count;
807 int space;
808
809 /*
810 * If first mbuf has no cluster, and has room for len bytes
811 * without shifting current data, pullup into it,
812 * otherwise allocate a new mbuf to prepend to the chain.
813 */
814 if ((n->m_flags & M_EXT) == 0 &&
815 n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
816 if (n->m_len >= len)
817 return (n);
818 m = n;
819 n = n->m_next;
820 len -= m->m_len;
821 } else {
822 if (len > MHLEN)
823 goto bad;
824 MGET(m, M_DONTWAIT, n->m_type);
825 if (m == 0)
826 goto bad;
827 MCLAIM(m, n->m_owner);
828 m->m_len = 0;
829 if (n->m_flags & M_PKTHDR) {
830 M_COPY_PKTHDR(m, n);
831 m_tag_delete_chain(n, NULL);
832 n->m_flags &= ~M_PKTHDR;
833 }
834 }
835 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
836 do {
837 count = min(min(max(len, max_protohdr), space), n->m_len);
838 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t),
839 (unsigned)count);
840 len -= count;
841 m->m_len += count;
842 n->m_len -= count;
843 space -= count;
844 if (n->m_len)
845 n->m_data += count;
846 else
847 n = m_free(n);
848 } while (len > 0 && n);
849 if (len > 0) {
850 (void) m_free(m);
851 goto bad;
852 }
853 m->m_next = n;
854 return (m);
855 bad:
856 m_freem(n);
857 MPFail++;
858 return (NULL);
859 }
860
861 /*
862 * Like m_pullup(), except a new mbuf is always allocated, and we allow
863 * the amount of empty space before the data in the new mbuf to be specified
864 * (in the event that the caller expects to prepend later).
865 */
866 int MSFail;
867
868 struct mbuf *
869 m_copyup(struct mbuf *n, int len, int dstoff)
870 {
871 struct mbuf *m;
872 int count, space;
873
874 if (len > (MHLEN - dstoff))
875 goto bad;
876 MGET(m, M_DONTWAIT, n->m_type);
877 if (m == NULL)
878 goto bad;
879 MCLAIM(m, n->m_owner);
880 m->m_len = 0;
881 if (n->m_flags & M_PKTHDR) {
882 M_COPY_PKTHDR(m, n);
883 m_tag_delete_chain(m, NULL);
884 n->m_flags &= ~M_PKTHDR;
885 }
886 m->m_data += dstoff;
887 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
888 do {
889 count = min(min(max(len, max_protohdr), space), n->m_len);
890 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t),
891 (unsigned)count);
892 len -= count;
893 m->m_len += count;
894 n->m_len -= count;
895 space -= count;
896 if (n->m_len)
897 n->m_data += count;
898 else
899 n = m_free(n);
900 } while (len > 0 && n);
901 if (len > 0) {
902 (void) m_free(m);
903 goto bad;
904 }
905 m->m_next = n;
906 return (m);
907 bad:
908 m_freem(n);
909 MSFail++;
910 return (NULL);
911 }
912
913 /*
914 * Partition an mbuf chain in two pieces, returning the tail --
915 * all but the first len0 bytes. In case of failure, it returns NULL and
916 * attempts to restore the chain to its original state.
917 */
918 struct mbuf *
919 m_split(struct mbuf *m0, int len0, int wait)
920 {
921
922 return m_split0(m0, len0, wait, 1);
923 }
924
925 static struct mbuf *
926 m_split0(struct mbuf *m0, int len0, int wait, int copyhdr)
927 {
928 struct mbuf *m, *n;
929 unsigned len = len0, remain, len_save;
930
931 for (m = m0; m && len > m->m_len; m = m->m_next)
932 len -= m->m_len;
933 if (m == 0)
934 return (NULL);
935 remain = m->m_len - len;
936 if (copyhdr && (m0->m_flags & M_PKTHDR)) {
937 MGETHDR(n, wait, m0->m_type);
938 if (n == 0)
939 return (NULL);
940 MCLAIM(m, m0->m_owner);
941 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
942 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
943 len_save = m0->m_pkthdr.len;
944 m0->m_pkthdr.len = len0;
945 if (m->m_flags & M_EXT)
946 goto extpacket;
947 if (remain > MHLEN) {
948 /* m can't be the lead packet */
949 MH_ALIGN(n, 0);
950 n->m_next = m_split(m, len, wait);
951 if (n->m_next == 0) {
952 (void) m_free(n);
953 m0->m_pkthdr.len = len_save;
954 return (NULL);
955 } else
956 return (n);
957 } else
958 MH_ALIGN(n, remain);
959 } else if (remain == 0) {
960 n = m->m_next;
961 m->m_next = 0;
962 return (n);
963 } else {
964 MGET(n, wait, m->m_type);
965 if (n == 0)
966 return (NULL);
967 MCLAIM(n, m->m_owner);
968 M_ALIGN(n, remain);
969 }
970 extpacket:
971 if (m->m_flags & M_EXT) {
972 n->m_ext = m->m_ext;
973 MCLADDREFERENCE(m, n);
974 n->m_data = m->m_data + len;
975 } else {
976 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + len, remain);
977 }
978 n->m_len = remain;
979 m->m_len = len;
980 n->m_next = m->m_next;
981 m->m_next = 0;
982 return (n);
983 }
984 /*
985 * Routine to copy from device local memory into mbufs.
986 */
987 struct mbuf *
988 m_devget(char *buf, int totlen, int off0, struct ifnet *ifp,
989 void (*copy)(const void *from, void *to, size_t len))
990 {
991 struct mbuf *m;
992 struct mbuf *top = 0, **mp = ⊤
993 int off = off0, len;
994 char *cp;
995 char *epkt;
996
997 cp = buf;
998 epkt = cp + totlen;
999 if (off) {
1000 /*
1001 * If 'off' is non-zero, packet is trailer-encapsulated,
1002 * so we have to skip the type and length fields.
1003 */
1004 cp += off + 2 * sizeof(u_int16_t);
1005 totlen -= 2 * sizeof(u_int16_t);
1006 }
1007 MGETHDR(m, M_DONTWAIT, MT_DATA);
1008 if (m == 0)
1009 return (NULL);
1010 m->m_pkthdr.rcvif = ifp;
1011 m->m_pkthdr.len = totlen;
1012 m->m_len = MHLEN;
1013
1014 while (totlen > 0) {
1015 if (top) {
1016 MGET(m, M_DONTWAIT, MT_DATA);
1017 if (m == 0) {
1018 m_freem(top);
1019 return (NULL);
1020 }
1021 m->m_len = MLEN;
1022 }
1023 len = min(totlen, epkt - cp);
1024 if (len >= MINCLSIZE) {
1025 MCLGET(m, M_DONTWAIT);
1026 if ((m->m_flags & M_EXT) == 0) {
1027 m_free(m);
1028 m_freem(top);
1029 return (NULL);
1030 }
1031 m->m_len = len = min(len, MCLBYTES);
1032 } else {
1033 /*
1034 * Place initial small packet/header at end of mbuf.
1035 */
1036 if (len < m->m_len) {
1037 if (top == 0 && len + max_linkhdr <= m->m_len)
1038 m->m_data += max_linkhdr;
1039 m->m_len = len;
1040 } else
1041 len = m->m_len;
1042 }
1043 if (copy)
1044 copy(cp, mtod(m, caddr_t), (size_t)len);
1045 else
1046 memcpy(mtod(m, caddr_t), cp, (size_t)len);
1047 cp += len;
1048 *mp = m;
1049 mp = &m->m_next;
1050 totlen -= len;
1051 if (cp == epkt)
1052 cp = buf;
1053 }
1054 return (top);
1055 }
1056
1057 /*
1058 * Copy data from a buffer back into the indicated mbuf chain,
1059 * starting "off" bytes from the beginning, extending the mbuf
1060 * chain if necessary.
1061 */
1062 void
1063 m_copyback(struct mbuf *m0, int off, int len, caddr_t cp)
1064 {
1065 #if defined(DEBUG)
1066 struct mbuf *origm = m0;
1067 int error;
1068 #endif /* defined(DEBUG) */
1069
1070 if (m0 == NULL)
1071 return;
1072
1073 #if defined(DEBUG)
1074 error =
1075 #endif /* defined(DEBUG) */
1076 m_copyback0(&m0, off, len, cp,
1077 M_COPYBACK0_COPYBACK|M_COPYBACK0_EXTEND, M_DONTWAIT);
1078
1079 #if defined(DEBUG)
1080 if (error != 0 || (m0 != NULL && origm != m0))
1081 panic("m_copyback");
1082 #endif /* defined(DEBUG) */
1083 }
1084
1085 struct mbuf *
1086 m_copyback_cow(struct mbuf *m0, int off, int len, caddr_t cp, int how)
1087 {
1088 int error;
1089
1090 /* don't support chain expansion */
1091 KDASSERT(off + len <= m_length(m0));
1092
1093 error = m_copyback0(&m0, off, len, cp,
1094 M_COPYBACK0_COPYBACK|M_COPYBACK0_COW, how);
1095 if (error) {
1096 /*
1097 * no way to recover from partial success.
1098 * just free the chain.
1099 */
1100 m_freem(m0);
1101 return NULL;
1102 }
1103 return m0;
1104 }
1105
1106 /*
1107 * m_makewritable: ensure the specified range writable.
1108 */
1109 int
1110 m_makewritable(struct mbuf **mp, int off, int len, int how)
1111 {
1112 int error;
1113 #if defined(DEBUG)
1114 struct mbuf *n;
1115 int origlen, reslen;
1116
1117 origlen = m_length(*mp);
1118 #endif /* defined(DEBUG) */
1119
1120 #if 0 /* M_COPYALL is large enough */
1121 if (len == M_COPYALL)
1122 len = m_length(*mp) - off; /* XXX */
1123 #endif
1124
1125 error = m_copyback0(mp, off, len, NULL,
1126 M_COPYBACK0_PRESERVE|M_COPYBACK0_COW, how);
1127
1128 #if defined(DEBUG)
1129 reslen = 0;
1130 for (n = *mp; n; n = n->m_next)
1131 reslen += n->m_len;
1132 if (origlen != reslen)
1133 panic("m_makewritable: length changed");
1134 if (((*mp)->m_flags & M_PKTHDR) != 0 && reslen != (*mp)->m_pkthdr.len)
1135 panic("m_makewritable: inconsist");
1136 #endif /* defined(DEBUG) */
1137
1138 return error;
1139 }
1140
1141 int
1142 m_copyback0(struct mbuf **mp0, int off, int len, caddr_t cp, int flags, int how)
1143 {
1144 int mlen;
1145 struct mbuf *m, *n;
1146 struct mbuf **mp;
1147 int totlen = 0;
1148
1149 KASSERT(mp0 != NULL);
1150 KASSERT(*mp0 != NULL);
1151 KASSERT((flags & M_COPYBACK0_PRESERVE) == 0 || cp == NULL);
1152 KASSERT((flags & M_COPYBACK0_COPYBACK) == 0 || cp != NULL);
1153
1154 mp = mp0;
1155 m = *mp;
1156 while (off > (mlen = m->m_len)) {
1157 off -= mlen;
1158 totlen += mlen;
1159 if (m->m_next == 0) {
1160 if ((flags & M_COPYBACK0_EXTEND) == 0)
1161 goto out;
1162 n = m_getclr(how, m->m_type);
1163 if (n == 0)
1164 goto out;
1165 n->m_len = min(MLEN, len + off);
1166 m->m_next = n;
1167 }
1168 mp = &m->m_next;
1169 m = m->m_next;
1170 }
1171 while (len > 0) {
1172 mlen = m->m_len - off;
1173 if (mlen != 0 && M_READONLY(m)) {
1174 char *datap;
1175 int eatlen;
1176
1177 /*
1178 * this mbuf is read-only.
1179 * allocate a new writable mbuf and try again.
1180 */
1181
1182 #if defined(DIAGNOSTIC)
1183 if ((flags & M_COPYBACK0_COW) == 0)
1184 panic("m_copyback0: read-only");
1185 #endif /* defined(DIAGNOSTIC) */
1186
1187 /*
1188 * if we're going to write into the middle of
1189 * a mbuf, split it first.
1190 */
1191 if (off > 0 && len < mlen) {
1192 n = m_split0(m, off, how, 0);
1193 if (n == NULL)
1194 goto enobufs;
1195 m->m_next = n;
1196 mp = &m->m_next;
1197 m = n;
1198 off = 0;
1199 continue;
1200 }
1201
1202 /*
1203 * XXX TODO coalesce into the trailingspace of
1204 * the previous mbuf when possible.
1205 */
1206
1207 /*
1208 * allocate a new mbuf. copy packet header if needed.
1209 */
1210 MGET(n, how, m->m_type);
1211 if (n == NULL)
1212 goto enobufs;
1213 MCLAIM(n, m->m_owner);
1214 if (off == 0 && (m->m_flags & M_PKTHDR) != 0) {
1215 /* XXX M_MOVE_PKTHDR */
1216 M_COPY_PKTHDR(n, m);
1217 n->m_len = MHLEN;
1218 } else {
1219 if (len >= MINCLSIZE)
1220 MCLGET(n, M_DONTWAIT);
1221 n->m_len =
1222 (n->m_flags & M_EXT) ? MCLBYTES : MLEN;
1223 }
1224 if (n->m_len > len)
1225 n->m_len = len;
1226
1227 /*
1228 * free the region which has been overwritten.
1229 * copying data from old mbufs if requested.
1230 */
1231 if (flags & M_COPYBACK0_PRESERVE)
1232 datap = mtod(n, char *);
1233 else
1234 datap = NULL;
1235 eatlen = n->m_len;
1236 KDASSERT(off == 0 || eatlen >= mlen);
1237 if (off > 0) {
1238 KDASSERT(len >= mlen);
1239 m->m_len = off;
1240 m->m_next = n;
1241 if (datap) {
1242 m_copydata(m, off, mlen, datap);
1243 datap += mlen;
1244 }
1245 eatlen -= mlen;
1246 mp = &m->m_next;
1247 m = m->m_next;
1248 }
1249 while (m != NULL && M_READONLY(m) &&
1250 n->m_type == m->m_type && eatlen > 0) {
1251 mlen = min(eatlen, m->m_len);
1252 if (datap) {
1253 m_copydata(m, 0, mlen, datap);
1254 datap += mlen;
1255 }
1256 m->m_data += mlen;
1257 m->m_len -= mlen;
1258 eatlen -= mlen;
1259 if (m->m_len == 0)
1260 *mp = m = m_free(m);
1261 }
1262 if (eatlen > 0)
1263 n->m_len -= eatlen;
1264 n->m_next = m;
1265 *mp = m = n;
1266 continue;
1267 }
1268 mlen = min(mlen, len);
1269 if (flags & M_COPYBACK0_COPYBACK) {
1270 memcpy(mtod(m, caddr_t) + off, cp, (unsigned)mlen);
1271 cp += mlen;
1272 }
1273 len -= mlen;
1274 mlen += off;
1275 off = 0;
1276 totlen += mlen;
1277 if (len == 0)
1278 break;
1279 if (m->m_next == 0) {
1280 if ((flags & M_COPYBACK0_EXTEND) == 0)
1281 goto out;
1282 n = m_get(how, m->m_type);
1283 if (n == 0)
1284 break;
1285 n->m_len = min(MLEN, len);
1286 m->m_next = n;
1287 }
1288 mp = &m->m_next;
1289 m = m->m_next;
1290 }
1291 out: if (((m = *mp0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
1292 m->m_pkthdr.len = totlen;
1293
1294 return 0;
1295
1296 enobufs:
1297 return ENOBUFS;
1298 }
1299
1300 /*
1301 * Apply function f to the data in an mbuf chain starting "off" bytes from the
1302 * beginning, continuing for "len" bytes.
1303 */
1304 int
1305 m_apply(struct mbuf *m, int off, int len,
1306 int (*f)(void *, caddr_t, unsigned int), void *arg)
1307 {
1308 unsigned int count;
1309 int rval;
1310
1311 KASSERT(len >= 0);
1312 KASSERT(off >= 0);
1313
1314 while (off > 0) {
1315 KASSERT(m != NULL);
1316 if (off < m->m_len)
1317 break;
1318 off -= m->m_len;
1319 m = m->m_next;
1320 }
1321 while (len > 0) {
1322 KASSERT(m != NULL);
1323 count = min(m->m_len - off, len);
1324
1325 rval = (*f)(arg, mtod(m, caddr_t) + off, count);
1326 if (rval)
1327 return (rval);
1328
1329 len -= count;
1330 off = 0;
1331 m = m->m_next;
1332 }
1333
1334 return (0);
1335 }
1336
1337 /*
1338 * Return a pointer to mbuf/offset of location in mbuf chain.
1339 */
1340 struct mbuf *
1341 m_getptr(struct mbuf *m, int loc, int *off)
1342 {
1343
1344 while (loc >= 0) {
1345 /* Normal end of search */
1346 if (m->m_len > loc) {
1347 *off = loc;
1348 return (m);
1349 } else {
1350 loc -= m->m_len;
1351
1352 if (m->m_next == NULL) {
1353 if (loc == 0) {
1354 /* Point at the end of valid data */
1355 *off = m->m_len;
1356 return (m);
1357 } else
1358 return (NULL);
1359 } else
1360 m = m->m_next;
1361 }
1362 }
1363
1364 return (NULL);
1365 }
Cache object: 678bdf1912acfeb0ac440edc30c85a0d
|