FreeBSD/Linux Kernel Cross Reference
sys/net/bpf.c
1 /*-
2 * Copyright (c) 1990, 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 *
5 * This code is derived from the Stanford/CMU enet packet filter,
6 * (net/enet.c) distributed as part of 4.3BSD, and code contributed
7 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence
8 * Berkeley Laboratory.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 4. Neither the name of the University nor the names of its contributors
19 * may be used to endorse or promote products derived from this software
20 * without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
34 * @(#)bpf.c 8.4 (Berkeley) 1/9/95
35 */
36
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD: releng/9.1/sys/net/bpf.c 236410 2012-06-01 17:17:46Z jkim $");
39
40 #include "opt_bpf.h"
41 #include "opt_compat.h"
42 #include "opt_netgraph.h"
43
44 #include <sys/types.h>
45 #include <sys/param.h>
46 #include <sys/systm.h>
47 #include <sys/conf.h>
48 #include <sys/fcntl.h>
49 #include <sys/jail.h>
50 #include <sys/malloc.h>
51 #include <sys/mbuf.h>
52 #include <sys/time.h>
53 #include <sys/priv.h>
54 #include <sys/proc.h>
55 #include <sys/signalvar.h>
56 #include <sys/filio.h>
57 #include <sys/sockio.h>
58 #include <sys/ttycom.h>
59 #include <sys/uio.h>
60
61 #include <sys/event.h>
62 #include <sys/file.h>
63 #include <sys/poll.h>
64 #include <sys/proc.h>
65
66 #include <sys/socket.h>
67
68 #include <net/if.h>
69 #include <net/bpf.h>
70 #include <net/bpf_buffer.h>
71 #ifdef BPF_JITTER
72 #include <net/bpf_jitter.h>
73 #endif
74 #include <net/bpf_zerocopy.h>
75 #include <net/bpfdesc.h>
76 #include <net/vnet.h>
77
78 #include <netinet/in.h>
79 #include <netinet/if_ether.h>
80 #include <sys/kernel.h>
81 #include <sys/sysctl.h>
82
83 #include <net80211/ieee80211_freebsd.h>
84
85 #include <security/mac/mac_framework.h>
86
87 MALLOC_DEFINE(M_BPF, "BPF", "BPF data");
88
89 #if defined(DEV_BPF) || defined(NETGRAPH_BPF)
90
91 #define PRINET 26 /* interruptible */
92
93 #define SIZEOF_BPF_HDR(type) \
94 (offsetof(type, bh_hdrlen) + sizeof(((type *)0)->bh_hdrlen))
95
96 #ifdef COMPAT_FREEBSD32
97 #include <sys/mount.h>
98 #include <compat/freebsd32/freebsd32.h>
99 #define BPF_ALIGNMENT32 sizeof(int32_t)
100 #define BPF_WORDALIGN32(x) (((x)+(BPF_ALIGNMENT32-1))&~(BPF_ALIGNMENT32-1))
101
102 #ifndef BURN_BRIDGES
103 /*
104 * 32-bit version of structure prepended to each packet. We use this header
105 * instead of the standard one for 32-bit streams. We mark the a stream as
106 * 32-bit the first time we see a 32-bit compat ioctl request.
107 */
108 struct bpf_hdr32 {
109 struct timeval32 bh_tstamp; /* time stamp */
110 uint32_t bh_caplen; /* length of captured portion */
111 uint32_t bh_datalen; /* original length of packet */
112 uint16_t bh_hdrlen; /* length of bpf header (this struct
113 plus alignment padding) */
114 };
115 #endif
116
117 struct bpf_program32 {
118 u_int bf_len;
119 uint32_t bf_insns;
120 };
121
122 struct bpf_dltlist32 {
123 u_int bfl_len;
124 u_int bfl_list;
125 };
126
127 #define BIOCSETF32 _IOW('B', 103, struct bpf_program32)
128 #define BIOCSRTIMEOUT32 _IOW('B', 109, struct timeval32)
129 #define BIOCGRTIMEOUT32 _IOR('B', 110, struct timeval32)
130 #define BIOCGDLTLIST32 _IOWR('B', 121, struct bpf_dltlist32)
131 #define BIOCSETWF32 _IOW('B', 123, struct bpf_program32)
132 #define BIOCSETFNR32 _IOW('B', 130, struct bpf_program32)
133 #endif
134
135 /*
136 * bpf_iflist is a list of BPF interface structures, each corresponding to a
137 * specific DLT. The same network interface might have several BPF interface
138 * structures registered by different layers in the stack (i.e., 802.11
139 * frames, ethernet frames, etc).
140 */
141 static LIST_HEAD(, bpf_if) bpf_iflist;
142 static struct mtx bpf_mtx; /* bpf global lock */
143 static int bpf_bpfd_cnt;
144
145 static void bpf_attachd(struct bpf_d *, struct bpf_if *);
146 static void bpf_detachd(struct bpf_d *);
147 static void bpf_freed(struct bpf_d *);
148 static int bpf_movein(struct uio *, int, struct ifnet *, struct mbuf **,
149 struct sockaddr *, int *, struct bpf_insn *);
150 static int bpf_setif(struct bpf_d *, struct ifreq *);
151 static void bpf_timed_out(void *);
152 static __inline void
153 bpf_wakeup(struct bpf_d *);
154 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int,
155 void (*)(struct bpf_d *, caddr_t, u_int, void *, u_int),
156 struct bintime *);
157 static void reset_d(struct bpf_d *);
158 static int bpf_setf(struct bpf_d *, struct bpf_program *, u_long cmd);
159 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *);
160 static int bpf_setdlt(struct bpf_d *, u_int);
161 static void filt_bpfdetach(struct knote *);
162 static int filt_bpfread(struct knote *, long);
163 static void bpf_drvinit(void *);
164 static int bpf_stats_sysctl(SYSCTL_HANDLER_ARGS);
165
166 SYSCTL_NODE(_net, OID_AUTO, bpf, CTLFLAG_RW, 0, "bpf sysctl");
167 int bpf_maxinsns = BPF_MAXINSNS;
168 SYSCTL_INT(_net_bpf, OID_AUTO, maxinsns, CTLFLAG_RW,
169 &bpf_maxinsns, 0, "Maximum bpf program instructions");
170 static int bpf_zerocopy_enable = 0;
171 SYSCTL_INT(_net_bpf, OID_AUTO, zerocopy_enable, CTLFLAG_RW,
172 &bpf_zerocopy_enable, 0, "Enable new zero-copy BPF buffer sessions");
173 SYSCTL_NODE(_net_bpf, OID_AUTO, stats, CTLFLAG_MPSAFE | CTLFLAG_RW,
174 bpf_stats_sysctl, "bpf statistics portal");
175
176 static d_open_t bpfopen;
177 static d_read_t bpfread;
178 static d_write_t bpfwrite;
179 static d_ioctl_t bpfioctl;
180 static d_poll_t bpfpoll;
181 static d_kqfilter_t bpfkqfilter;
182
183 static struct cdevsw bpf_cdevsw = {
184 .d_version = D_VERSION,
185 .d_open = bpfopen,
186 .d_read = bpfread,
187 .d_write = bpfwrite,
188 .d_ioctl = bpfioctl,
189 .d_poll = bpfpoll,
190 .d_name = "bpf",
191 .d_kqfilter = bpfkqfilter,
192 };
193
194 static struct filterops bpfread_filtops = {
195 .f_isfd = 1,
196 .f_detach = filt_bpfdetach,
197 .f_event = filt_bpfread,
198 };
199
200 /*
201 * Wrapper functions for various buffering methods. If the set of buffer
202 * modes expands, we will probably want to introduce a switch data structure
203 * similar to protosw, et.
204 */
205 static void
206 bpf_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset, void *src,
207 u_int len)
208 {
209
210 BPFD_LOCK_ASSERT(d);
211
212 switch (d->bd_bufmode) {
213 case BPF_BUFMODE_BUFFER:
214 return (bpf_buffer_append_bytes(d, buf, offset, src, len));
215
216 case BPF_BUFMODE_ZBUF:
217 d->bd_zcopy++;
218 return (bpf_zerocopy_append_bytes(d, buf, offset, src, len));
219
220 default:
221 panic("bpf_buf_append_bytes");
222 }
223 }
224
225 static void
226 bpf_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset, void *src,
227 u_int len)
228 {
229
230 BPFD_LOCK_ASSERT(d);
231
232 switch (d->bd_bufmode) {
233 case BPF_BUFMODE_BUFFER:
234 return (bpf_buffer_append_mbuf(d, buf, offset, src, len));
235
236 case BPF_BUFMODE_ZBUF:
237 d->bd_zcopy++;
238 return (bpf_zerocopy_append_mbuf(d, buf, offset, src, len));
239
240 default:
241 panic("bpf_buf_append_mbuf");
242 }
243 }
244
245 /*
246 * This function gets called when the free buffer is re-assigned.
247 */
248 static void
249 bpf_buf_reclaimed(struct bpf_d *d)
250 {
251
252 BPFD_LOCK_ASSERT(d);
253
254 switch (d->bd_bufmode) {
255 case BPF_BUFMODE_BUFFER:
256 return;
257
258 case BPF_BUFMODE_ZBUF:
259 bpf_zerocopy_buf_reclaimed(d);
260 return;
261
262 default:
263 panic("bpf_buf_reclaimed");
264 }
265 }
266
267 /*
268 * If the buffer mechanism has a way to decide that a held buffer can be made
269 * free, then it is exposed via the bpf_canfreebuf() interface. (1) is
270 * returned if the buffer can be discarded, (0) is returned if it cannot.
271 */
272 static int
273 bpf_canfreebuf(struct bpf_d *d)
274 {
275
276 BPFD_LOCK_ASSERT(d);
277
278 switch (d->bd_bufmode) {
279 case BPF_BUFMODE_ZBUF:
280 return (bpf_zerocopy_canfreebuf(d));
281 }
282 return (0);
283 }
284
285 /*
286 * Allow the buffer model to indicate that the current store buffer is
287 * immutable, regardless of the appearance of space. Return (1) if the
288 * buffer is writable, and (0) if not.
289 */
290 static int
291 bpf_canwritebuf(struct bpf_d *d)
292 {
293
294 BPFD_LOCK_ASSERT(d);
295
296 switch (d->bd_bufmode) {
297 case BPF_BUFMODE_ZBUF:
298 return (bpf_zerocopy_canwritebuf(d));
299 }
300 return (1);
301 }
302
303 /*
304 * Notify buffer model that an attempt to write to the store buffer has
305 * resulted in a dropped packet, in which case the buffer may be considered
306 * full.
307 */
308 static void
309 bpf_buffull(struct bpf_d *d)
310 {
311
312 BPFD_LOCK_ASSERT(d);
313
314 switch (d->bd_bufmode) {
315 case BPF_BUFMODE_ZBUF:
316 bpf_zerocopy_buffull(d);
317 break;
318 }
319 }
320
321 /*
322 * Notify the buffer model that a buffer has moved into the hold position.
323 */
324 void
325 bpf_bufheld(struct bpf_d *d)
326 {
327
328 BPFD_LOCK_ASSERT(d);
329
330 switch (d->bd_bufmode) {
331 case BPF_BUFMODE_ZBUF:
332 bpf_zerocopy_bufheld(d);
333 break;
334 }
335 }
336
337 static void
338 bpf_free(struct bpf_d *d)
339 {
340
341 switch (d->bd_bufmode) {
342 case BPF_BUFMODE_BUFFER:
343 return (bpf_buffer_free(d));
344
345 case BPF_BUFMODE_ZBUF:
346 return (bpf_zerocopy_free(d));
347
348 default:
349 panic("bpf_buf_free");
350 }
351 }
352
353 static int
354 bpf_uiomove(struct bpf_d *d, caddr_t buf, u_int len, struct uio *uio)
355 {
356
357 if (d->bd_bufmode != BPF_BUFMODE_BUFFER)
358 return (EOPNOTSUPP);
359 return (bpf_buffer_uiomove(d, buf, len, uio));
360 }
361
362 static int
363 bpf_ioctl_sblen(struct bpf_d *d, u_int *i)
364 {
365
366 if (d->bd_bufmode != BPF_BUFMODE_BUFFER)
367 return (EOPNOTSUPP);
368 return (bpf_buffer_ioctl_sblen(d, i));
369 }
370
371 static int
372 bpf_ioctl_getzmax(struct thread *td, struct bpf_d *d, size_t *i)
373 {
374
375 if (d->bd_bufmode != BPF_BUFMODE_ZBUF)
376 return (EOPNOTSUPP);
377 return (bpf_zerocopy_ioctl_getzmax(td, d, i));
378 }
379
380 static int
381 bpf_ioctl_rotzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz)
382 {
383
384 if (d->bd_bufmode != BPF_BUFMODE_ZBUF)
385 return (EOPNOTSUPP);
386 return (bpf_zerocopy_ioctl_rotzbuf(td, d, bz));
387 }
388
389 static int
390 bpf_ioctl_setzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz)
391 {
392
393 if (d->bd_bufmode != BPF_BUFMODE_ZBUF)
394 return (EOPNOTSUPP);
395 return (bpf_zerocopy_ioctl_setzbuf(td, d, bz));
396 }
397
398 /*
399 * General BPF functions.
400 */
401 static int
402 bpf_movein(struct uio *uio, int linktype, struct ifnet *ifp, struct mbuf **mp,
403 struct sockaddr *sockp, int *hdrlen, struct bpf_insn *wfilter)
404 {
405 const struct ieee80211_bpf_params *p;
406 struct ether_header *eh;
407 struct mbuf *m;
408 int error;
409 int len;
410 int hlen;
411 int slen;
412
413 /*
414 * Build a sockaddr based on the data link layer type.
415 * We do this at this level because the ethernet header
416 * is copied directly into the data field of the sockaddr.
417 * In the case of SLIP, there is no header and the packet
418 * is forwarded as is.
419 * Also, we are careful to leave room at the front of the mbuf
420 * for the link level header.
421 */
422 switch (linktype) {
423
424 case DLT_SLIP:
425 sockp->sa_family = AF_INET;
426 hlen = 0;
427 break;
428
429 case DLT_EN10MB:
430 sockp->sa_family = AF_UNSPEC;
431 /* XXX Would MAXLINKHDR be better? */
432 hlen = ETHER_HDR_LEN;
433 break;
434
435 case DLT_FDDI:
436 sockp->sa_family = AF_IMPLINK;
437 hlen = 0;
438 break;
439
440 case DLT_RAW:
441 sockp->sa_family = AF_UNSPEC;
442 hlen = 0;
443 break;
444
445 case DLT_NULL:
446 /*
447 * null interface types require a 4 byte pseudo header which
448 * corresponds to the address family of the packet.
449 */
450 sockp->sa_family = AF_UNSPEC;
451 hlen = 4;
452 break;
453
454 case DLT_ATM_RFC1483:
455 /*
456 * en atm driver requires 4-byte atm pseudo header.
457 * though it isn't standard, vpi:vci needs to be
458 * specified anyway.
459 */
460 sockp->sa_family = AF_UNSPEC;
461 hlen = 12; /* XXX 4(ATM_PH) + 3(LLC) + 5(SNAP) */
462 break;
463
464 case DLT_PPP:
465 sockp->sa_family = AF_UNSPEC;
466 hlen = 4; /* This should match PPP_HDRLEN */
467 break;
468
469 case DLT_IEEE802_11: /* IEEE 802.11 wireless */
470 sockp->sa_family = AF_IEEE80211;
471 hlen = 0;
472 break;
473
474 case DLT_IEEE802_11_RADIO: /* IEEE 802.11 wireless w/ phy params */
475 sockp->sa_family = AF_IEEE80211;
476 sockp->sa_len = 12; /* XXX != 0 */
477 hlen = sizeof(struct ieee80211_bpf_params);
478 break;
479
480 default:
481 return (EIO);
482 }
483
484 len = uio->uio_resid;
485
486 if (len - hlen > ifp->if_mtu)
487 return (EMSGSIZE);
488
489 if ((unsigned)len > MJUM16BYTES)
490 return (EIO);
491
492 if (len <= MHLEN)
493 MGETHDR(m, M_WAIT, MT_DATA);
494 else if (len <= MCLBYTES)
495 m = m_getcl(M_WAIT, MT_DATA, M_PKTHDR);
496 else
497 m = m_getjcl(M_WAIT, MT_DATA, M_PKTHDR,
498 #if (MJUMPAGESIZE > MCLBYTES)
499 len <= MJUMPAGESIZE ? MJUMPAGESIZE :
500 #endif
501 (len <= MJUM9BYTES ? MJUM9BYTES : MJUM16BYTES));
502 m->m_pkthdr.len = m->m_len = len;
503 m->m_pkthdr.rcvif = NULL;
504 *mp = m;
505
506 if (m->m_len < hlen) {
507 error = EPERM;
508 goto bad;
509 }
510
511 error = uiomove(mtod(m, u_char *), len, uio);
512 if (error)
513 goto bad;
514
515 slen = bpf_filter(wfilter, mtod(m, u_char *), len, len);
516 if (slen == 0) {
517 error = EPERM;
518 goto bad;
519 }
520
521 /* Check for multicast destination */
522 switch (linktype) {
523 case DLT_EN10MB:
524 eh = mtod(m, struct ether_header *);
525 if (ETHER_IS_MULTICAST(eh->ether_dhost)) {
526 if (bcmp(ifp->if_broadcastaddr, eh->ether_dhost,
527 ETHER_ADDR_LEN) == 0)
528 m->m_flags |= M_BCAST;
529 else
530 m->m_flags |= M_MCAST;
531 }
532 break;
533 }
534
535 /*
536 * Make room for link header, and copy it to sockaddr
537 */
538 if (hlen != 0) {
539 if (sockp->sa_family == AF_IEEE80211) {
540 /*
541 * Collect true length from the parameter header
542 * NB: sockp is known to be zero'd so if we do a
543 * short copy unspecified parameters will be
544 * zero.
545 * NB: packet may not be aligned after stripping
546 * bpf params
547 * XXX check ibp_vers
548 */
549 p = mtod(m, const struct ieee80211_bpf_params *);
550 hlen = p->ibp_len;
551 if (hlen > sizeof(sockp->sa_data)) {
552 error = EINVAL;
553 goto bad;
554 }
555 }
556 bcopy(m->m_data, sockp->sa_data, hlen);
557 }
558 *hdrlen = hlen;
559
560 return (0);
561 bad:
562 m_freem(m);
563 return (error);
564 }
565
566 /*
567 * Attach file to the bpf interface, i.e. make d listen on bp.
568 */
569 static void
570 bpf_attachd(struct bpf_d *d, struct bpf_if *bp)
571 {
572 /*
573 * Point d at bp, and add d to the interface's list of listeners.
574 * Finally, point the driver's bpf cookie at the interface so
575 * it will divert packets to bpf.
576 */
577 BPFIF_LOCK(bp);
578 d->bd_bif = bp;
579 LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next);
580
581 bpf_bpfd_cnt++;
582 BPFIF_UNLOCK(bp);
583
584 EVENTHANDLER_INVOKE(bpf_track, bp->bif_ifp, bp->bif_dlt, 1);
585 }
586
587 /*
588 * Detach a file from its interface.
589 */
590 static void
591 bpf_detachd(struct bpf_d *d)
592 {
593 int error;
594 struct bpf_if *bp;
595 struct ifnet *ifp;
596
597 bp = d->bd_bif;
598 BPFIF_LOCK(bp);
599 BPFD_LOCK(d);
600 ifp = d->bd_bif->bif_ifp;
601
602 /*
603 * Remove d from the interface's descriptor list.
604 */
605 LIST_REMOVE(d, bd_next);
606
607 bpf_bpfd_cnt--;
608 d->bd_bif = NULL;
609 BPFD_UNLOCK(d);
610 BPFIF_UNLOCK(bp);
611
612 EVENTHANDLER_INVOKE(bpf_track, ifp, bp->bif_dlt, 0);
613
614 /*
615 * Check if this descriptor had requested promiscuous mode.
616 * If so, turn it off.
617 */
618 if (d->bd_promisc) {
619 d->bd_promisc = 0;
620 CURVNET_SET(ifp->if_vnet);
621 error = ifpromisc(ifp, 0);
622 CURVNET_RESTORE();
623 if (error != 0 && error != ENXIO) {
624 /*
625 * ENXIO can happen if a pccard is unplugged
626 * Something is really wrong if we were able to put
627 * the driver into promiscuous mode, but can't
628 * take it out.
629 */
630 if_printf(bp->bif_ifp,
631 "bpf_detach: ifpromisc failed (%d)\n", error);
632 }
633 }
634 }
635
636 /*
637 * Close the descriptor by detaching it from its interface,
638 * deallocating its buffers, and marking it free.
639 */
640 static void
641 bpf_dtor(void *data)
642 {
643 struct bpf_d *d = data;
644
645 BPFD_LOCK(d);
646 if (d->bd_state == BPF_WAITING)
647 callout_stop(&d->bd_callout);
648 d->bd_state = BPF_IDLE;
649 BPFD_UNLOCK(d);
650 funsetown(&d->bd_sigio);
651 mtx_lock(&bpf_mtx);
652 if (d->bd_bif)
653 bpf_detachd(d);
654 mtx_unlock(&bpf_mtx);
655 #ifdef MAC
656 mac_bpfdesc_destroy(d);
657 #endif /* MAC */
658 seldrain(&d->bd_sel);
659 knlist_destroy(&d->bd_sel.si_note);
660 callout_drain(&d->bd_callout);
661 bpf_freed(d);
662 free(d, M_BPF);
663 }
664
665 /*
666 * Open ethernet device. Returns ENXIO for illegal minor device number,
667 * EBUSY if file is open by another process.
668 */
669 /* ARGSUSED */
670 static int
671 bpfopen(struct cdev *dev, int flags, int fmt, struct thread *td)
672 {
673 struct bpf_d *d;
674 int error;
675
676 d = malloc(sizeof(*d), M_BPF, M_WAITOK | M_ZERO);
677 error = devfs_set_cdevpriv(d, bpf_dtor);
678 if (error != 0) {
679 free(d, M_BPF);
680 return (error);
681 }
682
683 /*
684 * For historical reasons, perform a one-time initialization call to
685 * the buffer routines, even though we're not yet committed to a
686 * particular buffer method.
687 */
688 bpf_buffer_init(d);
689 d->bd_bufmode = BPF_BUFMODE_BUFFER;
690 d->bd_sig = SIGIO;
691 d->bd_direction = BPF_D_INOUT;
692 d->bd_pid = td->td_proc->p_pid;
693 #ifdef MAC
694 mac_bpfdesc_init(d);
695 mac_bpfdesc_create(td->td_ucred, d);
696 #endif
697 mtx_init(&d->bd_mtx, devtoname(dev), "bpf cdev lock", MTX_DEF);
698 callout_init_mtx(&d->bd_callout, &d->bd_mtx, 0);
699 knlist_init_mtx(&d->bd_sel.si_note, &d->bd_mtx);
700
701 return (0);
702 }
703
704 /*
705 * bpfread - read next chunk of packets from buffers
706 */
707 static int
708 bpfread(struct cdev *dev, struct uio *uio, int ioflag)
709 {
710 struct bpf_d *d;
711 int error;
712 int non_block;
713 int timed_out;
714
715 error = devfs_get_cdevpriv((void **)&d);
716 if (error != 0)
717 return (error);
718
719 /*
720 * Restrict application to use a buffer the same size as
721 * as kernel buffers.
722 */
723 if (uio->uio_resid != d->bd_bufsize)
724 return (EINVAL);
725
726 non_block = ((ioflag & O_NONBLOCK) != 0);
727
728 BPFD_LOCK(d);
729 d->bd_pid = curthread->td_proc->p_pid;
730 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) {
731 BPFD_UNLOCK(d);
732 return (EOPNOTSUPP);
733 }
734 if (d->bd_state == BPF_WAITING)
735 callout_stop(&d->bd_callout);
736 timed_out = (d->bd_state == BPF_TIMED_OUT);
737 d->bd_state = BPF_IDLE;
738 /*
739 * If the hold buffer is empty, then do a timed sleep, which
740 * ends when the timeout expires or when enough packets
741 * have arrived to fill the store buffer.
742 */
743 while (d->bd_hbuf == NULL) {
744 if (d->bd_slen != 0) {
745 /*
746 * A packet(s) either arrived since the previous
747 * read or arrived while we were asleep.
748 */
749 if (d->bd_immediate || non_block || timed_out) {
750 /*
751 * Rotate the buffers and return what's here
752 * if we are in immediate mode, non-blocking
753 * flag is set, or this descriptor timed out.
754 */
755 ROTATE_BUFFERS(d);
756 break;
757 }
758 }
759
760 /*
761 * No data is available, check to see if the bpf device
762 * is still pointed at a real interface. If not, return
763 * ENXIO so that the userland process knows to rebind
764 * it before using it again.
765 */
766 if (d->bd_bif == NULL) {
767 BPFD_UNLOCK(d);
768 return (ENXIO);
769 }
770
771 if (non_block) {
772 BPFD_UNLOCK(d);
773 return (EWOULDBLOCK);
774 }
775 error = msleep(d, &d->bd_mtx, PRINET|PCATCH,
776 "bpf", d->bd_rtout);
777 if (error == EINTR || error == ERESTART) {
778 BPFD_UNLOCK(d);
779 return (error);
780 }
781 if (error == EWOULDBLOCK) {
782 /*
783 * On a timeout, return what's in the buffer,
784 * which may be nothing. If there is something
785 * in the store buffer, we can rotate the buffers.
786 */
787 if (d->bd_hbuf)
788 /*
789 * We filled up the buffer in between
790 * getting the timeout and arriving
791 * here, so we don't need to rotate.
792 */
793 break;
794
795 if (d->bd_slen == 0) {
796 BPFD_UNLOCK(d);
797 return (0);
798 }
799 ROTATE_BUFFERS(d);
800 break;
801 }
802 }
803 /*
804 * At this point, we know we have something in the hold slot.
805 */
806 BPFD_UNLOCK(d);
807
808 /*
809 * Move data from hold buffer into user space.
810 * We know the entire buffer is transferred since
811 * we checked above that the read buffer is bpf_bufsize bytes.
812 *
813 * XXXRW: More synchronization needed here: what if a second thread
814 * issues a read on the same fd at the same time? Don't want this
815 * getting invalidated.
816 */
817 error = bpf_uiomove(d, d->bd_hbuf, d->bd_hlen, uio);
818
819 BPFD_LOCK(d);
820 d->bd_fbuf = d->bd_hbuf;
821 d->bd_hbuf = NULL;
822 d->bd_hlen = 0;
823 bpf_buf_reclaimed(d);
824 BPFD_UNLOCK(d);
825
826 return (error);
827 }
828
829 /*
830 * If there are processes sleeping on this descriptor, wake them up.
831 */
832 static __inline void
833 bpf_wakeup(struct bpf_d *d)
834 {
835
836 BPFD_LOCK_ASSERT(d);
837 if (d->bd_state == BPF_WAITING) {
838 callout_stop(&d->bd_callout);
839 d->bd_state = BPF_IDLE;
840 }
841 wakeup(d);
842 if (d->bd_async && d->bd_sig && d->bd_sigio)
843 pgsigio(&d->bd_sigio, d->bd_sig, 0);
844
845 selwakeuppri(&d->bd_sel, PRINET);
846 KNOTE_LOCKED(&d->bd_sel.si_note, 0);
847 }
848
849 static void
850 bpf_timed_out(void *arg)
851 {
852 struct bpf_d *d = (struct bpf_d *)arg;
853
854 BPFD_LOCK_ASSERT(d);
855
856 if (callout_pending(&d->bd_callout) || !callout_active(&d->bd_callout))
857 return;
858 if (d->bd_state == BPF_WAITING) {
859 d->bd_state = BPF_TIMED_OUT;
860 if (d->bd_slen != 0)
861 bpf_wakeup(d);
862 }
863 }
864
865 static int
866 bpf_ready(struct bpf_d *d)
867 {
868
869 BPFD_LOCK_ASSERT(d);
870
871 if (!bpf_canfreebuf(d) && d->bd_hlen != 0)
872 return (1);
873 if ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) &&
874 d->bd_slen != 0)
875 return (1);
876 return (0);
877 }
878
879 static int
880 bpfwrite(struct cdev *dev, struct uio *uio, int ioflag)
881 {
882 struct bpf_d *d;
883 struct ifnet *ifp;
884 struct mbuf *m, *mc;
885 struct sockaddr dst;
886 int error, hlen;
887
888 error = devfs_get_cdevpriv((void **)&d);
889 if (error != 0)
890 return (error);
891
892 d->bd_pid = curthread->td_proc->p_pid;
893 d->bd_wcount++;
894 if (d->bd_bif == NULL) {
895 d->bd_wdcount++;
896 return (ENXIO);
897 }
898
899 ifp = d->bd_bif->bif_ifp;
900
901 if ((ifp->if_flags & IFF_UP) == 0) {
902 d->bd_wdcount++;
903 return (ENETDOWN);
904 }
905
906 if (uio->uio_resid == 0) {
907 d->bd_wdcount++;
908 return (0);
909 }
910
911 bzero(&dst, sizeof(dst));
912 m = NULL;
913 hlen = 0;
914 error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp,
915 &m, &dst, &hlen, d->bd_wfilter);
916 if (error) {
917 d->bd_wdcount++;
918 return (error);
919 }
920 d->bd_wfcount++;
921 if (d->bd_hdrcmplt)
922 dst.sa_family = pseudo_AF_HDRCMPLT;
923
924 if (d->bd_feedback) {
925 mc = m_dup(m, M_DONTWAIT);
926 if (mc != NULL)
927 mc->m_pkthdr.rcvif = ifp;
928 /* Set M_PROMISC for outgoing packets to be discarded. */
929 if (d->bd_direction == BPF_D_INOUT)
930 m->m_flags |= M_PROMISC;
931 } else
932 mc = NULL;
933
934 m->m_pkthdr.len -= hlen;
935 m->m_len -= hlen;
936 m->m_data += hlen; /* XXX */
937
938 CURVNET_SET(ifp->if_vnet);
939 #ifdef MAC
940 BPFD_LOCK(d);
941 mac_bpfdesc_create_mbuf(d, m);
942 if (mc != NULL)
943 mac_bpfdesc_create_mbuf(d, mc);
944 BPFD_UNLOCK(d);
945 #endif
946
947 error = (*ifp->if_output)(ifp, m, &dst, NULL);
948 if (error)
949 d->bd_wdcount++;
950
951 if (mc != NULL) {
952 if (error == 0)
953 (*ifp->if_input)(ifp, mc);
954 else
955 m_freem(mc);
956 }
957 CURVNET_RESTORE();
958
959 return (error);
960 }
961
962 /*
963 * Reset a descriptor by flushing its packet buffer and clearing the receive
964 * and drop counts. This is doable for kernel-only buffers, but with
965 * zero-copy buffers, we can't write to (or rotate) buffers that are
966 * currently owned by userspace. It would be nice if we could encapsulate
967 * this logic in the buffer code rather than here.
968 */
969 static void
970 reset_d(struct bpf_d *d)
971 {
972
973 mtx_assert(&d->bd_mtx, MA_OWNED);
974
975 if ((d->bd_hbuf != NULL) &&
976 (d->bd_bufmode != BPF_BUFMODE_ZBUF || bpf_canfreebuf(d))) {
977 /* Free the hold buffer. */
978 d->bd_fbuf = d->bd_hbuf;
979 d->bd_hbuf = NULL;
980 d->bd_hlen = 0;
981 bpf_buf_reclaimed(d);
982 }
983 if (bpf_canwritebuf(d))
984 d->bd_slen = 0;
985 d->bd_rcount = 0;
986 d->bd_dcount = 0;
987 d->bd_fcount = 0;
988 d->bd_wcount = 0;
989 d->bd_wfcount = 0;
990 d->bd_wdcount = 0;
991 d->bd_zcopy = 0;
992 }
993
994 /*
995 * FIONREAD Check for read packet available.
996 * SIOCGIFADDR Get interface address - convenient hook to driver.
997 * BIOCGBLEN Get buffer len [for read()].
998 * BIOCSETF Set read filter.
999 * BIOCSETFNR Set read filter without resetting descriptor.
1000 * BIOCSETWF Set write filter.
1001 * BIOCFLUSH Flush read packet buffer.
1002 * BIOCPROMISC Put interface into promiscuous mode.
1003 * BIOCGDLT Get link layer type.
1004 * BIOCGETIF Get interface name.
1005 * BIOCSETIF Set interface.
1006 * BIOCSRTIMEOUT Set read timeout.
1007 * BIOCGRTIMEOUT Get read timeout.
1008 * BIOCGSTATS Get packet stats.
1009 * BIOCIMMEDIATE Set immediate mode.
1010 * BIOCVERSION Get filter language version.
1011 * BIOCGHDRCMPLT Get "header already complete" flag
1012 * BIOCSHDRCMPLT Set "header already complete" flag
1013 * BIOCGDIRECTION Get packet direction flag
1014 * BIOCSDIRECTION Set packet direction flag
1015 * BIOCGTSTAMP Get time stamp format and resolution.
1016 * BIOCSTSTAMP Set time stamp format and resolution.
1017 * BIOCLOCK Set "locked" flag
1018 * BIOCFEEDBACK Set packet feedback mode.
1019 * BIOCSETZBUF Set current zero-copy buffer locations.
1020 * BIOCGETZMAX Get maximum zero-copy buffer size.
1021 * BIOCROTZBUF Force rotation of zero-copy buffer
1022 * BIOCSETBUFMODE Set buffer mode.
1023 * BIOCGETBUFMODE Get current buffer mode.
1024 */
1025 /* ARGSUSED */
1026 static int
1027 bpfioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags,
1028 struct thread *td)
1029 {
1030 struct bpf_d *d;
1031 int error;
1032
1033 error = devfs_get_cdevpriv((void **)&d);
1034 if (error != 0)
1035 return (error);
1036
1037 /*
1038 * Refresh PID associated with this descriptor.
1039 */
1040 BPFD_LOCK(d);
1041 d->bd_pid = td->td_proc->p_pid;
1042 if (d->bd_state == BPF_WAITING)
1043 callout_stop(&d->bd_callout);
1044 d->bd_state = BPF_IDLE;
1045 BPFD_UNLOCK(d);
1046
1047 if (d->bd_locked == 1) {
1048 switch (cmd) {
1049 case BIOCGBLEN:
1050 case BIOCFLUSH:
1051 case BIOCGDLT:
1052 case BIOCGDLTLIST:
1053 #ifdef COMPAT_FREEBSD32
1054 case BIOCGDLTLIST32:
1055 #endif
1056 case BIOCGETIF:
1057 case BIOCGRTIMEOUT:
1058 #ifdef COMPAT_FREEBSD32
1059 case BIOCGRTIMEOUT32:
1060 #endif
1061 case BIOCGSTATS:
1062 case BIOCVERSION:
1063 case BIOCGRSIG:
1064 case BIOCGHDRCMPLT:
1065 case BIOCSTSTAMP:
1066 case BIOCFEEDBACK:
1067 case FIONREAD:
1068 case BIOCLOCK:
1069 case BIOCSRTIMEOUT:
1070 #ifdef COMPAT_FREEBSD32
1071 case BIOCSRTIMEOUT32:
1072 #endif
1073 case BIOCIMMEDIATE:
1074 case TIOCGPGRP:
1075 case BIOCROTZBUF:
1076 break;
1077 default:
1078 return (EPERM);
1079 }
1080 }
1081 #ifdef COMPAT_FREEBSD32
1082 /*
1083 * If we see a 32-bit compat ioctl, mark the stream as 32-bit so
1084 * that it will get 32-bit packet headers.
1085 */
1086 switch (cmd) {
1087 case BIOCSETF32:
1088 case BIOCSETFNR32:
1089 case BIOCSETWF32:
1090 case BIOCGDLTLIST32:
1091 case BIOCGRTIMEOUT32:
1092 case BIOCSRTIMEOUT32:
1093 d->bd_compat32 = 1;
1094 }
1095 #endif
1096
1097 CURVNET_SET(TD_TO_VNET(td));
1098 switch (cmd) {
1099
1100 default:
1101 error = EINVAL;
1102 break;
1103
1104 /*
1105 * Check for read packet available.
1106 */
1107 case FIONREAD:
1108 {
1109 int n;
1110
1111 BPFD_LOCK(d);
1112 n = d->bd_slen;
1113 if (d->bd_hbuf)
1114 n += d->bd_hlen;
1115 BPFD_UNLOCK(d);
1116
1117 *(int *)addr = n;
1118 break;
1119 }
1120
1121 case SIOCGIFADDR:
1122 {
1123 struct ifnet *ifp;
1124
1125 if (d->bd_bif == NULL)
1126 error = EINVAL;
1127 else {
1128 ifp = d->bd_bif->bif_ifp;
1129 error = (*ifp->if_ioctl)(ifp, cmd, addr);
1130 }
1131 break;
1132 }
1133
1134 /*
1135 * Get buffer len [for read()].
1136 */
1137 case BIOCGBLEN:
1138 *(u_int *)addr = d->bd_bufsize;
1139 break;
1140
1141 /*
1142 * Set buffer length.
1143 */
1144 case BIOCSBLEN:
1145 error = bpf_ioctl_sblen(d, (u_int *)addr);
1146 break;
1147
1148 /*
1149 * Set link layer read filter.
1150 */
1151 case BIOCSETF:
1152 case BIOCSETFNR:
1153 case BIOCSETWF:
1154 #ifdef COMPAT_FREEBSD32
1155 case BIOCSETF32:
1156 case BIOCSETFNR32:
1157 case BIOCSETWF32:
1158 #endif
1159 error = bpf_setf(d, (struct bpf_program *)addr, cmd);
1160 break;
1161
1162 /*
1163 * Flush read packet buffer.
1164 */
1165 case BIOCFLUSH:
1166 BPFD_LOCK(d);
1167 reset_d(d);
1168 BPFD_UNLOCK(d);
1169 break;
1170
1171 /*
1172 * Put interface into promiscuous mode.
1173 */
1174 case BIOCPROMISC:
1175 if (d->bd_bif == NULL) {
1176 /*
1177 * No interface attached yet.
1178 */
1179 error = EINVAL;
1180 break;
1181 }
1182 if (d->bd_promisc == 0) {
1183 error = ifpromisc(d->bd_bif->bif_ifp, 1);
1184 if (error == 0)
1185 d->bd_promisc = 1;
1186 }
1187 break;
1188
1189 /*
1190 * Get current data link type.
1191 */
1192 case BIOCGDLT:
1193 if (d->bd_bif == NULL)
1194 error = EINVAL;
1195 else
1196 *(u_int *)addr = d->bd_bif->bif_dlt;
1197 break;
1198
1199 /*
1200 * Get a list of supported data link types.
1201 */
1202 #ifdef COMPAT_FREEBSD32
1203 case BIOCGDLTLIST32:
1204 {
1205 struct bpf_dltlist32 *list32;
1206 struct bpf_dltlist dltlist;
1207
1208 list32 = (struct bpf_dltlist32 *)addr;
1209 dltlist.bfl_len = list32->bfl_len;
1210 dltlist.bfl_list = PTRIN(list32->bfl_list);
1211 if (d->bd_bif == NULL)
1212 error = EINVAL;
1213 else {
1214 error = bpf_getdltlist(d, &dltlist);
1215 if (error == 0)
1216 list32->bfl_len = dltlist.bfl_len;
1217 }
1218 break;
1219 }
1220 #endif
1221
1222 case BIOCGDLTLIST:
1223 if (d->bd_bif == NULL)
1224 error = EINVAL;
1225 else
1226 error = bpf_getdltlist(d, (struct bpf_dltlist *)addr);
1227 break;
1228
1229 /*
1230 * Set data link type.
1231 */
1232 case BIOCSDLT:
1233 if (d->bd_bif == NULL)
1234 error = EINVAL;
1235 else
1236 error = bpf_setdlt(d, *(u_int *)addr);
1237 break;
1238
1239 /*
1240 * Get interface name.
1241 */
1242 case BIOCGETIF:
1243 if (d->bd_bif == NULL)
1244 error = EINVAL;
1245 else {
1246 struct ifnet *const ifp = d->bd_bif->bif_ifp;
1247 struct ifreq *const ifr = (struct ifreq *)addr;
1248
1249 strlcpy(ifr->ifr_name, ifp->if_xname,
1250 sizeof(ifr->ifr_name));
1251 }
1252 break;
1253
1254 /*
1255 * Set interface.
1256 */
1257 case BIOCSETIF:
1258 error = bpf_setif(d, (struct ifreq *)addr);
1259 break;
1260
1261 /*
1262 * Set read timeout.
1263 */
1264 case BIOCSRTIMEOUT:
1265 #ifdef COMPAT_FREEBSD32
1266 case BIOCSRTIMEOUT32:
1267 #endif
1268 {
1269 struct timeval *tv = (struct timeval *)addr;
1270 #ifdef COMPAT_FREEBSD32
1271 struct timeval32 *tv32;
1272 struct timeval tv64;
1273
1274 if (cmd == BIOCSRTIMEOUT32) {
1275 tv32 = (struct timeval32 *)addr;
1276 tv = &tv64;
1277 tv->tv_sec = tv32->tv_sec;
1278 tv->tv_usec = tv32->tv_usec;
1279 } else
1280 #endif
1281 tv = (struct timeval *)addr;
1282
1283 /*
1284 * Subtract 1 tick from tvtohz() since this isn't
1285 * a one-shot timer.
1286 */
1287 if ((error = itimerfix(tv)) == 0)
1288 d->bd_rtout = tvtohz(tv) - 1;
1289 break;
1290 }
1291
1292 /*
1293 * Get read timeout.
1294 */
1295 case BIOCGRTIMEOUT:
1296 #ifdef COMPAT_FREEBSD32
1297 case BIOCGRTIMEOUT32:
1298 #endif
1299 {
1300 struct timeval *tv;
1301 #ifdef COMPAT_FREEBSD32
1302 struct timeval32 *tv32;
1303 struct timeval tv64;
1304
1305 if (cmd == BIOCGRTIMEOUT32)
1306 tv = &tv64;
1307 else
1308 #endif
1309 tv = (struct timeval *)addr;
1310
1311 tv->tv_sec = d->bd_rtout / hz;
1312 tv->tv_usec = (d->bd_rtout % hz) * tick;
1313 #ifdef COMPAT_FREEBSD32
1314 if (cmd == BIOCGRTIMEOUT32) {
1315 tv32 = (struct timeval32 *)addr;
1316 tv32->tv_sec = tv->tv_sec;
1317 tv32->tv_usec = tv->tv_usec;
1318 }
1319 #endif
1320
1321 break;
1322 }
1323
1324 /*
1325 * Get packet stats.
1326 */
1327 case BIOCGSTATS:
1328 {
1329 struct bpf_stat *bs = (struct bpf_stat *)addr;
1330
1331 /* XXXCSJP overflow */
1332 bs->bs_recv = d->bd_rcount;
1333 bs->bs_drop = d->bd_dcount;
1334 break;
1335 }
1336
1337 /*
1338 * Set immediate mode.
1339 */
1340 case BIOCIMMEDIATE:
1341 d->bd_immediate = *(u_int *)addr;
1342 break;
1343
1344 case BIOCVERSION:
1345 {
1346 struct bpf_version *bv = (struct bpf_version *)addr;
1347
1348 bv->bv_major = BPF_MAJOR_VERSION;
1349 bv->bv_minor = BPF_MINOR_VERSION;
1350 break;
1351 }
1352
1353 /*
1354 * Get "header already complete" flag
1355 */
1356 case BIOCGHDRCMPLT:
1357 *(u_int *)addr = d->bd_hdrcmplt;
1358 break;
1359
1360 /*
1361 * Set "header already complete" flag
1362 */
1363 case BIOCSHDRCMPLT:
1364 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0;
1365 break;
1366
1367 /*
1368 * Get packet direction flag
1369 */
1370 case BIOCGDIRECTION:
1371 *(u_int *)addr = d->bd_direction;
1372 break;
1373
1374 /*
1375 * Set packet direction flag
1376 */
1377 case BIOCSDIRECTION:
1378 {
1379 u_int direction;
1380
1381 direction = *(u_int *)addr;
1382 switch (direction) {
1383 case BPF_D_IN:
1384 case BPF_D_INOUT:
1385 case BPF_D_OUT:
1386 d->bd_direction = direction;
1387 break;
1388 default:
1389 error = EINVAL;
1390 }
1391 }
1392 break;
1393
1394 /*
1395 * Get packet timestamp format and resolution.
1396 */
1397 case BIOCGTSTAMP:
1398 *(u_int *)addr = d->bd_tstamp;
1399 break;
1400
1401 /*
1402 * Set packet timestamp format and resolution.
1403 */
1404 case BIOCSTSTAMP:
1405 {
1406 u_int func;
1407
1408 func = *(u_int *)addr;
1409 if (BPF_T_VALID(func))
1410 d->bd_tstamp = func;
1411 else
1412 error = EINVAL;
1413 }
1414 break;
1415
1416 case BIOCFEEDBACK:
1417 d->bd_feedback = *(u_int *)addr;
1418 break;
1419
1420 case BIOCLOCK:
1421 d->bd_locked = 1;
1422 break;
1423
1424 case FIONBIO: /* Non-blocking I/O */
1425 break;
1426
1427 case FIOASYNC: /* Send signal on receive packets */
1428 d->bd_async = *(int *)addr;
1429 break;
1430
1431 case FIOSETOWN:
1432 error = fsetown(*(int *)addr, &d->bd_sigio);
1433 break;
1434
1435 case FIOGETOWN:
1436 *(int *)addr = fgetown(&d->bd_sigio);
1437 break;
1438
1439 /* This is deprecated, FIOSETOWN should be used instead. */
1440 case TIOCSPGRP:
1441 error = fsetown(-(*(int *)addr), &d->bd_sigio);
1442 break;
1443
1444 /* This is deprecated, FIOGETOWN should be used instead. */
1445 case TIOCGPGRP:
1446 *(int *)addr = -fgetown(&d->bd_sigio);
1447 break;
1448
1449 case BIOCSRSIG: /* Set receive signal */
1450 {
1451 u_int sig;
1452
1453 sig = *(u_int *)addr;
1454
1455 if (sig >= NSIG)
1456 error = EINVAL;
1457 else
1458 d->bd_sig = sig;
1459 break;
1460 }
1461 case BIOCGRSIG:
1462 *(u_int *)addr = d->bd_sig;
1463 break;
1464
1465 case BIOCGETBUFMODE:
1466 *(u_int *)addr = d->bd_bufmode;
1467 break;
1468
1469 case BIOCSETBUFMODE:
1470 /*
1471 * Allow the buffering mode to be changed as long as we
1472 * haven't yet committed to a particular mode. Our
1473 * definition of commitment, for now, is whether or not a
1474 * buffer has been allocated or an interface attached, since
1475 * that's the point where things get tricky.
1476 */
1477 switch (*(u_int *)addr) {
1478 case BPF_BUFMODE_BUFFER:
1479 break;
1480
1481 case BPF_BUFMODE_ZBUF:
1482 if (bpf_zerocopy_enable)
1483 break;
1484 /* FALLSTHROUGH */
1485
1486 default:
1487 CURVNET_RESTORE();
1488 return (EINVAL);
1489 }
1490
1491 BPFD_LOCK(d);
1492 if (d->bd_sbuf != NULL || d->bd_hbuf != NULL ||
1493 d->bd_fbuf != NULL || d->bd_bif != NULL) {
1494 BPFD_UNLOCK(d);
1495 CURVNET_RESTORE();
1496 return (EBUSY);
1497 }
1498 d->bd_bufmode = *(u_int *)addr;
1499 BPFD_UNLOCK(d);
1500 break;
1501
1502 case BIOCGETZMAX:
1503 error = bpf_ioctl_getzmax(td, d, (size_t *)addr);
1504 break;
1505
1506 case BIOCSETZBUF:
1507 error = bpf_ioctl_setzbuf(td, d, (struct bpf_zbuf *)addr);
1508 break;
1509
1510 case BIOCROTZBUF:
1511 error = bpf_ioctl_rotzbuf(td, d, (struct bpf_zbuf *)addr);
1512 break;
1513 }
1514 CURVNET_RESTORE();
1515 return (error);
1516 }
1517
1518 /*
1519 * Set d's packet filter program to fp. If this file already has a filter,
1520 * free it and replace it. Returns EINVAL for bogus requests.
1521 */
1522 static int
1523 bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd)
1524 {
1525 struct bpf_insn *fcode, *old;
1526 u_int wfilter, flen, size;
1527 #ifdef BPF_JITTER
1528 bpf_jit_filter *ofunc;
1529 #endif
1530 #ifdef COMPAT_FREEBSD32
1531 struct bpf_program32 *fp32;
1532 struct bpf_program fp_swab;
1533
1534 switch (cmd) {
1535 case BIOCSETF32:
1536 case BIOCSETWF32:
1537 case BIOCSETFNR32:
1538 fp32 = (struct bpf_program32 *)fp;
1539 fp_swab.bf_len = fp32->bf_len;
1540 fp_swab.bf_insns = (struct bpf_insn *)(uintptr_t)fp32->bf_insns;
1541 fp = &fp_swab;
1542 switch (cmd) {
1543 case BIOCSETF32:
1544 cmd = BIOCSETF;
1545 break;
1546 case BIOCSETWF32:
1547 cmd = BIOCSETWF;
1548 break;
1549 }
1550 break;
1551 }
1552 #endif
1553 if (cmd == BIOCSETWF) {
1554 old = d->bd_wfilter;
1555 wfilter = 1;
1556 #ifdef BPF_JITTER
1557 ofunc = NULL;
1558 #endif
1559 } else {
1560 wfilter = 0;
1561 old = d->bd_rfilter;
1562 #ifdef BPF_JITTER
1563 ofunc = d->bd_bfilter;
1564 #endif
1565 }
1566 if (fp->bf_insns == NULL) {
1567 if (fp->bf_len != 0)
1568 return (EINVAL);
1569 BPFD_LOCK(d);
1570 if (wfilter)
1571 d->bd_wfilter = NULL;
1572 else {
1573 d->bd_rfilter = NULL;
1574 #ifdef BPF_JITTER
1575 d->bd_bfilter = NULL;
1576 #endif
1577 if (cmd == BIOCSETF)
1578 reset_d(d);
1579 }
1580 BPFD_UNLOCK(d);
1581 if (old != NULL)
1582 free((caddr_t)old, M_BPF);
1583 #ifdef BPF_JITTER
1584 if (ofunc != NULL)
1585 bpf_destroy_jit_filter(ofunc);
1586 #endif
1587 return (0);
1588 }
1589 flen = fp->bf_len;
1590 if (flen > bpf_maxinsns)
1591 return (EINVAL);
1592
1593 size = flen * sizeof(*fp->bf_insns);
1594 fcode = (struct bpf_insn *)malloc(size, M_BPF, M_WAITOK);
1595 if (copyin((caddr_t)fp->bf_insns, (caddr_t)fcode, size) == 0 &&
1596 bpf_validate(fcode, (int)flen)) {
1597 BPFD_LOCK(d);
1598 if (wfilter)
1599 d->bd_wfilter = fcode;
1600 else {
1601 d->bd_rfilter = fcode;
1602 #ifdef BPF_JITTER
1603 d->bd_bfilter = bpf_jitter(fcode, flen);
1604 #endif
1605 if (cmd == BIOCSETF)
1606 reset_d(d);
1607 }
1608 BPFD_UNLOCK(d);
1609 if (old != NULL)
1610 free((caddr_t)old, M_BPF);
1611 #ifdef BPF_JITTER
1612 if (ofunc != NULL)
1613 bpf_destroy_jit_filter(ofunc);
1614 #endif
1615
1616 return (0);
1617 }
1618 free((caddr_t)fcode, M_BPF);
1619 return (EINVAL);
1620 }
1621
1622 /*
1623 * Detach a file from its current interface (if attached at all) and attach
1624 * to the interface indicated by the name stored in ifr.
1625 * Return an errno or 0.
1626 */
1627 static int
1628 bpf_setif(struct bpf_d *d, struct ifreq *ifr)
1629 {
1630 struct bpf_if *bp;
1631 struct ifnet *theywant;
1632
1633 theywant = ifunit(ifr->ifr_name);
1634 if (theywant == NULL || theywant->if_bpf == NULL)
1635 return (ENXIO);
1636
1637 bp = theywant->if_bpf;
1638
1639 /*
1640 * Behavior here depends on the buffering model. If we're using
1641 * kernel memory buffers, then we can allocate them here. If we're
1642 * using zero-copy, then the user process must have registered
1643 * buffers by the time we get here. If not, return an error.
1644 *
1645 * XXXRW: There are locking issues here with multi-threaded use: what
1646 * if two threads try to set the interface at once?
1647 */
1648 switch (d->bd_bufmode) {
1649 case BPF_BUFMODE_BUFFER:
1650 if (d->bd_sbuf == NULL)
1651 bpf_buffer_alloc(d);
1652 KASSERT(d->bd_sbuf != NULL, ("bpf_setif: bd_sbuf NULL"));
1653 break;
1654
1655 case BPF_BUFMODE_ZBUF:
1656 if (d->bd_sbuf == NULL)
1657 return (EINVAL);
1658 break;
1659
1660 default:
1661 panic("bpf_setif: bufmode %d", d->bd_bufmode);
1662 }
1663 if (bp != d->bd_bif) {
1664 if (d->bd_bif)
1665 /*
1666 * Detach if attached to something else.
1667 */
1668 bpf_detachd(d);
1669
1670 bpf_attachd(d, bp);
1671 }
1672 BPFD_LOCK(d);
1673 reset_d(d);
1674 BPFD_UNLOCK(d);
1675 return (0);
1676 }
1677
1678 /*
1679 * Support for select() and poll() system calls
1680 *
1681 * Return true iff the specific operation will not block indefinitely.
1682 * Otherwise, return false but make a note that a selwakeup() must be done.
1683 */
1684 static int
1685 bpfpoll(struct cdev *dev, int events, struct thread *td)
1686 {
1687 struct bpf_d *d;
1688 int revents;
1689
1690 if (devfs_get_cdevpriv((void **)&d) != 0 || d->bd_bif == NULL)
1691 return (events &
1692 (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM));
1693
1694 /*
1695 * Refresh PID associated with this descriptor.
1696 */
1697 revents = events & (POLLOUT | POLLWRNORM);
1698 BPFD_LOCK(d);
1699 d->bd_pid = td->td_proc->p_pid;
1700 if (events & (POLLIN | POLLRDNORM)) {
1701 if (bpf_ready(d))
1702 revents |= events & (POLLIN | POLLRDNORM);
1703 else {
1704 selrecord(td, &d->bd_sel);
1705 /* Start the read timeout if necessary. */
1706 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
1707 callout_reset(&d->bd_callout, d->bd_rtout,
1708 bpf_timed_out, d);
1709 d->bd_state = BPF_WAITING;
1710 }
1711 }
1712 }
1713 BPFD_UNLOCK(d);
1714 return (revents);
1715 }
1716
1717 /*
1718 * Support for kevent() system call. Register EVFILT_READ filters and
1719 * reject all others.
1720 */
1721 int
1722 bpfkqfilter(struct cdev *dev, struct knote *kn)
1723 {
1724 struct bpf_d *d;
1725
1726 if (devfs_get_cdevpriv((void **)&d) != 0 ||
1727 kn->kn_filter != EVFILT_READ)
1728 return (1);
1729
1730 /*
1731 * Refresh PID associated with this descriptor.
1732 */
1733 BPFD_LOCK(d);
1734 d->bd_pid = curthread->td_proc->p_pid;
1735 kn->kn_fop = &bpfread_filtops;
1736 kn->kn_hook = d;
1737 knlist_add(&d->bd_sel.si_note, kn, 1);
1738 BPFD_UNLOCK(d);
1739
1740 return (0);
1741 }
1742
1743 static void
1744 filt_bpfdetach(struct knote *kn)
1745 {
1746 struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
1747
1748 knlist_remove(&d->bd_sel.si_note, kn, 0);
1749 }
1750
1751 static int
1752 filt_bpfread(struct knote *kn, long hint)
1753 {
1754 struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
1755 int ready;
1756
1757 BPFD_LOCK_ASSERT(d);
1758 ready = bpf_ready(d);
1759 if (ready) {
1760 kn->kn_data = d->bd_slen;
1761 if (d->bd_hbuf)
1762 kn->kn_data += d->bd_hlen;
1763 } else if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
1764 callout_reset(&d->bd_callout, d->bd_rtout,
1765 bpf_timed_out, d);
1766 d->bd_state = BPF_WAITING;
1767 }
1768
1769 return (ready);
1770 }
1771
1772 #define BPF_TSTAMP_NONE 0
1773 #define BPF_TSTAMP_FAST 1
1774 #define BPF_TSTAMP_NORMAL 2
1775 #define BPF_TSTAMP_EXTERN 3
1776
1777 static int
1778 bpf_ts_quality(int tstype)
1779 {
1780
1781 if (tstype == BPF_T_NONE)
1782 return (BPF_TSTAMP_NONE);
1783 if ((tstype & BPF_T_FAST) != 0)
1784 return (BPF_TSTAMP_FAST);
1785
1786 return (BPF_TSTAMP_NORMAL);
1787 }
1788
1789 static int
1790 bpf_gettime(struct bintime *bt, int tstype, struct mbuf *m)
1791 {
1792 struct m_tag *tag;
1793 int quality;
1794
1795 quality = bpf_ts_quality(tstype);
1796 if (quality == BPF_TSTAMP_NONE)
1797 return (quality);
1798
1799 if (m != NULL) {
1800 tag = m_tag_locate(m, MTAG_BPF, MTAG_BPF_TIMESTAMP, NULL);
1801 if (tag != NULL) {
1802 *bt = *(struct bintime *)(tag + 1);
1803 return (BPF_TSTAMP_EXTERN);
1804 }
1805 }
1806 if (quality == BPF_TSTAMP_NORMAL)
1807 binuptime(bt);
1808 else
1809 getbinuptime(bt);
1810
1811 return (quality);
1812 }
1813
1814 /*
1815 * Incoming linkage from device drivers. Process the packet pkt, of length
1816 * pktlen, which is stored in a contiguous buffer. The packet is parsed
1817 * by each process' filter, and if accepted, stashed into the corresponding
1818 * buffer.
1819 */
1820 void
1821 bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
1822 {
1823 struct bintime bt;
1824 struct bpf_d *d;
1825 #ifdef BPF_JITTER
1826 bpf_jit_filter *bf;
1827 #endif
1828 u_int slen;
1829 int gottime;
1830
1831 gottime = BPF_TSTAMP_NONE;
1832 BPFIF_LOCK(bp);
1833 LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
1834 BPFD_LOCK(d);
1835 ++d->bd_rcount;
1836 /*
1837 * NB: We dont call BPF_CHECK_DIRECTION() here since there is no
1838 * way for the caller to indiciate to us whether this packet
1839 * is inbound or outbound. In the bpf_mtap() routines, we use
1840 * the interface pointers on the mbuf to figure it out.
1841 */
1842 #ifdef BPF_JITTER
1843 bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL;
1844 if (bf != NULL)
1845 slen = (*(bf->func))(pkt, pktlen, pktlen);
1846 else
1847 #endif
1848 slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen);
1849 if (slen != 0) {
1850 d->bd_fcount++;
1851 if (gottime < bpf_ts_quality(d->bd_tstamp))
1852 gottime = bpf_gettime(&bt, d->bd_tstamp, NULL);
1853 #ifdef MAC
1854 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
1855 #endif
1856 catchpacket(d, pkt, pktlen, slen,
1857 bpf_append_bytes, &bt);
1858 }
1859 BPFD_UNLOCK(d);
1860 }
1861 BPFIF_UNLOCK(bp);
1862 }
1863
1864 #define BPF_CHECK_DIRECTION(d, r, i) \
1865 (((d)->bd_direction == BPF_D_IN && (r) != (i)) || \
1866 ((d)->bd_direction == BPF_D_OUT && (r) == (i)))
1867
1868 /*
1869 * Incoming linkage from device drivers, when packet is in an mbuf chain.
1870 */
1871 void
1872 bpf_mtap(struct bpf_if *bp, struct mbuf *m)
1873 {
1874 struct bintime bt;
1875 struct bpf_d *d;
1876 #ifdef BPF_JITTER
1877 bpf_jit_filter *bf;
1878 #endif
1879 u_int pktlen, slen;
1880 int gottime;
1881
1882 /* Skip outgoing duplicate packets. */
1883 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) {
1884 m->m_flags &= ~M_PROMISC;
1885 return;
1886 }
1887
1888 pktlen = m_length(m, NULL);
1889
1890 gottime = BPF_TSTAMP_NONE;
1891 BPFIF_LOCK(bp);
1892 LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
1893 if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp))
1894 continue;
1895 BPFD_LOCK(d);
1896 ++d->bd_rcount;
1897 #ifdef BPF_JITTER
1898 bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL;
1899 /* XXX We cannot handle multiple mbufs. */
1900 if (bf != NULL && m->m_next == NULL)
1901 slen = (*(bf->func))(mtod(m, u_char *), pktlen, pktlen);
1902 else
1903 #endif
1904 slen = bpf_filter(d->bd_rfilter, (u_char *)m, pktlen, 0);
1905 if (slen != 0) {
1906 d->bd_fcount++;
1907 if (gottime < bpf_ts_quality(d->bd_tstamp))
1908 gottime = bpf_gettime(&bt, d->bd_tstamp, m);
1909 #ifdef MAC
1910 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
1911 #endif
1912 catchpacket(d, (u_char *)m, pktlen, slen,
1913 bpf_append_mbuf, &bt);
1914 }
1915 BPFD_UNLOCK(d);
1916 }
1917 BPFIF_UNLOCK(bp);
1918 }
1919
1920 /*
1921 * Incoming linkage from device drivers, when packet is in
1922 * an mbuf chain and to be prepended by a contiguous header.
1923 */
1924 void
1925 bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m)
1926 {
1927 struct bintime bt;
1928 struct mbuf mb;
1929 struct bpf_d *d;
1930 u_int pktlen, slen;
1931 int gottime;
1932
1933 /* Skip outgoing duplicate packets. */
1934 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) {
1935 m->m_flags &= ~M_PROMISC;
1936 return;
1937 }
1938
1939 pktlen = m_length(m, NULL);
1940 /*
1941 * Craft on-stack mbuf suitable for passing to bpf_filter.
1942 * Note that we cut corners here; we only setup what's
1943 * absolutely needed--this mbuf should never go anywhere else.
1944 */
1945 mb.m_next = m;
1946 mb.m_data = data;
1947 mb.m_len = dlen;
1948 pktlen += dlen;
1949
1950 gottime = BPF_TSTAMP_NONE;
1951 BPFIF_LOCK(bp);
1952 LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
1953 if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp))
1954 continue;
1955 BPFD_LOCK(d);
1956 ++d->bd_rcount;
1957 slen = bpf_filter(d->bd_rfilter, (u_char *)&mb, pktlen, 0);
1958 if (slen != 0) {
1959 d->bd_fcount++;
1960 if (gottime < bpf_ts_quality(d->bd_tstamp))
1961 gottime = bpf_gettime(&bt, d->bd_tstamp, m);
1962 #ifdef MAC
1963 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
1964 #endif
1965 catchpacket(d, (u_char *)&mb, pktlen, slen,
1966 bpf_append_mbuf, &bt);
1967 }
1968 BPFD_UNLOCK(d);
1969 }
1970 BPFIF_UNLOCK(bp);
1971 }
1972
1973 #undef BPF_CHECK_DIRECTION
1974
1975 #undef BPF_TSTAMP_NONE
1976 #undef BPF_TSTAMP_FAST
1977 #undef BPF_TSTAMP_NORMAL
1978 #undef BPF_TSTAMP_EXTERN
1979
1980 static int
1981 bpf_hdrlen(struct bpf_d *d)
1982 {
1983 int hdrlen;
1984
1985 hdrlen = d->bd_bif->bif_hdrlen;
1986 #ifndef BURN_BRIDGES
1987 if (d->bd_tstamp == BPF_T_NONE ||
1988 BPF_T_FORMAT(d->bd_tstamp) == BPF_T_MICROTIME)
1989 #ifdef COMPAT_FREEBSD32
1990 if (d->bd_compat32)
1991 hdrlen += SIZEOF_BPF_HDR(struct bpf_hdr32);
1992 else
1993 #endif
1994 hdrlen += SIZEOF_BPF_HDR(struct bpf_hdr);
1995 else
1996 #endif
1997 hdrlen += SIZEOF_BPF_HDR(struct bpf_xhdr);
1998 #ifdef COMPAT_FREEBSD32
1999 if (d->bd_compat32)
2000 hdrlen = BPF_WORDALIGN32(hdrlen);
2001 else
2002 #endif
2003 hdrlen = BPF_WORDALIGN(hdrlen);
2004
2005 return (hdrlen - d->bd_bif->bif_hdrlen);
2006 }
2007
2008 static void
2009 bpf_bintime2ts(struct bintime *bt, struct bpf_ts *ts, int tstype)
2010 {
2011 struct bintime bt2;
2012 struct timeval tsm;
2013 struct timespec tsn;
2014
2015 if ((tstype & BPF_T_MONOTONIC) == 0) {
2016 bt2 = *bt;
2017 bintime_add(&bt2, &boottimebin);
2018 bt = &bt2;
2019 }
2020 switch (BPF_T_FORMAT(tstype)) {
2021 case BPF_T_MICROTIME:
2022 bintime2timeval(bt, &tsm);
2023 ts->bt_sec = tsm.tv_sec;
2024 ts->bt_frac = tsm.tv_usec;
2025 break;
2026 case BPF_T_NANOTIME:
2027 bintime2timespec(bt, &tsn);
2028 ts->bt_sec = tsn.tv_sec;
2029 ts->bt_frac = tsn.tv_nsec;
2030 break;
2031 case BPF_T_BINTIME:
2032 ts->bt_sec = bt->sec;
2033 ts->bt_frac = bt->frac;
2034 break;
2035 }
2036 }
2037
2038 /*
2039 * Move the packet data from interface memory (pkt) into the
2040 * store buffer. "cpfn" is the routine called to do the actual data
2041 * transfer. bcopy is passed in to copy contiguous chunks, while
2042 * bpf_append_mbuf is passed in to copy mbuf chains. In the latter case,
2043 * pkt is really an mbuf.
2044 */
2045 static void
2046 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
2047 void (*cpfn)(struct bpf_d *, caddr_t, u_int, void *, u_int),
2048 struct bintime *bt)
2049 {
2050 struct bpf_xhdr hdr;
2051 #ifndef BURN_BRIDGES
2052 struct bpf_hdr hdr_old;
2053 #ifdef COMPAT_FREEBSD32
2054 struct bpf_hdr32 hdr32_old;
2055 #endif
2056 #endif
2057 int caplen, curlen, hdrlen, totlen;
2058 int do_wakeup = 0;
2059 int do_timestamp;
2060 int tstype;
2061
2062 BPFD_LOCK_ASSERT(d);
2063
2064 /*
2065 * Detect whether user space has released a buffer back to us, and if
2066 * so, move it from being a hold buffer to a free buffer. This may
2067 * not be the best place to do it (for example, we might only want to
2068 * run this check if we need the space), but for now it's a reliable
2069 * spot to do it.
2070 */
2071 if (d->bd_fbuf == NULL && bpf_canfreebuf(d)) {
2072 d->bd_fbuf = d->bd_hbuf;
2073 d->bd_hbuf = NULL;
2074 d->bd_hlen = 0;
2075 bpf_buf_reclaimed(d);
2076 }
2077
2078 /*
2079 * Figure out how many bytes to move. If the packet is
2080 * greater or equal to the snapshot length, transfer that
2081 * much. Otherwise, transfer the whole packet (unless
2082 * we hit the buffer size limit).
2083 */
2084 hdrlen = bpf_hdrlen(d);
2085 totlen = hdrlen + min(snaplen, pktlen);
2086 if (totlen > d->bd_bufsize)
2087 totlen = d->bd_bufsize;
2088
2089 /*
2090 * Round up the end of the previous packet to the next longword.
2091 *
2092 * Drop the packet if there's no room and no hope of room
2093 * If the packet would overflow the storage buffer or the storage
2094 * buffer is considered immutable by the buffer model, try to rotate
2095 * the buffer and wakeup pending processes.
2096 */
2097 #ifdef COMPAT_FREEBSD32
2098 if (d->bd_compat32)
2099 curlen = BPF_WORDALIGN32(d->bd_slen);
2100 else
2101 #endif
2102 curlen = BPF_WORDALIGN(d->bd_slen);
2103 if (curlen + totlen > d->bd_bufsize || !bpf_canwritebuf(d)) {
2104 if (d->bd_fbuf == NULL) {
2105 /*
2106 * There's no room in the store buffer, and no
2107 * prospect of room, so drop the packet. Notify the
2108 * buffer model.
2109 */
2110 bpf_buffull(d);
2111 ++d->bd_dcount;
2112 return;
2113 }
2114 ROTATE_BUFFERS(d);
2115 do_wakeup = 1;
2116 curlen = 0;
2117 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT)
2118 /*
2119 * Immediate mode is set, or the read timeout has already
2120 * expired during a select call. A packet arrived, so the
2121 * reader should be woken up.
2122 */
2123 do_wakeup = 1;
2124 caplen = totlen - hdrlen;
2125 tstype = d->bd_tstamp;
2126 do_timestamp = tstype != BPF_T_NONE;
2127 #ifndef BURN_BRIDGES
2128 if (tstype == BPF_T_NONE || BPF_T_FORMAT(tstype) == BPF_T_MICROTIME) {
2129 struct bpf_ts ts;
2130 if (do_timestamp)
2131 bpf_bintime2ts(bt, &ts, tstype);
2132 #ifdef COMPAT_FREEBSD32
2133 if (d->bd_compat32) {
2134 bzero(&hdr32_old, sizeof(hdr32_old));
2135 if (do_timestamp) {
2136 hdr32_old.bh_tstamp.tv_sec = ts.bt_sec;
2137 hdr32_old.bh_tstamp.tv_usec = ts.bt_frac;
2138 }
2139 hdr32_old.bh_datalen = pktlen;
2140 hdr32_old.bh_hdrlen = hdrlen;
2141 hdr32_old.bh_caplen = caplen;
2142 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr32_old,
2143 sizeof(hdr32_old));
2144 goto copy;
2145 }
2146 #endif
2147 bzero(&hdr_old, sizeof(hdr_old));
2148 if (do_timestamp) {
2149 hdr_old.bh_tstamp.tv_sec = ts.bt_sec;
2150 hdr_old.bh_tstamp.tv_usec = ts.bt_frac;
2151 }
2152 hdr_old.bh_datalen = pktlen;
2153 hdr_old.bh_hdrlen = hdrlen;
2154 hdr_old.bh_caplen = caplen;
2155 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr_old,
2156 sizeof(hdr_old));
2157 goto copy;
2158 }
2159 #endif
2160
2161 /*
2162 * Append the bpf header. Note we append the actual header size, but
2163 * move forward the length of the header plus padding.
2164 */
2165 bzero(&hdr, sizeof(hdr));
2166 if (do_timestamp)
2167 bpf_bintime2ts(bt, &hdr.bh_tstamp, tstype);
2168 hdr.bh_datalen = pktlen;
2169 hdr.bh_hdrlen = hdrlen;
2170 hdr.bh_caplen = caplen;
2171 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr, sizeof(hdr));
2172
2173 /*
2174 * Copy the packet data into the store buffer and update its length.
2175 */
2176 #ifndef BURN_BRIDGES
2177 copy:
2178 #endif
2179 (*cpfn)(d, d->bd_sbuf, curlen + hdrlen, pkt, caplen);
2180 d->bd_slen = curlen + totlen;
2181
2182 if (do_wakeup)
2183 bpf_wakeup(d);
2184 }
2185
2186 /*
2187 * Free buffers currently in use by a descriptor.
2188 * Called on close.
2189 */
2190 static void
2191 bpf_freed(struct bpf_d *d)
2192 {
2193
2194 /*
2195 * We don't need to lock out interrupts since this descriptor has
2196 * been detached from its interface and it yet hasn't been marked
2197 * free.
2198 */
2199 bpf_free(d);
2200 if (d->bd_rfilter != NULL) {
2201 free((caddr_t)d->bd_rfilter, M_BPF);
2202 #ifdef BPF_JITTER
2203 if (d->bd_bfilter != NULL)
2204 bpf_destroy_jit_filter(d->bd_bfilter);
2205 #endif
2206 }
2207 if (d->bd_wfilter != NULL)
2208 free((caddr_t)d->bd_wfilter, M_BPF);
2209 mtx_destroy(&d->bd_mtx);
2210 }
2211
2212 /*
2213 * Attach an interface to bpf. dlt is the link layer type; hdrlen is the
2214 * fixed size of the link header (variable length headers not yet supported).
2215 */
2216 void
2217 bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
2218 {
2219
2220 bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf);
2221 }
2222
2223 /*
2224 * Attach an interface to bpf. ifp is a pointer to the structure
2225 * defining the interface to be attached, dlt is the link layer type,
2226 * and hdrlen is the fixed size of the link header (variable length
2227 * headers are not yet supporrted).
2228 */
2229 void
2230 bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp)
2231 {
2232 struct bpf_if *bp;
2233
2234 bp = malloc(sizeof(*bp), M_BPF, M_NOWAIT | M_ZERO);
2235 if (bp == NULL)
2236 panic("bpfattach");
2237
2238 LIST_INIT(&bp->bif_dlist);
2239 bp->bif_ifp = ifp;
2240 bp->bif_dlt = dlt;
2241 mtx_init(&bp->bif_mtx, "bpf interface lock", NULL, MTX_DEF);
2242 KASSERT(*driverp == NULL, ("bpfattach2: driverp already initialized"));
2243 *driverp = bp;
2244
2245 mtx_lock(&bpf_mtx);
2246 LIST_INSERT_HEAD(&bpf_iflist, bp, bif_next);
2247 mtx_unlock(&bpf_mtx);
2248
2249 bp->bif_hdrlen = hdrlen;
2250
2251 if (bootverbose)
2252 if_printf(ifp, "bpf attached\n");
2253 }
2254
2255 /*
2256 * Detach bpf from an interface. This involves detaching each descriptor
2257 * associated with the interface, and leaving bd_bif NULL. Notify each
2258 * descriptor as it's detached so that any sleepers wake up and get
2259 * ENXIO.
2260 */
2261 void
2262 bpfdetach(struct ifnet *ifp)
2263 {
2264 struct bpf_if *bp;
2265 struct bpf_d *d;
2266 #ifdef INVARIANTS
2267 int ndetached;
2268
2269 ndetached = 0;
2270 #endif
2271
2272 /* Find all bpf_if struct's which reference ifp and detach them. */
2273 do {
2274 mtx_lock(&bpf_mtx);
2275 LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2276 if (ifp == bp->bif_ifp)
2277 break;
2278 }
2279 if (bp != NULL)
2280 LIST_REMOVE(bp, bif_next);
2281 mtx_unlock(&bpf_mtx);
2282
2283 if (bp != NULL) {
2284 #ifdef INVARIANTS
2285 ndetached++;
2286 #endif
2287 while ((d = LIST_FIRST(&bp->bif_dlist)) != NULL) {
2288 bpf_detachd(d);
2289 BPFD_LOCK(d);
2290 bpf_wakeup(d);
2291 BPFD_UNLOCK(d);
2292 }
2293 mtx_destroy(&bp->bif_mtx);
2294 free(bp, M_BPF);
2295 }
2296 } while (bp != NULL);
2297
2298 #ifdef INVARIANTS
2299 if (ndetached == 0)
2300 printf("bpfdetach: %s was not attached\n", ifp->if_xname);
2301 #endif
2302 }
2303
2304 /*
2305 * Get a list of available data link type of the interface.
2306 */
2307 static int
2308 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl)
2309 {
2310 int n, error;
2311 struct ifnet *ifp;
2312 struct bpf_if *bp;
2313
2314 ifp = d->bd_bif->bif_ifp;
2315 n = 0;
2316 error = 0;
2317 mtx_lock(&bpf_mtx);
2318 LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2319 if (bp->bif_ifp != ifp)
2320 continue;
2321 if (bfl->bfl_list != NULL) {
2322 if (n >= bfl->bfl_len) {
2323 mtx_unlock(&bpf_mtx);
2324 return (ENOMEM);
2325 }
2326 error = copyout(&bp->bif_dlt,
2327 bfl->bfl_list + n, sizeof(u_int));
2328 }
2329 n++;
2330 }
2331 mtx_unlock(&bpf_mtx);
2332 bfl->bfl_len = n;
2333 return (error);
2334 }
2335
2336 /*
2337 * Set the data link type of a BPF instance.
2338 */
2339 static int
2340 bpf_setdlt(struct bpf_d *d, u_int dlt)
2341 {
2342 int error, opromisc;
2343 struct ifnet *ifp;
2344 struct bpf_if *bp;
2345
2346 if (d->bd_bif->bif_dlt == dlt)
2347 return (0);
2348 ifp = d->bd_bif->bif_ifp;
2349 mtx_lock(&bpf_mtx);
2350 LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2351 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt)
2352 break;
2353 }
2354 mtx_unlock(&bpf_mtx);
2355 if (bp != NULL) {
2356 opromisc = d->bd_promisc;
2357 bpf_detachd(d);
2358 bpf_attachd(d, bp);
2359 BPFD_LOCK(d);
2360 reset_d(d);
2361 BPFD_UNLOCK(d);
2362 if (opromisc) {
2363 error = ifpromisc(bp->bif_ifp, 1);
2364 if (error)
2365 if_printf(bp->bif_ifp,
2366 "bpf_setdlt: ifpromisc failed (%d)\n",
2367 error);
2368 else
2369 d->bd_promisc = 1;
2370 }
2371 }
2372 return (bp == NULL ? EINVAL : 0);
2373 }
2374
2375 static void
2376 bpf_drvinit(void *unused)
2377 {
2378 struct cdev *dev;
2379
2380 mtx_init(&bpf_mtx, "bpf global lock", NULL, MTX_DEF);
2381 LIST_INIT(&bpf_iflist);
2382
2383 dev = make_dev(&bpf_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "bpf");
2384 /* For compatibility */
2385 make_dev_alias(dev, "bpf0");
2386 }
2387
2388 /*
2389 * Zero out the various packet counters associated with all of the bpf
2390 * descriptors. At some point, we will probably want to get a bit more
2391 * granular and allow the user to specify descriptors to be zeroed.
2392 */
2393 static void
2394 bpf_zero_counters(void)
2395 {
2396 struct bpf_if *bp;
2397 struct bpf_d *bd;
2398
2399 mtx_lock(&bpf_mtx);
2400 LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2401 BPFIF_LOCK(bp);
2402 LIST_FOREACH(bd, &bp->bif_dlist, bd_next) {
2403 BPFD_LOCK(bd);
2404 bd->bd_rcount = 0;
2405 bd->bd_dcount = 0;
2406 bd->bd_fcount = 0;
2407 bd->bd_wcount = 0;
2408 bd->bd_wfcount = 0;
2409 bd->bd_zcopy = 0;
2410 BPFD_UNLOCK(bd);
2411 }
2412 BPFIF_UNLOCK(bp);
2413 }
2414 mtx_unlock(&bpf_mtx);
2415 }
2416
2417 static void
2418 bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd)
2419 {
2420
2421 bzero(d, sizeof(*d));
2422 BPFD_LOCK_ASSERT(bd);
2423 d->bd_structsize = sizeof(*d);
2424 d->bd_immediate = bd->bd_immediate;
2425 d->bd_promisc = bd->bd_promisc;
2426 d->bd_hdrcmplt = bd->bd_hdrcmplt;
2427 d->bd_direction = bd->bd_direction;
2428 d->bd_feedback = bd->bd_feedback;
2429 d->bd_async = bd->bd_async;
2430 d->bd_rcount = bd->bd_rcount;
2431 d->bd_dcount = bd->bd_dcount;
2432 d->bd_fcount = bd->bd_fcount;
2433 d->bd_sig = bd->bd_sig;
2434 d->bd_slen = bd->bd_slen;
2435 d->bd_hlen = bd->bd_hlen;
2436 d->bd_bufsize = bd->bd_bufsize;
2437 d->bd_pid = bd->bd_pid;
2438 strlcpy(d->bd_ifname,
2439 bd->bd_bif->bif_ifp->if_xname, IFNAMSIZ);
2440 d->bd_locked = bd->bd_locked;
2441 d->bd_wcount = bd->bd_wcount;
2442 d->bd_wdcount = bd->bd_wdcount;
2443 d->bd_wfcount = bd->bd_wfcount;
2444 d->bd_zcopy = bd->bd_zcopy;
2445 d->bd_bufmode = bd->bd_bufmode;
2446 }
2447
2448 static int
2449 bpf_stats_sysctl(SYSCTL_HANDLER_ARGS)
2450 {
2451 struct xbpf_d *xbdbuf, *xbd, zerostats;
2452 int index, error;
2453 struct bpf_if *bp;
2454 struct bpf_d *bd;
2455
2456 /*
2457 * XXX This is not technically correct. It is possible for non
2458 * privileged users to open bpf devices. It would make sense
2459 * if the users who opened the devices were able to retrieve
2460 * the statistics for them, too.
2461 */
2462 error = priv_check(req->td, PRIV_NET_BPF);
2463 if (error)
2464 return (error);
2465 /*
2466 * Check to see if the user is requesting that the counters be
2467 * zeroed out. Explicitly check that the supplied data is zeroed,
2468 * as we aren't allowing the user to set the counters currently.
2469 */
2470 if (req->newptr != NULL) {
2471 if (req->newlen != sizeof(zerostats))
2472 return (EINVAL);
2473 bzero(&zerostats, sizeof(zerostats));
2474 xbd = req->newptr;
2475 if (bcmp(xbd, &zerostats, sizeof(*xbd)) != 0)
2476 return (EINVAL);
2477 bpf_zero_counters();
2478 return (0);
2479 }
2480 if (req->oldptr == NULL)
2481 return (SYSCTL_OUT(req, 0, bpf_bpfd_cnt * sizeof(*xbd)));
2482 if (bpf_bpfd_cnt == 0)
2483 return (SYSCTL_OUT(req, 0, 0));
2484 xbdbuf = malloc(req->oldlen, M_BPF, M_WAITOK);
2485 mtx_lock(&bpf_mtx);
2486 if (req->oldlen < (bpf_bpfd_cnt * sizeof(*xbd))) {
2487 mtx_unlock(&bpf_mtx);
2488 free(xbdbuf, M_BPF);
2489 return (ENOMEM);
2490 }
2491 index = 0;
2492 LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2493 BPFIF_LOCK(bp);
2494 LIST_FOREACH(bd, &bp->bif_dlist, bd_next) {
2495 xbd = &xbdbuf[index++];
2496 BPFD_LOCK(bd);
2497 bpfstats_fill_xbpf(xbd, bd);
2498 BPFD_UNLOCK(bd);
2499 }
2500 BPFIF_UNLOCK(bp);
2501 }
2502 mtx_unlock(&bpf_mtx);
2503 error = SYSCTL_OUT(req, xbdbuf, index * sizeof(*xbd));
2504 free(xbdbuf, M_BPF);
2505 return (error);
2506 }
2507
2508 SYSINIT(bpfdev,SI_SUB_DRIVERS,SI_ORDER_MIDDLE,bpf_drvinit,NULL);
2509
2510 #else /* !DEV_BPF && !NETGRAPH_BPF */
2511 /*
2512 * NOP stubs to allow bpf-using drivers to load and function.
2513 *
2514 * A 'better' implementation would allow the core bpf functionality
2515 * to be loaded at runtime.
2516 */
2517 static struct bpf_if bp_null;
2518
2519 void
2520 bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
2521 {
2522 }
2523
2524 void
2525 bpf_mtap(struct bpf_if *bp, struct mbuf *m)
2526 {
2527 }
2528
2529 void
2530 bpf_mtap2(struct bpf_if *bp, void *d, u_int l, struct mbuf *m)
2531 {
2532 }
2533
2534 void
2535 bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
2536 {
2537
2538 bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf);
2539 }
2540
2541 void
2542 bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp)
2543 {
2544
2545 *driverp = &bp_null;
2546 }
2547
2548 void
2549 bpfdetach(struct ifnet *ifp)
2550 {
2551 }
2552
2553 u_int
2554 bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen)
2555 {
2556 return -1; /* "no filter" behaviour */
2557 }
2558
2559 int
2560 bpf_validate(const struct bpf_insn *f, int len)
2561 {
2562 return 0; /* false */
2563 }
2564
2565 #endif /* !DEV_BPF && !NETGRAPH_BPF */
Cache object: 19bcf752a79b690daaac04eb6a1c0dba
|