The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/net/bpf_zerocopy.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2007 Seccuris Inc.
    3  * All rights reserved.
    4  *
    5  * This sofware was developed by Robert N. M. Watson under contract to
    6  * Seccuris Inc.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  */
   29 
   30 #include <sys/cdefs.h>
   31 __FBSDID("$FreeBSD: releng/9.0/sys/net/bpf_zerocopy.c 216699 2010-12-25 21:26:56Z alc $");
   32 
   33 #include "opt_bpf.h"
   34 
   35 #include <sys/param.h>
   36 #include <sys/lock.h>
   37 #include <sys/malloc.h>
   38 #include <sys/mbuf.h>
   39 #include <sys/mutex.h>
   40 #include <sys/proc.h>
   41 #include <sys/sf_buf.h>
   42 #include <sys/socket.h>
   43 #include <sys/uio.h>
   44 
   45 #include <machine/atomic.h>
   46 
   47 #include <net/if.h>
   48 #include <net/bpf.h>
   49 #include <net/bpf_zerocopy.h>
   50 #include <net/bpfdesc.h>
   51 
   52 #include <vm/vm.h>
   53 #include <vm/pmap.h>
   54 #include <vm/vm_extern.h>
   55 #include <vm/vm_map.h>
   56 #include <vm/vm_page.h>
   57 
   58 /*
   59  * Zero-copy buffer scheme for BPF: user space "donates" two buffers, which
   60  * are mapped into the kernel address space using sf_bufs and used directly
   61  * by BPF.  Memory is wired since page faults cannot be tolerated in the
   62  * contexts where the buffers are copied to (locks held, interrupt context,
   63  * etc).  Access to shared memory buffers is synchronized using a header on
   64  * each buffer, allowing the number of system calls to go to zero as BPF
   65  * reaches saturation (buffers filled as fast as they can be drained by the
   66  * user process).  Full details of the protocol for communicating between the
   67  * user process and BPF may be found in bpf(4).
   68  */
   69 
   70 /*
   71  * Maximum number of pages per buffer.  Since all BPF devices use two, the
   72  * maximum per device is 2*BPF_MAX_PAGES.  Resource limits on the number of
   73  * sf_bufs may be an issue, so do not set this too high.  On older systems,
   74  * kernel address space limits may also be an issue.
   75  */
   76 #define BPF_MAX_PAGES   512
   77 
   78 /*
   79  * struct zbuf describes a memory buffer loaned by a user process to the
   80  * kernel.  We represent this as a series of pages managed using an array of
   81  * sf_bufs.  Even though the memory is contiguous in user space, it may not
   82  * be mapped contiguously in the kernel (i.e., a set of physically
   83  * non-contiguous pages in the direct map region) so we must implement
   84  * scatter-gather copying.  One significant mitigating factor is that on
   85  * systems with a direct memory map, we can avoid TLB misses.
   86  *
   87  * At the front of the shared memory region is a bpf_zbuf_header, which
   88  * contains shared control data to allow user space and the kernel to
   89  * synchronize; this is included in zb_size, but not bpf_bufsize, so that BPF
   90  * knows that the space is not available.
   91  */
   92 struct zbuf {
   93         vm_offset_t      zb_uaddr;      /* User address at time of setup. */
   94         size_t           zb_size;       /* Size of buffer, incl. header. */
   95         u_int            zb_numpages;   /* Number of pages. */
   96         int              zb_flags;      /* Flags on zbuf. */
   97         struct sf_buf   **zb_pages;     /* Pages themselves. */
   98         struct bpf_zbuf_header  *zb_header;     /* Shared header. */
   99 };
  100 
  101 /*
  102  * When a buffer has been assigned to userspace, flag it as such, as the
  103  * buffer may remain in the store position as a result of the user process
  104  * not yet having acknowledged the buffer in the hold position yet.
  105  */
  106 #define ZBUF_FLAG_ASSIGNED      0x00000001      /* Set when owned by user. */
  107 
  108 /*
  109  * Release a page we've previously wired.
  110  */
  111 static void
  112 zbuf_page_free(vm_page_t pp)
  113 {
  114 
  115         vm_page_lock(pp);
  116         vm_page_unwire(pp, 0);
  117         if (pp->wire_count == 0 && pp->object == NULL)
  118                 vm_page_free(pp);
  119         vm_page_unlock(pp);
  120 }
  121 
  122 /*
  123  * Free an sf_buf with attached page.
  124  */
  125 static void
  126 zbuf_sfbuf_free(struct sf_buf *sf)
  127 {
  128         vm_page_t pp;
  129 
  130         pp = sf_buf_page(sf);
  131         sf_buf_free(sf);
  132         zbuf_page_free(pp);
  133 }
  134 
  135 /*
  136  * Free a zbuf, including its page array, sbufs, and pages.  Allow partially
  137  * allocated zbufs to be freed so that it may be used even during a zbuf
  138  * setup.
  139  */
  140 static void
  141 zbuf_free(struct zbuf *zb)
  142 {
  143         int i;
  144 
  145         for (i = 0; i < zb->zb_numpages; i++) {
  146                 if (zb->zb_pages[i] != NULL)
  147                         zbuf_sfbuf_free(zb->zb_pages[i]);
  148         }
  149         free(zb->zb_pages, M_BPF);
  150         free(zb, M_BPF);
  151 }
  152 
  153 /*
  154  * Given a user pointer to a page of user memory, return an sf_buf for the
  155  * page.  Because we may be requesting quite a few sf_bufs, prefer failure to
  156  * deadlock and use SFB_NOWAIT.
  157  */
  158 static struct sf_buf *
  159 zbuf_sfbuf_get(struct vm_map *map, vm_offset_t uaddr)
  160 {
  161         struct sf_buf *sf;
  162         vm_page_t pp;
  163 
  164         if (vm_fault_quick_hold_pages(map, uaddr, PAGE_SIZE, VM_PROT_READ |
  165             VM_PROT_WRITE, &pp, 1) < 0)
  166                 return (NULL);
  167         vm_page_lock(pp);
  168         vm_page_wire(pp);
  169         vm_page_unhold(pp);
  170         vm_page_unlock(pp);
  171         sf = sf_buf_alloc(pp, SFB_NOWAIT);
  172         if (sf == NULL) {
  173                 zbuf_page_free(pp);
  174                 return (NULL);
  175         }
  176         return (sf);
  177 }
  178 
  179 /*
  180  * Create a zbuf describing a range of user address space memory.  Validate
  181  * page alignment, size requirements, etc.
  182  */
  183 static int
  184 zbuf_setup(struct thread *td, vm_offset_t uaddr, size_t len,
  185     struct zbuf **zbp)
  186 {
  187         struct zbuf *zb;
  188         struct vm_map *map;
  189         int error, i;
  190 
  191         *zbp = NULL;
  192 
  193         /*
  194          * User address must be page-aligned.
  195          */
  196         if (uaddr & PAGE_MASK)
  197                 return (EINVAL);
  198 
  199         /*
  200          * Length must be an integer number of full pages.
  201          */
  202         if (len & PAGE_MASK)
  203                 return (EINVAL);
  204 
  205         /*
  206          * Length must not exceed per-buffer resource limit.
  207          */
  208         if ((len / PAGE_SIZE) > BPF_MAX_PAGES)
  209                 return (EINVAL);
  210 
  211         /*
  212          * Allocate the buffer and set up each page with is own sf_buf.
  213          */
  214         error = 0;
  215         zb = malloc(sizeof(*zb), M_BPF, M_ZERO | M_WAITOK);
  216         zb->zb_uaddr = uaddr;
  217         zb->zb_size = len;
  218         zb->zb_numpages = len / PAGE_SIZE;
  219         zb->zb_pages = malloc(sizeof(struct sf_buf *) *
  220             zb->zb_numpages, M_BPF, M_ZERO | M_WAITOK);
  221         map = &td->td_proc->p_vmspace->vm_map;
  222         for (i = 0; i < zb->zb_numpages; i++) {
  223                 zb->zb_pages[i] = zbuf_sfbuf_get(map,
  224                     uaddr + (i * PAGE_SIZE));
  225                 if (zb->zb_pages[i] == NULL) {
  226                         error = EFAULT;
  227                         goto error;
  228                 }
  229         }
  230         zb->zb_header =
  231             (struct bpf_zbuf_header *)sf_buf_kva(zb->zb_pages[0]);
  232         bzero(zb->zb_header, sizeof(*zb->zb_header));
  233         *zbp = zb;
  234         return (0);
  235 
  236 error:
  237         zbuf_free(zb);
  238         return (error);
  239 }
  240 
  241 /*
  242  * Copy bytes from a source into the specified zbuf.  The caller is
  243  * responsible for performing bounds checking, etc.
  244  */
  245 void
  246 bpf_zerocopy_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset,
  247     void *src, u_int len)
  248 {
  249         u_int count, page, poffset;
  250         u_char *src_bytes;
  251         struct zbuf *zb;
  252 
  253         KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
  254             ("bpf_zerocopy_append_bytes: not in zbuf mode"));
  255         KASSERT(buf != NULL, ("bpf_zerocopy_append_bytes: NULL buf"));
  256 
  257         src_bytes = (u_char *)src;
  258         zb = (struct zbuf *)buf;
  259 
  260         KASSERT((zb->zb_flags & ZBUF_FLAG_ASSIGNED) == 0,
  261             ("bpf_zerocopy_append_bytes: ZBUF_FLAG_ASSIGNED"));
  262 
  263         /*
  264          * Scatter-gather copy to user pages mapped into kernel address space
  265          * using sf_bufs: copy up to a page at a time.
  266          */
  267         offset += sizeof(struct bpf_zbuf_header);
  268         page = offset / PAGE_SIZE;
  269         poffset = offset % PAGE_SIZE;
  270         while (len > 0) {
  271                 KASSERT(page < zb->zb_numpages, ("bpf_zerocopy_append_bytes:"
  272                    " page overflow (%d p %d np)\n", page, zb->zb_numpages));
  273 
  274                 count = min(len, PAGE_SIZE - poffset);
  275                 bcopy(src_bytes, ((u_char *)sf_buf_kva(zb->zb_pages[page])) +
  276                     poffset, count);
  277                 poffset += count;
  278                 if (poffset == PAGE_SIZE) {
  279                         poffset = 0;
  280                         page++;
  281                 }
  282                 KASSERT(poffset < PAGE_SIZE,
  283                     ("bpf_zerocopy_append_bytes: page offset overflow (%d)",
  284                     poffset));
  285                 len -= count;
  286                 src_bytes += count;
  287         }
  288 }
  289 
  290 /*
  291  * Copy bytes from an mbuf chain to the specified zbuf: copying will be
  292  * scatter-gather both from mbufs, which may be fragmented over memory, and
  293  * to pages, which may not be contiguously mapped in kernel address space.
  294  * As with bpf_zerocopy_append_bytes(), the caller is responsible for
  295  * checking that this will not exceed the buffer limit.
  296  */
  297 void
  298 bpf_zerocopy_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset,
  299     void *src, u_int len)
  300 {
  301         u_int count, moffset, page, poffset;
  302         const struct mbuf *m;
  303         struct zbuf *zb;
  304 
  305         KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
  306             ("bpf_zerocopy_append_mbuf not in zbuf mode"));
  307         KASSERT(buf != NULL, ("bpf_zerocopy_append_mbuf: NULL buf"));
  308 
  309         m = (struct mbuf *)src;
  310         zb = (struct zbuf *)buf;
  311 
  312         KASSERT((zb->zb_flags & ZBUF_FLAG_ASSIGNED) == 0,
  313             ("bpf_zerocopy_append_mbuf: ZBUF_FLAG_ASSIGNED"));
  314 
  315         /*
  316          * Scatter gather both from an mbuf chain and to a user page set
  317          * mapped into kernel address space using sf_bufs.  If we're lucky,
  318          * each mbuf requires one copy operation, but if page alignment and
  319          * mbuf alignment work out less well, we'll be doing two copies per
  320          * mbuf.
  321          */
  322         offset += sizeof(struct bpf_zbuf_header);
  323         page = offset / PAGE_SIZE;
  324         poffset = offset % PAGE_SIZE;
  325         moffset = 0;
  326         while (len > 0) {
  327                 KASSERT(page < zb->zb_numpages,
  328                     ("bpf_zerocopy_append_mbuf: page overflow (%d p %d "
  329                     "np)\n", page, zb->zb_numpages));
  330                 KASSERT(m != NULL,
  331                     ("bpf_zerocopy_append_mbuf: end of mbuf chain"));
  332 
  333                 count = min(m->m_len - moffset, len);
  334                 count = min(count, PAGE_SIZE - poffset);
  335                 bcopy(mtod(m, u_char *) + moffset,
  336                     ((u_char *)sf_buf_kva(zb->zb_pages[page])) + poffset,
  337                     count);
  338                 poffset += count;
  339                 if (poffset == PAGE_SIZE) {
  340                         poffset = 0;
  341                         page++;
  342                 }
  343                 KASSERT(poffset < PAGE_SIZE,
  344                     ("bpf_zerocopy_append_mbuf: page offset overflow (%d)",
  345                     poffset));
  346                 moffset += count;
  347                 if (moffset == m->m_len) {
  348                         m = m->m_next;
  349                         moffset = 0;
  350                 }
  351                 len -= count;
  352         }
  353 }
  354 
  355 /*
  356  * Notification from the BPF framework that a buffer in the store position is
  357  * rejecting packets and may be considered full.  We mark the buffer as
  358  * immutable and assign to userspace so that it is immediately available for
  359  * the user process to access.
  360  */
  361 void
  362 bpf_zerocopy_buffull(struct bpf_d *d)
  363 {
  364         struct zbuf *zb;
  365 
  366         KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
  367             ("bpf_zerocopy_buffull: not in zbuf mode"));
  368 
  369         zb = (struct zbuf *)d->bd_sbuf;
  370         KASSERT(zb != NULL, ("bpf_zerocopy_buffull: zb == NULL"));
  371 
  372         if ((zb->zb_flags & ZBUF_FLAG_ASSIGNED) == 0) {
  373                 zb->zb_flags |= ZBUF_FLAG_ASSIGNED;
  374                 zb->zb_header->bzh_kernel_len = d->bd_slen;
  375                 atomic_add_rel_int(&zb->zb_header->bzh_kernel_gen, 1);
  376         }
  377 }
  378 
  379 /*
  380  * Notification from the BPF framework that a buffer has moved into the held
  381  * slot on a descriptor.  Zero-copy BPF will update the shared page to let
  382  * the user process know and flag the buffer as assigned if it hasn't already
  383  * been marked assigned due to filling while it was in the store position.
  384  *
  385  * Note: identical logic as in bpf_zerocopy_buffull(), except that we operate
  386  * on bd_hbuf and bd_hlen.
  387  */
  388 void
  389 bpf_zerocopy_bufheld(struct bpf_d *d)
  390 {
  391         struct zbuf *zb;
  392 
  393         KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
  394             ("bpf_zerocopy_bufheld: not in zbuf mode"));
  395 
  396         zb = (struct zbuf *)d->bd_hbuf;
  397         KASSERT(zb != NULL, ("bpf_zerocopy_bufheld: zb == NULL"));
  398 
  399         if ((zb->zb_flags & ZBUF_FLAG_ASSIGNED) == 0) {
  400                 zb->zb_flags |= ZBUF_FLAG_ASSIGNED;
  401                 zb->zb_header->bzh_kernel_len = d->bd_hlen;
  402                 atomic_add_rel_int(&zb->zb_header->bzh_kernel_gen, 1);
  403         }
  404 }
  405 
  406 /*
  407  * Notification from the BPF framework that the free buffer has been been
  408  * rotated out of the held position to the free position.  This happens when
  409  * the user acknowledges the held buffer.
  410  */
  411 void
  412 bpf_zerocopy_buf_reclaimed(struct bpf_d *d)
  413 {
  414         struct zbuf *zb;
  415 
  416         KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
  417             ("bpf_zerocopy_reclaim_buf: not in zbuf mode"));
  418 
  419         KASSERT(d->bd_fbuf != NULL,
  420             ("bpf_zerocopy_buf_reclaimed: NULL free buf"));
  421         zb = (struct zbuf *)d->bd_fbuf;
  422         zb->zb_flags &= ~ZBUF_FLAG_ASSIGNED;
  423 }
  424 
  425 /*
  426  * Query from the BPF framework regarding whether the buffer currently in the
  427  * held position can be moved to the free position, which can be indicated by
  428  * the user process making their generation number equal to the kernel
  429  * generation number.
  430  */
  431 int
  432 bpf_zerocopy_canfreebuf(struct bpf_d *d)
  433 {
  434         struct zbuf *zb;
  435 
  436         KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
  437             ("bpf_zerocopy_canfreebuf: not in zbuf mode"));
  438 
  439         zb = (struct zbuf *)d->bd_hbuf;
  440         if (zb == NULL)
  441                 return (0);
  442         if (zb->zb_header->bzh_kernel_gen ==
  443             atomic_load_acq_int(&zb->zb_header->bzh_user_gen))
  444                 return (1);
  445         return (0);
  446 }
  447 
  448 /*
  449  * Query from the BPF framework as to whether or not the buffer current in
  450  * the store position can actually be written to.  This may return false if
  451  * the store buffer is assigned to userspace before the hold buffer is
  452  * acknowledged.
  453  */
  454 int
  455 bpf_zerocopy_canwritebuf(struct bpf_d *d)
  456 {
  457         struct zbuf *zb;
  458 
  459         KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
  460             ("bpf_zerocopy_canwritebuf: not in zbuf mode"));
  461 
  462         zb = (struct zbuf *)d->bd_sbuf;
  463         KASSERT(zb != NULL, ("bpf_zerocopy_canwritebuf: bd_sbuf NULL"));
  464 
  465         if (zb->zb_flags & ZBUF_FLAG_ASSIGNED)
  466                 return (0);
  467         return (1);
  468 }
  469 
  470 /*
  471  * Free zero copy buffers at request of descriptor.
  472  */
  473 void
  474 bpf_zerocopy_free(struct bpf_d *d)
  475 {
  476         struct zbuf *zb;
  477 
  478         KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
  479             ("bpf_zerocopy_free: not in zbuf mode"));
  480 
  481         zb = (struct zbuf *)d->bd_sbuf;
  482         if (zb != NULL)
  483                 zbuf_free(zb);
  484         zb = (struct zbuf *)d->bd_hbuf;
  485         if (zb != NULL)
  486                 zbuf_free(zb);
  487         zb = (struct zbuf *)d->bd_fbuf;
  488         if (zb != NULL)
  489                 zbuf_free(zb);
  490 }
  491 
  492 /*
  493  * Ioctl to return the maximum buffer size.
  494  */
  495 int
  496 bpf_zerocopy_ioctl_getzmax(struct thread *td, struct bpf_d *d, size_t *i)
  497 {
  498 
  499         KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
  500             ("bpf_zerocopy_ioctl_getzmax: not in zbuf mode"));
  501 
  502         *i = BPF_MAX_PAGES * PAGE_SIZE;
  503         return (0);
  504 }
  505 
  506 /*
  507  * Ioctl to force rotation of the two buffers, if there's any data available.
  508  * This can be used by user space to implement timeouts when waiting for a
  509  * buffer to fill.
  510  */
  511 int
  512 bpf_zerocopy_ioctl_rotzbuf(struct thread *td, struct bpf_d *d,
  513     struct bpf_zbuf *bz)
  514 {
  515         struct zbuf *bzh;
  516 
  517         bzero(bz, sizeof(*bz));
  518         BPFD_LOCK(d);
  519         if (d->bd_hbuf == NULL && d->bd_slen != 0) {
  520                 ROTATE_BUFFERS(d);
  521                 bzh = (struct zbuf *)d->bd_hbuf;
  522                 bz->bz_bufa = (void *)bzh->zb_uaddr;
  523                 bz->bz_buflen = d->bd_hlen;
  524         }
  525         BPFD_UNLOCK(d);
  526         return (0);
  527 }
  528 
  529 /*
  530  * Ioctl to configure zero-copy buffers -- may be done only once.
  531  */
  532 int
  533 bpf_zerocopy_ioctl_setzbuf(struct thread *td, struct bpf_d *d,
  534     struct bpf_zbuf *bz)
  535 {
  536         struct zbuf *zba, *zbb;
  537         int error;
  538 
  539         KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
  540             ("bpf_zerocopy_ioctl_setzbuf: not in zbuf mode"));
  541 
  542         /*
  543          * Must set both buffers.  Cannot clear them.
  544          */
  545         if (bz->bz_bufa == NULL || bz->bz_bufb == NULL)
  546                 return (EINVAL);
  547 
  548         /*
  549          * Buffers must have a size greater than 0.  Alignment and other size
  550          * validity checking is done in zbuf_setup().
  551          */
  552         if (bz->bz_buflen == 0)
  553                 return (EINVAL);
  554 
  555         /*
  556          * Allocate new buffers.
  557          */
  558         error = zbuf_setup(td, (vm_offset_t)bz->bz_bufa, bz->bz_buflen,
  559             &zba);
  560         if (error)
  561                 return (error);
  562         error = zbuf_setup(td, (vm_offset_t)bz->bz_bufb, bz->bz_buflen,
  563             &zbb);
  564         if (error) {
  565                 zbuf_free(zba);
  566                 return (error);
  567         }
  568 
  569         /*
  570          * We only allow buffers to be installed once, so atomically check
  571          * that no buffers are currently installed and install new buffers.
  572          */
  573         BPFD_LOCK(d);
  574         if (d->bd_hbuf != NULL || d->bd_sbuf != NULL || d->bd_fbuf != NULL ||
  575             d->bd_bif != NULL) {
  576                 BPFD_UNLOCK(d);
  577                 zbuf_free(zba);
  578                 zbuf_free(zbb);
  579                 return (EINVAL);
  580         }
  581 
  582         /*
  583          * Point BPF descriptor at buffers; initialize sbuf as zba so that
  584          * it is always filled first in the sequence, per bpf(4).
  585          */
  586         d->bd_fbuf = (caddr_t)zbb;
  587         d->bd_sbuf = (caddr_t)zba;
  588         d->bd_slen = 0;
  589         d->bd_hlen = 0;
  590 
  591         /*
  592          * We expose only the space left in the buffer after the size of the
  593          * shared management region.
  594          */
  595         d->bd_bufsize = bz->bz_buflen - sizeof(struct bpf_zbuf_header);
  596         BPFD_UNLOCK(d);
  597         return (0);
  598 }

Cache object: d6ca90f44fa5ad4afbf3d26c1cd258f9


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.