FreeBSD/Linux Kernel Cross Reference
sys/dev/netmap/netmap_generic.c

     /*-
      * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
      *
      * Copyright (C) 2013-2016 Vincenzo Maffione
      * Copyright (C) 2013-2016 Luigi Rizzo
      * All rights reserved.
      *
      * Redistribution and use in source and binary forms, with or without
      * modification, are permitted provided that the following conditions
     * are met:
     *   1. Redistributions of source code must retain the above copyright
     *      notice, this list of conditions and the following disclaimer.
     *   2. Redistributions in binary form must reproduce the above copyright
     *      notice, this list of conditions and the following disclaimer in the
     *      documentation and/or other materials provided with the distribution.
     *
     * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
     * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
     * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     * SUCH DAMAGE.
     */
    
    /*
     * This module implements netmap support on top of standard,
     * unmodified device drivers.
     *
     * A NIOCREGIF request is handled here if the device does not
     * have native support. TX and RX rings are emulated as follows:
     *
     * NIOCREGIF
     *      We preallocate a block of TX mbufs (roughly as many as
     *      tx descriptors; the number is not critical) to speed up
     *      operation during transmissions. The refcount on most of
     *      these buffers is artificially bumped up so we can recycle
     *      them more easily. Also, the destructor is intercepted
     *      so we use it as an interrupt notification to wake up
     *      processes blocked on a poll().
     *
     *      For each receive ring we allocate one "struct mbq"
     *      (an mbuf tailq plus a spinlock). We intercept packets
     *      (through if_input)
     *      on the receive path and put them in the mbq from which
     *      netmap receive routines can grab them.
     *
     * TX:
     *      in the generic_txsync() routine, netmap buffers are copied
     *      (or linked, in a future) to the preallocated mbufs
     *      and pushed to the transmit queue. Some of these mbufs
     *      (those with NS_REPORT, or otherwise every half ring)
     *      have the refcount=1, others have refcount=2.
     *      When the destructor is invoked, we take that as
     *      a notification that all mbufs up to that one in
     *      the specific ring have been completed, and generate
     *      the equivalent of a transmit interrupt.
     *
     * RX:
     *
     */
    
    #ifdef __FreeBSD__
    
    #include <sys/cdefs.h> /* prerequisite */
    __FBSDID("$FreeBSD$");
    
    #include <sys/types.h>
    #include <sys/errno.h>
    #include <sys/malloc.h>
    #include <sys/lock.h>   /* PROT_EXEC */
    #include <sys/rwlock.h>
    #include <sys/socket.h> /* sockaddrs */
    #include <sys/selinfo.h>
    #include <net/if.h>
    #include <net/if_types.h>
    #include <net/if_var.h>
    #include <machine/bus.h>        /* bus_dmamap_* in netmap_kern.h */
    
    #include <net/netmap.h>
    #include <dev/netmap/netmap_kern.h>
    #include <dev/netmap/netmap_mem2.h>
    
    #define MBUF_RXQ(m)     ((m)->m_pkthdr.flowid)
    #define smp_mb()
    
    #elif defined _WIN32
    
    #include "win_glue.h"
    
    #define MBUF_TXQ(m)     0//((m)->m_pkthdr.flowid)
    #define MBUF_RXQ(m)         0//((m)->m_pkthdr.flowid)
    #define smp_mb()                //XXX: to be correctly defined
    
    #else /* linux */
   
   #include "bsd_glue.h"
   
   #include <linux/ethtool.h>      /* struct ethtool_ops, get_ringparam */
   #include <linux/hrtimer.h>
   
   static inline struct mbuf *
   nm_os_get_mbuf(struct ifnet *ifp, int len)
   {
           return alloc_skb(LL_RESERVED_SPACE(ifp) + len +
                            ifp->needed_tailroom, GFP_ATOMIC);
   }
   
   #endif /* linux */
   
   
   /* Common headers. */
   #include <net/netmap.h>
   #include <dev/netmap/netmap_kern.h>
   #include <dev/netmap/netmap_mem2.h>
   
   
   #define for_each_kring_n(_i, _k, _karr, _n) \
           for ((_k)=*(_karr), (_i) = 0; (_i) < (_n); (_i)++, (_k) = (_karr)[(_i)])
   
   #define for_each_tx_kring(_i, _k, _na) \
                   for_each_kring_n(_i, _k, (_na)->tx_rings, (_na)->num_tx_rings)
   #define for_each_tx_kring_h(_i, _k, _na) \
                   for_each_kring_n(_i, _k, (_na)->tx_rings, (_na)->num_tx_rings + 1)
   
   #define for_each_rx_kring(_i, _k, _na) \
                   for_each_kring_n(_i, _k, (_na)->rx_rings, (_na)->num_rx_rings)
   #define for_each_rx_kring_h(_i, _k, _na) \
                   for_each_kring_n(_i, _k, (_na)->rx_rings, (_na)->num_rx_rings + 1)
   
   
   /* ======================== PERFORMANCE STATISTICS =========================== */
   
   #ifdef RATE_GENERIC
   #define IFRATE(x) x
   struct rate_stats {
           unsigned long txpkt;
           unsigned long txsync;
           unsigned long txirq;
           unsigned long txrepl;
           unsigned long txdrop;
           unsigned long rxpkt;
           unsigned long rxirq;
           unsigned long rxsync;
   };
   
   struct rate_context {
           unsigned refcount;
           struct timer_list timer;
           struct rate_stats new;
           struct rate_stats old;
   };
   
   #define RATE_PRINTK(_NAME_) \
           printk( #_NAME_ " = %lu Hz\n", (cur._NAME_ - ctx->old._NAME_)/RATE_PERIOD);
   #define RATE_PERIOD  2
   static void rate_callback(unsigned long arg)
   {
           struct rate_context * ctx = (struct rate_context *)arg;
           struct rate_stats cur = ctx->new;
           int r;
   
           RATE_PRINTK(txpkt);
           RATE_PRINTK(txsync);
           RATE_PRINTK(txirq);
           RATE_PRINTK(txrepl);
           RATE_PRINTK(txdrop);
           RATE_PRINTK(rxpkt);
           RATE_PRINTK(rxsync);
           RATE_PRINTK(rxirq);
           printk("\n");
   
           ctx->old = cur;
           r = mod_timer(&ctx->timer, jiffies +
                           msecs_to_jiffies(RATE_PERIOD * 1000));
           if (unlikely(r))
                   nm_prerr("mod_timer() failed");
   }
   
   static struct rate_context rate_ctx;
   
   void generic_rate(int txp, int txs, int txi, int rxp, int rxs, int rxi)
   {
           if (txp) rate_ctx.new.txpkt++;
           if (txs) rate_ctx.new.txsync++;
           if (txi) rate_ctx.new.txirq++;
           if (rxp) rate_ctx.new.rxpkt++;
           if (rxs) rate_ctx.new.rxsync++;
           if (rxi) rate_ctx.new.rxirq++;
   }
   
   #else /* !RATE */
   #define IFRATE(x)
   #endif /* !RATE */
   
   
   /* ========== GENERIC (EMULATED) NETMAP ADAPTER SUPPORT ============= */
   
   /*
    * Wrapper used by the generic adapter layer to notify
    * the poller threads. Differently from netmap_rx_irq(), we check
    * only NAF_NETMAP_ON instead of NAF_NATIVE_ON to enable the irq.
    */
   void
   netmap_generic_irq(struct netmap_adapter *na, u_int q, u_int *work_done)
   {
           if (unlikely(!nm_netmap_on(na)))
                   return;
   
           netmap_common_irq(na, q, work_done);
   #ifdef RATE_GENERIC
           if (work_done)
                   rate_ctx.new.rxirq++;
           else
                   rate_ctx.new.txirq++;
   #endif  /* RATE_GENERIC */
   }
   
   static int
   generic_netmap_unregister(struct netmap_adapter *na)
   {
           struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na;
           struct netmap_kring *kring = NULL;
           int i, r;
   
           if (na->active_fds == 0) {
                   na->na_flags &= ~NAF_NETMAP_ON;
   
                   /* Stop intercepting packets on the RX path. */
                   nm_os_catch_rx(gna, 0);
   
                   /* Release packet steering control. */
                   nm_os_catch_tx(gna, 0);
           }
   
           netmap_krings_mode_commit(na, /*onoff=*/0);
   
           for_each_rx_kring(r, kring, na) {
                   /* Free the mbufs still pending in the RX queues,
                    * that did not end up into the corresponding netmap
                    * RX rings. */
                   mbq_safe_purge(&kring->rx_queue);
                   nm_os_mitigation_cleanup(&gna->mit[r]);
           }
   
           /* Decrement reference counter for the mbufs in the
            * TX pools. These mbufs can be still pending in drivers,
            * (e.g. this happens with virtio-net driver, which
            * does lazy reclaiming of transmitted mbufs). */
           for_each_tx_kring(r, kring, na) {
                   /* We must remove the destructor on the TX event,
                    * because the destructor invokes netmap code, and
                    * the netmap module may disappear before the
                    * TX event is consumed. */
                   mtx_lock_spin(&kring->tx_event_lock);
                   if (kring->tx_event) {
                           SET_MBUF_DESTRUCTOR(kring->tx_event, NULL);
                   }
                   kring->tx_event = NULL;
                   mtx_unlock_spin(&kring->tx_event_lock);
           }
   
           if (na->active_fds == 0) {
                   nm_os_free(gna->mit);
   
                   for_each_rx_kring(r, kring, na) {
                           mbq_safe_fini(&kring->rx_queue);
                   }
   
                   for_each_tx_kring(r, kring, na) {
                           mtx_destroy(&kring->tx_event_lock);
                           if (kring->tx_pool == NULL) {
                                   continue;
                           }
   
                           for (i=0; i<na->num_tx_desc; i++) {
                                   if (kring->tx_pool[i]) {
                                           m_freem(kring->tx_pool[i]);
                                   }
                           }
                           nm_os_free(kring->tx_pool);
                           kring->tx_pool = NULL;
                   }
   
   #ifdef RATE_GENERIC
                   if (--rate_ctx.refcount == 0) {
                           nm_prinf("del_timer()");
                           del_timer(&rate_ctx.timer);
                   }
   #endif
                   nm_prinf("Emulated adapter for %s deactivated", na->name);
           }
   
           return 0;
   }
   
   /* Enable/disable netmap mode for a generic network interface. */
   static int
   generic_netmap_register(struct netmap_adapter *na, int enable)
   {
           struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na;
           struct netmap_kring *kring = NULL;
           int error;
           int i, r;
   
           if (!na) {
                   return EINVAL;
           }
   
           if (!enable) {
                   /* This is actually an unregif. */
                   return generic_netmap_unregister(na);
           }
   
           if (na->active_fds == 0) {
                   nm_prinf("Emulated adapter for %s activated", na->name);
                   /* Do all memory allocations when (na->active_fds == 0), to
                    * simplify error management. */
   
                   /* Allocate memory for mitigation support on all the rx queues. */
                   gna->mit = nm_os_malloc(na->num_rx_rings * sizeof(struct nm_generic_mit));
                   if (!gna->mit) {
                           nm_prerr("mitigation allocation failed");
                           error = ENOMEM;
                           goto out;
                   }
   
                   for_each_rx_kring(r, kring, na) {
                           /* Init mitigation support. */
                           nm_os_mitigation_init(&gna->mit[r], r, na);
   
                           /* Initialize the rx queue, as generic_rx_handler() can
                            * be called as soon as nm_os_catch_rx() returns.
                            */
                           mbq_safe_init(&kring->rx_queue);
                   }
   
                   /*
                    * Prepare mbuf pools (parallel to the tx rings), for packet
                    * transmission. Don't preallocate the mbufs here, it's simpler
                    * to leave this task to txsync.
                    */
                   for_each_tx_kring(r, kring, na) {
                           kring->tx_pool = NULL;
                   }
                   for_each_tx_kring(r, kring, na) {
                           kring->tx_pool =
                                   nm_os_malloc(na->num_tx_desc * sizeof(struct mbuf *));
                           if (!kring->tx_pool) {
                                   nm_prerr("tx_pool allocation failed");
                                   error = ENOMEM;
                                   goto free_tx_pools;
                           }
                           mtx_init(&kring->tx_event_lock, "tx_event_lock",
                                    NULL, MTX_SPIN);
                   }
           }
   
           netmap_krings_mode_commit(na, /*onoff=*/1);
   
           for_each_tx_kring(r, kring, na) {
                   /* Initialize tx_pool and tx_event. */
                   for (i=0; i<na->num_tx_desc; i++) {
                           kring->tx_pool[i] = NULL;
                   }
   
                   kring->tx_event = NULL;
           }
   
           if (na->active_fds == 0) {
                   /* Prepare to intercept incoming traffic. */
                   error = nm_os_catch_rx(gna, 1);
                   if (error) {
                           nm_prerr("nm_os_catch_rx(1) failed (%d)", error);
                           goto free_tx_pools;
                   }
   
                   /* Let netmap control the packet steering. */
                   error = nm_os_catch_tx(gna, 1);
                   if (error) {
                           nm_prerr("nm_os_catch_tx(1) failed (%d)", error);
                           goto catch_rx;
                   }
   
                   na->na_flags |= NAF_NETMAP_ON;
   
   #ifdef RATE_GENERIC
                   if (rate_ctx.refcount == 0) {
                           nm_prinf("setup_timer()");
                           memset(&rate_ctx, 0, sizeof(rate_ctx));
                           setup_timer(&rate_ctx.timer, &rate_callback, (unsigned long)&rate_ctx);
                           if (mod_timer(&rate_ctx.timer, jiffies + msecs_to_jiffies(1500))) {
                                   nm_prerr("Error: mod_timer()");
                           }
                   }
                   rate_ctx.refcount++;
   #endif /* RATE */
           }
   
           return 0;
   
           /* Here (na->active_fds == 0) holds. */
   catch_rx:
           nm_os_catch_rx(gna, 0);
   free_tx_pools:
           for_each_tx_kring(r, kring, na) {
                   mtx_destroy(&kring->tx_event_lock);
                   if (kring->tx_pool == NULL) {
                           continue;
                   }
                   nm_os_free(kring->tx_pool);
                   kring->tx_pool = NULL;
           }
           for_each_rx_kring(r, kring, na) {
                   mbq_safe_fini(&kring->rx_queue);
           }
           nm_os_free(gna->mit);
   out:
   
           return error;
   }
   
   /*
    * Callback invoked when the device driver frees an mbuf used
    * by netmap to transmit a packet. This usually happens when
    * the NIC notifies the driver that transmission is completed.
    */
   static void
   generic_mbuf_destructor(struct mbuf *m)
   {
           struct netmap_adapter *na = NA(GEN_TX_MBUF_IFP(m));
           struct netmap_kring *kring;
           unsigned int r = MBUF_TXQ(m);
           unsigned int r_orig = r;
   
           if (unlikely(!nm_netmap_on(na) || r >= na->num_tx_rings)) {
                   nm_prerr("Error: no netmap adapter on device %p",
                     GEN_TX_MBUF_IFP(m));
                   return;
           }
   
           /*
            * First, clear the event mbuf.
            * In principle, the event 'm' should match the one stored
            * on ring 'r'. However we check it explicitly to stay
            * safe against lower layers (qdisc, driver, etc.) changing
            * MBUF_TXQ(m) under our feet. If the match is not found
            * on 'r', we try to see if it belongs to some other ring.
            */
           for (;;) {
                   bool match = false;
   
                   kring = na->tx_rings[r];
                   mtx_lock_spin(&kring->tx_event_lock);
                   if (kring->tx_event == m) {
                           kring->tx_event = NULL;
                           match = true;
                   }
                   mtx_unlock_spin(&kring->tx_event_lock);
   
                   if (match) {
                           if (r != r_orig) {
                                   nm_prlim(1, "event %p migrated: ring %u --> %u",
                                         m, r_orig, r);
                           }
                           break;
                   }
   
                   if (++r == na->num_tx_rings) r = 0;
   
                   if (r == r_orig) {
                           nm_prlim(1, "Cannot match event %p", m);
                           return;
                   }
           }
   
           /* Second, wake up clients. They will reclaim the event through
            * txsync. */
           netmap_generic_irq(na, r, NULL);
   #ifdef __FreeBSD__
           void_mbuf_dtor(m);
   #endif
   }
   
   /* Record completed transmissions and update hwtail.
    *
    * The oldest tx buffer not yet completed is at nr_hwtail + 1,
    * nr_hwcur is the first unsent buffer.
    */
   static u_int
   generic_netmap_tx_clean(struct netmap_kring *kring, int txqdisc)
   {
           u_int const lim = kring->nkr_num_slots - 1;
           u_int nm_i = nm_next(kring->nr_hwtail, lim);
           u_int hwcur = kring->nr_hwcur;
           u_int n = 0;
           struct mbuf **tx_pool = kring->tx_pool;
   
           nm_prdis("hwcur = %d, hwtail = %d", kring->nr_hwcur, kring->nr_hwtail);
   
           while (nm_i != hwcur) { /* buffers not completed */
                   struct mbuf *m = tx_pool[nm_i];
   
                   if (txqdisc) {
                           if (m == NULL) {
                                   /* Nothing to do, this is going
                                    * to be replenished. */
                                   nm_prlim(3, "Is this happening?");
   
                           } else if (MBUF_QUEUED(m)) {
                                   break; /* Not dequeued yet. */
   
                           } else if (MBUF_REFCNT(m) != 1) {
                                   /* This mbuf has been dequeued but is still busy
                                    * (refcount is 2).
                                    * Leave it to the driver and replenish. */
                                   m_freem(m);
                                   tx_pool[nm_i] = NULL;
                           }
   
                   } else {
                           if (unlikely(m == NULL)) {
                                   int event_consumed;
   
                                   /* This slot was used to place an event. */
                                   mtx_lock_spin(&kring->tx_event_lock);
                                   event_consumed = (kring->tx_event == NULL);
                                   mtx_unlock_spin(&kring->tx_event_lock);
                                   if (!event_consumed) {
                                           /* The event has not been consumed yet,
                                            * still busy in the driver. */
                                           break;
                                   }
                                   /* The event has been consumed, we can go
                                    * ahead. */
   
                           } else if (MBUF_REFCNT(m) != 1) {
                                   /* This mbuf is still busy: its refcnt is 2. */
                                   break;
                           }
                   }
   
                   n++;
                   nm_i = nm_next(nm_i, lim);
           }
           kring->nr_hwtail = nm_prev(nm_i, lim);
           nm_prdis("tx completed [%d] -> hwtail %d", n, kring->nr_hwtail);
   
           return n;
   }
   
   /* Compute a slot index in the middle between inf and sup. */
   static inline u_int
   ring_middle(u_int inf, u_int sup, u_int lim)
   {
           u_int n = lim + 1;
           u_int e;
   
           if (sup >= inf) {
                   e = (sup + inf) / 2;
           } else { /* wrap around */
                   e = (sup + n + inf) / 2;
                   if (e >= n) {
                           e -= n;
                   }
           }
   
           if (unlikely(e >= n)) {
                   nm_prerr("This cannot happen");
                   e = 0;
           }
   
           return e;
   }
   
   static void
   generic_set_tx_event(struct netmap_kring *kring, u_int hwcur)
   {
           u_int lim = kring->nkr_num_slots - 1;
           struct mbuf *m;
           u_int e;
           u_int ntc = nm_next(kring->nr_hwtail, lim); /* next to clean */
   
           if (ntc == hwcur) {
                   return; /* all buffers are free */
           }
   
           /*
            * We have pending packets in the driver between hwtail+1
            * and hwcur, and we have to chose one of these slot to
            * generate a notification.
            * There is a race but this is only called within txsync which
            * does a double check.
            */
   #if 0
           /* Choose a slot in the middle, so that we don't risk ending
            * up in a situation where the client continuously wake up,
            * fills one or a few TX slots and go to sleep again. */
           e = ring_middle(ntc, hwcur, lim);
   #else
           /* Choose the first pending slot, to be safe against driver
            * reordering mbuf transmissions. */
           e = ntc;
   #endif
   
           m = kring->tx_pool[e];
           if (m == NULL) {
                   /* An event is already in place. */
                   return;
           }
   
           mtx_lock_spin(&kring->tx_event_lock);
           if (kring->tx_event) {
                   /* An event is already in place. */
                   mtx_unlock_spin(&kring->tx_event_lock);
                   return;
           }
   
           SET_MBUF_DESTRUCTOR(m, generic_mbuf_destructor);
           kring->tx_event = m;
           mtx_unlock_spin(&kring->tx_event_lock);
   
           kring->tx_pool[e] = NULL;
   
           nm_prdis("Request Event at %d mbuf %p refcnt %d", e, m, m ? MBUF_REFCNT(m) : -2 );
   
           /* Decrement the refcount. This will free it if we lose the race
            * with the driver. */
           m_freem(m);
           smp_mb();
   }
   
   
   /*
    * generic_netmap_txsync() transforms netmap buffers into mbufs
    * and passes them to the standard device driver
    * (ndo_start_xmit() or ifp->if_transmit() ).
    * On linux this is not done directly, but using dev_queue_xmit(),
    * since it implements the TX flow control (and takes some locks).
    */
   static int
   generic_netmap_txsync(struct netmap_kring *kring, int flags)
   {
           struct netmap_adapter *na = kring->na;
           struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na;
           struct ifnet *ifp = na->ifp;
           struct netmap_ring *ring = kring->ring;
           u_int nm_i;     /* index into the netmap ring */ // j
           u_int const lim = kring->nkr_num_slots - 1;
           u_int const head = kring->rhead;
           u_int ring_nr = kring->ring_id;
   
           IFRATE(rate_ctx.new.txsync++);
   
           rmb();
   
           /*
            * First part: process new packets to send.
            */
           nm_i = kring->nr_hwcur;
           if (nm_i != head) {     /* we have new packets to send */
                   struct nm_os_gen_arg a;
                   u_int event = -1;
   #ifdef __FreeBSD__
                   struct epoch_tracker et;
   
                   NET_EPOCH_ENTER(et);
   #endif
   
                   if (gna->txqdisc && nm_kr_txempty(kring)) {
                           /* In txqdisc mode, we ask for a delayed notification,
                            * but only when cur == hwtail, which means that the
                            * client is going to block. */
                           event = ring_middle(nm_i, head, lim);
                           nm_prdis("Place txqdisc event (hwcur=%u,event=%u,"
                                 "head=%u,hwtail=%u)", nm_i, event, head,
                                 kring->nr_hwtail);
                   }
   
                   a.ifp = ifp;
                   a.ring_nr = ring_nr;
                   a.head = a.tail = NULL;
   
                   while (nm_i != head) {
                           struct netmap_slot *slot = &ring->slot[nm_i];
                           u_int len = slot->len;
                           void *addr = NMB(na, slot);
                           /* device-specific */
                           struct mbuf *m;
                           int tx_ret;
   
                           NM_CHECK_ADDR_LEN(na, addr, len);
   
                           /* Tale a mbuf from the tx pool (replenishing the pool
                            * entry if necessary) and copy in the user packet. */
                           m = kring->tx_pool[nm_i];
                           if (unlikely(m == NULL)) {
                                   kring->tx_pool[nm_i] = m =
                                           nm_os_get_mbuf(ifp, NETMAP_BUF_SIZE(na));
                                   if (m == NULL) {
                                           nm_prlim(2, "Failed to replenish mbuf");
                                           /* Here we could schedule a timer which
                                            * retries to replenish after a while,
                                            * and notifies the client when it
                                            * manages to replenish some slots. In
                                            * any case we break early to avoid
                                            * crashes. */
                                           break;
                                   }
                                   IFRATE(rate_ctx.new.txrepl++);
                           }
   
                           a.m = m;
                           a.addr = addr;
                           a.len = len;
                           a.qevent = (nm_i == event);
                           /* When not in txqdisc mode, we should ask
                            * notifications when NS_REPORT is set, or roughly
                            * every half ring. To optimize this, we set a
                            * notification event when the client runs out of
                            * TX ring space, or when transmission fails. In
                            * the latter case we also break early.
                            */
                           tx_ret = nm_os_generic_xmit_frame(&a);
                           if (unlikely(tx_ret)) {
                                   if (!gna->txqdisc) {
                                           /*
                                            * No room for this mbuf in the device driver.
                                            * Request a notification FOR A PREVIOUS MBUF,
                                            * then call generic_netmap_tx_clean(kring) to do the
                                            * double check and see if we can free more buffers.
                                            * If there is space continue, else break;
                                            * NOTE: the double check is necessary if the problem
                                            * occurs in the txsync call after selrecord().
                                            * Also, we need some way to tell the caller that not
                                            * all buffers were queued onto the device (this was
                                            * not a problem with native netmap driver where space
                                            * is preallocated). The bridge has a similar problem
                                            * and we solve it there by dropping the excess packets.
                                            */
                                           generic_set_tx_event(kring, nm_i);
                                           if (generic_netmap_tx_clean(kring, gna->txqdisc)) {
                                                   /* space now available */
                                                   continue;
                                           } else {
                                                   break;
                                           }
                                   }
   
                                   /* In txqdisc mode, the netmap-aware qdisc
                                    * queue has the same length as the number of
                                    * netmap slots (N). Since tail is advanced
                                    * only when packets are dequeued, qdisc
                                    * queue overrun cannot happen, so
                                    * nm_os_generic_xmit_frame() did not fail
                                    * because of that.
                                    * However, packets can be dropped because
                                    * carrier is off, or because our qdisc is
                                    * being deactivated, or possibly for other
                                    * reasons. In these cases, we just let the
                                    * packet to be dropped. */
                                   IFRATE(rate_ctx.new.txdrop++);
                           }
   
                           slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);
                           nm_i = nm_next(nm_i, lim);
                           IFRATE(rate_ctx.new.txpkt++);
                   }
                   if (a.head != NULL) {
                           a.addr = NULL;
                           nm_os_generic_xmit_frame(&a);
                   }
                   /* Update hwcur to the next slot to transmit. Here nm_i
                    * is not necessarily head, we could break early. */
                   kring->nr_hwcur = nm_i;
   
   #ifdef __FreeBSD__
                   NET_EPOCH_EXIT(et);
   #endif
           }
   
           /*
            * Second, reclaim completed buffers
            */
           if (!gna->txqdisc && (flags & NAF_FORCE_RECLAIM || nm_kr_txempty(kring))) {
                   /* No more available slots? Set a notification event
                    * on a netmap slot that will be cleaned in the future.
                    * No doublecheck is performed, since txsync() will be
                    * called twice by netmap_poll().
                    */
                   generic_set_tx_event(kring, nm_i);
           }
   
           generic_netmap_tx_clean(kring, gna->txqdisc);
   
           return 0;
   }
   
   
   /*
    * This handler is registered (through nm_os_catch_rx())
    * within the attached network interface
    * in the RX subsystem, so that every mbuf passed up by
    * the driver can be stolen to the network stack.
    * Stolen packets are put in a queue where the
    * generic_netmap_rxsync() callback can extract them.
    * Returns 1 if the packet was stolen, 0 otherwise.
    */
   int
   generic_rx_handler(struct ifnet *ifp, struct mbuf *m)
   {
           struct netmap_adapter *na = NA(ifp);
           struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na;
           struct netmap_kring *kring;
           u_int work_done;
           u_int r = MBUF_RXQ(m); /* receive ring number */
   
           if (r >= na->num_rx_rings) {
                   r = r % na->num_rx_rings;
           }
   
           kring = na->rx_rings[r];
   
           if (kring->nr_mode == NKR_NETMAP_OFF) {
                   /* We must not intercept this mbuf. */
                   return 0;
           }
   
           /* limit the size of the queue */
           if (unlikely(!gna->rxsg && MBUF_LEN(m) > NETMAP_BUF_SIZE(na))) {
                   /* This may happen when GRO/LRO features are enabled for
                    * the NIC driver when the generic adapter does not
                    * support RX scatter-gather. */
                   nm_prlim(2, "Warning: driver pushed up big packet "
                                   "(size=%d)", (int)MBUF_LEN(m));
                   if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1);
                   m_freem(m);
           } else if (unlikely(mbq_len(&kring->rx_queue) > na->num_rx_desc)) {
                   if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1);
                   m_freem(m);
           } else {
                   mbq_safe_enqueue(&kring->rx_queue, m);
           }
   
           if (netmap_generic_mit < 32768) {
                   /* no rx mitigation, pass notification up */
                   netmap_generic_irq(na, r, &work_done);
           } else {
                   /* same as send combining, filter notification if there is a
                    * pending timer, otherwise pass it up and start a timer.
                    */
                   if (likely(nm_os_mitigation_active(&gna->mit[r]))) {
                           /* Record that there is some pending work. */
                           gna->mit[r].mit_pending = 1;
                   } else {
                           netmap_generic_irq(na, r, &work_done);
                           nm_os_mitigation_start(&gna->mit[r]);
                   }
           }
   
           /* We have intercepted the mbuf. */
           return 1;
   }
   
   /*
    * generic_netmap_rxsync() extracts mbufs from the queue filled by
    * generic_netmap_rx_handler() and puts their content in the netmap
    * receive ring.
    * Access must be protected because the rx handler is asynchronous,
    */
   static int
   generic_netmap_rxsync(struct netmap_kring *kring, int flags)
   {
           struct netmap_ring *ring = kring->ring;
           struct netmap_adapter *na = kring->na;
           u_int nm_i;     /* index into the netmap ring */ //j,
           u_int n;
           u_int const lim = kring->nkr_num_slots - 1;
           u_int const head = kring->rhead;
           int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
   
           /* Adapter-specific variables. */
           u_int nm_buf_len = NETMAP_BUF_SIZE(na);
           struct mbq tmpq;
           struct mbuf *m;
           int avail; /* in bytes */
           int mlen;
           int copy;
   
           if (head > lim)
                   return netmap_ring_reinit(kring);
   
           IFRATE(rate_ctx.new.rxsync++);
   
           /*
            * First part: skip past packets that userspace has released.
            * This can possibly make room for the second part.
            */
           nm_i = kring->nr_hwcur;
           if (nm_i != head) {
                   /* Userspace has released some packets. */
                   for (n = 0; nm_i != head; n++) {
                           struct netmap_slot *slot = &ring->slot[nm_i];
   
                           slot->flags &= ~NS_BUF_CHANGED;
                           nm_i = nm_next(nm_i, lim);
                   }
                   kring->nr_hwcur = head;
           }
   
           /*
            * Second part: import newly received packets.
            */
           if (!netmap_no_pendintr && !force_update) {
                   return 0;
           }
   
           nm_i = kring->nr_hwtail; /* First empty slot in the receive ring. */
   
           /* Compute the available space (in bytes) in this netmap ring.
            * The first slot that is not considered in is the one before
            * nr_hwcur. */
   
           avail = nm_prev(kring->nr_hwcur, lim) - nm_i;
           if (avail < 0)
                   avail += lim + 1;
           avail *= nm_buf_len;
   
           /* First pass: While holding the lock on the RX mbuf queue,
            * extract as many mbufs as they fit the available space,
            * and put them in a temporary queue.
            * To avoid performing a per-mbuf division (mlen / nm_buf_len) to
            * to update avail, we do the update in a while loop that we
            * also use to set the RX slots, but without performing the copy. */
           mbq_init(&tmpq);
           mbq_lock(&kring->rx_queue);
           for (n = 0;; n++) {
                   m = mbq_peek(&kring->rx_queue);
                   if (!m) {
                           /* No more packets from the driver. */
                           break;
                   }
   
                   mlen = MBUF_LEN(m);
                   if (mlen > avail) {
                           /* No more space in the ring. */
                           break;
                   }
   
                   mbq_dequeue(&kring->rx_queue);
   
                   while (mlen) {
                           copy = nm_buf_len;
                           if (mlen < copy) {
                                   copy = mlen;
                           }
                           mlen -= copy;
                           avail -= nm_buf_len;
   
                           ring->slot[nm_i].len = copy;
                           ring->slot[nm_i].flags = (mlen ? NS_MOREFRAG : 0);
                           nm_i = nm_next(nm_i, lim);
                   }
   
                   mbq_enqueue(&tmpq, m);
           }
           mbq_unlock(&kring->rx_queue);
   
           /* Second pass: Drain the temporary queue, going over the used RX slots,
            * and perform the copy out of the RX queue lock. */
           nm_i = kring->nr_hwtail;
   
           for (;;) {
                   void *nmaddr;
                   int ofs = 0;
                   int morefrag;
   
                   m = mbq_dequeue(&tmpq);
                   if (!m) {
                           break;
                   }
   
                   do {
                           nmaddr = NMB(na, &ring->slot[nm_i]);
                           /* We only check the address here on generic rx rings. */
                           if (nmaddr == NETMAP_BUF_BASE(na)) { /* Bad buffer */
                                   m_freem(m);
                                   mbq_purge(&tmpq);
                                   mbq_fini(&tmpq);
                                   return netmap_ring_reinit(kring);
                           }
   
                           copy = ring->slot[nm_i].len;
                           m_copydata(m, ofs, copy, nmaddr);
                           ofs += copy;
                          morefrag = ring->slot[nm_i].flags & NS_MOREFRAG;
                          nm_i = nm_next(nm_i, lim);
                  } while (morefrag);
  
                  m_freem(m);
          }
  
          mbq_fini(&tmpq);
  
          if (n) {
                  kring->nr_hwtail = nm_i;
                  IFRATE(rate_ctx.new.rxpkt += n);
          }
          kring->nr_kflags &= ~NKR_PENDINTR;
  
          return 0;
  }
  
  static void
  generic_netmap_dtor(struct netmap_adapter *na)
  {
          struct netmap_generic_adapter *gna = (struct netmap_generic_adapter*)na;
          struct ifnet *ifp = netmap_generic_getifp(gna);
          struct netmap_adapter *prev_na = gna->prev;
  
          if (prev_na != NULL) {
                  netmap_adapter_put(prev_na);
                  if (nm_iszombie(na)) {
                          /*
                           * The driver has been removed without releasing
                           * the reference so we need to do it here.
                           */
                          netmap_adapter_put(prev_na);
                  }
                  nm_prinf("Native netmap adapter for %s restored", prev_na->name);
          }
          NM_RESTORE_NA(ifp, prev_na);
          /*
           * netmap_detach_common(), that it's called after this function,
           * overrides WNA(ifp) if na->ifp is not NULL.
           */
          na->ifp = NULL;
          nm_prinf("Emulated netmap adapter for %s destroyed", na->name);
  }
  
  int
  na_is_generic(struct netmap_adapter *na)
  {
          return na->nm_register == generic_netmap_register;
  }
  
  /*
   * generic_netmap_attach() makes it possible to use netmap on
   * a device without native netmap support.
   * This is less performant than native support but potentially
   * faster than raw sockets or similar schemes.
   *
   * In this "emulated" mode, netmap rings do not necessarily
   * have the same size as those in the NIC. We use a default
   * value and possibly override it if the OS has ways to fetch the
   * actual configuration.
   */
  int
  generic_netmap_attach(struct ifnet *ifp)
  {
          struct netmap_adapter *na;
          struct netmap_generic_adapter *gna;
          int retval;
          u_int num_tx_desc, num_rx_desc;
  
  #ifdef __FreeBSD__
          if (ifp->if_type == IFT_LOOP) {
                  nm_prerr("if_loop is not supported by %s", __func__);
                  return EINVAL;
          }
  #endif
  
          if (NM_NA_CLASH(ifp)) {
                  /* If NA(ifp) is not null but there is no valid netmap
                   * adapter it means that someone else is using the same
                   * pointer (e.g. ax25_ptr on linux). This happens for
                   * instance when also PF_RING is in use. */
                  nm_prerr("Error: netmap adapter hook is busy");
                  return EBUSY;
          }
  
          num_tx_desc = num_rx_desc = netmap_generic_ringsize; /* starting point */
  
          nm_os_generic_find_num_desc(ifp, &num_tx_desc, &num_rx_desc); /* ignore errors */
          if (num_tx_desc == 0 || num_rx_desc == 0) {
                  nm_prerr("Device has no hw slots (tx %u, rx %u)", num_tx_desc, num_rx_desc);
                  return EINVAL;
          }
  
          gna = nm_os_malloc(sizeof(*gna));
          if (gna == NULL) {
                  nm_prerr("no memory on attach, give up");
                  return ENOMEM;
          }
          na = (struct netmap_adapter *)gna;
          strlcpy(na->name, ifp->if_xname, sizeof(na->name));
          na->ifp = ifp;
          na->num_tx_desc = num_tx_desc;
          na->num_rx_desc = num_rx_desc;
          na->rx_buf_maxsize = 32768;
          na->nm_register = &generic_netmap_register;
          na->nm_txsync = &generic_netmap_txsync;
          na->nm_rxsync = &generic_netmap_rxsync;
          na->nm_dtor = &generic_netmap_dtor;
          /* when using generic, NAF_NETMAP_ON is set so we force
           * NAF_SKIP_INTR to use the regular interrupt handler
           */
          na->na_flags = NAF_SKIP_INTR | NAF_HOST_RINGS;
  
          nm_prdis("[GNA] num_tx_queues(%d), real_num_tx_queues(%d), len(%lu)",
                          ifp->num_tx_queues, ifp->real_num_tx_queues,
                          ifp->tx_queue_len);
          nm_prdis("[GNA] num_rx_queues(%d), real_num_rx_queues(%d)",
                          ifp->num_rx_queues, ifp->real_num_rx_queues);
  
          nm_os_generic_find_num_queues(ifp, &na->num_tx_rings, &na->num_rx_rings);
  
          retval = netmap_attach_common(na);
          if (retval) {
                  nm_os_free(gna);
                  return retval;
          }
  
          if (NM_NA_VALID(ifp)) {
                  gna->prev = NA(ifp); /* save old na */
                  netmap_adapter_get(gna->prev);
          }
          NM_ATTACH_NA(ifp, na);
  
          nm_os_generic_set_features(gna);
  
          nm_prinf("Emulated adapter for %s created (prev was %s)", na->name,
              gna->prev ? gna->prev->name : "NULL");
  
          return retval;
  }

Cache object: a39cc254d137940e3aec77ecb42e8e54