nicvf_main.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*
    2  * Copyright (C) 2015 Cavium Inc.
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  *
   14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24  * SUCH DAMAGE.
   25  *
   26  * $FreeBSD$
   27  *
   28  */
   29 #include <sys/cdefs.h>
   30 __FBSDID("$FreeBSD$");
   31 
   32 #include "opt_inet.h"
   33 #include "opt_inet6.h"
   34 
   35 #include <sys/param.h>
   36 #include <sys/systm.h>
   37 #include <sys/bitset.h>
   38 #include <sys/bitstring.h>
   39 #include <sys/bus.h>
   40 #include <sys/endian.h>
   41 #include <sys/kernel.h>
   42 #include <sys/malloc.h>
   43 #include <sys/mbuf.h>
   44 #include <sys/module.h>
   45 #include <sys/rman.h>
   46 #include <sys/pciio.h>
   47 #include <sys/pcpu.h>
   48 #include <sys/proc.h>
   49 #include <sys/socket.h>
   50 #include <sys/sockio.h>
   51 #include <sys/stdatomic.h>
   52 #include <sys/cpuset.h>
   53 #include <sys/lock.h>
   54 #include <sys/mutex.h>
   55 #include <sys/smp.h>
   56 #include <sys/taskqueue.h>
   57 
   58 #include <net/bpf.h>
   59 #include <net/ethernet.h>
   60 #include <net/if.h>
   61 #include <net/if_var.h>
   62 #include <net/if_arp.h>
   63 #include <net/if_dl.h>
   64 #include <net/if_media.h>
   65 #include <net/if_types.h>
   66 #include <net/if_vlan_var.h>
   67 
   68 #include <netinet/in.h>
   69 #include <netinet/ip.h>
   70 #include <netinet/if_ether.h>
   71 #include <netinet/tcp_lro.h>
   72 
   73 #include <dev/pci/pcireg.h>
   74 #include <dev/pci/pcivar.h>
   75 
   76 #include <sys/dnv.h>
   77 #include <sys/nv.h>
   78 #include <sys/iov_schema.h>
   79 
   80 #include <machine/bus.h>
   81 
   82 #include "thunder_bgx.h"
   83 #include "nic_reg.h"
   84 #include "nic.h"
   85 #include "nicvf_queues.h"
   86 
   87 #define VNIC_VF_DEVSTR          "Cavium Thunder NIC Virtual Function Driver"
   88 
   89 #define VNIC_VF_REG_RID         PCIR_BAR(PCI_CFG_REG_BAR_NUM)
   90 
   91 /* Lock for core interface settings */
   92 #define NICVF_CORE_LOCK_INIT(nic)                               \
   93     sx_init(&(nic)->core_sx, device_get_nameunit((nic)->dev))
   94 
   95 #define NICVF_CORE_LOCK_DESTROY(nic)                            \
   96     sx_destroy(&(nic)->core_sx)
   97 
   98 #define NICVF_CORE_LOCK(nic)            sx_xlock(&(nic)->core_sx)
   99 #define NICVF_CORE_UNLOCK(nic)          sx_xunlock(&(nic)->core_sx)
  100 
  101 #define NICVF_CORE_LOCK_ASSERT(nic)     sx_assert(&(nic)->core_sx, SA_XLOCKED)
  102 
  103 #define SPEED_10        10
  104 #define SPEED_100       100
  105 #define SPEED_1000      1000
  106 #define SPEED_10000     10000
  107 #define SPEED_40000     40000
  108 
  109 MALLOC_DEFINE(M_NICVF, "nicvf", "ThunderX VNIC VF dynamic memory");
  110 
  111 static int nicvf_probe(device_t);
  112 static int nicvf_attach(device_t);
  113 static int nicvf_detach(device_t);
  114 
  115 static device_method_t nicvf_methods[] = {
  116         /* Device interface */
  117         DEVMETHOD(device_probe,         nicvf_probe),
  118         DEVMETHOD(device_attach,        nicvf_attach),
  119         DEVMETHOD(device_detach,        nicvf_detach),
  120 
  121         DEVMETHOD_END,
  122 };
  123 
  124 static driver_t nicvf_driver = {
  125         "vnic",
  126         nicvf_methods,
  127         sizeof(struct nicvf),
  128 };
  129 
  130 static devclass_t nicvf_devclass;
  131 
  132 DRIVER_MODULE(vnicvf, pci, nicvf_driver, nicvf_devclass, 0, 0);
  133 MODULE_VERSION(vnicvf, 1);
  134 MODULE_DEPEND(vnicvf, pci, 1, 1, 1);
  135 MODULE_DEPEND(vnicvf, ether, 1, 1, 1);
  136 MODULE_DEPEND(vnicvf, vnicpf, 1, 1, 1);
  137 
  138 static int nicvf_allocate_misc_interrupt(struct nicvf *);
  139 static int nicvf_enable_misc_interrupt(struct nicvf *);
  140 static int nicvf_allocate_net_interrupts(struct nicvf *);
  141 static void nicvf_release_all_interrupts(struct nicvf *);
  142 static int nicvf_update_hw_max_frs(struct nicvf *, int);
  143 static int nicvf_hw_set_mac_addr(struct nicvf *, uint8_t *);
  144 static void nicvf_config_cpi(struct nicvf *);
  145 static int nicvf_rss_init(struct nicvf *);
  146 static int nicvf_init_resources(struct nicvf *);
  147 
  148 static int nicvf_setup_ifnet(struct nicvf *);
  149 static int nicvf_setup_ifmedia(struct nicvf *);
  150 static void nicvf_hw_addr_random(uint8_t *);
  151 
  152 static int nicvf_if_ioctl(struct ifnet *, u_long, caddr_t);
  153 static void nicvf_if_init(void *);
  154 static void nicvf_if_init_locked(struct nicvf *);
  155 static int nicvf_if_transmit(struct ifnet *, struct mbuf *);
  156 static void nicvf_if_qflush(struct ifnet *);
  157 static uint64_t nicvf_if_getcounter(struct ifnet *, ift_counter);
  158 
  159 static int nicvf_stop_locked(struct nicvf *);
  160 
  161 static void nicvf_media_status(struct ifnet *, struct ifmediareq *);
  162 static int nicvf_media_change(struct ifnet *);
  163 
  164 static void nicvf_tick_stats(void *);
  165 
  166 static int
  167 nicvf_probe(device_t dev)
  168 {
  169         uint16_t vendor_id;
  170         uint16_t device_id;
  171 
  172         vendor_id = pci_get_vendor(dev);
  173         device_id = pci_get_device(dev);
  174 
  175         if (vendor_id != PCI_VENDOR_ID_CAVIUM)
  176                 return (ENXIO);
  177 
  178         if (device_id == PCI_DEVICE_ID_THUNDER_NIC_VF ||
  179             device_id == PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF) {
  180                 device_set_desc(dev, VNIC_VF_DEVSTR);
  181                 return (BUS_PROBE_DEFAULT);
  182         }
  183 
  184         return (ENXIO);
  185 }
  186 
  187 static int
  188 nicvf_attach(device_t dev)
  189 {
  190         struct nicvf *nic;
  191         int rid, qcount;
  192         int err = 0;
  193         uint8_t hwaddr[ETHER_ADDR_LEN];
  194         uint8_t zeromac[] = {[0 ... (ETHER_ADDR_LEN - 1)] = 0};
  195 
  196         nic = device_get_softc(dev);
  197         nic->dev = dev;
  198         nic->pnicvf = nic;
  199 
  200         NICVF_CORE_LOCK_INIT(nic);
  201         /* Enable HW TSO on Pass2 */
  202         if (!pass1_silicon(dev))
  203                 nic->hw_tso = TRUE;
  204 
  205         rid = VNIC_VF_REG_RID;
  206         nic->reg_base = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
  207             RF_ACTIVE);
  208         if (nic->reg_base == NULL) {
  209                 device_printf(dev, "Could not allocate registers memory\n");
  210                 return (ENXIO);
  211         }
  212 
  213         qcount = MAX_CMP_QUEUES_PER_QS;
  214         nic->max_queues = qcount;
  215 
  216         err = nicvf_set_qset_resources(nic);
  217         if (err != 0)
  218                 goto err_free_res;
  219 
  220         /* Check if PF is alive and get MAC address for this VF */
  221         err = nicvf_allocate_misc_interrupt(nic);
  222         if (err != 0)
  223                 goto err_free_res;
  224 
  225         NICVF_CORE_LOCK(nic);
  226         err = nicvf_enable_misc_interrupt(nic);
  227         NICVF_CORE_UNLOCK(nic);
  228         if (err != 0)
  229                 goto err_release_intr;
  230 
  231         err = nicvf_allocate_net_interrupts(nic);
  232         if (err != 0) {
  233                 device_printf(dev,
  234                     "Could not allocate network interface interrupts\n");
  235                 goto err_free_ifnet;
  236         }
  237 
  238         /* If no MAC address was obtained we generate random one */
  239         if (memcmp(nic->hwaddr, zeromac, ETHER_ADDR_LEN) == 0) {
  240                 nicvf_hw_addr_random(hwaddr);
  241                 memcpy(nic->hwaddr, hwaddr, ETHER_ADDR_LEN);
  242                 NICVF_CORE_LOCK(nic);
  243                 nicvf_hw_set_mac_addr(nic, hwaddr);
  244                 NICVF_CORE_UNLOCK(nic);
  245         }
  246 
  247         /* Configure CPI alorithm */
  248         nic->cpi_alg = CPI_ALG_NONE;
  249         NICVF_CORE_LOCK(nic);
  250         nicvf_config_cpi(nic);
  251         /* Configure receive side scaling */
  252         if (nic->qs->rq_cnt > 1)
  253                 nicvf_rss_init(nic);
  254         NICVF_CORE_UNLOCK(nic);
  255 
  256         err = nicvf_setup_ifnet(nic);
  257         if (err != 0) {
  258                 device_printf(dev, "Could not set-up ifnet\n");
  259                 goto err_release_intr;
  260         }
  261 
  262         err = nicvf_setup_ifmedia(nic);
  263         if (err != 0) {
  264                 device_printf(dev, "Could not set-up ifmedia\n");
  265                 goto err_free_ifnet;
  266         }
  267 
  268         mtx_init(&nic->stats_mtx, "VNIC stats", NULL, MTX_DEF);
  269         callout_init_mtx(&nic->stats_callout, &nic->stats_mtx, 0);
  270 
  271         ether_ifattach(nic->ifp, nic->hwaddr);
  272 
  273         return (0);
  274 
  275 err_free_ifnet:
  276         if_free(nic->ifp);
  277 err_release_intr:
  278         nicvf_release_all_interrupts(nic);
  279 err_free_res:
  280         bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(nic->reg_base),
  281             nic->reg_base);
  282 
  283         return (err);
  284 }
  285 
  286 static int
  287 nicvf_detach(device_t dev)
  288 {
  289         struct nicvf *nic;
  290 
  291         nic = device_get_softc(dev);
  292 
  293         NICVF_CORE_LOCK(nic);
  294         /* Shut down the port and release ring resources */
  295         nicvf_stop_locked(nic);
  296         /* Release stats lock */
  297         mtx_destroy(&nic->stats_mtx);
  298         /* Release interrupts */
  299         nicvf_release_all_interrupts(nic);
  300         /* Release memory resource */
  301         if (nic->reg_base != NULL) {
  302                 bus_release_resource(dev, SYS_RES_MEMORY,
  303                     rman_get_rid(nic->reg_base), nic->reg_base);
  304         }
  305 
  306         /* Remove all ifmedia configurations */
  307         ifmedia_removeall(&nic->if_media);
  308         /* Free this ifnet */
  309         if_free(nic->ifp);
  310         NICVF_CORE_UNLOCK(nic);
  311         /* Finally destroy the lock */
  312         NICVF_CORE_LOCK_DESTROY(nic);
  313 
  314         return (0);
  315 }
  316 
  317 static void
  318 nicvf_hw_addr_random(uint8_t *hwaddr)
  319 {
  320         uint32_t rnd;
  321         uint8_t addr[ETHER_ADDR_LEN];
  322 
  323         /*
  324          * Create randomized MAC address.
  325          * Set 'bsd' + random 24 low-order bits.
  326          */
  327         rnd = arc4random() & 0x00ffffff;
  328         addr[0] = 'b';
  329         addr[1] = 's';
  330         addr[2] = 'd';
  331         addr[3] = rnd >> 16;
  332         addr[4] = rnd >> 8;
  333         addr[5] = rnd >> 0;
  334 
  335         memcpy(hwaddr, addr, ETHER_ADDR_LEN);
  336 }
  337 
  338 static int
  339 nicvf_setup_ifnet(struct nicvf *nic)
  340 {
  341         struct ifnet *ifp;
  342 
  343         ifp = if_alloc(IFT_ETHER);
  344         if (ifp == NULL) {
  345                 device_printf(nic->dev, "Could not allocate ifnet structure\n");
  346                 return (ENOMEM);
  347         }
  348 
  349         nic->ifp = ifp;
  350 
  351         if_setsoftc(ifp, nic);
  352         if_initname(ifp, device_get_name(nic->dev), device_get_unit(nic->dev));
  353         if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
  354 
  355         if_settransmitfn(ifp, nicvf_if_transmit);
  356         if_setqflushfn(ifp, nicvf_if_qflush);
  357         if_setioctlfn(ifp, nicvf_if_ioctl);
  358         if_setinitfn(ifp, nicvf_if_init);
  359         if_setgetcounterfn(ifp, nicvf_if_getcounter);
  360 
  361         if_setmtu(ifp, ETHERMTU);
  362 
  363         /* Reset caps */
  364         if_setcapabilities(ifp, 0);
  365 
  366         /* Set the default values */
  367         if_setcapabilitiesbit(ifp, IFCAP_VLAN_MTU | IFCAP_JUMBO_MTU, 0);
  368         if_setcapabilitiesbit(ifp, IFCAP_LRO, 0);
  369         if (nic->hw_tso) {
  370                 /* TSO */
  371                 if_setcapabilitiesbit(ifp, IFCAP_TSO4, 0);
  372                 /* TSO parameters */
  373                 if_sethwtsomax(ifp, NICVF_TSO_MAXSIZE);
  374                 if_sethwtsomaxsegcount(ifp, NICVF_TSO_NSEGS);
  375                 if_sethwtsomaxsegsize(ifp, MCLBYTES);
  376         }
  377         /* IP/TCP/UDP HW checksums */
  378         if_setcapabilitiesbit(ifp, IFCAP_HWCSUM, 0);
  379         if_setcapabilitiesbit(ifp, IFCAP_HWSTATS, 0);
  380         /*
  381          * HW offload enable
  382          */
  383         if_clearhwassist(ifp);
  384         if_sethwassistbits(ifp, (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP), 0);
  385         if (nic->hw_tso)
  386                 if_sethwassistbits(ifp, (CSUM_TSO), 0);
  387         if_setcapenable(ifp, if_getcapabilities(ifp));
  388 
  389         return (0);
  390 }
  391 
  392 static int
  393 nicvf_setup_ifmedia(struct nicvf *nic)
  394 {
  395 
  396         ifmedia_init(&nic->if_media, IFM_IMASK, nicvf_media_change,
  397             nicvf_media_status);
  398 
  399         /*
  400          * Advertise availability of all possible connection types,
  401          * even though not all are possible at the same time.
  402          */
  403 
  404         ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_10_T | IFM_FDX),
  405             0, NULL);
  406         ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_100_TX | IFM_FDX),
  407             0, NULL);
  408         ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_1000_T | IFM_FDX),
  409             0, NULL);
  410         ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_10G_SR | IFM_FDX),
  411             0, NULL);
  412         ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_40G_CR4 | IFM_FDX),
  413             0, NULL);
  414         ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_AUTO | IFM_FDX),
  415             0, NULL);
  416 
  417         ifmedia_set(&nic->if_media, (IFM_ETHER | IFM_AUTO | IFM_FDX));
  418 
  419         return (0);
  420 }
  421 
  422 static int
  423 nicvf_if_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
  424 {
  425         struct nicvf *nic;
  426         struct rcv_queue *rq;
  427         struct ifreq *ifr;
  428         uint32_t flags;
  429         int mask, err;
  430         int rq_idx;
  431 #if defined(INET) || defined(INET6)
  432         struct ifaddr *ifa;
  433         boolean_t avoid_reset = FALSE;
  434 #endif
  435 
  436         nic = if_getsoftc(ifp);
  437         ifr = (struct ifreq *)data;
  438 #if defined(INET) || defined(INET6)
  439         ifa = (struct ifaddr *)data;
  440 #endif
  441         err = 0;
  442         switch (cmd) {
  443         case SIOCSIFADDR:
  444 #ifdef INET
  445                 if (ifa->ifa_addr->sa_family == AF_INET)
  446                         avoid_reset = TRUE;
  447 #endif
  448 #ifdef INET6
  449                 if (ifa->ifa_addr->sa_family == AF_INET6)
  450                         avoid_reset = TRUE;
  451 #endif
  452 
  453 #if defined(INET) || defined(INET6)
  454                 /* Avoid reinitialization unless it's necessary */
  455                 if (avoid_reset) {
  456                         if_setflagbits(ifp, IFF_UP, 0);
  457                         if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
  458                                 nicvf_if_init(nic);
  459 #ifdef INET
  460                         if (!(if_getflags(ifp) & IFF_NOARP))
  461                                 arp_ifinit(ifp, ifa);
  462 #endif
  463 
  464                         return (0);
  465                 }
  466 #endif
  467                 err = ether_ioctl(ifp, cmd, data);
  468                 break;
  469         case SIOCSIFMTU:
  470                 if (ifr->ifr_mtu < NIC_HW_MIN_FRS ||
  471                     ifr->ifr_mtu > NIC_HW_MAX_FRS) {
  472                         err = EINVAL;
  473                 } else {
  474                         NICVF_CORE_LOCK(nic);
  475                         err = nicvf_update_hw_max_frs(nic, ifr->ifr_mtu);
  476                         if (err == 0)
  477                                 if_setmtu(ifp, ifr->ifr_mtu);
  478                         NICVF_CORE_UNLOCK(nic);
  479                 }
  480                 break;
  481         case SIOCSIFFLAGS:
  482                 NICVF_CORE_LOCK(nic);
  483                 flags = if_getflags(ifp);
  484                 if (flags & IFF_UP) {
  485                         if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
  486                                 if ((flags ^ nic->if_flags) & IFF_PROMISC) {
  487                                         /* Change promiscous mode */
  488 #if 0 /* XXX */
  489                                         nicvf_set_promiscous(nic);
  490 #endif
  491                                 }
  492 
  493                                 if ((flags ^ nic->if_flags) & IFF_ALLMULTI) {
  494                                         /* Change multicasting settings */
  495 #if 0 /* XXX */
  496                                         nicvf_set_multicast(nic);
  497 #endif
  498                                 }
  499                         } else {
  500                                 nicvf_if_init_locked(nic);
  501                         }
  502                 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
  503                         nicvf_stop_locked(nic);
  504 
  505                 nic->if_flags = flags;
  506                 NICVF_CORE_UNLOCK(nic);
  507                 break;
  508 
  509         case SIOCADDMULTI:
  510         case SIOCDELMULTI:
  511                 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
  512 #if 0
  513                         NICVF_CORE_LOCK(nic);
  514                         /* ARM64TODO */
  515                         nicvf_set_multicast(nic);
  516                         NICVF_CORE_UNLOCK(nic);
  517 #endif
  518                 }
  519                 break;
  520 
  521         case SIOCSIFMEDIA:
  522         case SIOCGIFMEDIA:
  523                 err = ifmedia_ioctl(ifp, ifr, &nic->if_media, cmd);
  524                 break;
  525 
  526         case SIOCSIFCAP:
  527                 mask = if_getcapenable(ifp) ^ ifr->ifr_reqcap;
  528                 if (mask & IFCAP_VLAN_MTU) {
  529                         /* No work to do except acknowledge the change took. */
  530                         if_togglecapenable(ifp, IFCAP_VLAN_MTU);
  531                 }
  532                 if (mask & IFCAP_TXCSUM)
  533                         if_togglecapenable(ifp, IFCAP_TXCSUM);
  534                 if (mask & IFCAP_RXCSUM)
  535                         if_togglecapenable(ifp, IFCAP_RXCSUM);
  536                 if ((mask & IFCAP_TSO4) && nic->hw_tso)
  537                         if_togglecapenable(ifp, IFCAP_TSO4);
  538                 if (mask & IFCAP_LRO) {
  539                         /*
  540                          * Lock the driver for a moment to avoid
  541                          * mismatch in per-queue settings.
  542                          */
  543                         NICVF_CORE_LOCK(nic);
  544                         if_togglecapenable(ifp, IFCAP_LRO);
  545                         if ((if_getdrvflags(nic->ifp) & IFF_DRV_RUNNING) != 0) {
  546                                 /*
  547                                  * Now disable LRO for subsequent packets.
  548                                  * Atomicity of this change is not necessary
  549                                  * as we don't need precise toggle of this
  550                                  * feature for all threads processing the
  551                                  * completion queue.
  552                                  */
  553                                 for (rq_idx = 0;
  554                                     rq_idx < nic->qs->rq_cnt; rq_idx++) {
  555                                         rq = &nic->qs->rq[rq_idx];
  556                                         rq->lro_enabled = !rq->lro_enabled;
  557                                 }
  558                         }
  559                         NICVF_CORE_UNLOCK(nic);
  560                 }
  561 
  562                 break;
  563 
  564         default:
  565                 err = ether_ioctl(ifp, cmd, data);
  566                 break;
  567         }
  568 
  569         return (err);
  570 }
  571 
  572 static void
  573 nicvf_if_init_locked(struct nicvf *nic)
  574 {
  575         struct queue_set *qs = nic->qs;
  576         struct ifnet *ifp;
  577         int qidx;
  578         int err;
  579         caddr_t if_addr;
  580 
  581         NICVF_CORE_LOCK_ASSERT(nic);
  582         ifp = nic->ifp;
  583 
  584         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0)
  585                 nicvf_stop_locked(nic);
  586 
  587         err = nicvf_enable_misc_interrupt(nic);
  588         if (err != 0) {
  589                 if_printf(ifp, "Could not reenable Mbox interrupt\n");
  590                 return;
  591         }
  592 
  593         /* Get the latest MAC address */
  594         if_addr = if_getlladdr(ifp);
  595         /* Update MAC address if changed */
  596         if (memcmp(nic->hwaddr, if_addr, ETHER_ADDR_LEN) != 0) {
  597                 memcpy(nic->hwaddr, if_addr, ETHER_ADDR_LEN);
  598                 nicvf_hw_set_mac_addr(nic, if_addr);
  599         }
  600 
  601         /* Initialize the queues */
  602         err = nicvf_init_resources(nic);
  603         if (err != 0)
  604                 goto error;
  605 
  606         /* Make sure queue initialization is written */
  607         wmb();
  608 
  609         nicvf_reg_write(nic, NIC_VF_INT, ~0UL);
  610         /* Enable Qset err interrupt */
  611         nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0);
  612 
  613         /* Enable completion queue interrupt */
  614         for (qidx = 0; qidx < qs->cq_cnt; qidx++)
  615                 nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx);
  616 
  617         /* Enable RBDR threshold interrupt */
  618         for (qidx = 0; qidx < qs->rbdr_cnt; qidx++)
  619                 nicvf_enable_intr(nic, NICVF_INTR_RBDR, qidx);
  620 
  621         nic->drv_stats.txq_stop = 0;
  622         nic->drv_stats.txq_wake = 0;
  623 
  624         /* Activate network interface */
  625         if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
  626 
  627         /* Schedule callout to update stats */
  628         callout_reset(&nic->stats_callout, hz, nicvf_tick_stats, nic);
  629 
  630         return;
  631 
  632 error:
  633         /* Something went very wrong. Disable this ifnet for good */
  634         if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
  635 }
  636 
  637 static void
  638 nicvf_if_init(void *if_softc)
  639 {
  640         struct nicvf *nic = if_softc;
  641 
  642         NICVF_CORE_LOCK(nic);
  643         nicvf_if_init_locked(nic);
  644         NICVF_CORE_UNLOCK(nic);
  645 }
  646 
  647 static int
  648 nicvf_if_transmit(struct ifnet *ifp, struct mbuf *mbuf)
  649 {
  650         struct nicvf *nic = if_getsoftc(ifp);
  651         struct queue_set *qs = nic->qs;
  652         struct snd_queue *sq;
  653         struct mbuf *mtmp;
  654         int qidx;
  655         int err = 0;
  656 
  657         if (__predict_false(qs == NULL)) {
  658                 panic("%s: missing queue set for %s", __func__,
  659                     device_get_nameunit(nic->dev));
  660         }
  661 
  662         /* Select queue */
  663         if (M_HASHTYPE_GET(mbuf) != M_HASHTYPE_NONE)
  664                 qidx = mbuf->m_pkthdr.flowid % qs->sq_cnt;
  665         else
  666                 qidx = curcpu % qs->sq_cnt;
  667 
  668         sq = &qs->sq[qidx];
  669 
  670         if (mbuf->m_next != NULL &&
  671             (mbuf->m_pkthdr.csum_flags &
  672             (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP)) != 0) {
  673                 if (M_WRITABLE(mbuf) == 0) {
  674                         mtmp = m_dup(mbuf, M_NOWAIT);
  675                         m_freem(mbuf);
  676                         if (mtmp == NULL)
  677                                 return (ENOBUFS);
  678                         mbuf = mtmp;
  679                 }
  680         }
  681 
  682         err = drbr_enqueue(ifp, sq->br, mbuf);
  683         if (((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
  684             IFF_DRV_RUNNING) || !nic->link_up || (err != 0)) {
  685                 /*
  686                  * Try to enqueue packet to the ring buffer.
  687                  * If the driver is not active, link down or enqueue operation
  688                  * failed, return with the appropriate error code.
  689                  */
  690                 return (err);
  691         }
  692 
  693         if (NICVF_TX_TRYLOCK(sq) != 0) {
  694                 err = nicvf_xmit_locked(sq);
  695                 NICVF_TX_UNLOCK(sq);
  696                 return (err);
  697         } else
  698                 taskqueue_enqueue(sq->snd_taskq, &sq->snd_task);
  699 
  700         return (0);
  701 }
  702 
  703 static void
  704 nicvf_if_qflush(struct ifnet *ifp)
  705 {
  706         struct nicvf *nic;
  707         struct queue_set *qs;
  708         struct snd_queue *sq;
  709         struct mbuf *mbuf;
  710         size_t idx;
  711 
  712         nic = if_getsoftc(ifp);
  713         qs = nic->qs;
  714 
  715         for (idx = 0; idx < qs->sq_cnt; idx++) {
  716                 sq = &qs->sq[idx];
  717                 NICVF_TX_LOCK(sq);
  718                 while ((mbuf = buf_ring_dequeue_sc(sq->br)) != NULL)
  719                         m_freem(mbuf);
  720                 NICVF_TX_UNLOCK(sq);
  721         }
  722         if_qflush(ifp);
  723 }
  724 
  725 static uint64_t
  726 nicvf_if_getcounter(struct ifnet *ifp, ift_counter cnt)
  727 {
  728         struct nicvf *nic;
  729         struct nicvf_hw_stats *hw_stats;
  730         struct nicvf_drv_stats *drv_stats;
  731 
  732         nic = if_getsoftc(ifp);
  733         hw_stats = &nic->hw_stats;
  734         drv_stats = &nic->drv_stats;
  735 
  736         switch (cnt) {
  737         case IFCOUNTER_IPACKETS:
  738                 return (drv_stats->rx_frames_ok);
  739         case IFCOUNTER_OPACKETS:
  740                 return (drv_stats->tx_frames_ok);
  741         case IFCOUNTER_IBYTES:
  742                 return (hw_stats->rx_bytes);
  743         case IFCOUNTER_OBYTES:
  744                 return (hw_stats->tx_bytes_ok);
  745         case IFCOUNTER_IMCASTS:
  746                 return (hw_stats->rx_mcast_frames);
  747         case IFCOUNTER_COLLISIONS:
  748                 return (0);
  749         case IFCOUNTER_IQDROPS:
  750                 return (drv_stats->rx_drops);
  751         case IFCOUNTER_OQDROPS:
  752                 return (drv_stats->tx_drops);
  753         default:
  754                 return (if_get_counter_default(ifp, cnt));
  755         }
  756 
  757 }
  758 
  759 static void
  760 nicvf_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
  761 {
  762         struct nicvf *nic = if_getsoftc(ifp);
  763 
  764         NICVF_CORE_LOCK(nic);
  765 
  766         ifmr->ifm_status = IFM_AVALID;
  767         ifmr->ifm_active = IFM_ETHER;
  768 
  769         if (nic->link_up) {
  770                 /* Device attached to working network */
  771                 ifmr->ifm_status |= IFM_ACTIVE;
  772         }
  773 
  774         switch (nic->speed) {
  775         case SPEED_10:
  776                 ifmr->ifm_active |= IFM_10_T;
  777                 break;
  778         case SPEED_100:
  779                 ifmr->ifm_active |= IFM_100_TX;
  780                 break;
  781         case SPEED_1000:
  782                 ifmr->ifm_active |= IFM_1000_T;
  783                 break;
  784         case SPEED_10000:
  785                 ifmr->ifm_active |= IFM_10G_SR;
  786                 break;
  787         case SPEED_40000:
  788                 ifmr->ifm_active |= IFM_40G_CR4;
  789                 break;
  790         default:
  791                 ifmr->ifm_active |= IFM_AUTO;
  792                 break;
  793         }
  794 
  795         if (nic->duplex)
  796                 ifmr->ifm_active |= IFM_FDX;
  797         else
  798                 ifmr->ifm_active |= IFM_HDX;
  799 
  800         NICVF_CORE_UNLOCK(nic);
  801 }
  802 
  803 static int
  804 nicvf_media_change(struct ifnet *ifp __unused)
  805 {
  806 
  807         return (0);
  808 }
  809 
  810 /* Register read/write APIs */
  811 void
  812 nicvf_reg_write(struct nicvf *nic, bus_space_handle_t offset, uint64_t val)
  813 {
  814 
  815         bus_write_8(nic->reg_base, offset, val);
  816 }
  817 
  818 uint64_t
  819 nicvf_reg_read(struct nicvf *nic, uint64_t offset)
  820 {
  821 
  822         return (bus_read_8(nic->reg_base, offset));
  823 }
  824 
  825 void
  826 nicvf_queue_reg_write(struct nicvf *nic, bus_space_handle_t offset,
  827     uint64_t qidx, uint64_t val)
  828 {
  829 
  830         bus_write_8(nic->reg_base, offset + (qidx << NIC_Q_NUM_SHIFT), val);
  831 }
  832 
  833 uint64_t
  834 nicvf_queue_reg_read(struct nicvf *nic, bus_space_handle_t offset,
  835     uint64_t qidx)
  836 {
  837 
  838         return (bus_read_8(nic->reg_base, offset + (qidx << NIC_Q_NUM_SHIFT)));
  839 }
  840 
  841 /* VF -> PF mailbox communication */
  842 static void
  843 nicvf_write_to_mbx(struct nicvf *nic, union nic_mbx *mbx)
  844 {
  845         uint64_t *msg = (uint64_t *)mbx;
  846 
  847         nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 0, msg[0]);
  848         nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 8, msg[1]);
  849 }
  850 
  851 int
  852 nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx)
  853 {
  854         int timeout = NIC_MBOX_MSG_TIMEOUT * 10;
  855         int sleep = 2;
  856 
  857         NICVF_CORE_LOCK_ASSERT(nic);
  858 
  859         nic->pf_acked = FALSE;
  860         nic->pf_nacked = FALSE;
  861 
  862         nicvf_write_to_mbx(nic, mbx);
  863 
  864         /* Wait for previous message to be acked, timeout 2sec */
  865         while (!nic->pf_acked) {
  866                 if (nic->pf_nacked)
  867                         return (EINVAL);
  868 
  869                 DELAY(sleep * 1000);
  870 
  871                 if (nic->pf_acked)
  872                         break;
  873                 timeout -= sleep;
  874                 if (!timeout) {
  875                         device_printf(nic->dev,
  876                                    "PF didn't ack to mbox msg %d from VF%d\n",
  877                                    (mbx->msg.msg & 0xFF), nic->vf_id);
  878 
  879                         return (EBUSY);
  880                 }
  881         }
  882         return (0);
  883 }
  884 
  885 /*
  886  * Checks if VF is able to comminicate with PF
  887  * and also gets the VNIC number this VF is associated to.
  888  */
  889 static int
  890 nicvf_check_pf_ready(struct nicvf *nic)
  891 {
  892         union nic_mbx mbx = {};
  893 
  894         mbx.msg.msg = NIC_MBOX_MSG_READY;
  895         if (nicvf_send_msg_to_pf(nic, &mbx)) {
  896                 device_printf(nic->dev,
  897                            "PF didn't respond to READY msg\n");
  898                 return 0;
  899         }
  900 
  901         return 1;
  902 }
  903 
  904 static void
  905 nicvf_read_bgx_stats(struct nicvf *nic, struct bgx_stats_msg *bgx)
  906 {
  907 
  908         if (bgx->rx)
  909                 nic->bgx_stats.rx_stats[bgx->idx] = bgx->stats;
  910         else
  911                 nic->bgx_stats.tx_stats[bgx->idx] = bgx->stats;
  912 }
  913 
  914 static void
  915 nicvf_handle_mbx_intr(struct nicvf *nic)
  916 {
  917         union nic_mbx mbx = {};
  918         uint64_t *mbx_data;
  919         uint64_t mbx_addr;
  920         int i;
  921 
  922         mbx_addr = NIC_VF_PF_MAILBOX_0_1;
  923         mbx_data = (uint64_t *)&mbx;
  924 
  925         for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) {
  926                 *mbx_data = nicvf_reg_read(nic, mbx_addr);
  927                 mbx_data++;
  928                 mbx_addr += sizeof(uint64_t);
  929         }
  930 
  931         switch (mbx.msg.msg) {
  932         case NIC_MBOX_MSG_READY:
  933                 nic->pf_acked = TRUE;
  934                 nic->vf_id = mbx.nic_cfg.vf_id & 0x7F;
  935                 nic->tns_mode = mbx.nic_cfg.tns_mode & 0x7F;
  936                 nic->node = mbx.nic_cfg.node_id;
  937                 memcpy(nic->hwaddr, mbx.nic_cfg.mac_addr, ETHER_ADDR_LEN);
  938                 nic->loopback_supported = mbx.nic_cfg.loopback_supported;
  939                 nic->link_up = FALSE;
  940                 nic->duplex = 0;
  941                 nic->speed = 0;
  942                 break;
  943         case NIC_MBOX_MSG_ACK:
  944                 nic->pf_acked = TRUE;
  945                 break;
  946         case NIC_MBOX_MSG_NACK:
  947                 nic->pf_nacked = TRUE;
  948                 break;
  949         case NIC_MBOX_MSG_RSS_SIZE:
  950                 nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size;
  951                 nic->pf_acked = TRUE;
  952                 break;
  953         case NIC_MBOX_MSG_BGX_STATS:
  954                 nicvf_read_bgx_stats(nic, &mbx.bgx_stats);
  955                 nic->pf_acked = TRUE;
  956                 break;
  957         case NIC_MBOX_MSG_BGX_LINK_CHANGE:
  958                 nic->pf_acked = TRUE;
  959                 nic->link_up = mbx.link_status.link_up;
  960                 nic->duplex = mbx.link_status.duplex;
  961                 nic->speed = mbx.link_status.speed;
  962                 if (nic->link_up) {
  963                         if_setbaudrate(nic->ifp, nic->speed * 1000000);
  964                         if_link_state_change(nic->ifp, LINK_STATE_UP);
  965                 } else {
  966                         if_setbaudrate(nic->ifp, 0);
  967                         if_link_state_change(nic->ifp, LINK_STATE_DOWN);
  968                 }
  969                 break;
  970         default:
  971                 device_printf(nic->dev,
  972                            "Invalid message from PF, msg 0x%x\n", mbx.msg.msg);
  973                 break;
  974         }
  975         nicvf_clear_intr(nic, NICVF_INTR_MBOX, 0);
  976 }
  977 
  978 static int
  979 nicvf_update_hw_max_frs(struct nicvf *nic, int mtu)
  980 {
  981         union nic_mbx mbx = {};
  982 
  983         mbx.frs.msg = NIC_MBOX_MSG_SET_MAX_FRS;
  984         mbx.frs.max_frs = mtu;
  985         mbx.frs.vf_id = nic->vf_id;
  986 
  987         return nicvf_send_msg_to_pf(nic, &mbx);
  988 }
  989 
  990 static int
  991 nicvf_hw_set_mac_addr(struct nicvf *nic, uint8_t *hwaddr)
  992 {
  993         union nic_mbx mbx = {};
  994 
  995         mbx.mac.msg = NIC_MBOX_MSG_SET_MAC;
  996         mbx.mac.vf_id = nic->vf_id;
  997         memcpy(mbx.mac.mac_addr, hwaddr, ETHER_ADDR_LEN);
  998 
  999         return (nicvf_send_msg_to_pf(nic, &mbx));
 1000 }
 1001 
 1002 static void
 1003 nicvf_config_cpi(struct nicvf *nic)
 1004 {
 1005         union nic_mbx mbx = {};
 1006 
 1007         mbx.cpi_cfg.msg = NIC_MBOX_MSG_CPI_CFG;
 1008         mbx.cpi_cfg.vf_id = nic->vf_id;
 1009         mbx.cpi_cfg.cpi_alg = nic->cpi_alg;
 1010         mbx.cpi_cfg.rq_cnt = nic->qs->rq_cnt;
 1011 
 1012         nicvf_send_msg_to_pf(nic, &mbx);
 1013 }
 1014 
 1015 static void
 1016 nicvf_get_rss_size(struct nicvf *nic)
 1017 {
 1018         union nic_mbx mbx = {};
 1019 
 1020         mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE;
 1021         mbx.rss_size.vf_id = nic->vf_id;
 1022         nicvf_send_msg_to_pf(nic, &mbx);
 1023 }
 1024 
 1025 static void
 1026 nicvf_config_rss(struct nicvf *nic)
 1027 {
 1028         union nic_mbx mbx = {};
 1029         struct nicvf_rss_info *rss;
 1030         int ind_tbl_len;
 1031         int i, nextq;
 1032 
 1033         rss = &nic->rss_info;
 1034         ind_tbl_len = rss->rss_size;
 1035         nextq = 0;
 1036 
 1037         mbx.rss_cfg.vf_id = nic->vf_id;
 1038         mbx.rss_cfg.hash_bits = rss->hash_bits;
 1039         while (ind_tbl_len != 0) {
 1040                 mbx.rss_cfg.tbl_offset = nextq;
 1041                 mbx.rss_cfg.tbl_len = MIN(ind_tbl_len,
 1042                     RSS_IND_TBL_LEN_PER_MBX_MSG);
 1043                 mbx.rss_cfg.msg = mbx.rss_cfg.tbl_offset ?
 1044                     NIC_MBOX_MSG_RSS_CFG_CONT : NIC_MBOX_MSG_RSS_CFG;
 1045 
 1046                 for (i = 0; i < mbx.rss_cfg.tbl_len; i++)
 1047                         mbx.rss_cfg.ind_tbl[i] = rss->ind_tbl[nextq++];
 1048 
 1049                 nicvf_send_msg_to_pf(nic, &mbx);
 1050 
 1051                 ind_tbl_len -= mbx.rss_cfg.tbl_len;
 1052         }
 1053 }
 1054 
 1055 static void
 1056 nicvf_set_rss_key(struct nicvf *nic)
 1057 {
 1058         struct nicvf_rss_info *rss;
 1059         uint64_t key_addr;
 1060         int idx;
 1061 
 1062         rss = &nic->rss_info;
 1063         key_addr = NIC_VNIC_RSS_KEY_0_4;
 1064 
 1065         for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) {
 1066                 nicvf_reg_write(nic, key_addr, rss->key[idx]);
 1067                 key_addr += sizeof(uint64_t);
 1068         }
 1069 }
 1070 
 1071 static int
 1072 nicvf_rss_init(struct nicvf *nic)
 1073 {
 1074         struct nicvf_rss_info *rss;
 1075         int idx;
 1076 
 1077         nicvf_get_rss_size(nic);
 1078 
 1079         rss = &nic->rss_info;
 1080         if (nic->cpi_alg != CPI_ALG_NONE) {
 1081                 rss->enable = FALSE;
 1082                 rss->hash_bits = 0;
 1083                 return (ENXIO);
 1084         }
 1085 
 1086         rss->enable = TRUE;
 1087 
 1088         /* Using the HW reset value for now */
 1089         rss->key[0] = 0xFEED0BADFEED0BADUL;
 1090         rss->key[1] = 0xFEED0BADFEED0BADUL;
 1091         rss->key[2] = 0xFEED0BADFEED0BADUL;
 1092         rss->key[3] = 0xFEED0BADFEED0BADUL;
 1093         rss->key[4] = 0xFEED0BADFEED0BADUL;
 1094 
 1095         nicvf_set_rss_key(nic);
 1096 
 1097         rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA;
 1098         nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss->cfg);
 1099 
 1100         rss->hash_bits = fls(rss->rss_size) - 1;
 1101         for (idx = 0; idx < rss->rss_size; idx++)
 1102                 rss->ind_tbl[idx] = idx % nic->rx_queues;
 1103 
 1104         nicvf_config_rss(nic);
 1105 
 1106         return (0);
 1107 }
 1108 
 1109 static int
 1110 nicvf_init_resources(struct nicvf *nic)
 1111 {
 1112         int err;
 1113         union nic_mbx mbx = {};
 1114 
 1115         mbx.msg.msg = NIC_MBOX_MSG_CFG_DONE;
 1116 
 1117         /* Enable Qset */
 1118         nicvf_qset_config(nic, TRUE);
 1119 
 1120         /* Initialize queues and HW for data transfer */
 1121         err = nicvf_config_data_transfer(nic, TRUE);
 1122         if (err) {
 1123                 device_printf(nic->dev,
 1124                     "Failed to alloc/config VF's QSet resources\n");
 1125                 return (err);
 1126         }
 1127 
 1128         /* Send VF config done msg to PF */
 1129         nicvf_write_to_mbx(nic, &mbx);
 1130 
 1131         return (0);
 1132 }
 1133 
 1134 static void
 1135 nicvf_misc_intr_handler(void *arg)
 1136 {
 1137         struct nicvf *nic = (struct nicvf *)arg;
 1138         uint64_t intr;
 1139 
 1140         intr = nicvf_reg_read(nic, NIC_VF_INT);
 1141         /* Check for spurious interrupt */
 1142         if (!(intr & NICVF_INTR_MBOX_MASK))
 1143                 return;
 1144 
 1145         nicvf_handle_mbx_intr(nic);
 1146 }
 1147 
 1148 static int
 1149 nicvf_intr_handler(void *arg)
 1150 {
 1151         struct nicvf *nic;
 1152         struct cmp_queue *cq;
 1153         int qidx;
 1154 
 1155         cq = (struct cmp_queue *)arg;
 1156         nic = cq->nic;
 1157         qidx = cq->idx;
 1158 
 1159         /* Disable interrupts */
 1160         nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
 1161 
 1162         taskqueue_enqueue(cq->cmp_taskq, &cq->cmp_task);
 1163 
 1164         /* Clear interrupt */
 1165         nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx);
 1166 
 1167         return (FILTER_HANDLED);
 1168 }
 1169 
 1170 static void
 1171 nicvf_rbdr_intr_handler(void *arg)
 1172 {
 1173         struct nicvf *nic;
 1174         struct queue_set *qs;
 1175         struct rbdr *rbdr;
 1176         int qidx;
 1177 
 1178         nic = (struct nicvf *)arg;
 1179 
 1180         /* Disable RBDR interrupt and schedule softirq */
 1181         for (qidx = 0; qidx < nic->qs->rbdr_cnt; qidx++) {
 1182                 if (!nicvf_is_intr_enabled(nic, NICVF_INTR_RBDR, qidx))
 1183                         continue;
 1184                 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
 1185 
 1186                 qs = nic->qs;
 1187                 rbdr = &qs->rbdr[qidx];
 1188                 taskqueue_enqueue(rbdr->rbdr_taskq, &rbdr->rbdr_task_nowait);
 1189                 /* Clear interrupt */
 1190                 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
 1191         }
 1192 }
 1193 
 1194 static void
 1195 nicvf_qs_err_intr_handler(void *arg)
 1196 {
 1197         struct nicvf *nic = (struct nicvf *)arg;
 1198         struct queue_set *qs = nic->qs;
 1199 
 1200         /* Disable Qset err interrupt and schedule softirq */
 1201         nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
 1202         taskqueue_enqueue(qs->qs_err_taskq, &qs->qs_err_task);
 1203         nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
 1204 
 1205 }
 1206 
 1207 static int
 1208 nicvf_enable_msix(struct nicvf *nic)
 1209 {
 1210         struct pci_devinfo *dinfo;
 1211         int rid, count;
 1212         int ret;
 1213 
 1214         dinfo = device_get_ivars(nic->dev);
 1215         rid = dinfo->cfg.msix.msix_table_bar;
 1216         nic->msix_table_res =
 1217             bus_alloc_resource_any(nic->dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
 1218         if (nic->msix_table_res == NULL) {
 1219                 device_printf(nic->dev,
 1220                     "Could not allocate memory for MSI-X table\n");
 1221                 return (ENXIO);
 1222         }
 1223 
 1224         count = nic->num_vec = NIC_VF_MSIX_VECTORS;
 1225 
 1226         ret = pci_alloc_msix(nic->dev, &count);
 1227         if ((ret != 0) || (count != nic->num_vec)) {
 1228                 device_printf(nic->dev,
 1229                     "Request for #%d msix vectors failed, error: %d\n",
 1230                     nic->num_vec, ret);
 1231                 return (ret);
 1232         }
 1233 
 1234         nic->msix_enabled = 1;
 1235         return (0);
 1236 }
 1237 
 1238 static void
 1239 nicvf_disable_msix(struct nicvf *nic)
 1240 {
 1241 
 1242         if (nic->msix_enabled) {
 1243                 pci_release_msi(nic->dev);
 1244                 nic->msix_enabled = 0;
 1245                 nic->num_vec = 0;
 1246         }
 1247 }
 1248 
 1249 static void
 1250 nicvf_release_all_interrupts(struct nicvf *nic)
 1251 {
 1252         struct resource *res;
 1253         int irq;
 1254         int err;
 1255 
 1256         /* Free registered interrupts */
 1257         for (irq = 0; irq < nic->num_vec; irq++) {
 1258                 res = nic->msix_entries[irq].irq_res;
 1259                 if (res == NULL)
 1260                         continue;
 1261                 /* Teardown interrupt first */
 1262                 if (nic->msix_entries[irq].handle != NULL) {
 1263                         err = bus_teardown_intr(nic->dev,
 1264                             nic->msix_entries[irq].irq_res,
 1265                             nic->msix_entries[irq].handle);
 1266                         KASSERT(err == 0,
 1267                             ("ERROR: Unable to teardown interrupt %d", irq));
 1268                         nic->msix_entries[irq].handle = NULL;
 1269                 }
 1270 
 1271                 bus_release_resource(nic->dev, SYS_RES_IRQ,
 1272                             rman_get_rid(res), nic->msix_entries[irq].irq_res);
 1273                 nic->msix_entries[irq].irq_res = NULL;
 1274         }
 1275         /* Disable MSI-X */
 1276         nicvf_disable_msix(nic);
 1277 }
 1278 
 1279 /*
 1280  * Initialize MSIX vectors and register MISC interrupt.
 1281  * Send READY message to PF to check if its alive
 1282  */
 1283 static int
 1284 nicvf_allocate_misc_interrupt(struct nicvf *nic)
 1285 {
 1286         struct resource *res;
 1287         int irq, rid;
 1288         int ret = 0;
 1289 
 1290         /* Return if mailbox interrupt is already registered */
 1291         if (nic->msix_enabled)
 1292                 return (0);
 1293 
 1294         /* Enable MSI-X */
 1295         if (nicvf_enable_msix(nic) != 0)
 1296                 return (ENXIO);
 1297 
 1298         irq = NICVF_INTR_ID_MISC;
 1299         rid = irq + 1;
 1300         nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev,
 1301             SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE));
 1302         if (nic->msix_entries[irq].irq_res == NULL) {
 1303                 device_printf(nic->dev,
 1304                     "Could not allocate Mbox interrupt for VF%d\n",
 1305                     device_get_unit(nic->dev));
 1306                 return (ENXIO);
 1307         }
 1308 
 1309         ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res,
 1310             (INTR_MPSAFE | INTR_TYPE_MISC), NULL, nicvf_misc_intr_handler, nic,
 1311             &nic->msix_entries[irq].handle);
 1312         if (ret != 0) {
 1313                 res = nic->msix_entries[irq].irq_res;
 1314                 bus_release_resource(nic->dev, SYS_RES_IRQ,
 1315                             rman_get_rid(res), res);
 1316                 nic->msix_entries[irq].irq_res = NULL;
 1317                 return (ret);
 1318         }
 1319 
 1320         return (0);
 1321 }
 1322 
 1323 static int
 1324 nicvf_enable_misc_interrupt(struct nicvf *nic)
 1325 {
 1326 
 1327         /* Enable mailbox interrupt */
 1328         nicvf_enable_intr(nic, NICVF_INTR_MBOX, 0);
 1329 
 1330         /* Check if VF is able to communicate with PF */
 1331         if (!nicvf_check_pf_ready(nic)) {
 1332                 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
 1333                 return (ENXIO);
 1334         }
 1335 
 1336         return (0);
 1337 }
 1338 
 1339 static void
 1340 nicvf_release_net_interrupts(struct nicvf *nic)
 1341 {
 1342         struct resource *res;
 1343         int irq;
 1344         int err;
 1345 
 1346         for_each_cq_irq(irq) {
 1347                 res = nic->msix_entries[irq].irq_res;
 1348                 if (res == NULL)
 1349                         continue;
 1350                 /* Teardown active interrupts first */
 1351                 if (nic->msix_entries[irq].handle != NULL) {
 1352                         err = bus_teardown_intr(nic->dev,
 1353                             nic->msix_entries[irq].irq_res,
 1354                             nic->msix_entries[irq].handle);
 1355                         KASSERT(err == 0,
 1356                             ("ERROR: Unable to teardown CQ interrupt %d",
 1357                             (irq - NICVF_INTR_ID_CQ)));
 1358                         if (err != 0)
 1359                                 continue;
 1360                 }
 1361 
 1362                 /* Release resource */
 1363                 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res),
 1364                     res);
 1365                 nic->msix_entries[irq].irq_res = NULL;
 1366         }
 1367 
 1368         for_each_rbdr_irq(irq) {
 1369                 res = nic->msix_entries[irq].irq_res;
 1370                 if (res == NULL)
 1371                         continue;
 1372                 /* Teardown active interrupts first */
 1373                 if (nic->msix_entries[irq].handle != NULL) {
 1374                         err = bus_teardown_intr(nic->dev,
 1375                             nic->msix_entries[irq].irq_res,
 1376                             nic->msix_entries[irq].handle);
 1377                         KASSERT(err == 0,
 1378                             ("ERROR: Unable to teardown RDBR interrupt %d",
 1379                             (irq - NICVF_INTR_ID_RBDR)));
 1380                         if (err != 0)
 1381                                 continue;
 1382                 }
 1383 
 1384                 /* Release resource */
 1385                 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res),
 1386                     res);
 1387                 nic->msix_entries[irq].irq_res = NULL;
 1388         }
 1389 
 1390         irq = NICVF_INTR_ID_QS_ERR;
 1391         res = nic->msix_entries[irq].irq_res;
 1392         if (res != NULL) {
 1393                 /* Teardown active interrupts first */
 1394                 if (nic->msix_entries[irq].handle != NULL) {
 1395                         err = bus_teardown_intr(nic->dev,
 1396                             nic->msix_entries[irq].irq_res,
 1397                             nic->msix_entries[irq].handle);
 1398                         KASSERT(err == 0,
 1399                             ("ERROR: Unable to teardown QS Error interrupt %d",
 1400                             irq));
 1401                         if (err != 0)
 1402                                 return;
 1403                 }
 1404 
 1405                 /* Release resource */
 1406                 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res),
 1407                     res);
 1408                 nic->msix_entries[irq].irq_res = NULL;
 1409         }
 1410 }
 1411 
 1412 static int
 1413 nicvf_allocate_net_interrupts(struct nicvf *nic)
 1414 {
 1415         u_int cpuid;
 1416         int irq, rid;
 1417         int qidx;
 1418         int ret = 0;
 1419 
 1420         /* MSI-X must be configured by now */
 1421         if (!nic->msix_enabled) {
 1422                 device_printf(nic->dev, "Cannot alloacte queue interrups. "
 1423                     "MSI-X interrupts disabled.\n");
 1424                 return (ENXIO);
 1425         }
 1426 
 1427         /* Register CQ interrupts */
 1428         for_each_cq_irq(irq) {
 1429                 if (irq >= (NICVF_INTR_ID_CQ + nic->qs->cq_cnt))
 1430                         break;
 1431 
 1432                 qidx = irq - NICVF_INTR_ID_CQ;
 1433                 rid = irq + 1;
 1434                 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev,
 1435                     SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE));
 1436                 if (nic->msix_entries[irq].irq_res == NULL) {
 1437                         device_printf(nic->dev,
 1438                             "Could not allocate CQ interrupt %d for VF%d\n",
 1439                             (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev));
 1440                         ret = ENXIO;
 1441                         goto error;
 1442                 }
 1443                 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res,
 1444                     (INTR_MPSAFE | INTR_TYPE_NET), nicvf_intr_handler,
 1445                     NULL, &nic->qs->cq[qidx], &nic->msix_entries[irq].handle);
 1446                 if (ret != 0) {
 1447                         device_printf(nic->dev,
 1448                             "Could not setup CQ interrupt %d for VF%d\n",
 1449                             (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev));
 1450                         goto error;
 1451                 }
 1452                 cpuid = (device_get_unit(nic->dev) * CMP_QUEUE_CNT) + qidx;
 1453                 cpuid %= mp_ncpus;
 1454                 /*
 1455                  * Save CPU ID for later use when system-wide RSS is enabled.
 1456                  * It will be used to pit the CQ task to the same CPU that got
 1457                  * interrupted.
 1458                  */
 1459                 nic->qs->cq[qidx].cmp_cpuid = cpuid;
 1460                 if (bootverbose) {
 1461                         device_printf(nic->dev, "bind CQ%d IRQ to CPU%d\n",
 1462                             qidx, cpuid);
 1463                 }
 1464                 /* Bind interrupts to the given CPU */
 1465                 bus_bind_intr(nic->dev, nic->msix_entries[irq].irq_res, cpuid);
 1466         }
 1467 
 1468         /* Register RBDR interrupt */
 1469         for_each_rbdr_irq(irq) {
 1470                 if (irq >= (NICVF_INTR_ID_RBDR + nic->qs->rbdr_cnt))
 1471                         break;
 1472 
 1473                 rid = irq + 1;
 1474                 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev,
 1475                     SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE));
 1476                 if (nic->msix_entries[irq].irq_res == NULL) {
 1477                         device_printf(nic->dev,
 1478                             "Could not allocate RBDR interrupt %d for VF%d\n",
 1479                             (irq - NICVF_INTR_ID_RBDR),
 1480                             device_get_unit(nic->dev));
 1481                         ret = ENXIO;
 1482                         goto error;
 1483                 }
 1484                 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res,
 1485                     (INTR_MPSAFE | INTR_TYPE_NET), NULL,
 1486                     nicvf_rbdr_intr_handler, nic,
 1487                     &nic->msix_entries[irq].handle);
 1488                 if (ret != 0) {
 1489                         device_printf(nic->dev,
 1490                             "Could not setup RBDR interrupt %d for VF%d\n",
 1491                             (irq - NICVF_INTR_ID_RBDR),
 1492                             device_get_unit(nic->dev));
 1493                         goto error;
 1494                 }
 1495         }
 1496 
 1497         /* Register QS error interrupt */
 1498         irq = NICVF_INTR_ID_QS_ERR;
 1499         rid = irq + 1;
 1500         nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev,
 1501             SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE));
 1502         if (nic->msix_entries[irq].irq_res == NULL) {
 1503                 device_printf(nic->dev,
 1504                     "Could not allocate QS Error interrupt for VF%d\n",
 1505                     device_get_unit(nic->dev));
 1506                 ret = ENXIO;
 1507                 goto error;
 1508         }
 1509         ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res,
 1510             (INTR_MPSAFE | INTR_TYPE_NET), NULL, nicvf_qs_err_intr_handler,
 1511             nic, &nic->msix_entries[irq].handle);
 1512         if (ret != 0) {
 1513                 device_printf(nic->dev,
 1514                     "Could not setup QS Error interrupt for VF%d\n",
 1515                     device_get_unit(nic->dev));
 1516                 goto error;
 1517         }
 1518 
 1519         return (0);
 1520 error:
 1521         nicvf_release_net_interrupts(nic);
 1522         return (ret);
 1523 }
 1524 
 1525 static int
 1526 nicvf_stop_locked(struct nicvf *nic)
 1527 {
 1528         struct ifnet *ifp;
 1529         int qidx;
 1530         struct queue_set *qs = nic->qs;
 1531         union nic_mbx mbx = {};
 1532 
 1533         NICVF_CORE_LOCK_ASSERT(nic);
 1534         /* Stop callout. Can block here since holding SX lock */
 1535         callout_drain(&nic->stats_callout);
 1536 
 1537         ifp = nic->ifp;
 1538 
 1539         mbx.msg.msg = NIC_MBOX_MSG_SHUTDOWN;
 1540         nicvf_send_msg_to_pf(nic, &mbx);
 1541 
 1542         /* Disable RBDR & QS error interrupts */
 1543         for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) {
 1544                 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
 1545                 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
 1546         }
 1547         nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
 1548         nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
 1549 
 1550         /* Deactivate network interface */
 1551         if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
 1552 
 1553         /* Free resources */
 1554         nicvf_config_data_transfer(nic, FALSE);
 1555 
 1556         /* Disable HW Qset */
 1557         nicvf_qset_config(nic, FALSE);
 1558 
 1559         /* disable mailbox interrupt */
 1560         nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
 1561 
 1562         return (0);
 1563 }
 1564 
 1565 static void
 1566 nicvf_update_stats(struct nicvf *nic)
 1567 {
 1568         int qidx;
 1569         struct nicvf_hw_stats *stats = &nic->hw_stats;
 1570         struct nicvf_drv_stats *drv_stats = &nic->drv_stats;
 1571         struct queue_set *qs = nic->qs;
 1572 
 1573 #define GET_RX_STATS(reg) \
 1574     nicvf_reg_read(nic, NIC_VNIC_RX_STAT_0_13 | ((reg) << 3))
 1575 #define GET_TX_STATS(reg) \
 1576     nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | ((reg) << 3))
 1577 
 1578         stats->rx_bytes = GET_RX_STATS(RX_OCTS);
 1579         stats->rx_ucast_frames = GET_RX_STATS(RX_UCAST);
 1580         stats->rx_bcast_frames = GET_RX_STATS(RX_BCAST);
 1581         stats->rx_mcast_frames = GET_RX_STATS(RX_MCAST);
 1582         stats->rx_fcs_errors = GET_RX_STATS(RX_FCS);
 1583         stats->rx_l2_errors = GET_RX_STATS(RX_L2ERR);
 1584         stats->rx_drop_red = GET_RX_STATS(RX_RED);
 1585         stats->rx_drop_red_bytes = GET_RX_STATS(RX_RED_OCTS);
 1586         stats->rx_drop_overrun = GET_RX_STATS(RX_ORUN);
 1587         stats->rx_drop_overrun_bytes = GET_RX_STATS(RX_ORUN_OCTS);
 1588         stats->rx_drop_bcast = GET_RX_STATS(RX_DRP_BCAST);
 1589         stats->rx_drop_mcast = GET_RX_STATS(RX_DRP_MCAST);
 1590         stats->rx_drop_l3_bcast = GET_RX_STATS(RX_DRP_L3BCAST);
 1591         stats->rx_drop_l3_mcast = GET_RX_STATS(RX_DRP_L3MCAST);
 1592 
 1593         stats->tx_bytes_ok = GET_TX_STATS(TX_OCTS);
 1594         stats->tx_ucast_frames_ok = GET_TX_STATS(TX_UCAST);
 1595         stats->tx_bcast_frames_ok = GET_TX_STATS(TX_BCAST);
 1596         stats->tx_mcast_frames_ok = GET_TX_STATS(TX_MCAST);
 1597         stats->tx_drops = GET_TX_STATS(TX_DROP);
 1598 
 1599         drv_stats->tx_frames_ok = stats->tx_ucast_frames_ok +
 1600             stats->tx_bcast_frames_ok + stats->tx_mcast_frames_ok;
 1601         drv_stats->rx_drops = stats->rx_drop_red + stats->rx_drop_overrun;
 1602         drv_stats->tx_drops = stats->tx_drops;
 1603 
 1604         /* Update RQ and SQ stats */
 1605         for (qidx = 0; qidx < qs->rq_cnt; qidx++)
 1606                 nicvf_update_rq_stats(nic, qidx);
 1607         for (qidx = 0; qidx < qs->sq_cnt; qidx++)
 1608                 nicvf_update_sq_stats(nic, qidx);
 1609 }
 1610 
 1611 static void
 1612 nicvf_tick_stats(void *arg)
 1613 {
 1614         struct nicvf *nic;
 1615 
 1616         nic = (struct nicvf *)arg;
 1617 
 1618         /* Read the statistics */
 1619         nicvf_update_stats(nic);
 1620 
 1621         callout_reset(&nic->stats_callout, hz, nicvf_tick_stats, nic);
 1622 }
Cache object: 3fd538e55b1824768c8b50cccd903c8f
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/vnic/nicvf_main.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/vnic/nicvf_main.c
