nicvf_main.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*
    2  * Copyright (C) 2015 Cavium Inc.
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  *
   14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24  * SUCH DAMAGE.
   25  *
   26  * $FreeBSD$
   27  *
   28  */
   29 #include <sys/cdefs.h>
   30 __FBSDID("$FreeBSD$");
   31 
   32 #include "opt_inet.h"
   33 #include "opt_inet6.h"
   34 
   35 #include <sys/param.h>
   36 #include <sys/systm.h>
   37 #include <sys/bitset.h>
   38 #include <sys/bitstring.h>
   39 #include <sys/bus.h>
   40 #include <sys/endian.h>
   41 #include <sys/kernel.h>
   42 #include <sys/malloc.h>
   43 #include <sys/mbuf.h>
   44 #include <sys/module.h>
   45 #include <sys/rman.h>
   46 #include <sys/pciio.h>
   47 #include <sys/pcpu.h>
   48 #include <sys/proc.h>
   49 #include <sys/socket.h>
   50 #include <sys/sockio.h>
   51 #include <sys/stdatomic.h>
   52 #include <sys/cpuset.h>
   53 #include <sys/lock.h>
   54 #include <sys/mutex.h>
   55 #include <sys/smp.h>
   56 #include <sys/taskqueue.h>
   57 
   58 #include <net/bpf.h>
   59 #include <net/ethernet.h>
   60 #include <net/if.h>
   61 #include <net/if_var.h>
   62 #include <net/if_arp.h>
   63 #include <net/if_dl.h>
   64 #include <net/if_media.h>
   65 #include <net/if_types.h>
   66 #include <net/if_vlan_var.h>
   67 
   68 #include <netinet/in.h>
   69 #include <netinet/ip.h>
   70 #include <netinet/if_ether.h>
   71 #include <netinet/tcp_lro.h>
   72 
   73 #include <dev/pci/pcireg.h>
   74 #include <dev/pci/pcivar.h>
   75 
   76 #include <sys/dnv.h>
   77 #include <sys/nv.h>
   78 #include <sys/iov_schema.h>
   79 
   80 #include <machine/bus.h>
   81 
   82 #include "thunder_bgx.h"
   83 #include "nic_reg.h"
   84 #include "nic.h"
   85 #include "nicvf_queues.h"
   86 
   87 #define VNIC_VF_DEVSTR          "Cavium Thunder NIC Virtual Function Driver"
   88 
   89 #define VNIC_VF_REG_RID         PCIR_BAR(PCI_CFG_REG_BAR_NUM)
   90 
   91 /* Lock for core interface settings */
   92 #define NICVF_CORE_LOCK_INIT(nic)                               \
   93     sx_init(&(nic)->core_sx, device_get_nameunit((nic)->dev))
   94 
   95 #define NICVF_CORE_LOCK_DESTROY(nic)                            \
   96     sx_destroy(&(nic)->core_sx)
   97 
   98 #define NICVF_CORE_LOCK(nic)            sx_xlock(&(nic)->core_sx)
   99 #define NICVF_CORE_UNLOCK(nic)          sx_xunlock(&(nic)->core_sx)
  100 
  101 #define NICVF_CORE_LOCK_ASSERT(nic)     sx_assert(&(nic)->core_sx, SA_XLOCKED)
  102 
  103 #define SPEED_10        10
  104 #define SPEED_100       100
  105 #define SPEED_1000      1000
  106 #define SPEED_10000     10000
  107 #define SPEED_40000     40000
  108 
  109 MALLOC_DEFINE(M_NICVF, "nicvf", "ThunderX VNIC VF dynamic memory");
  110 
  111 static int nicvf_probe(device_t);
  112 static int nicvf_attach(device_t);
  113 static int nicvf_detach(device_t);
  114 
  115 static device_method_t nicvf_methods[] = {
  116         /* Device interface */
  117         DEVMETHOD(device_probe,         nicvf_probe),
  118         DEVMETHOD(device_attach,        nicvf_attach),
  119         DEVMETHOD(device_detach,        nicvf_detach),
  120 
  121         DEVMETHOD_END,
  122 };
  123 
  124 static driver_t nicvf_driver = {
  125         "vnic",
  126         nicvf_methods,
  127         sizeof(struct nicvf),
  128 };
  129 
  130 static devclass_t nicvf_devclass;
  131 
  132 DRIVER_MODULE(vnicvf, pci, nicvf_driver, nicvf_devclass, 0, 0);
  133 MODULE_VERSION(vnicvf, 1);
  134 MODULE_DEPEND(vnicvf, pci, 1, 1, 1);
  135 MODULE_DEPEND(vnicvf, ether, 1, 1, 1);
  136 MODULE_DEPEND(vnicvf, vnicpf, 1, 1, 1);
  137 
  138 static int nicvf_allocate_misc_interrupt(struct nicvf *);
  139 static int nicvf_enable_misc_interrupt(struct nicvf *);
  140 static int nicvf_allocate_net_interrupts(struct nicvf *);
  141 static void nicvf_release_all_interrupts(struct nicvf *);
  142 static int nicvf_update_hw_max_frs(struct nicvf *, int);
  143 static int nicvf_hw_set_mac_addr(struct nicvf *, uint8_t *);
  144 static void nicvf_config_cpi(struct nicvf *);
  145 static int nicvf_rss_init(struct nicvf *);
  146 static int nicvf_init_resources(struct nicvf *);
  147 
  148 static int nicvf_setup_ifnet(struct nicvf *);
  149 static int nicvf_setup_ifmedia(struct nicvf *);
  150 static void nicvf_hw_addr_random(uint8_t *);
  151 
  152 static int nicvf_if_ioctl(struct ifnet *, u_long, caddr_t);
  153 static void nicvf_if_init(void *);
  154 static void nicvf_if_init_locked(struct nicvf *);
  155 static int nicvf_if_transmit(struct ifnet *, struct mbuf *);
  156 static void nicvf_if_qflush(struct ifnet *);
  157 static uint64_t nicvf_if_getcounter(struct ifnet *, ift_counter);
  158 
  159 static int nicvf_stop_locked(struct nicvf *);
  160 
  161 static void nicvf_media_status(struct ifnet *, struct ifmediareq *);
  162 static int nicvf_media_change(struct ifnet *);
  163 
  164 static void nicvf_tick_stats(void *);
  165 
  166 static int
  167 nicvf_probe(device_t dev)
  168 {
  169         uint16_t vendor_id;
  170         uint16_t device_id;
  171 
  172         vendor_id = pci_get_vendor(dev);
  173         device_id = pci_get_device(dev);
  174 
  175         if (vendor_id != PCI_VENDOR_ID_CAVIUM)
  176                 return (ENXIO);
  177 
  178         if (device_id == PCI_DEVICE_ID_THUNDER_NIC_VF ||
  179             device_id == PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF) {
  180                 device_set_desc(dev, VNIC_VF_DEVSTR);
  181                 return (BUS_PROBE_DEFAULT);
  182         }
  183 
  184         return (ENXIO);
  185 }
  186 
  187 static int
  188 nicvf_attach(device_t dev)
  189 {
  190         struct nicvf *nic;
  191         int rid, qcount;
  192         int err = 0;
  193         uint8_t hwaddr[ETHER_ADDR_LEN];
  194         uint8_t zeromac[] = {[0 ... (ETHER_ADDR_LEN - 1)] = 0};
  195 
  196         nic = device_get_softc(dev);
  197         nic->dev = dev;
  198         nic->pnicvf = nic;
  199 
  200         NICVF_CORE_LOCK_INIT(nic);
  201         /* Enable HW TSO on Pass2 */
  202         if (!pass1_silicon(dev))
  203                 nic->hw_tso = TRUE;
  204 
  205         rid = VNIC_VF_REG_RID;
  206         nic->reg_base = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
  207             RF_ACTIVE);
  208         if (nic->reg_base == NULL) {
  209                 device_printf(dev, "Could not allocate registers memory\n");
  210                 return (ENXIO);
  211         }
  212 
  213         qcount = MAX_CMP_QUEUES_PER_QS;
  214         nic->max_queues = qcount;
  215 
  216         err = nicvf_set_qset_resources(nic);
  217         if (err != 0)
  218                 goto err_free_res;
  219 
  220         /* Check if PF is alive and get MAC address for this VF */
  221         err = nicvf_allocate_misc_interrupt(nic);
  222         if (err != 0)
  223                 goto err_free_res;
  224 
  225         NICVF_CORE_LOCK(nic);
  226         err = nicvf_enable_misc_interrupt(nic);
  227         NICVF_CORE_UNLOCK(nic);
  228         if (err != 0)
  229                 goto err_release_intr;
  230 
  231         err = nicvf_allocate_net_interrupts(nic);
  232         if (err != 0) {
  233                 device_printf(dev,
  234                     "Could not allocate network interface interrupts\n");
  235                 goto err_free_ifnet;
  236         }
  237 
  238         /* If no MAC address was obtained we generate random one */
  239         if (memcmp(nic->hwaddr, zeromac, ETHER_ADDR_LEN) == 0) {
  240                 nicvf_hw_addr_random(hwaddr);
  241                 memcpy(nic->hwaddr, hwaddr, ETHER_ADDR_LEN);
  242                 NICVF_CORE_LOCK(nic);
  243                 nicvf_hw_set_mac_addr(nic, hwaddr);
  244                 NICVF_CORE_UNLOCK(nic);
  245         }
  246 
  247         /* Configure CPI alorithm */
  248         nic->cpi_alg = CPI_ALG_NONE;
  249         NICVF_CORE_LOCK(nic);
  250         nicvf_config_cpi(nic);
  251         /* Configure receive side scaling */
  252         if (nic->qs->rq_cnt > 1)
  253                 nicvf_rss_init(nic);
  254         NICVF_CORE_UNLOCK(nic);
  255 
  256         err = nicvf_setup_ifnet(nic);
  257         if (err != 0) {
  258                 device_printf(dev, "Could not set-up ifnet\n");
  259                 goto err_release_intr;
  260         }
  261 
  262         err = nicvf_setup_ifmedia(nic);
  263         if (err != 0) {
  264                 device_printf(dev, "Could not set-up ifmedia\n");
  265                 goto err_free_ifnet;
  266         }
  267 
  268         mtx_init(&nic->stats_mtx, "VNIC stats", NULL, MTX_DEF);
  269         callout_init_mtx(&nic->stats_callout, &nic->stats_mtx, 0);
  270 
  271         ether_ifattach(nic->ifp, nic->hwaddr);
  272 
  273         return (0);
  274 
  275 err_free_ifnet:
  276         if_free(nic->ifp);
  277 err_release_intr:
  278         nicvf_release_all_interrupts(nic);
  279 err_free_res:
  280         bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(nic->reg_base),
  281             nic->reg_base);
  282 
  283         return (err);
  284 }
  285 
  286 static int
  287 nicvf_detach(device_t dev)
  288 {
  289         struct nicvf *nic;
  290 
  291         nic = device_get_softc(dev);
  292 
  293         NICVF_CORE_LOCK(nic);
  294         /* Shut down the port and release ring resources */
  295         nicvf_stop_locked(nic);
  296         /* Release stats lock */
  297         mtx_destroy(&nic->stats_mtx);
  298         /* Release interrupts */
  299         nicvf_release_all_interrupts(nic);
  300         /* Release memory resource */
  301         if (nic->reg_base != NULL) {
  302                 bus_release_resource(dev, SYS_RES_MEMORY,
  303                     rman_get_rid(nic->reg_base), nic->reg_base);
  304         }
  305 
  306         /* Remove all ifmedia configurations */
  307         ifmedia_removeall(&nic->if_media);
  308         /* Free this ifnet */
  309         if_free(nic->ifp);
  310         NICVF_CORE_UNLOCK(nic);
  311         /* Finally destroy the lock */
  312         NICVF_CORE_LOCK_DESTROY(nic);
  313 
  314         return (0);
  315 }
  316 
  317 static void
  318 nicvf_hw_addr_random(uint8_t *hwaddr)
  319 {
  320         uint32_t rnd;
  321         uint8_t addr[ETHER_ADDR_LEN];
  322 
  323         /*
  324          * Create randomized MAC address.
  325          * Set 'bsd' + random 24 low-order bits.
  326          */
  327         rnd = arc4random() & 0x00ffffff;
  328         addr[0] = 'b';
  329         addr[1] = 's';
  330         addr[2] = 'd';
  331         addr[3] = rnd >> 16;
  332         addr[4] = rnd >> 8;
  333         addr[5] = rnd >> 0;
  334 
  335         memcpy(hwaddr, addr, ETHER_ADDR_LEN);
  336 }
  337 
  338 static int
  339 nicvf_setup_ifnet(struct nicvf *nic)
  340 {
  341         struct ifnet *ifp;
  342 
  343         ifp = if_alloc(IFT_ETHER);
  344         if (ifp == NULL) {
  345                 device_printf(nic->dev, "Could not allocate ifnet structure\n");
  346                 return (ENOMEM);
  347         }
  348 
  349         nic->ifp = ifp;
  350 
  351         if_setsoftc(ifp, nic);
  352         if_initname(ifp, device_get_name(nic->dev), device_get_unit(nic->dev));
  353         if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
  354 
  355         if_settransmitfn(ifp, nicvf_if_transmit);
  356         if_setqflushfn(ifp, nicvf_if_qflush);
  357         if_setioctlfn(ifp, nicvf_if_ioctl);
  358         if_setinitfn(ifp, nicvf_if_init);
  359         if_setgetcounterfn(ifp, nicvf_if_getcounter);
  360 
  361         if_setmtu(ifp, ETHERMTU);
  362 
  363         /* Reset caps */
  364         if_setcapabilities(ifp, 0);
  365 
  366         /* Set the default values */
  367         if_setcapabilitiesbit(ifp, IFCAP_VLAN_MTU | IFCAP_JUMBO_MTU, 0);
  368         if_setcapabilitiesbit(ifp, IFCAP_LRO, 0);
  369         if (nic->hw_tso) {
  370                 /* TSO */
  371                 if_setcapabilitiesbit(ifp, IFCAP_TSO4, 0);
  372                 /* TSO parameters */
  373                 if_sethwtsomax(ifp, NICVF_TSO_MAXSIZE);
  374                 if_sethwtsomaxsegcount(ifp, NICVF_TSO_NSEGS);
  375                 if_sethwtsomaxsegsize(ifp, MCLBYTES);
  376         }
  377         /* IP/TCP/UDP HW checksums */
  378         if_setcapabilitiesbit(ifp, IFCAP_HWCSUM, 0);
  379         if_setcapabilitiesbit(ifp, IFCAP_HWSTATS, 0);
  380         /*
  381          * HW offload enable
  382          */
  383         if_clearhwassist(ifp);
  384         if_sethwassistbits(ifp, (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP), 0);
  385         if (nic->hw_tso)
  386                 if_sethwassistbits(ifp, (CSUM_TSO), 0);
  387         if_setcapenable(ifp, if_getcapabilities(ifp));
  388 
  389         return (0);
  390 }
  391 
  392 static int
  393 nicvf_setup_ifmedia(struct nicvf *nic)
  394 {
  395 
  396         ifmedia_init(&nic->if_media, IFM_IMASK, nicvf_media_change,
  397             nicvf_media_status);
  398 
  399         /*
  400          * Advertise availability of all possible connection types,
  401          * even though not all are possible at the same time.
  402          */
  403 
  404         ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_10_T | IFM_FDX),
  405             0, NULL);
  406         ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_100_TX | IFM_FDX),
  407             0, NULL);
  408         ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_1000_T | IFM_FDX),
  409             0, NULL);
  410         ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_10G_SR | IFM_FDX),
  411             0, NULL);
  412         ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_40G_CR4 | IFM_FDX),
  413             0, NULL);
  414         ifmedia_add(&nic->if_media, (IFM_ETHER | IFM_AUTO | IFM_FDX),
  415             0, NULL);
  416 
  417         ifmedia_set(&nic->if_media, (IFM_ETHER | IFM_AUTO | IFM_FDX));
  418 
  419         return (0);
  420 }
  421 
  422 static int
  423 nicvf_if_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
  424 {
  425         struct nicvf *nic;
  426         struct rcv_queue *rq;
  427         struct ifreq *ifr;
  428         uint32_t flags;
  429         int mask, err;
  430         int rq_idx;
  431 #if defined(INET) || defined(INET6)
  432         struct ifaddr *ifa;
  433         boolean_t avoid_reset = FALSE;
  434 #endif
  435 
  436         nic = if_getsoftc(ifp);
  437         ifr = (struct ifreq *)data;
  438 #if defined(INET) || defined(INET6)
  439         ifa = (struct ifaddr *)data;
  440 #endif
  441         err = 0;
  442         switch (cmd) {
  443         case SIOCSIFADDR:
  444 #ifdef INET
  445                 if (ifa->ifa_addr->sa_family == AF_INET)
  446                         avoid_reset = TRUE;
  447 #endif
  448 #ifdef INET6
  449                 if (ifa->ifa_addr->sa_family == AF_INET6)
  450                         avoid_reset = TRUE;
  451 #endif
  452 
  453 #if defined(INET) || defined(INET6)
  454                 /* Avoid reinitialization unless it's necessary */
  455                 if (avoid_reset) {
  456                         if_setflagbits(ifp, IFF_UP, 0);
  457                         if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
  458                                 nicvf_if_init(nic);
  459 #ifdef INET
  460                         if (!(if_getflags(ifp) & IFF_NOARP))
  461                                 arp_ifinit(ifp, ifa);
  462 #endif
  463 
  464                         return (0);
  465                 }
  466 #endif
  467                 err = ether_ioctl(ifp, cmd, data);
  468                 break;
  469         case SIOCSIFMTU:
  470                 if (ifr->ifr_mtu < NIC_HW_MIN_FRS ||
  471                     ifr->ifr_mtu > NIC_HW_MAX_FRS) {
  472                         err = EINVAL;
  473                 } else {
  474                         NICVF_CORE_LOCK(nic);
  475                         err = nicvf_update_hw_max_frs(nic, ifr->ifr_mtu);
  476                         if (err == 0)
  477                                 if_setmtu(ifp, ifr->ifr_mtu);
  478                         NICVF_CORE_UNLOCK(nic);
  479                 }
  480                 break;
  481         case SIOCSIFFLAGS:
  482                 NICVF_CORE_LOCK(nic);
  483                 flags = if_getflags(ifp);
  484                 if (flags & IFF_UP) {
  485                         if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
  486                                 if ((flags ^ nic->if_flags) & IFF_PROMISC) {
  487                                         /* Change promiscous mode */
  488 #if 0
  489                                         /* ARM64TODO */
  490                                         nicvf_set_promiscous(nic);
  491 #endif
  492                                 }
  493 
  494                                 if ((flags ^ nic->if_flags) & IFF_ALLMULTI) {
  495                                         /* Change multicasting settings */
  496 #if 0
  497                                         /* ARM64TODO */
  498                                         nicvf_set_multicast(nic);
  499 #endif
  500                                 }
  501                         } else {
  502                                 nicvf_if_init_locked(nic);
  503                         }
  504                 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
  505                         nicvf_stop_locked(nic);
  506 
  507                 nic->if_flags = flags;
  508                 NICVF_CORE_UNLOCK(nic);
  509                 break;
  510 
  511         case SIOCADDMULTI:
  512         case SIOCDELMULTI:
  513                 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
  514 #if 0
  515                         NICVF_CORE_LOCK(nic);
  516                         /* ARM64TODO */
  517                         nicvf_set_multicast(nic);
  518                         NICVF_CORE_UNLOCK(nic);
  519 #endif
  520                 }
  521                 break;
  522 
  523         case SIOCSIFMEDIA:
  524         case SIOCGIFMEDIA:
  525                 err = ifmedia_ioctl(ifp, ifr, &nic->if_media, cmd);
  526                 break;
  527 
  528         case SIOCSIFCAP:
  529                 mask = if_getcapenable(ifp) ^ ifr->ifr_reqcap;
  530                 if (mask & IFCAP_VLAN_MTU) {
  531                         /* No work to do except acknowledge the change took. */
  532                         if_togglecapenable(ifp, IFCAP_VLAN_MTU);
  533                 }
  534                 if (mask & IFCAP_TXCSUM)
  535                         if_togglecapenable(ifp, IFCAP_TXCSUM);
  536                 if (mask & IFCAP_RXCSUM)
  537                         if_togglecapenable(ifp, IFCAP_RXCSUM);
  538                 if ((mask & IFCAP_TSO4) && nic->hw_tso)
  539                         if_togglecapenable(ifp, IFCAP_TSO4);
  540                 if (mask & IFCAP_LRO) {
  541                         /*
  542                          * Lock the driver for a moment to avoid
  543                          * mismatch in per-queue settings.
  544                          */
  545                         NICVF_CORE_LOCK(nic);
  546                         if_togglecapenable(ifp, IFCAP_LRO);
  547                         if ((if_getdrvflags(nic->ifp) & IFF_DRV_RUNNING) != 0) {
  548                                 /*
  549                                  * Now disable LRO for subsequent packets.
  550                                  * Atomicity of this change is not necessary
  551                                  * as we don't need precise toggle of this
  552                                  * feature for all threads processing the
  553                                  * completion queue.
  554                                  */
  555                                 for (rq_idx = 0;
  556                                     rq_idx < nic->qs->rq_cnt; rq_idx++) {
  557                                         rq = &nic->qs->rq[rq_idx];
  558                                         rq->lro_enabled = !rq->lro_enabled;
  559                                 }
  560                         }
  561                         NICVF_CORE_UNLOCK(nic);
  562                 }
  563 
  564                 break;
  565 
  566         default:
  567                 err = ether_ioctl(ifp, cmd, data);
  568                 break;
  569         }
  570 
  571         return (err);
  572 }
  573 
  574 static void
  575 nicvf_if_init_locked(struct nicvf *nic)
  576 {
  577         struct queue_set *qs = nic->qs;
  578         struct ifnet *ifp;
  579         int qidx;
  580         int err;
  581         caddr_t if_addr;
  582 
  583         NICVF_CORE_LOCK_ASSERT(nic);
  584         ifp = nic->ifp;
  585 
  586         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0)
  587                 nicvf_stop_locked(nic);
  588 
  589         err = nicvf_enable_misc_interrupt(nic);
  590         if (err != 0) {
  591                 if_printf(ifp, "Could not reenable Mbox interrupt\n");
  592                 return;
  593         }
  594 
  595         /* Get the latest MAC address */
  596         if_addr = if_getlladdr(ifp);
  597         /* Update MAC address if changed */
  598         if (memcmp(nic->hwaddr, if_addr, ETHER_ADDR_LEN) != 0) {
  599                 memcpy(nic->hwaddr, if_addr, ETHER_ADDR_LEN);
  600                 nicvf_hw_set_mac_addr(nic, if_addr);
  601         }
  602 
  603         /* Initialize the queues */
  604         err = nicvf_init_resources(nic);
  605         if (err != 0)
  606                 goto error;
  607 
  608         /* Make sure queue initialization is written */
  609         wmb();
  610 
  611         nicvf_reg_write(nic, NIC_VF_INT, ~0UL);
  612         /* Enable Qset err interrupt */
  613         nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0);
  614 
  615         /* Enable completion queue interrupt */
  616         for (qidx = 0; qidx < qs->cq_cnt; qidx++)
  617                 nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx);
  618 
  619         /* Enable RBDR threshold interrupt */
  620         for (qidx = 0; qidx < qs->rbdr_cnt; qidx++)
  621                 nicvf_enable_intr(nic, NICVF_INTR_RBDR, qidx);
  622 
  623         nic->drv_stats.txq_stop = 0;
  624         nic->drv_stats.txq_wake = 0;
  625 
  626         /* Activate network interface */
  627         if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
  628 
  629         /* Schedule callout to update stats */
  630         callout_reset(&nic->stats_callout, hz, nicvf_tick_stats, nic);
  631 
  632         return;
  633 
  634 error:
  635         /* Something went very wrong. Disable this ifnet for good */
  636         if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
  637 }
  638 
  639 static void
  640 nicvf_if_init(void *if_softc)
  641 {
  642         struct nicvf *nic = if_softc;
  643 
  644         NICVF_CORE_LOCK(nic);
  645         nicvf_if_init_locked(nic);
  646         NICVF_CORE_UNLOCK(nic);
  647 }
  648 
  649 static int
  650 nicvf_if_transmit(struct ifnet *ifp, struct mbuf *mbuf)
  651 {
  652         struct nicvf *nic = if_getsoftc(ifp);
  653         struct queue_set *qs = nic->qs;
  654         struct snd_queue *sq;
  655         struct mbuf *mtmp;
  656         int qidx;
  657         int err = 0;
  658 
  659         if (__predict_false(qs == NULL)) {
  660                 panic("%s: missing queue set for %s", __func__,
  661                     device_get_nameunit(nic->dev));
  662         }
  663 
  664         /* Select queue */
  665         if (M_HASHTYPE_GET(mbuf) != M_HASHTYPE_NONE)
  666                 qidx = mbuf->m_pkthdr.flowid % qs->sq_cnt;
  667         else
  668                 qidx = curcpu % qs->sq_cnt;
  669 
  670         sq = &qs->sq[qidx];
  671 
  672         if (mbuf->m_next != NULL &&
  673             (mbuf->m_pkthdr.csum_flags &
  674             (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP)) != 0) {
  675                 if (M_WRITABLE(mbuf) == 0) {
  676                         mtmp = m_dup(mbuf, M_NOWAIT);
  677                         m_freem(mbuf);
  678                         if (mtmp == NULL)
  679                                 return (ENOBUFS);
  680                         mbuf = mtmp;
  681                 }
  682         }
  683 
  684         err = drbr_enqueue(ifp, sq->br, mbuf);
  685         if (((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
  686             IFF_DRV_RUNNING) || !nic->link_up || (err != 0)) {
  687                 /*
  688                  * Try to enqueue packet to the ring buffer.
  689                  * If the driver is not active, link down or enqueue operation
  690                  * failed, return with the appropriate error code.
  691                  */
  692                 return (err);
  693         }
  694 
  695         if (NICVF_TX_TRYLOCK(sq) != 0) {
  696                 err = nicvf_xmit_locked(sq);
  697                 NICVF_TX_UNLOCK(sq);
  698                 return (err);
  699         } else
  700                 taskqueue_enqueue(sq->snd_taskq, &sq->snd_task);
  701 
  702         return (0);
  703 }
  704 
  705 static void
  706 nicvf_if_qflush(struct ifnet *ifp)
  707 {
  708         struct nicvf *nic;
  709         struct queue_set *qs;
  710         struct snd_queue *sq;
  711         struct mbuf *mbuf;
  712         size_t idx;
  713 
  714         nic = if_getsoftc(ifp);
  715         qs = nic->qs;
  716 
  717         for (idx = 0; idx < qs->sq_cnt; idx++) {
  718                 sq = &qs->sq[idx];
  719                 NICVF_TX_LOCK(sq);
  720                 while ((mbuf = buf_ring_dequeue_sc(sq->br)) != NULL)
  721                         m_freem(mbuf);
  722                 NICVF_TX_UNLOCK(sq);
  723         }
  724         if_qflush(ifp);
  725 }
  726 
  727 static uint64_t
  728 nicvf_if_getcounter(struct ifnet *ifp, ift_counter cnt)
  729 {
  730         struct nicvf *nic;
  731         struct nicvf_hw_stats *hw_stats;
  732         struct nicvf_drv_stats *drv_stats;
  733 
  734         nic = if_getsoftc(ifp);
  735         hw_stats = &nic->hw_stats;
  736         drv_stats = &nic->drv_stats;
  737 
  738         switch (cnt) {
  739         case IFCOUNTER_IPACKETS:
  740                 return (drv_stats->rx_frames_ok);
  741         case IFCOUNTER_OPACKETS:
  742                 return (drv_stats->tx_frames_ok);
  743         case IFCOUNTER_IBYTES:
  744                 return (hw_stats->rx_bytes);
  745         case IFCOUNTER_OBYTES:
  746                 return (hw_stats->tx_bytes_ok);
  747         case IFCOUNTER_IMCASTS:
  748                 return (hw_stats->rx_mcast_frames);
  749         case IFCOUNTER_COLLISIONS:
  750                 return (0);
  751         case IFCOUNTER_IQDROPS:
  752                 return (drv_stats->rx_drops);
  753         case IFCOUNTER_OQDROPS:
  754                 return (drv_stats->tx_drops);
  755         default:
  756                 return (if_get_counter_default(ifp, cnt));
  757         }
  758 
  759 }
  760 
  761 static void
  762 nicvf_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
  763 {
  764         struct nicvf *nic = if_getsoftc(ifp);
  765 
  766         NICVF_CORE_LOCK(nic);
  767 
  768         ifmr->ifm_status = IFM_AVALID;
  769         ifmr->ifm_active = IFM_ETHER;
  770 
  771         if (nic->link_up) {
  772                 /* Device attached to working network */
  773                 ifmr->ifm_status |= IFM_ACTIVE;
  774         }
  775 
  776         switch (nic->speed) {
  777         case SPEED_10:
  778                 ifmr->ifm_active |= IFM_10_T;
  779                 break;
  780         case SPEED_100:
  781                 ifmr->ifm_active |= IFM_100_TX;
  782                 break;
  783         case SPEED_1000:
  784                 ifmr->ifm_active |= IFM_1000_T;
  785                 break;
  786         case SPEED_10000:
  787                 ifmr->ifm_active |= IFM_10G_SR;
  788                 break;
  789         case SPEED_40000:
  790                 ifmr->ifm_active |= IFM_40G_CR4;
  791                 break;
  792         default:
  793                 ifmr->ifm_active |= IFM_AUTO;
  794                 break;
  795         }
  796 
  797         if (nic->duplex)
  798                 ifmr->ifm_active |= IFM_FDX;
  799         else
  800                 ifmr->ifm_active |= IFM_HDX;
  801 
  802         NICVF_CORE_UNLOCK(nic);
  803 }
  804 
  805 static int
  806 nicvf_media_change(struct ifnet *ifp __unused)
  807 {
  808 
  809         return (0);
  810 }
  811 
  812 /* Register read/write APIs */
  813 void
  814 nicvf_reg_write(struct nicvf *nic, bus_space_handle_t offset, uint64_t val)
  815 {
  816 
  817         bus_write_8(nic->reg_base, offset, val);
  818 }
  819 
  820 uint64_t
  821 nicvf_reg_read(struct nicvf *nic, uint64_t offset)
  822 {
  823 
  824         return (bus_read_8(nic->reg_base, offset));
  825 }
  826 
  827 void
  828 nicvf_queue_reg_write(struct nicvf *nic, bus_space_handle_t offset,
  829     uint64_t qidx, uint64_t val)
  830 {
  831 
  832         bus_write_8(nic->reg_base, offset + (qidx << NIC_Q_NUM_SHIFT), val);
  833 }
  834 
  835 uint64_t
  836 nicvf_queue_reg_read(struct nicvf *nic, bus_space_handle_t offset,
  837     uint64_t qidx)
  838 {
  839 
  840         return (bus_read_8(nic->reg_base, offset + (qidx << NIC_Q_NUM_SHIFT)));
  841 }
  842 
  843 /* VF -> PF mailbox communication */
  844 static void
  845 nicvf_write_to_mbx(struct nicvf *nic, union nic_mbx *mbx)
  846 {
  847         uint64_t *msg = (uint64_t *)mbx;
  848 
  849         nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 0, msg[0]);
  850         nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 8, msg[1]);
  851 }
  852 
  853 int
  854 nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx)
  855 {
  856         int timeout = NIC_MBOX_MSG_TIMEOUT * 10;
  857         int sleep = 2;
  858 
  859         NICVF_CORE_LOCK_ASSERT(nic);
  860 
  861         nic->pf_acked = FALSE;
  862         nic->pf_nacked = FALSE;
  863 
  864         nicvf_write_to_mbx(nic, mbx);
  865 
  866         /* Wait for previous message to be acked, timeout 2sec */
  867         while (!nic->pf_acked) {
  868                 if (nic->pf_nacked)
  869                         return (EINVAL);
  870 
  871                 DELAY(sleep * 1000);
  872 
  873                 if (nic->pf_acked)
  874                         break;
  875                 timeout -= sleep;
  876                 if (!timeout) {
  877                         device_printf(nic->dev,
  878                                    "PF didn't ack to mbox msg %d from VF%d\n",
  879                                    (mbx->msg.msg & 0xFF), nic->vf_id);
  880 
  881                         return (EBUSY);
  882                 }
  883         }
  884         return (0);
  885 }
  886 
  887 /*
  888  * Checks if VF is able to comminicate with PF
  889  * and also gets the VNIC number this VF is associated to.
  890  */
  891 static int
  892 nicvf_check_pf_ready(struct nicvf *nic)
  893 {
  894         union nic_mbx mbx = {};
  895 
  896         mbx.msg.msg = NIC_MBOX_MSG_READY;
  897         if (nicvf_send_msg_to_pf(nic, &mbx)) {
  898                 device_printf(nic->dev,
  899                            "PF didn't respond to READY msg\n");
  900                 return 0;
  901         }
  902 
  903         return 1;
  904 }
  905 
  906 static void
  907 nicvf_read_bgx_stats(struct nicvf *nic, struct bgx_stats_msg *bgx)
  908 {
  909 
  910         if (bgx->rx)
  911                 nic->bgx_stats.rx_stats[bgx->idx] = bgx->stats;
  912         else
  913                 nic->bgx_stats.tx_stats[bgx->idx] = bgx->stats;
  914 }
  915 
  916 static void
  917 nicvf_handle_mbx_intr(struct nicvf *nic)
  918 {
  919         union nic_mbx mbx = {};
  920         uint64_t *mbx_data;
  921         uint64_t mbx_addr;
  922         int i;
  923 
  924         mbx_addr = NIC_VF_PF_MAILBOX_0_1;
  925         mbx_data = (uint64_t *)&mbx;
  926 
  927         for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) {
  928                 *mbx_data = nicvf_reg_read(nic, mbx_addr);
  929                 mbx_data++;
  930                 mbx_addr += sizeof(uint64_t);
  931         }
  932 
  933         switch (mbx.msg.msg) {
  934         case NIC_MBOX_MSG_READY:
  935                 nic->pf_acked = TRUE;
  936                 nic->vf_id = mbx.nic_cfg.vf_id & 0x7F;
  937                 nic->tns_mode = mbx.nic_cfg.tns_mode & 0x7F;
  938                 nic->node = mbx.nic_cfg.node_id;
  939                 memcpy(nic->hwaddr, mbx.nic_cfg.mac_addr, ETHER_ADDR_LEN);
  940                 nic->loopback_supported = mbx.nic_cfg.loopback_supported;
  941                 nic->link_up = FALSE;
  942                 nic->duplex = 0;
  943                 nic->speed = 0;
  944                 break;
  945         case NIC_MBOX_MSG_ACK:
  946                 nic->pf_acked = TRUE;
  947                 break;
  948         case NIC_MBOX_MSG_NACK:
  949                 nic->pf_nacked = TRUE;
  950                 break;
  951         case NIC_MBOX_MSG_RSS_SIZE:
  952                 nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size;
  953                 nic->pf_acked = TRUE;
  954                 break;
  955         case NIC_MBOX_MSG_BGX_STATS:
  956                 nicvf_read_bgx_stats(nic, &mbx.bgx_stats);
  957                 nic->pf_acked = TRUE;
  958                 break;
  959         case NIC_MBOX_MSG_BGX_LINK_CHANGE:
  960                 nic->pf_acked = TRUE;
  961                 nic->link_up = mbx.link_status.link_up;
  962                 nic->duplex = mbx.link_status.duplex;
  963                 nic->speed = mbx.link_status.speed;
  964                 if (nic->link_up) {
  965                         if_setbaudrate(nic->ifp, nic->speed * 1000000);
  966                         if_link_state_change(nic->ifp, LINK_STATE_UP);
  967                 } else {
  968                         if_setbaudrate(nic->ifp, 0);
  969                         if_link_state_change(nic->ifp, LINK_STATE_DOWN);
  970                 }
  971                 break;
  972         default:
  973                 device_printf(nic->dev,
  974                            "Invalid message from PF, msg 0x%x\n", mbx.msg.msg);
  975                 break;
  976         }
  977         nicvf_clear_intr(nic, NICVF_INTR_MBOX, 0);
  978 }
  979 
  980 static int
  981 nicvf_update_hw_max_frs(struct nicvf *nic, int mtu)
  982 {
  983         union nic_mbx mbx = {};
  984 
  985         mbx.frs.msg = NIC_MBOX_MSG_SET_MAX_FRS;
  986         mbx.frs.max_frs = mtu;
  987         mbx.frs.vf_id = nic->vf_id;
  988 
  989         return nicvf_send_msg_to_pf(nic, &mbx);
  990 }
  991 
  992 static int
  993 nicvf_hw_set_mac_addr(struct nicvf *nic, uint8_t *hwaddr)
  994 {
  995         union nic_mbx mbx = {};
  996 
  997         mbx.mac.msg = NIC_MBOX_MSG_SET_MAC;
  998         mbx.mac.vf_id = nic->vf_id;
  999         memcpy(mbx.mac.mac_addr, hwaddr, ETHER_ADDR_LEN);
 1000 
 1001         return (nicvf_send_msg_to_pf(nic, &mbx));
 1002 }
 1003 
 1004 static void
 1005 nicvf_config_cpi(struct nicvf *nic)
 1006 {
 1007         union nic_mbx mbx = {};
 1008 
 1009         mbx.cpi_cfg.msg = NIC_MBOX_MSG_CPI_CFG;
 1010         mbx.cpi_cfg.vf_id = nic->vf_id;
 1011         mbx.cpi_cfg.cpi_alg = nic->cpi_alg;
 1012         mbx.cpi_cfg.rq_cnt = nic->qs->rq_cnt;
 1013 
 1014         nicvf_send_msg_to_pf(nic, &mbx);
 1015 }
 1016 
 1017 static void
 1018 nicvf_get_rss_size(struct nicvf *nic)
 1019 {
 1020         union nic_mbx mbx = {};
 1021 
 1022         mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE;
 1023         mbx.rss_size.vf_id = nic->vf_id;
 1024         nicvf_send_msg_to_pf(nic, &mbx);
 1025 }
 1026 
 1027 static void
 1028 nicvf_config_rss(struct nicvf *nic)
 1029 {
 1030         union nic_mbx mbx = {};
 1031         struct nicvf_rss_info *rss;
 1032         int ind_tbl_len;
 1033         int i, nextq;
 1034 
 1035         rss = &nic->rss_info;
 1036         ind_tbl_len = rss->rss_size;
 1037         nextq = 0;
 1038 
 1039         mbx.rss_cfg.vf_id = nic->vf_id;
 1040         mbx.rss_cfg.hash_bits = rss->hash_bits;
 1041         while (ind_tbl_len != 0) {
 1042                 mbx.rss_cfg.tbl_offset = nextq;
 1043                 mbx.rss_cfg.tbl_len = MIN(ind_tbl_len,
 1044                     RSS_IND_TBL_LEN_PER_MBX_MSG);
 1045                 mbx.rss_cfg.msg = mbx.rss_cfg.tbl_offset ?
 1046                     NIC_MBOX_MSG_RSS_CFG_CONT : NIC_MBOX_MSG_RSS_CFG;
 1047 
 1048                 for (i = 0; i < mbx.rss_cfg.tbl_len; i++)
 1049                         mbx.rss_cfg.ind_tbl[i] = rss->ind_tbl[nextq++];
 1050 
 1051                 nicvf_send_msg_to_pf(nic, &mbx);
 1052 
 1053                 ind_tbl_len -= mbx.rss_cfg.tbl_len;
 1054         }
 1055 }
 1056 
 1057 static void
 1058 nicvf_set_rss_key(struct nicvf *nic)
 1059 {
 1060         struct nicvf_rss_info *rss;
 1061         uint64_t key_addr;
 1062         int idx;
 1063 
 1064         rss = &nic->rss_info;
 1065         key_addr = NIC_VNIC_RSS_KEY_0_4;
 1066 
 1067         for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) {
 1068                 nicvf_reg_write(nic, key_addr, rss->key[idx]);
 1069                 key_addr += sizeof(uint64_t);
 1070         }
 1071 }
 1072 
 1073 static int
 1074 nicvf_rss_init(struct nicvf *nic)
 1075 {
 1076         struct nicvf_rss_info *rss;
 1077         int idx;
 1078 
 1079         nicvf_get_rss_size(nic);
 1080 
 1081         rss = &nic->rss_info;
 1082         if (nic->cpi_alg != CPI_ALG_NONE) {
 1083                 rss->enable = FALSE;
 1084                 rss->hash_bits = 0;
 1085                 return (ENXIO);
 1086         }
 1087 
 1088         rss->enable = TRUE;
 1089 
 1090         /* Using the HW reset value for now */
 1091         rss->key[0] = 0xFEED0BADFEED0BADUL;
 1092         rss->key[1] = 0xFEED0BADFEED0BADUL;
 1093         rss->key[2] = 0xFEED0BADFEED0BADUL;
 1094         rss->key[3] = 0xFEED0BADFEED0BADUL;
 1095         rss->key[4] = 0xFEED0BADFEED0BADUL;
 1096 
 1097         nicvf_set_rss_key(nic);
 1098 
 1099         rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA;
 1100         nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss->cfg);
 1101 
 1102         rss->hash_bits = fls(rss->rss_size) - 1;
 1103         for (idx = 0; idx < rss->rss_size; idx++)
 1104                 rss->ind_tbl[idx] = idx % nic->rx_queues;
 1105 
 1106         nicvf_config_rss(nic);
 1107 
 1108         return (0);
 1109 }
 1110 
 1111 static int
 1112 nicvf_init_resources(struct nicvf *nic)
 1113 {
 1114         int err;
 1115         union nic_mbx mbx = {};
 1116 
 1117         mbx.msg.msg = NIC_MBOX_MSG_CFG_DONE;
 1118 
 1119         /* Enable Qset */
 1120         nicvf_qset_config(nic, TRUE);
 1121 
 1122         /* Initialize queues and HW for data transfer */
 1123         err = nicvf_config_data_transfer(nic, TRUE);
 1124         if (err) {
 1125                 device_printf(nic->dev,
 1126                     "Failed to alloc/config VF's QSet resources\n");
 1127                 return (err);
 1128         }
 1129 
 1130         /* Send VF config done msg to PF */
 1131         nicvf_write_to_mbx(nic, &mbx);
 1132 
 1133         return (0);
 1134 }
 1135 
 1136 static void
 1137 nicvf_misc_intr_handler(void *arg)
 1138 {
 1139         struct nicvf *nic = (struct nicvf *)arg;
 1140         uint64_t intr;
 1141 
 1142         intr = nicvf_reg_read(nic, NIC_VF_INT);
 1143         /* Check for spurious interrupt */
 1144         if (!(intr & NICVF_INTR_MBOX_MASK))
 1145                 return;
 1146 
 1147         nicvf_handle_mbx_intr(nic);
 1148 }
 1149 
 1150 static int
 1151 nicvf_intr_handler(void *arg)
 1152 {
 1153         struct nicvf *nic;
 1154         struct cmp_queue *cq;
 1155         int qidx;
 1156 
 1157         cq = (struct cmp_queue *)arg;
 1158         nic = cq->nic;
 1159         qidx = cq->idx;
 1160 
 1161         /* Disable interrupts */
 1162         nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
 1163 
 1164         taskqueue_enqueue(cq->cmp_taskq, &cq->cmp_task);
 1165 
 1166         /* Clear interrupt */
 1167         nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx);
 1168 
 1169         return (FILTER_HANDLED);
 1170 }
 1171 
 1172 static void
 1173 nicvf_rbdr_intr_handler(void *arg)
 1174 {
 1175         struct nicvf *nic;
 1176         struct queue_set *qs;
 1177         struct rbdr *rbdr;
 1178         int qidx;
 1179 
 1180         nic = (struct nicvf *)arg;
 1181 
 1182         /* Disable RBDR interrupt and schedule softirq */
 1183         for (qidx = 0; qidx < nic->qs->rbdr_cnt; qidx++) {
 1184                 if (!nicvf_is_intr_enabled(nic, NICVF_INTR_RBDR, qidx))
 1185                         continue;
 1186                 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
 1187 
 1188                 qs = nic->qs;
 1189                 rbdr = &qs->rbdr[qidx];
 1190                 taskqueue_enqueue(rbdr->rbdr_taskq, &rbdr->rbdr_task_nowait);
 1191                 /* Clear interrupt */
 1192                 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
 1193         }
 1194 }
 1195 
 1196 static void
 1197 nicvf_qs_err_intr_handler(void *arg)
 1198 {
 1199         struct nicvf *nic = (struct nicvf *)arg;
 1200         struct queue_set *qs = nic->qs;
 1201 
 1202         /* Disable Qset err interrupt and schedule softirq */
 1203         nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
 1204         taskqueue_enqueue(qs->qs_err_taskq, &qs->qs_err_task);
 1205         nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
 1206 
 1207 }
 1208 
 1209 static int
 1210 nicvf_enable_msix(struct nicvf *nic)
 1211 {
 1212         struct pci_devinfo *dinfo;
 1213         int rid, count;
 1214         int ret;
 1215 
 1216         dinfo = device_get_ivars(nic->dev);
 1217         rid = dinfo->cfg.msix.msix_table_bar;
 1218         nic->msix_table_res =
 1219             bus_alloc_resource_any(nic->dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
 1220         if (nic->msix_table_res == NULL) {
 1221                 device_printf(nic->dev,
 1222                     "Could not allocate memory for MSI-X table\n");
 1223                 return (ENXIO);
 1224         }
 1225 
 1226         count = nic->num_vec = NIC_VF_MSIX_VECTORS;
 1227 
 1228         ret = pci_alloc_msix(nic->dev, &count);
 1229         if ((ret != 0) || (count != nic->num_vec)) {
 1230                 device_printf(nic->dev,
 1231                     "Request for #%d msix vectors failed, error: %d\n",
 1232                     nic->num_vec, ret);
 1233                 return (ret);
 1234         }
 1235 
 1236         nic->msix_enabled = 1;
 1237         return (0);
 1238 }
 1239 
 1240 static void
 1241 nicvf_disable_msix(struct nicvf *nic)
 1242 {
 1243 
 1244         if (nic->msix_enabled) {
 1245                 pci_release_msi(nic->dev);
 1246                 nic->msix_enabled = 0;
 1247                 nic->num_vec = 0;
 1248         }
 1249 }
 1250 
 1251 static void
 1252 nicvf_release_all_interrupts(struct nicvf *nic)
 1253 {
 1254         struct resource *res;
 1255         int irq;
 1256         int err;
 1257 
 1258         /* Free registered interrupts */
 1259         for (irq = 0; irq < nic->num_vec; irq++) {
 1260                 res = nic->msix_entries[irq].irq_res;
 1261                 if (res == NULL)
 1262                         continue;
 1263                 /* Teardown interrupt first */
 1264                 if (nic->msix_entries[irq].handle != NULL) {
 1265                         err = bus_teardown_intr(nic->dev,
 1266                             nic->msix_entries[irq].irq_res,
 1267                             nic->msix_entries[irq].handle);
 1268                         KASSERT(err == 0,
 1269                             ("ERROR: Unable to teardown interrupt %d", irq));
 1270                         nic->msix_entries[irq].handle = NULL;
 1271                 }
 1272 
 1273                 bus_release_resource(nic->dev, SYS_RES_IRQ,
 1274                             rman_get_rid(res), nic->msix_entries[irq].irq_res);
 1275                 nic->msix_entries[irq].irq_res = NULL;
 1276         }
 1277         /* Disable MSI-X */
 1278         nicvf_disable_msix(nic);
 1279 }
 1280 
 1281 /*
 1282  * Initialize MSIX vectors and register MISC interrupt.
 1283  * Send READY message to PF to check if its alive
 1284  */
 1285 static int
 1286 nicvf_allocate_misc_interrupt(struct nicvf *nic)
 1287 {
 1288         struct resource *res;
 1289         int irq, rid;
 1290         int ret = 0;
 1291 
 1292         /* Return if mailbox interrupt is already registered */
 1293         if (nic->msix_enabled)
 1294                 return (0);
 1295 
 1296         /* Enable MSI-X */
 1297         if (nicvf_enable_msix(nic) != 0)
 1298                 return (ENXIO);
 1299 
 1300         irq = NICVF_INTR_ID_MISC;
 1301         rid = irq + 1;
 1302         nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev,
 1303             SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE));
 1304         if (nic->msix_entries[irq].irq_res == NULL) {
 1305                 device_printf(nic->dev,
 1306                     "Could not allocate Mbox interrupt for VF%d\n",
 1307                     device_get_unit(nic->dev));
 1308                 return (ENXIO);
 1309         }
 1310 
 1311         ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res,
 1312             (INTR_MPSAFE | INTR_TYPE_MISC), NULL, nicvf_misc_intr_handler, nic,
 1313             &nic->msix_entries[irq].handle);
 1314         if (ret != 0) {
 1315                 res = nic->msix_entries[irq].irq_res;
 1316                 bus_release_resource(nic->dev, SYS_RES_IRQ,
 1317                             rman_get_rid(res), res);
 1318                 nic->msix_entries[irq].irq_res = NULL;
 1319                 return (ret);
 1320         }
 1321 
 1322         return (0);
 1323 }
 1324 
 1325 static int
 1326 nicvf_enable_misc_interrupt(struct nicvf *nic)
 1327 {
 1328 
 1329         /* Enable mailbox interrupt */
 1330         nicvf_enable_intr(nic, NICVF_INTR_MBOX, 0);
 1331 
 1332         /* Check if VF is able to communicate with PF */
 1333         if (!nicvf_check_pf_ready(nic)) {
 1334                 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
 1335                 return (ENXIO);
 1336         }
 1337 
 1338         return (0);
 1339 }
 1340 
 1341 static void
 1342 nicvf_release_net_interrupts(struct nicvf *nic)
 1343 {
 1344         struct resource *res;
 1345         int irq;
 1346         int err;
 1347 
 1348         for_each_cq_irq(irq) {
 1349                 res = nic->msix_entries[irq].irq_res;
 1350                 if (res == NULL)
 1351                         continue;
 1352                 /* Teardown active interrupts first */
 1353                 if (nic->msix_entries[irq].handle != NULL) {
 1354                         err = bus_teardown_intr(nic->dev,
 1355                             nic->msix_entries[irq].irq_res,
 1356                             nic->msix_entries[irq].handle);
 1357                         KASSERT(err == 0,
 1358                             ("ERROR: Unable to teardown CQ interrupt %d",
 1359                             (irq - NICVF_INTR_ID_CQ)));
 1360                         if (err != 0)
 1361                                 continue;
 1362                 }
 1363 
 1364                 /* Release resource */
 1365                 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res),
 1366                     res);
 1367                 nic->msix_entries[irq].irq_res = NULL;
 1368         }
 1369 
 1370         for_each_rbdr_irq(irq) {
 1371                 res = nic->msix_entries[irq].irq_res;
 1372                 if (res == NULL)
 1373                         continue;
 1374                 /* Teardown active interrupts first */
 1375                 if (nic->msix_entries[irq].handle != NULL) {
 1376                         err = bus_teardown_intr(nic->dev,
 1377                             nic->msix_entries[irq].irq_res,
 1378                             nic->msix_entries[irq].handle);
 1379                         KASSERT(err == 0,
 1380                             ("ERROR: Unable to teardown RDBR interrupt %d",
 1381                             (irq - NICVF_INTR_ID_RBDR)));
 1382                         if (err != 0)
 1383                                 continue;
 1384                 }
 1385 
 1386                 /* Release resource */
 1387                 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res),
 1388                     res);
 1389                 nic->msix_entries[irq].irq_res = NULL;
 1390         }
 1391 
 1392         irq = NICVF_INTR_ID_QS_ERR;
 1393         res = nic->msix_entries[irq].irq_res;
 1394         if (res != NULL) {
 1395                 /* Teardown active interrupts first */
 1396                 if (nic->msix_entries[irq].handle != NULL) {
 1397                         err = bus_teardown_intr(nic->dev,
 1398                             nic->msix_entries[irq].irq_res,
 1399                             nic->msix_entries[irq].handle);
 1400                         KASSERT(err == 0,
 1401                             ("ERROR: Unable to teardown QS Error interrupt %d",
 1402                             irq));
 1403                         if (err != 0)
 1404                                 return;
 1405                 }
 1406 
 1407                 /* Release resource */
 1408                 bus_release_resource(nic->dev, SYS_RES_IRQ, rman_get_rid(res),
 1409                     res);
 1410                 nic->msix_entries[irq].irq_res = NULL;
 1411         }
 1412 }
 1413 
 1414 static int
 1415 nicvf_allocate_net_interrupts(struct nicvf *nic)
 1416 {
 1417         u_int cpuid;
 1418         int irq, rid;
 1419         int qidx;
 1420         int ret = 0;
 1421 
 1422         /* MSI-X must be configured by now */
 1423         if (!nic->msix_enabled) {
 1424                 device_printf(nic->dev, "Cannot alloacte queue interrups. "
 1425                     "MSI-X interrupts disabled.\n");
 1426                 return (ENXIO);
 1427         }
 1428 
 1429         /* Register CQ interrupts */
 1430         for_each_cq_irq(irq) {
 1431                 if (irq >= (NICVF_INTR_ID_CQ + nic->qs->cq_cnt))
 1432                         break;
 1433 
 1434                 qidx = irq - NICVF_INTR_ID_CQ;
 1435                 rid = irq + 1;
 1436                 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev,
 1437                     SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE));
 1438                 if (nic->msix_entries[irq].irq_res == NULL) {
 1439                         device_printf(nic->dev,
 1440                             "Could not allocate CQ interrupt %d for VF%d\n",
 1441                             (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev));
 1442                         ret = ENXIO;
 1443                         goto error;
 1444                 }
 1445                 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res,
 1446                     (INTR_MPSAFE | INTR_TYPE_NET), nicvf_intr_handler,
 1447                     NULL, &nic->qs->cq[qidx], &nic->msix_entries[irq].handle);
 1448                 if (ret != 0) {
 1449                         device_printf(nic->dev,
 1450                             "Could not setup CQ interrupt %d for VF%d\n",
 1451                             (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev));
 1452                         goto error;
 1453                 }
 1454                 cpuid = (device_get_unit(nic->dev) * CMP_QUEUE_CNT) + qidx;
 1455                 cpuid %= mp_ncpus;
 1456                 /*
 1457                  * Save CPU ID for later use when system-wide RSS is enabled.
 1458                  * It will be used to pit the CQ task to the same CPU that got
 1459                  * interrupted.
 1460                  */
 1461                 nic->qs->cq[qidx].cmp_cpuid = cpuid;
 1462                 if (bootverbose) {
 1463                         device_printf(nic->dev, "bind CQ%d IRQ to CPU%d\n",
 1464                             qidx, cpuid);
 1465                 }
 1466                 /* Bind interrupts to the given CPU */
 1467                 bus_bind_intr(nic->dev, nic->msix_entries[irq].irq_res, cpuid);
 1468         }
 1469 
 1470         /* Register RBDR interrupt */
 1471         for_each_rbdr_irq(irq) {
 1472                 if (irq >= (NICVF_INTR_ID_RBDR + nic->qs->rbdr_cnt))
 1473                         break;
 1474 
 1475                 rid = irq + 1;
 1476                 nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev,
 1477                     SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE));
 1478                 if (nic->msix_entries[irq].irq_res == NULL) {
 1479                         device_printf(nic->dev,
 1480                             "Could not allocate RBDR interrupt %d for VF%d\n",
 1481                             (irq - NICVF_INTR_ID_RBDR),
 1482                             device_get_unit(nic->dev));
 1483                         ret = ENXIO;
 1484                         goto error;
 1485                 }
 1486                 ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res,
 1487                     (INTR_MPSAFE | INTR_TYPE_NET), NULL,
 1488                     nicvf_rbdr_intr_handler, nic,
 1489                     &nic->msix_entries[irq].handle);
 1490                 if (ret != 0) {
 1491                         device_printf(nic->dev,
 1492                             "Could not setup RBDR interrupt %d for VF%d\n",
 1493                             (irq - NICVF_INTR_ID_RBDR),
 1494                             device_get_unit(nic->dev));
 1495                         goto error;
 1496                 }
 1497         }
 1498 
 1499         /* Register QS error interrupt */
 1500         irq = NICVF_INTR_ID_QS_ERR;
 1501         rid = irq + 1;
 1502         nic->msix_entries[irq].irq_res = bus_alloc_resource_any(nic->dev,
 1503             SYS_RES_IRQ, &rid, (RF_SHAREABLE | RF_ACTIVE));
 1504         if (nic->msix_entries[irq].irq_res == NULL) {
 1505                 device_printf(nic->dev,
 1506                     "Could not allocate QS Error interrupt for VF%d\n",
 1507                     device_get_unit(nic->dev));
 1508                 ret = ENXIO;
 1509                 goto error;
 1510         }
 1511         ret = bus_setup_intr(nic->dev, nic->msix_entries[irq].irq_res,
 1512             (INTR_MPSAFE | INTR_TYPE_NET), NULL, nicvf_qs_err_intr_handler,
 1513             nic, &nic->msix_entries[irq].handle);
 1514         if (ret != 0) {
 1515                 device_printf(nic->dev,
 1516                     "Could not setup QS Error interrupt for VF%d\n",
 1517                     device_get_unit(nic->dev));
 1518                 goto error;
 1519         }
 1520 
 1521         return (0);
 1522 error:
 1523         nicvf_release_net_interrupts(nic);
 1524         return (ret);
 1525 }
 1526 
 1527 static int
 1528 nicvf_stop_locked(struct nicvf *nic)
 1529 {
 1530         struct ifnet *ifp;
 1531         int qidx;
 1532         struct queue_set *qs = nic->qs;
 1533         union nic_mbx mbx = {};
 1534 
 1535         NICVF_CORE_LOCK_ASSERT(nic);
 1536         /* Stop callout. Can block here since holding SX lock */
 1537         callout_drain(&nic->stats_callout);
 1538 
 1539         ifp = nic->ifp;
 1540 
 1541         mbx.msg.msg = NIC_MBOX_MSG_SHUTDOWN;
 1542         nicvf_send_msg_to_pf(nic, &mbx);
 1543 
 1544         /* Disable RBDR & QS error interrupts */
 1545         for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) {
 1546                 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
 1547                 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
 1548         }
 1549         nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
 1550         nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
 1551 
 1552         /* Deactivate network interface */
 1553         if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
 1554 
 1555         /* Free resources */
 1556         nicvf_config_data_transfer(nic, FALSE);
 1557 
 1558         /* Disable HW Qset */
 1559         nicvf_qset_config(nic, FALSE);
 1560 
 1561         /* disable mailbox interrupt */
 1562         nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
 1563 
 1564         return (0);
 1565 }
 1566 
 1567 static void
 1568 nicvf_update_stats(struct nicvf *nic)
 1569 {
 1570         int qidx;
 1571         struct nicvf_hw_stats *stats = &nic->hw_stats;
 1572         struct nicvf_drv_stats *drv_stats = &nic->drv_stats;
 1573         struct queue_set *qs = nic->qs;
 1574 
 1575 #define GET_RX_STATS(reg) \
 1576     nicvf_reg_read(nic, NIC_VNIC_RX_STAT_0_13 | ((reg) << 3))
 1577 #define GET_TX_STATS(reg) \
 1578     nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | ((reg) << 3))
 1579 
 1580         stats->rx_bytes = GET_RX_STATS(RX_OCTS);
 1581         stats->rx_ucast_frames = GET_RX_STATS(RX_UCAST);
 1582         stats->rx_bcast_frames = GET_RX_STATS(RX_BCAST);
 1583         stats->rx_mcast_frames = GET_RX_STATS(RX_MCAST);
 1584         stats->rx_fcs_errors = GET_RX_STATS(RX_FCS);
 1585         stats->rx_l2_errors = GET_RX_STATS(RX_L2ERR);
 1586         stats->rx_drop_red = GET_RX_STATS(RX_RED);
 1587         stats->rx_drop_red_bytes = GET_RX_STATS(RX_RED_OCTS);
 1588         stats->rx_drop_overrun = GET_RX_STATS(RX_ORUN);
 1589         stats->rx_drop_overrun_bytes = GET_RX_STATS(RX_ORUN_OCTS);
 1590         stats->rx_drop_bcast = GET_RX_STATS(RX_DRP_BCAST);
 1591         stats->rx_drop_mcast = GET_RX_STATS(RX_DRP_MCAST);
 1592         stats->rx_drop_l3_bcast = GET_RX_STATS(RX_DRP_L3BCAST);
 1593         stats->rx_drop_l3_mcast = GET_RX_STATS(RX_DRP_L3MCAST);
 1594 
 1595         stats->tx_bytes_ok = GET_TX_STATS(TX_OCTS);
 1596         stats->tx_ucast_frames_ok = GET_TX_STATS(TX_UCAST);
 1597         stats->tx_bcast_frames_ok = GET_TX_STATS(TX_BCAST);
 1598         stats->tx_mcast_frames_ok = GET_TX_STATS(TX_MCAST);
 1599         stats->tx_drops = GET_TX_STATS(TX_DROP);
 1600 
 1601         drv_stats->tx_frames_ok = stats->tx_ucast_frames_ok +
 1602             stats->tx_bcast_frames_ok + stats->tx_mcast_frames_ok;
 1603         drv_stats->rx_drops = stats->rx_drop_red + stats->rx_drop_overrun;
 1604         drv_stats->tx_drops = stats->tx_drops;
 1605 
 1606         /* Update RQ and SQ stats */
 1607         for (qidx = 0; qidx < qs->rq_cnt; qidx++)
 1608                 nicvf_update_rq_stats(nic, qidx);
 1609         for (qidx = 0; qidx < qs->sq_cnt; qidx++)
 1610                 nicvf_update_sq_stats(nic, qidx);
 1611 }
 1612 
 1613 static void
 1614 nicvf_tick_stats(void *arg)
 1615 {
 1616         struct nicvf *nic;
 1617 
 1618         nic = (struct nicvf *)arg;
 1619 
 1620         /* Read the statistics */
 1621         nicvf_update_stats(nic);
 1622 
 1623         callout_reset(&nic->stats_callout, hz, nicvf_tick_stats, nic);
 1624 }
Cache object: 5f0850f81bc63ebe3efbdb5186c35bb2
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/vnic/nicvf_main.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/vnic/nicvf_main.c
