sfxge_ev.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3  *
    4  * Copyright (c) 2010-2016 Solarflare Communications Inc.
    5  * All rights reserved.
    6  *
    7  * This software was developed in part by Philip Paeps under contract for
    8  * Solarflare Communications, Inc.
    9  *
   10  * Redistribution and use in source and binary forms, with or without
   11  * modification, are permitted provided that the following conditions are met:
   12  *
   13  * 1. Redistributions of source code must retain the above copyright notice,
   14  *    this list of conditions and the following disclaimer.
   15  * 2. Redistributions in binary form must reproduce the above copyright notice,
   16  *    this list of conditions and the following disclaimer in the documentation
   17  *    and/or other materials provided with the distribution.
   18  *
   19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   20  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
   21  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
   22  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
   23  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
   24  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
   25  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
   26  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
   27  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
   28  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
   29  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   30  *
   31  * The views and conclusions contained in the software and documentation are
   32  * those of the authors and should not be interpreted as representing official
   33  * policies, either expressed or implied, of the FreeBSD Project.
   34  */
   35 
   36 #include <sys/cdefs.h>
   37 __FBSDID("$FreeBSD$");
   38 
   39 #include <sys/param.h>
   40 #include <sys/kernel.h>
   41 #include <sys/malloc.h>
   42 #include <sys/param.h>
   43 #include <sys/queue.h>
   44 #include <sys/systm.h>
   45 #include <sys/taskqueue.h>
   46 
   47 #include "common/efx.h"
   48 
   49 #include "sfxge.h"
   50 
   51 static void
   52 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
   53 {
   54         struct sfxge_softc *sc;
   55         unsigned int index;
   56         struct sfxge_rxq *rxq;
   57         struct sfxge_txq *txq;
   58 
   59         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
   60 
   61         sc = evq->sc;
   62         index = evq->index;
   63         rxq = sc->rxq[index];
   64 
   65         if ((txq = evq->txq) != NULL) {
   66                 evq->txq = NULL;
   67                 evq->txqs = &(evq->txq);
   68 
   69                 do {
   70                         struct sfxge_txq *next;
   71 
   72                         next = txq->next;
   73                         txq->next = NULL;
   74 
   75                         KASSERT(txq->evq_index == index,
   76                             ("txq->evq_index != index"));
   77 
   78                         if (txq->pending != txq->completed)
   79                                 sfxge_tx_qcomplete(txq, evq);
   80 
   81                         txq = next;
   82                 } while (txq != NULL);
   83         }
   84 
   85         if (rxq->pending != rxq->completed)
   86                 sfxge_rx_qcomplete(rxq, eop);
   87 }
   88 
   89 static struct sfxge_rxq *
   90 sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label)
   91 {
   92         struct sfxge_rxq *rxq;
   93 
   94         KASSERT(label == 0, ("unexpected rxq label != 0"));
   95 
   96         rxq = evq->sc->rxq[evq->index];
   97 
   98         KASSERT(rxq != NULL, ("rxq == NULL"));
   99         KASSERT(evq->index == rxq->index, ("evq->index != rxq->index"));
  100 
  101         return (rxq);
  102 }
  103 
  104 static boolean_t
  105 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
  106             uint16_t flags)
  107 {
  108         struct sfxge_evq *evq;
  109         struct sfxge_softc *sc;
  110         struct sfxge_rxq *rxq;
  111         unsigned int stop;
  112         unsigned int delta;
  113         struct sfxge_rx_sw_desc *rx_desc;
  114 
  115         evq = arg;
  116         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  117 
  118         sc = evq->sc;
  119 
  120         if (evq->exception)
  121                 goto done;
  122 
  123         rxq = sfxge_get_rxq_by_label(evq, label);
  124         if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
  125                 goto done;
  126 
  127         stop = (id + 1) & rxq->ptr_mask;
  128         id = rxq->pending & rxq->ptr_mask;
  129         delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop);
  130         rxq->pending += delta;
  131 
  132         if (delta != 1) {
  133                 if ((delta <= 0) ||
  134                     (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) {
  135                         evq->exception = B_TRUE;
  136 
  137                         device_printf(sc->dev, "RX completion out of order"
  138                                                   " (id=%#x delta=%u flags=%#x); resetting\n",
  139                                                   id, delta, flags);
  140                         sfxge_schedule_reset(sc);
  141 
  142                         goto done;
  143                 }
  144         }
  145 
  146         rx_desc = &rxq->queue[id];
  147 
  148         prefetch_read_many(rx_desc->mbuf);
  149 
  150         for (; id != stop; id = (id + 1) & rxq->ptr_mask) {
  151                 rx_desc = &rxq->queue[id];
  152                 KASSERT(rx_desc->flags == EFX_DISCARD,
  153                                 ("rx_desc->flags != EFX_DISCARD"));
  154                 rx_desc->flags = flags;
  155 
  156                 KASSERT(size < (1 << 16), ("size > (1 << 16)"));
  157                 rx_desc->size = (uint16_t)size;
  158         }
  159 
  160         evq->rx_done++;
  161 
  162         if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
  163                 sfxge_ev_qcomplete(evq, B_FALSE);
  164 
  165 done:
  166         return (evq->rx_done >= SFXGE_EV_BATCH);
  167 }
  168 
  169 static boolean_t
  170 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
  171 {
  172         struct sfxge_evq *evq;
  173         struct sfxge_softc *sc;
  174 
  175         evq = (struct sfxge_evq *)arg;
  176         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  177 
  178         sc = evq->sc;
  179 
  180         DBGPRINT(sc->dev, "[%d] %s", evq->index,
  181                           (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" :
  182                           (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" :
  183                           (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" :
  184                           (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" :
  185                           (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" :
  186                           (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" :
  187                           (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" :
  188                           (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" :
  189                           (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" :
  190                           "UNKNOWN");
  191 
  192         evq->exception = B_TRUE;
  193 
  194         if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
  195                 device_printf(sc->dev,
  196                               "hardware exception (code=%u); resetting\n",
  197                               code);
  198                 sfxge_schedule_reset(sc);
  199         }
  200 
  201         return (B_FALSE);
  202 }
  203 
  204 static boolean_t
  205 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
  206 {
  207         struct sfxge_evq *evq;
  208         struct sfxge_softc *sc;
  209         struct sfxge_rxq *rxq;
  210         unsigned int index;
  211         uint16_t magic;
  212 
  213         evq = (struct sfxge_evq *)arg;
  214         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  215 
  216         sc = evq->sc;
  217         rxq = sc->rxq[rxq_index];
  218 
  219         KASSERT(rxq != NULL, ("rxq == NULL"));
  220 
  221         /* Resend a software event on the correct queue */
  222         index = rxq->index;
  223         if (index == evq->index) {
  224                 sfxge_rx_qflush_done(rxq);
  225                 return (B_FALSE);
  226         }
  227 
  228         evq = sc->evq[index];
  229         magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq);
  230 
  231         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
  232             ("evq not started"));
  233         efx_ev_qpost(evq->common, magic);
  234 
  235         return (B_FALSE);
  236 }
  237 
  238 static boolean_t
  239 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
  240 {
  241         struct sfxge_evq *evq;
  242         struct sfxge_softc *sc;
  243         struct sfxge_rxq *rxq;
  244         unsigned int index;
  245         uint16_t magic;
  246 
  247         evq = (struct sfxge_evq *)arg;
  248         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  249 
  250         sc = evq->sc;
  251         rxq = sc->rxq[rxq_index];
  252 
  253         KASSERT(rxq != NULL, ("rxq == NULL"));
  254 
  255         /* Resend a software event on the correct queue */
  256         index = rxq->index;
  257         evq = sc->evq[index];
  258         magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq);
  259 
  260         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
  261             ("evq not started"));
  262         efx_ev_qpost(evq->common, magic);
  263 
  264         return (B_FALSE);
  265 }
  266 
  267 static struct sfxge_txq *
  268 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)
  269 {
  270         unsigned int index;
  271 
  272         KASSERT((evq->sc->txq_dynamic_cksum_toggle_supported) ? (label == 0) :
  273                 ((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||
  274                  (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM)),
  275                 ("unexpected txq label"));
  276 
  277         index = (evq->index == 0) ?
  278                 label : (evq->index - 1 + SFXGE_EVQ0_N_TXQ(evq->sc));
  279         return (evq->sc->txq[index]);
  280 }
  281 
  282 static boolean_t
  283 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
  284 {
  285         struct sfxge_evq *evq;
  286         struct sfxge_txq *txq;
  287         unsigned int stop;
  288         unsigned int delta;
  289 
  290         evq = (struct sfxge_evq *)arg;
  291         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  292 
  293         txq = sfxge_get_txq_by_label(evq, label);
  294 
  295         KASSERT(txq != NULL, ("txq == NULL"));
  296         KASSERT(evq->index == txq->evq_index,
  297             ("evq->index != txq->evq_index"));
  298 
  299         if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED))
  300                 goto done;
  301 
  302         stop = (id + 1) & txq->ptr_mask;
  303         id = txq->pending & txq->ptr_mask;
  304 
  305         delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop);
  306         txq->pending += delta;
  307 
  308         evq->tx_done++;
  309 
  310         if (txq->next == NULL &&
  311             evq->txqs != &(txq->next)) {
  312                 *(evq->txqs) = txq;
  313                 evq->txqs = &(txq->next);
  314         }
  315 
  316         if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
  317                 sfxge_tx_qcomplete(txq, evq);
  318 
  319 done:
  320         return (evq->tx_done >= SFXGE_EV_BATCH);
  321 }
  322 
  323 static boolean_t
  324 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
  325 {
  326         struct sfxge_evq *evq;
  327         struct sfxge_softc *sc;
  328         struct sfxge_txq *txq;
  329         uint16_t magic;
  330 
  331         evq = (struct sfxge_evq *)arg;
  332         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  333 
  334         sc = evq->sc;
  335         txq = sc->txq[txq_index];
  336 
  337         KASSERT(txq != NULL, ("txq == NULL"));
  338         KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
  339             ("txq not initialized"));
  340 
  341         if (txq->evq_index == evq->index) {
  342                 sfxge_tx_qflush_done(txq);
  343                 return (B_FALSE);
  344         }
  345 
  346         /* Resend a software event on the correct queue */
  347         evq = sc->evq[txq->evq_index];
  348         magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq);
  349 
  350         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
  351             ("evq not started"));
  352         efx_ev_qpost(evq->common, magic);
  353 
  354         return (B_FALSE);
  355 }
  356 
  357 static boolean_t
  358 sfxge_ev_software(void *arg, uint16_t magic)
  359 {
  360         struct sfxge_evq *evq;
  361         unsigned int label;
  362 
  363         evq = (struct sfxge_evq *)arg;
  364         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  365 
  366         label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
  367         magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
  368 
  369         switch (magic) {
  370         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE):
  371                 sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label));
  372                 break;
  373 
  374         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED):
  375                 sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label));
  376                 break;
  377 
  378         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL):
  379                 sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label));
  380                 break;
  381 
  382         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): {
  383                 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);
  384 
  385                 KASSERT(txq != NULL, ("txq == NULL"));
  386                 KASSERT(evq->index == txq->evq_index,
  387                     ("evq->index != txq->evq_index"));
  388 
  389                 sfxge_tx_qflush_done(txq);
  390                 break;
  391         }
  392         default:
  393                 break;
  394         }
  395 
  396         return (B_FALSE);
  397 }
  398 
  399 static boolean_t
  400 sfxge_ev_sram(void *arg, uint32_t code)
  401 {
  402         (void)arg;
  403         (void)code;
  404 
  405         switch (code) {
  406         case EFX_SRAM_UPDATE:
  407                 EFSYS_PROBE(sram_update);
  408                 break;
  409 
  410         case EFX_SRAM_CLEAR:
  411                 EFSYS_PROBE(sram_clear);
  412                 break;
  413 
  414         case EFX_SRAM_ILLEGAL_CLEAR:
  415                 EFSYS_PROBE(sram_illegal_clear);
  416                 break;
  417 
  418         default:
  419                 KASSERT(B_FALSE, ("Impossible SRAM event"));
  420                 break;
  421         }
  422 
  423         return (B_FALSE);
  424 }
  425 
  426 static boolean_t
  427 sfxge_ev_timer(void *arg, uint32_t index)
  428 {
  429         (void)arg;
  430         (void)index;
  431 
  432         return (B_FALSE);
  433 }
  434 
  435 static boolean_t
  436 sfxge_ev_wake_up(void *arg, uint32_t index)
  437 {
  438         (void)arg;
  439         (void)index;
  440 
  441         return (B_FALSE);
  442 }
  443 
  444 #if EFSYS_OPT_QSTATS
  445 
  446 static void
  447 sfxge_evq_stat_update(struct sfxge_evq *evq)
  448 {
  449         clock_t now;
  450 
  451         SFXGE_EVQ_LOCK(evq);
  452 
  453         if (__predict_false(evq->init_state != SFXGE_EVQ_STARTED))
  454                 goto out;
  455 
  456         now = ticks;
  457         if ((unsigned int)(now - evq->stats_update_time) < (unsigned int)hz)
  458                 goto out;
  459 
  460         evq->stats_update_time = now;
  461         efx_ev_qstats_update(evq->common, evq->stats);
  462 
  463 out:
  464         SFXGE_EVQ_UNLOCK(evq);
  465 }
  466 
  467 static int
  468 sfxge_evq_stat_handler(SYSCTL_HANDLER_ARGS)
  469 {
  470         struct sfxge_evq *evq = arg1;
  471         struct sfxge_softc *sc = evq->sc;
  472         unsigned int id = arg2;
  473 
  474         SFXGE_ADAPTER_LOCK(sc);
  475 
  476         sfxge_evq_stat_update(evq);
  477 
  478         SFXGE_ADAPTER_UNLOCK(sc);
  479 
  480         return (SYSCTL_OUT(req, &evq->stats[id], sizeof(evq->stats[id])));
  481 }
  482 
  483 static int
  484 sfxge_evq_stat_init(struct sfxge_evq *evq)
  485 {
  486         struct sfxge_softc *sc = evq->sc;
  487         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
  488         char name[16];
  489         struct sysctl_oid *evq_stats_node;
  490         unsigned int id;
  491 
  492         snprintf(name, sizeof(name), "%u", evq->index);
  493         evq_stats_node = SYSCTL_ADD_NODE(ctx,
  494             SYSCTL_CHILDREN(sc->evqs_stats_node), OID_AUTO, name,
  495             CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "");
  496         if (evq_stats_node == NULL)
  497                 return (ENOMEM);
  498 
  499         for (id = 0; id < EV_NQSTATS; id++) {
  500                 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(evq_stats_node),
  501                     OID_AUTO, efx_ev_qstat_name(sc->enp, id),
  502                     CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, evq, id,
  503                     sfxge_evq_stat_handler, "Q", "");
  504         }
  505 
  506         return (0);
  507 }
  508 
  509 static void
  510 sfxge_ev_stat_update(struct sfxge_softc *sc)
  511 {
  512         struct sfxge_evq *evq;
  513         unsigned int index;
  514         clock_t now;
  515         unsigned int id;
  516 
  517         SFXGE_ADAPTER_LOCK(sc);
  518 
  519         now = ticks;
  520         if ((unsigned int)(now - sc->ev_stats_update_time) < (unsigned int)hz)
  521                 goto out;
  522 
  523         sc->ev_stats_update_time = now;
  524 
  525         memset(sc->ev_stats, 0, sizeof(sc->ev_stats));
  526 
  527         /* Update and add event counts from each event queue in turn */
  528         for (index = 0; index < sc->evq_count; index++) {
  529                 evq = sc->evq[index];
  530                 sfxge_evq_stat_update(evq);
  531                 for (id = 0; id < EV_NQSTATS; id++)
  532                         sc->ev_stats[id] += evq->stats[id];
  533         }
  534 out:
  535         SFXGE_ADAPTER_UNLOCK(sc);
  536 }
  537 
  538 static int
  539 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
  540 {
  541         struct sfxge_softc *sc = arg1;
  542         unsigned int id = arg2;
  543 
  544         sfxge_ev_stat_update(sc);
  545 
  546         return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])));
  547 }
  548 
  549 static void
  550 sfxge_ev_stat_init(struct sfxge_softc *sc)
  551 {
  552         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
  553         struct sysctl_oid_list *stat_list;
  554         unsigned int id;
  555         char name[40];
  556 
  557         stat_list = SYSCTL_CHILDREN(sc->stats_node);
  558 
  559         for (id = 0; id < EV_NQSTATS; id++) {
  560                 snprintf(name, sizeof(name), "ev_%s",
  561                          efx_ev_qstat_name(sc->enp, id));
  562                 SYSCTL_ADD_PROC(ctx, stat_list, OID_AUTO, name,
  563                     CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE,
  564                     sc, id, sfxge_ev_stat_handler, "Q", "");
  565         }
  566 }
  567 
  568 #endif /* EFSYS_OPT_QSTATS */
  569 
  570 static void
  571 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
  572 {
  573         struct sfxge_evq *evq;
  574         efx_evq_t *eep;
  575 
  576         evq = sc->evq[idx];
  577         eep = evq->common;
  578 
  579         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
  580             ("evq->init_state != SFXGE_EVQ_STARTED"));
  581 
  582         (void)efx_ev_qmoderate(eep, us);
  583 }
  584 
  585 static int
  586 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
  587 {
  588         struct sfxge_softc *sc = arg1;
  589         struct sfxge_intr *intr = &sc->intr;
  590         unsigned int moderation;
  591         int error;
  592         unsigned int index;
  593 
  594         SFXGE_ADAPTER_LOCK(sc);
  595 
  596         if (req->newptr != NULL) {
  597                 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
  598                     != 0)
  599                         goto out;
  600 
  601                 /* We may not be calling efx_ev_qmoderate() now,
  602                  * so we have to range-check the value ourselves.
  603                  */
  604                 if (moderation >
  605                     efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) {
  606                         error = EINVAL;
  607                         goto out;
  608                 }
  609 
  610                 sc->ev_moderation = moderation;
  611                 if (intr->state == SFXGE_INTR_STARTED) {
  612                         for (index = 0; index < sc->evq_count; index++)
  613                                 sfxge_ev_qmoderate(sc, index, moderation);
  614                 }
  615         } else {
  616                 error = SYSCTL_OUT(req, &sc->ev_moderation,
  617                                    sizeof(sc->ev_moderation));
  618         }
  619 
  620 out:
  621         SFXGE_ADAPTER_UNLOCK(sc);
  622 
  623         return (error);
  624 }
  625 
  626 static boolean_t
  627 sfxge_ev_initialized(void *arg)
  628 {
  629         struct sfxge_evq *evq;
  630 
  631         evq = (struct sfxge_evq *)arg;
  632         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  633 
  634         /* Init done events may be duplicated on 7xxx */
  635         KASSERT(evq->init_state == SFXGE_EVQ_STARTING ||
  636                 evq->init_state == SFXGE_EVQ_STARTED,
  637             ("evq not starting"));
  638 
  639         evq->init_state = SFXGE_EVQ_STARTED;
  640 
  641         return (0);
  642 }
  643 
  644 static boolean_t
  645 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
  646 {
  647         struct sfxge_evq *evq;
  648         struct sfxge_softc *sc;
  649 
  650         evq = (struct sfxge_evq *)arg;
  651         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  652 
  653         sc = evq->sc;
  654 
  655         sfxge_mac_link_update(sc, link_mode);
  656 
  657         return (0);
  658 }
  659 
  660 static const efx_ev_callbacks_t sfxge_ev_callbacks = {
  661         .eec_initialized        = sfxge_ev_initialized,
  662         .eec_rx                 = sfxge_ev_rx,
  663         .eec_tx                 = sfxge_ev_tx,
  664         .eec_exception          = sfxge_ev_exception,
  665         .eec_rxq_flush_done     = sfxge_ev_rxq_flush_done,
  666         .eec_rxq_flush_failed   = sfxge_ev_rxq_flush_failed,
  667         .eec_txq_flush_done     = sfxge_ev_txq_flush_done,
  668         .eec_software           = sfxge_ev_software,
  669         .eec_sram               = sfxge_ev_sram,
  670         .eec_wake_up            = sfxge_ev_wake_up,
  671         .eec_timer              = sfxge_ev_timer,
  672         .eec_link_change        = sfxge_ev_link_change,
  673 };
  674 
  675 int
  676 sfxge_ev_qpoll(struct sfxge_evq *evq)
  677 {
  678         int rc;
  679 
  680         SFXGE_EVQ_LOCK(evq);
  681 
  682         if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING &&
  683                             evq->init_state != SFXGE_EVQ_STARTED)) {
  684                 rc = EINVAL;
  685                 goto fail;
  686         }
  687 
  688         /* Synchronize the DMA memory for reading */
  689         bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
  690             BUS_DMASYNC_POSTREAD);
  691 
  692         KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
  693         KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
  694         KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
  695         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
  696 
  697         /* Poll the queue */
  698         efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
  699 
  700         evq->rx_done = 0;
  701         evq->tx_done = 0;
  702 
  703         /* Perform any pending completion processing */
  704         sfxge_ev_qcomplete(evq, B_TRUE);
  705 
  706         /* Re-prime the event queue for interrupts */
  707         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
  708                 goto fail;
  709 
  710         SFXGE_EVQ_UNLOCK(evq);
  711 
  712         return (0);
  713 
  714 fail:
  715         SFXGE_EVQ_UNLOCK(evq);
  716         return (rc);
  717 }
  718 
  719 static void
  720 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
  721 {
  722         struct sfxge_evq *evq;
  723 
  724         evq = sc->evq[index];
  725 
  726         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
  727             ("evq->init_state != SFXGE_EVQ_STARTED"));
  728 
  729         SFXGE_EVQ_LOCK(evq);
  730         evq->init_state = SFXGE_EVQ_INITIALIZED;
  731         evq->read_ptr = 0;
  732         evq->exception = B_FALSE;
  733 
  734 #if EFSYS_OPT_QSTATS
  735         /* Add event counts before discarding the common evq state */
  736         efx_ev_qstats_update(evq->common, evq->stats);
  737 #endif
  738 
  739         efx_ev_qdestroy(evq->common);
  740         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
  741             EFX_EVQ_NBUFS(evq->entries));
  742         SFXGE_EVQ_UNLOCK(evq);
  743 }
  744 
  745 static int
  746 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
  747 {
  748         struct sfxge_evq *evq;
  749         efsys_mem_t *esmp;
  750         int count;
  751         int rc;
  752 
  753         evq = sc->evq[index];
  754         esmp = &evq->mem;
  755 
  756         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
  757             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
  758 
  759         /* Clear all events. */
  760         (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));
  761 
  762         /* Program the buffer table. */
  763         if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
  764             EFX_EVQ_NBUFS(evq->entries))) != 0)
  765                 return (rc);
  766 
  767         /* Create the common code event queue. */
  768         if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,
  769             evq->buf_base_id, sc->ev_moderation, EFX_EVQ_FLAGS_TYPE_AUTO,
  770             &evq->common)) != 0)
  771                 goto fail;
  772 
  773         SFXGE_EVQ_LOCK(evq);
  774 
  775         /* Prime the event queue for interrupts */
  776         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
  777                 goto fail2;
  778 
  779         evq->init_state = SFXGE_EVQ_STARTING;
  780 
  781         SFXGE_EVQ_UNLOCK(evq);
  782 
  783         /* Wait for the initialization event */
  784         count = 0;
  785         do {
  786                 /* Pause for 100 ms */
  787                 pause("sfxge evq init", hz / 10);
  788 
  789                 /* Check to see if the test event has been processed */
  790                 if (evq->init_state == SFXGE_EVQ_STARTED)
  791                         goto done;
  792 
  793         } while (++count < 20);
  794 
  795         rc = ETIMEDOUT;
  796         goto fail3;
  797 
  798 done:
  799         return (0);
  800 
  801 fail3:
  802         SFXGE_EVQ_LOCK(evq);
  803         evq->init_state = SFXGE_EVQ_INITIALIZED;
  804 fail2:
  805         SFXGE_EVQ_UNLOCK(evq);
  806         efx_ev_qdestroy(evq->common);
  807 fail:
  808         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
  809             EFX_EVQ_NBUFS(evq->entries));
  810 
  811         return (rc);
  812 }
  813 
  814 void
  815 sfxge_ev_stop(struct sfxge_softc *sc)
  816 {
  817         struct sfxge_intr *intr __diagused;
  818         efx_nic_t *enp;
  819         int index;
  820 
  821         intr = &sc->intr;
  822         enp = sc->enp;
  823 
  824         KASSERT(intr->state == SFXGE_INTR_STARTED,
  825             ("Interrupts not started"));
  826 
  827         /* Stop the event queue(s) */
  828         index = sc->evq_count;
  829         while (--index >= 0)
  830                 sfxge_ev_qstop(sc, index);
  831 
  832         /* Tear down the event module */
  833         efx_ev_fini(enp);
  834 }
  835 
  836 int
  837 sfxge_ev_start(struct sfxge_softc *sc)
  838 {
  839         struct sfxge_intr *intr __diagused;
  840         int index;
  841         int rc;
  842 
  843         intr = &sc->intr;
  844 
  845         KASSERT(intr->state == SFXGE_INTR_STARTED,
  846             ("intr->state != SFXGE_INTR_STARTED"));
  847 
  848         /* Initialize the event module */
  849         if ((rc = efx_ev_init(sc->enp)) != 0)
  850                 return (rc);
  851 
  852         /* Start the event queues */
  853         for (index = 0; index < sc->evq_count; index++) {
  854                 if ((rc = sfxge_ev_qstart(sc, index)) != 0)
  855                         goto fail;
  856         }
  857 
  858         return (0);
  859 
  860 fail:
  861         /* Stop the event queue(s) */
  862         while (--index >= 0)
  863                 sfxge_ev_qstop(sc, index);
  864 
  865         /* Tear down the event module */
  866         efx_ev_fini(sc->enp);
  867 
  868         return (rc);
  869 }
  870 
  871 static void
  872 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
  873 {
  874         struct sfxge_evq *evq;
  875 
  876         evq = sc->evq[index];
  877 
  878         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
  879             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
  880         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
  881 
  882         sfxge_dma_free(&evq->mem);
  883 
  884         sc->evq[index] = NULL;
  885 
  886         SFXGE_EVQ_LOCK_DESTROY(evq);
  887 
  888         free(evq, M_SFXGE);
  889 }
  890 
  891 static int
  892 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
  893 {
  894         struct sfxge_evq *evq;
  895         efsys_mem_t *esmp;
  896         int rc;
  897 
  898         KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
  899 
  900         evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
  901         evq->sc = sc;
  902         evq->index = index;
  903         sc->evq[index] = evq;
  904         esmp = &evq->mem;
  905 
  906         /* Build an event queue with room for one event per tx and rx buffer,
  907          * plus some extra for link state events and MCDI completions.
  908          * There are three tx queues in the first event queue and one in
  909          * other.
  910          */
  911         if (index == 0)
  912                 evq->entries =
  913                         ROUNDUP_POW_OF_TWO(sc->rxq_entries +
  914                                            3 * sc->txq_entries +
  915                                            128);
  916         else
  917                 evq->entries =
  918                         ROUNDUP_POW_OF_TWO(sc->rxq_entries +
  919                                            sc->txq_entries +
  920                                            128);
  921 
  922         /* Initialise TX completion list */
  923         evq->txqs = &evq->txq;
  924 
  925         /* Allocate DMA space. */
  926         if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)
  927                 return (rc);
  928 
  929         /* Allocate buffer table entries. */
  930         sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),
  931                                  &evq->buf_base_id);
  932 
  933         SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index);
  934 
  935         evq->init_state = SFXGE_EVQ_INITIALIZED;
  936 
  937 #if EFSYS_OPT_QSTATS
  938         rc = sfxge_evq_stat_init(evq);
  939         if (rc != 0)
  940                 goto fail_evq_stat_init;
  941 #endif
  942 
  943         return (0);
  944 
  945 #if EFSYS_OPT_QSTATS
  946 fail_evq_stat_init:
  947         evq->init_state = SFXGE_EVQ_UNINITIALIZED;
  948         SFXGE_EVQ_LOCK_DESTROY(evq);
  949         sfxge_dma_free(esmp);
  950         sc->evq[index] = NULL;
  951         free(evq, M_SFXGE);
  952 
  953         return (rc);
  954 #endif
  955 }
  956 
  957 void
  958 sfxge_ev_fini(struct sfxge_softc *sc)
  959 {
  960         struct sfxge_intr *intr __diagused;
  961         int index;
  962 
  963         intr = &sc->intr;
  964 
  965         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
  966             ("intr->state != SFXGE_INTR_INITIALIZED"));
  967 
  968         sc->ev_moderation = 0;
  969 
  970         /* Tear down the event queue(s). */
  971         index = sc->evq_count;
  972         while (--index >= 0)
  973                 sfxge_ev_qfini(sc, index);
  974 
  975         sc->evq_count = 0;
  976 }
  977 
  978 int
  979 sfxge_ev_init(struct sfxge_softc *sc)
  980 {
  981         struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
  982         struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
  983         struct sfxge_intr *intr;
  984         int index;
  985         int rc;
  986 
  987         intr = &sc->intr;
  988 
  989         sc->evq_count = intr->n_alloc;
  990 
  991         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
  992             ("intr->state != SFXGE_INTR_INITIALIZED"));
  993 
  994         /* Set default interrupt moderation; add a sysctl to
  995          * read and change it.
  996          */
  997         sc->ev_moderation = SFXGE_MODERATION;
  998         SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
  999             OID_AUTO, "int_mod", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
 1000             sc, 0, sfxge_int_mod_handler, "IU",
 1001             "sfxge interrupt moderation (us)");
 1002 
 1003 #if EFSYS_OPT_QSTATS
 1004         sc->evqs_stats_node = SYSCTL_ADD_NODE(
 1005             device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(sc->stats_node),
 1006             OID_AUTO, "evq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
 1007             "Event queues stats");
 1008         if (sc->evqs_stats_node == NULL) {
 1009                 rc = ENOMEM;
 1010                 goto fail_evqs_stats_node;
 1011         }
 1012 #endif
 1013 
 1014         /*
 1015          * Initialize the event queue(s) - one per interrupt.
 1016          */
 1017         for (index = 0; index < sc->evq_count; index++) {
 1018                 if ((rc = sfxge_ev_qinit(sc, index)) != 0)
 1019                         goto fail;
 1020         }
 1021 
 1022 #if EFSYS_OPT_QSTATS
 1023         sfxge_ev_stat_init(sc);
 1024 #endif
 1025 
 1026         return (0);
 1027 
 1028 fail:
 1029         while (--index >= 0)
 1030                 sfxge_ev_qfini(sc, index);
 1031 
 1032 #if EFSYS_OPT_QSTATS
 1033 fail_evqs_stats_node:
 1034 #endif
 1035         sc->evq_count = 0;
 1036         return (rc);
 1037 }
Cache object: 82a125ce5e8d0b4222955cfec2fe17c8
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/sfxge/sfxge_ev.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/sfxge/sfxge_ev.c
