sfxge_ev.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * Copyright (c) 2010-2016 Solarflare Communications Inc.
    3  * All rights reserved.
    4  *
    5  * This software was developed in part by Philip Paeps under contract for
    6  * Solarflare Communications, Inc.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions are met:
   10  *
   11  * 1. Redistributions of source code must retain the above copyright notice,
   12  *    this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright notice,
   14  *    this list of conditions and the following disclaimer in the documentation
   15  *    and/or other materials provided with the distribution.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   18  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
   19  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
   20  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
   21  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
   22  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
   23  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
   24  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
   25  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
   26  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
   27  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   28  *
   29  * The views and conclusions contained in the software and documentation are
   30  * those of the authors and should not be interpreted as representing official
   31  * policies, either expressed or implied, of the FreeBSD Project.
   32  */
   33 
   34 #include <sys/cdefs.h>
   35 __FBSDID("$FreeBSD: releng/11.2/sys/dev/sfxge/sfxge_ev.c 331722 2018-03-29 02:50:57Z eadler $");
   36 
   37 #include <sys/param.h>
   38 #include <sys/kernel.h>
   39 #include <sys/malloc.h>
   40 #include <sys/param.h>
   41 #include <sys/queue.h>
   42 #include <sys/systm.h>
   43 #include <sys/taskqueue.h>
   44 
   45 #include "common/efx.h"
   46 
   47 #include "sfxge.h"
   48 
   49 static void
   50 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
   51 {
   52         struct sfxge_softc *sc;
   53         unsigned int index;
   54         struct sfxge_rxq *rxq;
   55         struct sfxge_txq *txq;
   56 
   57         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
   58 
   59         sc = evq->sc;
   60         index = evq->index;
   61         rxq = sc->rxq[index];
   62 
   63         if ((txq = evq->txq) != NULL) {
   64                 evq->txq = NULL;
   65                 evq->txqs = &(evq->txq);
   66 
   67                 do {
   68                         struct sfxge_txq *next;
   69 
   70                         next = txq->next;
   71                         txq->next = NULL;
   72 
   73                         KASSERT(txq->evq_index == index,
   74                             ("txq->evq_index != index"));
   75 
   76                         if (txq->pending != txq->completed)
   77                                 sfxge_tx_qcomplete(txq, evq);
   78 
   79                         txq = next;
   80                 } while (txq != NULL);
   81         }
   82 
   83         if (rxq->pending != rxq->completed)
   84                 sfxge_rx_qcomplete(rxq, eop);
   85 }
   86 
   87 static struct sfxge_rxq *
   88 sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label)
   89 {
   90         struct sfxge_rxq *rxq;
   91 
   92         KASSERT(label == 0, ("unexpected rxq label != 0"));
   93 
   94         rxq = evq->sc->rxq[evq->index];
   95 
   96         KASSERT(rxq != NULL, ("rxq == NULL"));
   97         KASSERT(evq->index == rxq->index, ("evq->index != rxq->index"));
   98 
   99         return (rxq);
  100 }
  101 
  102 static boolean_t
  103 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
  104             uint16_t flags)
  105 {
  106         struct sfxge_evq *evq;
  107         struct sfxge_softc *sc;
  108         struct sfxge_rxq *rxq;
  109         unsigned int stop;
  110         unsigned int delta;
  111         struct sfxge_rx_sw_desc *rx_desc;
  112 
  113         evq = arg;
  114         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  115 
  116         sc = evq->sc;
  117 
  118         if (evq->exception)
  119                 goto done;
  120 
  121         rxq = sfxge_get_rxq_by_label(evq, label);
  122         if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
  123                 goto done;
  124 
  125         stop = (id + 1) & rxq->ptr_mask;
  126         id = rxq->pending & rxq->ptr_mask;
  127         delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop);
  128         rxq->pending += delta;
  129 
  130         if (delta != 1) {
  131                 if ((delta <= 0) ||
  132                     (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) {
  133                         evq->exception = B_TRUE;
  134 
  135                         device_printf(sc->dev, "RX completion out of order"
  136                                                   " (id=%#x delta=%u flags=%#x); resetting\n",
  137                                                   id, delta, flags);
  138                         sfxge_schedule_reset(sc);
  139 
  140                         goto done;
  141                 }
  142         }
  143 
  144         rx_desc = &rxq->queue[id];
  145 
  146         prefetch_read_many(rx_desc->mbuf);
  147 
  148         for (; id != stop; id = (id + 1) & rxq->ptr_mask) {
  149                 rx_desc = &rxq->queue[id];
  150                 KASSERT(rx_desc->flags == EFX_DISCARD,
  151                                 ("rx_desc->flags != EFX_DISCARD"));
  152                 rx_desc->flags = flags;
  153 
  154                 KASSERT(size < (1 << 16), ("size > (1 << 16)"));
  155                 rx_desc->size = (uint16_t)size;
  156         }
  157 
  158         evq->rx_done++;
  159 
  160         if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
  161                 sfxge_ev_qcomplete(evq, B_FALSE);
  162 
  163 done:
  164         return (evq->rx_done >= SFXGE_EV_BATCH);
  165 }
  166 
  167 static boolean_t
  168 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
  169 {
  170         struct sfxge_evq *evq;
  171         struct sfxge_softc *sc;
  172 
  173         evq = (struct sfxge_evq *)arg;
  174         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  175 
  176         sc = evq->sc;
  177 
  178         DBGPRINT(sc->dev, "[%d] %s", evq->index,
  179                           (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" :
  180                           (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" :
  181                           (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" :
  182                           (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" :
  183                           (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" :
  184                           (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" :
  185                           (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" :
  186                           (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" :
  187                           (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" :
  188                           "UNKNOWN");
  189 
  190         evq->exception = B_TRUE;
  191 
  192         if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
  193                 device_printf(sc->dev,
  194                               "hardware exception (code=%u); resetting\n",
  195                               code);
  196                 sfxge_schedule_reset(sc);
  197         }
  198 
  199         return (B_FALSE);
  200 }
  201 
  202 static boolean_t
  203 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
  204 {
  205         struct sfxge_evq *evq;
  206         struct sfxge_softc *sc;
  207         struct sfxge_rxq *rxq;
  208         unsigned int index;
  209         uint16_t magic;
  210 
  211         evq = (struct sfxge_evq *)arg;
  212         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  213 
  214         sc = evq->sc;
  215         rxq = sc->rxq[rxq_index];
  216 
  217         KASSERT(rxq != NULL, ("rxq == NULL"));
  218 
  219         /* Resend a software event on the correct queue */
  220         index = rxq->index;
  221         if (index == evq->index) {
  222                 sfxge_rx_qflush_done(rxq);
  223                 return (B_FALSE);
  224         }
  225 
  226         evq = sc->evq[index];
  227         magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq);
  228 
  229         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
  230             ("evq not started"));
  231         efx_ev_qpost(evq->common, magic);
  232 
  233         return (B_FALSE);
  234 }
  235 
  236 static boolean_t
  237 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
  238 {
  239         struct sfxge_evq *evq;
  240         struct sfxge_softc *sc;
  241         struct sfxge_rxq *rxq;
  242         unsigned int index;
  243         uint16_t magic;
  244 
  245         evq = (struct sfxge_evq *)arg;
  246         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  247 
  248         sc = evq->sc;
  249         rxq = sc->rxq[rxq_index];
  250 
  251         KASSERT(rxq != NULL, ("rxq == NULL"));
  252 
  253         /* Resend a software event on the correct queue */
  254         index = rxq->index;
  255         evq = sc->evq[index];
  256         magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq);
  257 
  258         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
  259             ("evq not started"));
  260         efx_ev_qpost(evq->common, magic);
  261 
  262         return (B_FALSE);
  263 }
  264 
  265 static struct sfxge_txq *
  266 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)
  267 {
  268         unsigned int index;
  269 
  270         KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||
  271             (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label"));
  272         index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES);
  273         return (evq->sc->txq[index]);
  274 }
  275 
  276 static boolean_t
  277 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
  278 {
  279         struct sfxge_evq *evq;
  280         struct sfxge_txq *txq;
  281         unsigned int stop;
  282         unsigned int delta;
  283 
  284         evq = (struct sfxge_evq *)arg;
  285         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  286 
  287         txq = sfxge_get_txq_by_label(evq, label);
  288 
  289         KASSERT(txq != NULL, ("txq == NULL"));
  290         KASSERT(evq->index == txq->evq_index,
  291             ("evq->index != txq->evq_index"));
  292 
  293         if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED))
  294                 goto done;
  295 
  296         stop = (id + 1) & txq->ptr_mask;
  297         id = txq->pending & txq->ptr_mask;
  298 
  299         delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop);
  300         txq->pending += delta;
  301 
  302         evq->tx_done++;
  303 
  304         if (txq->next == NULL &&
  305             evq->txqs != &(txq->next)) {
  306                 *(evq->txqs) = txq;
  307                 evq->txqs = &(txq->next);
  308         }
  309 
  310         if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
  311                 sfxge_tx_qcomplete(txq, evq);
  312 
  313 done:
  314         return (evq->tx_done >= SFXGE_EV_BATCH);
  315 }
  316 
  317 static boolean_t
  318 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
  319 {
  320         struct sfxge_evq *evq;
  321         struct sfxge_softc *sc;
  322         struct sfxge_txq *txq;
  323         uint16_t magic;
  324 
  325         evq = (struct sfxge_evq *)arg;
  326         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  327 
  328         sc = evq->sc;
  329         txq = sc->txq[txq_index];
  330 
  331         KASSERT(txq != NULL, ("txq == NULL"));
  332         KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
  333             ("txq not initialized"));
  334 
  335         if (txq->evq_index == evq->index) {
  336                 sfxge_tx_qflush_done(txq);
  337                 return (B_FALSE);
  338         }
  339 
  340         /* Resend a software event on the correct queue */
  341         evq = sc->evq[txq->evq_index];
  342         magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq);
  343 
  344         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
  345             ("evq not started"));
  346         efx_ev_qpost(evq->common, magic);
  347 
  348         return (B_FALSE);
  349 }
  350 
  351 static boolean_t
  352 sfxge_ev_software(void *arg, uint16_t magic)
  353 {
  354         struct sfxge_evq *evq;
  355         struct sfxge_softc *sc;
  356         unsigned int label;
  357 
  358         evq = (struct sfxge_evq *)arg;
  359         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  360 
  361         sc = evq->sc;
  362 
  363         label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
  364         magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
  365 
  366         switch (magic) {
  367         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE):
  368                 sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label));
  369                 break;
  370 
  371         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED):
  372                 sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label));
  373                 break;
  374 
  375         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL):
  376                 sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label));
  377                 break;
  378 
  379         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): {
  380                 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);
  381 
  382                 KASSERT(txq != NULL, ("txq == NULL"));
  383                 KASSERT(evq->index == txq->evq_index,
  384                     ("evq->index != txq->evq_index"));
  385 
  386                 sfxge_tx_qflush_done(txq);
  387                 break;
  388         }
  389         default:
  390                 break;
  391         }
  392 
  393         return (B_FALSE);
  394 }
  395 
  396 static boolean_t
  397 sfxge_ev_sram(void *arg, uint32_t code)
  398 {
  399         (void)arg;
  400         (void)code;
  401 
  402         switch (code) {
  403         case EFX_SRAM_UPDATE:
  404                 EFSYS_PROBE(sram_update);
  405                 break;
  406 
  407         case EFX_SRAM_CLEAR:
  408                 EFSYS_PROBE(sram_clear);
  409                 break;
  410 
  411         case EFX_SRAM_ILLEGAL_CLEAR:
  412                 EFSYS_PROBE(sram_illegal_clear);
  413                 break;
  414 
  415         default:
  416                 KASSERT(B_FALSE, ("Impossible SRAM event"));
  417                 break;
  418         }
  419 
  420         return (B_FALSE);
  421 }
  422 
  423 static boolean_t
  424 sfxge_ev_timer(void *arg, uint32_t index)
  425 {
  426         (void)arg;
  427         (void)index;
  428 
  429         return (B_FALSE);
  430 }
  431 
  432 static boolean_t
  433 sfxge_ev_wake_up(void *arg, uint32_t index)
  434 {
  435         (void)arg;
  436         (void)index;
  437 
  438         return (B_FALSE);
  439 }
  440 
  441 #if EFSYS_OPT_QSTATS
  442 
  443 static void
  444 sfxge_ev_stat_update(struct sfxge_softc *sc)
  445 {
  446         struct sfxge_evq *evq;
  447         unsigned int index;
  448         clock_t now;
  449 
  450         SFXGE_ADAPTER_LOCK(sc);
  451 
  452         if (__predict_false(sc->evq[0]->init_state != SFXGE_EVQ_STARTED))
  453                 goto out;
  454 
  455         now = ticks;
  456         if ((unsigned int)(now - sc->ev_stats_update_time) < (unsigned int)hz)
  457                 goto out;
  458 
  459         sc->ev_stats_update_time = now;
  460 
  461         /* Add event counts from each event queue in turn */
  462         for (index = 0; index < sc->evq_count; index++) {
  463                 evq = sc->evq[index];
  464                 SFXGE_EVQ_LOCK(evq);
  465                 efx_ev_qstats_update(evq->common, sc->ev_stats);
  466                 SFXGE_EVQ_UNLOCK(evq);
  467         }
  468 out:
  469         SFXGE_ADAPTER_UNLOCK(sc);
  470 }
  471 
  472 static int
  473 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
  474 {
  475         struct sfxge_softc *sc = arg1;
  476         unsigned int id = arg2;
  477 
  478         sfxge_ev_stat_update(sc);
  479 
  480         return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])));
  481 }
  482 
  483 static void
  484 sfxge_ev_stat_init(struct sfxge_softc *sc)
  485 {
  486         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
  487         struct sysctl_oid_list *stat_list;
  488         unsigned int id;
  489         char name[40];
  490 
  491         stat_list = SYSCTL_CHILDREN(sc->stats_node);
  492 
  493         for (id = 0; id < EV_NQSTATS; id++) {
  494                 snprintf(name, sizeof(name), "ev_%s",
  495                          efx_ev_qstat_name(sc->enp, id));
  496                 SYSCTL_ADD_PROC(
  497                         ctx, stat_list,
  498                         OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
  499                         sc, id, sfxge_ev_stat_handler, "Q",
  500                         "");
  501         }
  502 }
  503 
  504 #endif /* EFSYS_OPT_QSTATS */
  505 
  506 static void
  507 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
  508 {
  509         struct sfxge_evq *evq;
  510         efx_evq_t *eep;
  511 
  512         evq = sc->evq[idx];
  513         eep = evq->common;
  514 
  515         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
  516             ("evq->init_state != SFXGE_EVQ_STARTED"));
  517 
  518         (void)efx_ev_qmoderate(eep, us);
  519 }
  520 
  521 static int
  522 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
  523 {
  524         struct sfxge_softc *sc = arg1;
  525         struct sfxge_intr *intr = &sc->intr;
  526         unsigned int moderation;
  527         int error;
  528         unsigned int index;
  529 
  530         SFXGE_ADAPTER_LOCK(sc);
  531 
  532         if (req->newptr != NULL) {
  533                 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
  534                     != 0)
  535                         goto out;
  536 
  537                 /* We may not be calling efx_ev_qmoderate() now,
  538                  * so we have to range-check the value ourselves.
  539                  */
  540                 if (moderation >
  541                     efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) {
  542                         error = EINVAL;
  543                         goto out;
  544                 }
  545 
  546                 sc->ev_moderation = moderation;
  547                 if (intr->state == SFXGE_INTR_STARTED) {
  548                         for (index = 0; index < sc->evq_count; index++)
  549                                 sfxge_ev_qmoderate(sc, index, moderation);
  550                 }
  551         } else {
  552                 error = SYSCTL_OUT(req, &sc->ev_moderation,
  553                                    sizeof(sc->ev_moderation));
  554         }
  555 
  556 out:
  557         SFXGE_ADAPTER_UNLOCK(sc);
  558 
  559         return (error);
  560 }
  561 
  562 static boolean_t
  563 sfxge_ev_initialized(void *arg)
  564 {
  565         struct sfxge_evq *evq;
  566 
  567         evq = (struct sfxge_evq *)arg;
  568         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  569 
  570         /* Init done events may be duplicated on 7xxx */
  571         KASSERT(evq->init_state == SFXGE_EVQ_STARTING ||
  572                 evq->init_state == SFXGE_EVQ_STARTED,
  573             ("evq not starting"));
  574 
  575         evq->init_state = SFXGE_EVQ_STARTED;
  576 
  577         return (0);
  578 }
  579 
  580 static boolean_t
  581 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
  582 {
  583         struct sfxge_evq *evq;
  584         struct sfxge_softc *sc;
  585 
  586         evq = (struct sfxge_evq *)arg;
  587         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  588 
  589         sc = evq->sc;
  590 
  591         sfxge_mac_link_update(sc, link_mode);
  592 
  593         return (0);
  594 }
  595 
  596 static const efx_ev_callbacks_t sfxge_ev_callbacks = {
  597         .eec_initialized        = sfxge_ev_initialized,
  598         .eec_rx                 = sfxge_ev_rx,
  599         .eec_tx                 = sfxge_ev_tx,
  600         .eec_exception          = sfxge_ev_exception,
  601         .eec_rxq_flush_done     = sfxge_ev_rxq_flush_done,
  602         .eec_rxq_flush_failed   = sfxge_ev_rxq_flush_failed,
  603         .eec_txq_flush_done     = sfxge_ev_txq_flush_done,
  604         .eec_software           = sfxge_ev_software,
  605         .eec_sram               = sfxge_ev_sram,
  606         .eec_wake_up            = sfxge_ev_wake_up,
  607         .eec_timer              = sfxge_ev_timer,
  608         .eec_link_change        = sfxge_ev_link_change,
  609 };
  610 
  611 
  612 int
  613 sfxge_ev_qpoll(struct sfxge_evq *evq)
  614 {
  615         int rc;
  616 
  617         SFXGE_EVQ_LOCK(evq);
  618 
  619         if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING &&
  620                             evq->init_state != SFXGE_EVQ_STARTED)) {
  621                 rc = EINVAL;
  622                 goto fail;
  623         }
  624 
  625         /* Synchronize the DMA memory for reading */
  626         bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
  627             BUS_DMASYNC_POSTREAD);
  628 
  629         KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
  630         KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
  631         KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
  632         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
  633 
  634         /* Poll the queue */
  635         efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
  636 
  637         evq->rx_done = 0;
  638         evq->tx_done = 0;
  639 
  640         /* Perform any pending completion processing */
  641         sfxge_ev_qcomplete(evq, B_TRUE);
  642 
  643         /* Re-prime the event queue for interrupts */
  644         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
  645                 goto fail;
  646 
  647         SFXGE_EVQ_UNLOCK(evq);
  648 
  649         return (0);
  650 
  651 fail:
  652         SFXGE_EVQ_UNLOCK(evq);
  653         return (rc);
  654 }
  655 
  656 static void
  657 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
  658 {
  659         struct sfxge_evq *evq;
  660 
  661         evq = sc->evq[index];
  662 
  663         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
  664             ("evq->init_state != SFXGE_EVQ_STARTED"));
  665 
  666         SFXGE_EVQ_LOCK(evq);
  667         evq->init_state = SFXGE_EVQ_INITIALIZED;
  668         evq->read_ptr = 0;
  669         evq->exception = B_FALSE;
  670 
  671 #if EFSYS_OPT_QSTATS
  672         /* Add event counts before discarding the common evq state */
  673         efx_ev_qstats_update(evq->common, sc->ev_stats);
  674 #endif
  675 
  676         efx_ev_qdestroy(evq->common);
  677         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
  678             EFX_EVQ_NBUFS(evq->entries));
  679         SFXGE_EVQ_UNLOCK(evq);
  680 }
  681 
  682 static int
  683 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
  684 {
  685         struct sfxge_evq *evq;
  686         efsys_mem_t *esmp;
  687         int count;
  688         int rc;
  689 
  690         evq = sc->evq[index];
  691         esmp = &evq->mem;
  692 
  693         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
  694             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
  695 
  696         /* Clear all events. */
  697         (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));
  698 
  699         /* Program the buffer table. */
  700         if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
  701             EFX_EVQ_NBUFS(evq->entries))) != 0)
  702                 return (rc);
  703 
  704         /* Create the common code event queue. */
  705         if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,
  706             evq->buf_base_id, sc->ev_moderation, EFX_EVQ_FLAGS_TYPE_AUTO,
  707             &evq->common)) != 0)
  708                 goto fail;
  709 
  710         SFXGE_EVQ_LOCK(evq);
  711 
  712         /* Prime the event queue for interrupts */
  713         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
  714                 goto fail2;
  715 
  716         evq->init_state = SFXGE_EVQ_STARTING;
  717 
  718         SFXGE_EVQ_UNLOCK(evq);
  719 
  720         /* Wait for the initialization event */
  721         count = 0;
  722         do {
  723                 /* Pause for 100 ms */
  724                 pause("sfxge evq init", hz / 10);
  725 
  726                 /* Check to see if the test event has been processed */
  727                 if (evq->init_state == SFXGE_EVQ_STARTED)
  728                         goto done;
  729 
  730         } while (++count < 20);
  731 
  732         rc = ETIMEDOUT;
  733         goto fail3;
  734 
  735 done:
  736         return (0);
  737 
  738 fail3:
  739         SFXGE_EVQ_LOCK(evq);
  740         evq->init_state = SFXGE_EVQ_INITIALIZED;
  741 fail2:
  742         SFXGE_EVQ_UNLOCK(evq);
  743         efx_ev_qdestroy(evq->common);
  744 fail:
  745         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
  746             EFX_EVQ_NBUFS(evq->entries));
  747 
  748         return (rc);
  749 }
  750 
  751 void
  752 sfxge_ev_stop(struct sfxge_softc *sc)
  753 {
  754         struct sfxge_intr *intr;
  755         efx_nic_t *enp;
  756         int index;
  757 
  758         intr = &sc->intr;
  759         enp = sc->enp;
  760 
  761         KASSERT(intr->state == SFXGE_INTR_STARTED,
  762             ("Interrupts not started"));
  763 
  764         /* Stop the event queue(s) */
  765         index = sc->evq_count;
  766         while (--index >= 0)
  767                 sfxge_ev_qstop(sc, index);
  768 
  769         /* Tear down the event module */
  770         efx_ev_fini(enp);
  771 }
  772 
  773 int
  774 sfxge_ev_start(struct sfxge_softc *sc)
  775 {
  776         struct sfxge_intr *intr;
  777         int index;
  778         int rc;
  779 
  780         intr = &sc->intr;
  781 
  782         KASSERT(intr->state == SFXGE_INTR_STARTED,
  783             ("intr->state != SFXGE_INTR_STARTED"));
  784 
  785         /* Initialize the event module */
  786         if ((rc = efx_ev_init(sc->enp)) != 0)
  787                 return (rc);
  788 
  789         /* Start the event queues */
  790         for (index = 0; index < sc->evq_count; index++) {
  791                 if ((rc = sfxge_ev_qstart(sc, index)) != 0)
  792                         goto fail;
  793         }
  794 
  795         return (0);
  796 
  797 fail:
  798         /* Stop the event queue(s) */
  799         while (--index >= 0)
  800                 sfxge_ev_qstop(sc, index);
  801 
  802         /* Tear down the event module */
  803         efx_ev_fini(sc->enp);
  804 
  805         return (rc);
  806 }
  807 
  808 static void
  809 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
  810 {
  811         struct sfxge_evq *evq;
  812 
  813         evq = sc->evq[index];
  814 
  815         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
  816             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
  817         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
  818 
  819         sfxge_dma_free(&evq->mem);
  820 
  821         sc->evq[index] = NULL;
  822 
  823         SFXGE_EVQ_LOCK_DESTROY(evq);
  824 
  825         free(evq, M_SFXGE);
  826 }
  827 
  828 static int
  829 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
  830 {
  831         struct sfxge_evq *evq;
  832         efsys_mem_t *esmp;
  833         int rc;
  834 
  835         KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
  836 
  837         evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
  838         evq->sc = sc;
  839         evq->index = index;
  840         sc->evq[index] = evq;
  841         esmp = &evq->mem;
  842 
  843         /* Build an event queue with room for one event per tx and rx buffer,
  844          * plus some extra for link state events and MCDI completions.
  845          * There are three tx queues in the first event queue and one in
  846          * other.
  847          */
  848         if (index == 0)
  849                 evq->entries =
  850                         ROUNDUP_POW_OF_TWO(sc->rxq_entries +
  851                                            3 * sc->txq_entries +
  852                                            128);
  853         else
  854                 evq->entries =
  855                         ROUNDUP_POW_OF_TWO(sc->rxq_entries +
  856                                            sc->txq_entries +
  857                                            128);
  858 
  859         /* Initialise TX completion list */
  860         evq->txqs = &evq->txq;
  861 
  862         /* Allocate DMA space. */
  863         if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)
  864                 return (rc);
  865 
  866         /* Allocate buffer table entries. */
  867         sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),
  868                                  &evq->buf_base_id);
  869 
  870         SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index);
  871 
  872         evq->init_state = SFXGE_EVQ_INITIALIZED;
  873 
  874         return (0);
  875 }
  876 
  877 void
  878 sfxge_ev_fini(struct sfxge_softc *sc)
  879 {
  880         struct sfxge_intr *intr;
  881         int index;
  882 
  883         intr = &sc->intr;
  884 
  885         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
  886             ("intr->state != SFXGE_INTR_INITIALIZED"));
  887 
  888         sc->ev_moderation = 0;
  889 
  890         /* Tear down the event queue(s). */
  891         index = sc->evq_count;
  892         while (--index >= 0)
  893                 sfxge_ev_qfini(sc, index);
  894 
  895         sc->evq_count = 0;
  896 }
  897 
  898 int
  899 sfxge_ev_init(struct sfxge_softc *sc)
  900 {
  901         struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
  902         struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
  903         struct sfxge_intr *intr;
  904         int index;
  905         int rc;
  906 
  907         intr = &sc->intr;
  908 
  909         sc->evq_count = intr->n_alloc;
  910 
  911         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
  912             ("intr->state != SFXGE_INTR_INITIALIZED"));
  913 
  914         /* Set default interrupt moderation; add a sysctl to
  915          * read and change it.
  916          */
  917         sc->ev_moderation = SFXGE_MODERATION;
  918         SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
  919                         OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW,
  920                         sc, 0, sfxge_int_mod_handler, "IU",
  921                         "sfxge interrupt moderation (us)");
  922 
  923         /*
  924          * Initialize the event queue(s) - one per interrupt.
  925          */
  926         for (index = 0; index < sc->evq_count; index++) {
  927                 if ((rc = sfxge_ev_qinit(sc, index)) != 0)
  928                         goto fail;
  929         }
  930 
  931 #if EFSYS_OPT_QSTATS
  932         sfxge_ev_stat_init(sc);
  933 #endif
  934 
  935         return (0);
  936 
  937 fail:
  938         while (--index >= 0)
  939                 sfxge_ev_qfini(sc, index);
  940 
  941         sc->evq_count = 0;
  942         return (rc);
  943 }
Cache object: 5ea1df4b541bc497136249b7ca692663
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/sfxge/sfxge_ev.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/sfxge/sfxge_ev.c
