FreeBSD/Linux Kernel Cross Reference
sys/dev/ice/ice_iflib_txrx.c

     /* SPDX-License-Identifier: BSD-3-Clause */
     /*  Copyright (c) 2021, Intel Corporation
      *  All rights reserved.
      *
      *  Redistribution and use in source and binary forms, with or without
      *  modification, are permitted provided that the following conditions are met:
      *
      *   1. Redistributions of source code must retain the above copyright notice,
      *      this list of conditions and the following disclaimer.
     *
     *   2. Redistributions in binary form must reproduce the above copyright
     *      notice, this list of conditions and the following disclaimer in the
     *      documentation and/or other materials provided with the distribution.
     *
     *   3. Neither the name of the Intel Corporation nor the names of its
     *      contributors may be used to endorse or promote products derived from
     *      this software without specific prior written permission.
     *
     *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
     *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
     *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     *  POSSIBILITY OF SUCH DAMAGE.
     */
    /*$FreeBSD$*/
    
    /**
     * @file ice_iflib_txrx.c
     * @brief iflib Tx/Rx hotpath
     *
     * Main location for the iflib Tx/Rx hotpath implementation.
     *
     * Contains the implementation for the iflib function callbacks and the
     * if_txrx ops structure.
     */
    
    #include "ice_iflib.h"
    
    /* Tx/Rx hotpath utility functions */
    #include "ice_common_txrx.h"
    
    /*
     * iflib txrx method declarations
     */
    static int ice_ift_txd_encap(void *arg, if_pkt_info_t pi);
    static int ice_ift_rxd_pkt_get(void *arg, if_rxd_info_t ri);
    static void ice_ift_txd_flush(void *arg, uint16_t txqid, qidx_t pidx);
    static int ice_ift_txd_credits_update(void *arg, uint16_t txqid, bool clear);
    static int ice_ift_rxd_available(void *arg, uint16_t rxqid, qidx_t pidx, qidx_t budget);
    static void ice_ift_rxd_flush(void *arg, uint16_t rxqid, uint8_t flidx, qidx_t pidx);
    static void ice_ift_rxd_refill(void *arg, if_rxd_update_t iru);
    static qidx_t ice_ift_queue_select(void *arg, struct mbuf *m);
    
    /* Macro to help extract the NIC mode flexible Rx descriptor fields from the
     * advanced 32byte Rx descriptors.
     */
    #define RX_FLEX_NIC(desc, field) \
            (((struct ice_32b_rx_flex_desc_nic *)desc)->field)
    
    /**
     * @var ice_txrx
     * @brief Tx/Rx operations for the iflib stack
     *
     * Structure defining the Tx and Rx related operations that iflib can request
     * the driver to perform. These are the main entry points for the hot path of
     * the transmit and receive paths in the iflib driver.
     */
    struct if_txrx ice_txrx = {
            .ift_txd_encap = ice_ift_txd_encap,
            .ift_txd_flush = ice_ift_txd_flush,
            .ift_txd_credits_update = ice_ift_txd_credits_update,
            .ift_rxd_available = ice_ift_rxd_available,
            .ift_rxd_pkt_get = ice_ift_rxd_pkt_get,
            .ift_rxd_refill = ice_ift_rxd_refill,
            .ift_rxd_flush = ice_ift_rxd_flush,
            .ift_txq_select = ice_ift_queue_select,
    };
    
    /**
     * ice_ift_txd_encap - prepare Tx descriptors for a packet
     * @arg: the iflib softc structure pointer
     * @pi: packet info
     *
     * Prepares and encapsulates the given packet into into Tx descriptors, in
     * preparation for sending to the transmit engine. Sets the necessary context
     * descriptors for TSO and other offloads, and prepares the last descriptor
     * for the writeback status.
     *
     * Return 0 on success, non-zero error code on failure.
     */
    static int
    ice_ift_txd_encap(void *arg, if_pkt_info_t pi)
    {
           struct ice_softc *sc = (struct ice_softc *)arg;
           struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[pi->ipi_qsidx];
           int nsegs = pi->ipi_nsegs;
           bus_dma_segment_t *segs = pi->ipi_segs;
           struct ice_tx_desc *txd = NULL;
           int i, j, mask, pidx_last;
           u32 cmd, off;
   
           cmd = off = 0;
           i = pi->ipi_pidx;
   
           /* Set up the TSO/CSUM offload */
           if (pi->ipi_csum_flags & ICE_CSUM_OFFLOAD) {
                   /* Set up the TSO context descriptor if required */
                   if (pi->ipi_csum_flags & CSUM_TSO) {
                           if (ice_tso_detect_sparse(pi))
                                   return (EFBIG);
                           i = ice_tso_setup(txq, pi);
                   }
                   ice_tx_setup_offload(txq, pi, &cmd, &off);
           }
           if (pi->ipi_mflags & M_VLANTAG)
                   cmd |= ICE_TX_DESC_CMD_IL2TAG1;
   
           mask = txq->desc_count - 1;
           for (j = 0; j < nsegs; j++) {
                   bus_size_t seglen;
   
                   txd = &txq->tx_base[i];
                   seglen = segs[j].ds_len;
   
                   txd->buf_addr = htole64(segs[j].ds_addr);
                   txd->cmd_type_offset_bsz =
                       htole64(ICE_TX_DESC_DTYPE_DATA
                       | ((u64)cmd  << ICE_TXD_QW1_CMD_S)
                       | ((u64)off << ICE_TXD_QW1_OFFSET_S)
                       | ((u64)seglen  << ICE_TXD_QW1_TX_BUF_SZ_S)
                       | ((u64)htole16(pi->ipi_vtag) << ICE_TXD_QW1_L2TAG1_S));
   
                   txq->stats.tx_bytes += seglen;
                   pidx_last = i;
                   i = (i+1) & mask;
           }
   
           /* Set the last descriptor for report */
   #define ICE_TXD_CMD (ICE_TX_DESC_CMD_EOP | ICE_TX_DESC_CMD_RS)
           txd->cmd_type_offset_bsz |=
               htole64(((u64)ICE_TXD_CMD << ICE_TXD_QW1_CMD_S));
   
           /* Add to report status array */
           txq->tx_rsq[txq->tx_rs_pidx] = pidx_last;
           txq->tx_rs_pidx = (txq->tx_rs_pidx+1) & mask;
           MPASS(txq->tx_rs_pidx != txq->tx_rs_cidx);
   
           pi->ipi_new_pidx = i;
   
           ++txq->stats.tx_packets;
           return (0);
   }
   
   /**
    * ice_ift_txd_flush - Flush Tx descriptors to hardware
    * @arg: device specific softc pointer
    * @txqid: the Tx queue to flush
    * @pidx: descriptor index to advance tail to
    *
    * Advance the Transmit Descriptor Tail (TDT). This indicates to hardware that
    * frames are available for transmit.
    */
   static void
   ice_ift_txd_flush(void *arg, uint16_t txqid, qidx_t pidx)
   {
           struct ice_softc *sc = (struct ice_softc *)arg;
           struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[txqid];
           struct ice_hw *hw = &sc->hw;
   
           wr32(hw, txq->tail, pidx);
   }
   
   /**
    * ice_ift_txd_credits_update - cleanup Tx descriptors
    * @arg: device private softc
    * @txqid: the Tx queue to update
    * @clear: if false, only report, do not actually clean
    *
    * If clear is false, iflib is asking if we *could* clean up any Tx
    * descriptors.
    *
    * If clear is true, iflib is requesting to cleanup and reclaim used Tx
    * descriptors.
    */
   static int
   ice_ift_txd_credits_update(void *arg, uint16_t txqid, bool clear)
   {
           struct ice_softc *sc = (struct ice_softc *)arg;
           struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[txqid];
   
           qidx_t processed = 0;
           qidx_t cur, prev, ntxd, rs_cidx;
           int32_t delta;
           bool is_done;
   
           rs_cidx = txq->tx_rs_cidx;
           if (rs_cidx == txq->tx_rs_pidx)
                   return (0);
           cur = txq->tx_rsq[rs_cidx];
           MPASS(cur != QIDX_INVALID);
           is_done = ice_is_tx_desc_done(&txq->tx_base[cur]);
   
           if (!is_done)
                   return (0);
           else if (clear == false)
                   return (1);
   
           prev = txq->tx_cidx_processed;
           ntxd = txq->desc_count;
           do {
                   MPASS(prev != cur);
                   delta = (int32_t)cur - (int32_t)prev;
                   if (delta < 0)
                           delta += ntxd;
                   MPASS(delta > 0);
                   processed += delta;
                   prev = cur;
                   rs_cidx = (rs_cidx + 1) & (ntxd-1);
                   if (rs_cidx == txq->tx_rs_pidx)
                           break;
                   cur = txq->tx_rsq[rs_cidx];
                   MPASS(cur != QIDX_INVALID);
                   is_done = ice_is_tx_desc_done(&txq->tx_base[cur]);
           } while (is_done);
   
           txq->tx_rs_cidx = rs_cidx;
           txq->tx_cidx_processed = prev;
   
           return (processed);
   }
   
   /**
    * ice_ift_rxd_available - Return number of available Rx packets
    * @arg: device private softc
    * @rxqid: the Rx queue id
    * @pidx: descriptor start point
    * @budget: maximum Rx budget
    *
    * Determines how many Rx packets are available on the queue, up to a maximum
    * of the given budget.
    */
   static int
   ice_ift_rxd_available(void *arg, uint16_t rxqid, qidx_t pidx, qidx_t budget)
   {
           struct ice_softc *sc = (struct ice_softc *)arg;
           struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[rxqid];
           union ice_32b_rx_flex_desc *rxd;
           uint16_t status0;
           int cnt, i, nrxd;
   
           nrxd = rxq->desc_count;
   
           for (cnt = 0, i = pidx; cnt < nrxd - 1 && cnt < budget;) {
                   rxd = &rxq->rx_base[i];
                   status0 = le16toh(rxd->wb.status_error0);
   
                   if ((status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S)) == 0)
                           break;
                   if (++i == nrxd)
                           i = 0;
                   if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S))
                           cnt++;
           }
   
           return (cnt);
   }
   
   /**
    * ice_ift_rxd_pkt_get - Called by iflib to send data to upper layer
    * @arg: device specific softc
    * @ri: receive packet info
    *
    * This function is called by iflib, and executes in ithread context. It is
    * called by iflib to obtain data which has been DMA'ed into host memory.
    * Returns zero on success, and EBADMSG on failure.
    */
   static int
   ice_ift_rxd_pkt_get(void *arg, if_rxd_info_t ri)
   {
           struct ice_softc *sc = (struct ice_softc *)arg;
           if_softc_ctx_t scctx = sc->scctx;
           struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[ri->iri_qsidx];
           union ice_32b_rx_flex_desc *cur;
           u16 status0, plen, ptype;
           bool eop;
           size_t cidx;
           int i;
   
           cidx = ri->iri_cidx;
           i = 0;
           do {
                   /* 5 descriptor receive limit */
                   MPASS(i < ICE_MAX_RX_SEGS);
   
                   cur = &rxq->rx_base[cidx];
                   status0 = le16toh(cur->wb.status_error0);
                   plen = le16toh(cur->wb.pkt_len) &
                           ICE_RX_FLX_DESC_PKT_LEN_M;
   
                   /* we should never be called without a valid descriptor */
                   MPASS((status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S)) != 0);
   
                   ri->iri_len += plen;
   
                   cur->wb.status_error0 = 0;
                   eop = (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S));
   
                   ri->iri_frags[i].irf_flid = 0;
                   ri->iri_frags[i].irf_idx = cidx;
                   ri->iri_frags[i].irf_len = plen;
                   if (++cidx == rxq->desc_count)
                           cidx = 0;
                   i++;
           } while (!eop);
   
           /* End of Packet reached; cur is eop/last descriptor */
   
           /* Make sure packets with bad L2 values are discarded.
            * This bit is only valid in the last descriptor.
            */
           if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_RXE_S)) {
                   rxq->stats.desc_errs++;
                   return (EBADMSG);
           }
   
           /* Get VLAN tag information if one is in descriptor */
           if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_L2TAG1P_S)) {
                   ri->iri_vtag = le16toh(cur->wb.l2tag1);
                   ri->iri_flags |= M_VLANTAG;
           }
   
           /* Capture soft statistics for this Rx queue */
           rxq->stats.rx_packets++;
           rxq->stats.rx_bytes += ri->iri_len;
   
           /* Get packet type and set checksum flags */
           ptype = le16toh(cur->wb.ptype_flex_flags0) &
                   ICE_RX_FLEX_DESC_PTYPE_M;
           if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0)
                   ice_rx_checksum(rxq, &ri->iri_csum_flags,
                                   &ri->iri_csum_data, status0, ptype);
   
           /* Set remaining iflib RX descriptor info fields */
           ri->iri_flowid = le32toh(RX_FLEX_NIC(&cur->wb, rss_hash));
           ri->iri_rsstype = ice_ptype_to_hash(ptype);
           ri->iri_nfrags = i;
           return (0);
   }
   
   /**
    * ice_ift_rxd_refill - Prepare Rx descriptors for re-use by hardware
    * @arg: device specific softc structure
    * @iru: the Rx descriptor update structure
    *
    * Update the Rx descriptor indices for a given queue, assigning new physical
    * addresses to the descriptors, preparing them for re-use by the hardware.
    */
   static void
   ice_ift_rxd_refill(void *arg, if_rxd_update_t iru)
   {
           struct ice_softc *sc = (struct ice_softc *)arg;
           struct ice_rx_queue *rxq;
           uint32_t next_pidx;
           int i;
           uint64_t *paddrs;
           uint32_t pidx;
           uint16_t qsidx, count;
   
           paddrs = iru->iru_paddrs;
           pidx = iru->iru_pidx;
           qsidx = iru->iru_qsidx;
           count = iru->iru_count;
   
           rxq = &(sc->pf_vsi.rx_queues[qsidx]);
   
           for (i = 0, next_pidx = pidx; i < count; i++) {
                   rxq->rx_base[next_pidx].read.pkt_addr = htole64(paddrs[i]);
                   if (++next_pidx == (uint32_t)rxq->desc_count)
                           next_pidx = 0;
           }
   }
   
   /**
    * ice_ift_rxd_flush - Flush Rx descriptors to hardware
    * @arg: device specific softc pointer
    * @rxqid: the Rx queue to flush
    * @flidx: unused parameter
    * @pidx: descriptor index to advance tail to
    *
    * Advance the Receive Descriptor Tail (RDT). This indicates to hardware that
    * software is done with the descriptor and it can be recycled.
    */
   static void
   ice_ift_rxd_flush(void *arg, uint16_t rxqid, uint8_t flidx __unused,
                     qidx_t pidx)
   {
           struct ice_softc *sc = (struct ice_softc *)arg;
           struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[rxqid];
           struct ice_hw *hw = &sc->hw;
   
           wr32(hw, rxq->tail, pidx);
   }
   
   static qidx_t
   ice_ift_queue_select(void *arg, struct mbuf *m)
   {
           struct ice_softc *sc = (struct ice_softc *)arg;
           struct ice_vsi *vsi = &sc->pf_vsi;
           u16 tc_base_queue, tc_qcount;
           u8 up, tc;
   
   #ifdef ALTQ
           /* Included to match default iflib behavior */
           /* Only go out on default queue if ALTQ is enabled */
           struct ifnet *ifp = (struct ifnet *)iflib_get_ifp(sc->ctx);
           if (ALTQ_IS_ENABLED(&ifp->if_snd))
                   return (0);
   #endif
   
           if (!ice_test_state(&sc->state, ICE_STATE_MULTIPLE_TCS)) {
                   if (M_HASHTYPE_GET(m)) {
                           /* Default iflib queue selection method */
                           return (m->m_pkthdr.flowid % sc->pf_vsi.num_tx_queues);
                   } else
                           return (0);
           }
   
           /* Use default TC unless overridden */
           tc = 0; /* XXX: Get default TC for traffic if >1 TC? */
   
           if (m->m_flags & M_VLANTAG) {
                   up = EVL_PRIOFTAG(m->m_pkthdr.ether_vtag);
                   tc = sc->hw.port_info->qos_cfg.local_dcbx_cfg.etscfg.prio_table[up];
           }
   
           tc_base_queue = vsi->tc_info[tc].qoffset;
           tc_qcount = vsi->tc_info[tc].qcount_tx;
   
           if (M_HASHTYPE_GET(m))
                   return ((m->m_pkthdr.flowid % tc_qcount) + tc_base_queue);
           else
                   return (tc_base_queue);
   }

Cache object: 652bdc0fcb3b6316d9f6308b85ea7e30