The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/dev/virtio/virtqueue.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3  *
    4  * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice unmodified, this list of conditions, and the following
   12  *    disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
   26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   27  */
   28 
   29 /*
   30  * Implements the virtqueue interface as basically described
   31  * in the original VirtIO paper.
   32  */
   33 
   34 #include <sys/cdefs.h>
   35 __FBSDID("$FreeBSD$");
   36 
   37 #include <sys/param.h>
   38 #include <sys/systm.h>
   39 #include <sys/kernel.h>
   40 #include <sys/malloc.h>
   41 #include <sys/sdt.h>
   42 #include <sys/sglist.h>
   43 #include <vm/vm.h>
   44 #include <vm/pmap.h>
   45 
   46 #include <machine/cpu.h>
   47 #include <machine/bus.h>
   48 #include <machine/atomic.h>
   49 #include <machine/resource.h>
   50 #include <sys/bus.h>
   51 #include <sys/rman.h>
   52 
   53 #include <dev/virtio/virtio.h>
   54 #include <dev/virtio/virtqueue.h>
   55 #include <dev/virtio/virtio_ring.h>
   56 
   57 #include "virtio_bus_if.h"
   58 
   59 struct virtqueue {
   60         device_t                 vq_dev;
   61         uint16_t                 vq_queue_index;
   62         uint16_t                 vq_nentries;
   63         uint32_t                 vq_flags;
   64 #define VIRTQUEUE_FLAG_MODERN    0x0001
   65 #define VIRTQUEUE_FLAG_INDIRECT  0x0002
   66 #define VIRTQUEUE_FLAG_EVENT_IDX 0x0004
   67 
   68         int                      vq_max_indirect_size;
   69         bus_size_t               vq_notify_offset;
   70         virtqueue_intr_t        *vq_intrhand;
   71         void                    *vq_intrhand_arg;
   72 
   73         struct vring             vq_ring;
   74         uint16_t                 vq_free_cnt;
   75         uint16_t                 vq_queued_cnt;
   76         /*
   77          * Head of the free chain in the descriptor table. If
   78          * there are no free descriptors, this will be set to
   79          * VQ_RING_DESC_CHAIN_END.
   80          */
   81         uint16_t                 vq_desc_head_idx;
   82         /*
   83          * Last consumed descriptor in the used table,
   84          * trails vq_ring.used->idx.
   85          */
   86         uint16_t                 vq_used_cons_idx;
   87 
   88         void                    *vq_ring_mem;
   89         int                      vq_indirect_mem_size;
   90         int                      vq_alignment;
   91         int                      vq_ring_size;
   92         char                     vq_name[VIRTQUEUE_MAX_NAME_SZ];
   93 
   94         struct vq_desc_extra {
   95                 void              *cookie;
   96                 struct vring_desc *indirect;
   97                 vm_paddr_t         indirect_paddr;
   98                 uint16_t           ndescs;
   99         } vq_descx[0];
  100 };
  101 
  102 /*
  103  * The maximum virtqueue size is 2^15. Use that value as the end of
  104  * descriptor chain terminator since it will never be a valid index
  105  * in the descriptor table. This is used to verify we are correctly
  106  * handling vq_free_cnt.
  107  */
  108 #define VQ_RING_DESC_CHAIN_END 32768
  109 
  110 #define VQASSERT(_vq, _exp, _msg, ...)                          \
  111     KASSERT((_exp),("%s: %s - "_msg, __func__, (_vq)->vq_name,  \
  112         ##__VA_ARGS__))
  113 
  114 #define VQ_RING_ASSERT_VALID_IDX(_vq, _idx)                     \
  115     VQASSERT((_vq), (_idx) < (_vq)->vq_nentries,                \
  116         "invalid ring index: %d, max: %d", (_idx),              \
  117         (_vq)->vq_nentries)
  118 
  119 #define VQ_RING_ASSERT_CHAIN_TERM(_vq)                          \
  120     VQASSERT((_vq), (_vq)->vq_desc_head_idx ==                  \
  121         VQ_RING_DESC_CHAIN_END, "full ring terminated "         \
  122         "incorrectly: head idx: %d", (_vq)->vq_desc_head_idx)
  123 
  124 static int      virtqueue_init_indirect(struct virtqueue *vq, int);
  125 static void     virtqueue_free_indirect(struct virtqueue *vq);
  126 static void     virtqueue_init_indirect_list(struct virtqueue *,
  127                     struct vring_desc *);
  128 
  129 static void     vq_ring_init(struct virtqueue *);
  130 static void     vq_ring_update_avail(struct virtqueue *, uint16_t);
  131 static uint16_t vq_ring_enqueue_segments(struct virtqueue *,
  132                     struct vring_desc *, uint16_t, struct sglist *, int, int);
  133 static int      vq_ring_use_indirect(struct virtqueue *, int);
  134 static void     vq_ring_enqueue_indirect(struct virtqueue *, void *,
  135                     struct sglist *, int, int);
  136 static int      vq_ring_enable_interrupt(struct virtqueue *, uint16_t);
  137 static int      vq_ring_must_notify_host(struct virtqueue *);
  138 static void     vq_ring_notify_host(struct virtqueue *);
  139 static void     vq_ring_free_chain(struct virtqueue *, uint16_t);
  140 
  141 SDT_PROVIDER_DEFINE(virtqueue);
  142 SDT_PROBE_DEFINE6(virtqueue, , enqueue_segments, entry, "struct virtqueue *",
  143     "struct vring_desc *", "uint16_t", "struct sglist *", "int", "int");
  144 SDT_PROBE_DEFINE1(virtqueue, , enqueue_segments, return, "uint16_t");
  145 
  146 #define vq_modern(_vq)          (((_vq)->vq_flags & VIRTQUEUE_FLAG_MODERN) != 0)
  147 #define vq_htog16(_vq, _val)    virtio_htog16(vq_modern(_vq), _val)
  148 #define vq_htog32(_vq, _val)    virtio_htog32(vq_modern(_vq), _val)
  149 #define vq_htog64(_vq, _val)    virtio_htog64(vq_modern(_vq), _val)
  150 #define vq_gtoh16(_vq, _val)    virtio_gtoh16(vq_modern(_vq), _val)
  151 #define vq_gtoh32(_vq, _val)    virtio_gtoh32(vq_modern(_vq), _val)
  152 #define vq_gtoh64(_vq, _val)    virtio_gtoh64(vq_modern(_vq), _val)
  153 
  154 int
  155 virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size,
  156     bus_size_t notify_offset, int align, vm_paddr_t highaddr,
  157     struct vq_alloc_info *info, struct virtqueue **vqp)
  158 {
  159         struct virtqueue *vq;
  160         int error;
  161 
  162         *vqp = NULL;
  163         error = 0;
  164 
  165         if (size == 0) {
  166                 device_printf(dev,
  167                     "virtqueue %d (%s) does not exist (size is zero)\n",
  168                     queue, info->vqai_name);
  169                 return (ENODEV);
  170         } else if (!powerof2(size)) {
  171                 device_printf(dev,
  172                     "virtqueue %d (%s) size is not a power of 2: %d\n",
  173                     queue, info->vqai_name, size);
  174                 return (ENXIO);
  175         } else if (info->vqai_maxindirsz > VIRTIO_MAX_INDIRECT) {
  176                 device_printf(dev, "virtqueue %d (%s) requested too many "
  177                     "indirect descriptors: %d, max %d\n",
  178                     queue, info->vqai_name, info->vqai_maxindirsz,
  179                     VIRTIO_MAX_INDIRECT);
  180                 return (EINVAL);
  181         }
  182 
  183         vq = malloc(sizeof(struct virtqueue) +
  184             size * sizeof(struct vq_desc_extra), M_DEVBUF, M_NOWAIT | M_ZERO);
  185         if (vq == NULL) {
  186                 device_printf(dev, "cannot allocate virtqueue\n");
  187                 return (ENOMEM);
  188         }
  189 
  190         vq->vq_dev = dev;
  191         strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name));
  192         vq->vq_queue_index = queue;
  193         vq->vq_notify_offset = notify_offset;
  194         vq->vq_alignment = align;
  195         vq->vq_nentries = size;
  196         vq->vq_free_cnt = size;
  197         vq->vq_intrhand = info->vqai_intr;
  198         vq->vq_intrhand_arg = info->vqai_intr_arg;
  199 
  200         if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_F_VERSION_1) != 0)
  201                 vq->vq_flags |= VIRTQUEUE_FLAG_MODERN;
  202         if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0)
  203                 vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX;
  204 
  205         if (info->vqai_maxindirsz > 1) {
  206                 error = virtqueue_init_indirect(vq, info->vqai_maxindirsz);
  207                 if (error)
  208                         goto fail;
  209         }
  210 
  211         vq->vq_ring_size = round_page(vring_size(size, align));
  212         vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF,
  213             M_NOWAIT | M_ZERO, 0, highaddr, PAGE_SIZE, 0);
  214         if (vq->vq_ring_mem == NULL) {
  215                 device_printf(dev,
  216                     "cannot allocate memory for virtqueue ring\n");
  217                 error = ENOMEM;
  218                 goto fail;
  219         }
  220 
  221         vq_ring_init(vq);
  222         virtqueue_disable_intr(vq);
  223 
  224         *vqp = vq;
  225 
  226 fail:
  227         if (error)
  228                 virtqueue_free(vq);
  229 
  230         return (error);
  231 }
  232 
  233 static int
  234 virtqueue_init_indirect(struct virtqueue *vq, int indirect_size)
  235 {
  236         device_t dev;
  237         struct vq_desc_extra *dxp;
  238         int i, size;
  239 
  240         dev = vq->vq_dev;
  241 
  242         if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) {
  243                 /*
  244                  * Indirect descriptors requested by the driver but not
  245                  * negotiated. Return zero to keep the initialization
  246                  * going: we'll run fine without.
  247                  */
  248                 if (bootverbose)
  249                         device_printf(dev, "virtqueue %d (%s) requested "
  250                             "indirect descriptors but not negotiated\n",
  251                             vq->vq_queue_index, vq->vq_name);
  252                 return (0);
  253         }
  254 
  255         size = indirect_size * sizeof(struct vring_desc);
  256         vq->vq_max_indirect_size = indirect_size;
  257         vq->vq_indirect_mem_size = size;
  258         vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT;
  259 
  260         for (i = 0; i < vq->vq_nentries; i++) {
  261                 dxp = &vq->vq_descx[i];
  262 
  263                 dxp->indirect = malloc(size, M_DEVBUF, M_NOWAIT);
  264                 if (dxp->indirect == NULL) {
  265                         device_printf(dev, "cannot allocate indirect list\n");
  266                         return (ENOMEM);
  267                 }
  268 
  269                 dxp->indirect_paddr = vtophys(dxp->indirect);
  270                 virtqueue_init_indirect_list(vq, dxp->indirect);
  271         }
  272 
  273         return (0);
  274 }
  275 
  276 static void
  277 virtqueue_free_indirect(struct virtqueue *vq)
  278 {
  279         struct vq_desc_extra *dxp;
  280         int i;
  281 
  282         for (i = 0; i < vq->vq_nentries; i++) {
  283                 dxp = &vq->vq_descx[i];
  284 
  285                 if (dxp->indirect == NULL)
  286                         break;
  287 
  288                 free(dxp->indirect, M_DEVBUF);
  289                 dxp->indirect = NULL;
  290                 dxp->indirect_paddr = 0;
  291         }
  292 
  293         vq->vq_flags &= ~VIRTQUEUE_FLAG_INDIRECT;
  294         vq->vq_indirect_mem_size = 0;
  295 }
  296 
  297 static void
  298 virtqueue_init_indirect_list(struct virtqueue *vq,
  299     struct vring_desc *indirect)
  300 {
  301         int i;
  302 
  303         bzero(indirect, vq->vq_indirect_mem_size);
  304 
  305         for (i = 0; i < vq->vq_max_indirect_size - 1; i++)
  306                 indirect[i].next = vq_gtoh16(vq, i + 1);
  307         indirect[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END);
  308 }
  309 
  310 int
  311 virtqueue_reinit(struct virtqueue *vq, uint16_t size)
  312 {
  313         struct vq_desc_extra *dxp;
  314         int i;
  315 
  316         if (vq->vq_nentries != size) {
  317                 device_printf(vq->vq_dev,
  318                     "%s: '%s' changed size; old=%hu, new=%hu\n",
  319                     __func__, vq->vq_name, vq->vq_nentries, size);
  320                 return (EINVAL);
  321         }
  322 
  323         /* Warn if the virtqueue was not properly cleaned up. */
  324         if (vq->vq_free_cnt != vq->vq_nentries) {
  325                 device_printf(vq->vq_dev,
  326                     "%s: warning '%s' virtqueue not empty, "
  327                     "leaking %d entries\n", __func__, vq->vq_name,
  328                     vq->vq_nentries - vq->vq_free_cnt);
  329         }
  330 
  331         vq->vq_desc_head_idx = 0;
  332         vq->vq_used_cons_idx = 0;
  333         vq->vq_queued_cnt = 0;
  334         vq->vq_free_cnt = vq->vq_nentries;
  335 
  336         /* To be safe, reset all our allocated memory. */
  337         bzero(vq->vq_ring_mem, vq->vq_ring_size);
  338         for (i = 0; i < vq->vq_nentries; i++) {
  339                 dxp = &vq->vq_descx[i];
  340                 dxp->cookie = NULL;
  341                 dxp->ndescs = 0;
  342                 if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
  343                         virtqueue_init_indirect_list(vq, dxp->indirect);
  344         }
  345 
  346         vq_ring_init(vq);
  347         virtqueue_disable_intr(vq);
  348 
  349         return (0);
  350 }
  351 
  352 void
  353 virtqueue_free(struct virtqueue *vq)
  354 {
  355 
  356         if (vq->vq_free_cnt != vq->vq_nentries) {
  357                 device_printf(vq->vq_dev, "%s: freeing non-empty virtqueue, "
  358                     "leaking %d entries\n", vq->vq_name,
  359                     vq->vq_nentries - vq->vq_free_cnt);
  360         }
  361 
  362         if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
  363                 virtqueue_free_indirect(vq);
  364 
  365         if (vq->vq_ring_mem != NULL) {
  366                 contigfree(vq->vq_ring_mem, vq->vq_ring_size, M_DEVBUF);
  367                 vq->vq_ring_size = 0;
  368                 vq->vq_ring_mem = NULL;
  369         }
  370 
  371         free(vq, M_DEVBUF);
  372 }
  373 
  374 vm_paddr_t
  375 virtqueue_paddr(struct virtqueue *vq)
  376 {
  377 
  378         return (vtophys(vq->vq_ring_mem));
  379 }
  380 
  381 vm_paddr_t
  382 virtqueue_desc_paddr(struct virtqueue *vq)
  383 {
  384 
  385         return (vtophys(vq->vq_ring.desc));
  386 }
  387 
  388 vm_paddr_t
  389 virtqueue_avail_paddr(struct virtqueue *vq)
  390 {
  391 
  392         return (vtophys(vq->vq_ring.avail));
  393 }
  394 
  395 vm_paddr_t
  396 virtqueue_used_paddr(struct virtqueue *vq)
  397 {
  398 
  399         return (vtophys(vq->vq_ring.used));
  400 }
  401 
  402 uint16_t
  403 virtqueue_index(struct virtqueue *vq)
  404 {
  405 
  406         return (vq->vq_queue_index);
  407 }
  408 
  409 int
  410 virtqueue_size(struct virtqueue *vq)
  411 {
  412 
  413         return (vq->vq_nentries);
  414 }
  415 
  416 int
  417 virtqueue_nfree(struct virtqueue *vq)
  418 {
  419 
  420         return (vq->vq_free_cnt);
  421 }
  422 
  423 int
  424 virtqueue_empty(struct virtqueue *vq)
  425 {
  426 
  427         return (vq->vq_nentries == vq->vq_free_cnt);
  428 }
  429 
  430 int
  431 virtqueue_full(struct virtqueue *vq)
  432 {
  433 
  434         return (vq->vq_free_cnt == 0);
  435 }
  436 
  437 void
  438 virtqueue_notify(struct virtqueue *vq)
  439 {
  440 
  441         /* Ensure updated avail->idx is visible to host. */
  442         mb();
  443 
  444         if (vq_ring_must_notify_host(vq))
  445                 vq_ring_notify_host(vq);
  446         vq->vq_queued_cnt = 0;
  447 }
  448 
  449 int
  450 virtqueue_nused(struct virtqueue *vq)
  451 {
  452         uint16_t used_idx, nused;
  453 
  454         used_idx = vq_htog16(vq, vq->vq_ring.used->idx);
  455 
  456         nused = (uint16_t)(used_idx - vq->vq_used_cons_idx);
  457         VQASSERT(vq, nused <= vq->vq_nentries, "used more than available");
  458 
  459         return (nused);
  460 }
  461 
  462 int
  463 virtqueue_intr_filter(struct virtqueue *vq)
  464 {
  465 
  466         if (vq->vq_used_cons_idx == vq_htog16(vq, vq->vq_ring.used->idx))
  467                 return (0);
  468 
  469         virtqueue_disable_intr(vq);
  470 
  471         return (1);
  472 }
  473 
  474 void
  475 virtqueue_intr(struct virtqueue *vq)
  476 {
  477 
  478         vq->vq_intrhand(vq->vq_intrhand_arg);
  479 }
  480 
  481 int
  482 virtqueue_enable_intr(struct virtqueue *vq)
  483 {
  484 
  485         return (vq_ring_enable_interrupt(vq, 0));
  486 }
  487 
  488 int
  489 virtqueue_postpone_intr(struct virtqueue *vq, vq_postpone_t hint)
  490 {
  491         uint16_t ndesc, avail_idx;
  492 
  493         avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx);
  494         ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx);
  495 
  496         switch (hint) {
  497         case VQ_POSTPONE_SHORT:
  498                 ndesc = ndesc / 4;
  499                 break;
  500         case VQ_POSTPONE_LONG:
  501                 ndesc = (ndesc * 3) / 4;
  502                 break;
  503         case VQ_POSTPONE_EMPTIED:
  504                 break;
  505         }
  506 
  507         return (vq_ring_enable_interrupt(vq, ndesc));
  508 }
  509 
  510 /*
  511  * Note this is only considered a hint to the host.
  512  */
  513 void
  514 virtqueue_disable_intr(struct virtqueue *vq)
  515 {
  516 
  517         if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
  518                 vring_used_event(&vq->vq_ring) = vq_gtoh16(vq,
  519                     vq->vq_used_cons_idx - vq->vq_nentries - 1);
  520                 return;
  521         }
  522 
  523         vq->vq_ring.avail->flags |= vq_gtoh16(vq, VRING_AVAIL_F_NO_INTERRUPT);
  524 }
  525 
  526 int
  527 virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg,
  528     int readable, int writable)
  529 {
  530         struct vq_desc_extra *dxp;
  531         int needed;
  532         uint16_t head_idx, idx;
  533 
  534         needed = readable + writable;
  535 
  536         VQASSERT(vq, cookie != NULL, "enqueuing with no cookie");
  537         VQASSERT(vq, needed == sg->sg_nseg,
  538             "segment count mismatch, %d, %d", needed, sg->sg_nseg);
  539         VQASSERT(vq,
  540             needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size,
  541             "too many segments to enqueue: %d, %d/%d", needed,
  542             vq->vq_nentries, vq->vq_max_indirect_size);
  543 
  544         if (needed < 1)
  545                 return (EINVAL);
  546         if (vq->vq_free_cnt == 0)
  547                 return (ENOSPC);
  548 
  549         if (vq_ring_use_indirect(vq, needed)) {
  550                 vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable);
  551                 return (0);
  552         } else if (vq->vq_free_cnt < needed)
  553                 return (EMSGSIZE);
  554 
  555         head_idx = vq->vq_desc_head_idx;
  556         VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
  557         dxp = &vq->vq_descx[head_idx];
  558 
  559         VQASSERT(vq, dxp->cookie == NULL,
  560             "cookie already exists for index %d", head_idx);
  561         dxp->cookie = cookie;
  562         dxp->ndescs = needed;
  563 
  564         idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx,
  565             sg, readable, writable);
  566 
  567         vq->vq_desc_head_idx = idx;
  568         vq->vq_free_cnt -= needed;
  569         if (vq->vq_free_cnt == 0)
  570                 VQ_RING_ASSERT_CHAIN_TERM(vq);
  571         else
  572                 VQ_RING_ASSERT_VALID_IDX(vq, idx);
  573 
  574         vq_ring_update_avail(vq, head_idx);
  575 
  576         return (0);
  577 }
  578 
  579 void *
  580 virtqueue_dequeue(struct virtqueue *vq, uint32_t *len)
  581 {
  582         struct vring_used_elem *uep;
  583         void *cookie;
  584         uint16_t used_idx, desc_idx;
  585 
  586         if (vq->vq_used_cons_idx == vq_htog16(vq, vq->vq_ring.used->idx))
  587                 return (NULL);
  588 
  589         used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1);
  590         uep = &vq->vq_ring.used->ring[used_idx];
  591 
  592         rmb();
  593         desc_idx = (uint16_t) vq_htog32(vq, uep->id);
  594         if (len != NULL)
  595                 *len = vq_htog32(vq, uep->len);
  596 
  597         vq_ring_free_chain(vq, desc_idx);
  598 
  599         cookie = vq->vq_descx[desc_idx].cookie;
  600         VQASSERT(vq, cookie != NULL, "no cookie for index %d", desc_idx);
  601         vq->vq_descx[desc_idx].cookie = NULL;
  602 
  603         return (cookie);
  604 }
  605 
  606 void *
  607 virtqueue_poll(struct virtqueue *vq, uint32_t *len)
  608 {
  609         void *cookie;
  610 
  611         VIRTIO_BUS_POLL(vq->vq_dev);
  612         while ((cookie = virtqueue_dequeue(vq, len)) == NULL) {
  613                 cpu_spinwait();
  614                 VIRTIO_BUS_POLL(vq->vq_dev);
  615         }
  616 
  617         return (cookie);
  618 }
  619 
  620 void *
  621 virtqueue_drain(struct virtqueue *vq, int *last)
  622 {
  623         void *cookie;
  624         int idx;
  625 
  626         cookie = NULL;
  627         idx = *last;
  628 
  629         while (idx < vq->vq_nentries && cookie == NULL) {
  630                 if ((cookie = vq->vq_descx[idx].cookie) != NULL) {
  631                         vq->vq_descx[idx].cookie = NULL;
  632                         /* Free chain to keep free count consistent. */
  633                         vq_ring_free_chain(vq, idx);
  634                 }
  635                 idx++;
  636         }
  637 
  638         *last = idx;
  639 
  640         return (cookie);
  641 }
  642 
  643 void
  644 virtqueue_dump(struct virtqueue *vq)
  645 {
  646 
  647         if (vq == NULL)
  648                 return;
  649 
  650         printf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; "
  651             "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; "
  652             "used.idx=%d; used_event_idx=%d; avail.flags=0x%x; used.flags=0x%x\n",
  653             vq->vq_name, vq->vq_nentries, vq->vq_free_cnt, virtqueue_nused(vq),
  654             vq->vq_queued_cnt, vq->vq_desc_head_idx,
  655             vq_htog16(vq, vq->vq_ring.avail->idx), vq->vq_used_cons_idx,
  656             vq_htog16(vq, vq->vq_ring.used->idx),
  657             vq_htog16(vq, vring_used_event(&vq->vq_ring)),
  658             vq_htog16(vq, vq->vq_ring.avail->flags),
  659             vq_htog16(vq, vq->vq_ring.used->flags));
  660 }
  661 
  662 static void
  663 vq_ring_init(struct virtqueue *vq)
  664 {
  665         struct vring *vr;
  666         char *ring_mem;
  667         int i, size;
  668 
  669         ring_mem = vq->vq_ring_mem;
  670         size = vq->vq_nentries;
  671         vr = &vq->vq_ring;
  672 
  673         vring_init(vr, size, ring_mem, vq->vq_alignment);
  674 
  675         for (i = 0; i < size - 1; i++)
  676                 vr->desc[i].next = vq_gtoh16(vq, i + 1);
  677         vr->desc[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END);
  678 }
  679 
  680 static void
  681 vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx)
  682 {
  683         uint16_t avail_idx, avail_ring_idx;
  684 
  685         /*
  686          * Place the head of the descriptor chain into the next slot and make
  687          * it usable to the host. The chain is made available now rather than
  688          * deferring to virtqueue_notify() in the hopes that if the host is
  689          * currently running on another CPU, we can keep it processing the new
  690          * descriptor.
  691          */
  692         avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx);
  693         avail_ring_idx = avail_idx & (vq->vq_nentries - 1);
  694         vq->vq_ring.avail->ring[avail_ring_idx] = vq_gtoh16(vq, desc_idx);
  695 
  696         wmb();
  697         vq->vq_ring.avail->idx = vq_gtoh16(vq, avail_idx + 1);
  698 
  699         /* Keep pending count until virtqueue_notify(). */
  700         vq->vq_queued_cnt++;
  701 }
  702 
  703 static uint16_t
  704 vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc,
  705     uint16_t head_idx, struct sglist *sg, int readable, int writable)
  706 {
  707         struct sglist_seg *seg;
  708         struct vring_desc *dp;
  709         int i, needed;
  710         uint16_t idx;
  711 
  712         SDT_PROBE6(virtqueue, , enqueue_segments, entry, vq, desc, head_idx,
  713             sg, readable, writable);
  714 
  715         needed = readable + writable;
  716 
  717         for (i = 0, idx = head_idx, seg = sg->sg_segs;
  718              i < needed;
  719              i++, idx = vq_htog16(vq, dp->next), seg++) {
  720                 VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END,
  721                     "premature end of free desc chain");
  722 
  723                 dp = &desc[idx];
  724                 dp->addr = vq_gtoh64(vq, seg->ss_paddr);
  725                 dp->len = vq_gtoh32(vq, seg->ss_len);
  726                 dp->flags = 0;
  727 
  728                 if (i < needed - 1)
  729                         dp->flags |= vq_gtoh16(vq, VRING_DESC_F_NEXT);
  730                 if (i >= readable)
  731                         dp->flags |= vq_gtoh16(vq, VRING_DESC_F_WRITE);
  732         }
  733 
  734         SDT_PROBE1(virtqueue, , enqueue_segments, return, idx);
  735         return (idx);
  736 }
  737 
  738 static int
  739 vq_ring_use_indirect(struct virtqueue *vq, int needed)
  740 {
  741 
  742         if ((vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) == 0)
  743                 return (0);
  744 
  745         if (vq->vq_max_indirect_size < needed)
  746                 return (0);
  747 
  748         if (needed < 2)
  749                 return (0);
  750 
  751         return (1);
  752 }
  753 
  754 static void
  755 vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie,
  756     struct sglist *sg, int readable, int writable)
  757 {
  758         struct vring_desc *dp;
  759         struct vq_desc_extra *dxp;
  760         int needed;
  761         uint16_t head_idx;
  762 
  763         needed = readable + writable;
  764         VQASSERT(vq, needed <= vq->vq_max_indirect_size,
  765             "enqueuing too many indirect descriptors");
  766 
  767         head_idx = vq->vq_desc_head_idx;
  768         VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
  769         dp = &vq->vq_ring.desc[head_idx];
  770         dxp = &vq->vq_descx[head_idx];
  771 
  772         VQASSERT(vq, dxp->cookie == NULL,
  773             "cookie already exists for index %d", head_idx);
  774         dxp->cookie = cookie;
  775         dxp->ndescs = 1;
  776 
  777         dp->addr = vq_gtoh64(vq, dxp->indirect_paddr);
  778         dp->len = vq_gtoh32(vq, needed * sizeof(struct vring_desc));
  779         dp->flags = vq_gtoh16(vq, VRING_DESC_F_INDIRECT);
  780 
  781         vq_ring_enqueue_segments(vq, dxp->indirect, 0,
  782             sg, readable, writable);
  783 
  784         vq->vq_desc_head_idx = vq_htog16(vq, dp->next);
  785         vq->vq_free_cnt--;
  786         if (vq->vq_free_cnt == 0)
  787                 VQ_RING_ASSERT_CHAIN_TERM(vq);
  788         else
  789                 VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx);
  790 
  791         vq_ring_update_avail(vq, head_idx);
  792 }
  793 
  794 static int
  795 vq_ring_enable_interrupt(struct virtqueue *vq, uint16_t ndesc)
  796 {
  797 
  798         /*
  799          * Enable interrupts, making sure we get the latest index of
  800          * what's already been consumed.
  801          */
  802         if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
  803                 vring_used_event(&vq->vq_ring) =
  804                     vq_gtoh16(vq, vq->vq_used_cons_idx + ndesc);
  805         } else {
  806                 vq->vq_ring.avail->flags &=
  807                     vq_gtoh16(vq, ~VRING_AVAIL_F_NO_INTERRUPT);
  808         }
  809 
  810         mb();
  811 
  812         /*
  813          * Enough items may have already been consumed to meet our threshold
  814          * since we last checked. Let our caller know so it processes the new
  815          * entries.
  816          */
  817         if (virtqueue_nused(vq) > ndesc)
  818                 return (1);
  819 
  820         return (0);
  821 }
  822 
  823 static int
  824 vq_ring_must_notify_host(struct virtqueue *vq)
  825 {
  826         uint16_t new_idx, prev_idx, event_idx, flags;
  827 
  828         if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
  829                 new_idx = vq_htog16(vq, vq->vq_ring.avail->idx);
  830                 prev_idx = new_idx - vq->vq_queued_cnt;
  831                 event_idx = vq_htog16(vq, vring_avail_event(&vq->vq_ring));
  832 
  833                 return (vring_need_event(event_idx, new_idx, prev_idx) != 0);
  834         }
  835 
  836         flags = vq->vq_ring.used->flags;
  837         return ((flags & vq_gtoh16(vq, VRING_USED_F_NO_NOTIFY)) == 0);
  838 }
  839 
  840 static void
  841 vq_ring_notify_host(struct virtqueue *vq)
  842 {
  843 
  844         VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index,
  845             vq->vq_notify_offset);
  846 }
  847 
  848 static void
  849 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
  850 {
  851         struct vring_desc *dp;
  852         struct vq_desc_extra *dxp;
  853 
  854         VQ_RING_ASSERT_VALID_IDX(vq, desc_idx);
  855         dp = &vq->vq_ring.desc[desc_idx];
  856         dxp = &vq->vq_descx[desc_idx];
  857 
  858         if (vq->vq_free_cnt == 0)
  859                 VQ_RING_ASSERT_CHAIN_TERM(vq);
  860 
  861         vq->vq_free_cnt += dxp->ndescs;
  862         dxp->ndescs--;
  863 
  864         if ((dp->flags & vq_gtoh16(vq, VRING_DESC_F_INDIRECT)) == 0) {
  865                 while (dp->flags & vq_gtoh16(vq, VRING_DESC_F_NEXT)) {
  866                         uint16_t next_idx = vq_htog16(vq, dp->next);
  867                         VQ_RING_ASSERT_VALID_IDX(vq, next_idx);
  868                         dp = &vq->vq_ring.desc[next_idx];
  869                         dxp->ndescs--;
  870                 }
  871         }
  872 
  873         VQASSERT(vq, dxp->ndescs == 0,
  874             "failed to free entire desc chain, remaining: %d", dxp->ndescs);
  875 
  876         /*
  877          * We must append the existing free chain, if any, to the end of
  878          * newly freed chain. If the virtqueue was completely used, then
  879          * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
  880          */
  881         dp->next = vq_gtoh16(vq, vq->vq_desc_head_idx);
  882         vq->vq_desc_head_idx = desc_idx;
  883 }

Cache object: 287ac72f527c7471d34fbbe67fc18709


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.