FreeBSD/Linux Kernel Cross Reference
sys/dev/vmware/vmci/vmci_kernel_if.c

     /*-
      * Copyright (c) 2018 VMware, Inc.
      *
      * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0)
      */
     
     /* This file implements defines and helper functions. */
     
     #include <sys/cdefs.h>
    __FBSDID("$FreeBSD$");
    
    #include <sys/malloc.h>
    #include <sys/proc.h>
    #include <sys/uio.h>
    
    #include <machine/bus.h>
    
    #include "vmci.h"
    #include "vmci_defs.h"
    #include "vmci_kernel_defs.h"
    #include "vmci_kernel_if.h"
    #include "vmci_queue.h"
    
    struct vmci_queue_kernel_if {
            size_t                  num_pages;      /* Num pages incl. header. */
            struct vmci_dma_alloc   *dmas;          /* For dma alloc. */
    };
    
    /*
     *------------------------------------------------------------------------------
     *
     * vmci_init_lock
     *
     *     Initializes the lock. Must be called before use.
     *
     * Results:
     *     Always VMCI_SUCCESS.
     *
     * Side effects:
     *     Thread can block.
     *
     *------------------------------------------------------------------------------
     */
    
    int
    vmci_init_lock(vmci_lock *lock, char *name)
    {
    
            mtx_init(lock, name, NULL, MTX_DEF | MTX_NOWITNESS);
            return (VMCI_SUCCESS);
    }
    
    /*
     *------------------------------------------------------------------------------
     *
     * vmci_cleanup_lock
     *
     *     Cleanup the lock. Must be called before deallocating lock.
     *
     * Results:
     *     None
     *
     * Side effects:
     *     Deletes kernel lock state
     *
     *------------------------------------------------------------------------------
     */
    
    void
    vmci_cleanup_lock(vmci_lock *lock)
    {
    
            if (mtx_initialized(lock))
                    mtx_destroy(lock);
    }
    
    /*
     *------------------------------------------------------------------------------
     *
     * vmci_grab_lock
     *
     *     Grabs the given lock.
     *
     * Results:
     *      None
     *
     * Side effects:
     *      Thread can block.
     *
     *------------------------------------------------------------------------------
     */
    
    void
    vmci_grab_lock(vmci_lock *lock)
    {
    
            mtx_lock(lock);
    }
    
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_release_lock
    *
    *     Releases the given lock.
    *
    * Results:
    *     None
    *
    * Side effects:
    *     A thread blocked on this lock may wake up.
    *
    *------------------------------------------------------------------------------
    */
   
   void
   vmci_release_lock(vmci_lock *lock)
   {
   
           mtx_unlock(lock);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_grab_lock_bh
    *
    *     Grabs the given lock.
    *
    * Results:
    *     None
    *
    * Side effects:
    *     None.
    *
    *------------------------------------------------------------------------------
    */
   
   void
   vmci_grab_lock_bh(vmci_lock *lock)
   {
   
           mtx_lock(lock);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_release_lock_bh
    *
    *     Releases the given lock.
    *
    * Results:
    *     None
    *
    * Side effects:
    *     None.
    *
    *------------------------------------------------------------------------------
    */
   
   void
   vmci_release_lock_bh(vmci_lock *lock)
   {
   
           mtx_unlock(lock);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_initialized_lock
    *
    *     Returns whether a lock has been initialized.
    *
    * Results:
    *     Return 1 if initialized or 0 if unininitialized.
    *
    * Side effects:
    *     None
    *
    *------------------------------------------------------------------------------
    */
   
   int
   vmci_initialized_lock(vmci_lock *lock)
   {
   
           return mtx_initialized(lock);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_alloc_kernel_mem
    *
    *     Allocate physically contiguous memory for the VMCI driver.
    *
    * Results:
    *     The address allocated or NULL on error.
    *
    *
    * Side effects:
    *     Memory may be allocated.
    *
    *------------------------------------------------------------------------------
    */
   
   void *
   vmci_alloc_kernel_mem(size_t size, int flags)
   {
           void *ptr;
   
           if ((flags & VMCI_MEMORY_ATOMIC) != 0)
                   ptr = contigmalloc(size, M_DEVBUF, M_NOWAIT, 0, 0xFFFFFFFF,
                       8, 1024 * 1024);
           else
                   ptr = contigmalloc(size, M_DEVBUF, M_WAITOK, 0, 0xFFFFFFFF,
                       8, 1024 * 1024);
   
           return (ptr);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_free_kernel_mem
    *
    *     Free kernel memory allocated for the VMCI driver.
    *
    * Results:
    *     None.
    *
    * Side effects:
    *     Memory is freed.
    *
    *------------------------------------------------------------------------------
    */
   
   void
   vmci_free_kernel_mem(void *ptr, size_t size)
   {
   
           contigfree(ptr, size, M_DEVBUF);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_can_schedule_delayed_work --
    *
    *     Checks to see if the given platform supports delayed work callbacks.
    *
    * Results:
    *     true if it does. false otherwise.
    *
    * Side effects:
    *     None.
    *
    *------------------------------------------------------------------------------
    */
   
   bool
   vmci_can_schedule_delayed_work(void)
   {
   
           return (true);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_schedule_delayed_work --
    *
    *     Schedule the specified callback.
    *
    * Results:
    *     Zero on success, error code otherwise.
    *
    * Side effects:
    *     None.
    *
    *------------------------------------------------------------------------------
    */
   
   int
   vmci_schedule_delayed_work(vmci_work_fn *work_fn, void *data)
   {
   
           return (vmci_schedule_delayed_work_fn(work_fn, data));
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_create_event --
    *
    * Results:
    *     None.
    *
    * Side effects:
    *     None.
    *
    *------------------------------------------------------------------------------
    */
   
   void
   vmci_create_event(vmci_event *event)
   {
   
           sema_init(event, 0, "vmci_event");
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_destroy_event --
    *
    * Results:
    *     None.
    *
    * Side effects:
    *     None.
    *
    *------------------------------------------------------------------------------
    */
   
   void
   vmci_destroy_event(vmci_event *event)
   {
   
           if (mtx_owned(&event->sema_mtx))
                   sema_destroy(event);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_signal_event --
    *
    * Results:
    *     None.
    *
    * Side effects:
    *     None.
    *
    *------------------------------------------------------------------------------
    */
   
   void
   vmci_signal_event(vmci_event *event)
   {
   
           sema_post(event);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_wait_on_event --
    *
    * Results:
    *     None.
    *
    * Side effects:
    *     None.
    *
    *------------------------------------------------------------------------------
    */
   
   void
   vmci_wait_on_event(vmci_event *event, vmci_event_release_cb release_cb,
       void *client_data)
   {
   
           release_cb(client_data);
           sema_wait(event);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_mutex_init --
    *
    *     Initializes the mutex. Must be called before use.
    *
    * Results:
    *     Success.
    *
    * Side effects:
    *     None.
    *
    *------------------------------------------------------------------------------
    */
   
   int
   vmci_mutex_init(vmci_mutex *mutex, char *name)
   {
   
           mtx_init(mutex, name, NULL, MTX_DEF | MTX_NOWITNESS);
           return (VMCI_SUCCESS);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_mutex_destroy --
    *
    *     Destroys the mutex.
    *
    * Results:
    *     None.
    *
    * Side effects:
    *     None.
    *
    *------------------------------------------------------------------------------
    */
   
   void
   vmci_mutex_destroy(vmci_mutex *mutex)
   {
   
           mtx_destroy(mutex);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_mutex_acquire --
    *
    *     Acquires the mutex.
    *
    * Results:
    *     None.
    *
    * Side effects:
    *     Thread may block.
    *
    *------------------------------------------------------------------------------
    */
   
   void
   vmci_mutex_acquire(vmci_mutex *mutex)
   {
   
           mtx_lock(mutex);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_mutex_release --
    *
    *     Releases the mutex.
    *
    * Results:
    *     None.
    *
    * Side effects:
    *     May wake up the thread blocking on this mutex.
    *
    *------------------------------------------------------------------------------
    */
   
   void
   vmci_mutex_release(vmci_mutex *mutex)
   {
   
           mtx_unlock(mutex);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_mutex_initialized
    *
    *     Returns whether a mutex has been initialized.
    *
    * Results:
    *     Return 1 if initialized or 0 if unininitialized.
    *
    * Side effects:
    *     None
    *
    *------------------------------------------------------------------------------
    */
   
   int
   vmci_mutex_initialized(vmci_mutex *mutex)
   {
   
           return mtx_initialized(mutex);
   }
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_alloc_queue --
    *
    *     Allocates kernel queue pages of specified size with IOMMU mappings, plus
    *     space for the queue structure/kernel interface and the queue header.
    *
    * Results:
    *     Pointer to the queue on success, NULL otherwise.
    *
    * Side effects:
    *     Memory is allocated.
    *
    *------------------------------------------------------------------------------
    */
   
   void *
   vmci_alloc_queue(uint64_t size, uint32_t flags)
   {
           struct vmci_queue *queue;
           size_t i;
           const size_t num_pages = CEILING(size, PAGE_SIZE) + 1;
           const size_t dmas_size = num_pages * sizeof(struct vmci_dma_alloc);
           const size_t queue_size =
               sizeof(*queue) + sizeof(*(queue->kernel_if)) + dmas_size;
   
           /* Size should be enforced by vmci_qpair_alloc(), double-check here. */
           if (size > VMCI_MAX_GUEST_QP_MEMORY) {
                   ASSERT(false);
                   return (NULL);
           }
   
           queue = malloc(queue_size, M_DEVBUF, M_NOWAIT);
           if (!queue)
                   return (NULL);
   
           queue->q_header = NULL;
           queue->saved_header = NULL;
           queue->kernel_if = (struct vmci_queue_kernel_if *)(queue + 1);
           queue->kernel_if->num_pages = num_pages;
           queue->kernel_if->dmas = (struct vmci_dma_alloc *)(queue->kernel_if +
               1);
           for (i = 0; i < num_pages; i++) {
                   vmci_dma_malloc(PAGE_SIZE, 1, &queue->kernel_if->dmas[i]);
                   if (!queue->kernel_if->dmas[i].dma_vaddr) {
                           /* Size excl. the header. */
                           vmci_free_queue(queue, i * PAGE_SIZE);
                           return (NULL);
                   }
           }
   
           /* Queue header is the first page. */
           queue->q_header = (void *)queue->kernel_if->dmas[0].dma_vaddr;
   
           return ((void *)queue);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_free_queue --
    *
    *     Frees kernel VA space for a given queue and its queue header, and frees
    *     physical data pages.
    *
    * Results:
    *     None.
    *
    * Side effects:
    *     Memory is freed.
    *
    *------------------------------------------------------------------------------
    */
   
   void
   vmci_free_queue(void *q, uint64_t size)
   {
           struct vmci_queue *queue = q;
   
           if (queue) {
                   const size_t num_pages = CEILING(size, PAGE_SIZE) + 1;
                   uint64_t i;
   
                   /* Given size doesn't include header, so add in a page here. */
                   for (i = 0; i < num_pages; i++)
                           vmci_dma_free(&queue->kernel_if->dmas[i]);
                   free(queue, M_DEVBUF);
           }
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_alloc_ppn_set --
    *
    *     Allocates two list of PPNs --- one for the pages in the produce queue,
    *     and the other for the pages in the consume queue. Intializes the list of
    *     PPNs with the page frame numbers of the KVA for the two queues (and the
    *     queue headers).
    *
    * Results:
    *     Success or failure.
    *
    * Side effects:
    *     Memory may be allocated.
    *
    *-----------------------------------------------------------------------------
    */
   
   int
   vmci_alloc_ppn_set(void *prod_q, uint64_t num_produce_pages, void *cons_q,
       uint64_t num_consume_pages, struct ppn_set *ppn_set)
   {
           struct vmci_queue *consume_q = cons_q;
           struct vmci_queue *produce_q = prod_q;
           vmci_ppn_list consume_ppns;
           vmci_ppn_list produce_ppns;
           uint64_t i;
   
           if (!produce_q || !num_produce_pages || !consume_q ||
               !num_consume_pages || !ppn_set)
                   return (VMCI_ERROR_INVALID_ARGS);
   
           if (ppn_set->initialized)
                   return (VMCI_ERROR_ALREADY_EXISTS);
   
           produce_ppns =
               vmci_alloc_kernel_mem(num_produce_pages * sizeof(*produce_ppns),
               VMCI_MEMORY_NORMAL);
           if (!produce_ppns)
                   return (VMCI_ERROR_NO_MEM);
   
           consume_ppns =
               vmci_alloc_kernel_mem(num_consume_pages * sizeof(*consume_ppns),
               VMCI_MEMORY_NORMAL);
           if (!consume_ppns) {
                   vmci_free_kernel_mem(produce_ppns,
                       num_produce_pages * sizeof(*produce_ppns));
                   return (VMCI_ERROR_NO_MEM);
           }
   
           for (i = 0; i < num_produce_pages; i++) {
                   unsigned long pfn;
   
                   produce_ppns[i] =
                       pfn = produce_q->kernel_if->dmas[i].dma_paddr >> PAGE_SHIFT;
   
                   /*
                    * Fail allocation if PFN isn't supported by hypervisor.
                    */
   
                   if (sizeof(pfn) >
                       sizeof(*produce_ppns) && pfn != produce_ppns[i])
                           goto ppn_error;
           }
           for (i = 0; i < num_consume_pages; i++) {
                   unsigned long pfn;
   
                   consume_ppns[i] =
                       pfn = consume_q->kernel_if->dmas[i].dma_paddr >> PAGE_SHIFT;
   
                   /*
                    * Fail allocation if PFN isn't supported by hypervisor.
                    */
   
                   if (sizeof(pfn) >
                       sizeof(*consume_ppns) && pfn != consume_ppns[i])
                           goto ppn_error;
           }
   
           ppn_set->num_produce_pages = num_produce_pages;
           ppn_set->num_consume_pages = num_consume_pages;
           ppn_set->produce_ppns = produce_ppns;
           ppn_set->consume_ppns = consume_ppns;
           ppn_set->initialized = true;
           return (VMCI_SUCCESS);
   
   ppn_error:
           vmci_free_kernel_mem(produce_ppns, num_produce_pages *
               sizeof(*produce_ppns));
           vmci_free_kernel_mem(consume_ppns, num_consume_pages *
               sizeof(*consume_ppns));
           return (VMCI_ERROR_INVALID_ARGS);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_free_ppn_set --
    *
    *     Frees the two list of PPNs for a queue pair.
    *
    * Results:
    *     None.
    *
    * Side effects:
    *     None.
    *
    *------------------------------------------------------------------------------
    */
   
   void
   vmci_free_ppn_set(struct ppn_set *ppn_set)
   {
   
           ASSERT(ppn_set);
           if (ppn_set->initialized) {
                   /* Do not call these functions on NULL inputs. */
                   ASSERT(ppn_set->produce_ppns && ppn_set->consume_ppns);
                   vmci_free_kernel_mem(ppn_set->produce_ppns,
                       ppn_set->num_produce_pages *
                       sizeof(*ppn_set->produce_ppns));
                   vmci_free_kernel_mem(ppn_set->consume_ppns,
                       ppn_set->num_consume_pages *
                       sizeof(*ppn_set->consume_ppns));
           }
           memset(ppn_set, 0, sizeof(*ppn_set));
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_populate_ppn_list --
    *
    *     Populates the list of PPNs in the hypercall structure with the PPNS
    *     of the produce queue and the consume queue.
    *
    * Results:
    *     VMCI_SUCCESS.
    *
    * Side effects:
    *     None.
    *
    *------------------------------------------------------------------------------
    */
   
   int
   vmci_populate_ppn_list(uint8_t *call_buf, const struct ppn_set *ppn_set)
   {
   
           ASSERT(call_buf && ppn_set && ppn_set->initialized);
           memcpy(call_buf, ppn_set->produce_ppns,
               ppn_set->num_produce_pages * sizeof(*ppn_set->produce_ppns));
           memcpy(call_buf + ppn_set->num_produce_pages *
               sizeof(*ppn_set->produce_ppns), ppn_set->consume_ppns,
               ppn_set->num_consume_pages * sizeof(*ppn_set->consume_ppns));
   
           return (VMCI_SUCCESS);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_memcpy_{to,from}iovec --
    *
    *     These helper routines will copy the specified bytes to/from memory that's
    *     specified as a struct iovec.  The routines can not verify the correctness
    *     of the struct iovec's contents.
    *
    * Results:
    *      None.
    *
    * Side effects:
    *      None.
    *
    *------------------------------------------------------------------------------
    */
   
   static inline void
   vmci_memcpy_toiovec(struct iovec *iov, uint8_t *src, size_t len)
   {
   
           while (len > 0) {
                   if (iov->iov_len) {
                           size_t to_copy = MIN(iov->iov_len, len);
                           memcpy(iov->iov_base, src, to_copy);
                           src += to_copy;
                           len -= to_copy;
                           iov->iov_base = (void *)((uintptr_t) iov->iov_base +
                               to_copy);
                           iov->iov_len -= to_copy;
                   }
                   iov++;
           }
   }
   
   static inline void
   vmci_memcpy_fromiovec(uint8_t *dst, struct iovec *iov, size_t len)
   {
   
           while (len > 0) {
                   if (iov->iov_len) {
                           size_t to_copy = MIN(iov->iov_len, len);
                           memcpy(dst, iov->iov_base, to_copy);
                           dst += to_copy;
                           len -= to_copy;
                           iov->iov_base = (void *)((uintptr_t) iov->iov_base +
                               to_copy);
                           iov->iov_len -= to_copy;
                   }
                   iov++;
           }
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * __vmci_memcpy_to_queue --
    *
    *     Copies from a given buffer or iovector to a VMCI Queue. Assumes that
    *     offset + size does not wrap around in the queue.
    *
    * Results:
    *     Zero on success, negative error code on failure.
    *
    * Side effects:
    *     None.
    *
    *------------------------------------------------------------------------------
    */
   
   #pragma GCC diagnostic ignored "-Wcast-qual"
   static int
   __vmci_memcpy_to_queue(struct vmci_queue *queue, uint64_t queue_offset,
       const void *src, size_t size, bool is_iovec)
   {
           struct vmci_queue_kernel_if *kernel_if = queue->kernel_if;
           size_t bytes_copied = 0;
   
           while (bytes_copied < size) {
                   const uint64_t page_index =
                       (queue_offset + bytes_copied) / PAGE_SIZE;
                   const size_t page_offset =
                       (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
                   void *va;
                   size_t to_copy;
   
                   /* Skip header. */
                   va = (void *)kernel_if->dmas[page_index + 1].dma_vaddr;
   
                   ASSERT(va);
                   /*
                    * Fill up the page if we have enough payload, or else
                    * copy the remaining bytes.
                    */
                   to_copy = MIN(PAGE_SIZE - page_offset, size - bytes_copied);
   
                   if (is_iovec) {
                           struct iovec *iov = (struct iovec *)src;
   
                           /* The iovec will track bytes_copied internally. */
                           vmci_memcpy_fromiovec((uint8_t *)va + page_offset,
                               iov, to_copy);
                   } else
                           memcpy((uint8_t *)va + page_offset,
                               (uint8_t *)src + bytes_copied, to_copy);
                   bytes_copied += to_copy;
           }
   
           return (VMCI_SUCCESS);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * __vmci_memcpy_from_queue --
    *
    *     Copies to a given buffer or iovector from a VMCI Queue. Assumes that
    *     offset + size does not wrap around in the queue.
    *
    * Results:
    *     Zero on success, negative error code on failure.
    *
    * Side effects:
    *     None.
    *
    *------------------------------------------------------------------------------
    */
   
   static int
   __vmci_memcpy_from_queue(void *dest, const struct vmci_queue *queue,
       uint64_t queue_offset, size_t size, bool is_iovec)
   {
           struct vmci_queue_kernel_if *kernel_if = queue->kernel_if;
           size_t bytes_copied = 0;
   
           while (bytes_copied < size) {
                   const uint64_t page_index =
                       (queue_offset + bytes_copied) / PAGE_SIZE;
                   const size_t page_offset =
                       (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
                   void *va;
                   size_t to_copy;
   
                   /* Skip header. */
                   va = (void *)kernel_if->dmas[page_index + 1].dma_vaddr;
   
                   ASSERT(va);
                   /*
                    * Fill up the page if we have enough payload, or else
                    * copy the remaining bytes.
                    */
                   to_copy = MIN(PAGE_SIZE - page_offset, size - bytes_copied);
   
                   if (is_iovec) {
                           struct iovec *iov = (struct iovec *)dest;
   
                           /* The iovec will track bytesCopied internally. */
                           vmci_memcpy_toiovec(iov, (uint8_t *)va +
                               page_offset, to_copy);
                   } else
                           memcpy((uint8_t *)dest + bytes_copied,
                               (uint8_t *)va + page_offset, to_copy);
   
                   bytes_copied += to_copy;
           }
   
           return (VMCI_SUCCESS);
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_memcpy_to_queue --
    *
    *     Copies from a given buffer to a VMCI Queue.
    *
    * Results:
    *     Zero on success, negative error code on failure.
    *
    * Side effects:
    *     None.
    *
    *------------------------------------------------------------------------------
    */
   
   int
   vmci_memcpy_to_queue(struct vmci_queue *queue, uint64_t queue_offset,
       const void *src, size_t src_offset, size_t size, int buf_type,
       bool can_block)
   {
   
           ASSERT(can_block);
   
           return (__vmci_memcpy_to_queue(queue, queue_offset,
               (uint8_t *)src + src_offset, size, false));
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_memcpy_from_queue --
    *
    *      Copies to a given buffer from a VMCI Queue.
    *
    * Results:
    *      Zero on success, negative error code on failure.
    *
    * Side effects:
    *      None.
    *
    *------------------------------------------------------------------------------
    */
   
   int
   vmci_memcpy_from_queue(void *dest, size_t dest_offset,
       const struct vmci_queue *queue, uint64_t queue_offset, size_t size,
       int buf_type, bool can_block)
   {
   
           ASSERT(can_block);
   
           return (__vmci_memcpy_from_queue((uint8_t *)dest + dest_offset,
               queue, queue_offset, size, false));
   }
   
   /*
    *------------------------------------------------------------------------------
    *
    * vmci_memcpy_to_queue_local --
    *
    *     Copies from a given buffer to a local VMCI queue. This is the
    *     same as a regular copy.
    *
    * Results:
    *     Zero on success, negative error code on failure.
    *
    * Side effects:
    *     None.
    *
    *------------------------------------------------------------------------------
    */
   
   int
   vmci_memcpy_to_queue_local(struct vmci_queue *queue, uint64_t queue_offset,
       const void *src, size_t src_offset, size_t size, int buf_type,
       bool can_block)
   {
   
           ASSERT(can_block);
   
           return (__vmci_memcpy_to_queue(queue, queue_offset,
               (uint8_t *)src + src_offset, size, false));
   }
  
  /*
   *------------------------------------------------------------------------------
   *
   * vmci_memcpy_from_queue_local --
   *
   *     Copies to a given buffer from a VMCI Queue.
   *
   * Results:
   *     Zero on success, negative error code on failure.
   *
   * Side effects:
   *     None.
   *
   *------------------------------------------------------------------------------
   */
  
  int
  vmci_memcpy_from_queue_local(void *dest, size_t dest_offset,
      const struct vmci_queue *queue, uint64_t queue_offset, size_t size,
      int buf_type, bool can_block)
  {
  
          ASSERT(can_block);
  
          return (__vmci_memcpy_from_queue((uint8_t *)dest + dest_offset,
              queue, queue_offset, size, false));
  }
  
  /*------------------------------------------------------------------------------
   *
   * vmci_memcpy_to_queue_v --
   *
   *     Copies from a given iovec from a VMCI Queue.
   *
   * Results:
   *     Zero on success, negative error code on failure.
   *
   * Side effects:
   *     None.
   *
   *------------------------------------------------------------------------------
   */
  
  int
  vmci_memcpy_to_queue_v(struct vmci_queue *queue, uint64_t queue_offset,
      const void *src, size_t src_offset, size_t size, int buf_type,
      bool can_block)
  {
  
          ASSERT(can_block);
  
          /*
           * We ignore src_offset because src is really a struct iovec * and will
           * maintain offset internally.
           */
          return (__vmci_memcpy_to_queue(queue, queue_offset, src, size,
              true));
  }
  
  /*
   *------------------------------------------------------------------------------
   *
   * vmci_memcpy_from_queue_v --
   *
   *     Copies to a given iovec from a VMCI Queue.
   *
   * Results:
   *     Zero on success, negative error code on failure.
   *
   * Side effects:
   *     None.
   *
   *------------------------------------------------------------------------------
   */
  
  int
  vmci_memcpy_from_queue_v(void *dest, size_t dest_offset,
      const struct vmci_queue *queue, uint64_t queue_offset, size_t size,
      int buf_type, bool can_block)
  {
  
          ASSERT(can_block);
  
          /*
           * We ignore dest_offset because dest is really a struct iovec * and
           * will maintain offset internally.
           */
          return (__vmci_memcpy_from_queue(dest, queue, queue_offset, size,
              true));
  }
  
  /*
   *------------------------------------------------------------------------------
   *
   * vmci_read_port_bytes --
   *
   *     Copy memory from an I/O port to kernel memory.
   *
   * Results:
   *     No results.
   *
   * Side effects:
   *     None.
   *
   *------------------------------------------------------------------------------
   */
  
  void
  vmci_read_port_bytes(vmci_io_handle handle, vmci_io_port port, uint8_t *buffer,
      size_t buffer_length)
  {
  
          insb(port, buffer, buffer_length);
  }

Cache object: 4d308753c60e434e422b117fbe6379cd