The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/vmm/vmm.c

Version: -  FREEBSD  -  FREEBSD11  -  FREEBSD10  -  FREEBSD9  -  FREEBSD92  -  FREEBSD91  -  FREEBSD90  -  FREEBSD8  -  FREEBSD82  -  FREEBSD81  -  FREEBSD80  -  FREEBSD7  -  FREEBSD74  -  FREEBSD73  -  FREEBSD72  -  FREEBSD71  -  FREEBSD70  -  FREEBSD6  -  FREEBSD64  -  FREEBSD63  -  FREEBSD62  -  FREEBSD61  -  FREEBSD60  -  FREEBSD5  -  FREEBSD55  -  FREEBSD54  -  FREEBSD53  -  FREEBSD52  -  FREEBSD51  -  FREEBSD50  -  FREEBSD4  -  FREEBSD3  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2011 NetApp, Inc.
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  *
   14  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
   15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
   18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24  * SUCH DAMAGE.
   25  *
   26  * $FreeBSD: stable/11/sys/amd64/vmm/vmm.c 306471 2016-09-30 01:39:18Z jhb $
   27  */
   28 
   29 #include <sys/cdefs.h>
   30 __FBSDID("$FreeBSD: stable/11/sys/amd64/vmm/vmm.c 306471 2016-09-30 01:39:18Z jhb $");
   31 
   32 #include <sys/param.h>
   33 #include <sys/systm.h>
   34 #include <sys/kernel.h>
   35 #include <sys/module.h>
   36 #include <sys/sysctl.h>
   37 #include <sys/malloc.h>
   38 #include <sys/pcpu.h>
   39 #include <sys/lock.h>
   40 #include <sys/mutex.h>
   41 #include <sys/proc.h>
   42 #include <sys/rwlock.h>
   43 #include <sys/sched.h>
   44 #include <sys/smp.h>
   45 #include <sys/systm.h>
   46 
   47 #include <vm/vm.h>
   48 #include <vm/vm_object.h>
   49 #include <vm/vm_page.h>
   50 #include <vm/pmap.h>
   51 #include <vm/vm_map.h>
   52 #include <vm/vm_extern.h>
   53 #include <vm/vm_param.h>
   54 
   55 #include <machine/cpu.h>
   56 #include <machine/pcb.h>
   57 #include <machine/smp.h>
   58 #include <x86/psl.h>
   59 #include <x86/apicreg.h>
   60 
   61 #include <machine/vmm.h>
   62 #include <machine/vmm_dev.h>
   63 #include <machine/vmm_instruction_emul.h>
   64 
   65 #include "vmm_ioport.h"
   66 #include "vmm_ktr.h"
   67 #include "vmm_host.h"
   68 #include "vmm_mem.h"
   69 #include "vmm_util.h"
   70 #include "vatpic.h"
   71 #include "vatpit.h"
   72 #include "vhpet.h"
   73 #include "vioapic.h"
   74 #include "vlapic.h"
   75 #include "vpmtmr.h"
   76 #include "vrtc.h"
   77 #include "vmm_stat.h"
   78 #include "vmm_lapic.h"
   79 
   80 #include "io/ppt.h"
   81 #include "io/iommu.h"
   82 
   83 struct vlapic;
   84 
   85 /*
   86  * Initialization:
   87  * (a) allocated when vcpu is created
   88  * (i) initialized when vcpu is created and when it is reinitialized
   89  * (o) initialized the first time the vcpu is created
   90  * (x) initialized before use
   91  */
   92 struct vcpu {
   93         struct mtx      mtx;            /* (o) protects 'state' and 'hostcpu' */
   94         enum vcpu_state state;          /* (o) vcpu state */
   95         int             hostcpu;        /* (o) vcpu's host cpu */
   96         int             reqidle;        /* (i) request vcpu to idle */
   97         struct vlapic   *vlapic;        /* (i) APIC device model */
   98         enum x2apic_state x2apic_state; /* (i) APIC mode */
   99         uint64_t        exitintinfo;    /* (i) events pending at VM exit */
  100         int             nmi_pending;    /* (i) NMI pending */
  101         int             extint_pending; /* (i) INTR pending */
  102         int     exception_pending;      /* (i) exception pending */
  103         int     exc_vector;             /* (x) exception collateral */
  104         int     exc_errcode_valid;
  105         uint32_t exc_errcode;
  106         struct savefpu  *guestfpu;      /* (a,i) guest fpu state */
  107         uint64_t        guest_xcr0;     /* (i) guest %xcr0 register */
  108         void            *stats;         /* (a,i) statistics */
  109         struct vm_exit  exitinfo;       /* (x) exit reason and collateral */
  110         uint64_t        nextrip;        /* (x) next instruction to execute */
  111 };
  112 
  113 #define vcpu_lock_initialized(v) mtx_initialized(&((v)->mtx))
  114 #define vcpu_lock_init(v)       mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN)
  115 #define vcpu_lock(v)            mtx_lock_spin(&((v)->mtx))
  116 #define vcpu_unlock(v)          mtx_unlock_spin(&((v)->mtx))
  117 #define vcpu_assert_locked(v)   mtx_assert(&((v)->mtx), MA_OWNED)
  118 
  119 struct mem_seg {
  120         size_t  len;
  121         bool    sysmem;
  122         struct vm_object *object;
  123 };
  124 #define VM_MAX_MEMSEGS  3
  125 
  126 struct mem_map {
  127         vm_paddr_t      gpa;
  128         size_t          len;
  129         vm_ooffset_t    segoff;
  130         int             segid;
  131         int             prot;
  132         int             flags;
  133 };
  134 #define VM_MAX_MEMMAPS  4
  135 
  136 /*
  137  * Initialization:
  138  * (o) initialized the first time the VM is created
  139  * (i) initialized when VM is created and when it is reinitialized
  140  * (x) initialized before use
  141  */
  142 struct vm {
  143         void            *cookie;                /* (i) cpu-specific data */
  144         void            *iommu;                 /* (x) iommu-specific data */
  145         struct vhpet    *vhpet;                 /* (i) virtual HPET */
  146         struct vioapic  *vioapic;               /* (i) virtual ioapic */
  147         struct vatpic   *vatpic;                /* (i) virtual atpic */
  148         struct vatpit   *vatpit;                /* (i) virtual atpit */
  149         struct vpmtmr   *vpmtmr;                /* (i) virtual ACPI PM timer */
  150         struct vrtc     *vrtc;                  /* (o) virtual RTC */
  151         volatile cpuset_t active_cpus;          /* (i) active vcpus */
  152         int             suspend;                /* (i) stop VM execution */
  153         volatile cpuset_t suspended_cpus;       /* (i) suspended vcpus */
  154         volatile cpuset_t halted_cpus;          /* (x) cpus in a hard halt */
  155         cpuset_t        rendezvous_req_cpus;    /* (x) rendezvous requested */
  156         cpuset_t        rendezvous_done_cpus;   /* (x) rendezvous finished */
  157         void            *rendezvous_arg;        /* (x) rendezvous func/arg */
  158         vm_rendezvous_func_t rendezvous_func;
  159         struct mtx      rendezvous_mtx;         /* (o) rendezvous lock */
  160         struct mem_map  mem_maps[VM_MAX_MEMMAPS]; /* (i) guest address space */
  161         struct mem_seg  mem_segs[VM_MAX_MEMSEGS]; /* (o) guest memory regions */
  162         struct vmspace  *vmspace;               /* (o) guest's address space */
  163         char            name[VM_MAX_NAMELEN];   /* (o) virtual machine name */
  164         struct vcpu     vcpu[VM_MAXCPU];        /* (i) guest vcpus */
  165 };
  166 
  167 static int vmm_initialized;
  168 
  169 static struct vmm_ops *ops;
  170 #define VMM_INIT(num)   (ops != NULL ? (*ops->init)(num) : 0)
  171 #define VMM_CLEANUP()   (ops != NULL ? (*ops->cleanup)() : 0)
  172 #define VMM_RESUME()    (ops != NULL ? (*ops->resume)() : 0)
  173 
  174 #define VMINIT(vm, pmap) (ops != NULL ? (*ops->vminit)(vm, pmap): NULL)
  175 #define VMRUN(vmi, vcpu, rip, pmap, evinfo) \
  176         (ops != NULL ? (*ops->vmrun)(vmi, vcpu, rip, pmap, evinfo) : ENXIO)
  177 #define VMCLEANUP(vmi)  (ops != NULL ? (*ops->vmcleanup)(vmi) : NULL)
  178 #define VMSPACE_ALLOC(min, max) \
  179         (ops != NULL ? (*ops->vmspace_alloc)(min, max) : NULL)
  180 #define VMSPACE_FREE(vmspace) \
  181         (ops != NULL ? (*ops->vmspace_free)(vmspace) : ENXIO)
  182 #define VMGETREG(vmi, vcpu, num, retval)                \
  183         (ops != NULL ? (*ops->vmgetreg)(vmi, vcpu, num, retval) : ENXIO)
  184 #define VMSETREG(vmi, vcpu, num, val)           \
  185         (ops != NULL ? (*ops->vmsetreg)(vmi, vcpu, num, val) : ENXIO)
  186 #define VMGETDESC(vmi, vcpu, num, desc)         \
  187         (ops != NULL ? (*ops->vmgetdesc)(vmi, vcpu, num, desc) : ENXIO)
  188 #define VMSETDESC(vmi, vcpu, num, desc)         \
  189         (ops != NULL ? (*ops->vmsetdesc)(vmi, vcpu, num, desc) : ENXIO)
  190 #define VMGETCAP(vmi, vcpu, num, retval)        \
  191         (ops != NULL ? (*ops->vmgetcap)(vmi, vcpu, num, retval) : ENXIO)
  192 #define VMSETCAP(vmi, vcpu, num, val)           \
  193         (ops != NULL ? (*ops->vmsetcap)(vmi, vcpu, num, val) : ENXIO)
  194 #define VLAPIC_INIT(vmi, vcpu)                  \
  195         (ops != NULL ? (*ops->vlapic_init)(vmi, vcpu) : NULL)
  196 #define VLAPIC_CLEANUP(vmi, vlapic)             \
  197         (ops != NULL ? (*ops->vlapic_cleanup)(vmi, vlapic) : NULL)
  198 
  199 #define fpu_start_emulating()   load_cr0(rcr0() | CR0_TS)
  200 #define fpu_stop_emulating()    clts()
  201 
  202 static MALLOC_DEFINE(M_VM, "vm", "vm");
  203 
  204 /* statistics */
  205 static VMM_STAT(VCPU_TOTAL_RUNTIME, "vcpu total runtime");
  206 
  207 SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW, NULL, NULL);
  208 
  209 /*
  210  * Halt the guest if all vcpus are executing a HLT instruction with
  211  * interrupts disabled.
  212  */
  213 static int halt_detection_enabled = 1;
  214 SYSCTL_INT(_hw_vmm, OID_AUTO, halt_detection, CTLFLAG_RDTUN,
  215     &halt_detection_enabled, 0,
  216     "Halt VM if all vcpus execute HLT with interrupts disabled");
  217 
  218 static int vmm_ipinum;
  219 SYSCTL_INT(_hw_vmm, OID_AUTO, ipinum, CTLFLAG_RD, &vmm_ipinum, 0,
  220     "IPI vector used for vcpu notifications");
  221 
  222 static int trace_guest_exceptions;
  223 SYSCTL_INT(_hw_vmm, OID_AUTO, trace_guest_exceptions, CTLFLAG_RDTUN,
  224     &trace_guest_exceptions, 0,
  225     "Trap into hypervisor on all guest exceptions and reflect them back");
  226 
  227 static void vm_free_memmap(struct vm *vm, int ident);
  228 static bool sysmem_mapping(struct vm *vm, struct mem_map *mm);
  229 static void vcpu_notify_event_locked(struct vcpu *vcpu, bool lapic_intr);
  230 
  231 #ifdef KTR
  232 static const char *
  233 vcpu_state2str(enum vcpu_state state)
  234 {
  235 
  236         switch (state) {
  237         case VCPU_IDLE:
  238                 return ("idle");
  239         case VCPU_FROZEN:
  240                 return ("frozen");
  241         case VCPU_RUNNING:
  242                 return ("running");
  243         case VCPU_SLEEPING:
  244                 return ("sleeping");
  245         default:
  246                 return ("unknown");
  247         }
  248 }
  249 #endif
  250 
  251 static void
  252 vcpu_cleanup(struct vm *vm, int i, bool destroy)
  253 {
  254         struct vcpu *vcpu = &vm->vcpu[i];
  255 
  256         VLAPIC_CLEANUP(vm->cookie, vcpu->vlapic);
  257         if (destroy) {
  258                 vmm_stat_free(vcpu->stats);     
  259                 fpu_save_area_free(vcpu->guestfpu);
  260         }
  261 }
  262 
  263 static void
  264 vcpu_init(struct vm *vm, int vcpu_id, bool create)
  265 {
  266         struct vcpu *vcpu;
  267 
  268         KASSERT(vcpu_id >= 0 && vcpu_id < VM_MAXCPU,
  269             ("vcpu_init: invalid vcpu %d", vcpu_id));
  270           
  271         vcpu = &vm->vcpu[vcpu_id];
  272 
  273         if (create) {
  274                 KASSERT(!vcpu_lock_initialized(vcpu), ("vcpu %d already "
  275                     "initialized", vcpu_id));
  276                 vcpu_lock_init(vcpu);
  277                 vcpu->state = VCPU_IDLE;
  278                 vcpu->hostcpu = NOCPU;
  279                 vcpu->guestfpu = fpu_save_area_alloc();
  280                 vcpu->stats = vmm_stat_alloc();
  281         }
  282 
  283         vcpu->vlapic = VLAPIC_INIT(vm->cookie, vcpu_id);
  284         vm_set_x2apic_state(vm, vcpu_id, X2APIC_DISABLED);
  285         vcpu->reqidle = 0;
  286         vcpu->exitintinfo = 0;
  287         vcpu->nmi_pending = 0;
  288         vcpu->extint_pending = 0;
  289         vcpu->exception_pending = 0;
  290         vcpu->guest_xcr0 = XFEATURE_ENABLED_X87;
  291         fpu_save_area_reset(vcpu->guestfpu);
  292         vmm_stat_init(vcpu->stats);
  293 }
  294 
  295 int
  296 vcpu_trace_exceptions(struct vm *vm, int vcpuid)
  297 {
  298 
  299         return (trace_guest_exceptions);
  300 }
  301 
  302 struct vm_exit *
  303 vm_exitinfo(struct vm *vm, int cpuid)
  304 {
  305         struct vcpu *vcpu;
  306 
  307         if (cpuid < 0 || cpuid >= VM_MAXCPU)
  308                 panic("vm_exitinfo: invalid cpuid %d", cpuid);
  309 
  310         vcpu = &vm->vcpu[cpuid];
  311 
  312         return (&vcpu->exitinfo);
  313 }
  314 
  315 static void
  316 vmm_resume(void)
  317 {
  318         VMM_RESUME();
  319 }
  320 
  321 static int
  322 vmm_init(void)
  323 {
  324         int error;
  325 
  326         vmm_host_state_init();
  327 
  328         vmm_ipinum = lapic_ipi_alloc(&IDTVEC(justreturn));
  329         if (vmm_ipinum < 0)
  330                 vmm_ipinum = IPI_AST;
  331 
  332         error = vmm_mem_init();
  333         if (error)
  334                 return (error);
  335         
  336         if (vmm_is_intel())
  337                 ops = &vmm_ops_intel;
  338         else if (vmm_is_amd())
  339                 ops = &vmm_ops_amd;
  340         else
  341                 return (ENXIO);
  342 
  343         vmm_resume_p = vmm_resume;
  344 
  345         return (VMM_INIT(vmm_ipinum));
  346 }
  347 
  348 static int
  349 vmm_handler(module_t mod, int what, void *arg)
  350 {
  351         int error;
  352 
  353         switch (what) {
  354         case MOD_LOAD:
  355                 vmmdev_init();
  356                 error = vmm_init();
  357                 if (error == 0)
  358                         vmm_initialized = 1;
  359                 break;
  360         case MOD_UNLOAD:
  361                 error = vmmdev_cleanup();
  362                 if (error == 0) {
  363                         vmm_resume_p = NULL;
  364                         iommu_cleanup();
  365                         if (vmm_ipinum != IPI_AST)
  366                                 lapic_ipi_free(vmm_ipinum);
  367                         error = VMM_CLEANUP();
  368                         /*
  369                          * Something bad happened - prevent new
  370                          * VMs from being created
  371                          */
  372                         if (error)
  373                                 vmm_initialized = 0;
  374                 }
  375                 break;
  376         default:
  377                 error = 0;
  378                 break;
  379         }
  380         return (error);
  381 }
  382 
  383 static moduledata_t vmm_kmod = {
  384         "vmm",
  385         vmm_handler,
  386         NULL
  387 };
  388 
  389 /*
  390  * vmm initialization has the following dependencies:
  391  *
  392  * - VT-x initialization requires smp_rendezvous() and therefore must happen
  393  *   after SMP is fully functional (after SI_SUB_SMP).
  394  */
  395 DECLARE_MODULE(vmm, vmm_kmod, SI_SUB_SMP + 1, SI_ORDER_ANY);
  396 MODULE_VERSION(vmm, 1);
  397 
  398 static void
  399 vm_init(struct vm *vm, bool create)
  400 {
  401         int i;
  402 
  403         vm->cookie = VMINIT(vm, vmspace_pmap(vm->vmspace));
  404         vm->iommu = NULL;
  405         vm->vioapic = vioapic_init(vm);
  406         vm->vhpet = vhpet_init(vm);
  407         vm->vatpic = vatpic_init(vm);
  408         vm->vatpit = vatpit_init(vm);
  409         vm->vpmtmr = vpmtmr_init(vm);
  410         if (create)
  411                 vm->vrtc = vrtc_init(vm);
  412 
  413         CPU_ZERO(&vm->active_cpus);
  414 
  415         vm->suspend = 0;
  416         CPU_ZERO(&vm->suspended_cpus);
  417 
  418         for (i = 0; i < VM_MAXCPU; i++)
  419                 vcpu_init(vm, i, create);
  420 }
  421 
  422 int
  423 vm_create(const char *name, struct vm **retvm)
  424 {
  425         struct vm *vm;
  426         struct vmspace *vmspace;
  427 
  428         /*
  429          * If vmm.ko could not be successfully initialized then don't attempt
  430          * to create the virtual machine.
  431          */
  432         if (!vmm_initialized)
  433                 return (ENXIO);
  434 
  435         if (name == NULL || strlen(name) >= VM_MAX_NAMELEN)
  436                 return (EINVAL);
  437 
  438         vmspace = VMSPACE_ALLOC(0, VM_MAXUSER_ADDRESS);
  439         if (vmspace == NULL)
  440                 return (ENOMEM);
  441 
  442         vm = malloc(sizeof(struct vm), M_VM, M_WAITOK | M_ZERO);
  443         strcpy(vm->name, name);
  444         vm->vmspace = vmspace;
  445         mtx_init(&vm->rendezvous_mtx, "vm rendezvous lock", 0, MTX_DEF);
  446 
  447         vm_init(vm, true);
  448 
  449         *retvm = vm;
  450         return (0);
  451 }
  452 
  453 static void
  454 vm_cleanup(struct vm *vm, bool destroy)
  455 {
  456         struct mem_map *mm;
  457         int i;
  458 
  459         ppt_unassign_all(vm);
  460 
  461         if (vm->iommu != NULL)
  462                 iommu_destroy_domain(vm->iommu);
  463 
  464         if (destroy)
  465                 vrtc_cleanup(vm->vrtc);
  466         else
  467                 vrtc_reset(vm->vrtc);
  468         vpmtmr_cleanup(vm->vpmtmr);
  469         vatpit_cleanup(vm->vatpit);
  470         vhpet_cleanup(vm->vhpet);
  471         vatpic_cleanup(vm->vatpic);
  472         vioapic_cleanup(vm->vioapic);
  473 
  474         for (i = 0; i < VM_MAXCPU; i++)
  475                 vcpu_cleanup(vm, i, destroy);
  476 
  477         VMCLEANUP(vm->cookie);
  478 
  479         /*
  480          * System memory is removed from the guest address space only when
  481          * the VM is destroyed. This is because the mapping remains the same
  482          * across VM reset.
  483          *
  484          * Device memory can be relocated by the guest (e.g. using PCI BARs)
  485          * so those mappings are removed on a VM reset.
  486          */
  487         for (i = 0; i < VM_MAX_MEMMAPS; i++) {
  488                 mm = &vm->mem_maps[i];
  489                 if (destroy || !sysmem_mapping(vm, mm))
  490                         vm_free_memmap(vm, i);
  491         }
  492 
  493         if (destroy) {
  494                 for (i = 0; i < VM_MAX_MEMSEGS; i++)
  495                         vm_free_memseg(vm, i);
  496 
  497                 VMSPACE_FREE(vm->vmspace);
  498                 vm->vmspace = NULL;
  499         }
  500 }
  501 
  502 void
  503 vm_destroy(struct vm *vm)
  504 {
  505         vm_cleanup(vm, true);
  506         free(vm, M_VM);
  507 }
  508 
  509 int
  510 vm_reinit(struct vm *vm)
  511 {
  512         int error;
  513 
  514         /*
  515          * A virtual machine can be reset only if all vcpus are suspended.
  516          */
  517         if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) {
  518                 vm_cleanup(vm, false);
  519                 vm_init(vm, false);
  520                 error = 0;
  521         } else {
  522                 error = EBUSY;
  523         }
  524 
  525         return (error);
  526 }
  527 
  528 const char *
  529 vm_name(struct vm *vm)
  530 {
  531         return (vm->name);
  532 }
  533 
  534 int
  535 vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa)
  536 {
  537         vm_object_t obj;
  538 
  539         if ((obj = vmm_mmio_alloc(vm->vmspace, gpa, len, hpa)) == NULL)
  540                 return (ENOMEM);
  541         else
  542                 return (0);
  543 }
  544 
  545 int
  546 vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len)
  547 {
  548 
  549         vmm_mmio_free(vm->vmspace, gpa, len);
  550         return (0);
  551 }
  552 
  553 /*
  554  * Return 'true' if 'gpa' is allocated in the guest address space.
  555  *
  556  * This function is called in the context of a running vcpu which acts as
  557  * an implicit lock on 'vm->mem_maps[]'.
  558  */
  559 bool
  560 vm_mem_allocated(struct vm *vm, int vcpuid, vm_paddr_t gpa)
  561 {
  562         struct mem_map *mm;
  563         int i;
  564 
  565 #ifdef INVARIANTS
  566         int hostcpu, state;
  567         state = vcpu_get_state(vm, vcpuid, &hostcpu);
  568         KASSERT(state == VCPU_RUNNING && hostcpu == curcpu,
  569             ("%s: invalid vcpu state %d/%d", __func__, state, hostcpu));
  570 #endif
  571 
  572         for (i = 0; i < VM_MAX_MEMMAPS; i++) {
  573                 mm = &vm->mem_maps[i];
  574                 if (mm->len != 0 && gpa >= mm->gpa && gpa < mm->gpa + mm->len)
  575                         return (true);          /* 'gpa' is sysmem or devmem */
  576         }
  577 
  578         if (ppt_is_mmio(vm, gpa))
  579                 return (true);                  /* 'gpa' is pci passthru mmio */
  580 
  581         return (false);
  582 }
  583 
  584 int
  585 vm_alloc_memseg(struct vm *vm, int ident, size_t len, bool sysmem)
  586 {
  587         struct mem_seg *seg;
  588         vm_object_t obj;
  589 
  590         if (ident < 0 || ident >= VM_MAX_MEMSEGS)
  591                 return (EINVAL);
  592 
  593         if (len == 0 || (len & PAGE_MASK))
  594                 return (EINVAL);
  595 
  596         seg = &vm->mem_segs[ident];
  597         if (seg->object != NULL) {
  598                 if (seg->len == len && seg->sysmem == sysmem)
  599                         return (EEXIST);
  600                 else
  601                         return (EINVAL);
  602         }
  603 
  604         obj = vm_object_allocate(OBJT_DEFAULT, len >> PAGE_SHIFT);
  605         if (obj == NULL)
  606                 return (ENOMEM);
  607 
  608         seg->len = len;
  609         seg->object = obj;
  610         seg->sysmem = sysmem;
  611         return (0);
  612 }
  613 
  614 int
  615 vm_get_memseg(struct vm *vm, int ident, size_t *len, bool *sysmem,
  616     vm_object_t *objptr)
  617 {
  618         struct mem_seg *seg;
  619 
  620         if (ident < 0 || ident >= VM_MAX_MEMSEGS)
  621                 return (EINVAL);
  622 
  623         seg = &vm->mem_segs[ident];
  624         if (len)
  625                 *len = seg->len;
  626         if (sysmem)
  627                 *sysmem = seg->sysmem;
  628         if (objptr)
  629                 *objptr = seg->object;
  630         return (0);
  631 }
  632 
  633 void
  634 vm_free_memseg(struct vm *vm, int ident)
  635 {
  636         struct mem_seg *seg;
  637 
  638         KASSERT(ident >= 0 && ident < VM_MAX_MEMSEGS,
  639             ("%s: invalid memseg ident %d", __func__, ident));
  640 
  641         seg = &vm->mem_segs[ident];
  642         if (seg->object != NULL) {
  643                 vm_object_deallocate(seg->object);
  644                 bzero(seg, sizeof(struct mem_seg));
  645         }
  646 }
  647 
  648 int
  649 vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t first,
  650     size_t len, int prot, int flags)
  651 {
  652         struct mem_seg *seg;
  653         struct mem_map *m, *map;
  654         vm_ooffset_t last;
  655         int i, error;
  656 
  657         if (prot == 0 || (prot & ~(VM_PROT_ALL)) != 0)
  658                 return (EINVAL);
  659 
  660         if (flags & ~VM_MEMMAP_F_WIRED)
  661                 return (EINVAL);
  662 
  663         if (segid < 0 || segid >= VM_MAX_MEMSEGS)
  664                 return (EINVAL);
  665 
  666         seg = &vm->mem_segs[segid];
  667         if (seg->object == NULL)
  668                 return (EINVAL);
  669 
  670         last = first + len;
  671         if (first < 0 || first >= last || last > seg->len)
  672                 return (EINVAL);
  673 
  674         if ((gpa | first | last) & PAGE_MASK)
  675                 return (EINVAL);
  676 
  677         map = NULL;
  678         for (i = 0; i < VM_MAX_MEMMAPS; i++) {
  679                 m = &vm->mem_maps[i];
  680                 if (m->len == 0) {
  681                         map = m;
  682                         break;
  683                 }
  684         }
  685 
  686         if (map == NULL)
  687                 return (ENOSPC);
  688 
  689         error = vm_map_find(&vm->vmspace->vm_map, seg->object, first, &gpa,
  690             len, 0, VMFS_NO_SPACE, prot, prot, 0);
  691         if (error != KERN_SUCCESS)
  692                 return (EFAULT);
  693 
  694         vm_object_reference(seg->object);
  695 
  696         if (flags & VM_MEMMAP_F_WIRED) {
  697                 error = vm_map_wire(&vm->vmspace->vm_map, gpa, gpa + len,
  698                     VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
  699                 if (error != KERN_SUCCESS) {
  700                         vm_map_remove(&vm->vmspace->vm_map, gpa, gpa + len);
  701                         return (EFAULT);
  702                 }
  703         }
  704 
  705         map->gpa = gpa;
  706         map->len = len;
  707         map->segoff = first;
  708         map->segid = segid;
  709         map->prot = prot;
  710         map->flags = flags;
  711         return (0);
  712 }
  713 
  714 int
  715 vm_mmap_getnext(struct vm *vm, vm_paddr_t *gpa, int *segid,
  716     vm_ooffset_t *segoff, size_t *len, int *prot, int *flags)
  717 {
  718         struct mem_map *mm, *mmnext;
  719         int i;
  720 
  721         mmnext = NULL;
  722         for (i = 0; i < VM_MAX_MEMMAPS; i++) {
  723                 mm = &vm->mem_maps[i];
  724                 if (mm->len == 0 || mm->gpa < *gpa)
  725                         continue;
  726                 if (mmnext == NULL || mm->gpa < mmnext->gpa)
  727                         mmnext = mm;
  728         }
  729 
  730         if (mmnext != NULL) {
  731                 *gpa = mmnext->gpa;
  732                 if (segid)
  733                         *segid = mmnext->segid;
  734                 if (segoff)
  735                         *segoff = mmnext->segoff;
  736                 if (len)
  737                         *len = mmnext->len;
  738                 if (prot)
  739                         *prot = mmnext->prot;
  740                 if (flags)
  741                         *flags = mmnext->flags;
  742                 return (0);
  743         } else {
  744                 return (ENOENT);
  745         }
  746 }
  747 
  748 static void
  749 vm_free_memmap(struct vm *vm, int ident)
  750 {
  751         struct mem_map *mm;
  752         int error;
  753 
  754         mm = &vm->mem_maps[ident];
  755         if (mm->len) {
  756                 error = vm_map_remove(&vm->vmspace->vm_map, mm->gpa,
  757                     mm->gpa + mm->len);
  758                 KASSERT(error == KERN_SUCCESS, ("%s: vm_map_remove error %d",
  759                     __func__, error));
  760                 bzero(mm, sizeof(struct mem_map));
  761         }
  762 }
  763 
  764 static __inline bool
  765 sysmem_mapping(struct vm *vm, struct mem_map *mm)
  766 {
  767 
  768         if (mm->len != 0 && vm->mem_segs[mm->segid].sysmem)
  769                 return (true);
  770         else
  771                 return (false);
  772 }
  773 
  774 static vm_paddr_t
  775 sysmem_maxaddr(struct vm *vm)
  776 {
  777         struct mem_map *mm;
  778         vm_paddr_t maxaddr;
  779         int i;
  780 
  781         maxaddr = 0;
  782         for (i = 0; i < VM_MAX_MEMMAPS; i++) {
  783                 mm = &vm->mem_maps[i];
  784                 if (sysmem_mapping(vm, mm)) {
  785                         if (maxaddr < mm->gpa + mm->len)
  786                                 maxaddr = mm->gpa + mm->len;
  787                 }
  788         }
  789         return (maxaddr);
  790 }
  791 
  792 static void
  793 vm_iommu_modify(struct vm *vm, boolean_t map)
  794 {
  795         int i, sz;
  796         vm_paddr_t gpa, hpa;
  797         struct mem_map *mm;
  798         void *vp, *cookie, *host_domain;
  799 
  800         sz = PAGE_SIZE;
  801         host_domain = iommu_host_domain();
  802 
  803         for (i = 0; i < VM_MAX_MEMMAPS; i++) {
  804                 mm = &vm->mem_maps[i];
  805                 if (!sysmem_mapping(vm, mm))
  806                         continue;
  807 
  808                 if (map) {
  809                         KASSERT((mm->flags & VM_MEMMAP_F_IOMMU) == 0,
  810                             ("iommu map found invalid memmap %#lx/%#lx/%#x",
  811                             mm->gpa, mm->len, mm->flags));
  812                         if ((mm->flags & VM_MEMMAP_F_WIRED) == 0)
  813                                 continue;
  814                         mm->flags |= VM_MEMMAP_F_IOMMU;
  815                 } else {
  816                         if ((mm->flags & VM_MEMMAP_F_IOMMU) == 0)
  817                                 continue;
  818                         mm->flags &= ~VM_MEMMAP_F_IOMMU;
  819                         KASSERT((mm->flags & VM_MEMMAP_F_WIRED) != 0,
  820                             ("iommu unmap found invalid memmap %#lx/%#lx/%#x",
  821                             mm->gpa, mm->len, mm->flags));
  822                 }
  823 
  824                 gpa = mm->gpa;
  825                 while (gpa < mm->gpa + mm->len) {
  826                         vp = vm_gpa_hold(vm, -1, gpa, PAGE_SIZE, VM_PROT_WRITE,
  827                                          &cookie);
  828                         KASSERT(vp != NULL, ("vm(%s) could not map gpa %#lx",
  829                             vm_name(vm), gpa));
  830 
  831                         vm_gpa_release(cookie);
  832 
  833                         hpa = DMAP_TO_PHYS((uintptr_t)vp);
  834                         if (map) {
  835                                 iommu_create_mapping(vm->iommu, gpa, hpa, sz);
  836                                 iommu_remove_mapping(host_domain, hpa, sz);
  837                         } else {
  838                                 iommu_remove_mapping(vm->iommu, gpa, sz);
  839                                 iommu_create_mapping(host_domain, hpa, hpa, sz);
  840                         }
  841 
  842                         gpa += PAGE_SIZE;
  843                 }
  844         }
  845 
  846         /*
  847          * Invalidate the cached translations associated with the domain
  848          * from which pages were removed.
  849          */
  850         if (map)
  851                 iommu_invalidate_tlb(host_domain);
  852         else
  853                 iommu_invalidate_tlb(vm->iommu);
  854 }
  855 
  856 #define vm_iommu_unmap(vm)      vm_iommu_modify((vm), FALSE)
  857 #define vm_iommu_map(vm)        vm_iommu_modify((vm), TRUE)
  858 
  859 int
  860 vm_unassign_pptdev(struct vm *vm, int bus, int slot, int func)
  861 {
  862         int error;
  863 
  864         error = ppt_unassign_device(vm, bus, slot, func);
  865         if (error)
  866                 return (error);
  867 
  868         if (ppt_assigned_devices(vm) == 0)
  869                 vm_iommu_unmap(vm);
  870 
  871         return (0);
  872 }
  873 
  874 int
  875 vm_assign_pptdev(struct vm *vm, int bus, int slot, int func)
  876 {
  877         int error;
  878         vm_paddr_t maxaddr;
  879 
  880         /* Set up the IOMMU to do the 'gpa' to 'hpa' translation */
  881         if (ppt_assigned_devices(vm) == 0) {
  882                 KASSERT(vm->iommu == NULL,
  883                     ("vm_assign_pptdev: iommu must be NULL"));
  884                 maxaddr = sysmem_maxaddr(vm);
  885                 vm->iommu = iommu_create_domain(maxaddr);
  886                 if (vm->iommu == NULL)
  887                         return (ENXIO);
  888                 vm_iommu_map(vm);
  889         }
  890 
  891         error = ppt_assign_device(vm, bus, slot, func);
  892         return (error);
  893 }
  894 
  895 void *
  896 vm_gpa_hold(struct vm *vm, int vcpuid, vm_paddr_t gpa, size_t len, int reqprot,
  897             void **cookie)
  898 {
  899         int i, count, pageoff;
  900         struct mem_map *mm;
  901         vm_page_t m;
  902 #ifdef INVARIANTS
  903         /*
  904          * All vcpus are frozen by ioctls that modify the memory map
  905          * (e.g. VM_MMAP_MEMSEG). Therefore 'vm->memmap[]' stability is
  906          * guaranteed if at least one vcpu is in the VCPU_FROZEN state.
  907          */
  908         int state;
  909         KASSERT(vcpuid >= -1 && vcpuid < VM_MAXCPU, ("%s: invalid vcpuid %d",
  910             __func__, vcpuid));
  911         for (i = 0; i < VM_MAXCPU; i++) {
  912                 if (vcpuid != -1 && vcpuid != i)
  913                         continue;
  914                 state = vcpu_get_state(vm, i, NULL);
  915                 KASSERT(state == VCPU_FROZEN, ("%s: invalid vcpu state %d",
  916                     __func__, state));
  917         }
  918 #endif
  919         pageoff = gpa & PAGE_MASK;
  920         if (len > PAGE_SIZE - pageoff)
  921                 panic("vm_gpa_hold: invalid gpa/len: 0x%016lx/%lu", gpa, len);
  922 
  923         count = 0;
  924         for (i = 0; i < VM_MAX_MEMMAPS; i++) {
  925                 mm = &vm->mem_maps[i];
  926                 if (sysmem_mapping(vm, mm) && gpa >= mm->gpa &&
  927                     gpa < mm->gpa + mm->len) {
  928                         count = vm_fault_quick_hold_pages(&vm->vmspace->vm_map,
  929                             trunc_page(gpa), PAGE_SIZE, reqprot, &m, 1);
  930                         break;
  931                 }
  932         }
  933 
  934         if (count == 1) {
  935                 *cookie = m;
  936                 return ((void *)(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)) + pageoff));
  937         } else {
  938                 *cookie = NULL;
  939                 return (NULL);
  940         }
  941 }
  942 
  943 void
  944 vm_gpa_release(void *cookie)
  945 {
  946         vm_page_t m = cookie;
  947 
  948         vm_page_lock(m);
  949         vm_page_unhold(m);
  950         vm_page_unlock(m);
  951 }
  952 
  953 int
  954 vm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval)
  955 {
  956 
  957         if (vcpu < 0 || vcpu >= VM_MAXCPU)
  958                 return (EINVAL);
  959 
  960         if (reg >= VM_REG_LAST)
  961                 return (EINVAL);
  962 
  963         return (VMGETREG(vm->cookie, vcpu, reg, retval));
  964 }
  965 
  966 int
  967 vm_set_register(struct vm *vm, int vcpuid, int reg, uint64_t val)
  968 {
  969         struct vcpu *vcpu;
  970         int error;
  971 
  972         if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
  973                 return (EINVAL);
  974 
  975         if (reg >= VM_REG_LAST)
  976                 return (EINVAL);
  977 
  978         error = VMSETREG(vm->cookie, vcpuid, reg, val);
  979         if (error || reg != VM_REG_GUEST_RIP)
  980                 return (error);
  981 
  982         /* Set 'nextrip' to match the value of %rip */
  983         VCPU_CTR1(vm, vcpuid, "Setting nextrip to %#lx", val);
  984         vcpu = &vm->vcpu[vcpuid];
  985         vcpu->nextrip = val;
  986         return (0);
  987 }
  988 
  989 static boolean_t
  990 is_descriptor_table(int reg)
  991 {
  992 
  993         switch (reg) {
  994         case VM_REG_GUEST_IDTR:
  995         case VM_REG_GUEST_GDTR:
  996                 return (TRUE);
  997         default:
  998                 return (FALSE);
  999         }
 1000 }
 1001 
 1002 static boolean_t
 1003 is_segment_register(int reg)
 1004 {
 1005         
 1006         switch (reg) {
 1007         case VM_REG_GUEST_ES:
 1008         case VM_REG_GUEST_CS:
 1009         case VM_REG_GUEST_SS:
 1010         case VM_REG_GUEST_DS:
 1011         case VM_REG_GUEST_FS:
 1012         case VM_REG_GUEST_GS:
 1013         case VM_REG_GUEST_TR:
 1014         case VM_REG_GUEST_LDTR:
 1015                 return (TRUE);
 1016         default:
 1017                 return (FALSE);
 1018         }
 1019 }
 1020 
 1021 int
 1022 vm_get_seg_desc(struct vm *vm, int vcpu, int reg,
 1023                 struct seg_desc *desc)
 1024 {
 1025 
 1026         if (vcpu < 0 || vcpu >= VM_MAXCPU)
 1027                 return (EINVAL);
 1028 
 1029         if (!is_segment_register(reg) && !is_descriptor_table(reg))
 1030                 return (EINVAL);
 1031 
 1032         return (VMGETDESC(vm->cookie, vcpu, reg, desc));
 1033 }
 1034 
 1035 int
 1036 vm_set_seg_desc(struct vm *vm, int vcpu, int reg,
 1037                 struct seg_desc *desc)
 1038 {
 1039         if (vcpu < 0 || vcpu >= VM_MAXCPU)
 1040                 return (EINVAL);
 1041 
 1042         if (!is_segment_register(reg) && !is_descriptor_table(reg))
 1043                 return (EINVAL);
 1044 
 1045         return (VMSETDESC(vm->cookie, vcpu, reg, desc));
 1046 }
 1047 
 1048 static void
 1049 restore_guest_fpustate(struct vcpu *vcpu)
 1050 {
 1051 
 1052         /* flush host state to the pcb */
 1053         fpuexit(curthread);
 1054 
 1055         /* restore guest FPU state */
 1056         fpu_stop_emulating();
 1057         fpurestore(vcpu->guestfpu);
 1058 
 1059         /* restore guest XCR0 if XSAVE is enabled in the host */
 1060         if (rcr4() & CR4_XSAVE)
 1061                 load_xcr(0, vcpu->guest_xcr0);
 1062 
 1063         /*
 1064          * The FPU is now "dirty" with the guest's state so turn on emulation
 1065          * to trap any access to the FPU by the host.
 1066          */
 1067         fpu_start_emulating();
 1068 }
 1069 
 1070 static void
 1071 save_guest_fpustate(struct vcpu *vcpu)
 1072 {
 1073 
 1074         if ((rcr0() & CR0_TS) == 0)
 1075                 panic("fpu emulation not enabled in host!");
 1076 
 1077         /* save guest XCR0 and restore host XCR0 */
 1078         if (rcr4() & CR4_XSAVE) {
 1079                 vcpu->guest_xcr0 = rxcr(0);
 1080                 load_xcr(0, vmm_get_host_xcr0());
 1081         }
 1082 
 1083         /* save guest FPU state */
 1084         fpu_stop_emulating();
 1085         fpusave(vcpu->guestfpu);
 1086         fpu_start_emulating();
 1087 }
 1088 
 1089 static VMM_STAT(VCPU_IDLE_TICKS, "number of ticks vcpu was idle");
 1090 
 1091 static int
 1092 vcpu_set_state_locked(struct vm *vm, int vcpuid, enum vcpu_state newstate,
 1093     bool from_idle)
 1094 {
 1095         struct vcpu *vcpu;
 1096         int error;
 1097 
 1098         vcpu = &vm->vcpu[vcpuid];
 1099         vcpu_assert_locked(vcpu);
 1100 
 1101         /*
 1102          * State transitions from the vmmdev_ioctl() must always begin from
 1103          * the VCPU_IDLE state. This guarantees that there is only a single
 1104          * ioctl() operating on a vcpu at any point.
 1105          */
 1106         if (from_idle) {
 1107                 while (vcpu->state != VCPU_IDLE) {
 1108                         vcpu->reqidle = 1;
 1109                         vcpu_notify_event_locked(vcpu, false);
 1110                         VCPU_CTR1(vm, vcpuid, "vcpu state change from %s to "
 1111                             "idle requested", vcpu_state2str(vcpu->state));
 1112                         msleep_spin(&vcpu->state, &vcpu->mtx, "vmstat", hz);
 1113                 }
 1114         } else {
 1115                 KASSERT(vcpu->state != VCPU_IDLE, ("invalid transition from "
 1116                     "vcpu idle state"));
 1117         }
 1118 
 1119         if (vcpu->state == VCPU_RUNNING) {
 1120                 KASSERT(vcpu->hostcpu == curcpu, ("curcpu %d and hostcpu %d "
 1121                     "mismatch for running vcpu", curcpu, vcpu->hostcpu));
 1122         } else {
 1123                 KASSERT(vcpu->hostcpu == NOCPU, ("Invalid hostcpu %d for a "
 1124                     "vcpu that is not running", vcpu->hostcpu));
 1125         }
 1126 
 1127         /*
 1128          * The following state transitions are allowed:
 1129          * IDLE -> FROZEN -> IDLE
 1130          * FROZEN -> RUNNING -> FROZEN
 1131          * FROZEN -> SLEEPING -> FROZEN
 1132          */
 1133         switch (vcpu->state) {
 1134         case VCPU_IDLE:
 1135         case VCPU_RUNNING:
 1136         case VCPU_SLEEPING:
 1137                 error = (newstate != VCPU_FROZEN);
 1138                 break;
 1139         case VCPU_FROZEN:
 1140                 error = (newstate == VCPU_FROZEN);
 1141                 break;
 1142         default:
 1143                 error = 1;
 1144                 break;
 1145         }
 1146 
 1147         if (error)
 1148                 return (EBUSY);
 1149 
 1150         VCPU_CTR2(vm, vcpuid, "vcpu state changed from %s to %s",
 1151             vcpu_state2str(vcpu->state), vcpu_state2str(newstate));
 1152 
 1153         vcpu->state = newstate;
 1154         if (newstate == VCPU_RUNNING)
 1155                 vcpu->hostcpu = curcpu;
 1156         else
 1157                 vcpu->hostcpu = NOCPU;
 1158 
 1159         if (newstate == VCPU_IDLE)
 1160                 wakeup(&vcpu->state);
 1161 
 1162         return (0);
 1163 }
 1164 
 1165 static void
 1166 vcpu_require_state(struct vm *vm, int vcpuid, enum vcpu_state newstate)
 1167 {
 1168         int error;
 1169 
 1170         if ((error = vcpu_set_state(vm, vcpuid, newstate, false)) != 0)
 1171                 panic("Error %d setting state to %d\n", error, newstate);
 1172 }
 1173 
 1174 static void
 1175 vcpu_require_state_locked(struct vm *vm, int vcpuid, enum vcpu_state newstate)
 1176 {
 1177         int error;
 1178 
 1179         if ((error = vcpu_set_state_locked(vm, vcpuid, newstate, false)) != 0)
 1180                 panic("Error %d setting state to %d", error, newstate);
 1181 }
 1182 
 1183 static void
 1184 vm_set_rendezvous_func(struct vm *vm, vm_rendezvous_func_t func)
 1185 {
 1186 
 1187         KASSERT(mtx_owned(&vm->rendezvous_mtx), ("rendezvous_mtx not locked"));
 1188 
 1189         /*
 1190          * Update 'rendezvous_func' and execute a write memory barrier to
 1191          * ensure that it is visible across all host cpus. This is not needed
 1192          * for correctness but it does ensure that all the vcpus will notice
 1193          * that the rendezvous is requested immediately.
 1194          */
 1195         vm->rendezvous_func = func;
 1196         wmb();
 1197 }
 1198 
 1199 #define RENDEZVOUS_CTR0(vm, vcpuid, fmt)                                \
 1200         do {                                                            \
 1201                 if (vcpuid >= 0)                                        \
 1202                         VCPU_CTR0(vm, vcpuid, fmt);                     \
 1203                 else                                                    \
 1204                         VM_CTR0(vm, fmt);                               \
 1205         } while (0)
 1206 
 1207 static void
 1208 vm_handle_rendezvous(struct vm *vm, int vcpuid)
 1209 {
 1210 
 1211         KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < VM_MAXCPU),
 1212             ("vm_handle_rendezvous: invalid vcpuid %d", vcpuid));
 1213 
 1214         mtx_lock(&vm->rendezvous_mtx);
 1215         while (vm->rendezvous_func != NULL) {
 1216                 /* 'rendezvous_req_cpus' must be a subset of 'active_cpus' */
 1217                 CPU_AND(&vm->rendezvous_req_cpus, &vm->active_cpus);
 1218 
 1219                 if (vcpuid != -1 &&
 1220                     CPU_ISSET(vcpuid, &vm->rendezvous_req_cpus) &&
 1221                     !CPU_ISSET(vcpuid, &vm->rendezvous_done_cpus)) {
 1222                         VCPU_CTR0(vm, vcpuid, "Calling rendezvous func");
 1223                         (*vm->rendezvous_func)(vm, vcpuid, vm->rendezvous_arg);
 1224                         CPU_SET(vcpuid, &vm->rendezvous_done_cpus);
 1225                 }
 1226                 if (CPU_CMP(&vm->rendezvous_req_cpus,
 1227                     &vm->rendezvous_done_cpus) == 0) {
 1228                         VCPU_CTR0(vm, vcpuid, "Rendezvous completed");
 1229                         vm_set_rendezvous_func(vm, NULL);
 1230                         wakeup(&vm->rendezvous_func);
 1231                         break;
 1232                 }
 1233                 RENDEZVOUS_CTR0(vm, vcpuid, "Wait for rendezvous completion");
 1234                 mtx_sleep(&vm->rendezvous_func, &vm->rendezvous_mtx, 0,
 1235                     "vmrndv", 0);
 1236         }
 1237         mtx_unlock(&vm->rendezvous_mtx);
 1238 }
 1239 
 1240 /*
 1241  * Emulate a guest 'hlt' by sleeping until the vcpu is ready to run.
 1242  */
 1243 static int
 1244 vm_handle_hlt(struct vm *vm, int vcpuid, bool intr_disabled, bool *retu)
 1245 {
 1246         struct vcpu *vcpu;
 1247         const char *wmesg;
 1248         int t, vcpu_halted, vm_halted;
 1249 
 1250         KASSERT(!CPU_ISSET(vcpuid, &vm->halted_cpus), ("vcpu already halted"));
 1251 
 1252         vcpu = &vm->vcpu[vcpuid];
 1253         vcpu_halted = 0;
 1254         vm_halted = 0;
 1255 
 1256         vcpu_lock(vcpu);
 1257         while (1) {
 1258                 /*
 1259                  * Do a final check for pending NMI or interrupts before
 1260                  * really putting this thread to sleep. Also check for
 1261                  * software events that would cause this vcpu to wakeup.
 1262                  *
 1263                  * These interrupts/events could have happened after the
 1264                  * vcpu returned from VMRUN() and before it acquired the
 1265                  * vcpu lock above.
 1266                  */
 1267                 if (vm->rendezvous_func != NULL || vm->suspend || vcpu->reqidle)
 1268                         break;
 1269                 if (vm_nmi_pending(vm, vcpuid))
 1270                         break;
 1271                 if (!intr_disabled) {
 1272                         if (vm_extint_pending(vm, vcpuid) ||
 1273                             vlapic_pending_intr(vcpu->vlapic, NULL)) {
 1274                                 break;
 1275                         }
 1276                 }
 1277 
 1278                 /* Don't go to sleep if the vcpu thread needs to yield */
 1279                 if (vcpu_should_yield(vm, vcpuid))
 1280                         break;
 1281 
 1282                 /*
 1283                  * Some Linux guests implement "halt" by having all vcpus
 1284                  * execute HLT with interrupts disabled. 'halted_cpus' keeps
 1285                  * track of the vcpus that have entered this state. When all
 1286                  * vcpus enter the halted state the virtual machine is halted.
 1287                  */
 1288                 if (intr_disabled) {
 1289                         wmesg = "vmhalt";
 1290                         VCPU_CTR0(vm, vcpuid, "Halted");
 1291                         if (!vcpu_halted && halt_detection_enabled) {
 1292                                 vcpu_halted = 1;
 1293                                 CPU_SET_ATOMIC(vcpuid, &vm->halted_cpus);
 1294                         }
 1295                         if (CPU_CMP(&vm->halted_cpus, &vm->active_cpus) == 0) {
 1296                                 vm_halted = 1;
 1297                                 break;
 1298                         }
 1299                 } else {
 1300                         wmesg = "vmidle";
 1301                 }
 1302 
 1303                 t = ticks;
 1304                 vcpu_require_state_locked(vm, vcpuid, VCPU_SLEEPING);
 1305                 /*
 1306                  * XXX msleep_spin() cannot be interrupted by signals so
 1307                  * wake up periodically to check pending signals.
 1308                  */
 1309                 msleep_spin(vcpu, &vcpu->mtx, wmesg, hz);
 1310                 vcpu_require_state_locked(vm, vcpuid, VCPU_FROZEN);
 1311                 vmm_stat_incr(vm, vcpuid, VCPU_IDLE_TICKS, ticks - t);
 1312         }
 1313 
 1314         if (vcpu_halted)
 1315                 CPU_CLR_ATOMIC(vcpuid, &vm->halted_cpus);
 1316 
 1317         vcpu_unlock(vcpu);
 1318 
 1319         if (vm_halted)
 1320                 vm_suspend(vm, VM_SUSPEND_HALT);
 1321 
 1322         return (0);
 1323 }
 1324 
 1325 static int
 1326 vm_handle_paging(struct vm *vm, int vcpuid, bool *retu)
 1327 {
 1328         int rv, ftype;
 1329         struct vm_map *map;
 1330         struct vcpu *vcpu;
 1331         struct vm_exit *vme;
 1332 
 1333         vcpu = &vm->vcpu[vcpuid];
 1334         vme = &vcpu->exitinfo;
 1335 
 1336         KASSERT(vme->inst_length == 0, ("%s: invalid inst_length %d",
 1337             __func__, vme->inst_length));
 1338 
 1339         ftype = vme->u.paging.fault_type;
 1340         KASSERT(ftype == VM_PROT_READ ||
 1341             ftype == VM_PROT_WRITE || ftype == VM_PROT_EXECUTE,
 1342             ("vm_handle_paging: invalid fault_type %d", ftype));
 1343 
 1344         if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) {
 1345                 rv = pmap_emulate_accessed_dirty(vmspace_pmap(vm->vmspace),
 1346                     vme->u.paging.gpa, ftype);
 1347                 if (rv == 0) {
 1348                         VCPU_CTR2(vm, vcpuid, "%s bit emulation for gpa %#lx",
 1349                             ftype == VM_PROT_READ ? "accessed" : "dirty",
 1350                             vme->u.paging.gpa);
 1351                         goto done;
 1352                 }
 1353         }
 1354 
 1355         map = &vm->vmspace->vm_map;
 1356         rv = vm_fault(map, vme->u.paging.gpa, ftype, VM_FAULT_NORMAL);
 1357 
 1358         VCPU_CTR3(vm, vcpuid, "vm_handle_paging rv = %d, gpa = %#lx, "
 1359             "ftype = %d", rv, vme->u.paging.gpa, ftype);
 1360 
 1361         if (rv != KERN_SUCCESS)
 1362                 return (EFAULT);
 1363 done:
 1364         return (0);
 1365 }
 1366 
 1367 static int
 1368 vm_handle_inst_emul(struct vm *vm, int vcpuid, bool *retu)
 1369 {
 1370         struct vie *vie;
 1371         struct vcpu *vcpu;
 1372         struct vm_exit *vme;
 1373         uint64_t gla, gpa, cs_base;
 1374         struct vm_guest_paging *paging;
 1375         mem_region_read_t mread;
 1376         mem_region_write_t mwrite;
 1377         enum vm_cpu_mode cpu_mode;
 1378         int cs_d, error, fault;
 1379 
 1380         vcpu = &vm->vcpu[vcpuid];
 1381         vme = &vcpu->exitinfo;
 1382 
 1383         KASSERT(vme->inst_length == 0, ("%s: invalid inst_length %d",
 1384             __func__, vme->inst_length));
 1385 
 1386         gla = vme->u.inst_emul.gla;
 1387         gpa = vme->u.inst_emul.gpa;
 1388         cs_base = vme->u.inst_emul.cs_base;
 1389         cs_d = vme->u.inst_emul.cs_d;
 1390         vie = &vme->u.inst_emul.vie;
 1391         paging = &vme->u.inst_emul.paging;
 1392         cpu_mode = paging->cpu_mode;
 1393 
 1394         VCPU_CTR1(vm, vcpuid, "inst_emul fault accessing gpa %#lx", gpa);
 1395 
 1396         /* Fetch, decode and emulate the faulting instruction */
 1397         if (vie->num_valid == 0) {
 1398                 error = vmm_fetch_instruction(vm, vcpuid, paging, vme->rip +
 1399                     cs_base, VIE_INST_SIZE, vie, &fault);
 1400         } else {
 1401                 /*
 1402                  * The instruction bytes have already been copied into 'vie'
 1403                  */
 1404                 error = fault = 0;
 1405         }
 1406         if (error || fault)
 1407                 return (error);
 1408 
 1409         if (vmm_decode_instruction(vm, vcpuid, gla, cpu_mode, cs_d, vie) != 0) {
 1410                 VCPU_CTR1(vm, vcpuid, "Error decoding instruction at %#lx",
 1411                     vme->rip + cs_base);
 1412                 *retu = true;       /* dump instruction bytes in userspace */
 1413                 return (0);
 1414         }
 1415 
 1416         /*
 1417          * Update 'nextrip' based on the length of the emulated instruction.
 1418          */
 1419         vme->inst_length = vie->num_processed;
 1420         vcpu->nextrip += vie->num_processed;
 1421         VCPU_CTR1(vm, vcpuid, "nextrip updated to %#lx after instruction "
 1422             "decoding", vcpu->nextrip);
 1423  
 1424         /* return to userland unless this is an in-kernel emulated device */
 1425         if (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE) {
 1426                 mread = lapic_mmio_read;
 1427                 mwrite = lapic_mmio_write;
 1428         } else if (gpa >= VIOAPIC_BASE && gpa < VIOAPIC_BASE + VIOAPIC_SIZE) {
 1429                 mread = vioapic_mmio_read;
 1430                 mwrite = vioapic_mmio_write;
 1431         } else if (gpa >= VHPET_BASE && gpa < VHPET_BASE + VHPET_SIZE) {
 1432                 mread = vhpet_mmio_read;
 1433                 mwrite = vhpet_mmio_write;
 1434         } else {
 1435                 *retu = true;
 1436                 return (0);
 1437         }
 1438 
 1439         error = vmm_emulate_instruction(vm, vcpuid, gpa, vie, paging,
 1440             mread, mwrite, retu);
 1441 
 1442         return (error);
 1443 }
 1444 
 1445 static int
 1446 vm_handle_suspend(struct vm *vm, int vcpuid, bool *retu)
 1447 {
 1448         int i, done;
 1449         struct vcpu *vcpu;
 1450 
 1451         done = 0;
 1452         vcpu = &vm->vcpu[vcpuid];
 1453 
 1454         CPU_SET_ATOMIC(vcpuid, &vm->suspended_cpus);
 1455 
 1456         /*
 1457          * Wait until all 'active_cpus' have suspended themselves.
 1458          *
 1459          * Since a VM may be suspended at any time including when one or
 1460          * more vcpus are doing a rendezvous we need to call the rendezvous
 1461          * handler while we are waiting to prevent a deadlock.
 1462          */
 1463         vcpu_lock(vcpu);
 1464         while (1) {
 1465                 if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) {
 1466                         VCPU_CTR0(vm, vcpuid, "All vcpus suspended");
 1467                         break;
 1468                 }
 1469 
 1470                 if (vm->rendezvous_func == NULL) {
 1471                         VCPU_CTR0(vm, vcpuid, "Sleeping during suspend");
 1472                         vcpu_require_state_locked(vm, vcpuid, VCPU_SLEEPING);
 1473                         msleep_spin(vcpu, &vcpu->mtx, "vmsusp", hz);
 1474                         vcpu_require_state_locked(vm, vcpuid, VCPU_FROZEN);
 1475                 } else {
 1476                         VCPU_CTR0(vm, vcpuid, "Rendezvous during suspend");
 1477                         vcpu_unlock(vcpu);
 1478                         vm_handle_rendezvous(vm, vcpuid);
 1479                         vcpu_lock(vcpu);
 1480                 }
 1481         }
 1482         vcpu_unlock(vcpu);
 1483 
 1484         /*
 1485          * Wakeup the other sleeping vcpus and return to userspace.
 1486          */
 1487         for (i = 0; i < VM_MAXCPU; i++) {
 1488                 if (CPU_ISSET(i, &vm->suspended_cpus)) {
 1489                         vcpu_notify_event(vm, i, false);
 1490                 }
 1491         }
 1492 
 1493         *retu = true;
 1494         return (0);
 1495 }
 1496 
 1497 static int
 1498 vm_handle_reqidle(struct vm *vm, int vcpuid, bool *retu)
 1499 {
 1500         struct vcpu *vcpu = &vm->vcpu[vcpuid];
 1501 
 1502         vcpu_lock(vcpu);
 1503         KASSERT(vcpu->reqidle, ("invalid vcpu reqidle %d", vcpu->reqidle));
 1504         vcpu->reqidle = 0;
 1505         vcpu_unlock(vcpu);
 1506         *retu = true;
 1507         return (0);
 1508 }
 1509 
 1510 int
 1511 vm_suspend(struct vm *vm, enum vm_suspend_how how)
 1512 {
 1513         int i;
 1514 
 1515         if (how <= VM_SUSPEND_NONE || how >= VM_SUSPEND_LAST)
 1516                 return (EINVAL);
 1517 
 1518         if (atomic_cmpset_int(&vm->suspend, 0, how) == 0) {
 1519                 VM_CTR2(vm, "virtual machine already suspended %d/%d",
 1520                     vm->suspend, how);
 1521                 return (EALREADY);
 1522         }
 1523 
 1524         VM_CTR1(vm, "virtual machine successfully suspended %d", how);
 1525 
 1526         /*
 1527          * Notify all active vcpus that they are now suspended.
 1528          */
 1529         for (i = 0; i < VM_MAXCPU; i++) {
 1530                 if (CPU_ISSET(i, &vm->active_cpus))
 1531                         vcpu_notify_event(vm, i, false);
 1532         }
 1533 
 1534         return (0);
 1535 }
 1536 
 1537 void
 1538 vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip)
 1539 {
 1540         struct vm_exit *vmexit;
 1541 
 1542         KASSERT(vm->suspend > VM_SUSPEND_NONE && vm->suspend < VM_SUSPEND_LAST,
 1543             ("vm_exit_suspended: invalid suspend type %d", vm->suspend));
 1544 
 1545         vmexit = vm_exitinfo(vm, vcpuid);
 1546         vmexit->rip = rip;
 1547         vmexit->inst_length = 0;
 1548         vmexit->exitcode = VM_EXITCODE_SUSPENDED;
 1549         vmexit->u.suspended.how = vm->suspend;
 1550 }
 1551 
 1552 void
 1553 vm_exit_rendezvous(struct vm *vm, int vcpuid, uint64_t rip)
 1554 {
 1555         struct vm_exit *vmexit;
 1556 
 1557         KASSERT(vm->rendezvous_func != NULL, ("rendezvous not in progress"));
 1558 
 1559         vmexit = vm_exitinfo(vm, vcpuid);
 1560         vmexit->rip = rip;
 1561         vmexit->inst_length = 0;
 1562         vmexit->exitcode = VM_EXITCODE_RENDEZVOUS;
 1563         vmm_stat_incr(vm, vcpuid, VMEXIT_RENDEZVOUS, 1);
 1564 }
 1565 
 1566 void
 1567 vm_exit_reqidle(struct vm *vm, int vcpuid, uint64_t rip)
 1568 {
 1569         struct vm_exit *vmexit;
 1570 
 1571         vmexit = vm_exitinfo(vm, vcpuid);
 1572         vmexit->rip = rip;
 1573         vmexit->inst_length = 0;
 1574         vmexit->exitcode = VM_EXITCODE_REQIDLE;
 1575         vmm_stat_incr(vm, vcpuid, VMEXIT_REQIDLE, 1);
 1576 }
 1577 
 1578 void
 1579 vm_exit_astpending(struct vm *vm, int vcpuid, uint64_t rip)
 1580 {
 1581         struct vm_exit *vmexit;
 1582 
 1583         vmexit = vm_exitinfo(vm, vcpuid);
 1584         vmexit->rip = rip;
 1585         vmexit->inst_length = 0;
 1586         vmexit->exitcode = VM_EXITCODE_BOGUS;
 1587         vmm_stat_incr(vm, vcpuid, VMEXIT_ASTPENDING, 1);
 1588 }
 1589 
 1590 int
 1591 vm_run(struct vm *vm, struct vm_run *vmrun)
 1592 {
 1593         struct vm_eventinfo evinfo;
 1594         int error, vcpuid;
 1595         struct vcpu *vcpu;
 1596         struct pcb *pcb;
 1597         uint64_t tscval;
 1598         struct vm_exit *vme;
 1599         bool retu, intr_disabled;
 1600         pmap_t pmap;
 1601 
 1602         vcpuid = vmrun->cpuid;
 1603 
 1604         if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 1605                 return (EINVAL);
 1606 
 1607         if (!CPU_ISSET(vcpuid, &vm->active_cpus))
 1608                 return (EINVAL);
 1609 
 1610         if (CPU_ISSET(vcpuid, &vm->suspended_cpus))
 1611                 return (EINVAL);
 1612 
 1613         pmap = vmspace_pmap(vm->vmspace);
 1614         vcpu = &vm->vcpu[vcpuid];
 1615         vme = &vcpu->exitinfo;
 1616         evinfo.rptr = &vm->rendezvous_func;
 1617         evinfo.sptr = &vm->suspend;
 1618         evinfo.iptr = &vcpu->reqidle;
 1619 restart:
 1620         critical_enter();
 1621 
 1622         KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active),
 1623             ("vm_run: absurd pm_active"));
 1624 
 1625         tscval = rdtsc();
 1626 
 1627         pcb = PCPU_GET(curpcb);
 1628         set_pcb_flags(pcb, PCB_FULL_IRET);
 1629 
 1630         restore_guest_fpustate(vcpu);
 1631 
 1632         vcpu_require_state(vm, vcpuid, VCPU_RUNNING);
 1633         error = VMRUN(vm->cookie, vcpuid, vcpu->nextrip, pmap, &evinfo);
 1634         vcpu_require_state(vm, vcpuid, VCPU_FROZEN);
 1635 
 1636         save_guest_fpustate(vcpu);
 1637 
 1638         vmm_stat_incr(vm, vcpuid, VCPU_TOTAL_RUNTIME, rdtsc() - tscval);
 1639 
 1640         critical_exit();
 1641 
 1642         if (error == 0) {
 1643                 retu = false;
 1644                 vcpu->nextrip = vme->rip + vme->inst_length;
 1645                 switch (vme->exitcode) {
 1646                 case VM_EXITCODE_REQIDLE:
 1647                         error = vm_handle_reqidle(vm, vcpuid, &retu);
 1648                         break;
 1649                 case VM_EXITCODE_SUSPENDED:
 1650                         error = vm_handle_suspend(vm, vcpuid, &retu);
 1651                         break;
 1652                 case VM_EXITCODE_IOAPIC_EOI:
 1653                         vioapic_process_eoi(vm, vcpuid,
 1654                             vme->u.ioapic_eoi.vector);
 1655                         break;
 1656                 case VM_EXITCODE_RENDEZVOUS:
 1657                         vm_handle_rendezvous(vm, vcpuid);
 1658                         error = 0;
 1659                         break;
 1660                 case VM_EXITCODE_HLT:
 1661                         intr_disabled = ((vme->u.hlt.rflags & PSL_I) == 0);
 1662                         error = vm_handle_hlt(vm, vcpuid, intr_disabled, &retu);
 1663                         break;
 1664                 case VM_EXITCODE_PAGING:
 1665                         error = vm_handle_paging(vm, vcpuid, &retu);
 1666                         break;
 1667                 case VM_EXITCODE_INST_EMUL:
 1668                         error = vm_handle_inst_emul(vm, vcpuid, &retu);
 1669                         break;
 1670                 case VM_EXITCODE_INOUT:
 1671                 case VM_EXITCODE_INOUT_STR:
 1672                         error = vm_handle_inout(vm, vcpuid, vme, &retu);
 1673                         break;
 1674                 case VM_EXITCODE_MONITOR:
 1675                 case VM_EXITCODE_MWAIT:
 1676                         vm_inject_ud(vm, vcpuid);
 1677                         break;
 1678                 default:
 1679                         retu = true;    /* handled in userland */
 1680                         break;
 1681                 }
 1682         }
 1683 
 1684         if (error == 0 && retu == false)
 1685                 goto restart;
 1686 
 1687         VCPU_CTR2(vm, vcpuid, "retu %d/%d", error, vme->exitcode);
 1688 
 1689         /* copy the exit information */
 1690         bcopy(vme, &vmrun->vm_exit, sizeof(struct vm_exit));
 1691         return (error);
 1692 }
 1693 
 1694 int
 1695 vm_restart_instruction(void *arg, int vcpuid)
 1696 {
 1697         struct vm *vm;
 1698         struct vcpu *vcpu;
 1699         enum vcpu_state state;
 1700         uint64_t rip;
 1701         int error;
 1702 
 1703         vm = arg;
 1704         if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 1705                 return (EINVAL);
 1706 
 1707         vcpu = &vm->vcpu[vcpuid];
 1708         state = vcpu_get_state(vm, vcpuid, NULL);
 1709         if (state == VCPU_RUNNING) {
 1710                 /*
 1711                  * When a vcpu is "running" the next instruction is determined
 1712                  * by adding 'rip' and 'inst_length' in the vcpu's 'exitinfo'.
 1713                  * Thus setting 'inst_length' to zero will cause the current
 1714                  * instruction to be restarted.
 1715                  */
 1716                 vcpu->exitinfo.inst_length = 0;
 1717                 VCPU_CTR1(vm, vcpuid, "restarting instruction at %#lx by "
 1718                     "setting inst_length to zero", vcpu->exitinfo.rip);
 1719         } else if (state == VCPU_FROZEN) {
 1720                 /*
 1721                  * When a vcpu is "frozen" it is outside the critical section
 1722                  * around VMRUN() and 'nextrip' points to the next instruction.
 1723                  * Thus instruction restart is achieved by setting 'nextrip'
 1724                  * to the vcpu's %rip.
 1725                  */
 1726                 error = vm_get_register(vm, vcpuid, VM_REG_GUEST_RIP, &rip);
 1727                 KASSERT(!error, ("%s: error %d getting rip", __func__, error));
 1728                 VCPU_CTR2(vm, vcpuid, "restarting instruction by updating "
 1729                     "nextrip from %#lx to %#lx", vcpu->nextrip, rip);
 1730                 vcpu->nextrip = rip;
 1731         } else {
 1732                 panic("%s: invalid state %d", __func__, state);
 1733         }
 1734         return (0);
 1735 }
 1736 
 1737 int
 1738 vm_exit_intinfo(struct vm *vm, int vcpuid, uint64_t info)
 1739 {
 1740         struct vcpu *vcpu;
 1741         int type, vector;
 1742 
 1743         if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 1744                 return (EINVAL);
 1745 
 1746         vcpu = &vm->vcpu[vcpuid];
 1747 
 1748         if (info & VM_INTINFO_VALID) {
 1749                 type = info & VM_INTINFO_TYPE;
 1750                 vector = info & 0xff;
 1751                 if (type == VM_INTINFO_NMI && vector != IDT_NMI)
 1752                         return (EINVAL);
 1753                 if (type == VM_INTINFO_HWEXCEPTION && vector >= 32)
 1754                         return (EINVAL);
 1755                 if (info & VM_INTINFO_RSVD)
 1756                         return (EINVAL);
 1757         } else {
 1758                 info = 0;
 1759         }
 1760         VCPU_CTR2(vm, vcpuid, "%s: info1(%#lx)", __func__, info);
 1761         vcpu->exitintinfo = info;
 1762         return (0);
 1763 }
 1764 
 1765 enum exc_class {
 1766         EXC_BENIGN,
 1767         EXC_CONTRIBUTORY,
 1768         EXC_PAGEFAULT
 1769 };
 1770 
 1771 #define IDT_VE  20      /* Virtualization Exception (Intel specific) */
 1772 
 1773 static enum exc_class
 1774 exception_class(uint64_t info)
 1775 {
 1776         int type, vector;
 1777 
 1778         KASSERT(info & VM_INTINFO_VALID, ("intinfo must be valid: %#lx", info));
 1779         type = info & VM_INTINFO_TYPE;
 1780         vector = info & 0xff;
 1781 
 1782         /* Table 6-4, "Interrupt and Exception Classes", Intel SDM, Vol 3 */
 1783         switch (type) {
 1784         case VM_INTINFO_HWINTR:
 1785         case VM_INTINFO_SWINTR:
 1786         case VM_INTINFO_NMI:
 1787                 return (EXC_BENIGN);
 1788         default:
 1789                 /*
 1790                  * Hardware exception.
 1791                  *
 1792                  * SVM and VT-x use identical type values to represent NMI,
 1793                  * hardware interrupt and software interrupt.
 1794                  *
 1795                  * SVM uses type '3' for all exceptions. VT-x uses type '3'
 1796                  * for exceptions except #BP and #OF. #BP and #OF use a type
 1797                  * value of '5' or '6'. Therefore we don't check for explicit
 1798                  * values of 'type' to classify 'intinfo' into a hardware
 1799                  * exception.
 1800                  */
 1801                 break;
 1802         }
 1803 
 1804         switch (vector) {
 1805         case IDT_PF:
 1806         case IDT_VE:
 1807                 return (EXC_PAGEFAULT);
 1808         case IDT_DE:
 1809         case IDT_TS:
 1810         case IDT_NP:
 1811         case IDT_SS:
 1812         case IDT_GP:
 1813                 return (EXC_CONTRIBUTORY);
 1814         default:
 1815                 return (EXC_BENIGN);
 1816         }
 1817 }
 1818 
 1819 static int
 1820 nested_fault(struct vm *vm, int vcpuid, uint64_t info1, uint64_t info2,
 1821     uint64_t *retinfo)
 1822 {
 1823         enum exc_class exc1, exc2;
 1824         int type1, vector1;
 1825 
 1826         KASSERT(info1 & VM_INTINFO_VALID, ("info1 %#lx is not valid", info1));
 1827         KASSERT(info2 & VM_INTINFO_VALID, ("info2 %#lx is not valid", info2));
 1828 
 1829         /*
 1830          * If an exception occurs while attempting to call the double-fault
 1831          * handler the processor enters shutdown mode (aka triple fault).
 1832          */
 1833         type1 = info1 & VM_INTINFO_TYPE;
 1834         vector1 = info1 & 0xff;
 1835         if (type1 == VM_INTINFO_HWEXCEPTION && vector1 == IDT_DF) {
 1836                 VCPU_CTR2(vm, vcpuid, "triple fault: info1(%#lx), info2(%#lx)",
 1837                     info1, info2);
 1838                 vm_suspend(vm, VM_SUSPEND_TRIPLEFAULT);
 1839                 *retinfo = 0;
 1840                 return (0);
 1841         }
 1842 
 1843         /*
 1844          * Table 6-5 "Conditions for Generating a Double Fault", Intel SDM, Vol3
 1845          */
 1846         exc1 = exception_class(info1);
 1847         exc2 = exception_class(info2);
 1848         if ((exc1 == EXC_CONTRIBUTORY && exc2 == EXC_CONTRIBUTORY) ||
 1849             (exc1 == EXC_PAGEFAULT && exc2 != EXC_BENIGN)) {
 1850                 /* Convert nested fault into a double fault. */
 1851                 *retinfo = IDT_DF;
 1852                 *retinfo |= VM_INTINFO_VALID | VM_INTINFO_HWEXCEPTION;
 1853                 *retinfo |= VM_INTINFO_DEL_ERRCODE;
 1854         } else {
 1855                 /* Handle exceptions serially */
 1856                 *retinfo = info2;
 1857         }
 1858         return (1);
 1859 }
 1860 
 1861 static uint64_t
 1862 vcpu_exception_intinfo(struct vcpu *vcpu)
 1863 {
 1864         uint64_t info = 0;
 1865 
 1866         if (vcpu->exception_pending) {
 1867                 info = vcpu->exc_vector & 0xff;
 1868                 info |= VM_INTINFO_VALID | VM_INTINFO_HWEXCEPTION;
 1869                 if (vcpu->exc_errcode_valid) {
 1870                         info |= VM_INTINFO_DEL_ERRCODE;
 1871                         info |= (uint64_t)vcpu->exc_errcode << 32;
 1872                 }
 1873         }
 1874         return (info);
 1875 }
 1876 
 1877 int
 1878 vm_entry_intinfo(struct vm *vm, int vcpuid, uint64_t *retinfo)
 1879 {
 1880         struct vcpu *vcpu;
 1881         uint64_t info1, info2;
 1882         int valid;
 1883 
 1884         KASSERT(vcpuid >= 0 && vcpuid < VM_MAXCPU, ("invalid vcpu %d", vcpuid));
 1885 
 1886         vcpu = &vm->vcpu[vcpuid];
 1887 
 1888         info1 = vcpu->exitintinfo;
 1889         vcpu->exitintinfo = 0;
 1890 
 1891         info2 = 0;
 1892         if (vcpu->exception_pending) {
 1893                 info2 = vcpu_exception_intinfo(vcpu);
 1894                 vcpu->exception_pending = 0;
 1895                 VCPU_CTR2(vm, vcpuid, "Exception %d delivered: %#lx",
 1896                     vcpu->exc_vector, info2);
 1897         }
 1898 
 1899         if ((info1 & VM_INTINFO_VALID) && (info2 & VM_INTINFO_VALID)) {
 1900                 valid = nested_fault(vm, vcpuid, info1, info2, retinfo);
 1901         } else if (info1 & VM_INTINFO_VALID) {
 1902                 *retinfo = info1;
 1903                 valid = 1;
 1904         } else if (info2 & VM_INTINFO_VALID) {
 1905                 *retinfo = info2;
 1906                 valid = 1;
 1907         } else {
 1908                 valid = 0;
 1909         }
 1910 
 1911         if (valid) {
 1912                 VCPU_CTR4(vm, vcpuid, "%s: info1(%#lx), info2(%#lx), "
 1913                     "retinfo(%#lx)", __func__, info1, info2, *retinfo);
 1914         }
 1915 
 1916         return (valid);
 1917 }
 1918 
 1919 int
 1920 vm_get_intinfo(struct vm *vm, int vcpuid, uint64_t *info1, uint64_t *info2)
 1921 {
 1922         struct vcpu *vcpu;
 1923 
 1924         if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 1925                 return (EINVAL);
 1926 
 1927         vcpu = &vm->vcpu[vcpuid];
 1928         *info1 = vcpu->exitintinfo;
 1929         *info2 = vcpu_exception_intinfo(vcpu);
 1930         return (0);
 1931 }
 1932 
 1933 int
 1934 vm_inject_exception(struct vm *vm, int vcpuid, int vector, int errcode_valid,
 1935     uint32_t errcode, int restart_instruction)
 1936 {
 1937         struct vcpu *vcpu;
 1938         uint64_t regval;
 1939         int error;
 1940 
 1941         if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 1942                 return (EINVAL);
 1943 
 1944         if (vector < 0 || vector >= 32)
 1945                 return (EINVAL);
 1946 
 1947         /*
 1948          * A double fault exception should never be injected directly into
 1949          * the guest. It is a derived exception that results from specific
 1950          * combinations of nested faults.
 1951          */
 1952         if (vector == IDT_DF)
 1953                 return (EINVAL);
 1954 
 1955         vcpu = &vm->vcpu[vcpuid];
 1956 
 1957         if (vcpu->exception_pending) {
 1958                 VCPU_CTR2(vm, vcpuid, "Unable to inject exception %d due to "
 1959                     "pending exception %d", vector, vcpu->exc_vector);
 1960                 return (EBUSY);
 1961         }
 1962 
 1963         if (errcode_valid) {
 1964                 /*
 1965                  * Exceptions don't deliver an error code in real mode.
 1966                  */
 1967                 error = vm_get_register(vm, vcpuid, VM_REG_GUEST_CR0, &regval);
 1968                 KASSERT(!error, ("%s: error %d getting CR0", __func__, error));
 1969                 if (!(regval & CR0_PE))
 1970                         errcode_valid = 0;
 1971         }
 1972 
 1973         /*
 1974          * From section 26.6.1 "Interruptibility State" in Intel SDM:
 1975          *
 1976          * Event blocking by "STI" or "MOV SS" is cleared after guest executes
 1977          * one instruction or incurs an exception.
 1978          */
 1979         error = vm_set_register(vm, vcpuid, VM_REG_GUEST_INTR_SHADOW, 0);
 1980         KASSERT(error == 0, ("%s: error %d clearing interrupt shadow",
 1981             __func__, error));
 1982 
 1983         if (restart_instruction)
 1984                 vm_restart_instruction(vm, vcpuid);
 1985 
 1986         vcpu->exception_pending = 1;
 1987         vcpu->exc_vector = vector;
 1988         vcpu->exc_errcode = errcode;
 1989         vcpu->exc_errcode_valid = errcode_valid;
 1990         VCPU_CTR1(vm, vcpuid, "Exception %d pending", vector);
 1991         return (0);
 1992 }
 1993 
 1994 void
 1995 vm_inject_fault(void *vmarg, int vcpuid, int vector, int errcode_valid,
 1996     int errcode)
 1997 {
 1998         struct vm *vm;
 1999         int error, restart_instruction;
 2000 
 2001         vm = vmarg;
 2002         restart_instruction = 1;
 2003 
 2004         error = vm_inject_exception(vm, vcpuid, vector, errcode_valid,
 2005             errcode, restart_instruction);
 2006         KASSERT(error == 0, ("vm_inject_exception error %d", error));
 2007 }
 2008 
 2009 void
 2010 vm_inject_pf(void *vmarg, int vcpuid, int error_code, uint64_t cr2)
 2011 {
 2012         struct vm *vm;
 2013         int error;
 2014 
 2015         vm = vmarg;
 2016         VCPU_CTR2(vm, vcpuid, "Injecting page fault: error_code %#x, cr2 %#lx",
 2017             error_code, cr2);
 2018 
 2019         error = vm_set_register(vm, vcpuid, VM_REG_GUEST_CR2, cr2);
 2020         KASSERT(error == 0, ("vm_set_register(cr2) error %d", error));
 2021 
 2022         vm_inject_fault(vm, vcpuid, IDT_PF, 1, error_code);
 2023 }
 2024 
 2025 static VMM_STAT(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu");
 2026 
 2027 int
 2028 vm_inject_nmi(struct vm *vm, int vcpuid)
 2029 {
 2030         struct vcpu *vcpu;
 2031 
 2032         if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 2033                 return (EINVAL);
 2034 
 2035         vcpu = &vm->vcpu[vcpuid];
 2036 
 2037         vcpu->nmi_pending = 1;
 2038         vcpu_notify_event(vm, vcpuid, false);
 2039         return (0);
 2040 }
 2041 
 2042 int
 2043 vm_nmi_pending(struct vm *vm, int vcpuid)
 2044 {
 2045         struct vcpu *vcpu;
 2046 
 2047         if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 2048                 panic("vm_nmi_pending: invalid vcpuid %d", vcpuid);
 2049 
 2050         vcpu = &vm->vcpu[vcpuid];
 2051 
 2052         return (vcpu->nmi_pending);
 2053 }
 2054 
 2055 void
 2056 vm_nmi_clear(struct vm *vm, int vcpuid)
 2057 {
 2058         struct vcpu *vcpu;
 2059 
 2060         if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 2061                 panic("vm_nmi_pending: invalid vcpuid %d", vcpuid);
 2062 
 2063         vcpu = &vm->vcpu[vcpuid];
 2064 
 2065         if (vcpu->nmi_pending == 0)
 2066                 panic("vm_nmi_clear: inconsistent nmi_pending state");
 2067 
 2068         vcpu->nmi_pending = 0;
 2069         vmm_stat_incr(vm, vcpuid, VCPU_NMI_COUNT, 1);
 2070 }
 2071 
 2072 static VMM_STAT(VCPU_EXTINT_COUNT, "number of ExtINTs delivered to vcpu");
 2073 
 2074 int
 2075 vm_inject_extint(struct vm *vm, int vcpuid)
 2076 {
 2077         struct vcpu *vcpu;
 2078 
 2079         if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 2080                 return (EINVAL);
 2081 
 2082         vcpu = &vm->vcpu[vcpuid];
 2083 
 2084         vcpu->extint_pending = 1;
 2085         vcpu_notify_event(vm, vcpuid, false);
 2086         return (0);
 2087 }
 2088 
 2089 int
 2090 vm_extint_pending(struct vm *vm, int vcpuid)
 2091 {
 2092         struct vcpu *vcpu;
 2093 
 2094         if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 2095                 panic("vm_extint_pending: invalid vcpuid %d", vcpuid);
 2096 
 2097         vcpu = &vm->vcpu[vcpuid];
 2098 
 2099         return (vcpu->extint_pending);
 2100 }
 2101 
 2102 void
 2103 vm_extint_clear(struct vm *vm, int vcpuid)
 2104 {
 2105         struct vcpu *vcpu;
 2106 
 2107         if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 2108                 panic("vm_extint_pending: invalid vcpuid %d", vcpuid);
 2109 
 2110         vcpu = &vm->vcpu[vcpuid];
 2111 
 2112         if (vcpu->extint_pending == 0)
 2113                 panic("vm_extint_clear: inconsistent extint_pending state");
 2114 
 2115         vcpu->extint_pending = 0;
 2116         vmm_stat_incr(vm, vcpuid, VCPU_EXTINT_COUNT, 1);
 2117 }
 2118 
 2119 int
 2120 vm_get_capability(struct vm *vm, int vcpu, int type, int *retval)
 2121 {
 2122         if (vcpu < 0 || vcpu >= VM_MAXCPU)
 2123                 return (EINVAL);
 2124 
 2125         if (type < 0 || type >= VM_CAP_MAX)
 2126                 return (EINVAL);
 2127 
 2128         return (VMGETCAP(vm->cookie, vcpu, type, retval));
 2129 }
 2130 
 2131 int
 2132 vm_set_capability(struct vm *vm, int vcpu, int type, int val)
 2133 {
 2134         if (vcpu < 0 || vcpu >= VM_MAXCPU)
 2135                 return (EINVAL);
 2136 
 2137         if (type < 0 || type >= VM_CAP_MAX)
 2138                 return (EINVAL);
 2139 
 2140         return (VMSETCAP(vm->cookie, vcpu, type, val));
 2141 }
 2142 
 2143 struct vlapic *
 2144 vm_lapic(struct vm *vm, int cpu)
 2145 {
 2146         return (vm->vcpu[cpu].vlapic);
 2147 }
 2148 
 2149 struct vioapic *
 2150 vm_ioapic(struct vm *vm)
 2151 {
 2152 
 2153         return (vm->vioapic);
 2154 }
 2155 
 2156 struct vhpet *
 2157 vm_hpet(struct vm *vm)
 2158 {
 2159 
 2160         return (vm->vhpet);
 2161 }
 2162 
 2163 boolean_t
 2164 vmm_is_pptdev(int bus, int slot, int func)
 2165 {
 2166         int found, i, n;
 2167         int b, s, f;
 2168         char *val, *cp, *cp2;
 2169 
 2170         /*
 2171          * XXX
 2172          * The length of an environment variable is limited to 128 bytes which
 2173          * puts an upper limit on the number of passthru devices that may be
 2174          * specified using a single environment variable.
 2175          *
 2176          * Work around this by scanning multiple environment variable
 2177          * names instead of a single one - yuck!
 2178          */
 2179         const char *names[] = { "pptdevs", "pptdevs2", "pptdevs3", NULL };
 2180 
 2181         /* set pptdevs="1/2/3 4/5/6 7/8/9 10/11/12" */
 2182         found = 0;
 2183         for (i = 0; names[i] != NULL && !found; i++) {
 2184                 cp = val = kern_getenv(names[i]);
 2185                 while (cp != NULL && *cp != '\0') {
 2186                         if ((cp2 = strchr(cp, ' ')) != NULL)
 2187                                 *cp2 = '\0';
 2188 
 2189                         n = sscanf(cp, "%d/%d/%d", &b, &s, &f);
 2190                         if (n == 3 && bus == b && slot == s && func == f) {
 2191                                 found = 1;
 2192                                 break;
 2193                         }
 2194                 
 2195                         if (cp2 != NULL)
 2196                                 *cp2++ = ' ';
 2197 
 2198                         cp = cp2;
 2199                 }
 2200                 freeenv(val);
 2201         }
 2202         return (found);
 2203 }
 2204 
 2205 void *
 2206 vm_iommu_domain(struct vm *vm)
 2207 {
 2208 
 2209         return (vm->iommu);
 2210 }
 2211 
 2212 int
 2213 vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state newstate,
 2214     bool from_idle)
 2215 {
 2216         int error;
 2217         struct vcpu *vcpu;
 2218 
 2219         if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 2220                 panic("vm_set_run_state: invalid vcpuid %d", vcpuid);
 2221 
 2222         vcpu = &vm->vcpu[vcpuid];
 2223 
 2224         vcpu_lock(vcpu);
 2225         error = vcpu_set_state_locked(vm, vcpuid, newstate, from_idle);
 2226         vcpu_unlock(vcpu);
 2227 
 2228         return (error);
 2229 }
 2230 
 2231 enum vcpu_state
 2232 vcpu_get_state(struct vm *vm, int vcpuid, int *hostcpu)
 2233 {
 2234         struct vcpu *vcpu;
 2235         enum vcpu_state state;
 2236 
 2237         if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 2238                 panic("vm_get_run_state: invalid vcpuid %d", vcpuid);
 2239 
 2240         vcpu = &vm->vcpu[vcpuid];
 2241 
 2242         vcpu_lock(vcpu);
 2243         state = vcpu->state;
 2244         if (hostcpu != NULL)
 2245                 *hostcpu = vcpu->hostcpu;
 2246         vcpu_unlock(vcpu);
 2247 
 2248         return (state);
 2249 }
 2250 
 2251 int
 2252 vm_activate_cpu(struct vm *vm, int vcpuid)
 2253 {
 2254 
 2255         if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 2256                 return (EINVAL);
 2257 
 2258         if (CPU_ISSET(vcpuid, &vm->active_cpus))
 2259                 return (EBUSY);
 2260 
 2261         VCPU_CTR0(vm, vcpuid, "activated");
 2262         CPU_SET_ATOMIC(vcpuid, &vm->active_cpus);
 2263         return (0);
 2264 }
 2265 
 2266 cpuset_t
 2267 vm_active_cpus(struct vm *vm)
 2268 {
 2269 
 2270         return (vm->active_cpus);
 2271 }
 2272 
 2273 cpuset_t
 2274 vm_suspended_cpus(struct vm *vm)
 2275 {
 2276 
 2277         return (vm->suspended_cpus);
 2278 }
 2279 
 2280 void *
 2281 vcpu_stats(struct vm *vm, int vcpuid)
 2282 {
 2283 
 2284         return (vm->vcpu[vcpuid].stats);
 2285 }
 2286 
 2287 int
 2288 vm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state)
 2289 {
 2290         if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 2291                 return (EINVAL);
 2292 
 2293         *state = vm->vcpu[vcpuid].x2apic_state;
 2294 
 2295         return (0);
 2296 }
 2297 
 2298 int
 2299 vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
 2300 {
 2301         if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 2302                 return (EINVAL);
 2303 
 2304         if (state >= X2APIC_STATE_LAST)
 2305                 return (EINVAL);
 2306 
 2307         vm->vcpu[vcpuid].x2apic_state = state;
 2308 
 2309         vlapic_set_x2apic_state(vm, vcpuid, state);
 2310 
 2311         return (0);
 2312 }
 2313 
 2314 /*
 2315  * This function is called to ensure that a vcpu "sees" a pending event
 2316  * as soon as possible:
 2317  * - If the vcpu thread is sleeping then it is woken up.
 2318  * - If the vcpu is running on a different host_cpu then an IPI will be directed
 2319  *   to the host_cpu to cause the vcpu to trap into the hypervisor.
 2320  */
 2321 static void
 2322 vcpu_notify_event_locked(struct vcpu *vcpu, bool lapic_intr)
 2323 {
 2324         int hostcpu;
 2325 
 2326         hostcpu = vcpu->hostcpu;
 2327         if (vcpu->state == VCPU_RUNNING) {
 2328                 KASSERT(hostcpu != NOCPU, ("vcpu running on invalid hostcpu"));
 2329                 if (hostcpu != curcpu) {
 2330                         if (lapic_intr) {
 2331                                 vlapic_post_intr(vcpu->vlapic, hostcpu,
 2332                                     vmm_ipinum);
 2333                         } else {
 2334                                 ipi_cpu(hostcpu, vmm_ipinum);
 2335                         }
 2336                 } else {
 2337                         /*
 2338                          * If the 'vcpu' is running on 'curcpu' then it must
 2339                          * be sending a notification to itself (e.g. SELF_IPI).
 2340                          * The pending event will be picked up when the vcpu
 2341                          * transitions back to guest context.
 2342                          */
 2343                 }
 2344         } else {
 2345                 KASSERT(hostcpu == NOCPU, ("vcpu state %d not consistent "
 2346                     "with hostcpu %d", vcpu->state, hostcpu));
 2347                 if (vcpu->state == VCPU_SLEEPING)
 2348                         wakeup_one(vcpu);
 2349         }
 2350 }
 2351 
 2352 void
 2353 vcpu_notify_event(struct vm *vm, int vcpuid, bool lapic_intr)
 2354 {
 2355         struct vcpu *vcpu = &vm->vcpu[vcpuid];
 2356 
 2357         vcpu_lock(vcpu);
 2358         vcpu_notify_event_locked(vcpu, lapic_intr);
 2359         vcpu_unlock(vcpu);
 2360 }
 2361 
 2362 struct vmspace *
 2363 vm_get_vmspace(struct vm *vm)
 2364 {
 2365 
 2366         return (vm->vmspace);
 2367 }
 2368 
 2369 int
 2370 vm_apicid2vcpuid(struct vm *vm, int apicid)
 2371 {
 2372         /*
 2373          * XXX apic id is assumed to be numerically identical to vcpu id
 2374          */
 2375         return (apicid);
 2376 }
 2377 
 2378 void
 2379 vm_smp_rendezvous(struct vm *vm, int vcpuid, cpuset_t dest,
 2380     vm_rendezvous_func_t func, void *arg)
 2381 {
 2382         int i;
 2383 
 2384         /*
 2385          * Enforce that this function is called without any locks
 2386          */
 2387         WITNESS_WARN(WARN_PANIC, NULL, "vm_smp_rendezvous");
 2388         KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < VM_MAXCPU),
 2389             ("vm_smp_rendezvous: invalid vcpuid %d", vcpuid));
 2390 
 2391 restart:
 2392         mtx_lock(&vm->rendezvous_mtx);
 2393         if (vm->rendezvous_func != NULL) {
 2394                 /*
 2395                  * If a rendezvous is already in progress then we need to
 2396                  * call the rendezvous handler in case this 'vcpuid' is one
 2397                  * of the targets of the rendezvous.
 2398                  */
 2399                 RENDEZVOUS_CTR0(vm, vcpuid, "Rendezvous already in progress");
 2400                 mtx_unlock(&vm->rendezvous_mtx);
 2401                 vm_handle_rendezvous(vm, vcpuid);
 2402                 goto restart;
 2403         }
 2404         KASSERT(vm->rendezvous_func == NULL, ("vm_smp_rendezvous: previous "
 2405             "rendezvous is still in progress"));
 2406 
 2407         RENDEZVOUS_CTR0(vm, vcpuid, "Initiating rendezvous");
 2408         vm->rendezvous_req_cpus = dest;
 2409         CPU_ZERO(&vm->rendezvous_done_cpus);
 2410         vm->rendezvous_arg = arg;
 2411         vm_set_rendezvous_func(vm, func);
 2412         mtx_unlock(&vm->rendezvous_mtx);
 2413 
 2414         /*
 2415          * Wake up any sleeping vcpus and trigger a VM-exit in any running
 2416          * vcpus so they handle the rendezvous as soon as possible.
 2417          */
 2418         for (i = 0; i < VM_MAXCPU; i++) {
 2419                 if (CPU_ISSET(i, &dest))
 2420                         vcpu_notify_event(vm, i, false);
 2421         }
 2422 
 2423         vm_handle_rendezvous(vm, vcpuid);
 2424 }
 2425 
 2426 struct vatpic *
 2427 vm_atpic(struct vm *vm)
 2428 {
 2429         return (vm->vatpic);
 2430 }
 2431 
 2432 struct vatpit *
 2433 vm_atpit(struct vm *vm)
 2434 {
 2435         return (vm->vatpit);
 2436 }
 2437 
 2438 struct vpmtmr *
 2439 vm_pmtmr(struct vm *vm)
 2440 {
 2441 
 2442         return (vm->vpmtmr);
 2443 }
 2444 
 2445 struct vrtc *
 2446 vm_rtc(struct vm *vm)
 2447 {
 2448 
 2449         return (vm->vrtc);
 2450 }
 2451 
 2452 enum vm_reg_name
 2453 vm_segment_name(int seg)
 2454 {
 2455         static enum vm_reg_name seg_names[] = {
 2456                 VM_REG_GUEST_ES,
 2457                 VM_REG_GUEST_CS,
 2458                 VM_REG_GUEST_SS,
 2459                 VM_REG_GUEST_DS,
 2460                 VM_REG_GUEST_FS,
 2461                 VM_REG_GUEST_GS
 2462         };
 2463 
 2464         KASSERT(seg >= 0 && seg < nitems(seg_names),
 2465             ("%s: invalid segment encoding %d", __func__, seg));
 2466         return (seg_names[seg]);
 2467 }
 2468 
 2469 void
 2470 vm_copy_teardown(struct vm *vm, int vcpuid, struct vm_copyinfo *copyinfo,
 2471     int num_copyinfo)
 2472 {
 2473         int idx;
 2474 
 2475         for (idx = 0; idx < num_copyinfo; idx++) {
 2476                 if (copyinfo[idx].cookie != NULL)
 2477                         vm_gpa_release(copyinfo[idx].cookie);
 2478         }
 2479         bzero(copyinfo, num_copyinfo * sizeof(struct vm_copyinfo));
 2480 }
 2481 
 2482 int
 2483 vm_copy_setup(struct vm *vm, int vcpuid, struct vm_guest_paging *paging,
 2484     uint64_t gla, size_t len, int prot, struct vm_copyinfo *copyinfo,
 2485     int num_copyinfo, int *fault)
 2486 {
 2487         int error, idx, nused;
 2488         size_t n, off, remaining;
 2489         void *hva, *cookie;
 2490         uint64_t gpa;
 2491 
 2492         bzero(copyinfo, sizeof(struct vm_copyinfo) * num_copyinfo);
 2493 
 2494         nused = 0;
 2495         remaining = len;
 2496         while (remaining > 0) {
 2497                 KASSERT(nused < num_copyinfo, ("insufficient vm_copyinfo"));
 2498                 error = vm_gla2gpa(vm, vcpuid, paging, gla, prot, &gpa, fault);
 2499                 if (error || *fault)
 2500                         return (error);
 2501                 off = gpa & PAGE_MASK;
 2502                 n = min(remaining, PAGE_SIZE - off);
 2503                 copyinfo[nused].gpa = gpa;
 2504                 copyinfo[nused].len = n;
 2505                 remaining -= n;
 2506                 gla += n;
 2507                 nused++;
 2508         }
 2509 
 2510         for (idx = 0; idx < nused; idx++) {
 2511                 hva = vm_gpa_hold(vm, vcpuid, copyinfo[idx].gpa,
 2512                     copyinfo[idx].len, prot, &cookie);
 2513                 if (hva == NULL)
 2514                         break;
 2515                 copyinfo[idx].hva = hva;
 2516                 copyinfo[idx].cookie = cookie;
 2517         }
 2518 
 2519         if (idx != nused) {
 2520                 vm_copy_teardown(vm, vcpuid, copyinfo, num_copyinfo);
 2521                 return (EFAULT);
 2522         } else {
 2523                 *fault = 0;
 2524                 return (0);
 2525         }
 2526 }
 2527 
 2528 void
 2529 vm_copyin(struct vm *vm, int vcpuid, struct vm_copyinfo *copyinfo, void *kaddr,
 2530     size_t len)
 2531 {
 2532         char *dst;
 2533         int idx;
 2534         
 2535         dst = kaddr;
 2536         idx = 0;
 2537         while (len > 0) {
 2538                 bcopy(copyinfo[idx].hva, dst, copyinfo[idx].len);
 2539                 len -= copyinfo[idx].len;
 2540                 dst += copyinfo[idx].len;
 2541                 idx++;
 2542         }
 2543 }
 2544 
 2545 void
 2546 vm_copyout(struct vm *vm, int vcpuid, const void *kaddr,
 2547     struct vm_copyinfo *copyinfo, size_t len)
 2548 {
 2549         const char *src;
 2550         int idx;
 2551 
 2552         src = kaddr;
 2553         idx = 0;
 2554         while (len > 0) {
 2555                 bcopy(src, copyinfo[idx].hva, copyinfo[idx].len);
 2556                 len -= copyinfo[idx].len;
 2557                 src += copyinfo[idx].len;
 2558                 idx++;
 2559         }
 2560 }
 2561 
 2562 /*
 2563  * Return the amount of in-use and wired memory for the VM. Since
 2564  * these are global stats, only return the values with for vCPU 0
 2565  */
 2566 VMM_STAT_DECLARE(VMM_MEM_RESIDENT);
 2567 VMM_STAT_DECLARE(VMM_MEM_WIRED);
 2568 
 2569 static void
 2570 vm_get_rescnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat)
 2571 {
 2572 
 2573         if (vcpu == 0) {
 2574                 vmm_stat_set(vm, vcpu, VMM_MEM_RESIDENT,
 2575                     PAGE_SIZE * vmspace_resident_count(vm->vmspace));
 2576         }       
 2577 }
 2578 
 2579 static void
 2580 vm_get_wiredcnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat)
 2581 {
 2582 
 2583         if (vcpu == 0) {
 2584                 vmm_stat_set(vm, vcpu, VMM_MEM_WIRED,
 2585                     PAGE_SIZE * pmap_wired_count(vmspace_pmap(vm->vmspace)));
 2586         }       
 2587 }
 2588 
 2589 VMM_STAT_FUNC(VMM_MEM_RESIDENT, "Resident memory", vm_get_rescnt);
 2590 VMM_STAT_FUNC(VMM_MEM_WIRED, "Wired memory", vm_get_wiredcnt);

Cache object: 7980b4a7271030dceb7cdd83da7db247


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.