The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/i386/xen/xen_machdep.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  *
    3  * Copyright (c) 2004 Christian Limpach.
    4  * Copyright (c) 2004-2006,2008 Kip Macy
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice, this list of conditions and the following disclaimer.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice, this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  * 3. All advertising materials mentioning features or use of this software
   16  *    must display the following acknowledgement:
   17  *      This product includes software developed by Christian Limpach.
   18  * 4. The name of the author may not be used to endorse or promote products
   19  *    derived from this software without specific prior written permission.
   20  *
   21  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   22  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   23  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   24  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   25  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   26  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   27  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   28  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   29  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
   30  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   31  */
   32 
   33 #include <sys/cdefs.h>
   34 __FBSDID("$FreeBSD: releng/10.3/sys/i386/xen/xen_machdep.c 271132 2014-09-04 20:47:14Z emaste $");
   35 
   36 #include <sys/param.h>
   37 #include <sys/systm.h>
   38 #include <sys/bus.h>
   39 #include <sys/ktr.h>
   40 #include <sys/lock.h>
   41 #include <sys/mount.h>
   42 #include <sys/malloc.h>
   43 #include <sys/mutex.h>
   44 #include <sys/kernel.h>
   45 #include <sys/proc.h>
   46 #include <sys/reboot.h>
   47 #include <sys/rwlock.h>
   48 #include <sys/sysproto.h>
   49 #include <sys/boot.h>
   50 
   51 #include <xen/xen-os.h>
   52 
   53 #include <vm/vm.h>
   54 #include <vm/pmap.h>
   55 #include <machine/segments.h>
   56 #include <machine/pcb.h>
   57 #include <machine/stdarg.h>
   58 #include <machine/vmparam.h>
   59 #include <machine/cpu.h>
   60 #include <machine/intr_machdep.h>
   61 #include <machine/md_var.h>
   62 #include <machine/asmacros.h>
   63 
   64 
   65 
   66 #include <xen/hypervisor.h>
   67 #include <machine/xen/xenvar.h>
   68 #include <machine/xen/xenfunc.h>
   69 #include <machine/xen/xenpmap.h>
   70 #include <machine/xen/xenfunc.h>
   71 #include <xen/interface/memory.h>
   72 #include <machine/xen/features.h>
   73 #ifdef SMP
   74 #include <machine/privatespace.h>
   75 #endif
   76 
   77 
   78 #include <vm/vm_page.h>
   79 
   80 
   81 #define IDTVEC(name)    __CONCAT(X,name)
   82 
   83 extern inthand_t
   84 IDTVEC(div), IDTVEC(dbg), IDTVEC(nmi), IDTVEC(bpt), IDTVEC(ofl),
   85         IDTVEC(bnd), IDTVEC(ill), IDTVEC(dna), IDTVEC(fpusegm),
   86         IDTVEC(tss), IDTVEC(missing), IDTVEC(stk), IDTVEC(prot),
   87         IDTVEC(page), IDTVEC(mchk), IDTVEC(rsvd), IDTVEC(fpu), IDTVEC(align),
   88         IDTVEC(xmm), IDTVEC(lcall_syscall), IDTVEC(int0x80_syscall);
   89 
   90 
   91 int xendebug_flags; 
   92 start_info_t *xen_start_info;
   93 shared_info_t *HYPERVISOR_shared_info;
   94 xen_pfn_t *xen_machine_phys = machine_to_phys_mapping;
   95 xen_pfn_t *xen_phys_machine;
   96 xen_pfn_t *xen_pfn_to_mfn_frame_list[16];
   97 xen_pfn_t *xen_pfn_to_mfn_frame_list_list;
   98 int preemptable, init_first;
   99 extern unsigned int avail_space;
  100 int xen_vector_callback_enabled = 0;
  101 enum xen_domain_type xen_domain_type = XEN_PV_DOMAIN;
  102 
  103 void ni_cli(void);
  104 void ni_sti(void);
  105 
  106 
  107 void
  108 ni_cli(void)
  109 {
  110         CTR0(KTR_SPARE2, "ni_cli disabling interrupts");
  111         __asm__("pushl %edx;"
  112                 "pushl %eax;"
  113                 );
  114         __cli();
  115         __asm__("popl %eax;"
  116                 "popl %edx;"
  117                 );
  118 }
  119 
  120 
  121 void
  122 ni_sti(void)
  123 {
  124         __asm__("pushl %edx;"
  125                 "pushl %esi;"
  126                 "pushl %eax;"
  127                 );
  128         __sti();
  129         __asm__("popl %eax;"
  130                 "popl %esi;"
  131                 "popl %edx;"
  132                 );
  133 }
  134 
  135 void
  136 force_evtchn_callback(void)
  137 {
  138     (void)HYPERVISOR_xen_version(0, NULL);
  139 }
  140 
  141 /*
  142  * Modify the cmd_line by converting ',' to NULLs so that it is in a  format 
  143  * suitable for the static env vars.
  144  */
  145 char *
  146 xen_setbootenv(char *cmd_line)
  147 {
  148         char *cmd_line_next;
  149     
  150         /* Skip leading spaces */
  151         for (; *cmd_line == ' '; cmd_line++);
  152 
  153         xc_printf("xen_setbootenv(): cmd_line='%s'\n", cmd_line);
  154 
  155         for (cmd_line_next = cmd_line; strsep(&cmd_line_next, ",") != NULL;);
  156         return cmd_line;
  157 }
  158 
  159 int 
  160 xen_boothowto(char *envp)
  161 {
  162         int i, howto = 0;
  163 
  164         /* get equivalents from the environment */
  165         for (i = 0; howto_names[i].ev != NULL; i++)
  166                 if (getenv(howto_names[i].ev) != NULL)
  167                         howto |= howto_names[i].mask;
  168         return howto;
  169 }
  170 
  171 #define XC_PRINTF_BUFSIZE 1024
  172 void
  173 xc_printf(const char *fmt, ...)
  174 {
  175         __va_list ap;
  176         int retval;
  177         static char buf[XC_PRINTF_BUFSIZE];
  178 
  179         va_start(ap, fmt);
  180         retval = vsnprintf(buf, XC_PRINTF_BUFSIZE - 1, fmt, ap);
  181         va_end(ap);
  182         buf[retval] = 0;
  183         (void)HYPERVISOR_console_write(buf, retval);
  184 }
  185 
  186 
  187 #define XPQUEUE_SIZE 128
  188 
  189 struct mmu_log {
  190         char *file;
  191         int line;
  192 };
  193 
  194 #ifdef SMP
  195 /* per-cpu queues and indices */
  196 #ifdef INVARIANTS
  197 static struct mmu_log xpq_queue_log[XEN_LEGACY_MAX_VCPUS][XPQUEUE_SIZE];
  198 #endif
  199 
  200 static int xpq_idx[XEN_LEGACY_MAX_VCPUS];
  201 static mmu_update_t xpq_queue[XEN_LEGACY_MAX_VCPUS][XPQUEUE_SIZE];
  202 
  203 #define XPQ_QUEUE_LOG xpq_queue_log[vcpu]
  204 #define XPQ_QUEUE xpq_queue[vcpu]
  205 #define XPQ_IDX xpq_idx[vcpu]
  206 #define SET_VCPU() int vcpu = smp_processor_id()
  207 #else
  208         
  209 static mmu_update_t xpq_queue[XPQUEUE_SIZE];
  210 #ifdef INVARIANTS
  211 static struct mmu_log xpq_queue_log[XPQUEUE_SIZE];
  212 #endif
  213 static int xpq_idx = 0;
  214 
  215 #define XPQ_QUEUE_LOG xpq_queue_log
  216 #define XPQ_QUEUE xpq_queue
  217 #define XPQ_IDX xpq_idx
  218 #define SET_VCPU()
  219 #endif /* !SMP */
  220 
  221 #define XPQ_IDX_INC atomic_add_int(&XPQ_IDX, 1);
  222 
  223 #if 0
  224 static void
  225 xen_dump_queue(void)
  226 {
  227         int _xpq_idx = XPQ_IDX;
  228         int i;
  229 
  230         if (_xpq_idx <= 1)
  231                 return;
  232 
  233         xc_printf("xen_dump_queue(): %u entries\n", _xpq_idx);
  234         for (i = 0; i < _xpq_idx; i++) {
  235                 xc_printf(" val: %llx ptr: %llx\n", XPQ_QUEUE[i].val,
  236                     XPQ_QUEUE[i].ptr);
  237         }
  238 }
  239 #endif
  240 
  241 
  242 static __inline void
  243 _xen_flush_queue(void)
  244 {
  245         SET_VCPU();
  246         int _xpq_idx = XPQ_IDX;
  247         int error, i;
  248 
  249 #ifdef INVARIANTS
  250         if (__predict_true(gdtset))
  251                 CRITICAL_ASSERT(curthread);
  252 #endif
  253 
  254         XPQ_IDX = 0;
  255         /* Make sure index is cleared first to avoid double updates. */
  256         error = HYPERVISOR_mmu_update((mmu_update_t *)&XPQ_QUEUE,
  257                                       _xpq_idx, NULL, DOMID_SELF);
  258     
  259 #if 0
  260         if (__predict_true(gdtset))
  261         for (i = _xpq_idx; i > 0;) {
  262                 if (i >= 3) {
  263                         CTR6(KTR_PMAP, "mmu:val: %lx ptr: %lx val: %lx "
  264                             "ptr: %lx val: %lx ptr: %lx",
  265                             (XPQ_QUEUE[i-1].val & 0xffffffff),
  266                             (XPQ_QUEUE[i-1].ptr & 0xffffffff),
  267                             (XPQ_QUEUE[i-2].val & 0xffffffff),
  268                             (XPQ_QUEUE[i-2].ptr & 0xffffffff),
  269                             (XPQ_QUEUE[i-3].val & 0xffffffff),
  270                             (XPQ_QUEUE[i-3].ptr & 0xffffffff));
  271                             i -= 3;
  272                 } else if (i == 2) {
  273                         CTR4(KTR_PMAP, "mmu: val: %lx ptr: %lx val: %lx ptr: %lx",
  274                             (XPQ_QUEUE[i-1].val & 0xffffffff),
  275                             (XPQ_QUEUE[i-1].ptr & 0xffffffff),
  276                             (XPQ_QUEUE[i-2].val & 0xffffffff),
  277                             (XPQ_QUEUE[i-2].ptr & 0xffffffff));
  278                         i = 0;
  279                 } else {
  280                         CTR2(KTR_PMAP, "mmu: val: %lx ptr: %lx", 
  281                             (XPQ_QUEUE[i-1].val & 0xffffffff),
  282                             (XPQ_QUEUE[i-1].ptr & 0xffffffff));
  283                         i = 0;
  284                 }
  285         }
  286 #endif  
  287         if (__predict_false(error < 0)) {
  288                 for (i = 0; i < _xpq_idx; i++)
  289                         printf("val: %llx ptr: %llx\n",
  290                             XPQ_QUEUE[i].val, XPQ_QUEUE[i].ptr);
  291                 panic("Failed to execute MMU updates: %d", error);
  292         }
  293 
  294 }
  295 
  296 void
  297 xen_flush_queue(void)
  298 {
  299         SET_VCPU();
  300 
  301         if (__predict_true(gdtset))
  302                 critical_enter();
  303         if (XPQ_IDX != 0) _xen_flush_queue();
  304         if (__predict_true(gdtset))
  305                 critical_exit();
  306 }
  307 
  308 static __inline void
  309 xen_increment_idx(void)
  310 {
  311         SET_VCPU();
  312 
  313         XPQ_IDX++;
  314         if (__predict_false(XPQ_IDX == XPQUEUE_SIZE))
  315                 xen_flush_queue();
  316 }
  317 
  318 void
  319 xen_check_queue(void)
  320 {
  321 #ifdef INVARIANTS
  322         SET_VCPU();
  323         
  324         KASSERT(XPQ_IDX == 0, ("pending operations XPQ_IDX=%d", XPQ_IDX));
  325 #endif
  326 }
  327 
  328 void
  329 xen_invlpg(vm_offset_t va)
  330 {
  331         struct mmuext_op op;
  332         op.cmd = MMUEXT_INVLPG_ALL;
  333         op.arg1.linear_addr = va & ~PAGE_MASK;
  334         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  335 }
  336 
  337 void
  338 xen_load_cr3(u_int val)
  339 {
  340         struct mmuext_op op;
  341 #ifdef INVARIANTS
  342         SET_VCPU();
  343         
  344         KASSERT(XPQ_IDX == 0, ("pending operations XPQ_IDX=%d", XPQ_IDX));
  345 #endif
  346         op.cmd = MMUEXT_NEW_BASEPTR;
  347         op.arg1.mfn = xpmap_ptom(val) >> PAGE_SHIFT;
  348         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  349 }
  350 
  351 #ifdef KTR
  352 static __inline u_int
  353 rebp(void)
  354 {
  355         u_int   data;
  356 
  357         __asm __volatile("movl 4(%%ebp),%0" : "=r" (data));     
  358         return (data);
  359 }
  360 #endif
  361 
  362 u_int
  363 read_eflags(void)
  364 {
  365         vcpu_info_t *_vcpu;
  366         u_int eflags;
  367 
  368         eflags = _read_eflags();
  369         _vcpu = &HYPERVISOR_shared_info->vcpu_info[smp_processor_id()]; 
  370         if (_vcpu->evtchn_upcall_mask)
  371                 eflags &= ~PSL_I;
  372 
  373         return (eflags);
  374 }
  375 
  376 void
  377 write_eflags(u_int eflags)
  378 {
  379         u_int intr;
  380 
  381         CTR2(KTR_SPARE2, "%x xen_restore_flags eflags %x", rebp(), eflags);
  382         intr = ((eflags & PSL_I) == 0);
  383         __restore_flags(intr);
  384         _write_eflags(eflags);
  385 }
  386 
  387 void
  388 xen_cli(void)
  389 {
  390         CTR1(KTR_SPARE2, "%x xen_cli disabling interrupts", rebp());
  391         __cli();
  392 }
  393 
  394 void
  395 xen_sti(void)
  396 {
  397         CTR1(KTR_SPARE2, "%x xen_sti enabling interrupts", rebp());
  398         __sti();
  399 }
  400 
  401 u_int
  402 xen_rcr2(void)
  403 {
  404 
  405         return (HYPERVISOR_shared_info->vcpu_info[curcpu].arch.cr2);
  406 }
  407 
  408 void
  409 _xen_machphys_update(vm_paddr_t mfn, vm_paddr_t pfn, char *file, int line)
  410 {
  411         SET_VCPU();
  412         
  413         if (__predict_true(gdtset))
  414                 critical_enter();
  415         XPQ_QUEUE[XPQ_IDX].ptr = (mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE;
  416         XPQ_QUEUE[XPQ_IDX].val = pfn;
  417 #ifdef INVARIANTS
  418         XPQ_QUEUE_LOG[XPQ_IDX].file = file;
  419         XPQ_QUEUE_LOG[XPQ_IDX].line = line;     
  420 #endif          
  421         xen_increment_idx();
  422         if (__predict_true(gdtset))
  423                 critical_exit();
  424 }
  425 
  426 extern struct rwlock pvh_global_lock;
  427 
  428 void
  429 _xen_queue_pt_update(vm_paddr_t ptr, vm_paddr_t val, char *file, int line)
  430 {
  431         SET_VCPU();
  432 
  433         if (__predict_true(gdtset))     
  434                 rw_assert(&pvh_global_lock, RA_WLOCKED);
  435 
  436         KASSERT((ptr & 7) == 0, ("misaligned update"));
  437         
  438         if (__predict_true(gdtset))
  439                 critical_enter();
  440         
  441         XPQ_QUEUE[XPQ_IDX].ptr = ((uint64_t)ptr) | MMU_NORMAL_PT_UPDATE;
  442         XPQ_QUEUE[XPQ_IDX].val = (uint64_t)val;
  443 #ifdef INVARIANTS
  444         XPQ_QUEUE_LOG[XPQ_IDX].file = file;
  445         XPQ_QUEUE_LOG[XPQ_IDX].line = line;     
  446 #endif  
  447         xen_increment_idx();
  448         if (__predict_true(gdtset))
  449                 critical_exit();
  450 }
  451 
  452 void 
  453 xen_pgdpt_pin(vm_paddr_t ma)
  454 {
  455         struct mmuext_op op;
  456         op.cmd = MMUEXT_PIN_L3_TABLE;
  457         op.arg1.mfn = ma >> PAGE_SHIFT;
  458         xen_flush_queue();
  459         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  460 }
  461 
  462 void 
  463 xen_pgd_pin(vm_paddr_t ma)
  464 {
  465         struct mmuext_op op;
  466         op.cmd = MMUEXT_PIN_L2_TABLE;
  467         op.arg1.mfn = ma >> PAGE_SHIFT;
  468         xen_flush_queue();
  469         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  470 }
  471 
  472 void 
  473 xen_pgd_unpin(vm_paddr_t ma)
  474 {
  475         struct mmuext_op op;
  476         op.cmd = MMUEXT_UNPIN_TABLE;
  477         op.arg1.mfn = ma >> PAGE_SHIFT;
  478         xen_flush_queue();
  479         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  480 }
  481 
  482 void 
  483 xen_pt_pin(vm_paddr_t ma)
  484 {
  485         struct mmuext_op op;
  486         op.cmd = MMUEXT_PIN_L1_TABLE;
  487         op.arg1.mfn = ma >> PAGE_SHIFT;
  488         xen_flush_queue();
  489         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  490 }
  491 
  492 void 
  493 xen_pt_unpin(vm_paddr_t ma)
  494 {
  495         struct mmuext_op op;
  496         op.cmd = MMUEXT_UNPIN_TABLE;
  497         op.arg1.mfn = ma >> PAGE_SHIFT;
  498         xen_flush_queue();
  499         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  500 }
  501 
  502 void 
  503 xen_set_ldt(vm_paddr_t ptr, unsigned long len)
  504 {
  505         struct mmuext_op op;
  506         op.cmd = MMUEXT_SET_LDT;
  507         op.arg1.linear_addr = ptr;
  508         op.arg2.nr_ents = len;
  509         xen_flush_queue();
  510         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  511 }
  512 
  513 void xen_tlb_flush(void)
  514 {
  515         struct mmuext_op op;
  516         op.cmd = MMUEXT_TLB_FLUSH_LOCAL;
  517         xen_flush_queue();
  518         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  519 }
  520 
  521 void
  522 xen_update_descriptor(union descriptor *table, union descriptor *entry)
  523 {
  524         vm_paddr_t pa;
  525         pt_entry_t *ptp;
  526 
  527         ptp = vtopte((vm_offset_t)table);
  528         pa = (*ptp & PG_FRAME) | ((vm_offset_t)table & PAGE_MASK);
  529         if (HYPERVISOR_update_descriptor(pa, *(uint64_t *)entry))
  530                 panic("HYPERVISOR_update_descriptor failed\n");
  531 }
  532 
  533 
  534 #if 0
  535 /*
  536  * Bitmap is indexed by page number. If bit is set, the page is part of a
  537  * xen_create_contiguous_region() area of memory.
  538  */
  539 unsigned long *contiguous_bitmap;
  540 
  541 static void 
  542 contiguous_bitmap_set(unsigned long first_page, unsigned long nr_pages)
  543 {
  544         unsigned long start_off, end_off, curr_idx, end_idx;
  545 
  546         curr_idx  = first_page / BITS_PER_LONG;
  547         start_off = first_page & (BITS_PER_LONG-1);
  548         end_idx   = (first_page + nr_pages) / BITS_PER_LONG;
  549         end_off   = (first_page + nr_pages) & (BITS_PER_LONG-1);
  550 
  551         if (curr_idx == end_idx) {
  552                 contiguous_bitmap[curr_idx] |=
  553                         ((1UL<<end_off)-1) & -(1UL<<start_off);
  554         } else {
  555                 contiguous_bitmap[curr_idx] |= -(1UL<<start_off);
  556                 while ( ++curr_idx < end_idx )
  557                         contiguous_bitmap[curr_idx] = ~0UL;
  558                 contiguous_bitmap[curr_idx] |= (1UL<<end_off)-1;
  559         }
  560 }
  561 
  562 static void 
  563 contiguous_bitmap_clear(unsigned long first_page, unsigned long nr_pages)
  564 {
  565         unsigned long start_off, end_off, curr_idx, end_idx;
  566 
  567         curr_idx  = first_page / BITS_PER_LONG;
  568         start_off = first_page & (BITS_PER_LONG-1);
  569         end_idx   = (first_page + nr_pages) / BITS_PER_LONG;
  570         end_off   = (first_page + nr_pages) & (BITS_PER_LONG-1);
  571 
  572         if (curr_idx == end_idx) {
  573                 contiguous_bitmap[curr_idx] &=
  574                         -(1UL<<end_off) | ((1UL<<start_off)-1);
  575         } else {
  576                 contiguous_bitmap[curr_idx] &= (1UL<<start_off)-1;
  577                 while ( ++curr_idx != end_idx )
  578                         contiguous_bitmap[curr_idx] = 0;
  579                 contiguous_bitmap[curr_idx] &= -(1UL<<end_off);
  580         }
  581 }
  582 #endif
  583 
  584 /* Ensure multi-page extents are contiguous in machine memory. */
  585 int 
  586 xen_create_contiguous_region(vm_page_t pages, int npages)
  587 {
  588         unsigned long  mfn, i, flags;
  589         int order;
  590         struct xen_memory_reservation reservation = {
  591                 .nr_extents   = 1,
  592                 .extent_order = 0,
  593                 .domid        = DOMID_SELF
  594         };
  595         set_xen_guest_handle(reservation.extent_start, &mfn);
  596         
  597         balloon_lock(flags);
  598 
  599         /* can currently only handle power of two allocation */
  600         PANIC_IF(ffs(npages) != fls(npages));
  601 
  602         /* 0. determine order */
  603         order = (ffs(npages) == fls(npages)) ? fls(npages) - 1 : fls(npages);
  604         
  605         /* 1. give away machine pages. */
  606         for (i = 0; i < (1 << order); i++) {
  607                 int pfn;
  608                 pfn = VM_PAGE_TO_PHYS(&pages[i]) >> PAGE_SHIFT;
  609                 mfn = PFNTOMFN(pfn);
  610                 PFNTOMFN(pfn) = INVALID_P2M_ENTRY;
  611                 PANIC_IF(HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation) != 1);
  612         }
  613 
  614 
  615         /* 2. Get a new contiguous memory extent. */
  616         reservation.extent_order = order;
  617         /* xenlinux hardcodes this because of aacraid - maybe set to 0 if we're not 
  618          * running with a broxen driver XXXEN
  619          */
  620         reservation.address_bits = 31; 
  621         if (HYPERVISOR_memory_op(XENMEM_increase_reservation, &reservation) != 1)
  622                 goto fail;
  623 
  624         /* 3. Map the new extent in place of old pages. */
  625         for (i = 0; i < (1 << order); i++) {
  626                 int pfn;
  627                 pfn = VM_PAGE_TO_PHYS(&pages[i]) >> PAGE_SHIFT;
  628                 xen_machphys_update(mfn+i, pfn);
  629                 PFNTOMFN(pfn) = mfn+i;
  630         }
  631 
  632         xen_tlb_flush();
  633 
  634 #if 0
  635         contiguous_bitmap_set(VM_PAGE_TO_PHYS(&pages[0]) >> PAGE_SHIFT, 1UL << order);
  636 #endif
  637 
  638         balloon_unlock(flags);
  639 
  640         return 0;
  641 
  642  fail:
  643         reservation.extent_order = 0;
  644         reservation.address_bits = 0;
  645 
  646         for (i = 0; i < (1 << order); i++) {
  647                 int pfn;
  648                 pfn = VM_PAGE_TO_PHYS(&pages[i]) >> PAGE_SHIFT;
  649                 PANIC_IF(HYPERVISOR_memory_op(
  650                         XENMEM_increase_reservation, &reservation) != 1);
  651                 xen_machphys_update(mfn, pfn);
  652                 PFNTOMFN(pfn) = mfn;
  653         }
  654 
  655         xen_tlb_flush();
  656 
  657         balloon_unlock(flags);
  658 
  659         return ENOMEM;
  660 }
  661 
  662 void 
  663 xen_destroy_contiguous_region(void *addr, int npages)
  664 {
  665         unsigned long  mfn, i, flags, order, pfn0;
  666         struct xen_memory_reservation reservation = {
  667                 .nr_extents   = 1,
  668                 .extent_order = 0,
  669                 .domid        = DOMID_SELF
  670         };
  671         set_xen_guest_handle(reservation.extent_start, &mfn);
  672         
  673         pfn0 = vtophys(addr) >> PAGE_SHIFT;
  674 #if 0
  675         scrub_pages(vstart, 1 << order);
  676 #endif
  677         /* can currently only handle power of two allocation */
  678         PANIC_IF(ffs(npages) != fls(npages));
  679 
  680         /* 0. determine order */
  681         order = (ffs(npages) == fls(npages)) ? fls(npages) - 1 : fls(npages);
  682 
  683         balloon_lock(flags);
  684 
  685 #if 0
  686         contiguous_bitmap_clear(vtophys(addr) >> PAGE_SHIFT, 1UL << order);
  687 #endif
  688 
  689         /* 1. Zap current PTEs, giving away the underlying pages. */
  690         for (i = 0; i < (1 << order); i++) {
  691                 int pfn;
  692                 uint64_t new_val = 0;
  693                 pfn = vtomach((char *)addr + i*PAGE_SIZE) >> PAGE_SHIFT;
  694 
  695                 PANIC_IF(HYPERVISOR_update_va_mapping((vm_offset_t)((char *)addr + (i * PAGE_SIZE)), new_val, 0));
  696                 PFNTOMFN(pfn) = INVALID_P2M_ENTRY;
  697                 PANIC_IF(HYPERVISOR_memory_op(
  698                         XENMEM_decrease_reservation, &reservation) != 1);
  699         }
  700 
  701         /* 2. Map new pages in place of old pages. */
  702         for (i = 0; i < (1 << order); i++) {
  703                 int pfn;
  704                 uint64_t new_val;
  705                 pfn = pfn0 + i;
  706                 PANIC_IF(HYPERVISOR_memory_op(XENMEM_increase_reservation, &reservation) != 1);
  707                 
  708                 new_val = mfn << PAGE_SHIFT;
  709                 PANIC_IF(HYPERVISOR_update_va_mapping((vm_offset_t)addr + (i * PAGE_SIZE), 
  710                                                       new_val, PG_KERNEL));
  711                 xen_machphys_update(mfn, pfn);
  712                 PFNTOMFN(pfn) = mfn;
  713         }
  714 
  715         xen_tlb_flush();
  716 
  717         balloon_unlock(flags);
  718 }
  719 
  720 extern  vm_offset_t     proc0kstack;
  721 extern int vm86paddr, vm86phystk;
  722 char *bootmem_start, *bootmem_current, *bootmem_end;
  723 
  724 pteinfo_t *pteinfo_list;
  725 void initvalues(start_info_t *startinfo);
  726 
  727 struct xenstore_domain_interface;
  728 extern struct xenstore_domain_interface *xen_store;
  729 
  730 char *console_page;
  731 
  732 void *
  733 bootmem_alloc(unsigned int size) 
  734 {
  735         char *retptr;
  736         
  737         retptr = bootmem_current;
  738         PANIC_IF(retptr + size > bootmem_end);
  739         bootmem_current += size;
  740 
  741         return retptr;
  742 }
  743 
  744 void 
  745 bootmem_free(void *ptr, unsigned int size) 
  746 {
  747         char *tptr;
  748         
  749         tptr = ptr;
  750         PANIC_IF(tptr != bootmem_current - size ||
  751                 bootmem_current - size < bootmem_start);        
  752 
  753         bootmem_current -= size;
  754 }
  755 
  756 #if 0
  757 static vm_paddr_t
  758 xpmap_mtop2(vm_paddr_t mpa)
  759 {
  760         return ((machine_to_phys_mapping[mpa >> PAGE_SHIFT] << PAGE_SHIFT)
  761             ) | (mpa & ~PG_FRAME);
  762 }
  763 
  764 static pd_entry_t 
  765 xpmap_get_bootpde(vm_paddr_t va)
  766 {
  767 
  768         return ((pd_entry_t *)xen_start_info->pt_base)[va >> 22];
  769 }
  770 
  771 static pd_entry_t
  772 xpmap_get_vbootpde(vm_paddr_t va)
  773 {
  774         pd_entry_t pde;
  775 
  776         pde = xpmap_get_bootpde(va);
  777         if ((pde & PG_V) == 0)
  778                 return (pde & ~PG_FRAME);
  779         return (pde & ~PG_FRAME) |
  780                 (xpmap_mtop2(pde & PG_FRAME) + KERNBASE);
  781 }
  782 
  783 static pt_entry_t 8*
  784 xpmap_get_bootptep(vm_paddr_t va)
  785 {
  786         pd_entry_t pde;
  787 
  788         pde = xpmap_get_vbootpde(va);
  789         if ((pde & PG_V) == 0)
  790                 return (void *)-1;
  791 #define PT_MASK         0x003ff000      /* page table address bits */
  792         return &(((pt_entry_t *)(pde & PG_FRAME))[(va & PT_MASK) >> PAGE_SHIFT]);
  793 }
  794 
  795 static pt_entry_t
  796 xpmap_get_bootpte(vm_paddr_t va)
  797 {
  798 
  799         return xpmap_get_bootptep(va)[0];
  800 }
  801 #endif
  802 
  803 
  804 #ifdef ADD_ISA_HOLE
  805 static void
  806 shift_phys_machine(unsigned long *phys_machine, int nr_pages)
  807 {
  808 
  809         unsigned long *tmp_page, *current_page, *next_page;
  810         int i;
  811 
  812         tmp_page = bootmem_alloc(PAGE_SIZE);
  813         current_page = phys_machine + nr_pages - (PAGE_SIZE/sizeof(unsigned long));  
  814         next_page = current_page - (PAGE_SIZE/sizeof(unsigned long));  
  815         bcopy(phys_machine, tmp_page, PAGE_SIZE);
  816 
  817         while (current_page > phys_machine) { 
  818                 /*  save next page */
  819                 bcopy(next_page, tmp_page, PAGE_SIZE);
  820                 /* shift down page */
  821                 bcopy(current_page, next_page, PAGE_SIZE);
  822                 /*  finish swap */
  823                 bcopy(tmp_page, current_page, PAGE_SIZE);
  824           
  825                 current_page -= (PAGE_SIZE/sizeof(unsigned long));
  826                 next_page -= (PAGE_SIZE/sizeof(unsigned long));
  827         }
  828         bootmem_free(tmp_page, PAGE_SIZE);      
  829         
  830         for (i = 0; i < nr_pages; i++) {
  831                 xen_machphys_update(phys_machine[i], i);
  832         }
  833         memset(phys_machine, INVALID_P2M_ENTRY, PAGE_SIZE);
  834 
  835 }
  836 #endif /* ADD_ISA_HOLE */
  837 
  838 /*
  839  * Build a directory of the pages that make up our Physical to Machine
  840  * mapping table. The Xen suspend/restore code uses this to find our
  841  * mapping table.
  842  */
  843 static void
  844 init_frame_list_list(void *arg)
  845 {
  846         unsigned long nr_pages = xen_start_info->nr_pages;
  847 #define FPP     (PAGE_SIZE/sizeof(xen_pfn_t))
  848         int i, j, k;
  849 
  850         xen_pfn_to_mfn_frame_list_list = malloc(PAGE_SIZE, M_DEVBUF, M_WAITOK);
  851         for (i = 0, j = 0, k = -1; i < nr_pages;
  852              i += FPP, j++) {
  853                 if ((j & (FPP - 1)) == 0) {
  854                         k++;
  855                         xen_pfn_to_mfn_frame_list[k] =
  856                                 malloc(PAGE_SIZE, M_DEVBUF, M_WAITOK);
  857                         xen_pfn_to_mfn_frame_list_list[k] =
  858                                 VTOMFN(xen_pfn_to_mfn_frame_list[k]);
  859                         j = 0;
  860                 }
  861                 xen_pfn_to_mfn_frame_list[k][j] = 
  862                         VTOMFN(&xen_phys_machine[i]);
  863         }
  864 
  865         HYPERVISOR_shared_info->arch.max_pfn = nr_pages;
  866         HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list
  867                 = VTOMFN(xen_pfn_to_mfn_frame_list_list);
  868 }       
  869 SYSINIT(init_fll, SI_SUB_DEVFS, SI_ORDER_ANY, init_frame_list_list, NULL);
  870 
  871 extern unsigned long physfree;
  872 
  873 int pdir, curoffset;
  874 extern int nkpt;
  875 
  876 extern uint32_t kernbase;
  877 
  878 void
  879 initvalues(start_info_t *startinfo)
  880 { 
  881         vm_offset_t cur_space, cur_space_pt;
  882         struct physdev_set_iopl set_iopl;
  883         
  884         int l3_pages, l2_pages, l1_pages, offset;
  885         vm_paddr_t console_page_ma, xen_store_ma;
  886         vm_offset_t tmpva;
  887         vm_paddr_t shinfo;
  888 #ifdef PAE
  889         vm_paddr_t IdlePDPTma, IdlePDPTnewma;
  890         vm_paddr_t IdlePTDnewma[4];
  891         pd_entry_t *IdlePDPTnew, *IdlePTDnew;
  892         vm_paddr_t IdlePTDma[4];
  893 #else
  894         vm_paddr_t IdlePTDma[1];
  895 #endif
  896         unsigned long i;
  897         int ncpus = MAXCPU;
  898 
  899         nkpt = min(
  900                 min(
  901                         max((startinfo->nr_pages >> NPGPTD_SHIFT), nkpt),
  902                     NPGPTD*NPDEPG - KPTDI),
  903                     (HYPERVISOR_VIRT_START - KERNBASE) >> PDRSHIFT);
  904 
  905         HYPERVISOR_vm_assist(VMASST_CMD_enable, VMASST_TYPE_4gb_segments);      
  906 #ifdef notyet
  907         /*
  908          * need to install handler
  909          */
  910         HYPERVISOR_vm_assist(VMASST_CMD_enable, VMASST_TYPE_4gb_segments_notify);       
  911 #endif  
  912         xen_start_info = startinfo;
  913         xen_phys_machine = (xen_pfn_t *)startinfo->mfn_list;
  914 
  915         IdlePTD = (pd_entry_t *)((uint8_t *)startinfo->pt_base + PAGE_SIZE);
  916         l1_pages = 0;
  917         
  918 #ifdef PAE
  919         l3_pages = 1;
  920         l2_pages = 0;
  921         IdlePDPT = (pd_entry_t *)startinfo->pt_base;
  922         IdlePDPTma = VTOM(startinfo->pt_base);
  923         for (i = (KERNBASE >> 30);
  924              (i < 4) && (IdlePDPT[i] != 0); i++)
  925                         l2_pages++;
  926         /*
  927          * Note that only one page directory has been allocated at this point.
  928          * Thus, if KERNBASE
  929          */
  930         for (i = 0; i < l2_pages; i++)
  931                 IdlePTDma[i] = VTOM(IdlePTD + i*PAGE_SIZE);
  932 
  933         l2_pages = (l2_pages == 0) ? 1 : l2_pages;
  934 #else   
  935         l3_pages = 0;
  936         l2_pages = 1;
  937 #endif
  938         for (i = (((KERNBASE>>18) & PAGE_MASK)>>PAGE_SHIFT);
  939              (i<l2_pages*NPDEPG) && (i<(VM_MAX_KERNEL_ADDRESS>>PDRSHIFT)); i++) {
  940                 
  941                 if (IdlePTD[i] == 0)
  942                         break;
  943                 l1_pages++;
  944         }
  945 
  946         /* number of pages allocated after the pts + 1*/;
  947         cur_space = xen_start_info->pt_base +
  948             (l3_pages + l2_pages + l1_pages + 1)*PAGE_SIZE;
  949 
  950         xc_printf("initvalues(): wooh - availmem=%x,%x\n", avail_space,
  951             cur_space);
  952 
  953         xc_printf("KERNBASE=%x,pt_base=%x, VTOPFN(base)=%x, nr_pt_frames=%x\n",
  954             KERNBASE,xen_start_info->pt_base, VTOPFN(xen_start_info->pt_base),
  955             xen_start_info->nr_pt_frames);
  956         xendebug_flags = 0; /* 0xffffffff; */
  957 
  958 #ifdef ADD_ISA_HOLE
  959         shift_phys_machine(xen_phys_machine, xen_start_info->nr_pages);
  960 #endif
  961         XENPRINTF("IdlePTD %p\n", IdlePTD);
  962         XENPRINTF("nr_pages: %ld shared_info: 0x%lx flags: 0x%lx pt_base: 0x%lx "
  963                   "mod_start: 0x%lx mod_len: 0x%lx\n",
  964                   xen_start_info->nr_pages, xen_start_info->shared_info, 
  965                   xen_start_info->flags, xen_start_info->pt_base, 
  966                   xen_start_info->mod_start, xen_start_info->mod_len);
  967 
  968 #ifdef PAE
  969         IdlePDPTnew = (pd_entry_t *)cur_space; cur_space += PAGE_SIZE;
  970         bzero(IdlePDPTnew, PAGE_SIZE);
  971 
  972         IdlePDPTnewma =  VTOM(IdlePDPTnew);
  973         IdlePTDnew = (pd_entry_t *)cur_space; cur_space += 4*PAGE_SIZE;
  974         bzero(IdlePTDnew, 4*PAGE_SIZE);
  975 
  976         for (i = 0; i < 4; i++) 
  977                 IdlePTDnewma[i] = VTOM((uint8_t *)IdlePTDnew + i*PAGE_SIZE);
  978         /*
  979          * L3
  980          *
  981          * Copy the 4 machine addresses of the new PTDs in to the PDPT
  982          * 
  983          */
  984         for (i = 0; i < 4; i++)
  985                 IdlePDPTnew[i] = IdlePTDnewma[i] | PG_V;
  986 
  987         __asm__("nop;");
  988         /*
  989          *
  990          * re-map the new PDPT read-only
  991          */
  992         PT_SET_MA(IdlePDPTnew, IdlePDPTnewma | PG_V);
  993         /*
  994          * 
  995          * Unpin the current PDPT
  996          */
  997         xen_pt_unpin(IdlePDPTma);
  998 
  999 #endif  /* PAE */
 1000 
 1001         /* Map proc0's KSTACK */
 1002         proc0kstack = cur_space; cur_space += (KSTACK_PAGES * PAGE_SIZE);
 1003         xc_printf("proc0kstack=%u\n", proc0kstack);
 1004 
 1005         /* vm86/bios stack */
 1006         cur_space += PAGE_SIZE;
 1007 
 1008         /* Map space for the vm86 region */
 1009         vm86paddr = (vm_offset_t)cur_space;
 1010         cur_space += (PAGE_SIZE * 3);
 1011 
 1012         /* allocate 4 pages for bootmem allocator */
 1013         bootmem_start = bootmem_current = (char *)cur_space;
 1014         cur_space += (4 * PAGE_SIZE);
 1015         bootmem_end = (char *)cur_space;
 1016         
 1017         /* allocate pages for gdt */
 1018         gdt = (union descriptor *)cur_space;
 1019         cur_space += PAGE_SIZE*ncpus;
 1020 
 1021         /* allocate page for ldt */
 1022         ldt = (union descriptor *)cur_space; cur_space += PAGE_SIZE;
 1023         cur_space += PAGE_SIZE;
 1024         
 1025         /* unmap remaining pages from initial chunk
 1026          *
 1027          */
 1028         for (tmpva = cur_space; tmpva < (((uint32_t)&kernbase) + (l1_pages<<PDRSHIFT));
 1029              tmpva += PAGE_SIZE) {
 1030                 bzero((char *)tmpva, PAGE_SIZE);
 1031                 PT_SET_MA(tmpva, (vm_paddr_t)0);
 1032         }
 1033 
 1034         PT_UPDATES_FLUSH();
 1035 
 1036         memcpy(((uint8_t *)IdlePTDnew) + ((unsigned int)(KERNBASE >> 18)),
 1037             ((uint8_t *)IdlePTD) + ((KERNBASE >> 18) & PAGE_MASK),
 1038             l1_pages*sizeof(pt_entry_t));
 1039 
 1040         for (i = 0; i < 4; i++) {
 1041                 PT_SET_MA((uint8_t *)IdlePTDnew + i*PAGE_SIZE,
 1042                     IdlePTDnewma[i] | PG_V);
 1043         }
 1044         xen_load_cr3(VTOP(IdlePDPTnew));
 1045         xen_pgdpt_pin(VTOM(IdlePDPTnew));
 1046 
 1047         /* allocate remainder of nkpt pages */
 1048         cur_space_pt = cur_space;
 1049         for (offset = (KERNBASE >> PDRSHIFT), i = l1_pages; i < nkpt;
 1050              i++, cur_space += PAGE_SIZE) {
 1051                 pdir = (offset + i) / NPDEPG;
 1052                 curoffset = ((offset + i) % NPDEPG);
 1053                 if (((offset + i) << PDRSHIFT) == VM_MAX_KERNEL_ADDRESS)
 1054                         break;
 1055 
 1056                 /*
 1057                  * make sure that all the initial page table pages
 1058                  * have been zeroed
 1059                  */
 1060                 PT_SET_MA(cur_space, VTOM(cur_space) | PG_V | PG_RW);
 1061                 bzero((char *)cur_space, PAGE_SIZE);
 1062                 PT_SET_MA(cur_space, (vm_paddr_t)0);
 1063                 xen_pt_pin(VTOM(cur_space));
 1064                 xen_queue_pt_update((vm_paddr_t)(IdlePTDnewma[pdir] +
 1065                         curoffset*sizeof(vm_paddr_t)), 
 1066                     VTOM(cur_space) | PG_KERNEL);
 1067                 PT_UPDATES_FLUSH();
 1068         }
 1069         
 1070         for (i = 0; i < 4; i++) {
 1071                 pdir = (PTDPTDI + i) / NPDEPG;
 1072                 curoffset = (PTDPTDI + i) % NPDEPG;
 1073 
 1074                 xen_queue_pt_update((vm_paddr_t)(IdlePTDnewma[pdir] +
 1075                         curoffset*sizeof(vm_paddr_t)), 
 1076                     IdlePTDnewma[i] | PG_V);
 1077         }
 1078 
 1079         PT_UPDATES_FLUSH();
 1080         
 1081         IdlePTD = IdlePTDnew;
 1082         IdlePDPT = IdlePDPTnew;
 1083         IdlePDPTma = IdlePDPTnewma;
 1084         
 1085         HYPERVISOR_shared_info = (shared_info_t *)cur_space;
 1086         cur_space += PAGE_SIZE;
 1087 
 1088         xen_store = (struct xenstore_domain_interface *)cur_space;
 1089         cur_space += PAGE_SIZE;
 1090 
 1091         console_page = (char *)cur_space;
 1092         cur_space += PAGE_SIZE;
 1093         
 1094         /*
 1095          * shared_info is an unsigned long so this will randomly break if
 1096          * it is allocated above 4GB - I guess people are used to that
 1097          * sort of thing with Xen ... sigh
 1098          */
 1099         shinfo = xen_start_info->shared_info;
 1100         PT_SET_MA(HYPERVISOR_shared_info, shinfo | PG_KERNEL);
 1101         
 1102         xc_printf("#4\n");
 1103 
 1104         xen_store_ma = (((vm_paddr_t)xen_start_info->store_mfn) << PAGE_SHIFT);
 1105         PT_SET_MA(xen_store, xen_store_ma | PG_KERNEL);
 1106         console_page_ma = (((vm_paddr_t)xen_start_info->console.domU.mfn) << PAGE_SHIFT);
 1107         PT_SET_MA(console_page, console_page_ma | PG_KERNEL);
 1108 
 1109         xc_printf("#5\n");
 1110 
 1111         set_iopl.iopl = 1;
 1112         PANIC_IF(HYPERVISOR_physdev_op(PHYSDEVOP_SET_IOPL, &set_iopl));
 1113         xc_printf("#6\n");
 1114 #if 0
 1115         /* add page table for KERNBASE */
 1116         xen_queue_pt_update(IdlePTDma + KPTDI*sizeof(vm_paddr_t), 
 1117                             VTOM(cur_space) | PG_KERNEL);
 1118         xen_flush_queue();
 1119 #ifdef PAE      
 1120         xen_queue_pt_update(pdir_shadow_ma[3] + KPTDI*sizeof(vm_paddr_t), 
 1121                             VTOM(cur_space) | PG_V | PG_A);
 1122 #else
 1123         xen_queue_pt_update(pdir_shadow_ma + KPTDI*sizeof(vm_paddr_t), 
 1124                             VTOM(cur_space) | PG_V | PG_A);
 1125 #endif  
 1126         xen_flush_queue();
 1127         cur_space += PAGE_SIZE;
 1128         xc_printf("#6\n");
 1129 #endif /* 0 */  
 1130 #ifdef notyet
 1131         if (xen_start_info->flags & SIF_INITDOMAIN) {
 1132                 /* Map first megabyte */
 1133                 for (i = 0; i < (256 << PAGE_SHIFT); i += PAGE_SIZE) 
 1134                         PT_SET_MA(KERNBASE + i, i | PG_KERNEL | PG_NC_PCD);
 1135                 xen_flush_queue();
 1136         }
 1137 #endif
 1138         /*
 1139          * re-map kernel text read-only
 1140          *
 1141          */
 1142         for (i = (((vm_offset_t)&btext) & ~PAGE_MASK);
 1143              i < (((vm_offset_t)&etext) & ~PAGE_MASK); i += PAGE_SIZE)
 1144                 PT_SET_MA(i, VTOM(i) | PG_V | PG_A);
 1145         
 1146         xc_printf("#7\n");
 1147         physfree = VTOP(cur_space);
 1148         init_first = physfree >> PAGE_SHIFT;
 1149         IdlePTD = (pd_entry_t *)VTOP(IdlePTD);
 1150         IdlePDPT = (pd_entry_t *)VTOP(IdlePDPT);
 1151         setup_xen_features();
 1152         xc_printf("#8, proc0kstack=%u\n", proc0kstack);
 1153 }
 1154 
 1155 
 1156 trap_info_t trap_table[] = {
 1157         { 0,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(div)},
 1158         { 1,   0|4, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(dbg)},
 1159         { 3,   3|4, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(bpt)},
 1160         { 4,   3, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(ofl)},
 1161         /* This is UPL on Linux and KPL on BSD */
 1162         { 5,   3, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(bnd)},
 1163         { 6,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(ill)},
 1164         { 7,   0|4, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(dna)},
 1165         /*
 1166          * { 8,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(XXX)},
 1167          *   no handler for double fault
 1168          */
 1169         { 9,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(fpusegm)},
 1170         {10,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(tss)},
 1171         {11,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(missing)},
 1172         {12,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(stk)},
 1173         {13,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(prot)},
 1174         {14,   0|4, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(page)},
 1175         {15,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(rsvd)},
 1176         {16,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(fpu)},
 1177         {17,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(align)},
 1178         {18,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(mchk)},
 1179         {19,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(xmm)},
 1180         {0x80, 3, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(int0x80_syscall)},
 1181         {  0, 0,           0, 0 }
 1182 };
 1183 
 1184 /* Perform a multicall and check that individual calls succeeded. */
 1185 int
 1186 HYPERVISOR_multicall(struct multicall_entry * call_list, int nr_calls)
 1187 {
 1188         int ret = 0;
 1189         int i;
 1190 
 1191         /* Perform the multicall. */
 1192         PANIC_IF(_HYPERVISOR_multicall(call_list, nr_calls));
 1193 
 1194         /* Check the results of individual hypercalls. */
 1195         for (i = 0; i < nr_calls; i++)
 1196                 if (__predict_false(call_list[i].result < 0))
 1197                         ret++;
 1198         if (__predict_false(ret > 0))
 1199                 panic("%d multicall(s) failed: cpu %d\n",
 1200                     ret, smp_processor_id());
 1201 
 1202         /* If we didn't panic already, everything succeeded. */
 1203         return (0);
 1204 }
 1205 
 1206 /********** CODE WORTH KEEPING ABOVE HERE *****************/ 
 1207 
 1208 void xen_failsafe_handler(void);
 1209 
 1210 void
 1211 xen_failsafe_handler(void)
 1212 {
 1213 
 1214         panic("xen_failsafe_handler called!\n");
 1215 }
 1216 
 1217 void xen_handle_thread_switch(struct pcb *pcb);
 1218 
 1219 /* This is called by cpu_switch() when switching threads. */
 1220 /* The pcb arg refers to the process control block of the */
 1221 /* next thread which is to run */
 1222 void
 1223 xen_handle_thread_switch(struct pcb *pcb)
 1224 {
 1225     uint32_t *a = (uint32_t *)&PCPU_GET(fsgs_gdt)[0];
 1226     uint32_t *b = (uint32_t *)&pcb->pcb_fsd;
 1227     multicall_entry_t mcl[3];
 1228     int i = 0;
 1229 
 1230     /* Notify Xen of task switch */
 1231     mcl[i].op = __HYPERVISOR_stack_switch;
 1232     mcl[i].args[0] = GSEL(GDATA_SEL, SEL_KPL);
 1233     mcl[i++].args[1] = (unsigned long)pcb;
 1234 
 1235     /* Check for update of fsd */
 1236     if (*a != *b || *(a+1) != *(b+1)) {
 1237         mcl[i].op = __HYPERVISOR_update_descriptor;
 1238         *(uint64_t *)&mcl[i].args[0] = vtomach((vm_offset_t)a);
 1239         *(uint64_t *)&mcl[i++].args[2] = *(uint64_t *)b;
 1240     }    
 1241 
 1242     a += 2;
 1243     b += 2;
 1244 
 1245     /* Check for update of gsd */
 1246     if (*a != *b || *(a+1) != *(b+1)) {
 1247         mcl[i].op = __HYPERVISOR_update_descriptor;
 1248         *(uint64_t *)&mcl[i].args[0] = vtomach((vm_offset_t)a);
 1249         *(uint64_t *)&mcl[i++].args[2] = *(uint64_t *)b;
 1250     }    
 1251 
 1252     (void)HYPERVISOR_multicall(mcl, i);
 1253 }

Cache object: 43b154f4c1d685b19859d12b7f05f216


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.