The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/i386/xen/xen_machdep.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  *
    3  * Copyright (c) 2004 Christian Limpach.
    4  * Copyright (c) 2004-2006,2008 Kip Macy
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice, this list of conditions and the following disclaimer.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice, this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  * 3. All advertising materials mentioning features or use of this software
   16  *    must display the following acknowledgement:
   17  *      This product includes software developed by Christian Limpach.
   18  * 4. The name of the author may not be used to endorse or promote products
   19  *    derived from this software without specific prior written permission.
   20  *
   21  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   22  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   23  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   24  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   25  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   26  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   27  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   28  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   29  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
   30  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   31  */
   32 
   33 #include <sys/cdefs.h>
   34 __FBSDID("$FreeBSD: releng/8.3/sys/i386/xen/xen_machdep.c 231759 2012-02-15 14:23:01Z ken $");
   35 
   36 #include <sys/param.h>
   37 #include <sys/systm.h>
   38 #include <sys/bus.h>
   39 #include <sys/ktr.h>
   40 #include <sys/lock.h>
   41 #include <sys/mount.h>
   42 #include <sys/malloc.h>
   43 #include <sys/mutex.h>
   44 #include <sys/kernel.h>
   45 #include <sys/proc.h>
   46 #include <sys/reboot.h>
   47 #include <sys/sysproto.h>
   48 
   49 #include <machine/xen/xen-os.h>
   50 
   51 #include <vm/vm.h>
   52 #include <vm/pmap.h>
   53 #include <machine/segments.h>
   54 #include <machine/pcb.h>
   55 #include <machine/stdarg.h>
   56 #include <machine/vmparam.h>
   57 #include <machine/cpu.h>
   58 #include <machine/intr_machdep.h>
   59 #include <machine/md_var.h>
   60 #include <machine/asmacros.h>
   61 
   62 
   63 
   64 #include <xen/hypervisor.h>
   65 #include <machine/xen/xenvar.h>
   66 #include <machine/xen/xenfunc.h>
   67 #include <machine/xen/xenpmap.h>
   68 #include <machine/xen/xenfunc.h>
   69 #include <xen/interface/memory.h>
   70 #include <machine/xen/features.h>
   71 #ifdef SMP
   72 #include <machine/privatespace.h>
   73 #endif
   74 
   75 
   76 #include <vm/vm_page.h>
   77 
   78 
   79 #define IDTVEC(name)    __CONCAT(X,name)
   80 
   81 extern inthand_t
   82 IDTVEC(div), IDTVEC(dbg), IDTVEC(nmi), IDTVEC(bpt), IDTVEC(ofl),
   83         IDTVEC(bnd), IDTVEC(ill), IDTVEC(dna), IDTVEC(fpusegm),
   84         IDTVEC(tss), IDTVEC(missing), IDTVEC(stk), IDTVEC(prot),
   85         IDTVEC(page), IDTVEC(mchk), IDTVEC(rsvd), IDTVEC(fpu), IDTVEC(align),
   86         IDTVEC(xmm), IDTVEC(lcall_syscall), IDTVEC(int0x80_syscall);
   87 
   88 
   89 int xendebug_flags; 
   90 start_info_t *xen_start_info;
   91 shared_info_t *HYPERVISOR_shared_info;
   92 xen_pfn_t *xen_machine_phys = machine_to_phys_mapping;
   93 xen_pfn_t *xen_phys_machine;
   94 xen_pfn_t *xen_pfn_to_mfn_frame_list[16];
   95 xen_pfn_t *xen_pfn_to_mfn_frame_list_list;
   96 int preemptable, init_first;
   97 extern unsigned int avail_space;
   98 
   99 void ni_cli(void);
  100 void ni_sti(void);
  101 
  102 
  103 void
  104 ni_cli(void)
  105 {
  106         CTR0(KTR_SPARE2, "ni_cli disabling interrupts");
  107         __asm__("pushl %edx;"
  108                 "pushl %eax;"
  109                 );
  110         __cli();
  111         __asm__("popl %eax;"
  112                 "popl %edx;"
  113                 );
  114 }
  115 
  116 
  117 void
  118 ni_sti(void)
  119 {
  120         __asm__("pushl %edx;"
  121                 "pushl %esi;"
  122                 "pushl %eax;"
  123                 );
  124         __sti();
  125         __asm__("popl %eax;"
  126                 "popl %esi;"
  127                 "popl %edx;"
  128                 );
  129 }
  130 
  131 /*
  132  * Modify the cmd_line by converting ',' to NULLs so that it is in a  format 
  133  * suitable for the static env vars.
  134  */
  135 char *
  136 xen_setbootenv(char *cmd_line)
  137 {
  138         char *cmd_line_next;
  139     
  140         /* Skip leading spaces */
  141         for (; *cmd_line == ' '; cmd_line++);
  142 
  143         printk("xen_setbootenv(): cmd_line='%s'\n", cmd_line);
  144 
  145         for (cmd_line_next = cmd_line; strsep(&cmd_line_next, ",") != NULL;);
  146         return cmd_line;
  147 }
  148 
  149 static struct 
  150 {
  151         const char      *ev;
  152         int             mask;
  153 } howto_names[] = {
  154         {"boot_askname",        RB_ASKNAME},
  155         {"boot_single", RB_SINGLE},
  156         {"boot_nosync", RB_NOSYNC},
  157         {"boot_halt",   RB_ASKNAME},
  158         {"boot_serial", RB_SERIAL},
  159         {"boot_cdrom",  RB_CDROM},
  160         {"boot_gdb",    RB_GDB},
  161         {"boot_gdb_pause",      RB_RESERVED1},
  162         {"boot_verbose",        RB_VERBOSE},
  163         {"boot_multicons",      RB_MULTIPLE},
  164         {NULL,  0}
  165 };
  166 
  167 int 
  168 xen_boothowto(char *envp)
  169 {
  170         int i, howto = 0;
  171 
  172         /* get equivalents from the environment */
  173         for (i = 0; howto_names[i].ev != NULL; i++)
  174                 if (getenv(howto_names[i].ev) != NULL)
  175                         howto |= howto_names[i].mask;
  176         return howto;
  177 }
  178 
  179 #define PRINTK_BUFSIZE 1024
  180 void
  181 printk(const char *fmt, ...)
  182 {
  183         __va_list ap;
  184         int retval;
  185         static char buf[PRINTK_BUFSIZE];
  186 
  187         va_start(ap, fmt);
  188         retval = vsnprintf(buf, PRINTK_BUFSIZE - 1, fmt, ap);
  189         va_end(ap);
  190         buf[retval] = 0;
  191         (void)HYPERVISOR_console_write(buf, retval);
  192 }
  193 
  194 
  195 #define XPQUEUE_SIZE 128
  196 
  197 struct mmu_log {
  198         char *file;
  199         int line;
  200 };
  201 
  202 #ifdef SMP
  203 /* per-cpu queues and indices */
  204 #ifdef INVARIANTS
  205 static struct mmu_log xpq_queue_log[MAX_VIRT_CPUS][XPQUEUE_SIZE];
  206 #endif
  207 
  208 static int xpq_idx[MAX_VIRT_CPUS];  
  209 static mmu_update_t xpq_queue[MAX_VIRT_CPUS][XPQUEUE_SIZE];
  210 
  211 #define XPQ_QUEUE_LOG xpq_queue_log[vcpu]
  212 #define XPQ_QUEUE xpq_queue[vcpu]
  213 #define XPQ_IDX xpq_idx[vcpu]
  214 #define SET_VCPU() int vcpu = smp_processor_id()
  215 #else
  216         
  217 static mmu_update_t xpq_queue[XPQUEUE_SIZE];
  218 static struct mmu_log xpq_queue_log[XPQUEUE_SIZE];
  219 static int xpq_idx = 0;
  220 
  221 #define XPQ_QUEUE_LOG xpq_queue_log
  222 #define XPQ_QUEUE xpq_queue
  223 #define XPQ_IDX xpq_idx
  224 #define SET_VCPU()
  225 #endif /* !SMP */
  226 
  227 #define XPQ_IDX_INC atomic_add_int(&XPQ_IDX, 1);
  228 
  229 #if 0
  230 static void
  231 xen_dump_queue(void)
  232 {
  233         int _xpq_idx = XPQ_IDX;
  234         int i;
  235 
  236         if (_xpq_idx <= 1)
  237                 return;
  238 
  239         printk("xen_dump_queue(): %u entries\n", _xpq_idx);
  240         for (i = 0; i < _xpq_idx; i++) {
  241                 printk(" val: %llx ptr: %llx\n", XPQ_QUEUE[i].val, XPQ_QUEUE[i].ptr);
  242         }
  243 }
  244 #endif
  245 
  246 
  247 static __inline void
  248 _xen_flush_queue(void)
  249 {
  250         SET_VCPU();
  251         int _xpq_idx = XPQ_IDX;
  252         int error, i;
  253 
  254 #ifdef INVARIANTS
  255         if (__predict_true(gdtset))
  256                 CRITICAL_ASSERT(curthread);
  257 #endif
  258 
  259         XPQ_IDX = 0;
  260         /* Make sure index is cleared first to avoid double updates. */
  261         error = HYPERVISOR_mmu_update((mmu_update_t *)&XPQ_QUEUE,
  262                                       _xpq_idx, NULL, DOMID_SELF);
  263     
  264 #if 0
  265         if (__predict_true(gdtset))
  266         for (i = _xpq_idx; i > 0;) {
  267                 if (i >= 3) {
  268                         CTR6(KTR_PMAP, "mmu:val: %lx ptr: %lx val: %lx "
  269                             "ptr: %lx val: %lx ptr: %lx",
  270                             (XPQ_QUEUE[i-1].val & 0xffffffff),
  271                             (XPQ_QUEUE[i-1].ptr & 0xffffffff),
  272                             (XPQ_QUEUE[i-2].val & 0xffffffff),
  273                             (XPQ_QUEUE[i-2].ptr & 0xffffffff),
  274                             (XPQ_QUEUE[i-3].val & 0xffffffff),
  275                             (XPQ_QUEUE[i-3].ptr & 0xffffffff));
  276                             i -= 3;
  277                 } else if (i == 2) {
  278                         CTR4(KTR_PMAP, "mmu: val: %lx ptr: %lx val: %lx ptr: %lx",
  279                             (XPQ_QUEUE[i-1].val & 0xffffffff),
  280                             (XPQ_QUEUE[i-1].ptr & 0xffffffff),
  281                             (XPQ_QUEUE[i-2].val & 0xffffffff),
  282                             (XPQ_QUEUE[i-2].ptr & 0xffffffff));
  283                         i = 0;
  284                 } else {
  285                         CTR2(KTR_PMAP, "mmu: val: %lx ptr: %lx", 
  286                             (XPQ_QUEUE[i-1].val & 0xffffffff),
  287                             (XPQ_QUEUE[i-1].ptr & 0xffffffff));
  288                         i = 0;
  289                 }
  290         }
  291 #endif  
  292         if (__predict_false(error < 0)) {
  293                 for (i = 0; i < _xpq_idx; i++)
  294                         printf("val: %llx ptr: %llx\n",
  295                             XPQ_QUEUE[i].val, XPQ_QUEUE[i].ptr);
  296                 panic("Failed to execute MMU updates: %d", error);
  297         }
  298 
  299 }
  300 
  301 void
  302 xen_flush_queue(void)
  303 {
  304         SET_VCPU();
  305 
  306         if (__predict_true(gdtset))
  307                 critical_enter();
  308         if (XPQ_IDX != 0) _xen_flush_queue();
  309         if (__predict_true(gdtset))
  310                 critical_exit();
  311 }
  312 
  313 static __inline void
  314 xen_increment_idx(void)
  315 {
  316         SET_VCPU();
  317 
  318         XPQ_IDX++;
  319         if (__predict_false(XPQ_IDX == XPQUEUE_SIZE))
  320                 xen_flush_queue();
  321 }
  322 
  323 void
  324 xen_check_queue(void)
  325 {
  326 #ifdef INVARIANTS
  327         SET_VCPU();
  328         
  329         KASSERT(XPQ_IDX == 0, ("pending operations XPQ_IDX=%d", XPQ_IDX));
  330 #endif
  331 }
  332 
  333 void
  334 xen_invlpg(vm_offset_t va)
  335 {
  336         struct mmuext_op op;
  337         op.cmd = MMUEXT_INVLPG_ALL;
  338         op.arg1.linear_addr = va & ~PAGE_MASK;
  339         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  340 }
  341 
  342 void
  343 xen_load_cr3(u_int val)
  344 {
  345         struct mmuext_op op;
  346 #ifdef INVARIANTS
  347         SET_VCPU();
  348         
  349         KASSERT(XPQ_IDX == 0, ("pending operations XPQ_IDX=%d", XPQ_IDX));
  350 #endif
  351         op.cmd = MMUEXT_NEW_BASEPTR;
  352         op.arg1.mfn = xpmap_ptom(val) >> PAGE_SHIFT;
  353         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  354 }
  355 
  356 #ifdef KTR
  357 static __inline u_int
  358 rebp(void)
  359 {
  360         u_int   data;
  361 
  362         __asm __volatile("movl 4(%%ebp),%0" : "=r" (data));     
  363         return (data);
  364 }
  365 #endif
  366 
  367 u_int
  368 read_eflags(void)
  369 {
  370         vcpu_info_t *_vcpu;
  371         u_int eflags;
  372 
  373         eflags = _read_eflags();
  374         _vcpu = &HYPERVISOR_shared_info->vcpu_info[smp_processor_id()]; 
  375         if (_vcpu->evtchn_upcall_mask)
  376                 eflags &= ~PSL_I;
  377 
  378         return (eflags);
  379 }
  380 
  381 void
  382 write_eflags(u_int eflags)
  383 {
  384         u_int intr;
  385 
  386         CTR2(KTR_SPARE2, "%x xen_restore_flags eflags %x", rebp(), eflags);
  387         intr = ((eflags & PSL_I) == 0);
  388         __restore_flags(intr);
  389         _write_eflags(eflags);
  390 }
  391 
  392 void
  393 xen_cli(void)
  394 {
  395         CTR1(KTR_SPARE2, "%x xen_cli disabling interrupts", rebp());
  396         __cli();
  397 }
  398 
  399 void
  400 xen_sti(void)
  401 {
  402         CTR1(KTR_SPARE2, "%x xen_sti enabling interrupts", rebp());
  403         __sti();
  404 }
  405 
  406 u_int
  407 xen_rcr2(void)
  408 {
  409 
  410         return (HYPERVISOR_shared_info->vcpu_info[curcpu].arch.cr2);
  411 }
  412 
  413 void
  414 _xen_machphys_update(vm_paddr_t mfn, vm_paddr_t pfn, char *file, int line)
  415 {
  416         SET_VCPU();
  417         
  418         if (__predict_true(gdtset))
  419                 critical_enter();
  420         XPQ_QUEUE[XPQ_IDX].ptr = (mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE;
  421         XPQ_QUEUE[XPQ_IDX].val = pfn;
  422 #ifdef INVARIANTS
  423         XPQ_QUEUE_LOG[XPQ_IDX].file = file;
  424         XPQ_QUEUE_LOG[XPQ_IDX].line = line;     
  425 #endif          
  426         xen_increment_idx();
  427         if (__predict_true(gdtset))
  428                 critical_exit();
  429 }
  430 
  431 void
  432 _xen_queue_pt_update(vm_paddr_t ptr, vm_paddr_t val, char *file, int line)
  433 {
  434         SET_VCPU();
  435 
  436         if (__predict_true(gdtset))     
  437                 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  438 
  439         KASSERT((ptr & 7) == 0, ("misaligned update"));
  440         
  441         if (__predict_true(gdtset))
  442                 critical_enter();
  443         
  444         XPQ_QUEUE[XPQ_IDX].ptr = ((uint64_t)ptr) | MMU_NORMAL_PT_UPDATE;
  445         XPQ_QUEUE[XPQ_IDX].val = (uint64_t)val;
  446 #ifdef INVARIANTS
  447         XPQ_QUEUE_LOG[XPQ_IDX].file = file;
  448         XPQ_QUEUE_LOG[XPQ_IDX].line = line;     
  449 #endif  
  450         xen_increment_idx();
  451         if (__predict_true(gdtset))
  452                 critical_exit();
  453 }
  454 
  455 void 
  456 xen_pgdpt_pin(vm_paddr_t ma)
  457 {
  458         struct mmuext_op op;
  459         op.cmd = MMUEXT_PIN_L3_TABLE;
  460         op.arg1.mfn = ma >> PAGE_SHIFT;
  461         xen_flush_queue();
  462         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  463 }
  464 
  465 void 
  466 xen_pgd_pin(vm_paddr_t ma)
  467 {
  468         struct mmuext_op op;
  469         op.cmd = MMUEXT_PIN_L2_TABLE;
  470         op.arg1.mfn = ma >> PAGE_SHIFT;
  471         xen_flush_queue();
  472         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  473 }
  474 
  475 void 
  476 xen_pgd_unpin(vm_paddr_t ma)
  477 {
  478         struct mmuext_op op;
  479         op.cmd = MMUEXT_UNPIN_TABLE;
  480         op.arg1.mfn = ma >> PAGE_SHIFT;
  481         xen_flush_queue();
  482         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  483 }
  484 
  485 void 
  486 xen_pt_pin(vm_paddr_t ma)
  487 {
  488         struct mmuext_op op;
  489         op.cmd = MMUEXT_PIN_L1_TABLE;
  490         op.arg1.mfn = ma >> PAGE_SHIFT;
  491         xen_flush_queue();
  492         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  493 }
  494 
  495 void 
  496 xen_pt_unpin(vm_paddr_t ma)
  497 {
  498         struct mmuext_op op;
  499         op.cmd = MMUEXT_UNPIN_TABLE;
  500         op.arg1.mfn = ma >> PAGE_SHIFT;
  501         xen_flush_queue();
  502         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  503 }
  504 
  505 void 
  506 xen_set_ldt(vm_paddr_t ptr, unsigned long len)
  507 {
  508         struct mmuext_op op;
  509         op.cmd = MMUEXT_SET_LDT;
  510         op.arg1.linear_addr = ptr;
  511         op.arg2.nr_ents = len;
  512         xen_flush_queue();
  513         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  514 }
  515 
  516 void xen_tlb_flush(void)
  517 {
  518         struct mmuext_op op;
  519         op.cmd = MMUEXT_TLB_FLUSH_LOCAL;
  520         xen_flush_queue();
  521         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  522 }
  523 
  524 void
  525 xen_update_descriptor(union descriptor *table, union descriptor *entry)
  526 {
  527         vm_paddr_t pa;
  528         pt_entry_t *ptp;
  529 
  530         ptp = vtopte((vm_offset_t)table);
  531         pa = (*ptp & PG_FRAME) | ((vm_offset_t)table & PAGE_MASK);
  532         if (HYPERVISOR_update_descriptor(pa, *(uint64_t *)entry))
  533                 panic("HYPERVISOR_update_descriptor failed\n");
  534 }
  535 
  536 
  537 #if 0
  538 /*
  539  * Bitmap is indexed by page number. If bit is set, the page is part of a
  540  * xen_create_contiguous_region() area of memory.
  541  */
  542 unsigned long *contiguous_bitmap;
  543 
  544 static void 
  545 contiguous_bitmap_set(unsigned long first_page, unsigned long nr_pages)
  546 {
  547         unsigned long start_off, end_off, curr_idx, end_idx;
  548 
  549         curr_idx  = first_page / BITS_PER_LONG;
  550         start_off = first_page & (BITS_PER_LONG-1);
  551         end_idx   = (first_page + nr_pages) / BITS_PER_LONG;
  552         end_off   = (first_page + nr_pages) & (BITS_PER_LONG-1);
  553 
  554         if (curr_idx == end_idx) {
  555                 contiguous_bitmap[curr_idx] |=
  556                         ((1UL<<end_off)-1) & -(1UL<<start_off);
  557         } else {
  558                 contiguous_bitmap[curr_idx] |= -(1UL<<start_off);
  559                 while ( ++curr_idx < end_idx )
  560                         contiguous_bitmap[curr_idx] = ~0UL;
  561                 contiguous_bitmap[curr_idx] |= (1UL<<end_off)-1;
  562         }
  563 }
  564 
  565 static void 
  566 contiguous_bitmap_clear(unsigned long first_page, unsigned long nr_pages)
  567 {
  568         unsigned long start_off, end_off, curr_idx, end_idx;
  569 
  570         curr_idx  = first_page / BITS_PER_LONG;
  571         start_off = first_page & (BITS_PER_LONG-1);
  572         end_idx   = (first_page + nr_pages) / BITS_PER_LONG;
  573         end_off   = (first_page + nr_pages) & (BITS_PER_LONG-1);
  574 
  575         if (curr_idx == end_idx) {
  576                 contiguous_bitmap[curr_idx] &=
  577                         -(1UL<<end_off) | ((1UL<<start_off)-1);
  578         } else {
  579                 contiguous_bitmap[curr_idx] &= (1UL<<start_off)-1;
  580                 while ( ++curr_idx != end_idx )
  581                         contiguous_bitmap[curr_idx] = 0;
  582                 contiguous_bitmap[curr_idx] &= -(1UL<<end_off);
  583         }
  584 }
  585 #endif
  586 
  587 /* Ensure multi-page extents are contiguous in machine memory. */
  588 int 
  589 xen_create_contiguous_region(vm_page_t pages, int npages)
  590 {
  591         unsigned long  mfn, i, flags;
  592         int order;
  593         struct xen_memory_reservation reservation = {
  594                 .nr_extents   = 1,
  595                 .extent_order = 0,
  596                 .domid        = DOMID_SELF
  597         };
  598         set_xen_guest_handle(reservation.extent_start, &mfn);
  599         
  600         balloon_lock(flags);
  601 
  602         /* can currently only handle power of two allocation */
  603         PANIC_IF(ffs(npages) != fls(npages));
  604 
  605         /* 0. determine order */
  606         order = (ffs(npages) == fls(npages)) ? fls(npages) - 1 : fls(npages);
  607         
  608         /* 1. give away machine pages. */
  609         for (i = 0; i < (1 << order); i++) {
  610                 int pfn;
  611                 pfn = VM_PAGE_TO_PHYS(&pages[i]) >> PAGE_SHIFT;
  612                 mfn = PFNTOMFN(pfn);
  613                 PFNTOMFN(pfn) = INVALID_P2M_ENTRY;
  614                 PANIC_IF(HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation) != 1);
  615         }
  616 
  617 
  618         /* 2. Get a new contiguous memory extent. */
  619         reservation.extent_order = order;
  620         /* xenlinux hardcodes this because of aacraid - maybe set to 0 if we're not 
  621          * running with a broxen driver XXXEN
  622          */
  623         reservation.address_bits = 31; 
  624         if (HYPERVISOR_memory_op(XENMEM_increase_reservation, &reservation) != 1)
  625                 goto fail;
  626 
  627         /* 3. Map the new extent in place of old pages. */
  628         for (i = 0; i < (1 << order); i++) {
  629                 int pfn;
  630                 pfn = VM_PAGE_TO_PHYS(&pages[i]) >> PAGE_SHIFT;
  631                 xen_machphys_update(mfn+i, pfn);
  632                 PFNTOMFN(pfn) = mfn+i;
  633         }
  634 
  635         xen_tlb_flush();
  636 
  637 #if 0
  638         contiguous_bitmap_set(VM_PAGE_TO_PHYS(&pages[0]) >> PAGE_SHIFT, 1UL << order);
  639 #endif
  640 
  641         balloon_unlock(flags);
  642 
  643         return 0;
  644 
  645  fail:
  646         reservation.extent_order = 0;
  647         reservation.address_bits = 0;
  648 
  649         for (i = 0; i < (1 << order); i++) {
  650                 int pfn;
  651                 pfn = VM_PAGE_TO_PHYS(&pages[i]) >> PAGE_SHIFT;
  652                 PANIC_IF(HYPERVISOR_memory_op(
  653                         XENMEM_increase_reservation, &reservation) != 1);
  654                 xen_machphys_update(mfn, pfn);
  655                 PFNTOMFN(pfn) = mfn;
  656         }
  657 
  658         xen_tlb_flush();
  659 
  660         balloon_unlock(flags);
  661 
  662         return ENOMEM;
  663 }
  664 
  665 void 
  666 xen_destroy_contiguous_region(void *addr, int npages)
  667 {
  668         unsigned long  mfn, i, flags, order, pfn0;
  669         struct xen_memory_reservation reservation = {
  670                 .nr_extents   = 1,
  671                 .extent_order = 0,
  672                 .domid        = DOMID_SELF
  673         };
  674         set_xen_guest_handle(reservation.extent_start, &mfn);
  675         
  676         pfn0 = vtophys(addr) >> PAGE_SHIFT;
  677 #if 0
  678         scrub_pages(vstart, 1 << order);
  679 #endif
  680         /* can currently only handle power of two allocation */
  681         PANIC_IF(ffs(npages) != fls(npages));
  682 
  683         /* 0. determine order */
  684         order = (ffs(npages) == fls(npages)) ? fls(npages) - 1 : fls(npages);
  685 
  686         balloon_lock(flags);
  687 
  688 #if 0
  689         contiguous_bitmap_clear(vtophys(addr) >> PAGE_SHIFT, 1UL << order);
  690 #endif
  691 
  692         /* 1. Zap current PTEs, giving away the underlying pages. */
  693         for (i = 0; i < (1 << order); i++) {
  694                 int pfn;
  695                 uint64_t new_val = 0;
  696                 pfn = vtomach((char *)addr + i*PAGE_SIZE) >> PAGE_SHIFT;
  697 
  698                 PANIC_IF(HYPERVISOR_update_va_mapping((vm_offset_t)((char *)addr + (i * PAGE_SIZE)), new_val, 0));
  699                 PFNTOMFN(pfn) = INVALID_P2M_ENTRY;
  700                 PANIC_IF(HYPERVISOR_memory_op(
  701                         XENMEM_decrease_reservation, &reservation) != 1);
  702         }
  703 
  704         /* 2. Map new pages in place of old pages. */
  705         for (i = 0; i < (1 << order); i++) {
  706                 int pfn;
  707                 uint64_t new_val;
  708                 pfn = pfn0 + i;
  709                 PANIC_IF(HYPERVISOR_memory_op(XENMEM_increase_reservation, &reservation) != 1);
  710                 
  711                 new_val = mfn << PAGE_SHIFT;
  712                 PANIC_IF(HYPERVISOR_update_va_mapping((vm_offset_t)addr + (i * PAGE_SIZE), 
  713                                                       new_val, PG_KERNEL));
  714                 xen_machphys_update(mfn, pfn);
  715                 PFNTOMFN(pfn) = mfn;
  716         }
  717 
  718         xen_tlb_flush();
  719 
  720         balloon_unlock(flags);
  721 }
  722 
  723 extern unsigned long cpu0prvpage;
  724 extern unsigned long *SMPpt;
  725 extern  struct user     *proc0uarea;
  726 extern  vm_offset_t     proc0kstack;
  727 extern int vm86paddr, vm86phystk;
  728 char *bootmem_start, *bootmem_current, *bootmem_end;
  729 
  730 pteinfo_t *pteinfo_list;
  731 void initvalues(start_info_t *startinfo);
  732 
  733 struct xenstore_domain_interface;
  734 extern struct xenstore_domain_interface *xen_store;
  735 
  736 char *console_page;
  737 
  738 void *
  739 bootmem_alloc(unsigned int size) 
  740 {
  741         char *retptr;
  742         
  743         retptr = bootmem_current;
  744         PANIC_IF(retptr + size > bootmem_end);
  745         bootmem_current += size;
  746 
  747         return retptr;
  748 }
  749 
  750 void 
  751 bootmem_free(void *ptr, unsigned int size) 
  752 {
  753         char *tptr;
  754         
  755         tptr = ptr;
  756         PANIC_IF(tptr != bootmem_current - size ||
  757                 bootmem_current - size < bootmem_start);        
  758 
  759         bootmem_current -= size;
  760 }
  761 
  762 #if 0
  763 static vm_paddr_t
  764 xpmap_mtop2(vm_paddr_t mpa)
  765 {
  766         return ((machine_to_phys_mapping[mpa >> PAGE_SHIFT] << PAGE_SHIFT)
  767             ) | (mpa & ~PG_FRAME);
  768 }
  769 
  770 static pd_entry_t 
  771 xpmap_get_bootpde(vm_paddr_t va)
  772 {
  773 
  774         return ((pd_entry_t *)xen_start_info->pt_base)[va >> 22];
  775 }
  776 
  777 static pd_entry_t
  778 xpmap_get_vbootpde(vm_paddr_t va)
  779 {
  780         pd_entry_t pde;
  781 
  782         pde = xpmap_get_bootpde(va);
  783         if ((pde & PG_V) == 0)
  784                 return (pde & ~PG_FRAME);
  785         return (pde & ~PG_FRAME) |
  786                 (xpmap_mtop2(pde & PG_FRAME) + KERNBASE);
  787 }
  788 
  789 static pt_entry_t 8*
  790 xpmap_get_bootptep(vm_paddr_t va)
  791 {
  792         pd_entry_t pde;
  793 
  794         pde = xpmap_get_vbootpde(va);
  795         if ((pde & PG_V) == 0)
  796                 return (void *)-1;
  797 #define PT_MASK         0x003ff000      /* page table address bits */
  798         return &(((pt_entry_t *)(pde & PG_FRAME))[(va & PT_MASK) >> PAGE_SHIFT]);
  799 }
  800 
  801 static pt_entry_t
  802 xpmap_get_bootpte(vm_paddr_t va)
  803 {
  804 
  805         return xpmap_get_bootptep(va)[0];
  806 }
  807 #endif
  808 
  809 
  810 #ifdef ADD_ISA_HOLE
  811 static void
  812 shift_phys_machine(unsigned long *phys_machine, int nr_pages)
  813 {
  814 
  815         unsigned long *tmp_page, *current_page, *next_page;
  816         int i;
  817 
  818         tmp_page = bootmem_alloc(PAGE_SIZE);
  819         current_page = phys_machine + nr_pages - (PAGE_SIZE/sizeof(unsigned long));  
  820         next_page = current_page - (PAGE_SIZE/sizeof(unsigned long));  
  821         bcopy(phys_machine, tmp_page, PAGE_SIZE);
  822 
  823         while (current_page > phys_machine) { 
  824                 /*  save next page */
  825                 bcopy(next_page, tmp_page, PAGE_SIZE);
  826                 /* shift down page */
  827                 bcopy(current_page, next_page, PAGE_SIZE);
  828                 /*  finish swap */
  829                 bcopy(tmp_page, current_page, PAGE_SIZE);
  830           
  831                 current_page -= (PAGE_SIZE/sizeof(unsigned long));
  832                 next_page -= (PAGE_SIZE/sizeof(unsigned long));
  833         }
  834         bootmem_free(tmp_page, PAGE_SIZE);      
  835         
  836         for (i = 0; i < nr_pages; i++) {
  837                 xen_machphys_update(phys_machine[i], i);
  838         }
  839         memset(phys_machine, INVALID_P2M_ENTRY, PAGE_SIZE);
  840 
  841 }
  842 #endif /* ADD_ISA_HOLE */
  843 
  844 /*
  845  * Build a directory of the pages that make up our Physical to Machine
  846  * mapping table. The Xen suspend/restore code uses this to find our
  847  * mapping table.
  848  */
  849 static void
  850 init_frame_list_list(void *arg)
  851 {
  852         unsigned long nr_pages = xen_start_info->nr_pages;
  853 #define FPP     (PAGE_SIZE/sizeof(xen_pfn_t))
  854         int i, j, k;
  855 
  856         xen_pfn_to_mfn_frame_list_list = malloc(PAGE_SIZE, M_DEVBUF, M_WAITOK);
  857         for (i = 0, j = 0, k = -1; i < nr_pages;
  858              i += FPP, j++) {
  859                 if ((j & (FPP - 1)) == 0) {
  860                         k++;
  861                         xen_pfn_to_mfn_frame_list[k] =
  862                                 malloc(PAGE_SIZE, M_DEVBUF, M_WAITOK);
  863                         xen_pfn_to_mfn_frame_list_list[k] =
  864                                 VTOMFN(xen_pfn_to_mfn_frame_list[k]);
  865                         j = 0;
  866                 }
  867                 xen_pfn_to_mfn_frame_list[k][j] = 
  868                         VTOMFN(&xen_phys_machine[i]);
  869         }
  870 
  871         HYPERVISOR_shared_info->arch.max_pfn = nr_pages;
  872         HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list
  873                 = VTOMFN(xen_pfn_to_mfn_frame_list_list);
  874 }       
  875 SYSINIT(init_fll, SI_SUB_DEVFS, SI_ORDER_ANY, init_frame_list_list, NULL);
  876 
  877 extern unsigned long physfree;
  878 
  879 int pdir, curoffset;
  880 extern int nkpt;
  881 
  882 void
  883 initvalues(start_info_t *startinfo)
  884 { 
  885         int l3_pages, l2_pages, l1_pages, offset;
  886         vm_offset_t cur_space, cur_space_pt;
  887         struct physdev_set_iopl set_iopl;
  888         
  889         vm_paddr_t KPTphys, IdlePTDma;
  890         vm_paddr_t console_page_ma, xen_store_ma;
  891         vm_offset_t KPTphysoff, tmpva;
  892         vm_paddr_t shinfo;
  893 #ifdef PAE
  894         vm_paddr_t IdlePDPTma, IdlePDPTnewma;
  895         vm_paddr_t IdlePTDnewma[4];
  896         pd_entry_t *IdlePDPTnew, *IdlePTDnew;
  897 #else
  898         vm_paddr_t pdir_shadow_ma;
  899 #endif
  900         unsigned long i;
  901         int ncpus = MAXCPU;
  902 
  903         nkpt = min(
  904                 min(
  905                         max((startinfo->nr_pages >> NPGPTD_SHIFT), nkpt),
  906                     NPGPTD*NPDEPG - KPTDI),
  907                     (HYPERVISOR_VIRT_START - KERNBASE) >> PDRSHIFT);
  908 
  909         HYPERVISOR_vm_assist(VMASST_CMD_enable, VMASST_TYPE_4gb_segments);      
  910 #ifdef notyet
  911         /*
  912          * need to install handler
  913          */
  914         HYPERVISOR_vm_assist(VMASST_CMD_enable, VMASST_TYPE_4gb_segments_notify);       
  915 #endif  
  916         xen_start_info = startinfo;
  917         xen_phys_machine = (xen_pfn_t *)startinfo->mfn_list;
  918 
  919         IdlePTD = (pd_entry_t *)((uint8_t *)startinfo->pt_base + PAGE_SIZE);
  920         l1_pages = 0;
  921         
  922 #ifdef PAE
  923         l3_pages = 1;
  924         l2_pages = 0;
  925         IdlePDPT = (pd_entry_t *)startinfo->pt_base;
  926         IdlePDPTma = xpmap_ptom(VTOP(startinfo->pt_base));
  927         for (i = (KERNBASE >> 30);
  928              (i < 4) && (IdlePDPT[i] != 0); i++)
  929                         l2_pages++;
  930         /*
  931          * Note that only one page directory has been allocated at this point.
  932          * Thus, if KERNBASE
  933          */
  934 #if 0
  935         for (i = 0; i < l2_pages; i++)
  936                 IdlePTDma[i] = xpmap_ptom(VTOP(IdlePTD + i*PAGE_SIZE));
  937 #endif
  938         
  939         l2_pages = (l2_pages == 0) ? 1 : l2_pages;
  940 #else   
  941         l3_pages = 0;
  942         l2_pages = 1;
  943 #endif
  944         for (i = (((KERNBASE>>18) & PAGE_MASK)>>PAGE_SHIFT);
  945              (i<l2_pages*NPDEPG) && (i<(VM_MAX_KERNEL_ADDRESS>>PDRSHIFT)); i++) {
  946                 
  947                 if (IdlePTD[i] == 0)
  948                         break;
  949                 l1_pages++;
  950         }
  951         
  952         /* number of pages allocated after the pts + 1*/;
  953         cur_space = xen_start_info->pt_base +
  954             ((xen_start_info->nr_pt_frames) + 3 )*PAGE_SIZE;
  955         printk("initvalues(): wooh - availmem=%x,%x\n", avail_space, cur_space);
  956 
  957         printk("KERNBASE=%x,pt_base=%x, VTOPFN(base)=%x, nr_pt_frames=%x\n",
  958             KERNBASE,xen_start_info->pt_base, VTOPFN(xen_start_info->pt_base),
  959             xen_start_info->nr_pt_frames);
  960         xendebug_flags = 0; /* 0xffffffff; */
  961 
  962         /* allocate 4 pages for bootmem allocator */
  963         bootmem_start = bootmem_current = (char *)cur_space;
  964         cur_space += (4 * PAGE_SIZE);
  965         bootmem_end = (char *)cur_space;
  966 
  967         /* allocate page for gdt */
  968         gdt = (union descriptor *)cur_space;
  969         cur_space += PAGE_SIZE*ncpus;
  970 
  971         /* allocate page for ldt */
  972         ldt = (union descriptor *)cur_space; cur_space += PAGE_SIZE;
  973         cur_space += PAGE_SIZE;
  974         
  975         HYPERVISOR_shared_info = (shared_info_t *)cur_space;
  976         cur_space += PAGE_SIZE;
  977 
  978         xen_store = (struct xenstore_domain_interface *)cur_space;
  979         cur_space += PAGE_SIZE;
  980 
  981         console_page = (char *)cur_space;
  982         cur_space += PAGE_SIZE;
  983 
  984 #ifdef ADD_ISA_HOLE
  985         shift_phys_machine(xen_phys_machine, xen_start_info->nr_pages);
  986 #endif
  987         /* 
  988          * pre-zero unused mapped pages - mapped on 4MB boundary
  989          */
  990 #ifdef PAE
  991         IdlePDPT = (pd_entry_t *)startinfo->pt_base;
  992         IdlePDPTma = xpmap_ptom(VTOP(startinfo->pt_base));
  993         /*
  994          * Note that only one page directory has been allocated at this point.
  995          * Thus, if KERNBASE
  996          */
  997         IdlePTD = (pd_entry_t *)((uint8_t *)startinfo->pt_base + PAGE_SIZE);
  998         IdlePTDma = xpmap_ptom(VTOP(IdlePTD));
  999         l3_pages = 1;
 1000 #else   
 1001         IdlePTD = (pd_entry_t *)startinfo->pt_base;
 1002         IdlePTDma = xpmap_ptom(VTOP(startinfo->pt_base));
 1003         l3_pages = 0;
 1004 #endif
 1005         l2_pages = 1;
 1006         l1_pages = xen_start_info->nr_pt_frames - l2_pages - l3_pages;
 1007 
 1008         KPTphysoff = (l2_pages + l3_pages)*PAGE_SIZE;
 1009 
 1010         KPTphys = xpmap_ptom(VTOP(startinfo->pt_base + KPTphysoff));
 1011         XENPRINTF("IdlePTD %p\n", IdlePTD);
 1012         XENPRINTF("nr_pages: %ld shared_info: 0x%lx flags: 0x%lx pt_base: 0x%lx "
 1013                   "mod_start: 0x%lx mod_len: 0x%lx\n",
 1014                   xen_start_info->nr_pages, xen_start_info->shared_info, 
 1015                   xen_start_info->flags, xen_start_info->pt_base, 
 1016                   xen_start_info->mod_start, xen_start_info->mod_len);
 1017         /* Map proc0's KSTACK */
 1018 
 1019         proc0kstack = cur_space; cur_space += (KSTACK_PAGES * PAGE_SIZE);
 1020         printk("proc0kstack=%u\n", proc0kstack);
 1021 
 1022         /* vm86/bios stack */
 1023         cur_space += PAGE_SIZE;
 1024 
 1025         /* Map space for the vm86 region */
 1026         vm86paddr = (vm_offset_t)cur_space;
 1027         cur_space += (PAGE_SIZE * 3);
 1028 
 1029 #ifdef PAE
 1030         IdlePDPTnew = (pd_entry_t *)cur_space; cur_space += PAGE_SIZE;
 1031         bzero(IdlePDPTnew, PAGE_SIZE);
 1032 
 1033         IdlePDPTnewma =  xpmap_ptom(VTOP(IdlePDPTnew));
 1034         IdlePTDnew = (pd_entry_t *)cur_space; cur_space += 4*PAGE_SIZE;
 1035         bzero(IdlePTDnew, 4*PAGE_SIZE);
 1036 
 1037         for (i = 0; i < 4; i++) 
 1038                 IdlePTDnewma[i] =
 1039                     xpmap_ptom(VTOP((uint8_t *)IdlePTDnew + i*PAGE_SIZE));
 1040         /*
 1041          * L3
 1042          *
 1043          * Copy the 4 machine addresses of the new PTDs in to the PDPT
 1044          * 
 1045          */
 1046         for (i = 0; i < 4; i++)
 1047                 IdlePDPTnew[i] = IdlePTDnewma[i] | PG_V;
 1048 
 1049         __asm__("nop;");
 1050         /*
 1051          *
 1052          * re-map the new PDPT read-only
 1053          */
 1054         PT_SET_MA(IdlePDPTnew, IdlePDPTnewma | PG_V);
 1055         /*
 1056          * 
 1057          * Unpin the current PDPT
 1058          */
 1059         xen_pt_unpin(IdlePDPTma);
 1060         
 1061         for (i = 0; i < 20; i++) {
 1062                 int startidx = ((KERNBASE >> 18) & PAGE_MASK) >> 3;
 1063 
 1064                 if (IdlePTD[startidx + i] == 0) {
 1065                         l1_pages = i;
 1066                         break;
 1067                 }       
 1068         }
 1069 
 1070 #endif  /* PAE */
 1071         
 1072         /* unmap remaining pages from initial 4MB chunk
 1073          *
 1074          */
 1075         for (tmpva = cur_space; (tmpva & ((1<<22)-1)) != 0; tmpva += PAGE_SIZE) {
 1076                 bzero((char *)tmpva, PAGE_SIZE);
 1077                 PT_SET_MA(tmpva, (vm_paddr_t)0);
 1078         }
 1079         
 1080         PT_UPDATES_FLUSH();
 1081 
 1082         memcpy(((uint8_t *)IdlePTDnew) + ((unsigned int)(KERNBASE >> 18)),
 1083             ((uint8_t *)IdlePTD) + ((KERNBASE >> 18) & PAGE_MASK),
 1084             l1_pages*sizeof(pt_entry_t));
 1085 
 1086         for (i = 0; i < 4; i++) {
 1087                 PT_SET_MA((uint8_t *)IdlePTDnew + i*PAGE_SIZE,
 1088                     IdlePTDnewma[i] | PG_V);
 1089         }
 1090         xen_load_cr3(VTOP(IdlePDPTnew));
 1091         xen_pgdpt_pin(xpmap_ptom(VTOP(IdlePDPTnew)));
 1092 
 1093         /* allocate remainder of nkpt pages */
 1094         cur_space_pt = cur_space;
 1095         for (offset = (KERNBASE >> PDRSHIFT), i = l1_pages; i < nkpt;
 1096              i++, cur_space += PAGE_SIZE) {
 1097                 pdir = (offset + i) / NPDEPG;
 1098                 curoffset = ((offset + i) % NPDEPG);
 1099                 if (((offset + i) << PDRSHIFT) == VM_MAX_KERNEL_ADDRESS)
 1100                         break;
 1101 
 1102                 /*
 1103                  * make sure that all the initial page table pages
 1104                  * have been zeroed
 1105                  */
 1106                 PT_SET_MA(cur_space_pt,
 1107                     xpmap_ptom(VTOP(cur_space)) | PG_V | PG_RW);
 1108                 bzero((char *)cur_space_pt, PAGE_SIZE);
 1109                 PT_SET_MA(cur_space_pt, (vm_paddr_t)0);
 1110                 xen_pt_pin(xpmap_ptom(VTOP(cur_space)));
 1111                 xen_queue_pt_update((vm_paddr_t)(IdlePTDnewma[pdir] +
 1112                         curoffset*sizeof(vm_paddr_t)), 
 1113                     xpmap_ptom(VTOP(cur_space)) | PG_KERNEL);
 1114                 PT_UPDATES_FLUSH();
 1115         }
 1116         
 1117         for (i = 0; i < 4; i++) {
 1118                 pdir = (PTDPTDI + i) / NPDEPG;
 1119                 curoffset = (PTDPTDI + i) % NPDEPG;
 1120 
 1121                 xen_queue_pt_update((vm_paddr_t)(IdlePTDnewma[pdir] +
 1122                         curoffset*sizeof(vm_paddr_t)), 
 1123                     IdlePTDnewma[i] | PG_V);
 1124         }
 1125 
 1126         PT_UPDATES_FLUSH();
 1127         
 1128         IdlePTD = IdlePTDnew;
 1129         IdlePDPT = IdlePDPTnew;
 1130         IdlePDPTma = IdlePDPTnewma;
 1131         
 1132         /*
 1133          * shared_info is an unsigned long so this will randomly break if
 1134          * it is allocated above 4GB - I guess people are used to that
 1135          * sort of thing with Xen ... sigh
 1136          */
 1137         shinfo = xen_start_info->shared_info;
 1138         PT_SET_MA(HYPERVISOR_shared_info, shinfo | PG_KERNEL);
 1139         
 1140         printk("#4\n");
 1141 
 1142         xen_store_ma = (((vm_paddr_t)xen_start_info->store_mfn) << PAGE_SHIFT);
 1143         PT_SET_MA(xen_store, xen_store_ma | PG_KERNEL);
 1144         console_page_ma = (((vm_paddr_t)xen_start_info->console.domU.mfn) << PAGE_SHIFT);
 1145         PT_SET_MA(console_page, console_page_ma | PG_KERNEL);
 1146 
 1147         printk("#5\n");
 1148 
 1149         set_iopl.iopl = 1;
 1150         PANIC_IF(HYPERVISOR_physdev_op(PHYSDEVOP_SET_IOPL, &set_iopl));
 1151         printk("#6\n");
 1152 #if 0
 1153         /* add page table for KERNBASE */
 1154         xen_queue_pt_update(IdlePTDma + KPTDI*sizeof(vm_paddr_t), 
 1155                             xpmap_ptom(VTOP(cur_space) | PG_KERNEL));
 1156         xen_flush_queue();
 1157 #ifdef PAE      
 1158         xen_queue_pt_update(pdir_shadow_ma[3] + KPTDI*sizeof(vm_paddr_t), 
 1159                             xpmap_ptom(VTOP(cur_space) | PG_V | PG_A));
 1160 #else
 1161         xen_queue_pt_update(pdir_shadow_ma + KPTDI*sizeof(vm_paddr_t), 
 1162                             xpmap_ptom(VTOP(cur_space) | PG_V | PG_A));
 1163 #endif  
 1164         xen_flush_queue();
 1165         cur_space += PAGE_SIZE;
 1166         printk("#6\n");
 1167 #endif /* 0 */  
 1168 #ifdef notyet
 1169         if (xen_start_info->flags & SIF_INITDOMAIN) {
 1170                 /* Map first megabyte */
 1171                 for (i = 0; i < (256 << PAGE_SHIFT); i += PAGE_SIZE) 
 1172                         PT_SET_MA(KERNBASE + i, i | PG_KERNEL | PG_NC_PCD);
 1173                 xen_flush_queue();
 1174         }
 1175 #endif
 1176         /*
 1177          * re-map kernel text read-only
 1178          *
 1179          */
 1180         for (i = (((vm_offset_t)&btext) & ~PAGE_MASK);
 1181              i < (((vm_offset_t)&etext) & ~PAGE_MASK); i += PAGE_SIZE)
 1182                 PT_SET_MA(i, xpmap_ptom(VTOP(i)) | PG_V | PG_A);
 1183         
 1184         printk("#7\n");
 1185         physfree = VTOP(cur_space);
 1186         init_first = physfree >> PAGE_SHIFT;
 1187         IdlePTD = (pd_entry_t *)VTOP(IdlePTD);
 1188         IdlePDPT = (pd_entry_t *)VTOP(IdlePDPT);
 1189         setup_xen_features();
 1190         printk("#8, proc0kstack=%u\n", proc0kstack);
 1191 }
 1192 
 1193 
 1194 trap_info_t trap_table[] = {
 1195         { 0,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(div)},
 1196         { 1,   0|4, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(dbg)},
 1197         { 3,   3|4, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(bpt)},
 1198         { 4,   3, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(ofl)},
 1199         /* This is UPL on Linux and KPL on BSD */
 1200         { 5,   3, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(bnd)},
 1201         { 6,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(ill)},
 1202         { 7,   0|4, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(dna)},
 1203         /*
 1204          * { 8,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(XXX)},
 1205          *   no handler for double fault
 1206          */
 1207         { 9,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(fpusegm)},
 1208         {10,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(tss)},
 1209         {11,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(missing)},
 1210         {12,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(stk)},
 1211         {13,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(prot)},
 1212         {14,   0|4, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(page)},
 1213         {15,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(rsvd)},
 1214         {16,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(fpu)},
 1215         {17,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(align)},
 1216         {18,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(mchk)},
 1217         {19,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(xmm)},
 1218         {0x80, 3, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(int0x80_syscall)},
 1219         {  0, 0,           0, 0 }
 1220 };
 1221 
 1222 /* Perform a multicall and check that individual calls succeeded. */
 1223 int
 1224 HYPERVISOR_multicall(struct multicall_entry * call_list, int nr_calls)
 1225 {
 1226         int ret = 0;
 1227         int i;
 1228 
 1229         /* Perform the multicall. */
 1230         PANIC_IF(_HYPERVISOR_multicall(call_list, nr_calls));
 1231 
 1232         /* Check the results of individual hypercalls. */
 1233         for (i = 0; i < nr_calls; i++)
 1234                 if (unlikely(call_list[i].result < 0))
 1235                         ret++;
 1236         if (unlikely(ret > 0))
 1237                 panic("%d multicall(s) failed: cpu %d\n",
 1238                     ret, smp_processor_id());
 1239 
 1240         /* If we didn't panic already, everything succeeded. */
 1241         return (0);
 1242 }
 1243 
 1244 /********** CODE WORTH KEEPING ABOVE HERE *****************/ 
 1245 
 1246 void xen_failsafe_handler(void);
 1247 
 1248 void
 1249 xen_failsafe_handler(void)
 1250 {
 1251 
 1252         panic("xen_failsafe_handler called!\n");
 1253 }
 1254 
 1255 void xen_handle_thread_switch(struct pcb *pcb);
 1256 
 1257 /* This is called by cpu_switch() when switching threads. */
 1258 /* The pcb arg refers to the process control block of the */
 1259 /* next thread which is to run */
 1260 void
 1261 xen_handle_thread_switch(struct pcb *pcb)
 1262 {
 1263     uint32_t *a = (uint32_t *)&PCPU_GET(fsgs_gdt)[0];
 1264     uint32_t *b = (uint32_t *)&pcb->pcb_fsd;
 1265     multicall_entry_t mcl[3];
 1266     int i = 0;
 1267 
 1268     /* Notify Xen of task switch */
 1269     mcl[i].op = __HYPERVISOR_stack_switch;
 1270     mcl[i].args[0] = GSEL(GDATA_SEL, SEL_KPL);
 1271     mcl[i++].args[1] = (unsigned long)pcb;
 1272 
 1273     /* Check for update of fsd */
 1274     if (*a != *b || *(a+1) != *(b+1)) {
 1275         mcl[i].op = __HYPERVISOR_update_descriptor;
 1276         *(uint64_t *)&mcl[i].args[0] = vtomach((vm_offset_t)a);
 1277         *(uint64_t *)&mcl[i++].args[2] = *(uint64_t *)b;
 1278     }    
 1279 
 1280     a += 2;
 1281     b += 2;
 1282 
 1283     /* Check for update of gsd */
 1284     if (*a != *b || *(a+1) != *(b+1)) {
 1285         mcl[i].op = __HYPERVISOR_update_descriptor;
 1286         *(uint64_t *)&mcl[i].args[0] = vtomach((vm_offset_t)a);
 1287         *(uint64_t *)&mcl[i++].args[2] = *(uint64_t *)b;
 1288     }    
 1289 
 1290     (void)HYPERVISOR_multicall(mcl, i);
 1291 }

Cache object: eb5a0eb40752086ceeb297f80b09a5bb


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.