FreeBSD/Linux Kernel Cross Reference
sys/osfmk/ppc/pmap.c

     /*
      * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
      *
      * @APPLE_LICENSE_HEADER_START@
      * 
      * The contents of this file constitute Original Code as defined in and
      * are subject to the Apple Public Source License Version 1.1 (the
      * "License").  You may not use this file except in compliance with the
      * License.  Please obtain a copy of the License at
     * http://www.apple.com/publicsource and read it before using this file.
     * 
     * This Original Code and all software distributed under the License are
     * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
     * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
     * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
     * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
     * License for the specific language governing rights and limitations
     * under the License.
     * 
     * @APPLE_LICENSE_HEADER_END@
     */
    /*
     * @OSF_COPYRIGHT@
     */
    /*
     * Mach Operating System
     * Copyright (c) 1990,1991,1992 The University of Utah and
     * the Center for Software Science (CSS).
     * Copyright (c) 1991,1987 Carnegie Mellon University.
     * All rights reserved.
     *
     * Permission to use, copy, modify and distribute this software and its
     * documentation is hereby granted, provided that both the copyright
     * notice and this permission notice appear in all copies of the
     * software, derivative works or modified versions, and any portions
     * thereof, and that both notices appear in supporting documentation,
     * and that all advertising materials mentioning features or use of
     * this software display the following acknowledgement: ``This product
     * includes software developed by the Center for Software Science at
     * the University of Utah.''
     *
     * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSS ALLOW FREE USE OF
     * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
     * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
     * THIS SOFTWARE.
     *
     * CSS requests users of this software to return to css-dist@cs.utah.edu any
     * improvements that they make and grant CSS redistribution rights.
     *
     * Carnegie Mellon requests users of this software to return to
     *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
     *  School of Computer Science
     *  Carnegie Mellon University
     *  Pittsburgh PA 15213-3890
     * any improvements or extensions that they make and grant Carnegie Mellon
     * the rights to redistribute these changes.
     *
     *      Utah $Hdr: pmap.c 1.28 92/06/23$
     *      Author: Mike Hibler, Bob Wheeler, University of Utah CSS, 10/90
     */
     
    /*
     *      Manages physical address maps for powerpc.
     *
     *      In addition to hardware address maps, this
     *      module is called upon to provide software-use-only
     *      maps which may or may not be stored in the same
     *      form as hardware maps.  These pseudo-maps are
     *      used to store intermediate results from copy
     *      operations to and from address spaces.
     *
     *      Since the information managed by this module is
     *      also stored by the logical address mapping module,
     *      this module may throw away valid virtual-to-physical
     *      mappings at almost any time.  However, invalidations
     *      of virtual-to-physical mappings must be done as
     *      requested.
     *
     *      In order to cope with hardware architectures which
     *      make virtual-to-physical map invalidates expensive,
     *      this module may delay invalidate or reduced protection
     *      operations until such time as they are actually
     *      necessary.  This module is given full information to
     *      when physical maps must be made correct.
     *      
     */
    
    #include <zone_debug.h>
    #include <debug.h>
    #include <mach_kgdb.h>
    #include <mach_vm_debug.h>
    #include <db_machine_commands.h>
    
    #include <kern/thread.h>
    #include <kern/simple_lock.h>
    #include <mach/vm_attributes.h>
    #include <mach/vm_param.h>
    #include <vm/vm_kern.h>
    #include <kern/spl.h>
   
   #include <kern/misc_protos.h>
   #include <ppc/misc_protos.h>
   #include <ppc/proc_reg.h>
   
   #include <vm/pmap.h>
   #include <vm/vm_map.h>
   #include <vm/vm_page.h>
   
   #include <ppc/pmap.h>
   #include <ppc/mem.h>
   #include <ppc/mappings.h>
   
   #include <ppc/new_screen.h>
   #include <ppc/Firmware.h>
   #include <ppc/savearea.h>
   #include <ppc/cpu_internal.h>
   #include <ppc/exception.h>
   #include <ppc/low_trace.h>
   #include <ppc/lowglobals.h>
   #include <ddb/db_output.h>
   #include <machine/cpu_capabilities.h>
   
   #include <vm/vm_protos.h> /* must be last */
   
   
   extern unsigned int     avail_remaining;
   unsigned int    debugbackpocket;                                                        /* (TEST/DEBUG) */
   
   vm_offset_t             first_free_virt;
   int             current_free_region;                                            /* Used in pmap_next_page */
   
   pmapTransTab *pmapTrans;                                                                        /* Point to the hash to pmap translations */
   struct phys_entry *phys_table;
   
   /* forward */
   static void pmap_map_physical(void);
   static void pmap_map_iohole(addr64_t paddr, addr64_t size);
   void pmap_activate(pmap_t pmap, thread_t th, int which_cpu);
   void pmap_deactivate(pmap_t pmap, thread_t th, int which_cpu);
   
   extern void hw_hash_init(void);
   
   /*  NOTE:  kernel_pmap_store must be in V=R storage and aligned!!!!!!!!!!!!!! */
   
   extern struct pmap      kernel_pmap_store;
   pmap_t          kernel_pmap;                    /* Pointer to kernel pmap and anchor for in-use pmaps */                
   addr64_t        kernel_pmap_phys;               /* Pointer to kernel pmap and anchor for in-use pmaps, physical address */              
   pmap_t          cursor_pmap;                    /* Pointer to last pmap allocated or previous if removed from in-use list */
   pmap_t          sharedPmap;                             /* Pointer to common pmap for 64-bit address spaces */
   struct zone     *pmap_zone;                             /* zone of pmap structures */
   boolean_t       pmap_initialized = FALSE;
   
   int ppc_max_pmaps;                                      /* Maximum number of concurrent address spaces allowed. This is machine dependent */    
   addr64_t vm_max_address;                        /* Maximum effective address supported */
   addr64_t vm_max_physical;                       /* Maximum physical address supported */
   
   /*
    * Physical-to-virtual translations are handled by inverted page table
    * structures, phys_tables.  Multiple mappings of a single page are handled
    * by linking the affected mapping structures. We initialise one region
    * for phys_tables of the physical memory we know about, but more may be
    * added as it is discovered (eg. by drivers).
    */
   
   /*
    *      free pmap list. caches the first free_pmap_max pmaps that are freed up
    */
   int             free_pmap_max = 32;
   int             free_pmap_count;
   pmap_t  free_pmap_list;
   decl_simple_lock_data(,free_pmap_lock)
   
   /*
    * Function to get index into phys_table for a given physical address
    */
   
   struct phys_entry *pmap_find_physentry(ppnum_t pa)
   {
           int i;
           unsigned int entry;
   
           for (i = pmap_mem_regions_count - 1; i >= 0; i--) {
                   if (pa < pmap_mem_regions[i].mrStart) continue; /* See if we fit in this region */
                   if (pa > pmap_mem_regions[i].mrEnd) continue;   /* Check the end too */
                   
                   entry = (unsigned int)pmap_mem_regions[i].mrPhysTab + ((pa - pmap_mem_regions[i].mrStart) * sizeof(phys_entry_t));
                   return (struct phys_entry *)entry;
           }
   //      kprintf("DEBUG - pmap_find_physentry: page 0x%08X not found\n", pa);
           return 0;
   }
   
   /*
    * kern_return_t
    * pmap_add_physical_memory(vm_offset_t spa, vm_offset_t epa,
    *                          boolean_t available, unsigned int attr)
    *
    *      THIS IS NOT SUPPORTED
    */
   kern_return_t
   pmap_add_physical_memory(
           __unused vm_offset_t spa, 
           __unused vm_offset_t epa,
           __unused boolean_t available,
           __unused unsigned int attr)
   {
           
           panic("Forget it! You can't map no more memory, you greedy puke!\n");
           return KERN_SUCCESS;
   }
   
   /*
    * pmap_map(va, spa, epa, prot)
    *      is called during boot to map memory in the kernel's address map.
    *      A virtual address range starting at "va" is mapped to the physical
    *      address range "spa" to "epa" with machine independent protection
    *      "prot".
    *
    *      "va", "spa", and "epa" are byte addresses and must be on machine
    *      independent page boundaries.
    *
    *      Pages with a contiguous virtual address range, the same protection, and attributes.
    *      therefore, we map it with a single block.
    *
    *      Note that this call will only map into 32-bit space
    *
    */
   
   vm_offset_t
   pmap_map(
           vm_offset_t va,
           vm_offset_t spa,
           vm_offset_t epa,
           vm_prot_t prot)
   {
   
           addr64_t colladr;
           
           if (spa == epa) return(va);
   
           assert(epa > spa);
   
           colladr = mapping_make(kernel_pmap, (addr64_t)va, (ppnum_t)(spa >> 12), (mmFlgBlock | mmFlgPerm), (epa - spa) >> 12, prot & VM_PROT_ALL);
   
           if(colladr) {                                                                                   /* Was something already mapped in the range? */
                   panic("pmap_map: attempt to map previously mapped range - va = %08X, pa = %08X, epa = %08X, collision = %016llX\n",
                           va, spa, epa, colladr);
           }                               
           return(va);
   }
   
   /*
    * pmap_map_physical()
    *      Maps physical memory into the kernel's address map beginning at lgPMWvaddr, the
    *  physical memory window.
    *
    */
   void
   pmap_map_physical()
   {
           unsigned region;
           uint64_t msize, size;
           addr64_t paddr, vaddr, colladdr;
   
           /* Iterate over physical memory regions, block mapping each into the kernel's address map */    
           for (region = 0; region < (unsigned)pmap_mem_regions_count; region++) {
                   paddr = ((addr64_t)pmap_mem_regions[region].mrStart << 12);     /* Get starting physical address */
                   size  = (((addr64_t)pmap_mem_regions[region].mrEnd + 1) << 12) - paddr;
   
                   vaddr = paddr + lowGlo.lgPMWvaddr;                                      /* Get starting virtual address */
   
                   while (size > 0) {
                           
                           msize = ((size > 0x0000020000000000ULL) ? 0x0000020000000000ULL : size);        /* Get size, but no more than 2TBs */
                           
                           colladdr = mapping_make(kernel_pmap, vaddr, (paddr >> 12),
                                   (mmFlgBlock | mmFlgPerm), (msize >> 12),
                                   (VM_PROT_READ | VM_PROT_WRITE));
                           if (colladdr) {
                                   panic ("pmap_map_physical: mapping failure - va = %016llX, pa = %08X, size = %08X, collision = %016llX\n",
                                              vaddr, (paddr >> 12), (msize >> 12), colladdr);
                           }
   
                           vaddr = vaddr + (uint64_t)msize;                                /* Point to the next virtual addr */
                           paddr = paddr + (uint64_t)msize;                                /* Point to the next physical addr */
                           size  -= msize;
                   }
           }
   }
   
   /*
    * pmap_map_iohole(addr64_t paddr, addr64_t size)
    *      Maps an I/O hole into the kernel's address map at its proper offset in
    *      the physical memory window.
    *
    */
   void
   pmap_map_iohole(addr64_t paddr, addr64_t size)
   {
   
           addr64_t vaddr, colladdr, msize;
           uint32_t psize;
   
           vaddr = paddr + lowGlo.lgPMWvaddr;                                              /* Get starting virtual address */              
   
           while (size > 0) {
   
                   msize = ((size > 0x0000020000000000ULL) ? 0x0000020000000000ULL : size);        /* Get size, but no more than 2TBs */
                   
                   colladdr = mapping_make(kernel_pmap, vaddr, (paddr >> 12),
                           (mmFlgBlock | mmFlgPerm | mmFlgGuarded | mmFlgCInhib), (msize >> 12),
                           (VM_PROT_READ | VM_PROT_WRITE));
                   if (colladdr) {
                           panic ("pmap_map_iohole: mapping failed - va = %016llX, pa = %08X, size = %08X, collision = %016llX\n",
                                      vaddr, (paddr >> 12), (msize >> 12), colladdr);
                   }
   
                   vaddr = vaddr + (uint64_t)msize;                                        /* Point to the next virtual addr */
                   paddr = paddr + (uint64_t)msize;                                        /* Point to the next physical addr */
                   size  -= msize;
           }       
   }
   
   /*
    *      Bootstrap the system enough to run with virtual memory.
    *      Map the kernel's code and data, and allocate the system page table.
    *      Called with mapping done by BATs. Page_size must already be set.
    *
    *      Parameters:
    *      msize:  Total memory present
    *      first_avail:    First virtual address available
    *      kmapsize:       Size of kernel text and data
    */
   void
   pmap_bootstrap(uint64_t msize, vm_offset_t *first_avail, unsigned int kmapsize)
   {
           vm_offset_t     addr;
           vm_size_t               size;
           unsigned int    i, num, mapsize, vmpagesz, vmmapsz, nbits;
           signed                  bank;
           uint64_t                tmemsize;
           uint_t                  htslop;
           vm_offset_t             first_used_addr, PCAsize;
           struct phys_entry *phys_entry;
   
           *first_avail = round_page(*first_avail);                                /* Make sure we start out on a page boundary */
           vm_last_addr = VM_MAX_KERNEL_ADDRESS;                                   /* Set the highest address know to VM */
   
           /*
            * Initialize kernel pmap
            */
           kernel_pmap = &kernel_pmap_store;
           kernel_pmap_phys = (addr64_t)&kernel_pmap_store;
           cursor_pmap = &kernel_pmap_store;
   
           kernel_pmap->pmap_link.next = (queue_t)kernel_pmap;             /* Set up anchor forward */
           kernel_pmap->pmap_link.prev = (queue_t)kernel_pmap;             /* Set up anchor reverse */
           kernel_pmap->ref_count = 1;
           kernel_pmap->pmapFlags = pmapKeyDef;                                    /* Set the default keys */
           kernel_pmap->pmapCCtl = pmapCCtlVal;                                    /* Initialize cache control */
           kernel_pmap->space = PPC_SID_KERNEL;
           kernel_pmap->pmapvr = 0;                                                                /* Virtual = Real  */
   
   /*
    *  IBM's recommended hash table size is one PTEG for every 2 physical pages.
    *  However, we have found that OSX rarely uses more than 4 PTEs in a PTEG
    *  with this size table.  Therefore, by default we allocate a hash table
    *  one half IBM's recommended size, ie one PTEG per 4 pages.  The "ht_shift" boot-arg
    *  can be used to override the default hash table size.
    *      We will allocate the hash table in physical RAM, outside of kernel virtual memory,
    *      at the top of the highest bank that will contain it.
    *      Note that "bank" doesn't refer to a physical memory slot here, it is a range of
    *      physically contiguous memory.
    *
    *      The PCA will go there as well, immediately before the hash table.
    */
    
           nbits = cntlzw(((msize << 1) - 1) >> 32);                               /* Get first bit in upper half */
           if (nbits == 32)                                        /* If upper half was empty, find bit in bottom half */
           nbits = nbits + cntlzw((uint_t)((msize << 1) - 1));
           tmemsize = 0x8000000000000000ULL >> nbits;              /* Get memory size rounded up to power of 2 */
           
       /* Calculate hash table size:  First, make sure we don't overflow 32-bit arithmetic. */
           if (tmemsize > 0x0000002000000000ULL)
           tmemsize = 0x0000002000000000ULL;
   
       /* Second, calculate IBM recommended hash table size, ie one PTEG per 2 physical pages */
           hash_table_size = (uint_t)(tmemsize >> 13) * PerProcTable[0].ppe_vaddr->pf.pfPTEG;
       
       /* Third, cut this in half to produce the OSX default, ie one PTEG per 4 physical pages */
       hash_table_size >>= 1;
       
       /* Fourth, adjust default size per "ht_shift" boot arg */
       if (hash_table_shift >= 0)                              /* if positive, make size bigger */
           hash_table_size <<= hash_table_shift;
       else                                                    /* if "ht_shift" is negative, make smaller */
           hash_table_size >>= (-hash_table_shift);
       
       /* Fifth, make sure we are at least minimum size */
           if (hash_table_size < (256 * 1024))
           hash_table_size = (256 * 1024);
   
           while(1) {                                                                                              /* Try to fit hash table in PCA into contiguous memory */
   
                   if(hash_table_size < (256 * 1024)) {                            /* Have we dropped too short? This should never, ever happen */
                           panic("pmap_bootstrap: Can't find space for hash table\n");     /* This will never print, system isn't up far enough... */
                   }
   
                   PCAsize = (hash_table_size / PerProcTable[0].ppe_vaddr->pf.pfPTEG) * sizeof(PCA_t);     /* Get total size of PCA table */
                   PCAsize = round_page(PCAsize);                                  /* Make sure it is at least a page long */
           
                   for(bank = pmap_mem_regions_count - 1; bank >= 0; bank--) {     /* Search backwards through banks */
                           
                           hash_table_base = ((addr64_t)pmap_mem_regions[bank].mrEnd << 12) - hash_table_size + PAGE_SIZE; /* Get tenative address */
                           
                           htslop = hash_table_base & (hash_table_size - 1);       /* Get the extra that we will round down when we align */
                           hash_table_base = hash_table_base & -(addr64_t)hash_table_size; /* Round down to correct boundary */
                           
                           if((hash_table_base - round_page(PCAsize)) >= ((addr64_t)pmap_mem_regions[bank].mrStart << 12)) break;  /* Leave if we fit */
                   }
                   
                   if(bank >= 0) break;                                                            /* We are done if we found a suitable bank */
                   
                   hash_table_size = hash_table_size >> 1;                         /* Try the next size down */
           }
   
           if(htslop) {                                                                                    /* If there was slop (i.e., wasted pages for alignment) add a new region */
                   for(i = pmap_mem_regions_count - 1; i >= (unsigned)bank; i--) { /* Copy from end to our bank, including our bank */
                           pmap_mem_regions[i + 1].mrStart  = pmap_mem_regions[i].mrStart; /* Set the start of the bank */
                           pmap_mem_regions[i + 1].mrAStart = pmap_mem_regions[i].mrAStart;        /* Set the start of allocatable area */
                           pmap_mem_regions[i + 1].mrEnd    = pmap_mem_regions[i].mrEnd;   /* Set the end address of bank */
                           pmap_mem_regions[i + 1].mrAEnd   = pmap_mem_regions[i].mrAEnd;  /* Set the end address of allocatable area */
                   }
                   
                   pmap_mem_regions[i + 1].mrStart  = (hash_table_base + hash_table_size) >> 12;   /* Set the start of the next bank to the start of the slop area */
                   pmap_mem_regions[i + 1].mrAStart = (hash_table_base + hash_table_size) >> 12;   /* Set the start of allocatable area to the start of the slop area */
                   pmap_mem_regions[i].mrEnd        = (hash_table_base + hash_table_size - 4096) >> 12;    /* Set the end of our bank to the end of the hash table */
                   
           }               
           
           pmap_mem_regions[bank].mrAEnd = (hash_table_base - PCAsize - 4096) >> 12;       /* Set the maximum allocatable in this bank */
           
           hw_hash_init();                                                                                 /* Initiaize the hash table and PCA */
           hw_setup_trans();                                                                               /* Set up hardware registers needed for translation */
           
   /*
    *      The hash table is now all initialized and so is the PCA.  Go on to do the rest of it.
    *      This allocation is from the bottom up.
    */     
           
           num = atop_64(msize);                                                                           /* Get number of pages in all of memory */
   
   /* Figure out how much we need to allocate */
   
           size = (vm_size_t) (
                   (InitialSaveBloks * PAGE_SIZE) +                                        /* Allow space for the initial context saveareas */
                   (BackPocketSaveBloks * PAGE_SIZE) +                                     /* For backpocket saveareas */
                   trcWork.traceSize +                                                             /* Size of trace table */
                   ((((1 << maxAdrSpb) * sizeof(pmapTransTab)) + 4095) & -4096) +  /* Size of pmap translate table */
                   (((num * sizeof(struct phys_entry)) + 4095) & -4096)    /* For the physical entries */
           );
   
           mapsize = size = round_page(size);                                              /* Get size of area to map that we just calculated */
           mapsize = mapsize + kmapsize;                                                   /* Account for the kernel text size */
   
           vmpagesz = round_page(num * sizeof(struct vm_page));    /* Allow for all vm_pages needed to map physical mem */
           vmmapsz = round_page((num / 8) * sizeof(struct vm_map_entry));  /* Allow for vm_maps */
           
           mapsize = mapsize + vmpagesz + vmmapsz;                                 /* Add the VM system estimates into the grand total */
   
           mapsize = mapsize + (4 * 1024 * 1024);                                  /* Allow for 4 meg of extra mappings */
           mapsize = ((mapsize / PAGE_SIZE) + MAPPERBLOK - 1) / MAPPERBLOK;        /* Get number of blocks of mappings we need */
           mapsize = mapsize + ((mapsize  + MAPPERBLOK - 1) / MAPPERBLOK); /* Account for the mappings themselves */
   
           size = size + (mapsize * PAGE_SIZE);                                    /* Get the true size we need */
   
           /* hash table must be aligned to its size */
   
           addr = *first_avail;                                                                    /* Set the address to start allocations */
           first_used_addr = addr;                                                                 /* Remember where we started */
   
           bzero((char *)addr, size);                                                              /* Clear everything that we are allocating */
   
           savearea_init(addr);                                                                    /* Initialize the savearea chains and data */
   
           addr = (vm_offset_t)((unsigned int)addr + ((InitialSaveBloks + BackPocketSaveBloks) * PAGE_SIZE));      /* Point past saveareas */
   
           trcWork.traceCurr = (unsigned int)addr;                                 /* Set first trace slot to use */
           trcWork.traceStart = (unsigned int)addr;                                /* Set start of trace table */
           trcWork.traceEnd = (unsigned int)addr + trcWork.traceSize;              /* Set end of trace table */
   
           addr = (vm_offset_t)trcWork.traceEnd;                                   /* Set next allocatable location */
                   
           pmapTrans = (pmapTransTab *)addr;                                               /* Point to the pmap to hash translation table */
                   
           pmapTrans[PPC_SID_KERNEL].pmapPAddr = (addr64_t)((uintptr_t)kernel_pmap);       /* Initialize the kernel pmap in the translate table */
           pmapTrans[PPC_SID_KERNEL].pmapVAddr = CAST_DOWN(unsigned int, kernel_pmap);  /* Initialize the kernel pmap in the translate table */
                   
           addr += ((((1 << maxAdrSpb) * sizeof(pmapTransTab)) + 4095) & -4096);   /* Point past pmap translate table */
   
   /*      NOTE: the phys_table must be within the first 2GB of physical RAM. This makes sure we only need to do 32-bit arithmetic */
   
           phys_entry = (struct phys_entry *) addr;                                /* Get pointer to physical table */
   
           for (bank = 0; bank < pmap_mem_regions_count; bank++) { /* Set pointer and initialize all banks of ram */
                   
                   pmap_mem_regions[bank].mrPhysTab = phys_entry;          /* Set pointer to the physical table for this bank */
                   
                   phys_entry = phys_entry + (pmap_mem_regions[bank].mrEnd - pmap_mem_regions[bank].mrStart + 1);  /* Point to the next */
           }
   
           addr += (((num * sizeof(struct phys_entry)) + 4095) & -4096);   /* Step on past the physical entries */
           
   /*
    *              Remaining space is for mapping entries.  Tell the initializer routine that
    *              the mapping system can't release this block because it's permanently assigned
    */
   
           mapping_init();                                                                                 /* Initialize the mapping tables */
   
           for(i = addr; i < first_used_addr + size; i += PAGE_SIZE) {     /* Add initial mapping blocks */
                   mapping_free_init(i, 1, 0);                                                     /* Pass block address and say that this one is not releasable */
           }
           mapCtl.mapcmin = MAPPERBLOK;                                                    /* Make sure we only adjust one at a time */
   
           /* Map V=R the page tables */
           pmap_map(first_used_addr, first_used_addr,
                    round_page(first_used_addr + size), VM_PROT_READ | VM_PROT_WRITE);
   
           *first_avail = round_page(first_used_addr + size);              /* Set next available page */
           first_free_virt = *first_avail;                                                 /* Ditto */
           
           /* For 64-bit machines, block map physical memory and the I/O hole into kernel space */
           if(BootProcInfo.pf.Available & pf64Bit) {                               /* Are we on a 64-bit machine? */
                   lowGlo.lgPMWvaddr = PHYS_MEM_WINDOW_VADDR;                      /* Initialize the physical memory window's virtual address */
   
                   pmap_map_physical();                                                            /* Block map physical memory into the window */
                   
                   pmap_map_iohole(IO_MEM_WINDOW_VADDR, IO_MEM_WINDOW_SIZE);
                                                                                                                           /* Block map the I/O hole */
           }
   
           /* All the rest of memory is free - add it to the free
            * regions so that it can be allocated by pmap_steal
            */
   
           pmap_mem_regions[0].mrAStart = (*first_avail >> 12);    /* Set up the free area to start allocations (always in the first bank) */
   
           current_free_region = 0;                                                                /* Set that we will start allocating in bank 0 */
           avail_remaining = 0;                                                                    /* Clear free page count */
           for(bank = 0; bank < pmap_mem_regions_count; bank++) {  /* Total up all of the pages in the system that are available */
                   avail_remaining += (pmap_mem_regions[bank].mrAEnd - pmap_mem_regions[bank].mrAStart) + 1;       /* Add in allocatable pages in this bank */
           }
   
   
   }
   
   /*
    * pmap_init(spa, epa)
    *      finishes the initialization of the pmap module.
    *      This procedure is called from vm_mem_init() in vm/vm_init.c
    *      to initialize any remaining data structures that the pmap module
    *      needs to map virtual memory (VM is already ON).
    *
    *      Note that the pmap needs to be sized and aligned to
    *      a power of two.  This is because it is used both in virtual and
    *      real so it can't span a page boundary.
    */
   
   void
   pmap_init(void)
   {
   
           pmap_zone = zinit(pmapSize, 400 * pmapSize, 4096, "pmap");
   #if     ZONE_DEBUG
           zone_debug_disable(pmap_zone);          /* Can't debug this one 'cause it messes with size and alignment */
   #endif  /* ZONE_DEBUG */
   
           pmap_initialized = TRUE;
   
           /*
            *      Initialize list of freed up pmaps
            */
           free_pmap_list = 0;                                     /* Set that there are no free pmaps */
           free_pmap_count = 0;
           simple_lock_init(&free_pmap_lock, 0);
           
   }
   
   unsigned int pmap_free_pages(void)
   {
           return avail_remaining;
   }
   
   /*
    *      This function allocates physical pages.
    */
   
   /* Non-optimal, but only used for virtual memory startup.
    * Allocate memory from a table of free physical addresses
    * If there are no more free entries, too bad. 
    */
   
   boolean_t pmap_next_page(ppnum_t *addrp)
   {
                   int i;
   
           if(current_free_region >= pmap_mem_regions_count) return FALSE; /* Return failure if we have used everything... */
           
           for(i = current_free_region; i < pmap_mem_regions_count; i++) { /* Find the next bank with free pages */
                   if(pmap_mem_regions[i].mrAStart <= pmap_mem_regions[i].mrAEnd) break;   /* Found one */
           }
           
           current_free_region = i;                                                                                /* Set our current bank */
           if(i >= pmap_mem_regions_count) return FALSE;                                   /* Couldn't find a free page */
   
           *addrp = pmap_mem_regions[i].mrAStart;                                  /* Allocate the page */
           pmap_mem_regions[i].mrAStart = pmap_mem_regions[i].mrAStart + 1;        /* Set the next one to go */
           avail_remaining--;                                                                                              /* Drop free count */
   
           return TRUE;
   }
   
   void pmap_virtual_space(
           vm_offset_t *startp,
           vm_offset_t *endp)
   {
           *startp = round_page(first_free_virt);
           *endp   = vm_last_addr;
   }
   
   /*
    * pmap_create
    *
    * Create and return a physical map.
    *
    * If the size specified for the map is zero, the map is an actual physical
    * map, and may be referenced by the hardware.
    *
    * A pmap is either in the free list or in the in-use list.  The only use
    * of the in-use list (aside from debugging) is to handle the VSID wrap situation.
    * Whenever a new pmap is allocated (i.e., not recovered from the free list). The
    * in-use list is matched until a hole in the VSID sequence is found. (Note
    * that the in-use pmaps are queued in VSID sequence order.) This is all done
    * while free_pmap_lock is held.
    *
    * If the size specified is non-zero, the map will be used in software 
    * only, and is bounded by that size.
    */
   pmap_t
   pmap_create(vm_map_size_t size)
   {
           pmap_t pmap, ckpmap, fore;
           int s;
           unsigned int currSID;
           addr64_t physpmap;
   
           /*
            * A software use-only map doesn't even need a pmap structure.
            */
           if (size)
                   return(PMAP_NULL);
   
           /* 
            * If there is a pmap in the pmap free list, reuse it. 
            * Note that we use free_pmap_list for all chaining of pmaps, both to
            * the free list and the in use chain (anchored from kernel_pmap).
            */
           s = splhigh();
           simple_lock(&free_pmap_lock);
           
           if(free_pmap_list) {                                                    /* Any free? */
                   pmap = free_pmap_list;                                          /* Yes, allocate it */
                   free_pmap_list = (pmap_t)pmap->freepmap;        /* Dequeue this one (we chain free ones through freepmap) */
                   free_pmap_count--;
           }
           else {
                   simple_unlock(&free_pmap_lock);                         /* Unlock just in case */
                   splx(s);
   
                   pmap = (pmap_t) zalloc(pmap_zone);                      /* Get one */
                   if (pmap == PMAP_NULL) return(PMAP_NULL);       /* Handle out-of-memory condition */
                   
                   bzero((char *)pmap, pmapSize);                          /* Clean up the pmap */
                   
                   s = splhigh();
                   simple_lock(&free_pmap_lock);                           /* Lock it back up      */
                   
                   ckpmap = cursor_pmap;                                           /* Get starting point for free ID search */
                   currSID = ckpmap->spaceNum;                                     /* Get the actual space ID number */
   
                   while(1) {                                                                      /* Keep trying until something happens */
                   
                           currSID = (currSID + 1) & (maxAdrSp - 1);       /* Get the next in the sequence */
                           if(((currSID * incrVSID) & (maxAdrSp - 1)) == invalSpace) continue;     /* Skip the space we have reserved */
                           ckpmap = (pmap_t)ckpmap->pmap_link.next;        /* On to the next in-use pmap */
           
                           if(ckpmap->spaceNum != currSID) break;  /* If we are out of sequence, this is free */
                           
                           if(ckpmap == cursor_pmap) {                             /* See if we have 2^20 already allocated */
                                   panic("pmap_create: Maximum number (%d) active address spaces reached\n", maxAdrSp);    /* Die pig dog */
                           }
                   }
   
                   pmap->space = (currSID * incrVSID) & (maxAdrSp - 1);    /* Calculate the actual VSID */
                   pmap->spaceNum = currSID;                                       /* Set the space ID number */
   /*
    *              Now we link into the chain just before the out of sequence guy.
    */
   
                   fore = (pmap_t)ckpmap->pmap_link.prev;          /* Get the current's previous */
                   pmap->pmap_link.next = (queue_t)ckpmap;         /* My next points to the current */
                   fore->pmap_link.next = (queue_t)pmap;           /* Current's previous's next points to me */
                   pmap->pmap_link.prev = (queue_t)fore;           /* My prev points to what the current pointed to */
                   ckpmap->pmap_link.prev = (queue_t)pmap;         /* Current's prev points to me */
                   
                   physpmap = ((addr64_t)pmap_find_phys(kernel_pmap, (addr64_t)((uintptr_t)pmap)) << 12) | (addr64_t)((unsigned int)pmap & 0xFFF); /* Get the physical address of the pmap */
                   
                   pmap->pmapvr = (addr64_t)((uintptr_t)pmap) ^ physpmap;  /* Make V to R translation mask */
                   
                   pmapTrans[pmap->space].pmapPAddr = physpmap;    /* Set translate table physical to point to us */
                   pmapTrans[pmap->space].pmapVAddr = CAST_DOWN(unsigned int, pmap);       /* Set translate table virtual to point to us */
           }
   
           pmap->pmapVmmExt = 0;                                                   /* Clear VMM extension block vaddr */
           pmap->pmapVmmExtPhys = 0;                                               /*  and the paddr, too */
           pmap->pmapFlags = pmapKeyDef;                                   /* Set default key */
           pmap->pmapCCtl = pmapCCtlVal;                                   /* Initialize cache control */
           pmap->ref_count = 1;
           pmap->stats.resident_count = 0;
           pmap->stats.wired_count = 0;
           pmap->pmapSCSubTag = 0x0000000000000000ULL;             /* Make sure this is clean an tidy */
           simple_unlock(&free_pmap_lock);
   
           splx(s);
           return(pmap);
   }
   
   /* 
    * pmap_destroy
    * 
    * Gives up a reference to the specified pmap.  When the reference count 
    * reaches zero the pmap structure is added to the pmap free list.
    *
    * Should only be called if the map contains no valid mappings.
    */
   void
   pmap_destroy(pmap_t pmap)
   {
           int ref_count;
           spl_t s;
           pmap_t fore, aft;
   
           if (pmap == PMAP_NULL)
                   return;
   
           ref_count=hw_atomic_sub(&pmap->ref_count, 1);                   /* Back off the count */
           if(ref_count>0) return;                                                                 /* Still more users, leave now... */
   
           if(ref_count < 0)                                                                               /* Did we go too far? */
                   panic("pmap_destroy(): ref_count < 0");
   
           if (!(pmap->pmapFlags & pmapVMgsaa)) {                                  /* Don't try this for a shadow assist guest */
                   pmap_unmap_sharedpage(pmap);                                            /* Remove any mapping of page -1 */
           }
           
   #ifdef notdef
           if(pmap->stats.resident_count != 0)
                   panic("PMAP_DESTROY: pmap not empty");
   #else
           if(pmap->stats.resident_count != 0) {
                   pmap_remove(pmap, 0, 0xFFFFFFFFFFFFF000ULL);
           }
   #endif
   
           /* 
            * Add the pmap to the pmap free list. 
            */
   
           s = splhigh();
           /* 
            * Add the pmap to the pmap free list. 
            */
           simple_lock(&free_pmap_lock);
           
           if (free_pmap_count <= free_pmap_max) {         /* Do we have enough spares? */
                   
                   pmap->freepmap = free_pmap_list;                /* Queue in front */
                   free_pmap_list = pmap;
                   free_pmap_count++;
                   simple_unlock(&free_pmap_lock);
   
           } else {
                   if(cursor_pmap == pmap) cursor_pmap = (pmap_t)pmap->pmap_link.prev;     /* If we are releasing the cursor, back up */
                   fore = (pmap_t)pmap->pmap_link.prev;
                   aft  = (pmap_t)pmap->pmap_link.next;
                   fore->pmap_link.next = pmap->pmap_link.next;    /* My previous's next is my next */
                   aft->pmap_link.prev = pmap->pmap_link.prev;             /* My next's previous is my previous */ 
                   simple_unlock(&free_pmap_lock);
                   pmapTrans[pmap->space].pmapPAddr = -1;                  /* Invalidate the translate table physical */
                   pmapTrans[pmap->space].pmapVAddr = -1;                  /* Invalidate the translate table virtual */
                   zfree(pmap_zone, pmap);
           }
           splx(s);
   }
   
   /*
    * pmap_reference(pmap)
    *      gains a reference to the specified pmap.
    */
   void
   pmap_reference(pmap_t pmap)
   {
           if (pmap != PMAP_NULL) hw_atomic_add(&pmap->ref_count, 1);      /* Bump the count */
   }
   
   /*
    * pmap_remove_some_phys
    *
    *      Removes mappings of the associated page from the specified pmap
    *
    */
   void pmap_remove_some_phys(
                pmap_t pmap,
                vm_offset_t pa)
   {
           register struct phys_entry      *pp;
           register struct mapping         *mp;
           unsigned int pindex;
   
           if (pmap == PMAP_NULL) {                                        /* This should never be called with a null pmap */
                   panic("pmap_remove_some_phys: null pmap\n");
           }
   
           pp = mapping_phys_lookup(pa, &pindex);          /* Get physical entry */
           if (pp == 0) return;                                            /* Leave if not in physical RAM */
   
           do {                                                                            /* Keep going until we toss all pages from this pmap */
                   if (pmap->pmapFlags & pmapVMhost) {
                           mp = hw_purge_phys(pp);                         /* Toss a map */
                           switch ((unsigned int)mp & mapRetCode) {
                                   case mapRtOK:
                                           mapping_free(mp);                       /* Return mapping to free inventory */
                                           break;
                                   case mapRtGuest:
                                           break;                                          /* Don't try to return a guest mapping */
                                   case mapRtEmpty:
                                           break;                                          /* Physent chain empty, we're done */
                                   case mapRtNotFnd:                               
                                           break;                                          /* Mapping disappeared on us, retry */  
                                   default:
                                           panic("pmap_remove_some_phys: hw_purge_phys failed - pp = %08X, pmap = %08X, code = %08X\n",
                                                           pp, pmap, mp);          /* Handle failure with our usual lack of tact */
                           }
                   } else { 
                           mp = hw_purge_space(pp, pmap);          /* Toss a map */
                           switch ((unsigned int)mp & mapRetCode) {
                                   case mapRtOK:
                                           mapping_free(mp);                       /* Return mapping to free inventory */
                                           break;
                                   case mapRtEmpty:
                                           break;                                          /* Physent chain empty, we're done */
                                   case mapRtNotFnd:                               
                                           break;                                          /* Mapping disappeared on us, retry */  
                                   default:
                                           panic("pmap_remove_some_phys: hw_purge_phys failed - pp = %08X, pmap = %08X, code = %08X\n",
                                                           pp, pmap, mp);          /* Handle failure with our usual lack of tact */
                           }
                   }
           } while (mapRtEmpty != ((unsigned int)mp & mapRetCode));
   
   #if DEBUG       
           if ((pmap->pmapFlags & pmapVMhost) && !pmap_verify_free(pa)) 
                   panic("pmap_remove_some_phys: cruft left behind - pa = %08X, pmap = %08X\n", pa, pmap);
   #endif
   
           return;                                                                         /* Leave... */
   }
   
   /*
    * pmap_remove(pmap, s, e)
    *      unmaps all virtual addresses v in the virtual address
    *      range determined by [s, e) and pmap.
    *      s and e must be on machine independent page boundaries and
    *      s must be less than or equal to e.
    *
    *      Note that pmap_remove does not remove any mappings in nested pmaps. We just 
    *      skip those segments.
    */
   void
   pmap_remove(
               pmap_t pmap,
               addr64_t sva,
               addr64_t eva)
   {
           addr64_t                va, endva;
   
           if (pmap == PMAP_NULL) return;                                  /* Leave if software pmap */
   
   
           /* It is just possible that eva might have wrapped around to zero,
            * and sometimes we get asked to liberate something of size zero
            * even though it's dumb (eg. after zero length read_overwrites)
            */
           assert(eva >= sva);
   
           /* If these are not page aligned the loop might not terminate */
           assert((sva == trunc_page_64(sva)) && (eva == trunc_page_64(eva)));
   
           va = sva & -4096LL;                                                     /* Round start down to a page */
           endva = eva & -4096LL;                                          /* Round end down to a page */
   
           while(1) {                                                                      /* Go until we finish the range */
                   va = mapping_remove(pmap, va);                  /* Remove the mapping and see what's next */
                   va = va & -4096LL;                                              /* Make sure the "not found" indication is clear */
                   if((va == 0) || (va >= endva)) break;   /* End loop if we finish range or run off the end */
           }
   
   }
   
   /*
    *      Routine:
    *              pmap_page_protect
    *
    *      Function:
    *              Lower the permission for all mappings to a given page.
    */
   void
   pmap_page_protect(
           ppnum_t pa,
           vm_prot_t prot)
   {
           register struct phys_entry      *pp;
           boolean_t                       remove;
           unsigned int            pindex;
           mapping_t                       *mp;
   
   
           switch (prot) {
                   case VM_PROT_READ:
                   case VM_PROT_READ|VM_PROT_EXECUTE:
                           remove = FALSE;
                           break;
                   case VM_PROT_ALL:
                           return;
                   default:
                           remove = TRUE;
                           break;
           }
   
   
           pp = mapping_phys_lookup(pa, &pindex);          /* Get physical entry */
           if (pp == 0) return;                                            /* Leave if not in physical RAM */
   
           if (remove) {                                                           /* If the protection was set to none, we'll remove all mappings */
                   
                   do {                                                                    /* Keep going until we toss all pages from this physical page */
                           mp = hw_purge_phys(pp);                         /* Toss a map */
                           switch ((unsigned int)mp & mapRetCode) {
                                   case mapRtOK:
                                                           mapping_free(mp);       /* Return mapping to free inventory */
                                                           break;
                                   case mapRtGuest:
                                                           break;                          /* Don't try to return a guest mapping */
                                   case mapRtNotFnd:
                                                           break;                          /* Mapping disappeared on us, retry */
                                   case mapRtEmpty:
                                                           break;                          /* Physent chain empty, we're done */
                                   default:        panic("pmap_page_protect: hw_purge_phys failed - pp = %08X, code = %08X\n",
                                                                     pp, mp);              /* Handle failure with our usual lack of tact */
                           }
                   } while (mapRtEmpty != ((unsigned int)mp & mapRetCode));
   
   #if DEBUG
                   if (!pmap_verify_free(pa)) 
                           panic("pmap_page_protect: cruft left behind - pa = %08X\n", pa);
   #endif
   
                   return;                                                                 /* Leave... */
           }
   
   /*      When we get here, it means that we are to change the protection for a 
    *      physical page.  
    */
    
           mapping_protect_phys(pa, prot & VM_PROT_ALL);   /* Change protection of all mappings to page. */
   
   }
   
   /*
    *      Routine:
    *              pmap_disconnect
    *
    *      Function:
    *              Disconnect all mappings for this page and return reference and change status
    *              in generic format.
    *
    */
   unsigned int pmap_disconnect(
          ppnum_t pa)
  {
          register struct phys_entry *pp;
          unsigned int                            pindex;
          mapping_t                                  *mp;
          
          pp = mapping_phys_lookup(pa, &pindex);          /* Get physical entry */
          if (pp == 0) return (0);                                        /* Return null ref and chg if not in physical RAM */
          do {                                                                            /* Iterate until all mappings are dead and gone */
                  mp = hw_purge_phys(pp);                                 /* Disconnect a mapping */
                  if (!mp) break;                                                 /* All mappings are gone, leave the loop */
                  switch ((unsigned int)mp & mapRetCode) {
                          case mapRtOK:
                                                  mapping_free(mp);               /* Return mapping to free inventory */
                                                  break;
                          case mapRtGuest:
                                                  break;                                  /* Don't try to return a guest mapping */
                          case mapRtNotFnd:
                                                  break;                                  /* Mapping disappeared on us, retry */
                          case mapRtEmpty:
                                                  break;                                  /* Physent chain empty, we're done */
                          default:        panic("hw_purge_phys: hw_purge_phys failed - pp = %08X, code = %08X\n",
                                                            pp, mp);                      /* Handle failure with our usual lack of tact */
                  }
          } while (mapRtEmpty != ((unsigned int)mp & mapRetCode));
  
  #if DEBUG
          if (!pmap_verify_free(pa)) 
                  panic("pmap_disconnect: cruft left behind - pa = %08X\n", pa);
  #endif
  
          return (mapping_tst_refmod(pa));                        /* Return page ref and chg in generic format */
  }
  
  /*
   * pmap_protect(pmap, s, e, prot)
   *      changes the protection on all virtual addresses v in the 
   *      virtual address range determined by [s, e] and pmap to prot.
   *      s and e must be on machine independent page boundaries and
   *      s must be less than or equal to e.
   *
   *      Note that any requests to change the protection of a nested pmap are
   *      ignored. Those changes MUST be done by calling this with the correct pmap.
   */
  void pmap_protect(
               pmap_t pmap,
               vm_map_offset_t sva, 
               vm_map_offset_t eva,
               vm_prot_t prot)
  {
  
          addr64_t va, endva;
  
          if (pmap == PMAP_NULL) return;                          /* Do nothing if no pmap */
  
          if (prot == VM_PROT_NONE) {                                     /* Should we kill the address range?? */
                  pmap_remove(pmap, (addr64_t)sva, (addr64_t)eva);        /* Yeah, dump 'em */
                  return;                                                                 /* Leave... */
          }
  
          va = sva & -4096LL;                                                     /* Round start down to a page */
          endva = eva & -4096LL;                                          /* Round end down to a page */
  
          while(1) {                                                                      /* Go until we finish the range */
                  mapping_protect(pmap, va, prot & VM_PROT_ALL, &va);     /* Change the protection and see what's next */
                  if((va == 0) || (va >= endva)) break;   /* End loop if we finish range or run off the end */
          }
  
  }
  
  
  
  /*
   * pmap_enter
   *
   * Create a translation for the virtual address (virt) to the physical
   * address (phys) in the pmap with the protection requested. If the
   * translation is wired then we can not allow a full page fault, i.e., 
   * the mapping control block is not eligible to be stolen in a low memory
   * condition.
   *
   * NB: This is the only routine which MAY NOT lazy-evaluate
   *     or lose information.  That is, this routine must actually
   *     insert this page into the given map NOW.
   */
  void
  pmap_enter(pmap_t pmap, vm_map_offset_t va, ppnum_t pa, vm_prot_t prot, 
                  unsigned int flags, __unused boolean_t wired)
  {
          unsigned int            mflags;
          addr64_t                        colva;
          
          if (pmap == PMAP_NULL) return;                                  /* Leave if software pmap */
  
          mflags = 0;                                                                             /* Make sure this is initialized to nothing special */
          if(!(flags & VM_WIMG_USE_DEFAULT)) {                    /* Are they supplying the attributes? */
                  mflags = mmFlgUseAttr | (flags & VM_MEM_GUARDED) | ((flags & VM_MEM_NOT_CACHEABLE) >> 1);       /* Convert to our mapping_make flags */
          }
          
  /*
   *      It is possible to hang here if another processor is remapping any pages we collide with and are removing
   */ 
  
          while(1) {                                                                              /* Keep trying the enter until it goes in */
          
                  colva = mapping_make(pmap, va, pa, mflags, 1, prot & VM_PROT_ALL);      /* Enter the mapping into the pmap */
                  
                  if(!colva) break;                                                       /* If there were no collisions, we are done... */
                  
                  mapping_remove(pmap, colva);                            /* Remove the mapping that collided */
          }
  }
  
  /*
   *              Enters translations for odd-sized V=F blocks.
   *
   *              The higher level VM map should be locked to insure that we don't have a
   *              double diddle here.
   *
   *              We panic if we get a block that overlaps with another. We do not merge adjacent
   *              blocks because removing any address within a block removes the entire block and if
   *              would really mess things up if we trashed too much.
   *
   *              Once a block is mapped, it is unmutable, that is, protection, catch mode, etc. can
   *              not be changed.  The block must be unmapped and then remapped with the new stuff.
   *              We also do not keep track of reference or change flags.
   *
   *              Any block that is larger than 256MB must be a multiple of 32MB.  We panic if it is not.
   *
   *              Note that pmap_map_block_rc is the same but doesn't panic if collision.
   *
   */
   
  void pmap_map_block(pmap_t pmap, addr64_t va, ppnum_t pa, uint32_t size, vm_prot_t prot, int attr, unsigned int flags) {        /* Map an autogenned block */
  
          unsigned int            mflags;
          addr64_t                        colva;
  
          
          if (pmap == PMAP_NULL) {                                                /* Did they give us a pmap? */
                  panic("pmap_map_block: null pmap\n");           /* No, like that's dumb... */
          }
  
  //      kprintf("pmap_map_block: (%08X) va = %016llX, pa = %08X, size = %08X, prot = %08X, attr = %08X, flags = %08X\n",        /* (BRINGUP) */
  //              current_thread(), va, pa, size, prot, attr, flags);     /* (BRINGUP) */
  
          mflags = mmFlgBlock | mmFlgUseAttr | (attr & VM_MEM_GUARDED) | ((attr & VM_MEM_NOT_CACHEABLE) >> 1);    /* Convert to our mapping_make flags */
          if(flags) mflags |= mmFlgPerm;                                  /* Mark permanent if requested */
          
          colva = mapping_make(pmap, va, pa, mflags, size, prot); /* Enter the mapping into the pmap */
          
          if(colva) {                                                                             /* If there was a collision, panic */
                  panic("pmap_map_block: mapping error %d, pmap = %08X, va = %016llX\n", (uint32_t)(colva & mapRetCode), pmap, va);
          }
          
          return;                                                                                 /* Return */
  }
  
  int pmap_map_block_rc(pmap_t pmap, addr64_t va, ppnum_t pa, uint32_t size, vm_prot_t prot, int attr, unsigned int flags) {      /* Map an autogenned block */
  
          unsigned int            mflags;
          addr64_t                        colva;
  
          
          if (pmap == PMAP_NULL) {                                                /* Did they give us a pmap? */
                  panic("pmap_map_block_rc: null pmap\n");        /* No, like that's dumb... */
          }
  
          mflags = mmFlgBlock | mmFlgUseAttr | (attr & VM_MEM_GUARDED) | ((attr & VM_MEM_NOT_CACHEABLE) >> 1);    /* Convert to our mapping_make flags */
          if(flags) mflags |= mmFlgPerm;                                  /* Mark permanent if requested */
  
          colva = mapping_make(pmap, va, pa, mflags, size, prot); /* Enter the mapping into the pmap */
          
          if(colva) return 0;                                                             /* If there was a collision, fail */
          
          return 1;                                                                               /* Return true of we worked */
  }
  
  /*
   * pmap_extract(pmap, va)
   *      returns the physical address corrsponding to the 
   *      virtual address specified by pmap and va if the
   *      virtual address is mapped and 0 if it is not.
   *      Note: we assume nothing is ever mapped to phys 0.
   *
   *      NOTE: This call always will fail for physical addresses greater than 0xFFFFF000.
   */
  vm_offset_t pmap_extract(pmap_t pmap, vm_map_offset_t va) {
  
          spl_t                                   spl;
          register struct mapping *mp;
          register vm_offset_t    pa;
          addr64_t                                nextva;
          ppnum_t                                 ppoffset;
          unsigned int                    gva;
  
  #ifdef BOGUSCOMPAT
          panic("pmap_extract: THIS CALL IS BOGUS. NEVER USE IT EVER. So there...\n");    /* Don't use this */
  #else
  
          gva = (unsigned int)va;                                                 /* Make sure we don't have a sign */
  
          spl = splhigh();                                                                /* We can't allow any loss of control here */
          
          mp = mapping_find(pmap, (addr64_t)gva, &nextva,1);      /* Find the mapping for this address */
          
          if(!mp) {                                                                               /* Is the page mapped? */
                  splx(spl);                                                                      /* Enable interrupts */
                  return 0;                                                                       /* Pass back 0 if not found */
          }
  
          ppoffset = (ppnum_t)(((gva & -4096LL) - (mp->mpVAddr & -4096LL)) >> 12);        /* Get offset from va to base va */
          
          
          pa = mp->mpPAddr + ppoffset;                                    /* Remember ppage because mapping may vanish after drop call */
                          
          mapping_drop_busy(mp);                                                  /* We have everything we need from the mapping */
          splx(spl);                                                                              /* Restore 'rupts */
  
          if(pa > maxPPage32) return 0;                                   /* Force large addresses to fail */
          
          pa = (pa << 12) | (va & 0xFFF);                                 /* Convert physical page number to address */
          
  #endif
          return pa;                                                                              /* Return physical address or 0 */
  }
  
  /*
   * ppnum_t pmap_find_phys(pmap, addr64_t va)
   *      returns the physical page corrsponding to the 
   *      virtual address specified by pmap and va if the
   *      virtual address is mapped and 0 if it is not.
   *      Note: we assume nothing is ever mapped to phys 0.
   *
   */
  ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va) {
  
          spl_t                                   spl;
          register struct mapping *mp;
          ppnum_t                                 pa, ppoffset;
          addr64_t                                nextva;
  
          spl = splhigh();                                                                /* We can't allow any loss of control here */
          
          mp = mapping_find(pmap, va, &nextva, 1);                /* Find the mapping for this address */
          
          if(!mp) {                                                                               /* Is the page mapped? */
                  splx(spl);                                                                      /* Enable interrupts */
                  return 0;                                                                       /* Pass back 0 if not found */
          }
                  
          
          ppoffset = (ppnum_t)(((va & -4096LL) - (mp->mpVAddr & -4096LL)) >> 12); /* Get offset from va to base va */
          
          pa = mp->mpPAddr + ppoffset;                                    /* Get the actual physical address */
  
          mapping_drop_busy(mp);                                                  /* We have everything we need from the mapping */
  
          splx(spl);                                                                              /* Restore 'rupts */
          return pa;                                                                              /* Return physical address or 0 */
  }
  
  
  /*
   *      pmap_attributes:
   *
   *      Set/Get special memory attributes; not implemented.
   *
   *      Note: 'VAL_GET_INFO' is used to return info about a page.
   *        If less than 1 page is specified, return the physical page
   *        mapping and a count of the number of mappings to that page.
   *        If more than one page is specified, return the number
   *        of resident pages and the number of shared (more than
   *        one mapping) pages in the range;
   *
   *
   */
  kern_return_t
  pmap_attribute(
          __unused pmap_t                         pmap,
          __unused vm_map_offset_t                address,
          __unused vm_map_size_t                  size,
          __unused vm_machine_attribute_t         attribute,
          __unused vm_machine_attribute_val_t*    value)  
  {
          
          return KERN_INVALID_ARGUMENT;
  
  }
  
  /*
   * pmap_attribute_cache_sync(vm_offset_t pa)
   * 
   * Invalidates all of the instruction cache on a physical page and
   * pushes any dirty data from the data cache for the same physical page
   */
   
  kern_return_t pmap_attribute_cache_sync(ppnum_t pp, vm_size_t size,
                                  __unused vm_machine_attribute_t  attribute,
                                  __unused vm_machine_attribute_val_t* value) {
          
          spl_t s;
          unsigned int i, npages;
          
          npages = round_page(size) >> 12;                        /* Get the number of pages to do */
          
          for(i = 0; i < npages; i++) {                           /* Do all requested pages */
                  s = splhigh();                                                  /* No interruptions here */
                  sync_ppage(pp + i);                                             /* Go flush data cache and invalidate icache */
                  splx(s);                                                                /* Allow interruptions */
          }
          
          return KERN_SUCCESS;
  }
  
  /*
   * pmap_sync_page_data_phys(ppnum_t pa)
   * 
   * Invalidates all of the instruction cache on a physical page and
   * pushes any dirty data from the data cache for the same physical page
   */
   
  void pmap_sync_page_data_phys(ppnum_t pa) {
          
          spl_t s;
          
          s = splhigh();                                                          /* No interruptions here */
          sync_ppage(pa);                                                         /* Sync up dem caches */
          splx(s);                                                                        /* Allow interruptions */
          return;
  }
  
  void
  pmap_sync_page_attributes_phys(ppnum_t pa)
  {
          pmap_sync_page_data_phys(pa);
  }
  
  /*
   * pmap_collect
   * 
   * Garbage collects the physical map system for pages that are no longer used.
   * It isn't implemented or needed or wanted.
   */
  void
  pmap_collect(__unused pmap_t pmap)
  {
          return;
  }
  
  /*
   *      Routine:        pmap_activate
   *      Function:
   *              Binds the given physical map to the given
   *              processor, and returns a hardware map description.
   *              It isn't implemented or needed or wanted.
   */
  void
  pmap_activate(
          __unused pmap_t pmap,
          __unused thread_t th,
          __unused int which_cpu)
  {
          return;
  }
  /*
   * pmap_deactivate:
   * It isn't implemented or needed or wanted.
   */
  void
  pmap_deactivate(
          __unused pmap_t pmap,
          __unused thread_t th,
          __unused int which_cpu)
  {
          return;
  }
  
  
  /*
   * pmap_pageable(pmap, s, e, pageable)
   *      Make the specified pages (by pmap, offset)
   *      pageable (or not) as requested.
   *
   *      A page which is not pageable may not take
   *      a fault; therefore, its page table entry
   *      must remain valid for the duration.
   *
   *      This routine is merely advisory; pmap_enter()
   *      will specify that these pages are to be wired
   *      down (or not) as appropriate.
   *
   *      (called from vm/vm_fault.c).
   */
  void
  pmap_pageable(
          __unused pmap_t                         pmap,
          __unused vm_map_offset_t        start,
          __unused vm_map_offset_t        end,
          __unused boolean_t                      pageable)
  {
  
          return;                                                                                         /* This is not used... */
  
  }
  /*
   *      Routine:        pmap_change_wiring
   *      NOT USED ANYMORE.
   */
  void
  pmap_change_wiring(
          __unused pmap_t                         pmap,
          __unused vm_map_offset_t        va,
          __unused boolean_t                      wired)
  {
          return;                                                                                         /* This is not used... */
  }
  
  /*
   * pmap_modify_pages(pmap, s, e)
   *      sets the modified bit on all virtual addresses v in the 
   *      virtual address range determined by [s, e] and pmap,
   *      s and e must be on machine independent page boundaries and
   *      s must be less than or equal to e.
   *
   *  Note that this function will not descend nested pmaps.
   */
  void
  pmap_modify_pages(
               pmap_t pmap,
               vm_map_offset_t sva, 
               vm_map_offset_t eva)
  {
          spl_t           spl;
          mapping_t       *mp;
          ppnum_t         pa;
          addr64_t                va, endva;
          unsigned int    savetype;
  
          if (pmap == PMAP_NULL) return;                                  /* If no pmap, can't do it... */
          
          va = sva & -4096;                                                               /* Round to page */
          endva = eva & -4096;                                                    /* Round to page */
  
          while (va < endva) {                                                    /* Walk through all pages */
  
                  spl = splhigh();                                                        /* We can't allow any loss of control here */
          
                  mp = mapping_find(pmap, (addr64_t)va, &va, 0);  /* Find the mapping for this address */
                  
                  if(!mp) {                                                                       /* Is the page mapped? */
                          splx(spl);                                                              /* Page not mapped, restore interruptions */
                          if((va == 0) || (va >= endva)) break;   /* We are done if there are no more or we hit the end... */
                          continue;                                                               /* We are not done and there is more to check... */
                  }
                  
                  savetype = mp->mpFlags & mpType;                        /* Remember the type */
                  pa = mp->mpPAddr;                                                       /* Remember ppage because mapping may vanish after drop call */
          
                  mapping_drop_busy(mp);                                          /* We have everything we need from the mapping */
          
                  splx(spl);                                                                      /* Restore 'rupts */
          
                  if(savetype != mpNormal) continue;                      /* Can't mess around with these guys... */      
                  
                  mapping_set_mod(pa);                                            /* Set the modfied bit for this page */
                  
                  if(va == 0) break;                                                      /* We hit the end of the pmap, might as well leave now... */
          }
          return;                                                                                 /* Leave... */
  }
  
  /*
   * pmap_clear_modify(phys)
   *      clears the hardware modified ("dirty") bit for one
   *      machine independant page starting at the given
   *      physical address.  phys must be aligned on a machine
   *      independant page boundary.
   */
  void
  pmap_clear_modify(ppnum_t pa)
  {
  
          mapping_clr_mod(pa);                            /* Clear all change bits for physical page */
  
  }
  
  /*
   * pmap_is_modified(phys)
   *      returns TRUE if the given physical page has been modified 
   *      since the last call to pmap_clear_modify().
   */
  boolean_t
  pmap_is_modified(register ppnum_t pa)
  {
          return mapping_tst_mod(pa);     /* Check for modified */
          
  }
  
  /*
   * pmap_clear_reference(phys)
   *      clears the hardware referenced bit in the given machine
   *      independant physical page.  
   *
   */
  void
  pmap_clear_reference(ppnum_t pa)
  {
          mapping_clr_ref(pa);                    /* Check for modified */
  }
  
  /*
   * pmap_is_referenced(phys)
   *      returns TRUE if the given physical page has been referenced 
   *      since the last call to pmap_clear_reference().
   */
  boolean_t
  pmap_is_referenced(ppnum_t pa)
  {
          return mapping_tst_ref(pa);     /* Check for referenced */
  }
  
  /*
   * pmap_get_refmod(phys)
   *  returns the referenced and modified bits of the specified
   *  physical page.
   */
  unsigned int
  pmap_get_refmod(ppnum_t pa)
  {
          return (mapping_tst_refmod(pa));
  }
  
  /*
   * pmap_clear_refmod(phys, mask)
   *  clears the referenced and modified bits as specified by the mask
   *  of the specified physical page.
   */
  void
  pmap_clear_refmod(ppnum_t pa, unsigned int mask)
  {
          mapping_clr_refmod(pa, mask);
  }
  
  /*
   * pmap_eligible_for_execute(ppnum_t pa)
   *      return true if physical address is eligible to contain executable code;
   *  otherwise, return false
   */
  boolean_t
  pmap_eligible_for_execute(ppnum_t pa)
  {
          phys_entry_t *physent;
          unsigned int  pindex;
  
          physent = mapping_phys_lookup(pa, &pindex);                             /* Get physical entry */
  
          if((!physent) || (physent->ppLink & ppG))
                  return 0;                                                                                       /* If there is no physical entry or marked guarded,
                                                                         the entry is not eligible for execute */
  
          return 1;                                                                                               /* Otherwise, entry is eligible for execute */
  }
  
  #if     MACH_VM_DEBUG
  int
  pmap_list_resident_pages(
          __unused pmap_t         pmap,
          __unused vm_offset_t    *listp,
          __unused int            space)
  {
          return 0;
  }
  #endif  /* MACH_VM_DEBUG */
  
  /*
   * Locking:
   *      spl: VM
   */
  void
  pmap_copy_part_page(
          vm_offset_t     src,
          vm_offset_t     src_offset,
          vm_offset_t     dst,
          vm_offset_t     dst_offset,
          vm_size_t       len)
  {
          addr64_t fsrc, fdst;
  
          assert(((dst <<12) & PAGE_MASK+dst_offset+len) <= PAGE_SIZE);
          assert(((src <<12) & PAGE_MASK+src_offset+len) <= PAGE_SIZE);
  
          fsrc = ((addr64_t)src << 12) + src_offset;
          fdst = ((addr64_t)dst << 12) + dst_offset;
  
          phys_copy(fsrc, fdst, len);                                                             /* Copy the stuff physically */
  }
  
  void
  pmap_zero_part_page(
          __unused vm_offset_t            p,
          __unused vm_offset_t    offset,
          __unused vm_size_t      len)
  {
      panic("pmap_zero_part_page");
  }
  
  boolean_t pmap_verify_free(ppnum_t pa) {
  
          struct phys_entry       *pp;
          unsigned int pindex;
  
          pp = mapping_phys_lookup(pa, &pindex);  /* Get physical entry */
          if (pp == 0) return FALSE;                                      /* If there isn't one, show no mapping... */
  
          if(pp->ppLink & ~(ppLock | ppFlags)) return FALSE;      /* We have at least one mapping */
          return TRUE;                                                            /* No mappings */
  }
  
  
  /* Determine if we need to switch space and set up for it if so */
  
  void pmap_switch(pmap_t map)
  {
          hw_blow_seg(lowGlo.lgUMWvaddr);                                 /* Blow off the first segment */
          hw_blow_seg(lowGlo.lgUMWvaddr + 0x10000000ULL); /* Blow off the second segment */
  
  /* when changing to kernel space, don't bother
   * doing anything, the kernel is mapped from here already.
   */
          if (map->space == PPC_SID_KERNEL) {                     /* Are we switching into kernel space? */
                  return;                                                                 /* If so, we don't do anything... */
          }
          
          hw_set_user_space(map);                                         /* Indicate if we need to load the SRs or not */
          return;                                                                         /* Bye, bye, butterfly... */
  }
  
  /*
   *      kern_return_t pmap_nest(grand, subord, vstart, size)
   *
   *      grand  = the pmap that we will nest subord into
   *      subord = the pmap that goes into the grand
   *      vstart  = start of range in pmap to be inserted
   *      nstart  = start of range in pmap nested pmap
   *      size   = Size of nest area (up to 2TB)
   *
   *      Inserts a pmap into another.  This is used to implement shared segments.
   *      On the current PPC processors, this is limited to segment (256MB) aligned
   *      segment sized ranges.
   *
   *      We actually kinda allow recursive nests.  The gating factor is that we do not allow 
   *      nesting on top of something that is already mapped, i.e., the range must be empty.
   *
   *      Note that we depend upon higher level VM locks to insure that things don't change while
   *      we are doing this.  For example, VM should not be doing any pmap enters while it is nesting
   *      or do 2 nests at once.
   */
  
  kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t nstart, uint64_t size) {
                  
          addr64_t vend, colladdr;
          unsigned int msize;
          int nlists;
          mapping_t *mp;
          
          if(size & 0x0FFFFFFFULL) return KERN_INVALID_VALUE;     /* We can only do this for multiples of 256MB */
          if((size >> 25) > 65536)  return KERN_INVALID_VALUE;    /* Max size we can nest is 2TB */
          if(vstart & 0x0FFFFFFFULL) return KERN_INVALID_VALUE;   /* We can only do this aligned to 256MB */
          if(nstart & 0x0FFFFFFFULL) return KERN_INVALID_VALUE;   /* We can only do this aligned to 256MB */
          
          if(size == 0) {                                                         /*      Is the size valid? */
                  panic("pmap_nest: size is invalid - %016llX\n", size);
          }
          
          msize = (size >> 25) - 1;                                                       /* Change size to blocks of 32MB */
          
          nlists = mapSetLists(grand);                                            /* Set number of lists this will be on */
  
          mp = mapping_alloc(nlists);                                                     /* Get a spare mapping block */
          
          mp->mpFlags = 0x01000000 | mpNest | mpPerm | mpBSu | nlists;    /* Make this a permanent nested pmap with a 32MB basic size unit */
                                                                                                                  /* Set the flags. Make sure busy count is 1 */
          mp->mpSpace = subord->space;                                            /* Set the address space/pmap lookup ID */
          mp->u.mpBSize = msize;                                                          /* Set the size */
          mp->mpPte = 0;                                                                          /* Set the PTE invalid */
          mp->mpPAddr = 0;                                                                        /* Set the physical page number */
          mp->mpVAddr = vstart;                                                           /* Set the address */
          mp->mpNestReloc = nstart - vstart;                                      /* Set grand to nested vaddr relocation value */
          
          colladdr = hw_add_map(grand, mp);                                       /* Go add the mapping to the pmap */
          
          if(colladdr) {                                                                          /* Did it collide? */
                  vend = vstart + size - 4096;                                    /* Point to the last page we would cover in nest */     
                  panic("pmap_nest: attempt to nest into a non-empty range - pmap = %08X, start = %016llX, end = %016llX\n",
                          grand, vstart, vend);
          }
          
          return KERN_SUCCESS;
  }
  
  /*
   *      kern_return_t pmap_unnest(grand, vaddr)
   *
   *      grand  = the pmap that we will nest subord into
   *      vaddr  = start of range in pmap to be unnested
   *
   *      Removes a pmap from another.  This is used to implement shared segments.
   *      On the current PPC processors, this is limited to segment (256MB) aligned
   *      segment sized ranges.
   */
  
  kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr) {
                          
          unsigned int tstamp, i, mycpu;
          addr64_t nextva;
          spl_t s;
          mapping_t *mp;
                  
          s = splhigh();                                                                          /* Make sure interruptions are disabled */
  
          mp = mapping_find(grand, vaddr, &nextva, 0);            /* Find the nested map */
  
          if(((unsigned int)mp & mapRetCode) != mapRtOK) {        /* See if it was even nested */
                  panic("pmap_unnest: Attempt to unnest an unnested segment - va = %016llX\n", vaddr);
          }
  
          if((mp->mpFlags & mpType) != mpNest) {                          /* Did we find something other than a nest? */
                  panic("pmap_unnest: Attempt to unnest something that is not a nest - va = %016llX\n", vaddr);
          }
          
          if(mp->mpVAddr != vaddr) {                                                      /* Make sure the address is the same */
                  panic("pmap_unnest: Attempt to unnest something that is not at start of nest - va = %016llX\n", vaddr);
          }
  
          (void)hw_atomic_and(&mp->mpFlags, ~mpPerm);                     /* Show that this mapping is now removable */
          
          mapping_drop_busy(mp);                                                          /* Go ahead and release the mapping now */
  
          splx(s);                                                                                        /* Restore 'rupts */
                  
          (void)mapping_remove(grand, vaddr);                                     /* Toss the nested pmap mapping */
          
          invalidateSegs(grand);                                                          /* Invalidate the pmap segment cache */
          
  /*
   *      Note that the following will force the segment registers to be reloaded 
   *      on all processors (if they are using the pmap we just changed) before returning.
   *
   *      This is needed.  The reason is that until the segment register is 
   *      reloaded, another thread in the same task on a different processor will
   *      be able to access memory that it isn't allowed to anymore.  That can happen
   *      because access to the subordinate pmap is being removed, but the pmap is still
   *      valid.
   *
   *      Note that we only kick the other processor if we see that it was using the pmap while we
   *      were changing it.
   */
  
  
          for(i=0; i < real_ncpus; i++) {                                         /* Cycle through processors */
                  disable_preemption();
                  mycpu = cpu_number();                                                           /* Who am I? Am I just a dream? */
                  if((unsigned int)grand == PerProcTable[i].ppe_vaddr->ppUserPmapVirt) {  /* Is this guy using the changed pmap? */
                          
                          PerProcTable[i].ppe_vaddr->ppInvSeg = 1;        /* Show that we need to invalidate the segments */
                          
                          if(i != mycpu) {
                  
                                  tstamp = PerProcTable[i].ppe_vaddr->ruptStamp[1];               /* Save the processor's last interrupt time stamp */
                                  if(cpu_signal(i, SIGPcpureq, CPRQsegload, 0) == KERN_SUCCESS) { /* Make sure we see the pmap change */
                                          if(!hw_cpu_wcng(&PerProcTable[i].ppe_vaddr->ruptStamp[1], tstamp, LockTimeOut)) {       /* Wait for the other processors to enter debug */
                                                  panic("pmap_unnest: Other processor (%d) did not see interruption request\n", i);
                                          }
                                  }
                          }
                  }
                  enable_preemption();
          }
  
          return KERN_SUCCESS;                                                            /* Bye, bye, butterfly... */
  }
  
  
  /*
   *      void MapUserMemoryWindowInit(void)
   *
   *      Initialize anything we need to in order to map user address space slices into
   *      the kernel.  Primarily used for copy in/out.
   *
   *      Currently we only support one 512MB slot for this purpose.  There are two special
   *      mappings defined for the purpose: the special pmap nest, and linkage mapping.
   *
   *      The special pmap nest (which is allocated in this function) is used as a place holder
   *      in the kernel's pmap search list. It is 512MB long and covers the address range
   *      starting at lgUMWvaddr.  It points to no actual memory and when the fault handler 
   *      hits in it, it knows to look in the per_proc and start using the linkage
   *      mapping contained therin.
   *
   *      The linkage mapping is used to glue the user address space slice into the 
   *      kernel.  It contains the relocation information used to transform the faulting
   *      kernel address into the user address space.  It also provides the link to the
   *      user's pmap.  This is pointed to by the per_proc and is switched in and out
   *      whenever there is a context switch.
   *
   */
  
  void MapUserMemoryWindowInit(void) {
                  
          addr64_t colladdr;
          int nlists;
          mapping_t *mp;
          
          nlists = mapSetLists(kernel_pmap);                                      /* Set number of lists this will be on */
          
          mp = mapping_alloc(nlists);                                                     /* Get a spare mapping block */
  
          mp->mpFlags = 0x01000000 | mpLinkage | mpPerm | mpBSu | nlists; /* Make this a permanent nested pmap with a 32MB basic size unit */
                                                                                                                  /* Set the flags. Make sure busy count is 1 */
          mp->mpSpace = kernel_pmap->space;                                       /* Set the address space/pmap lookup ID */
          mp->u.mpBSize = 15;                                                                     /* Set the size to 2 segments in 32MB chunks - 1 */
          mp->mpPte = 0;                                                                          /* Means nothing */
          mp->mpPAddr = 0;                                                                        /* Means nothing */
          mp->mpVAddr = lowGlo.lgUMWvaddr;                                        /* Set the address range we cover */
          mp->mpNestReloc = 0;                                                            /* Means nothing */
          
          colladdr = hw_add_map(kernel_pmap, mp);                         /* Go add the mapping to the pmap */
          
          if(colladdr) {                                                                          /* Did it collide? */
                  panic("MapUserMemoryWindowInit: MapUserMemoryWindow range already mapped\n");
          }
          
          return;
  }
  
  /*
   *      addr64_t MapUserMemoryWindow(vm_map_t map, vm_offset_t va, size)
   *
   *      map  = the vm_map that we are mapping into the kernel
   *      va = start of the address range we are mapping
   *      Note that we do not test validty, we chose to trust our fellows...
   *
   *      Maps a 512M slice of a user address space into a predefined kernel range
   *      on a per-thread basis. We map only the first 256M segment, allowing the
   *  second 256M segment to fault in as needed. This allows our clients to access
   *  an arbitrarily aligned operand up to 256M in size.
   *
   *  In the future, the restriction of a predefined range may be loosened.
   *
   *      Builds the proper linkage map to map the user range
   *  We will round this down to the previous segment boundary and calculate
   *      the relocation to the kernel slot
   *
   *      We always make a segment table entry here if we need to.  This is mainly because of
   *      copyin/out and if we don't, there will be multiple segment faults for
   *      each system call.  I have seen upwards of 30000 per second.
   *
   *      We do check, however, to see if the slice is already mapped and if so,
   *      we just exit.  This is done for performance reasons.  It was found that 
   *      there was a considerable boost in copyin/out performance if we did not
   *      invalidate the segment at ReleaseUserAddressSpace time, so we dumped the
   *      restriction that you had to bracket MapUserMemoryWindow.  Further, there 
   *      is a yet further boost if you didn't need to map it each time.  The theory
   *      behind this is that many times copies are to or from the same segment and
   *      done multiple times within the same system call.  To take advantage of that,
   *      we check umwSpace and umwRelo to see if we've already got it.  
   *
   *      We also need to half-invalidate the slice when we context switch or go
   *      back to user state.  A half-invalidate does not clear the actual mapping,
   *      but it does force the MapUserMemoryWindow function to reload the segment
   *      register/SLBE.  If this is not done, we can end up some pretty severe
   *      performance penalties. If we map a slice, and the cached space/relocation is
   *      the same, we won't reload the segment registers.  Howver, since we ran someone else,
   *      our SR is cleared and we will take a fault.  This is reasonable if we block
   *      while copying (e.g., we took a page fault), but it is not reasonable when we 
   *      just start.  For this reason, we half-invalidate to make sure that the SR is
   *      explicitly reloaded.
   *       
   *      Note that we do not go to the trouble of making a pmap segment cache
   *      entry for these guys because they are very short term -- 99.99% of the time
   *      they will be unmapped before the next context switch.
   *
   */
  
  addr64_t MapUserMemoryWindow(
          vm_map_t map,
          addr64_t va) {
                  
          addr64_t baddrs, reladd;
          thread_t thread;
          mapping_t *mp;
          
          baddrs = va & 0xFFFFFFFFF0000000ULL;                            /* Isolate the segment */
          thread = current_thread();                                                      /* Remember our activation */
  
          reladd = baddrs - lowGlo.lgUMWvaddr;                            /* Get the relocation from user to kernel */
          
          if((thread->machine.umwSpace == map->pmap->space) && (thread->machine.umwRelo == reladd)) {     /* Already mapped? */
                  return ((va & 0x0FFFFFFFULL) | lowGlo.lgUMWvaddr);      /* Pass back the kernel address we are to use */
          }
  
          disable_preemption();                                                           /* Don't move... */     
          
          mp = (mapping_t *)&(getPerProc()->ppUMWmp);                     /* Make up for C */
          thread->machine.umwRelo = reladd;                                       /* Relocation from user to kernel */
          mp->mpNestReloc = reladd;                                                       /* Relocation from user to kernel */
          
          thread->machine.umwSpace = map->pmap->space;            /* Set the address space/pmap lookup ID */
          mp->mpSpace = map->pmap->space;                                         /* Set the address space/pmap lookup ID */
          
  /*
   *      Here we make an assumption that we are going to be using the base pmap's address space.
   *      If we are wrong, and that would be very, very, very rare, the fault handler will fix us up.
   */ 
  
          hw_map_seg(map->pmap,  lowGlo.lgUMWvaddr, baddrs);      /* Make the entry for the first segment */
  
          enable_preemption();                                                            /* Let's move */
          return ((va & 0x0FFFFFFFULL) | lowGlo.lgUMWvaddr);      /* Pass back the kernel address we are to use */
  }
  
  
  /*
   *      kern_return_t pmap_boot_map(size)
   *
   *      size   = size of virtual address range to be mapped
   *
   *      This function is used to assign a range of virtual addresses before VM in 
   *      initialized.  It starts at VM_MAX_KERNEL_ADDRESS and works downward.
   *      The variable vm_last_addr contains the current highest possible VM
   *      assignable address.  It is a panic to attempt to call this after VM has
   *      started up.  The only problem is, is that we may not have the serial or
   *      framebuffer mapped, so we'll never know we died.........
   */
  
  vm_offset_t pmap_boot_map(vm_size_t size) {
                          
          if(kernel_map != VM_MAP_NULL) {                         /* Has VM already started? */
                  panic("pmap_boot_map: VM started\n");
          }
          
          size = round_page(size);                                        /* Make sure this is in pages */
          vm_last_addr = vm_last_addr - size;                     /* Allocate the memory */
          return (vm_last_addr + 1);                                      /* Return the vaddr we just allocated */
  
  }
  
  
  /*
   *      void pmap_init_sharedpage(void);
   *
   *      Hack map for the 64-bit commpage
   */
  
  void pmap_init_sharedpage(vm_offset_t cpg){
          
          addr64_t cva, cpoff;
          ppnum_t cpphys;
          
          sharedPmap = pmap_create(0);                            /* Get a pmap to hold the common segment */
          if(!sharedPmap) {                                                       /* Check for errors */
                  panic("pmap_init_sharedpage: couldn't make sharedPmap\n");
          }
  
          for(cpoff = 0; cpoff < _COMM_PAGE_AREA_USED; cpoff += 4096) {   /* Step along now */
          
                  cpphys = pmap_find_phys(kernel_pmap, (addr64_t)cpg + cpoff);
                  if(!cpphys) {
                          panic("pmap_init_sharedpage: compage %08X not mapped in kernel\n", cpg + cpoff);
                  }
                  
                  cva = mapping_make(sharedPmap, (addr64_t)((uint32_t)_COMM_PAGE_BASE_ADDRESS) + cpoff,
                          cpphys, mmFlgPerm, 1, VM_PROT_READ);    /* Map the page read only */
                  if(cva) {                                                               /* Check for errors */
                          panic("pmap_init_sharedpage: couldn't map commpage page - cva = %016llX\n", cva);
                  }
          
          }
                  
          return;
  }
  
  
  /*
   *      void pmap_map_sharedpage(pmap_t pmap);
   *
   *      Maps the last segment in a 64-bit address space
   *
   *      
   */
  
  void pmap_map_sharedpage(task_t task, pmap_t pmap){
          
          kern_return_t ret;
  
          if(task_has_64BitAddr(task) || _cpu_capabilities & k64Bit) {    /* Should we map the 64-bit page -1? */
                  ret = pmap_nest(pmap, sharedPmap, 0xFFFFFFFFF0000000ULL, 0x00000000F0000000ULL,
                          0x0000000010000000ULL);                         /* Nest the highest possible segment to map comm page */
                  if(ret != KERN_SUCCESS) {                               /* Did it work? */
                          panic("pmap_map_sharedpage: couldn't nest shared page - ret = %08X\n", ret);
                  }
          }
  
          return;
  }
  
  
  /*
   *      void pmap_unmap_sharedpage(pmap_t pmap);
   *
   *      Unmaps the last segment in a 64-bit address space
   *
   */
  
  void pmap_unmap_sharedpage(pmap_t pmap){
          
          kern_return_t ret;
          mapping_t *mp;
          boolean_t inter;
          int gotnest;
          addr64_t nextva;
  
          if(BootProcInfo.pf.Available & pf64Bit) {               /* Are we on a 64-bit machine? */
                  
                  inter  = ml_set_interrupts_enabled(FALSE);      /* Disable interruptions for now */
                  mp = hw_find_map(pmap, 0xFFFFFFFFF0000000ULL, &nextva); /* Find the mapping for this address */
                  if((unsigned int)mp == mapRtBadLk) {            /* Did we lock up ok? */
                          panic("pmap_unmap_sharedpage: mapping lock failure - rc = %08X, pmap = %08X\n", mp, pmap);      /* Die... */
                  }
                  
                  gotnest = 0;                                                            /* Assume nothing here */
                  if(mp) {
                          gotnest = ((mp->mpFlags & mpType) == mpNest);
                                                                                                          /* Remember if we have a nest here */
                          mapping_drop_busy(mp);                                  /* We have everything we need from the mapping */
                  }
                  ml_set_interrupts_enabled(inter);                       /* Put interrupts back to what they were */
                  
                  if(!gotnest) return;                                            /* Leave if there isn't any nesting here */
                  
                  ret = pmap_unnest(pmap, 0xFFFFFFFFF0000000ULL); /* Unnest the max 64-bit page */
                  
                  if(ret != KERN_SUCCESS) {                                       /* Did it work? */
                          panic("pmap_unmap_sharedpage: couldn't unnest shared page - ret = %08X\n", ret);
                  }
          }
          
          return;
  }
  
  
  /* temporary workaround */
  boolean_t
  coredumpok(
          __unused vm_map_t map,
          __unused vm_offset_t va)
  {
          return TRUE;
  }

Cache object: e1bc91c65e61319a3df76eb200f7474b