FreeBSD/Linux Kernel Cross Reference
sys/intel/pmap.c

     /*
      * Mach Operating System
      * Copyright (c) 1993,1992,1991,1990,1989,1988 Carnegie Mellon University
      * All Rights Reserved.
      * 
      * Permission to use, copy, modify and distribute this software and its
      * documentation is hereby granted, provided that both the copyright
      * notice and this permission notice appear in all copies of the
      * software, derivative works or modified versions, and any portions
     * thereof, and that both notices appear in supporting documentation.
     * 
     * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
     * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
     * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
     * 
     * Carnegie Mellon requests users of this software to return to
     * 
     *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
     *  School of Computer Science
     *  Carnegie Mellon University
     *  Pittsburgh PA 15213-3890
     * 
     * any improvements or extensions that they make and grant Carnegie Mellon
     * the rights to redistribute these changes.
     */
    /*
     * HISTORY
     * $Log:        pmap.c,v $
     * Revision 2.19  93/11/17  16:55:03  dbg
     *      Cleaned up lint.  ANSI-fied.
     *      [93/06/17            dbg]
     * 
     * Revision 2.18  93/01/14  17:32:25  danner
     *      Lock pmap_object!
     *      [92/09/22            dbg]
     * 
     * Revision 2.17  92/04/06  23:12:15  rpd
     *      It's time to let us map page 0.
     *      [92/04/03            rvb]
     * 
     * Revision 2.16  92/04/01  19:32:48  rpd
     *      Removed pmap_remove_attributes.
     *      [92/03/25            rpd]
     * 
     * Revision 2.15  92/01/14  16:44:01  rpd
     *      Removed pmap_list_resident_pages.
     *      [91/12/31            rpd]
     * 
     * Revision 2.14  91/12/10  16:32:15  jsb
     *      Fixes from Intel
     *      [91/12/10  15:51:38  jsb]
     * 
     * Revision 2.13  91/11/18  17:37:09  rvb
     *      Up morevm for NORMA.
     * 
     * Revision 2.12  91/08/28  11:13:08  jsb
     *      From Intel SSD: add data cache flush in INVALIDATE_TLB to work around
     *      some more subtle unknown bug in page table caching; allow user access
     *      to various bits of I/O space?
     *      [91/08/26  18:27:04  jsb]
     * 
     * Revision 2.11  91/06/17  15:45:52  jsb
     *      Fixed reference to XEOD_OFF_PH for i860ipsc dcm module.
     *      [91/06/17  10:43:40  jsb]
     * 
     * Revision 2.10  91/06/06  17:05:03  jsb
     *      Defined SPLVM, SPLX as null (vs. splvm, splx) in uniprocessor case.
     *      [91/05/13  17:12:34  jsb]
     * 
     * Revision 2.9  91/05/18  14:31:14  rpd
     *      Moved pmap_free_pages, pmap_next_page to a model-dependent file.
     *      [91/05/15            rpd]
     * 
     *      Make sure hole_start and hole_end are page-aligned.
     *      [91/05/01            rpd]
     * 
     *      Removed pmap_update.
     *      [91/04/12            rpd]
     * 
     *      Added inuse_ptepages_count.
     *      Added vm_page_fictitious_addr assertions.
     *      [91/04/10            rpd]
     *      Added check_simple_locks to pmap_expand.
     *      [91/03/31            rpd]
     *      Changed vm_page_init to vm_page_insert.
     *      Added pmap_free_pages, pmap_next_page, pmap_virtual_space.
     *      [91/03/25            rpd]
     * 
     * Revision 2.8  91/05/14  16:30:24  mrt
     *      Correcting copyright
     * 
     * Revision 2.7  91/05/08  12:46:31  dbg
     *      Add volatile declarations where needed.
     *      Move pmap_valid_page to model_dependent file.
     *      [91/04/26  14:41:31  dbg]
     * 
     * Revision 2.6  91/03/16  14:47:31  rpd
     *      Removed some incorrect (?) assertions.
     *      [91/03/13  14:18:51  rpd]
    * 
    *      Updated for new kmem_alloc interface.
    *      [91/03/03            rpd]
    *      Added continuation argument to VM_PAGE_WAIT.
    *      [91/02/05            rpd]
    * 
    * Revision 2.5  91/02/14  14:08:11  mrt
    *      Fixed pmap_expand to use vm_page_grab/VM_PAGE_WAIT.
    *      [91/01/12            rpd]
    * 
    * Revision 2.4  91/02/05  17:20:34  mrt
    *      Changed to new Mach copyright
    *      [91/01/31  18:17:35  mrt]
    * 
    * Revision 2.3  91/01/08  15:12:47  rpd
    *      Changed pmap_collect to ignore the kernel pmap.
    *      [91/01/03            rpd]
    * 
    * Revision 2.2  90/12/04  14:50:28  jsb
    *      First checkin (for intel directory).
    *      [90/12/03  21:54:31  jsb]
    * 
    * Revision 2.9  90/11/26  14:48:44  rvb
    *      Slight error in pmap_valid_page.  Pages > last_addr
    *      must be invalid. (They are probably device buffers.)
    *      [90/11/23  10:00:56  rvb]
    * 
    * Revision 2.8  90/11/24  15:14:47  jsb
    *      Replaced "0x1000" in pmap_valid_page with "first_addr".
    *      [90/11/24  11:49:04  jsb]
    * 
    * Revision 2.7  90/11/05  14:27:27  rpd
    *      Replace (va < vm_first_phys || va > vm_last_phys) with test
    *      using valid page.  Otherwise, video buffer memory is treated as
    *      valid memory and setting dirty bits leads to disasterous results.
    *      [90/11/05            rvb]
    * 
    *      Define pmap_valid_page: [0x1000..cnvmem * 1024) and
    *                              [first_avail..)
    *      as useable memory
    *      [90/09/05            rvb]
    * 
    * Revision 2.6  90/09/09  14:31:39  rpd
    *      Use decl_simple_lock_data.
    *      [90/08/30            rpd]
    * 
    * Revision 2.5  90/08/06  15:07:05  rwd
    *      Fix bugs in pmap_remove, pmap_protect, phys_attribute routines.
    *      Allocate pte pages directly from vm_resident page list, via a
    *      pmap_object.
    *      [90/07/17            dbg]
    * 
    * Revision 2.4  90/06/19  22:57:46  rpd
    *      Made MOREVM a variable; increased to 28 meg.
    *      Commented out pte_to_phys assertions.
    *      [90/06/04            rpd]
    * 
    * Revision 2.3  90/06/02  14:48:40  rpd
    *      Added dummy pmap_list_resident_pages, under MACH_VM_DEBUG.
    *      [90/05/31            rpd]
    * 
    * Revision 2.2  90/05/03  15:37:04  dbg
    *      Define separate Write and User bits instead of protection codes.
    *      Write-protect kernel data by invalidating it; the 386 ignores
    *      write permission in supervisor mode.
    *      [90/03/25            dbg]
    * 
    *      Fix pmap_collect to look for VA that maps page table page.
    *      Since page table pages are allocated with kmem_alloc, their
    *      virtual and physical addresses are not necessarily the same.
    *      Rewrite pmap_remove to skip address range when PDE is invalid.
    *      Combine pmap_remove_all and pmap_copy_on_write into pmap_page_protect.
    *      Add reference bits.
    *      [90/03/21            dbg]
    * 
    *      Fix for pure kernel.  kpde and kptes are dynamically allocated
    *      by assembly code.  Reverse CHIPBUG test (what was this, Bob?)
    *      [90/02/14            dbg]
    * 
    * Revision 1.8.1.3  89/12/28  12:43:18  rvb
    *      v_avail gets phystokv(av_start), in case esym != end.
    *      [89/12/26            rvb]
    * 
    * Revision 1.8.1.2  89/12/21  17:59:15  rvb
    *      Revision 1.11  89/11/27  22:54:27  kupfer
    *      kernacc() moved here from locore (from Lance).
    * 
    *      Revision 1.10  89/10/24  13:31:38  lance
    *      Eliminate the boot-time `pause that refreshes'
    * 
    * Revision 1.8  89/09/20  17:26:47  rvb
    *      The OLIVETTI CACHE bug strikes again.  I am leaving this code in
    *      as it for now so we can sync up.  BUT all this stuff is going to
    *      be on a run time switch or a ifdef real soon.
    *      [89/09/20            rvb]
    * 
    * Revision 1.7  89/07/17  10:38:18  rvb
    *      pmap_map_bd now flushes the tlb with a call to pmap_update.
    *      [Lance Berc]
    * 
    * Revision 1.6  89/04/05  12:59:14  rvb
    *      Can not use zone anymore for directory, since alignment is not
    *      guaranteed.  Besides the directory is a page.
    *      [89/03/30            rvb]
    * 
    *      Move extern out of function scope for gcc.
    *      [89/03/04            rvb]
    * 
    * Revision 1.5  89/03/09  20:03:25  rpd
    *      More cleanup.
    * 
    * Revision 1.4  89/02/26  12:33:06  gm0w
    *      Changes for cleanup.
    * 
    * 31-Dec-88  Robert Baron (rvb) at Carnegie-Mellon University
    *      Derived from MACH2.0 vax release.
    *
    * 17-Jan-88  David Golub (dbg) at Carnegie-Mellon University
    *      Use cpus_idle, not the scheduler's cpu_idle, to determine when a
    *      cpu does not need to be interrupted.  The two are not
    *      synchronized.
    *
    */
   
   /*
    *      File:   pmap.c
    *      Author: Avadis Tevanian, Jr., Michael Wayne Young
    *      (These guys wrote the Vax version)
    *
    *      Physical Map management code for Intel i386, i486, and i860.
    *
    *      Manages physical address maps.
    *
    *      In addition to hardware address maps, this
    *      module is called upon to provide software-use-only
    *      maps which may or may not be stored in the same
    *      form as hardware maps.  These pseudo-maps are
    *      used to store intermediate results from copy
    *      operations to and from address spaces.
    *
    *      Since the information managed by this module is
    *      also stored by the logical address mapping module,
    *      this module may throw away valid virtual-to-physical
    *      mappings at almost any time.  However, invalidations
    *      of virtual-to-physical mappings must be done as
    *      requested.
    *
    *      In order to cope with hardware architectures which
    *      make virtual-to-physical map invalidates expensive,
    *      this module may delay invalidate or reduced protection
    *      operations until such time as they are actually
    *      necessary.  This module is given full information as
    *      to which processors are currently using which maps,
    *      and to when physical maps must be made correct.
    */
   
   #include <cpus.h>
   
   #include <mach/machine/vm_types.h>
   
   #include <mach/boolean.h>
   
   #include <kern/kern_io.h>
   #include <kern/memory.h>
   #include <kern/thread.h>
   #include <kern/zalloc.h>
   
   #include <kern/lock.h>
   
   #include <vm/pmap.h>
   #include <vm/vm_map.h>
   #include <vm/vm_kern.h>
   #include <mach/vm_param.h>
   #include <mach/vm_prot.h>
   #include <vm/vm_object.h>
   #include <vm/vm_page.h>
   #include <vm/vm_user.h>
   
   #include <mach/machine/vm_param.h>
   #include <machine/thread.h>
   #if     i860
   #include <i860ipsc/nodehw.h>
   #endif
   
   #ifdef  ORC
   #define OLIVETTICACHE   1
   #endif  /* ORC */
   
   #ifndef OLIVETTICACHE
   #define WRITE_PTE(pte_p, pte_entry)             *(pte_p) = (pte_entry);
   #define WRITE_PTE_FAST(pte_p, pte_entry)        *(pte_p) = (pte_entry);
   #else   /* OLIVETTICACHE */
   
   /* This gross kludgery is needed for Olivetti XP7 & XP9 boxes to get
    * around an apparent hardware bug. Other than at startup it doesn't
    * affect run-time performacne very much, so we leave it in for all
    * machines.
    */
   extern  unsigned        *pstart();
   #define CACHE_LINE      8
   #define CACHE_SIZE      512
   #define CACHE_PAGE      0x1000;
   
   #define WRITE_PTE(pte_p, pte_entry) { write_pte(pte_p, pte_entry); }
   
   void write_pte(
           pt_entry_t      *pte_p,
           pt_entry_t      pte_entry)
   {
           unsigned long count;
           volatile unsigned long hold, *addr1, *addr2;
   
           if ( pte_entry != *pte_p )
                   *pte_p = pte_entry;
           else {
                   /* This isn't necessarily the optimal algorithm */
                   addr1 = (unsigned long *)pstart;
                   for (count = 0; count < CACHE_SIZE; count++) {
                           addr2 = addr1 + CACHE_PAGE;
                           hold = *addr1;          /* clear cache bank - A - */
                           hold = *addr2;          /* clear cache bank - B - */
                           addr1 += CACHE_LINE;
                   }
           }
   }
   
   #define WRITE_PTE_FAST(pte_p, pte_entry)*pte_p = pte_entry;
   
   #endif  /* OLIVETTICACHE */
   
   /*
    *      Private data structures.
    */
   
   /*
    *      For each vm_page_t, there is a list of all currently
    *      valid virtual mappings of that page.  An entry is
    *      a pv_entry_t; the list is the pv_table.
    */
   
   typedef struct pv_entry {
           struct pv_entry *next;          /* next pv_entry */
           pmap_t          pmap;           /* pmap where mapping lies */
           vm_offset_t     va;             /* virtual address for mapping */
   } *pv_entry_t;
   
   #define PV_ENTRY_NULL   ((pv_entry_t) 0)
   
   pv_entry_t      pv_head_table;          /* array of entries, one per page */
   
   /*
    *      pv_list entries are kept on a list that can only be accessed
    *      with the pmap system locked (at SPLVM, not in the cpus_active set).
    *      The list is refilled from the pv_list_zone if it becomes empty.
    */
   pv_entry_t      pv_free_list;           /* free list at SPLVM */
   decl_simple_lock_data(, pv_free_list_lock)
   
   #define PV_ALLOC(pv_e) { \
           simple_lock(&pv_free_list_lock); \
           if ((pv_e = pv_free_list) != 0) { \
               pv_free_list = pv_e->next; \
           } \
           simple_unlock(&pv_free_list_lock); \
   }
   
   #define PV_FREE(pv_e) { \
           simple_lock(&pv_free_list_lock); \
           pv_e->next = pv_free_list; \
           pv_free_list = pv_e; \
           simple_unlock(&pv_free_list_lock); \
   }
   
   zone_t          pv_list_zone;           /* zone of pv_entry structures */
   
   /*
    *      Each entry in the pv_head_table is locked by a bit in the
    *      pv_lock_table.  The lock bits are accessed by the physical
    *      address of the page they lock.
    */
   
   char    *pv_lock_table;         /* pointer to array of bits */
   #define pv_lock_table_size(n)   (((n)+BYTE_SIZE-1)/BYTE_SIZE)
   
   /*
    *      First and last physical addresses that we maintain any information
    *      for.  Initialized to zero so that pmap operations done before
    *      pmap_init won't touch any non-existent structures.
    */
   vm_offset_t     vm_first_phys = (vm_offset_t) 0;
   vm_offset_t     vm_last_phys  = (vm_offset_t) 0;
   boolean_t       pmap_initialized = FALSE;/* Has pmap_init completed? */
   
   /*
    *      Index into pv_head table, its lock bits, and the modify/reference
    *      bits starting at vm_first_phys.
    */
   
   #define pa_index(pa)    (atop(pa - vm_first_phys))
   
   #define pai_to_pvh(pai)         (&pv_head_table[pai])
   #define lock_pvh_pai(pai)       (bit_lock(pai, pv_lock_table))
   #define unlock_pvh_pai(pai)     (bit_unlock(pai, pv_lock_table))
   
   /*
    *      Array of physical page attribites for managed pages.
    *      One byte per physical page.
    */
   char    *pmap_phys_attributes;
   
   /*
    *      Physical page attributes.  Copy bits from PTE definition.
    */
   #define PHYS_MODIFIED   INTEL_PTE_MOD   /* page modified */
   #define PHYS_REFERENCED INTEL_PTE_REF   /* page referenced */
   
   /*
    *      Amount of virtual memory mapped by one
    *      page-directory entry.
    */
   #define PDE_MAPPED_SIZE         (pdetova(1))
   
   /*
    *      We allocate page table pages directly from the VM system
    *      through this object.  It maps physical memory.
    */
   vm_object_t     pmap_object = VM_OBJECT_NULL;
   
   /*
    *      Locking and TLB invalidation
    */
   
   /*
    *      Locking Protocols:
    *
    *      There are two structures in the pmap module that need locking:
    *      the pmaps themselves, and the per-page pv_lists (which are locked
    *      by locking the pv_lock_table entry that corresponds to the pv_head
    *      for the list in question.)  Most routines want to lock a pmap and
    *      then do operations in it that require pv_list locking -- however
    *      pmap_remove_all and pmap_copy_on_write operate on a physical page
    *      basis and want to do the locking in the reverse order, i.e. lock
    *      a pv_list and then go through all the pmaps referenced by that list.
    *      To protect against deadlock between these two cases, the pmap_lock
    *      is used.  There are three different locking protocols as a result:
    *
    *  1.  pmap operations only (pmap_extract, pmap_access, ...)  Lock only
    *              the pmap.
    *
    *  2.  pmap-based operations (pmap_enter, pmap_remove, ...)  Get a read
    *              lock on the pmap_lock (shared read), then lock the pmap
    *              and finally the pv_lists as needed [i.e. pmap lock before
    *              pv_list lock.]
    *
    *  3.  pv_list-based operations (pmap_remove_all, pmap_copy_on_write, ...)
    *              Get a write lock on the pmap_lock (exclusive write); this
    *              also guaranteees exclusive access to the pv_lists.  Lock the
    *              pmaps as needed.
    *
    *      At no time may any routine hold more than one pmap lock or more than
    *      one pv_list lock.  Because interrupt level routines can allocate
    *      mbufs and cause pmap_enter's, the pmap_lock and the lock on the
    *      kernel_pmap can only be held at splvm.
    */
   
   #if     NCPUS > 1
   /*
    *      We raise the interrupt level to splvm, to block interprocessor
    *      interrupts during pmap operations.  We must take the CPU out of
    *      the cpus_active set while interrupts are blocked.
    */
   #define SPLVM(spl)      { \
           spl = splvm(); \
           i_bit_clear(cpu_number(), &cpus_active); \
   }
   
   #define SPLX(spl)       { \
           i_bit_set(cpu_number(), &cpus_active); \
           splx(spl); \
   }
   
   /*
    *      Lock on pmap system
    */
   lock_data_t     pmap_system_lock;
   
   #define PMAP_READ_LOCK(pmap, spl) { \
           SPLVM(spl); \
           lock_read(&pmap_system_lock); \
           simple_lock(&(pmap)->lock); \
   }
   
   #define PMAP_WRITE_LOCK(spl) { \
           SPLVM(spl); \
           lock_write(&pmap_system_lock); \
   }
   
   #define PMAP_READ_UNLOCK(pmap, spl) { \
           simple_unlock(&(pmap)->lock); \
           lock_read_done(&pmap_system_lock); \
           SPLX(spl); \
   }
   
   #define PMAP_WRITE_UNLOCK(spl) { \
           lock_write_done(&pmap_system_lock); \
           SPLX(spl); \
   }
   
   #define PMAP_WRITE_TO_READ_LOCK(pmap) { \
           simple_lock(&(pmap)->lock); \
           lock_write_to_read(&pmap_system_lock); \
   }
   
   #define LOCK_PVH(index)         (lock_pvh_pai(index))
   
   #define UNLOCK_PVH(index)       (unlock_pvh_pai(index))
   
   #define PMAP_UPDATE_TLBS(pmap, s, e) \
   { \
           cpu_set cpu_mask = 1 << cpu_number(); \
           cpu_set users; \
    \
           /* Since the pmap is locked, other updates are locked */ \
           /* out, and any pmap_activate has finished. */ \
    \
           /* find other cpus using the pmap */ \
           users = (pmap)->cpus_using & ~cpu_mask; \
           if (users) { \
               /* signal them, and wait for them to finish */ \
               /* using the pmap */ \
               signal_cpus(users, (pmap), (s), (e)); \
               while ((pmap)->cpus_using & cpus_active & ~cpu_mask) \
                   continue; \
           } \
    \
           /* invalidate our own TLB if pmap is in use */ \
           if ((pmap)->cpus_using & cpu_mask) { \
               INVALIDATE_TLB((s), (e)); \
           } \
   }
   
   #else   /* NCPUS > 1 */
   
   #define SPLVM(spl)
   #define SPLX(spl)
   
   #define PMAP_READ_LOCK(pmap, spl)       SPLVM(spl)
   #define PMAP_WRITE_LOCK(spl)            SPLVM(spl)
   #define PMAP_READ_UNLOCK(pmap, spl)     SPLX(spl)
   #define PMAP_WRITE_UNLOCK(spl)          SPLX(spl)
   #define PMAP_WRITE_TO_READ_LOCK(pmap)
   
   #define LOCK_PVH(index)
   #define UNLOCK_PVH(index)
   
   #define PMAP_UPDATE_TLBS(pmap, s, e) { \
           /* invalidate our own TLB if pmap is in use */ \
           if ((pmap)->cpus_using) { \
               INVALIDATE_TLB((s), (e)); \
           } \
   }
   
   #endif  /* NCPUS > 1 */
   
   #define MAX_TBIS_SIZE   32              /* > this -> TBIA */ /* XXX */
   
   #if     i860
   /* Do a data cache flush until we find the caching bug XXX prp */
   #define INVALIDATE_TLB(s, e) { \
           flush(); \
           flush_tlb(); \
   }
   #else   /* i860 */
   #define INVALIDATE_TLB(s, e) { \
           flush_tlb(); \
   }
   #endif  /* i860 */
   
   
   #if     NCPUS > 1
   /*
    *      Structures to keep track of pending TLB invalidations
    */
   
   #define UPDATE_LIST_SIZE        4
   
   struct pmap_update_item {
           pmap_t          pmap;           /* pmap to invalidate */
           vm_offset_t     start;          /* start address to invalidate */
           vm_offset_t     end;            /* end address to invalidate */
   } ;
   
   typedef struct pmap_update_item *pmap_update_item_t;
   
   /*
    *      List of pmap updates.  If the list overflows,
    *      the last entry is changed to invalidate all.
    */
   struct pmap_update_list {
           decl_simple_lock_data(, lock)
           int                     count;
           struct pmap_update_item item[UPDATE_LIST_SIZE];
   } ;
   typedef struct pmap_update_list *pmap_update_list_t;
   
   struct pmap_update_list cpu_update_list[NCPUS];
   
   /*
    *      List of cpus that are actively using mapped memory.  Any
    *      pmap update operation must wait for all cpus in this list.
    *      Update operations must still be queued to cpus not in this
    *      list.
    */
   cpu_set                 cpus_active;
   
   /*
    *      List of cpus that are idle, but still operating, and will want
    *      to see any kernel pmap updates when they become active.
    */
   cpu_set                 cpus_idle;
   
   /*
    *      Quick test for pmap update requests.
    */
   volatile boolean_t      cpu_update_needed[NCPUS];
   
   #endif  /* NCPUS > 1 */
   
   /*
    *      Other useful macros.
    */
   #define current_pmap()          (vm_map_pmap(current_thread()->task->map))
   #define pmap_in_use(pmap, cpu)  (((pmap)->cpus_using & (1 << (cpu))) != 0)
   
   struct pmap     kernel_pmap_store;
   pmap_t          kernel_pmap;
   
   struct zone     *pmap_zone;             /* zone of pmap structures */
   
   int             pmap_debug = 0;         /* flag for debugging prints */
   int             ptes_per_vm_page;       /* number of hardware ptes needed
                                              to map one VM page. */
   unsigned int    inuse_ptepages_count = 0;       /* debugging */
   
   extern char end;
   /*
    * Page directory for kernel.
    */
   pt_entry_t      *kpde = 0;      /* set by start.s - keep out of bss */
   #if i860
   extern pt_entry_t kpde_page;
   #endif
   
   #if     NCPUS > 1
   void signal_cpus(cpu_set, pmap_t, vm_offset_t, vm_offset_t);
                                           /* forward */
   #endif  /* NCPUS > 1 */
   
   #if     i860
   /*
    * Paging flag
    */
   int             paging_enabled = 0;
   #endif
   
   /*
    *      Given an offset and a map, compute the address of the
    *      pte.  If the address is invalid with respect to the map
    *      then PT_ENTRY_NULL is returned (and the map may need to grow).
    *
    *      This is only used internally.
    */
   pt_entry_t *pmap_pte(
           register pmap_t         pmap,
           register vm_offset_t    addr)
   {
   #if i860
           pt_entry_t              *ptp;
   #else
           register pt_entry_t     *ptp;
   #endif
           register pt_entry_t     pte;
   
           if (pmap->dirbase == 0)
                   return(PT_ENTRY_NULL);
           pte = pmap->dirbase[pdenum(addr)];
           if ((pte & INTEL_PTE_VALID) == 0)
                   return(PT_ENTRY_NULL);
           ptp = (pt_entry_t *)ptetokv(pte);
           return &ptp[ptenum(addr)];
   
   }
   
   #define pmap_pde(pmap, addr) (&(pmap)->dirbase[pdenum(addr)])
   
   #define DEBUG_PTE_PAGE  0
   
   #if     DEBUG_PTE_PAGE
   void ptep_check(
           ptep_t  ptep)
   {
           register pt_entry_t     *pte, *epte;
           int                     ctu, ctw;
   
           /* check the use and wired counts */
           if (ptep == PTE_PAGE_NULL)
                   return;
           pte = pmap_pte(ptep->pmap, ptep->va);
           epte = pte + INTEL_PGBYTES/sizeof(pt_entry_t);
           ctu = 0;
           ctw = 0;
           while (pte < epte) {
                   if (pte->pfn != 0) {
                           ctu++;
                           if (pte->wired)
                                   ctw++;
                   }
                   pte += ptes_per_vm_page;
           }
   
           if (ctu != ptep->use_count || ctw != ptep->wired_count) {
                   printf("use %d wired %d - actual use %d wired %d\n",
                           ptep->use_count, ptep->wired_count, ctu, ctw);
                   panic("pte count");
           }
   }
   #endif  /* DEBUG_PTE_PAGE */
   
   /*
    *      Map memory at initialization.  The physical addresses being
    *      mapped are not managed and are never unmapped.
    *
    *      For now, VM is already on, we only need to map the
    *      specified memory.
    */
   vm_offset_t pmap_map(
           register vm_offset_t    virt,
           register vm_offset_t    start,
           register vm_offset_t    end,
           register int            prot)
   {
           register int            ps;
   
           ps = PAGE_SIZE;
           while (start < end) {
                   pmap_enter(kernel_pmap, virt, start, prot, FALSE);
                   virt += ps;
                   start += ps;
           }
           return virt;
   }
   
   /*
    *      Back-door routine for mapping kernel VM at initialization.  
    *      Useful for mapping memory outside the range
    *      [vm_first_phys, vm_last_phys) (i.e., devices).
    *      Otherwise like pmap_map.
   #if     i860
    *      Sets no-cache bit.
   #endif
    */
   vm_offset_t pmap_map_bd(
           register vm_offset_t    virt,
           register vm_offset_t    start,
           register vm_offset_t    end,
           vm_prot_t               prot)
   {
           register pt_entry_t     template;
           register pt_entry_t     *pte;
   
           template = pa_to_pte(start)
   #if     i860
                   | INTEL_PTE_NCACHE
   #endif
                   | INTEL_PTE_VALID;
           if (prot & VM_PROT_WRITE)
               template |= INTEL_PTE_WRITE;
   
           while (start < end) {
                   pte = pmap_pte(kernel_pmap, virt);
                   if (pte == PT_ENTRY_NULL)
                           panic("pmap_map_bd: Invalid kernel address\n");
                   WRITE_PTE_FAST(pte, template)
                   pte_increment_pa(template);
                   virt += PAGE_SIZE;
                   start += PAGE_SIZE;
           }
           return virt;
   }
   
   extern int              cnvmem;
   extern  char            *first_avail;
   extern  vm_offset_t     virtual_avail, virtual_end;
   extern  vm_offset_t     avail_start, avail_end;
   
   /*
    *      Bootstrap the system enough to run with virtual memory.
    *      Map the kernel's code and data, and allocate the system page table.
    *      Called with mapping OFF.  Page_size must already be set.
    *
    *      Parameters:
    *      load_start:     PA where kernel was loaded
    *      avail_start     PA of first available physical page -
    *                         after kernel page tables
    *      avail_end       PA of last available physical page
    *      virtual_avail   VA of first available page -
    *                         after kernel page tables
    *      virtual_end     VA of last available page -
    *                         end of kernel address space
    *
    *      &start_text     start of kernel text
    *      &etext          end of kernel text
    */
   
   vm_size_t morevm = 40 * 1024 * 1024;    /* VM space for kernel map */
   
   void pmap_bootstrap(
           vm_offset_t     load_start)
   {
           vm_offset_t     va, tva;
           pt_entry_t      template;
           pt_entry_t      *pde, *pte, *ptend;
   #if     i860
           vm_offset_t     sva;
           pt_entry_t      *pt_pte, *tpt;
           pt_entry_t      *ppde, *ppte;
   
           /*
            * Mapping is turned OFF, we must reference only physical addresses.
            * The load image of the system is to be mapped 1-1 physical = virtual.
            *
            * This code will only work if VM_MIN_KERNEL_ADDRESS
            * equals PHYS_RAM_ADDRESS.
            */
   #endif
   
           /*
            *      Set ptes_per_vm_page for general use.
            */
           ptes_per_vm_page = page_size / INTEL_PGBYTES;
   
           /*
            *      The kernel's pmap is statically allocated so we don't
            *      have to use pmap_create, which is unlikely to work
            *      correctly at this part of the boot sequence.
            */
   
           kernel_pmap = &kernel_pmap_store;
   
   #if     NCPUS > 1
           lock_init(&pmap_system_lock, FALSE);    /* NOT a sleep lock */
   #endif  /* NCPUS > 1 */
   
           simple_lock_init(&kernel_pmap->lock);
   
           kernel_pmap->ref_count = 1;
   
           /*
            *      The kernel page directory has been allocated;
            *      its virtual address is in kpde.
            *
   #if     i860
            *      No kernel page table pages have been allocated
   #else
            *      Enough kernel page table pages have been allocated
   #endif
            *      to map low system memory, kernel text, kernel data/bss,
            *      kdb's symbols, and the page directory and page tables.
            *
            *      No other physical memory has been allocated.
            */
   #if     i860
           kpde = &kpde_page;
           kernel_pmap->dirbase = kpde;
   #endif
   
           /*
            * Start mapping virtual memory to physical memory, 1-1,
   #if     i860
            * from load point to end of memory,
            * virtual = physical.
   #else
            * at end of mapped memory.
   #endif
            */
           virtual_avail = phystokv(avail_start);
           virtual_end = phystokv(avail_end);
   
   #if     i860
           bzero(kpde, INTEL_PGBYTES);
   #endif
           pde = kpde;
   #if     i860
           pde += pdenum(load_start);
           pte = 0; ptend = 0;
   #else
           pde += pdenum(virtual_avail);
           if (pte_to_pa(*pde) == 0) {
               /* This pte has not been allocated */
               pte = 0; ptend = 0;
           }
           else {
               pte = (pt_entry_t *)ptetokv(*pde);
                                                   /* first pte of page */
               ptend = pte+NPTES;                  /* last pte of page */
               pte += ptenum(virtual_avail);               /* point to pte that
                                                      maps first avail VA */
               pde++;      /* point pde to first empty slot */
           }
   #endif
   
   #if     i860
           template = pa_to_pte(load_start)
   #else
           template = pa_to_pte(avail_start)
   #endif
                   | INTEL_PTE_VALID | INTEL_PTE_WRITE;
   
   #if     i860
           tva = virtual_end;
           sva = virtual_avail;
           for (va = load_start; va < tva; va += INTEL_PGBYTES) {
   #else
           for (va = virtual_avail; va < virtual_end; va += INTEL_PGBYTES) {
   #endif
               if (pte >= ptend) {
                   pte = (pt_entry_t *)virtual_avail;
                   ptend = pte + NPTES;
                   virtual_avail = (vm_offset_t)ptend;
   #if     i860
                   *pde = pa_to_pte((vm_offset_t)pte)
   #else
                   *pde = pa_to_pte((vm_offset_t)pte - VM_MIN_KERNEL_ADDRESS)
   #endif
                           | INTEL_PTE_VALID
                           | INTEL_PTE_WRITE;
                   pde++;
               }
               WRITE_PTE_FAST(pte, template)
               pte++;
               pte_increment_pa(template);
           }
   #if     i860
           /* kvtophys should now work in phys range */
   
           /*
            * Mark page table pages non-cacheable
            */
   
           pt_pte = (pt_entry_t *)pte_to_pa(*(kpde + pdenum(sva))) + ptenum(sva);
   
           for (va = load_start; va < tva; va += INTEL_PGBYTES*NPTES) {
                   /* Mark page table non-cacheable */
                   *pt_pte |= INTEL_PTE_NCACHE;
                   pt_pte++;
           }
   
           /*
            * Map I/O space
            */
   
           ppde = kpde;
           ppde += pdenum(IO_BASE);
   
           if (pte_to_pa(*ppde) == 0) {
                   /* This pte has not been allocated */
                   ppte = (pt_entry_t *)kvtophys(virtual_avail);
                   ptend = ppte + NPTES;
                   virtual_avail = phystokv((vm_offset_t)ptend);
                   *ppde = pa_to_pte((vm_offset_t)ppte)
                           | INTEL_PTE_VALID
                           | INTEL_PTE_WRITE;
                   pte = ptend;
   
                   /* Mark page table non-cacheable */
                   *pt_pte |= INTEL_PTE_NCACHE;
                   pt_pte++;
   
                   bzero(ppte, INTEL_PGBYTES);
           } else {
                   ppte = (pt_entry_t *)(*ppde);   /* first pte of page */
           }
           *ppde |= INTEL_PTE_USER;
   
   
           WRITE_PTE(ppte + ptenum(FIFO_ADDR),
                     pa_to_pte(FIFO_ADDR_PH)
                   | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_NCACHE);
   
           WRITE_PTE(ppte + ptenum(FIFO_ADDR + XEOD_OFF),
                     pa_to_pte(FIFO_ADDR_PH + XEOD_OFF_PH)
                   | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_NCACHE);
   
   /* XXX Allowed user access to control reg - cfj */
           WRITE_PTE(ppte + ptenum(CSR_ADDR),
                     pa_to_pte(CSR_ADDR_PH)
                   | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_NCACHE | INTEL_PTE_USER);
   
   /* XXX Allowed user access to perf reg - cfj */
           WRITE_PTE(ppte + ptenum(PERFCNT_ADDR),
                    pa_to_pte(PERFCNT_ADDR_PH)
                  | INTEL_PTE_VALID | INTEL_PTE_USER | INTEL_PTE_NCACHE | INTEL_PTE_USER);
  
          WRITE_PTE(ppte + ptenum(UART_ADDR),
                    pa_to_pte(UART_ADDR_PH)
                  | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_NCACHE);
  
          WRITE_PTE(ppte + ptenum(0xFFFFF000),
                    pa_to_pte(avail_end)
                  | INTEL_PTE_VALID | INTEL_PTE_WRITE);
          avail_start = kvtophys(virtual_avail);
  #else
          avail_start = virtual_avail - VM_MIN_KERNEL_ADDRESS;
  #endif
  
  /*
   *      startup requires additional virtual memory (for tables, buffers, 
   *      etc.).  The kd driver may also require some of that memory to
   *      access the graphics board.
   *
   */
          *(int *)&template = 0;
          virtual_end += morevm;
          for (tva = va; tva < virtual_end; tva += INTEL_PGBYTES) {
              if (pte >= ptend) {
  #if i860
                  pte = (pt_entry_t *)kvtophys(virtual_avail);
  #else
                  pte = (pt_entry_t *)virtual_avail;
  #endif
                  ptend = pte + NPTES;
  #if i860
                  virtual_avail = phystokv((vm_offset_t)ptend);
  #else
                  virtual_avail = (vm_offset_t)ptend;
  #endif
                  avail_start += INTEL_PGBYTES;
  #if i860
                  *pde = pa_to_pte((vm_offset_t)pte)
  #else
                  *pde = pa_to_pte((vm_offset_t)pte - VM_MIN_KERNEL_ADDRESS)
  #endif
                          | INTEL_PTE_VALID
                          | INTEL_PTE_WRITE;
                  pde++;
  #if i860
                  /* Mark page table non-cacheable */
                  *pt_pte |= INTEL_PTE_NCACHE;
                  pt_pte++;
  #endif
              }
              WRITE_PTE_FAST(pte, template)
              pte++;
          }
          virtual_avail = va;
  /*
   *      c.f. comment above
   *
   */
          virtual_end = va + morevm;
          while (pte < ptend)
              *pte++ = 0;
  /*
   *      invalidate virtual addresses at 0
   */
          kpde[0] = 0;
  #if i860
  #else
          kernel_pmap->dirbase = kpde;
  #endif
          printf("Kernel virtual space from 0x%x to 0x%x.\n",
  #if i860
                          sva, virtual_end);
  #else
                          VM_MIN_KERNEL_ADDRESS, virtual_end);
  #endif
          printf("Available physical space from 0x%x to 0x%x\n",
                          avail_start, avail_end);
  #if i860
  /*
   *      Turn on mapping
   */
  
          flush_and_ctxsw(kernel_pmap->dirbase);
          paging_enabled = 1;
  
          printf("Paging enabled.\n");
  #endif
  }
  
  void pmap_virtual_space(
          vm_offset_t *startp,
          vm_offset_t *endp)
  {
          *startp = virtual_avail;
          *endp = virtual_end;
  }
  
  /*
   *      Initialize the pmap module.
   *      Called by vm_init, to initialize any structures that the pmap
   *      system needs to map virtual memory.
   */
  void pmap_init(void)
  {
          register long           npages;
          vm_offset_t             addr;
          register vm_size_t      s;
  #if     NCPUS > 1
          int                     i;
  #endif
  
          /*
           *      Allocate memory for the pv_head_table and its lock bits,
           *      the modify bit array, and the pte_page table.
           */
  
          npages = atop(avail_end - avail_start);
          s = (vm_size_t) (sizeof(struct pv_entry) * npages
                                  + pv_lock_table_size(npages)
                                  + npages);
  
          s = round_page(s);
          if (kmem_alloc_wired(kernel_map, &addr, s) != KERN_SUCCESS)
                  panic("pmap_init");
          bzero((void *) addr, s);
  
          /*
           *      Allocate the structures first to preserve word-alignment.
           */
          pv_head_table = (pv_entry_t) addr;
          addr = (vm_offset_t) (pv_head_table + npages);
  
          pv_lock_table = (char *) addr;
          addr = (vm_offset_t) (pv_lock_table + pv_lock_table_size(npages));
  
          pmap_phys_attributes = (char *) addr;
  
          /*
           *      Create the zone of physical maps,
           *      and of the physical-to-virtual entries.
           */
          s = (vm_size_t) sizeof(struct pmap);
          pmap_zone = zinit(s, 400*s, 4096, FALSE, "pmap"); /* XXX */
          s = (vm_size_t) sizeof(struct pv_entry);
          pv_list_zone = zinit(s, 10000*s, 4096, FALSE, "pv_list"); /* XXX */
  
  #if     NCPUS > 1
          /*
           *      Set up the pmap request lists
           */
          for (i = 0; i < NCPUS; i++) {
              pmap_update_list_t  up = &cpu_update_list[i];
  
              simple_lock_init(&up->lock);
              up->count = 0;
          }
  #endif  /* NCPUS > 1 */
  
          /*
           *      Only now, when all of the data structures are allocated,
           *      can we set vm_first_phys and vm_last_phys.  If we set them
           *      too soon, the kmem_alloc_wired above will try to use these
           *      data structures and blow up.
           */
  
          vm_first_phys = avail_start;
          vm_last_phys = avail_end;
          pmap_initialized = TRUE;
  }
  
  extern boolean_t        pmap_valid_page(vm_offset_t pa);
  
  #define valid_page(x) (pmap_initialized && pmap_valid_page(x))
  
  boolean_t pmap_verify_free(
          vm_offset_t     phys)
  {
          pv_entry_t      pv_h;
          int             pai;
          boolean_t       result;
  #if     NCPUS > 1
          spl_t           spl;
  #endif
  
          assert(phys != vm_page_fictitious_addr);
          if (!pmap_initialized)
                  return TRUE;
  
          if (!pmap_valid_page(phys))
                  return FALSE;
  
          PMAP_WRITE_LOCK(spl);
  
          pai = pa_index(phys);
          pv_h = pai_to_pvh(pai);
  
          result = (pv_h->pmap == PMAP_NULL);
          PMAP_WRITE_UNLOCK(spl);
  
          return result;
  }
  
  /*
   *      Routine:        pmap_page_table_page_alloc
   *
   *      Allocates a new physical page to be used as a page-table page.
   *
   *      Must be called with the pmap system and the pmap unlocked,
   *      since these must be unlocked to use vm_page_grab.
   */
  vm_offset_t
  pmap_page_table_page_alloc(void)
  {
          register vm_page_t      m;
          register vm_offset_t    pa;
  
          check_simple_locks();
  
          /*
           *      We cannot allocate the pmap_object in pmap_init,
           *      because it is called before the zone package is up.
           *      Allocate it now if it is missing.
           */
          if (pmap_object == VM_OBJECT_NULL)
              pmap_object = vm_object_allocate(mem_size);
  
          /*
           *      Allocate a VM page for the level 2 page table entries.
           */
          while ((m = vm_page_grab()) == VM_PAGE_NULL)
                  VM_PAGE_WAIT(CONTINUE_NULL);
  
          /*
           *      Map the page to its physical address so that it
           *      can be found later.
           */
          pa = m->phys_addr;
          vm_object_lock(pmap_object);
          vm_page_insert(m, pmap_object, pa);
          vm_page_lock_queues();
          vm_page_wire(m);
          inuse_ptepages_count++;
          vm_page_unlock_queues();
          vm_object_unlock(pmap_object);
  
          /*
           *      Zero the page.
           */
          bzero((void *) phystokv(pa), PAGE_SIZE);
  
  #if     i860
          /*
           *      Mark the page table page(s) non-cacheable.
           */
          {
              int         i = ptes_per_vm_page;
              pt_entry_t  *pdp;
  
              pdp = pmap_pte(kernel_pmap, pa);
              do {
                  *pdp |= INTEL_PTE_NCACHE;
                  pdp++;
              } while (--i > 0);
          }
  #endif
          return pa;
  }
  
  /*
   *      Deallocate a page-table page.
   *      The page-table page must have all mappings removed,
   *      and be removed from its page directory.
   */
  void
  pmap_page_table_page_dealloc(
          vm_offset_t     pa)
  {
          vm_page_t       m;
  
          vm_object_lock(pmap_object);
          m = vm_page_lookup(pmap_object, pa);
          vm_page_lock_queues();
          vm_page_free(m);
          inuse_ptepages_count--;
          vm_page_unlock_queues();
          vm_object_unlock(pmap_object);
  }
  
  /*
   *      Create and return a physical map.
   *
   *      If the size specified for the map
   *      is zero, the map is an actual physical
   *      map, and may be referenced by the
   *      hardware.
   *
   *      If the size specified is non-zero,
   *      the map will be used in software only, and
   *      is bounded by that size.
   */
  pmap_t pmap_create(
          vm_size_t       size)
  {
          register pmap_t                 p;
          register pmap_statistics_t      stats;
  
          /*
           *      A software use-only map doesn't even need a map.
           */
  
          if (size != 0) {
                  return PMAP_NULL;
          }
  
  /*
   *      Allocate a pmap struct from the pmap_zone.  Then allocate
   *      the page descriptor table from the pd_zone.
   */
  
          p = (pmap_t) zalloc(pmap_zone);
          if (p == PMAP_NULL)
                  panic("pmap_create");
  
          if (kmem_alloc_wired(kernel_map,
                               (vm_offset_t *)&p->dirbase, INTEL_PGBYTES)
                                                          != KERN_SUCCESS)
                  panic("pmap_create");
  
          bcopy(kpde, p->dirbase, INTEL_PGBYTES);
          p->ref_count = 1;
  
          simple_lock_init(&p->lock);
          p->cpus_using = 0;
  
          /*
           *      Initialize statistics.
           */
  
          stats = &p->stats;
          stats->resident_count = 0;
          stats->wired_count = 0;
  
          return p;
  }
  
  /*
   *      Retire the given physical map from service.
   *      Should only be called if the map contains
   *      no valid mappings.
   */
  
  void pmap_destroy(
          register pmap_t p)
  {
          register pt_entry_t     *pdep;
          register vm_offset_t    pa;
          register int            c;
          register vm_page_t      m;
  #if     NCPUS > 1
          spl_t                   s;
  #endif
  
          if (p == PMAP_NULL)
                  return;
  
          SPLVM(s);
          simple_lock(&p->lock);
          c = --p->ref_count;
          simple_unlock(&p->lock);
          SPLX(s);
  
          if (c != 0) {
              return;     /* still in use */
          }
  
          /*
           *      Free the memory maps, then the
           *      pmap structure.
           */
          for (pdep = p->dirbase;
               pdep < &p->dirbase[pdenum(VM_MIN_KERNEL_ADDRESS)];
               pdep += ptes_per_vm_page) {
              if (*pdep & INTEL_PTE_VALID) {
                  pa = pte_to_pa(*pdep);
                  vm_object_lock(pmap_object);
                  m = vm_page_lookup(pmap_object, pa);
                  if (m == VM_PAGE_NULL)
                      panic("pmap_destroy: pte page not in object");
                  vm_page_lock_queues();
                  vm_page_free(m);
                  inuse_ptepages_count--;
                  vm_page_unlock_queues();
                  vm_object_unlock(pmap_object);
              }
          }
          kmem_free(kernel_map, (vm_offset_t) p->dirbase, INTEL_PGBYTES);
          zfree(pmap_zone, (vm_offset_t) p);
  }
  
  /*
   *      Add a reference to the specified pmap.
   */
  
  void pmap_reference(
          register pmap_t p)
  {
  #if     NCPUS > 1
          spl_t   s;
  #endif
  
          if (p != PMAP_NULL) {
                  SPLVM(s);
                  simple_lock(&p->lock);
                  p->ref_count++;
                  simple_unlock(&p->lock);
                  SPLX(s);
          }
  }
  
  /*
   *      Remove a range of hardware page-table entries.
   *      The entries given are the first (inclusive)
   *      and last (exclusive) entries for the VM pages.
   *      The virtual address is the va for the first pte.
   *
   *      The pmap must be locked.
   *      If the pmap is not the kernel pmap, the range must lie
   *      entirely within one pte-page.  This is NOT checked.
   *      Assumes that the pte-page exists.
   */
  
  /* static */
  void pmap_remove_range(
          pmap_t                  pmap,
          vm_offset_t             va,
          pt_entry_t              *spte,
          pt_entry_t              *epte)
  {
          register pt_entry_t     *cpte;
          int                     num_removed, num_unwired;
          int                     pai;
          vm_offset_t             pa;
  
  #if     DEBUG_PTE_PAGE
          if (pmap != kernel_pmap)
                  ptep_check(get_pte_page(spte));
  #endif  /* DEBUG_PTE_PAGE */
          num_removed = 0;
          num_unwired = 0;
  
          for (cpte = spte; cpte < epte;
               cpte += ptes_per_vm_page, va += PAGE_SIZE) {
  
              if (*cpte == 0)
                  continue;
              pa = pte_to_pa(*cpte);
  
              num_removed++;
              if (*cpte & INTEL_PTE_WIRED)
                  num_unwired++;
  
              if (!valid_page(pa)) {
  
                  /*
                   *      Outside range of managed physical memory.
                   *      Just remove the mappings.
                   */
                  register int    i = ptes_per_vm_page;
                  register pt_entry_t     *lpte = cpte;
                  do {
                      *lpte = 0;
                      lpte++;
                  } while (--i > 0);
                  continue;
              }
  
              pai = pa_index(pa);
              LOCK_PVH(pai);
  
              /*
               *  Get the modify and reference bits.
               */
              {
                  register int            i;
                  register pt_entry_t     *lpte;
  
                  i = ptes_per_vm_page;
                  lpte = cpte;
                  do {
                      pmap_phys_attributes[pai] |=
                          *lpte & (PHYS_MODIFIED|PHYS_REFERENCED);
                      *lpte = 0;
                      lpte++;
                  } while (--i > 0);
              }
  
              /*
               *  Remove the mapping from the pvlist for
               *  this physical page.
               */
              {
                  register pv_entry_t     pv_h, prev, cur;
  
                  pv_h = pai_to_pvh(pai);
                  if (pv_h->pmap == PMAP_NULL) {
                      panic("pmap_remove: null pv_list!");
                  }
                  if (pv_h->va == va && pv_h->pmap == pmap) {
                      /*
                       * Header is the pv_entry.  Copy the next one
                       * to header and free the next one (we cannot
                       * free the header)
                       */
                      cur = pv_h->next;
                      if (cur != PV_ENTRY_NULL) {
                          *pv_h = *cur;
                          PV_FREE(cur);
                      }
                      else {
                          pv_h->pmap = PMAP_NULL;
                      }
                  }
                  else {
                      cur = pv_h;
                      do {
                          prev = cur;
                          if ((cur = prev->next) == PV_ENTRY_NULL) {
                              panic("pmap-remove: mapping not in pv_list!");
                          }
                      } while (cur->va != va || cur->pmap != pmap);
                      prev->next = cur->next;
                      PV_FREE(cur);
                  }
                  UNLOCK_PVH(pai);
              }
          }
  
          /*
           *      Update the counts
           */
          pmap->stats.resident_count -= num_removed;
          pmap->stats.wired_count -= num_unwired;
  }
  
  /*
   *      Remove the given range of addresses
   *      from the specified map.
   *
   *      It is assumed that the start and end are properly
   *      rounded to the hardware page size.
   */
  
  void pmap_remove(
          pmap_t          map,
          vm_offset_t     s,
          vm_offset_t     e)
  {
          register pt_entry_t     *pde;
          register pt_entry_t     *spte, *epte;
          vm_offset_t             l;
  #if     NCPUS > 1
          spl_t                   spl;
  #endif
  
          if (map == PMAP_NULL)
                  return;
  
          PMAP_READ_LOCK(map, spl);
  
          /*
           *      Invalidate the translation buffer first
           */
          PMAP_UPDATE_TLBS(map, s, e);
  
          pde = pmap_pde(map, s);
          while (s < e) {
              l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1);
              if (l > e)
                  l = e;
              if (*pde & INTEL_PTE_VALID) {
                  spte = (pt_entry_t *)ptetokv(*pde);
                  spte = &spte[ptenum(s)];
                  epte = &spte[intel_btop(l-s)];
                  pmap_remove_range(map, s, spte, epte);
              }
              s = l;
              pde++;
          }
  
          PMAP_READ_UNLOCK(map, spl);
  }
  
  /*
   *      Routine:        pmap_page_protect
   *
   *      Function:
   *              Lower the permission for all mappings to a given
   *              page.
   */
  void pmap_page_protect(
          vm_offset_t     phys,
          vm_prot_t       prot)
  {
          pv_entry_t              pv_h, prev;
          register pv_entry_t     pv_e;
          register pt_entry_t     *pte;
          int                     pai;
          register pmap_t         pmap;
          boolean_t               remove;
  #if     NCPUS > 1
          spl_t                   spl;
  #endif
  
          assert(phys != vm_page_fictitious_addr);
          if (!valid_page(phys)) {
              /*
               *  Not a managed page.
               */
              return;
          }
  
          /*
           * Determine the new protection.
           */
          switch (prot) {
              case VM_PROT_READ:
              case VM_PROT_READ|VM_PROT_EXECUTE:
                  remove = FALSE;
                  break;
              case VM_PROT_ALL:
                  return; /* nothing to do */
              default:
                  remove = TRUE;
                  break;
          }
  
          /*
           *      Lock the pmap system first, since we will be changing
           *      several pmaps.
           */
  
          PMAP_WRITE_LOCK(spl);
  
          pai = pa_index(phys);
          pv_h = pai_to_pvh(pai);
  
          /*
           * Walk down PV list, changing or removing all mappings.
           * We do not have to lock the pv_list because we have
           * the entire pmap system locked.
           */
          if (pv_h->pmap != PMAP_NULL) {
  
              prev = pv_e = pv_h;
              do {
                  pmap = pv_e->pmap;
                  /*
                   * Lock the pmap to block pmap_extract and similar routines.
                   */
                  simple_lock(&pmap->lock);
  
                  {
                      register vm_offset_t va;
  
                      va = pv_e->va;
                      pte = pmap_pte(pmap, va);
  
                      /*
                       * Consistency checks.
                       */
                      /* assert(*pte & INTEL_PTE_VALID); XXX */
                      /* assert(pte_to_phys(*pte) == phys); */
  
                      /*
                       * Invalidate TLBs for all CPUs using this mapping.
                       */
                      PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
                  }
  
                  /*
                   * Remove the mapping if new protection is NONE
                   * or if write-protecting a kernel mapping.
                   */
                  if (remove || pmap == kernel_pmap) {
                      /*
                       * Remove the mapping, collecting any modify bits.
                       */
                      if (*pte & INTEL_PTE_WIRED)
                          panic("pmap_remove_all removing a wired page");
  
                      {
                          register int    i = ptes_per_vm_page;
  
                          do {
                              pmap_phys_attributes[pai] |=
                                  *pte & (PHYS_MODIFIED|PHYS_REFERENCED);
                              *pte++ = 0;
                          } while (--i > 0);
                      }
  
                      pmap->stats.resident_count--;
  
                      /*
                       * Remove the pv_entry.
                       */
                      if (pv_e == pv_h) {
                          /*
                           * Fix up head later.
                           */
                          pv_h->pmap = PMAP_NULL;
                      }
                      else {
                          /*
                           * Delete this entry.
                           */
                          prev->next = pv_e->next;
                          PV_FREE(pv_e);
                      }
                  }
                  else {
                      /*
                       * Write-protect.
                       */
                      register int i = ptes_per_vm_page;
  
                      do {
                          *pte &= ~INTEL_PTE_WRITE;
                          pte++;
                      } while (--i > 0);
  
                      /*
                       * Advance prev.
                       */
                      prev = pv_e;
                  }
  
                  simple_unlock(&pmap->lock);
  
              } while ((pv_e = prev->next) != PV_ENTRY_NULL);
  
              /*
               * If pv_head mapping was removed, fix it up.
               */
              if (pv_h->pmap == PMAP_NULL) {
                  pv_e = pv_h->next;
                  if (pv_e != PV_ENTRY_NULL) {
                      *pv_h = *pv_e;
                      PV_FREE(pv_e);
                  }
              }
          }
  
          PMAP_WRITE_UNLOCK(spl);
  }
  
  /*
   *      Set the physical protection on the
   *      specified range of this map as requested.
   *      Will not increase permissions.
   */
  void pmap_protect(
          pmap_t          map,
          vm_offset_t     s,
          vm_offset_t     e,
          vm_prot_t       prot)
  {
          register pt_entry_t     *pde;
          register pt_entry_t     *spte, *epte;
          vm_offset_t             l;
  #if     NCPUS > 1
          spl_t                   spl;
  #endif
  
          if (map == PMAP_NULL)
                  return;
  
          /*
           * Determine the new protection.
           */
          switch (prot) {
              case VM_PROT_READ:
              case VM_PROT_READ|VM_PROT_EXECUTE:
                  break;
              case VM_PROT_READ|VM_PROT_WRITE:
              case VM_PROT_ALL:
                  return; /* nothing to do */
              default:
                  pmap_remove(map, s, e);
                  return;
          }
  
          /*
           * If write-protecting in the kernel pmap,
           * remove the mappings; the i386 ignores
           * the write-permission bit in kernel mode.
           *
           * XXX should be #if'd for i386
           */
          if (map == kernel_pmap) {
              pmap_remove(map, s, e);
              return;
          }
  
          SPLVM(spl);
          simple_lock(&map->lock);
  
          /*
           *      Invalidate the translation buffer first
           */
          PMAP_UPDATE_TLBS(map, s, e);
  
          pde = pmap_pde(map, s);
          while (s < e) {
              l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1);
              if (l > e)
                  l = e;
              if (*pde & INTEL_PTE_VALID) {
                  spte = (pt_entry_t *)ptetokv(*pde);
                  spte = &spte[ptenum(s)];
                  epte = &spte[intel_btop(l-s)];
  
                  while (spte < epte) {
                      if (*spte & INTEL_PTE_VALID)
                          *spte &= ~INTEL_PTE_WRITE;
                      spte++;
                  }
              }
              s = l;
              pde++;
          }
  
          simple_unlock(&map->lock);
          SPLX(spl);
  }
  
  /*
   *      Insert the given physical page (p) at
   *      the specified virtual address (v) in the
   *      target physical map with the protection requested.
   *
   *      If specified, the page will be wired down, meaning
   *      that the related pte can not be reclaimed.
   *
   *      NB:  This is the only routine which MAY NOT lazy-evaluate
   *      or lose information.  That is, this routine must actually
   *      insert this page into the given map NOW.
   */
  void pmap_enter(
          register pmap_t         pmap,
          vm_offset_t             v,
          register vm_offset_t    pa,
          vm_prot_t               prot,
          boolean_t               wired)
  {
          register pt_entry_t     *pte;
          register pv_entry_t     pv_h;
          register int            i, pai;
          pv_entry_t              pv_e;
          pt_entry_t              template;
          vm_offset_t             old_pa;
  #if     NCPUS > 1
          spl_t                   spl;
  #endif
  
          assert(pa != vm_page_fictitious_addr);
  if (pmap_debug) printf("pmap(%x, %x)\n", v, pa);
          if (pmap == PMAP_NULL)
                  return;
  
          if (pmap == kernel_pmap && (prot & VM_PROT_WRITE) == 0
              && !wired /* hack for io_wire */ ) {
              /*
               *  Because the 386 ignores write protection in kernel mode,
               *  we cannot enter a read-only kernel mapping, and must
               *  remove an existing mapping if changing it.
               *
               *  XXX should be #if'd for i386
               */
              PMAP_READ_LOCK(pmap, spl);
  
              pte = pmap_pte(pmap, v);
              if (pte != PT_ENTRY_NULL && *pte != 0) {
                  /*
                   *      Invalidate the translation buffer,
                   *      then remove the mapping.
                   */
                  PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE);
                  pmap_remove_range(pmap, v, pte,
                                    pte + ptes_per_vm_page);
              }
              PMAP_READ_UNLOCK(pmap, spl);
              return;
          }
  
          /*
           *      Must allocate a new pvlist entry while we're unlocked;
           *      zalloc may cause pageout (which will lock the pmap system).
           *      If we determine we need a pvlist entry, we will unlock
           *      and allocate one.  Then we will retry, throughing away
           *      the allocated entry later (if we no longer need it).
           */
          pv_e = PV_ENTRY_NULL;
  Retry:
          PMAP_READ_LOCK(pmap, spl);
  
          /*
           *      Expand pmap to include this pte.  Assume that
           *      pmap is always expanded to include enough hardware
           *      pages to map one VM page.
           */
  
          while ((pte = pmap_pte(pmap, v)) == PT_ENTRY_NULL) {
              /*
               * Need to allocate a new page-table page.
               */
              vm_offset_t ptp;
              pt_entry_t  *pdp;
              int         i;
  
              if (pmap == kernel_pmap) {
                  /*
                   * Would have to enter the new page-table page in
                   * EVERY pmap.
                   */
                  panic("pmap_expand kernel pmap to %#x", v);
              }
  
              /*
               * Unlock the pmap and allocate a new page-table page.
               */
              PMAP_READ_UNLOCK(pmap, spl);
  
              ptp = pmap_page_table_page_alloc();
  
              /*
               * Re-lock the pmap and check that another thread has
               * not already allocated the page-table page.  If it
               * has, discard the new page-table page (and try
               * again to make sure).
               */
              PMAP_READ_LOCK(pmap, spl);
  
              if (pmap_pte(pmap, v) != PT_ENTRY_NULL) {
                  /*
                   * Oops...
                   */
                  PMAP_READ_UNLOCK(pmap, spl);
                  pmap_page_table_page_dealloc(ptp);
                  PMAP_READ_LOCK(pmap, spl);
                  continue;
              }
  
              /*
               * Enter the new page table page in the page directory.
               */
              i = ptes_per_vm_page;
              pdp = &pmap->dirbase[pdenum(v) & ~(i-1)];
              do {
                  *pdp = pa_to_pte(ptp) | INTEL_PTE_VALID
                                        | INTEL_PTE_USER
                                        | INTEL_PTE_WRITE;
                  pdp++;
                  ptp += INTEL_PGBYTES;
              } while (--i > 0);
  #if     i860
              /*
               * Flush the data cache.
               */
              flush();
  #endif  /* i860 */
  
              /*
               * Now, get the address of the page-table entry.
               */
              continue;
          }
  
          /*
           *      Special case if the physical page is already mapped
           *      at this address.
           */
          old_pa = pte_to_pa(*pte);
          if (*pte && old_pa == pa) {
              /*
               *  May be changing its wired attribute or protection
               */
                  
              if (wired && !(*pte & INTEL_PTE_WIRED))
                  pmap->stats.wired_count++;
              else if (!wired && (*pte & INTEL_PTE_WIRED))
                  pmap->stats.wired_count--;
  
              template = pa_to_pte(pa) | INTEL_PTE_VALID;
              if (pmap != kernel_pmap)
                  template |= INTEL_PTE_USER;
              if (prot & VM_PROT_WRITE)
                  template |= INTEL_PTE_WRITE;
              if (wired)
                  template |= INTEL_PTE_WIRED;
              PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE);
              i = ptes_per_vm_page;
              do {
                  if (*pte & INTEL_PTE_MOD)
                      template |= INTEL_PTE_MOD;
                  WRITE_PTE(pte, template)
                  pte++;
                  pte_increment_pa(template);
              } while (--i > 0);
          }
          else {
  
              /*
               *  Remove old mapping from the PV list if necessary.
               */
              if (*pte) {
                  /*
                   *      Invalidate the translation buffer,
                   *      then remove the mapping.
                   */
                  PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE);
  
                  /*
                   *      Don't free the pte page if removing last
                   *      mapping - we will immediately replace it.
                   */
                  pmap_remove_range(pmap, v, pte,
                                    pte + ptes_per_vm_page);
              }
  
              if (valid_page(pa)) {
  
                  /*
                   *      Enter the mapping in the PV list for this
                   *      physical page.
                   */
  
                  pai = pa_index(pa);
                  LOCK_PVH(pai);
                  pv_h = pai_to_pvh(pai);
  
                  if (pv_h->pmap == PMAP_NULL) {
                      /*
                       *  No mappings yet
                       */
                      pv_h->va = v;
                      pv_h->pmap = pmap;
                      pv_h->next = PV_ENTRY_NULL;
                  }
                  else {
  #if     DEBUG
                      {
                          /* check that this mapping is not already there */
                          pv_entry_t      e = pv_h;
                          while (e != PV_ENTRY_NULL) {
                              if (e->pmap == pmap && e->va == v)
                                  panic("pmap_enter: already in pv_list");
                              e = e->next;
                          }
                      }
  #endif  /* DEBUG */
                      
                      /*
                       *  Add new pv_entry after header.
                       */
                      if (pv_e == PV_ENTRY_NULL) {
                          PV_ALLOC(pv_e);
                          if (pv_e == PV_ENTRY_NULL) {
                              UNLOCK_PVH(pai);
                              PMAP_READ_UNLOCK(pmap, spl);
  
                              /*
                               * Refill from zone.
                               */
                              pv_e = (pv_entry_t) zalloc(pv_list_zone);
                              goto Retry;
                          }
                      }
                      pv_e->va = v;
                      pv_e->pmap = pmap;
                      pv_e->next = pv_h->next;
                      pv_h->next = pv_e;
                      /*
                       *  Remember that we used the pvlist entry.
                       */
                      pv_e = PV_ENTRY_NULL;
                  }
                  UNLOCK_PVH(pai);
              }
  
              /*
               *  And count the mapping.
               */
  
              pmap->stats.resident_count++;
              if (wired)
                  pmap->stats.wired_count++;
  
              /*
               *  Build a template to speed up entering -
               *  only the pfn changes.
               */
              template = pa_to_pte(pa) | INTEL_PTE_VALID;
              if (pmap != kernel_pmap)
                  template |= INTEL_PTE_USER;
              if (prot & VM_PROT_WRITE)
                  template |= INTEL_PTE_WRITE;
              if (wired)
                  template |= INTEL_PTE_WIRED;
              i = ptes_per_vm_page;
              do {
                  WRITE_PTE(pte, template)
                  pte++;
                  pte_increment_pa(template);
              } while (--i > 0);
          }
  
          if (pv_e != PV_ENTRY_NULL) {
              PV_FREE(pv_e);
          }
  
          PMAP_READ_UNLOCK(pmap, spl);
  }
  
  /*
   *      Routine:        pmap_change_wiring
   *      Function:       Change the wiring attribute for a map/virtual-address
   *                      pair.
   *      In/out conditions:
   *                      The mapping must already exist in the pmap.
   */
  void pmap_change_wiring(
          register pmap_t map,
          vm_offset_t     v,
          boolean_t       wired)
  {
          register pt_entry_t     *pte;
          register int            i;
  #if     NCPUS > 1
          spl_t                   spl;
  #endif
  
          /*
           *      We must grab the pmap system lock because we may
           *      change a pte_page queue.
           */
          PMAP_READ_LOCK(map, spl);
  
          if ((pte = pmap_pte(map, v)) == PT_ENTRY_NULL)
                  panic("pmap_change_wiring: pte missing");
  
          if (wired && !(*pte & INTEL_PTE_WIRED)) {
              /*
               *  wiring down mapping
               */
              map->stats.wired_count++;
              i = ptes_per_vm_page;
              do {
                  *pte++ |= INTEL_PTE_WIRED;
              } while (--i > 0);
          }
          else if (!wired && (*pte & INTEL_PTE_WIRED)) {
              /*
               *  unwiring mapping
               */
              map->stats.wired_count--;
              i = ptes_per_vm_page;
              do {
                  *pte &= ~INTEL_PTE_WIRED;
              } while (--i > 0);
          }
  
          PMAP_READ_UNLOCK(map, spl);
  }
  
  /*
   *      Routine:        pmap_extract
   *      Function:
   *              Extract the physical page address associated
   *              with the given map/virtual_address pair.
   */
  
  vm_offset_t pmap_extract(
          register pmap_t pmap,
          vm_offset_t     va)
  {
          register pt_entry_t     *pte;
          register vm_offset_t    pa;
  #if     NCPUS > 1
          spl_t                   spl;
  #endif
  
          SPLVM(spl);
          simple_lock(&pmap->lock);
          if ((pte = pmap_pte(pmap, va)) == PT_ENTRY_NULL)
              pa = (vm_offset_t) 0;
          else if (!(*pte & INTEL_PTE_VALID))
              pa = (vm_offset_t) 0;
          else
              pa = pte_to_pa(*pte) + (va & INTEL_OFFMASK);
          simple_unlock(&pmap->lock);
          SPLX(spl);
          return pa;
  }
  
  /*
   *      Copy the range specified by src_addr/len
   *      from the source map to the range dst_addr/len
   *      in the destination map.
   *
   *      This routine is only advisory and need not do anything.
   */
  #if     0
  void pmap_copy(
          pmap_t          dst_pmap,
          pmap_t          src_pmap,
          vm_offset_t     dst_addr,
          vm_size_t       len,
          vm_offset_t     src_addr)
  {
  #ifdef  lint
          dst_pmap++; src_pmap++; dst_addr++; len++; src_addr++;
  #endif  /* lint */
  }
  #endif  /* 0 */
  
  /*
   *      Routine:        pmap_collect
   *      Function:
   *              Garbage collects the physical map system for
   *              pages which are no longer used.
   *              Success need not be guaranteed -- that is, there
   *              may well be pages which are not referenced, but
   *              others may be collected.
   *      Usage:
   *              Called by the pageout daemon when pages are scarce.
   */
  void pmap_collect(
          pmap_t          p)
  {
          register pt_entry_t     *pdp, *ptp;
          pt_entry_t              *eptp;
          vm_offset_t             pa;
          boolean_t               wired;
  #if     NCPUS > 1
          spl_t                   spl;
  #endif
  
          if (p == PMAP_NULL)
                  return;
  
          if (p == kernel_pmap)
                  return;
  
          /*
           *      Garbage collect map.
           */
          PMAP_READ_LOCK(p, spl);
          PMAP_UPDATE_TLBS(p, VM_MIN_ADDRESS, VM_MAX_ADDRESS);
  
          for (pdp = p->dirbase;
               pdp < &p->dirbase[pdenum(VM_MIN_KERNEL_ADDRESS)];
               pdp += ptes_per_vm_page)
          {
              if (*pdp & INTEL_PTE_VALID) {
  
                  pa = pte_to_pa(*pdp);
                  ptp = (pt_entry_t *)phystokv(pa);
                  eptp = ptp + NPTES*ptes_per_vm_page;
  
                  /*
                   * If the pte page has any wired mappings, we cannot
                   * free it.
                   */
                  wired = FALSE;
                  {
                      register pt_entry_t *ptep;
                      for (ptep = ptp; ptep < eptp; ptep++) {
                          if (*ptep & INTEL_PTE_WIRED) {
                              wired = TRUE;
                              break;
                          }
                      }
                  }
                  if (!wired) {
                      /*
                       * Remove the virtual addresses mapped by this pte page.
                       */
                      pmap_remove_range(p,
                                  pdetova(pdp - p->dirbase),
                                  ptp,
                                  eptp);
  
                      /*
                       * Invalidate the page directory pointer.
                       */
                      {
                          register int i = ptes_per_vm_page;
                          register pt_entry_t *pdep = pdp;
                          do {
                              *pdep++ = 0;
                          } while (--i > 0);
                      }
  
                      PMAP_READ_UNLOCK(p, spl);
  
                      /*
                       * And free the pte page itself.
                       */
                      {
                          register vm_page_t m;
  
                          vm_object_lock(pmap_object);
                          m = vm_page_lookup(pmap_object, pa);
                          if (m == VM_PAGE_NULL)
                              panic("pmap_collect: pte page not in object");
                          vm_page_lock_queues();
                          vm_page_free(m);
                          inuse_ptepages_count--;
                          vm_page_unlock_queues();
                          vm_object_unlock(pmap_object);
                      }
  
                      PMAP_READ_LOCK(p, spl);
                  }
              }
          }
          PMAP_READ_UNLOCK(p, spl);
          return;
  
  }
  
  /*
   *      Routine:        pmap_activate
   *      Function:
   *              Binds the given physical map to the given
   *              processor.
   */
  #if     0
  void pmap_activate(
          register pmap_t my_pmap,
          thread_t        th,
          int             my_cpu)
  {
          PMAP_ACTIVATE(my_pmap, th, my_cpu);
  }
  #endif  /* 0 */
  
  /*
   *      Routine:        pmap_deactivate
   *      Function:
   *              Indicates that the given physical map is no longer
   *              in use on the specified processor.  (This is a macro
   *              in pmap.h)
   */
  #if     0
  void pmap_deactivate(
          pmap_t          pmap,
          thread_t        th,
          int             which_cpu)
  {
  #ifdef  lint
          pmap++; th++; which_cpu++;
  #endif  /* lint */
          PMAP_DEACTIVATE(pmap, th, which_cpu);
  }
  #endif  /* 0 */
  
  /*
   *      Routine:        pmap_kernel
   *      Function:
   *              Returns the physical map handle for the kernel.
   */
  #if     0
  pmap_t pmap_kernel(void)
  {
          return (kernel_pmap);
  }
  #endif  /* 0 */
  
  /*
   *      pmap_zero_page zeros the specified (machine independent) page.
   *      See machine/phys.c or machine/phys.s for implementation.
   */
  #if     0
  pmap_zero_page(
          register vm_offset_t    phys)
  {
          register int    i;
  
          assert(phys != vm_page_fictitious_addr);
          i = PAGE_SIZE / INTEL_PGBYTES;
          phys = intel_pfn(phys);
  
          while (i--)
                  zero_phys(phys++);
  }
  #endif  /* 0 */
  
  /*
   *      pmap_copy_page copies the specified (machine independent) page.
   *      See machine/phys.c or machine/phys.s for implementation.
   */
  #if     0
  pmap_copy_page(
          vm_offset_t     src,
          vm_offset_t     dst)
  {
          int     i;
  
          assert(src != vm_page_fictitious_addr);
          assert(dst != vm_page_fictitious_addr);
          i = PAGE_SIZE / INTEL_PGBYTES;
  
          while (i--) {
                  copy_phys(intel_pfn(src), intel_pfn(dst));
                  src += INTEL_PGBYTES;
                  dst += INTEL_PGBYTES;
          }
  }
  #endif  /* 0 */
  
  /*
   *      Routine:        pmap_pageable
   *      Function:
   *              Make the specified pages (by pmap, offset)
   *              pageable (or not) as requested.
   *
   *              A page which is not pageable may not take
   *              a fault; therefore, its page table entry
   *              must remain valid for the duration.
   *
   *              This routine is merely advisory; pmap_enter
   *              will specify that these pages are to be wired
   *              down (or not) as appropriate.
   */
  void pmap_pageable(
          pmap_t          pmap,
          vm_offset_t     start,
          vm_offset_t     end,
          boolean_t       pageable)
  {
  #ifdef  lint
          pmap++; start++; end++; pageable++;
  #endif  /* lint */
  }
  
  /*
   *      Clear specified attribute bits.
   */
  void
  phys_attribute_clear(
          vm_offset_t     phys,
          int             bits)
  {
          pv_entry_t              pv_h;
          register pv_entry_t     pv_e;
          register pt_entry_t     *pte;
          int                     pai;
          register pmap_t         pmap;
  #if     NCPUS > 1
          spl_t                   spl;
  #endif
  
          assert(phys != vm_page_fictitious_addr);
          if (!valid_page(phys)) {
              /*
               *  Not a managed page.
               */
              return;
          }
  
          /*
           *      Lock the pmap system first, since we will be changing
           *      several pmaps.
           */
  
          PMAP_WRITE_LOCK(spl);
  
          pai = pa_index(phys);
          pv_h = pai_to_pvh(pai);
  
          /*
           * Walk down PV list, clearing all modify or reference bits.
           * We do not have to lock the pv_list because we have
           * the entire pmap system locked.
           */
          if (pv_h->pmap != PMAP_NULL) {
              /*
               * There are some mappings.
               */
              for (pv_e = pv_h; pv_e != PV_ENTRY_NULL; pv_e = pv_e->next) {
  
                  pmap = pv_e->pmap;
                  /*
                   * Lock the pmap to block pmap_extract and similar routines.
                   */
                  simple_lock(&pmap->lock);
  
                  {
                      register vm_offset_t va;
  
                      va = pv_e->va;
                      pte = pmap_pte(pmap, va);
  
  #if     0
                      /*
                       * Consistency checks.
                       */
                      assert(*pte & INTEL_PTE_VALID);
                      /* assert(pte_to_phys(*pte) == phys); */
  #endif
  
                      /*
                       * Invalidate TLBs for all CPUs using this mapping.
                       */
                      PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
                  }
  
                  /*
                   * Clear modify or reference bits.
                   */
                  {
                      register int        i = ptes_per_vm_page;
                      do {
                          *pte &= ~bits;
                      } while (--i > 0);
                  }
                  simple_unlock(&pmap->lock);
              }
          }
  
          pmap_phys_attributes[pai] &= ~bits;
  
          PMAP_WRITE_UNLOCK(spl);
  }
  
  /*
   *      Check specified attribute bits.
   */
  boolean_t
  phys_attribute_test(
          vm_offset_t     phys,
          int             bits)
  {
          pv_entry_t              pv_h;
          register pv_entry_t     pv_e;
          register pt_entry_t     *pte;
          int                     pai;
          register pmap_t         pmap;
  #if     NCPUS > 1
          spl_t                   spl;
  #endif
  
          assert(phys != vm_page_fictitious_addr);
          if (!valid_page(phys)) {
              /*
               *  Not a managed page.
               */
              return FALSE;
          }
  
          /*
           *      Lock the pmap system first, since we will be checking
           *      several pmaps.
           */
  
          PMAP_WRITE_LOCK(spl);
  
          pai = pa_index(phys);
          pv_h = pai_to_pvh(pai);
  
          if (pmap_phys_attributes[pai] & bits) {
              PMAP_WRITE_UNLOCK(spl);
              return TRUE;
          }
  
          /*
           * Walk down PV list, checking all mappings.
           * We do not have to lock the pv_list because we have
           * the entire pmap system locked.
           */
          if (pv_h->pmap != PMAP_NULL) {
              /*
               * There are some mappings.
               */
              for (pv_e = pv_h; pv_e != PV_ENTRY_NULL; pv_e = pv_e->next) {
  
                  pmap = pv_e->pmap;
                  /*
                   * Lock the pmap to block pmap_extract and similar routines.
                   */
                  simple_lock(&pmap->lock);
  
                  {
                      register vm_offset_t va;
  
                      va = pv_e->va;
                      pte = pmap_pte(pmap, va);
  
  #if     0
                      /*
                       * Consistency checks.
                       */
                      assert(*pte & INTEL_PTE_VALID);
                      /* assert(pte_to_phys(*pte) == phys); */
  #endif
                  }
  
                  /*
                   * Check modify or reference bits.
                   */
                  {
                      register int        i = ptes_per_vm_page;
  
                      do {
                          if (*pte & bits) {
                              simple_unlock(&pmap->lock);
                              PMAP_WRITE_UNLOCK(spl);
                              return TRUE;
                          }
                      } while (--i > 0);
                  }
                  simple_unlock(&pmap->lock);
              }
          }
          PMAP_WRITE_UNLOCK(spl);
          return FALSE;
  }
  
  /*
   *      Clear the modify bits on the specified physical page.
   */
  
  void pmap_clear_modify(
          register vm_offset_t    phys)
  {
          phys_attribute_clear(phys, PHYS_MODIFIED);
  }
  
  /*
   *      pmap_is_modified:
   *
   *      Return whether or not the specified physical page is modified
   *      by any physical maps.
   */
  
  boolean_t pmap_is_modified(
          register vm_offset_t    phys)
  {
          return phys_attribute_test(phys, PHYS_MODIFIED);
  }
  
  /*
   *      pmap_clear_reference:
   *
   *      Clear the reference bit on the specified physical page.
   */
  
  void pmap_clear_reference(
          vm_offset_t     phys)
  {
          phys_attribute_clear(phys, PHYS_REFERENCED);
  }
  
  /*
   *      pmap_is_referenced:
   *
   *      Return whether or not the specified physical page is referenced
   *      by any physical maps.
   */
  
  boolean_t pmap_is_referenced(
          vm_offset_t     phys)
  {
          return phys_attribute_test(phys, PHYS_REFERENCED);
  }
  
  #if     NCPUS > 1
  /*
  *           TLB Coherence Code (TLB "shootdown" code)
  * 
  * Threads that belong to the same task share the same address space and
  * hence share a pmap.  However, they  may run on distinct cpus and thus
  * have distinct TLBs that cache page table entries. In order to guarantee
  * the TLBs are consistent, whenever a pmap is changed, all threads that
  * are active in that pmap must have their TLB updated. To keep track of
  * this information, the set of cpus that are currently using a pmap is
  * maintained within each pmap structure (cpus_using). Pmap_activate() and
  * pmap_deactivate add and remove, respectively, a cpu from this set.
  * Since the TLBs are not addressable over the bus, each processor must
  * flush its own TLB; a processor that needs to invalidate another TLB
  * needs to interrupt the processor that owns that TLB to signal the
  * update.
  * 
  * Whenever a pmap is updated, the lock on that pmap is locked, and all
  * cpus using the pmap are signaled to invalidate. All threads that need
  * to activate a pmap must wait for the lock to clear to await any updates
  * in progress before using the pmap. They must ACQUIRE the lock to add
  * their cpu to the cpus_using set. An implicit assumption made
  * throughout the TLB code is that all kernel code that runs at or higher
  * than splvm blocks out update interrupts, and that such code does not
  * touch pageable pages.
  * 
  * A shootdown interrupt serves another function besides signaling a
  * processor to invalidate. The interrupt routine (pmap_update_interrupt)
  * waits for the both the pmap lock (and the kernel pmap lock) to clear,
  * preventing user code from making implicit pmap updates while the
  * sending processor is performing its update. (This could happen via a
  * user data write reference that turns on the modify bit in the page
  * table). It must wait for any kernel updates that may have started
  * concurrently with a user pmap update because the IPC code
  * changes mappings.
  * Spinning on the VALUES of the locks is sufficient (rather than
  * having to acquire the locks) because any updates that occur subsequent
  * to finding the lock unlocked will be signaled via another interrupt.
  * (This assumes the interrupt is cleared before the low level interrupt code 
  * calls pmap_update_interrupt()). 
  * 
  * The signaling processor must wait for any implicit updates in progress
  * to terminate before continuing with its update. Thus it must wait for an
  * acknowledgement of the interrupt from each processor for which such
  * references could be made. For maintaining this information, a set
  * cpus_active is used. A cpu is in this set if and only if it can 
  * use a pmap. When pmap_update_interrupt() is entered, a cpu is removed from
  * this set; when all such cpus are removed, it is safe to update.
  * 
  * Before attempting to acquire the update lock on a pmap, a cpu (A) must
  * be at least at the priority of the interprocessor interrupt
  * (splip<=splvm). Otherwise, A could grab a lock and be interrupted by a
  * kernel update; it would spin forever in pmap_update_interrupt() trying
  * to acquire the user pmap lock it had already acquired. Furthermore A
  * must remove itself from cpus_active.  Otherwise, another cpu holding
  * the lock (B) could be in the process of sending an update signal to A,
  * and thus be waiting for A to remove itself from cpus_active. If A is
  * spinning on the lock at priority this will never happen and a deadlock
  * will result.
  */
  
  /*
   *      Signal another CPU that it must flush its TLB
   */
  void signal_cpus(
          cpu_set         use_list,
          pmap_t          pmap,
          vm_offset_t     start,
          vm_offset_t     end)
  {
          register int            which_cpu, j;
          register pmap_update_list_t     update_list_p;
  
          while ((which_cpu = ffs(use_list)) != 0) {
              which_cpu -= 1;     /* convert to 0 origin */
  
              update_list_p = &cpu_update_list[which_cpu];
              simple_lock(&update_list_p->lock);
  
              j = update_list_p->count;
              if (j >= UPDATE_LIST_SIZE) {
                  /*
                   *      list overflowed.  Change last item to
                   *      indicate overflow.
                   */
                  update_list_p->item[UPDATE_LIST_SIZE-1].pmap  = kernel_pmap;
                  update_list_p->item[UPDATE_LIST_SIZE-1].start = VM_MIN_ADDRESS;
                  update_list_p->item[UPDATE_LIST_SIZE-1].end   = VM_MAX_KERNEL_ADDRESS;
              }
              else {
                  update_list_p->item[j].pmap  = pmap;
                  update_list_p->item[j].start = start;
                  update_list_p->item[j].end   = end;
                  update_list_p->count = j+1;
              }
              cpu_update_needed[which_cpu] = TRUE;
              simple_unlock(&update_list_p->lock);
  
              if ((cpus_idle & (1 << which_cpu)) == 0)
                  interrupt_processor(which_cpu);
              use_list &= ~(1 << which_cpu);
          }
  }
  
  void process_pmap_updates(
          register pmap_t         my_pmap)
  {
          register int            my_cpu = cpu_number();
          register pmap_update_list_t     update_list_p;
          register int            j;
          register pmap_t         pmap;
  
          update_list_p = &cpu_update_list[my_cpu];
          simple_lock(&update_list_p->lock);
  
          for (j = 0; j < update_list_p->count; j++) {
              pmap = update_list_p->item[j].pmap;
              if (pmap == my_pmap ||
                  pmap == kernel_pmap) {
  
                  INVALIDATE_TLB(update_list_p->item[j].start,
                                  update_list_p->item[j].end);
              }
          }
          update_list_p->count = 0;
          cpu_update_needed[my_cpu] = FALSE;
          simple_unlock(&update_list_p->lock);
  }
  
  /*
   *      Interrupt routine for TBIA requested from other processor.
   */
  void pmap_update_interrupt(void)
  {
          register int            my_cpu;
          register pmap_t         my_pmap;
          int                     s;
  
          my_cpu = cpu_number();
  
          /*
           *      Exit now if we're idle.  We'll pick up the update request
           *      when we go active, and we must not put ourselves back in
           *      the active set because we'll never process the interrupt
           *      while we're idle (thus hanging the system).
           */
          if (cpus_idle & (1 << my_cpu))
              return;
  
          if (current_thread() == THREAD_NULL)
              my_pmap = kernel_pmap;
          else {
              my_pmap = current_pmap();
              if (!pmap_in_use(my_pmap, my_cpu))
                  my_pmap = kernel_pmap;
          }
  
          /*
           *      Raise spl to splvm (above splip) to block out pmap_extract
           *      from IO code (which would put this cpu back in the active
           *      set).
           */
          s = splvm();
  
          do {
  
              /*
               *  Indicate that we're not using either user or kernel
               *  pmap.
               */
              i_bit_clear(my_cpu, &cpus_active);
  
              /*
               *  Wait for any pmap updates in progress, on either user
               *  or kernel pmap.
               */
              while (*(volatile int *)&my_pmap->lock.lock_data ||
                     *(volatile int *)&kernel_pmap->lock.lock_data)
                  continue;
  
              process_pmap_updates(my_pmap);
  
              i_bit_set(my_cpu, &cpus_active);
  
          } while (cpu_update_needed[my_cpu]);
          
          splx(s);
  }
  #else   /* NCPUS > 1 */
  /*
   *      Dummy routine to satisfy external reference.
   */
  void pmap_update_interrupt(void)
  {
          /* should never be called. */
  }
  #endif  /* NCPUS > 1 */
  
  #if     i860    /* akp */
  void set_dirbase(
          register vm_offset_t    dirbase)
  {
          /*flush();*/
          /*flush_tlb();*/
          flush_and_ctxsw(dirbase);
  }
  #endif  /* i860 */

Cache object: 686635158c25464bbd24b932619f0897