[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/vm/vm_fault.c

Version: -  FREEBSD  -  FREEBSD10  -  FREEBSD9  -  FREEBSD92  -  FREEBSD91  -  FREEBSD90  -  FREEBSD8  -  FREEBSD82  -  FREEBSD81  -  FREEBSD80  -  FREEBSD7  -  FREEBSD74  -  FREEBSD73  -  FREEBSD72  -  FREEBSD71  -  FREEBSD70  -  FREEBSD6  -  FREEBSD64  -  FREEBSD63  -  FREEBSD62  -  FREEBSD61  -  FREEBSD60  -  FREEBSD5  -  FREEBSD55  -  FREEBSD54  -  FREEBSD53  -  FREEBSD52  -  FREEBSD51  -  FREEBSD50  -  FREEBSD4  -  FREEBSD3  -  FREEBSD22  -  cheribsd  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1  -  FREEBSD-LIBC  -  FREEBSD8-LIBC  -  FREEBSD7-LIBC  -  FREEBSD6-LIBC  -  GLIBC27 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  * Copyright (c) 1991, 1993
    3  *      The Regents of the University of California.  All rights reserved.
    4  * Copyright (c) 1994 John S. Dyson
    5  * All rights reserved.
    6  * Copyright (c) 1994 David Greenman
    7  * All rights reserved.
    8  *
    9  *
   10  * This code is derived from software contributed to Berkeley by
   11  * The Mach Operating System project at Carnegie-Mellon University.
   12  *
   13  * Redistribution and use in source and binary forms, with or without
   14  * modification, are permitted provided that the following conditions
   15  * are met:
   16  * 1. Redistributions of source code must retain the above copyright
   17  *    notice, this list of conditions and the following disclaimer.
   18  * 2. Redistributions in binary form must reproduce the above copyright
   19  *    notice, this list of conditions and the following disclaimer in the
   20  *    documentation and/or other materials provided with the distribution.
   21  * 3. All advertising materials mentioning features or use of this software
   22  *    must display the following acknowledgement:
   23  *      This product includes software developed by the University of
   24  *      California, Berkeley and its contributors.
   25  * 4. Neither the name of the University nor the names of its contributors
   26  *    may be used to endorse or promote products derived from this software
   27  *    without specific prior written permission.
   28  *
   29  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   30  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   31  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   32  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   33  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   34  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   35  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   36  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   37  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   38  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   39  * SUCH DAMAGE.
   40  *
   41  *      from: @(#)vm_fault.c    8.4 (Berkeley) 1/12/94
   42  *
   43  *
   44  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
   45  * All rights reserved.
   46  *
   47  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
   48  *
   49  * Permission to use, copy, modify and distribute this software and
   50  * its documentation is hereby granted, provided that both the copyright
   51  * notice and this permission notice appear in all copies of the
   52  * software, derivative works or modified versions, and any portions
   53  * thereof, and that both notices appear in supporting documentation.
   54  *
   55  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
   56  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
   57  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
   58  *
   59  * Carnegie Mellon requests users of this software to return to
   60  *
   61  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
   62  *  School of Computer Science
   63  *  Carnegie Mellon University
   64  *  Pittsburgh PA 15213-3890
   65  *
   66  * any improvements or extensions that they make and grant Carnegie the
   67  * rights to redistribute these changes.
   68  */
   69 
   70 /*
   71  *      Page fault handling module.
   72  */
   73 
   74 #include <sys/cdefs.h>
   75 __FBSDID("$FreeBSD: src/sys/vm/vm_fault.c,v 1.182 2003/11/10 00:44:00 mini Exp $");
   76 
   77 #include <sys/param.h>
   78 #include <sys/systm.h>
   79 #include <sys/kernel.h>
   80 #include <sys/lock.h>
   81 #include <sys/mutex.h>
   82 #include <sys/proc.h>
   83 #include <sys/resourcevar.h>
   84 #include <sys/sysctl.h>
   85 #include <sys/vmmeter.h>
   86 #include <sys/vnode.h>
   87 
   88 #include <vm/vm.h>
   89 #include <vm/vm_param.h>
   90 #include <vm/pmap.h>
   91 #include <vm/vm_map.h>
   92 #include <vm/vm_object.h>
   93 #include <vm/vm_page.h>
   94 #include <vm/vm_pageout.h>
   95 #include <vm/vm_kern.h>
   96 #include <vm/vm_pager.h>
   97 #include <vm/vnode_pager.h>
   98 #include <vm/vm_extern.h>
   99 
  100 #define PFBAK 4
  101 #define PFFOR 4
  102 #define PAGEORDER_SIZE (PFBAK+PFFOR)
  103 
  104 static int prefault_pageorder[] = {
  105         -1 * PAGE_SIZE, 1 * PAGE_SIZE,
  106         -2 * PAGE_SIZE, 2 * PAGE_SIZE,
  107         -3 * PAGE_SIZE, 3 * PAGE_SIZE,
  108         -4 * PAGE_SIZE, 4 * PAGE_SIZE
  109 };
  110 
  111 static int vm_fault_additional_pages(vm_page_t, int, int, vm_page_t *, int *);
  112 static void vm_fault_prefault(pmap_t, vm_offset_t, vm_map_entry_t);
  113 
  114 #define VM_FAULT_READ_AHEAD 8
  115 #define VM_FAULT_READ_BEHIND 7
  116 #define VM_FAULT_READ (VM_FAULT_READ_AHEAD+VM_FAULT_READ_BEHIND+1)
  117 
  118 struct faultstate {
  119         vm_page_t m;
  120         vm_object_t object;
  121         vm_pindex_t pindex;
  122         vm_page_t first_m;
  123         vm_object_t     first_object;
  124         vm_pindex_t first_pindex;
  125         vm_map_t map;
  126         vm_map_entry_t entry;
  127         int lookup_still_valid;
  128         struct vnode *vp;
  129 };
  130 
  131 static __inline void
  132 release_page(struct faultstate *fs)
  133 {
  134         vm_page_lock_queues();
  135         vm_page_wakeup(fs->m);
  136         vm_page_deactivate(fs->m);
  137         vm_page_unlock_queues();
  138         fs->m = NULL;
  139 }
  140 
  141 static __inline void
  142 unlock_map(struct faultstate *fs)
  143 {
  144         if (fs->lookup_still_valid) {
  145                 vm_map_lookup_done(fs->map, fs->entry);
  146                 fs->lookup_still_valid = FALSE;
  147         }
  148 }
  149 
  150 static void
  151 _unlock_things(struct faultstate *fs, int dealloc)
  152 {
  153 
  154         vm_object_pip_wakeup(fs->object);
  155         VM_OBJECT_UNLOCK(fs->object);
  156         if (fs->object != fs->first_object) {
  157                 VM_OBJECT_LOCK(fs->first_object);
  158                 vm_page_lock_queues();
  159                 vm_page_free(fs->first_m);
  160                 vm_page_unlock_queues();
  161                 vm_object_pip_wakeup(fs->first_object);
  162                 VM_OBJECT_UNLOCK(fs->first_object);
  163                 fs->first_m = NULL;
  164         }
  165         if (dealloc) {
  166                 vm_object_deallocate(fs->first_object);
  167         }
  168         unlock_map(fs); 
  169         if (fs->vp != NULL) { 
  170                 vput(fs->vp);
  171                 fs->vp = NULL;
  172         }
  173 }
  174 
  175 #define unlock_things(fs) _unlock_things(fs, 0)
  176 #define unlock_and_deallocate(fs) _unlock_things(fs, 1)
  177 
  178 /*
  179  * TRYPAGER - used by vm_fault to calculate whether the pager for the
  180  *            current object *might* contain the page.
  181  *
  182  *            default objects are zero-fill, there is no real pager.
  183  */
  184 #define TRYPAGER        (fs.object->type != OBJT_DEFAULT && \
  185                         (((fault_flags & VM_FAULT_WIRE_MASK) == 0) || wired))
  186 
  187 /*
  188  *      vm_fault:
  189  *
  190  *      Handle a page fault occurring at the given address,
  191  *      requiring the given permissions, in the map specified.
  192  *      If successful, the page is inserted into the
  193  *      associated physical map.
  194  *
  195  *      NOTE: the given address should be truncated to the
  196  *      proper page address.
  197  *
  198  *      KERN_SUCCESS is returned if the page fault is handled; otherwise,
  199  *      a standard error specifying why the fault is fatal is returned.
  200  *
  201  *
  202  *      The map in question must be referenced, and remains so.
  203  *      Caller may hold no locks.
  204  */
  205 int
  206 vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
  207          int fault_flags)
  208 {
  209         vm_prot_t prot;
  210         int is_first_object_locked, result;
  211         boolean_t growstack, wired;
  212         int map_generation;
  213         vm_object_t next_object;
  214         vm_page_t marray[VM_FAULT_READ];
  215         int hardfault;
  216         int faultcount;
  217         struct faultstate fs;
  218 
  219         hardfault = 0;
  220         growstack = TRUE;
  221         atomic_add_int(&cnt.v_vm_faults, 1);
  222 
  223         mtx_lock(&Giant);
  224 RetryFault:;
  225 
  226         /*
  227          * Find the backing store object and offset into it to begin the
  228          * search.
  229          */
  230         fs.map = map;
  231         result = vm_map_lookup(&fs.map, vaddr, fault_type, &fs.entry,
  232             &fs.first_object, &fs.first_pindex, &prot, &wired);
  233         if (result != KERN_SUCCESS) {
  234                 if (result != KERN_PROTECTION_FAILURE ||
  235                     (fault_flags & VM_FAULT_WIRE_MASK) != VM_FAULT_USER_WIRE) {
  236                         if (growstack && result == KERN_INVALID_ADDRESS &&
  237                             map != kernel_map && curproc != NULL) {
  238                                 result = vm_map_growstack(curproc, vaddr);
  239                                 if (result != KERN_SUCCESS) {
  240                                         mtx_unlock(&Giant);
  241                                         return (KERN_FAILURE);
  242                                 }
  243                                 growstack = FALSE;
  244                                 goto RetryFault;
  245                         }
  246                         mtx_unlock(&Giant);
  247                         return (result);
  248                 }
  249 
  250                 /*
  251                  * If we are user-wiring a r/w segment, and it is COW, then
  252                  * we need to do the COW operation.  Note that we don't COW
  253                  * currently RO sections now, because it is NOT desirable
  254                  * to COW .text.  We simply keep .text from ever being COW'ed
  255                  * and take the heat that one cannot debug wired .text sections.
  256                  */
  257                 result = vm_map_lookup(&fs.map, vaddr,
  258                         VM_PROT_READ|VM_PROT_WRITE|VM_PROT_OVERRIDE_WRITE,
  259                         &fs.entry, &fs.first_object, &fs.first_pindex, &prot, &wired);
  260                 if (result != KERN_SUCCESS) {
  261                         mtx_unlock(&Giant);
  262                         return (result);
  263                 }
  264 
  265                 /*
  266                  * If we don't COW now, on a user wire, the user will never
  267                  * be able to write to the mapping.  If we don't make this
  268                  * restriction, the bookkeeping would be nearly impossible.
  269                  *
  270                  * XXX The following assignment modifies the map without
  271                  * holding a write lock on it.
  272                  */
  273                 if ((fs.entry->protection & VM_PROT_WRITE) == 0)
  274                         fs.entry->max_protection &= ~VM_PROT_WRITE;
  275         }
  276 
  277         map_generation = fs.map->timestamp;
  278 
  279         if (fs.entry->eflags & MAP_ENTRY_NOFAULT) {
  280                 panic("vm_fault: fault on nofault entry, addr: %lx",
  281                     (u_long)vaddr);
  282         }
  283 
  284         /*
  285          * Make a reference to this object to prevent its disposal while we
  286          * are messing with it.  Once we have the reference, the map is free
  287          * to be diddled.  Since objects reference their shadows (and copies),
  288          * they will stay around as well.
  289          *
  290          * Bump the paging-in-progress count to prevent size changes (e.g. 
  291          * truncation operations) during I/O.  This must be done after
  292          * obtaining the vnode lock in order to avoid possible deadlocks.
  293          *
  294          * XXX vnode_pager_lock() can block without releasing the map lock.
  295          */
  296         vm_object_reference(fs.first_object);
  297         VM_OBJECT_LOCK(fs.first_object);
  298         fs.vp = vnode_pager_lock(fs.first_object);
  299         vm_object_pip_add(fs.first_object, 1);
  300 
  301         fs.lookup_still_valid = TRUE;
  302 
  303         if (wired)
  304                 fault_type = prot;
  305 
  306         fs.first_m = NULL;
  307 
  308         /*
  309          * Search for the page at object/offset.
  310          */
  311         fs.object = fs.first_object;
  312         fs.pindex = fs.first_pindex;
  313         while (TRUE) {
  314                 /*
  315                  * If the object is dead, we stop here
  316                  */
  317                 if (fs.object->flags & OBJ_DEAD) {
  318                         unlock_and_deallocate(&fs);
  319                         mtx_unlock(&Giant);
  320                         return (KERN_PROTECTION_FAILURE);
  321                 }
  322 
  323                 /*
  324                  * See if page is resident
  325                  */
  326                 fs.m = vm_page_lookup(fs.object, fs.pindex);
  327                 if (fs.m != NULL) {
  328                         int queue, s;
  329 
  330                         /* 
  331                          * check for page-based copy on write.
  332                          * We check fs.object == fs.first_object so
  333                          * as to ensure the legacy COW mechanism is
  334                          * used when the page in question is part of
  335                          * a shadow object.  Otherwise, vm_page_cowfault()
  336                          * removes the page from the backing object, 
  337                          * which is not what we want.
  338                          */
  339                         vm_page_lock_queues();
  340                         if ((fs.m->cow) && 
  341                             (fault_type & VM_PROT_WRITE) &&
  342                             (fs.object == fs.first_object)) {
  343                                 s = splvm();
  344                                 vm_page_cowfault(fs.m);
  345                                 splx(s);
  346                                 vm_page_unlock_queues();
  347                                 unlock_and_deallocate(&fs);
  348                                 goto RetryFault;
  349                         }
  350 
  351                         /*
  352                          * Wait/Retry if the page is busy.  We have to do this
  353                          * if the page is busy via either PG_BUSY or 
  354                          * vm_page_t->busy because the vm_pager may be using
  355                          * vm_page_t->busy for pageouts ( and even pageins if
  356                          * it is the vnode pager ), and we could end up trying
  357                          * to pagein and pageout the same page simultaneously.
  358                          *
  359                          * We can theoretically allow the busy case on a read
  360                          * fault if the page is marked valid, but since such
  361                          * pages are typically already pmap'd, putting that
  362                          * special case in might be more effort then it is 
  363                          * worth.  We cannot under any circumstances mess
  364                          * around with a vm_page_t->busy page except, perhaps,
  365                          * to pmap it.
  366                          */
  367                         if ((fs.m->flags & PG_BUSY) || fs.m->busy) {
  368                                 vm_page_unlock_queues();
  369                                 unlock_things(&fs);
  370                                 vm_page_lock_queues();
  371                                 if (!vm_page_sleep_if_busy(fs.m, TRUE, "vmpfw"))
  372                                         vm_page_unlock_queues();
  373                                 cnt.v_intrans++;
  374                                 vm_object_deallocate(fs.first_object);
  375                                 goto RetryFault;
  376                         }
  377                         queue = fs.m->queue;
  378 
  379                         s = splvm();
  380                         vm_pageq_remove_nowakeup(fs.m);
  381                         splx(s);
  382 
  383                         if ((queue - fs.m->pc) == PQ_CACHE && vm_page_count_severe()) {
  384                                 vm_page_activate(fs.m);
  385                                 vm_page_unlock_queues();
  386                                 unlock_and_deallocate(&fs);
  387                                 VM_WAITPFAULT;
  388                                 goto RetryFault;
  389                         }
  390 
  391                         /*
  392                          * Mark page busy for other processes, and the 
  393                          * pagedaemon.  If it still isn't completely valid
  394                          * (readable), jump to readrest, else break-out ( we
  395                          * found the page ).
  396                          */
  397                         vm_page_busy(fs.m);
  398                         vm_page_unlock_queues();
  399                         if (((fs.m->valid & VM_PAGE_BITS_ALL) != VM_PAGE_BITS_ALL) &&
  400                                 fs.m->object != kernel_object && fs.m->object != kmem_object) {
  401                                 goto readrest;
  402                         }
  403 
  404                         break;
  405                 }
  406 
  407                 /*
  408                  * Page is not resident, If this is the search termination
  409                  * or the pager might contain the page, allocate a new page.
  410                  */
  411                 if (TRYPAGER || fs.object == fs.first_object) {
  412                         if (fs.pindex >= fs.object->size) {
  413                                 unlock_and_deallocate(&fs);
  414                                 mtx_unlock(&Giant);
  415                                 return (KERN_PROTECTION_FAILURE);
  416                         }
  417 
  418                         /*
  419                          * Allocate a new page for this object/offset pair.
  420                          */
  421                         fs.m = NULL;
  422                         if (!vm_page_count_severe()) {
  423                                 fs.m = vm_page_alloc(fs.object, fs.pindex,
  424                                     (fs.vp || fs.object->backing_object)? VM_ALLOC_NORMAL: VM_ALLOC_ZERO);
  425                         }
  426                         if (fs.m == NULL) {
  427                                 unlock_and_deallocate(&fs);
  428                                 VM_WAITPFAULT;
  429                                 goto RetryFault;
  430                         }
  431                 }
  432 
  433 readrest:
  434                 /*
  435                  * We have found a valid page or we have allocated a new page.
  436                  * The page thus may not be valid or may not be entirely 
  437                  * valid.
  438                  *
  439                  * Attempt to fault-in the page if there is a chance that the
  440                  * pager has it, and potentially fault in additional pages
  441                  * at the same time.
  442                  */
  443                 if (TRYPAGER) {
  444                         int rv;
  445                         int reqpage;
  446                         int ahead, behind;
  447                         u_char behavior = vm_map_entry_behavior(fs.entry);
  448 
  449                         if (behavior == MAP_ENTRY_BEHAV_RANDOM) {
  450                                 ahead = 0;
  451                                 behind = 0;
  452                         } else {
  453                                 behind = (vaddr - fs.entry->start) >> PAGE_SHIFT;
  454                                 if (behind > VM_FAULT_READ_BEHIND)
  455                                         behind = VM_FAULT_READ_BEHIND;
  456 
  457                                 ahead = ((fs.entry->end - vaddr) >> PAGE_SHIFT) - 1;
  458                                 if (ahead > VM_FAULT_READ_AHEAD)
  459                                         ahead = VM_FAULT_READ_AHEAD;
  460                         }
  461                         is_first_object_locked = FALSE;
  462                         if ((behavior == MAP_ENTRY_BEHAV_SEQUENTIAL ||
  463                              (behavior != MAP_ENTRY_BEHAV_RANDOM &&
  464                               fs.pindex >= fs.entry->lastr &&
  465                               fs.pindex < fs.entry->lastr + VM_FAULT_READ)) &&
  466                             (fs.first_object == fs.object ||
  467                              (is_first_object_locked = VM_OBJECT_TRYLOCK(fs.first_object))) &&
  468                             fs.first_object->type != OBJT_DEVICE) {
  469                                 vm_pindex_t firstpindex, tmppindex;
  470 
  471                                 if (fs.first_pindex < 2 * VM_FAULT_READ)
  472                                         firstpindex = 0;
  473                                 else
  474                                         firstpindex = fs.first_pindex - 2 * VM_FAULT_READ;
  475 
  476                                 vm_page_lock_queues();
  477                                 /*
  478                                  * note: partially valid pages cannot be 
  479                                  * included in the lookahead - NFS piecemeal
  480                                  * writes will barf on it badly.
  481                                  */
  482                                 for (tmppindex = fs.first_pindex - 1;
  483                                         tmppindex >= firstpindex;
  484                                         --tmppindex) {
  485                                         vm_page_t mt;
  486 
  487                                         mt = vm_page_lookup(fs.first_object, tmppindex);
  488                                         if (mt == NULL || (mt->valid != VM_PAGE_BITS_ALL))
  489                                                 break;
  490                                         if (mt->busy ||
  491                                                 (mt->flags & (PG_BUSY | PG_FICTITIOUS | PG_UNMANAGED)) ||
  492                                                 mt->hold_count ||
  493                                                 mt->wire_count) 
  494                                                 continue;
  495                                         if (mt->dirty == 0)
  496                                                 vm_page_test_dirty(mt);
  497                                         if (mt->dirty) {
  498                                                 pmap_remove_all(mt);
  499                                                 vm_page_deactivate(mt);
  500                                         } else {
  501                                                 vm_page_cache(mt);
  502                                         }
  503                                 }
  504                                 vm_page_unlock_queues();
  505                                 ahead += behind;
  506                                 behind = 0;
  507                         }
  508                         if (is_first_object_locked)
  509                                 VM_OBJECT_UNLOCK(fs.first_object);
  510                         /*
  511                          * now we find out if any other pages should be paged
  512                          * in at this time this routine checks to see if the
  513                          * pages surrounding this fault reside in the same
  514                          * object as the page for this fault.  If they do,
  515                          * then they are faulted in also into the object.  The
  516                          * array "marray" returned contains an array of
  517                          * vm_page_t structs where one of them is the
  518                          * vm_page_t passed to the routine.  The reqpage
  519                          * return value is the index into the marray for the
  520                          * vm_page_t passed to the routine.
  521                          *
  522                          * fs.m plus the additional pages are PG_BUSY'd.
  523                          *
  524                          * XXX vm_fault_additional_pages() can block
  525                          * without releasing the map lock.
  526                          */
  527                         faultcount = vm_fault_additional_pages(
  528                             fs.m, behind, ahead, marray, &reqpage);
  529 
  530                         /*
  531                          * update lastr imperfectly (we do not know how much
  532                          * getpages will actually read), but good enough.
  533                          *
  534                          * XXX The following assignment modifies the map
  535                          * without holding a write lock on it.
  536                          */
  537                         fs.entry->lastr = fs.pindex + faultcount - behind;
  538 
  539                         /*
  540                          * Call the pager to retrieve the data, if any, after
  541                          * releasing the lock on the map.  We hold a ref on
  542                          * fs.object and the pages are PG_BUSY'd.
  543                          */
  544                         unlock_map(&fs);
  545 
  546                         rv = faultcount ?
  547                             vm_pager_get_pages(fs.object, marray, faultcount,
  548                                 reqpage) : VM_PAGER_FAIL;
  549 
  550                         if (rv == VM_PAGER_OK) {
  551                                 /*
  552                                  * Found the page. Leave it busy while we play
  553                                  * with it.
  554                                  */
  555 
  556                                 /*
  557                                  * Relookup in case pager changed page. Pager
  558                                  * is responsible for disposition of old page
  559                                  * if moved.
  560                                  */
  561                                 fs.m = vm_page_lookup(fs.object, fs.pindex);
  562                                 if (!fs.m) {
  563                                         unlock_and_deallocate(&fs);
  564                                         goto RetryFault;
  565                                 }
  566 
  567                                 hardfault++;
  568                                 break; /* break to PAGE HAS BEEN FOUND */
  569                         }
  570                         /*
  571                          * Remove the bogus page (which does not exist at this
  572                          * object/offset); before doing so, we must get back
  573                          * our object lock to preserve our invariant.
  574                          *
  575                          * Also wake up any other process that may want to bring
  576                          * in this page.
  577                          *
  578                          * If this is the top-level object, we must leave the
  579                          * busy page to prevent another process from rushing
  580                          * past us, and inserting the page in that object at
  581                          * the same time that we are.
  582                          */
  583                         if (rv == VM_PAGER_ERROR)
  584                                 printf("vm_fault: pager read error, pid %d (%s)\n",
  585                                     curproc->p_pid, curproc->p_comm);
  586                         /*
  587                          * Data outside the range of the pager or an I/O error
  588                          */
  589                         /*
  590                          * XXX - the check for kernel_map is a kludge to work
  591                          * around having the machine panic on a kernel space
  592                          * fault w/ I/O error.
  593                          */
  594                         if (((fs.map != kernel_map) && (rv == VM_PAGER_ERROR)) ||
  595                                 (rv == VM_PAGER_BAD)) {
  596                                 vm_page_lock_queues();
  597                                 vm_page_free(fs.m);
  598                                 vm_page_unlock_queues();
  599                                 fs.m = NULL;
  600                                 unlock_and_deallocate(&fs);
  601                                 mtx_unlock(&Giant);
  602                                 return ((rv == VM_PAGER_ERROR) ? KERN_FAILURE : KERN_PROTECTION_FAILURE);
  603                         }
  604                         if (fs.object != fs.first_object) {
  605                                 vm_page_lock_queues();
  606                                 vm_page_free(fs.m);
  607                                 vm_page_unlock_queues();
  608                                 fs.m = NULL;
  609                                 /*
  610                                  * XXX - we cannot just fall out at this
  611                                  * point, m has been freed and is invalid!
  612                                  */
  613                         }
  614                 }
  615 
  616                 /*
  617                  * We get here if the object has default pager (or unwiring) 
  618                  * or the pager doesn't have the page.
  619                  */
  620                 if (fs.object == fs.first_object)
  621                         fs.first_m = fs.m;
  622 
  623                 /*
  624                  * Move on to the next object.  Lock the next object before
  625                  * unlocking the current one.
  626                  */
  627                 fs.pindex += OFF_TO_IDX(fs.object->backing_object_offset);
  628                 next_object = fs.object->backing_object;
  629                 if (next_object == NULL) {
  630                         /*
  631                          * If there's no object left, fill the page in the top
  632                          * object with zeros.
  633                          */
  634                         if (fs.object != fs.first_object) {
  635                                 vm_object_pip_wakeup(fs.object);
  636                                 VM_OBJECT_UNLOCK(fs.object);
  637 
  638                                 fs.object = fs.first_object;
  639                                 fs.pindex = fs.first_pindex;
  640                                 fs.m = fs.first_m;
  641                                 VM_OBJECT_LOCK(fs.object);
  642                         }
  643                         fs.first_m = NULL;
  644 
  645                         /*
  646                          * Zero the page if necessary and mark it valid.
  647                          */
  648                         if ((fs.m->flags & PG_ZERO) == 0) {
  649                                 pmap_zero_page(fs.m);
  650                         } else {
  651                                 cnt.v_ozfod++;
  652                         }
  653                         cnt.v_zfod++;
  654                         fs.m->valid = VM_PAGE_BITS_ALL;
  655                         break;  /* break to PAGE HAS BEEN FOUND */
  656                 } else {
  657                         KASSERT(fs.object != next_object,
  658                             ("object loop %p", next_object));
  659                         VM_OBJECT_LOCK(next_object);
  660                         vm_object_pip_add(next_object, 1);
  661                         if (fs.object != fs.first_object)
  662                                 vm_object_pip_wakeup(fs.object);
  663                         VM_OBJECT_UNLOCK(fs.object);
  664                         fs.object = next_object;
  665                 }
  666         }
  667 
  668         KASSERT((fs.m->flags & PG_BUSY) != 0,
  669             ("vm_fault: not busy after main loop"));
  670 
  671         /*
  672          * PAGE HAS BEEN FOUND. [Loop invariant still holds -- the object lock
  673          * is held.]
  674          */
  675 
  676         /*
  677          * If the page is being written, but isn't already owned by the
  678          * top-level object, we have to copy it into a new page owned by the
  679          * top-level object.
  680          */
  681         if (fs.object != fs.first_object) {
  682                 /*
  683                  * We only really need to copy if we want to write it.
  684                  */
  685                 if (fault_type & VM_PROT_WRITE) {
  686                         /*
  687                          * This allows pages to be virtually copied from a 
  688                          * backing_object into the first_object, where the 
  689                          * backing object has no other refs to it, and cannot
  690                          * gain any more refs.  Instead of a bcopy, we just 
  691                          * move the page from the backing object to the 
  692                          * first object.  Note that we must mark the page 
  693                          * dirty in the first object so that it will go out 
  694                          * to swap when needed.
  695                          */
  696                         is_first_object_locked = FALSE;
  697                         if (
  698                                 /*
  699                                  * Only one shadow object
  700                                  */
  701                                 (fs.object->shadow_count == 1) &&
  702                                 /*
  703                                  * No COW refs, except us
  704                                  */
  705                                 (fs.object->ref_count == 1) &&
  706                                 /*
  707                                  * No one else can look this object up
  708                                  */
  709                                 (fs.object->handle == NULL) &&
  710                                 /*
  711                                  * No other ways to look the object up
  712                                  */
  713                                 ((fs.object->type == OBJT_DEFAULT) ||
  714                                  (fs.object->type == OBJT_SWAP)) &&
  715                             (is_first_object_locked = VM_OBJECT_TRYLOCK(fs.first_object)) &&
  716                                 /*
  717                                  * We don't chase down the shadow chain
  718                                  */
  719                             fs.object == fs.first_object->backing_object) {
  720                                 vm_page_lock_queues();
  721                                 /*
  722                                  * get rid of the unnecessary page
  723                                  */
  724                                 pmap_remove_all(fs.first_m);
  725                                 vm_page_free(fs.first_m);
  726                                 /*
  727                                  * grab the page and put it into the 
  728                                  * process'es object.  The page is 
  729                                  * automatically made dirty.
  730                                  */
  731                                 vm_page_rename(fs.m, fs.first_object, fs.first_pindex);
  732                                 vm_page_busy(fs.m);
  733                                 vm_page_unlock_queues();
  734                                 fs.first_m = fs.m;
  735                                 fs.m = NULL;
  736                                 cnt.v_cow_optim++;
  737                         } else {
  738                                 /*
  739                                  * Oh, well, lets copy it.
  740                                  */
  741                                 pmap_copy_page(fs.m, fs.first_m);
  742                                 fs.first_m->valid = VM_PAGE_BITS_ALL;
  743                         }
  744                         if (fs.m) {
  745                                 /*
  746                                  * We no longer need the old page or object.
  747                                  */
  748                                 release_page(&fs);
  749                         }
  750                         /*
  751                          * fs.object != fs.first_object due to above 
  752                          * conditional
  753                          */
  754                         vm_object_pip_wakeup(fs.object);
  755                         VM_OBJECT_UNLOCK(fs.object);
  756                         /*
  757                          * Only use the new page below...
  758                          */
  759                         fs.object = fs.first_object;
  760                         fs.pindex = fs.first_pindex;
  761                         fs.m = fs.first_m;
  762                         if (!is_first_object_locked)
  763                                 VM_OBJECT_LOCK(fs.object);
  764                         cnt.v_cow_faults++;
  765                 } else {
  766                         prot &= ~VM_PROT_WRITE;
  767                 }
  768         }
  769 
  770         /*
  771          * We must verify that the maps have not changed since our last
  772          * lookup.
  773          */
  774         if (!fs.lookup_still_valid &&
  775                 (fs.map->timestamp != map_generation)) {
  776                 vm_object_t retry_object;
  777                 vm_pindex_t retry_pindex;
  778                 vm_prot_t retry_prot;
  779 
  780                 /*
  781                  * Since map entries may be pageable, make sure we can take a
  782                  * page fault on them.
  783                  */
  784 
  785                 /*
  786                  * Unlock vnode before the lookup to avoid deadlock.   E.G.
  787                  * avoid a deadlock between the inode and exec_map that can
  788                  * occur due to locks being obtained in different orders.
  789                  */
  790                 if (fs.vp != NULL) {
  791                         vput(fs.vp);
  792                         fs.vp = NULL;
  793                 }
  794                 
  795                 if (fs.map->infork) {
  796                         release_page(&fs);
  797                         unlock_and_deallocate(&fs);
  798                         goto RetryFault;
  799                 }
  800                 VM_OBJECT_UNLOCK(fs.object);
  801 
  802                 /*
  803                  * To avoid trying to write_lock the map while another process
  804                  * has it read_locked (in vm_map_wire), we do not try for
  805                  * write permission.  If the page is still writable, we will
  806                  * get write permission.  If it is not, or has been marked
  807                  * needs_copy, we enter the mapping without write permission,
  808                  * and will merely take another fault.
  809                  */
  810                 result = vm_map_lookup(&fs.map, vaddr, fault_type & ~VM_PROT_WRITE,
  811                     &fs.entry, &retry_object, &retry_pindex, &retry_prot, &wired);
  812                 map_generation = fs.map->timestamp;
  813 
  814                 VM_OBJECT_LOCK(fs.object);
  815                 /*
  816                  * If we don't need the page any longer, put it on the active
  817                  * list (the easiest thing to do here).  If no one needs it,
  818                  * pageout will grab it eventually.
  819                  */
  820                 if (result != KERN_SUCCESS) {
  821                         release_page(&fs);
  822                         unlock_and_deallocate(&fs);
  823                         mtx_unlock(&Giant);
  824                         return (result);
  825                 }
  826                 fs.lookup_still_valid = TRUE;
  827 
  828                 if ((retry_object != fs.first_object) ||
  829                     (retry_pindex != fs.first_pindex)) {
  830                         release_page(&fs);
  831                         unlock_and_deallocate(&fs);
  832                         goto RetryFault;
  833                 }
  834                 /*
  835                  * Check whether the protection has changed or the object has
  836                  * been copied while we left the map unlocked. Changing from
  837                  * read to write permission is OK - we leave the page
  838                  * write-protected, and catch the write fault. Changing from
  839                  * write to read permission means that we can't mark the page
  840                  * write-enabled after all.
  841                  */
  842                 prot &= retry_prot;
  843         }
  844 
  845         /*
  846          * Put this page into the physical map. We had to do the unlock above
  847          * because pmap_enter may cause other faults.   We don't put the page
  848          * back on the active queue until later so that the page-out daemon
  849          * won't find us (yet).
  850          */
  851 
  852         if (prot & VM_PROT_WRITE) {
  853                 vm_page_lock_queues();
  854                 vm_page_flag_set(fs.m, PG_WRITEABLE);
  855                 vm_object_set_writeable_dirty(fs.m->object);
  856 
  857                 /*
  858                  * If the fault is a write, we know that this page is being
  859                  * written NOW so dirty it explicitly to save on 
  860                  * pmap_is_modified() calls later.
  861                  *
  862                  * If this is a NOSYNC mmap we do not want to set PG_NOSYNC
  863                  * if the page is already dirty to prevent data written with
  864                  * the expectation of being synced from not being synced.
  865                  * Likewise if this entry does not request NOSYNC then make
  866                  * sure the page isn't marked NOSYNC.  Applications sharing
  867                  * data should use the same flags to avoid ping ponging.
  868                  *
  869                  * Also tell the backing pager, if any, that it should remove
  870                  * any swap backing since the page is now dirty.
  871                  */
  872                 if (fs.entry->eflags & MAP_ENTRY_NOSYNC) {
  873                         if (fs.m->dirty == 0)
  874                                 vm_page_flag_set(fs.m, PG_NOSYNC);
  875                 } else {
  876                         vm_page_flag_clear(fs.m, PG_NOSYNC);
  877                 }
  878                 vm_page_unlock_queues();
  879                 if (fault_flags & VM_FAULT_DIRTY) {
  880                         int s;
  881                         vm_page_dirty(fs.m);
  882                         s = splvm();
  883                         vm_pager_page_unswapped(fs.m);
  884                         splx(s);
  885                 }
  886         }
  887 
  888         /*
  889          * Page had better still be busy
  890          */
  891         KASSERT(fs.m->flags & PG_BUSY,
  892                 ("vm_fault: page %p not busy!", fs.m));
  893         /*
  894          * Sanity check: page must be completely valid or it is not fit to
  895          * map into user space.  vm_pager_get_pages() ensures this.
  896          */
  897         if (fs.m->valid != VM_PAGE_BITS_ALL) {
  898                 vm_page_zero_invalid(fs.m, TRUE);
  899                 printf("Warning: page %p partially invalid on fault\n", fs.m);
  900         }
  901         unlock_things(&fs);
  902 
  903         pmap_enter(fs.map->pmap, vaddr, fs.m, prot, wired);
  904         if (((fault_flags & VM_FAULT_WIRE_MASK) == 0) && (wired == 0)) {
  905                 vm_fault_prefault(fs.map->pmap, vaddr, fs.entry);
  906         }
  907         vm_page_lock_queues();
  908         vm_page_flag_clear(fs.m, PG_ZERO);
  909         vm_page_flag_set(fs.m, PG_REFERENCED);
  910 
  911         /*
  912          * If the page is not wired down, then put it where the pageout daemon
  913          * can find it.
  914          */
  915         if (fault_flags & VM_FAULT_WIRE_MASK) {
  916                 if (wired)
  917                         vm_page_wire(fs.m);
  918                 else
  919                         vm_page_unwire(fs.m, 1);
  920         } else {
  921                 vm_page_activate(fs.m);
  922         }
  923         vm_page_wakeup(fs.m);
  924         vm_page_unlock_queues();
  925 
  926         PROC_LOCK(curproc);
  927         if ((curproc->p_sflag & PS_INMEM) && curproc->p_stats) {
  928                 if (hardfault) {
  929                         curproc->p_stats->p_ru.ru_majflt++;
  930                 } else {
  931                         curproc->p_stats->p_ru.ru_minflt++;
  932                 }
  933         }
  934         PROC_UNLOCK(curproc);
  935 
  936         /*
  937          * Unlock everything, and return
  938          */
  939         vm_object_deallocate(fs.first_object);
  940         mtx_unlock(&Giant);
  941         return (KERN_SUCCESS);
  942 }
  943 
  944 /*
  945  * vm_fault_prefault provides a quick way of clustering
  946  * pagefaults into a processes address space.  It is a "cousin"
  947  * of vm_map_pmap_enter, except it runs at page fault time instead
  948  * of mmap time.
  949  */
  950 static void
  951 vm_fault_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry)
  952 {
  953         int i;
  954         vm_offset_t addr, starta;
  955         vm_pindex_t pindex;
  956         vm_page_t m, mpte;
  957         vm_object_t object;
  958 
  959         if (!curthread || (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)))
  960                 return;
  961 
  962         object = entry->object.vm_object;
  963 
  964         starta = addra - PFBAK * PAGE_SIZE;
  965         if (starta < entry->start) {
  966                 starta = entry->start;
  967         } else if (starta > addra) {
  968                 starta = 0;
  969         }
  970 
  971         mpte = NULL;
  972         for (i = 0; i < PAGEORDER_SIZE; i++) {
  973                 vm_object_t backing_object, lobject;
  974 
  975                 addr = addra + prefault_pageorder[i];
  976                 if (addr > addra + (PFFOR * PAGE_SIZE))
  977                         addr = 0;
  978 
  979                 if (addr < starta || addr >= entry->end)
  980                         continue;
  981 
  982                 if (!pmap_is_prefaultable(pmap, addr))
  983                         continue;
  984 
  985                 pindex = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT;
  986                 lobject = object;
  987                 VM_OBJECT_LOCK(lobject);
  988                 while ((m = vm_page_lookup(lobject, pindex)) == NULL &&
  989                     lobject->type == OBJT_DEFAULT &&
  990                     (backing_object = lobject->backing_object) != NULL) {
  991                         if (lobject->backing_object_offset & PAGE_MASK)
  992                                 break;
  993                         pindex += lobject->backing_object_offset >> PAGE_SHIFT;
  994                         VM_OBJECT_LOCK(backing_object);
  995                         VM_OBJECT_UNLOCK(lobject);
  996                         lobject = backing_object;
  997                 }
  998                 /*
  999                  * give-up when a page is not in memory
 1000                  */
 1001                 if (m == NULL) {
 1002                         VM_OBJECT_UNLOCK(lobject);
 1003                         break;
 1004                 }
 1005                 vm_page_lock_queues();
 1006                 if (((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) &&
 1007                         (m->busy == 0) &&
 1008                     (m->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) {
 1009 
 1010                         if ((m->queue - m->pc) == PQ_CACHE) {
 1011                                 vm_page_deactivate(m);
 1012                         }
 1013                         vm_page_busy(m);
 1014                         vm_page_unlock_queues();
 1015                         VM_OBJECT_UNLOCK(lobject);
 1016                         mpte = pmap_enter_quick(pmap, addr, m, mpte);
 1017                         VM_OBJECT_LOCK(lobject);
 1018                         vm_page_lock_queues();
 1019                         vm_page_wakeup(m);
 1020                 }
 1021                 vm_page_unlock_queues();
 1022                 VM_OBJECT_UNLOCK(lobject);
 1023         }
 1024 }
 1025 
 1026 /*
 1027  *      vm_fault_quick:
 1028  *
 1029  *      Ensure that the requested virtual address, which may be in userland,
 1030  *      is valid.  Fault-in the page if necessary.  Return -1 on failure.
 1031  */
 1032 int
 1033 vm_fault_quick(caddr_t v, int prot)
 1034 {
 1035         int r;
 1036 
 1037         if (prot & VM_PROT_WRITE)
 1038                 r = subyte(v, fubyte(v));
 1039         else
 1040                 r = fubyte(v);
 1041         return(r);
 1042 }
 1043 
 1044 /*
 1045  *      vm_fault_wire:
 1046  *
 1047  *      Wire down a range of virtual addresses in a map.
 1048  */
 1049 int
 1050 vm_fault_wire(map, start, end, user_wire)
 1051         vm_map_t map;
 1052         vm_offset_t start, end;
 1053         boolean_t user_wire;
 1054 {
 1055         vm_offset_t va;
 1056         int rv;
 1057 
 1058         /*
 1059          * We simulate a fault to get the page and enter it in the physical
 1060          * map.  For user wiring, we only ask for read access on currently
 1061          * read-only sections.
 1062          */
 1063         for (va = start; va < end; va += PAGE_SIZE) {
 1064                 rv = vm_fault(map, va,
 1065                     user_wire ? VM_PROT_READ : VM_PROT_READ | VM_PROT_WRITE,
 1066                     user_wire ? VM_FAULT_USER_WIRE : VM_FAULT_CHANGE_WIRING);
 1067                 if (rv) {
 1068                         if (va != start)
 1069                                 vm_fault_unwire(map, start, va);
 1070                         return (rv);
 1071                 }
 1072         }
 1073         return (KERN_SUCCESS);
 1074 }
 1075 
 1076 /*
 1077  *      vm_fault_unwire:
 1078  *
 1079  *      Unwire a range of virtual addresses in a map.
 1080  */
 1081 void
 1082 vm_fault_unwire(map, start, end)
 1083         vm_map_t map;
 1084         vm_offset_t start, end;
 1085 {
 1086         vm_paddr_t pa;
 1087         vm_offset_t va;
 1088         pmap_t pmap;
 1089 
 1090         pmap = vm_map_pmap(map);
 1091 
 1092         mtx_lock(&Giant);
 1093         /*
 1094          * Since the pages are wired down, we must be able to get their
 1095          * mappings from the physical map system.
 1096          */
 1097         for (va = start; va < end; va += PAGE_SIZE) {
 1098                 pa = pmap_extract(pmap, va);
 1099                 if (pa != 0) {
 1100                         pmap_change_wiring(pmap, va, FALSE);
 1101                         vm_page_lock_queues();
 1102                         vm_page_unwire(PHYS_TO_VM_PAGE(pa), 1);
 1103                         vm_page_unlock_queues();
 1104                 }
 1105         }
 1106         mtx_unlock(&Giant);
 1107 }
 1108 
 1109 /*
 1110  *      Routine:
 1111  *              vm_fault_copy_entry
 1112  *      Function:
 1113  *              Copy all of the pages from a wired-down map entry to another.
 1114  *
 1115  *      In/out conditions:
 1116  *              The source and destination maps must be locked for write.
 1117  *              The source map entry must be wired down (or be a sharing map
 1118  *              entry corresponding to a main map entry that is wired down).
 1119  */
 1120 void
 1121 vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry)
 1122         vm_map_t dst_map;
 1123         vm_map_t src_map;
 1124         vm_map_entry_t dst_entry;
 1125         vm_map_entry_t src_entry;
 1126 {
 1127         vm_object_t backing_object, dst_object, object;
 1128         vm_object_t src_object;
 1129         vm_ooffset_t dst_offset;
 1130         vm_ooffset_t src_offset;
 1131         vm_pindex_t pindex;
 1132         vm_prot_t prot;
 1133         vm_offset_t vaddr;
 1134         vm_page_t dst_m;
 1135         vm_page_t src_m;
 1136 
 1137 #ifdef  lint
 1138         src_map++;
 1139 #endif  /* lint */
 1140 
 1141         src_object = src_entry->object.vm_object;
 1142         src_offset = src_entry->offset;
 1143 
 1144         /*
 1145          * Create the top-level object for the destination entry. (Doesn't
 1146          * actually shadow anything - we copy the pages directly.)
 1147          */
 1148         dst_object = vm_object_allocate(OBJT_DEFAULT,
 1149             (vm_size_t) OFF_TO_IDX(dst_entry->end - dst_entry->start));
 1150 
 1151         VM_OBJECT_LOCK(dst_object);
 1152         dst_entry->object.vm_object = dst_object;
 1153         dst_entry->offset = 0;
 1154 
 1155         prot = dst_entry->max_protection;
 1156 
 1157         /*
 1158          * Loop through all of the pages in the entry's range, copying each
 1159          * one from the source object (it should be there) to the destination
 1160          * object.
 1161          */
 1162         for (vaddr = dst_entry->start, dst_offset = 0;
 1163             vaddr < dst_entry->end;
 1164             vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) {
 1165 
 1166                 /*
 1167                  * Allocate a page in the destination object
 1168                  */
 1169                 do {
 1170                         dst_m = vm_page_alloc(dst_object,
 1171                                 OFF_TO_IDX(dst_offset), VM_ALLOC_NORMAL);
 1172                         if (dst_m == NULL) {
 1173                                 VM_OBJECT_UNLOCK(dst_object);
 1174                                 VM_WAIT;
 1175                                 VM_OBJECT_LOCK(dst_object);
 1176                         }
 1177                 } while (dst_m == NULL);
 1178 
 1179                 /*
 1180                  * Find the page in the source object, and copy it in.
 1181                  * (Because the source is wired down, the page will be in
 1182                  * memory.)
 1183                  */
 1184                 VM_OBJECT_LOCK(src_object);
 1185                 object = src_object;
 1186                 pindex = 0;
 1187                 while ((src_m = vm_page_lookup(object, pindex +
 1188                     OFF_TO_IDX(dst_offset + src_offset))) == NULL &&
 1189                     (src_entry->protection & VM_PROT_WRITE) == 0 &&
 1190                     (backing_object = object->backing_object) != NULL) {
 1191                         /*
 1192                          * Allow fallback to backing objects if we are reading.
 1193                          */
 1194                         VM_OBJECT_LOCK(backing_object);
 1195                         pindex += OFF_TO_IDX(object->backing_object_offset);
 1196                         VM_OBJECT_UNLOCK(object);
 1197                         object = backing_object;
 1198                 }
 1199                 if (src_m == NULL)
 1200                         panic("vm_fault_copy_wired: page missing");
 1201                 pmap_copy_page(src_m, dst_m);
 1202                 VM_OBJECT_UNLOCK(object);
 1203                 dst_m->valid = VM_PAGE_BITS_ALL;
 1204                 VM_OBJECT_UNLOCK(dst_object);
 1205 
 1206                 /*
 1207                  * Enter it in the pmap...
 1208                  */
 1209                 pmap_enter(dst_map->pmap, vaddr, dst_m, prot, FALSE);
 1210                 VM_OBJECT_LOCK(dst_object);
 1211                 vm_page_lock_queues();
 1212                 if ((prot & VM_PROT_WRITE) != 0)
 1213                         vm_page_flag_set(dst_m, PG_WRITEABLE);
 1214 
 1215                 /*
 1216                  * Mark it no longer busy, and put it on the active list.
 1217                  */
 1218                 vm_page_activate(dst_m);
 1219                 vm_page_wakeup(dst_m);
 1220                 vm_page_unlock_queues();
 1221         }
 1222         VM_OBJECT_UNLOCK(dst_object);
 1223 }
 1224 
 1225 
 1226 /*
 1227  * This routine checks around the requested page for other pages that
 1228  * might be able to be faulted in.  This routine brackets the viable
 1229  * pages for the pages to be paged in.
 1230  *
 1231  * Inputs:
 1232  *      m, rbehind, rahead
 1233  *
 1234  * Outputs:
 1235  *  marray (array of vm_page_t), reqpage (index of requested page)
 1236  *
 1237  * Return value:
 1238  *  number of pages in marray
 1239  *
 1240  * This routine can't block.
 1241  */
 1242 static int
 1243 vm_fault_additional_pages(m, rbehind, rahead, marray, reqpage)
 1244         vm_page_t m;
 1245         int rbehind;
 1246         int rahead;
 1247         vm_page_t *marray;
 1248         int *reqpage;
 1249 {
 1250         int i,j;
 1251         vm_object_t object;
 1252         vm_pindex_t pindex, startpindex, endpindex, tpindex;
 1253         vm_page_t rtm;
 1254         int cbehind, cahead;
 1255 
 1256         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 1257 
 1258         object = m->object;
 1259         pindex = m->pindex;
 1260 
 1261         /*
 1262          * we don't fault-ahead for device pager
 1263          */
 1264         if (object->type == OBJT_DEVICE) {
 1265                 *reqpage = 0;
 1266                 marray[0] = m;
 1267                 return 1;
 1268         }
 1269 
 1270         /*
 1271          * if the requested page is not available, then give up now
 1272          */
 1273         if (!vm_pager_has_page(object, pindex, &cbehind, &cahead)) {
 1274                 return 0;
 1275         }
 1276 
 1277         if ((cbehind == 0) && (cahead == 0)) {
 1278                 *reqpage = 0;
 1279                 marray[0] = m;
 1280                 return 1;
 1281         }
 1282 
 1283         if (rahead > cahead) {
 1284                 rahead = cahead;
 1285         }
 1286 
 1287         if (rbehind > cbehind) {
 1288                 rbehind = cbehind;
 1289         }
 1290 
 1291         /*
 1292          * try to do any readahead that we might have free pages for.
 1293          */
 1294         if ((rahead + rbehind) >
 1295                 ((cnt.v_free_count + cnt.v_cache_count) - cnt.v_free_reserved)) {
 1296                 pagedaemon_wakeup();
 1297                 marray[0] = m;
 1298                 *reqpage = 0;
 1299                 return 1;
 1300         }
 1301 
 1302         /*
 1303          * scan backward for the read behind pages -- in memory 
 1304          */
 1305         if (pindex > 0) {
 1306                 if (rbehind > pindex) {
 1307                         rbehind = pindex;
 1308                         startpindex = 0;
 1309                 } else {
 1310                         startpindex = pindex - rbehind;
 1311                 }
 1312 
 1313                 for (tpindex = pindex - 1; tpindex >= startpindex; tpindex -= 1) {
 1314                         if (vm_page_lookup(object, tpindex)) {
 1315                                 startpindex = tpindex + 1;
 1316                                 break;
 1317                         }
 1318                         if (tpindex == 0)
 1319                                 break;
 1320                 }
 1321 
 1322                 for (i = 0, tpindex = startpindex; tpindex < pindex; i++, tpindex++) {
 1323 
 1324                         rtm = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);
 1325                         if (rtm == NULL) {
 1326                                 vm_page_lock_queues();
 1327                                 for (j = 0; j < i; j++) {
 1328                                         vm_page_free(marray[j]);
 1329                                 }
 1330                                 vm_page_unlock_queues();
 1331                                 marray[0] = m;
 1332                                 *reqpage = 0;
 1333                                 return 1;
 1334                         }
 1335 
 1336                         marray[i] = rtm;
 1337                 }
 1338         } else {
 1339                 startpindex = 0;
 1340                 i = 0;
 1341         }
 1342 
 1343         marray[i] = m;
 1344         /* page offset of the required page */
 1345         *reqpage = i;
 1346 
 1347         tpindex = pindex + 1;
 1348         i++;
 1349 
 1350         /*
 1351          * scan forward for the read ahead pages
 1352          */
 1353         endpindex = tpindex + rahead;
 1354         if (endpindex > object->size)
 1355                 endpindex = object->size;
 1356 
 1357         for (; tpindex < endpindex; i++, tpindex++) {
 1358 
 1359                 if (vm_page_lookup(object, tpindex)) {
 1360                         break;
 1361                 }
 1362 
 1363                 rtm = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);
 1364                 if (rtm == NULL) {
 1365                         break;
 1366                 }
 1367 
 1368                 marray[i] = rtm;
 1369         }
 1370 
 1371         /* return number of bytes of pages */
 1372         return i;
 1373 }

Cache object: 36e91e3ec36227cd731934838bc96c3b


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.