The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/uvm/uvm_pager.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*      $NetBSD: uvm_pager.c,v 1.65 2005/01/01 21:00:06 yamt Exp $      */
    2 
    3 /*
    4  *
    5  * Copyright (c) 1997 Charles D. Cranor and Washington University.
    6  * All rights reserved.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  * 3. All advertising materials mentioning features or use of this software
   17  *    must display the following acknowledgement:
   18  *      This product includes software developed by Charles D. Cranor and
   19  *      Washington University.
   20  * 4. The name of the author may not be used to endorse or promote products
   21  *    derived from this software without specific prior written permission.
   22  *
   23  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   24  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   25  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   26  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   27  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   28  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   29  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   30  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   31  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
   32  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   33  *
   34  * from: Id: uvm_pager.c,v 1.1.2.23 1998/02/02 20:38:06 chuck Exp
   35  */
   36 
   37 /*
   38  * uvm_pager.c: generic functions used to assist the pagers.
   39  */
   40 
   41 #include <sys/cdefs.h>
   42 __KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.65 2005/01/01 21:00:06 yamt Exp $");
   43 
   44 #include "opt_uvmhist.h"
   45 
   46 #include <sys/param.h>
   47 #include <sys/systm.h>
   48 #include <sys/proc.h>
   49 #include <sys/malloc.h>
   50 #include <sys/pool.h>
   51 #include <sys/vnode.h>
   52 
   53 #define UVM_PAGER
   54 #include <uvm/uvm.h>
   55 
   56 struct pool *uvm_aiobuf_pool;
   57 
   58 /*
   59  * list of uvm pagers in the system
   60  */
   61 
   62 struct uvm_pagerops * const uvmpagerops[] = {
   63         &aobj_pager,
   64         &uvm_deviceops,
   65         &uvm_vnodeops,
   66         &ubc_pager,
   67 };
   68 
   69 /*
   70  * the pager map: provides KVA for I/O
   71  */
   72 
   73 struct vm_map *pager_map;               /* XXX */
   74 struct simplelock pager_map_wanted_lock;
   75 boolean_t pager_map_wanted;     /* locked by pager map */
   76 static vaddr_t emergva;
   77 static boolean_t emerginuse;
   78 
   79 /*
   80  * uvm_pager_init: init pagers (at boot time)
   81  */
   82 
   83 void
   84 uvm_pager_init()
   85 {
   86         u_int lcv;
   87         vaddr_t sva, eva;
   88 
   89         /*
   90          * init pager map
   91          */
   92 
   93         sva = 0;
   94         pager_map = uvm_km_suballoc(kernel_map, &sva, &eva, PAGER_MAP_SIZE, 0,
   95             FALSE, NULL);
   96         simple_lock_init(&pager_map_wanted_lock);
   97         pager_map_wanted = FALSE;
   98         emergva = uvm_km_valloc(kernel_map, round_page(MAXPHYS));
   99         emerginuse = FALSE;
  100 
  101         /*
  102          * init ASYNC I/O queue
  103          */
  104 
  105         TAILQ_INIT(&uvm.aio_done);
  106 
  107         /*
  108          * call pager init functions
  109          */
  110         for (lcv = 0 ; lcv < sizeof(uvmpagerops)/sizeof(struct uvm_pagerops *);
  111             lcv++) {
  112                 if (uvmpagerops[lcv]->pgo_init)
  113                         uvmpagerops[lcv]->pgo_init();
  114         }
  115 }
  116 
  117 /*
  118  * uvm_pagermapin: map pages into KVA (pager_map) for I/O that needs mappings
  119  *
  120  * we basically just map in a blank map entry to reserve the space in the
  121  * map and then use pmap_enter() to put the mappings in by hand.
  122  */
  123 
  124 vaddr_t
  125 uvm_pagermapin(pps, npages, flags)
  126         struct vm_page **pps;
  127         int npages;
  128         int flags;
  129 {
  130         vsize_t size;
  131         vaddr_t kva;
  132         vaddr_t cva;
  133         struct vm_page *pp;
  134         vm_prot_t prot;
  135         UVMHIST_FUNC("uvm_pagermapin"); UVMHIST_CALLED(maphist);
  136 
  137         UVMHIST_LOG(maphist,"(pps=0x%x, npages=%d)", pps, npages,0,0);
  138 
  139         /*
  140          * compute protection.  outgoing I/O only needs read
  141          * access to the page, whereas incoming needs read/write.
  142          */
  143 
  144         prot = VM_PROT_READ;
  145         if (flags & UVMPAGER_MAPIN_READ)
  146                 prot |= VM_PROT_WRITE;
  147 
  148 ReStart:
  149         size = npages << PAGE_SHIFT;
  150         kva = 0;                        /* let system choose VA */
  151 
  152         if (uvm_map(pager_map, &kva, size, NULL,
  153               UVM_UNKNOWN_OFFSET, 0, UVM_FLAG_NOMERGE) != 0) {
  154                 if (curproc == uvm.pagedaemon_proc) {
  155                         simple_lock(&pager_map_wanted_lock);
  156                         if (emerginuse) {
  157                                 UVM_UNLOCK_AND_WAIT(&emergva,
  158                                     &pager_map_wanted_lock, FALSE,
  159                                     "emergva", 0);
  160                                 goto ReStart;
  161                         }
  162                         emerginuse = TRUE;
  163                         simple_unlock(&pager_map_wanted_lock);
  164                         kva = emergva;
  165                         /* The shift implicitly truncates to PAGE_SIZE */
  166                         KASSERT(npages <= (MAXPHYS >> PAGE_SHIFT));
  167                         goto enter;
  168                 }
  169                 if ((flags & UVMPAGER_MAPIN_WAITOK) == 0) {
  170                         UVMHIST_LOG(maphist,"<- NOWAIT failed", 0,0,0,0);
  171                         return(0);
  172                 }
  173                 simple_lock(&pager_map_wanted_lock);
  174                 pager_map_wanted = TRUE;
  175                 UVMHIST_LOG(maphist, "  SLEEPING on pager_map",0,0,0,0);
  176                 UVM_UNLOCK_AND_WAIT(pager_map, &pager_map_wanted_lock, FALSE,
  177                     "pager_map", 0);
  178                 goto ReStart;
  179         }
  180 
  181 enter:
  182         /* got it */
  183         for (cva = kva ; size != 0 ; size -= PAGE_SIZE, cva += PAGE_SIZE) {
  184                 pp = *pps++;
  185                 KASSERT(pp);
  186                 KASSERT(pp->flags & PG_BUSY);
  187                 pmap_kenter_pa(cva, VM_PAGE_TO_PHYS(pp), prot);
  188         }
  189         pmap_update(vm_map_pmap(pager_map));
  190 
  191         UVMHIST_LOG(maphist, "<- done (KVA=0x%x)", kva,0,0,0);
  192         return(kva);
  193 }
  194 
  195 /*
  196  * uvm_pagermapout: remove pager_map mapping
  197  *
  198  * we remove our mappings by hand and then remove the mapping (waking
  199  * up anyone wanting space).
  200  */
  201 
  202 void
  203 uvm_pagermapout(kva, npages)
  204         vaddr_t kva;
  205         int npages;
  206 {
  207         vsize_t size = npages << PAGE_SHIFT;
  208         struct vm_map_entry *entries;
  209         UVMHIST_FUNC("uvm_pagermapout"); UVMHIST_CALLED(maphist);
  210 
  211         UVMHIST_LOG(maphist, " (kva=0x%x, npages=%d)", kva, npages,0,0);
  212 
  213         /*
  214          * duplicate uvm_unmap, but add in pager_map_wanted handling.
  215          */
  216 
  217         pmap_kremove(kva, npages << PAGE_SHIFT);
  218         if (kva == emergva) {
  219                 simple_lock(&pager_map_wanted_lock);
  220                 emerginuse = FALSE;
  221                 wakeup(&emergva);
  222                 simple_unlock(&pager_map_wanted_lock);
  223                 return;
  224         }
  225 
  226         vm_map_lock(pager_map);
  227         uvm_unmap_remove(pager_map, kva, kva + size, &entries, NULL);
  228         simple_lock(&pager_map_wanted_lock);
  229         if (pager_map_wanted) {
  230                 pager_map_wanted = FALSE;
  231                 wakeup(pager_map);
  232         }
  233         simple_unlock(&pager_map_wanted_lock);
  234         vm_map_unlock(pager_map);
  235         if (entries)
  236                 uvm_unmap_detach(entries, 0);
  237         pmap_update(pmap_kernel());
  238         UVMHIST_LOG(maphist,"<- done",0,0,0,0);
  239 }
  240 
  241 /*
  242  * interrupt-context iodone handler for nested i/o bufs.
  243  *
  244  * => must be at splbio().
  245  */
  246 
  247 void
  248 uvm_aio_biodone1(bp)
  249         struct buf *bp;
  250 {
  251         struct buf *mbp = bp->b_private;
  252 
  253         KASSERT(mbp != bp);
  254         if (bp->b_flags & B_ERROR) {
  255                 mbp->b_flags |= B_ERROR;
  256                 mbp->b_error = bp->b_error;
  257         }
  258         mbp->b_resid -= bp->b_bcount;
  259         pool_put(&bufpool, bp);
  260         if (mbp->b_resid == 0) {
  261                 biodone(mbp);
  262         }
  263 }
  264 
  265 /*
  266  * interrupt-context iodone handler for single-buf i/os
  267  * or the top-level buf of a nested-buf i/o.
  268  *
  269  * => must be at splbio().
  270  */
  271 
  272 void
  273 uvm_aio_biodone(bp)
  274         struct buf *bp;
  275 {
  276         /* reset b_iodone for when this is a single-buf i/o. */
  277         bp->b_iodone = uvm_aio_aiodone;
  278 
  279         simple_lock(&uvm.aiodoned_lock);        /* locks uvm.aio_done */
  280         TAILQ_INSERT_TAIL(&uvm.aio_done, bp, b_freelist);
  281         wakeup(&uvm.aiodoned);
  282         simple_unlock(&uvm.aiodoned_lock);
  283 }
  284 
  285 /*
  286  * uvm_aio_aiodone: do iodone processing for async i/os.
  287  * this should be called in thread context, not interrupt context.
  288  */
  289 
  290 void
  291 uvm_aio_aiodone(bp)
  292         struct buf *bp;
  293 {
  294         int npages = bp->b_bufsize >> PAGE_SHIFT;
  295         struct vm_page *pg, *pgs[npages];
  296         struct uvm_object *uobj;
  297         struct simplelock *slock;
  298         int s, i, error, swslot;
  299         boolean_t write, swap;
  300         UVMHIST_FUNC("uvm_aio_aiodone"); UVMHIST_CALLED(ubchist);
  301         UVMHIST_LOG(ubchist, "bp %p", bp, 0,0,0);
  302 
  303         error = (bp->b_flags & B_ERROR) ? (bp->b_error ? bp->b_error : EIO) : 0;
  304         write = (bp->b_flags & B_READ) == 0;
  305         /* XXXUBC B_NOCACHE is for swap pager, should be done differently */
  306         if (write && !(bp->b_flags & B_NOCACHE) && bioops.io_pageiodone) {
  307                 (*bioops.io_pageiodone)(bp);
  308         }
  309 
  310         uobj = NULL;
  311         for (i = 0; i < npages; i++) {
  312                 pgs[i] = uvm_pageratop((vaddr_t)bp->b_data + (i << PAGE_SHIFT));
  313                 UVMHIST_LOG(ubchist, "pgs[%d] = %p", i, pgs[i],0,0);
  314         }
  315         uvm_pagermapout((vaddr_t)bp->b_data, npages);
  316 
  317         swslot = 0;
  318         slock = NULL;
  319         pg = pgs[0];
  320         swap = (pg->uanon != NULL && pg->uobject == NULL) ||
  321                 (pg->pqflags & PQ_AOBJ) != 0;
  322         if (!swap) {
  323                 uobj = pg->uobject;
  324                 slock = &uobj->vmobjlock;
  325                 simple_lock(slock);
  326                 uvm_lock_pageq();
  327         } else if (error) {
  328                 if (pg->uobject != NULL) {
  329                         swslot = uao_find_swslot(pg->uobject,
  330                             pg->offset >> PAGE_SHIFT);
  331                 } else {
  332                         swslot = pg->uanon->an_swslot;
  333                 }
  334                 KASSERT(swslot);
  335         }
  336         for (i = 0; i < npages; i++) {
  337                 pg = pgs[i];
  338                 KASSERT(swap || pg->uobject == uobj);
  339                 UVMHIST_LOG(ubchist, "pg %p", pg, 0,0,0);
  340 
  341                 /*
  342                  * for swap i/os, lock each page's object (or anon)
  343                  * individually since each page may need a different lock.
  344                  */
  345 
  346                 if (swap) {
  347                         if (pg->uobject != NULL) {
  348                                 slock = &pg->uobject->vmobjlock;
  349                         } else {
  350                                 slock = &pg->uanon->an_lock;
  351                         }
  352                         simple_lock(slock);
  353                         uvm_lock_pageq();
  354                 }
  355 
  356                 /*
  357                  * process errors.  for reads, just mark the page to be freed.
  358                  * for writes, if the error was ENOMEM, we assume this was
  359                  * a transient failure so we mark the page dirty so that
  360                  * we'll try to write it again later.  for all other write
  361                  * errors, we assume the error is permanent, thus the data
  362                  * in the page is lost.  bummer.
  363                  */
  364 
  365                 if (error) {
  366                         int slot;
  367                         if (!write) {
  368                                 pg->flags |= PG_RELEASED;
  369                                 continue;
  370                         } else if (error == ENOMEM) {
  371                                 if (pg->flags & PG_PAGEOUT) {
  372                                         pg->flags &= ~PG_PAGEOUT;
  373                                         uvmexp.paging--;
  374                                 }
  375                                 pg->flags &= ~PG_CLEAN;
  376                                 uvm_pageactivate(pg);
  377                                 slot = 0;
  378                         } else
  379                                 slot = SWSLOT_BAD;
  380 
  381                         if (swap) {
  382                                 if (pg->uobject != NULL) {
  383                                         int oldslot;
  384                                         oldslot = uao_set_swslot(pg->uobject,
  385                                                 pg->offset >> PAGE_SHIFT, slot);
  386                                         KASSERT(oldslot == swslot + i);
  387                                 } else {
  388                                         KASSERT(pg->uanon->an_swslot ==
  389                                                 swslot + i);
  390                                         pg->uanon->an_swslot = slot;
  391                                 }
  392                         }
  393                 }
  394 
  395                 /*
  396                  * if the page is PG_FAKE, this must have been a read to
  397                  * initialize the page.  clear PG_FAKE and activate the page.
  398                  * we must also clear the pmap "modified" flag since it may
  399                  * still be set from the page's previous identity.
  400                  */
  401 
  402                 if (pg->flags & PG_FAKE) {
  403                         KASSERT(!write);
  404                         pg->flags &= ~PG_FAKE;
  405                         uvm_pageactivate(pg);
  406                         pmap_clear_modify(pg);
  407                 }
  408 
  409                 /*
  410                  * do accounting for pagedaemon i/o and arrange to free
  411                  * the pages instead of just unbusying them.
  412                  */
  413 
  414                 if (pg->flags & PG_PAGEOUT) {
  415                         pg->flags &= ~PG_PAGEOUT;
  416                         uvmexp.paging--;
  417                         uvmexp.pdfreed++;
  418                         pg->flags |= PG_RELEASED;
  419                 }
  420 
  421                 /*
  422                  * for swap pages, unlock everything for this page now.
  423                  */
  424 
  425                 if (swap) {
  426                         if (pg->uobject == NULL && pg->uanon->an_ref == 0 &&
  427                             (pg->flags & PG_RELEASED) != 0) {
  428                                 uvm_unlock_pageq();
  429                                 uvm_anon_release(pg->uanon);
  430                         } else {
  431                                 uvm_page_unbusy(&pg, 1);
  432                                 uvm_unlock_pageq();
  433                                 simple_unlock(slock);
  434                         }
  435                 }
  436         }
  437         if (!swap) {
  438                 uvm_page_unbusy(pgs, npages);
  439                 uvm_unlock_pageq();
  440                 simple_unlock(slock);
  441         } else {
  442                 KASSERT(write);
  443 
  444                 /* these pages are now only in swap. */
  445                 simple_lock(&uvm.swap_data_lock);
  446                 KASSERT(uvmexp.swpgonly + npages <= uvmexp.swpginuse);
  447                 if (error != ENOMEM)
  448                         uvmexp.swpgonly += npages;
  449                 simple_unlock(&uvm.swap_data_lock);
  450                 if (error) {
  451                         if (error != ENOMEM)
  452                                 uvm_swap_markbad(swslot, npages);
  453                         else
  454                                 uvm_swap_free(swslot, npages);
  455                 }
  456                 uvmexp.pdpending--;
  457         }
  458         s = splbio();
  459         if (write && (bp->b_flags & B_AGE) != 0) {
  460                 vwakeup(bp);
  461         }
  462         pool_put(&bufpool, bp);
  463         splx(s);
  464 }

Cache object: 0ae9f55ed26f26afe7ff650c1a3dbb8c


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.