The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/uvm/uvm_loan.c

Version: -  FREEBSD  -  FREEBSD11  -  FREEBSD10  -  FREEBSD9  -  FREEBSD92  -  FREEBSD91  -  FREEBSD90  -  FREEBSD8  -  FREEBSD82  -  FREEBSD81  -  FREEBSD80  -  FREEBSD7  -  FREEBSD74  -  FREEBSD73  -  FREEBSD72  -  FREEBSD71  -  FREEBSD70  -  FREEBSD6  -  FREEBSD64  -  FREEBSD63  -  FREEBSD62  -  FREEBSD61  -  FREEBSD60  -  FREEBSD5  -  FREEBSD55  -  FREEBSD54  -  FREEBSD53  -  FREEBSD52  -  FREEBSD51  -  FREEBSD50  -  FREEBSD4  -  FREEBSD3  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*      $NetBSD: uvm_loan.c,v 1.62 2006/11/01 10:18:27 yamt Exp $       */
    2 
    3 /*
    4  *
    5  * Copyright (c) 1997 Charles D. Cranor and Washington University.
    6  * All rights reserved.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  * 3. All advertising materials mentioning features or use of this software
   17  *    must display the following acknowledgement:
   18  *      This product includes software developed by Charles D. Cranor and
   19  *      Washington University.
   20  * 4. The name of the author may not be used to endorse or promote products
   21  *    derived from this software without specific prior written permission.
   22  *
   23  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   24  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   25  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   26  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   27  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   28  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   29  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   30  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   31  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
   32  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   33  *
   34  * from: Id: uvm_loan.c,v 1.1.6.4 1998/02/06 05:08:43 chs Exp
   35  */
   36 
   37 /*
   38  * uvm_loan.c: page loanout handler
   39  */
   40 
   41 #include <sys/cdefs.h>
   42 __KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.62 2006/11/01 10:18:27 yamt Exp $");
   43 
   44 #include <sys/param.h>
   45 #include <sys/systm.h>
   46 #include <sys/kernel.h>
   47 #include <sys/proc.h>
   48 #include <sys/malloc.h>
   49 #include <sys/mman.h>
   50 
   51 #include <uvm/uvm.h>
   52 
   53 /*
   54  * "loaned" pages are pages which are (read-only, copy-on-write) loaned
   55  * from the VM system to other parts of the kernel.   this allows page
   56  * copying to be avoided (e.g. you can loan pages from objs/anons to
   57  * the mbuf system).
   58  *
   59  * there are 3 types of loans possible:
   60  *  O->K  uvm_object page to wired kernel page (e.g. mbuf data area)
   61  *  A->K  anon page to wired kernel page (e.g. mbuf data area)
   62  *  O->A  uvm_object to anon loan (e.g. vnode page to an anon)
   63  * note that it possible to have an O page loaned to both an A and K
   64  * at the same time.
   65  *
   66  * loans are tracked by pg->loan_count.  an O->A page will have both
   67  * a uvm_object and a vm_anon, but PQ_ANON will not be set.   this sort
   68  * of page is considered "owned" by the uvm_object (not the anon).
   69  *
   70  * each loan of a page to the kernel bumps the pg->wire_count.  the
   71  * kernel mappings for these pages will be read-only and wired.  since
   72  * the page will also be wired, it will not be a candidate for pageout,
   73  * and thus will never be pmap_page_protect()'d with VM_PROT_NONE.  a
   74  * write fault in the kernel to one of these pages will not cause
   75  * copy-on-write.  instead, the page fault is considered fatal.  this
   76  * is because the kernel mapping will have no way to look up the
   77  * object/anon which the page is owned by.  this is a good side-effect,
   78  * since a kernel write to a loaned page is an error.
   79  *
   80  * owners that want to free their pages and discover that they are
   81  * loaned out simply "disown" them (the page becomes an orphan).  these
   82  * pages should be freed when the last loan is dropped.   in some cases
   83  * an anon may "adopt" an orphaned page.
   84  *
   85  * locking: to read pg->loan_count either the owner or the page queues
   86  * must be locked.   to modify pg->loan_count, both the owner of the page
   87  * and the PQs must be locked.   pg->flags is (as always) locked by
   88  * the owner of the page.
   89  *
   90  * note that locking from the "loaned" side is tricky since the object
   91  * getting the loaned page has no reference to the page's owner and thus
   92  * the owner could "die" at any time.   in order to prevent the owner
   93  * from dying the page queues should be locked.   this forces us to sometimes
   94  * use "try" locking.
   95  *
   96  * loans are typically broken by the following events:
   97  *  1. user-level xwrite fault to a loaned page
   98  *  2. pageout of clean+inactive O->A loaned page
   99  *  3. owner frees page (e.g. pager flush)
  100  *
  101  * note that loaning a page causes all mappings of the page to become
  102  * read-only (via pmap_page_protect).   this could have an unexpected
  103  * effect on normal "wired" pages if one is not careful (XXX).
  104  */
  105 
  106 /*
  107  * local prototypes
  108  */
  109 
  110 static int      uvm_loananon(struct uvm_faultinfo *, void ***,
  111                              int, struct vm_anon *);
  112 static int      uvm_loanuobj(struct uvm_faultinfo *, void ***,
  113                              int, vaddr_t);
  114 static int      uvm_loanzero(struct uvm_faultinfo *, void ***, int);
  115 static void     uvm_unloananon(struct vm_anon **, int);
  116 static void     uvm_unloanpage(struct vm_page **, int);
  117 static int      uvm_loanpage(struct vm_page **, int);
  118 
  119 
  120 /*
  121  * inlines
  122  */
  123 
  124 /*
  125  * uvm_loanentry: loan out pages in a map entry (helper fn for uvm_loan())
  126  *
  127  * => "ufi" is the result of a successful map lookup (meaning that
  128  *      on entry the map is locked by the caller)
  129  * => we may unlock and then relock the map if needed (for I/O)
  130  * => we put our output result in "output"
  131  * => we always return with the map unlocked
  132  * => possible return values:
  133  *      -1 == error, map is unlocked
  134  *       0 == map relock error (try again!), map is unlocked
  135  *      >0 == number of pages we loaned, map is unlocked
  136  *
  137  * NOTE: We can live with this being an inline, because it is only called
  138  * from one place.
  139  */
  140 
  141 static inline int
  142 uvm_loanentry(struct uvm_faultinfo *ufi, void ***output, int flags)
  143 {
  144         vaddr_t curaddr = ufi->orig_rvaddr;
  145         vsize_t togo = ufi->size;
  146         struct vm_aref *aref = &ufi->entry->aref;
  147         struct uvm_object *uobj = ufi->entry->object.uvm_obj;
  148         struct vm_anon *anon;
  149         int rv, result = 0;
  150 
  151         UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
  152 
  153         /*
  154          * lock us the rest of the way down (we unlock before return)
  155          */
  156         if (aref->ar_amap)
  157                 amap_lock(aref->ar_amap);
  158 
  159         /*
  160          * loop until done
  161          */
  162         while (togo) {
  163 
  164                 /*
  165                  * find the page we want.   check the anon layer first.
  166                  */
  167 
  168                 if (aref->ar_amap) {
  169                         anon = amap_lookup(aref, curaddr - ufi->entry->start);
  170                 } else {
  171                         anon = NULL;
  172                 }
  173 
  174                 /* locked: map, amap, uobj */
  175                 if (anon) {
  176                         rv = uvm_loananon(ufi, output, flags, anon);
  177                 } else if (uobj) {
  178                         rv = uvm_loanuobj(ufi, output, flags, curaddr);
  179                 } else if (UVM_ET_ISCOPYONWRITE(ufi->entry)) {
  180                         rv = uvm_loanzero(ufi, output, flags);
  181                 } else {
  182                         uvmfault_unlockall(ufi, aref->ar_amap, uobj, NULL);
  183                         rv = -1;
  184                 }
  185                 /* locked: if (rv > 0) => map, amap, uobj  [o.w. unlocked] */
  186                 LOCK_ASSERT(rv > 0 || aref->ar_amap == NULL ||
  187                     !simple_lock_held(&aref->ar_amap->am_l));
  188                 LOCK_ASSERT(rv > 0 || uobj == NULL ||
  189                     !simple_lock_held(&uobj->vmobjlock));
  190 
  191                 /* total failure */
  192                 if (rv < 0) {
  193                         UVMHIST_LOG(loanhist, "failure %d", rv, 0,0,0);
  194                         return (-1);
  195                 }
  196 
  197                 /* relock failed, need to do another lookup */
  198                 if (rv == 0) {
  199                         UVMHIST_LOG(loanhist, "relock failure %d", result
  200                             ,0,0,0);
  201                         return (result);
  202                 }
  203 
  204                 /*
  205                  * got it... advance to next page
  206                  */
  207 
  208                 result++;
  209                 togo -= PAGE_SIZE;
  210                 curaddr += PAGE_SIZE;
  211         }
  212 
  213         /*
  214          * unlock what we locked, unlock the maps and return
  215          */
  216 
  217         if (aref->ar_amap)
  218                 amap_unlock(aref->ar_amap);
  219         uvmfault_unlockmaps(ufi, FALSE);
  220         UVMHIST_LOG(loanhist, "done %d", result, 0,0,0);
  221         return (result);
  222 }
  223 
  224 /*
  225  * normal functions
  226  */
  227 
  228 /*
  229  * uvm_loan: loan pages in a map out to anons or to the kernel
  230  *
  231  * => map should be unlocked
  232  * => start and len should be multiples of PAGE_SIZE
  233  * => result is either an array of anon's or vm_pages (depending on flags)
  234  * => flag values: UVM_LOAN_TOANON - loan to anons
  235  *                 UVM_LOAN_TOPAGE - loan to wired kernel page
  236  *    one and only one of these flags must be set!
  237  * => returns 0 (success), or an appropriate error number
  238  */
  239 
  240 int
  241 uvm_loan(struct vm_map *map, vaddr_t start, vsize_t len, void *v, int flags)
  242 {
  243         struct uvm_faultinfo ufi;
  244         void **result, **output;
  245         int rv, error;
  246 
  247         UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
  248 
  249         /*
  250          * ensure that one and only one of the flags is set
  251          */
  252 
  253         KASSERT(((flags & UVM_LOAN_TOANON) == 0) ^
  254                 ((flags & UVM_LOAN_TOPAGE) == 0));
  255         KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
  256 
  257         /*
  258          * "output" is a pointer to the current place to put the loaned page.
  259          */
  260 
  261         result = v;
  262         output = &result[0];    /* start at the beginning ... */
  263 
  264         /*
  265          * while we've got pages to do
  266          */
  267 
  268         while (len > 0) {
  269 
  270                 /*
  271                  * fill in params for a call to uvmfault_lookup
  272                  */
  273 
  274                 ufi.orig_map = map;
  275                 ufi.orig_rvaddr = start;
  276                 ufi.orig_size = len;
  277 
  278                 /*
  279                  * do the lookup, the only time this will fail is if we hit on
  280                  * an unmapped region (an error)
  281                  */
  282 
  283                 if (!uvmfault_lookup(&ufi, FALSE)) {
  284                         error = ENOENT;
  285                         goto fail;
  286                 }
  287 
  288                 /*
  289                  * map now locked.  now do the loanout...
  290                  */
  291 
  292                 rv = uvm_loanentry(&ufi, &output, flags);
  293                 if (rv < 0) {
  294                         /* all unlocked due to error */
  295                         error = EINVAL;
  296                         goto fail;
  297                 }
  298 
  299                 /*
  300                  * done!  the map is unlocked.  advance, if possible.
  301                  *
  302                  * XXXCDC: could be recoded to hold the map lock with
  303                  *         smarter code (but it only happens on map entry
  304                  *         boundaries, so it isn't that bad).
  305                  */
  306 
  307                 if (rv) {
  308                         rv <<= PAGE_SHIFT;
  309                         len -= rv;
  310                         start += rv;
  311                 }
  312         }
  313         UVMHIST_LOG(loanhist, "success", 0,0,0,0);
  314         return 0;
  315 
  316 fail:
  317         /*
  318          * failed to complete loans.  drop any loans and return failure code.
  319          * map is already unlocked.
  320          */
  321 
  322         if (output - result) {
  323                 if (flags & UVM_LOAN_TOANON) {
  324                         uvm_unloananon((struct vm_anon **)result,
  325                             output - result);
  326                 } else {
  327                         uvm_unloanpage((struct vm_page **)result,
  328                             output - result);
  329                 }
  330         }
  331         UVMHIST_LOG(loanhist, "error %d", error,0,0,0);
  332         return (error);
  333 }
  334 
  335 /*
  336  * uvm_loananon: loan a page from an anon out
  337  *
  338  * => called with map, amap, uobj locked
  339  * => return value:
  340  *      -1 = fatal error, everything is unlocked, abort.
  341  *       0 = lookup in ufi went stale, everything unlocked, relookup and
  342  *              try again
  343  *       1 = got it, everything still locked
  344  */
  345 
  346 int
  347 uvm_loananon(struct uvm_faultinfo *ufi, void ***output, int flags,
  348     struct vm_anon *anon)
  349 {
  350         struct vm_page *pg;
  351         int error;
  352 
  353         UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
  354 
  355         /*
  356          * if we are loaning to "another" anon then it is easy, we just
  357          * bump the reference count on the current anon and return a
  358          * pointer to it (it becomes copy-on-write shared).
  359          */
  360 
  361         if (flags & UVM_LOAN_TOANON) {
  362                 simple_lock(&anon->an_lock);
  363                 pg = anon->an_page;
  364                 if (pg && (pg->pqflags & PQ_ANON) != 0 && anon->an_ref == 1) {
  365                         if (pg->wire_count > 0) {
  366                                 UVMHIST_LOG(loanhist, "->A wired %p", pg,0,0,0);
  367                                 uvmfault_unlockall(ufi,
  368                                     ufi->entry->aref.ar_amap,
  369                                     ufi->entry->object.uvm_obj, anon);
  370                                 return (-1);
  371                         }
  372                         pmap_page_protect(pg, VM_PROT_READ);
  373                 }
  374                 anon->an_ref++;
  375                 **output = anon;
  376                 (*output)++;
  377                 simple_unlock(&anon->an_lock);
  378                 UVMHIST_LOG(loanhist, "->A done", 0,0,0,0);
  379                 return (1);
  380         }
  381 
  382         /*
  383          * we are loaning to a kernel-page.   we need to get the page
  384          * resident so we can wire it.   uvmfault_anonget will handle
  385          * this for us.
  386          */
  387 
  388         simple_lock(&anon->an_lock);
  389         error = uvmfault_anonget(ufi, ufi->entry->aref.ar_amap, anon);
  390 
  391         /*
  392          * if we were unable to get the anon, then uvmfault_anonget has
  393          * unlocked everything and returned an error code.
  394          */
  395 
  396         if (error) {
  397                 UVMHIST_LOG(loanhist, "error %d", error,0,0,0);
  398 
  399                 /* need to refault (i.e. refresh our lookup) ? */
  400                 if (error == ERESTART) {
  401                         return (0);
  402                 }
  403 
  404                 /* "try again"?   sleep a bit and retry ... */
  405                 if (error == EAGAIN) {
  406                         tsleep(&lbolt, PVM, "loanagain", 0);
  407                         return (0);
  408                 }
  409 
  410                 /* otherwise flag it as an error */
  411                 return (-1);
  412         }
  413 
  414         /*
  415          * we have the page and its owner locked: do the loan now.
  416          */
  417 
  418         pg = anon->an_page;
  419         uvm_lock_pageq();
  420         if (pg->wire_count > 0) {
  421                 uvm_unlock_pageq();
  422                 UVMHIST_LOG(loanhist, "->K wired %p", pg,0,0,0);
  423                 KASSERT(pg->uobject == NULL);
  424                 uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap,
  425                     NULL, anon);
  426                 return (-1);
  427         }
  428         if (pg->loan_count == 0) {
  429                 pmap_page_protect(pg, VM_PROT_READ);
  430         }
  431         pg->loan_count++;
  432         uvm_pagedequeue(pg);
  433         uvm_unlock_pageq();
  434         **output = pg;
  435         (*output)++;
  436 
  437         /* unlock anon and return success */
  438         if (pg->uobject)
  439                 simple_unlock(&pg->uobject->vmobjlock);
  440         simple_unlock(&anon->an_lock);
  441         UVMHIST_LOG(loanhist, "->K done", 0,0,0,0);
  442         return (1);
  443 }
  444 
  445 /*
  446  * uvm_loanpage: loan out pages to kernel (->K)
  447  *
  448  * => pages should be object-owned and the object should be locked.
  449  * => in the case of error, the object might be unlocked and relocked.
  450  * => caller should busy the pages beforehand.
  451  * => pages will be unbusied.
  452  * => fail with EBUSY if meet a wired page.
  453  */
  454 static int
  455 uvm_loanpage(struct vm_page **pgpp, int npages)
  456 {
  457         int i;
  458         int error = 0;
  459 
  460         UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
  461 
  462         for (i = 0; i < npages; i++) {
  463                 struct vm_page *pg = pgpp[i];
  464 
  465                 KASSERT(pg->uobject != NULL);
  466                 KASSERT(pg->uobject == pgpp[0]->uobject);
  467                 KASSERT(!(pg->flags & (PG_RELEASED|PG_PAGEOUT)));
  468                 LOCK_ASSERT(simple_lock_held(&pg->uobject->vmobjlock));
  469                 KASSERT(pg->flags & PG_BUSY);
  470 
  471                 uvm_lock_pageq();
  472                 if (pg->wire_count > 0) {
  473                         uvm_unlock_pageq();
  474                         UVMHIST_LOG(loanhist, "wired %p", pg,0,0,0);
  475                         error = EBUSY;
  476                         break;
  477                 }
  478                 if (pg->loan_count == 0) {
  479                         pmap_page_protect(pg, VM_PROT_READ);
  480                 }
  481                 pg->loan_count++;
  482                 uvm_pagedequeue(pg);
  483                 uvm_unlock_pageq();
  484         }
  485 
  486         uvm_page_unbusy(pgpp, npages);
  487 
  488         if (error) {
  489                 /*
  490                  * backout what we've done
  491                  */
  492                 struct simplelock *slock = &pgpp[0]->uobject->vmobjlock;
  493 
  494                 simple_unlock(slock);
  495                 uvm_unloan(pgpp, i, UVM_LOAN_TOPAGE);
  496                 simple_lock(slock);
  497         }
  498 
  499         UVMHIST_LOG(loanhist, "done %d", error,0,0,0);
  500         return error;
  501 }
  502 
  503 /*
  504  * XXX UBC temp limit
  505  * number of pages to get at once.
  506  * should be <= MAX_READ_AHEAD in genfs_vnops.c
  507  */
  508 #define UVM_LOAN_GET_CHUNK      16
  509 
  510 /*
  511  * uvm_loanuobjpages: loan pages from a uobj out (O->K)
  512  *
  513  * => uobj shouldn't be locked.  (we'll lock it)
  514  * => fail with EBUSY if we meet a wired page.
  515  */
  516 int
  517 uvm_loanuobjpages(struct uvm_object *uobj, voff_t pgoff, int orignpages,
  518     struct vm_page **origpgpp)
  519 {
  520         int ndone; /* # of pages loaned out */
  521         struct vm_page **pgpp;
  522         int error;
  523         int i;
  524         struct simplelock *slock;
  525 
  526         pgpp = origpgpp;
  527         for (ndone = 0; ndone < orignpages; ) {
  528                 int npages;
  529                 /* npendloan: # of pages busied but not loand out yet. */
  530                 int npendloan = 0xdead; /* XXX gcc */
  531 reget:
  532                 npages = MIN(UVM_LOAN_GET_CHUNK, orignpages - ndone);
  533                 simple_lock(&uobj->vmobjlock);
  534                 error = (*uobj->pgops->pgo_get)(uobj,
  535                     pgoff + (ndone << PAGE_SHIFT), pgpp, &npages, 0,
  536                     VM_PROT_READ, 0, PGO_SYNCIO);
  537                 if (error == EAGAIN) {
  538                         tsleep(&lbolt, PVM, "nfsread", 0);
  539                         continue;
  540                 }
  541                 if (error)
  542                         goto fail;
  543 
  544                 KASSERT(npages > 0);
  545 
  546                 /* loan and unbusy pages */
  547                 slock = NULL;
  548                 for (i = 0; i < npages; i++) {
  549                         struct simplelock *nextslock; /* slock for next page */
  550                         struct vm_page *pg = *pgpp;
  551 
  552                         /* XXX assuming that the page is owned by uobj */
  553                         KASSERT(pg->uobject != NULL);
  554                         nextslock = &pg->uobject->vmobjlock;
  555 
  556                         if (slock != nextslock) {
  557                                 if (slock) {
  558                                         KASSERT(npendloan > 0);
  559                                         error = uvm_loanpage(pgpp - npendloan,
  560                                             npendloan);
  561                                         simple_unlock(slock);
  562                                         if (error)
  563                                                 goto fail;
  564                                         ndone += npendloan;
  565                                         KASSERT(origpgpp + ndone == pgpp);
  566                                 }
  567                                 slock = nextslock;
  568                                 npendloan = 0;
  569                                 simple_lock(slock);
  570                         }
  571 
  572                         if ((pg->flags & PG_RELEASED) != 0) {
  573                                 /*
  574                                  * release pages and try again.
  575                                  */
  576                                 simple_unlock(slock);
  577                                 for (; i < npages; i++) {
  578                                         pg = pgpp[i];
  579                                         slock = &pg->uobject->vmobjlock;
  580 
  581                                         simple_lock(slock);
  582                                         uvm_lock_pageq();
  583                                         uvm_page_unbusy(&pg, 1);
  584                                         uvm_unlock_pageq();
  585                                         simple_unlock(slock);
  586                                 }
  587                                 goto reget;
  588                         }
  589 
  590                         npendloan++;
  591                         pgpp++;
  592                         KASSERT(origpgpp + ndone + npendloan == pgpp);
  593                 }
  594                 KASSERT(slock != NULL);
  595                 KASSERT(npendloan > 0);
  596                 error = uvm_loanpage(pgpp - npendloan, npendloan);
  597                 simple_unlock(slock);
  598                 if (error)
  599                         goto fail;
  600                 ndone += npendloan;
  601                 KASSERT(origpgpp + ndone == pgpp);
  602         }
  603 
  604         return 0;
  605 
  606 fail:
  607         uvm_unloan(origpgpp, ndone, UVM_LOAN_TOPAGE);
  608 
  609         return error;
  610 }
  611 
  612 /*
  613  * uvm_loanuobj: loan a page from a uobj out
  614  *
  615  * => called with map, amap, uobj locked
  616  * => return value:
  617  *      -1 = fatal error, everything is unlocked, abort.
  618  *       0 = lookup in ufi went stale, everything unlocked, relookup and
  619  *              try again
  620  *       1 = got it, everything still locked
  621  */
  622 
  623 static int
  624 uvm_loanuobj(struct uvm_faultinfo *ufi, void ***output, int flags, vaddr_t va)
  625 {
  626         struct vm_amap *amap = ufi->entry->aref.ar_amap;
  627         struct uvm_object *uobj = ufi->entry->object.uvm_obj;
  628         struct vm_page *pg;
  629         struct vm_anon *anon;
  630         int error, npages;
  631         boolean_t locked;
  632 
  633         UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
  634 
  635         /*
  636          * first we must make sure the page is resident.
  637          *
  638          * XXXCDC: duplicate code with uvm_fault().
  639          */
  640 
  641         simple_lock(&uobj->vmobjlock);
  642         if (uobj->pgops->pgo_get) {     /* try locked pgo_get */
  643                 npages = 1;
  644                 pg = NULL;
  645                 error = (*uobj->pgops->pgo_get)(uobj,
  646                     va - ufi->entry->start + ufi->entry->offset,
  647                     &pg, &npages, 0, VM_PROT_READ, MADV_NORMAL, PGO_LOCKED);
  648         } else {
  649                 error = EIO;            /* must have pgo_get op */
  650         }
  651 
  652         /*
  653          * check the result of the locked pgo_get.  if there is a problem,
  654          * then we fail the loan.
  655          */
  656 
  657         if (error && error != EBUSY) {
  658                 uvmfault_unlockall(ufi, amap, uobj, NULL);
  659                 return (-1);
  660         }
  661 
  662         /*
  663          * if we need to unlock for I/O, do so now.
  664          */
  665 
  666         if (error == EBUSY) {
  667                 uvmfault_unlockall(ufi, amap, NULL, NULL);
  668 
  669                 /* locked: uobj */
  670                 npages = 1;
  671                 error = (*uobj->pgops->pgo_get)(uobj,
  672                     va - ufi->entry->start + ufi->entry->offset,
  673                     &pg, &npages, 0, VM_PROT_READ, MADV_NORMAL, PGO_SYNCIO);
  674                 /* locked: <nothing> */
  675 
  676                 if (error) {
  677                         if (error == EAGAIN) {
  678                                 tsleep(&lbolt, PVM, "fltagain2", 0);
  679                                 return (0);
  680                         }
  681                         return (-1);
  682                 }
  683 
  684                 /*
  685                  * pgo_get was a success.   attempt to relock everything.
  686                  */
  687 
  688                 locked = uvmfault_relock(ufi);
  689                 if (locked && amap)
  690                         amap_lock(amap);
  691                 uobj = pg->uobject;
  692                 simple_lock(&uobj->vmobjlock);
  693 
  694                 /*
  695                  * verify that the page has not be released and re-verify
  696                  * that amap slot is still free.   if there is a problem we
  697                  * drop our lock (thus force a lookup refresh/retry).
  698                  */
  699 
  700                 if ((pg->flags & PG_RELEASED) != 0 ||
  701                     (locked && amap && amap_lookup(&ufi->entry->aref,
  702                     ufi->orig_rvaddr - ufi->entry->start))) {
  703                         if (locked)
  704                                 uvmfault_unlockall(ufi, amap, NULL, NULL);
  705                         locked = FALSE;
  706                 }
  707 
  708                 /*
  709                  * didn't get the lock?   release the page and retry.
  710                  */
  711 
  712                 if (locked == FALSE) {
  713                         if (pg->flags & PG_WANTED) {
  714                                 wakeup(pg);
  715                         }
  716                         if (pg->flags & PG_RELEASED) {
  717                                 uvm_lock_pageq();
  718                                 uvm_pagefree(pg);
  719                                 uvm_unlock_pageq();
  720                                 simple_unlock(&uobj->vmobjlock);
  721                                 return (0);
  722                         }
  723                         uvm_lock_pageq();
  724                         uvm_pageactivate(pg);
  725                         uvm_unlock_pageq();
  726                         pg->flags &= ~(PG_BUSY|PG_WANTED);
  727                         UVM_PAGE_OWN(pg, NULL);
  728                         simple_unlock(&uobj->vmobjlock);
  729                         return (0);
  730                 }
  731         }
  732 
  733         KASSERT(uobj == pg->uobject);
  734 
  735         /*
  736          * at this point we have the page we want ("pg") marked PG_BUSY for us
  737          * and we have all data structures locked.  do the loanout.  page can
  738          * not be PG_RELEASED (we caught this above).
  739          */
  740 
  741         if ((flags & UVM_LOAN_TOANON) == 0) {
  742                 if (uvm_loanpage(&pg, 1)) {
  743                         uvmfault_unlockall(ufi, amap, uobj, NULL);
  744                         return (-1);
  745                 }
  746                 simple_unlock(&uobj->vmobjlock);
  747                 **output = pg;
  748                 (*output)++;
  749                 return (1);
  750         }
  751 
  752         /*
  753          * must be a loan to an anon.   check to see if there is already
  754          * an anon associated with this page.  if so, then just return
  755          * a reference to this object.   the page should already be
  756          * mapped read-only because it is already on loan.
  757          */
  758 
  759         if (pg->uanon) {
  760                 anon = pg->uanon;
  761                 simple_lock(&anon->an_lock);
  762                 anon->an_ref++;
  763                 simple_unlock(&anon->an_lock);
  764                 if (pg->flags & PG_WANTED) {
  765                         wakeup(pg);
  766                 }
  767                 pg->flags &= ~(PG_WANTED|PG_BUSY);
  768                 UVM_PAGE_OWN(pg, NULL);
  769                 simple_unlock(&uobj->vmobjlock);
  770                 **output = anon;
  771                 (*output)++;
  772                 return (1);
  773         }
  774 
  775         /*
  776          * need to allocate a new anon
  777          */
  778 
  779         anon = uvm_analloc();
  780         if (anon == NULL) {
  781                 goto fail;
  782         }
  783         anon->an_page = pg;
  784         pg->uanon = anon;
  785         uvm_lock_pageq();
  786         if (pg->wire_count > 0) {
  787                 uvm_unlock_pageq();
  788                 UVMHIST_LOG(loanhist, "wired %p", pg,0,0,0);
  789                 pg->uanon = NULL;
  790                 anon->an_page = NULL;
  791                 anon->an_ref--;
  792                 simple_unlock(&anon->an_lock);
  793                 uvm_anfree(anon);
  794                 goto fail;
  795         }
  796         if (pg->loan_count == 0) {
  797                 pmap_page_protect(pg, VM_PROT_READ);
  798         }
  799         pg->loan_count++;
  800         uvm_pageactivate(pg);
  801         uvm_unlock_pageq();
  802         if (pg->flags & PG_WANTED) {
  803                 wakeup(pg);
  804         }
  805         pg->flags &= ~(PG_WANTED|PG_BUSY);
  806         UVM_PAGE_OWN(pg, NULL);
  807         simple_unlock(&uobj->vmobjlock);
  808         simple_unlock(&anon->an_lock);
  809         **output = anon;
  810         (*output)++;
  811         return (1);
  812 
  813 fail:
  814         UVMHIST_LOG(loanhist, "fail", 0,0,0,0);
  815         /*
  816          * unlock everything and bail out.
  817          */
  818         if (pg->flags & PG_WANTED) {
  819                 wakeup(pg);
  820         }
  821         pg->flags &= ~(PG_WANTED|PG_BUSY);
  822         UVM_PAGE_OWN(pg, NULL);
  823         uvmfault_unlockall(ufi, amap, uobj, NULL);
  824         return (-1);
  825 }
  826 
  827 /*
  828  * uvm_loanzero: loan a zero-fill page out
  829  *
  830  * => called with map, amap, uobj locked
  831  * => return value:
  832  *      -1 = fatal error, everything is unlocked, abort.
  833  *       0 = lookup in ufi went stale, everything unlocked, relookup and
  834  *              try again
  835  *       1 = got it, everything still locked
  836  */
  837 
  838 static struct uvm_object uvm_loanzero_object;
  839 
  840 static int
  841 uvm_loanzero(struct uvm_faultinfo *ufi, void ***output, int flags)
  842 {
  843         struct vm_anon *anon;
  844         struct vm_page *pg;
  845         struct vm_amap *amap = ufi->entry->aref.ar_amap;
  846 
  847         UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
  848 again:
  849         simple_lock(&uvm_loanzero_object.vmobjlock);
  850 
  851         /*
  852          * first, get ahold of our single zero page.
  853          */
  854 
  855         if (__predict_false((pg =
  856                              TAILQ_FIRST(&uvm_loanzero_object.memq)) == NULL)) {
  857                 while ((pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL,
  858                                            UVM_PGA_ZERO)) == NULL) {
  859                         simple_unlock(&uvm_loanzero_object.vmobjlock);
  860                         uvmfault_unlockall(ufi, amap, NULL, NULL);
  861                         uvm_wait("loanzero");
  862                         if (!uvmfault_relock(ufi)) {
  863                                 return (0);
  864                         }
  865                         if (amap) {
  866                                 amap_lock(amap);
  867                         }
  868                         goto again;
  869                 }
  870 
  871                 /* got a zero'd page. */
  872                 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
  873                 pg->flags |= PG_RDONLY;
  874                 uvm_lock_pageq();
  875                 uvm_pageactivate(pg);
  876                 uvm_unlock_pageq();
  877                 UVM_PAGE_OWN(pg, NULL);
  878         }
  879 
  880         if ((flags & UVM_LOAN_TOANON) == 0) {   /* loaning to kernel-page */
  881                 uvm_lock_pageq();
  882                 pg->loan_count++;
  883                 uvm_pagedequeue(pg);
  884                 uvm_unlock_pageq();
  885                 simple_unlock(&uvm_loanzero_object.vmobjlock);
  886                 **output = pg;
  887                 (*output)++;
  888                 return (1);
  889         }
  890 
  891         /*
  892          * loaning to an anon.  check to see if there is already an anon
  893          * associated with this page.  if so, then just return a reference
  894          * to this object.
  895          */
  896 
  897         if (pg->uanon) {
  898                 anon = pg->uanon;
  899                 simple_lock(&anon->an_lock);
  900                 anon->an_ref++;
  901                 simple_unlock(&anon->an_lock);
  902                 simple_unlock(&uvm_loanzero_object.vmobjlock);
  903                 **output = anon;
  904                 (*output)++;
  905                 return (1);
  906         }
  907 
  908         /*
  909          * need to allocate a new anon
  910          */
  911 
  912         anon = uvm_analloc();
  913         if (anon == NULL) {
  914                 /* out of swap causes us to fail */
  915                 simple_unlock(&uvm_loanzero_object.vmobjlock);
  916                 uvmfault_unlockall(ufi, amap, NULL, NULL);
  917                 return (-1);
  918         }
  919         anon->an_page = pg;
  920         pg->uanon = anon;
  921         uvm_lock_pageq();
  922         pg->loan_count++;
  923         uvm_pageactivate(pg);
  924         uvm_unlock_pageq();
  925         simple_unlock(&anon->an_lock);
  926         simple_unlock(&uvm_loanzero_object.vmobjlock);
  927         **output = anon;
  928         (*output)++;
  929         return (1);
  930 }
  931 
  932 
  933 /*
  934  * uvm_unloananon: kill loans on anons (basically a normal ref drop)
  935  *
  936  * => we expect all our resources to be unlocked
  937  */
  938 
  939 static void
  940 uvm_unloananon(struct vm_anon **aloans, int nanons)
  941 {
  942         struct vm_anon *anon;
  943 
  944         while (nanons-- > 0) {
  945                 int refs;
  946 
  947                 anon = *aloans++;
  948                 simple_lock(&anon->an_lock);
  949                 refs = --anon->an_ref;
  950                 simple_unlock(&anon->an_lock);
  951 
  952                 if (refs == 0) {
  953                         uvm_anfree(anon);
  954                 }
  955         }
  956 }
  957 
  958 /*
  959  * uvm_unloanpage: kill loans on pages loaned out to the kernel
  960  *
  961  * => we expect all our resources to be unlocked
  962  */
  963 
  964 static void
  965 uvm_unloanpage(struct vm_page **ploans, int npages)
  966 {
  967         struct vm_page *pg;
  968         struct simplelock *slock;
  969 
  970         uvm_lock_pageq();
  971         while (npages-- > 0) {
  972                 pg = *ploans++;
  973 
  974                 /*
  975                  * do a little dance to acquire the object or anon lock
  976                  * as appropriate.  we are locking in the wrong order,
  977                  * so we have to do a try-lock here.
  978                  */
  979 
  980                 slock = NULL;
  981                 while (pg->uobject != NULL || pg->uanon != NULL) {
  982                         if (pg->uobject != NULL) {
  983                                 slock = &pg->uobject->vmobjlock;
  984                         } else {
  985                                 slock = &pg->uanon->an_lock;
  986                         }
  987                         if (simple_lock_try(slock)) {
  988                                 break;
  989                         }
  990                         uvm_unlock_pageq();
  991                         uvm_lock_pageq();
  992                         slock = NULL;
  993                 }
  994 
  995                 /*
  996                  * drop our loan.  if page is owned by an anon but
  997                  * PQ_ANON is not set, the page was loaned to the anon
  998                  * from an object which dropped ownership, so resolve
  999                  * this by turning the anon's loan into real ownership
 1000                  * (ie. decrement loan_count again and set PQ_ANON).
 1001                  * after all this, if there are no loans left, put the
 1002                  * page back a paging queue (if the page is owned by
 1003                  * an anon) or free it (if the page is now unowned).
 1004                  */
 1005 
 1006                 KASSERT(pg->loan_count > 0);
 1007                 pg->loan_count--;
 1008                 if (pg->uobject == NULL && pg->uanon != NULL &&
 1009                     (pg->pqflags & PQ_ANON) == 0) {
 1010                         KASSERT(pg->loan_count > 0);
 1011                         pg->loan_count--;
 1012                         pg->pqflags |= PQ_ANON;
 1013                 }
 1014                 if (pg->loan_count == 0) {
 1015                         if (pg->uobject == NULL && pg->uanon == NULL) {
 1016                                 KASSERT((pg->flags & PG_BUSY) == 0);
 1017                                 uvm_pagefree(pg);
 1018                         } else {
 1019                                 uvm_pageactivate(pg);
 1020                         }
 1021                 } else if (pg->loan_count == 1 && pg->uobject != NULL &&
 1022                            pg->uanon != NULL) {
 1023                         uvm_pageactivate(pg);
 1024                 }
 1025                 if (slock != NULL) {
 1026                         simple_unlock(slock);
 1027                 }
 1028         }
 1029         uvm_unlock_pageq();
 1030 }
 1031 
 1032 /*
 1033  * uvm_unloan: kill loans on pages or anons.
 1034  */
 1035 
 1036 void
 1037 uvm_unloan(void *v, int npages, int flags)
 1038 {
 1039         if (flags & UVM_LOAN_TOANON) {
 1040                 uvm_unloananon(v, npages);
 1041         } else {
 1042                 uvm_unloanpage(v, npages);
 1043         }
 1044 }
 1045 
 1046 /*
 1047  * Minimal pager for uvm_loanzero_object.  We need to provide a "put"
 1048  * method, because the page can end up on a paging queue, and the
 1049  * page daemon will want to call pgo_put when it encounters the page
 1050  * on the inactive list.
 1051  */
 1052 
 1053 static int
 1054 ulz_put(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
 1055 {
 1056         struct vm_page *pg;
 1057 
 1058         KDASSERT(uobj == &uvm_loanzero_object);
 1059 
 1060         /*
 1061          * Don't need to do any work here if we're not freeing pages.
 1062          */
 1063 
 1064         if ((flags & PGO_FREE) == 0) {
 1065                 simple_unlock(&uobj->vmobjlock);
 1066                 return 0;
 1067         }
 1068 
 1069         /*
 1070          * we don't actually want to ever free the uvm_loanzero_page, so
 1071          * just reactivate or dequeue it.
 1072          */
 1073 
 1074         pg = TAILQ_FIRST(&uobj->memq);
 1075         KASSERT(pg != NULL);
 1076         KASSERT(TAILQ_NEXT(pg, listq) == NULL);
 1077 
 1078         uvm_lock_pageq();
 1079         if (pg->uanon)
 1080                 uvm_pageactivate(pg);
 1081         else
 1082                 uvm_pagedequeue(pg);
 1083         uvm_unlock_pageq();
 1084 
 1085         simple_unlock(&uobj->vmobjlock);
 1086         return 0;
 1087 }
 1088 
 1089 static struct uvm_pagerops ulz_pager = {
 1090         NULL,           /* init */
 1091         NULL,           /* reference */
 1092         NULL,           /* detach */
 1093         NULL,           /* fault */
 1094         NULL,           /* get */
 1095         ulz_put,        /* put */
 1096 };
 1097 
 1098 /*
 1099  * uvm_loan_init(): initialize the uvm_loan() facility.
 1100  */
 1101 
 1102 void
 1103 uvm_loan_init(void)
 1104 {
 1105 
 1106         simple_lock_init(&uvm_loanzero_object.vmobjlock);
 1107         TAILQ_INIT(&uvm_loanzero_object.memq);
 1108         uvm_loanzero_object.pgops = &ulz_pager;
 1109 
 1110         UVMHIST_INIT(loanhist, 300);
 1111 }
 1112 
 1113 /*
 1114  * uvm_loanbreak: break loan on a uobj page
 1115  *
 1116  * => called with uobj locked
 1117  * => the page should be busy
 1118  * => return value:
 1119  *      newly allocated page if succeeded
 1120  */
 1121 struct vm_page *
 1122 uvm_loanbreak(struct vm_page *uobjpage)
 1123 {
 1124         struct vm_page *pg;
 1125 #ifdef DIAGNOSTIC
 1126         struct uvm_object *uobj = uobjpage->uobject;
 1127 #endif
 1128 
 1129         KASSERT(uobj != NULL);
 1130         LOCK_ASSERT(simple_lock_held(&uobj->vmobjlock));
 1131         KASSERT(uobjpage->flags & PG_BUSY);
 1132 
 1133         /* alloc new un-owned page */
 1134         pg = uvm_pagealloc(NULL, 0, NULL, 0);
 1135         if (pg == NULL)
 1136                 return NULL;
 1137 
 1138         /*
 1139          * copy the data from the old page to the new
 1140          * one and clear the fake flags on the new page (keep it busy).
 1141          * force a reload of the old page by clearing it from all
 1142          * pmaps.
 1143          * transfer dirtiness of the old page to the new page.
 1144          * then lock the page queues to rename the pages.
 1145          */
 1146 
 1147         uvm_pagecopy(uobjpage, pg);     /* old -> new */
 1148         pg->flags &= ~PG_FAKE;
 1149         pmap_page_protect(uobjpage, VM_PROT_NONE);
 1150         if ((uobjpage->flags & PG_CLEAN) != 0 && !pmap_clear_modify(uobjpage)) {
 1151                 pmap_clear_modify(pg);
 1152                 pg->flags |= PG_CLEAN;
 1153         } else {
 1154                 /* uvm_pagecopy marked it dirty */
 1155                 KASSERT((pg->flags & PG_CLEAN) == 0);
 1156                 /* a object with a dirty page should be dirty. */
 1157                 KASSERT(!UVM_OBJ_IS_CLEAN(uobj));
 1158         }
 1159         if (uobjpage->flags & PG_WANTED)
 1160                 wakeup(uobjpage);
 1161         /* uobj still locked */
 1162         uobjpage->flags &= ~(PG_WANTED|PG_BUSY);
 1163         UVM_PAGE_OWN(uobjpage, NULL);
 1164 
 1165         uvm_lock_pageq();
 1166 
 1167         /*
 1168          * replace uobjpage with new page.
 1169          */
 1170 
 1171         uvm_pagereplace(uobjpage, pg);
 1172 
 1173         /*
 1174          * if the page is no longer referenced by
 1175          * an anon (i.e. we are breaking an O->K
 1176          * loan), then remove it from any pageq's.
 1177          */
 1178         if (uobjpage->uanon == NULL)
 1179                 uvm_pagedequeue(uobjpage);
 1180 
 1181         /*
 1182          * at this point we have absolutely no
 1183          * control over uobjpage
 1184          */
 1185 
 1186         /* install new page */
 1187         uvm_pageactivate(pg);
 1188         uvm_unlock_pageq();
 1189 
 1190         /*
 1191          * done!  loan is broken and "pg" is
 1192          * PG_BUSY.   it can now replace uobjpage.
 1193          */
 1194 
 1195         return pg;
 1196 }

Cache object: 02bb2397ef7e56fcc619d3999de2596c


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.