The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System, Second Edition

[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/uvm/uvm_loan.c

Version: -  FREEBSD  -  FREEBSD10  -  FREEBSD9  -  FREEBSD92  -  FREEBSD91  -  FREEBSD90  -  FREEBSD8  -  FREEBSD82  -  FREEBSD81  -  FREEBSD80  -  FREEBSD7  -  FREEBSD74  -  FREEBSD73  -  FREEBSD72  -  FREEBSD71  -  FREEBSD70  -  FREEBSD6  -  FREEBSD64  -  FREEBSD63  -  FREEBSD62  -  FREEBSD61  -  FREEBSD60  -  FREEBSD5  -  FREEBSD55  -  FREEBSD54  -  FREEBSD53  -  FREEBSD52  -  FREEBSD51  -  FREEBSD50  -  FREEBSD4  -  FREEBSD3  -  FREEBSD22  -  cheribsd  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1  -  FREEBSD-LIBC  -  FREEBSD8-LIBC  -  FREEBSD7-LIBC  -  FREEBSD6-LIBC  -  GLIBC27 
SearchContext: -  none  -  3  -  10 

    1 /*      $NetBSD: uvm_loan.c,v 1.79 2011/04/23 18:14:12 rmind Exp $      */
    2 
    3 /*
    4  * Copyright (c) 1997 Charles D. Cranor and Washington University.
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice, this list of conditions and the following disclaimer.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice, this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  *
   16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   21  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   22  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   23  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
   25  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   26  *
   27  * from: Id: uvm_loan.c,v 1.1.6.4 1998/02/06 05:08:43 chs Exp
   28  */
   29 
   30 /*
   31  * uvm_loan.c: page loanout handler
   32  */
   33 
   34 #include <sys/cdefs.h>
   35 __KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.79 2011/04/23 18:14:12 rmind Exp $");
   36 
   37 #include <sys/param.h>
   38 #include <sys/systm.h>
   39 #include <sys/kernel.h>
   40 #include <sys/mman.h>
   41 
   42 #include <uvm/uvm.h>
   43 
   44 /*
   45  * "loaned" pages are pages which are (read-only, copy-on-write) loaned
   46  * from the VM system to other parts of the kernel.   this allows page
   47  * copying to be avoided (e.g. you can loan pages from objs/anons to
   48  * the mbuf system).
   49  *
   50  * there are 3 types of loans possible:
   51  *  O->K  uvm_object page to wired kernel page (e.g. mbuf data area)
   52  *  A->K  anon page to wired kernel page (e.g. mbuf data area)
   53  *  O->A  uvm_object to anon loan (e.g. vnode page to an anon)
   54  * note that it possible to have an O page loaned to both an A and K
   55  * at the same time.
   56  *
   57  * loans are tracked by pg->loan_count.  an O->A page will have both
   58  * a uvm_object and a vm_anon, but PQ_ANON will not be set.   this sort
   59  * of page is considered "owned" by the uvm_object (not the anon).
   60  *
   61  * each loan of a page to the kernel bumps the pg->wire_count.  the
   62  * kernel mappings for these pages will be read-only and wired.  since
   63  * the page will also be wired, it will not be a candidate for pageout,
   64  * and thus will never be pmap_page_protect()'d with VM_PROT_NONE.  a
   65  * write fault in the kernel to one of these pages will not cause
   66  * copy-on-write.  instead, the page fault is considered fatal.  this
   67  * is because the kernel mapping will have no way to look up the
   68  * object/anon which the page is owned by.  this is a good side-effect,
   69  * since a kernel write to a loaned page is an error.
   70  *
   71  * owners that want to free their pages and discover that they are
   72  * loaned out simply "disown" them (the page becomes an orphan).  these
   73  * pages should be freed when the last loan is dropped.   in some cases
   74  * an anon may "adopt" an orphaned page.
   75  *
   76  * locking: to read pg->loan_count either the owner or the page queues
   77  * must be locked.   to modify pg->loan_count, both the owner of the page
   78  * and the PQs must be locked.   pg->flags is (as always) locked by
   79  * the owner of the page.
   80  *
   81  * note that locking from the "loaned" side is tricky since the object
   82  * getting the loaned page has no reference to the page's owner and thus
   83  * the owner could "die" at any time.   in order to prevent the owner
   84  * from dying the page queues should be locked.   this forces us to sometimes
   85  * use "try" locking.
   86  *
   87  * loans are typically broken by the following events:
   88  *  1. user-level xwrite fault to a loaned page
   89  *  2. pageout of clean+inactive O->A loaned page
   90  *  3. owner frees page (e.g. pager flush)
   91  *
   92  * note that loaning a page causes all mappings of the page to become
   93  * read-only (via pmap_page_protect).   this could have an unexpected
   94  * effect on normal "wired" pages if one is not careful (XXX).
   95  */
   96 
   97 /*
   98  * local prototypes
   99  */
  100 
  101 static int      uvm_loananon(struct uvm_faultinfo *, void ***,
  102                              int, struct vm_anon *);
  103 static int      uvm_loanuobj(struct uvm_faultinfo *, void ***,
  104                              int, vaddr_t);
  105 static int      uvm_loanzero(struct uvm_faultinfo *, void ***, int);
  106 static void     uvm_unloananon(struct vm_anon **, int);
  107 static void     uvm_unloanpage(struct vm_page **, int);
  108 static int      uvm_loanpage(struct vm_page **, int);
  109 
  110 
  111 /*
  112  * inlines
  113  */
  114 
  115 /*
  116  * uvm_loanentry: loan out pages in a map entry (helper fn for uvm_loan())
  117  *
  118  * => "ufi" is the result of a successful map lookup (meaning that
  119  *      on entry the map is locked by the caller)
  120  * => we may unlock and then relock the map if needed (for I/O)
  121  * => we put our output result in "output"
  122  * => we always return with the map unlocked
  123  * => possible return values:
  124  *      -1 == error, map is unlocked
  125  *       0 == map relock error (try again!), map is unlocked
  126  *      >0 == number of pages we loaned, map is unlocked
  127  *
  128  * NOTE: We can live with this being an inline, because it is only called
  129  * from one place.
  130  */
  131 
  132 static inline int
  133 uvm_loanentry(struct uvm_faultinfo *ufi, void ***output, int flags)
  134 {
  135         vaddr_t curaddr = ufi->orig_rvaddr;
  136         vsize_t togo = ufi->size;
  137         struct vm_aref *aref = &ufi->entry->aref;
  138         struct uvm_object *uobj = ufi->entry->object.uvm_obj;
  139         struct vm_anon *anon;
  140         int rv, result = 0;
  141 
  142         UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
  143 
  144         /*
  145          * lock us the rest of the way down (we unlock before return)
  146          */
  147         if (aref->ar_amap)
  148                 amap_lock(aref->ar_amap);
  149 
  150         /*
  151          * loop until done
  152          */
  153         while (togo) {
  154 
  155                 /*
  156                  * find the page we want.   check the anon layer first.
  157                  */
  158 
  159                 if (aref->ar_amap) {
  160                         anon = amap_lookup(aref, curaddr - ufi->entry->start);
  161                 } else {
  162                         anon = NULL;
  163                 }
  164 
  165                 /* locked: map, amap, uobj */
  166                 if (anon) {
  167                         rv = uvm_loananon(ufi, output, flags, anon);
  168                 } else if (uobj) {
  169                         rv = uvm_loanuobj(ufi, output, flags, curaddr);
  170                 } else if (UVM_ET_ISCOPYONWRITE(ufi->entry)) {
  171                         rv = uvm_loanzero(ufi, output, flags);
  172                 } else {
  173                         uvmfault_unlockall(ufi, aref->ar_amap, uobj, NULL);
  174                         rv = -1;
  175                 }
  176                 /* locked: if (rv > 0) => map, amap, uobj  [o.w. unlocked] */
  177                 KASSERT(rv > 0 || aref->ar_amap == NULL ||
  178                     !mutex_owned(&aref->ar_amap->am_l));
  179                 KASSERT(rv > 0 || uobj == NULL ||
  180                     !mutex_owned(&uobj->vmobjlock));
  181 
  182                 /* total failure */
  183                 if (rv < 0) {
  184                         UVMHIST_LOG(loanhist, "failure %d", rv, 0,0,0);
  185                         return (-1);
  186                 }
  187 
  188                 /* relock failed, need to do another lookup */
  189                 if (rv == 0) {
  190                         UVMHIST_LOG(loanhist, "relock failure %d", result
  191                             ,0,0,0);
  192                         return (result);
  193                 }
  194 
  195                 /*
  196                  * got it... advance to next page
  197                  */
  198 
  199                 result++;
  200                 togo -= PAGE_SIZE;
  201                 curaddr += PAGE_SIZE;
  202         }
  203 
  204         /*
  205          * unlock what we locked, unlock the maps and return
  206          */
  207 
  208         if (aref->ar_amap)
  209                 amap_unlock(aref->ar_amap);
  210         uvmfault_unlockmaps(ufi, false);
  211         UVMHIST_LOG(loanhist, "done %d", result, 0,0,0);
  212         return (result);
  213 }
  214 
  215 /*
  216  * normal functions
  217  */
  218 
  219 /*
  220  * uvm_loan: loan pages in a map out to anons or to the kernel
  221  *
  222  * => map should be unlocked
  223  * => start and len should be multiples of PAGE_SIZE
  224  * => result is either an array of anon's or vm_pages (depending on flags)
  225  * => flag values: UVM_LOAN_TOANON - loan to anons
  226  *                 UVM_LOAN_TOPAGE - loan to wired kernel page
  227  *    one and only one of these flags must be set!
  228  * => returns 0 (success), or an appropriate error number
  229  */
  230 
  231 int
  232 uvm_loan(struct vm_map *map, vaddr_t start, vsize_t len, void *v, int flags)
  233 {
  234         struct uvm_faultinfo ufi;
  235         void **result, **output;
  236         int rv, error;
  237 
  238         UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
  239 
  240         /*
  241          * ensure that one and only one of the flags is set
  242          */
  243 
  244         KASSERT(((flags & UVM_LOAN_TOANON) == 0) ^
  245                 ((flags & UVM_LOAN_TOPAGE) == 0));
  246         KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
  247 
  248         /*
  249          * "output" is a pointer to the current place to put the loaned page.
  250          */
  251 
  252         result = v;
  253         output = &result[0];    /* start at the beginning ... */
  254 
  255         /*
  256          * while we've got pages to do
  257          */
  258 
  259         while (len > 0) {
  260 
  261                 /*
  262                  * fill in params for a call to uvmfault_lookup
  263                  */
  264 
  265                 ufi.orig_map = map;
  266                 ufi.orig_rvaddr = start;
  267                 ufi.orig_size = len;
  268 
  269                 /*
  270                  * do the lookup, the only time this will fail is if we hit on
  271                  * an unmapped region (an error)
  272                  */
  273 
  274                 if (!uvmfault_lookup(&ufi, false)) {
  275                         error = ENOENT;
  276                         goto fail;
  277                 }
  278 
  279                 /*
  280                  * map now locked.  now do the loanout...
  281                  */
  282 
  283                 rv = uvm_loanentry(&ufi, &output, flags);
  284                 if (rv < 0) {
  285                         /* all unlocked due to error */
  286                         error = EINVAL;
  287                         goto fail;
  288                 }
  289 
  290                 /*
  291                  * done!  the map is unlocked.  advance, if possible.
  292                  *
  293                  * XXXCDC: could be recoded to hold the map lock with
  294                  *         smarter code (but it only happens on map entry
  295                  *         boundaries, so it isn't that bad).
  296                  */
  297 
  298                 if (rv) {
  299                         rv <<= PAGE_SHIFT;
  300                         len -= rv;
  301                         start += rv;
  302                 }
  303         }
  304         UVMHIST_LOG(loanhist, "success", 0,0,0,0);
  305         return 0;
  306 
  307 fail:
  308         /*
  309          * failed to complete loans.  drop any loans and return failure code.
  310          * map is already unlocked.
  311          */
  312 
  313         if (output - result) {
  314                 if (flags & UVM_LOAN_TOANON) {
  315                         uvm_unloananon((struct vm_anon **)result,
  316                             output - result);
  317                 } else {
  318                         uvm_unloanpage((struct vm_page **)result,
  319                             output - result);
  320                 }
  321         }
  322         UVMHIST_LOG(loanhist, "error %d", error,0,0,0);
  323         return (error);
  324 }
  325 
  326 /*
  327  * uvm_loananon: loan a page from an anon out
  328  *
  329  * => called with map, amap, uobj locked
  330  * => return value:
  331  *      -1 = fatal error, everything is unlocked, abort.
  332  *       0 = lookup in ufi went stale, everything unlocked, relookup and
  333  *              try again
  334  *       1 = got it, everything still locked
  335  */
  336 
  337 int
  338 uvm_loananon(struct uvm_faultinfo *ufi, void ***output, int flags,
  339     struct vm_anon *anon)
  340 {
  341         struct vm_page *pg;
  342         int error;
  343 
  344         UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
  345 
  346         /*
  347          * if we are loaning to "another" anon then it is easy, we just
  348          * bump the reference count on the current anon and return a
  349          * pointer to it (it becomes copy-on-write shared).
  350          */
  351 
  352         if (flags & UVM_LOAN_TOANON) {
  353                 mutex_enter(&anon->an_lock);
  354                 pg = anon->an_page;
  355                 if (pg && (pg->pqflags & PQ_ANON) != 0 && anon->an_ref == 1) {
  356                         if (pg->wire_count > 0) {
  357                                 UVMHIST_LOG(loanhist, "->A wired %p", pg,0,0,0);
  358                                 uvmfault_unlockall(ufi,
  359                                     ufi->entry->aref.ar_amap,
  360                                     ufi->entry->object.uvm_obj, anon);
  361                                 return (-1);
  362                         }
  363                         pmap_page_protect(pg, VM_PROT_READ);
  364                 }
  365                 anon->an_ref++;
  366                 **output = anon;
  367                 (*output)++;
  368                 mutex_exit(&anon->an_lock);
  369                 UVMHIST_LOG(loanhist, "->A done", 0,0,0,0);
  370                 return (1);
  371         }
  372 
  373         /*
  374          * we are loaning to a kernel-page.   we need to get the page
  375          * resident so we can wire it.   uvmfault_anonget will handle
  376          * this for us.
  377          */
  378 
  379         mutex_enter(&anon->an_lock);
  380         error = uvmfault_anonget(ufi, ufi->entry->aref.ar_amap, anon);
  381 
  382         /*
  383          * if we were unable to get the anon, then uvmfault_anonget has
  384          * unlocked everything and returned an error code.
  385          */
  386 
  387         if (error) {
  388                 UVMHIST_LOG(loanhist, "error %d", error,0,0,0);
  389 
  390                 /* need to refault (i.e. refresh our lookup) ? */
  391                 if (error == ERESTART) {
  392                         return (0);
  393                 }
  394 
  395                 /* "try again"?   sleep a bit and retry ... */
  396                 if (error == EAGAIN) {
  397                         kpause("loanagain", false, hz/2, NULL);
  398                         return (0);
  399                 }
  400 
  401                 /* otherwise flag it as an error */
  402                 return (-1);
  403         }
  404 
  405         /*
  406          * we have the page and its owner locked: do the loan now.
  407          */
  408 
  409         pg = anon->an_page;
  410         mutex_enter(&uvm_pageqlock);
  411         if (pg->wire_count > 0) {
  412                 mutex_exit(&uvm_pageqlock);
  413                 UVMHIST_LOG(loanhist, "->K wired %p", pg,0,0,0);
  414                 KASSERT(pg->uobject == NULL);
  415                 uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap,
  416                     NULL, anon);
  417                 return (-1);
  418         }
  419         if (pg->loan_count == 0) {
  420                 pmap_page_protect(pg, VM_PROT_READ);
  421         }
  422         pg->loan_count++;
  423         uvm_pageactivate(pg);
  424         mutex_exit(&uvm_pageqlock);
  425         **output = pg;
  426         (*output)++;
  427 
  428         /* unlock anon and return success */
  429         if (pg->uobject)
  430                 mutex_exit(&pg->uobject->vmobjlock);
  431         mutex_exit(&anon->an_lock);
  432         UVMHIST_LOG(loanhist, "->K done", 0,0,0,0);
  433         return (1);
  434 }
  435 
  436 /*
  437  * uvm_loanpage: loan out pages to kernel (->K)
  438  *
  439  * => pages should be object-owned and the object should be locked.
  440  * => in the case of error, the object might be unlocked and relocked.
  441  * => caller should busy the pages beforehand.
  442  * => pages will be unbusied.
  443  * => fail with EBUSY if meet a wired page.
  444  */
  445 static int
  446 uvm_loanpage(struct vm_page **pgpp, int npages)
  447 {
  448         int i;
  449         int error = 0;
  450 
  451         UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
  452 
  453         for (i = 0; i < npages; i++) {
  454                 struct vm_page *pg = pgpp[i];
  455 
  456                 KASSERT(pg->uobject != NULL);
  457                 KASSERT(pg->uobject == pgpp[0]->uobject);
  458                 KASSERT(!(pg->flags & (PG_RELEASED|PG_PAGEOUT)));
  459                 KASSERT(mutex_owned(&pg->uobject->vmobjlock));
  460                 KASSERT(pg->flags & PG_BUSY);
  461 
  462                 mutex_enter(&uvm_pageqlock);
  463                 if (pg->wire_count > 0) {
  464                         mutex_exit(&uvm_pageqlock);
  465                         UVMHIST_LOG(loanhist, "wired %p", pg,0,0,0);
  466                         error = EBUSY;
  467                         break;
  468                 }
  469                 if (pg->loan_count == 0) {
  470                         pmap_page_protect(pg, VM_PROT_READ);
  471                 }
  472                 pg->loan_count++;
  473                 uvm_pageactivate(pg);
  474                 mutex_exit(&uvm_pageqlock);
  475         }
  476 
  477         uvm_page_unbusy(pgpp, npages);
  478 
  479         if (error) {
  480                 /*
  481                  * backout what we've done
  482                  */
  483                 kmutex_t *slock = &pgpp[0]->uobject->vmobjlock;
  484 
  485                 mutex_exit(slock);
  486                 uvm_unloan(pgpp, i, UVM_LOAN_TOPAGE);
  487                 mutex_enter(slock);
  488         }
  489 
  490         UVMHIST_LOG(loanhist, "done %d", error,0,0,0);
  491         return error;
  492 }
  493 
  494 /*
  495  * XXX UBC temp limit
  496  * number of pages to get at once.
  497  * should be <= MAX_READ_AHEAD in genfs_vnops.c
  498  */
  499 #define UVM_LOAN_GET_CHUNK      16
  500 
  501 /*
  502  * uvm_loanuobjpages: loan pages from a uobj out (O->K)
  503  *
  504  * => uobj shouldn't be locked.  (we'll lock it)
  505  * => fail with EBUSY if we meet a wired page.
  506  */
  507 int
  508 uvm_loanuobjpages(struct uvm_object *uobj, voff_t pgoff, int orignpages,
  509     struct vm_page **origpgpp)
  510 {
  511         int ndone; /* # of pages loaned out */
  512         struct vm_page **pgpp;
  513         int error;
  514         int i;
  515         kmutex_t *slock;
  516 
  517         pgpp = origpgpp;
  518         for (ndone = 0; ndone < orignpages; ) {
  519                 int npages;
  520                 /* npendloan: # of pages busied but not loand out yet. */
  521                 int npendloan = 0xdead; /* XXX gcc */
  522 reget:
  523                 npages = MIN(UVM_LOAN_GET_CHUNK, orignpages - ndone);
  524                 mutex_enter(&uobj->vmobjlock);
  525                 error = (*uobj->pgops->pgo_get)(uobj,
  526                     pgoff + (ndone << PAGE_SHIFT), pgpp, &npages, 0,
  527                     VM_PROT_READ, 0, PGO_SYNCIO);
  528                 if (error == EAGAIN) {
  529                         kpause("loanuopg", false, hz/2, NULL);
  530                         continue;
  531                 }
  532                 if (error)
  533                         goto fail;
  534 
  535                 KASSERT(npages > 0);
  536 
  537                 /* loan and unbusy pages */
  538                 slock = NULL;
  539                 for (i = 0; i < npages; i++) {
  540                         kmutex_t *nextslock; /* slock for next page */
  541                         struct vm_page *pg = *pgpp;
  542 
  543                         /* XXX assuming that the page is owned by uobj */
  544                         KASSERT(pg->uobject != NULL);
  545                         nextslock = &pg->uobject->vmobjlock;
  546 
  547                         if (slock != nextslock) {
  548                                 if (slock) {
  549                                         KASSERT(npendloan > 0);
  550                                         error = uvm_loanpage(pgpp - npendloan,
  551                                             npendloan);
  552                                         mutex_exit(slock);
  553                                         if (error)
  554                                                 goto fail;
  555                                         ndone += npendloan;
  556                                         KASSERT(origpgpp + ndone == pgpp);
  557                                 }
  558                                 slock = nextslock;
  559                                 npendloan = 0;
  560                                 mutex_enter(slock);
  561                         }
  562 
  563                         if ((pg->flags & PG_RELEASED) != 0) {
  564                                 /*
  565                                  * release pages and try again.
  566                                  */
  567                                 mutex_exit(slock);
  568                                 for (; i < npages; i++) {
  569                                         pg = pgpp[i];
  570                                         slock = &pg->uobject->vmobjlock;
  571 
  572                                         mutex_enter(slock);
  573                                         mutex_enter(&uvm_pageqlock);
  574                                         uvm_page_unbusy(&pg, 1);
  575                                         mutex_exit(&uvm_pageqlock);
  576                                         mutex_exit(slock);
  577                                 }
  578                                 goto reget;
  579                         }
  580 
  581                         npendloan++;
  582                         pgpp++;
  583                         KASSERT(origpgpp + ndone + npendloan == pgpp);
  584                 }
  585                 KASSERT(slock != NULL);
  586                 KASSERT(npendloan > 0);
  587                 error = uvm_loanpage(pgpp - npendloan, npendloan);
  588                 mutex_exit(slock);
  589                 if (error)
  590                         goto fail;
  591                 ndone += npendloan;
  592                 KASSERT(origpgpp + ndone == pgpp);
  593         }
  594 
  595         return 0;
  596 
  597 fail:
  598         uvm_unloan(origpgpp, ndone, UVM_LOAN_TOPAGE);
  599 
  600         return error;
  601 }
  602 
  603 /*
  604  * uvm_loanuobj: loan a page from a uobj out
  605  *
  606  * => called with map, amap, uobj locked
  607  * => return value:
  608  *      -1 = fatal error, everything is unlocked, abort.
  609  *       0 = lookup in ufi went stale, everything unlocked, relookup and
  610  *              try again
  611  *       1 = got it, everything still locked
  612  */
  613 
  614 static int
  615 uvm_loanuobj(struct uvm_faultinfo *ufi, void ***output, int flags, vaddr_t va)
  616 {
  617         struct vm_amap *amap = ufi->entry->aref.ar_amap;
  618         struct uvm_object *uobj = ufi->entry->object.uvm_obj;
  619         struct vm_page *pg;
  620         struct vm_anon *anon;
  621         int error, npages;
  622         bool locked;
  623 
  624         UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
  625 
  626         /*
  627          * first we must make sure the page is resident.
  628          *
  629          * XXXCDC: duplicate code with uvm_fault().
  630          */
  631 
  632         /* locked: maps(read), amap(if there) */
  633         mutex_enter(&uobj->vmobjlock);
  634         /* locked: maps(read), amap(if there), uobj */
  635 
  636         if (uobj->pgops->pgo_get) {     /* try locked pgo_get */
  637                 npages = 1;
  638                 pg = NULL;
  639                 error = (*uobj->pgops->pgo_get)(uobj,
  640                     va - ufi->entry->start + ufi->entry->offset,
  641                     &pg, &npages, 0, VM_PROT_READ, MADV_NORMAL, PGO_LOCKED);
  642         } else {
  643                 error = EIO;            /* must have pgo_get op */
  644         }
  645 
  646         /*
  647          * check the result of the locked pgo_get.  if there is a problem,
  648          * then we fail the loan.
  649          */
  650 
  651         if (error && error != EBUSY) {
  652                 uvmfault_unlockall(ufi, amap, uobj, NULL);
  653                 return (-1);
  654         }
  655 
  656         /*
  657          * if we need to unlock for I/O, do so now.
  658          */
  659 
  660         if (error == EBUSY) {
  661                 uvmfault_unlockall(ufi, amap, NULL, NULL);
  662 
  663                 /* locked: uobj */
  664                 npages = 1;
  665                 error = (*uobj->pgops->pgo_get)(uobj,
  666                     va - ufi->entry->start + ufi->entry->offset,
  667                     &pg, &npages, 0, VM_PROT_READ, MADV_NORMAL, PGO_SYNCIO);
  668                 /* locked: <nothing> */
  669 
  670                 if (error) {
  671                         if (error == EAGAIN) {
  672                                 kpause("fltagain2", false, hz/2, NULL);
  673                                 return (0);
  674                         }
  675                         return (-1);
  676                 }
  677 
  678                 /*
  679                  * pgo_get was a success.   attempt to relock everything.
  680                  */
  681 
  682                 locked = uvmfault_relock(ufi);
  683                 if (locked && amap)
  684                         amap_lock(amap);
  685                 uobj = pg->uobject;
  686                 mutex_enter(&uobj->vmobjlock);
  687 
  688                 /*
  689                  * verify that the page has not be released and re-verify
  690                  * that amap slot is still free.   if there is a problem we
  691                  * drop our lock (thus force a lookup refresh/retry).
  692                  */
  693 
  694                 if ((pg->flags & PG_RELEASED) != 0 ||
  695                     (locked && amap && amap_lookup(&ufi->entry->aref,
  696                     ufi->orig_rvaddr - ufi->entry->start))) {
  697                         if (locked)
  698                                 uvmfault_unlockall(ufi, amap, NULL, NULL);
  699                         locked = false;
  700                 }
  701 
  702                 /*
  703                  * didn't get the lock?   release the page and retry.
  704                  */
  705 
  706                 if (locked == false) {
  707                         if (pg->flags & PG_WANTED) {
  708                                 wakeup(pg);
  709                         }
  710                         if (pg->flags & PG_RELEASED) {
  711                                 mutex_enter(&uvm_pageqlock);
  712                                 uvm_pagefree(pg);
  713                                 mutex_exit(&uvm_pageqlock);
  714                                 mutex_exit(&uobj->vmobjlock);
  715                                 return (0);
  716                         }
  717                         mutex_enter(&uvm_pageqlock);
  718                         uvm_pageactivate(pg);
  719                         mutex_exit(&uvm_pageqlock);
  720                         pg->flags &= ~(PG_BUSY|PG_WANTED);
  721                         UVM_PAGE_OWN(pg, NULL);
  722                         mutex_exit(&uobj->vmobjlock);
  723                         return (0);
  724                 }
  725         }
  726 
  727         KASSERT(uobj == pg->uobject);
  728 
  729         /*
  730          * at this point we have the page we want ("pg") marked PG_BUSY for us
  731          * and we have all data structures locked.  do the loanout.  page can
  732          * not be PG_RELEASED (we caught this above).
  733          */
  734 
  735         if ((flags & UVM_LOAN_TOANON) == 0) {
  736                 if (uvm_loanpage(&pg, 1)) {
  737                         uvmfault_unlockall(ufi, amap, uobj, NULL);
  738                         return (-1);
  739                 }
  740                 mutex_exit(&uobj->vmobjlock);
  741                 **output = pg;
  742                 (*output)++;
  743                 return (1);
  744         }
  745 
  746         /*
  747          * must be a loan to an anon.   check to see if there is already
  748          * an anon associated with this page.  if so, then just return
  749          * a reference to this object.   the page should already be
  750          * mapped read-only because it is already on loan.
  751          */
  752 
  753         if (pg->uanon) {
  754                 anon = pg->uanon;
  755                 mutex_enter(&anon->an_lock);
  756                 anon->an_ref++;
  757                 mutex_exit(&anon->an_lock);
  758                 if (pg->flags & PG_WANTED) {
  759                         wakeup(pg);
  760                 }
  761                 pg->flags &= ~(PG_WANTED|PG_BUSY);
  762                 UVM_PAGE_OWN(pg, NULL);
  763                 mutex_exit(&uobj->vmobjlock);
  764                 **output = anon;
  765                 (*output)++;
  766                 return (1);
  767         }
  768 
  769         /*
  770          * need to allocate a new anon
  771          */
  772 
  773         anon = uvm_analloc();
  774         if (anon == NULL) {
  775                 goto fail;
  776         }
  777         anon->an_page = pg;
  778         pg->uanon = anon;
  779         mutex_enter(&uvm_pageqlock);
  780         if (pg->wire_count > 0) {
  781                 mutex_exit(&uvm_pageqlock);
  782                 UVMHIST_LOG(loanhist, "wired %p", pg,0,0,0);
  783                 pg->uanon = NULL;
  784                 anon->an_page = NULL;
  785                 anon->an_ref--;
  786                 mutex_exit(&anon->an_lock);
  787                 uvm_anfree(anon);
  788                 goto fail;
  789         }
  790         if (pg->loan_count == 0) {
  791                 pmap_page_protect(pg, VM_PROT_READ);
  792         }
  793         pg->loan_count++;
  794         uvm_pageactivate(pg);
  795         mutex_exit(&uvm_pageqlock);
  796         if (pg->flags & PG_WANTED) {
  797                 wakeup(pg);
  798         }
  799         pg->flags &= ~(PG_WANTED|PG_BUSY);
  800         UVM_PAGE_OWN(pg, NULL);
  801         mutex_exit(&uobj->vmobjlock);
  802         mutex_exit(&anon->an_lock);
  803         **output = anon;
  804         (*output)++;
  805         return (1);
  806 
  807 fail:
  808         UVMHIST_LOG(loanhist, "fail", 0,0,0,0);
  809         /*
  810          * unlock everything and bail out.
  811          */
  812         if (pg->flags & PG_WANTED) {
  813                 wakeup(pg);
  814         }
  815         pg->flags &= ~(PG_WANTED|PG_BUSY);
  816         UVM_PAGE_OWN(pg, NULL);
  817         uvmfault_unlockall(ufi, amap, uobj, NULL);
  818         return (-1);
  819 }
  820 
  821 /*
  822  * uvm_loanzero: loan a zero-fill page out
  823  *
  824  * => called with map, amap, uobj locked
  825  * => return value:
  826  *      -1 = fatal error, everything is unlocked, abort.
  827  *       0 = lookup in ufi went stale, everything unlocked, relookup and
  828  *              try again
  829  *       1 = got it, everything still locked
  830  */
  831 
  832 static struct uvm_object uvm_loanzero_object;
  833 
  834 static int
  835 uvm_loanzero(struct uvm_faultinfo *ufi, void ***output, int flags)
  836 {
  837         struct vm_anon *anon;
  838         struct vm_page *pg;
  839         struct vm_amap *amap = ufi->entry->aref.ar_amap;
  840 
  841         UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
  842 again:
  843         mutex_enter(&uvm_loanzero_object.vmobjlock);
  844 
  845         /*
  846          * first, get ahold of our single zero page.
  847          */
  848 
  849         if (__predict_false((pg =
  850                              TAILQ_FIRST(&uvm_loanzero_object.memq)) == NULL)) {
  851                 while ((pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL,
  852                                            UVM_PGA_ZERO)) == NULL) {
  853                         mutex_exit(&uvm_loanzero_object.vmobjlock);
  854                         uvmfault_unlockall(ufi, amap, NULL, NULL);
  855                         uvm_wait("loanzero");
  856                         if (!uvmfault_relock(ufi)) {
  857                                 return (0);
  858                         }
  859                         if (amap) {
  860                                 amap_lock(amap);
  861                         }
  862                         goto again;
  863                 }
  864 
  865                 /* got a zero'd page. */
  866                 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
  867                 pg->flags |= PG_RDONLY;
  868                 mutex_enter(&uvm_pageqlock);
  869                 uvm_pageactivate(pg);
  870                 mutex_exit(&uvm_pageqlock);
  871                 UVM_PAGE_OWN(pg, NULL);
  872         }
  873 
  874         if ((flags & UVM_LOAN_TOANON) == 0) {   /* loaning to kernel-page */
  875                 mutex_enter(&uvm_pageqlock);
  876                 pg->loan_count++;
  877                 mutex_exit(&uvm_pageqlock);
  878                 mutex_exit(&uvm_loanzero_object.vmobjlock);
  879                 **output = pg;
  880                 (*output)++;
  881                 return (1);
  882         }
  883 
  884         /*
  885          * loaning to an anon.  check to see if there is already an anon
  886          * associated with this page.  if so, then just return a reference
  887          * to this object.
  888          */
  889 
  890         if (pg->uanon) {
  891                 anon = pg->uanon;
  892                 mutex_enter(&anon->an_lock);
  893                 anon->an_ref++;
  894                 mutex_exit(&anon->an_lock);
  895                 mutex_exit(&uvm_loanzero_object.vmobjlock);
  896                 **output = anon;
  897                 (*output)++;
  898                 return (1);
  899         }
  900 
  901         /*
  902          * need to allocate a new anon
  903          */
  904 
  905         anon = uvm_analloc();
  906         if (anon == NULL) {
  907                 /* out of swap causes us to fail */
  908                 mutex_exit(&uvm_loanzero_object.vmobjlock);
  909                 uvmfault_unlockall(ufi, amap, NULL, NULL);
  910                 return (-1);
  911         }
  912         anon->an_page = pg;
  913         pg->uanon = anon;
  914         mutex_enter(&uvm_pageqlock);
  915         pg->loan_count++;
  916         uvm_pageactivate(pg);
  917         mutex_exit(&uvm_pageqlock);
  918         mutex_exit(&anon->an_lock);
  919         mutex_exit(&uvm_loanzero_object.vmobjlock);
  920         **output = anon;
  921         (*output)++;
  922         return (1);
  923 }
  924 
  925 
  926 /*
  927  * uvm_unloananon: kill loans on anons (basically a normal ref drop)
  928  *
  929  * => we expect all our resources to be unlocked
  930  */
  931 
  932 static void
  933 uvm_unloananon(struct vm_anon **aloans, int nanons)
  934 {
  935         struct vm_anon *anon;
  936 
  937         while (nanons-- > 0) {
  938                 int refs;
  939 
  940                 anon = *aloans++;
  941                 mutex_enter(&anon->an_lock);
  942                 refs = --anon->an_ref;
  943                 mutex_exit(&anon->an_lock);
  944 
  945                 if (refs == 0) {
  946                         uvm_anfree(anon);
  947                 }
  948         }
  949 }
  950 
  951 /*
  952  * uvm_unloanpage: kill loans on pages loaned out to the kernel
  953  *
  954  * => we expect all our resources to be unlocked
  955  */
  956 
  957 static void
  958 uvm_unloanpage(struct vm_page **ploans, int npages)
  959 {
  960         struct vm_page *pg;
  961         kmutex_t *slock;
  962 
  963         mutex_enter(&uvm_pageqlock);
  964         while (npages-- > 0) {
  965                 pg = *ploans++;
  966 
  967                 /*
  968                  * do a little dance to acquire the object or anon lock
  969                  * as appropriate.  we are locking in the wrong order,
  970                  * so we have to do a try-lock here.
  971                  */
  972 
  973                 slock = NULL;
  974                 while (pg->uobject != NULL || pg->uanon != NULL) {
  975                         if (pg->uobject != NULL) {
  976                                 slock = &pg->uobject->vmobjlock;
  977                         } else {
  978                                 slock = &pg->uanon->an_lock;
  979                         }
  980                         if (mutex_tryenter(slock)) {
  981                                 break;
  982                         }
  983                         mutex_exit(&uvm_pageqlock);
  984                         /* XXX Better than yielding but inadequate. */
  985                         kpause("livelock", false, 1, NULL);
  986                         mutex_enter(&uvm_pageqlock);
  987                         slock = NULL;
  988                 }
  989 
  990                 /*
  991                  * drop our loan.  if page is owned by an anon but
  992                  * PQ_ANON is not set, the page was loaned to the anon
  993                  * from an object which dropped ownership, so resolve
  994                  * this by turning the anon's loan into real ownership
  995                  * (ie. decrement loan_count again and set PQ_ANON).
  996                  * after all this, if there are no loans left, put the
  997                  * page back a paging queue (if the page is owned by
  998                  * an anon) or free it (if the page is now unowned).
  999                  */
 1000 
 1001                 KASSERT(pg->loan_count > 0);
 1002                 pg->loan_count--;
 1003                 if (pg->uobject == NULL && pg->uanon != NULL &&
 1004                     (pg->pqflags & PQ_ANON) == 0) {
 1005                         KASSERT(pg->loan_count > 0);
 1006                         pg->loan_count--;
 1007                         pg->pqflags |= PQ_ANON;
 1008                 }
 1009                 if (pg->loan_count == 0 && pg->uobject == NULL &&
 1010                     pg->uanon == NULL) {
 1011                         KASSERT((pg->flags & PG_BUSY) == 0);
 1012                         uvm_pagefree(pg);
 1013                 }
 1014                 if (slock != NULL) {
 1015                         mutex_exit(slock);
 1016                 }
 1017         }
 1018         mutex_exit(&uvm_pageqlock);
 1019 }
 1020 
 1021 /*
 1022  * uvm_unloan: kill loans on pages or anons.
 1023  */
 1024 
 1025 void
 1026 uvm_unloan(void *v, int npages, int flags)
 1027 {
 1028         if (flags & UVM_LOAN_TOANON) {
 1029                 uvm_unloananon(v, npages);
 1030         } else {
 1031                 uvm_unloanpage(v, npages);
 1032         }
 1033 }
 1034 
 1035 /*
 1036  * Minimal pager for uvm_loanzero_object.  We need to provide a "put"
 1037  * method, because the page can end up on a paging queue, and the
 1038  * page daemon will want to call pgo_put when it encounters the page
 1039  * on the inactive list.
 1040  */
 1041 
 1042 static int
 1043 ulz_put(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
 1044 {
 1045         struct vm_page *pg;
 1046 
 1047         KDASSERT(uobj == &uvm_loanzero_object);
 1048 
 1049         /*
 1050          * Don't need to do any work here if we're not freeing pages.
 1051          */
 1052 
 1053         if ((flags & PGO_FREE) == 0) {
 1054                 mutex_exit(&uobj->vmobjlock);
 1055                 return 0;
 1056         }
 1057 
 1058         /*
 1059          * we don't actually want to ever free the uvm_loanzero_page, so
 1060          * just reactivate or dequeue it.
 1061          */
 1062 
 1063         pg = TAILQ_FIRST(&uobj->memq);
 1064         KASSERT(pg != NULL);
 1065         KASSERT(TAILQ_NEXT(pg, listq.queue) == NULL);
 1066 
 1067         mutex_enter(&uvm_pageqlock);
 1068         if (pg->uanon)
 1069                 uvm_pageactivate(pg);
 1070         else
 1071                 uvm_pagedequeue(pg);
 1072         mutex_exit(&uvm_pageqlock);
 1073 
 1074         mutex_exit(&uobj->vmobjlock);
 1075         return 0;
 1076 }
 1077 
 1078 static const struct uvm_pagerops ulz_pager = {
 1079         .pgo_put = ulz_put,
 1080 };
 1081 
 1082 /*
 1083  * uvm_loan_init(): initialize the uvm_loan() facility.
 1084  */
 1085 
 1086 void
 1087 uvm_loan_init(void)
 1088 {
 1089 
 1090         UVM_OBJ_INIT(&uvm_loanzero_object, &ulz_pager, 0);
 1091 
 1092         UVMHIST_INIT(loanhist, 300);
 1093 }
 1094 
 1095 /*
 1096  * uvm_loanbreak: break loan on a uobj page
 1097  *
 1098  * => called with uobj locked
 1099  * => the page should be busy
 1100  * => return value:
 1101  *      newly allocated page if succeeded
 1102  */
 1103 struct vm_page *
 1104 uvm_loanbreak(struct vm_page *uobjpage)
 1105 {
 1106         struct vm_page *pg;
 1107 #ifdef DIAGNOSTIC
 1108         struct uvm_object *uobj = uobjpage->uobject;
 1109 #endif
 1110 
 1111         KASSERT(uobj != NULL);
 1112         KASSERT(mutex_owned(&uobj->vmobjlock));
 1113         KASSERT(uobjpage->flags & PG_BUSY);
 1114 
 1115         /* alloc new un-owned page */
 1116         pg = uvm_pagealloc(NULL, 0, NULL, 0);
 1117         if (pg == NULL)
 1118                 return NULL;
 1119 
 1120         /*
 1121          * copy the data from the old page to the new
 1122          * one and clear the fake flags on the new page (keep it busy).
 1123          * force a reload of the old page by clearing it from all
 1124          * pmaps.
 1125          * transfer dirtiness of the old page to the new page.
 1126          * then lock the page queues to rename the pages.
 1127          */
 1128 
 1129         uvm_pagecopy(uobjpage, pg);     /* old -> new */
 1130         pg->flags &= ~PG_FAKE;
 1131         pmap_page_protect(uobjpage, VM_PROT_NONE);
 1132         if ((uobjpage->flags & PG_CLEAN) != 0 && !pmap_clear_modify(uobjpage)) {
 1133                 pmap_clear_modify(pg);
 1134                 pg->flags |= PG_CLEAN;
 1135         } else {
 1136                 /* uvm_pagecopy marked it dirty */
 1137                 KASSERT((pg->flags & PG_CLEAN) == 0);
 1138                 /* a object with a dirty page should be dirty. */
 1139                 KASSERT(!UVM_OBJ_IS_CLEAN(uobj));
 1140         }
 1141         if (uobjpage->flags & PG_WANTED)
 1142                 wakeup(uobjpage);
 1143         /* uobj still locked */
 1144         uobjpage->flags &= ~(PG_WANTED|PG_BUSY);
 1145         UVM_PAGE_OWN(uobjpage, NULL);
 1146 
 1147         mutex_enter(&uvm_pageqlock);
 1148 
 1149         /*
 1150          * replace uobjpage with new page.
 1151          */
 1152 
 1153         uvm_pagereplace(uobjpage, pg);
 1154 
 1155         /*
 1156          * if the page is no longer referenced by
 1157          * an anon (i.e. we are breaking an O->K
 1158          * loan), then remove it from any pageq's.
 1159          */
 1160         if (uobjpage->uanon == NULL)
 1161                 uvm_pagedequeue(uobjpage);
 1162 
 1163         /*
 1164          * at this point we have absolutely no
 1165          * control over uobjpage
 1166          */
 1167 
 1168         /* install new page */
 1169         uvm_pageactivate(pg);
 1170         mutex_exit(&uvm_pageqlock);
 1171 
 1172         /*
 1173          * done!  loan is broken and "pg" is
 1174          * PG_BUSY.   it can now replace uobjpage.
 1175          */
 1176 
 1177         return pg;
 1178 }
 1179 
 1180 int
 1181 uvm_loanbreak_anon(struct vm_anon *anon, struct uvm_object *uobj)
 1182 {
 1183         struct vm_page *pg;
 1184 
 1185         KASSERT(mutex_owned(&anon->an_lock));
 1186         KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
 1187 
 1188         /* get new un-owned replacement page */
 1189         pg = uvm_pagealloc(NULL, 0, NULL, 0);
 1190         if (pg == NULL) {
 1191                 return ENOMEM;
 1192         }
 1193 
 1194         /*
 1195          * copy data, kill loan, and drop uobj lock (if any)
 1196          */
 1197         /* copy old -> new */
 1198         uvm_pagecopy(anon->an_page, pg);
 1199 
 1200         /* force reload */
 1201         pmap_page_protect(anon->an_page, VM_PROT_NONE);
 1202         mutex_enter(&uvm_pageqlock);      /* KILL loan */
 1203 
 1204         anon->an_page->uanon = NULL;
 1205         /* in case we owned */
 1206         anon->an_page->pqflags &= ~PQ_ANON;
 1207 
 1208         if (uobj) {
 1209                 /* if we were receiver of loan */
 1210                 anon->an_page->loan_count--;
 1211         } else {
 1212                 /*
 1213                  * we were the lender (A->K); need to remove the page from
 1214                  * pageq's.
 1215                  */
 1216                 uvm_pagedequeue(anon->an_page);
 1217         }
 1218 
 1219         if (uobj) {
 1220                 mutex_exit(&uobj->vmobjlock);
 1221         }
 1222 
 1223         /* install new page in anon */
 1224         anon->an_page = pg;
 1225         pg->uanon = anon;
 1226         pg->pqflags |= PQ_ANON;
 1227 
 1228         uvm_pageactivate(pg);
 1229         mutex_exit(&uvm_pageqlock);
 1230 
 1231         pg->flags &= ~(PG_BUSY|PG_FAKE);
 1232         UVM_PAGE_OWN(pg, NULL);
 1233 
 1234         /* done! */
 1235 
 1236         return 0;
 1237 }

Cache object: b0f0d0447b25dcc75853214b18af1f87


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.