The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/uvm/uvm_anon.c

Version: -  FREEBSD  -  FREEBSD11  -  FREEBSD10  -  FREEBSD9  -  FREEBSD92  -  FREEBSD91  -  FREEBSD90  -  FREEBSD8  -  FREEBSD82  -  FREEBSD81  -  FREEBSD80  -  FREEBSD7  -  FREEBSD74  -  FREEBSD73  -  FREEBSD72  -  FREEBSD71  -  FREEBSD70  -  FREEBSD6  -  FREEBSD64  -  FREEBSD63  -  FREEBSD62  -  FREEBSD61  -  FREEBSD60  -  FREEBSD5  -  FREEBSD55  -  FREEBSD54  -  FREEBSD53  -  FREEBSD52  -  FREEBSD51  -  FREEBSD50  -  FREEBSD4  -  FREEBSD3  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*      $NetBSD: uvm_anon.c,v 1.31 2004/09/01 11:53:38 yamt Exp $       */
    2 
    3 /*
    4  *
    5  * Copyright (c) 1997 Charles D. Cranor and Washington University.
    6  * All rights reserved.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  * 3. All advertising materials mentioning features or use of this software
   17  *    must display the following acknowledgement:
   18  *      This product includes software developed by Charles D. Cranor and
   19  *      Washington University.
   20  * 4. The name of the author may not be used to endorse or promote products
   21  *    derived from this software without specific prior written permission.
   22  *
   23  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   24  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   25  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   26  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   27  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   28  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   29  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   30  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   31  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
   32  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   33  */
   34 
   35 /*
   36  * uvm_anon.c: uvm anon ops
   37  */
   38 
   39 #include <sys/cdefs.h>
   40 __KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.31 2004/09/01 11:53:38 yamt Exp $");
   41 
   42 #include "opt_uvmhist.h"
   43 
   44 #include <sys/param.h>
   45 #include <sys/systm.h>
   46 #include <sys/proc.h>
   47 #include <sys/malloc.h>
   48 #include <sys/pool.h>
   49 #include <sys/kernel.h>
   50 
   51 #include <uvm/uvm.h>
   52 #include <uvm/uvm_swap.h>
   53 
   54 /*
   55  * anonblock_list: global list of anon blocks,
   56  * locked by swap_syscall_lock (since we never remove
   57  * anything from this list and we only add to it via swapctl(2)).
   58  */
   59 
   60 struct uvm_anonblock {
   61         LIST_ENTRY(uvm_anonblock) list;
   62         int count;
   63         struct vm_anon *anons;
   64 };
   65 static LIST_HEAD(anonlist, uvm_anonblock) anonblock_list;
   66 
   67 
   68 static boolean_t anon_pagein(struct vm_anon *);
   69 
   70 
   71 /*
   72  * allocate anons
   73  */
   74 void
   75 uvm_anon_init()
   76 {
   77         int nanon = uvmexp.free - (uvmexp.free / 16); /* XXXCDC ??? */
   78 
   79         simple_lock_init(&uvm.afreelock);
   80         LIST_INIT(&anonblock_list);
   81 
   82         /*
   83          * Allocate the initial anons.
   84          */
   85         uvm_anon_add(nanon);
   86 }
   87 
   88 /*
   89  * add some more anons to the free pool.  called when we add
   90  * more swap space.
   91  *
   92  * => swap_syscall_lock should be held (protects anonblock_list).
   93  */
   94 int
   95 uvm_anon_add(count)
   96         int     count;
   97 {
   98         struct uvm_anonblock *anonblock;
   99         struct vm_anon *anon;
  100         int lcv, needed;
  101 
  102         simple_lock(&uvm.afreelock);
  103         uvmexp.nanonneeded += count;
  104         needed = uvmexp.nanonneeded - uvmexp.nanon;
  105         simple_unlock(&uvm.afreelock);
  106 
  107         if (needed <= 0) {
  108                 return 0;
  109         }
  110         anon = (void *)uvm_km_alloc(kernel_map, sizeof(*anon) * needed);
  111         if (anon == NULL) {
  112                 simple_lock(&uvm.afreelock);
  113                 uvmexp.nanonneeded -= count;
  114                 simple_unlock(&uvm.afreelock);
  115                 return ENOMEM;
  116         }
  117         MALLOC(anonblock, void *, sizeof(*anonblock), M_UVMAMAP, M_WAITOK);
  118 
  119         anonblock->count = needed;
  120         anonblock->anons = anon;
  121         LIST_INSERT_HEAD(&anonblock_list, anonblock, list);
  122         memset(anon, 0, sizeof(*anon) * needed);
  123 
  124         simple_lock(&uvm.afreelock);
  125         uvmexp.nanon += needed;
  126         uvmexp.nfreeanon += needed;
  127         for (lcv = 0; lcv < needed; lcv++) {
  128                 simple_lock_init(&anon[lcv].an_lock);
  129                 anon[lcv].u.an_nxt = uvm.afree;
  130                 uvm.afree = &anon[lcv];
  131         }
  132         simple_unlock(&uvm.afreelock);
  133         return 0;
  134 }
  135 
  136 /*
  137  * remove anons from the free pool.
  138  */
  139 void
  140 uvm_anon_remove(count)
  141         int count;
  142 {
  143         /*
  144          * we never actually free any anons, to avoid allocation overhead.
  145          * XXX someday we might want to try to free anons.
  146          */
  147 
  148         simple_lock(&uvm.afreelock);
  149         uvmexp.nanonneeded -= count;
  150         simple_unlock(&uvm.afreelock);
  151 }
  152 
  153 /*
  154  * allocate an anon
  155  *
  156  * => new anon is returned locked!
  157  */
  158 struct vm_anon *
  159 uvm_analloc()
  160 {
  161         struct vm_anon *a;
  162 
  163         simple_lock(&uvm.afreelock);
  164         a = uvm.afree;
  165         if (a) {
  166                 uvm.afree = a->u.an_nxt;
  167                 uvmexp.nfreeanon--;
  168                 a->an_ref = 1;
  169                 a->an_swslot = 0;
  170                 a->u.an_page = NULL;            /* so we can free quickly */
  171                 LOCK_ASSERT(simple_lock_held(&a->an_lock) == 0);
  172                 simple_lock(&a->an_lock);
  173         }
  174         simple_unlock(&uvm.afreelock);
  175         return(a);
  176 }
  177 
  178 /*
  179  * uvm_anfree: free a single anon structure
  180  *
  181  * => caller must remove anon from its amap before calling (if it was in
  182  *      an amap).
  183  * => anon must be unlocked and have a zero reference count.
  184  * => we may lock the pageq's.
  185  */
  186 
  187 void
  188 uvm_anfree(anon)
  189         struct vm_anon *anon;
  190 {
  191         struct vm_page *pg;
  192         UVMHIST_FUNC("uvm_anfree"); UVMHIST_CALLED(maphist);
  193         UVMHIST_LOG(maphist,"(anon=0x%x)", anon, 0,0,0);
  194 
  195         KASSERT(anon->an_ref == 0);
  196         LOCK_ASSERT(!simple_lock_held(&anon->an_lock));
  197 
  198         /*
  199          * get page
  200          */
  201 
  202         pg = anon->u.an_page;
  203 
  204         /*
  205          * if there is a resident page and it is loaned, then anon may not
  206          * own it.   call out to uvm_anon_lockpage() to ensure the real owner
  207          * of the page has been identified and locked.
  208          */
  209 
  210         if (pg && pg->loan_count) {
  211                 simple_lock(&anon->an_lock);
  212                 pg = uvm_anon_lockloanpg(anon);
  213                 simple_unlock(&anon->an_lock);
  214         }
  215 
  216         /*
  217          * if we have a resident page, we must dispose of it before freeing
  218          * the anon.
  219          */
  220 
  221         if (pg) {
  222 
  223                 /*
  224                  * if the page is owned by a uobject (now locked), then we must
  225                  * kill the loan on the page rather than free it.
  226                  */
  227 
  228                 if (pg->uobject) {
  229                         uvm_lock_pageq();
  230                         KASSERT(pg->loan_count > 0);
  231                         pg->loan_count--;
  232                         pg->uanon = NULL;
  233                         uvm_unlock_pageq();
  234                         simple_unlock(&pg->uobject->vmobjlock);
  235                 } else {
  236 
  237                         /*
  238                          * page has no uobject, so we must be the owner of it.
  239                          */
  240 
  241                         KASSERT((pg->flags & PG_RELEASED) == 0);
  242                         simple_lock(&anon->an_lock);
  243                         pmap_page_protect(pg, VM_PROT_NONE);
  244 
  245                         /*
  246                          * if the page is busy, mark it as PG_RELEASED
  247                          * so that uvm_anon_release will release it later.
  248                          */
  249 
  250                         if (pg->flags & PG_BUSY) {
  251                                 pg->flags |= PG_RELEASED;
  252                                 simple_unlock(&anon->an_lock);
  253                                 return;
  254                         }
  255                         uvm_lock_pageq();
  256                         uvm_pagefree(pg);
  257                         uvm_unlock_pageq();
  258                         simple_unlock(&anon->an_lock);
  259                         UVMHIST_LOG(maphist, "anon 0x%x, page 0x%x: "
  260                                     "freed now!", anon, pg, 0, 0);
  261                 }
  262         }
  263         if (pg == NULL && anon->an_swslot > 0) {
  264                 /* this page is no longer only in swap. */
  265                 simple_lock(&uvm.swap_data_lock);
  266                 KASSERT(uvmexp.swpgonly > 0);
  267                 uvmexp.swpgonly--;
  268                 simple_unlock(&uvm.swap_data_lock);
  269         }
  270 
  271         /*
  272          * free any swap resources.
  273          */
  274 
  275         uvm_anon_dropswap(anon);
  276 
  277         /*
  278          * now that we've stripped the data areas from the anon,
  279          * free the anon itself.
  280          */
  281 
  282         KASSERT(anon->u.an_page == NULL);
  283         KASSERT(anon->an_swslot == 0);
  284 
  285         simple_lock(&uvm.afreelock);
  286         anon->u.an_nxt = uvm.afree;
  287         uvm.afree = anon;
  288         uvmexp.nfreeanon++;
  289         simple_unlock(&uvm.afreelock);
  290         UVMHIST_LOG(maphist,"<- done!",0,0,0,0);
  291 }
  292 
  293 /*
  294  * uvm_anon_dropswap:  release any swap resources from this anon.
  295  *
  296  * => anon must be locked or have a reference count of 0.
  297  */
  298 void
  299 uvm_anon_dropswap(anon)
  300         struct vm_anon *anon;
  301 {
  302         UVMHIST_FUNC("uvm_anon_dropswap"); UVMHIST_CALLED(maphist);
  303 
  304         if (anon->an_swslot == 0)
  305                 return;
  306 
  307         UVMHIST_LOG(maphist,"freeing swap for anon %p, paged to swslot 0x%x",
  308                     anon, anon->an_swslot, 0, 0);
  309         uvm_swap_free(anon->an_swslot, 1);
  310         anon->an_swslot = 0;
  311 }
  312 
  313 /*
  314  * uvm_anon_lockloanpg: given a locked anon, lock its resident page
  315  *
  316  * => anon is locked by caller
  317  * => on return: anon is locked
  318  *               if there is a resident page:
  319  *                      if it has a uobject, it is locked by us
  320  *                      if it is ownerless, we take over as owner
  321  *               we return the resident page (it can change during
  322  *               this function)
  323  * => note that the only time an anon has an ownerless resident page
  324  *      is if the page was loaned from a uvm_object and the uvm_object
  325  *      disowned it
  326  * => this only needs to be called when you want to do an operation
  327  *      on an anon's resident page and that page has a non-zero loan
  328  *      count.
  329  */
  330 struct vm_page *
  331 uvm_anon_lockloanpg(anon)
  332         struct vm_anon *anon;
  333 {
  334         struct vm_page *pg;
  335         boolean_t locked = FALSE;
  336 
  337         LOCK_ASSERT(simple_lock_held(&anon->an_lock));
  338 
  339         /*
  340          * loop while we have a resident page that has a non-zero loan count.
  341          * if we successfully get our lock, we will "break" the loop.
  342          * note that the test for pg->loan_count is not protected -- this
  343          * may produce false positive results.   note that a false positive
  344          * result may cause us to do more work than we need to, but it will
  345          * not produce an incorrect result.
  346          */
  347 
  348         while (((pg = anon->u.an_page) != NULL) && pg->loan_count != 0) {
  349 
  350                 /*
  351                  * quickly check to see if the page has an object before
  352                  * bothering to lock the page queues.   this may also produce
  353                  * a false positive result, but that's ok because we do a real
  354                  * check after that.
  355                  */
  356 
  357                 if (pg->uobject) {
  358                         uvm_lock_pageq();
  359                         if (pg->uobject) {
  360                                 locked =
  361                                     simple_lock_try(&pg->uobject->vmobjlock);
  362                         } else {
  363                                 /* object disowned before we got PQ lock */
  364                                 locked = TRUE;
  365                         }
  366                         uvm_unlock_pageq();
  367 
  368                         /*
  369                          * if we didn't get a lock (try lock failed), then we
  370                          * toggle our anon lock and try again
  371                          */
  372 
  373                         if (!locked) {
  374                                 simple_unlock(&anon->an_lock);
  375 
  376                                 /*
  377                                  * someone locking the object has a chance to
  378                                  * lock us right now
  379                                  */
  380 
  381                                 simple_lock(&anon->an_lock);
  382                                 continue;
  383                         }
  384                 }
  385 
  386                 /*
  387                  * if page is un-owned [i.e. the object dropped its ownership],
  388                  * then we can take over as owner!
  389                  */
  390 
  391                 if (pg->uobject == NULL && (pg->pqflags & PQ_ANON) == 0) {
  392                         uvm_lock_pageq();
  393                         pg->pqflags |= PQ_ANON;
  394                         pg->loan_count--;
  395                         uvm_unlock_pageq();
  396                 }
  397                 break;
  398         }
  399         return(pg);
  400 }
  401 
  402 
  403 
  404 /*
  405  * page in every anon that is paged out to a range of swslots.
  406  *
  407  * swap_syscall_lock should be held (protects anonblock_list).
  408  */
  409 
  410 boolean_t
  411 anon_swap_off(startslot, endslot)
  412         int startslot, endslot;
  413 {
  414         struct uvm_anonblock *anonblock;
  415 
  416         LIST_FOREACH(anonblock, &anonblock_list, list) {
  417                 int i;
  418 
  419                 /*
  420                  * loop thru all the anons in the anonblock,
  421                  * paging in where needed.
  422                  */
  423 
  424                 for (i = 0; i < anonblock->count; i++) {
  425                         struct vm_anon *anon = &anonblock->anons[i];
  426                         int slot;
  427 
  428                         /*
  429                          * lock anon to work on it.
  430                          */
  431 
  432                         simple_lock(&anon->an_lock);
  433 
  434                         /*
  435                          * is this anon's swap slot in range?
  436                          */
  437 
  438                         slot = anon->an_swslot;
  439                         if (slot >= startslot && slot < endslot) {
  440                                 boolean_t rv;
  441 
  442                                 /*
  443                                  * yup, page it in.
  444                                  */
  445 
  446                                 /* locked: anon */
  447                                 rv = anon_pagein(anon);
  448                                 /* unlocked: anon */
  449 
  450                                 if (rv) {
  451                                         return rv;
  452                                 }
  453                         } else {
  454 
  455                                 /*
  456                                  * nope, unlock and proceed.
  457                                  */
  458 
  459                                 simple_unlock(&anon->an_lock);
  460                         }
  461                 }
  462         }
  463         return FALSE;
  464 }
  465 
  466 
  467 /*
  468  * fetch an anon's page.
  469  *
  470  * => anon must be locked, and is unlocked upon return.
  471  * => returns TRUE if pagein was aborted due to lack of memory.
  472  */
  473 
  474 static boolean_t
  475 anon_pagein(anon)
  476         struct vm_anon *anon;
  477 {
  478         struct vm_page *pg;
  479         struct uvm_object *uobj;
  480         int rv;
  481 
  482         /* locked: anon */
  483         LOCK_ASSERT(simple_lock_held(&anon->an_lock));
  484 
  485         rv = uvmfault_anonget(NULL, NULL, anon);
  486 
  487         /*
  488          * if rv == 0, anon is still locked, else anon
  489          * is unlocked
  490          */
  491 
  492         switch (rv) {
  493         case 0:
  494                 break;
  495 
  496         case EIO:
  497         case ERESTART:
  498 
  499                 /*
  500                  * nothing more to do on errors.
  501                  * ERESTART can only mean that the anon was freed,
  502                  * so again there's nothing to do.
  503                  */
  504 
  505                 return FALSE;
  506 
  507         default:
  508                 return TRUE;
  509         }
  510 
  511         /*
  512          * ok, we've got the page now.
  513          * mark it as dirty, clear its swslot and un-busy it.
  514          */
  515 
  516         pg = anon->u.an_page;
  517         uobj = pg->uobject;
  518         if (anon->an_swslot > 0)
  519                 uvm_swap_free(anon->an_swslot, 1);
  520         anon->an_swslot = 0;
  521         pg->flags &= ~(PG_CLEAN);
  522 
  523         /*
  524          * deactivate the page (to put it on a page queue)
  525          */
  526 
  527         pmap_clear_reference(pg);
  528         uvm_lock_pageq();
  529         if (pg->wire_count == 0)
  530                 uvm_pagedeactivate(pg);
  531         uvm_unlock_pageq();
  532 
  533         if (pg->flags & PG_WANTED) {
  534                 wakeup(pg);
  535                 pg->flags &= ~(PG_WANTED);
  536         }
  537 
  538         /*
  539          * unlock the anon and we're done.
  540          */
  541 
  542         simple_unlock(&anon->an_lock);
  543         if (uobj) {
  544                 simple_unlock(&uobj->vmobjlock);
  545         }
  546         return FALSE;
  547 }
  548 
  549 /*
  550  * uvm_anon_release: release an anon and its page.
  551  *
  552  * => caller must lock the anon.
  553  */
  554 
  555 void
  556 uvm_anon_release(anon)
  557         struct vm_anon *anon;
  558 {
  559         struct vm_page *pg = anon->u.an_page;
  560 
  561         LOCK_ASSERT(simple_lock_held(&anon->an_lock));
  562 
  563         KASSERT(pg != NULL);
  564         KASSERT((pg->flags & PG_RELEASED) != 0);
  565         KASSERT((pg->flags & PG_BUSY) != 0);
  566         KASSERT(pg->uobject == NULL);
  567         KASSERT(pg->uanon == anon);
  568         KASSERT(pg->loan_count == 0);
  569         KASSERT(anon->an_ref == 0);
  570 
  571         uvm_lock_pageq();
  572         uvm_pagefree(pg);
  573         uvm_unlock_pageq();
  574         simple_unlock(&anon->an_lock);
  575 
  576         KASSERT(anon->u.an_page == NULL);
  577 
  578         uvm_anfree(anon);
  579 }

Cache object: 0741e6bbb6f53fe952ec018caa2bc27b


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.