phys_pager.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * Copyright (c) 2000 Peter Wemm
    3  *
    4  * Redistribution and use in source and binary forms, with or without
    5  * modification, are permitted provided that the following conditions
    6  * are met:
    7  * 1. Redistributions of source code must retain the above copyright
    8  *    notice, this list of conditions and the following disclaimer.
    9  * 2. Redistributions in binary form must reproduce the above copyright
   10  *    notice, this list of conditions and the following disclaimer in the
   11  *    documentation and/or other materials provided with the distribution.
   12  *
   13  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
   14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   16  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
   17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   23  * SUCH DAMAGE.
   24  */
   25 
   26 #include <sys/cdefs.h>
   27 __FBSDID("$FreeBSD$");
   28 
   29 #include <sys/param.h>
   30 #include <sys/systm.h>
   31 #include <sys/conf.h>
   32 #include <sys/kernel.h>
   33 #include <sys/lock.h>
   34 #include <sys/proc.h>
   35 #include <sys/mutex.h>
   36 #include <sys/mman.h>
   37 #include <sys/rwlock.h>
   38 #include <sys/sysctl.h>
   39 
   40 #include <vm/vm.h>
   41 #include <vm/vm_param.h>
   42 #include <vm/vm_object.h>
   43 #include <vm/vm_page.h>
   44 #include <vm/vm_pageout.h>
   45 #include <vm/vm_pager.h>
   46 
   47 /* list of phys pager objects */
   48 static struct pagerlst phys_pager_object_list;
   49 /* protect access to phys_pager_object_list */
   50 static struct mtx phys_pager_mtx;
   51 
   52 static void
   53 phys_pager_init(void)
   54 {
   55 
   56         TAILQ_INIT(&phys_pager_object_list);
   57         mtx_init(&phys_pager_mtx, "phys_pager list", NULL, MTX_DEF);
   58 }
   59 
   60 static vm_object_t
   61 phys_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
   62     vm_ooffset_t foff, struct ucred *cred)
   63 {
   64         vm_object_t object, object1;
   65         vm_pindex_t pindex;
   66 
   67         /*
   68          * Offset should be page aligned.
   69          */
   70         if (foff & PAGE_MASK)
   71                 return (NULL);
   72 
   73         pindex = OFF_TO_IDX(foff + PAGE_MASK + size);
   74 
   75         if (handle != NULL) {
   76                 mtx_lock(&phys_pager_mtx);
   77                 /*
   78                  * Look up pager, creating as necessary.
   79                  */
   80                 object1 = NULL;
   81                 object = vm_pager_object_lookup(&phys_pager_object_list, handle);
   82                 if (object == NULL) {
   83                         /*
   84                          * Allocate object and associate it with the pager.
   85                          */
   86                         mtx_unlock(&phys_pager_mtx);
   87                         object1 = vm_object_allocate(OBJT_PHYS, pindex);
   88                         mtx_lock(&phys_pager_mtx);
   89                         object = vm_pager_object_lookup(&phys_pager_object_list,
   90                             handle);
   91                         if (object != NULL) {
   92                                 /*
   93                                  * We raced with other thread while
   94                                  * allocating object.
   95                                  */
   96                                 if (pindex > object->size)
   97                                         object->size = pindex;
   98                         } else {
   99                                 object = object1;
  100                                 object1 = NULL;
  101                                 object->handle = handle;
  102                                 vm_object_set_flag(object, OBJ_POPULATE);
  103                                 TAILQ_INSERT_TAIL(&phys_pager_object_list,
  104                                     object, pager_object_list);
  105                         }
  106                 } else {
  107                         if (pindex > object->size)
  108                                 object->size = pindex;
  109                 }
  110                 mtx_unlock(&phys_pager_mtx);
  111                 vm_object_deallocate(object1);
  112         } else {
  113                 object = vm_object_allocate(OBJT_PHYS, pindex);
  114                 vm_object_set_flag(object, OBJ_POPULATE);
  115         }
  116 
  117         return (object);
  118 }
  119 
  120 static void
  121 phys_pager_dealloc(vm_object_t object)
  122 {
  123 
  124         if (object->handle != NULL) {
  125                 VM_OBJECT_WUNLOCK(object);
  126                 mtx_lock(&phys_pager_mtx);
  127                 TAILQ_REMOVE(&phys_pager_object_list, object, pager_object_list);
  128                 mtx_unlock(&phys_pager_mtx);
  129                 VM_OBJECT_WLOCK(object);
  130         }
  131         object->handle = NULL;
  132         object->type = OBJT_DEAD;
  133 }
  134 
  135 /*
  136  * Fill as many pages as vm_fault has allocated for us.
  137  */
  138 static int
  139 phys_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind,
  140     int *rahead)
  141 {
  142         int i;
  143 
  144         VM_OBJECT_ASSERT_WLOCKED(object);
  145         for (i = 0; i < count; i++) {
  146                 if (m[i]->valid == 0) {
  147                         if ((m[i]->flags & PG_ZERO) == 0)
  148                                 pmap_zero_page(m[i]);
  149                         m[i]->valid = VM_PAGE_BITS_ALL;
  150                 }
  151                 KASSERT(m[i]->valid == VM_PAGE_BITS_ALL,
  152                     ("phys_pager_getpages: partially valid page %p", m[i]));
  153                 KASSERT(m[i]->dirty == 0,
  154                     ("phys_pager_getpages: dirty page %p", m[i]));
  155         }
  156         if (rbehind)
  157                 *rbehind = 0;
  158         if (rahead)
  159                 *rahead = 0;
  160         return (VM_PAGER_OK);
  161 }
  162 
  163 /*
  164  * Implement a pretty aggressive clustered getpages strategy.  Hint that
  165  * everything in an entire 4MB window should be prefaulted at once.
  166  *
  167  * 4MB (1024 slots per page table page) is convenient for x86,
  168  * but may not be for other arches.
  169  */
  170 #ifndef PHYSCLUSTER
  171 #define PHYSCLUSTER 1024
  172 #endif
  173 static int phys_pager_cluster = PHYSCLUSTER;
  174 SYSCTL_INT(_vm, OID_AUTO, phys_pager_cluster, CTLFLAG_RWTUN,
  175     &phys_pager_cluster, 0,
  176     "prefault window size for phys pager");
  177 
  178 /*
  179  * Max hint to vm_page_alloc() about the further allocation needs
  180  * inside the phys_pager_populate() loop.  The number of bits used to
  181  * implement VM_ALLOC_COUNT() determines the hard limit on this value.
  182  * That limit is currently 65535.
  183  */
  184 #define PHYSALLOC       16
  185 
  186 static int
  187 phys_pager_populate(vm_object_t object, vm_pindex_t pidx,
  188     int fault_type __unused, vm_prot_t max_prot __unused, vm_pindex_t *first,
  189     vm_pindex_t *last)
  190 {
  191         vm_page_t m;
  192         vm_pindex_t base, end, i;
  193         int ahead;
  194 
  195         base = rounddown(pidx, phys_pager_cluster);
  196         end = base + phys_pager_cluster - 1;
  197         if (end >= object->size)
  198                 end = object->size - 1;
  199         if (*first > base)
  200                 base = *first;
  201         if (end > *last)
  202                 end = *last;
  203         *first = base;
  204         *last = end;
  205 
  206         for (i = base; i <= end; i++) {
  207 retry:
  208                 m = vm_page_lookup(object, i);
  209                 if (m == NULL) {
  210                         ahead = MIN(end - i, PHYSALLOC);
  211                         m = vm_page_alloc(object, i, VM_ALLOC_NORMAL |
  212                             VM_ALLOC_ZERO | VM_ALLOC_WAITFAIL |
  213                             VM_ALLOC_COUNT(ahead));
  214                         if (m == NULL)
  215                                 goto retry;
  216                         if ((m->flags & PG_ZERO) == 0)
  217                                 pmap_zero_page(m);
  218                         m->valid = VM_PAGE_BITS_ALL;
  219                 } else if (vm_page_xbusied(m)) {
  220                         vm_page_lock(m);
  221                         VM_OBJECT_WUNLOCK(object);
  222                         vm_page_busy_sleep(m, "physb", true);
  223                         VM_OBJECT_WLOCK(object);
  224                         goto retry;
  225                 } else {
  226                         vm_page_xbusy(m);
  227                         if (m->valid != VM_PAGE_BITS_ALL)
  228                                 vm_page_zero_invalid(m, TRUE);
  229                 }
  230 
  231                 KASSERT(m->valid == VM_PAGE_BITS_ALL,
  232                     ("phys_pager_populate: partially valid page %p", m));
  233                 KASSERT(m->dirty == 0,
  234                     ("phys_pager_populate: dirty page %p", m));
  235         }
  236         return (VM_PAGER_OK);
  237 }
  238 
  239 static void
  240 phys_pager_putpages(vm_object_t object, vm_page_t *m, int count, boolean_t sync,
  241     int *rtvals)
  242 {
  243 
  244         panic("phys_pager_putpage called");
  245 }
  246 
  247 static boolean_t
  248 phys_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,
  249     int *after)
  250 {
  251         vm_pindex_t base, end;
  252 
  253         base = rounddown(pindex, phys_pager_cluster);
  254         end = base + phys_pager_cluster - 1;
  255         if (before != NULL)
  256                 *before = pindex - base;
  257         if (after != NULL)
  258                 *after = end - pindex;
  259         return (TRUE);
  260 }
  261 
  262 struct pagerops physpagerops = {
  263         .pgo_init =     phys_pager_init,
  264         .pgo_alloc =    phys_pager_alloc,
  265         .pgo_dealloc =  phys_pager_dealloc,
  266         .pgo_getpages = phys_pager_getpages,
  267         .pgo_putpages = phys_pager_putpages,
  268         .pgo_haspage =  phys_pager_haspage,
  269         .pgo_populate = phys_pager_populate,
  270 };
Cache object: e4a570f6399ab1af4c6b31c73c492a96
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/vm/phys_pager.c

FreeBSD/Linux Kernel Cross Reference
sys/vm/phys_pager.c
