The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/vm/vm_kern.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  * Copyright (c) 1991, 1993
    3  *      The Regents of the University of California.  All rights reserved.
    4  *
    5  * This code is derived from software contributed to Berkeley by
    6  * The Mach Operating System project at Carnegie-Mellon University.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  * 3. All advertising materials mentioning features or use of this software
   17  *    must display the following acknowledgement:
   18  *      This product includes software developed by the University of
   19  *      California, Berkeley and its contributors.
   20  * 4. Neither the name of the University nor the names of its contributors
   21  *    may be used to endorse or promote products derived from this software
   22  *    without specific prior written permission.
   23  *
   24  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   25  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   27  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   28  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   30  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   32  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   33  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   34  * SUCH DAMAGE.
   35  *
   36  *      from: @(#)vm_kern.c     8.3 (Berkeley) 1/12/94
   37  *
   38  *
   39  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
   40  * All rights reserved.
   41  *
   42  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
   43  *
   44  * Permission to use, copy, modify and distribute this software and
   45  * its documentation is hereby granted, provided that both the copyright
   46  * notice and this permission notice appear in all copies of the
   47  * software, derivative works or modified versions, and any portions
   48  * thereof, and that both notices appear in supporting documentation.
   49  *
   50  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
   51  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
   52  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
   53  *
   54  * Carnegie Mellon requests users of this software to return to
   55  *
   56  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
   57  *  School of Computer Science
   58  *  Carnegie Mellon University
   59  *  Pittsburgh PA 15213-3890
   60  *
   61  * any improvements or extensions that they make and grant Carnegie the
   62  * rights to redistribute these changes.
   63  *
   64  * $FreeBSD: src/sys/vm/vm_kern.c,v 1.27.2.4 1999/09/05 08:24:26 peter Exp $
   65  */
   66 
   67 /*
   68  *      Kernel memory management.
   69  */
   70 
   71 #include <sys/param.h>
   72 #include <sys/systm.h>
   73 #include <sys/kernel.h>
   74 #include <sys/proc.h>
   75 #include <sys/malloc.h>
   76 #include <sys/syslog.h>
   77 #include <sys/queue.h>
   78 #include <sys/vmmeter.h>
   79 
   80 #include <vm/vm.h>
   81 #include <vm/vm_param.h>
   82 #include <vm/vm_prot.h>
   83 #include <vm/lock.h>
   84 #include <vm/pmap.h>
   85 #include <vm/vm_map.h>
   86 #include <vm/vm_object.h>
   87 #include <vm/vm_page.h>
   88 #include <vm/vm_pageout.h>
   89 #include <vm/vm_kern.h>
   90 #include <vm/vm_extern.h>
   91 
   92 vm_map_t kernel_map=0;
   93 vm_map_t kmem_map=0;
   94 vm_map_t exec_map=0;
   95 vm_map_t clean_map=0;
   96 vm_map_t u_map=0;
   97 vm_map_t buffer_map=0;
   98 vm_map_t mb_map=0;
   99 int mb_map_full=0;
  100 vm_map_t io_map=0;
  101 vm_map_t phys_map=0;
  102 
  103 /*
  104  *      kmem_alloc_pageable:
  105  *
  106  *      Allocate pageable memory to the kernel's address map.
  107  *      "map" must be kernel_map or a submap of kernel_map.
  108  */
  109 
  110 vm_offset_t
  111 kmem_alloc_pageable(map, size)
  112         vm_map_t map;
  113         register vm_size_t size;
  114 {
  115         vm_offset_t addr;
  116         register int result;
  117 
  118         size = round_page(size);
  119         addr = vm_map_min(map);
  120         result = vm_map_find(map, NULL, (vm_offset_t) 0,
  121             &addr, size, TRUE, VM_PROT_ALL, VM_PROT_ALL, 0);
  122         if (result != KERN_SUCCESS) {
  123                 return (0);
  124         }
  125         return (addr);
  126 }
  127 
  128 /*
  129  *      Allocate wired-down memory in the kernel's address map
  130  *      or a submap.
  131  */
  132 vm_offset_t
  133 kmem_alloc(map, size)
  134         register vm_map_t map;
  135         register vm_size_t size;
  136 {
  137         vm_offset_t addr;
  138         register vm_offset_t offset;
  139         vm_offset_t i;
  140 
  141         size = round_page(size);
  142 
  143         /*
  144          * Use the kernel object for wired-down kernel pages. Assume that no
  145          * region of the kernel object is referenced more than once.
  146          */
  147 
  148         /*
  149          * Locate sufficient space in the map.  This will give us the final
  150          * virtual address for the new memory, and thus will tell us the
  151          * offset within the kernel map.
  152          */
  153         vm_map_lock(map);
  154         if (vm_map_findspace(map, 0, size, &addr)) {
  155                 vm_map_unlock(map);
  156                 return (0);
  157         }
  158         offset = addr - VM_MIN_KERNEL_ADDRESS;
  159         vm_object_reference(kernel_object);
  160         vm_map_insert(map, kernel_object, offset, addr, addr + size,
  161                 VM_PROT_ALL, VM_PROT_ALL, 0);
  162         vm_map_unlock(map);
  163 
  164         /*
  165          * Guarantee that there are pages already in this object before
  166          * calling vm_map_pageable.  This is to prevent the following
  167          * scenario:
  168          *
  169          * 1) Threads have swapped out, so that there is a pager for the
  170          * kernel_object. 2) The kmsg zone is empty, and so we are
  171          * kmem_allocing a new page for it. 3) vm_map_pageable calls vm_fault;
  172          * there is no page, but there is a pager, so we call
  173          * pager_data_request.  But the kmsg zone is empty, so we must
  174          * kmem_alloc. 4) goto 1 5) Even if the kmsg zone is not empty: when
  175          * we get the data back from the pager, it will be (very stale)
  176          * non-zero data.  kmem_alloc is defined to return zero-filled memory.
  177          *
  178          * We're intentionally not activating the pages we allocate to prevent a
  179          * race with page-out.  vm_map_pageable will wire the pages.
  180          */
  181 
  182         for (i = 0; i < size; i += PAGE_SIZE) {
  183                 vm_page_t mem;
  184 
  185                 while ((mem = vm_page_alloc(kernel_object,
  186                         OFF_TO_IDX(offset + i), VM_ALLOC_ZERO)) == NULL) {
  187                         VM_WAIT;
  188                 }
  189                 if ((mem->flags & PG_ZERO) == 0)
  190                         vm_page_zero_fill(mem);
  191                 mem->flags &= ~(PG_BUSY|PG_ZERO);
  192                 mem->valid = VM_PAGE_BITS_ALL;
  193         }
  194 
  195         /*
  196          * And finally, mark the data as non-pageable.
  197          */
  198 
  199         (void) vm_map_pageable(map, (vm_offset_t) addr, addr + size, FALSE);
  200 
  201         return (addr);
  202 }
  203 
  204 /*
  205  *      kmem_free:
  206  *
  207  *      Release a region of kernel virtual memory allocated
  208  *      with kmem_alloc, and return the physical pages
  209  *      associated with that region.
  210  */
  211 void
  212 kmem_free(map, addr, size)
  213         vm_map_t map;
  214         register vm_offset_t addr;
  215         vm_size_t size;
  216 {
  217         (void) vm_map_remove(map, trunc_page(addr), round_page(addr + size));
  218 }
  219 
  220 /*
  221  *      kmem_suballoc:
  222  *
  223  *      Allocates a map to manage a subrange
  224  *      of the kernel virtual address space.
  225  *
  226  *      Arguments are as follows:
  227  *
  228  *      parent          Map to take range from
  229  *      size            Size of range to find
  230  *      min, max        Returned endpoints of map
  231  *      pageable        Can the region be paged
  232  */
  233 vm_map_t
  234 kmem_suballoc(parent, min, max, size, pageable)
  235         register vm_map_t parent;
  236         vm_offset_t *min, *max;
  237         register vm_size_t size;
  238         boolean_t pageable;
  239 {
  240         register int ret;
  241         vm_map_t result;
  242 
  243         size = round_page(size);
  244 
  245         *min = (vm_offset_t) vm_map_min(parent);
  246         ret = vm_map_find(parent, NULL, (vm_offset_t) 0,
  247             min, size, TRUE, VM_PROT_ALL, VM_PROT_ALL, 0);
  248         if (ret != KERN_SUCCESS) {
  249                 printf("kmem_suballoc: bad status return of %d.\n", ret);
  250                 panic("kmem_suballoc");
  251         }
  252         *max = *min + size;
  253         pmap_reference(vm_map_pmap(parent));
  254         result = vm_map_create(vm_map_pmap(parent), *min, *max, pageable);
  255         if (result == NULL)
  256                 panic("kmem_suballoc: cannot create submap");
  257         if ((ret = vm_map_submap(parent, *min, *max, result)) != KERN_SUCCESS)
  258                 panic("kmem_suballoc: unable to change range to submap");
  259         return (result);
  260 }
  261 
  262 /*
  263  * Allocate wired-down memory in the kernel's address map for the higher
  264  * level kernel memory allocator (kern/kern_malloc.c).  We cannot use
  265  * kmem_alloc() because we may need to allocate memory at interrupt
  266  * level where we cannot block (canwait == FALSE).
  267  *
  268  * This routine has its own private kernel submap (kmem_map) and object
  269  * (kmem_object).  This, combined with the fact that only malloc uses
  270  * this routine, ensures that we will never block in map or object waits.
  271  *
  272  * Note that this still only works in a uni-processor environment and
  273  * when called at splhigh().
  274  *
  275  * We don't worry about expanding the map (adding entries) since entries
  276  * for wired maps are statically allocated.
  277  */
  278 vm_offset_t
  279 kmem_malloc(map, size, waitflag)
  280         register vm_map_t map;
  281         register vm_size_t size;
  282         boolean_t waitflag;
  283 {
  284         register vm_offset_t offset, i;
  285         vm_map_entry_t entry;
  286         vm_offset_t addr;
  287         vm_page_t m;
  288 
  289         if (map != kmem_map && map != mb_map)
  290                 panic("kmem_malloc: map != {kmem,mb}_map");
  291 
  292         size = round_page(size);
  293         addr = vm_map_min(map);
  294 
  295         /*
  296          * Locate sufficient space in the map.  This will give us the final
  297          * virtual address for the new memory, and thus will tell us the
  298          * offset within the kernel map.
  299          */
  300         vm_map_lock(map);
  301         if (vm_map_findspace(map, 0, size, &addr)) {
  302                 vm_map_unlock(map);
  303                 if (map == mb_map) {
  304                         mb_map_full = TRUE;
  305                         log(LOG_ERR, "Out of mbuf clusters - increase maxusers!\n");
  306                         return (0);
  307                 }
  308                 if (waitflag == M_WAITOK)
  309                         panic("kmem_malloc: kmem_map too small");
  310                 return (0);
  311         }
  312         offset = addr - VM_MIN_KERNEL_ADDRESS;
  313         vm_object_reference(kmem_object);
  314         vm_map_insert(map, kmem_object, offset, addr, addr + size,
  315                 VM_PROT_ALL, VM_PROT_ALL, 0);
  316 
  317         for (i = 0; i < size; i += PAGE_SIZE) {
  318 retry:
  319                 m = vm_page_alloc(kmem_object, OFF_TO_IDX(offset + i),
  320                         (waitflag == M_NOWAIT) ? VM_ALLOC_INTERRUPT : VM_ALLOC_SYSTEM);
  321 
  322                 /*
  323                  * Ran out of space, free everything up and return. Don't need
  324                  * to lock page queues here as we know that the pages we got
  325                  * aren't on any queues.
  326                  */
  327                 if (m == NULL) {
  328                         if (waitflag == M_WAITOK) {
  329                                 VM_WAIT;
  330                                 goto retry;
  331                         }
  332                         while (i != 0) {
  333                                 i -= PAGE_SIZE;
  334                                 m = vm_page_lookup(kmem_object,
  335                                         OFF_TO_IDX(offset + i));
  336                                 PAGE_WAKEUP(m);
  337                                 vm_page_free(m);
  338                         }
  339                         vm_map_delete(map, addr, addr + size);
  340                         vm_map_unlock(map);
  341                         return (0);
  342                 }
  343                 m->flags &= ~PG_ZERO;
  344                 m->valid = VM_PAGE_BITS_ALL;
  345         }
  346 
  347         /*
  348          * Mark map entry as non-pageable. Assert: vm_map_insert() will never
  349          * be able to extend the previous entry so there will be a new entry
  350          * exactly corresponding to this address range and it will have
  351          * wired_count == 0.
  352          */
  353         if (!vm_map_lookup_entry(map, addr, &entry) ||
  354             entry->start != addr || entry->end != addr + size ||
  355             entry->wired_count)
  356                 panic("kmem_malloc: entry not found or misaligned");
  357         entry->wired_count++;
  358 
  359         vm_map_simplify_entry(map, entry);
  360 
  361         /*
  362          * Loop thru pages, entering them in the pmap. (We cannot add them to
  363          * the wired count without wrapping the vm_page_queue_lock in
  364          * splimp...)
  365          */
  366         for (i = 0; i < size; i += PAGE_SIZE) {
  367                 m = vm_page_lookup(kmem_object, OFF_TO_IDX(offset + i));
  368                 vm_page_wire(m);
  369                 PAGE_WAKEUP(m);
  370                 pmap_enter(kernel_pmap, addr + i, VM_PAGE_TO_PHYS(m),
  371                         VM_PROT_ALL, 1);
  372                 m->flags |= PG_MAPPED|PG_WRITEABLE;
  373         }
  374         vm_map_unlock(map);
  375 
  376         return (addr);
  377 }
  378 
  379 /*
  380  *      kmem_alloc_wait
  381  *
  382  *      Allocates pageable memory from a sub-map of the kernel.  If the submap
  383  *      has no room, the caller sleeps waiting for more memory in the submap.
  384  *
  385  */
  386 vm_offset_t
  387 kmem_alloc_wait(map, size)
  388         vm_map_t map;
  389         vm_size_t size;
  390 {
  391         vm_offset_t addr;
  392 
  393         size = round_page(size);
  394 
  395         for (;;) {
  396                 /*
  397                  * To make this work for more than one map, use the map's lock
  398                  * to lock out sleepers/wakers.
  399                  */
  400                 vm_map_lock(map);
  401                 if (vm_map_findspace(map, 0, size, &addr) == 0)
  402                         break;
  403                 /* no space now; see if we can ever get space */
  404                 if (vm_map_max(map) - vm_map_min(map) < size) {
  405                         vm_map_unlock(map);
  406                         return (0);
  407                 }
  408                 vm_map_unlock(map);
  409                 tsleep(map, PVM, "kmaw", 0);
  410         }
  411         vm_map_insert(map, NULL, (vm_offset_t) 0, addr, addr + size, VM_PROT_ALL, VM_PROT_ALL, 0);
  412         vm_map_unlock(map);
  413         return (addr);
  414 }
  415 
  416 /*
  417  *      kmem_free_wakeup
  418  *
  419  *      Returns memory to a submap of the kernel, and wakes up any processes
  420  *      waiting for memory in that map.
  421  */
  422 void
  423 kmem_free_wakeup(map, addr, size)
  424         vm_map_t map;
  425         vm_offset_t addr;
  426         vm_size_t size;
  427 {
  428         vm_map_lock(map);
  429         (void) vm_map_delete(map, trunc_page(addr), round_page(addr + size));
  430         wakeup(map);
  431         vm_map_unlock(map);
  432 }
  433 
  434 /*
  435  * Create the kernel map; insert a mapping covering kernel text, data, bss,
  436  * and all space allocated thus far (`boostrap' data).  The new map will thus
  437  * map the range between VM_MIN_KERNEL_ADDRESS and `start' as allocated, and
  438  * the range between `start' and `end' as free.
  439  */
  440 void
  441 kmem_init(start, end)
  442         vm_offset_t start, end;
  443 {
  444         register vm_map_t m;
  445 
  446         m = vm_map_create(kernel_pmap, VM_MIN_KERNEL_ADDRESS, end, FALSE);
  447         vm_map_lock(m);
  448         /* N.B.: cannot use kgdb to debug, starting with this assignment ... */
  449         kernel_map = m;
  450         (void) vm_map_insert(m, NULL, (vm_offset_t) 0,
  451             VM_MIN_KERNEL_ADDRESS, start, VM_PROT_ALL, VM_PROT_ALL, 0);
  452         /* ... and ending with the completion of the above `insert' */
  453         vm_map_unlock(m);
  454 }

Cache object: 4aae73c02d16ec919d0bc7a5c3fb39f3


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.