FreeBSD/Linux Kernel Cross Reference
sys/vm/vm_kern.c
1 /*
2 * Copyright (c) 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 *
5 * This code is derived from software contributed to Berkeley by
6 * The Mach Operating System project at Carnegie-Mellon University.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. All advertising materials mentioning features or use of this software
17 * must display the following acknowledgement:
18 * This product includes software developed by the University of
19 * California, Berkeley and its contributors.
20 * 4. Neither the name of the University nor the names of its contributors
21 * may be used to endorse or promote products derived from this software
22 * without specific prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 *
36 * from: @(#)vm_kern.c 8.3 (Berkeley) 1/12/94
37 *
38 *
39 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
40 * All rights reserved.
41 *
42 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
43 *
44 * Permission to use, copy, modify and distribute this software and
45 * its documentation is hereby granted, provided that both the copyright
46 * notice and this permission notice appear in all copies of the
47 * software, derivative works or modified versions, and any portions
48 * thereof, and that both notices appear in supporting documentation.
49 *
50 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
51 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
52 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
53 *
54 * Carnegie Mellon requests users of this software to return to
55 *
56 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
57 * School of Computer Science
58 * Carnegie Mellon University
59 * Pittsburgh PA 15213-3890
60 *
61 * any improvements or extensions that they make and grant Carnegie the
62 * rights to redistribute these changes.
63 *
64 * $FreeBSD: releng/5.1/sys/vm/vm_kern.c 113489 2003-04-15 01:16:05Z alc $
65 */
66
67 /*
68 * Kernel memory management.
69 */
70
71 #include <sys/param.h>
72 #include <sys/systm.h>
73 #include <sys/kernel.h> /* for ticks and hz */
74 #include <sys/lock.h>
75 #include <sys/mutex.h>
76 #include <sys/proc.h>
77 #include <sys/malloc.h>
78
79 #include <vm/vm.h>
80 #include <vm/vm_param.h>
81 #include <vm/pmap.h>
82 #include <vm/vm_map.h>
83 #include <vm/vm_object.h>
84 #include <vm/vm_page.h>
85 #include <vm/vm_pageout.h>
86 #include <vm/vm_extern.h>
87
88 vm_map_t kernel_map=0;
89 vm_map_t kmem_map=0;
90 vm_map_t exec_map=0;
91 vm_map_t clean_map=0;
92 vm_map_t buffer_map=0;
93
94 /*
95 * kmem_alloc_pageable:
96 *
97 * Allocate pageable memory to the kernel's address map.
98 * "map" must be kernel_map or a submap of kernel_map.
99 */
100 vm_offset_t
101 kmem_alloc_pageable(map, size)
102 vm_map_t map;
103 vm_size_t size;
104 {
105 vm_offset_t addr;
106 int result;
107
108 size = round_page(size);
109 addr = vm_map_min(map);
110 result = vm_map_find(map, NULL, 0,
111 &addr, size, TRUE, VM_PROT_ALL, VM_PROT_ALL, 0);
112 if (result != KERN_SUCCESS) {
113 return (0);
114 }
115 return (addr);
116 }
117
118 /*
119 * kmem_alloc_nofault:
120 *
121 * Same as kmem_alloc_pageable, except that it create a nofault entry.
122 */
123 vm_offset_t
124 kmem_alloc_nofault(map, size)
125 vm_map_t map;
126 vm_size_t size;
127 {
128 vm_offset_t addr;
129 int result;
130
131 size = round_page(size);
132 addr = vm_map_min(map);
133 result = vm_map_find(map, NULL, 0,
134 &addr, size, TRUE, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT);
135 if (result != KERN_SUCCESS) {
136 return (0);
137 }
138 return (addr);
139 }
140
141 /*
142 * Allocate wired-down memory in the kernel's address map
143 * or a submap.
144 */
145 vm_offset_t
146 kmem_alloc(map, size)
147 vm_map_t map;
148 vm_size_t size;
149 {
150 vm_offset_t addr;
151 vm_offset_t offset;
152 vm_offset_t i;
153
154 GIANT_REQUIRED;
155
156 size = round_page(size);
157
158 /*
159 * Use the kernel object for wired-down kernel pages. Assume that no
160 * region of the kernel object is referenced more than once.
161 */
162
163 /*
164 * Locate sufficient space in the map. This will give us the final
165 * virtual address for the new memory, and thus will tell us the
166 * offset within the kernel map.
167 */
168 vm_map_lock(map);
169 if (vm_map_findspace(map, vm_map_min(map), size, &addr)) {
170 vm_map_unlock(map);
171 return (0);
172 }
173 offset = addr - VM_MIN_KERNEL_ADDRESS;
174 vm_object_reference(kernel_object);
175 vm_map_insert(map, kernel_object, offset, addr, addr + size,
176 VM_PROT_ALL, VM_PROT_ALL, 0);
177 vm_map_unlock(map);
178
179 /*
180 * Guarantee that there are pages already in this object before
181 * calling vm_map_pageable. This is to prevent the following
182 * scenario:
183 *
184 * 1) Threads have swapped out, so that there is a pager for the
185 * kernel_object. 2) The kmsg zone is empty, and so we are
186 * kmem_allocing a new page for it. 3) vm_map_pageable calls vm_fault;
187 * there is no page, but there is a pager, so we call
188 * pager_data_request. But the kmsg zone is empty, so we must
189 * kmem_alloc. 4) goto 1 5) Even if the kmsg zone is not empty: when
190 * we get the data back from the pager, it will be (very stale)
191 * non-zero data. kmem_alloc is defined to return zero-filled memory.
192 *
193 * We're intentionally not activating the pages we allocate to prevent a
194 * race with page-out. vm_map_pageable will wire the pages.
195 */
196 for (i = 0; i < size; i += PAGE_SIZE) {
197 vm_page_t mem;
198
199 mem = vm_page_grab(kernel_object, OFF_TO_IDX(offset + i),
200 VM_ALLOC_ZERO | VM_ALLOC_RETRY);
201 if ((mem->flags & PG_ZERO) == 0)
202 pmap_zero_page(mem);
203 vm_page_lock_queues();
204 mem->valid = VM_PAGE_BITS_ALL;
205 vm_page_flag_clear(mem, PG_ZERO);
206 vm_page_wakeup(mem);
207 vm_page_unlock_queues();
208 }
209
210 /*
211 * And finally, mark the data as non-pageable.
212 */
213 (void) vm_map_wire(map, addr, addr + size, FALSE);
214
215 return (addr);
216 }
217
218 /*
219 * kmem_free:
220 *
221 * Release a region of kernel virtual memory allocated
222 * with kmem_alloc, and return the physical pages
223 * associated with that region.
224 *
225 * This routine may not block on kernel maps.
226 */
227 void
228 kmem_free(map, addr, size)
229 vm_map_t map;
230 vm_offset_t addr;
231 vm_size_t size;
232 {
233
234 (void) vm_map_remove(map, trunc_page(addr), round_page(addr + size));
235 }
236
237 /*
238 * kmem_suballoc:
239 *
240 * Allocates a map to manage a subrange
241 * of the kernel virtual address space.
242 *
243 * Arguments are as follows:
244 *
245 * parent Map to take range from
246 * min, max Returned endpoints of map
247 * size Size of range to find
248 */
249 vm_map_t
250 kmem_suballoc(parent, min, max, size)
251 vm_map_t parent;
252 vm_offset_t *min, *max;
253 vm_size_t size;
254 {
255 int ret;
256 vm_map_t result;
257
258 GIANT_REQUIRED;
259
260 size = round_page(size);
261
262 *min = (vm_offset_t) vm_map_min(parent);
263 ret = vm_map_find(parent, NULL, (vm_offset_t) 0,
264 min, size, TRUE, VM_PROT_ALL, VM_PROT_ALL, 0);
265 if (ret != KERN_SUCCESS) {
266 printf("kmem_suballoc: bad status return of %d.\n", ret);
267 panic("kmem_suballoc");
268 }
269 *max = *min + size;
270 result = vm_map_create(vm_map_pmap(parent), *min, *max);
271 if (result == NULL)
272 panic("kmem_suballoc: cannot create submap");
273 if (vm_map_submap(parent, *min, *max, result) != KERN_SUCCESS)
274 panic("kmem_suballoc: unable to change range to submap");
275 return (result);
276 }
277
278 /*
279 * kmem_malloc:
280 *
281 * Allocate wired-down memory in the kernel's address map for the higher
282 * level kernel memory allocator (kern/kern_malloc.c). We cannot use
283 * kmem_alloc() because we may need to allocate memory at interrupt
284 * level where we cannot block (canwait == FALSE).
285 *
286 * This routine has its own private kernel submap (kmem_map) and object
287 * (kmem_object). This, combined with the fact that only malloc uses
288 * this routine, ensures that we will never block in map or object waits.
289 *
290 * Note that this still only works in a uni-processor environment and
291 * when called at splhigh().
292 *
293 * We don't worry about expanding the map (adding entries) since entries
294 * for wired maps are statically allocated.
295 *
296 * NOTE: This routine is not supposed to block if M_NOWAIT is set, but
297 * I have not verified that it actually does not block.
298 *
299 * `map' is ONLY allowed to be kmem_map or one of the mbuf submaps to
300 * which we never free.
301 */
302 vm_offset_t
303 kmem_malloc(map, size, flags)
304 vm_map_t map;
305 vm_size_t size;
306 int flags;
307 {
308 vm_offset_t offset, i;
309 vm_map_entry_t entry;
310 vm_offset_t addr;
311 vm_page_t m;
312 int pflags;
313
314 if ((flags & M_NOWAIT) == 0)
315 GIANT_REQUIRED;
316
317 size = round_page(size);
318 addr = vm_map_min(map);
319
320 /*
321 * Locate sufficient space in the map. This will give us the final
322 * virtual address for the new memory, and thus will tell us the
323 * offset within the kernel map.
324 */
325 vm_map_lock(map);
326 if (vm_map_findspace(map, vm_map_min(map), size, &addr)) {
327 vm_map_unlock(map);
328 if (map != kmem_map) {
329 static int last_report; /* when we did it (in ticks) */
330 if (ticks < last_report ||
331 (ticks - last_report) >= hz) {
332 last_report = ticks;
333 printf("Out of mbuf address space!\n");
334 printf("Consider increasing NMBCLUSTERS\n");
335 }
336 return (0);
337 }
338 if ((flags & M_NOWAIT) == 0)
339 panic("kmem_malloc(%ld): kmem_map too small: %ld total allocated",
340 (long)size, (long)map->size);
341 return (0);
342 }
343 offset = addr - VM_MIN_KERNEL_ADDRESS;
344 vm_object_reference(kmem_object);
345 vm_map_insert(map, kmem_object, offset, addr, addr + size,
346 VM_PROT_ALL, VM_PROT_ALL, 0);
347
348 /*
349 * Note: if M_NOWAIT specified alone, allocate from
350 * interrupt-safe queues only (just the free list). If
351 * M_USE_RESERVE is also specified, we can also
352 * allocate from the cache. Neither of the latter two
353 * flags may be specified from an interrupt since interrupts
354 * are not allowed to mess with the cache queue.
355 */
356
357 if ((flags & (M_NOWAIT|M_USE_RESERVE)) == M_NOWAIT)
358 pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED;
359 else
360 pflags = VM_ALLOC_SYSTEM | VM_ALLOC_WIRED;
361
362 if (flags & M_ZERO)
363 pflags |= VM_ALLOC_ZERO;
364
365 VM_OBJECT_LOCK(kmem_object);
366 for (i = 0; i < size; i += PAGE_SIZE) {
367 retry:
368 m = vm_page_alloc(kmem_object, OFF_TO_IDX(offset + i), pflags);
369
370 /*
371 * Ran out of space, free everything up and return. Don't need
372 * to lock page queues here as we know that the pages we got
373 * aren't on any queues.
374 */
375 if (m == NULL) {
376 if ((flags & M_NOWAIT) == 0) {
377 VM_OBJECT_UNLOCK(kmem_object);
378 vm_map_unlock(map);
379 VM_WAIT;
380 vm_map_lock(map);
381 VM_OBJECT_LOCK(kmem_object);
382 goto retry;
383 }
384 /*
385 * Free the pages before removing the map entry.
386 * They are already marked busy. Calling
387 * vm_map_delete before the pages has been freed or
388 * unbusied will cause a deadlock.
389 */
390 while (i != 0) {
391 i -= PAGE_SIZE;
392 m = vm_page_lookup(kmem_object,
393 OFF_TO_IDX(offset + i));
394 vm_page_lock_queues();
395 vm_page_unwire(m, 0);
396 vm_page_free(m);
397 vm_page_unlock_queues();
398 }
399 VM_OBJECT_UNLOCK(kmem_object);
400 vm_map_delete(map, addr, addr + size);
401 vm_map_unlock(map);
402 return (0);
403 }
404 if (flags & M_ZERO && (m->flags & PG_ZERO) == 0)
405 pmap_zero_page(m);
406 vm_page_lock_queues();
407 vm_page_flag_clear(m, PG_ZERO);
408 m->valid = VM_PAGE_BITS_ALL;
409 vm_page_unlock_queues();
410 }
411 VM_OBJECT_UNLOCK(kmem_object);
412
413 /*
414 * Mark map entry as non-pageable. Assert: vm_map_insert() will never
415 * be able to extend the previous entry so there will be a new entry
416 * exactly corresponding to this address range and it will have
417 * wired_count == 0.
418 */
419 if (!vm_map_lookup_entry(map, addr, &entry) ||
420 entry->start != addr || entry->end != addr + size ||
421 entry->wired_count != 0)
422 panic("kmem_malloc: entry not found or misaligned");
423 entry->wired_count = 1;
424
425 vm_map_simplify_entry(map, entry);
426
427 /*
428 * Loop thru pages, entering them in the pmap. (We cannot add them to
429 * the wired count without wrapping the vm_page_queue_lock in
430 * splimp...)
431 */
432 for (i = 0; i < size; i += PAGE_SIZE) {
433 VM_OBJECT_LOCK(kmem_object);
434 m = vm_page_lookup(kmem_object, OFF_TO_IDX(offset + i));
435 VM_OBJECT_UNLOCK(kmem_object);
436 /*
437 * Because this is kernel_pmap, this call will not block.
438 */
439 pmap_enter(kernel_pmap, addr + i, m, VM_PROT_ALL, 1);
440 vm_page_lock_queues();
441 vm_page_flag_set(m, PG_WRITEABLE | PG_REFERENCED);
442 vm_page_wakeup(m);
443 vm_page_unlock_queues();
444 }
445 vm_map_unlock(map);
446
447 return (addr);
448 }
449
450 /*
451 * kmem_alloc_wait:
452 *
453 * Allocates pageable memory from a sub-map of the kernel. If the submap
454 * has no room, the caller sleeps waiting for more memory in the submap.
455 *
456 * This routine may block.
457 */
458 vm_offset_t
459 kmem_alloc_wait(map, size)
460 vm_map_t map;
461 vm_size_t size;
462 {
463 vm_offset_t addr;
464
465 size = round_page(size);
466
467 for (;;) {
468 /*
469 * To make this work for more than one map, use the map's lock
470 * to lock out sleepers/wakers.
471 */
472 vm_map_lock(map);
473 if (vm_map_findspace(map, vm_map_min(map), size, &addr) == 0)
474 break;
475 /* no space now; see if we can ever get space */
476 if (vm_map_max(map) - vm_map_min(map) < size) {
477 vm_map_unlock(map);
478 return (0);
479 }
480 map->needs_wakeup = TRUE;
481 vm_map_unlock_and_wait(map, FALSE);
482 }
483 vm_map_insert(map, NULL, 0, addr, addr + size, VM_PROT_ALL, VM_PROT_ALL, 0);
484 vm_map_unlock(map);
485 return (addr);
486 }
487
488 /*
489 * kmem_free_wakeup:
490 *
491 * Returns memory to a submap of the kernel, and wakes up any processes
492 * waiting for memory in that map.
493 */
494 void
495 kmem_free_wakeup(map, addr, size)
496 vm_map_t map;
497 vm_offset_t addr;
498 vm_size_t size;
499 {
500
501 vm_map_lock(map);
502 (void) vm_map_delete(map, trunc_page(addr), round_page(addr + size));
503 if (map->needs_wakeup) {
504 map->needs_wakeup = FALSE;
505 vm_map_wakeup(map);
506 }
507 vm_map_unlock(map);
508 }
509
510 /*
511 * kmem_init:
512 *
513 * Create the kernel map; insert a mapping covering kernel text,
514 * data, bss, and all space allocated thus far (`boostrap' data). The
515 * new map will thus map the range between VM_MIN_KERNEL_ADDRESS and
516 * `start' as allocated, and the range between `start' and `end' as free.
517 */
518 void
519 kmem_init(start, end)
520 vm_offset_t start, end;
521 {
522 vm_map_t m;
523
524 m = vm_map_create(kernel_pmap, VM_MIN_KERNEL_ADDRESS, end);
525 m->system_map = 1;
526 vm_map_lock(m);
527 /* N.B.: cannot use kgdb to debug, starting with this assignment ... */
528 kernel_map = m;
529 (void) vm_map_insert(m, NULL, (vm_ooffset_t) 0,
530 VM_MIN_KERNEL_ADDRESS, start, VM_PROT_ALL, VM_PROT_ALL, 0);
531 /* ... and ending with the completion of the above `insert' */
532 vm_map_unlock(m);
533 }
Cache object: 533cbed50e59ff6283ba700df4fc5a5b
|