FreeBSD/Linux Kernel Cross Reference
sys/vm/vm_map.c
1 /*-
2 * Copyright (c) 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 *
5 * This code is derived from software contributed to Berkeley by
6 * The Mach Operating System project at Carnegie-Mellon University.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 4. Neither the name of the University nor the names of its contributors
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 *
32 * from: @(#)vm_map.c 8.3 (Berkeley) 1/12/94
33 *
34 *
35 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
36 * All rights reserved.
37 *
38 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
39 *
40 * Permission to use, copy, modify and distribute this software and
41 * its documentation is hereby granted, provided that both the copyright
42 * notice and this permission notice appear in all copies of the
43 * software, derivative works or modified versions, and any portions
44 * thereof, and that both notices appear in supporting documentation.
45 *
46 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
47 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
48 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
49 *
50 * Carnegie Mellon requests users of this software to return to
51 *
52 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
53 * School of Computer Science
54 * Carnegie Mellon University
55 * Pittsburgh PA 15213-3890
56 *
57 * any improvements or extensions that they make and grant Carnegie the
58 * rights to redistribute these changes.
59 */
60
61 /*
62 * Virtual memory mapping module.
63 */
64
65 #include <sys/cdefs.h>
66 __FBSDID("$FreeBSD: releng/9.1/sys/vm/vm_map.c 267018 2014-06-03 19:03:11Z delphij $");
67
68 #include <sys/param.h>
69 #include <sys/systm.h>
70 #include <sys/kernel.h>
71 #include <sys/ktr.h>
72 #include <sys/lock.h>
73 #include <sys/mutex.h>
74 #include <sys/proc.h>
75 #include <sys/vmmeter.h>
76 #include <sys/mman.h>
77 #include <sys/vnode.h>
78 #include <sys/racct.h>
79 #include <sys/resourcevar.h>
80 #include <sys/file.h>
81 #include <sys/sysctl.h>
82 #include <sys/sysent.h>
83 #include <sys/shm.h>
84
85 #include <vm/vm.h>
86 #include <vm/vm_param.h>
87 #include <vm/pmap.h>
88 #include <vm/vm_map.h>
89 #include <vm/vm_page.h>
90 #include <vm/vm_object.h>
91 #include <vm/vm_pager.h>
92 #include <vm/vm_kern.h>
93 #include <vm/vm_extern.h>
94 #include <vm/vnode_pager.h>
95 #include <vm/swap_pager.h>
96 #include <vm/uma.h>
97
98 /*
99 * Virtual memory maps provide for the mapping, protection,
100 * and sharing of virtual memory objects. In addition,
101 * this module provides for an efficient virtual copy of
102 * memory from one map to another.
103 *
104 * Synchronization is required prior to most operations.
105 *
106 * Maps consist of an ordered doubly-linked list of simple
107 * entries; a self-adjusting binary search tree of these
108 * entries is used to speed up lookups.
109 *
110 * Since portions of maps are specified by start/end addresses,
111 * which may not align with existing map entries, all
112 * routines merely "clip" entries to these start/end values.
113 * [That is, an entry is split into two, bordering at a
114 * start or end value.] Note that these clippings may not
115 * always be necessary (as the two resulting entries are then
116 * not changed); however, the clipping is done for convenience.
117 *
118 * As mentioned above, virtual copy operations are performed
119 * by copying VM object references from one map to
120 * another, and then marking both regions as copy-on-write.
121 */
122
123 static struct mtx map_sleep_mtx;
124 static uma_zone_t mapentzone;
125 static uma_zone_t kmapentzone;
126 static uma_zone_t mapzone;
127 static uma_zone_t vmspace_zone;
128 static struct vm_object kmapentobj;
129 static int vmspace_zinit(void *mem, int size, int flags);
130 static void vmspace_zfini(void *mem, int size);
131 static int vm_map_zinit(void *mem, int ize, int flags);
132 static void vm_map_zfini(void *mem, int size);
133 static void _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min,
134 vm_offset_t max);
135 static void vm_map_entry_deallocate(vm_map_entry_t entry, boolean_t system_map);
136 static void vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry);
137 #ifdef INVARIANTS
138 static void vm_map_zdtor(void *mem, int size, void *arg);
139 static void vmspace_zdtor(void *mem, int size, void *arg);
140 #endif
141
142 #define ENTRY_CHARGED(e) ((e)->cred != NULL || \
143 ((e)->object.vm_object != NULL && (e)->object.vm_object->cred != NULL && \
144 !((e)->eflags & MAP_ENTRY_NEEDS_COPY)))
145
146 /*
147 * PROC_VMSPACE_{UN,}LOCK() can be a noop as long as vmspaces are type
148 * stable.
149 */
150 #define PROC_VMSPACE_LOCK(p) do { } while (0)
151 #define PROC_VMSPACE_UNLOCK(p) do { } while (0)
152
153 /*
154 * VM_MAP_RANGE_CHECK: [ internal use only ]
155 *
156 * Asserts that the starting and ending region
157 * addresses fall within the valid range of the map.
158 */
159 #define VM_MAP_RANGE_CHECK(map, start, end) \
160 { \
161 if (start < vm_map_min(map)) \
162 start = vm_map_min(map); \
163 if (end > vm_map_max(map)) \
164 end = vm_map_max(map); \
165 if (start > end) \
166 start = end; \
167 }
168
169 /*
170 * vm_map_startup:
171 *
172 * Initialize the vm_map module. Must be called before
173 * any other vm_map routines.
174 *
175 * Map and entry structures are allocated from the general
176 * purpose memory pool with some exceptions:
177 *
178 * - The kernel map and kmem submap are allocated statically.
179 * - Kernel map entries are allocated out of a static pool.
180 *
181 * These restrictions are necessary since malloc() uses the
182 * maps and requires map entries.
183 */
184
185 void
186 vm_map_startup(void)
187 {
188 mtx_init(&map_sleep_mtx, "vm map sleep mutex", NULL, MTX_DEF);
189 mapzone = uma_zcreate("MAP", sizeof(struct vm_map), NULL,
190 #ifdef INVARIANTS
191 vm_map_zdtor,
192 #else
193 NULL,
194 #endif
195 vm_map_zinit, vm_map_zfini, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
196 uma_prealloc(mapzone, MAX_KMAP);
197 kmapentzone = uma_zcreate("KMAP ENTRY", sizeof(struct vm_map_entry),
198 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
199 UMA_ZONE_MTXCLASS | UMA_ZONE_VM);
200 uma_prealloc(kmapentzone, MAX_KMAPENT);
201 mapentzone = uma_zcreate("MAP ENTRY", sizeof(struct vm_map_entry),
202 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
203 }
204
205 static void
206 vmspace_zfini(void *mem, int size)
207 {
208 struct vmspace *vm;
209
210 vm = (struct vmspace *)mem;
211 vm_map_zfini(&vm->vm_map, sizeof(vm->vm_map));
212 }
213
214 static int
215 vmspace_zinit(void *mem, int size, int flags)
216 {
217 struct vmspace *vm;
218
219 vm = (struct vmspace *)mem;
220
221 vm->vm_map.pmap = NULL;
222 (void)vm_map_zinit(&vm->vm_map, sizeof(vm->vm_map), flags);
223 return (0);
224 }
225
226 static void
227 vm_map_zfini(void *mem, int size)
228 {
229 vm_map_t map;
230
231 map = (vm_map_t)mem;
232 mtx_destroy(&map->system_mtx);
233 sx_destroy(&map->lock);
234 }
235
236 static int
237 vm_map_zinit(void *mem, int size, int flags)
238 {
239 vm_map_t map;
240
241 map = (vm_map_t)mem;
242 map->nentries = 0;
243 map->size = 0;
244 mtx_init(&map->system_mtx, "system map", NULL, MTX_DEF | MTX_DUPOK);
245 sx_init(&map->lock, "user map");
246 return (0);
247 }
248
249 #ifdef INVARIANTS
250 static void
251 vmspace_zdtor(void *mem, int size, void *arg)
252 {
253 struct vmspace *vm;
254
255 vm = (struct vmspace *)mem;
256
257 vm_map_zdtor(&vm->vm_map, sizeof(vm->vm_map), arg);
258 }
259 static void
260 vm_map_zdtor(void *mem, int size, void *arg)
261 {
262 vm_map_t map;
263
264 map = (vm_map_t)mem;
265 KASSERT(map->nentries == 0,
266 ("map %p nentries == %d on free.",
267 map, map->nentries));
268 KASSERT(map->size == 0,
269 ("map %p size == %lu on free.",
270 map, (unsigned long)map->size));
271 }
272 #endif /* INVARIANTS */
273
274 /*
275 * Allocate a vmspace structure, including a vm_map and pmap,
276 * and initialize those structures. The refcnt is set to 1.
277 */
278 struct vmspace *
279 vmspace_alloc(min, max)
280 vm_offset_t min, max;
281 {
282 struct vmspace *vm;
283
284 vm = uma_zalloc(vmspace_zone, M_WAITOK);
285 if (vm->vm_map.pmap == NULL && !pmap_pinit(vmspace_pmap(vm))) {
286 uma_zfree(vmspace_zone, vm);
287 return (NULL);
288 }
289 CTR1(KTR_VM, "vmspace_alloc: %p", vm);
290 _vm_map_init(&vm->vm_map, vmspace_pmap(vm), min, max);
291 vm->vm_refcnt = 1;
292 vm->vm_shm = NULL;
293 vm->vm_swrss = 0;
294 vm->vm_tsize = 0;
295 vm->vm_dsize = 0;
296 vm->vm_ssize = 0;
297 vm->vm_taddr = 0;
298 vm->vm_daddr = 0;
299 vm->vm_maxsaddr = 0;
300 return (vm);
301 }
302
303 void
304 vm_init2(void)
305 {
306 uma_zone_set_obj(kmapentzone, &kmapentobj, lmin(cnt.v_page_count,
307 (VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / PAGE_SIZE) / 8 +
308 maxproc * 2 + maxfiles);
309 vmspace_zone = uma_zcreate("VMSPACE", sizeof(struct vmspace), NULL,
310 #ifdef INVARIANTS
311 vmspace_zdtor,
312 #else
313 NULL,
314 #endif
315 vmspace_zinit, vmspace_zfini, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
316 }
317
318 static void
319 vmspace_container_reset(struct proc *p)
320 {
321
322 #ifdef RACCT
323 PROC_LOCK(p);
324 racct_set(p, RACCT_DATA, 0);
325 racct_set(p, RACCT_STACK, 0);
326 racct_set(p, RACCT_RSS, 0);
327 racct_set(p, RACCT_MEMLOCK, 0);
328 racct_set(p, RACCT_VMEM, 0);
329 PROC_UNLOCK(p);
330 #endif
331 }
332
333 static inline void
334 vmspace_dofree(struct vmspace *vm)
335 {
336
337 CTR1(KTR_VM, "vmspace_free: %p", vm);
338
339 /*
340 * Make sure any SysV shm is freed, it might not have been in
341 * exit1().
342 */
343 shmexit(vm);
344
345 /*
346 * Lock the map, to wait out all other references to it.
347 * Delete all of the mappings and pages they hold, then call
348 * the pmap module to reclaim anything left.
349 */
350 (void)vm_map_remove(&vm->vm_map, vm->vm_map.min_offset,
351 vm->vm_map.max_offset);
352
353 pmap_release(vmspace_pmap(vm));
354 vm->vm_map.pmap = NULL;
355 uma_zfree(vmspace_zone, vm);
356 }
357
358 void
359 vmspace_free(struct vmspace *vm)
360 {
361
362 if (vm->vm_refcnt == 0)
363 panic("vmspace_free: attempt to free already freed vmspace");
364
365 if (atomic_fetchadd_int(&vm->vm_refcnt, -1) == 1)
366 vmspace_dofree(vm);
367 }
368
369 void
370 vmspace_exitfree(struct proc *p)
371 {
372 struct vmspace *vm;
373
374 PROC_VMSPACE_LOCK(p);
375 vm = p->p_vmspace;
376 p->p_vmspace = NULL;
377 PROC_VMSPACE_UNLOCK(p);
378 KASSERT(vm == &vmspace0, ("vmspace_exitfree: wrong vmspace"));
379 vmspace_free(vm);
380 }
381
382 void
383 vmspace_exit(struct thread *td)
384 {
385 int refcnt;
386 struct vmspace *vm;
387 struct proc *p;
388
389 /*
390 * Release user portion of address space.
391 * This releases references to vnodes,
392 * which could cause I/O if the file has been unlinked.
393 * Need to do this early enough that we can still sleep.
394 *
395 * The last exiting process to reach this point releases as
396 * much of the environment as it can. vmspace_dofree() is the
397 * slower fallback in case another process had a temporary
398 * reference to the vmspace.
399 */
400
401 p = td->td_proc;
402 vm = p->p_vmspace;
403 atomic_add_int(&vmspace0.vm_refcnt, 1);
404 do {
405 refcnt = vm->vm_refcnt;
406 if (refcnt > 1 && p->p_vmspace != &vmspace0) {
407 /* Switch now since other proc might free vmspace */
408 PROC_VMSPACE_LOCK(p);
409 p->p_vmspace = &vmspace0;
410 PROC_VMSPACE_UNLOCK(p);
411 pmap_activate(td);
412 }
413 } while (!atomic_cmpset_int(&vm->vm_refcnt, refcnt, refcnt - 1));
414 if (refcnt == 1) {
415 if (p->p_vmspace != vm) {
416 /* vmspace not yet freed, switch back */
417 PROC_VMSPACE_LOCK(p);
418 p->p_vmspace = vm;
419 PROC_VMSPACE_UNLOCK(p);
420 pmap_activate(td);
421 }
422 pmap_remove_pages(vmspace_pmap(vm));
423 /* Switch now since this proc will free vmspace */
424 PROC_VMSPACE_LOCK(p);
425 p->p_vmspace = &vmspace0;
426 PROC_VMSPACE_UNLOCK(p);
427 pmap_activate(td);
428 vmspace_dofree(vm);
429 }
430 vmspace_container_reset(p);
431 }
432
433 /* Acquire reference to vmspace owned by another process. */
434
435 struct vmspace *
436 vmspace_acquire_ref(struct proc *p)
437 {
438 struct vmspace *vm;
439 int refcnt;
440
441 PROC_VMSPACE_LOCK(p);
442 vm = p->p_vmspace;
443 if (vm == NULL) {
444 PROC_VMSPACE_UNLOCK(p);
445 return (NULL);
446 }
447 do {
448 refcnt = vm->vm_refcnt;
449 if (refcnt <= 0) { /* Avoid 0->1 transition */
450 PROC_VMSPACE_UNLOCK(p);
451 return (NULL);
452 }
453 } while (!atomic_cmpset_int(&vm->vm_refcnt, refcnt, refcnt + 1));
454 if (vm != p->p_vmspace) {
455 PROC_VMSPACE_UNLOCK(p);
456 vmspace_free(vm);
457 return (NULL);
458 }
459 PROC_VMSPACE_UNLOCK(p);
460 return (vm);
461 }
462
463 void
464 _vm_map_lock(vm_map_t map, const char *file, int line)
465 {
466
467 if (map->system_map)
468 mtx_lock_flags_(&map->system_mtx, 0, file, line);
469 else
470 sx_xlock_(&map->lock, file, line);
471 map->timestamp++;
472 }
473
474 static void
475 vm_map_process_deferred(void)
476 {
477 struct thread *td;
478 vm_map_entry_t entry, next;
479 vm_object_t object;
480
481 td = curthread;
482 entry = td->td_map_def_user;
483 td->td_map_def_user = NULL;
484 while (entry != NULL) {
485 next = entry->next;
486 if ((entry->eflags & MAP_ENTRY_VN_WRITECNT) != 0) {
487 /*
488 * Decrement the object's writemappings and
489 * possibly the vnode's v_writecount.
490 */
491 KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0,
492 ("Submap with writecount"));
493 object = entry->object.vm_object;
494 KASSERT(object != NULL, ("No object for writecount"));
495 vnode_pager_release_writecount(object, entry->start,
496 entry->end);
497 }
498 vm_map_entry_deallocate(entry, FALSE);
499 entry = next;
500 }
501 }
502
503 void
504 _vm_map_unlock(vm_map_t map, const char *file, int line)
505 {
506
507 if (map->system_map)
508 mtx_unlock_flags_(&map->system_mtx, 0, file, line);
509 else {
510 sx_xunlock_(&map->lock, file, line);
511 vm_map_process_deferred();
512 }
513 }
514
515 void
516 _vm_map_lock_read(vm_map_t map, const char *file, int line)
517 {
518
519 if (map->system_map)
520 mtx_lock_flags_(&map->system_mtx, 0, file, line);
521 else
522 sx_slock_(&map->lock, file, line);
523 }
524
525 void
526 _vm_map_unlock_read(vm_map_t map, const char *file, int line)
527 {
528
529 if (map->system_map)
530 mtx_unlock_flags_(&map->system_mtx, 0, file, line);
531 else {
532 sx_sunlock_(&map->lock, file, line);
533 vm_map_process_deferred();
534 }
535 }
536
537 int
538 _vm_map_trylock(vm_map_t map, const char *file, int line)
539 {
540 int error;
541
542 error = map->system_map ?
543 !mtx_trylock_flags_(&map->system_mtx, 0, file, line) :
544 !sx_try_xlock_(&map->lock, file, line);
545 if (error == 0)
546 map->timestamp++;
547 return (error == 0);
548 }
549
550 int
551 _vm_map_trylock_read(vm_map_t map, const char *file, int line)
552 {
553 int error;
554
555 error = map->system_map ?
556 !mtx_trylock_flags_(&map->system_mtx, 0, file, line) :
557 !sx_try_slock_(&map->lock, file, line);
558 return (error == 0);
559 }
560
561 /*
562 * _vm_map_lock_upgrade: [ internal use only ]
563 *
564 * Tries to upgrade a read (shared) lock on the specified map to a write
565 * (exclusive) lock. Returns the value "" if the upgrade succeeds and a
566 * non-zero value if the upgrade fails. If the upgrade fails, the map is
567 * returned without a read or write lock held.
568 *
569 * Requires that the map be read locked.
570 */
571 int
572 _vm_map_lock_upgrade(vm_map_t map, const char *file, int line)
573 {
574 unsigned int last_timestamp;
575
576 if (map->system_map) {
577 mtx_assert_(&map->system_mtx, MA_OWNED, file, line);
578 } else {
579 if (!sx_try_upgrade_(&map->lock, file, line)) {
580 last_timestamp = map->timestamp;
581 sx_sunlock_(&map->lock, file, line);
582 vm_map_process_deferred();
583 /*
584 * If the map's timestamp does not change while the
585 * map is unlocked, then the upgrade succeeds.
586 */
587 sx_xlock_(&map->lock, file, line);
588 if (last_timestamp != map->timestamp) {
589 sx_xunlock_(&map->lock, file, line);
590 return (1);
591 }
592 }
593 }
594 map->timestamp++;
595 return (0);
596 }
597
598 void
599 _vm_map_lock_downgrade(vm_map_t map, const char *file, int line)
600 {
601
602 if (map->system_map) {
603 mtx_assert_(&map->system_mtx, MA_OWNED, file, line);
604 } else
605 sx_downgrade_(&map->lock, file, line);
606 }
607
608 /*
609 * vm_map_locked:
610 *
611 * Returns a non-zero value if the caller holds a write (exclusive) lock
612 * on the specified map and the value "" otherwise.
613 */
614 int
615 vm_map_locked(vm_map_t map)
616 {
617
618 if (map->system_map)
619 return (mtx_owned(&map->system_mtx));
620 else
621 return (sx_xlocked(&map->lock));
622 }
623
624 #ifdef INVARIANTS
625 static void
626 _vm_map_assert_locked(vm_map_t map, const char *file, int line)
627 {
628
629 if (map->system_map)
630 mtx_assert_(&map->system_mtx, MA_OWNED, file, line);
631 else
632 sx_assert_(&map->lock, SA_XLOCKED, file, line);
633 }
634
635 #define VM_MAP_ASSERT_LOCKED(map) \
636 _vm_map_assert_locked(map, LOCK_FILE, LOCK_LINE)
637 #else
638 #define VM_MAP_ASSERT_LOCKED(map)
639 #endif
640
641 /*
642 * _vm_map_unlock_and_wait:
643 *
644 * Atomically releases the lock on the specified map and puts the calling
645 * thread to sleep. The calling thread will remain asleep until either
646 * vm_map_wakeup() is performed on the map or the specified timeout is
647 * exceeded.
648 *
649 * WARNING! This function does not perform deferred deallocations of
650 * objects and map entries. Therefore, the calling thread is expected to
651 * reacquire the map lock after reawakening and later perform an ordinary
652 * unlock operation, such as vm_map_unlock(), before completing its
653 * operation on the map.
654 */
655 int
656 _vm_map_unlock_and_wait(vm_map_t map, int timo, const char *file, int line)
657 {
658
659 mtx_lock(&map_sleep_mtx);
660 if (map->system_map)
661 mtx_unlock_flags_(&map->system_mtx, 0, file, line);
662 else
663 sx_xunlock_(&map->lock, file, line);
664 return (msleep(&map->root, &map_sleep_mtx, PDROP | PVM, "vmmaps",
665 timo));
666 }
667
668 /*
669 * vm_map_wakeup:
670 *
671 * Awaken any threads that have slept on the map using
672 * vm_map_unlock_and_wait().
673 */
674 void
675 vm_map_wakeup(vm_map_t map)
676 {
677
678 /*
679 * Acquire and release map_sleep_mtx to prevent a wakeup()
680 * from being performed (and lost) between the map unlock
681 * and the msleep() in _vm_map_unlock_and_wait().
682 */
683 mtx_lock(&map_sleep_mtx);
684 mtx_unlock(&map_sleep_mtx);
685 wakeup(&map->root);
686 }
687
688 void
689 vm_map_busy(vm_map_t map)
690 {
691
692 VM_MAP_ASSERT_LOCKED(map);
693 map->busy++;
694 }
695
696 void
697 vm_map_unbusy(vm_map_t map)
698 {
699
700 VM_MAP_ASSERT_LOCKED(map);
701 KASSERT(map->busy, ("vm_map_unbusy: not busy"));
702 if (--map->busy == 0 && (map->flags & MAP_BUSY_WAKEUP)) {
703 vm_map_modflags(map, 0, MAP_BUSY_WAKEUP);
704 wakeup(&map->busy);
705 }
706 }
707
708 void
709 vm_map_wait_busy(vm_map_t map)
710 {
711
712 VM_MAP_ASSERT_LOCKED(map);
713 while (map->busy) {
714 vm_map_modflags(map, MAP_BUSY_WAKEUP, 0);
715 if (map->system_map)
716 msleep(&map->busy, &map->system_mtx, 0, "mbusy", 0);
717 else
718 sx_sleep(&map->busy, &map->lock, 0, "mbusy", 0);
719 }
720 map->timestamp++;
721 }
722
723 long
724 vmspace_resident_count(struct vmspace *vmspace)
725 {
726 return pmap_resident_count(vmspace_pmap(vmspace));
727 }
728
729 long
730 vmspace_wired_count(struct vmspace *vmspace)
731 {
732 return pmap_wired_count(vmspace_pmap(vmspace));
733 }
734
735 /*
736 * vm_map_create:
737 *
738 * Creates and returns a new empty VM map with
739 * the given physical map structure, and having
740 * the given lower and upper address bounds.
741 */
742 vm_map_t
743 vm_map_create(pmap_t pmap, vm_offset_t min, vm_offset_t max)
744 {
745 vm_map_t result;
746
747 result = uma_zalloc(mapzone, M_WAITOK);
748 CTR1(KTR_VM, "vm_map_create: %p", result);
749 _vm_map_init(result, pmap, min, max);
750 return (result);
751 }
752
753 /*
754 * Initialize an existing vm_map structure
755 * such as that in the vmspace structure.
756 */
757 static void
758 _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max)
759 {
760
761 map->header.next = map->header.prev = &map->header;
762 map->needs_wakeup = FALSE;
763 map->system_map = 0;
764 map->pmap = pmap;
765 map->min_offset = min;
766 map->max_offset = max;
767 map->flags = 0;
768 map->root = NULL;
769 map->timestamp = 0;
770 map->busy = 0;
771 }
772
773 void
774 vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max)
775 {
776
777 _vm_map_init(map, pmap, min, max);
778 mtx_init(&map->system_mtx, "system map", NULL, MTX_DEF | MTX_DUPOK);
779 sx_init(&map->lock, "user map");
780 }
781
782 /*
783 * vm_map_entry_dispose: [ internal use only ]
784 *
785 * Inverse of vm_map_entry_create.
786 */
787 static void
788 vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry)
789 {
790 uma_zfree(map->system_map ? kmapentzone : mapentzone, entry);
791 }
792
793 /*
794 * vm_map_entry_create: [ internal use only ]
795 *
796 * Allocates a VM map entry for insertion.
797 * No entry fields are filled in.
798 */
799 static vm_map_entry_t
800 vm_map_entry_create(vm_map_t map)
801 {
802 vm_map_entry_t new_entry;
803
804 if (map->system_map)
805 new_entry = uma_zalloc(kmapentzone, M_NOWAIT);
806 else
807 new_entry = uma_zalloc(mapentzone, M_WAITOK);
808 if (new_entry == NULL)
809 panic("vm_map_entry_create: kernel resources exhausted");
810 return (new_entry);
811 }
812
813 /*
814 * vm_map_entry_set_behavior:
815 *
816 * Set the expected access behavior, either normal, random, or
817 * sequential.
818 */
819 static inline void
820 vm_map_entry_set_behavior(vm_map_entry_t entry, u_char behavior)
821 {
822 entry->eflags = (entry->eflags & ~MAP_ENTRY_BEHAV_MASK) |
823 (behavior & MAP_ENTRY_BEHAV_MASK);
824 }
825
826 /*
827 * vm_map_entry_set_max_free:
828 *
829 * Set the max_free field in a vm_map_entry.
830 */
831 static inline void
832 vm_map_entry_set_max_free(vm_map_entry_t entry)
833 {
834
835 entry->max_free = entry->adj_free;
836 if (entry->left != NULL && entry->left->max_free > entry->max_free)
837 entry->max_free = entry->left->max_free;
838 if (entry->right != NULL && entry->right->max_free > entry->max_free)
839 entry->max_free = entry->right->max_free;
840 }
841
842 /*
843 * vm_map_entry_splay:
844 *
845 * The Sleator and Tarjan top-down splay algorithm with the
846 * following variation. Max_free must be computed bottom-up, so
847 * on the downward pass, maintain the left and right spines in
848 * reverse order. Then, make a second pass up each side to fix
849 * the pointers and compute max_free. The time bound is O(log n)
850 * amortized.
851 *
852 * The new root is the vm_map_entry containing "addr", or else an
853 * adjacent entry (lower or higher) if addr is not in the tree.
854 *
855 * The map must be locked, and leaves it so.
856 *
857 * Returns: the new root.
858 */
859 static vm_map_entry_t
860 vm_map_entry_splay(vm_offset_t addr, vm_map_entry_t root)
861 {
862 vm_map_entry_t llist, rlist;
863 vm_map_entry_t ltree, rtree;
864 vm_map_entry_t y;
865
866 /* Special case of empty tree. */
867 if (root == NULL)
868 return (root);
869
870 /*
871 * Pass One: Splay down the tree until we find addr or a NULL
872 * pointer where addr would go. llist and rlist are the two
873 * sides in reverse order (bottom-up), with llist linked by
874 * the right pointer and rlist linked by the left pointer in
875 * the vm_map_entry. Wait until Pass Two to set max_free on
876 * the two spines.
877 */
878 llist = NULL;
879 rlist = NULL;
880 for (;;) {
881 /* root is never NULL in here. */
882 if (addr < root->start) {
883 y = root->left;
884 if (y == NULL)
885 break;
886 if (addr < y->start && y->left != NULL) {
887 /* Rotate right and put y on rlist. */
888 root->left = y->right;
889 y->right = root;
890 vm_map_entry_set_max_free(root);
891 root = y->left;
892 y->left = rlist;
893 rlist = y;
894 } else {
895 /* Put root on rlist. */
896 root->left = rlist;
897 rlist = root;
898 root = y;
899 }
900 } else if (addr >= root->end) {
901 y = root->right;
902 if (y == NULL)
903 break;
904 if (addr >= y->end && y->right != NULL) {
905 /* Rotate left and put y on llist. */
906 root->right = y->left;
907 y->left = root;
908 vm_map_entry_set_max_free(root);
909 root = y->right;
910 y->right = llist;
911 llist = y;
912 } else {
913 /* Put root on llist. */
914 root->right = llist;
915 llist = root;
916 root = y;
917 }
918 } else
919 break;
920 }
921
922 /*
923 * Pass Two: Walk back up the two spines, flip the pointers
924 * and set max_free. The subtrees of the root go at the
925 * bottom of llist and rlist.
926 */
927 ltree = root->left;
928 while (llist != NULL) {
929 y = llist->right;
930 llist->right = ltree;
931 vm_map_entry_set_max_free(llist);
932 ltree = llist;
933 llist = y;
934 }
935 rtree = root->right;
936 while (rlist != NULL) {
937 y = rlist->left;
938 rlist->left = rtree;
939 vm_map_entry_set_max_free(rlist);
940 rtree = rlist;
941 rlist = y;
942 }
943
944 /*
945 * Final assembly: add ltree and rtree as subtrees of root.
946 */
947 root->left = ltree;
948 root->right = rtree;
949 vm_map_entry_set_max_free(root);
950
951 return (root);
952 }
953
954 /*
955 * vm_map_entry_{un,}link:
956 *
957 * Insert/remove entries from maps.
958 */
959 static void
960 vm_map_entry_link(vm_map_t map,
961 vm_map_entry_t after_where,
962 vm_map_entry_t entry)
963 {
964
965 CTR4(KTR_VM,
966 "vm_map_entry_link: map %p, nentries %d, entry %p, after %p", map,
967 map->nentries, entry, after_where);
968 VM_MAP_ASSERT_LOCKED(map);
969 map->nentries++;
970 entry->prev = after_where;
971 entry->next = after_where->next;
972 entry->next->prev = entry;
973 after_where->next = entry;
974
975 if (after_where != &map->header) {
976 if (after_where != map->root)
977 vm_map_entry_splay(after_where->start, map->root);
978 entry->right = after_where->right;
979 entry->left = after_where;
980 after_where->right = NULL;
981 after_where->adj_free = entry->start - after_where->end;
982 vm_map_entry_set_max_free(after_where);
983 } else {
984 entry->right = map->root;
985 entry->left = NULL;
986 }
987 entry->adj_free = (entry->next == &map->header ? map->max_offset :
988 entry->next->start) - entry->end;
989 vm_map_entry_set_max_free(entry);
990 map->root = entry;
991 }
992
993 static void
994 vm_map_entry_unlink(vm_map_t map,
995 vm_map_entry_t entry)
996 {
997 vm_map_entry_t next, prev, root;
998
999 VM_MAP_ASSERT_LOCKED(map);
1000 if (entry != map->root)
1001 vm_map_entry_splay(entry->start, map->root);
1002 if (entry->left == NULL)
1003 root = entry->right;
1004 else {
1005 root = vm_map_entry_splay(entry->start, entry->left);
1006 root->right = entry->right;
1007 root->adj_free = (entry->next == &map->header ? map->max_offset :
1008 entry->next->start) - root->end;
1009 vm_map_entry_set_max_free(root);
1010 }
1011 map->root = root;
1012
1013 prev = entry->prev;
1014 next = entry->next;
1015 next->prev = prev;
1016 prev->next = next;
1017 map->nentries--;
1018 CTR3(KTR_VM, "vm_map_entry_unlink: map %p, nentries %d, entry %p", map,
1019 map->nentries, entry);
1020 }
1021
1022 /*
1023 * vm_map_entry_resize_free:
1024 *
1025 * Recompute the amount of free space following a vm_map_entry
1026 * and propagate that value up the tree. Call this function after
1027 * resizing a map entry in-place, that is, without a call to
1028 * vm_map_entry_link() or _unlink().
1029 *
1030 * The map must be locked, and leaves it so.
1031 */
1032 static void
1033 vm_map_entry_resize_free(vm_map_t map, vm_map_entry_t entry)
1034 {
1035
1036 /*
1037 * Using splay trees without parent pointers, propagating
1038 * max_free up the tree is done by moving the entry to the
1039 * root and making the change there.
1040 */
1041 if (entry != map->root)
1042 map->root = vm_map_entry_splay(entry->start, map->root);
1043
1044 entry->adj_free = (entry->next == &map->header ? map->max_offset :
1045 entry->next->start) - entry->end;
1046 vm_map_entry_set_max_free(entry);
1047 }
1048
1049 /*
1050 * vm_map_lookup_entry: [ internal use only ]
1051 *
1052 * Finds the map entry containing (or
1053 * immediately preceding) the specified address
1054 * in the given map; the entry is returned
1055 * in the "entry" parameter. The boolean
1056 * result indicates whether the address is
1057 * actually contained in the map.
1058 */
1059 boolean_t
1060 vm_map_lookup_entry(
1061 vm_map_t map,
1062 vm_offset_t address,
1063 vm_map_entry_t *entry) /* OUT */
1064 {
1065 vm_map_entry_t cur;
1066 boolean_t locked;
1067
1068 /*
1069 * If the map is empty, then the map entry immediately preceding
1070 * "address" is the map's header.
1071 */
1072 cur = map->root;
1073 if (cur == NULL)
1074 *entry = &map->header;
1075 else if (address >= cur->start && cur->end > address) {
1076 *entry = cur;
1077 return (TRUE);
1078 } else if ((locked = vm_map_locked(map)) ||
1079 sx_try_upgrade(&map->lock)) {
1080 /*
1081 * Splay requires a write lock on the map. However, it only
1082 * restructures the binary search tree; it does not otherwise
1083 * change the map. Thus, the map's timestamp need not change
1084 * on a temporary upgrade.
1085 */
1086 map->root = cur = vm_map_entry_splay(address, cur);
1087 if (!locked)
1088 sx_downgrade(&map->lock);
1089
1090 /*
1091 * If "address" is contained within a map entry, the new root
1092 * is that map entry. Otherwise, the new root is a map entry
1093 * immediately before or after "address".
1094 */
1095 if (address >= cur->start) {
1096 *entry = cur;
1097 if (cur->end > address)
1098 return (TRUE);
1099 } else
1100 *entry = cur->prev;
1101 } else
1102 /*
1103 * Since the map is only locked for read access, perform a
1104 * standard binary search tree lookup for "address".
1105 */
1106 for (;;) {
1107 if (address < cur->start) {
1108 if (cur->left == NULL) {
1109 *entry = cur->prev;
1110 break;
1111 }
1112 cur = cur->left;
1113 } else if (cur->end > address) {
1114 *entry = cur;
1115 return (TRUE);
1116 } else {
1117 if (cur->right == NULL) {
1118 *entry = cur;
1119 break;
1120 }
1121 cur = cur->right;
1122 }
1123 }
1124 return (FALSE);
1125 }
1126
1127 /*
1128 * vm_map_insert:
1129 *
1130 * Inserts the given whole VM object into the target
1131 * map at the specified address range. The object's
1132 * size should match that of the address range.
1133 *
1134 * Requires that the map be locked, and leaves it so.
1135 *
1136 * If object is non-NULL, ref count must be bumped by caller
1137 * prior to making call to account for the new entry.
1138 */
1139 int
1140 vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
1141 vm_offset_t start, vm_offset_t end, vm_prot_t prot, vm_prot_t max,
1142 int cow)
1143 {
1144 vm_map_entry_t new_entry;
1145 vm_map_entry_t prev_entry;
1146 vm_map_entry_t temp_entry;
1147 vm_eflags_t protoeflags;
1148 struct ucred *cred;
1149 vm_inherit_t inheritance;
1150 boolean_t charge_prev_obj;
1151
1152 VM_MAP_ASSERT_LOCKED(map);
1153
1154 /*
1155 * Check that the start and end points are not bogus.
1156 */
1157 if ((start < map->min_offset) || (end > map->max_offset) ||
1158 (start >= end))
1159 return (KERN_INVALID_ADDRESS);
1160
1161 /*
1162 * Find the entry prior to the proposed starting address; if it's part
1163 * of an existing entry, this range is bogus.
1164 */
1165 if (vm_map_lookup_entry(map, start, &temp_entry))
1166 return (KERN_NO_SPACE);
1167
1168 prev_entry = temp_entry;
1169
1170 /*
1171 * Assert that the next entry doesn't overlap the end point.
1172 */
1173 if ((prev_entry->next != &map->header) &&
1174 (prev_entry->next->start < end))
1175 return (KERN_NO_SPACE);
1176
1177 protoeflags = 0;
1178 charge_prev_obj = FALSE;
1179
1180 if (cow & MAP_COPY_ON_WRITE)
1181 protoeflags |= MAP_ENTRY_COW|MAP_ENTRY_NEEDS_COPY;
1182
1183 if (cow & MAP_NOFAULT) {
1184 protoeflags |= MAP_ENTRY_NOFAULT;
1185
1186 KASSERT(object == NULL,
1187 ("vm_map_insert: paradoxical MAP_NOFAULT request"));
1188 }
1189 if (cow & MAP_DISABLE_SYNCER)
1190 protoeflags |= MAP_ENTRY_NOSYNC;
1191 if (cow & MAP_DISABLE_COREDUMP)
1192 protoeflags |= MAP_ENTRY_NOCOREDUMP;
1193 if (cow & MAP_VN_WRITECOUNT)
1194 protoeflags |= MAP_ENTRY_VN_WRITECNT;
1195 if (cow & MAP_INHERIT_SHARE)
1196 inheritance = VM_INHERIT_SHARE;
1197 else
1198 inheritance = VM_INHERIT_DEFAULT;
1199
1200 cred = NULL;
1201 KASSERT((object != kmem_object && object != kernel_object) ||
1202 ((object == kmem_object || object == kernel_object) &&
1203 !(protoeflags & MAP_ENTRY_NEEDS_COPY)),
1204 ("kmem or kernel object and cow"));
1205 if (cow & (MAP_ACC_NO_CHARGE | MAP_NOFAULT))
1206 goto charged;
1207 if ((cow & MAP_ACC_CHARGED) || ((prot & VM_PROT_WRITE) &&
1208 ((protoeflags & MAP_ENTRY_NEEDS_COPY) || object == NULL))) {
1209 if (!(cow & MAP_ACC_CHARGED) && !swap_reserve(end - start))
1210 return (KERN_RESOURCE_SHORTAGE);
1211 KASSERT(object == NULL || (protoeflags & MAP_ENTRY_NEEDS_COPY) ||
1212 object->cred == NULL,
1213 ("OVERCOMMIT: vm_map_insert o %p", object));
1214 cred = curthread->td_ucred;
1215 crhold(cred);
1216 if (object == NULL && !(protoeflags & MAP_ENTRY_NEEDS_COPY))
1217 charge_prev_obj = TRUE;
1218 }
1219
1220 charged:
1221 /* Expand the kernel pmap, if necessary. */
1222 if (map == kernel_map && end > kernel_vm_end)
1223 pmap_growkernel(end);
1224 if (object != NULL) {
1225 /*
1226 * OBJ_ONEMAPPING must be cleared unless this mapping
1227 * is trivially proven to be the only mapping for any
1228 * of the object's pages. (Object granularity
1229 * reference counting is insufficient to recognize
1230 * aliases with precision.)
1231 */
1232 VM_OBJECT_LOCK(object);
1233 if (object->ref_count > 1 || object->shadow_count != 0)
1234 vm_object_clear_flag(object, OBJ_ONEMAPPING);
1235 VM_OBJECT_UNLOCK(object);
1236 }
1237 else if ((prev_entry != &map->header) &&
1238 (prev_entry->eflags == protoeflags) &&
1239 (cow & (MAP_ENTRY_GROWS_DOWN | MAP_ENTRY_GROWS_UP)) == 0 &&
1240 (prev_entry->end == start) &&
1241 (prev_entry->wired_count == 0) &&
1242 (prev_entry->cred == cred ||
1243 (prev_entry->object.vm_object != NULL &&
1244 (prev_entry->object.vm_object->cred == cred))) &&
1245 vm_object_coalesce(prev_entry->object.vm_object,
1246 prev_entry->offset,
1247 (vm_size_t)(prev_entry->end - prev_entry->start),
1248 (vm_size_t)(end - prev_entry->end), charge_prev_obj)) {
1249 /*
1250 * We were able to extend the object. Determine if we
1251 * can extend the previous map entry to include the
1252 * new range as well.
1253 */
1254 if ((prev_entry->inheritance == inheritance) &&
1255 (prev_entry->protection == prot) &&
1256 (prev_entry->max_protection == max)) {
1257 map->size += (end - prev_entry->end);
1258 prev_entry->end = end;
1259 vm_map_entry_resize_free(map, prev_entry);
1260 vm_map_simplify_entry(map, prev_entry);
1261 if (cred != NULL)
1262 crfree(cred);
1263 return (KERN_SUCCESS);
1264 }
1265
1266 /*
1267 * If we can extend the object but cannot extend the
1268 * map entry, we have to create a new map entry. We
1269 * must bump the ref count on the extended object to
1270 * account for it. object may be NULL.
1271 */
1272 object = prev_entry->object.vm_object;
1273 offset = prev_entry->offset +
1274 (prev_entry->end - prev_entry->start);
1275 vm_object_reference(object);
1276 if (cred != NULL && object != NULL && object->cred != NULL &&
1277 !(prev_entry->eflags & MAP_ENTRY_NEEDS_COPY)) {
1278 /* Object already accounts for this uid. */
1279 crfree(cred);
1280 cred = NULL;
1281 }
1282 }
1283
1284 /*
1285 * NOTE: if conditionals fail, object can be NULL here. This occurs
1286 * in things like the buffer map where we manage kva but do not manage
1287 * backing objects.
1288 */
1289
1290 /*
1291 * Create a new entry
1292 */
1293 new_entry = vm_map_entry_create(map);
1294 new_entry->start = start;
1295 new_entry->end = end;
1296 new_entry->cred = NULL;
1297
1298 new_entry->eflags = protoeflags;
1299 new_entry->object.vm_object = object;
1300 new_entry->offset = offset;
1301 new_entry->avail_ssize = 0;
1302
1303 new_entry->inheritance = inheritance;
1304 new_entry->protection = prot;
1305 new_entry->max_protection = max;
1306 new_entry->wired_count = 0;
1307 new_entry->read_ahead = VM_FAULT_READ_AHEAD_INIT;
1308 new_entry->next_read = OFF_TO_IDX(offset);
1309
1310 KASSERT(cred == NULL || !ENTRY_CHARGED(new_entry),
1311 ("OVERCOMMIT: vm_map_insert leaks vm_map %p", new_entry));
1312 new_entry->cred = cred;
1313
1314 /*
1315 * Insert the new entry into the list
1316 */
1317 vm_map_entry_link(map, prev_entry, new_entry);
1318 map->size += new_entry->end - new_entry->start;
1319
1320 /*
1321 * It may be possible to merge the new entry with the next and/or
1322 * previous entries. However, due to MAP_STACK_* being a hack, a
1323 * panic can result from merging such entries.
1324 */
1325 if ((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) == 0)
1326 vm_map_simplify_entry(map, new_entry);
1327
1328 if (cow & (MAP_PREFAULT|MAP_PREFAULT_PARTIAL)) {
1329 vm_map_pmap_enter(map, start, prot,
1330 object, OFF_TO_IDX(offset), end - start,
1331 cow & MAP_PREFAULT_PARTIAL);
1332 }
1333
1334 return (KERN_SUCCESS);
1335 }
1336
1337 /*
1338 * vm_map_findspace:
1339 *
1340 * Find the first fit (lowest VM address) for "length" free bytes
1341 * beginning at address >= start in the given map.
1342 *
1343 * In a vm_map_entry, "adj_free" is the amount of free space
1344 * adjacent (higher address) to this entry, and "max_free" is the
1345 * maximum amount of contiguous free space in its subtree. This
1346 * allows finding a free region in one path down the tree, so
1347 * O(log n) amortized with splay trees.
1348 *
1349 * The map must be locked, and leaves it so.
1350 *
1351 * Returns: 0 on success, and starting address in *addr,
1352 * 1 if insufficient space.
1353 */
1354 int
1355 vm_map_findspace(vm_map_t map, vm_offset_t start, vm_size_t length,
1356 vm_offset_t *addr) /* OUT */
1357 {
1358 vm_map_entry_t entry;
1359 vm_offset_t st;
1360
1361 /*
1362 * Request must fit within min/max VM address and must avoid
1363 * address wrap.
1364 */
1365 if (start < map->min_offset)
1366 start = map->min_offset;
1367 if (start + length > map->max_offset || start + length < start)
1368 return (1);
1369
1370 /* Empty tree means wide open address space. */
1371 if (map->root == NULL) {
1372 *addr = start;
1373 return (0);
1374 }
1375
1376 /*
1377 * After splay, if start comes before root node, then there
1378 * must be a gap from start to the root.
1379 */
1380 map->root = vm_map_entry_splay(start, map->root);
1381 if (start + length <= map->root->start) {
1382 *addr = start;
1383 return (0);
1384 }
1385
1386 /*
1387 * Root is the last node that might begin its gap before
1388 * start, and this is the last comparison where address
1389 * wrap might be a problem.
1390 */
1391 st = (start > map->root->end) ? start : map->root->end;
1392 if (length <= map->root->end + map->root->adj_free - st) {
1393 *addr = st;
1394 return (0);
1395 }
1396
1397 /* With max_free, can immediately tell if no solution. */
1398 entry = map->root->right;
1399 if (entry == NULL || length > entry->max_free)
1400 return (1);
1401
1402 /*
1403 * Search the right subtree in the order: left subtree, root,
1404 * right subtree (first fit). The previous splay implies that
1405 * all regions in the right subtree have addresses > start.
1406 */
1407 while (entry != NULL) {
1408 if (entry->left != NULL && entry->left->max_free >= length)
1409 entry = entry->left;
1410 else if (entry->adj_free >= length) {
1411 *addr = entry->end;
1412 return (0);
1413 } else
1414 entry = entry->right;
1415 }
1416
1417 /* Can't get here, so panic if we do. */
1418 panic("vm_map_findspace: max_free corrupt");
1419 }
1420
1421 int
1422 vm_map_fixed(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
1423 vm_offset_t start, vm_size_t length, vm_prot_t prot,
1424 vm_prot_t max, int cow)
1425 {
1426 vm_offset_t end;
1427 int result;
1428
1429 end = start + length;
1430 vm_map_lock(map);
1431 VM_MAP_RANGE_CHECK(map, start, end);
1432 (void) vm_map_delete(map, start, end);
1433 result = vm_map_insert(map, object, offset, start, end, prot,
1434 max, cow);
1435 vm_map_unlock(map);
1436 return (result);
1437 }
1438
1439 /*
1440 * vm_map_find finds an unallocated region in the target address
1441 * map with the given length. The search is defined to be
1442 * first-fit from the specified address; the region found is
1443 * returned in the same parameter.
1444 *
1445 * If object is non-NULL, ref count must be bumped by caller
1446 * prior to making call to account for the new entry.
1447 */
1448 int
1449 vm_map_find(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
1450 vm_offset_t *addr, /* IN/OUT */
1451 vm_size_t length, int find_space, vm_prot_t prot,
1452 vm_prot_t max, int cow)
1453 {
1454 vm_offset_t start;
1455 int result;
1456
1457 start = *addr;
1458 vm_map_lock(map);
1459 do {
1460 if (find_space != VMFS_NO_SPACE) {
1461 if (vm_map_findspace(map, start, length, addr)) {
1462 vm_map_unlock(map);
1463 return (KERN_NO_SPACE);
1464 }
1465 switch (find_space) {
1466 case VMFS_ALIGNED_SPACE:
1467 pmap_align_superpage(object, offset, addr,
1468 length);
1469 break;
1470 #ifdef VMFS_TLB_ALIGNED_SPACE
1471 case VMFS_TLB_ALIGNED_SPACE:
1472 pmap_align_tlb(addr);
1473 break;
1474 #endif
1475 default:
1476 break;
1477 }
1478
1479 start = *addr;
1480 }
1481 result = vm_map_insert(map, object, offset, start, start +
1482 length, prot, max, cow);
1483 } while (result == KERN_NO_SPACE && (find_space == VMFS_ALIGNED_SPACE
1484 #ifdef VMFS_TLB_ALIGNED_SPACE
1485 || find_space == VMFS_TLB_ALIGNED_SPACE
1486 #endif
1487 ));
1488 vm_map_unlock(map);
1489 return (result);
1490 }
1491
1492 /*
1493 * vm_map_simplify_entry:
1494 *
1495 * Simplify the given map entry by merging with either neighbor. This
1496 * routine also has the ability to merge with both neighbors.
1497 *
1498 * The map must be locked.
1499 *
1500 * This routine guarentees that the passed entry remains valid (though
1501 * possibly extended). When merging, this routine may delete one or
1502 * both neighbors.
1503 */
1504 void
1505 vm_map_simplify_entry(vm_map_t map, vm_map_entry_t entry)
1506 {
1507 vm_map_entry_t next, prev;
1508 vm_size_t prevsize, esize;
1509
1510 if (entry->eflags & (MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_IS_SUB_MAP))
1511 return;
1512
1513 prev = entry->prev;
1514 if (prev != &map->header) {
1515 prevsize = prev->end - prev->start;
1516 if ( (prev->end == entry->start) &&
1517 (prev->object.vm_object == entry->object.vm_object) &&
1518 (!prev->object.vm_object ||
1519 (prev->offset + prevsize == entry->offset)) &&
1520 (prev->eflags == entry->eflags) &&
1521 (prev->protection == entry->protection) &&
1522 (prev->max_protection == entry->max_protection) &&
1523 (prev->inheritance == entry->inheritance) &&
1524 (prev->wired_count == entry->wired_count) &&
1525 (prev->cred == entry->cred)) {
1526 vm_map_entry_unlink(map, prev);
1527 entry->start = prev->start;
1528 entry->offset = prev->offset;
1529 if (entry->prev != &map->header)
1530 vm_map_entry_resize_free(map, entry->prev);
1531
1532 /*
1533 * If the backing object is a vnode object,
1534 * vm_object_deallocate() calls vrele().
1535 * However, vrele() does not lock the vnode
1536 * because the vnode has additional
1537 * references. Thus, the map lock can be kept
1538 * without causing a lock-order reversal with
1539 * the vnode lock.
1540 *
1541 * Since we count the number of virtual page
1542 * mappings in object->un_pager.vnp.writemappings,
1543 * the writemappings value should not be adjusted
1544 * when the entry is disposed of.
1545 */
1546 if (prev->object.vm_object)
1547 vm_object_deallocate(prev->object.vm_object);
1548 if (prev->cred != NULL)
1549 crfree(prev->cred);
1550 vm_map_entry_dispose(map, prev);
1551 }
1552 }
1553
1554 next = entry->next;
1555 if (next != &map->header) {
1556 esize = entry->end - entry->start;
1557 if ((entry->end == next->start) &&
1558 (next->object.vm_object == entry->object.vm_object) &&
1559 (!entry->object.vm_object ||
1560 (entry->offset + esize == next->offset)) &&
1561 (next->eflags == entry->eflags) &&
1562 (next->protection == entry->protection) &&
1563 (next->max_protection == entry->max_protection) &&
1564 (next->inheritance == entry->inheritance) &&
1565 (next->wired_count == entry->wired_count) &&
1566 (next->cred == entry->cred)) {
1567 vm_map_entry_unlink(map, next);
1568 entry->end = next->end;
1569 vm_map_entry_resize_free(map, entry);
1570
1571 /*
1572 * See comment above.
1573 */
1574 if (next->object.vm_object)
1575 vm_object_deallocate(next->object.vm_object);
1576 if (next->cred != NULL)
1577 crfree(next->cred);
1578 vm_map_entry_dispose(map, next);
1579 }
1580 }
1581 }
1582 /*
1583 * vm_map_clip_start: [ internal use only ]
1584 *
1585 * Asserts that the given entry begins at or after
1586 * the specified address; if necessary,
1587 * it splits the entry into two.
1588 */
1589 #define vm_map_clip_start(map, entry, startaddr) \
1590 { \
1591 if (startaddr > entry->start) \
1592 _vm_map_clip_start(map, entry, startaddr); \
1593 }
1594
1595 /*
1596 * This routine is called only when it is known that
1597 * the entry must be split.
1598 */
1599 static void
1600 _vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t start)
1601 {
1602 vm_map_entry_t new_entry;
1603
1604 VM_MAP_ASSERT_LOCKED(map);
1605
1606 /*
1607 * Split off the front portion -- note that we must insert the new
1608 * entry BEFORE this one, so that this entry has the specified
1609 * starting address.
1610 */
1611 vm_map_simplify_entry(map, entry);
1612
1613 /*
1614 * If there is no object backing this entry, we might as well create
1615 * one now. If we defer it, an object can get created after the map
1616 * is clipped, and individual objects will be created for the split-up
1617 * map. This is a bit of a hack, but is also about the best place to
1618 * put this improvement.
1619 */
1620 if (entry->object.vm_object == NULL && !map->system_map) {
1621 vm_object_t object;
1622 object = vm_object_allocate(OBJT_DEFAULT,
1623 atop(entry->end - entry->start));
1624 entry->object.vm_object = object;
1625 entry->offset = 0;
1626 if (entry->cred != NULL) {
1627 object->cred = entry->cred;
1628 object->charge = entry->end - entry->start;
1629 entry->cred = NULL;
1630 }
1631 } else if (entry->object.vm_object != NULL &&
1632 ((entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) &&
1633 entry->cred != NULL) {
1634 VM_OBJECT_LOCK(entry->object.vm_object);
1635 KASSERT(entry->object.vm_object->cred == NULL,
1636 ("OVERCOMMIT: vm_entry_clip_start: both cred e %p", entry));
1637 entry->object.vm_object->cred = entry->cred;
1638 entry->object.vm_object->charge = entry->end - entry->start;
1639 VM_OBJECT_UNLOCK(entry->object.vm_object);
1640 entry->cred = NULL;
1641 }
1642
1643 new_entry = vm_map_entry_create(map);
1644 *new_entry = *entry;
1645
1646 new_entry->end = start;
1647 entry->offset += (start - entry->start);
1648 entry->start = start;
1649 if (new_entry->cred != NULL)
1650 crhold(entry->cred);
1651
1652 vm_map_entry_link(map, entry->prev, new_entry);
1653
1654 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
1655 vm_object_reference(new_entry->object.vm_object);
1656 /*
1657 * The object->un_pager.vnp.writemappings for the
1658 * object of MAP_ENTRY_VN_WRITECNT type entry shall be
1659 * kept as is here. The virtual pages are
1660 * re-distributed among the clipped entries, so the sum is
1661 * left the same.
1662 */
1663 }
1664 }
1665
1666 /*
1667 * vm_map_clip_end: [ internal use only ]
1668 *
1669 * Asserts that the given entry ends at or before
1670 * the specified address; if necessary,
1671 * it splits the entry into two.
1672 */
1673 #define vm_map_clip_end(map, entry, endaddr) \
1674 { \
1675 if ((endaddr) < (entry->end)) \
1676 _vm_map_clip_end((map), (entry), (endaddr)); \
1677 }
1678
1679 /*
1680 * This routine is called only when it is known that
1681 * the entry must be split.
1682 */
1683 static void
1684 _vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t end)
1685 {
1686 vm_map_entry_t new_entry;
1687
1688 VM_MAP_ASSERT_LOCKED(map);
1689
1690 /*
1691 * If there is no object backing this entry, we might as well create
1692 * one now. If we defer it, an object can get created after the map
1693 * is clipped, and individual objects will be created for the split-up
1694 * map. This is a bit of a hack, but is also about the best place to
1695 * put this improvement.
1696 */
1697 if (entry->object.vm_object == NULL && !map->system_map) {
1698 vm_object_t object;
1699 object = vm_object_allocate(OBJT_DEFAULT,
1700 atop(entry->end - entry->start));
1701 entry->object.vm_object = object;
1702 entry->offset = 0;
1703 if (entry->cred != NULL) {
1704 object->cred = entry->cred;
1705 object->charge = entry->end - entry->start;
1706 entry->cred = NULL;
1707 }
1708 } else if (entry->object.vm_object != NULL &&
1709 ((entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) &&
1710 entry->cred != NULL) {
1711 VM_OBJECT_LOCK(entry->object.vm_object);
1712 KASSERT(entry->object.vm_object->cred == NULL,
1713 ("OVERCOMMIT: vm_entry_clip_end: both cred e %p", entry));
1714 entry->object.vm_object->cred = entry->cred;
1715 entry->object.vm_object->charge = entry->end - entry->start;
1716 VM_OBJECT_UNLOCK(entry->object.vm_object);
1717 entry->cred = NULL;
1718 }
1719
1720 /*
1721 * Create a new entry and insert it AFTER the specified entry
1722 */
1723 new_entry = vm_map_entry_create(map);
1724 *new_entry = *entry;
1725
1726 new_entry->start = entry->end = end;
1727 new_entry->offset += (end - entry->start);
1728 if (new_entry->cred != NULL)
1729 crhold(entry->cred);
1730
1731 vm_map_entry_link(map, entry, new_entry);
1732
1733 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
1734 vm_object_reference(new_entry->object.vm_object);
1735 }
1736 }
1737
1738 /*
1739 * vm_map_submap: [ kernel use only ]
1740 *
1741 * Mark the given range as handled by a subordinate map.
1742 *
1743 * This range must have been created with vm_map_find,
1744 * and no other operations may have been performed on this
1745 * range prior to calling vm_map_submap.
1746 *
1747 * Only a limited number of operations can be performed
1748 * within this rage after calling vm_map_submap:
1749 * vm_fault
1750 * [Don't try vm_map_copy!]
1751 *
1752 * To remove a submapping, one must first remove the
1753 * range from the superior map, and then destroy the
1754 * submap (if desired). [Better yet, don't try it.]
1755 */
1756 int
1757 vm_map_submap(
1758 vm_map_t map,
1759 vm_offset_t start,
1760 vm_offset_t end,
1761 vm_map_t submap)
1762 {
1763 vm_map_entry_t entry;
1764 int result = KERN_INVALID_ARGUMENT;
1765
1766 vm_map_lock(map);
1767
1768 VM_MAP_RANGE_CHECK(map, start, end);
1769
1770 if (vm_map_lookup_entry(map, start, &entry)) {
1771 vm_map_clip_start(map, entry, start);
1772 } else
1773 entry = entry->next;
1774
1775 vm_map_clip_end(map, entry, end);
1776
1777 if ((entry->start == start) && (entry->end == end) &&
1778 ((entry->eflags & MAP_ENTRY_COW) == 0) &&
1779 (entry->object.vm_object == NULL)) {
1780 entry->object.sub_map = submap;
1781 entry->eflags |= MAP_ENTRY_IS_SUB_MAP;
1782 result = KERN_SUCCESS;
1783 }
1784 vm_map_unlock(map);
1785
1786 return (result);
1787 }
1788
1789 /*
1790 * The maximum number of pages to map
1791 */
1792 #define MAX_INIT_PT 96
1793
1794 /*
1795 * vm_map_pmap_enter:
1796 *
1797 * Preload read-only mappings for the given object's resident pages into
1798 * the given map. This eliminates the soft faults on process startup and
1799 * immediately after an mmap(2). Because these are speculative mappings,
1800 * cached pages are not reactivated and mapped.
1801 */
1802 void
1803 vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot,
1804 vm_object_t object, vm_pindex_t pindex, vm_size_t size, int flags)
1805 {
1806 vm_offset_t start;
1807 vm_page_t p, p_start;
1808 vm_pindex_t psize, tmpidx;
1809
1810 if ((prot & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0 || object == NULL)
1811 return;
1812 VM_OBJECT_LOCK(object);
1813 if (object->type == OBJT_DEVICE || object->type == OBJT_SG) {
1814 pmap_object_init_pt(map->pmap, addr, object, pindex, size);
1815 goto unlock_return;
1816 }
1817
1818 psize = atop(size);
1819
1820 if ((flags & MAP_PREFAULT_PARTIAL) && psize > MAX_INIT_PT &&
1821 object->resident_page_count > MAX_INIT_PT)
1822 goto unlock_return;
1823
1824 if (psize + pindex > object->size) {
1825 if (object->size < pindex)
1826 goto unlock_return;
1827 psize = object->size - pindex;
1828 }
1829
1830 start = 0;
1831 p_start = NULL;
1832
1833 p = vm_page_find_least(object, pindex);
1834 /*
1835 * Assert: the variable p is either (1) the page with the
1836 * least pindex greater than or equal to the parameter pindex
1837 * or (2) NULL.
1838 */
1839 for (;
1840 p != NULL && (tmpidx = p->pindex - pindex) < psize;
1841 p = TAILQ_NEXT(p, listq)) {
1842 /*
1843 * don't allow an madvise to blow away our really
1844 * free pages allocating pv entries.
1845 */
1846 if ((flags & MAP_PREFAULT_MADVISE) &&
1847 cnt.v_free_count < cnt.v_free_reserved) {
1848 psize = tmpidx;
1849 break;
1850 }
1851 if (p->valid == VM_PAGE_BITS_ALL) {
1852 if (p_start == NULL) {
1853 start = addr + ptoa(tmpidx);
1854 p_start = p;
1855 }
1856 } else if (p_start != NULL) {
1857 pmap_enter_object(map->pmap, start, addr +
1858 ptoa(tmpidx), p_start, prot);
1859 p_start = NULL;
1860 }
1861 }
1862 if (p_start != NULL)
1863 pmap_enter_object(map->pmap, start, addr + ptoa(psize),
1864 p_start, prot);
1865 unlock_return:
1866 VM_OBJECT_UNLOCK(object);
1867 }
1868
1869 /*
1870 * vm_map_protect:
1871 *
1872 * Sets the protection of the specified address
1873 * region in the target map. If "set_max" is
1874 * specified, the maximum protection is to be set;
1875 * otherwise, only the current protection is affected.
1876 */
1877 int
1878 vm_map_protect(vm_map_t map, vm_offset_t start, vm_offset_t end,
1879 vm_prot_t new_prot, boolean_t set_max)
1880 {
1881 vm_map_entry_t current, entry;
1882 vm_object_t obj;
1883 struct ucred *cred;
1884 vm_prot_t old_prot;
1885
1886 vm_map_lock(map);
1887
1888 VM_MAP_RANGE_CHECK(map, start, end);
1889
1890 if (vm_map_lookup_entry(map, start, &entry)) {
1891 vm_map_clip_start(map, entry, start);
1892 } else {
1893 entry = entry->next;
1894 }
1895
1896 /*
1897 * Make a first pass to check for protection violations.
1898 */
1899 current = entry;
1900 while ((current != &map->header) && (current->start < end)) {
1901 if (current->eflags & MAP_ENTRY_IS_SUB_MAP) {
1902 vm_map_unlock(map);
1903 return (KERN_INVALID_ARGUMENT);
1904 }
1905 if ((new_prot & current->max_protection) != new_prot) {
1906 vm_map_unlock(map);
1907 return (KERN_PROTECTION_FAILURE);
1908 }
1909 current = current->next;
1910 }
1911
1912
1913 /*
1914 * Do an accounting pass for private read-only mappings that
1915 * now will do cow due to allowed write (e.g. debugger sets
1916 * breakpoint on text segment)
1917 */
1918 for (current = entry; (current != &map->header) &&
1919 (current->start < end); current = current->next) {
1920
1921 vm_map_clip_end(map, current, end);
1922
1923 if (set_max ||
1924 ((new_prot & ~(current->protection)) & VM_PROT_WRITE) == 0 ||
1925 ENTRY_CHARGED(current)) {
1926 continue;
1927 }
1928
1929 cred = curthread->td_ucred;
1930 obj = current->object.vm_object;
1931
1932 if (obj == NULL || (current->eflags & MAP_ENTRY_NEEDS_COPY)) {
1933 if (!swap_reserve(current->end - current->start)) {
1934 vm_map_unlock(map);
1935 return (KERN_RESOURCE_SHORTAGE);
1936 }
1937 crhold(cred);
1938 current->cred = cred;
1939 continue;
1940 }
1941
1942 VM_OBJECT_LOCK(obj);
1943 if (obj->type != OBJT_DEFAULT && obj->type != OBJT_SWAP) {
1944 VM_OBJECT_UNLOCK(obj);
1945 continue;
1946 }
1947
1948 /*
1949 * Charge for the whole object allocation now, since
1950 * we cannot distinguish between non-charged and
1951 * charged clipped mapping of the same object later.
1952 */
1953 KASSERT(obj->charge == 0,
1954 ("vm_map_protect: object %p overcharged\n", obj));
1955 if (!swap_reserve(ptoa(obj->size))) {
1956 VM_OBJECT_UNLOCK(obj);
1957 vm_map_unlock(map);
1958 return (KERN_RESOURCE_SHORTAGE);
1959 }
1960
1961 crhold(cred);
1962 obj->cred = cred;
1963 obj->charge = ptoa(obj->size);
1964 VM_OBJECT_UNLOCK(obj);
1965 }
1966
1967 /*
1968 * Go back and fix up protections. [Note that clipping is not
1969 * necessary the second time.]
1970 */
1971 current = entry;
1972 while ((current != &map->header) && (current->start < end)) {
1973 old_prot = current->protection;
1974
1975 if (set_max)
1976 current->protection =
1977 (current->max_protection = new_prot) &
1978 old_prot;
1979 else
1980 current->protection = new_prot;
1981
1982 if ((current->eflags & (MAP_ENTRY_COW | MAP_ENTRY_USER_WIRED))
1983 == (MAP_ENTRY_COW | MAP_ENTRY_USER_WIRED) &&
1984 (current->protection & VM_PROT_WRITE) != 0 &&
1985 (old_prot & VM_PROT_WRITE) == 0) {
1986 vm_fault_copy_entry(map, map, current, current, NULL);
1987 }
1988
1989 /*
1990 * When restricting access, update the physical map. Worry
1991 * about copy-on-write here.
1992 */
1993 if ((old_prot & ~current->protection) != 0) {
1994 #define MASK(entry) (((entry)->eflags & MAP_ENTRY_COW) ? ~VM_PROT_WRITE : \
1995 VM_PROT_ALL)
1996 pmap_protect(map->pmap, current->start,
1997 current->end,
1998 current->protection & MASK(current));
1999 #undef MASK
2000 }
2001 vm_map_simplify_entry(map, current);
2002 current = current->next;
2003 }
2004 vm_map_unlock(map);
2005 return (KERN_SUCCESS);
2006 }
2007
2008 /*
2009 * vm_map_madvise:
2010 *
2011 * This routine traverses a processes map handling the madvise
2012 * system call. Advisories are classified as either those effecting
2013 * the vm_map_entry structure, or those effecting the underlying
2014 * objects.
2015 */
2016 int
2017 vm_map_madvise(
2018 vm_map_t map,
2019 vm_offset_t start,
2020 vm_offset_t end,
2021 int behav)
2022 {
2023 vm_map_entry_t current, entry;
2024 int modify_map = 0;
2025
2026 /*
2027 * Some madvise calls directly modify the vm_map_entry, in which case
2028 * we need to use an exclusive lock on the map and we need to perform
2029 * various clipping operations. Otherwise we only need a read-lock
2030 * on the map.
2031 */
2032 switch(behav) {
2033 case MADV_NORMAL:
2034 case MADV_SEQUENTIAL:
2035 case MADV_RANDOM:
2036 case MADV_NOSYNC:
2037 case MADV_AUTOSYNC:
2038 case MADV_NOCORE:
2039 case MADV_CORE:
2040 modify_map = 1;
2041 vm_map_lock(map);
2042 break;
2043 case MADV_WILLNEED:
2044 case MADV_DONTNEED:
2045 case MADV_FREE:
2046 vm_map_lock_read(map);
2047 break;
2048 default:
2049 return (KERN_INVALID_ARGUMENT);
2050 }
2051
2052 /*
2053 * Locate starting entry and clip if necessary.
2054 */
2055 VM_MAP_RANGE_CHECK(map, start, end);
2056
2057 if (vm_map_lookup_entry(map, start, &entry)) {
2058 if (modify_map)
2059 vm_map_clip_start(map, entry, start);
2060 } else {
2061 entry = entry->next;
2062 }
2063
2064 if (modify_map) {
2065 /*
2066 * madvise behaviors that are implemented in the vm_map_entry.
2067 *
2068 * We clip the vm_map_entry so that behavioral changes are
2069 * limited to the specified address range.
2070 */
2071 for (current = entry;
2072 (current != &map->header) && (current->start < end);
2073 current = current->next
2074 ) {
2075 if (current->eflags & MAP_ENTRY_IS_SUB_MAP)
2076 continue;
2077
2078 vm_map_clip_end(map, current, end);
2079
2080 switch (behav) {
2081 case MADV_NORMAL:
2082 vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_NORMAL);
2083 break;
2084 case MADV_SEQUENTIAL:
2085 vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_SEQUENTIAL);
2086 break;
2087 case MADV_RANDOM:
2088 vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_RANDOM);
2089 break;
2090 case MADV_NOSYNC:
2091 current->eflags |= MAP_ENTRY_NOSYNC;
2092 break;
2093 case MADV_AUTOSYNC:
2094 current->eflags &= ~MAP_ENTRY_NOSYNC;
2095 break;
2096 case MADV_NOCORE:
2097 current->eflags |= MAP_ENTRY_NOCOREDUMP;
2098 break;
2099 case MADV_CORE:
2100 current->eflags &= ~MAP_ENTRY_NOCOREDUMP;
2101 break;
2102 default:
2103 break;
2104 }
2105 vm_map_simplify_entry(map, current);
2106 }
2107 vm_map_unlock(map);
2108 } else {
2109 vm_pindex_t pstart, pend;
2110
2111 /*
2112 * madvise behaviors that are implemented in the underlying
2113 * vm_object.
2114 *
2115 * Since we don't clip the vm_map_entry, we have to clip
2116 * the vm_object pindex and count.
2117 */
2118 for (current = entry;
2119 (current != &map->header) && (current->start < end);
2120 current = current->next
2121 ) {
2122 vm_offset_t useStart;
2123
2124 if (current->eflags & MAP_ENTRY_IS_SUB_MAP)
2125 continue;
2126
2127 pstart = OFF_TO_IDX(current->offset);
2128 pend = pstart + atop(current->end - current->start);
2129 useStart = current->start;
2130
2131 if (current->start < start) {
2132 pstart += atop(start - current->start);
2133 useStart = start;
2134 }
2135 if (current->end > end)
2136 pend -= atop(current->end - end);
2137
2138 if (pstart >= pend)
2139 continue;
2140
2141 vm_object_madvise(current->object.vm_object, pstart,
2142 pend, behav);
2143 if (behav == MADV_WILLNEED) {
2144 vm_map_pmap_enter(map,
2145 useStart,
2146 current->protection,
2147 current->object.vm_object,
2148 pstart,
2149 ptoa(pend - pstart),
2150 MAP_PREFAULT_MADVISE
2151 );
2152 }
2153 }
2154 vm_map_unlock_read(map);
2155 }
2156 return (0);
2157 }
2158
2159
2160 /*
2161 * vm_map_inherit:
2162 *
2163 * Sets the inheritance of the specified address
2164 * range in the target map. Inheritance
2165 * affects how the map will be shared with
2166 * child maps at the time of vmspace_fork.
2167 */
2168 int
2169 vm_map_inherit(vm_map_t map, vm_offset_t start, vm_offset_t end,
2170 vm_inherit_t new_inheritance)
2171 {
2172 vm_map_entry_t entry;
2173 vm_map_entry_t temp_entry;
2174
2175 switch (new_inheritance) {
2176 case VM_INHERIT_NONE:
2177 case VM_INHERIT_COPY:
2178 case VM_INHERIT_SHARE:
2179 break;
2180 default:
2181 return (KERN_INVALID_ARGUMENT);
2182 }
2183 vm_map_lock(map);
2184 VM_MAP_RANGE_CHECK(map, start, end);
2185 if (vm_map_lookup_entry(map, start, &temp_entry)) {
2186 entry = temp_entry;
2187 vm_map_clip_start(map, entry, start);
2188 } else
2189 entry = temp_entry->next;
2190 while ((entry != &map->header) && (entry->start < end)) {
2191 vm_map_clip_end(map, entry, end);
2192 entry->inheritance = new_inheritance;
2193 vm_map_simplify_entry(map, entry);
2194 entry = entry->next;
2195 }
2196 vm_map_unlock(map);
2197 return (KERN_SUCCESS);
2198 }
2199
2200 /*
2201 * vm_map_unwire:
2202 *
2203 * Implements both kernel and user unwiring.
2204 */
2205 int
2206 vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end,
2207 int flags)
2208 {
2209 vm_map_entry_t entry, first_entry, tmp_entry;
2210 vm_offset_t saved_start;
2211 unsigned int last_timestamp;
2212 int rv;
2213 boolean_t need_wakeup, result, user_unwire;
2214
2215 user_unwire = (flags & VM_MAP_WIRE_USER) ? TRUE : FALSE;
2216 vm_map_lock(map);
2217 VM_MAP_RANGE_CHECK(map, start, end);
2218 if (!vm_map_lookup_entry(map, start, &first_entry)) {
2219 if (flags & VM_MAP_WIRE_HOLESOK)
2220 first_entry = first_entry->next;
2221 else {
2222 vm_map_unlock(map);
2223 return (KERN_INVALID_ADDRESS);
2224 }
2225 }
2226 last_timestamp = map->timestamp;
2227 entry = first_entry;
2228 while (entry != &map->header && entry->start < end) {
2229 if (entry->eflags & MAP_ENTRY_IN_TRANSITION) {
2230 /*
2231 * We have not yet clipped the entry.
2232 */
2233 saved_start = (start >= entry->start) ? start :
2234 entry->start;
2235 entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
2236 if (vm_map_unlock_and_wait(map, 0)) {
2237 /*
2238 * Allow interruption of user unwiring?
2239 */
2240 }
2241 vm_map_lock(map);
2242 if (last_timestamp+1 != map->timestamp) {
2243 /*
2244 * Look again for the entry because the map was
2245 * modified while it was unlocked.
2246 * Specifically, the entry may have been
2247 * clipped, merged, or deleted.
2248 */
2249 if (!vm_map_lookup_entry(map, saved_start,
2250 &tmp_entry)) {
2251 if (flags & VM_MAP_WIRE_HOLESOK)
2252 tmp_entry = tmp_entry->next;
2253 else {
2254 if (saved_start == start) {
2255 /*
2256 * First_entry has been deleted.
2257 */
2258 vm_map_unlock(map);
2259 return (KERN_INVALID_ADDRESS);
2260 }
2261 end = saved_start;
2262 rv = KERN_INVALID_ADDRESS;
2263 goto done;
2264 }
2265 }
2266 if (entry == first_entry)
2267 first_entry = tmp_entry;
2268 else
2269 first_entry = NULL;
2270 entry = tmp_entry;
2271 }
2272 last_timestamp = map->timestamp;
2273 continue;
2274 }
2275 vm_map_clip_start(map, entry, start);
2276 vm_map_clip_end(map, entry, end);
2277 /*
2278 * Mark the entry in case the map lock is released. (See
2279 * above.)
2280 */
2281 entry->eflags |= MAP_ENTRY_IN_TRANSITION;
2282 /*
2283 * Check the map for holes in the specified region.
2284 * If VM_MAP_WIRE_HOLESOK was specified, skip this check.
2285 */
2286 if (((flags & VM_MAP_WIRE_HOLESOK) == 0) &&
2287 (entry->end < end && (entry->next == &map->header ||
2288 entry->next->start > entry->end))) {
2289 end = entry->end;
2290 rv = KERN_INVALID_ADDRESS;
2291 goto done;
2292 }
2293 /*
2294 * If system unwiring, require that the entry is system wired.
2295 */
2296 if (!user_unwire &&
2297 vm_map_entry_system_wired_count(entry) == 0) {
2298 end = entry->end;
2299 rv = KERN_INVALID_ARGUMENT;
2300 goto done;
2301 }
2302 entry = entry->next;
2303 }
2304 rv = KERN_SUCCESS;
2305 done:
2306 need_wakeup = FALSE;
2307 if (first_entry == NULL) {
2308 result = vm_map_lookup_entry(map, start, &first_entry);
2309 if (!result && (flags & VM_MAP_WIRE_HOLESOK))
2310 first_entry = first_entry->next;
2311 else
2312 KASSERT(result, ("vm_map_unwire: lookup failed"));
2313 }
2314 entry = first_entry;
2315 while (entry != &map->header && entry->start < end) {
2316 if (rv == KERN_SUCCESS && (!user_unwire ||
2317 (entry->eflags & MAP_ENTRY_USER_WIRED))) {
2318 if (user_unwire)
2319 entry->eflags &= ~MAP_ENTRY_USER_WIRED;
2320 entry->wired_count--;
2321 if (entry->wired_count == 0) {
2322 /*
2323 * Retain the map lock.
2324 */
2325 vm_fault_unwire(map, entry->start, entry->end,
2326 entry->object.vm_object != NULL &&
2327 (entry->object.vm_object->type == OBJT_DEVICE ||
2328 entry->object.vm_object->type == OBJT_SG));
2329 }
2330 }
2331 KASSERT(entry->eflags & MAP_ENTRY_IN_TRANSITION,
2332 ("vm_map_unwire: in-transition flag missing"));
2333 entry->eflags &= ~MAP_ENTRY_IN_TRANSITION;
2334 if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) {
2335 entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP;
2336 need_wakeup = TRUE;
2337 }
2338 vm_map_simplify_entry(map, entry);
2339 entry = entry->next;
2340 }
2341 vm_map_unlock(map);
2342 if (need_wakeup)
2343 vm_map_wakeup(map);
2344 return (rv);
2345 }
2346
2347 /*
2348 * vm_map_wire:
2349 *
2350 * Implements both kernel and user wiring.
2351 */
2352 int
2353 vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end,
2354 int flags)
2355 {
2356 vm_map_entry_t entry, first_entry, tmp_entry;
2357 vm_offset_t saved_end, saved_start;
2358 unsigned int last_timestamp;
2359 int rv;
2360 boolean_t fictitious, need_wakeup, result, user_wire;
2361 vm_prot_t prot;
2362
2363 prot = 0;
2364 if (flags & VM_MAP_WIRE_WRITE)
2365 prot |= VM_PROT_WRITE;
2366 user_wire = (flags & VM_MAP_WIRE_USER) ? TRUE : FALSE;
2367 vm_map_lock(map);
2368 VM_MAP_RANGE_CHECK(map, start, end);
2369 if (!vm_map_lookup_entry(map, start, &first_entry)) {
2370 if (flags & VM_MAP_WIRE_HOLESOK)
2371 first_entry = first_entry->next;
2372 else {
2373 vm_map_unlock(map);
2374 return (KERN_INVALID_ADDRESS);
2375 }
2376 }
2377 last_timestamp = map->timestamp;
2378 entry = first_entry;
2379 while (entry != &map->header && entry->start < end) {
2380 if (entry->eflags & MAP_ENTRY_IN_TRANSITION) {
2381 /*
2382 * We have not yet clipped the entry.
2383 */
2384 saved_start = (start >= entry->start) ? start :
2385 entry->start;
2386 entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
2387 if (vm_map_unlock_and_wait(map, 0)) {
2388 /*
2389 * Allow interruption of user wiring?
2390 */
2391 }
2392 vm_map_lock(map);
2393 if (last_timestamp + 1 != map->timestamp) {
2394 /*
2395 * Look again for the entry because the map was
2396 * modified while it was unlocked.
2397 * Specifically, the entry may have been
2398 * clipped, merged, or deleted.
2399 */
2400 if (!vm_map_lookup_entry(map, saved_start,
2401 &tmp_entry)) {
2402 if (flags & VM_MAP_WIRE_HOLESOK)
2403 tmp_entry = tmp_entry->next;
2404 else {
2405 if (saved_start == start) {
2406 /*
2407 * first_entry has been deleted.
2408 */
2409 vm_map_unlock(map);
2410 return (KERN_INVALID_ADDRESS);
2411 }
2412 end = saved_start;
2413 rv = KERN_INVALID_ADDRESS;
2414 goto done;
2415 }
2416 }
2417 if (entry == first_entry)
2418 first_entry = tmp_entry;
2419 else
2420 first_entry = NULL;
2421 entry = tmp_entry;
2422 }
2423 last_timestamp = map->timestamp;
2424 continue;
2425 }
2426 vm_map_clip_start(map, entry, start);
2427 vm_map_clip_end(map, entry, end);
2428 /*
2429 * Mark the entry in case the map lock is released. (See
2430 * above.)
2431 */
2432 entry->eflags |= MAP_ENTRY_IN_TRANSITION;
2433 if ((entry->protection & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0
2434 || (entry->protection & prot) != prot) {
2435 entry->eflags |= MAP_ENTRY_WIRE_SKIPPED;
2436 if ((flags & VM_MAP_WIRE_HOLESOK) == 0) {
2437 end = entry->end;
2438 rv = KERN_INVALID_ADDRESS;
2439 goto done;
2440 }
2441 goto next_entry;
2442 }
2443 if (entry->wired_count == 0) {
2444 entry->wired_count++;
2445 saved_start = entry->start;
2446 saved_end = entry->end;
2447 fictitious = entry->object.vm_object != NULL &&
2448 (entry->object.vm_object->type == OBJT_DEVICE ||
2449 entry->object.vm_object->type == OBJT_SG);
2450 /*
2451 * Release the map lock, relying on the in-transition
2452 * mark. Mark the map busy for fork.
2453 */
2454 vm_map_busy(map);
2455 vm_map_unlock(map);
2456 rv = vm_fault_wire(map, saved_start, saved_end,
2457 fictitious);
2458 vm_map_lock(map);
2459 vm_map_unbusy(map);
2460 if (last_timestamp + 1 != map->timestamp) {
2461 /*
2462 * Look again for the entry because the map was
2463 * modified while it was unlocked. The entry
2464 * may have been clipped, but NOT merged or
2465 * deleted.
2466 */
2467 result = vm_map_lookup_entry(map, saved_start,
2468 &tmp_entry);
2469 KASSERT(result, ("vm_map_wire: lookup failed"));
2470 if (entry == first_entry)
2471 first_entry = tmp_entry;
2472 else
2473 first_entry = NULL;
2474 entry = tmp_entry;
2475 while (entry->end < saved_end) {
2476 if (rv != KERN_SUCCESS) {
2477 KASSERT(entry->wired_count == 1,
2478 ("vm_map_wire: bad count"));
2479 entry->wired_count = -1;
2480 }
2481 entry = entry->next;
2482 }
2483 }
2484 last_timestamp = map->timestamp;
2485 if (rv != KERN_SUCCESS) {
2486 KASSERT(entry->wired_count == 1,
2487 ("vm_map_wire: bad count"));
2488 /*
2489 * Assign an out-of-range value to represent
2490 * the failure to wire this entry.
2491 */
2492 entry->wired_count = -1;
2493 end = entry->end;
2494 goto done;
2495 }
2496 } else if (!user_wire ||
2497 (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) {
2498 entry->wired_count++;
2499 }
2500 /*
2501 * Check the map for holes in the specified region.
2502 * If VM_MAP_WIRE_HOLESOK was specified, skip this check.
2503 */
2504 next_entry:
2505 if (((flags & VM_MAP_WIRE_HOLESOK) == 0) &&
2506 (entry->end < end && (entry->next == &map->header ||
2507 entry->next->start > entry->end))) {
2508 end = entry->end;
2509 rv = KERN_INVALID_ADDRESS;
2510 goto done;
2511 }
2512 entry = entry->next;
2513 }
2514 rv = KERN_SUCCESS;
2515 done:
2516 need_wakeup = FALSE;
2517 if (first_entry == NULL) {
2518 result = vm_map_lookup_entry(map, start, &first_entry);
2519 if (!result && (flags & VM_MAP_WIRE_HOLESOK))
2520 first_entry = first_entry->next;
2521 else
2522 KASSERT(result, ("vm_map_wire: lookup failed"));
2523 }
2524 entry = first_entry;
2525 while (entry != &map->header && entry->start < end) {
2526 if ((entry->eflags & MAP_ENTRY_WIRE_SKIPPED) != 0)
2527 goto next_entry_done;
2528 if (rv == KERN_SUCCESS) {
2529 if (user_wire)
2530 entry->eflags |= MAP_ENTRY_USER_WIRED;
2531 } else if (entry->wired_count == -1) {
2532 /*
2533 * Wiring failed on this entry. Thus, unwiring is
2534 * unnecessary.
2535 */
2536 entry->wired_count = 0;
2537 } else {
2538 if (!user_wire ||
2539 (entry->eflags & MAP_ENTRY_USER_WIRED) == 0)
2540 entry->wired_count--;
2541 if (entry->wired_count == 0) {
2542 /*
2543 * Retain the map lock.
2544 */
2545 vm_fault_unwire(map, entry->start, entry->end,
2546 entry->object.vm_object != NULL &&
2547 (entry->object.vm_object->type == OBJT_DEVICE ||
2548 entry->object.vm_object->type == OBJT_SG));
2549 }
2550 }
2551 next_entry_done:
2552 KASSERT(entry->eflags & MAP_ENTRY_IN_TRANSITION,
2553 ("vm_map_wire: in-transition flag missing"));
2554 entry->eflags &= ~(MAP_ENTRY_IN_TRANSITION|MAP_ENTRY_WIRE_SKIPPED);
2555 if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) {
2556 entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP;
2557 need_wakeup = TRUE;
2558 }
2559 vm_map_simplify_entry(map, entry);
2560 entry = entry->next;
2561 }
2562 vm_map_unlock(map);
2563 if (need_wakeup)
2564 vm_map_wakeup(map);
2565 return (rv);
2566 }
2567
2568 /*
2569 * vm_map_sync
2570 *
2571 * Push any dirty cached pages in the address range to their pager.
2572 * If syncio is TRUE, dirty pages are written synchronously.
2573 * If invalidate is TRUE, any cached pages are freed as well.
2574 *
2575 * If the size of the region from start to end is zero, we are
2576 * supposed to flush all modified pages within the region containing
2577 * start. Unfortunately, a region can be split or coalesced with
2578 * neighboring regions, making it difficult to determine what the
2579 * original region was. Therefore, we approximate this requirement by
2580 * flushing the current region containing start.
2581 *
2582 * Returns an error if any part of the specified range is not mapped.
2583 */
2584 int
2585 vm_map_sync(
2586 vm_map_t map,
2587 vm_offset_t start,
2588 vm_offset_t end,
2589 boolean_t syncio,
2590 boolean_t invalidate)
2591 {
2592 vm_map_entry_t current;
2593 vm_map_entry_t entry;
2594 vm_size_t size;
2595 vm_object_t object;
2596 vm_ooffset_t offset;
2597 unsigned int last_timestamp;
2598 boolean_t failed;
2599
2600 vm_map_lock_read(map);
2601 VM_MAP_RANGE_CHECK(map, start, end);
2602 if (!vm_map_lookup_entry(map, start, &entry)) {
2603 vm_map_unlock_read(map);
2604 return (KERN_INVALID_ADDRESS);
2605 } else if (start == end) {
2606 start = entry->start;
2607 end = entry->end;
2608 }
2609 /*
2610 * Make a first pass to check for user-wired memory and holes.
2611 */
2612 for (current = entry; current != &map->header && current->start < end;
2613 current = current->next) {
2614 if (invalidate && (current->eflags & MAP_ENTRY_USER_WIRED)) {
2615 vm_map_unlock_read(map);
2616 return (KERN_INVALID_ARGUMENT);
2617 }
2618 if (end > current->end &&
2619 (current->next == &map->header ||
2620 current->end != current->next->start)) {
2621 vm_map_unlock_read(map);
2622 return (KERN_INVALID_ADDRESS);
2623 }
2624 }
2625
2626 if (invalidate)
2627 pmap_remove(map->pmap, start, end);
2628 failed = FALSE;
2629
2630 /*
2631 * Make a second pass, cleaning/uncaching pages from the indicated
2632 * objects as we go.
2633 */
2634 for (current = entry; current != &map->header && current->start < end;) {
2635 offset = current->offset + (start - current->start);
2636 size = (end <= current->end ? end : current->end) - start;
2637 if (current->eflags & MAP_ENTRY_IS_SUB_MAP) {
2638 vm_map_t smap;
2639 vm_map_entry_t tentry;
2640 vm_size_t tsize;
2641
2642 smap = current->object.sub_map;
2643 vm_map_lock_read(smap);
2644 (void) vm_map_lookup_entry(smap, offset, &tentry);
2645 tsize = tentry->end - offset;
2646 if (tsize < size)
2647 size = tsize;
2648 object = tentry->object.vm_object;
2649 offset = tentry->offset + (offset - tentry->start);
2650 vm_map_unlock_read(smap);
2651 } else {
2652 object = current->object.vm_object;
2653 }
2654 vm_object_reference(object);
2655 last_timestamp = map->timestamp;
2656 vm_map_unlock_read(map);
2657 if (!vm_object_sync(object, offset, size, syncio, invalidate))
2658 failed = TRUE;
2659 start += size;
2660 vm_object_deallocate(object);
2661 vm_map_lock_read(map);
2662 if (last_timestamp == map->timestamp ||
2663 !vm_map_lookup_entry(map, start, ¤t))
2664 current = current->next;
2665 }
2666
2667 vm_map_unlock_read(map);
2668 return (failed ? KERN_FAILURE : KERN_SUCCESS);
2669 }
2670
2671 /*
2672 * vm_map_entry_unwire: [ internal use only ]
2673 *
2674 * Make the region specified by this entry pageable.
2675 *
2676 * The map in question should be locked.
2677 * [This is the reason for this routine's existence.]
2678 */
2679 static void
2680 vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry)
2681 {
2682 vm_fault_unwire(map, entry->start, entry->end,
2683 entry->object.vm_object != NULL &&
2684 (entry->object.vm_object->type == OBJT_DEVICE ||
2685 entry->object.vm_object->type == OBJT_SG));
2686 entry->wired_count = 0;
2687 }
2688
2689 static void
2690 vm_map_entry_deallocate(vm_map_entry_t entry, boolean_t system_map)
2691 {
2692
2693 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0)
2694 vm_object_deallocate(entry->object.vm_object);
2695 uma_zfree(system_map ? kmapentzone : mapentzone, entry);
2696 }
2697
2698 /*
2699 * vm_map_entry_delete: [ internal use only ]
2700 *
2701 * Deallocate the given entry from the target map.
2702 */
2703 static void
2704 vm_map_entry_delete(vm_map_t map, vm_map_entry_t entry)
2705 {
2706 vm_object_t object;
2707 vm_pindex_t offidxstart, offidxend, count, size1;
2708 vm_ooffset_t size;
2709
2710 vm_map_entry_unlink(map, entry);
2711 object = entry->object.vm_object;
2712 size = entry->end - entry->start;
2713 map->size -= size;
2714
2715 if (entry->cred != NULL) {
2716 swap_release_by_cred(size, entry->cred);
2717 crfree(entry->cred);
2718 }
2719
2720 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
2721 (object != NULL)) {
2722 KASSERT(entry->cred == NULL || object->cred == NULL ||
2723 (entry->eflags & MAP_ENTRY_NEEDS_COPY),
2724 ("OVERCOMMIT vm_map_entry_delete: both cred %p", entry));
2725 count = OFF_TO_IDX(size);
2726 offidxstart = OFF_TO_IDX(entry->offset);
2727 offidxend = offidxstart + count;
2728 VM_OBJECT_LOCK(object);
2729 if (object->ref_count != 1 &&
2730 ((object->flags & (OBJ_NOSPLIT|OBJ_ONEMAPPING)) == OBJ_ONEMAPPING ||
2731 object == kernel_object || object == kmem_object)) {
2732 vm_object_collapse(object);
2733
2734 /*
2735 * The option OBJPR_NOTMAPPED can be passed here
2736 * because vm_map_delete() already performed
2737 * pmap_remove() on the only mapping to this range
2738 * of pages.
2739 */
2740 vm_object_page_remove(object, offidxstart, offidxend,
2741 OBJPR_NOTMAPPED);
2742 if (object->type == OBJT_SWAP)
2743 swap_pager_freespace(object, offidxstart, count);
2744 if (offidxend >= object->size &&
2745 offidxstart < object->size) {
2746 size1 = object->size;
2747 object->size = offidxstart;
2748 if (object->cred != NULL) {
2749 size1 -= object->size;
2750 KASSERT(object->charge >= ptoa(size1),
2751 ("vm_map_entry_delete: object->charge < 0"));
2752 swap_release_by_cred(ptoa(size1), object->cred);
2753 object->charge -= ptoa(size1);
2754 }
2755 }
2756 }
2757 VM_OBJECT_UNLOCK(object);
2758 } else
2759 entry->object.vm_object = NULL;
2760 if (map->system_map)
2761 vm_map_entry_deallocate(entry, TRUE);
2762 else {
2763 entry->next = curthread->td_map_def_user;
2764 curthread->td_map_def_user = entry;
2765 }
2766 }
2767
2768 /*
2769 * vm_map_delete: [ internal use only ]
2770 *
2771 * Deallocates the given address range from the target
2772 * map.
2773 */
2774 int
2775 vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end)
2776 {
2777 vm_map_entry_t entry;
2778 vm_map_entry_t first_entry;
2779
2780 VM_MAP_ASSERT_LOCKED(map);
2781
2782 /*
2783 * Find the start of the region, and clip it
2784 */
2785 if (!vm_map_lookup_entry(map, start, &first_entry))
2786 entry = first_entry->next;
2787 else {
2788 entry = first_entry;
2789 vm_map_clip_start(map, entry, start);
2790 }
2791
2792 /*
2793 * Step through all entries in this region
2794 */
2795 while ((entry != &map->header) && (entry->start < end)) {
2796 vm_map_entry_t next;
2797
2798 /*
2799 * Wait for wiring or unwiring of an entry to complete.
2800 * Also wait for any system wirings to disappear on
2801 * user maps.
2802 */
2803 if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0 ||
2804 (vm_map_pmap(map) != kernel_pmap &&
2805 vm_map_entry_system_wired_count(entry) != 0)) {
2806 unsigned int last_timestamp;
2807 vm_offset_t saved_start;
2808 vm_map_entry_t tmp_entry;
2809
2810 saved_start = entry->start;
2811 entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
2812 last_timestamp = map->timestamp;
2813 (void) vm_map_unlock_and_wait(map, 0);
2814 vm_map_lock(map);
2815 if (last_timestamp + 1 != map->timestamp) {
2816 /*
2817 * Look again for the entry because the map was
2818 * modified while it was unlocked.
2819 * Specifically, the entry may have been
2820 * clipped, merged, or deleted.
2821 */
2822 if (!vm_map_lookup_entry(map, saved_start,
2823 &tmp_entry))
2824 entry = tmp_entry->next;
2825 else {
2826 entry = tmp_entry;
2827 vm_map_clip_start(map, entry,
2828 saved_start);
2829 }
2830 }
2831 continue;
2832 }
2833 vm_map_clip_end(map, entry, end);
2834
2835 next = entry->next;
2836
2837 /*
2838 * Unwire before removing addresses from the pmap; otherwise,
2839 * unwiring will put the entries back in the pmap.
2840 */
2841 if (entry->wired_count != 0) {
2842 vm_map_entry_unwire(map, entry);
2843 }
2844
2845 pmap_remove(map->pmap, entry->start, entry->end);
2846
2847 /*
2848 * Delete the entry only after removing all pmap
2849 * entries pointing to its pages. (Otherwise, its
2850 * page frames may be reallocated, and any modify bits
2851 * will be set in the wrong object!)
2852 */
2853 vm_map_entry_delete(map, entry);
2854 entry = next;
2855 }
2856 return (KERN_SUCCESS);
2857 }
2858
2859 /*
2860 * vm_map_remove:
2861 *
2862 * Remove the given address range from the target map.
2863 * This is the exported form of vm_map_delete.
2864 */
2865 int
2866 vm_map_remove(vm_map_t map, vm_offset_t start, vm_offset_t end)
2867 {
2868 int result;
2869
2870 vm_map_lock(map);
2871 VM_MAP_RANGE_CHECK(map, start, end);
2872 result = vm_map_delete(map, start, end);
2873 vm_map_unlock(map);
2874 return (result);
2875 }
2876
2877 /*
2878 * vm_map_check_protection:
2879 *
2880 * Assert that the target map allows the specified privilege on the
2881 * entire address region given. The entire region must be allocated.
2882 *
2883 * WARNING! This code does not and should not check whether the
2884 * contents of the region is accessible. For example a smaller file
2885 * might be mapped into a larger address space.
2886 *
2887 * NOTE! This code is also called by munmap().
2888 *
2889 * The map must be locked. A read lock is sufficient.
2890 */
2891 boolean_t
2892 vm_map_check_protection(vm_map_t map, vm_offset_t start, vm_offset_t end,
2893 vm_prot_t protection)
2894 {
2895 vm_map_entry_t entry;
2896 vm_map_entry_t tmp_entry;
2897
2898 if (!vm_map_lookup_entry(map, start, &tmp_entry))
2899 return (FALSE);
2900 entry = tmp_entry;
2901
2902 while (start < end) {
2903 if (entry == &map->header)
2904 return (FALSE);
2905 /*
2906 * No holes allowed!
2907 */
2908 if (start < entry->start)
2909 return (FALSE);
2910 /*
2911 * Check protection associated with entry.
2912 */
2913 if ((entry->protection & protection) != protection)
2914 return (FALSE);
2915 /* go to next entry */
2916 start = entry->end;
2917 entry = entry->next;
2918 }
2919 return (TRUE);
2920 }
2921
2922 /*
2923 * vm_map_copy_entry:
2924 *
2925 * Copies the contents of the source entry to the destination
2926 * entry. The entries *must* be aligned properly.
2927 */
2928 static void
2929 vm_map_copy_entry(
2930 vm_map_t src_map,
2931 vm_map_t dst_map,
2932 vm_map_entry_t src_entry,
2933 vm_map_entry_t dst_entry,
2934 vm_ooffset_t *fork_charge)
2935 {
2936 vm_object_t src_object;
2937 vm_map_entry_t fake_entry;
2938 vm_offset_t size;
2939 struct ucred *cred;
2940 int charged;
2941
2942 VM_MAP_ASSERT_LOCKED(dst_map);
2943
2944 if ((dst_entry->eflags|src_entry->eflags) & MAP_ENTRY_IS_SUB_MAP)
2945 return;
2946
2947 if (src_entry->wired_count == 0) {
2948
2949 /*
2950 * If the source entry is marked needs_copy, it is already
2951 * write-protected.
2952 */
2953 if ((src_entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) {
2954 pmap_protect(src_map->pmap,
2955 src_entry->start,
2956 src_entry->end,
2957 src_entry->protection & ~VM_PROT_WRITE);
2958 }
2959
2960 /*
2961 * Make a copy of the object.
2962 */
2963 size = src_entry->end - src_entry->start;
2964 if ((src_object = src_entry->object.vm_object) != NULL) {
2965 VM_OBJECT_LOCK(src_object);
2966 charged = ENTRY_CHARGED(src_entry);
2967 if ((src_object->handle == NULL) &&
2968 (src_object->type == OBJT_DEFAULT ||
2969 src_object->type == OBJT_SWAP)) {
2970 vm_object_collapse(src_object);
2971 if ((src_object->flags & (OBJ_NOSPLIT|OBJ_ONEMAPPING)) == OBJ_ONEMAPPING) {
2972 vm_object_split(src_entry);
2973 src_object = src_entry->object.vm_object;
2974 }
2975 }
2976 vm_object_reference_locked(src_object);
2977 vm_object_clear_flag(src_object, OBJ_ONEMAPPING);
2978 if (src_entry->cred != NULL &&
2979 !(src_entry->eflags & MAP_ENTRY_NEEDS_COPY)) {
2980 KASSERT(src_object->cred == NULL,
2981 ("OVERCOMMIT: vm_map_copy_entry: cred %p",
2982 src_object));
2983 src_object->cred = src_entry->cred;
2984 src_object->charge = size;
2985 }
2986 VM_OBJECT_UNLOCK(src_object);
2987 dst_entry->object.vm_object = src_object;
2988 if (charged) {
2989 cred = curthread->td_ucred;
2990 crhold(cred);
2991 dst_entry->cred = cred;
2992 *fork_charge += size;
2993 if (!(src_entry->eflags &
2994 MAP_ENTRY_NEEDS_COPY)) {
2995 crhold(cred);
2996 src_entry->cred = cred;
2997 *fork_charge += size;
2998 }
2999 }
3000 src_entry->eflags |= (MAP_ENTRY_COW|MAP_ENTRY_NEEDS_COPY);
3001 dst_entry->eflags |= (MAP_ENTRY_COW|MAP_ENTRY_NEEDS_COPY);
3002 dst_entry->offset = src_entry->offset;
3003 if (src_entry->eflags & MAP_ENTRY_VN_WRITECNT) {
3004 /*
3005 * MAP_ENTRY_VN_WRITECNT cannot
3006 * indicate write reference from
3007 * src_entry, since the entry is
3008 * marked as needs copy. Allocate a
3009 * fake entry that is used to
3010 * decrement object->un_pager.vnp.writecount
3011 * at the appropriate time. Attach
3012 * fake_entry to the deferred list.
3013 */
3014 fake_entry = vm_map_entry_create(dst_map);
3015 fake_entry->eflags = MAP_ENTRY_VN_WRITECNT;
3016 src_entry->eflags &= ~MAP_ENTRY_VN_WRITECNT;
3017 vm_object_reference(src_object);
3018 fake_entry->object.vm_object = src_object;
3019 fake_entry->start = src_entry->start;
3020 fake_entry->end = src_entry->end;
3021 fake_entry->next = curthread->td_map_def_user;
3022 curthread->td_map_def_user = fake_entry;
3023 }
3024 } else {
3025 dst_entry->object.vm_object = NULL;
3026 dst_entry->offset = 0;
3027 if (src_entry->cred != NULL) {
3028 dst_entry->cred = curthread->td_ucred;
3029 crhold(dst_entry->cred);
3030 *fork_charge += size;
3031 }
3032 }
3033
3034 pmap_copy(dst_map->pmap, src_map->pmap, dst_entry->start,
3035 dst_entry->end - dst_entry->start, src_entry->start);
3036 } else {
3037 /*
3038 * Of course, wired down pages can't be set copy-on-write.
3039 * Cause wired pages to be copied into the new map by
3040 * simulating faults (the new pages are pageable)
3041 */
3042 vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry,
3043 fork_charge);
3044 }
3045 }
3046
3047 /*
3048 * vmspace_map_entry_forked:
3049 * Update the newly-forked vmspace each time a map entry is inherited
3050 * or copied. The values for vm_dsize and vm_tsize are approximate
3051 * (and mostly-obsolete ideas in the face of mmap(2) et al.)
3052 */
3053 static void
3054 vmspace_map_entry_forked(const struct vmspace *vm1, struct vmspace *vm2,
3055 vm_map_entry_t entry)
3056 {
3057 vm_size_t entrysize;
3058 vm_offset_t newend;
3059
3060 entrysize = entry->end - entry->start;
3061 vm2->vm_map.size += entrysize;
3062 if (entry->eflags & (MAP_ENTRY_GROWS_DOWN | MAP_ENTRY_GROWS_UP)) {
3063 vm2->vm_ssize += btoc(entrysize);
3064 } else if (entry->start >= (vm_offset_t)vm1->vm_daddr &&
3065 entry->start < (vm_offset_t)vm1->vm_daddr + ctob(vm1->vm_dsize)) {
3066 newend = MIN(entry->end,
3067 (vm_offset_t)vm1->vm_daddr + ctob(vm1->vm_dsize));
3068 vm2->vm_dsize += btoc(newend - entry->start);
3069 } else if (entry->start >= (vm_offset_t)vm1->vm_taddr &&
3070 entry->start < (vm_offset_t)vm1->vm_taddr + ctob(vm1->vm_tsize)) {
3071 newend = MIN(entry->end,
3072 (vm_offset_t)vm1->vm_taddr + ctob(vm1->vm_tsize));
3073 vm2->vm_tsize += btoc(newend - entry->start);
3074 }
3075 }
3076
3077 /*
3078 * vmspace_fork:
3079 * Create a new process vmspace structure and vm_map
3080 * based on those of an existing process. The new map
3081 * is based on the old map, according to the inheritance
3082 * values on the regions in that map.
3083 *
3084 * XXX It might be worth coalescing the entries added to the new vmspace.
3085 *
3086 * The source map must not be locked.
3087 */
3088 struct vmspace *
3089 vmspace_fork(struct vmspace *vm1, vm_ooffset_t *fork_charge)
3090 {
3091 struct vmspace *vm2;
3092 vm_map_t new_map, old_map;
3093 vm_map_entry_t new_entry, old_entry;
3094 vm_object_t object;
3095 int locked;
3096
3097 old_map = &vm1->vm_map;
3098 /* Copy immutable fields of vm1 to vm2. */
3099 vm2 = vmspace_alloc(old_map->min_offset, old_map->max_offset);
3100 if (vm2 == NULL)
3101 return (NULL);
3102 vm2->vm_taddr = vm1->vm_taddr;
3103 vm2->vm_daddr = vm1->vm_daddr;
3104 vm2->vm_maxsaddr = vm1->vm_maxsaddr;
3105 vm_map_lock(old_map);
3106 if (old_map->busy)
3107 vm_map_wait_busy(old_map);
3108 new_map = &vm2->vm_map;
3109 locked = vm_map_trylock(new_map); /* trylock to silence WITNESS */
3110 KASSERT(locked, ("vmspace_fork: lock failed"));
3111
3112 old_entry = old_map->header.next;
3113
3114 while (old_entry != &old_map->header) {
3115 if (old_entry->eflags & MAP_ENTRY_IS_SUB_MAP)
3116 panic("vm_map_fork: encountered a submap");
3117
3118 switch (old_entry->inheritance) {
3119 case VM_INHERIT_NONE:
3120 break;
3121
3122 case VM_INHERIT_SHARE:
3123 /*
3124 * Clone the entry, creating the shared object if necessary.
3125 */
3126 object = old_entry->object.vm_object;
3127 if (object == NULL) {
3128 object = vm_object_allocate(OBJT_DEFAULT,
3129 atop(old_entry->end - old_entry->start));
3130 old_entry->object.vm_object = object;
3131 old_entry->offset = 0;
3132 if (old_entry->cred != NULL) {
3133 object->cred = old_entry->cred;
3134 object->charge = old_entry->end -
3135 old_entry->start;
3136 old_entry->cred = NULL;
3137 }
3138 }
3139
3140 /*
3141 * Add the reference before calling vm_object_shadow
3142 * to insure that a shadow object is created.
3143 */
3144 vm_object_reference(object);
3145 if (old_entry->eflags & MAP_ENTRY_NEEDS_COPY) {
3146 vm_object_shadow(&old_entry->object.vm_object,
3147 &old_entry->offset,
3148 old_entry->end - old_entry->start);
3149 old_entry->eflags &= ~MAP_ENTRY_NEEDS_COPY;
3150 /* Transfer the second reference too. */
3151 vm_object_reference(
3152 old_entry->object.vm_object);
3153
3154 /*
3155 * As in vm_map_simplify_entry(), the
3156 * vnode lock will not be acquired in
3157 * this call to vm_object_deallocate().
3158 */
3159 vm_object_deallocate(object);
3160 object = old_entry->object.vm_object;
3161 }
3162 VM_OBJECT_LOCK(object);
3163 vm_object_clear_flag(object, OBJ_ONEMAPPING);
3164 if (old_entry->cred != NULL) {
3165 KASSERT(object->cred == NULL, ("vmspace_fork both cred"));
3166 object->cred = old_entry->cred;
3167 object->charge = old_entry->end - old_entry->start;
3168 old_entry->cred = NULL;
3169 }
3170 VM_OBJECT_UNLOCK(object);
3171
3172 /*
3173 * Clone the entry, referencing the shared object.
3174 */
3175 new_entry = vm_map_entry_create(new_map);
3176 *new_entry = *old_entry;
3177 new_entry->eflags &= ~(MAP_ENTRY_USER_WIRED |
3178 MAP_ENTRY_IN_TRANSITION);
3179 new_entry->wired_count = 0;
3180 if (new_entry->eflags & MAP_ENTRY_VN_WRITECNT) {
3181 object = new_entry->object.vm_object;
3182 KASSERT(((struct vnode *)object->handle)->
3183 v_writecount > 0,
3184 ("vmspace_fork: v_writecount"));
3185 KASSERT(object->un_pager.vnp.writemappings > 0,
3186 ("vmspace_fork: vnp.writecount"));
3187 vnode_pager_update_writecount(object,
3188 new_entry->start, new_entry->end);
3189 }
3190
3191 /*
3192 * Insert the entry into the new map -- we know we're
3193 * inserting at the end of the new map.
3194 */
3195 vm_map_entry_link(new_map, new_map->header.prev,
3196 new_entry);
3197 vmspace_map_entry_forked(vm1, vm2, new_entry);
3198
3199 /*
3200 * Update the physical map
3201 */
3202 pmap_copy(new_map->pmap, old_map->pmap,
3203 new_entry->start,
3204 (old_entry->end - old_entry->start),
3205 old_entry->start);
3206 break;
3207
3208 case VM_INHERIT_COPY:
3209 /*
3210 * Clone the entry and link into the map.
3211 */
3212 new_entry = vm_map_entry_create(new_map);
3213 *new_entry = *old_entry;
3214 /*
3215 * Copied entry is COW over the old object.
3216 */
3217 new_entry->eflags &= ~(MAP_ENTRY_USER_WIRED |
3218 MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_VN_WRITECNT);
3219 new_entry->wired_count = 0;
3220 new_entry->object.vm_object = NULL;
3221 new_entry->cred = NULL;
3222 vm_map_entry_link(new_map, new_map->header.prev,
3223 new_entry);
3224 vmspace_map_entry_forked(vm1, vm2, new_entry);
3225 vm_map_copy_entry(old_map, new_map, old_entry,
3226 new_entry, fork_charge);
3227 break;
3228 }
3229 old_entry = old_entry->next;
3230 }
3231 /*
3232 * Use inlined vm_map_unlock() to postpone handling the deferred
3233 * map entries, which cannot be done until both old_map and
3234 * new_map locks are released.
3235 */
3236 sx_xunlock(&old_map->lock);
3237 sx_xunlock(&new_map->lock);
3238 vm_map_process_deferred();
3239
3240 return (vm2);
3241 }
3242
3243 int
3244 vm_map_stack(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize,
3245 vm_prot_t prot, vm_prot_t max, int cow)
3246 {
3247 vm_map_entry_t new_entry, prev_entry;
3248 vm_offset_t bot, top;
3249 vm_size_t init_ssize;
3250 int orient, rv;
3251 rlim_t vmemlim;
3252
3253 /*
3254 * The stack orientation is piggybacked with the cow argument.
3255 * Extract it into orient and mask the cow argument so that we
3256 * don't pass it around further.
3257 * NOTE: We explicitly allow bi-directional stacks.
3258 */
3259 orient = cow & (MAP_STACK_GROWS_DOWN|MAP_STACK_GROWS_UP);
3260 KASSERT(orient != 0, ("No stack grow direction"));
3261
3262 if (addrbos < vm_map_min(map) ||
3263 addrbos > vm_map_max(map) ||
3264 addrbos + max_ssize < addrbos)
3265 return (KERN_NO_SPACE);
3266
3267 init_ssize = (max_ssize < sgrowsiz) ? max_ssize : sgrowsiz;
3268
3269 PROC_LOCK(curthread->td_proc);
3270 vmemlim = lim_cur(curthread->td_proc, RLIMIT_VMEM);
3271 PROC_UNLOCK(curthread->td_proc);
3272
3273 vm_map_lock(map);
3274
3275 /* If addr is already mapped, no go */
3276 if (vm_map_lookup_entry(map, addrbos, &prev_entry)) {
3277 vm_map_unlock(map);
3278 return (KERN_NO_SPACE);
3279 }
3280
3281 /* If we would blow our VMEM resource limit, no go */
3282 if (map->size + init_ssize > vmemlim) {
3283 vm_map_unlock(map);
3284 return (KERN_NO_SPACE);
3285 }
3286
3287 /*
3288 * If we can't accomodate max_ssize in the current mapping, no go.
3289 * However, we need to be aware that subsequent user mappings might
3290 * map into the space we have reserved for stack, and currently this
3291 * space is not protected.
3292 *
3293 * Hopefully we will at least detect this condition when we try to
3294 * grow the stack.
3295 */
3296 if ((prev_entry->next != &map->header) &&
3297 (prev_entry->next->start < addrbos + max_ssize)) {
3298 vm_map_unlock(map);
3299 return (KERN_NO_SPACE);
3300 }
3301
3302 /*
3303 * We initially map a stack of only init_ssize. We will grow as
3304 * needed later. Depending on the orientation of the stack (i.e.
3305 * the grow direction) we either map at the top of the range, the
3306 * bottom of the range or in the middle.
3307 *
3308 * Note: we would normally expect prot and max to be VM_PROT_ALL,
3309 * and cow to be 0. Possibly we should eliminate these as input
3310 * parameters, and just pass these values here in the insert call.
3311 */
3312 if (orient == MAP_STACK_GROWS_DOWN)
3313 bot = addrbos + max_ssize - init_ssize;
3314 else if (orient == MAP_STACK_GROWS_UP)
3315 bot = addrbos;
3316 else
3317 bot = round_page(addrbos + max_ssize/2 - init_ssize/2);
3318 top = bot + init_ssize;
3319 rv = vm_map_insert(map, NULL, 0, bot, top, prot, max, cow);
3320
3321 /* Now set the avail_ssize amount. */
3322 if (rv == KERN_SUCCESS) {
3323 if (prev_entry != &map->header)
3324 vm_map_clip_end(map, prev_entry, bot);
3325 new_entry = prev_entry->next;
3326 if (new_entry->end != top || new_entry->start != bot)
3327 panic("Bad entry start/end for new stack entry");
3328
3329 new_entry->avail_ssize = max_ssize - init_ssize;
3330 if (orient & MAP_STACK_GROWS_DOWN)
3331 new_entry->eflags |= MAP_ENTRY_GROWS_DOWN;
3332 if (orient & MAP_STACK_GROWS_UP)
3333 new_entry->eflags |= MAP_ENTRY_GROWS_UP;
3334 }
3335
3336 vm_map_unlock(map);
3337 return (rv);
3338 }
3339
3340 static int stack_guard_page = 0;
3341 TUNABLE_INT("security.bsd.stack_guard_page", &stack_guard_page);
3342 SYSCTL_INT(_security_bsd, OID_AUTO, stack_guard_page, CTLFLAG_RW,
3343 &stack_guard_page, 0,
3344 "Insert stack guard page ahead of the growable segments.");
3345
3346 /* Attempts to grow a vm stack entry. Returns KERN_SUCCESS if the
3347 * desired address is already mapped, or if we successfully grow
3348 * the stack. Also returns KERN_SUCCESS if addr is outside the
3349 * stack range (this is strange, but preserves compatibility with
3350 * the grow function in vm_machdep.c).
3351 */
3352 int
3353 vm_map_growstack(struct proc *p, vm_offset_t addr)
3354 {
3355 vm_map_entry_t next_entry, prev_entry;
3356 vm_map_entry_t new_entry, stack_entry;
3357 struct vmspace *vm = p->p_vmspace;
3358 vm_map_t map = &vm->vm_map;
3359 vm_offset_t end;
3360 size_t grow_amount, max_grow;
3361 rlim_t stacklim, vmemlim;
3362 int is_procstack, rv;
3363 struct ucred *cred;
3364 #ifdef notyet
3365 uint64_t limit;
3366 #endif
3367 #ifdef RACCT
3368 int error;
3369 #endif
3370
3371 Retry:
3372 PROC_LOCK(p);
3373 stacklim = lim_cur(p, RLIMIT_STACK);
3374 vmemlim = lim_cur(p, RLIMIT_VMEM);
3375 PROC_UNLOCK(p);
3376
3377 vm_map_lock_read(map);
3378
3379 /* If addr is already in the entry range, no need to grow.*/
3380 if (vm_map_lookup_entry(map, addr, &prev_entry)) {
3381 vm_map_unlock_read(map);
3382 return (KERN_SUCCESS);
3383 }
3384
3385 next_entry = prev_entry->next;
3386 if (!(prev_entry->eflags & MAP_ENTRY_GROWS_UP)) {
3387 /*
3388 * This entry does not grow upwards. Since the address lies
3389 * beyond this entry, the next entry (if one exists) has to
3390 * be a downward growable entry. The entry list header is
3391 * never a growable entry, so it suffices to check the flags.
3392 */
3393 if (!(next_entry->eflags & MAP_ENTRY_GROWS_DOWN)) {
3394 vm_map_unlock_read(map);
3395 return (KERN_SUCCESS);
3396 }
3397 stack_entry = next_entry;
3398 } else {
3399 /*
3400 * This entry grows upward. If the next entry does not at
3401 * least grow downwards, this is the entry we need to grow.
3402 * otherwise we have two possible choices and we have to
3403 * select one.
3404 */
3405 if (next_entry->eflags & MAP_ENTRY_GROWS_DOWN) {
3406 /*
3407 * We have two choices; grow the entry closest to
3408 * the address to minimize the amount of growth.
3409 */
3410 if (addr - prev_entry->end <= next_entry->start - addr)
3411 stack_entry = prev_entry;
3412 else
3413 stack_entry = next_entry;
3414 } else
3415 stack_entry = prev_entry;
3416 }
3417
3418 if (stack_entry == next_entry) {
3419 KASSERT(stack_entry->eflags & MAP_ENTRY_GROWS_DOWN, ("foo"));
3420 KASSERT(addr < stack_entry->start, ("foo"));
3421 end = (prev_entry != &map->header) ? prev_entry->end :
3422 stack_entry->start - stack_entry->avail_ssize;
3423 grow_amount = roundup(stack_entry->start - addr, PAGE_SIZE);
3424 max_grow = stack_entry->start - end;
3425 } else {
3426 KASSERT(stack_entry->eflags & MAP_ENTRY_GROWS_UP, ("foo"));
3427 KASSERT(addr >= stack_entry->end, ("foo"));
3428 end = (next_entry != &map->header) ? next_entry->start :
3429 stack_entry->end + stack_entry->avail_ssize;
3430 grow_amount = roundup(addr + 1 - stack_entry->end, PAGE_SIZE);
3431 max_grow = end - stack_entry->end;
3432 }
3433
3434 if (grow_amount > stack_entry->avail_ssize) {
3435 vm_map_unlock_read(map);
3436 return (KERN_NO_SPACE);
3437 }
3438
3439 /*
3440 * If there is no longer enough space between the entries nogo, and
3441 * adjust the available space. Note: this should only happen if the
3442 * user has mapped into the stack area after the stack was created,
3443 * and is probably an error.
3444 *
3445 * This also effectively destroys any guard page the user might have
3446 * intended by limiting the stack size.
3447 */
3448 if (grow_amount + (stack_guard_page ? PAGE_SIZE : 0) > max_grow) {
3449 if (vm_map_lock_upgrade(map))
3450 goto Retry;
3451
3452 stack_entry->avail_ssize = max_grow;
3453
3454 vm_map_unlock(map);
3455 return (KERN_NO_SPACE);
3456 }
3457
3458 is_procstack = (addr >= (vm_offset_t)vm->vm_maxsaddr) ? 1 : 0;
3459
3460 /*
3461 * If this is the main process stack, see if we're over the stack
3462 * limit.
3463 */
3464 if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > stacklim)) {
3465 vm_map_unlock_read(map);
3466 return (KERN_NO_SPACE);
3467 }
3468 #ifdef RACCT
3469 PROC_LOCK(p);
3470 if (is_procstack &&
3471 racct_set(p, RACCT_STACK, ctob(vm->vm_ssize) + grow_amount)) {
3472 PROC_UNLOCK(p);
3473 vm_map_unlock_read(map);
3474 return (KERN_NO_SPACE);
3475 }
3476 PROC_UNLOCK(p);
3477 #endif
3478
3479 /* Round up the grow amount modulo SGROWSIZ */
3480 grow_amount = roundup (grow_amount, sgrowsiz);
3481 if (grow_amount > stack_entry->avail_ssize)
3482 grow_amount = stack_entry->avail_ssize;
3483 if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > stacklim)) {
3484 grow_amount = trunc_page((vm_size_t)stacklim) -
3485 ctob(vm->vm_ssize);
3486 }
3487 #ifdef notyet
3488 PROC_LOCK(p);
3489 limit = racct_get_available(p, RACCT_STACK);
3490 PROC_UNLOCK(p);
3491 if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > limit))
3492 grow_amount = limit - ctob(vm->vm_ssize);
3493 #endif
3494
3495 /* If we would blow our VMEM resource limit, no go */
3496 if (map->size + grow_amount > vmemlim) {
3497 vm_map_unlock_read(map);
3498 rv = KERN_NO_SPACE;
3499 goto out;
3500 }
3501 #ifdef RACCT
3502 PROC_LOCK(p);
3503 if (racct_set(p, RACCT_VMEM, map->size + grow_amount)) {
3504 PROC_UNLOCK(p);
3505 vm_map_unlock_read(map);
3506 rv = KERN_NO_SPACE;
3507 goto out;
3508 }
3509 PROC_UNLOCK(p);
3510 #endif
3511
3512 if (vm_map_lock_upgrade(map))
3513 goto Retry;
3514
3515 if (stack_entry == next_entry) {
3516 /*
3517 * Growing downward.
3518 */
3519 /* Get the preliminary new entry start value */
3520 addr = stack_entry->start - grow_amount;
3521
3522 /*
3523 * If this puts us into the previous entry, cut back our
3524 * growth to the available space. Also, see the note above.
3525 */
3526 if (addr < end) {
3527 stack_entry->avail_ssize = max_grow;
3528 addr = end;
3529 if (stack_guard_page)
3530 addr += PAGE_SIZE;
3531 }
3532
3533 rv = vm_map_insert(map, NULL, 0, addr, stack_entry->start,
3534 next_entry->protection, next_entry->max_protection, 0);
3535
3536 /* Adjust the available stack space by the amount we grew. */
3537 if (rv == KERN_SUCCESS) {
3538 if (prev_entry != &map->header)
3539 vm_map_clip_end(map, prev_entry, addr);
3540 new_entry = prev_entry->next;
3541 KASSERT(new_entry == stack_entry->prev, ("foo"));
3542 KASSERT(new_entry->end == stack_entry->start, ("foo"));
3543 KASSERT(new_entry->start == addr, ("foo"));
3544 grow_amount = new_entry->end - new_entry->start;
3545 new_entry->avail_ssize = stack_entry->avail_ssize -
3546 grow_amount;
3547 stack_entry->eflags &= ~MAP_ENTRY_GROWS_DOWN;
3548 new_entry->eflags |= MAP_ENTRY_GROWS_DOWN;
3549 }
3550 } else {
3551 /*
3552 * Growing upward.
3553 */
3554 addr = stack_entry->end + grow_amount;
3555
3556 /*
3557 * If this puts us into the next entry, cut back our growth
3558 * to the available space. Also, see the note above.
3559 */
3560 if (addr > end) {
3561 stack_entry->avail_ssize = end - stack_entry->end;
3562 addr = end;
3563 if (stack_guard_page)
3564 addr -= PAGE_SIZE;
3565 }
3566
3567 grow_amount = addr - stack_entry->end;
3568 cred = stack_entry->cred;
3569 if (cred == NULL && stack_entry->object.vm_object != NULL)
3570 cred = stack_entry->object.vm_object->cred;
3571 if (cred != NULL && !swap_reserve_by_cred(grow_amount, cred))
3572 rv = KERN_NO_SPACE;
3573 /* Grow the underlying object if applicable. */
3574 else if (stack_entry->object.vm_object == NULL ||
3575 vm_object_coalesce(stack_entry->object.vm_object,
3576 stack_entry->offset,
3577 (vm_size_t)(stack_entry->end - stack_entry->start),
3578 (vm_size_t)grow_amount, cred != NULL)) {
3579 map->size += (addr - stack_entry->end);
3580 /* Update the current entry. */
3581 stack_entry->end = addr;
3582 stack_entry->avail_ssize -= grow_amount;
3583 vm_map_entry_resize_free(map, stack_entry);
3584 rv = KERN_SUCCESS;
3585
3586 if (next_entry != &map->header)
3587 vm_map_clip_start(map, next_entry, addr);
3588 } else
3589 rv = KERN_FAILURE;
3590 }
3591
3592 if (rv == KERN_SUCCESS && is_procstack)
3593 vm->vm_ssize += btoc(grow_amount);
3594
3595 vm_map_unlock(map);
3596
3597 /*
3598 * Heed the MAP_WIREFUTURE flag if it was set for this process.
3599 */
3600 if (rv == KERN_SUCCESS && (map->flags & MAP_WIREFUTURE)) {
3601 vm_map_wire(map,
3602 (stack_entry == next_entry) ? addr : addr - grow_amount,
3603 (stack_entry == next_entry) ? stack_entry->start : addr,
3604 (p->p_flag & P_SYSTEM)
3605 ? VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES
3606 : VM_MAP_WIRE_USER|VM_MAP_WIRE_NOHOLES);
3607 }
3608
3609 out:
3610 #ifdef RACCT
3611 if (rv != KERN_SUCCESS) {
3612 PROC_LOCK(p);
3613 error = racct_set(p, RACCT_VMEM, map->size);
3614 KASSERT(error == 0, ("decreasing RACCT_VMEM failed"));
3615 error = racct_set(p, RACCT_STACK, ctob(vm->vm_ssize));
3616 KASSERT(error == 0, ("decreasing RACCT_STACK failed"));
3617 PROC_UNLOCK(p);
3618 }
3619 #endif
3620
3621 return (rv);
3622 }
3623
3624 /*
3625 * Unshare the specified VM space for exec. If other processes are
3626 * mapped to it, then create a new one. The new vmspace is null.
3627 */
3628 int
3629 vmspace_exec(struct proc *p, vm_offset_t minuser, vm_offset_t maxuser)
3630 {
3631 struct vmspace *oldvmspace = p->p_vmspace;
3632 struct vmspace *newvmspace;
3633
3634 KASSERT((curthread->td_pflags & TDP_EXECVMSPC) == 0,
3635 ("vmspace_exec recursed"));
3636 newvmspace = vmspace_alloc(minuser, maxuser);
3637 if (newvmspace == NULL)
3638 return (ENOMEM);
3639 newvmspace->vm_swrss = oldvmspace->vm_swrss;
3640 /*
3641 * This code is written like this for prototype purposes. The
3642 * goal is to avoid running down the vmspace here, but let the
3643 * other process's that are still using the vmspace to finally
3644 * run it down. Even though there is little or no chance of blocking
3645 * here, it is a good idea to keep this form for future mods.
3646 */
3647 PROC_VMSPACE_LOCK(p);
3648 p->p_vmspace = newvmspace;
3649 PROC_VMSPACE_UNLOCK(p);
3650 if (p == curthread->td_proc)
3651 pmap_activate(curthread);
3652 curthread->td_pflags |= TDP_EXECVMSPC;
3653 return (0);
3654 }
3655
3656 /*
3657 * Unshare the specified VM space for forcing COW. This
3658 * is called by rfork, for the (RFMEM|RFPROC) == 0 case.
3659 */
3660 int
3661 vmspace_unshare(struct proc *p)
3662 {
3663 struct vmspace *oldvmspace = p->p_vmspace;
3664 struct vmspace *newvmspace;
3665 vm_ooffset_t fork_charge;
3666
3667 if (oldvmspace->vm_refcnt == 1)
3668 return (0);
3669 fork_charge = 0;
3670 newvmspace = vmspace_fork(oldvmspace, &fork_charge);
3671 if (newvmspace == NULL)
3672 return (ENOMEM);
3673 if (!swap_reserve_by_cred(fork_charge, p->p_ucred)) {
3674 vmspace_free(newvmspace);
3675 return (ENOMEM);
3676 }
3677 PROC_VMSPACE_LOCK(p);
3678 p->p_vmspace = newvmspace;
3679 PROC_VMSPACE_UNLOCK(p);
3680 if (p == curthread->td_proc)
3681 pmap_activate(curthread);
3682 vmspace_free(oldvmspace);
3683 return (0);
3684 }
3685
3686 /*
3687 * vm_map_lookup:
3688 *
3689 * Finds the VM object, offset, and
3690 * protection for a given virtual address in the
3691 * specified map, assuming a page fault of the
3692 * type specified.
3693 *
3694 * Leaves the map in question locked for read; return
3695 * values are guaranteed until a vm_map_lookup_done
3696 * call is performed. Note that the map argument
3697 * is in/out; the returned map must be used in
3698 * the call to vm_map_lookup_done.
3699 *
3700 * A handle (out_entry) is returned for use in
3701 * vm_map_lookup_done, to make that fast.
3702 *
3703 * If a lookup is requested with "write protection"
3704 * specified, the map may be changed to perform virtual
3705 * copying operations, although the data referenced will
3706 * remain the same.
3707 */
3708 int
3709 vm_map_lookup(vm_map_t *var_map, /* IN/OUT */
3710 vm_offset_t vaddr,
3711 vm_prot_t fault_typea,
3712 vm_map_entry_t *out_entry, /* OUT */
3713 vm_object_t *object, /* OUT */
3714 vm_pindex_t *pindex, /* OUT */
3715 vm_prot_t *out_prot, /* OUT */
3716 boolean_t *wired) /* OUT */
3717 {
3718 vm_map_entry_t entry;
3719 vm_map_t map = *var_map;
3720 vm_prot_t prot;
3721 vm_prot_t fault_type = fault_typea;
3722 vm_object_t eobject;
3723 vm_size_t size;
3724 struct ucred *cred;
3725
3726 RetryLookup:;
3727
3728 vm_map_lock_read(map);
3729
3730 /*
3731 * Lookup the faulting address.
3732 */
3733 if (!vm_map_lookup_entry(map, vaddr, out_entry)) {
3734 vm_map_unlock_read(map);
3735 return (KERN_INVALID_ADDRESS);
3736 }
3737
3738 entry = *out_entry;
3739
3740 /*
3741 * Handle submaps.
3742 */
3743 if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
3744 vm_map_t old_map = map;
3745
3746 *var_map = map = entry->object.sub_map;
3747 vm_map_unlock_read(old_map);
3748 goto RetryLookup;
3749 }
3750
3751 /*
3752 * Check whether this task is allowed to have this page.
3753 */
3754 prot = entry->protection;
3755 fault_type &= (VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE);
3756 if ((fault_type & prot) != fault_type || prot == VM_PROT_NONE) {
3757 vm_map_unlock_read(map);
3758 return (KERN_PROTECTION_FAILURE);
3759 }
3760 if ((entry->eflags & MAP_ENTRY_USER_WIRED) &&
3761 (entry->eflags & MAP_ENTRY_COW) &&
3762 (fault_type & VM_PROT_WRITE)) {
3763 vm_map_unlock_read(map);
3764 return (KERN_PROTECTION_FAILURE);
3765 }
3766 if ((fault_typea & VM_PROT_COPY) != 0 &&
3767 (entry->max_protection & VM_PROT_WRITE) == 0 &&
3768 (entry->eflags & MAP_ENTRY_COW) == 0) {
3769 vm_map_unlock_read(map);
3770 return (KERN_PROTECTION_FAILURE);
3771 }
3772
3773 /*
3774 * If this page is not pageable, we have to get it for all possible
3775 * accesses.
3776 */
3777 *wired = (entry->wired_count != 0);
3778 if (*wired)
3779 fault_type = entry->protection;
3780 size = entry->end - entry->start;
3781 /*
3782 * If the entry was copy-on-write, we either ...
3783 */
3784 if (entry->eflags & MAP_ENTRY_NEEDS_COPY) {
3785 /*
3786 * If we want to write the page, we may as well handle that
3787 * now since we've got the map locked.
3788 *
3789 * If we don't need to write the page, we just demote the
3790 * permissions allowed.
3791 */
3792 if ((fault_type & VM_PROT_WRITE) != 0 ||
3793 (fault_typea & VM_PROT_COPY) != 0) {
3794 /*
3795 * Make a new object, and place it in the object
3796 * chain. Note that no new references have appeared
3797 * -- one just moved from the map to the new
3798 * object.
3799 */
3800 if (vm_map_lock_upgrade(map))
3801 goto RetryLookup;
3802
3803 if (entry->cred == NULL) {
3804 /*
3805 * The debugger owner is charged for
3806 * the memory.
3807 */
3808 cred = curthread->td_ucred;
3809 crhold(cred);
3810 if (!swap_reserve_by_cred(size, cred)) {
3811 crfree(cred);
3812 vm_map_unlock(map);
3813 return (KERN_RESOURCE_SHORTAGE);
3814 }
3815 entry->cred = cred;
3816 }
3817 vm_object_shadow(&entry->object.vm_object,
3818 &entry->offset, size);
3819 entry->eflags &= ~MAP_ENTRY_NEEDS_COPY;
3820 eobject = entry->object.vm_object;
3821 if (eobject->cred != NULL) {
3822 /*
3823 * The object was not shadowed.
3824 */
3825 swap_release_by_cred(size, entry->cred);
3826 crfree(entry->cred);
3827 entry->cred = NULL;
3828 } else if (entry->cred != NULL) {
3829 VM_OBJECT_LOCK(eobject);
3830 eobject->cred = entry->cred;
3831 eobject->charge = size;
3832 VM_OBJECT_UNLOCK(eobject);
3833 entry->cred = NULL;
3834 }
3835
3836 vm_map_lock_downgrade(map);
3837 } else {
3838 /*
3839 * We're attempting to read a copy-on-write page --
3840 * don't allow writes.
3841 */
3842 prot &= ~VM_PROT_WRITE;
3843 }
3844 }
3845
3846 /*
3847 * Create an object if necessary.
3848 */
3849 if (entry->object.vm_object == NULL &&
3850 !map->system_map) {
3851 if (vm_map_lock_upgrade(map))
3852 goto RetryLookup;
3853 entry->object.vm_object = vm_object_allocate(OBJT_DEFAULT,
3854 atop(size));
3855 entry->offset = 0;
3856 if (entry->cred != NULL) {
3857 VM_OBJECT_LOCK(entry->object.vm_object);
3858 entry->object.vm_object->cred = entry->cred;
3859 entry->object.vm_object->charge = size;
3860 VM_OBJECT_UNLOCK(entry->object.vm_object);
3861 entry->cred = NULL;
3862 }
3863 vm_map_lock_downgrade(map);
3864 }
3865
3866 /*
3867 * Return the object/offset from this entry. If the entry was
3868 * copy-on-write or empty, it has been fixed up.
3869 */
3870 *pindex = OFF_TO_IDX((vaddr - entry->start) + entry->offset);
3871 *object = entry->object.vm_object;
3872
3873 *out_prot = prot;
3874 return (KERN_SUCCESS);
3875 }
3876
3877 /*
3878 * vm_map_lookup_locked:
3879 *
3880 * Lookup the faulting address. A version of vm_map_lookup that returns
3881 * KERN_FAILURE instead of blocking on map lock or memory allocation.
3882 */
3883 int
3884 vm_map_lookup_locked(vm_map_t *var_map, /* IN/OUT */
3885 vm_offset_t vaddr,
3886 vm_prot_t fault_typea,
3887 vm_map_entry_t *out_entry, /* OUT */
3888 vm_object_t *object, /* OUT */
3889 vm_pindex_t *pindex, /* OUT */
3890 vm_prot_t *out_prot, /* OUT */
3891 boolean_t *wired) /* OUT */
3892 {
3893 vm_map_entry_t entry;
3894 vm_map_t map = *var_map;
3895 vm_prot_t prot;
3896 vm_prot_t fault_type = fault_typea;
3897
3898 /*
3899 * Lookup the faulting address.
3900 */
3901 if (!vm_map_lookup_entry(map, vaddr, out_entry))
3902 return (KERN_INVALID_ADDRESS);
3903
3904 entry = *out_entry;
3905
3906 /*
3907 * Fail if the entry refers to a submap.
3908 */
3909 if (entry->eflags & MAP_ENTRY_IS_SUB_MAP)
3910 return (KERN_FAILURE);
3911
3912 /*
3913 * Check whether this task is allowed to have this page.
3914 */
3915 prot = entry->protection;
3916 fault_type &= VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE;
3917 if ((fault_type & prot) != fault_type)
3918 return (KERN_PROTECTION_FAILURE);
3919 if ((entry->eflags & MAP_ENTRY_USER_WIRED) &&
3920 (entry->eflags & MAP_ENTRY_COW) &&
3921 (fault_type & VM_PROT_WRITE))
3922 return (KERN_PROTECTION_FAILURE);
3923
3924 /*
3925 * If this page is not pageable, we have to get it for all possible
3926 * accesses.
3927 */
3928 *wired = (entry->wired_count != 0);
3929 if (*wired)
3930 fault_type = entry->protection;
3931
3932 if (entry->eflags & MAP_ENTRY_NEEDS_COPY) {
3933 /*
3934 * Fail if the entry was copy-on-write for a write fault.
3935 */
3936 if (fault_type & VM_PROT_WRITE)
3937 return (KERN_FAILURE);
3938 /*
3939 * We're attempting to read a copy-on-write page --
3940 * don't allow writes.
3941 */
3942 prot &= ~VM_PROT_WRITE;
3943 }
3944
3945 /*
3946 * Fail if an object should be created.
3947 */
3948 if (entry->object.vm_object == NULL && !map->system_map)
3949 return (KERN_FAILURE);
3950
3951 /*
3952 * Return the object/offset from this entry. If the entry was
3953 * copy-on-write or empty, it has been fixed up.
3954 */
3955 *pindex = OFF_TO_IDX((vaddr - entry->start) + entry->offset);
3956 *object = entry->object.vm_object;
3957
3958 *out_prot = prot;
3959 return (KERN_SUCCESS);
3960 }
3961
3962 /*
3963 * vm_map_lookup_done:
3964 *
3965 * Releases locks acquired by a vm_map_lookup
3966 * (according to the handle returned by that lookup).
3967 */
3968 void
3969 vm_map_lookup_done(vm_map_t map, vm_map_entry_t entry)
3970 {
3971 /*
3972 * Unlock the main-level map
3973 */
3974 vm_map_unlock_read(map);
3975 }
3976
3977 #include "opt_ddb.h"
3978 #ifdef DDB
3979 #include <sys/kernel.h>
3980
3981 #include <ddb/ddb.h>
3982
3983 /*
3984 * vm_map_print: [ debug ]
3985 */
3986 DB_SHOW_COMMAND(map, vm_map_print)
3987 {
3988 static int nlines;
3989 /* XXX convert args. */
3990 vm_map_t map = (vm_map_t)addr;
3991 boolean_t full = have_addr;
3992
3993 vm_map_entry_t entry;
3994
3995 db_iprintf("Task map %p: pmap=%p, nentries=%d, version=%u\n",
3996 (void *)map,
3997 (void *)map->pmap, map->nentries, map->timestamp);
3998 nlines++;
3999
4000 if (!full && db_indent)
4001 return;
4002
4003 db_indent += 2;
4004 for (entry = map->header.next; entry != &map->header;
4005 entry = entry->next) {
4006 db_iprintf("map entry %p: start=%p, end=%p\n",
4007 (void *)entry, (void *)entry->start, (void *)entry->end);
4008 nlines++;
4009 {
4010 static char *inheritance_name[4] =
4011 {"share", "copy", "none", "donate_copy"};
4012
4013 db_iprintf(" prot=%x/%x/%s",
4014 entry->protection,
4015 entry->max_protection,
4016 inheritance_name[(int)(unsigned char)entry->inheritance]);
4017 if (entry->wired_count != 0)
4018 db_printf(", wired");
4019 }
4020 if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
4021 db_printf(", share=%p, offset=0x%jx\n",
4022 (void *)entry->object.sub_map,
4023 (uintmax_t)entry->offset);
4024 nlines++;
4025 if ((entry->prev == &map->header) ||
4026 (entry->prev->object.sub_map !=
4027 entry->object.sub_map)) {
4028 db_indent += 2;
4029 vm_map_print((db_expr_t)(intptr_t)
4030 entry->object.sub_map,
4031 full, 0, (char *)0);
4032 db_indent -= 2;
4033 }
4034 } else {
4035 if (entry->cred != NULL)
4036 db_printf(", ruid %d", entry->cred->cr_ruid);
4037 db_printf(", object=%p, offset=0x%jx",
4038 (void *)entry->object.vm_object,
4039 (uintmax_t)entry->offset);
4040 if (entry->object.vm_object && entry->object.vm_object->cred)
4041 db_printf(", obj ruid %d charge %jx",
4042 entry->object.vm_object->cred->cr_ruid,
4043 (uintmax_t)entry->object.vm_object->charge);
4044 if (entry->eflags & MAP_ENTRY_COW)
4045 db_printf(", copy (%s)",
4046 (entry->eflags & MAP_ENTRY_NEEDS_COPY) ? "needed" : "done");
4047 db_printf("\n");
4048 nlines++;
4049
4050 if ((entry->prev == &map->header) ||
4051 (entry->prev->object.vm_object !=
4052 entry->object.vm_object)) {
4053 db_indent += 2;
4054 vm_object_print((db_expr_t)(intptr_t)
4055 entry->object.vm_object,
4056 full, 0, (char *)0);
4057 nlines += 4;
4058 db_indent -= 2;
4059 }
4060 }
4061 }
4062 db_indent -= 2;
4063 if (db_indent == 0)
4064 nlines = 0;
4065 }
4066
4067
4068 DB_SHOW_COMMAND(procvm, procvm)
4069 {
4070 struct proc *p;
4071
4072 if (have_addr) {
4073 p = (struct proc *) addr;
4074 } else {
4075 p = curproc;
4076 }
4077
4078 db_printf("p = %p, vmspace = %p, map = %p, pmap = %p\n",
4079 (void *)p, (void *)p->p_vmspace, (void *)&p->p_vmspace->vm_map,
4080 (void *)vmspace_pmap(p->p_vmspace));
4081
4082 vm_map_print((db_expr_t)(intptr_t)&p->p_vmspace->vm_map, 1, 0, NULL);
4083 }
4084
4085 #endif /* DDB */
Cache object: 572b8ea82865d1487f824190bed60e9c
|