Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/vm/vm_kern.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* 2 * Mach Operating System 3 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University 4 * All Rights Reserved. 5 * 6 * Permission to use, copy, modify and distribute this software and its 7 * documentation is hereby granted, provided that both the copyright 8 * notice and this permission notice appear in all copies of the 9 * software, derivative works or modified versions, and any portions 10 * thereof, and that both notices appear in supporting documentation. 11 * 12 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 13 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 14 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 15 * 16 * Carnegie Mellon requests users of this software to return to 17 * 18 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 19 * School of Computer Science 20 * Carnegie Mellon University 21 * Pittsburgh PA 15213-3890 22 * 23 * any improvements or extensions that they make and grant Carnegie Mellon 24 * the rights to redistribute these changes. 25 */ 26 /* 27 * HISTORY 28 * $Log: vm_kern.c,v $ 29 * Revision 2.24 93/08/10 15:12:33 mrt 30 * Included support for projected buffers: projected_buffer_allocate, 31 * projected_buffer_map, projected_buffer_deallocate, projected_buffer_collect 32 * and projected_buffer_in_range. Projected buffers are buffers shared 33 * between the kernel and user tasks, and which may be persistent or not 34 * (be deallocated from the kernel map automatically when the last user 35 * reference is deallocated). The intended use is for device drivers. 36 * The user is denied direct manipulation of these map entries. 37 * [93/02/16 09:28:21 jcb] 38 * 39 * Revision 2.23 93/01/14 18:01:03 danner 40 * 64bit cleanup. 41 * [92/12/01 af] 42 * 43 * Revision 2.22 92/08/03 18:00:41 jfriedl 44 * removed silly prototypes 45 * [92/08/02 jfriedl] 46 * 47 * Revision 2.21 92/05/21 17:26:02 jfriedl 48 * Added types for functions that didn't have them explicitly. 49 * [92/05/16 jfriedl] 50 * 51 * Revision 2.20 92/04/01 19:36:35 rpd 52 * Change kmem_io_map_copyout to handle multiple page lists. 53 * Don't call pmap_change_wiring from kmem_io_map_deallocate. 54 * [92/03/20 14:13:50 dlb] 55 * 56 * Revision 2.19 92/02/23 19:50:53 elf 57 * kmem_io_map_deallocate() now calls pmap_remove() -- VM 58 * code optimization makes this necessary. Optimized 59 * out kmem_alloc_pageable() call from kmem_io_map_copyout(). 60 * [92/01/07 16:37:38 dlb] 61 * 62 * Revision 2.18 92/01/14 16:47:57 rpd 63 * Added copyinmap and copyoutmap. 64 * [91/12/16 rpd] 65 * 66 * Revision 2.17 91/08/28 11:18:00 jsb 67 * Delete kmem_fault_wire, io_wire, io_unwire - Replaced by 68 * kmem_io_{copyout,deallocate}. 69 * [91/08/06 17:17:40 dlb] 70 * 71 * Make kmem_io_map_deallocate return void. 72 * [91/08/05 17:45:39 dlb] 73 * 74 * New interfaces for kmem_io_map routines. 75 * [91/08/02 17:07:40 dlb] 76 * 77 * New and improved io wiring support based on vm page lists: 78 * kmem_io_map_{copyout,deallocate}. io_wire and io_unwire will 79 * go away when the device logic fully supports this. 80 * [91/07/31 15:12:02 dlb] 81 * 82 * Revision 2.16 91/07/30 15:47:20 rvb 83 * Fixed io_wire to allocate an object when the entry doesn't have one. 84 * [91/06/27 rpd] 85 * 86 * Revision 2.15 91/05/18 14:40:31 rpd 87 * Added kmem_alloc_aligned. 88 * [91/05/02 rpd] 89 * Added VM_FAULT_FICTITIOUS_SHORTAGE. 90 * Revised vm_map_find_entry to allow coalescing of entries. 91 * Fixed deadlock problem in kmem_alloc. 92 * [91/03/29 rpd] 93 * Fixed kmem_init to not create a zero-size entry. 94 * [91/03/25 rpd] 95 * 96 * Revision 2.14 91/05/14 17:49:15 mrt 97 * Correcting copyright 98 * 99 * Revision 2.13 91/03/16 15:05:20 rpd 100 * Fixed kmem_alloc_pages and kmem_remap_pages 101 * to not hold locks across pmap_enter. 102 * [91/03/11 rpd] 103 * Added kmem_realloc. Changed kmem_alloc, kmem_alloc_wired, and 104 * kmem_alloc_pageable to return error codes. Changed kmem_alloc 105 * to not use the kernel object and to not zero memory. 106 * Changed kmem_alloc_wired to use the kernel object. 107 * [91/03/07 16:49:52 rpd] 108 * 109 * Added resume, continuation arguments to vm_fault_page. 110 * Added continuation argument to VM_PAGE_WAIT. 111 * [91/02/05 rpd] 112 * 113 * Revision 2.12 91/02/05 17:58:22 mrt 114 * Changed to new Mach copyright 115 * [91/02/01 16:32:19 mrt] 116 * 117 * Revision 2.11 91/01/08 16:44:59 rpd 118 * Changed VM_WAIT to VM_PAGE_WAIT. 119 * [90/11/13 rpd] 120 * 121 * Revision 2.10 90/10/12 13:05:35 rpd 122 * Only activate the page returned by vm_fault_page if it isn't 123 * already on a pageout queue. 124 * [90/10/09 22:33:09 rpd] 125 * 126 * Revision 2.9 90/06/19 23:01:54 rpd 127 * Picked up vm_submap_object. 128 * [90/06/08 rpd] 129 * 130 * Revision 2.8 90/06/02 15:10:43 rpd 131 * Purged MACH_XP_FPD. 132 * [90/03/26 23:12:33 rpd] 133 * 134 * Revision 2.7 90/02/22 20:05:39 dbg 135 * Update to vm_map.h. 136 * Remove kmem_alloc_wait, kmem_free_wakeup, vm_move. 137 * Fix copy_user_to_physical_page to test for kernel tasks. 138 * Simplify v_to_p allocation. 139 * Change PAGE_WAKEUP to PAGE_WAKEUP_DONE to reflect the 140 * fact that it clears the busy flag. 141 * [90/01/25 dbg] 142 * 143 * Revision 2.6 90/01/22 23:09:12 af 144 * Undone VM_PROT_DEFAULT change, moved to vm_prot.h 145 * [90/01/20 17:28:57 af] 146 * 147 * Revision 2.5 90/01/19 14:35:57 rwd 148 * Get new version from rfr 149 * [90/01/10 rwd] 150 * 151 * Revision 2.4 90/01/11 11:47:44 dbg 152 * Remove kmem_mb_alloc and mb_map. 153 * [89/12/11 dbg] 154 * 155 * Revision 2.3 89/11/29 14:17:43 af 156 * Redefine VM_PROT_DEFAULT locally for mips. 157 * Might migrate in the final place sometimes. 158 * 159 * Revision 2.2 89/09/08 11:28:19 dbg 160 * Add special wiring code for IO memory. 161 * [89/08/10 dbg] 162 * 163 * Add keep_wired argument to vm_move. 164 * [89/07/14 dbg] 165 * 166 * 28-Apr-89 David Golub (dbg) at Carnegie-Mellon University 167 * Changes for MACH_KERNEL: 168 * . Optimize kmem_alloc. Add kmem_alloc_wired. 169 * . Remove non-MACH include files. 170 * . Change vm_move to call vm_map_move. 171 * . Clean up fast_pager_data option. 172 * 173 * Revision 2.14 89/04/22 15:35:28 gm0w 174 * Added code in kmem_mb_alloc to verify that requested allocation 175 * will fit in the map. 176 * [89/04/14 gm0w] 177 * 178 * Revision 2.13 89/04/18 21:25:45 mwyoung 179 * Recent history: 180 * Add call to vm_map_simplify to reduce kernel map fragmentation. 181 * History condensation: 182 * Added routines for copying user data to physical 183 * addresses. [rfr, mwyoung] 184 * Added routines for sleep/wakeup forms, interrupt-time 185 * allocation. [dbg] 186 * Created. [avie, mwyoung, dbg] 187 * 188 */ 189 /* 190 * File: vm/vm_kern.c 191 * Author: Avadis Tevanian, Jr., Michael Wayne Young 192 * Date: 1985 193 * 194 * Kernel memory management. 195 */ 196 197 #include <mach/kern_return.h> 198 #include <mach/vm_param.h> 199 #include <kern/assert.h> 200 #include <kern/lock.h> 201 #include <kern/thread.h> 202 #include <vm/vm_fault.h> 203 #include <vm/vm_kern.h> 204 #include <vm/vm_map.h> 205 #include <vm/vm_object.h> 206 #include <vm/vm_page.h> 207 #include <vm/vm_pageout.h> 208 209 210 211 /* 212 * Variables exported by this module. 213 */ 214 215 vm_map_t kernel_map; 216 vm_map_t kernel_pageable_map; 217 218 extern void kmem_alloc_pages(); 219 extern void kmem_remap_pages(); 220 221 /* 222 * projected_buffer_allocate 223 * 224 * Allocate a wired-down buffer shared between kernel and user task. 225 * Fresh, zero-filled memory is allocated. 226 * If persistence is false, this buffer can only be deallocated from 227 * user task using projected_buffer_deallocate, and deallocation 228 * from user task also deallocates the buffer from the kernel map. 229 * projected_buffer_collect is called from vm_map_deallocate to 230 * automatically deallocate projected buffers on task_deallocate. 231 * Sharing with more than one user task is achieved by using 232 * projected_buffer_map for the second and subsequent tasks. 233 * The user is precluded from manipulating the VM entry of this buffer 234 * (i.e. changing protection, inheritance or machine attributes). 235 */ 236 237 kern_return_t 238 projected_buffer_allocate(map, size, persistence, kernel_p, 239 user_p, protection, inheritance) 240 vm_map_t map; 241 vm_size_t size; 242 int persistence; 243 vm_offset_t *kernel_p; 244 vm_offset_t *user_p; 245 vm_prot_t protection; 246 vm_inherit_t inheritance; /*Currently only VM_INHERIT_NONE supported*/ 247 { 248 vm_object_t object; 249 vm_map_entry_t u_entry, k_entry; 250 vm_offset_t addr; 251 vm_size_t r_size; 252 kern_return_t kr; 253 254 if (map == VM_MAP_NULL || map == kernel_map) 255 return(KERN_INVALID_ARGUMENT); 256 257 /* 258 * Allocate a new object. 259 */ 260 261 size = round_page(size); 262 object = vm_object_allocate(size); 263 264 vm_map_lock(kernel_map); 265 kr = vm_map_find_entry(kernel_map, &addr, size, (vm_offset_t) 0, 266 VM_OBJECT_NULL, &k_entry); 267 if (kr != KERN_SUCCESS) { 268 vm_map_unlock(kernel_map); 269 vm_object_deallocate(object); 270 return kr; 271 } 272 273 k_entry->object.vm_object = object; 274 if (!persistence) 275 k_entry->projected_on = (vm_map_entry_t) -1; 276 /*Mark entry so as to automatically deallocate it when 277 last corresponding user entry is deallocated*/ 278 vm_map_unlock(kernel_map); 279 *kernel_p = addr; 280 281 vm_map_lock(map); 282 kr = vm_map_find_entry(map, &addr, size, (vm_offset_t) 0, 283 VM_OBJECT_NULL, &u_entry); 284 if (kr != KERN_SUCCESS) { 285 vm_map_unlock(map); 286 vm_map_lock(kernel_map); 287 vm_map_entry_delete(kernel_map, k_entry); 288 vm_map_unlock(kernel_map); 289 vm_object_deallocate(object); 290 return kr; 291 } 292 293 u_entry->object.vm_object = object; 294 vm_object_reference(object); 295 u_entry->projected_on = k_entry; 296 /*Creates coupling with kernel mapping of the buffer, and 297 also guarantees that user cannot directly manipulate 298 buffer VM entry*/ 299 u_entry->protection = protection; 300 u_entry->max_protection = protection; 301 u_entry->inheritance = inheritance; 302 vm_map_unlock(map); 303 *user_p = addr; 304 305 /* 306 * Allocate wired-down memory in the object, 307 * and enter it in the kernel pmap. 308 */ 309 kmem_alloc_pages(object, 0, 310 *kernel_p, *kernel_p + size, 311 VM_PROT_READ | VM_PROT_WRITE); 312 bzero(*kernel_p, size); /*Zero fill*/ 313 314 /* Set up physical mappings for user pmap */ 315 316 pmap_pageable(map->pmap, *user_p, *user_p + size, FALSE); 317 for (r_size = 0; r_size < size; r_size += PAGE_SIZE) { 318 addr = pmap_extract(kernel_pmap, *kernel_p + r_size); 319 pmap_enter(map->pmap, *user_p + r_size, addr, 320 protection, TRUE); 321 } 322 323 return(KERN_SUCCESS); 324 } 325 326 327 /* 328 * projected_buffer_map 329 * 330 * Map an area of kernel memory onto a task's address space. 331 * No new memory is allocated; the area must previously exist in the 332 * kernel memory map. 333 */ 334 335 kern_return_t 336 projected_buffer_map(map, kernel_addr, size, user_p, protection, inheritance) 337 vm_map_t map; 338 vm_offset_t kernel_addr; 339 vm_size_t size; 340 vm_offset_t *user_p; 341 vm_prot_t protection; 342 vm_inherit_t inheritance; /*Currently only VM_INHERIT_NONE supported*/ 343 { 344 vm_object_t object; 345 vm_map_entry_t u_entry, k_entry; 346 vm_offset_t physical_addr, user_addr; 347 vm_size_t r_size; 348 kern_return_t kr; 349 350 /* 351 * Find entry in kernel map 352 */ 353 354 size = round_page(size); 355 if (map == VM_MAP_NULL || map == kernel_map || 356 !vm_map_lookup_entry(kernel_map, kernel_addr, &k_entry) || 357 kernel_addr + size > k_entry->vme_end) 358 return(KERN_INVALID_ARGUMENT); 359 360 361 /* 362 * Create entry in user task 363 */ 364 365 vm_map_lock(map); 366 kr = vm_map_find_entry(map, &user_addr, size, (vm_offset_t) 0, 367 VM_OBJECT_NULL, &u_entry); 368 if (kr != KERN_SUCCESS) { 369 vm_map_unlock(map); 370 return kr; 371 } 372 373 u_entry->object.vm_object = k_entry->object.vm_object; 374 vm_object_reference(k_entry->object.vm_object); 375 u_entry->offset = kernel_addr - k_entry->vme_start + k_entry->offset; 376 u_entry->projected_on = k_entry; 377 /*Creates coupling with kernel mapping of the buffer, and 378 also guarantees that user cannot directly manipulate 379 buffer VM entry*/ 380 u_entry->protection = protection; 381 u_entry->max_protection = protection; 382 u_entry->inheritance = inheritance; 383 u_entry->wired_count = k_entry->wired_count; 384 vm_map_unlock(map); 385 *user_p = user_addr; 386 387 /* Set up physical mappings for user pmap */ 388 389 pmap_pageable(map->pmap, user_addr, user_addr + size, 390 !k_entry->wired_count); 391 for (r_size = 0; r_size < size; r_size += PAGE_SIZE) { 392 physical_addr = pmap_extract(kernel_pmap, kernel_addr + r_size); 393 pmap_enter(map->pmap, user_addr + r_size, physical_addr, 394 protection, k_entry->wired_count); 395 } 396 397 return(KERN_SUCCESS); 398 } 399 400 401 /* 402 * projected_buffer_deallocate 403 * 404 * Unmap projected buffer from task's address space. 405 * May also unmap buffer from kernel map, if buffer is not 406 * persistent and only the kernel reference remains. 407 */ 408 409 kern_return_t 410 projected_buffer_deallocate(map, start, end) 411 vm_map_t map; 412 vm_offset_t start, end; 413 { 414 vm_map_entry_t entry, k_entry; 415 416 vm_map_lock(map); 417 if (map == VM_MAP_NULL || map == kernel_map || 418 !vm_map_lookup_entry(map, start, &entry) || 419 end > entry->vme_end || 420 /*Check corresponding kernel entry*/ 421 (k_entry = entry->projected_on) == 0) { 422 vm_map_unlock(map); 423 return(KERN_INVALID_ARGUMENT); 424 } 425 426 /*Prepare for deallocation*/ 427 if (entry->vme_start < start) 428 _vm_map_clip_start(map, entry, start); 429 if (entry->vme_end > end) 430 _vm_map_clip_end(map, entry, end); 431 if (map->first_free == entry) /*Adjust first_free hint*/ 432 map->first_free = entry->vme_prev; 433 entry->projected_on = 0; /*Needed to allow deletion*/ 434 entry->wired_count = 0; /*Avoid unwire fault*/ 435 vm_map_entry_delete(map, entry); 436 vm_map_unlock(map); 437 438 /*Check if the buffer is not persistent and only the 439 kernel mapping remains, and if so delete it*/ 440 vm_map_lock(kernel_map); 441 if (k_entry->projected_on == (vm_map_entry_t) -1 && 442 k_entry->object.vm_object->ref_count == 1) { 443 if (kernel_map->first_free == k_entry) 444 kernel_map->first_free = k_entry->vme_prev; 445 k_entry->projected_on = 0; /*Allow unwire fault*/ 446 vm_map_entry_delete(kernel_map, k_entry); 447 } 448 vm_map_unlock(kernel_map); 449 return(KERN_SUCCESS); 450 } 451 452 453 /* 454 * projected_buffer_collect 455 * 456 * Unmap all projected buffers from task's address space. 457 */ 458 459 kern_return_t 460 projected_buffer_collect(map) 461 vm_map_t map; 462 { 463 vm_map_entry_t entry, next; 464 465 if (map == VM_MAP_NULL || map == kernel_map) 466 return(KERN_INVALID_ARGUMENT); 467 468 for (entry = vm_map_first_entry(map); 469 entry != vm_map_to_entry(map); 470 entry = next) { 471 next = entry->vme_next; 472 if (entry->projected_on != 0) 473 projected_buffer_deallocate(map, entry->vme_start, entry->vme_end); 474 } 475 return(KERN_SUCCESS); 476 } 477 478 479 /* 480 * projected_buffer_in_range 481 * 482 * Verifies whether a projected buffer exists in the address range 483 * given. 484 */ 485 486 boolean_t 487 projected_buffer_in_range(map, start, end) 488 vm_map_t map; 489 vm_offset_t start, end; 490 { 491 vm_map_entry_t entry; 492 493 if (map == VM_MAP_NULL || map == kernel_map) 494 return(FALSE); 495 496 /*Find first entry*/ 497 if (!vm_map_lookup_entry(map, start, &entry)) 498 entry = entry->vme_next; 499 500 while (entry != vm_map_to_entry(map) && entry->projected_on == 0 && 501 entry->vme_start <= end) { 502 entry = entry->vme_next; 503 } 504 return(entry != vm_map_to_entry(map) && entry->vme_start <= end); 505 } 506 507 508 /* 509 * kmem_alloc: 510 * 511 * Allocate wired-down memory in the kernel's address map 512 * or a submap. The memory is not zero-filled. 513 */ 514 515 kern_return_t 516 kmem_alloc(map, addrp, size) 517 vm_map_t map; 518 vm_offset_t *addrp; 519 vm_size_t size; 520 { 521 vm_object_t object; 522 vm_map_entry_t entry; 523 vm_offset_t addr; 524 kern_return_t kr; 525 526 /* 527 * Allocate a new object. We must do this before locking 528 * the map, lest we risk deadlock with the default pager: 529 * device_read_alloc uses kmem_alloc, 530 * which tries to allocate an object, 531 * which uses kmem_alloc_wired to get memory, 532 * which blocks for pages. 533 * then the default pager needs to read a block 534 * to process a memory_object_data_write, 535 * and device_read_alloc calls kmem_alloc 536 * and deadlocks on the map lock. 537 */ 538 539 size = round_page(size); 540 object = vm_object_allocate(size); 541 542 vm_map_lock(map); 543 kr = vm_map_find_entry(map, &addr, size, (vm_offset_t) 0, 544 VM_OBJECT_NULL, &entry); 545 if (kr != KERN_SUCCESS) { 546 vm_map_unlock(map); 547 vm_object_deallocate(object); 548 return kr; 549 } 550 551 entry->object.vm_object = object; 552 entry->offset = 0; 553 554 /* 555 * Since we have not given out this address yet, 556 * it is safe to unlock the map. 557 */ 558 vm_map_unlock(map); 559 560 /* 561 * Allocate wired-down memory in the kernel_object, 562 * for this entry, and enter it in the kernel pmap. 563 */ 564 kmem_alloc_pages(object, 0, 565 addr, addr + size, 566 VM_PROT_DEFAULT); 567 568 /* 569 * Return the memory, not zeroed. 570 */ 571 *addrp = addr; 572 return KERN_SUCCESS; 573 } 574 575 /* 576 * kmem_realloc: 577 * 578 * Reallocate wired-down memory in the kernel's address map 579 * or a submap. Newly allocated pages are not zeroed. 580 * This can only be used on regions allocated with kmem_alloc. 581 * 582 * If successful, the pages in the old region are mapped twice. 583 * The old region is unchanged. Use kmem_free to get rid of it. 584 */ 585 kern_return_t kmem_realloc(map, oldaddr, oldsize, newaddrp, newsize) 586 vm_map_t map; 587 vm_offset_t oldaddr; 588 vm_size_t oldsize; 589 vm_offset_t *newaddrp; 590 vm_size_t newsize; 591 { 592 vm_offset_t oldmin, oldmax; 593 vm_offset_t newaddr; 594 vm_object_t object; 595 vm_map_entry_t oldentry, newentry; 596 kern_return_t kr; 597 598 oldmin = trunc_page(oldaddr); 599 oldmax = round_page(oldaddr + oldsize); 600 oldsize = oldmax - oldmin; 601 newsize = round_page(newsize); 602 603 /* 604 * Find space for the new region. 605 */ 606 607 vm_map_lock(map); 608 kr = vm_map_find_entry(map, &newaddr, newsize, (vm_offset_t) 0, 609 VM_OBJECT_NULL, &newentry); 610 if (kr != KERN_SUCCESS) { 611 vm_map_unlock(map); 612 return kr; 613 } 614 615 /* 616 * Find the VM object backing the old region. 617 */ 618 619 if (!vm_map_lookup_entry(map, oldmin, &oldentry)) 620 panic("kmem_realloc"); 621 object = oldentry->object.vm_object; 622 623 /* 624 * Increase the size of the object and 625 * fill in the new region. 626 */ 627 628 vm_object_reference(object); 629 vm_object_lock(object); 630 if (object->size != oldsize) 631 panic("kmem_realloc"); 632 object->size = newsize; 633 vm_object_unlock(object); 634 635 newentry->object.vm_object = object; 636 newentry->offset = 0; 637 638 /* 639 * Since we have not given out this address yet, 640 * it is safe to unlock the map. We are trusting 641 * that nobody will play with either region. 642 */ 643 644 vm_map_unlock(map); 645 646 /* 647 * Remap the pages in the old region and 648 * allocate more pages for the new region. 649 */ 650 651 kmem_remap_pages(object, 0, 652 newaddr, newaddr + oldsize, 653 VM_PROT_DEFAULT); 654 kmem_alloc_pages(object, oldsize, 655 newaddr + oldsize, newaddr + newsize, 656 VM_PROT_DEFAULT); 657 658 *newaddrp = newaddr; 659 return KERN_SUCCESS; 660 } 661 662 /* 663 * kmem_alloc_wired: 664 * 665 * Allocate wired-down memory in the kernel's address map 666 * or a submap. The memory is not zero-filled. 667 * 668 * The memory is allocated in the kernel_object. 669 * It may not be copied with vm_map_copy, and 670 * it may not be reallocated with kmem_realloc. 671 */ 672 673 kern_return_t 674 kmem_alloc_wired(map, addrp, size) 675 vm_map_t map; 676 vm_offset_t *addrp; 677 vm_size_t size; 678 { 679 vm_map_entry_t entry; 680 vm_offset_t offset; 681 vm_offset_t addr; 682 kern_return_t kr; 683 684 /* 685 * Use the kernel object for wired-down kernel pages. 686 * Assume that no region of the kernel object is 687 * referenced more than once. We want vm_map_find_entry 688 * to extend an existing entry if possible. 689 */ 690 691 size = round_page(size); 692 vm_map_lock(map); 693 kr = vm_map_find_entry(map, &addr, size, (vm_offset_t) 0, 694 kernel_object, &entry); 695 if (kr != KERN_SUCCESS) { 696 vm_map_unlock(map); 697 return kr; 698 } 699 700 /* 701 * Since we didn't know where the new region would 702 * start, we couldn't supply the correct offset into 703 * the kernel object. We only initialize the entry 704 * if we aren't extending an existing entry. 705 */ 706 707 offset = addr - VM_MIN_KERNEL_ADDRESS; 708 709 if (entry->object.vm_object == VM_OBJECT_NULL) { 710 vm_object_reference(kernel_object); 711 712 entry->object.vm_object = kernel_object; 713 entry->offset = offset; 714 } 715 716 /* 717 * Since we have not given out this address yet, 718 * it is safe to unlock the map. 719 */ 720 vm_map_unlock(map); 721 722 /* 723 * Allocate wired-down memory in the kernel_object, 724 * for this entry, and enter it in the kernel pmap. 725 */ 726 kmem_alloc_pages(kernel_object, offset, 727 addr, addr + size, 728 VM_PROT_DEFAULT); 729 730 /* 731 * Return the memory, not zeroed. 732 */ 733 *addrp = addr; 734 return KERN_SUCCESS; 735 } 736 737 /* 738 * kmem_alloc_aligned: 739 * 740 * Like kmem_alloc_wired, except that the memory is aligned. 741 * The size should be a power-of-2. 742 */ 743 744 kern_return_t 745 kmem_alloc_aligned(map, addrp, size) 746 vm_map_t map; 747 vm_offset_t *addrp; 748 vm_size_t size; 749 { 750 vm_map_entry_t entry; 751 vm_offset_t offset; 752 vm_offset_t addr; 753 kern_return_t kr; 754 755 if ((size & (size - 1)) != 0) 756 panic("kmem_alloc_aligned"); 757 758 /* 759 * Use the kernel object for wired-down kernel pages. 760 * Assume that no region of the kernel object is 761 * referenced more than once. We want vm_map_find_entry 762 * to extend an existing entry if possible. 763 */ 764 765 size = round_page(size); 766 vm_map_lock(map); 767 kr = vm_map_find_entry(map, &addr, size, size - 1, 768 kernel_object, &entry); 769 if (kr != KERN_SUCCESS) { 770 vm_map_unlock(map); 771 return kr; 772 } 773 774 /* 775 * Since we didn't know where the new region would 776 * start, we couldn't supply the correct offset into 777 * the kernel object. We only initialize the entry 778 * if we aren't extending an existing entry. 779 */ 780 781 offset = addr - VM_MIN_KERNEL_ADDRESS; 782 783 if (entry->object.vm_object == VM_OBJECT_NULL) { 784 vm_object_reference(kernel_object); 785 786 entry->object.vm_object = kernel_object; 787 entry->offset = offset; 788 } 789 790 /* 791 * Since we have not given out this address yet, 792 * it is safe to unlock the map. 793 */ 794 vm_map_unlock(map); 795 796 /* 797 * Allocate wired-down memory in the kernel_object, 798 * for this entry, and enter it in the kernel pmap. 799 */ 800 kmem_alloc_pages(kernel_object, offset, 801 addr, addr + size, 802 VM_PROT_DEFAULT); 803 804 /* 805 * Return the memory, not zeroed. 806 */ 807 *addrp = addr; 808 return KERN_SUCCESS; 809 } 810 811 /* 812 * kmem_alloc_pageable: 813 * 814 * Allocate pageable memory in the kernel's address map. 815 */ 816 817 kern_return_t 818 kmem_alloc_pageable(map, addrp, size) 819 vm_map_t map; 820 vm_offset_t *addrp; 821 vm_size_t size; 822 { 823 vm_offset_t addr; 824 kern_return_t kr; 825 826 addr = vm_map_min(map); 827 kr = vm_map_enter(map, &addr, round_page(size), 828 (vm_offset_t) 0, TRUE, 829 VM_OBJECT_NULL, (vm_offset_t) 0, FALSE, 830 VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); 831 if (kr != KERN_SUCCESS) 832 return kr; 833 834 *addrp = addr; 835 return KERN_SUCCESS; 836 } 837 838 /* 839 * kmem_free: 840 * 841 * Release a region of kernel virtual memory allocated 842 * with kmem_alloc, kmem_alloc_wired, or kmem_alloc_pageable, 843 * and return the physical pages associated with that region. 844 */ 845 846 void 847 kmem_free(map, addr, size) 848 vm_map_t map; 849 vm_offset_t addr; 850 vm_size_t size; 851 { 852 kern_return_t kr; 853 854 kr = vm_map_remove(map, trunc_page(addr), round_page(addr + size)); 855 if (kr != KERN_SUCCESS) 856 panic("kmem_free"); 857 } 858 859 /* 860 * Allocate new wired pages in an object. 861 * The object is assumed to be mapped into the kernel map or 862 * a submap. 863 */ 864 void 865 kmem_alloc_pages(object, offset, start, end, protection) 866 register vm_object_t object; 867 register vm_offset_t offset; 868 register vm_offset_t start, end; 869 vm_prot_t protection; 870 { 871 /* 872 * Mark the pmap region as not pageable. 873 */ 874 pmap_pageable(kernel_pmap, start, end, FALSE); 875 876 while (start < end) { 877 register vm_page_t mem; 878 879 vm_object_lock(object); 880 881 /* 882 * Allocate a page 883 */ 884 while ((mem = vm_page_alloc(object, offset)) 885 == VM_PAGE_NULL) { 886 vm_object_unlock(object); 887 VM_PAGE_WAIT((void (*)()) 0); 888 vm_object_lock(object); 889 } 890 891 /* 892 * Wire it down 893 */ 894 vm_page_lock_queues(); 895 vm_page_wire(mem); 896 vm_page_unlock_queues(); 897 vm_object_unlock(object); 898 899 /* 900 * Enter it in the kernel pmap 901 */ 902 PMAP_ENTER(kernel_pmap, start, mem, 903 protection, TRUE); 904 905 vm_object_lock(object); 906 PAGE_WAKEUP_DONE(mem); 907 vm_object_unlock(object); 908 909 start += PAGE_SIZE; 910 offset += PAGE_SIZE; 911 } 912 } 913 914 /* 915 * Remap wired pages in an object into a new region. 916 * The object is assumed to be mapped into the kernel map or 917 * a submap. 918 */ 919 void 920 kmem_remap_pages(object, offset, start, end, protection) 921 register vm_object_t object; 922 register vm_offset_t offset; 923 register vm_offset_t start, end; 924 vm_prot_t protection; 925 { 926 /* 927 * Mark the pmap region as not pageable. 928 */ 929 pmap_pageable(kernel_pmap, start, end, FALSE); 930 931 while (start < end) { 932 register vm_page_t mem; 933 934 vm_object_lock(object); 935 936 /* 937 * Find a page 938 */ 939 if ((mem = vm_page_lookup(object, offset)) == VM_PAGE_NULL) 940 panic("kmem_remap_pages"); 941 942 /* 943 * Wire it down (again) 944 */ 945 vm_page_lock_queues(); 946 vm_page_wire(mem); 947 vm_page_unlock_queues(); 948 vm_object_unlock(object); 949 950 /* 951 * Enter it in the kernel pmap. The page isn't busy, 952 * but this shouldn't be a problem because it is wired. 953 */ 954 PMAP_ENTER(kernel_pmap, start, mem, 955 protection, TRUE); 956 957 start += PAGE_SIZE; 958 offset += PAGE_SIZE; 959 } 960 } 961 962 /* 963 * kmem_suballoc: 964 * 965 * Allocates a map to manage a subrange 966 * of the kernel virtual address space. 967 * 968 * Arguments are as follows: 969 * 970 * parent Map to take range from 971 * size Size of range to find 972 * min, max Returned endpoints of map 973 * pageable Can the region be paged 974 */ 975 976 vm_map_t 977 kmem_suballoc(parent, min, max, size, pageable) 978 vm_map_t parent; 979 vm_offset_t *min, *max; 980 vm_size_t size; 981 boolean_t pageable; 982 { 983 vm_map_t map; 984 vm_offset_t addr; 985 kern_return_t kr; 986 987 size = round_page(size); 988 989 /* 990 * Need reference on submap object because it is internal 991 * to the vm_system. vm_object_enter will never be called 992 * on it (usual source of reference for vm_map_enter). 993 */ 994 vm_object_reference(vm_submap_object); 995 996 addr = (vm_offset_t) vm_map_min(parent); 997 kr = vm_map_enter(parent, &addr, size, 998 (vm_offset_t) 0, TRUE, 999 vm_submap_object, (vm_offset_t) 0, FALSE, 1000 VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); 1001 if (kr != KERN_SUCCESS) 1002 panic("kmem_suballoc"); 1003 1004 pmap_reference(vm_map_pmap(parent)); 1005 map = vm_map_create(vm_map_pmap(parent), addr, addr + size, pageable); 1006 if (map == VM_MAP_NULL) 1007 panic("kmem_suballoc"); 1008 1009 kr = vm_map_submap(parent, addr, addr + size, map); 1010 if (kr != KERN_SUCCESS) 1011 panic("kmem_suballoc"); 1012 1013 *min = addr; 1014 *max = addr + size; 1015 return map; 1016 } 1017 1018 /* 1019 * kmem_init: 1020 * 1021 * Initialize the kernel's virtual memory map, taking 1022 * into account all memory allocated up to this time. 1023 */ 1024 void kmem_init(start, end) 1025 vm_offset_t start; 1026 vm_offset_t end; 1027 { 1028 kernel_map = vm_map_create(pmap_kernel(), 1029 VM_MIN_KERNEL_ADDRESS, end, 1030 FALSE); 1031 1032 /* 1033 * Reserve virtual memory allocated up to this time. 1034 */ 1035 1036 if (start != VM_MIN_KERNEL_ADDRESS) { 1037 vm_offset_t addr = VM_MIN_KERNEL_ADDRESS; 1038 (void) vm_map_enter(kernel_map, 1039 &addr, start - VM_MIN_KERNEL_ADDRESS, 1040 (vm_offset_t) 0, TRUE, 1041 VM_OBJECT_NULL, (vm_offset_t) 0, FALSE, 1042 VM_PROT_DEFAULT, VM_PROT_ALL, 1043 VM_INHERIT_DEFAULT); 1044 } 1045 } 1046 1047 /* 1048 * New and improved IO wiring support. 1049 */ 1050 1051 /* 1052 * kmem_io_map_copyout: 1053 * 1054 * Establish temporary mapping in designated map for the memory 1055 * passed in. Memory format must be a page_list vm_map_copy. 1056 * Mapping is READ-ONLY. 1057 */ 1058 1059 kern_return_t 1060 kmem_io_map_copyout(map, addr, alloc_addr, alloc_size, copy, min_size) 1061 vm_map_t map; 1062 vm_offset_t *addr; /* actual addr of data */ 1063 vm_offset_t *alloc_addr; /* page aligned addr */ 1064 vm_size_t *alloc_size; /* size allocated */ 1065 vm_map_copy_t copy; 1066 vm_size_t min_size; /* Do at least this much */ 1067 { 1068 vm_offset_t myaddr, offset; 1069 vm_size_t mysize, copy_size; 1070 kern_return_t ret; 1071 register 1072 vm_page_t *page_list; 1073 vm_map_copy_t new_copy; 1074 register 1075 int i; 1076 1077 assert(copy->type == VM_MAP_COPY_PAGE_LIST); 1078 assert(min_size != 0); 1079 1080 /* 1081 * Figure out the size in vm pages. 1082 */ 1083 min_size += copy->offset - trunc_page(copy->offset); 1084 min_size = round_page(min_size); 1085 mysize = round_page(copy->offset + copy->size) - 1086 trunc_page(copy->offset); 1087 1088 /* 1089 * If total size is larger than one page list and 1090 * we don't have to do more than one page list, then 1091 * only do one page list. 1092 * 1093 * XXX Could be much smarter about this ... like trimming length 1094 * XXX if we need more than one page list but not all of them. 1095 */ 1096 1097 copy_size = ptoa(copy->cpy_npages); 1098 if (mysize > copy_size && copy_size > min_size) 1099 mysize = copy_size; 1100 1101 /* 1102 * Allocate some address space in the map (must be kernel 1103 * space). 1104 */ 1105 myaddr = vm_map_min(map); 1106 ret = vm_map_enter(map, &myaddr, mysize, 1107 (vm_offset_t) 0, TRUE, 1108 VM_OBJECT_NULL, (vm_offset_t) 0, FALSE, 1109 VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); 1110 1111 if (ret != KERN_SUCCESS) 1112 return(ret); 1113 1114 /* 1115 * Tell the pmap module that this will be wired, and 1116 * enter the mappings. 1117 */ 1118 pmap_pageable(vm_map_pmap(map), myaddr, myaddr + mysize, TRUE); 1119 1120 *addr = myaddr + (copy->offset - trunc_page(copy->offset)); 1121 *alloc_addr = myaddr; 1122 *alloc_size = mysize; 1123 1124 offset = myaddr; 1125 page_list = ©->cpy_page_list[0]; 1126 while (TRUE) { 1127 for ( i = 0; i < copy->cpy_npages; i++, offset += PAGE_SIZE) { 1128 PMAP_ENTER(vm_map_pmap(map), offset, *page_list, 1129 VM_PROT_READ, TRUE); 1130 page_list++; 1131 } 1132 1133 if (offset == (myaddr + mysize)) 1134 break; 1135 1136 /* 1137 * Onward to the next page_list. The extend_cont 1138 * leaves the current page list's pages alone; 1139 * they'll be cleaned up at discard. Reset this 1140 * copy's continuation to discard the next one. 1141 */ 1142 vm_map_copy_invoke_extend_cont(copy, &new_copy, &ret); 1143 1144 if (ret != KERN_SUCCESS) { 1145 kmem_io_map_deallocate(map, myaddr, mysize); 1146 return(ret); 1147 } 1148 copy->cpy_cont = vm_map_copy_discard_cont; 1149 copy->cpy_cont_args = (char *) new_copy; 1150 copy = new_copy; 1151 page_list = ©->cpy_page_list[0]; 1152 } 1153 1154 return(ret); 1155 } 1156 1157 /* 1158 * kmem_io_map_deallocate: 1159 * 1160 * Get rid of the mapping established by kmem_io_map_copyout. 1161 * Assumes that addr and size have been rounded to page boundaries. 1162 * (e.g., the alloc_addr and alloc_size returned by kmem_io_map_copyout) 1163 */ 1164 1165 void 1166 kmem_io_map_deallocate(map, addr, size) 1167 vm_map_t map; 1168 vm_offset_t addr; 1169 vm_size_t size; 1170 { 1171 /* 1172 * Remove the mappings. The pmap_remove is needed. 1173 */ 1174 1175 pmap_remove(vm_map_pmap(map), addr, addr + size); 1176 vm_map_remove(map, addr, addr + size); 1177 } 1178 1179 /* 1180 * Routine: copyinmap 1181 * Purpose: 1182 * Like copyin, except that fromaddr is an address 1183 * in the specified VM map. This implementation 1184 * is incomplete; it handles the current user map 1185 * and the kernel map/submaps. 1186 */ 1187 1188 int copyinmap(map, fromaddr, toaddr, length) 1189 vm_map_t map; 1190 char *fromaddr, *toaddr; 1191 int length; 1192 { 1193 if (vm_map_pmap(map) == kernel_pmap) { 1194 /* assume a correct copy */ 1195 bcopy(fromaddr, toaddr, length); 1196 return 0; 1197 } 1198 1199 if (current_map() == map) 1200 return copyin( fromaddr, toaddr, length); 1201 1202 return 1; 1203 } 1204 1205 /* 1206 * Routine: copyoutmap 1207 * Purpose: 1208 * Like copyout, except that toaddr is an address 1209 * in the specified VM map. This implementation 1210 * is incomplete; it handles the current user map 1211 * and the kernel map/submaps. 1212 */ 1213 1214 int copyoutmap(map, fromaddr, toaddr, length) 1215 vm_map_t map; 1216 char *fromaddr, *toaddr; 1217 int length; 1218 { 1219 if (vm_map_pmap(map) == kernel_pmap) { 1220 /* assume a correct copy */ 1221 bcopy(fromaddr, toaddr, length); 1222 return 0; 1223 } 1224 1225 if (current_map() == map) 1226 return copyout(fromaddr, toaddr, length); 1227 1228 return 1; 1229 }
Cache object: a86153434b2f818c6d2510e5c85bece9
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]