Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/vm/vm_mmap.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 1988 University of Utah. 3 * Copyright (c) 1991, 1993 4 * The Regents of the University of California. All rights reserved. 5 * 6 * This code is derived from software contributed to Berkeley by 7 * the Systems Programming Group of the University of Utah Computer 8 * Science Department. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 4. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * from: Utah $Hdr: vm_mmap.c 1.6 91/10/21$ 35 * 36 * @(#)vm_mmap.c 8.4 (Berkeley) 1/12/94 37 */ 38 39 /* 40 * Mapped file (mmap) interface to VM 41 */ 42 43 #include <sys/cdefs.h> 44 __FBSDID("$FreeBSD$"); 45 46 #include "opt_compat.h" 47 #include "opt_hwpmc_hooks.h" 48 49 #include <sys/param.h> 50 #include <sys/systm.h> 51 #include <sys/capability.h> 52 #include <sys/kernel.h> 53 #include <sys/lock.h> 54 #include <sys/mutex.h> 55 #include <sys/sysproto.h> 56 #include <sys/filedesc.h> 57 #include <sys/priv.h> 58 #include <sys/proc.h> 59 #include <sys/procctl.h> 60 #include <sys/racct.h> 61 #include <sys/resource.h> 62 #include <sys/resourcevar.h> 63 #include <sys/sysctl.h> 64 #include <sys/vnode.h> 65 #include <sys/fcntl.h> 66 #include <sys/file.h> 67 #include <sys/mman.h> 68 #include <sys/mount.h> 69 #include <sys/conf.h> 70 #include <sys/stat.h> 71 #include <sys/syscallsubr.h> 72 #include <sys/sysent.h> 73 #include <sys/vmmeter.h> 74 75 #include <security/mac/mac_framework.h> 76 77 #include <vm/vm.h> 78 #include <vm/vm_param.h> 79 #include <vm/pmap.h> 80 #include <vm/vm_map.h> 81 #include <vm/vm_object.h> 82 #include <vm/vm_page.h> 83 #include <vm/vm_pager.h> 84 #include <vm/vm_pageout.h> 85 #include <vm/vm_extern.h> 86 #include <vm/vm_page.h> 87 #include <vm/vnode_pager.h> 88 89 #ifdef HWPMC_HOOKS 90 #include <sys/pmckern.h> 91 #endif 92 93 int old_mlock = 1; 94 SYSCTL_INT(_vm, OID_AUTO, old_mlock, CTLFLAG_RW | CTLFLAG_TUN, &old_mlock, 0, 95 "Do not apply RLIMIT_MEMLOCK on mlockall"); 96 TUNABLE_INT("vm.old_mlock", &old_mlock); 97 98 #ifndef _SYS_SYSPROTO_H_ 99 struct sbrk_args { 100 int incr; 101 }; 102 #endif 103 104 static int vm_mmap_vnode(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *, 105 int *, struct vnode *, vm_ooffset_t *, vm_object_t *, boolean_t *); 106 static int vm_mmap_cdev(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *, 107 int *, struct cdev *, vm_ooffset_t *, vm_object_t *); 108 static int vm_mmap_shm(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *, 109 int *, struct shmfd *, vm_ooffset_t, vm_object_t *); 110 111 /* 112 * MPSAFE 113 */ 114 /* ARGSUSED */ 115 int 116 sys_sbrk(td, uap) 117 struct thread *td; 118 struct sbrk_args *uap; 119 { 120 /* Not yet implemented */ 121 return (EOPNOTSUPP); 122 } 123 124 #ifndef _SYS_SYSPROTO_H_ 125 struct sstk_args { 126 int incr; 127 }; 128 #endif 129 130 /* 131 * MPSAFE 132 */ 133 /* ARGSUSED */ 134 int 135 sys_sstk(td, uap) 136 struct thread *td; 137 struct sstk_args *uap; 138 { 139 /* Not yet implemented */ 140 return (EOPNOTSUPP); 141 } 142 143 #if defined(COMPAT_43) 144 #ifndef _SYS_SYSPROTO_H_ 145 struct getpagesize_args { 146 int dummy; 147 }; 148 #endif 149 150 /* ARGSUSED */ 151 int 152 ogetpagesize(td, uap) 153 struct thread *td; 154 struct getpagesize_args *uap; 155 { 156 /* MP SAFE */ 157 td->td_retval[0] = PAGE_SIZE; 158 return (0); 159 } 160 #endif /* COMPAT_43 */ 161 162 163 /* 164 * Memory Map (mmap) system call. Note that the file offset 165 * and address are allowed to be NOT page aligned, though if 166 * the MAP_FIXED flag it set, both must have the same remainder 167 * modulo the PAGE_SIZE (POSIX 1003.1b). If the address is not 168 * page-aligned, the actual mapping starts at trunc_page(addr) 169 * and the return value is adjusted up by the page offset. 170 * 171 * Generally speaking, only character devices which are themselves 172 * memory-based, such as a video framebuffer, can be mmap'd. Otherwise 173 * there would be no cache coherency between a descriptor and a VM mapping 174 * both to the same character device. 175 */ 176 #ifndef _SYS_SYSPROTO_H_ 177 struct mmap_args { 178 void *addr; 179 size_t len; 180 int prot; 181 int flags; 182 int fd; 183 long pad; 184 off_t pos; 185 }; 186 #endif 187 188 /* 189 * MPSAFE 190 */ 191 int 192 sys_mmap(td, uap) 193 struct thread *td; 194 struct mmap_args *uap; 195 { 196 #ifdef HWPMC_HOOKS 197 struct pmckern_map_in pkm; 198 #endif 199 struct file *fp; 200 struct vnode *vp; 201 vm_offset_t addr; 202 vm_size_t size, pageoff; 203 vm_prot_t cap_maxprot, prot, maxprot; 204 void *handle; 205 objtype_t handle_type; 206 int align, error, flags; 207 off_t pos; 208 struct vmspace *vms = td->td_proc->p_vmspace; 209 cap_rights_t rights; 210 211 addr = (vm_offset_t) uap->addr; 212 size = uap->len; 213 prot = uap->prot & VM_PROT_ALL; 214 flags = uap->flags; 215 pos = uap->pos; 216 217 fp = NULL; 218 219 /* 220 * Enforce the constraints. 221 * Mapping of length 0 is only allowed for old binaries. 222 * Anonymous mapping shall specify -1 as filedescriptor and 223 * zero position for new code. Be nice to ancient a.out 224 * binaries and correct pos for anonymous mapping, since old 225 * ld.so sometimes issues anonymous map requests with non-zero 226 * pos. 227 */ 228 if (!SV_CURPROC_FLAG(SV_AOUT)) { 229 if ((uap->len == 0 && curproc->p_osrel >= P_OSREL_MAP_ANON) || 230 ((flags & MAP_ANON) != 0 && (uap->fd != -1 || pos != 0))) 231 return (EINVAL); 232 } else { 233 if ((flags & MAP_ANON) != 0) 234 pos = 0; 235 } 236 237 if (flags & MAP_STACK) { 238 if ((uap->fd != -1) || 239 ((prot & (PROT_READ | PROT_WRITE)) != (PROT_READ | PROT_WRITE))) 240 return (EINVAL); 241 flags |= MAP_ANON; 242 pos = 0; 243 } 244 245 /* 246 * Align the file position to a page boundary, 247 * and save its page offset component. 248 */ 249 pageoff = (pos & PAGE_MASK); 250 pos -= pageoff; 251 252 /* Adjust size for rounding (on both ends). */ 253 size += pageoff; /* low end... */ 254 size = (vm_size_t) round_page(size); /* hi end */ 255 256 /* Ensure alignment is at least a page and fits in a pointer. */ 257 align = flags & MAP_ALIGNMENT_MASK; 258 if (align != 0 && align != MAP_ALIGNED_SUPER && 259 (align >> MAP_ALIGNMENT_SHIFT >= sizeof(void *) * NBBY || 260 align >> MAP_ALIGNMENT_SHIFT < PAGE_SHIFT)) 261 return (EINVAL); 262 263 /* 264 * Check for illegal addresses. Watch out for address wrap... Note 265 * that VM_*_ADDRESS are not constants due to casts (argh). 266 */ 267 if (flags & MAP_FIXED) { 268 /* 269 * The specified address must have the same remainder 270 * as the file offset taken modulo PAGE_SIZE, so it 271 * should be aligned after adjustment by pageoff. 272 */ 273 addr -= pageoff; 274 if (addr & PAGE_MASK) 275 return (EINVAL); 276 277 /* Address range must be all in user VM space. */ 278 if (addr < vm_map_min(&vms->vm_map) || 279 addr + size > vm_map_max(&vms->vm_map)) 280 return (EINVAL); 281 if (addr + size < addr) 282 return (EINVAL); 283 } else { 284 /* 285 * XXX for non-fixed mappings where no hint is provided or 286 * the hint would fall in the potential heap space, 287 * place it after the end of the largest possible heap. 288 * 289 * There should really be a pmap call to determine a reasonable 290 * location. 291 */ 292 PROC_LOCK(td->td_proc); 293 if (addr == 0 || 294 (addr >= round_page((vm_offset_t)vms->vm_taddr) && 295 addr < round_page((vm_offset_t)vms->vm_daddr + 296 lim_max(td->td_proc, RLIMIT_DATA)))) 297 addr = round_page((vm_offset_t)vms->vm_daddr + 298 lim_max(td->td_proc, RLIMIT_DATA)); 299 PROC_UNLOCK(td->td_proc); 300 } 301 if (flags & MAP_ANON) { 302 /* 303 * Mapping blank space is trivial. 304 */ 305 handle = NULL; 306 handle_type = OBJT_DEFAULT; 307 maxprot = VM_PROT_ALL; 308 cap_maxprot = VM_PROT_ALL; 309 } else { 310 /* 311 * Mapping file, get fp for validation and don't let the 312 * descriptor disappear on us if we block. Check capability 313 * rights, but also return the maximum rights to be combined 314 * with maxprot later. 315 */ 316 rights = CAP_MMAP; 317 if (prot & PROT_READ) 318 rights |= CAP_READ; 319 if ((flags & MAP_SHARED) != 0) { 320 if (prot & PROT_WRITE) 321 rights |= CAP_WRITE; 322 } 323 if (prot & PROT_EXEC) 324 rights |= CAP_MAPEXEC; 325 if ((error = fget_mmap(td, uap->fd, rights, &cap_maxprot, 326 &fp)) != 0) 327 goto done; 328 if (fp->f_type == DTYPE_SHM) { 329 handle = fp->f_data; 330 handle_type = OBJT_SWAP; 331 maxprot = VM_PROT_NONE; 332 333 /* FREAD should always be set. */ 334 if (fp->f_flag & FREAD) 335 maxprot |= VM_PROT_EXECUTE | VM_PROT_READ; 336 if (fp->f_flag & FWRITE) 337 maxprot |= VM_PROT_WRITE; 338 goto map; 339 } 340 if (fp->f_type != DTYPE_VNODE) { 341 error = ENODEV; 342 goto done; 343 } 344 #if defined(COMPAT_FREEBSD7) || defined(COMPAT_FREEBSD6) || \ 345 defined(COMPAT_FREEBSD5) || defined(COMPAT_FREEBSD4) 346 /* 347 * POSIX shared-memory objects are defined to have 348 * kernel persistence, and are not defined to support 349 * read(2)/write(2) -- or even open(2). Thus, we can 350 * use MAP_ASYNC to trade on-disk coherence for speed. 351 * The shm_open(3) library routine turns on the FPOSIXSHM 352 * flag to request this behavior. 353 */ 354 if (fp->f_flag & FPOSIXSHM) 355 flags |= MAP_NOSYNC; 356 #endif 357 vp = fp->f_vnode; 358 /* 359 * Ensure that file and memory protections are 360 * compatible. Note that we only worry about 361 * writability if mapping is shared; in this case, 362 * current and max prot are dictated by the open file. 363 * XXX use the vnode instead? Problem is: what 364 * credentials do we use for determination? What if 365 * proc does a setuid? 366 */ 367 if (vp->v_mount != NULL && vp->v_mount->mnt_flag & MNT_NOEXEC) 368 maxprot = VM_PROT_NONE; 369 else 370 maxprot = VM_PROT_EXECUTE; 371 if (fp->f_flag & FREAD) { 372 maxprot |= VM_PROT_READ; 373 } else if (prot & PROT_READ) { 374 error = EACCES; 375 goto done; 376 } 377 /* 378 * If we are sharing potential changes (either via 379 * MAP_SHARED or via the implicit sharing of character 380 * device mappings), and we are trying to get write 381 * permission although we opened it without asking 382 * for it, bail out. 383 */ 384 if ((flags & MAP_SHARED) != 0) { 385 if ((fp->f_flag & FWRITE) != 0) { 386 maxprot |= VM_PROT_WRITE; 387 } else if ((prot & PROT_WRITE) != 0) { 388 error = EACCES; 389 goto done; 390 } 391 } else if (vp->v_type != VCHR || (fp->f_flag & FWRITE) != 0) { 392 maxprot |= VM_PROT_WRITE; 393 cap_maxprot |= VM_PROT_WRITE; 394 } 395 handle = (void *)vp; 396 handle_type = OBJT_VNODE; 397 } 398 map: 399 td->td_fpop = fp; 400 maxprot &= cap_maxprot; 401 error = vm_mmap(&vms->vm_map, &addr, size, prot, maxprot, 402 flags, handle_type, handle, pos); 403 td->td_fpop = NULL; 404 #ifdef HWPMC_HOOKS 405 /* inform hwpmc(4) if an executable is being mapped */ 406 if (error == 0 && handle_type == OBJT_VNODE && 407 (prot & PROT_EXEC)) { 408 pkm.pm_file = handle; 409 pkm.pm_address = (uintptr_t) addr; 410 PMC_CALL_HOOK(td, PMC_FN_MMAP, (void *) &pkm); 411 } 412 #endif 413 if (error == 0) 414 td->td_retval[0] = (register_t) (addr + pageoff); 415 done: 416 if (fp) 417 fdrop(fp, td); 418 419 return (error); 420 } 421 422 int 423 freebsd6_mmap(struct thread *td, struct freebsd6_mmap_args *uap) 424 { 425 struct mmap_args oargs; 426 427 oargs.addr = uap->addr; 428 oargs.len = uap->len; 429 oargs.prot = uap->prot; 430 oargs.flags = uap->flags; 431 oargs.fd = uap->fd; 432 oargs.pos = uap->pos; 433 return (sys_mmap(td, &oargs)); 434 } 435 436 #ifdef COMPAT_43 437 #ifndef _SYS_SYSPROTO_H_ 438 struct ommap_args { 439 caddr_t addr; 440 int len; 441 int prot; 442 int flags; 443 int fd; 444 long pos; 445 }; 446 #endif 447 int 448 ommap(td, uap) 449 struct thread *td; 450 struct ommap_args *uap; 451 { 452 struct mmap_args nargs; 453 static const char cvtbsdprot[8] = { 454 0, 455 PROT_EXEC, 456 PROT_WRITE, 457 PROT_EXEC | PROT_WRITE, 458 PROT_READ, 459 PROT_EXEC | PROT_READ, 460 PROT_WRITE | PROT_READ, 461 PROT_EXEC | PROT_WRITE | PROT_READ, 462 }; 463 464 #define OMAP_ANON 0x0002 465 #define OMAP_COPY 0x0020 466 #define OMAP_SHARED 0x0010 467 #define OMAP_FIXED 0x0100 468 469 nargs.addr = uap->addr; 470 nargs.len = uap->len; 471 nargs.prot = cvtbsdprot[uap->prot & 0x7]; 472 #ifdef COMPAT_FREEBSD32 473 #if defined(__amd64__) || defined(__ia64__) 474 if (i386_read_exec && SV_PROC_FLAG(td->td_proc, SV_ILP32) && 475 nargs.prot != 0) 476 nargs.prot |= PROT_EXEC; 477 #endif 478 #endif 479 nargs.flags = 0; 480 if (uap->flags & OMAP_ANON) 481 nargs.flags |= MAP_ANON; 482 if (uap->flags & OMAP_COPY) 483 nargs.flags |= MAP_COPY; 484 if (uap->flags & OMAP_SHARED) 485 nargs.flags |= MAP_SHARED; 486 else 487 nargs.flags |= MAP_PRIVATE; 488 if (uap->flags & OMAP_FIXED) 489 nargs.flags |= MAP_FIXED; 490 nargs.fd = uap->fd; 491 nargs.pos = uap->pos; 492 return (sys_mmap(td, &nargs)); 493 } 494 #endif /* COMPAT_43 */ 495 496 497 #ifndef _SYS_SYSPROTO_H_ 498 struct msync_args { 499 void *addr; 500 size_t len; 501 int flags; 502 }; 503 #endif 504 /* 505 * MPSAFE 506 */ 507 int 508 sys_msync(td, uap) 509 struct thread *td; 510 struct msync_args *uap; 511 { 512 vm_offset_t addr; 513 vm_size_t size, pageoff; 514 int flags; 515 vm_map_t map; 516 int rv; 517 518 addr = (vm_offset_t) uap->addr; 519 size = uap->len; 520 flags = uap->flags; 521 522 pageoff = (addr & PAGE_MASK); 523 addr -= pageoff; 524 size += pageoff; 525 size = (vm_size_t) round_page(size); 526 if (addr + size < addr) 527 return (EINVAL); 528 529 if ((flags & (MS_ASYNC|MS_INVALIDATE)) == (MS_ASYNC|MS_INVALIDATE)) 530 return (EINVAL); 531 532 map = &td->td_proc->p_vmspace->vm_map; 533 534 /* 535 * Clean the pages and interpret the return value. 536 */ 537 rv = vm_map_sync(map, addr, addr + size, (flags & MS_ASYNC) == 0, 538 (flags & MS_INVALIDATE) != 0); 539 switch (rv) { 540 case KERN_SUCCESS: 541 return (0); 542 case KERN_INVALID_ADDRESS: 543 return (ENOMEM); 544 case KERN_INVALID_ARGUMENT: 545 return (EBUSY); 546 case KERN_FAILURE: 547 return (EIO); 548 default: 549 return (EINVAL); 550 } 551 } 552 553 #ifndef _SYS_SYSPROTO_H_ 554 struct munmap_args { 555 void *addr; 556 size_t len; 557 }; 558 #endif 559 /* 560 * MPSAFE 561 */ 562 int 563 sys_munmap(td, uap) 564 struct thread *td; 565 struct munmap_args *uap; 566 { 567 #ifdef HWPMC_HOOKS 568 struct pmckern_map_out pkm; 569 vm_map_entry_t entry; 570 #endif 571 vm_offset_t addr; 572 vm_size_t size, pageoff; 573 vm_map_t map; 574 575 addr = (vm_offset_t) uap->addr; 576 size = uap->len; 577 if (size == 0) 578 return (EINVAL); 579 580 pageoff = (addr & PAGE_MASK); 581 addr -= pageoff; 582 size += pageoff; 583 size = (vm_size_t) round_page(size); 584 if (addr + size < addr) 585 return (EINVAL); 586 587 /* 588 * Check for illegal addresses. Watch out for address wrap... 589 */ 590 map = &td->td_proc->p_vmspace->vm_map; 591 if (addr < vm_map_min(map) || addr + size > vm_map_max(map)) 592 return (EINVAL); 593 vm_map_lock(map); 594 #ifdef HWPMC_HOOKS 595 /* 596 * Inform hwpmc if the address range being unmapped contains 597 * an executable region. 598 */ 599 pkm.pm_address = (uintptr_t) NULL; 600 if (vm_map_lookup_entry(map, addr, &entry)) { 601 for (; 602 entry != &map->header && entry->start < addr + size; 603 entry = entry->next) { 604 if (vm_map_check_protection(map, entry->start, 605 entry->end, VM_PROT_EXECUTE) == TRUE) { 606 pkm.pm_address = (uintptr_t) addr; 607 pkm.pm_size = (size_t) size; 608 break; 609 } 610 } 611 } 612 #endif 613 vm_map_delete(map, addr, addr + size); 614 615 #ifdef HWPMC_HOOKS 616 /* downgrade the lock to prevent a LOR with the pmc-sx lock */ 617 vm_map_lock_downgrade(map); 618 if (pkm.pm_address != (uintptr_t) NULL) 619 PMC_CALL_HOOK(td, PMC_FN_MUNMAP, (void *) &pkm); 620 vm_map_unlock_read(map); 621 #else 622 vm_map_unlock(map); 623 #endif 624 /* vm_map_delete returns nothing but KERN_SUCCESS anyway */ 625 return (0); 626 } 627 628 #ifndef _SYS_SYSPROTO_H_ 629 struct mprotect_args { 630 const void *addr; 631 size_t len; 632 int prot; 633 }; 634 #endif 635 /* 636 * MPSAFE 637 */ 638 int 639 sys_mprotect(td, uap) 640 struct thread *td; 641 struct mprotect_args *uap; 642 { 643 vm_offset_t addr; 644 vm_size_t size, pageoff; 645 vm_prot_t prot; 646 647 addr = (vm_offset_t) uap->addr; 648 size = uap->len; 649 prot = uap->prot & VM_PROT_ALL; 650 651 pageoff = (addr & PAGE_MASK); 652 addr -= pageoff; 653 size += pageoff; 654 size = (vm_size_t) round_page(size); 655 if (addr + size < addr) 656 return (EINVAL); 657 658 switch (vm_map_protect(&td->td_proc->p_vmspace->vm_map, addr, 659 addr + size, prot, FALSE)) { 660 case KERN_SUCCESS: 661 return (0); 662 case KERN_PROTECTION_FAILURE: 663 return (EACCES); 664 case KERN_RESOURCE_SHORTAGE: 665 return (ENOMEM); 666 } 667 return (EINVAL); 668 } 669 670 #ifndef _SYS_SYSPROTO_H_ 671 struct minherit_args { 672 void *addr; 673 size_t len; 674 int inherit; 675 }; 676 #endif 677 /* 678 * MPSAFE 679 */ 680 int 681 sys_minherit(td, uap) 682 struct thread *td; 683 struct minherit_args *uap; 684 { 685 vm_offset_t addr; 686 vm_size_t size, pageoff; 687 vm_inherit_t inherit; 688 689 addr = (vm_offset_t)uap->addr; 690 size = uap->len; 691 inherit = uap->inherit; 692 693 pageoff = (addr & PAGE_MASK); 694 addr -= pageoff; 695 size += pageoff; 696 size = (vm_size_t) round_page(size); 697 if (addr + size < addr) 698 return (EINVAL); 699 700 switch (vm_map_inherit(&td->td_proc->p_vmspace->vm_map, addr, 701 addr + size, inherit)) { 702 case KERN_SUCCESS: 703 return (0); 704 case KERN_PROTECTION_FAILURE: 705 return (EACCES); 706 } 707 return (EINVAL); 708 } 709 710 #ifndef _SYS_SYSPROTO_H_ 711 struct madvise_args { 712 void *addr; 713 size_t len; 714 int behav; 715 }; 716 #endif 717 718 /* 719 * MPSAFE 720 */ 721 /* ARGSUSED */ 722 int 723 sys_madvise(td, uap) 724 struct thread *td; 725 struct madvise_args *uap; 726 { 727 vm_offset_t start, end; 728 vm_map_t map; 729 int flags; 730 731 /* 732 * Check for our special case, advising the swap pager we are 733 * "immortal." 734 */ 735 if (uap->behav == MADV_PROTECT) { 736 flags = PPROT_SET; 737 return (kern_procctl(td, P_PID, td->td_proc->p_pid, 738 PROC_SPROTECT, &flags)); 739 } 740 741 /* 742 * Check for illegal behavior 743 */ 744 if (uap->behav < 0 || uap->behav > MADV_CORE) 745 return (EINVAL); 746 /* 747 * Check for illegal addresses. Watch out for address wrap... Note 748 * that VM_*_ADDRESS are not constants due to casts (argh). 749 */ 750 map = &td->td_proc->p_vmspace->vm_map; 751 if ((vm_offset_t)uap->addr < vm_map_min(map) || 752 (vm_offset_t)uap->addr + uap->len > vm_map_max(map)) 753 return (EINVAL); 754 if (((vm_offset_t) uap->addr + uap->len) < (vm_offset_t) uap->addr) 755 return (EINVAL); 756 757 /* 758 * Since this routine is only advisory, we default to conservative 759 * behavior. 760 */ 761 start = trunc_page((vm_offset_t) uap->addr); 762 end = round_page((vm_offset_t) uap->addr + uap->len); 763 764 if (vm_map_madvise(map, start, end, uap->behav)) 765 return (EINVAL); 766 return (0); 767 } 768 769 #ifndef _SYS_SYSPROTO_H_ 770 struct mincore_args { 771 const void *addr; 772 size_t len; 773 char *vec; 774 }; 775 #endif 776 777 /* 778 * MPSAFE 779 */ 780 /* ARGSUSED */ 781 int 782 sys_mincore(td, uap) 783 struct thread *td; 784 struct mincore_args *uap; 785 { 786 vm_offset_t addr, first_addr; 787 vm_offset_t end, cend; 788 pmap_t pmap; 789 vm_map_t map; 790 char *vec; 791 int error = 0; 792 int vecindex, lastvecindex; 793 vm_map_entry_t current; 794 vm_map_entry_t entry; 795 vm_object_t object; 796 vm_paddr_t locked_pa; 797 vm_page_t m; 798 vm_pindex_t pindex; 799 int mincoreinfo; 800 unsigned int timestamp; 801 boolean_t locked; 802 803 /* 804 * Make sure that the addresses presented are valid for user 805 * mode. 806 */ 807 first_addr = addr = trunc_page((vm_offset_t) uap->addr); 808 end = addr + (vm_size_t)round_page(uap->len); 809 map = &td->td_proc->p_vmspace->vm_map; 810 if (end > vm_map_max(map) || end < addr) 811 return (ENOMEM); 812 813 /* 814 * Address of byte vector 815 */ 816 vec = uap->vec; 817 818 pmap = vmspace_pmap(td->td_proc->p_vmspace); 819 820 vm_map_lock_read(map); 821 RestartScan: 822 timestamp = map->timestamp; 823 824 if (!vm_map_lookup_entry(map, addr, &entry)) { 825 vm_map_unlock_read(map); 826 return (ENOMEM); 827 } 828 829 /* 830 * Do this on a map entry basis so that if the pages are not 831 * in the current processes address space, we can easily look 832 * up the pages elsewhere. 833 */ 834 lastvecindex = -1; 835 for (current = entry; 836 (current != &map->header) && (current->start < end); 837 current = current->next) { 838 839 /* 840 * check for contiguity 841 */ 842 if (current->end < end && 843 (entry->next == &map->header || 844 current->next->start > current->end)) { 845 vm_map_unlock_read(map); 846 return (ENOMEM); 847 } 848 849 /* 850 * ignore submaps (for now) or null objects 851 */ 852 if ((current->eflags & MAP_ENTRY_IS_SUB_MAP) || 853 current->object.vm_object == NULL) 854 continue; 855 856 /* 857 * limit this scan to the current map entry and the 858 * limits for the mincore call 859 */ 860 if (addr < current->start) 861 addr = current->start; 862 cend = current->end; 863 if (cend > end) 864 cend = end; 865 866 /* 867 * scan this entry one page at a time 868 */ 869 while (addr < cend) { 870 /* 871 * Check pmap first, it is likely faster, also 872 * it can provide info as to whether we are the 873 * one referencing or modifying the page. 874 */ 875 object = NULL; 876 locked_pa = 0; 877 retry: 878 m = NULL; 879 mincoreinfo = pmap_mincore(pmap, addr, &locked_pa); 880 if (locked_pa != 0) { 881 /* 882 * The page is mapped by this process but not 883 * both accessed and modified. It is also 884 * managed. Acquire the object lock so that 885 * other mappings might be examined. 886 */ 887 m = PHYS_TO_VM_PAGE(locked_pa); 888 if (m->object != object) { 889 if (object != NULL) 890 VM_OBJECT_UNLOCK(object); 891 object = m->object; 892 locked = VM_OBJECT_TRYLOCK(object); 893 vm_page_unlock(m); 894 if (!locked) { 895 VM_OBJECT_LOCK(object); 896 vm_page_lock(m); 897 goto retry; 898 } 899 } else 900 vm_page_unlock(m); 901 KASSERT(m->valid == VM_PAGE_BITS_ALL, 902 ("mincore: page %p is mapped but invalid", 903 m)); 904 } else if (mincoreinfo == 0) { 905 /* 906 * The page is not mapped by this process. If 907 * the object implements managed pages, then 908 * determine if the page is resident so that 909 * the mappings might be examined. 910 */ 911 if (current->object.vm_object != object) { 912 if (object != NULL) 913 VM_OBJECT_UNLOCK(object); 914 object = current->object.vm_object; 915 VM_OBJECT_LOCK(object); 916 } 917 if (object->type == OBJT_DEFAULT || 918 object->type == OBJT_SWAP || 919 object->type == OBJT_VNODE) { 920 pindex = OFF_TO_IDX(current->offset + 921 (addr - current->start)); 922 m = vm_page_lookup(object, pindex); 923 if (m == NULL && 924 vm_page_is_cached(object, pindex)) 925 mincoreinfo = MINCORE_INCORE; 926 if (m != NULL && m->valid == 0) 927 m = NULL; 928 if (m != NULL) 929 mincoreinfo = MINCORE_INCORE; 930 } 931 } 932 if (m != NULL) { 933 /* Examine other mappings to the page. */ 934 if (m->dirty == 0 && pmap_is_modified(m)) 935 vm_page_dirty(m); 936 if (m->dirty != 0) 937 mincoreinfo |= MINCORE_MODIFIED_OTHER; 938 /* 939 * The first test for PGA_REFERENCED is an 940 * optimization. The second test is 941 * required because a concurrent pmap 942 * operation could clear the last reference 943 * and set PGA_REFERENCED before the call to 944 * pmap_is_referenced(). 945 */ 946 if ((m->aflags & PGA_REFERENCED) != 0 || 947 pmap_is_referenced(m) || 948 (m->aflags & PGA_REFERENCED) != 0) 949 mincoreinfo |= MINCORE_REFERENCED_OTHER; 950 } 951 if (object != NULL) 952 VM_OBJECT_UNLOCK(object); 953 954 /* 955 * subyte may page fault. In case it needs to modify 956 * the map, we release the lock. 957 */ 958 vm_map_unlock_read(map); 959 960 /* 961 * calculate index into user supplied byte vector 962 */ 963 vecindex = OFF_TO_IDX(addr - first_addr); 964 965 /* 966 * If we have skipped map entries, we need to make sure that 967 * the byte vector is zeroed for those skipped entries. 968 */ 969 while ((lastvecindex + 1) < vecindex) { 970 ++lastvecindex; 971 error = subyte(vec + lastvecindex, 0); 972 if (error) { 973 error = EFAULT; 974 goto done2; 975 } 976 } 977 978 /* 979 * Pass the page information to the user 980 */ 981 error = subyte(vec + vecindex, mincoreinfo); 982 if (error) { 983 error = EFAULT; 984 goto done2; 985 } 986 987 /* 988 * If the map has changed, due to the subyte, the previous 989 * output may be invalid. 990 */ 991 vm_map_lock_read(map); 992 if (timestamp != map->timestamp) 993 goto RestartScan; 994 995 lastvecindex = vecindex; 996 addr += PAGE_SIZE; 997 } 998 } 999 1000 /* 1001 * subyte may page fault. In case it needs to modify 1002 * the map, we release the lock. 1003 */ 1004 vm_map_unlock_read(map); 1005 1006 /* 1007 * Zero the last entries in the byte vector. 1008 */ 1009 vecindex = OFF_TO_IDX(end - first_addr); 1010 while ((lastvecindex + 1) < vecindex) { 1011 ++lastvecindex; 1012 error = subyte(vec + lastvecindex, 0); 1013 if (error) { 1014 error = EFAULT; 1015 goto done2; 1016 } 1017 } 1018 1019 /* 1020 * If the map has changed, due to the subyte, the previous 1021 * output may be invalid. 1022 */ 1023 vm_map_lock_read(map); 1024 if (timestamp != map->timestamp) 1025 goto RestartScan; 1026 vm_map_unlock_read(map); 1027 done2: 1028 return (error); 1029 } 1030 1031 #ifndef _SYS_SYSPROTO_H_ 1032 struct mlock_args { 1033 const void *addr; 1034 size_t len; 1035 }; 1036 #endif 1037 /* 1038 * MPSAFE 1039 */ 1040 int 1041 sys_mlock(td, uap) 1042 struct thread *td; 1043 struct mlock_args *uap; 1044 { 1045 struct proc *proc; 1046 vm_offset_t addr, end, last, start; 1047 vm_size_t npages, size; 1048 vm_map_t map; 1049 unsigned long nsize; 1050 int error; 1051 1052 error = priv_check(td, PRIV_VM_MLOCK); 1053 if (error) 1054 return (error); 1055 addr = (vm_offset_t)uap->addr; 1056 size = uap->len; 1057 last = addr + size; 1058 start = trunc_page(addr); 1059 end = round_page(last); 1060 if (last < addr || end < addr) 1061 return (EINVAL); 1062 npages = atop(end - start); 1063 if (npages > vm_page_max_wired) 1064 return (ENOMEM); 1065 proc = td->td_proc; 1066 map = &proc->p_vmspace->vm_map; 1067 PROC_LOCK(proc); 1068 nsize = ptoa(npages + pmap_wired_count(map->pmap)); 1069 if (nsize > lim_cur(proc, RLIMIT_MEMLOCK)) { 1070 PROC_UNLOCK(proc); 1071 return (ENOMEM); 1072 } 1073 PROC_UNLOCK(proc); 1074 if (npages + cnt.v_wire_count > vm_page_max_wired) 1075 return (EAGAIN); 1076 #ifdef RACCT 1077 PROC_LOCK(proc); 1078 error = racct_set(proc, RACCT_MEMLOCK, nsize); 1079 PROC_UNLOCK(proc); 1080 if (error != 0) 1081 return (ENOMEM); 1082 #endif 1083 error = vm_map_wire(map, start, end, 1084 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 1085 #ifdef RACCT 1086 if (error != KERN_SUCCESS) { 1087 PROC_LOCK(proc); 1088 racct_set(proc, RACCT_MEMLOCK, 1089 ptoa(pmap_wired_count(map->pmap))); 1090 PROC_UNLOCK(proc); 1091 } 1092 #endif 1093 return (error == KERN_SUCCESS ? 0 : ENOMEM); 1094 } 1095 1096 #ifndef _SYS_SYSPROTO_H_ 1097 struct mlockall_args { 1098 int how; 1099 }; 1100 #endif 1101 1102 /* 1103 * MPSAFE 1104 */ 1105 int 1106 sys_mlockall(td, uap) 1107 struct thread *td; 1108 struct mlockall_args *uap; 1109 { 1110 vm_map_t map; 1111 int error; 1112 1113 map = &td->td_proc->p_vmspace->vm_map; 1114 error = priv_check(td, PRIV_VM_MLOCK); 1115 if (error) 1116 return (error); 1117 1118 if ((uap->how == 0) || ((uap->how & ~(MCL_CURRENT|MCL_FUTURE)) != 0)) 1119 return (EINVAL); 1120 1121 /* 1122 * If wiring all pages in the process would cause it to exceed 1123 * a hard resource limit, return ENOMEM. 1124 */ 1125 if (!old_mlock && uap->how & MCL_CURRENT) { 1126 PROC_LOCK(td->td_proc); 1127 if (map->size > lim_cur(td->td_proc, RLIMIT_MEMLOCK)) { 1128 PROC_UNLOCK(td->td_proc); 1129 return (ENOMEM); 1130 } 1131 PROC_UNLOCK(td->td_proc); 1132 } 1133 #ifdef RACCT 1134 PROC_LOCK(td->td_proc); 1135 error = racct_set(td->td_proc, RACCT_MEMLOCK, map->size); 1136 PROC_UNLOCK(td->td_proc); 1137 if (error != 0) 1138 return (ENOMEM); 1139 #endif 1140 1141 if (uap->how & MCL_FUTURE) { 1142 vm_map_lock(map); 1143 vm_map_modflags(map, MAP_WIREFUTURE, 0); 1144 vm_map_unlock(map); 1145 error = 0; 1146 } 1147 1148 if (uap->how & MCL_CURRENT) { 1149 /* 1150 * P1003.1-2001 mandates that all currently mapped pages 1151 * will be memory resident and locked (wired) upon return 1152 * from mlockall(). vm_map_wire() will wire pages, by 1153 * calling vm_fault_wire() for each page in the region. 1154 */ 1155 error = vm_map_wire(map, vm_map_min(map), vm_map_max(map), 1156 VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK); 1157 error = (error == KERN_SUCCESS ? 0 : EAGAIN); 1158 } 1159 #ifdef RACCT 1160 if (error != KERN_SUCCESS) { 1161 PROC_LOCK(td->td_proc); 1162 racct_set(td->td_proc, RACCT_MEMLOCK, 1163 ptoa(pmap_wired_count(map->pmap))); 1164 PROC_UNLOCK(td->td_proc); 1165 } 1166 #endif 1167 1168 return (error); 1169 } 1170 1171 #ifndef _SYS_SYSPROTO_H_ 1172 struct munlockall_args { 1173 register_t dummy; 1174 }; 1175 #endif 1176 1177 /* 1178 * MPSAFE 1179 */ 1180 int 1181 sys_munlockall(td, uap) 1182 struct thread *td; 1183 struct munlockall_args *uap; 1184 { 1185 vm_map_t map; 1186 int error; 1187 1188 map = &td->td_proc->p_vmspace->vm_map; 1189 error = priv_check(td, PRIV_VM_MUNLOCK); 1190 if (error) 1191 return (error); 1192 1193 /* Clear the MAP_WIREFUTURE flag from this vm_map. */ 1194 vm_map_lock(map); 1195 vm_map_modflags(map, 0, MAP_WIREFUTURE); 1196 vm_map_unlock(map); 1197 1198 /* Forcibly unwire all pages. */ 1199 error = vm_map_unwire(map, vm_map_min(map), vm_map_max(map), 1200 VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK); 1201 #ifdef RACCT 1202 if (error == KERN_SUCCESS) { 1203 PROC_LOCK(td->td_proc); 1204 racct_set(td->td_proc, RACCT_MEMLOCK, 0); 1205 PROC_UNLOCK(td->td_proc); 1206 } 1207 #endif 1208 1209 return (error); 1210 } 1211 1212 #ifndef _SYS_SYSPROTO_H_ 1213 struct munlock_args { 1214 const void *addr; 1215 size_t len; 1216 }; 1217 #endif 1218 /* 1219 * MPSAFE 1220 */ 1221 int 1222 sys_munlock(td, uap) 1223 struct thread *td; 1224 struct munlock_args *uap; 1225 { 1226 vm_offset_t addr, end, last, start; 1227 vm_size_t size; 1228 #ifdef RACCT 1229 vm_map_t map; 1230 #endif 1231 int error; 1232 1233 error = priv_check(td, PRIV_VM_MUNLOCK); 1234 if (error) 1235 return (error); 1236 addr = (vm_offset_t)uap->addr; 1237 size = uap->len; 1238 last = addr + size; 1239 start = trunc_page(addr); 1240 end = round_page(last); 1241 if (last < addr || end < addr) 1242 return (EINVAL); 1243 error = vm_map_unwire(&td->td_proc->p_vmspace->vm_map, start, end, 1244 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 1245 #ifdef RACCT 1246 if (error == KERN_SUCCESS) { 1247 PROC_LOCK(td->td_proc); 1248 map = &td->td_proc->p_vmspace->vm_map; 1249 racct_set(td->td_proc, RACCT_MEMLOCK, 1250 ptoa(pmap_wired_count(map->pmap))); 1251 PROC_UNLOCK(td->td_proc); 1252 } 1253 #endif 1254 return (error == KERN_SUCCESS ? 0 : ENOMEM); 1255 } 1256 1257 /* 1258 * vm_mmap_vnode() 1259 * 1260 * Helper function for vm_mmap. Perform sanity check specific for mmap 1261 * operations on vnodes. 1262 * 1263 * For VCHR vnodes, the vnode lock is held over the call to 1264 * vm_mmap_cdev() to keep vp->v_rdev valid. 1265 */ 1266 int 1267 vm_mmap_vnode(struct thread *td, vm_size_t objsize, 1268 vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp, 1269 struct vnode *vp, vm_ooffset_t *foffp, vm_object_t *objp, 1270 boolean_t *writecounted) 1271 { 1272 struct vattr va; 1273 vm_object_t obj; 1274 vm_offset_t foff; 1275 struct mount *mp; 1276 struct ucred *cred; 1277 int error, flags, locktype, vfslocked; 1278 1279 mp = vp->v_mount; 1280 cred = td->td_ucred; 1281 if ((*maxprotp & VM_PROT_WRITE) && (*flagsp & MAP_SHARED)) 1282 locktype = LK_EXCLUSIVE; 1283 else 1284 locktype = LK_SHARED; 1285 vfslocked = VFS_LOCK_GIANT(mp); 1286 if ((error = vget(vp, locktype, td)) != 0) { 1287 VFS_UNLOCK_GIANT(vfslocked); 1288 return (error); 1289 } 1290 foff = *foffp; 1291 flags = *flagsp; 1292 obj = vp->v_object; 1293 if (vp->v_type == VREG) { 1294 /* 1295 * Get the proper underlying object 1296 */ 1297 if (obj == NULL) { 1298 error = EINVAL; 1299 goto done; 1300 } 1301 if (obj->handle != vp) { 1302 vput(vp); 1303 vp = (struct vnode *)obj->handle; 1304 /* 1305 * Bypass filesystems obey the mpsafety of the 1306 * underlying fs. 1307 */ 1308 error = vget(vp, locktype, td); 1309 if (error != 0) { 1310 VFS_UNLOCK_GIANT(vfslocked); 1311 return (error); 1312 } 1313 } 1314 if (locktype == LK_EXCLUSIVE) { 1315 *writecounted = TRUE; 1316 vnode_pager_update_writecount(obj, 0, objsize); 1317 } 1318 } else if (vp->v_type == VCHR) { 1319 error = vm_mmap_cdev(td, objsize, prot, maxprotp, flagsp, 1320 vp->v_rdev, foffp, objp); 1321 if (error == 0) 1322 goto mark_atime; 1323 goto done; 1324 } else { 1325 error = EINVAL; 1326 goto done; 1327 } 1328 if ((error = VOP_GETATTR(vp, &va, cred))) 1329 goto done; 1330 #ifdef MAC 1331 error = mac_vnode_check_mmap(cred, vp, prot, flags); 1332 if (error != 0) 1333 goto done; 1334 #endif 1335 if ((flags & MAP_SHARED) != 0) { 1336 if ((va.va_flags & (SF_SNAPSHOT|IMMUTABLE|APPEND)) != 0) { 1337 if (prot & PROT_WRITE) { 1338 error = EPERM; 1339 goto done; 1340 } 1341 *maxprotp &= ~VM_PROT_WRITE; 1342 } 1343 } 1344 /* 1345 * If it is a regular file without any references 1346 * we do not need to sync it. 1347 * Adjust object size to be the size of actual file. 1348 */ 1349 objsize = round_page(va.va_size); 1350 if (va.va_nlink == 0) 1351 flags |= MAP_NOSYNC; 1352 obj = vm_pager_allocate(OBJT_VNODE, vp, objsize, prot, foff, cred); 1353 if (obj == NULL) { 1354 error = ENOMEM; 1355 goto done; 1356 } 1357 *objp = obj; 1358 *flagsp = flags; 1359 1360 mark_atime: 1361 vfs_mark_atime(vp, cred); 1362 1363 done: 1364 if (error != 0 && *writecounted) { 1365 *writecounted = FALSE; 1366 vnode_pager_update_writecount(obj, objsize, 0); 1367 } 1368 vput(vp); 1369 VFS_UNLOCK_GIANT(vfslocked); 1370 return (error); 1371 } 1372 1373 /* 1374 * vm_mmap_cdev() 1375 * 1376 * MPSAFE 1377 * 1378 * Helper function for vm_mmap. Perform sanity check specific for mmap 1379 * operations on cdevs. 1380 */ 1381 int 1382 vm_mmap_cdev(struct thread *td, vm_size_t objsize, 1383 vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp, 1384 struct cdev *cdev, vm_ooffset_t *foff, vm_object_t *objp) 1385 { 1386 vm_object_t obj; 1387 struct cdevsw *dsw; 1388 int error, flags, ref; 1389 1390 flags = *flagsp; 1391 1392 dsw = dev_refthread(cdev, &ref); 1393 if (dsw == NULL) 1394 return (ENXIO); 1395 if (dsw->d_flags & D_MMAP_ANON) { 1396 dev_relthread(cdev, ref); 1397 *maxprotp = VM_PROT_ALL; 1398 *flagsp |= MAP_ANON; 1399 return (0); 1400 } 1401 /* 1402 * cdevs do not provide private mappings of any kind. 1403 */ 1404 if ((*maxprotp & VM_PROT_WRITE) == 0 && 1405 (prot & PROT_WRITE) != 0) { 1406 dev_relthread(cdev, ref); 1407 return (EACCES); 1408 } 1409 if (flags & (MAP_PRIVATE|MAP_COPY)) { 1410 dev_relthread(cdev, ref); 1411 return (EINVAL); 1412 } 1413 /* 1414 * Force device mappings to be shared. 1415 */ 1416 flags |= MAP_SHARED; 1417 #ifdef MAC_XXX 1418 error = mac_cdev_check_mmap(td->td_ucred, cdev, prot); 1419 if (error != 0) { 1420 dev_relthread(cdev, ref); 1421 return (error); 1422 } 1423 #endif 1424 /* 1425 * First, try d_mmap_single(). If that is not implemented 1426 * (returns ENODEV), fall back to using the device pager. 1427 * Note that d_mmap_single() must return a reference to the 1428 * object (it needs to bump the reference count of the object 1429 * it returns somehow). 1430 * 1431 * XXX assumes VM_PROT_* == PROT_* 1432 */ 1433 error = dsw->d_mmap_single(cdev, foff, objsize, objp, (int)prot); 1434 dev_relthread(cdev, ref); 1435 if (error != ENODEV) 1436 return (error); 1437 obj = vm_pager_allocate(OBJT_DEVICE, cdev, objsize, prot, *foff, 1438 td->td_ucred); 1439 if (obj == NULL) 1440 return (EINVAL); 1441 *objp = obj; 1442 *flagsp = flags; 1443 return (0); 1444 } 1445 1446 /* 1447 * vm_mmap_shm() 1448 * 1449 * MPSAFE 1450 * 1451 * Helper function for vm_mmap. Perform sanity check specific for mmap 1452 * operations on shm file descriptors. 1453 */ 1454 int 1455 vm_mmap_shm(struct thread *td, vm_size_t objsize, 1456 vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp, 1457 struct shmfd *shmfd, vm_ooffset_t foff, vm_object_t *objp) 1458 { 1459 int error; 1460 1461 if ((*flagsp & MAP_SHARED) != 0 && 1462 (*maxprotp & VM_PROT_WRITE) == 0 && 1463 (prot & PROT_WRITE) != 0) 1464 return (EACCES); 1465 #ifdef MAC 1466 error = mac_posixshm_check_mmap(td->td_ucred, shmfd, prot, *flagsp); 1467 if (error != 0) 1468 return (error); 1469 #endif 1470 error = shm_mmap(shmfd, objsize, foff, objp); 1471 if (error) 1472 return (error); 1473 return (0); 1474 } 1475 1476 /* 1477 * vm_mmap() 1478 * 1479 * MPSAFE 1480 * 1481 * Internal version of mmap. Currently used by mmap, exec, and sys5 1482 * shared memory. Handle is either a vnode pointer or NULL for MAP_ANON. 1483 */ 1484 int 1485 vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot, 1486 vm_prot_t maxprot, int flags, 1487 objtype_t handle_type, void *handle, 1488 vm_ooffset_t foff) 1489 { 1490 boolean_t fitit; 1491 vm_object_t object = NULL; 1492 struct thread *td = curthread; 1493 int docow, error, findspace, rv; 1494 boolean_t writecounted; 1495 1496 if (size == 0) 1497 return (0); 1498 1499 size = round_page(size); 1500 1501 PROC_LOCK(td->td_proc); 1502 if (td->td_proc->p_vmspace->vm_map.size + size > 1503 lim_cur(td->td_proc, RLIMIT_VMEM)) { 1504 if (!old_mlock && map->flags & MAP_WIREFUTURE) { 1505 if (ptoa(pmap_wired_count(map->pmap)) + size > 1506 lim_cur(td->td_proc, RLIMIT_MEMLOCK)) { 1507 racct_set_force(td->td_proc, RACCT_VMEM, 1508 map->size); 1509 PROC_UNLOCK(td->td_proc); 1510 return (ENOMEM); 1511 } 1512 error = racct_set(td->td_proc, RACCT_MEMLOCK, 1513 ptoa(pmap_wired_count(map->pmap)) + size); 1514 if (error != 0) { 1515 racct_set_force(td->td_proc, RACCT_VMEM, 1516 map->size); 1517 PROC_UNLOCK(td->td_proc); 1518 return (error); 1519 } 1520 } 1521 PROC_UNLOCK(td->td_proc); 1522 return (ENOMEM); 1523 } 1524 if (racct_set(td->td_proc, RACCT_VMEM, 1525 td->td_proc->p_vmspace->vm_map.size + size)) { 1526 PROC_UNLOCK(td->td_proc); 1527 return (ENOMEM); 1528 } 1529 PROC_UNLOCK(td->td_proc); 1530 1531 /* 1532 * We currently can only deal with page aligned file offsets. 1533 * The check is here rather than in the syscall because the 1534 * kernel calls this function internally for other mmaping 1535 * operations (such as in exec) and non-aligned offsets will 1536 * cause pmap inconsistencies...so we want to be sure to 1537 * disallow this in all cases. 1538 */ 1539 if (foff & PAGE_MASK) 1540 return (EINVAL); 1541 1542 if ((flags & MAP_FIXED) == 0) { 1543 fitit = TRUE; 1544 *addr = round_page(*addr); 1545 } else { 1546 if (*addr != trunc_page(*addr)) 1547 return (EINVAL); 1548 fitit = FALSE; 1549 } 1550 writecounted = FALSE; 1551 1552 /* 1553 * Lookup/allocate object. 1554 */ 1555 switch (handle_type) { 1556 case OBJT_DEVICE: 1557 error = vm_mmap_cdev(td, size, prot, &maxprot, &flags, 1558 handle, &foff, &object); 1559 break; 1560 case OBJT_VNODE: 1561 error = vm_mmap_vnode(td, size, prot, &maxprot, &flags, 1562 handle, &foff, &object, &writecounted); 1563 break; 1564 case OBJT_SWAP: 1565 error = vm_mmap_shm(td, size, prot, &maxprot, &flags, 1566 handle, foff, &object); 1567 break; 1568 case OBJT_DEFAULT: 1569 if (handle == NULL) { 1570 error = 0; 1571 break; 1572 } 1573 /* FALLTHROUGH */ 1574 default: 1575 error = EINVAL; 1576 break; 1577 } 1578 if (error) 1579 return (error); 1580 if (flags & MAP_ANON) { 1581 object = NULL; 1582 docow = 0; 1583 /* 1584 * Unnamed anonymous regions always start at 0. 1585 */ 1586 if (handle == 0) 1587 foff = 0; 1588 } else if (flags & MAP_PREFAULT_READ) 1589 docow = MAP_PREFAULT; 1590 else 1591 docow = MAP_PREFAULT_PARTIAL; 1592 1593 if ((flags & (MAP_ANON|MAP_SHARED)) == 0) 1594 docow |= MAP_COPY_ON_WRITE; 1595 if (flags & MAP_NOSYNC) 1596 docow |= MAP_DISABLE_SYNCER; 1597 if (flags & MAP_NOCORE) 1598 docow |= MAP_DISABLE_COREDUMP; 1599 /* Shared memory is also shared with children. */ 1600 if (flags & MAP_SHARED) 1601 docow |= MAP_INHERIT_SHARE; 1602 if (writecounted) 1603 docow |= MAP_VN_WRITECOUNT; 1604 1605 if (flags & MAP_STACK) 1606 rv = vm_map_stack(map, *addr, size, prot, maxprot, 1607 docow | MAP_STACK_GROWS_DOWN); 1608 else if (fitit) { 1609 if ((flags & MAP_ALIGNMENT_MASK) == MAP_ALIGNED_SUPER) 1610 findspace = VMFS_SUPER_SPACE; 1611 else if ((flags & MAP_ALIGNMENT_MASK) != 0) 1612 findspace = VMFS_ALIGNED_SPACE(flags >> 1613 MAP_ALIGNMENT_SHIFT); 1614 else 1615 findspace = VMFS_OPTIMAL_SPACE; 1616 rv = vm_map_find(map, object, foff, addr, size, findspace, 1617 prot, maxprot, docow); 1618 } else 1619 rv = vm_map_fixed(map, object, foff, *addr, size, 1620 prot, maxprot, docow); 1621 1622 if (rv == KERN_SUCCESS) { 1623 /* 1624 * If the process has requested that all future mappings 1625 * be wired, then heed this. 1626 */ 1627 if (map->flags & MAP_WIREFUTURE) { 1628 vm_map_wire(map, *addr, *addr + size, 1629 VM_MAP_WIRE_USER | ((flags & MAP_STACK) ? 1630 VM_MAP_WIRE_HOLESOK : VM_MAP_WIRE_NOHOLES)); 1631 } 1632 } else { 1633 /* 1634 * If this mapping was accounted for in the vnode's 1635 * writecount, then undo that now. 1636 */ 1637 if (writecounted) 1638 vnode_pager_release_writecount(object, 0, size); 1639 /* 1640 * Lose the object reference. Will destroy the 1641 * object if it's an unnamed anonymous mapping 1642 * or named anonymous without other references. 1643 */ 1644 vm_object_deallocate(object); 1645 } 1646 return (vm_mmap_to_errno(rv)); 1647 } 1648 1649 /* 1650 * Translate a Mach VM return code to zero on success or the appropriate errno 1651 * on failure. 1652 */ 1653 int 1654 vm_mmap_to_errno(int rv) 1655 { 1656 1657 switch (rv) { 1658 case KERN_SUCCESS: 1659 return (0); 1660 case KERN_INVALID_ADDRESS: 1661 case KERN_NO_SPACE: 1662 return (ENOMEM); 1663 case KERN_PROTECTION_FAILURE: 1664 return (EACCES); 1665 default: 1666 return (EINVAL); 1667 } 1668 }
Cache object: 1647983b8bba287f87f243b2ea3518fa
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]