Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/kern/sysv_shm.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* $NetBSD: sysv_shm.c,v 1.141 2019/10/09 17:47:13 chs Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, and by Mindaugas Rasiukevicius. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1994 Adam Glass and Charles M. Hannum. All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 3. All advertising materials mentioning features or use of this software 45 * must display the following acknowledgement: 46 * This product includes software developed by Adam Glass and Charles M. 47 * Hannum. 48 * 4. The names of the authors may not be used to endorse or promote products 49 * derived from this software without specific prior written permission. 50 * 51 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 52 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 53 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 54 * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, 55 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 56 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 57 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 58 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 59 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 60 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 61 */ 62 63 #include <sys/cdefs.h> 64 __KERNEL_RCSID(0, "$NetBSD: sysv_shm.c,v 1.141 2019/10/09 17:47:13 chs Exp $"); 65 66 #ifdef _KERNEL_OPT 67 #include "opt_sysv.h" 68 #endif 69 70 #include <sys/param.h> 71 #include <sys/kernel.h> 72 #include <sys/kmem.h> 73 #include <sys/shm.h> 74 #include <sys/mutex.h> 75 #include <sys/mman.h> 76 #include <sys/stat.h> 77 #include <sys/sysctl.h> 78 #include <sys/mount.h> /* XXX for <sys/syscallargs.h> */ 79 #include <sys/syscallargs.h> 80 #include <sys/queue.h> 81 #include <sys/kauth.h> 82 83 #include <uvm/uvm_extern.h> 84 #include <uvm/uvm_object.h> 85 86 struct shmmap_entry { 87 SLIST_ENTRY(shmmap_entry) next; 88 vaddr_t va; 89 int shmid; 90 }; 91 92 int shm_nused __cacheline_aligned; 93 struct shmid_ds * shmsegs __read_mostly; 94 95 static kmutex_t shm_lock __cacheline_aligned; 96 static kcondvar_t * shm_cv __cacheline_aligned; 97 static int shm_last_free __cacheline_aligned; 98 static size_t shm_committed __cacheline_aligned; 99 static int shm_use_phys __read_mostly; 100 101 static kcondvar_t shm_realloc_cv; 102 static bool shm_realloc_state; 103 static u_int shm_realloc_disable; 104 105 struct shmmap_state { 106 unsigned int nitems; 107 unsigned int nrefs; 108 SLIST_HEAD(, shmmap_entry) entries; 109 }; 110 111 extern int kern_has_sysvshm; 112 113 SYSCTL_SETUP_PROTO(sysctl_ipc_shm_setup); 114 115 #ifdef SHMDEBUG 116 #define SHMPRINTF(a) printf a 117 #else 118 #define SHMPRINTF(a) 119 #endif 120 121 static int shmrealloc(int); 122 123 /* 124 * Find the shared memory segment permission by the index. Only used by 125 * compat_linux to implement SHM_STAT. 126 */ 127 int 128 shm_find_segment_perm_by_index(int index, struct ipc_perm *perm) 129 { 130 struct shmid_ds *shmseg; 131 132 mutex_enter(&shm_lock); 133 if (index < 0 || index >= shminfo.shmmni) { 134 mutex_exit(&shm_lock); 135 return EINVAL; 136 } 137 shmseg = &shmsegs[index]; 138 memcpy(perm, &shmseg->shm_perm, sizeof(*perm)); 139 mutex_exit(&shm_lock); 140 return 0; 141 } 142 143 /* 144 * Find the shared memory segment by the identifier. 145 * => must be called with shm_lock held; 146 */ 147 static struct shmid_ds * 148 shm_find_segment_by_shmid(int shmid) 149 { 150 int segnum; 151 struct shmid_ds *shmseg; 152 153 KASSERT(mutex_owned(&shm_lock)); 154 155 segnum = IPCID_TO_IX(shmid); 156 if (segnum < 0 || segnum >= shminfo.shmmni) 157 return NULL; 158 shmseg = &shmsegs[segnum]; 159 if ((shmseg->shm_perm.mode & SHMSEG_ALLOCATED) == 0) 160 return NULL; 161 if ((shmseg->shm_perm.mode & 162 (SHMSEG_REMOVED|SHMSEG_RMLINGER)) == SHMSEG_REMOVED) 163 return NULL; 164 if (shmseg->shm_perm._seq != IPCID_TO_SEQ(shmid)) 165 return NULL; 166 167 return shmseg; 168 } 169 170 /* 171 * Free memory segment. 172 * => must be called with shm_lock held; 173 */ 174 static void 175 shm_free_segment(int segnum) 176 { 177 struct shmid_ds *shmseg; 178 size_t size; 179 bool wanted; 180 181 KASSERT(mutex_owned(&shm_lock)); 182 183 shmseg = &shmsegs[segnum]; 184 SHMPRINTF(("shm freeing key 0x%lx seq 0x%x\n", 185 shmseg->shm_perm._key, shmseg->shm_perm._seq)); 186 187 size = (shmseg->shm_segsz + PGOFSET) & ~PGOFSET; 188 wanted = (shmseg->shm_perm.mode & SHMSEG_WANTED); 189 190 shmseg->_shm_internal = NULL; 191 shm_committed -= btoc(size); 192 shm_nused--; 193 shmseg->shm_perm.mode = SHMSEG_FREE; 194 shm_last_free = segnum; 195 if (wanted == true) 196 cv_broadcast(&shm_cv[segnum]); 197 } 198 199 /* 200 * Delete entry from the shm map. 201 * => must be called with shm_lock held; 202 */ 203 static struct uvm_object * 204 shm_delete_mapping(struct shmmap_state *shmmap_s, 205 struct shmmap_entry *shmmap_se) 206 { 207 struct uvm_object *uobj = NULL; 208 struct shmid_ds *shmseg; 209 int segnum; 210 211 KASSERT(mutex_owned(&shm_lock)); 212 213 segnum = IPCID_TO_IX(shmmap_se->shmid); 214 shmseg = &shmsegs[segnum]; 215 SLIST_REMOVE(&shmmap_s->entries, shmmap_se, shmmap_entry, next); 216 shmmap_s->nitems--; 217 shmseg->shm_dtime = time_second; 218 if ((--shmseg->shm_nattch <= 0) && 219 (shmseg->shm_perm.mode & SHMSEG_REMOVED)) { 220 uobj = shmseg->_shm_internal; 221 shm_free_segment(segnum); 222 } 223 224 return uobj; 225 } 226 227 /* 228 * Get a non-shared shm map for that vmspace. Note, that memory 229 * allocation might be performed with lock held. 230 */ 231 static struct shmmap_state * 232 shmmap_getprivate(struct proc *p) 233 { 234 struct shmmap_state *oshmmap_s, *shmmap_s; 235 struct shmmap_entry *oshmmap_se, *shmmap_se; 236 237 KASSERT(mutex_owned(&shm_lock)); 238 239 /* 1. A shm map with refcnt = 1, used by ourselves, thus return */ 240 oshmmap_s = (struct shmmap_state *)p->p_vmspace->vm_shm; 241 if (oshmmap_s && oshmmap_s->nrefs == 1) 242 return oshmmap_s; 243 244 /* 2. No shm map preset - create a fresh one */ 245 shmmap_s = kmem_zalloc(sizeof(struct shmmap_state), KM_SLEEP); 246 shmmap_s->nrefs = 1; 247 SLIST_INIT(&shmmap_s->entries); 248 p->p_vmspace->vm_shm = (void *)shmmap_s; 249 250 if (oshmmap_s == NULL) 251 return shmmap_s; 252 253 SHMPRINTF(("shmmap_getprivate: vm %p split (%d entries), was used by %d\n", 254 p->p_vmspace, oshmmap_s->nitems, oshmmap_s->nrefs)); 255 256 /* 3. A shared shm map, copy to a fresh one and adjust refcounts */ 257 SLIST_FOREACH(oshmmap_se, &oshmmap_s->entries, next) { 258 shmmap_se = kmem_alloc(sizeof(struct shmmap_entry), KM_SLEEP); 259 shmmap_se->va = oshmmap_se->va; 260 shmmap_se->shmid = oshmmap_se->shmid; 261 SLIST_INSERT_HEAD(&shmmap_s->entries, shmmap_se, next); 262 } 263 shmmap_s->nitems = oshmmap_s->nitems; 264 oshmmap_s->nrefs--; 265 266 return shmmap_s; 267 } 268 269 /* 270 * Lock/unlock the memory. 271 * => must be called with shm_lock held; 272 */ 273 static int 274 shm_memlock(struct shmid_ds *shmseg, int shmid, int cmd) 275 { 276 size_t size; 277 int error; 278 279 KASSERT(mutex_owned(&shm_lock)); 280 281 size = round_page(shmseg->shm_segsz); 282 283 if (cmd == SHM_LOCK && (shmseg->shm_perm.mode & SHMSEG_WIRED) == 0) { 284 /* Wire the object and map, then tag it */ 285 error = uvm_obj_wirepages(shmseg->_shm_internal, 286 0, size, NULL); 287 if (error) 288 return EIO; 289 shmseg->shm_perm.mode |= SHMSEG_WIRED; 290 291 } else if (cmd == SHM_UNLOCK && 292 (shmseg->shm_perm.mode & SHMSEG_WIRED) != 0) { 293 /* Unwire the object, then untag it */ 294 uvm_obj_unwirepages(shmseg->_shm_internal, 0, size); 295 shmseg->shm_perm.mode &= ~SHMSEG_WIRED; 296 } 297 298 return 0; 299 } 300 301 /* 302 * Unmap shared memory. 303 */ 304 int 305 sys_shmdt(struct lwp *l, const struct sys_shmdt_args *uap, register_t *retval) 306 { 307 /* { 308 syscallarg(const void *) shmaddr; 309 } */ 310 struct proc *p = l->l_proc; 311 struct shmmap_state *shmmap_s1, *shmmap_s; 312 struct shmmap_entry *shmmap_se; 313 struct uvm_object *uobj; 314 struct shmid_ds *shmseg; 315 size_t size; 316 317 mutex_enter(&shm_lock); 318 /* In case of reallocation, we will wait for completion */ 319 while (__predict_false(shm_realloc_state)) 320 cv_wait(&shm_realloc_cv, &shm_lock); 321 322 shmmap_s1 = (struct shmmap_state *)p->p_vmspace->vm_shm; 323 if (shmmap_s1 == NULL) { 324 mutex_exit(&shm_lock); 325 return EINVAL; 326 } 327 328 /* Find the map entry */ 329 SLIST_FOREACH(shmmap_se, &shmmap_s1->entries, next) 330 if (shmmap_se->va == (vaddr_t)SCARG(uap, shmaddr)) 331 break; 332 if (shmmap_se == NULL) { 333 mutex_exit(&shm_lock); 334 return EINVAL; 335 } 336 337 shmmap_s = shmmap_getprivate(p); 338 if (shmmap_s != shmmap_s1) { 339 /* Map has been copied, lookup entry in new map */ 340 SLIST_FOREACH(shmmap_se, &shmmap_s->entries, next) 341 if (shmmap_se->va == (vaddr_t)SCARG(uap, shmaddr)) 342 break; 343 if (shmmap_se == NULL) { 344 mutex_exit(&shm_lock); 345 return EINVAL; 346 } 347 } 348 349 SHMPRINTF(("shmdt: vm %p: remove %d @%lx\n", 350 p->p_vmspace, shmmap_se->shmid, shmmap_se->va)); 351 352 /* Delete the entry from shm map */ 353 uobj = shm_delete_mapping(shmmap_s, shmmap_se); 354 shmseg = &shmsegs[IPCID_TO_IX(shmmap_se->shmid)]; 355 size = (shmseg->shm_segsz + PGOFSET) & ~PGOFSET; 356 mutex_exit(&shm_lock); 357 358 uvm_deallocate(&p->p_vmspace->vm_map, shmmap_se->va, size); 359 if (uobj != NULL) { 360 uao_detach(uobj); 361 } 362 kmem_free(shmmap_se, sizeof(struct shmmap_entry)); 363 364 return 0; 365 } 366 367 /* 368 * Map shared memory. 369 */ 370 int 371 sys_shmat(struct lwp *l, const struct sys_shmat_args *uap, register_t *retval) 372 { 373 /* { 374 syscallarg(int) shmid; 375 syscallarg(const void *) shmaddr; 376 syscallarg(int) shmflg; 377 } */ 378 int error, flags = 0; 379 struct proc *p = l->l_proc; 380 kauth_cred_t cred = l->l_cred; 381 struct shmid_ds *shmseg; 382 struct shmmap_state *shmmap_s; 383 struct shmmap_entry *shmmap_se; 384 struct uvm_object *uobj; 385 struct vmspace *vm; 386 vaddr_t attach_va; 387 vm_prot_t prot; 388 vsize_t size; 389 390 /* Allocate a new map entry and set it */ 391 shmmap_se = kmem_alloc(sizeof(struct shmmap_entry), KM_SLEEP); 392 shmmap_se->shmid = SCARG(uap, shmid); 393 394 mutex_enter(&shm_lock); 395 /* In case of reallocation, we will wait for completion */ 396 while (__predict_false(shm_realloc_state)) 397 cv_wait(&shm_realloc_cv, &shm_lock); 398 399 shmseg = shm_find_segment_by_shmid(SCARG(uap, shmid)); 400 if (shmseg == NULL) { 401 error = EINVAL; 402 goto err; 403 } 404 error = ipcperm(cred, &shmseg->shm_perm, 405 (SCARG(uap, shmflg) & SHM_RDONLY) ? IPC_R : IPC_R|IPC_W); 406 if (error) 407 goto err; 408 409 vm = p->p_vmspace; 410 shmmap_s = (struct shmmap_state *)vm->vm_shm; 411 if (shmmap_s && shmmap_s->nitems >= shminfo.shmseg) { 412 error = EMFILE; 413 goto err; 414 } 415 416 size = (shmseg->shm_segsz + PGOFSET) & ~PGOFSET; 417 prot = VM_PROT_READ; 418 if ((SCARG(uap, shmflg) & SHM_RDONLY) == 0) 419 prot |= VM_PROT_WRITE; 420 if (SCARG(uap, shmaddr)) { 421 flags |= UVM_FLAG_FIXED; 422 if (SCARG(uap, shmflg) & SHM_RND) 423 attach_va = 424 (vaddr_t)SCARG(uap, shmaddr) & ~(SHMLBA-1); 425 else if (((vaddr_t)SCARG(uap, shmaddr) & (SHMLBA-1)) == 0) 426 attach_va = (vaddr_t)SCARG(uap, shmaddr); 427 else { 428 error = EINVAL; 429 goto err; 430 } 431 } else { 432 /* This is just a hint to uvm_map() about where to put it. */ 433 attach_va = p->p_emul->e_vm_default_addr(p, 434 (vaddr_t)vm->vm_daddr, size, 435 p->p_vmspace->vm_map.flags & VM_MAP_TOPDOWN); 436 } 437 438 /* 439 * Create a map entry, add it to the list and increase the counters. 440 */ 441 shmmap_s = shmmap_getprivate(p); 442 SLIST_INSERT_HEAD(&shmmap_s->entries, shmmap_se, next); 443 shmmap_s->nitems++; 444 shmseg->shm_lpid = p->p_pid; 445 shmseg->shm_nattch++; 446 447 /* 448 * Map the segment into the address space. 449 */ 450 uobj = shmseg->_shm_internal; 451 uao_reference(uobj); 452 error = uvm_map(&vm->vm_map, &attach_va, size, uobj, 0, 0, 453 UVM_MAPFLAG(prot, prot, UVM_INH_SHARE, UVM_ADV_RANDOM, flags)); 454 if (error) 455 goto err_detach; 456 457 /* Set the new address, and update the time */ 458 shmmap_se->va = attach_va; 459 shmseg->shm_atime = time_second; 460 retval[0] = attach_va; 461 SHMPRINTF(("shmat: vm %p: add %d @%lx\n", 462 p->p_vmspace, shmmap_se->shmid, attach_va)); 463 err: 464 mutex_exit(&shm_lock); 465 if (error && shmmap_se) { 466 kmem_free(shmmap_se, sizeof(struct shmmap_entry)); 467 } 468 return error; 469 470 err_detach: 471 uao_detach(uobj); 472 uobj = shm_delete_mapping(shmmap_s, shmmap_se); 473 mutex_exit(&shm_lock); 474 if (uobj != NULL) { 475 uao_detach(uobj); 476 } 477 kmem_free(shmmap_se, sizeof(struct shmmap_entry)); 478 return error; 479 } 480 481 /* 482 * Shared memory control operations. 483 */ 484 int 485 sys___shmctl50(struct lwp *l, const struct sys___shmctl50_args *uap, 486 register_t *retval) 487 { 488 /* { 489 syscallarg(int) shmid; 490 syscallarg(int) cmd; 491 syscallarg(struct shmid_ds *) buf; 492 } */ 493 struct shmid_ds shmbuf; 494 int cmd, error; 495 496 cmd = SCARG(uap, cmd); 497 if (cmd == IPC_SET) { 498 error = copyin(SCARG(uap, buf), &shmbuf, sizeof(shmbuf)); 499 if (error) 500 return error; 501 } 502 503 error = shmctl1(l, SCARG(uap, shmid), cmd, 504 (cmd == IPC_SET || cmd == IPC_STAT) ? &shmbuf : NULL); 505 506 if (error == 0 && cmd == IPC_STAT) 507 error = copyout(&shmbuf, SCARG(uap, buf), sizeof(shmbuf)); 508 509 return error; 510 } 511 512 int 513 shmctl1(struct lwp *l, int shmid, int cmd, struct shmid_ds *shmbuf) 514 { 515 struct uvm_object *uobj = NULL; 516 kauth_cred_t cred = l->l_cred; 517 struct shmid_ds *shmseg; 518 int error = 0; 519 520 mutex_enter(&shm_lock); 521 /* In case of reallocation, we will wait for completion */ 522 while (__predict_false(shm_realloc_state)) 523 cv_wait(&shm_realloc_cv, &shm_lock); 524 525 shmseg = shm_find_segment_by_shmid(shmid); 526 if (shmseg == NULL) { 527 mutex_exit(&shm_lock); 528 return EINVAL; 529 } 530 531 switch (cmd) { 532 case IPC_STAT: 533 if ((error = ipcperm(cred, &shmseg->shm_perm, IPC_R)) != 0) 534 break; 535 memset(shmbuf, 0, sizeof *shmbuf); 536 shmbuf->shm_perm = shmseg->shm_perm; 537 shmbuf->shm_perm.mode &= 0777; 538 shmbuf->shm_segsz = shmseg->shm_segsz; 539 shmbuf->shm_lpid = shmseg->shm_lpid; 540 shmbuf->shm_cpid = shmseg->shm_cpid; 541 shmbuf->shm_nattch = shmseg->shm_nattch; 542 shmbuf->shm_atime = shmseg->shm_atime; 543 shmbuf->shm_dtime = shmseg->shm_dtime; 544 shmbuf->shm_ctime = shmseg->shm_ctime; 545 break; 546 case IPC_SET: 547 if ((error = ipcperm(cred, &shmseg->shm_perm, IPC_M)) != 0) 548 break; 549 shmseg->shm_perm.uid = shmbuf->shm_perm.uid; 550 shmseg->shm_perm.gid = shmbuf->shm_perm.gid; 551 shmseg->shm_perm.mode = 552 (shmseg->shm_perm.mode & ~ACCESSPERMS) | 553 (shmbuf->shm_perm.mode & ACCESSPERMS); 554 shmseg->shm_ctime = time_second; 555 break; 556 case IPC_RMID: 557 if ((error = ipcperm(cred, &shmseg->shm_perm, IPC_M)) != 0) 558 break; 559 shmseg->shm_perm._key = IPC_PRIVATE; 560 shmseg->shm_perm.mode |= SHMSEG_REMOVED; 561 if (shmseg->shm_nattch <= 0) { 562 uobj = shmseg->_shm_internal; 563 shm_free_segment(IPCID_TO_IX(shmid)); 564 } 565 break; 566 case SHM_LOCK: 567 case SHM_UNLOCK: 568 if ((error = kauth_authorize_system(cred, 569 KAUTH_SYSTEM_SYSVIPC, 570 (cmd == SHM_LOCK) ? KAUTH_REQ_SYSTEM_SYSVIPC_SHM_LOCK : 571 KAUTH_REQ_SYSTEM_SYSVIPC_SHM_UNLOCK, NULL, NULL, NULL)) != 0) 572 break; 573 error = shm_memlock(shmseg, shmid, cmd); 574 break; 575 default: 576 error = EINVAL; 577 } 578 579 mutex_exit(&shm_lock); 580 if (uobj != NULL) 581 uao_detach(uobj); 582 return error; 583 } 584 585 /* 586 * Try to take an already existing segment. 587 * => must be called with shm_lock held; 588 * => called from one place, thus, inline; 589 */ 590 static inline int 591 shmget_existing(struct lwp *l, const struct sys_shmget_args *uap, int mode, 592 register_t *retval) 593 { 594 struct shmid_ds *shmseg; 595 kauth_cred_t cred = l->l_cred; 596 int segnum, error; 597 again: 598 KASSERT(mutex_owned(&shm_lock)); 599 600 /* Find segment by key */ 601 for (segnum = 0; segnum < shminfo.shmmni; segnum++) 602 if ((shmsegs[segnum].shm_perm.mode & SHMSEG_ALLOCATED) && 603 shmsegs[segnum].shm_perm._key == SCARG(uap, key)) 604 break; 605 if (segnum == shminfo.shmmni) { 606 /* Not found */ 607 return -1; 608 } 609 610 shmseg = &shmsegs[segnum]; 611 if (shmseg->shm_perm.mode & SHMSEG_REMOVED) { 612 /* 613 * This segment is in the process of being allocated. Wait 614 * until it's done, and look the key up again (in case the 615 * allocation failed or it was freed). 616 */ 617 shmseg->shm_perm.mode |= SHMSEG_WANTED; 618 error = cv_wait_sig(&shm_cv[segnum], &shm_lock); 619 if (error) 620 return error; 621 goto again; 622 } 623 624 /* 625 * First check the flags, to generate a useful error when a 626 * segment already exists. 627 */ 628 if ((SCARG(uap, shmflg) & (IPC_CREAT | IPC_EXCL)) == 629 (IPC_CREAT | IPC_EXCL)) 630 return EEXIST; 631 632 /* Check the permission and segment size. */ 633 error = ipcperm(cred, &shmseg->shm_perm, mode); 634 if (error) 635 return error; 636 if (SCARG(uap, size) && SCARG(uap, size) > shmseg->shm_segsz) 637 return EINVAL; 638 639 *retval = IXSEQ_TO_IPCID(segnum, shmseg->shm_perm); 640 return 0; 641 } 642 643 int 644 sys_shmget(struct lwp *l, const struct sys_shmget_args *uap, register_t *retval) 645 { 646 /* { 647 syscallarg(key_t) key; 648 syscallarg(size_t) size; 649 syscallarg(int) shmflg; 650 } */ 651 struct shmid_ds *shmseg; 652 kauth_cred_t cred = l->l_cred; 653 key_t key = SCARG(uap, key); 654 size_t size; 655 int error, mode, segnum; 656 bool lockmem; 657 658 mode = SCARG(uap, shmflg) & ACCESSPERMS; 659 if (SCARG(uap, shmflg) & _SHM_RMLINGER) 660 mode |= SHMSEG_RMLINGER; 661 662 SHMPRINTF(("shmget: key 0x%lx size 0x%zx shmflg 0x%x mode 0x%x\n", 663 SCARG(uap, key), SCARG(uap, size), SCARG(uap, shmflg), mode)); 664 665 mutex_enter(&shm_lock); 666 /* In case of reallocation, we will wait for completion */ 667 while (__predict_false(shm_realloc_state)) 668 cv_wait(&shm_realloc_cv, &shm_lock); 669 670 if (key != IPC_PRIVATE) { 671 error = shmget_existing(l, uap, mode, retval); 672 if (error != -1) { 673 mutex_exit(&shm_lock); 674 return error; 675 } 676 if ((SCARG(uap, shmflg) & IPC_CREAT) == 0) { 677 mutex_exit(&shm_lock); 678 return ENOENT; 679 } 680 } 681 error = 0; 682 683 /* 684 * Check the for the limits. 685 */ 686 size = SCARG(uap, size); 687 if (size < shminfo.shmmin || size > shminfo.shmmax) { 688 mutex_exit(&shm_lock); 689 return EINVAL; 690 } 691 if (shm_nused >= shminfo.shmmni) { 692 mutex_exit(&shm_lock); 693 return ENOSPC; 694 } 695 size = round_page(size); 696 if (shm_committed + btoc(size) > shminfo.shmall) { 697 mutex_exit(&shm_lock); 698 return ENOMEM; 699 } 700 701 /* Find the first available segment */ 702 if (shm_last_free < 0) { 703 for (segnum = 0; segnum < shminfo.shmmni; segnum++) 704 if (shmsegs[segnum].shm_perm.mode & SHMSEG_FREE) 705 break; 706 KASSERT(segnum < shminfo.shmmni); 707 } else { 708 segnum = shm_last_free; 709 shm_last_free = -1; 710 } 711 712 /* 713 * Initialize the segment. 714 * We will drop the lock while allocating the memory, thus mark the 715 * segment present, but removed, that no other thread could take it. 716 * Also, disable reallocation, while lock is dropped. 717 */ 718 shmseg = &shmsegs[segnum]; 719 shmseg->shm_perm.mode = SHMSEG_ALLOCATED | SHMSEG_REMOVED; 720 shm_committed += btoc(size); 721 shm_nused++; 722 lockmem = shm_use_phys; 723 shm_realloc_disable++; 724 mutex_exit(&shm_lock); 725 726 /* Allocate the memory object and lock it if needed */ 727 shmseg->_shm_internal = uao_create(size, 0); 728 if (lockmem) { 729 /* Wire the pages and tag it */ 730 error = uvm_obj_wirepages(shmseg->_shm_internal, 0, size, NULL); 731 if (error) { 732 uao_detach(shmseg->_shm_internal); 733 mutex_enter(&shm_lock); 734 shm_free_segment(segnum); 735 shm_realloc_disable--; 736 mutex_exit(&shm_lock); 737 return error; 738 } 739 } 740 741 /* 742 * Please note, while segment is marked, there are no need to hold the 743 * lock, while setting it (except shm_perm.mode). 744 */ 745 shmseg->shm_perm._key = SCARG(uap, key); 746 shmseg->shm_perm._seq = (shmseg->shm_perm._seq + 1) & 0x7fff; 747 *retval = IXSEQ_TO_IPCID(segnum, shmseg->shm_perm); 748 749 shmseg->shm_perm.cuid = shmseg->shm_perm.uid = kauth_cred_geteuid(cred); 750 shmseg->shm_perm.cgid = shmseg->shm_perm.gid = kauth_cred_getegid(cred); 751 shmseg->shm_segsz = SCARG(uap, size); 752 shmseg->shm_cpid = l->l_proc->p_pid; 753 shmseg->shm_lpid = shmseg->shm_nattch = 0; 754 shmseg->shm_atime = shmseg->shm_dtime = 0; 755 shmseg->shm_ctime = time_second; 756 757 /* 758 * Segment is initialized. 759 * Enter the lock, mark as allocated, and notify waiters (if any). 760 * Also, unmark the state of reallocation. 761 */ 762 mutex_enter(&shm_lock); 763 shmseg->shm_perm.mode = (shmseg->shm_perm.mode & SHMSEG_WANTED) | 764 (mode & (ACCESSPERMS | SHMSEG_RMLINGER)) | 765 SHMSEG_ALLOCATED | (lockmem ? SHMSEG_WIRED : 0); 766 if (shmseg->shm_perm.mode & SHMSEG_WANTED) { 767 shmseg->shm_perm.mode &= ~SHMSEG_WANTED; 768 cv_broadcast(&shm_cv[segnum]); 769 } 770 shm_realloc_disable--; 771 cv_broadcast(&shm_realloc_cv); 772 mutex_exit(&shm_lock); 773 774 return error; 775 } 776 777 void 778 shmfork(struct vmspace *vm1, struct vmspace *vm2) 779 { 780 struct shmmap_state *shmmap_s; 781 struct shmmap_entry *shmmap_se; 782 783 SHMPRINTF(("shmfork %p->%p\n", vm1, vm2)); 784 mutex_enter(&shm_lock); 785 vm2->vm_shm = vm1->vm_shm; 786 if (vm1->vm_shm) { 787 shmmap_s = (struct shmmap_state *)vm1->vm_shm; 788 SLIST_FOREACH(shmmap_se, &shmmap_s->entries, next) 789 shmsegs[IPCID_TO_IX(shmmap_se->shmid)].shm_nattch++; 790 shmmap_s->nrefs++; 791 } 792 mutex_exit(&shm_lock); 793 } 794 795 void 796 shmexit(struct vmspace *vm) 797 { 798 struct shmmap_state *shmmap_s; 799 struct shmmap_entry *shmmap_se; 800 801 mutex_enter(&shm_lock); 802 shmmap_s = (struct shmmap_state *)vm->vm_shm; 803 if (shmmap_s == NULL) { 804 mutex_exit(&shm_lock); 805 return; 806 } 807 vm->vm_shm = NULL; 808 809 if (--shmmap_s->nrefs > 0) { 810 SHMPRINTF(("shmexit: vm %p drop ref (%d entries), refs = %d\n", 811 vm, shmmap_s->nitems, shmmap_s->nrefs)); 812 SLIST_FOREACH(shmmap_se, &shmmap_s->entries, next) { 813 shmsegs[IPCID_TO_IX(shmmap_se->shmid)].shm_nattch--; 814 } 815 mutex_exit(&shm_lock); 816 return; 817 } 818 819 SHMPRINTF(("shmexit: vm %p cleanup (%d entries)\n", vm, shmmap_s->nitems)); 820 if (shmmap_s->nitems == 0) { 821 mutex_exit(&shm_lock); 822 kmem_free(shmmap_s, sizeof(struct shmmap_state)); 823 return; 824 } 825 826 /* 827 * Delete the entry from shm map. 828 */ 829 for (;;) { 830 struct shmid_ds *shmseg; 831 struct uvm_object *uobj; 832 size_t sz; 833 834 shmmap_se = SLIST_FIRST(&shmmap_s->entries); 835 KASSERT(shmmap_se != NULL); 836 837 shmseg = &shmsegs[IPCID_TO_IX(shmmap_se->shmid)]; 838 sz = (shmseg->shm_segsz + PGOFSET) & ~PGOFSET; 839 /* shm_delete_mapping() removes from the list. */ 840 uobj = shm_delete_mapping(shmmap_s, shmmap_se); 841 mutex_exit(&shm_lock); 842 843 uvm_deallocate(&vm->vm_map, shmmap_se->va, sz); 844 if (uobj != NULL) { 845 uao_detach(uobj); 846 } 847 kmem_free(shmmap_se, sizeof(struct shmmap_entry)); 848 849 if (SLIST_EMPTY(&shmmap_s->entries)) { 850 break; 851 } 852 mutex_enter(&shm_lock); 853 KASSERT(!SLIST_EMPTY(&shmmap_s->entries)); 854 } 855 kmem_free(shmmap_s, sizeof(struct shmmap_state)); 856 } 857 858 static int 859 shmrealloc(int newshmni) 860 { 861 vaddr_t v; 862 struct shmid_ds *oldshmsegs, *newshmsegs; 863 kcondvar_t *newshm_cv, *oldshm_cv; 864 size_t sz; 865 int i, lsegid, oldshmni; 866 867 if (newshmni < 1) 868 return EINVAL; 869 870 /* Allocate new memory area */ 871 sz = ALIGN(newshmni * sizeof(struct shmid_ds)) + 872 ALIGN(newshmni * sizeof(kcondvar_t)); 873 sz = round_page(sz); 874 v = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 875 if (v == 0) 876 return ENOMEM; 877 878 mutex_enter(&shm_lock); 879 while (shm_realloc_state || shm_realloc_disable) 880 cv_wait(&shm_realloc_cv, &shm_lock); 881 882 /* 883 * Get the number of last segment. Fail we are trying to 884 * reallocate less memory than we use. 885 */ 886 lsegid = 0; 887 for (i = 0; i < shminfo.shmmni; i++) 888 if ((shmsegs[i].shm_perm.mode & SHMSEG_FREE) == 0) 889 lsegid = i; 890 if (lsegid >= newshmni) { 891 mutex_exit(&shm_lock); 892 uvm_km_free(kernel_map, v, sz, UVM_KMF_WIRED); 893 return EBUSY; 894 } 895 shm_realloc_state = true; 896 897 newshmsegs = (void *)v; 898 newshm_cv = (void *)((uintptr_t)newshmsegs + 899 ALIGN(newshmni * sizeof(struct shmid_ds))); 900 901 /* Copy all memory to the new area */ 902 for (i = 0; i < shm_nused; i++) { 903 cv_init(&newshm_cv[i], "shmwait"); 904 (void)memcpy(&newshmsegs[i], &shmsegs[i], 905 sizeof(newshmsegs[0])); 906 } 907 908 /* Mark as free all new segments, if there is any */ 909 for (; i < newshmni; i++) { 910 cv_init(&newshm_cv[i], "shmwait"); 911 newshmsegs[i].shm_perm.mode = SHMSEG_FREE; 912 newshmsegs[i].shm_perm._seq = 0; 913 } 914 915 oldshmsegs = shmsegs; 916 oldshmni = shminfo.shmmni; 917 shminfo.shmmni = newshmni; 918 shmsegs = newshmsegs; 919 shm_cv = newshm_cv; 920 921 /* Reallocation completed - notify all waiters, if any */ 922 shm_realloc_state = false; 923 cv_broadcast(&shm_realloc_cv); 924 mutex_exit(&shm_lock); 925 926 /* Release now unused resources. */ 927 oldshm_cv = (void *)((uintptr_t)oldshmsegs + 928 ALIGN(oldshmni * sizeof(struct shmid_ds))); 929 for (i = 0; i < oldshmni; i++) 930 cv_destroy(&oldshm_cv[i]); 931 932 sz = ALIGN(oldshmni * sizeof(struct shmid_ds)) + 933 ALIGN(oldshmni * sizeof(kcondvar_t)); 934 sz = round_page(sz); 935 uvm_km_free(kernel_map, (vaddr_t)oldshmsegs, sz, UVM_KMF_WIRED); 936 937 return 0; 938 } 939 940 int 941 shminit(void) 942 { 943 vaddr_t v; 944 size_t sz; 945 int i; 946 947 mutex_init(&shm_lock, MUTEX_DEFAULT, IPL_NONE); 948 cv_init(&shm_realloc_cv, "shmrealc"); 949 950 /* Allocate the wired memory for our structures */ 951 sz = ALIGN(shminfo.shmmni * sizeof(struct shmid_ds)) + 952 ALIGN(shminfo.shmmni * sizeof(kcondvar_t)); 953 sz = round_page(sz); 954 v = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 955 if (v == 0) { 956 printf("sysv_shm: cannot allocate memory"); 957 return ENOMEM; 958 } 959 shmsegs = (void *)v; 960 shm_cv = (void *)((uintptr_t)shmsegs + 961 ALIGN(shminfo.shmmni * sizeof(struct shmid_ds))); 962 963 if (shminfo.shmmax == 0) 964 shminfo.shmmax = uimax(physmem / 4, 1024) * PAGE_SIZE; 965 else 966 shminfo.shmmax *= PAGE_SIZE; 967 shminfo.shmall = shminfo.shmmax / PAGE_SIZE; 968 969 for (i = 0; i < shminfo.shmmni; i++) { 970 cv_init(&shm_cv[i], "shmwait"); 971 shmsegs[i].shm_perm.mode = SHMSEG_FREE; 972 shmsegs[i].shm_perm._seq = 0; 973 } 974 shm_last_free = 0; 975 shm_nused = 0; 976 shm_committed = 0; 977 shm_realloc_disable = 0; 978 shm_realloc_state = false; 979 980 kern_has_sysvshm = 1; 981 982 /* Load the callback function pointers for the uvm subsystem */ 983 uvm_shmexit = shmexit; 984 uvm_shmfork = shmfork; 985 986 return 0; 987 } 988 989 int 990 shmfini(void) 991 { 992 size_t sz; 993 int i; 994 vaddr_t v = (vaddr_t)shmsegs; 995 996 mutex_enter(&shm_lock); 997 if (shm_nused) { 998 mutex_exit(&shm_lock); 999 return 1; 1000 } 1001 1002 /* Clear the callback function pointers for the uvm subsystem */ 1003 uvm_shmexit = NULL; 1004 uvm_shmfork = NULL; 1005 1006 /* Destroy all condvars */ 1007 for (i = 0; i < shminfo.shmmni; i++) 1008 cv_destroy(&shm_cv[i]); 1009 cv_destroy(&shm_realloc_cv); 1010 1011 /* Free the allocated/wired memory */ 1012 sz = ALIGN(shminfo.shmmni * sizeof(struct shmid_ds)) + 1013 ALIGN(shminfo.shmmni * sizeof(kcondvar_t)); 1014 sz = round_page(sz); 1015 uvm_km_free(kernel_map, v, sz, UVM_KMF_WIRED); 1016 1017 /* Release and destroy our mutex */ 1018 mutex_exit(&shm_lock); 1019 mutex_destroy(&shm_lock); 1020 1021 kern_has_sysvshm = 0; 1022 1023 return 0; 1024 } 1025 1026 static int 1027 sysctl_ipc_shmmni(SYSCTLFN_ARGS) 1028 { 1029 int newsize, error; 1030 struct sysctlnode node; 1031 node = *rnode; 1032 node.sysctl_data = &newsize; 1033 1034 newsize = shminfo.shmmni; 1035 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1036 if (error || newp == NULL) 1037 return error; 1038 1039 sysctl_unlock(); 1040 error = shmrealloc(newsize); 1041 sysctl_relock(); 1042 return error; 1043 } 1044 1045 static int 1046 sysctl_ipc_shmmaxpgs(SYSCTLFN_ARGS) 1047 { 1048 uint32_t newsize; 1049 int error; 1050 struct sysctlnode node; 1051 node = *rnode; 1052 node.sysctl_data = &newsize; 1053 1054 newsize = shminfo.shmall; 1055 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1056 if (error || newp == NULL) 1057 return error; 1058 1059 if (newsize < 1) 1060 return EINVAL; 1061 1062 shminfo.shmall = newsize; 1063 shminfo.shmmax = (uint64_t)shminfo.shmall * PAGE_SIZE; 1064 1065 return 0; 1066 } 1067 1068 static int 1069 sysctl_ipc_shmmax(SYSCTLFN_ARGS) 1070 { 1071 uint64_t newsize; 1072 int error; 1073 struct sysctlnode node; 1074 node = *rnode; 1075 node.sysctl_data = &newsize; 1076 1077 newsize = shminfo.shmmax; 1078 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1079 if (error || newp == NULL) 1080 return error; 1081 1082 if (newsize < PAGE_SIZE) 1083 return EINVAL; 1084 1085 shminfo.shmmax = round_page(newsize); 1086 shminfo.shmall = shminfo.shmmax >> PAGE_SHIFT; 1087 1088 return 0; 1089 } 1090 1091 SYSCTL_SETUP(sysctl_ipc_shm_setup, "sysctl kern.ipc subtree setup") 1092 { 1093 1094 sysctl_createv(clog, 0, NULL, NULL, 1095 CTLFLAG_PERMANENT, 1096 CTLTYPE_NODE, "ipc", 1097 SYSCTL_DESCR("SysV IPC options"), 1098 NULL, 0, NULL, 0, 1099 CTL_KERN, KERN_SYSVIPC, CTL_EOL); 1100 sysctl_createv(clog, 0, NULL, NULL, 1101 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1102 CTLTYPE_QUAD, "shmmax", 1103 SYSCTL_DESCR("Max shared memory segment size in bytes"), 1104 sysctl_ipc_shmmax, 0, &shminfo.shmmax, 0, 1105 CTL_KERN, KERN_SYSVIPC, KERN_SYSVIPC_SHMMAX, CTL_EOL); 1106 sysctl_createv(clog, 0, NULL, NULL, 1107 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1108 CTLTYPE_INT, "shmmni", 1109 SYSCTL_DESCR("Max number of shared memory identifiers"), 1110 sysctl_ipc_shmmni, 0, &shminfo.shmmni, 0, 1111 CTL_KERN, KERN_SYSVIPC, KERN_SYSVIPC_SHMMNI, CTL_EOL); 1112 sysctl_createv(clog, 0, NULL, NULL, 1113 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1114 CTLTYPE_INT, "shmseg", 1115 SYSCTL_DESCR("Max shared memory segments per process"), 1116 NULL, 0, &shminfo.shmseg, 0, 1117 CTL_KERN, KERN_SYSVIPC, KERN_SYSVIPC_SHMSEG, CTL_EOL); 1118 sysctl_createv(clog, 0, NULL, NULL, 1119 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1120 CTLTYPE_INT, "shmmaxpgs", 1121 SYSCTL_DESCR("Max amount of shared memory in pages"), 1122 sysctl_ipc_shmmaxpgs, 0, &shminfo.shmall, 0, 1123 CTL_KERN, KERN_SYSVIPC, KERN_SYSVIPC_SHMMAXPGS, CTL_EOL); 1124 sysctl_createv(clog, 0, NULL, NULL, 1125 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1126 CTLTYPE_INT, "shm_use_phys", 1127 SYSCTL_DESCR("Enable/disable locking of shared memory in " 1128 "physical memory"), NULL, 0, &shm_use_phys, 0, 1129 CTL_KERN, KERN_SYSVIPC, KERN_SYSVIPC_SHMUSEPHYS, CTL_EOL); 1130 }
Cache object: a966e2d871c84b4886e69edcdcd92add
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]