Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/bsd/vm/vnode_pager.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* 2 * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_LICENSE_HEADER_START@ 5 * 6 * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved. 7 * 8 * This file contains Original Code and/or Modifications of Original Code 9 * as defined in and that are subject to the Apple Public Source License 10 * Version 2.0 (the 'License'). You may not use this file except in 11 * compliance with the License. Please obtain a copy of the License at 12 * http://www.opensource.apple.com/apsl/ and read it before using this 13 * file. 14 * 15 * The Original Code and all software distributed under the License are 16 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 17 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 18 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 20 * Please see the License for the specific language governing rights and 21 * limitations under the License. 22 * 23 * @APPLE_LICENSE_HEADER_END@ 24 */ 25 /* 26 * Mach Operating System 27 * Copyright (c) 1987 Carnegie-Mellon University 28 * All rights reserved. The CMU software License Agreement specifies 29 * the terms and conditions for use and redistribution. 30 */ 31 /* 32 * File: vnode_pager.c 33 * 34 * "Swap" pager that pages to/from vnodes. Also 35 * handles demand paging from files. 36 * 37 */ 38 39 #include <mach/boolean.h> 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/proc.h> 43 #include <sys/buf.h> 44 #include <sys/uio.h> 45 #include <sys/vnode.h> 46 #include <sys/namei.h> 47 #include <sys/mount.h> 48 #include <sys/ubc.h> 49 #include <sys/lock.h> 50 51 #include <mach/mach_types.h> 52 #include <mach/memory_object_types.h> 53 54 #include <vm/vm_map.h> 55 #include <vm/vm_kern.h> 56 #include <kern/zalloc.h> 57 #include <kern/kalloc.h> 58 #include <libkern/libkern.h> 59 60 #include <vm/vnode_pager.h> 61 #include <vm/vm_pageout.h> 62 63 #include <kern/assert.h> 64 #include <sys/kdebug.h> 65 66 unsigned int vp_pagein=0; 67 unsigned int vp_pgodirty=0; 68 unsigned int vp_pgoclean=0; 69 unsigned int dp_pgouts=0; /* Default pager pageouts */ 70 unsigned int dp_pgins=0; /* Default pager pageins */ 71 72 vm_object_offset_t 73 vnode_pager_get_filesize(struct vnode *vp) 74 { 75 if (UBCINVALID(vp)) { 76 return (vm_object_offset_t) 0; 77 } 78 79 return (vm_object_offset_t) ubc_getsize(vp); 80 81 } 82 83 pager_return_t 84 vnode_pageout(struct vnode *vp, 85 upl_t upl, 86 vm_offset_t upl_offset, 87 vm_object_offset_t f_offset, 88 vm_size_t size, 89 int flags, 90 int *errorp) 91 { 92 int result = PAGER_SUCCESS; 93 struct proc *p = current_proc(); 94 int error = 0; 95 int blkno=0, s; 96 int cnt, isize; 97 int pg_index; 98 int offset; 99 struct buf *bp; 100 boolean_t funnel_state; 101 upl_page_info_t *pl; 102 upl_t vpupl = NULL; 103 104 funnel_state = thread_funnel_set(kernel_flock, TRUE); 105 106 isize = (int)size; 107 108 if (isize <= 0) { 109 result = error = PAGER_ERROR; 110 goto out; 111 } 112 UBCINFOCHECK("vnode_pageout", vp); 113 114 if (UBCINVALID(vp)) { 115 result = error = PAGER_ERROR; 116 117 if (upl && !(flags & UPL_NOCOMMIT)) 118 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY); 119 goto out; 120 } 121 if (upl) { 122 /* 123 * This is a pageout from the Default pager, 124 * just go ahead and call VOP_PAGEOUT 125 */ 126 dp_pgouts++; 127 128 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 129 size, 1, 0, 0, 0); 130 131 if (error = VOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset, 132 (size_t)size, p->p_ucred, flags)) 133 result = error = PAGER_ERROR; 134 135 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 136 size, 1, 0, 0, 0); 137 138 goto out; 139 } 140 ubc_create_upl(vp, f_offset, isize, &vpupl, &pl, UPL_FOR_PAGEOUT | UPL_COPYOUT_FROM | UPL_SET_LITE); 141 142 if (vpupl == (upl_t) 0) { 143 result = error = PAGER_ABSENT; 144 goto out; 145 } 146 /* 147 * if we get here, we've created the upl and 148 * are responsible for commiting/aborting it 149 * regardless of what the caller has passed in 150 */ 151 flags &= ~UPL_NOCOMMIT; 152 153 if (ubc_getsize(vp) == 0) { 154 for (offset = 0; isize; isize -= PAGE_SIZE, 155 offset += PAGE_SIZE) { 156 blkno = ubc_offtoblk(vp, (off_t)f_offset); 157 f_offset += PAGE_SIZE; 158 if ((bp = incore(vp, blkno)) && 159 ISSET(bp->b_flags, B_BUSY)) { 160 ubc_upl_abort_range(vpupl, offset, PAGE_SIZE, 161 UPL_ABORT_FREE_ON_EMPTY); 162 result = error = PAGER_ERROR; 163 continue; 164 } else if (bp) { 165 bremfree(bp); 166 SET(bp->b_flags, B_BUSY | B_INVAL); 167 brelse(bp); 168 } 169 ubc_upl_commit_range(vpupl, offset, PAGE_SIZE, 170 UPL_COMMIT_FREE_ON_EMPTY); 171 } 172 goto out; 173 } 174 pg_index = 0; 175 offset = 0; 176 177 while (isize) { 178 int xsize; 179 int num_of_pages; 180 181 if ( !upl_valid_page(pl, pg_index)) { 182 ubc_upl_abort_range(vpupl, offset, PAGE_SIZE, 183 UPL_ABORT_FREE_ON_EMPTY); 184 offset += PAGE_SIZE; 185 isize -= PAGE_SIZE; 186 pg_index++; 187 188 continue; 189 } 190 if ( !upl_dirty_page(pl, pg_index)) { 191 /* 192 * if the page is not dirty and reached here it is 193 * marked precious or it is due to invalidation in 194 * memory_object_lock request as part of truncation 195 * We also get here from vm_object_terminate() 196 * So all you need to do in these 197 * cases is to invalidate incore buffer if it is there 198 * Note we must not sleep here if B_BUSY - that is 199 * a lock inversion which causes deadlock. 200 */ 201 blkno = ubc_offtoblk(vp, (off_t)(f_offset + offset)); 202 s = splbio(); 203 vp_pgoclean++; 204 if (vp->v_tag == VT_NFS) { 205 /* check with nfs if page is OK to drop */ 206 error = nfs_buf_page_inval(vp, (off_t)(f_offset + offset)); 207 splx(s); 208 if (error) { 209 ubc_upl_abort_range(vpupl, offset, PAGE_SIZE, 210 UPL_ABORT_FREE_ON_EMPTY); 211 result = error = PAGER_ERROR; 212 offset += PAGE_SIZE; 213 isize -= PAGE_SIZE; 214 pg_index++; 215 continue; 216 } 217 } else if ((bp = incore(vp, blkno)) && 218 ISSET(bp->b_flags, B_BUSY | B_NEEDCOMMIT)) { 219 splx(s); 220 ubc_upl_abort_range(vpupl, offset, PAGE_SIZE, 221 UPL_ABORT_FREE_ON_EMPTY); 222 result = error = PAGER_ERROR; 223 offset += PAGE_SIZE; 224 isize -= PAGE_SIZE; 225 pg_index++; 226 continue; 227 } else if (bp) { 228 bremfree(bp); 229 SET(bp->b_flags, B_BUSY | B_INVAL ); 230 splx(s); 231 brelse(bp); 232 } else 233 splx(s); 234 235 ubc_upl_commit_range(vpupl, offset, PAGE_SIZE, 236 UPL_COMMIT_FREE_ON_EMPTY); 237 offset += PAGE_SIZE; 238 isize -= PAGE_SIZE; 239 pg_index++; 240 241 continue; 242 } 243 vp_pgodirty++; 244 245 num_of_pages = 1; 246 xsize = isize - PAGE_SIZE; 247 248 while (xsize) { 249 if ( !upl_valid_page(pl, pg_index + num_of_pages)) 250 break; 251 if ( !upl_dirty_page(pl, pg_index + num_of_pages)) 252 break; 253 num_of_pages++; 254 xsize -= PAGE_SIZE; 255 } 256 xsize = num_of_pages * PAGE_SIZE; 257 258 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 259 xsize, 0, 0, 0, 0); 260 261 if (error = VOP_PAGEOUT(vp, vpupl, (vm_offset_t)offset, 262 (off_t)(f_offset + offset), xsize, 263 p->p_ucred, flags)) 264 result = error = PAGER_ERROR; 265 266 KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 267 xsize, 0, 0, 0, 0); 268 269 offset += xsize; 270 isize -= xsize; 271 pg_index += num_of_pages; 272 } 273 out: 274 if (errorp) 275 *errorp = result; 276 277 thread_funnel_set(kernel_flock, funnel_state); 278 279 return (error); 280 } 281 282 283 pager_return_t 284 vnode_pagein( 285 struct vnode *vp, 286 upl_t upl, 287 vm_offset_t upl_offset, 288 vm_object_offset_t f_offset, 289 vm_size_t size, 290 int flags, 291 int *errorp) 292 { 293 struct proc *p = current_proc(); 294 upl_page_info_t *pl; 295 int result = PAGER_SUCCESS; 296 int error = 0; 297 int xfer_size; 298 int pages_in_upl; 299 int start_pg; 300 int last_pg; 301 int first_pg; 302 int xsize; 303 int abort_needed = 1; 304 boolean_t funnel_state; 305 306 307 funnel_state = thread_funnel_set(kernel_flock, TRUE); 308 309 UBCINFOCHECK("vnode_pagein", vp); 310 311 if (UBCINVALID(vp)) { 312 result = PAGER_ERROR; 313 error = PAGER_ERROR; 314 if (upl && !(flags & UPL_NOCOMMIT)) { 315 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR); 316 } 317 goto out; 318 } 319 if (upl == (upl_t)NULL) { 320 if (size > (MAX_UPL_TRANSFER * PAGE_SIZE)) { 321 result = PAGER_ERROR; 322 error = PAGER_ERROR; 323 goto out; 324 } 325 ubc_create_upl(vp, f_offset, size, &upl, &pl, UPL_RET_ONLY_ABSENT | UPL_SET_LITE); 326 327 if (upl == (upl_t)NULL) { 328 result = PAGER_ABSENT; 329 error = PAGER_ABSENT; 330 goto out; 331 } 332 upl_offset = 0; 333 /* 334 * if we get here, we've created the upl and 335 * are responsible for commiting/aborting it 336 * regardless of what the caller has passed in 337 */ 338 flags &= ~UPL_NOCOMMIT; 339 340 vp_pagein++; 341 } else { 342 pl = ubc_upl_pageinfo(upl); 343 344 dp_pgins++; 345 } 346 pages_in_upl = size / PAGE_SIZE; 347 first_pg = upl_offset / PAGE_SIZE; 348 349 /* 350 * before we start marching forward, we must make sure we end on 351 * a present page, otherwise we will be working with a freed 352 * upl 353 */ 354 for (last_pg = pages_in_upl - 1; last_pg >= first_pg; last_pg--) { 355 if (upl_page_present(pl, last_pg)) 356 break; 357 } 358 pages_in_upl = last_pg + 1; 359 360 for (last_pg = first_pg; last_pg < pages_in_upl;) { 361 /* 362 * scan the upl looking for the next 363 * page that is present.... if all of the 364 * pages are absent, we're done 365 */ 366 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) { 367 if (upl_page_present(pl, last_pg)) 368 break; 369 } 370 if (last_pg == pages_in_upl) 371 break; 372 373 /* 374 * if we get here, we've sitting on a page 375 * that is present... we want to skip over 376 * any range of 'valid' pages... if this takes 377 * us to the end of the request, than we're done 378 */ 379 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) { 380 if (!upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg)) 381 break; 382 } 383 if (last_pg > start_pg) { 384 /* 385 * we've found a range of valid pages 386 * if we've got COMMIT responsibility 387 * commit this range of pages back to the 388 * cache unchanged 389 */ 390 xsize = (last_pg - start_pg) * PAGE_SIZE; 391 392 if (!(flags & UPL_NOCOMMIT)) 393 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY); 394 395 abort_needed = 0; 396 } 397 if (last_pg == pages_in_upl) 398 break; 399 400 if (!upl_page_present(pl, last_pg)) 401 /* 402 * if we found a range of valid pages 403 * terminated by a non-present page 404 * than start over 405 */ 406 continue; 407 408 /* 409 * scan from the found invalid page looking for a valid 410 * or non-present page before the end of the upl is reached, if we 411 * find one, then it will be the last page of the request to 412 * 'cluster_io' 413 */ 414 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) { 415 if (upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg)) 416 break; 417 } 418 if (last_pg > start_pg) { 419 int xoff; 420 421 xsize = (last_pg - start_pg) * PAGE_SIZE; 422 xoff = start_pg * PAGE_SIZE; 423 424 if (error = VOP_PAGEIN(vp, upl, (vm_offset_t) xoff, 425 (off_t)f_offset + xoff, 426 xsize, p->p_ucred, 427 flags)) { 428 result = PAGER_ERROR; 429 error = PAGER_ERROR; 430 431 } 432 abort_needed = 0; 433 } 434 } 435 if (!(flags & UPL_NOCOMMIT) && abort_needed) 436 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY); 437 out: 438 if (errorp) 439 *errorp = result; 440 thread_funnel_set(kernel_flock, funnel_state); 441 442 return (error); 443 } 444 445 void 446 vnode_pager_shutdown() 447 { 448 int i; 449 extern struct bs_map bs_port_table[]; 450 struct vnode *vp; 451 452 for(i = 0; i < MAX_BACKING_STORE; i++) { 453 vp = (struct vnode *)(bs_port_table[i]).vp; 454 if (vp) { 455 (bs_port_table[i]).vp = 0; 456 ubc_rele(vp); 457 /* get rid of macx_swapon() namei() reference */ 458 vrele(vp); 459 460 /* get rid of macx_swapon() "extra" reference */ 461 vrele(vp); 462 } 463 } 464 } 465 466 467 void * 468 upl_get_internal_page_list(upl_t upl) 469 { 470 return(UPL_GET_INTERNAL_PAGE_LIST(upl)); 471 472 }
Cache object: 44a4665b14016a553cf31febbb6406a9
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]