FreeBSD/Linux Kernel Cross Reference
sys/uvm/uvm_loan.c
1 /* $NetBSD: uvm_loan.c,v 1.52 2004/11/23 04:51:56 yamt Exp $ */
2
3 /*
4 *
5 * Copyright (c) 1997 Charles D. Cranor and Washington University.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. All advertising materials mentioning features or use of this software
17 * must display the following acknowledgement:
18 * This product includes software developed by Charles D. Cranor and
19 * Washington University.
20 * 4. The name of the author may not be used to endorse or promote products
21 * derived from this software without specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 *
34 * from: Id: uvm_loan.c,v 1.1.6.4 1998/02/06 05:08:43 chs Exp
35 */
36
37 /*
38 * uvm_loan.c: page loanout handler
39 */
40
41 #include <sys/cdefs.h>
42 __KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.52 2004/11/23 04:51:56 yamt Exp $");
43
44 #include <sys/param.h>
45 #include <sys/systm.h>
46 #include <sys/kernel.h>
47 #include <sys/proc.h>
48 #include <sys/malloc.h>
49 #include <sys/mman.h>
50
51 #include <uvm/uvm.h>
52
53 /*
54 * "loaned" pages are pages which are (read-only, copy-on-write) loaned
55 * from the VM system to other parts of the kernel. this allows page
56 * copying to be avoided (e.g. you can loan pages from objs/anons to
57 * the mbuf system).
58 *
59 * there are 3 types of loans possible:
60 * O->K uvm_object page to wired kernel page (e.g. mbuf data area)
61 * A->K anon page to wired kernel page (e.g. mbuf data area)
62 * O->A uvm_object to anon loan (e.g. vnode page to an anon)
63 * note that it possible to have an O page loaned to both an A and K
64 * at the same time.
65 *
66 * loans are tracked by pg->loan_count. an O->A page will have both
67 * a uvm_object and a vm_anon, but PQ_ANON will not be set. this sort
68 * of page is considered "owned" by the uvm_object (not the anon).
69 *
70 * each loan of a page to the kernel bumps the pg->wire_count. the
71 * kernel mappings for these pages will be read-only and wired. since
72 * the page will also be wired, it will not be a candidate for pageout,
73 * and thus will never be pmap_page_protect()'d with VM_PROT_NONE. a
74 * write fault in the kernel to one of these pages will not cause
75 * copy-on-write. instead, the page fault is considered fatal. this
76 * is because the kernel mapping will have no way to look up the
77 * object/anon which the page is owned by. this is a good side-effect,
78 * since a kernel write to a loaned page is an error.
79 *
80 * owners that want to free their pages and discover that they are
81 * loaned out simply "disown" them (the page becomes an orphan). these
82 * pages should be freed when the last loan is dropped. in some cases
83 * an anon may "adopt" an orphaned page.
84 *
85 * locking: to read pg->loan_count either the owner or the page queues
86 * must be locked. to modify pg->loan_count, both the owner of the page
87 * and the PQs must be locked. pg->flags is (as always) locked by
88 * the owner of the page.
89 *
90 * note that locking from the "loaned" side is tricky since the object
91 * getting the loaned page has no reference to the page's owner and thus
92 * the owner could "die" at any time. in order to prevent the owner
93 * from dying the page queues should be locked. this forces us to sometimes
94 * use "try" locking.
95 *
96 * loans are typically broken by the following events:
97 * 1. user-level xwrite fault to a loaned page
98 * 2. pageout of clean+inactive O->A loaned page
99 * 3. owner frees page (e.g. pager flush)
100 *
101 * note that loaning a page causes all mappings of the page to become
102 * read-only (via pmap_page_protect). this could have an unexpected
103 * effect on normal "wired" pages if one is not careful (XXX).
104 */
105
106 /*
107 * local prototypes
108 */
109
110 static int uvm_loananon(struct uvm_faultinfo *, void ***,
111 int, struct vm_anon *);
112 static int uvm_loanentry(struct uvm_faultinfo *, void ***, int);
113 static int uvm_loanuobj(struct uvm_faultinfo *, void ***,
114 int, vaddr_t);
115 static int uvm_loanzero(struct uvm_faultinfo *, void ***, int);
116 static void uvm_unloananon(struct vm_anon **, int);
117 static void uvm_unloanpage(struct vm_page **, int);
118 static int uvm_loanpage(struct vm_page **, int);
119
120
121 /*
122 * inlines
123 */
124
125 /*
126 * uvm_loanentry: loan out pages in a map entry (helper fn for uvm_loan())
127 *
128 * => "ufi" is the result of a successful map lookup (meaning that
129 * on entry the map is locked by the caller)
130 * => we may unlock and then relock the map if needed (for I/O)
131 * => we put our output result in "output"
132 * => we always return with the map unlocked
133 * => possible return values:
134 * -1 == error, map is unlocked
135 * 0 == map relock error (try again!), map is unlocked
136 * >0 == number of pages we loaned, map is unlocked
137 */
138
139 static __inline int
140 uvm_loanentry(ufi, output, flags)
141 struct uvm_faultinfo *ufi;
142 void ***output;
143 int flags;
144 {
145 vaddr_t curaddr = ufi->orig_rvaddr;
146 vsize_t togo = ufi->size;
147 struct vm_aref *aref = &ufi->entry->aref;
148 struct uvm_object *uobj = ufi->entry->object.uvm_obj;
149 struct vm_anon *anon;
150 int rv, result = 0;
151
152 UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
153
154 /*
155 * lock us the rest of the way down (we unlock before return)
156 */
157 if (aref->ar_amap)
158 amap_lock(aref->ar_amap);
159 if (uobj)
160 simple_lock(&uobj->vmobjlock);
161
162 /*
163 * loop until done
164 */
165 while (togo) {
166
167 /*
168 * find the page we want. check the anon layer first.
169 */
170
171 if (aref->ar_amap) {
172 anon = amap_lookup(aref, curaddr - ufi->entry->start);
173 } else {
174 anon = NULL;
175 }
176
177 /* locked: map, amap, uobj */
178 if (anon) {
179 rv = uvm_loananon(ufi, output, flags, anon);
180 } else if (uobj) {
181 rv = uvm_loanuobj(ufi, output, flags, curaddr);
182 } else if (UVM_ET_ISCOPYONWRITE(ufi->entry)) {
183 rv = uvm_loanzero(ufi, output, flags);
184 } else {
185 uvmfault_unlockall(ufi, aref->ar_amap, uobj, NULL);
186 rv = -1;
187 }
188 /* locked: if (rv > 0) => map, amap, uobj [o.w. unlocked] */
189 LOCK_ASSERT(rv > 0 || aref->ar_amap == NULL ||
190 !simple_lock_held(&aref->ar_amap->am_l));
191 LOCK_ASSERT(rv > 0 || uobj == NULL ||
192 !simple_lock_held(&uobj->vmobjlock));
193
194 /* total failure */
195 if (rv < 0) {
196 UVMHIST_LOG(loanhist, "failure %d", rv, 0,0,0);
197 return (-1);
198 }
199
200 /* relock failed, need to do another lookup */
201 if (rv == 0) {
202 UVMHIST_LOG(loanhist, "relock failure %d", result
203 ,0,0,0);
204 return (result);
205 }
206
207 /*
208 * got it... advance to next page
209 */
210
211 result++;
212 togo -= PAGE_SIZE;
213 curaddr += PAGE_SIZE;
214 }
215
216 /*
217 * unlock what we locked, unlock the maps and return
218 */
219
220 if (aref->ar_amap)
221 amap_unlock(aref->ar_amap);
222 if (uobj)
223 simple_unlock(&uobj->vmobjlock);
224 uvmfault_unlockmaps(ufi, FALSE);
225 UVMHIST_LOG(loanhist, "done %d", result, 0,0,0);
226 return (result);
227 }
228
229 /*
230 * normal functions
231 */
232
233 /*
234 * uvm_loan: loan pages in a map out to anons or to the kernel
235 *
236 * => map should be unlocked
237 * => start and len should be multiples of PAGE_SIZE
238 * => result is either an array of anon's or vm_pages (depending on flags)
239 * => flag values: UVM_LOAN_TOANON - loan to anons
240 * UVM_LOAN_TOPAGE - loan to wired kernel page
241 * one and only one of these flags must be set!
242 * => returns 0 (success), or an appropriate error number
243 */
244
245 int
246 uvm_loan(map, start, len, v, flags)
247 struct vm_map *map;
248 vaddr_t start;
249 vsize_t len;
250 void *v;
251 int flags;
252 {
253 struct uvm_faultinfo ufi;
254 void **result, **output;
255 int rv, error;
256
257 UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
258
259 /*
260 * ensure that one and only one of the flags is set
261 */
262
263 KASSERT(((flags & UVM_LOAN_TOANON) == 0) ^
264 ((flags & UVM_LOAN_TOPAGE) == 0));
265 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
266
267 /*
268 * "output" is a pointer to the current place to put the loaned page.
269 */
270
271 result = v;
272 output = &result[0]; /* start at the beginning ... */
273
274 /*
275 * while we've got pages to do
276 */
277
278 while (len > 0) {
279
280 /*
281 * fill in params for a call to uvmfault_lookup
282 */
283
284 ufi.orig_map = map;
285 ufi.orig_rvaddr = start;
286 ufi.orig_size = len;
287
288 /*
289 * do the lookup, the only time this will fail is if we hit on
290 * an unmapped region (an error)
291 */
292
293 if (!uvmfault_lookup(&ufi, FALSE)) {
294 error = ENOENT;
295 goto fail;
296 }
297
298 /*
299 * map now locked. now do the loanout...
300 */
301
302 rv = uvm_loanentry(&ufi, &output, flags);
303 if (rv < 0) {
304 /* all unlocked due to error */
305 error = EINVAL;
306 goto fail;
307 }
308
309 /*
310 * done! the map is unlocked. advance, if possible.
311 *
312 * XXXCDC: could be recoded to hold the map lock with
313 * smarter code (but it only happens on map entry
314 * boundaries, so it isn't that bad).
315 */
316
317 if (rv) {
318 rv <<= PAGE_SHIFT;
319 len -= rv;
320 start += rv;
321 }
322 }
323 UVMHIST_LOG(loanhist, "success", 0,0,0,0);
324 return 0;
325
326 fail:
327 /*
328 * failed to complete loans. drop any loans and return failure code.
329 * map is already unlocked.
330 */
331
332 if (output - result) {
333 if (flags & UVM_LOAN_TOANON) {
334 uvm_unloananon((struct vm_anon **)result,
335 output - result);
336 } else {
337 uvm_unloanpage((struct vm_page **)result,
338 output - result);
339 }
340 }
341 UVMHIST_LOG(loanhist, "error %d", error,0,0,0);
342 return (error);
343 }
344
345 /*
346 * uvm_loananon: loan a page from an anon out
347 *
348 * => called with map, amap, uobj locked
349 * => return value:
350 * -1 = fatal error, everything is unlocked, abort.
351 * 0 = lookup in ufi went stale, everything unlocked, relookup and
352 * try again
353 * 1 = got it, everything still locked
354 */
355
356 int
357 uvm_loananon(ufi, output, flags, anon)
358 struct uvm_faultinfo *ufi;
359 void ***output;
360 int flags;
361 struct vm_anon *anon;
362 {
363 struct vm_page *pg;
364 int error;
365
366 UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
367
368 /*
369 * if we are loaning to "another" anon then it is easy, we just
370 * bump the reference count on the current anon and return a
371 * pointer to it (it becomes copy-on-write shared).
372 */
373
374 if (flags & UVM_LOAN_TOANON) {
375 simple_lock(&anon->an_lock);
376 pg = anon->u.an_page;
377 if (pg && (pg->pqflags & PQ_ANON) != 0 && anon->an_ref == 1) {
378 if (pg->wire_count > 0) {
379 UVMHIST_LOG(loanhist, "->A wired %p", pg,0,0,0);
380 uvmfault_unlockall(ufi,
381 ufi->entry->aref.ar_amap,
382 ufi->entry->object.uvm_obj, anon);
383 return (-1);
384 }
385 pmap_page_protect(pg, VM_PROT_READ);
386 }
387 anon->an_ref++;
388 **output = anon;
389 (*output)++;
390 simple_unlock(&anon->an_lock);
391 UVMHIST_LOG(loanhist, "->A done", 0,0,0,0);
392 return (1);
393 }
394
395 /*
396 * we are loaning to a kernel-page. we need to get the page
397 * resident so we can wire it. uvmfault_anonget will handle
398 * this for us.
399 */
400
401 simple_lock(&anon->an_lock);
402 error = uvmfault_anonget(ufi, ufi->entry->aref.ar_amap, anon);
403
404 /*
405 * if we were unable to get the anon, then uvmfault_anonget has
406 * unlocked everything and returned an error code.
407 */
408
409 if (error) {
410 UVMHIST_LOG(loanhist, "error %d", error,0,0,0);
411
412 /* need to refault (i.e. refresh our lookup) ? */
413 if (error == ERESTART) {
414 return (0);
415 }
416
417 /* "try again"? sleep a bit and retry ... */
418 if (error == EAGAIN) {
419 tsleep(&lbolt, PVM, "loanagain", 0);
420 return (0);
421 }
422
423 /* otherwise flag it as an error */
424 return (-1);
425 }
426
427 /*
428 * we have the page and its owner locked: do the loan now.
429 */
430
431 pg = anon->u.an_page;
432 uvm_lock_pageq();
433 if (pg->wire_count > 0) {
434 uvm_unlock_pageq();
435 UVMHIST_LOG(loanhist, "->K wired %p", pg,0,0,0);
436 KASSERT(pg->uobject == NULL);
437 uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap,
438 ufi->entry->object.uvm_obj, anon);
439 return (-1);
440 }
441 if (pg->loan_count == 0) {
442 pmap_page_protect(pg, VM_PROT_READ);
443 }
444 pg->loan_count++;
445 uvm_pagedequeue(pg);
446 uvm_unlock_pageq();
447 **output = pg;
448 (*output)++;
449
450 /* unlock anon and return success */
451 if (pg->uobject) /* XXXCDC: what if this is our uobj? bad */
452 simple_unlock(&pg->uobject->vmobjlock);
453 simple_unlock(&anon->an_lock);
454 UVMHIST_LOG(loanhist, "->K done", 0,0,0,0);
455 return (1);
456 }
457
458 /*
459 * uvm_loanpage: loan out pages to kernel (->K)
460 *
461 * => pages should be object-owned and the object should be locked.
462 * => in the case of error, the object might be unlocked and relocked.
463 * => caller should busy the pages beforehand.
464 * => pages will be unbusied.
465 * => fail with EBUSY if meet a wired page.
466 */
467 static int
468 uvm_loanpage(pgpp, npages)
469 struct vm_page **pgpp;
470 int npages;
471 {
472 int i;
473 int error = 0;
474
475 UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
476
477 for (i = 0; i < npages; i++) {
478 struct vm_page *pg = pgpp[i];
479
480 KASSERT(pg->uobject != NULL);
481 KASSERT(pg->uobject == pgpp[0]->uobject);
482 KASSERT(!(pg->flags & (PG_RELEASED|PG_PAGEOUT)));
483 LOCK_ASSERT(simple_lock_held(&pg->uobject->vmobjlock));
484 KASSERT(pg->flags & PG_BUSY);
485
486 uvm_lock_pageq();
487 if (pg->wire_count > 0) {
488 uvm_unlock_pageq();
489 UVMHIST_LOG(loanhist, "wired %p", pg,0,0,0);
490 error = EBUSY;
491 break;
492 }
493 if (pg->loan_count == 0) {
494 pmap_page_protect(pg, VM_PROT_READ);
495 }
496 pg->loan_count++;
497 uvm_pagedequeue(pg);
498 uvm_unlock_pageq();
499 }
500
501 uvm_page_unbusy(pgpp, npages);
502
503 if (error) {
504 /*
505 * backout what we've done
506 */
507 struct simplelock *slock = &pgpp[0]->uobject->vmobjlock;
508
509 simple_unlock(slock);
510 uvm_unloan(pgpp, i, UVM_LOAN_TOPAGE);
511 simple_lock(slock);
512 }
513
514 UVMHIST_LOG(loanhist, "done %d", error,0,0,0);
515 return error;
516 }
517
518 /*
519 * XXX UBC temp limit
520 * number of pages to get at once.
521 * should be <= MAX_READ_AHEAD in genfs_vnops.c
522 */
523 #define UVM_LOAN_GET_CHUNK 16
524
525 /*
526 * uvm_loanuobjpages: loan pages from a uobj out (O->K)
527 *
528 * => uobj shouldn't be locked. (we'll lock it)
529 * => fail with EBUSY if we meet a wired page.
530 */
531 int
532 uvm_loanuobjpages(uobj, pgoff, orignpages, origpgpp)
533 struct uvm_object *uobj;
534 voff_t pgoff;
535 int orignpages;
536 struct vm_page **origpgpp;
537 {
538 int ndone; /* # of pages loaned out */
539 struct vm_page **pgpp;
540 int error;
541 int i;
542 struct simplelock *slock;
543
544 pgpp = origpgpp;
545 for (ndone = 0; ndone < orignpages; ) {
546 int npages;
547 /* npendloan: # of pages busied but not loand out yet. */
548 int npendloan = 0xdead; /* XXX gcc */
549 reget:
550 npages = MIN(UVM_LOAN_GET_CHUNK, orignpages - ndone);
551 simple_lock(&uobj->vmobjlock);
552 error = (*uobj->pgops->pgo_get)(uobj,
553 pgoff + (ndone << PAGE_SHIFT), pgpp, &npages, 0,
554 VM_PROT_READ, 0, PGO_SYNCIO);
555 if (error == EAGAIN) {
556 tsleep(&lbolt, PVM, "nfsread", 0);
557 continue;
558 }
559 if (error)
560 goto fail;
561
562 KASSERT(npages > 0);
563
564 /* loan and unbusy pages */
565 slock = NULL;
566 for (i = 0; i < npages; i++) {
567 struct simplelock *nextslock; /* slock for next page */
568 struct vm_page *pg = *pgpp;
569
570 /* XXX assuming that the page is owned by uobj */
571 KASSERT(pg->uobject != NULL);
572 nextslock = &pg->uobject->vmobjlock;
573
574 if (slock != nextslock) {
575 if (slock) {
576 KASSERT(npendloan > 0);
577 error = uvm_loanpage(pgpp - npendloan,
578 npendloan);
579 simple_unlock(slock);
580 if (error)
581 goto fail;
582 ndone += npendloan;
583 KASSERT(origpgpp + ndone == pgpp);
584 }
585 slock = nextslock;
586 npendloan = 0;
587 simple_lock(slock);
588 }
589
590 if ((pg->flags & PG_RELEASED) != 0) {
591 /*
592 * release pages and try again.
593 */
594 simple_unlock(slock);
595 for (; i < npages; i++) {
596 pg = pgpp[i];
597 slock = &pg->uobject->vmobjlock;
598
599 simple_lock(slock);
600 uvm_lock_pageq();
601 uvm_page_unbusy(&pg, 1);
602 uvm_unlock_pageq();
603 simple_unlock(slock);
604 }
605 goto reget;
606 }
607
608 npendloan++;
609 pgpp++;
610 KASSERT(origpgpp + ndone + npendloan == pgpp);
611 }
612 KASSERT(slock != NULL);
613 KASSERT(npendloan > 0);
614 error = uvm_loanpage(pgpp - npendloan, npendloan);
615 simple_unlock(slock);
616 if (error)
617 goto fail;
618 ndone += npendloan;
619 KASSERT(origpgpp + ndone == pgpp);
620 }
621
622 return 0;
623
624 fail:
625 uvm_unloan(origpgpp, ndone, UVM_LOAN_TOPAGE);
626
627 return error;
628 }
629
630 /*
631 * uvm_loanuobj: loan a page from a uobj out
632 *
633 * => called with map, amap, uobj locked
634 * => return value:
635 * -1 = fatal error, everything is unlocked, abort.
636 * 0 = lookup in ufi went stale, everything unlocked, relookup and
637 * try again
638 * 1 = got it, everything still locked
639 */
640
641 static int
642 uvm_loanuobj(ufi, output, flags, va)
643 struct uvm_faultinfo *ufi;
644 void ***output;
645 int flags;
646 vaddr_t va;
647 {
648 struct vm_amap *amap = ufi->entry->aref.ar_amap;
649 struct uvm_object *uobj = ufi->entry->object.uvm_obj;
650 struct vm_page *pg;
651 struct vm_anon *anon;
652 int error, npages;
653 boolean_t locked;
654
655 UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
656
657 /*
658 * first we must make sure the page is resident.
659 *
660 * XXXCDC: duplicate code with uvm_fault().
661 */
662
663 if (uobj->pgops->pgo_get) { /* try locked pgo_get */
664 npages = 1;
665 pg = NULL;
666 error = (*uobj->pgops->pgo_get)(uobj,
667 va - ufi->entry->start + ufi->entry->offset,
668 &pg, &npages, 0, VM_PROT_READ, MADV_NORMAL, PGO_LOCKED);
669 } else {
670 error = EIO; /* must have pgo_get op */
671 }
672
673 /*
674 * check the result of the locked pgo_get. if there is a problem,
675 * then we fail the loan.
676 */
677
678 if (error && error != EBUSY) {
679 uvmfault_unlockall(ufi, amap, uobj, NULL);
680 return (-1);
681 }
682
683 /*
684 * if we need to unlock for I/O, do so now.
685 */
686
687 if (error == EBUSY) {
688 uvmfault_unlockall(ufi, amap, NULL, NULL);
689
690 /* locked: uobj */
691 npages = 1;
692 error = (*uobj->pgops->pgo_get)(uobj,
693 va - ufi->entry->start + ufi->entry->offset,
694 &pg, &npages, 0, VM_PROT_READ, MADV_NORMAL, PGO_SYNCIO);
695 /* locked: <nothing> */
696
697 if (error) {
698 if (error == EAGAIN) {
699 tsleep(&lbolt, PVM, "fltagain2", 0);
700 return (0);
701 }
702 return (-1);
703 }
704
705 /*
706 * pgo_get was a success. attempt to relock everything.
707 */
708
709 locked = uvmfault_relock(ufi);
710 if (locked && amap)
711 amap_lock(amap);
712 simple_lock(&uobj->vmobjlock);
713
714 /*
715 * verify that the page has not be released and re-verify
716 * that amap slot is still free. if there is a problem we
717 * drop our lock (thus force a lookup refresh/retry).
718 */
719
720 if ((pg->flags & PG_RELEASED) != 0 ||
721 (locked && amap && amap_lookup(&ufi->entry->aref,
722 ufi->orig_rvaddr - ufi->entry->start))) {
723 if (locked)
724 uvmfault_unlockall(ufi, amap, NULL, NULL);
725 locked = FALSE;
726 }
727
728 /*
729 * didn't get the lock? release the page and retry.
730 */
731
732 if (locked == FALSE) {
733 if (pg->flags & PG_WANTED) {
734 wakeup(pg);
735 }
736 if (pg->flags & PG_RELEASED) {
737 uvm_lock_pageq();
738 uvm_pagefree(pg);
739 uvm_unlock_pageq();
740 return (0);
741 }
742 uvm_lock_pageq();
743 uvm_pageactivate(pg);
744 uvm_unlock_pageq();
745 pg->flags &= ~(PG_BUSY|PG_WANTED);
746 UVM_PAGE_OWN(pg, NULL);
747 simple_unlock(&uobj->vmobjlock);
748 return (0);
749 }
750 }
751
752 /*
753 * at this point we have the page we want ("pg") marked PG_BUSY for us
754 * and we have all data structures locked. do the loanout. page can
755 * not be PG_RELEASED (we caught this above).
756 */
757
758 if ((flags & UVM_LOAN_TOANON) == 0) {
759 if (uvm_loanpage(&pg, 1)) {
760 uvmfault_unlockall(ufi, amap, uobj, NULL);
761 return (-1);
762 }
763 **output = pg;
764 (*output)++;
765 return (1);
766 }
767
768 /*
769 * must be a loan to an anon. check to see if there is already
770 * an anon associated with this page. if so, then just return
771 * a reference to this object. the page should already be
772 * mapped read-only because it is already on loan.
773 */
774
775 if (pg->uanon) {
776 anon = pg->uanon;
777 simple_lock(&anon->an_lock);
778 anon->an_ref++;
779 simple_unlock(&anon->an_lock);
780 if (pg->flags & PG_WANTED) {
781 wakeup(pg);
782 }
783 pg->flags &= ~(PG_WANTED|PG_BUSY);
784 UVM_PAGE_OWN(pg, NULL);
785 **output = anon;
786 (*output)++;
787 return (1);
788 }
789
790 /*
791 * need to allocate a new anon
792 */
793
794 anon = uvm_analloc();
795 if (anon == NULL) {
796 goto fail;
797 }
798 anon->u.an_page = pg;
799 pg->uanon = anon;
800 uvm_lock_pageq();
801 if (pg->wire_count > 0) {
802 uvm_unlock_pageq();
803 UVMHIST_LOG(loanhist, "wired %p", pg,0,0,0);
804 pg->uanon = NULL;
805 anon->u.an_page = NULL;
806 anon->an_ref--;
807 simple_unlock(&anon->an_lock);
808 uvm_anfree(anon);
809 goto fail;
810 }
811 if (pg->loan_count == 0) {
812 pmap_page_protect(pg, VM_PROT_READ);
813 }
814 pg->loan_count++;
815 uvm_pageactivate(pg);
816 uvm_unlock_pageq();
817 if (pg->flags & PG_WANTED) {
818 wakeup(pg);
819 }
820 pg->flags &= ~(PG_WANTED|PG_BUSY);
821 UVM_PAGE_OWN(pg, NULL);
822 simple_unlock(&anon->an_lock);
823 **output = anon;
824 (*output)++;
825 return (1);
826
827 fail:
828 UVMHIST_LOG(loanhist, "fail", 0,0,0,0);
829 /*
830 * unlock everything and bail out.
831 */
832 if (pg->flags & PG_WANTED) {
833 wakeup(pg);
834 }
835 pg->flags &= ~(PG_WANTED|PG_BUSY);
836 UVM_PAGE_OWN(pg, NULL);
837 uvmfault_unlockall(ufi, amap, uobj, NULL);
838 return (-1);
839 }
840
841 /*
842 * uvm_loanzero: loan a zero-fill page out
843 *
844 * => called with map, amap, uobj locked
845 * => return value:
846 * -1 = fatal error, everything is unlocked, abort.
847 * 0 = lookup in ufi went stale, everything unlocked, relookup and
848 * try again
849 * 1 = got it, everything still locked
850 */
851
852 static struct uvm_object uvm_loanzero_object;
853
854 static int
855 uvm_loanzero(ufi, output, flags)
856 struct uvm_faultinfo *ufi;
857 void ***output;
858 int flags;
859 {
860 struct vm_anon *anon;
861 struct vm_page *pg;
862 struct uvm_object *uobj = ufi->entry->object.uvm_obj;
863 struct vm_amap *amap = ufi->entry->aref.ar_amap;
864
865 UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
866 again:
867 simple_lock(&uvm_loanzero_object.vmobjlock);
868
869 /*
870 * first, get ahold of our single zero page.
871 */
872
873 if (__predict_false((pg =
874 TAILQ_FIRST(&uvm_loanzero_object.memq)) == NULL)) {
875 while ((pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL,
876 UVM_PGA_ZERO)) == NULL) {
877 simple_unlock(&uvm_loanzero_object.vmobjlock);
878 uvmfault_unlockall(ufi, amap, uobj, NULL);
879 uvm_wait("loanzero");
880 if (!uvmfault_relock(ufi)) {
881 return (0);
882 }
883 if (amap) {
884 amap_lock(amap);
885 }
886 if (uobj) {
887 simple_lock(&uobj->vmobjlock);
888 }
889 goto again;
890 }
891
892 /* got a zero'd page. */
893 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
894 pg->flags |= PG_RDONLY;
895 uvm_lock_pageq();
896 uvm_pageactivate(pg);
897 uvm_unlock_pageq();
898 UVM_PAGE_OWN(pg, NULL);
899 }
900
901 if ((flags & UVM_LOAN_TOANON) == 0) { /* loaning to kernel-page */
902 uvm_lock_pageq();
903 pg->loan_count++;
904 uvm_pagedequeue(pg);
905 uvm_unlock_pageq();
906 simple_unlock(&uvm_loanzero_object.vmobjlock);
907 **output = pg;
908 (*output)++;
909 return (1);
910 }
911
912 /*
913 * loaning to an anon. check to see if there is already an anon
914 * associated with this page. if so, then just return a reference
915 * to this object.
916 */
917
918 if (pg->uanon) {
919 anon = pg->uanon;
920 simple_lock(&anon->an_lock);
921 anon->an_ref++;
922 simple_unlock(&anon->an_lock);
923 simple_unlock(&uvm_loanzero_object.vmobjlock);
924 **output = anon;
925 (*output)++;
926 return (1);
927 }
928
929 /*
930 * need to allocate a new anon
931 */
932
933 anon = uvm_analloc();
934 if (anon == NULL) {
935 /* out of swap causes us to fail */
936 simple_unlock(&uvm_loanzero_object.vmobjlock);
937 uvmfault_unlockall(ufi, amap, uobj, NULL);
938 return (-1);
939 }
940 anon->u.an_page = pg;
941 pg->uanon = anon;
942 uvm_lock_pageq();
943 pg->loan_count++;
944 uvm_pageactivate(pg);
945 uvm_unlock_pageq();
946 simple_unlock(&uvm_loanzero_object.vmobjlock);
947 **output = anon;
948 (*output)++;
949 return (1);
950 }
951
952
953 /*
954 * uvm_unloananon: kill loans on anons (basically a normal ref drop)
955 *
956 * => we expect all our resources to be unlocked
957 */
958
959 static void
960 uvm_unloananon(aloans, nanons)
961 struct vm_anon **aloans;
962 int nanons;
963 {
964 struct vm_anon *anon;
965
966 while (nanons-- > 0) {
967 int refs;
968
969 anon = *aloans++;
970 simple_lock(&anon->an_lock);
971 refs = --anon->an_ref;
972 simple_unlock(&anon->an_lock);
973
974 if (refs == 0) {
975 uvm_anfree(anon);
976 }
977 }
978 }
979
980 /*
981 * uvm_unloanpage: kill loans on pages loaned out to the kernel
982 *
983 * => we expect all our resources to be unlocked
984 */
985
986 static void
987 uvm_unloanpage(ploans, npages)
988 struct vm_page **ploans;
989 int npages;
990 {
991 struct vm_page *pg;
992 struct simplelock *slock;
993
994 uvm_lock_pageq();
995 while (npages-- > 0) {
996 pg = *ploans++;
997
998 /*
999 * do a little dance to acquire the object or anon lock
1000 * as appropriate. we are locking in the wrong order,
1001 * so we have to do a try-lock here.
1002 */
1003
1004 slock = NULL;
1005 while (pg->uobject != NULL || pg->uanon != NULL) {
1006 if (pg->uobject != NULL) {
1007 slock = &pg->uobject->vmobjlock;
1008 } else {
1009 slock = &pg->uanon->an_lock;
1010 }
1011 if (simple_lock_try(slock)) {
1012 break;
1013 }
1014 uvm_unlock_pageq();
1015 uvm_lock_pageq();
1016 slock = NULL;
1017 }
1018
1019 /*
1020 * drop our loan. if page is owned by an anon but
1021 * PQ_ANON is not set, the page was loaned to the anon
1022 * from an object which dropped ownership, so resolve
1023 * this by turning the anon's loan into real ownership
1024 * (ie. decrement loan_count again and set PQ_ANON).
1025 * after all this, if there are no loans left, put the
1026 * page back a paging queue (if the page is owned by
1027 * an anon) or free it (if the page is now unowned).
1028 */
1029
1030 KASSERT(pg->loan_count > 0);
1031 pg->loan_count--;
1032 if (pg->uobject == NULL && pg->uanon != NULL &&
1033 (pg->pqflags & PQ_ANON) == 0) {
1034 KASSERT(pg->loan_count > 0);
1035 pg->loan_count--;
1036 pg->pqflags |= PQ_ANON;
1037 }
1038 if (pg->loan_count == 0) {
1039 if (pg->uobject == NULL && pg->uanon == NULL) {
1040 KASSERT((pg->flags & PG_BUSY) == 0);
1041 uvm_pagefree(pg);
1042 } else {
1043 uvm_pageactivate(pg);
1044 }
1045 } else if (pg->loan_count == 1 && pg->uobject != NULL &&
1046 pg->uanon != NULL) {
1047 uvm_pageactivate(pg);
1048 }
1049 if (slock != NULL) {
1050 simple_unlock(slock);
1051 }
1052 }
1053 uvm_unlock_pageq();
1054 }
1055
1056 /*
1057 * uvm_unloan: kill loans on pages or anons.
1058 */
1059
1060 void
1061 uvm_unloan(void *v, int npages, int flags)
1062 {
1063 if (flags & UVM_LOAN_TOANON) {
1064 uvm_unloananon(v, npages);
1065 } else {
1066 uvm_unloanpage(v, npages);
1067 }
1068 }
1069
1070 /*
1071 * Minimal pager for uvm_loanzero_object. We need to provide a "put"
1072 * method, because the page can end up on a paging queue, and the
1073 * page daemon will want to call pgo_put when it encounters the page
1074 * on the inactive list.
1075 */
1076
1077 static int
1078 ulz_put(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
1079 {
1080 struct vm_page *pg;
1081
1082 KDASSERT(uobj == &uvm_loanzero_object);
1083
1084 /*
1085 * Don't need to do any work here if we're not freeing pages.
1086 */
1087
1088 if ((flags & PGO_FREE) == 0) {
1089 simple_unlock(&uobj->vmobjlock);
1090 return 0;
1091 }
1092
1093 /*
1094 * we don't actually want to ever free the uvm_loanzero_page, so
1095 * just reactivate or dequeue it.
1096 */
1097
1098 pg = TAILQ_FIRST(&uobj->memq);
1099 KASSERT(pg != NULL);
1100 KASSERT(TAILQ_NEXT(pg, listq) == NULL);
1101
1102 uvm_lock_pageq();
1103 if (pg->uanon)
1104 uvm_pageactivate(pg);
1105 else
1106 uvm_pagedequeue(pg);
1107 uvm_unlock_pageq();
1108
1109 simple_unlock(&uobj->vmobjlock);
1110 return 0;
1111 }
1112
1113 static struct uvm_pagerops ulz_pager = {
1114 NULL, /* init */
1115 NULL, /* reference */
1116 NULL, /* detach */
1117 NULL, /* fault */
1118 NULL, /* get */
1119 ulz_put, /* put */
1120 };
1121
1122 /*
1123 * uvm_loan_init(): initialize the uvm_loan() facility.
1124 */
1125
1126 void
1127 uvm_loan_init(void)
1128 {
1129
1130 simple_lock_init(&uvm_loanzero_object.vmobjlock);
1131 TAILQ_INIT(&uvm_loanzero_object.memq);
1132 uvm_loanzero_object.pgops = &ulz_pager;
1133
1134 UVMHIST_INIT(loanhist, 300);
1135 }
1136
1137 /*
1138 * uvm_loanbreak: break loan on a uobj page
1139 *
1140 * => called with uobj locked
1141 * => the page should be busy
1142 * => return value:
1143 * newly allocated page if succeeded
1144 */
1145 struct vm_page *
1146 uvm_loanbreak(struct vm_page *uobjpage)
1147 {
1148 struct vm_page *pg;
1149 #ifdef DIAGNOSTIC
1150 struct uvm_object *uobj = uobjpage->uobject;
1151 #endif
1152
1153 KASSERT(uobj != NULL);
1154 LOCK_ASSERT(simple_lock_held(&uobj->vmobjlock));
1155 KASSERT(uobjpage->flags & PG_BUSY);
1156
1157 /* alloc new un-owned page */
1158 pg = uvm_pagealloc(NULL, 0, NULL, 0);
1159 if (pg == NULL)
1160 return NULL;
1161
1162 /*
1163 * copy the data from the old page to the new
1164 * one and clear the fake/clean flags on the
1165 * new page (keep it busy). force a reload
1166 * of the old page by clearing it from all
1167 * pmaps. then lock the page queues to
1168 * rename the pages.
1169 */
1170
1171 uvm_pagecopy(uobjpage, pg); /* old -> new */
1172 pg->flags &= ~(PG_FAKE|PG_CLEAN);
1173 pmap_page_protect(uobjpage, VM_PROT_NONE);
1174 if (uobjpage->flags & PG_WANTED)
1175 wakeup(uobjpage);
1176 /* uobj still locked */
1177 uobjpage->flags &= ~(PG_WANTED|PG_BUSY);
1178 UVM_PAGE_OWN(uobjpage, NULL);
1179
1180 uvm_lock_pageq();
1181
1182 /*
1183 * replace uobjpage with new page.
1184 */
1185
1186 uvm_pagereplace(uobjpage, pg);
1187
1188 /*
1189 * if the page is no longer referenced by
1190 * an anon (i.e. we are breaking an O->K
1191 * loan), then remove it from any pageq's.
1192 */
1193 if (uobjpage->uanon == NULL)
1194 uvm_pagedequeue(uobjpage);
1195
1196 /*
1197 * at this point we have absolutely no
1198 * control over uobjpage
1199 */
1200
1201 /* install new page */
1202 uvm_pageactivate(pg);
1203 uvm_unlock_pageq();
1204
1205 /*
1206 * done! loan is broken and "pg" is
1207 * PG_BUSY. it can now replace uobjpage.
1208 */
1209
1210 return pg;
1211 }
Cache object: a18044c7f381d6ea4ddc290d45ee003d
|