FreeBSD/Linux Kernel Cross Reference
sys/uvm/uvm_loan.c
1 /* $NetBSD: uvm_loan.c,v 1.79 2011/04/23 18:14:12 rmind Exp $ */
2
3 /*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 *
27 * from: Id: uvm_loan.c,v 1.1.6.4 1998/02/06 05:08:43 chs Exp
28 */
29
30 /*
31 * uvm_loan.c: page loanout handler
32 */
33
34 #include <sys/cdefs.h>
35 __KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.79 2011/04/23 18:14:12 rmind Exp $");
36
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/kernel.h>
40 #include <sys/mman.h>
41
42 #include <uvm/uvm.h>
43
44 /*
45 * "loaned" pages are pages which are (read-only, copy-on-write) loaned
46 * from the VM system to other parts of the kernel. this allows page
47 * copying to be avoided (e.g. you can loan pages from objs/anons to
48 * the mbuf system).
49 *
50 * there are 3 types of loans possible:
51 * O->K uvm_object page to wired kernel page (e.g. mbuf data area)
52 * A->K anon page to wired kernel page (e.g. mbuf data area)
53 * O->A uvm_object to anon loan (e.g. vnode page to an anon)
54 * note that it possible to have an O page loaned to both an A and K
55 * at the same time.
56 *
57 * loans are tracked by pg->loan_count. an O->A page will have both
58 * a uvm_object and a vm_anon, but PQ_ANON will not be set. this sort
59 * of page is considered "owned" by the uvm_object (not the anon).
60 *
61 * each loan of a page to the kernel bumps the pg->wire_count. the
62 * kernel mappings for these pages will be read-only and wired. since
63 * the page will also be wired, it will not be a candidate for pageout,
64 * and thus will never be pmap_page_protect()'d with VM_PROT_NONE. a
65 * write fault in the kernel to one of these pages will not cause
66 * copy-on-write. instead, the page fault is considered fatal. this
67 * is because the kernel mapping will have no way to look up the
68 * object/anon which the page is owned by. this is a good side-effect,
69 * since a kernel write to a loaned page is an error.
70 *
71 * owners that want to free their pages and discover that they are
72 * loaned out simply "disown" them (the page becomes an orphan). these
73 * pages should be freed when the last loan is dropped. in some cases
74 * an anon may "adopt" an orphaned page.
75 *
76 * locking: to read pg->loan_count either the owner or the page queues
77 * must be locked. to modify pg->loan_count, both the owner of the page
78 * and the PQs must be locked. pg->flags is (as always) locked by
79 * the owner of the page.
80 *
81 * note that locking from the "loaned" side is tricky since the object
82 * getting the loaned page has no reference to the page's owner and thus
83 * the owner could "die" at any time. in order to prevent the owner
84 * from dying the page queues should be locked. this forces us to sometimes
85 * use "try" locking.
86 *
87 * loans are typically broken by the following events:
88 * 1. user-level xwrite fault to a loaned page
89 * 2. pageout of clean+inactive O->A loaned page
90 * 3. owner frees page (e.g. pager flush)
91 *
92 * note that loaning a page causes all mappings of the page to become
93 * read-only (via pmap_page_protect). this could have an unexpected
94 * effect on normal "wired" pages if one is not careful (XXX).
95 */
96
97 /*
98 * local prototypes
99 */
100
101 static int uvm_loananon(struct uvm_faultinfo *, void ***,
102 int, struct vm_anon *);
103 static int uvm_loanuobj(struct uvm_faultinfo *, void ***,
104 int, vaddr_t);
105 static int uvm_loanzero(struct uvm_faultinfo *, void ***, int);
106 static void uvm_unloananon(struct vm_anon **, int);
107 static void uvm_unloanpage(struct vm_page **, int);
108 static int uvm_loanpage(struct vm_page **, int);
109
110
111 /*
112 * inlines
113 */
114
115 /*
116 * uvm_loanentry: loan out pages in a map entry (helper fn for uvm_loan())
117 *
118 * => "ufi" is the result of a successful map lookup (meaning that
119 * on entry the map is locked by the caller)
120 * => we may unlock and then relock the map if needed (for I/O)
121 * => we put our output result in "output"
122 * => we always return with the map unlocked
123 * => possible return values:
124 * -1 == error, map is unlocked
125 * 0 == map relock error (try again!), map is unlocked
126 * >0 == number of pages we loaned, map is unlocked
127 *
128 * NOTE: We can live with this being an inline, because it is only called
129 * from one place.
130 */
131
132 static inline int
133 uvm_loanentry(struct uvm_faultinfo *ufi, void ***output, int flags)
134 {
135 vaddr_t curaddr = ufi->orig_rvaddr;
136 vsize_t togo = ufi->size;
137 struct vm_aref *aref = &ufi->entry->aref;
138 struct uvm_object *uobj = ufi->entry->object.uvm_obj;
139 struct vm_anon *anon;
140 int rv, result = 0;
141
142 UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
143
144 /*
145 * lock us the rest of the way down (we unlock before return)
146 */
147 if (aref->ar_amap)
148 amap_lock(aref->ar_amap);
149
150 /*
151 * loop until done
152 */
153 while (togo) {
154
155 /*
156 * find the page we want. check the anon layer first.
157 */
158
159 if (aref->ar_amap) {
160 anon = amap_lookup(aref, curaddr - ufi->entry->start);
161 } else {
162 anon = NULL;
163 }
164
165 /* locked: map, amap, uobj */
166 if (anon) {
167 rv = uvm_loananon(ufi, output, flags, anon);
168 } else if (uobj) {
169 rv = uvm_loanuobj(ufi, output, flags, curaddr);
170 } else if (UVM_ET_ISCOPYONWRITE(ufi->entry)) {
171 rv = uvm_loanzero(ufi, output, flags);
172 } else {
173 uvmfault_unlockall(ufi, aref->ar_amap, uobj, NULL);
174 rv = -1;
175 }
176 /* locked: if (rv > 0) => map, amap, uobj [o.w. unlocked] */
177 KASSERT(rv > 0 || aref->ar_amap == NULL ||
178 !mutex_owned(&aref->ar_amap->am_l));
179 KASSERT(rv > 0 || uobj == NULL ||
180 !mutex_owned(&uobj->vmobjlock));
181
182 /* total failure */
183 if (rv < 0) {
184 UVMHIST_LOG(loanhist, "failure %d", rv, 0,0,0);
185 return (-1);
186 }
187
188 /* relock failed, need to do another lookup */
189 if (rv == 0) {
190 UVMHIST_LOG(loanhist, "relock failure %d", result
191 ,0,0,0);
192 return (result);
193 }
194
195 /*
196 * got it... advance to next page
197 */
198
199 result++;
200 togo -= PAGE_SIZE;
201 curaddr += PAGE_SIZE;
202 }
203
204 /*
205 * unlock what we locked, unlock the maps and return
206 */
207
208 if (aref->ar_amap)
209 amap_unlock(aref->ar_amap);
210 uvmfault_unlockmaps(ufi, false);
211 UVMHIST_LOG(loanhist, "done %d", result, 0,0,0);
212 return (result);
213 }
214
215 /*
216 * normal functions
217 */
218
219 /*
220 * uvm_loan: loan pages in a map out to anons or to the kernel
221 *
222 * => map should be unlocked
223 * => start and len should be multiples of PAGE_SIZE
224 * => result is either an array of anon's or vm_pages (depending on flags)
225 * => flag values: UVM_LOAN_TOANON - loan to anons
226 * UVM_LOAN_TOPAGE - loan to wired kernel page
227 * one and only one of these flags must be set!
228 * => returns 0 (success), or an appropriate error number
229 */
230
231 int
232 uvm_loan(struct vm_map *map, vaddr_t start, vsize_t len, void *v, int flags)
233 {
234 struct uvm_faultinfo ufi;
235 void **result, **output;
236 int rv, error;
237
238 UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
239
240 /*
241 * ensure that one and only one of the flags is set
242 */
243
244 KASSERT(((flags & UVM_LOAN_TOANON) == 0) ^
245 ((flags & UVM_LOAN_TOPAGE) == 0));
246 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
247
248 /*
249 * "output" is a pointer to the current place to put the loaned page.
250 */
251
252 result = v;
253 output = &result[0]; /* start at the beginning ... */
254
255 /*
256 * while we've got pages to do
257 */
258
259 while (len > 0) {
260
261 /*
262 * fill in params for a call to uvmfault_lookup
263 */
264
265 ufi.orig_map = map;
266 ufi.orig_rvaddr = start;
267 ufi.orig_size = len;
268
269 /*
270 * do the lookup, the only time this will fail is if we hit on
271 * an unmapped region (an error)
272 */
273
274 if (!uvmfault_lookup(&ufi, false)) {
275 error = ENOENT;
276 goto fail;
277 }
278
279 /*
280 * map now locked. now do the loanout...
281 */
282
283 rv = uvm_loanentry(&ufi, &output, flags);
284 if (rv < 0) {
285 /* all unlocked due to error */
286 error = EINVAL;
287 goto fail;
288 }
289
290 /*
291 * done! the map is unlocked. advance, if possible.
292 *
293 * XXXCDC: could be recoded to hold the map lock with
294 * smarter code (but it only happens on map entry
295 * boundaries, so it isn't that bad).
296 */
297
298 if (rv) {
299 rv <<= PAGE_SHIFT;
300 len -= rv;
301 start += rv;
302 }
303 }
304 UVMHIST_LOG(loanhist, "success", 0,0,0,0);
305 return 0;
306
307 fail:
308 /*
309 * failed to complete loans. drop any loans and return failure code.
310 * map is already unlocked.
311 */
312
313 if (output - result) {
314 if (flags & UVM_LOAN_TOANON) {
315 uvm_unloananon((struct vm_anon **)result,
316 output - result);
317 } else {
318 uvm_unloanpage((struct vm_page **)result,
319 output - result);
320 }
321 }
322 UVMHIST_LOG(loanhist, "error %d", error,0,0,0);
323 return (error);
324 }
325
326 /*
327 * uvm_loananon: loan a page from an anon out
328 *
329 * => called with map, amap, uobj locked
330 * => return value:
331 * -1 = fatal error, everything is unlocked, abort.
332 * 0 = lookup in ufi went stale, everything unlocked, relookup and
333 * try again
334 * 1 = got it, everything still locked
335 */
336
337 int
338 uvm_loananon(struct uvm_faultinfo *ufi, void ***output, int flags,
339 struct vm_anon *anon)
340 {
341 struct vm_page *pg;
342 int error;
343
344 UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
345
346 /*
347 * if we are loaning to "another" anon then it is easy, we just
348 * bump the reference count on the current anon and return a
349 * pointer to it (it becomes copy-on-write shared).
350 */
351
352 if (flags & UVM_LOAN_TOANON) {
353 mutex_enter(&anon->an_lock);
354 pg = anon->an_page;
355 if (pg && (pg->pqflags & PQ_ANON) != 0 && anon->an_ref == 1) {
356 if (pg->wire_count > 0) {
357 UVMHIST_LOG(loanhist, "->A wired %p", pg,0,0,0);
358 uvmfault_unlockall(ufi,
359 ufi->entry->aref.ar_amap,
360 ufi->entry->object.uvm_obj, anon);
361 return (-1);
362 }
363 pmap_page_protect(pg, VM_PROT_READ);
364 }
365 anon->an_ref++;
366 **output = anon;
367 (*output)++;
368 mutex_exit(&anon->an_lock);
369 UVMHIST_LOG(loanhist, "->A done", 0,0,0,0);
370 return (1);
371 }
372
373 /*
374 * we are loaning to a kernel-page. we need to get the page
375 * resident so we can wire it. uvmfault_anonget will handle
376 * this for us.
377 */
378
379 mutex_enter(&anon->an_lock);
380 error = uvmfault_anonget(ufi, ufi->entry->aref.ar_amap, anon);
381
382 /*
383 * if we were unable to get the anon, then uvmfault_anonget has
384 * unlocked everything and returned an error code.
385 */
386
387 if (error) {
388 UVMHIST_LOG(loanhist, "error %d", error,0,0,0);
389
390 /* need to refault (i.e. refresh our lookup) ? */
391 if (error == ERESTART) {
392 return (0);
393 }
394
395 /* "try again"? sleep a bit and retry ... */
396 if (error == EAGAIN) {
397 kpause("loanagain", false, hz/2, NULL);
398 return (0);
399 }
400
401 /* otherwise flag it as an error */
402 return (-1);
403 }
404
405 /*
406 * we have the page and its owner locked: do the loan now.
407 */
408
409 pg = anon->an_page;
410 mutex_enter(&uvm_pageqlock);
411 if (pg->wire_count > 0) {
412 mutex_exit(&uvm_pageqlock);
413 UVMHIST_LOG(loanhist, "->K wired %p", pg,0,0,0);
414 KASSERT(pg->uobject == NULL);
415 uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap,
416 NULL, anon);
417 return (-1);
418 }
419 if (pg->loan_count == 0) {
420 pmap_page_protect(pg, VM_PROT_READ);
421 }
422 pg->loan_count++;
423 uvm_pageactivate(pg);
424 mutex_exit(&uvm_pageqlock);
425 **output = pg;
426 (*output)++;
427
428 /* unlock anon and return success */
429 if (pg->uobject)
430 mutex_exit(&pg->uobject->vmobjlock);
431 mutex_exit(&anon->an_lock);
432 UVMHIST_LOG(loanhist, "->K done", 0,0,0,0);
433 return (1);
434 }
435
436 /*
437 * uvm_loanpage: loan out pages to kernel (->K)
438 *
439 * => pages should be object-owned and the object should be locked.
440 * => in the case of error, the object might be unlocked and relocked.
441 * => caller should busy the pages beforehand.
442 * => pages will be unbusied.
443 * => fail with EBUSY if meet a wired page.
444 */
445 static int
446 uvm_loanpage(struct vm_page **pgpp, int npages)
447 {
448 int i;
449 int error = 0;
450
451 UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
452
453 for (i = 0; i < npages; i++) {
454 struct vm_page *pg = pgpp[i];
455
456 KASSERT(pg->uobject != NULL);
457 KASSERT(pg->uobject == pgpp[0]->uobject);
458 KASSERT(!(pg->flags & (PG_RELEASED|PG_PAGEOUT)));
459 KASSERT(mutex_owned(&pg->uobject->vmobjlock));
460 KASSERT(pg->flags & PG_BUSY);
461
462 mutex_enter(&uvm_pageqlock);
463 if (pg->wire_count > 0) {
464 mutex_exit(&uvm_pageqlock);
465 UVMHIST_LOG(loanhist, "wired %p", pg,0,0,0);
466 error = EBUSY;
467 break;
468 }
469 if (pg->loan_count == 0) {
470 pmap_page_protect(pg, VM_PROT_READ);
471 }
472 pg->loan_count++;
473 uvm_pageactivate(pg);
474 mutex_exit(&uvm_pageqlock);
475 }
476
477 uvm_page_unbusy(pgpp, npages);
478
479 if (error) {
480 /*
481 * backout what we've done
482 */
483 kmutex_t *slock = &pgpp[0]->uobject->vmobjlock;
484
485 mutex_exit(slock);
486 uvm_unloan(pgpp, i, UVM_LOAN_TOPAGE);
487 mutex_enter(slock);
488 }
489
490 UVMHIST_LOG(loanhist, "done %d", error,0,0,0);
491 return error;
492 }
493
494 /*
495 * XXX UBC temp limit
496 * number of pages to get at once.
497 * should be <= MAX_READ_AHEAD in genfs_vnops.c
498 */
499 #define UVM_LOAN_GET_CHUNK 16
500
501 /*
502 * uvm_loanuobjpages: loan pages from a uobj out (O->K)
503 *
504 * => uobj shouldn't be locked. (we'll lock it)
505 * => fail with EBUSY if we meet a wired page.
506 */
507 int
508 uvm_loanuobjpages(struct uvm_object *uobj, voff_t pgoff, int orignpages,
509 struct vm_page **origpgpp)
510 {
511 int ndone; /* # of pages loaned out */
512 struct vm_page **pgpp;
513 int error;
514 int i;
515 kmutex_t *slock;
516
517 pgpp = origpgpp;
518 for (ndone = 0; ndone < orignpages; ) {
519 int npages;
520 /* npendloan: # of pages busied but not loand out yet. */
521 int npendloan = 0xdead; /* XXX gcc */
522 reget:
523 npages = MIN(UVM_LOAN_GET_CHUNK, orignpages - ndone);
524 mutex_enter(&uobj->vmobjlock);
525 error = (*uobj->pgops->pgo_get)(uobj,
526 pgoff + (ndone << PAGE_SHIFT), pgpp, &npages, 0,
527 VM_PROT_READ, 0, PGO_SYNCIO);
528 if (error == EAGAIN) {
529 kpause("loanuopg", false, hz/2, NULL);
530 continue;
531 }
532 if (error)
533 goto fail;
534
535 KASSERT(npages > 0);
536
537 /* loan and unbusy pages */
538 slock = NULL;
539 for (i = 0; i < npages; i++) {
540 kmutex_t *nextslock; /* slock for next page */
541 struct vm_page *pg = *pgpp;
542
543 /* XXX assuming that the page is owned by uobj */
544 KASSERT(pg->uobject != NULL);
545 nextslock = &pg->uobject->vmobjlock;
546
547 if (slock != nextslock) {
548 if (slock) {
549 KASSERT(npendloan > 0);
550 error = uvm_loanpage(pgpp - npendloan,
551 npendloan);
552 mutex_exit(slock);
553 if (error)
554 goto fail;
555 ndone += npendloan;
556 KASSERT(origpgpp + ndone == pgpp);
557 }
558 slock = nextslock;
559 npendloan = 0;
560 mutex_enter(slock);
561 }
562
563 if ((pg->flags & PG_RELEASED) != 0) {
564 /*
565 * release pages and try again.
566 */
567 mutex_exit(slock);
568 for (; i < npages; i++) {
569 pg = pgpp[i];
570 slock = &pg->uobject->vmobjlock;
571
572 mutex_enter(slock);
573 mutex_enter(&uvm_pageqlock);
574 uvm_page_unbusy(&pg, 1);
575 mutex_exit(&uvm_pageqlock);
576 mutex_exit(slock);
577 }
578 goto reget;
579 }
580
581 npendloan++;
582 pgpp++;
583 KASSERT(origpgpp + ndone + npendloan == pgpp);
584 }
585 KASSERT(slock != NULL);
586 KASSERT(npendloan > 0);
587 error = uvm_loanpage(pgpp - npendloan, npendloan);
588 mutex_exit(slock);
589 if (error)
590 goto fail;
591 ndone += npendloan;
592 KASSERT(origpgpp + ndone == pgpp);
593 }
594
595 return 0;
596
597 fail:
598 uvm_unloan(origpgpp, ndone, UVM_LOAN_TOPAGE);
599
600 return error;
601 }
602
603 /*
604 * uvm_loanuobj: loan a page from a uobj out
605 *
606 * => called with map, amap, uobj locked
607 * => return value:
608 * -1 = fatal error, everything is unlocked, abort.
609 * 0 = lookup in ufi went stale, everything unlocked, relookup and
610 * try again
611 * 1 = got it, everything still locked
612 */
613
614 static int
615 uvm_loanuobj(struct uvm_faultinfo *ufi, void ***output, int flags, vaddr_t va)
616 {
617 struct vm_amap *amap = ufi->entry->aref.ar_amap;
618 struct uvm_object *uobj = ufi->entry->object.uvm_obj;
619 struct vm_page *pg;
620 struct vm_anon *anon;
621 int error, npages;
622 bool locked;
623
624 UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
625
626 /*
627 * first we must make sure the page is resident.
628 *
629 * XXXCDC: duplicate code with uvm_fault().
630 */
631
632 /* locked: maps(read), amap(if there) */
633 mutex_enter(&uobj->vmobjlock);
634 /* locked: maps(read), amap(if there), uobj */
635
636 if (uobj->pgops->pgo_get) { /* try locked pgo_get */
637 npages = 1;
638 pg = NULL;
639 error = (*uobj->pgops->pgo_get)(uobj,
640 va - ufi->entry->start + ufi->entry->offset,
641 &pg, &npages, 0, VM_PROT_READ, MADV_NORMAL, PGO_LOCKED);
642 } else {
643 error = EIO; /* must have pgo_get op */
644 }
645
646 /*
647 * check the result of the locked pgo_get. if there is a problem,
648 * then we fail the loan.
649 */
650
651 if (error && error != EBUSY) {
652 uvmfault_unlockall(ufi, amap, uobj, NULL);
653 return (-1);
654 }
655
656 /*
657 * if we need to unlock for I/O, do so now.
658 */
659
660 if (error == EBUSY) {
661 uvmfault_unlockall(ufi, amap, NULL, NULL);
662
663 /* locked: uobj */
664 npages = 1;
665 error = (*uobj->pgops->pgo_get)(uobj,
666 va - ufi->entry->start + ufi->entry->offset,
667 &pg, &npages, 0, VM_PROT_READ, MADV_NORMAL, PGO_SYNCIO);
668 /* locked: <nothing> */
669
670 if (error) {
671 if (error == EAGAIN) {
672 kpause("fltagain2", false, hz/2, NULL);
673 return (0);
674 }
675 return (-1);
676 }
677
678 /*
679 * pgo_get was a success. attempt to relock everything.
680 */
681
682 locked = uvmfault_relock(ufi);
683 if (locked && amap)
684 amap_lock(amap);
685 uobj = pg->uobject;
686 mutex_enter(&uobj->vmobjlock);
687
688 /*
689 * verify that the page has not be released and re-verify
690 * that amap slot is still free. if there is a problem we
691 * drop our lock (thus force a lookup refresh/retry).
692 */
693
694 if ((pg->flags & PG_RELEASED) != 0 ||
695 (locked && amap && amap_lookup(&ufi->entry->aref,
696 ufi->orig_rvaddr - ufi->entry->start))) {
697 if (locked)
698 uvmfault_unlockall(ufi, amap, NULL, NULL);
699 locked = false;
700 }
701
702 /*
703 * didn't get the lock? release the page and retry.
704 */
705
706 if (locked == false) {
707 if (pg->flags & PG_WANTED) {
708 wakeup(pg);
709 }
710 if (pg->flags & PG_RELEASED) {
711 mutex_enter(&uvm_pageqlock);
712 uvm_pagefree(pg);
713 mutex_exit(&uvm_pageqlock);
714 mutex_exit(&uobj->vmobjlock);
715 return (0);
716 }
717 mutex_enter(&uvm_pageqlock);
718 uvm_pageactivate(pg);
719 mutex_exit(&uvm_pageqlock);
720 pg->flags &= ~(PG_BUSY|PG_WANTED);
721 UVM_PAGE_OWN(pg, NULL);
722 mutex_exit(&uobj->vmobjlock);
723 return (0);
724 }
725 }
726
727 KASSERT(uobj == pg->uobject);
728
729 /*
730 * at this point we have the page we want ("pg") marked PG_BUSY for us
731 * and we have all data structures locked. do the loanout. page can
732 * not be PG_RELEASED (we caught this above).
733 */
734
735 if ((flags & UVM_LOAN_TOANON) == 0) {
736 if (uvm_loanpage(&pg, 1)) {
737 uvmfault_unlockall(ufi, amap, uobj, NULL);
738 return (-1);
739 }
740 mutex_exit(&uobj->vmobjlock);
741 **output = pg;
742 (*output)++;
743 return (1);
744 }
745
746 /*
747 * must be a loan to an anon. check to see if there is already
748 * an anon associated with this page. if so, then just return
749 * a reference to this object. the page should already be
750 * mapped read-only because it is already on loan.
751 */
752
753 if (pg->uanon) {
754 anon = pg->uanon;
755 mutex_enter(&anon->an_lock);
756 anon->an_ref++;
757 mutex_exit(&anon->an_lock);
758 if (pg->flags & PG_WANTED) {
759 wakeup(pg);
760 }
761 pg->flags &= ~(PG_WANTED|PG_BUSY);
762 UVM_PAGE_OWN(pg, NULL);
763 mutex_exit(&uobj->vmobjlock);
764 **output = anon;
765 (*output)++;
766 return (1);
767 }
768
769 /*
770 * need to allocate a new anon
771 */
772
773 anon = uvm_analloc();
774 if (anon == NULL) {
775 goto fail;
776 }
777 anon->an_page = pg;
778 pg->uanon = anon;
779 mutex_enter(&uvm_pageqlock);
780 if (pg->wire_count > 0) {
781 mutex_exit(&uvm_pageqlock);
782 UVMHIST_LOG(loanhist, "wired %p", pg,0,0,0);
783 pg->uanon = NULL;
784 anon->an_page = NULL;
785 anon->an_ref--;
786 mutex_exit(&anon->an_lock);
787 uvm_anfree(anon);
788 goto fail;
789 }
790 if (pg->loan_count == 0) {
791 pmap_page_protect(pg, VM_PROT_READ);
792 }
793 pg->loan_count++;
794 uvm_pageactivate(pg);
795 mutex_exit(&uvm_pageqlock);
796 if (pg->flags & PG_WANTED) {
797 wakeup(pg);
798 }
799 pg->flags &= ~(PG_WANTED|PG_BUSY);
800 UVM_PAGE_OWN(pg, NULL);
801 mutex_exit(&uobj->vmobjlock);
802 mutex_exit(&anon->an_lock);
803 **output = anon;
804 (*output)++;
805 return (1);
806
807 fail:
808 UVMHIST_LOG(loanhist, "fail", 0,0,0,0);
809 /*
810 * unlock everything and bail out.
811 */
812 if (pg->flags & PG_WANTED) {
813 wakeup(pg);
814 }
815 pg->flags &= ~(PG_WANTED|PG_BUSY);
816 UVM_PAGE_OWN(pg, NULL);
817 uvmfault_unlockall(ufi, amap, uobj, NULL);
818 return (-1);
819 }
820
821 /*
822 * uvm_loanzero: loan a zero-fill page out
823 *
824 * => called with map, amap, uobj locked
825 * => return value:
826 * -1 = fatal error, everything is unlocked, abort.
827 * 0 = lookup in ufi went stale, everything unlocked, relookup and
828 * try again
829 * 1 = got it, everything still locked
830 */
831
832 static struct uvm_object uvm_loanzero_object;
833
834 static int
835 uvm_loanzero(struct uvm_faultinfo *ufi, void ***output, int flags)
836 {
837 struct vm_anon *anon;
838 struct vm_page *pg;
839 struct vm_amap *amap = ufi->entry->aref.ar_amap;
840
841 UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
842 again:
843 mutex_enter(&uvm_loanzero_object.vmobjlock);
844
845 /*
846 * first, get ahold of our single zero page.
847 */
848
849 if (__predict_false((pg =
850 TAILQ_FIRST(&uvm_loanzero_object.memq)) == NULL)) {
851 while ((pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL,
852 UVM_PGA_ZERO)) == NULL) {
853 mutex_exit(&uvm_loanzero_object.vmobjlock);
854 uvmfault_unlockall(ufi, amap, NULL, NULL);
855 uvm_wait("loanzero");
856 if (!uvmfault_relock(ufi)) {
857 return (0);
858 }
859 if (amap) {
860 amap_lock(amap);
861 }
862 goto again;
863 }
864
865 /* got a zero'd page. */
866 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
867 pg->flags |= PG_RDONLY;
868 mutex_enter(&uvm_pageqlock);
869 uvm_pageactivate(pg);
870 mutex_exit(&uvm_pageqlock);
871 UVM_PAGE_OWN(pg, NULL);
872 }
873
874 if ((flags & UVM_LOAN_TOANON) == 0) { /* loaning to kernel-page */
875 mutex_enter(&uvm_pageqlock);
876 pg->loan_count++;
877 mutex_exit(&uvm_pageqlock);
878 mutex_exit(&uvm_loanzero_object.vmobjlock);
879 **output = pg;
880 (*output)++;
881 return (1);
882 }
883
884 /*
885 * loaning to an anon. check to see if there is already an anon
886 * associated with this page. if so, then just return a reference
887 * to this object.
888 */
889
890 if (pg->uanon) {
891 anon = pg->uanon;
892 mutex_enter(&anon->an_lock);
893 anon->an_ref++;
894 mutex_exit(&anon->an_lock);
895 mutex_exit(&uvm_loanzero_object.vmobjlock);
896 **output = anon;
897 (*output)++;
898 return (1);
899 }
900
901 /*
902 * need to allocate a new anon
903 */
904
905 anon = uvm_analloc();
906 if (anon == NULL) {
907 /* out of swap causes us to fail */
908 mutex_exit(&uvm_loanzero_object.vmobjlock);
909 uvmfault_unlockall(ufi, amap, NULL, NULL);
910 return (-1);
911 }
912 anon->an_page = pg;
913 pg->uanon = anon;
914 mutex_enter(&uvm_pageqlock);
915 pg->loan_count++;
916 uvm_pageactivate(pg);
917 mutex_exit(&uvm_pageqlock);
918 mutex_exit(&anon->an_lock);
919 mutex_exit(&uvm_loanzero_object.vmobjlock);
920 **output = anon;
921 (*output)++;
922 return (1);
923 }
924
925
926 /*
927 * uvm_unloananon: kill loans on anons (basically a normal ref drop)
928 *
929 * => we expect all our resources to be unlocked
930 */
931
932 static void
933 uvm_unloananon(struct vm_anon **aloans, int nanons)
934 {
935 struct vm_anon *anon;
936
937 while (nanons-- > 0) {
938 int refs;
939
940 anon = *aloans++;
941 mutex_enter(&anon->an_lock);
942 refs = --anon->an_ref;
943 mutex_exit(&anon->an_lock);
944
945 if (refs == 0) {
946 uvm_anfree(anon);
947 }
948 }
949 }
950
951 /*
952 * uvm_unloanpage: kill loans on pages loaned out to the kernel
953 *
954 * => we expect all our resources to be unlocked
955 */
956
957 static void
958 uvm_unloanpage(struct vm_page **ploans, int npages)
959 {
960 struct vm_page *pg;
961 kmutex_t *slock;
962
963 mutex_enter(&uvm_pageqlock);
964 while (npages-- > 0) {
965 pg = *ploans++;
966
967 /*
968 * do a little dance to acquire the object or anon lock
969 * as appropriate. we are locking in the wrong order,
970 * so we have to do a try-lock here.
971 */
972
973 slock = NULL;
974 while (pg->uobject != NULL || pg->uanon != NULL) {
975 if (pg->uobject != NULL) {
976 slock = &pg->uobject->vmobjlock;
977 } else {
978 slock = &pg->uanon->an_lock;
979 }
980 if (mutex_tryenter(slock)) {
981 break;
982 }
983 mutex_exit(&uvm_pageqlock);
984 /* XXX Better than yielding but inadequate. */
985 kpause("livelock", false, 1, NULL);
986 mutex_enter(&uvm_pageqlock);
987 slock = NULL;
988 }
989
990 /*
991 * drop our loan. if page is owned by an anon but
992 * PQ_ANON is not set, the page was loaned to the anon
993 * from an object which dropped ownership, so resolve
994 * this by turning the anon's loan into real ownership
995 * (ie. decrement loan_count again and set PQ_ANON).
996 * after all this, if there are no loans left, put the
997 * page back a paging queue (if the page is owned by
998 * an anon) or free it (if the page is now unowned).
999 */
1000
1001 KASSERT(pg->loan_count > 0);
1002 pg->loan_count--;
1003 if (pg->uobject == NULL && pg->uanon != NULL &&
1004 (pg->pqflags & PQ_ANON) == 0) {
1005 KASSERT(pg->loan_count > 0);
1006 pg->loan_count--;
1007 pg->pqflags |= PQ_ANON;
1008 }
1009 if (pg->loan_count == 0 && pg->uobject == NULL &&
1010 pg->uanon == NULL) {
1011 KASSERT((pg->flags & PG_BUSY) == 0);
1012 uvm_pagefree(pg);
1013 }
1014 if (slock != NULL) {
1015 mutex_exit(slock);
1016 }
1017 }
1018 mutex_exit(&uvm_pageqlock);
1019 }
1020
1021 /*
1022 * uvm_unloan: kill loans on pages or anons.
1023 */
1024
1025 void
1026 uvm_unloan(void *v, int npages, int flags)
1027 {
1028 if (flags & UVM_LOAN_TOANON) {
1029 uvm_unloananon(v, npages);
1030 } else {
1031 uvm_unloanpage(v, npages);
1032 }
1033 }
1034
1035 /*
1036 * Minimal pager for uvm_loanzero_object. We need to provide a "put"
1037 * method, because the page can end up on a paging queue, and the
1038 * page daemon will want to call pgo_put when it encounters the page
1039 * on the inactive list.
1040 */
1041
1042 static int
1043 ulz_put(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
1044 {
1045 struct vm_page *pg;
1046
1047 KDASSERT(uobj == &uvm_loanzero_object);
1048
1049 /*
1050 * Don't need to do any work here if we're not freeing pages.
1051 */
1052
1053 if ((flags & PGO_FREE) == 0) {
1054 mutex_exit(&uobj->vmobjlock);
1055 return 0;
1056 }
1057
1058 /*
1059 * we don't actually want to ever free the uvm_loanzero_page, so
1060 * just reactivate or dequeue it.
1061 */
1062
1063 pg = TAILQ_FIRST(&uobj->memq);
1064 KASSERT(pg != NULL);
1065 KASSERT(TAILQ_NEXT(pg, listq.queue) == NULL);
1066
1067 mutex_enter(&uvm_pageqlock);
1068 if (pg->uanon)
1069 uvm_pageactivate(pg);
1070 else
1071 uvm_pagedequeue(pg);
1072 mutex_exit(&uvm_pageqlock);
1073
1074 mutex_exit(&uobj->vmobjlock);
1075 return 0;
1076 }
1077
1078 static const struct uvm_pagerops ulz_pager = {
1079 .pgo_put = ulz_put,
1080 };
1081
1082 /*
1083 * uvm_loan_init(): initialize the uvm_loan() facility.
1084 */
1085
1086 void
1087 uvm_loan_init(void)
1088 {
1089
1090 UVM_OBJ_INIT(&uvm_loanzero_object, &ulz_pager, 0);
1091
1092 UVMHIST_INIT(loanhist, 300);
1093 }
1094
1095 /*
1096 * uvm_loanbreak: break loan on a uobj page
1097 *
1098 * => called with uobj locked
1099 * => the page should be busy
1100 * => return value:
1101 * newly allocated page if succeeded
1102 */
1103 struct vm_page *
1104 uvm_loanbreak(struct vm_page *uobjpage)
1105 {
1106 struct vm_page *pg;
1107 #ifdef DIAGNOSTIC
1108 struct uvm_object *uobj = uobjpage->uobject;
1109 #endif
1110
1111 KASSERT(uobj != NULL);
1112 KASSERT(mutex_owned(&uobj->vmobjlock));
1113 KASSERT(uobjpage->flags & PG_BUSY);
1114
1115 /* alloc new un-owned page */
1116 pg = uvm_pagealloc(NULL, 0, NULL, 0);
1117 if (pg == NULL)
1118 return NULL;
1119
1120 /*
1121 * copy the data from the old page to the new
1122 * one and clear the fake flags on the new page (keep it busy).
1123 * force a reload of the old page by clearing it from all
1124 * pmaps.
1125 * transfer dirtiness of the old page to the new page.
1126 * then lock the page queues to rename the pages.
1127 */
1128
1129 uvm_pagecopy(uobjpage, pg); /* old -> new */
1130 pg->flags &= ~PG_FAKE;
1131 pmap_page_protect(uobjpage, VM_PROT_NONE);
1132 if ((uobjpage->flags & PG_CLEAN) != 0 && !pmap_clear_modify(uobjpage)) {
1133 pmap_clear_modify(pg);
1134 pg->flags |= PG_CLEAN;
1135 } else {
1136 /* uvm_pagecopy marked it dirty */
1137 KASSERT((pg->flags & PG_CLEAN) == 0);
1138 /* a object with a dirty page should be dirty. */
1139 KASSERT(!UVM_OBJ_IS_CLEAN(uobj));
1140 }
1141 if (uobjpage->flags & PG_WANTED)
1142 wakeup(uobjpage);
1143 /* uobj still locked */
1144 uobjpage->flags &= ~(PG_WANTED|PG_BUSY);
1145 UVM_PAGE_OWN(uobjpage, NULL);
1146
1147 mutex_enter(&uvm_pageqlock);
1148
1149 /*
1150 * replace uobjpage with new page.
1151 */
1152
1153 uvm_pagereplace(uobjpage, pg);
1154
1155 /*
1156 * if the page is no longer referenced by
1157 * an anon (i.e. we are breaking an O->K
1158 * loan), then remove it from any pageq's.
1159 */
1160 if (uobjpage->uanon == NULL)
1161 uvm_pagedequeue(uobjpage);
1162
1163 /*
1164 * at this point we have absolutely no
1165 * control over uobjpage
1166 */
1167
1168 /* install new page */
1169 uvm_pageactivate(pg);
1170 mutex_exit(&uvm_pageqlock);
1171
1172 /*
1173 * done! loan is broken and "pg" is
1174 * PG_BUSY. it can now replace uobjpage.
1175 */
1176
1177 return pg;
1178 }
1179
1180 int
1181 uvm_loanbreak_anon(struct vm_anon *anon, struct uvm_object *uobj)
1182 {
1183 struct vm_page *pg;
1184
1185 KASSERT(mutex_owned(&anon->an_lock));
1186 KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1187
1188 /* get new un-owned replacement page */
1189 pg = uvm_pagealloc(NULL, 0, NULL, 0);
1190 if (pg == NULL) {
1191 return ENOMEM;
1192 }
1193
1194 /*
1195 * copy data, kill loan, and drop uobj lock (if any)
1196 */
1197 /* copy old -> new */
1198 uvm_pagecopy(anon->an_page, pg);
1199
1200 /* force reload */
1201 pmap_page_protect(anon->an_page, VM_PROT_NONE);
1202 mutex_enter(&uvm_pageqlock); /* KILL loan */
1203
1204 anon->an_page->uanon = NULL;
1205 /* in case we owned */
1206 anon->an_page->pqflags &= ~PQ_ANON;
1207
1208 if (uobj) {
1209 /* if we were receiver of loan */
1210 anon->an_page->loan_count--;
1211 } else {
1212 /*
1213 * we were the lender (A->K); need to remove the page from
1214 * pageq's.
1215 */
1216 uvm_pagedequeue(anon->an_page);
1217 }
1218
1219 if (uobj) {
1220 mutex_exit(&uobj->vmobjlock);
1221 }
1222
1223 /* install new page in anon */
1224 anon->an_page = pg;
1225 pg->uanon = anon;
1226 pg->pqflags |= PQ_ANON;
1227
1228 uvm_pageactivate(pg);
1229 mutex_exit(&uvm_pageqlock);
1230
1231 pg->flags &= ~(PG_BUSY|PG_FAKE);
1232 UVM_PAGE_OWN(pg, NULL);
1233
1234 /* done! */
1235
1236 return 0;
1237 }
Cache object: b0f0d0447b25dcc75853214b18af1f87
|