FreeBSD/Linux Kernel Cross Reference
sys/vm/vm_pageout.c
1 /*
2 * Copyright (c) 1991 Regents of the University of California.
3 * All rights reserved.
4 * Copyright (c) 1994 John S. Dyson
5 * All rights reserved.
6 * Copyright (c) 1994 David Greenman
7 * All rights reserved.
8 *
9 * This code is derived from software contributed to Berkeley by
10 * The Mach Operating System project at Carnegie-Mellon University.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. All advertising materials mentioning features or use of this software
21 * must display the following acknowledgement:
22 * This product includes software developed by the University of
23 * California, Berkeley and its contributors.
24 * 4. Neither the name of the University nor the names of its contributors
25 * may be used to endorse or promote products derived from this software
26 * without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 * SUCH DAMAGE.
39 *
40 * from: @(#)vm_pageout.c 7.4 (Berkeley) 5/7/91
41 *
42 *
43 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
44 * All rights reserved.
45 *
46 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
47 *
48 * Permission to use, copy, modify and distribute this software and
49 * its documentation is hereby granted, provided that both the copyright
50 * notice and this permission notice appear in all copies of the
51 * software, derivative works or modified versions, and any portions
52 * thereof, and that both notices appear in supporting documentation.
53 *
54 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
55 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
56 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
57 *
58 * Carnegie Mellon requests users of this software to return to
59 *
60 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
61 * School of Computer Science
62 * Carnegie Mellon University
63 * Pittsburgh PA 15213-3890
64 *
65 * any improvements or extensions that they make and grant Carnegie the
66 * rights to redistribute these changes.
67 *
68 * $FreeBSD: src/sys/vm/vm_pageout.c,v 1.86.2.3 1999/09/05 08:24:36 peter Exp $
69 */
70
71 /*
72 * The proverbial page-out daemon.
73 */
74
75 #include <sys/param.h>
76 #include <sys/systm.h>
77 #include <sys/kernel.h>
78 #include <sys/proc.h>
79 #include <sys/resourcevar.h>
80 #include <sys/malloc.h>
81 #include <sys/kernel.h>
82 #include <sys/signalvar.h>
83 #include <sys/vnode.h>
84 #include <sys/vmmeter.h>
85 #include <sys/sysctl.h>
86
87 #include <vm/vm.h>
88 #include <vm/vm_param.h>
89 #include <vm/vm_prot.h>
90 #include <vm/lock.h>
91 #include <vm/vm_object.h>
92 #include <vm/vm_page.h>
93 #include <vm/vm_map.h>
94 #include <vm/vm_pageout.h>
95 #include <vm/vm_kern.h>
96 #include <vm/vm_pager.h>
97 #include <vm/swap_pager.h>
98 #include <vm/vm_extern.h>
99
100 /*
101 * System initialization
102 */
103
104 /* the kernel process "vm_pageout"*/
105 static void vm_pageout __P((void));
106 static int vm_pageout_clean __P((vm_page_t, int));
107 static int vm_pageout_scan __P((void));
108 static int vm_pageout_free_page_calc __P((vm_size_t count));
109 struct proc *pageproc;
110
111 static struct kproc_desc page_kp = {
112 "pagedaemon",
113 vm_pageout,
114 &pageproc
115 };
116 SYSINIT_KT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_FIRST, kproc_start, &page_kp)
117
118 #if !defined(NO_SWAPPING)
119 /* the kernel process "vm_daemon"*/
120 static void vm_daemon __P((void));
121 static struct proc *vmproc;
122
123 static struct kproc_desc vm_kp = {
124 "vmdaemon",
125 vm_daemon,
126 &vmproc
127 };
128 SYSINIT_KT(vmdaemon, SI_SUB_KTHREAD_VM, SI_ORDER_FIRST, kproc_start, &vm_kp)
129 #endif
130
131
132 int vm_pages_needed; /* Event on which pageout daemon sleeps */
133
134 int vm_pageout_pages_needed; /* flag saying that the pageout daemon needs pages */
135
136 extern int npendingio;
137 #if !defined(NO_SWAPPING)
138 static int vm_pageout_req_swapout; /* XXX */
139 static int vm_daemon_needed;
140 #endif
141 extern int nswiodone;
142 extern int vm_swap_size;
143 extern int vfs_update_wakeup;
144 int vm_pageout_algorithm_lru=0;
145 #if defined(NO_SWAPPING)
146 int vm_swapping_enabled=0;
147 #else
148 int vm_swapping_enabled=1;
149 #endif
150
151 SYSCTL_INT(_vm, VM_PAGEOUT_ALGORITHM, pageout_algorithm,
152 CTLFLAG_RW, &vm_pageout_algorithm_lru, 0, "");
153
154 #if defined(NO_SWAPPING)
155 SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swapping_enabled,
156 CTLFLAG_RD, &vm_swapping_enabled, 0, "");
157 #else
158 SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swapping_enabled,
159 CTLFLAG_RW, &vm_swapping_enabled, 0, "");
160 #endif
161
162 #define MAXLAUNDER (cnt.v_page_count > 1800 ? 32 : 16)
163
164 #define VM_PAGEOUT_PAGE_COUNT 16
165 int vm_pageout_page_count = VM_PAGEOUT_PAGE_COUNT;
166
167 int vm_page_max_wired; /* XXX max # of wired pages system-wide */
168
169 #if !defined(NO_SWAPPING)
170 typedef void freeer_fcn_t __P((vm_map_t, vm_object_t, vm_pindex_t, int));
171 static void vm_pageout_map_deactivate_pages __P((vm_map_t, vm_pindex_t));
172 static freeer_fcn_t vm_pageout_object_deactivate_pages;
173 static void vm_req_vmdaemon __P((void));
174 #endif
175
176 /*
177 * vm_pageout_clean:
178 *
179 * Clean the page and remove it from the laundry.
180 *
181 * We set the busy bit to cause potential page faults on this page to
182 * block.
183 *
184 * And we set pageout-in-progress to keep the object from disappearing
185 * during pageout. This guarantees that the page won't move from the
186 * inactive queue. (However, any other page on the inactive queue may
187 * move!)
188 */
189 static int
190 vm_pageout_clean(m, sync)
191 vm_page_t m;
192 int sync;
193 {
194 register vm_object_t object;
195 vm_page_t mc[2*vm_pageout_page_count];
196 int pageout_count;
197 int i, forward_okay, backward_okay, page_base;
198 vm_pindex_t pindex = m->pindex;
199
200 object = m->object;
201
202 /*
203 * If not OBJT_SWAP, additional memory may be needed to do the pageout.
204 * Try to avoid the deadlock.
205 */
206 if ((sync != VM_PAGEOUT_FORCE) &&
207 (object->type == OBJT_DEFAULT) &&
208 ((cnt.v_free_count + cnt.v_cache_count) < cnt.v_pageout_free_min))
209 return 0;
210
211 /*
212 * Don't mess with the page if it's busy.
213 */
214 if ((!sync && m->hold_count != 0) ||
215 ((m->busy != 0) || (m->flags & PG_BUSY)))
216 return 0;
217
218 /*
219 * Try collapsing before it's too late.
220 */
221 if (!sync && object->backing_object) {
222 vm_object_collapse(object);
223 }
224
225 mc[vm_pageout_page_count] = m;
226 pageout_count = 1;
227 page_base = vm_pageout_page_count;
228 forward_okay = TRUE;
229 if (pindex != 0)
230 backward_okay = TRUE;
231 else
232 backward_okay = FALSE;
233 /*
234 * Scan object for clusterable pages.
235 *
236 * We can cluster ONLY if: ->> the page is NOT
237 * clean, wired, busy, held, or mapped into a
238 * buffer, and one of the following:
239 * 1) The page is inactive, or a seldom used
240 * active page.
241 * -or-
242 * 2) we force the issue.
243 */
244 for (i = 1; (i < vm_pageout_page_count) && (forward_okay || backward_okay); i++) {
245 vm_page_t p;
246
247 /*
248 * See if forward page is clusterable.
249 */
250 if (forward_okay) {
251 /*
252 * Stop forward scan at end of object.
253 */
254 if ((pindex + i) > object->size) {
255 forward_okay = FALSE;
256 goto do_backward;
257 }
258 p = vm_page_lookup(object, pindex + i);
259 if (p) {
260 if (((p->queue - p->pc) == PQ_CACHE) ||
261 (p->flags & PG_BUSY) || p->busy) {
262 forward_okay = FALSE;
263 goto do_backward;
264 }
265 vm_page_test_dirty(p);
266 if ((p->dirty & p->valid) != 0 &&
267 ((p->queue == PQ_INACTIVE) ||
268 (sync == VM_PAGEOUT_FORCE)) &&
269 (p->wire_count == 0) &&
270 (p->hold_count == 0)) {
271 mc[vm_pageout_page_count + i] = p;
272 pageout_count++;
273 if (pageout_count == vm_pageout_page_count)
274 break;
275 } else {
276 forward_okay = FALSE;
277 }
278 } else {
279 forward_okay = FALSE;
280 }
281 }
282 do_backward:
283 /*
284 * See if backward page is clusterable.
285 */
286 if (backward_okay) {
287 /*
288 * Stop backward scan at beginning of object.
289 */
290 if ((pindex - i) == 0) {
291 backward_okay = FALSE;
292 }
293 p = vm_page_lookup(object, pindex - i);
294 if (p) {
295 if (((p->queue - p->pc) == PQ_CACHE) ||
296 (p->flags & PG_BUSY) || p->busy) {
297 backward_okay = FALSE;
298 continue;
299 }
300 vm_page_test_dirty(p);
301 if ((p->dirty & p->valid) != 0 &&
302 ((p->queue == PQ_INACTIVE) ||
303 (sync == VM_PAGEOUT_FORCE)) &&
304 (p->wire_count == 0) &&
305 (p->hold_count == 0)) {
306 mc[vm_pageout_page_count - i] = p;
307 pageout_count++;
308 page_base--;
309 if (pageout_count == vm_pageout_page_count)
310 break;
311 } else {
312 backward_okay = FALSE;
313 }
314 } else {
315 backward_okay = FALSE;
316 }
317 }
318 }
319
320 /*
321 * we allow reads during pageouts...
322 */
323 for (i = page_base; i < (page_base + pageout_count); i++) {
324 mc[i]->flags |= PG_BUSY;
325 vm_page_protect(mc[i], VM_PROT_READ);
326 }
327
328 return vm_pageout_flush(&mc[page_base], pageout_count, sync);
329 }
330
331 int
332 vm_pageout_flush(mc, count, sync)
333 vm_page_t *mc;
334 int count;
335 int sync;
336 {
337 register vm_object_t object;
338 int pageout_status[count];
339 int anyok = 0;
340 int i;
341
342 object = mc[0]->object;
343 object->paging_in_progress += count;
344
345 vm_pager_put_pages(object, mc, count,
346 ((sync || (object == kernel_object)) ? TRUE : FALSE),
347 pageout_status);
348
349 for (i = 0; i < count; i++) {
350 vm_page_t mt = mc[i];
351
352 switch (pageout_status[i]) {
353 case VM_PAGER_OK:
354 ++anyok;
355 break;
356 case VM_PAGER_PEND:
357 ++anyok;
358 break;
359 case VM_PAGER_BAD:
360 /*
361 * Page outside of range of object. Right now we
362 * essentially lose the changes by pretending it
363 * worked.
364 */
365 pmap_clear_modify(VM_PAGE_TO_PHYS(mt));
366 mt->dirty = 0;
367 break;
368 case VM_PAGER_ERROR:
369 case VM_PAGER_FAIL:
370 /*
371 * If page couldn't be paged out, then reactivate the
372 * page so it doesn't clog the inactive list. (We
373 * will try paging out it again later).
374 */
375 if (mt->queue == PQ_INACTIVE)
376 vm_page_activate(mt);
377 break;
378 case VM_PAGER_AGAIN:
379 break;
380 }
381
382
383 /*
384 * If the operation is still going, leave the page busy to
385 * block all other accesses. Also, leave the paging in
386 * progress indicator set so that we don't attempt an object
387 * collapse.
388 */
389 if (pageout_status[i] != VM_PAGER_PEND) {
390 vm_object_pip_wakeup(object);
391 PAGE_WAKEUP(mt);
392 }
393 }
394 return anyok;
395 }
396
397 #if !defined(NO_SWAPPING)
398 /*
399 * vm_pageout_object_deactivate_pages
400 *
401 * deactivate enough pages to satisfy the inactive target
402 * requirements or if vm_page_proc_limit is set, then
403 * deactivate all of the pages in the object and its
404 * backing_objects.
405 *
406 * The object and map must be locked.
407 */
408 static void
409 vm_pageout_object_deactivate_pages(map, object, desired, map_remove_only)
410 vm_map_t map;
411 vm_object_t object;
412 vm_pindex_t desired;
413 int map_remove_only;
414 {
415 register vm_page_t p, next;
416 int rcount;
417 int remove_mode;
418 int s;
419
420 if (object->type == OBJT_DEVICE)
421 return;
422
423 while (object) {
424 if (vm_map_pmap(map)->pm_stats.resident_count <= desired)
425 return;
426 if (object->paging_in_progress)
427 return;
428
429 remove_mode = map_remove_only;
430 if (object->shadow_count > 1)
431 remove_mode = 1;
432 /*
433 * scan the objects entire memory queue
434 */
435 rcount = object->resident_page_count;
436 p = TAILQ_FIRST(&object->memq);
437 while (p && (rcount-- > 0)) {
438 int refcount;
439 if (vm_map_pmap(map)->pm_stats.resident_count <= desired)
440 return;
441 next = TAILQ_NEXT(p, listq);
442 cnt.v_pdpages++;
443 if (p->wire_count != 0 ||
444 p->hold_count != 0 ||
445 p->busy != 0 ||
446 (p->flags & PG_BUSY) ||
447 !pmap_page_exists(vm_map_pmap(map), VM_PAGE_TO_PHYS(p))) {
448 p = next;
449 continue;
450 }
451
452 refcount = pmap_ts_referenced(VM_PAGE_TO_PHYS(p));
453 if (refcount) {
454 p->flags |= PG_REFERENCED;
455 } else if (p->flags & PG_REFERENCED) {
456 refcount = 1;
457 }
458
459 if ((p->queue != PQ_ACTIVE) &&
460 (p->flags & PG_REFERENCED)) {
461 vm_page_activate(p);
462 p->act_count += refcount;
463 p->flags &= ~PG_REFERENCED;
464 } else if (p->queue == PQ_ACTIVE) {
465 if ((p->flags & PG_REFERENCED) == 0) {
466 p->act_count -= min(p->act_count, ACT_DECLINE);
467 if (!remove_mode && (vm_pageout_algorithm_lru || (p->act_count == 0))) {
468 vm_page_protect(p, VM_PROT_NONE);
469 vm_page_deactivate(p);
470 } else {
471 s = splvm();
472 TAILQ_REMOVE(&vm_page_queue_active, p, pageq);
473 TAILQ_INSERT_TAIL(&vm_page_queue_active, p, pageq);
474 splx(s);
475 }
476 } else {
477 p->flags &= ~PG_REFERENCED;
478 if (p->act_count < (ACT_MAX - ACT_ADVANCE))
479 p->act_count += ACT_ADVANCE;
480 s = splvm();
481 TAILQ_REMOVE(&vm_page_queue_active, p, pageq);
482 TAILQ_INSERT_TAIL(&vm_page_queue_active, p, pageq);
483 splx(s);
484 }
485 } else if (p->queue == PQ_INACTIVE) {
486 vm_page_protect(p, VM_PROT_NONE);
487 }
488 p = next;
489 }
490 object = object->backing_object;
491 }
492 return;
493 }
494
495 /*
496 * deactivate some number of pages in a map, try to do it fairly, but
497 * that is really hard to do.
498 */
499 static void
500 vm_pageout_map_deactivate_pages(map, desired)
501 vm_map_t map;
502 vm_pindex_t desired;
503 {
504 vm_map_entry_t tmpe;
505 vm_object_t obj, bigobj;
506
507 vm_map_reference(map);
508 if (!lock_try_write(&map->lock)) {
509 vm_map_deallocate(map);
510 return;
511 }
512
513 bigobj = NULL;
514
515 /*
516 * first, search out the biggest object, and try to free pages from
517 * that.
518 */
519 tmpe = map->header.next;
520 while (tmpe != &map->header) {
521 if ((tmpe->eflags & (MAP_ENTRY_IS_A_MAP|MAP_ENTRY_IS_SUB_MAP)) == 0) {
522 obj = tmpe->object.vm_object;
523 if ((obj != NULL) && (obj->shadow_count <= 1) &&
524 ((bigobj == NULL) ||
525 (bigobj->resident_page_count < obj->resident_page_count))) {
526 bigobj = obj;
527 }
528 }
529 tmpe = tmpe->next;
530 }
531
532 if (bigobj)
533 vm_pageout_object_deactivate_pages(map, bigobj, desired, 0);
534
535 /*
536 * Next, hunt around for other pages to deactivate. We actually
537 * do this search sort of wrong -- .text first is not the best idea.
538 */
539 tmpe = map->header.next;
540 while (tmpe != &map->header) {
541 if (vm_map_pmap(map)->pm_stats.resident_count <= desired)
542 break;
543 if ((tmpe->eflags & (MAP_ENTRY_IS_A_MAP|MAP_ENTRY_IS_SUB_MAP)) == 0) {
544 obj = tmpe->object.vm_object;
545 if (obj)
546 vm_pageout_object_deactivate_pages(map, obj, desired, 0);
547 }
548 tmpe = tmpe->next;
549 };
550
551 /*
552 * Remove all mappings if a process is swapped out, this will free page
553 * table pages.
554 */
555 if (desired == 0)
556 pmap_remove(vm_map_pmap(map),
557 VM_MIN_ADDRESS, VM_MAXUSER_ADDRESS);
558 vm_map_unlock(map);
559 vm_map_deallocate(map);
560 return;
561 }
562 #endif
563
564 /*
565 * vm_pageout_scan does the dirty work for the pageout daemon.
566 */
567 static int
568 vm_pageout_scan()
569 {
570 vm_page_t m, next;
571 int page_shortage, addl_page_shortage, maxscan, maxlaunder, pcount;
572 int pages_freed;
573 struct proc *p, *bigproc;
574 vm_offset_t size, bigsize;
575 vm_object_t object;
576 int force_wakeup = 0;
577 int vnodes_skipped = 0;
578 int s;
579
580 /*
581 * Start scanning the inactive queue for pages we can free. We keep
582 * scanning until we have enough free pages or we have scanned through
583 * the entire queue. If we encounter dirty pages, we start cleaning
584 * them.
585 */
586
587 pages_freed = 0;
588 addl_page_shortage = 0;
589
590 maxlaunder = (cnt.v_inactive_target > MAXLAUNDER) ?
591 MAXLAUNDER : cnt.v_inactive_target;
592 rescan0:
593 maxscan = cnt.v_inactive_count;
594 for( m = TAILQ_FIRST(&vm_page_queue_inactive);
595
596 (m != NULL) && (maxscan-- > 0) &&
597 ((cnt.v_cache_count + cnt.v_free_count) <
598 (cnt.v_cache_min + cnt.v_free_target));
599
600 m = next) {
601
602 cnt.v_pdpages++;
603
604 if (m->queue != PQ_INACTIVE) {
605 goto rescan0;
606 }
607
608 next = TAILQ_NEXT(m, pageq);
609
610 if (m->hold_count) {
611 s = splvm();
612 TAILQ_REMOVE(&vm_page_queue_inactive, m, pageq);
613 TAILQ_INSERT_TAIL(&vm_page_queue_inactive, m, pageq);
614 splx(s);
615 addl_page_shortage++;
616 continue;
617 }
618 /*
619 * Dont mess with busy pages, keep in the front of the
620 * queue, most likely are being paged out.
621 */
622 if (m->busy || (m->flags & PG_BUSY)) {
623 addl_page_shortage++;
624 continue;
625 }
626
627 if (m->object->ref_count == 0) {
628 m->flags &= ~PG_REFERENCED;
629 pmap_clear_reference(VM_PAGE_TO_PHYS(m));
630 } else if (((m->flags & PG_REFERENCED) == 0) &&
631 pmap_ts_referenced(VM_PAGE_TO_PHYS(m))) {
632 vm_page_activate(m);
633 continue;
634 }
635
636 if ((m->flags & PG_REFERENCED) != 0) {
637 m->flags &= ~PG_REFERENCED;
638 pmap_clear_reference(VM_PAGE_TO_PHYS(m));
639 vm_page_activate(m);
640 continue;
641 }
642
643 if (m->dirty == 0) {
644 vm_page_test_dirty(m);
645 } else if (m->dirty != 0) {
646 m->dirty = VM_PAGE_BITS_ALL;
647 }
648
649 if (m->valid == 0) {
650 vm_page_protect(m, VM_PROT_NONE);
651 vm_page_free(m);
652 cnt.v_dfree++;
653 ++pages_freed;
654 } else if (m->dirty == 0) {
655 vm_page_cache(m);
656 ++pages_freed;
657 } else if (maxlaunder > 0) {
658 int written;
659 struct vnode *vp = NULL;
660
661 object = m->object;
662 if (object->flags & OBJ_DEAD) {
663 s = splvm();
664 TAILQ_REMOVE(&vm_page_queue_inactive, m, pageq);
665 TAILQ_INSERT_TAIL(&vm_page_queue_inactive, m, pageq);
666 splx(s);
667 continue;
668 }
669
670 if (object->type == OBJT_VNODE) {
671 vp = object->handle;
672 if (VOP_ISLOCKED(vp) || vget(vp, 1)) {
673 if ((m->queue == PQ_INACTIVE) &&
674 (m->hold_count == 0) &&
675 (m->busy == 0) &&
676 (m->flags & PG_BUSY) == 0) {
677 s = splvm();
678 TAILQ_REMOVE(&vm_page_queue_inactive, m, pageq);
679 TAILQ_INSERT_TAIL(&vm_page_queue_inactive, m, pageq);
680 splx(s);
681 }
682 if (object->flags & OBJ_MIGHTBEDIRTY)
683 ++vnodes_skipped;
684 continue;
685 }
686
687 /*
688 * The page might have been moved to another queue
689 * during potential blocking in vget() above.
690 */
691 if (m->queue != PQ_INACTIVE) {
692 if (object->flags & OBJ_MIGHTBEDIRTY)
693 ++vnodes_skipped;
694 vput(vp);
695 continue;
696 }
697
698 /*
699 * The page may have been busied during the blocking in
700 * vput(); We don't move the page back onto the end of
701 * the queue so that statistics are more correct if we don't.
702 */
703 if (m->busy || (m->flags & PG_BUSY)) {
704 vput(vp);
705 continue;
706 }
707
708 /*
709 * If the page has become held, then skip it
710 */
711 if (m->hold_count) {
712 s = splvm();
713 TAILQ_REMOVE(&vm_page_queue_inactive, m, pageq);
714 TAILQ_INSERT_TAIL(&vm_page_queue_inactive, m, pageq);
715 splx(s);
716 if (object->flags & OBJ_MIGHTBEDIRTY)
717 ++vnodes_skipped;
718 vput(vp);
719 continue;
720 }
721 }
722
723 /*
724 * If a page is dirty, then it is either being washed
725 * (but not yet cleaned) or it is still in the
726 * laundry. If it is still in the laundry, then we
727 * start the cleaning operation.
728 */
729 written = vm_pageout_clean(m, 0);
730
731 if (vp)
732 vput(vp);
733
734 maxlaunder -= written;
735 }
736 }
737
738 /*
739 * Compute the page shortage. If we are still very low on memory be
740 * sure that we will move a minimal amount of pages from active to
741 * inactive.
742 */
743
744 page_shortage = (cnt.v_inactive_target + cnt.v_cache_min) -
745 (cnt.v_free_count + cnt.v_inactive_count + cnt.v_cache_count);
746 if (page_shortage <= 0) {
747 if (pages_freed == 0) {
748 page_shortage = cnt.v_free_min - cnt.v_free_count;
749 } else {
750 page_shortage = 1;
751 }
752 }
753 if (addl_page_shortage) {
754 if (page_shortage < 0)
755 page_shortage = 0;
756 page_shortage += addl_page_shortage;
757 }
758
759 pcount = cnt.v_active_count;
760 m = TAILQ_FIRST(&vm_page_queue_active);
761 while ((m != NULL) && (pcount-- > 0) && (page_shortage > 0)) {
762 int refcount;
763
764 if (m->queue != PQ_ACTIVE) {
765 break;
766 }
767
768 next = TAILQ_NEXT(m, pageq);
769 /*
770 * Don't deactivate pages that are busy.
771 */
772 if ((m->busy != 0) ||
773 (m->flags & PG_BUSY) ||
774 (m->hold_count != 0)) {
775 s = splvm();
776 TAILQ_REMOVE(&vm_page_queue_active, m, pageq);
777 TAILQ_INSERT_TAIL(&vm_page_queue_active, m, pageq);
778 splx(s);
779 m = next;
780 continue;
781 }
782
783 /*
784 * The count for pagedaemon pages is done after checking the
785 * page for eligbility...
786 */
787 cnt.v_pdpages++;
788
789 refcount = 0;
790 if (m->object->ref_count != 0) {
791 if (m->flags & PG_REFERENCED) {
792 refcount += 1;
793 }
794 refcount += pmap_ts_referenced(VM_PAGE_TO_PHYS(m));
795 if (refcount) {
796 m->act_count += ACT_ADVANCE + refcount;
797 if (m->act_count > ACT_MAX)
798 m->act_count = ACT_MAX;
799 }
800 }
801
802 m->flags &= ~PG_REFERENCED;
803
804 if (refcount && (m->object->ref_count != 0)) {
805 s = splvm();
806 TAILQ_REMOVE(&vm_page_queue_active, m, pageq);
807 TAILQ_INSERT_TAIL(&vm_page_queue_active, m, pageq);
808 splx(s);
809 } else {
810 m->act_count -= min(m->act_count, ACT_DECLINE);
811 if (vm_pageout_algorithm_lru ||
812 (m->object->ref_count == 0) || (m->act_count == 0)) {
813 --page_shortage;
814 if (m->object->ref_count == 0) {
815 vm_page_protect(m, VM_PROT_NONE);
816 if (m->dirty == 0)
817 vm_page_cache(m);
818 else
819 vm_page_deactivate(m);
820 } else {
821 vm_page_deactivate(m);
822 }
823 } else {
824 s = splvm();
825 TAILQ_REMOVE(&vm_page_queue_active, m, pageq);
826 TAILQ_INSERT_TAIL(&vm_page_queue_active, m, pageq);
827 splx(s);
828 }
829 }
830 m = next;
831 }
832
833 s = splvm();
834 /*
835 * We try to maintain some *really* free pages, this allows interrupt
836 * code to be guaranteed space.
837 */
838 while (cnt.v_free_count < cnt.v_free_reserved) {
839 static int cache_rover = 0;
840 m = vm_page_list_find(PQ_CACHE, cache_rover);
841 if (!m)
842 break;
843 cache_rover = (cache_rover + PQ_PRIME2) & PQ_L2_MASK;
844 vm_page_free(m);
845 cnt.v_dfree++;
846 }
847 splx(s);
848
849 /*
850 * If we didn't get enough free pages, and we have skipped a vnode
851 * in a writeable object, wakeup the sync daemon. And kick swapout
852 * if we did not get enough free pages.
853 */
854 if ((cnt.v_cache_count + cnt.v_free_count) <
855 (cnt.v_free_target + cnt.v_cache_min) ) {
856 if (vnodes_skipped &&
857 (cnt.v_cache_count + cnt.v_free_count) < cnt.v_free_min) {
858 if (!vfs_update_wakeup) {
859 vfs_update_wakeup = 1;
860 wakeup(&vfs_update_wakeup);
861 }
862 }
863 #if !defined(NO_SWAPPING)
864 if (vm_swapping_enabled &&
865 (cnt.v_free_count + cnt.v_cache_count < cnt.v_free_target)) {
866 vm_req_vmdaemon();
867 vm_pageout_req_swapout = 1;
868 }
869 #endif
870 }
871
872
873 /*
874 * make sure that we have swap space -- if we are low on memory and
875 * swap -- then kill the biggest process.
876 */
877 if ((vm_swap_size == 0 || swap_pager_full) &&
878 ((cnt.v_free_count + cnt.v_cache_count) < cnt.v_free_min)) {
879 bigproc = NULL;
880 bigsize = 0;
881 for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) {
882 /*
883 * if this is a system process, skip it
884 */
885 if ((p->p_flag & P_SYSTEM) || (p->p_pid == 1) ||
886 ((p->p_pid < 48) && (vm_swap_size != 0))) {
887 continue;
888 }
889 /*
890 * if the process is in a non-running type state,
891 * don't touch it.
892 */
893 if (p->p_stat != SRUN && p->p_stat != SSLEEP) {
894 continue;
895 }
896 /*
897 * get the process size
898 */
899 size = p->p_vmspace->vm_pmap.pm_stats.resident_count;
900 /*
901 * if the this process is bigger than the biggest one
902 * remember it.
903 */
904 if (size > bigsize) {
905 bigproc = p;
906 bigsize = size;
907 }
908 }
909 if (bigproc != NULL) {
910 killproc(bigproc, "out of swap space");
911 bigproc->p_estcpu = 0;
912 bigproc->p_nice = PRIO_MIN;
913 resetpriority(bigproc);
914 wakeup(&cnt.v_free_count);
915 }
916 }
917 return force_wakeup;
918 }
919
920 static int
921 vm_pageout_free_page_calc(count)
922 vm_size_t count;
923 {
924 if (count < cnt.v_page_count)
925 return 0;
926 /*
927 * free_reserved needs to include enough for the largest swap pager
928 * structures plus enough for any pv_entry structs when paging.
929 */
930 if (cnt.v_page_count > 1024)
931 cnt.v_free_min = 4 + (cnt.v_page_count - 1024) / 200;
932 else
933 cnt.v_free_min = 4;
934 cnt.v_pageout_free_min = (2*MAXBSIZE)/PAGE_SIZE +
935 cnt.v_interrupt_free_min;
936 cnt.v_free_reserved = vm_pageout_page_count +
937 cnt.v_pageout_free_min + (count / 768) + PQ_L2_SIZE;
938 cnt.v_free_min += cnt.v_free_reserved;
939 return 1;
940 }
941
942
943 #ifdef unused
944 int
945 vm_pageout_free_pages(object, add)
946 vm_object_t object;
947 int add;
948 {
949 return vm_pageout_free_page_calc(object->size);
950 }
951 #endif
952
953 /*
954 * vm_pageout is the high level pageout daemon.
955 */
956 static void
957 vm_pageout()
958 {
959 /*
960 * Initialize some paging parameters.
961 */
962
963 cnt.v_interrupt_free_min = 2;
964 if (cnt.v_page_count < 2000)
965 vm_pageout_page_count = 8;
966
967 vm_pageout_free_page_calc(cnt.v_page_count);
968 /*
969 * free_reserved needs to include enough for the largest swap pager
970 * structures plus enough for any pv_entry structs when paging.
971 */
972 cnt.v_free_target = 3 * cnt.v_free_min + cnt.v_free_reserved;
973
974 if (cnt.v_free_count > 1024) {
975 cnt.v_cache_max = (cnt.v_free_count - 1024) / 2;
976 cnt.v_cache_min = (cnt.v_free_count - 1024) / 8;
977 cnt.v_inactive_target = 2*cnt.v_cache_min + 192;
978 } else {
979 cnt.v_cache_min = 0;
980 cnt.v_cache_max = 0;
981 cnt.v_inactive_target = cnt.v_free_count / 4;
982 }
983
984 /* XXX does not really belong here */
985 if (vm_page_max_wired == 0)
986 vm_page_max_wired = cnt.v_free_count / 3;
987
988
989 swap_pager_swap_init();
990 /*
991 * The pageout daemon is never done, so loop forever.
992 */
993 while (TRUE) {
994 int inactive_target;
995 int s = splvm();
996 if (!vm_pages_needed ||
997 ((cnt.v_free_count + cnt.v_cache_count) > cnt.v_free_min)) {
998 vm_pages_needed = 0;
999 tsleep(&vm_pages_needed, PVM, "psleep", 0);
1000 } else if (!vm_pages_needed) {
1001 tsleep(&vm_pages_needed, PVM, "psleep", hz/10);
1002 }
1003 inactive_target =
1004 (cnt.v_page_count - cnt.v_wire_count) / 4;
1005 if (inactive_target < 2*cnt.v_free_min)
1006 inactive_target = 2*cnt.v_free_min;
1007 cnt.v_inactive_target = inactive_target;
1008 if (vm_pages_needed)
1009 cnt.v_pdwakeups++;
1010 vm_pages_needed = 0;
1011 splx(s);
1012 vm_pager_sync();
1013 vm_pageout_scan();
1014 vm_pager_sync();
1015 wakeup(&cnt.v_free_count);
1016 }
1017 }
1018
1019 void
1020 pagedaemon_wakeup()
1021 {
1022 if (!vm_pages_needed && curproc != pageproc) {
1023 vm_pages_needed++;
1024 wakeup(&vm_pages_needed);
1025 }
1026 }
1027
1028 #if !defined(NO_SWAPPING)
1029 static void
1030 vm_req_vmdaemon()
1031 {
1032 static int lastrun = 0;
1033
1034 if ((ticks > (lastrun + hz)) || (ticks < lastrun)) {
1035 wakeup(&vm_daemon_needed);
1036 lastrun = ticks;
1037 }
1038 }
1039
1040 static void
1041 vm_daemon()
1042 {
1043 vm_object_t object;
1044 struct proc *p;
1045
1046 while (TRUE) {
1047 tsleep(&vm_daemon_needed, PUSER, "psleep", 0);
1048 if (vm_pageout_req_swapout) {
1049 swapout_procs();
1050 vm_pageout_req_swapout = 0;
1051 }
1052 /*
1053 * scan the processes for exceeding their rlimits or if
1054 * process is swapped out -- deactivate pages
1055 */
1056
1057 for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) {
1058 quad_t limit;
1059 vm_offset_t size;
1060
1061 /*
1062 * if this is a system process or if we have already
1063 * looked at this process, skip it.
1064 */
1065 if (p->p_flag & (P_SYSTEM | P_WEXIT)) {
1066 continue;
1067 }
1068 /*
1069 * if the process is in a non-running type state,
1070 * don't touch it.
1071 */
1072 if (p->p_stat != SRUN && p->p_stat != SSLEEP) {
1073 continue;
1074 }
1075 /*
1076 * get a limit
1077 */
1078 limit = qmin(p->p_rlimit[RLIMIT_RSS].rlim_cur,
1079 p->p_rlimit[RLIMIT_RSS].rlim_max);
1080
1081 /*
1082 * let processes that are swapped out really be
1083 * swapped out set the limit to nothing (will force a
1084 * swap-out.)
1085 */
1086 if ((p->p_flag & P_INMEM) == 0)
1087 limit = 0; /* XXX */
1088
1089 size = p->p_vmspace->vm_pmap.pm_stats.resident_count * PAGE_SIZE;
1090 if (limit >= 0 && size >= limit) {
1091 vm_pageout_map_deactivate_pages(&p->p_vmspace->vm_map,
1092 (vm_pindex_t)(limit >> PAGE_SHIFT) );
1093 }
1094 }
1095
1096 /*
1097 * we remove cached objects that have no RSS...
1098 */
1099 restart:
1100 object = TAILQ_FIRST(&vm_object_cached_list);
1101 while (object) {
1102 /*
1103 * if there are no resident pages -- get rid of the object
1104 */
1105 if (object->resident_page_count == 0) {
1106 vm_object_reference(object);
1107 pager_cache(object, FALSE);
1108 goto restart;
1109 }
1110 object = TAILQ_NEXT(object, cached_list);
1111 }
1112 }
1113 }
1114 #endif
Cache object: 3937e85327dbda6522ce3db968ea2542
|