FreeBSD/Linux Kernel Cross Reference
sys/i386/i386/pmap.c
1 /*
2 * Copyright (c) 1991 Regents of the University of California.
3 * All rights reserved.
4 * Copyright (c) 1994 John S. Dyson
5 * All rights reserved.
6 * Copyright (c) 1994 David Greenman
7 * All rights reserved.
8 *
9 * This code is derived from software contributed to Berkeley by
10 * the Systems Programming Group of the University of Utah Computer
11 * Science Department and William Jolitz of UUNET Technologies Inc.
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 * 3. All advertising materials mentioning features or use of this software
22 * must display the following acknowledgement:
23 * This product includes software developed by the University of
24 * California, Berkeley and its contributors.
25 * 4. Neither the name of the University nor the names of its contributors
26 * may be used to endorse or promote products derived from this software
27 * without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39 * SUCH DAMAGE.
40 *
41 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91
42 * $FreeBSD: src/sys/i386/i386/pmap.c,v 1.128.2.8 1999/09/05 08:11:13 peter Exp $
43 */
44
45 /*
46 * Manages physical address maps.
47 *
48 * In addition to hardware address maps, this
49 * module is called upon to provide software-use-only
50 * maps which may or may not be stored in the same
51 * form as hardware maps. These pseudo-maps are
52 * used to store intermediate results from copy
53 * operations to and from address spaces.
54 *
55 * Since the information managed by this module is
56 * also stored by the logical address mapping module,
57 * this module may throw away valid virtual-to-physical
58 * mappings at almost any time. However, invalidations
59 * of virtual-to-physical mappings must be done as
60 * requested.
61 *
62 * In order to cope with hardware architectures which
63 * make virtual-to-physical map invalidates expensive,
64 * this module may delay invalidate or reduced protection
65 * operations until such time as they are actually
66 * necessary. This module is given full information as
67 * to which processors are currently using which maps,
68 * and to when physical maps must be made correct.
69 */
70
71 #include "opt_cpu.h"
72
73 #define PMAP_LOCK 1
74 #define PMAP_PVLIST 1
75
76 #include "opt_msgbuf.h"
77
78 #include <sys/param.h>
79 #include <sys/systm.h>
80 #include <sys/proc.h>
81 #include <sys/malloc.h>
82 #include <sys/msgbuf.h>
83 #include <sys/queue.h>
84 #include <sys/vmmeter.h>
85 #include <sys/mman.h>
86
87 #include <vm/vm.h>
88 #include <vm/vm_param.h>
89 #include <vm/vm_prot.h>
90 #include <vm/lock.h>
91 #include <vm/vm_kern.h>
92 #include <vm/vm_page.h>
93 #include <vm/vm_map.h>
94 #include <vm/vm_object.h>
95 #include <vm/vm_extern.h>
96 #include <vm/vm_pageout.h>
97 #include <vm/vm_pager.h>
98
99 #include <machine/pcb.h>
100 #include <machine/cputypes.h>
101 #include <machine/md_var.h>
102 #include <machine/specialreg.h>
103
104 #define PMAP_KEEP_PDIRS
105 #ifndef PMAP_SHPGPERPROC
106 #define PMAP_SHPGPERPROC 200
107 #endif
108
109 #if defined(DIAGNOSTIC)
110 #define PMAP_DIAGNOSTIC
111 #endif
112
113 #if !defined(PMAP_DIAGNOSTIC)
114 #define PMAP_INLINE __inline
115 #else
116 #define PMAP_INLINE
117 #endif
118
119 #define PTPHINT
120
121 static void init_pv_entries __P((int));
122
123 /*
124 * Get PDEs and PTEs for user/kernel address space
125 */
126 #define pmap_pde(m, v) (&((m)->pm_pdir[(vm_offset_t)(v) >> PDRSHIFT]))
127 #define pdir_pde(m, v) (m[(vm_offset_t)(v) >> PDRSHIFT])
128
129 #define pmap_pde_v(pte) ((*(int *)pte & PG_V) != 0)
130 #define pmap_pte_w(pte) ((*(int *)pte & PG_W) != 0)
131 #define pmap_pte_m(pte) ((*(int *)pte & PG_M) != 0)
132 #define pmap_pte_u(pte) ((*(int *)pte & PG_A) != 0)
133 #define pmap_pte_v(pte) ((*(int *)pte & PG_V) != 0)
134
135 #define pmap_pte_set_w(pte, v) ((v)?(*(int *)pte |= PG_W):(*(int *)pte &= ~PG_W))
136 #define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v)))
137
138 /*
139 * Given a map and a machine independent protection code,
140 * convert to a vax protection code.
141 */
142 #define pte_prot(m, p) (protection_codes[p])
143 static int protection_codes[8];
144
145 #define pa_index(pa) atop((pa) - vm_first_phys)
146 #define pa_to_pvh(pa) (&pv_table[pa_index(pa)])
147
148 static struct pmap kernel_pmap_store;
149 pmap_t kernel_pmap;
150
151 vm_offset_t avail_start; /* PA of first available physical page */
152 vm_offset_t avail_end; /* PA of last available physical page */
153 vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */
154 vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */
155 static boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */
156 static vm_offset_t vm_first_phys;
157 static int pgeflag; /* PG_G or-in */
158
159 static int nkpt;
160 static vm_page_t nkpg;
161 vm_offset_t kernel_vm_end;
162
163 extern vm_offset_t clean_sva, clean_eva;
164 extern int cpu_class;
165
166 #define PV_FREELIST_MIN ((PAGE_SIZE / sizeof (struct pv_entry)) / 2)
167
168 /*
169 * Data for the pv entry allocation mechanism
170 */
171 static int pv_freelistcnt;
172 TAILQ_HEAD (,pv_entry) pv_freelist = {0};
173 static vm_offset_t pvva;
174 static int npvvapg;
175
176 /*
177 * All those kernel PT submaps that BSD is so fond of
178 */
179 pt_entry_t *CMAP1 = 0;
180 static pt_entry_t *CMAP2, *ptmmap;
181 static pv_table_t *pv_table;
182 caddr_t CADDR1 = 0, ptvmmap = 0;
183 static caddr_t CADDR2;
184 static pt_entry_t *msgbufmap;
185 struct msgbuf *msgbufp=0;
186
187 pt_entry_t *PMAP1 = 0;
188 unsigned *PADDR1 = 0;
189
190 static PMAP_INLINE void free_pv_entry __P((pv_entry_t pv));
191 static unsigned * get_ptbase __P((pmap_t pmap));
192 static pv_entry_t get_pv_entry __P((void));
193 static void i386_protection_init __P((void));
194 static void pmap_alloc_pv_entry __P((void));
195 static void pmap_changebit __P((vm_offset_t pa, int bit, boolean_t setem));
196
197 static PMAP_INLINE int pmap_is_managed __P((vm_offset_t pa));
198 static void pmap_remove_all __P((vm_offset_t pa));
199 static vm_page_t pmap_enter_quick __P((pmap_t pmap, vm_offset_t va,
200 vm_offset_t pa, vm_page_t mpte));
201 static int pmap_remove_pte __P((struct pmap *pmap, unsigned *ptq,
202 vm_offset_t sva));
203 static void pmap_remove_page __P((struct pmap *pmap, vm_offset_t va));
204 static int pmap_remove_entry __P((struct pmap *pmap, pv_table_t *pv,
205 vm_offset_t va));
206 static boolean_t pmap_testbit __P((vm_offset_t pa, int bit));
207 static void pmap_insert_entry __P((pmap_t pmap, vm_offset_t va,
208 vm_page_t mpte, vm_offset_t pa));
209
210 static vm_page_t pmap_allocpte __P((pmap_t pmap, vm_offset_t va));
211
212 static int pmap_release_free_page __P((pmap_t pmap, vm_page_t p));
213 static vm_page_t _pmap_allocpte __P((pmap_t pmap, unsigned ptepindex));
214 static unsigned * pmap_pte_quick __P((pmap_t pmap, vm_offset_t va));
215 static vm_page_t pmap_page_alloc __P((vm_object_t object, vm_pindex_t pindex));
216 static vm_page_t pmap_page_lookup __P((vm_object_t object, vm_pindex_t pindex));
217 static int pmap_unuse_pt __P((pmap_t, vm_offset_t, vm_page_t));
218
219 #define PDSTACKMAX 6
220 static vm_offset_t pdstack[PDSTACKMAX];
221 static int pdstackptr;
222
223 /*
224 * Routine: pmap_pte
225 * Function:
226 * Extract the page table entry associated
227 * with the given map/virtual_address pair.
228 */
229
230 PMAP_INLINE unsigned *
231 pmap_pte(pmap, va)
232 register pmap_t pmap;
233 vm_offset_t va;
234 {
235 if (pmap && *pmap_pde(pmap, va)) {
236 return get_ptbase(pmap) + i386_btop(va);
237 }
238 return (0);
239 }
240
241 /*
242 * Bootstrap the system enough to run with virtual memory.
243 *
244 * On the i386 this is called after mapping has already been enabled
245 * and just syncs the pmap module with what has already been done.
246 * [We can't call it easily with mapping off since the kernel is not
247 * mapped with PA == VA, hence we would have to relocate every address
248 * from the linked base (virtual) address "KERNBASE" to the actual
249 * (physical) address starting relative to 0]
250 */
251 void
252 pmap_bootstrap(firstaddr, loadaddr)
253 vm_offset_t firstaddr;
254 vm_offset_t loadaddr;
255 {
256 vm_offset_t va;
257 pt_entry_t *pte;
258
259 avail_start = firstaddr;
260
261 /*
262 * XXX The calculation of virtual_avail is wrong. It's NKPT*PAGE_SIZE too
263 * large. It should instead be correctly calculated in locore.s and
264 * not based on 'first' (which is a physical address, not a virtual
265 * address, for the start of unused physical memory). The kernel
266 * page tables are NOT double mapped and thus should not be included
267 * in this calculation.
268 */
269 virtual_avail = (vm_offset_t) KERNBASE + firstaddr;
270 virtual_end = VM_MAX_KERNEL_ADDRESS;
271
272 /*
273 * Initialize protection array.
274 */
275 i386_protection_init();
276
277 /*
278 * The kernel's pmap is statically allocated so we don't have to use
279 * pmap_create, which is unlikely to work correctly at this part of
280 * the boot sequence (XXX and which no longer exists).
281 */
282 kernel_pmap = &kernel_pmap_store;
283
284 kernel_pmap->pm_pdir = (pd_entry_t *) (KERNBASE + IdlePTD);
285
286 kernel_pmap->pm_count = 1;
287 #if PMAP_PVLIST
288 TAILQ_INIT(&kernel_pmap->pm_pvlist);
289 #endif
290 nkpt = NKPT;
291
292 /*
293 * Reserve some special page table entries/VA space for temporary
294 * mapping of pages.
295 */
296 #define SYSMAP(c, p, v, n) \
297 v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n);
298
299 va = virtual_avail;
300 pte = (pt_entry_t *) pmap_pte(kernel_pmap, va);
301
302 /*
303 * CMAP1/CMAP2 are used for zeroing and copying pages.
304 */
305 SYSMAP(caddr_t, CMAP1, CADDR1, 1)
306 SYSMAP(caddr_t, CMAP2, CADDR2, 1)
307
308 /*
309 * ptvmmap is used for reading arbitrary physical pages via /dev/mem.
310 * XXX ptmmap is not used.
311 */
312 SYSMAP(caddr_t, ptmmap, ptvmmap, 1)
313
314 /*
315 * msgbufp is used to map the system message buffer.
316 * XXX msgbufmap is not used.
317 */
318 SYSMAP(struct msgbuf *, msgbufmap, msgbufp,
319 atop(round_page(MSGBUF_SIZE)))
320
321 /*
322 * ptemap is used for pmap_pte_quick
323 */
324 SYSMAP(unsigned *, PMAP1, PADDR1, 1);
325
326 virtual_avail = va;
327
328 *(int *) CMAP1 = *(int *) CMAP2 = *(int *) PTD = 0;
329 invltlb();
330 if (cpu_feature & CPUID_PGE)
331 pgeflag = PG_G;
332 else
333 pgeflag = 0;
334
335 }
336
337 /*
338 * Initialize the pmap module.
339 * Called by vm_init, to initialize any structures that the pmap
340 * system needs to map virtual memory.
341 * pmap_init has been enhanced to support in a fairly consistant
342 * way, discontiguous physical memory.
343 */
344 void
345 pmap_init(phys_start, phys_end)
346 vm_offset_t phys_start, phys_end;
347 {
348 vm_offset_t addr;
349 vm_size_t s;
350 int i, npg;
351
352 /*
353 * calculate the number of pv_entries needed
354 */
355 vm_first_phys = phys_avail[0];
356 for (i = 0; phys_avail[i + 1]; i += 2);
357 npg = (phys_avail[(i - 2) + 1] - vm_first_phys) / PAGE_SIZE;
358
359 /*
360 * Allocate memory for random pmap data structures. Includes the
361 * pv_head_table.
362 */
363 s = (vm_size_t) (sizeof(pv_table_t) * npg);
364 s = round_page(s);
365
366 addr = (vm_offset_t) kmem_alloc(kernel_map, s);
367 pv_table = (pv_table_t *) addr;
368 for(i = 0; i < npg; i++) {
369 vm_offset_t pa;
370 TAILQ_INIT(&pv_table[i].pv_list);
371 pv_table[i].pv_list_count = 0;
372 pa = vm_first_phys + i * PAGE_SIZE;
373 pv_table[i].pv_vm_page = PHYS_TO_VM_PAGE(pa);
374 }
375 TAILQ_INIT(&pv_freelist);
376
377 /*
378 * init the pv free list
379 */
380 init_pv_entries(npg);
381 /*
382 * Now it is safe to enable pv_table recording.
383 */
384 pmap_initialized = TRUE;
385 }
386
387 /*
388 * Used to map a range of physical addresses into kernel
389 * virtual address space.
390 *
391 * For now, VM is already on, we only need to map the
392 * specified memory.
393 */
394 vm_offset_t
395 pmap_map(virt, start, end, prot)
396 vm_offset_t virt;
397 vm_offset_t start;
398 vm_offset_t end;
399 int prot;
400 {
401 while (start < end) {
402 pmap_enter(kernel_pmap, virt, start, prot, FALSE);
403 virt += PAGE_SIZE;
404 start += PAGE_SIZE;
405 }
406 return (virt);
407 }
408
409
410 /***************************************************
411 * Low level helper routines.....
412 ***************************************************/
413
414 #if defined(PMAP_DIAGNOSTIC)
415
416 /*
417 * This code checks for non-writeable/modified pages.
418 * This should be an invalid condition.
419 */
420 static int
421 pmap_nw_modified(pt_entry_t ptea) {
422 int pte;
423
424 pte = (int) ptea;
425
426 if ((pte & (PG_M|PG_RW)) == PG_M)
427 return 1;
428 else
429 return 0;
430 }
431 #endif
432
433
434 /*
435 * this routine defines the region(s) of memory that should
436 * not be tested for the modified bit.
437 */
438 static PMAP_INLINE int
439 pmap_track_modified( vm_offset_t va) {
440 if ((va < clean_sva) || (va >= clean_eva))
441 return 1;
442 else
443 return 0;
444 }
445
446 static PMAP_INLINE void
447 invltlb_1pg( vm_offset_t va) {
448 #if defined(I386_CPU)
449 if (cpu_class == CPUCLASS_386) {
450 invltlb();
451 } else
452 #endif
453 {
454 invlpg(va);
455 }
456 }
457
458 static PMAP_INLINE void
459 invltlb_2pg( vm_offset_t va1, vm_offset_t va2) {
460 #if defined(I386_CPU)
461 if (cpu_class == CPUCLASS_386) {
462 invltlb();
463 } else
464 #endif
465 {
466 invlpg(va1);
467 invlpg(va2);
468 }
469 }
470
471 static unsigned *
472 get_ptbase(pmap)
473 pmap_t pmap;
474 {
475 unsigned frame = (unsigned) pmap->pm_pdir[PTDPTDI] & PG_FRAME;
476
477 /* are we current address space or kernel? */
478 if (pmap == kernel_pmap || frame == (((unsigned) PTDpde) & PG_FRAME)) {
479 return (unsigned *) PTmap;
480 }
481 /* otherwise, we are alternate address space */
482 if (frame != (((unsigned) APTDpde) & PG_FRAME)) {
483 APTDpde = (pd_entry_t) (frame | PG_RW | PG_V);
484 invltlb();
485 }
486 return (unsigned *) APTmap;
487 }
488
489 /*
490 * Super fast pmap_pte routine best used when scanning
491 * the pv lists. This eliminates many coarse-grained
492 * invltlb calls. Note that many of the pv list
493 * scans are across different pmaps. It is very wasteful
494 * to do an entire invltlb for checking a single mapping.
495 */
496
497 static unsigned *
498 pmap_pte_quick(pmap, va)
499 register pmap_t pmap;
500 vm_offset_t va;
501 {
502 unsigned pde, newpf;
503 if (pde = (unsigned) pmap->pm_pdir[va >> PDRSHIFT]) {
504 unsigned frame = (unsigned) pmap->pm_pdir[PTDPTDI] & PG_FRAME;
505 unsigned index = i386_btop(va);
506 /* are we current address space or kernel? */
507 if ((pmap == kernel_pmap) ||
508 (frame == (((unsigned) PTDpde) & PG_FRAME))) {
509 return (unsigned *) PTmap + index;
510 }
511 newpf = pde & PG_FRAME;
512 if ( ((* (unsigned *) PMAP1) & PG_FRAME) != newpf) {
513 * (unsigned *) PMAP1 = newpf | PG_RW | PG_V;
514 invltlb_1pg((vm_offset_t) PADDR1);
515 }
516 return PADDR1 + ((unsigned) index & (NPTEPG - 1));
517 }
518 return (0);
519 }
520
521 /*
522 * Routine: pmap_extract
523 * Function:
524 * Extract the physical page address associated
525 * with the given map/virtual_address pair.
526 */
527 vm_offset_t
528 pmap_extract(pmap, va)
529 register pmap_t pmap;
530 vm_offset_t va;
531 {
532 vm_offset_t rtval;
533 if (pmap && *pmap_pde(pmap, va)) {
534 unsigned *pte;
535 pte = get_ptbase(pmap) + i386_btop(va);
536 rtval = ((*pte & PG_FRAME) | (va & PAGE_MASK));
537 return rtval;
538 }
539 return 0;
540
541 }
542
543 /*
544 * determine if a page is managed (memory vs. device)
545 */
546 static PMAP_INLINE int
547 pmap_is_managed(pa)
548 vm_offset_t pa;
549 {
550 int i;
551
552 if (!pmap_initialized)
553 return 0;
554
555 for (i = 0; phys_avail[i + 1]; i += 2) {
556 if (pa < phys_avail[i + 1] && pa >= phys_avail[i])
557 return 1;
558 }
559 return 0;
560 }
561
562
563 /***************************************************
564 * Low level mapping routines.....
565 ***************************************************/
566
567 /*
568 * Add a list of wired pages to the kva
569 * this routine is only used for temporary
570 * kernel mappings that do not need to have
571 * page modification or references recorded.
572 * Note that old mappings are simply written
573 * over. The page *must* be wired.
574 */
575 void
576 pmap_qenter(va, m, count)
577 vm_offset_t va;
578 vm_page_t *m;
579 int count;
580 {
581 int i;
582 register unsigned *pte;
583
584 for (i = 0; i < count; i++) {
585 vm_offset_t tva = va + i * PAGE_SIZE;
586 unsigned npte = VM_PAGE_TO_PHYS(m[i]) | PG_RW | PG_V | pgeflag;
587 unsigned opte;
588 pte = (unsigned *)vtopte(tva);
589 opte = *pte;
590 *pte = npte;
591 if (opte)
592 invltlb_1pg(tva);
593 }
594 }
595
596 /*
597 * this routine jerks page mappings from the
598 * kernel -- it is meant only for temporary mappings.
599 */
600 void
601 pmap_qremove(va, count)
602 vm_offset_t va;
603 int count;
604 {
605 int i;
606 register unsigned *pte;
607
608 for (i = 0; i < count; i++) {
609 pte = (unsigned *)vtopte(va);
610 *pte = 0;
611 invltlb_1pg(va);
612 va += PAGE_SIZE;
613 }
614 }
615
616 /*
617 * add a wired page to the kva
618 * note that in order for the mapping to take effect -- you
619 * should do a invltlb after doing the pmap_kenter...
620 */
621 PMAP_INLINE void
622 pmap_kenter(va, pa)
623 vm_offset_t va;
624 register vm_offset_t pa;
625 {
626 register unsigned *pte;
627 unsigned npte, opte;
628
629 npte = pa | PG_RW | PG_V | pgeflag;
630 pte = (unsigned *)vtopte(va);
631 opte = *pte;
632 *pte = npte;
633 if (opte)
634 invltlb_1pg(va);
635 }
636
637 /*
638 * remove a page from the kernel pagetables
639 */
640 PMAP_INLINE void
641 pmap_kremove(va)
642 vm_offset_t va;
643 {
644 register unsigned *pte;
645
646 pte = (unsigned *)vtopte(va);
647 *pte = 0;
648 invltlb_1pg(va);
649 }
650
651 static vm_page_t
652 pmap_page_alloc(object, pindex)
653 vm_object_t object;
654 vm_pindex_t pindex;
655 {
656 vm_page_t m;
657 m = vm_page_alloc(object, pindex, VM_ALLOC_ZERO);
658 if (m == NULL) {
659 VM_WAIT;
660 }
661 return m;
662 }
663
664 static vm_page_t
665 pmap_page_lookup(object, pindex)
666 vm_object_t object;
667 vm_pindex_t pindex;
668 {
669 vm_page_t m;
670 retry:
671 m = vm_page_lookup(object, pindex);
672 if (m) {
673 if (m->flags & PG_BUSY) {
674 m->flags |= PG_WANTED;
675 tsleep(m, PVM, "pplookp", 0);
676 goto retry;
677 }
678 }
679
680 return m;
681 }
682
683 /*
684 * Create the UPAGES for a new process.
685 * This routine directly affects the fork perf for a process.
686 */
687 void
688 pmap_new_proc(p)
689 struct proc *p;
690 {
691 int i;
692 vm_object_t upobj;
693 pmap_t pmap;
694 vm_page_t m;
695 struct user *up;
696 unsigned *ptep, *ptek;
697
698 pmap = &p->p_vmspace->vm_pmap;
699
700 /*
701 * allocate object for the upages
702 */
703 upobj = vm_object_allocate( OBJT_DEFAULT,
704 UPAGES);
705 p->p_vmspace->vm_upages_obj = upobj;
706
707 /* get a kernel virtual address for the UPAGES for this proc */
708 up = (struct user *) kmem_alloc_pageable(u_map, UPAGES * PAGE_SIZE);
709 if (up == NULL)
710 panic("vm_fork: u_map allocation failed");
711
712 /*
713 * Allocate the ptp and incr the hold count appropriately
714 */
715 m = pmap_allocpte(pmap, (vm_offset_t) kstack);
716 m->hold_count += (UPAGES - 1);
717
718 ptep = (unsigned *) pmap_pte(pmap, (vm_offset_t) kstack);
719 ptek = (unsigned *) vtopte((vm_offset_t) up);
720
721 for(i=0;i<UPAGES;i++) {
722 /*
723 * Get a kernel stack page
724 */
725 while ((m = vm_page_alloc(upobj,
726 i, VM_ALLOC_NORMAL)) == NULL) {
727 VM_WAIT;
728 }
729
730 /*
731 * Wire the page
732 */
733 m->wire_count++;
734 ++cnt.v_wire_count;
735
736 /*
737 * Enter the page into both the kernel and the process
738 * address space.
739 */
740 *(ptep + i) = VM_PAGE_TO_PHYS(m) | PG_RW | PG_V;
741 *(ptek + i) = VM_PAGE_TO_PHYS(m) | PG_RW | PG_V;
742
743 m->flags &= ~(PG_ZERO|PG_BUSY);
744 m->flags |= PG_MAPPED|PG_WRITEABLE;
745 m->valid = VM_PAGE_BITS_ALL;
746 }
747
748 pmap->pm_stats.resident_count += UPAGES;
749 p->p_addr = up;
750 }
751
752 /*
753 * Dispose the UPAGES for a process that has exited.
754 * This routine directly impacts the exit perf of a process.
755 */
756 void
757 pmap_dispose_proc(p)
758 struct proc *p;
759 {
760 int i;
761 vm_object_t upobj;
762 pmap_t pmap;
763 vm_page_t m;
764 unsigned *ptep, *ptek;
765
766 pmap = &p->p_vmspace->vm_pmap;
767 ptep = (unsigned *) pmap_pte(pmap, (vm_offset_t) kstack);
768 ptek = (unsigned *) vtopte((vm_offset_t) p->p_addr);
769
770 upobj = p->p_vmspace->vm_upages_obj;
771
772 for(i=0;i<UPAGES;i++) {
773 if ((m = vm_page_lookup(upobj, i)) == NULL)
774 panic("pmap_dispose_proc: upage already missing???");
775 *(ptep + i) = 0;
776 *(ptek + i) = 0;
777 pmap_unuse_pt(pmap, (vm_offset_t) kstack + i * PAGE_SIZE, NULL);
778 vm_page_unwire(m);
779 vm_page_free(m);
780 }
781 pmap->pm_stats.resident_count -= UPAGES;
782
783 kmem_free(u_map, (vm_offset_t)p->p_addr, ctob(UPAGES));
784 }
785
786 /*
787 * Allow the UPAGES for a process to be prejudicially paged out.
788 */
789 void
790 pmap_swapout_proc(p)
791 struct proc *p;
792 {
793 int i;
794 vm_object_t upobj;
795 pmap_t pmap;
796 vm_page_t m;
797 unsigned *pte;
798
799 pmap = &p->p_vmspace->vm_pmap;
800 pte = (unsigned *) pmap_pte(pmap, (vm_offset_t) kstack);
801
802 upobj = p->p_vmspace->vm_upages_obj;
803 /*
804 * let the upages be paged
805 */
806 for(i=0;i<UPAGES;i++) {
807 if ((m = vm_page_lookup(upobj, i)) == NULL)
808 panic("pmap_pageout_proc: upage already missing???");
809 m->dirty = VM_PAGE_BITS_ALL;
810 *(pte + i) = 0;
811 pmap_unuse_pt(pmap, (vm_offset_t) kstack + i * PAGE_SIZE, NULL);
812
813 vm_page_unwire(m);
814 vm_page_deactivate(m);
815 pmap_kremove( (vm_offset_t) p->p_addr + PAGE_SIZE * i);
816 }
817 pmap->pm_stats.resident_count -= UPAGES;
818 }
819
820 /*
821 * Bring the UPAGES for a specified process back in.
822 */
823 void
824 pmap_swapin_proc(p)
825 struct proc *p;
826 {
827 int i;
828 vm_object_t upobj;
829 pmap_t pmap;
830 vm_page_t m;
831 unsigned *pte;
832
833 pmap = &p->p_vmspace->vm_pmap;
834 /*
835 * Allocate the ptp and incr the hold count appropriately
836 */
837 m = pmap_allocpte(pmap, (vm_offset_t) kstack);
838 m->hold_count += (UPAGES - 1);
839 pte = (unsigned *) pmap_pte(pmap, (vm_offset_t) kstack);
840
841 upobj = p->p_vmspace->vm_upages_obj;
842 for(i=0;i<UPAGES;i++) {
843 int s;
844 s = splvm();
845 retry:
846 if ((m = vm_page_lookup(upobj, i)) == NULL) {
847 if ((m = vm_page_alloc(upobj, i, VM_ALLOC_NORMAL)) == NULL) {
848 VM_WAIT;
849 goto retry;
850 }
851 } else {
852 if ((m->flags & PG_BUSY) || m->busy) {
853 m->flags |= PG_WANTED;
854 tsleep(m, PVM, "swinuw",0);
855 goto retry;
856 }
857 m->flags |= PG_BUSY;
858 }
859 vm_page_wire(m);
860 splx(s);
861
862 *(pte+i) = VM_PAGE_TO_PHYS(m) | PG_RW | PG_V;
863 pmap_kenter(((vm_offset_t) p->p_addr) + i * PAGE_SIZE,
864 VM_PAGE_TO_PHYS(m));
865
866 if (m->valid != VM_PAGE_BITS_ALL) {
867 int rv;
868 rv = vm_pager_get_pages(upobj, &m, 1, 0);
869 if (rv != VM_PAGER_OK)
870 panic("faultin: cannot get upages for proc: %d\n", p->p_pid);
871 m->valid = VM_PAGE_BITS_ALL;
872 }
873 PAGE_WAKEUP(m);
874 m->flags |= PG_MAPPED|PG_WRITEABLE;
875 }
876 pmap->pm_stats.resident_count += UPAGES;
877 }
878
879 /***************************************************
880 * Page table page management routines.....
881 ***************************************************/
882
883 /*
884 * This routine unholds page table pages, and if the hold count
885 * drops to zero, then it decrements the wire count.
886 */
887 static int
888 _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) {
889 int s;
890
891 if (m->flags & PG_BUSY) {
892 s = splvm();
893 while (m->flags & PG_BUSY) {
894 m->flags |= PG_WANTED;
895 tsleep(m, PVM, "pmuwpt", 0);
896 }
897 splx(s);
898 }
899
900 if (m->hold_count == 0) {
901 vm_offset_t pteva;
902 /*
903 * unmap the page table page
904 */
905 pmap->pm_pdir[m->pindex] = 0;
906 --pmap->pm_stats.resident_count;
907 if ((((unsigned)pmap->pm_pdir[PTDPTDI]) & PG_FRAME) ==
908 (((unsigned) PTDpde) & PG_FRAME)) {
909 /*
910 * Do a invltlb to make the invalidated mapping
911 * take effect immediately.
912 */
913 pteva = UPT_MIN_ADDRESS + i386_ptob(m->pindex);
914 invltlb_1pg(pteva);
915 }
916
917 #if defined(PTPHINT)
918 if (pmap->pm_ptphint == m)
919 pmap->pm_ptphint = NULL;
920 #endif
921
922 /*
923 * If the page is finally unwired, simply free it.
924 */
925 --m->wire_count;
926 if (m->wire_count == 0) {
927
928 if (m->flags & PG_WANTED) {
929 m->flags &= ~PG_WANTED;
930 wakeup(m);
931 }
932
933 vm_page_free_zero(m);
934 --cnt.v_wire_count;
935 }
936 return 1;
937 }
938 return 0;
939 }
940
941 __inline static int
942 pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) {
943 vm_page_unhold(m);
944 if (m->hold_count == 0)
945 return _pmap_unwire_pte_hold(pmap, m);
946 else
947 return 0;
948 }
949
950 /*
951 * After removing a page table entry, this routine is used to
952 * conditionally free the page, and manage the hold/wire counts.
953 */
954 static int
955 pmap_unuse_pt(pmap, va, mpte)
956 pmap_t pmap;
957 vm_offset_t va;
958 vm_page_t mpte;
959 {
960 unsigned ptepindex;
961 if (va >= UPT_MIN_ADDRESS)
962 return 0;
963
964 if (mpte == NULL) {
965 ptepindex = (va >> PDRSHIFT);
966 #if defined(PTPHINT)
967 if (pmap->pm_ptphint &&
968 (pmap->pm_ptphint->pindex == ptepindex)) {
969 mpte = pmap->pm_ptphint;
970 } else {
971 mpte = pmap_page_lookup( pmap->pm_pteobj, ptepindex);
972 pmap->pm_ptphint = mpte;
973 }
974 #else
975 mpte = pmap_page_lookup( pmap->pm_pteobj, ptepindex);
976 #endif
977 }
978
979 return pmap_unwire_pte_hold(pmap, mpte);
980 }
981
982 /*
983 * Initialize a preallocated and zeroed pmap structure,
984 * such as one in a vmspace structure.
985 */
986 void
987 pmap_pinit(pmap)
988 register struct pmap *pmap;
989 {
990 vm_page_t ptdpg;
991 /*
992 * No need to allocate page table space yet but we do need a valid
993 * page directory table.
994 */
995
996 if (pdstackptr > 0) {
997 --pdstackptr;
998 pmap->pm_pdir = (pd_entry_t *)pdstack[pdstackptr];
999 } else {
1000 pmap->pm_pdir =
1001 (pd_entry_t *)kmem_alloc_pageable(kernel_map, PAGE_SIZE);
1002 }
1003
1004 /*
1005 * allocate object for the ptes
1006 */
1007 pmap->pm_pteobj = vm_object_allocate( OBJT_DEFAULT, PTDPTDI + 1);
1008
1009 /*
1010 * allocate the page directory page
1011 */
1012 retry:
1013 ptdpg = pmap_page_alloc( pmap->pm_pteobj, PTDPTDI);
1014 if (ptdpg == NULL)
1015 goto retry;
1016
1017 ptdpg->wire_count = 1;
1018 ++cnt.v_wire_count;
1019
1020 ptdpg->flags &= ~(PG_MAPPED|PG_BUSY); /* not mapped normally */
1021 ptdpg->valid = VM_PAGE_BITS_ALL;
1022
1023 pmap_kenter((vm_offset_t) pmap->pm_pdir, VM_PAGE_TO_PHYS(ptdpg));
1024 if ((ptdpg->flags & PG_ZERO) == 0)
1025 bzero(pmap->pm_pdir, PAGE_SIZE);
1026
1027 /* wire in kernel global address entries */
1028 bcopy(PTD + KPTDI, pmap->pm_pdir + KPTDI, nkpt * PTESIZE);
1029
1030 /* install self-referential address mapping entry */
1031 *(unsigned *) (pmap->pm_pdir + PTDPTDI) =
1032 VM_PAGE_TO_PHYS(ptdpg) | PG_V | PG_RW;
1033
1034 pmap->pm_flags = 0;
1035 pmap->pm_count = 1;
1036 pmap->pm_ptphint = NULL;
1037 #if PMAP_PVLIST
1038 TAILQ_INIT(&pmap->pm_pvlist);
1039 #endif
1040 }
1041
1042 static int
1043 pmap_release_free_page(pmap, p)
1044 struct pmap *pmap;
1045 vm_page_t p;
1046 {
1047 int s;
1048 unsigned *pde = (unsigned *) pmap->pm_pdir;
1049 /*
1050 * This code optimizes the case of freeing non-busy
1051 * page-table pages. Those pages are zero now, and
1052 * might as well be placed directly into the zero queue.
1053 */
1054 s = splvm();
1055 if (p->flags & PG_BUSY) {
1056 p->flags |= PG_WANTED;
1057 tsleep(p, PVM, "pmaprl", 0);
1058 splx(s);
1059 return 0;
1060 }
1061
1062 if (p->flags & PG_WANTED) {
1063 p->flags &= ~PG_WANTED;
1064 wakeup(p);
1065 }
1066
1067 /*
1068 * Remove the page table page from the processes address space.
1069 */
1070 pde[p->pindex] = 0;
1071 --pmap->pm_stats.resident_count;
1072
1073 if (p->hold_count) {
1074 panic("pmap_release: freeing held page table page");
1075 }
1076 /*
1077 * Page directory pages need to have the kernel
1078 * stuff cleared, so they can go into the zero queue also.
1079 */
1080 if (p->pindex == PTDPTDI) {
1081 bzero(pde + KPTDI, nkpt * PTESIZE);
1082 pde[APTDPTDI] = 0;
1083 pmap_kremove((vm_offset_t) pmap->pm_pdir);
1084 }
1085
1086 #if defined(PTPHINT)
1087 if (pmap->pm_ptphint &&
1088 (pmap->pm_ptphint->pindex == p->pindex))
1089 pmap->pm_ptphint = NULL;
1090 #endif
1091
1092 vm_page_free_zero(p);
1093 splx(s);
1094 return 1;
1095 }
1096
1097 /*
1098 * this routine is called if the page table page is not
1099 * mapped correctly.
1100 */
1101 static vm_page_t
1102 _pmap_allocpte(pmap, ptepindex)
1103 pmap_t pmap;
1104 unsigned ptepindex;
1105 {
1106 vm_offset_t pteva, ptepa;
1107 vm_page_t m;
1108 int needszero = 0;
1109
1110 /*
1111 * Find or fabricate a new pagetable page
1112 */
1113 retry:
1114 m = vm_page_lookup(pmap->pm_pteobj, ptepindex);
1115 if (m == NULL) {
1116 m = pmap_page_alloc(pmap->pm_pteobj, ptepindex);
1117 if (m == NULL)
1118 goto retry;
1119 if ((m->flags & PG_ZERO) == 0)
1120 needszero = 1;
1121 m->flags &= ~(PG_ZERO|PG_BUSY);
1122 m->valid = VM_PAGE_BITS_ALL;
1123 } else {
1124 if ((m->flags & PG_BUSY) || m->busy) {
1125 m->flags |= PG_WANTED;
1126 tsleep(m, PVM, "ptewai", 0);
1127 goto retry;
1128 }
1129 }
1130
1131 if (m->queue != PQ_NONE) {
1132 int s = splvm();
1133 vm_page_unqueue(m);
1134 splx(s);
1135 }
1136
1137 if (m->wire_count == 0)
1138 ++cnt.v_wire_count;
1139 ++m->wire_count;
1140
1141 /*
1142 * Increment the hold count for the page table page
1143 * (denoting a new mapping.)
1144 */
1145 ++m->hold_count;
1146
1147 /*
1148 * Map the pagetable page into the process address space, if
1149 * it isn't already there.
1150 */
1151
1152 pmap->pm_stats.resident_count++;
1153
1154 ptepa = VM_PAGE_TO_PHYS(m);
1155 pmap->pm_pdir[ptepindex] = (pd_entry_t) (ptepa | PG_U | PG_RW | PG_V);
1156
1157 #if defined(PTPHINT)
1158 /*
1159 * Set the page table hint
1160 */
1161 pmap->pm_ptphint = m;
1162 #endif
1163
1164 /*
1165 * Try to use the new mapping, but if we cannot, then
1166 * do it with the routine that maps the page explicitly.
1167 */
1168 if (needszero) {
1169 if ((((unsigned)pmap->pm_pdir[PTDPTDI]) & PG_FRAME) ==
1170 (((unsigned) PTDpde) & PG_FRAME)) {
1171 pteva = UPT_MIN_ADDRESS + i386_ptob(ptepindex);
1172 bzero((caddr_t) pteva, PAGE_SIZE);
1173 } else {
1174 pmap_zero_page(ptepa);
1175 }
1176 }
1177
1178 m->valid = VM_PAGE_BITS_ALL;
1179 m->flags |= PG_MAPPED;
1180
1181 return m;
1182 }
1183
1184 static vm_page_t
1185 pmap_allocpte(pmap, va)
1186 pmap_t pmap;
1187 vm_offset_t va;
1188 {
1189 unsigned ptepindex;
1190 vm_offset_t ptepa;
1191 vm_page_t m;
1192
1193 /*
1194 * Calculate pagetable page index
1195 */
1196 ptepindex = va >> PDRSHIFT;
1197
1198 /*
1199 * Get the page directory entry
1200 */
1201 ptepa = (vm_offset_t) pmap->pm_pdir[ptepindex];
1202
1203 /*
1204 * If the page table page is mapped, we just increment the
1205 * hold count, and activate it.
1206 */
1207 if (ptepa) {
1208 #if defined(PTPHINT)
1209 /*
1210 * In order to get the page table page, try the
1211 * hint first.
1212 */
1213 if (pmap->pm_ptphint &&
1214 (pmap->pm_ptphint->pindex == ptepindex)) {
1215 m = pmap->pm_ptphint;
1216 } else {
1217 m = pmap_page_lookup( pmap->pm_pteobj, ptepindex);
1218 pmap->pm_ptphint = m;
1219 }
1220 #else
1221 m = pmap_page_lookup( pmap->pm_pteobj, ptepindex);
1222 #endif
1223 ++m->hold_count;
1224 return m;
1225 }
1226 /*
1227 * Here if the pte page isn't mapped, or if it has been deallocated.
1228 */
1229 return _pmap_allocpte(pmap, ptepindex);
1230 }
1231
1232
1233 /***************************************************
1234 * Pmap allocation/deallocation routines.
1235 ***************************************************/
1236
1237 /*
1238 * Release any resources held by the given physical map.
1239 * Called when a pmap initialized by pmap_pinit is being released.
1240 * Should only be called if the map contains no valid mappings.
1241 */
1242 void
1243 pmap_release(pmap)
1244 register struct pmap *pmap;
1245 {
1246 vm_page_t p,n,ptdpg;
1247 vm_object_t object = pmap->pm_pteobj;
1248
1249 #if defined(DIAGNOSTIC)
1250 if (object->ref_count != 1)
1251 panic("pmap_release: pteobj reference count != 1");
1252 #endif
1253
1254 ptdpg = NULL;
1255 retry:
1256 for (p = TAILQ_FIRST(&object->memq); p != NULL; p = n) {
1257 n = TAILQ_NEXT(p, listq);
1258 if (p->pindex == PTDPTDI) {
1259 ptdpg = p;
1260 continue;
1261 }
1262 if (!pmap_release_free_page(pmap, p))
1263 goto retry;
1264 }
1265
1266 if (ptdpg && !pmap_release_free_page(pmap, ptdpg))
1267 goto retry;
1268
1269 vm_object_deallocate(object);
1270 if (pdstackptr < PDSTACKMAX) {
1271 pdstack[pdstackptr] = (vm_offset_t) pmap->pm_pdir;
1272 ++pdstackptr;
1273 } else {
1274 kmem_free(kernel_map, (vm_offset_t) pmap->pm_pdir, PAGE_SIZE);
1275 }
1276 pmap->pm_pdir = 0;
1277 }
1278
1279 /*
1280 * grow the number of kernel page table entries, if needed
1281 */
1282 void
1283 pmap_growkernel(vm_offset_t addr)
1284 {
1285 struct proc *p;
1286 struct pmap *pmap;
1287 int s;
1288
1289 s = splhigh();
1290 if (kernel_vm_end == 0) {
1291 kernel_vm_end = KERNBASE;
1292 nkpt = 0;
1293 while (pdir_pde(PTD, kernel_vm_end)) {
1294 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
1295 ++nkpt;
1296 }
1297 }
1298 addr = (addr + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
1299 while (kernel_vm_end < addr) {
1300 if (pdir_pde(PTD, kernel_vm_end)) {
1301 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
1302 continue;
1303 }
1304 ++nkpt;
1305 if (!nkpg) {
1306 vm_offset_t ptpkva = (vm_offset_t) vtopte(addr);
1307 /*
1308 * This index is bogus, but out of the way
1309 */
1310 vm_pindex_t ptpidx = (ptpkva >> PAGE_SHIFT);
1311 nkpg = vm_page_alloc(kernel_object,
1312 ptpidx, VM_ALLOC_SYSTEM);
1313 if (!nkpg)
1314 panic("pmap_growkernel: no memory to grow kernel");
1315 vm_page_wire(nkpg);
1316 vm_page_remove(nkpg);
1317 pmap_zero_page(VM_PAGE_TO_PHYS(nkpg));
1318 }
1319 pdir_pde(PTD, kernel_vm_end) = (pd_entry_t) (VM_PAGE_TO_PHYS(nkpg) | PG_V | PG_RW);
1320 nkpg = NULL;
1321
1322 for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) {
1323 if (p->p_vmspace) {
1324 pmap = &p->p_vmspace->vm_pmap;
1325 *pmap_pde(pmap, kernel_vm_end) = pdir_pde(PTD, kernel_vm_end);
1326 }
1327 }
1328 *pmap_pde(kernel_pmap, kernel_vm_end) = pdir_pde(PTD, kernel_vm_end);
1329 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
1330 }
1331 splx(s);
1332 }
1333
1334 /*
1335 * Retire the given physical map from service.
1336 * Should only be called if the map contains
1337 * no valid mappings.
1338 */
1339 void
1340 pmap_destroy(pmap)
1341 register pmap_t pmap;
1342 {
1343 int count;
1344
1345 if (pmap == NULL)
1346 return;
1347
1348 count = --pmap->pm_count;
1349 if (count == 0) {
1350 pmap_release(pmap);
1351 free((caddr_t) pmap, M_VMPMAP);
1352 }
1353 }
1354
1355 /*
1356 * Add a reference to the specified pmap.
1357 */
1358 void
1359 pmap_reference(pmap)
1360 pmap_t pmap;
1361 {
1362 if (pmap != NULL) {
1363 pmap->pm_count++;
1364 }
1365 }
1366
1367 /***************************************************
1368 * page management routines.
1369 ***************************************************/
1370
1371 /*
1372 * free the pv_entry back to the free list
1373 */
1374 static PMAP_INLINE void
1375 free_pv_entry(pv)
1376 pv_entry_t pv;
1377 {
1378 ++pv_freelistcnt;
1379 TAILQ_INSERT_HEAD(&pv_freelist, pv, pv_list);
1380 }
1381
1382 /*
1383 * get a new pv_entry, allocating a block from the system
1384 * when needed.
1385 * the memory allocation is performed bypassing the malloc code
1386 * because of the possibility of allocations at interrupt time.
1387 */
1388 static pv_entry_t
1389 get_pv_entry()
1390 {
1391 pv_entry_t tmp;
1392
1393 /*
1394 * get more pv_entry pages if needed
1395 */
1396 if (pv_freelistcnt < PV_FREELIST_MIN || !TAILQ_FIRST(&pv_freelist)) {
1397 pmap_alloc_pv_entry();
1398 }
1399 /*
1400 * get a pv_entry off of the free list
1401 */
1402 --pv_freelistcnt;
1403 tmp = TAILQ_FIRST(&pv_freelist);
1404 TAILQ_REMOVE(&pv_freelist, tmp, pv_list);
1405 return tmp;
1406 }
1407
1408 /*
1409 * This *strange* allocation routine eliminates the possibility of a malloc
1410 * failure (*FATAL*) for a pv_entry_t data structure.
1411 * also -- this code is MUCH MUCH faster than the malloc equiv...
1412 * We really need to do the slab allocator thingie here.
1413 */
1414 static void
1415 pmap_alloc_pv_entry()
1416 {
1417 /*
1418 * do we have any pre-allocated map-pages left?
1419 */
1420 if (npvvapg) {
1421 vm_page_t m;
1422
1423 /*
1424 * allocate a physical page out of the vm system
1425 */
1426 m = vm_page_alloc(kernel_object,
1427 OFF_TO_IDX(pvva - vm_map_min(kernel_map)),
1428 VM_ALLOC_INTERRUPT);
1429 if (m) {
1430 int newentries;
1431 int i;
1432 pv_entry_t entry;
1433
1434 newentries = (PAGE_SIZE / sizeof(struct pv_entry));
1435 /*
1436 * wire the page
1437 */
1438 vm_page_wire(m);
1439 m->flags &= ~PG_BUSY;
1440 /*
1441 * let the kernel see it
1442 */
1443 pmap_kenter(pvva, VM_PAGE_TO_PHYS(m));
1444
1445 entry = (pv_entry_t) pvva;
1446 /*
1447 * update the allocation pointers
1448 */
1449 pvva += PAGE_SIZE;
1450 --npvvapg;
1451
1452 /*
1453 * free the entries into the free list
1454 */
1455 for (i = 0; i < newentries; i++) {
1456 free_pv_entry(entry);
1457 entry++;
1458 }
1459 }
1460 }
1461 if (!TAILQ_FIRST(&pv_freelist))
1462 panic("get_pv_entry: cannot get a pv_entry_t");
1463 }
1464
1465 /*
1466 * init the pv_entry allocation system
1467 */
1468 void
1469 init_pv_entries(npg)
1470 int npg;
1471 {
1472 /*
1473 * Allocate enough kvm space for one entry per page, and
1474 * each process having PMAP_SHPGPERPROC pages shared with other
1475 * processes. (The system can panic if this is too small, but also
1476 * can fail on bootup if this is too big.)
1477 * XXX The pv management mechanism needs to be fixed so that systems
1478 * with lots of shared mappings amongst lots of processes will still
1479 * work. The fix will likely be that once we run out of pv entries
1480 * we will free other entries (and the associated mappings), with
1481 * some policy yet to be determined.
1482 */
1483 npvvapg = ((PMAP_SHPGPERPROC * maxproc + npg) * sizeof(struct pv_entry)
1484 + PAGE_SIZE - 1) / PAGE_SIZE;
1485 pvva = kmem_alloc_pageable(kernel_map, npvvapg * PAGE_SIZE);
1486 /*
1487 * get the first batch of entries
1488 */
1489 pmap_alloc_pv_entry();
1490 }
1491
1492 /*
1493 * If it is the first entry on the list, it is actually
1494 * in the header and we must copy the following entry up
1495 * to the header. Otherwise we must search the list for
1496 * the entry. In either case we free the now unused entry.
1497 */
1498
1499 static int
1500 pmap_remove_entry(pmap, ppv, va)
1501 struct pmap *pmap;
1502 pv_table_t *ppv;
1503 vm_offset_t va;
1504 {
1505 pv_entry_t pv;
1506 int rtval;
1507 int s;
1508
1509 s = splvm();
1510 #if PMAP_PVLIST
1511 if (ppv->pv_list_count < pmap->pm_stats.resident_count) {
1512 #endif
1513 for (pv = TAILQ_FIRST(&ppv->pv_list);
1514 pv;
1515 pv = TAILQ_NEXT(pv, pv_list)) {
1516 if (pmap == pv->pv_pmap && va == pv->pv_va)
1517 break;
1518 }
1519 #if PMAP_PVLIST
1520 } else {
1521 for (pv = TAILQ_FIRST(&pmap->pm_pvlist);
1522 pv;
1523 pv = TAILQ_NEXT(pv, pv_plist)) {
1524 if (va == pv->pv_va)
1525 break;
1526 }
1527 }
1528 #endif
1529
1530 rtval = 0;
1531 if (pv) {
1532 rtval = pmap_unuse_pt(pmap, va, pv->pv_ptem);
1533 TAILQ_REMOVE(&ppv->pv_list, pv, pv_list);
1534 --ppv->pv_list_count;
1535 if (TAILQ_FIRST(&ppv->pv_list) == NULL) {
1536 ppv->pv_vm_page->flags &= ~(PG_MAPPED|PG_WRITEABLE);
1537 }
1538
1539 #if PMAP_PVLIST
1540 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
1541 #endif
1542 free_pv_entry(pv);
1543 }
1544
1545 splx(s);
1546 return rtval;
1547 }
1548
1549 /*
1550 * Create a pv entry for page at pa for
1551 * (pmap, va).
1552 */
1553 static void
1554 pmap_insert_entry(pmap, va, mpte, pa)
1555 pmap_t pmap;
1556 vm_offset_t va;
1557 vm_page_t mpte;
1558 vm_offset_t pa;
1559 {
1560
1561 int s;
1562 pv_entry_t pv;
1563 pv_table_t *ppv;
1564
1565 s = splvm();
1566 pv = get_pv_entry();
1567 pv->pv_va = va;
1568 pv->pv_pmap = pmap;
1569 pv->pv_ptem = mpte;
1570
1571 #if PMAP_PVLIST
1572 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist);
1573 #endif
1574
1575 ppv = pa_to_pvh(pa);
1576 TAILQ_INSERT_TAIL(&ppv->pv_list, pv, pv_list);
1577 ++ppv->pv_list_count;
1578
1579 splx(s);
1580 }
1581
1582 /*
1583 * pmap_remove_pte: do the things to unmap a page in a process
1584 */
1585 static int
1586 pmap_remove_pte(pmap, ptq, va)
1587 struct pmap *pmap;
1588 unsigned *ptq;
1589 vm_offset_t va;
1590 {
1591 unsigned oldpte;
1592 pv_table_t *ppv;
1593
1594 oldpte = *ptq;
1595 *ptq = 0;
1596 if (oldpte & PG_W)
1597 pmap->pm_stats.wired_count -= 1;
1598 /*
1599 * Machines that don't support invlpg, also don't support
1600 * PG_G.
1601 */
1602 if (oldpte & PG_G)
1603 invlpg(va);
1604 pmap->pm_stats.resident_count -= 1;
1605 if (oldpte & PG_MANAGED) {
1606 ppv = pa_to_pvh(oldpte);
1607 if (oldpte & PG_M) {
1608 #if defined(PMAP_DIAGNOSTIC)
1609 if (pmap_nw_modified((pt_entry_t) oldpte)) {
1610 printf("pmap_remove: modified page not writable: va: 0x%lx, pte: 0x%lx\n", va, (int) oldpte);
1611 }
1612 #endif
1613 if (pmap_track_modified(va))
1614 ppv->pv_vm_page->dirty = VM_PAGE_BITS_ALL;
1615 }
1616 return pmap_remove_entry(pmap, ppv, va);
1617 } else {
1618 return pmap_unuse_pt(pmap, va, NULL);
1619 }
1620
1621 return 0;
1622 }
1623
1624 /*
1625 * Remove a single page from a process address space
1626 */
1627 static void
1628 pmap_remove_page(pmap, va)
1629 struct pmap *pmap;
1630 register vm_offset_t va;
1631 {
1632 register unsigned *ptq;
1633
1634 /*
1635 * if there is no pte for this address, just skip it!!!
1636 */
1637 if (*pmap_pde(pmap, va) == 0) {
1638 return;
1639 }
1640
1641 /*
1642 * get a local va for mappings for this pmap.
1643 */
1644 ptq = get_ptbase(pmap) + i386_btop(va);
1645 if (*ptq) {
1646 (void) pmap_remove_pte(pmap, ptq, va);
1647 invltlb_1pg(va);
1648 }
1649 return;
1650 }
1651
1652 /*
1653 * Remove the given range of addresses from the specified map.
1654 *
1655 * It is assumed that the start and end are properly
1656 * rounded to the page size.
1657 */
1658 void
1659 pmap_remove(pmap, sva, eva)
1660 struct pmap *pmap;
1661 register vm_offset_t sva;
1662 register vm_offset_t eva;
1663 {
1664 register unsigned *ptbase;
1665 vm_offset_t pdnxt;
1666 vm_offset_t ptpaddr;
1667 vm_offset_t sindex, eindex;
1668 int anyvalid;
1669
1670 if (pmap == NULL)
1671 return;
1672
1673 if (pmap->pm_stats.resident_count == 0)
1674 return;
1675
1676 /*
1677 * special handling of removing one page. a very
1678 * common operation and easy to short circuit some
1679 * code.
1680 */
1681 if ((sva + PAGE_SIZE) == eva) {
1682 pmap_remove_page(pmap, sva);
1683 return;
1684 }
1685
1686 anyvalid = 0;
1687
1688 /*
1689 * Get a local virtual address for the mappings that are being
1690 * worked with.
1691 */
1692 ptbase = get_ptbase(pmap);
1693
1694 sindex = i386_btop(sva);
1695 eindex = i386_btop(eva);
1696
1697 for (; sindex < eindex; sindex = pdnxt) {
1698
1699 /*
1700 * Calculate index for next page table.
1701 */
1702 pdnxt = ((sindex + NPTEPG) & ~(NPTEPG - 1));
1703 if (pmap->pm_stats.resident_count == 0)
1704 break;
1705 ptpaddr = (vm_offset_t) *pmap_pde(pmap, i386_ptob(sindex));
1706
1707 /*
1708 * Weed out invalid mappings. Note: we assume that the page
1709 * directory table is always allocated, and in kernel virtual.
1710 */
1711 if (ptpaddr == 0)
1712 continue;
1713
1714 /*
1715 * Limit our scan to either the end of the va represented
1716 * by the current page table page, or to the end of the
1717 * range being removed.
1718 */
1719 if (pdnxt > eindex) {
1720 pdnxt = eindex;
1721 }
1722
1723 for ( ;sindex != pdnxt; sindex++) {
1724 vm_offset_t va;
1725 if (ptbase[sindex] == 0) {
1726 continue;
1727 }
1728 va = i386_ptob(sindex);
1729
1730 anyvalid++;
1731 if (pmap_remove_pte(pmap,
1732 ptbase + sindex, va))
1733 break;
1734 }
1735 }
1736
1737 if (anyvalid) {
1738 invltlb();
1739 }
1740 }
1741
1742 /*
1743 * Routine: pmap_remove_all
1744 * Function:
1745 * Removes this physical page from
1746 * all physical maps in which it resides.
1747 * Reflects back modify bits to the pager.
1748 *
1749 * Notes:
1750 * Original versions of this routine were very
1751 * inefficient because they iteratively called
1752 * pmap_remove (slow...)
1753 */
1754
1755 static void
1756 pmap_remove_all(pa)
1757 vm_offset_t pa;
1758 {
1759 register pv_entry_t pv;
1760 pv_table_t *ppv;
1761 register unsigned *pte, tpte;
1762 int nmodify;
1763 int update_needed;
1764 int s;
1765
1766 nmodify = 0;
1767 update_needed = 0;
1768 #if defined(PMAP_DIAGNOSTIC)
1769 /*
1770 * XXX this makes pmap_page_protect(NONE) illegal for non-managed
1771 * pages!
1772 */
1773 if (!pmap_is_managed(pa)) {
1774 panic("pmap_page_protect: illegal for unmanaged page, va: 0x%lx", pa);
1775 }
1776 #endif
1777
1778 s = splvm();
1779 ppv = pa_to_pvh(pa);
1780 while ((pv = TAILQ_FIRST(&ppv->pv_list)) != NULL) {
1781 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
1782
1783 pv->pv_pmap->pm_stats.resident_count--;
1784
1785 tpte = *pte;
1786 *pte = 0;
1787 if (tpte & PG_W)
1788 pv->pv_pmap->pm_stats.wired_count--;
1789 /*
1790 * Update the vm_page_t clean and reference bits.
1791 */
1792 if (tpte & PG_M) {
1793 #if defined(PMAP_DIAGNOSTIC)
1794 if (pmap_nw_modified((pt_entry_t) tpte)) {
1795 printf("pmap_remove_all: modified page not writable: va: 0x%lx, pte: 0x%lx\n", pv->pv_va, tpte);
1796 }
1797 #endif
1798 if (pmap_track_modified(pv->pv_va))
1799 ppv->pv_vm_page->dirty = VM_PAGE_BITS_ALL;
1800 }
1801 if (!update_needed &&
1802 ((!curproc || (&curproc->p_vmspace->vm_pmap == pv->pv_pmap)) ||
1803 (pv->pv_pmap == kernel_pmap))) {
1804 update_needed = 1;
1805 }
1806
1807 #if PMAP_PVLIST
1808 TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist);
1809 #endif
1810 TAILQ_REMOVE(&ppv->pv_list, pv, pv_list);
1811 --ppv->pv_list_count;
1812 pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem);
1813 free_pv_entry(pv);
1814 }
1815 ppv->pv_vm_page->flags &= ~(PG_MAPPED|PG_WRITEABLE);
1816
1817
1818 if (update_needed)
1819 invltlb();
1820 splx(s);
1821 return;
1822 }
1823
1824 /*
1825 * Set the physical protection on the
1826 * specified range of this map as requested.
1827 */
1828 void
1829 pmap_protect(pmap, sva, eva, prot)
1830 register pmap_t pmap;
1831 vm_offset_t sva, eva;
1832 vm_prot_t prot;
1833 {
1834 register unsigned *ptbase;
1835 vm_offset_t pdnxt;
1836 vm_offset_t ptpaddr;
1837 vm_offset_t sindex, eindex;
1838 int anychanged;
1839
1840
1841 if (pmap == NULL)
1842 return;
1843
1844 if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
1845 pmap_remove(pmap, sva, eva);
1846 return;
1847 }
1848 if (prot & VM_PROT_WRITE) {
1849 return;
1850 }
1851
1852 anychanged = 0;
1853
1854 ptbase = get_ptbase(pmap);
1855
1856 sindex = i386_btop(sva);
1857 eindex = i386_btop(eva);
1858
1859 for (; sindex < eindex; sindex = pdnxt) {
1860
1861 pdnxt = ((sindex + NPTEPG) & ~(NPTEPG - 1));
1862 ptpaddr = (vm_offset_t) *pmap_pde(pmap, i386_ptob(sindex));
1863
1864 /*
1865 * Weed out invalid mappings. Note: we assume that the page
1866 * directory table is always allocated, and in kernel virtual.
1867 */
1868 if (ptpaddr == 0)
1869 continue;
1870
1871 if (pdnxt > eindex) {
1872 pdnxt = eindex;
1873 }
1874
1875 for (; sindex != pdnxt; sindex++) {
1876
1877 unsigned pbits = ptbase[sindex];
1878
1879 if (pbits & PG_RW) {
1880 if (pbits & PG_M) {
1881 vm_offset_t sva = i386_ptob(sindex);
1882 if (pmap_track_modified(sva)) {
1883 vm_page_t m = PHYS_TO_VM_PAGE(pbits);
1884 m->dirty = VM_PAGE_BITS_ALL;
1885 }
1886 }
1887 ptbase[sindex] = pbits & ~(PG_M|PG_RW);
1888 anychanged = 1;
1889 }
1890 }
1891 }
1892 if (anychanged)
1893 invltlb();
1894 }
1895
1896 /*
1897 * Insert the given physical page (p) at
1898 * the specified virtual address (v) in the
1899 * target physical map with the protection requested.
1900 *
1901 * If specified, the page will be wired down, meaning
1902 * that the related pte can not be reclaimed.
1903 *
1904 * NB: This is the only routine which MAY NOT lazy-evaluate
1905 * or lose information. That is, this routine must actually
1906 * insert this page into the given map NOW.
1907 */
1908 void
1909 pmap_enter(pmap, va, pa, prot, wired)
1910 register pmap_t pmap;
1911 vm_offset_t va;
1912 register vm_offset_t pa;
1913 vm_prot_t prot;
1914 boolean_t wired;
1915 {
1916 register unsigned *pte;
1917 vm_offset_t opa;
1918 vm_offset_t origpte, newpte;
1919 vm_page_t mpte;
1920
1921 if (pmap == NULL)
1922 return;
1923
1924 va &= PG_FRAME;
1925 #ifdef PMAP_DIAGNOSTIC
1926 if (va > VM_MAX_KERNEL_ADDRESS)
1927 panic("pmap_enter: toobig");
1928 if ((va >= UPT_MIN_ADDRESS) && (va < UPT_MAX_ADDRESS))
1929 panic("pmap_enter: invalid to pmap_enter page table pages (va: 0x%x)", va);
1930 #endif
1931
1932 mpte = NULL;
1933 /*
1934 * In the case that a page table page is not
1935 * resident, we are creating it here.
1936 */
1937 if (va < UPT_MIN_ADDRESS)
1938 mpte = pmap_allocpte(pmap, va);
1939
1940 pte = pmap_pte(pmap, va);
1941 /*
1942 * Page Directory table entry not valid, we need a new PT page
1943 */
1944 if (pte == NULL) {
1945 panic("pmap_enter: invalid page directory, pdir=%p, va=0x%lx\n",
1946 pmap->pm_pdir[PTDPTDI], va);
1947 }
1948
1949 origpte = *(vm_offset_t *)pte;
1950 pa &= PG_FRAME;
1951 opa = origpte & PG_FRAME;
1952
1953 /*
1954 * Mapping has not changed, must be protection or wiring change.
1955 */
1956 if (origpte && (opa == pa)) {
1957 /*
1958 * Wiring change, just update stats. We don't worry about
1959 * wiring PT pages as they remain resident as long as there
1960 * are valid mappings in them. Hence, if a user page is wired,
1961 * the PT page will be also.
1962 */
1963 if (wired && ((origpte & PG_W) == 0))
1964 pmap->pm_stats.wired_count++;
1965 else if (!wired && (origpte & PG_W))
1966 pmap->pm_stats.wired_count--;
1967
1968 #if defined(PMAP_DIAGNOSTIC)
1969 if (pmap_nw_modified((pt_entry_t) origpte)) {
1970 printf("pmap_enter: modified page not writable: va: 0x%lx, pte: 0x%lx\n", va, origpte);
1971 }
1972 #endif
1973
1974 /*
1975 * We might be turning off write access to the page,
1976 * so we go ahead and sense modify status.
1977 */
1978 if (origpte & PG_MANAGED) {
1979 vm_page_t m;
1980 if (origpte & PG_M) {
1981 if (pmap_track_modified(va)) {
1982 m = PHYS_TO_VM_PAGE(pa);
1983 m->dirty = VM_PAGE_BITS_ALL;
1984 }
1985 }
1986 pa |= PG_MANAGED;
1987 }
1988
1989 if (mpte)
1990 --mpte->hold_count;
1991
1992 goto validate;
1993 }
1994 /*
1995 * Mapping has changed, invalidate old range and fall through to
1996 * handle validating new mapping.
1997 */
1998 if (opa) {
1999 int err;
2000 err = pmap_remove_pte(pmap, pte, va);
2001 if (err)
2002 panic("pmap_enter: pte vanished, va: 0x%x", va);
2003 }
2004
2005 /*
2006 * Enter on the PV list if part of our managed memory Note that we
2007 * raise IPL while manipulating pv_table since pmap_enter can be
2008 * called at interrupt time.
2009 */
2010 if (pmap_is_managed(pa)) {
2011 pmap_insert_entry(pmap, va, mpte, pa);
2012 pa |= PG_MANAGED;
2013 }
2014
2015 /*
2016 * Increment counters
2017 */
2018 pmap->pm_stats.resident_count++;
2019 if (wired)
2020 pmap->pm_stats.wired_count++;
2021
2022 validate:
2023 /*
2024 * Now validate mapping with desired protection/wiring.
2025 */
2026 newpte = (vm_offset_t) (pa | pte_prot(pmap, prot) | PG_V);
2027
2028 if (wired)
2029 newpte |= PG_W;
2030 if (va < UPT_MIN_ADDRESS)
2031 newpte |= PG_U;
2032 if (pmap == kernel_pmap)
2033 newpte |= pgeflag;
2034
2035 /*
2036 * if the mapping or permission bits are different, we need
2037 * to update the pte.
2038 */
2039 if ((origpte & ~(PG_M|PG_A)) != newpte) {
2040 *pte = newpte;
2041 if (origpte)
2042 invltlb_1pg(va);
2043 }
2044 }
2045
2046 /*
2047 * this code makes some *MAJOR* assumptions:
2048 * 1. Current pmap & pmap exists.
2049 * 2. Not wired.
2050 * 3. Read access.
2051 * 4. No page table pages.
2052 * 5. Tlbflush is deferred to calling procedure.
2053 * 6. Page IS managed.
2054 * but is *MUCH* faster than pmap_enter...
2055 */
2056
2057 static vm_page_t
2058 pmap_enter_quick(pmap, va, pa, mpte)
2059 register pmap_t pmap;
2060 vm_offset_t va;
2061 register vm_offset_t pa;
2062 vm_page_t mpte;
2063 {
2064 register unsigned *pte;
2065
2066 /*
2067 * In the case that a page table page is not
2068 * resident, we are creating it here.
2069 */
2070 if (va < UPT_MIN_ADDRESS) {
2071 unsigned ptepindex;
2072 vm_offset_t ptepa;
2073
2074 /*
2075 * Calculate pagetable page index
2076 */
2077 ptepindex = va >> PDRSHIFT;
2078 if (mpte && (mpte->pindex == ptepindex)) {
2079 ++mpte->hold_count;
2080 } else {
2081 retry:
2082 /*
2083 * Get the page directory entry
2084 */
2085 ptepa = (vm_offset_t) pmap->pm_pdir[ptepindex];
2086
2087 /*
2088 * If the page table page is mapped, we just increment
2089 * the hold count, and activate it.
2090 */
2091 if (ptepa) {
2092 #if defined(PTPHINT)
2093 if (pmap->pm_ptphint &&
2094 (pmap->pm_ptphint->pindex == ptepindex)) {
2095 mpte = pmap->pm_ptphint;
2096 } else {
2097 mpte = pmap_page_lookup( pmap->pm_pteobj, ptepindex);
2098 pmap->pm_ptphint = mpte;
2099 }
2100 #else
2101 mpte = pmap_page_lookup( pmap->pm_pteobj, ptepindex);
2102 #endif
2103 if (mpte == NULL)
2104 goto retry;
2105 ++mpte->hold_count;
2106 } else {
2107 mpte = _pmap_allocpte(pmap, ptepindex);
2108 }
2109 }
2110 } else {
2111 mpte = NULL;
2112 }
2113
2114 /*
2115 * This call to vtopte makes the assumption that we are
2116 * entering the page into the current pmap. In order to support
2117 * quick entry into any pmap, one would likely use pmap_pte_quick.
2118 * But that isn't as quick as vtopte.
2119 */
2120 pte = (unsigned *)vtopte(va);
2121 if (*pte) {
2122 if (mpte)
2123 pmap_unwire_pte_hold(pmap, mpte);
2124 return 0;
2125 }
2126
2127 /*
2128 * Enter on the PV list if part of our managed memory Note that we
2129 * raise IPL while manipulating pv_table since pmap_enter can be
2130 * called at interrupt time.
2131 */
2132 pmap_insert_entry(pmap, va, mpte, pa);
2133
2134 /*
2135 * Increment counters
2136 */
2137 pmap->pm_stats.resident_count++;
2138
2139 /*
2140 * Now validate mapping with RO protection
2141 */
2142 *pte = pa | PG_V | PG_U | PG_MANAGED;
2143
2144 return mpte;
2145 }
2146
2147 #define MAX_INIT_PT (96)
2148 /*
2149 * pmap_object_init_pt preloads the ptes for a given object
2150 * into the specified pmap. This eliminates the blast of soft
2151 * faults on process startup and immediately after an mmap.
2152 */
2153 void
2154 pmap_object_init_pt(pmap, addr, object, pindex, size, limit)
2155 pmap_t pmap;
2156 vm_offset_t addr;
2157 vm_object_t object;
2158 vm_pindex_t pindex;
2159 vm_size_t size;
2160 int limit;
2161 {
2162 vm_offset_t tmpidx;
2163 int psize;
2164 vm_page_t p, mpte;
2165 int objpgs;
2166
2167 psize = i386_btop(size);
2168
2169 if (!pmap || (object->type != OBJT_VNODE) ||
2170 (limit && (psize > MAX_INIT_PT) &&
2171 (object->resident_page_count > MAX_INIT_PT))) {
2172 return;
2173 }
2174
2175 if (psize + pindex > object->size)
2176 psize = object->size - pindex;
2177
2178 mpte = NULL;
2179 /*
2180 * if we are processing a major portion of the object, then scan the
2181 * entire thing.
2182 */
2183 if (psize > (object->size >> 2)) {
2184 objpgs = psize;
2185
2186 for (p = TAILQ_FIRST(&object->memq);
2187 ((objpgs > 0) && (p != NULL));
2188 p = TAILQ_NEXT(p, listq)) {
2189
2190 tmpidx = p->pindex;
2191 if (tmpidx < pindex) {
2192 continue;
2193 }
2194 tmpidx -= pindex;
2195 if (tmpidx >= psize) {
2196 continue;
2197 }
2198 if (((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) &&
2199 (p->busy == 0) &&
2200 (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) {
2201 if ((p->queue - p->pc) == PQ_CACHE)
2202 vm_page_deactivate(p);
2203 p->flags |= PG_BUSY;
2204 mpte = pmap_enter_quick(pmap,
2205 addr + i386_ptob(tmpidx),
2206 VM_PAGE_TO_PHYS(p), mpte);
2207 p->flags |= PG_MAPPED;
2208 PAGE_WAKEUP(p);
2209 }
2210 objpgs -= 1;
2211 }
2212 } else {
2213 /*
2214 * else lookup the pages one-by-one.
2215 */
2216 for (tmpidx = 0; tmpidx < psize; tmpidx += 1) {
2217 p = vm_page_lookup(object, tmpidx + pindex);
2218 if (p &&
2219 ((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) &&
2220 (p->busy == 0) &&
2221 (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) {
2222 if ((p->queue - p->pc) == PQ_CACHE)
2223 vm_page_deactivate(p);
2224 p->flags |= PG_BUSY;
2225 mpte = pmap_enter_quick(pmap,
2226 addr + i386_ptob(tmpidx),
2227 VM_PAGE_TO_PHYS(p), mpte);
2228 p->flags |= PG_MAPPED;
2229 PAGE_WAKEUP(p);
2230 }
2231 }
2232 }
2233 return;
2234 }
2235
2236 /*
2237 * pmap_prefault provides a quick way of clustering
2238 * pagefaults into a processes address space. It is a "cousin"
2239 * of pmap_object_init_pt, except it runs at page fault time instead
2240 * of mmap time.
2241 */
2242 #define PFBAK 2
2243 #define PFFOR 2
2244 #define PAGEORDER_SIZE (PFBAK+PFFOR)
2245
2246 static int pmap_prefault_pageorder[] = {
2247 -PAGE_SIZE, PAGE_SIZE, -2 * PAGE_SIZE, 2 * PAGE_SIZE
2248 };
2249
2250 void
2251 pmap_prefault(pmap, addra, entry, object)
2252 pmap_t pmap;
2253 vm_offset_t addra;
2254 vm_map_entry_t entry;
2255 vm_object_t object;
2256 {
2257 int i;
2258 vm_offset_t starta;
2259 vm_offset_t addr;
2260 vm_pindex_t pindex;
2261 vm_page_t m, mpte;
2262
2263 if (entry->object.vm_object != object)
2264 return;
2265
2266 if (!curproc || (pmap != &curproc->p_vmspace->vm_pmap))
2267 return;
2268
2269 starta = addra - PFBAK * PAGE_SIZE;
2270 if (starta < entry->start) {
2271 starta = entry->start;
2272 } else if (starta > addra) {
2273 starta = 0;
2274 }
2275
2276 mpte = NULL;
2277 for (i = 0; i < PAGEORDER_SIZE; i++) {
2278 vm_object_t lobject;
2279 unsigned *pte;
2280
2281 addr = addra + pmap_prefault_pageorder[i];
2282 if (addr < starta || addr >= entry->end)
2283 continue;
2284
2285 if ((*pmap_pde(pmap, addr)) == NULL)
2286 continue;
2287
2288 pte = (unsigned *) vtopte(addr);
2289 if (*pte)
2290 continue;
2291
2292 pindex = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT;
2293 lobject = object;
2294 for (m = vm_page_lookup(lobject, pindex);
2295 (!m && (lobject->type == OBJT_DEFAULT) && (lobject->backing_object));
2296 lobject = lobject->backing_object) {
2297 if (lobject->backing_object_offset & PAGE_MASK)
2298 break;
2299 pindex += (lobject->backing_object_offset >> PAGE_SHIFT);
2300 m = vm_page_lookup(lobject->backing_object, pindex);
2301 }
2302
2303 /*
2304 * give-up when a page is not in memory
2305 */
2306 if (m == NULL)
2307 break;
2308
2309 if (((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) &&
2310 (m->busy == 0) &&
2311 (m->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) {
2312
2313 if ((m->queue - m->pc) == PQ_CACHE) {
2314 vm_page_deactivate(m);
2315 }
2316 m->flags |= PG_BUSY;
2317 mpte = pmap_enter_quick(pmap, addr,
2318 VM_PAGE_TO_PHYS(m), mpte);
2319 m->flags |= PG_MAPPED;
2320 PAGE_WAKEUP(m);
2321 }
2322 }
2323 }
2324
2325 /*
2326 * Routine: pmap_change_wiring
2327 * Function: Change the wiring attribute for a map/virtual-address
2328 * pair.
2329 * In/out conditions:
2330 * The mapping must already exist in the pmap.
2331 */
2332 void
2333 pmap_change_wiring(pmap, va, wired)
2334 register pmap_t pmap;
2335 vm_offset_t va;
2336 boolean_t wired;
2337 {
2338 register unsigned *pte;
2339
2340 if (pmap == NULL)
2341 return;
2342
2343 pte = pmap_pte(pmap, va);
2344
2345 if (wired && !pmap_pte_w(pte))
2346 pmap->pm_stats.wired_count++;
2347 else if (!wired && pmap_pte_w(pte))
2348 pmap->pm_stats.wired_count--;
2349
2350 /*
2351 * Wiring is not a hardware characteristic so there is no need to
2352 * invalidate TLB.
2353 */
2354 pmap_pte_set_w(pte, wired);
2355 }
2356
2357
2358
2359 /*
2360 * Copy the range specified by src_addr/len
2361 * from the source map to the range dst_addr/len
2362 * in the destination map.
2363 *
2364 * This routine is only advisory and need not do anything.
2365 */
2366
2367 void
2368 pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr)
2369 pmap_t dst_pmap, src_pmap;
2370 vm_offset_t dst_addr;
2371 vm_size_t len;
2372 vm_offset_t src_addr;
2373 {
2374 vm_offset_t addr;
2375 vm_offset_t end_addr = src_addr + len;
2376 vm_offset_t pdnxt;
2377 unsigned src_frame, dst_frame;
2378
2379 if (dst_addr != src_addr)
2380 return;
2381
2382 src_frame = ((unsigned) src_pmap->pm_pdir[PTDPTDI]) & PG_FRAME;
2383 if (src_frame != (((unsigned) PTDpde) & PG_FRAME)) {
2384 return;
2385 }
2386
2387 dst_frame = ((unsigned) dst_pmap->pm_pdir[PTDPTDI]) & PG_FRAME;
2388 if (dst_frame != (((unsigned) APTDpde) & PG_FRAME)) {
2389 APTDpde = (pd_entry_t) (dst_frame | PG_RW | PG_V);
2390 invltlb();
2391 }
2392
2393 for(addr = src_addr; addr < end_addr; addr = pdnxt) {
2394 unsigned *src_pte, *dst_pte;
2395 vm_page_t dstmpte, srcmpte;
2396 vm_offset_t srcptepaddr;
2397 unsigned ptepindex;
2398
2399 if (addr >= UPT_MIN_ADDRESS)
2400 panic("pmap_copy: invalid to pmap_copy page tables\n");
2401
2402 pdnxt = ((addr + PAGE_SIZE*NPTEPG) & ~(PAGE_SIZE*NPTEPG - 1));
2403 ptepindex = addr >> PDRSHIFT;
2404
2405 srcptepaddr = (vm_offset_t) src_pmap->pm_pdir[ptepindex];
2406 if (srcptepaddr == 0)
2407 continue;
2408
2409 srcmpte = vm_page_lookup(src_pmap->pm_pteobj, ptepindex);
2410 if ((srcmpte->hold_count == 0) || (srcmpte->flags & PG_BUSY))
2411 continue;
2412
2413 if (pdnxt > end_addr)
2414 pdnxt = end_addr;
2415
2416 src_pte = (unsigned *) vtopte(addr);
2417 dst_pte = (unsigned *) avtopte(addr);
2418 while (addr < pdnxt) {
2419 unsigned ptetemp;
2420 ptetemp = *src_pte;
2421 /*
2422 * we only virtual copy managed pages
2423 */
2424 if ((ptetemp & PG_MANAGED) != 0) {
2425 /*
2426 * We have to check after allocpte for the
2427 * pte still being around... allocpte can
2428 * block.
2429 */
2430 dstmpte = pmap_allocpte(dst_pmap, addr);
2431 if ((*dst_pte == 0) && (ptetemp = *src_pte)) {
2432 /*
2433 * Clear the modified and
2434 * accessed (referenced) bits
2435 * during the copy.
2436 */
2437 *dst_pte = ptetemp & ~(PG_M|PG_A);
2438 dst_pmap->pm_stats.resident_count++;
2439 pmap_insert_entry(dst_pmap, addr,
2440 dstmpte,
2441 (ptetemp & PG_FRAME));
2442 } else {
2443 pmap_unwire_pte_hold(dst_pmap, dstmpte);
2444 }
2445 if (dstmpte->hold_count >= srcmpte->hold_count)
2446 break;
2447 }
2448 addr += PAGE_SIZE;
2449 ++src_pte;
2450 ++dst_pte;
2451 }
2452 }
2453 }
2454
2455 /*
2456 * Routine: pmap_kernel
2457 * Function:
2458 * Returns the physical map handle for the kernel.
2459 */
2460 pmap_t
2461 pmap_kernel()
2462 {
2463 return (kernel_pmap);
2464 }
2465
2466 /*
2467 * pmap_zero_page zeros the specified (machine independent)
2468 * page by mapping the page into virtual memory and using
2469 * bzero to clear its contents, one machine dependent page
2470 * at a time.
2471 */
2472 void
2473 pmap_zero_page(phys)
2474 vm_offset_t phys;
2475 {
2476 if (*(int *) CMAP2)
2477 panic("pmap_zero_page: CMAP busy");
2478
2479 *(int *) CMAP2 = PG_V | PG_RW | (phys & PG_FRAME);
2480 bzero(CADDR2, PAGE_SIZE);
2481 *(int *) CMAP2 = 0;
2482 invltlb_1pg((vm_offset_t) CADDR2);
2483 }
2484
2485 /*
2486 * pmap_copy_page copies the specified (machine independent)
2487 * page by mapping the page into virtual memory and using
2488 * bcopy to copy the page, one machine dependent page at a
2489 * time.
2490 */
2491 void
2492 pmap_copy_page(src, dst)
2493 vm_offset_t src;
2494 vm_offset_t dst;
2495 {
2496 if (*(int *) CMAP1 || *(int *) CMAP2)
2497 panic("pmap_copy_page: CMAP busy");
2498
2499 *(int *) CMAP1 = PG_V | PG_RW | (src & PG_FRAME);
2500 *(int *) CMAP2 = PG_V | PG_RW | (dst & PG_FRAME);
2501
2502 bcopy(CADDR1, CADDR2, PAGE_SIZE);
2503
2504 *(int *) CMAP1 = 0;
2505 *(int *) CMAP2 = 0;
2506 invltlb_2pg( (vm_offset_t) CADDR1, (vm_offset_t) CADDR2);
2507 }
2508
2509
2510 /*
2511 * Routine: pmap_pageable
2512 * Function:
2513 * Make the specified pages (by pmap, offset)
2514 * pageable (or not) as requested.
2515 *
2516 * A page which is not pageable may not take
2517 * a fault; therefore, its page table entry
2518 * must remain valid for the duration.
2519 *
2520 * This routine is merely advisory; pmap_enter
2521 * will specify that these pages are to be wired
2522 * down (or not) as appropriate.
2523 */
2524 void
2525 pmap_pageable(pmap, sva, eva, pageable)
2526 pmap_t pmap;
2527 vm_offset_t sva, eva;
2528 boolean_t pageable;
2529 {
2530 }
2531
2532 /*
2533 * this routine returns true if a physical page resides
2534 * in the given pmap.
2535 */
2536 boolean_t
2537 pmap_page_exists(pmap, pa)
2538 pmap_t pmap;
2539 vm_offset_t pa;
2540 {
2541 register pv_entry_t pv;
2542 pv_table_t *ppv;
2543 int s;
2544
2545 if (!pmap_is_managed(pa))
2546 return FALSE;
2547
2548 s = splvm();
2549
2550 ppv = pa_to_pvh(pa);
2551 /*
2552 * Not found, check current mappings returning immediately if found.
2553 */
2554 for (pv = TAILQ_FIRST(&ppv->pv_list);
2555 pv;
2556 pv = TAILQ_NEXT(pv, pv_list)) {
2557 if (pv->pv_pmap == pmap) {
2558 splx(s);
2559 return TRUE;
2560 }
2561 }
2562 splx(s);
2563 return (FALSE);
2564 }
2565
2566 #define PMAP_REMOVE_PAGES_CURPROC_ONLY
2567 /*
2568 * Remove all pages from specified address space
2569 * this aids process exit speeds. Also, this code
2570 * is special cased for current process only, but
2571 * can have the more generic (and slightly slower)
2572 * mode enabled. This is much faster than pmap_remove
2573 * in the case of running down an entire address space.
2574 */
2575 void
2576 pmap_remove_pages(pmap, sva, eva)
2577 pmap_t pmap;
2578 vm_offset_t sva, eva;
2579 {
2580 unsigned *pte, tpte;
2581 pv_table_t *ppv;
2582 pv_entry_t pv, npv;
2583 int s;
2584
2585 #if PMAP_PVLIST
2586
2587 #ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY
2588 if (!curproc || (pmap != &curproc->p_vmspace->vm_pmap)) {
2589 printf("warning: pmap_remove_pages called with non-current pmap\n");
2590 return;
2591 }
2592 #endif
2593
2594 s = splvm();
2595 for(pv = TAILQ_FIRST(&pmap->pm_pvlist);
2596 pv;
2597 pv = npv) {
2598
2599 if (pv->pv_va >= eva || pv->pv_va < sva) {
2600 npv = TAILQ_NEXT(pv, pv_plist);
2601 continue;
2602 }
2603
2604 #ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY
2605 pte = (unsigned *)vtopte(pv->pv_va);
2606 #else
2607 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
2608 #endif
2609 tpte = *pte;
2610
2611 /*
2612 * We cannot remove wired pages from a process' mapping at this time
2613 */
2614 if (tpte & PG_W) {
2615 npv = TAILQ_NEXT(pv, pv_plist);
2616 continue;
2617 }
2618 *pte = 0;
2619
2620 ppv = pa_to_pvh(tpte);
2621
2622 pv->pv_pmap->pm_stats.resident_count--;
2623
2624 /*
2625 * Update the vm_page_t clean and reference bits.
2626 */
2627 if (tpte & PG_M) {
2628 ppv->pv_vm_page->dirty = VM_PAGE_BITS_ALL;
2629 }
2630
2631
2632 npv = TAILQ_NEXT(pv, pv_plist);
2633 TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist);
2634
2635 --ppv->pv_list_count;
2636 TAILQ_REMOVE(&ppv->pv_list, pv, pv_list);
2637 if (TAILQ_FIRST(&ppv->pv_list) == NULL) {
2638 ppv->pv_vm_page->flags &= ~(PG_MAPPED|PG_WRITEABLE);
2639 }
2640
2641 pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem);
2642 free_pv_entry(pv);
2643 }
2644 splx(s);
2645 invltlb();
2646 #endif
2647 }
2648
2649 /*
2650 * pmap_testbit tests bits in pte's
2651 * note that the testbit/changebit routines are inline,
2652 * and a lot of things compile-time evaluate.
2653 */
2654 static boolean_t
2655 pmap_testbit(pa, bit)
2656 register vm_offset_t pa;
2657 int bit;
2658 {
2659 register pv_entry_t pv;
2660 pv_table_t *ppv;
2661 unsigned *pte;
2662 int s;
2663
2664 if (!pmap_is_managed(pa))
2665 return FALSE;
2666
2667 ppv = pa_to_pvh(pa);
2668 if (TAILQ_FIRST(&ppv->pv_list) == NULL)
2669 return FALSE;
2670
2671 s = splvm();
2672
2673 for (pv = TAILQ_FIRST(&ppv->pv_list);
2674 pv;
2675 pv = TAILQ_NEXT(pv, pv_list)) {
2676
2677 /*
2678 * if the bit being tested is the modified bit, then
2679 * mark clean_map and ptes as never
2680 * modified.
2681 */
2682 if (bit & (PG_A|PG_M)) {
2683 if (!pmap_track_modified(pv->pv_va))
2684 continue;
2685 }
2686
2687 #if defined(PMAP_DIAGNOSTIC)
2688 if (!pv->pv_pmap) {
2689 printf("Null pmap (tb) at va: 0x%lx\n", pv->pv_va);
2690 continue;
2691 }
2692 #endif
2693 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
2694 if (*pte & bit) {
2695 splx(s);
2696 return TRUE;
2697 }
2698 }
2699 splx(s);
2700 return (FALSE);
2701 }
2702
2703 /*
2704 * this routine is used to modify bits in ptes
2705 */
2706 static void
2707 pmap_changebit(pa, bit, setem)
2708 vm_offset_t pa;
2709 int bit;
2710 boolean_t setem;
2711 {
2712 register pv_entry_t pv;
2713 pv_table_t *ppv;
2714 register unsigned *pte;
2715 int changed;
2716 int s;
2717
2718 if (!pmap_is_managed(pa))
2719 return;
2720
2721 s = splvm();
2722 changed = 0;
2723 ppv = pa_to_pvh(pa);
2724
2725 /*
2726 * Loop over all current mappings setting/clearing as appropos If
2727 * setting RO do we need to clear the VAC?
2728 */
2729 for (pv = TAILQ_FIRST(&ppv->pv_list);
2730 pv;
2731 pv = TAILQ_NEXT(pv, pv_list)) {
2732
2733 /*
2734 * don't write protect pager mappings
2735 */
2736 if (!setem && (bit == PG_RW)) {
2737 if (!pmap_track_modified(pv->pv_va))
2738 continue;
2739 }
2740
2741 #if defined(PMAP_DIAGNOSTIC)
2742 if (!pv->pv_pmap) {
2743 printf("Null pmap (cb) at va: 0x%lx\n", pv->pv_va);
2744 continue;
2745 }
2746 #endif
2747
2748 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
2749
2750 if (setem) {
2751 *(int *)pte |= bit;
2752 changed = 1;
2753 } else {
2754 vm_offset_t pbits = *(vm_offset_t *)pte;
2755 if (pbits & bit) {
2756 changed = 1;
2757 if (bit == PG_RW) {
2758 if (pbits & PG_M) {
2759 ppv->pv_vm_page->dirty = VM_PAGE_BITS_ALL;
2760 }
2761 *(int *)pte = pbits & ~(PG_M|PG_RW);
2762 } else {
2763 *(int *)pte = pbits & ~bit;
2764 }
2765 }
2766 }
2767 }
2768 splx(s);
2769 if (changed)
2770 invltlb();
2771 }
2772
2773 /*
2774 * pmap_page_protect:
2775 *
2776 * Lower the permission for all mappings to a given page.
2777 */
2778 void
2779 pmap_page_protect(phys, prot)
2780 vm_offset_t phys;
2781 vm_prot_t prot;
2782 {
2783 if ((prot & VM_PROT_WRITE) == 0) {
2784 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) {
2785 pmap_changebit(phys, PG_RW, FALSE);
2786 } else {
2787 pmap_remove_all(phys);
2788 }
2789 }
2790 }
2791
2792 vm_offset_t
2793 pmap_phys_address(ppn)
2794 int ppn;
2795 {
2796 return (i386_ptob(ppn));
2797 }
2798
2799 /*
2800 * pmap_is_referenced:
2801 *
2802 * Return whether or not the specified physical page was referenced
2803 * by any physical maps.
2804 */
2805 boolean_t
2806 pmap_is_referenced(vm_offset_t pa)
2807 {
2808 register pv_entry_t pv;
2809 pv_table_t *ppv;
2810 unsigned *pte;
2811 int s;
2812
2813 if (!pmap_is_managed(pa))
2814 return FALSE;
2815
2816 ppv = pa_to_pvh(pa);
2817
2818 s = splvm();
2819 /*
2820 * Not found, check current mappings returning immediately if found.
2821 */
2822 for (pv = TAILQ_FIRST(&ppv->pv_list);
2823 pv;
2824 pv = TAILQ_NEXT(pv, pv_list)) {
2825
2826 /*
2827 * if the bit being tested is the modified bit, then
2828 * mark clean_map and ptes as never
2829 * modified.
2830 */
2831 if (!pmap_track_modified(pv->pv_va))
2832 continue;
2833
2834 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
2835 if ((int) *pte & PG_A) {
2836 splx(s);
2837 return TRUE;
2838 }
2839 }
2840 splx(s);
2841 return (FALSE);
2842 }
2843
2844 /*
2845 * pmap_ts_referenced:
2846 *
2847 * Return the count of reference bits for a page, clearing all of them.
2848 *
2849 */
2850 int
2851 pmap_ts_referenced(vm_offset_t pa)
2852 {
2853 register pv_entry_t pv;
2854 pv_table_t *ppv;
2855 unsigned *pte;
2856 int s;
2857 int rtval = 0;
2858
2859 if (!pmap_is_managed(pa))
2860 return FALSE;
2861
2862 s = splvm();
2863
2864 ppv = pa_to_pvh(pa);
2865
2866 if (TAILQ_FIRST(&ppv->pv_list) == NULL) {
2867 splx(s);
2868 return 0;
2869 }
2870
2871 /*
2872 * Not found, check current mappings returning immediately if found.
2873 */
2874 for (pv = TAILQ_FIRST(&ppv->pv_list);
2875 pv;
2876 pv = TAILQ_NEXT(pv, pv_list)) {
2877 /*
2878 * if the bit being tested is the modified bit, then
2879 * mark clean_map and ptes as never
2880 * modified.
2881 */
2882 if (!pmap_track_modified(pv->pv_va))
2883 continue;
2884
2885 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
2886 if (pte == NULL) {
2887 continue;
2888 }
2889 if (*pte & PG_A) {
2890 rtval++;
2891 *pte &= ~PG_A;
2892 }
2893 }
2894 splx(s);
2895 if (rtval) {
2896 invltlb();
2897 }
2898 return (rtval);
2899 }
2900
2901 /*
2902 * pmap_is_modified:
2903 *
2904 * Return whether or not the specified physical page was modified
2905 * in any physical maps.
2906 */
2907 boolean_t
2908 pmap_is_modified(vm_offset_t pa)
2909 {
2910 return pmap_testbit((pa), PG_M);
2911 }
2912
2913 /*
2914 * Clear the modify bits on the specified physical page.
2915 */
2916 void
2917 pmap_clear_modify(vm_offset_t pa)
2918 {
2919 pmap_changebit((pa), PG_M, FALSE);
2920 }
2921
2922 /*
2923 * pmap_clear_reference:
2924 *
2925 * Clear the reference bit on the specified physical page.
2926 */
2927 void
2928 pmap_clear_reference(vm_offset_t pa)
2929 {
2930 pmap_changebit((pa), PG_A, FALSE);
2931 }
2932
2933 /*
2934 * Miscellaneous support routines follow
2935 */
2936
2937 static void
2938 i386_protection_init()
2939 {
2940 register int *kp, prot;
2941
2942 kp = protection_codes;
2943 for (prot = 0; prot < 8; prot++) {
2944 switch (prot) {
2945 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE:
2946 /*
2947 * Read access is also 0. There isn't any execute bit,
2948 * so just make it readable.
2949 */
2950 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_NONE:
2951 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_EXECUTE:
2952 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_EXECUTE:
2953 *kp++ = 0;
2954 break;
2955 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE:
2956 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_EXECUTE:
2957 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_NONE:
2958 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE:
2959 *kp++ = PG_RW;
2960 break;
2961 }
2962 }
2963 }
2964
2965 /*
2966 * Map a set of physical memory pages into the kernel virtual
2967 * address space. Return a pointer to where it is mapped. This
2968 * routine is intended to be used for mapping device memory,
2969 * NOT real memory. The non-cacheable bits are set on each
2970 * mapped page.
2971 */
2972 void *
2973 pmap_mapdev(pa, size)
2974 vm_offset_t pa;
2975 vm_size_t size;
2976 {
2977 vm_offset_t va, tmpva;
2978 unsigned *pte;
2979
2980 size = roundup(size, PAGE_SIZE);
2981
2982 va = kmem_alloc_pageable(kernel_map, size);
2983 if (!va)
2984 panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
2985
2986 pa = pa & PG_FRAME;
2987 for (tmpva = va; size > 0;) {
2988 pte = (unsigned *)vtopte(tmpva);
2989 *pte = pa | PG_RW | PG_V;
2990 size -= PAGE_SIZE;
2991 tmpva += PAGE_SIZE;
2992 pa += PAGE_SIZE;
2993 }
2994 invltlb();
2995
2996 return ((void *) va);
2997 }
2998
2999 /*
3000 * perform the pmap work for mincore
3001 */
3002 int
3003 pmap_mincore(pmap, addr)
3004 pmap_t pmap;
3005 vm_offset_t addr;
3006 {
3007
3008 unsigned *ptep, pte;
3009 int val = 0;
3010
3011 ptep = pmap_pte(pmap, addr);
3012 if (ptep == 0) {
3013 return 0;
3014 }
3015
3016 if (pte = *ptep) {
3017 vm_offset_t pa;
3018 val = MINCORE_INCORE;
3019 pa = pte & PG_FRAME;
3020
3021 /*
3022 * Modified by us
3023 */
3024 if (pte & PG_M)
3025 val |= MINCORE_MODIFIED|MINCORE_MODIFIED_OTHER;
3026 /*
3027 * Modified by someone
3028 */
3029 else if (PHYS_TO_VM_PAGE(pa)->dirty ||
3030 pmap_is_modified(pa))
3031 val |= MINCORE_MODIFIED_OTHER;
3032 /*
3033 * Referenced by us
3034 */
3035 if (pte & PG_U)
3036 val |= MINCORE_REFERENCED|MINCORE_REFERENCED_OTHER;
3037
3038 /*
3039 * Referenced by someone
3040 */
3041 else if ((PHYS_TO_VM_PAGE(pa)->flags & PG_REFERENCED) ||
3042 pmap_is_referenced(pa))
3043 val |= MINCORE_REFERENCED_OTHER;
3044 }
3045 return val;
3046 }
3047
3048 #if defined(PMAP_DEBUG)
3049 pmap_pid_dump(int pid) {
3050 pmap_t pmap;
3051 struct proc *p;
3052 int npte = 0;
3053 int index;
3054 for (p = allproc.lh_first; p != NULL; p = p->p_list.le_next) {
3055 if (p->p_pid != pid)
3056 continue;
3057
3058 if (p->p_vmspace) {
3059 int i,j;
3060 index = 0;
3061 pmap = &p->p_vmspace->vm_pmap;
3062 for(i=0;i<1024;i++) {
3063 pd_entry_t *pde;
3064 unsigned *pte;
3065 unsigned base = i << PDRSHIFT;
3066
3067 pde = &pmap->pm_pdir[i];
3068 if (pde && pmap_pde_v(pde)) {
3069 for(j=0;j<1024;j++) {
3070 unsigned va = base + (j << PAGE_SHIFT);
3071 if (va >= (vm_offset_t) VM_MIN_KERNEL_ADDRESS) {
3072 if (index) {
3073 index = 0;
3074 printf("\n");
3075 }
3076 return npte;
3077 }
3078 pte = pmap_pte_quick( pmap, va);
3079 if (pte && pmap_pte_v(pte)) {
3080 vm_offset_t pa;
3081 vm_page_t m;
3082 pa = *(int *)pte;
3083 m = PHYS_TO_VM_PAGE((pa & PG_FRAME));
3084 printf("va: 0x%x, pt: 0x%x, h: %d, w: %d, f: 0x%x",
3085 va, pa, m->hold_count, m->wire_count, m->flags);
3086 npte++;
3087 index++;
3088 if (index >= 2) {
3089 index = 0;
3090 printf("\n");
3091 } else {
3092 printf(" ");
3093 }
3094 }
3095 }
3096 }
3097 }
3098 }
3099 }
3100 return npte;
3101 }
3102 #endif
3103
3104 #if defined(DEBUG)
3105
3106 static void pads __P((pmap_t pm));
3107 static void pmap_pvdump __P((vm_offset_t pa));
3108
3109 /* print address space of pmap*/
3110 static void
3111 pads(pm)
3112 pmap_t pm;
3113 {
3114 unsigned va, i, j;
3115 unsigned *ptep;
3116
3117 if (pm == kernel_pmap)
3118 return;
3119 for (i = 0; i < 1024; i++)
3120 if (pm->pm_pdir[i])
3121 for (j = 0; j < 1024; j++) {
3122 va = (i << PDRSHIFT) + (j << PAGE_SHIFT);
3123 if (pm == kernel_pmap && va < KERNBASE)
3124 continue;
3125 if (pm != kernel_pmap && va > UPT_MAX_ADDRESS)
3126 continue;
3127 ptep = pmap_pte_quick(pm, va);
3128 if (pmap_pte_v(ptep))
3129 printf("%x:%x ", va, *(int *) ptep);
3130 };
3131
3132 }
3133
3134 static void
3135 pmap_pvdump(pa)
3136 vm_offset_t pa;
3137 {
3138 pv_table_t *ppv;
3139 register pv_entry_t pv;
3140
3141 printf("pa %x", pa);
3142 ppv = pa_to_pvh(pa);
3143 for (pv = TAILQ_FIRST(&ppv->pv_list);
3144 pv;
3145 pv = TAILQ_NEXT(pv, pv_list)) {
3146 #ifdef used_to_be
3147 printf(" -> pmap %x, va %x, flags %x",
3148 pv->pv_pmap, pv->pv_va, pv->pv_flags);
3149 #endif
3150 printf(" -> pmap %x, va %x",
3151 pv->pv_pmap, pv->pv_va);
3152 pads(pv->pv_pmap);
3153 }
3154 printf(" ");
3155 }
3156 #endif
Cache object: 8d77ce20185432ff28c16a264dd85304
|