1 /*-
2 * Copyright (c) 1991 Regents of the University of California.
3 * All rights reserved.
4 * Copyright (c) 1994 John S. Dyson
5 * All rights reserved.
6 * Copyright (c) 1994 David Greenman
7 * All rights reserved.
8 *
9 * This code is derived from software contributed to Berkeley by
10 * the Systems Programming Group of the University of Utah Computer
11 * Science Department and William Jolitz of UUNET Technologies Inc.
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 * 3. All advertising materials mentioning features or use of this software
22 * must display the following acknowledgement:
23 * This product includes software developed by the University of
24 * California, Berkeley and its contributors.
25 * 4. Neither the name of the University nor the names of its contributors
26 * may be used to endorse or promote products derived from this software
27 * without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39 * SUCH DAMAGE.
40 *
41 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91
42 * $FreeBSD: releng/6.0/sys/sparc64/sparc64/pmap.c 147217 2005-06-10 03:33:36Z alc $
43 */
44
45 /*
46 * Manages physical address maps.
47 *
48 * In addition to hardware address maps, this module is called upon to
49 * provide software-use-only maps which may or may not be stored in the
50 * same form as hardware maps. These pseudo-maps are used to store
51 * intermediate results from copy operations to and from address spaces.
52 *
53 * Since the information managed by this module is also stored by the
54 * logical address mapping module, this module may throw away valid virtual
55 * to physical mappings at almost any time. However, invalidations of
56 * mappings must be done as requested.
57 *
58 * In order to cope with hardware architectures which make virtual to
59 * physical map invalidates expensive, this module may delay invalidate
60 * reduced protection operations until such time as they are actually
61 * necessary. This module is given full information as to which processors
62 * are currently using which maps, and to when physical maps must be made
63 * correct.
64 */
65
66 #include "opt_kstack_pages.h"
67 #include "opt_msgbuf.h"
68 #include "opt_pmap.h"
69
70 #include <sys/param.h>
71 #include <sys/kernel.h>
72 #include <sys/ktr.h>
73 #include <sys/lock.h>
74 #include <sys/msgbuf.h>
75 #include <sys/mutex.h>
76 #include <sys/proc.h>
77 #include <sys/smp.h>
78 #include <sys/sysctl.h>
79 #include <sys/systm.h>
80 #include <sys/vmmeter.h>
81
82 #include <dev/ofw/openfirm.h>
83
84 #include <vm/vm.h>
85 #include <vm/vm_param.h>
86 #include <vm/vm_kern.h>
87 #include <vm/vm_page.h>
88 #include <vm/vm_map.h>
89 #include <vm/vm_object.h>
90 #include <vm/vm_extern.h>
91 #include <vm/vm_pageout.h>
92 #include <vm/vm_pager.h>
93
94 #include <machine/cache.h>
95 #include <machine/frame.h>
96 #include <machine/instr.h>
97 #include <machine/md_var.h>
98 #include <machine/metadata.h>
99 #include <machine/ofw_mem.h>
100 #include <machine/smp.h>
101 #include <machine/tlb.h>
102 #include <machine/tte.h>
103 #include <machine/tsb.h>
104
105 #define PMAP_DEBUG
106
107 #ifndef PMAP_SHPGPERPROC
108 #define PMAP_SHPGPERPROC 200
109 #endif
110
111 /*
112 * Virtual and physical address of message buffer.
113 */
114 struct msgbuf *msgbufp;
115 vm_paddr_t msgbuf_phys;
116
117 int pmap_pagedaemon_waken;
118
119 /*
120 * Map of physical memory reagions.
121 */
122 vm_paddr_t phys_avail[128];
123 static struct ofw_mem_region mra[128];
124 struct ofw_mem_region sparc64_memreg[128];
125 int sparc64_nmemreg;
126 static struct ofw_map translations[128];
127 static int translations_size;
128
129 static vm_offset_t pmap_idle_map;
130 static vm_offset_t pmap_temp_map_1;
131 static vm_offset_t pmap_temp_map_2;
132
133 /*
134 * First and last available kernel virtual addresses.
135 */
136 vm_offset_t virtual_avail;
137 vm_offset_t virtual_end;
138 vm_offset_t kernel_vm_end;
139
140 vm_offset_t vm_max_kernel_address;
141
142 /*
143 * Kernel pmap.
144 */
145 struct pmap kernel_pmap_store;
146
147 /*
148 * Allocate physical memory for use in pmap_bootstrap.
149 */
150 static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size);
151
152 extern int tl1_immu_miss_patch_1[];
153 extern int tl1_immu_miss_patch_2[];
154 extern int tl1_dmmu_miss_patch_1[];
155 extern int tl1_dmmu_miss_patch_2[];
156 extern int tl1_dmmu_prot_patch_1[];
157 extern int tl1_dmmu_prot_patch_2[];
158
159 /*
160 * If user pmap is processed with pmap_remove and with pmap_remove and the
161 * resident count drops to 0, there are no more pages to remove, so we
162 * need not continue.
163 */
164 #define PMAP_REMOVE_DONE(pm) \
165 ((pm) != kernel_pmap && (pm)->pm_stats.resident_count == 0)
166
167 /*
168 * The threshold (in bytes) above which tsb_foreach() is used in pmap_remove()
169 * and pmap_protect() instead of trying each virtual address.
170 */
171 #define PMAP_TSB_THRESH ((TSB_SIZE / 2) * PAGE_SIZE)
172
173 SYSCTL_NODE(_debug, OID_AUTO, pmap_stats, CTLFLAG_RD, 0, "");
174
175 PMAP_STATS_VAR(pmap_nenter);
176 PMAP_STATS_VAR(pmap_nenter_update);
177 PMAP_STATS_VAR(pmap_nenter_replace);
178 PMAP_STATS_VAR(pmap_nenter_new);
179 PMAP_STATS_VAR(pmap_nkenter);
180 PMAP_STATS_VAR(pmap_nkenter_oc);
181 PMAP_STATS_VAR(pmap_nkenter_stupid);
182 PMAP_STATS_VAR(pmap_nkremove);
183 PMAP_STATS_VAR(pmap_nqenter);
184 PMAP_STATS_VAR(pmap_nqremove);
185 PMAP_STATS_VAR(pmap_ncache_enter);
186 PMAP_STATS_VAR(pmap_ncache_enter_c);
187 PMAP_STATS_VAR(pmap_ncache_enter_oc);
188 PMAP_STATS_VAR(pmap_ncache_enter_cc);
189 PMAP_STATS_VAR(pmap_ncache_enter_coc);
190 PMAP_STATS_VAR(pmap_ncache_enter_nc);
191 PMAP_STATS_VAR(pmap_ncache_enter_cnc);
192 PMAP_STATS_VAR(pmap_ncache_remove);
193 PMAP_STATS_VAR(pmap_ncache_remove_c);
194 PMAP_STATS_VAR(pmap_ncache_remove_oc);
195 PMAP_STATS_VAR(pmap_ncache_remove_cc);
196 PMAP_STATS_VAR(pmap_ncache_remove_coc);
197 PMAP_STATS_VAR(pmap_ncache_remove_nc);
198 PMAP_STATS_VAR(pmap_nzero_page);
199 PMAP_STATS_VAR(pmap_nzero_page_c);
200 PMAP_STATS_VAR(pmap_nzero_page_oc);
201 PMAP_STATS_VAR(pmap_nzero_page_nc);
202 PMAP_STATS_VAR(pmap_nzero_page_area);
203 PMAP_STATS_VAR(pmap_nzero_page_area_c);
204 PMAP_STATS_VAR(pmap_nzero_page_area_oc);
205 PMAP_STATS_VAR(pmap_nzero_page_area_nc);
206 PMAP_STATS_VAR(pmap_nzero_page_idle);
207 PMAP_STATS_VAR(pmap_nzero_page_idle_c);
208 PMAP_STATS_VAR(pmap_nzero_page_idle_oc);
209 PMAP_STATS_VAR(pmap_nzero_page_idle_nc);
210 PMAP_STATS_VAR(pmap_ncopy_page);
211 PMAP_STATS_VAR(pmap_ncopy_page_c);
212 PMAP_STATS_VAR(pmap_ncopy_page_oc);
213 PMAP_STATS_VAR(pmap_ncopy_page_nc);
214 PMAP_STATS_VAR(pmap_ncopy_page_dc);
215 PMAP_STATS_VAR(pmap_ncopy_page_doc);
216 PMAP_STATS_VAR(pmap_ncopy_page_sc);
217 PMAP_STATS_VAR(pmap_ncopy_page_soc);
218
219 PMAP_STATS_VAR(pmap_nnew_thread);
220 PMAP_STATS_VAR(pmap_nnew_thread_oc);
221
222 /*
223 * Quick sort callout for comparing memory regions.
224 */
225 static int mr_cmp(const void *a, const void *b);
226 static int om_cmp(const void *a, const void *b);
227 static int
228 mr_cmp(const void *a, const void *b)
229 {
230 const struct ofw_mem_region *mra;
231 const struct ofw_mem_region *mrb;
232
233 mra = a;
234 mrb = b;
235 if (mra->mr_start < mrb->mr_start)
236 return (-1);
237 else if (mra->mr_start > mrb->mr_start)
238 return (1);
239 else
240 return (0);
241 }
242 static int
243 om_cmp(const void *a, const void *b)
244 {
245 const struct ofw_map *oma;
246 const struct ofw_map *omb;
247
248 oma = a;
249 omb = b;
250 if (oma->om_start < omb->om_start)
251 return (-1);
252 else if (oma->om_start > omb->om_start)
253 return (1);
254 else
255 return (0);
256 }
257
258 /*
259 * Bootstrap the system enough to run with virtual memory.
260 */
261 void
262 pmap_bootstrap(vm_offset_t ekva)
263 {
264 struct pmap *pm;
265 struct tte *tp;
266 vm_offset_t off;
267 vm_offset_t va;
268 vm_paddr_t pa;
269 vm_size_t physsz;
270 vm_size_t virtsz;
271 ihandle_t pmem;
272 ihandle_t vmem;
273 int sz;
274 int i;
275 int j;
276
277 /*
278 * Find out what physical memory is available from the prom and
279 * initialize the phys_avail array. This must be done before
280 * pmap_bootstrap_alloc is called.
281 */
282 if ((pmem = OF_finddevice("/memory")) == -1)
283 panic("pmap_bootstrap: finddevice /memory");
284 if ((sz = OF_getproplen(pmem, "available")) == -1)
285 panic("pmap_bootstrap: getproplen /memory/available");
286 if (sizeof(phys_avail) < sz)
287 panic("pmap_bootstrap: phys_avail too small");
288 if (sizeof(mra) < sz)
289 panic("pmap_bootstrap: mra too small");
290 bzero(mra, sz);
291 if (OF_getprop(pmem, "available", mra, sz) == -1)
292 panic("pmap_bootstrap: getprop /memory/available");
293 sz /= sizeof(*mra);
294 CTR0(KTR_PMAP, "pmap_bootstrap: physical memory");
295 qsort(mra, sz, sizeof (*mra), mr_cmp);
296 physsz = 0;
297 getenv_quad("hw.physmem", &physmem);
298 for (i = 0, j = 0; i < sz; i++, j += 2) {
299 CTR2(KTR_PMAP, "start=%#lx size=%#lx", mra[i].mr_start,
300 mra[i].mr_size);
301 if (physmem != 0 && btoc(physsz + mra[i].mr_size) >= physmem) {
302 if (btoc(physsz) < physmem) {
303 phys_avail[j] = mra[i].mr_start;
304 phys_avail[j + 1] = mra[i].mr_start +
305 (ctob(physmem) - physsz);
306 physsz = ctob(physmem);
307 }
308 break;
309 }
310 phys_avail[j] = mra[i].mr_start;
311 phys_avail[j + 1] = mra[i].mr_start + mra[i].mr_size;
312 physsz += mra[i].mr_size;
313 }
314 physmem = btoc(physsz);
315
316 /*
317 * Calculate the size of kernel virtual memory, and the size and mask
318 * for the kernel tsb.
319 */
320 virtsz = roundup(physsz, PAGE_SIZE_4M << (PAGE_SHIFT - TTE_SHIFT));
321 vm_max_kernel_address = VM_MIN_KERNEL_ADDRESS + virtsz;
322 tsb_kernel_size = virtsz >> (PAGE_SHIFT - TTE_SHIFT);
323 tsb_kernel_mask = (tsb_kernel_size >> TTE_SHIFT) - 1;
324
325 /*
326 * Allocate the kernel tsb and lock it in the tlb.
327 */
328 pa = pmap_bootstrap_alloc(tsb_kernel_size);
329 if (pa & PAGE_MASK_4M)
330 panic("pmap_bootstrap: tsb unaligned\n");
331 tsb_kernel_phys = pa;
332 tsb_kernel = (struct tte *)(VM_MIN_KERNEL_ADDRESS - tsb_kernel_size);
333 pmap_map_tsb();
334 bzero(tsb_kernel, tsb_kernel_size);
335
336 /*
337 * Allocate and map the message buffer.
338 */
339 msgbuf_phys = pmap_bootstrap_alloc(MSGBUF_SIZE);
340 msgbufp = (struct msgbuf *)TLB_PHYS_TO_DIRECT(msgbuf_phys);
341
342 /*
343 * Patch the virtual address and the tsb mask into the trap table.
344 */
345
346 #define SETHI(rd, imm22) \
347 (EIF_OP(IOP_FORM2) | EIF_F2_RD(rd) | EIF_F2_OP2(INS0_SETHI) | \
348 EIF_IMM((imm22) >> 10, 22))
349 #define OR_R_I_R(rd, imm13, rs1) \
350 (EIF_OP(IOP_MISC) | EIF_F3_RD(rd) | EIF_F3_OP3(INS2_OR) | \
351 EIF_F3_RS1(rs1) | EIF_F3_I(1) | EIF_IMM(imm13, 13))
352
353 #define PATCH(addr) do { \
354 if (addr[0] != SETHI(IF_F2_RD(addr[0]), 0x0) || \
355 addr[1] != OR_R_I_R(IF_F3_RD(addr[1]), 0x0, IF_F3_RS1(addr[1])) || \
356 addr[2] != SETHI(IF_F2_RD(addr[2]), 0x0)) \
357 panic("pmap_boostrap: patched instructions have changed"); \
358 addr[0] |= EIF_IMM((tsb_kernel_mask) >> 10, 22); \
359 addr[1] |= EIF_IMM(tsb_kernel_mask, 10); \
360 addr[2] |= EIF_IMM(((vm_offset_t)tsb_kernel) >> 10, 22); \
361 flush(addr); \
362 flush(addr + 1); \
363 flush(addr + 2); \
364 } while (0)
365
366 PATCH(tl1_immu_miss_patch_1);
367 PATCH(tl1_immu_miss_patch_2);
368 PATCH(tl1_dmmu_miss_patch_1);
369 PATCH(tl1_dmmu_miss_patch_2);
370 PATCH(tl1_dmmu_prot_patch_1);
371 PATCH(tl1_dmmu_prot_patch_2);
372
373 /*
374 * Enter fake 8k pages for the 4MB kernel pages, so that
375 * pmap_kextract() will work for them.
376 */
377 for (i = 0; i < kernel_tlb_slots; i++) {
378 pa = kernel_tlbs[i].te_pa;
379 va = kernel_tlbs[i].te_va;
380 for (off = 0; off < PAGE_SIZE_4M; off += PAGE_SIZE) {
381 tp = tsb_kvtotte(va + off);
382 tp->tte_vpn = TV_VPN(va + off, TS_8K);
383 tp->tte_data = TD_V | TD_8K | TD_PA(pa + off) |
384 TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W;
385 }
386 }
387
388 /*
389 * Set the start and end of kva. The kernel is loaded at the first
390 * available 4 meg super page, so round up to the end of the page.
391 */
392 virtual_avail = roundup2(ekva, PAGE_SIZE_4M);
393 virtual_end = vm_max_kernel_address;
394 kernel_vm_end = vm_max_kernel_address;
395
396 /*
397 * Allocate kva space for temporary mappings.
398 */
399 pmap_idle_map = virtual_avail;
400 virtual_avail += PAGE_SIZE * DCACHE_COLORS;
401 pmap_temp_map_1 = virtual_avail;
402 virtual_avail += PAGE_SIZE * DCACHE_COLORS;
403 pmap_temp_map_2 = virtual_avail;
404 virtual_avail += PAGE_SIZE * DCACHE_COLORS;
405
406 /*
407 * Allocate a kernel stack with guard page for thread0 and map it into
408 * the kernel tsb. We must ensure that the virtual address is coloured
409 * properly, since we're allocating from phys_avail so the memory won't
410 * have an associated vm_page_t.
411 */
412 pa = pmap_bootstrap_alloc(roundup(KSTACK_PAGES, DCACHE_COLORS) *
413 PAGE_SIZE);
414 kstack0_phys = pa;
415 virtual_avail += roundup(KSTACK_GUARD_PAGES, DCACHE_COLORS) *
416 PAGE_SIZE;
417 kstack0 = virtual_avail;
418 virtual_avail += roundup(KSTACK_PAGES, DCACHE_COLORS) * PAGE_SIZE;
419 KASSERT(DCACHE_COLOR(kstack0) == DCACHE_COLOR(kstack0_phys),
420 ("pmap_bootstrap: kstack0 miscoloured"));
421 for (i = 0; i < KSTACK_PAGES; i++) {
422 pa = kstack0_phys + i * PAGE_SIZE;
423 va = kstack0 + i * PAGE_SIZE;
424 tp = tsb_kvtotte(va);
425 tp->tte_vpn = TV_VPN(va, TS_8K);
426 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW |
427 TD_CP | TD_CV | TD_P | TD_W;
428 }
429
430 /*
431 * Calculate the last available physical address.
432 */
433 for (i = 0; phys_avail[i + 2] != 0; i += 2)
434 ;
435 Maxmem = sparc64_btop(phys_avail[i + 1]);
436
437 /*
438 * Add the prom mappings to the kernel tsb.
439 */
440 if ((vmem = OF_finddevice("/virtual-memory")) == -1)
441 panic("pmap_bootstrap: finddevice /virtual-memory");
442 if ((sz = OF_getproplen(vmem, "translations")) == -1)
443 panic("pmap_bootstrap: getproplen translations");
444 if (sizeof(translations) < sz)
445 panic("pmap_bootstrap: translations too small");
446 bzero(translations, sz);
447 if (OF_getprop(vmem, "translations", translations, sz) == -1)
448 panic("pmap_bootstrap: getprop /virtual-memory/translations");
449 sz /= sizeof(*translations);
450 translations_size = sz;
451 CTR0(KTR_PMAP, "pmap_bootstrap: translations");
452 qsort(translations, sz, sizeof (*translations), om_cmp);
453 for (i = 0; i < sz; i++) {
454 CTR3(KTR_PMAP,
455 "translation: start=%#lx size=%#lx tte=%#lx",
456 translations[i].om_start, translations[i].om_size,
457 translations[i].om_tte);
458 if (translations[i].om_start < VM_MIN_PROM_ADDRESS ||
459 translations[i].om_start > VM_MAX_PROM_ADDRESS)
460 continue;
461 for (off = 0; off < translations[i].om_size;
462 off += PAGE_SIZE) {
463 va = translations[i].om_start + off;
464 tp = tsb_kvtotte(va);
465 tp->tte_vpn = TV_VPN(va, TS_8K);
466 tp->tte_data =
467 ((translations[i].om_tte &
468 ~(TD_SOFT_MASK << TD_SOFT_SHIFT)) | TD_EXEC) +
469 off;
470 }
471 }
472
473 /*
474 * Get the available physical memory ranges from /memory/reg. These
475 * are only used for kernel dumps, but it may not be wise to do prom
476 * calls in that situation.
477 */
478 if ((sz = OF_getproplen(pmem, "reg")) == -1)
479 panic("pmap_bootstrap: getproplen /memory/reg");
480 if (sizeof(sparc64_memreg) < sz)
481 panic("pmap_bootstrap: sparc64_memreg too small");
482 if (OF_getprop(pmem, "reg", sparc64_memreg, sz) == -1)
483 panic("pmap_bootstrap: getprop /memory/reg");
484 sparc64_nmemreg = sz / sizeof(*sparc64_memreg);
485
486 /*
487 * Initialize the kernel pmap (which is statically allocated).
488 * NOTE: PMAP_LOCK_INIT() is needed as part of the initialization
489 * but sparc64 start up is not ready to initialize mutexes yet.
490 * It is called in machdep.c.
491 */
492 pm = kernel_pmap;
493 for (i = 0; i < MAXCPU; i++)
494 pm->pm_context[i] = TLB_CTX_KERNEL;
495 pm->pm_active = ~0;
496
497 /* XXX flush all non-locked tlb entries */
498 }
499
500 void
501 pmap_map_tsb(void)
502 {
503 vm_offset_t va;
504 vm_paddr_t pa;
505 u_long data;
506 u_long s;
507 int i;
508
509 s = intr_disable();
510
511 /*
512 * Map the 4mb tsb pages.
513 */
514 for (i = 0; i < tsb_kernel_size; i += PAGE_SIZE_4M) {
515 va = (vm_offset_t)tsb_kernel + i;
516 pa = tsb_kernel_phys + i;
517 data = TD_V | TD_4M | TD_PA(pa) | TD_L | TD_CP | TD_CV |
518 TD_P | TD_W;
519 /* XXX - cheetah */
520 stxa(AA_DMMU_TAR, ASI_DMMU, TLB_TAR_VA(va) |
521 TLB_TAR_CTX(TLB_CTX_KERNEL));
522 stxa_sync(0, ASI_DTLB_DATA_IN_REG, data);
523 }
524
525 /*
526 * Set the secondary context to be the kernel context (needed for
527 * fp block operations in the kernel and the cache code).
528 */
529 stxa(AA_DMMU_SCXR, ASI_DMMU, TLB_CTX_KERNEL);
530 membar(Sync);
531
532 intr_restore(s);
533 }
534
535 /*
536 * Allocate a physical page of memory directly from the phys_avail map.
537 * Can only be called from pmap_bootstrap before avail start and end are
538 * calculated.
539 */
540 static vm_paddr_t
541 pmap_bootstrap_alloc(vm_size_t size)
542 {
543 vm_paddr_t pa;
544 int i;
545
546 size = round_page(size);
547 for (i = 0; phys_avail[i + 1] != 0; i += 2) {
548 if (phys_avail[i + 1] - phys_avail[i] < size)
549 continue;
550 pa = phys_avail[i];
551 phys_avail[i] += size;
552 return (pa);
553 }
554 panic("pmap_bootstrap_alloc");
555 }
556
557 /*
558 * Initialize a vm_page's machine-dependent fields.
559 */
560 void
561 pmap_page_init(vm_page_t m)
562 {
563
564 TAILQ_INIT(&m->md.tte_list);
565 m->md.color = DCACHE_COLOR(VM_PAGE_TO_PHYS(m));
566 m->md.flags = 0;
567 m->md.pmap = NULL;
568 }
569
570 /*
571 * Initialize the pmap module.
572 */
573 void
574 pmap_init(void)
575 {
576 vm_offset_t addr;
577 vm_size_t size;
578 int result;
579 int i;
580
581 for (i = 0; i < translations_size; i++) {
582 addr = translations[i].om_start;
583 size = translations[i].om_size;
584 if (addr < VM_MIN_PROM_ADDRESS || addr > VM_MAX_PROM_ADDRESS)
585 continue;
586 result = vm_map_find(kernel_map, NULL, 0, &addr, size, FALSE,
587 VM_PROT_ALL, VM_PROT_ALL, 0);
588 if (result != KERN_SUCCESS || addr != translations[i].om_start)
589 panic("pmap_init: vm_map_find");
590 }
591 }
592
593 /*
594 * Initialize the address space (zone) for the pv_entries. Set a
595 * high water mark so that the system can recover from excessive
596 * numbers of pv entries.
597 */
598 void
599 pmap_init2(void)
600 {
601 }
602
603 /*
604 * Extract the physical page address associated with the given
605 * map/virtual_address pair.
606 */
607 vm_paddr_t
608 pmap_extract(pmap_t pm, vm_offset_t va)
609 {
610 struct tte *tp;
611 vm_paddr_t pa;
612
613 if (pm == kernel_pmap)
614 return (pmap_kextract(va));
615 PMAP_LOCK(pm);
616 tp = tsb_tte_lookup(pm, va);
617 if (tp == NULL)
618 pa = 0;
619 else
620 pa = TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp));
621 PMAP_UNLOCK(pm);
622 return (pa);
623 }
624
625 /*
626 * Atomically extract and hold the physical page with the given
627 * pmap and virtual address pair if that mapping permits the given
628 * protection.
629 */
630 vm_page_t
631 pmap_extract_and_hold(pmap_t pm, vm_offset_t va, vm_prot_t prot)
632 {
633 struct tte *tp;
634 vm_page_t m;
635
636 m = NULL;
637 vm_page_lock_queues();
638 if (pm == kernel_pmap) {
639 if (va >= VM_MIN_DIRECT_ADDRESS) {
640 tp = NULL;
641 m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS(va));
642 vm_page_hold(m);
643 } else {
644 tp = tsb_kvtotte(va);
645 if ((tp->tte_data & TD_V) == 0)
646 tp = NULL;
647 }
648 } else {
649 PMAP_LOCK(pm);
650 tp = tsb_tte_lookup(pm, va);
651 }
652 if (tp != NULL && ((tp->tte_data & TD_SW) ||
653 (prot & VM_PROT_WRITE) == 0)) {
654 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp));
655 vm_page_hold(m);
656 }
657 vm_page_unlock_queues();
658 if (pm != kernel_pmap)
659 PMAP_UNLOCK(pm);
660 return (m);
661 }
662
663 /*
664 * Extract the physical page address associated with the given kernel virtual
665 * address.
666 */
667 vm_paddr_t
668 pmap_kextract(vm_offset_t va)
669 {
670 struct tte *tp;
671
672 if (va >= VM_MIN_DIRECT_ADDRESS)
673 return (TLB_DIRECT_TO_PHYS(va));
674 tp = tsb_kvtotte(va);
675 if ((tp->tte_data & TD_V) == 0)
676 return (0);
677 return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp)));
678 }
679
680 int
681 pmap_cache_enter(vm_page_t m, vm_offset_t va)
682 {
683 struct tte *tp;
684 int color;
685
686 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
687 KASSERT((m->flags & PG_FICTITIOUS) == 0,
688 ("pmap_cache_enter: fake page"));
689 PMAP_STATS_INC(pmap_ncache_enter);
690
691 /*
692 * Find the color for this virtual address and note the added mapping.
693 */
694 color = DCACHE_COLOR(va);
695 m->md.colors[color]++;
696
697 /*
698 * If all existing mappings have the same color, the mapping is
699 * cacheable.
700 */
701 if (m->md.color == color) {
702 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] == 0,
703 ("pmap_cache_enter: cacheable, mappings of other color"));
704 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m)))
705 PMAP_STATS_INC(pmap_ncache_enter_c);
706 else
707 PMAP_STATS_INC(pmap_ncache_enter_oc);
708 return (1);
709 }
710
711 /*
712 * If there are no mappings of the other color, and the page still has
713 * the wrong color, this must be a new mapping. Change the color to
714 * match the new mapping, which is cacheable. We must flush the page
715 * from the cache now.
716 */
717 if (m->md.colors[DCACHE_OTHER_COLOR(color)] == 0) {
718 KASSERT(m->md.colors[color] == 1,
719 ("pmap_cache_enter: changing color, not new mapping"));
720 dcache_page_inval(VM_PAGE_TO_PHYS(m));
721 m->md.color = color;
722 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m)))
723 PMAP_STATS_INC(pmap_ncache_enter_cc);
724 else
725 PMAP_STATS_INC(pmap_ncache_enter_coc);
726 return (1);
727 }
728
729 /*
730 * If the mapping is already non-cacheable, just return.
731 */
732 if (m->md.color == -1) {
733 PMAP_STATS_INC(pmap_ncache_enter_nc);
734 return (0);
735 }
736
737 PMAP_STATS_INC(pmap_ncache_enter_cnc);
738
739 /*
740 * Mark all mappings as uncacheable, flush any lines with the other
741 * color out of the dcache, and set the color to none (-1).
742 */
743 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) {
744 atomic_clear_long(&tp->tte_data, TD_CV);
745 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp));
746 }
747 dcache_page_inval(VM_PAGE_TO_PHYS(m));
748 m->md.color = -1;
749 return (0);
750 }
751
752 void
753 pmap_cache_remove(vm_page_t m, vm_offset_t va)
754 {
755 struct tte *tp;
756 int color;
757
758 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
759 CTR3(KTR_PMAP, "pmap_cache_remove: m=%p va=%#lx c=%d", m, va,
760 m->md.colors[DCACHE_COLOR(va)]);
761 KASSERT((m->flags & PG_FICTITIOUS) == 0,
762 ("pmap_cache_remove: fake page"));
763 KASSERT(m->md.colors[DCACHE_COLOR(va)] > 0,
764 ("pmap_cache_remove: no mappings %d <= 0",
765 m->md.colors[DCACHE_COLOR(va)]));
766 PMAP_STATS_INC(pmap_ncache_remove);
767
768 /*
769 * Find the color for this virtual address and note the removal of
770 * the mapping.
771 */
772 color = DCACHE_COLOR(va);
773 m->md.colors[color]--;
774
775 /*
776 * If the page is cacheable, just return and keep the same color, even
777 * if there are no longer any mappings.
778 */
779 if (m->md.color != -1) {
780 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m)))
781 PMAP_STATS_INC(pmap_ncache_remove_c);
782 else
783 PMAP_STATS_INC(pmap_ncache_remove_oc);
784 return;
785 }
786
787 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] != 0,
788 ("pmap_cache_remove: uncacheable, no mappings of other color"));
789
790 /*
791 * If the page is not cacheable (color is -1), and the number of
792 * mappings for this color is not zero, just return. There are
793 * mappings of the other color still, so remain non-cacheable.
794 */
795 if (m->md.colors[color] != 0) {
796 PMAP_STATS_INC(pmap_ncache_remove_nc);
797 return;
798 }
799
800 /*
801 * The number of mappings for this color is now zero. Recache the
802 * other colored mappings, and change the page color to the other
803 * color. There should be no lines in the data cache for this page,
804 * so flushing should not be needed.
805 */
806 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) {
807 atomic_set_long(&tp->tte_data, TD_CV);
808 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp));
809 }
810 m->md.color = DCACHE_OTHER_COLOR(color);
811
812 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m)))
813 PMAP_STATS_INC(pmap_ncache_remove_cc);
814 else
815 PMAP_STATS_INC(pmap_ncache_remove_coc);
816 }
817
818 /*
819 * Map a wired page into kernel virtual address space.
820 */
821 void
822 pmap_kenter(vm_offset_t va, vm_page_t m)
823 {
824 vm_offset_t ova;
825 struct tte *tp;
826 vm_page_t om;
827 u_long data;
828
829 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
830 PMAP_STATS_INC(pmap_nkenter);
831 tp = tsb_kvtotte(va);
832 CTR4(KTR_PMAP, "pmap_kenter: va=%#lx pa=%#lx tp=%p data=%#lx",
833 va, VM_PAGE_TO_PHYS(m), tp, tp->tte_data);
834 if (m->pc != DCACHE_COLOR(va)) {
835 CTR6(KTR_CT2,
836 "pmap_kenter: off colour va=%#lx pa=%#lx o=%p oc=%#lx ot=%d pi=%#lx",
837 va, VM_PAGE_TO_PHYS(m), m->object,
838 m->object ? m->object->pg_color : -1,
839 m->object ? m->object->type : -1,
840 m->pindex);
841 PMAP_STATS_INC(pmap_nkenter_oc);
842 }
843 if ((tp->tte_data & TD_V) != 0) {
844 om = PHYS_TO_VM_PAGE(TTE_GET_PA(tp));
845 ova = TTE_GET_VA(tp);
846 if (m == om && va == ova) {
847 PMAP_STATS_INC(pmap_nkenter_stupid);
848 return;
849 }
850 TAILQ_REMOVE(&om->md.tte_list, tp, tte_link);
851 pmap_cache_remove(om, ova);
852 if (va != ova)
853 tlb_page_demap(kernel_pmap, ova);
854 }
855 data = TD_V | TD_8K | VM_PAGE_TO_PHYS(m) | TD_REF | TD_SW | TD_CP |
856 TD_P | TD_W;
857 if (pmap_cache_enter(m, va) != 0)
858 data |= TD_CV;
859 tp->tte_vpn = TV_VPN(va, TS_8K);
860 tp->tte_data = data;
861 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link);
862 }
863
864 /*
865 * Map a wired page into kernel virtual address space. This additionally
866 * takes a flag argument wich is or'ed to the TTE data. This is used by
867 * bus_space_map().
868 * NOTE: if the mapping is non-cacheable, it's the caller's responsibility
869 * to flush entries that might still be in the cache, if applicable.
870 */
871 void
872 pmap_kenter_flags(vm_offset_t va, vm_paddr_t pa, u_long flags)
873 {
874 struct tte *tp;
875
876 tp = tsb_kvtotte(va);
877 CTR4(KTR_PMAP, "pmap_kenter_flags: va=%#lx pa=%#lx tp=%p data=%#lx",
878 va, pa, tp, tp->tte_data);
879 tp->tte_vpn = TV_VPN(va, TS_8K);
880 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_P | flags;
881 }
882
883 /*
884 * Remove a wired page from kernel virtual address space.
885 */
886 void
887 pmap_kremove(vm_offset_t va)
888 {
889 struct tte *tp;
890 vm_page_t m;
891
892 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
893 PMAP_STATS_INC(pmap_nkremove);
894 tp = tsb_kvtotte(va);
895 CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp,
896 tp->tte_data);
897 if ((tp->tte_data & TD_V) == 0)
898 return;
899 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp));
900 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link);
901 pmap_cache_remove(m, va);
902 TTE_ZERO(tp);
903 }
904
905 /*
906 * Inverse of pmap_kenter_flags, used by bus_space_unmap().
907 */
908 void
909 pmap_kremove_flags(vm_offset_t va)
910 {
911 struct tte *tp;
912
913 tp = tsb_kvtotte(va);
914 CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp,
915 tp->tte_data);
916 TTE_ZERO(tp);
917 }
918
919 /*
920 * Map a range of physical addresses into kernel virtual address space.
921 *
922 * The value passed in *virt is a suggested virtual address for the mapping.
923 * Architectures which can support a direct-mapped physical to virtual region
924 * can return the appropriate address within that region, leaving '*virt'
925 * unchanged.
926 */
927 vm_offset_t
928 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot)
929 {
930
931 return (TLB_PHYS_TO_DIRECT(start));
932 }
933
934 /*
935 * Map a list of wired pages into kernel virtual address space. This is
936 * intended for temporary mappings which do not need page modification or
937 * references recorded. Existing mappings in the region are overwritten.
938 */
939 void
940 pmap_qenter(vm_offset_t sva, vm_page_t *m, int count)
941 {
942 vm_offset_t va;
943 int locked;
944
945 PMAP_STATS_INC(pmap_nqenter);
946 va = sva;
947 if (!(locked = mtx_owned(&vm_page_queue_mtx)))
948 vm_page_lock_queues();
949 while (count-- > 0) {
950 pmap_kenter(va, *m);
951 va += PAGE_SIZE;
952 m++;
953 }
954 if (!locked)
955 vm_page_unlock_queues();
956 tlb_range_demap(kernel_pmap, sva, va);
957 }
958
959 /*
960 * Remove page mappings from kernel virtual address space. Intended for
961 * temporary mappings entered by pmap_qenter.
962 */
963 void
964 pmap_qremove(vm_offset_t sva, int count)
965 {
966 vm_offset_t va;
967 int locked;
968
969 PMAP_STATS_INC(pmap_nqremove);
970 va = sva;
971 if (!(locked = mtx_owned(&vm_page_queue_mtx)))
972 vm_page_lock_queues();
973 while (count-- > 0) {
974 pmap_kremove(va);
975 va += PAGE_SIZE;
976 }
977 if (!locked)
978 vm_page_unlock_queues();
979 tlb_range_demap(kernel_pmap, sva, va);
980 }
981
982 /*
983 * Initialize the pmap associated with process 0.
984 */
985 void
986 pmap_pinit0(pmap_t pm)
987 {
988 int i;
989
990 PMAP_LOCK_INIT(pm);
991 for (i = 0; i < MAXCPU; i++)
992 pm->pm_context[i] = 0;
993 pm->pm_active = 0;
994 pm->pm_tsb = NULL;
995 pm->pm_tsb_obj = NULL;
996 bzero(&pm->pm_stats, sizeof(pm->pm_stats));
997 }
998
999 /*
1000 * Initialize a preallocated and zeroed pmap structure, uch as one in a
1001 * vmspace structure.
1002 */
1003 void
1004 pmap_pinit(pmap_t pm)
1005 {
1006 vm_page_t ma[TSB_PAGES];
1007 vm_page_t m;
1008 int i;
1009
1010 PMAP_LOCK_INIT(pm);
1011
1012 /*
1013 * Allocate kva space for the tsb.
1014 */
1015 if (pm->pm_tsb == NULL) {
1016 pm->pm_tsb = (struct tte *)kmem_alloc_nofault(kernel_map,
1017 TSB_BSIZE);
1018 }
1019
1020 /*
1021 * Allocate an object for it.
1022 */
1023 if (pm->pm_tsb_obj == NULL)
1024 pm->pm_tsb_obj = vm_object_allocate(OBJT_DEFAULT, TSB_PAGES);
1025
1026 VM_OBJECT_LOCK(pm->pm_tsb_obj);
1027 for (i = 0; i < TSB_PAGES; i++) {
1028 m = vm_page_grab(pm->pm_tsb_obj, i, VM_ALLOC_NOBUSY |
1029 VM_ALLOC_RETRY | VM_ALLOC_WIRED | VM_ALLOC_ZERO);
1030
1031 vm_page_lock_queues();
1032 m->valid = VM_PAGE_BITS_ALL;
1033 m->md.pmap = pm;
1034 vm_page_unlock_queues();
1035
1036 ma[i] = m;
1037 }
1038 VM_OBJECT_UNLOCK(pm->pm_tsb_obj);
1039 pmap_qenter((vm_offset_t)pm->pm_tsb, ma, TSB_PAGES);
1040
1041 for (i = 0; i < MAXCPU; i++)
1042 pm->pm_context[i] = -1;
1043 pm->pm_active = 0;
1044 bzero(&pm->pm_stats, sizeof(pm->pm_stats));
1045 }
1046
1047 /*
1048 * Release any resources held by the given physical map.
1049 * Called when a pmap initialized by pmap_pinit is being released.
1050 * Should only be called if the map contains no valid mappings.
1051 */
1052 void
1053 pmap_release(pmap_t pm)
1054 {
1055 vm_object_t obj;
1056 vm_page_t m;
1057 struct pcpu *pc;
1058
1059 CTR2(KTR_PMAP, "pmap_release: ctx=%#x tsb=%p",
1060 pm->pm_context[PCPU_GET(cpuid)], pm->pm_tsb);
1061 KASSERT(pmap_resident_count(pm) == 0,
1062 ("pmap_release: resident pages %ld != 0",
1063 pmap_resident_count(pm)));
1064
1065 /*
1066 * After the pmap was freed, it might be reallocated to a new process.
1067 * When switching, this might lead us to wrongly assume that we need
1068 * not switch contexts because old and new pmap pointer are equal.
1069 * Therefore, make sure that this pmap is not referenced by any PCPU
1070 * pointer any more. This could happen in two cases:
1071 * - A process that referenced the pmap is currently exiting on a CPU.
1072 * However, it is guaranteed to not switch in any more after setting
1073 * its state to PRS_ZOMBIE.
1074 * - A process that referenced this pmap ran on a CPU, but we switched
1075 * to a kernel thread, leaving the pmap pointer unchanged.
1076 */
1077 mtx_lock_spin(&sched_lock);
1078 SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
1079 if (pc->pc_pmap == pm)
1080 pc->pc_pmap = NULL;
1081 }
1082 mtx_unlock_spin(&sched_lock);
1083
1084 obj = pm->pm_tsb_obj;
1085 VM_OBJECT_LOCK(obj);
1086 KASSERT(obj->ref_count == 1, ("pmap_release: tsbobj ref count != 1"));
1087 while (!TAILQ_EMPTY(&obj->memq)) {
1088 m = TAILQ_FIRST(&obj->memq);
1089 vm_page_lock_queues();
1090 if (vm_page_sleep_if_busy(m, FALSE, "pmaprl"))
1091 continue;
1092 KASSERT(m->hold_count == 0,
1093 ("pmap_release: freeing held tsb page"));
1094 m->md.pmap = NULL;
1095 m->wire_count--;
1096 atomic_subtract_int(&cnt.v_wire_count, 1);
1097 vm_page_free_zero(m);
1098 vm_page_unlock_queues();
1099 }
1100 VM_OBJECT_UNLOCK(obj);
1101 pmap_qremove((vm_offset_t)pm->pm_tsb, TSB_PAGES);
1102 PMAP_LOCK_DESTROY(pm);
1103 }
1104
1105 /*
1106 * Grow the number of kernel page table entries. Unneeded.
1107 */
1108 void
1109 pmap_growkernel(vm_offset_t addr)
1110 {
1111
1112 panic("pmap_growkernel: can't grow kernel");
1113 }
1114
1115 int
1116 pmap_remove_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp,
1117 vm_offset_t va)
1118 {
1119 vm_page_t m;
1120 u_long data;
1121
1122 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1123 data = atomic_readandclear_long(&tp->tte_data);
1124 if ((data & TD_FAKE) == 0) {
1125 m = PHYS_TO_VM_PAGE(TD_PA(data));
1126 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link);
1127 if ((data & TD_WIRED) != 0)
1128 pm->pm_stats.wired_count--;
1129 if ((data & TD_PV) != 0) {
1130 if ((data & TD_W) != 0 && pmap_track_modified(pm, va))
1131 vm_page_dirty(m);
1132 if ((data & TD_REF) != 0)
1133 vm_page_flag_set(m, PG_REFERENCED);
1134 if (TAILQ_EMPTY(&m->md.tte_list))
1135 vm_page_flag_clear(m, PG_WRITEABLE);
1136 pm->pm_stats.resident_count--;
1137 }
1138 pmap_cache_remove(m, va);
1139 }
1140 TTE_ZERO(tp);
1141 if (PMAP_REMOVE_DONE(pm))
1142 return (0);
1143 return (1);
1144 }
1145
1146 /*
1147 * Remove the given range of addresses from the specified map.
1148 */
1149 void
1150 pmap_remove(pmap_t pm, vm_offset_t start, vm_offset_t end)
1151 {
1152 struct tte *tp;
1153 vm_offset_t va;
1154
1155 CTR3(KTR_PMAP, "pmap_remove: ctx=%#lx start=%#lx end=%#lx",
1156 pm->pm_context[PCPU_GET(cpuid)], start, end);
1157 if (PMAP_REMOVE_DONE(pm))
1158 return;
1159 vm_page_lock_queues();
1160 PMAP_LOCK(pm);
1161 if (end - start > PMAP_TSB_THRESH) {
1162 tsb_foreach(pm, NULL, start, end, pmap_remove_tte);
1163 tlb_context_demap(pm);
1164 } else {
1165 for (va = start; va < end; va += PAGE_SIZE) {
1166 if ((tp = tsb_tte_lookup(pm, va)) != NULL) {
1167 if (!pmap_remove_tte(pm, NULL, tp, va))
1168 break;
1169 }
1170 }
1171 tlb_range_demap(pm, start, end - 1);
1172 }
1173 PMAP_UNLOCK(pm);
1174 vm_page_unlock_queues();
1175 }
1176
1177 void
1178 pmap_remove_all(vm_page_t m)
1179 {
1180 struct pmap *pm;
1181 struct tte *tpn;
1182 struct tte *tp;
1183 vm_offset_t va;
1184
1185 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1186 for (tp = TAILQ_FIRST(&m->md.tte_list); tp != NULL; tp = tpn) {
1187 tpn = TAILQ_NEXT(tp, tte_link);
1188 if ((tp->tte_data & TD_PV) == 0)
1189 continue;
1190 pm = TTE_GET_PMAP(tp);
1191 va = TTE_GET_VA(tp);
1192 PMAP_LOCK(pm);
1193 if ((tp->tte_data & TD_WIRED) != 0)
1194 pm->pm_stats.wired_count--;
1195 if ((tp->tte_data & TD_REF) != 0)
1196 vm_page_flag_set(m, PG_REFERENCED);
1197 if ((tp->tte_data & TD_W) != 0 &&
1198 pmap_track_modified(pm, va))
1199 vm_page_dirty(m);
1200 tp->tte_data &= ~TD_V;
1201 tlb_page_demap(pm, va);
1202 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link);
1203 pm->pm_stats.resident_count--;
1204 pmap_cache_remove(m, va);
1205 TTE_ZERO(tp);
1206 PMAP_UNLOCK(pm);
1207 }
1208 vm_page_flag_clear(m, PG_WRITEABLE);
1209 }
1210
1211 int
1212 pmap_protect_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp,
1213 vm_offset_t va)
1214 {
1215 u_long data;
1216 vm_page_t m;
1217
1218 data = atomic_clear_long(&tp->tte_data, TD_REF | TD_SW | TD_W);
1219 if ((data & TD_PV) != 0) {
1220 m = PHYS_TO_VM_PAGE(TD_PA(data));
1221 if ((data & TD_REF) != 0)
1222 vm_page_flag_set(m, PG_REFERENCED);
1223 if ((data & TD_W) != 0 && pmap_track_modified(pm, va))
1224 vm_page_dirty(m);
1225 }
1226 return (1);
1227 }
1228
1229 /*
1230 * Set the physical protection on the specified range of this map as requested.
1231 */
1232 void
1233 pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
1234 {
1235 vm_offset_t va;
1236 struct tte *tp;
1237
1238 CTR4(KTR_PMAP, "pmap_protect: ctx=%#lx sva=%#lx eva=%#lx prot=%#lx",
1239 pm->pm_context[PCPU_GET(cpuid)], sva, eva, prot);
1240
1241 if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
1242 pmap_remove(pm, sva, eva);
1243 return;
1244 }
1245
1246 if (prot & VM_PROT_WRITE)
1247 return;
1248
1249 vm_page_lock_queues();
1250 PMAP_LOCK(pm);
1251 if (eva - sva > PMAP_TSB_THRESH) {
1252 tsb_foreach(pm, NULL, sva, eva, pmap_protect_tte);
1253 tlb_context_demap(pm);
1254 } else {
1255 for (va = sva; va < eva; va += PAGE_SIZE) {
1256 if ((tp = tsb_tte_lookup(pm, va)) != NULL)
1257 pmap_protect_tte(pm, NULL, tp, va);
1258 }
1259 tlb_range_demap(pm, sva, eva - 1);
1260 }
1261 PMAP_UNLOCK(pm);
1262 vm_page_unlock_queues();
1263 }
1264
1265 /*
1266 * Map the given physical page at the specified virtual address in the
1267 * target pmap with the protection requested. If specified the page
1268 * will be wired down.
1269 */
1270 void
1271 pmap_enter(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot,
1272 boolean_t wired)
1273 {
1274 struct tte *tp;
1275 vm_paddr_t pa;
1276 u_long data;
1277 int i;
1278
1279 PMAP_STATS_INC(pmap_nenter);
1280 pa = VM_PAGE_TO_PHYS(m);
1281
1282 /*
1283 * If this is a fake page from the device_pager, but it covers actual
1284 * physical memory, convert to the real backing page.
1285 */
1286 if ((m->flags & PG_FICTITIOUS) != 0) {
1287 for (i = 0; phys_avail[i + 1] != 0; i += 2) {
1288 if (pa >= phys_avail[i] && pa <= phys_avail[i + 1]) {
1289 m = PHYS_TO_VM_PAGE(pa);
1290 break;
1291 }
1292 }
1293 }
1294
1295 CTR6(KTR_PMAP,
1296 "pmap_enter: ctx=%p m=%p va=%#lx pa=%#lx prot=%#x wired=%d",
1297 pm->pm_context[PCPU_GET(cpuid)], m, va, pa, prot, wired);
1298
1299 vm_page_lock_queues();
1300 PMAP_LOCK(pm);
1301
1302 /*
1303 * If there is an existing mapping, and the physical address has not
1304 * changed, must be protection or wiring change.
1305 */
1306 if ((tp = tsb_tte_lookup(pm, va)) != NULL && TTE_GET_PA(tp) == pa) {
1307 CTR0(KTR_PMAP, "pmap_enter: update");
1308 PMAP_STATS_INC(pmap_nenter_update);
1309
1310 /*
1311 * Wiring change, just update stats.
1312 */
1313 if (wired) {
1314 if ((tp->tte_data & TD_WIRED) == 0) {
1315 tp->tte_data |= TD_WIRED;
1316 pm->pm_stats.wired_count++;
1317 }
1318 } else {
1319 if ((tp->tte_data & TD_WIRED) != 0) {
1320 tp->tte_data &= ~TD_WIRED;
1321 pm->pm_stats.wired_count--;
1322 }
1323 }
1324
1325 /*
1326 * Save the old bits and clear the ones we're interested in.
1327 */
1328 data = tp->tte_data;
1329 tp->tte_data &= ~(TD_EXEC | TD_SW | TD_W);
1330
1331 /*
1332 * If we're turning off write permissions, sense modify status.
1333 */
1334 if ((prot & VM_PROT_WRITE) != 0) {
1335 tp->tte_data |= TD_SW;
1336 if (wired) {
1337 tp->tte_data |= TD_W;
1338 }
1339 } else if ((data & TD_W) != 0 &&
1340 pmap_track_modified(pm, va)) {
1341 vm_page_dirty(m);
1342 }
1343
1344 /*
1345 * If we're turning on execute permissions, flush the icache.
1346 */
1347 if ((prot & VM_PROT_EXECUTE) != 0) {
1348 if ((data & TD_EXEC) == 0) {
1349 icache_page_inval(pa);
1350 }
1351 tp->tte_data |= TD_EXEC;
1352 }
1353
1354 /*
1355 * Delete the old mapping.
1356 */
1357 tlb_page_demap(pm, TTE_GET_VA(tp));
1358 } else {
1359 /*
1360 * If there is an existing mapping, but its for a different
1361 * phsyical address, delete the old mapping.
1362 */
1363 if (tp != NULL) {
1364 CTR0(KTR_PMAP, "pmap_enter: replace");
1365 PMAP_STATS_INC(pmap_nenter_replace);
1366 pmap_remove_tte(pm, NULL, tp, va);
1367 tlb_page_demap(pm, va);
1368 } else {
1369 CTR0(KTR_PMAP, "pmap_enter: new");
1370 PMAP_STATS_INC(pmap_nenter_new);
1371 }
1372
1373 /*
1374 * Now set up the data and install the new mapping.
1375 */
1376 data = TD_V | TD_8K | TD_PA(pa);
1377 if (pm == kernel_pmap)
1378 data |= TD_P;
1379 if (prot & VM_PROT_WRITE)
1380 data |= TD_SW;
1381 if (prot & VM_PROT_EXECUTE) {
1382 data |= TD_EXEC;
1383 icache_page_inval(pa);
1384 }
1385
1386 /*
1387 * If its wired update stats. We also don't need reference or
1388 * modify tracking for wired mappings, so set the bits now.
1389 */
1390 if (wired) {
1391 pm->pm_stats.wired_count++;
1392 data |= TD_REF | TD_WIRED;
1393 if ((prot & VM_PROT_WRITE) != 0)
1394 data |= TD_W;
1395 }
1396
1397 tsb_tte_enter(pm, m, va, TS_8K, data);
1398 }
1399 vm_page_unlock_queues();
1400 PMAP_UNLOCK(pm);
1401 }
1402
1403 vm_page_t
1404 pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_page_t mpte)
1405 {
1406
1407 vm_page_busy(m);
1408 vm_page_unlock_queues();
1409 VM_OBJECT_UNLOCK(m->object);
1410 pmap_enter(pm, va, m, VM_PROT_READ | VM_PROT_EXECUTE, FALSE);
1411 VM_OBJECT_LOCK(m->object);
1412 vm_page_lock_queues();
1413 vm_page_wakeup(m);
1414 return (NULL);
1415 }
1416
1417 void
1418 pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object,
1419 vm_pindex_t pindex, vm_size_t size)
1420 {
1421
1422 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1423 KASSERT(object->type == OBJT_DEVICE,
1424 ("pmap_object_init_pt: non-device object"));
1425 }
1426
1427 /*
1428 * Change the wiring attribute for a map/virtual-address pair.
1429 * The mapping must already exist in the pmap.
1430 */
1431 void
1432 pmap_change_wiring(pmap_t pm, vm_offset_t va, boolean_t wired)
1433 {
1434 struct tte *tp;
1435 u_long data;
1436
1437 PMAP_LOCK(pm);
1438 if ((tp = tsb_tte_lookup(pm, va)) != NULL) {
1439 if (wired) {
1440 data = atomic_set_long(&tp->tte_data, TD_WIRED);
1441 if ((data & TD_WIRED) == 0)
1442 pm->pm_stats.wired_count++;
1443 } else {
1444 data = atomic_clear_long(&tp->tte_data, TD_WIRED);
1445 if ((data & TD_WIRED) != 0)
1446 pm->pm_stats.wired_count--;
1447 }
1448 }
1449 PMAP_UNLOCK(pm);
1450 }
1451
1452 static int
1453 pmap_copy_tte(pmap_t src_pmap, pmap_t dst_pmap, struct tte *tp, vm_offset_t va)
1454 {
1455 vm_page_t m;
1456 u_long data;
1457
1458 if ((tp->tte_data & TD_FAKE) != 0)
1459 return (1);
1460 if (tsb_tte_lookup(dst_pmap, va) == NULL) {
1461 data = tp->tte_data &
1462 ~(TD_PV | TD_REF | TD_SW | TD_CV | TD_W);
1463 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp));
1464 tsb_tte_enter(dst_pmap, m, va, TS_8K, data);
1465 }
1466 return (1);
1467 }
1468
1469 void
1470 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
1471 vm_size_t len, vm_offset_t src_addr)
1472 {
1473 struct tte *tp;
1474 vm_offset_t va;
1475
1476 if (dst_addr != src_addr)
1477 return;
1478 vm_page_lock_queues();
1479 if (dst_pmap < src_pmap) {
1480 PMAP_LOCK(dst_pmap);
1481 PMAP_LOCK(src_pmap);
1482 } else {
1483 PMAP_LOCK(src_pmap);
1484 PMAP_LOCK(dst_pmap);
1485 }
1486 if (len > PMAP_TSB_THRESH) {
1487 tsb_foreach(src_pmap, dst_pmap, src_addr, src_addr + len,
1488 pmap_copy_tte);
1489 tlb_context_demap(dst_pmap);
1490 } else {
1491 for (va = src_addr; va < src_addr + len; va += PAGE_SIZE) {
1492 if ((tp = tsb_tte_lookup(src_pmap, va)) != NULL)
1493 pmap_copy_tte(src_pmap, dst_pmap, tp, va);
1494 }
1495 tlb_range_demap(dst_pmap, src_addr, src_addr + len - 1);
1496 }
1497 vm_page_unlock_queues();
1498 PMAP_UNLOCK(src_pmap);
1499 PMAP_UNLOCK(dst_pmap);
1500 }
1501
1502 void
1503 pmap_zero_page(vm_page_t m)
1504 {
1505 struct tte *tp;
1506 vm_offset_t va;
1507 vm_paddr_t pa;
1508
1509 KASSERT((m->flags & PG_FICTITIOUS) == 0,
1510 ("pmap_zero_page: fake page"));
1511 PMAP_STATS_INC(pmap_nzero_page);
1512 pa = VM_PAGE_TO_PHYS(m);
1513 if (m->md.color == -1) {
1514 PMAP_STATS_INC(pmap_nzero_page_nc);
1515 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE);
1516 } else if (m->md.color == DCACHE_COLOR(pa)) {
1517 PMAP_STATS_INC(pmap_nzero_page_c);
1518 va = TLB_PHYS_TO_DIRECT(pa);
1519 cpu_block_zero((void *)va, PAGE_SIZE);
1520 } else {
1521 PMAP_STATS_INC(pmap_nzero_page_oc);
1522 PMAP_LOCK(kernel_pmap);
1523 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE);
1524 tp = tsb_kvtotte(va);
1525 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W;
1526 tp->tte_vpn = TV_VPN(va, TS_8K);
1527 cpu_block_zero((void *)va, PAGE_SIZE);
1528 tlb_page_demap(kernel_pmap, va);
1529 PMAP_UNLOCK(kernel_pmap);
1530 }
1531 }
1532
1533 void
1534 pmap_zero_page_area(vm_page_t m, int off, int size)
1535 {
1536 struct tte *tp;
1537 vm_offset_t va;
1538 vm_paddr_t pa;
1539
1540 KASSERT((m->flags & PG_FICTITIOUS) == 0,
1541 ("pmap_zero_page_area: fake page"));
1542 KASSERT(off + size <= PAGE_SIZE, ("pmap_zero_page_area: bad off/size"));
1543 PMAP_STATS_INC(pmap_nzero_page_area);
1544 pa = VM_PAGE_TO_PHYS(m);
1545 if (m->md.color == -1) {
1546 PMAP_STATS_INC(pmap_nzero_page_area_nc);
1547 aszero(ASI_PHYS_USE_EC, pa + off, size);
1548 } else if (m->md.color == DCACHE_COLOR(pa)) {
1549 PMAP_STATS_INC(pmap_nzero_page_area_c);
1550 va = TLB_PHYS_TO_DIRECT(pa);
1551 bzero((void *)(va + off), size);
1552 } else {
1553 PMAP_STATS_INC(pmap_nzero_page_area_oc);
1554 PMAP_LOCK(kernel_pmap);
1555 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE);
1556 tp = tsb_kvtotte(va);
1557 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W;
1558 tp->tte_vpn = TV_VPN(va, TS_8K);
1559 bzero((void *)(va + off), size);
1560 tlb_page_demap(kernel_pmap, va);
1561 PMAP_UNLOCK(kernel_pmap);
1562 }
1563 }
1564
1565 void
1566 pmap_zero_page_idle(vm_page_t m)
1567 {
1568 struct tte *tp;
1569 vm_offset_t va;
1570 vm_paddr_t pa;
1571
1572 KASSERT((m->flags & PG_FICTITIOUS) == 0,
1573 ("pmap_zero_page_idle: fake page"));
1574 PMAP_STATS_INC(pmap_nzero_page_idle);
1575 pa = VM_PAGE_TO_PHYS(m);
1576 if (m->md.color == -1) {
1577 PMAP_STATS_INC(pmap_nzero_page_idle_nc);
1578 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE);
1579 } else if (m->md.color == DCACHE_COLOR(pa)) {
1580 PMAP_STATS_INC(pmap_nzero_page_idle_c);
1581 va = TLB_PHYS_TO_DIRECT(pa);
1582 cpu_block_zero((void *)va, PAGE_SIZE);
1583 } else {
1584 PMAP_STATS_INC(pmap_nzero_page_idle_oc);
1585 va = pmap_idle_map + (m->md.color * PAGE_SIZE);
1586 tp = tsb_kvtotte(va);
1587 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W;
1588 tp->tte_vpn = TV_VPN(va, TS_8K);
1589 cpu_block_zero((void *)va, PAGE_SIZE);
1590 tlb_page_demap(kernel_pmap, va);
1591 }
1592 }
1593
1594 void
1595 pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
1596 {
1597 vm_offset_t vdst;
1598 vm_offset_t vsrc;
1599 vm_paddr_t pdst;
1600 vm_paddr_t psrc;
1601 struct tte *tp;
1602
1603 KASSERT((mdst->flags & PG_FICTITIOUS) == 0,
1604 ("pmap_copy_page: fake dst page"));
1605 KASSERT((msrc->flags & PG_FICTITIOUS) == 0,
1606 ("pmap_copy_page: fake src page"));
1607 PMAP_STATS_INC(pmap_ncopy_page);
1608 pdst = VM_PAGE_TO_PHYS(mdst);
1609 psrc = VM_PAGE_TO_PHYS(msrc);
1610 if (msrc->md.color == -1 && mdst->md.color == -1) {
1611 PMAP_STATS_INC(pmap_ncopy_page_nc);
1612 ascopy(ASI_PHYS_USE_EC, psrc, pdst, PAGE_SIZE);
1613 } else if (msrc->md.color == DCACHE_COLOR(psrc) &&
1614 mdst->md.color == DCACHE_COLOR(pdst)) {
1615 PMAP_STATS_INC(pmap_ncopy_page_c);
1616 vdst = TLB_PHYS_TO_DIRECT(pdst);
1617 vsrc = TLB_PHYS_TO_DIRECT(psrc);
1618 cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE);
1619 } else if (msrc->md.color == -1) {
1620 if (mdst->md.color == DCACHE_COLOR(pdst)) {
1621 PMAP_STATS_INC(pmap_ncopy_page_dc);
1622 vdst = TLB_PHYS_TO_DIRECT(pdst);
1623 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst,
1624 PAGE_SIZE);
1625 } else {
1626 PMAP_STATS_INC(pmap_ncopy_page_doc);
1627 PMAP_LOCK(kernel_pmap);
1628 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE);
1629 tp = tsb_kvtotte(vdst);
1630 tp->tte_data =
1631 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W;
1632 tp->tte_vpn = TV_VPN(vdst, TS_8K);
1633 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst,
1634 PAGE_SIZE);
1635 tlb_page_demap(kernel_pmap, vdst);
1636 PMAP_UNLOCK(kernel_pmap);
1637 }
1638 } else if (mdst->md.color == -1) {
1639 if (msrc->md.color == DCACHE_COLOR(psrc)) {
1640 PMAP_STATS_INC(pmap_ncopy_page_sc);
1641 vsrc = TLB_PHYS_TO_DIRECT(psrc);
1642 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst,
1643 PAGE_SIZE);
1644 } else {
1645 PMAP_STATS_INC(pmap_ncopy_page_soc);
1646 PMAP_LOCK(kernel_pmap);
1647 vsrc = pmap_temp_map_1 + (msrc->md.color * PAGE_SIZE);
1648 tp = tsb_kvtotte(vsrc);
1649 tp->tte_data =
1650 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W;
1651 tp->tte_vpn = TV_VPN(vsrc, TS_8K);
1652 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst,
1653 PAGE_SIZE);
1654 tlb_page_demap(kernel_pmap, vsrc);
1655 PMAP_UNLOCK(kernel_pmap);
1656 }
1657 } else {
1658 PMAP_STATS_INC(pmap_ncopy_page_oc);
1659 PMAP_LOCK(kernel_pmap);
1660 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE);
1661 tp = tsb_kvtotte(vdst);
1662 tp->tte_data =
1663 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W;
1664 tp->tte_vpn = TV_VPN(vdst, TS_8K);
1665 vsrc = pmap_temp_map_2 + (msrc->md.color * PAGE_SIZE);
1666 tp = tsb_kvtotte(vsrc);
1667 tp->tte_data =
1668 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W;
1669 tp->tte_vpn = TV_VPN(vsrc, TS_8K);
1670 cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE);
1671 tlb_page_demap(kernel_pmap, vdst);
1672 tlb_page_demap(kernel_pmap, vsrc);
1673 PMAP_UNLOCK(kernel_pmap);
1674 }
1675 }
1676
1677 /*
1678 * Returns true if the pmap's pv is one of the first
1679 * 16 pvs linked to from this page. This count may
1680 * be changed upwards or downwards in the future; it
1681 * is only necessary that true be returned for a small
1682 * subset of pmaps for proper page aging.
1683 */
1684 boolean_t
1685 pmap_page_exists_quick(pmap_t pm, vm_page_t m)
1686 {
1687 struct tte *tp;
1688 int loops;
1689
1690 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1691 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
1692 return (FALSE);
1693 loops = 0;
1694 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) {
1695 if ((tp->tte_data & TD_PV) == 0)
1696 continue;
1697 if (TTE_GET_PMAP(tp) == pm)
1698 return (TRUE);
1699 if (++loops >= 16)
1700 break;
1701 }
1702 return (FALSE);
1703 }
1704
1705 /*
1706 * Remove all pages from specified address space, this aids process exit
1707 * speeds. This is much faster than pmap_remove n the case of running down
1708 * an entire address space. Only works for the current pmap.
1709 */
1710 void
1711 pmap_remove_pages(pmap_t pm, vm_offset_t sva, vm_offset_t eva)
1712 {
1713 }
1714
1715 /*
1716 * Returns TRUE if the given page has a managed mapping.
1717 */
1718 boolean_t
1719 pmap_page_is_mapped(vm_page_t m)
1720 {
1721 struct tte *tp;
1722
1723 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1724 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
1725 return (FALSE);
1726 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) {
1727 if ((tp->tte_data & TD_PV) != 0)
1728 return (TRUE);
1729 }
1730 return (FALSE);
1731 }
1732
1733 /*
1734 * Lower the permission for all mappings to a given page.
1735 */
1736 void
1737 pmap_page_protect(vm_page_t m, vm_prot_t prot)
1738 {
1739
1740 KASSERT((m->flags & PG_FICTITIOUS) == 0,
1741 ("pmap_page_protect: fake page"));
1742 if ((prot & VM_PROT_WRITE) == 0) {
1743 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE))
1744 pmap_clear_write(m);
1745 else
1746 pmap_remove_all(m);
1747 }
1748 }
1749
1750 /*
1751 * pmap_ts_referenced:
1752 *
1753 * Return a count of reference bits for a page, clearing those bits.
1754 * It is not necessary for every reference bit to be cleared, but it
1755 * is necessary that 0 only be returned when there are truly no
1756 * reference bits set.
1757 *
1758 * XXX: The exact number of bits to check and clear is a matter that
1759 * should be tested and standardized at some point in the future for
1760 * optimal aging of shared pages.
1761 */
1762
1763 int
1764 pmap_ts_referenced(vm_page_t m)
1765 {
1766 struct tte *tpf;
1767 struct tte *tpn;
1768 struct tte *tp;
1769 u_long data;
1770 int count;
1771
1772 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1773 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
1774 return (0);
1775 count = 0;
1776 if ((tp = TAILQ_FIRST(&m->md.tte_list)) != NULL) {
1777 tpf = tp;
1778 do {
1779 tpn = TAILQ_NEXT(tp, tte_link);
1780 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link);
1781 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link);
1782 if ((tp->tte_data & TD_PV) == 0 ||
1783 !pmap_track_modified(TTE_GET_PMAP(tp),
1784 TTE_GET_VA(tp)))
1785 continue;
1786 data = atomic_clear_long(&tp->tte_data, TD_REF);
1787 if ((data & TD_REF) != 0 && ++count > 4)
1788 break;
1789 } while ((tp = tpn) != NULL && tp != tpf);
1790 }
1791 return (count);
1792 }
1793
1794 boolean_t
1795 pmap_is_modified(vm_page_t m)
1796 {
1797 struct tte *tp;
1798
1799 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1800 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
1801 return (FALSE);
1802 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) {
1803 if ((tp->tte_data & TD_PV) == 0 ||
1804 !pmap_track_modified(TTE_GET_PMAP(tp), TTE_GET_VA(tp)))
1805 continue;
1806 if ((tp->tte_data & TD_W) != 0)
1807 return (TRUE);
1808 }
1809 return (FALSE);
1810 }
1811
1812 /*
1813 * pmap_is_prefaultable:
1814 *
1815 * Return whether or not the specified virtual address is elgible
1816 * for prefault.
1817 */
1818 boolean_t
1819 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
1820 {
1821
1822 return (FALSE);
1823 }
1824
1825 void
1826 pmap_clear_modify(vm_page_t m)
1827 {
1828 struct tte *tp;
1829 u_long data;
1830
1831 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1832 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
1833 return;
1834 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) {
1835 if ((tp->tte_data & TD_PV) == 0)
1836 continue;
1837 data = atomic_clear_long(&tp->tte_data, TD_W);
1838 if ((data & TD_W) != 0)
1839 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp));
1840 }
1841 }
1842
1843 void
1844 pmap_clear_reference(vm_page_t m)
1845 {
1846 struct tte *tp;
1847 u_long data;
1848
1849 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1850 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
1851 return;
1852 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) {
1853 if ((tp->tte_data & TD_PV) == 0)
1854 continue;
1855 data = atomic_clear_long(&tp->tte_data, TD_REF);
1856 if ((data & TD_REF) != 0)
1857 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp));
1858 }
1859 }
1860
1861 void
1862 pmap_clear_write(vm_page_t m)
1863 {
1864 struct tte *tp;
1865 u_long data;
1866
1867 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1868 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0 ||
1869 (m->flags & PG_WRITEABLE) == 0)
1870 return;
1871 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) {
1872 if ((tp->tte_data & TD_PV) == 0)
1873 continue;
1874 data = atomic_clear_long(&tp->tte_data, TD_SW | TD_W);
1875 if ((data & TD_W) != 0) {
1876 if (pmap_track_modified(TTE_GET_PMAP(tp),
1877 TTE_GET_VA(tp)))
1878 vm_page_dirty(m);
1879 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp));
1880 }
1881 }
1882 vm_page_flag_clear(m, PG_WRITEABLE);
1883 }
1884
1885 int
1886 pmap_mincore(pmap_t pm, vm_offset_t addr)
1887 {
1888 /* TODO; */
1889 return (0);
1890 }
1891
1892 /*
1893 * Activate a user pmap. The pmap must be activated before its address space
1894 * can be accessed in any way.
1895 */
1896 void
1897 pmap_activate(struct thread *td)
1898 {
1899 struct vmspace *vm;
1900 struct pmap *pm;
1901 int context;
1902
1903 vm = td->td_proc->p_vmspace;
1904 pm = vmspace_pmap(vm);
1905
1906 mtx_lock_spin(&sched_lock);
1907
1908 context = PCPU_GET(tlb_ctx);
1909 if (context == PCPU_GET(tlb_ctx_max)) {
1910 tlb_flush_user();
1911 context = PCPU_GET(tlb_ctx_min);
1912 }
1913 PCPU_SET(tlb_ctx, context + 1);
1914
1915 pm->pm_context[PCPU_GET(cpuid)] = context;
1916 pm->pm_active |= PCPU_GET(cpumask);
1917 PCPU_SET(pmap, pm);
1918
1919 stxa(AA_DMMU_TSB, ASI_DMMU, pm->pm_tsb);
1920 stxa(AA_IMMU_TSB, ASI_IMMU, pm->pm_tsb);
1921 stxa(AA_DMMU_PCXR, ASI_DMMU, context);
1922 membar(Sync);
1923
1924 mtx_unlock_spin(&sched_lock);
1925 }
1926
1927 vm_offset_t
1928 pmap_addr_hint(vm_object_t object, vm_offset_t va, vm_size_t size)
1929 {
1930
1931 return (va);
1932 }
Cache object: 65d51b044c92b719534255ef8ffed555
|