FreeBSD/Linux Kernel Cross Reference
sys/uvm/uvm_bio.c
1 /* $NetBSD: uvm_bio.c,v 1.126 2021/04/01 06:26:26 simonb Exp $ */
2
3 /*
4 * Copyright (c) 1998 Chuck Silvers.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. The name of the author may not be used to endorse or promote products
16 * derived from this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 */
31
32 /*
33 * uvm_bio.c: buffered i/o object mapping cache
34 */
35
36 #include <sys/cdefs.h>
37 __KERNEL_RCSID(0, "$NetBSD: uvm_bio.c,v 1.126 2021/04/01 06:26:26 simonb Exp $");
38
39 #include "opt_uvmhist.h"
40 #include "opt_ubc.h"
41
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/kmem.h>
45 #include <sys/kernel.h>
46 #include <sys/proc.h>
47 #include <sys/sysctl.h>
48 #include <sys/vnode.h>
49 #include <sys/bitops.h> /* for ilog2() */
50
51 #include <uvm/uvm.h>
52 #include <uvm/uvm_pdpolicy.h>
53
54 #ifdef PMAP_DIRECT
55 # define UBC_USE_PMAP_DIRECT
56 #endif
57
58 /*
59 * local functions
60 */
61
62 static int ubc_fault(struct uvm_faultinfo *, vaddr_t, struct vm_page **,
63 int, int, vm_prot_t, int);
64 static struct ubc_map *ubc_find_mapping(struct uvm_object *, voff_t);
65 static int ubchash_stats(struct hashstat_sysctl *hs, bool fill);
66 #ifdef UBC_USE_PMAP_DIRECT
67 static int __noinline ubc_uiomove_direct(struct uvm_object *, struct uio *, vsize_t,
68 int, int);
69 static void __noinline ubc_zerorange_direct(struct uvm_object *, off_t, size_t, int);
70
71 /* XXX disabled by default until the kinks are worked out. */
72 bool ubc_direct = false;
73 #endif
74
75 /*
76 * local data structues
77 */
78
79 #define UBC_HASH(uobj, offset) \
80 (((((u_long)(uobj)) >> 8) + (((u_long)(offset)) >> PAGE_SHIFT)) & \
81 ubc_object.hashmask)
82
83 #define UBC_QUEUE(offset) \
84 (&ubc_object.inactive[(((u_long)(offset)) >> ubc_winshift) & \
85 (UBC_NQUEUES - 1)])
86
87 #define UBC_UMAP_ADDR(u) \
88 (vaddr_t)(ubc_object.kva + (((u) - ubc_object.umap) << ubc_winshift))
89
90
91 #define UMAP_PAGES_LOCKED 0x0001
92 #define UMAP_MAPPING_CACHED 0x0002
93
94 struct ubc_map {
95 struct uvm_object * uobj; /* mapped object */
96 voff_t offset; /* offset into uobj */
97 voff_t writeoff; /* write offset */
98 vsize_t writelen; /* write len */
99 int refcount; /* refcount on mapping */
100 int flags; /* extra state */
101 int advice;
102
103 LIST_ENTRY(ubc_map) hash; /* hash table */
104 TAILQ_ENTRY(ubc_map) inactive; /* inactive queue */
105 LIST_ENTRY(ubc_map) list; /* per-object list */
106 };
107
108 TAILQ_HEAD(ubc_inactive_head, ubc_map);
109 static struct ubc_object {
110 struct uvm_object uobj; /* glue for uvm_map() */
111 char *kva; /* where ubc_object is mapped */
112 struct ubc_map *umap; /* array of ubc_map's */
113
114 LIST_HEAD(, ubc_map) *hash; /* hashtable for cached ubc_map's */
115 u_long hashmask; /* mask for hashtable */
116
117 struct ubc_inactive_head *inactive;
118 /* inactive queues for ubc_map's */
119 } ubc_object;
120
121 const struct uvm_pagerops ubc_pager = {
122 .pgo_fault = ubc_fault,
123 /* ... rest are NULL */
124 };
125
126 /* Use value at least as big as maximum page size supported by architecture */
127 #define UBC_MAX_WINSHIFT \
128 ((1 << UBC_WINSHIFT) > MAX_PAGE_SIZE ? UBC_WINSHIFT : ilog2(MAX_PAGE_SIZE))
129
130 int ubc_nwins = UBC_NWINS;
131 const int ubc_winshift = UBC_MAX_WINSHIFT;
132 const int ubc_winsize = 1 << UBC_MAX_WINSHIFT;
133 #if defined(PMAP_PREFER)
134 int ubc_nqueues;
135 #define UBC_NQUEUES ubc_nqueues
136 #else
137 #define UBC_NQUEUES 1
138 #endif
139
140 #if defined(UBC_STATS)
141
142 #define UBC_EVCNT_DEFINE(name) \
143 struct evcnt ubc_evcnt_##name = \
144 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "ubc", #name); \
145 EVCNT_ATTACH_STATIC(ubc_evcnt_##name);
146 #define UBC_EVCNT_INCR(name) ubc_evcnt_##name.ev_count++
147
148 #else /* defined(UBC_STATS) */
149
150 #define UBC_EVCNT_DEFINE(name) /* nothing */
151 #define UBC_EVCNT_INCR(name) /* nothing */
152
153 #endif /* defined(UBC_STATS) */
154
155 UBC_EVCNT_DEFINE(wincachehit)
156 UBC_EVCNT_DEFINE(wincachemiss)
157 UBC_EVCNT_DEFINE(faultbusy)
158
159 /*
160 * ubc_init
161 *
162 * init pager private data structures.
163 */
164
165 void
166 ubc_init(void)
167 {
168 /*
169 * Make sure ubc_winshift is sane.
170 */
171 KASSERT(ubc_winshift >= PAGE_SHIFT);
172
173 /*
174 * init ubc_object.
175 * alloc and init ubc_map's.
176 * init inactive queues.
177 * alloc and init hashtable.
178 * map in ubc_object.
179 */
180
181 uvm_obj_init(&ubc_object.uobj, &ubc_pager, true, UVM_OBJ_KERN);
182
183 ubc_object.umap = kmem_zalloc(ubc_nwins * sizeof(struct ubc_map),
184 KM_SLEEP);
185 if (ubc_object.umap == NULL)
186 panic("ubc_init: failed to allocate ubc_map");
187
188 vaddr_t va = (vaddr_t)1L;
189 #ifdef PMAP_PREFER
190 PMAP_PREFER(0, &va, 0, 0); /* kernel is never topdown */
191 ubc_nqueues = va >> ubc_winshift;
192 if (ubc_nqueues == 0) {
193 ubc_nqueues = 1;
194 }
195 #endif
196 ubc_object.inactive = kmem_alloc(UBC_NQUEUES *
197 sizeof(struct ubc_inactive_head), KM_SLEEP);
198 for (int i = 0; i < UBC_NQUEUES; i++) {
199 TAILQ_INIT(&ubc_object.inactive[i]);
200 }
201 for (int i = 0; i < ubc_nwins; i++) {
202 struct ubc_map *umap;
203 umap = &ubc_object.umap[i];
204 TAILQ_INSERT_TAIL(&ubc_object.inactive[i & (UBC_NQUEUES - 1)],
205 umap, inactive);
206 }
207
208 ubc_object.hash = hashinit(ubc_nwins, HASH_LIST, true,
209 &ubc_object.hashmask);
210 for (int i = 0; i <= ubc_object.hashmask; i++) {
211 LIST_INIT(&ubc_object.hash[i]);
212 }
213
214 if (uvm_map(kernel_map, (vaddr_t *)&ubc_object.kva,
215 ubc_nwins << ubc_winshift, &ubc_object.uobj, 0, (vsize_t)va,
216 UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, UVM_INH_NONE,
217 UVM_ADV_RANDOM, UVM_FLAG_NOMERGE)) != 0) {
218 panic("ubc_init: failed to map ubc_object");
219 }
220
221 hashstat_register("ubchash", ubchash_stats);
222 }
223
224 void
225 ubchist_init(void)
226 {
227
228 UVMHIST_INIT(ubchist, 300);
229 }
230
231 /*
232 * ubc_fault_page: helper of ubc_fault to handle a single page.
233 *
234 * => Caller has UVM object locked.
235 * => Caller will perform pmap_update().
236 */
237
238 static inline int
239 ubc_fault_page(const struct uvm_faultinfo *ufi, const struct ubc_map *umap,
240 struct vm_page *pg, vm_prot_t prot, vm_prot_t access_type, vaddr_t va)
241 {
242 vm_prot_t mask;
243 int error;
244 bool rdonly;
245
246 KASSERT(rw_write_held(pg->uobject->vmobjlock));
247
248 KASSERT((pg->flags & PG_FAKE) == 0);
249 if (pg->flags & PG_RELEASED) {
250 uvm_pagefree(pg);
251 return 0;
252 }
253 if (pg->loan_count != 0) {
254
255 /*
256 * Avoid unneeded loan break, if possible.
257 */
258
259 if ((access_type & VM_PROT_WRITE) == 0) {
260 prot &= ~VM_PROT_WRITE;
261 }
262 if (prot & VM_PROT_WRITE) {
263 struct vm_page *newpg;
264
265 newpg = uvm_loanbreak(pg);
266 if (newpg == NULL) {
267 uvm_page_unbusy(&pg, 1);
268 return ENOMEM;
269 }
270 pg = newpg;
271 }
272 }
273
274 /*
275 * Note that a page whose backing store is partially allocated
276 * is marked as PG_RDONLY.
277 *
278 * it's a responsibility of ubc_alloc's caller to allocate backing
279 * blocks before writing to the window.
280 */
281
282 KASSERT((pg->flags & PG_RDONLY) == 0 ||
283 (access_type & VM_PROT_WRITE) == 0 ||
284 pg->offset < umap->writeoff ||
285 pg->offset + PAGE_SIZE > umap->writeoff + umap->writelen);
286
287 rdonly = uvm_pagereadonly_p(pg);
288 mask = rdonly ? ~VM_PROT_WRITE : VM_PROT_ALL;
289
290 error = pmap_enter(ufi->orig_map->pmap, va, VM_PAGE_TO_PHYS(pg),
291 prot & mask, PMAP_CANFAIL | (access_type & mask));
292
293 uvm_pagelock(pg);
294 uvm_pageactivate(pg);
295 uvm_pagewakeup(pg);
296 uvm_pageunlock(pg);
297 pg->flags &= ~PG_BUSY;
298 UVM_PAGE_OWN(pg, NULL);
299
300 return error;
301 }
302
303 /*
304 * ubc_fault: fault routine for ubc mapping
305 */
306
307 static int
308 ubc_fault(struct uvm_faultinfo *ufi, vaddr_t ign1, struct vm_page **ign2,
309 int ign3, int ign4, vm_prot_t access_type, int flags)
310 {
311 struct uvm_object *uobj;
312 struct ubc_map *umap;
313 vaddr_t va, eva, ubc_offset, slot_offset;
314 struct vm_page *pgs[howmany(ubc_winsize, MIN_PAGE_SIZE)];
315 int i, error, npages;
316 vm_prot_t prot;
317
318 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);
319
320 /*
321 * no need to try with PGO_LOCKED...
322 * we don't need to have the map locked since we know that
323 * no one will mess with it until our reference is released.
324 */
325
326 if (flags & PGO_LOCKED) {
327 uvmfault_unlockall(ufi, NULL, &ubc_object.uobj);
328 flags &= ~PGO_LOCKED;
329 }
330
331 va = ufi->orig_rvaddr;
332 ubc_offset = va - (vaddr_t)ubc_object.kva;
333 umap = &ubc_object.umap[ubc_offset >> ubc_winshift];
334 KASSERT(umap->refcount != 0);
335 KASSERT((umap->flags & UMAP_PAGES_LOCKED) == 0);
336 slot_offset = ubc_offset & (ubc_winsize - 1);
337
338 /*
339 * some platforms cannot write to individual bytes atomically, so
340 * software has to do read/modify/write of larger quantities instead.
341 * this means that the access_type for "write" operations
342 * can be VM_PROT_READ, which confuses us mightily.
343 *
344 * deal with this by resetting access_type based on the info
345 * that ubc_alloc() stores for us.
346 */
347
348 access_type = umap->writelen ? VM_PROT_WRITE : VM_PROT_READ;
349 UVMHIST_LOG(ubchist, "va %#jx ubc_offset %#jx access_type %jd",
350 va, ubc_offset, access_type, 0);
351
352 if ((access_type & VM_PROT_WRITE) != 0) {
353 #ifndef PRIxOFF /* XXX */
354 #define PRIxOFF "jx" /* XXX */
355 #endif /* XXX */
356 KASSERTMSG((trunc_page(umap->writeoff) <= slot_offset),
357 "out of range write: slot=%#"PRIxVSIZE" off=%#"PRIxOFF,
358 slot_offset, (intmax_t)umap->writeoff);
359 KASSERTMSG((slot_offset < umap->writeoff + umap->writelen),
360 "out of range write: slot=%#"PRIxVADDR
361 " off=%#"PRIxOFF" len=%#"PRIxVSIZE,
362 slot_offset, (intmax_t)umap->writeoff, umap->writelen);
363 }
364
365 /* no umap locking needed since we have a ref on the umap */
366 uobj = umap->uobj;
367
368 if ((access_type & VM_PROT_WRITE) == 0) {
369 npages = (ubc_winsize - slot_offset) >> PAGE_SHIFT;
370 } else {
371 npages = (round_page(umap->offset + umap->writeoff +
372 umap->writelen) - (umap->offset + slot_offset))
373 >> PAGE_SHIFT;
374 flags |= PGO_PASTEOF;
375 }
376
377 again:
378 memset(pgs, 0, sizeof (pgs));
379 rw_enter(uobj->vmobjlock, RW_WRITER);
380
381 UVMHIST_LOG(ubchist, "slot_offset %#jx writeoff %#jx writelen %#jx ",
382 slot_offset, umap->writeoff, umap->writelen, 0);
383 UVMHIST_LOG(ubchist, "getpages uobj %#jx offset %#jx npages %jd",
384 (uintptr_t)uobj, umap->offset + slot_offset, npages, 0);
385
386 error = (*uobj->pgops->pgo_get)(uobj, umap->offset + slot_offset, pgs,
387 &npages, 0, access_type, umap->advice, flags | PGO_NOBLOCKALLOC |
388 PGO_NOTIMESTAMP);
389 UVMHIST_LOG(ubchist, "getpages error %jd npages %jd", error, npages, 0,
390 0);
391
392 if (error == EAGAIN) {
393 kpause("ubc_fault", false, hz >> 2, NULL);
394 goto again;
395 }
396 if (error) {
397 return error;
398 }
399
400 /*
401 * For virtually-indexed, virtually-tagged caches we should avoid
402 * creating writable mappings when we do not absolutely need them,
403 * since the "compatible alias" trick does not work on such caches.
404 * Otherwise, we can always map the pages writable.
405 */
406
407 #ifdef PMAP_CACHE_VIVT
408 prot = VM_PROT_READ | access_type;
409 #else
410 prot = VM_PROT_READ | VM_PROT_WRITE;
411 #endif
412
413 va = ufi->orig_rvaddr;
414 eva = ufi->orig_rvaddr + (npages << PAGE_SHIFT);
415
416 UVMHIST_LOG(ubchist, "va %#jx eva %#jx", va, eva, 0, 0);
417
418 /*
419 * Note: normally all returned pages would have the same UVM object.
420 * However, layered file-systems and e.g. tmpfs, may return pages
421 * which belong to underlying UVM object. In such case, lock is
422 * shared amongst the objects.
423 */
424 rw_enter(uobj->vmobjlock, RW_WRITER);
425 for (i = 0; va < eva; i++, va += PAGE_SIZE) {
426 struct vm_page *pg;
427
428 UVMHIST_LOG(ubchist, "pgs[%jd] = %#jx", i, (uintptr_t)pgs[i],
429 0, 0);
430 pg = pgs[i];
431
432 if (pg == NULL || pg == PGO_DONTCARE) {
433 continue;
434 }
435 KASSERT(uobj->vmobjlock == pg->uobject->vmobjlock);
436 error = ubc_fault_page(ufi, umap, pg, prot, access_type, va);
437 if (error) {
438 /*
439 * Flush (there might be pages entered), drop the lock,
440 * and perform uvm_wait(). Note: page will re-fault.
441 */
442 pmap_update(ufi->orig_map->pmap);
443 rw_exit(uobj->vmobjlock);
444 uvm_wait("ubc_fault");
445 rw_enter(uobj->vmobjlock, RW_WRITER);
446 }
447 }
448 /* Must make VA visible before the unlock. */
449 pmap_update(ufi->orig_map->pmap);
450 rw_exit(uobj->vmobjlock);
451
452 return 0;
453 }
454
455 /*
456 * local functions
457 */
458
459 static struct ubc_map *
460 ubc_find_mapping(struct uvm_object *uobj, voff_t offset)
461 {
462 struct ubc_map *umap;
463
464 LIST_FOREACH(umap, &ubc_object.hash[UBC_HASH(uobj, offset)], hash) {
465 if (umap->uobj == uobj && umap->offset == offset) {
466 return umap;
467 }
468 }
469 return NULL;
470 }
471
472
473 /*
474 * ubc interface functions
475 */
476
477 /*
478 * ubc_alloc: allocate a file mapping window
479 */
480
481 static void * __noinline
482 ubc_alloc(struct uvm_object *uobj, voff_t offset, vsize_t *lenp, int advice,
483 int flags, struct vm_page **pgs, int *npagesp)
484 {
485 vaddr_t slot_offset, va;
486 struct ubc_map *umap;
487 voff_t umap_offset;
488 int error;
489 UVMHIST_FUNC(__func__);
490 UVMHIST_CALLARGS(ubchist, "uobj %#jx offset %#jx len %#jx",
491 (uintptr_t)uobj, offset, *lenp, 0);
492
493 KASSERT(*lenp > 0);
494 umap_offset = (offset & ~((voff_t)ubc_winsize - 1));
495 slot_offset = (vaddr_t)(offset & ((voff_t)ubc_winsize - 1));
496 *lenp = MIN(*lenp, ubc_winsize - slot_offset);
497 KASSERT(*lenp > 0);
498
499 rw_enter(ubc_object.uobj.vmobjlock, RW_WRITER);
500 again:
501 /*
502 * The UVM object is already referenced.
503 * Lock order: UBC object -> ubc_map::uobj.
504 */
505 umap = ubc_find_mapping(uobj, umap_offset);
506 if (umap == NULL) {
507 struct uvm_object *oobj;
508
509 UBC_EVCNT_INCR(wincachemiss);
510 umap = TAILQ_FIRST(UBC_QUEUE(offset));
511 if (umap == NULL) {
512 rw_exit(ubc_object.uobj.vmobjlock);
513 kpause("ubc_alloc", false, hz >> 2, NULL);
514 rw_enter(ubc_object.uobj.vmobjlock, RW_WRITER);
515 goto again;
516 }
517
518 va = UBC_UMAP_ADDR(umap);
519 oobj = umap->uobj;
520
521 /*
522 * Remove from old hash (if any), add to new hash.
523 */
524
525 if (oobj != NULL) {
526 /*
527 * Mapping must be removed before the list entry,
528 * since there is a race with ubc_purge().
529 */
530 if (umap->flags & UMAP_MAPPING_CACHED) {
531 umap->flags &= ~UMAP_MAPPING_CACHED;
532 rw_enter(oobj->vmobjlock, RW_WRITER);
533 pmap_remove(pmap_kernel(), va,
534 va + ubc_winsize);
535 pmap_update(pmap_kernel());
536 rw_exit(oobj->vmobjlock);
537 }
538 LIST_REMOVE(umap, hash);
539 LIST_REMOVE(umap, list);
540 } else {
541 KASSERT((umap->flags & UMAP_MAPPING_CACHED) == 0);
542 }
543 umap->uobj = uobj;
544 umap->offset = umap_offset;
545 LIST_INSERT_HEAD(&ubc_object.hash[UBC_HASH(uobj, umap_offset)],
546 umap, hash);
547 LIST_INSERT_HEAD(&uobj->uo_ubc, umap, list);
548 } else {
549 UBC_EVCNT_INCR(wincachehit);
550 va = UBC_UMAP_ADDR(umap);
551 }
552
553 if (umap->refcount == 0) {
554 TAILQ_REMOVE(UBC_QUEUE(offset), umap, inactive);
555 }
556
557 if (flags & UBC_WRITE) {
558 KASSERTMSG(umap->writeoff == 0 && umap->writelen == 0,
559 "ubc_alloc: concurrent writes to uobj %p", uobj);
560 umap->writeoff = slot_offset;
561 umap->writelen = *lenp;
562 }
563
564 umap->refcount++;
565 umap->advice = advice;
566 rw_exit(ubc_object.uobj.vmobjlock);
567 UVMHIST_LOG(ubchist, "umap %#jx refs %jd va %#jx flags %#jx",
568 (uintptr_t)umap, umap->refcount, (uintptr_t)va, flags);
569
570 if (flags & UBC_FAULTBUSY) {
571 int npages = (*lenp + (offset & (PAGE_SIZE - 1)) +
572 PAGE_SIZE - 1) >> PAGE_SHIFT;
573 int gpflags =
574 PGO_SYNCIO|PGO_OVERWRITE|PGO_PASTEOF|PGO_NOBLOCKALLOC|
575 PGO_NOTIMESTAMP;
576 int i;
577 KDASSERT(flags & UBC_WRITE);
578 KASSERT(npages <= *npagesp);
579 KASSERT(umap->refcount == 1);
580
581 UBC_EVCNT_INCR(faultbusy);
582 again_faultbusy:
583 rw_enter(uobj->vmobjlock, RW_WRITER);
584 if (umap->flags & UMAP_MAPPING_CACHED) {
585 umap->flags &= ~UMAP_MAPPING_CACHED;
586 pmap_remove(pmap_kernel(), va, va + ubc_winsize);
587 }
588 memset(pgs, 0, *npagesp * sizeof(pgs[0]));
589
590 error = (*uobj->pgops->pgo_get)(uobj, trunc_page(offset), pgs,
591 &npages, 0, VM_PROT_READ | VM_PROT_WRITE, advice, gpflags);
592 UVMHIST_LOG(ubchist, "faultbusy getpages %jd", error, 0, 0, 0);
593 if (error) {
594 /*
595 * Flush: the mapping above might have been removed.
596 */
597 pmap_update(pmap_kernel());
598 goto out;
599 }
600 for (i = 0; i < npages; i++) {
601 struct vm_page *pg = pgs[i];
602
603 KASSERT(pg->uobject == uobj);
604 if (pg->loan_count != 0) {
605 rw_enter(uobj->vmobjlock, RW_WRITER);
606 if (pg->loan_count != 0) {
607 pg = uvm_loanbreak(pg);
608 }
609 if (pg == NULL) {
610 pmap_kremove(va, ubc_winsize);
611 pmap_update(pmap_kernel());
612 uvm_page_unbusy(pgs, npages);
613 rw_exit(uobj->vmobjlock);
614 uvm_wait("ubc_alloc");
615 goto again_faultbusy;
616 }
617 rw_exit(uobj->vmobjlock);
618 pgs[i] = pg;
619 }
620 pmap_kenter_pa(
621 va + trunc_page(slot_offset) + (i << PAGE_SHIFT),
622 VM_PAGE_TO_PHYS(pg),
623 VM_PROT_READ | VM_PROT_WRITE, 0);
624 }
625 pmap_update(pmap_kernel());
626 umap->flags |= UMAP_PAGES_LOCKED;
627 *npagesp = npages;
628 } else {
629 KASSERT((umap->flags & UMAP_PAGES_LOCKED) == 0);
630 }
631
632 out:
633 return (void *)(va + slot_offset);
634 }
635
636 /*
637 * ubc_release: free a file mapping window.
638 */
639
640 static void __noinline
641 ubc_release(void *va, int flags, struct vm_page **pgs, int npages)
642 {
643 struct ubc_map *umap;
644 struct uvm_object *uobj;
645 vaddr_t umapva;
646 bool unmapped;
647 UVMHIST_FUNC(__func__);
648 UVMHIST_CALLARGS(ubchist, "va %#jx", (uintptr_t)va, 0, 0, 0);
649
650 umap = &ubc_object.umap[((char *)va - ubc_object.kva) >> ubc_winshift];
651 umapva = UBC_UMAP_ADDR(umap);
652 uobj = umap->uobj;
653 KASSERT(uobj != NULL);
654
655 if (umap->flags & UMAP_PAGES_LOCKED) {
656 const voff_t endoff = umap->writeoff + umap->writelen;
657 const voff_t zerolen = round_page(endoff) - endoff;
658
659 KASSERT(npages == (round_page(endoff) -
660 trunc_page(umap->writeoff)) >> PAGE_SHIFT);
661 KASSERT((umap->flags & UMAP_MAPPING_CACHED) == 0);
662 if (zerolen) {
663 memset((char *)umapva + endoff, 0, zerolen);
664 }
665 umap->flags &= ~UMAP_PAGES_LOCKED;
666 rw_enter(uobj->vmobjlock, RW_WRITER);
667 for (u_int i = 0; i < npages; i++) {
668 struct vm_page *pg = pgs[i];
669 #ifdef DIAGNOSTIC
670 paddr_t pa;
671 bool rv;
672 rv = pmap_extract(pmap_kernel(), umapva +
673 umap->writeoff + (i << PAGE_SHIFT), &pa);
674 KASSERT(rv);
675 KASSERT(PHYS_TO_VM_PAGE(pa) == pg);
676 #endif
677 pg->flags &= ~PG_FAKE;
678 KASSERTMSG(uvm_pagegetdirty(pg) ==
679 UVM_PAGE_STATUS_DIRTY,
680 "page %p not dirty", pg);
681 KASSERT(pg->loan_count == 0);
682 if (uvmpdpol_pageactivate_p(pg)) {
683 uvm_pagelock(pg);
684 uvm_pageactivate(pg);
685 uvm_pageunlock(pg);
686 }
687 }
688 pmap_kremove(umapva, ubc_winsize);
689 pmap_update(pmap_kernel());
690 uvm_page_unbusy(pgs, npages);
691 rw_exit(uobj->vmobjlock);
692 unmapped = true;
693 } else {
694 unmapped = false;
695 }
696
697 rw_enter(ubc_object.uobj.vmobjlock, RW_WRITER);
698 umap->writeoff = 0;
699 umap->writelen = 0;
700 umap->refcount--;
701 if (umap->refcount == 0) {
702 if (flags & UBC_UNMAP) {
703 /*
704 * Invalidate any cached mappings if requested.
705 * This is typically used to avoid leaving
706 * incompatible cache aliases around indefinitely.
707 */
708 rw_enter(uobj->vmobjlock, RW_WRITER);
709 pmap_remove(pmap_kernel(), umapva,
710 umapva + ubc_winsize);
711 pmap_update(pmap_kernel());
712 rw_exit(uobj->vmobjlock);
713
714 umap->flags &= ~UMAP_MAPPING_CACHED;
715 LIST_REMOVE(umap, hash);
716 LIST_REMOVE(umap, list);
717 umap->uobj = NULL;
718 TAILQ_INSERT_HEAD(UBC_QUEUE(umap->offset), umap,
719 inactive);
720 } else {
721 if (!unmapped) {
722 umap->flags |= UMAP_MAPPING_CACHED;
723 }
724 TAILQ_INSERT_TAIL(UBC_QUEUE(umap->offset), umap,
725 inactive);
726 }
727 }
728 UVMHIST_LOG(ubchist, "umap %#jx refs %jd", (uintptr_t)umap,
729 umap->refcount, 0, 0);
730 rw_exit(ubc_object.uobj.vmobjlock);
731 }
732
733 /*
734 * ubc_uiomove: move data to/from an object.
735 */
736
737 int
738 ubc_uiomove(struct uvm_object *uobj, struct uio *uio, vsize_t todo, int advice,
739 int flags)
740 {
741 const bool overwrite = (flags & UBC_FAULTBUSY) != 0;
742 struct vm_page *pgs[howmany(ubc_winsize, MIN_PAGE_SIZE)];
743 voff_t off;
744 int error, npages;
745
746 KASSERT(todo <= uio->uio_resid);
747 KASSERT(((flags & UBC_WRITE) != 0 && uio->uio_rw == UIO_WRITE) ||
748 ((flags & UBC_READ) != 0 && uio->uio_rw == UIO_READ));
749
750 #ifdef UBC_USE_PMAP_DIRECT
751 /*
752 * during direct access pages need to be held busy to prevent them
753 * changing identity, and therefore if we read or write an object
754 * into a mapped view of same we could deadlock while faulting.
755 *
756 * avoid the problem by disallowing direct access if the object
757 * might be visible somewhere via mmap().
758 *
759 * XXX concurrent reads cause thundering herd issues with PG_BUSY.
760 * In the future enable by default for writes or if ncpu<=2, and
761 * make the toggle override that.
762 */
763 if ((ubc_direct && (flags & UBC_ISMAPPED) == 0) ||
764 (flags & UBC_FAULTBUSY) != 0) {
765 return ubc_uiomove_direct(uobj, uio, todo, advice, flags);
766 }
767 #endif
768
769 off = uio->uio_offset;
770 error = 0;
771 while (todo > 0) {
772 vsize_t bytelen = todo;
773 void *win;
774
775 npages = __arraycount(pgs);
776 win = ubc_alloc(uobj, off, &bytelen, advice, flags, pgs,
777 &npages);
778 if (error == 0) {
779 error = uiomove(win, bytelen, uio);
780 }
781 if (error != 0 && overwrite) {
782 /*
783 * if we haven't initialized the pages yet,
784 * do it now. it's safe to use memset here
785 * because we just mapped the pages above.
786 */
787 memset(win, 0, bytelen);
788 }
789 ubc_release(win, flags, pgs, npages);
790 off += bytelen;
791 todo -= bytelen;
792 if (error != 0 && (flags & UBC_PARTIALOK) != 0) {
793 break;
794 }
795 }
796
797 return error;
798 }
799
800 /*
801 * ubc_zerorange: set a range of bytes in an object to zero.
802 */
803
804 void
805 ubc_zerorange(struct uvm_object *uobj, off_t off, size_t len, int flags)
806 {
807 struct vm_page *pgs[howmany(ubc_winsize, MIN_PAGE_SIZE)];
808 int npages;
809
810 #ifdef UBC_USE_PMAP_DIRECT
811 if (ubc_direct || (flags & UBC_FAULTBUSY) != 0) {
812 ubc_zerorange_direct(uobj, off, len, flags);
813 return;
814 }
815 #endif
816
817 /*
818 * XXXUBC invent kzero() and use it
819 */
820
821 while (len) {
822 void *win;
823 vsize_t bytelen = len;
824
825 npages = __arraycount(pgs);
826 win = ubc_alloc(uobj, off, &bytelen, UVM_ADV_NORMAL, UBC_WRITE,
827 pgs, &npages);
828 memset(win, 0, bytelen);
829 ubc_release(win, flags, pgs, npages);
830
831 off += bytelen;
832 len -= bytelen;
833 }
834 }
835
836 #ifdef UBC_USE_PMAP_DIRECT
837 /* Copy data using direct map */
838
839 /*
840 * ubc_alloc_direct: allocate a file mapping window using direct map
841 */
842 static int __noinline
843 ubc_alloc_direct(struct uvm_object *uobj, voff_t offset, vsize_t *lenp,
844 int advice, int flags, struct vm_page **pgs, int *npages)
845 {
846 voff_t pgoff;
847 int error;
848 int gpflags = flags | PGO_NOTIMESTAMP | PGO_SYNCIO;
849 int access_type = VM_PROT_READ;
850 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);
851
852 if (flags & UBC_WRITE) {
853 if (flags & UBC_FAULTBUSY)
854 gpflags |= PGO_OVERWRITE | PGO_NOBLOCKALLOC;
855 #if 0
856 KASSERT(!UVM_OBJ_NEEDS_WRITEFAULT(uobj));
857 #endif
858
859 /*
860 * Tell genfs_getpages() we already have the journal lock,
861 * allow allocation past current EOF.
862 */
863 gpflags |= PGO_JOURNALLOCKED | PGO_PASTEOF;
864 access_type |= VM_PROT_WRITE;
865 } else {
866 /* Don't need the empty blocks allocated, PG_RDONLY is okay */
867 gpflags |= PGO_NOBLOCKALLOC;
868 }
869
870 pgoff = (offset & PAGE_MASK);
871 *lenp = MIN(*lenp, ubc_winsize - pgoff);
872
873 again:
874 *npages = (*lenp + pgoff + PAGE_SIZE - 1) >> PAGE_SHIFT;
875 KASSERT((*npages * PAGE_SIZE) <= ubc_winsize);
876 KASSERT(*lenp + pgoff <= ubc_winsize);
877 memset(pgs, 0, *npages * sizeof(pgs[0]));
878
879 rw_enter(uobj->vmobjlock, RW_WRITER);
880 error = (*uobj->pgops->pgo_get)(uobj, trunc_page(offset), pgs,
881 npages, 0, access_type, advice, gpflags);
882 UVMHIST_LOG(ubchist, "alloc_direct getpages %jd", error, 0, 0, 0);
883 if (error) {
884 if (error == EAGAIN) {
885 kpause("ubc_alloc_directg", false, hz >> 2, NULL);
886 goto again;
887 }
888 return error;
889 }
890
891 rw_enter(uobj->vmobjlock, RW_WRITER);
892 for (int i = 0; i < *npages; i++) {
893 struct vm_page *pg = pgs[i];
894
895 KASSERT(pg != NULL);
896 KASSERT(pg != PGO_DONTCARE);
897 KASSERT((pg->flags & PG_FAKE) == 0 || (gpflags & PGO_OVERWRITE));
898 KASSERT(pg->uobject->vmobjlock == uobj->vmobjlock);
899
900 /* Avoid breaking loan if possible, only do it on write */
901 if ((flags & UBC_WRITE) && pg->loan_count != 0) {
902 pg = uvm_loanbreak(pg);
903 if (pg == NULL) {
904 uvm_page_unbusy(pgs, *npages);
905 rw_exit(uobj->vmobjlock);
906 uvm_wait("ubc_alloc_directl");
907 goto again;
908 }
909 pgs[i] = pg;
910 }
911
912 /* Page must be writable by now */
913 KASSERT((pg->flags & PG_RDONLY) == 0 || (flags & UBC_WRITE) == 0);
914
915 /*
916 * XXX For aobj pages. No managed mapping - mark the page
917 * dirty.
918 */
919 if ((flags & UBC_WRITE) != 0) {
920 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
921 }
922 }
923 rw_exit(uobj->vmobjlock);
924
925 return 0;
926 }
927
928 static void __noinline
929 ubc_direct_release(struct uvm_object *uobj,
930 int flags, struct vm_page **pgs, int npages)
931 {
932 rw_enter(uobj->vmobjlock, RW_WRITER);
933 for (int i = 0; i < npages; i++) {
934 struct vm_page *pg = pgs[i];
935
936 pg->flags &= ~PG_BUSY;
937 UVM_PAGE_OWN(pg, NULL);
938 if (pg->flags & PG_RELEASED) {
939 pg->flags &= ~PG_RELEASED;
940 uvm_pagefree(pg);
941 continue;
942 }
943
944 if (uvm_pagewanted_p(pg) || uvmpdpol_pageactivate_p(pg)) {
945 uvm_pagelock(pg);
946 uvm_pageactivate(pg);
947 uvm_pagewakeup(pg);
948 uvm_pageunlock(pg);
949 }
950
951 /* Page was changed, no longer fake and neither clean. */
952 if (flags & UBC_WRITE) {
953 KASSERTMSG(uvm_pagegetdirty(pg) ==
954 UVM_PAGE_STATUS_DIRTY,
955 "page %p not dirty", pg);
956 pg->flags &= ~PG_FAKE;
957 }
958 }
959 rw_exit(uobj->vmobjlock);
960 }
961
962 static int
963 ubc_uiomove_process(void *win, size_t len, void *arg)
964 {
965 struct uio *uio = (struct uio *)arg;
966
967 return uiomove(win, len, uio);
968 }
969
970 static int
971 ubc_zerorange_process(void *win, size_t len, void *arg)
972 {
973 memset(win, 0, len);
974 return 0;
975 }
976
977 static int __noinline
978 ubc_uiomove_direct(struct uvm_object *uobj, struct uio *uio, vsize_t todo, int advice,
979 int flags)
980 {
981 const bool overwrite = (flags & UBC_FAULTBUSY) != 0;
982 voff_t off;
983 int error, npages;
984 struct vm_page *pgs[howmany(ubc_winsize, MIN_PAGE_SIZE)];
985
986 KASSERT(todo <= uio->uio_resid);
987 KASSERT(((flags & UBC_WRITE) != 0 && uio->uio_rw == UIO_WRITE) ||
988 ((flags & UBC_READ) != 0 && uio->uio_rw == UIO_READ));
989
990 off = uio->uio_offset;
991 error = 0;
992 while (todo > 0) {
993 vsize_t bytelen = todo;
994
995 error = ubc_alloc_direct(uobj, off, &bytelen, advice, flags,
996 pgs, &npages);
997 if (error != 0) {
998 /* can't do anything, failed to get the pages */
999 break;
1000 }
1001
1002 if (error == 0) {
1003 error = uvm_direct_process(pgs, npages, off, bytelen,
1004 ubc_uiomove_process, uio);
1005 }
1006
1007 if (overwrite) {
1008 voff_t endoff;
1009
1010 /*
1011 * if we haven't initialized the pages yet due to an
1012 * error above, do it now.
1013 */
1014 if (error != 0) {
1015 (void) uvm_direct_process(pgs, npages, off,
1016 bytelen, ubc_zerorange_process, NULL);
1017 }
1018
1019 off += bytelen;
1020 todo -= bytelen;
1021 endoff = off & (PAGE_SIZE - 1);
1022
1023 /*
1024 * zero out the remaining portion of the final page
1025 * (if any).
1026 */
1027 if (todo == 0 && endoff != 0) {
1028 vsize_t zlen = PAGE_SIZE - endoff;
1029 (void) uvm_direct_process(pgs + npages - 1, 1,
1030 off, zlen, ubc_zerorange_process, NULL);
1031 }
1032 } else {
1033 off += bytelen;
1034 todo -= bytelen;
1035 }
1036
1037 ubc_direct_release(uobj, flags, pgs, npages);
1038
1039 if (error != 0 && ISSET(flags, UBC_PARTIALOK)) {
1040 break;
1041 }
1042 }
1043
1044 return error;
1045 }
1046
1047 static void __noinline
1048 ubc_zerorange_direct(struct uvm_object *uobj, off_t off, size_t todo, int flags)
1049 {
1050 int error, npages;
1051 struct vm_page *pgs[howmany(ubc_winsize, MIN_PAGE_SIZE)];
1052
1053 flags |= UBC_WRITE;
1054
1055 error = 0;
1056 while (todo > 0) {
1057 vsize_t bytelen = todo;
1058
1059 error = ubc_alloc_direct(uobj, off, &bytelen, UVM_ADV_NORMAL,
1060 flags, pgs, &npages);
1061 if (error != 0) {
1062 /* can't do anything, failed to get the pages */
1063 break;
1064 }
1065
1066 error = uvm_direct_process(pgs, npages, off, bytelen,
1067 ubc_zerorange_process, NULL);
1068
1069 ubc_direct_release(uobj, flags, pgs, npages);
1070
1071 off += bytelen;
1072 todo -= bytelen;
1073 }
1074 }
1075
1076 #endif /* UBC_USE_PMAP_DIRECT */
1077
1078 /*
1079 * ubc_purge: disassociate ubc_map structures from an empty uvm_object.
1080 */
1081
1082 void
1083 ubc_purge(struct uvm_object *uobj)
1084 {
1085 struct ubc_map *umap;
1086 vaddr_t va;
1087
1088 KASSERT(uobj->uo_npages == 0);
1089
1090 /*
1091 * Safe to check without lock held, as ubc_alloc() removes
1092 * the mapping and list entry in the correct order.
1093 */
1094 if (__predict_true(LIST_EMPTY(&uobj->uo_ubc))) {
1095 return;
1096 }
1097 rw_enter(ubc_object.uobj.vmobjlock, RW_WRITER);
1098 while ((umap = LIST_FIRST(&uobj->uo_ubc)) != NULL) {
1099 KASSERT(umap->refcount == 0);
1100 for (va = 0; va < ubc_winsize; va += PAGE_SIZE) {
1101 KASSERT(!pmap_extract(pmap_kernel(),
1102 va + UBC_UMAP_ADDR(umap), NULL));
1103 }
1104 LIST_REMOVE(umap, list);
1105 LIST_REMOVE(umap, hash);
1106 umap->flags &= ~UMAP_MAPPING_CACHED;
1107 umap->uobj = NULL;
1108 }
1109 rw_exit(ubc_object.uobj.vmobjlock);
1110 }
1111
1112 static int
1113 ubchash_stats(struct hashstat_sysctl *hs, bool fill)
1114 {
1115 struct ubc_map *umap;
1116 uint64_t chain;
1117
1118 strlcpy(hs->hash_name, "ubchash", sizeof(hs->hash_name));
1119 strlcpy(hs->hash_desc, "ubc object hash", sizeof(hs->hash_desc));
1120 if (!fill)
1121 return 0;
1122
1123 hs->hash_size = ubc_object.hashmask + 1;
1124
1125 for (size_t i = 0; i < hs->hash_size; i++) {
1126 chain = 0;
1127 rw_enter(ubc_object.uobj.vmobjlock, RW_READER);
1128 LIST_FOREACH(umap, &ubc_object.hash[i], hash) {
1129 chain++;
1130 }
1131 rw_exit(ubc_object.uobj.vmobjlock);
1132 if (chain > 0) {
1133 hs->hash_used++;
1134 hs->hash_items += chain;
1135 if (chain > hs->hash_maxchain)
1136 hs->hash_maxchain = chain;
1137 }
1138 preempt_point();
1139 }
1140
1141 return 0;
1142 }
Cache object: 93f561df632b7b798df23277d37db71d
|