FreeBSD/Linux Kernel Cross Reference
sys/uvm/uvm_device.c
1 /* $NetBSD: uvm_device.c,v 1.48 2006/09/03 21:37:22 christos Exp $ */
2
3 /*
4 *
5 * Copyright (c) 1997 Charles D. Cranor and Washington University.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. All advertising materials mentioning features or use of this software
17 * must display the following acknowledgement:
18 * This product includes software developed by Charles D. Cranor and
19 * Washington University.
20 * 4. The name of the author may not be used to endorse or promote products
21 * derived from this software without specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 *
34 * from: Id: uvm_device.c,v 1.1.2.9 1998/02/06 05:11:47 chs Exp
35 */
36
37 /*
38 * uvm_device.c: the device pager.
39 */
40
41 #include <sys/cdefs.h>
42 __KERNEL_RCSID(0, "$NetBSD: uvm_device.c,v 1.48 2006/09/03 21:37:22 christos Exp $");
43
44 #include "opt_uvmhist.h"
45
46 #include <sys/param.h>
47 #include <sys/systm.h>
48 #include <sys/conf.h>
49 #include <sys/proc.h>
50 #include <sys/malloc.h>
51 #include <sys/vnode.h>
52
53 #include <uvm/uvm.h>
54 #include <uvm/uvm_device.h>
55
56 /*
57 * private global data structure
58 *
59 * we keep a list of active device objects in the system.
60 */
61
62 LIST_HEAD(udv_list_struct, uvm_device);
63 static struct udv_list_struct udv_list;
64 static struct simplelock udv_lock;
65
66 /*
67 * functions
68 */
69
70 static void udv_init(void);
71 static void udv_reference(struct uvm_object *);
72 static void udv_detach(struct uvm_object *);
73 static int udv_fault(struct uvm_faultinfo *, vaddr_t,
74 struct vm_page **, int, int, vm_prot_t,
75 int);
76
77 /*
78 * master pager structure
79 */
80
81 struct uvm_pagerops uvm_deviceops = {
82 .pgo_init = udv_init,
83 .pgo_reference = udv_reference,
84 .pgo_detach = udv_detach,
85 .pgo_fault = udv_fault,
86 };
87
88 /*
89 * the ops!
90 */
91
92 /*
93 * udv_init
94 *
95 * init pager private data structures.
96 */
97
98 static void
99 udv_init(void)
100 {
101 LIST_INIT(&udv_list);
102 simple_lock_init(&udv_lock);
103 }
104
105 /*
106 * udv_attach
107 *
108 * get a VM object that is associated with a device. allocate a new
109 * one if needed.
110 *
111 * => caller must _not_ already be holding the lock on the uvm_object.
112 * => in fact, nothing should be locked so that we can sleep here.
113 */
114
115 struct uvm_object *
116 udv_attach(void *arg, vm_prot_t accessprot,
117 voff_t off, /* used only for access check */
118 vsize_t size /* used only for access check */)
119 {
120 dev_t device = *((dev_t *)arg);
121 struct uvm_device *udv, *lcv;
122 const struct cdevsw *cdev;
123 dev_type_mmap((*mapfn));
124
125 UVMHIST_FUNC("udv_attach"); UVMHIST_CALLED(maphist);
126
127 UVMHIST_LOG(maphist, "(device=0x%x)", device,0,0,0);
128
129 /*
130 * before we do anything, ensure this device supports mmap
131 */
132
133 cdev = cdevsw_lookup(device);
134 if (cdev == NULL)
135 return (NULL);
136 mapfn = cdev->d_mmap;
137 if (mapfn == NULL || mapfn == nommap || mapfn == nullmmap)
138 return(NULL);
139
140 /*
141 * Negative offsets on the object are not allowed.
142 */
143
144 if (off < 0)
145 return(NULL);
146
147 /*
148 * Check that the specified range of the device allows the
149 * desired protection.
150 *
151 * XXX assumes VM_PROT_* == PROT_*
152 * XXX clobbers off and size, but nothing else here needs them.
153 */
154
155 while (size != 0) {
156 if ((*mapfn)(device, off, accessprot) == -1)
157 return (NULL);
158 off += PAGE_SIZE; size -= PAGE_SIZE;
159 }
160
161 /*
162 * keep looping until we get it
163 */
164
165 for (;;) {
166
167 /*
168 * first, attempt to find it on the main list
169 */
170
171 simple_lock(&udv_lock);
172 LIST_FOREACH(lcv, &udv_list, u_list) {
173 if (device == lcv->u_device)
174 break;
175 }
176
177 /*
178 * got it on main list. put a hold on it and unlock udv_lock.
179 */
180
181 if (lcv) {
182
183 /*
184 * if someone else has a hold on it, sleep and start
185 * over again.
186 */
187
188 if (lcv->u_flags & UVM_DEVICE_HOLD) {
189 lcv->u_flags |= UVM_DEVICE_WANTED;
190 UVM_UNLOCK_AND_WAIT(lcv, &udv_lock, FALSE,
191 "udv_attach",0);
192 continue;
193 }
194
195 /* we are now holding it */
196 lcv->u_flags |= UVM_DEVICE_HOLD;
197 simple_unlock(&udv_lock);
198
199 /*
200 * bump reference count, unhold, return.
201 */
202
203 simple_lock(&lcv->u_obj.vmobjlock);
204 lcv->u_obj.uo_refs++;
205 simple_unlock(&lcv->u_obj.vmobjlock);
206
207 simple_lock(&udv_lock);
208 if (lcv->u_flags & UVM_DEVICE_WANTED)
209 wakeup(lcv);
210 lcv->u_flags &= ~(UVM_DEVICE_WANTED|UVM_DEVICE_HOLD);
211 simple_unlock(&udv_lock);
212 return(&lcv->u_obj);
213 }
214
215 /*
216 * did not find it on main list. need to malloc a new one.
217 */
218
219 simple_unlock(&udv_lock);
220 /* NOTE: we could sleep in the following malloc() */
221 MALLOC(udv, struct uvm_device *, sizeof(*udv), M_TEMP,
222 M_WAITOK);
223 simple_lock(&udv_lock);
224
225 /*
226 * now we have to double check to make sure no one added it
227 * to the list while we were sleeping...
228 */
229
230 LIST_FOREACH(lcv, &udv_list, u_list) {
231 if (device == lcv->u_device)
232 break;
233 }
234
235 /*
236 * did we lose a race to someone else?
237 * free our memory and retry.
238 */
239
240 if (lcv) {
241 simple_unlock(&udv_lock);
242 FREE(udv, M_TEMP);
243 continue;
244 }
245
246 /*
247 * we have it! init the data structures, add to list
248 * and return.
249 */
250
251 UVM_OBJ_INIT(&udv->u_obj, &uvm_deviceops, 1);
252 udv->u_flags = 0;
253 udv->u_device = device;
254 LIST_INSERT_HEAD(&udv_list, udv, u_list);
255 simple_unlock(&udv_lock);
256 return(&udv->u_obj);
257 }
258 /*NOTREACHED*/
259 }
260
261 /*
262 * udv_reference
263 *
264 * add a reference to a VM object. Note that the reference count must
265 * already be one (the passed in reference) so there is no chance of the
266 * udv being released or locked out here.
267 *
268 * => caller must call with object unlocked.
269 */
270
271 static void
272 udv_reference(struct uvm_object *uobj)
273 {
274 UVMHIST_FUNC("udv_reference"); UVMHIST_CALLED(maphist);
275
276 simple_lock(&uobj->vmobjlock);
277 uobj->uo_refs++;
278 UVMHIST_LOG(maphist, "<- done (uobj=0x%x, ref = %d)",
279 uobj, uobj->uo_refs,0,0);
280 simple_unlock(&uobj->vmobjlock);
281 }
282
283 /*
284 * udv_detach
285 *
286 * remove a reference to a VM object.
287 *
288 * => caller must call with object unlocked and map locked.
289 */
290
291 static void
292 udv_detach(struct uvm_object *uobj)
293 {
294 struct uvm_device *udv = (struct uvm_device *)uobj;
295 UVMHIST_FUNC("udv_detach"); UVMHIST_CALLED(maphist);
296
297 /*
298 * loop until done
299 */
300 again:
301 simple_lock(&uobj->vmobjlock);
302 if (uobj->uo_refs > 1) {
303 uobj->uo_refs--;
304 simple_unlock(&uobj->vmobjlock);
305 UVMHIST_LOG(maphist," <- done, uobj=0x%x, ref=%d",
306 uobj,uobj->uo_refs,0,0);
307 return;
308 }
309
310 /*
311 * is it being held? if so, wait until others are done.
312 */
313
314 simple_lock(&udv_lock);
315 if (udv->u_flags & UVM_DEVICE_HOLD) {
316 udv->u_flags |= UVM_DEVICE_WANTED;
317 simple_unlock(&uobj->vmobjlock);
318 UVM_UNLOCK_AND_WAIT(udv, &udv_lock, FALSE, "udv_detach",0);
319 goto again;
320 }
321
322 /*
323 * got it! nuke it now.
324 */
325
326 LIST_REMOVE(udv, u_list);
327 if (udv->u_flags & UVM_DEVICE_WANTED)
328 wakeup(udv);
329 simple_unlock(&udv_lock);
330 simple_unlock(&uobj->vmobjlock);
331 FREE(udv, M_TEMP);
332 UVMHIST_LOG(maphist," <- done, freed uobj=0x%x", uobj,0,0,0);
333 }
334
335 /*
336 * udv_fault: non-standard fault routine for device "pages"
337 *
338 * => rather than having a "get" function, we have a fault routine
339 * since we don't return vm_pages we need full control over the
340 * pmap_enter map in
341 * => all the usual fault data structured are locked by the caller
342 * (i.e. maps(read), amap (if any), uobj)
343 * => on return, we unlock all fault data structures
344 * => flags: PGO_ALLPAGES: get all of the pages
345 * PGO_LOCKED: fault data structures are locked
346 * XXX: currently PGO_LOCKED is always required ... consider removing
347 * it as a flag
348 * => NOTE: vaddr is the VA of pps[0] in ufi->entry, _NOT_ pps[centeridx]
349 */
350
351 static int
352 udv_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, struct vm_page **pps,
353 int npages, int centeridx, vm_prot_t access_type,
354 int flags)
355 {
356 struct vm_map_entry *entry = ufi->entry;
357 struct uvm_object *uobj = entry->object.uvm_obj;
358 struct uvm_device *udv = (struct uvm_device *)uobj;
359 const struct cdevsw *cdev;
360 vaddr_t curr_va;
361 off_t curr_offset;
362 paddr_t paddr, mdpgno;
363 int lcv, retval;
364 dev_t device;
365 paddr_t (*mapfn)(dev_t, off_t, int);
366 vm_prot_t mapprot;
367 UVMHIST_FUNC("udv_fault"); UVMHIST_CALLED(maphist);
368 UVMHIST_LOG(maphist," flags=%d", flags,0,0,0);
369
370 /*
371 * we do not allow device mappings to be mapped copy-on-write
372 * so we kill any attempt to do so here.
373 */
374
375 if (UVM_ET_ISCOPYONWRITE(entry)) {
376 UVMHIST_LOG(maphist, "<- failed -- COW entry (etype=0x%x)",
377 entry->etype, 0,0,0);
378 uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj, NULL);
379 return(EIO);
380 }
381
382 /*
383 * get device map function.
384 */
385
386 device = udv->u_device;
387 cdev = cdevsw_lookup(device);
388 if (cdev == NULL) {
389 uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj, NULL);
390 return (EIO);
391 }
392 mapfn = cdev->d_mmap;
393
394 /*
395 * now we must determine the offset in udv to use and the VA to
396 * use for pmap_enter. note that we always use orig_map's pmap
397 * for pmap_enter (even if we have a submap). since virtual
398 * addresses in a submap must match the main map, this is ok.
399 */
400
401 /* udv offset = (offset from start of entry) + entry's offset */
402 curr_offset = entry->offset + (vaddr - entry->start);
403 /* pmap va = vaddr (virtual address of pps[0]) */
404 curr_va = vaddr;
405
406 /*
407 * loop over the page range entering in as needed
408 */
409
410 retval = 0;
411 for (lcv = 0 ; lcv < npages ; lcv++, curr_offset += PAGE_SIZE,
412 curr_va += PAGE_SIZE) {
413 if ((flags & PGO_ALLPAGES) == 0 && lcv != centeridx)
414 continue;
415
416 if (pps[lcv] == PGO_DONTCARE)
417 continue;
418
419 mdpgno = (*mapfn)(device, curr_offset, access_type);
420 if (mdpgno == -1) {
421 retval = EIO;
422 break;
423 }
424 paddr = pmap_phys_address(mdpgno);
425 mapprot = ufi->entry->protection;
426 UVMHIST_LOG(maphist,
427 " MAPPING: device: pm=0x%x, va=0x%x, pa=0x%lx, at=%d",
428 ufi->orig_map->pmap, curr_va, paddr, mapprot);
429 if (pmap_enter(ufi->orig_map->pmap, curr_va, paddr,
430 mapprot, PMAP_CANFAIL | mapprot) != 0) {
431 /*
432 * pmap_enter() didn't have the resource to
433 * enter this mapping. Unlock everything,
434 * wait for the pagedaemon to free up some
435 * pages, and then tell uvm_fault() to start
436 * the fault again.
437 *
438 * XXX Needs some rethinking for the PGO_ALLPAGES
439 * XXX case.
440 */
441 uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap,
442 uobj, NULL);
443 pmap_update(ufi->orig_map->pmap); /* sync what we have so far */
444 uvm_wait("udv_fault");
445 return (ERESTART);
446 }
447 }
448
449 uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj, NULL);
450 pmap_update(ufi->orig_map->pmap);
451 return (retval);
452 }
Cache object: 58a4dfc3d9cb7116e91fb99c39e83ac0
|