FreeBSD/Linux Kernel Cross Reference
sys/xen/gnttab.c
1 /******************************************************************************
2 * gnttab.c
3 *
4 * Two sets of functionality:
5 * 1. Granting foreign access to our memory reservation.
6 * 2. Accessing others' memory reservations via grant references.
7 * (i.e., mechanisms for both sender and recipient of grant references)
8 *
9 * Copyright (c) 2005, Christopher Clark
10 * Copyright (c) 2004, K A Fraser
11 */
12
13 #include <sys/cdefs.h>
14 __FBSDID("$FreeBSD: releng/8.0/sys/xen/gnttab.c 189699 2009-03-11 15:30:12Z dfr $");
15
16 #include "opt_global.h"
17 #include "opt_pmap.h"
18
19 #include <sys/param.h>
20 #include <sys/systm.h>
21 #include <sys/bus.h>
22 #include <sys/conf.h>
23 #include <sys/module.h>
24 #include <sys/kernel.h>
25 #include <sys/lock.h>
26 #include <sys/malloc.h>
27 #include <sys/mman.h>
28
29 #include <machine/xen/xen-os.h>
30 #include <xen/hypervisor.h>
31 #include <machine/xen/synch_bitops.h>
32
33 #include <xen/hypervisor.h>
34 #include <xen/gnttab.h>
35
36 #include <vm/vm.h>
37 #include <vm/vm_kern.h>
38 #include <vm/vm_extern.h>
39 #include <vm/pmap.h>
40
41 #define cmpxchg(a, b, c) atomic_cmpset_int((volatile u_int *)(a),(b),(c))
42
43 /* External tools reserve first few grant table entries. */
44 #define NR_RESERVED_ENTRIES 8
45 #define GNTTAB_LIST_END 0xffffffff
46 #define GREFS_PER_GRANT_FRAME (PAGE_SIZE / sizeof(grant_entry_t))
47
48 static grant_ref_t **gnttab_list;
49 static unsigned int nr_grant_frames;
50 static unsigned int boot_max_nr_grant_frames;
51 static int gnttab_free_count;
52 static grant_ref_t gnttab_free_head;
53 static struct mtx gnttab_list_lock;
54
55 static grant_entry_t *shared;
56
57 static struct gnttab_free_callback *gnttab_free_callback_list = NULL;
58
59 static int gnttab_expand(unsigned int req_entries);
60
61 #define RPP (PAGE_SIZE / sizeof(grant_ref_t))
62 #define gnttab_entry(entry) (gnttab_list[(entry) / RPP][(entry) % RPP])
63
64 static int
65 get_free_entries(int count, int *entries)
66 {
67 int ref, error;
68 grant_ref_t head;
69
70 mtx_lock(&gnttab_list_lock);
71 if ((gnttab_free_count < count) &&
72 ((error = gnttab_expand(count - gnttab_free_count)) != 0)) {
73 mtx_unlock(&gnttab_list_lock);
74 return (error);
75 }
76 ref = head = gnttab_free_head;
77 gnttab_free_count -= count;
78 while (count-- > 1)
79 head = gnttab_entry(head);
80 gnttab_free_head = gnttab_entry(head);
81 gnttab_entry(head) = GNTTAB_LIST_END;
82 mtx_unlock(&gnttab_list_lock);
83
84 *entries = ref;
85 return (0);
86 }
87
88 static void
89 do_free_callbacks(void)
90 {
91 struct gnttab_free_callback *callback, *next;
92
93 callback = gnttab_free_callback_list;
94 gnttab_free_callback_list = NULL;
95
96 while (callback != NULL) {
97 next = callback->next;
98 if (gnttab_free_count >= callback->count) {
99 callback->next = NULL;
100 callback->fn(callback->arg);
101 } else {
102 callback->next = gnttab_free_callback_list;
103 gnttab_free_callback_list = callback;
104 }
105 callback = next;
106 }
107 }
108
109 static inline void
110 check_free_callbacks(void)
111 {
112 if (unlikely(gnttab_free_callback_list != NULL))
113 do_free_callbacks();
114 }
115
116 static void
117 put_free_entry(grant_ref_t ref)
118 {
119
120 mtx_lock(&gnttab_list_lock);
121 gnttab_entry(ref) = gnttab_free_head;
122 gnttab_free_head = ref;
123 gnttab_free_count++;
124 check_free_callbacks();
125 mtx_unlock(&gnttab_list_lock);
126 }
127
128 /*
129 * Public grant-issuing interface functions
130 */
131
132 int
133 gnttab_grant_foreign_access(domid_t domid, unsigned long frame, int readonly,
134 grant_ref_t *result)
135 {
136 int error, ref;
137
138 error = get_free_entries(1, &ref);
139
140 if (unlikely(error))
141 return (error);
142
143 shared[ref].frame = frame;
144 shared[ref].domid = domid;
145 wmb();
146 shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0);
147
148 if (result)
149 *result = ref;
150
151 return (0);
152 }
153
154 void
155 gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
156 unsigned long frame, int readonly)
157 {
158
159 shared[ref].frame = frame;
160 shared[ref].domid = domid;
161 wmb();
162 shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0);
163 }
164
165 int
166 gnttab_query_foreign_access(grant_ref_t ref)
167 {
168 uint16_t nflags;
169
170 nflags = shared[ref].flags;
171
172 return (nflags & (GTF_reading|GTF_writing));
173 }
174
175 int
176 gnttab_end_foreign_access_ref(grant_ref_t ref)
177 {
178 uint16_t flags, nflags;
179
180 nflags = shared[ref].flags;
181 do {
182 if ( (flags = nflags) & (GTF_reading|GTF_writing) ) {
183 printf("WARNING: g.e. still in use!\n");
184 return (0);
185 }
186 } while ((nflags = synch_cmpxchg(&shared[ref].flags, flags, 0)) !=
187 flags);
188
189 return (1);
190 }
191
192 void
193 gnttab_end_foreign_access(grant_ref_t ref, void *page)
194 {
195 if (gnttab_end_foreign_access_ref(ref)) {
196 put_free_entry(ref);
197 if (page != NULL) {
198 free(page, M_DEVBUF);
199 }
200 }
201 else {
202 /* XXX This needs to be fixed so that the ref and page are
203 placed on a list to be freed up later. */
204 printf("WARNING: leaking g.e. and page still in use!\n");
205 }
206 }
207
208 int
209 gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn,
210 grant_ref_t *result)
211 {
212 int error, ref;
213
214 error = get_free_entries(1, &ref);
215 if (unlikely(error))
216 return (error);
217
218 gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
219
220 *result = ref;
221 return (0);
222 }
223
224 void
225 gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
226 unsigned long pfn)
227 {
228 shared[ref].frame = pfn;
229 shared[ref].domid = domid;
230 wmb();
231 shared[ref].flags = GTF_accept_transfer;
232 }
233
234 unsigned long
235 gnttab_end_foreign_transfer_ref(grant_ref_t ref)
236 {
237 unsigned long frame;
238 uint16_t flags;
239
240 /*
241 * If a transfer is not even yet started, try to reclaim the grant
242 * reference and return failure (== 0).
243 */
244 while (!((flags = shared[ref].flags) & GTF_transfer_committed)) {
245 if ( synch_cmpxchg(&shared[ref].flags, flags, 0) == flags )
246 return (0);
247 cpu_relax();
248 }
249
250 /* If a transfer is in progress then wait until it is completed. */
251 while (!(flags & GTF_transfer_completed)) {
252 flags = shared[ref].flags;
253 cpu_relax();
254 }
255
256 /* Read the frame number /after/ reading completion status. */
257 rmb();
258 frame = shared[ref].frame;
259 KASSERT(frame != 0, ("grant table inconsistent"));
260
261 return (frame);
262 }
263
264 unsigned long
265 gnttab_end_foreign_transfer(grant_ref_t ref)
266 {
267 unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
268
269 put_free_entry(ref);
270 return (frame);
271 }
272
273 void
274 gnttab_free_grant_reference(grant_ref_t ref)
275 {
276
277 put_free_entry(ref);
278 }
279
280 void
281 gnttab_free_grant_references(grant_ref_t head)
282 {
283 grant_ref_t ref;
284 int count = 1;
285
286 if (head == GNTTAB_LIST_END)
287 return;
288
289 mtx_lock(&gnttab_list_lock);
290 ref = head;
291 while (gnttab_entry(ref) != GNTTAB_LIST_END) {
292 ref = gnttab_entry(ref);
293 count++;
294 }
295 gnttab_entry(ref) = gnttab_free_head;
296 gnttab_free_head = head;
297 gnttab_free_count += count;
298 check_free_callbacks();
299 mtx_unlock(&gnttab_list_lock);
300 }
301
302 int
303 gnttab_alloc_grant_references(uint16_t count, grant_ref_t *head)
304 {
305 int ref, error;
306
307 error = get_free_entries(count, &ref);
308 if (unlikely(error))
309 return (error);
310
311 *head = ref;
312 return (0);
313 }
314
315 int
316 gnttab_empty_grant_references(const grant_ref_t *private_head)
317 {
318
319 return (*private_head == GNTTAB_LIST_END);
320 }
321
322 int
323 gnttab_claim_grant_reference(grant_ref_t *private_head)
324 {
325 grant_ref_t g = *private_head;
326
327 if (unlikely(g == GNTTAB_LIST_END))
328 return (ENOSPC);
329 *private_head = gnttab_entry(g);
330 return (g);
331 }
332
333 void
334 gnttab_release_grant_reference(grant_ref_t *private_head, grant_ref_t release)
335 {
336
337 gnttab_entry(release) = *private_head;
338 *private_head = release;
339 }
340
341 void
342 gnttab_request_free_callback(struct gnttab_free_callback *callback,
343 void (*fn)(void *), void *arg, uint16_t count)
344 {
345
346 mtx_lock(&gnttab_list_lock);
347 if (callback->next)
348 goto out;
349 callback->fn = fn;
350 callback->arg = arg;
351 callback->count = count;
352 callback->next = gnttab_free_callback_list;
353 gnttab_free_callback_list = callback;
354 check_free_callbacks();
355 out:
356 mtx_unlock(&gnttab_list_lock);
357
358 }
359
360 void
361 gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
362 {
363 struct gnttab_free_callback **pcb;
364
365 mtx_lock(&gnttab_list_lock);
366 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
367 if (*pcb == callback) {
368 *pcb = callback->next;
369 break;
370 }
371 }
372 mtx_unlock(&gnttab_list_lock);
373 }
374
375
376 static int
377 grow_gnttab_list(unsigned int more_frames)
378 {
379 unsigned int new_nr_grant_frames, extra_entries, i;
380
381 new_nr_grant_frames = nr_grant_frames + more_frames;
382 extra_entries = more_frames * GREFS_PER_GRANT_FRAME;
383
384 for (i = nr_grant_frames; i < new_nr_grant_frames; i++)
385 {
386 gnttab_list[i] = (grant_ref_t *)
387 malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT);
388
389 if (!gnttab_list[i])
390 goto grow_nomem;
391 }
392
393 for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames;
394 i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
395 gnttab_entry(i) = i + 1;
396
397 gnttab_entry(i) = gnttab_free_head;
398 gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames;
399 gnttab_free_count += extra_entries;
400
401 nr_grant_frames = new_nr_grant_frames;
402
403 check_free_callbacks();
404
405 return (0);
406
407 grow_nomem:
408 for ( ; i >= nr_grant_frames; i--)
409 free(gnttab_list[i], M_DEVBUF);
410 return (ENOMEM);
411 }
412
413 static unsigned int
414 __max_nr_grant_frames(void)
415 {
416 struct gnttab_query_size query;
417 int rc;
418
419 query.dom = DOMID_SELF;
420
421 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
422 if ((rc < 0) || (query.status != GNTST_okay))
423 return (4); /* Legacy max supported number of frames */
424
425 return (query.max_nr_frames);
426 }
427
428 static inline
429 unsigned int max_nr_grant_frames(void)
430 {
431 unsigned int xen_max = __max_nr_grant_frames();
432
433 if (xen_max > boot_max_nr_grant_frames)
434 return (boot_max_nr_grant_frames);
435 return (xen_max);
436 }
437
438 #ifdef notyet
439 /*
440 * XXX needed for backend support
441 *
442 */
443 static int
444 map_pte_fn(pte_t *pte, struct page *pmd_page,
445 unsigned long addr, void *data)
446 {
447 unsigned long **frames = (unsigned long **)data;
448
449 set_pte_at(&init_mm, addr, pte, pfn_pte_ma((*frames)[0], PAGE_KERNEL));
450 (*frames)++;
451 return 0;
452 }
453
454 static int
455 unmap_pte_fn(pte_t *pte, struct page *pmd_page,
456 unsigned long addr, void *data)
457 {
458
459 set_pte_at(&init_mm, addr, pte, __pte(0));
460 return 0;
461 }
462 #endif
463
464 #ifndef XENHVM
465
466 static int
467 gnttab_map(unsigned int start_idx, unsigned int end_idx)
468 {
469 struct gnttab_setup_table setup;
470 u_long *frames;
471
472 unsigned int nr_gframes = end_idx + 1;
473 int i, rc;
474
475 frames = malloc(nr_gframes * sizeof(unsigned long), M_DEVBUF, M_NOWAIT);
476 if (!frames)
477 return (ENOMEM);
478
479 setup.dom = DOMID_SELF;
480 setup.nr_frames = nr_gframes;
481 set_xen_guest_handle(setup.frame_list, frames);
482
483 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
484 if (rc == -ENOSYS) {
485 free(frames, M_DEVBUF);
486 return (ENOSYS);
487 }
488 KASSERT(!(rc || setup.status),
489 ("unexpected result from grant_table_op"));
490
491 if (shared == NULL) {
492 vm_offset_t area;
493
494 area = kmem_alloc_nofault(kernel_map,
495 PAGE_SIZE * max_nr_grant_frames());
496 KASSERT(area, ("can't allocate VM space for grant table"));
497 shared = (grant_entry_t *)area;
498 }
499
500 for (i = 0; i < nr_gframes; i++)
501 PT_SET_MA(((caddr_t)shared) + i*PAGE_SIZE,
502 ((vm_paddr_t)frames[i]) << PAGE_SHIFT | PG_RW | PG_V);
503
504 free(frames, M_DEVBUF);
505
506 return (0);
507 }
508
509 int
510 gnttab_resume(void)
511 {
512
513 if (max_nr_grant_frames() < nr_grant_frames)
514 return (ENOSYS);
515 return (gnttab_map(0, nr_grant_frames - 1));
516 }
517
518 int
519 gnttab_suspend(void)
520 {
521 int i;
522
523 for (i = 0; i < nr_grant_frames; i++)
524 pmap_kremove((vm_offset_t) shared + i * PAGE_SIZE);
525
526 return (0);
527 }
528
529 #else /* XENHVM */
530
531 #include <dev/xen/xenpci/xenpcivar.h>
532
533 static vm_paddr_t resume_frames;
534
535 static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
536 {
537 struct xen_add_to_physmap xatp;
538 unsigned int i = end_idx;
539
540 /*
541 * Loop backwards, so that the first hypercall has the largest index,
542 * ensuring that the table will grow only once.
543 */
544 do {
545 xatp.domid = DOMID_SELF;
546 xatp.idx = i;
547 xatp.space = XENMAPSPACE_grant_table;
548 xatp.gpfn = (resume_frames >> PAGE_SHIFT) + i;
549 if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp))
550 panic("HYPERVISOR_memory_op failed to map gnttab");
551 } while (i-- > start_idx);
552
553 if (shared == NULL) {
554 vm_offset_t area;
555
556 area = kmem_alloc_nofault(kernel_map,
557 PAGE_SIZE * max_nr_grant_frames());
558 KASSERT(area, ("can't allocate VM space for grant table"));
559 shared = (grant_entry_t *)area;
560 }
561
562 for (i = start_idx; i <= end_idx; i++) {
563 pmap_kenter((vm_offset_t) shared + i * PAGE_SIZE,
564 resume_frames + i * PAGE_SIZE);
565 }
566
567 return (0);
568 }
569
570 int
571 gnttab_resume(void)
572 {
573 int error;
574 unsigned int max_nr_gframes, nr_gframes;
575
576 nr_gframes = nr_grant_frames;
577 max_nr_gframes = max_nr_grant_frames();
578 if (max_nr_gframes < nr_gframes)
579 return (ENOSYS);
580
581 if (!resume_frames) {
582 error = xenpci_alloc_space(PAGE_SIZE * max_nr_gframes,
583 &resume_frames);
584 if (error) {
585 printf("error mapping gnttab share frames\n");
586 return (error);
587 }
588 }
589
590 return (gnttab_map(0, nr_gframes - 1));
591 }
592
593 #endif
594
595 static int
596 gnttab_expand(unsigned int req_entries)
597 {
598 int error;
599 unsigned int cur, extra;
600
601 cur = nr_grant_frames;
602 extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) /
603 GREFS_PER_GRANT_FRAME);
604 if (cur + extra > max_nr_grant_frames())
605 return (ENOSPC);
606
607 error = gnttab_map(cur, cur + extra - 1);
608 if (!error)
609 error = grow_gnttab_list(extra);
610
611 return (error);
612 }
613
614 int
615 gnttab_init()
616 {
617 int i;
618 unsigned int max_nr_glist_frames;
619 unsigned int nr_init_grefs;
620
621 if (!is_running_on_xen())
622 return (ENODEV);
623
624 nr_grant_frames = 1;
625 boot_max_nr_grant_frames = __max_nr_grant_frames();
626
627 /* Determine the maximum number of frames required for the
628 * grant reference free list on the current hypervisor.
629 */
630 max_nr_glist_frames = (boot_max_nr_grant_frames *
631 GREFS_PER_GRANT_FRAME /
632 (PAGE_SIZE / sizeof(grant_ref_t)));
633
634 gnttab_list = malloc(max_nr_glist_frames * sizeof(grant_ref_t *),
635 M_DEVBUF, M_NOWAIT);
636
637 if (gnttab_list == NULL)
638 return (ENOMEM);
639
640 for (i = 0; i < nr_grant_frames; i++) {
641 gnttab_list[i] = (grant_ref_t *)
642 malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT);
643 if (gnttab_list[i] == NULL)
644 goto ini_nomem;
645 }
646
647 if (gnttab_resume())
648 return (ENODEV);
649
650 nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME;
651
652 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
653 gnttab_entry(i) = i + 1;
654
655 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
656 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
657 gnttab_free_head = NR_RESERVED_ENTRIES;
658
659 if (bootverbose)
660 printf("Grant table initialized\n");
661
662 return (0);
663
664 ini_nomem:
665 for (i--; i >= 0; i--)
666 free(gnttab_list[i], M_DEVBUF);
667 free(gnttab_list, M_DEVBUF);
668 return (ENOMEM);
669
670 }
671
672 MTX_SYSINIT(gnttab, &gnttab_list_lock, "GNTTAB LOCK", MTX_DEF);
673 //SYSINIT(gnttab, SI_SUB_PSEUDO, SI_ORDER_FIRST, gnttab_init, NULL);
Cache object: dfae4f4fbab39db4209f925fe835879e
|