FreeBSD/Linux Kernel Cross Reference
sys/xen/gnttab.c
1 /******************************************************************************
2 * gnttab.c
3 *
4 * Two sets of functionality:
5 * 1. Granting foreign access to our memory reservation.
6 * 2. Accessing others' memory reservations via grant references.
7 * (i.e., mechanisms for both sender and recipient of grant references)
8 *
9 * Copyright (c) 2005, Christopher Clark
10 * Copyright (c) 2004, K A Fraser
11 */
12
13 #include <sys/cdefs.h>
14 __FBSDID("$FreeBSD: releng/10.0/sys/xen/gnttab.c 255040 2013-08-29 19:52:18Z gibbs $");
15
16 #include "opt_global.h"
17 #include "opt_pmap.h"
18
19 #include <sys/param.h>
20 #include <sys/systm.h>
21 #include <sys/bus.h>
22 #include <sys/conf.h>
23 #include <sys/module.h>
24 #include <sys/kernel.h>
25 #include <sys/lock.h>
26 #include <sys/malloc.h>
27 #include <sys/mman.h>
28
29 #include <xen/xen-os.h>
30 #include <xen/hypervisor.h>
31 #include <machine/xen/synch_bitops.h>
32
33 #include <xen/hypervisor.h>
34 #include <xen/gnttab.h>
35
36 #include <vm/vm.h>
37 #include <vm/vm_kern.h>
38 #include <vm/vm_extern.h>
39 #include <vm/pmap.h>
40
41 #define cmpxchg(a, b, c) atomic_cmpset_int((volatile u_int *)(a),(b),(c))
42
43 /* External tools reserve first few grant table entries. */
44 #define NR_RESERVED_ENTRIES 8
45 #define GREFS_PER_GRANT_FRAME (PAGE_SIZE / sizeof(grant_entry_t))
46
47 static grant_ref_t **gnttab_list;
48 static unsigned int nr_grant_frames;
49 static unsigned int boot_max_nr_grant_frames;
50 static int gnttab_free_count;
51 static grant_ref_t gnttab_free_head;
52 static struct mtx gnttab_list_lock;
53
54 static grant_entry_t *shared;
55
56 static struct gnttab_free_callback *gnttab_free_callback_list = NULL;
57
58 static int gnttab_expand(unsigned int req_entries);
59
60 #define RPP (PAGE_SIZE / sizeof(grant_ref_t))
61 #define gnttab_entry(entry) (gnttab_list[(entry) / RPP][(entry) % RPP])
62
63 static int
64 get_free_entries(int count, int *entries)
65 {
66 int ref, error;
67 grant_ref_t head;
68
69 mtx_lock(&gnttab_list_lock);
70 if ((gnttab_free_count < count) &&
71 ((error = gnttab_expand(count - gnttab_free_count)) != 0)) {
72 mtx_unlock(&gnttab_list_lock);
73 return (error);
74 }
75 ref = head = gnttab_free_head;
76 gnttab_free_count -= count;
77 while (count-- > 1)
78 head = gnttab_entry(head);
79 gnttab_free_head = gnttab_entry(head);
80 gnttab_entry(head) = GNTTAB_LIST_END;
81 mtx_unlock(&gnttab_list_lock);
82
83 *entries = ref;
84 return (0);
85 }
86
87 static void
88 do_free_callbacks(void)
89 {
90 struct gnttab_free_callback *callback, *next;
91
92 callback = gnttab_free_callback_list;
93 gnttab_free_callback_list = NULL;
94
95 while (callback != NULL) {
96 next = callback->next;
97 if (gnttab_free_count >= callback->count) {
98 callback->next = NULL;
99 callback->fn(callback->arg);
100 } else {
101 callback->next = gnttab_free_callback_list;
102 gnttab_free_callback_list = callback;
103 }
104 callback = next;
105 }
106 }
107
108 static inline void
109 check_free_callbacks(void)
110 {
111 if (__predict_false(gnttab_free_callback_list != NULL))
112 do_free_callbacks();
113 }
114
115 static void
116 put_free_entry(grant_ref_t ref)
117 {
118
119 mtx_lock(&gnttab_list_lock);
120 gnttab_entry(ref) = gnttab_free_head;
121 gnttab_free_head = ref;
122 gnttab_free_count++;
123 check_free_callbacks();
124 mtx_unlock(&gnttab_list_lock);
125 }
126
127 /*
128 * Public grant-issuing interface functions
129 */
130
131 int
132 gnttab_grant_foreign_access(domid_t domid, unsigned long frame, int readonly,
133 grant_ref_t *result)
134 {
135 int error, ref;
136
137 error = get_free_entries(1, &ref);
138
139 if (__predict_false(error))
140 return (error);
141
142 shared[ref].frame = frame;
143 shared[ref].domid = domid;
144 wmb();
145 shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0);
146
147 if (result)
148 *result = ref;
149
150 return (0);
151 }
152
153 void
154 gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
155 unsigned long frame, int readonly)
156 {
157
158 shared[ref].frame = frame;
159 shared[ref].domid = domid;
160 wmb();
161 shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0);
162 }
163
164 int
165 gnttab_query_foreign_access(grant_ref_t ref)
166 {
167 uint16_t nflags;
168
169 nflags = shared[ref].flags;
170
171 return (nflags & (GTF_reading|GTF_writing));
172 }
173
174 int
175 gnttab_end_foreign_access_ref(grant_ref_t ref)
176 {
177 uint16_t flags, nflags;
178
179 nflags = shared[ref].flags;
180 do {
181 if ( (flags = nflags) & (GTF_reading|GTF_writing) ) {
182 printf("%s: WARNING: g.e. still in use!\n", __func__);
183 return (0);
184 }
185 } while ((nflags = synch_cmpxchg(&shared[ref].flags, flags, 0)) !=
186 flags);
187
188 return (1);
189 }
190
191 void
192 gnttab_end_foreign_access(grant_ref_t ref, void *page)
193 {
194 if (gnttab_end_foreign_access_ref(ref)) {
195 put_free_entry(ref);
196 if (page != NULL) {
197 free(page, M_DEVBUF);
198 }
199 }
200 else {
201 /* XXX This needs to be fixed so that the ref and page are
202 placed on a list to be freed up later. */
203 printf("%s: WARNING: leaking g.e. and page still in use!\n",
204 __func__);
205 }
206 }
207
208 void
209 gnttab_end_foreign_access_references(u_int count, grant_ref_t *refs)
210 {
211 grant_ref_t *last_ref;
212 grant_ref_t head;
213 grant_ref_t tail;
214
215 head = GNTTAB_LIST_END;
216 tail = *refs;
217 last_ref = refs + count;
218 while (refs != last_ref) {
219
220 if (gnttab_end_foreign_access_ref(*refs)) {
221 gnttab_entry(*refs) = head;
222 head = *refs;
223 } else {
224 /*
225 * XXX This needs to be fixed so that the ref
226 * is placed on a list to be freed up later.
227 */
228 printf("%s: WARNING: leaking g.e. still in use!\n",
229 __func__);
230 count--;
231 }
232 refs++;
233 }
234
235 if (count != 0) {
236 mtx_lock(&gnttab_list_lock);
237 gnttab_free_count += count;
238 gnttab_entry(tail) = gnttab_free_head;
239 gnttab_free_head = head;
240 mtx_unlock(&gnttab_list_lock);
241 }
242 }
243
244 int
245 gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn,
246 grant_ref_t *result)
247 {
248 int error, ref;
249
250 error = get_free_entries(1, &ref);
251 if (__predict_false(error))
252 return (error);
253
254 gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
255
256 *result = ref;
257 return (0);
258 }
259
260 void
261 gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
262 unsigned long pfn)
263 {
264 shared[ref].frame = pfn;
265 shared[ref].domid = domid;
266 wmb();
267 shared[ref].flags = GTF_accept_transfer;
268 }
269
270 unsigned long
271 gnttab_end_foreign_transfer_ref(grant_ref_t ref)
272 {
273 unsigned long frame;
274 uint16_t flags;
275
276 /*
277 * If a transfer is not even yet started, try to reclaim the grant
278 * reference and return failure (== 0).
279 */
280 while (!((flags = shared[ref].flags) & GTF_transfer_committed)) {
281 if ( synch_cmpxchg(&shared[ref].flags, flags, 0) == flags )
282 return (0);
283 cpu_relax();
284 }
285
286 /* If a transfer is in progress then wait until it is completed. */
287 while (!(flags & GTF_transfer_completed)) {
288 flags = shared[ref].flags;
289 cpu_relax();
290 }
291
292 /* Read the frame number /after/ reading completion status. */
293 rmb();
294 frame = shared[ref].frame;
295 KASSERT(frame != 0, ("grant table inconsistent"));
296
297 return (frame);
298 }
299
300 unsigned long
301 gnttab_end_foreign_transfer(grant_ref_t ref)
302 {
303 unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
304
305 put_free_entry(ref);
306 return (frame);
307 }
308
309 void
310 gnttab_free_grant_reference(grant_ref_t ref)
311 {
312
313 put_free_entry(ref);
314 }
315
316 void
317 gnttab_free_grant_references(grant_ref_t head)
318 {
319 grant_ref_t ref;
320 int count = 1;
321
322 if (head == GNTTAB_LIST_END)
323 return;
324
325 ref = head;
326 while (gnttab_entry(ref) != GNTTAB_LIST_END) {
327 ref = gnttab_entry(ref);
328 count++;
329 }
330 mtx_lock(&gnttab_list_lock);
331 gnttab_entry(ref) = gnttab_free_head;
332 gnttab_free_head = head;
333 gnttab_free_count += count;
334 check_free_callbacks();
335 mtx_unlock(&gnttab_list_lock);
336 }
337
338 int
339 gnttab_alloc_grant_references(uint16_t count, grant_ref_t *head)
340 {
341 int ref, error;
342
343 error = get_free_entries(count, &ref);
344 if (__predict_false(error))
345 return (error);
346
347 *head = ref;
348 return (0);
349 }
350
351 int
352 gnttab_empty_grant_references(const grant_ref_t *private_head)
353 {
354
355 return (*private_head == GNTTAB_LIST_END);
356 }
357
358 int
359 gnttab_claim_grant_reference(grant_ref_t *private_head)
360 {
361 grant_ref_t g = *private_head;
362
363 if (__predict_false(g == GNTTAB_LIST_END))
364 return (g);
365 *private_head = gnttab_entry(g);
366 return (g);
367 }
368
369 void
370 gnttab_release_grant_reference(grant_ref_t *private_head, grant_ref_t release)
371 {
372
373 gnttab_entry(release) = *private_head;
374 *private_head = release;
375 }
376
377 void
378 gnttab_request_free_callback(struct gnttab_free_callback *callback,
379 void (*fn)(void *), void *arg, uint16_t count)
380 {
381
382 mtx_lock(&gnttab_list_lock);
383 if (callback->next)
384 goto out;
385 callback->fn = fn;
386 callback->arg = arg;
387 callback->count = count;
388 callback->next = gnttab_free_callback_list;
389 gnttab_free_callback_list = callback;
390 check_free_callbacks();
391 out:
392 mtx_unlock(&gnttab_list_lock);
393
394 }
395
396 void
397 gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
398 {
399 struct gnttab_free_callback **pcb;
400
401 mtx_lock(&gnttab_list_lock);
402 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
403 if (*pcb == callback) {
404 *pcb = callback->next;
405 break;
406 }
407 }
408 mtx_unlock(&gnttab_list_lock);
409 }
410
411
412 static int
413 grow_gnttab_list(unsigned int more_frames)
414 {
415 unsigned int new_nr_grant_frames, extra_entries, i;
416
417 new_nr_grant_frames = nr_grant_frames + more_frames;
418 extra_entries = more_frames * GREFS_PER_GRANT_FRAME;
419
420 for (i = nr_grant_frames; i < new_nr_grant_frames; i++)
421 {
422 gnttab_list[i] = (grant_ref_t *)
423 malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT);
424
425 if (!gnttab_list[i])
426 goto grow_nomem;
427 }
428
429 for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames;
430 i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
431 gnttab_entry(i) = i + 1;
432
433 gnttab_entry(i) = gnttab_free_head;
434 gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames;
435 gnttab_free_count += extra_entries;
436
437 nr_grant_frames = new_nr_grant_frames;
438
439 check_free_callbacks();
440
441 return (0);
442
443 grow_nomem:
444 for ( ; i >= nr_grant_frames; i--)
445 free(gnttab_list[i], M_DEVBUF);
446 return (ENOMEM);
447 }
448
449 static unsigned int
450 __max_nr_grant_frames(void)
451 {
452 struct gnttab_query_size query;
453 int rc;
454
455 query.dom = DOMID_SELF;
456
457 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
458 if ((rc < 0) || (query.status != GNTST_okay))
459 return (4); /* Legacy max supported number of frames */
460
461 return (query.max_nr_frames);
462 }
463
464 static inline
465 unsigned int max_nr_grant_frames(void)
466 {
467 unsigned int xen_max = __max_nr_grant_frames();
468
469 if (xen_max > boot_max_nr_grant_frames)
470 return (boot_max_nr_grant_frames);
471 return (xen_max);
472 }
473
474 #ifdef notyet
475 /*
476 * XXX needed for backend support
477 *
478 */
479 static int
480 map_pte_fn(pte_t *pte, struct page *pmd_page,
481 unsigned long addr, void *data)
482 {
483 unsigned long **frames = (unsigned long **)data;
484
485 set_pte_at(&init_mm, addr, pte, pfn_pte_ma((*frames)[0], PAGE_KERNEL));
486 (*frames)++;
487 return 0;
488 }
489
490 static int
491 unmap_pte_fn(pte_t *pte, struct page *pmd_page,
492 unsigned long addr, void *data)
493 {
494
495 set_pte_at(&init_mm, addr, pte, __pte(0));
496 return 0;
497 }
498 #endif
499
500 #ifndef XENHVM
501
502 static int
503 gnttab_map(unsigned int start_idx, unsigned int end_idx)
504 {
505 struct gnttab_setup_table setup;
506 u_long *frames;
507
508 unsigned int nr_gframes = end_idx + 1;
509 int i, rc;
510
511 frames = malloc(nr_gframes * sizeof(unsigned long), M_DEVBUF, M_NOWAIT);
512 if (!frames)
513 return (ENOMEM);
514
515 setup.dom = DOMID_SELF;
516 setup.nr_frames = nr_gframes;
517 set_xen_guest_handle(setup.frame_list, frames);
518
519 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
520 if (rc == -ENOSYS) {
521 free(frames, M_DEVBUF);
522 return (ENOSYS);
523 }
524 KASSERT(!(rc || setup.status),
525 ("unexpected result from grant_table_op"));
526
527 if (shared == NULL) {
528 vm_offset_t area;
529
530 area = kva_alloc(PAGE_SIZE * max_nr_grant_frames());
531 KASSERT(area, ("can't allocate VM space for grant table"));
532 shared = (grant_entry_t *)area;
533 }
534
535 for (i = 0; i < nr_gframes; i++)
536 PT_SET_MA(((caddr_t)shared) + i*PAGE_SIZE,
537 ((vm_paddr_t)frames[i]) << PAGE_SHIFT | PG_RW | PG_V);
538
539 free(frames, M_DEVBUF);
540
541 return (0);
542 }
543
544 int
545 gnttab_resume(void)
546 {
547
548 if (max_nr_grant_frames() < nr_grant_frames)
549 return (ENOSYS);
550 return (gnttab_map(0, nr_grant_frames - 1));
551 }
552
553 int
554 gnttab_suspend(void)
555 {
556 int i;
557
558 for (i = 0; i < nr_grant_frames; i++)
559 pmap_kremove((vm_offset_t) shared + i * PAGE_SIZE);
560
561 return (0);
562 }
563
564 #else /* XENHVM */
565
566 #include <dev/xen/xenpci/xenpcivar.h>
567
568 static vm_paddr_t resume_frames;
569
570 static int
571 gnttab_map(unsigned int start_idx, unsigned int end_idx)
572 {
573 struct xen_add_to_physmap xatp;
574 unsigned int i = end_idx;
575
576 /*
577 * Loop backwards, so that the first hypercall has the largest index,
578 * ensuring that the table will grow only once.
579 */
580 do {
581 xatp.domid = DOMID_SELF;
582 xatp.idx = i;
583 xatp.space = XENMAPSPACE_grant_table;
584 xatp.gpfn = (resume_frames >> PAGE_SHIFT) + i;
585 if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp))
586 panic("HYPERVISOR_memory_op failed to map gnttab");
587 } while (i-- > start_idx);
588
589 if (shared == NULL) {
590 vm_offset_t area;
591
592 area = kva_alloc(PAGE_SIZE * max_nr_grant_frames());
593 KASSERT(area, ("can't allocate VM space for grant table"));
594 shared = (grant_entry_t *)area;
595 }
596
597 for (i = start_idx; i <= end_idx; i++) {
598 pmap_kenter((vm_offset_t) shared + i * PAGE_SIZE,
599 resume_frames + i * PAGE_SIZE);
600 }
601
602 return (0);
603 }
604
605 int
606 gnttab_resume(void)
607 {
608 int error;
609 unsigned int max_nr_gframes, nr_gframes;
610
611 nr_gframes = nr_grant_frames;
612 max_nr_gframes = max_nr_grant_frames();
613 if (max_nr_gframes < nr_gframes)
614 return (ENOSYS);
615
616 if (!resume_frames) {
617 error = xenpci_alloc_space(PAGE_SIZE * max_nr_gframes,
618 &resume_frames);
619 if (error) {
620 printf("error mapping gnttab share frames\n");
621 return (error);
622 }
623 }
624
625 return (gnttab_map(0, nr_gframes - 1));
626 }
627
628 #endif
629
630 static int
631 gnttab_expand(unsigned int req_entries)
632 {
633 int error;
634 unsigned int cur, extra;
635
636 cur = nr_grant_frames;
637 extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) /
638 GREFS_PER_GRANT_FRAME);
639 if (cur + extra > max_nr_grant_frames())
640 return (ENOSPC);
641
642 error = gnttab_map(cur, cur + extra - 1);
643 if (!error)
644 error = grow_gnttab_list(extra);
645
646 return (error);
647 }
648
649 int
650 gnttab_init()
651 {
652 int i;
653 unsigned int max_nr_glist_frames;
654 unsigned int nr_init_grefs;
655
656 if (!is_running_on_xen())
657 return (ENODEV);
658
659 nr_grant_frames = 1;
660 boot_max_nr_grant_frames = __max_nr_grant_frames();
661
662 /* Determine the maximum number of frames required for the
663 * grant reference free list on the current hypervisor.
664 */
665 max_nr_glist_frames = (boot_max_nr_grant_frames *
666 GREFS_PER_GRANT_FRAME /
667 (PAGE_SIZE / sizeof(grant_ref_t)));
668
669 gnttab_list = malloc(max_nr_glist_frames * sizeof(grant_ref_t *),
670 M_DEVBUF, M_NOWAIT);
671
672 if (gnttab_list == NULL)
673 return (ENOMEM);
674
675 for (i = 0; i < nr_grant_frames; i++) {
676 gnttab_list[i] = (grant_ref_t *)
677 malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT);
678 if (gnttab_list[i] == NULL)
679 goto ini_nomem;
680 }
681
682 if (gnttab_resume())
683 return (ENODEV);
684
685 nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME;
686
687 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
688 gnttab_entry(i) = i + 1;
689
690 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
691 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
692 gnttab_free_head = NR_RESERVED_ENTRIES;
693
694 if (bootverbose)
695 printf("Grant table initialized\n");
696
697 return (0);
698
699 ini_nomem:
700 for (i--; i >= 0; i--)
701 free(gnttab_list[i], M_DEVBUF);
702 free(gnttab_list, M_DEVBUF);
703 return (ENOMEM);
704
705 }
706
707 MTX_SYSINIT(gnttab, &gnttab_list_lock, "GNTTAB LOCK", MTX_DEF);
Cache object: 43a8b82b1552f168b40a1301b8839b30
|