The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/xen/gnttab.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /******************************************************************************
    2  * gnttab.c
    3  * 
    4  * Two sets of functionality:
    5  * 1. Granting foreign access to our memory reservation.
    6  * 2. Accessing others' memory reservations via grant references.
    7  * (i.e., mechanisms for both sender and recipient of grant references)
    8  * 
    9  * Copyright (c) 2005, Christopher Clark
   10  * Copyright (c) 2004, K A Fraser
   11  */
   12 
   13 #include <sys/cdefs.h>
   14 __FBSDID("$FreeBSD: releng/8.1/sys/xen/gnttab.c 199583 2009-11-20 15:27:52Z jhb $");
   15 
   16 #include "opt_global.h"
   17 #include "opt_pmap.h"
   18 
   19 #include <sys/param.h>
   20 #include <sys/systm.h>
   21 #include <sys/bus.h>
   22 #include <sys/conf.h>
   23 #include <sys/module.h>
   24 #include <sys/kernel.h>
   25 #include <sys/lock.h>
   26 #include <sys/malloc.h>
   27 #include <sys/mman.h>
   28 
   29 #include <machine/xen/xen-os.h>
   30 #include <xen/hypervisor.h>
   31 #include <machine/xen/synch_bitops.h>
   32 
   33 #include <xen/hypervisor.h>
   34 #include <xen/gnttab.h>
   35 
   36 #include <vm/vm.h>
   37 #include <vm/vm_kern.h>
   38 #include <vm/vm_extern.h>
   39 #include <vm/pmap.h>
   40 
   41 #define cmpxchg(a, b, c) atomic_cmpset_int((volatile u_int *)(a),(b),(c))
   42 
   43 /* External tools reserve first few grant table entries. */
   44 #define NR_RESERVED_ENTRIES 8
   45 #define GNTTAB_LIST_END 0xffffffff
   46 #define GREFS_PER_GRANT_FRAME (PAGE_SIZE / sizeof(grant_entry_t))
   47 
   48 static grant_ref_t **gnttab_list;
   49 static unsigned int nr_grant_frames;
   50 static unsigned int boot_max_nr_grant_frames;
   51 static int gnttab_free_count;
   52 static grant_ref_t gnttab_free_head;
   53 static struct mtx gnttab_list_lock;
   54 
   55 static grant_entry_t *shared;
   56 
   57 static struct gnttab_free_callback *gnttab_free_callback_list = NULL;
   58 
   59 static int gnttab_expand(unsigned int req_entries);
   60 
   61 #define RPP (PAGE_SIZE / sizeof(grant_ref_t))
   62 #define gnttab_entry(entry) (gnttab_list[(entry) / RPP][(entry) % RPP])
   63 
   64 static int
   65 get_free_entries(int count, int *entries)
   66 {
   67         int ref, error;
   68         grant_ref_t head;
   69         
   70         mtx_lock(&gnttab_list_lock);
   71         if ((gnttab_free_count < count) &&
   72             ((error = gnttab_expand(count - gnttab_free_count)) != 0)) {
   73                 mtx_unlock(&gnttab_list_lock);
   74                 return (error);
   75         }
   76         ref = head = gnttab_free_head;
   77         gnttab_free_count -= count;
   78         while (count-- > 1)
   79                 head = gnttab_entry(head);
   80         gnttab_free_head = gnttab_entry(head);
   81         gnttab_entry(head) = GNTTAB_LIST_END;
   82         mtx_unlock(&gnttab_list_lock);  
   83 
   84         *entries = ref;
   85         return (0);
   86 }
   87 
   88 static void
   89 do_free_callbacks(void)
   90 {
   91         struct gnttab_free_callback *callback, *next;
   92 
   93         callback = gnttab_free_callback_list;
   94         gnttab_free_callback_list = NULL;
   95 
   96         while (callback != NULL) {
   97                 next = callback->next;
   98                 if (gnttab_free_count >= callback->count) {
   99                         callback->next = NULL;
  100                         callback->fn(callback->arg);
  101                 } else {
  102                         callback->next = gnttab_free_callback_list;
  103                         gnttab_free_callback_list = callback;
  104                 }
  105                 callback = next;
  106         }
  107 }
  108 
  109 static inline void
  110 check_free_callbacks(void)
  111 {
  112         if (unlikely(gnttab_free_callback_list != NULL))
  113                 do_free_callbacks();
  114 }
  115 
  116 static void
  117 put_free_entry(grant_ref_t ref)
  118 {
  119 
  120         mtx_lock(&gnttab_list_lock);
  121         gnttab_entry(ref) = gnttab_free_head;
  122         gnttab_free_head = ref;
  123         gnttab_free_count++;
  124         check_free_callbacks();
  125         mtx_unlock(&gnttab_list_lock);  
  126 }
  127 
  128 /*
  129  * Public grant-issuing interface functions
  130  */
  131 
  132 int
  133 gnttab_grant_foreign_access(domid_t domid, unsigned long frame, int readonly,
  134         grant_ref_t *result)
  135 {
  136         int error, ref;
  137 
  138         error = get_free_entries(1, &ref);
  139         
  140         if (unlikely(error))
  141                 return (error);
  142 
  143         shared[ref].frame = frame;
  144         shared[ref].domid = domid;
  145         wmb();
  146         shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0);
  147 
  148         if (result)
  149                 *result = ref;
  150 
  151         return (0);
  152 }
  153 
  154 void
  155 gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
  156                                 unsigned long frame, int readonly)
  157 {
  158 
  159         shared[ref].frame = frame;
  160         shared[ref].domid = domid;
  161         wmb();
  162         shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0);
  163 }
  164 
  165 int
  166 gnttab_query_foreign_access(grant_ref_t ref)
  167 {
  168         uint16_t nflags;
  169         
  170         nflags = shared[ref].flags;
  171         
  172         return (nflags & (GTF_reading|GTF_writing));
  173 }
  174 
  175 int
  176 gnttab_end_foreign_access_ref(grant_ref_t ref)
  177 {
  178         uint16_t flags, nflags;
  179 
  180         nflags = shared[ref].flags;
  181         do {
  182                 if ( (flags = nflags) & (GTF_reading|GTF_writing) ) {
  183                         printf("WARNING: g.e. still in use!\n");
  184                         return (0);
  185                 }
  186         } while ((nflags = synch_cmpxchg(&shared[ref].flags, flags, 0)) !=
  187                flags);
  188 
  189         return (1);
  190 }
  191 
  192 void
  193 gnttab_end_foreign_access(grant_ref_t ref, void *page)
  194 {
  195         if (gnttab_end_foreign_access_ref(ref)) {
  196                 put_free_entry(ref);
  197                 if (page != NULL) {
  198                         free(page, M_DEVBUF);
  199                 }
  200         }
  201         else {
  202                 /* XXX This needs to be fixed so that the ref and page are
  203                    placed on a list to be freed up later. */
  204                 printf("WARNING: leaking g.e. and page still in use!\n");
  205         }
  206 }
  207 
  208 int
  209 gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn,
  210     grant_ref_t *result)
  211 {
  212         int error, ref;
  213 
  214         error = get_free_entries(1, &ref);
  215         if (unlikely(error))
  216                 return (error);
  217 
  218         gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
  219         
  220         *result = ref;
  221         return (0);
  222 }
  223 
  224 void
  225 gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
  226         unsigned long pfn)
  227 {
  228         shared[ref].frame = pfn;
  229         shared[ref].domid = domid;
  230         wmb();
  231         shared[ref].flags = GTF_accept_transfer;
  232 }
  233 
  234 unsigned long
  235 gnttab_end_foreign_transfer_ref(grant_ref_t ref)
  236 {
  237         unsigned long frame;
  238         uint16_t      flags;
  239 
  240         /*
  241          * If a transfer is not even yet started, try to reclaim the grant
  242          * reference and return failure (== 0).
  243          */
  244         while (!((flags = shared[ref].flags) & GTF_transfer_committed)) {
  245                 if ( synch_cmpxchg(&shared[ref].flags, flags, 0) == flags )
  246                         return (0);
  247                 cpu_relax();
  248         }
  249 
  250         /* If a transfer is in progress then wait until it is completed. */
  251         while (!(flags & GTF_transfer_completed)) {
  252                 flags = shared[ref].flags;
  253                 cpu_relax();
  254         }
  255 
  256         /* Read the frame number /after/ reading completion status. */
  257         rmb();
  258         frame = shared[ref].frame;
  259         KASSERT(frame != 0, ("grant table inconsistent"));
  260 
  261         return (frame);
  262 }
  263 
  264 unsigned long
  265 gnttab_end_foreign_transfer(grant_ref_t ref)
  266 {
  267         unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
  268 
  269         put_free_entry(ref);
  270         return (frame);
  271 }
  272 
  273 void
  274 gnttab_free_grant_reference(grant_ref_t ref)
  275 {
  276 
  277         put_free_entry(ref);
  278 }
  279 
  280 void
  281 gnttab_free_grant_references(grant_ref_t head)
  282 {
  283         grant_ref_t ref;
  284         int count = 1;
  285         
  286         if (head == GNTTAB_LIST_END)
  287                 return;
  288         
  289         mtx_lock(&gnttab_list_lock);
  290         ref = head;
  291         while (gnttab_entry(ref) != GNTTAB_LIST_END) {
  292                 ref = gnttab_entry(ref);
  293                 count++;
  294         }
  295         gnttab_entry(ref) = gnttab_free_head;
  296         gnttab_free_head = head;
  297         gnttab_free_count += count;
  298         check_free_callbacks();
  299         mtx_unlock(&gnttab_list_lock);
  300 }
  301 
  302 int
  303 gnttab_alloc_grant_references(uint16_t count, grant_ref_t *head)
  304 {
  305         int ref, error;
  306 
  307         error = get_free_entries(count, &ref);
  308         if (unlikely(error))
  309                 return (error);
  310 
  311         *head = ref;
  312         return (0);
  313 }
  314 
  315 int
  316 gnttab_empty_grant_references(const grant_ref_t *private_head)
  317 {
  318 
  319         return (*private_head == GNTTAB_LIST_END);
  320 }
  321 
  322 int
  323 gnttab_claim_grant_reference(grant_ref_t *private_head)
  324 {
  325         grant_ref_t g = *private_head;
  326 
  327         if (unlikely(g == GNTTAB_LIST_END))
  328                 return (ENOSPC);
  329         *private_head = gnttab_entry(g);
  330         return (g);
  331 }
  332 
  333 void
  334 gnttab_release_grant_reference(grant_ref_t *private_head, grant_ref_t  release)
  335 {
  336 
  337         gnttab_entry(release) = *private_head;
  338         *private_head = release;
  339 }
  340 
  341 void
  342 gnttab_request_free_callback(struct gnttab_free_callback *callback,
  343     void (*fn)(void *), void *arg, uint16_t count)
  344 {
  345 
  346         mtx_lock(&gnttab_list_lock);
  347         if (callback->next)
  348                 goto out;
  349         callback->fn = fn;
  350         callback->arg = arg;
  351         callback->count = count;
  352         callback->next = gnttab_free_callback_list;
  353         gnttab_free_callback_list = callback;
  354         check_free_callbacks();
  355  out:
  356         mtx_unlock(&gnttab_list_lock);
  357 
  358 }
  359 
  360 void
  361 gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
  362 {
  363         struct gnttab_free_callback **pcb;
  364 
  365         mtx_lock(&gnttab_list_lock);
  366         for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
  367                 if (*pcb == callback) {
  368                         *pcb = callback->next;
  369                         break;
  370                 }
  371         }
  372         mtx_unlock(&gnttab_list_lock);
  373 }
  374 
  375 
  376 static int
  377 grow_gnttab_list(unsigned int more_frames)
  378 {
  379         unsigned int new_nr_grant_frames, extra_entries, i;
  380 
  381         new_nr_grant_frames = nr_grant_frames + more_frames;
  382         extra_entries       = more_frames * GREFS_PER_GRANT_FRAME;
  383 
  384         for (i = nr_grant_frames; i < new_nr_grant_frames; i++)
  385         {
  386                 gnttab_list[i] = (grant_ref_t *)
  387                         malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT);
  388 
  389                 if (!gnttab_list[i])
  390                         goto grow_nomem;
  391         }
  392 
  393         for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames;
  394              i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
  395                 gnttab_entry(i) = i + 1;
  396 
  397         gnttab_entry(i) = gnttab_free_head;
  398         gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames;
  399         gnttab_free_count += extra_entries;
  400 
  401         nr_grant_frames = new_nr_grant_frames;
  402 
  403         check_free_callbacks();
  404 
  405         return (0);
  406         
  407 grow_nomem:
  408         for ( ; i >= nr_grant_frames; i--)
  409                 free(gnttab_list[i], M_DEVBUF);
  410         return (ENOMEM);
  411 }
  412 
  413 static unsigned int
  414 __max_nr_grant_frames(void)
  415 {
  416         struct gnttab_query_size query;
  417         int rc;
  418 
  419         query.dom = DOMID_SELF;
  420 
  421         rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
  422         if ((rc < 0) || (query.status != GNTST_okay))
  423                 return (4); /* Legacy max supported number of frames */
  424 
  425         return (query.max_nr_frames);
  426 }
  427 
  428 static inline
  429 unsigned int max_nr_grant_frames(void)
  430 {
  431         unsigned int xen_max = __max_nr_grant_frames();
  432 
  433         if (xen_max > boot_max_nr_grant_frames)
  434                 return (boot_max_nr_grant_frames);
  435         return (xen_max);
  436 }
  437 
  438 #ifdef notyet
  439 /*
  440  * XXX needed for backend support
  441  *
  442  */
  443 static int
  444 map_pte_fn(pte_t *pte, struct page *pmd_page,
  445                       unsigned long addr, void *data)
  446 {
  447         unsigned long **frames = (unsigned long **)data;
  448 
  449         set_pte_at(&init_mm, addr, pte, pfn_pte_ma((*frames)[0], PAGE_KERNEL));
  450         (*frames)++;
  451         return 0;
  452 }
  453 
  454 static int
  455 unmap_pte_fn(pte_t *pte, struct page *pmd_page,
  456                         unsigned long addr, void *data)
  457 {
  458 
  459         set_pte_at(&init_mm, addr, pte, __pte(0));
  460         return 0;
  461 }
  462 #endif
  463 
  464 #ifndef XENHVM
  465 
  466 static int
  467 gnttab_map(unsigned int start_idx, unsigned int end_idx)
  468 {
  469         struct gnttab_setup_table setup;
  470         u_long *frames;
  471 
  472         unsigned int nr_gframes = end_idx + 1;
  473         int i, rc;
  474 
  475         frames = malloc(nr_gframes * sizeof(unsigned long), M_DEVBUF, M_NOWAIT);
  476         if (!frames)
  477                 return (ENOMEM);
  478 
  479         setup.dom        = DOMID_SELF;
  480         setup.nr_frames  = nr_gframes;
  481         set_xen_guest_handle(setup.frame_list, frames);
  482 
  483         rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
  484         if (rc == -ENOSYS) {
  485                 free(frames, M_DEVBUF);
  486                 return (ENOSYS);
  487         }
  488         KASSERT(!(rc || setup.status),
  489             ("unexpected result from grant_table_op"));
  490 
  491         if (shared == NULL) {
  492                 vm_offset_t area;
  493                 
  494                 area = kmem_alloc_nofault(kernel_map,
  495                     PAGE_SIZE * max_nr_grant_frames());
  496                 KASSERT(area, ("can't allocate VM space for grant table"));
  497                 shared = (grant_entry_t *)area;
  498         }
  499 
  500         for (i = 0; i < nr_gframes; i++)
  501                 PT_SET_MA(((caddr_t)shared) + i*PAGE_SIZE, 
  502                     ((vm_paddr_t)frames[i]) << PAGE_SHIFT | PG_RW | PG_V);
  503 
  504         free(frames, M_DEVBUF);
  505         
  506         return (0);
  507 }
  508 
  509 int
  510 gnttab_resume(void)
  511 {
  512 
  513         if (max_nr_grant_frames() < nr_grant_frames)
  514                 return (ENOSYS);
  515         return (gnttab_map(0, nr_grant_frames - 1));
  516 }
  517 
  518 int
  519 gnttab_suspend(void)
  520 {       
  521         int i;
  522 
  523         for (i = 0; i < nr_grant_frames; i++)
  524                 pmap_kremove((vm_offset_t) shared + i * PAGE_SIZE);
  525 
  526         return (0);
  527 }
  528 
  529 #else /* XENHVM */
  530 
  531 #include <dev/xen/xenpci/xenpcivar.h>
  532 
  533 static vm_paddr_t resume_frames;
  534 
  535 static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
  536 {
  537         struct xen_add_to_physmap xatp;
  538         unsigned int i = end_idx;
  539 
  540         /*
  541          * Loop backwards, so that the first hypercall has the largest index,
  542          * ensuring that the table will grow only once.
  543          */
  544         do {
  545                 xatp.domid = DOMID_SELF;
  546                 xatp.idx = i;
  547                 xatp.space = XENMAPSPACE_grant_table;
  548                 xatp.gpfn = (resume_frames >> PAGE_SHIFT) + i;
  549                 if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp))
  550                         panic("HYPERVISOR_memory_op failed to map gnttab");
  551         } while (i-- > start_idx);
  552 
  553         if (shared == NULL) {
  554                 vm_offset_t area;
  555                 
  556                 area = kmem_alloc_nofault(kernel_map,
  557                     PAGE_SIZE * max_nr_grant_frames());
  558                 KASSERT(area, ("can't allocate VM space for grant table"));
  559                 shared = (grant_entry_t *)area;
  560         }
  561 
  562         for (i = start_idx; i <= end_idx; i++) {
  563                 pmap_kenter((vm_offset_t) shared + i * PAGE_SIZE,
  564                     resume_frames + i * PAGE_SIZE);
  565         }
  566 
  567         return (0);
  568 }
  569 
  570 int
  571 gnttab_resume(void)
  572 {
  573         int error;
  574         unsigned int max_nr_gframes, nr_gframes;
  575 
  576         nr_gframes = nr_grant_frames;
  577         max_nr_gframes = max_nr_grant_frames();
  578         if (max_nr_gframes < nr_gframes)
  579                 return (ENOSYS);
  580 
  581         if (!resume_frames) {
  582                 error = xenpci_alloc_space(PAGE_SIZE * max_nr_gframes,
  583                     &resume_frames);
  584                 if (error) {
  585                         printf("error mapping gnttab share frames\n");
  586                         return (error);
  587                 }
  588         }
  589 
  590         return (gnttab_map(0, nr_gframes - 1));
  591 }
  592 
  593 #endif
  594 
  595 static int
  596 gnttab_expand(unsigned int req_entries)
  597 {
  598         int error;
  599         unsigned int cur, extra;
  600 
  601         cur = nr_grant_frames;
  602         extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) /
  603                  GREFS_PER_GRANT_FRAME);
  604         if (cur + extra > max_nr_grant_frames())
  605                 return (ENOSPC);
  606 
  607         error = gnttab_map(cur, cur + extra - 1);
  608         if (!error)
  609                 error = grow_gnttab_list(extra);
  610 
  611         return (error);
  612 }
  613 
  614 int 
  615 gnttab_init()
  616 {
  617         int i;
  618         unsigned int max_nr_glist_frames;
  619         unsigned int nr_init_grefs;
  620 
  621         if (!is_running_on_xen())
  622                 return (ENODEV);
  623 
  624         nr_grant_frames = 1;
  625         boot_max_nr_grant_frames = __max_nr_grant_frames();
  626 
  627         /* Determine the maximum number of frames required for the
  628          * grant reference free list on the current hypervisor.
  629          */
  630         max_nr_glist_frames = (boot_max_nr_grant_frames *
  631                                GREFS_PER_GRANT_FRAME /
  632                                (PAGE_SIZE / sizeof(grant_ref_t)));
  633 
  634         gnttab_list = malloc(max_nr_glist_frames * sizeof(grant_ref_t *),
  635             M_DEVBUF, M_NOWAIT);
  636 
  637         if (gnttab_list == NULL)
  638                 return (ENOMEM);
  639 
  640         for (i = 0; i < nr_grant_frames; i++) {
  641                 gnttab_list[i] = (grant_ref_t *)
  642                         malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT);
  643                 if (gnttab_list[i] == NULL)
  644                         goto ini_nomem;
  645         }
  646         
  647         if (gnttab_resume())
  648                 return (ENODEV);
  649         
  650         nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME;
  651 
  652         for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
  653                 gnttab_entry(i) = i + 1;
  654 
  655         gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
  656         gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
  657         gnttab_free_head  = NR_RESERVED_ENTRIES;
  658 
  659         if (bootverbose)
  660                 printf("Grant table initialized\n");
  661 
  662         return (0);
  663 
  664 ini_nomem:
  665         for (i--; i >= 0; i--)
  666                 free(gnttab_list[i], M_DEVBUF);
  667         free(gnttab_list, M_DEVBUF);
  668         return (ENOMEM);
  669 
  670 }
  671 
  672 MTX_SYSINIT(gnttab, &gnttab_list_lock, "GNTTAB LOCK", MTX_DEF); 
  673 //SYSINIT(gnttab, SI_SUB_PSEUDO, SI_ORDER_FIRST, gnttab_init, NULL);

Cache object: da2e881884237dce23da99beeada98e3


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.