1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2006 Oleksandr Tymoshenko
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions, and the following disclaimer,
12 * without modification, immediately at the beginning of the file.
13 * 2. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred
29 */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33
34 /*
35 * MIPS bus dma support routines
36 */
37
38 #include <sys/param.h>
39 #include <sys/systm.h>
40 #include <sys/malloc.h>
41 #include <sys/bus.h>
42 #include <sys/busdma_bufalloc.h>
43 #include <sys/interrupt.h>
44 #include <sys/lock.h>
45 #include <sys/proc.h>
46 #include <sys/memdesc.h>
47 #include <sys/mutex.h>
48 #include <sys/ktr.h>
49 #include <sys/kernel.h>
50 #include <sys/sysctl.h>
51 #include <sys/uio.h>
52
53 #include <vm/uma.h>
54 #include <vm/vm.h>
55 #include <vm/vm_extern.h>
56 #include <vm/vm_kern.h>
57 #include <vm/vm_page.h>
58 #include <vm/vm_phys.h>
59 #include <vm/vm_map.h>
60
61 #include <machine/atomic.h>
62 #include <machine/bus.h>
63 #include <machine/cache.h>
64 #include <machine/cpufunc.h>
65 #include <machine/cpuinfo.h>
66 #include <machine/md_var.h>
67
68 #define MAX_BPAGES 64
69 #define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3
70 #define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4
71
72 /*
73 * On XBurst cores from Ingenic, cache-line writeback is local
74 * only, unless accompanied by invalidation. Invalidations force
75 * dirty line writeout and invalidation requests forwarded to
76 * other cores if other cores have the cache line dirty.
77 */
78 #if defined(SMP) && defined(CPU_XBURST)
79 #define BUS_DMA_FORCE_WBINV
80 #endif
81
82 struct bounce_zone;
83
84 struct bus_dma_tag {
85 bus_dma_tag_t parent;
86 bus_size_t alignment;
87 bus_addr_t boundary;
88 bus_addr_t lowaddr;
89 bus_addr_t highaddr;
90 bus_dma_filter_t *filter;
91 void *filterarg;
92 bus_size_t maxsize;
93 u_int nsegments;
94 bus_size_t maxsegsz;
95 int flags;
96 int ref_count;
97 int map_count;
98 bus_dma_lock_t *lockfunc;
99 void *lockfuncarg;
100 bus_dma_segment_t *segments;
101 struct bounce_zone *bounce_zone;
102 };
103
104 struct bounce_page {
105 vm_offset_t vaddr; /* kva of bounce buffer */
106 vm_offset_t vaddr_nocache; /* kva of bounce buffer uncached */
107 bus_addr_t busaddr; /* Physical address */
108 vm_offset_t datavaddr; /* kva of client data */
109 bus_addr_t dataaddr; /* client physical address */
110 bus_size_t datacount; /* client data count */
111 STAILQ_ENTRY(bounce_page) links;
112 };
113
114 struct sync_list {
115 vm_offset_t vaddr; /* kva of bounce buffer */
116 bus_addr_t busaddr; /* Physical address */
117 bus_size_t datacount; /* client data count */
118 };
119
120 int busdma_swi_pending;
121
122 struct bounce_zone {
123 STAILQ_ENTRY(bounce_zone) links;
124 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
125 int total_bpages;
126 int free_bpages;
127 int reserved_bpages;
128 int active_bpages;
129 int total_bounced;
130 int total_deferred;
131 int map_count;
132 bus_size_t alignment;
133 bus_addr_t lowaddr;
134 char zoneid[8];
135 char lowaddrid[20];
136 struct sysctl_ctx_list sysctl_tree;
137 struct sysctl_oid *sysctl_tree_top;
138 };
139
140 static struct mtx bounce_lock;
141 static int total_bpages;
142 static int busdma_zonecount;
143 static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
144
145 static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0,
146 "Busdma parameters");
147 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
148 "Total bounce pages");
149
150 #define DMAMAP_UNCACHEABLE 0x08
151 #define DMAMAP_CACHE_ALIGNED 0x10
152
153 struct bus_dmamap {
154 struct bp_list bpages;
155 int pagesneeded;
156 int pagesreserved;
157 bus_dma_tag_t dmat;
158 struct memdesc mem;
159 int flags;
160 TAILQ_ENTRY(bus_dmamap) freelist;
161 STAILQ_ENTRY(bus_dmamap) links;
162 bus_dmamap_callback_t *callback;
163 void *callback_arg;
164 int sync_count;
165 struct sync_list *slist;
166 };
167
168 static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
169 static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
170
171 static void init_bounce_pages(void *dummy);
172 static int alloc_bounce_zone(bus_dma_tag_t dmat);
173 static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
174 static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
175 int commit);
176 static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
177 vm_offset_t vaddr, bus_addr_t addr,
178 bus_size_t size);
179 static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
180
181 /* Default tag, as most drivers provide no parent tag. */
182 bus_dma_tag_t mips_root_dma_tag;
183
184 static uma_zone_t dmamap_zone; /* Cache of struct bus_dmamap items */
185
186 static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */
187 static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */
188
189 MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata");
190 MALLOC_DEFINE(M_BOUNCE, "bounce", "busdma bounce pages");
191
192 /*
193 * This is the ctor function passed to uma_zcreate() for the pool of dma maps.
194 * It'll need platform-specific changes if this code is copied.
195 */
196 static int
197 dmamap_ctor(void *mem, int size, void *arg, int flags)
198 {
199 bus_dmamap_t map;
200 bus_dma_tag_t dmat;
201
202 map = (bus_dmamap_t)mem;
203 dmat = (bus_dma_tag_t)arg;
204
205 dmat->map_count++;
206
207 bzero(map, sizeof(*map));
208 map->dmat = dmat;
209 STAILQ_INIT(&map->bpages);
210
211 return (0);
212 }
213
214 /*
215 * This is the dtor function passed to uma_zcreate() for the pool of dma maps.
216 * It may need platform-specific changes if this code is copied .
217 */
218 static void
219 dmamap_dtor(void *mem, int size, void *arg)
220 {
221 bus_dmamap_t map;
222
223 map = (bus_dmamap_t)mem;
224
225 map->dmat->map_count--;
226 }
227
228 static void
229 busdma_init(void *dummy)
230 {
231
232 /* Create a cache of maps for bus_dmamap_create(). */
233 dmamap_zone = uma_zcreate("dma maps", sizeof(struct bus_dmamap),
234 dmamap_ctor, dmamap_dtor, NULL, NULL, UMA_ALIGN_PTR, 0);
235
236 /* Create a cache of buffers in standard (cacheable) memory. */
237 standard_allocator = busdma_bufalloc_create("buffer",
238 mips_dcache_max_linesize, /* minimum_alignment */
239 NULL, /* uma_alloc func */
240 NULL, /* uma_free func */
241 0); /* uma_zcreate_flags */
242
243 /*
244 * Create a cache of buffers in uncacheable memory, to implement the
245 * BUS_DMA_COHERENT flag.
246 */
247 coherent_allocator = busdma_bufalloc_create("coherent",
248 mips_dcache_max_linesize, /* minimum_alignment */
249 busdma_bufalloc_alloc_uncacheable,
250 busdma_bufalloc_free_uncacheable,
251 0); /* uma_zcreate_flags */
252 }
253 SYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_FOURTH, busdma_init, NULL);
254
255 /*
256 * Return true if a match is made.
257 *
258 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'.
259 *
260 * If paddr is within the bounds of the dma tag then call the filter callback
261 * to check for a match, if there is no filter callback then assume a match.
262 */
263 static int
264 run_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
265 {
266 int retval;
267
268 retval = 0;
269
270 do {
271 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr)
272 || ((paddr & (dmat->alignment - 1)) != 0))
273 && (dmat->filter == NULL
274 || (*dmat->filter)(dmat->filterarg, paddr) != 0))
275 retval = 1;
276
277 dmat = dmat->parent;
278 } while (retval == 0 && dmat != NULL);
279 return (retval);
280 }
281
282 /*
283 * Check to see if the specified page is in an allowed DMA range.
284 */
285
286 static __inline int
287 _bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr)
288 {
289 int i;
290 for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) {
291 if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1])
292 || (lowaddr < phys_avail[i] &&
293 highaddr > phys_avail[i]))
294 return (1);
295 }
296 return (0);
297 }
298
299 /*
300 * Convenience function for manipulating driver locks from busdma (during
301 * busdma_swi, for example). Drivers that don't provide their own locks
302 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own
303 * non-mutex locking scheme don't have to use this at all.
304 */
305 void
306 busdma_lock_mutex(void *arg, bus_dma_lock_op_t op)
307 {
308 struct mtx *dmtx;
309
310 dmtx = (struct mtx *)arg;
311 switch (op) {
312 case BUS_DMA_LOCK:
313 mtx_lock(dmtx);
314 break;
315 case BUS_DMA_UNLOCK:
316 mtx_unlock(dmtx);
317 break;
318 default:
319 panic("Unknown operation 0x%x for busdma_lock_mutex!", op);
320 }
321 }
322
323 /*
324 * dflt_lock should never get called. It gets put into the dma tag when
325 * lockfunc == NULL, which is only valid if the maps that are associated
326 * with the tag are meant to never be defered.
327 * XXX Should have a way to identify which driver is responsible here.
328 */
329 static void
330 dflt_lock(void *arg, bus_dma_lock_op_t op)
331 {
332 #ifdef INVARIANTS
333 panic("driver error: busdma dflt_lock called");
334 #else
335 printf("DRIVER_ERROR: busdma dflt_lock called\n");
336 #endif
337 }
338
339 static __inline bus_dmamap_t
340 _busdma_alloc_dmamap(bus_dma_tag_t dmat)
341 {
342 struct sync_list *slist;
343 bus_dmamap_t map;
344
345 slist = malloc(sizeof(*slist) * dmat->nsegments, M_BUSDMA, M_NOWAIT);
346 if (slist == NULL)
347 return (NULL);
348 map = uma_zalloc_arg(dmamap_zone, dmat, M_NOWAIT);
349 if (map != NULL)
350 map->slist = slist;
351 else
352 free(slist, M_BUSDMA);
353 return (map);
354 }
355
356 static __inline void
357 _busdma_free_dmamap(bus_dmamap_t map)
358 {
359
360 free(map->slist, M_BUSDMA);
361 uma_zfree(dmamap_zone, map);
362 }
363
364 /*
365 * Allocate a device specific dma_tag.
366 */
367 #define SEG_NB 1024
368
369 int
370 bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
371 bus_addr_t boundary, bus_addr_t lowaddr,
372 bus_addr_t highaddr, bus_dma_filter_t *filter,
373 void *filterarg, bus_size_t maxsize, int nsegments,
374 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
375 void *lockfuncarg, bus_dma_tag_t *dmat)
376 {
377 bus_dma_tag_t newtag;
378 int error = 0;
379 /* Return a NULL tag on failure */
380 *dmat = NULL;
381 if (!parent)
382 parent = mips_root_dma_tag;
383
384 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_BUSDMA, M_NOWAIT);
385 if (newtag == NULL) {
386 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
387 __func__, newtag, 0, error);
388 return (ENOMEM);
389 }
390
391 newtag->parent = parent;
392 newtag->alignment = alignment;
393 newtag->boundary = boundary;
394 newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1);
395 newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1);
396 newtag->filter = filter;
397 newtag->filterarg = filterarg;
398 newtag->maxsize = maxsize;
399 newtag->nsegments = nsegments;
400 newtag->maxsegsz = maxsegsz;
401 newtag->flags = flags;
402 if (cpuinfo.cache_coherent_dma)
403 newtag->flags |= BUS_DMA_COHERENT;
404 newtag->ref_count = 1; /* Count ourself */
405 newtag->map_count = 0;
406 if (lockfunc != NULL) {
407 newtag->lockfunc = lockfunc;
408 newtag->lockfuncarg = lockfuncarg;
409 } else {
410 newtag->lockfunc = dflt_lock;
411 newtag->lockfuncarg = NULL;
412 }
413 newtag->segments = NULL;
414
415 /*
416 * Take into account any restrictions imposed by our parent tag
417 */
418 if (parent != NULL) {
419 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr);
420 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
421 if (newtag->boundary == 0)
422 newtag->boundary = parent->boundary;
423 else if (parent->boundary != 0)
424 newtag->boundary =
425 MIN(parent->boundary, newtag->boundary);
426 if ((newtag->filter != NULL) ||
427 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0))
428 newtag->flags |= BUS_DMA_COULD_BOUNCE;
429 if (newtag->filter == NULL) {
430 /*
431 * Short circuit looking at our parent directly
432 * since we have encapsulated all of its information
433 */
434 newtag->filter = parent->filter;
435 newtag->filterarg = parent->filterarg;
436 newtag->parent = parent->parent;
437 }
438 if (newtag->parent != NULL)
439 atomic_add_int(&parent->ref_count, 1);
440 }
441 if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr)
442 || newtag->alignment > 1)
443 newtag->flags |= BUS_DMA_COULD_BOUNCE;
444
445 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
446 (flags & BUS_DMA_ALLOCNOW) != 0) {
447 struct bounce_zone *bz;
448
449 /* Must bounce */
450
451 if ((error = alloc_bounce_zone(newtag)) != 0) {
452 free(newtag, M_BUSDMA);
453 return (error);
454 }
455 bz = newtag->bounce_zone;
456
457 if (ptoa(bz->total_bpages) < maxsize) {
458 int pages;
459
460 pages = atop(maxsize) - bz->total_bpages;
461
462 /* Add pages to our bounce pool */
463 if (alloc_bounce_pages(newtag, pages) < pages)
464 error = ENOMEM;
465 }
466 /* Performed initial allocation */
467 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
468 } else
469 newtag->bounce_zone = NULL;
470 if (error != 0)
471 free(newtag, M_BUSDMA);
472 else
473 *dmat = newtag;
474 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
475 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error);
476
477 return (error);
478 }
479
480 void
481 bus_dma_template_clone(bus_dma_template_t *t, bus_dma_tag_t dmat)
482 {
483
484 if (t == NULL || dmat == NULL)
485 return;
486
487 t->parent = dmat->parent;
488 t->alignment = dmat->alignment;
489 t->boundary = dmat->boundary;
490 t->lowaddr = dmat->lowaddr;
491 t->highaddr = dmat->highaddr;
492 t->maxsize = dmat->maxsize;
493 t->nsegments = dmat->nsegments;
494 t->maxsegsize = dmat->maxsegsz;
495 t->flags = dmat->flags;
496 t->lockfunc = dmat->lockfunc;
497 t->lockfuncarg = dmat->lockfuncarg;
498 }
499
500 int
501 bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain)
502 {
503
504 return (0);
505 }
506
507 int
508 bus_dma_tag_destroy(bus_dma_tag_t dmat)
509 {
510 #ifdef KTR
511 bus_dma_tag_t dmat_copy = dmat;
512 #endif
513
514 if (dmat != NULL) {
515 if (dmat->map_count != 0)
516 return (EBUSY);
517
518 while (dmat != NULL) {
519 bus_dma_tag_t parent;
520
521 parent = dmat->parent;
522 atomic_subtract_int(&dmat->ref_count, 1);
523 if (dmat->ref_count == 0) {
524 if (dmat->segments != NULL)
525 free(dmat->segments, M_BUSDMA);
526 free(dmat, M_BUSDMA);
527 /*
528 * Last reference count, so
529 * release our reference
530 * count on our parent.
531 */
532 dmat = parent;
533 } else
534 dmat = NULL;
535 }
536 }
537 CTR2(KTR_BUSDMA, "%s tag %p", __func__, dmat_copy);
538
539 return (0);
540 }
541
542 #include <sys/kdb.h>
543 /*
544 * Allocate a handle for mapping from kva/uva/physical
545 * address space into bus device space.
546 */
547 int
548 bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
549 {
550 bus_dmamap_t newmap;
551 int error = 0;
552
553 if (dmat->segments == NULL) {
554 dmat->segments = (bus_dma_segment_t *)malloc(
555 sizeof(bus_dma_segment_t) * dmat->nsegments, M_BUSDMA,
556 M_NOWAIT);
557 if (dmat->segments == NULL) {
558 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
559 __func__, dmat, ENOMEM);
560 return (ENOMEM);
561 }
562 }
563
564 newmap = _busdma_alloc_dmamap(dmat);
565 if (newmap == NULL) {
566 CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM);
567 return (ENOMEM);
568 }
569 *mapp = newmap;
570
571 /*
572 * Bouncing might be required if the driver asks for an active
573 * exclusion region, a data alignment that is stricter than 1, and/or
574 * an active address boundary.
575 */
576 if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
577 /* Must bounce */
578 struct bounce_zone *bz;
579 int maxpages;
580
581 if (dmat->bounce_zone == NULL) {
582 if ((error = alloc_bounce_zone(dmat)) != 0) {
583 _busdma_free_dmamap(newmap);
584 *mapp = NULL;
585 return (error);
586 }
587 }
588 bz = dmat->bounce_zone;
589
590 /* Initialize the new map */
591 STAILQ_INIT(&((*mapp)->bpages));
592
593 /*
594 * Attempt to add pages to our pool on a per-instance
595 * basis up to a sane limit.
596 */
597 maxpages = MAX_BPAGES;
598 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
599 || (bz->map_count > 0 && bz->total_bpages < maxpages)) {
600 int pages;
601
602 pages = MAX(atop(dmat->maxsize), 1);
603 pages = MIN(maxpages - bz->total_bpages, pages);
604 pages = MAX(pages, 1);
605 if (alloc_bounce_pages(dmat, pages) < pages)
606 error = ENOMEM;
607
608 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
609 if (error == 0)
610 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
611 } else {
612 error = 0;
613 }
614 }
615 bz->map_count++;
616 }
617
618 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
619 __func__, dmat, dmat->flags, error);
620
621 return (0);
622 }
623
624 /*
625 * Destroy a handle for mapping from kva/uva/physical
626 * address space into bus device space.
627 */
628 int
629 bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
630 {
631
632 if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) {
633 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
634 __func__, dmat, EBUSY);
635 return (EBUSY);
636 }
637 if (dmat->bounce_zone)
638 dmat->bounce_zone->map_count--;
639 _busdma_free_dmamap(map);
640 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
641 return (0);
642 }
643
644 /*
645 * Allocate a piece of memory that can be efficiently mapped into
646 * bus device space based on the constraints lited in the dma tag.
647 * A dmamap to for use with dmamap_load is also allocated.
648 */
649 int
650 bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddrp, int flags,
651 bus_dmamap_t *mapp)
652 {
653 bus_dmamap_t newmap = NULL;
654 busdma_bufalloc_t ba;
655 struct busdma_bufzone *bufzone;
656 vm_memattr_t memattr;
657 void *vaddr;
658
659 int mflags;
660
661 if (flags & BUS_DMA_NOWAIT)
662 mflags = M_NOWAIT;
663 else
664 mflags = M_WAITOK;
665 if (dmat->segments == NULL) {
666 dmat->segments = (bus_dma_segment_t *)malloc(
667 sizeof(bus_dma_segment_t) * dmat->nsegments, M_BUSDMA,
668 mflags);
669 if (dmat->segments == NULL) {
670 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
671 __func__, dmat, dmat->flags, ENOMEM);
672 return (ENOMEM);
673 }
674 }
675
676 newmap = _busdma_alloc_dmamap(dmat);
677 if (newmap == NULL) {
678 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
679 __func__, dmat, dmat->flags, ENOMEM);
680 return (ENOMEM);
681 }
682
683 /*
684 * If all the memory is coherent with DMA then we don't need to
685 * do anything special for a coherent mapping request.
686 */
687 if (dmat->flags & BUS_DMA_COHERENT)
688 flags &= ~BUS_DMA_COHERENT;
689
690 if (flags & BUS_DMA_COHERENT) {
691 memattr = VM_MEMATTR_UNCACHEABLE;
692 ba = coherent_allocator;
693 newmap->flags |= DMAMAP_UNCACHEABLE;
694 } else {
695 memattr = VM_MEMATTR_DEFAULT;
696 ba = standard_allocator;
697 }
698 /* All buffers we allocate are cache-aligned. */
699 newmap->flags |= DMAMAP_CACHE_ALIGNED;
700
701 if (flags & BUS_DMA_ZERO)
702 mflags |= M_ZERO;
703
704 /*
705 * Try to find a bufzone in the allocator that holds a cache of buffers
706 * of the right size for this request. If the buffer is too big to be
707 * held in the allocator cache, this returns NULL.
708 */
709 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize);
710
711 /*
712 * Allocate the buffer from the uma(9) allocator if...
713 * - It's small enough to be in the allocator (bufzone not NULL).
714 * - The alignment constraint isn't larger than the allocation size
715 * (the allocator aligns buffers to their size boundaries).
716 * - There's no need to handle lowaddr/highaddr exclusion zones.
717 * else allocate non-contiguous pages if...
718 * - The page count that could get allocated doesn't exceed
719 * nsegments also when the maximum segment size is less
720 * than PAGE_SIZE.
721 * - The alignment constraint isn't larger than a page boundary.
722 * - There are no boundary-crossing constraints.
723 * else allocate a block of contiguous pages because one or more of the
724 * constraints is something that only the contig allocator can fulfill.
725 */
726 if (bufzone != NULL && dmat->alignment <= bufzone->size &&
727 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) {
728 vaddr = uma_zalloc(bufzone->umazone, mflags);
729 } else if (dmat->nsegments >=
730 howmany(dmat->maxsize, MIN(dmat->maxsegsz, PAGE_SIZE)) &&
731 dmat->alignment <= PAGE_SIZE &&
732 (dmat->boundary % PAGE_SIZE) == 0) {
733 vaddr = (void *)kmem_alloc_attr(dmat->maxsize, mflags, 0,
734 dmat->lowaddr, memattr);
735 } else {
736 vaddr = (void *)kmem_alloc_contig(dmat->maxsize, mflags, 0,
737 dmat->lowaddr, dmat->alignment, dmat->boundary, memattr);
738 }
739 if (vaddr == NULL) {
740 _busdma_free_dmamap(newmap);
741 newmap = NULL;
742 } else {
743 newmap->sync_count = 0;
744 }
745 *vaddrp = vaddr;
746 *mapp = newmap;
747
748 return (vaddr == NULL ? ENOMEM : 0);
749 }
750
751 /*
752 * Free a piece of memory and it's allocated dmamap, that was allocated
753 * via bus_dmamem_alloc. Make the same choice for free/contigfree.
754 */
755 void
756 bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
757 {
758 struct busdma_bufzone *bufzone;
759 busdma_bufalloc_t ba;
760
761 if (map->flags & DMAMAP_UNCACHEABLE)
762 ba = coherent_allocator;
763 else
764 ba = standard_allocator;
765
766 free(map->slist, M_BUSDMA);
767 uma_zfree(dmamap_zone, map);
768
769 bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize);
770
771 if (bufzone != NULL && dmat->alignment <= bufzone->size &&
772 !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr))
773 uma_zfree(bufzone->umazone, vaddr);
774 else
775 kmem_free((vm_offset_t)vaddr, dmat->maxsize);
776 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags);
777 }
778
779 static void
780 _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
781 bus_size_t buflen, int flags)
782 {
783 bus_addr_t curaddr;
784 bus_size_t sgsize;
785
786 if (map->pagesneeded == 0) {
787 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d",
788 dmat->lowaddr, dmat->boundary, dmat->alignment);
789 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d",
790 map, map->pagesneeded);
791 /*
792 * Count the number of bounce pages
793 * needed in order to complete this transfer
794 */
795 curaddr = buf;
796 while (buflen != 0) {
797 sgsize = MIN(buflen, dmat->maxsegsz);
798 if (run_filter(dmat, curaddr) != 0) {
799 sgsize = MIN(sgsize, PAGE_SIZE);
800 map->pagesneeded++;
801 }
802 curaddr += sgsize;
803 buflen -= sgsize;
804 }
805 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
806 }
807 }
808
809 static void
810 _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
811 void *buf, bus_size_t buflen, int flags)
812 {
813 vm_offset_t vaddr;
814 vm_offset_t vendaddr;
815 bus_addr_t paddr;
816
817 if (map->pagesneeded == 0) {
818 CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d",
819 dmat->lowaddr, dmat->boundary, dmat->alignment);
820 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d",
821 map, map->pagesneeded);
822 /*
823 * Count the number of bounce pages
824 * needed in order to complete this transfer
825 */
826 vaddr = (vm_offset_t)buf;
827 vendaddr = (vm_offset_t)buf + buflen;
828
829 while (vaddr < vendaddr) {
830 bus_size_t sg_len;
831
832 KASSERT(kernel_pmap == pmap, ("pmap is not kernel pmap"));
833 sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK);
834 paddr = pmap_kextract(vaddr);
835 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
836 run_filter(dmat, paddr) != 0) {
837 sg_len = roundup2(sg_len, dmat->alignment);
838 map->pagesneeded++;
839 }
840 vaddr += sg_len;
841 }
842 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
843 }
844 }
845
846 static int
847 _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map,int flags)
848 {
849
850 /* Reserve Necessary Bounce Pages */
851 mtx_lock(&bounce_lock);
852 if (flags & BUS_DMA_NOWAIT) {
853 if (reserve_bounce_pages(dmat, map, 0) != 0) {
854 mtx_unlock(&bounce_lock);
855 return (ENOMEM);
856 }
857 } else {
858 if (reserve_bounce_pages(dmat, map, 1) != 0) {
859 /* Queue us for resources */
860 STAILQ_INSERT_TAIL(&bounce_map_waitinglist,
861 map, links);
862 mtx_unlock(&bounce_lock);
863 return (EINPROGRESS);
864 }
865 }
866 mtx_unlock(&bounce_lock);
867
868 return (0);
869 }
870
871 /*
872 * Add a single contiguous physical range to the segment list.
873 */
874 static int
875 _bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr,
876 bus_size_t sgsize, bus_dma_segment_t *segs, int *segp)
877 {
878 bus_addr_t baddr, bmask;
879 int seg;
880
881 /*
882 * Make sure we don't cross any boundaries.
883 */
884 bmask = ~(dmat->boundary - 1);
885 if (dmat->boundary > 0) {
886 baddr = (curaddr + dmat->boundary) & bmask;
887 if (sgsize > (baddr - curaddr))
888 sgsize = (baddr - curaddr);
889 }
890 /*
891 * Insert chunk into a segment, coalescing with
892 * the previous segment if possible.
893 */
894 seg = *segp;
895 if (seg >= 0 &&
896 curaddr == segs[seg].ds_addr + segs[seg].ds_len &&
897 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
898 (dmat->boundary == 0 ||
899 (segs[seg].ds_addr & bmask) == (curaddr & bmask))) {
900 segs[seg].ds_len += sgsize;
901 } else {
902 if (++seg >= dmat->nsegments)
903 return (0);
904 segs[seg].ds_addr = curaddr;
905 segs[seg].ds_len = sgsize;
906 }
907 *segp = seg;
908 return (sgsize);
909 }
910
911 /*
912 * Utility function to load a physical buffer. segp contains
913 * the starting segment on entrace, and the ending segment on exit.
914 */
915 int
916 _bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
917 vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs,
918 int *segp)
919 {
920 bus_addr_t curaddr;
921 bus_size_t sgsize;
922 int error;
923
924 if (segs == NULL)
925 segs = dmat->segments;
926
927 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
928 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
929 if (map->pagesneeded != 0) {
930 error = _bus_dmamap_reserve_pages(dmat, map, flags);
931 if (error)
932 return (error);
933 }
934 }
935
936 while (buflen > 0) {
937 curaddr = buf;
938 sgsize = MIN(buflen, dmat->maxsegsz);
939 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
940 map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
941 sgsize = MIN(sgsize, PAGE_SIZE);
942 curaddr = add_bounce_page(dmat, map, 0, curaddr,
943 sgsize);
944 }
945 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
946 segp);
947 if (sgsize == 0)
948 break;
949 buf += sgsize;
950 buflen -= sgsize;
951 }
952
953 /*
954 * Did we fit?
955 */
956 if (buflen != 0) {
957 bus_dmamap_unload(dmat, map);
958 return (EFBIG); /* XXX better return value here? */
959 }
960 return (0);
961 }
962
963 int
964 _bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map,
965 struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
966 bus_dma_segment_t *segs, int *segp)
967 {
968
969 return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags,
970 segs, segp));
971 }
972
973 /*
974 * Utility function to load a linear buffer. segp contains
975 * the starting segment on entrance, and the ending segment on exit.
976 * first indicates if this is the first invocation of this function.
977 */
978 int
979 _bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
980 bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs,
981 int *segp)
982 {
983 bus_size_t sgsize;
984 bus_addr_t curaddr;
985 struct sync_list *sl;
986 vm_offset_t vaddr = (vm_offset_t)buf;
987 int error = 0;
988
989 if (segs == NULL)
990 segs = dmat->segments;
991 if ((flags & BUS_DMA_LOAD_MBUF) != 0)
992 map->flags |= DMAMAP_CACHE_ALIGNED;
993
994 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
995 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
996 if (map->pagesneeded != 0) {
997 error = _bus_dmamap_reserve_pages(dmat, map, flags);
998 if (error)
999 return (error);
1000 }
1001 }
1002 CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, "
1003 "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment);
1004
1005 while (buflen > 0) {
1006 /*
1007 * Get the physical address for this segment.
1008 *
1009 * XXX Don't support checking for coherent mappings
1010 * XXX in user address space.
1011 */
1012 KASSERT(kernel_pmap == pmap, ("pmap is not kernel pmap"));
1013 curaddr = pmap_kextract(vaddr);
1014
1015 /*
1016 * Compute the segment size, and adjust counts.
1017 */
1018 sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK);
1019 if (sgsize > dmat->maxsegsz)
1020 sgsize = dmat->maxsegsz;
1021 if (buflen < sgsize)
1022 sgsize = buflen;
1023
1024 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
1025 map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
1026 curaddr = add_bounce_page(dmat, map, vaddr, curaddr,
1027 sgsize);
1028 } else {
1029 sl = &map->slist[map->sync_count - 1];
1030 if (map->sync_count == 0 ||
1031 vaddr != sl->vaddr + sl->datacount) {
1032 if (++map->sync_count > dmat->nsegments)
1033 goto cleanup;
1034 sl++;
1035 sl->vaddr = vaddr;
1036 sl->datacount = sgsize;
1037 sl->busaddr = curaddr;
1038 } else
1039 sl->datacount += sgsize;
1040 }
1041 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
1042 segp);
1043 if (sgsize == 0)
1044 break;
1045 vaddr += sgsize;
1046 buflen -= sgsize;
1047 }
1048
1049 cleanup:
1050 /*
1051 * Did we fit?
1052 */
1053 if (buflen != 0) {
1054 bus_dmamap_unload(dmat, map);
1055 error = EFBIG; /* XXX better return value here? */
1056 }
1057 return (error);
1058 }
1059
1060 void
1061 _bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
1062 struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg)
1063 {
1064
1065 KASSERT(dmat != NULL, ("dmatag is NULL"));
1066 KASSERT(map != NULL, ("dmamap is NULL"));
1067 map->mem = *mem;
1068 map->callback = callback;
1069 map->callback_arg = callback_arg;
1070 }
1071
1072 bus_dma_segment_t *
1073 _bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
1074 bus_dma_segment_t *segs, int nsegs, int error)
1075 {
1076
1077 if (segs == NULL)
1078 segs = dmat->segments;
1079 return (segs);
1080 }
1081
1082 /*
1083 * Release the mapping held by map.
1084 */
1085 void
1086 bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
1087 {
1088 struct bounce_page *bpage;
1089
1090 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
1091 STAILQ_REMOVE_HEAD(&map->bpages, links);
1092 free_bounce_page(dmat, bpage);
1093 }
1094 map->sync_count = 0;
1095 return;
1096 }
1097
1098 static void
1099 bus_dmamap_sync_buf(vm_offset_t buf, int len, bus_dmasync_op_t op, int aligned)
1100 {
1101 char tmp_cl[mips_dcache_max_linesize], tmp_clend[mips_dcache_max_linesize];
1102 vm_offset_t buf_cl, buf_clend;
1103 vm_size_t size_cl, size_clend;
1104 int cache_linesize_mask = mips_dcache_max_linesize - 1;
1105
1106 /*
1107 * dcache invalidation operates on cache line aligned addresses
1108 * and could modify areas of memory that share the same cache line
1109 * at the beginning and the ending of the buffer. In order to
1110 * prevent a data loss we save these chunks in temporary buffer
1111 * before invalidation and restore them afer it.
1112 *
1113 * If the aligned flag is set the buffer is either an mbuf or came from
1114 * our allocator caches. In both cases they are always sized and
1115 * aligned to cacheline boundaries, so we can skip preserving nearby
1116 * data if a transfer appears to overlap cachelines. An mbuf in
1117 * particular will usually appear to be overlapped because of offsetting
1118 * within the buffer to align the L3 headers, but we know that the bytes
1119 * preceeding that offset are part of the same mbuf memory and are not
1120 * unrelated adjacent data (and a rule of mbuf handling is that the cpu
1121 * is not allowed to touch the mbuf while dma is in progress, including
1122 * header fields).
1123 */
1124 if (aligned) {
1125 size_cl = 0;
1126 size_clend = 0;
1127 } else {
1128 buf_cl = buf & ~cache_linesize_mask;
1129 size_cl = buf & cache_linesize_mask;
1130 buf_clend = buf + len;
1131 size_clend = (mips_dcache_max_linesize -
1132 (buf_clend & cache_linesize_mask)) & cache_linesize_mask;
1133 }
1134
1135 switch (op) {
1136 case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE:
1137 case BUS_DMASYNC_POSTREAD:
1138
1139 /*
1140 * Save buffers that might be modified by invalidation
1141 */
1142 if (size_cl)
1143 memcpy (tmp_cl, (void*)buf_cl, size_cl);
1144 if (size_clend)
1145 memcpy (tmp_clend, (void*)buf_clend, size_clend);
1146 mips_dcache_inv_range(buf, len);
1147 /*
1148 * Restore them
1149 */
1150 if (size_cl)
1151 memcpy ((void*)buf_cl, tmp_cl, size_cl);
1152 if (size_clend)
1153 memcpy ((void*)buf_clend, tmp_clend, size_clend);
1154 /*
1155 * Copies above have brought corresponding memory
1156 * cache lines back into dirty state. Write them back
1157 * out and invalidate affected cache lines again if
1158 * necessary.
1159 */
1160 if (size_cl)
1161 mips_dcache_wbinv_range(buf_cl, size_cl);
1162 if (size_clend && (size_cl == 0 ||
1163 buf_clend - buf_cl > mips_dcache_max_linesize))
1164 mips_dcache_wbinv_range(buf_clend, size_clend);
1165 break;
1166
1167 case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE:
1168 mips_dcache_wbinv_range(buf, len);
1169 break;
1170
1171 case BUS_DMASYNC_PREREAD:
1172 /*
1173 * Save buffers that might be modified by invalidation
1174 */
1175 if (size_cl)
1176 memcpy (tmp_cl, (void *)buf_cl, size_cl);
1177 if (size_clend)
1178 memcpy (tmp_clend, (void *)buf_clend, size_clend);
1179 mips_dcache_inv_range(buf, len);
1180 /*
1181 * Restore them
1182 */
1183 if (size_cl)
1184 memcpy ((void *)buf_cl, tmp_cl, size_cl);
1185 if (size_clend)
1186 memcpy ((void *)buf_clend, tmp_clend, size_clend);
1187 /*
1188 * Copies above have brought corresponding memory
1189 * cache lines back into dirty state. Write them back
1190 * out and invalidate affected cache lines again if
1191 * necessary.
1192 */
1193 if (size_cl)
1194 mips_dcache_wbinv_range(buf_cl, size_cl);
1195 if (size_clend && (size_cl == 0 ||
1196 buf_clend - buf_cl > mips_dcache_max_linesize))
1197 mips_dcache_wbinv_range(buf_clend, size_clend);
1198 break;
1199
1200 case BUS_DMASYNC_PREWRITE:
1201 #ifdef BUS_DMA_FORCE_WBINV
1202 mips_dcache_wbinv_range(buf, len);
1203 #else
1204 mips_dcache_wb_range(buf, len);
1205 #endif
1206 break;
1207 }
1208 }
1209
1210 static void
1211 _bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
1212 {
1213 struct bounce_page *bpage;
1214
1215 STAILQ_FOREACH(bpage, &map->bpages, links) {
1216 if (op & BUS_DMASYNC_PREWRITE) {
1217 if (bpage->datavaddr != 0)
1218 bcopy((void *)bpage->datavaddr,
1219 (void *)(bpage->vaddr_nocache != 0 ?
1220 bpage->vaddr_nocache :
1221 bpage->vaddr),
1222 bpage->datacount);
1223 else
1224 physcopyout(bpage->dataaddr,
1225 (void *)(bpage->vaddr_nocache != 0 ?
1226 bpage->vaddr_nocache :
1227 bpage->vaddr),
1228 bpage->datacount);
1229 if (bpage->vaddr_nocache == 0) {
1230 #ifdef BUS_DMA_FORCE_WBINV
1231 mips_dcache_wbinv_range(bpage->vaddr,
1232 bpage->datacount);
1233 #else
1234 mips_dcache_wb_range(bpage->vaddr,
1235 bpage->datacount);
1236 #endif
1237 }
1238 dmat->bounce_zone->total_bounced++;
1239 }
1240 if (op & BUS_DMASYNC_POSTREAD) {
1241 if (bpage->vaddr_nocache == 0) {
1242 mips_dcache_inv_range(bpage->vaddr,
1243 bpage->datacount);
1244 }
1245 if (bpage->datavaddr != 0)
1246 bcopy((void *)(bpage->vaddr_nocache != 0 ?
1247 bpage->vaddr_nocache : bpage->vaddr),
1248 (void *)bpage->datavaddr, bpage->datacount);
1249 else
1250 physcopyin((void *)(bpage->vaddr_nocache != 0 ?
1251 bpage->vaddr_nocache : bpage->vaddr),
1252 bpage->dataaddr, bpage->datacount);
1253 dmat->bounce_zone->total_bounced++;
1254 }
1255 }
1256 }
1257
1258 void
1259 bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
1260 {
1261 struct sync_list *sl, *end;
1262 int aligned;
1263
1264 if (op == BUS_DMASYNC_POSTWRITE)
1265 return;
1266 if (STAILQ_FIRST(&map->bpages))
1267 _bus_dmamap_sync_bp(dmat, map, op);
1268
1269 if ((dmat->flags & BUS_DMA_COHERENT) ||
1270 (map->flags & DMAMAP_UNCACHEABLE)) {
1271 if (op & BUS_DMASYNC_PREWRITE)
1272 mips_sync();
1273 return;
1274 }
1275
1276 aligned = (map->flags & DMAMAP_CACHE_ALIGNED) ? 1 : 0;
1277
1278 CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags);
1279 if (map->sync_count) {
1280 end = &map->slist[map->sync_count];
1281 for (sl = &map->slist[0]; sl != end; sl++)
1282 bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op,
1283 aligned);
1284 }
1285 }
1286
1287 static void
1288 init_bounce_pages(void *dummy __unused)
1289 {
1290
1291 total_bpages = 0;
1292 STAILQ_INIT(&bounce_zone_list);
1293 STAILQ_INIT(&bounce_map_waitinglist);
1294 STAILQ_INIT(&bounce_map_callbacklist);
1295 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF);
1296 }
1297 SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL);
1298
1299 static struct sysctl_ctx_list *
1300 busdma_sysctl_tree(struct bounce_zone *bz)
1301 {
1302 return (&bz->sysctl_tree);
1303 }
1304
1305 static struct sysctl_oid *
1306 busdma_sysctl_tree_top(struct bounce_zone *bz)
1307 {
1308 return (bz->sysctl_tree_top);
1309 }
1310
1311 static int
1312 alloc_bounce_zone(bus_dma_tag_t dmat)
1313 {
1314 struct bounce_zone *bz;
1315
1316 /* Check to see if we already have a suitable zone */
1317 STAILQ_FOREACH(bz, &bounce_zone_list, links) {
1318 if ((dmat->alignment <= bz->alignment)
1319 && (dmat->lowaddr >= bz->lowaddr)) {
1320 dmat->bounce_zone = bz;
1321 return (0);
1322 }
1323 }
1324
1325 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_BUSDMA,
1326 M_NOWAIT | M_ZERO)) == NULL)
1327 return (ENOMEM);
1328
1329 STAILQ_INIT(&bz->bounce_page_list);
1330 bz->free_bpages = 0;
1331 bz->reserved_bpages = 0;
1332 bz->active_bpages = 0;
1333 bz->lowaddr = dmat->lowaddr;
1334 bz->alignment = MAX(dmat->alignment, PAGE_SIZE);
1335 bz->map_count = 0;
1336 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
1337 busdma_zonecount++;
1338 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr);
1339 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
1340 dmat->bounce_zone = bz;
1341
1342 sysctl_ctx_init(&bz->sysctl_tree);
1343 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree,
1344 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
1345 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "");
1346 if (bz->sysctl_tree_top == NULL) {
1347 sysctl_ctx_free(&bz->sysctl_tree);
1348 return (0); /* XXX error code? */
1349 }
1350
1351 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1352 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1353 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
1354 "Total bounce pages");
1355 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1356 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1357 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
1358 "Free bounce pages");
1359 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1360 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1361 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
1362 "Reserved bounce pages");
1363 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1364 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1365 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
1366 "Active bounce pages");
1367 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1368 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1369 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
1370 "Total bounce requests");
1371 SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1372 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1373 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
1374 "Total bounce requests that were deferred");
1375 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz),
1376 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1377 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, "");
1378 SYSCTL_ADD_UAUTO(busdma_sysctl_tree(bz),
1379 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1380 "alignment", CTLFLAG_RD, &bz->alignment, "");
1381
1382 return (0);
1383 }
1384
1385 static int
1386 alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
1387 {
1388 struct bounce_zone *bz;
1389 int count;
1390
1391 bz = dmat->bounce_zone;
1392 count = 0;
1393 while (numpages > 0) {
1394 struct bounce_page *bpage;
1395
1396 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_BUSDMA,
1397 M_NOWAIT | M_ZERO);
1398
1399 if (bpage == NULL)
1400 break;
1401 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_BOUNCE,
1402 M_NOWAIT, 0ul,
1403 bz->lowaddr,
1404 PAGE_SIZE,
1405 0);
1406 if (bpage->vaddr == 0) {
1407 free(bpage, M_BUSDMA);
1408 break;
1409 }
1410 bpage->busaddr = pmap_kextract(bpage->vaddr);
1411 bpage->vaddr_nocache =
1412 (vm_offset_t)pmap_mapdev(bpage->busaddr, PAGE_SIZE);
1413 mtx_lock(&bounce_lock);
1414 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
1415 total_bpages++;
1416 bz->total_bpages++;
1417 bz->free_bpages++;
1418 mtx_unlock(&bounce_lock);
1419 count++;
1420 numpages--;
1421 }
1422 return (count);
1423 }
1424
1425 static int
1426 reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
1427 {
1428 struct bounce_zone *bz;
1429 int pages;
1430
1431 mtx_assert(&bounce_lock, MA_OWNED);
1432 bz = dmat->bounce_zone;
1433 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved);
1434 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages))
1435 return (map->pagesneeded - (map->pagesreserved + pages));
1436 bz->free_bpages -= pages;
1437 bz->reserved_bpages += pages;
1438 map->pagesreserved += pages;
1439 pages = map->pagesneeded - map->pagesreserved;
1440
1441 return (pages);
1442 }
1443
1444 static bus_addr_t
1445 add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
1446 bus_addr_t addr, bus_size_t size)
1447 {
1448 struct bounce_zone *bz;
1449 struct bounce_page *bpage;
1450
1451 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));
1452 KASSERT(map != NULL, ("add_bounce_page: bad map %p", map));
1453
1454 bz = dmat->bounce_zone;
1455 if (map->pagesneeded == 0)
1456 panic("add_bounce_page: map doesn't need any pages");
1457 map->pagesneeded--;
1458
1459 if (map->pagesreserved == 0)
1460 panic("add_bounce_page: map doesn't need any pages");
1461 map->pagesreserved--;
1462
1463 mtx_lock(&bounce_lock);
1464 bpage = STAILQ_FIRST(&bz->bounce_page_list);
1465 if (bpage == NULL)
1466 panic("add_bounce_page: free page list is empty");
1467
1468 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
1469 bz->reserved_bpages--;
1470 bz->active_bpages++;
1471 mtx_unlock(&bounce_lock);
1472
1473 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1474 /* Page offset needs to be preserved. */
1475 bpage->vaddr |= addr & PAGE_MASK;
1476 bpage->busaddr |= addr & PAGE_MASK;
1477 }
1478 bpage->datavaddr = vaddr;
1479 bpage->dataaddr = addr;
1480 bpage->datacount = size;
1481 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
1482 return (bpage->busaddr);
1483 }
1484
1485 static void
1486 free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
1487 {
1488 struct bus_dmamap *map;
1489 struct bounce_zone *bz;
1490
1491 bz = dmat->bounce_zone;
1492 bpage->datavaddr = 0;
1493 bpage->datacount = 0;
1494 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1495 /*
1496 * Reset the bounce page to start at offset 0. Other uses
1497 * of this bounce page may need to store a full page of
1498 * data and/or assume it starts on a page boundary.
1499 */
1500 bpage->vaddr &= ~PAGE_MASK;
1501 bpage->busaddr &= ~PAGE_MASK;
1502 }
1503
1504 mtx_lock(&bounce_lock);
1505 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
1506 bz->free_bpages++;
1507 bz->active_bpages--;
1508 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
1509 if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
1510 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
1511 STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
1512 map, links);
1513 busdma_swi_pending = 1;
1514 bz->total_deferred++;
1515 swi_sched(vm_ih, 0);
1516 }
1517 }
1518 mtx_unlock(&bounce_lock);
1519 }
1520
1521 void
1522 busdma_swi(void)
1523 {
1524 bus_dma_tag_t dmat;
1525 struct bus_dmamap *map;
1526
1527 mtx_lock(&bounce_lock);
1528 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
1529 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
1530 mtx_unlock(&bounce_lock);
1531 dmat = map->dmat;
1532 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK);
1533 bus_dmamap_load_mem(map->dmat, map, &map->mem, map->callback,
1534 map->callback_arg, BUS_DMA_WAITOK);
1535 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK);
1536 mtx_lock(&bounce_lock);
1537 }
1538 mtx_unlock(&bounce_lock);
1539 }
Cache object: 1f1d2f5432ea681e99c42ee83cdad81d
|