Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/i386/i386/busdma_machdep.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 1997, 1998 Justin T. Gibbs. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions, and the following disclaimer, 10 * without modification, immediately at the beginning of the file. 11 * 2. The name of the author may not be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD: releng/8.4/sys/i386/i386/busdma_machdep.c 232354 2012-03-01 19:43:28Z emaste $"); 29 30 #include <sys/param.h> 31 #include <sys/kdb.h> 32 #include <ddb/ddb.h> 33 #include <ddb/db_output.h> 34 #include <sys/systm.h> 35 #include <sys/malloc.h> 36 #include <sys/bus.h> 37 #include <sys/interrupt.h> 38 #include <sys/kernel.h> 39 #include <sys/ktr.h> 40 #include <sys/lock.h> 41 #include <sys/proc.h> 42 #include <sys/mutex.h> 43 #include <sys/mbuf.h> 44 #include <sys/uio.h> 45 #include <sys/sysctl.h> 46 47 #include <vm/vm.h> 48 #include <vm/vm_page.h> 49 #include <vm/vm_map.h> 50 51 #include <machine/atomic.h> 52 #include <machine/bus.h> 53 #include <machine/md_var.h> 54 #include <machine/specialreg.h> 55 56 #define MAX_BPAGES 512 57 #define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 58 #define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 59 60 struct bounce_zone; 61 62 struct bus_dma_tag { 63 bus_dma_tag_t parent; 64 bus_size_t alignment; 65 bus_size_t boundary; 66 bus_addr_t lowaddr; 67 bus_addr_t highaddr; 68 bus_dma_filter_t *filter; 69 void *filterarg; 70 bus_size_t maxsize; 71 u_int nsegments; 72 bus_size_t maxsegsz; 73 int flags; 74 int ref_count; 75 int map_count; 76 bus_dma_lock_t *lockfunc; 77 void *lockfuncarg; 78 bus_dma_segment_t *segments; 79 struct bounce_zone *bounce_zone; 80 }; 81 82 struct bounce_page { 83 vm_offset_t vaddr; /* kva of bounce buffer */ 84 bus_addr_t busaddr; /* Physical address */ 85 vm_offset_t datavaddr; /* kva of client data */ 86 bus_size_t datacount; /* client data count */ 87 STAILQ_ENTRY(bounce_page) links; 88 }; 89 90 int busdma_swi_pending; 91 92 struct bounce_zone { 93 STAILQ_ENTRY(bounce_zone) links; 94 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 95 int total_bpages; 96 int free_bpages; 97 int reserved_bpages; 98 int active_bpages; 99 int total_bounced; 100 int total_deferred; 101 int map_count; 102 bus_size_t alignment; 103 bus_addr_t lowaddr; 104 char zoneid[8]; 105 char lowaddrid[20]; 106 struct sysctl_ctx_list sysctl_tree; 107 struct sysctl_oid *sysctl_tree_top; 108 }; 109 110 static struct mtx bounce_lock; 111 static int total_bpages; 112 static int busdma_zonecount; 113 static STAILQ_HEAD(, bounce_zone) bounce_zone_list; 114 115 SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 116 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 117 "Total bounce pages"); 118 119 struct bus_dmamap { 120 struct bp_list bpages; 121 int pagesneeded; 122 int pagesreserved; 123 bus_dma_tag_t dmat; 124 void *buf; /* unmapped buffer pointer */ 125 bus_size_t buflen; /* unmapped buffer length */ 126 bus_dmamap_callback_t *callback; 127 void *callback_arg; 128 STAILQ_ENTRY(bus_dmamap) links; 129 }; 130 131 static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 132 static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 133 static struct bus_dmamap nobounce_dmamap; 134 135 static void init_bounce_pages(void *dummy); 136 static int alloc_bounce_zone(bus_dma_tag_t dmat); 137 static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 138 static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 139 int commit); 140 static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 141 vm_offset_t vaddr, bus_size_t size); 142 static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 143 int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); 144 int _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 145 void *buf, bus_size_t buflen, int flags); 146 147 #ifdef XEN 148 #undef pmap_kextract 149 #define pmap_kextract pmap_kextract_ma 150 #endif 151 152 /* 153 * Return true if a match is made. 154 * 155 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. 156 * 157 * If paddr is within the bounds of the dma tag then call the filter callback 158 * to check for a match, if there is no filter callback then assume a match. 159 */ 160 int 161 run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 162 { 163 int retval; 164 165 retval = 0; 166 167 do { 168 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) 169 || ((paddr & (dmat->alignment - 1)) != 0)) 170 && (dmat->filter == NULL 171 || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 172 retval = 1; 173 174 dmat = dmat->parent; 175 } while (retval == 0 && dmat != NULL); 176 return (retval); 177 } 178 179 /* 180 * Convenience function for manipulating driver locks from busdma (during 181 * busdma_swi, for example). Drivers that don't provide their own locks 182 * should specify &Giant to dmat->lockfuncarg. Drivers that use their own 183 * non-mutex locking scheme don't have to use this at all. 184 */ 185 void 186 busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) 187 { 188 struct mtx *dmtx; 189 190 dmtx = (struct mtx *)arg; 191 switch (op) { 192 case BUS_DMA_LOCK: 193 mtx_lock(dmtx); 194 break; 195 case BUS_DMA_UNLOCK: 196 mtx_unlock(dmtx); 197 break; 198 default: 199 panic("Unknown operation 0x%x for busdma_lock_mutex!", op); 200 } 201 } 202 203 /* 204 * dflt_lock should never get called. It gets put into the dma tag when 205 * lockfunc == NULL, which is only valid if the maps that are associated 206 * with the tag are meant to never be defered. 207 * XXX Should have a way to identify which driver is responsible here. 208 */ 209 static void 210 dflt_lock(void *arg, bus_dma_lock_op_t op) 211 { 212 panic("driver error: busdma dflt_lock called"); 213 } 214 215 /* 216 * Allocate a device specific dma_tag. 217 */ 218 int 219 bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 220 bus_size_t boundary, bus_addr_t lowaddr, 221 bus_addr_t highaddr, bus_dma_filter_t *filter, 222 void *filterarg, bus_size_t maxsize, int nsegments, 223 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 224 void *lockfuncarg, bus_dma_tag_t *dmat) 225 { 226 bus_dma_tag_t newtag; 227 int error = 0; 228 229 #if defined(PAE) 230 /* Need at least a 4GB boundary, PAE limitations require 2GB */ 231 if (boundary == 0 || boundary > ((bus_addr_t)1 << 31)) 232 boundary = (bus_size_t)1 << 31; 233 #endif 234 235 /* Basic sanity checking */ 236 if (boundary != 0 && boundary < maxsegsz) 237 maxsegsz = boundary; 238 239 if (maxsegsz == 0) { 240 return (EINVAL); 241 } 242 243 /* Return a NULL tag on failure */ 244 *dmat = NULL; 245 246 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, 247 M_ZERO | M_NOWAIT); 248 if (newtag == NULL) { 249 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 250 __func__, newtag, 0, error); 251 return (ENOMEM); 252 } 253 254 newtag->parent = parent; 255 newtag->alignment = alignment; 256 newtag->boundary = boundary; 257 newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); 258 newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1); 259 newtag->filter = filter; 260 newtag->filterarg = filterarg; 261 newtag->maxsize = maxsize; 262 newtag->nsegments = nsegments; 263 newtag->maxsegsz = maxsegsz; 264 newtag->flags = flags; 265 newtag->ref_count = 1; /* Count ourself */ 266 newtag->map_count = 0; 267 if (lockfunc != NULL) { 268 newtag->lockfunc = lockfunc; 269 newtag->lockfuncarg = lockfuncarg; 270 } else { 271 newtag->lockfunc = dflt_lock; 272 newtag->lockfuncarg = NULL; 273 } 274 newtag->segments = NULL; 275 276 /* Take into account any restrictions imposed by our parent tag */ 277 if (parent != NULL) { 278 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 279 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 280 if (newtag->boundary == 0) 281 newtag->boundary = parent->boundary; 282 else if (parent->boundary != 0) 283 newtag->boundary = MIN(parent->boundary, 284 newtag->boundary); 285 if ((newtag->filter != NULL) || 286 ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0)) 287 newtag->flags |= BUS_DMA_COULD_BOUNCE; 288 if (newtag->filter == NULL) { 289 /* 290 * Short circuit looking at our parent directly 291 * since we have encapsulated all of its information 292 */ 293 newtag->filter = parent->filter; 294 newtag->filterarg = parent->filterarg; 295 newtag->parent = parent->parent; 296 } 297 if (newtag->parent != NULL) 298 atomic_add_int(&parent->ref_count, 1); 299 } 300 301 if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem) 302 || newtag->alignment > 1) 303 newtag->flags |= BUS_DMA_COULD_BOUNCE; 304 305 if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && 306 (flags & BUS_DMA_ALLOCNOW) != 0) { 307 struct bounce_zone *bz; 308 309 /* Must bounce */ 310 311 if ((error = alloc_bounce_zone(newtag)) != 0) { 312 free(newtag, M_DEVBUF); 313 return (error); 314 } 315 bz = newtag->bounce_zone; 316 317 if (ptoa(bz->total_bpages) < maxsize) { 318 int pages; 319 320 pages = atop(maxsize) - bz->total_bpages; 321 322 /* Add pages to our bounce pool */ 323 if (alloc_bounce_pages(newtag, pages) < pages) 324 error = ENOMEM; 325 } 326 /* Performed initial allocation */ 327 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 328 } 329 330 if (error != 0) { 331 free(newtag, M_DEVBUF); 332 } else { 333 *dmat = newtag; 334 } 335 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 336 __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); 337 return (error); 338 } 339 340 int 341 bus_dma_tag_destroy(bus_dma_tag_t dmat) 342 { 343 bus_dma_tag_t dmat_copy; 344 int error; 345 346 error = 0; 347 dmat_copy = dmat; 348 349 if (dmat != NULL) { 350 351 if (dmat->map_count != 0) { 352 error = EBUSY; 353 goto out; 354 } 355 356 while (dmat != NULL) { 357 bus_dma_tag_t parent; 358 359 parent = dmat->parent; 360 atomic_subtract_int(&dmat->ref_count, 1); 361 if (dmat->ref_count == 0) { 362 if (dmat->segments != NULL) 363 free(dmat->segments, M_DEVBUF); 364 free(dmat, M_DEVBUF); 365 /* 366 * Last reference count, so 367 * release our reference 368 * count on our parent. 369 */ 370 dmat = parent; 371 } else 372 dmat = NULL; 373 } 374 } 375 out: 376 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); 377 return (error); 378 } 379 380 /* 381 * Allocate a handle for mapping from kva/uva/physical 382 * address space into bus device space. 383 */ 384 int 385 bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 386 { 387 int error; 388 389 error = 0; 390 391 if (dmat->segments == NULL) { 392 dmat->segments = (bus_dma_segment_t *)malloc( 393 sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, 394 M_NOWAIT); 395 if (dmat->segments == NULL) { 396 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 397 __func__, dmat, ENOMEM); 398 return (ENOMEM); 399 } 400 } 401 402 /* 403 * Bouncing might be required if the driver asks for an active 404 * exclusion region, a data alignment that is stricter than 1, and/or 405 * an active address boundary. 406 */ 407 if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 408 409 /* Must bounce */ 410 struct bounce_zone *bz; 411 int maxpages; 412 413 if (dmat->bounce_zone == NULL) { 414 if ((error = alloc_bounce_zone(dmat)) != 0) 415 return (error); 416 } 417 bz = dmat->bounce_zone; 418 419 *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF, 420 M_NOWAIT | M_ZERO); 421 if (*mapp == NULL) { 422 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 423 __func__, dmat, ENOMEM); 424 return (ENOMEM); 425 } 426 427 /* Initialize the new map */ 428 STAILQ_INIT(&((*mapp)->bpages)); 429 430 /* 431 * Attempt to add pages to our pool on a per-instance 432 * basis up to a sane limit. 433 */ 434 if (dmat->alignment > 1) 435 maxpages = MAX_BPAGES; 436 else 437 maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->lowaddr)); 438 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 439 || (bz->map_count > 0 && bz->total_bpages < maxpages)) { 440 int pages; 441 442 pages = MAX(atop(dmat->maxsize), 1); 443 pages = MIN(maxpages - bz->total_bpages, pages); 444 pages = MAX(pages, 1); 445 if (alloc_bounce_pages(dmat, pages) < pages) 446 error = ENOMEM; 447 448 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 449 if (error == 0) 450 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 451 } else { 452 error = 0; 453 } 454 } 455 bz->map_count++; 456 } else { 457 *mapp = NULL; 458 } 459 if (error == 0) 460 dmat->map_count++; 461 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 462 __func__, dmat, dmat->flags, error); 463 return (error); 464 } 465 466 /* 467 * Destroy a handle for mapping from kva/uva/physical 468 * address space into bus device space. 469 */ 470 int 471 bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 472 { 473 if (map != NULL && map != &nobounce_dmamap) { 474 if (STAILQ_FIRST(&map->bpages) != NULL) { 475 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 476 __func__, dmat, EBUSY); 477 return (EBUSY); 478 } 479 if (dmat->bounce_zone) 480 dmat->bounce_zone->map_count--; 481 free(map, M_DEVBUF); 482 } 483 dmat->map_count--; 484 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 485 return (0); 486 } 487 488 489 /* 490 * Allocate a piece of memory that can be efficiently mapped into 491 * bus device space based on the constraints lited in the dma tag. 492 * A dmamap to for use with dmamap_load is also allocated. 493 */ 494 int 495 bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 496 bus_dmamap_t *mapp) 497 { 498 int mflags; 499 500 if (flags & BUS_DMA_NOWAIT) 501 mflags = M_NOWAIT; 502 else 503 mflags = M_WAITOK; 504 505 /* If we succeed, no mapping/bouncing will be required */ 506 *mapp = NULL; 507 508 if (dmat->segments == NULL) { 509 dmat->segments = (bus_dma_segment_t *)malloc( 510 sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, 511 mflags); 512 if (dmat->segments == NULL) { 513 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 514 __func__, dmat, dmat->flags, ENOMEM); 515 return (ENOMEM); 516 } 517 } 518 if (flags & BUS_DMA_ZERO) 519 mflags |= M_ZERO; 520 521 /* 522 * XXX: 523 * (dmat->alignment < dmat->maxsize) is just a quick hack; the exact 524 * alignment guarantees of malloc need to be nailed down, and the 525 * code below should be rewritten to take that into account. 526 * 527 * In the meantime, we'll warn the user if malloc gets it wrong. 528 */ 529 if ((dmat->maxsize <= PAGE_SIZE) && 530 (dmat->alignment < dmat->maxsize) && 531 dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) { 532 *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags); 533 } else { 534 /* 535 * XXX Use Contigmalloc until it is merged into this facility 536 * and handles multi-seg allocations. Nobody is doing 537 * multi-seg allocations yet though. 538 * XXX Certain AGP hardware does. 539 */ 540 *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags, 541 0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul, 542 dmat->boundary); 543 } 544 if (*vaddr == NULL) { 545 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 546 __func__, dmat, dmat->flags, ENOMEM); 547 return (ENOMEM); 548 } else if ((uintptr_t)*vaddr & (dmat->alignment - 1)) { 549 printf("bus_dmamem_alloc failed to align memory properly.\n"); 550 } 551 if (flags & BUS_DMA_NOCACHE) 552 pmap_change_attr((vm_offset_t)*vaddr, dmat->maxsize, 553 PAT_UNCACHEABLE); 554 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 555 __func__, dmat, dmat->flags, 0); 556 return (0); 557 } 558 559 /* 560 * Free a piece of memory and it's allociated dmamap, that was allocated 561 * via bus_dmamem_alloc. Make the same choice for free/contigfree. 562 */ 563 void 564 bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 565 { 566 /* 567 * dmamem does not need to be bounced, so the map should be 568 * NULL 569 */ 570 if (map != NULL) 571 panic("bus_dmamem_free: Invalid map freed\n"); 572 pmap_change_attr((vm_offset_t)vaddr, dmat->maxsize, PAT_WRITE_BACK); 573 if ((dmat->maxsize <= PAGE_SIZE) && 574 (dmat->alignment < dmat->maxsize) && 575 dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) 576 free(vaddr, M_DEVBUF); 577 else { 578 contigfree(vaddr, dmat->maxsize, M_DEVBUF); 579 } 580 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); 581 } 582 583 int 584 _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 585 void *buf, bus_size_t buflen, int flags) 586 { 587 vm_offset_t vaddr; 588 vm_offset_t vendaddr; 589 bus_addr_t paddr; 590 591 if ((map != &nobounce_dmamap && map->pagesneeded == 0)) { 592 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " 593 "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem), 594 dmat->boundary, dmat->alignment); 595 CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d", 596 map, &nobounce_dmamap, map->pagesneeded); 597 /* 598 * Count the number of bounce pages 599 * needed in order to complete this transfer 600 */ 601 vaddr = (vm_offset_t)buf; 602 vendaddr = (vm_offset_t)buf + buflen; 603 604 while (vaddr < vendaddr) { 605 bus_size_t sg_len; 606 607 sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK); 608 if (pmap) 609 paddr = pmap_extract(pmap, vaddr); 610 else 611 paddr = pmap_kextract(vaddr); 612 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 613 run_filter(dmat, paddr) != 0) { 614 sg_len = roundup2(sg_len, dmat->alignment); 615 map->pagesneeded++; 616 } 617 vaddr += sg_len; 618 } 619 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 620 } 621 622 /* Reserve Necessary Bounce Pages */ 623 if (map->pagesneeded != 0) { 624 mtx_lock(&bounce_lock); 625 if (flags & BUS_DMA_NOWAIT) { 626 if (reserve_bounce_pages(dmat, map, 0) != 0) { 627 mtx_unlock(&bounce_lock); 628 return (ENOMEM); 629 } 630 } else { 631 if (reserve_bounce_pages(dmat, map, 1) != 0) { 632 /* Queue us for resources */ 633 map->dmat = dmat; 634 map->buf = buf; 635 map->buflen = buflen; 636 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, 637 map, links); 638 mtx_unlock(&bounce_lock); 639 return (EINPROGRESS); 640 } 641 } 642 mtx_unlock(&bounce_lock); 643 } 644 645 return (0); 646 } 647 648 /* 649 * Utility function to load a linear buffer. lastaddrp holds state 650 * between invocations (for multiple-buffer loads). segp contains 651 * the starting segment on entrace, and the ending segment on exit. 652 * first indicates if this is the first invocation of this function. 653 */ 654 static __inline int 655 _bus_dmamap_load_buffer(bus_dma_tag_t dmat, 656 bus_dmamap_t map, 657 void *buf, bus_size_t buflen, 658 pmap_t pmap, 659 int flags, 660 bus_addr_t *lastaddrp, 661 bus_dma_segment_t *segs, 662 int *segp, 663 int first) 664 { 665 bus_size_t sgsize; 666 bus_addr_t curaddr, lastaddr, baddr, bmask; 667 vm_offset_t vaddr; 668 int seg, error; 669 670 if (map == NULL) 671 map = &nobounce_dmamap; 672 673 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { 674 error = _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); 675 if (error) 676 return (error); 677 } 678 679 vaddr = (vm_offset_t)buf; 680 lastaddr = *lastaddrp; 681 bmask = ~(dmat->boundary - 1); 682 683 for (seg = *segp; buflen > 0 ; ) { 684 bus_size_t max_sgsize; 685 686 /* 687 * Get the physical address for this segment. 688 */ 689 if (pmap) 690 curaddr = pmap_extract(pmap, vaddr); 691 else 692 curaddr = pmap_kextract(vaddr); 693 694 /* 695 * Compute the segment size, and adjust counts. 696 */ 697 max_sgsize = MIN(buflen, dmat->maxsegsz); 698 sgsize = PAGE_SIZE - ((vm_offset_t)curaddr & PAGE_MASK); 699 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && 700 map->pagesneeded != 0 && run_filter(dmat, curaddr)) { 701 sgsize = roundup2(sgsize, dmat->alignment); 702 sgsize = MIN(sgsize, max_sgsize); 703 curaddr = add_bounce_page(dmat, map, vaddr, sgsize); 704 } else { 705 sgsize = MIN(sgsize, max_sgsize); 706 } 707 708 /* 709 * Make sure we don't cross any boundaries. 710 */ 711 if (dmat->boundary > 0) { 712 baddr = (curaddr + dmat->boundary) & bmask; 713 if (sgsize > (baddr - curaddr)) 714 sgsize = (baddr - curaddr); 715 } 716 717 /* 718 * Insert chunk into a segment, coalescing with 719 * previous segment if possible. 720 */ 721 if (first) { 722 segs[seg].ds_addr = curaddr; 723 segs[seg].ds_len = sgsize; 724 first = 0; 725 } else { 726 if (curaddr == lastaddr && 727 (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && 728 (dmat->boundary == 0 || 729 (segs[seg].ds_addr & bmask) == (curaddr & bmask))) 730 segs[seg].ds_len += sgsize; 731 else { 732 if (++seg >= dmat->nsegments) 733 break; 734 segs[seg].ds_addr = curaddr; 735 segs[seg].ds_len = sgsize; 736 } 737 } 738 739 lastaddr = curaddr + sgsize; 740 vaddr += sgsize; 741 buflen -= sgsize; 742 } 743 744 *segp = seg; 745 *lastaddrp = lastaddr; 746 747 /* 748 * Did we fit? 749 */ 750 return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ 751 } 752 753 /* 754 * Map the buffer buf into bus space using the dmamap map. 755 */ 756 int 757 bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 758 bus_size_t buflen, bus_dmamap_callback_t *callback, 759 void *callback_arg, int flags) 760 { 761 bus_addr_t lastaddr = 0; 762 int error, nsegs = 0; 763 764 if (map != NULL) { 765 flags |= BUS_DMA_WAITOK; 766 map->callback = callback; 767 map->callback_arg = callback_arg; 768 } 769 770 error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, NULL, flags, 771 &lastaddr, dmat->segments, &nsegs, 1); 772 773 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 774 __func__, dmat, dmat->flags, error, nsegs + 1); 775 776 if (error == EINPROGRESS) { 777 return (error); 778 } 779 780 if (error) 781 (*callback)(callback_arg, dmat->segments, 0, error); 782 else 783 (*callback)(callback_arg, dmat->segments, nsegs + 1, 0); 784 785 /* 786 * Return ENOMEM to the caller so that it can pass it up the stack. 787 * This error only happens when NOWAIT is set, so deferal is disabled. 788 */ 789 if (error == ENOMEM) 790 return (error); 791 792 return (0); 793 } 794 795 796 /* 797 * Like _bus_dmamap_load(), but for mbufs. 798 */ 799 static __inline int 800 _bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 801 struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, 802 int flags) 803 { 804 int error; 805 806 M_ASSERTPKTHDR(m0); 807 808 flags |= BUS_DMA_NOWAIT; 809 *nsegs = 0; 810 error = 0; 811 if (m0->m_pkthdr.len <= dmat->maxsize) { 812 int first = 1; 813 bus_addr_t lastaddr = 0; 814 struct mbuf *m; 815 816 for (m = m0; m != NULL && error == 0; m = m->m_next) { 817 if (m->m_len > 0) { 818 error = _bus_dmamap_load_buffer(dmat, map, 819 m->m_data, m->m_len, 820 NULL, flags, &lastaddr, 821 segs, nsegs, first); 822 first = 0; 823 } 824 } 825 } else { 826 error = EINVAL; 827 } 828 829 /* XXX FIXME: Having to increment nsegs is really annoying */ 830 ++*nsegs; 831 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 832 __func__, dmat, dmat->flags, error, *nsegs); 833 return (error); 834 } 835 836 int 837 bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, 838 struct mbuf *m0, 839 bus_dmamap_callback2_t *callback, void *callback_arg, 840 int flags) 841 { 842 int nsegs, error; 843 844 error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, dmat->segments, &nsegs, 845 flags); 846 847 if (error) { 848 /* force "no valid mappings" in callback */ 849 (*callback)(callback_arg, dmat->segments, 0, 0, error); 850 } else { 851 (*callback)(callback_arg, dmat->segments, 852 nsegs, m0->m_pkthdr.len, error); 853 } 854 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 855 __func__, dmat, dmat->flags, error, nsegs); 856 return (error); 857 } 858 859 int 860 bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 861 struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, 862 int flags) 863 { 864 return (_bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, nsegs, flags)); 865 } 866 867 /* 868 * Like _bus_dmamap_load(), but for uios. 869 */ 870 int 871 bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, 872 struct uio *uio, 873 bus_dmamap_callback2_t *callback, void *callback_arg, 874 int flags) 875 { 876 bus_addr_t lastaddr; 877 int nsegs, error, first, i; 878 bus_size_t resid; 879 struct iovec *iov; 880 pmap_t pmap; 881 882 flags |= BUS_DMA_NOWAIT; 883 resid = uio->uio_resid; 884 iov = uio->uio_iov; 885 886 if (uio->uio_segflg == UIO_USERSPACE) { 887 KASSERT(uio->uio_td != NULL, 888 ("bus_dmamap_load_uio: USERSPACE but no proc")); 889 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); 890 } else 891 pmap = NULL; 892 893 nsegs = 0; 894 error = 0; 895 first = 1; 896 lastaddr = (bus_addr_t) 0; 897 for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 898 /* 899 * Now at the first iovec to load. Load each iovec 900 * until we have exhausted the residual count. 901 */ 902 bus_size_t minlen = 903 resid < iov[i].iov_len ? resid : iov[i].iov_len; 904 caddr_t addr = (caddr_t) iov[i].iov_base; 905 906 if (minlen > 0) { 907 error = _bus_dmamap_load_buffer(dmat, map, 908 addr, minlen, pmap, flags, &lastaddr, 909 dmat->segments, &nsegs, first); 910 first = 0; 911 912 resid -= minlen; 913 } 914 } 915 916 if (error) { 917 /* force "no valid mappings" in callback */ 918 (*callback)(callback_arg, dmat->segments, 0, 0, error); 919 } else { 920 (*callback)(callback_arg, dmat->segments, 921 nsegs+1, uio->uio_resid, error); 922 } 923 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 924 __func__, dmat, dmat->flags, error, nsegs + 1); 925 return (error); 926 } 927 928 /* 929 * Release the mapping held by map. 930 */ 931 void 932 _bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 933 { 934 struct bounce_page *bpage; 935 936 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 937 STAILQ_REMOVE_HEAD(&map->bpages, links); 938 free_bounce_page(dmat, bpage); 939 } 940 } 941 942 void 943 _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 944 { 945 struct bounce_page *bpage; 946 947 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 948 /* 949 * Handle data bouncing. We might also 950 * want to add support for invalidating 951 * the caches on broken hardware 952 */ 953 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " 954 "performing bounce", __func__, op, dmat, dmat->flags); 955 956 if (op & BUS_DMASYNC_PREWRITE) { 957 while (bpage != NULL) { 958 bcopy((void *)bpage->datavaddr, 959 (void *)bpage->vaddr, 960 bpage->datacount); 961 bpage = STAILQ_NEXT(bpage, links); 962 } 963 dmat->bounce_zone->total_bounced++; 964 } 965 966 if (op & BUS_DMASYNC_POSTREAD) { 967 while (bpage != NULL) { 968 bcopy((void *)bpage->vaddr, 969 (void *)bpage->datavaddr, 970 bpage->datacount); 971 bpage = STAILQ_NEXT(bpage, links); 972 } 973 dmat->bounce_zone->total_bounced++; 974 } 975 } 976 } 977 978 static void 979 init_bounce_pages(void *dummy __unused) 980 { 981 982 total_bpages = 0; 983 STAILQ_INIT(&bounce_zone_list); 984 STAILQ_INIT(&bounce_map_waitinglist); 985 STAILQ_INIT(&bounce_map_callbacklist); 986 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 987 } 988 SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 989 990 static struct sysctl_ctx_list * 991 busdma_sysctl_tree(struct bounce_zone *bz) 992 { 993 return (&bz->sysctl_tree); 994 } 995 996 static struct sysctl_oid * 997 busdma_sysctl_tree_top(struct bounce_zone *bz) 998 { 999 return (bz->sysctl_tree_top); 1000 } 1001 1002 static int 1003 alloc_bounce_zone(bus_dma_tag_t dmat) 1004 { 1005 struct bounce_zone *bz; 1006 1007 /* Check to see if we already have a suitable zone */ 1008 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1009 if ((dmat->alignment <= bz->alignment) 1010 && (dmat->lowaddr >= bz->lowaddr)) { 1011 dmat->bounce_zone = bz; 1012 return (0); 1013 } 1014 } 1015 1016 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 1017 M_NOWAIT | M_ZERO)) == NULL) 1018 return (ENOMEM); 1019 1020 STAILQ_INIT(&bz->bounce_page_list); 1021 bz->free_bpages = 0; 1022 bz->reserved_bpages = 0; 1023 bz->active_bpages = 0; 1024 bz->lowaddr = dmat->lowaddr; 1025 bz->alignment = MAX(dmat->alignment, PAGE_SIZE); 1026 bz->map_count = 0; 1027 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1028 busdma_zonecount++; 1029 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1030 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1031 dmat->bounce_zone = bz; 1032 1033 sysctl_ctx_init(&bz->sysctl_tree); 1034 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1035 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1036 CTLFLAG_RD, 0, ""); 1037 if (bz->sysctl_tree_top == NULL) { 1038 sysctl_ctx_free(&bz->sysctl_tree); 1039 return (0); /* XXX error code? */ 1040 } 1041 1042 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1043 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1044 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1045 "Total bounce pages"); 1046 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1047 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1048 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1049 "Free bounce pages"); 1050 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1051 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1052 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1053 "Reserved bounce pages"); 1054 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1055 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1056 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1057 "Active bounce pages"); 1058 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1059 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1060 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1061 "Total bounce requests"); 1062 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1063 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1064 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1065 "Total bounce requests that were deferred"); 1066 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1067 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1068 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1069 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1070 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1071 "alignment", CTLFLAG_RD, &bz->alignment, 0, ""); 1072 1073 return (0); 1074 } 1075 1076 static int 1077 alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1078 { 1079 struct bounce_zone *bz; 1080 int count; 1081 1082 bz = dmat->bounce_zone; 1083 count = 0; 1084 while (numpages > 0) { 1085 struct bounce_page *bpage; 1086 1087 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 1088 M_NOWAIT | M_ZERO); 1089 1090 if (bpage == NULL) 1091 break; 1092 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1093 M_NOWAIT, 0ul, 1094 bz->lowaddr, 1095 PAGE_SIZE, 1096 0); 1097 if (bpage->vaddr == 0) { 1098 free(bpage, M_DEVBUF); 1099 break; 1100 } 1101 bpage->busaddr = pmap_kextract(bpage->vaddr); 1102 mtx_lock(&bounce_lock); 1103 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1104 total_bpages++; 1105 bz->total_bpages++; 1106 bz->free_bpages++; 1107 mtx_unlock(&bounce_lock); 1108 count++; 1109 numpages--; 1110 } 1111 return (count); 1112 } 1113 1114 static int 1115 reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1116 { 1117 struct bounce_zone *bz; 1118 int pages; 1119 1120 mtx_assert(&bounce_lock, MA_OWNED); 1121 bz = dmat->bounce_zone; 1122 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1123 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1124 return (map->pagesneeded - (map->pagesreserved + pages)); 1125 bz->free_bpages -= pages; 1126 bz->reserved_bpages += pages; 1127 map->pagesreserved += pages; 1128 pages = map->pagesneeded - map->pagesreserved; 1129 1130 return (pages); 1131 } 1132 1133 static bus_addr_t 1134 add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1135 bus_size_t size) 1136 { 1137 struct bounce_zone *bz; 1138 struct bounce_page *bpage; 1139 1140 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1141 KASSERT(map != NULL && map != &nobounce_dmamap, 1142 ("add_bounce_page: bad map %p", map)); 1143 1144 bz = dmat->bounce_zone; 1145 if (map->pagesneeded == 0) 1146 panic("add_bounce_page: map doesn't need any pages"); 1147 map->pagesneeded--; 1148 1149 if (map->pagesreserved == 0) 1150 panic("add_bounce_page: map doesn't need any pages"); 1151 map->pagesreserved--; 1152 1153 mtx_lock(&bounce_lock); 1154 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1155 if (bpage == NULL) 1156 panic("add_bounce_page: free page list is empty"); 1157 1158 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1159 bz->reserved_bpages--; 1160 bz->active_bpages++; 1161 mtx_unlock(&bounce_lock); 1162 1163 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1164 /* Page offset needs to be preserved. */ 1165 bpage->vaddr |= vaddr & PAGE_MASK; 1166 bpage->busaddr |= vaddr & PAGE_MASK; 1167 } 1168 bpage->datavaddr = vaddr; 1169 bpage->datacount = size; 1170 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1171 return (bpage->busaddr); 1172 } 1173 1174 static void 1175 free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1176 { 1177 struct bus_dmamap *map; 1178 struct bounce_zone *bz; 1179 1180 bz = dmat->bounce_zone; 1181 bpage->datavaddr = 0; 1182 bpage->datacount = 0; 1183 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1184 /* 1185 * Reset the bounce page to start at offset 0. Other uses 1186 * of this bounce page may need to store a full page of 1187 * data and/or assume it starts on a page boundary. 1188 */ 1189 bpage->vaddr &= ~PAGE_MASK; 1190 bpage->busaddr &= ~PAGE_MASK; 1191 } 1192 1193 mtx_lock(&bounce_lock); 1194 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1195 bz->free_bpages++; 1196 bz->active_bpages--; 1197 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1198 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1199 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1200 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1201 map, links); 1202 busdma_swi_pending = 1; 1203 bz->total_deferred++; 1204 swi_sched(vm_ih, 0); 1205 } 1206 } 1207 mtx_unlock(&bounce_lock); 1208 } 1209 1210 void 1211 busdma_swi(void) 1212 { 1213 bus_dma_tag_t dmat; 1214 struct bus_dmamap *map; 1215 1216 mtx_lock(&bounce_lock); 1217 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1218 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1219 mtx_unlock(&bounce_lock); 1220 dmat = map->dmat; 1221 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); 1222 bus_dmamap_load(map->dmat, map, map->buf, map->buflen, 1223 map->callback, map->callback_arg, /*flags*/0); 1224 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); 1225 mtx_lock(&bounce_lock); 1226 } 1227 mtx_unlock(&bounce_lock); 1228 }
Cache object: 93a569e4f3adcd99aec37501561a5f0b
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]