Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/x86/iommu/busdma_dmar.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 2013 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 6 * under sponsorship from the FreeBSD Foundation. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/malloc.h> 36 #include <sys/bus.h> 37 #include <sys/conf.h> 38 #include <sys/interrupt.h> 39 #include <sys/kernel.h> 40 #include <sys/ktr.h> 41 #include <sys/lock.h> 42 #include <sys/proc.h> 43 #include <sys/memdesc.h> 44 #include <sys/mutex.h> 45 #include <sys/sysctl.h> 46 #include <sys/rman.h> 47 #include <sys/taskqueue.h> 48 #include <sys/tree.h> 49 #include <sys/uio.h> 50 #include <sys/vmem.h> 51 #include <dev/pci/pcireg.h> 52 #include <dev/pci/pcivar.h> 53 #include <vm/vm.h> 54 #include <vm/vm_extern.h> 55 #include <vm/vm_kern.h> 56 #include <vm/vm_object.h> 57 #include <vm/vm_page.h> 58 #include <vm/vm_map.h> 59 #include <machine/atomic.h> 60 #include <machine/bus.h> 61 #include <machine/md_var.h> 62 #include <machine/specialreg.h> 63 #include <x86/include/busdma_impl.h> 64 #include <x86/iommu/intel_reg.h> 65 #include <x86/iommu/busdma_dmar.h> 66 #include <x86/iommu/intel_dmar.h> 67 68 /* 69 * busdma_dmar.c, the implementation of the busdma(9) interface using 70 * DMAR units from Intel VT-d. 71 */ 72 73 static bool 74 dmar_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func) 75 { 76 char str[128], *env; 77 int default_bounce; 78 bool ret; 79 static const char bounce_str[] = "bounce"; 80 static const char dmar_str[] = "dmar"; 81 82 default_bounce = 0; 83 env = kern_getenv("hw.busdma.default"); 84 if (env != NULL) { 85 if (strcmp(env, bounce_str) == 0) 86 default_bounce = 1; 87 else if (strcmp(env, dmar_str) == 0) 88 default_bounce = 0; 89 freeenv(env); 90 } 91 92 snprintf(str, sizeof(str), "hw.busdma.pci%d.%d.%d.%d", 93 domain, bus, slot, func); 94 env = kern_getenv(str); 95 if (env == NULL) 96 return (default_bounce != 0); 97 if (strcmp(env, bounce_str) == 0) 98 ret = true; 99 else if (strcmp(env, dmar_str) == 0) 100 ret = false; 101 else 102 ret = default_bounce != 0; 103 freeenv(env); 104 return (ret); 105 } 106 107 /* 108 * Given original device, find the requester ID that will be seen by 109 * the DMAR unit and used for page table lookup. PCI bridges may take 110 * ownership of transactions from downstream devices, so it may not be 111 * the same as the BSF of the target device. In those cases, all 112 * devices downstream of the bridge must share a single mapping 113 * domain, and must collectively be assigned to use either DMAR or 114 * bounce mapping. 115 */ 116 device_t 117 dmar_get_requester(device_t dev, uint16_t *rid) 118 { 119 devclass_t pci_class; 120 device_t l, pci, pcib, pcip, pcibp, requester; 121 int cap_offset; 122 uint16_t pcie_flags; 123 bool bridge_is_pcie; 124 125 pci_class = devclass_find("pci"); 126 l = requester = dev; 127 128 *rid = pci_get_rid(dev); 129 130 /* 131 * Walk the bridge hierarchy from the target device to the 132 * host port to find the translating bridge nearest the DMAR 133 * unit. 134 */ 135 for (;;) { 136 pci = device_get_parent(l); 137 KASSERT(pci != NULL, ("dmar_get_requester(%s): NULL parent " 138 "for %s", device_get_name(dev), device_get_name(l))); 139 KASSERT(device_get_devclass(pci) == pci_class, 140 ("dmar_get_requester(%s): non-pci parent %s for %s", 141 device_get_name(dev), device_get_name(pci), 142 device_get_name(l))); 143 144 pcib = device_get_parent(pci); 145 KASSERT(pcib != NULL, ("dmar_get_requester(%s): NULL bridge " 146 "for %s", device_get_name(dev), device_get_name(pci))); 147 148 /* 149 * The parent of our "bridge" isn't another PCI bus, 150 * so pcib isn't a PCI->PCI bridge but rather a host 151 * port, and the requester ID won't be translated 152 * further. 153 */ 154 pcip = device_get_parent(pcib); 155 if (device_get_devclass(pcip) != pci_class) 156 break; 157 pcibp = device_get_parent(pcip); 158 159 if (pci_find_cap(l, PCIY_EXPRESS, &cap_offset) == 0) { 160 /* 161 * Do not stop the loop even if the target 162 * device is PCIe, because it is possible (but 163 * unlikely) to have a PCI->PCIe bridge 164 * somewhere in the hierarchy. 165 */ 166 l = pcib; 167 } else { 168 /* 169 * Device is not PCIe, it cannot be seen as a 170 * requester by DMAR unit. Check whether the 171 * bridge is PCIe. 172 */ 173 bridge_is_pcie = pci_find_cap(pcib, PCIY_EXPRESS, 174 &cap_offset) == 0; 175 requester = pcib; 176 177 /* 178 * Check for a buggy PCIe/PCI bridge that 179 * doesn't report the express capability. If 180 * the bridge above it is express but isn't a 181 * PCI bridge, then we know pcib is actually a 182 * PCIe/PCI bridge. 183 */ 184 if (!bridge_is_pcie && pci_find_cap(pcibp, 185 PCIY_EXPRESS, &cap_offset) == 0) { 186 pcie_flags = pci_read_config(pcibp, 187 cap_offset + PCIER_FLAGS, 2); 188 if ((pcie_flags & PCIEM_FLAGS_TYPE) != 189 PCIEM_TYPE_PCI_BRIDGE) 190 bridge_is_pcie = true; 191 } 192 193 if (bridge_is_pcie) { 194 /* 195 * The current device is not PCIe, but 196 * the bridge above it is. This is a 197 * PCIe->PCI bridge. Assume that the 198 * requester ID will be the secondary 199 * bus number with slot and function 200 * set to zero. 201 * 202 * XXX: Doesn't handle the case where 203 * the bridge is PCIe->PCI-X, and the 204 * bridge will only take ownership of 205 * requests in some cases. We should 206 * provide context entries with the 207 * same page tables for taken and 208 * non-taken transactions. 209 */ 210 *rid = PCI_RID(pci_get_bus(l), 0, 0); 211 l = pcibp; 212 } else { 213 /* 214 * Neither the device nor the bridge 215 * above it are PCIe. This is a 216 * conventional PCI->PCI bridge, which 217 * will use the bridge's BSF as the 218 * requester ID. 219 */ 220 *rid = pci_get_rid(pcib); 221 l = pcib; 222 } 223 } 224 } 225 return (requester); 226 } 227 228 struct dmar_ctx * 229 dmar_instantiate_ctx(struct dmar_unit *dmar, device_t dev, bool rmrr) 230 { 231 device_t requester; 232 struct dmar_ctx *ctx; 233 bool disabled; 234 uint16_t rid; 235 236 requester = dmar_get_requester(dev, &rid); 237 238 /* 239 * If the user requested the IOMMU disabled for the device, we 240 * cannot disable the DMAR, due to possibility of other 241 * devices on the same DMAR still requiring translation. 242 * Instead provide the identity mapping for the device 243 * context. 244 */ 245 disabled = dmar_bus_dma_is_dev_disabled(pci_get_domain(requester), 246 pci_get_bus(requester), pci_get_slot(requester), 247 pci_get_function(requester)); 248 ctx = dmar_get_ctx_for_dev(dmar, requester, rid, disabled, rmrr); 249 if (ctx == NULL) 250 return (NULL); 251 if (disabled) { 252 /* 253 * Keep the first reference on context, release the 254 * later refs. 255 */ 256 DMAR_LOCK(dmar); 257 if ((ctx->flags & DMAR_CTX_DISABLED) == 0) { 258 ctx->flags |= DMAR_CTX_DISABLED; 259 DMAR_UNLOCK(dmar); 260 } else { 261 dmar_free_ctx_locked(dmar, ctx); 262 } 263 ctx = NULL; 264 } 265 return (ctx); 266 } 267 268 bus_dma_tag_t 269 dmar_get_dma_tag(device_t dev, device_t child) 270 { 271 struct dmar_unit *dmar; 272 struct dmar_ctx *ctx; 273 bus_dma_tag_t res; 274 275 dmar = dmar_find(child); 276 /* Not in scope of any DMAR ? */ 277 if (dmar == NULL) 278 return (NULL); 279 if (!dmar->dma_enabled) 280 return (NULL); 281 dmar_quirks_pre_use(dmar); 282 dmar_instantiate_rmrr_ctxs(dmar); 283 284 ctx = dmar_instantiate_ctx(dmar, child, false); 285 res = ctx == NULL ? NULL : (bus_dma_tag_t)&ctx->ctx_tag; 286 return (res); 287 } 288 289 static MALLOC_DEFINE(M_DMAR_DMAMAP, "dmar_dmamap", "Intel DMAR DMA Map"); 290 291 static void dmar_bus_schedule_dmamap(struct dmar_unit *unit, 292 struct bus_dmamap_dmar *map); 293 294 static int 295 dmar_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 296 bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, 297 bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, 298 int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 299 void *lockfuncarg, bus_dma_tag_t *dmat) 300 { 301 struct bus_dma_tag_dmar *newtag, *oldtag; 302 int error; 303 304 *dmat = NULL; 305 error = common_bus_dma_tag_create(parent != NULL ? 306 &((struct bus_dma_tag_dmar *)parent)->common : NULL, alignment, 307 boundary, lowaddr, highaddr, filter, filterarg, maxsize, 308 nsegments, maxsegsz, flags, lockfunc, lockfuncarg, 309 sizeof(struct bus_dma_tag_dmar), (void **)&newtag); 310 if (error != 0) 311 goto out; 312 313 oldtag = (struct bus_dma_tag_dmar *)parent; 314 newtag->common.impl = &bus_dma_dmar_impl; 315 newtag->ctx = oldtag->ctx; 316 newtag->owner = oldtag->owner; 317 318 *dmat = (bus_dma_tag_t)newtag; 319 out: 320 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 321 __func__, newtag, (newtag != NULL ? newtag->common.flags : 0), 322 error); 323 return (error); 324 } 325 326 static int 327 dmar_bus_dma_tag_destroy(bus_dma_tag_t dmat1) 328 { 329 struct bus_dma_tag_dmar *dmat, *dmat_copy, *parent; 330 int error; 331 332 error = 0; 333 dmat_copy = dmat = (struct bus_dma_tag_dmar *)dmat1; 334 335 if (dmat != NULL) { 336 if (dmat->map_count != 0) { 337 error = EBUSY; 338 goto out; 339 } 340 while (dmat != NULL) { 341 parent = (struct bus_dma_tag_dmar *)dmat->common.parent; 342 if (atomic_fetchadd_int(&dmat->common.ref_count, -1) == 343 1) { 344 if (dmat == &dmat->ctx->ctx_tag) 345 dmar_free_ctx(dmat->ctx); 346 free(dmat->segments, M_DMAR_DMAMAP); 347 free(dmat, M_DEVBUF); 348 dmat = parent; 349 } else 350 dmat = NULL; 351 } 352 } 353 out: 354 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); 355 return (error); 356 } 357 358 static int 359 dmar_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 360 { 361 struct bus_dma_tag_dmar *tag; 362 struct bus_dmamap_dmar *map; 363 364 tag = (struct bus_dma_tag_dmar *)dmat; 365 map = malloc(sizeof(*map), M_DMAR_DMAMAP, M_NOWAIT | M_ZERO); 366 if (map == NULL) { 367 *mapp = NULL; 368 return (ENOMEM); 369 } 370 if (tag->segments == NULL) { 371 tag->segments = malloc(sizeof(bus_dma_segment_t) * 372 tag->common.nsegments, M_DMAR_DMAMAP, M_NOWAIT); 373 if (tag->segments == NULL) { 374 free(map, M_DMAR_DMAMAP); 375 *mapp = NULL; 376 return (ENOMEM); 377 } 378 } 379 TAILQ_INIT(&map->map_entries); 380 map->tag = tag; 381 map->locked = true; 382 map->cansleep = false; 383 tag->map_count++; 384 *mapp = (bus_dmamap_t)map; 385 386 return (0); 387 } 388 389 static int 390 dmar_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map1) 391 { 392 struct bus_dma_tag_dmar *tag; 393 struct bus_dmamap_dmar *map; 394 struct dmar_domain *domain; 395 396 tag = (struct bus_dma_tag_dmar *)dmat; 397 map = (struct bus_dmamap_dmar *)map1; 398 if (map != NULL) { 399 domain = tag->ctx->domain; 400 DMAR_DOMAIN_LOCK(domain); 401 if (!TAILQ_EMPTY(&map->map_entries)) { 402 DMAR_DOMAIN_UNLOCK(domain); 403 return (EBUSY); 404 } 405 DMAR_DOMAIN_UNLOCK(domain); 406 free(map, M_DMAR_DMAMAP); 407 } 408 tag->map_count--; 409 return (0); 410 } 411 412 413 static int 414 dmar_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 415 bus_dmamap_t *mapp) 416 { 417 struct bus_dma_tag_dmar *tag; 418 struct bus_dmamap_dmar *map; 419 int error, mflags; 420 vm_memattr_t attr; 421 422 error = dmar_bus_dmamap_create(dmat, flags, mapp); 423 if (error != 0) 424 return (error); 425 426 mflags = (flags & BUS_DMA_NOWAIT) != 0 ? M_NOWAIT : M_WAITOK; 427 mflags |= (flags & BUS_DMA_ZERO) != 0 ? M_ZERO : 0; 428 attr = (flags & BUS_DMA_NOCACHE) != 0 ? VM_MEMATTR_UNCACHEABLE : 429 VM_MEMATTR_DEFAULT; 430 431 tag = (struct bus_dma_tag_dmar *)dmat; 432 map = (struct bus_dmamap_dmar *)*mapp; 433 434 if (tag->common.maxsize < PAGE_SIZE && 435 tag->common.alignment <= tag->common.maxsize && 436 attr == VM_MEMATTR_DEFAULT) { 437 *vaddr = malloc(tag->common.maxsize, M_DEVBUF, mflags); 438 map->flags |= BUS_DMAMAP_DMAR_MALLOC; 439 } else { 440 *vaddr = (void *)kmem_alloc_attr(kernel_arena, 441 tag->common.maxsize, mflags, 0ul, BUS_SPACE_MAXADDR, 442 attr); 443 map->flags |= BUS_DMAMAP_DMAR_KMEM_ALLOC; 444 } 445 if (*vaddr == NULL) { 446 dmar_bus_dmamap_destroy(dmat, *mapp); 447 *mapp = NULL; 448 return (ENOMEM); 449 } 450 return (0); 451 } 452 453 static void 454 dmar_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map1) 455 { 456 struct bus_dma_tag_dmar *tag; 457 struct bus_dmamap_dmar *map; 458 459 tag = (struct bus_dma_tag_dmar *)dmat; 460 map = (struct bus_dmamap_dmar *)map1; 461 462 if ((map->flags & BUS_DMAMAP_DMAR_MALLOC) != 0) { 463 free(vaddr, M_DEVBUF); 464 map->flags &= ~BUS_DMAMAP_DMAR_MALLOC; 465 } else { 466 KASSERT((map->flags & BUS_DMAMAP_DMAR_KMEM_ALLOC) != 0, 467 ("dmar_bus_dmamem_free for non alloced map %p", map)); 468 kmem_free(kernel_arena, (vm_offset_t)vaddr, tag->common.maxsize); 469 map->flags &= ~BUS_DMAMAP_DMAR_KMEM_ALLOC; 470 } 471 472 dmar_bus_dmamap_destroy(dmat, map1); 473 } 474 475 static int 476 dmar_bus_dmamap_load_something1(struct bus_dma_tag_dmar *tag, 477 struct bus_dmamap_dmar *map, vm_page_t *ma, int offset, bus_size_t buflen, 478 int flags, bus_dma_segment_t *segs, int *segp, 479 struct dmar_map_entries_tailq *unroll_list) 480 { 481 struct dmar_ctx *ctx; 482 struct dmar_domain *domain; 483 struct dmar_map_entry *entry; 484 dmar_gaddr_t size; 485 bus_size_t buflen1; 486 int error, idx, gas_flags, seg; 487 488 KASSERT(offset < DMAR_PAGE_SIZE, ("offset %d", offset)); 489 if (segs == NULL) 490 segs = tag->segments; 491 ctx = tag->ctx; 492 domain = ctx->domain; 493 seg = *segp; 494 error = 0; 495 idx = 0; 496 while (buflen > 0) { 497 seg++; 498 if (seg >= tag->common.nsegments) { 499 error = EFBIG; 500 break; 501 } 502 buflen1 = buflen > tag->common.maxsegsz ? 503 tag->common.maxsegsz : buflen; 504 size = round_page(offset + buflen1); 505 506 /* 507 * (Too) optimistically allow split if there are more 508 * then one segments left. 509 */ 510 gas_flags = map->cansleep ? DMAR_GM_CANWAIT : 0; 511 if (seg + 1 < tag->common.nsegments) 512 gas_flags |= DMAR_GM_CANSPLIT; 513 514 error = dmar_gas_map(domain, &tag->common, size, offset, 515 DMAR_MAP_ENTRY_READ | DMAR_MAP_ENTRY_WRITE, 516 gas_flags, ma + idx, &entry); 517 if (error != 0) 518 break; 519 if ((gas_flags & DMAR_GM_CANSPLIT) != 0) { 520 KASSERT(size >= entry->end - entry->start, 521 ("split increased entry size %jx %jx %jx", 522 (uintmax_t)size, (uintmax_t)entry->start, 523 (uintmax_t)entry->end)); 524 size = entry->end - entry->start; 525 if (buflen1 > size) 526 buflen1 = size; 527 } else { 528 KASSERT(entry->end - entry->start == size, 529 ("no split allowed %jx %jx %jx", 530 (uintmax_t)size, (uintmax_t)entry->start, 531 (uintmax_t)entry->end)); 532 } 533 if (offset + buflen1 > size) 534 buflen1 = size - offset; 535 if (buflen1 > tag->common.maxsegsz) 536 buflen1 = tag->common.maxsegsz; 537 538 KASSERT(((entry->start + offset) & (tag->common.alignment - 1)) 539 == 0, 540 ("alignment failed: ctx %p start 0x%jx offset %x " 541 "align 0x%jx", ctx, (uintmax_t)entry->start, offset, 542 (uintmax_t)tag->common.alignment)); 543 KASSERT(entry->end <= tag->common.lowaddr || 544 entry->start >= tag->common.highaddr, 545 ("entry placement failed: ctx %p start 0x%jx end 0x%jx " 546 "lowaddr 0x%jx highaddr 0x%jx", ctx, 547 (uintmax_t)entry->start, (uintmax_t)entry->end, 548 (uintmax_t)tag->common.lowaddr, 549 (uintmax_t)tag->common.highaddr)); 550 KASSERT(dmar_test_boundary(entry->start + offset, buflen1, 551 tag->common.boundary), 552 ("boundary failed: ctx %p start 0x%jx end 0x%jx " 553 "boundary 0x%jx", ctx, (uintmax_t)entry->start, 554 (uintmax_t)entry->end, (uintmax_t)tag->common.boundary)); 555 KASSERT(buflen1 <= tag->common.maxsegsz, 556 ("segment too large: ctx %p start 0x%jx end 0x%jx " 557 "buflen1 0x%jx maxsegsz 0x%jx", ctx, 558 (uintmax_t)entry->start, (uintmax_t)entry->end, 559 (uintmax_t)buflen1, (uintmax_t)tag->common.maxsegsz)); 560 561 DMAR_DOMAIN_LOCK(domain); 562 TAILQ_INSERT_TAIL(&map->map_entries, entry, dmamap_link); 563 entry->flags |= DMAR_MAP_ENTRY_MAP; 564 DMAR_DOMAIN_UNLOCK(domain); 565 TAILQ_INSERT_TAIL(unroll_list, entry, unroll_link); 566 567 segs[seg].ds_addr = entry->start + offset; 568 segs[seg].ds_len = buflen1; 569 570 idx += OFF_TO_IDX(trunc_page(offset + buflen1)); 571 offset += buflen1; 572 offset &= DMAR_PAGE_MASK; 573 buflen -= buflen1; 574 } 575 if (error == 0) 576 *segp = seg; 577 return (error); 578 } 579 580 static int 581 dmar_bus_dmamap_load_something(struct bus_dma_tag_dmar *tag, 582 struct bus_dmamap_dmar *map, vm_page_t *ma, int offset, bus_size_t buflen, 583 int flags, bus_dma_segment_t *segs, int *segp) 584 { 585 struct dmar_ctx *ctx; 586 struct dmar_domain *domain; 587 struct dmar_map_entry *entry, *entry1; 588 struct dmar_map_entries_tailq unroll_list; 589 int error; 590 591 ctx = tag->ctx; 592 domain = ctx->domain; 593 atomic_add_long(&ctx->loads, 1); 594 595 TAILQ_INIT(&unroll_list); 596 error = dmar_bus_dmamap_load_something1(tag, map, ma, offset, 597 buflen, flags, segs, segp, &unroll_list); 598 if (error != 0) { 599 /* 600 * The busdma interface does not allow us to report 601 * partial buffer load, so unfortunately we have to 602 * revert all work done. 603 */ 604 DMAR_DOMAIN_LOCK(domain); 605 TAILQ_FOREACH_SAFE(entry, &unroll_list, unroll_link, 606 entry1) { 607 /* 608 * No entries other than what we have created 609 * during the failed run might have been 610 * inserted there in between, since we own ctx 611 * pglock. 612 */ 613 TAILQ_REMOVE(&map->map_entries, entry, dmamap_link); 614 TAILQ_REMOVE(&unroll_list, entry, unroll_link); 615 TAILQ_INSERT_TAIL(&domain->unload_entries, entry, 616 dmamap_link); 617 } 618 DMAR_DOMAIN_UNLOCK(domain); 619 taskqueue_enqueue(domain->dmar->delayed_taskqueue, 620 &domain->unload_task); 621 } 622 623 if (error == ENOMEM && (flags & BUS_DMA_NOWAIT) == 0 && 624 !map->cansleep) 625 error = EINPROGRESS; 626 if (error == EINPROGRESS) 627 dmar_bus_schedule_dmamap(domain->dmar, map); 628 return (error); 629 } 630 631 static int 632 dmar_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map1, 633 struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, 634 bus_dma_segment_t *segs, int *segp) 635 { 636 struct bus_dma_tag_dmar *tag; 637 struct bus_dmamap_dmar *map; 638 639 tag = (struct bus_dma_tag_dmar *)dmat; 640 map = (struct bus_dmamap_dmar *)map1; 641 return (dmar_bus_dmamap_load_something(tag, map, ma, ma_offs, tlen, 642 flags, segs, segp)); 643 } 644 645 static int 646 dmar_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map1, 647 vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs, 648 int *segp) 649 { 650 struct bus_dma_tag_dmar *tag; 651 struct bus_dmamap_dmar *map; 652 vm_page_t *ma; 653 vm_paddr_t pstart, pend; 654 int error, i, ma_cnt, offset; 655 656 tag = (struct bus_dma_tag_dmar *)dmat; 657 map = (struct bus_dmamap_dmar *)map1; 658 pstart = trunc_page(buf); 659 pend = round_page(buf + buflen); 660 offset = buf & PAGE_MASK; 661 ma_cnt = OFF_TO_IDX(pend - pstart); 662 ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, map->cansleep ? 663 M_WAITOK : M_NOWAIT); 664 if (ma == NULL) 665 return (ENOMEM); 666 for (i = 0; i < ma_cnt; i++) 667 ma[i] = PHYS_TO_VM_PAGE(pstart + i * PAGE_SIZE); 668 error = dmar_bus_dmamap_load_something(tag, map, ma, offset, buflen, 669 flags, segs, segp); 670 free(ma, M_DEVBUF); 671 return (error); 672 } 673 674 static int 675 dmar_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map1, void *buf, 676 bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, 677 int *segp) 678 { 679 struct bus_dma_tag_dmar *tag; 680 struct bus_dmamap_dmar *map; 681 vm_page_t *ma, fma; 682 vm_paddr_t pstart, pend, paddr; 683 int error, i, ma_cnt, offset; 684 685 tag = (struct bus_dma_tag_dmar *)dmat; 686 map = (struct bus_dmamap_dmar *)map1; 687 pstart = trunc_page((vm_offset_t)buf); 688 pend = round_page((vm_offset_t)buf + buflen); 689 offset = (vm_offset_t)buf & PAGE_MASK; 690 ma_cnt = OFF_TO_IDX(pend - pstart); 691 ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, map->cansleep ? 692 M_WAITOK : M_NOWAIT); 693 if (ma == NULL) 694 return (ENOMEM); 695 if (dumping) { 696 /* 697 * If dumping, do not attempt to call 698 * PHYS_TO_VM_PAGE() at all. It may return non-NULL 699 * but the vm_page returned might be not initialized, 700 * e.g. for the kernel itself. 701 */ 702 KASSERT(pmap == kernel_pmap, ("non-kernel address write")); 703 fma = malloc(sizeof(struct vm_page) * ma_cnt, M_DEVBUF, 704 M_ZERO | (map->cansleep ? M_WAITOK : M_NOWAIT)); 705 if (fma == NULL) { 706 free(ma, M_DEVBUF); 707 return (ENOMEM); 708 } 709 for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) { 710 paddr = pmap_kextract(pstart); 711 vm_page_initfake(&fma[i], paddr, VM_MEMATTR_DEFAULT); 712 ma[i] = &fma[i]; 713 } 714 } else { 715 fma = NULL; 716 for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) { 717 if (pmap == kernel_pmap) 718 paddr = pmap_kextract(pstart); 719 else 720 paddr = pmap_extract(pmap, pstart); 721 ma[i] = PHYS_TO_VM_PAGE(paddr); 722 KASSERT(VM_PAGE_TO_PHYS(ma[i]) == paddr, 723 ("PHYS_TO_VM_PAGE failed %jx %jx m %p", 724 (uintmax_t)paddr, (uintmax_t)VM_PAGE_TO_PHYS(ma[i]), 725 ma[i])); 726 } 727 } 728 error = dmar_bus_dmamap_load_something(tag, map, ma, offset, buflen, 729 flags, segs, segp); 730 free(ma, M_DEVBUF); 731 free(fma, M_DEVBUF); 732 return (error); 733 } 734 735 static void 736 dmar_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map1, 737 struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) 738 { 739 struct bus_dmamap_dmar *map; 740 741 if (map1 == NULL) 742 return; 743 map = (struct bus_dmamap_dmar *)map1; 744 map->mem = *mem; 745 map->tag = (struct bus_dma_tag_dmar *)dmat; 746 map->callback = callback; 747 map->callback_arg = callback_arg; 748 } 749 750 static bus_dma_segment_t * 751 dmar_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map1, 752 bus_dma_segment_t *segs, int nsegs, int error) 753 { 754 struct bus_dma_tag_dmar *tag; 755 struct bus_dmamap_dmar *map; 756 757 tag = (struct bus_dma_tag_dmar *)dmat; 758 map = (struct bus_dmamap_dmar *)map1; 759 760 if (!map->locked) { 761 KASSERT(map->cansleep, 762 ("map not locked and not sleepable context %p", map)); 763 764 /* 765 * We are called from the delayed context. Relock the 766 * driver. 767 */ 768 (tag->common.lockfunc)(tag->common.lockfuncarg, BUS_DMA_LOCK); 769 map->locked = true; 770 } 771 772 if (segs == NULL) 773 segs = tag->segments; 774 return (segs); 775 } 776 777 /* 778 * The limitations of busdma KPI forces the dmar to perform the actual 779 * unload, consisting of the unmapping of the map entries page tables, 780 * from the delayed context on i386, since page table page mapping 781 * might require a sleep to be successfull. The unfortunate 782 * consequence is that the DMA requests can be served some time after 783 * the bus_dmamap_unload() call returned. 784 * 785 * On amd64, we assume that sf allocation cannot fail. 786 */ 787 static void 788 dmar_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map1) 789 { 790 struct bus_dma_tag_dmar *tag; 791 struct bus_dmamap_dmar *map; 792 struct dmar_ctx *ctx; 793 struct dmar_domain *domain; 794 #if defined(__amd64__) 795 struct dmar_map_entries_tailq entries; 796 #endif 797 798 tag = (struct bus_dma_tag_dmar *)dmat; 799 map = (struct bus_dmamap_dmar *)map1; 800 ctx = tag->ctx; 801 domain = ctx->domain; 802 atomic_add_long(&ctx->unloads, 1); 803 804 #if defined(__i386__) 805 DMAR_DOMAIN_LOCK(domain); 806 TAILQ_CONCAT(&domain->unload_entries, &map->map_entries, dmamap_link); 807 DMAR_DOMAIN_UNLOCK(domain); 808 taskqueue_enqueue(domain->dmar->delayed_taskqueue, 809 &domain->unload_task); 810 #else /* defined(__amd64__) */ 811 TAILQ_INIT(&entries); 812 DMAR_DOMAIN_LOCK(domain); 813 TAILQ_CONCAT(&entries, &map->map_entries, dmamap_link); 814 DMAR_DOMAIN_UNLOCK(domain); 815 THREAD_NO_SLEEPING(); 816 dmar_domain_unload(domain, &entries, false); 817 THREAD_SLEEPING_OK(); 818 KASSERT(TAILQ_EMPTY(&entries), ("lazy dmar_ctx_unload %p", ctx)); 819 #endif 820 } 821 822 static void 823 dmar_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, 824 bus_dmasync_op_t op) 825 { 826 } 827 828 struct bus_dma_impl bus_dma_dmar_impl = { 829 .tag_create = dmar_bus_dma_tag_create, 830 .tag_destroy = dmar_bus_dma_tag_destroy, 831 .map_create = dmar_bus_dmamap_create, 832 .map_destroy = dmar_bus_dmamap_destroy, 833 .mem_alloc = dmar_bus_dmamem_alloc, 834 .mem_free = dmar_bus_dmamem_free, 835 .load_phys = dmar_bus_dmamap_load_phys, 836 .load_buffer = dmar_bus_dmamap_load_buffer, 837 .load_ma = dmar_bus_dmamap_load_ma, 838 .map_waitok = dmar_bus_dmamap_waitok, 839 .map_complete = dmar_bus_dmamap_complete, 840 .map_unload = dmar_bus_dmamap_unload, 841 .map_sync = dmar_bus_dmamap_sync 842 }; 843 844 static void 845 dmar_bus_task_dmamap(void *arg, int pending) 846 { 847 struct bus_dma_tag_dmar *tag; 848 struct bus_dmamap_dmar *map; 849 struct dmar_unit *unit; 850 851 unit = arg; 852 DMAR_LOCK(unit); 853 while ((map = TAILQ_FIRST(&unit->delayed_maps)) != NULL) { 854 TAILQ_REMOVE(&unit->delayed_maps, map, delay_link); 855 DMAR_UNLOCK(unit); 856 tag = map->tag; 857 map->cansleep = true; 858 map->locked = false; 859 bus_dmamap_load_mem((bus_dma_tag_t)tag, (bus_dmamap_t)map, 860 &map->mem, map->callback, map->callback_arg, 861 BUS_DMA_WAITOK); 862 map->cansleep = false; 863 if (map->locked) { 864 (tag->common.lockfunc)(tag->common.lockfuncarg, 865 BUS_DMA_UNLOCK); 866 } else 867 map->locked = true; 868 map->cansleep = false; 869 DMAR_LOCK(unit); 870 } 871 DMAR_UNLOCK(unit); 872 } 873 874 static void 875 dmar_bus_schedule_dmamap(struct dmar_unit *unit, struct bus_dmamap_dmar *map) 876 { 877 878 map->locked = false; 879 DMAR_LOCK(unit); 880 TAILQ_INSERT_TAIL(&unit->delayed_maps, map, delay_link); 881 DMAR_UNLOCK(unit); 882 taskqueue_enqueue(unit->delayed_taskqueue, &unit->dmamap_load_task); 883 } 884 885 int 886 dmar_init_busdma(struct dmar_unit *unit) 887 { 888 889 unit->dma_enabled = 1; 890 TUNABLE_INT_FETCH("hw.dmar.dma", &unit->dma_enabled); 891 TAILQ_INIT(&unit->delayed_maps); 892 TASK_INIT(&unit->dmamap_load_task, 0, dmar_bus_task_dmamap, unit); 893 unit->delayed_taskqueue = taskqueue_create("dmar", M_WAITOK, 894 taskqueue_thread_enqueue, &unit->delayed_taskqueue); 895 taskqueue_start_threads(&unit->delayed_taskqueue, 1, PI_DISK, 896 "dmar%d busdma taskq", unit->unit); 897 return (0); 898 } 899 900 void 901 dmar_fini_busdma(struct dmar_unit *unit) 902 { 903 904 if (unit->delayed_taskqueue == NULL) 905 return; 906 907 taskqueue_drain(unit->delayed_taskqueue, &unit->dmamap_load_task); 908 taskqueue_free(unit->delayed_taskqueue); 909 unit->delayed_taskqueue = NULL; 910 }
Cache object: 551f9e6faaf8799ad6e7fb9ce550ca10
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]