Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/x86/iommu/busdma_dmar.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2013 The FreeBSD Foundation 5 * All rights reserved. 6 * 7 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 8 * under sponsorship from the FreeBSD Foundation. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD: releng/12.0/sys/x86/iommu/busdma_dmar.c 340401 2018-11-13 18:21:47Z markj $"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/domainset.h> 38 #include <sys/malloc.h> 39 #include <sys/bus.h> 40 #include <sys/conf.h> 41 #include <sys/interrupt.h> 42 #include <sys/kernel.h> 43 #include <sys/ktr.h> 44 #include <sys/lock.h> 45 #include <sys/proc.h> 46 #include <sys/memdesc.h> 47 #include <sys/mutex.h> 48 #include <sys/sysctl.h> 49 #include <sys/rman.h> 50 #include <sys/taskqueue.h> 51 #include <sys/tree.h> 52 #include <sys/uio.h> 53 #include <sys/vmem.h> 54 #include <dev/pci/pcireg.h> 55 #include <dev/pci/pcivar.h> 56 #include <vm/vm.h> 57 #include <vm/vm_extern.h> 58 #include <vm/vm_kern.h> 59 #include <vm/vm_object.h> 60 #include <vm/vm_page.h> 61 #include <vm/vm_map.h> 62 #include <machine/atomic.h> 63 #include <machine/bus.h> 64 #include <machine/md_var.h> 65 #include <machine/specialreg.h> 66 #include <x86/include/busdma_impl.h> 67 #include <x86/iommu/intel_reg.h> 68 #include <x86/iommu/busdma_dmar.h> 69 #include <x86/iommu/intel_dmar.h> 70 71 /* 72 * busdma_dmar.c, the implementation of the busdma(9) interface using 73 * DMAR units from Intel VT-d. 74 */ 75 76 static bool 77 dmar_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func) 78 { 79 char str[128], *env; 80 int default_bounce; 81 bool ret; 82 static const char bounce_str[] = "bounce"; 83 static const char dmar_str[] = "dmar"; 84 85 default_bounce = 0; 86 env = kern_getenv("hw.busdma.default"); 87 if (env != NULL) { 88 if (strcmp(env, bounce_str) == 0) 89 default_bounce = 1; 90 else if (strcmp(env, dmar_str) == 0) 91 default_bounce = 0; 92 freeenv(env); 93 } 94 95 snprintf(str, sizeof(str), "hw.busdma.pci%d.%d.%d.%d", 96 domain, bus, slot, func); 97 env = kern_getenv(str); 98 if (env == NULL) 99 return (default_bounce != 0); 100 if (strcmp(env, bounce_str) == 0) 101 ret = true; 102 else if (strcmp(env, dmar_str) == 0) 103 ret = false; 104 else 105 ret = default_bounce != 0; 106 freeenv(env); 107 return (ret); 108 } 109 110 /* 111 * Given original device, find the requester ID that will be seen by 112 * the DMAR unit and used for page table lookup. PCI bridges may take 113 * ownership of transactions from downstream devices, so it may not be 114 * the same as the BSF of the target device. In those cases, all 115 * devices downstream of the bridge must share a single mapping 116 * domain, and must collectively be assigned to use either DMAR or 117 * bounce mapping. 118 */ 119 device_t 120 dmar_get_requester(device_t dev, uint16_t *rid) 121 { 122 devclass_t pci_class; 123 device_t l, pci, pcib, pcip, pcibp, requester; 124 int cap_offset; 125 uint16_t pcie_flags; 126 bool bridge_is_pcie; 127 128 pci_class = devclass_find("pci"); 129 l = requester = dev; 130 131 *rid = pci_get_rid(dev); 132 133 /* 134 * Walk the bridge hierarchy from the target device to the 135 * host port to find the translating bridge nearest the DMAR 136 * unit. 137 */ 138 for (;;) { 139 pci = device_get_parent(l); 140 KASSERT(pci != NULL, ("dmar_get_requester(%s): NULL parent " 141 "for %s", device_get_name(dev), device_get_name(l))); 142 KASSERT(device_get_devclass(pci) == pci_class, 143 ("dmar_get_requester(%s): non-pci parent %s for %s", 144 device_get_name(dev), device_get_name(pci), 145 device_get_name(l))); 146 147 pcib = device_get_parent(pci); 148 KASSERT(pcib != NULL, ("dmar_get_requester(%s): NULL bridge " 149 "for %s", device_get_name(dev), device_get_name(pci))); 150 151 /* 152 * The parent of our "bridge" isn't another PCI bus, 153 * so pcib isn't a PCI->PCI bridge but rather a host 154 * port, and the requester ID won't be translated 155 * further. 156 */ 157 pcip = device_get_parent(pcib); 158 if (device_get_devclass(pcip) != pci_class) 159 break; 160 pcibp = device_get_parent(pcip); 161 162 if (pci_find_cap(l, PCIY_EXPRESS, &cap_offset) == 0) { 163 /* 164 * Do not stop the loop even if the target 165 * device is PCIe, because it is possible (but 166 * unlikely) to have a PCI->PCIe bridge 167 * somewhere in the hierarchy. 168 */ 169 l = pcib; 170 } else { 171 /* 172 * Device is not PCIe, it cannot be seen as a 173 * requester by DMAR unit. Check whether the 174 * bridge is PCIe. 175 */ 176 bridge_is_pcie = pci_find_cap(pcib, PCIY_EXPRESS, 177 &cap_offset) == 0; 178 requester = pcib; 179 180 /* 181 * Check for a buggy PCIe/PCI bridge that 182 * doesn't report the express capability. If 183 * the bridge above it is express but isn't a 184 * PCI bridge, then we know pcib is actually a 185 * PCIe/PCI bridge. 186 */ 187 if (!bridge_is_pcie && pci_find_cap(pcibp, 188 PCIY_EXPRESS, &cap_offset) == 0) { 189 pcie_flags = pci_read_config(pcibp, 190 cap_offset + PCIER_FLAGS, 2); 191 if ((pcie_flags & PCIEM_FLAGS_TYPE) != 192 PCIEM_TYPE_PCI_BRIDGE) 193 bridge_is_pcie = true; 194 } 195 196 if (bridge_is_pcie) { 197 /* 198 * The current device is not PCIe, but 199 * the bridge above it is. This is a 200 * PCIe->PCI bridge. Assume that the 201 * requester ID will be the secondary 202 * bus number with slot and function 203 * set to zero. 204 * 205 * XXX: Doesn't handle the case where 206 * the bridge is PCIe->PCI-X, and the 207 * bridge will only take ownership of 208 * requests in some cases. We should 209 * provide context entries with the 210 * same page tables for taken and 211 * non-taken transactions. 212 */ 213 *rid = PCI_RID(pci_get_bus(l), 0, 0); 214 l = pcibp; 215 } else { 216 /* 217 * Neither the device nor the bridge 218 * above it are PCIe. This is a 219 * conventional PCI->PCI bridge, which 220 * will use the bridge's BSF as the 221 * requester ID. 222 */ 223 *rid = pci_get_rid(pcib); 224 l = pcib; 225 } 226 } 227 } 228 return (requester); 229 } 230 231 struct dmar_ctx * 232 dmar_instantiate_ctx(struct dmar_unit *dmar, device_t dev, bool rmrr) 233 { 234 device_t requester; 235 struct dmar_ctx *ctx; 236 bool disabled; 237 uint16_t rid; 238 239 requester = dmar_get_requester(dev, &rid); 240 241 /* 242 * If the user requested the IOMMU disabled for the device, we 243 * cannot disable the DMAR, due to possibility of other 244 * devices on the same DMAR still requiring translation. 245 * Instead provide the identity mapping for the device 246 * context. 247 */ 248 disabled = dmar_bus_dma_is_dev_disabled(pci_get_domain(requester), 249 pci_get_bus(requester), pci_get_slot(requester), 250 pci_get_function(requester)); 251 ctx = dmar_get_ctx_for_dev(dmar, requester, rid, disabled, rmrr); 252 if (ctx == NULL) 253 return (NULL); 254 if (disabled) { 255 /* 256 * Keep the first reference on context, release the 257 * later refs. 258 */ 259 DMAR_LOCK(dmar); 260 if ((ctx->flags & DMAR_CTX_DISABLED) == 0) { 261 ctx->flags |= DMAR_CTX_DISABLED; 262 DMAR_UNLOCK(dmar); 263 } else { 264 dmar_free_ctx_locked(dmar, ctx); 265 } 266 ctx = NULL; 267 } 268 return (ctx); 269 } 270 271 bus_dma_tag_t 272 dmar_get_dma_tag(device_t dev, device_t child) 273 { 274 struct dmar_unit *dmar; 275 struct dmar_ctx *ctx; 276 bus_dma_tag_t res; 277 278 dmar = dmar_find(child); 279 /* Not in scope of any DMAR ? */ 280 if (dmar == NULL) 281 return (NULL); 282 if (!dmar->dma_enabled) 283 return (NULL); 284 dmar_quirks_pre_use(dmar); 285 dmar_instantiate_rmrr_ctxs(dmar); 286 287 ctx = dmar_instantiate_ctx(dmar, child, false); 288 res = ctx == NULL ? NULL : (bus_dma_tag_t)&ctx->ctx_tag; 289 return (res); 290 } 291 292 static MALLOC_DEFINE(M_DMAR_DMAMAP, "dmar_dmamap", "Intel DMAR DMA Map"); 293 294 static void dmar_bus_schedule_dmamap(struct dmar_unit *unit, 295 struct bus_dmamap_dmar *map); 296 297 static int 298 dmar_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 299 bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, 300 bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, 301 int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 302 void *lockfuncarg, bus_dma_tag_t *dmat) 303 { 304 struct bus_dma_tag_dmar *newtag, *oldtag; 305 int error; 306 307 *dmat = NULL; 308 error = common_bus_dma_tag_create(parent != NULL ? 309 &((struct bus_dma_tag_dmar *)parent)->common : NULL, alignment, 310 boundary, lowaddr, highaddr, filter, filterarg, maxsize, 311 nsegments, maxsegsz, flags, lockfunc, lockfuncarg, 312 sizeof(struct bus_dma_tag_dmar), (void **)&newtag); 313 if (error != 0) 314 goto out; 315 316 oldtag = (struct bus_dma_tag_dmar *)parent; 317 newtag->common.impl = &bus_dma_dmar_impl; 318 newtag->ctx = oldtag->ctx; 319 newtag->owner = oldtag->owner; 320 321 *dmat = (bus_dma_tag_t)newtag; 322 out: 323 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 324 __func__, newtag, (newtag != NULL ? newtag->common.flags : 0), 325 error); 326 return (error); 327 } 328 329 static int 330 dmar_bus_dma_tag_set_domain(bus_dma_tag_t dmat) 331 { 332 333 return (0); 334 } 335 336 static int 337 dmar_bus_dma_tag_destroy(bus_dma_tag_t dmat1) 338 { 339 struct bus_dma_tag_dmar *dmat, *dmat_copy, *parent; 340 int error; 341 342 error = 0; 343 dmat_copy = dmat = (struct bus_dma_tag_dmar *)dmat1; 344 345 if (dmat != NULL) { 346 if (dmat->map_count != 0) { 347 error = EBUSY; 348 goto out; 349 } 350 while (dmat != NULL) { 351 parent = (struct bus_dma_tag_dmar *)dmat->common.parent; 352 if (atomic_fetchadd_int(&dmat->common.ref_count, -1) == 353 1) { 354 if (dmat == &dmat->ctx->ctx_tag) 355 dmar_free_ctx(dmat->ctx); 356 free_domain(dmat->segments, M_DMAR_DMAMAP); 357 free(dmat, M_DEVBUF); 358 dmat = parent; 359 } else 360 dmat = NULL; 361 } 362 } 363 out: 364 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); 365 return (error); 366 } 367 368 static int 369 dmar_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 370 { 371 struct bus_dma_tag_dmar *tag; 372 struct bus_dmamap_dmar *map; 373 374 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "%s", __func__); 375 376 tag = (struct bus_dma_tag_dmar *)dmat; 377 map = malloc_domainset(sizeof(*map), M_DMAR_DMAMAP, 378 DOMAINSET_PREF(tag->common.domain), M_NOWAIT | M_ZERO); 379 if (map == NULL) { 380 *mapp = NULL; 381 return (ENOMEM); 382 } 383 if (tag->segments == NULL) { 384 tag->segments = malloc_domainset(sizeof(bus_dma_segment_t) * 385 tag->common.nsegments, M_DMAR_DMAMAP, 386 DOMAINSET_PREF(tag->common.domain), M_NOWAIT); 387 if (tag->segments == NULL) { 388 free_domain(map, M_DMAR_DMAMAP); 389 *mapp = NULL; 390 return (ENOMEM); 391 } 392 } 393 TAILQ_INIT(&map->map_entries); 394 map->tag = tag; 395 map->locked = true; 396 map->cansleep = false; 397 tag->map_count++; 398 *mapp = (bus_dmamap_t)map; 399 400 return (0); 401 } 402 403 static int 404 dmar_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map1) 405 { 406 struct bus_dma_tag_dmar *tag; 407 struct bus_dmamap_dmar *map; 408 struct dmar_domain *domain; 409 410 tag = (struct bus_dma_tag_dmar *)dmat; 411 map = (struct bus_dmamap_dmar *)map1; 412 if (map != NULL) { 413 domain = tag->ctx->domain; 414 DMAR_DOMAIN_LOCK(domain); 415 if (!TAILQ_EMPTY(&map->map_entries)) { 416 DMAR_DOMAIN_UNLOCK(domain); 417 return (EBUSY); 418 } 419 DMAR_DOMAIN_UNLOCK(domain); 420 free_domain(map, M_DMAR_DMAMAP); 421 } 422 tag->map_count--; 423 return (0); 424 } 425 426 427 static int 428 dmar_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 429 bus_dmamap_t *mapp) 430 { 431 struct bus_dma_tag_dmar *tag; 432 struct bus_dmamap_dmar *map; 433 int error, mflags; 434 vm_memattr_t attr; 435 436 error = dmar_bus_dmamap_create(dmat, flags, mapp); 437 if (error != 0) 438 return (error); 439 440 mflags = (flags & BUS_DMA_NOWAIT) != 0 ? M_NOWAIT : M_WAITOK; 441 mflags |= (flags & BUS_DMA_ZERO) != 0 ? M_ZERO : 0; 442 attr = (flags & BUS_DMA_NOCACHE) != 0 ? VM_MEMATTR_UNCACHEABLE : 443 VM_MEMATTR_DEFAULT; 444 445 tag = (struct bus_dma_tag_dmar *)dmat; 446 map = (struct bus_dmamap_dmar *)*mapp; 447 448 if (tag->common.maxsize < PAGE_SIZE && 449 tag->common.alignment <= tag->common.maxsize && 450 attr == VM_MEMATTR_DEFAULT) { 451 *vaddr = malloc_domainset(tag->common.maxsize, M_DEVBUF, 452 DOMAINSET_PREF(tag->common.domain), mflags); 453 map->flags |= BUS_DMAMAP_DMAR_MALLOC; 454 } else { 455 *vaddr = (void *)kmem_alloc_attr_domainset( 456 DOMAINSET_PREF(tag->common.domain), tag->common.maxsize, 457 mflags, 0ul, BUS_SPACE_MAXADDR, attr); 458 map->flags |= BUS_DMAMAP_DMAR_KMEM_ALLOC; 459 } 460 if (*vaddr == NULL) { 461 dmar_bus_dmamap_destroy(dmat, *mapp); 462 *mapp = NULL; 463 return (ENOMEM); 464 } 465 return (0); 466 } 467 468 static void 469 dmar_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map1) 470 { 471 struct bus_dma_tag_dmar *tag; 472 struct bus_dmamap_dmar *map; 473 474 tag = (struct bus_dma_tag_dmar *)dmat; 475 map = (struct bus_dmamap_dmar *)map1; 476 477 if ((map->flags & BUS_DMAMAP_DMAR_MALLOC) != 0) { 478 free_domain(vaddr, M_DEVBUF); 479 map->flags &= ~BUS_DMAMAP_DMAR_MALLOC; 480 } else { 481 KASSERT((map->flags & BUS_DMAMAP_DMAR_KMEM_ALLOC) != 0, 482 ("dmar_bus_dmamem_free for non alloced map %p", map)); 483 kmem_free((vm_offset_t)vaddr, tag->common.maxsize); 484 map->flags &= ~BUS_DMAMAP_DMAR_KMEM_ALLOC; 485 } 486 487 dmar_bus_dmamap_destroy(dmat, map1); 488 } 489 490 static int 491 dmar_bus_dmamap_load_something1(struct bus_dma_tag_dmar *tag, 492 struct bus_dmamap_dmar *map, vm_page_t *ma, int offset, bus_size_t buflen, 493 int flags, bus_dma_segment_t *segs, int *segp, 494 struct dmar_map_entries_tailq *unroll_list) 495 { 496 struct dmar_ctx *ctx; 497 struct dmar_domain *domain; 498 struct dmar_map_entry *entry; 499 dmar_gaddr_t size; 500 bus_size_t buflen1; 501 int error, idx, gas_flags, seg; 502 503 KASSERT(offset < DMAR_PAGE_SIZE, ("offset %d", offset)); 504 if (segs == NULL) 505 segs = tag->segments; 506 ctx = tag->ctx; 507 domain = ctx->domain; 508 seg = *segp; 509 error = 0; 510 idx = 0; 511 while (buflen > 0) { 512 seg++; 513 if (seg >= tag->common.nsegments) { 514 error = EFBIG; 515 break; 516 } 517 buflen1 = buflen > tag->common.maxsegsz ? 518 tag->common.maxsegsz : buflen; 519 size = round_page(offset + buflen1); 520 521 /* 522 * (Too) optimistically allow split if there are more 523 * then one segments left. 524 */ 525 gas_flags = map->cansleep ? DMAR_GM_CANWAIT : 0; 526 if (seg + 1 < tag->common.nsegments) 527 gas_flags |= DMAR_GM_CANSPLIT; 528 529 error = dmar_gas_map(domain, &tag->common, size, offset, 530 DMAR_MAP_ENTRY_READ | DMAR_MAP_ENTRY_WRITE, 531 gas_flags, ma + idx, &entry); 532 if (error != 0) 533 break; 534 if ((gas_flags & DMAR_GM_CANSPLIT) != 0) { 535 KASSERT(size >= entry->end - entry->start, 536 ("split increased entry size %jx %jx %jx", 537 (uintmax_t)size, (uintmax_t)entry->start, 538 (uintmax_t)entry->end)); 539 size = entry->end - entry->start; 540 if (buflen1 > size) 541 buflen1 = size; 542 } else { 543 KASSERT(entry->end - entry->start == size, 544 ("no split allowed %jx %jx %jx", 545 (uintmax_t)size, (uintmax_t)entry->start, 546 (uintmax_t)entry->end)); 547 } 548 if (offset + buflen1 > size) 549 buflen1 = size - offset; 550 if (buflen1 > tag->common.maxsegsz) 551 buflen1 = tag->common.maxsegsz; 552 553 KASSERT(((entry->start + offset) & (tag->common.alignment - 1)) 554 == 0, 555 ("alignment failed: ctx %p start 0x%jx offset %x " 556 "align 0x%jx", ctx, (uintmax_t)entry->start, offset, 557 (uintmax_t)tag->common.alignment)); 558 KASSERT(entry->end <= tag->common.lowaddr || 559 entry->start >= tag->common.highaddr, 560 ("entry placement failed: ctx %p start 0x%jx end 0x%jx " 561 "lowaddr 0x%jx highaddr 0x%jx", ctx, 562 (uintmax_t)entry->start, (uintmax_t)entry->end, 563 (uintmax_t)tag->common.lowaddr, 564 (uintmax_t)tag->common.highaddr)); 565 KASSERT(dmar_test_boundary(entry->start + offset, buflen1, 566 tag->common.boundary), 567 ("boundary failed: ctx %p start 0x%jx end 0x%jx " 568 "boundary 0x%jx", ctx, (uintmax_t)entry->start, 569 (uintmax_t)entry->end, (uintmax_t)tag->common.boundary)); 570 KASSERT(buflen1 <= tag->common.maxsegsz, 571 ("segment too large: ctx %p start 0x%jx end 0x%jx " 572 "buflen1 0x%jx maxsegsz 0x%jx", ctx, 573 (uintmax_t)entry->start, (uintmax_t)entry->end, 574 (uintmax_t)buflen1, (uintmax_t)tag->common.maxsegsz)); 575 576 DMAR_DOMAIN_LOCK(domain); 577 TAILQ_INSERT_TAIL(&map->map_entries, entry, dmamap_link); 578 entry->flags |= DMAR_MAP_ENTRY_MAP; 579 DMAR_DOMAIN_UNLOCK(domain); 580 TAILQ_INSERT_TAIL(unroll_list, entry, unroll_link); 581 582 segs[seg].ds_addr = entry->start + offset; 583 segs[seg].ds_len = buflen1; 584 585 idx += OFF_TO_IDX(trunc_page(offset + buflen1)); 586 offset += buflen1; 587 offset &= DMAR_PAGE_MASK; 588 buflen -= buflen1; 589 } 590 if (error == 0) 591 *segp = seg; 592 return (error); 593 } 594 595 static int 596 dmar_bus_dmamap_load_something(struct bus_dma_tag_dmar *tag, 597 struct bus_dmamap_dmar *map, vm_page_t *ma, int offset, bus_size_t buflen, 598 int flags, bus_dma_segment_t *segs, int *segp) 599 { 600 struct dmar_ctx *ctx; 601 struct dmar_domain *domain; 602 struct dmar_map_entry *entry, *entry1; 603 struct dmar_map_entries_tailq unroll_list; 604 int error; 605 606 ctx = tag->ctx; 607 domain = ctx->domain; 608 atomic_add_long(&ctx->loads, 1); 609 610 TAILQ_INIT(&unroll_list); 611 error = dmar_bus_dmamap_load_something1(tag, map, ma, offset, 612 buflen, flags, segs, segp, &unroll_list); 613 if (error != 0) { 614 /* 615 * The busdma interface does not allow us to report 616 * partial buffer load, so unfortunately we have to 617 * revert all work done. 618 */ 619 DMAR_DOMAIN_LOCK(domain); 620 TAILQ_FOREACH_SAFE(entry, &unroll_list, unroll_link, 621 entry1) { 622 /* 623 * No entries other than what we have created 624 * during the failed run might have been 625 * inserted there in between, since we own ctx 626 * pglock. 627 */ 628 TAILQ_REMOVE(&map->map_entries, entry, dmamap_link); 629 TAILQ_REMOVE(&unroll_list, entry, unroll_link); 630 TAILQ_INSERT_TAIL(&domain->unload_entries, entry, 631 dmamap_link); 632 } 633 DMAR_DOMAIN_UNLOCK(domain); 634 taskqueue_enqueue(domain->dmar->delayed_taskqueue, 635 &domain->unload_task); 636 } 637 638 if (error == ENOMEM && (flags & BUS_DMA_NOWAIT) == 0 && 639 !map->cansleep) 640 error = EINPROGRESS; 641 if (error == EINPROGRESS) 642 dmar_bus_schedule_dmamap(domain->dmar, map); 643 return (error); 644 } 645 646 static int 647 dmar_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map1, 648 struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, 649 bus_dma_segment_t *segs, int *segp) 650 { 651 struct bus_dma_tag_dmar *tag; 652 struct bus_dmamap_dmar *map; 653 654 tag = (struct bus_dma_tag_dmar *)dmat; 655 map = (struct bus_dmamap_dmar *)map1; 656 return (dmar_bus_dmamap_load_something(tag, map, ma, ma_offs, tlen, 657 flags, segs, segp)); 658 } 659 660 static int 661 dmar_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map1, 662 vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs, 663 int *segp) 664 { 665 struct bus_dma_tag_dmar *tag; 666 struct bus_dmamap_dmar *map; 667 vm_page_t *ma; 668 vm_paddr_t pstart, pend; 669 int error, i, ma_cnt, offset; 670 671 tag = (struct bus_dma_tag_dmar *)dmat; 672 map = (struct bus_dmamap_dmar *)map1; 673 pstart = trunc_page(buf); 674 pend = round_page(buf + buflen); 675 offset = buf & PAGE_MASK; 676 ma_cnt = OFF_TO_IDX(pend - pstart); 677 ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, map->cansleep ? 678 M_WAITOK : M_NOWAIT); 679 if (ma == NULL) 680 return (ENOMEM); 681 for (i = 0; i < ma_cnt; i++) 682 ma[i] = PHYS_TO_VM_PAGE(pstart + i * PAGE_SIZE); 683 error = dmar_bus_dmamap_load_something(tag, map, ma, offset, buflen, 684 flags, segs, segp); 685 free(ma, M_DEVBUF); 686 return (error); 687 } 688 689 static int 690 dmar_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map1, void *buf, 691 bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, 692 int *segp) 693 { 694 struct bus_dma_tag_dmar *tag; 695 struct bus_dmamap_dmar *map; 696 vm_page_t *ma, fma; 697 vm_paddr_t pstart, pend, paddr; 698 int error, i, ma_cnt, offset; 699 700 tag = (struct bus_dma_tag_dmar *)dmat; 701 map = (struct bus_dmamap_dmar *)map1; 702 pstart = trunc_page((vm_offset_t)buf); 703 pend = round_page((vm_offset_t)buf + buflen); 704 offset = (vm_offset_t)buf & PAGE_MASK; 705 ma_cnt = OFF_TO_IDX(pend - pstart); 706 ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, map->cansleep ? 707 M_WAITOK : M_NOWAIT); 708 if (ma == NULL) 709 return (ENOMEM); 710 if (dumping) { 711 /* 712 * If dumping, do not attempt to call 713 * PHYS_TO_VM_PAGE() at all. It may return non-NULL 714 * but the vm_page returned might be not initialized, 715 * e.g. for the kernel itself. 716 */ 717 KASSERT(pmap == kernel_pmap, ("non-kernel address write")); 718 fma = malloc(sizeof(struct vm_page) * ma_cnt, M_DEVBUF, 719 M_ZERO | (map->cansleep ? M_WAITOK : M_NOWAIT)); 720 if (fma == NULL) { 721 free(ma, M_DEVBUF); 722 return (ENOMEM); 723 } 724 for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) { 725 paddr = pmap_kextract(pstart); 726 vm_page_initfake(&fma[i], paddr, VM_MEMATTR_DEFAULT); 727 ma[i] = &fma[i]; 728 } 729 } else { 730 fma = NULL; 731 for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) { 732 if (pmap == kernel_pmap) 733 paddr = pmap_kextract(pstart); 734 else 735 paddr = pmap_extract(pmap, pstart); 736 ma[i] = PHYS_TO_VM_PAGE(paddr); 737 KASSERT(VM_PAGE_TO_PHYS(ma[i]) == paddr, 738 ("PHYS_TO_VM_PAGE failed %jx %jx m %p", 739 (uintmax_t)paddr, (uintmax_t)VM_PAGE_TO_PHYS(ma[i]), 740 ma[i])); 741 } 742 } 743 error = dmar_bus_dmamap_load_something(tag, map, ma, offset, buflen, 744 flags, segs, segp); 745 free(ma, M_DEVBUF); 746 free(fma, M_DEVBUF); 747 return (error); 748 } 749 750 static void 751 dmar_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map1, 752 struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) 753 { 754 struct bus_dmamap_dmar *map; 755 756 if (map1 == NULL) 757 return; 758 map = (struct bus_dmamap_dmar *)map1; 759 map->mem = *mem; 760 map->tag = (struct bus_dma_tag_dmar *)dmat; 761 map->callback = callback; 762 map->callback_arg = callback_arg; 763 } 764 765 static bus_dma_segment_t * 766 dmar_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map1, 767 bus_dma_segment_t *segs, int nsegs, int error) 768 { 769 struct bus_dma_tag_dmar *tag; 770 struct bus_dmamap_dmar *map; 771 772 tag = (struct bus_dma_tag_dmar *)dmat; 773 map = (struct bus_dmamap_dmar *)map1; 774 775 if (!map->locked) { 776 KASSERT(map->cansleep, 777 ("map not locked and not sleepable context %p", map)); 778 779 /* 780 * We are called from the delayed context. Relock the 781 * driver. 782 */ 783 (tag->common.lockfunc)(tag->common.lockfuncarg, BUS_DMA_LOCK); 784 map->locked = true; 785 } 786 787 if (segs == NULL) 788 segs = tag->segments; 789 return (segs); 790 } 791 792 /* 793 * The limitations of busdma KPI forces the dmar to perform the actual 794 * unload, consisting of the unmapping of the map entries page tables, 795 * from the delayed context on i386, since page table page mapping 796 * might require a sleep to be successfull. The unfortunate 797 * consequence is that the DMA requests can be served some time after 798 * the bus_dmamap_unload() call returned. 799 * 800 * On amd64, we assume that sf allocation cannot fail. 801 */ 802 static void 803 dmar_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map1) 804 { 805 struct bus_dma_tag_dmar *tag; 806 struct bus_dmamap_dmar *map; 807 struct dmar_ctx *ctx; 808 struct dmar_domain *domain; 809 #if defined(__amd64__) 810 struct dmar_map_entries_tailq entries; 811 #endif 812 813 tag = (struct bus_dma_tag_dmar *)dmat; 814 map = (struct bus_dmamap_dmar *)map1; 815 ctx = tag->ctx; 816 domain = ctx->domain; 817 atomic_add_long(&ctx->unloads, 1); 818 819 #if defined(__i386__) 820 DMAR_DOMAIN_LOCK(domain); 821 TAILQ_CONCAT(&domain->unload_entries, &map->map_entries, dmamap_link); 822 DMAR_DOMAIN_UNLOCK(domain); 823 taskqueue_enqueue(domain->dmar->delayed_taskqueue, 824 &domain->unload_task); 825 #else /* defined(__amd64__) */ 826 TAILQ_INIT(&entries); 827 DMAR_DOMAIN_LOCK(domain); 828 TAILQ_CONCAT(&entries, &map->map_entries, dmamap_link); 829 DMAR_DOMAIN_UNLOCK(domain); 830 THREAD_NO_SLEEPING(); 831 dmar_domain_unload(domain, &entries, false); 832 THREAD_SLEEPING_OK(); 833 KASSERT(TAILQ_EMPTY(&entries), ("lazy dmar_ctx_unload %p", ctx)); 834 #endif 835 } 836 837 static void 838 dmar_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, 839 bus_dmasync_op_t op) 840 { 841 } 842 843 struct bus_dma_impl bus_dma_dmar_impl = { 844 .tag_create = dmar_bus_dma_tag_create, 845 .tag_destroy = dmar_bus_dma_tag_destroy, 846 .tag_set_domain = dmar_bus_dma_tag_set_domain, 847 .map_create = dmar_bus_dmamap_create, 848 .map_destroy = dmar_bus_dmamap_destroy, 849 .mem_alloc = dmar_bus_dmamem_alloc, 850 .mem_free = dmar_bus_dmamem_free, 851 .load_phys = dmar_bus_dmamap_load_phys, 852 .load_buffer = dmar_bus_dmamap_load_buffer, 853 .load_ma = dmar_bus_dmamap_load_ma, 854 .map_waitok = dmar_bus_dmamap_waitok, 855 .map_complete = dmar_bus_dmamap_complete, 856 .map_unload = dmar_bus_dmamap_unload, 857 .map_sync = dmar_bus_dmamap_sync, 858 }; 859 860 static void 861 dmar_bus_task_dmamap(void *arg, int pending) 862 { 863 struct bus_dma_tag_dmar *tag; 864 struct bus_dmamap_dmar *map; 865 struct dmar_unit *unit; 866 867 unit = arg; 868 DMAR_LOCK(unit); 869 while ((map = TAILQ_FIRST(&unit->delayed_maps)) != NULL) { 870 TAILQ_REMOVE(&unit->delayed_maps, map, delay_link); 871 DMAR_UNLOCK(unit); 872 tag = map->tag; 873 map->cansleep = true; 874 map->locked = false; 875 bus_dmamap_load_mem((bus_dma_tag_t)tag, (bus_dmamap_t)map, 876 &map->mem, map->callback, map->callback_arg, 877 BUS_DMA_WAITOK); 878 map->cansleep = false; 879 if (map->locked) { 880 (tag->common.lockfunc)(tag->common.lockfuncarg, 881 BUS_DMA_UNLOCK); 882 } else 883 map->locked = true; 884 map->cansleep = false; 885 DMAR_LOCK(unit); 886 } 887 DMAR_UNLOCK(unit); 888 } 889 890 static void 891 dmar_bus_schedule_dmamap(struct dmar_unit *unit, struct bus_dmamap_dmar *map) 892 { 893 894 map->locked = false; 895 DMAR_LOCK(unit); 896 TAILQ_INSERT_TAIL(&unit->delayed_maps, map, delay_link); 897 DMAR_UNLOCK(unit); 898 taskqueue_enqueue(unit->delayed_taskqueue, &unit->dmamap_load_task); 899 } 900 901 int 902 dmar_init_busdma(struct dmar_unit *unit) 903 { 904 905 unit->dma_enabled = 1; 906 TUNABLE_INT_FETCH("hw.dmar.dma", &unit->dma_enabled); 907 TAILQ_INIT(&unit->delayed_maps); 908 TASK_INIT(&unit->dmamap_load_task, 0, dmar_bus_task_dmamap, unit); 909 unit->delayed_taskqueue = taskqueue_create("dmar", M_WAITOK, 910 taskqueue_thread_enqueue, &unit->delayed_taskqueue); 911 taskqueue_start_threads(&unit->delayed_taskqueue, 1, PI_DISK, 912 "dmar%d busdma taskq", unit->unit); 913 return (0); 914 } 915 916 void 917 dmar_fini_busdma(struct dmar_unit *unit) 918 { 919 920 if (unit->delayed_taskqueue == NULL) 921 return; 922 923 taskqueue_drain(unit->delayed_taskqueue, &unit->dmamap_load_task); 924 taskqueue_free(unit->delayed_taskqueue); 925 unit->delayed_taskqueue = NULL; 926 }
Cache object: a15bbdcf51293587cc6be3dfc7918816
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]