1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2013 The FreeBSD Foundation
5 * All rights reserved.
6 *
7 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
8 * under sponsorship from the FreeBSD Foundation.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD: releng/12.0/sys/x86/iommu/busdma_dmar.c 340401 2018-11-13 18:21:47Z markj $");
34
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/domainset.h>
38 #include <sys/malloc.h>
39 #include <sys/bus.h>
40 #include <sys/conf.h>
41 #include <sys/interrupt.h>
42 #include <sys/kernel.h>
43 #include <sys/ktr.h>
44 #include <sys/lock.h>
45 #include <sys/proc.h>
46 #include <sys/memdesc.h>
47 #include <sys/mutex.h>
48 #include <sys/sysctl.h>
49 #include <sys/rman.h>
50 #include <sys/taskqueue.h>
51 #include <sys/tree.h>
52 #include <sys/uio.h>
53 #include <sys/vmem.h>
54 #include <dev/pci/pcireg.h>
55 #include <dev/pci/pcivar.h>
56 #include <vm/vm.h>
57 #include <vm/vm_extern.h>
58 #include <vm/vm_kern.h>
59 #include <vm/vm_object.h>
60 #include <vm/vm_page.h>
61 #include <vm/vm_map.h>
62 #include <machine/atomic.h>
63 #include <machine/bus.h>
64 #include <machine/md_var.h>
65 #include <machine/specialreg.h>
66 #include <x86/include/busdma_impl.h>
67 #include <x86/iommu/intel_reg.h>
68 #include <x86/iommu/busdma_dmar.h>
69 #include <x86/iommu/intel_dmar.h>
70
71 /*
72 * busdma_dmar.c, the implementation of the busdma(9) interface using
73 * DMAR units from Intel VT-d.
74 */
75
76 static bool
77 dmar_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func)
78 {
79 char str[128], *env;
80 int default_bounce;
81 bool ret;
82 static const char bounce_str[] = "bounce";
83 static const char dmar_str[] = "dmar";
84
85 default_bounce = 0;
86 env = kern_getenv("hw.busdma.default");
87 if (env != NULL) {
88 if (strcmp(env, bounce_str) == 0)
89 default_bounce = 1;
90 else if (strcmp(env, dmar_str) == 0)
91 default_bounce = 0;
92 freeenv(env);
93 }
94
95 snprintf(str, sizeof(str), "hw.busdma.pci%d.%d.%d.%d",
96 domain, bus, slot, func);
97 env = kern_getenv(str);
98 if (env == NULL)
99 return (default_bounce != 0);
100 if (strcmp(env, bounce_str) == 0)
101 ret = true;
102 else if (strcmp(env, dmar_str) == 0)
103 ret = false;
104 else
105 ret = default_bounce != 0;
106 freeenv(env);
107 return (ret);
108 }
109
110 /*
111 * Given original device, find the requester ID that will be seen by
112 * the DMAR unit and used for page table lookup. PCI bridges may take
113 * ownership of transactions from downstream devices, so it may not be
114 * the same as the BSF of the target device. In those cases, all
115 * devices downstream of the bridge must share a single mapping
116 * domain, and must collectively be assigned to use either DMAR or
117 * bounce mapping.
118 */
119 device_t
120 dmar_get_requester(device_t dev, uint16_t *rid)
121 {
122 devclass_t pci_class;
123 device_t l, pci, pcib, pcip, pcibp, requester;
124 int cap_offset;
125 uint16_t pcie_flags;
126 bool bridge_is_pcie;
127
128 pci_class = devclass_find("pci");
129 l = requester = dev;
130
131 *rid = pci_get_rid(dev);
132
133 /*
134 * Walk the bridge hierarchy from the target device to the
135 * host port to find the translating bridge nearest the DMAR
136 * unit.
137 */
138 for (;;) {
139 pci = device_get_parent(l);
140 KASSERT(pci != NULL, ("dmar_get_requester(%s): NULL parent "
141 "for %s", device_get_name(dev), device_get_name(l)));
142 KASSERT(device_get_devclass(pci) == pci_class,
143 ("dmar_get_requester(%s): non-pci parent %s for %s",
144 device_get_name(dev), device_get_name(pci),
145 device_get_name(l)));
146
147 pcib = device_get_parent(pci);
148 KASSERT(pcib != NULL, ("dmar_get_requester(%s): NULL bridge "
149 "for %s", device_get_name(dev), device_get_name(pci)));
150
151 /*
152 * The parent of our "bridge" isn't another PCI bus,
153 * so pcib isn't a PCI->PCI bridge but rather a host
154 * port, and the requester ID won't be translated
155 * further.
156 */
157 pcip = device_get_parent(pcib);
158 if (device_get_devclass(pcip) != pci_class)
159 break;
160 pcibp = device_get_parent(pcip);
161
162 if (pci_find_cap(l, PCIY_EXPRESS, &cap_offset) == 0) {
163 /*
164 * Do not stop the loop even if the target
165 * device is PCIe, because it is possible (but
166 * unlikely) to have a PCI->PCIe bridge
167 * somewhere in the hierarchy.
168 */
169 l = pcib;
170 } else {
171 /*
172 * Device is not PCIe, it cannot be seen as a
173 * requester by DMAR unit. Check whether the
174 * bridge is PCIe.
175 */
176 bridge_is_pcie = pci_find_cap(pcib, PCIY_EXPRESS,
177 &cap_offset) == 0;
178 requester = pcib;
179
180 /*
181 * Check for a buggy PCIe/PCI bridge that
182 * doesn't report the express capability. If
183 * the bridge above it is express but isn't a
184 * PCI bridge, then we know pcib is actually a
185 * PCIe/PCI bridge.
186 */
187 if (!bridge_is_pcie && pci_find_cap(pcibp,
188 PCIY_EXPRESS, &cap_offset) == 0) {
189 pcie_flags = pci_read_config(pcibp,
190 cap_offset + PCIER_FLAGS, 2);
191 if ((pcie_flags & PCIEM_FLAGS_TYPE) !=
192 PCIEM_TYPE_PCI_BRIDGE)
193 bridge_is_pcie = true;
194 }
195
196 if (bridge_is_pcie) {
197 /*
198 * The current device is not PCIe, but
199 * the bridge above it is. This is a
200 * PCIe->PCI bridge. Assume that the
201 * requester ID will be the secondary
202 * bus number with slot and function
203 * set to zero.
204 *
205 * XXX: Doesn't handle the case where
206 * the bridge is PCIe->PCI-X, and the
207 * bridge will only take ownership of
208 * requests in some cases. We should
209 * provide context entries with the
210 * same page tables for taken and
211 * non-taken transactions.
212 */
213 *rid = PCI_RID(pci_get_bus(l), 0, 0);
214 l = pcibp;
215 } else {
216 /*
217 * Neither the device nor the bridge
218 * above it are PCIe. This is a
219 * conventional PCI->PCI bridge, which
220 * will use the bridge's BSF as the
221 * requester ID.
222 */
223 *rid = pci_get_rid(pcib);
224 l = pcib;
225 }
226 }
227 }
228 return (requester);
229 }
230
231 struct dmar_ctx *
232 dmar_instantiate_ctx(struct dmar_unit *dmar, device_t dev, bool rmrr)
233 {
234 device_t requester;
235 struct dmar_ctx *ctx;
236 bool disabled;
237 uint16_t rid;
238
239 requester = dmar_get_requester(dev, &rid);
240
241 /*
242 * If the user requested the IOMMU disabled for the device, we
243 * cannot disable the DMAR, due to possibility of other
244 * devices on the same DMAR still requiring translation.
245 * Instead provide the identity mapping for the device
246 * context.
247 */
248 disabled = dmar_bus_dma_is_dev_disabled(pci_get_domain(requester),
249 pci_get_bus(requester), pci_get_slot(requester),
250 pci_get_function(requester));
251 ctx = dmar_get_ctx_for_dev(dmar, requester, rid, disabled, rmrr);
252 if (ctx == NULL)
253 return (NULL);
254 if (disabled) {
255 /*
256 * Keep the first reference on context, release the
257 * later refs.
258 */
259 DMAR_LOCK(dmar);
260 if ((ctx->flags & DMAR_CTX_DISABLED) == 0) {
261 ctx->flags |= DMAR_CTX_DISABLED;
262 DMAR_UNLOCK(dmar);
263 } else {
264 dmar_free_ctx_locked(dmar, ctx);
265 }
266 ctx = NULL;
267 }
268 return (ctx);
269 }
270
271 bus_dma_tag_t
272 dmar_get_dma_tag(device_t dev, device_t child)
273 {
274 struct dmar_unit *dmar;
275 struct dmar_ctx *ctx;
276 bus_dma_tag_t res;
277
278 dmar = dmar_find(child);
279 /* Not in scope of any DMAR ? */
280 if (dmar == NULL)
281 return (NULL);
282 if (!dmar->dma_enabled)
283 return (NULL);
284 dmar_quirks_pre_use(dmar);
285 dmar_instantiate_rmrr_ctxs(dmar);
286
287 ctx = dmar_instantiate_ctx(dmar, child, false);
288 res = ctx == NULL ? NULL : (bus_dma_tag_t)&ctx->ctx_tag;
289 return (res);
290 }
291
292 static MALLOC_DEFINE(M_DMAR_DMAMAP, "dmar_dmamap", "Intel DMAR DMA Map");
293
294 static void dmar_bus_schedule_dmamap(struct dmar_unit *unit,
295 struct bus_dmamap_dmar *map);
296
297 static int
298 dmar_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
299 bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr,
300 bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize,
301 int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
302 void *lockfuncarg, bus_dma_tag_t *dmat)
303 {
304 struct bus_dma_tag_dmar *newtag, *oldtag;
305 int error;
306
307 *dmat = NULL;
308 error = common_bus_dma_tag_create(parent != NULL ?
309 &((struct bus_dma_tag_dmar *)parent)->common : NULL, alignment,
310 boundary, lowaddr, highaddr, filter, filterarg, maxsize,
311 nsegments, maxsegsz, flags, lockfunc, lockfuncarg,
312 sizeof(struct bus_dma_tag_dmar), (void **)&newtag);
313 if (error != 0)
314 goto out;
315
316 oldtag = (struct bus_dma_tag_dmar *)parent;
317 newtag->common.impl = &bus_dma_dmar_impl;
318 newtag->ctx = oldtag->ctx;
319 newtag->owner = oldtag->owner;
320
321 *dmat = (bus_dma_tag_t)newtag;
322 out:
323 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
324 __func__, newtag, (newtag != NULL ? newtag->common.flags : 0),
325 error);
326 return (error);
327 }
328
329 static int
330 dmar_bus_dma_tag_set_domain(bus_dma_tag_t dmat)
331 {
332
333 return (0);
334 }
335
336 static int
337 dmar_bus_dma_tag_destroy(bus_dma_tag_t dmat1)
338 {
339 struct bus_dma_tag_dmar *dmat, *dmat_copy, *parent;
340 int error;
341
342 error = 0;
343 dmat_copy = dmat = (struct bus_dma_tag_dmar *)dmat1;
344
345 if (dmat != NULL) {
346 if (dmat->map_count != 0) {
347 error = EBUSY;
348 goto out;
349 }
350 while (dmat != NULL) {
351 parent = (struct bus_dma_tag_dmar *)dmat->common.parent;
352 if (atomic_fetchadd_int(&dmat->common.ref_count, -1) ==
353 1) {
354 if (dmat == &dmat->ctx->ctx_tag)
355 dmar_free_ctx(dmat->ctx);
356 free_domain(dmat->segments, M_DMAR_DMAMAP);
357 free(dmat, M_DEVBUF);
358 dmat = parent;
359 } else
360 dmat = NULL;
361 }
362 }
363 out:
364 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
365 return (error);
366 }
367
368 static int
369 dmar_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
370 {
371 struct bus_dma_tag_dmar *tag;
372 struct bus_dmamap_dmar *map;
373
374 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "%s", __func__);
375
376 tag = (struct bus_dma_tag_dmar *)dmat;
377 map = malloc_domainset(sizeof(*map), M_DMAR_DMAMAP,
378 DOMAINSET_PREF(tag->common.domain), M_NOWAIT | M_ZERO);
379 if (map == NULL) {
380 *mapp = NULL;
381 return (ENOMEM);
382 }
383 if (tag->segments == NULL) {
384 tag->segments = malloc_domainset(sizeof(bus_dma_segment_t) *
385 tag->common.nsegments, M_DMAR_DMAMAP,
386 DOMAINSET_PREF(tag->common.domain), M_NOWAIT);
387 if (tag->segments == NULL) {
388 free_domain(map, M_DMAR_DMAMAP);
389 *mapp = NULL;
390 return (ENOMEM);
391 }
392 }
393 TAILQ_INIT(&map->map_entries);
394 map->tag = tag;
395 map->locked = true;
396 map->cansleep = false;
397 tag->map_count++;
398 *mapp = (bus_dmamap_t)map;
399
400 return (0);
401 }
402
403 static int
404 dmar_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map1)
405 {
406 struct bus_dma_tag_dmar *tag;
407 struct bus_dmamap_dmar *map;
408 struct dmar_domain *domain;
409
410 tag = (struct bus_dma_tag_dmar *)dmat;
411 map = (struct bus_dmamap_dmar *)map1;
412 if (map != NULL) {
413 domain = tag->ctx->domain;
414 DMAR_DOMAIN_LOCK(domain);
415 if (!TAILQ_EMPTY(&map->map_entries)) {
416 DMAR_DOMAIN_UNLOCK(domain);
417 return (EBUSY);
418 }
419 DMAR_DOMAIN_UNLOCK(domain);
420 free_domain(map, M_DMAR_DMAMAP);
421 }
422 tag->map_count--;
423 return (0);
424 }
425
426
427 static int
428 dmar_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
429 bus_dmamap_t *mapp)
430 {
431 struct bus_dma_tag_dmar *tag;
432 struct bus_dmamap_dmar *map;
433 int error, mflags;
434 vm_memattr_t attr;
435
436 error = dmar_bus_dmamap_create(dmat, flags, mapp);
437 if (error != 0)
438 return (error);
439
440 mflags = (flags & BUS_DMA_NOWAIT) != 0 ? M_NOWAIT : M_WAITOK;
441 mflags |= (flags & BUS_DMA_ZERO) != 0 ? M_ZERO : 0;
442 attr = (flags & BUS_DMA_NOCACHE) != 0 ? VM_MEMATTR_UNCACHEABLE :
443 VM_MEMATTR_DEFAULT;
444
445 tag = (struct bus_dma_tag_dmar *)dmat;
446 map = (struct bus_dmamap_dmar *)*mapp;
447
448 if (tag->common.maxsize < PAGE_SIZE &&
449 tag->common.alignment <= tag->common.maxsize &&
450 attr == VM_MEMATTR_DEFAULT) {
451 *vaddr = malloc_domainset(tag->common.maxsize, M_DEVBUF,
452 DOMAINSET_PREF(tag->common.domain), mflags);
453 map->flags |= BUS_DMAMAP_DMAR_MALLOC;
454 } else {
455 *vaddr = (void *)kmem_alloc_attr_domainset(
456 DOMAINSET_PREF(tag->common.domain), tag->common.maxsize,
457 mflags, 0ul, BUS_SPACE_MAXADDR, attr);
458 map->flags |= BUS_DMAMAP_DMAR_KMEM_ALLOC;
459 }
460 if (*vaddr == NULL) {
461 dmar_bus_dmamap_destroy(dmat, *mapp);
462 *mapp = NULL;
463 return (ENOMEM);
464 }
465 return (0);
466 }
467
468 static void
469 dmar_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map1)
470 {
471 struct bus_dma_tag_dmar *tag;
472 struct bus_dmamap_dmar *map;
473
474 tag = (struct bus_dma_tag_dmar *)dmat;
475 map = (struct bus_dmamap_dmar *)map1;
476
477 if ((map->flags & BUS_DMAMAP_DMAR_MALLOC) != 0) {
478 free_domain(vaddr, M_DEVBUF);
479 map->flags &= ~BUS_DMAMAP_DMAR_MALLOC;
480 } else {
481 KASSERT((map->flags & BUS_DMAMAP_DMAR_KMEM_ALLOC) != 0,
482 ("dmar_bus_dmamem_free for non alloced map %p", map));
483 kmem_free((vm_offset_t)vaddr, tag->common.maxsize);
484 map->flags &= ~BUS_DMAMAP_DMAR_KMEM_ALLOC;
485 }
486
487 dmar_bus_dmamap_destroy(dmat, map1);
488 }
489
490 static int
491 dmar_bus_dmamap_load_something1(struct bus_dma_tag_dmar *tag,
492 struct bus_dmamap_dmar *map, vm_page_t *ma, int offset, bus_size_t buflen,
493 int flags, bus_dma_segment_t *segs, int *segp,
494 struct dmar_map_entries_tailq *unroll_list)
495 {
496 struct dmar_ctx *ctx;
497 struct dmar_domain *domain;
498 struct dmar_map_entry *entry;
499 dmar_gaddr_t size;
500 bus_size_t buflen1;
501 int error, idx, gas_flags, seg;
502
503 KASSERT(offset < DMAR_PAGE_SIZE, ("offset %d", offset));
504 if (segs == NULL)
505 segs = tag->segments;
506 ctx = tag->ctx;
507 domain = ctx->domain;
508 seg = *segp;
509 error = 0;
510 idx = 0;
511 while (buflen > 0) {
512 seg++;
513 if (seg >= tag->common.nsegments) {
514 error = EFBIG;
515 break;
516 }
517 buflen1 = buflen > tag->common.maxsegsz ?
518 tag->common.maxsegsz : buflen;
519 size = round_page(offset + buflen1);
520
521 /*
522 * (Too) optimistically allow split if there are more
523 * then one segments left.
524 */
525 gas_flags = map->cansleep ? DMAR_GM_CANWAIT : 0;
526 if (seg + 1 < tag->common.nsegments)
527 gas_flags |= DMAR_GM_CANSPLIT;
528
529 error = dmar_gas_map(domain, &tag->common, size, offset,
530 DMAR_MAP_ENTRY_READ | DMAR_MAP_ENTRY_WRITE,
531 gas_flags, ma + idx, &entry);
532 if (error != 0)
533 break;
534 if ((gas_flags & DMAR_GM_CANSPLIT) != 0) {
535 KASSERT(size >= entry->end - entry->start,
536 ("split increased entry size %jx %jx %jx",
537 (uintmax_t)size, (uintmax_t)entry->start,
538 (uintmax_t)entry->end));
539 size = entry->end - entry->start;
540 if (buflen1 > size)
541 buflen1 = size;
542 } else {
543 KASSERT(entry->end - entry->start == size,
544 ("no split allowed %jx %jx %jx",
545 (uintmax_t)size, (uintmax_t)entry->start,
546 (uintmax_t)entry->end));
547 }
548 if (offset + buflen1 > size)
549 buflen1 = size - offset;
550 if (buflen1 > tag->common.maxsegsz)
551 buflen1 = tag->common.maxsegsz;
552
553 KASSERT(((entry->start + offset) & (tag->common.alignment - 1))
554 == 0,
555 ("alignment failed: ctx %p start 0x%jx offset %x "
556 "align 0x%jx", ctx, (uintmax_t)entry->start, offset,
557 (uintmax_t)tag->common.alignment));
558 KASSERT(entry->end <= tag->common.lowaddr ||
559 entry->start >= tag->common.highaddr,
560 ("entry placement failed: ctx %p start 0x%jx end 0x%jx "
561 "lowaddr 0x%jx highaddr 0x%jx", ctx,
562 (uintmax_t)entry->start, (uintmax_t)entry->end,
563 (uintmax_t)tag->common.lowaddr,
564 (uintmax_t)tag->common.highaddr));
565 KASSERT(dmar_test_boundary(entry->start + offset, buflen1,
566 tag->common.boundary),
567 ("boundary failed: ctx %p start 0x%jx end 0x%jx "
568 "boundary 0x%jx", ctx, (uintmax_t)entry->start,
569 (uintmax_t)entry->end, (uintmax_t)tag->common.boundary));
570 KASSERT(buflen1 <= tag->common.maxsegsz,
571 ("segment too large: ctx %p start 0x%jx end 0x%jx "
572 "buflen1 0x%jx maxsegsz 0x%jx", ctx,
573 (uintmax_t)entry->start, (uintmax_t)entry->end,
574 (uintmax_t)buflen1, (uintmax_t)tag->common.maxsegsz));
575
576 DMAR_DOMAIN_LOCK(domain);
577 TAILQ_INSERT_TAIL(&map->map_entries, entry, dmamap_link);
578 entry->flags |= DMAR_MAP_ENTRY_MAP;
579 DMAR_DOMAIN_UNLOCK(domain);
580 TAILQ_INSERT_TAIL(unroll_list, entry, unroll_link);
581
582 segs[seg].ds_addr = entry->start + offset;
583 segs[seg].ds_len = buflen1;
584
585 idx += OFF_TO_IDX(trunc_page(offset + buflen1));
586 offset += buflen1;
587 offset &= DMAR_PAGE_MASK;
588 buflen -= buflen1;
589 }
590 if (error == 0)
591 *segp = seg;
592 return (error);
593 }
594
595 static int
596 dmar_bus_dmamap_load_something(struct bus_dma_tag_dmar *tag,
597 struct bus_dmamap_dmar *map, vm_page_t *ma, int offset, bus_size_t buflen,
598 int flags, bus_dma_segment_t *segs, int *segp)
599 {
600 struct dmar_ctx *ctx;
601 struct dmar_domain *domain;
602 struct dmar_map_entry *entry, *entry1;
603 struct dmar_map_entries_tailq unroll_list;
604 int error;
605
606 ctx = tag->ctx;
607 domain = ctx->domain;
608 atomic_add_long(&ctx->loads, 1);
609
610 TAILQ_INIT(&unroll_list);
611 error = dmar_bus_dmamap_load_something1(tag, map, ma, offset,
612 buflen, flags, segs, segp, &unroll_list);
613 if (error != 0) {
614 /*
615 * The busdma interface does not allow us to report
616 * partial buffer load, so unfortunately we have to
617 * revert all work done.
618 */
619 DMAR_DOMAIN_LOCK(domain);
620 TAILQ_FOREACH_SAFE(entry, &unroll_list, unroll_link,
621 entry1) {
622 /*
623 * No entries other than what we have created
624 * during the failed run might have been
625 * inserted there in between, since we own ctx
626 * pglock.
627 */
628 TAILQ_REMOVE(&map->map_entries, entry, dmamap_link);
629 TAILQ_REMOVE(&unroll_list, entry, unroll_link);
630 TAILQ_INSERT_TAIL(&domain->unload_entries, entry,
631 dmamap_link);
632 }
633 DMAR_DOMAIN_UNLOCK(domain);
634 taskqueue_enqueue(domain->dmar->delayed_taskqueue,
635 &domain->unload_task);
636 }
637
638 if (error == ENOMEM && (flags & BUS_DMA_NOWAIT) == 0 &&
639 !map->cansleep)
640 error = EINPROGRESS;
641 if (error == EINPROGRESS)
642 dmar_bus_schedule_dmamap(domain->dmar, map);
643 return (error);
644 }
645
646 static int
647 dmar_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map1,
648 struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
649 bus_dma_segment_t *segs, int *segp)
650 {
651 struct bus_dma_tag_dmar *tag;
652 struct bus_dmamap_dmar *map;
653
654 tag = (struct bus_dma_tag_dmar *)dmat;
655 map = (struct bus_dmamap_dmar *)map1;
656 return (dmar_bus_dmamap_load_something(tag, map, ma, ma_offs, tlen,
657 flags, segs, segp));
658 }
659
660 static int
661 dmar_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map1,
662 vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs,
663 int *segp)
664 {
665 struct bus_dma_tag_dmar *tag;
666 struct bus_dmamap_dmar *map;
667 vm_page_t *ma;
668 vm_paddr_t pstart, pend;
669 int error, i, ma_cnt, offset;
670
671 tag = (struct bus_dma_tag_dmar *)dmat;
672 map = (struct bus_dmamap_dmar *)map1;
673 pstart = trunc_page(buf);
674 pend = round_page(buf + buflen);
675 offset = buf & PAGE_MASK;
676 ma_cnt = OFF_TO_IDX(pend - pstart);
677 ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, map->cansleep ?
678 M_WAITOK : M_NOWAIT);
679 if (ma == NULL)
680 return (ENOMEM);
681 for (i = 0; i < ma_cnt; i++)
682 ma[i] = PHYS_TO_VM_PAGE(pstart + i * PAGE_SIZE);
683 error = dmar_bus_dmamap_load_something(tag, map, ma, offset, buflen,
684 flags, segs, segp);
685 free(ma, M_DEVBUF);
686 return (error);
687 }
688
689 static int
690 dmar_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map1, void *buf,
691 bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,
692 int *segp)
693 {
694 struct bus_dma_tag_dmar *tag;
695 struct bus_dmamap_dmar *map;
696 vm_page_t *ma, fma;
697 vm_paddr_t pstart, pend, paddr;
698 int error, i, ma_cnt, offset;
699
700 tag = (struct bus_dma_tag_dmar *)dmat;
701 map = (struct bus_dmamap_dmar *)map1;
702 pstart = trunc_page((vm_offset_t)buf);
703 pend = round_page((vm_offset_t)buf + buflen);
704 offset = (vm_offset_t)buf & PAGE_MASK;
705 ma_cnt = OFF_TO_IDX(pend - pstart);
706 ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, map->cansleep ?
707 M_WAITOK : M_NOWAIT);
708 if (ma == NULL)
709 return (ENOMEM);
710 if (dumping) {
711 /*
712 * If dumping, do not attempt to call
713 * PHYS_TO_VM_PAGE() at all. It may return non-NULL
714 * but the vm_page returned might be not initialized,
715 * e.g. for the kernel itself.
716 */
717 KASSERT(pmap == kernel_pmap, ("non-kernel address write"));
718 fma = malloc(sizeof(struct vm_page) * ma_cnt, M_DEVBUF,
719 M_ZERO | (map->cansleep ? M_WAITOK : M_NOWAIT));
720 if (fma == NULL) {
721 free(ma, M_DEVBUF);
722 return (ENOMEM);
723 }
724 for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) {
725 paddr = pmap_kextract(pstart);
726 vm_page_initfake(&fma[i], paddr, VM_MEMATTR_DEFAULT);
727 ma[i] = &fma[i];
728 }
729 } else {
730 fma = NULL;
731 for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) {
732 if (pmap == kernel_pmap)
733 paddr = pmap_kextract(pstart);
734 else
735 paddr = pmap_extract(pmap, pstart);
736 ma[i] = PHYS_TO_VM_PAGE(paddr);
737 KASSERT(VM_PAGE_TO_PHYS(ma[i]) == paddr,
738 ("PHYS_TO_VM_PAGE failed %jx %jx m %p",
739 (uintmax_t)paddr, (uintmax_t)VM_PAGE_TO_PHYS(ma[i]),
740 ma[i]));
741 }
742 }
743 error = dmar_bus_dmamap_load_something(tag, map, ma, offset, buflen,
744 flags, segs, segp);
745 free(ma, M_DEVBUF);
746 free(fma, M_DEVBUF);
747 return (error);
748 }
749
750 static void
751 dmar_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map1,
752 struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg)
753 {
754 struct bus_dmamap_dmar *map;
755
756 if (map1 == NULL)
757 return;
758 map = (struct bus_dmamap_dmar *)map1;
759 map->mem = *mem;
760 map->tag = (struct bus_dma_tag_dmar *)dmat;
761 map->callback = callback;
762 map->callback_arg = callback_arg;
763 }
764
765 static bus_dma_segment_t *
766 dmar_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map1,
767 bus_dma_segment_t *segs, int nsegs, int error)
768 {
769 struct bus_dma_tag_dmar *tag;
770 struct bus_dmamap_dmar *map;
771
772 tag = (struct bus_dma_tag_dmar *)dmat;
773 map = (struct bus_dmamap_dmar *)map1;
774
775 if (!map->locked) {
776 KASSERT(map->cansleep,
777 ("map not locked and not sleepable context %p", map));
778
779 /*
780 * We are called from the delayed context. Relock the
781 * driver.
782 */
783 (tag->common.lockfunc)(tag->common.lockfuncarg, BUS_DMA_LOCK);
784 map->locked = true;
785 }
786
787 if (segs == NULL)
788 segs = tag->segments;
789 return (segs);
790 }
791
792 /*
793 * The limitations of busdma KPI forces the dmar to perform the actual
794 * unload, consisting of the unmapping of the map entries page tables,
795 * from the delayed context on i386, since page table page mapping
796 * might require a sleep to be successfull. The unfortunate
797 * consequence is that the DMA requests can be served some time after
798 * the bus_dmamap_unload() call returned.
799 *
800 * On amd64, we assume that sf allocation cannot fail.
801 */
802 static void
803 dmar_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map1)
804 {
805 struct bus_dma_tag_dmar *tag;
806 struct bus_dmamap_dmar *map;
807 struct dmar_ctx *ctx;
808 struct dmar_domain *domain;
809 #if defined(__amd64__)
810 struct dmar_map_entries_tailq entries;
811 #endif
812
813 tag = (struct bus_dma_tag_dmar *)dmat;
814 map = (struct bus_dmamap_dmar *)map1;
815 ctx = tag->ctx;
816 domain = ctx->domain;
817 atomic_add_long(&ctx->unloads, 1);
818
819 #if defined(__i386__)
820 DMAR_DOMAIN_LOCK(domain);
821 TAILQ_CONCAT(&domain->unload_entries, &map->map_entries, dmamap_link);
822 DMAR_DOMAIN_UNLOCK(domain);
823 taskqueue_enqueue(domain->dmar->delayed_taskqueue,
824 &domain->unload_task);
825 #else /* defined(__amd64__) */
826 TAILQ_INIT(&entries);
827 DMAR_DOMAIN_LOCK(domain);
828 TAILQ_CONCAT(&entries, &map->map_entries, dmamap_link);
829 DMAR_DOMAIN_UNLOCK(domain);
830 THREAD_NO_SLEEPING();
831 dmar_domain_unload(domain, &entries, false);
832 THREAD_SLEEPING_OK();
833 KASSERT(TAILQ_EMPTY(&entries), ("lazy dmar_ctx_unload %p", ctx));
834 #endif
835 }
836
837 static void
838 dmar_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map,
839 bus_dmasync_op_t op)
840 {
841 }
842
843 struct bus_dma_impl bus_dma_dmar_impl = {
844 .tag_create = dmar_bus_dma_tag_create,
845 .tag_destroy = dmar_bus_dma_tag_destroy,
846 .tag_set_domain = dmar_bus_dma_tag_set_domain,
847 .map_create = dmar_bus_dmamap_create,
848 .map_destroy = dmar_bus_dmamap_destroy,
849 .mem_alloc = dmar_bus_dmamem_alloc,
850 .mem_free = dmar_bus_dmamem_free,
851 .load_phys = dmar_bus_dmamap_load_phys,
852 .load_buffer = dmar_bus_dmamap_load_buffer,
853 .load_ma = dmar_bus_dmamap_load_ma,
854 .map_waitok = dmar_bus_dmamap_waitok,
855 .map_complete = dmar_bus_dmamap_complete,
856 .map_unload = dmar_bus_dmamap_unload,
857 .map_sync = dmar_bus_dmamap_sync,
858 };
859
860 static void
861 dmar_bus_task_dmamap(void *arg, int pending)
862 {
863 struct bus_dma_tag_dmar *tag;
864 struct bus_dmamap_dmar *map;
865 struct dmar_unit *unit;
866
867 unit = arg;
868 DMAR_LOCK(unit);
869 while ((map = TAILQ_FIRST(&unit->delayed_maps)) != NULL) {
870 TAILQ_REMOVE(&unit->delayed_maps, map, delay_link);
871 DMAR_UNLOCK(unit);
872 tag = map->tag;
873 map->cansleep = true;
874 map->locked = false;
875 bus_dmamap_load_mem((bus_dma_tag_t)tag, (bus_dmamap_t)map,
876 &map->mem, map->callback, map->callback_arg,
877 BUS_DMA_WAITOK);
878 map->cansleep = false;
879 if (map->locked) {
880 (tag->common.lockfunc)(tag->common.lockfuncarg,
881 BUS_DMA_UNLOCK);
882 } else
883 map->locked = true;
884 map->cansleep = false;
885 DMAR_LOCK(unit);
886 }
887 DMAR_UNLOCK(unit);
888 }
889
890 static void
891 dmar_bus_schedule_dmamap(struct dmar_unit *unit, struct bus_dmamap_dmar *map)
892 {
893
894 map->locked = false;
895 DMAR_LOCK(unit);
896 TAILQ_INSERT_TAIL(&unit->delayed_maps, map, delay_link);
897 DMAR_UNLOCK(unit);
898 taskqueue_enqueue(unit->delayed_taskqueue, &unit->dmamap_load_task);
899 }
900
901 int
902 dmar_init_busdma(struct dmar_unit *unit)
903 {
904
905 unit->dma_enabled = 1;
906 TUNABLE_INT_FETCH("hw.dmar.dma", &unit->dma_enabled);
907 TAILQ_INIT(&unit->delayed_maps);
908 TASK_INIT(&unit->dmamap_load_task, 0, dmar_bus_task_dmamap, unit);
909 unit->delayed_taskqueue = taskqueue_create("dmar", M_WAITOK,
910 taskqueue_thread_enqueue, &unit->delayed_taskqueue);
911 taskqueue_start_threads(&unit->delayed_taskqueue, 1, PI_DISK,
912 "dmar%d busdma taskq", unit->unit);
913 return (0);
914 }
915
916 void
917 dmar_fini_busdma(struct dmar_unit *unit)
918 {
919
920 if (unit->delayed_taskqueue == NULL)
921 return;
922
923 taskqueue_drain(unit->delayed_taskqueue, &unit->dmamap_load_task);
924 taskqueue_free(unit->delayed_taskqueue);
925 unit->delayed_taskqueue = NULL;
926 }
Cache object: a15bbdcf51293587cc6be3dfc7918816
|