1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2013 The FreeBSD Foundation
5 * All rights reserved.
6 *
7 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
8 * under sponsorship from the FreeBSD Foundation.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD: releng/12.0/sys/x86/iommu/intel_ctx.c 326263 2017-11-27 15:11:47Z pfg $");
34
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/malloc.h>
38 #include <sys/bus.h>
39 #include <sys/interrupt.h>
40 #include <sys/kernel.h>
41 #include <sys/ktr.h>
42 #include <sys/limits.h>
43 #include <sys/lock.h>
44 #include <sys/memdesc.h>
45 #include <sys/mutex.h>
46 #include <sys/proc.h>
47 #include <sys/rwlock.h>
48 #include <sys/rman.h>
49 #include <sys/sysctl.h>
50 #include <sys/taskqueue.h>
51 #include <sys/tree.h>
52 #include <sys/uio.h>
53 #include <sys/vmem.h>
54 #include <vm/vm.h>
55 #include <vm/vm_extern.h>
56 #include <vm/vm_kern.h>
57 #include <vm/vm_object.h>
58 #include <vm/vm_page.h>
59 #include <vm/vm_pager.h>
60 #include <vm/vm_map.h>
61 #include <machine/atomic.h>
62 #include <machine/bus.h>
63 #include <machine/md_var.h>
64 #include <machine/specialreg.h>
65 #include <x86/include/busdma_impl.h>
66 #include <x86/iommu/intel_reg.h>
67 #include <x86/iommu/busdma_dmar.h>
68 #include <x86/iommu/intel_dmar.h>
69 #include <dev/pci/pcireg.h>
70 #include <dev/pci/pcivar.h>
71
72 static MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context");
73 static MALLOC_DEFINE(M_DMAR_DOMAIN, "dmar_dom", "Intel DMAR Domain");
74
75 static void dmar_domain_unload_task(void *arg, int pending);
76 static void dmar_unref_domain_locked(struct dmar_unit *dmar,
77 struct dmar_domain *domain);
78 static void dmar_domain_destroy(struct dmar_domain *domain);
79
80 static void
81 dmar_ensure_ctx_page(struct dmar_unit *dmar, int bus)
82 {
83 struct sf_buf *sf;
84 dmar_root_entry_t *re;
85 vm_page_t ctxm;
86
87 /*
88 * Allocated context page must be linked.
89 */
90 ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, DMAR_PGF_NOALLOC);
91 if (ctxm != NULL)
92 return;
93
94 /*
95 * Page not present, allocate and link. Note that other
96 * thread might execute this sequence in parallel. This
97 * should be safe, because the context entries written by both
98 * threads are equal.
99 */
100 TD_PREP_PINNED_ASSERT;
101 ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, DMAR_PGF_ZERO |
102 DMAR_PGF_WAITOK);
103 re = dmar_map_pgtbl(dmar->ctx_obj, 0, DMAR_PGF_NOALLOC, &sf);
104 re += bus;
105 dmar_pte_store(&re->r1, DMAR_ROOT_R1_P | (DMAR_ROOT_R1_CTP_MASK &
106 VM_PAGE_TO_PHYS(ctxm)));
107 dmar_flush_root_to_ram(dmar, re);
108 dmar_unmap_pgtbl(sf);
109 TD_PINNED_ASSERT;
110 }
111
112 static dmar_ctx_entry_t *
113 dmar_map_ctx_entry(struct dmar_ctx *ctx, struct sf_buf **sfp)
114 {
115 dmar_ctx_entry_t *ctxp;
116
117 ctxp = dmar_map_pgtbl(ctx->domain->dmar->ctx_obj, 1 +
118 PCI_RID2BUS(ctx->rid), DMAR_PGF_NOALLOC | DMAR_PGF_WAITOK, sfp);
119 ctxp += ctx->rid & 0xff;
120 return (ctxp);
121 }
122
123 static void
124 ctx_tag_init(struct dmar_ctx *ctx, device_t dev)
125 {
126 bus_addr_t maxaddr;
127
128 maxaddr = MIN(ctx->domain->end, BUS_SPACE_MAXADDR);
129 ctx->ctx_tag.common.ref_count = 1; /* Prevent free */
130 ctx->ctx_tag.common.impl = &bus_dma_dmar_impl;
131 ctx->ctx_tag.common.boundary = PCI_DMA_BOUNDARY;
132 ctx->ctx_tag.common.lowaddr = maxaddr;
133 ctx->ctx_tag.common.highaddr = maxaddr;
134 ctx->ctx_tag.common.maxsize = maxaddr;
135 ctx->ctx_tag.common.nsegments = BUS_SPACE_UNRESTRICTED;
136 ctx->ctx_tag.common.maxsegsz = maxaddr;
137 ctx->ctx_tag.ctx = ctx;
138 ctx->ctx_tag.owner = dev;
139 }
140
141 static void
142 ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp, bool move)
143 {
144 struct dmar_unit *unit;
145 struct dmar_domain *domain;
146 vm_page_t ctx_root;
147
148 domain = ctx->domain;
149 unit = domain->dmar;
150 KASSERT(move || (ctxp->ctx1 == 0 && ctxp->ctx2 == 0),
151 ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx",
152 unit->unit, pci_get_bus(ctx->ctx_tag.owner),
153 pci_get_slot(ctx->ctx_tag.owner),
154 pci_get_function(ctx->ctx_tag.owner),
155 ctxp->ctx1, ctxp->ctx2));
156 /*
157 * For update due to move, the store is not atomic. It is
158 * possible that DMAR read upper doubleword, while low
159 * doubleword is not yet updated. The domain id is stored in
160 * the upper doubleword, while the table pointer in the lower.
161 *
162 * There is no good solution, for the same reason it is wrong
163 * to clear P bit in the ctx entry for update.
164 */
165 dmar_pte_store1(&ctxp->ctx2, DMAR_CTX2_DID(domain->domain) |
166 domain->awlvl);
167 if ((domain->flags & DMAR_DOMAIN_IDMAP) != 0 &&
168 (unit->hw_ecap & DMAR_ECAP_PT) != 0) {
169 KASSERT(domain->pgtbl_obj == NULL,
170 ("ctx %p non-null pgtbl_obj", ctx));
171 dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_PASS | DMAR_CTX1_P);
172 } else {
173 ctx_root = dmar_pgalloc(domain->pgtbl_obj, 0, DMAR_PGF_NOALLOC);
174 dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_UNTR |
175 (DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) |
176 DMAR_CTX1_P);
177 }
178 dmar_flush_ctx_to_ram(unit, ctxp);
179 }
180
181 static int
182 dmar_flush_for_ctx_entry(struct dmar_unit *dmar, bool force)
183 {
184 int error;
185
186 /*
187 * If dmar declares Caching Mode as Set, follow 11.5 "Caching
188 * Mode Consideration" and do the (global) invalidation of the
189 * negative TLB entries.
190 */
191 if ((dmar->hw_cap & DMAR_CAP_CM) == 0 && !force)
192 return (0);
193 if (dmar->qi_enabled) {
194 dmar_qi_invalidate_ctx_glob_locked(dmar);
195 if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 || force)
196 dmar_qi_invalidate_iotlb_glob_locked(dmar);
197 return (0);
198 }
199 error = dmar_inv_ctx_glob(dmar);
200 if (error == 0 && ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 || force))
201 error = dmar_inv_iotlb_glob(dmar);
202 return (error);
203 }
204
205 static int
206 domain_init_rmrr(struct dmar_domain *domain, device_t dev)
207 {
208 struct dmar_map_entries_tailq rmrr_entries;
209 struct dmar_map_entry *entry, *entry1;
210 vm_page_t *ma;
211 dmar_gaddr_t start, end;
212 vm_pindex_t size, i;
213 int error, error1;
214
215 error = 0;
216 TAILQ_INIT(&rmrr_entries);
217 dmar_dev_parse_rmrr(domain, dev, &rmrr_entries);
218 TAILQ_FOREACH_SAFE(entry, &rmrr_entries, unroll_link, entry1) {
219 /*
220 * VT-d specification requires that the start of an
221 * RMRR entry is 4k-aligned. Buggy BIOSes put
222 * anything into the start and end fields. Truncate
223 * and round as neccesary.
224 *
225 * We also allow the overlapping RMRR entries, see
226 * dmar_gas_alloc_region().
227 */
228 start = entry->start;
229 end = entry->end;
230 entry->start = trunc_page(start);
231 entry->end = round_page(end);
232 if (entry->start == entry->end) {
233 /* Workaround for some AMI (?) BIOSes */
234 if (bootverbose) {
235 device_printf(dev, "BIOS bug: dmar%d RMRR "
236 "region (%jx, %jx) corrected\n",
237 domain->dmar->unit, start, end);
238 }
239 entry->end += DMAR_PAGE_SIZE * 0x20;
240 }
241 size = OFF_TO_IDX(entry->end - entry->start);
242 ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK);
243 for (i = 0; i < size; i++) {
244 ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i,
245 VM_MEMATTR_DEFAULT);
246 }
247 error1 = dmar_gas_map_region(domain, entry,
248 DMAR_MAP_ENTRY_READ | DMAR_MAP_ENTRY_WRITE,
249 DMAR_GM_CANWAIT, ma);
250 /*
251 * Non-failed RMRR entries are owned by context rb
252 * tree. Get rid of the failed entry, but do not stop
253 * the loop. Rest of the parsed RMRR entries are
254 * loaded and removed on the context destruction.
255 */
256 if (error1 == 0 && entry->end != entry->start) {
257 DMAR_LOCK(domain->dmar);
258 domain->refs++; /* XXXKIB prevent free */
259 domain->flags |= DMAR_DOMAIN_RMRR;
260 DMAR_UNLOCK(domain->dmar);
261 } else {
262 if (error1 != 0) {
263 device_printf(dev,
264 "dmar%d failed to map RMRR region (%jx, %jx) %d\n",
265 domain->dmar->unit, start, end, error1);
266 error = error1;
267 }
268 TAILQ_REMOVE(&rmrr_entries, entry, unroll_link);
269 dmar_gas_free_entry(domain, entry);
270 }
271 for (i = 0; i < size; i++)
272 vm_page_putfake(ma[i]);
273 free(ma, M_TEMP);
274 }
275 return (error);
276 }
277
278 static struct dmar_domain *
279 dmar_domain_alloc(struct dmar_unit *dmar, bool id_mapped)
280 {
281 struct dmar_domain *domain;
282 int error, id, mgaw;
283
284 id = alloc_unr(dmar->domids);
285 if (id == -1)
286 return (NULL);
287 domain = malloc(sizeof(*domain), M_DMAR_DOMAIN, M_WAITOK | M_ZERO);
288 domain->domain = id;
289 LIST_INIT(&domain->contexts);
290 RB_INIT(&domain->rb_root);
291 TAILQ_INIT(&domain->unload_entries);
292 TASK_INIT(&domain->unload_task, 0, dmar_domain_unload_task, domain);
293 mtx_init(&domain->lock, "dmardom", NULL, MTX_DEF);
294 domain->dmar = dmar;
295
296 /*
297 * For now, use the maximal usable physical address of the
298 * installed memory to calculate the mgaw on id_mapped domain.
299 * It is useful for the identity mapping, and less so for the
300 * virtualized bus address space.
301 */
302 domain->end = id_mapped ? ptoa(Maxmem) : BUS_SPACE_MAXADDR;
303 mgaw = dmar_maxaddr2mgaw(dmar, domain->end, !id_mapped);
304 error = domain_set_agaw(domain, mgaw);
305 if (error != 0)
306 goto fail;
307 if (!id_mapped)
308 /* Use all supported address space for remapping. */
309 domain->end = 1ULL << (domain->agaw - 1);
310
311 dmar_gas_init_domain(domain);
312
313 if (id_mapped) {
314 if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) {
315 domain->pgtbl_obj = domain_get_idmap_pgtbl(domain,
316 domain->end);
317 }
318 domain->flags |= DMAR_DOMAIN_IDMAP;
319 } else {
320 error = domain_alloc_pgtbl(domain);
321 if (error != 0)
322 goto fail;
323 /* Disable local apic region access */
324 error = dmar_gas_reserve_region(domain, 0xfee00000,
325 0xfeefffff + 1);
326 if (error != 0)
327 goto fail;
328 }
329 return (domain);
330
331 fail:
332 dmar_domain_destroy(domain);
333 return (NULL);
334 }
335
336 static struct dmar_ctx *
337 dmar_ctx_alloc(struct dmar_domain *domain, uint16_t rid)
338 {
339 struct dmar_ctx *ctx;
340
341 ctx = malloc(sizeof(*ctx), M_DMAR_CTX, M_WAITOK | M_ZERO);
342 ctx->domain = domain;
343 ctx->rid = rid;
344 ctx->refs = 1;
345 return (ctx);
346 }
347
348 static void
349 dmar_ctx_link(struct dmar_ctx *ctx)
350 {
351 struct dmar_domain *domain;
352
353 domain = ctx->domain;
354 DMAR_ASSERT_LOCKED(domain->dmar);
355 KASSERT(domain->refs >= domain->ctx_cnt,
356 ("dom %p ref underflow %d %d", domain, domain->refs,
357 domain->ctx_cnt));
358 domain->refs++;
359 domain->ctx_cnt++;
360 LIST_INSERT_HEAD(&domain->contexts, ctx, link);
361 }
362
363 static void
364 dmar_ctx_unlink(struct dmar_ctx *ctx)
365 {
366 struct dmar_domain *domain;
367
368 domain = ctx->domain;
369 DMAR_ASSERT_LOCKED(domain->dmar);
370 KASSERT(domain->refs > 0,
371 ("domain %p ctx dtr refs %d", domain, domain->refs));
372 KASSERT(domain->ctx_cnt >= domain->refs,
373 ("domain %p ctx dtr refs %d ctx_cnt %d", domain,
374 domain->refs, domain->ctx_cnt));
375 domain->refs--;
376 domain->ctx_cnt--;
377 LIST_REMOVE(ctx, link);
378 }
379
380 static void
381 dmar_domain_destroy(struct dmar_domain *domain)
382 {
383
384 KASSERT(TAILQ_EMPTY(&domain->unload_entries),
385 ("unfinished unloads %p", domain));
386 KASSERT(LIST_EMPTY(&domain->contexts),
387 ("destroying dom %p with contexts", domain));
388 KASSERT(domain->ctx_cnt == 0,
389 ("destroying dom %p with ctx_cnt %d", domain, domain->ctx_cnt));
390 KASSERT(domain->refs == 0,
391 ("destroying dom %p with refs %d", domain, domain->refs));
392 if ((domain->flags & DMAR_DOMAIN_GAS_INITED) != 0) {
393 DMAR_DOMAIN_LOCK(domain);
394 dmar_gas_fini_domain(domain);
395 DMAR_DOMAIN_UNLOCK(domain);
396 }
397 if ((domain->flags & DMAR_DOMAIN_PGTBL_INITED) != 0) {
398 if (domain->pgtbl_obj != NULL)
399 DMAR_DOMAIN_PGLOCK(domain);
400 domain_free_pgtbl(domain);
401 }
402 mtx_destroy(&domain->lock);
403 free_unr(domain->dmar->domids, domain->domain);
404 free(domain, M_DMAR_DOMAIN);
405 }
406
407 struct dmar_ctx *
408 dmar_get_ctx_for_dev(struct dmar_unit *dmar, device_t dev, uint16_t rid,
409 bool id_mapped, bool rmrr_init)
410 {
411 struct dmar_domain *domain, *domain1;
412 struct dmar_ctx *ctx, *ctx1;
413 dmar_ctx_entry_t *ctxp;
414 struct sf_buf *sf;
415 int bus, slot, func, error;
416 bool enable;
417
418 bus = pci_get_bus(dev);
419 slot = pci_get_slot(dev);
420 func = pci_get_function(dev);
421 enable = false;
422 TD_PREP_PINNED_ASSERT;
423 DMAR_LOCK(dmar);
424 ctx = dmar_find_ctx_locked(dmar, rid);
425 error = 0;
426 if (ctx == NULL) {
427 /*
428 * Perform the allocations which require sleep or have
429 * higher chance to succeed if the sleep is allowed.
430 */
431 DMAR_UNLOCK(dmar);
432 dmar_ensure_ctx_page(dmar, PCI_RID2BUS(rid));
433 domain1 = dmar_domain_alloc(dmar, id_mapped);
434 if (domain1 == NULL) {
435 TD_PINNED_ASSERT;
436 return (NULL);
437 }
438 if (!id_mapped) {
439 error = domain_init_rmrr(domain1, dev);
440 if (error != 0) {
441 dmar_domain_destroy(domain1);
442 TD_PINNED_ASSERT;
443 return (NULL);
444 }
445 }
446 ctx1 = dmar_ctx_alloc(domain1, rid);
447 ctxp = dmar_map_ctx_entry(ctx1, &sf);
448 DMAR_LOCK(dmar);
449
450 /*
451 * Recheck the contexts, other thread might have
452 * already allocated needed one.
453 */
454 ctx = dmar_find_ctx_locked(dmar, rid);
455 if (ctx == NULL) {
456 domain = domain1;
457 ctx = ctx1;
458 dmar_ctx_link(ctx);
459 ctx->ctx_tag.owner = dev;
460 ctx_tag_init(ctx, dev);
461
462 /*
463 * This is the first activated context for the
464 * DMAR unit. Enable the translation after
465 * everything is set up.
466 */
467 if (LIST_EMPTY(&dmar->domains))
468 enable = true;
469 LIST_INSERT_HEAD(&dmar->domains, domain, link);
470 ctx_id_entry_init(ctx, ctxp, false);
471 device_printf(dev,
472 "dmar%d pci%d:%d:%d:%d rid %x domain %d mgaw %d "
473 "agaw %d %s-mapped\n",
474 dmar->unit, dmar->segment, bus, slot,
475 func, rid, domain->domain, domain->mgaw,
476 domain->agaw, id_mapped ? "id" : "re");
477 dmar_unmap_pgtbl(sf);
478 } else {
479 dmar_unmap_pgtbl(sf);
480 dmar_domain_destroy(domain1);
481 /* Nothing needs to be done to destroy ctx1. */
482 free(ctx1, M_DMAR_CTX);
483 domain = ctx->domain;
484 ctx->refs++; /* tag referenced us */
485 }
486 } else {
487 domain = ctx->domain;
488 ctx->refs++; /* tag referenced us */
489 }
490
491 error = dmar_flush_for_ctx_entry(dmar, enable);
492 if (error != 0) {
493 dmar_free_ctx_locked(dmar, ctx);
494 TD_PINNED_ASSERT;
495 return (NULL);
496 }
497
498 /*
499 * The dmar lock was potentially dropped between check for the
500 * empty context list and now. Recheck the state of GCMD_TE
501 * to avoid unneeded command.
502 */
503 if (enable && !rmrr_init && (dmar->hw_gcmd & DMAR_GCMD_TE) == 0) {
504 error = dmar_enable_translation(dmar);
505 if (error != 0) {
506 dmar_free_ctx_locked(dmar, ctx);
507 TD_PINNED_ASSERT;
508 return (NULL);
509 }
510 }
511 DMAR_UNLOCK(dmar);
512 TD_PINNED_ASSERT;
513 return (ctx);
514 }
515
516 int
517 dmar_move_ctx_to_domain(struct dmar_domain *domain, struct dmar_ctx *ctx)
518 {
519 struct dmar_unit *dmar;
520 struct dmar_domain *old_domain;
521 dmar_ctx_entry_t *ctxp;
522 struct sf_buf *sf;
523 int error;
524
525 dmar = domain->dmar;
526 old_domain = ctx->domain;
527 if (domain == old_domain)
528 return (0);
529 KASSERT(old_domain->dmar == dmar,
530 ("domain %p %u moving between dmars %u %u", domain,
531 domain->domain, old_domain->dmar->unit, domain->dmar->unit));
532 TD_PREP_PINNED_ASSERT;
533
534 ctxp = dmar_map_ctx_entry(ctx, &sf);
535 DMAR_LOCK(dmar);
536 dmar_ctx_unlink(ctx);
537 ctx->domain = domain;
538 dmar_ctx_link(ctx);
539 ctx_id_entry_init(ctx, ctxp, true);
540 dmar_unmap_pgtbl(sf);
541 error = dmar_flush_for_ctx_entry(dmar, true);
542 /* If flush failed, rolling back would not work as well. */
543 printf("dmar%d rid %x domain %d->%d %s-mapped\n",
544 dmar->unit, ctx->rid, old_domain->domain, domain->domain,
545 (domain->flags & DMAR_DOMAIN_IDMAP) != 0 ? "id" : "re");
546 dmar_unref_domain_locked(dmar, old_domain);
547 TD_PINNED_ASSERT;
548 return (error);
549 }
550
551 static void
552 dmar_unref_domain_locked(struct dmar_unit *dmar, struct dmar_domain *domain)
553 {
554
555 DMAR_ASSERT_LOCKED(dmar);
556 KASSERT(domain->refs >= 1,
557 ("dmar %d domain %p refs %u", dmar->unit, domain, domain->refs));
558 KASSERT(domain->refs > domain->ctx_cnt,
559 ("dmar %d domain %p refs %d ctx_cnt %d", dmar->unit, domain,
560 domain->refs, domain->ctx_cnt));
561
562 if (domain->refs > 1) {
563 domain->refs--;
564 DMAR_UNLOCK(dmar);
565 return;
566 }
567
568 KASSERT((domain->flags & DMAR_DOMAIN_RMRR) == 0,
569 ("lost ref on RMRR domain %p", domain));
570
571 LIST_REMOVE(domain, link);
572 DMAR_UNLOCK(dmar);
573
574 taskqueue_drain(dmar->delayed_taskqueue, &domain->unload_task);
575 dmar_domain_destroy(domain);
576 }
577
578 void
579 dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx)
580 {
581 struct sf_buf *sf;
582 dmar_ctx_entry_t *ctxp;
583 struct dmar_domain *domain;
584
585 DMAR_ASSERT_LOCKED(dmar);
586 KASSERT(ctx->refs >= 1,
587 ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs));
588
589 /*
590 * If our reference is not last, only the dereference should
591 * be performed.
592 */
593 if (ctx->refs > 1) {
594 ctx->refs--;
595 DMAR_UNLOCK(dmar);
596 return;
597 }
598
599 KASSERT((ctx->flags & DMAR_CTX_DISABLED) == 0,
600 ("lost ref on disabled ctx %p", ctx));
601
602 /*
603 * Otherwise, the context entry must be cleared before the
604 * page table is destroyed. The mapping of the context
605 * entries page could require sleep, unlock the dmar.
606 */
607 DMAR_UNLOCK(dmar);
608 TD_PREP_PINNED_ASSERT;
609 ctxp = dmar_map_ctx_entry(ctx, &sf);
610 DMAR_LOCK(dmar);
611 KASSERT(ctx->refs >= 1,
612 ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs));
613
614 /*
615 * Other thread might have referenced the context, in which
616 * case again only the dereference should be performed.
617 */
618 if (ctx->refs > 1) {
619 ctx->refs--;
620 DMAR_UNLOCK(dmar);
621 dmar_unmap_pgtbl(sf);
622 TD_PINNED_ASSERT;
623 return;
624 }
625
626 KASSERT((ctx->flags & DMAR_CTX_DISABLED) == 0,
627 ("lost ref on disabled ctx %p", ctx));
628
629 /*
630 * Clear the context pointer and flush the caches.
631 * XXXKIB: cannot do this if any RMRR entries are still present.
632 */
633 dmar_pte_clear(&ctxp->ctx1);
634 ctxp->ctx2 = 0;
635 dmar_flush_ctx_to_ram(dmar, ctxp);
636 dmar_inv_ctx_glob(dmar);
637 if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) {
638 if (dmar->qi_enabled)
639 dmar_qi_invalidate_iotlb_glob_locked(dmar);
640 else
641 dmar_inv_iotlb_glob(dmar);
642 }
643 dmar_unmap_pgtbl(sf);
644 domain = ctx->domain;
645 dmar_ctx_unlink(ctx);
646 free(ctx, M_DMAR_CTX);
647 dmar_unref_domain_locked(dmar, domain);
648 TD_PINNED_ASSERT;
649 }
650
651 void
652 dmar_free_ctx(struct dmar_ctx *ctx)
653 {
654 struct dmar_unit *dmar;
655
656 dmar = ctx->domain->dmar;
657 DMAR_LOCK(dmar);
658 dmar_free_ctx_locked(dmar, ctx);
659 }
660
661 /*
662 * Returns with the domain locked.
663 */
664 struct dmar_ctx *
665 dmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid)
666 {
667 struct dmar_domain *domain;
668 struct dmar_ctx *ctx;
669
670 DMAR_ASSERT_LOCKED(dmar);
671
672 LIST_FOREACH(domain, &dmar->domains, link) {
673 LIST_FOREACH(ctx, &domain->contexts, link) {
674 if (ctx->rid == rid)
675 return (ctx);
676 }
677 }
678 return (NULL);
679 }
680
681 void
682 dmar_domain_free_entry(struct dmar_map_entry *entry, bool free)
683 {
684 struct dmar_domain *domain;
685
686 domain = entry->domain;
687 DMAR_DOMAIN_LOCK(domain);
688 if ((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0)
689 dmar_gas_free_region(domain, entry);
690 else
691 dmar_gas_free_space(domain, entry);
692 DMAR_DOMAIN_UNLOCK(domain);
693 if (free)
694 dmar_gas_free_entry(domain, entry);
695 else
696 entry->flags = 0;
697 }
698
699 void
700 dmar_domain_unload_entry(struct dmar_map_entry *entry, bool free)
701 {
702 struct dmar_unit *unit;
703
704 unit = entry->domain->dmar;
705 if (unit->qi_enabled) {
706 DMAR_LOCK(unit);
707 dmar_qi_invalidate_locked(entry->domain, entry->start,
708 entry->end - entry->start, &entry->gseq, true);
709 if (!free)
710 entry->flags |= DMAR_MAP_ENTRY_QI_NF;
711 TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link);
712 DMAR_UNLOCK(unit);
713 } else {
714 domain_flush_iotlb_sync(entry->domain, entry->start,
715 entry->end - entry->start);
716 dmar_domain_free_entry(entry, free);
717 }
718 }
719
720 static bool
721 dmar_domain_unload_emit_wait(struct dmar_domain *domain,
722 struct dmar_map_entry *entry)
723 {
724
725 if (TAILQ_NEXT(entry, dmamap_link) == NULL)
726 return (true);
727 return (domain->batch_no++ % dmar_batch_coalesce == 0);
728 }
729
730 void
731 dmar_domain_unload(struct dmar_domain *domain,
732 struct dmar_map_entries_tailq *entries, bool cansleep)
733 {
734 struct dmar_unit *unit;
735 struct dmar_map_entry *entry, *entry1;
736 int error;
737
738 unit = domain->dmar;
739
740 TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
741 KASSERT((entry->flags & DMAR_MAP_ENTRY_MAP) != 0,
742 ("not mapped entry %p %p", domain, entry));
743 error = domain_unmap_buf(domain, entry->start, entry->end -
744 entry->start, cansleep ? DMAR_PGF_WAITOK : 0);
745 KASSERT(error == 0, ("unmap %p error %d", domain, error));
746 if (!unit->qi_enabled) {
747 domain_flush_iotlb_sync(domain, entry->start,
748 entry->end - entry->start);
749 TAILQ_REMOVE(entries, entry, dmamap_link);
750 dmar_domain_free_entry(entry, true);
751 }
752 }
753 if (TAILQ_EMPTY(entries))
754 return;
755
756 KASSERT(unit->qi_enabled, ("loaded entry left"));
757 DMAR_LOCK(unit);
758 TAILQ_FOREACH(entry, entries, dmamap_link) {
759 dmar_qi_invalidate_locked(domain, entry->start, entry->end -
760 entry->start, &entry->gseq,
761 dmar_domain_unload_emit_wait(domain, entry));
762 }
763 TAILQ_CONCAT(&unit->tlb_flush_entries, entries, dmamap_link);
764 DMAR_UNLOCK(unit);
765 }
766
767 static void
768 dmar_domain_unload_task(void *arg, int pending)
769 {
770 struct dmar_domain *domain;
771 struct dmar_map_entries_tailq entries;
772
773 domain = arg;
774 TAILQ_INIT(&entries);
775
776 for (;;) {
777 DMAR_DOMAIN_LOCK(domain);
778 TAILQ_SWAP(&domain->unload_entries, &entries, dmar_map_entry,
779 dmamap_link);
780 DMAR_DOMAIN_UNLOCK(domain);
781 if (TAILQ_EMPTY(&entries))
782 break;
783 dmar_domain_unload(domain, &entries, true);
784 }
785 }
Cache object: a2dfce3d7b746227e0eb2bc1395bc182
|