1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2013 The FreeBSD Foundation
5 * All rights reserved.
6 *
7 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
8 * under sponsorship from the FreeBSD Foundation.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
34
35 #include "opt_acpi.h"
36
37 #include <sys/param.h>
38 #include <sys/bus.h>
39 #include <sys/kernel.h>
40 #include <sys/malloc.h>
41 #include <sys/memdesc.h>
42 #include <sys/module.h>
43 #include <sys/rman.h>
44 #include <sys/taskqueue.h>
45 #include <sys/time.h>
46 #include <sys/tree.h>
47 #include <sys/vmem.h>
48 #include <vm/vm.h>
49 #include <vm/vm_extern.h>
50 #include <vm/vm_kern.h>
51 #include <vm/vm_page.h>
52 #include <vm/vm_map.h>
53 #include <contrib/dev/acpica/include/acpi.h>
54 #include <contrib/dev/acpica/include/accommon.h>
55 #include <dev/acpica/acpivar.h>
56 #include <dev/pci/pcireg.h>
57 #include <machine/bus.h>
58 #include <machine/cpu.h>
59 #include <x86/include/busdma_impl.h>
60 #include <dev/iommu/busdma_iommu.h>
61 #include <x86/iommu/intel_reg.h>
62 #include <x86/iommu/intel_dmar.h>
63
64 static bool
65 dmar_qi_seq_processed(const struct dmar_unit *unit,
66 const struct iommu_qi_genseq *pseq)
67 {
68
69 return (pseq->gen < unit->inv_waitd_gen ||
70 (pseq->gen == unit->inv_waitd_gen &&
71 pseq->seq <= unit->inv_waitd_seq_hw));
72 }
73
74 static int
75 dmar_enable_qi(struct dmar_unit *unit)
76 {
77 int error;
78
79 DMAR_ASSERT_LOCKED(unit);
80 unit->hw_gcmd |= DMAR_GCMD_QIE;
81 dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
82 DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES)
83 != 0));
84 return (error);
85 }
86
87 static int
88 dmar_disable_qi(struct dmar_unit *unit)
89 {
90 int error;
91
92 DMAR_ASSERT_LOCKED(unit);
93 unit->hw_gcmd &= ~DMAR_GCMD_QIE;
94 dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
95 DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES)
96 == 0));
97 return (error);
98 }
99
100 static void
101 dmar_qi_advance_tail(struct dmar_unit *unit)
102 {
103
104 DMAR_ASSERT_LOCKED(unit);
105 dmar_write4(unit, DMAR_IQT_REG, unit->inv_queue_tail);
106 }
107
108 static void
109 dmar_qi_ensure(struct dmar_unit *unit, int descr_count)
110 {
111 uint32_t head;
112 int bytes;
113
114 DMAR_ASSERT_LOCKED(unit);
115 bytes = descr_count << DMAR_IQ_DESCR_SZ_SHIFT;
116 for (;;) {
117 if (bytes <= unit->inv_queue_avail)
118 break;
119 /* refill */
120 head = dmar_read4(unit, DMAR_IQH_REG);
121 head &= DMAR_IQH_MASK;
122 unit->inv_queue_avail = head - unit->inv_queue_tail -
123 DMAR_IQ_DESCR_SZ;
124 if (head <= unit->inv_queue_tail)
125 unit->inv_queue_avail += unit->inv_queue_size;
126 if (bytes <= unit->inv_queue_avail)
127 break;
128
129 /*
130 * No space in the queue, do busy wait. Hardware must
131 * make a progress. But first advance the tail to
132 * inform the descriptor streamer about entries we
133 * might have already filled, otherwise they could
134 * clog the whole queue..
135 */
136 dmar_qi_advance_tail(unit);
137 unit->inv_queue_full++;
138 cpu_spinwait();
139 }
140 unit->inv_queue_avail -= bytes;
141 }
142
143 static void
144 dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2)
145 {
146
147 DMAR_ASSERT_LOCKED(unit);
148 *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data1;
149 unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
150 KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
151 ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
152 (uintmax_t)unit->inv_queue_size));
153 unit->inv_queue_tail &= unit->inv_queue_size - 1;
154 *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data2;
155 unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
156 KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
157 ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
158 (uintmax_t)unit->inv_queue_size));
159 unit->inv_queue_tail &= unit->inv_queue_size - 1;
160 }
161
162 static void
163 dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr,
164 bool memw, bool fence)
165 {
166
167 DMAR_ASSERT_LOCKED(unit);
168 dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID |
169 (intr ? DMAR_IQ_DESCR_WAIT_IF : 0) |
170 (memw ? DMAR_IQ_DESCR_WAIT_SW : 0) |
171 (fence ? DMAR_IQ_DESCR_WAIT_FN : 0) |
172 (memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0),
173 memw ? unit->inv_waitd_seq_hw_phys : 0);
174 }
175
176 static void
177 dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct iommu_qi_genseq *pseq,
178 bool emit_wait)
179 {
180 struct iommu_qi_genseq gsec;
181 uint32_t seq;
182
183 KASSERT(pseq != NULL, ("wait descriptor with no place for seq"));
184 DMAR_ASSERT_LOCKED(unit);
185 if (unit->inv_waitd_seq == 0xffffffff) {
186 gsec.gen = unit->inv_waitd_gen;
187 gsec.seq = unit->inv_waitd_seq;
188 dmar_qi_ensure(unit, 1);
189 dmar_qi_emit_wait_descr(unit, gsec.seq, false, true, false);
190 dmar_qi_advance_tail(unit);
191 while (!dmar_qi_seq_processed(unit, &gsec))
192 cpu_spinwait();
193 unit->inv_waitd_gen++;
194 unit->inv_waitd_seq = 1;
195 }
196 seq = unit->inv_waitd_seq++;
197 pseq->gen = unit->inv_waitd_gen;
198 pseq->seq = seq;
199 if (emit_wait) {
200 dmar_qi_ensure(unit, 1);
201 dmar_qi_emit_wait_descr(unit, seq, true, true, false);
202 }
203 }
204
205 static void
206 dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct iommu_qi_genseq *gseq,
207 bool nowait)
208 {
209
210 DMAR_ASSERT_LOCKED(unit);
211 unit->inv_seq_waiters++;
212 while (!dmar_qi_seq_processed(unit, gseq)) {
213 if (cold || nowait) {
214 cpu_spinwait();
215 } else {
216 msleep(&unit->inv_seq_waiters, &unit->iommu.lock, 0,
217 "dmarse", hz);
218 }
219 }
220 unit->inv_seq_waiters--;
221 }
222
223 void
224 dmar_qi_invalidate_locked(struct dmar_domain *domain, iommu_gaddr_t base,
225 iommu_gaddr_t size, struct iommu_qi_genseq *pseq, bool emit_wait)
226 {
227 struct dmar_unit *unit;
228 iommu_gaddr_t isize;
229 int am;
230
231 unit = domain->dmar;
232 DMAR_ASSERT_LOCKED(unit);
233 for (; size > 0; base += isize, size -= isize) {
234 am = calc_am(unit, base, size, &isize);
235 dmar_qi_ensure(unit, 1);
236 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV |
237 DMAR_IQ_DESCR_IOTLB_PAGE | DMAR_IQ_DESCR_IOTLB_DW |
238 DMAR_IQ_DESCR_IOTLB_DR |
239 DMAR_IQ_DESCR_IOTLB_DID(domain->domain),
240 base | am);
241 }
242 dmar_qi_emit_wait_seq(unit, pseq, emit_wait);
243 dmar_qi_advance_tail(unit);
244 }
245
246 void
247 dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit)
248 {
249 struct iommu_qi_genseq gseq;
250
251 DMAR_ASSERT_LOCKED(unit);
252 dmar_qi_ensure(unit, 2);
253 dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0);
254 dmar_qi_emit_wait_seq(unit, &gseq, true);
255 dmar_qi_advance_tail(unit);
256 dmar_qi_wait_for_seq(unit, &gseq, false);
257 }
258
259 void
260 dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit)
261 {
262 struct iommu_qi_genseq gseq;
263
264 DMAR_ASSERT_LOCKED(unit);
265 dmar_qi_ensure(unit, 2);
266 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB |
267 DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0);
268 dmar_qi_emit_wait_seq(unit, &gseq, true);
269 dmar_qi_advance_tail(unit);
270 dmar_qi_wait_for_seq(unit, &gseq, false);
271 }
272
273 void
274 dmar_qi_invalidate_iec_glob(struct dmar_unit *unit)
275 {
276 struct iommu_qi_genseq gseq;
277
278 DMAR_ASSERT_LOCKED(unit);
279 dmar_qi_ensure(unit, 2);
280 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0);
281 dmar_qi_emit_wait_seq(unit, &gseq, true);
282 dmar_qi_advance_tail(unit);
283 dmar_qi_wait_for_seq(unit, &gseq, false);
284 }
285
286 void
287 dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt)
288 {
289 struct iommu_qi_genseq gseq;
290 u_int c, l;
291
292 DMAR_ASSERT_LOCKED(unit);
293 KASSERT(start < unit->irte_cnt && start < start + cnt &&
294 start + cnt <= unit->irte_cnt,
295 ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt));
296 for (; cnt > 0; cnt -= c, start += c) {
297 l = ffs(start | cnt) - 1;
298 c = 1 << l;
299 dmar_qi_ensure(unit, 1);
300 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV |
301 DMAR_IQ_DESCR_IEC_IDX | DMAR_IQ_DESCR_IEC_IIDX(start) |
302 DMAR_IQ_DESCR_IEC_IM(l), 0);
303 }
304 dmar_qi_ensure(unit, 1);
305 dmar_qi_emit_wait_seq(unit, &gseq, true);
306 dmar_qi_advance_tail(unit);
307
308 /*
309 * The caller of the function, in particular,
310 * dmar_ir_program_irte(), may be called from the context
311 * where the sleeping is forbidden (in fact, the
312 * intr_table_lock mutex may be held, locked from
313 * intr_shuffle_irqs()). Wait for the invalidation completion
314 * using the busy wait.
315 *
316 * The impact on the interrupt input setup code is small, the
317 * expected overhead is comparable with the chipset register
318 * read. It is more harmful for the parallel DMA operations,
319 * since we own the dmar unit lock until whole invalidation
320 * queue is processed, which includes requests possibly issued
321 * before our request.
322 */
323 dmar_qi_wait_for_seq(unit, &gseq, true);
324 }
325
326 int
327 dmar_qi_intr(void *arg)
328 {
329 struct dmar_unit *unit;
330
331 unit = arg;
332 KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled",
333 unit->iommu.unit));
334 taskqueue_enqueue(unit->qi_taskqueue, &unit->qi_task);
335 return (FILTER_HANDLED);
336 }
337
338 static void
339 dmar_qi_task(void *arg, int pending __unused)
340 {
341 struct dmar_unit *unit;
342 struct iommu_map_entry *entry;
343 uint32_t ics;
344
345 unit = arg;
346
347 DMAR_LOCK(unit);
348 for (;;) {
349 entry = TAILQ_FIRST(&unit->tlb_flush_entries);
350 if (entry == NULL)
351 break;
352 if (!dmar_qi_seq_processed(unit, &entry->gseq))
353 break;
354 TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link);
355 DMAR_UNLOCK(unit);
356 dmar_domain_free_entry(entry, (entry->flags &
357 IOMMU_MAP_ENTRY_QI_NF) == 0);
358 DMAR_LOCK(unit);
359 }
360 ics = dmar_read4(unit, DMAR_ICS_REG);
361 if ((ics & DMAR_ICS_IWC) != 0) {
362 ics = DMAR_ICS_IWC;
363 dmar_write4(unit, DMAR_ICS_REG, ics);
364 }
365 if (unit->inv_seq_waiters > 0)
366 wakeup(&unit->inv_seq_waiters);
367 DMAR_UNLOCK(unit);
368 }
369
370 int
371 dmar_init_qi(struct dmar_unit *unit)
372 {
373 uint64_t iqa;
374 uint32_t ics;
375 int qi_sz;
376
377 if (!DMAR_HAS_QI(unit) || (unit->hw_cap & DMAR_CAP_CM) != 0)
378 return (0);
379 unit->qi_enabled = 1;
380 TUNABLE_INT_FETCH("hw.dmar.qi", &unit->qi_enabled);
381 if (!unit->qi_enabled)
382 return (0);
383
384 TAILQ_INIT(&unit->tlb_flush_entries);
385 TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit);
386 unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK,
387 taskqueue_thread_enqueue, &unit->qi_taskqueue);
388 taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV,
389 "dmar%d qi taskq", unit->iommu.unit);
390
391 unit->inv_waitd_gen = 0;
392 unit->inv_waitd_seq = 1;
393
394 qi_sz = DMAR_IQA_QS_DEF;
395 TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz);
396 if (qi_sz > DMAR_IQA_QS_MAX)
397 qi_sz = DMAR_IQA_QS_MAX;
398 unit->inv_queue_size = (1ULL << qi_sz) * PAGE_SIZE;
399 /* Reserve one descriptor to prevent wraparound. */
400 unit->inv_queue_avail = unit->inv_queue_size - DMAR_IQ_DESCR_SZ;
401
402 /* The invalidation queue reads by DMARs are always coherent. */
403 unit->inv_queue = kmem_alloc_contig(unit->inv_queue_size, M_WAITOK |
404 M_ZERO, 0, dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
405 unit->inv_waitd_seq_hw_phys = pmap_kextract(
406 (vm_offset_t)&unit->inv_waitd_seq_hw);
407
408 DMAR_LOCK(unit);
409 dmar_write8(unit, DMAR_IQT_REG, 0);
410 iqa = pmap_kextract(unit->inv_queue);
411 iqa |= qi_sz;
412 dmar_write8(unit, DMAR_IQA_REG, iqa);
413 dmar_enable_qi(unit);
414 ics = dmar_read4(unit, DMAR_ICS_REG);
415 if ((ics & DMAR_ICS_IWC) != 0) {
416 ics = DMAR_ICS_IWC;
417 dmar_write4(unit, DMAR_ICS_REG, ics);
418 }
419 dmar_enable_qi_intr(unit);
420 DMAR_UNLOCK(unit);
421
422 return (0);
423 }
424
425 void
426 dmar_fini_qi(struct dmar_unit *unit)
427 {
428 struct iommu_qi_genseq gseq;
429
430 if (!unit->qi_enabled)
431 return;
432 taskqueue_drain(unit->qi_taskqueue, &unit->qi_task);
433 taskqueue_free(unit->qi_taskqueue);
434 unit->qi_taskqueue = NULL;
435
436 DMAR_LOCK(unit);
437 /* quisce */
438 dmar_qi_ensure(unit, 1);
439 dmar_qi_emit_wait_seq(unit, &gseq, true);
440 dmar_qi_advance_tail(unit);
441 dmar_qi_wait_for_seq(unit, &gseq, false);
442 /* only after the quisce, disable queue */
443 dmar_disable_qi_intr(unit);
444 dmar_disable_qi(unit);
445 KASSERT(unit->inv_seq_waiters == 0,
446 ("dmar%d: waiters on disabled queue", unit->iommu.unit));
447 DMAR_UNLOCK(unit);
448
449 kmem_free(unit->inv_queue, unit->inv_queue_size);
450 unit->inv_queue = 0;
451 unit->inv_queue_size = 0;
452 unit->qi_enabled = 0;
453 }
454
455 void
456 dmar_enable_qi_intr(struct dmar_unit *unit)
457 {
458 uint32_t iectl;
459
460 DMAR_ASSERT_LOCKED(unit);
461 KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported",
462 unit->iommu.unit));
463 iectl = dmar_read4(unit, DMAR_IECTL_REG);
464 iectl &= ~DMAR_IECTL_IM;
465 dmar_write4(unit, DMAR_IECTL_REG, iectl);
466 }
467
468 void
469 dmar_disable_qi_intr(struct dmar_unit *unit)
470 {
471 uint32_t iectl;
472
473 DMAR_ASSERT_LOCKED(unit);
474 KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported",
475 unit->iommu.unit));
476 iectl = dmar_read4(unit, DMAR_IECTL_REG);
477 dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM);
478 }
Cache object: 442517981a4a97a91afcbca48fc8f44d
|