Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/x86/iommu/intel_gas.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2013 The FreeBSD Foundation 5 * All rights reserved. 6 * 7 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 8 * under sponsorship from the FreeBSD Foundation. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD: releng/12.0/sys/x86/iommu/intel_gas.c 329521 2018-02-18 16:03:50Z markj $"); 34 35 #define RB_AUGMENT(entry) dmar_gas_augment_entry(entry) 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/malloc.h> 40 #include <sys/bus.h> 41 #include <sys/interrupt.h> 42 #include <sys/kernel.h> 43 #include <sys/ktr.h> 44 #include <sys/lock.h> 45 #include <sys/proc.h> 46 #include <sys/rwlock.h> 47 #include <sys/memdesc.h> 48 #include <sys/mutex.h> 49 #include <sys/sysctl.h> 50 #include <sys/rman.h> 51 #include <sys/taskqueue.h> 52 #include <sys/tree.h> 53 #include <sys/uio.h> 54 #include <sys/vmem.h> 55 #include <dev/pci/pcivar.h> 56 #include <vm/vm.h> 57 #include <vm/vm_extern.h> 58 #include <vm/vm_kern.h> 59 #include <vm/vm_object.h> 60 #include <vm/vm_page.h> 61 #include <vm/vm_map.h> 62 #include <vm/uma.h> 63 #include <machine/atomic.h> 64 #include <machine/bus.h> 65 #include <machine/md_var.h> 66 #include <machine/specialreg.h> 67 #include <x86/include/busdma_impl.h> 68 #include <x86/iommu/intel_reg.h> 69 #include <x86/iommu/busdma_dmar.h> 70 #include <x86/iommu/intel_dmar.h> 71 72 /* 73 * Guest Address Space management. 74 */ 75 76 static uma_zone_t dmar_map_entry_zone; 77 78 static void 79 intel_gas_init(void) 80 { 81 82 dmar_map_entry_zone = uma_zcreate("DMAR_MAP_ENTRY", 83 sizeof(struct dmar_map_entry), NULL, NULL, 84 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NODUMP); 85 } 86 SYSINIT(intel_gas, SI_SUB_DRIVERS, SI_ORDER_FIRST, intel_gas_init, NULL); 87 88 struct dmar_map_entry * 89 dmar_gas_alloc_entry(struct dmar_domain *domain, u_int flags) 90 { 91 struct dmar_map_entry *res; 92 93 KASSERT((flags & ~(DMAR_PGF_WAITOK)) == 0, 94 ("unsupported flags %x", flags)); 95 96 res = uma_zalloc(dmar_map_entry_zone, ((flags & DMAR_PGF_WAITOK) != 97 0 ? M_WAITOK : M_NOWAIT) | M_ZERO); 98 if (res != NULL) { 99 res->domain = domain; 100 atomic_add_int(&domain->entries_cnt, 1); 101 } 102 return (res); 103 } 104 105 void 106 dmar_gas_free_entry(struct dmar_domain *domain, struct dmar_map_entry *entry) 107 { 108 109 KASSERT(domain == entry->domain, 110 ("mismatched free domain %p entry %p entry->domain %p", domain, 111 entry, entry->domain)); 112 atomic_subtract_int(&domain->entries_cnt, 1); 113 uma_zfree(dmar_map_entry_zone, entry); 114 } 115 116 static int 117 dmar_gas_cmp_entries(struct dmar_map_entry *a, struct dmar_map_entry *b) 118 { 119 120 /* Last entry have zero size, so <= */ 121 KASSERT(a->start <= a->end, ("inverted entry %p (%jx, %jx)", 122 a, (uintmax_t)a->start, (uintmax_t)a->end)); 123 KASSERT(b->start <= b->end, ("inverted entry %p (%jx, %jx)", 124 b, (uintmax_t)b->start, (uintmax_t)b->end)); 125 KASSERT(a->end <= b->start || b->end <= a->start || 126 a->end == a->start || b->end == b->start, 127 ("overlapping entries %p (%jx, %jx) %p (%jx, %jx)", 128 a, (uintmax_t)a->start, (uintmax_t)a->end, 129 b, (uintmax_t)b->start, (uintmax_t)b->end)); 130 131 if (a->end < b->end) 132 return (-1); 133 else if (b->end < a->end) 134 return (1); 135 return (0); 136 } 137 138 static void 139 dmar_gas_augment_entry(struct dmar_map_entry *entry) 140 { 141 struct dmar_map_entry *l, *r; 142 143 for (; entry != NULL; entry = RB_PARENT(entry, rb_entry)) { 144 l = RB_LEFT(entry, rb_entry); 145 r = RB_RIGHT(entry, rb_entry); 146 if (l == NULL && r == NULL) { 147 entry->free_down = entry->free_after; 148 } else if (l == NULL && r != NULL) { 149 entry->free_down = MAX(entry->free_after, r->free_down); 150 } else if (/*l != NULL && */ r == NULL) { 151 entry->free_down = MAX(entry->free_after, l->free_down); 152 } else /* if (l != NULL && r != NULL) */ { 153 entry->free_down = MAX(entry->free_after, l->free_down); 154 entry->free_down = MAX(entry->free_down, r->free_down); 155 } 156 } 157 } 158 159 RB_GENERATE(dmar_gas_entries_tree, dmar_map_entry, rb_entry, 160 dmar_gas_cmp_entries); 161 162 static void 163 dmar_gas_fix_free(struct dmar_domain *domain, struct dmar_map_entry *entry) 164 { 165 struct dmar_map_entry *next; 166 167 next = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); 168 entry->free_after = (next != NULL ? next->start : domain->end) - 169 entry->end; 170 dmar_gas_augment_entry(entry); 171 } 172 173 #ifdef INVARIANTS 174 static void 175 dmar_gas_check_free(struct dmar_domain *domain) 176 { 177 struct dmar_map_entry *entry, *next, *l, *r; 178 dmar_gaddr_t v; 179 180 RB_FOREACH(entry, dmar_gas_entries_tree, &domain->rb_root) { 181 KASSERT(domain == entry->domain, 182 ("mismatched free domain %p entry %p entry->domain %p", 183 domain, entry, entry->domain)); 184 next = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); 185 if (next == NULL) { 186 MPASS(entry->free_after == domain->end - entry->end); 187 } else { 188 MPASS(entry->free_after = next->start - entry->end); 189 MPASS(entry->end <= next->start); 190 } 191 l = RB_LEFT(entry, rb_entry); 192 r = RB_RIGHT(entry, rb_entry); 193 if (l == NULL && r == NULL) { 194 MPASS(entry->free_down == entry->free_after); 195 } else if (l == NULL && r != NULL) { 196 MPASS(entry->free_down = MAX(entry->free_after, 197 r->free_down)); 198 } else if (r == NULL) { 199 MPASS(entry->free_down = MAX(entry->free_after, 200 l->free_down)); 201 } else { 202 v = MAX(entry->free_after, l->free_down); 203 v = MAX(v, r->free_down); 204 MPASS(entry->free_down == v); 205 } 206 } 207 } 208 #endif 209 210 static bool 211 dmar_gas_rb_insert(struct dmar_domain *domain, struct dmar_map_entry *entry) 212 { 213 struct dmar_map_entry *prev, *found; 214 215 found = RB_INSERT(dmar_gas_entries_tree, &domain->rb_root, entry); 216 dmar_gas_fix_free(domain, entry); 217 prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); 218 if (prev != NULL) 219 dmar_gas_fix_free(domain, prev); 220 return (found == NULL); 221 } 222 223 static void 224 dmar_gas_rb_remove(struct dmar_domain *domain, struct dmar_map_entry *entry) 225 { 226 struct dmar_map_entry *prev; 227 228 prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); 229 RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); 230 if (prev != NULL) 231 dmar_gas_fix_free(domain, prev); 232 } 233 234 void 235 dmar_gas_init_domain(struct dmar_domain *domain) 236 { 237 struct dmar_map_entry *begin, *end; 238 239 begin = dmar_gas_alloc_entry(domain, DMAR_PGF_WAITOK); 240 end = dmar_gas_alloc_entry(domain, DMAR_PGF_WAITOK); 241 242 DMAR_DOMAIN_LOCK(domain); 243 KASSERT(domain->entries_cnt == 2, ("dirty domain %p", domain)); 244 KASSERT(RB_EMPTY(&domain->rb_root), ("non-empty entries %p", domain)); 245 246 begin->start = 0; 247 begin->end = DMAR_PAGE_SIZE; 248 begin->free_after = domain->end - begin->end; 249 begin->flags = DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_UNMAPPED; 250 dmar_gas_rb_insert(domain, begin); 251 252 end->start = domain->end; 253 end->end = domain->end; 254 end->free_after = 0; 255 end->flags = DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_UNMAPPED; 256 dmar_gas_rb_insert(domain, end); 257 258 domain->first_place = begin; 259 domain->last_place = end; 260 domain->flags |= DMAR_DOMAIN_GAS_INITED; 261 DMAR_DOMAIN_UNLOCK(domain); 262 } 263 264 void 265 dmar_gas_fini_domain(struct dmar_domain *domain) 266 { 267 struct dmar_map_entry *entry, *entry1; 268 269 DMAR_DOMAIN_ASSERT_LOCKED(domain); 270 KASSERT(domain->entries_cnt == 2, ("domain still in use %p", domain)); 271 272 entry = RB_MIN(dmar_gas_entries_tree, &domain->rb_root); 273 KASSERT(entry->start == 0, ("start entry start %p", domain)); 274 KASSERT(entry->end == DMAR_PAGE_SIZE, ("start entry end %p", domain)); 275 KASSERT(entry->flags == DMAR_MAP_ENTRY_PLACE, 276 ("start entry flags %p", domain)); 277 RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); 278 dmar_gas_free_entry(domain, entry); 279 280 entry = RB_MAX(dmar_gas_entries_tree, &domain->rb_root); 281 KASSERT(entry->start == domain->end, ("end entry start %p", domain)); 282 KASSERT(entry->end == domain->end, ("end entry end %p", domain)); 283 KASSERT(entry->free_after == 0, ("end entry free_after %p", domain)); 284 KASSERT(entry->flags == DMAR_MAP_ENTRY_PLACE, 285 ("end entry flags %p", domain)); 286 RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); 287 dmar_gas_free_entry(domain, entry); 288 289 RB_FOREACH_SAFE(entry, dmar_gas_entries_tree, &domain->rb_root, 290 entry1) { 291 KASSERT((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0, 292 ("non-RMRR entry left %p", domain)); 293 RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); 294 dmar_gas_free_entry(domain, entry); 295 } 296 } 297 298 struct dmar_gas_match_args { 299 struct dmar_domain *domain; 300 dmar_gaddr_t size; 301 int offset; 302 const struct bus_dma_tag_common *common; 303 u_int gas_flags; 304 struct dmar_map_entry *entry; 305 }; 306 307 static bool 308 dmar_gas_match_one(struct dmar_gas_match_args *a, struct dmar_map_entry *prev, 309 dmar_gaddr_t end) 310 { 311 dmar_gaddr_t bs, start; 312 313 if (a->entry->start + a->size > end) 314 return (false); 315 316 /* DMAR_PAGE_SIZE to create gap after new entry. */ 317 if (a->entry->start < prev->end + DMAR_PAGE_SIZE || 318 a->entry->start + a->size + a->offset + DMAR_PAGE_SIZE > 319 prev->end + prev->free_after) 320 return (false); 321 322 /* No boundary crossing. */ 323 if (dmar_test_boundary(a->entry->start + a->offset, a->size, 324 a->common->boundary)) 325 return (true); 326 327 /* 328 * The start + offset to start + offset + size region crosses 329 * the boundary. Check if there is enough space after the 330 * next boundary after the prev->end. 331 */ 332 bs = rounddown2(a->entry->start + a->offset + a->common->boundary, 333 a->common->boundary); 334 start = roundup2(bs, a->common->alignment); 335 /* DMAR_PAGE_SIZE to create gap after new entry. */ 336 if (start + a->offset + a->size + DMAR_PAGE_SIZE <= 337 prev->end + prev->free_after && 338 start + a->offset + a->size <= end && 339 dmar_test_boundary(start + a->offset, a->size, 340 a->common->boundary)) { 341 a->entry->start = start; 342 return (true); 343 } 344 345 /* 346 * Not enough space to align at the requested boundary, or 347 * boundary is smaller than the size, but allowed to split. 348 * We already checked that start + size does not overlap end. 349 * 350 * XXXKIB. It is possible that bs is exactly at the start of 351 * the next entry, then we do not have gap. Ignore for now. 352 */ 353 if ((a->gas_flags & DMAR_GM_CANSPLIT) != 0) { 354 a->size = bs - a->entry->start; 355 return (true); 356 } 357 358 return (false); 359 } 360 361 static void 362 dmar_gas_match_insert(struct dmar_gas_match_args *a, 363 struct dmar_map_entry *prev) 364 { 365 struct dmar_map_entry *next; 366 bool found; 367 368 /* 369 * The prev->end is always aligned on the page size, which 370 * causes page alignment for the entry->start too. The size 371 * is checked to be multiple of the page size. 372 * 373 * The page sized gap is created between consequent 374 * allocations to ensure that out-of-bounds accesses fault. 375 */ 376 a->entry->end = a->entry->start + a->size; 377 378 next = RB_NEXT(dmar_gas_entries_tree, &a->domain->rb_root, prev); 379 KASSERT(next->start >= a->entry->end && 380 next->start - a->entry->start >= a->size && 381 prev->end <= a->entry->end, 382 ("dmar_gas_match_insert hole failed %p prev (%jx, %jx) " 383 "free_after %jx next (%jx, %jx) entry (%jx, %jx)", a->domain, 384 (uintmax_t)prev->start, (uintmax_t)prev->end, 385 (uintmax_t)prev->free_after, 386 (uintmax_t)next->start, (uintmax_t)next->end, 387 (uintmax_t)a->entry->start, (uintmax_t)a->entry->end)); 388 389 prev->free_after = a->entry->start - prev->end; 390 a->entry->free_after = next->start - a->entry->end; 391 392 found = dmar_gas_rb_insert(a->domain, a->entry); 393 KASSERT(found, ("found dup %p start %jx size %jx", 394 a->domain, (uintmax_t)a->entry->start, (uintmax_t)a->size)); 395 a->entry->flags = DMAR_MAP_ENTRY_MAP; 396 397 KASSERT(RB_PREV(dmar_gas_entries_tree, &a->domain->rb_root, 398 a->entry) == prev, 399 ("entry %p prev %p inserted prev %p", a->entry, prev, 400 RB_PREV(dmar_gas_entries_tree, &a->domain->rb_root, a->entry))); 401 KASSERT(RB_NEXT(dmar_gas_entries_tree, &a->domain->rb_root, 402 a->entry) == next, 403 ("entry %p next %p inserted next %p", a->entry, next, 404 RB_NEXT(dmar_gas_entries_tree, &a->domain->rb_root, a->entry))); 405 } 406 407 static int 408 dmar_gas_lowermatch(struct dmar_gas_match_args *a, struct dmar_map_entry *prev) 409 { 410 struct dmar_map_entry *l; 411 int ret; 412 413 if (prev->end < a->common->lowaddr) { 414 a->entry->start = roundup2(prev->end + DMAR_PAGE_SIZE, 415 a->common->alignment); 416 if (dmar_gas_match_one(a, prev, a->common->lowaddr)) { 417 dmar_gas_match_insert(a, prev); 418 return (0); 419 } 420 } 421 if (prev->free_down < a->size + a->offset + DMAR_PAGE_SIZE) 422 return (ENOMEM); 423 l = RB_LEFT(prev, rb_entry); 424 if (l != NULL) { 425 ret = dmar_gas_lowermatch(a, l); 426 if (ret == 0) 427 return (0); 428 } 429 l = RB_RIGHT(prev, rb_entry); 430 if (l != NULL) 431 return (dmar_gas_lowermatch(a, l)); 432 return (ENOMEM); 433 } 434 435 static int 436 dmar_gas_uppermatch(struct dmar_gas_match_args *a) 437 { 438 struct dmar_map_entry *next, *prev, find_entry; 439 440 find_entry.start = a->common->highaddr; 441 next = RB_NFIND(dmar_gas_entries_tree, &a->domain->rb_root, 442 &find_entry); 443 if (next == NULL) 444 return (ENOMEM); 445 prev = RB_PREV(dmar_gas_entries_tree, &a->domain->rb_root, next); 446 KASSERT(prev != NULL, ("no prev %p %jx", a->domain, 447 (uintmax_t)find_entry.start)); 448 for (;;) { 449 a->entry->start = prev->start + DMAR_PAGE_SIZE; 450 if (a->entry->start < a->common->highaddr) 451 a->entry->start = a->common->highaddr; 452 a->entry->start = roundup2(a->entry->start, 453 a->common->alignment); 454 if (dmar_gas_match_one(a, prev, a->domain->end)) { 455 dmar_gas_match_insert(a, prev); 456 return (0); 457 } 458 459 /* 460 * XXXKIB. This falls back to linear iteration over 461 * the free space in the high region. But high 462 * regions are almost unused, the code should be 463 * enough to cover the case, although in the 464 * non-optimal way. 465 */ 466 prev = next; 467 next = RB_NEXT(dmar_gas_entries_tree, &a->domain->rb_root, 468 prev); 469 KASSERT(next != NULL, ("no next %p %jx", a->domain, 470 (uintmax_t)find_entry.start)); 471 if (next->end >= a->domain->end) 472 return (ENOMEM); 473 } 474 } 475 476 static int 477 dmar_gas_find_space(struct dmar_domain *domain, 478 const struct bus_dma_tag_common *common, dmar_gaddr_t size, 479 int offset, u_int flags, struct dmar_map_entry *entry) 480 { 481 struct dmar_gas_match_args a; 482 int error; 483 484 DMAR_DOMAIN_ASSERT_LOCKED(domain); 485 KASSERT(entry->flags == 0, ("dirty entry %p %p", domain, entry)); 486 KASSERT((size & DMAR_PAGE_MASK) == 0, ("size %jx", (uintmax_t)size)); 487 488 a.domain = domain; 489 a.size = size; 490 a.offset = offset; 491 a.common = common; 492 a.gas_flags = flags; 493 a.entry = entry; 494 495 /* Handle lower region. */ 496 if (common->lowaddr > 0) { 497 error = dmar_gas_lowermatch(&a, RB_ROOT(&domain->rb_root)); 498 if (error == 0) 499 return (0); 500 KASSERT(error == ENOMEM, 501 ("error %d from dmar_gas_lowermatch", error)); 502 } 503 /* Handle upper region. */ 504 if (common->highaddr >= domain->end) 505 return (ENOMEM); 506 error = dmar_gas_uppermatch(&a); 507 KASSERT(error == ENOMEM, 508 ("error %d from dmar_gas_uppermatch", error)); 509 return (error); 510 } 511 512 static int 513 dmar_gas_alloc_region(struct dmar_domain *domain, struct dmar_map_entry *entry, 514 u_int flags) 515 { 516 struct dmar_map_entry *next, *prev; 517 bool found; 518 519 DMAR_DOMAIN_ASSERT_LOCKED(domain); 520 521 if ((entry->start & DMAR_PAGE_MASK) != 0 || 522 (entry->end & DMAR_PAGE_MASK) != 0) 523 return (EINVAL); 524 if (entry->start >= entry->end) 525 return (EINVAL); 526 if (entry->end >= domain->end) 527 return (EINVAL); 528 529 next = RB_NFIND(dmar_gas_entries_tree, &domain->rb_root, entry); 530 KASSERT(next != NULL, ("next must be non-null %p %jx", domain, 531 (uintmax_t)entry->start)); 532 prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, next); 533 /* prev could be NULL */ 534 535 /* 536 * Adapt to broken BIOSes which specify overlapping RMRR 537 * entries. 538 * 539 * XXXKIB: this does not handle a case when prev or next 540 * entries are completely covered by the current one, which 541 * extends both ways. 542 */ 543 if (prev != NULL && prev->end > entry->start && 544 (prev->flags & DMAR_MAP_ENTRY_PLACE) == 0) { 545 if ((prev->flags & DMAR_MAP_ENTRY_RMRR) == 0) 546 return (EBUSY); 547 entry->start = prev->end; 548 } 549 if (next != NULL && next->start < entry->end && 550 (next->flags & DMAR_MAP_ENTRY_PLACE) == 0) { 551 if ((next->flags & DMAR_MAP_ENTRY_RMRR) == 0) 552 return (EBUSY); 553 entry->end = next->start; 554 } 555 if (entry->end == entry->start) 556 return (0); 557 558 if (prev != NULL && prev->end > entry->start) { 559 /* This assumes that prev is the placeholder entry. */ 560 dmar_gas_rb_remove(domain, prev); 561 prev = NULL; 562 } 563 if (next != NULL && next->start < entry->end) { 564 dmar_gas_rb_remove(domain, next); 565 next = NULL; 566 } 567 568 found = dmar_gas_rb_insert(domain, entry); 569 KASSERT(found, ("found RMRR dup %p start %jx end %jx", 570 domain, (uintmax_t)entry->start, (uintmax_t)entry->end)); 571 entry->flags = DMAR_MAP_ENTRY_RMRR; 572 573 #ifdef INVARIANTS 574 struct dmar_map_entry *ip, *in; 575 ip = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); 576 in = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); 577 KASSERT(prev == NULL || ip == prev, 578 ("RMRR %p (%jx %jx) prev %p (%jx %jx) ins prev %p (%jx %jx)", 579 entry, entry->start, entry->end, prev, 580 prev == NULL ? 0 : prev->start, prev == NULL ? 0 : prev->end, 581 ip, ip == NULL ? 0 : ip->start, ip == NULL ? 0 : ip->end)); 582 KASSERT(next == NULL || in == next, 583 ("RMRR %p (%jx %jx) next %p (%jx %jx) ins next %p (%jx %jx)", 584 entry, entry->start, entry->end, next, 585 next == NULL ? 0 : next->start, next == NULL ? 0 : next->end, 586 in, in == NULL ? 0 : in->start, in == NULL ? 0 : in->end)); 587 #endif 588 589 return (0); 590 } 591 592 void 593 dmar_gas_free_space(struct dmar_domain *domain, struct dmar_map_entry *entry) 594 { 595 596 DMAR_DOMAIN_ASSERT_LOCKED(domain); 597 KASSERT((entry->flags & (DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_RMRR | 598 DMAR_MAP_ENTRY_MAP)) == DMAR_MAP_ENTRY_MAP, 599 ("permanent entry %p %p", domain, entry)); 600 601 dmar_gas_rb_remove(domain, entry); 602 entry->flags &= ~DMAR_MAP_ENTRY_MAP; 603 #ifdef INVARIANTS 604 if (dmar_check_free) 605 dmar_gas_check_free(domain); 606 #endif 607 } 608 609 void 610 dmar_gas_free_region(struct dmar_domain *domain, struct dmar_map_entry *entry) 611 { 612 struct dmar_map_entry *next, *prev; 613 614 DMAR_DOMAIN_ASSERT_LOCKED(domain); 615 KASSERT((entry->flags & (DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_RMRR | 616 DMAR_MAP_ENTRY_MAP)) == DMAR_MAP_ENTRY_RMRR, 617 ("non-RMRR entry %p %p", domain, entry)); 618 619 prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); 620 next = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); 621 dmar_gas_rb_remove(domain, entry); 622 entry->flags &= ~DMAR_MAP_ENTRY_RMRR; 623 624 if (prev == NULL) 625 dmar_gas_rb_insert(domain, domain->first_place); 626 if (next == NULL) 627 dmar_gas_rb_insert(domain, domain->last_place); 628 } 629 630 int 631 dmar_gas_map(struct dmar_domain *domain, 632 const struct bus_dma_tag_common *common, dmar_gaddr_t size, int offset, 633 u_int eflags, u_int flags, vm_page_t *ma, struct dmar_map_entry **res) 634 { 635 struct dmar_map_entry *entry; 636 int error; 637 638 KASSERT((flags & ~(DMAR_GM_CANWAIT | DMAR_GM_CANSPLIT)) == 0, 639 ("invalid flags 0x%x", flags)); 640 641 entry = dmar_gas_alloc_entry(domain, (flags & DMAR_GM_CANWAIT) != 0 ? 642 DMAR_PGF_WAITOK : 0); 643 if (entry == NULL) 644 return (ENOMEM); 645 DMAR_DOMAIN_LOCK(domain); 646 error = dmar_gas_find_space(domain, common, size, offset, flags, 647 entry); 648 if (error == ENOMEM) { 649 DMAR_DOMAIN_UNLOCK(domain); 650 dmar_gas_free_entry(domain, entry); 651 return (error); 652 } 653 #ifdef INVARIANTS 654 if (dmar_check_free) 655 dmar_gas_check_free(domain); 656 #endif 657 KASSERT(error == 0, 658 ("unexpected error %d from dmar_gas_find_entry", error)); 659 KASSERT(entry->end < domain->end, ("allocated GPA %jx, max GPA %jx", 660 (uintmax_t)entry->end, (uintmax_t)domain->end)); 661 entry->flags |= eflags; 662 DMAR_DOMAIN_UNLOCK(domain); 663 664 error = domain_map_buf(domain, entry->start, entry->end - entry->start, 665 ma, 666 ((eflags & DMAR_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) | 667 ((eflags & DMAR_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) | 668 ((eflags & DMAR_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) | 669 ((eflags & DMAR_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0), 670 (flags & DMAR_GM_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0); 671 if (error == ENOMEM) { 672 dmar_domain_unload_entry(entry, true); 673 return (error); 674 } 675 KASSERT(error == 0, 676 ("unexpected error %d from domain_map_buf", error)); 677 678 *res = entry; 679 return (0); 680 } 681 682 int 683 dmar_gas_map_region(struct dmar_domain *domain, struct dmar_map_entry *entry, 684 u_int eflags, u_int flags, vm_page_t *ma) 685 { 686 dmar_gaddr_t start; 687 int error; 688 689 KASSERT(entry->flags == 0, ("used RMRR entry %p %p %x", domain, 690 entry, entry->flags)); 691 KASSERT((flags & ~(DMAR_GM_CANWAIT)) == 0, 692 ("invalid flags 0x%x", flags)); 693 694 start = entry->start; 695 DMAR_DOMAIN_LOCK(domain); 696 error = dmar_gas_alloc_region(domain, entry, flags); 697 if (error != 0) { 698 DMAR_DOMAIN_UNLOCK(domain); 699 return (error); 700 } 701 entry->flags |= eflags; 702 DMAR_DOMAIN_UNLOCK(domain); 703 if (entry->end == entry->start) 704 return (0); 705 706 error = domain_map_buf(domain, entry->start, entry->end - entry->start, 707 ma + OFF_TO_IDX(start - entry->start), 708 ((eflags & DMAR_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) | 709 ((eflags & DMAR_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) | 710 ((eflags & DMAR_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) | 711 ((eflags & DMAR_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0), 712 (flags & DMAR_GM_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0); 713 if (error == ENOMEM) { 714 dmar_domain_unload_entry(entry, false); 715 return (error); 716 } 717 KASSERT(error == 0, 718 ("unexpected error %d from domain_map_buf", error)); 719 720 return (0); 721 } 722 723 int 724 dmar_gas_reserve_region(struct dmar_domain *domain, dmar_gaddr_t start, 725 dmar_gaddr_t end) 726 { 727 struct dmar_map_entry *entry; 728 int error; 729 730 entry = dmar_gas_alloc_entry(domain, DMAR_PGF_WAITOK); 731 entry->start = start; 732 entry->end = end; 733 DMAR_DOMAIN_LOCK(domain); 734 error = dmar_gas_alloc_region(domain, entry, DMAR_GM_CANWAIT); 735 if (error == 0) 736 entry->flags |= DMAR_MAP_ENTRY_UNMAPPED; 737 DMAR_DOMAIN_UNLOCK(domain); 738 if (error != 0) 739 dmar_gas_free_entry(domain, entry); 740 return (error); 741 }
Cache object: c547472e5b62bb302e11450872e22ca3
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]