Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/x86/iommu/intel_gas.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 2013 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 6 * under sponsorship from the FreeBSD Foundation. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #define RB_AUGMENT(entry) dmar_gas_augment_entry(entry) 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/malloc.h> 38 #include <sys/bus.h> 39 #include <sys/interrupt.h> 40 #include <sys/kernel.h> 41 #include <sys/ktr.h> 42 #include <sys/lock.h> 43 #include <sys/proc.h> 44 #include <sys/rwlock.h> 45 #include <sys/memdesc.h> 46 #include <sys/mutex.h> 47 #include <sys/sysctl.h> 48 #include <sys/rman.h> 49 #include <sys/taskqueue.h> 50 #include <sys/tree.h> 51 #include <sys/uio.h> 52 #include <sys/vmem.h> 53 #include <dev/pci/pcivar.h> 54 #include <vm/vm.h> 55 #include <vm/vm_extern.h> 56 #include <vm/vm_kern.h> 57 #include <vm/vm_object.h> 58 #include <vm/vm_page.h> 59 #include <vm/vm_map.h> 60 #include <vm/uma.h> 61 #include <machine/atomic.h> 62 #include <machine/bus.h> 63 #include <machine/md_var.h> 64 #include <machine/specialreg.h> 65 #include <x86/include/busdma_impl.h> 66 #include <x86/iommu/intel_reg.h> 67 #include <x86/iommu/busdma_dmar.h> 68 #include <x86/iommu/intel_dmar.h> 69 70 /* 71 * Guest Address Space management. 72 */ 73 74 static uma_zone_t dmar_map_entry_zone; 75 76 static void 77 intel_gas_init(void) 78 { 79 80 dmar_map_entry_zone = uma_zcreate("DMAR_MAP_ENTRY", 81 sizeof(struct dmar_map_entry), NULL, NULL, 82 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NODUMP); 83 } 84 SYSINIT(intel_gas, SI_SUB_DRIVERS, SI_ORDER_FIRST, intel_gas_init, NULL); 85 86 struct dmar_map_entry * 87 dmar_gas_alloc_entry(struct dmar_domain *domain, u_int flags) 88 { 89 struct dmar_map_entry *res; 90 91 KASSERT((flags & ~(DMAR_PGF_WAITOK)) == 0, 92 ("unsupported flags %x", flags)); 93 94 res = uma_zalloc(dmar_map_entry_zone, ((flags & DMAR_PGF_WAITOK) != 95 0 ? M_WAITOK : M_NOWAIT) | M_ZERO); 96 if (res != NULL) { 97 res->domain = domain; 98 atomic_add_int(&domain->entries_cnt, 1); 99 } 100 return (res); 101 } 102 103 void 104 dmar_gas_free_entry(struct dmar_domain *domain, struct dmar_map_entry *entry) 105 { 106 107 KASSERT(domain == entry->domain, 108 ("mismatched free domain %p entry %p entry->domain %p", domain, 109 entry, entry->domain)); 110 atomic_subtract_int(&domain->entries_cnt, 1); 111 uma_zfree(dmar_map_entry_zone, entry); 112 } 113 114 static int 115 dmar_gas_cmp_entries(struct dmar_map_entry *a, struct dmar_map_entry *b) 116 { 117 118 /* Last entry have zero size, so <= */ 119 KASSERT(a->start <= a->end, ("inverted entry %p (%jx, %jx)", 120 a, (uintmax_t)a->start, (uintmax_t)a->end)); 121 KASSERT(b->start <= b->end, ("inverted entry %p (%jx, %jx)", 122 b, (uintmax_t)b->start, (uintmax_t)b->end)); 123 KASSERT(a->end <= b->start || b->end <= a->start || 124 a->end == a->start || b->end == b->start, 125 ("overlapping entries %p (%jx, %jx) %p (%jx, %jx)", 126 a, (uintmax_t)a->start, (uintmax_t)a->end, 127 b, (uintmax_t)b->start, (uintmax_t)b->end)); 128 129 if (a->end < b->end) 130 return (-1); 131 else if (b->end < a->end) 132 return (1); 133 return (0); 134 } 135 136 static void 137 dmar_gas_augment_entry(struct dmar_map_entry *entry) 138 { 139 struct dmar_map_entry *l, *r; 140 141 for (; entry != NULL; entry = RB_PARENT(entry, rb_entry)) { 142 l = RB_LEFT(entry, rb_entry); 143 r = RB_RIGHT(entry, rb_entry); 144 if (l == NULL && r == NULL) { 145 entry->free_down = entry->free_after; 146 } else if (l == NULL && r != NULL) { 147 entry->free_down = MAX(entry->free_after, r->free_down); 148 } else if (/*l != NULL && */ r == NULL) { 149 entry->free_down = MAX(entry->free_after, l->free_down); 150 } else /* if (l != NULL && r != NULL) */ { 151 entry->free_down = MAX(entry->free_after, l->free_down); 152 entry->free_down = MAX(entry->free_down, r->free_down); 153 } 154 } 155 } 156 157 RB_GENERATE(dmar_gas_entries_tree, dmar_map_entry, rb_entry, 158 dmar_gas_cmp_entries); 159 160 static void 161 dmar_gas_fix_free(struct dmar_domain *domain, struct dmar_map_entry *entry) 162 { 163 struct dmar_map_entry *next; 164 165 next = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); 166 entry->free_after = (next != NULL ? next->start : domain->end) - 167 entry->end; 168 dmar_gas_augment_entry(entry); 169 } 170 171 #ifdef INVARIANTS 172 static void 173 dmar_gas_check_free(struct dmar_domain *domain) 174 { 175 struct dmar_map_entry *entry, *next, *l, *r; 176 dmar_gaddr_t v; 177 178 RB_FOREACH(entry, dmar_gas_entries_tree, &domain->rb_root) { 179 KASSERT(domain == entry->domain, 180 ("mismatched free domain %p entry %p entry->domain %p", 181 domain, entry, entry->domain)); 182 next = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); 183 if (next == NULL) { 184 MPASS(entry->free_after == domain->end - entry->end); 185 } else { 186 MPASS(entry->free_after = next->start - entry->end); 187 MPASS(entry->end <= next->start); 188 } 189 l = RB_LEFT(entry, rb_entry); 190 r = RB_RIGHT(entry, rb_entry); 191 if (l == NULL && r == NULL) { 192 MPASS(entry->free_down == entry->free_after); 193 } else if (l == NULL && r != NULL) { 194 MPASS(entry->free_down = MAX(entry->free_after, 195 r->free_down)); 196 } else if (r == NULL) { 197 MPASS(entry->free_down = MAX(entry->free_after, 198 l->free_down)); 199 } else { 200 v = MAX(entry->free_after, l->free_down); 201 v = MAX(v, r->free_down); 202 MPASS(entry->free_down == v); 203 } 204 } 205 } 206 #endif 207 208 static bool 209 dmar_gas_rb_insert(struct dmar_domain *domain, struct dmar_map_entry *entry) 210 { 211 struct dmar_map_entry *prev, *found; 212 213 found = RB_INSERT(dmar_gas_entries_tree, &domain->rb_root, entry); 214 dmar_gas_fix_free(domain, entry); 215 prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); 216 if (prev != NULL) 217 dmar_gas_fix_free(domain, prev); 218 return (found == NULL); 219 } 220 221 static void 222 dmar_gas_rb_remove(struct dmar_domain *domain, struct dmar_map_entry *entry) 223 { 224 struct dmar_map_entry *prev; 225 226 prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); 227 RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); 228 if (prev != NULL) 229 dmar_gas_fix_free(domain, prev); 230 } 231 232 void 233 dmar_gas_init_domain(struct dmar_domain *domain) 234 { 235 struct dmar_map_entry *begin, *end; 236 237 begin = dmar_gas_alloc_entry(domain, DMAR_PGF_WAITOK); 238 end = dmar_gas_alloc_entry(domain, DMAR_PGF_WAITOK); 239 240 DMAR_DOMAIN_LOCK(domain); 241 KASSERT(domain->entries_cnt == 2, ("dirty domain %p", domain)); 242 KASSERT(RB_EMPTY(&domain->rb_root), ("non-empty entries %p", domain)); 243 244 begin->start = 0; 245 begin->end = DMAR_PAGE_SIZE; 246 begin->free_after = domain->end - begin->end; 247 begin->flags = DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_UNMAPPED; 248 dmar_gas_rb_insert(domain, begin); 249 250 end->start = domain->end; 251 end->end = domain->end; 252 end->free_after = 0; 253 end->flags = DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_UNMAPPED; 254 dmar_gas_rb_insert(domain, end); 255 256 domain->first_place = begin; 257 domain->last_place = end; 258 domain->flags |= DMAR_DOMAIN_GAS_INITED; 259 DMAR_DOMAIN_UNLOCK(domain); 260 } 261 262 void 263 dmar_gas_fini_domain(struct dmar_domain *domain) 264 { 265 struct dmar_map_entry *entry, *entry1; 266 267 DMAR_DOMAIN_ASSERT_LOCKED(domain); 268 KASSERT(domain->entries_cnt == 2, ("domain still in use %p", domain)); 269 270 entry = RB_MIN(dmar_gas_entries_tree, &domain->rb_root); 271 KASSERT(entry->start == 0, ("start entry start %p", domain)); 272 KASSERT(entry->end == DMAR_PAGE_SIZE, ("start entry end %p", domain)); 273 KASSERT(entry->flags == DMAR_MAP_ENTRY_PLACE, 274 ("start entry flags %p", domain)); 275 RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); 276 dmar_gas_free_entry(domain, entry); 277 278 entry = RB_MAX(dmar_gas_entries_tree, &domain->rb_root); 279 KASSERT(entry->start == domain->end, ("end entry start %p", domain)); 280 KASSERT(entry->end == domain->end, ("end entry end %p", domain)); 281 KASSERT(entry->free_after == 0, ("end entry free_after %p", domain)); 282 KASSERT(entry->flags == DMAR_MAP_ENTRY_PLACE, 283 ("end entry flags %p", domain)); 284 RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); 285 dmar_gas_free_entry(domain, entry); 286 287 RB_FOREACH_SAFE(entry, dmar_gas_entries_tree, &domain->rb_root, 288 entry1) { 289 KASSERT((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0, 290 ("non-RMRR entry left %p", domain)); 291 RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); 292 dmar_gas_free_entry(domain, entry); 293 } 294 } 295 296 struct dmar_gas_match_args { 297 struct dmar_domain *domain; 298 dmar_gaddr_t size; 299 int offset; 300 const struct bus_dma_tag_common *common; 301 u_int gas_flags; 302 struct dmar_map_entry *entry; 303 }; 304 305 static bool 306 dmar_gas_match_one(struct dmar_gas_match_args *a, struct dmar_map_entry *prev, 307 dmar_gaddr_t end) 308 { 309 dmar_gaddr_t bs, start; 310 311 if (a->entry->start + a->size > end) 312 return (false); 313 314 /* DMAR_PAGE_SIZE to create gap after new entry. */ 315 if (a->entry->start < prev->end + DMAR_PAGE_SIZE || 316 a->entry->start + a->size + a->offset + DMAR_PAGE_SIZE > 317 prev->end + prev->free_after) 318 return (false); 319 320 /* No boundary crossing. */ 321 if (dmar_test_boundary(a->entry->start + a->offset, a->size, 322 a->common->boundary)) 323 return (true); 324 325 /* 326 * The start + offset to start + offset + size region crosses 327 * the boundary. Check if there is enough space after the 328 * next boundary after the prev->end. 329 */ 330 bs = rounddown2(a->entry->start + a->offset + a->common->boundary, 331 a->common->boundary); 332 start = roundup2(bs, a->common->alignment); 333 /* DMAR_PAGE_SIZE to create gap after new entry. */ 334 if (start + a->offset + a->size + DMAR_PAGE_SIZE <= 335 prev->end + prev->free_after && 336 start + a->offset + a->size <= end && 337 dmar_test_boundary(start + a->offset, a->size, 338 a->common->boundary)) { 339 a->entry->start = start; 340 return (true); 341 } 342 343 /* 344 * Not enough space to align at the requested boundary, or 345 * boundary is smaller than the size, but allowed to split. 346 * We already checked that start + size does not overlap end. 347 * 348 * XXXKIB. It is possible that bs is exactly at the start of 349 * the next entry, then we do not have gap. Ignore for now. 350 */ 351 if ((a->gas_flags & DMAR_GM_CANSPLIT) != 0) { 352 a->size = bs - a->entry->start; 353 return (true); 354 } 355 356 return (false); 357 } 358 359 static void 360 dmar_gas_match_insert(struct dmar_gas_match_args *a, 361 struct dmar_map_entry *prev) 362 { 363 struct dmar_map_entry *next; 364 bool found; 365 366 /* 367 * The prev->end is always aligned on the page size, which 368 * causes page alignment for the entry->start too. The size 369 * is checked to be multiple of the page size. 370 * 371 * The page sized gap is created between consequent 372 * allocations to ensure that out-of-bounds accesses fault. 373 */ 374 a->entry->end = a->entry->start + a->size; 375 376 next = RB_NEXT(dmar_gas_entries_tree, &a->domain->rb_root, prev); 377 KASSERT(next->start >= a->entry->end && 378 next->start - a->entry->start >= a->size && 379 prev->end <= a->entry->end, 380 ("dmar_gas_match_insert hole failed %p prev (%jx, %jx) " 381 "free_after %jx next (%jx, %jx) entry (%jx, %jx)", a->domain, 382 (uintmax_t)prev->start, (uintmax_t)prev->end, 383 (uintmax_t)prev->free_after, 384 (uintmax_t)next->start, (uintmax_t)next->end, 385 (uintmax_t)a->entry->start, (uintmax_t)a->entry->end)); 386 387 prev->free_after = a->entry->start - prev->end; 388 a->entry->free_after = next->start - a->entry->end; 389 390 found = dmar_gas_rb_insert(a->domain, a->entry); 391 KASSERT(found, ("found dup %p start %jx size %jx", 392 a->domain, (uintmax_t)a->entry->start, (uintmax_t)a->size)); 393 a->entry->flags = DMAR_MAP_ENTRY_MAP; 394 395 KASSERT(RB_PREV(dmar_gas_entries_tree, &a->domain->rb_root, 396 a->entry) == prev, 397 ("entry %p prev %p inserted prev %p", a->entry, prev, 398 RB_PREV(dmar_gas_entries_tree, &a->domain->rb_root, a->entry))); 399 KASSERT(RB_NEXT(dmar_gas_entries_tree, &a->domain->rb_root, 400 a->entry) == next, 401 ("entry %p next %p inserted next %p", a->entry, next, 402 RB_NEXT(dmar_gas_entries_tree, &a->domain->rb_root, a->entry))); 403 } 404 405 static int 406 dmar_gas_lowermatch(struct dmar_gas_match_args *a, struct dmar_map_entry *prev) 407 { 408 struct dmar_map_entry *l; 409 int ret; 410 411 if (prev->end < a->common->lowaddr) { 412 a->entry->start = roundup2(prev->end + DMAR_PAGE_SIZE, 413 a->common->alignment); 414 if (dmar_gas_match_one(a, prev, a->common->lowaddr)) { 415 dmar_gas_match_insert(a, prev); 416 return (0); 417 } 418 } 419 if (prev->free_down < a->size + a->offset + DMAR_PAGE_SIZE) 420 return (ENOMEM); 421 l = RB_LEFT(prev, rb_entry); 422 if (l != NULL) { 423 ret = dmar_gas_lowermatch(a, l); 424 if (ret == 0) 425 return (0); 426 } 427 l = RB_RIGHT(prev, rb_entry); 428 if (l != NULL) 429 return (dmar_gas_lowermatch(a, l)); 430 return (ENOMEM); 431 } 432 433 static int 434 dmar_gas_uppermatch(struct dmar_gas_match_args *a) 435 { 436 struct dmar_map_entry *next, *prev, find_entry; 437 438 find_entry.start = a->common->highaddr; 439 next = RB_NFIND(dmar_gas_entries_tree, &a->domain->rb_root, 440 &find_entry); 441 if (next == NULL) 442 return (ENOMEM); 443 prev = RB_PREV(dmar_gas_entries_tree, &a->domain->rb_root, next); 444 KASSERT(prev != NULL, ("no prev %p %jx", a->domain, 445 (uintmax_t)find_entry.start)); 446 for (;;) { 447 a->entry->start = prev->start + DMAR_PAGE_SIZE; 448 if (a->entry->start < a->common->highaddr) 449 a->entry->start = a->common->highaddr; 450 a->entry->start = roundup2(a->entry->start, 451 a->common->alignment); 452 if (dmar_gas_match_one(a, prev, a->domain->end)) { 453 dmar_gas_match_insert(a, prev); 454 return (0); 455 } 456 457 /* 458 * XXXKIB. This falls back to linear iteration over 459 * the free space in the high region. But high 460 * regions are almost unused, the code should be 461 * enough to cover the case, although in the 462 * non-optimal way. 463 */ 464 prev = next; 465 next = RB_NEXT(dmar_gas_entries_tree, &a->domain->rb_root, 466 prev); 467 KASSERT(next != NULL, ("no next %p %jx", a->domain, 468 (uintmax_t)find_entry.start)); 469 if (next->end >= a->domain->end) 470 return (ENOMEM); 471 } 472 } 473 474 static int 475 dmar_gas_find_space(struct dmar_domain *domain, 476 const struct bus_dma_tag_common *common, dmar_gaddr_t size, 477 int offset, u_int flags, struct dmar_map_entry *entry) 478 { 479 struct dmar_gas_match_args a; 480 int error; 481 482 DMAR_DOMAIN_ASSERT_LOCKED(domain); 483 KASSERT(entry->flags == 0, ("dirty entry %p %p", domain, entry)); 484 KASSERT((size & DMAR_PAGE_MASK) == 0, ("size %jx", (uintmax_t)size)); 485 486 a.domain = domain; 487 a.size = size; 488 a.offset = offset; 489 a.common = common; 490 a.gas_flags = flags; 491 a.entry = entry; 492 493 /* Handle lower region. */ 494 if (common->lowaddr > 0) { 495 error = dmar_gas_lowermatch(&a, RB_ROOT(&domain->rb_root)); 496 if (error == 0) 497 return (0); 498 KASSERT(error == ENOMEM, 499 ("error %d from dmar_gas_lowermatch", error)); 500 } 501 /* Handle upper region. */ 502 if (common->highaddr >= domain->end) 503 return (ENOMEM); 504 error = dmar_gas_uppermatch(&a); 505 KASSERT(error == ENOMEM, 506 ("error %d from dmar_gas_uppermatch", error)); 507 return (error); 508 } 509 510 static int 511 dmar_gas_alloc_region(struct dmar_domain *domain, struct dmar_map_entry *entry, 512 u_int flags) 513 { 514 struct dmar_map_entry *next, *prev; 515 bool found; 516 517 DMAR_DOMAIN_ASSERT_LOCKED(domain); 518 519 if ((entry->start & DMAR_PAGE_MASK) != 0 || 520 (entry->end & DMAR_PAGE_MASK) != 0) 521 return (EINVAL); 522 if (entry->start >= entry->end) 523 return (EINVAL); 524 if (entry->end >= domain->end) 525 return (EINVAL); 526 527 next = RB_NFIND(dmar_gas_entries_tree, &domain->rb_root, entry); 528 KASSERT(next != NULL, ("next must be non-null %p %jx", domain, 529 (uintmax_t)entry->start)); 530 prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, next); 531 /* prev could be NULL */ 532 533 /* 534 * Adapt to broken BIOSes which specify overlapping RMRR 535 * entries. 536 * 537 * XXXKIB: this does not handle a case when prev or next 538 * entries are completely covered by the current one, which 539 * extends both ways. 540 */ 541 if (prev != NULL && prev->end > entry->start && 542 (prev->flags & DMAR_MAP_ENTRY_PLACE) == 0) { 543 if ((prev->flags & DMAR_MAP_ENTRY_RMRR) == 0) 544 return (EBUSY); 545 entry->start = prev->end; 546 } 547 if (next != NULL && next->start < entry->end && 548 (next->flags & DMAR_MAP_ENTRY_PLACE) == 0) { 549 if ((next->flags & DMAR_MAP_ENTRY_RMRR) == 0) 550 return (EBUSY); 551 entry->end = next->start; 552 } 553 if (entry->end == entry->start) 554 return (0); 555 556 if (prev != NULL && prev->end > entry->start) { 557 /* This assumes that prev is the placeholder entry. */ 558 dmar_gas_rb_remove(domain, prev); 559 prev = NULL; 560 } 561 if (next != NULL && next->start < entry->end) { 562 dmar_gas_rb_remove(domain, next); 563 next = NULL; 564 } 565 566 found = dmar_gas_rb_insert(domain, entry); 567 KASSERT(found, ("found RMRR dup %p start %jx end %jx", 568 domain, (uintmax_t)entry->start, (uintmax_t)entry->end)); 569 entry->flags = DMAR_MAP_ENTRY_RMRR; 570 571 #ifdef INVARIANTS 572 struct dmar_map_entry *ip, *in; 573 ip = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); 574 in = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); 575 KASSERT(prev == NULL || ip == prev, 576 ("RMRR %p (%jx %jx) prev %p (%jx %jx) ins prev %p (%jx %jx)", 577 entry, entry->start, entry->end, prev, 578 prev == NULL ? 0 : prev->start, prev == NULL ? 0 : prev->end, 579 ip, ip == NULL ? 0 : ip->start, ip == NULL ? 0 : ip->end)); 580 KASSERT(next == NULL || in == next, 581 ("RMRR %p (%jx %jx) next %p (%jx %jx) ins next %p (%jx %jx)", 582 entry, entry->start, entry->end, next, 583 next == NULL ? 0 : next->start, next == NULL ? 0 : next->end, 584 in, in == NULL ? 0 : in->start, in == NULL ? 0 : in->end)); 585 #endif 586 587 return (0); 588 } 589 590 void 591 dmar_gas_free_space(struct dmar_domain *domain, struct dmar_map_entry *entry) 592 { 593 594 DMAR_DOMAIN_ASSERT_LOCKED(domain); 595 KASSERT((entry->flags & (DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_RMRR | 596 DMAR_MAP_ENTRY_MAP)) == DMAR_MAP_ENTRY_MAP, 597 ("permanent entry %p %p", domain, entry)); 598 599 dmar_gas_rb_remove(domain, entry); 600 entry->flags &= ~DMAR_MAP_ENTRY_MAP; 601 #ifdef INVARIANTS 602 if (dmar_check_free) 603 dmar_gas_check_free(domain); 604 #endif 605 } 606 607 void 608 dmar_gas_free_region(struct dmar_domain *domain, struct dmar_map_entry *entry) 609 { 610 struct dmar_map_entry *next, *prev; 611 612 DMAR_DOMAIN_ASSERT_LOCKED(domain); 613 KASSERT((entry->flags & (DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_RMRR | 614 DMAR_MAP_ENTRY_MAP)) == DMAR_MAP_ENTRY_RMRR, 615 ("non-RMRR entry %p %p", domain, entry)); 616 617 prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); 618 next = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); 619 dmar_gas_rb_remove(domain, entry); 620 entry->flags &= ~DMAR_MAP_ENTRY_RMRR; 621 622 if (prev == NULL) 623 dmar_gas_rb_insert(domain, domain->first_place); 624 if (next == NULL) 625 dmar_gas_rb_insert(domain, domain->last_place); 626 } 627 628 int 629 dmar_gas_map(struct dmar_domain *domain, 630 const struct bus_dma_tag_common *common, dmar_gaddr_t size, int offset, 631 u_int eflags, u_int flags, vm_page_t *ma, struct dmar_map_entry **res) 632 { 633 struct dmar_map_entry *entry; 634 int error; 635 636 KASSERT((flags & ~(DMAR_GM_CANWAIT | DMAR_GM_CANSPLIT)) == 0, 637 ("invalid flags 0x%x", flags)); 638 639 entry = dmar_gas_alloc_entry(domain, (flags & DMAR_GM_CANWAIT) != 0 ? 640 DMAR_PGF_WAITOK : 0); 641 if (entry == NULL) 642 return (ENOMEM); 643 DMAR_DOMAIN_LOCK(domain); 644 error = dmar_gas_find_space(domain, common, size, offset, flags, 645 entry); 646 if (error == ENOMEM) { 647 DMAR_DOMAIN_UNLOCK(domain); 648 dmar_gas_free_entry(domain, entry); 649 return (error); 650 } 651 #ifdef INVARIANTS 652 if (dmar_check_free) 653 dmar_gas_check_free(domain); 654 #endif 655 KASSERT(error == 0, 656 ("unexpected error %d from dmar_gas_find_entry", error)); 657 KASSERT(entry->end < domain->end, ("allocated GPA %jx, max GPA %jx", 658 (uintmax_t)entry->end, (uintmax_t)domain->end)); 659 entry->flags |= eflags; 660 DMAR_DOMAIN_UNLOCK(domain); 661 662 error = domain_map_buf(domain, entry->start, entry->end - entry->start, 663 ma, 664 ((eflags & DMAR_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) | 665 ((eflags & DMAR_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) | 666 ((eflags & DMAR_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) | 667 ((eflags & DMAR_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0), 668 (flags & DMAR_GM_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0); 669 if (error == ENOMEM) { 670 dmar_domain_unload_entry(entry, true); 671 return (error); 672 } 673 KASSERT(error == 0, 674 ("unexpected error %d from domain_map_buf", error)); 675 676 *res = entry; 677 return (0); 678 } 679 680 int 681 dmar_gas_map_region(struct dmar_domain *domain, struct dmar_map_entry *entry, 682 u_int eflags, u_int flags, vm_page_t *ma) 683 { 684 dmar_gaddr_t start; 685 int error; 686 687 KASSERT(entry->flags == 0, ("used RMRR entry %p %p %x", domain, 688 entry, entry->flags)); 689 KASSERT((flags & ~(DMAR_GM_CANWAIT)) == 0, 690 ("invalid flags 0x%x", flags)); 691 692 start = entry->start; 693 DMAR_DOMAIN_LOCK(domain); 694 error = dmar_gas_alloc_region(domain, entry, flags); 695 if (error != 0) { 696 DMAR_DOMAIN_UNLOCK(domain); 697 return (error); 698 } 699 entry->flags |= eflags; 700 DMAR_DOMAIN_UNLOCK(domain); 701 if (entry->end == entry->start) 702 return (0); 703 704 error = domain_map_buf(domain, entry->start, entry->end - entry->start, 705 ma + OFF_TO_IDX(start - entry->start), 706 ((eflags & DMAR_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) | 707 ((eflags & DMAR_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) | 708 ((eflags & DMAR_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) | 709 ((eflags & DMAR_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0), 710 (flags & DMAR_GM_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0); 711 if (error == ENOMEM) { 712 dmar_domain_unload_entry(entry, false); 713 return (error); 714 } 715 KASSERT(error == 0, 716 ("unexpected error %d from domain_map_buf", error)); 717 718 return (0); 719 } 720 721 int 722 dmar_gas_reserve_region(struct dmar_domain *domain, dmar_gaddr_t start, 723 dmar_gaddr_t end) 724 { 725 struct dmar_map_entry *entry; 726 int error; 727 728 entry = dmar_gas_alloc_entry(domain, DMAR_PGF_WAITOK); 729 entry->start = start; 730 entry->end = end; 731 DMAR_DOMAIN_LOCK(domain); 732 error = dmar_gas_alloc_region(domain, entry, DMAR_GM_CANWAIT); 733 if (error == 0) 734 entry->flags |= DMAR_MAP_ENTRY_UNMAPPED; 735 DMAR_DOMAIN_UNLOCK(domain); 736 if (error != 0) 737 dmar_gas_free_entry(domain, entry); 738 return (error); 739 }
Cache object: 63fcc1a1a4f683159fcbf745efcc5877
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]