FreeBSD/Linux Kernel Cross Reference
sys/kern/subr_pool.c
1 /* $NetBSD: subr_pool.c,v 1.93.2.1 2004/06/22 08:58:42 tron Exp $ */
2
3 /*-
4 * Copyright (c) 1997, 1999, 2000 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Paul Kranenburg; by Jason R. Thorpe of the Numerical Aerospace
9 * Simulation Facility, NASA Ames Research Center.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. All advertising materials mentioning features or use of this software
20 * must display the following acknowledgement:
21 * This product includes software developed by the NetBSD
22 * Foundation, Inc. and its contributors.
23 * 4. Neither the name of The NetBSD Foundation nor the names of its
24 * contributors may be used to endorse or promote products derived
25 * from this software without specific prior written permission.
26 *
27 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
29 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
31 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
37 * POSSIBILITY OF SUCH DAMAGE.
38 */
39
40 #include <sys/cdefs.h>
41 __KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.93.2.1 2004/06/22 08:58:42 tron Exp $");
42
43 #include "opt_pool.h"
44 #include "opt_poollog.h"
45 #include "opt_lockdebug.h"
46
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/proc.h>
50 #include <sys/errno.h>
51 #include <sys/kernel.h>
52 #include <sys/malloc.h>
53 #include <sys/lock.h>
54 #include <sys/pool.h>
55 #include <sys/syslog.h>
56
57 #include <uvm/uvm.h>
58
59 /*
60 * Pool resource management utility.
61 *
62 * Memory is allocated in pages which are split into pieces according to
63 * the pool item size. Each page is kept on one of three lists in the
64 * pool structure: `pr_emptypages', `pr_fullpages' and `pr_partpages',
65 * for empty, full and partially-full pages respectively. The individual
66 * pool items are on a linked list headed by `ph_itemlist' in each page
67 * header. The memory for building the page list is either taken from
68 * the allocated pages themselves (for small pool items) or taken from
69 * an internal pool of page headers (`phpool').
70 */
71
72 /* List of all pools */
73 TAILQ_HEAD(,pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head);
74
75 /* Private pool for page header structures */
76 static struct pool phpool;
77
78 #ifdef POOL_SUBPAGE
79 /* Pool of subpages for use by normal pools. */
80 static struct pool psppool;
81 #endif
82
83 /* # of seconds to retain page after last use */
84 int pool_inactive_time = 10;
85
86 /* Next candidate for drainage (see pool_drain()) */
87 static struct pool *drainpp;
88
89 /* This spin lock protects both pool_head and drainpp. */
90 struct simplelock pool_head_slock = SIMPLELOCK_INITIALIZER;
91
92 struct pool_item_header {
93 /* Page headers */
94 LIST_ENTRY(pool_item_header)
95 ph_pagelist; /* pool page list */
96 TAILQ_HEAD(,pool_item) ph_itemlist; /* chunk list for this page */
97 SPLAY_ENTRY(pool_item_header)
98 ph_node; /* Off-page page headers */
99 unsigned int ph_nmissing; /* # of chunks in use */
100 caddr_t ph_page; /* this page's address */
101 struct timeval ph_time; /* last referenced */
102 };
103
104 struct pool_item {
105 #ifdef DIAGNOSTIC
106 u_int pi_magic;
107 #endif
108 #define PI_MAGIC 0xdeadbeefU
109 /* Other entries use only this list entry */
110 TAILQ_ENTRY(pool_item) pi_list;
111 };
112
113 #define POOL_NEEDS_CATCHUP(pp) \
114 ((pp)->pr_nitems < (pp)->pr_minitems)
115
116 /*
117 * Pool cache management.
118 *
119 * Pool caches provide a way for constructed objects to be cached by the
120 * pool subsystem. This can lead to performance improvements by avoiding
121 * needless object construction/destruction; it is deferred until absolutely
122 * necessary.
123 *
124 * Caches are grouped into cache groups. Each cache group references
125 * up to 16 constructed objects. When a cache allocates an object
126 * from the pool, it calls the object's constructor and places it into
127 * a cache group. When a cache group frees an object back to the pool,
128 * it first calls the object's destructor. This allows the object to
129 * persist in constructed form while freed to the cache.
130 *
131 * Multiple caches may exist for each pool. This allows a single
132 * object type to have multiple constructed forms. The pool references
133 * each cache, so that when a pool is drained by the pagedaemon, it can
134 * drain each individual cache as well. Each time a cache is drained,
135 * the most idle cache group is freed to the pool in its entirety.
136 *
137 * Pool caches are layed on top of pools. By layering them, we can avoid
138 * the complexity of cache management for pools which would not benefit
139 * from it.
140 */
141
142 /* The cache group pool. */
143 static struct pool pcgpool;
144
145 static void pool_cache_reclaim(struct pool_cache *);
146
147 static int pool_catchup(struct pool *);
148 static void pool_prime_page(struct pool *, caddr_t,
149 struct pool_item_header *);
150 static void pool_update_curpage(struct pool *);
151
152 void *pool_allocator_alloc(struct pool *, int);
153 void pool_allocator_free(struct pool *, void *);
154
155 static void pool_print_pagelist(struct pool_pagelist *,
156 void (*)(const char *, ...));
157 static void pool_print1(struct pool *, const char *,
158 void (*)(const char *, ...));
159
160 static int pool_chk_page(struct pool *, const char *,
161 struct pool_item_header *);
162
163 /*
164 * Pool log entry. An array of these is allocated in pool_init().
165 */
166 struct pool_log {
167 const char *pl_file;
168 long pl_line;
169 int pl_action;
170 #define PRLOG_GET 1
171 #define PRLOG_PUT 2
172 void *pl_addr;
173 };
174
175 #ifdef POOL_DIAGNOSTIC
176 /* Number of entries in pool log buffers */
177 #ifndef POOL_LOGSIZE
178 #define POOL_LOGSIZE 10
179 #endif
180
181 int pool_logsize = POOL_LOGSIZE;
182
183 static __inline void
184 pr_log(struct pool *pp, void *v, int action, const char *file, long line)
185 {
186 int n = pp->pr_curlogentry;
187 struct pool_log *pl;
188
189 if ((pp->pr_roflags & PR_LOGGING) == 0)
190 return;
191
192 /*
193 * Fill in the current entry. Wrap around and overwrite
194 * the oldest entry if necessary.
195 */
196 pl = &pp->pr_log[n];
197 pl->pl_file = file;
198 pl->pl_line = line;
199 pl->pl_action = action;
200 pl->pl_addr = v;
201 if (++n >= pp->pr_logsize)
202 n = 0;
203 pp->pr_curlogentry = n;
204 }
205
206 static void
207 pr_printlog(struct pool *pp, struct pool_item *pi,
208 void (*pr)(const char *, ...))
209 {
210 int i = pp->pr_logsize;
211 int n = pp->pr_curlogentry;
212
213 if ((pp->pr_roflags & PR_LOGGING) == 0)
214 return;
215
216 /*
217 * Print all entries in this pool's log.
218 */
219 while (i-- > 0) {
220 struct pool_log *pl = &pp->pr_log[n];
221 if (pl->pl_action != 0) {
222 if (pi == NULL || pi == pl->pl_addr) {
223 (*pr)("\tlog entry %d:\n", i);
224 (*pr)("\t\taction = %s, addr = %p\n",
225 pl->pl_action == PRLOG_GET ? "get" : "put",
226 pl->pl_addr);
227 (*pr)("\t\tfile: %s at line %lu\n",
228 pl->pl_file, pl->pl_line);
229 }
230 }
231 if (++n >= pp->pr_logsize)
232 n = 0;
233 }
234 }
235
236 static __inline void
237 pr_enter(struct pool *pp, const char *file, long line)
238 {
239
240 if (__predict_false(pp->pr_entered_file != NULL)) {
241 printf("pool %s: reentrancy at file %s line %ld\n",
242 pp->pr_wchan, file, line);
243 printf(" previous entry at file %s line %ld\n",
244 pp->pr_entered_file, pp->pr_entered_line);
245 panic("pr_enter");
246 }
247
248 pp->pr_entered_file = file;
249 pp->pr_entered_line = line;
250 }
251
252 static __inline void
253 pr_leave(struct pool *pp)
254 {
255
256 if (__predict_false(pp->pr_entered_file == NULL)) {
257 printf("pool %s not entered?\n", pp->pr_wchan);
258 panic("pr_leave");
259 }
260
261 pp->pr_entered_file = NULL;
262 pp->pr_entered_line = 0;
263 }
264
265 static __inline void
266 pr_enter_check(struct pool *pp, void (*pr)(const char *, ...))
267 {
268
269 if (pp->pr_entered_file != NULL)
270 (*pr)("\n\tcurrently entered from file %s line %ld\n",
271 pp->pr_entered_file, pp->pr_entered_line);
272 }
273 #else
274 #define pr_log(pp, v, action, file, line)
275 #define pr_printlog(pp, pi, pr)
276 #define pr_enter(pp, file, line)
277 #define pr_leave(pp)
278 #define pr_enter_check(pp, pr)
279 #endif /* POOL_DIAGNOSTIC */
280
281 static __inline int
282 phtree_compare(struct pool_item_header *a, struct pool_item_header *b)
283 {
284 if (a->ph_page < b->ph_page)
285 return (-1);
286 else if (a->ph_page > b->ph_page)
287 return (1);
288 else
289 return (0);
290 }
291
292 SPLAY_PROTOTYPE(phtree, pool_item_header, ph_node, phtree_compare);
293 SPLAY_GENERATE(phtree, pool_item_header, ph_node, phtree_compare);
294
295 /*
296 * Return the pool page header based on page address.
297 */
298 static __inline struct pool_item_header *
299 pr_find_pagehead(struct pool *pp, caddr_t page)
300 {
301 struct pool_item_header *ph, tmp;
302
303 if ((pp->pr_roflags & PR_PHINPAGE) != 0)
304 return ((struct pool_item_header *)(page + pp->pr_phoffset));
305
306 tmp.ph_page = page;
307 ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp);
308 return ph;
309 }
310
311 /*
312 * Remove a page from the pool.
313 */
314 static __inline void
315 pr_rmpage(struct pool *pp, struct pool_item_header *ph,
316 struct pool_pagelist *pq)
317 {
318 int s;
319
320 LOCK_ASSERT(!simple_lock_held(&pp->pr_slock) || pq != NULL);
321
322 /*
323 * If the page was idle, decrement the idle page count.
324 */
325 if (ph->ph_nmissing == 0) {
326 #ifdef DIAGNOSTIC
327 if (pp->pr_nidle == 0)
328 panic("pr_rmpage: nidle inconsistent");
329 if (pp->pr_nitems < pp->pr_itemsperpage)
330 panic("pr_rmpage: nitems inconsistent");
331 #endif
332 pp->pr_nidle--;
333 }
334
335 pp->pr_nitems -= pp->pr_itemsperpage;
336
337 /*
338 * Unlink a page from the pool and release it (or queue it for release).
339 */
340 LIST_REMOVE(ph, ph_pagelist);
341 if ((pp->pr_roflags & PR_PHINPAGE) == 0)
342 SPLAY_REMOVE(phtree, &pp->pr_phtree, ph);
343 if (pq) {
344 LIST_INSERT_HEAD(pq, ph, ph_pagelist);
345 } else {
346 pool_allocator_free(pp, ph->ph_page);
347 if ((pp->pr_roflags & PR_PHINPAGE) == 0) {
348 s = splvm();
349 pool_put(&phpool, ph);
350 splx(s);
351 }
352 }
353 pp->pr_npages--;
354 pp->pr_npagefree++;
355
356 pool_update_curpage(pp);
357 }
358
359 /*
360 * Initialize the given pool resource structure.
361 *
362 * We export this routine to allow other kernel parts to declare
363 * static pools that must be initialized before malloc() is available.
364 */
365 void
366 pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags,
367 const char *wchan, struct pool_allocator *palloc)
368 {
369 int off, slack;
370 size_t trysize, phsize;
371 int s;
372
373 #ifdef POOL_DIAGNOSTIC
374 /*
375 * Always log if POOL_DIAGNOSTIC is defined.
376 */
377 if (pool_logsize != 0)
378 flags |= PR_LOGGING;
379 #endif
380
381 #ifdef POOL_SUBPAGE
382 /*
383 * XXX We don't provide a real `nointr' back-end
384 * yet; all sub-pages come from a kmem back-end.
385 * maybe some day...
386 */
387 if (palloc == NULL) {
388 extern struct pool_allocator pool_allocator_kmem_subpage;
389 palloc = &pool_allocator_kmem_subpage;
390 }
391 /*
392 * We'll assume any user-specified back-end allocator
393 * will deal with sub-pages, or simply don't care.
394 */
395 #else
396 if (palloc == NULL)
397 palloc = &pool_allocator_kmem;
398 #endif /* POOL_SUBPAGE */
399 if ((palloc->pa_flags & PA_INITIALIZED) == 0) {
400 if (palloc->pa_pagesz == 0) {
401 #ifdef POOL_SUBPAGE
402 if (palloc == &pool_allocator_kmem)
403 palloc->pa_pagesz = PAGE_SIZE;
404 else
405 palloc->pa_pagesz = POOL_SUBPAGE;
406 #else
407 palloc->pa_pagesz = PAGE_SIZE;
408 #endif /* POOL_SUBPAGE */
409 }
410
411 TAILQ_INIT(&palloc->pa_list);
412
413 simple_lock_init(&palloc->pa_slock);
414 palloc->pa_pagemask = ~(palloc->pa_pagesz - 1);
415 palloc->pa_pageshift = ffs(palloc->pa_pagesz) - 1;
416 palloc->pa_flags |= PA_INITIALIZED;
417 }
418
419 if (align == 0)
420 align = ALIGN(1);
421
422 if (size < sizeof(struct pool_item))
423 size = sizeof(struct pool_item);
424
425 size = roundup(size, align);
426 #ifdef DIAGNOSTIC
427 if (size > palloc->pa_pagesz)
428 panic("pool_init: pool item size (%lu) too large",
429 (u_long)size);
430 #endif
431
432 /*
433 * Initialize the pool structure.
434 */
435 LIST_INIT(&pp->pr_emptypages);
436 LIST_INIT(&pp->pr_fullpages);
437 LIST_INIT(&pp->pr_partpages);
438 TAILQ_INIT(&pp->pr_cachelist);
439 pp->pr_curpage = NULL;
440 pp->pr_npages = 0;
441 pp->pr_minitems = 0;
442 pp->pr_minpages = 0;
443 pp->pr_maxpages = UINT_MAX;
444 pp->pr_roflags = flags;
445 pp->pr_flags = 0;
446 pp->pr_size = size;
447 pp->pr_align = align;
448 pp->pr_wchan = wchan;
449 pp->pr_alloc = palloc;
450 pp->pr_nitems = 0;
451 pp->pr_nout = 0;
452 pp->pr_hardlimit = UINT_MAX;
453 pp->pr_hardlimit_warning = NULL;
454 pp->pr_hardlimit_ratecap.tv_sec = 0;
455 pp->pr_hardlimit_ratecap.tv_usec = 0;
456 pp->pr_hardlimit_warning_last.tv_sec = 0;
457 pp->pr_hardlimit_warning_last.tv_usec = 0;
458 pp->pr_drain_hook = NULL;
459 pp->pr_drain_hook_arg = NULL;
460
461 /*
462 * Decide whether to put the page header off page to avoid
463 * wasting too large a part of the page or too big item.
464 * Off-page page headers go on a hash table, so we can match
465 * a returned item with its header based on the page address.
466 * We use 1/16 of the page size and about 8 times of the item
467 * size as the threshold (XXX: tune)
468 *
469 * However, we'll put the header into the page if we can put
470 * it without wasting any items.
471 *
472 * Silently enforce `0 <= ioff < align'.
473 */
474 pp->pr_itemoffset = ioff %= align;
475 /* See the comment below about reserved bytes. */
476 trysize = palloc->pa_pagesz - ((align - ioff) % align);
477 phsize = ALIGN(sizeof(struct pool_item_header));
478 if (pp->pr_size < MIN(palloc->pa_pagesz / 16, phsize << 3) ||
479 trysize / pp->pr_size == (trysize - phsize) / pp->pr_size) {
480 /* Use the end of the page for the page header */
481 pp->pr_roflags |= PR_PHINPAGE;
482 pp->pr_phoffset = off = palloc->pa_pagesz - phsize;
483 } else {
484 /* The page header will be taken from our page header pool */
485 pp->pr_phoffset = 0;
486 off = palloc->pa_pagesz;
487 SPLAY_INIT(&pp->pr_phtree);
488 }
489
490 /*
491 * Alignment is to take place at `ioff' within the item. This means
492 * we must reserve up to `align - 1' bytes on the page to allow
493 * appropriate positioning of each item.
494 */
495 pp->pr_itemsperpage = (off - ((align - ioff) % align)) / pp->pr_size;
496 KASSERT(pp->pr_itemsperpage != 0);
497
498 /*
499 * Use the slack between the chunks and the page header
500 * for "cache coloring".
501 */
502 slack = off - pp->pr_itemsperpage * pp->pr_size;
503 pp->pr_maxcolor = (slack / align) * align;
504 pp->pr_curcolor = 0;
505
506 pp->pr_nget = 0;
507 pp->pr_nfail = 0;
508 pp->pr_nput = 0;
509 pp->pr_npagealloc = 0;
510 pp->pr_npagefree = 0;
511 pp->pr_hiwat = 0;
512 pp->pr_nidle = 0;
513
514 #ifdef POOL_DIAGNOSTIC
515 if (flags & PR_LOGGING) {
516 if (kmem_map == NULL ||
517 (pp->pr_log = malloc(pool_logsize * sizeof(struct pool_log),
518 M_TEMP, M_NOWAIT)) == NULL)
519 pp->pr_roflags &= ~PR_LOGGING;
520 pp->pr_curlogentry = 0;
521 pp->pr_logsize = pool_logsize;
522 }
523 #endif
524
525 pp->pr_entered_file = NULL;
526 pp->pr_entered_line = 0;
527
528 simple_lock_init(&pp->pr_slock);
529
530 /*
531 * Initialize private page header pool and cache magazine pool if we
532 * haven't done so yet.
533 * XXX LOCKING.
534 */
535 if (phpool.pr_size == 0) {
536 #ifdef POOL_SUBPAGE
537 pool_init(&phpool, sizeof(struct pool_item_header), 0, 0, 0,
538 "phpool", &pool_allocator_kmem);
539 pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0,
540 PR_RECURSIVE, "psppool", &pool_allocator_kmem);
541 #else
542 pool_init(&phpool, sizeof(struct pool_item_header), 0, 0,
543 0, "phpool", NULL);
544 #endif
545 pool_init(&pcgpool, sizeof(struct pool_cache_group), 0, 0,
546 0, "pcgpool", NULL);
547 }
548
549 /* Insert into the list of all pools. */
550 simple_lock(&pool_head_slock);
551 TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist);
552 simple_unlock(&pool_head_slock);
553
554 /* Insert this into the list of pools using this allocator. */
555 s = splvm();
556 simple_lock(&palloc->pa_slock);
557 TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list);
558 simple_unlock(&palloc->pa_slock);
559 splx(s);
560 }
561
562 /*
563 * De-commision a pool resource.
564 */
565 void
566 pool_destroy(struct pool *pp)
567 {
568 struct pool_item_header *ph;
569 struct pool_cache *pc;
570 int s;
571
572 /* Locking order: pool_allocator -> pool */
573 s = splvm();
574 simple_lock(&pp->pr_alloc->pa_slock);
575 TAILQ_REMOVE(&pp->pr_alloc->pa_list, pp, pr_alloc_list);
576 simple_unlock(&pp->pr_alloc->pa_slock);
577 splx(s);
578
579 /* Destroy all caches for this pool. */
580 while ((pc = TAILQ_FIRST(&pp->pr_cachelist)) != NULL)
581 pool_cache_destroy(pc);
582
583 #ifdef DIAGNOSTIC
584 if (pp->pr_nout != 0) {
585 pr_printlog(pp, NULL, printf);
586 panic("pool_destroy: pool busy: still out: %u",
587 pp->pr_nout);
588 }
589 #endif
590
591 /* Remove all pages */
592 while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL)
593 pr_rmpage(pp, ph, NULL);
594 KASSERT(LIST_EMPTY(&pp->pr_fullpages));
595 KASSERT(LIST_EMPTY(&pp->pr_partpages));
596
597 /* Remove from global pool list */
598 simple_lock(&pool_head_slock);
599 TAILQ_REMOVE(&pool_head, pp, pr_poollist);
600 if (drainpp == pp) {
601 drainpp = NULL;
602 }
603 simple_unlock(&pool_head_slock);
604
605 #ifdef POOL_DIAGNOSTIC
606 if ((pp->pr_roflags & PR_LOGGING) != 0)
607 free(pp->pr_log, M_TEMP);
608 #endif
609 }
610
611 void
612 pool_set_drain_hook(struct pool *pp, void (*fn)(void *, int), void *arg)
613 {
614
615 /* XXX no locking -- must be used just after pool_init() */
616 #ifdef DIAGNOSTIC
617 if (pp->pr_drain_hook != NULL)
618 panic("pool_set_drain_hook(%s): already set", pp->pr_wchan);
619 #endif
620 pp->pr_drain_hook = fn;
621 pp->pr_drain_hook_arg = arg;
622 }
623
624 static struct pool_item_header *
625 pool_alloc_item_header(struct pool *pp, caddr_t storage, int flags)
626 {
627 struct pool_item_header *ph;
628 int s;
629
630 LOCK_ASSERT(simple_lock_held(&pp->pr_slock) == 0);
631
632 if ((pp->pr_roflags & PR_PHINPAGE) != 0)
633 ph = (struct pool_item_header *) (storage + pp->pr_phoffset);
634 else {
635 s = splvm();
636 ph = pool_get(&phpool, flags);
637 splx(s);
638 }
639
640 return (ph);
641 }
642
643 /*
644 * Grab an item from the pool; must be called at appropriate spl level
645 */
646 void *
647 #ifdef POOL_DIAGNOSTIC
648 _pool_get(struct pool *pp, int flags, const char *file, long line)
649 #else
650 pool_get(struct pool *pp, int flags)
651 #endif
652 {
653 struct pool_item *pi;
654 struct pool_item_header *ph;
655 void *v;
656
657 #ifdef DIAGNOSTIC
658 if (__predict_false(curlwp == NULL && doing_shutdown == 0 &&
659 (flags & PR_WAITOK) != 0))
660 panic("pool_get: %s: must have NOWAIT", pp->pr_wchan);
661
662 #ifdef LOCKDEBUG
663 if (flags & PR_WAITOK)
664 simple_lock_only_held(NULL, "pool_get(PR_WAITOK)");
665 #endif
666 #endif /* DIAGNOSTIC */
667
668 simple_lock(&pp->pr_slock);
669 pr_enter(pp, file, line);
670
671 startover:
672 /*
673 * Check to see if we've reached the hard limit. If we have,
674 * and we can wait, then wait until an item has been returned to
675 * the pool.
676 */
677 #ifdef DIAGNOSTIC
678 if (__predict_false(pp->pr_nout > pp->pr_hardlimit)) {
679 pr_leave(pp);
680 simple_unlock(&pp->pr_slock);
681 panic("pool_get: %s: crossed hard limit", pp->pr_wchan);
682 }
683 #endif
684 if (__predict_false(pp->pr_nout == pp->pr_hardlimit)) {
685 if (pp->pr_drain_hook != NULL) {
686 /*
687 * Since the drain hook is going to free things
688 * back to the pool, unlock, call the hook, re-lock,
689 * and check the hardlimit condition again.
690 */
691 pr_leave(pp);
692 simple_unlock(&pp->pr_slock);
693 (*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags);
694 simple_lock(&pp->pr_slock);
695 pr_enter(pp, file, line);
696 if (pp->pr_nout < pp->pr_hardlimit)
697 goto startover;
698 }
699
700 if ((flags & PR_WAITOK) && !(flags & PR_LIMITFAIL)) {
701 /*
702 * XXX: A warning isn't logged in this case. Should
703 * it be?
704 */
705 pp->pr_flags |= PR_WANTED;
706 pr_leave(pp);
707 ltsleep(pp, PSWP, pp->pr_wchan, 0, &pp->pr_slock);
708 pr_enter(pp, file, line);
709 goto startover;
710 }
711
712 /*
713 * Log a message that the hard limit has been hit.
714 */
715 if (pp->pr_hardlimit_warning != NULL &&
716 ratecheck(&pp->pr_hardlimit_warning_last,
717 &pp->pr_hardlimit_ratecap))
718 log(LOG_ERR, "%s\n", pp->pr_hardlimit_warning);
719
720 pp->pr_nfail++;
721
722 pr_leave(pp);
723 simple_unlock(&pp->pr_slock);
724 return (NULL);
725 }
726
727 /*
728 * The convention we use is that if `curpage' is not NULL, then
729 * it points at a non-empty bucket. In particular, `curpage'
730 * never points at a page header which has PR_PHINPAGE set and
731 * has no items in its bucket.
732 */
733 if ((ph = pp->pr_curpage) == NULL) {
734 #ifdef DIAGNOSTIC
735 if (pp->pr_nitems != 0) {
736 simple_unlock(&pp->pr_slock);
737 printf("pool_get: %s: curpage NULL, nitems %u\n",
738 pp->pr_wchan, pp->pr_nitems);
739 panic("pool_get: nitems inconsistent");
740 }
741 #endif
742
743 /*
744 * Call the back-end page allocator for more memory.
745 * Release the pool lock, as the back-end page allocator
746 * may block.
747 */
748 pr_leave(pp);
749 simple_unlock(&pp->pr_slock);
750 v = pool_allocator_alloc(pp, flags);
751 if (__predict_true(v != NULL))
752 ph = pool_alloc_item_header(pp, v, flags);
753
754 if (__predict_false(v == NULL || ph == NULL)) {
755 if (v != NULL)
756 pool_allocator_free(pp, v);
757
758 simple_lock(&pp->pr_slock);
759 pr_enter(pp, file, line);
760
761 /*
762 * We were unable to allocate a page or item
763 * header, but we released the lock during
764 * allocation, so perhaps items were freed
765 * back to the pool. Check for this case.
766 */
767 if (pp->pr_curpage != NULL)
768 goto startover;
769
770 if ((flags & PR_WAITOK) == 0) {
771 pp->pr_nfail++;
772 pr_leave(pp);
773 simple_unlock(&pp->pr_slock);
774 return (NULL);
775 }
776
777 /*
778 * Wait for items to be returned to this pool.
779 *
780 * XXX: maybe we should wake up once a second and
781 * try again?
782 */
783 pp->pr_flags |= PR_WANTED;
784 /* PA_WANTED is already set on the allocator. */
785 pr_leave(pp);
786 ltsleep(pp, PSWP, pp->pr_wchan, 0, &pp->pr_slock);
787 pr_enter(pp, file, line);
788 goto startover;
789 }
790
791 /* We have more memory; add it to the pool */
792 simple_lock(&pp->pr_slock);
793 pr_enter(pp, file, line);
794 pool_prime_page(pp, v, ph);
795 pp->pr_npagealloc++;
796
797 /* Start the allocation process over. */
798 goto startover;
799 }
800 if (__predict_false((v = pi = TAILQ_FIRST(&ph->ph_itemlist)) == NULL)) {
801 pr_leave(pp);
802 simple_unlock(&pp->pr_slock);
803 panic("pool_get: %s: page empty", pp->pr_wchan);
804 }
805 #ifdef DIAGNOSTIC
806 if (__predict_false(pp->pr_nitems == 0)) {
807 pr_leave(pp);
808 simple_unlock(&pp->pr_slock);
809 printf("pool_get: %s: items on itemlist, nitems %u\n",
810 pp->pr_wchan, pp->pr_nitems);
811 panic("pool_get: nitems inconsistent");
812 }
813 #endif
814
815 #ifdef POOL_DIAGNOSTIC
816 pr_log(pp, v, PRLOG_GET, file, line);
817 #endif
818
819 #ifdef DIAGNOSTIC
820 if (__predict_false(pi->pi_magic != PI_MAGIC)) {
821 pr_printlog(pp, pi, printf);
822 panic("pool_get(%s): free list modified: magic=%x; page %p;"
823 " item addr %p\n",
824 pp->pr_wchan, pi->pi_magic, ph->ph_page, pi);
825 }
826 #endif
827
828 /*
829 * Remove from item list.
830 */
831 TAILQ_REMOVE(&ph->ph_itemlist, pi, pi_list);
832 pp->pr_nitems--;
833 pp->pr_nout++;
834 if (ph->ph_nmissing == 0) {
835 #ifdef DIAGNOSTIC
836 if (__predict_false(pp->pr_nidle == 0))
837 panic("pool_get: nidle inconsistent");
838 #endif
839 pp->pr_nidle--;
840
841 /*
842 * This page was previously empty. Move it to the list of
843 * partially-full pages. This page is already curpage.
844 */
845 LIST_REMOVE(ph, ph_pagelist);
846 LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist);
847 }
848 ph->ph_nmissing++;
849 if (TAILQ_EMPTY(&ph->ph_itemlist)) {
850 #ifdef DIAGNOSTIC
851 if (__predict_false(ph->ph_nmissing != pp->pr_itemsperpage)) {
852 pr_leave(pp);
853 simple_unlock(&pp->pr_slock);
854 panic("pool_get: %s: nmissing inconsistent",
855 pp->pr_wchan);
856 }
857 #endif
858 /*
859 * This page is now full. Move it to the full list
860 * and select a new current page.
861 */
862 LIST_REMOVE(ph, ph_pagelist);
863 LIST_INSERT_HEAD(&pp->pr_fullpages, ph, ph_pagelist);
864 pool_update_curpage(pp);
865 }
866
867 pp->pr_nget++;
868
869 /*
870 * If we have a low water mark and we are now below that low
871 * water mark, add more items to the pool.
872 */
873 if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) {
874 /*
875 * XXX: Should we log a warning? Should we set up a timeout
876 * to try again in a second or so? The latter could break
877 * a caller's assumptions about interrupt protection, etc.
878 */
879 }
880
881 pr_leave(pp);
882 simple_unlock(&pp->pr_slock);
883 return (v);
884 }
885
886 /*
887 * Internal version of pool_put(). Pool is already locked/entered.
888 */
889 static void
890 pool_do_put(struct pool *pp, void *v)
891 {
892 struct pool_item *pi = v;
893 struct pool_item_header *ph;
894 caddr_t page;
895 int s;
896
897 LOCK_ASSERT(simple_lock_held(&pp->pr_slock));
898
899 page = (caddr_t)((u_long)v & pp->pr_alloc->pa_pagemask);
900
901 #ifdef DIAGNOSTIC
902 if (__predict_false(pp->pr_nout == 0)) {
903 printf("pool %s: putting with none out\n",
904 pp->pr_wchan);
905 panic("pool_put");
906 }
907 #endif
908
909 if (__predict_false((ph = pr_find_pagehead(pp, page)) == NULL)) {
910 pr_printlog(pp, NULL, printf);
911 panic("pool_put: %s: page header missing", pp->pr_wchan);
912 }
913
914 #ifdef LOCKDEBUG
915 /*
916 * Check if we're freeing a locked simple lock.
917 */
918 simple_lock_freecheck((caddr_t)pi, ((caddr_t)pi) + pp->pr_size);
919 #endif
920
921 /*
922 * Return to item list.
923 */
924 #ifdef DIAGNOSTIC
925 pi->pi_magic = PI_MAGIC;
926 #endif
927 #ifdef DEBUG
928 {
929 int i, *ip = v;
930
931 for (i = 0; i < pp->pr_size / sizeof(int); i++) {
932 *ip++ = PI_MAGIC;
933 }
934 }
935 #endif
936
937 TAILQ_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list);
938 KDASSERT(ph->ph_nmissing != 0);
939 ph->ph_nmissing--;
940 pp->pr_nput++;
941 pp->pr_nitems++;
942 pp->pr_nout--;
943
944 /* Cancel "pool empty" condition if it exists */
945 if (pp->pr_curpage == NULL)
946 pp->pr_curpage = ph;
947
948 if (pp->pr_flags & PR_WANTED) {
949 pp->pr_flags &= ~PR_WANTED;
950 if (ph->ph_nmissing == 0)
951 pp->pr_nidle++;
952 wakeup((caddr_t)pp);
953 return;
954 }
955
956 /*
957 * If this page is now empty, do one of two things:
958 *
959 * (1) If we have more pages than the page high water mark,
960 * free the page back to the system. ONLY CONSIDER
961 * FREEING BACK A PAGE IF WE HAVE MORE THAN OUR MINIMUM PAGE
962 * CLAIM.
963 *
964 * (2) Otherwise, move the page to the empty page list.
965 *
966 * Either way, select a new current page (so we use a partially-full
967 * page if one is available).
968 */
969 if (ph->ph_nmissing == 0) {
970 pp->pr_nidle++;
971 if (pp->pr_npages > pp->pr_minpages &&
972 (pp->pr_npages > pp->pr_maxpages ||
973 (pp->pr_alloc->pa_flags & PA_WANT) != 0)) {
974 simple_unlock(&pp->pr_slock);
975 pr_rmpage(pp, ph, NULL);
976 simple_lock(&pp->pr_slock);
977 } else {
978 LIST_REMOVE(ph, ph_pagelist);
979 LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist);
980
981 /*
982 * Update the timestamp on the page. A page must
983 * be idle for some period of time before it can
984 * be reclaimed by the pagedaemon. This minimizes
985 * ping-pong'ing for memory.
986 */
987 s = splclock();
988 ph->ph_time = mono_time;
989 splx(s);
990 }
991 pool_update_curpage(pp);
992 }
993
994 /*
995 * If the page was previously completely full, move it to the
996 * partially-full list and make it the current page. The next
997 * allocation will get the item from this page, instead of
998 * further fragmenting the pool.
999 */
1000 else if (ph->ph_nmissing == (pp->pr_itemsperpage - 1)) {
1001 LIST_REMOVE(ph, ph_pagelist);
1002 LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist);
1003 pp->pr_curpage = ph;
1004 }
1005 }
1006
1007 /*
1008 * Return resource to the pool; must be called at appropriate spl level
1009 */
1010 #ifdef POOL_DIAGNOSTIC
1011 void
1012 _pool_put(struct pool *pp, void *v, const char *file, long line)
1013 {
1014
1015 simple_lock(&pp->pr_slock);
1016 pr_enter(pp, file, line);
1017
1018 pr_log(pp, v, PRLOG_PUT, file, line);
1019
1020 pool_do_put(pp, v);
1021
1022 pr_leave(pp);
1023 simple_unlock(&pp->pr_slock);
1024 }
1025 #undef pool_put
1026 #endif /* POOL_DIAGNOSTIC */
1027
1028 void
1029 pool_put(struct pool *pp, void *v)
1030 {
1031
1032 simple_lock(&pp->pr_slock);
1033
1034 pool_do_put(pp, v);
1035
1036 simple_unlock(&pp->pr_slock);
1037 }
1038
1039 #ifdef POOL_DIAGNOSTIC
1040 #define pool_put(h, v) _pool_put((h), (v), __FILE__, __LINE__)
1041 #endif
1042
1043 /*
1044 * Add N items to the pool.
1045 */
1046 int
1047 pool_prime(struct pool *pp, int n)
1048 {
1049 struct pool_item_header *ph = NULL;
1050 caddr_t cp;
1051 int newpages;
1052
1053 simple_lock(&pp->pr_slock);
1054
1055 newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
1056
1057 while (newpages-- > 0) {
1058 simple_unlock(&pp->pr_slock);
1059 cp = pool_allocator_alloc(pp, PR_NOWAIT);
1060 if (__predict_true(cp != NULL))
1061 ph = pool_alloc_item_header(pp, cp, PR_NOWAIT);
1062
1063 if (__predict_false(cp == NULL || ph == NULL)) {
1064 if (cp != NULL)
1065 pool_allocator_free(pp, cp);
1066 simple_lock(&pp->pr_slock);
1067 break;
1068 }
1069
1070 simple_lock(&pp->pr_slock);
1071 pool_prime_page(pp, cp, ph);
1072 pp->pr_npagealloc++;
1073 pp->pr_minpages++;
1074 }
1075
1076 if (pp->pr_minpages >= pp->pr_maxpages)
1077 pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */
1078
1079 simple_unlock(&pp->pr_slock);
1080 return (0);
1081 }
1082
1083 /*
1084 * Add a page worth of items to the pool.
1085 *
1086 * Note, we must be called with the pool descriptor LOCKED.
1087 */
1088 static void
1089 pool_prime_page(struct pool *pp, caddr_t storage, struct pool_item_header *ph)
1090 {
1091 struct pool_item *pi;
1092 caddr_t cp = storage;
1093 unsigned int align = pp->pr_align;
1094 unsigned int ioff = pp->pr_itemoffset;
1095 int n;
1096 int s;
1097
1098 LOCK_ASSERT(simple_lock_held(&pp->pr_slock));
1099
1100 #ifdef DIAGNOSTIC
1101 if (((u_long)cp & (pp->pr_alloc->pa_pagesz - 1)) != 0)
1102 panic("pool_prime_page: %s: unaligned page", pp->pr_wchan);
1103 #endif
1104
1105 /*
1106 * Insert page header.
1107 */
1108 LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist);
1109 TAILQ_INIT(&ph->ph_itemlist);
1110 ph->ph_page = storage;
1111 ph->ph_nmissing = 0;
1112 s = splclock();
1113 ph->ph_time = mono_time;
1114 splx(s);
1115 if ((pp->pr_roflags & PR_PHINPAGE) == 0)
1116 SPLAY_INSERT(phtree, &pp->pr_phtree, ph);
1117
1118 pp->pr_nidle++;
1119
1120 /*
1121 * Color this page.
1122 */
1123 cp = (caddr_t)(cp + pp->pr_curcolor);
1124 if ((pp->pr_curcolor += align) > pp->pr_maxcolor)
1125 pp->pr_curcolor = 0;
1126
1127 /*
1128 * Adjust storage to apply aligment to `pr_itemoffset' in each item.
1129 */
1130 if (ioff != 0)
1131 cp = (caddr_t)(cp + (align - ioff));
1132
1133 /*
1134 * Insert remaining chunks on the bucket list.
1135 */
1136 n = pp->pr_itemsperpage;
1137 pp->pr_nitems += n;
1138
1139 while (n--) {
1140 pi = (struct pool_item *)cp;
1141
1142 KASSERT(((((vaddr_t)pi) + ioff) & (align - 1)) == 0);
1143
1144 /* Insert on page list */
1145 TAILQ_INSERT_TAIL(&ph->ph_itemlist, pi, pi_list);
1146 #ifdef DIAGNOSTIC
1147 pi->pi_magic = PI_MAGIC;
1148 #endif
1149 cp = (caddr_t)(cp + pp->pr_size);
1150 }
1151
1152 /*
1153 * If the pool was depleted, point at the new page.
1154 */
1155 if (pp->pr_curpage == NULL)
1156 pp->pr_curpage = ph;
1157
1158 if (++pp->pr_npages > pp->pr_hiwat)
1159 pp->pr_hiwat = pp->pr_npages;
1160 }
1161
1162 /*
1163 * Used by pool_get() when nitems drops below the low water mark. This
1164 * is used to catch up pr_nitems with the low water mark.
1165 *
1166 * Note 1, we never wait for memory here, we let the caller decide what to do.
1167 *
1168 * Note 2, we must be called with the pool already locked, and we return
1169 * with it locked.
1170 */
1171 static int
1172 pool_catchup(struct pool *pp)
1173 {
1174 struct pool_item_header *ph = NULL;
1175 caddr_t cp;
1176 int error = 0;
1177
1178 while (POOL_NEEDS_CATCHUP(pp)) {
1179 /*
1180 * Call the page back-end allocator for more memory.
1181 *
1182 * XXX: We never wait, so should we bother unlocking
1183 * the pool descriptor?
1184 */
1185 simple_unlock(&pp->pr_slock);
1186 cp = pool_allocator_alloc(pp, PR_NOWAIT);
1187 if (__predict_true(cp != NULL))
1188 ph = pool_alloc_item_header(pp, cp, PR_NOWAIT);
1189 if (__predict_false(cp == NULL || ph == NULL)) {
1190 if (cp != NULL)
1191 pool_allocator_free(pp, cp);
1192 error = ENOMEM;
1193 simple_lock(&pp->pr_slock);
1194 break;
1195 }
1196 simple_lock(&pp->pr_slock);
1197 pool_prime_page(pp, cp, ph);
1198 pp->pr_npagealloc++;
1199 }
1200
1201 return (error);
1202 }
1203
1204 static void
1205 pool_update_curpage(struct pool *pp)
1206 {
1207
1208 pp->pr_curpage = LIST_FIRST(&pp->pr_partpages);
1209 if (pp->pr_curpage == NULL) {
1210 pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages);
1211 }
1212 }
1213
1214 void
1215 pool_setlowat(struct pool *pp, int n)
1216 {
1217
1218 simple_lock(&pp->pr_slock);
1219
1220 pp->pr_minitems = n;
1221 pp->pr_minpages = (n == 0)
1222 ? 0
1223 : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
1224
1225 /* Make sure we're caught up with the newly-set low water mark. */
1226 if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) {
1227 /*
1228 * XXX: Should we log a warning? Should we set up a timeout
1229 * to try again in a second or so? The latter could break
1230 * a caller's assumptions about interrupt protection, etc.
1231 */
1232 }
1233
1234 simple_unlock(&pp->pr_slock);
1235 }
1236
1237 void
1238 pool_sethiwat(struct pool *pp, int n)
1239 {
1240
1241 simple_lock(&pp->pr_slock);
1242
1243 pp->pr_maxpages = (n == 0)
1244 ? 0
1245 : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
1246
1247 simple_unlock(&pp->pr_slock);
1248 }
1249
1250 void
1251 pool_sethardlimit(struct pool *pp, int n, const char *warnmess, int ratecap)
1252 {
1253
1254 simple_lock(&pp->pr_slock);
1255
1256 pp->pr_hardlimit = n;
1257 pp->pr_hardlimit_warning = warnmess;
1258 pp->pr_hardlimit_ratecap.tv_sec = ratecap;
1259 pp->pr_hardlimit_warning_last.tv_sec = 0;
1260 pp->pr_hardlimit_warning_last.tv_usec = 0;
1261
1262 /*
1263 * In-line version of pool_sethiwat(), because we don't want to
1264 * release the lock.
1265 */
1266 pp->pr_maxpages = (n == 0)
1267 ? 0
1268 : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
1269
1270 simple_unlock(&pp->pr_slock);
1271 }
1272
1273 /*
1274 * Release all complete pages that have not been used recently.
1275 */
1276 int
1277 #ifdef POOL_DIAGNOSTIC
1278 _pool_reclaim(struct pool *pp, const char *file, long line)
1279 #else
1280 pool_reclaim(struct pool *pp)
1281 #endif
1282 {
1283 struct pool_item_header *ph, *phnext;
1284 struct pool_cache *pc;
1285 struct timeval curtime;
1286 struct pool_pagelist pq;
1287 struct timeval diff;
1288 int s;
1289
1290 if (pp->pr_drain_hook != NULL) {
1291 /*
1292 * The drain hook must be called with the pool unlocked.
1293 */
1294 (*pp->pr_drain_hook)(pp->pr_drain_hook_arg, PR_NOWAIT);
1295 }
1296
1297 if (simple_lock_try(&pp->pr_slock) == 0)
1298 return (0);
1299 pr_enter(pp, file, line);
1300
1301 LIST_INIT(&pq);
1302
1303 /*
1304 * Reclaim items from the pool's caches.
1305 */
1306 TAILQ_FOREACH(pc, &pp->pr_cachelist, pc_poollist)
1307 pool_cache_reclaim(pc);
1308
1309 s = splclock();
1310 curtime = mono_time;
1311 splx(s);
1312
1313 for (ph = LIST_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) {
1314 phnext = LIST_NEXT(ph, ph_pagelist);
1315
1316 /* Check our minimum page claim */
1317 if (pp->pr_npages <= pp->pr_minpages)
1318 break;
1319
1320 KASSERT(ph->ph_nmissing == 0);
1321 timersub(&curtime, &ph->ph_time, &diff);
1322 if (diff.tv_sec < pool_inactive_time)
1323 continue;
1324
1325 /*
1326 * If freeing this page would put us below
1327 * the low water mark, stop now.
1328 */
1329 if ((pp->pr_nitems - pp->pr_itemsperpage) <
1330 pp->pr_minitems)
1331 break;
1332
1333 pr_rmpage(pp, ph, &pq);
1334 }
1335
1336 pr_leave(pp);
1337 simple_unlock(&pp->pr_slock);
1338 if (LIST_EMPTY(&pq))
1339 return (0);
1340
1341 while ((ph = LIST_FIRST(&pq)) != NULL) {
1342 LIST_REMOVE(ph, ph_pagelist);
1343 pool_allocator_free(pp, ph->ph_page);
1344 if (pp->pr_roflags & PR_PHINPAGE) {
1345 continue;
1346 }
1347 s = splvm();
1348 pool_put(&phpool, ph);
1349 splx(s);
1350 }
1351
1352 return (1);
1353 }
1354
1355 /*
1356 * Drain pools, one at a time.
1357 *
1358 * Note, we must never be called from an interrupt context.
1359 */
1360 void
1361 pool_drain(void *arg)
1362 {
1363 struct pool *pp;
1364 int s;
1365
1366 pp = NULL;
1367 s = splvm();
1368 simple_lock(&pool_head_slock);
1369 if (drainpp == NULL) {
1370 drainpp = TAILQ_FIRST(&pool_head);
1371 }
1372 if (drainpp) {
1373 pp = drainpp;
1374 drainpp = TAILQ_NEXT(pp, pr_poollist);
1375 }
1376 simple_unlock(&pool_head_slock);
1377 pool_reclaim(pp);
1378 splx(s);
1379 }
1380
1381 /*
1382 * Diagnostic helpers.
1383 */
1384 void
1385 pool_print(struct pool *pp, const char *modif)
1386 {
1387 int s;
1388
1389 s = splvm();
1390 if (simple_lock_try(&pp->pr_slock) == 0) {
1391 printf("pool %s is locked; try again later\n",
1392 pp->pr_wchan);
1393 splx(s);
1394 return;
1395 }
1396 pool_print1(pp, modif, printf);
1397 simple_unlock(&pp->pr_slock);
1398 splx(s);
1399 }
1400
1401 void
1402 pool_printit(struct pool *pp, const char *modif, void (*pr)(const char *, ...))
1403 {
1404 int didlock = 0;
1405
1406 if (pp == NULL) {
1407 (*pr)("Must specify a pool to print.\n");
1408 return;
1409 }
1410
1411 /*
1412 * Called from DDB; interrupts should be blocked, and all
1413 * other processors should be paused. We can skip locking
1414 * the pool in this case.
1415 *
1416 * We do a simple_lock_try() just to print the lock
1417 * status, however.
1418 */
1419
1420 if (simple_lock_try(&pp->pr_slock) == 0)
1421 (*pr)("WARNING: pool %s is locked\n", pp->pr_wchan);
1422 else
1423 didlock = 1;
1424
1425 pool_print1(pp, modif, pr);
1426
1427 if (didlock)
1428 simple_unlock(&pp->pr_slock);
1429 }
1430
1431 static void
1432 pool_print_pagelist(struct pool_pagelist *pl, void (*pr)(const char *, ...))
1433 {
1434 struct pool_item_header *ph;
1435 #ifdef DIAGNOSTIC
1436 struct pool_item *pi;
1437 #endif
1438
1439 LIST_FOREACH(ph, pl, ph_pagelist) {
1440 (*pr)("\t\tpage %p, nmissing %d, time %lu,%lu\n",
1441 ph->ph_page, ph->ph_nmissing,
1442 (u_long)ph->ph_time.tv_sec,
1443 (u_long)ph->ph_time.tv_usec);
1444 #ifdef DIAGNOSTIC
1445 TAILQ_FOREACH(pi, &ph->ph_itemlist, pi_list) {
1446 if (pi->pi_magic != PI_MAGIC) {
1447 (*pr)("\t\t\titem %p, magic 0x%x\n",
1448 pi, pi->pi_magic);
1449 }
1450 }
1451 #endif
1452 }
1453 }
1454
1455 static void
1456 pool_print1(struct pool *pp, const char *modif, void (*pr)(const char *, ...))
1457 {
1458 struct pool_item_header *ph;
1459 struct pool_cache *pc;
1460 struct pool_cache_group *pcg;
1461 int i, print_log = 0, print_pagelist = 0, print_cache = 0;
1462 char c;
1463
1464 while ((c = *modif++) != '\0') {
1465 if (c == 'l')
1466 print_log = 1;
1467 if (c == 'p')
1468 print_pagelist = 1;
1469 if (c == 'c')
1470 print_cache = 1;
1471 }
1472
1473 (*pr)("POOL %s: size %u, align %u, ioff %u, roflags 0x%08x\n",
1474 pp->pr_wchan, pp->pr_size, pp->pr_align, pp->pr_itemoffset,
1475 pp->pr_roflags);
1476 (*pr)("\talloc %p\n", pp->pr_alloc);
1477 (*pr)("\tminitems %u, minpages %u, maxpages %u, npages %u\n",
1478 pp->pr_minitems, pp->pr_minpages, pp->pr_maxpages, pp->pr_npages);
1479 (*pr)("\titemsperpage %u, nitems %u, nout %u, hardlimit %u\n",
1480 pp->pr_itemsperpage, pp->pr_nitems, pp->pr_nout, pp->pr_hardlimit);
1481
1482 (*pr)("\n\tnget %lu, nfail %lu, nput %lu\n",
1483 pp->pr_nget, pp->pr_nfail, pp->pr_nput);
1484 (*pr)("\tnpagealloc %lu, npagefree %lu, hiwat %u, nidle %lu\n",
1485 pp->pr_npagealloc, pp->pr_npagefree, pp->pr_hiwat, pp->pr_nidle);
1486
1487 if (print_pagelist == 0)
1488 goto skip_pagelist;
1489
1490 if ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL)
1491 (*pr)("\n\tempty page list:\n");
1492 pool_print_pagelist(&pp->pr_emptypages, pr);
1493 if ((ph = LIST_FIRST(&pp->pr_fullpages)) != NULL)
1494 (*pr)("\n\tfull page list:\n");
1495 pool_print_pagelist(&pp->pr_fullpages, pr);
1496 if ((ph = LIST_FIRST(&pp->pr_partpages)) != NULL)
1497 (*pr)("\n\tpartial-page list:\n");
1498 pool_print_pagelist(&pp->pr_partpages, pr);
1499
1500 if (pp->pr_curpage == NULL)
1501 (*pr)("\tno current page\n");
1502 else
1503 (*pr)("\tcurpage %p\n", pp->pr_curpage->ph_page);
1504
1505 skip_pagelist:
1506 if (print_log == 0)
1507 goto skip_log;
1508
1509 (*pr)("\n");
1510 if ((pp->pr_roflags & PR_LOGGING) == 0)
1511 (*pr)("\tno log\n");
1512 else
1513 pr_printlog(pp, NULL, pr);
1514
1515 skip_log:
1516 if (print_cache == 0)
1517 goto skip_cache;
1518
1519 TAILQ_FOREACH(pc, &pp->pr_cachelist, pc_poollist) {
1520 (*pr)("\tcache %p: allocfrom %p freeto %p\n", pc,
1521 pc->pc_allocfrom, pc->pc_freeto);
1522 (*pr)("\t hits %lu misses %lu ngroups %lu nitems %lu\n",
1523 pc->pc_hits, pc->pc_misses, pc->pc_ngroups, pc->pc_nitems);
1524 TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) {
1525 (*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail);
1526 for (i = 0; i < PCG_NOBJECTS; i++) {
1527 if (pcg->pcg_objects[i].pcgo_pa !=
1528 POOL_PADDR_INVALID) {
1529 (*pr)("\t\t\t%p, 0x%llx\n",
1530 pcg->pcg_objects[i].pcgo_va,
1531 (unsigned long long)
1532 pcg->pcg_objects[i].pcgo_pa);
1533 } else {
1534 (*pr)("\t\t\t%p\n",
1535 pcg->pcg_objects[i].pcgo_va);
1536 }
1537 }
1538 }
1539 }
1540
1541 skip_cache:
1542 pr_enter_check(pp, pr);
1543 }
1544
1545 static int
1546 pool_chk_page(struct pool *pp, const char *label, struct pool_item_header *ph)
1547 {
1548 struct pool_item *pi;
1549 caddr_t page;
1550 int n;
1551
1552 page = (caddr_t)((u_long)ph & pp->pr_alloc->pa_pagemask);
1553 if (page != ph->ph_page &&
1554 (pp->pr_roflags & PR_PHINPAGE) != 0) {
1555 if (label != NULL)
1556 printf("%s: ", label);
1557 printf("pool(%p:%s): page inconsistency: page %p;"
1558 " at page head addr %p (p %p)\n", pp,
1559 pp->pr_wchan, ph->ph_page,
1560 ph, page);
1561 return 1;
1562 }
1563
1564 for (pi = TAILQ_FIRST(&ph->ph_itemlist), n = 0;
1565 pi != NULL;
1566 pi = TAILQ_NEXT(pi,pi_list), n++) {
1567
1568 #ifdef DIAGNOSTIC
1569 if (pi->pi_magic != PI_MAGIC) {
1570 if (label != NULL)
1571 printf("%s: ", label);
1572 printf("pool(%s): free list modified: magic=%x;"
1573 " page %p; item ordinal %d;"
1574 " addr %p (p %p)\n",
1575 pp->pr_wchan, pi->pi_magic, ph->ph_page,
1576 n, pi, page);
1577 panic("pool");
1578 }
1579 #endif
1580 page =
1581 (caddr_t)((u_long)pi & pp->pr_alloc->pa_pagemask);
1582 if (page == ph->ph_page)
1583 continue;
1584
1585 if (label != NULL)
1586 printf("%s: ", label);
1587 printf("pool(%p:%s): page inconsistency: page %p;"
1588 " item ordinal %d; addr %p (p %p)\n", pp,
1589 pp->pr_wchan, ph->ph_page,
1590 n, pi, page);
1591 return 1;
1592 }
1593 return 0;
1594 }
1595
1596
1597 int
1598 pool_chk(struct pool *pp, const char *label)
1599 {
1600 struct pool_item_header *ph;
1601 int r = 0;
1602
1603 simple_lock(&pp->pr_slock);
1604 LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) {
1605 r = pool_chk_page(pp, label, ph);
1606 if (r) {
1607 goto out;
1608 }
1609 }
1610 LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) {
1611 r = pool_chk_page(pp, label, ph);
1612 if (r) {
1613 goto out;
1614 }
1615 }
1616 LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) {
1617 r = pool_chk_page(pp, label, ph);
1618 if (r) {
1619 goto out;
1620 }
1621 }
1622
1623 out:
1624 simple_unlock(&pp->pr_slock);
1625 return (r);
1626 }
1627
1628 /*
1629 * pool_cache_init:
1630 *
1631 * Initialize a pool cache.
1632 *
1633 * NOTE: If the pool must be protected from interrupts, we expect
1634 * to be called at the appropriate interrupt priority level.
1635 */
1636 void
1637 pool_cache_init(struct pool_cache *pc, struct pool *pp,
1638 int (*ctor)(void *, void *, int),
1639 void (*dtor)(void *, void *),
1640 void *arg)
1641 {
1642
1643 TAILQ_INIT(&pc->pc_grouplist);
1644 simple_lock_init(&pc->pc_slock);
1645
1646 pc->pc_allocfrom = NULL;
1647 pc->pc_freeto = NULL;
1648 pc->pc_pool = pp;
1649
1650 pc->pc_ctor = ctor;
1651 pc->pc_dtor = dtor;
1652 pc->pc_arg = arg;
1653
1654 pc->pc_hits = 0;
1655 pc->pc_misses = 0;
1656
1657 pc->pc_ngroups = 0;
1658
1659 pc->pc_nitems = 0;
1660
1661 simple_lock(&pp->pr_slock);
1662 TAILQ_INSERT_TAIL(&pp->pr_cachelist, pc, pc_poollist);
1663 simple_unlock(&pp->pr_slock);
1664 }
1665
1666 /*
1667 * pool_cache_destroy:
1668 *
1669 * Destroy a pool cache.
1670 */
1671 void
1672 pool_cache_destroy(struct pool_cache *pc)
1673 {
1674 struct pool *pp = pc->pc_pool;
1675
1676 /* First, invalidate the entire cache. */
1677 pool_cache_invalidate(pc);
1678
1679 /* ...and remove it from the pool's cache list. */
1680 simple_lock(&pp->pr_slock);
1681 TAILQ_REMOVE(&pp->pr_cachelist, pc, pc_poollist);
1682 simple_unlock(&pp->pr_slock);
1683 }
1684
1685 static __inline void *
1686 pcg_get(struct pool_cache_group *pcg, paddr_t *pap)
1687 {
1688 void *object;
1689 u_int idx;
1690
1691 KASSERT(pcg->pcg_avail <= PCG_NOBJECTS);
1692 KASSERT(pcg->pcg_avail != 0);
1693 idx = --pcg->pcg_avail;
1694
1695 KASSERT(pcg->pcg_objects[idx].pcgo_va != NULL);
1696 object = pcg->pcg_objects[idx].pcgo_va;
1697 if (pap != NULL)
1698 *pap = pcg->pcg_objects[idx].pcgo_pa;
1699 pcg->pcg_objects[idx].pcgo_va = NULL;
1700
1701 return (object);
1702 }
1703
1704 static __inline void
1705 pcg_put(struct pool_cache_group *pcg, void *object, paddr_t pa)
1706 {
1707 u_int idx;
1708
1709 KASSERT(pcg->pcg_avail < PCG_NOBJECTS);
1710 idx = pcg->pcg_avail++;
1711
1712 KASSERT(pcg->pcg_objects[idx].pcgo_va == NULL);
1713 pcg->pcg_objects[idx].pcgo_va = object;
1714 pcg->pcg_objects[idx].pcgo_pa = pa;
1715 }
1716
1717 /*
1718 * pool_cache_get{,_paddr}:
1719 *
1720 * Get an object from a pool cache (optionally returning
1721 * the physical address of the object).
1722 */
1723 void *
1724 pool_cache_get_paddr(struct pool_cache *pc, int flags, paddr_t *pap)
1725 {
1726 struct pool_cache_group *pcg;
1727 void *object;
1728
1729 #ifdef LOCKDEBUG
1730 if (flags & PR_WAITOK)
1731 simple_lock_only_held(NULL, "pool_cache_get(PR_WAITOK)");
1732 #endif
1733
1734 simple_lock(&pc->pc_slock);
1735
1736 if ((pcg = pc->pc_allocfrom) == NULL) {
1737 TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) {
1738 if (pcg->pcg_avail != 0) {
1739 pc->pc_allocfrom = pcg;
1740 goto have_group;
1741 }
1742 }
1743
1744 /*
1745 * No groups with any available objects. Allocate
1746 * a new object, construct it, and return it to
1747 * the caller. We will allocate a group, if necessary,
1748 * when the object is freed back to the cache.
1749 */
1750 pc->pc_misses++;
1751 simple_unlock(&pc->pc_slock);
1752 object = pool_get(pc->pc_pool, flags);
1753 if (object != NULL && pc->pc_ctor != NULL) {
1754 if ((*pc->pc_ctor)(pc->pc_arg, object, flags) != 0) {
1755 pool_put(pc->pc_pool, object);
1756 return (NULL);
1757 }
1758 }
1759 if (object != NULL && pap != NULL) {
1760 #ifdef POOL_VTOPHYS
1761 *pap = POOL_VTOPHYS(object);
1762 #else
1763 *pap = POOL_PADDR_INVALID;
1764 #endif
1765 }
1766 return (object);
1767 }
1768
1769 have_group:
1770 pc->pc_hits++;
1771 pc->pc_nitems--;
1772 object = pcg_get(pcg, pap);
1773
1774 if (pcg->pcg_avail == 0)
1775 pc->pc_allocfrom = NULL;
1776
1777 simple_unlock(&pc->pc_slock);
1778
1779 return (object);
1780 }
1781
1782 /*
1783 * pool_cache_put{,_paddr}:
1784 *
1785 * Put an object back to the pool cache (optionally caching the
1786 * physical address of the object).
1787 */
1788 void
1789 pool_cache_put_paddr(struct pool_cache *pc, void *object, paddr_t pa)
1790 {
1791 struct pool_cache_group *pcg;
1792 int s;
1793
1794 simple_lock(&pc->pc_slock);
1795
1796 if ((pcg = pc->pc_freeto) == NULL) {
1797 TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) {
1798 if (pcg->pcg_avail != PCG_NOBJECTS) {
1799 pc->pc_freeto = pcg;
1800 goto have_group;
1801 }
1802 }
1803
1804 /*
1805 * No empty groups to free the object to. Attempt to
1806 * allocate one.
1807 */
1808 simple_unlock(&pc->pc_slock);
1809 s = splvm();
1810 pcg = pool_get(&pcgpool, PR_NOWAIT);
1811 splx(s);
1812 if (pcg != NULL) {
1813 memset(pcg, 0, sizeof(*pcg));
1814 simple_lock(&pc->pc_slock);
1815 pc->pc_ngroups++;
1816 TAILQ_INSERT_TAIL(&pc->pc_grouplist, pcg, pcg_list);
1817 if (pc->pc_freeto == NULL)
1818 pc->pc_freeto = pcg;
1819 goto have_group;
1820 }
1821
1822 /*
1823 * Unable to allocate a cache group; destruct the object
1824 * and free it back to the pool.
1825 */
1826 pool_cache_destruct_object(pc, object);
1827 return;
1828 }
1829
1830 have_group:
1831 pc->pc_nitems++;
1832 pcg_put(pcg, object, pa);
1833
1834 if (pcg->pcg_avail == PCG_NOBJECTS)
1835 pc->pc_freeto = NULL;
1836
1837 simple_unlock(&pc->pc_slock);
1838 }
1839
1840 /*
1841 * pool_cache_destruct_object:
1842 *
1843 * Force destruction of an object and its release back into
1844 * the pool.
1845 */
1846 void
1847 pool_cache_destruct_object(struct pool_cache *pc, void *object)
1848 {
1849
1850 if (pc->pc_dtor != NULL)
1851 (*pc->pc_dtor)(pc->pc_arg, object);
1852 pool_put(pc->pc_pool, object);
1853 }
1854
1855 /*
1856 * pool_cache_do_invalidate:
1857 *
1858 * This internal function implements pool_cache_invalidate() and
1859 * pool_cache_reclaim().
1860 */
1861 static void
1862 pool_cache_do_invalidate(struct pool_cache *pc, int free_groups,
1863 void (*putit)(struct pool *, void *))
1864 {
1865 struct pool_cache_group *pcg, *npcg;
1866 void *object;
1867 int s;
1868
1869 for (pcg = TAILQ_FIRST(&pc->pc_grouplist); pcg != NULL;
1870 pcg = npcg) {
1871 npcg = TAILQ_NEXT(pcg, pcg_list);
1872 while (pcg->pcg_avail != 0) {
1873 pc->pc_nitems--;
1874 object = pcg_get(pcg, NULL);
1875 if (pcg->pcg_avail == 0 && pc->pc_allocfrom == pcg)
1876 pc->pc_allocfrom = NULL;
1877 if (pc->pc_dtor != NULL)
1878 (*pc->pc_dtor)(pc->pc_arg, object);
1879 (*putit)(pc->pc_pool, object);
1880 }
1881 if (free_groups) {
1882 pc->pc_ngroups--;
1883 TAILQ_REMOVE(&pc->pc_grouplist, pcg, pcg_list);
1884 if (pc->pc_freeto == pcg)
1885 pc->pc_freeto = NULL;
1886 s = splvm();
1887 pool_put(&pcgpool, pcg);
1888 splx(s);
1889 }
1890 }
1891 }
1892
1893 /*
1894 * pool_cache_invalidate:
1895 *
1896 * Invalidate a pool cache (destruct and release all of the
1897 * cached objects).
1898 */
1899 void
1900 pool_cache_invalidate(struct pool_cache *pc)
1901 {
1902
1903 simple_lock(&pc->pc_slock);
1904 pool_cache_do_invalidate(pc, 0, pool_put);
1905 simple_unlock(&pc->pc_slock);
1906 }
1907
1908 /*
1909 * pool_cache_reclaim:
1910 *
1911 * Reclaim a pool cache for pool_reclaim().
1912 */
1913 static void
1914 pool_cache_reclaim(struct pool_cache *pc)
1915 {
1916
1917 simple_lock(&pc->pc_slock);
1918 pool_cache_do_invalidate(pc, 1, pool_do_put);
1919 simple_unlock(&pc->pc_slock);
1920 }
1921
1922 /*
1923 * Pool backend allocators.
1924 *
1925 * Each pool has a backend allocator that handles allocation, deallocation,
1926 * and any additional draining that might be needed.
1927 *
1928 * We provide two standard allocators:
1929 *
1930 * pool_allocator_kmem - the default when no allocator is specified
1931 *
1932 * pool_allocator_nointr - used for pools that will not be accessed
1933 * in interrupt context.
1934 */
1935 void *pool_page_alloc(struct pool *, int);
1936 void pool_page_free(struct pool *, void *);
1937
1938 struct pool_allocator pool_allocator_kmem = {
1939 pool_page_alloc, pool_page_free, 0,
1940 };
1941
1942 void *pool_page_alloc_nointr(struct pool *, int);
1943 void pool_page_free_nointr(struct pool *, void *);
1944
1945 struct pool_allocator pool_allocator_nointr = {
1946 pool_page_alloc_nointr, pool_page_free_nointr, 0,
1947 };
1948
1949 #ifdef POOL_SUBPAGE
1950 void *pool_subpage_alloc(struct pool *, int);
1951 void pool_subpage_free(struct pool *, void *);
1952
1953 struct pool_allocator pool_allocator_kmem_subpage = {
1954 pool_subpage_alloc, pool_subpage_free, 0,
1955 };
1956 #endif /* POOL_SUBPAGE */
1957
1958 /*
1959 * We have at least three different resources for the same allocation and
1960 * each resource can be depleted. First, we have the ready elements in the
1961 * pool. Then we have the resource (typically a vm_map) for this allocator.
1962 * Finally, we have physical memory. Waiting for any of these can be
1963 * unnecessary when any other is freed, but the kernel doesn't support
1964 * sleeping on multiple wait channels, so we have to employ another strategy.
1965 *
1966 * The caller sleeps on the pool (so that it can be awakened when an item
1967 * is returned to the pool), but we set PA_WANT on the allocator. When a
1968 * page is returned to the allocator and PA_WANT is set, pool_allocator_free
1969 * will wake up all sleeping pools belonging to this allocator.
1970 *
1971 * XXX Thundering herd.
1972 */
1973 void *
1974 pool_allocator_alloc(struct pool *org, int flags)
1975 {
1976 struct pool_allocator *pa = org->pr_alloc;
1977 struct pool *pp, *start;
1978 int s, freed;
1979 void *res;
1980
1981 LOCK_ASSERT(!simple_lock_held(&org->pr_slock));
1982
1983 do {
1984 if ((res = (*pa->pa_alloc)(org, flags)) != NULL)
1985 return (res);
1986 if ((flags & PR_WAITOK) == 0) {
1987 /*
1988 * We only run the drain hookhere if PR_NOWAIT.
1989 * In other cases, the hook will be run in
1990 * pool_reclaim().
1991 */
1992 if (org->pr_drain_hook != NULL) {
1993 (*org->pr_drain_hook)(org->pr_drain_hook_arg,
1994 flags);
1995 if ((res = (*pa->pa_alloc)(org, flags)) != NULL)
1996 return (res);
1997 }
1998 break;
1999 }
2000
2001 /*
2002 * Drain all pools, except "org", that use this
2003 * allocator. We do this to reclaim VA space.
2004 * pa_alloc is responsible for waiting for
2005 * physical memory.
2006 *
2007 * XXX We risk looping forever if start if someone
2008 * calls pool_destroy on "start". But there is no
2009 * other way to have potentially sleeping pool_reclaim,
2010 * non-sleeping locks on pool_allocator, and some
2011 * stirring of drained pools in the allocator.
2012 *
2013 * XXX Maybe we should use pool_head_slock for locking
2014 * the allocators?
2015 */
2016 freed = 0;
2017
2018 s = splvm();
2019 simple_lock(&pa->pa_slock);
2020 pp = start = TAILQ_FIRST(&pa->pa_list);
2021 do {
2022 TAILQ_REMOVE(&pa->pa_list, pp, pr_alloc_list);
2023 TAILQ_INSERT_TAIL(&pa->pa_list, pp, pr_alloc_list);
2024 if (pp == org)
2025 continue;
2026 simple_unlock(&pa->pa_slock);
2027 freed = pool_reclaim(pp);
2028 simple_lock(&pa->pa_slock);
2029 } while ((pp = TAILQ_FIRST(&pa->pa_list)) != start &&
2030 freed == 0);
2031
2032 if (freed == 0) {
2033 /*
2034 * We set PA_WANT here, the caller will most likely
2035 * sleep waiting for pages (if not, this won't hurt
2036 * that much), and there is no way to set this in
2037 * the caller without violating locking order.
2038 */
2039 pa->pa_flags |= PA_WANT;
2040 }
2041 simple_unlock(&pa->pa_slock);
2042 splx(s);
2043 } while (freed);
2044 return (NULL);
2045 }
2046
2047 void
2048 pool_allocator_free(struct pool *pp, void *v)
2049 {
2050 struct pool_allocator *pa = pp->pr_alloc;
2051 int s;
2052
2053 LOCK_ASSERT(!simple_lock_held(&pp->pr_slock));
2054
2055 (*pa->pa_free)(pp, v);
2056
2057 s = splvm();
2058 simple_lock(&pa->pa_slock);
2059 if ((pa->pa_flags & PA_WANT) == 0) {
2060 simple_unlock(&pa->pa_slock);
2061 splx(s);
2062 return;
2063 }
2064
2065 TAILQ_FOREACH(pp, &pa->pa_list, pr_alloc_list) {
2066 simple_lock(&pp->pr_slock);
2067 if ((pp->pr_flags & PR_WANTED) != 0) {
2068 pp->pr_flags &= ~PR_WANTED;
2069 wakeup(pp);
2070 }
2071 simple_unlock(&pp->pr_slock);
2072 }
2073 pa->pa_flags &= ~PA_WANT;
2074 simple_unlock(&pa->pa_slock);
2075 splx(s);
2076 }
2077
2078 void *
2079 pool_page_alloc(struct pool *pp, int flags)
2080 {
2081 boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE;
2082
2083 return ((void *) uvm_km_alloc_poolpage(waitok));
2084 }
2085
2086 void
2087 pool_page_free(struct pool *pp, void *v)
2088 {
2089
2090 uvm_km_free_poolpage((vaddr_t) v);
2091 }
2092
2093 #ifdef POOL_SUBPAGE
2094 /* Sub-page allocator, for machines with large hardware pages. */
2095 void *
2096 pool_subpage_alloc(struct pool *pp, int flags)
2097 {
2098 void *v;
2099 int s;
2100 s = splvm();
2101 v = pool_get(&psppool, flags);
2102 splx(s);
2103 return v;
2104 }
2105
2106 void
2107 pool_subpage_free(struct pool *pp, void *v)
2108 {
2109 int s;
2110 s = splvm();
2111 pool_put(&psppool, v);
2112 splx(s);
2113 }
2114
2115 /* We don't provide a real nointr allocator. Maybe later. */
2116 void *
2117 pool_page_alloc_nointr(struct pool *pp, int flags)
2118 {
2119
2120 return (pool_subpage_alloc(pp, flags));
2121 }
2122
2123 void
2124 pool_page_free_nointr(struct pool *pp, void *v)
2125 {
2126
2127 pool_subpage_free(pp, v);
2128 }
2129 #else
2130 void *
2131 pool_page_alloc_nointr(struct pool *pp, int flags)
2132 {
2133 boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE;
2134
2135 return ((void *) uvm_km_alloc_poolpage1(kernel_map,
2136 uvm.kernel_object, waitok));
2137 }
2138
2139 void
2140 pool_page_free_nointr(struct pool *pp, void *v)
2141 {
2142
2143 uvm_km_free_poolpage1(kernel_map, (vaddr_t) v);
2144 }
2145 #endif /* POOL_SUBPAGE */
Cache object: 2e6558ec5a2f311853f144108b1f864f
|