FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_mpipe.c
1 /*
2 * (MPSAFE)
3 *
4 * Copyright (c) 2003,2004 The DragonFly Project. All rights reserved.
5 *
6 * This code is derived from software contributed to The DragonFly Project
7 * by Matthew Dillon <dillon@backplane.com>
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 *
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in
17 * the documentation and/or other materials provided with the
18 * distribution.
19 * 3. Neither the name of The DragonFly Project nor the names of its
20 * contributors may be used to endorse or promote products derived
21 * from this software without specific, prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
26 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
27 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
28 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
29 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
30 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
31 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
32 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
33 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 */
36
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/kernel.h>
40 #include <sys/slaballoc.h>
41 #include <sys/mbuf.h>
42 #include <sys/vmmeter.h>
43 #include <sys/lock.h>
44 #include <sys/thread.h>
45 #include <sys/globaldata.h>
46 #include <sys/mpipe.h>
47 #include <sys/kthread.h>
48
49 #include <sys/thread2.h>
50
51 struct mpipe_callback {
52 STAILQ_ENTRY(mpipe_callback) entry;
53 void (*func)(void *arg1, void *arg2);
54 void *arg1;
55 void *arg2;
56 };
57
58 static MALLOC_DEFINE(M_MPIPEARY, "MPipe Array", "Auxiliary MPIPE structure");
59
60 static void mpipe_thread(void *arg);
61
62 /*
63 * Initialize a malloc pipeline for the specified malloc type and allocation
64 * size. Create an array to cache up to nom_count buffers and preallocate
65 * them.
66 */
67 void
68 mpipe_init(malloc_pipe_t mpipe, malloc_type_t type, int bytes,
69 int nnom, int nmax,
70 int mpflags,
71 void (*construct)(void *, void *),
72 void (*deconstruct)(void *, void *),
73 void *priv)
74 {
75 int n;
76
77 if (nnom < 1)
78 nnom = 1;
79 if (nmax < 0)
80 nmax = 0x7FFF0000; /* some very large number */
81 if (nmax < nnom)
82 nmax = nnom;
83 bzero(mpipe, sizeof(struct malloc_pipe));
84 mpipe->type = type;
85 mpipe->bytes = bytes;
86 mpipe->mpflags = mpflags;
87 mpipe->construct = construct;
88 mpipe->deconstruct = deconstruct;
89 mpipe->priv = priv;
90 if ((mpflags & MPF_NOZERO) == 0)
91 mpipe->mflags |= M_ZERO;
92 if (mpflags & MPF_INT)
93 mpipe->mflags |= M_USE_RESERVE | M_USE_INTERRUPT_RESERVE;
94 mpipe->ary_count = nnom;
95 mpipe->max_count = nmax;
96 mpipe->array = kmalloc(nnom * sizeof(mpipe->array[0]), M_MPIPEARY,
97 M_WAITOK | M_ZERO);
98
99 while (mpipe->free_count < nnom) {
100 n = mpipe->free_count;
101 mpipe->array[n] = kmalloc(bytes, mpipe->type, M_WAITOK | mpipe->mflags);
102 if (construct)
103 construct(mpipe->array[n], priv);
104 ++mpipe->free_count;
105 ++mpipe->total_count;
106 }
107 STAILQ_INIT(&mpipe->queue);
108
109 lwkt_token_init(&mpipe->token, "mpipe token");
110
111 /*
112 * Create a support thread for the mpipe queue
113 */
114 if (mpflags & MPF_CALLBACK) {
115 kthread_create(mpipe_thread, mpipe, &mpipe->thread,
116 "mpipe_%s", type->ks_shortdesc);
117 }
118 }
119
120 /*
121 * Destroy a previously initialized mpipe. This routine can also safely be
122 * called on an uninitialized mpipe structure if it was zero'd or mpipe_done()
123 * was previously called on it.
124 */
125 void
126 mpipe_done(malloc_pipe_t mpipe)
127 {
128 void *buf;
129 int n;
130
131 KKASSERT(mpipe->free_count == mpipe->total_count); /* no outstanding mem */
132
133 /*
134 * Clean up the kthread
135 */
136 lwkt_gettoken(&mpipe->token);
137 mpipe->mpflags |= MPF_EXITING;
138 while (mpipe->thread) {
139 wakeup(&mpipe->queue);
140 tsleep(mpipe, 0, "mpipex", 1);
141 }
142
143 /*
144 * Clean up the mpipe buffers
145 */
146 for (n = mpipe->free_count - 1; n >= 0; --n) {
147 buf = mpipe->array[n];
148 mpipe->array[n] = NULL;
149 KKASSERT(buf != NULL);
150 if (mpipe->deconstruct)
151 mpipe->deconstruct(buf, mpipe->priv);
152 kfree(buf, mpipe->type);
153 }
154 mpipe->free_count = 0;
155 mpipe->total_count = 0;
156 if (mpipe->array) {
157 kfree(mpipe->array, M_MPIPEARY);
158 mpipe->array = NULL;
159 }
160 lwkt_reltoken(&mpipe->token);
161 lwkt_token_uninit(&mpipe->token);
162 }
163
164 /*
165 * mpipe support thread for request failures when mpipe_alloc_callback()
166 * is called.
167 */
168 static void
169 mpipe_thread(void *arg)
170 {
171 malloc_pipe_t mpipe = arg;
172 struct mpipe_callback *mcb;
173
174 lwkt_gettoken(&mpipe->token);
175 while ((mpipe->mpflags & MPF_EXITING) == 0) {
176 while (mpipe->free_count &&
177 (mcb = STAILQ_FIRST(&mpipe->queue)) != NULL) {
178 STAILQ_REMOVE(&mpipe->queue, mcb, mpipe_callback, entry);
179 mcb->func(mcb->arg1, mcb->arg2);
180 kfree(mcb, M_MPIPEARY);
181 }
182 mpipe->mpflags |= MPF_QUEUEWAIT;
183 tsleep(&mpipe->queue, 0, "wait", 0);
184 }
185 mpipe->thread = NULL;
186 wakeup(mpipe);
187 lwkt_reltoken(&mpipe->token);
188 }
189
190
191 /*
192 * Allocate an entry (inline suppot routine). The allocation is guarenteed
193 * to return non-NULL up to the nominal count after which it may return NULL.
194 * Note that the implementation is defined to be allowed to block for short
195 * periods of time.
196 *
197 * Use mpipe_alloc_callback() for non-blocking operation with a callback
198 * Use mpipe_alloc_nowait() for non-blocking operation without a callback
199 * Use mpipe_alloc_waitok() for blocking operation & guarenteed non-NULL
200 */
201 static __inline
202 void *
203 _mpipe_alloc_locked(malloc_pipe_t mpipe, int mfailed)
204 {
205 void *buf;
206 int n;
207
208 if ((n = mpipe->free_count) != 0) {
209 /*
210 * Use a free entry if it exists.
211 */
212 --n;
213 buf = mpipe->array[n];
214 mpipe->array[n] = NULL; /* sanity check, not absolutely needed */
215 mpipe->free_count = n;
216 } else if (mpipe->total_count >= mpipe->max_count || mfailed) {
217 /*
218 * Return NULL if we have hit our limit
219 */
220 buf = NULL;
221 } else {
222 /*
223 * Otherwise try to malloc() non-blocking.
224 */
225 buf = kmalloc(mpipe->bytes, mpipe->type, M_NOWAIT | mpipe->mflags);
226 if (buf) {
227 ++mpipe->total_count;
228 if (mpipe->construct)
229 mpipe->construct(buf, mpipe->priv);
230 }
231 }
232 return(buf);
233 }
234
235 /*
236 * Nominal non-blocking mpipe allocation
237 */
238 void *
239 mpipe_alloc_nowait(malloc_pipe_t mpipe)
240 {
241 void *buf;
242
243 lwkt_gettoken(&mpipe->token);
244 buf = _mpipe_alloc_locked(mpipe, 0);
245 lwkt_reltoken(&mpipe->token);
246
247 return(buf);
248 }
249
250 /*
251 * non-blocking mpipe allocation with callback for retry.
252 *
253 * If NULL is returned func(arg) is queued and will be called back when
254 * space is likely (but not necessarily) available.
255 *
256 * If non-NULL is returned func(arg) is ignored.
257 */
258 void *
259 mpipe_alloc_callback(malloc_pipe_t mpipe, void (*func)(void *arg1, void *arg2),
260 void *arg1, void *arg2)
261 {
262 struct mpipe_callback *mcb;
263 void *buf;
264
265 lwkt_gettoken(&mpipe->token);
266 buf = _mpipe_alloc_locked(mpipe, 0);
267 if (buf == NULL) {
268 mcb = kmalloc(sizeof(*mcb), M_MPIPEARY, M_INTWAIT);
269 buf = _mpipe_alloc_locked(mpipe, 0);
270 if (buf == NULL) {
271 mcb->func = func;
272 mcb->arg1 = arg1;
273 mcb->arg2 = arg2;
274 STAILQ_INSERT_TAIL(&mpipe->queue, mcb, entry);
275 } else {
276 kfree(mcb, M_MPIPEARY);
277 }
278 }
279 lwkt_reltoken(&mpipe->token);
280
281 return(buf);
282 }
283
284 /*
285 * This function can be called to nominally wait until resources are
286 * available and mpipe_alloc_nowait() is likely to return non-NULL.
287 *
288 * NOTE: mpipe_alloc_nowait() can still return NULL.
289 */
290 void
291 mpipe_wait(malloc_pipe_t mpipe)
292 {
293 if (mpipe->free_count == 0) {
294 lwkt_gettoken(&mpipe->token);
295 while ((mpipe->mpflags & MPF_EXITING) == 0) {
296 if (mpipe->free_count)
297 break;
298 mpipe->mpflags |= MPF_QUEUEWAIT;
299 tsleep(&mpipe->queue, 0, "wait", 0);
300 }
301 lwkt_reltoken(&mpipe->token);
302 }
303 }
304
305 /*
306 * Allocate an entry, block until the allocation succeeds. This may cause
307 * us to block waiting for a prior allocation to be freed.
308 */
309 void *
310 mpipe_alloc_waitok(malloc_pipe_t mpipe)
311 {
312 void *buf;
313 int mfailed;
314
315 lwkt_gettoken(&mpipe->token);
316 mfailed = 0;
317 while ((buf = _mpipe_alloc_locked(mpipe, mfailed)) == NULL) {
318 /*
319 * Block if we have hit our limit
320 */
321 mpipe->pending = 1;
322 tsleep(mpipe, 0, "mpipe1", 0);
323 mfailed = 1;
324 }
325 lwkt_reltoken(&mpipe->token);
326
327 return(buf);
328 }
329
330 /*
331 * Free an entry, unblock any waiters. Allow NULL.
332 */
333 void
334 mpipe_free(malloc_pipe_t mpipe, void *buf)
335 {
336 int n;
337
338 if (buf == NULL)
339 return;
340
341 lwkt_gettoken(&mpipe->token);
342 if ((n = mpipe->free_count) < mpipe->ary_count) {
343 /*
344 * Free slot available in free array (LIFO)
345 */
346 mpipe->array[n] = buf;
347 ++mpipe->free_count;
348 if ((mpipe->mpflags & (MPF_CACHEDATA|MPF_NOZERO)) == 0)
349 bzero(buf, mpipe->bytes);
350 if (mpipe->mpflags & MPF_QUEUEWAIT) {
351 mpipe->mpflags &= ~MPF_QUEUEWAIT;
352 lwkt_reltoken(&mpipe->token);
353 wakeup(&mpipe->queue);
354 } else {
355 lwkt_reltoken(&mpipe->token);
356 }
357 /*
358 * Wakeup anyone blocked in mpipe_alloc_*().
359 */
360 if (mpipe->pending) {
361 mpipe->pending = 0;
362 wakeup(mpipe);
363 }
364 } else {
365 /*
366 * All the free slots are full, free the buffer directly.
367 */
368 --mpipe->total_count;
369 KKASSERT(mpipe->total_count >= mpipe->free_count);
370 if (mpipe->deconstruct)
371 mpipe->deconstruct(buf, mpipe->priv);
372 lwkt_reltoken(&mpipe->token);
373 kfree(buf, mpipe->type);
374 }
375 }
376
Cache object: eed69ad7d5998f7c9ed2889d942c846d
|