FreeBSD/Linux Kernel Cross Reference
sys/sys/mutex.h
1 /*-
2 * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * 3. Berkeley Software Design Inc's name may not be used to endorse or
13 * promote products derived from this software without specific prior
14 * written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * from BSDI $Id: mutex.h,v 2.7.2.35 2000/04/27 03:10:26 cp Exp $
29 * $FreeBSD: releng/6.3/sys/sys/mutex.h 173886 2007-11-24 19:45:58Z cvs2svn $
30 */
31
32 #ifndef _SYS_MUTEX_H_
33 #define _SYS_MUTEX_H_
34
35 #ifndef LOCORE
36 #include <sys/queue.h>
37 #include <sys/_lock.h>
38 #include <sys/_mutex.h>
39
40 #ifdef _KERNEL
41 #include <sys/pcpu.h>
42 #include <machine/atomic.h>
43 #include <machine/cpufunc.h>
44 #endif /* _KERNEL_ */
45 #endif /* !LOCORE */
46
47 #include <machine/mutex.h>
48
49 #ifdef _KERNEL
50
51 /*
52 * Mutex types and options passed to mtx_init(). MTX_QUIET and MTX_DUPOK
53 * can also be passed in.
54 */
55 #define MTX_DEF 0x00000000 /* DEFAULT (sleep) lock */
56 #define MTX_SPIN 0x00000001 /* Spin lock (disables interrupts) */
57 #define MTX_RECURSE 0x00000004 /* Option: lock allowed to recurse */
58 #define MTX_NOWITNESS 0x00000008 /* Don't do any witness checking. */
59
60 /*
61 * Option flags passed to certain lock/unlock routines, through the use
62 * of corresponding mtx_{lock,unlock}_flags() interface macros.
63 */
64 #define MTX_QUIET LOP_QUIET /* Don't log a mutex event */
65 #define MTX_DUPOK LOP_DUPOK /* Don't log a duplicate acquire */
66
67 /*
68 * State bits kept in mutex->mtx_lock, for the DEFAULT lock type. None of this,
69 * with the exception of MTX_UNOWNED, applies to spin locks.
70 */
71 #define MTX_RECURSED 0x00000001 /* lock recursed (for MTX_DEF only) */
72 #define MTX_CONTESTED 0x00000002 /* lock contested (for MTX_DEF only) */
73 #define MTX_UNOWNED 0x00000004 /* Cookie for free mutex */
74 #define MTX_FLAGMASK ~(MTX_RECURSED | MTX_CONTESTED)
75
76 /*
77 * Value stored in mutex->mtx_lock to denote a destroyed mutex.
78 */
79 #define MTX_DESTROYED (MTX_CONTESTED | MTX_UNOWNED)
80
81 #endif /* _KERNEL */
82
83 #ifndef LOCORE
84
85 /*
86 * XXX: Friendly reminder to fix things in MP code that is presently being
87 * XXX: worked on.
88 */
89 #define mp_fixme(string)
90
91 #ifdef _KERNEL
92
93 /*
94 * Prototypes
95 *
96 * NOTE: Functions prepended with `_' (underscore) are exported to other parts
97 * of the kernel via macros, thus allowing us to use the cpp LOCK_FILE
98 * and LOCK_LINE. These functions should not be called directly by any
99 * code using the API. Their macros cover their functionality.
100 *
101 * [See below for descriptions]
102 *
103 */
104 void mtx_init(struct mtx *m, const char *name, const char *type, int opts);
105 void mtx_destroy(struct mtx *m);
106 void mtx_sysinit(void *arg);
107 void mutex_init(void);
108 void _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts,
109 const char *file, int line);
110 void _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line);
111 #ifdef SMP
112 void _mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts,
113 const char *file, int line);
114 #endif
115 void _mtx_unlock_spin(struct mtx *m, int opts, const char *file, int line);
116 int _mtx_trylock(struct mtx *m, int opts, const char *file, int line);
117 void _mtx_lock_flags(struct mtx *m, int opts, const char *file, int line);
118 void _mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line);
119 void _mtx_lock_spin_flags(struct mtx *m, int opts, const char *file,
120 int line);
121 void _mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file,
122 int line);
123 #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
124 void _mtx_assert(struct mtx *m, int what, const char *file, int line);
125 #endif
126
127 /*
128 * We define our machine-independent (unoptimized) mutex micro-operations
129 * here, if they are not already defined in the machine-dependent mutex.h
130 */
131
132 /* Try to obtain mtx_lock once. */
133 #ifndef _obtain_lock
134 #define _obtain_lock(mp, tid) \
135 atomic_cmpset_acq_ptr(&(mp)->mtx_lock, MTX_UNOWNED, (tid))
136 #endif
137
138 /* Try to release mtx_lock if it is unrecursed and uncontested. */
139 #ifndef _release_lock
140 #define _release_lock(mp, tid) \
141 atomic_cmpset_rel_ptr(&(mp)->mtx_lock, (tid), MTX_UNOWNED)
142 #endif
143
144 /* Release mtx_lock quickly, assuming we own it. */
145 #ifndef _release_lock_quick
146 #define _release_lock_quick(mp) \
147 atomic_store_rel_ptr(&(mp)->mtx_lock, MTX_UNOWNED)
148 #endif
149
150 /*
151 * Obtain a sleep lock inline, or call the "hard" function if we can't get it
152 * easy.
153 */
154 #ifndef _get_sleep_lock
155 #define _get_sleep_lock(mp, tid, opts, file, line) do { \
156 uintptr_t _tid = (uintptr_t)(tid); \
157 \
158 if (!_obtain_lock((mp), _tid)) \
159 _mtx_lock_sleep((mp), _tid, (opts), (file), (line)); \
160 } while (0)
161 #endif
162
163 /*
164 * Obtain a spin lock inline, or call the "hard" function if we can't get it
165 * easy. For spinlocks, we handle recursion inline (it turns out that function
166 * calls can be significantly expensive on some architectures).
167 * Since spin locks are not _too_ common, inlining this code is not too big
168 * a deal.
169 */
170 #ifndef _get_spin_lock
171 #ifdef SMP
172 #define _get_spin_lock(mp, tid, opts, file, line) do { \
173 uintptr_t _tid = (uintptr_t)(tid); \
174 \
175 spinlock_enter(); \
176 if (!_obtain_lock((mp), _tid)) { \
177 if ((mp)->mtx_lock == _tid) \
178 (mp)->mtx_recurse++; \
179 else \
180 _mtx_lock_spin((mp), _tid, (opts), (file), (line)); \
181 } \
182 } while (0)
183 #else /* SMP */
184 #define _get_spin_lock(mp, tid, opts, file, line) do { \
185 uintptr_t _tid = (uintptr_t)(tid); \
186 \
187 spinlock_enter(); \
188 if ((mp)->mtx_lock == _tid) \
189 (mp)->mtx_recurse++; \
190 else { \
191 KASSERT((mp)->mtx_lock == MTX_UNOWNED, ("corrupt spinlock")); \
192 (mp)->mtx_lock = _tid; \
193 } \
194 } while (0)
195 #endif /* SMP */
196 #endif
197
198 /*
199 * Release a sleep lock inline, or call the "hard" function if we can't do it
200 * easy.
201 */
202 #ifndef _rel_sleep_lock
203 #define _rel_sleep_lock(mp, tid, opts, file, line) do { \
204 uintptr_t _tid = (uintptr_t)(tid); \
205 \
206 if (!_release_lock((mp), _tid)) \
207 _mtx_unlock_sleep((mp), (opts), (file), (line)); \
208 } while (0)
209 #endif
210
211 /*
212 * For spinlocks, we can handle everything inline, as it's pretty simple and
213 * a function call would be too expensive (at least on some architectures).
214 * Since spin locks are not _too_ common, inlining this code is not too big
215 * a deal.
216 *
217 * Since we always perform a spinlock_enter() when attempting to acquire a
218 * spin lock, we need to always perform a matching spinlock_exit() when
219 * releasing a spin lock. This includes the recursion cases.
220 */
221 #ifndef _rel_spin_lock
222 #ifdef SMP
223 #define _rel_spin_lock(mp) do { \
224 if (mtx_recursed((mp))) \
225 (mp)->mtx_recurse--; \
226 else \
227 _release_lock_quick((mp)); \
228 spinlock_exit(); \
229 } while (0)
230 #else /* SMP */
231 #define _rel_spin_lock(mp) do { \
232 if (mtx_recursed((mp))) \
233 (mp)->mtx_recurse--; \
234 else \
235 (mp)->mtx_lock = MTX_UNOWNED; \
236 spinlock_exit(); \
237 } while (0)
238 #endif /* SMP */
239 #endif
240
241 /*
242 * Exported lock manipulation interface.
243 *
244 * mtx_lock(m) locks MTX_DEF mutex `m'
245 *
246 * mtx_lock_spin(m) locks MTX_SPIN mutex `m'
247 *
248 * mtx_unlock(m) unlocks MTX_DEF mutex `m'
249 *
250 * mtx_unlock_spin(m) unlocks MTX_SPIN mutex `m'
251 *
252 * mtx_lock_spin_flags(m, opts) and mtx_lock_flags(m, opts) locks mutex `m'
253 * and passes option flags `opts' to the "hard" function, if required.
254 * With these routines, it is possible to pass flags such as MTX_QUIET
255 * to the appropriate lock manipulation routines.
256 *
257 * mtx_trylock(m) attempts to acquire MTX_DEF mutex `m' but doesn't sleep if
258 * it cannot. Rather, it returns 0 on failure and non-zero on success.
259 * It does NOT handle recursion as we assume that if a caller is properly
260 * using this part of the interface, he will know that the lock in question
261 * is _not_ recursed.
262 *
263 * mtx_trylock_flags(m, opts) is used the same way as mtx_trylock() but accepts
264 * relevant option flags `opts.'
265 *
266 * mtx_initialized(m) returns non-zero if the lock `m' has been initialized.
267 *
268 * mtx_owned(m) returns non-zero if the current thread owns the lock `m'
269 *
270 * mtx_recursed(m) returns non-zero if the lock `m' is presently recursed.
271 */
272 #define mtx_lock(m) mtx_lock_flags((m), 0)
273 #define mtx_lock_spin(m) mtx_lock_spin_flags((m), 0)
274 #define mtx_trylock(m) mtx_trylock_flags((m), 0)
275 #define mtx_unlock(m) mtx_unlock_flags((m), 0)
276 #define mtx_unlock_spin(m) mtx_unlock_spin_flags((m), 0)
277
278 struct mtx_pool;
279
280 struct mtx_pool *mtx_pool_create(const char *mtx_name, int pool_size, int opts);
281 void mtx_pool_destroy(struct mtx_pool **poolp);
282 struct mtx *mtx_pool_find(struct mtx_pool *pool, void *ptr);
283 struct mtx *mtx_pool_alloc(struct mtx_pool *pool);
284 #define mtx_pool_lock(pool, ptr) \
285 mtx_lock(mtx_pool_find((pool), (ptr)))
286 #define mtx_pool_lock_spin(pool, ptr) \
287 mtx_lock_spin(mtx_pool_find((pool), (ptr)))
288 #define mtx_pool_unlock(pool, ptr) \
289 mtx_unlock(mtx_pool_find((pool), (ptr)))
290 #define mtx_pool_unlock_spin(pool, ptr) \
291 mtx_unlock_spin(mtx_pool_find((pool), (ptr)))
292
293 /*
294 * mtxpool_lockbuilder is a pool of sleep locks that is not witness
295 * checked and should only be used for building higher level locks.
296 *
297 * mtxpool_sleep is a general purpose pool of sleep mutexes.
298 */
299 extern struct mtx_pool *mtxpool_lockbuilder;
300 extern struct mtx_pool *mtxpool_sleep;
301
302 #ifndef LOCK_DEBUG
303 #error LOCK_DEBUG not defined, include <sys/lock.h> before <sys/mutex.h>
304 #endif
305 #if LOCK_DEBUG > 0 || defined(MUTEX_NOINLINE)
306 #define mtx_lock_flags(m, opts) \
307 _mtx_lock_flags((m), (opts), LOCK_FILE, LOCK_LINE)
308 #define mtx_unlock_flags(m, opts) \
309 _mtx_unlock_flags((m), (opts), LOCK_FILE, LOCK_LINE)
310 #define mtx_lock_spin_flags(m, opts) \
311 _mtx_lock_spin_flags((m), (opts), LOCK_FILE, LOCK_LINE)
312 #define mtx_unlock_spin_flags(m, opts) \
313 _mtx_unlock_spin_flags((m), (opts), LOCK_FILE, LOCK_LINE)
314 #else /* LOCK_DEBUG == 0 && !MUTEX_NOINLINE */
315 #define mtx_lock_flags(m, opts) \
316 _get_sleep_lock((m), curthread, (opts), LOCK_FILE, LOCK_LINE)
317 #define mtx_unlock_flags(m, opts) \
318 _rel_sleep_lock((m), curthread, (opts), LOCK_FILE, LOCK_LINE)
319 #define mtx_lock_spin_flags(m, opts) \
320 _get_spin_lock((m), curthread, (opts), LOCK_FILE, LOCK_LINE)
321 #define mtx_unlock_spin_flags(m, opts) \
322 _rel_spin_lock((m))
323 #endif /* LOCK_DEBUG > 0 || MUTEX_NOINLINE */
324
325 #define mtx_trylock_flags(m, opts) \
326 _mtx_trylock((m), (opts), LOCK_FILE, LOCK_LINE)
327
328 #define mtx_initialized(m) lock_initalized(&(m)->mtx_object)
329
330 #define mtx_owned(m) (((m)->mtx_lock & MTX_FLAGMASK) == (uintptr_t)curthread)
331
332 #define mtx_recursed(m) ((m)->mtx_recurse != 0)
333
334 #define mtx_name(m) ((m)->mtx_object.lo_name)
335
336 /*
337 * Global locks.
338 */
339 extern struct mtx sched_lock;
340 extern struct mtx Giant;
341
342 /*
343 * Giant lock manipulation and clean exit macros.
344 * Used to replace return with an exit Giant and return.
345 *
346 * Note that DROP_GIANT*() needs to be paired with PICKUP_GIANT()
347 * The #ifndef is to allow lint-like tools to redefine DROP_GIANT.
348 */
349 #ifndef DROP_GIANT
350 #define DROP_GIANT() \
351 do { \
352 int _giantcnt; \
353 WITNESS_SAVE_DECL(Giant); \
354 \
355 if (mtx_owned(&Giant)) \
356 WITNESS_SAVE(&Giant.mtx_object, Giant); \
357 for (_giantcnt = 0; mtx_owned(&Giant); _giantcnt++) \
358 mtx_unlock(&Giant)
359
360 #define PICKUP_GIANT() \
361 mtx_assert(&Giant, MA_NOTOWNED); \
362 while (_giantcnt--) \
363 mtx_lock(&Giant); \
364 if (mtx_owned(&Giant)) \
365 WITNESS_RESTORE(&Giant.mtx_object, Giant); \
366 } while (0)
367
368 #define PARTIAL_PICKUP_GIANT() \
369 mtx_assert(&Giant, MA_NOTOWNED); \
370 while (_giantcnt--) \
371 mtx_lock(&Giant); \
372 if (mtx_owned(&Giant)) \
373 WITNESS_RESTORE(&Giant.mtx_object, Giant)
374 #endif
375
376 /*
377 * Network MPSAFE temporary workarounds. When debug_mpsafenet
378 * is 1 the network is assumed to operate without Giant on the
379 * input path and protocols that require Giant must collect it
380 * on entry. When 0 Giant is grabbed in the network interface
381 * ISR's and in the netisr path and there is no need to grab
382 * the Giant lock. Note that, unlike GIANT_PICKUP() and
383 * GIANT_DROP(), these macros directly wrap mutex operations
384 * without special recursion handling.
385 *
386 * This mechanism is intended as temporary until everything of
387 * importance is properly locked. Note: the semantics for
388 * NET_{LOCK,UNLOCK}_GIANT() are not the same as DROP_GIANT()
389 * and PICKUP_GIANT(), as they are plain mutex operations
390 * without a recursion counter.
391 */
392 extern int debug_mpsafenet; /* defined in net/netisr.c */
393 #define NET_LOCK_GIANT() do { \
394 if (!debug_mpsafenet) \
395 mtx_lock(&Giant); \
396 } while (0)
397 #define NET_UNLOCK_GIANT() do { \
398 if (!debug_mpsafenet) \
399 mtx_unlock(&Giant); \
400 } while (0)
401 #define NET_ASSERT_GIANT() do { \
402 if (!debug_mpsafenet) \
403 mtx_assert(&Giant, MA_OWNED); \
404 } while (0)
405 #define NET_CALLOUT_MPSAFE (debug_mpsafenet ? CALLOUT_MPSAFE : 0)
406
407 #define UGAR(rval) do { \
408 int _val = (rval); \
409 mtx_unlock(&Giant); \
410 return (_val); \
411 } while (0)
412
413 struct mtx_args {
414 struct mtx *ma_mtx;
415 const char *ma_desc;
416 int ma_opts;
417 };
418
419 #define MTX_SYSINIT(name, mtx, desc, opts) \
420 static struct mtx_args name##_args = { \
421 (mtx), \
422 (desc), \
423 (opts) \
424 }; \
425 SYSINIT(name##_mtx_sysinit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \
426 mtx_sysinit, &name##_args); \
427 SYSUNINIT(name##_mtx_sysuninit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \
428 mtx_destroy, (mtx))
429
430 /*
431 * The INVARIANTS-enabled mtx_assert() functionality.
432 *
433 * The constants need to be defined for INVARIANT_SUPPORT infrastructure
434 * support as _mtx_assert() itself uses them and the latter implies that
435 * _mtx_assert() must build.
436 */
437 #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
438 #define MA_OWNED 0x01
439 #define MA_NOTOWNED 0x02
440 #define MA_RECURSED 0x04
441 #define MA_NOTRECURSED 0x08
442 #endif
443
444 #ifdef INVARIANTS
445 #define mtx_assert(m, what) \
446 _mtx_assert((m), (what), __FILE__, __LINE__)
447
448 #define GIANT_REQUIRED mtx_assert(&Giant, MA_OWNED)
449
450 #else /* INVARIANTS */
451 #define mtx_assert(m, what)
452 #define GIANT_REQUIRED
453 #endif /* INVARIANTS */
454
455 /*
456 * Common lock type names.
457 */
458 #define MTX_NETWORK_LOCK "network driver"
459
460 #endif /* _KERNEL */
461 #endif /* !LOCORE */
462 #endif /* _SYS_MUTEX_H_ */
Cache object: c86fd24de82ae8f18ccbbc948ea9d322
|