FreeBSD/Linux Kernel Cross Reference
sys/kern/kern_lock.c
1 /* $NetBSD: kern_lock.c,v 1.146 2008/07/02 14:47:34 matt Exp $ */
2
3 /*-
4 * Copyright (c) 2002, 2006, 2007, 2008 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9 * NASA Ames Research Center, and by Andrew Doran.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 #include <sys/cdefs.h>
34 __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.146 2008/07/02 14:47:34 matt Exp $");
35
36 #include <sys/param.h>
37 #include <sys/proc.h>
38 #include <sys/lock.h>
39 #include <sys/systm.h>
40 #include <sys/kernel.h>
41 #include <sys/lockdebug.h>
42 #include <sys/cpu.h>
43 #include <sys/syslog.h>
44 #include <sys/atomic.h>
45
46 #include <machine/stdarg.h>
47 #include <machine/lock.h>
48
49 #include <dev/lockstat.h>
50
51 #define RETURN_ADDRESS (uintptr_t)__builtin_return_address(0)
52
53 bool kernel_lock_dodebug;
54
55 __cpu_simple_lock_t kernel_lock[CACHE_LINE_SIZE / sizeof(__cpu_simple_lock_t)]
56 __aligned(CACHE_LINE_SIZE);
57
58 #if defined(DEBUG) || defined(LKM)
59 void
60 assert_sleepable(void)
61 {
62 const char *reason;
63
64 if (panicstr != NULL) {
65 return;
66 }
67
68 LOCKDEBUG_BARRIER(kernel_lock, 1);
69
70 reason = NULL;
71 if (CURCPU_IDLE_P() && !cold) {
72 reason = "idle";
73 }
74 if (cpu_intr_p()) {
75 reason = "interrupt";
76 }
77 if ((curlwp->l_pflag & LP_INTR) != 0) {
78 reason = "softint";
79 }
80
81 if (reason) {
82 panic("%s: %s caller=%p", __func__, reason,
83 (void *)RETURN_ADDRESS);
84 }
85 }
86 #endif /* defined(DEBUG) || defined(LKM) */
87
88 /*
89 * Functions for manipulating the kernel_lock. We put them here
90 * so that they show up in profiles.
91 */
92
93 #define _KERNEL_LOCK_ABORT(msg) \
94 LOCKDEBUG_ABORT(kernel_lock, &_kernel_lock_ops, __func__, msg)
95
96 #ifdef LOCKDEBUG
97 #define _KERNEL_LOCK_ASSERT(cond) \
98 do { \
99 if (!(cond)) \
100 _KERNEL_LOCK_ABORT("assertion failed: " #cond); \
101 } while (/* CONSTCOND */ 0)
102 #else
103 #define _KERNEL_LOCK_ASSERT(cond) /* nothing */
104 #endif
105
106 void _kernel_lock_dump(volatile void *);
107
108 lockops_t _kernel_lock_ops = {
109 "Kernel lock",
110 LOCKOPS_SPIN,
111 _kernel_lock_dump
112 };
113
114 /*
115 * Initialize the kernel lock.
116 */
117 void
118 kernel_lock_init(void)
119 {
120
121 CTASSERT(CACHE_LINE_SIZE >= sizeof(__cpu_simple_lock_t));
122 __cpu_simple_lock_init(kernel_lock);
123 kernel_lock_dodebug = LOCKDEBUG_ALLOC(kernel_lock, &_kernel_lock_ops,
124 RETURN_ADDRESS);
125 }
126
127 /*
128 * Print debugging information about the kernel lock.
129 */
130 void
131 _kernel_lock_dump(volatile void *junk)
132 {
133 struct cpu_info *ci = curcpu();
134
135 (void)junk;
136
137 printf_nolog("curcpu holds : %18d wanted by: %#018lx\n",
138 ci->ci_biglock_count, (long)ci->ci_biglock_wanted);
139 }
140
141 /*
142 * Acquire 'nlocks' holds on the kernel lock. If 'l' is non-null, the
143 * acquisition is from process context.
144 */
145 void
146 _kernel_lock(int nlocks)
147 {
148 struct cpu_info *ci;
149 LOCKSTAT_TIMER(spintime);
150 LOCKSTAT_FLAG(lsflag);
151 struct lwp *owant;
152 u_int spins;
153 int s;
154 struct lwp *l = curlwp;
155
156 _KERNEL_LOCK_ASSERT(nlocks > 0);
157
158 s = splvm();
159 ci = curcpu();
160 if (ci->ci_biglock_count != 0) {
161 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock));
162 ci->ci_biglock_count += nlocks;
163 l->l_blcnt += nlocks;
164 splx(s);
165 return;
166 }
167
168 _KERNEL_LOCK_ASSERT(l->l_blcnt == 0);
169 LOCKDEBUG_WANTLOCK(kernel_lock_dodebug, kernel_lock, RETURN_ADDRESS,
170 false, false);
171
172 if (__cpu_simple_lock_try(kernel_lock)) {
173 ci->ci_biglock_count = nlocks;
174 l->l_blcnt = nlocks;
175 LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL,
176 RETURN_ADDRESS, 0);
177 splx(s);
178 return;
179 }
180
181 /*
182 * To remove the ordering constraint between adaptive mutexes
183 * and kernel_lock we must make it appear as if this thread is
184 * blocking. For non-interlocked mutex release, a store fence
185 * is required to ensure that the result of any mutex_exit()
186 * by the current LWP becomes visible on the bus before the set
187 * of ci->ci_biglock_wanted becomes visible.
188 */
189 membar_producer();
190 owant = ci->ci_biglock_wanted;
191 ci->ci_biglock_wanted = l;
192
193 /*
194 * Spin until we acquire the lock. Once we have it, record the
195 * time spent with lockstat.
196 */
197 LOCKSTAT_ENTER(lsflag);
198 LOCKSTAT_START_TIMER(lsflag, spintime);
199
200 spins = 0;
201 do {
202 splx(s);
203 while (__SIMPLELOCK_LOCKED_P(kernel_lock)) {
204 if (SPINLOCK_SPINOUT(spins)) {
205 extern int start_init_exec;
206 if (!start_init_exec)
207 _KERNEL_LOCK_ABORT("spinout");
208 }
209 SPINLOCK_BACKOFF_HOOK;
210 SPINLOCK_SPIN_HOOK;
211 }
212 s = splvm();
213 } while (!__cpu_simple_lock_try(kernel_lock));
214
215 ci->ci_biglock_count = nlocks;
216 l->l_blcnt = nlocks;
217 LOCKSTAT_STOP_TIMER(lsflag, spintime);
218 LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL,
219 RETURN_ADDRESS, 0);
220 if (owant == NULL) {
221 LOCKSTAT_EVENT_RA(lsflag, kernel_lock,
222 LB_KERNEL_LOCK | LB_SPIN, 1, spintime, RETURN_ADDRESS);
223 }
224 LOCKSTAT_EXIT(lsflag);
225 splx(s);
226
227 /*
228 * Now that we have kernel_lock, reset ci_biglock_wanted. This
229 * store must be unbuffered (immediately visible on the bus) in
230 * order for non-interlocked mutex release to work correctly.
231 * It must be visible before a mutex_exit() can execute on this
232 * processor.
233 *
234 * Note: only where CAS is available in hardware will this be
235 * an unbuffered write, but non-interlocked release cannot be
236 * done on CPUs without CAS in hardware.
237 */
238 (void)atomic_swap_ptr(&ci->ci_biglock_wanted, owant);
239
240 /*
241 * Issue a memory barrier as we have acquired a lock. This also
242 * prevents stores from a following mutex_exit() being reordered
243 * to occur before our store to ci_biglock_wanted above.
244 */
245 membar_enter();
246 }
247
248 /*
249 * Release 'nlocks' holds on the kernel lock. If 'nlocks' is zero, release
250 * all holds. If 'l' is non-null, the release is from process context.
251 */
252 void
253 _kernel_unlock(int nlocks, int *countp)
254 {
255 struct cpu_info *ci;
256 u_int olocks;
257 int s;
258 struct lwp *l = curlwp;
259
260 _KERNEL_LOCK_ASSERT(nlocks < 2);
261
262 olocks = l->l_blcnt;
263
264 if (olocks == 0) {
265 _KERNEL_LOCK_ASSERT(nlocks <= 0);
266 if (countp != NULL)
267 *countp = 0;
268 return;
269 }
270
271 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock));
272
273 if (nlocks == 0)
274 nlocks = olocks;
275 else if (nlocks == -1) {
276 nlocks = 1;
277 _KERNEL_LOCK_ASSERT(olocks == 1);
278 }
279 s = splvm();
280 ci = curcpu();
281 _KERNEL_LOCK_ASSERT(ci->ci_biglock_count >= l->l_blcnt);
282 if (ci->ci_biglock_count == nlocks) {
283 LOCKDEBUG_UNLOCKED(kernel_lock_dodebug, kernel_lock,
284 RETURN_ADDRESS, 0);
285 ci->ci_biglock_count = 0;
286 __cpu_simple_unlock(kernel_lock);
287 l->l_blcnt -= nlocks;
288 splx(s);
289 if (l->l_dopreempt)
290 kpreempt(0);
291 } else {
292 ci->ci_biglock_count -= nlocks;
293 l->l_blcnt -= nlocks;
294 splx(s);
295 }
296
297 if (countp != NULL)
298 *countp = olocks;
299 }
Cache object: 4937ebf794a3290c774998c38e601c23
|