1 /*-
2 * Copyright (c) 2003
3 * Bill Paul <wpaul@windriver.com>. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. All advertising materials mentioning features or use of this software
14 * must display the following acknowledgement:
15 * This product includes software developed by Bill Paul.
16 * 4. Neither the name of the author nor the names of any co-contributors
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
30 * THE POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD: releng/6.4/sys/compat/ndis/subr_hal.c 152137 2005-11-06 19:39:41Z wpaul $");
35
36 #include <sys/param.h>
37 #include <sys/types.h>
38 #include <sys/errno.h>
39
40 #include <sys/callout.h>
41 #include <sys/kernel.h>
42 #include <sys/lock.h>
43 #include <sys/mutex.h>
44 #include <sys/proc.h>
45 #include <sys/sched.h>
46 #include <sys/module.h>
47
48 #include <sys/systm.h>
49 #include <machine/clock.h>
50 #include <machine/bus.h>
51
52 #include <sys/bus.h>
53 #include <sys/rman.h>
54
55 #include <compat/ndis/pe_var.h>
56 #include <compat/ndis/resource_var.h>
57 #include <compat/ndis/cfg_var.h>
58 #include <compat/ndis/ntoskrnl_var.h>
59 #include <compat/ndis/hal_var.h>
60
61 static void KeStallExecutionProcessor(uint32_t);
62 static void WRITE_PORT_BUFFER_ULONG(uint32_t *,
63 uint32_t *, uint32_t);
64 static void WRITE_PORT_BUFFER_USHORT(uint16_t *,
65 uint16_t *, uint32_t);
66 static void WRITE_PORT_BUFFER_UCHAR(uint8_t *,
67 uint8_t *, uint32_t);
68 static void WRITE_PORT_ULONG(uint32_t *, uint32_t);
69 static void WRITE_PORT_USHORT(uint16_t *, uint16_t);
70 static void WRITE_PORT_UCHAR(uint8_t *, uint8_t);
71 static uint32_t READ_PORT_ULONG(uint32_t *);
72 static uint16_t READ_PORT_USHORT(uint16_t *);
73 static uint8_t READ_PORT_UCHAR(uint8_t *);
74 static void READ_PORT_BUFFER_ULONG(uint32_t *,
75 uint32_t *, uint32_t);
76 static void READ_PORT_BUFFER_USHORT(uint16_t *,
77 uint16_t *, uint32_t);
78 static void READ_PORT_BUFFER_UCHAR(uint8_t *,
79 uint8_t *, uint32_t);
80 static uint64_t KeQueryPerformanceCounter(uint64_t *);
81 static void _KeLowerIrql(uint8_t);
82 static uint8_t KeRaiseIrqlToDpcLevel(void);
83 static void dummy (void);
84
85 #define NDIS_MAXCPUS 64
86 static struct mtx disp_lock[NDIS_MAXCPUS];
87
88 int
89 hal_libinit()
90 {
91 image_patch_table *patch;
92 int i;
93
94 for (i = 0; i < NDIS_MAXCPUS; i++)
95 mtx_init(&disp_lock[i], "HAL preemption lock",
96 "HAL lock", MTX_RECURSE|MTX_DEF);
97
98 patch = hal_functbl;
99 while (patch->ipt_func != NULL) {
100 windrv_wrap((funcptr)patch->ipt_func,
101 (funcptr *)&patch->ipt_wrap,
102 patch->ipt_argcnt, patch->ipt_ftype);
103 patch++;
104 }
105
106
107 return(0);
108 }
109
110 int
111 hal_libfini()
112 {
113 image_patch_table *patch;
114 int i;
115
116 for (i = 0; i < NDIS_MAXCPUS; i++)
117 mtx_destroy(&disp_lock[i]);
118
119 patch = hal_functbl;
120 while (patch->ipt_func != NULL) {
121 windrv_unwrap(patch->ipt_wrap);
122 patch++;
123 }
124
125 return(0);
126 }
127
128 static void
129 KeStallExecutionProcessor(usecs)
130 uint32_t usecs;
131 {
132 DELAY(usecs);
133 return;
134 }
135
136 static void
137 WRITE_PORT_ULONG(port, val)
138 uint32_t *port;
139 uint32_t val;
140 {
141 bus_space_write_4(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port, val);
142 return;
143 }
144
145 static void
146 WRITE_PORT_USHORT(port, val)
147 uint16_t *port;
148 uint16_t val;
149 {
150 bus_space_write_2(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port, val);
151 return;
152 }
153
154 static void
155 WRITE_PORT_UCHAR(port, val)
156 uint8_t *port;
157 uint8_t val;
158 {
159 bus_space_write_1(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port, val);
160 return;
161 }
162
163 static void
164 WRITE_PORT_BUFFER_ULONG(port, val, cnt)
165 uint32_t *port;
166 uint32_t *val;
167 uint32_t cnt;
168 {
169 bus_space_write_multi_4(NDIS_BUS_SPACE_IO, 0x0,
170 (bus_size_t)port, val, cnt);
171 return;
172 }
173
174 static void
175 WRITE_PORT_BUFFER_USHORT(port, val, cnt)
176 uint16_t *port;
177 uint16_t *val;
178 uint32_t cnt;
179 {
180 bus_space_write_multi_2(NDIS_BUS_SPACE_IO, 0x0,
181 (bus_size_t)port, val, cnt);
182 return;
183 }
184
185 static void
186 WRITE_PORT_BUFFER_UCHAR(port, val, cnt)
187 uint8_t *port;
188 uint8_t *val;
189 uint32_t cnt;
190 {
191 bus_space_write_multi_1(NDIS_BUS_SPACE_IO, 0x0,
192 (bus_size_t)port, val, cnt);
193 return;
194 }
195
196 static uint16_t
197 READ_PORT_USHORT(port)
198 uint16_t *port;
199 {
200 return(bus_space_read_2(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port));
201 }
202
203 static uint32_t
204 READ_PORT_ULONG(port)
205 uint32_t *port;
206 {
207 return(bus_space_read_4(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port));
208 }
209
210 static uint8_t
211 READ_PORT_UCHAR(port)
212 uint8_t *port;
213 {
214 return(bus_space_read_1(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port));
215 }
216
217 static void
218 READ_PORT_BUFFER_ULONG(port, val, cnt)
219 uint32_t *port;
220 uint32_t *val;
221 uint32_t cnt;
222 {
223 bus_space_read_multi_4(NDIS_BUS_SPACE_IO, 0x0,
224 (bus_size_t)port, val, cnt);
225 return;
226 }
227
228 static void
229 READ_PORT_BUFFER_USHORT(port, val, cnt)
230 uint16_t *port;
231 uint16_t *val;
232 uint32_t cnt;
233 {
234 bus_space_read_multi_2(NDIS_BUS_SPACE_IO, 0x0,
235 (bus_size_t)port, val, cnt);
236 return;
237 }
238
239 static void
240 READ_PORT_BUFFER_UCHAR(port, val, cnt)
241 uint8_t *port;
242 uint8_t *val;
243 uint32_t cnt;
244 {
245 bus_space_read_multi_1(NDIS_BUS_SPACE_IO, 0x0,
246 (bus_size_t)port, val, cnt);
247 return;
248 }
249
250 /*
251 * The spinlock implementation in Windows differs from that of FreeBSD.
252 * The basic operation of spinlocks involves two steps: 1) spin in a
253 * tight loop while trying to acquire a lock, 2) after obtaining the
254 * lock, disable preemption. (Note that on uniprocessor systems, you're
255 * allowed to skip the first step and just lock out pre-emption, since
256 * it's not possible for you to be in contention with another running
257 * thread.) Later, you release the lock then re-enable preemption.
258 * The difference between Windows and FreeBSD lies in how preemption
259 * is disabled. In FreeBSD, it's done using critical_enter(), which on
260 * the x86 arch translates to a cli instruction. This masks off all
261 * interrupts, and effectively stops the scheduler from ever running
262 * so _nothing_ can execute except the current thread. In Windows,
263 * preemption is disabled by raising the processor IRQL to DISPATCH_LEVEL.
264 * This stops other threads from running, but does _not_ block device
265 * interrupts. This means ISRs can still run, and they can make other
266 * threads runable, but those other threads won't be able to execute
267 * until the current thread lowers the IRQL to something less than
268 * DISPATCH_LEVEL.
269 *
270 * There's another commonly used IRQL in Windows, which is APC_LEVEL.
271 * An APC is an Asynchronous Procedure Call, which differs from a DPC
272 * (Defered Procedure Call) in that a DPC is queued up to run in
273 * another thread, while an APC runs in the thread that scheduled
274 * it (similar to a signal handler in a UNIX process). We don't
275 * actually support the notion of APCs in FreeBSD, so for now, the
276 * only IRQLs we're interested in are DISPATCH_LEVEL and PASSIVE_LEVEL.
277 *
278 * To simulate DISPATCH_LEVEL, we raise the current thread's priority
279 * to PI_REALTIME, which is the highest we can give it. This should,
280 * if I understand things correctly, prevent anything except for an
281 * interrupt thread from preempting us. PASSIVE_LEVEL is basically
282 * everything else.
283 *
284 * Be aware that, at least on the x86 arch, the Windows spinlock
285 * functions are divided up in peculiar ways. The actual spinlock
286 * functions are KfAcquireSpinLock() and KfReleaseSpinLock(), and
287 * they live in HAL.dll. Meanwhile, KeInitializeSpinLock(),
288 * KefAcquireSpinLockAtDpcLevel() and KefReleaseSpinLockFromDpcLevel()
289 * live in ntoskrnl.exe. Most Windows source code will call
290 * KeAcquireSpinLock() and KeReleaseSpinLock(), but these are just
291 * macros that call KfAcquireSpinLock() and KfReleaseSpinLock().
292 * KefAcquireSpinLockAtDpcLevel() and KefReleaseSpinLockFromDpcLevel()
293 * perform the lock aquisition/release functions without doing the
294 * IRQL manipulation, and are used when one is already running at
295 * DISPATCH_LEVEL. Make sense? Good.
296 *
297 * According to the Microsoft documentation, any thread that calls
298 * KeAcquireSpinLock() must be running at IRQL <= DISPATCH_LEVEL. If
299 * we detect someone trying to acquire a spinlock from DEVICE_LEVEL
300 * or HIGH_LEVEL, we panic.
301 *
302 * Alternate sleep-lock-based spinlock implementation
303 * --------------------------------------------------
304 *
305 * The earlier spinlock implementation was arguably a bit of a hack
306 * and presented several problems. It was basically designed to provide
307 * the functionality of spinlocks without incurring the wrath of
308 * WITNESS. We could get away with using both our spinlock implementation
309 * and FreeBSD sleep locks at the same time, but if WITNESS knew what
310 * we were really up to, it would have spanked us rather severely.
311 *
312 * There's another method we can use based entirely on sleep locks.
313 * First, it's important to realize that everything we're locking
314 * resides inside Project Evil itself: any critical data being locked
315 * by drivers belongs to the drivers, and should not be referenced
316 * by any other OS code outside of the NDISulator. The priority-based
317 * locking scheme has system-wide effects, just like real spinlocks
318 * (blocking preemption affects the whole CPU), but since we keep all
319 * our critical data private, we can use a simpler mechanism that
320 * affects only code/threads directly related to Project Evil.
321 *
322 * The idea is to create a sleep lock mutex for each CPU in the system.
323 * When a CPU running in the NDISulator wants to acquire a spinlock, it
324 * does the following:
325 * - Pin ourselves to the current CPU
326 * - Acquire the mutex for the current CPU
327 * - Spin on the spinlock variable using atomic test and set, just like
328 * a real spinlock.
329 * - Once we have the lock, we execute our critical code
330 *
331 * To give up the lock, we do:
332 * - Clear the spinlock variable with an atomic op
333 * - Release the per-CPU mutex
334 * - Unpin ourselves from the current CPU.
335 *
336 * On a uniprocessor system, this means all threads that access protected
337 * data are serialized through the per-CPU mutex. After one thread
338 * acquires the 'spinlock,' any other thread that uses a spinlock on the
339 * current CPU will block on the per-CPU mutex, which has the same general
340 * effect of blocking pre-emption, but _only_ for those threads that are
341 * running NDISulator code.
342 *
343 * On a multiprocessor system, threads on different CPUs all block on
344 * their respective per-CPU mutex, and the atomic test/set operation
345 * on the spinlock variable provides inter-CPU synchronization, though
346 * only for threads running NDISulator code.
347 *
348 * This method solves an important problem. In Windows, you're allowed
349 * to do an ExAllocatePoolWithTag() with a spinlock held, provided you
350 * allocate from NonPagedPool. This implies an atomic heap allocation
351 * that will not cause the current thread to sleep. (You can't sleep
352 * while holding real spinlock: clowns will eat you.) But in FreeBSD,
353 * malloc(9) _always_ triggers the acquisition of a sleep lock, even
354 * when you use M_NOWAIT. This is not a problem for FreeBSD native
355 * code: you're allowed to sleep in things like interrupt threads. But
356 * it is a problem with the old priority-based spinlock implementation:
357 * even though we get away with it most of the time, we really can't
358 * do a malloc(9) after doing a KeAcquireSpinLock() or KeRaiseIrql().
359 * With the new implementation, it's not a problem: you're allowed to
360 * acquire more than one sleep lock (as long as you avoid lock order
361 * reversals).
362 *
363 * The one drawback to this approach is that now we have a lot of
364 * contention on one per-CPU mutex within the NDISulator code. Whether
365 * or not this is preferable to the expected Windows spinlock behavior
366 * of blocking pre-emption is debatable.
367 */
368
369 uint8_t
370 KfAcquireSpinLock(lock)
371 kspin_lock *lock;
372 {
373 uint8_t oldirql;
374
375 KeRaiseIrql(DISPATCH_LEVEL, &oldirql);
376 KeAcquireSpinLockAtDpcLevel(lock);
377
378 return(oldirql);
379 }
380
381 void
382 KfReleaseSpinLock(lock, newirql)
383 kspin_lock *lock;
384 uint8_t newirql;
385 {
386 KeReleaseSpinLockFromDpcLevel(lock);
387 KeLowerIrql(newirql);
388
389 return;
390 }
391
392 uint8_t
393 KeGetCurrentIrql()
394 {
395 if (mtx_owned(&disp_lock[curthread->td_oncpu]))
396 return(DISPATCH_LEVEL);
397 return(PASSIVE_LEVEL);
398 }
399
400 static uint64_t
401 KeQueryPerformanceCounter(freq)
402 uint64_t *freq;
403 {
404 if (freq != NULL)
405 *freq = hz;
406
407 return((uint64_t)ticks);
408 }
409
410 uint8_t
411 KfRaiseIrql(irql)
412 uint8_t irql;
413 {
414 uint8_t oldirql;
415
416 oldirql = KeGetCurrentIrql();
417
418 /* I am so going to hell for this. */
419 if (oldirql > irql)
420 panic("IRQL_NOT_LESS_THAN");
421
422 if (oldirql != DISPATCH_LEVEL) {
423 sched_pin();
424 mtx_lock(&disp_lock[curthread->td_oncpu]);
425 }
426 /*printf("RAISE IRQL: %d %d\n", irql, oldirql);*/
427
428 return(oldirql);
429 }
430
431 void
432 KfLowerIrql(oldirql)
433 uint8_t oldirql;
434 {
435 if (oldirql == DISPATCH_LEVEL)
436 return;
437
438 if (KeGetCurrentIrql() != DISPATCH_LEVEL)
439 panic("IRQL_NOT_GREATER_THAN");
440
441 mtx_unlock(&disp_lock[curthread->td_oncpu]);
442 sched_unpin();
443
444 return;
445 }
446
447 static uint8_t
448 KeRaiseIrqlToDpcLevel(void)
449 {
450 uint8_t irql;
451
452 KeRaiseIrql(DISPATCH_LEVEL, &irql);
453 return(irql);
454 }
455
456 static void
457 _KeLowerIrql(oldirql)
458 uint8_t oldirql;
459 {
460 KeLowerIrql(oldirql);
461 return;
462 }
463
464 static void dummy()
465 {
466 printf ("hal dummy called...\n");
467 return;
468 }
469
470 image_patch_table hal_functbl[] = {
471 IMPORT_SFUNC(KeStallExecutionProcessor, 1),
472 IMPORT_SFUNC(WRITE_PORT_ULONG, 2),
473 IMPORT_SFUNC(WRITE_PORT_USHORT, 2),
474 IMPORT_SFUNC(WRITE_PORT_UCHAR, 2),
475 IMPORT_SFUNC(WRITE_PORT_BUFFER_ULONG, 3),
476 IMPORT_SFUNC(WRITE_PORT_BUFFER_USHORT, 3),
477 IMPORT_SFUNC(WRITE_PORT_BUFFER_UCHAR, 3),
478 IMPORT_SFUNC(READ_PORT_ULONG, 1),
479 IMPORT_SFUNC(READ_PORT_USHORT, 1),
480 IMPORT_SFUNC(READ_PORT_UCHAR, 1),
481 IMPORT_SFUNC(READ_PORT_BUFFER_ULONG, 3),
482 IMPORT_SFUNC(READ_PORT_BUFFER_USHORT, 3),
483 IMPORT_SFUNC(READ_PORT_BUFFER_UCHAR, 3),
484 IMPORT_FFUNC(KfAcquireSpinLock, 1),
485 IMPORT_FFUNC(KfReleaseSpinLock, 1),
486 IMPORT_SFUNC(KeGetCurrentIrql, 0),
487 IMPORT_SFUNC(KeQueryPerformanceCounter, 1),
488 IMPORT_FFUNC(KfLowerIrql, 1),
489 IMPORT_FFUNC(KfRaiseIrql, 1),
490 IMPORT_SFUNC(KeRaiseIrqlToDpcLevel, 0),
491 #undef KeLowerIrql
492 IMPORT_SFUNC_MAP(KeLowerIrql, _KeLowerIrql, 1),
493
494 /*
495 * This last entry is a catch-all for any function we haven't
496 * implemented yet. The PE import list patching routine will
497 * use it for any function that doesn't have an explicit match
498 * in this table.
499 */
500
501 { NULL, (FUNC)dummy, NULL, 0, WINDRV_WRAP_STDCALL },
502
503 /* End of list. */
504
505 { NULL, NULL, NULL }
506 };
Cache object: 810b463d2ac9ad2d7517d46017785afc
|