1 /* gamma_dma.c -- DMA support for GMX 2000 -*- linux-c -*-
2 * Created: Fri Mar 19 14:30:16 1999 by faith@precisioninsight.com
3 *
4 * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
5 * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
6 * All Rights Reserved.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the "Software"),
10 * to deal in the Software without restriction, including without limitation
11 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 * and/or sell copies of the Software, and to permit persons to whom the
13 * Software is furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the next
16 * paragraph) shall be included in all copies or substantial portions of the
17 * Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
23 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
24 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25 * DEALINGS IN THE SOFTWARE.
26 *
27 * Authors:
28 * Rickard E. (Rik) Faith <faith@valinux.com>
29 *
30 * $FreeBSD: releng/5.0/sys/dev/drm/gamma_dma.c 95693 2002-04-29 00:25:10Z anholt $
31 */
32
33
34 #ifdef __linux__
35 #define __NO_VERSION__
36 #include <linux/interrupt.h> /* For task queue support */
37 #include <linux/delay.h>
38 #endif /* __linux__ */
39
40 #include "dev/drm/gamma.h"
41 #include "dev/drm/drmP.h"
42 #include "dev/drm/gamma_drv.h"
43
44
45 static __inline__ void gamma_dma_dispatch(drm_device_t *dev, unsigned long address,
46 unsigned long length)
47 {
48 drm_gamma_private_t *dev_priv =
49 (drm_gamma_private_t *)dev->dev_private;
50
51 GAMMA_WRITE(GAMMA_DMAADDRESS, DRM_OS_VTOPHYS((void *)address));
52 while (GAMMA_READ(GAMMA_GCOMMANDSTATUS) != 4)
53 ;
54 GAMMA_WRITE(GAMMA_DMACOUNT, length / 4);
55 }
56
57 void gamma_dma_quiescent_single(drm_device_t *dev)
58 {
59 drm_gamma_private_t *dev_priv =
60 (drm_gamma_private_t *)dev->dev_private;
61
62 while (GAMMA_READ(GAMMA_DMACOUNT))
63 ;
64 while (GAMMA_READ(GAMMA_INFIFOSPACE) < 3)
65 ;
66
67 GAMMA_WRITE(GAMMA_FILTERMODE, 1 << 10);
68 GAMMA_WRITE(GAMMA_SYNC, 0);
69
70 do {
71 while (!GAMMA_READ(GAMMA_OUTFIFOWORDS))
72 ;
73 } while (GAMMA_READ(GAMMA_OUTPUTFIFO) != GAMMA_SYNC_TAG);
74 }
75
76 void gamma_dma_quiescent_dual(drm_device_t *dev)
77 {
78 drm_gamma_private_t *dev_priv =
79 (drm_gamma_private_t *)dev->dev_private;
80
81 while (GAMMA_READ(GAMMA_DMACOUNT))
82 ;
83 while (GAMMA_READ(GAMMA_INFIFOSPACE) < 3)
84 ;
85
86 GAMMA_WRITE(GAMMA_BROADCASTMASK, 3);
87
88 GAMMA_WRITE(GAMMA_FILTERMODE, 1 << 10);
89 GAMMA_WRITE(GAMMA_SYNC, 0);
90
91 /* Read from first MX */
92 do {
93 while (!GAMMA_READ(GAMMA_OUTFIFOWORDS))
94 ;
95 } while (GAMMA_READ(GAMMA_OUTPUTFIFO) != GAMMA_SYNC_TAG);
96
97 /* Read from second MX */
98 do {
99 while (!GAMMA_READ(GAMMA_OUTFIFOWORDS + 0x10000))
100 ;
101 } while (GAMMA_READ(GAMMA_OUTPUTFIFO + 0x10000) != GAMMA_SYNC_TAG);
102 }
103
104 void gamma_dma_ready(drm_device_t *dev)
105 {
106 drm_gamma_private_t *dev_priv =
107 (drm_gamma_private_t *)dev->dev_private;
108
109 while (GAMMA_READ(GAMMA_DMACOUNT))
110 ;
111 }
112
113 static __inline__ int gamma_dma_is_ready(drm_device_t *dev)
114 {
115 drm_gamma_private_t *dev_priv =
116 (drm_gamma_private_t *)dev->dev_private;
117
118 return !GAMMA_READ(GAMMA_DMACOUNT);
119 }
120
121 void gamma_dma_service( DRM_OS_IRQ_ARGS)
122 {
123 drm_device_t *dev = (drm_device_t *)device;
124 drm_device_dma_t *dma = dev->dma;
125 drm_gamma_private_t *dev_priv =
126 (drm_gamma_private_t *)dev->dev_private;
127
128 atomic_inc(&dev->counts[6]); /* _DRM_STAT_IRQ */
129 GAMMA_WRITE(GAMMA_GDELAYTIMER, 0xc350/2); /* 0x05S */
130 GAMMA_WRITE(GAMMA_GCOMMANDINTFLAGS, 8);
131 GAMMA_WRITE(GAMMA_GINTFLAGS, 0x2001);
132 if (gamma_dma_is_ready(dev)) {
133 /* Free previous buffer */
134 if (test_and_set_bit(0, &dev->dma_flag)) return;
135 if (dma->this_buffer) {
136 gamma_free_buffer(dev, dma->this_buffer);
137 dma->this_buffer = NULL;
138 }
139 clear_bit(0, &dev->dma_flag);
140
141 #ifdef __linux__
142 /* XXX: Does FreeBSD need something here?*/
143 /* Dispatch new buffer */
144 queue_task(&dev->tq, &tq_immediate);
145 mark_bh(IMMEDIATE_BH);
146 #endif /* __linux__ */
147 }
148 }
149
150 /* Only called by gamma_dma_schedule. */
151 static int gamma_do_dma(drm_device_t *dev, int locked)
152 {
153 unsigned long address;
154 unsigned long length;
155 drm_buf_t *buf;
156 int retcode = 0;
157 drm_device_dma_t *dma = dev->dma;
158 #if DRM_DMA_HISTOGRAM
159 cycles_t dma_start, dma_stop;
160 #endif
161
162 if (test_and_set_bit(0, &dev->dma_flag)) return DRM_OS_ERR(EBUSY);
163
164 #if DRM_DMA_HISTOGRAM
165 dma_start = get_cycles();
166 #endif
167
168 if (!dma->next_buffer) {
169 DRM_ERROR("No next_buffer\n");
170 clear_bit(0, &dev->dma_flag);
171 return DRM_OS_ERR(EINVAL);
172 }
173
174 buf = dma->next_buffer;
175 address = (unsigned long)buf->address;
176 length = buf->used;
177
178 DRM_DEBUG("context %d, buffer %d (%ld bytes)\n",
179 buf->context, buf->idx, length);
180
181 if (buf->list == DRM_LIST_RECLAIM) {
182 gamma_clear_next_buffer(dev);
183 gamma_free_buffer(dev, buf);
184 clear_bit(0, &dev->dma_flag);
185 return DRM_OS_ERR(EINVAL);
186 }
187
188 if (!length) {
189 DRM_ERROR("0 length buffer\n");
190 gamma_clear_next_buffer(dev);
191 gamma_free_buffer(dev, buf);
192 clear_bit(0, &dev->dma_flag);
193 return 0;
194 }
195
196 if (!gamma_dma_is_ready(dev)) {
197 clear_bit(0, &dev->dma_flag);
198 return DRM_OS_ERR(EBUSY);
199 }
200
201 if (buf->while_locked) {
202 if (!_DRM_LOCK_IS_HELD(dev->lock.hw_lock->lock)) {
203 DRM_ERROR("Dispatching buffer %d from pid %d"
204 " \"while locked\", but no lock held\n",
205 buf->idx, buf->pid);
206 }
207 } else {
208 if (!locked && !gamma_lock_take(&dev->lock.hw_lock->lock,
209 DRM_KERNEL_CONTEXT)) {
210 clear_bit(0, &dev->dma_flag);
211 return DRM_OS_ERR(EBUSY);
212 }
213 }
214
215 if (dev->last_context != buf->context
216 && !(dev->queuelist[buf->context]->flags
217 & _DRM_CONTEXT_PRESERVED)) {
218 /* PRE: dev->last_context != buf->context */
219 if (DRM(context_switch)(dev, dev->last_context,
220 buf->context)) {
221 DRM(clear_next_buffer)(dev);
222 DRM(free_buffer)(dev, buf);
223 }
224 retcode = DRM_OS_ERR(EBUSY);
225 goto cleanup;
226
227 /* POST: we will wait for the context
228 switch and will dispatch on a later call
229 when dev->last_context == buf->context.
230 NOTE WE HOLD THE LOCK THROUGHOUT THIS
231 TIME! */
232 }
233
234 gamma_clear_next_buffer(dev);
235 buf->pending = 1;
236 buf->waiting = 0;
237 buf->list = DRM_LIST_PEND;
238 #if DRM_DMA_HISTOGRAM
239 buf->time_dispatched = get_cycles();
240 #endif
241
242 gamma_dma_dispatch(dev, address, length);
243 gamma_free_buffer(dev, dma->this_buffer);
244 dma->this_buffer = buf;
245
246 atomic_inc(&dev->counts[7]); /* _DRM_STAT_DMA */
247 atomic_add(length, &dev->counts[8]); /* _DRM_STAT_PRIMARY */
248
249 if (!buf->while_locked && !dev->context_flag && !locked) {
250 if (gamma_lock_free(dev, &dev->lock.hw_lock->lock,
251 DRM_KERNEL_CONTEXT)) {
252 DRM_ERROR("\n");
253 }
254 }
255 cleanup:
256
257 clear_bit(0, &dev->dma_flag);
258
259 #if DRM_DMA_HISTOGRAM
260 dma_stop = get_cycles();
261 atomic_inc(&dev->histo.dma[gamma_histogram_slot(dma_stop - dma_start)]);
262 #endif
263
264 return retcode;
265 }
266
267 static void gamma_dma_timer_bh(unsigned long dev)
268 {
269 gamma_dma_schedule((drm_device_t *)dev, 0);
270 }
271
272 void gamma_dma_immediate_bh(DRM_OS_TASKQUEUE_ARGS)
273 {
274 gamma_dma_schedule(dev, 0);
275 }
276
277 int gamma_dma_schedule(drm_device_t *dev, int locked)
278 {
279 int next;
280 drm_queue_t *q;
281 drm_buf_t *buf;
282 int retcode = 0;
283 int processed = 0;
284 int missed;
285 int expire = 20;
286 drm_device_dma_t *dma = dev->dma;
287 #if DRM_DMA_HISTOGRAM
288 cycles_t schedule_start;
289 #endif
290
291 if (test_and_set_bit(0, &dev->interrupt_flag)) {
292 /* Not reentrant */
293 atomic_inc(&dev->counts[10]); /* _DRM_STAT_MISSED */
294 return DRM_OS_ERR(EBUSY);
295 }
296 missed = atomic_read(&dev->counts[10]);
297
298 #if DRM_DMA_HISTOGRAM
299 schedule_start = get_cycles();
300 #endif
301
302 again:
303 if (dev->context_flag) {
304 clear_bit(0, &dev->interrupt_flag);
305 return DRM_OS_ERR(EBUSY);
306 }
307 if (dma->next_buffer) {
308 /* Unsent buffer that was previously
309 selected, but that couldn't be sent
310 because the lock could not be obtained
311 or the DMA engine wasn't ready. Try
312 again. */
313 if (!(retcode = gamma_do_dma(dev, locked))) ++processed;
314 } else {
315 do {
316 next = gamma_select_queue(dev, gamma_dma_timer_bh);
317 if (next >= 0) {
318 q = dev->queuelist[next];
319 buf = gamma_waitlist_get(&q->waitlist);
320 dma->next_buffer = buf;
321 dma->next_queue = q;
322 if (buf && buf->list == DRM_LIST_RECLAIM) {
323 gamma_clear_next_buffer(dev);
324 gamma_free_buffer(dev, buf);
325 }
326 }
327 } while (next >= 0 && !dma->next_buffer);
328 if (dma->next_buffer) {
329 if (!(retcode = gamma_do_dma(dev, locked))) {
330 ++processed;
331 }
332 }
333 }
334
335 if (--expire) {
336 if (missed != atomic_read(&dev->counts[10])) {
337 if (gamma_dma_is_ready(dev)) goto again;
338 }
339 if (processed && gamma_dma_is_ready(dev)) {
340 processed = 0;
341 goto again;
342 }
343 }
344
345 clear_bit(0, &dev->interrupt_flag);
346
347 #if DRM_DMA_HISTOGRAM
348 atomic_inc(&dev->histo.schedule[gamma_histogram_slot(get_cycles()
349 - schedule_start)]);
350 #endif
351 return retcode;
352 }
353
354 static int gamma_dma_priority(drm_device_t *dev, drm_dma_t *d)
355 {
356 unsigned long address;
357 unsigned long length;
358 int must_free = 0;
359 int retcode = 0;
360 int i;
361 int idx;
362 drm_buf_t *buf;
363 drm_buf_t *last_buf = NULL;
364 drm_device_dma_t *dma = dev->dma;
365 #ifdef __linux__
366 DECLARE_WAITQUEUE(entry, current);
367 #endif /* __linux__ */
368 #ifdef __FreeBSD__
369 static int never;
370 #endif /* __FreeBSD__ */
371
372 /* Turn off interrupt handling */
373 while (test_and_set_bit(0, &dev->interrupt_flag)) {
374 #ifdef __linux__
375 schedule();
376 if (signal_pending(current)) return DRM_OS_ERR(EINTR);
377 #endif /* __linux__ */
378 #ifdef __FreeBSD__
379 retcode = tsleep(&never, PZERO|PCATCH, "gamp1", 1);
380 if (retcode)
381 return retcode;
382 #endif /* __FreeBSD__ */
383 }
384 if (!(d->flags & _DRM_DMA_WHILE_LOCKED)) {
385 while (!gamma_lock_take(&dev->lock.hw_lock->lock,
386 DRM_KERNEL_CONTEXT)) {
387 #ifdef __linux__
388 schedule();
389 if (signal_pending(current)) {
390 clear_bit(0, &dev->interrupt_flag);
391 return DRM_OS_ERR(EINTR);
392 }
393 #endif /* __linux__ */
394 #ifdef __FreeBSD__
395 retcode = tsleep(&never, PZERO|PCATCH, "gamp2", 1);
396 if (retcode)
397 return retcode;
398 #endif /* __FreeBSD__ */
399 }
400 ++must_free;
401 }
402
403 for (i = 0; i < d->send_count; i++) {
404 idx = d->send_indices[i];
405 if (idx < 0 || idx >= dma->buf_count) {
406 DRM_ERROR("Index %d (of %d max)\n",
407 d->send_indices[i], dma->buf_count - 1);
408 continue;
409 }
410 buf = dma->buflist[ idx ];
411 if (buf->pid != DRM_OS_CURRENTPID) {
412 DRM_ERROR("Process %d using buffer owned by %d\n",
413 DRM_OS_CURRENTPID, buf->pid);
414 retcode = DRM_OS_ERR(EINVAL);
415 goto cleanup;
416 }
417 if (buf->list != DRM_LIST_NONE) {
418 DRM_ERROR("Process %d using %d's buffer on list %d\n",
419 DRM_OS_CURRENTPID, buf->pid, buf->list);
420 retcode = DRM_OS_ERR(EINVAL);
421 goto cleanup;
422 }
423 /* This isn't a race condition on
424 buf->list, since our concern is the
425 buffer reclaim during the time the
426 process closes the /dev/drm? handle, so
427 it can't also be doing DMA. */
428 buf->list = DRM_LIST_PRIO;
429 buf->used = d->send_sizes[i];
430 buf->context = d->context;
431 buf->while_locked = d->flags & _DRM_DMA_WHILE_LOCKED;
432 address = (unsigned long)buf->address;
433 length = buf->used;
434 if (!length) {
435 DRM_ERROR("0 length buffer\n");
436 }
437 if (buf->pending) {
438 DRM_ERROR("Sending pending buffer:"
439 " buffer %d, offset %d\n",
440 d->send_indices[i], i);
441 retcode = DRM_OS_ERR(EINVAL);
442 goto cleanup;
443 }
444 if (buf->waiting) {
445 DRM_ERROR("Sending waiting buffer:"
446 " buffer %d, offset %d\n",
447 d->send_indices[i], i);
448 retcode = DRM_OS_ERR(EINVAL);
449 goto cleanup;
450 }
451 buf->pending = 1;
452
453 if (dev->last_context != buf->context
454 && !(dev->queuelist[buf->context]->flags
455 & _DRM_CONTEXT_PRESERVED)) {
456 #ifdef __linux__
457 add_wait_queue(&dev->context_wait, &entry);
458 current->state = TASK_INTERRUPTIBLE;
459 #endif /* __linux__ */
460 /* PRE: dev->last_context != buf->context */
461 DRM(context_switch)(dev, dev->last_context,
462 buf->context);
463 /* POST: we will wait for the context
464 switch and will dispatch on a later call
465 when dev->last_context == buf->context.
466 NOTE WE HOLD THE LOCK THROUGHOUT THIS
467 TIME! */
468 #ifdef __linux__
469 schedule();
470 current->state = TASK_RUNNING;
471 remove_wait_queue(&dev->context_wait, &entry);
472 if (signal_pending(current)) {
473 retcode = DRM_OS_ERR(EINTR);
474 goto cleanup;
475 }
476 #endif /* __linux__ */
477 #ifdef __FreeBSD__
478 retcode = tsleep(&dev->context_wait, PZERO|PCATCH,
479 "gamctx", 0);
480 if (retcode)
481 goto cleanup;
482 #endif /* __FreeBSD__ */
483 if (dev->last_context != buf->context) {
484 DRM_ERROR("Context mismatch: %d %d\n",
485 dev->last_context,
486 buf->context);
487 }
488 }
489
490 #if DRM_DMA_HISTOGRAM
491 buf->time_queued = get_cycles();
492 buf->time_dispatched = buf->time_queued;
493 #endif
494 gamma_dma_dispatch(dev, address, length);
495 atomic_inc(&dev->counts[9]); /* _DRM_STAT_SPECIAL */
496 atomic_add(length, &dev->counts[8]); /* _DRM_STAT_PRIMARY */
497
498 if (last_buf) {
499 gamma_free_buffer(dev, last_buf);
500 }
501 last_buf = buf;
502 }
503
504
505 cleanup:
506 if (last_buf) {
507 gamma_dma_ready(dev);
508 gamma_free_buffer(dev, last_buf);
509 }
510
511 if (must_free && !dev->context_flag) {
512 if (gamma_lock_free(dev, &dev->lock.hw_lock->lock,
513 DRM_KERNEL_CONTEXT)) {
514 DRM_ERROR("\n");
515 }
516 }
517 clear_bit(0, &dev->interrupt_flag);
518 return retcode;
519 }
520
521 static int gamma_dma_send_buffers(drm_device_t *dev, drm_dma_t *d)
522 {
523 drm_buf_t *last_buf = NULL;
524 int retcode = 0;
525 drm_device_dma_t *dma = dev->dma;
526 #ifdef __linux__
527 DECLARE_WAITQUEUE(entry, current);
528 #endif /* __linux__ */
529
530 if (d->flags & _DRM_DMA_BLOCK) {
531 last_buf = dma->buflist[d->send_indices[d->send_count-1]];
532 #ifdef __linux__
533 add_wait_queue(&last_buf->dma_wait, &entry);
534 #endif /* __linux__ */
535 #ifdef __FreeBSD__
536 atomic_inc(&last_buf->dma_wait);
537 #endif /* __FreeBSD__ */
538 }
539
540 if ((retcode = gamma_dma_enqueue(dev, d))) {
541 if (d->flags & _DRM_DMA_BLOCK)
542 #ifdef __linux__
543 remove_wait_queue(&last_buf->dma_wait, &entry);
544 #endif /* __linux__ */
545 #ifdef __FreeBSD__
546 atomic_dec(&last_buf->dma_wait);
547 #endif /* __FreeBSD__ */
548 return retcode;
549 }
550
551 gamma_dma_schedule(dev, 0);
552
553 if (d->flags & _DRM_DMA_BLOCK) {
554 DRM_DEBUG("%d waiting\n", DRM_OS_CURRENTPID);
555 #ifdef __linux__
556 for (;;) {
557 current->state = TASK_INTERRUPTIBLE;
558 if (!last_buf->waiting && !last_buf->pending)
559 break; /* finished */
560 schedule();
561 if (signal_pending(current)) {
562 retcode = DRM_OS_ERR(EINTR); /* Can't restart */
563 break;
564 }
565 }
566 current->state = TASK_RUNNING;
567 remove_wait_queue(&last_buf->dma_wait, &entry);
568 #endif /* __linux__ */
569 #ifdef __FreeBSD__
570 for (;;) {
571 retcode = tsleep(&last_buf->dma_wait, PZERO|PCATCH,
572 "gamdw", 0);
573 if (!last_buf->waiting
574 && !last_buf->pending)
575 break; /* finished */
576 if (retcode)
577 break;
578 }
579 atomic_dec(&last_buf->dma_wait);
580 #endif /* __FreeBSD__ */
581 DRM_DEBUG("%d running\n", DRM_OS_CURRENTPID);
582 if (!retcode
583 || (last_buf->list==DRM_LIST_PEND && !last_buf->pending)) {
584 #ifdef __linux__
585 if (!waitqueue_active(&last_buf->dma_wait)) {
586 #endif /* __linux__ */
587 #ifdef __FreeBSD__
588 if (!last_buf->dma_wait) {
589 #endif /* __FreeBSD__ */
590 gamma_free_buffer(dev, last_buf);
591 }
592 }
593 if (retcode) {
594 DRM_ERROR("ctx%d w%d p%d c%d i%d l%d %d/%d\n",
595 d->context,
596 last_buf->waiting,
597 last_buf->pending,
598 DRM_WAITCOUNT(dev, d->context),
599 last_buf->idx,
600 last_buf->list,
601 last_buf->pid,
602 DRM_OS_CURRENTPID);
603 }
604 }
605 return retcode;
606 }
607
608 int gamma_dma( DRM_OS_IOCTL )
609 {
610 DRM_OS_DEVICE;
611 drm_device_dma_t *dma = dev->dma;
612 int retcode = 0;
613 drm_dma_t d;
614
615 DRM_OS_KRNFROMUSR(d, (drm_dma_t *) data, sizeof(d));
616
617 if (d.send_count < 0 || d.send_count > dma->buf_count) {
618 DRM_ERROR("Process %d trying to send %d buffers (of %d max)\n",
619 DRM_OS_CURRENTPID, d.send_count, dma->buf_count);
620 return DRM_OS_ERR(EINVAL);
621 }
622
623 if (d.request_count < 0 || d.request_count > dma->buf_count) {
624 DRM_ERROR("Process %d trying to get %d buffers (of %d max)\n",
625 DRM_OS_CURRENTPID, d.request_count, dma->buf_count);
626 return DRM_OS_ERR(EINVAL);
627 }
628
629 if (d.send_count) {
630 if (d.flags & _DRM_DMA_PRIORITY)
631 retcode = gamma_dma_priority(dev, &d);
632 else
633 retcode = gamma_dma_send_buffers(dev, &d);
634 }
635
636 d.granted_count = 0;
637
638 if (!retcode && d.request_count) {
639 retcode = gamma_dma_get_buffers(dev, &d);
640 }
641
642 DRM_DEBUG("%d returning, granted = %d\n",
643 DRM_OS_CURRENTPID, d.granted_count);
644 DRM_OS_KRNTOUSR((drm_dma_t *) data, d, sizeof(d));
645
646 return retcode;
647 }
Cache object: e83e58ef89ca69c4777206932a0bedc5
|