313 	drm_via_blitq_t *blitq = dev_priv->blit_queues + engine;  in via_dmablit_handler()  local
320 		  engine, from_irq, (unsigned long) blitq);  in via_dmablit_handler()
323 		spin_lock(&blitq->blit_lock);  in via_dmablit_handler()
325 		spin_lock_irqsave(&blitq->blit_lock, irqsave);  in via_dmablit_handler()
327 	done_transfer = blitq->is_active &&  in via_dmablit_handler()
329 	done_transfer = done_transfer || (blitq->aborting && !(status & VIA_DMA_CSR_DE));  in via_dmablit_handler()
331 	cur = blitq->cur;  in via_dmablit_handler()
334 		blitq->blits[cur]->aborted = blitq->aborting;  in via_dmablit_handler()
335 		blitq->done_blit_handle++;  in via_dmablit_handler()
336 		wake_up(blitq->blit_queue + cur);  in via_dmablit_handler()
341 		blitq->cur = cur;  in via_dmablit_handler()
349 		blitq->is_active = 0;  in via_dmablit_handler()
350 		blitq->aborting = 0;  in via_dmablit_handler()
351 		schedule_work(&blitq->wq);  in via_dmablit_handler()
353 	} else if (blitq->is_active && time_after_eq(jiffies, blitq->end)) {  in via_dmablit_handler()
360 		blitq->aborting = 1;  in via_dmablit_handler()
361 		blitq->end = jiffies + HZ;  in via_dmablit_handler()
364 	if (!blitq->is_active) {  in via_dmablit_handler()
365 		if (blitq->num_outstanding) {  in via_dmablit_handler()
366 			via_fire_dmablit(dev, blitq->blits[cur], engine);  in via_dmablit_handler()
367 			blitq->is_active = 1;  in via_dmablit_handler()
368 			blitq->cur = cur;  in via_dmablit_handler()
369 			blitq->num_outstanding--;  in via_dmablit_handler()
370 			blitq->end = jiffies + HZ;  in via_dmablit_handler()
371 			if (!timer_pending(&blitq->poll_timer))  in via_dmablit_handler()
372 				mod_timer(&blitq->poll_timer, jiffies + 1);  in via_dmablit_handler()
374 			if (timer_pending(&blitq->poll_timer))  in via_dmablit_handler()
375 				del_timer(&blitq->poll_timer);  in via_dmablit_handler()
381 		spin_unlock(&blitq->blit_lock);  in via_dmablit_handler()
383 		spin_unlock_irqrestore(&blitq->blit_lock, irqsave);  in via_dmablit_handler()
393 via_dmablit_active(drm_via_blitq_t *blitq, int engine, uint32_t handle, wait_queue_head_t **queue)  in via_dmablit_active()  argument
399 	spin_lock_irqsave(&blitq->blit_lock, irqsave);  in via_dmablit_active()
405 	active = ((blitq->done_blit_handle - handle) > (1 << 23)) &&  in via_dmablit_active()
406 		((blitq->cur_blit_handle - handle) <= (1 << 23));  in via_dmablit_active()
409 		slot = handle - blitq->done_blit_handle + blitq->cur - 1;  in via_dmablit_active()
412 		*queue = blitq->blit_queue + slot;  in via_dmablit_active()
415 	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);  in via_dmablit_active()
429 	drm_via_blitq_t *blitq = dev_priv->blit_queues + engine;  in via_dmablit_sync()  local
433 	if (via_dmablit_active(blitq, engine, handle, &queue)) {  in via_dmablit_sync()
435 			    !via_dmablit_active(blitq, engine, handle, NULL));  in via_dmablit_sync()
457 	drm_via_blitq_t *blitq = from_timer(blitq, t, poll_timer);  in via_dmablit_timer()  local
458 	struct drm_device *dev = blitq->dev;  in via_dmablit_timer()
460 		(blitq - ((drm_via_private_t *)dev->dev_private)->blit_queues);  in via_dmablit_timer()
467 	if (!timer_pending(&blitq->poll_timer)) {  in via_dmablit_timer()
468 		mod_timer(&blitq->poll_timer, jiffies + 1);  in via_dmablit_timer()
493 	drm_via_blitq_t *blitq = container_of(work, drm_via_blitq_t, wq);  in via_dmablit_workqueue()  local
494 	struct drm_device *dev = blitq->dev;  in via_dmablit_workqueue()
501 		  (blitq - ((drm_via_private_t *)dev->dev_private)->blit_queues));  in via_dmablit_workqueue()
503 	spin_lock_irqsave(&blitq->blit_lock, irqsave);  in via_dmablit_workqueue()
505 	while (blitq->serviced != blitq->cur) {  in via_dmablit_workqueue()
507 		cur_released = blitq->serviced++;  in via_dmablit_workqueue()
511 		if (blitq->serviced >= VIA_NUM_BLIT_SLOTS)  in via_dmablit_workqueue()
512 			blitq->serviced = 0;  in via_dmablit_workqueue()
514 		cur_sg = blitq->blits[cur_released];  in via_dmablit_workqueue()
515 		blitq->num_free++;  in via_dmablit_workqueue()
517 		spin_unlock_irqrestore(&blitq->blit_lock, irqsave);  in via_dmablit_workqueue()
519 		wake_up(&blitq->busy_queue);  in via_dmablit_workqueue()
524 		spin_lock_irqsave(&blitq->blit_lock, irqsave);  in via_dmablit_workqueue()
527 	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);  in via_dmablit_workqueue()
541 	drm_via_blitq_t *blitq;  in via_init_dmablit()  local
546 		blitq = dev_priv->blit_queues + i;  in via_init_dmablit()
547 		blitq->dev = dev;  in via_init_dmablit()
548 		blitq->cur_blit_handle = 0;  in via_init_dmablit()
549 		blitq->done_blit_handle = 0;  in via_init_dmablit()
550 		blitq->head = 0;  in via_init_dmablit()
551 		blitq->cur = 0;  in via_init_dmablit()
552 		blitq->serviced = 0;  in via_init_dmablit()
553 		blitq->num_free = VIA_NUM_BLIT_SLOTS - 1;  in via_init_dmablit()
554 		blitq->num_outstanding = 0;  in via_init_dmablit()
555 		blitq->is_active = 0;  in via_init_dmablit()
556 		blitq->aborting = 0;  in via_init_dmablit()
557 		spin_lock_init(&blitq->blit_lock);  in via_init_dmablit()
559 			init_waitqueue_head(blitq->blit_queue + j);  in via_init_dmablit()
560 		init_waitqueue_head(&blitq->busy_queue);  in via_init_dmablit()
561 		INIT_WORK(&blitq->wq, via_dmablit_workqueue);  in via_init_dmablit()
562 		timer_setup(&blitq->poll_timer, via_dmablit_timer, 0);  in via_init_dmablit()
675 via_dmablit_grab_slot(drm_via_blitq_t *blitq, int engine)  in via_dmablit_grab_slot()  argument
680 	DRM_DEBUG("Num free is %d\n", blitq->num_free);  in via_dmablit_grab_slot()
681 	spin_lock_irqsave(&blitq->blit_lock, irqsave);  in via_dmablit_grab_slot()
682 	while (blitq->num_free == 0) {  in via_dmablit_grab_slot()
683 		spin_unlock_irqrestore(&blitq->blit_lock, irqsave);  in via_dmablit_grab_slot()
685 		DRM_WAIT_ON(ret, blitq->busy_queue, HZ, blitq->num_free > 0);  in via_dmablit_grab_slot()
689 		spin_lock_irqsave(&blitq->blit_lock, irqsave);  in via_dmablit_grab_slot()
692 	blitq->num_free--;  in via_dmablit_grab_slot()
693 	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);  in via_dmablit_grab_slot()
703 via_dmablit_release_slot(drm_via_blitq_t *blitq)  in via_dmablit_release_slot()  argument
707 	spin_lock_irqsave(&blitq->blit_lock, irqsave);  in via_dmablit_release_slot()
708 	blitq->num_free++;  in via_dmablit_release_slot()
709 	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);  in via_dmablit_release_slot()
710 	wake_up(&blitq->busy_queue);  in via_dmablit_release_slot()
723 	drm_via_blitq_t *blitq;  in via_dmablit()  local
734 	blitq = dev_priv->blit_queues + engine;  in via_dmablit()
735 	if (0 != (ret = via_dmablit_grab_slot(blitq, engine)))  in via_dmablit()
738 		via_dmablit_release_slot(blitq);  in via_dmablit()
742 		via_dmablit_release_slot(blitq);  in via_dmablit()
746 	spin_lock_irqsave(&blitq->blit_lock, irqsave);  in via_dmablit()
748 	blitq->blits[blitq->head++] = vsg;  in via_dmablit()
749 	if (blitq->head >= VIA_NUM_BLIT_SLOTS)  in via_dmablit()
750 		blitq->head = 0;  in via_dmablit()
751 	blitq->num_outstanding++;  in via_dmablit()
752 	xfer->sync.sync_handle = ++blitq->cur_blit_handle;  in via_dmablit()
754 	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);  in via_dmablit()