67 	dev_priv->dma_low += 8;					\
75 	dev_priv->dma_low += 8;			\
78 static void via_cmdbuf_start(drm_via_private_t *dev_priv);
79 static void via_cmdbuf_pause(drm_via_private_t *dev_priv);
80 static void via_cmdbuf_reset(drm_via_private_t *dev_priv);
81 static void via_cmdbuf_rewind(drm_via_private_t *dev_priv);
82 static int via_wait_idle(drm_via_private_t *dev_priv);
83 static void via_pad_cache(drm_via_private_t *dev_priv, int qwords);
89 static uint32_t via_cmdbuf_space(drm_via_private_t *dev_priv)  in via_cmdbuf_space()  argument
91 	uint32_t agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;  in via_cmdbuf_space()
92 	uint32_t hw_addr = *(dev_priv->hw_addr_ptr) - agp_base;  in via_cmdbuf_space()
94 	return ((hw_addr <= dev_priv->dma_low) ?  in via_cmdbuf_space()
95 		(dev_priv->dma_high + hw_addr - dev_priv->dma_low) :  in via_cmdbuf_space()
96 		(hw_addr - dev_priv->dma_low));  in via_cmdbuf_space()
103 static uint32_t via_cmdbuf_lag(drm_via_private_t *dev_priv)  in via_cmdbuf_lag()  argument
105 	uint32_t agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;  in via_cmdbuf_lag()
106 	uint32_t hw_addr = *(dev_priv->hw_addr_ptr) - agp_base;  in via_cmdbuf_lag()
108 	return ((hw_addr <= dev_priv->dma_low) ?  in via_cmdbuf_lag()
109 		(dev_priv->dma_low - hw_addr) :  in via_cmdbuf_lag()
110 		(dev_priv->dma_wrap + dev_priv->dma_low - hw_addr));  in via_cmdbuf_lag()
118 via_cmdbuf_wait(drm_via_private_t *dev_priv, unsigned int size)  in via_cmdbuf_wait()  argument
120 	uint32_t agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;  in via_cmdbuf_wait()
124 	hw_addr_ptr = dev_priv->hw_addr_ptr;  in via_cmdbuf_wait()
125 	cur_addr = dev_priv->dma_low;  in via_cmdbuf_wait()
149 static inline uint32_t *via_check_dma(drm_via_private_t * dev_priv,  in via_check_dma()  argument
152 	if ((dev_priv->dma_low + size + 4 * CMDBUF_ALIGNMENT_SIZE) >  in via_check_dma()
153 	    dev_priv->dma_high) {  in via_check_dma()
154 		via_cmdbuf_rewind(dev_priv);  in via_check_dma()
156 	if (via_cmdbuf_wait(dev_priv, size) != 0)  in via_check_dma()
159 	return (uint32_t *) (dev_priv->dma_ptr + dev_priv->dma_low);  in via_check_dma()
165 		drm_via_private_t *dev_priv =  in via_dma_cleanup()  local
168 		if (dev_priv->ring.virtual_start) {  in via_dma_cleanup()
169 			via_cmdbuf_reset(dev_priv);  in via_dma_cleanup()
171 			drm_legacy_ioremapfree(&dev_priv->ring.map, dev);  in via_dma_cleanup()
172 			dev_priv->ring.virtual_start = NULL;  in via_dma_cleanup()
181 			  drm_via_private_t *dev_priv,  in via_initialize()  argument
184 	if (!dev_priv || !dev_priv->mmio) {  in via_initialize()
189 	if (dev_priv->ring.virtual_start != NULL) {  in via_initialize()
199 	if (dev_priv->chipset == VIA_DX9_0) {  in via_initialize()
204 	dev_priv->ring.map.offset = dev->agp->base + init->offset;  in via_initialize()
205 	dev_priv->ring.map.size = init->size;  in via_initialize()
206 	dev_priv->ring.map.type = 0;  in via_initialize()
207 	dev_priv->ring.map.flags = 0;  in via_initialize()
208 	dev_priv->ring.map.mtrr = 0;  in via_initialize()
210 	drm_legacy_ioremap(&dev_priv->ring.map, dev);  in via_initialize()
212 	if (dev_priv->ring.map.handle == NULL) {  in via_initialize()
219 	dev_priv->ring.virtual_start = dev_priv->ring.map.handle;  in via_initialize()
221 	dev_priv->dma_ptr = dev_priv->ring.virtual_start;  in via_initialize()
222 	dev_priv->dma_low = 0;  in via_initialize()
223 	dev_priv->dma_high = init->size;  in via_initialize()
224 	dev_priv->dma_wrap = init->size;  in via_initialize()
225 	dev_priv->dma_offset = init->offset;  in via_initialize()
226 	dev_priv->last_pause_ptr = NULL;  in via_initialize()
227 	dev_priv->hw_addr_ptr =  in via_initialize()
228 		(volatile uint32_t *)((char *)dev_priv->mmio->handle +  in via_initialize()
231 	via_cmdbuf_start(dev_priv);  in via_initialize()
238 	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;  in via_dma_init()  local
247 			retcode = via_initialize(dev, dev_priv, init);  in via_dma_init()
256 		retcode = (dev_priv->ring.virtual_start != NULL) ?  in via_dma_init()
269 	drm_via_private_t *dev_priv;  in via_dispatch_cmdbuffer()  local
273 	dev_priv = (drm_via_private_t *) dev->dev_private;  in via_dispatch_cmdbuffer()
275 	if (dev_priv->ring.virtual_start == NULL) {  in via_dispatch_cmdbuffer()
283 	if (copy_from_user(dev_priv->pci_buf, cmd->buf, cmd->size))  in via_dispatch_cmdbuffer()
293 	     via_verify_command_stream((uint32_t *) dev_priv->pci_buf,  in via_dispatch_cmdbuffer()
298 	vb = via_check_dma(dev_priv, (cmd->size < 0x100) ? 0x102 : cmd->size);  in via_dispatch_cmdbuffer()
302 	memcpy(vb, dev_priv->pci_buf, cmd->size);  in via_dispatch_cmdbuffer()
304 	dev_priv->dma_low += cmd->size;  in via_dispatch_cmdbuffer()
312 		via_pad_cache(dev_priv, (0x100 - cmd->size) >> 3);  in via_dispatch_cmdbuffer()
313 	via_cmdbuf_pause(dev_priv);  in via_dispatch_cmdbuffer()
320 	drm_via_private_t *dev_priv = dev->dev_private;  in via_driver_dma_quiescent()  local
322 	if (!via_wait_idle(dev_priv))  in via_driver_dma_quiescent()
351 	drm_via_private_t *dev_priv = dev->dev_private;  in via_dispatch_pci_cmdbuffer()  local
356 	if (copy_from_user(dev_priv->pci_buf, cmd->buf, cmd->size))  in via_dispatch_pci_cmdbuffer()
360 	     via_verify_command_stream((uint32_t *) dev_priv->pci_buf,  in via_dispatch_pci_cmdbuffer()
366 	    via_parse_command_stream(dev, (const uint32_t *)dev_priv->pci_buf,  in via_dispatch_pci_cmdbuffer()
384 static inline uint32_t *via_align_buffer(drm_via_private_t *dev_priv,  in via_align_buffer()  argument
397 static inline uint32_t *via_get_dma(drm_via_private_t *dev_priv)  in via_get_dma()  argument
399 	return (uint32_t *) (dev_priv->dma_ptr + dev_priv->dma_low);  in via_get_dma()
407 static int via_hook_segment(drm_via_private_t *dev_priv,  in via_hook_segment()  argument
412 	volatile uint32_t *paused_at = dev_priv->last_pause_ptr;  in via_hook_segment()
418 	(void) *(volatile uint32_t *)(via_get_dma(dev_priv) - 1);  in via_hook_segment()
424 	reader = *(dev_priv->hw_addr_ptr);  in via_hook_segment()
425 	ptr = ((volatile char *)paused_at - dev_priv->dma_ptr) +  in via_hook_segment()
426 		dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr + 4;  in via_hook_segment()
428 	dev_priv->last_pause_ptr = via_get_dma(dev_priv) - 1;  in via_hook_segment()
437 	diff = (uint32_t) (ptr - reader) - dev_priv->dma_diff;  in via_hook_segment()
440 		paused = (via_read(dev_priv, 0x41c) & 0x80000000);  in via_hook_segment()
443 		reader = *(dev_priv->hw_addr_ptr);  in via_hook_segment()
444 		diff = (uint32_t) (ptr - reader) - dev_priv->dma_diff;  in via_hook_segment()
447 	paused = via_read(dev_priv, 0x41c) & 0x80000000;  in via_hook_segment()
450 		reader = *(dev_priv->hw_addr_ptr);  in via_hook_segment()
451 		diff = (uint32_t) (ptr - reader) - dev_priv->dma_diff;  in via_hook_segment()
452 		diff &= (dev_priv->dma_high - 1);  in via_hook_segment()
453 		if (diff != 0 && diff < (dev_priv->dma_high >> 1)) {  in via_hook_segment()
456 				  ptr, reader, dev_priv->dma_diff);  in via_hook_segment()
464 			via_write(dev_priv, VIA_REG_TRANSET, (HC_ParaType_PreCR << 16));  in via_hook_segment()
465 			via_write(dev_priv, VIA_REG_TRANSPACE, pause_addr_hi);  in via_hook_segment()
466 			via_write(dev_priv, VIA_REG_TRANSPACE, pause_addr_lo);  in via_hook_segment()
467 			via_read(dev_priv, VIA_REG_TRANSPACE);  in via_hook_segment()
473 static int via_wait_idle(drm_via_private_t *dev_priv)  in via_wait_idle()  argument
477 	while (!(via_read(dev_priv, VIA_REG_STATUS) & VIA_VR_QUEUE_BUSY) && --count)  in via_wait_idle()
480 	while (count && (via_read(dev_priv, VIA_REG_STATUS) &  in via_wait_idle()
487 static uint32_t *via_align_cmd(drm_via_private_t *dev_priv, uint32_t cmd_type,  in via_align_cmd()  argument
497 		via_cmdbuf_wait(dev_priv, 2 * CMDBUF_ALIGNMENT_SIZE);  in via_align_cmd()
499 	vb = via_get_dma(dev_priv);  in via_align_cmd()
502 	agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;  in via_align_cmd()
504 	    ((dev_priv->dma_low & CMDBUF_ALIGNMENT_MASK) >> 3);  in via_align_cmd()
507 	    agp_base + dev_priv->dma_low - 8 + (qw_pad_count << 3);  in via_align_cmd()
512 	vb = via_align_buffer(dev_priv, vb, qw_pad_count - 1);  in via_align_cmd()
517 static void via_cmdbuf_start(drm_via_private_t *dev_priv)  in via_cmdbuf_start()  argument
528 	dev_priv->dma_low = 0;  in via_cmdbuf_start()
530 	agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;  in via_cmdbuf_start()
532 	end_addr = agp_base + dev_priv->dma_high;  in via_cmdbuf_start()
539 	dev_priv->last_pause_ptr =  in via_cmdbuf_start()
540 	    via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0,  in via_cmdbuf_start()
544 	(void) *(volatile uint32_t *)dev_priv->last_pause_ptr;  in via_cmdbuf_start()
546 	via_write(dev_priv, VIA_REG_TRANSET, (HC_ParaType_PreCR << 16));  in via_cmdbuf_start()
547 	via_write(dev_priv, VIA_REG_TRANSPACE, command);  in via_cmdbuf_start()
548 	via_write(dev_priv, VIA_REG_TRANSPACE, start_addr_lo);  in via_cmdbuf_start()
549 	via_write(dev_priv, VIA_REG_TRANSPACE, end_addr_lo);  in via_cmdbuf_start()
551 	via_write(dev_priv, VIA_REG_TRANSPACE, pause_addr_hi);  in via_cmdbuf_start()
552 	via_write(dev_priv, VIA_REG_TRANSPACE, pause_addr_lo);  in via_cmdbuf_start()
554 	via_write(dev_priv, VIA_REG_TRANSPACE, command | HC_HAGPCMNT_MASK);  in via_cmdbuf_start()
555 	via_read(dev_priv, VIA_REG_TRANSPACE);  in via_cmdbuf_start()
557 	dev_priv->dma_diff = 0;  in via_cmdbuf_start()
560 	while (!(via_read(dev_priv, 0x41c) & 0x80000000) && count--);  in via_cmdbuf_start()
562 	reader = *(dev_priv->hw_addr_ptr);  in via_cmdbuf_start()
563 	ptr = ((volatile char *)dev_priv->last_pause_ptr - dev_priv->dma_ptr) +  in via_cmdbuf_start()
564 	    dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr + 4;  in via_cmdbuf_start()
573 	dev_priv->dma_diff = ptr - reader;  in via_cmdbuf_start()
576 static void via_pad_cache(drm_via_private_t *dev_priv, int qwords)  in via_pad_cache()  argument
580 	via_cmdbuf_wait(dev_priv, qwords + 2);  in via_pad_cache()
581 	vb = via_get_dma(dev_priv);  in via_pad_cache()
583 	via_align_buffer(dev_priv, vb, qwords);  in via_pad_cache()
586 static inline void via_dummy_bitblt(drm_via_private_t *dev_priv)  in via_dummy_bitblt()  argument
588 	uint32_t *vb = via_get_dma(dev_priv);  in via_dummy_bitblt()
594 static void via_cmdbuf_jump(drm_via_private_t *dev_priv)  in via_cmdbuf_jump()  argument
602 	agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;  in via_cmdbuf_jump()
603 	via_align_cmd(dev_priv, HC_HAGPBpID_JUMP, 0, &jump_addr_hi,  in via_cmdbuf_jump()
606 	dev_priv->dma_wrap = dev_priv->dma_low;  in via_cmdbuf_jump()
612 	dev_priv->dma_low = 0;  in via_cmdbuf_jump()
613 	if (via_cmdbuf_wait(dev_priv, CMDBUF_ALIGNMENT_SIZE) != 0)  in via_cmdbuf_jump()
616 	via_dummy_bitblt(dev_priv);  in via_cmdbuf_jump()
617 	via_dummy_bitblt(dev_priv);  in via_cmdbuf_jump()
620 	    via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,  in via_cmdbuf_jump()
622 	via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,  in via_cmdbuf_jump()
626 	dma_low_save1 = dev_priv->dma_low;  in via_cmdbuf_jump()
638 		via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,  in via_cmdbuf_jump()
640 	via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,  in via_cmdbuf_jump()
644 	dma_low_save2 = dev_priv->dma_low;  in via_cmdbuf_jump()
645 	dev_priv->dma_low = dma_low_save1;  in via_cmdbuf_jump()
646 	via_hook_segment(dev_priv, jump_addr_hi, jump_addr_lo, 0);  in via_cmdbuf_jump()
647 	dev_priv->dma_low = dma_low_save2;  in via_cmdbuf_jump()
648 	via_hook_segment(dev_priv, pause_addr_hi, pause_addr_lo, 0);  in via_cmdbuf_jump()
652 static void via_cmdbuf_rewind(drm_via_private_t *dev_priv)  in via_cmdbuf_rewind()  argument
654 	via_cmdbuf_jump(dev_priv);  in via_cmdbuf_rewind()
657 static void via_cmdbuf_flush(drm_via_private_t *dev_priv, uint32_t cmd_type)  in via_cmdbuf_flush()  argument
661 	via_align_cmd(dev_priv, cmd_type, 0, &pause_addr_hi, &pause_addr_lo, 0);  in via_cmdbuf_flush()
662 	via_hook_segment(dev_priv, pause_addr_hi, pause_addr_lo, 0);  in via_cmdbuf_flush()
665 static void via_cmdbuf_pause(drm_via_private_t *dev_priv)  in via_cmdbuf_pause()  argument
667 	via_cmdbuf_flush(dev_priv, HC_HAGPBpID_PAUSE);  in via_cmdbuf_pause()
670 static void via_cmdbuf_reset(drm_via_private_t *dev_priv)  in via_cmdbuf_reset()  argument
672 	via_cmdbuf_flush(dev_priv, HC_HAGPBpID_STOP);  in via_cmdbuf_reset()
673 	via_wait_idle(dev_priv);  in via_cmdbuf_reset()
685 	drm_via_private_t *dev_priv;  in via_cmdbuf_size()  local
690 	dev_priv = (drm_via_private_t *) dev->dev_private;  in via_cmdbuf_size()
692 	if (dev_priv->ring.virtual_start == NULL) {  in via_cmdbuf_size()
701 		while (((tmp_size = via_cmdbuf_space(dev_priv)) < d_siz->size)  in via_cmdbuf_size()
712 		while (((tmp_size = via_cmdbuf_lag(dev_priv)) > d_siz->size)  in via_cmdbuf_size()