31 static inline uint32_t reset_pkt_id(struct dma_iproc_pax_ring_data *ring)  in reset_pkt_id()  argument
33 	return ring->pkt_id = 0x0;  in reset_pkt_id()
39 static inline uint32_t alloc_pkt_id(struct dma_iproc_pax_ring_data *ring)  in alloc_pkt_id()  argument
41 	ring->pkt_id = (ring->pkt_id + 1) % 32;  in alloc_pkt_id()
42 	return ring->pkt_id;  in alloc_pkt_id()
45 static inline uint32_t curr_pkt_id(struct dma_iproc_pax_ring_data *ring)  in curr_pkt_id()  argument
47 	return ring->pkt_id;  in curr_pkt_id()
50 static inline uint32_t curr_toggle_val(struct dma_iproc_pax_ring_data *ring)  in curr_toggle_val()  argument
52 	return ring->curr.toggle;  in curr_toggle_val()
145 static void *next_desc_addr(struct dma_iproc_pax_ring_data *ring)  in next_desc_addr()  argument
150 	curr = (uintptr_t)ring->curr.write_ptr + PAX_DMA_RM_DESC_BDWIDTH;  in next_desc_addr()
156 		uintptr_t last = (uintptr_t)ring->bd +  in next_desc_addr()
158 		ring->curr.toggle = (ring->curr.toggle == 0) ? 1 : 0;  in next_desc_addr()
162 			curr = (uintptr_t)ring->bd;  in next_desc_addr()
167 	ring->curr.write_ptr = (void *)curr;  in next_desc_addr()
184 static void prepare_ring(struct dma_iproc_pax_ring_data *ring)  in prepare_ring()  argument
191 	memset(ring->bd, 0x0, PAX_DMA_RM_DESC_RING_SIZE * PAX_DMA_NUM_BD_BUFFS);  in prepare_ring()
192 	memset(ring->cmpl, 0x0, PAX_DMA_RM_CMPL_RING_SIZE);  in prepare_ring()
195 	rm_write_header_desc(ring->bd, 0x0, reset_pkt_id(ring),  in prepare_ring()
199 	curr = (uintptr_t)ring->bd;  in prepare_ring()
213 			next = (uintptr_t)ring->bd;  in prepare_ring()
221 	ring->curr.write_ptr = ring->bd;  in prepare_ring()
223 	ring->curr.toggle = 1;  in prepare_ring()
225 	ring->curr.cmpl_rd_offs = 0;  in prepare_ring()
228 	ring->curr.sync_data.signature = PAX_DMA_WRITE_SYNC_SIGNATURE;  in prepare_ring()
229 	ring->curr.sync_data.ring = ring->idx;  in prepare_ring()
231 	ring->curr.sync_data.opaque = 0x0;  in prepare_ring()
233 	ring->curr.sync_data.total_pkts = 0x0;  in prepare_ring()
364 	uintptr_t desc = (uintptr_t)pd->ring[idx].bd;  in init_ring()
365 	uintptr_t cmpl = (uintptr_t)pd->ring[idx].cmpl;  in init_ring()
423 			      struct dma_iproc_pax_ring_data *ring)  in poll_on_write_sync()  argument
433 	sent = &(ring->curr.sync_data);  in poll_on_write_sync()
436 	pci32[0] = ring->sync_pci.addr_lo;  in poll_on_write_sync()
437 	pci32[1] = ring->sync_pci.addr_hi;  in poll_on_write_sync()
459 		LOG_DBG("[ring %d]: not recvd write sync!\n", ring->idx);  in poll_on_write_sync()
471 	struct dma_iproc_pax_ring_data *ring = &(pd->ring[idx]);  in process_cmpl_event()  local
476 	rd_offs = ring->curr.cmpl_rd_offs;  in process_cmpl_event()
482 	ring->curr.cmpl_rd_offs = wr_offs;  in process_cmpl_event()
495 	c = (struct cmpl_pkt *)((uintptr_t)ring->cmpl +  in process_cmpl_event()
503 	if ((ring->curr.opq != c->opq) && (is_outstanding != 0)) {  in process_cmpl_event()
505 			idx, ring->curr.opq, c->opq, is_outstanding);  in process_cmpl_event()
517 	if (ring->dma_callback) {  in process_cmpl_event()
518 		ring->dma_callback(dev, ring->callback_arg, idx, ret);  in process_cmpl_event()
530 	struct dma_iproc_pax_ring_data *ring = &(pd->ring[idx]);  in peek_ring_cmpl()  local
533 	rd_offs = ring->curr.cmpl_rd_offs;  in peek_ring_cmpl()
581 			k_sem_give(&pd->ring[idx].alert);  in rm_isr()
621 		k_mutex_init(&pd->ring[r].lock);  in dma_iproc_pax_init()
623 		k_sem_init(&pd->ring[r].alert, 0, 1);  in dma_iproc_pax_init()
625 		pd->ring[r].idx = r;  in dma_iproc_pax_init()
626 		pd->ring[r].ring_base = cfg->rm_base +  in dma_iproc_pax_init()
628 		LOG_DBG("RING%d,VERSION:0x%x\n", pd->ring[r].idx,  in dma_iproc_pax_init()
632 		pd->ring[r].ring_mem = (void *)((uintptr_t)cfg->bd_memory_base +  in dma_iproc_pax_init()
635 		if (!pd->ring[r].ring_mem) {  in dma_iproc_pax_init()
640 		mem_aligned = ((uintptr_t)pd->ring[r].ring_mem +  in dma_iproc_pax_init()
644 		pd->ring[r].cmpl = (void *)mem_aligned;  in dma_iproc_pax_init()
645 		pd->ring[r].bd = (void *)(mem_aligned +  in dma_iproc_pax_init()
647 		pd->ring[r].payload = (void *)((uintptr_t)pd->ring[r].bd +  in dma_iproc_pax_init()
652 			pd->ring[r].idx,  in dma_iproc_pax_init()
653 			pd->ring[r].ring_mem,  in dma_iproc_pax_init()
656 			pd->ring[r].idx,  in dma_iproc_pax_init()
657 			pd->ring[r].bd,  in dma_iproc_pax_init()
658 			pd->ring[r].cmpl,  in dma_iproc_pax_init()
659 			pd->ring[r].payload);  in dma_iproc_pax_init()
662 		prepare_ring(&(pd->ring[r]));  in dma_iproc_pax_init()
724 	struct dma_iproc_pax_ring_data *ring;  in wait_for_pkt_completion()  local
726 	ring = &(pd->ring[idx]);  in wait_for_pkt_completion()
729 	if (k_sem_take(&ring->alert, K_MSEC(PAX_DMA_TIMEOUT)) != 0) {  in wait_for_pkt_completion()
746 	struct dma_iproc_pax_ring_data *ring;  in dma_iproc_pax_do_xfer()  local
752 	ring = &(pd->ring[idx]);  in dma_iproc_pax_do_xfer()
753 	pl = ring->payload;  in dma_iproc_pax_do_xfer()
760 	if ((ring->sync_pci.addr_lo == 0x0) &&  in dma_iproc_pax_do_xfer()
761 	    (ring->sync_pci.addr_hi == 0x0)) {  in dma_iproc_pax_do_xfer()
768 		ring->sync_pci.addr_lo = sync.addr_lo + idx * 4;  in dma_iproc_pax_do_xfer()
769 		ring->sync_pci.addr_hi = sync.addr_hi;  in dma_iproc_pax_do_xfer()
771 			ring->sync_pci.addr_hi,  in dma_iproc_pax_do_xfer()
772 			ring->sync_pci.addr_lo);  in dma_iproc_pax_do_xfer()
776 	ring->curr.sync_data.opaque = ring->curr.opq;  in dma_iproc_pax_do_xfer()
777 	ring->curr.sync_data.total_pkts = pl_len;  in dma_iproc_pax_do_xfer()
778 	memcpy((void *)&ring->sync_loc,  in dma_iproc_pax_do_xfer()
779 	       (void *)&(ring->curr.sync_data), 4);  in dma_iproc_pax_do_xfer()
780 	sync_pl.pci_addr = ring->sync_pci.addr_lo |  in dma_iproc_pax_do_xfer()
781 			   (uint64_t)ring->sync_pci.addr_hi << 32;  in dma_iproc_pax_do_xfer()
782 	sync_pl.axi_addr = (uintptr_t)&ring->sync_loc;  in dma_iproc_pax_do_xfer()
788 	hdr = (void *)ring->curr.write_ptr;  in dma_iproc_pax_do_xfer()
790 	toggle_bit = ring->curr.toggle;  in dma_iproc_pax_do_xfer()
792 	ring->curr.opq = curr_pkt_id(ring);  in dma_iproc_pax_do_xfer()
799 		rm_write_dma_header_desc(next_desc_addr(ring), pl + cnt);  in dma_iproc_pax_do_xfer()
800 		rm_write_axi_addr_desc(next_desc_addr(ring), pl + cnt);  in dma_iproc_pax_do_xfer()
801 		rm_write_pci_addr_desc(next_desc_addr(ring), pl + cnt);  in dma_iproc_pax_do_xfer()
803 		rm_write_header_desc(next_desc_addr(ring),  in dma_iproc_pax_do_xfer()
804 				     curr_toggle_val(ring),  in dma_iproc_pax_do_xfer()
805 				     curr_pkt_id(ring),  in dma_iproc_pax_do_xfer()
810 	rm_write_dma_header_desc(next_desc_addr(ring), &sync_pl);  in dma_iproc_pax_do_xfer()
811 	rm_write_axi_addr_desc(next_desc_addr(ring), &sync_pl);  in dma_iproc_pax_do_xfer()
812 	rm_write_pci_addr_desc(next_desc_addr(ring), &sync_pl);  in dma_iproc_pax_do_xfer()
815 	rm_write_header_next_desc(next_desc_addr(ring), ring, alloc_pkt_id(ring),  in dma_iproc_pax_do_xfer()
833 	ret = poll_on_write_sync(dev, ring);  in dma_iproc_pax_do_xfer()
834 	k_mutex_lock(&ring->lock, K_FOREVER);  in dma_iproc_pax_do_xfer()
835 	ring->ring_active = 0;  in dma_iproc_pax_do_xfer()
836 	k_mutex_unlock(&ring->lock);  in dma_iproc_pax_do_xfer()
839 	ring->ring_active = 0;  in dma_iproc_pax_do_xfer()
850 	struct dma_iproc_pax_ring_data *ring;  in dma_iproc_pax_configure()  local
863 	ring = &(pd->ring[channel]);  in dma_iproc_pax_configure()
864 	k_mutex_lock(&ring->lock, K_FOREVER);  in dma_iproc_pax_configure()
872 	if (ring->ring_active) {  in dma_iproc_pax_configure()
877 	ring->ring_active = 1;  in dma_iproc_pax_configure()
884 		ring->payload->direction = CARD_TO_HOST;  in dma_iproc_pax_configure()
885 		ring->payload->pci_addr = cfg->head_block->dest_address;  in dma_iproc_pax_configure()
886 		ring->payload->axi_addr = cfg->head_block->source_address;  in dma_iproc_pax_configure()
892 		ring->payload->direction = HOST_TO_CARD;  in dma_iproc_pax_configure()
893 		ring->payload->pci_addr = cfg->head_block->source_address;  in dma_iproc_pax_configure()
894 		ring->payload->axi_addr = cfg->head_block->dest_address;  in dma_iproc_pax_configure()
896 		ring->ring_active = 0;  in dma_iproc_pax_configure()
906 		ring->ring_active = 0;  in dma_iproc_pax_configure()
913 		ring->ring_active = 0;  in dma_iproc_pax_configure()
921 		ring->ring_active = 0;  in dma_iproc_pax_configure()
929 		ring->ring_active = 0;  in dma_iproc_pax_configure()
934 	ring->payload->xfer_sz = xfer_sz;  in dma_iproc_pax_configure()
936 	ring->dma_callback = cfg->dma_callback;  in dma_iproc_pax_configure()
937 	ring->callback_arg = cfg->user_data;  in dma_iproc_pax_configure()
939 	k_mutex_unlock(&ring->lock);  in dma_iproc_pax_configure()
949 	struct dma_iproc_pax_ring_data *ring;  in dma_iproc_pax_transfer_start()  local
955 	ring = &(pd->ring[channel]);  in dma_iproc_pax_transfer_start()
957 	ret = dma_iproc_pax_do_xfer(dev, channel, ring->payload, 1);  in dma_iproc_pax_transfer_start()