34 static inline uint32_t reset_pkt_id(struct dma_iproc_pax_ring_data *ring)  in reset_pkt_id()  argument
36 	return ring->pkt_id = 0x0;  in reset_pkt_id()
39 static inline uint32_t alloc_pkt_id(struct dma_iproc_pax_ring_data *ring)  in alloc_pkt_id()  argument
41 	ring->pkt_id = (ring->pkt_id + 1) % 32;  in alloc_pkt_id()
42 	return ring->pkt_id;  in alloc_pkt_id()
45 static inline uint32_t curr_pkt_id(struct dma_iproc_pax_ring_data *ring)  in curr_pkt_id()  argument
47 	return ring->pkt_id;  in curr_pkt_id()
50 static inline uint32_t curr_toggle_val(struct dma_iproc_pax_ring_data *ring)  in curr_toggle_val()  argument
52 	return ring->curr.toggle;  in curr_toggle_val()
149 static inline void *get_curr_desc_addr(struct dma_iproc_pax_ring_data *ring)  in get_curr_desc_addr()  argument
154 	curr = (uintptr_t)ring->curr.write_ptr;  in get_curr_desc_addr()
160 		uintptr_t last = (uintptr_t)ring->bd +  in get_curr_desc_addr()
162 		nxt->toggle = ring->curr.toggle;  in get_curr_desc_addr()
163 		ring->curr.toggle = (ring->curr.toggle == 0) ? 1 : 0;  in get_curr_desc_addr()
167 			curr = (uintptr_t)ring->bd;  in get_curr_desc_addr()
171 		ring->descs_inflight++;  in get_curr_desc_addr()
174 	ring->curr.write_ptr = (void *)(curr + PAX_DMA_RM_DESC_BDWIDTH);  in get_curr_desc_addr()
175 	ring->descs_inflight++;  in get_curr_desc_addr()
193 static void prepare_ring(struct dma_iproc_pax_ring_data *ring)  in prepare_ring()  argument
202 	memset(ring->bd, 0x0, PAX_DMA_RM_DESC_RING_SIZE * PAX_DMA_NUM_BD_BUFFS);  in prepare_ring()
203 	memset(ring->cmpl, 0x0, PAX_DMA_RM_CMPL_RING_SIZE);  in prepare_ring()
209 	curr = (uintptr_t)ring->bd;  in prepare_ring()
232 			next = (uintptr_t)ring->bd;  in prepare_ring()
240 	ring->curr.write_ptr = ring->bd;  in prepare_ring()
242 	ring->curr.toggle = 1;  in prepare_ring()
244 	ring->curr.cmpl_rd_offs = 0;  in prepare_ring()
246 	ring->descs_inflight = 0;  in prepare_ring()
249 	ring->curr.sync_data.signature = PAX_DMA_WRITE_SYNC_SIGNATURE;  in prepare_ring()
250 	ring->curr.sync_data.ring = ring->idx;  in prepare_ring()
252 	ring->curr.sync_data.opaque = 0x0;  in prepare_ring()
254 	ring->curr.sync_data.total_pkts = 0x0;  in prepare_ring()
403 	struct dma_iproc_pax_ring_data *ring = &(pd->ring[idx]);  in write_doorbell()  local
405 	sys_write32(ring->descs_inflight,  in write_doorbell()
407 	ring->descs_inflight = 0;  in write_doorbell()
428 	uintptr_t desc = (uintptr_t)pd->ring[idx].bd;  in init_ring()
429 	uintptr_t cmpl = (uintptr_t)pd->ring[idx].cmpl;  in init_ring()
507 			      struct dma_iproc_pax_ring_data *ring)  in poll_on_write_sync()  argument
517 	sent = &(ring->curr.sync_data);  in poll_on_write_sync()
520 	pci32[0] = ring->sync_pci.addr_lo;  in poll_on_write_sync()
521 	pci32[1] = ring->sync_pci.addr_hi;  in poll_on_write_sync()
543 		LOG_ERR("[ring %d]: not recvd write sync!\n", ring->idx);  in poll_on_write_sync()
555 	struct dma_iproc_pax_ring_data *ring = &(pd->ring[idx]);  in process_cmpl_event()  local
560 	rd_offs = ring->curr.cmpl_rd_offs;  in process_cmpl_event()
566 	ring->curr.cmpl_rd_offs = wr_offs;  in process_cmpl_event()
579 	c = (struct cmpl_pkt *)((uintptr_t)ring->cmpl +  in process_cmpl_event()
587 	if ((ring->curr.opq != c->opq) && (is_outstanding != 0)) {  in process_cmpl_event()
589 			idx, ring->curr.opq, c->opq, is_outstanding);  in process_cmpl_event()
601 	if (ring->dma_callback) {  in process_cmpl_event()
602 		ring->dma_callback(dev, ring->callback_arg, idx, ret);  in process_cmpl_event()
606 	ring->total_pkt_count = 0;  in process_cmpl_event()
617 	struct dma_iproc_pax_ring_data *ring = &(pd->ring[idx]);  in peek_ring_cmpl()  local
620 	rd_offs = ring->curr.cmpl_rd_offs;  in peek_ring_cmpl()
664 			k_sem_give(&pd->ring[idx].alert);  in rm_isr()
701 		k_mutex_init(&pd->ring[r].lock);  in dma_iproc_pax_init()
703 		k_sem_init(&pd->ring[r].alert, 0, 1);  in dma_iproc_pax_init()
705 		pd->ring[r].idx = r;  in dma_iproc_pax_init()
706 		pd->ring[r].ring_base = cfg->rm_base +  in dma_iproc_pax_init()
708 		LOG_DBG("RING%d,VERSION:0x%x\n", pd->ring[r].idx,  in dma_iproc_pax_init()
712 		pd->ring[r].ring_mem = (void *)((uintptr_t)cfg->bd_memory_base +  in dma_iproc_pax_init()
714 		if (!pd->ring[r].ring_mem) {  in dma_iproc_pax_init()
719 		mem_aligned = ((uintptr_t)pd->ring[r].ring_mem +  in dma_iproc_pax_init()
723 		pd->ring[r].cmpl = (void *)mem_aligned;  in dma_iproc_pax_init()
724 		pd->ring[r].bd = (void *)(mem_aligned +  in dma_iproc_pax_init()
726 		pd->ring[r].sync_loc = (void *)((uintptr_t)pd->ring[r].bd +  in dma_iproc_pax_init()
731 			pd->ring[r].idx,  in dma_iproc_pax_init()
732 			pd->ring[r].ring_mem,  in dma_iproc_pax_init()
735 			pd->ring[r].idx,  in dma_iproc_pax_init()
736 			pd->ring[r].bd,  in dma_iproc_pax_init()
737 			pd->ring[r].cmpl,  in dma_iproc_pax_init()
738 			pd->ring[r].sync_loc);  in dma_iproc_pax_init()
741 		prepare_ring(&(pd->ring[r]));  in dma_iproc_pax_init()
766 static int dma_iproc_pax_gen_desc(struct dma_iproc_pax_ring_data *ring,  in dma_iproc_pax_gen_desc()  argument
778 		ring->current_hdr = (uintptr_t)get_curr_desc_addr(ring);  in dma_iproc_pax_gen_desc()
779 		rm_write_header_desc((void *)ring->current_hdr,  in dma_iproc_pax_gen_desc()
780 				     curr_toggle_val(ring),  in dma_iproc_pax_gen_desc()
781 				     curr_pkt_id(ring),  in dma_iproc_pax_gen_desc()
784 		ring->total_pkt_count++;  in dma_iproc_pax_gen_desc()
787 	rm_write_pcie_desc(get_curr_desc_addr(ring),  in dma_iproc_pax_gen_desc()
788 			   curr_toggle_val(ring), pci_addr);  in dma_iproc_pax_gen_desc()
790 	rm_write_src_dst_desc(get_curr_desc_addr(ring),  in dma_iproc_pax_gen_desc()
791 			      is_mega, curr_toggle_val(ring),  in dma_iproc_pax_gen_desc()
796 	hdr = (struct rm_header *)ring->current_hdr;  in dma_iproc_pax_gen_desc()
806 				     struct dma_iproc_pax_ring_data *ring,  in dma_iproc_pax_gen_packets()  argument
843 			dma_iproc_pax_gen_desc(ring, true, pci_addr,  in dma_iproc_pax_gen_packets()
853 			dma_iproc_pax_gen_desc(ring, false, pci_addr, axi_addr,  in dma_iproc_pax_gen_packets()
902 	struct dma_iproc_pax_ring_data *ring;  in wait_for_pkt_completion()  local
904 	ring = &(pd->ring[idx]);  in wait_for_pkt_completion()
906 	if (k_sem_take(&ring->alert, K_MSEC(PAX_DMA_TIMEOUT)) != 0) {  in wait_for_pkt_completion()
922 	struct dma_iproc_pax_ring_data *ring;  in dma_iproc_pax_process_dma_blocks()  local
933 	ring = &(pd->ring[idx]);  in dma_iproc_pax_process_dma_blocks()
940 	if ((ring->sync_pci.addr_lo == 0x0) &&  in dma_iproc_pax_process_dma_blocks()
941 	    (ring->sync_pci.addr_hi == 0x0)) {  in dma_iproc_pax_process_dma_blocks()
946 		ring->sync_pci.addr_lo = sync.addr_lo + idx * 4;  in dma_iproc_pax_process_dma_blocks()
947 		ring->sync_pci.addr_hi = sync.addr_hi;  in dma_iproc_pax_process_dma_blocks()
949 			ring->sync_pci.addr_hi,  in dma_iproc_pax_process_dma_blocks()
950 			ring->sync_pci.addr_lo);  in dma_iproc_pax_process_dma_blocks()
954 	ring->curr.sync_data.opaque = ring->curr.opq;  in dma_iproc_pax_process_dma_blocks()
955 	ring->curr.sync_data.total_pkts = config->block_count;  in dma_iproc_pax_process_dma_blocks()
956 	memcpy((void *)ring->sync_loc,  in dma_iproc_pax_process_dma_blocks()
957 	       (void *)&(ring->curr.sync_data), 4);  in dma_iproc_pax_process_dma_blocks()
958 	sync_pl.dest_address = ring->sync_pci.addr_lo |  in dma_iproc_pax_process_dma_blocks()
959 			   (uint64_t)ring->sync_pci.addr_hi << 32;  in dma_iproc_pax_process_dma_blocks()
960 	sync_pl.source_address = (uintptr_t)ring->sync_loc;  in dma_iproc_pax_process_dma_blocks()
964 	toggle_bit = ring->curr.toggle;  in dma_iproc_pax_process_dma_blocks()
966 	ring->curr.opq = curr_pkt_id(ring);  in dma_iproc_pax_process_dma_blocks()
970 		ret = dma_iproc_pax_gen_packets(dev, ring,  in dma_iproc_pax_process_dma_blocks()
986 	ring->non_hdr_bd_count = 0;  in dma_iproc_pax_process_dma_blocks()
987 	dma_iproc_pax_gen_packets(dev, ring, MEMORY_TO_PERIPHERAL, &sync_pl,  in dma_iproc_pax_process_dma_blocks()
990 	alloc_pkt_id(ring);  in dma_iproc_pax_process_dma_blocks()
999 	struct dma_iproc_pax_ring_data *ring;  in dma_iproc_pax_configure()  local
1007 	ring = &(pd->ring[channel]);  in dma_iproc_pax_configure()
1008 	k_mutex_lock(&ring->lock, K_FOREVER);  in dma_iproc_pax_configure()
1010 	if (ring->ring_active) {  in dma_iproc_pax_configure()
1022 	ring->ring_active = 1;  in dma_iproc_pax_configure()
1026 		ring->ring_active = 0;  in dma_iproc_pax_configure()
1030 	ring->dma_callback = cfg->dma_callback;  in dma_iproc_pax_configure()
1031 	ring->callback_arg = cfg->user_data;  in dma_iproc_pax_configure()
1033 	k_mutex_unlock(&ring->lock);  in dma_iproc_pax_configure()
1042 	struct dma_iproc_pax_ring_data *ring;  in dma_iproc_pax_transfer_start()  local
1049 	ring = &(pd->ring[channel]);  in dma_iproc_pax_transfer_start()
1050 	set_pkt_count(dev, channel, ring->total_pkt_count);  in dma_iproc_pax_transfer_start()
1059 	ret = wait_for_pkt_completion(dev, channel, ring->total_pkt_count);  in dma_iproc_pax_transfer_start()
1064 	ret = poll_on_write_sync(dev, ring);  in dma_iproc_pax_transfer_start()
1067 	k_mutex_lock(&ring->lock, K_FOREVER);  in dma_iproc_pax_transfer_start()
1068 	ring->ring_active = 0;  in dma_iproc_pax_transfer_start()
1069 	k_mutex_unlock(&ring->lock);  in dma_iproc_pax_transfer_start()