drivers/dma/dma_iproc_pax_v2.c

/*
 * Copyright 2020 Broadcom
 *
 * SPDX-License-Identifier: Apache-2.0
 */

#define DT_DRV_COMPAT brcm_iproc_pax_dma_v2

#include <zephyr/arch/cpu.h>
#include <zephyr/cache.h>
#include <errno.h>
#include <zephyr/init.h>
#include <zephyr/kernel.h>
#include <zephyr/linker/sections.h>
#include <soc.h>
#include <string.h>
#include <zephyr/toolchain.h>
#include <zephyr/types.h>
#include <zephyr/drivers/dma.h>
#include <zephyr/drivers/pcie/endpoint/pcie_ep.h>
#include "dma_iproc_pax_v2.h"

#define LOG_LEVEL CONFIG_DMA_LOG_LEVEL
#include <zephyr/logging/log.h>
#include <zephyr/irq.h>
LOG_MODULE_REGISTER(dma_iproc_pax_v2);

/* Driver runtime data for PAX DMA and RM */
static struct dma_iproc_pax_data pax_dma_data;

/**
 * @brief Opaque/packet id allocator, range 0 to 31
 */
static inline uint32_t reset_pkt_id(struct dma_iproc_pax_ring_data *ring)
{
	return ring->pkt_id = 0x0;
}

static inline uint32_t alloc_pkt_id(struct dma_iproc_pax_ring_data *ring)
{
	ring->pkt_id = (ring->pkt_id + 1) % 32;
	return ring->pkt_id;
}

static inline uint32_t curr_pkt_id(struct dma_iproc_pax_ring_data *ring)
{
	return ring->pkt_id;
}

static inline uint32_t curr_toggle_val(struct dma_iproc_pax_ring_data *ring)
{
	return ring->curr.toggle;
}

/**
 * @brief Populate header descriptor
 */
static inline void rm_write_header_desc(void *desc, uint32_t toggle,
				  uint32_t opq, uint32_t bdcount,
				  uint64_t pci_addr)
{
	struct rm_header *r = (struct rm_header *)desc;

	r->opq = opq;
	r->bdf = 0x0;
	r->res1 = 0x0;
	/* DMA descriptor count init value */
	r->bdcount = bdcount;
	r->prot = 0x0;
	r->res2 = 0x0;
	/* No packet extension, start and end set to '1' */
	r->start = 1;
	r->end = 1;
	/* RM header type */
	r->type = PAX_DMA_TYPE_RM_HEADER;
	r->pcie_addr_msb = PAX_DMA_PCI_ADDR_HI_MSB8(pci_addr);
	r->res3 = 0x0;
	r->res4 = 0x0;
#ifdef CONFIG_DMA_IPROC_PAX_TOGGLE_MODE
	r->toggle = toggle;
#elif CONFIG_DMA_IPROC_PAX_DOORBELL_MODE
	r->toggle = 0;
#endif
}

/**
 * @brief Populate pcie descriptor
 */
static inline void rm_write_pcie_desc(void *desc,
			     uint32_t toggle,
			     uint64_t pci_addr)
{
	struct pcie_desc *pcie = (struct pcie_desc *)desc;

	pcie->pcie_addr_lsb = pci_addr;
	pcie->res1 = 0x0;
	/* PCIE header type */
	pcie->type = PAX_DMA_TYPE_PCIE_DESC;
#ifdef CONFIG_DMA_IPROC_PAX_TOGGLE_MODE
	pcie->toggle = toggle;
#elif CONFIG_DMA_IPROC_PAX_DOORBELL_MODE
	pcie->toggle = 0;
#endif
}

/**
 * @brief Populate src/destination descriptor
 */
static inline void rm_write_src_dst_desc(void *desc_ptr,
				bool is_mega,
				uint32_t toggle,
				uint64_t axi_addr,
				uint32_t size,
				enum pax_dma_dir direction)
{
	struct src_dst_desc *desc;

	desc = (struct src_dst_desc *)desc_ptr;
	desc->axi_addr = axi_addr;
	desc->length = size;
#ifdef CONFIG_DMA_IPROC_PAX_TOGGLE_MODE
	desc->toggle = toggle;
#elif CONFIG_DMA_IPROC_PAX_DOORBELL_MODE
	desc->toggle = 0;
#endif

	if (direction == CARD_TO_HOST) {
		desc->type = is_mega ?
			     PAX_DMA_TYPE_MEGA_SRC_DESC : PAX_DMA_TYPE_SRC_DESC;
	} else {
		desc->type = is_mega ?
			     PAX_DMA_TYPE_MEGA_DST_DESC : PAX_DMA_TYPE_DST_DESC;
	}
}

#ifdef CONFIG_DMA_IPROC_PAX_TOGGLE_MODE
static void init_toggle(void *desc, uint32_t toggle)
{
	struct rm_header *r = (struct rm_header *)desc;

	r->toggle = toggle;
}
#endif

/**
 * @brief Return current descriptor memory address and
 *        increment to point to next descriptor memory address.
 */
static inline void *get_curr_desc_addr(struct dma_iproc_pax_ring_data *ring)
{
	struct next_ptr_desc *nxt;
	uintptr_t curr;

	curr = (uintptr_t)ring->curr.write_ptr;
	/* if hit next table ptr, skip to next location, flip toggle */
	nxt = (struct next_ptr_desc *)curr;
	if (nxt->type == PAX_DMA_TYPE_NEXT_PTR) {
		LOG_DBG("hit next_ptr@0x%lx %d, next_table@0x%lx\n",
			curr, nxt->toggle, (uintptr_t)nxt->addr);
		uintptr_t last = (uintptr_t)ring->bd +
			     PAX_DMA_RM_DESC_RING_SIZE * PAX_DMA_NUM_BD_BUFFS;
		nxt->toggle = ring->curr.toggle;
		ring->curr.toggle = (ring->curr.toggle == 0) ? 1 : 0;
		/* move to next addr, wrap around if hits end */
		curr += PAX_DMA_RM_DESC_BDWIDTH;
		if (curr == last) {
			curr = (uintptr_t)ring->bd;
			LOG_DBG("hit end of desc:0x%lx, wrap to 0x%lx\n",
				last, curr);
		}
		ring->descs_inflight++;
	}

	ring->curr.write_ptr = (void *)(curr + PAX_DMA_RM_DESC_BDWIDTH);
	ring->descs_inflight++;

	return (void *)curr;
}

/**
 * @brief Populate next ptr descriptor
 */
static void rm_write_next_table_desc(void *desc, void *next_ptr,
				     uint32_t toggle)
{
	struct next_ptr_desc *nxt = (struct next_ptr_desc *)desc;

	nxt->addr = (uintptr_t)next_ptr;
	nxt->type = PAX_DMA_TYPE_NEXT_PTR;
	nxt->toggle = toggle;
}

static void prepare_ring(struct dma_iproc_pax_ring_data *ring)
{
	uintptr_t curr, next, last;
	int buff_count = PAX_DMA_NUM_BD_BUFFS;
#ifdef CONFIG_DMA_IPROC_PAX_TOGGLE_MODE
	uint32_t toggle;
#endif

	/* zero out descriptor area */
	memset(ring->bd, 0x0, PAX_DMA_RM_DESC_RING_SIZE * PAX_DMA_NUM_BD_BUFFS);
	memset(ring->cmpl, 0x0, PAX_DMA_RM_CMPL_RING_SIZE);

	/* start with first buffer, valid toggle is 0x1 */
#ifdef CONFIG_DMA_IPROC_PAX_TOGGLE_MODE
	toggle = 0x1;
#endif
	curr = (uintptr_t)ring->bd;
	next = curr + PAX_DMA_RM_DESC_RING_SIZE;
	last = curr + PAX_DMA_RM_DESC_RING_SIZE * PAX_DMA_NUM_BD_BUFFS;
	do {
#ifdef CONFIG_DMA_IPROC_PAX_TOGGLE_MODE
		init_toggle((void *)curr, toggle);
		/* Place next_table desc as last BD entry on each buffer */
		rm_write_next_table_desc(PAX_DMA_NEXT_TBL_ADDR((void *)curr),
					 (void *)next, toggle);
#elif CONFIG_DMA_IPROC_PAX_DOORBELL_MODE
		/* Place next_table desc as last BD entry on each buffer */
		rm_write_next_table_desc(PAX_DMA_NEXT_TBL_ADDR((void *)curr),
					 (void *)next, 0);
#endif

#ifdef CONFIG_DMA_IPROC_PAX_TOGGLE_MODE
		/* valid toggle flips for each buffer */
		toggle = toggle ? 0x0 : 0x1;
#endif
		curr += PAX_DMA_RM_DESC_RING_SIZE;
		next += PAX_DMA_RM_DESC_RING_SIZE;
		/* last entry, chain back to first buffer */
		if (next == last) {
			next = (uintptr_t)ring->bd;
		}

	} while (--buff_count);

	dma_mb();

	/* start programming from first RM header */
	ring->curr.write_ptr = ring->bd;
	/* valid toggle starts with 1 after reset */
	ring->curr.toggle = 1;
	/* completion read offset */
	ring->curr.cmpl_rd_offs = 0;
	/* inflight descs */
	ring->descs_inflight = 0;

	/* init sync data for the ring */
	ring->curr.sync_data.signature = PAX_DMA_WRITE_SYNC_SIGNATURE;
	ring->curr.sync_data.ring = ring->idx;
	/* pkt id for active dma xfer */
	ring->curr.sync_data.opaque = 0x0;
	/* pkt count for active dma xfer */
	ring->curr.sync_data.total_pkts = 0x0;
}

static int init_rm(struct dma_iproc_pax_data *pd)
{
	int ret = -ETIMEDOUT, timeout = 1000;

	k_mutex_lock(&pd->dma_lock, K_FOREVER);
	/* Wait for Ring Manager ready */
	do {
		LOG_DBG("Waiting for RM HW init\n");
		if ((sys_read32(RM_COMM_REG(pd, RM_COMM_MAIN_HW_INIT_DONE)) &
		    RM_COMM_MAIN_HW_INIT_DONE_MASK)) {
			ret = 0;
			break;
		}
		k_sleep(K_MSEC(1));
	} while (--timeout);
	k_mutex_unlock(&pd->dma_lock);

	if (!timeout) {
		LOG_WRN("RM HW Init timedout!\n");
	} else {
		LOG_INF("PAX DMA RM HW Init Done\n");
	}

	return ret;
}

static void rm_cfg_start(struct dma_iproc_pax_data *pd)
{
	uint32_t val;

	k_mutex_lock(&pd->dma_lock, K_FOREVER);

	/* set config done 0, enable toggle mode */
	val = sys_read32(RM_COMM_REG(pd, RM_COMM_CONTROL));
	val &= ~RM_COMM_CONTROL_CONFIG_DONE;
	sys_write32(val, RM_COMM_REG(pd, RM_COMM_CONTROL));

	val &= ~(RM_COMM_CONTROL_MODE_MASK << RM_COMM_CONTROL_MODE_SHIFT);

#ifdef CONFIG_DMA_IPROC_PAX_DOORBELL_MODE
	val |= (RM_COMM_CONTROL_MODE_DOORBELL <<
		RM_COMM_CONTROL_MODE_SHIFT);
#elif CONFIG_DMA_IPROC_PAX_TOGGLE_MODE
	val |= (RM_COMM_CONTROL_MODE_ALL_BD_TOGGLE <<
		RM_COMM_CONTROL_MODE_SHIFT);
#endif
	sys_write32(val, RM_COMM_REG(pd, RM_COMM_CONTROL));
	sys_write32(RM_COMM_MSI_DISABLE_MASK,
		    RM_COMM_REG(pd, RM_COMM_MSI_DISABLE));

	val = sys_read32(RM_COMM_REG(pd, RM_COMM_AXI_READ_BURST_THRESHOLD));
	val &= ~(RM_COMM_THRESHOLD_CFG_RD_FIFO_MAX_THRESHOLD_MASK <<
		 RM_COMM_THRESHOLD_CFG_RD_FIFO_MAX_THRESHOLD_SHIFT);
	val |= RM_COMM_THRESHOLD_CFG_RD_FIFO_MAX_THRESHOLD_SHIFT_VAL <<
	       RM_COMM_THRESHOLD_CFG_RD_FIFO_MAX_THRESHOLD_SHIFT;
	sys_write32(val, RM_COMM_REG(pd, RM_COMM_AXI_READ_BURST_THRESHOLD));

	val = sys_read32(RM_COMM_REG(pd, RM_COMM_FIFO_FULL_THRESHOLD));
	val &= ~(RM_COMM_PKT_ALIGNMENT_BD_FIFO_FULL_THRESHOLD_MASK <<
		 RM_COMM_PKT_ALIGNMENT_BD_FIFO_FULL_THRESHOLD_SHIFT);
	val |= RM_COMM_PKT_ALIGNMENT_BD_FIFO_FULL_THRESHOLD_VAL <<
	       RM_COMM_PKT_ALIGNMENT_BD_FIFO_FULL_THRESHOLD_SHIFT;

	val &= ~(RM_COMM_BD_FIFO_FULL_THRESHOLD_MASK <<
		 RM_COMM_BD_FIFO_FULL_THRESHOLD_SHIFT);
	val |= RM_COMM_BD_FIFO_FULL_THRESHOLD_VAL <<
	       RM_COMM_BD_FIFO_FULL_THRESHOLD_SHIFT;
	sys_write32(val, RM_COMM_REG(pd, RM_COMM_FIFO_FULL_THRESHOLD));

	/* Enable Line interrupt */
	val = sys_read32(RM_COMM_REG(pd, RM_COMM_CONTROL));
	val |= RM_COMM_CONTROL_LINE_INTR_EN;
	sys_write32(val, RM_COMM_REG(pd, RM_COMM_CONTROL));

	/* Enable AE_TIMEOUT */
	sys_write32(RM_COMM_AE_TIMEOUT_VAL,
		    RM_COMM_REG(pd, RM_COMM_AE_TIMEOUT));
	val = sys_read32(RM_COMM_REG(pd, RM_COMM_CONTROL));
	val |= RM_COMM_CONTROL_AE_TIMEOUT_EN;
	sys_write32(val, RM_COMM_REG(pd, RM_COMM_CONTROL));

	/* AE (Acceleration Engine) grouping to group '0' */
	val = sys_read32(RM_COMM_REG(pd, RM_AE0_AE_CONTROL));
	val &= ~RM_AE_CTRL_AE_GROUP_MASK;
	sys_write32(val, RM_COMM_REG(pd, RM_AE0_AE_CONTROL));
	val |= RM_AE_CONTROL_ACTIVE;
	sys_write32(val, RM_COMM_REG(pd, RM_AE0_AE_CONTROL));

	/* AXI read/write channel enable */
	val = sys_read32(RM_COMM_REG(pd, RM_COMM_AXI_CONTROL));
	val |= (RM_COMM_AXI_CONTROL_RD_CH_EN | RM_COMM_AXI_CONTROL_WR_CH_EN);
	sys_write32(val, RM_COMM_REG(pd, RM_COMM_AXI_CONTROL));

	/* Tune RM control programming for 4 rings */
	sys_write32(RM_COMM_TIMER_CONTROL0_VAL,
		    RM_COMM_REG(pd, RM_COMM_TIMER_CONTROL_0));
	sys_write32(RM_COMM_TIMER_CONTROL1_VAL,
		    RM_COMM_REG(pd, RM_COMM_TIMER_CONTROL_1));
	val = sys_read32(RM_COMM_REG(pd, RM_COMM_BURST_LENGTH));
	val |= RM_COMM_BD_FETCH_CACHE_ALIGNED_DISABLED;
	val |= RM_COMM_VALUE_FOR_DDR_ADDR_GEN_VAL <<
	       RM_COMM_VALUE_FOR_DDR_ADDR_GEN_SHIFT;
	val |= RM_COMM_VALUE_FOR_TOGGLE_VAL << RM_COMM_VALUE_FOR_TOGGLE_SHIFT;
	sys_write32(val, RM_COMM_REG(pd, RM_COMM_BURST_LENGTH));

	val = sys_read32(RM_COMM_REG(pd, RM_COMM_BD_FETCH_MODE_CONTROL));
	val |= RM_COMM_DISABLE_GRP_BD_FIFO_FLOW_CONTROL_FOR_PKT_ALIGNMENT;
	val |= RM_COMM_DISABLE_PKT_ALIGNMENT_BD_FIFO_FLOW_CONTROL;
	sys_write32(val, RM_COMM_REG(pd, RM_COMM_BD_FETCH_MODE_CONTROL));

	/* Set Sequence max count to the max supported value */
	val = sys_read32(RM_COMM_REG(pd, RM_COMM_MASK_SEQUENCE_MAX_COUNT));
	val = (val | RING_MASK_SEQ_MAX_COUNT_MASK);
	sys_write32(val, RM_COMM_REG(pd, RM_COMM_MASK_SEQUENCE_MAX_COUNT));

	k_mutex_unlock(&pd->dma_lock);
}

static void rm_ring_clear_stats(struct dma_iproc_pax_data *pd,
				enum ring_idx idx)
{
	/* Read ring Tx, Rx, and Outstanding counts to clear */
	sys_read32(RM_RING_REG(pd, idx, RING_NUM_REQ_RECV_LS));
	sys_read32(RM_RING_REG(pd, idx, RING_NUM_REQ_RECV_MS));
	sys_read32(RM_RING_REG(pd, idx, RING_NUM_REQ_TRANS_LS));
	sys_read32(RM_RING_REG(pd, idx, RING_NUM_REQ_TRANS_MS));
	sys_read32(RM_RING_REG(pd, idx, RING_NUM_REQ_OUTSTAND));
}

static void rm_cfg_finish(struct dma_iproc_pax_data *pd)
{
	uint32_t val;

	k_mutex_lock(&pd->dma_lock, K_FOREVER);

	/* set Ring config done */
	val = sys_read32(RM_COMM_REG(pd, RM_COMM_CONTROL));
	val |= RM_COMM_CONTROL_CONFIG_DONE;
	sys_write32(val, RM_COMM_REG(pd, RM_COMM_CONTROL));

	k_mutex_unlock(&pd->dma_lock);
}

static inline void write_doorbell(struct dma_iproc_pax_data *pd,
				  enum ring_idx idx)
{
	struct dma_iproc_pax_ring_data *ring = &(pd->ring[idx]);

	sys_write32(ring->descs_inflight,
		    RM_RING_REG(pd, idx, RING_DOORBELL_BD_WRITE_COUNT));
	ring->descs_inflight = 0;
}

static inline void set_ring_active(struct dma_iproc_pax_data *pd,
				   enum ring_idx idx,
				   bool active)
{
	uint32_t val;

	val = sys_read32(RM_RING_REG(pd, idx, RING_CONTROL));
	if (active) {
		val |= RING_CONTROL_ACTIVE;
	} else {
		val &= ~RING_CONTROL_ACTIVE;
	}
	sys_write32(val, RM_RING_REG(pd, idx, RING_CONTROL));
}

static int init_ring(struct dma_iproc_pax_data *pd, enum ring_idx idx)
{
	uint32_t val;
	uintptr_t desc = (uintptr_t)pd->ring[idx].bd;
	uintptr_t cmpl = (uintptr_t)pd->ring[idx].cmpl;
	int timeout = 5000, ret = 0;

	k_mutex_lock(&pd->dma_lock, K_FOREVER);

	/*  Read cmpl write ptr incase previous dma stopped */
	sys_read32(RM_RING_REG(pd, idx, RING_CMPL_WRITE_PTR));

	/* Inactivate ring */
	sys_write32(0x0, RM_RING_REG(pd, idx, RING_CONTROL));

	/* set Ring config done */
	val = sys_read32(RM_COMM_REG(pd, RM_COMM_CONTROL));
	val |= RM_COMM_CONTROL_CONFIG_DONE;
	sys_write32(val,  RM_COMM_REG(pd, RM_COMM_CONTROL));
	/* Flush ring before loading new descriptor */
	sys_write32(RING_CONTROL_FLUSH, RM_RING_REG(pd, idx,
						    RING_CONTROL));
	do {
		if (sys_read32(RM_RING_REG(pd, idx, RING_FLUSH_DONE)) & RING_FLUSH_DONE_MASK) {
			break;
		}
		k_busy_wait(1);
	} while (--timeout);

	if (!timeout) {
		LOG_WRN("Ring %d flush timedout!\n", idx);
		ret = -ETIMEDOUT;
		goto err;
	}

	/* clear ring after flush */
	sys_write32(0x0, RM_RING_REG(pd, idx, RING_CONTROL));

	/* Clear Ring config done */
	val = sys_read32(RM_COMM_REG(pd, RM_COMM_CONTROL));
	val &= ~(RM_COMM_CONTROL_CONFIG_DONE);
	sys_write32(val,  RM_COMM_REG(pd, RM_COMM_CONTROL));
	/* ring group id set to '0' */
	val = sys_read32(RM_COMM_REG(pd, RM_COMM_CTRL_REG(idx)));
	val &= ~RING_COMM_CTRL_AE_GROUP_MASK;
	sys_write32(val, RM_COMM_REG(pd, RM_COMM_CTRL_REG(idx)));

	/* DDR update control, set timeout value */
	val = RING_DDR_CONTROL_COUNT(RING_DDR_CONTROL_COUNT_VAL) |
	      RING_DDR_CONTROL_TIMER(RING_DDR_CONTROL_TIMER_VAL) |
	      RING_DDR_CONTROL_ENABLE;

	sys_write32(val, RM_RING_REG(pd, idx, RING_CMPL_WR_PTR_DDR_CONTROL));
	/* Disable Ring MSI Timeout */
	sys_write32(RING_DISABLE_MSI_TIMEOUT_VALUE,
		    RM_RING_REG(pd, idx, RING_DISABLE_MSI_TIMEOUT));

	/* BD and CMPL desc queue start address */
	sys_write32((uint32_t)desc, RM_RING_REG(pd, idx, RING_BD_START_ADDR));
	sys_write32((uint32_t)cmpl, RM_RING_REG(pd, idx, RING_CMPL_START_ADDR));
	val = sys_read32(RM_RING_REG(pd, idx, RING_BD_READ_PTR));

	/* keep ring inactive after init to avoid BD poll */
#ifdef CONFIG_DMA_IPROC_PAX_TOGGLE_MODE
	set_ring_active(pd, idx, false);
#elif CONFIG_DMA_IPROC_PAX_DOORBELL_MODE
	set_ring_active(pd, idx, true);
#endif

#if !defined(CONFIG_DMA_IPROC_PAX_POLL_MODE)
	/* Enable ring completion interrupt */
	sys_write32(0x0, RM_RING_REG(pd, idx,
				     RING_COMPLETION_INTERRUPT_STAT_MASK));
#endif
	rm_ring_clear_stats(pd, idx);
err:
	k_mutex_unlock(&pd->dma_lock);

	return ret;
}

static int poll_on_write_sync(const struct device *dev,
			      struct dma_iproc_pax_ring_data *ring)
{
	const struct dma_iproc_pax_cfg *cfg = dev->config;
	struct dma_iproc_pax_write_sync_data sync_rd, *recv, *sent;
	uint64_t pci_addr;
	uint32_t *pci32, *axi32;
	uint32_t zero_init = 0, timeout = PAX_DMA_MAX_SYNC_WAIT;
	int ret;

	recv = &sync_rd;
	sent = &(ring->curr.sync_data);
	/* form host pci sync address */
	pci32 = (uint32_t *)&pci_addr;
	pci32[0] = ring->sync_pci.addr_lo;
	pci32[1] = ring->sync_pci.addr_hi;
	axi32 = (uint32_t *)&sync_rd;

	do {
		ret = pcie_ep_xfer_data_memcpy(cfg->pcie_dev, pci_addr,
					       (uintptr_t *)axi32, 4,
					       PCIE_OB_LOWMEM, HOST_TO_DEVICE);

		if (memcmp((void *)recv, (void *)sent, 4) == 0) {
			/* clear the sync word */
			ret = pcie_ep_xfer_data_memcpy(cfg->pcie_dev, pci_addr,
						       (uintptr_t *)&zero_init,
						       4, PCIE_OB_LOWMEM,
						       DEVICE_TO_HOST);
			dma_mb();
			ret = 0;
			break;
		}
		k_busy_wait(1);
	} while (--timeout);

	if (!timeout) {
		LOG_ERR("[ring %d]: not recvd write sync!\n", ring->idx);
		ret = -ETIMEDOUT;
	}

	return ret;
}

static int process_cmpl_event(const struct device *dev,
			      enum ring_idx idx, uint32_t pl_len)
{
	struct dma_iproc_pax_data *pd = dev->data;
	uint32_t wr_offs, rd_offs, ret = 0;
	struct dma_iproc_pax_ring_data *ring = &(pd->ring[idx]);
	struct cmpl_pkt *c;
	uint32_t is_outstanding;

	/* cmpl read offset, unprocessed cmpl location */
	rd_offs = ring->curr.cmpl_rd_offs;

	wr_offs = sys_read32(RM_RING_REG(pd, idx,
					 RING_CMPL_WRITE_PTR));

	/* Update read ptr to "processed" */
	ring->curr.cmpl_rd_offs = wr_offs;

	/*
	 * Ensure consistency of completion descriptor
	 * The completion desc is updated by RM via AXI stream
	 * CPU need to ensure the memory operations are completed
	 * before reading cmpl area, by a "dsb"
	 * If Dcache enabled, need to invalidate the cachelines to
	 * read updated cmpl desc. The cache API also issues dsb.
	 */
	dma_mb();

	/* Decode cmpl pkt id to verify */
	c = (struct cmpl_pkt *)((uintptr_t)ring->cmpl +
	    PAX_DMA_CMPL_DESC_SIZE * PAX_DMA_CURR_CMPL_IDX(wr_offs));

	LOG_DBG("RING%d WR_PTR:%d opq:%d, rm_status:%x dma_status:%x\n",
		idx, wr_offs, c->opq, c->rm_status, c->dma_status);

	is_outstanding = sys_read32(RM_RING_REG(pd, idx,
						RING_NUM_REQ_OUTSTAND));
	if ((ring->curr.opq != c->opq) && (is_outstanding != 0)) {
		LOG_ERR("RING%d: pkt id should be %d, rcvd %d outst=%d\n",
			idx, ring->curr.opq, c->opq, is_outstanding);
		ret = -EIO;
	}
	/* check for completion AE timeout */
	if (c->rm_status == RM_COMPLETION_AE_TIMEOUT) {
		LOG_ERR("RING%d WR_PTR:%d rm_status:%x AE Timeout!\n",
			idx, wr_offs, c->rm_status);
		/* TBD: Issue full card reset to restore operations */
		LOG_ERR("Needs Card Reset to recover!\n");
		ret = -ETIMEDOUT;
	}

	if (ring->dma_callback) {
		ring->dma_callback(dev, ring->callback_arg, idx, ret);
	}

	/* clear total packet count and non header bd count */
	ring->total_pkt_count = 0;

	return ret;
}

#ifdef CONFIG_DMA_IPROC_PAX_POLL_MODE
static int peek_ring_cmpl(const struct device *dev,
			  enum ring_idx idx, uint32_t pl_len)
{
	struct dma_iproc_pax_data *pd = dev->data;
	uint32_t wr_offs, rd_offs, timeout = PAX_DMA_MAX_POLL_WAIT;
	struct dma_iproc_pax_ring_data *ring = &(pd->ring[idx]);

	/* cmpl read offset, unprocessed cmpl location */
	rd_offs = ring->curr.cmpl_rd_offs;

	/* poll write_ptr until cmpl received for all buffers */
	do {
		wr_offs = sys_read32(RM_RING_REG(pd, idx,
						 RING_CMPL_WRITE_PTR));
		if (PAX_DMA_GET_CMPL_COUNT(wr_offs, rd_offs) >= pl_len)
			break;
		k_busy_wait(1);
	} while (--timeout);

	if (timeout == 0) {
		LOG_ERR("RING%d timeout, rcvd %d, expected %d!\n",
			idx, PAX_DMA_GET_CMPL_COUNT(wr_offs, rd_offs), pl_len);
		/* More debug info on current dma instance */
		LOG_ERR("WR_PTR:%x RD_PTR%x\n", wr_offs, rd_offs);
		return  -ETIMEDOUT;
	}

	return process_cmpl_event(dev, idx, pl_len);
}
#else
static void rm_isr(const struct device *dev)
{
	uint32_t status, err_stat, idx;
	struct dma_iproc_pax_data *pd = dev->data;

	err_stat =
	sys_read32(RM_COMM_REG(pd,
			       RM_COMM_AE_INTERFACE_GROUP_0_INTERRUPT_MASK));
	sys_write32(err_stat,
		    RM_COMM_REG(pd,
				RM_COMM_AE_INTERFACE_GROUP_0_INTERRUPT_CLEAR));

	/* alert waiting thread to process, for each completed ring */
	for (idx = PAX_DMA_RING0; idx < PAX_DMA_RINGS_MAX; idx++) {
		status =
		sys_read32(RM_RING_REG(pd, idx,
				       RING_COMPLETION_INTERRUPT_STAT));
		sys_write32(status,
			    RM_RING_REG(pd, idx,
					RING_COMPLETION_INTERRUPT_STAT_CLEAR));
		if (status & 0x1) {
			k_sem_give(&pd->ring[idx].alert);
		}
	}
}
#endif

static int dma_iproc_pax_init(const struct device *dev)
{
	const struct dma_iproc_pax_cfg *cfg = dev->config;
	struct dma_iproc_pax_data *pd = dev->data;
	int r;
	uintptr_t mem_aligned;

	if (!device_is_ready(cfg->pcie_dev)) {
		LOG_ERR("PCIe device not ready");
		return -ENODEV;
	}

	pd->dma_base = cfg->dma_base;
	pd->rm_comm_base = cfg->rm_comm_base;
	pd->used_rings = (cfg->use_rings < PAX_DMA_RINGS_MAX) ?
			 cfg->use_rings : PAX_DMA_RINGS_MAX;

	/* dma/rm access lock */
	k_mutex_init(&pd->dma_lock);

	/* Ring Manager H/W init */
	if (init_rm(pd)) {
		return -ETIMEDOUT;
	}

	/* common rm config */
	rm_cfg_start(pd);

	/* individual ring config */
	for (r = 0; r < pd->used_rings; r++) {
		/* per-ring mutex lock */
		k_mutex_init(&pd->ring[r].lock);
		/* Init alerts */
		k_sem_init(&pd->ring[r].alert, 0, 1);

		pd->ring[r].idx = r;
		pd->ring[r].ring_base = cfg->rm_base +
					PAX_DMA_RING_ADDR_OFFSET(r);
		LOG_DBG("RING%d,VERSION:0x%x\n", pd->ring[r].idx,
			sys_read32(RM_RING_REG(pd, r, RING_VER)));

		/* Allocate for 2 BD buffers + cmpl buffer + sync location */
		pd->ring[r].ring_mem = (void *)((uintptr_t)cfg->bd_memory_base +
					r * PAX_DMA_PER_RING_ALLOC_SIZE);
		if (!pd->ring[r].ring_mem) {
			LOG_ERR("RING%d failed to alloc desc memory!\n", r);
			return -ENOMEM;
		}
		/* Find 8K aligned address within allocated region */
		mem_aligned = ((uintptr_t)pd->ring[r].ring_mem +
			       PAX_DMA_RING_ALIGN - 1) &
			       ~(PAX_DMA_RING_ALIGN - 1);

		pd->ring[r].cmpl = (void *)mem_aligned;
		pd->ring[r].bd = (void *)(mem_aligned +
					  PAX_DMA_RM_CMPL_RING_SIZE);
		pd->ring[r].sync_loc = (void *)((uintptr_t)pd->ring[r].bd +
				      PAX_DMA_RM_DESC_RING_SIZE *
				      PAX_DMA_NUM_BD_BUFFS);

		LOG_DBG("Ring%d,allocated Mem:0x%p Size %d\n",
			pd->ring[r].idx,
			pd->ring[r].ring_mem,
			PAX_DMA_PER_RING_ALLOC_SIZE);
		LOG_DBG("Ring%d,BD:0x%p, CMPL:0x%p, SYNC_LOC:0x%p\n",
			pd->ring[r].idx,
			pd->ring[r].bd,
			pd->ring[r].cmpl,
			pd->ring[r].sync_loc);

		/* Prepare ring desc table */
		prepare_ring(&(pd->ring[r]));

		/* initialize ring */
		init_ring(pd, r);
	}

	/* set ring config done */
	rm_cfg_finish(pd);

#ifndef CONFIG_DMA_IPROC_PAX_POLL_MODE
	/* Register and enable RM interrupt */
	IRQ_CONNECT(DT_INST_IRQN(0),
		    DT_INST_IRQ(0, priority),
		    rm_isr,
		    DEVICE_DT_INST_GET(0),
		    0);
	irq_enable(DT_INST_IRQN(0));
#else
	LOG_INF("%s PAX DMA rings in poll mode!\n", dev->name);
#endif
	LOG_INF("%s RM setup %d rings\n", dev->name, pd->used_rings);

	return 0;
}

static int dma_iproc_pax_gen_desc(struct dma_iproc_pax_ring_data *ring,
				  bool is_mega,
				  uint64_t pci_addr,
				  uint64_t axi_addr,
				  uint32_t length,
				  enum pax_dma_dir dir,
				  uint32_t *non_hdr_bd_count)
{
	struct rm_header *hdr;

	if (*non_hdr_bd_count == 0) {
		/* Generate Header BD */
		ring->current_hdr = (uintptr_t)get_curr_desc_addr(ring);
		rm_write_header_desc((void *)ring->current_hdr,
				     curr_toggle_val(ring),
				     curr_pkt_id(ring),
				     PAX_DMA_RM_DESC_BDCOUNT,
				     pci_addr);
		ring->total_pkt_count++;
	}

	rm_write_pcie_desc(get_curr_desc_addr(ring),
			   curr_toggle_val(ring), pci_addr);
	*non_hdr_bd_count = *non_hdr_bd_count + 1;
	rm_write_src_dst_desc(get_curr_desc_addr(ring),
			      is_mega, curr_toggle_val(ring),
			      axi_addr, length, dir);
	*non_hdr_bd_count = *non_hdr_bd_count + 1;

	/* Update Header BD with bd count */
	hdr = (struct rm_header *)ring->current_hdr;
	hdr->bdcount = *non_hdr_bd_count;
	if (*non_hdr_bd_count == MAX_BD_COUNT_PER_HEADER) {
		*non_hdr_bd_count = 0;
	}

	return 0;
}

static int dma_iproc_pax_gen_packets(const struct device *dev,
				     struct dma_iproc_pax_ring_data *ring,
				     uint32_t direction,
				     struct dma_block_config *config,
				     uint32_t *non_hdr_bd_count)
{
	uint32_t outstanding, remaining_len;
	uint32_t offset, curr, mega_len;
	uint64_t axi_addr;
	uint64_t pci_addr;
	enum pax_dma_dir dir;

	switch (direction) {
	case MEMORY_TO_PERIPHERAL:
		pci_addr = config->dest_address;
		axi_addr = config->source_address;
		dir = CARD_TO_HOST;
		break;
	case PERIPHERAL_TO_MEMORY:
		axi_addr = config->dest_address;
		pci_addr = config->source_address;
		dir = HOST_TO_CARD;
		break;
	default:
		LOG_ERR("not supported transfer direction");
		return -EINVAL;
	}

	outstanding = config->block_size;
	offset = 0;
	while (outstanding) {
		curr = MIN(outstanding, PAX_DMA_MAX_SZ_PER_BD);
		mega_len = curr / PAX_DMA_MEGA_LENGTH_MULTIPLE;
		remaining_len = curr % PAX_DMA_MEGA_LENGTH_MULTIPLE;
		pci_addr = pci_addr + offset;
		axi_addr = axi_addr + offset;

		if (mega_len) {
			dma_iproc_pax_gen_desc(ring, true, pci_addr,
					       axi_addr, mega_len, dir,
					       non_hdr_bd_count);
			offset = offset + mega_len *
				PAX_DMA_MEGA_LENGTH_MULTIPLE;
		}

		if (remaining_len) {
			pci_addr = pci_addr + offset;
			axi_addr = axi_addr + offset;
			dma_iproc_pax_gen_desc(ring, false, pci_addr, axi_addr,
					       remaining_len, dir,
					       non_hdr_bd_count);
			offset = offset + remaining_len;
		}

		outstanding =  outstanding - curr;
	}

	return 0;
}

#ifdef CONFIG_DMA_IPROC_PAX_POLL_MODE
static void set_pkt_count(const struct device *dev,
			  enum ring_idx idx,
			  uint32_t pl_len)
{
	/* Nothing needs to be programmed here in poll mode */
}

static int wait_for_pkt_completion(const struct device *dev,
				   enum ring_idx idx,
				   uint32_t pl_len)
{
	/* poll for completion */
	return peek_ring_cmpl(dev, idx, pl_len);
}
#else
static void set_pkt_count(const struct device *dev,
			  enum ring_idx idx,
			  uint32_t pl_len)
{
	struct dma_iproc_pax_data *pd = dev->data;
	uint32_t val;

	/* program packet count for interrupt assertion */
	val = sys_read32(RM_RING_REG(pd, idx,
				     RING_CMPL_WR_PTR_DDR_CONTROL));
	val &= ~RING_DDR_CONTROL_COUNT_MASK;
	val |= RING_DDR_CONTROL_COUNT(pl_len);
	sys_write32(val, RM_RING_REG(pd, idx,
				     RING_CMPL_WR_PTR_DDR_CONTROL));
}

static int wait_for_pkt_completion(const struct device *dev,
				   enum ring_idx idx,
				   uint32_t pl_len)
{
	struct dma_iproc_pax_data *pd = dev->data;
	struct dma_iproc_pax_ring_data *ring;

	ring = &(pd->ring[idx]);
	/* wait for sg dma completion alert */
	if (k_sem_take(&ring->alert, K_MSEC(PAX_DMA_TIMEOUT)) != 0) {
		LOG_ERR("PAX DMA [ring %d] Timeout!\n", idx);
		return -ETIMEDOUT;
	}

	return process_cmpl_event(dev, idx, pl_len);
}
#endif

static int dma_iproc_pax_process_dma_blocks(const struct device *dev,
					    enum ring_idx idx,
					    struct dma_config *config)
{
	struct dma_iproc_pax_data *pd = dev->data;
	const struct dma_iproc_pax_cfg *cfg = dev->config;
	int ret = 0;
	struct dma_iproc_pax_ring_data *ring;
	uint32_t toggle_bit, non_hdr_bd_count = 0;
	struct dma_block_config sync_pl;
	struct dma_iproc_pax_addr64 sync;
	struct dma_block_config *block_config = config->head_block;

	if (block_config == NULL) {
		LOG_ERR("head_block is NULL\n");
		return -EINVAL;
	}

	ring = &(pd->ring[idx]);

	/*
	 * Host sync buffer isn't ready at zephyr/driver init-time
	 * Read the host address location once at first DMA write
	 * on that ring.
	 */
	if ((ring->sync_pci.addr_lo == 0x0) &&
	    (ring->sync_pci.addr_hi == 0x0)) {
		/* populate sync data location */
		LOG_DBG("sync addr loc 0x%x\n", cfg->scr_addr_loc);
		sync.addr_lo = sys_read32(cfg->scr_addr_loc + 4);
		sync.addr_hi = sys_read32(cfg->scr_addr_loc);
		ring->sync_pci.addr_lo = sync.addr_lo + idx * 4;
		ring->sync_pci.addr_hi = sync.addr_hi;
		LOG_DBG("ring:%d,sync addr:0x%x.0x%x\n", idx,
			ring->sync_pci.addr_hi,
			ring->sync_pci.addr_lo);
	}

	/* account extra sync packet */
	ring->curr.sync_data.opaque = ring->curr.opq;
	ring->curr.sync_data.total_pkts = config->block_count;
	memcpy((void *)ring->sync_loc,
	       (void *)&(ring->curr.sync_data), 4);
	sync_pl.dest_address = ring->sync_pci.addr_lo |
			   (uint64_t)ring->sync_pci.addr_hi << 32;
	sync_pl.source_address = (uintptr_t)ring->sync_loc;
	sync_pl.block_size = 4; /* 4-bytes */

	/* current toggle bit */
	toggle_bit = ring->curr.toggle;
	/* current opq value for cmpl check */
	ring->curr.opq = curr_pkt_id(ring);

	/* Form descriptors for total block counts */
	while (block_config != NULL) {
		ret = dma_iproc_pax_gen_packets(dev, ring,
						config->channel_direction,
						block_config,
						&non_hdr_bd_count);
		if (ret) {
			goto err;
		}
		block_config = block_config->next_block;
	}

	/*
	 * Write sync payload descriptors should go with separate RM header
	 * as RM implementation allows all the BD's in a header packet should
	 * have same data transfer direction. Setting non_hdr_bd_count to 0,
	 * helps generate separate packet.
	 */
	ring->non_hdr_bd_count = 0;
	dma_iproc_pax_gen_packets(dev, ring, MEMORY_TO_PERIPHERAL, &sync_pl,
				  &non_hdr_bd_count);

	alloc_pkt_id(ring);
err:
	return ret;
}

static int dma_iproc_pax_configure(const struct device *dev, uint32_t channel,
				   struct dma_config *cfg)
{
	struct dma_iproc_pax_data *pd = dev->data;
	struct dma_iproc_pax_ring_data *ring;
	int ret = 0;

	if (channel >= PAX_DMA_RINGS_MAX) {
		LOG_ERR("Invalid ring/channel %d\n", channel);
		return -EINVAL;
	}

	ring = &(pd->ring[channel]);
	k_mutex_lock(&ring->lock, K_FOREVER);

	if (ring->ring_active) {
		ret = -EBUSY;
		goto err;
	}

	if (cfg->block_count >= RM_V2_MAX_BLOCK_COUNT) {
		LOG_ERR("Dma block count[%d] supported exceeds limit[%d]\n",
			cfg->block_count, RM_V2_MAX_BLOCK_COUNT);
		ret = -ENOTSUP;
		goto err;
	}

	ring->ring_active = 1;
	ret = dma_iproc_pax_process_dma_blocks(dev, channel, cfg);

	if (ret) {
		ring->ring_active = 0;
		goto err;
	}

	ring->dma_callback = cfg->dma_callback;
	ring->callback_arg = cfg->user_data;
err:
	k_mutex_unlock(&ring->lock);
	return ret;
}

static int dma_iproc_pax_transfer_start(const struct device *dev,
					uint32_t channel)
{
	int ret = 0;
	struct dma_iproc_pax_data *pd = dev->data;
	struct dma_iproc_pax_ring_data *ring;

	if (channel >= PAX_DMA_RINGS_MAX) {
		LOG_ERR("Invalid ring %d\n", channel);
		return -EINVAL;
	}

	ring = &(pd->ring[channel]);
	set_pkt_count(dev, channel, ring->total_pkt_count);

#ifdef CONFIG_DMA_IPROC_PAX_DOORBELL_MODE
	write_doorbell(pd, channel);
#elif CONFIG_DMA_IPROC_PAX_TOGGLE_MODE
	/* activate the ring */
	set_ring_active(pd, channel, true);
#endif

	ret = wait_for_pkt_completion(dev, channel, ring->total_pkt_count);
	if (ret) {
		goto err_ret;
	}

	ret = poll_on_write_sync(dev, ring);

err_ret:
	k_mutex_lock(&ring->lock, K_FOREVER);
	ring->ring_active = 0;
	k_mutex_unlock(&ring->lock);

#ifdef CONFIG_DMA_IPROC_PAX_TOGGLE_MODE
	/* deactivate the ring until next active transfer */
	set_ring_active(pd, channel, false);
#endif
	return ret;
}

static int dma_iproc_pax_transfer_stop(const struct device *dev,
				       uint32_t channel)
{
	return 0;
}

static const struct dma_driver_api pax_dma_driver_api = {
	.config = dma_iproc_pax_configure,
	.start = dma_iproc_pax_transfer_start,
	.stop = dma_iproc_pax_transfer_stop,
};

static const struct dma_iproc_pax_cfg pax_dma_cfg = {
	.dma_base = DT_INST_REG_ADDR_BY_NAME(0, dme_regs),
	.rm_base = DT_INST_REG_ADDR_BY_NAME(0, rm_ring_regs),
	.rm_comm_base = DT_INST_REG_ADDR_BY_NAME(0, rm_comm_regs),
	.use_rings = DT_INST_PROP(0, dma_channels),
	.bd_memory_base = (void *)DT_INST_PROP_BY_IDX(0, bd_memory, 0),
	.scr_addr_loc = DT_INST_PROP(0, scr_addr_loc),
	.pcie_dev = DEVICE_DT_GET(DT_INST_PHANDLE(0, pcie_ep)),
};

DEVICE_DT_INST_DEFINE(0,
		    &dma_iproc_pax_init,
		    NULL,
		    &pax_dma_data,
		    &pax_dma_cfg,
		    POST_KERNEL,
		    CONFIG_DMA_IPROC_PAX_V2_INIT_PRIORITY,
		    &pax_dma_driver_api);